diff --git a/clang-tools-extra/clang-tidy/GlobList.cpp b/clang-tools-extra/clang-tidy/GlobList.cpp
index 43945fcdc074c32565466a6e1d25c1a026b4f6ca..863bb4672852d20a329a0c7ec83cb32e1735ca60 100644
--- a/clang-tools-extra/clang-tidy/GlobList.cpp
+++ b/clang-tools-extra/clang-tidy/GlobList.cpp
@@ -15,7 +15,7 @@ using namespace tidy;
 // Returns true if GlobList starts with the negative indicator ('-'), removes it
 // from the GlobList.
 static bool consumeNegativeIndicator(StringRef &GlobList) {
-  GlobList = GlobList.trim(" \r\n");
+  GlobList = GlobList.trim();
   if (GlobList.startswith("-")) {
     GlobList = GlobList.substr(1);
     return true;
@@ -27,7 +27,7 @@ static bool consumeNegativeIndicator(StringRef &GlobList) {
 // removes it and the trailing comma from the GlobList.
 static llvm::Regex consumeGlob(StringRef &GlobList) {
   StringRef UntrimmedGlob = GlobList.substr(0, GlobList.find(','));
-  StringRef Glob = UntrimmedGlob.trim(' ');
+  StringRef Glob = UntrimmedGlob.trim();
   GlobList = GlobList.substr(UntrimmedGlob.size() + 1);
   SmallString<128> RegexText("^");
   StringRef MetaChars("()^$|*+?.[]\\{}");
diff --git a/clang-tools-extra/clang-tidy/altera/AlteraTidyModule.cpp b/clang-tools-extra/clang-tidy/altera/AlteraTidyModule.cpp
index a328f05da5d0346944394b14279811bc1b5c1de3..a6c29e03f7aa80f6800d00226bc1a02f3ea4a228 100644
--- a/clang-tools-extra/clang-tidy/altera/AlteraTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/altera/AlteraTidyModule.cpp
@@ -12,6 +12,7 @@
 #include "KernelNameRestrictionCheck.h"
 #include "SingleWorkItemBarrierCheck.h"
 #include "StructPackAlignCheck.h"
+#include "UnrollLoopsCheck.h"
 
 using namespace clang::ast_matchers;
 
@@ -28,6 +29,7 @@ public:
         "altera-single-work-item-barrier");
     CheckFactories.registerCheck<StructPackAlignCheck>(
         "altera-struct-pack-align");
+    CheckFactories.registerCheck<UnrollLoopsCheck>("altera-unroll-loops");
   }
 };
 
diff --git a/clang-tools-extra/clang-tidy/altera/CMakeLists.txt b/clang-tools-extra/clang-tidy/altera/CMakeLists.txt
index 0765b9735cf9a622d6fe26bc547d8f134cd0fbe5..c8a883d0750c57061cc53c5a781d52b9feab4f80 100644
--- a/clang-tools-extra/clang-tidy/altera/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/altera/CMakeLists.txt
@@ -8,6 +8,7 @@ add_clang_library(clangTidyAlteraModule
   KernelNameRestrictionCheck.cpp
   SingleWorkItemBarrierCheck.cpp
   StructPackAlignCheck.cpp
+  UnrollLoopsCheck.cpp
 
   LINK_LIBS
   clangTidy
diff --git a/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.cpp b/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..40ba3913467f22b3bb4c9e6b247a7b72ab32ed22
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.cpp
@@ -0,0 +1,277 @@
+//===--- UnrollLoopsCheck.cpp - clang-tidy --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UnrollLoopsCheck.h"
+#include "clang/AST/APValue.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/AST/OperationKinds.h"
+#include "clang/AST/ParentMapContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include <math.h>
+
+using namespace clang::ast_matchers;
+
+namespace clang {
+namespace tidy {
+namespace altera {
+
+UnrollLoopsCheck::UnrollLoopsCheck(StringRef Name, ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      MaxLoopIterations(Options.get("MaxLoopIterations", 100U)) {}
+
+void UnrollLoopsCheck::registerMatchers(MatchFinder *Finder) {
+  const auto HasLoopBound = hasDescendant(
+      varDecl(allOf(matchesName("__end*"),
+                    hasDescendant(integerLiteral().bind("cxx_loop_bound")))));
+  const auto CXXForRangeLoop =
+      cxxForRangeStmt(anyOf(HasLoopBound, unless(HasLoopBound)));
+  const auto AnyLoop = anyOf(forStmt(), whileStmt(), doStmt(), CXXForRangeLoop);
+  Finder->addMatcher(
+      stmt(allOf(AnyLoop, unless(hasDescendant(stmt(AnyLoop))))).bind("loop"),
+      this);
+}
+
+void UnrollLoopsCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *Loop = Result.Nodes.getNodeAs<Stmt>("loop");
+  const auto *CXXLoopBound =
+      Result.Nodes.getNodeAs<IntegerLiteral>("cxx_loop_bound");
+  const ASTContext *Context = Result.Context;
+  switch (unrollType(Loop, Result.Context)) {
+  case NotUnrolled:
+    diag(Loop->getBeginLoc(),
+         "kernel performance could be improved by unrolling this loop with a "
+         "'#pragma unroll' directive");
+    break;
+  case PartiallyUnrolled:
+    // Loop already partially unrolled, do nothing.
+    break;
+  case FullyUnrolled:
+    if (hasKnownBounds(Loop, CXXLoopBound, Context)) {
+      if (hasLargeNumIterations(Loop, CXXLoopBound, Context)) {
+        diag(Loop->getBeginLoc(),
+             "loop likely has a large number of iterations and thus "
+             "cannot be fully unrolled; to partially unroll this loop, use "
+             "the '#pragma unroll <num>' directive");
+        return;
+      }
+      return;
+    }
+    if (isa<WhileStmt, DoStmt>(Loop)) {
+      diag(Loop->getBeginLoc(),
+           "full unrolling requested, but loop bounds may not be known; to "
+           "partially unroll this loop, use the '#pragma unroll <num>' "
+           "directive",
+           DiagnosticIDs::Note);
+      break;
+    }
+    diag(Loop->getBeginLoc(),
+         "full unrolling requested, but loop bounds are not known; to "
+         "partially unroll this loop, use the '#pragma unroll <num>' "
+         "directive");
+    break;
+  }
+}
+
+enum UnrollLoopsCheck::UnrollType
+UnrollLoopsCheck::unrollType(const Stmt *Statement, ASTContext *Context) {
+  const DynTypedNodeList Parents = Context->getParents<Stmt>(*Statement);
+  for (const DynTypedNode &Parent : Parents) {
+    const auto *ParentStmt = Parent.get<AttributedStmt>();
+    if (!ParentStmt)
+      continue;
+    for (const Attr *Attribute : ParentStmt->getAttrs()) {
+      const auto *LoopHint = dyn_cast<LoopHintAttr>(Attribute);
+      if (!LoopHint)
+        continue;
+      switch (LoopHint->getState()) {
+      case LoopHintAttr::Numeric:
+        return PartiallyUnrolled;
+      case LoopHintAttr::Disable:
+        return NotUnrolled;
+      case LoopHintAttr::Full:
+        return FullyUnrolled;
+      case LoopHintAttr::Enable:
+        return FullyUnrolled;
+      case LoopHintAttr::AssumeSafety:
+        return NotUnrolled;
+      case LoopHintAttr::FixedWidth:
+        return NotUnrolled;
+      case LoopHintAttr::ScalableWidth:
+        return NotUnrolled;
+      }
+    }
+  }
+  return NotUnrolled;
+}
+
+bool UnrollLoopsCheck::hasKnownBounds(const Stmt *Statement,
+                                      const IntegerLiteral *CXXLoopBound,
+                                      const ASTContext *Context) {
+  if (isa<CXXForRangeStmt>(Statement))
+    return CXXLoopBound != nullptr;
+  // Too many possibilities in a while statement, so always recommend partial
+  // unrolling for these.
+  if (isa<WhileStmt, DoStmt>(Statement))
+    return false;
+  // The last loop type is a for loop.
+  const auto *ForLoop = dyn_cast<ForStmt>(Statement);
+  if (!ForLoop)
+    llvm_unreachable("Unknown loop");
+  const Stmt *Initializer = ForLoop->getInit();
+  const Expr *Conditional = ForLoop->getCond();
+  const Expr *Increment = ForLoop->getInc();
+  if (!Initializer || !Conditional || !Increment)
+    return false;
+  // If the loop variable value isn't known, loop bounds are unknown.
+  if (const auto *InitDeclStatement = dyn_cast<DeclStmt>(Initializer)) {
+    if (const auto *VariableDecl =
+            dyn_cast<VarDecl>(InitDeclStatement->getSingleDecl())) {
+      APValue *Evaluation = VariableDecl->evaluateValue();
+      if (!Evaluation || !Evaluation->hasValue())
+        return false;
+    }
+  }
+  // If increment is unary and not one of ++ and --, loop bounds are unknown.
+  if (const auto *Op = dyn_cast<UnaryOperator>(Increment))
+    if (!Op->isIncrementDecrementOp())
+      return false;
+
+  if (isa<BinaryOperator>(Conditional)) {
+    const auto *BinaryOp = dyn_cast<BinaryOperator>(Conditional);
+    const Expr *LHS = BinaryOp->getLHS();
+    const Expr *RHS = BinaryOp->getRHS();
+    // If both sides are value dependent or constant, loop bounds are unknown.
+    return LHS->isEvaluatable(*Context) != RHS->isEvaluatable(*Context);
+  }
+  return false; // If it's not a binary operator, loop bounds are unknown.
+}
+
+const Expr *UnrollLoopsCheck::getCondExpr(const Stmt *Statement) {
+  if (const auto *ForLoop = dyn_cast<ForStmt>(Statement))
+    return ForLoop->getCond();
+  if (const auto *WhileLoop = dyn_cast<WhileStmt>(Statement))
+    return WhileLoop->getCond();
+  if (const auto *DoWhileLoop = dyn_cast<DoStmt>(Statement))
+    return DoWhileLoop->getCond();
+  if (const auto *CXXRangeLoop = dyn_cast<CXXForRangeStmt>(Statement))
+    return CXXRangeLoop->getCond();
+  llvm_unreachable("Unknown loop");
+}
+
+bool UnrollLoopsCheck::hasLargeNumIterations(const Stmt *Statement,
+                                             const IntegerLiteral *CXXLoopBound,
+                                             const ASTContext *Context) {
+  // Because hasKnownBounds is called before this, if this is true, then
+  // CXXLoopBound is also matched.
+  if (isa<CXXForRangeStmt>(Statement)) {
+    assert(CXXLoopBound && "CXX ranged for loop has no loop bound");
+    return exprHasLargeNumIterations(CXXLoopBound, Context);
+  }
+  const auto *ForLoop = dyn_cast<ForStmt>(Statement);
+  assert(ForLoop && "Unknown loop");
+  const Stmt *Initializer = ForLoop->getInit();
+  const Expr *Conditional = ForLoop->getCond();
+  const Expr *Increment = ForLoop->getInc();
+  int InitValue;
+  // If the loop variable value isn't known, we can't know the loop bounds.
+  if (const auto *InitDeclStatement = dyn_cast<DeclStmt>(Initializer)) {
+    if (const auto *VariableDecl =
+            dyn_cast<VarDecl>(InitDeclStatement->getSingleDecl())) {
+      APValue *Evaluation = VariableDecl->evaluateValue();
+      if (!Evaluation || !Evaluation->isInt())
+        return true;
+      InitValue = Evaluation->getInt().getExtValue();
+    }
+  }
+  assert(isa<BinaryOperator>(Conditional) &&
+         "Conditional is not a binary operator");
+  int EndValue;
+  const auto *BinaryOp = dyn_cast<BinaryOperator>(Conditional);
+  if (!extractValue(EndValue, BinaryOp, Context))
+    return true;
+
+  double Iterations;
+
+  // If increment is unary and not one of ++, --, we can't know the loop bounds.
+  if (const auto *Op = dyn_cast<UnaryOperator>(Increment)) {
+    if (Op->isIncrementOp())
+      Iterations = EndValue - InitValue;
+    else if (Op->isDecrementOp())
+      Iterations = InitValue - EndValue;
+    else
+      llvm_unreachable("Unary operator neither increment nor decrement");
+  }
+
+  // If increment is binary and not one of +, -, *, /, we can't know the loop
+  // bounds.
+  if (const auto *Op = dyn_cast<BinaryOperator>(Increment)) {
+    int ConstantValue;
+    if (!extractValue(ConstantValue, Op, Context))
+      return true;
+    switch (Op->getOpcode()) {
+    case (BO_AddAssign):
+      Iterations = ceil(float(EndValue - InitValue) / ConstantValue);
+      break;
+    case (BO_SubAssign):
+      Iterations = ceil(float(InitValue - EndValue) / ConstantValue);
+      break;
+    case (BO_MulAssign):
+      Iterations = 1 + (log(EndValue) - log(InitValue)) / log(ConstantValue);
+      break;
+    case (BO_DivAssign):
+      Iterations = 1 + (log(InitValue) - log(EndValue)) / log(ConstantValue);
+      break;
+    default:
+      // All other operators are not handled; assume large bounds.
+      return true;
+    }
+  }
+  return Iterations > MaxLoopIterations;
+}
+
+bool UnrollLoopsCheck::extractValue(int &Value, const BinaryOperator *Op,
+                                    const ASTContext *Context) {
+  const Expr *LHS = Op->getLHS();
+  const Expr *RHS = Op->getRHS();
+  Expr::EvalResult Result;
+  if (LHS->isEvaluatable(*Context))
+    LHS->EvaluateAsRValue(Result, *Context);
+  else if (RHS->isEvaluatable(*Context))
+    RHS->EvaluateAsRValue(Result, *Context);
+  else
+    return false; // Cannot evalue either side.
+  if (!Result.Val.isInt())
+    return false; // Cannot check number of iterations, return false to be
+                  // safe.
+  Value = Result.Val.getInt().getExtValue();
+  return true;
+}
+
+bool UnrollLoopsCheck::exprHasLargeNumIterations(const Expr *Expression,
+                                                 const ASTContext *Context) {
+  Expr::EvalResult Result;
+  if (Expression->EvaluateAsRValue(Result, *Context)) {
+    if (!Result.Val.isInt())
+      return false; // Cannot check number of iterations, return false to be
+                    // safe.
+    // The following assumes values go from 0 to Val in increments of 1.
+    return Result.Val.getInt() > MaxLoopIterations;
+  }
+  // Cannot evaluate Expression as an r-value, so cannot check number of
+  // iterations.
+  return false;
+}
+
+void UnrollLoopsCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "MaxLoopIterations", MaxLoopIterations);
+}
+
+} // namespace altera
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.h b/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.h
new file mode 100644
index 0000000000000000000000000000000000000000..8a63e9173e0aa6ad8df89ef9c0878bbc29ea73ab
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/altera/UnrollLoopsCheck.h
@@ -0,0 +1,78 @@
+//===--- UnrollLoopsCheck.h - clang-tidy ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ALTERA_UNROLLLOOPSCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ALTERA_UNROLLLOOPSCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang {
+namespace tidy {
+namespace altera {
+
+/// Finds inner loops that have not been unrolled, as well as fully unrolled
+/// loops with unknown loop bounds or a large number of iterations.
+///
+/// Unrolling inner loops could improve the performance of OpenCL kernels.
+/// However, if they have unknown loop bounds or a large number of iterations,
+/// they cannot be fully unrolled, and should be partially unrolled.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/altera-unroll-loops.html
+class UnrollLoopsCheck : public ClangTidyCheck {
+public:
+  UnrollLoopsCheck(StringRef Name, ClangTidyContext *Context);
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+
+private:
+  /// Recommend partial unrolling if number of loop iterations is greater than
+  /// MaxLoopIterations.
+  const unsigned MaxLoopIterations;
+  /// The kind of unrolling, if any, applied to a given loop.
+  enum UnrollType {
+    // This loop has no #pragma unroll directive associated with it.
+    NotUnrolled,
+    // This loop has a #pragma unroll directive associated with it.
+    FullyUnrolled,
+    // This loop has a #pragma unroll <num> directive associated with it.
+    PartiallyUnrolled
+  };
+  /// Attempts to extract an integer value from either side of the
+  /// BinaryOperator. Returns true and saves the result to &value if successful,
+  /// returns false otherwise.
+  bool extractValue(int &Value, const BinaryOperator *Op,
+                    const ASTContext *Context);
+  /// Returns true if the given loop statement has a large number of iterations,
+  /// as determined by the integer value in the loop's condition expression,
+  /// if one exists.
+  bool hasLargeNumIterations(const Stmt *Statement,
+                             const IntegerLiteral *CXXLoopBound,
+                             const ASTContext *Context);
+  /// Checks one hand side of the binary operator to ascertain if the upper
+  /// bound on the number of loops is greater than max_loop_iterations or not.
+  /// If the expression is not evaluatable or not an integer, returns false.
+  bool exprHasLargeNumIterations(const Expr *Expression,
+                                 const ASTContext *Context);
+  /// Returns the type of unrolling, if any, associated with the given
+  /// statement.
+  enum UnrollType unrollType(const Stmt *Statement, ASTContext *Context);
+  /// Returns the condition expression within a given for statement. If there is
+  /// none, or if the Statement is not a loop, then returns a NULL pointer.
+  const Expr *getCondExpr(const Stmt *Statement);
+  /// Returns True if the loop statement has known bounds.
+  bool hasKnownBounds(const Stmt *Statement, const IntegerLiteral *CXXLoopBound,
+                      const ASTContext *Context);
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+};
+
+} // namespace altera
+} // namespace tidy
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_ALTERA_UNROLLLOOPSCHECK_H
diff --git a/clang-tools-extra/clang-tidy/bugprone/TerminatingContinueCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/TerminatingContinueCheck.cpp
index 43402a2212182e13ad9740e87cf888e97298f006..65da4c325de4c78b9af0616f8f5fc12b6dcbf959 100644
--- a/clang-tools-extra/clang-tidy/bugprone/TerminatingContinueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/TerminatingContinueCheck.cpp
@@ -26,10 +26,11 @@ void TerminatingContinueCheck::registerMatchers(MatchFinder *Finder) {
              equalsBoundNode("closestLoop"));
 
   Finder->addMatcher(
-      continueStmt(hasAncestor(stmt(anyOf(forStmt(), whileStmt(),
-                                          cxxForRangeStmt(), doStmt()))
-                                   .bind("closestLoop")),
-                   hasAncestor(DoWithFalse))
+      continueStmt(
+          hasAncestor(stmt(anyOf(forStmt(), whileStmt(), cxxForRangeStmt(),
+                                 doStmt(), switchStmt()))
+                          .bind("closestLoop")),
+          hasAncestor(DoWithFalse))
           .bind("continue"),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.cpp b/clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.cpp
index b108f75fbc7a5091941152583ae9d0dd3c6240d1..ebe9658d8b2b1ea14e3bea3b6f313e10c006acd1 100644
--- a/clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.cpp
+++ b/clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.cpp
@@ -9,7 +9,6 @@
 #include "BufferDerefCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
-#include "clang/StaticAnalyzer/Checkers/MPIFunctionClassifier.h"
 #include "clang/Tooling/FixIt.h"
 
 using namespace clang::ast_matchers;
@@ -23,13 +22,15 @@ void BufferDerefCheck::registerMatchers(MatchFinder *Finder) {
 }
 
 void BufferDerefCheck::check(const MatchFinder::MatchResult &Result) {
-  static ento::mpi::MPIFunctionClassifier FuncClassifier(*Result.Context);
   const auto *CE = Result.Nodes.getNodeAs<CallExpr>("CE");
   if (!CE->getDirectCallee())
     return;
 
+  if (!FuncClassifier)
+    FuncClassifier.emplace(*Result.Context);
+
   const IdentifierInfo *Identifier = CE->getDirectCallee()->getIdentifier();
-  if (!Identifier || !FuncClassifier.isMPIType(Identifier))
+  if (!Identifier || !FuncClassifier->isMPIType(Identifier))
     return;
 
   // These containers are used, to capture the type and expression of a buffer.
@@ -60,18 +61,18 @@ void BufferDerefCheck::check(const MatchFinder::MatchResult &Result) {
   // Collect buffer types and argument expressions for all buffers used in the
   // MPI call expression. The number passed to the lambda corresponds to the
   // argument index of the currently verified MPI function call.
-  if (FuncClassifier.isPointToPointType(Identifier)) {
+  if (FuncClassifier->isPointToPointType(Identifier)) {
     AddBuffer(0);
-  } else if (FuncClassifier.isCollectiveType(Identifier)) {
-    if (FuncClassifier.isReduceType(Identifier)) {
+  } else if (FuncClassifier->isCollectiveType(Identifier)) {
+    if (FuncClassifier->isReduceType(Identifier)) {
       AddBuffer(0);
       AddBuffer(1);
-    } else if (FuncClassifier.isScatterType(Identifier) ||
-               FuncClassifier.isGatherType(Identifier) ||
-               FuncClassifier.isAlltoallType(Identifier)) {
+    } else if (FuncClassifier->isScatterType(Identifier) ||
+               FuncClassifier->isGatherType(Identifier) ||
+               FuncClassifier->isAlltoallType(Identifier)) {
       AddBuffer(0);
       AddBuffer(3);
-    } else if (FuncClassifier.isBcastType(Identifier)) {
+    } else if (FuncClassifier->isBcastType(Identifier)) {
       AddBuffer(0);
     }
   }
@@ -126,6 +127,7 @@ void BufferDerefCheck::checkBuffers(ArrayRef<const Type *> BufferTypes,
   }
 }
 
+void BufferDerefCheck::onEndOfTranslationUnit() { FuncClassifier.reset(); }
 } // namespace mpi
 } // namespace tidy
 } // namespace clang
diff --git a/clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.h b/clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.h
index 040e3f790e61d1d12b0cfaa7f00b055ce3950b4e..a3be5a8224e00f42b13d863f56ea8e7943ffe961 100644
--- a/clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.h
+++ b/clang-tools-extra/clang-tidy/mpi/BufferDerefCheck.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MPI_BUFFER_DEREF_H
 
 #include "../ClangTidyCheck.h"
+#include "clang/StaticAnalyzer/Checkers/MPIFunctionClassifier.h"
 
 namespace clang {
 namespace tidy {
@@ -30,6 +31,7 @@ public:
       : ClangTidyCheck(Name, Context) {}
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  void onEndOfTranslationUnit() override;
 
 private:
   /// Checks for all buffers in an MPI call if they are sufficiently
@@ -41,6 +43,8 @@ private:
                     ArrayRef<const Expr *> BufferExprs);
 
   enum class IndirectionType : unsigned char { Pointer, Array };
+
+  Optional<ento::mpi::MPIFunctionClassifier> FuncClassifier;
 };
 
 } // namespace mpi
diff --git a/clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.cpp b/clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.cpp
index cc60ea365c2f33fe51e32a853b672634f0ff6465..fb96ce77a13df15db91ebf3b8b562e6e08cc10b7 100644
--- a/clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.cpp
+++ b/clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.cpp
@@ -8,7 +8,6 @@
 
 #include "TypeMismatchCheck.h"
 #include "clang/Lex/Lexer.h"
-#include "clang/StaticAnalyzer/Checkers/MPIFunctionClassifier.h"
 #include "clang/Tooling/FixIt.h"
 #include <map>
 #include <unordered_set>
@@ -241,13 +240,15 @@ void TypeMismatchCheck::registerMatchers(MatchFinder *Finder) {
 }
 
 void TypeMismatchCheck::check(const MatchFinder::MatchResult &Result) {
-  static ento::mpi::MPIFunctionClassifier FuncClassifier(*Result.Context);
   const auto *const CE = Result.Nodes.getNodeAs<CallExpr>("CE");
   if (!CE->getDirectCallee())
     return;
 
+  if (!FuncClassifier)
+    FuncClassifier.emplace(*Result.Context);
+
   const IdentifierInfo *Identifier = CE->getDirectCallee()->getIdentifier();
-  if (!Identifier || !FuncClassifier.isMPIType(Identifier))
+  if (!Identifier || !FuncClassifier->isMPIType(Identifier))
     return;
 
   // These containers are used, to capture buffer, MPI datatype pairs.
@@ -281,18 +282,18 @@ void TypeMismatchCheck::check(const MatchFinder::MatchResult &Result) {
   };
 
   // Collect all buffer, MPI datatype pairs for the inspected call expression.
-  if (FuncClassifier.isPointToPointType(Identifier)) {
+  if (FuncClassifier->isPointToPointType(Identifier)) {
     AddPair(0, 2);
-  } else if (FuncClassifier.isCollectiveType(Identifier)) {
-    if (FuncClassifier.isReduceType(Identifier)) {
+  } else if (FuncClassifier->isCollectiveType(Identifier)) {
+    if (FuncClassifier->isReduceType(Identifier)) {
       AddPair(0, 3);
       AddPair(1, 3);
-    } else if (FuncClassifier.isScatterType(Identifier) ||
-               FuncClassifier.isGatherType(Identifier) ||
-               FuncClassifier.isAlltoallType(Identifier)) {
+    } else if (FuncClassifier->isScatterType(Identifier) ||
+               FuncClassifier->isGatherType(Identifier) ||
+               FuncClassifier->isAlltoallType(Identifier)) {
       AddPair(0, 2);
       AddPair(3, 5);
-    } else if (FuncClassifier.isBcastType(Identifier)) {
+    } else if (FuncClassifier->isBcastType(Identifier)) {
       AddPair(0, 2);
     }
   }
@@ -331,6 +332,7 @@ void TypeMismatchCheck::checkArguments(ArrayRef<const Type *> BufferTypes,
   }
 }
 
+void TypeMismatchCheck::onEndOfTranslationUnit() { FuncClassifier.reset(); }
 } // namespace mpi
 } // namespace tidy
 } // namespace clang
diff --git a/clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.h b/clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.h
index a563e295c7b769fe165d1077a2d38f874d8aed50..d09ba270495b38d936ae75be6065806e40557f4e 100644
--- a/clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.h
+++ b/clang-tools-extra/clang-tidy/mpi/TypeMismatchCheck.h
@@ -11,6 +11,7 @@
 
 #include "../ClangTidyCheck.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/StaticAnalyzer/Checkers/MPIFunctionClassifier.h"
 
 namespace clang {
 namespace tidy {
@@ -31,6 +32,8 @@ public:
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
+  void onEndOfTranslationUnit() override;
+
 private:
   /// Check if the buffer type MPI datatype pairs match.
   ///
@@ -41,6 +44,8 @@ private:
   void checkArguments(ArrayRef<const Type *> BufferTypes,
                       ArrayRef<const Expr *> BufferExprs,
                       ArrayRef<StringRef> MPIDatatypes, const LangOptions &LO);
+
+  Optional<ento::mpi::MPIFunctionClassifier> FuncClassifier;
 };
 
 } // namespace mpi
diff --git a/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.cpp b/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.cpp
index 1123238c186b56624d400fe9e83d40f9876a4bc0..fe25f7a7ccbccdd704aa49185f6f885c3bda6657 100644
--- a/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "BracesAroundStatementsCheck.h"
+#include "../utils/LexerUtils.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Lex/Lexer.h"
@@ -16,10 +17,9 @@ using namespace clang::ast_matchers;
 namespace clang {
 namespace tidy {
 namespace readability {
-namespace {
 
-tok::TokenKind getTokenKind(SourceLocation Loc, const SourceManager &SM,
-                            const ASTContext *Context) {
+static tok::TokenKind getTokenKind(SourceLocation Loc, const SourceManager &SM,
+                                   const ASTContext *Context) {
   Token Tok;
   SourceLocation Beginning =
       Lexer::GetBeginningOfToken(Loc, SM, Context->getLangOpts());
@@ -33,9 +33,9 @@ tok::TokenKind getTokenKind(SourceLocation Loc, const SourceManager &SM,
   return Tok.getKind();
 }
 
-SourceLocation forwardSkipWhitespaceAndComments(SourceLocation Loc,
-                                                const SourceManager &SM,
-                                                const ASTContext *Context) {
+static SourceLocation
+forwardSkipWhitespaceAndComments(SourceLocation Loc, const SourceManager &SM,
+                                 const ASTContext *Context) {
   assert(Loc.isValid());
   for (;;) {
     while (isWhitespace(*SM.getCharacterData(Loc)))
@@ -50,31 +50,15 @@ SourceLocation forwardSkipWhitespaceAndComments(SourceLocation Loc,
   }
 }
 
-SourceLocation findEndLocation(SourceLocation LastTokenLoc,
-                               const SourceManager &SM,
-                               const ASTContext *Context) {
+static SourceLocation findEndLocation(const Stmt &S, const SourceManager &SM,
+                                      const ASTContext *Context) {
   SourceLocation Loc =
-      Lexer::GetBeginningOfToken(LastTokenLoc, SM, Context->getLangOpts());
-  // Loc points to the beginning of the last (non-comment non-ws) token
-  // before end or ';'.
-  assert(Loc.isValid());
-  bool SkipEndWhitespaceAndComments = true;
-  tok::TokenKind TokKind = getTokenKind(Loc, SM, Context);
-  if (TokKind == tok::NUM_TOKENS || TokKind == tok::semi ||
-      TokKind == tok::r_brace) {
-    // If we are at ";" or "}", we found the last token. We could use as well
-    // `if (isa<NullStmt>(S))`, but it wouldn't work for nested statements.
-    SkipEndWhitespaceAndComments = false;
-  }
+      utils::lexer::getUnifiedEndLoc(S, SM, Context->getLangOpts());
+  if (!Loc.isValid())
+    return Loc;
 
-  Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, Context->getLangOpts());
-  // Loc points past the last token before end or after ';'.
-  if (SkipEndWhitespaceAndComments) {
-    Loc = forwardSkipWhitespaceAndComments(Loc, SM, Context);
-    tok::TokenKind TokKind = getTokenKind(Loc, SM, Context);
-    if (TokKind == tok::semi)
-      Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, Context->getLangOpts());
-  }
+  // Start searching right after S.
+  Loc = Loc.getLocWithOffset(1);
 
   for (;;) {
     assert(Loc.isValid());
@@ -109,8 +93,6 @@ SourceLocation findEndLocation(SourceLocation LastTokenLoc,
   return Loc;
 }
 
-} // namespace
-
 BracesAroundStatementsCheck::BracesAroundStatementsCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
@@ -123,10 +105,7 @@ void BracesAroundStatementsCheck::storeOptions(
 }
 
 void BracesAroundStatementsCheck::registerMatchers(MatchFinder *Finder) {
-  Finder->addMatcher(
-      ifStmt(unless(allOf(isConstexpr(), isInTemplateInstantiation())))
-          .bind("if"),
-      this);
+  Finder->addMatcher(ifStmt().bind("if"), this);
   Finder->addMatcher(whileStmt().bind("while"), this);
   Finder->addMatcher(doStmt().bind("do"), this);
   Finder->addMatcher(forStmt().bind("for"), this);
@@ -224,13 +203,6 @@ bool BracesAroundStatementsCheck::checkStmt(
   const SourceManager &SM = *Result.SourceManager;
   const ASTContext *Context = Result.Context;
 
-  // Treat macros.
-  CharSourceRange FileRange = Lexer::makeFileCharRange(
-      CharSourceRange::getTokenRange(S->getSourceRange()), SM,
-      Context->getLangOpts());
-  if (FileRange.isInvalid())
-    return false;
-
   // Convert InitialLoc to file location, if it's on the same macro expansion
   // level as the start of the statement. We also need file locations for
   // Lexer::getLocForEndOfToken working properly.
@@ -250,13 +222,12 @@ bool BracesAroundStatementsCheck::checkStmt(
     EndLoc = EndLocHint;
     ClosingInsertion = "} ";
   } else {
-    const auto FREnd = FileRange.getEnd().getLocWithOffset(-1);
-    EndLoc = findEndLocation(FREnd, SM, Context);
+    EndLoc = findEndLocation(*S, SM, Context);
     ClosingInsertion = "\n}";
   }
 
   assert(StartLoc.isValid());
-  assert(EndLoc.isValid());
+
   // Don't require braces for statements spanning less than certain number of
   // lines.
   if (ShortStatementLines && !ForceBracesStmts.erase(S)) {
@@ -267,6 +238,20 @@ bool BracesAroundStatementsCheck::checkStmt(
   }
 
   auto Diag = diag(StartLoc, "statement should be inside braces");
+
+  // Change only if StartLoc and EndLoc are on the same macro expansion level.
+  // This will also catch invalid EndLoc.
+  // Example: LLVM_DEBUG( for(...) do_something() );
+  // In this case fix-it cannot be provided as the semicolon which is not
+  // visible here is part of the macro. Adding braces here would require adding
+  // another semicolon.
+  if (Lexer::makeFileCharRange(
+          CharSourceRange::getTokenRange(SourceRange(
+              SM.getSpellingLoc(StartLoc), SM.getSpellingLoc(EndLoc))),
+          SM, Context->getLangOpts())
+          .isInvalid())
+    return false;
+
   Diag << FixItHint::CreateInsertion(StartLoc, " {")
        << FixItHint::CreateInsertion(EndLoc, ClosingInsertion);
   return true;
diff --git a/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.h b/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.h
index 7c019c6cb5521ddc6d5f55e1356432e711a64c54..1270cfe10d193e78c704a92867325653ae40a277 100644
--- a/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.h
@@ -55,6 +55,9 @@ private:
   template <typename IfOrWhileStmt>
   SourceLocation findRParenLoc(const IfOrWhileStmt *S, const SourceManager &SM,
                                const ASTContext *Context);
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
 private:
   std::set<const Stmt *> ForceBracesStmts;
diff --git a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt
index ecf37b5b9157001555d65441aa77668cd5015953..78a3851f66bef65f69f8cbea75a07b389099da20 100644
--- a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt
@@ -10,7 +10,6 @@ add_clang_library(clangTidyReadabilityModule
   ContainerSizeEmptyCheck.cpp
   ConvertMemberFunctionsToStatic.cpp
   DeleteNullPointerCheck.cpp
-  DeletedDefaultCheck.cpp
   ElseAfterReturnCheck.cpp
   FunctionCognitiveComplexityCheck.cpp
   FunctionSizeCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.cpp b/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.cpp
deleted file mode 100644
index ff2f00b94e36c0117d29397d23ac6462b059bb56..0000000000000000000000000000000000000000
--- a/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.cpp
+++ /dev/null
@@ -1,68 +0,0 @@
-//===--- DeletedDefaultCheck.cpp - clang-tidy------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "DeletedDefaultCheck.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/ASTMatchers/ASTMatchFinder.h"
-
-using namespace clang::ast_matchers;
-
-namespace clang {
-namespace tidy {
-namespace readability {
-
-void DeletedDefaultCheck::registerMatchers(MatchFinder *Finder) {
-  // We match constructors/assignment operators that are:
-  //   - explicitly marked '= default'
-  //   - actually deleted
-  //   - not in template instantiation.
-  // We bind the declaration to "method-decl" and also to "constructor" when
-  // it is a constructor.
-
-  Finder->addMatcher(
-      cxxMethodDecl(anyOf(cxxConstructorDecl().bind("constructor"),
-                          isCopyAssignmentOperator(),
-                          isMoveAssignmentOperator()),
-                    isDefaulted(), unless(isImplicit()), isDeleted(),
-                    unless(isInstantiated()))
-          .bind("method-decl"),
-      this);
-}
-
-void DeletedDefaultCheck::check(const MatchFinder::MatchResult &Result) {
-  const StringRef Message = "%0 is explicitly defaulted but implicitly "
-                            "deleted, probably because %1; definition can "
-                            "either be removed or explicitly deleted";
-  if (const auto *Constructor =
-          Result.Nodes.getNodeAs<CXXConstructorDecl>("constructor")) {
-    auto Diag = diag(Constructor->getBeginLoc(), Message);
-    if (Constructor->isDefaultConstructor()) {
-      Diag << "default constructor"
-           << "a non-static data member or a base class is lacking a default "
-              "constructor";
-    } else if (Constructor->isCopyConstructor()) {
-      Diag << "copy constructor"
-           << "a non-static data member or a base class is not copyable";
-    } else if (Constructor->isMoveConstructor()) {
-      Diag << "move constructor"
-           << "a non-static data member or a base class is neither copyable "
-              "nor movable";
-    }
-  } else if (const auto *Assignment =
-                 Result.Nodes.getNodeAs<CXXMethodDecl>("method-decl")) {
-    diag(Assignment->getBeginLoc(), Message)
-        << (Assignment->isCopyAssignmentOperator() ? "copy assignment operator"
-                                                   : "move assignment operator")
-        << "a base class or a non-static data member is not assignable, e.g. "
-           "because the latter is marked 'const'";
-  }
-}
-
-} // namespace readability
-} // namespace tidy
-} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.h b/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.h
deleted file mode 100644
index ab7f141417d01a0e1eb0c249925a0d267e079dbe..0000000000000000000000000000000000000000
--- a/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.h
+++ /dev/null
@@ -1,35 +0,0 @@
-//===--- DeletedDefaultCheck.h - clang-tidy----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_DELETED_DEFAULT_H
-#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_DELETED_DEFAULT_H
-
-#include "../ClangTidyCheck.h"
-
-namespace clang {
-namespace tidy {
-namespace readability {
-
-/// Checks when a constructor or an assignment operator is marked as '= default'
-/// but is actually deleted by the compiler.
-///
-/// For the user-facing documentation see:
-/// http://clang.llvm.org/extra/clang-tidy/checks/readability-deleted-default.html
-class DeletedDefaultCheck : public ClangTidyCheck {
-public:
-  DeletedDefaultCheck(StringRef Name, ClangTidyContext *Context)
-      : ClangTidyCheck(Name, Context) {}
-  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
-  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
-};
-
-} // namespace readability
-} // namespace tidy
-} // namespace clang
-
-#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_DELETED_DEFAULT_H
diff --git a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp
index 89bb02e78cc6320e1450475476958e14cef5caf1..0558b41016379da2c1184b9f8831c173fc55bb20 100644
--- a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp
@@ -171,8 +171,7 @@ void ElseAfterReturnCheck::registerPPCallbacks(const SourceManager &SM,
 void ElseAfterReturnCheck::registerMatchers(MatchFinder *Finder) {
   const auto InterruptsControlFlow = stmt(anyOf(
       returnStmt().bind(InterruptingStr), continueStmt().bind(InterruptingStr),
-      breakStmt().bind(InterruptingStr),
-      expr(ignoringImplicit(cxxThrowExpr().bind(InterruptingStr)))));
+      breakStmt().bind(InterruptingStr), cxxThrowExpr().bind(InterruptingStr)));
   Finder->addMatcher(
       compoundStmt(
           forEach(ifStmt(unless(isConstexpr()),
diff --git a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.h b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.h
index 440cf4b637b73042bfc684642eed7f554780b88f..d3fbc0ac0abe03c9973fd15081fd6dd039066311 100644
--- a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.h
@@ -28,6 +28,9 @@ public:
                            Preprocessor *ModuleExpanderPP) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
   using ConditionalBranchMap =
       llvm::DenseMap<FileID, SmallVector<SourceRange, 1>>;
diff --git a/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.cpp b/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.cpp
index c28424b11f279976b9ca941784999552122119cc..b3945b5a932f5af060ef463fbb055c7d65f6358a 100644
--- a/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.cpp
@@ -294,8 +294,7 @@ void InconsistentDeclarationParameterNameCheck::storeOptions(
 
 void InconsistentDeclarationParameterNameCheck::registerMatchers(
     MatchFinder *Finder) {
-  Finder->addMatcher(functionDecl(unless(isImplicit()), hasOtherDeclarations())
-                         .bind("functionDecl"),
+  Finder->addMatcher(functionDecl(hasOtherDeclarations()).bind("functionDecl"),
                      this);
 }
 
diff --git a/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.h b/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.h
index aac2f5060aa9c77f8429c7c93c376cbc26b46c4f..ca9640fc5a6040f02319e3137067aeb28693e6c1 100644
--- a/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/InconsistentDeclarationParameterNameCheck.h
@@ -33,6 +33,9 @@ public:
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
 private:
   void markRedeclarationsAsVisited(const FunctionDecl *FunctionDeclaration);
diff --git a/clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.cpp b/clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.cpp
index 77f93f1999a9dcaf8ce64bc12317351fbaac67ce..7b5a22f449c4bbd588e8941a7f4cb6e925aa27a7 100644
--- a/clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.cpp
@@ -106,11 +106,7 @@ void MisleadingIndentationCheck::missingBracesCheck(const SourceManager &SM,
 }
 
 void MisleadingIndentationCheck::registerMatchers(MatchFinder *Finder) {
-  Finder->addMatcher(
-      ifStmt(allOf(hasElse(stmt()),
-                   unless(allOf(isConstexpr(), isInTemplateInstantiation()))))
-          .bind("if"),
-      this);
+  Finder->addMatcher(ifStmt(hasElse(stmt())).bind("if"), this);
   Finder->addMatcher(
       compoundStmt(has(stmt(anyOf(ifStmt(), forStmt(), whileStmt()))))
           .bind("compound"),
diff --git a/clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.h b/clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.h
index ed5ba5bad1209f48dbe1f69dbef52e2f79e1526b..a9067032fa945f8935f9ff0fab16421859ab0d6e 100644
--- a/clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/MisleadingIndentationCheck.h
@@ -27,6 +27,9 @@ public:
       : ClangTidyCheck(Name, Context) {}
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
 private:
   void danglingElseCheck(const SourceManager &SM, ASTContext *Context,
diff --git a/clang-tools-extra/clang-tidy/readability/NamedParameterCheck.cpp b/clang-tools-extra/clang-tidy/readability/NamedParameterCheck.cpp
index 5e018ce6172c6213a2c16ad8ff606bab1e892864..4f81dc49ded7c6114ec97e19b06dc74d0daa34f0 100644
--- a/clang-tools-extra/clang-tidy/readability/NamedParameterCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/NamedParameterCheck.cpp
@@ -18,7 +18,7 @@ namespace tidy {
 namespace readability {
 
 void NamedParameterCheck::registerMatchers(ast_matchers::MatchFinder *Finder) {
-  Finder->addMatcher(functionDecl(unless(isInstantiated())).bind("decl"), this);
+  Finder->addMatcher(functionDecl().bind("decl"), this);
 }
 
 void NamedParameterCheck::check(const MatchFinder::MatchResult &Result) {
@@ -26,10 +26,6 @@ void NamedParameterCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *Function = Result.Nodes.getNodeAs<FunctionDecl>("decl");
   SmallVector<std::pair<const FunctionDecl *, unsigned>, 4> UnnamedParams;
 
-  // Ignore implicitly generated members.
-  if (Function->isImplicit())
-    return;
-
   // Ignore declarations without a definition if we're not dealing with an
   // overriden method.
   const FunctionDecl *Definition = nullptr;
diff --git a/clang-tools-extra/clang-tidy/readability/NamedParameterCheck.h b/clang-tools-extra/clang-tidy/readability/NamedParameterCheck.h
index 33a51b4c8dc9126d53163ac577ba090cfcc0ff5a..f946e00a004440c34a9d95426fea5726e04a17ee 100644
--- a/clang-tools-extra/clang-tidy/readability/NamedParameterCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/NamedParameterCheck.h
@@ -32,6 +32,9 @@ public:
       : ClangTidyCheck(Name, Context) {}
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 };
 
 } // namespace readability
diff --git a/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.cpp b/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.cpp
index 1044df0032d8046bb56693ae3b9f1ecf239ecb77..c9ebf7b6f8ce7812bf510776ee23fcce2c767d42 100644
--- a/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.cpp
@@ -18,7 +18,7 @@ namespace readability {
 
 void NonConstParameterCheck::registerMatchers(MatchFinder *Finder) {
   // Add parameters to Parameters.
-  Finder->addMatcher(parmVarDecl(unless(isInstantiated())).bind("Parm"), this);
+  Finder->addMatcher(parmVarDecl().bind("Parm"), this);
 
   // C++ constructor.
   Finder->addMatcher(cxxConstructorDecl().bind("Ctor"), this);
@@ -28,13 +28,11 @@ void NonConstParameterCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(declRefExpr().bind("Ref"), this);
 
   // Analyse parameter usage in function.
-  Finder->addMatcher(
-      traverse(TK_AsIs,
-               stmt(anyOf(unaryOperator(hasAnyOperatorName("++", "--")),
-                          binaryOperator(), callExpr(), returnStmt(),
-                          cxxConstructExpr()))
-                   .bind("Mark")),
-      this);
+  Finder->addMatcher(stmt(anyOf(unaryOperator(hasAnyOperatorName("++", "--")),
+                                binaryOperator(), callExpr(), returnStmt(),
+                                cxxConstructExpr()))
+                         .bind("Mark"),
+                     this);
   Finder->addMatcher(varDecl(hasInitializer(anything())).bind("Mark"), this);
 }
 
diff --git a/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.h b/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.h
index 39959e619096017b8dfcc2900a2b43c9e511bec0..a6179d6aa1e8978dadc3a22a785cff88fcec73f4 100644
--- a/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.h
@@ -26,6 +26,9 @@ public:
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void onEndOfTranslationUnit() override;
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
 private:
   /// Parameter info.
diff --git a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
index bbd2e24e503b6cb0a3eccc64de9566a87451fcd6..088b9f09082e0303c6c238f8572e479963ccb2de 100644
--- a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
@@ -15,7 +15,6 @@
 #include "ContainerSizeEmptyCheck.h"
 #include "ConvertMemberFunctionsToStatic.h"
 #include "DeleteNullPointerCheck.h"
-#include "DeletedDefaultCheck.h"
 #include "ElseAfterReturnCheck.h"
 #include "FunctionCognitiveComplexityCheck.h"
 #include "FunctionSizeCheck.h"
@@ -67,8 +66,6 @@ public:
         "readability-convert-member-functions-to-static");
     CheckFactories.registerCheck<DeleteNullPointerCheck>(
         "readability-delete-null-pointer");
-    CheckFactories.registerCheck<DeletedDefaultCheck>(
-        "readability-deleted-default");
     CheckFactories.registerCheck<ElseAfterReturnCheck>(
         "readability-else-after-return");
     CheckFactories.registerCheck<FunctionCognitiveComplexityCheck>(
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.cpp
index 9e336cb4cf15d33076094b6fd82b41019686dfae..6af77635aa2b1be9c246c133f656f7c23737559a 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.cpp
@@ -32,10 +32,10 @@ bool isLocationInMacroExpansion(const SourceManager &SM, SourceLocation Loc) {
 
 void RedundantControlFlowCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
-      functionDecl(
-          isDefinition(), returns(voidType()),
-          has(compoundStmt(hasAnySubstatement(returnStmt(unless(has(expr())))))
-                  .bind("return"))),
+      functionDecl(isDefinition(), returns(voidType()),
+                   hasBody(compoundStmt(hasAnySubstatement(
+                                            returnStmt(unless(has(expr())))))
+                               .bind("return"))),
       this);
   Finder->addMatcher(
       mapAnyOf(forStmt, cxxForRangeStmt, whileStmt, doStmt)
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.h b/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.h
index d4513e6f49a77f21acbab75d700deccadf818c0e..6d91c208587fecc2f3c8f7733bdbdeaebb0d5d4b 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.h
@@ -29,6 +29,10 @@ public:
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
+
 private:
   void
   checkRedundantReturn(const ast_matchers::MatchFinder::MatchResult &Result,
diff --git a/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.cpp b/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.cpp
index 2d14b49e0eea908c15ae21752ac810d8bc4d69b9..36294cd0e9b820f54396bd0db2544cc248bc2481 100644
--- a/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.cpp
@@ -32,7 +32,7 @@ void SimplifySubscriptExprCheck::registerMatchers(MatchFinder *Finder) {
           llvm::SmallVector<StringRef, 8>(Types.begin(), Types.end()))))));
 
   Finder->addMatcher(
-      arraySubscriptExpr(hasBase(ignoringParenImpCasts(
+      arraySubscriptExpr(hasBase(
           cxxMemberCallExpr(
               has(memberExpr().bind("member")),
               on(hasType(qualType(
@@ -40,7 +40,7 @@ void SimplifySubscriptExprCheck::registerMatchers(MatchFinder *Finder) {
                                hasDescendant(substTemplateTypeParmType()))),
                   anyOf(TypesMatcher, pointerType(pointee(TypesMatcher)))))),
               callee(namedDecl(hasName("data"))))
-              .bind("call")))),
+              .bind("call"))),
       this);
 }
 
diff --git a/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.h b/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.h
index 4f43cdb726df5d17015932e99823aaf2cc0e2a15..6caaf49de2a3d7b40d1516089ba3e50b25a8071d 100644
--- a/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/SimplifySubscriptExprCheck.h
@@ -28,6 +28,9 @@ public:
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap& Opts) override;
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
 private:
   const std::vector<std::string> Types;
diff --git a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
index e7d70f06bdb407835be262236c265c03c879ad11..df4a39e11ce43c780890d670db31d04781771611 100644
--- a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
@@ -39,8 +39,7 @@ void StaticAccessedThroughInstanceCheck::storeOptions(
 void StaticAccessedThroughInstanceCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       memberExpr(hasDeclaration(anyOf(cxxMethodDecl(isStaticStorageClass()),
-                                      varDecl(hasStaticStorageDuration()))),
-                 unless(isInTemplateInstantiation()))
+                                      varDecl(hasStaticStorageDuration()))))
           .bind("memberExpression"),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.h b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.h
index d12e8211651f3a83e81c37b895ea84396bec0a48..f80f3605a01fe4ab935e05bdddbd650fa8d57cd6 100644
--- a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.h
@@ -30,6 +30,9 @@ public:
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
 private:
   const unsigned NameSpecifierNestingThreshold;
diff --git a/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.cpp b/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.cpp
index 45194a8e3d97ebb5a647cf51e34263a97f674c56..23f8dbbacb6859943728fc3d58d537f668923ef1 100644
--- a/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.cpp
@@ -39,17 +39,14 @@ void UniqueptrDeleteReleaseCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       cxxDeleteExpr(
           unless(isInTemplateInstantiation()),
-          has(expr(ignoringParenImpCasts(
-              cxxMemberCallExpr(
-                  callee(
-                      memberExpr(hasObjectExpression(allOf(
-                                     unless(isTypeDependent()),
-                                     anyOf(hasType(UniquePtrWithDefaultDelete),
-                                           hasType(pointsTo(
-                                               UniquePtrWithDefaultDelete))))),
-                                 member(cxxMethodDecl(hasName("release"))))
-                          .bind("release_expr")))
-                  .bind("release_call")))))
+          has(cxxMemberCallExpr(
+                  callee(memberExpr(hasObjectExpression(anyOf(
+                                        hasType(UniquePtrWithDefaultDelete),
+                                        hasType(pointsTo(
+                                            UniquePtrWithDefaultDelete)))),
+                                    member(cxxMethodDecl(hasName("release"))))
+                             .bind("release_expr")))
+                  .bind("release_call")))
           .bind("delete"),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.h b/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.h
index 88bb82539ac508eaf98907f708230f522290e02c..a840ac722d19ce138bad524a8389c448233f397d 100644
--- a/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.h
@@ -26,6 +26,9 @@ public:
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
 private:
   const bool PreferResetCall;
diff --git a/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.cpp b/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.cpp
index c4fbeeb4777aa4eca1946281dfb7ac8f718a2e7b..827711e92e8772b1be1982158dcd3987fe7094a5 100644
--- a/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.cpp
@@ -196,12 +196,11 @@ void UppercaseLiteralSuffixCheck::registerMatchers(MatchFinder *Finder) {
   // Sadly, we can't check whether the literal has suffix or not.
   // E.g. i32 suffix still results in 'BuiltinType::Kind::Int'.
   // And such an info is not stored in the *Literal itself.
-  Finder->addMatcher(traverse(TK_AsIs,
+  Finder->addMatcher(
       stmt(eachOf(integerLiteral().bind(IntegerLiteralCheck::Name),
                   floatLiteral().bind(FloatingLiteralCheck::Name)),
            unless(anyOf(hasParent(userDefinedLiteral()),
-                        hasAncestor(isImplicit()),
-                        hasAncestor(substNonTypeTemplateParmExpr()))))),
+                        hasAncestor(substNonTypeTemplateParmExpr())))),
       this);
 }
 
diff --git a/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.h b/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.h
index c31fc06cb4a9e6a5a5b257c772a1c53c56a7a962..1f6e6e6fd572c4770f1610a834bbdbf36eaf16a0 100644
--- a/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/UppercaseLiteralSuffixCheck.h
@@ -28,6 +28,9 @@ public:
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  llvm::Optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
 private:
   template <typename LiteralType>
diff --git a/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp b/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp
index ca8b1b59f89eb58824566c4dc50deb4a8088c2ca..88828f72e6bb4f9a1cfe42cdada114efa6de8b50 100644
--- a/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp
+++ b/clang-tools-extra/clang-tidy/utils/LexerUtils.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "LexerUtils.h"
+#include "clang/AST/AST.h"
 #include "clang/Basic/SourceManager.h"
 
 namespace clang {
@@ -148,6 +149,70 @@ llvm::Optional<Token> getQualifyingToken(tok::TokenKind TK,
   return LastMatchAfterTemplate != None ? LastMatchAfterTemplate
                                         : LastMatchBeforeTemplate;
 }
+
+static bool breakAndReturnEnd(const Stmt &S) {
+  return isa<CompoundStmt, DeclStmt, NullStmt>(S);
+}
+
+static bool breakAndReturnEndPlus1Token(const Stmt &S) {
+  return isa<Expr, DoStmt, ReturnStmt, BreakStmt, ContinueStmt, GotoStmt, SEHLeaveStmt>(S);
+}
+
+// Given a Stmt which does not include it's semicolon this method returns the
+// SourceLocation of the semicolon.
+static SourceLocation getSemicolonAfterStmtEndLoc(const SourceLocation &EndLoc,
+                                                  const SourceManager &SM,
+                                                  const LangOptions &LangOpts) {
+
+  if (EndLoc.isMacroID()) {
+    // Assuming EndLoc points to a function call foo within macro F.
+    // This method is supposed to return location of the semicolon within
+    // those macro arguments:
+    //  F     (      foo()               ;   )
+    //  ^ EndLoc         ^ SpellingLoc   ^ next token of SpellingLoc
+    const SourceLocation SpellingLoc = SM.getSpellingLoc(EndLoc);
+    Optional<Token> NextTok =
+        findNextTokenSkippingComments(SpellingLoc, SM, LangOpts);
+
+    // Was the next token found successfully?
+    // All macro issues are simply resolved by ensuring it's a semicolon.
+    if (NextTok && NextTok->is(tok::TokenKind::semi)) {
+      // Ideally this would return `F` with spelling location `;` (NextTok)
+      // following the examle above. For now simply return NextTok location.
+      return NextTok->getLocation();
+    }
+
+    // Fallthrough to 'normal handling'.
+    //  F     (      foo()              ) ;
+    //  ^ EndLoc         ^ SpellingLoc  ) ^ next token of EndLoc
+  }
+
+  Optional<Token> NextTok = findNextTokenSkippingComments(EndLoc, SM, LangOpts);
+
+  // Testing for semicolon again avoids some issues with macros.
+  if (NextTok && NextTok->is(tok::TokenKind::semi))
+    return NextTok->getLocation();
+
+  return SourceLocation();
+}
+
+SourceLocation getUnifiedEndLoc(const Stmt &S, const SourceManager &SM,
+                                const LangOptions &LangOpts) {
+
+  const Stmt *LastChild = &S;
+  while (!LastChild->children().empty() && !breakAndReturnEnd(*LastChild) &&
+         !breakAndReturnEndPlus1Token(*LastChild)) {
+    for (const Stmt *Child : LastChild->children())
+      LastChild = Child;
+  }
+
+  if (!breakAndReturnEnd(*LastChild) &&
+      breakAndReturnEndPlus1Token(*LastChild))
+    return getSemicolonAfterStmtEndLoc(S.getEndLoc(), SM, LangOpts);
+
+  return S.getEndLoc();
+}
+
 } // namespace lexer
 } // namespace utils
 } // namespace tidy
diff --git a/clang-tools-extra/clang-tidy/utils/LexerUtils.h b/clang-tools-extra/clang-tidy/utils/LexerUtils.h
index 8781b0571e7af5bc7613586b15dbaded70039c13..79ba16ded96800de6a22e2ce642180dde0be29bc 100644
--- a/clang-tools-extra/clang-tidy/utils/LexerUtils.h
+++ b/clang-tools-extra/clang-tidy/utils/LexerUtils.h
@@ -14,6 +14,9 @@
 #include "clang/Lex/Lexer.h"
 
 namespace clang {
+
+class Stmt;
+
 namespace tidy {
 namespace utils {
 namespace lexer {
@@ -104,6 +107,11 @@ llvm::Optional<Token> getQualifyingToken(tok::TokenKind TK,
                                          const ASTContext &Context,
                                          const SourceManager &SM);
 
+/// Stmt->getEndLoc does not always behave the same way depending on Token type.
+/// See implementation for exceptions.
+SourceLocation getUnifiedEndLoc(const Stmt &S, const SourceManager &SM,
+                                const LangOptions &LangOpts);
+
 } // namespace lexer
 } // namespace utils
 } // namespace tidy
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index cd13e013aa502b208bce75e3e237705c93014cf6..aef849d8d8d97791113739bcc43408074a7364e5 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -518,6 +518,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
   if (Params.capabilities.WorkDoneProgress)
     BackgroundIndexProgressState = BackgroundIndexProgress::Empty;
   BackgroundIndexSkipCreate = Params.capabilities.ImplicitProgressCreation;
+  Opts.ImplicitCancellation = !Params.capabilities.CancelsStaleRequests;
 
   llvm::json::Object ServerCaps{
       {"textDocumentSync",
@@ -779,7 +780,7 @@ void ClangdLSPServer::onWorkspaceSymbol(
     const WorkspaceSymbolParams &Params,
     Callback<std::vector<SymbolInformation>> Reply) {
   Server->workspaceSymbols(
-      Params.query, Opts.CodeComplete.Limit,
+      Params.query, Params.limit.getValueOr(Opts.CodeComplete.Limit),
       [Reply = std::move(Reply),
        this](llvm::Expected<std::vector<SymbolInformation>> Items) mutable {
         if (!Items)
@@ -1030,21 +1031,24 @@ void ClangdLSPServer::onCompletion(const CompletionParams &Params,
     vlog("ignored auto-triggered completion, preceding char did not match");
     return Reply(CompletionList());
   }
-  Server->codeComplete(
-      Params.textDocument.uri.file(), Params.position, Opts.CodeComplete,
-      [Reply = std::move(Reply),
-       this](llvm::Expected<CodeCompleteResult> List) mutable {
-        if (!List)
-          return Reply(List.takeError());
-        CompletionList LSPList;
-        LSPList.isIncomplete = List->HasMore;
-        for (const auto &R : List->Completions) {
-          CompletionItem C = R.render(Opts.CodeComplete);
-          C.kind = adjustKindToCapability(C.kind, SupportedCompletionItemKinds);
-          LSPList.items.push_back(std::move(C));
-        }
-        return Reply(std::move(LSPList));
-      });
+  auto Opts = this->Opts.CodeComplete;
+  if (Params.limit && *Params.limit >= 0)
+    Opts.Limit = *Params.limit;
+  Server->codeComplete(Params.textDocument.uri.file(), Params.position, Opts,
+                       [Reply = std::move(Reply), Opts,
+                        this](llvm::Expected<CodeCompleteResult> List) mutable {
+                         if (!List)
+                           return Reply(List.takeError());
+                         CompletionList LSPList;
+                         LSPList.isIncomplete = List->HasMore;
+                         for (const auto &R : List->Completions) {
+                           CompletionItem C = R.render(Opts);
+                           C.kind = adjustKindToCapability(
+                               C.kind, SupportedCompletionItemKinds);
+                           LSPList.items.push_back(std::move(C));
+                         }
+                         return Reply(std::move(LSPList));
+                       });
 }
 
 void ClangdLSPServer::onSignatureHelp(const TextDocumentPositionParams &Params,
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index 164e387bd454878e5a4f2a700e1e66085e8801ee..557689774b148da1f4eadd98dcca9bd899bda9ef 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -9,6 +9,7 @@
 #include "ClangdServer.h"
 #include "CodeComplete.h"
 #include "Config.h"
+#include "Diagnostics.h"
 #include "DumpAST.h"
 #include "FindSymbols.h"
 #include "Format.h"
@@ -81,7 +82,9 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
     if (FIndex)
       FIndex->updateMain(Path, AST);
 
-    std::vector<Diag> Diagnostics = AST.getDiagnostics();
+    assert(AST.getDiagnostics().hasValue() &&
+           "We issue callback only with fresh preambles");
+    std::vector<Diag> Diagnostics = *AST.getDiagnostics();
     if (ServerCallbacks)
       Publish([&]() {
         ServerCallbacks->onDiagnosticsReady(Path, AST.version(),
@@ -150,6 +153,8 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
       DynamicIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex() : nullptr),
       ClangTidyProvider(Opts.ClangTidyProvider),
       WorkspaceRoot(Opts.WorkspaceRoot),
+      Transient(Opts.ImplicitCancellation ? TUScheduler::InvalidateOnUpdate
+                                          : TUScheduler::NoInvalidation),
       DirtyFS(std::make_unique<DraftStoreFS>(TFS, DraftMgr)) {
   // Pass a callback into `WorkScheduler` to extract symbols from a newly
   // parsed file and rebuild the file index synchronously each time an AST
@@ -492,7 +497,7 @@ void ClangdServer::prepareRename(PathRef File, Position Pos,
     // prepareRename is latency-sensitive: we don't query the index, as we
     // only need main-file references
     auto Results =
-        clangd::rename({Pos, NewName.getValueOr("__clangd_rename_dummy"),
+        clangd::rename({Pos, NewName.getValueOr("__clangd_rename_placeholder"),
                         InpAST->AST, File, /*FS=*/nullptr,
                         /*Index=*/nullptr, RenameOpts});
     if (!Results) {
@@ -593,7 +598,7 @@ void ClangdServer::enumerateTweaks(
   };
 
   WorkScheduler->runWithAST("EnumerateTweaks", File, std::move(Action),
-                            TUScheduler::InvalidateOnUpdate);
+                            Transient);
 }
 
 void ClangdServer::applyTweak(PathRef File, Range Sel, StringRef TweakID,
@@ -683,8 +688,7 @@ void ClangdServer::findDocumentHighlights(
         CB(clangd::findDocumentHighlights(InpAST->AST, Pos));
       };
 
-  WorkScheduler->runWithAST("Highlights", File, std::move(Action),
-                            TUScheduler::InvalidateOnUpdate);
+  WorkScheduler->runWithAST("Highlights", File, std::move(Action), Transient);
 }
 
 void ClangdServer::findHover(PathRef File, Position Pos,
@@ -698,8 +702,7 @@ void ClangdServer::findHover(PathRef File, Position Pos,
     CB(clangd::getHover(InpAST->AST, Pos, std::move(Style), Index));
   };
 
-  WorkScheduler->runWithAST("Hover", File, std::move(Action),
-                            TUScheduler::InvalidateOnUpdate);
+  WorkScheduler->runWithAST("Hover", File, std::move(Action), Transient);
 }
 
 void ClangdServer::typeHierarchy(PathRef File, Position Pos, int Resolve,
@@ -771,7 +774,7 @@ void ClangdServer::documentSymbols(llvm::StringRef File,
         CB(clangd::getDocumentSymbols(InpAST->AST));
       };
   WorkScheduler->runWithAST("DocumentSymbols", File, std::move(Action),
-                            TUScheduler::InvalidateOnUpdate);
+                            Transient);
 }
 
 void ClangdServer::foldingRanges(llvm::StringRef File,
@@ -783,7 +786,7 @@ void ClangdServer::foldingRanges(llvm::StringRef File,
         CB(clangd::getFoldingRanges(InpAST->AST));
       };
   WorkScheduler->runWithAST("FoldingRanges", File, std::move(Action),
-                            TUScheduler::InvalidateOnUpdate);
+                            Transient);
 }
 
 void ClangdServer::findImplementations(
@@ -850,7 +853,7 @@ void ClangdServer::documentLinks(PathRef File,
         CB(clangd::getDocumentLinks(InpAST->AST));
       };
   WorkScheduler->runWithAST("DocumentLinks", File, std::move(Action),
-                            TUScheduler::InvalidateOnUpdate);
+                            Transient);
 }
 
 void ClangdServer::semanticHighlights(
@@ -862,7 +865,7 @@ void ClangdServer::semanticHighlights(
         CB(clangd::getSemanticHighlightings(InpAST->AST));
       };
   WorkScheduler->runWithAST("SemanticHighlights", File, std::move(Action),
-                            TUScheduler::InvalidateOnUpdate);
+                            Transient);
 }
 
 void ClangdServer::getAST(PathRef File, Range R,
@@ -902,6 +905,21 @@ void ClangdServer::customAction(PathRef File, llvm::StringRef Name,
   WorkScheduler->runWithAST(Name, File, std::move(Action));
 }
 
+void ClangdServer::diagnostics(PathRef File, Callback<std::vector<Diag>> CB) {
+  auto Action =
+      [CB = std::move(CB)](llvm::Expected<InputsAndAST> InpAST) mutable {
+        if (!InpAST)
+          return CB(InpAST.takeError());
+        if (auto Diags = InpAST->AST.getDiagnostics())
+          return CB(*Diags);
+        // FIXME: Use ServerCancelled error once it is settled in LSP-3.17.
+        return CB(llvm::make_error<LSPError>("server is busy parsing includes",
+                                             ErrorCode::InternalError));
+      };
+
+  WorkScheduler->runWithAST("Diagnostics", File, std::move(Action));
+}
+
 llvm::StringMap<TUScheduler::FileStats> ClangdServer::fileStats() const {
   return WorkScheduler->fileStats();
 }
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index e76ef65922ee076afbf0623d9bbb90359eeb06af..37ac30f70cb4c9c8027bbff5501d027738c12840 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -12,6 +12,7 @@
 #include "../clang-tidy/ClangTidyOptions.h"
 #include "CodeComplete.h"
 #include "ConfigProvider.h"
+#include "Diagnostics.h"
 #include "DraftStore.h"
 #include "FeatureModule.h"
 #include "GlobalCompilationDatabase.h"
@@ -40,6 +41,7 @@
 #include <string>
 #include <type_traits>
 #include <utility>
+#include <vector>
 
 namespace clang {
 namespace clangd {
@@ -64,6 +66,8 @@ public:
     virtual ~Callbacks() = default;
 
     /// Called by ClangdServer when \p Diagnostics for \p File are ready.
+    /// These pushed diagnostics might correspond to an older version of the
+    /// file, they do not interfere with "pull-based" ClangdServer::diagnostics.
     /// May be called concurrently for separate files, not for a single file.
     virtual void onDiagnosticsReady(PathRef File, llvm::StringRef Version,
                                     std::vector<Diag> Diagnostics) {}
@@ -146,6 +150,12 @@ public:
         /*RebuildRatio=*/1,
     };
 
+    /// Cancel certain requests if the file changes before they begin running.
+    /// This is useful for "transient" actions like enumerateTweaks that were
+    /// likely implicitly generated, and avoids redundant work if clients forget
+    /// to cancel. Clients that always cancel stale requests should clear this.
+    bool ImplicitCancellation = true;
+
     /// Clangd will execute compiler drivers matching one of these globs to
     /// fetch system include path.
     std::vector<std::string> QueryDriverGlobs;
@@ -339,6 +349,14 @@ public:
   void customAction(PathRef File, llvm::StringRef Name,
                     Callback<InputsAndAST> Action);
 
+  /// Fetches diagnostics for current version of the \p File. This might fail if
+  /// server is busy (building a preamble) and would require a long time to
+  /// prepare diagnostics. If it fails, clients should wait for
+  /// onSemanticsMaybeChanged and then retry.
+  /// These 'pulled' diagnostics do not interfere with the diagnostics 'pushed'
+  /// to Callbacks::onDiagnosticsReady, and clients may use either or both.
+  void diagnostics(PathRef File, Callback<std::vector<Diag>> CB);
+
   /// Returns estimated memory usage and other statistics for each of the
   /// currently open files.
   /// Overall memory usage of clangd may be significantly more than reported
@@ -391,6 +409,8 @@ private:
 
   llvm::Optional<std::string> WorkspaceRoot;
   llvm::Optional<TUScheduler> WorkScheduler;
+  // Invalidation policy used for actions that we assume are "transient".
+  TUScheduler::ASTActionInvalidation Transient;
 
   // Store of the current versions of the open documents.
   // Only written from the main thread (despite being threadsafe).
diff --git a/clang-tools-extra/clangd/CompileCommands.cpp b/clang-tools-extra/clangd/CompileCommands.cpp
index b55d1b03dee64c80ed288f7097d9fcb7c7ffd8bc..7966b7dfa8a3dcb54391030d5c88393ddf80d515 100644
--- a/clang-tools-extra/clangd/CompileCommands.cpp
+++ b/clang-tools-extra/clangd/CompileCommands.cpp
@@ -96,9 +96,9 @@ std::string detectClangPath() {
     if (auto PathCC = llvm::sys::findProgramByName(Name))
       return resolve(std::move(*PathCC));
   // Fallback: a nonexistent 'clang' binary next to clangd.
-  static int Dummy;
+  static int StaticForMainAddr;
   std::string ClangdExecutable =
-      llvm::sys::fs::getMainExecutable("clangd", (void *)&Dummy);
+      llvm::sys::fs::getMainExecutable("clangd", (void *)&StaticForMainAddr);
   SmallString<128> ClangPath;
   ClangPath = llvm::sys::path::parent_path(ClangdExecutable);
   llvm::sys::path::append(ClangPath, "clang");
@@ -120,8 +120,9 @@ const llvm::Optional<std::string> detectSysroot() {
 }
 
 std::string detectStandardResourceDir() {
-  static int Dummy; // Just an address in this process.
-  return CompilerInvocation::GetResourcesPath("clangd", (void *)&Dummy);
+  static int StaticForMainAddr; // Just an address in this process.
+  return CompilerInvocation::GetResourcesPath("clangd",
+                                              (void *)&StaticForMainAddr);
 }
 
 // The path passed to argv[0] is important:
diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp
index 9aed3c4679f5e260398fe24c74c3dc0018dadc74..8f402ae0615356fbb7715ac8e345c97e8500348a 100644
--- a/clang-tools-extra/clangd/ConfigCompile.cpp
+++ b/clang-tools-extra/clangd/ConfigCompile.cpp
@@ -371,7 +371,7 @@ struct FragmentCompiler {
   }
 
   void compile(Fragment::DiagnosticsBlock &&F) {
-    std::vector<llvm::StringRef> Normalized;
+    std::vector<std::string> Normalized;
     for (const auto &Suppressed : F.Suppress) {
       if (*Suppressed == "*") {
         Out.Apply.push_back([&](const Params &, Config &C) {
@@ -380,15 +380,16 @@ struct FragmentCompiler {
         });
         return;
       }
-      Normalized.push_back(normalizeSuppressedCode(*Suppressed));
+      Normalized.push_back(normalizeSuppressedCode(*Suppressed).str());
     }
     if (!Normalized.empty())
-      Out.Apply.push_back([Normalized](const Params &, Config &C) {
-        if (C.Diagnostics.SuppressAll)
-          return;
-        for (llvm::StringRef N : Normalized)
-          C.Diagnostics.Suppress.insert(N);
-      });
+      Out.Apply.push_back(
+          [Normalized(std::move(Normalized))](const Params &, Config &C) {
+            if (C.Diagnostics.SuppressAll)
+              return;
+            for (llvm::StringRef N : Normalized)
+              C.Diagnostics.Suppress.insert(N);
+          });
 
     compile(std::move(F.ClangTidy));
   }
diff --git a/clang-tools-extra/clangd/FindSymbols.cpp b/clang-tools-extra/clangd/FindSymbols.cpp
index bda5dcadf12eefe12ae64b1d53a98b39cbd13b78..e4846ac7a59d34ac09bd6a2724e4b0b532f5c72e 100644
--- a/clang-tools-extra/clangd/FindSymbols.cpp
+++ b/clang-tools-extra/clangd/FindSymbols.cpp
@@ -376,10 +376,10 @@ public:
 
   /// Builds the document outline for the generated AST.
   std::vector<DocumentSymbol> build() {
-    SymBuilder DummyRoot;
+    SymBuilder Root;
     for (auto &TopLevel : AST.getLocalTopLevelDecls())
-      traverseDecl(TopLevel, DummyRoot);
-    return std::move(std::move(DummyRoot).build().children);
+      traverseDecl(TopLevel, Root);
+    return std::move(std::move(Root).build().children);
   }
 
 private:
diff --git a/clang-tools-extra/clangd/Format.cpp b/clang-tools-extra/clangd/Format.cpp
index d68a5bc672d0a3d10c41c6a2c97f1ab8c422f37c..3963bc21d4033a930b36c7343dab846d2e00cbb8 100644
--- a/clang-tools-extra/clangd/Format.cpp
+++ b/clang-tools-extra/clangd/Format.cpp
@@ -23,7 +23,7 @@ namespace {
 /// as it isn't sure where the errors are and so can't correct.
 /// When editing, it's reasonable to assume code before the cursor is complete.
 void closeBrackets(std::string &Code, const format::FormatStyle &Style) {
-  SourceManagerForFile FileSM("dummy.cpp", Code);
+  SourceManagerForFile FileSM("mock_file.cpp", Code);
   auto &SM = FileSM.get();
   FileID FID = SM.getMainFileID();
   Lexer Lex(FID, SM.getBufferOrFake(FID), SM,
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index 24038f0ec35f1e7b7d2ea82bc90f1da4b67d0d90..fca19428192e92d2a3f81bbffb6efd00d85cd4d3 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -264,9 +264,11 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
   StoreDiags ASTDiags;
 
   llvm::Optional<PreamblePatch> Patch;
+  bool PreserveDiags = true;
   if (Preamble) {
     Patch = PreamblePatch::create(Filename, Inputs, *Preamble);
     Patch->apply(*CI);
+    PreserveDiags = Patch->preserveDiagnostics();
   }
   auto Clang = prepareCompilerInstance(
       std::move(CI), PreamblePCH,
@@ -421,6 +423,8 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
   // tokens from running the preprocessor inside the checks (only
   // modernize-use-trailing-return-type does that today).
   syntax::TokenBuffer Tokens = std::move(CollectTokens).consume();
+  // Makes SelectionTree build much faster.
+  Tokens.indexExpandedTokens();
   std::vector<Decl *> ParsedDecls = Action->takeTopLevelDecls();
   // AST traversals should exclude the preamble, to avoid performance cliffs.
   Clang->getASTContext().setTraversalScope(ParsedDecls);
@@ -441,14 +445,20 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
   // CompilerInstance won't run this callback, do it directly.
   ASTDiags.EndSourceFile();
 
-  std::vector<Diag> Diags = CompilerInvocationDiags;
-  // Add diagnostics from the preamble, if any.
-  if (Preamble)
-    Diags.insert(Diags.end(), Preamble->Diags.begin(), Preamble->Diags.end());
-  // Finally, add diagnostics coming from the AST.
-  {
-    std::vector<Diag> D = ASTDiags.take(CTContext.getPointer());
-    Diags.insert(Diags.end(), D.begin(), D.end());
+  llvm::Optional<std::vector<Diag>> Diags;
+  // FIXME: Also skip generation of diagnostics alltogether to speed up ast
+  // builds when we are patching a stale preamble.
+  if (PreserveDiags) {
+    Diags = CompilerInvocationDiags;
+    // Add diagnostics from the preamble, if any.
+    if (Preamble)
+      Diags->insert(Diags->end(), Preamble->Diags.begin(),
+                    Preamble->Diags.end());
+    // Finally, add diagnostics coming from the AST.
+    {
+      std::vector<Diag> D = ASTDiags.take(CTContext.getPointer());
+      Diags->insert(Diags->end(), D.begin(), D.end());
+    }
   }
   return ParsedAST(Inputs.Version, std::move(Preamble), std::move(Clang),
                    std::move(Action), std::move(Tokens), std::move(Macros),
@@ -493,14 +503,12 @@ llvm::ArrayRef<Decl *> ParsedAST::getLocalTopLevelDecls() {
 
 const MainFileMacros &ParsedAST::getMacros() const { return Macros; }
 
-const std::vector<Diag> &ParsedAST::getDiagnostics() const { return Diags; }
-
 std::size_t ParsedAST::getUsedBytes() const {
   auto &AST = getASTContext();
   // FIXME(ibiryukov): we do not account for the dynamically allocated part of
   // Message and Fixes inside each diagnostic.
-  std::size_t Total =
-      clangd::getUsedBytes(LocalTopLevelDecls) + clangd::getUsedBytes(Diags);
+  std::size_t Total = clangd::getUsedBytes(LocalTopLevelDecls) +
+                      (Diags ? clangd::getUsedBytes(*Diags) : 0);
 
   // FIXME: the rest of the function is almost a direct copy-paste from
   // libclang's clang_getCXTUResourceUsage. We could share the implementation.
@@ -541,8 +549,8 @@ ParsedAST::ParsedAST(llvm::StringRef Version,
                      std::unique_ptr<FrontendAction> Action,
                      syntax::TokenBuffer Tokens, MainFileMacros Macros,
                      std::vector<Decl *> LocalTopLevelDecls,
-                     std::vector<Diag> Diags, IncludeStructure Includes,
-                     CanonicalIncludes CanonIncludes)
+                     llvm::Optional<std::vector<Diag>> Diags,
+                     IncludeStructure Includes, CanonicalIncludes CanonIncludes)
     : Version(Version), Preamble(std::move(Preamble)), Clang(std::move(Clang)),
       Action(std::move(Action)), Tokens(std::move(Tokens)),
       Macros(std::move(Macros)), Diags(std::move(Diags)),
diff --git a/clang-tools-extra/clangd/ParsedAST.h b/clang-tools-extra/clangd/ParsedAST.h
index 263a097b14cba72fff15e22306ece6a3651c8cfd..c1ce6fce702975294cdd2d60973d08776f236533 100644
--- a/clang-tools-extra/clangd/ParsedAST.h
+++ b/clang-tools-extra/clangd/ParsedAST.h
@@ -88,7 +88,9 @@ public:
   /// (These should be const, but RecursiveASTVisitor requires Decl*).
   ArrayRef<Decl *> getLocalTopLevelDecls();
 
-  const std::vector<Diag> &getDiagnostics() const;
+  const llvm::Optional<std::vector<Diag>> &getDiagnostics() const {
+    return Diags;
+  }
 
   /// Returns the estimated size of the AST and the accessory structures, in
   /// bytes. Does not include the size of the preamble.
@@ -120,7 +122,7 @@ private:
             std::unique_ptr<CompilerInstance> Clang,
             std::unique_ptr<FrontendAction> Action, syntax::TokenBuffer Tokens,
             MainFileMacros Macros, std::vector<Decl *> LocalTopLevelDecls,
-            std::vector<Diag> Diags, IncludeStructure Includes,
+            llvm::Optional<std::vector<Diag>> Diags, IncludeStructure Includes,
             CanonicalIncludes CanonIncludes);
 
   std::string Version;
@@ -142,8 +144,8 @@ private:
 
   /// All macro definitions and expansions in the main file.
   MainFileMacros Macros;
-  // Data, stored after parsing.
-  std::vector<Diag> Diags;
+  // Data, stored after parsing. None if AST was built with a stale preamble.
+  llvm::Optional<std::vector<Diag>> Diags;
   // Top-level decls inside the current file. Not that this does not include
   // top-level decls from the preamble.
   std::vector<Decl *> LocalTopLevelDecls;
diff --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h
index 1de1d6cfe5fad974b9496872166396b95bfecb8c..5b9d17840214ed3d46a5017407ad0ec29afa9e5f 100644
--- a/clang-tools-extra/clangd/Preamble.h
+++ b/clang-tools-extra/clangd/Preamble.h
@@ -126,6 +126,9 @@ public:
   /// Returns textual patch contents.
   llvm::StringRef text() const { return PatchContents; }
 
+  /// Whether diagnostics generated using this patch are trustable.
+  bool preserveDiagnostics() const { return PatchContents.empty(); }
+
 private:
   PreamblePatch() = default;
   std::string PatchContents;
diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp
index b3ff124df4de7534c986e3ffeef43d0f78a00ba5..099c8531d341b8590a80584e39531b6aede5f8fe 100644
--- a/clang-tools-extra/clangd/Protocol.cpp
+++ b/clang-tools-extra/clangd/Protocol.cpp
@@ -414,6 +414,12 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R,
     if (auto Implicit = Window->getBoolean("implicitWorkDoneProgressCreate"))
       R.ImplicitProgressCreation = *Implicit;
   }
+  if (auto *General = O->getObject("general")) {
+    if (auto *StaleRequestSupport = General->getObject("staleRequestSupport")) {
+      if (auto Cancel = StaleRequestSupport->getBoolean("cancel"))
+        R.CancelsStaleRequests = *Cancel;
+    }
+  }
   if (auto *OffsetEncoding = O->get("offsetEncoding")) {
     R.offsetEncoding.emplace();
     if (!fromJSON(*OffsetEncoding, *R.offsetEncoding,
@@ -744,7 +750,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &O, const SymbolDetails &S) {
 bool fromJSON(const llvm::json::Value &Params, WorkspaceSymbolParams &R,
               llvm::json::Path P) {
   llvm::json::ObjectMapper O(Params, P);
-  return O && O.map("query", R.query);
+  return O && O.map("query", R.query) &&
+         mapOptOrNull(Params, "limit", R.limit, P);
 }
 
 llvm::json::Value toJSON(const Command &C) {
@@ -845,7 +852,8 @@ bool fromJSON(const llvm::json::Value &Params, CompletionContext &R,
 
 bool fromJSON(const llvm::json::Value &Params, CompletionParams &R,
               llvm::json::Path P) {
-  if (!fromJSON(Params, static_cast<TextDocumentPositionParams &>(R), P))
+  if (!fromJSON(Params, static_cast<TextDocumentPositionParams &>(R), P) ||
+      !mapOptOrNull(Params, "limit", R.limit, P))
     return false;
   if (auto *Context = Params.getAsObject()->get("context"))
     return fromJSON(*Context, R.context, P.field("context"));
diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h
index c6074abcb04e9e7718674a4c5bfbeb2a8048d04a..8e90f1f478314699edacf6cb55377b9c15dec65e 100644
--- a/clang-tools-extra/clangd/Protocol.h
+++ b/clang-tools-extra/clangd/Protocol.h
@@ -475,6 +475,10 @@ struct ClientCapabilities {
   /// window.implicitWorkDoneProgressCreate
   bool ImplicitProgressCreation = false;
 
+  /// Whether the client claims to cancel stale requests.
+  /// general.staleRequestSupport.cancel
+  bool CancelsStaleRequests = false;
+
   /// Whether the client implementation supports a refresh request sent from the
   /// server to the client.
   bool SemanticTokenRefreshSupport = false;
@@ -1052,8 +1056,13 @@ bool operator==(const SymbolDetails &, const SymbolDetails &);
 
 /// The parameters of a Workspace Symbol Request.
 struct WorkspaceSymbolParams {
-  /// A non-empty query string
+  /// A query string to filter symbols by.
+  /// Clients may send an empty string here to request all the symbols.
   std::string query;
+
+  /// Max results to return, overriding global default. 0 means no limit.
+  /// Clangd extension.
+  llvm::Optional<int> limit;
 };
 bool fromJSON(const llvm::json::Value &, WorkspaceSymbolParams &,
               llvm::json::Path);
@@ -1102,6 +1111,10 @@ bool fromJSON(const llvm::json::Value &, CompletionContext &, llvm::json::Path);
 
 struct CompletionParams : TextDocumentPositionParams {
   CompletionContext context;
+
+  /// Max results to return, overriding global default. 0 means no limit.
+  /// Clangd extension.
+  llvm::Optional<int> limit;
 };
 bool fromJSON(const llvm::json::Value &, CompletionParams &, llvm::json::Path);
 
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index 0b4965c427150458c813b7b400ee4c0aea291ecc..cf06eac01a346d7e98de11f1b2d2b99a03503ae5 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -294,7 +294,7 @@ public:
   HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
     Loc = getHighlightableSpellingToken(Loc, SourceMgr);
     if (Loc.isInvalid())
-      return Dummy;
+      return InvalidHighlightingToken;
     const auto *Tok = TB.spelledTokenAt(Loc);
     assert(Tok);
     return addToken(
@@ -395,7 +395,8 @@ private:
   const SourceManager &SourceMgr;
   const LangOptions &LangOpts;
   std::vector<HighlightingToken> Tokens;
-  HighlightingToken Dummy; // returned from addToken(InvalidLoc)
+  // returned from addToken(InvalidLoc)
+  HighlightingToken InvalidHighlightingToken;
 };
 
 llvm::Optional<HighlightingModifier> scopeModifier(const NamedDecl *D) {
diff --git a/clang-tools-extra/clangd/SourceCode.cpp b/clang-tools-extra/clangd/SourceCode.cpp
index 8faed3e046aa399d8b7ebf60f8b7f2c32c54f886..5a9cf05ea818a74a7a09f8c983be604b7dc80b59 100644
--- a/clang-tools-extra/clangd/SourceCode.cpp
+++ b/clang-tools-extra/clangd/SourceCode.cpp
@@ -599,7 +599,7 @@ lex(llvm::StringRef Code, const LangOptions &LangOpts,
         Action) {
   // FIXME: InMemoryFileAdapter crashes unless the buffer is null terminated!
   std::string NullTerminatedCode = Code.str();
-  SourceManagerForFile FileSM("dummy.cpp", NullTerminatedCode);
+  SourceManagerForFile FileSM("mock_file_name.cpp", NullTerminatedCode);
   auto &SM = FileSM.get();
   for (const auto &Tok : syntax::tokenize(SM.getMainFileID(), SM, LangOpts))
     Action(Tok, SM);
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index 1f821f8edd1e06842a6742e7865e433cdcf8dfae..65bbbcd28b40978ee4c352a15930ffdb13f043ac 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -162,10 +162,10 @@ SymbolLocation toIndexLocation(const Location &Loc, std::string &URIStorage) {
 SymbolLocation getPreferredLocation(const Location &ASTLoc,
                                     const SymbolLocation &IdxLoc,
                                     std::string &Scratch) {
-  // Also use a dummy symbol for the index location so that other fields (e.g.
+  // Also use a mock symbol for the index location so that other fields (e.g.
   // definition) are not factored into the preference.
   Symbol ASTSym, IdxSym;
-  ASTSym.ID = IdxSym.ID = SymbolID("dummy_id");
+  ASTSym.ID = IdxSym.ID = SymbolID("mock_symbol_id");
   ASTSym.CanonicalDeclaration = toIndexLocation(ASTLoc, Scratch);
   IdxSym.CanonicalDeclaration = IdxLoc;
   auto Merged = mergeSymbol(ASTSym, IdxSym);
diff --git a/clang-tools-extra/clangd/fuzzer/CMakeLists.txt b/clang-tools-extra/clangd/fuzzer/CMakeLists.txt
index 778b61158304c518bd4c876a145d86f18a0c1c77..18cab4b41e1a0d820c807161fcb97b5b6e015792 100644
--- a/clang-tools-extra/clangd/fuzzer/CMakeLists.txt
+++ b/clang-tools-extra/clangd/fuzzer/CMakeLists.txt
@@ -9,7 +9,7 @@ set(LLVM_LINK_COMPONENTS
 # This fuzzer runs on oss-fuzz, so keep it around even if it looks unreferenced.
 add_llvm_fuzzer(clangd-fuzzer
   clangd-fuzzer.cpp
-  DUMMY_MAIN DummyClangdMain.cpp
+  DUMMY_MAIN FuzzerClangdMain.cpp
   )
 
 clang_target_link_libraries(clangd-fuzzer
diff --git a/clang-tools-extra/clangd/fuzzer/DummyClangdMain.cpp b/clang-tools-extra/clangd/fuzzer/FuzzerClangdMain.cpp
similarity index 91%
rename from clang-tools-extra/clangd/fuzzer/DummyClangdMain.cpp
rename to clang-tools-extra/clangd/fuzzer/FuzzerClangdMain.cpp
index cd5a61217511d203bf972776eb67d46e5d691bae..7b10dbb78201cc92c1ef4ae885a7fbe273ee89f9 100644
--- a/clang-tools-extra/clangd/fuzzer/DummyClangdMain.cpp
+++ b/clang-tools-extra/clangd/fuzzer/FuzzerClangdMain.cpp
@@ -1,4 +1,4 @@
-//===---- DummyClangdMain.cpp - Entry point to sanity check the fuzzer ----===//
+//===--- FuzzerClangdMain.cpp - Entry point to sanity check the fuzzer ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/clang-tools-extra/clangd/index/remote/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/CMakeLists.txt
index eaa000b745e5d871edb7748b1b9810a9c1470b18..2aa6e9b6cfa9d796ad886b622e3de1e59665535f 100644
--- a/clang-tools-extra/clangd/index/remote/CMakeLists.txt
+++ b/clang-tools-extra/clangd/index/remote/CMakeLists.txt
@@ -1,5 +1,7 @@
 if (CLANGD_ENABLE_REMOTE)
   generate_protos(RemoteIndexProto "Index.proto")
+  generate_protos(MonitoringServiceProto "MonitoringService.proto"
+    GRPC)
   generate_protos(RemoteIndexServiceProto "Service.proto"
     DEPENDS "Index.proto"
     GRPC)
@@ -8,6 +10,7 @@ if (CLANGD_ENABLE_REMOTE)
   target_link_libraries(RemoteIndexServiceProto
     PRIVATE
     RemoteIndexProto
+    MonitoringServiceProto
     )
   include_directories(${CMAKE_CURRENT_BINARY_DIR})
   include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../)
@@ -36,6 +39,6 @@ if (CLANGD_ENABLE_REMOTE)
   add_subdirectory(marshalling)
   add_subdirectory(server)
 else()
-  # Provides a dummy implementation of clangdRemoteIndex.
+  # Provides a no-op implementation of clangdRemoteIndex.
   add_subdirectory(unimplemented)
 endif()
diff --git a/clang-tools-extra/clangd/index/remote/MonitoringService.proto b/clang-tools-extra/clangd/index/remote/MonitoringService.proto
new file mode 100644
index 0000000000000000000000000000000000000000..75d807c19005b0c0d503d3af0cf471b06e243dce
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/MonitoringService.proto
@@ -0,0 +1,27 @@
+//===--- MonitoringService.proto - CLangd Remote index monitoring service -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+syntax = "proto2";
+
+package clang.clangd.remote.v1;
+
+message MonitoringInfoRequest {}
+message MonitoringInfoReply {
+  // Time since the server started (in seconds).
+  optional uint64 uptime_seconds = 1;
+  // Time since the index was built on the indexing machine.
+  optional uint64 index_age_seconds = 2;
+  // ID of the indexed commit in Version Control System.
+  optional string index_commit_hash = 3;
+  // URL to the index file.
+  optional string index_link = 4;
+}
+
+service Monitor {
+  rpc MonitoringInfo(MonitoringInfoRequest) returns (MonitoringInfoReply) {}
+}
diff --git a/clang-tools-extra/clangd/index/remote/Service.proto b/clang-tools-extra/clangd/index/remote/Service.proto
index 4e39ff9ec6660eea76f8a965cb52bd1f812ed7db..7c7efa530200d7b66441d9f153dd6500b98f38c7 100644
--- a/clang-tools-extra/clangd/index/remote/Service.proto
+++ b/clang-tools-extra/clangd/index/remote/Service.proto
@@ -23,4 +23,3 @@ service SymbolIndex {
 
   rpc Relations(RelationsRequest) returns (stream RelationsReply) {}
 }
-
diff --git a/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
index 90b62caf524c2b4090695a9ba96131190db47a0d..f1131e53c9192db03dfd718dec310bad4cc4dfbd 100644
--- a/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
+++ b/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
@@ -14,6 +14,7 @@ target_link_libraries(clangd-index-server
   clangDaemon
   clangdSupport
 
+  MonitoringServiceProto
   RemoteIndexProto
   RemoteIndexServiceProto
   clangdRemoteMarshalling
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index be0e844a1f8002c56fcbec4750040994b6b505dd..f3cf131bb8a589b4b61b8c9a269a0c0bd570528d 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -8,7 +8,10 @@
 
 #include "Features.inc"
 #include "Index.pb.h"
+#include "MonitoringService.grpc.pb.h"
+#include "MonitoringService.pb.h"
 #include "Service.grpc.pb.h"
+#include "Service.pb.h"
 #include "index/Index.h"
 #include "index/Serialization.h"
 #include "index/Symbol.h"
@@ -288,11 +291,46 @@ private:
   clangd::SymbolIndex &Index;
 };
 
+class Monitor final : public v1::Monitor::Service {
+public:
+  Monitor(llvm::sys::TimePoint<> IndexAge)
+      : StartTime(std::chrono::system_clock::now()), IndexBuildTime(IndexAge) {}
+
+  void updateIndex(llvm::sys::TimePoint<> UpdateTime) {
+    IndexBuildTime.exchange(UpdateTime);
+  }
+
+private:
+  // FIXME(kirillbobyrev): Most fields should be populated when the index
+  // reloads (probably in adjacent metadata.txt file next to loaded .idx) but
+  // they aren't right now.
+  grpc::Status MonitoringInfo(grpc::ServerContext *Context,
+                              const v1::MonitoringInfoRequest *Request,
+                              v1::MonitoringInfoReply *Reply) override {
+    Reply->set_uptime_seconds(std::chrono::duration_cast<std::chrono::seconds>(
+                                  std::chrono::system_clock::now() - StartTime)
+                                  .count());
+    // FIXME(kirillbobyrev): We are currently making use of the last
+    // modification time of the index artifact to deduce its age. This is wrong
+    // as it doesn't account for the indexing delay. Propagate some metadata
+    // with the index artifacts to indicate time of the commit we indexed.
+    Reply->set_index_age_seconds(
+        std::chrono::duration_cast<std::chrono::seconds>(
+            std::chrono::system_clock::now() - IndexBuildTime.load())
+            .count());
+    return grpc::Status::OK;
+  }
+
+  const llvm::sys::TimePoint<> StartTime;
+  std::atomic<llvm::sys::TimePoint<>> IndexBuildTime;
+};
+
 // Detect changes in \p IndexPath file and load new versions of the index
 // whenever they become available.
 void hotReload(clangd::SwapIndex &Index, llvm::StringRef IndexPath,
                llvm::vfs::Status &LastStatus,
-               llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &FS) {
+               llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &FS,
+               Monitor &Monitor) {
   auto Status = FS->status(IndexPath);
   // Requested file is same as loaded index: no reload is needed.
   if (!Status || (Status->getLastModificationTime() ==
@@ -309,12 +347,13 @@ void hotReload(clangd::SwapIndex &Index, llvm::StringRef IndexPath,
     return;
   }
   Index.reset(std::move(NewIndex));
+  Monitor.updateIndex(Status->getLastModificationTime());
   log("New index version loaded. Last modification time: {0}, size: {1} bytes.",
       Status->getLastModificationTime(), Status->getSize());
 }
 
 void runServerAndWait(clangd::SymbolIndex &Index, llvm::StringRef ServerAddress,
-                      llvm::StringRef IndexPath) {
+                      llvm::StringRef IndexPath, Monitor &Monitor) {
   RemoteIndexServer Service(Index, IndexRoot);
 
   grpc::EnableDefaultHealthCheckService(true);
@@ -327,6 +366,7 @@ void runServerAndWait(clangd::SymbolIndex &Index, llvm::StringRef ServerAddress,
   Builder.AddChannelArgument(GRPC_ARG_MAX_CONNECTION_IDLE_MS,
                              IdleTimeoutSeconds * 1000);
   Builder.RegisterService(&Service);
+  Builder.RegisterService(&Monitor);
   std::unique_ptr<grpc::Server> Server(Builder.BuildAndStart());
   log("Server listening on {0}", ServerAddress);
 
@@ -425,16 +465,18 @@ int main(int argc, char *argv[]) {
   }
   clang::clangd::SwapIndex Index(std::move(SymIndex));
 
-  std::thread HotReloadThread([&Index, &Status, &FS]() {
+  Monitor Monitor(Status->getLastModificationTime());
+
+  std::thread HotReloadThread([&Index, &Status, &FS, &Monitor]() {
     llvm::vfs::Status LastStatus = *Status;
     static constexpr auto RefreshFrequency = std::chrono::seconds(30);
     while (!clang::clangd::shutdownRequested()) {
-      hotReload(Index, llvm::StringRef(IndexPath), LastStatus, FS);
+      hotReload(Index, llvm::StringRef(IndexPath), LastStatus, FS, Monitor);
       std::this_thread::sleep_for(RefreshFrequency);
     }
   });
 
-  runServerAndWait(Index, ServerAddress, IndexPath);
+  runServerAndWait(Index, ServerAddress, IndexPath, Monitor);
 
   HotReloadThread.join();
 }
diff --git a/clang-tools-extra/clangd/quality/CompletionModel.cmake b/clang-tools-extra/clangd/quality/CompletionModel.cmake
index 60c6d2aa8433000df5ce2131d4572ba7718beb2a..41bc2ed1890b0275ed0fde4eb98f323c96db4568 100644
--- a/clang-tools-extra/clangd/quality/CompletionModel.cmake
+++ b/clang-tools-extra/clangd/quality/CompletionModel.cmake
@@ -5,8 +5,8 @@
 # will define a C++ class called ${cpp_class} - which may be a
 # namespace-qualified class name.
 function(gen_decision_forest model filename cpp_class)
-  set(model_compiler ${CMAKE_SOURCE_DIR}/../clang-tools-extra/clangd/quality/CompletionModelCodegen.py)
-  
+  set(model_compiler ${LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR}/clangd/quality/CompletionModelCodegen.py)
+
   set(output_dir ${CMAKE_CURRENT_BINARY_DIR})
   set(header_file ${output_dir}/${filename}.h)
   set(cpp_file ${output_dir}/${filename}.cpp)
diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp
index 853fc57bb906e569bd696e5d82cb4dd9832f5b46..5431046836ca85cb4f18e1008a5300f7bc42b850 100644
--- a/clang-tools-extra/clangd/refactor/Rename.cpp
+++ b/clang-tools-extra/clangd/refactor/Rename.cpp
@@ -22,14 +22,17 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/ParentMapContext.h"
 #include "clang/AST/Stmt.h"
+#include "clang/Basic/CharInfo.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Tooling/Syntax/Tokens.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
 #include <algorithm>
 
 namespace clang {
@@ -178,8 +181,7 @@ enum class ReasonToReject {
   UnsupportedSymbol,
   AmbiguousSymbol,
 
-  // name validation.
-  RenameToKeywords,
+  // name validation. FIXME: reconcile with InvalidName
   SameName,
 };
 
@@ -241,8 +243,6 @@ llvm::Error makeError(ReasonToReject Reason) {
       return "symbol is not a supported kind (e.g. namespace, macro)";
     case ReasonToReject::AmbiguousSymbol:
       return "there are multiple symbols at the given location";
-    case ReasonToReject::RenameToKeywords:
-      return "the chosen name is a keyword";
     case ReasonToReject::SameName:
       return "new name is the same as the old name";
     }
@@ -437,6 +437,7 @@ struct InvalidName {
   enum Kind {
     Keywords,
     Conflict,
+    BadIdentifier,
   };
   Kind K;
   std::string Details;
@@ -447,6 +448,8 @@ std::string toString(InvalidName::Kind K) {
     return "Keywords";
   case InvalidName::Conflict:
     return "Conflict";
+  case InvalidName::BadIdentifier:
+    return "BadIdentifier";
   }
   llvm_unreachable("unhandled InvalidName kind");
 }
@@ -459,12 +462,31 @@ llvm::Error makeError(InvalidName Reason) {
                            Reason.Details);
     case InvalidName::Conflict:
       return llvm::formatv("conflict with the symbol in {0}", Reason.Details);
+    case InvalidName::BadIdentifier:
+      return llvm::formatv("the chosen name \"{0}\" is not a valid identifier",
+                           Reason.Details);
     }
     llvm_unreachable("unhandled InvalidName kind");
   };
   return error("invalid name: {0}", Message(Reason));
 }
 
+static bool mayBeValidIdentifier(llvm::StringRef Ident) {
+  assert(llvm::json::isUTF8(Ident));
+  if (Ident.empty())
+    return false;
+  // We don't check all the rules for non-ascii characters (most are allowed).
+  bool AllowDollar = true; // lenient
+  if (llvm::isASCII(Ident.front()) &&
+      !isIdentifierHead(Ident.front(), AllowDollar))
+    return false;
+  for (char C : Ident) {
+    if (llvm::isASCII(C) && !isIdentifierBody(C, AllowDollar))
+      return false;
+  }
+  return true;
+}
+
 // Check if we can rename the given RenameDecl into NewName.
 // Return details if the rename would produce a conflict.
 llvm::Optional<InvalidName> checkName(const NamedDecl &RenameDecl,
@@ -476,6 +498,8 @@ llvm::Optional<InvalidName> checkName(const NamedDecl &RenameDecl,
   llvm::Optional<InvalidName> Result;
   if (isKeyword(NewName, ASTCtx.getLangOpts()))
     Result = InvalidName{InvalidName::Keywords, NewName.str()};
+  else if (!mayBeValidIdentifier(NewName))
+    Result = InvalidName{InvalidName::BadIdentifier, NewName.str()};
   else {
     // Name conflict detection.
     // Function conflicts are subtle (overloading), so ignore them.
diff --git a/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp b/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
index c603861c3d6963c98d1f403ef152b5b8ee7139cb..a4db11f1a364078edc585fc011e4011c3635e35b 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
@@ -376,7 +376,7 @@ bool eligibleForExtraction(const SelectionTree::Node *N) {
   if (llvm::isa<DeclRefExpr>(E) || llvm::isa<MemberExpr>(E))
     return false;
 
-  // Extracting Exprs like a = 1 gives dummy = a = 1 which isn't useful.
+  // Extracting Exprs like a = 1 gives placeholder = a = 1 which isn't useful.
   // FIXME: we could still hoist the assignment, and leave the variable there?
   ParsedBinaryOperator BinOp;
   if (BinOp.parse(*N) && BinaryOperator::isAssignmentOp(BinOp.Kind))
@@ -387,7 +387,7 @@ bool eligibleForExtraction(const SelectionTree::Node *N) {
   if (!Parent)
     return false;
   // We don't want to extract expressions used as statements, that would leave
-  // a `dummy;` around that has no effect.
+  // a `placeholder;` around that has no effect.
   // Unfortunately because the AST doesn't have ExprStmt, we have to check in
   // this roundabout way.
   if (childExprIsStmt(Parent->ASTNode.get<Stmt>(),
@@ -422,7 +422,7 @@ const SelectionTree::Node *computeExtractedExpr(const SelectionTree::Node *N) {
       llvm::isa<MemberExpr>(SelectedExpr))
     if (const SelectionTree::Node *Call = getCallExpr(N))
       TargetNode = Call;
-  // Extracting Exprs like a = 1 gives dummy = a = 1 which isn't useful.
+  // Extracting Exprs like a = 1 gives placeholder = a = 1 which isn't useful.
   if (const BinaryOperator *BinOpExpr =
           dyn_cast_or_null<BinaryOperator>(SelectedExpr)) {
     if (BinOpExpr->getOpcode() == BinaryOperatorKind::BO_Assign)
@@ -433,13 +433,13 @@ const SelectionTree::Node *computeExtractedExpr(const SelectionTree::Node *N) {
   return TargetNode;
 }
 
-/// Extracts an expression to the variable dummy
+/// Extracts an expression to the variable placeholder
 /// Before:
 /// int x = 5 + 4 * 3;
 ///         ^^^^^
 /// After:
-/// auto dummy = 5 + 4;
-/// int x = dummy * 3;
+/// auto placeholder = 5 + 4;
+/// int x = placeholder * 3;
 class ExtractVariable : public Tweak {
 public:
   const char *id() const override final;
@@ -476,7 +476,7 @@ bool ExtractVariable::prepare(const Selection &Inputs) {
 Expected<Tweak::Effect> ExtractVariable::apply(const Selection &Inputs) {
   tooling::Replacements Result;
   // FIXME: get variable name from user or suggest based on type
-  std::string VarName = "dummy";
+  std::string VarName = "placeholder";
   SourceRange Range = Target->getExtractionChars();
   // insert new variable declaration
   if (auto Err = Result.add(Target->insertDeclaration(VarName, Range)))
diff --git a/clang-tools-extra/clangd/support/Trace.cpp b/clang-tools-extra/clangd/support/Trace.cpp
index d69b1c2bbde593f2da0054b047d421f394c29b0f..9cfc58c377331dc44294fd29f5f26750263c55d0 100644
--- a/clang-tools-extra/clangd/support/Trace.cpp
+++ b/clang-tools-extra/clangd/support/Trace.cpp
@@ -112,14 +112,14 @@ private:
             "s",
             llvm::json::Object{{"id", FlowID},
                                {"name", "Context crosses threads"},
-                               {"cat", "dummy"}},
+                               {"cat", "mock_cat"}},
             (*Parent)->TID, (*Parent)->StartTime);
         Tracer->jsonEvent(
             "f",
             llvm::json::Object{{"id", FlowID},
                                {"bp", "e"},
                                {"name", "Context crosses threads"},
-                               {"cat", "dummy"}},
+                               {"cat", "mock_cat"}},
             TID);
       }
     }
diff --git a/clang-tools-extra/clangd/tool/Check.cpp b/clang-tools-extra/clangd/tool/Check.cpp
index 00a4609e5134c71fa27f5d64dad53e3e9e3beffb..20b86daff8af3d46a758f092a66f44004f2b299c 100644
--- a/clang-tools-extra/clangd/tool/Check.cpp
+++ b/clang-tools-extra/clangd/tool/Check.cpp
@@ -181,7 +181,7 @@ public:
       elog("Failed to build AST");
       return false;
     }
-    ErrCount += showErrors(llvm::makeArrayRef(AST->getDiagnostics())
+    ErrCount += showErrors(llvm::makeArrayRef(*AST->getDiagnostics())
                                .drop_front(Preamble->Diags.size()));
 
     if (Opts.BuildDynamicSymbolIndex) {
diff --git a/clang-tools-extra/clangd/unittests/ClangdTests.cpp b/clang-tools-extra/clangd/unittests/ClangdTests.cpp
index 15320e8bd8e8009c17f46b40fb4f93008e6f5ef7..49e1f7aa93b678183010f6fd54a24bd7dc7a4c91 100644
--- a/clang-tools-extra/clangd/unittests/ClangdTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ClangdTests.cpp
@@ -407,9 +407,9 @@ TEST(ClangdServerTest, SearchLibDir) {
 
   // Put crtbegin.o into LibDir/64 to trick clang into thinking there's a gcc
   // installation there.
-  SmallString<64> DummyLibFile;
-  llvm::sys::path::append(DummyLibFile, LibDir, "64", "crtbegin.o");
-  FS.Files[DummyLibFile] = "";
+  SmallString<64> MockLibFile;
+  llvm::sys::path::append(MockLibFile, LibDir, "64", "crtbegin.o");
+  FS.Files[MockLibFile] = "";
 
   SmallString<64> IncludeDir("/randomusr/include/c++");
   llvm::sys::path::append(IncludeDir, Version);
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 0ff1e83b7613e313df0b5bcf1fa17bb37b231483..a57ae49f91591da48c874e2b1e2f04df88a3a67f 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -1253,6 +1253,19 @@ TEST(SignatureHelpTest, Overloads) {
   EXPECT_EQ(0, Results.activeParameter);
 }
 
+TEST(SignatureHelpTest, OverloadInitListRegression) {
+  auto Results = signatures(R"cpp(
+    struct A {int x;};
+    struct B {B(A);};
+    void f();
+    int main() {
+      B b({1});
+      f(^);
+    }
+  )cpp");
+  EXPECT_THAT(Results.signatures, UnorderedElementsAre(Sig("f() -> void")));
+}
+
 TEST(SignatureHelpTest, DefaultArgs) {
   auto Results = signatures(R"cpp(
     void bar(int x, int y = 0);
diff --git a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
index aebb231f39f93056cdc77df8bcf032f838bf27d4..d5b4a08a422953e5a632c47041193808374e7c6a 100644
--- a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
@@ -135,7 +135,7 @@ o]]();
   auto TU = TestTU::withCode(Test.code());
   TU.ClangTidyProvider = addTidyChecks("google-explicit-constructor");
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       ElementsAre(
           // This range spans lines.
           AllOf(Diag(Test.range("typo"),
@@ -173,14 +173,14 @@ o]]();
 TEST(DiagnosticsTest, FlagsMatter) {
   Annotations Test("[[void]] main() {} // error-ok");
   auto TU = TestTU::withCode(Test.code());
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               ElementsAre(AllOf(Diag(Test.range(), "'main' must return 'int'"),
                                 WithFix(Fix(Test.range(), "int",
                                             "change 'void' to 'int'")))));
   // Same code built as C gets different diagnostics.
   TU.Filename = "Plain.c";
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       ElementsAre(AllOf(
           Diag(Test.range(), "return type of 'main' is not 'int'"),
           WithFix(Fix(Test.range(), "int", "change return type to 'int'")))));
@@ -192,7 +192,7 @@ TEST(DiagnosticsTest, DiagnosticPreamble) {
   )cpp");
 
   auto TU = TestTU::withCode(Test.code());
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               ElementsAre(::testing::AllOf(
                   Diag(Test.range(), "'not-found.h' file not found"),
                   DiagSource(Diag::Clang), DiagName("pp_file_not_found"))));
@@ -209,7 +209,7 @@ TEST(DiagnosticsTest, DeduplicatedClangTidyDiagnostics) {
                                        "hicpp-uppercase-literal-suffix");
   // Verify that we filter out the duplicated diagnostic message.
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(::testing::AllOf(
           Diag(Test.range(),
                "floating point literal has suffix 'f', which is not uppercase"),
@@ -229,7 +229,7 @@ TEST(DiagnosticsTest, DeduplicatedClangTidyDiagnostics) {
   // The check doesn't handle template instantiations which ends up emitting
   // duplicated messages, verify that we deduplicate them.
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(::testing::AllOf(
           Diag(Test.range(),
                "floating point literal has suffix 'f', which is not uppercase"),
@@ -254,7 +254,7 @@ TEST(DiagnosticsTest, ClangTidy) {
                                        "modernize-deprecated-headers,"
                                        "modernize-use-trailing-return-type");
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(
           AllOf(Diag(Test.range("deprecated"),
                      "inclusion of deprecated C++ header 'assert.h'; consider "
@@ -296,7 +296,7 @@ TEST(DiagnosticsTest, ClangTidyEOF) {
   TU.AdditionalFiles["a.h"] = TU.AdditionalFiles["b.h"] = "";
   TU.ClangTidyProvider = addTidyChecks("llvm-include-order");
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       Contains(AllOf(Diag(Test.range(), "#includes are not sorted properly"),
                      DiagSource(Diag::ClangTidy),
                      DiagName("llvm-include-order"))));
@@ -314,7 +314,7 @@ TEST(DiagnosticTest, TemplatesInHeaders) {
   TestTU TU = TestTU::withCode(Main.code());
   TU.HeaderCode = Header.code().str();
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       ElementsAre(AllOf(
           Diag(Main.range(), "in template: base specifier must name a class"),
           WithNote(Diag(Header.range(), "error occurred here"),
@@ -340,7 +340,7 @@ TEST(DiagnosticTest, MakeUnique) {
     }
     }
   )cpp";
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(
                   Diag(Main.range(),
                        "in template: "
@@ -368,7 +368,7 @@ TEST(DiagnosticTest, NoMultipleDiagnosticInFlight) {
   TestTU TU = TestTU::withCode(Main.code());
   TU.ClangTidyProvider = addTidyChecks("modernize-loop-convert");
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(::testing::AllOf(
           Diag(Main.range(), "use range-based for loop instead"),
           DiagSource(Diag::ClangTidy), DiagName("modernize-loop-convert"))));
@@ -384,14 +384,14 @@ TEST(DiagnosticTest, RespectsDiagnosticConfig) {
   )cpp");
   auto TU = TestTU::withCode(Main.code());
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       ElementsAre(Diag(Main.range(), "use of undeclared identifier 'unknown'"),
                   Diag(Main.range("ret"),
                        "void function 'x' should not return a value")));
   Config Cfg;
   Cfg.Diagnostics.Suppress.insert("return-type");
   WithContextValue WithCfg(Config::Key, std::move(Cfg));
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               ElementsAre(Diag(Main.range(),
                                "use of undeclared identifier 'unknown'")));
 }
@@ -413,7 +413,7 @@ TEST(DiagnosticTest, ClangTidySuppressionComment) {
   TestTU TU = TestTU::withCode(Main.code());
   TU.ClangTidyProvider = addTidyChecks("bugprone-integer-division");
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(::testing::AllOf(
           Diag(Main.range(), "result of integer division used in a floating "
                              "point context; possible loss of precision"),
@@ -431,7 +431,7 @@ TEST(DiagnosticTest, ClangTidyWarningAsError) {
   TU.ClangTidyProvider =
       addTidyChecks("bugprone-integer-division", "bugprone-integer-division");
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(::testing::AllOf(
           Diag(Main.range(), "result of integer division used in a floating "
                              "point context; possible loss of precision"),
@@ -450,7 +450,7 @@ TEST(DiagnosticTest, LongFixMessages) {
   )cpp");
   TestTU TU = TestTU::withCode(Source.code());
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       ElementsAre(WithFix(Fix(
           Source.range(),
           "somereallyreallyreallyreallyreallyreallyreallyreallylongidentifier",
@@ -466,7 +466,7 @@ n]] = 10; // error-ok
     }
   )cpp");
   TU.Code = std::string(Source.code());
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               ElementsAre(WithFix(
                   Fix(Source.range(), "ident", "change 'ide\\…' to 'ident'"))));
 }
@@ -481,7 +481,7 @@ TEST(DiagnosticTest, ClangTidySuppressionCommentTrumpsWarningAsError) {
   TestTU TU = TestTU::withCode(Main.code());
   TU.ClangTidyProvider =
       addTidyChecks("bugprone-integer-division", "bugprone-integer-division");
-  EXPECT_THAT(TU.build().getDiagnostics(), UnorderedElementsAre());
+  EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre());
 }
 
 TEST(DiagnosticTest, ClangTidyNoLiteralDataInMacroToken) {
@@ -496,7 +496,7 @@ TEST(DiagnosticTest, ClangTidyNoLiteralDataInMacroToken) {
   )cpp");
   TestTU TU = TestTU::withCode(Main.code());
   TU.ClangTidyProvider = addTidyChecks("bugprone-bad-signal-to-kill-thread");
-  EXPECT_THAT(TU.build().getDiagnostics(), UnorderedElementsAre()); // no-crash
+  EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre()); // no-crash
 }
 
 TEST(DiagnosticTest, ElseAfterReturnRange) {
@@ -513,7 +513,7 @@ TEST(DiagnosticTest, ElseAfterReturnRange) {
   TestTU TU = TestTU::withCode(Main.code());
   TU.ClangTidyProvider = addTidyChecks("llvm-else-after-return");
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       ElementsAre(Diag(Main.range(), "do not use 'else' after 'return'")));
 }
 
@@ -532,7 +532,7 @@ TEST(DiagnosticsTest, Preprocessor) {
     #endif
     )cpp");
   EXPECT_THAT(
-      TestTU::withCode(Test.code()).build().getDiagnostics(),
+      *TestTU::withCode(Test.code()).build().getDiagnostics(),
       ElementsAre(Diag(Test.range(), "use of undeclared identifier 'b'")));
 }
 
@@ -542,7 +542,7 @@ TEST(DiagnosticsTest, IgnoreVerify) {
   )cpp");
   TU.ExtraArgs.push_back("-Xclang");
   TU.ExtraArgs.push_back("-verify");
-  EXPECT_THAT(TU.build().getDiagnostics(), IsEmpty());
+  EXPECT_THAT(*TU.build().getDiagnostics(), IsEmpty());
 }
 
 // Recursive main-file include is diagnosed, and doesn't crash.
@@ -552,7 +552,7 @@ TEST(DiagnosticsTest, RecursivePreamble) {
     int symbol;
   )cpp");
   TU.Filename = "foo.h";
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               ElementsAre(DiagName("pp_including_mainfile_in_preamble")));
   EXPECT_THAT(TU.build().getLocalTopLevelDecls(), SizeIs(1));
 }
@@ -565,7 +565,7 @@ TEST(DiagnosticsTest, RecursivePreamblePragmaOnce) {
     int symbol;
   )cpp");
   TU.Filename = "foo.h";
-  EXPECT_THAT(TU.build().getDiagnostics(), IsEmpty());
+  EXPECT_THAT(*TU.build().getDiagnostics(), IsEmpty());
   EXPECT_THAT(TU.build().getLocalTopLevelDecls(), SizeIs(1));
 }
 
@@ -581,7 +581,7 @@ TEST(DiagnosticsTest, RecursivePreambleIfndefGuard) {
   )cpp");
   TU.Filename = "foo.h";
   // FIXME: should be no errors here.
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               ElementsAre(DiagName("pp_including_mainfile_in_preamble")));
   EXPECT_THAT(TU.build().getLocalTopLevelDecls(), SizeIs(1));
 }
@@ -598,7 +598,7 @@ TEST(DiagnosticsTest, InsideMacros) {
       return $bar[[TEN]];
     }
     )cpp");
-  EXPECT_THAT(TestTU::withCode(Test.code()).build().getDiagnostics(),
+  EXPECT_THAT(*TestTU::withCode(Test.code()).build().getDiagnostics(),
               ElementsAre(Diag(Test.range("foo"),
                                "cannot initialize return object of type "
                                "'int *' with an rvalue of type 'int'"),
@@ -614,7 +614,7 @@ TEST(DiagnosticsTest, NoFixItInMacro) {
     [[Define]](main) // error-ok
   )cpp");
   auto TU = TestTU::withCode(Test.code());
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               ElementsAre(AllOf(Diag(Test.range(), "'main' must return 'int'"),
                                 Not(WithFix(_)))));
 }
@@ -625,7 +625,7 @@ TEST(ClangdTest, MSAsm) {
   llvm::InitializeAllTargetInfos(); // As in ClangdMain
   auto TU = TestTU::withCode("void fn() { __asm { cmp cl,64 } }");
   TU.ExtraArgs = {"-fms-extensions"};
-  EXPECT_THAT(TU.build().getDiagnostics(), IsEmpty());
+  EXPECT_THAT(*TU.build().getDiagnostics(), IsEmpty());
 }
 
 TEST(DiagnosticsTest, ToLSP) {
@@ -783,7 +783,7 @@ class T {
   TU.ExternalIndex = Index.get();
 
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAreArray(
           {AllOf(Diag(Test.range("nested"),
                       "incomplete type 'ns::X' named in nested name specifier"),
@@ -868,7 +868,7 @@ int main() {
       MemIndex::build(std::move(Slab).build(), RefSlab(), RelationSlab());
   TU.ExternalIndex = Index.get();
 
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(
                   Diag(Test.range("base"), "base class has incomplete type"),
                   Diag(Test.range("access"),
@@ -901,7 +901,7 @@ using Type = ns::$template[[Foo]]<int>;
   TU.ExternalIndex = Index.get();
 
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(
           AllOf(Diag(Test.range("unqualified1"), "unknown type name 'X'"),
                 DiagName("unknown_typename"),
@@ -946,7 +946,7 @@ void foo() {
        SymbolWithHeader{"na::nb::X", "unittest:///b.h", "\"b.h\""}});
   TU.ExternalIndex = Index.get();
 
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(AllOf(
                   Diag(Test.range("unqualified"), "unknown type name 'X'"),
                   DiagName("unknown_typename"),
@@ -967,7 +967,7 @@ TEST(IncludeFixerTest, NoCrashMemebrAccess) {
   TU.ExternalIndex = Index.get();
 
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(Diag(Test.range(), "no member named 'xy' in 'X'")));
 }
 
@@ -1002,7 +1002,7 @@ void bar(X *x) {
   TU.ExternalIndex = Index.get();
 
   auto Parsed = TU.build();
-  for (const auto &D : Parsed.getDiagnostics()) {
+  for (const auto &D : *Parsed.getDiagnostics()) {
     if (D.Fixes.size() != 1) {
       ADD_FAILURE() << "D.Fixes.size() != 1";
       continue;
@@ -1027,7 +1027,7 @@ void g() {  ns::$[[scope]]::X_Y();  }
   TU.ExternalIndex = Index.get();
 
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(AllOf(
           Diag(Test.range(), "no member named 'scope' in namespace 'ns'"),
           DiagName("no_member"),
@@ -1055,7 +1055,7 @@ void f() {
   TU.ExternalIndex = Index.get();
 
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       UnorderedElementsAre(
           AllOf(
               Diag(Test.range("q1"), "use of undeclared identifier 'clangd'; "
@@ -1098,7 +1098,7 @@ namespace c {
       SymbolWithHeader{"a::X", "unittest:///x.h", "\"x.h\""});
   TU.ExternalIndex = Index.get();
 
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(AllOf(
                   Diag(Test.range(), "no type named 'X' in namespace 'a'"),
                   DiagName("typename_nested_not_found"),
@@ -1124,7 +1124,7 @@ TEST(IncludeFixerTest, NoCrashOnTemplateInstantiations) {
   TU.ExternalIndex = Index.get();
 
   EXPECT_THAT(
-      TU.build().getDiagnostics(),
+      *TU.build().getDiagnostics(),
       ElementsAre(Diag(Test.range(), "use of undeclared identifier 'a'")));
 }
 
@@ -1135,7 +1135,7 @@ TEST(DiagsInHeaders, DiagInsideHeader) {
   Annotations Header("[[no_type_spec]]; // error-ok");
   TestTU TU = TestTU::withCode(Main.code());
   TU.AdditionalFiles = {{"a.h", std::string(Header.code())}};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(AllOf(
                   Diag(Main.range(), "in included file: C++ requires a "
                                      "type specifier for all declarations"),
@@ -1149,7 +1149,7 @@ TEST(DiagsInHeaders, DiagInTransitiveInclude) {
   TestTU TU = TestTU::withCode(Main.code());
   TU.AdditionalFiles = {{"a.h", "#include \"b.h\""},
                         {"b.h", "no_type_spec; // error-ok"}};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(
                   Diag(Main.range(), "in included file: C++ requires a "
                                      "type specifier for all declarations")));
@@ -1163,7 +1163,7 @@ TEST(DiagsInHeaders, DiagInMultipleHeaders) {
   TestTU TU = TestTU::withCode(Main.code());
   TU.AdditionalFiles = {{"a.h", "no_type_spec; // error-ok"},
                         {"b.h", "no_type_spec; // error-ok"}};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(
                   Diag(Main.range("a"), "in included file: C++ requires a type "
                                         "specifier for all declarations"),
@@ -1180,7 +1180,7 @@ TEST(DiagsInHeaders, PreferExpansionLocation) {
   TU.AdditionalFiles = {
       {"a.h", "#include \"b.h\"\n"},
       {"b.h", "#ifndef X\n#define X\nno_type_spec; // error-ok\n#endif"}};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(Diag(Main.range(),
                                         "in included file: C++ requires a type "
                                         "specifier for all declarations")));
@@ -1198,7 +1198,7 @@ TEST(DiagsInHeaders, PreferExpansionLocationMacros) {
       {"a.h", "#include \"c.h\"\n"},
       {"b.h", "#include \"c.h\"\n"},
       {"c.h", "#ifndef X\n#define X\nno_type_spec; // error-ok\n#endif"}};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(
                   Diag(Main.range(), "in included file: C++ requires a "
                                      "type specifier for all declarations")));
@@ -1227,7 +1227,7 @@ TEST(DiagsInHeaders, LimitDiagsOutsideMainFile) {
       no_type_spec_9;
       no_type_spec_10;
       #endif)cpp"}};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(
                   Diag(Main.range(), "in included file: C++ requires a "
                                      "type specifier for all declarations")));
@@ -1242,7 +1242,7 @@ TEST(DiagsInHeaders, OnlyErrorOrFatal) {
     int x = 5/0;)cpp");
   TestTU TU = TestTU::withCode(Main.code());
   TU.AdditionalFiles = {{"a.h", std::string(Header.code())}};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(AllOf(
                   Diag(Main.range(), "in included file: C++ requires "
                                      "a type specifier for all declarations"),
@@ -1260,7 +1260,7 @@ TEST(DiagsInHeaders, OnlyDefaultErrorOrFatal) {
   TU.AdditionalFiles = {{"a.h", std::string(Header.code())}};
   // promote warnings to errors.
   TU.ExtraArgs = {"-Werror", "-Wunused"};
-  EXPECT_THAT(TU.build().getDiagnostics(), IsEmpty());
+  EXPECT_THAT(*TU.build().getDiagnostics(), IsEmpty());
 }
 
 TEST(DiagsInHeaders, FromNonWrittenSources) {
@@ -1273,7 +1273,7 @@ TEST(DiagsInHeaders, FromNonWrittenSources) {
   TestTU TU = TestTU::withCode(Main.code());
   TU.AdditionalFiles = {{"a.h", std::string(Header.code())}};
   TU.ExtraArgs = {"-DFOO=NOOO"};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(AllOf(
                   Diag(Main.range(),
                        "in included file: use of undeclared identifier 'NOOO'"),
@@ -1291,7 +1291,7 @@ TEST(DiagsInHeaders, ErrorFromMacroExpansion) {
   X;)cpp");
   TestTU TU = TestTU::withCode(Main.code());
   TU.AdditionalFiles = {{"a.h", std::string(Header.code())}};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(
                   Diag(Main.range(), "in included file: use of undeclared "
                                      "identifier 'foo'; did you mean 'fo'?")));
@@ -1308,7 +1308,7 @@ TEST(DiagsInHeaders, ErrorFromMacroArgument) {
   X(foo);)cpp");
   TestTU TU = TestTU::withCode(Main.code());
   TU.AdditionalFiles = {{"a.h", std::string(Header.code())}};
-  EXPECT_THAT(TU.build().getDiagnostics(),
+  EXPECT_THAT(*TU.build().getDiagnostics(),
               UnorderedElementsAre(
                   Diag(Main.range(), "in included file: use of undeclared "
                                      "identifier 'foo'; did you mean 'fo'?")));
@@ -1320,7 +1320,7 @@ TEST(IgnoreDiags, FromNonWrittenInclude) {
   TU.AdditionalFiles = {{"a.h", "void main();"}};
   // The diagnostic "main must return int" is from the header, we don't attempt
   // to render it in the main file as there is no written location there.
-  EXPECT_THAT(TU.build().getDiagnostics(), UnorderedElementsAre());
+  EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre());
 }
 
 TEST(ToLSPDiag, RangeIsInMain) {
diff --git a/clang-tools-extra/clangd/unittests/GlobalCompilationDatabaseTests.cpp b/clang-tools-extra/clangd/unittests/GlobalCompilationDatabaseTests.cpp
index 2ec64128485bf22761426b8269ff1241413ec7ca..9c02f697d46c6ed59ca91f1bf901f2964321786a 100644
--- a/clang-tools-extra/clangd/unittests/GlobalCompilationDatabaseTests.cpp
+++ b/clang-tools-extra/clangd/unittests/GlobalCompilationDatabaseTests.cpp
@@ -463,7 +463,9 @@ MATCHER_P2(hasFlag, Flag, Path, "") {
   return true;
 }
 
-auto hasFlag(llvm::StringRef Flag) { return hasFlag(Flag, "dummy.cc"); }
+auto hasFlag(llvm::StringRef Flag) {
+  return hasFlag(Flag, "mock_file_name.cc");
+}
 
 TEST_F(DirectoryBasedGlobalCompilationDatabaseCacheTest, Cacheable) {
   MockFS FS;
@@ -507,15 +509,15 @@ TEST_F(DirectoryBasedGlobalCompilationDatabaseCacheTest, Cacheable) {
   // compile_commands.json takes precedence over compile_flags.txt.
   FS.Files["foo/compile_commands.json"] =
       llvm::formatv(R"json([{
-    "file": "{0}/foo/dummy.cc",
-    "command": "clang -DBAZ dummy.cc",
+    "file": "{0}/foo/mock_file.cc",
+    "command": "clang -DBAZ mock_file.cc",
     "directory": "{0}/foo",
   }])json",
                     llvm::sys::path::convert_to_slash(testRoot()));
   EXPECT_EQ(FooBar, lookupCDB(GDB, testPath("foo/test.cc"), Stale))
       << "cache still valid";
   auto Baz = lookupCDB(GDB, testPath("foo/test.cc"), Fresh);
-  EXPECT_THAT(Baz, hasFlag("-DBAZ", testPath("foo/dummy.cc")))
+  EXPECT_THAT(Baz, hasFlag("-DBAZ", testPath("foo/mock_file.cc")))
       << "compile_commands overrides compile_flags";
 
   // Removing compile_commands.json reveals compile_flags.txt again.
diff --git a/clang-tools-extra/clangd/unittests/ModulesTests.cpp b/clang-tools-extra/clangd/unittests/ModulesTests.cpp
index 83d6b28d6dfc36f8f95d008211fcf63426403a60..b56b91836508f1537562931e6138d26e5126eb42 100644
--- a/clang-tools-extra/clangd/unittests/ModulesTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ModulesTests.cpp
@@ -61,7 +61,7 @@ TEST(Modules, PreambleBuildVisibility) {
       header "module.h"
     }
 )modulemap";
-  EXPECT_TRUE(TU.build().getDiagnostics().empty());
+  EXPECT_TRUE(TU.build().getDiagnostics()->empty());
 }
 
 TEST(Modules, Diagnostic) {
diff --git a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
index b96d1243d2437b8d3de1166989faec4bf86e2bc6..5435648cd9be15d89eace426eda1942d390b98c4 100644
--- a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
@@ -511,7 +511,7 @@ TEST(ParsedASTTest, ReplayPreambleForTidyCheckers) {
     auto PatchedAST = ParsedAST::build(testPath(TU.Filename), TU.inputs(FS),
                                        std::move(CI), {}, BaselinePreamble);
     ASSERT_TRUE(PatchedAST);
-    EXPECT_TRUE(PatchedAST->getDiagnostics().empty());
+    EXPECT_FALSE(PatchedAST->getDiagnostics());
   }
 
   // Then ensure correctness by making sure includes were seen only once.
@@ -526,7 +526,7 @@ TEST(ParsedASTTest, ReplayPreambleForTidyCheckers) {
   auto PatchedAST = ParsedAST::build(testPath(TU.Filename), TU.inputs(FS),
                                      std::move(CI), {}, BaselinePreamble);
   ASSERT_TRUE(PatchedAST);
-  EXPECT_TRUE(PatchedAST->getDiagnostics().empty());
+  EXPECT_FALSE(PatchedAST->getDiagnostics());
   EXPECT_THAT(Includes,
               ElementsAre(WithFileName(testPath("__preamble_patch__.h")),
                           WithFileName("b.h"), WithFileName("a.h")));
@@ -569,7 +569,7 @@ TEST(ParsedASTTest, PatchesAdditionalIncludes) {
   auto PatchedAST = ParsedAST::build(testPath("foo.cpp"), Inputs, std::move(CI),
                                      {}, EmptyPreamble);
   ASSERT_TRUE(PatchedAST);
-  ASSERT_TRUE(PatchedAST->getDiagnostics().empty());
+  ASSERT_FALSE(PatchedAST->getDiagnostics());
 
   // Ensure source location information is correct, including resolved paths.
   EXPECT_THAT(PatchedAST->getIncludeStructure().MainFileIncludes,
diff --git a/clang-tools-extra/clangd/unittests/PreambleTests.cpp b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
index 4eee9effb82427bb82061b4f5233303e78fb8145..70a14241a8ac43a2d97b00ae705b17943cb2184a 100644
--- a/clang-tools-extra/clangd/unittests/PreambleTests.cpp
+++ b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
@@ -274,8 +274,12 @@ TEST(PreamblePatchTest, Define) {
 
     auto AST = createPatchedAST("", Modified.code());
     ASSERT_TRUE(AST);
-    EXPECT_THAT(AST->getDiagnostics(),
-                Not(Contains(Field(&Diag::Range, Modified.range()))));
+    std::vector<Range> MacroRefRanges;
+    for (auto &M : AST->getMacros().MacroRefs) {
+      for (auto &O : M.getSecond())
+        MacroRefRanges.push_back(O.Rng);
+    }
+    EXPECT_THAT(MacroRefRanges, Contains(Modified.range()));
   }
 }
 
@@ -298,8 +302,6 @@ TEST(PreamblePatchTest, OrderingPreserved) {
 
   auto AST = createPatchedAST(Baseline, Modified.code());
   ASSERT_TRUE(AST);
-  EXPECT_THAT(AST->getDiagnostics(),
-              Not(Contains(Field(&Diag::Range, Modified.range()))));
 }
 
 TEST(PreamblePatchTest, LocateMacroAtWorks) {
@@ -535,6 +537,15 @@ TEST(PreamblePatch, ModifiedBounds) {
               ExpectedBounds.PreambleEndsAtStartOfLine);
   }
 }
+
+TEST(PreamblePatch, DropsDiagnostics) {
+  llvm::StringLiteral Code = "#define FOO\nx;/* error-ok */";
+  // First check that this code generates diagnostics.
+  EXPECT_THAT(*TestTU::withCode(Code).build().getDiagnostics(),
+              testing::Not(testing::IsEmpty()));
+  // Ensure they are dropeed when a patched preamble is used.
+  EXPECT_FALSE(createPatchedAST("", Code)->getDiagnostics());
+}
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/RenameTests.cpp b/clang-tools-extra/clangd/unittests/RenameTests.cpp
index ca0e7ff243061ae3ae736dd24a0cd1aa4d64c299..f917e30cd7fe0be2cd9ef8826e304774e3c00a2a 100644
--- a/clang-tools-extra/clangd/unittests/RenameTests.cpp
+++ b/clang-tools-extra/clangd/unittests/RenameTests.cpp
@@ -844,7 +844,7 @@ TEST(RenameTest, Renameable) {
     const char *Code;
     const char* ErrorMessage; // null if no error
     bool IsHeaderFile;
-    llvm::StringRef NewName = "DummyName";
+    llvm::StringRef NewName = "MockName";
   };
   const bool HeaderFile = true;
   Case Cases[] = {
@@ -1240,6 +1240,21 @@ TEST(RenameTest, PrepareRename) {
               testing::HasSubstr("keyword"));
   EXPECT_THAT(Tracer.takeMetric("rename_name_invalid", "Keywords"),
               ElementsAre(1));
+
+  for (std::string BadIdent : {"foo!bar", "123foo", "😀@"}) {
+    Results = runPrepareRename(Server, FooCCPath, FooCC.point(),
+                               /*NewName=*/BadIdent, {});
+    EXPECT_FALSE(Results);
+    EXPECT_THAT(llvm::toString(Results.takeError()),
+                testing::HasSubstr("identifier"));
+    EXPECT_THAT(Tracer.takeMetric("rename_name_invalid", "BadIdentifier"),
+                ElementsAre(1));
+  }
+  for (std::string GoodIdent : {"fooBar", "__foo$", "😀"}) {
+    Results = runPrepareRename(Server, FooCCPath, FooCC.point(),
+                               /*NewName=*/GoodIdent, {});
+    EXPECT_TRUE(bool(Results));
+  }
 }
 
 TEST(CrossFileRenameTests, DirtyBuffer) {
diff --git a/clang-tools-extra/clangd/unittests/SelectionTests.cpp b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
index e9c689f329ab9ed6a35652d8e3b58b24771abfda..a063c84a6a4ca77335dbb88f447e40ce48e39b09 100644
--- a/clang-tools-extra/clangd/unittests/SelectionTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
@@ -581,7 +581,7 @@ TEST(SelectionTest, PathologicalPreprocessor) {
   auto TU = TestTU::withCode(Test.code());
   TU.AdditionalFiles["Expand.inc"] = "MACRO\n";
   auto AST = TU.build();
-  EXPECT_THAT(AST.getDiagnostics(), ::testing::IsEmpty());
+  EXPECT_THAT(*AST.getDiagnostics(), ::testing::IsEmpty());
   auto T = makeSelectionTree(Case, AST);
 
   EXPECT_EQ("BreakStmt", T.commonAncestor()->kind());
diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
index 22b6ea2296d2b403696a38cea7b2f0fdd174a5af..d68cb3efa3d6d373659225e6447f68947b4c9630 100644
--- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
+++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
@@ -71,7 +71,7 @@ MATCHER_P2(TUState, PreambleActivity, ASTActivity, "") {
   return true;
 }
 
-// Dummy ContextProvider to verify the provider is invoked & contexts are used.
+// Simple ContextProvider to verify the provider is invoked & contexts are used.
 static Key<std::string> BoundPath;
 Context bindPath(PathRef F) {
   return Context::current().derive(BoundPath, F.str());
@@ -121,7 +121,7 @@ protected:
     class CaptureDiags : public ParsingCallbacks {
     public:
       void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override {
-        reportDiagnostics(File, AST.getDiagnostics(), Publish);
+        reportDiagnostics(File, *AST.getDiagnostics(), Publish);
       }
 
       void onFailedAST(PathRef File, llvm::StringRef Version,
diff --git a/clang-tools-extra/clangd/unittests/TestTU.cpp b/clang-tools-extra/clangd/unittests/TestTU.cpp
index 8d336b3f4e19f6803bff14353ebfca7179b86a26..1c6e54774c0361729e3892cf8c15af2e6e3b49be 100644
--- a/clang-tools-extra/clangd/unittests/TestTU.cpp
+++ b/clang-tools-extra/clangd/unittests/TestTU.cpp
@@ -113,6 +113,11 @@ ParsedAST TestTU::build() const {
     ADD_FAILURE() << "Failed to build code:\n" << Code;
     llvm_unreachable("Failed to build TestTU!");
   }
+  if (!AST->getDiagnostics()) {
+    ADD_FAILURE() << "TestTU should always build an AST with a fresh Preamble"
+                  << Code;
+    return std::move(*AST);
+  }
   // Check for error diagnostics and report gtest failures (unless expected).
   // This guards against accidental syntax errors silently subverting tests.
   // error-ok is awfully primitive - using clang -verify would be nicer.
@@ -128,7 +133,8 @@ ParsedAST TestTU::build() const {
     return false;
   }();
   if (!ErrorOk) {
-    for (const auto &D : AST->getDiagnostics())
+    // We always build AST with a fresh preamble in TestTU.
+    for (const auto &D : *AST->getDiagnostics())
       if (D.Severity >= DiagnosticsEngine::Error) {
         ADD_FAILURE()
             << "TestTU failed to build (suppress with /*error-ok*/): \n"
@@ -189,7 +195,7 @@ const NamedDecl &findDecl(ParsedAST &AST, llvm::StringRef QName) {
                            llvm::StringRef Name) -> const NamedDecl & {
     auto LookupRes = Scope.lookup(DeclarationName(&Ctx.Idents.get(Name)));
     assert(!LookupRes.empty() && "Lookup failed");
-    assert(LookupRes.size() == 1 && "Lookup returned multiple results");
+    assert(LookupRes.isSingleResult() && "Lookup returned multiple results");
     return *LookupRes.front();
   };
 
diff --git a/clang-tools-extra/clangd/unittests/TestTU.h b/clang-tools-extra/clangd/unittests/TestTU.h
index 18b490332b1a757ad2b88b8c6c6dd9c0745cb219..169cab045ea1ed7e652f1fa2e1522d78da1cb6dd 100644
--- a/clang-tools-extra/clangd/unittests/TestTU.h
+++ b/clang-tools-extra/clangd/unittests/TestTU.h
@@ -78,6 +78,7 @@ struct TestTU {
 
   // By default, build() will report Error diagnostics as GTest errors.
   // Suppress this behavior by adding an 'error-ok' comment to the code.
+  // The result will always have getDiagnostics() populated.
   ParsedAST build() const;
   std::shared_ptr<const PreambleData>
   preamble(PreambleParsedCallback PreambleCallback = nullptr) const;
diff --git a/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp b/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp
index 08f936ce8b553bbbf13ba1dbe03cdc7c41f57950..09f90fd6e6b58367b287969a84bb821b7e3b6b02 100644
--- a/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp
+++ b/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp
@@ -398,7 +398,7 @@ TEST(TypeHierarchy, RecursiveHierarchyUnbounded) {
 
   // The compiler should produce a diagnostic for hitting the
   // template instantiation depth.
-  ASSERT_TRUE(!AST.getDiagnostics().empty());
+  ASSERT_TRUE(!AST.getDiagnostics()->empty());
 
   // Make sure getTypeHierarchy() doesn't get into an infinite recursion.
   // The parent is reported as "S" because "S<0>" is an invalid instantiation.
diff --git a/clang-tools-extra/clangd/unittests/tweaks/ExtractVariableTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/ExtractVariableTests.cpp
index 4e9223cfe553f8dfdb5d6bb3866d32c354fb6b12..5862d4938d4a284b70d22c885fcdaa53b58a3348 100644
--- a/clang-tools-extra/clangd/unittests/tweaks/ExtractVariableTests.cpp
+++ b/clang-tools-extra/clangd/unittests/tweaks/ExtractVariableTests.cpp
@@ -131,7 +131,7 @@ TEST_F(ExtractVariableTest, Test) {
                    int a = 5 * (4 + (3 [[- 1)]]);
                  })cpp",
        R"cpp(void varDecl() {
-                   auto dummy = (3 - 1); int a = 5 * (4 + dummy);
+                   auto placeholder = (3 - 1); int a = 5 * (4 + placeholder);
                  })cpp"},
       // FIXME: extraction from switch case
       /*{R"cpp(void f(int a) {
@@ -146,11 +146,11 @@ TEST_F(ExtractVariableTest, Test) {
                    }
              })cpp",
        R"cpp(void f(int a) {
-               auto dummy = 1 + 2; if(1)
+               auto placeholder = 1 + 2; if(1)
                  while(a < 1)
                    switch (1) {
                        case 1:
-                         a = dummy;
+                         a = placeholder;
                          break;
                        default:
                          break;
@@ -164,11 +164,11 @@ TEST_F(ExtractVariableTest, Test) {
        /*FIXME: It should be extracted like this.
         R"cpp(#define PLUS(x) x++
               void f(int a) {
-                auto dummy = 1+a; int y = PLUS(dummy);
+                auto placeholder = 1+a; int y = PLUS(placeholder);
               })cpp"},*/
        R"cpp(#define PLUS(x) x++
                  void f(int a) {
-                   auto dummy = PLUS(1+a); int y = dummy;
+                   auto placeholder = PLUS(1+a); int y = placeholder;
                  })cpp"},
       // ensure InsertionPoint isn't inside a macro
       {R"cpp(#define LOOP(x) while (1) {a = x;}
@@ -178,8 +178,8 @@ TEST_F(ExtractVariableTest, Test) {
                  })cpp",
        R"cpp(#define LOOP(x) while (1) {a = x;}
                  void f(int a) {
-                   auto dummy = 3; if(1)
-                    LOOP(5 + dummy)
+                   auto placeholder = 3; if(1)
+                    LOOP(5 + placeholder)
                  })cpp"},
       {R"cpp(#define LOOP(x) do {x;} while(1);
                  void f(int a) {
@@ -188,15 +188,15 @@ TEST_F(ExtractVariableTest, Test) {
                  })cpp",
        R"cpp(#define LOOP(x) do {x;} while(1);
                  void f(int a) {
-                   auto dummy = 3; if(1)
-                    LOOP(5 + dummy)
+                   auto placeholder = 3; if(1)
+                    LOOP(5 + placeholder)
                  })cpp"},
       // attribute testing
       {R"cpp(void f(int a) {
                     [ [gsl::suppress("type")] ] for (;;) a = [[1]] + 1;
                  })cpp",
        R"cpp(void f(int a) {
-                    auto dummy = 1; [ [gsl::suppress("type")] ] for (;;) a = dummy + 1;
+                    auto placeholder = 1; [ [gsl::suppress("type")] ] for (;;) a = placeholder + 1;
                  })cpp"},
       // MemberExpr
       {R"cpp(class T {
@@ -206,7 +206,7 @@ TEST_F(ExtractVariableTest, Test) {
                  };)cpp",
        R"cpp(class T {
                    T f() {
-                     auto dummy = T().f(); return dummy.f();
+                     auto placeholder = T().f(); return placeholder.f();
                    }
                  };)cpp"},
       // Function DeclRefExpr
@@ -214,7 +214,7 @@ TEST_F(ExtractVariableTest, Test) {
                    return [[f]]();
                  })cpp",
        R"cpp(int f() {
-                   auto dummy = f(); return dummy;
+                   auto placeholder = f(); return placeholder;
                  })cpp"},
       // FIXME: Wrong result for \[\[clang::uninitialized\]\] int b = [[1]];
       // since the attr is inside the DeclStmt and the bounds of
@@ -225,33 +225,33 @@ TEST_F(ExtractVariableTest, Test) {
                    int x = 1 + [[2 + 3 + 4]] + 5;
                  })cpp",
        R"cpp(void f() {
-                   auto dummy = 2 + 3 + 4; int x = 1 + dummy + 5;
+                   auto placeholder = 2 + 3 + 4; int x = 1 + placeholder + 5;
                  })cpp"},
       {R"cpp(void f() {
                    int x = [[1 + 2 + 3]] + 4 + 5;
                  })cpp",
        R"cpp(void f() {
-                   auto dummy = 1 + 2 + 3; int x = dummy + 4 + 5;
+                   auto placeholder = 1 + 2 + 3; int x = placeholder + 4 + 5;
                  })cpp"},
       {R"cpp(void f() {
                    int x = 1 + 2 + [[3 + 4 + 5]];
                  })cpp",
        R"cpp(void f() {
-                   auto dummy = 3 + 4 + 5; int x = 1 + 2 + dummy;
+                   auto placeholder = 3 + 4 + 5; int x = 1 + 2 + placeholder;
                  })cpp"},
       // Non-associative operations have no special support
       {R"cpp(void f() {
                    int x = 1 - [[2 - 3 - 4]] - 5;
                  })cpp",
        R"cpp(void f() {
-                   auto dummy = 1 - 2 - 3 - 4; int x = dummy - 5;
+                   auto placeholder = 1 - 2 - 3 - 4; int x = placeholder - 5;
                  })cpp"},
       // A mix of associative operators isn't associative.
       {R"cpp(void f() {
                    int x = 0 + 1 * [[2 + 3]] * 4 + 5;
                  })cpp",
        R"cpp(void f() {
-                   auto dummy = 1 * 2 + 3 * 4; int x = 0 + dummy + 5;
+                   auto placeholder = 1 * 2 + 3 * 4; int x = 0 + placeholder + 5;
                  })cpp"},
       // Overloaded operators are supported, we assume associativity
       // as if they were built-in.
@@ -269,7 +269,7 @@ TEST_F(ExtractVariableTest, Test) {
                  S operator+(S, S);
 
                  void f() {
-                   auto dummy = S(2) + S(3) + S(4); S x = S(1) + dummy + S(5);
+                   auto placeholder = S(2) + S(3) + S(4); S x = S(1) + placeholder + S(5);
                  })cpp"},
       // Don't try to analyze across macro boundaries
       // FIXME: it'd be nice to do this someday (in a safe way)
@@ -279,7 +279,7 @@ TEST_F(ExtractVariableTest, Test) {
                  })cpp",
        R"cpp(#define ECHO(X) X
                  void f() {
-                   auto dummy = 1 + ECHO(2 + 3) + 4; int x = dummy + 5;
+                   auto placeholder = 1 + ECHO(2 + 3) + 4; int x = placeholder + 5;
                  })cpp"},
       {R"cpp(#define ECHO(X) X
                  void f() {
@@ -287,7 +287,7 @@ TEST_F(ExtractVariableTest, Test) {
                  })cpp",
        R"cpp(#define ECHO(X) X
                  void f() {
-                   auto dummy = 1 + ECHO(2) + ECHO(3) + 4; int x = dummy + 5;
+                   auto placeholder = 1 + ECHO(2) + ECHO(3) + 4; int x = placeholder + 5;
                  })cpp"},
   };
   for (const auto &IO : InputOutputs) {
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 91207090902d8b2fa12451c72d00a915d63a7fbf..2e38c7d86dc9c26b4dbe44dbb198c111f2aabd9d 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -83,6 +83,12 @@ New checks
   Finds ``pthread_setcanceltype`` function calls where a thread's cancellation
   type is set to asynchronous.
 
+- New :doc:`altera-unroll-loops
+  <clang-tidy/checks/altera-unroll-loops>` check.
+
+  Finds inner loops that have not been unrolled, as well as fully unrolled
+  loops with unknown loops bounds or a large number of iterations.
+
 - New :doc:`cppcoreguidelines-prefer-member-initializer
   <clang-tidy/checks/cppcoreguidelines-prefer-member-initializer>` check.
 
@@ -112,11 +118,10 @@ Changes in existing checks
   function or assignment to ``nullptr``.
   Added support for pointers to ``std::unique_ptr``.
 
-Deprecated checks
-^^^^^^^^^^^^^^^^^
+Removed checks
+^^^^^^^^^^^^^^
 
-- The :doc:`readability-deleted-default
-  <clang-tidy/checks/readability-deleted-default>` check has been deprecated.
+- The readability-deleted-default check has been removed.
   
   The clang warning `Wdefaulted-function-deleted
   <https://clang.llvm.org/docs/DiagnosticsReference.html#wdefaulted-function-deleted>`_
diff --git a/clang-tools-extra/docs/clang-tidy/checks/altera-unroll-loops.rst b/clang-tools-extra/docs/clang-tidy/checks/altera-unroll-loops.rst
new file mode 100644
index 0000000000000000000000000000000000000000..419a1edf02112e12e9da0f46d00d39a75fc2530a
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/altera-unroll-loops.rst
@@ -0,0 +1,105 @@
+.. title:: clang-tidy - altera-unroll-loops
+
+altera-unroll-loops
+===================
+
+Finds inner loops that have not been unrolled, as well as fully unrolled loops
+with unknown loop bounds or a large number of iterations.
+
+Unrolling inner loops could improve the performance of OpenCL kernels. However,
+if they have unknown loop bounds or a large number of iterations, they cannot
+be fully unrolled, and should be partially unrolled.
+
+Notes:
+
+- This check is unable to determine the number of iterations in a ``while`` or
+  ``do..while`` loop; hence if such a loop is fully unrolled, a note is emitted
+  advising the user to partially unroll instead.
+
+- In ``for`` loops, our check only works with simple arithmetic increments (
+  ``+``, ``-``, ``*``, ``/``). For all other increments, partial unrolling is
+  advised.
+
+- Depending on the exit condition, the calculations for determining if the
+  number of iterations is large may be off by 1. This should not be an issue
+  since the cut-off is generally arbitrary.
+
+Based on the `Altera SDK for OpenCL: Best Practices Guide
+<https://www.altera.com/en_US/pdfs/literature/hb/opencl-sdk/aocl_optimization_guide.pdf>`_.
+
+.. code-block:: c++
+
+   for (int i = 0; i < 10; i++) {  // ok: outer loops should not be unrolled
+      int j = 0;
+      do {  // warning: this inner do..while loop should be unrolled
+         j++;
+      } while (j < 15);
+
+      int k = 0;
+      #pragma unroll
+      while (k < 20) {  // ok: this inner loop is already unrolled
+         k++;
+      }
+   }
+
+   int A[1000];
+   #pragma unroll
+   // warning: this loop is large and should be partially unrolled
+   for (int a : A) {
+      printf("%d", a);
+   }
+
+   #pragma unroll 5
+   // ok: this loop is large, but is partially unrolled
+   for (int a : A) {
+      printf("%d", a);
+   }
+
+   #pragma unroll
+   // warning: this loop is large and should be partially unrolled
+   for (int i = 0; i < 1000; ++i) {
+      printf("%d", i);
+   }
+
+   #pragma unroll 5
+   // ok: this loop is large, but is partially unrolled
+   for (int i = 0; i < 1000; ++i) {
+      printf("%d", i);
+   }
+
+   #pragma unroll
+   // warning: << operator not supported, recommend partial unrolling
+   for (int i = 0; i < 1000; i<<1) {
+      printf("%d", i);
+   }
+
+   std::vector<int> someVector (100, 0);
+   int i = 0;
+   #pragma unroll
+   // note: loop may be large, recommend partial unrolling
+   while (i < someVector.size()) {
+      someVector[i]++;
+   }
+
+   #pragma unroll
+   // note: loop may be large, recommend partial unrolling
+   while (true) {
+      printf("In loop");
+   }
+
+   #pragma unroll 5
+   // ok: loop may be large, but is partially unrolled
+   while (i < someVector.size()) {
+      someVector[i]++;
+   }
+
+Options
+-------
+
+.. option:: MaxLoopIterations
+
+   Defines the maximum number of loop iterations that a fully unrolled loop
+   can have. By default, it is set to `100`.
+
+   In practice, this refers to the integer value of the upper bound
+   within the loop statement's condition expression.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index e53c0e7049632953325e27d1801017236fa50502..bdce63cd26b6e5632a10459d86af099577abb8cb 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -33,6 +33,7 @@ Clang-Tidy Checks
    `altera-kernel-name-restriction <altera-kernel-name-restriction.html>`_,
    `altera-single-work-item-barrier <altera-single-work-item-barrier.html>`_,
    `altera-struct-pack-align <altera-struct-pack-align.html>`_, "Yes"
+   `altera-unroll-loops <altera-unroll-loops.html>`_,
    `android-cloexec-accept <android-cloexec-accept.html>`_, "Yes"
    `android-cloexec-accept4 <android-cloexec-accept4.html>`_,
    `android-cloexec-creat <android-cloexec-creat.html>`_, "Yes"
@@ -280,7 +281,6 @@ Clang-Tidy Checks
    `readability-container-size-empty <readability-container-size-empty.html>`_, "Yes"
    `readability-convert-member-functions-to-static <readability-convert-member-functions-to-static.html>`_,
    `readability-delete-null-pointer <readability-delete-null-pointer.html>`_, "Yes"
-   `readability-deleted-default <readability-deleted-default.html>`_,
    `readability-else-after-return <readability-else-after-return.html>`_, "Yes"
    `readability-function-cognitive-complexity <readability-function-cognitive-complexity.html>`_,
    `readability-function-size <readability-function-size.html>`_,
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-deleted-default.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-deleted-default.rst
deleted file mode 100644
index 5f2083e00061aabaa7fb3837dd2f1c296c5febb0..0000000000000000000000000000000000000000
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-deleted-default.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-.. title:: clang-tidy - readability-deleted-default
-
-readability-deleted-default
-===========================
-
-This check has been deprecated prefer to make use of the `Wdefaulted-function-deleted
-<https://clang.llvm.org/docs/DiagnosticsReference.html#wdefaulted-function-deleted>`_
-flag.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/altera-unroll-loops.cpp b/clang-tools-extra/test/clang-tidy/checkers/altera-unroll-loops.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..06c1cbaf3a28abb886810753a9a94935a9cabdc5
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/altera-unroll-loops.cpp
@@ -0,0 +1,516 @@
+// RUN: %check_clang_tidy %s altera-unroll-loops %t -- -config="{CheckOptions: [{key: "altera-unroll-loops.MaxLoopIterations", value: 50}]}" -header-filter=.*
+// RUN: %check_clang_tidy -check-suffix=MULT %s altera-unroll-loops %t -- -config="{CheckOptions: [{key: "altera-unroll-loops.MaxLoopIterations", value: 5}]}" -header-filter=.* "--" -DMULT
+
+#ifdef MULT
+// For loops with *= and /= increments.
+void for_loop_mult_div_increments(int *A) {
+// *=
+#pragma unroll
+  for (int i = 2; i <= 32; i *= 2)
+    A[i]++; // OK
+
+#pragma unroll
+  for (int i = 2; i <= 64; i *= 2)
+    // CHECK-MESSAGES-MULT: :[[@LINE-1]]:3: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++; // Not OK
+
+// /=
+#pragma unroll
+  for (int i = 32; i >= 2; i /= 2)
+    A[i]++; // OK
+
+#pragma unroll
+  for (int i = 64; i >= 2; i /= 2)
+    // CHECK-MESSAGES-MULT: :[[@LINE-1]]:3: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++; // Not OK
+}
+#else
+// Cannot determine loop bounds for while loops.
+void while_loops(int *A) {
+  // Recommend unrolling loops that aren't already unrolled.
+  int j = 0;
+  while (j < 2000) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+    A[1] += j;
+    j++;
+  }
+
+  do {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+    A[2] += j;
+    j++;
+  } while (j < 2000);
+
+// If a while loop is fully unrolled, add a note recommending partial
+// unrolling.
+#pragma unroll
+  while (j < 2000) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: note: full unrolling requested, but loop bounds may not be known; to partially unroll this loop, use the '#pragma unroll <num>' directive
+    A[j]++;
+  }
+
+#pragma unroll
+  do {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: note: full unrolling requested, but loop bounds may not be known; to partially unroll this loop, use the '#pragma unroll <num>' directive
+    A[j]++;
+  } while (j < 2000);
+
+// While loop is partially unrolled, no action needed.
+#pragma unroll 4
+  while (j < 2000) {
+    A[j]++;
+  }
+
+#pragma unroll 4
+  do {
+    A[j]++;
+  } while (j < 2000);
+}
+
+// Range-based for loops.
+void cxx_for_loops(int *A, int vectorSize) {
+  // Loop with known array size should be unrolled.
+  int a[] = {0, 1, 2, 3, 4, 5};
+  for (int k : a) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+    A[k]++;
+  }
+
+// Loop with known size correctly unrolled.
+#pragma unroll
+  for (int k : a) {
+    A[k]++;
+  }
+
+  // Loop with unknown size should be partially unrolled.
+  int b[vectorSize];
+#pragma unroll
+  for (int k : b) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    k++;
+  }
+
+// Loop with unknown size correctly unrolled.
+#pragma unroll 5
+  for (int k : b) {
+    k++;
+  }
+
+  // Loop with large size should be partially unrolled.
+  int c[51];
+#pragma unroll
+  for (int k : c) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[k]++;
+  }
+
+// Loop with large size correctly unrolled.
+#pragma unroll 5
+  for (int k : c) {
+    A[k]++;
+  }
+}
+
+// Simple for loops.
+void for_loops(int *A, int size) {
+  // Recommend unrolling loops that aren't already unrolled.
+  for (int i = 0; i < 2000; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+    A[0] += i;
+  }
+
+// Loop with known size correctly unrolled.
+#pragma unroll
+  for (int i = 0; i < 50; ++i) {
+    A[i]++;
+  }
+
+// Loop with unknown size should be partially unrolled.
+#pragma unroll
+  for (int i = 0; i < size; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++;
+  }
+
+#pragma unroll
+  for (;;) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[0]++;
+  }
+
+  int i = 0;
+#pragma unroll
+  for (; i < size; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++;
+  }
+
+#pragma unroll
+  for (int i = 0;; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++;
+  }
+
+#pragma unroll
+  for (int i = 0; i < size;) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++;
+  }
+
+#pragma unroll
+  for (int i = size; i < 50; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++;
+  }
+
+#pragma unroll
+  for (int i = 0; true; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++;
+  }
+
+#pragma unroll
+  for (int i = 0; i == i; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++;
+  }
+
+// Loop with unknown size correctly unrolled.
+#pragma unroll 5
+  for (int i = 0; i < size; ++i) {
+    A[i]++;
+  }
+
+// Loop with large size should be partially unrolled.
+#pragma unroll
+  for (int i = 0; i < 51; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++;
+  }
+
+// Loop with large size correctly unrolled.
+#pragma unroll 5
+  for (int i = 0; i < 51; ++i) {
+    A[i]++;
+  }
+}
+
+// For loops with different increments.
+void for_loop_increments(int *A) {
+// ++
+#pragma unroll
+  for (int i = 0; i < 50; ++i)
+    A[i]++; // OK
+
+#pragma unroll
+  for (int i = 0; i < 51; ++i)
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++; // Not OK
+
+// --
+#pragma unroll
+  for (int i = 50; i > 0; --i)
+    A[i]++; // OK
+
+#pragma unroll
+  for (int i = 51; i > 0; --i)
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++; // Not OK
+
+// +=
+#pragma unroll
+  for (int i = 0; i < 100; i += 2)
+    A[i]++; // OK
+
+#pragma unroll
+  for (int i = 0; i < 101; i += 2)
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++; // Not OK
+
+// -=
+#pragma unroll
+  for (int i = 100; i > 0; i -= 2)
+    A[i]++; // OK
+
+#pragma unroll
+  for (int i = 101; i > 0; i -= 2)
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    A[i]++; // Not OK
+}
+
+// Inner loops should be unrolled.
+void nested_simple_loops(int *A) {
+  for (int i = 0; i < 1000; ++i) {
+    for (int j = 0; j < 2000; ++j) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+      A[0] += i + j;
+    }
+  }
+
+  for (int i = 0; i < 1000; ++i) {
+    int j = 0;
+    while (j < 2000) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+      A[1] += i + j;
+      j++;
+    }
+  }
+
+  for (int i = 0; i < 1000; ++i) {
+    int j = 0;
+    do {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+      A[2] += i + j;
+      j++;
+    } while (j < 2000);
+  }
+
+  int i = 0;
+  while (i < 1000) {
+    for (int j = 0; j < 2000; ++j) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+      A[3] += i + j;
+    }
+    i++;
+  }
+
+  i = 0;
+  while (i < 1000) {
+    int j = 0;
+    while (j < 2000) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+      A[4] += i + j;
+      j++;
+    }
+    i++;
+  }
+
+  i = 0;
+  while (i < 1000) {
+    int j = 0;
+    do {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+      A[5] += i + j;
+      j++;
+    } while (j < 2000);
+    i++;
+  }
+
+  i = 0;
+  do {
+    for (int j = 0; j < 2000; ++j) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+      A[6] += i + j;
+    }
+    i++;
+  } while (i < 1000);
+
+  i = 0;
+  do {
+    int j = 0;
+    while (j < 2000) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+      A[7] += i + j;
+      j++;
+    }
+    i++;
+  } while (i < 1000);
+
+  i = 0;
+  do {
+    int j = 0;
+    do {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+      A[8] += i + j;
+      j++;
+    } while (j < 2000);
+    i++;
+  } while (i < 1000);
+
+  for (int i = 0; i < 100; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+    A[i]++;
+  }
+
+  i = 0;
+  while (i < 100) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+    i++;
+  }
+
+  i = 0;
+  do {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: kernel performance could be improved by unrolling this loop with a '#pragma unroll' directive [altera-unroll-loops]
+    i++;
+  } while (i < 100);
+}
+
+// These loops are all correctly unrolled.
+void unrolled_nested_simple_loops(int *A) {
+  for (int i = 0; i < 1000; ++i) {
+#pragma unroll
+    for (int j = 0; j < 50; ++j) {
+      A[0] += i + j;
+    }
+  }
+
+  for (int i = 0; i < 1000; ++i) {
+    int j = 0;
+#pragma unroll 5
+    while (j < 50) {
+      A[1] += i + j;
+      j++;
+    }
+  }
+
+  for (int i = 0; i < 1000; ++i) {
+    int j = 0;
+#pragma unroll 5
+    do {
+      A[2] += i + j;
+      j++;
+    } while (j < 50);
+  }
+
+  int i = 0;
+  while (i < 1000) {
+#pragma unroll
+    for (int j = 0; j < 50; ++j) {
+      A[3] += i + j;
+    }
+    i++;
+  }
+
+  i = 0;
+  while (i < 1000) {
+    int j = 0;
+#pragma unroll 5
+    while (50 > j) {
+      A[4] += i + j;
+      j++;
+    }
+    i++;
+  }
+
+  i = 0;
+  while (1000 > i) {
+    int j = 0;
+#pragma unroll 5
+    do {
+      A[5] += i + j;
+      j++;
+    } while (j < 50);
+    i++;
+  }
+
+  i = 0;
+  do {
+#pragma unroll
+    for (int j = 0; j < 50; ++j) {
+      A[6] += i + j;
+    }
+    i++;
+  } while (i < 1000);
+
+  i = 0;
+  do {
+    int j = 0;
+#pragma unroll 5
+    while (j < 50) {
+      A[7] += i + j;
+      j++;
+    }
+    i++;
+  } while (i < 1000);
+
+  i = 0;
+  do {
+    int j = 0;
+#pragma unroll 5
+    do {
+      A[8] += i + j;
+      j++;
+    } while (j < 50);
+    i++;
+  } while (i < 1000);
+}
+
+// These inner loops are large and should be partially unrolled.
+void unrolled_nested_simple_loops_large_num_iterations(int *A) {
+  for (int i = 0; i < 1000; ++i) {
+#pragma unroll
+    for (int j = 0; j < 51; ++j) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+      A[0] += i + j;
+    }
+  }
+
+  int i = 0;
+  while (i < 1000) {
+#pragma unroll
+    for (int j = 0; j < 51; ++j) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+      A[3] += i + j;
+    }
+    i++;
+  }
+
+  i = 0;
+  do {
+#pragma unroll
+    for (int j = 0; j < 51; ++j) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+      A[6] += i + j;
+    }
+    i++;
+  } while (i < 1000);
+
+  i = 0;
+  do {
+    int j = 0;
+#pragma unroll
+    do {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: note: full unrolling requested, but loop bounds may not be known; to partially unroll this loop, use the '#pragma unroll <num>' directive
+      A[8] += i + j;
+      j++;
+    } while (j < 51);
+    i++;
+  } while (i < 1000);
+
+  i = 0;
+  int a[51];
+  do {
+#pragma unroll
+    for (int k : a) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: loop likely has a large number of iterations and thus cannot be fully unrolled; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+      A[k]++;
+    }
+  } while (i < 1000);
+}
+
+// These loops have unknown bounds and should be partially unrolled.
+void fully_unrolled_unknown_bounds(int vectorSize) {
+  int someVector[101];
+
+// There is no loop condition
+#pragma unroll
+  for (;;) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    someVector[0]++;
+  }
+
+#pragma unroll
+  for (int i = 0; 1 < 5; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    someVector[i]++;
+  }
+
+// Both sides are value-dependent
+#pragma unroll
+  for (int i = 0; i < vectorSize; ++i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: full unrolling requested, but loop bounds are not known; to partially unroll this loop, use the '#pragma unroll <num>' directive [altera-unroll-loops]
+    someVector[i]++;
+  }
+}
+#endif
+// There are no fix-its for this check
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone-terminating-continue.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone-terminating-continue.cpp
index 4bdcbc42fc477c24d4d90b6b6511cb1f18bbe4d8..04fc4a80ea7d489e99e7d4e4265bba6f1d072291 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone-terminating-continue.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone-terminating-continue.cpp
@@ -32,6 +32,15 @@ void f() {
     // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: 'continue' in loop with false condition is equivalent to 'break' [bugprone-terminating-continue]
     // CHECK-FIXES: if (x > 0) break;
   } while (false);
+
+  switch (2) {
+  case 2:
+    do {
+      continue; // LoopInSwitch
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: 'continue' in loop with false condition is equivalent to 'break' [bugprone-terminating-continue]
+      // CHECK-FIXES: break; // LoopInSwitch
+    } while (0);
+  }
 }
 
 void g() {
@@ -62,4 +71,12 @@ void g() {
       if (n>2) continue;
     }
   } while (false);
+
+  do {
+    switch (2) {
+    case 1:
+    case 2:
+      continue;
+    }
+  } while (false);
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-braces-around-statements.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-braces-around-statements.cpp
index 8dc5bf150fa322b837d42f2e4ed7077567e944fa..494c2b780a7cda656c5793f1dfc211afed1d3324 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability-braces-around-statements.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability-braces-around-statements.cpp
@@ -74,30 +74,41 @@ void test() {
     do_something("for");
   // CHECK-MESSAGES: :[[@LINE-2]]:11: warning: statement should be inside braces
   // CHECK-FIXES: for (;;) {
-  // CHECK-FIXES: }
+  // CHECK-FIXES-NEXT: do_something("for");
+  // CHECK-FIXES-NEXT: }
+
   for (;;) {
-    do_something("for");
+    do_something("for-ok");
   }
   for (;;)
     ;
   // CHECK-MESSAGES: :[[@LINE-2]]:11: warning: statement should be inside braces
   // CHECK-FIXES: for (;;) {
-  // CHECK-FIXES: }
+  // CHECK-FIXES-NEXT: ;
+  // CHECK-FIXES-NEXT: }
 
   int arr[4] = {1, 2, 3, 4};
   for (int a : arr)
     do_something("for-range");
   // CHECK-MESSAGES: :[[@LINE-2]]:20: warning: statement should be inside braces
   // CHECK-FIXES: for (int a : arr) {
-  // CHECK-FIXES: }
-  for (int a : arr) {
+  // CHECK-FIXES-NEXT: do_something("for-range");
+  // CHECK-FIXES-NEXT: }
+  for (int &assign : arr)
+    assign = 7;
+  // CHECK-MESSAGES: :[[@LINE-2]]:26: warning: statement should be inside braces
+  // CHECK-FIXES: for (int &assign : arr) {
+  // CHECK-FIXES-NEXT: assign = 7;
+  // CHECK-FIXES-NEXT: }
+  for (int ok : arr) {
     do_something("for-range");
   }
-  for (int a : arr)
+  for (int NullStmt : arr)
     ;
-  // CHECK-MESSAGES: :[[@LINE-2]]:20: warning: statement should be inside braces
-  // CHECK-FIXES: for (int a : arr) {
-  // CHECK-FIXES: }
+  // CHECK-MESSAGES: :[[@LINE-2]]:27: warning: statement should be inside braces
+  // CHECK-FIXES: for (int NullStmt : arr) {
+  // CHECK-FIXES-NEXT: ;
+  // CHECK-FIXES-NEXT: }
 
   while (cond("while1"))
     do_something("while");
@@ -143,14 +154,19 @@ void test() {
   // CHECK-FIXES-NEXT: }
 
   if (cond("ifif3"))
-    // comment
+    // comment1
     if (cond("ifif4")) {
-      // comment
-      /*comment*/; // comment
+      // comment2
+      /*comment3*/; // comment4
     }
   // CHECK-MESSAGES: :[[@LINE-6]]:21: warning: statement should be inside braces
   // CHECK-FIXES: if (cond("ifif3")) {
-  // CHECK-FIXES: }
+  // CHECK-FIXES-NEXT: // comment1
+  // CHECK-FIXES-NEXT: if (cond("ifif4")) {
+  // CHECK-FIXES-NEXT: // comment2
+  // CHECK-FIXES-NEXT: /*comment3*/; // comment4
+  // CHECK-FIXES-NEXT: }
+  // CHECK-FIXES-NEXT: }
 
   if (cond("ifif5"))
     ; /* multi-line
@@ -170,6 +186,161 @@ void test() {
   // CHECK-FIXES-NEXT: }
   // CHECK-FIXES-NEXT: }
   // CHECK-FIXES-NEXT: }
+
+  int S;
+  if (cond("assign with brackets"))
+    S = {5};
+  // CHECK-MESSAGES: :[[@LINE-2]]:36: warning: statement should be inside braces
+  // CHECK-FIXES: if (cond("assign with brackets")) {
+  // CHECK-FIXES-NEXT: S = {5};
+  // CHECK-FIXES-NEXT: }
+
+  if (cond("assign with brackets 2"))
+    S = {  5  } /* comment1 */ ; /* comment2 */
+  // CHECK-MESSAGES: :[[@LINE-2]]:38: warning: statement should be inside braces
+  // CHECK-FIXES: if (cond("assign with brackets 2")) {
+  // CHECK-FIXES-NEXT: S = {  5  } /* comment1 */ ; /* comment2 */
+  // CHECK-FIXES-NEXT: }
+
+  if (cond("return"))
+    return;
+  // CHECK-MESSAGES: :[[@LINE-2]]:22: warning: statement should be inside braces
+  // CHECK-FIXES: if (cond("return")) {
+  // CHECK-FIXES-NEXT: return;
+  // CHECK-FIXES-NEXT: }
+
+  while (cond("break and continue")) {
+    // CHECK-FIXES: while (cond("break and continue")) {
+    if (true)
+      break;
+    // CHECK-MESSAGES: :[[@LINE-2]]:14: warning: statement should be inside braces
+    // CHECK-FIXES: {{^}}    if (true) {{{$}}
+    // CHECK-FIXES-NEXT: {{^}}      break;{{$}}
+    // CHECK-FIXES-NEXT: {{^ *}}}{{$}}
+    if (false)
+      continue;
+    // CHECK-MESSAGES: :[[@LINE-2]]:15: warning: statement should be inside braces
+    // CHECK-FIXES: {{^}}    if (false) {{{$}}
+    // CHECK-FIXES-NEXT: {{^}}      continue;{{$}}
+    // CHECK-FIXES-NEXT: {{^ *}}}{{$}}
+  } //end
+  // CHECK-FIXES: } //end
+
+  if (cond("decl 1"))
+    int s;
+  else
+    int t;
+  // CHECK-MESSAGES: :[[@LINE-4]]:22: warning: statement should be inside braces
+  // CHECK-MESSAGES: :[[@LINE-3]]:7: warning: statement should be inside braces
+  // CHECK-FIXES: if (cond("decl 1")) {
+  // CHECK-FIXES-NEXT: int s;
+  // CHECK-FIXES-NEXT: } else {
+  // CHECK-FIXES-NEXT: int t;
+  // CHECK-FIXES-NEXT: }
+
+  if (cond("decl 2"))
+    int s = (5);
+  else
+    int t = (5);
+  // CHECK-MESSAGES: :[[@LINE-4]]:22: warning: statement should be inside braces
+  // CHECK-MESSAGES: :[[@LINE-3]]:7: warning: statement should be inside braces
+  // CHECK-FIXES: if (cond("decl 2")) {
+  // CHECK-FIXES-NEXT: int s = (5);
+  // CHECK-FIXES-NEXT: } else {
+  // CHECK-FIXES-NEXT: int t = (5);
+  // CHECK-FIXES-NEXT: }
+
+  if (cond("decl 3"))
+    int s = {6};
+  else
+    int t = {6};
+  // CHECK-MESSAGES: :[[@LINE-4]]:22: warning: statement should be inside braces
+  // CHECK-MESSAGES: :[[@LINE-3]]:7: warning: statement should be inside braces
+  // CHECK-FIXES: if (cond("decl 3")) {
+  // CHECK-FIXES-NEXT: int s = {6};
+  // CHECK-FIXES-NEXT: } else {
+  // CHECK-FIXES-NEXT: int t = {6};
+  // CHECK-FIXES-NEXT: }
+}
+
+void test_whitespace() {
+  while(cond("preserve empty lines"))
+    if(cond("using continue within if"))
+      continue;
+
+
+  test();
+
+  // CHECK-MESSAGES: :[[@LINE-7]]:{{[0-9]+}}: warning: statement should be inside braces
+  // CHECK-MESSAGES: :[[@LINE-7]]:{{[0-9]+}}: warning: statement should be inside braces
+  // CHECK-FIXES: {{^}}  while(cond("preserve empty lines")) {{{$}}
+  // CHECK-FIXES-NEXT: {{^}}    if(cond("using continue within if")) {{{$}}
+  // CHECK-FIXES-NEXT: {{^      continue;$}}
+  // The closing brace is added at beginning of line, clang-format can be
+  // applied afterwards.
+  // CHECK-FIXES-NEXT: {{^}$}}
+  // CHECK-FIXES-NEXT: {{^}$}}
+  // Following whitespace is assumed to not to belong to the else branch.
+  // However the check is not possible with CHECK-FIXES-NEXT.
+  // CHECK-FIXES: {{^}}  test();{{$}}
+
+  if (cond("preserve empty lines"))
+ 
+  
+    int s;
+   
+    
+  else
+ 
+  
+    int t;
+   
+    
+  test();
+
+  // CHECK-MESSAGES: :[[@LINE-14]]:{{[0-9]+}}: warning: statement should be inside braces
+  // CHECK-MESSAGES: :[[@LINE-9]]:{{[0-9]+}}: warning: statement should be inside braces
+  // CHECK-FIXES: {{^}}  if (cond("preserve empty lines")) {{{$}}
+  // CHECK-FIXES-NEXT: {{^ $}}
+  // CHECK-FIXES-NEXT: {{^  $}}
+  // CHECK-FIXES-NEXT: {{^    int s;$}}
+  // CHECK-FIXES-NEXT: {{^   $}}
+  // CHECK-FIXES-NEXT: {{^    $}}
+  // CHECK-FIXES-NEXT: {{^  } else {$}}
+  // CHECK-FIXES-NEXT: {{^ $}}
+  // CHECK-FIXES-NEXT: {{^  $}}
+  // CHECK-FIXES-NEXT: {{^    int t;$}}
+  // The closing brace is added at beginning of line, clang-format can be
+  // applied afterwards.
+  // CHECK-FIXES-NEXT: {{^}$}}
+  // Following whitespace is assumed to not to belong to the else branch.
+  // CHECK-FIXES-NEXT: {{^   $}}
+  // CHECK-FIXES-NEXT: {{^    $}}
+  // CHECK-FIXES-NEXT: {{^}}  test();{{$}}
+}
+
+int test_return_int() {
+  if (cond("return5"))
+    return 5;
+  // CHECK-MESSAGES: :[[@LINE-2]]:23: warning: statement should be inside braces
+  // CHECK-FIXES: if (cond("return5")) {
+  // CHECK-FIXES-NEXT: return 5;
+  // CHECK-FIXES-NEXT: }
+
+  if (cond("return{6}"))
+    return {6};
+  // CHECK-MESSAGES: :[[@LINE-2]]:25: warning: statement should be inside braces
+  // CHECK-FIXES: if (cond("return{6}")) {
+  // CHECK-FIXES-NEXT: return {6};
+  // CHECK-FIXES-NEXT: }
+
+  // From https://bugs.llvm.org/show_bug.cgi?id=25970
+  if (cond("25970")) return {25970};
+  return {!25970};
+  // CHECK-MESSAGES: :[[@LINE-2]]:21: warning: statement should be inside braces
+  // CHECK-FIXES: if (cond("25970")) { return {25970};
+  // CHECK-FIXES-NEXT: }
+  // CHECK-FIXES-NEXT: return {!25970};
 }
 
 void f(const char *p) {
@@ -203,4 +374,86 @@ int test_macros(bool b) {
   // CHECK-FIXES: {{^}}    for (;;) {{{$}}
   // CHECK-FIXES-NEXT: {{^      ;$}}
   // CHECK-FIXES-NEXT: {{^}$}}
+
+
+  #define WRAP(X) { X; }
+  // This is to ensure no other CHECK-FIXES matches the macro definition:
+  // CHECK-FIXES: WRAP
+
+  // Use-case: LLVM_DEBUG({ for(...) do_something(); });
+  WRAP({
+    for (;;)
+      do_something("for in wrapping macro 1");
+    });
+  // CHECK-MESSAGES: :[[@LINE-3]]:13: warning: statement should be inside braces
+  // CHECK-FIXES: for (;;) {
+  // CHECK-FIXES-NEXT: do_something("for in wrapping macro 1");
+  // CHECK-FIXES-NEXT: }
+
+  // Use-case: LLVM_DEBUG( for(...) do_something(); );
+  WRAP(
+    for (;;)
+      do_something("for in wrapping macro 2");
+    );
+  // CHECK-MESSAGES: :[[@LINE-3]]:13: warning: statement should be inside braces
+  // CHECK-FIXES: for (;;) {
+  // CHECK-FIXES-NEXT: do_something("for in wrapping macro 2");
+  // CHECK-FIXES-NEXT: }
+
+  // Use-case: LLVM_DEBUG( for(...) do_something() );
+  // This is not supported and this test ensure it's correctly not changed.
+  // We don't want to add the `}` into the Macro and there is no other way
+  // to add it except for introduction of a NullStmt.
+  WRAP(
+    for (;;)
+        do_something("for in wrapping macro 3")
+    );
+  // CHECK-MESSAGES: :[[@LINE-3]]:13: warning: statement should be inside braces
+  // CHECK-FIXES: WRAP(
+  // CHECK-FIXES-NEXT: for (;;)
+  // CHECK-FIXES-NEXT: do_something("for in wrapping macro 3")
+  // CHECK-FIXES-NEXT: );
+
+  // Taken from https://bugs.llvm.org/show_bug.cgi?id=22785
+  int i;
+  #define MACRO_1 i++
+  #define MACRO_2
+  if( i % 3) i--;
+  else if( i % 2) MACRO_1;
+  else MACRO_2;
+  // CHECK-MESSAGES: :[[@LINE-3]]:13: warning: statement should be inside braces
+  // CHECK-MESSAGES: :[[@LINE-3]]:18: warning: statement should be inside braces
+  // CHECK-MESSAGES: :[[@LINE-3]]:7: warning: statement should be inside braces
+  // CHECK-FIXES: if( i % 3) { i--;
+  // CHECK-FIXES-NEXT: } else if( i % 2) { MACRO_1;
+  // CHECK-FIXES-NEXT: } else { MACRO_2;
+  // CHECK-FIXES-NEXT: }
+
+  // Taken from https://bugs.llvm.org/show_bug.cgi?id=22785
+  #define M(x) x
+
+  if (b)
+    return 1;
+  else
+    return 2;
+  // CHECK-MESSAGES: :[[@LINE-4]]:9: warning: statement should be inside braces
+  // CHECK-MESSAGES: :[[@LINE-3]]:7: warning: statement should be inside braces
+  // CHECK-FIXES: if (b) {
+  // CHECK-FIXES-NEXT: return 1;
+  // CHECK-FIXES-NEXT: } else {
+  // CHECK-FIXES-NEXT: return 2;
+  // CHECK-FIXES-NEXT: }
+
+  if (b)
+    return 1;
+  else
+    M(return 2);
+  // CHECK-MESSAGES: :[[@LINE-4]]:9: warning: statement should be inside braces
+  // CHECK-MESSAGES: :[[@LINE-3]]:7: warning: statement should be inside braces
+  // CHECK-FIXES: if (b) {
+  // CHECK-FIXES-NEXT: return 1;
+  // CHECK-FIXES-NEXT: } else {
+  // CHECK-FIXES-NEXT: M(return 2);
+  // CHECK-FIXES-NEXT: }
+
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-deleted-default.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-deleted-default.cpp
deleted file mode 100644
index 232f224128a69e7221201bcf9798f4f72dc8489c..0000000000000000000000000000000000000000
--- a/clang-tools-extra/test/clang-tidy/checkers/readability-deleted-default.cpp
+++ /dev/null
@@ -1,127 +0,0 @@
-// RUN: %check_clang_tidy %s readability-deleted-default %t -- -- -fno-ms-compatibility
-
-class NoDefault {
-public:
-  NoDefault() = delete;
-  NoDefault(NoDefault &&Other) = delete;
-  NoDefault(const NoDefault &Other) = delete;
-};
-
-class MissingEverything {
-public:
-  MissingEverything() = default;
-  // CHECK-MESSAGES: warning: default constructor is explicitly defaulted but implicitly deleted, probably because a non-static data member or a base class is lacking a default constructor; definition can either be removed or explicitly deleted [readability-deleted-default]
-  MissingEverything(MissingEverything &&Other) = default;
-  // CHECK-MESSAGES: warning: move constructor is explicitly defaulted but implicitly deleted, probably because a non-static data member or a base class is neither copyable nor movable; definition can either be removed or explicitly deleted [readability-deleted-default]
-  MissingEverything(const MissingEverything &Other) = default;
-  // CHECK-MESSAGES: warning: copy constructor is explicitly defaulted but implicitly deleted, probably because a non-static data member or a base class is not copyable; definition can either be removed or explicitly deleted [readability-deleted-default]
-  MissingEverything &operator=(MissingEverything &&Other) = default;
-  // CHECK-MESSAGES: warning: move assignment operator is explicitly defaulted but implicitly deleted, probably because a base class or a non-static data member is not assignable, e.g. because the latter is marked 'const'; definition can either be removed or explicitly deleted [readability-deleted-default]
-  MissingEverything &operator=(const MissingEverything &Other) = default;
-  // CHECK-MESSAGES: warning: copy assignment operator is explicitly defaulted but implicitly deleted, probably because a base class or a non-static data member is not assignable, e.g. because the latter is marked 'const'; definition can either be removed or explicitly deleted [readability-deleted-default]
-
-private:
-  NoDefault ND;
-};
-
-class NotAssignable {
-public:
-  NotAssignable(NotAssignable &&Other) = default;
-  NotAssignable(const NotAssignable &Other) = default;
-  NotAssignable &operator=(NotAssignable &&Other) = default;
-  // CHECK-MESSAGES: warning: move assignment operator is explicitly defaulted but implicitly deleted
-  NotAssignable &operator=(const NotAssignable &Other) = default;
-  // CHECK-MESSAGES: warning: copy assignment operator is explicitly defaulted but implicitly deleted
-
-private:
-  const int I = 0;
-};
-
-class Movable {
-public:
-  Movable() = default;
-  Movable(Movable &&Other) = default;
-  Movable(const Movable &Other) = delete;
-  Movable &operator=(Movable &&Other) = default;
-  Movable &operator=(const Movable &Other) = delete;
-};
-
-class NotCopyable {
-public:
-  NotCopyable(NotCopyable &&Other) = default;
-  NotCopyable(const NotCopyable &Other) = default;
-  // CHECK-MESSAGES: warning: copy constructor is explicitly defaulted but implicitly deleted
-  NotCopyable &operator=(NotCopyable &&Other) = default;
-  NotCopyable &operator=(const NotCopyable &Other) = default;
-  // CHECK-MESSAGES: warning: copy assignment operator is explicitly defaulted but implicitly deleted
-private:
-  Movable M;
-};
-
-template <typename T> class Templated {
-public:
-  // No warning here, it is a templated class.
-  Templated() = default;
-  Templated(Templated &&Other) = default;
-  Templated(const Templated &Other) = default;
-  Templated &operator=(Templated &&Other) = default;
-  Templated &operator=(const Templated &Other) = default;
-
-  class InnerTemplated {
-  public:
-    // This class is not in itself templated, but we still don't have warning.
-    InnerTemplated() = default;
-    InnerTemplated(InnerTemplated &&Other) = default;
-    InnerTemplated(const InnerTemplated &Other) = default;
-    InnerTemplated &operator=(InnerTemplated &&Other) = default;
-    InnerTemplated &operator=(const InnerTemplated &Other) = default;
-
-  private:
-    T TVar;
-  };
-
-  class InnerNotTemplated {
-  public:
-    // This one could technically have warnings, but currently doesn't.
-    InnerNotTemplated() = default;
-    InnerNotTemplated(InnerNotTemplated &&Other) = default;
-    InnerNotTemplated(const InnerNotTemplated &Other) = default;
-    InnerNotTemplated &operator=(InnerNotTemplated &&Other) = default;
-    InnerNotTemplated &operator=(const InnerNotTemplated &Other) = default;
-
-  private:
-    int I;
-  };
-
-private:
-  const T TVar{};
-};
-
-int FunctionWithInnerClass() {
-  class InnerNotAssignable {
-  public:
-    InnerNotAssignable &operator=(InnerNotAssignable &&Other) = default;
-    // CHECK-MESSAGES: warning: move assignment operator is explicitly defaulted but implicitly deleted
-  private:
-    const int I = 0;
-  };
-  return 1;
-};
-
-template <typename T>
-int TemplateFunctionWithInnerClass() {
-  class InnerNotAssignable {
-  public:
-    InnerNotAssignable &operator=(InnerNotAssignable &&Other) = default;
-  private:
-    const T TVar{};
-  };
-  return 1;
-};
-
-void Foo() {
-  Templated<const int> V1;
-  Templated<int>::InnerTemplated V2;
-  Templated<float>::InnerNotTemplated V3;
-  TemplateFunctionWithInnerClass<int>();
-}
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-file/config-file-spaces b/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-file/config-file-spaces
new file mode 100644
index 0000000000000000000000000000000000000000..4aa1f846ade65e859b4bb98801bcb21a1e60fd07
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-file/config-file-spaces
@@ -0,0 +1,9 @@
+Checks: "
+  -*
+  ,
+  hicpp-uppercase-literal-suffix
+  ,hicpp-use-auto
+
+
+  ,  hicpp-use-emplace
+"
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/config-file.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/config-file.cpp
index 49028d198f75adb0a550423c7981efcda13bfaba..16b216a1d4f6918816adf7893a9a65db4b4e0b27 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/config-file.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/config-file.cpp
@@ -1,2 +1,8 @@
 // RUN: clang-tidy -config-file=%S/Inputs/config-file/config-file -dump-config -- | FileCheck %s -check-prefix=CHECK-BASE
 // CHECK-BASE: Checks: {{.*}}hicpp-uppercase-literal-suffix
+// RUN: clang-tidy -config-file=%S/Inputs/config-file/config-file-spaces --list-checks -- | FileCheck %s -check-prefix=CHECK-SPACES
+// CHECK-SPACES: Enabled checks:
+// CHECK-SPACES-NEXT: hicpp-uppercase-literal-suffix
+// CHECK-SPACES-NEXT: hicpp-use-auto
+// CHECK-SPACES-NEXT: hicpp-use-emplace
+// CHECK-SPACES-EMPTY:
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 7af05c331e9467b39092d32e1ef0983959c533d8..95cdbd8f6663a082f292632dec6712969d2deb42 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -467,7 +467,8 @@ option(CLANG_ENABLE_STATIC_ANALYZER
 
 option(CLANG_ENABLE_PROTO_FUZZER "Build Clang protobuf fuzzer." OFF)
 
-option(CLANG_ROUND_TRIP_CC1_ARGS "Round-trip command line arguments in -cc1." OFF)
+option(CLANG_ROUND_TRIP_CC1_ARGS
+  "Round-trip command line arguments in -cc1." ${LLVM_ENABLE_ASSERTIONS})
 
 if(NOT CLANG_ENABLE_STATIC_ANALYZER AND CLANG_ENABLE_ARCMT)
   message(FATAL_ERROR "Cannot disable static analyzer while enabling ARCMT or Z3")
@@ -640,6 +641,11 @@ if (CLANG_ENABLE_BOOTSTRAP)
   set(BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${NEXT_CLANG_STAGE}-bins/)
 
   if(BOOTSTRAP_LLVM_ENABLE_LLD)
+    # adding lld to clang-bootstrap-deps without having it enabled in
+    # LLVM_ENABLE_PROJECTS just generates a cryptic error message.
+    if (NOT "lld" IN_LIST LLVM_ENABLE_PROJECTS)
+      message(FATAL_ERROR "LLD is enabled in the boostrap build, but lld is not in LLVM_ENABLE_PROJECTS")
+    endif()
     add_dependencies(clang-bootstrap-deps lld)
   endif()
 
@@ -859,13 +865,24 @@ if (CLANG_ENABLE_BOOTSTRAP)
     set(CLANG_BOOTSTRAP_TARGETS check-llvm check-clang check-all)
   endif()
   foreach(target ${CLANG_BOOTSTRAP_TARGETS})
-    # exclude from main target
-    set_target_properties(${NEXT_CLANG_STAGE} PROPERTIES _EP_${target}_EXCLUDE_FROM_MAIN On)
+    # Install targets have side effects, so we always want to execute them.
+    # "install" is reserved by CMake and can't be used as a step name for
+    # ExternalProject_Add_Step, so we can match against "^install-" instead of
+    # "^install" to get a tighter match. CMake's installation scripts already
+    # skip up-to-date files, so there's no behavior change if you install to the
+    # same destination multiple times.
+    if(target MATCHES "^install-")
+      set(step_always ON)
+    else()
+      set(step_always OFF)
+    endif()
 
     ExternalProject_Add_Step(${NEXT_CLANG_STAGE} ${target}
       COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target ${target}
       COMMENT "Performing ${target} for '${NEXT_CLANG_STAGE}'"
       DEPENDEES configure
+      ALWAYS ${step_always}
+      EXCLUDE_FROM_MAIN ON
       USES_TERMINAL 1
     )
 
diff --git a/clang/cmake/caches/3-stage-base.cmake b/clang/cmake/caches/3-stage-base.cmake
index 88ab5d77f16fd4f77f22569e25fdd04145c568e6..31391aa4defceb7f308ef54abdaf5059ef3eaad5 100644
--- a/clang/cmake/caches/3-stage-base.cmake
+++ b/clang/cmake/caches/3-stage-base.cmake
@@ -1,14 +1,34 @@
 set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "")
 set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
 set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "")
-set(BOOTSTRAP_LLVM_ENABLE_LTO ON CACHE BOOL "")
 
-# Use LLD do have less requirements on system linker, unless we're on an apple
-# platform where the system compiler is to be prefered.
 if(APPLE)
+  # Use LLD to have fewer requirements on system linker, unless we're on an apple
+  # platform where the system compiler is to be preferred.
+  set(BOOTSTRAP_LLVM_ENABLE_LLD OFF CACHE BOOL "")
+  set(BOOTSTRAP_LLVM_ENABLE_LTO ON CACHE BOOL "")
+elseif(CMAKE_HOST_UNIX)
+  # s390/SystemZ is unsupported by LLD, so don't try to enable LTO if it
+  # cannot work.
+  # We do our own uname business here since the appropriate variables from CMake
+  # and llvm are not yet available.
+  find_program(CMAKE_UNAME uname /bin /usr/bin /usr/local/bin )
+  if(CMAKE_UNAME)
+    exec_program(${CMAKE_UNAME} ARGS -m OUTPUT_VARIABLE CMAKE_HOST_SYSTEM_PROCESSOR
+        RETURN_VALUE val)
+  endif(CMAKE_UNAME)
+
+  if("${CMAKE_HOST_SYSTEM_PROCESSOR}" MATCHES "s390")
+    set(BOOTSTRAP_LLVM_ENABLE_LTO OFF CACHE BOOL "")
     set(BOOTSTRAP_LLVM_ENABLE_LLD OFF CACHE BOOL "")
-else()
+  else()
+    set(BOOTSTRAP_LLVM_ENABLE_LTO ON CACHE BOOL "")
     set(BOOTSTRAP_LLVM_ENABLE_LLD ON CACHE BOOL "")
+  endif()
+
+else()
+  set(BOOTSTRAP_LLVM_ENABLE_LTO ON CACHE BOOL "")
+  set(BOOTSTRAP_LLVM_ENABLE_LLD ON CACHE BOOL "")
 endif()
 
 
diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 667e33c700d5f24970e3b65dfa986a3634530e60..27391bdafb3e21e962afed8b57818d99c00e3ead 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -263,6 +263,7 @@ set(LLVM_TOOLCHAIN_TOOLS
   llvm-elfabi
   llvm-gsymutil
   llvm-lib
+  llvm-lipo
   llvm-mt
   llvm-nm
   llvm-objcopy
diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst
index 0038dccd53f991a416854d05ec73c7e821a24ed9..d895587c458a4219ac0aa572e78a0e6a27a1d8bd 100644
--- a/clang/docs/ClangCommandLineReference.rst
+++ b/clang/docs/ClangCommandLineReference.rst
@@ -18,9 +18,9 @@ GCC-compatible ``clang`` and ``clang++`` drivers.
 
 
 .. program:: clang
-.. option:: -B<dir>, --prefix <arg>, --prefix=<arg>
+.. option:: -B<prefix>, --prefix <arg>, --prefix=<arg>
 
-Add <dir> to search path for binaries and object files used implicitly
+Search $prefix/$triple-$file and $prefix$file for executables, libraries, includes, and data files used by the compiler. $prefix may or may not be a directory
 
 .. option:: -F<arg>
 
@@ -256,7 +256,7 @@ Build this module as a system module. Only used with -emit-module
 
 .. option:: --gcc-toolchain=<arg>, -gcc-toolchain <arg>
 
-Use the gcc toolchain at the given directory
+Search for GCC installation in the specified directory on targets which commonly use GCC. The directory usually contains 'lib{,32,64}/gcc{,-cross}/$triple' and 'include'. If specified, sysroot is skipped for GCC detection. Note: executables (e.g. ld) used by the compiler are not overridden by the selected GCC installation
 
 .. option:: -gcodeview
 
@@ -2797,10 +2797,6 @@ Use packed stack layout (SystemZ only).
 
 Specify maximum number of prefixes to use for padding
 
-.. option:: -mpie-copy-relocations, -mno-pie-copy-relocations
-
-Use copy relocations support for PIE builds
-
 .. option:: -mprefer-vector-width=<arg>
 
 Specifies preferred vector width for auto-vectorization. Defaults to 'none' which allows target specific decisions.
@@ -3798,4 +3794,3 @@ undef all system defines
 .. option:: -z <arg>
 
 Pass -z <arg> to the linker
-
diff --git a/clang/docs/DiagnosticsReference.rst b/clang/docs/DiagnosticsReference.rst
index 04d7f74d5bfc9400c4f7bc5612118e09725eaa69..730077f33397aafcd2c7743263b41e0510b65a0c 100644
--- a/clang/docs/DiagnosticsReference.rst
+++ b/clang/docs/DiagnosticsReference.rst
@@ -851,6 +851,13 @@ This diagnostic is enabled by default.
 |:warning:`warning:` |nbsp| :diagtext:`cast from function call of type` |nbsp| :placeholder:`A` |nbsp| :diagtext:`to non-matching type` |nbsp| :placeholder:`B`|
 +--------------------------------------------------------------------------------------------------------------------------------------------------------------+
 
+-Wcast-function-type
+-------------------
+**Diagnostic text:**
+
++--------------------------------------------------------------------------------------------------------------------------------------------------------------+
+|:warning:`warning:` |nbsp| :diagtext:`cast from` |nbsp| :placeholder:`A` |nbsp| :diagtext:`to` |nbsp| :placeholder:`B` |nbsp| :diagtext:`converts to incompatible function types`|
++--------------------------------------------------------------------------------------------------------------------------------------------------------------+
 
 -Wbinary-literal
 ----------------
diff --git a/clang/docs/HardwareAssistedAddressSanitizerDesign.rst b/clang/docs/HardwareAssistedAddressSanitizerDesign.rst
index b97fbb91a43a24c8eda6e73318d0170516e7c198..f89ca117427ad7efc8651cb0122828864d65042a 100644
--- a/clang/docs/HardwareAssistedAddressSanitizerDesign.rst
+++ b/clang/docs/HardwareAssistedAddressSanitizerDesign.rst
@@ -19,13 +19,17 @@ The redzones, the quarantine, and, to a less extent, the shadow, are the
 sources of AddressSanitizer's memory overhead.
 See the `AddressSanitizer paper`_ for details.
 
-AArch64 has the `Address Tagging`_ (or top-byte-ignore, TBI), a hardware feature that allows
-software to use 8 most significant bits of a 64-bit pointer as
+AArch64 has `Address Tagging`_ (or top-byte-ignore, TBI), a hardware feature that allows
+software to use the 8 most significant bits of a 64-bit pointer as
 a tag. HWASAN uses `Address Tagging`_
 to implement a memory safety tool, similar to :doc:`AddressSanitizer`,
 but with smaller memory overhead and slightly different (mostly better)
 accuracy guarantees.
 
+Intel's `Linear Address Masking`_ (LAM) also provides address tagging for
+x86_64, though it is not widely available in hardware yet.  For x86_64, HWASAN
+has a limited implementation using page aliasing instead.
+
 Algorithm
 =========
 * Every heap/stack/global memory object is forcibly aligned by `TG` bytes
@@ -266,7 +270,15 @@ before every load and store by compiler instrumentation, but this variant
 will have limited deployability since not all of the code is
 typically instrumented.
 
-The HWASAN's approach is not applicable to 32-bit architectures.
+On x86_64, HWASAN utilizes page aliasing to place tags in userspace address
+bits.  Currently only heap tagging is supported.  The page aliases rely on
+shared memory, which will cause heap memory to be shared between processes if
+the application calls ``fork()``.  Therefore x86_64 is really only safe for
+applications that do not fork.
+
+HWASAN does not currently support 32-bit architectures since they do not
+support `Address Tagging`_ and the address space is too constrained to easily
+implement page aliasing.
 
 
 Related Work
@@ -284,4 +296,4 @@ Related Work
 .. _SPARC ADI: https://lazytyped.blogspot.com/2017/09/getting-started-with-adi.html
 .. _AddressSanitizer paper: https://www.usenix.org/system/files/conference/atc12/atc12-final39.pdf
 .. _Address Tagging: http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/ch12s05s01.html
-
+.. _Linear Address Masking: https://software.intel.com/content/www/us/en/develop/download/intel-architecture-instruction-set-extensions-programming-reference.html
diff --git a/clang/docs/InternalsManual.rst b/clang/docs/InternalsManual.rst
index 32c8f2dad5aa69af5ed807bfa9130a7ff17068db..a547154721e9e936c0592577baf320be03450fbd 100644
--- a/clang/docs/InternalsManual.rst
+++ b/clang/docs/InternalsManual.rst
@@ -2795,12 +2795,14 @@ implementing a keyword attribute, the parsing of the keyword and creation of the
 ``ParsedAttr`` object must be done manually.
 
 Eventually, ``Sema::ProcessDeclAttributeList()`` is called with a ``Decl`` and
-an ``ParsedAttr``, at which point the parsed attribute can be transformed
+a ``ParsedAttr``, at which point the parsed attribute can be transformed
 into a semantic attribute. The process by which a parsed attribute is converted
 into a semantic attribute depends on the attribute definition and semantic
 requirements of the attribute. The end result, however, is that the semantic
 attribute object is attached to the ``Decl`` object, and can be obtained by a
-call to ``Decl::getAttr<T>()``.
+call to ``Decl::getAttr<T>()``. Similarly, for statement attributes,
+``Sema::ProcessStmtAttributes()`` is called with a ``Stmt`` a list of
+``ParsedAttr`` objects to be converted into a semantic attribute.
 
 The structure of the semantic attribute is also governed by the attribute
 definition given in Attr.td. This definition is used to automatically generate
@@ -2820,12 +2822,13 @@ semantic) type, or one of its derivatives. Most attributes will derive from the
 later redeclarations of the ``Decl`` it is associated with.
 ``InheritableParamAttr`` is similar to ``InheritableAttr``, except that the
 attribute is written on a parameter instead of a declaration. If the attribute
-is intended to apply to a type instead of a declaration, such an attribute
-should derive from ``TypeAttr``, and will generally not be given an AST
-representation. (Note that this document does not cover the creation of type
-attributes.) An attribute that inherits from ``IgnoredAttr`` is parsed, but will
-generate an ignored attribute diagnostic when used, which may be useful when an
-attribute is supported by another vendor but not supported by clang.
+applies to statements, it should inherit from ``StmtAttr`. If the attribute is
+intended to apply to a type instead of a declaration, such an attribute should
+derive from ``TypeAttr``, and will generally not be given an AST representation.
+(Note that this document does not cover the creation of type attributes.) An
+attribute that inherits from ``IgnoredAttr`` is parsed, but will generate an
+ignored attribute diagnostic when used, which may be useful when an attribute is
+supported by another vendor but not supported by clang.
 
 The definition will specify several key pieces of information, such as the
 semantic name of the attribute, the spellings the attribute supports, the
@@ -2854,10 +2857,11 @@ are created implicitly. The following spellings are accepted:
   ``Declspec``  Spelled with a Microsoft-style ``__declspec(attr)`` syntax.
   ``Keyword``   The attribute is spelled as a keyword, and required custom
                 parsing.
-  ``GCC``       Specifies two spellings: the first is a GNU-style spelling, and
-                the second is a C++-style spelling with the ``gnu`` namespace.
-                Attributes should only specify this spelling for attributes
-                supported by GCC.
+  ``GCC``       Specifies two or three spellings: the first is a GNU-style
+                spelling, the second is a C++-style spelling with the ``gnu``
+                namespace, and the third is an optional C-style spelling with
+                the ``gnu`` namespace. Attributes should only specify this
+                spelling for attributes supported by GCC.
   ``Clang``     Specifies two or three spellings: the first is a GNU-style
                 spelling, the second is a C++-style spelling with the ``clang``
                 namespace, and the third is an optional C-style spelling with
@@ -2871,19 +2875,16 @@ are created implicitly. The following spellings are accepted:
 
 Subjects
 ~~~~~~~~
-Attributes appertain to one or more ``Decl`` subjects. If the attribute attempts
-to attach to a subject that is not in the subject list, a diagnostic is issued
+Attributes appertain to one or more subjects. If the attribute attempts to 
+attach to a subject that is not in the subject list, a diagnostic is issued
 automatically. Whether the diagnostic is a warning or an error depends on how
 the attribute's ``SubjectList`` is defined, but the default behavior is to warn.
 The diagnostics displayed to the user are automatically determined based on the
 subjects in the list, but a custom diagnostic parameter can also be specified in
 the ``SubjectList``. The diagnostics generated for subject list violations are
-either ``diag::warn_attribute_wrong_decl_type`` or
-``diag::err_attribute_wrong_decl_type``, and the parameter enumeration is found
-in `include/clang/Sema/ParsedAttr.h
-<https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Sema/ParsedAttr.h>`_
-If a previously unused Decl node is added to the ``SubjectList``, the logic used
-to automatically determine the diagnostic parameter in `utils/TableGen/ClangAttrEmitter.cpp
+calculated automatically or specified by the subject list itself. If a
+previously unused Decl node is added to the ``SubjectList``, the logic used to
+automatically determine the diagnostic parameter in `utils/TableGen/ClangAttrEmitter.cpp
 <https://github.com/llvm/llvm-project/blob/main/clang/utils/TableGen/ClangAttrEmitter.cpp>`_
 may need to be updated.
 
@@ -2897,8 +2898,8 @@ instance, a ``NonBitField`` SubsetSubject appertains to a ``FieldDecl``, and
 tests whether the given FieldDecl is a bit field. When a SubsetSubject is
 specified in a SubjectList, a custom diagnostic parameter must also be provided.
 
-Diagnostic checking for attribute subject lists is automated except when
-``HasCustomParsing`` is set to ``1``.
+Diagnostic checking for attribute subject lists for declaration and statement
+attributes is automated except when ``HasCustomParsing`` is set to ``1``.
 
 Documentation
 ~~~~~~~~~~~~~
@@ -3045,8 +3046,8 @@ the switch statement. Please do not implement handling logic directly in the
 
 Unless otherwise specified by the attribute definition, common semantic checking
 of the parsed attribute is handled automatically. This includes diagnosing
-parsed attributes that do not appertain to the given ``Decl``, ensuring the
-correct minimum number of arguments are passed, etc.
+parsed attributes that do not appertain to the given ``Decl`` or ``Stmt``,
+ensuring the correct minimum number of arguments are passed, etc.
 
 If the attribute adds additional warnings, define a ``DiagGroup`` in
 `include/clang/Basic/DiagnosticGroups.td
@@ -3072,6 +3073,10 @@ The ``clang::Decl`` object can be queried for the presence or absence of an
 attribute using ``hasAttr<T>()``. To obtain a pointer to the semantic
 representation of the attribute, ``getAttr<T>`` may be used.
 
+The ``clang::AttributedStmt`` object can  be queried for the presence or absence
+of an attribute by calling ``getAttrs()`` and looping over the list of
+attributes.
+
 How to add an expression or statement
 -------------------------------------
 
diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html
index 0d2570bcd58f2de047ea09cbed3f0b8a1818d10e..b8fb27b126fc1e742808e6f09ff0ab43f223907f 100644
--- a/clang/docs/LibASTMatchersReference.html
+++ b/clang/docs/LibASTMatchersReference.html
@@ -1337,6 +1337,16 @@ Example matches 'a', L'a'
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('coawaitExpr0')"><a name="coawaitExpr0Anchor">coawaitExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CoawaitExpr.html">CoawaitExpr</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="coawaitExpr0"><pre>Matches co_await expressions.
+
+Given
+  co_await 1;
+coawaitExpr()
+  matches 'co_await 1'
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('compoundLiteralExpr0')"><a name="compoundLiteralExpr0Anchor">compoundLiteralExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="compoundLiteralExpr0"><pre>Matches compound (i.e. non-scalar) literals
 
@@ -1383,6 +1393,26 @@ continueStmt()
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('coreturnStmt0')"><a name="coreturnStmt0Anchor">coreturnStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CoreturnStmt.html">CoreturnStmt</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="coreturnStmt0"><pre>Matches co_return statements.
+
+Given
+  while (true) { co_return; }
+coreturnStmt()
+  matches 'co_return'
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('coyieldExpr0')"><a name="coyieldExpr0Anchor">coyieldExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CoyieldExpr.html">CoyieldExpr</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="coyieldExpr0"><pre>Matches co_yield expressions.
+
+Given
+  co_yield 1;
+coyieldExpr()
+  matches 'co_yield 1'
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cudaKernelCallExpr0')"><a name="cudaKernelCallExpr0Anchor">cudaKernelCallExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CUDAKernelCallExpr.html">CUDAKernelCallExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cudaKernelCallExpr0"><pre>Matches CUDA kernel call expression.
 
@@ -1698,6 +1728,11 @@ defaultStmt()
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('dependentCoawaitExpr0')"><a name="dependentCoawaitExpr0Anchor">dependentCoawaitExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentCoawaitExpr.html">DependentCoawaitExpr</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="dependentCoawaitExpr0"><pre>Matches co_await expressions where the type of the promise is dependent
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('designatedInitExpr0')"><a name="designatedInitExpr0Anchor">designatedInitExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DesignatedInitExpr.html">DesignatedInitExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="designatedInitExpr0"><pre>Matches C99 designated initializer expressions [C99 6.7.8].
 
diff --git a/clang/docs/OpenCLSupport.rst b/clang/docs/OpenCLSupport.rst
index f7d6b1f7f2c954b50dea4d921cd136befae15c0a..8af579a082a97c154d5e94e778b25347070e181c 100644
--- a/clang/docs/OpenCLSupport.rst
+++ b/clang/docs/OpenCLSupport.rst
@@ -360,7 +360,7 @@ implementation status.
 +------------------------------+--------------------------------------------------------------+----------------------+---------------------------------------------------------------------------+
 | Predefined macros            | New version macro                                            | :good:`done`         | https://reviews.llvm.org/D88300                                           |
 +------------------------------+--------------------------------------------------------------+----------------------+---------------------------------------------------------------------------+
-| Predefined macros            | Feature macros                                               | :part:`worked on`    | https://reviews.llvm.org/D89869                                           |
+| Predefined macros            | Feature macros                                               | :good:`done`         | https://reviews.llvm.org/D95776                                           |
 +------------------------------+--------------------------------------------------------------+----------------------+---------------------------------------------------------------------------+
 | Feature optionality          | Generic address space                                        | :none:`unclaimed`    |                                                                           |
 +------------------------------+--------------------------------------------------------------+----------------------+---------------------------------------------------------------------------+
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index c78445b9be6f1417fd8e95bf39c5742df6cd2c15..d4c9f53b82c0b9e9e51553c06127e4ac852c22e0 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -72,6 +72,13 @@ Modified Compiler Flags
 -----------------------
 
 - -Wshadow now also checks for shadowed structured bindings
+- ``-B <prefix>`` (when ``<prefix>`` is a directory) was overloaded to additionally
+  detect GCC installations under ``<prefix>`` (``lib{,32,64}/gcc{,-cross}/$triple``).
+  This behavior was incompatible with GCC, caused interop issues with
+  ``--gcc-toolchain``, and was thus dropped. Specify ``--gcc-toolchain=<dir>``
+  instead. ``-B``'s other GCC-compatible semantics are preserved:
+  ``$prefix/$triple-$file`` and ``$prefix$file`` are searched for executables,
+  libraries, includes, and data files used by the compiler.
 
 Removed Compiler Flags
 -------------------------
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 28de4e3aac6f74cc240dc10618f00ff2bb2afd2c..6c8d297e618fc7c74c058684caeef61601553350 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -674,6 +674,11 @@ control the crash diagnostics.
 The -fno-crash-diagnostics flag can be helpful for speeding the process
 of generating a delta reduced test case.
 
+.. option:: -fcrash-diagnostics-dir=<dir>
+
+  Specify where to write the crash diagnostics files; defaults to the
+  usual location for temporary files.
+
 Clang is also capable of generating preprocessed source file(s) and associated
 run script(s) even without a crash. This is specially useful when trying to
 generate a reproducer for warnings or errors while using modules.
@@ -3232,6 +3237,18 @@ compiling ``.cl`` file ``-cl-std=clc++``, ``-cl-std=CLC++``, ``-std=clc++`` or
 
      clang -cl-std=clc++ test.cl
 
+Alternatively, files with ``.clcpp`` extension are compiled with the C++ for OpenCL
+mode.
+
+   .. code-block:: console
+
+     clang test.clcpp
+
+C++ for OpenCL kernel sources can also be compiled online in drivers supporting 
+`cl_ext_cxx_for_opencl
+<https://www.khronos.org/registry/OpenCL/extensions/ext/cl_ext_cxx_for_opencl.html>`_
+extension.
+
 Constructing and destroying global objects
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -3629,6 +3646,8 @@ Execute ``clang-cl /?`` to see a list of supported options:
       -fcomplete-member-pointers
                               Require member pointer base types to be complete if they would be significant under the Microsoft ABI
       -fcoverage-mapping      Generate coverage mapping to enable code coverage analysis
+      -fcrash-diagnostics-dir=<dir>
+                              Put crash-report files in <dir>
       -fdebug-macro           Emit macro debug information
       -fdelayed-template-parsing
                               Parse templated function definitions at the end of the translation unit
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index b052501c6b24073339ff4938be55467337353f5b..bcc063051b8c3bfaae106c86f069203ba3d7ed4d 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2576,7 +2576,11 @@ enum CXCursorKind {
    */
   CXCursor_OMPCanonicalLoop = 289,
 
-  CXCursor_LastStmt = CXCursor_OMPCanonicalLoop,
+  /** OpenMP interop directive.
+   */
+  CXCursor_OMPInteropDirective = 290,
+
+  CXCursor_LastStmt = CXCursor_OMPInteropDirective,
 
   /**
    * Cursor that represents the translation unit itself.
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index fa7cb268654d1b055420debadb26beecac8e6aa2..0f7e8e1782531af6681b34428b118b2ffd1e765b 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -625,6 +625,9 @@ private:
   std::unique_ptr<interp::Context> InterpContext;
   std::unique_ptr<ParentMapContext> ParentMapCtx;
 
+  /// Keeps track of the deallocated DeclListNodes for future reuse.
+  DeclListNode *ListNodeFreeList = nullptr;
+
 public:
   IdentifierTable &Idents;
   SelectorTable &Selectors;
@@ -676,6 +679,24 @@ public:
   }
   void Deallocate(void *Ptr) const {}
 
+  /// Allocates a \c DeclListNode or returns one from the \c ListNodeFreeList
+  /// pool.
+  DeclListNode *AllocateDeclListNode(clang::NamedDecl *ND) {
+    if (DeclListNode *Alloc = ListNodeFreeList) {
+      ListNodeFreeList = Alloc->Rest.dyn_cast<DeclListNode*>();
+      Alloc->D = ND;
+      Alloc->Rest = nullptr;
+      return Alloc;
+    }
+    return new (*this) DeclListNode(ND);
+  }
+  /// Deallcates a \c DeclListNode by returning it to the \c ListNodeFreeList
+  /// pool.
+  void DeallocateDeclListNode(DeclListNode *N) {
+    N->Rest = ListNodeFreeList;
+    ListNodeFreeList = N;
+  }
+
   /// Return the total amount of physical memory allocated for representing
   /// AST nodes and type information.
   size_t getASTAllocatedMemory() const {
diff --git a/clang/include/clang/AST/ASTImporter.h b/clang/include/clang/AST/ASTImporter.h
index 630d220deff70baec5fc9ac34b8406bb2c6ce0df..17e673a8471a73632a52cf0b8a5316038f2b7743 100644
--- a/clang/include/clang/AST/ASTImporter.h
+++ b/clang/include/clang/AST/ASTImporter.h
@@ -344,6 +344,12 @@ class TypeSourceInfo;
     Import(ExprWithCleanups::CleanupObject From);
 
     /// Import the given type from the "from" context into the "to"
+    /// context.
+    ///
+    /// \returns The equivalent type in the "to" context, or the import error.
+    llvm::Expected<const Type *> Import(const Type *FromT);
+
+    /// Import the given qualified type from the "from" context into the "to"
     /// context. A null type is imported as a null type (no error).
     ///
     /// \returns The equivalent type in the "to" context, or the import error.
diff --git a/clang/include/clang/AST/CXXInheritance.h b/clang/include/clang/AST/CXXInheritance.h
index 709f08bff82aee84794a3c3d8b4781891fce0fc2..946b9e318baa277cd4733b625b65fc0138589ae4 100644
--- a/clang/include/clang/AST/CXXInheritance.h
+++ b/clang/include/clang/AST/CXXInheritance.h
@@ -76,9 +76,8 @@ public:
 
   CXXBasePath() = default;
 
-  /// The set of declarations found inside this base class
-  /// subobject.
-  DeclContext::lookup_result Decls;
+  /// The declarations found inside this base class subobject.
+  DeclContext::lookup_iterator Decls;
 
   void clear() {
     SmallVectorImpl<CXXBasePathElement>::clear();
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index e1153478305a1f998b214d627df05dfb606ddf18..b84519988fc421f2acc3c0f293139d03ffbee28c 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -579,6 +579,16 @@ public:
     AnonOrFirstNamespaceAndInline.setInt(Inline);
   }
 
+  /// Returns true if the inline qualifier for \c Name is redundant.
+  bool isRedundantInlineQualifierFor(DeclarationName Name) const {
+    if (!isInline())
+      return false;
+    auto X = lookup(Name);
+    auto Y = getParent()->lookup(Name);
+    return std::distance(X.begin(), X.end()) ==
+      std::distance(Y.begin(), Y.end());
+  }
+
   /// Get the original (first) namespace declaration.
   NamespaceDecl *getOriginalNamespace();
 
diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index 24d72f1cf6cda8bce4470e6fd9eb732c116b0ef5..787504c46904a8bd23bb7117def0cb7e81b2c413 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -1220,65 +1220,110 @@ public:
 
   void print(raw_ostream &OS) const override;
 };
+} // namespace clang
 
-/// The results of name lookup within a DeclContext. This is either a
-/// single result (with no stable storage) or a collection of results (with
-/// stable storage provided by the lookup table).
-class DeclContextLookupResult {
-  using ResultTy = ArrayRef<NamedDecl *>;
-
-  ResultTy Result;
-
-  // If there is only one lookup result, it would be invalidated by
-  // reallocations of the name table, so store it separately.
-  NamedDecl *Single = nullptr;
-
-  static NamedDecl *const SingleElementDummyList;
+// Required to determine the layout of the PointerUnion<NamedDecl*> before
+// seeing the NamedDecl definition being first used in DeclListNode::operator*.
+namespace llvm {
+  template <> struct PointerLikeTypeTraits<::clang::NamedDecl *> {
+    static inline void *getAsVoidPointer(::clang::NamedDecl *P) { return P; }
+    static inline ::clang::NamedDecl *getFromVoidPointer(void *P) {
+      return static_cast<::clang::NamedDecl *>(P);
+    }
+    static constexpr int NumLowBitsAvailable = 3;
+  };
+}
 
+namespace clang {
+/// A list storing NamedDecls in the lookup tables.
+class DeclListNode {
+  friend class ASTContext; // allocate, deallocate nodes.
+  friend class StoredDeclsList;
 public:
-  DeclContextLookupResult() = default;
-  DeclContextLookupResult(ArrayRef<NamedDecl *> Result)
-      : Result(Result) {}
-  DeclContextLookupResult(NamedDecl *Single)
-      : Result(SingleElementDummyList), Single(Single) {}
-
-  class iterator;
-
-  using IteratorBase =
-      llvm::iterator_adaptor_base<iterator, ResultTy::iterator,
-                                  std::random_access_iterator_tag, NamedDecl *>;
-
-  class iterator : public IteratorBase {
-    value_type SingleElement;
+  using Decls = llvm::PointerUnion<NamedDecl*, DeclListNode*>;
+  class iterator {
+    friend class DeclContextLookupResult;
+    friend class StoredDeclsList;
 
+    Decls Ptr;
+    iterator(Decls Node) : Ptr(Node) { }
   public:
-    explicit iterator(pointer Pos, value_type Single = nullptr)
-        : IteratorBase(Pos), SingleElement(Single) {}
+    using difference_type = ptrdiff_t;
+    using value_type = NamedDecl*;
+    using pointer = void;
+    using reference = value_type;
+    using iterator_category = std::forward_iterator_tag;
+
+    iterator() = default;
 
     reference operator*() const {
-      return SingleElement ? SingleElement : IteratorBase::operator*();
+      assert(Ptr && "dereferencing end() iterator");
+      if (DeclListNode *CurNode = Ptr.dyn_cast<DeclListNode*>())
+        return CurNode->D;
+      return Ptr.get<NamedDecl*>();
     }
+    void operator->() const { } // Unsupported.
+    bool operator==(const iterator &X) const { return Ptr == X.Ptr; }
+    bool operator!=(const iterator &X) const { return Ptr != X.Ptr; }
+    inline iterator &operator++() { // ++It
+      assert(!Ptr.isNull() && "Advancing empty iterator");
+
+      if (DeclListNode *CurNode = Ptr.dyn_cast<DeclListNode*>())
+        Ptr = CurNode->Rest;
+      else
+        Ptr = nullptr;
+      return *this;
+    }
+    iterator operator++(int) { // It++
+      iterator temp = *this;
+      ++(*this);
+      return temp;
+    }
+    // Enables the pattern for (iterator I =..., E = I.end(); I != E; ++I)
+    iterator end() { return iterator(); }
   };
+private:
+  NamedDecl *D = nullptr;
+  Decls Rest = nullptr;
+  DeclListNode(NamedDecl *ND) : D(ND) {}
+};
+
+/// The results of name lookup within a DeclContext.
+class DeclContextLookupResult {
+  using Decls = DeclListNode::Decls;
 
+  /// When in collection form, this is what the Data pointer points to.
+  Decls Result;
+
+public:
+  DeclContextLookupResult() = default;
+  DeclContextLookupResult(Decls Result) : Result(Result) {}
+
+  using iterator = DeclListNode::iterator;
   using const_iterator = iterator;
-  using pointer = iterator::pointer;
   using reference = iterator::reference;
 
-  iterator begin() const { return iterator(Result.begin(), Single); }
-  iterator end() const { return iterator(Result.end(), Single); }
-
-  bool empty() const { return Result.empty(); }
-  pointer data() const { return Single ? &Single : Result.data(); }
-  size_t size() const { return Single ? 1 : Result.size(); }
-  reference front() const { return Single ? Single : Result.front(); }
-  reference back() const { return Single ? Single : Result.back(); }
-  reference operator[](size_t N) const { return Single ? Single : Result[N]; }
-
-  // FIXME: Remove this from the interface
-  DeclContextLookupResult slice(size_t N) const {
-    DeclContextLookupResult Sliced = Result.slice(N);
-    Sliced.Single = Single;
-    return Sliced;
+  iterator begin() { return iterator(Result); }
+  iterator end() { return iterator(); }
+  const_iterator begin() const {
+    return const_cast<DeclContextLookupResult*>(this)->begin();
+  }
+  const_iterator end() const { return iterator(); }
+
+  bool empty() const { return Result.isNull();  }
+  bool isSingleResult() const { return Result.dyn_cast<NamedDecl*>(); }
+  reference front() const { return *begin(); }
+
+  // Find the first declaration of the given type in the list. Note that this
+  // is not in general the earliest-declared declaration, and should only be
+  // used when it's not possible for there to be more than one match or where
+  // it doesn't matter which one is found.
+  template<class T> T *find_first() const {
+    for (auto *D : *this)
+      if (T *Decl = dyn_cast<T>(D))
+        return Decl;
+
+    return nullptr;
   }
 };
 
diff --git a/clang/include/clang/AST/DeclContextInternals.h b/clang/include/clang/AST/DeclContextInternals.h
index e6a4cd4381e4aa53fc7f30f4f60e6f8454b67310..3556044098c466e1b7b15b0431db34db2bd0f1ca 100644
--- a/clang/include/clang/AST/DeclContextInternals.h
+++ b/clang/include/clang/AST/DeclContextInternals.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_AST_DECLCONTEXTINTERNALS_H
 #define LLVM_CLANG_AST_DECLCONTEXTINTERNALS_H
 
+#include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclCXX.h"
@@ -21,7 +22,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/PointerUnion.h"
-#include "llvm/ADT/SmallVector.h"
 #include <algorithm>
 #include <cassert>
 
@@ -31,231 +31,287 @@ class DependentDiagnostic;
 
 /// An array of decls optimized for the common case of only containing
 /// one entry.
-struct StoredDeclsList {
-  /// When in vector form, this is what the Data pointer points to.
-  using DeclsTy = SmallVector<NamedDecl *, 4>;
+class StoredDeclsList {
+  using Decls = DeclListNode::Decls;
 
   /// A collection of declarations, with a flag to indicate if we have
   /// further external declarations.
-  using DeclsAndHasExternalTy = llvm::PointerIntPair<DeclsTy *, 1, bool>;
+  using DeclsAndHasExternalTy = llvm::PointerIntPair<Decls, 1, bool>;
 
   /// The stored data, which will be either a pointer to a NamedDecl,
-  /// or a pointer to a vector with a flag to indicate if there are further
+  /// or a pointer to a list with a flag to indicate if there are further
   /// external declarations.
-  llvm::PointerUnion<NamedDecl *, DeclsAndHasExternalTy> Data;
+  DeclsAndHasExternalTy Data;
+
+  template<typename Fn>
+  void erase_if(Fn ShouldErase) {
+    Decls List = Data.getPointer();
+    if (!List)
+      return;
+    ASTContext &C = getASTContext();
+    DeclListNode::Decls NewHead = nullptr;
+    DeclListNode::Decls *NewLast = nullptr;
+    DeclListNode::Decls *NewTail = &NewHead;
+    while (true) {
+      if (!ShouldErase(*DeclListNode::iterator(List))) {
+        NewLast = NewTail;
+        *NewTail = List;
+        if (auto *Node = List.dyn_cast<DeclListNode*>()) {
+          NewTail = &Node->Rest;
+          List = Node->Rest;
+        } else {
+          break;
+        }
+      } else if (DeclListNode *N = List.dyn_cast<DeclListNode*>()) {
+        List = N->Rest;
+        C.DeallocateDeclListNode(N);
+      } else {
+        // We're discarding the last declaration in the list. The last node we
+        // want to keep (if any) will be of the form DeclListNode(D, <rest>);
+        // replace it with just D.
+        if (NewLast) {
+          DeclListNode *Node = NewLast->get<DeclListNode*>();
+          *NewLast = Node->D;
+          C.DeallocateDeclListNode(Node);
+        }
+        break;
+      }
+    }
+    Data.setPointer(NewHead);
+
+    assert(llvm::find_if(getLookupResult(), ShouldErase) ==
+           getLookupResult().end() && "Still exists!");
+  }
+
+  void erase(NamedDecl *ND) {
+    erase_if([ND](NamedDecl *D) { return D == ND; });
+  }
 
 public:
   StoredDeclsList() = default;
 
   StoredDeclsList(StoredDeclsList &&RHS) : Data(RHS.Data) {
-    RHS.Data = (NamedDecl *)nullptr;
+    RHS.Data.setPointer(nullptr);
+    RHS.Data.setInt(0);
+  }
+
+  void MaybeDeallocList() {
+    if (isNull())
+      return;
+    // If this is a list-form, free the list.
+    ASTContext &C = getASTContext();
+    Decls List = Data.getPointer();
+    while (DeclListNode *ToDealloc = List.dyn_cast<DeclListNode *>()) {
+      List = ToDealloc->Rest;
+      C.DeallocateDeclListNode(ToDealloc);
+    }
   }
 
   ~StoredDeclsList() {
-    // If this is a vector-form, free the vector.
-    if (DeclsTy *Vector = getAsVector())
-      delete Vector;
+    MaybeDeallocList();
   }
 
   StoredDeclsList &operator=(StoredDeclsList &&RHS) {
-    if (DeclsTy *Vector = getAsVector())
-      delete Vector;
+    MaybeDeallocList();
+
     Data = RHS.Data;
-    RHS.Data = (NamedDecl *)nullptr;
+    RHS.Data.setPointer(nullptr);
+    RHS.Data.setInt(0);
     return *this;
   }
 
-  bool isNull() const { return Data.isNull(); }
+  bool isNull() const { return Data.getPointer().isNull(); }
 
-  NamedDecl *getAsDecl() const {
-    return Data.dyn_cast<NamedDecl *>();
+  ASTContext &getASTContext() {
+    assert(!isNull() && "No ASTContext.");
+    if (NamedDecl *ND = getAsDecl())
+      return ND->getASTContext();
+    return getAsList()->D->getASTContext();
   }
 
-  DeclsAndHasExternalTy getAsVectorAndHasExternal() const {
-    return Data.dyn_cast<DeclsAndHasExternalTy>();
+  DeclsAndHasExternalTy getAsListAndHasExternal() const { return Data; }
+
+  NamedDecl *getAsDecl() const {
+    return getAsListAndHasExternal().getPointer().dyn_cast<NamedDecl *>();
   }
 
-  DeclsTy *getAsVector() const {
-    return getAsVectorAndHasExternal().getPointer();
+  DeclListNode *getAsList() const {
+    return getAsListAndHasExternal().getPointer().dyn_cast<DeclListNode*>();
   }
 
   bool hasExternalDecls() const {
-    return getAsVectorAndHasExternal().getInt();
+    return getAsListAndHasExternal().getInt();
   }
 
   void setHasExternalDecls() {
-    if (DeclsTy *Vec = getAsVector())
-      Data = DeclsAndHasExternalTy(Vec, true);
-    else {
-      DeclsTy *VT = new DeclsTy();
-      if (NamedDecl *OldD = getAsDecl())
-        VT->push_back(OldD);
-      Data = DeclsAndHasExternalTy(VT, true);
-    }
-  }
-
-  void setOnlyValue(NamedDecl *ND) {
-    assert(!getAsVector() && "Not inline");
-    Data = ND;
-    // Make sure that Data is a plain NamedDecl* so we can use its address
-    // at getLookupResult.
-    assert(*(NamedDecl **)&Data == ND &&
-           "PointerUnion mangles the NamedDecl pointer!");
+    Data.setInt(1);
   }
 
   void remove(NamedDecl *D) {
     assert(!isNull() && "removing from empty list");
-    if (NamedDecl *Singleton = getAsDecl()) {
-      assert(Singleton == D && "list is different singleton");
-      (void)Singleton;
-      Data = (NamedDecl *)nullptr;
-      return;
-    }
-
-    DeclsTy &Vec = *getAsVector();
-    DeclsTy::iterator I = llvm::find(Vec, D);
-    assert(I != Vec.end() && "list does not contain decl");
-    Vec.erase(I);
-
-    assert(llvm::find(Vec, D) == Vec.end() && "list still contains decl");
+    erase(D);
   }
 
-  /// Remove any declarations which were imported from an external
-  /// AST source.
+  /// Remove any declarations which were imported from an external AST source.
   void removeExternalDecls() {
-    if (isNull()) {
-      // Nothing to do.
-    } else if (NamedDecl *Singleton = getAsDecl()) {
-      if (Singleton->isFromASTFile())
-        *this = StoredDeclsList();
-    } else {
-      DeclsTy &Vec = *getAsVector();
-      Vec.erase(std::remove_if(Vec.begin(), Vec.end(),
-                               [](Decl *D) { return D->isFromASTFile(); }),
-                Vec.end());
-      // Don't have any external decls any more.
-      Data = DeclsAndHasExternalTy(&Vec, false);
-    }
+    erase_if([](NamedDecl *ND) { return ND->isFromASTFile(); });
+
+    // Don't have any pending external decls any more.
+    Data.setInt(0);
   }
 
-  /// getLookupResult - Return an array of all the decls that this list
-  /// represents.
-  DeclContext::lookup_result getLookupResult() {
-    if (isNull())
-      return DeclContext::lookup_result();
+  void replaceExternalDecls(ArrayRef<NamedDecl*> Decls) {
+    // Remove all declarations that are either external or are replaced with
+    // external declarations.
+    erase_if([Decls](NamedDecl *ND) {
+      if (ND->isFromASTFile())
+        return true;
+      for (NamedDecl *D : Decls)
+        if (D->declarationReplaces(ND, /*IsKnownNewer=*/false))
+          return true;
+      return false;
+    });
 
-    // If we have a single NamedDecl, return it.
-    if (NamedDecl *ND = getAsDecl()) {
-      assert(!isNull() && "Empty list isn't allowed");
+    // Don't have any pending external decls any more.
+    Data.setInt(0);
+
+    if (Decls.empty())
+      return;
+
+    // Convert Decls into a list, in order.
+    ASTContext &C = Decls.front()->getASTContext();
+    DeclListNode::Decls DeclsAsList = Decls.back();
+    for (size_t I = Decls.size() - 1; I != 0; --I) {
+      DeclListNode *Node = C.AllocateDeclListNode(Decls[I - 1]);
+      Node->Rest = DeclsAsList;
+      DeclsAsList = Node;
+    }
 
-      // Data is a raw pointer to a NamedDecl*, return it.
-      return DeclContext::lookup_result(ND);
+    DeclListNode::Decls Head = Data.getPointer();
+    if (Head.isNull()) {
+      Data.setPointer(DeclsAsList);
+      return;
     }
 
-    assert(getAsVector() && "Must have a vector at this point");
-    DeclsTy &Vector = *getAsVector();
+    // Find the end of the existing list.
+    // FIXME: It would be possible to preserve information from erase_if to
+    // avoid this rescan looking for the end of the list.
+    DeclListNode::Decls *Tail = &Head;
+    while (DeclListNode *Node = Tail->dyn_cast<DeclListNode *>())
+      Tail = &Node->Rest;
+
+    // Append the Decls.
+    DeclListNode *Node = C.AllocateDeclListNode(Tail->get<NamedDecl *>());
+    Node->Rest = DeclsAsList;
+    *Tail = Node;
+    Data.setPointer(Head);
+  }
 
-    // Otherwise, we have a range result.
-    return DeclContext::lookup_result(Vector);
+  /// Return an array of all the decls that this list represents.
+  DeclContext::lookup_result getLookupResult() const {
+    return DeclContext::lookup_result(Data.getPointer());
   }
 
-  /// HandleRedeclaration - If this is a redeclaration of an existing decl,
-  /// replace the old one with D and return true.  Otherwise return false.
-  bool HandleRedeclaration(NamedDecl *D, bool IsKnownNewer) {
+  /// If this is a redeclaration of an existing decl, replace the old one with
+  /// D. Otherwise, append D.
+  void addOrReplaceDecl(NamedDecl *D) {
+    const bool IsKnownNewer = true;
+
+    if (isNull()) {
+      Data.setPointer(D);
+      return;
+    }
+
     // Most decls only have one entry in their list, special case it.
     if (NamedDecl *OldD = getAsDecl()) {
-      if (!D->declarationReplaces(OldD, IsKnownNewer))
-        return false;
-      setOnlyValue(D);
-      return true;
+      if (D->declarationReplaces(OldD, IsKnownNewer)) {
+        Data.setPointer(D);
+        return;
+      }
+
+      // Add D after OldD.
+      ASTContext &C = D->getASTContext();
+      DeclListNode *Node = C.AllocateDeclListNode(OldD);
+      Node->Rest = D;
+      Data.setPointer(Node);
+      return;
     }
 
+    // FIXME: Move the assert before the single decl case when we fix the
+    // duplication coming from the ASTReader reading builtin types.
+    assert(!llvm::is_contained(getLookupResult(), D) && "Already exists!");
     // Determine if this declaration is actually a redeclaration.
-    DeclsTy &Vec = *getAsVector();
-    for (DeclsTy::iterator OD = Vec.begin(), ODEnd = Vec.end();
-         OD != ODEnd; ++OD) {
-      NamedDecl *OldD = *OD;
-      if (D->declarationReplaces(OldD, IsKnownNewer)) {
-        *OD = D;
-        return true;
+    for (DeclListNode *N = getAsList(); /*return in loop*/;
+         N = N->Rest.dyn_cast<DeclListNode *>()) {
+      if (D->declarationReplaces(N->D, IsKnownNewer)) {
+        N->D = D;
+        return;
+      }
+      if (auto *ND = N->Rest.dyn_cast<NamedDecl *>()) {
+        if (D->declarationReplaces(ND, IsKnownNewer)) {
+          N->Rest = D;
+          return;
+        }
+
+        // Add D after ND.
+        ASTContext &C = D->getASTContext();
+        DeclListNode *Node = C.AllocateDeclListNode(ND);
+        N->Rest = Node;
+        Node->Rest = D;
+        return;
       }
     }
-
-    return false;
   }
 
-  /// AddSubsequentDecl - This is called on the second and later decl when it is
-  /// not a redeclaration to merge it into the appropriate place in our list.
-  void AddSubsequentDecl(NamedDecl *D) {
-    assert(!isNull() && "don't AddSubsequentDecl when we have no decls");
+  /// Add a declaration to the list without checking if it replaces anything.
+  void prependDeclNoReplace(NamedDecl *D) {
+    if (isNull()) {
+      Data.setPointer(D);
+      return;
+    }
 
-    // If this is the second decl added to the list, convert this to vector
-    // form.
-    if (NamedDecl *OldD = getAsDecl()) {
-      DeclsTy *VT = new DeclsTy();
-      VT->push_back(OldD);
-      Data = DeclsAndHasExternalTy(VT, false);
+    ASTContext &C = D->getASTContext();
+    DeclListNode *Node = C.AllocateDeclListNode(D);
+    Node->Rest = Data.getPointer();
+    Data.setPointer(Node);
+  }
+
+  LLVM_DUMP_METHOD void dump() const {
+    Decls D = Data.getPointer();
+    if (!D) {
+      llvm::errs() << "<null>\n";
+      return;
     }
 
-    DeclsTy &Vec = *getAsVector();
-
-    // Using directives end up in a special entry which contains only
-    // other using directives, so all this logic is wasted for them.
-    // But avoiding the logic wastes time in the far-more-common case
-    // that we're *not* adding a new using directive.
-
-    // Tag declarations always go at the end of the list so that an
-    // iterator which points at the first tag will start a span of
-    // decls that only contains tags.
-    if (D->hasTagIdentifierNamespace())
-      Vec.push_back(D);
-
-    // Resolved using declarations go at the front of the list so that
-    // they won't show up in other lookup results.  Unresolved using
-    // declarations (which are always in IDNS_Using | IDNS_Ordinary)
-    // follow that so that the using declarations will be contiguous.
-    else if (D->getIdentifierNamespace() & Decl::IDNS_Using) {
-      DeclsTy::iterator I = Vec.begin();
-      if (D->getIdentifierNamespace() != Decl::IDNS_Using) {
-        while (I != Vec.end() &&
-               (*I)->getIdentifierNamespace() == Decl::IDNS_Using)
-          ++I;
+    while (true) {
+      if (auto *Node = D.dyn_cast<DeclListNode*>()) {
+        llvm::errs() << '[' << Node->D << "] -> ";
+        D = Node->Rest;
+      } else {
+        llvm::errs() << '[' << D.get<NamedDecl*>() << "]\n";
+        return;
       }
-      Vec.insert(I, D);
-
-    // All other declarations go at the end of the list, but before any
-    // tag declarations.  But we can be clever about tag declarations
-    // because there can only ever be one in a scope.
-    } else if (!Vec.empty() && Vec.back()->hasTagIdentifierNamespace()) {
-      NamedDecl *TagD = Vec.back();
-      Vec.back() = D;
-      Vec.push_back(TagD);
-    } else
-      Vec.push_back(D);
+    }
   }
 };
 
 class StoredDeclsMap
     : public llvm::SmallDenseMap<DeclarationName, StoredDeclsList, 4> {
-public:
-  static void DestroyAll(StoredDeclsMap *Map, bool Dependent);
-
-private:
   friend class ASTContext; // walks the chain deleting these
   friend class DeclContext;
 
   llvm::PointerIntPair<StoredDeclsMap*, 1> Previous;
+public:
+  static void DestroyAll(StoredDeclsMap *Map, bool Dependent);
 };
 
 class DependentStoredDeclsMap : public StoredDeclsMap {
-public:
-  DependentStoredDeclsMap() = default;
-
-private:
   friend class DeclContext; // iterates over diagnostics
   friend class DependentDiagnostic;
 
   DependentDiagnostic *FirstDiagnostic = nullptr;
+public:
+  DependentStoredDeclsMap() = default;
 };
 
 } // namespace clang
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 958f2b9e0e6f2ab04a83ba25c1589f7d2c62b638..f71eb15feea272126f47b6cd85d07f7c571a2932 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -7366,15 +7366,244 @@ public:
   }
 };
 
+/// This represents the 'init' clause in '#pragma omp ...' directives.
+///
+/// \code
+/// #pragma omp interop init(target:obj)
+/// \endcode
+class OMPInitClause final
+    : public OMPVarListClause<OMPInitClause>,
+      private llvm::TrailingObjects<OMPInitClause, Expr *> {
+  friend class OMPClauseReader;
+  friend OMPVarListClause;
+  friend TrailingObjects;
+
+  /// Location of interop variable.
+  SourceLocation VarLoc;
+
+  bool IsTarget = false;
+  bool IsTargetSync = false;
+
+  void setInteropVar(Expr *E) { varlist_begin()[0] = E; }
+
+  void setIsTarget(bool V) { IsTarget = V; }
+
+  void setIsTargetSync(bool V) { IsTargetSync = V; }
+
+  /// Sets the location of the interop variable.
+  void setVarLoc(SourceLocation Loc) { VarLoc = Loc; }
+
+  /// Build 'init' clause.
+  ///
+  /// \param IsTarget Uses the 'target' interop-type.
+  /// \param IsTargetSync Uses the 'targetsync' interop-type.
+  /// \param StartLoc Starting location of the clause.
+  /// \param LParenLoc Location of '('.
+  /// \param VarLoc Location of the interop variable.
+  /// \param EndLoc Ending location of the clause.
+  /// \param N Number of expressions.
+  OMPInitClause(bool IsTarget, bool IsTargetSync, SourceLocation StartLoc,
+                SourceLocation LParenLoc, SourceLocation VarLoc,
+                SourceLocation EndLoc, unsigned N)
+      : OMPVarListClause<OMPInitClause>(llvm::omp::OMPC_init, StartLoc,
+                                        LParenLoc, EndLoc, N),
+        VarLoc(VarLoc), IsTarget(IsTarget), IsTargetSync(IsTargetSync) {}
+
+  /// Build an empty clause.
+  OMPInitClause(unsigned N)
+      : OMPVarListClause<OMPInitClause>(llvm::omp::OMPC_init, SourceLocation(),
+                                        SourceLocation(), SourceLocation(), N) {
+  }
+
+public:
+  /// Creates a fully specified clause.
+  ///
+  /// \param C AST context.
+  /// \param InteropVar The interop variable.
+  /// \param PrefExprs The list of preference expressions.
+  /// \param IsTarget Uses the 'target' interop-type.
+  /// \param IsTargetSync Uses the 'targetsync' interop-type.
+  /// \param StartLoc Starting location of the clause.
+  /// \param LParenLoc Location of '('.
+  /// \param VarLoc Location of the interop variable.
+  /// \param EndLoc Ending location of the clause.
+  static OMPInitClause *Create(const ASTContext &C, Expr *InteropVar,
+                               ArrayRef<Expr *> PrefExprs, bool IsTarget,
+                               bool IsTargetSync, SourceLocation StartLoc,
+                               SourceLocation LParenLoc, SourceLocation VarLoc,
+                               SourceLocation EndLoc);
+
+  /// Creates an empty clause with \a N expressions.
+  ///
+  /// \param C AST context.
+  /// \param N Number of expression items.
+  static OMPInitClause *CreateEmpty(const ASTContext &C, unsigned N);
+
+  /// Returns the location of the interop variable.
+  SourceLocation getVarLoc() const { return VarLoc; }
+
+  /// Returns the interop variable.
+  Expr *getInteropVar() { return varlist_begin()[0]; }
+  const Expr *getInteropVar() const { return varlist_begin()[0]; }
+
+  /// Returns true is interop-type 'target' is used.
+  bool getIsTarget() const { return IsTarget; }
+
+  /// Returns true is interop-type 'targetsync' is used.
+  bool getIsTargetSync() const { return IsTargetSync; }
+
+  child_range children() {
+    return child_range(reinterpret_cast<Stmt **>(varlist_begin()),
+                       reinterpret_cast<Stmt **>(varlist_end()));
+  }
+
+  const_child_range children() const {
+    auto Children = const_cast<OMPInitClause *>(this)->children();
+    return const_child_range(Children.begin(), Children.end());
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  using prefs_iterator = MutableArrayRef<Expr *>::iterator;
+  using const_prefs_iterator = ArrayRef<const Expr *>::iterator;
+  using prefs_range = llvm::iterator_range<prefs_iterator>;
+  using const_prefs_range = llvm::iterator_range<const_prefs_iterator>;
+
+  prefs_range prefs() {
+    return prefs_range(reinterpret_cast<Expr **>(std::next(varlist_begin())),
+                       reinterpret_cast<Expr **>(varlist_end()));
+  }
+
+  const_prefs_range prefs() const {
+    auto Prefs = const_cast<OMPInitClause *>(this)->prefs();
+    return const_prefs_range(Prefs.begin(), Prefs.end());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_init;
+  }
+};
+
+/// This represents the 'use' clause in '#pragma omp ...' directives.
+///
+/// \code
+/// #pragma omp interop use(obj)
+/// \endcode
+class OMPUseClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// Location of interop variable.
+  SourceLocation VarLoc;
+
+  /// The interop variable.
+  Stmt *InteropVar = nullptr;
+
+  /// Set the interop variable.
+  void setInteropVar(Expr *E) { InteropVar = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+  /// Sets the location of the interop variable.
+  void setVarLoc(SourceLocation Loc) { VarLoc = Loc; }
+
+public:
+  /// Build 'use' clause with and interop variable expression \a InteropVar.
+  ///
+  /// \param InteropVar The interop variable.
+  /// \param StartLoc Starting location of the clause.
+  /// \param LParenLoc Location of '('.
+  /// \param VarLoc Location of the interop variable.
+  /// \param EndLoc Ending location of the clause.
+  OMPUseClause(Expr *InteropVar, SourceLocation StartLoc,
+               SourceLocation LParenLoc, SourceLocation VarLoc,
+               SourceLocation EndLoc)
+      : OMPClause(llvm::omp::OMPC_use, StartLoc, EndLoc), LParenLoc(LParenLoc),
+        VarLoc(VarLoc), InteropVar(InteropVar) {}
+
+  /// Build an empty clause.
+  OMPUseClause()
+      : OMPClause(llvm::omp::OMPC_use, SourceLocation(), SourceLocation()) {}
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the location of the interop variable.
+  SourceLocation getVarLoc() const { return VarLoc; }
+
+  /// Returns the interop variable.
+  Expr *getInteropVar() const { return cast<Expr>(InteropVar); }
+
+  child_range children() { return child_range(&InteropVar, &InteropVar + 1); }
+
+  const_child_range children() const {
+    return const_child_range(&InteropVar, &InteropVar + 1);
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_use;
+  }
+};
+
 /// This represents 'destroy' clause in the '#pragma omp depobj'
-/// directive.
+/// directive or the '#pragma omp interop' directive..
 ///
 /// \code
 /// #pragma omp depobj(a) destroy
+/// #pragma omp interop destroy(obj)
 /// \endcode
-/// In this example directive '#pragma omp depobj' has 'destroy' clause.
+/// In these examples directive '#pragma omp depobj' and '#pragma omp interop'
+/// have a 'destroy' clause. The 'interop' directive includes an object.
 class OMPDestroyClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// Location of interop variable.
+  SourceLocation VarLoc;
+
+  /// The interop variable.
+  Stmt *InteropVar = nullptr;
+
+  /// Set the interop variable.
+  void setInteropVar(Expr *E) { InteropVar = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+  /// Sets the location of the interop variable.
+  void setVarLoc(SourceLocation Loc) { VarLoc = Loc; }
+
 public:
+  /// Build 'destroy' clause with an interop variable expression \a InteropVar.
+  ///
+  /// \param InteropVar The interop variable.
+  /// \param StartLoc Starting location of the clause.
+  /// \param LParenLoc Location of '('.
+  /// \param VarLoc Location of the interop variable.
+  /// \param EndLoc Ending location of the clause.
+  OMPDestroyClause(Expr *InteropVar, SourceLocation StartLoc,
+                   SourceLocation LParenLoc, SourceLocation VarLoc,
+                   SourceLocation EndLoc)
+      : OMPClause(llvm::omp::OMPC_destroy, StartLoc, EndLoc),
+        LParenLoc(LParenLoc), VarLoc(VarLoc), InteropVar(InteropVar) {}
+
   /// Build 'destroy' clause.
   ///
   /// \param StartLoc Starting location of the clause.
@@ -7387,11 +7616,24 @@ public:
       : OMPClause(llvm::omp::OMPC_destroy, SourceLocation(), SourceLocation()) {
   }
 
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the location of the interop variable.
+  SourceLocation getVarLoc() const { return VarLoc; }
+
+  /// Returns the interop variable.
+  Expr *getInteropVar() const { return cast_or_null<Expr>(InteropVar); }
+
   child_range children() {
+    if (InteropVar)
+      return child_range(&InteropVar, &InteropVar + 1);
     return child_range(child_iterator(), child_iterator());
   }
 
   const_child_range children() const {
+    if (InteropVar)
+      return const_child_range(&InteropVar, &InteropVar + 1);
     return const_child_range(const_child_iterator(), const_child_iterator());
   }
 
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 8ec2c882a9f2f988d6d35cd3fffdbbd327ee438f..256f73338bd27d01f39f0e1e7688ae26cd311c7e 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -2968,6 +2968,9 @@ DEF_TRAVERSE_STMT(OMPTargetTeamsDistributeParallelForSimdDirective,
 DEF_TRAVERSE_STMT(OMPTargetTeamsDistributeSimdDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPInteropDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 // OpenMP clauses.
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::TraverseOMPClause(OMPClause *C) {
@@ -3195,7 +3198,20 @@ bool RecursiveASTVisitor<Derived>::VisitOMPNogroupClause(OMPNogroupClause *) {
 }
 
 template <typename Derived>
-bool RecursiveASTVisitor<Derived>::VisitOMPDestroyClause(OMPDestroyClause *) {
+bool RecursiveASTVisitor<Derived>::VisitOMPInitClause(OMPInitClause *C) {
+  TRY_TO(VisitOMPClauseList(C));
+  return true;
+}
+
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPUseClause(OMPUseClause *C) {
+  TRY_TO(TraverseStmt(C->getInteropVar()));
+  return true;
+}
+
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPDestroyClause(OMPDestroyClause *C) {
+  TRY_TO(TraverseStmt(C->getInteropVar()));
   return true;
 }
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 32b7778aa487847cd5cc60677d1db1c26f772504..6ddc29446a8c8ccc67812f7226bb0424470d99eb 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5086,6 +5086,59 @@ public:
   }
 };
 
+/// This represents '#pragma omp interop' directive.
+///
+/// \code
+/// #pragma omp interop init(target:obj) device(x) depend(inout:y) nowait
+/// \endcode
+/// In this example directive '#pragma omp interop' has
+/// clauses 'init', 'device', 'depend' and 'nowait'.
+///
+class OMPInteropDirective final : public OMPExecutableDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Build directive with the given start and end location.
+  ///
+  /// \param StartLoc Starting location of the directive.
+  /// \param EndLoc Ending location of the directive.
+  ///
+  OMPInteropDirective(SourceLocation StartLoc, SourceLocation EndLoc)
+      : OMPExecutableDirective(OMPInteropDirectiveClass,
+                               llvm::omp::OMPD_interop, StartLoc, EndLoc) {}
+
+  /// Build an empty directive.
+  ///
+  explicit OMPInteropDirective()
+      : OMPExecutableDirective(OMPInteropDirectiveClass,
+                               llvm::omp::OMPD_interop, SourceLocation(),
+                               SourceLocation()) {}
+
+public:
+  /// Creates directive.
+  ///
+  /// \param C AST context.
+  /// \param StartLoc Starting location of the directive.
+  /// \param EndLoc Ending Location of the directive.
+  /// \param Clauses The directive's clauses.
+  ///
+  static OMPInteropDirective *Create(const ASTContext &C,
+                                     SourceLocation StartLoc,
+                                     SourceLocation EndLoc,
+                                     ArrayRef<OMPClause *> Clauses);
+
+  /// Creates an empty directive.
+  ///
+  /// \param C AST context.
+  ///
+  static OMPInteropDirective *CreateEmpty(const ASTContext &C,
+                                          unsigned NumClauses, EmptyShell);
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPInteropDirectiveClass;
+  }
+};
+
 } // end namespace clang
 
 #endif
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 64e8cecf48228612d241738abbb11a0c63e43be1..fd0e9d6d7c1f738197fba31553f0d6cfb5630fdb 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -2162,6 +2162,17 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, BreakStmt> breakStmt;
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ContinueStmt>
     continueStmt;
 
+/// Matches co_return statements.
+///
+/// Given
+/// \code
+///   while (true) { co_return; }
+/// \endcode
+/// coreturnStmt()
+///   matches 'co_return'
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, CoreturnStmt>
+    coreturnStmt;
+
 /// Matches return statements.
 ///
 /// Given
@@ -2379,6 +2390,30 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, UserDefinedLiteral>
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CompoundLiteralExpr>
     compoundLiteralExpr;
 
+/// Matches co_await expressions.
+///
+/// Given
+/// \code
+///   co_await 1;
+/// \endcode
+/// coawaitExpr()
+///   matches 'co_await 1'
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, CoawaitExpr>
+    coawaitExpr;
+/// Matches co_await expressions where the type of the promise is dependent
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, DependentCoawaitExpr>
+    dependentCoawaitExpr;
+/// Matches co_yield expressions.
+///
+/// Given
+/// \code
+///   co_yield 1;
+/// \endcode
+/// coyieldExpr()
+///   matches 'co_yield 1'
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, CoyieldExpr>
+    coyieldExpr;
+
 /// Matches nullptr literal.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXNullPtrLiteralExpr>
     cxxNullPtrLiteralExpr;
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index 53b37b338a55a52a6dea67e51703d119f92fda0e..e8f427bafa2579b5c474f42a2e291db9f32bfcd2 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -541,12 +541,18 @@ public:
   /// Convert \c this into a \c Matcher<T> by applying dyn_cast<> to the
   /// argument.
   /// \c To must be a base class of \c T.
-  template <typename To>
-  Matcher<To> dynCastTo() const {
+  template <typename To> Matcher<To> dynCastTo() const LLVM_LVALUE_FUNCTION {
     static_assert(std::is_base_of<To, T>::value, "Invalid dynCast call.");
     return Matcher<To>(Implementation);
   }
 
+#if LLVM_HAS_RVALUE_REFERENCE_THIS
+  template <typename To> Matcher<To> dynCastTo() && {
+    static_assert(std::is_base_of<To, T>::value, "Invalid dynCast call.");
+    return Matcher<To>(std::move(Implementation));
+  }
+#endif
+
   /// Forwards the call to the underlying MatcherInterface<T> pointer.
   bool matches(const T &Node,
                ASTMatchFinder *Finder,
@@ -563,7 +569,13 @@ public:
   ///
   /// The returned matcher keeps the same restrictions as \c this and remembers
   /// that it is meant to support nodes of type \c T.
-  operator DynTypedMatcher() const { return Implementation; }
+  operator DynTypedMatcher() const LLVM_LVALUE_FUNCTION {
+    return Implementation;
+  }
+
+#if LLVM_HAS_RVALUE_REFERENCE_THIS
+  operator DynTypedMatcher() && { return std::move(Implementation); }
+#endif
 
   /// Allows the conversion of a \c Matcher<Type> to a \c
   /// Matcher<QualType>.
@@ -870,7 +882,7 @@ private:
                Names, getOperatorSpelling(Node.getOverloadedOperator()));
   }
 
-  const std::vector<std::string> Names;
+  std::vector<std::string> Names;
 };
 
 /// Matches named declarations with a specific name.
@@ -904,8 +916,8 @@ class HasNameMatcher : public SingleNodeMatcherInterface<NamedDecl> {
   /// It is slower but simple and works on all cases.
   bool matchesNodeFullSlow(const NamedDecl &Node) const;
 
-  const bool UseUnqualifiedMatch;
-  const std::vector<std::string> Names;
+  bool UseUnqualifiedMatch;
+  std::vector<std::string> Names;
 };
 
 /// Trampoline function to use VariadicFunction<> to construct a
@@ -926,7 +938,7 @@ class HasDeclarationMatcher : public MatcherInterface<T> {
   static_assert(std::is_same<DeclMatcherT, Matcher<Decl>>::value,
                 "instantiated with wrong types");
 
-  const DynTypedMatcher InnerMatcher;
+  DynTypedMatcher InnerMatcher;
 
 public:
   explicit HasDeclarationMatcher(const Matcher<Decl> &InnerMatcher)
@@ -1315,20 +1327,36 @@ public:
   VariadicOperatorMatcher(DynTypedMatcher::VariadicOperator Op, Ps &&... Params)
       : Op(Op), Params(std::forward<Ps>(Params)...) {}
 
-  template <typename T> operator Matcher<T>() const {
+  template <typename T> operator Matcher<T>() const LLVM_LVALUE_FUNCTION {
     return DynTypedMatcher::constructVariadic(
                Op, ASTNodeKind::getFromNodeKind<T>(),
                getMatchers<T>(std::index_sequence_for<Ps...>()))
         .template unconditionalConvertTo<T>();
   }
 
+#if LLVM_HAS_RVALUE_REFERENCE_THIS
+  template <typename T> operator Matcher<T>() && {
+    return DynTypedMatcher::constructVariadic(
+               Op, ASTNodeKind::getFromNodeKind<T>(),
+               getMatchers<T>(std::index_sequence_for<Ps...>()))
+        .template unconditionalConvertTo<T>();
+  }
+#endif
 private:
   // Helper method to unpack the tuple into a vector.
   template <typename T, std::size_t... Is>
-  std::vector<DynTypedMatcher> getMatchers(std::index_sequence<Is...>) const {
+  std::vector<DynTypedMatcher>
+  getMatchers(std::index_sequence<Is...>) const LLVM_LVALUE_FUNCTION {
     return {Matcher<T>(std::get<Is>(Params))...};
   }
 
+#if LLVM_HAS_RVALUE_REFERENCE_THIS
+  template <typename T, std::size_t... Is>
+  std::vector<DynTypedMatcher> getMatchers(std::index_sequence<Is...>) && {
+    return {Matcher<T>(std::get<Is>(std::move(Params)))...};
+  }
+#endif
+
   const DynTypedMatcher::VariadicOperator Op;
   std::tuple<Ps...> Params;
 };
@@ -1417,12 +1445,18 @@ public:
 
   using ReturnTypes = ToTypes;
 
-  template <typename To> operator Matcher<To>() const {
+  template <typename To> operator Matcher<To>() const LLVM_LVALUE_FUNCTION {
     return Matcher<To>(new ArgumentAdapterT<To, T>(InnerMatcher));
   }
 
+#if LLVM_HAS_RVALUE_REFERENCE_THIS
+  template <typename To> operator Matcher<To>() && {
+    return Matcher<To>(new ArgumentAdapterT<To, T>(std::move(InnerMatcher)));
+  }
+#endif
+
 private:
-  const Matcher<T> InnerMatcher;
+  Matcher<T> InnerMatcher;
 };
 
 /// Converts a \c Matcher<T> to a matcher of desired type \c To by
@@ -1464,7 +1498,7 @@ struct ArgumentAdaptingMatcherFunc {
 };
 
 template <typename T> class TraversalMatcher : public MatcherInterface<T> {
-  const DynTypedMatcher InnerMatcher;
+  DynTypedMatcher InnerMatcher;
   clang::TraversalKind Traversal;
 
 public:
@@ -1490,13 +1524,22 @@ public:
   TraversalWrapper(TraversalKind TK, const MatcherType &InnerMatcher)
       : TK(TK), InnerMatcher(InnerMatcher) {}
 
-  template <typename T> operator Matcher<T>() const {
+  template <typename T> operator Matcher<T>() const LLVM_LVALUE_FUNCTION {
     return internal::DynTypedMatcher::constructRestrictedWrapper(
                new internal::TraversalMatcher<T>(TK, InnerMatcher),
                ASTNodeKind::getFromNodeKind<T>())
         .template unconditionalConvertTo<T>();
   }
 
+#if LLVM_HAS_RVALUE_REFERENCE_THIS
+  template <typename T> operator Matcher<T>() && {
+    return internal::DynTypedMatcher::constructRestrictedWrapper(
+               new internal::TraversalMatcher<T>(TK, std::move(InnerMatcher)),
+               ASTNodeKind::getFromNodeKind<T>())
+        .template unconditionalConvertTo<T>();
+  }
+#endif
+
 private:
   TraversalKind TK;
   MatcherType InnerMatcher;
@@ -1522,15 +1565,23 @@ public:
 
   using ReturnTypes = typename ExtractFunctionArgMeta<ReturnTypesF>::type;
 
-  template <typename T>
-  operator Matcher<T>() const {
+  template <typename T> operator Matcher<T>() const LLVM_LVALUE_FUNCTION {
     static_assert(TypeListContainsSuperOf<ReturnTypes, T>::value,
                   "right polymorphic conversion");
     return Matcher<T>(new_from_tuple<MatcherT<T, ParamTypes...>>(Params));
   }
 
+#if LLVM_HAS_RVALUE_REFERENCE_THIS
+  template <typename T> operator Matcher<T>() && {
+    static_assert(TypeListContainsSuperOf<ReturnTypes, T>::value,
+                  "right polymorphic conversion");
+    return Matcher<T>(
+        new_from_tuple<MatcherT<T, ParamTypes...>>(std::move(Params)));
+  }
+#endif
+
 private:
-  const std::tuple<ParamTypes...> Params;
+  std::tuple<ParamTypes...> Params;
 };
 
 /// Matches nodes of type T that have child nodes of type ChildT for
@@ -1539,7 +1590,7 @@ private:
 /// ChildT must be an AST base type.
 template <typename T, typename ChildT>
 class HasMatcher : public MatcherInterface<T> {
-  const DynTypedMatcher InnerMatcher;
+  DynTypedMatcher InnerMatcher;
 
 public:
   explicit HasMatcher(const Matcher<ChildT> &InnerMatcher)
@@ -1562,7 +1613,7 @@ class ForEachMatcher : public MatcherInterface<T> {
   static_assert(IsBaseType<ChildT>::value,
                 "for each only accepts base type matcher");
 
-  const DynTypedMatcher InnerMatcher;
+  DynTypedMatcher InnerMatcher;
 
 public:
   explicit ForEachMatcher(const Matcher<ChildT> &InnerMatcher)
@@ -1592,7 +1643,7 @@ class HasDescendantMatcher : public MatcherInterface<T> {
   static_assert(IsBaseType<DescendantT>::value,
                 "has descendant only accepts base type matcher");
 
-  const DynTypedMatcher DescendantMatcher;
+  DynTypedMatcher DescendantMatcher;
 
 public:
   explicit HasDescendantMatcher(const Matcher<DescendantT> &DescendantMatcher)
@@ -1614,7 +1665,7 @@ class HasParentMatcher : public MatcherInterface<T> {
   static_assert(IsBaseType<ParentT>::value,
                 "has parent only accepts base type matcher");
 
-  const DynTypedMatcher ParentMatcher;
+  DynTypedMatcher ParentMatcher;
 
 public:
   explicit HasParentMatcher(const Matcher<ParentT> &ParentMatcher)
@@ -1636,7 +1687,7 @@ class HasAncestorMatcher : public MatcherInterface<T> {
   static_assert(IsBaseType<AncestorT>::value,
                 "has ancestor only accepts base type matcher");
 
-  const DynTypedMatcher AncestorMatcher;
+  DynTypedMatcher AncestorMatcher;
 
 public:
   explicit HasAncestorMatcher(const Matcher<AncestorT> &AncestorMatcher)
@@ -1660,7 +1711,7 @@ class ForEachDescendantMatcher : public MatcherInterface<T> {
   static_assert(IsBaseType<DescendantT>::value,
                 "for each descendant only accepts base type matcher");
 
-  const DynTypedMatcher DescendantMatcher;
+  DynTypedMatcher DescendantMatcher;
 
 public:
   explicit ForEachDescendantMatcher(
@@ -1693,7 +1744,7 @@ public:
   }
 
 private:
-  const ValueT ExpectedValue;
+  ValueT ExpectedValue;
 };
 
 /// Template specializations to easily write matchers for floating point
@@ -1726,7 +1777,7 @@ inline bool ValueEqualsMatcher<FloatingLiteral, llvm::APFloat>::matchesNode(
 /// \c Matcher<T> matches.
 template <typename TLoc, typename T>
 class LocMatcher : public MatcherInterface<TLoc> {
-  const DynTypedMatcher InnerMatcher;
+  DynTypedMatcher InnerMatcher;
 
 public:
   explicit LocMatcher(const Matcher<T> &InnerMatcher)
@@ -1750,7 +1801,7 @@ private:
 ///
 /// Used to implement the \c loc() matcher.
 class TypeLocTypeMatcher : public MatcherInterface<TypeLoc> {
-  const DynTypedMatcher InnerMatcher;
+  DynTypedMatcher InnerMatcher;
 
 public:
   explicit TypeLocTypeMatcher(const Matcher<QualType> &InnerMatcher)
@@ -1769,7 +1820,7 @@ public:
 /// another node of type \c T that can be reached using a given traverse
 /// function.
 template <typename T> class TypeTraverseMatcher : public MatcherInterface<T> {
-  const DynTypedMatcher InnerMatcher;
+  DynTypedMatcher InnerMatcher;
 
 public:
   explicit TypeTraverseMatcher(const Matcher<QualType> &InnerMatcher,
@@ -1794,7 +1845,7 @@ private:
 /// given traverse function.
 template <typename T>
 class TypeLocTraverseMatcher : public MatcherInterface<T> {
-  const DynTypedMatcher InnerMatcher;
+  DynTypedMatcher InnerMatcher;
 
 public:
   explicit TypeLocTraverseMatcher(const Matcher<TypeLoc> &InnerMatcher,
@@ -1849,7 +1900,7 @@ public:
   };
 
 private:
-  const Matcher<InnerTBase> InnerMatcher;
+  Matcher<InnerTBase> InnerMatcher;
 };
 
 /// A simple memoizer of T(*)() functions.
@@ -2193,7 +2244,7 @@ private:
     return BinaryOperator::getOpcodeStr(*optBinaryOpcode);
   }
 
-  const std::vector<std::string> Names;
+  std::vector<std::string> Names;
 };
 
 using HasOpNameMatcher =
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersMacros.h b/clang/include/clang/ASTMatchers/ASTMatchersMacros.h
index b7fe907f9414235164da0572cae8b66374487590..592a3898a2959151fb69d0e617057dbcaed369e5 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersMacros.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersMacros.h
@@ -143,7 +143,7 @@
                      *Builder) const override;                                 \
                                                                                \
   private:                                                                     \
-    ParamType const Param;                                                     \
+    ParamType Param;                                                           \
   };                                                                           \
   }                                                                            \
   inline ::clang::ast_matchers::internal::Matcher<Type> DefineMatcher(         \
@@ -151,7 +151,7 @@
     return ::clang::ast_matchers::internal::makeMatcher(                       \
         new internal::matcher_##DefineMatcher##OverloadId##Matcher(Param));    \
   }                                                                            \
-  typedef ::clang::ast_matchers::internal::Matcher<Type>(                      \
+  typedef ::clang::ast_matchers::internal::Matcher<Type> (                     \
       &DefineMatcher##_Type##OverloadId)(ParamType const &Param);              \
   inline bool internal::matcher_##DefineMatcher##OverloadId##Matcher::matches( \
       const Type &Node,                                                        \
@@ -192,8 +192,8 @@
                      *Builder) const override;                                 \
                                                                                \
   private:                                                                     \
-    ParamType1 const Param1;                                                   \
-    ParamType2 const Param2;                                                   \
+    ParamType1 Param1;                                                         \
+    ParamType2 Param2;                                                         \
   };                                                                           \
   }                                                                            \
   inline ::clang::ast_matchers::internal::Matcher<Type> DefineMatcher(         \
@@ -202,7 +202,7 @@
         new internal::matcher_##DefineMatcher##OverloadId##Matcher(Param1,     \
                                                                    Param2));   \
   }                                                                            \
-  typedef ::clang::ast_matchers::internal::Matcher<Type>(                      \
+  typedef ::clang::ast_matchers::internal::Matcher<Type> (                     \
       &DefineMatcher##_Type##OverloadId)(ParamType1 const &Param1,             \
                                          ParamType2 const &Param2);            \
   inline bool internal::matcher_##DefineMatcher##OverloadId##Matcher::matches( \
@@ -281,7 +281,7 @@
                      *Builder) const override;                                 \
                                                                                \
   private:                                                                     \
-    ParamType const Param;                                                     \
+    ParamType Param;                                                           \
   };                                                                           \
   }                                                                            \
   inline ::clang::ast_matchers::internal::PolymorphicMatcher<                  \
@@ -333,8 +333,8 @@
                      *Builder) const override;                                 \
                                                                                \
   private:                                                                     \
-    ParamType1 const Param1;                                                   \
-    ParamType2 const Param2;                                                   \
+    ParamType1 Param1;                                                         \
+    ParamType2 Param2;                                                         \
   };                                                                           \
   }                                                                            \
   inline ::clang::ast_matchers::internal::PolymorphicMatcher<                  \
@@ -469,7 +469,7 @@
                      *Builder) const override;                                 \
                                                                                \
   private:                                                                     \
-    std::shared_ptr<llvm::Regex> const Param;                                  \
+    std::shared_ptr<llvm::Regex> Param;                                        \
   };                                                                           \
   }                                                                            \
   inline ::clang::ast_matchers::internal::Matcher<Type> DefineMatcher(         \
@@ -521,7 +521,7 @@
                      *Builder) const override;                                 \
                                                                                \
   private:                                                                     \
-    std::shared_ptr<llvm::Regex> const Param;                                  \
+    std::shared_ptr<llvm::Regex> Param;                                        \
   };                                                                           \
   }                                                                            \
   inline ::clang::ast_matchers::internal::PolymorphicMatcher<                  \
diff --git a/clang/include/clang/Analysis/Analyses/CalledOnceCheck.h b/clang/include/clang/Analysis/Analyses/CalledOnceCheck.h
index fc574c680a444bc0428817a27c9a38d746d2715a..a0c767bf92d22b9afce9bdd56a1f616437ddc07c 100644
--- a/clang/include/clang/Analysis/Analyses/CalledOnceCheck.h
+++ b/clang/include/clang/Analysis/Analyses/CalledOnceCheck.h
@@ -17,6 +17,7 @@
 namespace clang {
 
 class AnalysisDeclContext;
+class BlockDecl;
 class CFG;
 class Decl;
 class DeclContext;
@@ -79,6 +80,7 @@ public:
   /// the path containing the call and not containing the call.  This helps us
   /// to pinpoint a bad path for the user.
   /// \param Parameter -- parameter that should be called once.
+  /// \param Function -- function declaration where the problem occured.
   /// \param Where -- the least common ancestor statement.
   /// \param Reason -- a reason describing the path without a call.
   /// \param IsCalledDirectly -- true, if parameter actually gets called on
@@ -86,9 +88,22 @@ public:
   /// collection, passed as a parameter, etc.).
   /// \param IsCompletionHandler -- true, if parameter is a completion handler.
   virtual void handleNeverCalled(const ParmVarDecl *Parameter,
-                                 const Stmt *Where, NeverCalledReason Reason,
+                                 const Decl *Function, const Stmt *Where,
+                                 NeverCalledReason Reason,
                                  bool IsCalledDirectly,
                                  bool IsCompletionHandler) {}
+
+  /// Called when the block is guaranteed to be called exactly once.
+  /// It means that we can be stricter with what we report on that block.
+  /// \param Block -- block declaration that is known to be called exactly once.
+  virtual void
+  handleBlockThatIsGuaranteedToBeCalledOnce(const BlockDecl *Block) {}
+
+  /// Called when the block has no guarantees about how many times it can get
+  /// called.
+  /// It means that we should be more lenient with reporting warnings in it.
+  /// \param Block -- block declaration in question.
+  virtual void handleBlockWithNoGuarantees(const BlockDecl *Block) {}
 };
 
 /// Check given CFG for 'called once' parameter violations.
diff --git a/clang/include/clang/Analysis/AnyCall.h b/clang/include/clang/Analysis/AnyCall.h
index 16371eb1da1835dba388780072eb4dd4dc1782e3..846ff7719ce152d2da97c65f9f6b396c6d91ad36 100644
--- a/clang/include/clang/Analysis/AnyCall.h
+++ b/clang/include/clang/Analysis/AnyCall.h
@@ -107,8 +107,8 @@ public:
 
   }
 
-  /// If {@code E} is a generic call (to ObjC method /function/block/etc),
-  /// return a constructed {@code AnyCall} object. Return None otherwise.
+  /// If @c E is a generic call (to ObjC method /function/block/etc),
+  /// return a constructed @c AnyCall object. Return None otherwise.
   static Optional<AnyCall> forExpr(const Expr *E) {
     if (const auto *ME = dyn_cast<ObjCMessageExpr>(E)) {
       return AnyCall(ME);
@@ -127,8 +127,8 @@ public:
     }
   }
 
-  /// If {@code D} is a callable (Objective-C method or a function), return
-  /// a constructed {@code AnyCall} object. Return None otherwise.
+  /// If @c D is a callable (Objective-C method or a function), return
+  /// a constructed @c AnyCall object. Return None otherwise.
   // FIXME: block support.
   static Optional<AnyCall> forDecl(const Decl *D) {
     if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
@@ -186,7 +186,7 @@ public:
   }
 
   /// \returns Function identifier if it is a named declaration,
-  /// {@code nullptr} otherwise.
+  /// @c nullptr otherwise.
   const IdentifierInfo *getIdentifier() const {
     if (const auto *ND = dyn_cast_or_null<NamedDecl>(D))
       return ND->getIdentifier();
diff --git a/clang/include/clang/Analysis/RetainSummaryManager.h b/clang/include/clang/Analysis/RetainSummaryManager.h
index 6acefb563d8ca45068d4397d9df309ce3f7390b3..b7ccb03178302392bfca1b94dccde65c183a8057 100644
--- a/clang/include/clang/Analysis/RetainSummaryManager.h
+++ b/clang/include/clang/Analysis/RetainSummaryManager.h
@@ -613,8 +613,8 @@ class RetainSummaryManager {
     const FunctionType *FT,
     bool &AllowAnnotations);
 
-  /// Apply the annotation of {@code pd} in function {@code FD}
-  /// to the resulting summary stored in out-parameter {@code Template}.
+  /// Apply the annotation of @c pd in function @c FD
+  /// to the resulting summary stored in out-parameter @c Template.
   /// \return whether an annotation was applied.
   bool applyParamAnnotationEffect(const ParmVarDecl *pd, unsigned parm_idx,
                                   const NamedDecl *FD,
@@ -715,8 +715,8 @@ private:
   /// Set argument types for arguments which are not doing anything.
   void updateSummaryForArgumentTypes(const AnyCall &C, const RetainSummary *&RS);
 
-  /// Determine whether a declaration {@code D} of correspondent type (return
-  /// type for functions/methods) {@code QT} has any of the given attributes,
+  /// Determine whether a declaration @c D of correspondent type (return
+  /// type for functions/methods) @c QT has any of the given attributes,
   /// provided they pass necessary validation checks AND tracking the given
   /// attribute is enabled.
   /// Returns the object kind corresponding to the present attribute, or None,
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 664eb566a70322ee283511cb44edf4210d7c87c6..c7b68856aab03ddf28c170511e77a251898f751b 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -730,7 +730,8 @@ def XRayLogArgs : InheritableAttr {
 
 def PatchableFunctionEntry
     : InheritableAttr,
-      TargetSpecificAttr<TargetArch<["aarch64", "aarch64_be", "x86", "x86_64"]>> {
+      TargetSpecificAttr<TargetArch<
+          ["aarch64", "aarch64_be", "riscv32", "riscv64", "x86", "x86_64"]>> {
   let Spellings = [GCC<"patchable_function_entry">];
   let Subjects = SubjectList<[Function, ObjCMethod]>;
   let Args = [UnsignedArgument<"Count">, DefaultIntArgument<"Offset", 0>];
@@ -1182,9 +1183,9 @@ def OpenCLKernel : InheritableAttr {
 
 def OpenCLUnrollHint : StmtAttr {
   let Spellings = [GNU<"opencl_unroll_hint">];
-//  let Subjects = SubjectList<[ForStmt, CXXForRangeStmt, WhileStmt, DoStmt],
-//                             ErrorDiag, "'for', 'while', and 'do' statements">;
-  let Args = [UnsignedArgument<"UnrollHint">];
+  let Subjects = SubjectList<[ForStmt, CXXForRangeStmt, WhileStmt, DoStmt],
+                             ErrorDiag, "'for', 'while', and 'do' statements">;
+  let Args = [UnsignedArgument<"UnrollHint", /*opt*/1>];
   let Documentation = [OpenCLUnrollHintDocs];
 }
 
@@ -1325,7 +1326,10 @@ def FallThrough : StmtAttr {
   let Spellings = [CXX11<"", "fallthrough", 201603>,
                    C2x<"", "fallthrough", 201904>,
                    CXX11<"clang", "fallthrough">, GCC<"fallthrough">];
-//  let Subjects = [NullStmt];
+  // The attribute only applies to a NullStmt, but we have special fix-it
+  // behavior if applied to a case label.
+  let Subjects = SubjectList<[NullStmt, SwitchCase], ErrorDiag,
+                             "empty statements">;
   let Documentation = [FallthroughDocs];
 }
 
@@ -1343,7 +1347,8 @@ def NoMerge : DeclOrStmtAttr {
   let Spellings = [Clang<"nomerge">];
   let Documentation = [NoMergeDocs];
   let InheritEvenIfAlreadyPresent = 1;
-  let Subjects = SubjectList<[Function], ErrorDiag, "functions and statements">;
+  let Subjects = SubjectList<[Function, Stmt], ErrorDiag,
+                             "functions and statements">;
   let SimpleHandler = 1;
 }
 
@@ -3466,6 +3471,7 @@ def LoopHint : Attr {
   }];
 
   let Documentation = [LoopHintDocs, UnrollHintDocs];
+  let HasCustomParsing = 1;
 }
 
 def CapturedRecord : InheritableAttr {
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 77d3bd1fdcd6eb2be593a6f913e9bd8efc3cb654..ed3d75b14f5e636a7d9da1cc0c8ce75388c476fe 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -3026,16 +3026,55 @@ It is only supported when using the Microsoft C++ ABI.
 def LifetimeBoundDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
-The ``lifetimebound`` attribute indicates that a resource owned by
-a function parameter or implicit object parameter
-is retained by the return value of the annotated function
-(or, for a parameter of a constructor, in the value of the constructed object).
-It is only supported in C++.
+The ``lifetimebound`` attribute on a function parameter or implicit object
+parameter indicates that objects that are referred to by that parameter may
+also be referred to by the return value of the annotated function (or, for a
+parameter of a constructor, by the value of the constructed object). It is only
+supported in C++.
+
+By default, a reference is considered to refer to its referenced object, a
+pointer is considered to refer to its pointee, a ``std::initializer_list<T>``
+is considered to refer to its underlying array, and aggregates (arrays and
+simple ``struct``s) are considered to refer to all objects that their
+transitive subobjects refer to.
+
+Clang warns if it is able to detect that an object or reference refers to
+another object with a shorter lifetime. For example, Clang will warn if a
+function returns a reference to a local variable, or if a reference is bound to
+a temporary object whose lifetime is not extended. By using the
+``lifetimebound`` attribute, this determination can be extended to look through
+user-declared functions. For example:
 
-This attribute provides an experimental implementation of the facility
-described in the C++ committee paper `P0936R0 <http://wg21.link/p0936r0>`_,
-and is subject to change as the design of the corresponding functionality
-changes.
+.. code-block:: c++
+
+    // Returns m[key] if key is present, or default_value if not.
+    template<typename T, typename U>
+    const U &get_or_default(const std::map<T, U> &m [[clang::lifetimebound]],
+                            const T &key, /* note, not lifetimebound */
+                            const U &default_value [[clang::lifetimebound]]);
+
+    std::map<std::string, std::string> m;
+    // warning: temporary "bar"s that might be bound to local reference 'val'
+    // will be destroyed at the end of the full-expression
+    const std::string &val = get_or_default(m, "foo"s, "bar"s);
+
+    // No warning in this case.
+    std::string def_val = "bar"s;
+    const std::string &val = get_or_default(m, "foo"s, def_val);
+
+The attribute can be applied to the implicit ``this`` parameter of a member
+function by writing the attribute after the function type:
+
+.. code-block:: c++
+
+    struct string {
+      // The returned pointer should not outlive ``*this``.
+      const char *data() const [[clang::lifetimebound]];
+    };
+
+This attribute is inspired by the C++ committee paper `P0936R0
+<http://wg21.link/p0936r0>`_, but does not affect whether temporary objects
+have their lifetimes extended.
   }];
 }
 
@@ -4810,6 +4849,9 @@ def PatchableFunctionEntryDocs : Documentation {
 before the function entry and N-M NOPs after the function entry. This attribute
 takes precedence over the command line option ``-fpatchable-function-entry=N,M``.
 ``M`` defaults to 0 if omitted.
+
+This attribute is only supported on
+aarch64/aarch64-be/riscv32/riscv64/i386/x86-64 targets.
 }];
 }
 
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index e39b37c31ad4a69779662f2d18aec196568e5742..3a531b7a7ffbe6bca100b0756387c5bae7135aa2 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -1640,6 +1640,9 @@ BUILTIN(__builtin_os_log_format, "v*v*cC*.", "p:0:nt")
 // OpenMP 4.0
 LANGBUILTIN(omp_is_initial_device, "i", "nc", OMP_LANG)
 
+// CUDA/HIP
+LANGBUILTIN(__builtin_get_device_side_mangled_name, "cC*.", "ncT", CUDA_LANG)
+
 // Builtins for XRay
 BUILTIN(__xray_customevent, "vcC*z", "")
 BUILTIN(__xray_typedevent, "vzcC*z", "")
diff --git a/clang/include/clang/Basic/Builtins.h b/clang/include/clang/Basic/Builtins.h
index 15bfcf797917c8f221f18a646013665f133a0996..efd6cb897293e473f973966dda3ae32e00c70771 100644
--- a/clang/include/clang/Basic/Builtins.h
+++ b/clang/include/clang/Basic/Builtins.h
@@ -36,6 +36,7 @@ enum LanguageID {
   OCLC20_LANG = 0x20, // builtin for OpenCL C 2.0 only.
   OCLC1X_LANG = 0x40, // builtin for OpenCL C 1.x only.
   OMP_LANG = 0x80,    // builtin requires OpenMP.
+  CUDA_LANG = 0x100,  // builtin requires CUDA.
   ALL_LANGUAGES = C_LANG | CXX_LANG | OBJC_LANG, // builtin for all languages.
   ALL_GNU_LANGUAGES = ALL_LANGUAGES | GNU_LANG,  // builtin requires GNU mode.
   ALL_MS_LANGUAGES = ALL_LANGUAGES | MS_LANG,    // builtin requires MS mode.
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 415d8cb3e73ac24b64a3244c9bef9c31e657c3bd..9677b1aadb51801455ddd2ebae775ec192b3d1ad 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -193,13 +193,13 @@ TARGET_BUILTIN(__builtin_amdgcn_fmed3h, "hhhh", "nc", "gfx9-insts")
 // Deep learning builtins.
 //===----------------------------------------------------------------------===//
 
-TARGET_BUILTIN(__builtin_amdgcn_fdot2, "fV2hV2hfIb", "nc", "dot2-insts")
+TARGET_BUILTIN(__builtin_amdgcn_fdot2, "fV2hV2hfIb", "nc", "dot7-insts")
 TARGET_BUILTIN(__builtin_amdgcn_sdot2, "SiV2SsV2SsSiIb", "nc", "dot2-insts")
 TARGET_BUILTIN(__builtin_amdgcn_udot2, "UiV2UsV2UsUiIb", "nc", "dot2-insts")
 TARGET_BUILTIN(__builtin_amdgcn_sdot4, "SiSiSiSiIb", "nc", "dot1-insts")
-TARGET_BUILTIN(__builtin_amdgcn_udot4, "UiUiUiUiIb", "nc", "dot2-insts")
+TARGET_BUILTIN(__builtin_amdgcn_udot4, "UiUiUiUiIb", "nc", "dot7-insts")
 TARGET_BUILTIN(__builtin_amdgcn_sdot8, "SiSiSiSiIb", "nc", "dot1-insts")
-TARGET_BUILTIN(__builtin_amdgcn_udot8, "UiUiUiUiIb", "nc", "dot2-insts")
+TARGET_BUILTIN(__builtin_amdgcn_udot8, "UiUiUiUiIb", "nc", "dot7-insts")
 
 //===----------------------------------------------------------------------===//
 // GFX10+ only builtins.
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index 39c66f5daeb16b64336478292a29802124ae844c..66c35a9a82bedb52be470c8dce2b1acadd067a0a 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -39,6 +39,7 @@ BUILTIN(__builtin_altivec_vadduws, "V4UiV4UiV4Ui", "")
 BUILTIN(__builtin_altivec_vaddeuqm, "V1ULLLiV1ULLLiV1ULLLiV1ULLLi","")
 BUILTIN(__builtin_altivec_vaddcuq, "V1ULLLiV1ULLLiV1ULLLi","")
 BUILTIN(__builtin_altivec_vaddecuq, "V1ULLLiV1ULLLiV1ULLLiV1ULLLi","")
+BUILTIN(__builtin_altivec_vadduqm, "V1ULLLiV16UcV16Uc","")
 
 BUILTIN(__builtin_altivec_vsubsbs, "V16ScV16ScV16Sc", "")
 BUILTIN(__builtin_altivec_vsububs, "V16UcV16UcV16Uc", "")
@@ -49,6 +50,7 @@ BUILTIN(__builtin_altivec_vsubuws, "V4UiV4UiV4Ui", "")
 BUILTIN(__builtin_altivec_vsubeuqm, "V1ULLLiV1ULLLiV1ULLLiV1ULLLi","")
 BUILTIN(__builtin_altivec_vsubcuq, "V1ULLLiV1ULLLiV1ULLLi","")
 BUILTIN(__builtin_altivec_vsubecuq, "V1ULLLiV1ULLLiV1ULLLiV1ULLLi","")
+BUILTIN(__builtin_altivec_vsubuqm, "V1ULLLiV16UcV16Uc","")
 
 BUILTIN(__builtin_altivec_vavgsb, "V16ScV16ScV16Sc", "")
 BUILTIN(__builtin_altivec_vavgub, "V16UcV16UcV16Uc", "")
diff --git a/clang/include/clang/Basic/BuiltinsRISCV.def b/clang/include/clang/Basic/BuiltinsRISCV.def
index e76c853787c9c9af352832bab5af1293fcf0b9e9..c91b3d1b1f5c1cadff44ab4e9f538c833a45a9f6 100644
--- a/clang/include/clang/Basic/BuiltinsRISCV.def
+++ b/clang/include/clang/Basic/BuiltinsRISCV.def
@@ -17,3 +17,5 @@
 
 #include "clang/Basic/riscv_vector_builtins.inc"
 
+#undef BUILTIN
+#undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsWebAssembly.def b/clang/include/clang/Basic/BuiltinsWebAssembly.def
index 84f346bcb928a23a056c3f481c620b3c56fb73fb..6ea59026cd02a6c29008e792c9ce45b441ab7b44 100644
--- a/clang/include/clang/Basic/BuiltinsWebAssembly.def
+++ b/clang/include/clang/Basic/BuiltinsWebAssembly.def
@@ -84,15 +84,15 @@ TARGET_BUILTIN(__builtin_wasm_replace_lane_i64x2, "V2LLiV2LLiIiLLi", "nc", "simd
 TARGET_BUILTIN(__builtin_wasm_replace_lane_f32x4, "V4fV4fIif", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_replace_lane_f64x2, "V2dV2dIid", "nc", "simd128")
 
-TARGET_BUILTIN(__builtin_wasm_add_saturate_s_i8x16, "V16ScV16ScV16Sc", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_add_saturate_u_i8x16, "V16UcV16UcV16Uc", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_add_saturate_s_i16x8, "V8sV8sV8s", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_add_saturate_u_i16x8, "V8UsV8UsV8Us", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_add_sat_s_i8x16, "V16ScV16ScV16Sc", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_add_sat_u_i8x16, "V16UcV16UcV16Uc", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_add_sat_s_i16x8, "V8sV8sV8s", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_add_sat_u_i16x8, "V8UsV8UsV8Us", "nc", "simd128")
 
-TARGET_BUILTIN(__builtin_wasm_sub_saturate_s_i8x16, "V16ScV16ScV16Sc", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_sub_saturate_u_i8x16, "V16UcV16UcV16Uc", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_sub_saturate_s_i16x8, "V8sV8sV8s", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_sub_saturate_u_i16x8, "V8UsV8UsV8Us", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_sub_sat_s_i8x16, "V16ScV16ScV16Sc", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_sub_sat_u_i8x16, "V16UcV16UcV16Uc", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_sub_sat_s_i16x8, "V8sV8sV8s", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_sub_sat_u_i16x8, "V8UsV8UsV8Us", "nc", "simd128")
 
 TARGET_BUILTIN(__builtin_wasm_abs_i8x16, "V16ScV16Sc", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_abs_i16x8, "V8sV8s", "nc", "simd128")
@@ -116,7 +116,7 @@ TARGET_BUILTIN(__builtin_wasm_avgr_u_i16x8, "V8UsV8UsV8Us", "nc", "simd128")
 
 TARGET_BUILTIN(__builtin_wasm_popcnt_i8x16, "V16ScV16Sc", "nc", "simd128")
 
-TARGET_BUILTIN(__builtin_wasm_q15mulr_saturate_s_i16x8, "V8sV8sV8s", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_q15mulr_sat_s_i16x8, "V8sV8sV8s", "nc", "simd128")
 
 TARGET_BUILTIN(__builtin_wasm_extmul_low_i8x16_s_i16x8, "V8sV16ScV16Sc", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_extmul_high_i8x16_s_i16x8, "V8sV16ScV16Sc", "nc", "simd128")
@@ -141,21 +141,16 @@ TARGET_BUILTIN(__builtin_wasm_extadd_pairwise_i16x8_u_i32x4, "V4UiV8Us", "nc", "
 
 TARGET_BUILTIN(__builtin_wasm_bitselect, "V4iV4iV4iV4i", "nc", "simd128")
 
-TARGET_BUILTIN(__builtin_wasm_signselect_i8x16, "V16ScV16ScV16ScV16Sc", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_signselect_i16x8, "V8sV8sV8sV8s", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_signselect_i32x4, "V4iV4iV4iV4i", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_signselect_i64x2, "V2LLiV2LLiV2LLiV2LLi", "nc", "simd128")
-
 TARGET_BUILTIN(__builtin_wasm_shuffle_v8x16, "V16ScV16ScV16ScIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIi", "nc", "simd128")
 
 TARGET_BUILTIN(__builtin_wasm_any_true_i8x16, "iV16Sc", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_any_true_i16x8, "iV8s", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_any_true_i32x4, "iV4i", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_any_true_i64x2, "iV2LLi", "nc", "unimplemented-simd128")
+TARGET_BUILTIN(__builtin_wasm_any_true_i64x2, "iV2LLi", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_all_true_i8x16, "iV16Sc", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_all_true_i16x8, "iV8s", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_all_true_i32x4, "iV4i", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_all_true_i64x2, "iV2LLi", "nc", "unimplemented-simd128")
+TARGET_BUILTIN(__builtin_wasm_all_true_i64x2, "iV2LLi", "nc", "simd128")
 
 TARGET_BUILTIN(__builtin_wasm_bitmask_i8x16, "iV16Sc", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_bitmask_i16x8, "iV8s", "nc", "simd128")
@@ -188,11 +183,6 @@ TARGET_BUILTIN(__builtin_wasm_dot_s_i32x4_i16x8, "V4iV8sV8s", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_sqrt_f32x4, "V4fV4f", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_sqrt_f64x2, "V2dV2d", "nc", "simd128")
 
-TARGET_BUILTIN(__builtin_wasm_qfma_f32x4, "V4fV4fV4fV4f", "nc", "unimplemented-simd128")
-TARGET_BUILTIN(__builtin_wasm_qfms_f32x4, "V4fV4fV4fV4f", "nc", "unimplemented-simd128")
-TARGET_BUILTIN(__builtin_wasm_qfma_f64x2, "V2dV2dV2dV2d", "nc", "unimplemented-simd128")
-TARGET_BUILTIN(__builtin_wasm_qfms_f64x2, "V2dV2dV2dV2d", "nc", "unimplemented-simd128")
-
 TARGET_BUILTIN(__builtin_wasm_trunc_saturate_s_i32x4_f32x4, "V4iV4f", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_trunc_saturate_u_i32x4_f32x4, "V4iV4f", "nc", "simd128")
 
@@ -201,18 +191,15 @@ TARGET_BUILTIN(__builtin_wasm_narrow_u_i8x16_i16x8, "V16UcV8UsV8Us", "nc", "simd
 TARGET_BUILTIN(__builtin_wasm_narrow_s_i16x8_i32x4, "V8sV4iV4i", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_narrow_u_i16x8_i32x4, "V8UsV4UiV4Ui", "nc", "simd128")
 
-TARGET_BUILTIN(__builtin_wasm_widen_low_s_i32x4_i64x2, "V2LLiV4i", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_high_s_i32x4_i64x2, "V2LLiV4i", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_low_u_i32x4_i64x2, "V2LLUiV4Ui", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_high_u_i32x4_i64x2, "V2LLUiV4Ui", "nc", "simd128")
-
-TARGET_BUILTIN(__builtin_wasm_widen_s_i8x16_i32x4, "V4iV16ScIi", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_widen_u_i8x16_i32x4, "V4UiV16UcIi", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_extend_low_s_i32x4_i64x2, "V2LLiV4i", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_extend_high_s_i32x4_i64x2, "V2LLiV4i", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_extend_low_u_i32x4_i64x2, "V2LLUiV4Ui", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_extend_high_u_i32x4_i64x2, "V2LLUiV4Ui", "nc", "simd128")
 
 TARGET_BUILTIN(__builtin_wasm_convert_low_s_i32x4_f64x2, "V2dV4i", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_convert_low_u_i32x4_f64x2, "V2dV4Ui", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_trunc_saturate_zero_s_f64x2_i32x4, "V4iV2d", "nc", "simd128")
-TARGET_BUILTIN(__builtin_wasm_trunc_saturate_zero_u_f64x2_i32x4, "V4UiV2d", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_trunc_sat_zero_s_f64x2_i32x4, "V4iV2d", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_trunc_sat_zero_u_f64x2_i32x4, "V4UiV2d", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_demote_zero_f64x2_f32x4, "V4fV2d", "nc", "simd128")
 TARGET_BUILTIN(__builtin_wasm_promote_low_f32x4_f64x2, "V2dV4f", "nc", "simd128")
 
@@ -230,8 +217,5 @@ TARGET_BUILTIN(__builtin_wasm_store64_lane, "vLLi*V2LLiIi", "n", "simd128")
 
 TARGET_BUILTIN(__builtin_wasm_eq_i64x2, "V2LLiV2LLiV2LLi", "nc", "simd128")
 
-TARGET_BUILTIN(__builtin_wasm_prefetch_t, "vv*", "n", "simd128")
-TARGET_BUILTIN(__builtin_wasm_prefetch_nt, "vv*", "n", "simd128")
-
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index bbda74044a1c17e2a04c498aec7fd032d8fd8b6c..4c354734dff872833842151565a9cb6d7b256515 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -70,6 +70,10 @@ CODEGENOPT(DebugPassManager, 1, 0) ///< Prints debug information for the new
 CODEGENOPT(DisableRedZone    , 1, 0) ///< Set when -mno-red-zone is enabled.
 CODEGENOPT(EmitCallSiteInfo, 1, 0) ///< Emit call site info only in the case of
                                    ///< '-g' + 'O>0' level.
+CODEGENOPT(EnableDIPreservationVerify, 1, 0) ///< Enable di preservation verify
+                                             ///< each (it means check
+                                             ///< the original debug info
+                                             ///< metadata preservation).
 CODEGENOPT(IndirectTlsSegRefs, 1, 0) ///< Set when -mno-tls-direct-seg-refs
                                      ///< is specified.
 CODEGENOPT(DisableTailCalls  , 1, 0) ///< Do not emit tail calls.
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index b38df2da97de12a47d3fea0dcce5d5f90eb9cb36..778340b34272563dc7569db7e2cea7b26eb4cecf 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -190,6 +190,10 @@ public:
   /// The ABI to use for passing floating point arguments.
   std::string FloatABI;
 
+  /// The file to use for dumping bug report by `Debugify` for original
+  /// debug info.
+  std::string DIBugsReportFilePath;
+
   /// The floating-point denormal mode to use.
   llvm::DenormalMode FPDenormalMode = llvm::DenormalMode::getIEEE();
 
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 50f56bc79644ff817c46f63c076a914f4a87da9e..03f7036b82355ea33f627eed9244e31d44e01d54 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -344,6 +344,10 @@ def warn_drv_disabling_vptr_no_rtti_default : Warning<
 def warn_drv_object_size_disabled_O0 : Warning<
   "the object size sanitizer has no effect at -O0, but is explicitly enabled: %0">,
   InGroup<InvalidCommandLineArgument>, DefaultWarnNoWerror;
+def warn_ignoring_verify_debuginfo_preserve_export : Warning<
+  "ignoring -fverify-debuginfo-preserve-export=%0 because "
+  "-fverify-debuginfo-preserve wasn't enabled">,
+  InGroup<UnusedCommandLineArgument>;
 def err_invalid_branch_protection: Error <
   "invalid branch protection option '%0' in '%1'">;
 def err_invalid_sls_hardening : Error<
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 34876e9945b5b0354a8d2e89847ae6eb4bd98fd3..85f798013a3d41741308ed0d37d649b19cfae32a 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -253,22 +253,30 @@ def : DiagGroup<"c++1z-compat-mangling", [CXX17CompatMangling]>;
 // Name of this warning in GCC.
 def NoexceptType : DiagGroup<"noexcept-type", [CXX17CompatMangling]>;
 
-// Warnings for C2x code which is not compatible with prior C standards.
+// Warnings for C code which is not compatible with previous C standards.
 def CPre2xCompat : DiagGroup<"pre-c2x-compat">;
 def CPre2xCompatPedantic : DiagGroup<"pre-c2x-compat-pedantic",
                                      [CPre2xCompat]>;
 
-// Warnings for C++1y code which is not compatible with prior C++ standards.
-def CXXPre14Compat : DiagGroup<"c++98-c++11-compat">;
-def CXXPre14CompatPedantic : DiagGroup<"c++98-c++11-compat-pedantic",
+// Warnings for C++ code which is not compatible with previous C++ standards.
+def CXXPre14Compat : DiagGroup<"pre-c++14-compat">;
+def : DiagGroup<"c++98-c++11-compat", [CXXPre14Compat]>;
+def CXXPre14CompatPedantic : DiagGroup<"pre-c++14-compat-pedantic",
                                        [CXXPre14Compat,
                                         CXXPre14CompatBinaryLiteral]>;
-def CXXPre17Compat : DiagGroup<"c++98-c++11-c++14-compat">;
-def CXXPre17CompatPedantic : DiagGroup<"c++98-c++11-c++14-compat-pedantic",
+def : DiagGroup<"c++98-c++11-compat-pedantic", [CXXPre14CompatPedantic]>;
+def CXXPre17Compat : DiagGroup<"pre-c++17-compat">;
+def : DiagGroup<"c++98-c++11-c++14-compat", [CXXPre17Compat]>;
+def CXXPre17CompatPedantic : DiagGroup<"pre-c++17-compat-pedantic",
                                        [CXXPre17Compat]>;
-def CXXPre20Compat : DiagGroup<"c++98-c++11-c++14-c++17-compat">;
-def CXXPre20CompatPedantic : DiagGroup<"c++98-c++11-c++14-c++17-compat-pedantic",
+def : DiagGroup<"c++98-c++11-c++14-compat-pedantic",
+                [CXXPre17CompatPedantic]>;
+def CXXPre20Compat : DiagGroup<"pre-c++20-compat">;
+def : DiagGroup<"c++98-c++11-c++14-c++17-compat", [CXXPre20Compat]>;
+def CXXPre20CompatPedantic : DiagGroup<"pre-c++20-compat-pedantic",
                                        [CXXPre20Compat]>;
+def : DiagGroup<"c++98-c++11-c++14-c++17-compat-pedantic",
+                [CXXPre20CompatPedantic]>;
 def CXXPre2bCompat : DiagGroup<"pre-c++2b-compat">;
 def CXXPre2bCompatPedantic :
   DiagGroup<"pre-c++2b-compat-pedantic", [CXXPre2bCompat]>;
@@ -491,6 +499,7 @@ def PrivateExtern : DiagGroup<"private-extern">;
 def SelTypeCast : DiagGroup<"cast-of-sel-type">;
 def FunctionDefInObjCContainer : DiagGroup<"function-def-in-objc-container">;
 def BadFunctionCast : DiagGroup<"bad-function-cast">;
+def CastFunctionType : DiagGroup<"cast-function-type">;
 def ObjCPropertyImpl : DiagGroup<"objc-property-implementation">;
 def ObjCPropertyNoAttribute : DiagGroup<"objc-property-no-attribute">;
 def ObjCPropertyAssignOnObjectType : DiagGroup<"objc-property-assign-on-object-type">;
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 4608905f1b2661edf46891696280396488367fbe..b0f9b317a020d6333db0f94ef1d94d38ef3984d9 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -949,9 +949,6 @@ def err_expected_lambda_body : Error<"expected body of lambda expression">;
 def warn_cxx98_compat_lambda : Warning<
   "lambda expressions are incompatible with C++98">,
   InGroup<CXX98Compat>, DefaultIgnore;
-def err_lambda_missing_parens : Error<
-  "lambda requires '()' before %select{'mutable'|return type|"
-  "attribute specifier|'constexpr'|'consteval'|'requires' clause}0">;
 def err_lambda_decl_specifier_repeated : Error<
   "%select{'mutable'|'constexpr'|'consteval'}0 cannot appear multiple times in a lambda declarator">;
 def err_lambda_capture_misplaced_ellipsis : Error<
@@ -964,6 +961,9 @@ def err_capture_default_first : Error<
 def ext_decl_attrs_on_lambda : ExtWarn<
   "an attribute specifier sequence in this position is a C++2b extension">,
   InGroup<CXX2b>;
+def ext_lambda_missing_parens : ExtWarn<
+  "lambda without a parameter clause is a C++2b extension">,
+  InGroup<CXX2b>;
 def warn_cxx20_compat_decl_attrs_on_lambda : Warning<
   "an attribute specifier sequence in this position is incompatible with C++ "
   "standards before C++2b">, InGroup<CXXPre2bCompat>, DefaultIgnore;
@@ -1400,6 +1400,13 @@ def err_omp_variant_ctx_second_match_extension : Error<
   "only a single match extension allowed per OpenMP context selector">;
 def err_omp_invalid_dsa: Error<
   "data-sharing attribute '%0' in '%1' clause requires OpenMP version %2 or above">;
+def err_omp_expected_punc_after_interop_mod : Error<
+  "expected ',' after interop modifier">;
+def err_omp_expected_interop_type : Error<
+  "expected interop type: 'target' and/or 'targetsync'">;
+def warn_omp_more_one_interop_type
+  : Warning<"interop type '%0' cannot be specified more than once">,
+    InGroup<OpenMPClauses>;
 
 // Pragma loop support.
 def err_pragma_loop_missing_argument : Error<
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index cb15ab3dc5b7d3c885e4993cd49bc0f5c56889f2..e653a223a9c66528872d7dbeeebb81bc0c429c29 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -5776,6 +5776,9 @@ def note_goto_ms_asm_label : Note<
 def warn_unused_label : Warning<"unused label %0">,
   InGroup<UnusedLabel>, DefaultIgnore;
 
+def err_continue_from_cond_var_init : Error<
+  "cannot jump from this continue statement to the loop increment; "
+  "jump bypasses initialization of loop condition variable">;
 def err_goto_into_protected_scope : Error<
   "cannot jump from this goto statement to its label">;
 def ext_goto_into_protected_scope : ExtWarn<
@@ -8300,6 +8303,9 @@ def note_cuda_device_builtin_surftex_should_be_template_class : Note<
     "%0 needs to be instantiated from a class template with proper "
     "template arguments">;
 
+def err_hip_invalid_args_builtin_mangled_name : Error<
+    "invalid argument: symbol must be a device-side function or global variable">;
+
 def warn_non_pod_vararg_with_format_string : Warning<
   "cannot pass %select{non-POD|non-trivial}0 object of type %1 to variadic "
   "%select{function|block|method|constructor}2; expected type from format "
@@ -8383,6 +8389,9 @@ def note_change_calling_conv_fixit : Note<
 def warn_bad_function_cast : Warning<
   "cast from function call of type %0 to non-matching type %1">,
   InGroup<BadFunctionCast>, DefaultIgnore;
+def warn_cast_function_type : Warning<
+  "cast from %0 to %1 converts to incompatible function types">,
+  InGroup<CastFunctionType>, DefaultIgnore;
 def err_cast_pointer_to_non_pointer_int : Error<
   "pointer cannot be cast to type %0">;
 def err_cast_to_bfloat16 : Error<"cannot type-cast to __bf16">;
@@ -8603,6 +8612,15 @@ def warn_initializer_out_of_order : Warning<
   "%select{field|base class}0 %1 will be initialized after "
   "%select{field|base}2 %3">,
   InGroup<ReorderCtor>, DefaultIgnore;
+
+def warn_some_initializers_out_of_order : Warning<
+  "initializer order does not match the declaration order">,
+  InGroup<ReorderCtor>, DefaultIgnore;
+
+def note_initializer_out_of_order : Note<
+  "%select{field|base class}0 %1 will be initialized after "
+  "%select{field|base}2 %3">;
+
 def warn_abstract_vbase_init_ignored : Warning<
   "initializer for virtual base class %0 of abstract class %1 "
   "will never be used">,
@@ -10587,6 +10605,17 @@ def note_omp_protected_structured_block
     : Note<"jump bypasses OpenMP structured block">;
 def note_omp_exits_structured_block
     : Note<"jump exits scope of OpenMP structured block">;
+def err_omp_interop_variable_expected : Error<
+  "expected%select{| non-const}0 variable of type 'omp_interop_t'">;
+def err_omp_interop_variable_wrong_type : Error<
+  "interop variable must be of type 'omp_interop_t'">;
+def err_omp_interop_prefer_type : Error<
+  "prefer_list item must be a string literal or constant integral "
+  "expression">;
+def err_omp_interop_bad_depend_clause : Error<
+  "'depend' clause requires the 'targetsync' interop type">;
+def err_omp_interop_var_multiple_actions : Error<
+  "interop variable %0 used in multiple action clauses">;
 } // end of OpenMP category
 
 let CategoryName = "Related Result Type Issue" in {
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 1a2358ed83a8be1a94bd54aa0f7f4729e06d4f29..3f34ec3cb1c36d2af43742d44873d05306c06cbd 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -250,8 +250,8 @@ LANGOPT(GPUMaxThreadsPerBlock, 32, 1024, "default max threads per block for kern
 LANGOPT(GPUDeferDiag, 1, 0, "defer host/device related diagnostic messages for CUDA/HIP")
 LANGOPT(GPUExcludeWrongSideOverloads, 1, 0, "always exclude wrong side overloads in overloading resolution for CUDA/HIP")
 
-LANGOPT(SYCL              , 1, 0, "SYCL")
 LANGOPT(SYCLIsDevice      , 1, 0, "Generate code for SYCL device")
+LANGOPT(SYCLIsHost        , 1, 0, "SYCL host compilation")
 ENUM_LANGOPT(SYCLVersion  , SYCLMajorVersion, 1, SYCL_None, "Version of the SYCL standard used")
 
 LANGOPT(HIPUseNewLaunchAPI, 1, 0, "Use new kernel launching API for HIP")
diff --git a/clang/include/clang/Basic/LangStandard.h b/clang/include/clang/Basic/LangStandard.h
index f82ce05a6369282683e9f61c7a572f52616f0895..b0785409628c420e4c07ee01167badba6f82d7ba 100644
--- a/clang/include/clang/Basic/LangStandard.h
+++ b/clang/include/clang/Basic/LangStandard.h
@@ -32,6 +32,7 @@ enum class Language : uint8_t {
   ObjC,
   ObjCXX,
   OpenCL,
+  OpenCLCXX,
   CUDA,
   RenderScript,
   HIP,
diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h
index 82ea1f462949a8fc27ab1b944f8cbb798a6055ba..16f34d11398afe4e5bbf7f6e3e58263019a8d2c0 100644
--- a/clang/include/clang/Basic/Module.h
+++ b/clang/include/clang/Basic/Module.h
@@ -637,7 +637,7 @@ public:
   }
 
   /// Print the module map for this module to the given stream.
-  void print(raw_ostream &OS, unsigned Indent = 0) const;
+  void print(raw_ostream &OS, unsigned Indent = 0, bool Dump = false) const;
 
   /// Dump the contents of this module to the given output stream.
   void dump() const;
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index 66df4d363e0eadbfa2cdcde68fc865d1d0b4536a..4aac8a32ef976707d11fd05cd04f194a50226834 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -275,3 +275,4 @@ def OMPTargetTeamsDistributeDirective : StmtNode<OMPLoopDirective>;
 def OMPTargetTeamsDistributeParallelForDirective : StmtNode<OMPLoopDirective>;
 def OMPTargetTeamsDistributeParallelForSimdDirective : StmtNode<OMPLoopDirective>;
 def OMPTargetTeamsDistributeSimdDirective : StmtNode<OMPLoopDirective>;
+def OMPInteropDirective : StmtNode<OMPExecutableDirective>;
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 9791cb6bbee703909674b4eed449e35f8f009ff4..01ca9ca0d2a821b9f6c48c18075ed76f863a24c2 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -155,6 +155,10 @@ protected:
   /// zero-length bitfield.
   unsigned UseZeroLengthBitfieldAlignment : 1;
 
+  /// Whether zero length bitfield alignment is respected if they are the
+  /// leading members.
+  unsigned UseLeadingZeroLengthBitfield : 1;
+
   ///  Whether explicit bit field alignment attributes are honored.
   unsigned UseExplicitBitFieldAlignment : 1;
 
@@ -768,6 +772,12 @@ public:
     return UseZeroLengthBitfieldAlignment;
   }
 
+  /// Check whether zero length bitfield alignment is respected if they are
+  /// leading members.
+  bool useLeadingZeroLengthBitfield() const {
+    return UseLeadingZeroLengthBitfield;
+  }
+
   /// Get the fixed alignment value in bits for a member that follows
   /// a zero length bitfield.
   unsigned getZeroLengthBitfieldBoundary() const {
diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td
index c69b5be1798c906fd8497930689a2d6368def3e1..2932ee1ab0f2379b387fd7e20e781a1efa5e692d 100644
--- a/clang/include/clang/Basic/riscv_vector.td
+++ b/clang/include/clang/Basic/riscv_vector.td
@@ -84,6 +84,13 @@
 //      elements of the same width
 //   S: given a vector type, computes its equivalent one for LMUL=1. This is a
 //      no-op if the vector was already LMUL=1
+//   (Log2EEW:Value): Log2EEW value could be 3/4/5/6 (8/16/32/64), given a
+//      vector type (SEW and LMUL) and EEW (8/16/32/64), computes its
+//      equivalent integer vector type with EEW and corresponding ELMUL (elmul =
+//      (eew/sew) * lmul). For example, vector type is __rvv_float16m4
+//      (SEW=16, LMUL=4) and Log2EEW is 3 (EEW=8), and then equivalent vector
+//      type is __rvv_uint8m2_t (elmul=(8/16)*4 = 2). Ignore to define a new
+//      builtins if its equivalent type has illegal lmul.
 //
 // Following with the example above, if t is "i", then "Ue" will yield unsigned
 // int and "Fv" will yield __rvv_float32m1_t (again assuming LMUL=1), Fw would
@@ -150,8 +157,9 @@ class RVVBuiltin<string suffix, string prototype, string type_range,
   // This builtin has a granted vector length parameter in the last position.
   bit HasVL = true;
 
-  // This builtin supports function overloading and has a mangled name.
-  bit HasGeneric = true;
+  // This builtin supports non-masked function overloading api.
+  // All masked operations support overloading api.
+  bit HasNoMaskedOverloaded = true;
 
   // Reads or writes "memory" or has other side-effects.
   bit HasSideEffects = false;
@@ -159,18 +167,33 @@ class RVVBuiltin<string suffix, string prototype, string type_range,
   // This builtin is valid for the given Log2LMULs.
   list<int> Log2LMUL = [0, 1, 2, 3, -1, -2, -3];
 
-  // Emit the automatic clang codegen. It describes what types we have to use
+  // Manual code in clang codegen riscv_vector_builtin_cg.inc
+  code ManualCodegen = [{}];
+  code ManualCodegenMask = [{}];
+
+  // When emit the automatic clang codegen, it describes what types we have to use
   // to obtain the specific LLVM intrinsic. -1 means the return type, otherwise,
   // k >= 0 meaning the k-th operand (counting from zero) of the codegen'd
   // parameter of the unmasked version. k can't be the mask operand's position.
   list<int> IntrinsicTypes = [];
 
+  // When the order of the parameters of clang builtin do not match the order of
+  // C/C++ api, we use permutation index to mapping the operand from clang
+  // builtin to C/C++. It is parameter of the unmasked version without VL
+  // operand. If empty, the default permutation is [0, 1, 2, ...].
+  list<int> PermuteOperands = [];
+
   // If these names are not empty, this is the ID of the LLVM intrinsic
   // we want to lower to.
   string IRName = NAME;
 
   // If HasMask, this is the ID of the LLVM intrinsic we want to lower to.
   string IRNameMask = NAME #"_mask";
+
+  // If non empty, this is the code emitted in the header, otherwise
+  // an automatic definition in header is emitted.
+  string HeaderCode = "";
+
 }
 
 //===----------------------------------------------------------------------===//
@@ -195,6 +218,173 @@ multiclass RVVBinBuiltinSet<string intrinsic_name, string type_range,
   }
 }
 
+defvar TypeList = ["c","s","i","l","f","d"];
+defvar EEWList = [["8", "(Log2EEW:3)"],
+                  ["16", "(Log2EEW:4)"],
+                  ["32", "(Log2EEW:5)"],
+                  ["64", "(Log2EEW:6)"]];
+
+class IsFloat<string type> {
+  bit val = !or(!eq(type, "h"), !eq(type, "f"), !eq(type, "d"));
+}
+
+multiclass RVVVLEBuiltin<list<string> types> {
+  let Name = NAME # "_v",
+      IRName = "vle",
+      IRNameMask ="vle_mask",
+      HasNoMaskedOverloaded = false,
+      ManualCodegen = [{
+        IntrinsicTypes = {ResultType, Ops[1]->getType()};
+        Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo());
+      }],
+      ManualCodegenMask= [{
+        IntrinsicTypes = {ResultType, Ops[3]->getType()};
+        Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
+      }] in {
+    foreach type = types in {
+      def : RVVBuiltin<"v", "vPCe", type>;
+      if !not(IsFloat<type>.val) then {
+        def : RVVBuiltin<"Uv", "UvPCUe", type>;
+      }
+    }
+  }
+}
+
+multiclass RVVIndexedLoad<string op> {
+  let ManualCodegen = [{
+        IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType()};
+        Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo());
+      }],
+      ManualCodegenMask = [{
+        IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops[4]->getType()};
+        Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
+      }] in {
+      foreach type = TypeList in {
+        foreach eew_list = EEWList in {
+          defvar eew = eew_list[0];
+          defvar eew_type = eew_list[1];
+          let Name = op # eew # "_v", IRName = op, IRNameMask = op # "_mask" in {
+            def: RVVBuiltin<"v", "vPCe" # eew_type # "Uv", type>;
+              if !not(IsFloat<type>.val) then {
+                def: RVVBuiltin<"Uv", "UvPCUe" # eew_type # "Uv", type>;
+              }
+          }
+        }
+      }
+  }
+}
+
+multiclass RVVVSEBuiltin<list<string> types> {
+  let Name = NAME # "_v",
+      IRName = "vse",
+      IRNameMask = "vse_mask",
+      HasMaskedOffOperand = false,
+      PermuteOperands = [1, 0], // C/C++ Operand: (ptr, value, vl). Builtin: (value, ptr, vl)
+      ManualCodegen = [{
+        Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
+        IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()};
+      }],
+      ManualCodegenMask= [{
+        Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
+        IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()};
+      }] in {
+    foreach type = types in {
+      def : RVVBuiltin<"v", "0vPe", type>;
+      if !not(IsFloat<type>.val) then {
+        def : RVVBuiltin<"Uv", "0UvPUe", type>;
+      }
+    }
+  }
+}
+
+// 6. Configuration-Setting Instructions
+// 6.1. vsetvli/vsetvl instructions
+let HasVL = false,
+    HasMask = false,
+    HasSideEffects = true,
+    Log2LMUL = [0],
+    ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type
+{
+  // vsetvl is a macro because for it require constant integers in SEW and LMUL.
+  let HeaderCode =
+[{
+#define vsetvl_e8mf8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 5)
+#define vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6)
+#define vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7)
+#define vsetvl_e8m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 0)
+#define vsetvl_e8m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 1)
+#define vsetvl_e8m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 2)
+#define vsetvl_e8m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 3)
+
+#define vsetvl_e16mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 6)
+#define vsetvl_e16mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 7)
+#define vsetvl_e16m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 0)
+#define vsetvl_e16m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 1)
+#define vsetvl_e16m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 2)
+#define vsetvl_e16m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 3)
+
+#define vsetvl_e32mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 7)
+#define vsetvl_e32m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 0)
+#define vsetvl_e32m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 1)
+#define vsetvl_e32m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 2)
+#define vsetvl_e32m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 3)
+
+#define vsetvl_e64m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 0)
+#define vsetvl_e64m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 1)
+#define vsetvl_e64m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 2)
+#define vsetvl_e64m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 3)
+
+}] in
+  def vsetvli : RVVBuiltin<"", "zzKzKz", "i">;
+
+  let HeaderCode =
+[{
+#define vsetvlmax_e8mf8() __builtin_rvv_vsetvlimax(0, 5)
+#define vsetvlmax_e8mf4() __builtin_rvv_vsetvlimax(0, 6)
+#define vsetvlmax_e8mf2() __builtin_rvv_vsetvlimax(0, 7)
+#define vsetvlmax_e8m1() __builtin_rvv_vsetvlimax(0, 0)
+#define vsetvlmax_e8m2() __builtin_rvv_vsetvlimax(0, 1)
+#define vsetvlmax_e8m4() __builtin_rvv_vsetvlimax(0, 2)
+#define vsetvlmax_e8m8() __builtin_rvv_vsetvlimax(0, 3)
+
+#define vsetvlmax_e16mf4() __builtin_rvv_vsetvlimax(1, 6)
+#define vsetvlmax_e16mf2() __builtin_rvv_vsetvlimax(1, 7)
+#define vsetvlmax_e16m1() __builtin_rvv_vsetvlimax(1, 0)
+#define vsetvlmax_e16m2() __builtin_rvv_vsetvlimax(1, 1)
+#define vsetvlmax_e16m4() __builtin_rvv_vsetvlimax(1, 2)
+#define vsetvlmax_e16m8() __builtin_rvv_vsetvlimax(1, 3)
+
+#define vsetvlmax_e32mf2() __builtin_rvv_vsetvlimax(2, 7)
+#define vsetvlmax_e32m1() __builtin_rvv_vsetvlimax(2, 0)
+#define vsetvlmax_e32m2() __builtin_rvv_vsetvlimax(2, 1)
+#define vsetvlmax_e32m4() __builtin_rvv_vsetvlimax(2, 2)
+#define vsetvlmax_e32m8() __builtin_rvv_vsetvlimax(2, 3)
+
+#define vsetvlmax_e64m1() __builtin_rvv_vsetvlimax(3, 0)
+#define vsetvlmax_e64m2() __builtin_rvv_vsetvlimax(3, 1)
+#define vsetvlmax_e64m4() __builtin_rvv_vsetvlimax(3, 2)
+#define vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3)
+
+}] in
+  def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">;
+}
+
+// 7. Vector Loads and Stores
+// 7.4. Vector Unit-Stride Instructions
+defm vle8: RVVVLEBuiltin<["c"]>;
+defm vle16: RVVVLEBuiltin<["s"]>;
+defm vle32: RVVVLEBuiltin<["i","f"]>;
+defm vle64: RVVVLEBuiltin<["l","d"]>;
+
+defm vse8 : RVVVSEBuiltin<["c"]>;
+defm vse16: RVVVSEBuiltin<["s"]>;
+defm vse32: RVVVSEBuiltin<["i","f"]>;
+defm vse64: RVVVSEBuiltin<["l","d"]>;
+
+// 7.6. Vector Indexed Instructions
+defm : RVVIndexedLoad<"vluxei">;
+defm : RVVIndexedLoad<"vloxei">;
+
 // 12. Vector Integer Arithmetic Instructions
 // 12.1. Vector Single-Width Integer Add and Subtract
 defm vadd : RVVBinBuiltinSet<"vadd", "csil",
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 54c20620910b347ad216c319ed037828d09a8eff..469c000c952cadb1a0d406d380216682a7941264 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -157,16 +157,16 @@ public:
   std::string HostBits, HostMachine, HostSystem, HostRelease;
 
   /// The file to log CC_PRINT_PROC_STAT_FILE output to, if enabled.
-  const char *CCPrintStatReportFilename;
+  std::string CCPrintStatReportFilename;
 
   /// The file to log CC_PRINT_OPTIONS output to, if enabled.
-  const char *CCPrintOptionsFilename;
+  std::string CCPrintOptionsFilename;
 
   /// The file to log CC_PRINT_HEADERS output to, if enabled.
-  const char *CCPrintHeadersFilename;
+  std::string CCPrintHeadersFilename;
 
   /// The file to log CC_LOG_DIAGNOSTICS output to, if enabled.
-  const char *CCLogDiagnosticsFilename;
+  std::string CCLogDiagnosticsFilename;
 
   /// A list of inputs and their types for the given arguments.
   typedef SmallVector<std::pair<types::ID, const llvm::opt::Arg *>, 16>
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index d4fb2bf653b7c93ee398e6372984c35f848c70d5..46ff9bd655569dcae7846235a5df0f2b7367be9a 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -601,8 +601,14 @@ def _HASH_HASH_HASH : Flag<["-"], "###">, Flags<[NoXarchOption, CoreOption, Flan
 def _DASH_DASH : Option<["--"], "", KIND_REMAINING_ARGS>,
     Flags<[NoXarchOption, CoreOption]>;
 def A : JoinedOrSeparate<["-"], "A">, Flags<[RenderJoined]>, Group<gfortran_Group>;
-def B : JoinedOrSeparate<["-"], "B">, MetaVarName<"<dir>">,
-    HelpText<"Add <dir> to search path for binaries and object files used implicitly">;
+def B : JoinedOrSeparate<["-"], "B">, MetaVarName<"<prefix>">,
+    HelpText<"Search $prefix/$triple-$file and $prefix$file for executables, libraries, "
+    "includes, and data files used by the compiler. $prefix may or may not be a directory">;
+def gcc_toolchain : Joined<["--"], "gcc-toolchain=">, Flags<[NoXarchOption]>,
+  HelpText<"Search for GCC installation in the specified directory on targets which commonly use GCC. "
+  "The directory usually contains 'lib{,32,64}/gcc{,-cross}/$triple' and 'include'. If specified, "
+  "sysroot is skipped for GCC detection. Note: executables (e.g. ld) used by the compiler are not "
+  "overridden by the selected GCC installation">;
 def CC : Flag<["-"], "CC">, Flags<[CC1Option]>, Group<Preprocessor_Group>,
     HelpText<"Include comments from within macros in preprocessed output">,
     MarshallingInfoFlag<PreprocessorOutputOpts<"ShowMacroComments">>;
@@ -980,11 +986,6 @@ def dependency_dot : Separate<["-"], "dependency-dot">, Flags<[CC1Option]>,
 def module_dependency_dir : Separate<["-"], "module-dependency-dir">,
   Flags<[CC1Option]>, HelpText<"Directory to dump module dependencies to">,
   MarshallingInfoString<DependencyOutputOpts<"ModuleDependencyOutputDir">>;
-def module_dir : Separate<["-"], "module-dir">, Flags<[FlangOption,FC1Option]>, MetaVarName<"<dir>">,
-  HelpText<"Put MODULE files in <dir>">,
-  DocBrief<[{This option specifies where to put .mod files for compiled modules.
-It is also added to the list of directories to be searched by an USE statement.
-The default is the current directory.}]>;
 def dsym_dir : JoinedOrSeparate<["-"], "dsym-dir">,
   Flags<[NoXarchOption, RenderAsInput]>,
   HelpText<"Directory to output dSYM's (if any) to">, MetaVarName<"<dir>">;
@@ -1125,6 +1126,7 @@ def fcoverage_compilation_dir_EQ : Joined<["-"], "fcoverage-compilation-dir=">,
     HelpText<"The compilation directory to embed in the coverage mapping.">,
     MarshallingInfoString<CodeGenOpts<"CoverageCompilationDir">>;
 def ffile_compilation_dir_EQ : Joined<["-"], "ffile-compilation-dir=">, Group<f_Group>,
+    Flags<[CoreOption]>,
     HelpText<"The compilation directory to embed in the debug info and coverage mapping.">;
 defm debug_info_for_profiling : BoolFOption<"debug-info-for-profiling",
   CodeGenOpts<"DebugInfoForProfiling">, DefaultFalse,
@@ -1277,7 +1279,9 @@ def fconstexpr_backtrace_limit_EQ : Joined<["-"], "fconstexpr-backtrace-limit=">
                                     Group<f_Group>;
 def fno_crash_diagnostics : Flag<["-"], "fno-crash-diagnostics">, Group<f_clang_Group>, Flags<[NoArgumentUnused, CoreOption]>,
   HelpText<"Disable auto-generation of preprocessed source files and a script for reproduction during a clang crash">;
-def fcrash_diagnostics_dir : Joined<["-"], "fcrash-diagnostics-dir=">, Group<f_clang_Group>, Flags<[NoArgumentUnused, CoreOption]>;
+def fcrash_diagnostics_dir : Joined<["-"], "fcrash-diagnostics-dir=">,
+  Group<f_clang_Group>, Flags<[NoArgumentUnused, CoreOption]>,
+  HelpText<"Put crash-report files in <dir>">, MetaVarName<"<dir>">;
 def fcreate_profile : Flag<["-"], "fcreate-profile">, Group<f_Group>;
 defm cxx_exceptions: BoolFOption<"cxx-exceptions",
   LangOpts<"CXXExceptions">, DefaultFalse,
@@ -2373,7 +2377,10 @@ defm signed_char : BoolFOption<"signed-char",
   LangOpts<"CharIsSigned">, DefaultTrue,
   NegFlag<SetFalse, [CC1Option], "char is unsigned">, PosFlag<SetTrue, [], "char is signed">>,
   ShouldParseIf<!strconcat("!", open_cl.KeyPath)>;
-def fsplit_stack : Flag<["-"], "fsplit-stack">, Group<f_Group>;
+defm split_stack : BoolFOption<"split-stack",
+  CodeGenOpts<"EnableSegmentedStacks">, DefaultFalse,
+  NegFlag<SetFalse, [], "Wouldn't use segmented stack">, 
+  PosFlag<SetTrue, [CC1Option], "Use segmented stack">>;
 def fstack_protector_all : Flag<["-"], "fstack-protector-all">, Group<f_Group>,
   HelpText<"Enable stack protectors for all functions">;
 defm stack_clash_protection : BoolFOption<"stack-clash-protection",
@@ -3129,8 +3136,6 @@ def mharden_sls_EQ : Joined<["-"], "mharden-sls=">,
   HelpText<"Select straight-line speculation hardening scope">;
 
 def msimd128 : Flag<["-"], "msimd128">, Group<m_wasm_Features_Group>;
-def munimplemented_simd128 : Flag<["-"], "munimplemented-simd128">, Group<m_wasm_Features_Group>;
-def mno_unimplemented_simd128 : Flag<["-"], "mno-unimplemented-simd128">, Group<m_wasm_Features_Group>;
 def mno_simd128 : Flag<["-"], "mno-simd128">, Group<m_wasm_Features_Group>;
 def mnontrapping_fptoint : Flag<["-"], "mnontrapping-fptoint">, Group<m_wasm_Features_Group>;
 def mno_nontrapping_fptoint : Flag<["-"], "mno-nontrapping-fptoint">, Group<m_wasm_Features_Group>;
@@ -3248,7 +3253,9 @@ def mno_longcall : Flag<["-"], "mno-longcall">,
     Group<m_ppc_Features_Group>;
 def mmma: Flag<["-"], "mmma">, Group<m_ppc_Features_Group>;
 def mno_mma: Flag<["-"], "mno-mma">, Group<m_ppc_Features_Group>;
-def mrop_protection : Flag<["-"], "mrop-protection">,
+def mrop_protect : Flag<["-"], "mrop-protect">,
+    Group<m_ppc_Features_Group>;
+def mprivileged : Flag<["-"], "mprivileged">,
     Group<m_ppc_Features_Group>;
 def maix_struct_return : Flag<["-"], "maix-struct-return">,
   Group<m_Group>, Flags<[CC1Option]>,
@@ -3323,10 +3330,6 @@ def mstack_protector_guard_offset_EQ : Joined<["-"], "mstack-protector-guard-off
 def mstack_protector_guard_reg_EQ : Joined<["-"], "mstack-protector-guard-reg=">, Group<m_Group>, Flags<[CC1Option]>,
   HelpText<"Use the given reg for addressing the stack-protector guard">,
   MarshallingInfoString<CodeGenOpts<"StackProtectorGuardReg">, [{"none"}]>;
-def mpie_copy_relocations : Flag<["-"], "mpie-copy-relocations">,
-  Alias<fdirect_access_external_data>, Group<m_Group>;
-def mno_pie_copy_relocations : Flag<["-"], "mno-pie-copy-relocations">,
-  Alias<fno_direct_access_external_data>, Group<m_Group>;
 def mfentry : Flag<["-"], "mfentry">, HelpText<"Insert calls to fentry at function entry (x86/SystemZ only)">,
   Flags<[CC1Option]>, Group<m_Group>,
   MarshallingInfoFlag<CodeGenOpts<"CallFEntry">>;
@@ -3537,8 +3540,8 @@ def pagezero__size : JoinedOrSeparate<["-"], "pagezero_size">;
 def pass_exit_codes : Flag<["-", "--"], "pass-exit-codes">, Flags<[Unsupported]>;
 def pedantic_errors : Flag<["-", "--"], "pedantic-errors">, Group<pedantic_Group>, Flags<[CC1Option]>,
   MarshallingInfoFlag<DiagnosticOpts<"PedanticErrors">>;
-def pedantic : Flag<["-", "--"], "pedantic">, Group<pedantic_Group>, Flags<[CC1Option]>,
-  MarshallingInfoFlag<DiagnosticOpts<"Pedantic">>;
+def pedantic : Flag<["-", "--"], "pedantic">, Group<pedantic_Group>, Flags<[CC1Option,FlangOption,FC1Option]>,
+  HelpText<"Warn on language extensions">, MarshallingInfoFlag<DiagnosticOpts<"Pedantic">>;
 def pg : Flag<["-"], "pg">, HelpText<"Enable mcount instrumentation">, Flags<[CC1Option]>,
   MarshallingInfoFlag<CodeGenOpts<"InstrumentForProfiling">>;
 def pipe : Flag<["-", "--"], "pipe">,
@@ -3572,6 +3575,8 @@ def print_targets : Flag<["-", "--"], "print-targets">,
   HelpText<"Print the registered targets">;
 def print_rocm_search_dirs : Flag<["-", "--"], "print-rocm-search-dirs">,
   HelpText<"Print the paths used for finding ROCm installation">;
+def print_runtime_dir : Flag<["-", "--"], "print-runtime-dir">,
+  HelpText<"Print the directory pathname containing clangs runtime libraries">;
 def private__bundle : Flag<["-"], "private_bundle">;
 def pthreads : Flag<["-"], "pthreads">;
 defm pthread : BoolOption<"", "pthread",
@@ -3638,7 +3643,7 @@ def static_libgcc : Flag<["-"], "static-libgcc">;
 def static_libstdcxx : Flag<["-"], "static-libstdc++">;
 def static : Flag<["-", "--"], "static">, Group<Link_Group>, Flags<[NoArgumentUnused]>;
 def std_default_EQ : Joined<["-"], "std-default=">;
-def std_EQ : Joined<["-", "--"], "std=">, Flags<[CC1Option]>,
+def std_EQ : Joined<["-", "--"], "std=">, Flags<[CC1Option,FlangOption,FC1Option]>,
   Group<CompileOnly_Group>, HelpText<"Language standard to compile for">,
   ValuesCode<[{
     const char *Values =
@@ -3677,8 +3682,6 @@ def print_supported_cpus : Flag<["-", "--"], "print-supported-cpus">,
   MarshallingInfoFlag<FrontendOpts<"PrintSupportedCPUs">>;
 def mcpu_EQ_QUESTION : Flag<["-"], "mcpu=?">, Alias<print_supported_cpus>;
 def mtune_EQ_QUESTION : Flag<["-"], "mtune=?">, Alias<print_supported_cpus>;
-def gcc_toolchain : Joined<["--"], "gcc-toolchain=">, Flags<[NoXarchOption]>,
-  HelpText<"Use the gcc toolchain at the given directory">;
 def time : Flag<["-"], "time">,
   HelpText<"Time individual commands">;
 def traditional_cpp : Flag<["-", "--"], "traditional-cpp">, Flags<[CC1Option]>,
@@ -4224,7 +4227,6 @@ defm devirtualize_speculatively : BooleanFFlag<"devirtualize-speculatively">,
 
 // Generic gfortran options.
 def A_DASH : Joined<["-"], "A-">, Group<gfortran_Group>;
-def J : JoinedOrSeparate<["-"], "J">, Flags<[RenderJoined,FlangOption,FC1Option]>, Group<gfortran_Group>, Alias<module_dir>;
 def cpp : Flag<["-"], "cpp">, Group<gfortran_Group>;
 def nocpp : Flag<["-"], "nocpp">, Group<gfortran_Group>;
 def static_libgfortran : Flag<["-"], "static-libgfortran">, Group<gfortran_Group>;
@@ -4266,7 +4268,6 @@ defm f2c : BooleanFFlag<"f2c">, Group<gfortran_Group>;
 defm frontend_optimize : BooleanFFlag<"frontend-optimize">, Group<gfortran_Group>;
 defm init_local_zero : BooleanFFlag<"init-local-zero">, Group<gfortran_Group>;
 defm integer_4_integer_8 : BooleanFFlag<"integer-4-integer-8">, Group<gfortran_Group>;
-defm intrinsic_modules_path : BooleanFFlag<"intrinsic-modules-path">, Group<gfortran_Group>;
 defm max_identifier_length : BooleanFFlag<"max-identifier-length">, Group<gfortran_Group>;
 defm module_private : BooleanFFlag<"module-private">, Group<gfortran_Group>;
 defm pack_derived : BooleanFFlag<"pack-derived">, Group<gfortran_Group>;
@@ -4288,15 +4289,10 @@ defm underscoring : BooleanFFlag<"underscoring">, Group<gfortran_Group>;
 defm whole_file : BooleanFFlag<"whole-file">, Group<gfortran_Group>;
 
 // C++ SYCL options
-defm sycl : BoolOption<"f", "sycl",
-  LangOpts<"SYCL">, DefaultFalse,
-  PosFlag<SetTrue, [CC1Option], "Enable">, NegFlag<SetFalse, [], "Disable">,
-  BothFlags<[CoreOption], " SYCL kernels compilation for device">>,
-  Group<sycl_Group>;
-def sycl_std_EQ : Joined<["-"], "sycl-std=">, Group<sycl_Group>, Flags<[CC1Option, NoArgumentUnused, CoreOption]>,
-  HelpText<"SYCL language standard to compile for.">, Values<"2017,121,1.2.1,sycl-1.2.1">,
-  NormalizedValues<["SYCL_2017", "SYCL_2017", "SYCL_2017", "SYCL_2017"]>, NormalizedValuesScope<"LangOptions">,
-  MarshallingInfoEnum<LangOpts<"SYCLVersion">, "SYCL_None">, ShouldParseIf<fsycl.KeyPath>;
+def fsycl : Flag<["-"], "fsycl">, Flags<[NoXarchOption, CoreOption]>,
+  Group<sycl_Group>, HelpText<"Enables SYCL kernels compilation for device">;
+def fno_sycl : Flag<["-"], "fno-sycl">, Flags<[NoXarchOption, CoreOption]>,
+  Group<sycl_Group>, HelpText<"Disables SYCL kernels compilation for device">;
 
 //===----------------------------------------------------------------------===//
 // FLangOption + CoreOption + NoXarchOption
@@ -4312,6 +4308,12 @@ def Xflang : Separate<["-"], "Xflang">,
 //===----------------------------------------------------------------------===//
 let Flags = [FC1Option, FlangOption, FlangOnlyOption] in {
 
+def module_dir : Separate<["-"], "module-dir">, MetaVarName<"<dir>">,
+  HelpText<"Put MODULE files in <dir>">,
+  DocBrief<[{This option specifies where to put .mod files for compiled modules.
+It is also added to the list of directories to be searched by an USE statement.
+The default is the current directory.}]>;
+
 def ffixed_form : Flag<["-"], "ffixed-form">, Group<f_Group>,
   HelpText<"Process source files in fixed form">;
 def ffree_form : Flag<["-"], "ffree-form">, Group<f_Group>,
@@ -4347,8 +4349,17 @@ def fimplicit_none : Flag<["-"], "fimplicit-none">, Group<f_Group>,
 def fno_implicit_none : Flag<["-"], "fno-implicit-none">, Group<f_Group>;
 def falternative_parameter_statement : Flag<["-"], "falternative-parameter-statement">, Group<f_Group>,
   HelpText<"Enable the old style PARAMETER statement">;
+def fintrinsic_modules_path : Separate<["-"], "fintrinsic-modules-path">,  Group<f_Group>, MetaVarName<"<dir>">,
+  HelpText<"Specify where to find the compiled intrinsic modules">,
+  DocBrief<[{This option specifies the location of pre-compiled intrinsic modules, 
+  if they are not in the default location expected by the compiler.}]>;
 }
 
+def J : JoinedOrSeparate<["-"], "J">,
+  Flags<[RenderJoined, FlangOption, FC1Option, FlangOnlyOption]>,
+  Group<gfortran_Group>,
+  Alias<module_dir>;
+
 //===----------------------------------------------------------------------===//
 // FC1 Options
 //===----------------------------------------------------------------------===//
@@ -4374,6 +4385,8 @@ def fdebug_pre_fir_tree : Flag<["-"], "fdebug-pre-fir-tree">, Group<Action_Group
   HelpText<"Dump the pre-FIR tree">;
 def fdebug_module_writer : Flag<["-"],"fdebug-module-writer">, 
   HelpText<"Enable debug messages while writing module files">;
+def fget_symbols_sources : Flag<["-"], "fget-symbols-sources">, Group<Action_Group>,
+  HelpText<"Dump symbols and their source code locations">;
 
 }
 
@@ -4749,9 +4762,6 @@ def mtp : Separate<["-"], "mtp">,
 def mlimit_float_precision : Separate<["-"], "mlimit-float-precision">,
   HelpText<"Limit float precision to the given value">,
   MarshallingInfoString<CodeGenOpts<"LimitFloatPrecision">>;
-def split_stacks : Flag<["-"], "split-stacks">,
-  HelpText<"Try to use a split stack if possible.">,
-  MarshallingInfoFlag<CodeGenOpts<"EnableSegmentedStacks">>;
 def mregparm : Separate<["-"], "mregparm">,
   HelpText<"Limit the number of registers available for integer arguments">,
   MarshallingInfoInt<CodeGenOpts<"NumRegisterParameters">>;
@@ -4872,6 +4882,18 @@ def fexperimental_debug_variable_locations : Flag<["-"],
     "fexperimental-debug-variable-locations">,
     HelpText<"Use experimental new value-tracking variable locations">,
     MarshallingInfoFlag<CodeGenOpts<"ValueTrackingVariableLocations">>;
+def fverify_debuginfo_preserve
+    : Flag<["-"], "fverify-debuginfo-preserve">,
+      HelpText<"Enable Debug Info Metadata preservation testing in "
+               "optimizations.">,
+      MarshallingInfoFlag<CodeGenOpts<"EnableDIPreservationVerify">>;
+def fverify_debuginfo_preserve_export
+    : Joined<["-"], "fverify-debuginfo-preserve-export=">,
+      MetaVarName<"<file>">,
+      HelpText<"Export debug info (by testing original Debug Info) failures "
+               "into specified (JSON) file (should be abs path as we use "
+               "append mode to insert new JSON objects).">,
+      MarshallingInfoString<CodeGenOpts<"DIBugsReportFilePath">>;
 // The driver option takes the key as a parameter to the -msign-return-address=
 // and -mbranch-protection= options, but CC1 has a separate option so we
 // don't have to parse the parameter twice.
@@ -5551,11 +5573,22 @@ def fopenmp_host_ir_file_path : Separate<["-"], "fopenmp-host-ir-file-path">,
 
 def fsycl_is_device : Flag<["-"], "fsycl-is-device">,
   HelpText<"Generate code for SYCL device.">,
-  MarshallingInfoFlag<LangOpts<"SYCLIsDevice">>,
-  ShouldParseIf<fsycl.KeyPath>;
+  MarshallingInfoFlag<LangOpts<"SYCLIsDevice">>;
+def fsycl_is_host : Flag<["-"], "fsycl-is-host">,
+  HelpText<"SYCL host compilation">,
+  MarshallingInfoFlag<LangOpts<"SYCLIsHost">>;
 
 } // let Flags = [CC1Option, NoDriverOption]
 
+def sycl_std_EQ : Joined<["-"], "sycl-std=">, Group<sycl_Group>,
+  Flags<[CC1Option, NoArgumentUnused, CoreOption]>,
+  HelpText<"SYCL language standard to compile for.">,
+  Values<"2017,121,1.2.1,sycl-1.2.1">,
+  NormalizedValues<["SYCL_2017", "SYCL_2017", "SYCL_2017", "SYCL_2017"]>,
+  NormalizedValuesScope<"LangOptions">,
+  MarshallingInfoEnum<LangOpts<"SYCLVersion">, "SYCL_None">,
+  ShouldParseIf<!strconcat(fsycl_is_device.KeyPath, "||", fsycl_is_host.KeyPath)>;
+
 defm cuda_approx_transcendentals : BoolFOption<"cuda-approx-transcendentals",
   LangOpts<"CUDADeviceApproxTranscendentals">, DefaultFalse,
   PosFlag<SetTrue, [CC1Option], "Use">, NegFlag<SetFalse, [], "Don't use">,
diff --git a/clang/include/clang/Driver/Types.def b/clang/include/clang/Driver/Types.def
index 79e8d109cd97e78459747dad893b0f41b7a5898e..997eea445c225b20fab9bc030a8320dc2e4c6570 100644
--- a/clang/include/clang/Driver/Types.def
+++ b/clang/include/clang/Driver/Types.def
@@ -38,6 +38,7 @@
 TYPE("cpp-output",               PP_C,         INVALID,         "i",      phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("c",                        C,            PP_C,            "c",      phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("cl",                       CL,           PP_C,            "cl",     phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
+TYPE("clcpp",                    CLCXX,        PP_CXX,          "clcpp",  phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("cuda-cpp-output",          PP_CUDA,      INVALID,         "cui",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("cuda",                     CUDA,         PP_CUDA,         "cu",     phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("cuda",                     CUDA_DEVICE,  PP_CUDA,         "cu",     phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
diff --git a/clang/include/clang/Lex/MacroInfo.h b/clang/include/clang/Lex/MacroInfo.h
index 550abf35c84166d5288ec542378550bc135da971..0347a7a37186b31565d571bb6cb2ce80f2bfda35 100644
--- a/clang/include/clang/Lex/MacroInfo.h
+++ b/clang/include/clang/Lex/MacroInfo.h
@@ -521,7 +521,7 @@ public:
   }
 
   static void Profile(llvm::FoldingSetNodeID &ID, Module *OwningModule,
-                      IdentifierInfo *II) {
+                      const IdentifierInfo *II) {
     ID.AddPointer(OwningModule);
     ID.AddPointer(II);
   }
diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h
index f6423e5b425896181116fc6b80ff93af51d20c6f..64562e6760df8d180eb7bf31c2b2e52b57a5aaf1 100644
--- a/clang/include/clang/Lex/ModuleMap.h
+++ b/clang/include/clang/Lex/ModuleMap.h
@@ -696,6 +696,9 @@ public:
 
   module_iterator module_begin() const { return Modules.begin(); }
   module_iterator module_end()   const { return Modules.end(); }
+  llvm::iterator_range<module_iterator> modules() const {
+    return {module_begin(), module_end()};
+  }
 
   /// Cache a module load.  M might be nullptr.
   void cacheModuleLoad(const IdentifierInfo &II, Module *M) {
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 68139cb24b31d083670e2f255b6dc53aff566e11..d89c4753f8d1a884c064dc8698046ccf609d2eab 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -1151,7 +1151,7 @@ public:
   /// Register an exported macro for a module and identifier.
   ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
                               ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
-  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
+  ModuleMacro *getModuleMacro(Module *Mod, const IdentifierInfo *II);
 
   /// Get the list of leaf (non-overridden) module macros for a name.
   ArrayRef<ModuleMacro*> getLeafModuleMacros(const IdentifierInfo *II) const {
@@ -1163,6 +1163,11 @@ public:
     return None;
   }
 
+  /// Get the list of submodules that we're currently building.
+  ArrayRef<BuildingSubmoduleInfo> getBuildingSubmodules() const {
+    return BuildingSubmoduleStack;
+  }
+
   /// \{
   /// Iterators for the macro history table. Currently defined macros have
   /// IdentifierInfo::hasMacroDefinition() set and an empty
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 09a0dd2cf23392ad2fa0bb69f788f56f60ff0ed0..290b451771a6849d80bffd74394c63d7ec0e471d 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -941,8 +941,8 @@ private:
     bool isActive;
 
   public:
-    explicit TentativeParsingAction(Parser& p) : P(p) {
-      PrevPreferredType = P.PreferredType;
+    explicit TentativeParsingAction(Parser &p)
+        : P(p), PrevPreferredType(P.PreferredType) {
       PrevTok = P.Tok;
       PrevTentativelyDeclaredIdentifierCount =
           P.TentativelyDeclaredIdentifiers.size();
@@ -1991,7 +1991,8 @@ private:
   Sema::ConditionResult ParseCXXCondition(StmtResult *InitStmt,
                                           SourceLocation Loc,
                                           Sema::ConditionKind CK,
-                                          ForRangeInfo *FRI = nullptr);
+                                          ForRangeInfo *FRI = nullptr,
+                                          bool EnterForConditionScope = false);
 
   //===--------------------------------------------------------------------===//
   // C++ Coroutines
@@ -3292,6 +3293,14 @@ private:
   /// '(' { <allocator> [ '(' <allocator_traits> ')' ] }+ ')'
   OMPClause *ParseOpenMPUsesAllocatorClause(OpenMPDirectiveKind DKind);
 
+  /// Parses clause with an interop variable of kind \a Kind.
+  ///
+  /// \param Kind Kind of current clause.
+  /// \param ParseOnly true to skip the clause's semantic actions and return
+  /// nullptr.
+  //
+  OMPClause *ParseOpenMPInteropClause(OpenMPClauseKind Kind, bool ParseOnly);
+
 public:
   /// Parses simple expression in parens for single-expression clauses of OpenMP
   /// constructs.
diff --git a/clang/include/clang/Sema/AnalysisBasedWarnings.h b/clang/include/clang/Sema/AnalysisBasedWarnings.h
index e13fe955eaf486b07a98440a7bb2a40b1b758582..49b69c585ff79cdeb48450b661f971822f026f97 100644
--- a/clang/include/clang/Sema/AnalysisBasedWarnings.h
+++ b/clang/include/clang/Sema/AnalysisBasedWarnings.h
@@ -14,6 +14,7 @@
 #define LLVM_CLANG_SEMA_ANALYSISBASEDWARNINGS_H
 
 #include "llvm/ADT/DenseMap.h"
+#include <memory>
 
 namespace clang {
 
@@ -47,6 +48,9 @@ private:
   Sema &S;
   Policy DefaultPolicy;
 
+  class InterProceduralData;
+  std::unique_ptr<InterProceduralData> IPData;
+
   enum VisitFlag { NotVisited = 0, Visited = 1, Pending = 2 };
   llvm::DenseMap<const FunctionDecl*, VisitFlag> VisitedFD;
 
@@ -88,6 +92,7 @@ private:
 
 public:
   AnalysisBasedWarnings(Sema &s);
+  ~AnalysisBasedWarnings();
 
   void IssueWarnings(Policy P, FunctionScopeInfo *fscope,
                      const Decl *D, QualType BlockType);
@@ -97,6 +102,7 @@ public:
   void PrintStats() const;
 };
 
-}} // end namespace clang::sema
+} // namespace sema
+} // namespace clang
 
 #endif
diff --git a/clang/include/clang/Sema/Initialization.h b/clang/include/clang/Sema/Initialization.h
index dcdfa3c3cf64a34d16d2542d7d4d60224b70a095..a764a36d33127fd3a21617c2d9687ae1013a6871 100644
--- a/clang/include/clang/Sema/Initialization.h
+++ b/clang/include/clang/Sema/Initialization.h
@@ -187,8 +187,8 @@ private:
     ObjCMethodDecl *MethodDecl;
 
     /// When Kind == EK_Parameter, the ParmVarDecl, with the
-    /// low bit indicating whether the parameter is "consumed".
-    uintptr_t Parameter;
+    /// integer indicating whether the parameter is "consumed".
+    llvm::PointerIntPair<ParmVarDecl *, 1> Parameter;
 
     /// When Kind == EK_Temporary or EK_CompoundLiteralInit, the type
     /// source information for the temporary.
@@ -197,9 +197,9 @@ private:
     struct LN LocAndNRVO;
 
     /// When Kind == EK_Base, the base specifier that provides the
-    /// base class. The lower bit specifies whether the base is an inherited
+    /// base class. The integer specifies whether the base is an inherited
     /// virtual base.
-    uintptr_t Base;
+    llvm::PointerIntPair<const CXXBaseSpecifier *, 1> Base;
 
     /// When Kind == EK_ArrayElement, EK_VectorElement, or
     /// EK_ComplexElement, the index of the array or vector element being
@@ -252,15 +252,14 @@ public:
 
   /// Create the initialization entity for a parameter.
   static InitializedEntity InitializeParameter(ASTContext &Context,
-                                               const ParmVarDecl *Parm) {
+                                               ParmVarDecl *Parm) {
     return InitializeParameter(Context, Parm, Parm->getType());
   }
 
   /// Create the initialization entity for a parameter, but use
   /// another type.
-  static InitializedEntity InitializeParameter(ASTContext &Context,
-                                               const ParmVarDecl *Parm,
-                                               QualType Type) {
+  static InitializedEntity
+  InitializeParameter(ASTContext &Context, ParmVarDecl *Parm, QualType Type) {
     bool Consumed = (Context.getLangOpts().ObjCAutoRefCount &&
                      Parm->hasAttr<NSConsumedAttr>());
 
@@ -269,8 +268,7 @@ public:
     Entity.Type =
       Context.getVariableArrayDecayedType(Type.getUnqualifiedType());
     Entity.Parent = nullptr;
-    Entity.Parameter
-      = (static_cast<uintptr_t>(Consumed) | reinterpret_cast<uintptr_t>(Parm));
+    Entity.Parameter = {Parm, Consumed};
     return Entity;
   }
 
@@ -283,7 +281,7 @@ public:
     Entity.Kind = EK_Parameter;
     Entity.Type = Context.getVariableArrayDecayedType(Type);
     Entity.Parent = nullptr;
-    Entity.Parameter = (Consumed);
+    Entity.Parameter = {nullptr, Consumed};
     return Entity;
   }
 
@@ -466,19 +464,19 @@ public:
   /// parameter.
   bool isParameterConsumed() const {
     assert(isParameterKind() && "Not a parameter");
-    return (Parameter & 1);
+    return Parameter.getInt();
   }
 
   /// Retrieve the base specifier.
   const CXXBaseSpecifier *getBaseSpecifier() const {
     assert(getKind() == EK_Base && "Not a base specifier");
-    return reinterpret_cast<const CXXBaseSpecifier *>(Base & ~0x1);
+    return Base.getPointer();
   }
 
   /// Return whether the base is an inherited virtual base.
   bool isInheritedVirtualBase() const {
     assert(getKind() == EK_Base && "Not a base specifier");
-    return Base & 0x1;
+    return Base.getInt();
   }
 
   /// Determine whether this is an array new with an unknown bound.
diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h
index 0d731d9150a8a322edd335e657f8753d12f45bf1..a3d82fcd84f7a9ddb86a964460a7d432dfb5e5e0 100644
--- a/clang/include/clang/Sema/ParsedAttr.h
+++ b/clang/include/clang/Sema/ParsedAttr.h
@@ -39,6 +39,7 @@ class IdentifierInfo;
 class LangOptions;
 class ParsedAttr;
 class Sema;
+class Stmt;
 class TargetInfo;
 
 struct ParsedAttrInfo {
@@ -80,6 +81,11 @@ struct ParsedAttrInfo {
                                     const Decl *D) const {
     return true;
   }
+  /// Check if this attribute appertains to St, and issue a diagnostic if not.
+  virtual bool diagAppertainsToStmt(Sema &S, const ParsedAttr &Attr,
+                                    const Stmt *St) const {
+    return true;
+  }
   /// Check if this attribute is allowed by the language we are compiling, and
   /// issue a diagnostic if not.
   virtual bool diagLangOpts(Sema &S, const ParsedAttr &Attr) const {
@@ -592,6 +598,7 @@ public:
   unsigned getMaxArgs() const;
   bool hasVariadicArg() const;
   bool diagnoseAppertainsTo(class Sema &S, const Decl *D) const;
+  bool diagnoseAppertainsTo(class Sema &S, const Stmt *St) const;
   bool appliesToDecl(const Decl *D, attr::SubjectMatchRule MatchRule) const;
   void getMatchRules(const LangOptions &LangOpts,
                      SmallVectorImpl<std::pair<attr::SubjectMatchRule, bool>>
diff --git a/clang/include/clang/Sema/Scope.h b/clang/include/clang/Sema/Scope.h
index b7260f15fe1b9e96985d3683e0eca8f37868b5e5..b499ba1e7c2afd94f4c48b416e44695fad3333fd 100644
--- a/clang/include/clang/Sema/Scope.h
+++ b/clang/include/clang/Sema/Scope.h
@@ -129,11 +129,17 @@ public:
     /// This is a compound statement scope.
     CompoundStmtScope = 0x400000,
 
-    /// We are between inheritance colon and the real class/struct definition scope.
+    /// We are between inheritance colon and the real class/struct definition
+    /// scope.
     ClassInheritanceScope = 0x800000,
 
     /// This is the scope of a C++ catch statement.
     CatchScope = 0x1000000,
+
+    /// This is a scope in which a condition variable is currently being
+    /// parsed. If such a scope is a ContinueScope, it's invalid to jump to the
+    /// continue block from here.
+    ConditionVarScope = 0x2000000,
   };
 
 private:
@@ -247,6 +253,17 @@ public:
     return const_cast<Scope*>(this)->getContinueParent();
   }
 
+  // Set whether we're in the scope of a condition variable, where 'continue'
+  // is disallowed despite being a continue scope.
+  void setIsConditionVarScope(bool InConditionVarScope) {
+    Flags = (Flags & ~ConditionVarScope) |
+            (InConditionVarScope ? ConditionVarScope : 0);
+  }
+
+  bool isConditionVarScope() const {
+    return Flags & ConditionVarScope;
+  }
+
   /// getBreakParent - Return the closest scope that a break statement
   /// would be affected by.
   Scope *getBreakParent() {
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index a919740aa662f78fb7291164e66cd9e3ba107271..efabc78b45ba9de0ec55995448404741b104a7a6 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -286,14 +286,13 @@ public:
   }
 };
 
-/// Keeps track of expected type during expression parsing. The type is tied to
-/// a particular token, all functions that update or consume the type take a
-/// start location of the token they are looking at as a parameter. This allows
-/// to avoid updating the type on hot paths in the parser.
+/// Tracks expected type during expression parsing, for use in code completion.
+/// The type is tied to a particular token, all functions that update or consume
+/// the type take a start location of the token they are looking at as a
+/// parameter. This avoids updating the type on hot paths in the parser.
 class PreferredTypeBuilder {
 public:
-  PreferredTypeBuilder() = default;
-  explicit PreferredTypeBuilder(QualType Type) : Type(Type) {}
+  PreferredTypeBuilder(bool Enabled) : Enabled(Enabled) {}
 
   void enterCondition(Sema &S, SourceLocation Tok);
   void enterReturn(Sema &S, SourceLocation Tok);
@@ -307,6 +306,9 @@ public:
   /// Clients should be very careful when using this funciton, as it stores a
   /// function_ref, clients should make sure all calls to get() with the same
   /// location happen while function_ref is alive.
+  ///
+  /// The callback should also emit signature help as a side-effect, but only
+  /// if the completion point has been reached.
   void enterFunctionArgument(SourceLocation Tok,
                              llvm::function_ref<QualType()> ComputeType);
 
@@ -319,8 +321,14 @@ public:
   /// Handles all type casts, including C-style cast, C++ casts, etc.
   void enterTypeCast(SourceLocation Tok, QualType CastType);
 
+  /// Get the expected type associated with this location, if any.
+  ///
+  /// If the location is a function argument, determining the expected type
+  /// involves considering all function overloads and the arguments so far.
+  /// In this case, signature help for these function overloads will be reported
+  /// as a side-effect (only if the completion point has been reached).
   QualType get(SourceLocation Tok) const {
-    if (Tok != ExpectedLoc)
+    if (!Enabled || Tok != ExpectedLoc)
       return QualType();
     if (!Type.isNull())
       return Type;
@@ -330,6 +338,7 @@ public:
   }
 
 private:
+  bool Enabled;
   /// Start position of a token for which we store expected type.
   SourceLocation ExpectedLoc;
   /// Expected type for a token starting at ExpectedLoc.
@@ -2693,8 +2702,7 @@ public:
   void ActOnParamUnparsedDefaultArgument(Decl *param, SourceLocation EqualLoc,
                                          SourceLocation ArgLoc);
   void ActOnParamDefaultArgumentError(Decl *param, SourceLocation EqualLoc);
-  ExprResult ConvertParamDefaultArgument(const ParmVarDecl *Param,
-                                         Expr *DefaultArg,
+  ExprResult ConvertParamDefaultArgument(ParmVarDecl *Param, Expr *DefaultArg,
                                          SourceLocation EqualLoc);
   void SetParamDefaultArgument(ParmVarDecl *Param, Expr *DefaultArg,
                                SourceLocation EqualLoc);
@@ -4251,6 +4259,13 @@ public:
 
   void checkUnusedDeclAttributes(Declarator &D);
 
+  /// Handles semantic checking for features that are common to all attributes,
+  /// such as checking whether a parameter was properly specified, or the
+  /// correct number of arguments were passed, etc. Returns true if the
+  /// attribute has been diagnosed.
+  bool checkCommonAttributeFeatures(const Decl *D, const ParsedAttr &A);
+  bool checkCommonAttributeFeatures(const Stmt *S, const ParsedAttr &A);
+
   /// Determine if type T is a valid subject for a nonnull and similar
   /// attributes. By default, we look through references (the behavior used by
   /// nonnull), but if the second parameter is true, then we treat a reference
@@ -5565,6 +5580,9 @@ public:
   ExprResult ActOnAsTypeExpr(Expr *E, ParsedType ParsedDestTy,
                              SourceLocation BuiltinLoc,
                              SourceLocation RParenLoc);
+  ExprResult BuildAsTypeExpr(Expr *E, QualType DestTy,
+                             SourceLocation BuiltinLoc,
+                             SourceLocation RParenLoc);
 
   //===---------------------------- C++ Features --------------------------===//
 
@@ -10781,6 +10799,10 @@ public:
   StmtResult ActOnOpenMPTargetTeamsDistributeSimdDirective(
       ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
       SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp interop'.
+  StmtResult ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
+                                         SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
 
   /// Checks correctness of linear modifiers.
   bool CheckOpenMPLinearModifier(OpenMPLinearClauseKind LinKind,
@@ -10970,9 +10992,26 @@ public:
   /// Called on well-formed 'relaxed' clause.
   OMPClause *ActOnOpenMPRelaxedClause(SourceLocation StartLoc,
                                       SourceLocation EndLoc);
+
+  /// Called on well-formed 'init' clause.
+  OMPClause *ActOnOpenMPInitClause(Expr *InteropVar, ArrayRef<Expr *> PrefExprs,
+                                   bool IsTarget, bool IsTargetSync,
+                                   SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation VarLoc,
+                                   SourceLocation EndLoc);
+
+  /// Called on well-formed 'use' clause.
+  OMPClause *ActOnOpenMPUseClause(Expr *InteropVar, SourceLocation StartLoc,
+                                  SourceLocation LParenLoc,
+                                  SourceLocation VarLoc, SourceLocation EndLoc);
+
   /// Called on well-formed 'destroy' clause.
-  OMPClause *ActOnOpenMPDestroyClause(SourceLocation StartLoc,
+  OMPClause *ActOnOpenMPDestroyClause(Expr *InteropVar, SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation VarLoc,
                                       SourceLocation EndLoc);
+
   /// Called on well-formed 'threads' clause.
   OMPClause *ActOnOpenMPThreadsClause(SourceLocation StartLoc,
                                       SourceLocation EndLoc);
@@ -12216,8 +12255,14 @@ public:
                                       const VirtSpecifiers *VS = nullptr);
   void CodeCompleteBracketDeclarator(Scope *S);
   void CodeCompleteCase(Scope *S);
-  /// Reports signatures for a call to CodeCompleteConsumer and returns the
-  /// preferred type for the current argument. Returned type can be null.
+  /// Determines the preferred type of the current function argument, by
+  /// examining the signatures of all possible overloads.
+  /// Returns null if unknown or ambiguous, or if code completion is off.
+  ///
+  /// If the code completion point has been reached, also reports the function
+  /// signatures that were considered.
+  ///
+  /// FIXME: rename to GuessCallArgumentType to reduce confusion.
   QualType ProduceCallSignatureHelp(Scope *S, Expr *Fn, ArrayRef<Expr *> Args,
                                     SourceLocation OpenParLoc);
   QualType ProduceConstructorSignatureHelp(Scope *S, QualType Type,
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 2c4356a738f9556930fa0f08fd99356321500785..64f15e75bc2c895d6e3b0ca08aaf7269ca934af7 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1940,6 +1940,7 @@ enum StmtCode {
   STMT_OMP_TARGET_TEAMS_DISTRIBUTE_PARALLEL_FOR_DIRECTIVE,
   STMT_OMP_TARGET_TEAMS_DISTRIBUTE_PARALLEL_FOR_SIMD_DIRECTIVE,
   STMT_OMP_TARGET_TEAMS_DISTRIBUTE_SIMD_DIRECTIVE,
+  STMT_OMP_INTEROP_DIRECTIVE,
   EXPR_OMP_ARRAY_SECTION,
   EXPR_OMP_ARRAY_SHAPING,
   EXPR_OMP_ITERATOR,
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 12073a38a77a5171b0fda1275f30afc8ab5f82dc..ea67d6990a8a0b626282f0ed6f5ec361011173da 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -84,7 +84,7 @@ class RecordDecl;
 class Sema;
 class SourceManager;
 class Stmt;
-struct StoredDeclsList;
+class StoredDeclsList;
 class SwitchCase;
 class TemplateParameterList;
 class Token;
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h
index 58a88f452ed95dbcf2a6111c0a3977bef3ad7798..2975d50de33341d256823a792b2a9c0f3538abfd 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h
@@ -93,7 +93,7 @@ enum class TrackingKind {
   /// gathered about the tracked expression value as possible.
   Thorough,
   /// Specifies that a more moderate tracking should be used for the expression
-  /// value. This will essentially make sure that functions relevant to the it
+  /// value. This will essentially make sure that functions relevant to it
   /// aren't pruned, but otherwise relies on the user reading the code or
   /// following the arrows.
   Condition
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
index 062a604a7551bbeeb4682854d2cd683f7438badb..392bc484bf62c388cb76212e754d4da96efd3001 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
@@ -21,6 +21,7 @@ extern const char *const UnixAPI;
 extern const char *const CXXObjectLifecycle;
 extern const char *const CXXMoveSemantics;
 extern const char *const SecurityError;
+extern const char *const UnusedCode;
 } // namespace categories
 } // namespace ento
 } // namespace clang
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
index bc5d5f57cd68744787af81df33e902bfa9be9542..4a118074463d1b23a8e2d526fe0e9176f0d3d4f1 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
@@ -16,6 +16,8 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SimpleConstraintManager.h"
+#include "llvm/ADT/APSInt.h"
+#include "llvm/Support/Allocator.h"
 
 namespace clang {
 
@@ -24,21 +26,19 @@ namespace ento {
 /// A Range represents the closed range [from, to].  The caller must
 /// guarantee that from <= to.  Note that Range is immutable, so as not
 /// to subvert RangeSet's immutability.
-class Range : public std::pair<const llvm::APSInt *, const llvm::APSInt *> {
+class Range {
 public:
-  Range(const llvm::APSInt &from, const llvm::APSInt &to)
-      : std::pair<const llvm::APSInt *, const llvm::APSInt *>(&from, &to) {
-    assert(from <= to);
+  Range(const llvm::APSInt &From, const llvm::APSInt &To) : Impl(&From, &To) {
+    assert(From <= To);
   }
 
-  Range(const llvm::APSInt &point)
-      : std::pair<const llvm::APSInt *, const llvm::APSInt *>(&point, &point) {}
+  Range(const llvm::APSInt &Point) : Range(Point, Point) {}
 
-  bool Includes(const llvm::APSInt &v) const {
-    return *first <= v && v <= *second;
+  bool Includes(const llvm::APSInt &Point) const {
+    return From() <= Point && Point <= To();
   }
-  const llvm::APSInt &From() const { return *first; }
-  const llvm::APSInt &To() const { return *second; }
+  const llvm::APSInt &From() const { return *Impl.first; }
+  const llvm::APSInt &To() const { return *Impl.second; }
   const llvm::APSInt *getConcreteValue() const {
     return &From() == &To() ? &From() : nullptr;
   }
@@ -47,93 +47,264 @@ public:
     ID.AddPointer(&From());
     ID.AddPointer(&To());
   }
-};
+  void dump(raw_ostream &OS) const;
 
-class RangeTrait : public llvm::ImutContainerInfo<Range> {
-public:
-  // When comparing if one Range is less than another, we should compare
-  // the actual APSInt values instead of their pointers.  This keeps the order
-  // consistent (instead of comparing by pointer values) and can potentially
-  // be used to speed up some of the operations in RangeSet.
-  static inline bool isLess(key_type_ref lhs, key_type_ref rhs) {
-    return *lhs.first < *rhs.first ||
-           (!(*rhs.first < *lhs.first) && *lhs.second < *rhs.second);
-  }
+  // In order to keep non-overlapping ranges sorted, we can compare only From
+  // points.
+  bool operator<(const Range &RHS) const { return From() < RHS.From(); }
+
+  bool operator==(const Range &RHS) const { return Impl == RHS.Impl; }
+  bool operator!=(const Range &RHS) const { return !operator==(RHS); }
+
+private:
+  std::pair<const llvm::APSInt *, const llvm::APSInt *> Impl;
 };
 
-/// RangeSet contains a set of ranges. If the set is empty, then
-///  there the value of a symbol is overly constrained and there are no
-///  possible values for that symbol.
+/// @class RangeSet is a persistent set of non-overlapping ranges.
+///
+/// New RangeSet objects can be ONLY produced by RangeSet::Factory object, which
+/// also supports the most common operations performed on range sets.
+///
+/// Empty set corresponds to an overly constrained symbol meaning that there
+/// are no possible values for that symbol.
 class RangeSet {
-  typedef llvm::ImmutableSet<Range, RangeTrait> PrimRangeSet;
-  PrimRangeSet ranges; // no need to make const, since it is an
-                       // ImmutableSet - this allows default operator=
-                       // to work.
 public:
-  typedef PrimRangeSet::Factory Factory;
-  typedef PrimRangeSet::iterator iterator;
-
-  RangeSet(PrimRangeSet RS) : ranges(RS) {}
-
-  /// Create a new set with all ranges of this set and RS.
-  /// Possible intersections are not checked here.
-  RangeSet addRange(Factory &F, const RangeSet &RS) {
-    PrimRangeSet Ranges(RS.ranges);
-    for (const auto &range : ranges)
-      Ranges = F.add(Ranges, range);
-    return RangeSet(Ranges);
-  }
-
-  iterator begin() const { return ranges.begin(); }
-  iterator end() const { return ranges.end(); }
+  class Factory;
 
-  bool isEmpty() const { return ranges.isEmpty(); }
+private:
+  // We use llvm::SmallVector as the underlying container for the following
+  // reasons:
+  //
+  //   * Range sets are usually very simple, 1 or 2 ranges.
+  //     That's why llvm::ImmutableSet is not perfect.
+  //
+  //   * Ranges in sets are NOT overlapping, so it is natural to keep them
+  //     sorted for efficient operations and queries.  For this reason,
+  //     llvm::SmallSet doesn't fit the requirements, it is not sorted when it
+  //     is a vector.
+  //
+  //   * Range set operations usually a bit harder than add/remove a range.
+  //     Complex operations might do many of those for just one range set.
+  //     Formerly it used to be llvm::ImmutableSet, which is inefficient for our
+  //     purposes as we want to make these operations BOTH immutable AND
+  //     efficient.
+  //
+  //   * Iteration over ranges is widespread and a more cache-friendly
+  //     structure is preferred.
+  using ImplType = llvm::SmallVector<Range, 4>;
+
+  struct ContainerType : public ImplType, public llvm::FoldingSetNode {
+    void Profile(llvm::FoldingSetNodeID &ID) const {
+      for (const Range &It : *this) {
+        It.Profile(ID);
+      }
+    }
+  };
+  // This is a non-owning pointer to an actual container.
+  // The memory is fully managed by the factory and is alive as long as the
+  // factory itself is alive.
+  // It is a pointer as opposed to a reference, so we can easily reassign
+  // RangeSet objects.
+  using UnderlyingType = const ContainerType *;
+  UnderlyingType Impl;
 
-  /// Construct a new RangeSet representing '{ [from, to] }'.
-  RangeSet(Factory &F, const llvm::APSInt &from, const llvm::APSInt &to)
-      : ranges(F.add(F.getEmptySet(), Range(from, to))) {}
+public:
+  using const_iterator = ImplType::const_iterator;
+
+  const_iterator begin() const { return Impl->begin(); }
+  const_iterator end() const { return Impl->end(); }
+  size_t size() const { return Impl->size(); }
+
+  bool isEmpty() const { return Impl->empty(); }
+
+  class Factory {
+  public:
+    Factory(BasicValueFactory &BV) : ValueFactory(BV) {}
+
+    /// Create a new set with all ranges from both LHS and RHS.
+    /// Possible intersections are not checked here.
+    ///
+    /// Complexity: O(N + M)
+    ///             where N = size(LHS), M = size(RHS)
+    RangeSet add(RangeSet LHS, RangeSet RHS);
+    /// Create a new set with all ranges from the original set plus the new one.
+    /// Possible intersections are not checked here.
+    ///
+    /// Complexity: O(N)
+    ///             where N = size(Original)
+    RangeSet add(RangeSet Original, Range Element);
+    /// Create a new set with all ranges from the original set plus the point.
+    /// Possible intersections are not checked here.
+    ///
+    /// Complexity: O(N)
+    ///             where N = size(Original)
+    RangeSet add(RangeSet Original, const llvm::APSInt &Point);
+
+    RangeSet getEmptySet() { return &EmptySet; }
+
+    /// Create a new set with just one range.
+    /// @{
+    RangeSet getRangeSet(Range Origin);
+    RangeSet getRangeSet(const llvm::APSInt &From, const llvm::APSInt &To) {
+      return getRangeSet(Range(From, To));
+    }
+    RangeSet getRangeSet(const llvm::APSInt &Origin) {
+      return getRangeSet(Origin, Origin);
+    }
+    /// @}
+
+    /// Intersect the given range sets.
+    ///
+    /// Complexity: O(N + M)
+    ///             where N = size(LHS), M = size(RHS)
+    RangeSet intersect(RangeSet LHS, RangeSet RHS);
+    /// Intersect the given set with the closed range [Lower, Upper].
+    ///
+    /// Unlike the Range type, this range uses modular arithmetic, corresponding
+    /// to the common treatment of C integer overflow. Thus, if the Lower bound
+    /// is greater than the Upper bound, the range is taken to wrap around. This
+    /// is equivalent to taking the intersection with the two ranges [Min,
+    /// Upper] and [Lower, Max], or, alternatively, /removing/ all integers
+    /// between Upper and Lower.
+    ///
+    /// Complexity: O(N)
+    ///             where N = size(What)
+    RangeSet intersect(RangeSet What, llvm::APSInt Lower, llvm::APSInt Upper);
+    /// Intersect the given range with the given point.
+    ///
+    /// The result can be either an empty set or a set containing the given
+    /// point depending on whether the point is in the range set.
+    ///
+    /// Complexity: O(logN)
+    ///             where N = size(What)
+    RangeSet intersect(RangeSet What, llvm::APSInt Point);
+
+    /// Delete the given point from the range set.
+    ///
+    /// Complexity: O(N)
+    ///             where N = size(From)
+    RangeSet deletePoint(RangeSet From, const llvm::APSInt &Point);
+    /// Negate the given range set.
+    ///
+    /// Turn all [A, B] ranges to [-B, -A], when "-" is a C-like unary minus
+    /// operation under the values of the type.
+    ///
+    /// We also handle MIN because applying unary minus to MIN does not change
+    /// it.
+    /// Example 1:
+    /// char x = -128;        // -128 is a MIN value in a range of 'char'
+    /// char y = -x;          // y: -128
+    ///
+    /// Example 2:
+    /// unsigned char x = 0;  // 0 is a MIN value in a range of 'unsigned char'
+    /// unsigned char y = -x; // y: 0
+    ///
+    /// And it makes us to separate the range
+    /// like [MIN, N] to [MIN, MIN] U [-N, MAX].
+    /// For instance, whole range is {-128..127} and subrange is [-128,-126],
+    /// thus [-128,-127,-126,...] negates to [-128,...,126,127].
+    ///
+    /// Negate restores disrupted ranges on bounds,
+    /// e.g. [MIN, B] => [MIN, MIN] U [-B, MAX] => [MIN, B].
+    ///
+    /// Negate is a self-inverse function, i.e. negate(negate(R)) == R.
+    ///
+    /// Complexity: O(N)
+    ///             where N = size(What)
+    RangeSet negate(RangeSet What);
+
+  private:
+    /// Return a persistent version of the given container.
+    RangeSet makePersistent(ContainerType &&From);
+    /// Construct a new persistent version of the given container.
+    ContainerType *construct(ContainerType &&From);
+
+    RangeSet intersect(const ContainerType &LHS, const ContainerType &RHS);
+
+    // Many operations include producing new APSInt values and that's why
+    // we need this factory.
+    BasicValueFactory &ValueFactory;
+    // Allocator for all the created containers.
+    // Containers might own their own memory and that's why it is specific
+    // for the type, so it calls container destructors upon deletion.
+    llvm::SpecificBumpPtrAllocator<ContainerType> Arena;
+    // Usually we deal with the same ranges and range sets over and over.
+    // Here we track all created containers and try not to repeat ourselves.
+    llvm::FoldingSet<ContainerType> Cache;
+    static ContainerType EmptySet;
+  };
+
+  RangeSet(const RangeSet &) = default;
+  RangeSet &operator=(const RangeSet &) = default;
+  RangeSet(RangeSet &&) = default;
+  RangeSet &operator=(RangeSet &&) = default;
+  ~RangeSet() = default;
+
+  /// Construct a new RangeSet representing '{ [From, To] }'.
+  RangeSet(Factory &F, const llvm::APSInt &From, const llvm::APSInt &To)
+      : RangeSet(F.getRangeSet(From, To)) {}
 
   /// Construct a new RangeSet representing the given point as a range.
-  RangeSet(Factory &F, const llvm::APSInt &point) : RangeSet(F, point, point) {}
+  RangeSet(Factory &F, const llvm::APSInt &Point)
+      : RangeSet(F.getRangeSet(Point)) {}
+
+  static void Profile(llvm::FoldingSetNodeID &ID, const RangeSet &RS) {
+    ID.AddPointer(RS.Impl);
+  }
 
   /// Profile - Generates a hash profile of this RangeSet for use
   ///  by FoldingSet.
-  void Profile(llvm::FoldingSetNodeID &ID) const { ranges.Profile(ID); }
+  void Profile(llvm::FoldingSetNodeID &ID) const { Profile(ID, *this); }
 
   /// getConcreteValue - If a symbol is contrained to equal a specific integer
   ///  constant then this method returns that value.  Otherwise, it returns
   ///  NULL.
   const llvm::APSInt *getConcreteValue() const {
-    return ranges.isSingleton() ? ranges.begin()->getConcreteValue() : nullptr;
+    return Impl->size() == 1 ? begin()->getConcreteValue() : nullptr;
   }
 
-  /// Get a minimal value covered by the ranges in the set
+  /// Get the minimal value covered by the ranges in the set.
+  ///
+  /// Complexity: O(1)
   const llvm::APSInt &getMinValue() const;
-  /// Get a maximal value covered by the ranges in the set
+  /// Get the maximal value covered by the ranges in the set.
+  ///
+  /// Complexity: O(1)
   const llvm::APSInt &getMaxValue() const;
 
-private:
-  void IntersectInRange(BasicValueFactory &BV, Factory &F,
-                        const llvm::APSInt &Lower, const llvm::APSInt &Upper,
-                        PrimRangeSet &newRanges, PrimRangeSet::iterator &i,
-                        PrimRangeSet::iterator &e) const;
+  /// Test whether the given point is contained by any of the ranges.
+  ///
+  /// Complexity: O(logN)
+  ///             where N = size(this)
+  bool contains(llvm::APSInt Point) const { return containsImpl(Point); }
+
+  void dump(raw_ostream &OS) const;
+
+  bool operator==(const RangeSet &Other) const { return *Impl == *Other.Impl; }
+  bool operator!=(const RangeSet &Other) const { return !(*this == Other); }
 
+private:
+  /* implicit */ RangeSet(ContainerType *RawContainer) : Impl(RawContainer) {}
+  /* implicit */ RangeSet(UnderlyingType Ptr) : Impl(Ptr) {}
+
+  /// Pin given points to the type represented by the current range set.
+  ///
+  /// This makes parameter points to be in-out parameters.
+  /// In order to maintain consistent types across all of the ranges in the set
+  /// and to keep all the operations to compare ONLY points of the same type, we
+  /// need to pin every point before any operation.
+  ///
+  /// @Returns true if the given points can be converted to the target type
+  ///          without changing the values (i.e. trivially) and false otherwise.
+  /// @{
   bool pin(llvm::APSInt &Lower, llvm::APSInt &Upper) const;
+  bool pin(llvm::APSInt &Point) const;
+  /// @}
 
-public:
-  RangeSet Intersect(BasicValueFactory &BV, Factory &F, llvm::APSInt Lower,
-                     llvm::APSInt Upper) const;
-  RangeSet Intersect(BasicValueFactory &BV, Factory &F,
-                     const RangeSet &Other) const;
-  RangeSet Negate(BasicValueFactory &BV, Factory &F) const;
-  RangeSet Delete(BasicValueFactory &BV, Factory &F,
-                  const llvm::APSInt &Point) const;
-
-  void print(raw_ostream &os) const;
-
-  bool operator==(const RangeSet &other) const {
-    return ranges == other.ranges;
-  }
+  // This version of this function modifies its arguments (pins it).
+  bool containsImpl(llvm::APSInt &Point) const;
+
+  friend class Factory;
 };
 
 using ConstraintMap = llvm::ImmutableMap<SymbolRef, RangeSet>;
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
index 1c106ed4b765a90d7ab10f6263254189632f30de..b4fa27f531e328697c21d5cad7b29a923b74714d 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
@@ -22,19 +22,10 @@ namespace dependencies{
 
 /// The full dependencies and module graph for a specific input.
 struct FullDependencies {
-  /// The name of the C++20 module this translation unit exports. This may
-  /// include `:` for C++20 module partitons.
+  /// The identifier of the C++20 module this translation unit exports.
   ///
-  /// If the translation unit is not a module then this will be empty.
-  std::string ExportedModuleName;
-
-  /// The context hash represents the set of compiler options that may make one
-  /// version of a module incompatible with another. This includes things like
-  /// language mode, predefined macros, header search paths, etc...
-  ///
-  /// Modules with the same name but a different \c ContextHash should be
-  /// treated as separate modules for the purpose of a build.
-  std::string ContextHash;
+  /// If the translation unit is not a module then \c ID.ModuleName is empty.
+  ModuleID ID;
 
   /// A collection of absolute paths to files that this translation unit
   /// directly depends on, not including transitive dependencies.
@@ -45,7 +36,7 @@ struct FullDependencies {
   ///
   /// This may include modules with a different context hash when it can be
   /// determined that the differences are benign for this compilation.
-  std::vector<ClangModuleDep> ClangModuleDeps;
+  std::vector<ModuleID> ClangModuleDeps;
 
   /// A partial addtional set of command line arguments that can be used to
   /// build this translation unit.
@@ -65,8 +56,8 @@ struct FullDependencies {
   ///                         transitive set of dependencies for this
   ///                         compilation.
   std::vector<std::string> getAdditionalCommandLine(
-      std::function<StringRef(ClangModuleDep)> LookupPCMPath,
-      std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const;
+      std::function<StringRef(ModuleID)> LookupPCMPath,
+      std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const;
 };
 
 struct FullDependenciesResult {
diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
index c490bb38c167d07333710ead792941646f26d3f5..87bb1b86c279e4ec7aeb1ffd15636ce0db3205e3 100644
--- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
+++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
@@ -28,16 +28,9 @@ namespace dependencies {
 
 class DependencyConsumer;
 
-/// This is used to refer to a specific module.
-///
-/// See \c ModuleDeps for details about what these members mean.
-struct ClangModuleDep {
-  std::string ModuleName;
-  std::string ContextHash;
-};
-
-struct ModuleDeps {
-  /// The name of the module. This may include `:` for C++20 module partitons,
+/// This is used to identify a specific module.
+struct ModuleID {
+  /// The name of the module. This may include `:` for C++20 module partitions,
   /// or a header-name for C++20 header units.
   std::string ModuleName;
 
@@ -48,6 +41,11 @@ struct ModuleDeps {
   /// Modules with the same name but a different \c ContextHash should be
   /// treated as separate modules for the purpose of a build.
   std::string ContextHash;
+};
+
+struct ModuleDeps {
+  /// The identifier of the module.
+  ModuleID ID;
 
   /// The path to the modulemap file which defines this module.
   ///
@@ -62,12 +60,12 @@ struct ModuleDeps {
   /// on, not including transitive dependencies.
   llvm::StringSet<> FileDeps;
 
-  /// A list of modules this module directly depends on, not including
-  /// transitive dependencies.
+  /// A list of module identifiers this module directly depends on, not
+  /// including transitive dependencies.
   ///
   /// This may include modules with a different context hash when it can be
   /// determined that the differences are benign for this compilation.
-  std::vector<ClangModuleDep> ClangModuleDeps;
+  std::vector<ModuleID> ClangModuleDeps;
 
   /// A partial command line that can be used to build this module.
   ///
@@ -89,8 +87,8 @@ struct ModuleDeps {
   ///                         transitive set of dependencies for this
   ///                         compilation.
   std::vector<std::string> getFullCommandLine(
-      std::function<StringRef(ClangModuleDep)> LookupPCMPath,
-      std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const;
+      std::function<StringRef(ModuleID)> LookupPCMPath,
+      std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const;
 };
 
 namespace detail {
@@ -98,14 +96,18 @@ namespace detail {
 /// modules in \c Modules transitively, along with other needed arguments to
 /// use explicitly built modules.
 void appendCommonModuleArguments(
-    llvm::ArrayRef<ClangModuleDep> Modules,
-    std::function<StringRef(ClangModuleDep)> LookupPCMPath,
-    std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps,
+    llvm::ArrayRef<ModuleID> Modules,
+    std::function<StringRef(ModuleID)> LookupPCMPath,
+    std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps,
     std::vector<std::string> &Result);
 } // namespace detail
 
 class ModuleDepCollector;
 
+/// Callback that records textual includes and direct modular includes/imports
+/// during preprocessing. At the end of the main file, it also collects
+/// transitive modular dependencies and passes everything to the
+/// \c DependencyConsumer of the parent \c ModuleDepCollector.
 class ModuleDepCollectorPP final : public PPCallbacks {
 public:
   ModuleDepCollectorPP(CompilerInstance &I, ModuleDepCollector &MDC)
@@ -126,11 +128,18 @@ public:
   void EndOfMainFile() override;
 
 private:
+  /// The compiler instance for the current translation unit.
   CompilerInstance &Instance;
+  /// The parent dependency collector.
   ModuleDepCollector &MDC;
-  llvm::DenseSet<const Module *> DirectDeps;
+  /// Working set of direct modular dependencies.
+  llvm::DenseSet<const Module *> DirectModularDeps;
 
   void handleImport(const Module *Imported);
+
+  /// Traverses the previously collected direct modular dependencies to discover
+  /// transitive modular dependencies and fills the parent \c ModuleDepCollector
+  /// with both.
   void handleTopLevelModule(const Module *M);
   void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD,
                            llvm::DenseSet<const Module *> &AddedModules);
@@ -138,6 +147,8 @@ private:
                     llvm::DenseSet<const Module *> &AddedModules);
 };
 
+/// Collects modular and non-modular dependencies of the main file by attaching
+/// \c ModuleDepCollectorPP to the preprocessor.
 class ModuleDepCollector final : public DependencyCollector {
 public:
   ModuleDepCollector(std::unique_ptr<DependencyOutputOptions> Opts,
@@ -149,12 +160,20 @@ public:
 private:
   friend ModuleDepCollectorPP;
 
+  /// The compiler instance for the current translation unit.
   CompilerInstance &Instance;
+  /// The consumer of collected dependency information.
   DependencyConsumer &Consumer;
+  /// Path to the main source file.
   std::string MainFile;
+  /// The module hash identifying the compilation conditions.
   std::string ContextHash;
-  std::vector<std::string> MainDeps;
-  std::unordered_map<std::string, ModuleDeps> Deps;
+  /// Non-modular file dependencies. This includes the main source file and
+  /// textually included header files.
+  std::vector<std::string> FileDeps;
+  /// Direct and transitive modular dependencies of the main source file.
+  std::unordered_map<std::string, ModuleDeps> ModularDeps;
+  /// Options that control the dependency output generation.
   std::unique_ptr<DependencyOutputOptions> Opts;
 };
 
diff --git a/clang/include/clang/Tooling/NodeIntrospection.h b/clang/include/clang/Tooling/NodeIntrospection.h
index abaa58b674a1c910712c5d0fd1fc2962bcb9cda8..3b40e68df24e7fb41c9559563c3d4d5172dcfca4 100644
--- a/clang/include/clang/Tooling/NodeIntrospection.h
+++ b/clang/include/clang/Tooling/NodeIntrospection.h
@@ -22,6 +22,7 @@
 namespace clang {
 
 class Stmt;
+class Decl;
 
 namespace tooling {
 
@@ -78,6 +79,7 @@ struct NodeLocationAccessors {
 
 namespace NodeIntrospection {
 NodeLocationAccessors GetLocations(clang::Stmt const *Object);
+NodeLocationAccessors GetLocations(clang::Decl const *Object);
 NodeLocationAccessors GetLocations(clang::DynTypedNode const &Node);
 } // namespace NodeIntrospection
 } // namespace tooling
diff --git a/clang/include/clang/Tooling/Syntax/Tokens.h b/clang/include/clang/Tooling/Syntax/Tokens.h
index 98320bd54d6f6c1ae60add6d1ef87cda6b0f8299..e4bc1553c2d6032e1e6fc8dee2ec6a77ba34c40e 100644
--- a/clang/include/clang/Tooling/Syntax/Tokens.h
+++ b/clang/include/clang/Tooling/Syntax/Tokens.h
@@ -34,6 +34,7 @@
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Lex/Token.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
@@ -192,8 +193,13 @@ public:
     return ExpandedTokens;
   }
 
+  /// Builds a cache to make future calls to expandedToken(SourceRange) faster.
+  /// Creates an index only once. Further calls to it will be no-op.
+  void indexExpandedTokens();
+
   /// Returns the subrange of expandedTokens() corresponding to the closed
   /// token range R.
+  /// Consider calling indexExpandedTokens() before for faster lookups.
   llvm::ArrayRef<syntax::Token> expandedTokens(SourceRange R) const;
 
   /// Returns the subrange of spelled tokens corresponding to AST node spanning
@@ -366,6 +372,8 @@ private:
   /// same stream as 'clang -E' (excluding the preprocessor directives like
   /// #file, etc.).
   std::vector<syntax::Token> ExpandedTokens;
+  // Index of ExpandedTokens for faster lookups by SourceLocation.
+  llvm::DenseMap<SourceLocation, unsigned> ExpandedTokIndex;
   llvm::DenseMap<FileID, MarkedFile> Files;
   // The value is never null, pointer instead of reference to avoid disabling
   // implicit assignment operator.
diff --git a/clang/lib/ARCMigrate/ObjCMT.cpp b/clang/lib/ARCMigrate/ObjCMT.cpp
index 68a51a49c71823ac89ac97e71e0066642d976e92..c8069b51567c2eeabe180ed7834d4dfa4534561c 100644
--- a/clang/lib/ARCMigrate/ObjCMT.cpp
+++ b/clang/lib/ARCMigrate/ObjCMT.cpp
@@ -613,7 +613,7 @@ ClassImplementsAllMethodsAndProperties(ASTContext &Ctx,
         continue;
       HasAtleastOneRequiredProperty = true;
       DeclContext::lookup_result R = IDecl->lookup(Property->getDeclName());
-      if (R.size() == 0) {
+      if (R.empty()) {
         // Relax the rule and look into class's implementation for a synthesize
         // or dynamic declaration. Class is implementing a property coming from
         // another protocol. This still makes the target protocol as conforming.
@@ -621,14 +621,12 @@ ClassImplementsAllMethodsAndProperties(ASTContext &Ctx,
                                   Property->getDeclName().getAsIdentifierInfo(),
                                   Property->getQueryKind()))
           return false;
-      }
-      else if (ObjCPropertyDecl *ClassProperty = dyn_cast<ObjCPropertyDecl>(R[0])) {
-          if ((ClassProperty->getPropertyAttributes()
-              != Property->getPropertyAttributes()) ||
-              !Ctx.hasSameType(ClassProperty->getType(), Property->getType()))
-            return false;
-      }
-      else
+      } else if (auto *ClassProperty = R.find_first<ObjCPropertyDecl>()) {
+        if ((ClassProperty->getPropertyAttributes() !=
+             Property->getPropertyAttributes()) ||
+            !Ctx.hasSameType(ClassProperty->getType(), Property->getType()))
+          return false;
+      } else
         return false;
     }
 
@@ -645,12 +643,12 @@ ClassImplementsAllMethodsAndProperties(ASTContext &Ctx,
       if (MD->getImplementationControl() == ObjCMethodDecl::Optional)
         continue;
       DeclContext::lookup_result R = ImpDecl->lookup(MD->getDeclName());
-      if (R.size() == 0)
+      if (R.empty())
         return false;
       bool match = false;
       HasAtleastOneRequiredMethod = true;
-      for (unsigned I = 0, N = R.size(); I != N; ++I)
-        if (ObjCMethodDecl *ImpMD = dyn_cast<ObjCMethodDecl>(R[0]))
+      for (NamedDecl *ND : R)
+        if (ObjCMethodDecl *ImpMD = dyn_cast<ObjCMethodDecl>(ND))
           if (Ctx.ObjCMethodsAreEqual(MD, ImpMD)) {
             match = true;
             break;
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 3de0262fc19282394b611ffc16d75d9aa458779c..5a8b71f5c9b0cb08becb42c678ef02b43815db9b 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -10970,6 +10970,9 @@ void ASTContext::forEachMultiversionedFunctionVersion(
   assert(FD->isMultiVersion() && "Only valid for multiversioned functions");
   llvm::SmallDenseSet<const FunctionDecl*, 4> SeenDecls;
   FD = FD->getMostRecentDecl();
+  // FIXME: The order of traversal here matters and depends on the order of
+  // lookup results, which happens to be (mostly) oldest-to-newest, but we
+  // shouldn't rely on that.
   for (auto *CurDecl :
        FD->getDeclContext()->getRedeclContext()->lookup(FD->getDeclName())) {
     FunctionDecl *CurFD = CurDecl->getAsFunction()->getMostRecentDecl();
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index f4dfc54b36cbb1eaafcfeea96d94a735a4a91d3a..ef7a3ea8a66c2173b55201281b62b97469d3f3a9 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -358,6 +358,8 @@ namespace clang {
     ExpectedType VisitDecltypeType(const DecltypeType *T);
     ExpectedType VisitUnaryTransformType(const UnaryTransformType *T);
     ExpectedType VisitAutoType(const AutoType *T);
+    ExpectedType VisitDeducedTemplateSpecializationType(
+        const DeducedTemplateSpecializationType *T);
     ExpectedType VisitInjectedClassNameType(const InjectedClassNameType *T);
     // FIXME: DependentDecltypeType
     ExpectedType VisitRecordType(const RecordType *T);
@@ -574,6 +576,7 @@ namespace clang {
 
     // Importing expressions
     ExpectedStmt VisitExpr(Expr *E);
+    ExpectedStmt VisitSourceLocExpr(SourceLocExpr *E);
     ExpectedStmt VisitVAArgExpr(VAArgExpr *E);
     ExpectedStmt VisitChooseExpr(ChooseExpr *E);
     ExpectedStmt VisitGNUNullExpr(GNUNullExpr *E);
@@ -1375,6 +1378,20 @@ ExpectedType ASTNodeImporter::VisitAutoType(const AutoType *T) {
       ToTemplateArgs);
 }
 
+ExpectedType ASTNodeImporter::VisitDeducedTemplateSpecializationType(
+    const DeducedTemplateSpecializationType *T) {
+  // FIXME: Make sure that the "to" context supports C++17!
+  Expected<TemplateName> ToTemplateNameOrErr = import(T->getTemplateName());
+  if (!ToTemplateNameOrErr)
+    return ToTemplateNameOrErr.takeError();
+  ExpectedType ToDeducedTypeOrErr = import(T->getDeducedType());
+  if (!ToDeducedTypeOrErr)
+    return ToDeducedTypeOrErr.takeError();
+
+  return Importer.getToContext().getDeducedTemplateSpecializationType(
+      *ToTemplateNameOrErr, *ToDeducedTypeOrErr, T->isDependentType());
+}
+
 ExpectedType ASTNodeImporter::VisitInjectedClassNameType(
     const InjectedClassNameType *T) {
   Expected<CXXRecordDecl *> ToDeclOrErr = import(T->getDecl());
@@ -3631,6 +3648,10 @@ ExpectedDecl ASTNodeImporter::VisitFieldDecl(FieldDecl *D) {
   auto ToInitializer = importChecked(Err, D->getInClassInitializer());
   if (Err)
     return std::move(Err);
+  const Type *ToCapturedVLAType = nullptr;
+  if (Error Err = Importer.importInto(
+          ToCapturedVLAType, cast_or_null<Type>(D->getCapturedVLAType())))
+    return std::move(Err);
 
   FieldDecl *ToField;
   if (GetImportedOrCreateDecl(ToField, D, Importer.getToContext(), DC,
@@ -3644,6 +3665,8 @@ ExpectedDecl ASTNodeImporter::VisitFieldDecl(FieldDecl *D) {
   if (ToInitializer)
     ToField->setInClassInitializer(ToInitializer);
   ToField->setImplicit(D->isImplicit());
+  if (ToCapturedVLAType)
+    ToField->setCapturedVLAType(cast<VariableArrayType>(ToCapturedVLAType));
   LexicalDC->addDeclInternal(ToField);
   return ToField;
 }
@@ -4018,6 +4041,7 @@ ExpectedDecl ASTNodeImporter::VisitVarDecl(VarDecl *D) {
                               D->getStorageClass()))
     return ToVar;
 
+  ToVar->setTSCSpec(D->getTSCSpec());
   ToVar->setQualifierInfo(ToQualifierLoc);
   ToVar->setAccess(D->getAccess());
   ToVar->setLexicalDeclContext(LexicalDC);
@@ -5064,6 +5088,11 @@ ExpectedDecl ASTNodeImporter::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
   auto FoundDecls = Importer.findDeclsInToCtx(DC, Name);
   for (auto *FoundDecl : FoundDecls) {
     if (auto *FoundProp = dyn_cast<ObjCPropertyDecl>(FoundDecl)) {
+      // Instance and class properties can share the same name but are different
+      // declarations.
+      if (FoundProp->isInstanceProperty() != D->isInstanceProperty())
+        continue;
+
       // Check property types.
       if (!Importer.IsStructurallyEquivalent(D->getType(),
                                              FoundProp->getType())) {
@@ -6482,6 +6511,21 @@ ExpectedStmt ASTNodeImporter::VisitExpr(Expr *E) {
   return make_error<ImportError>(ImportError::UnsupportedConstruct);
 }
 
+ExpectedStmt ASTNodeImporter::VisitSourceLocExpr(SourceLocExpr *E) {
+  Error Err = Error::success();
+  auto BLoc = importChecked(Err, E->getBeginLoc());
+  auto RParenLoc = importChecked(Err, E->getEndLoc());
+  if (Err)
+    return std::move(Err);
+  auto ParentContextOrErr = Importer.ImportContext(E->getParentContext());
+  if (!ParentContextOrErr)
+    return ParentContextOrErr.takeError();
+
+  return new (Importer.getToContext())
+      SourceLocExpr(Importer.getToContext(), E->getIdentKind(), BLoc, RParenLoc,
+                    *ParentContextOrErr);
+}
+
 ExpectedStmt ASTNodeImporter::VisitVAArgExpr(VAArgExpr *E) {
 
   Error Err = Error::success();
@@ -8151,28 +8195,37 @@ ASTImporter::Import(ExprWithCleanups::CleanupObject From) {
   return make_error<ImportError>(ImportError::UnsupportedConstruct);
 }
 
-Expected<QualType> ASTImporter::Import(QualType FromT) {
-  if (FromT.isNull())
-    return QualType{};
-
-  const Type *FromTy = FromT.getTypePtr();
+Expected<const Type *> ASTImporter::Import(const Type *FromT) {
+  if (!FromT)
+    return FromT;
 
   // Check whether we've already imported this type.
-  llvm::DenseMap<const Type *, const Type *>::iterator Pos
-    = ImportedTypes.find(FromTy);
+  llvm::DenseMap<const Type *, const Type *>::iterator Pos =
+      ImportedTypes.find(FromT);
   if (Pos != ImportedTypes.end())
-    return ToContext.getQualifiedType(Pos->second, FromT.getLocalQualifiers());
+    return Pos->second;
 
   // Import the type
   ASTNodeImporter Importer(*this);
-  ExpectedType ToTOrErr = Importer.Visit(FromTy);
+  ExpectedType ToTOrErr = Importer.Visit(FromT);
   if (!ToTOrErr)
     return ToTOrErr.takeError();
 
   // Record the imported type.
-  ImportedTypes[FromTy] = (*ToTOrErr).getTypePtr();
+  ImportedTypes[FromT] = ToTOrErr->getTypePtr();
+
+  return ToTOrErr->getTypePtr();
+}
+
+Expected<QualType> ASTImporter::Import(QualType FromT) {
+  if (FromT.isNull())
+    return QualType{};
+
+  Expected<const Type *> ToTyOrErr = Import(FromT.getTypePtr());
+  if (!ToTyOrErr)
+    return ToTyOrErr.takeError();
 
-  return ToContext.getQualifiedType(*ToTOrErr, FromT.getLocalQualifiers());
+  return ToContext.getQualifiedType(*ToTyOrErr, FromT.getLocalQualifiers());
 }
 
 Expected<TypeSourceInfo *> ASTImporter::Import(TypeSourceInfo *FromTSI) {
diff --git a/clang/lib/AST/CXXInheritance.cpp b/clang/lib/AST/CXXInheritance.cpp
index c87bcf31d1209c5762a243d2da7bb4d9d16f1b02..9027fa7a7515d34407f09e909de385abc03c9efe 100644
--- a/clang/lib/AST/CXXInheritance.cpp
+++ b/clang/lib/AST/CXXInheritance.cpp
@@ -386,9 +386,9 @@ static bool isOrdinaryMember(const NamedDecl *ND) {
 
 static bool findOrdinaryMember(const CXXRecordDecl *RD, CXXBasePath &Path,
                                DeclarationName Name) {
-  Path.Decls = RD->lookup(Name);
-  for (NamedDecl *ND : Path.Decls)
-    if (isOrdinaryMember(ND))
+  Path.Decls = RD->lookup(Name).begin();
+  for (DeclContext::lookup_iterator I = Path.Decls, E = I.end(); I != E; ++I)
+    if (isOrdinaryMember(*I))
       return true;
 
   return false;
@@ -453,9 +453,10 @@ std::vector<const NamedDecl *> CXXRecordDecl::lookupDependentName(
           },
           Paths, /*LookupInDependent=*/true))
     return Results;
-  for (const NamedDecl *ND : Paths.front().Decls) {
-    if (isOrdinaryMember(ND) && Filter(ND))
-      Results.push_back(ND);
+  for (DeclContext::lookup_iterator I = Paths.front().Decls, E = I.end();
+       I != E; ++I) {
+    if (isOrdinaryMember(*I) && Filter(*I))
+      Results.push_back(*I);
   }
   return Results;
 }
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 0f6ae2eacc3b4bd17b3a43bdd0d19d95d0efb92e..a32540e7f3b07da8df529a23f2bdeb781b0f870b 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -1612,8 +1612,7 @@ void NamedDecl::printNestedNameSpecifier(raw_ostream &OS,
 
     // Suppress inline namespace if it doesn't make the result ambiguous.
     if (P.SuppressInlineNamespace && Ctx->isInlineNamespace() && NameInScope &&
-        Ctx->lookup(NameInScope).size() ==
-            Ctx->getParent()->lookup(NameInScope).size())
+        cast<NamespaceDecl>(Ctx)->isRedundantInlineQualifierFor(NameInScope))
       continue;
 
     // Skip non-named contexts such as linkage specifications and ExportDecls.
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index c26d6d1a42ea03de24c5ea5c81c9b108d37a0be8..6d438cf05590af9aeeac07b52edb63850f55ebdb 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -1394,39 +1394,7 @@ ExternalASTSource::SetExternalVisibleDeclsForName(const DeclContext *DC,
     DC->reconcileExternalVisibleStorage();
 
   StoredDeclsList &List = (*Map)[Name];
-
-  // Clear out any old external visible declarations, to avoid quadratic
-  // performance in the redeclaration checks below.
-  List.removeExternalDecls();
-
-  if (!List.isNull()) {
-    // We have both existing declarations and new declarations for this name.
-    // Some of the declarations may simply replace existing ones. Handle those
-    // first.
-    llvm::SmallVector<unsigned, 8> Skip;
-    for (unsigned I = 0, N = Decls.size(); I != N; ++I)
-      if (List.HandleRedeclaration(Decls[I], /*IsKnownNewer*/false))
-        Skip.push_back(I);
-    Skip.push_back(Decls.size());
-
-    // Add in any new declarations.
-    unsigned SkipPos = 0;
-    for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
-      if (I == Skip[SkipPos])
-        ++SkipPos;
-      else
-        List.AddSubsequentDecl(Decls[I]);
-    }
-  } else {
-    // Convert the array to a StoredDeclsList.
-    for (auto *D : Decls) {
-      if (List.isNull())
-        List.setOnlyValue(D);
-      else
-        List.AddSubsequentDecl(D);
-    }
-  }
-
+  List.replaceExternalDecls(Decls);
   return List.getLookupResult();
 }
 
@@ -1538,10 +1506,7 @@ void DeclContext::removeDecl(Decl *D) {
       if (Map) {
         StoredDeclsMap::iterator Pos = Map->find(ND->getDeclName());
         assert(Pos != Map->end() && "no lookup entry for decl");
-        // Remove the decl only if it is contained.
-        StoredDeclsList::DeclsTy *Vec = Pos->second.getAsVector();
-        if ((Vec && is_contained(*Vec, ND)) || Pos->second.getAsDecl() == ND)
-          Pos->second.remove(ND);
+        Pos->second.remove(ND);
       }
     } while (DC->isTransparentContext() && (DC = DC->getParent()));
   }
@@ -1658,8 +1623,6 @@ void DeclContext::buildLookupImpl(DeclContext *DCtx, bool Internal) {
   }
 }
 
-NamedDecl *const DeclContextLookupResult::SingleElementDummyList = nullptr;
-
 DeclContext::lookup_result
 DeclContext::lookup(DeclarationName Name) const {
   assert(getDeclKind() != Decl::LinkageSpec &&
@@ -1935,23 +1898,11 @@ void DeclContext::makeDeclVisibleInContextImpl(NamedDecl *D, bool Internal) {
     // In this case, we never try to replace an existing declaration; we'll
     // handle that when we finalize the list of declarations for this name.
     DeclNameEntries.setHasExternalDecls();
-    DeclNameEntries.AddSubsequentDecl(D);
-    return;
-  }
-
-  if (DeclNameEntries.isNull()) {
-    DeclNameEntries.setOnlyValue(D);
-    return;
-  }
-
-  if (DeclNameEntries.HandleRedeclaration(D, /*IsKnownNewer*/!Internal)) {
-    // This declaration has replaced an existing one for which
-    // declarationReplaces returns true.
+    DeclNameEntries.prependDeclNoReplace(D);
     return;
   }
 
-  // Put this declaration into the appropriate slot.
-  DeclNameEntries.AddSubsequentDecl(D);
+  DeclNameEntries.addOrReplaceDecl(D);
 }
 
 UsingDirectiveDecl *DeclContext::udir_iterator::operator*() const {
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 4213beb915af157b88288de55a06923639306ede..624b1bfde4e64e902362704743b369f34dc89054 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -12472,25 +12472,6 @@ void DataRecursiveIntBinOpEvaluator::process(EvalResult &Result) {
 }
 
 namespace {
-/// Used when we determine that we should fail, but can keep evaluating prior to
-/// noting that we had a failure.
-class DelayedNoteFailureRAII {
-  EvalInfo &Info;
-  bool NoteFailure;
-
-public:
-  DelayedNoteFailureRAII(EvalInfo &Info, bool NoteFailure = true)
-      : Info(Info), NoteFailure(NoteFailure) {}
-  ~DelayedNoteFailureRAII() {
-    if (NoteFailure) {
-      bool ContinueAfterFailure = Info.noteFailure();
-      (void)ContinueAfterFailure;
-      assert(ContinueAfterFailure &&
-             "Shouldn't have kept evaluating on failure.");
-    }
-  }
-};
-
 enum class CmpResult {
   Unequal,
   Less,
@@ -12858,12 +12839,14 @@ bool RecordExprEvaluator::VisitBinCmp(const BinaryOperator *E) {
 }
 
 bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
-  // We don't call noteFailure immediately because the assignment happens after
-  // we evaluate LHS and RHS.
-  if (!Info.keepEvaluatingAfterFailure() && E->isAssignmentOp())
-    return Error(E);
+  // We don't support assignment in C. C++ assignments don't get here because
+  // assignment is an lvalue in C++.
+  if (E->isAssignmentOp()) {
+    Error(E);
+    if (!Info.noteFailure())
+      return false;
+  }
 
-  DelayedNoteFailureRAII MaybeNoteFailureLater(Info, E->isAssignmentOp());
   if (DataRecursiveIntBinOpEvaluator::shouldEnqueue(E))
     return DataRecursiveIntBinOpEvaluator(*this, Result).Traverse(E);
 
diff --git a/clang/lib/AST/ExternalASTMerger.cpp b/clang/lib/AST/ExternalASTMerger.cpp
index 88bbe90a4e909bf25819c37f887f86ba5abef933..c7789b707b21997efc12e30f09e5f381e1c98e56 100644
--- a/clang/lib/AST/ExternalASTMerger.cpp
+++ b/clang/lib/AST/ExternalASTMerger.cpp
@@ -64,24 +64,24 @@ LookupSameContext(Source<TranslationUnitDecl *> SourceTU, const DeclContext *DC,
   Source<DeclarationName> SourceName = *SourceNameOrErr;
   DeclContext::lookup_result SearchResult =
       SourceParentDC.get()->lookup(SourceName.get());
-  size_t SearchResultSize = SearchResult.size();
-  if (SearchResultSize == 0 || SearchResultSize > 1) {
-    // There are two cases here.  First, we might not find the name.
-    // We might also find multiple copies, in which case we have no
-    // guarantee that the one we wanted is the one we pick.  (E.g.,
-    // if we have two specializations of the same template it is
-    // very hard to determine which is the one you want.)
-    //
-    // The Origins map fixes this problem by allowing the origin to be
-    // explicitly recorded, so we trigger that recording by returning
-    // nothing (rather than a possibly-inaccurate guess) here.
-    return nullptr;
-  } else {
-    NamedDecl *SearchResultDecl = SearchResult[0];
+
+  // There are two cases here. First, we might not find the name.
+  // We might also find multiple copies, in which case we have no
+  // guarantee that the one we wanted is the one we pick.  (E.g.,
+  // if we have two specializations of the same template it is
+  // very hard to determine which is the one you want.)
+  //
+  // The Origins map fixes this problem by allowing the origin to be
+  // explicitly recorded, so we trigger that recording by returning
+  // nothing (rather than a possibly-inaccurate guess) here.
+  if (SearchResult.isSingleResult()) {
+    NamedDecl *SearchResultDecl = SearchResult.front();
     if (isa<DeclContext>(SearchResultDecl) &&
         SearchResultDecl->getKind() == DC->getDeclKind())
       return cast<DeclContext>(SearchResultDecl)->getPrimaryContext();
     return nullptr; // This type of lookup is unsupported
+  } else {
+    return nullptr;
   }
 }
 
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index ba96fda6cd57a92fd0f782ca0d19b62c7f829e16..3e6e29207f08cd9bda24b810978e1a17b6072cff 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -640,7 +640,7 @@ bool ItaniumMangleContextImpl::isUniqueInternalLinkageDecl(
 
   // For C functions without prototypes, return false as their
   // names should not be mangled.
-  if (!FD->getType()->getAs<FunctionProtoType>())
+  if (!FD->hasPrototype())
     return false;
 
   if (isInternalLinkageDecl(ND))
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index e2da71b211a4e057ac1fb53a4cca40848837401a..254b426064087e89940f2446cec4d3d3f4458ae0 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1513,6 +1513,27 @@ OMPAffinityClause *OMPAffinityClause::CreateEmpty(const ASTContext &C,
   return new (Mem) OMPAffinityClause(N);
 }
 
+OMPInitClause *OMPInitClause::Create(const ASTContext &C, Expr *InteropVar,
+                                     ArrayRef<Expr *> PrefExprs, bool IsTarget,
+                                     bool IsTargetSync, SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation VarLoc,
+                                     SourceLocation EndLoc) {
+
+  void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(PrefExprs.size() + 1));
+  auto *Clause =
+      new (Mem) OMPInitClause(IsTarget, IsTargetSync, StartLoc, LParenLoc,
+                              VarLoc, EndLoc, PrefExprs.size() + 1);
+  Clause->setInteropVar(InteropVar);
+  llvm::copy(PrefExprs, Clause->getTrailingObjects<Expr *>() + 1);
+  return Clause;
+}
+
+OMPInitClause *OMPInitClause::CreateEmpty(const ASTContext &C, unsigned N) {
+  void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(N));
+  return new (Mem) OMPInitClause(N);
+}
+
 //===----------------------------------------------------------------------===//
 //  OpenMP clauses printing methods
 //===----------------------------------------------------------------------===//
@@ -1755,8 +1776,44 @@ void OMPClausePrinter::VisitOMPHintClause(OMPHintClause *Node) {
   OS << ")";
 }
 
-void OMPClausePrinter::VisitOMPDestroyClause(OMPDestroyClause *) {
+void OMPClausePrinter::VisitOMPInitClause(OMPInitClause *Node) {
+  OS << "init(";
+  bool First = true;
+  for (const Expr *E : Node->prefs()) {
+    if (First)
+      OS << "prefer_type(";
+    else
+      OS << ",";
+    E->printPretty(OS, nullptr, Policy);
+    First = false;
+  }
+  if (!First)
+    OS << "), ";
+  if (Node->getIsTarget())
+    OS << "target";
+  if (Node->getIsTargetSync()) {
+    if (Node->getIsTarget())
+      OS << ", ";
+    OS << "targetsync";
+  }
+  OS << " : ";
+  Node->getInteropVar()->printPretty(OS, nullptr, Policy);
+  OS << ")";
+}
+
+void OMPClausePrinter::VisitOMPUseClause(OMPUseClause *Node) {
+  OS << "use(";
+  Node->getInteropVar()->printPretty(OS, nullptr, Policy);
+  OS << ")";
+}
+
+void OMPClausePrinter::VisitOMPDestroyClause(OMPDestroyClause *Node) {
   OS << "destroy";
+  if (Expr *E = Node->getInteropVar()) {
+    OS << "(";
+    E->printPretty(OS, nullptr, Policy);
+    OS << ")";
+  }
 }
 
 template<typename T>
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index 95d69fa5b11ad0f41a82b4d05c1f4943a9a5216c..eb9bfc20342fd17c97f6fce4a055680478c523b3 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -1627,12 +1627,17 @@ void ItaniumRecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
     // Some such targets do honor it on zero-width bitfields.
     if (FieldSize == 0 &&
         Context.getTargetInfo().useZeroLengthBitfieldAlignment()) {
-      // The alignment to round up to is the max of the field's natural
-      // alignment and a target-specific fixed value (sometimes zero).
-      unsigned ZeroLengthBitfieldBoundary =
-        Context.getTargetInfo().getZeroLengthBitfieldBoundary();
-      FieldAlign = std::max(FieldAlign, ZeroLengthBitfieldBoundary);
-
+      // Some targets don't honor leading zero-width bitfield.
+      if (!IsUnion && FieldOffset == 0 &&
+          !Context.getTargetInfo().useLeadingZeroLengthBitfield())
+        FieldAlign = 1;
+      else {
+        // The alignment to round up to is the max of the field's natural
+        // alignment and a target-specific fixed value (sometimes zero).
+        unsigned ZeroLengthBitfieldBoundary =
+            Context.getTargetInfo().getZeroLengthBitfieldBoundary();
+        FieldAlign = std::max(FieldAlign, ZeroLengthBitfieldBoundary);
+      }
     // If that doesn't apply, just ignore the field alignment.
     } else {
       FieldAlign = 1;
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 5a6b31bae62438789c7bd183c9f575fac8fd922b..fa77b862f3d0e5586ee874b8c2d2d5e1968d7558 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -1944,3 +1944,18 @@ OMPTargetTeamsDistributeSimdDirective::CreateEmpty(const ASTContext &C,
       numLoopChildren(CollapsedNum, OMPD_target_teams_distribute_simd),
       CollapsedNum);
 }
+
+OMPInteropDirective *
+OMPInteropDirective::Create(const ASTContext &C, SourceLocation StartLoc,
+                            SourceLocation EndLoc,
+                            ArrayRef<OMPClause *> Clauses) {
+  return createDirective<OMPInteropDirective>(
+      C, Clauses, /*AssociatedStmt=*/nullptr, /*NumChildren=*/0, StartLoc,
+      EndLoc);
+}
+
+OMPInteropDirective *OMPInteropDirective::CreateEmpty(const ASTContext &C,
+                                                      unsigned NumClauses,
+                                                      EmptyShell) {
+  return createEmptyDirective<OMPInteropDirective>(C, NumClauses);
+}
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 82071f5d7aaa8bc956931cf94fc6823be140c886..ca35c6dccbf8cb6b4c06ea2edbab6c1869c413f0 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -962,6 +962,11 @@ void StmtPrinter::VisitOMPTargetTeamsDistributeSimdDirective(
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPInteropDirective(OMPInteropDirective *Node) {
+  Indent() << "#pragma omp interop";
+  PrintOMPExecutableDirective(Node);
+}
+
 //===----------------------------------------------------------------------===//
 //  Expr printing methods.
 //===----------------------------------------------------------------------===//
@@ -1165,6 +1170,10 @@ void StmtPrinter::VisitIntegerLiteral(IntegerLiteral *Node) {
   case BuiltinType::ULong:     OS << "UL"; break;
   case BuiltinType::LongLong:  OS << "LL"; break;
   case BuiltinType::ULongLong: OS << "ULL"; break;
+  case BuiltinType::Int128:
+    break; // no suffix.
+  case BuiltinType::UInt128:
+    break; // no suffix.
   }
 }
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 761d027b337800ffe3f4b62250d3e333753cca93..bf130ed4ff3d34872ac9109d67953219b1aadfb6 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -543,7 +543,19 @@ void OMPClauseProfiler::VisitOMPSIMDClause(const OMPSIMDClause *) {}
 
 void OMPClauseProfiler::VisitOMPNogroupClause(const OMPNogroupClause *) {}
 
-void OMPClauseProfiler::VisitOMPDestroyClause(const OMPDestroyClause *) {}
+void OMPClauseProfiler::VisitOMPInitClause(const OMPInitClause *C) {
+  VisitOMPClauseList(C);
+}
+
+void OMPClauseProfiler::VisitOMPUseClause(const OMPUseClause *C) {
+  if (C->getInteropVar())
+    Profiler->VisitStmt(C->getInteropVar());
+}
+
+void OMPClauseProfiler::VisitOMPDestroyClause(const OMPDestroyClause *C) {
+  if (C->getInteropVar())
+    Profiler->VisitStmt(C->getInteropVar());
+}
 
 template<typename T>
 void OMPClauseProfiler::VisitOMPClauseList(T *Node) {
@@ -1128,6 +1140,10 @@ void StmtProfiler::VisitOMPTargetTeamsDistributeSimdDirective(
   VisitOMPLoopDirective(S);
 }
 
+void StmtProfiler::VisitOMPInteropDirective(const OMPInteropDirective *S) {
+  VisitOMPExecutableDirective(S);
+}
+
 void StmtProfiler::VisitExpr(const Expr *S) {
   VisitStmt(S);
 }
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 51289ce45ab927b23c43d23c009ece5f9199e6cd..611c30d9c7677652ef7335ecc3951a5b967a7870 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2229,10 +2229,11 @@ bool Type::isIncompleteType(NamedDecl **Def) const {
     return !Rec->isCompleteDefinition();
   }
   case ConstantArray:
+  case VariableArray:
     // An array is incomplete if its element type is incomplete
     // (C++ [dcl.array]p1).
-    // We don't handle variable arrays (they're not allowed in C++) or
-    // dependent-sized arrays (dependent types are never treated as incomplete).
+    // We don't handle dependent-sized arrays (dependent types are never treated
+    // as incomplete).
     return cast<ArrayType>(CanonicalType)->getElementType()
              ->isIncompleteType(Def);
   case IncompleteArray:
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 167b9fdb35ec1b905dff736da9a65b332a767b51..70c29d3da97faa29d2349341d42135e798fe477d 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -1284,8 +1284,7 @@ void TypePrinter::AppendScope(DeclContext *DC, raw_ostream &OS,
     // Only suppress an inline namespace if the name has the same lookup
     // results in the enclosing namespace.
     if (Policy.SuppressInlineNamespace && NS->isInline() && NameInScope &&
-        DC->getParent()->lookup(NameInScope).size() ==
-            DC->lookup(NameInScope).size())
+        NS->isRedundantInlineQualifierFor(NameInScope))
       return AppendScope(DC->getParent(), OS, NameInScope);
 
     AppendScope(DC->getParent(), OS, NS->getDeclName());
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index b20a6042566185375fc78210fa05d93d5503639d..c2001070de558f507eb304ba5c0952c837abc8e9 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -883,6 +883,7 @@ const internal::VariadicDynCastAllOfMatcher<Stmt, WhileStmt> whileStmt;
 const internal::VariadicDynCastAllOfMatcher<Stmt, DoStmt> doStmt;
 const internal::VariadicDynCastAllOfMatcher<Stmt, BreakStmt> breakStmt;
 const internal::VariadicDynCastAllOfMatcher<Stmt, ContinueStmt> continueStmt;
+const internal::VariadicDynCastAllOfMatcher<Stmt, CoreturnStmt> coreturnStmt;
 const internal::VariadicDynCastAllOfMatcher<Stmt, ReturnStmt> returnStmt;
 const internal::VariadicDynCastAllOfMatcher<Stmt, GotoStmt> gotoStmt;
 const internal::VariadicDynCastAllOfMatcher<Stmt, LabelStmt> labelStmt;
@@ -915,6 +916,12 @@ const internal::VariadicDynCastAllOfMatcher<Stmt, CompoundLiteralExpr>
 const internal::VariadicDynCastAllOfMatcher<Stmt, CXXNullPtrLiteralExpr>
     cxxNullPtrLiteralExpr;
 const internal::VariadicDynCastAllOfMatcher<Stmt, ChooseExpr> chooseExpr;
+const internal::VariadicDynCastAllOfMatcher<Stmt, CoawaitExpr>
+    coawaitExpr;
+const internal::VariadicDynCastAllOfMatcher<Stmt, DependentCoawaitExpr>
+    dependentCoawaitExpr;
+const internal::VariadicDynCastAllOfMatcher<Stmt, CoyieldExpr>
+    coyieldExpr;
 const internal::VariadicDynCastAllOfMatcher<Stmt, GNUNullExpr> gnuNullExpr;
 const internal::VariadicDynCastAllOfMatcher<Stmt, GenericSelectionExpr>
     genericSelectionExpr;
diff --git a/clang/lib/ASTMatchers/Dynamic/Marshallers.h b/clang/lib/ASTMatchers/Dynamic/Marshallers.h
index 3ffa0d6af21789046c7b74bcfc4d9bc3380a24c0..783fb203c40826eb3252ae868cc9bb076f604742 100644
--- a/clang/lib/ASTMatchers/Dynamic/Marshallers.h
+++ b/clang/lib/ASTMatchers/Dynamic/Marshallers.h
@@ -492,9 +492,11 @@ template <typename ResultT, typename ArgT,
 VariantMatcher
 variadicMatcherDescriptor(StringRef MatcherName, SourceRange NameRange,
                           ArrayRef<ParserValue> Args, Diagnostics *Error) {
-  ArgT **InnerArgs = new ArgT *[Args.size()]();
+  SmallVector<ArgT *, 8> InnerArgsPtr;
+  InnerArgsPtr.resize_for_overwrite(Args.size());
+  SmallVector<ArgT, 8> InnerArgs;
+  InnerArgs.reserve(Args.size());
 
-  bool HasError = false;
   for (size_t i = 0, e = Args.size(); i != e; ++i) {
     using ArgTraits = ArgTypeTraits<ArgT>;
 
@@ -503,8 +505,7 @@ variadicMatcherDescriptor(StringRef MatcherName, SourceRange NameRange,
     if (!ArgTraits::hasCorrectType(Value)) {
       Error->addError(Arg.Range, Error->ET_RegistryWrongArgType)
           << (i + 1) << ArgTraits::getKind().asString() << Value.getTypeAsString();
-      HasError = true;
-      break;
+      return {};
     }
     if (!ArgTraits::hasCorrectValue(Value)) {
       if (llvm::Optional<std::string> BestGuess =
@@ -521,24 +522,12 @@ variadicMatcherDescriptor(StringRef MatcherName, SourceRange NameRange,
             << (i + 1) << ArgTraits::getKind().asString()
             << Value.getTypeAsString();
       }
-      HasError = true;
-      break;
+      return {};
     }
-
-    InnerArgs[i] = new ArgT(ArgTraits::get(Value));
-  }
-
-  VariantMatcher Out;
-  if (!HasError) {
-    Out = outvalueToVariantMatcher(Func(llvm::makeArrayRef(InnerArgs,
-                                                           Args.size())));
+    InnerArgs.set_size(i + 1);
+    InnerArgsPtr[i] = new (&InnerArgs[i]) ArgT(ArgTraits::get(Value));
   }
-
-  for (size_t i = 0, e = Args.size(); i != e; ++i) {
-    delete InnerArgs[i];
-  }
-  delete[] InnerArgs;
-  return Out;
+  return outvalueToVariantMatcher(Func(InnerArgsPtr));
 }
 
 /// Matcher descriptor for variadic functions.
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index bfc46b7742d68cd70c1424da00861be8c5f3449c..8e595deac2cd1c213f7a7db30f0fb961431a0365 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -166,11 +166,14 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(complexType);
   REGISTER_MATCHER(compoundLiteralExpr);
   REGISTER_MATCHER(compoundStmt);
+  REGISTER_MATCHER(coawaitExpr);
   REGISTER_MATCHER(conditionalOperator);
   REGISTER_MATCHER(constantArrayType);
   REGISTER_MATCHER(constantExpr);
   REGISTER_MATCHER(containsDeclaration);
   REGISTER_MATCHER(continueStmt);
+  REGISTER_MATCHER(coreturnStmt);
+  REGISTER_MATCHER(coyieldExpr);
   REGISTER_MATCHER(cudaKernelCallExpr);
   REGISTER_MATCHER(cxxBindTemporaryExpr);
   REGISTER_MATCHER(cxxBoolLiteral);
@@ -214,6 +217,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(decltypeType);
   REGISTER_MATCHER(deducedTemplateSpecializationType);
   REGISTER_MATCHER(defaultStmt);
+  REGISTER_MATCHER(dependentCoawaitExpr);
   REGISTER_MATCHER(dependentSizedArrayType);
   REGISTER_MATCHER(designatedInitExpr);
   REGISTER_MATCHER(designatorCountIs);
diff --git a/clang/lib/Analysis/CalledOnceCheck.cpp b/clang/lib/Analysis/CalledOnceCheck.cpp
index 2438c50d7e4e6c7b6bb9b42b3fa535e78611626e..00bb51a1c0d3dd79e1d6a2a9512393ce4c83fa3b 100644
--- a/clang/lib/Analysis/CalledOnceCheck.cpp
+++ b/clang/lib/Analysis/CalledOnceCheck.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Analysis/Analyses/CalledOnceCheck.h"
+#include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
@@ -57,6 +58,20 @@ constexpr llvm::StringLiteral CONVENTIONAL_SUFFIXES[] = {
 constexpr llvm::StringLiteral CONVENTIONAL_CONDITIONS[] = {
     "error", "cancel", "shouldCall", "done", "OK", "success"};
 
+struct KnownCalledOnceParameter {
+  llvm::StringLiteral FunctionName;
+  unsigned ParamIndex;
+};
+constexpr KnownCalledOnceParameter KNOWN_CALLED_ONCE_PARAMETERS[] = {
+    {llvm::StringLiteral{"dispatch_async"}, 1},
+    {llvm::StringLiteral{"dispatch_async_and_wait"}, 1},
+    {llvm::StringLiteral{"dispatch_after"}, 2},
+    {llvm::StringLiteral{"dispatch_sync"}, 1},
+    {llvm::StringLiteral{"dispatch_once"}, 1},
+    {llvm::StringLiteral{"dispatch_barrier_async"}, 1},
+    {llvm::StringLiteral{"dispatch_barrier_async_and_wait"}, 1},
+    {llvm::StringLiteral{"dispatch_barrier_sync"}, 1}};
+
 class ParameterStatus {
 public:
   // Status kind is basically the main part of parameter's status.
@@ -797,8 +812,12 @@ private:
       }
     }
 
-    // Early exit if we don't have parameters for extra analysis.
-    if (NotCalledOnEveryPath.none() && NotUsedOnEveryPath.none())
+    // Early exit if we don't have parameters for extra analysis...
+    if (NotCalledOnEveryPath.none() && NotUsedOnEveryPath.none() &&
+        // ... or if we've seen variables with cleanup functions.
+        // We can't reason that we've seen every path in this case,
+        // and thus abandon reporting any warnings that imply that.
+        !FunctionHasCleanupVars)
       return;
 
     // We are looking for a pair of blocks A, B so that the following is true:
@@ -867,16 +886,14 @@ private:
     // Let's check if any of the call arguments is a point of interest.
     for (const auto &Argument : llvm::enumerate(Arguments)) {
       if (auto Index = getIndexOfExpression(Argument.value())) {
-        ParameterStatus &CurrentParamStatus = CurrentState.getStatusFor(*Index);
-
         if (shouldBeCalledOnce(CallOrMessage, Argument.index())) {
           // If the corresponding parameter is marked as 'called_once' we should
           // consider it as a call.
           processCallFor(*Index, CallOrMessage);
-        } else if (CurrentParamStatus.getKind() == ParameterStatus::NotCalled) {
+        } else {
           // Otherwise, we mark this parameter as escaped, which can be
           // interpreted both as called or not called depending on the context.
-          CurrentParamStatus = ParameterStatus::Escaped;
+          processEscapeFor(*Index);
         }
         // Otherwise, let's keep the state as it is.
       }
@@ -910,6 +927,16 @@ private:
     }
   }
 
+  /// Process escape of the parameter with the given index
+  void processEscapeFor(unsigned Index) {
+    ParameterStatus &CurrentParamStatus = CurrentState.getStatusFor(Index);
+
+    // Escape overrides whatever error we think happened.
+    if (CurrentParamStatus.isErrorStatus()) {
+      CurrentParamStatus = ParameterStatus::Escaped;
+    }
+  }
+
   void findAndReportNotCalledBranches(const CFGBlock *Parent, unsigned Index,
                                       bool IsEscape = false) {
     for (const CFGBlock *Succ : Parent->succs()) {
@@ -921,9 +948,9 @@ private:
                "Block should have at least two successors at this point");
         if (auto Clarification = NotCalledClarifier::clarify(Parent, Succ)) {
           const ParmVarDecl *Parameter = getParameter(Index);
-          Handler.handleNeverCalled(Parameter, Clarification->Location,
-                                    Clarification->Reason, !IsEscape,
-                                    !isExplicitlyMarked(Parameter));
+          Handler.handleNeverCalled(
+              Parameter, AC.getDecl(), Clarification->Location,
+              Clarification->Reason, !IsEscape, !isExplicitlyMarked(Parameter));
         }
       }
     }
@@ -1083,6 +1110,91 @@ private:
     return false;
   }
 
+  // Return a call site where the block is called exactly once or null otherwise
+  const Expr *getBlockGuaraneedCallSite(const BlockExpr *Block) const {
+    ParentMap &PM = AC.getParentMap();
+
+    // We don't want to track the block through assignments and so on, instead
+    // we simply see how the block used and if it's used directly in a call,
+    // we decide based on call to what it is.
+    //
+    // In order to do this, we go up the parents of the block looking for
+    // a call or a message expressions.  These might not be immediate parents
+    // of the actual block expression due to casts and parens, so we skip them.
+    for (const Stmt *Prev = Block, *Current = PM.getParent(Block);
+         Current != nullptr; Prev = Current, Current = PM.getParent(Current)) {
+      // Skip no-op (for our case) operations.
+      if (isa<CastExpr>(Current) || isa<ParenExpr>(Current))
+        continue;
+
+      // At this point, Prev represents our block as an immediate child of the
+      // call.
+      if (const auto *Call = dyn_cast<CallExpr>(Current)) {
+        // It might be the call of the Block itself...
+        if (Call->getCallee() == Prev)
+          return Call;
+
+        // ...or it can be an indirect call of the block.
+        return shouldBlockArgumentBeCalledOnce(Call, Prev) ? Call : nullptr;
+      }
+      if (const auto *Message = dyn_cast<ObjCMessageExpr>(Current)) {
+        return shouldBlockArgumentBeCalledOnce(Message, Prev) ? Message
+                                                              : nullptr;
+      }
+
+      break;
+    }
+
+    return nullptr;
+  }
+
+  template <class CallLikeExpr>
+  bool shouldBlockArgumentBeCalledOnce(const CallLikeExpr *CallOrMessage,
+                                       const Stmt *BlockArgument) const {
+    // CallExpr::arguments does not interact nicely with llvm::enumerate.
+    llvm::ArrayRef<const Expr *> Arguments = llvm::makeArrayRef(
+        CallOrMessage->getArgs(), CallOrMessage->getNumArgs());
+
+    for (const auto &Argument : llvm::enumerate(Arguments)) {
+      if (Argument.value() == BlockArgument) {
+        return shouldBlockArgumentBeCalledOnce(CallOrMessage, Argument.index());
+      }
+    }
+
+    return false;
+  }
+
+  bool shouldBlockArgumentBeCalledOnce(const CallExpr *Call,
+                                       unsigned ParamIndex) const {
+    const FunctionDecl *Function = Call->getDirectCallee();
+    return shouldBlockArgumentBeCalledOnce(Function, ParamIndex) ||
+           shouldBeCalledOnce(Call, ParamIndex);
+  }
+
+  bool shouldBlockArgumentBeCalledOnce(const ObjCMessageExpr *Message,
+                                       unsigned ParamIndex) const {
+    // At the moment, we don't have any Obj-C methods we want to specifically
+    // check in here.
+    return shouldBeCalledOnce(Message, ParamIndex);
+  }
+
+  static bool shouldBlockArgumentBeCalledOnce(const FunctionDecl *Function,
+                                              unsigned ParamIndex) {
+    // There is a list of important API functions that while not following
+    // conventions nor being directly annotated, still guarantee that the
+    // callback parameter will be called exactly once.
+    //
+    // Here we check if this is the case.
+    return Function &&
+           llvm::any_of(KNOWN_CALLED_ONCE_PARAMETERS,
+                        [Function, ParamIndex](
+                            const KnownCalledOnceParameter &Reference) {
+                          return Reference.FunctionName ==
+                                     Function->getName() &&
+                                 Reference.ParamIndex == ParamIndex;
+                        });
+  }
+
   /// Return true if the analyzed function is actually a default implementation
   /// of the method that has to be overriden.
   ///
@@ -1365,11 +1477,7 @@ private:
   /// Check given parameter that was discovered to escape.
   void checkEscapee(const ParmVarDecl &Parameter) {
     if (auto Index = getIndex(Parameter)) {
-      ParameterStatus &CurrentParamStatus = CurrentState.getStatusFor(*Index);
-
-      if (CurrentParamStatus.getKind() == ParameterStatus::NotCalled) {
-        CurrentParamStatus = ParameterStatus::Escaped;
-      }
+      processEscapeFor(*Index);
     }
   }
 
@@ -1433,17 +1541,44 @@ public:
   }
 
   void VisitBlockExpr(const BlockExpr *Block) {
+    // Block expressions are tricky.  It is a very common practice to capture
+    // completion handlers by blocks and use them there.
+    // For this reason, it is important to analyze blocks and report warnings
+    // for completion handler misuse in blocks.
+    //
+    // However, it can be quite difficult to track how the block itself is being
+    // used.  The full precise anlysis of that will be similar to alias analysis
+    // for completion handlers and can be too heavyweight for a compile-time
+    // diagnostic.  Instead, we judge about the immediate use of the block.
+    //
+    // Here, we try to find a call expression where we know due to conventions,
+    // annotations, or other reasons that the block is called once and only
+    // once.
+    const Expr *CalledOnceCallSite = getBlockGuaraneedCallSite(Block);
+
+    // We need to report this information to the handler because in the
+    // situation when we know that the block is called exactly once, we can be
+    // stricter in terms of reported diagnostics.
+    if (CalledOnceCallSite) {
+      Handler.handleBlockThatIsGuaranteedToBeCalledOnce(Block->getBlockDecl());
+    } else {
+      Handler.handleBlockWithNoGuarantees(Block->getBlockDecl());
+    }
+
     for (const auto &Capture : Block->getBlockDecl()->captures()) {
-      // If a block captures a tracked parameter, it should be
-      // considered escaped.
-      // On one hand, blocks that do that should definitely call it on
-      // every path.  However, it is not guaranteed that the block
-      // itself gets called whenever it gets created.
-      //
-      // Because we don't want to track blocks and whether they get called,
-      // we consider such parameters simply escaped.
       if (const auto *Param = dyn_cast<ParmVarDecl>(Capture.getVariable())) {
-        checkEscapee(*Param);
+        if (auto Index = getIndex(*Param)) {
+          if (CalledOnceCallSite) {
+            // The call site of a block can be considered a call site of the
+            // captured parameter we track.
+            processCallFor(*Index, CalledOnceCallSite);
+          } else {
+            // We still should consider this block as an escape for parameter,
+            // if we don't know about its call site or the number of time it
+            // can be invoked.
+            processEscapeFor(*Index);
+          }
+        }
       }
     }
   }
@@ -1470,6 +1605,10 @@ public:
         if (Var->getInit()) {
           checkEscapee(Var->getInit());
         }
+
+        if (Var->hasAttr<CleanupAttr>()) {
+          FunctionHasCleanupVars = true;
+        }
       }
     }
   }
@@ -1538,6 +1677,13 @@ private:
   // around.
   bool SuppressOnConventionalErrorPaths = false;
 
+  // The user can annotate variable declarations with cleanup functions, which
+  // essentially imposes a custom destructor logic on that variable.
+  // It is possible to use it, however, to call tracked parameters on all exits
+  // from the function.  For this reason, we track the fact that the function
+  // actually has these.
+  bool FunctionHasCleanupVars = false;
+
   State CurrentState;
   ParamSizedVector<const ParmVarDecl *> TrackedParams;
   CFGSizedVector<State> States;
diff --git a/clang/lib/Analysis/RetainSummaryManager.cpp b/clang/lib/Analysis/RetainSummaryManager.cpp
index 00bc854a88049623d104a461731dee011e2634a5..ecda47a67c1d70361f8fee61ba3cd9493777a908 100644
--- a/clang/lib/Analysis/RetainSummaryManager.cpp
+++ b/clang/lib/Analysis/RetainSummaryManager.cpp
@@ -881,8 +881,8 @@ RetainSummaryManager::getRetEffectFromAnnotations(QualType RetTy,
   return None;
 }
 
-/// \return Whether the chain of typedefs starting from {@code QT}
-/// has a typedef with a given name {@code Name}.
+/// \return Whether the chain of typedefs starting from @c QT
+/// has a typedef with a given name @c Name.
 static bool hasTypedefNamed(QualType QT,
                             StringRef Name) {
   while (auto *T = dyn_cast<TypedefType>(QT)) {
diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp
index 21583e92c72d6444ca34a490edbd2eebaee7ac5b..84e0e91f597fe9717eabeebf275879e0ac97699f 100644
--- a/clang/lib/Analysis/ThreadSafety.cpp
+++ b/clang/lib/Analysis/ThreadSafety.cpp
@@ -2051,15 +2051,11 @@ void BuildLockset::VisitCallExpr(const CallExpr *Exp) {
 
     if (ME && MD) {
       if (ME->isArrow()) {
-        if (MD->isConst())
-          checkPtAccess(CE->getImplicitObjectArgument(), AK_Read);
-        else // FIXME -- should be AK_Written
-          checkPtAccess(CE->getImplicitObjectArgument(), AK_Read);
+        // Should perhaps be AK_Written if !MD->isConst().
+        checkPtAccess(CE->getImplicitObjectArgument(), AK_Read);
       } else {
-        if (MD->isConst())
-          checkAccess(CE->getImplicitObjectArgument(), AK_Read);
-        else     // FIXME -- should be AK_Written
-          checkAccess(CE->getImplicitObjectArgument(), AK_Read);
+        // Should perhaps be AK_Written if !MD->isConst().
+        checkAccess(CE->getImplicitObjectArgument(), AK_Read);
       }
     }
 
diff --git a/clang/lib/Basic/Builtins.cpp b/clang/lib/Basic/Builtins.cpp
index 0cd89df41b67acd2440f9e34cd721f93696285e0..49afaa9ba6a3e0844c5ed2bf884340356336e007 100644
--- a/clang/lib/Basic/Builtins.cpp
+++ b/clang/lib/Basic/Builtins.cpp
@@ -75,12 +75,13 @@ bool Builtin::Context::builtinIsSupported(const Builtin::Info &BuiltinInfo,
   bool OclCUnsupported = !LangOpts.OpenCL &&
                          (BuiltinInfo.Langs & ALL_OCLC_LANGUAGES);
   bool OpenMPUnsupported = !LangOpts.OpenMP && BuiltinInfo.Langs == OMP_LANG;
+  bool CUDAUnsupported = !LangOpts.CUDA && BuiltinInfo.Langs == CUDA_LANG;
   bool CPlusPlusUnsupported =
       !LangOpts.CPlusPlus && BuiltinInfo.Langs == CXX_LANG;
   return !BuiltinsUnsupported && !MathBuiltinsUnsupported && !OclCUnsupported &&
          !OclC1Unsupported && !OclC2Unsupported && !OpenMPUnsupported &&
          !GnuModeUnsupported && !MSModeUnsupported && !ObjCUnsupported &&
-         !CPlusPlusUnsupported;
+         !CPlusPlusUnsupported && !CUDAUnsupported;
 }
 
 /// initializeBuiltins - Mark the identifiers for all the builtins with their
diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index 8730a5d5f4e709f6f489e3624b5721c0687f8f3f..8d26149cd39d9045b9b8cf461a000f3fcf4773f4 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -429,7 +429,7 @@ void Module::buildVisibleModulesCache() const {
   }
 }
 
-void Module::print(raw_ostream &OS, unsigned Indent) const {
+void Module::print(raw_ostream &OS, unsigned Indent, bool Dump) const {
   OS.indent(Indent);
   if (IsFramework)
     OS << "framework ";
@@ -535,7 +535,7 @@ void Module::print(raw_ostream &OS, unsigned Indent) const {
     // the module. Regular inferred submodules are OK, as we need to look at all
     // those header files anyway.
     if (!(*MI)->IsInferred || (*MI)->IsFramework)
-      (*MI)->print(OS, Indent + 2);
+      (*MI)->print(OS, Indent + 2, Dump);
 
   for (unsigned I = 0, N = Exports.size(); I != N; ++I) {
     OS.indent(Indent + 2);
@@ -559,6 +559,13 @@ void Module::print(raw_ostream &OS, unsigned Indent) const {
     OS << "\n";
   }
 
+  if (Dump) {
+    for (Module *M : Imports) {
+      OS.indent(Indent + 2);
+      llvm::errs() << "import " << M->getFullModuleName() << "\n";
+    }
+  }
+
   for (unsigned I = 0, N = DirectUses.size(); I != N; ++I) {
     OS.indent(Indent + 2);
     OS << "use ";
@@ -619,7 +626,7 @@ void Module::print(raw_ostream &OS, unsigned Indent) const {
 }
 
 LLVM_DUMP_METHOD void Module::dump() const {
-  print(llvm::errs());
+  print(llvm::errs(), 0, true);
 }
 
 void VisibleModuleSet::setVisible(Module *M, SourceLocation Loc,
diff --git a/clang/lib/Basic/TargetInfo.cpp b/clang/lib/Basic/TargetInfo.cpp
index bc3c607dd74e8264dc36b69a4528b4012d7a2cd2..468c8a24498a0b0d88b88446854c3f4a6a365f67 100644
--- a/clang/lib/Basic/TargetInfo.cpp
+++ b/clang/lib/Basic/TargetInfo.cpp
@@ -104,6 +104,7 @@ TargetInfo::TargetInfo(const llvm::Triple &T) : TargetOpts(), Triple(T) {
   UseSignedCharForObjCBool = true;
   UseBitFieldTypeAlignment = true;
   UseZeroLengthBitfieldAlignment = false;
+  UseLeadingZeroLengthBitfield = true;
   UseExplicitBitFieldAlignment = true;
   ZeroLengthBitfieldBoundary = 0;
   HalfFormat = &llvm::APFloat::IEEEhalf();
diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp
index 793a471194fe12d030a912748f508425fbdd5463..8df5cb7a3a61acec8e75629a3a67271db89693ce 100644
--- a/clang/lib/Basic/Targets.cpp
+++ b/clang/lib/Basic/Targets.cpp
@@ -595,13 +595,13 @@ TargetInfo *AllocateTarget(const llvm::Triple &Triple,
     }
 
   case llvm::Triple::spir: {
-    if (Triple.getOS() != llvm::Triple::UnknownOS ||
+    if (os != llvm::Triple::UnknownOS ||
         Triple.getEnvironment() != llvm::Triple::UnknownEnvironment)
       return nullptr;
     return new SPIR32TargetInfo(Triple, Opts);
   }
   case llvm::Triple::spir64: {
-    if (Triple.getOS() != llvm::Triple::UnknownOS ||
+    if (os != llvm::Triple::UnknownOS ||
         Triple.getEnvironment() != llvm::Triple::UnknownEnvironment)
       return nullptr;
     return new SPIR64TargetInfo(Triple, Opts);
@@ -611,7 +611,7 @@ TargetInfo *AllocateTarget(const llvm::Triple &Triple,
         Triple.getVendor() != llvm::Triple::UnknownVendor ||
         !Triple.isOSBinFormatWasm())
       return nullptr;
-    switch (Triple.getOS()) {
+    switch (os) {
       case llvm::Triple::WASI:
         return new WASITargetInfo<WebAssembly32TargetInfo>(Triple, Opts);
       case llvm::Triple::Emscripten:
@@ -626,7 +626,7 @@ TargetInfo *AllocateTarget(const llvm::Triple &Triple,
         Triple.getVendor() != llvm::Triple::UnknownVendor ||
         !Triple.isOSBinFormatWasm())
       return nullptr;
-    switch (Triple.getOS()) {
+    switch (os) {
       case llvm::Triple::WASI:
         return new WASITargetInfo<WebAssembly64TargetInfo>(Triple, Opts);
       case llvm::Triple::Emscripten:
diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp
index a84422e412ff90fb6b41d22a31cacda40afb93af..75115db1250b9a6457b3160265559ccf447fd0b7 100644
--- a/clang/lib/Basic/Targets/AMDGPU.cpp
+++ b/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -183,6 +183,7 @@ bool AMDGPUTargetInfo::initFeatureMap(
       Features["dot2-insts"] = true;
       Features["dot5-insts"] = true;
       Features["dot6-insts"] = true;
+      Features["dot7-insts"] = true;
       Features["dl-insts"] = true;
       Features["flat-address-space"] = true;
       Features["16-bit-insts"] = true;
@@ -200,6 +201,7 @@ bool AMDGPUTargetInfo::initFeatureMap(
       Features["dot2-insts"] = true;
       Features["dot5-insts"] = true;
       Features["dot6-insts"] = true;
+      Features["dot7-insts"] = true;
       LLVM_FALLTHROUGH;
     case GK_GFX1010:
       Features["dl-insts"] = true;
@@ -227,6 +229,7 @@ bool AMDGPUTargetInfo::initFeatureMap(
       Features["dl-insts"] = true;
       Features["dot1-insts"] = true;
       Features["dot2-insts"] = true;
+      Features["dot7-insts"] = true;
       LLVM_FALLTHROUGH;
     case GK_GFX90C:
     case GK_GFX909:
diff --git a/clang/lib/Basic/Targets/Hexagon.cpp b/clang/lib/Basic/Targets/Hexagon.cpp
index ba10459e96908b5e9e94ba0053ea5bc6918400ef..d1613fb229307f8a829e0ba636e8d88e8d8cf9be 100644
--- a/clang/lib/Basic/Targets/Hexagon.cpp
+++ b/clang/lib/Basic/Targets/Hexagon.cpp
@@ -136,7 +136,7 @@ const char *const HexagonTargetInfo::GCCRegNames[] = {
     "r9",  "r10", "r11", "r12", "r13", "r14", "r15", "r16", "r17",
     "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26",
     "r27", "r28", "r29", "r30", "r31", "p0",  "p1",  "p2",  "p3",
-    "sa0", "lc0", "sa1", "lc1", "m0",  "m1",  "usr", "ugp",
+    "sa0", "lc0", "sa1", "lc1", "m0",  "m1",  "usr", "ugp", "cs0", "cs1",
     "r1:0", "r3:2", "r5:4", "r7:6", "r9:8", "r11:10", "r13:12", "r15:14",
     "r17:16", "r19:18", "r21:20", "r23:22", "r25:24", "r27:26", "r29:28",
     "r31:30"
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index 539466c4f678fd959c9d675af03b17b83ee57d1c..568f759bfa0d7b14d5b90f0d098abfc805c69ea4 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -261,6 +261,9 @@ public:
     case llvm::Triple::arm:
       this->MCountName = "__mcount";
       break;
+    case llvm::Triple::riscv32:
+    case llvm::Triple::riscv64:
+      break;
     }
   }
 };
@@ -491,6 +494,9 @@ public:
     case llvm::Triple::sparcv9:
       this->MCountName = "_mcount";
       break;
+    case llvm::Triple::riscv32:
+    case llvm::Triple::riscv64:
+      break;
     }
   }
 };
@@ -790,6 +796,7 @@ public:
     this->WCharType = TargetInfo::UnsignedInt;
     this->UseBitFieldTypeAlignment = false;
     this->UseZeroLengthBitfieldAlignment = true;
+    this->UseLeadingZeroLengthBitfield = false;
     this->ZeroLengthBitfieldBoundary = 32;
     this->MinGlobalAlign = 0;
     this->DefaultAlignForAttributeAligned = 128;
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index 57f5de1d0c668600d9c55e0948d5b824e4189968..0703691f4c5b3063da5fea271076a005a7ad5042 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -66,8 +66,10 @@ bool PPCTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       PairedVectorMemops = true;
     } else if (Feature == "+mma") {
       HasMMA = true;
-    } else if (Feature == "+rop-protection") {
-      HasROPProtection = true;
+    } else if (Feature == "+rop-protect") {
+      HasROPProtect = true;
+    } else if (Feature == "+privileged") {
+      HasPrivileged = true;
     }
     // TODO: Finish this list and add an assert that we've handled them
     // all.
@@ -195,8 +197,10 @@ void PPCTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__POWER9_VECTOR__");
   if (HasMMA)
     Builder.defineMacro("__MMA__");
-  if (HasROPProtection)
-    Builder.defineMacro("__ROP_PROTECTION__");
+  if (HasROPProtect)
+    Builder.defineMacro("__ROP_PROTECT__");
+  if (HasPrivileged)
+    Builder.defineMacro("__PRIVILEGED__");
   if (HasP10Vector)
     Builder.defineMacro("__POWER10_VECTOR__");
   if (HasPCRelativeMemops)
@@ -325,8 +329,10 @@ bool PPCTargetInfo::initFeatureMap(
                         .Case("pwr8", true)
                         .Default(false);
 
-  // ROP Protection is off by default.
-  Features["rop-protection"] = false;
+  // ROP Protect is off by default.
+  Features["rop-protect"] = false;
+  // Privileged instructions are off by default.
+  Features["privileged"] = false;
 
   Features["spe"] = llvm::StringSwitch<bool>(CPU)
                         .Case("8548", true)
@@ -365,9 +371,15 @@ bool PPCTargetInfo::initFeatureMap(
   }
 
   if (!(ArchDefs & ArchDefinePwr8) &&
-      llvm::find(FeaturesVec, "+rop-protection") != FeaturesVec.end()) {
-    // We can turn on ROP Protection on Power 8 and above.
-    Diags.Report(diag::err_opt_not_valid_with_opt) << "-mrop-protection" << CPU;
+      llvm::find(FeaturesVec, "+rop-protect") != FeaturesVec.end()) {
+    // We can turn on ROP Protect on Power 8 and above.
+    Diags.Report(diag::err_opt_not_valid_with_opt) << "-mrop-protect" << CPU;
+    return false;
+  }
+
+  if (!(ArchDefs & ArchDefinePwr8) &&
+      llvm::find(FeaturesVec, "+privileged") != FeaturesVec.end()) {
+    Diags.Report(diag::err_opt_not_valid_with_opt) << "-mprivileged" << CPU;
     return false;
   }
 
@@ -409,7 +421,8 @@ bool PPCTargetInfo::hasFeature(StringRef Feature) const {
       .Case("pcrelative-memops", HasPCRelativeMemops)
       .Case("spe", HasSPE)
       .Case("mma", HasMMA)
-      .Case("rop-protection", HasROPProtection)
+      .Case("rop-protect", HasROPProtect)
+      .Case("privileged", HasPrivileged)
       .Default(false);
 }
 
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index 0f9713a4c015844ead4822f54b8bc523349000da..1c6c63bdfa90b477fa79d3943574970bd6442d2f 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -59,7 +59,8 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
   // Target cpu features.
   bool HasAltivec = false;
   bool HasMMA = false;
-  bool HasROPProtection = false;
+  bool HasROPProtect = false;
+  bool HasPrivileged = false;
   bool HasVSX = false;
   bool HasP8Vector = false;
   bool HasP8Crypto = false;
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index d908de3497eb3ac96bec41c6dd3df69aed6645fb..de4abfa20a7add41db037c5f8190c8501859dbb0 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -59,6 +59,7 @@ public:
     WCharType = SignedInt;
     WIntType = UnsignedInt;
     HasRISCVVTypes = true;
+    MCountName = "_mcount";
   }
 
   bool setCPU(const std::string &Name) override {
diff --git a/clang/lib/Basic/Targets/WebAssembly.cpp b/clang/lib/Basic/Targets/WebAssembly.cpp
index 89babe85794db46feb827f8a7767e5f73a2924bd..2a5055c3d534b0b642fbf49c753a8008336d9d92 100644
--- a/clang/lib/Basic/Targets/WebAssembly.cpp
+++ b/clang/lib/Basic/Targets/WebAssembly.cpp
@@ -46,7 +46,6 @@ bool WebAssemblyTargetInfo::setABI(const std::string &Name) {
 bool WebAssemblyTargetInfo::hasFeature(StringRef Feature) const {
   return llvm::StringSwitch<bool>(Feature)
       .Case("simd128", SIMDLevel >= SIMD128)
-      .Case("unimplemented-simd128", SIMDLevel >= UnimplementedSIMD128)
       .Case("nontrapping-fptoint", HasNontrappingFPToInt)
       .Case("sign-ext", HasSignExt)
       .Case("exception-handling", HasExceptionHandling)
@@ -73,8 +72,6 @@ void WebAssemblyTargetInfo::getTargetDefines(const LangOptions &Opts,
   defineCPUMacros(Builder, "wasm", /*Tuning=*/false);
   if (SIMDLevel >= SIMD128)
     Builder.defineMacro("__wasm_simd128__");
-  if (SIMDLevel >= UnimplementedSIMD128)
-    Builder.defineMacro("__wasm_unimplemented_simd128__");
   if (HasNontrappingFPToInt)
     Builder.defineMacro("__wasm_nontrapping_fptoint__");
   if (HasSignExt)
@@ -99,9 +96,6 @@ void WebAssemblyTargetInfo::setSIMDLevel(llvm::StringMap<bool> &Features,
                                          SIMDEnum Level, bool Enabled) {
   if (Enabled) {
     switch (Level) {
-    case UnimplementedSIMD128:
-      Features["unimplemented-simd128"] = true;
-      LLVM_FALLTHROUGH;
     case SIMD128:
       Features["simd128"] = true;
       LLVM_FALLTHROUGH;
@@ -115,9 +109,6 @@ void WebAssemblyTargetInfo::setSIMDLevel(llvm::StringMap<bool> &Features,
   case NoSIMD:
   case SIMD128:
     Features["simd128"] = false;
-    LLVM_FALLTHROUGH;
-  case UnimplementedSIMD128:
-    Features["unimplemented-simd128"] = false;
     break;
   }
 }
@@ -127,8 +118,6 @@ void WebAssemblyTargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
                                               bool Enabled) const {
   if (Name == "simd128")
     setSIMDLevel(Features, SIMD128, Enabled);
-  else if (Name == "unimplemented-simd128")
-    setSIMDLevel(Features, UnimplementedSIMD128, Enabled);
   else
     Features[Name] = Enabled;
 }
@@ -160,14 +149,6 @@ bool WebAssemblyTargetInfo::handleTargetFeatures(
       SIMDLevel = std::min(SIMDLevel, SIMDEnum(SIMD128 - 1));
       continue;
     }
-    if (Feature == "+unimplemented-simd128") {
-      SIMDLevel = std::max(SIMDLevel, SIMDEnum(UnimplementedSIMD128));
-      continue;
-    }
-    if (Feature == "-unimplemented-simd128") {
-      SIMDLevel = std::min(SIMDLevel, SIMDEnum(UnimplementedSIMD128 - 1));
-      continue;
-    }
     if (Feature == "+nontrapping-fptoint") {
       HasNontrappingFPToInt = true;
       continue;
diff --git a/clang/lib/Basic/Targets/WebAssembly.h b/clang/lib/Basic/Targets/WebAssembly.h
index 9150d849f6017fae4425fd1a42dbc7694a66a407..be5b66a9580b14a6c1dbecc2d756518171913b6e 100644
--- a/clang/lib/Basic/Targets/WebAssembly.h
+++ b/clang/lib/Basic/Targets/WebAssembly.h
@@ -27,7 +27,6 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyTargetInfo : public TargetInfo {
   enum SIMDEnum {
     NoSIMD,
     SIMD128,
-    UnimplementedSIMD128,
   } SIMDLevel = NoSIMD;
 
   bool HasNontrappingFPToInt = false;
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 37f9067e0b2e1b4c184c914777cb4efb163af451..6de482ea74f521ca418f9d6c3216aa5a8f2f78e9 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -81,6 +81,7 @@
 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
 #include "llvm/Transforms/Utils.h"
 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
+#include "llvm/Transforms/Utils/Debugify.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
 #include "llvm/Transforms/Utils/SymbolRewriter.h"
@@ -945,7 +946,16 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
   if (TM)
     TheModule->setDataLayout(TM->createDataLayout());
 
-  legacy::PassManager PerModulePasses;
+  DebugifyCustomPassManager PerModulePasses;
+  DebugInfoPerPassMap DIPreservationMap;
+  if (CodeGenOpts.EnableDIPreservationVerify) {
+    PerModulePasses.setDebugifyMode(DebugifyMode::OriginalDebugInfo);
+    PerModulePasses.setDIPreservationMap(DIPreservationMap);
+
+    if (!CodeGenOpts.DIBugsReportFilePath.empty())
+      PerModulePasses.setOrigDIVerifyBugsReportFilePath(
+          CodeGenOpts.DIBugsReportFilePath);
+  }
   PerModulePasses.add(
       createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
 
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index c7256e240a31eb0b6cc7d32afd35259dd0a8c934..8ac36e4a6b64b1a649fcc573ef1c9477d6b7c07c 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -427,6 +427,8 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
     else
       switch ((llvm::AtomicOrderingCABI)FOS) {
       case llvm::AtomicOrderingCABI::relaxed:
+      // 31.7.2.18: "The failure argument shall not be memory_order_release
+      // nor memory_order_acq_rel". Fallback to monotonic.
       case llvm::AtomicOrderingCABI::release:
       case llvm::AtomicOrderingCABI::acq_rel:
         FailureOrder = llvm::AtomicOrdering::Monotonic;
@@ -439,12 +441,10 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
         FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
         break;
       }
-    if (isStrongerThan(FailureOrder, SuccessOrder)) {
-      // Don't assert on undefined behavior "failure argument shall be no
-      // stronger than the success argument".
-      FailureOrder =
-          llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
-    }
+    // Prior to c++17, "the failure argument shall be no stronger than the
+    // success argument". This condition has been lifted and the only
+    // precondition is 31.7.2.18. Effectively treat this as a DR and skip
+    // language version checks.
     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
                       FailureOrder, Scope);
     return;
@@ -454,8 +454,7 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
                    *SeqCstBB = nullptr;
   MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
-  if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
-      SuccessOrder != llvm::AtomicOrdering::Release)
+  if (SuccessOrder != llvm::AtomicOrdering::Monotonic)
     AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
   if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
     SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
@@ -479,8 +478,9 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
                       Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
     CGF.Builder.CreateBr(ContBB);
-    SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
-                AcquireBB);
+    if (SuccessOrder != llvm::AtomicOrdering::Release)
+      SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
+                  AcquireBB);
     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
                 AcquireBB);
   }
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index c8d3d4923817e81b11b374c20012274b48c173b0..ee3d9e3e97ec873eac66fc5d3a472ef98dc4ba68 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CGCUDARuntime.h"
 #include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
 #include "CGOpenCLRuntime.h"
@@ -5060,6 +5061,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val");
     return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
   }
+
+  case Builtin::BI__builtin_get_device_side_mangled_name: {
+    auto Name = CGM.getCUDARuntime().getDeviceSideName(
+        cast<DeclRefExpr>(E->getArg(0)->IgnoreImpCasts())->getDecl());
+    auto Str = CGM.GetAddrOfConstantCString(Name, "");
+    llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0),
+                               llvm::ConstantInt::get(SizeTy, 0)};
+    auto *Ptr = llvm::ConstantExpr::getGetElementPtr(Str.getElementType(),
+                                                     Str.getPointer(), Zeros);
+    return RValue::get(Ptr);
+  }
   }
 
   // If this is an alias for a lib function (e.g. __builtin_sin), emit
@@ -10622,17 +10634,23 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
   }
   case NEON::BI__builtin_neon_vrndnh_f16: {
     Ops.push_back(EmitScalarExpr(E->getArg(0)));
-    Int = Intrinsic::aarch64_neon_frintn;
+    Int = Builder.getIsFPConstrained()
+              ? Intrinsic::experimental_constrained_roundeven
+              : Intrinsic::roundeven;
     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn");
   }
   case NEON::BI__builtin_neon_vrndn_v:
   case NEON::BI__builtin_neon_vrndnq_v: {
-    Int = Intrinsic::aarch64_neon_frintn;
+    Int = Builder.getIsFPConstrained()
+              ? Intrinsic::experimental_constrained_roundeven
+              : Intrinsic::roundeven;
     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
   }
   case NEON::BI__builtin_neon_vrndns_f32: {
     Ops.push_back(EmitScalarExpr(E->getArg(0)));
-    Int = Intrinsic::aarch64_neon_frintn;
+    Int = Builder.getIsFPConstrained()
+              ? Intrinsic::experimental_constrained_roundeven
+              : Intrinsic::roundeven;
     return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn");
   }
   case NEON::BI__builtin_neon_vrndph_f16: {
@@ -15024,6 +15042,18 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
     return Builder.CreateCall(F, X);
   }
+  case PPC::BI__builtin_altivec_vadduqm:
+  case PPC::BI__builtin_altivec_vsubuqm: {
+    llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
+    Ops[0] =
+        Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int128Ty, 1));
+    Ops[1] =
+        Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(Int128Ty, 1));
+    if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
+      return Builder.CreateAdd(Ops[0], Ops[1], "vadduqm");
+    else
+      return Builder.CreateSub(Ops[0], Ops[1], "vsubuqm");
+  }
   // Copy sign
   case PPC::BI__builtin_vsx_xvcpsgnsp:
   case PPC::BI__builtin_vsx_xvcpsgndp: {
@@ -17190,31 +17220,31 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
       llvm_unreachable("unexpected builtin ID");
     }
   }
-  case WebAssembly::BI__builtin_wasm_add_saturate_s_i8x16:
-  case WebAssembly::BI__builtin_wasm_add_saturate_u_i8x16:
-  case WebAssembly::BI__builtin_wasm_add_saturate_s_i16x8:
-  case WebAssembly::BI__builtin_wasm_add_saturate_u_i16x8:
-  case WebAssembly::BI__builtin_wasm_sub_saturate_s_i8x16:
-  case WebAssembly::BI__builtin_wasm_sub_saturate_u_i8x16:
-  case WebAssembly::BI__builtin_wasm_sub_saturate_s_i16x8:
-  case WebAssembly::BI__builtin_wasm_sub_saturate_u_i16x8: {
+  case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
+  case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
+  case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
+  case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
+  case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
+  case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
+  case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
+  case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8: {
     unsigned IntNo;
     switch (BuiltinID) {
-    case WebAssembly::BI__builtin_wasm_add_saturate_s_i8x16:
-    case WebAssembly::BI__builtin_wasm_add_saturate_s_i16x8:
+    case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
+    case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
       IntNo = Intrinsic::sadd_sat;
       break;
-    case WebAssembly::BI__builtin_wasm_add_saturate_u_i8x16:
-    case WebAssembly::BI__builtin_wasm_add_saturate_u_i16x8:
+    case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
+    case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
       IntNo = Intrinsic::uadd_sat;
       break;
-    case WebAssembly::BI__builtin_wasm_sub_saturate_s_i8x16:
-    case WebAssembly::BI__builtin_wasm_sub_saturate_s_i16x8:
-      IntNo = Intrinsic::wasm_sub_saturate_signed;
+    case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
+    case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
+      IntNo = Intrinsic::wasm_sub_sat_signed;
       break;
-    case WebAssembly::BI__builtin_wasm_sub_saturate_u_i8x16:
-    case WebAssembly::BI__builtin_wasm_sub_saturate_u_i16x8:
-      IntNo = Intrinsic::wasm_sub_saturate_unsigned;
+    case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
+    case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8:
+      IntNo = Intrinsic::wasm_sub_sat_unsigned;
       break;
     default:
       llvm_unreachable("unexpected builtin ID");
@@ -17282,11 +17312,10 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
                                         ConvertType(E->getType()));
     return Builder.CreateCall(Callee, {LHS, RHS});
   }
-  case WebAssembly::BI__builtin_wasm_q15mulr_saturate_s_i16x8: {
+  case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
     Value *LHS = EmitScalarExpr(E->getArg(0));
     Value *RHS = EmitScalarExpr(E->getArg(1));
-    Function *Callee =
-        CGM.getIntrinsic(Intrinsic::wasm_q15mulr_saturate_signed);
+    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_q15mulr_sat_signed);
     return Builder.CreateCall(Callee, {LHS, RHS});
   }
   case WebAssembly::BI__builtin_wasm_extmul_low_i8x16_s_i16x8:
@@ -17362,17 +17391,6 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
         CGM.getIntrinsic(Intrinsic::wasm_bitselect, ConvertType(E->getType()));
     return Builder.CreateCall(Callee, {V1, V2, C});
   }
-  case WebAssembly::BI__builtin_wasm_signselect_i8x16:
-  case WebAssembly::BI__builtin_wasm_signselect_i16x8:
-  case WebAssembly::BI__builtin_wasm_signselect_i32x4:
-  case WebAssembly::BI__builtin_wasm_signselect_i64x2: {
-    Value *V1 = EmitScalarExpr(E->getArg(0));
-    Value *V2 = EmitScalarExpr(E->getArg(1));
-    Value *C = EmitScalarExpr(E->getArg(2));
-    Function *Callee =
-        CGM.getIntrinsic(Intrinsic::wasm_signselect, ConvertType(E->getType()));
-    return Builder.CreateCall(Callee, {V1, V2, C});
-  }
   case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
     Value *LHS = EmitScalarExpr(E->getArg(0));
     Value *RHS = EmitScalarExpr(E->getArg(1));
@@ -17440,29 +17458,6 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
     Function *Callee = CGM.getIntrinsic(Intrinsic::sqrt, Vec->getType());
     return Builder.CreateCall(Callee, {Vec});
   }
-  case WebAssembly::BI__builtin_wasm_qfma_f32x4:
-  case WebAssembly::BI__builtin_wasm_qfms_f32x4:
-  case WebAssembly::BI__builtin_wasm_qfma_f64x2:
-  case WebAssembly::BI__builtin_wasm_qfms_f64x2: {
-    Value *A = EmitScalarExpr(E->getArg(0));
-    Value *B = EmitScalarExpr(E->getArg(1));
-    Value *C = EmitScalarExpr(E->getArg(2));
-    unsigned IntNo;
-    switch (BuiltinID) {
-    case WebAssembly::BI__builtin_wasm_qfma_f32x4:
-    case WebAssembly::BI__builtin_wasm_qfma_f64x2:
-      IntNo = Intrinsic::wasm_qfma;
-      break;
-    case WebAssembly::BI__builtin_wasm_qfms_f32x4:
-    case WebAssembly::BI__builtin_wasm_qfms_f64x2:
-      IntNo = Intrinsic::wasm_qfms;
-      break;
-    default:
-      llvm_unreachable("unexpected builtin ID");
-    }
-    Function *Callee = CGM.getIntrinsic(IntNo, A->getType());
-    return Builder.CreateCall(Callee, {A, B, C});
-  }
   case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
   case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
   case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
@@ -17486,24 +17481,24 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
         CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Low->getType()});
     return Builder.CreateCall(Callee, {Low, High});
   }
-  case WebAssembly::BI__builtin_wasm_widen_low_s_i32x4_i64x2:
-  case WebAssembly::BI__builtin_wasm_widen_high_s_i32x4_i64x2:
-  case WebAssembly::BI__builtin_wasm_widen_low_u_i32x4_i64x2:
-  case WebAssembly::BI__builtin_wasm_widen_high_u_i32x4_i64x2: {
+  case WebAssembly::BI__builtin_wasm_extend_low_s_i32x4_i64x2:
+  case WebAssembly::BI__builtin_wasm_extend_high_s_i32x4_i64x2:
+  case WebAssembly::BI__builtin_wasm_extend_low_u_i32x4_i64x2:
+  case WebAssembly::BI__builtin_wasm_extend_high_u_i32x4_i64x2: {
     Value *Vec = EmitScalarExpr(E->getArg(0));
     unsigned IntNo;
     switch (BuiltinID) {
-    case WebAssembly::BI__builtin_wasm_widen_low_s_i32x4_i64x2:
-      IntNo = Intrinsic::wasm_widen_low_signed;
+    case WebAssembly::BI__builtin_wasm_extend_low_s_i32x4_i64x2:
+      IntNo = Intrinsic::wasm_extend_low_signed;
       break;
-    case WebAssembly::BI__builtin_wasm_widen_high_s_i32x4_i64x2:
-      IntNo = Intrinsic::wasm_widen_high_signed;
+    case WebAssembly::BI__builtin_wasm_extend_high_s_i32x4_i64x2:
+      IntNo = Intrinsic::wasm_extend_high_signed;
       break;
-    case WebAssembly::BI__builtin_wasm_widen_low_u_i32x4_i64x2:
-      IntNo = Intrinsic::wasm_widen_low_unsigned;
+    case WebAssembly::BI__builtin_wasm_extend_low_u_i32x4_i64x2:
+      IntNo = Intrinsic::wasm_extend_low_unsigned;
       break;
-    case WebAssembly::BI__builtin_wasm_widen_high_u_i32x4_i64x2:
-      IntNo = Intrinsic::wasm_widen_high_unsigned;
+    case WebAssembly::BI__builtin_wasm_extend_high_u_i32x4_i64x2:
+      IntNo = Intrinsic::wasm_extend_high_unsigned;
       break;
     default:
       llvm_unreachable("unexpected builtin ID");
@@ -17511,26 +17506,6 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
     Function *Callee = CGM.getIntrinsic(IntNo);
     return Builder.CreateCall(Callee, Vec);
   }
-  case WebAssembly::BI__builtin_wasm_widen_s_i8x16_i32x4:
-  case WebAssembly::BI__builtin_wasm_widen_u_i8x16_i32x4: {
-    Value *Vec = EmitScalarExpr(E->getArg(0));
-    llvm::APSInt SubVecConst =
-        *E->getArg(1)->getIntegerConstantExpr(getContext());
-    Value *SubVec = llvm::ConstantInt::get(getLLVMContext(), SubVecConst);
-    unsigned IntNo;
-    switch (BuiltinID) {
-    case WebAssembly::BI__builtin_wasm_widen_s_i8x16_i32x4:
-      IntNo = Intrinsic::wasm_widen_signed;
-      break;
-    case WebAssembly::BI__builtin_wasm_widen_u_i8x16_i32x4:
-      IntNo = Intrinsic::wasm_widen_unsigned;
-      break;
-    default:
-      llvm_unreachable("unexpected builtin ID");
-    }
-    Function *Callee = CGM.getIntrinsic(IntNo);
-    return Builder.CreateCall(Callee, {Vec, SubVec});
-  }
   case WebAssembly::BI__builtin_wasm_convert_low_s_i32x4_f64x2:
   case WebAssembly::BI__builtin_wasm_convert_low_u_i32x4_f64x2: {
     Value *Vec = EmitScalarExpr(E->getArg(0));
@@ -17548,16 +17523,16 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
     Function *Callee = CGM.getIntrinsic(IntNo);
     return Builder.CreateCall(Callee, Vec);
   }
-  case WebAssembly::BI__builtin_wasm_trunc_saturate_zero_s_f64x2_i32x4:
-  case WebAssembly::BI__builtin_wasm_trunc_saturate_zero_u_f64x2_i32x4: {
+  case WebAssembly::BI__builtin_wasm_trunc_sat_zero_s_f64x2_i32x4:
+  case WebAssembly::BI__builtin_wasm_trunc_sat_zero_u_f64x2_i32x4: {
     Value *Vec = EmitScalarExpr(E->getArg(0));
     unsigned IntNo;
     switch (BuiltinID) {
-    case WebAssembly::BI__builtin_wasm_trunc_saturate_zero_s_f64x2_i32x4:
-      IntNo = Intrinsic::wasm_trunc_saturate_zero_signed;
+    case WebAssembly::BI__builtin_wasm_trunc_sat_zero_s_f64x2_i32x4:
+      IntNo = Intrinsic::wasm_trunc_sat_zero_signed;
       break;
-    case WebAssembly::BI__builtin_wasm_trunc_saturate_zero_u_f64x2_i32x4:
-      IntNo = Intrinsic::wasm_trunc_saturate_zero_unsigned;
+    case WebAssembly::BI__builtin_wasm_trunc_sat_zero_u_f64x2_i32x4:
+      IntNo = Intrinsic::wasm_trunc_sat_zero_unsigned;
       break;
     default:
       llvm_unreachable("unexpected builtin ID");
@@ -17645,16 +17620,6 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_shuffle);
     return Builder.CreateCall(Callee, Ops);
   }
-  case WebAssembly::BI__builtin_wasm_prefetch_t: {
-    Value *Ptr = EmitScalarExpr(E->getArg(0));
-    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_prefetch_t);
-    return Builder.CreateCall(Callee, Ptr);
-  }
-  case WebAssembly::BI__builtin_wasm_prefetch_nt: {
-    Value *Ptr = EmitScalarExpr(E->getArg(0));
-    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_prefetch_nt);
-    return Builder.CreateCall(Callee, Ptr);
-  }
   default:
     return nullptr;
   }
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 3a311ab395e402b6020cf2e2867b756b36bde236..d53a623b258c11dddb1f6cdce60cf3554fd95794 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CGCUDARuntime.h"
+#include "CGCXXABI.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "clang/AST/Decl.h"
@@ -260,10 +261,15 @@ std::string CGNVCUDARuntime::getDeviceSideName(const NamedDecl *ND) {
   else
     GD = GlobalDecl(ND);
   std::string DeviceSideName;
-  if (DeviceMC->shouldMangleDeclName(ND)) {
+  MangleContext *MC;
+  if (CGM.getLangOpts().CUDAIsDevice)
+    MC = &CGM.getCXXABI().getMangleContext();
+  else
+    MC = DeviceMC.get();
+  if (MC->shouldMangleDeclName(ND)) {
     SmallString<256> Buffer;
     llvm::raw_svector_ostream Out(Buffer);
-    DeviceMC->mangleName(GD, Out);
+    MC->mangleName(GD, Out);
     DeviceSideName = std::string(Out.str());
   } else
     DeviceSideName = std::string(ND->getIdentifier()->getName());
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index dc73e3260891c2b48715f7fa442accc52e06af1f..1d71148d67e67fe469db9509a63bac54355752fd 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2290,7 +2290,7 @@ void CodeGenModule::ConstructAttributeList(
   // Attach attributes to inalloca argument.
   if (IRFunctionArgs.hasInallocaArg()) {
     llvm::AttrBuilder Attrs;
-    Attrs.addAttribute(llvm::Attribute::InAlloca);
+    Attrs.addInAllocaAttr(FI.getArgStruct());
     ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
         llvm::AttributeSet::get(getLLVMContext(), Attrs);
   }
diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp
index 5c57ad0685d58eca04975215f38a085a1622168b..038238c8404672df2982f7b1b30384bfa41fc38a 100644
--- a/clang/lib/CodeGen/CGCoroutine.cpp
+++ b/clang/lib/CodeGen/CGCoroutine.cpp
@@ -556,6 +556,8 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
       {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr});
   createCoroData(*this, CurCoro, CoroId);
   CurCoro.Data->SuspendBB = RetBB;
+  assert(ShouldEmitLifetimeMarkers &&
+         "Must emit lifetime intrinsics for coroutines");
 
   // Backend is allowed to elide memory allocations, to help it, emit
   // auto mem = coro.alloc() ? 0 : ... allocation code ...;
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 468c2b78b488d374b54b50478bb2246b47565028..c80249a9c9fc10289d47862ddf205d101cea358b 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3522,7 +3522,7 @@ void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit,
                                            llvm::DIScope *&FDContext,
                                            llvm::DINodeArray &TParamsArray,
                                            llvm::DINode::DIFlags &Flags) {
-  const auto *FD = cast<FunctionDecl>(GD.getDecl());
+  const auto *FD = cast<FunctionDecl>(GD.getCanonicalDecl().getDecl());
   Name = getFunctionName(FD);
   // Use mangled name as linkage name for C/C++ functions.
   if (FD->hasPrototype()) {
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index e41eeef0433113a730de8282d20751d0afecd203..466ff096b585713f9b3fc9f1fd49ab8a9cdac582 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -409,6 +409,7 @@ class InlinedOpenMPRegionRAII {
   llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
   FieldDecl *LambdaThisCaptureField = nullptr;
   const CodeGen::CGBlockInfo *BlockInfo = nullptr;
+  bool NoInheritance = false;
 
 public:
   /// Constructs region for combined constructs.
@@ -416,16 +417,19 @@ public:
   /// a list of functions used for code generation of implicitly inlined
   /// regions.
   InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
-                          OpenMPDirectiveKind Kind, bool HasCancel)
-      : CGF(CGF) {
+                          OpenMPDirectiveKind Kind, bool HasCancel,
+                          bool NoInheritance = true)
+      : CGF(CGF), NoInheritance(NoInheritance) {
     // Start emission for the construct.
     CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
         CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
-    std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
-    LambdaThisCaptureField = CGF.LambdaThisCaptureField;
-    CGF.LambdaThisCaptureField = nullptr;
-    BlockInfo = CGF.BlockInfo;
-    CGF.BlockInfo = nullptr;
+    if (NoInheritance) {
+      std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
+      LambdaThisCaptureField = CGF.LambdaThisCaptureField;
+      CGF.LambdaThisCaptureField = nullptr;
+      BlockInfo = CGF.BlockInfo;
+      CGF.BlockInfo = nullptr;
+    }
   }
 
   ~InlinedOpenMPRegionRAII() {
@@ -434,9 +438,11 @@ public:
         cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
     delete CGF.CapturedStmtInfo;
     CGF.CapturedStmtInfo = OldCSI;
-    std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
-    CGF.LambdaThisCaptureField = LambdaThisCaptureField;
-    CGF.BlockInfo = BlockInfo;
+    if (NoInheritance) {
+      std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
+      CGF.LambdaThisCaptureField = LambdaThisCaptureField;
+      CGF.BlockInfo = BlockInfo;
+    }
   }
 };
 
@@ -1028,7 +1034,7 @@ LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
       getThreadIDVariable()->getType()->castAs<PointerType>());
 }
 
-void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
+void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
   if (!CGF.HaveInsertPoint())
     return;
   // 1.2.2 OpenMP Language Terminology
@@ -1037,6 +1043,8 @@ void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
   // The point of exit cannot be a branch out of the structured block.
   // longjmp() and throw() must not violate the entry/exit criteria.
   CGF.EHStack.pushTerminate();
+  if (S)
+    CGF.incrementProfileCounter(S);
   CodeGen(CGF);
   CGF.EHStack.popTerminate();
 }
@@ -3857,7 +3865,7 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
           // Processing for implicitly captured variables.
           InlinedOpenMPRegionRAII Region(
               CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
-              /*HasCancel=*/false);
+              /*HasCancel=*/false, /*NoInheritance=*/true);
           SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
         }
         if (Type->isArrayType()) {
@@ -6218,7 +6226,9 @@ void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
                                            bool HasCancel) {
   if (!CGF.HaveInsertPoint())
     return;
-  InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
+  InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
+                                 InnerKind != OMPD_critical &&
+                                     InnerKind != OMPD_master);
   CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
 }
 
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 021eda31ee715048b86cf29924c920a8821e2b2e..fb719efb1a35d9849facffd7b7bf7bcce2ef656b 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -375,6 +375,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
     EmitOMPTargetTeamsDistributeSimdDirective(
         cast<OMPTargetTeamsDistributeSimdDirective>(*S));
     break;
+  case Stmt::OMPInteropDirectiveClass:
+    llvm_unreachable("Interop directive not supported yet.");
+    break;
   }
 }
 
@@ -818,8 +821,11 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
     llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
     if (ConditionScope.requiresCleanups())
       ExitBlock = createBasicBlock("while.exit");
-    llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
-        S.getCond(), getProfileCount(S.getBody()), S.getBody());
+    llvm::MDNode *Weights =
+        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
+    if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
+      BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
+          BoolCondVal, Stmt::getLikelihood(S.getBody()));
     Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights);
 
     if (ExitBlock != LoopExit.getBlock()) {
@@ -945,8 +951,8 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
   // Start the loop with a block that tests the condition.
   // If there's an increment, the continue scope will be overwritten
   // later.
-  JumpDest Continue = getJumpDestInCurrentScope("for.cond");
-  llvm::BasicBlock *CondBlock = Continue.getBlock();
+  JumpDest CondDest = getJumpDestInCurrentScope("for.cond");
+  llvm::BasicBlock *CondBlock = CondDest.getBlock();
   EmitBlock(CondBlock);
 
   bool LoopMustProgress = false;
@@ -964,24 +970,33 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
                  SourceLocToDebugLoc(R.getBegin()),
                  SourceLocToDebugLoc(R.getEnd()), LoopMustProgress);
 
-  // If the for loop doesn't have an increment we can just use the
-  // condition as the continue block.  Otherwise we'll need to create
-  // a block for it (in the current scope, i.e. in the scope of the
-  // condition), and that we will become our continue block.
-  if (S.getInc())
-    Continue = getJumpDestInCurrentScope("for.inc");
-
-  // Store the blocks to use for break and continue.
-  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
-
   // Create a cleanup scope for the condition variable cleanups.
   LexicalScope ConditionScope(*this, S.getSourceRange());
 
+  // If the for loop doesn't have an increment we can just use the condition as
+  // the continue block. Otherwise, if there is no condition variable, we can
+  // form the continue block now. If there is a condition variable, we can't
+  // form the continue block until after we've emitted the condition, because
+  // the condition is in scope in the increment, but Sema's jump diagnostics
+  // ensure that there are no continues from the condition variable that jump
+  // to the loop increment.
+  JumpDest Continue;
+  if (!S.getInc())
+    Continue = CondDest;
+  else if (!S.getConditionVariable())
+    Continue = getJumpDestInCurrentScope("for.inc");
+  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
+
   if (S.getCond()) {
     // If the for statement has a condition scope, emit the local variable
     // declaration.
     if (S.getConditionVariable()) {
       EmitDecl(*S.getConditionVariable());
+
+      // We have entered the condition variable's scope, so we're now able to
+      // jump to the continue block.
+      Continue = S.getInc() ? getJumpDestInCurrentScope("for.inc") : CondDest;
+      BreakContinueStack.back().ContinueBlock = Continue;
     }
 
     llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
@@ -996,8 +1011,11 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // C99 6.8.5p2/p4: The first substatement is executed if the expression
     // compares unequal to 0.  The condition must be a scalar type.
     llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
-    llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
-        S.getCond(), getProfileCount(S.getBody()), S.getBody());
+    llvm::MDNode *Weights =
+        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
+    if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
+      BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
+          BoolCondVal, Stmt::getLikelihood(S.getBody()));
 
     if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal))
       if (C->isOne())
@@ -1082,8 +1100,11 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
   // The body is executed if the expression, contextually converted
   // to bool, is true.
   llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
-  llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
-      S.getCond(), getProfileCount(S.getBody()), S.getBody());
+  llvm::MDNode *Weights =
+      createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
+  if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
+    BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
+        BoolCondVal, Stmt::getLikelihood(S.getBody()));
   Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
 
   if (ExitBlock != LoopExit.getBlock()) {
@@ -1357,7 +1378,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
     // this case.
     (*SwitchWeights)[0] += ThisCount;
   } else if (SwitchLikelihood)
-    Weights = createBranchWeights(LH);
+    Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
 
   Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index a00ae74fa165faad8dfe34cdfd2b3a9d03daf694..600312e15ef4fce76ab713487c7ff9d81308bde2 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -496,6 +496,13 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
   if (LargestVectorWidth)
     CurFn->addFnAttr("min-legal-vector-width", llvm::utostr(LargestVectorWidth));
 
+  // Add vscale attribute if appropriate.
+  if (getLangOpts().ArmSveVectorBits) {
+    unsigned VScale = getLangOpts().ArmSveVectorBits / 128;
+    CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(getLLVMContext(),
+                                                             VScale, VScale));
+  }
+
   // If we generated an unreachable return block, delete it now.
   if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
     Builder.ClearInsertionPoint();
@@ -1311,10 +1318,16 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
 
   Stmt *Body = FD->getBody();
 
-  // Initialize helper which will detect jumps which can cause invalid lifetime
-  // markers.
-  if (Body && ShouldEmitLifetimeMarkers)
-    Bypasses.Init(Body);
+  if (Body) {
+    // Coroutines always emit lifetime markers.
+    if (isa<CoroutineBodyStmt>(Body))
+      ShouldEmitLifetimeMarkers = true;
+
+    // Initialize helper which will detect jumps which can cause invalid
+    // lifetime markers.
+    if (ShouldEmitLifetimeMarkers)
+      Bypasses.Init(Body);
+  }
 
   // Emit the standard function prologue.
   StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
@@ -1764,10 +1777,19 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
     return;
   }
 
+  // Emit the code with the fully general case.
+  llvm::Value *CondV;
+  {
+    ApplyDebugLocation DL(*this, Cond);
+    CondV = EvaluateExprAsBool(Cond);
+  }
+
+  llvm::MDNode *Weights = nullptr;
+  llvm::MDNode *Unpredictable = nullptr;
+
   // If the branch has a condition wrapped by __builtin_unpredictable,
   // create metadata that specifies that the branch is unpredictable.
   // Don't bother if not optimizing because that metadata would not be used.
-  llvm::MDNode *Unpredictable = nullptr;
   auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
   if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
     auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
@@ -1777,18 +1799,17 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
     }
   }
 
-  llvm::MDNode *Weights = createBranchWeights(LH);
-  if (!Weights) {
+  // If there is a Likelihood knowledge for the cond, lower it.
+  // Note that if not optimizing this won't emit anything.
+  llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
+  if (CondV != NewCondV)
+    CondV = NewCondV;
+  else {
+    // Otherwise, lower profile counts. Note that we do this even at -O0.
     uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
     Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
   }
 
-  // Emit the code with the fully general case.
-  llvm::Value *CondV;
-  {
-    ApplyDebugLocation DL(*this, Cond);
-    CondV = EvaluateExprAsBool(Cond);
-  }
   Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
 }
 
@@ -2632,35 +2653,26 @@ llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
   return llvm::DebugLoc();
 }
 
-static Optional<std::pair<uint32_t, uint32_t>>
-getLikelihoodWeights(Stmt::Likelihood LH) {
+llvm::Value *
+CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
+                                                      Stmt::Likelihood LH) {
   switch (LH) {
-  case Stmt::LH_Unlikely:
-    return std::pair<uint32_t, uint32_t>(llvm::UnlikelyBranchWeight,
-                                         llvm::LikelyBranchWeight);
   case Stmt::LH_None:
-    return None;
+    return Cond;
   case Stmt::LH_Likely:
-    return std::pair<uint32_t, uint32_t>(llvm::LikelyBranchWeight,
-                                         llvm::UnlikelyBranchWeight);
+  case Stmt::LH_Unlikely:
+    // Don't generate llvm.expect on -O0 as the backend won't use it for
+    // anything.
+    if (CGM.getCodeGenOpts().OptimizationLevel == 0)
+      return Cond;
+    llvm::Type *CondTy = Cond->getType();
+    assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
+    llvm::Function *FnExpect =
+        CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);
+    llvm::Value *ExpectedValueOfCond =
+        llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);
+    return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
+                              Cond->getName() + ".expval");
   }
   llvm_unreachable("Unknown Likelihood");
 }
-
-llvm::MDNode *CodeGenFunction::createBranchWeights(Stmt::Likelihood LH) const {
-  Optional<std::pair<uint32_t, uint32_t>> LHW = getLikelihoodWeights(LH);
-  if (!LHW)
-    return nullptr;
-
-  llvm::MDBuilder MDHelper(CGM.getLLVMContext());
-  return MDHelper.createBranchWeights(LHW->first, LHW->second);
-}
-
-llvm::MDNode *CodeGenFunction::createProfileOrBranchWeightsForLoop(
-    const Stmt *Cond, uint64_t LoopCount, const Stmt *Body) const {
-  llvm::MDNode *Weights = createProfileWeightsForLoop(Cond, LoopCount);
-  if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
-    Weights = createBranchWeights(Stmt::getLikelihood(Body));
-
-  return Weights;
-}
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index fc342bbdbd711e1a6c746fd6f5e94ca474f99001..c14918912323e411841f60f69dff27e269b86e5c 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -1445,8 +1445,9 @@ private:
   };
   OpenMPCancelExitStack OMPCancelStack;
 
-  /// Calculate branch weights for the likelihood attribute
-  llvm::MDNode *createBranchWeights(Stmt::Likelihood LH) const;
+  /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
+  llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
+                                                    Stmt::Likelihood LH);
 
   CodeGenPGO PGO;
 
@@ -1457,13 +1458,6 @@ private:
   llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
                                             uint64_t LoopCount) const;
 
-  /// Calculate the branch weight for PGO data or the likelihood attribute.
-  /// The function tries to get the weight of \ref createProfileWeightsForLoop.
-  /// If that fails it gets the weight of \ref createBranchWeights.
-  llvm::MDNode *createProfileOrBranchWeightsForLoop(const Stmt *Cond,
-                                                    uint64_t LoopCount,
-                                                    const Stmt *Body) const;
-
 public:
   /// Increment the profiler's counter for the given statement by \p StepV.
   /// If \p StepV is null, the default increment is 1.
@@ -1892,8 +1886,9 @@ private:
   /// function attribute.
   unsigned LargestVectorWidth = 0;
 
-  /// True if we need emit the life-time markers.
-  const bool ShouldEmitLifetimeMarkers;
+  /// True if we need emit the life-time markers. This is initially set in
+  /// the constructor, but could be overwritten to true if this is a coroutine.
+  bool ShouldEmitLifetimeMarkers;
 
   /// Add OpenCL kernel arg metadata and the kernel attribute metadata to
   /// the function metadata.
@@ -4673,7 +4668,6 @@ public:
 
   struct MultiVersionResolverOption {
     llvm::Function *Function;
-    FunctionDecl *FD;
     struct Conds {
       StringRef Architecture;
       llvm::SmallVector<StringRef, 8> Features;
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 75854f69b110b41ee32a37b98ae150fa25936e65..3a197e85ef7b37d4611fd2dc5a8022c217c65428 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -978,20 +978,14 @@ static bool shouldAssumeDSOLocal(const CodeGenModule &CGM,
   if (TT.isOSBinFormatCOFF() || (TT.isOSWindows() && TT.isOSBinFormatMachO()))
     return true;
 
-  const auto &CGOpts = CGM.getCodeGenOpts();
-  llvm::Reloc::Model RM = CGOpts.RelocationModel;
-  const auto &LOpts = CGM.getLangOpts();
-
-  if (TT.isOSBinFormatMachO()) {
-    if (RM == llvm::Reloc::Static)
-      return true;
-    return GV->isStrongDefinitionForLinker();
-  }
-
   // Only handle COFF and ELF for now.
   if (!TT.isOSBinFormatELF())
     return false;
 
+  // If this is not an executable, don't assume anything is local.
+  const auto &CGOpts = CGM.getCodeGenOpts();
+  llvm::Reloc::Model RM = CGOpts.RelocationModel;
+  const auto &LOpts = CGM.getLangOpts();
   if (RM != llvm::Reloc::Static && !LOpts.PIE) {
     // On ELF, if -fno-semantic-interposition is specified and the target
     // supports local aliases, there will be neither CC1
@@ -6265,9 +6259,8 @@ CodeGenModule::createOpenCLIntToSamplerConversion(const Expr *E,
   llvm::Constant *C = ConstantEmitter(CGF).emitAbstract(E, E->getType());
   auto *SamplerT = getOpenCLRuntime().getSamplerType(E->getType().getTypePtr());
   auto *FTy = llvm::FunctionType::get(SamplerT, {C->getType()}, false);
-  auto *Call = CGF.Builder.CreateCall(
+  auto *Call = CGF.EmitRuntimeCall(
       CreateRuntimeFunction(FTy, "__translate_sampler_initializer"), {C});
-  Call->setCallingConv(Call->getCalledFunction()->getCallingConv());
   return Call;
 }
 
diff --git a/clang/lib/Driver/Compilation.cpp b/clang/lib/Driver/Compilation.cpp
index d33055739080432817961ee3e081a161bff0b4cb..f28c23a59940dda932e43bcb1487400cf3e48dec 100644
--- a/clang/lib/Driver/Compilation.cpp
+++ b/clang/lib/Driver/Compilation.cpp
@@ -170,10 +170,11 @@ int Compilation::ExecuteCommand(const Command &C,
 
     // Follow gcc implementation of CC_PRINT_OPTIONS; we could also cache the
     // output stream.
-    if (getDriver().CCPrintOptions && getDriver().CCPrintOptionsFilename) {
+    if (getDriver().CCPrintOptions &&
+        !getDriver().CCPrintOptionsFilename.empty()) {
       std::error_code EC;
       OwnedStream.reset(new llvm::raw_fd_ostream(
-          getDriver().CCPrintOptionsFilename, EC,
+          getDriver().CCPrintOptionsFilename.c_str(), EC,
           llvm::sys::fs::OF_Append | llvm::sys::fs::OF_Text));
       if (EC) {
         getDriver().Diag(diag::err_drv_cc_print_options_failure)
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index dbd365e7c9bcb698109bae1fba5c9e6f2e4fcee9..171d3d5b5b88c5585a23961dd14b95cf8304aa9c 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -135,14 +135,13 @@ Driver::Driver(StringRef ClangExecutable, StringRef TargetTriple,
     : Diags(Diags), VFS(std::move(VFS)), Mode(GCCMode),
       SaveTemps(SaveTempsNone), BitcodeEmbed(EmbedNone), LTOMode(LTOK_None),
       ClangExecutable(ClangExecutable), SysRoot(DEFAULT_SYSROOT),
-      DriverTitle(Title), CCPrintStatReportFilename(nullptr),
-      CCPrintOptionsFilename(nullptr), CCPrintHeadersFilename(nullptr),
-      CCLogDiagnosticsFilename(nullptr), CCCPrintBindings(false),
-      CCPrintOptions(false), CCPrintHeaders(false), CCLogDiagnostics(false),
-      CCGenDiagnostics(false), CCPrintProcessStats(false),
-      TargetTriple(TargetTriple), CCCGenericGCCName(""), Saver(Alloc),
-      CheckInputsExist(true), GenReproducer(false),
-      SuppressMissingInputWarning(false) {
+      DriverTitle(Title), CCPrintStatReportFilename(), CCPrintOptionsFilename(),
+      CCPrintHeadersFilename(), CCLogDiagnosticsFilename(),
+      CCCPrintBindings(false), CCPrintOptions(false), CCPrintHeaders(false),
+      CCLogDiagnostics(false), CCGenDiagnostics(false),
+      CCPrintProcessStats(false), TargetTriple(TargetTriple),
+      CCCGenericGCCName(""), Saver(Alloc), CheckInputsExist(true),
+      GenReproducer(false), SuppressMissingInputWarning(false) {
   // Provide a sane fallback if no VFS is specified.
   if (!this->VFS)
     this->VFS = llvm::vfs::getRealFileSystem();
@@ -1824,6 +1823,15 @@ bool Driver::HandleImmediateArgs(const Compilation &C) {
     return false;
   }
 
+  if (C.getArgs().hasArg(options::OPT_print_runtime_dir)) {
+    if (auto RuntimePath = TC.getRuntimePath()) {
+      llvm::outs() << *RuntimePath << '\n';
+      return false;
+    }
+    llvm::outs() << TC.getCompilerRTPath() << '\n';
+    return false;
+  }
+
   // FIXME: The following handlers should use a callback mechanism, we don't
   // know what the client would like to do.
   if (Arg *A = C.getArgs().getLastArg(options::OPT_print_file_name_EQ)) {
@@ -4048,7 +4056,7 @@ void Driver::BuildJobs(Compilation &C) const {
       else
         LinkingOutput = getDefaultImageName();
 
-      if (!CCPrintStatReportFilename) {
+      if (CCPrintStatReportFilename.empty()) {
         using namespace llvm;
         // Human readable output.
         outs() << sys::path::filename(Cmd.getExecutable()) << ": "
@@ -4071,8 +4079,9 @@ void Driver::BuildJobs(Compilation &C) const {
             << '\n';
         Out.flush();
         std::error_code EC;
-        llvm::raw_fd_ostream OS(CCPrintStatReportFilename, EC,
-                                llvm::sys::fs::OF_Append);
+        llvm::raw_fd_ostream OS(CCPrintStatReportFilename.c_str(), EC,
+                                llvm::sys::fs::OF_Append |
+                                    llvm::sys::fs::OF_Text);
         if (EC)
           return;
         auto L = OS.lock();
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 217ba56c33518e2217b6379cb005f645acbe78bb..6b747f06439fa70d0699ce10a9a0caa9c2698068 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -916,7 +916,8 @@ void ToolChain::AddClangCXXStdlibIsystemArgs(
     const llvm::opt::ArgList &DriverArgs,
     llvm::opt::ArgStringList &CC1Args) const {
   DriverArgs.ClaimAllArgs(options::OPT_stdlibxx_isystem);
-  if (!DriverArgs.hasArg(options::OPT_nostdincxx))
+  if (!DriverArgs.hasArg(options::OPT_nostdinc, options::OPT_nostdincxx,
+                         options::OPT_nostdlibinc))
     for (const auto &P :
          DriverArgs.getAllArgValues(options::OPT_stdlibxx_isystem))
       addSystemInclude(DriverArgs, CC1Args, P);
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index d6cf5a868555304a09db27379fdc6b87a3dd3623..2ce040cfca01ac825ebe02031ed4f97a643f5c33 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -186,6 +186,12 @@ RocmInstallationDetector::getInstallationPathCandidates() {
     ROCmSearchDirs.emplace_back(RocmPathArg.str());
     DoPrintROCmSearchDirs();
     return ROCmSearchDirs;
+  } else if (const char *RocmPathEnv = ::getenv("ROCM_PATH")) {
+    if (!StringRef(RocmPathEnv).empty()) {
+      ROCmSearchDirs.emplace_back(RocmPathEnv);
+      DoPrintROCmSearchDirs();
+      return ROCmSearchDirs;
+    }
   }
 
   // Try to find relative to the compiler binary.
@@ -247,6 +253,43 @@ RocmInstallationDetector::getInstallationPathCandidates() {
 
   ROCmSearchDirs.emplace_back(D.SysRoot + "/opt/rocm",
                               /*StrictChecking=*/true);
+
+  // Find the latest /opt/rocm-{release} directory.
+  std::error_code EC;
+  std::string LatestROCm;
+  llvm::VersionTuple LatestVer;
+  // Get ROCm version from ROCm directory name.
+  auto GetROCmVersion = [](StringRef DirName) {
+    llvm::VersionTuple V;
+    std::string VerStr = DirName.drop_front(strlen("rocm-")).str();
+    // The ROCm directory name follows the format of
+    // rocm-{major}.{minor}.{subMinor}[-{build}]
+    std::replace(VerStr.begin(), VerStr.end(), '-', '.');
+    V.tryParse(VerStr);
+    return V;
+  };
+  for (llvm::vfs::directory_iterator
+           File = D.getVFS().dir_begin(D.SysRoot + "/opt", EC),
+           FileEnd;
+       File != FileEnd && !EC; File.increment(EC)) {
+    llvm::StringRef FileName = llvm::sys::path::filename(File->path());
+    if (!FileName.startswith("rocm-"))
+      continue;
+    if (LatestROCm.empty()) {
+      LatestROCm = FileName.str();
+      LatestVer = GetROCmVersion(LatestROCm);
+      continue;
+    }
+    auto Ver = GetROCmVersion(FileName);
+    if (LatestVer < Ver) {
+      LatestROCm = FileName.str();
+      LatestVer = Ver;
+    }
+  }
+  if (!LatestROCm.empty())
+    ROCmSearchDirs.emplace_back(D.SysRoot + "/opt/" + LatestROCm,
+                                /*StrictChecking=*/true);
+
   DoPrintROCmSearchDirs();
   return ROCmSearchDirs;
 }
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index c531f87228eb5c3725990cadd3d37104a26e972b..78461a6e22fc0906378348bf828e4f80cd5d18ef 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2082,6 +2082,12 @@ void Clang::AddRISCVTargetArgs(const ArgList &Args,
     CmdArgs.push_back("-mllvm");
     CmdArgs.push_back("-epi-pipeline");
 
+    // In EPI we assume vectors of at least 64 bits, even if the spec
+    // says >128. Our implementation predates this and we can't change
+    // that now. Should impact only the vectorizer, mostly.
+    CmdArgs.push_back("-mllvm");
+    CmdArgs.push_back("-riscv-v-vector-bits-min=64");
+
     // FIXME: Remove this.
     CmdArgs.push_back("-mllvm");
     CmdArgs.push_back("-disable-vpred-sdags");
@@ -2354,7 +2360,8 @@ void Clang::DumpCompilationDatabaseFragmentToDir(
       Twine(llvm::sys::path::filename(Input.getFilename())) + ".%%%%.json");
   int FD;
   SmallString<256> TempPath;
-  Err = llvm::sys::fs::createUniqueFile(Path, FD, TempPath);
+  Err = llvm::sys::fs::createUniqueFile(Path, FD, TempPath,
+                                        llvm::sys::fs::OF_Text);
   if (Err) {
     Driver.Diag(diag::err_drv_compilationdatabase) << Path << Err.message();
     return;
@@ -4269,7 +4276,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   if (Args.hasFlag(options::OPT_fsycl, options::OPT_fno_sycl, false)) {
-    CmdArgs.push_back("-fsycl");
     CmdArgs.push_back("-fsycl-is-device");
 
     if (Arg *A = Args.getLastArg(options::OPT_sycl_std_EQ)) {
@@ -4900,8 +4906,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                   options::OPT_fno_experimental_relative_cxx_abi_vtables);
 
   // Handle segmented stacks.
-  if (Args.hasArg(options::OPT_fsplit_stack))
-    CmdArgs.push_back("-split-stacks");
+  if (Args.hasFlag(options::OPT_fsplit_stack, options::OPT_fno_split_stack,
+                   false))
+    CmdArgs.push_back("-fsplit-stack");
 
   RenderFloatingPointOptions(TC, D, OFastEnabled, Args, CmdArgs, JA);
 
@@ -5134,8 +5141,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (D.CCPrintHeaders && !D.CCGenDiagnostics) {
     CmdArgs.push_back("-header-include-file");
-    CmdArgs.push_back(D.CCPrintHeadersFilename ? D.CCPrintHeadersFilename
-                                               : "-");
+    CmdArgs.push_back(!D.CCPrintHeadersFilename.empty()
+                          ? D.CCPrintHeadersFilename.c_str()
+                          : "-");
     CmdArgs.push_back("-sys-header-deps");
   }
   Args.AddLastArg(CmdArgs, options::OPT_P);
@@ -5143,8 +5151,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (D.CCLogDiagnostics && !D.CCGenDiagnostics) {
     CmdArgs.push_back("-diagnostic-log-file");
-    CmdArgs.push_back(D.CCLogDiagnosticsFilename ? D.CCLogDiagnosticsFilename
-                                                 : "-");
+    CmdArgs.push_back(!D.CCLogDiagnosticsFilename.empty()
+                          ? D.CCLogDiagnosticsFilename.c_str()
+                          : "-");
   }
 
   // Give the gen diagnostics more chances to succeed, by avoiding intentional
@@ -5641,8 +5650,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Arg *A = Args.getLastArg(options::OPT_fpatchable_function_entry_EQ)) {
     StringRef S0 = A->getValue(), S = S0;
     unsigned Size, Offset = 0;
-    if (!Triple.isAArch64() && Triple.getArch() != llvm::Triple::x86 &&
-        Triple.getArch() != llvm::Triple::x86_64)
+    if (!Triple.isAArch64() && !Triple.isRISCV() && !Triple.isX86())
       D.Diag(diag::err_drv_unsupported_opt_for_target)
           << A->getAsString(Args) << TripleStr;
     else if (S.consumeInteger(10, Size) ||
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index a09a69f946ef78b796a8cc6ca79ce06b5f7357cd..bc59b6beafc7857042b8362816b61739f86b98ef 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -373,6 +373,18 @@ void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args,
       D.Diag(diag::err_drv_bitcode_unsupported_on_toolchain);
   }
 
+  // If GlobalISel is enabled, pass it through to LLVM.
+  if (Arg *A = Args.getLastArg(options::OPT_fglobal_isel,
+                               options::OPT_fno_global_isel)) {
+    if (A->getOption().matches(options::OPT_fglobal_isel)) {
+      CmdArgs.push_back("-mllvm");
+      CmdArgs.push_back("-global-isel");
+      // Disable abort and fall back to SDAG silently.
+      CmdArgs.push_back("-mllvm");
+      CmdArgs.push_back("-global-isel-abort=0");
+    }
+  }
+
   Args.AddLastArg(CmdArgs, options::OPT_prebind);
   Args.AddLastArg(CmdArgs, options::OPT_noprebind);
   Args.AddLastArg(CmdArgs, options::OPT_nofixprebinding);
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 153a1dfc85927712e593c817925721be710b7f74..bf2a19e7c54ae7617bd06783927ee7480e1f25ef 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -42,7 +42,9 @@ void Flang::AddPreprocessingOptions(const ArgList &Args,
 
 void Flang::AddOtherOptions(const ArgList &Args, ArgStringList &CmdArgs) const {
   Args.AddAllArgs(CmdArgs,
-                  {options::OPT_module_dir, options::OPT_fdebug_module_writer});
+                  {options::OPT_module_dir, options::OPT_fdebug_module_writer,
+                   options::OPT_fintrinsic_modules_path, options::OPT_pedantic,
+                   options::OPT_std_EQ});
 }
 
 void Flang::ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Driver/ToolChains/FreeBSD.cpp b/clang/lib/Driver/ToolChains/FreeBSD.cpp
index d59bb6f8c3b0aa4398f7889197d982c755f12276..c508af655ac2cbdda0bd6f3e36c1528984a0c66d 100644
--- a/clang/lib/Driver/ToolChains/FreeBSD.cpp
+++ b/clang/lib/Driver/ToolChains/FreeBSD.cpp
@@ -409,8 +409,8 @@ void FreeBSD::addLibCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
 void FreeBSD::addLibStdCxxIncludePaths(
     const llvm::opt::ArgList &DriverArgs,
     llvm::opt::ArgStringList &CC1Args) const {
-  addLibStdCXXIncludePaths(getDriver().SysRoot, "/usr/include/c++/4.2", "", "",
-                           "", "", DriverArgs, CC1Args);
+  addLibStdCXXIncludePaths(getDriver().SysRoot + "/usr/include/c++/4.2", "", "",
+                           DriverArgs, CC1Args);
 }
 
 void FreeBSD::AddCXXStdlibLibArgs(const ArgList &Args,
diff --git a/clang/lib/Driver/ToolChains/Fuchsia.cpp b/clang/lib/Driver/ToolChains/Fuchsia.cpp
index 8e086010a984a22f18688812b56406fb620df385..25b7e4ed1d526ef65eace194738dbbd4c5b9c0dc 100644
--- a/clang/lib/Driver/ToolChains/Fuchsia.cpp
+++ b/clang/lib/Driver/ToolChains/Fuchsia.cpp
@@ -95,6 +95,8 @@ void fuchsia::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     std::string Dyld = D.DyldPrefix;
     if (SanArgs.needsAsanRt() && SanArgs.needsSharedRt())
       Dyld += "asan/";
+    if (SanArgs.needsHwasanRt() && SanArgs.needsSharedRt())
+      Dyld += "hwasan/";
     if (SanArgs.needsTsanRt() && SanArgs.needsSharedRt())
       Dyld += "tsan/";
     Dyld += "ld.so.1";
@@ -210,23 +212,41 @@ Fuchsia::Fuchsia(const Driver &D, const llvm::Triple &Triple,
                           .flag("+fsanitize=address")
                           .flag("-fexceptions")
                           .flag("+fno-exceptions"));
+  // HWASan has higher priority because we always want the instrumentated
+  // version.
+  Multilibs.push_back(
+      Multilib("hwasan", {}, {}, 4).flag("+fsanitize=hwaddress"));
+  // Use the hwasan+noexcept variant with HWASan and -fno-exceptions.
+  Multilibs.push_back(Multilib("hwasan+noexcept", {}, {}, 5)
+                          .flag("+fsanitize=hwaddress")
+                          .flag("-fexceptions")
+                          .flag("+fno-exceptions"));
   // Use the relative vtables ABI.
   // TODO: Remove these multilibs once relative vtables are enabled by default
   // for Fuchsia.
-  Multilibs.push_back(Multilib("relative-vtables", {}, {}, 4)
+  Multilibs.push_back(Multilib("relative-vtables", {}, {}, 6)
                           .flag("+fexperimental-relative-c++-abi-vtables"));
-  Multilibs.push_back(Multilib("relative-vtables+noexcept", {}, {}, 5)
+  Multilibs.push_back(Multilib("relative-vtables+noexcept", {}, {}, 7)
                           .flag("+fexperimental-relative-c++-abi-vtables")
                           .flag("-fexceptions")
                           .flag("+fno-exceptions"));
-  Multilibs.push_back(Multilib("relative-vtables+asan", {}, {}, 6)
+  Multilibs.push_back(Multilib("relative-vtables+asan", {}, {}, 8)
                           .flag("+fexperimental-relative-c++-abi-vtables")
                           .flag("+fsanitize=address"));
-  Multilibs.push_back(Multilib("relative-vtables+asan+noexcept", {}, {}, 7)
+  Multilibs.push_back(Multilib("relative-vtables+asan+noexcept", {}, {}, 9)
                           .flag("+fexperimental-relative-c++-abi-vtables")
                           .flag("+fsanitize=address")
                           .flag("-fexceptions")
                           .flag("+fno-exceptions"));
+  Multilibs.push_back(Multilib("relative-vtables+hwasan", {}, {}, 10)
+                          .flag("+fexperimental-relative-c++-abi-vtables")
+                          .flag("+fsanitize=hwaddress"));
+  Multilibs.push_back(Multilib("relative-vtables+hwasan+noexcept", {}, {}, 11)
+                          .flag("+fexperimental-relative-c++-abi-vtables")
+                          .flag("+fsanitize=hwaddress")
+                          .flag("-fexceptions")
+                          .flag("+fno-exceptions"));
+
   Multilibs.FilterOut([&](const Multilib &M) {
     std::vector<std::string> RD = FilePaths(M);
     return std::all_of(RD.begin(), RD.end(), [&](std::string P) {
@@ -239,6 +259,8 @@ Fuchsia::Fuchsia(const Driver &D, const llvm::Triple &Triple,
       Args.hasFlag(options::OPT_fexceptions, options::OPT_fno_exceptions, true),
       "fexceptions", Flags);
   addMultilibFlag(getSanitizerArgs().needsAsanRt(), "fsanitize=address", Flags);
+  addMultilibFlag(getSanitizerArgs().needsHwasanRt(), "fsanitize=hwaddress",
+                  Flags);
 
   addMultilibFlag(
       Args.hasFlag(options::OPT_fexperimental_relative_cxx_abi_vtables,
@@ -368,6 +390,7 @@ void Fuchsia::AddCXXStdlibLibArgs(const ArgList &Args,
 SanitizerMask Fuchsia::getSupportedSanitizers() const {
   SanitizerMask Res = ToolChain::getSupportedSanitizers();
   Res |= SanitizerKind::Address;
+  Res |= SanitizerKind::HWAddress;
   Res |= SanitizerKind::PointerCompare;
   Res |= SanitizerKind::PointerSubtract;
   Res |= SanitizerKind::Fuzzer;
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index 8bf825fa6a79818c43a79be8822943e113a84876..c216cff46254187244fd06fc5bdba7864530500b 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -1869,10 +1869,18 @@ Generic_GCC::GCCVersion Generic_GCC::GCCVersion::Parse(StringRef VersionText) {
   return GoodVersion;
 }
 
-static llvm::StringRef getGCCToolchainDir(const ArgList &Args) {
+static llvm::StringRef getGCCToolchainDir(const ArgList &Args,
+                                          llvm::StringRef SysRoot) {
   const Arg *A = Args.getLastArg(clang::driver::options::OPT_gcc_toolchain);
   if (A)
     return A->getValue();
+
+  // If we have a SysRoot, ignore GCC_INSTALL_PREFIX.
+  // GCC_INSTALL_PREFIX specifies the gcc installation for the default
+  // sysroot and is likely not valid with a different sysroot.
+  if (!SysRoot.empty())
+    return "";
+
   return GCC_INSTALL_PREFIX;
 }
 
@@ -1901,10 +1909,8 @@ void Generic_GCC::GCCInstallationDetector::init(
                            CandidateBiarchTripleAliases);
 
   // Compute the set of prefixes for our search.
-  SmallVector<std::string, 8> Prefixes(D.PrefixDirs.begin(),
-                                       D.PrefixDirs.end());
-
-  StringRef GCCToolchainDir = getGCCToolchainDir(Args);
+  SmallVector<std::string, 8> Prefixes;
+  StringRef GCCToolchainDir = getGCCToolchainDir(Args, D.SysRoot);
   if (GCCToolchainDir != "") {
     if (GCCToolchainDir.back() == '/')
       GCCToolchainDir = GCCToolchainDir.drop_back(); // remove the /
@@ -1949,7 +1955,8 @@ void Generic_GCC::GCCInstallationDetector::init(
 
   // Loop over the various components which exist and select the best GCC
   // installation available. GCC installs are ranked by version number.
-  Version = GCCVersion::Parse("0.0.0");
+  const GCCVersion VersionZero = GCCVersion::Parse("0.0.0");
+  Version = VersionZero;
   for (const std::string &Prefix : Prefixes) {
     auto &VFS = D.getVFS();
     if (!VFS.exists(Prefix))
@@ -1982,6 +1989,10 @@ void Generic_GCC::GCCInstallationDetector::init(
         ScanLibDirForGCCTriple(TargetTriple, Args, LibDir, Candidate, true,
                                GCCDirExists, GCCCrossDirExists);
     }
+
+    // Skip other prefixes once a GCC installation is found.
+    if (Version > VersionZero)
+      break;
   }
 }
 
@@ -2098,12 +2109,12 @@ void Generic_GCC::GCCInstallationDetector::AddDefaultGCCPrefixes(
   static const char *const X32LibDirs[] = {"/libx32"};
   static const char *const X86LibDirs[] = {"/lib32", "/lib"};
   static const char *const X86Triples[] = {
-      "i686-linux-gnu",       "i686-pc-linux-gnu",     "i486-linux-gnu",
-      "i386-linux-gnu",       "i386-redhat-linux6E",   "i686-redhat-linux",
-      "i586-redhat-linux",    "i386-redhat-linux",     "i586-suse-linux",
-      "i486-slackware-linux", "i686-montavista-linux", "i586-linux-gnu",
-      "i686-linux-android",   "i386-gnu",              "i486-gnu",
-      "i586-gnu",             "i686-gnu"};
+      "i586-linux-gnu",     "i686-linux-gnu",
+      "i686-pc-linux-gnu",  "i386-redhat-linux6E",
+      "i686-redhat-linux",  "i386-redhat-linux",
+      "i586-suse-linux",    "i686-montavista-linux",
+      "i686-linux-android", "i386-gnu",
+  };
 
   static const char *const M68kLibDirs[] = {"/lib"};
   static const char *const M68kTriples[] = {
@@ -2508,7 +2519,6 @@ void Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple(
     const llvm::Triple &TargetTriple, const ArgList &Args,
     const std::string &LibDir, StringRef CandidateTriple,
     bool NeedsBiarchSuffix, bool GCCDirExists, bool GCCCrossDirExists) {
-  llvm::Triple::ArchType TargetArch = TargetTriple.getArch();
   // Locations relative to the system lib directory where GCC's triple-specific
   // directories might reside.
   struct GCCLibSuffix {
@@ -2532,24 +2542,7 @@ void Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple(
       // files in that location, not just GCC installation data.
       {CandidateTriple.str(), "..",
        TargetTriple.getVendor() == llvm::Triple::Freescale ||
-       TargetTriple.getVendor() == llvm::Triple::OpenEmbedded},
-
-      // Natively multiarch systems sometimes put the GCC triple-specific
-      // directory within their multiarch lib directory, resulting in the
-      // triple appearing twice.
-      {CandidateTriple.str() + "/gcc/" + CandidateTriple.str(), "../../..",
-       TargetTriple.getOS() != llvm::Triple::Solaris},
-
-      // Deal with cases (on Ubuntu) where the system architecture could be i386
-      // but the GCC target architecture could be (say) i686.
-      // FIXME: It may be worthwhile to generalize this and look for a second
-      // triple.
-      {"i386-linux-gnu/gcc/" + CandidateTriple.str(), "../../..",
-       (TargetArch == llvm::Triple::x86 &&
-        TargetTriple.getOS() != llvm::Triple::Solaris)},
-      {"i386-gnu/gcc/" + CandidateTriple.str(), "../../..",
-       (TargetArch == llvm::Triple::x86 &&
-        TargetTriple.getOS() != llvm::Triple::Solaris)}};
+           TargetTriple.getVendor() == llvm::Triple::OpenEmbedded}};
 
   for (auto &Suffix : Suffixes) {
     if (!Suffix.Active)
@@ -2786,15 +2779,6 @@ bool Generic_GCC::IsIntegratedAssemblerDefault() const {
   }
 }
 
-static void addMultilibsFilePaths(const Driver &D, const MultilibSet &Multilibs,
-                                  const Multilib &Multilib,
-                                  StringRef InstallPath,
-                                  ToolChain::path_list &Paths) {
-  if (const auto &PathsCallback = Multilibs.filePathsCallback())
-    for (const auto &Path : PathsCallback(Multilib))
-      addPathIfExists(D, InstallPath + Path, Paths);
-}
-
 void Generic_GCC::PushPPaths(ToolChain::path_list &PPaths) {
   // Cross-compiling binutils and GCC installations (vanilla and openSUSE at
   // least) put various tools in a triple-prefixed directory off of the parent
@@ -2821,12 +2805,13 @@ void Generic_GCC::AddMultilibPaths(const Driver &D,
     const std::string &LibPath =
         std::string(GCCInstallation.getParentLibPath());
 
-    // Add toolchain / multilib specific file paths.
-    addMultilibsFilePaths(D, Multilibs, SelectedMultilib,
-                          GCCInstallation.getInstallPath(), Paths);
-
     // Sourcery CodeBench MIPS toolchain holds some libraries under
     // a biarch-like suffix of the GCC installation.
+    if (const auto &PathsCallback = Multilibs.filePathsCallback())
+      for (const auto &Path : PathsCallback(SelectedMultilib))
+        addPathIfExists(D, GCCInstallation.getInstallPath() + Path, Paths);
+
+    // Add lib/gcc/$triple/$version, with an optional /multilib suffix.
     addPathIfExists(
         D, GCCInstallation.getInstallPath() + SelectedMultilib.gccSuffix(),
         Paths);
@@ -2863,10 +2848,8 @@ void Generic_GCC::AddMultilibPaths(const Driver &D,
     // the cross. Note that GCC does include some of these directories in some
     // configurations but this seems somewhere between questionable and simply
     // a bug.
-    if (StringRef(LibPath).startswith(SysRoot)) {
-      addPathIfExists(D, LibPath + "/" + MultiarchTriple, Paths);
+    if (StringRef(LibPath).startswith(SysRoot))
       addPathIfExists(D, LibPath + "/../" + OSLibDir, Paths);
-    }
   }
 }
 
@@ -2874,24 +2857,7 @@ void Generic_GCC::AddMultiarchPaths(const Driver &D,
                                     const std::string &SysRoot,
                                     const std::string &OSLibDir,
                                     path_list &Paths) {
-  // Try walking via the GCC triple path in case of biarch or multiarch GCC
-  // installations with strange symlinks.
   if (GCCInstallation.isValid()) {
-    addPathIfExists(D,
-                    SysRoot + "/usr/lib/" + GCCInstallation.getTriple().str() +
-                        "/../../" + OSLibDir,
-                    Paths);
-
-    // Add the 'other' biarch variant path
-    Multilib BiarchSibling;
-    if (GCCInstallation.getBiarchSibling(BiarchSibling)) {
-      addPathIfExists(
-          D, GCCInstallation.getInstallPath() + BiarchSibling.gccSuffix(),
-                      Paths);
-    }
-
-    // See comments above on the multilib variant for details of why this is
-    // included even from outside the sysroot.
     const std::string &LibPath =
         std::string(GCCInstallation.getParentLibPath());
     const llvm::Triple &GCCTriple = GCCInstallation.getTriple();
@@ -2899,31 +2865,33 @@ void Generic_GCC::AddMultiarchPaths(const Driver &D,
     addPathIfExists(
         D, LibPath + "/../" + GCCTriple.str() + "/lib" + Multilib.osSuffix(),
                     Paths);
-
-    // See comments above on the multilib variant for details of why this is
-    // only included from within the sysroot.
-    if (StringRef(LibPath).startswith(SysRoot))
-      addPathIfExists(D, LibPath, Paths);
   }
 }
 
 void Generic_GCC::AddMultilibIncludeArgs(const ArgList &DriverArgs,
                                          ArgStringList &CC1Args) const {
   // Add include directories specific to the selected multilib set and multilib.
-  if (GCCInstallation.isValid()) {
-    const auto &Callback = Multilibs.includeDirsCallback();
-    if (Callback) {
-      for (const auto &Path : Callback(GCCInstallation.getMultilib()))
-        addExternCSystemIncludeIfExists(
-            DriverArgs, CC1Args, GCCInstallation.getInstallPath() + Path);
-    }
+  if (!GCCInstallation.isValid())
+    return;
+  // gcc TOOL_INCLUDE_DIR.
+  const llvm::Triple &GCCTriple = GCCInstallation.getTriple();
+  std::string LibPath(GCCInstallation.getParentLibPath());
+  addSystemInclude(DriverArgs, CC1Args,
+                   Twine(LibPath) + "/../" + GCCTriple.str() + "/include");
+
+  const auto &Callback = Multilibs.includeDirsCallback();
+  if (Callback) {
+    for (const auto &Path : Callback(GCCInstallation.getMultilib()))
+      addExternCSystemIncludeIfExists(DriverArgs, CC1Args,
+                                      GCCInstallation.getInstallPath() + Path);
   }
 }
 
 void Generic_GCC::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
                                                ArgStringList &CC1Args) const {
-  if (DriverArgs.hasArg(options::OPT_nostdlibinc) ||
-      DriverArgs.hasArg(options::OPT_nostdincxx))
+  if (DriverArgs.hasArg(options::OPT_nostdinc) ||
+      DriverArgs.hasArg(options::OPT_nostdincxx) ||
+      DriverArgs.hasArg(options::OPT_nostdlibinc))
     return;
 
   switch (GetCXXStdlibType(DriverArgs)) {
@@ -2982,36 +2950,41 @@ Generic_GCC::addLibCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
     return;
 }
 
-/// Helper to add the variant paths of a libstdc++ installation.
-bool Generic_GCC::addLibStdCXXIncludePaths(
-    Twine Base, Twine Suffix, StringRef GCCTriple, StringRef GCCMultiarchTriple,
-    StringRef TargetMultiarchTriple, Twine IncludeSuffix,
-    const ArgList &DriverArgs, ArgStringList &CC1Args) const {
-  if (!getVFS().exists(Base + Suffix))
+bool Generic_GCC::addLibStdCXXIncludePaths(Twine IncludeDir, StringRef Triple,
+                                           Twine IncludeSuffix,
+                                           const llvm::opt::ArgList &DriverArgs,
+                                           llvm::opt::ArgStringList &CC1Args,
+                                           bool DetectDebian) const {
+  if (!getVFS().exists(IncludeDir))
     return false;
 
-  addSystemInclude(DriverArgs, CC1Args, Base + Suffix);
-
-  // The vanilla GCC layout of libstdc++ headers uses a triple subdirectory. If
-  // that path exists or we have neither a GCC nor target multiarch triple, use
-  // this vanilla search path.
-  if ((GCCMultiarchTriple.empty() && TargetMultiarchTriple.empty()) ||
-      getVFS().exists(Base + Suffix + "/" + GCCTriple + IncludeSuffix)) {
-    addSystemInclude(DriverArgs, CC1Args,
-                     Base + Suffix + "/" + GCCTriple + IncludeSuffix);
-  } else {
-    // Otherwise try to use multiarch naming schemes which have normalized the
-    // triples and put the triple before the suffix.
-    //
-    // GCC surprisingly uses *both* the GCC triple with a multilib suffix and
-    // the target triple, so we support that here.
-    addSystemInclude(DriverArgs, CC1Args,
-                     Base + "/" + GCCMultiarchTriple + Suffix + IncludeSuffix);
-    addSystemInclude(DriverArgs, CC1Args,
-                     Base + "/" + TargetMultiarchTriple + Suffix);
+  // GPLUSPLUS_INCLUDE_DIR
+  addSystemInclude(DriverArgs, CC1Args, IncludeDir);
+  // GPLUSPLUS_TOOL_INCLUDE_DIR. If Triple is not empty, add a target-dependent
+  // include directory.
+  if (!Triple.empty()) {
+    if (DetectDebian) {
+      // Debian native gcc has an awful patch g++-multiarch-incdir.diff which
+      // uses include/x86_64-linux-gnu/c++/10$IncludeSuffix instead of
+      // include/c++/10/x86_64-linux-gnu$IncludeSuffix.
+      std::string Dir = IncludeDir.str();
+      StringRef Include =
+          llvm::sys::path::parent_path(llvm::sys::path::parent_path(Dir));
+      std::string Path =
+          (Include + "/" + Triple + Dir.substr(Include.size()) + IncludeSuffix)
+              .str();
+      if (getVFS().exists(Path))
+        addSystemInclude(DriverArgs, CC1Args, Path);
+      else
+        addSystemInclude(DriverArgs, CC1Args,
+                         IncludeDir + "/" + Triple + IncludeSuffix);
+    } else {
+      addSystemInclude(DriverArgs, CC1Args,
+                       IncludeDir + "/" + Triple + IncludeSuffix);
+    }
   }
-
-  addSystemInclude(DriverArgs, CC1Args, Base + Suffix + "/backward");
+  // GPLUSPLUS_BACKWARD_INCLUDE_DIR
+  addSystemInclude(DriverArgs, CC1Args, IncludeDir + "/backward");
   return true;
 }
 
@@ -3029,17 +3002,17 @@ Generic_GCC::addGCCLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
   StringRef InstallDir = GCCInstallation.getInstallPath();
   StringRef TripleStr = GCCInstallation.getTriple().str();
   const Multilib &Multilib = GCCInstallation.getMultilib();
-  const std::string GCCMultiarchTriple = getMultiarchTriple(
-      getDriver(), GCCInstallation.getTriple(), getDriver().SysRoot);
-  const std::string TargetMultiarchTriple =
-      getMultiarchTriple(getDriver(), getTriple(), getDriver().SysRoot);
   const GCCVersion &Version = GCCInstallation.getVersion();
 
-  // The primary search for libstdc++ supports multiarch variants.
-  if (addLibStdCXXIncludePaths(LibDir.str() + "/../include",
-                               "/c++/" + Version.Text, TripleStr,
-                               GCCMultiarchTriple, TargetMultiarchTriple,
-                               Multilib.includeSuffix(), DriverArgs, CC1Args))
+  // Try /../$triple/include/c++/$version then /../include/c++/$version.
+  if (addLibStdCXXIncludePaths(
+          LibDir.str() + "/../" + TripleStr + "/include/c++/" + Version.Text,
+          TripleStr, Multilib.includeSuffix(), DriverArgs, CC1Args))
+    return true;
+  // Detect Debian g++-multiarch-incdir.diff.
+  if (addLibStdCXXIncludePaths(LibDir.str() + "/../include/c++/" + Version.Text,
+                               TripleStr, Multilib.includeSuffix(), DriverArgs,
+                               CC1Args, /*Debian=*/true))
     return true;
 
   // Otherwise, fall back on a bunch of options which don't use multiarch
@@ -3054,9 +3027,7 @@ Generic_GCC::addGCCLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
   };
 
   for (const auto &IncludePath : LibStdCXXIncludePathCandidates) {
-    if (addLibStdCXXIncludePaths(IncludePath, /*Suffix*/ "", TripleStr,
-                                 /*GCCMultiarchTriple*/ "",
-                                 /*TargetMultiarchTriple*/ "",
+    if (addLibStdCXXIncludePaths(IncludePath, TripleStr,
                                  Multilib.includeSuffix(), DriverArgs, CC1Args))
       return true;
   }
diff --git a/clang/lib/Driver/ToolChains/Gnu.h b/clang/lib/Driver/ToolChains/Gnu.h
index 90d3bafc1f00b9feee11cb72b580450ea8a8b489..bcd0807174b01e991dba5b1383163c542522763f 100644
--- a/clang/lib/Driver/ToolChains/Gnu.h
+++ b/clang/lib/Driver/ToolChains/Gnu.h
@@ -351,12 +351,11 @@ protected:
   addGCCLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
                            llvm::opt::ArgStringList &CC1Args) const;
 
-  bool addLibStdCXXIncludePaths(Twine Base, Twine Suffix, StringRef GCCTriple,
-                                StringRef GCCMultiarchTriple,
-                                StringRef TargetMultiarchTriple,
+  bool addLibStdCXXIncludePaths(Twine IncludeDir, StringRef Triple,
                                 Twine IncludeSuffix,
                                 const llvm::opt::ArgList &DriverArgs,
-                                llvm::opt::ArgStringList &CC1Args) const;
+                                llvm::opt::ArgStringList &CC1Args,
+                                bool DetectDebian = false) const;
 
   /// @}
 
diff --git a/clang/lib/Driver/ToolChains/Haiku.cpp b/clang/lib/Driver/ToolChains/Haiku.cpp
index 18f550c9ceca1e9fda6ba2c80744eaa31e44c554..a79f0f7622addc394d842f4f044f81828dec7059 100644
--- a/clang/lib/Driver/ToolChains/Haiku.cpp
+++ b/clang/lib/Driver/ToolChains/Haiku.cpp
@@ -29,6 +29,6 @@ void Haiku::addLibCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
 
 void Haiku::addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
                                      llvm::opt::ArgStringList &CC1Args) const {
-  addLibStdCXXIncludePaths(getDriver().SysRoot, "/system/develop/headers/c++",
-                           getTriple().str(), "", "", "", DriverArgs, CC1Args);
+  addLibStdCXXIncludePaths(getDriver().SysRoot + "/system/develop/headers/c++",
+                           getTriple().str(), "", DriverArgs, CC1Args);
 }
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index fb54f73bcd4c890d2b344c1aa5bbabf5d4515571..e58b666dbfc006991b9a29316aa5416904bde618 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -613,15 +613,15 @@ void HexagonToolChain::addLibCxxIncludePaths(
     llvm::opt::ArgStringList &CC1Args) const {
   const Driver &D = getDriver();
   if (!D.SysRoot.empty() && getTriple().isMusl())
-    addLibStdCXXIncludePaths(D.SysRoot + "/usr/include/c++/v1", "", "", "", "",
-                             "", DriverArgs, CC1Args);
-  else if (getTriple().isMusl())
-    addLibStdCXXIncludePaths("/usr/include/c++/v1", "", "", "", "", "",
+    addLibStdCXXIncludePaths(D.SysRoot + "/usr/include/c++/v1", "", "",
                              DriverArgs, CC1Args);
+  else if (getTriple().isMusl())
+    addLibStdCXXIncludePaths("/usr/include/c++/v1", "", "", DriverArgs,
+                             CC1Args);
   else {
     std::string TargetDir = getHexagonTargetDir(D.InstalledDir, D.PrefixDirs);
-    addLibStdCXXIncludePaths(TargetDir, "/hexagon/include/c++/v1", "", "", "",
-                             "", DriverArgs, CC1Args);
+    addLibStdCXXIncludePaths(TargetDir + "/hexagon/include/c++/v1", "", "",
+                             DriverArgs, CC1Args);
   }
 }
 void HexagonToolChain::addLibStdCxxIncludePaths(
@@ -629,7 +629,7 @@ void HexagonToolChain::addLibStdCxxIncludePaths(
     llvm::opt::ArgStringList &CC1Args) const {
   const Driver &D = getDriver();
   std::string TargetDir = getHexagonTargetDir(D.InstalledDir, D.PrefixDirs);
-  addLibStdCXXIncludePaths(TargetDir, "/hexagon/include/c++", "", "", "", "",
+  addLibStdCXXIncludePaths(TargetDir + "/hexagon/include/c++", "", "",
                            DriverArgs, CC1Args);
 }
 
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 9b01602bbc978e72cd60f593444f36f48ee6d984..200a8dbc0ec5f03c1cb920ed417b87c7733a5188 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -58,70 +58,42 @@ std::string Linux::getMultiarchTriple(const Driver &D,
   // regardless of what the actual target triple is.
   case llvm::Triple::arm:
   case llvm::Triple::thumb:
-    if (IsAndroid) {
+    if (IsAndroid)
       return "arm-linux-androideabi";
-    } else if (TargetEnvironment == llvm::Triple::GNUEABIHF) {
-      if (D.getVFS().exists(SysRoot + "/lib/arm-linux-gnueabihf"))
-        return "arm-linux-gnueabihf";
-    } else {
-      if (D.getVFS().exists(SysRoot + "/lib/arm-linux-gnueabi"))
-        return "arm-linux-gnueabi";
-    }
-    break;
+    if (TargetEnvironment == llvm::Triple::GNUEABIHF)
+      return "arm-linux-gnueabihf";
+    return "arm-linux-gnueabi";
   case llvm::Triple::armeb:
   case llvm::Triple::thumbeb:
-    if (TargetEnvironment == llvm::Triple::GNUEABIHF) {
-      if (D.getVFS().exists(SysRoot + "/lib/armeb-linux-gnueabihf"))
-        return "armeb-linux-gnueabihf";
-    } else {
-      if (D.getVFS().exists(SysRoot + "/lib/armeb-linux-gnueabi"))
-        return "armeb-linux-gnueabi";
-    }
-    break;
+    if (TargetEnvironment == llvm::Triple::GNUEABIHF)
+      return "armeb-linux-gnueabihf";
+    return "armeb-linux-gnueabi";
   case llvm::Triple::x86:
     if (IsAndroid)
       return "i686-linux-android";
-    if (D.getVFS().exists(SysRoot + "/lib/i386-linux-gnu"))
-      return "i386-linux-gnu";
-    break;
+    return "i386-linux-gnu";
   case llvm::Triple::x86_64:
     if (IsAndroid)
       return "x86_64-linux-android";
-    // We don't want this for x32, otherwise it will match x86_64 libs
-    if (TargetEnvironment != llvm::Triple::GNUX32 &&
-        D.getVFS().exists(SysRoot + "/lib/x86_64-linux-gnu"))
-      return "x86_64-linux-gnu";
-    break;
+    if (TargetEnvironment == llvm::Triple::GNUX32)
+      return "x86_64-linux-gnux32";
+    return "x86_64-linux-gnu";
   case llvm::Triple::aarch64:
     if (IsAndroid)
       return "aarch64-linux-android";
-    if (D.getVFS().exists(SysRoot + "/lib/aarch64-linux-gnu"))
-      return "aarch64-linux-gnu";
-    break;
+    return "aarch64-linux-gnu";
   case llvm::Triple::aarch64_be:
-    if (D.getVFS().exists(SysRoot + "/lib/aarch64_be-linux-gnu"))
-      return "aarch64_be-linux-gnu";
-    break;
+    return "aarch64_be-linux-gnu";
 
   case llvm::Triple::m68k:
-    if (D.getVFS().exists(SysRoot + "/lib/m68k-linux-gnu"))
-      return "m68k-linux-gnu";
-    break;
+    return "m68k-linux-gnu";
 
-  case llvm::Triple::mips: {
-    std::string MT = IsMipsR6 ? "mipsisa32r6-linux-gnu" : "mips-linux-gnu";
-    if (D.getVFS().exists(SysRoot + "/lib/" + MT))
-      return MT;
-    break;
-  }
-  case llvm::Triple::mipsel: {
+  case llvm::Triple::mips:
+    return IsMipsR6 ? "mipsisa32r6-linux-gnu" : "mips-linux-gnu";
+  case llvm::Triple::mipsel:
     if (IsAndroid)
       return "mipsel-linux-android";
-    std::string MT = IsMipsR6 ? "mipsisa32r6el-linux-gnu" : "mipsel-linux-gnu";
-    if (D.getVFS().exists(SysRoot + "/lib/" + MT))
-      return MT;
-    break;
-  }
+    return IsMipsR6 ? "mipsisa32r6el-linux-gnu" : "mipsel-linux-gnu";
   case llvm::Triple::mips64: {
     std::string MT = std::string(IsMipsR6 ? "mipsisa64r6" : "mips64") +
                      "-linux-" + (IsMipsN32Abi ? "gnuabin32" : "gnuabi64");
@@ -145,33 +117,19 @@ std::string Linux::getMultiarchTriple(const Driver &D,
   case llvm::Triple::ppc:
     if (D.getVFS().exists(SysRoot + "/lib/powerpc-linux-gnuspe"))
       return "powerpc-linux-gnuspe";
-    if (D.getVFS().exists(SysRoot + "/lib/powerpc-linux-gnu"))
-      return "powerpc-linux-gnu";
-    break;
+    return "powerpc-linux-gnu";
   case llvm::Triple::ppcle:
-    if (D.getVFS().exists(SysRoot + "/lib/powerpcle-linux-gnu"))
-      return "powerpcle-linux-gnu";
-    break;
+    return "powerpcle-linux-gnu";
   case llvm::Triple::ppc64:
-    if (D.getVFS().exists(SysRoot + "/lib/powerpc64-linux-gnu"))
-      return "powerpc64-linux-gnu";
-    break;
+    return "powerpc64-linux-gnu";
   case llvm::Triple::ppc64le:
-    if (D.getVFS().exists(SysRoot + "/lib/powerpc64le-linux-gnu"))
-      return "powerpc64le-linux-gnu";
-    break;
+    return "powerpc64le-linux-gnu";
   case llvm::Triple::sparc:
-    if (D.getVFS().exists(SysRoot + "/lib/sparc-linux-gnu"))
-      return "sparc-linux-gnu";
-    break;
+    return "sparc-linux-gnu";
   case llvm::Triple::sparcv9:
-    if (D.getVFS().exists(SysRoot + "/lib/sparc64-linux-gnu"))
-      return "sparc64-linux-gnu";
-    break;
+    return "sparc64-linux-gnu";
   case llvm::Triple::systemz:
-    if (D.getVFS().exists(SysRoot + "/lib/s390x-linux-gnu"))
-      return "s390x-linux-gnu";
-    break;
+    return "s390x-linux-gnu";
   }
   return TargetTriple.str();
 }
@@ -307,16 +265,6 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
 
   Generic_GCC::AddMultilibPaths(D, SysRoot, OSLibDir, MultiarchTriple, Paths);
 
-  // Similar to the logic for GCC above, if we currently running Clang inside
-  // of the requested system root, add its parent library paths to
-  // those searched.
-  // FIXME: It's not clear whether we should use the driver's installed
-  // directory ('Dir' below) or the ResourceDir.
-  if (StringRef(D.Dir).startswith(SysRoot)) {
-    addPathIfExists(D, D.Dir + "/../lib/" + MultiarchTriple, Paths);
-    addPathIfExists(D, D.Dir + "/../" + OSLibDir, Paths);
-  }
-
   addPathIfExists(D, SysRoot + "/lib/" + MultiarchTriple, Paths);
   addPathIfExists(D, SysRoot + "/lib/../" + OSLibDir, Paths);
 
@@ -464,6 +412,14 @@ std::string Linux::getDynamicLinker(const ArgList &Args) const {
       ArchName = "armeb";
       IsArm = true;
       break;
+    case llvm::Triple::x86:
+      ArchName = "i386";
+      break;
+    case llvm::Triple::x86_64:
+      ArchName = Triple.getEnvironment() == llvm::Triple::MuslX32
+                     ? "x32"
+                     : Triple.getArchName().str();
+      break;
     default:
       ArchName = Triple.getArchName().str();
     }
@@ -598,9 +554,10 @@ void Linux::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
   if (DriverArgs.hasArg(clang::driver::options::OPT_nostdinc))
     return;
 
-  if (!DriverArgs.hasArg(options::OPT_nostdlibinc))
-    addSystemInclude(DriverArgs, CC1Args, SysRoot + "/usr/local/include");
-
+  // Add 'include' in the resource directory, which is similar to
+  // GCC_INCLUDE_DIR (private headers) in GCC. Note: the include directory
+  // contains some files conflicting with system /usr/include. musl systems
+  // prefer the /usr/include copies which are more relevant.
   SmallString<128> ResourceDirInclude(D.ResourceDir);
   llvm::sys::path::append(ResourceDirInclude, "include");
   if (!DriverArgs.hasArg(options::OPT_nobuiltininc) &&
@@ -610,6 +567,11 @@ void Linux::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
   if (DriverArgs.hasArg(options::OPT_nostdlibinc))
     return;
 
+  // LOCAL_INCLUDE_DIR
+  addSystemInclude(DriverArgs, CC1Args, SysRoot + "/usr/local/include");
+  // TOOL_INCLUDE_DIR
+  AddMultilibIncludeArgs(DriverArgs, CC1Args);
+
   // Check for configure-time C include directories.
   StringRef CIncludeDirs(C_INCLUDE_DIRS);
   if (CIncludeDirs != "") {
@@ -623,177 +585,13 @@ void Linux::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
     return;
   }
 
-  // Lacking those, try to detect the correct set of system includes for the
-  // target triple.
-
-  AddMultilibIncludeArgs(DriverArgs, CC1Args);
-
-  // Implement generic Debian multiarch support.
-  const StringRef X86_64MultiarchIncludeDirs[] = {
-      "/usr/include/x86_64-linux-gnu",
-
-      // FIXME: These are older forms of multiarch. It's not clear that they're
-      // in use in any released version of Debian, so we should consider
-      // removing them.
-      "/usr/include/i686-linux-gnu/64", "/usr/include/i486-linux-gnu/64"};
-  const StringRef X86MultiarchIncludeDirs[] = {
-      "/usr/include/i386-linux-gnu",
-
-      // FIXME: These are older forms of multiarch. It's not clear that they're
-      // in use in any released version of Debian, so we should consider
-      // removing them.
-      "/usr/include/x86_64-linux-gnu/32", "/usr/include/i686-linux-gnu",
-      "/usr/include/i486-linux-gnu"};
-  const StringRef AArch64MultiarchIncludeDirs[] = {
-      "/usr/include/aarch64-linux-gnu"};
-  const StringRef ARMMultiarchIncludeDirs[] = {
-      "/usr/include/arm-linux-gnueabi"};
-  const StringRef ARMHFMultiarchIncludeDirs[] = {
-      "/usr/include/arm-linux-gnueabihf"};
-  const StringRef ARMEBMultiarchIncludeDirs[] = {
-      "/usr/include/armeb-linux-gnueabi"};
-  const StringRef ARMEBHFMultiarchIncludeDirs[] = {
-      "/usr/include/armeb-linux-gnueabihf"};
-  const StringRef M68kMultiarchIncludeDirs[] = {"/usr/include/m68k-linux-gnu"};
-  const StringRef MIPSMultiarchIncludeDirs[] = {"/usr/include/mips-linux-gnu"};
-  const StringRef MIPSELMultiarchIncludeDirs[] = {
-      "/usr/include/mipsel-linux-gnu"};
-  const StringRef MIPS64MultiarchIncludeDirs[] = {
-      "/usr/include/mips64-linux-gnuabi64"};
-  const StringRef MIPS64ELMultiarchIncludeDirs[] = {
-      "/usr/include/mips64el-linux-gnuabi64"};
-  const StringRef MIPSN32MultiarchIncludeDirs[] = {
-      "/usr/include/mips64-linux-gnuabin32"};
-  const StringRef MIPSN32ELMultiarchIncludeDirs[] = {
-      "/usr/include/mips64el-linux-gnuabin32"};
-  const StringRef MIPSR6MultiarchIncludeDirs[] = {
-      "/usr/include/mipsisa32-linux-gnu"};
-  const StringRef MIPSR6ELMultiarchIncludeDirs[] = {
-      "/usr/include/mipsisa32r6el-linux-gnu"};
-  const StringRef MIPS64R6MultiarchIncludeDirs[] = {
-      "/usr/include/mipsisa64r6-linux-gnuabi64"};
-  const StringRef MIPS64R6ELMultiarchIncludeDirs[] = {
-      "/usr/include/mipsisa64r6el-linux-gnuabi64"};
-  const StringRef MIPSN32R6MultiarchIncludeDirs[] = {
-      "/usr/include/mipsisa64r6-linux-gnuabin32"};
-  const StringRef MIPSN32R6ELMultiarchIncludeDirs[] = {
-      "/usr/include/mipsisa64r6el-linux-gnuabin32"};
-  const StringRef PPCMultiarchIncludeDirs[] = {
-      "/usr/include/powerpc-linux-gnu",
-      "/usr/include/powerpc-linux-gnuspe"};
-  const StringRef PPCLEMultiarchIncludeDirs[] = {
-      "/usr/include/powerpcle-linux-gnu"};
-  const StringRef PPC64MultiarchIncludeDirs[] = {
-      "/usr/include/powerpc64-linux-gnu"};
-  const StringRef PPC64LEMultiarchIncludeDirs[] = {
-      "/usr/include/powerpc64le-linux-gnu"};
-  const StringRef SparcMultiarchIncludeDirs[] = {
-      "/usr/include/sparc-linux-gnu"};
-  const StringRef Sparc64MultiarchIncludeDirs[] = {
-      "/usr/include/sparc64-linux-gnu"};
-  const StringRef SYSTEMZMultiarchIncludeDirs[] = {
-      "/usr/include/s390x-linux-gnu"};
-  ArrayRef<StringRef> MultiarchIncludeDirs;
-  switch (getTriple().getArch()) {
-  case llvm::Triple::x86_64:
-    MultiarchIncludeDirs = X86_64MultiarchIncludeDirs;
-    break;
-  case llvm::Triple::x86:
-    MultiarchIncludeDirs = X86MultiarchIncludeDirs;
-    break;
-  case llvm::Triple::aarch64:
-  case llvm::Triple::aarch64_be:
-    MultiarchIncludeDirs = AArch64MultiarchIncludeDirs;
-    break;
-  case llvm::Triple::arm:
-  case llvm::Triple::thumb:
-    if (getTriple().getEnvironment() == llvm::Triple::GNUEABIHF)
-      MultiarchIncludeDirs = ARMHFMultiarchIncludeDirs;
-    else
-      MultiarchIncludeDirs = ARMMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::armeb:
-  case llvm::Triple::thumbeb:
-    if (getTriple().getEnvironment() == llvm::Triple::GNUEABIHF)
-      MultiarchIncludeDirs = ARMEBHFMultiarchIncludeDirs;
-    else
-      MultiarchIncludeDirs = ARMEBMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::m68k:
-    MultiarchIncludeDirs = M68kMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::mips:
-    if (getTriple().getSubArch() == llvm::Triple::MipsSubArch_r6)
-      MultiarchIncludeDirs = MIPSR6MultiarchIncludeDirs;
-    else
-      MultiarchIncludeDirs = MIPSMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::mipsel:
-    if (getTriple().getSubArch() == llvm::Triple::MipsSubArch_r6)
-      MultiarchIncludeDirs = MIPSR6ELMultiarchIncludeDirs;
-    else
-      MultiarchIncludeDirs = MIPSELMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::mips64:
-    if (getTriple().getSubArch() == llvm::Triple::MipsSubArch_r6)
-      if (getTriple().getEnvironment() == llvm::Triple::GNUABIN32)
-        MultiarchIncludeDirs = MIPSN32R6MultiarchIncludeDirs;
-      else
-        MultiarchIncludeDirs = MIPS64R6MultiarchIncludeDirs;
-    else if (getTriple().getEnvironment() == llvm::Triple::GNUABIN32)
-      MultiarchIncludeDirs = MIPSN32MultiarchIncludeDirs;
-    else
-      MultiarchIncludeDirs = MIPS64MultiarchIncludeDirs;
-    break;
-  case llvm::Triple::mips64el:
-    if (getTriple().getSubArch() == llvm::Triple::MipsSubArch_r6)
-      if (getTriple().getEnvironment() == llvm::Triple::GNUABIN32)
-        MultiarchIncludeDirs = MIPSN32R6ELMultiarchIncludeDirs;
-      else
-        MultiarchIncludeDirs = MIPS64R6ELMultiarchIncludeDirs;
-    else if (getTriple().getEnvironment() == llvm::Triple::GNUABIN32)
-      MultiarchIncludeDirs = MIPSN32ELMultiarchIncludeDirs;
-    else
-      MultiarchIncludeDirs = MIPS64ELMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::ppc:
-    MultiarchIncludeDirs = PPCMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::ppcle:
-    MultiarchIncludeDirs = PPCLEMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::ppc64:
-    MultiarchIncludeDirs = PPC64MultiarchIncludeDirs;
-    break;
-  case llvm::Triple::ppc64le:
-    MultiarchIncludeDirs = PPC64LEMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::sparc:
-    MultiarchIncludeDirs = SparcMultiarchIncludeDirs;
-    break;
-  case llvm::Triple::sparcv9:
-    MultiarchIncludeDirs = Sparc64MultiarchIncludeDirs;
-    break;
-  case llvm::Triple::systemz:
-    MultiarchIncludeDirs = SYSTEMZMultiarchIncludeDirs;
-    break;
-  default:
-    break;
-  }
-
-  const std::string AndroidMultiarchIncludeDir =
-      std::string("/usr/include/") +
-      getMultiarchTriple(D, getTriple(), SysRoot);
-  const StringRef AndroidMultiarchIncludeDirs[] = {AndroidMultiarchIncludeDir};
-  if (getTriple().isAndroid())
-    MultiarchIncludeDirs = AndroidMultiarchIncludeDirs;
-
-  for (StringRef Dir : MultiarchIncludeDirs) {
-    if (D.getVFS().exists(SysRoot + Dir)) {
-      addExternCSystemInclude(DriverArgs, CC1Args, SysRoot + Dir);
-      break;
-    }
-  }
+  // On systems using multiarch and Android, add /usr/include/$triple before
+  // /usr/include.
+  std::string MultiarchIncludeDir = getMultiarchTriple(D, getTriple(), SysRoot);
+  if (!MultiarchIncludeDir.empty() &&
+      D.getVFS().exists(SysRoot + "/usr/include/" + MultiarchIncludeDir))
+    addExternCSystemInclude(DriverArgs, CC1Args,
+                            SysRoot + "/usr/include/" + MultiarchIncludeDir);
 
   if (getTriple().getOS() == llvm::Triple::RTEMS)
     return;
@@ -837,9 +635,7 @@ void Linux::addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
   };
 
   for (const auto &IncludePath : LibStdCXXIncludePathCandidates) {
-    if (addLibStdCXXIncludePaths(IncludePath, /*Suffix*/ "", TripleStr,
-                                 /*GCCMultiarchTriple*/ "",
-                                 /*TargetMultiarchTriple*/ "",
+    if (addLibStdCXXIncludePaths(IncludePath, TripleStr,
                                  Multilib.includeSuffix(), DriverArgs, CC1Args))
       break;
   }
diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp
index 96de02378ca2554c2afb54382b56f2aef01d3e8a..877919e114642d6b8d3268c7b04170b2a23858e3 100644
--- a/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -11,6 +11,7 @@
 #include "Darwin.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/Version.h"
+#include "clang/Config/config.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/DriverDiagnostic.h"
@@ -577,7 +578,10 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // translate 'lld' into 'lld-link', and in the case of the regular msvc
   // linker, we need to use a special search algorithm.
   llvm::SmallString<128> linkPath;
-  StringRef Linker = Args.getLastArgValue(options::OPT_fuse_ld_EQ, "link");
+  StringRef Linker
+    = Args.getLastArgValue(options::OPT_fuse_ld_EQ, CLANG_DEFAULT_LINKER);
+  if (Linker.empty())
+    Linker = "link";
   if (Linker.equals_lower("lld"))
     Linker = "lld-link";
 
diff --git a/clang/lib/Driver/ToolChains/Myriad.cpp b/clang/lib/Driver/ToolChains/Myriad.cpp
index ab0df5d8f1683ef9023e5e738a13a738c87f7f1d..f31466633104c932daaf6e36a81dc2723c789b30 100644
--- a/clang/lib/Driver/ToolChains/Myriad.cpp
+++ b/clang/lib/Driver/ToolChains/Myriad.cpp
@@ -260,7 +260,7 @@ void MyriadToolChain::addLibStdCxxIncludePaths(
   const Multilib &Multilib = GCCInstallation.getMultilib();
   addLibStdCXXIncludePaths(
       LibDir.str() + "/../" + TripleStr.str() + "/include/c++/" + Version.Text,
-      "", TripleStr, "", "", Multilib.includeSuffix(), DriverArgs, CC1Args);
+      TripleStr, Multilib.includeSuffix(), DriverArgs, CC1Args);
 }
 
 // MyriadToolChain handles several triples:
diff --git a/clang/lib/Driver/ToolChains/NetBSD.cpp b/clang/lib/Driver/ToolChains/NetBSD.cpp
index 48bf061c6650d94ad8799f8326f45e90f463c5dd..1ce5a2a203c24482179ef120b9ae856c9c19d12a 100644
--- a/clang/lib/Driver/ToolChains/NetBSD.cpp
+++ b/clang/lib/Driver/ToolChains/NetBSD.cpp
@@ -452,8 +452,8 @@ void NetBSD::addLibCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
 
 void NetBSD::addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
                                       llvm::opt::ArgStringList &CC1Args) const {
-  addLibStdCXXIncludePaths(getDriver().SysRoot, "/usr/include/g++", "", "", "",
-                           "", DriverArgs, CC1Args);
+  addLibStdCXXIncludePaths(getDriver().SysRoot + "/usr/include/g++", "", "",
+                           DriverArgs, CC1Args);
 }
 
 llvm::ExceptionHandling NetBSD::GetExceptionModel(const ArgList &Args) const {
diff --git a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
index 0dc12c7a84b5640ffe5fb6cfc0c67d766940b5c2..9a29cc0985fc961ece74f5c976b72225044077d9 100644
--- a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
+++ b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
@@ -112,7 +112,8 @@ void RISCVToolChain::addLibStdCxxIncludePaths(
   StringRef TripleStr = GCCInstallation.getTriple().str();
   const Multilib &Multilib = GCCInstallation.getMultilib();
   addLibStdCXXIncludePaths(computeSysRoot() + "/include/c++/" + Version.Text,
-      "", TripleStr, "", "", Multilib.includeSuffix(), DriverArgs, CC1Args);
+                           TripleStr, Multilib.includeSuffix(), DriverArgs,
+                           CC1Args);
 }
 
 std::string RISCVToolChain::computeSysRoot() const {
diff --git a/clang/lib/Driver/ToolChains/Solaris.cpp b/clang/lib/Driver/ToolChains/Solaris.cpp
index 4ed4d839ad106c179afc319420455e39212d6cff..4d1af094f4814826f413ffe260752dd96bcadaee 100644
--- a/clang/lib/Driver/ToolChains/Solaris.cpp
+++ b/clang/lib/Driver/ToolChains/Solaris.cpp
@@ -283,9 +283,7 @@ void Solaris::addLibStdCxxIncludePaths(
   const GCCVersion &Version = GCCInstallation.getVersion();
 
   // The primary search for libstdc++ supports multiarch variants.
-  addLibStdCXXIncludePaths(LibDir.str() + "/../include", "/c++/" + Version.Text,
-                           TripleStr,
-                           /*GCCMultiarchTriple*/ "",
-                           /*TargetMultiarchTriple*/ "",
-                           Multilib.includeSuffix(), DriverArgs, CC1Args);
+  addLibStdCXXIncludePaths(LibDir.str() + "/../include/c++/" + Version.Text,
+                           TripleStr, Multilib.includeSuffix(), DriverArgs,
+                           CC1Args);
 }
diff --git a/clang/lib/Driver/ToolChains/WebAssembly.cpp b/clang/lib/Driver/ToolChains/WebAssembly.cpp
index 8c4d99b8ad07ef0e77dc3d9fe92305cba62a7588..e1ca90b195e207b6fe65008d97b673bab2d20cfd 100644
--- a/clang/lib/Driver/ToolChains/WebAssembly.cpp
+++ b/clang/lib/Driver/ToolChains/WebAssembly.cpp
@@ -294,6 +294,36 @@ void WebAssembly::addClangTargetOptions(const ArgList &DriverArgs,
     CC1Args.push_back("-target-feature");
     CC1Args.push_back("+exception-handling");
   }
+
+  for (const Arg *A : DriverArgs.filtered(options::OPT_mllvm)) {
+    StringRef Opt = A->getValue(0);
+    if (Opt.startswith("-emscripten-cxx-exceptions-allowed")) {
+      // '-mllvm -emscripten-cxx-exceptions-allowed' should be used with
+      // '-mllvm -enable-emscripten-cxx-exceptions'
+      bool EmExceptionArgExists = false;
+      for (const Arg *A : DriverArgs.filtered(options::OPT_mllvm)) {
+        if (StringRef(A->getValue(0)) == "-enable-emscripten-cxx-exceptions") {
+          EmExceptionArgExists = true;
+          break;
+        }
+      }
+      if (!EmExceptionArgExists)
+        getDriver().Diag(diag::err_drv_argument_only_allowed_with)
+            << "-mllvm -emscripten-cxx-exceptions-allowed"
+            << "-mllvm -enable-emscripten-cxx-exceptions";
+
+      // Prevent functions specified in -emscripten-cxx-exceptions-allowed list
+      // from being inlined before reaching the wasm backend.
+      StringRef FuncNamesStr = Opt.split('=').second;
+      SmallVector<StringRef, 4> FuncNames;
+      FuncNamesStr.split(FuncNames, ',');
+      for (auto Name : FuncNames) {
+        CC1Args.push_back("-mllvm");
+        CC1Args.push_back(DriverArgs.MakeArgString("--force-attribute=" + Name +
+                                                   ":noinline"));
+      }
+    }
+  }
 }
 
 ToolChain::RuntimeLibType WebAssembly::GetDefaultRuntimeLibType() const {
diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 9bdebe2dd761e1c30cfbc29bf098a88aae275201..b7ccdf23cbaaf5bc7cf072d5a8a3a015adc483d3 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -126,7 +126,7 @@ bool types::isAcceptedByClang(ID Id) {
 
   case TY_Asm:
   case TY_C: case TY_PP_C:
-  case TY_CL:
+  case TY_CL: case TY_CLCXX:
   case TY_CUDA: case TY_PP_CUDA:
   case TY_CUDA_DEVICE:
   case TY_HIP:
@@ -160,7 +160,7 @@ bool types::isObjC(ID Id) {
   }
 }
 
-bool types::isOpenCL(ID Id) { return Id == TY_CL; }
+bool types::isOpenCL(ID Id) { return Id == TY_CL || Id == TY_CLCXX; }
 
 bool types::isCXX(ID Id) {
   switch (Id) {
@@ -249,6 +249,7 @@ types::ID types::lookupTypeForExtension(llvm::StringRef Ext) {
            .Case("cc", TY_CXX)
            .Case("CC", TY_CXX)
            .Case("cl", TY_CL)
+           .Case("clcpp", TY_CLCXX)
            .Case("cp", TY_CXX)
            .Case("cu", TY_CUDA)
            .Case("hh", TY_CXXHeader)
@@ -396,6 +397,7 @@ ID types::lookupHeaderTypeForSourceType(ID Id) {
   case types::TY_ObjCXX:
     return types::TY_ObjCXXHeader;
   case types::TY_CL:
+  case types::TY_CLCXX:
     return types::TY_CLHeader;
   }
 }
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 1404d4a8eaeb3fb5f9389ad4fc29b64c6c4b515a..e424fcb34219661cd1593379542be67cac7e1157 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -649,7 +649,6 @@ void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, unsigned AddLevels,
     nextToken();
 
   Line->Level = InitialLevel;
-  FormatTok->setBlockKind(BK_Block);
 
   if (PPStartHash == PPEndHash) {
     Line->MatchingOpeningBlockLineIndex = OpeningLineIndex;
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index ac68fe0aa01af2ed854ce1d8edb8a28da205a714..6016f8d131c747c5b0e4c67a39d58df0115ffbc6 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -278,6 +278,14 @@ AlignTokenSequence(unsigned Start, unsigned End, unsigned Column, F &&Matches,
   //          double z);
   // In the above example, we need to take special care to ensure that
   // 'double z' is indented along with it's owning function 'b'.
+  // The same holds for calling a function:
+  //   double a = foo(x);
+  //   int    b = bar(foo(y),
+  //            foor(z));
+  // Similar for broken string literals:
+  //   double x = 3.14;
+  //   auto s   = "Hello"
+  //          "World";
   // Special handling is required for 'nested' ternary operators.
   SmallVector<unsigned, 16> ScopeStack;
 
@@ -298,8 +306,12 @@ AlignTokenSequence(unsigned Start, unsigned End, unsigned Column, F &&Matches,
       ScopeStack.push_back(i);
 
     bool InsideNestedScope = ScopeStack.size() != 0;
+    bool ContinuedStringLiteral = i > Start &&
+                                  Changes[i].Tok->is(tok::string_literal) &&
+                                  Changes[i - 1].Tok->is(tok::string_literal);
+    bool SkipMatchCheck = InsideNestedScope || ContinuedStringLiteral;
 
-    if (Changes[i].NewlinesBefore > 0 && !InsideNestedScope) {
+    if (Changes[i].NewlinesBefore > 0 && !SkipMatchCheck) {
       Shift = 0;
       FoundMatchOnLine = false;
     }
@@ -307,7 +319,7 @@ AlignTokenSequence(unsigned Start, unsigned End, unsigned Column, F &&Matches,
     // If this is the first matching token to be aligned, remember by how many
     // spaces it has to be shifted, so the rest of the changes on the line are
     // shifted by the same amount
-    if (!FoundMatchOnLine && !InsideNestedScope && Matches(Changes[i])) {
+    if (!FoundMatchOnLine && !SkipMatchCheck && Matches(Changes[i])) {
       FoundMatchOnLine = true;
       Shift = Column - Changes[i].StartOfTokenColumn;
       Changes[i].Spaces += Shift;
@@ -317,15 +329,41 @@ AlignTokenSequence(unsigned Start, unsigned End, unsigned Column, F &&Matches,
     // as mentioned in the ScopeStack comment.
     if (InsideNestedScope && Changes[i].NewlinesBefore > 0) {
       unsigned ScopeStart = ScopeStack.back();
-      if (Changes[ScopeStart - 1].Tok->is(TT_FunctionDeclarationName) ||
-          (ScopeStart > Start + 1 &&
-           Changes[ScopeStart - 2].Tok->is(TT_FunctionDeclarationName)) ||
-          Changes[i].Tok->is(TT_ConditionalExpr) ||
-          (Changes[i].Tok->Previous &&
-           Changes[i].Tok->Previous->is(TT_ConditionalExpr)))
+      auto ShouldShiftBeAdded = [&] {
+        // Function declaration
+        if (Changes[ScopeStart - 1].Tok->is(TT_FunctionDeclarationName))
+          return true;
+
+        // Continued function declaration
+        if (ScopeStart > Start + 1 &&
+            Changes[ScopeStart - 2].Tok->is(TT_FunctionDeclarationName))
+          return true;
+
+        // Continued function call
+        if (ScopeStart > Start + 1 &&
+            Changes[ScopeStart - 2].Tok->is(tok::identifier) &&
+            Changes[ScopeStart - 1].Tok->is(tok::l_paren))
+          return true;
+
+        // Ternary operator
+        if (Changes[i].Tok->is(TT_ConditionalExpr))
+          return true;
+
+        // Continued ternary operator
+        if (Changes[i].Tok->Previous &&
+            Changes[i].Tok->Previous->is(TT_ConditionalExpr))
+          return true;
+
+        return false;
+      };
+
+      if (ShouldShiftBeAdded())
         Changes[i].Spaces += Shift;
     }
 
+    if (ContinuedStringLiteral)
+      Changes[i].Spaces += Shift;
+
     assert(Shift >= 0);
     Changes[i].StartOfTokenColumn += Shift;
     if (i + 1 != Changes.size())
@@ -434,7 +472,10 @@ static unsigned AlignTokens(
         AlignCurrentSequence();
 
       // A new line starts, re-initialize line status tracking bools.
-      FoundMatchOnLine = false;
+      // Keep the match state if a string literal is continued on this line.
+      if (i == 0 || !Changes[i].Tok->is(tok::string_literal) ||
+          !Changes[i - 1].Tok->is(tok::string_literal))
+        FoundMatchOnLine = false;
       LineIsComment = true;
     }
 
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index d40240b5b5273740fc7cc80f0a3f0b304e658779..284b20cb400a5343609e286a0c3bfbdf0300f41d 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -814,15 +814,18 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
     TempPath += OutputExtension;
     TempPath += ".tmp";
     int fd;
-    std::error_code EC =
-        llvm::sys::fs::createUniqueFile(TempPath, fd, TempPath);
+    std::error_code EC = llvm::sys::fs::createUniqueFile(
+        TempPath, fd, TempPath,
+        Binary ? llvm::sys::fs::OF_None : llvm::sys::fs::OF_Text);
 
     if (CreateMissingDirectories &&
         EC == llvm::errc::no_such_file_or_directory) {
       StringRef Parent = llvm::sys::path::parent_path(OutputPath);
       EC = llvm::sys::fs::create_directories(Parent);
       if (!EC) {
-        EC = llvm::sys::fs::createUniqueFile(TempPath, fd, TempPath);
+        EC = llvm::sys::fs::createUniqueFile(TempPath, fd, TempPath,
+                                             Binary ? llvm::sys::fs::OF_None
+                                                    : llvm::sys::fs::OF_Text);
       }
     }
 
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 3de2f8f37bc650cfb8f353daf97498efaba2fcdd..7797c16c75dea9690586d06bf964caf477d8551d 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -466,6 +466,11 @@ static bool FixupInvocation(CompilerInvocation &Invocation,
     LangOpts.NewAlignOverride = 0;
   }
 
+  // Prevent the user from specifying both -fsycl-is-device and -fsycl-is-host.
+  if (LangOpts.SYCLIsDevice && LangOpts.SYCLIsHost)
+    Diags.Report(diag::err_drv_argument_not_allowed_with) << "-fsycl-is-device"
+                                                          << "-fsycl-is-host";
+
   if (Args.hasArg(OPT_fgnu89_inline) && LangOpts.CPlusPlus)
     Diags.Report(diag::err_drv_argument_not_allowed_with)
         << "-fgnu89-inline" << GetInputKindName(IK);
@@ -515,7 +520,9 @@ static bool FixupInvocation(CompilerInvocation &Invocation,
 static unsigned getOptimizationLevel(ArgList &Args, InputKind IK,
                                      DiagnosticsEngine &Diags) {
   unsigned DefaultOpt = llvm::CodeGenOpt::None;
-  if (IK.getLanguage() == Language::OpenCL && !Args.hasArg(OPT_cl_opt_disable))
+  if ((IK.getLanguage() == Language::OpenCL ||
+       IK.getLanguage() == Language::OpenCLCXX) &&
+      !Args.hasArg(OPT_cl_opt_disable))
     DefaultOpt = llvm::CodeGenOpt::Default;
 
   if (Arg *A = Args.getLastArg(options::OPT_O_Group)) {
@@ -1641,6 +1648,12 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
       llvm::is_contained(DebugEntryValueArchs, T.getArch()))
     Opts.EmitCallSiteInfo = true;
 
+  if (!Opts.EnableDIPreservationVerify && Opts.DIBugsReportFilePath.size()) {
+    Diags.Report(diag::warn_ignoring_verify_debuginfo_preserve_export)
+        << Opts.DIBugsReportFilePath;
+    Opts.DIBugsReportFilePath = "";
+  }
+
   Opts.NewStructPathTBAA = !Args.hasArg(OPT_no_struct_path_tbaa) &&
                            Args.hasArg(OPT_new_struct_path_tbaa);
   Opts.OptimizeSize = getOptimizationLevelSize(Args);
@@ -2505,6 +2518,9 @@ static void GenerateFrontendArgs(const FrontendOptions &Opts,
     case Language::OpenCL:
       Lang = "cl";
       break;
+    case Language::OpenCLCXX:
+      Lang = "clcpp";
+      break;
     case Language::CUDA:
       Lang = "cuda";
       break;
@@ -2693,6 +2709,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     DashX = llvm::StringSwitch<InputKind>(XValue)
                 .Case("c", Language::C)
                 .Case("cl", Language::OpenCL)
+                .Case("clcpp", Language::OpenCLCXX)
                 .Case("cuda", Language::CUDA)
                 .Case("hip", Language::HIP)
                 .Case("c++", Language::CXX)
@@ -3058,6 +3075,9 @@ void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
     case Language::OpenCL:
       LangStd = LangStandard::lang_opencl10;
       break;
+    case Language::OpenCLCXX:
+      LangStd = LangStandard::lang_openclcpp;
+      break;
     case Language::CUDA:
       LangStd = LangStandard::lang_cuda;
       break;
@@ -3193,7 +3213,11 @@ static bool IsInputCompatibleWithStandard(InputKind IK,
     return S.getLanguage() == Language::C;
 
   case Language::OpenCL:
-    return S.getLanguage() == Language::OpenCL;
+    return S.getLanguage() == Language::OpenCL ||
+           S.getLanguage() == Language::OpenCLCXX;
+
+  case Language::OpenCLCXX:
+    return S.getLanguage() == Language::OpenCLCXX;
 
   case Language::CXX:
   case Language::ObjCXX:
@@ -3230,6 +3254,8 @@ static const StringRef GetInputKindName(InputKind IK) {
     return "Objective-C++";
   case Language::OpenCL:
     return "OpenCL";
+  case Language::OpenCLCXX:
+    return "C++ for OpenCL";
   case Language::CUDA:
     return "CUDA";
   case Language::RenderScript:
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index 38b6f753134cfd71bc488d4006c14762d8b34295..0a84971c748cb2fc1c90c721d45d7178c8c0f1b3 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -795,7 +795,7 @@ void PreprocessOnlyAction::ExecuteAction() {
 void PrintPreprocessedAction::ExecuteAction() {
   CompilerInstance &CI = getCompilerInstance();
   // Output file may need to be set to 'Binary', to avoid converting Unix style
-  // line feeds (<LF>) to Microsoft style line feeds (<CR><LF>).
+  // line feeds (<LF>) to Microsoft style line feeds (<CR><LF>) on Windows.
   //
   // Look to see what type of line endings the file uses. If there's a
   // CRLF, then we won't open the file up in binary mode. If there is
@@ -807,30 +807,35 @@ void PrintPreprocessedAction::ExecuteAction() {
   // all of their source code on a single line. However, that is still a
   // concern, so if we scan for too long, we'll just assume the file should
   // be opened in binary mode.
-  bool BinaryMode = true;
-  const SourceManager& SM = CI.getSourceManager();
-  if (llvm::Optional<llvm::MemoryBufferRef> Buffer =
-          SM.getBufferOrNone(SM.getMainFileID())) {
-    const char *cur = Buffer->getBufferStart();
-    const char *end = Buffer->getBufferEnd();
-    const char *next = (cur != end) ? cur + 1 : end;
-
-    // Limit ourselves to only scanning 256 characters into the source
-    // file.  This is mostly a sanity check in case the file has no
-    // newlines whatsoever.
-    if (end - cur > 256) end = cur + 256;
-
-    while (next < end) {
-      if (*cur == 0x0D) {  // CR
-        if (*next == 0x0A)  // CRLF
-          BinaryMode = false;
-
-        break;
-      } else if (*cur == 0x0A)  // LF
-        break;
-
-      ++cur;
-      ++next;
+
+  bool BinaryMode = false;
+  if (llvm::Triple(LLVM_HOST_TRIPLE).isOSWindows()) {
+    BinaryMode = true;
+    const SourceManager &SM = CI.getSourceManager();
+    if (llvm::Optional<llvm::MemoryBufferRef> Buffer =
+            SM.getBufferOrNone(SM.getMainFileID())) {
+      const char *cur = Buffer->getBufferStart();
+      const char *end = Buffer->getBufferEnd();
+      const char *next = (cur != end) ? cur + 1 : end;
+
+      // Limit ourselves to only scanning 256 characters into the source
+      // file.  This is mostly a sanity check in case the file has no
+      // newlines whatsoever.
+      if (end - cur > 256)
+        end = cur + 256;
+
+      while (next < end) {
+        if (*cur == 0x0D) {  // CR
+          if (*next == 0x0A) // CRLF
+            BinaryMode = false;
+
+          break;
+        } else if (*cur == 0x0A) // LF
+          break;
+
+        ++cur;
+        ++next;
+      }
     }
   }
 
@@ -862,6 +867,7 @@ void PrintPreambleAction::ExecuteAction() {
   case Language::ObjC:
   case Language::ObjCXX:
   case Language::OpenCL:
+  case Language::OpenCLCXX:
   case Language::CUDA:
   case Language::HIP:
     break;
diff --git a/clang/lib/Frontend/FrontendOptions.cpp b/clang/lib/Frontend/FrontendOptions.cpp
index 4ea13cf0784fcdf09b1bc55fca566b7eadbc9430..37ac428a80033b611a138ac13ba4339079e9ec32 100644
--- a/clang/lib/Frontend/FrontendOptions.cpp
+++ b/clang/lib/Frontend/FrontendOptions.cpp
@@ -29,6 +29,7 @@ InputKind FrontendOptions::getInputKindForExtension(StringRef Extension) {
       .Case("cppm", Language::CXX)
       .Case("iim", InputKind(Language::CXX).getPreprocessed())
       .Case("cl", Language::OpenCL)
+      .Case("clcpp", Language::OpenCLCXX)
       .Cases("cu", "cuh", Language::CUDA)
       .Case("hip", Language::HIP)
       .Cases("ll", "bc", Language::LLVM_IR)
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 3d69316f59a0b5a6cebfe7a4477ef4b7e0bd3722..fa2fc67fa875ede78a5dfacc2ca2bd89c3e6e482 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -474,7 +474,7 @@ static void InitializeStandardPredefinedMacros(const TargetInfo &TI,
       Builder.defineMacro("__FAST_RELAXED_MATH__");
   }
 
-  if (LangOpts.SYCL) {
+  if (LangOpts.SYCLIsDevice || LangOpts.SYCLIsHost) {
     // SYCL Version is set to a value when building SYCL applications
     if (LangOpts.getSYCLVersion() == LangOptions::SYCL_2017)
       Builder.defineMacro("CL_SYCL_LANGUAGE_VERSION", "121");
diff --git a/clang/lib/Frontend/Rewrite/FrontendActions.cpp b/clang/lib/Frontend/Rewrite/FrontendActions.cpp
index 13e668a47a2fe4258dec6fc006b81b66193b06a9..45960068220bd695aa4eebeb24b89820e4f216d4 100644
--- a/clang/lib/Frontend/Rewrite/FrontendActions.cpp
+++ b/clang/lib/Frontend/Rewrite/FrontendActions.cpp
@@ -185,7 +185,7 @@ RewriteObjCAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
 void RewriteMacrosAction::ExecuteAction() {
   CompilerInstance &CI = getCompilerInstance();
   std::unique_ptr<raw_ostream> OS =
-      CI.createDefaultOutputFile(/*Binary=*/false, getCurrentFileOrBufferName());
+      CI.createDefaultOutputFile(/*Binary=*/true, getCurrentFileOrBufferName());
   if (!OS) return;
 
   RewriteMacrosInInput(CI.getPreprocessor(), OS.get());
@@ -194,7 +194,7 @@ void RewriteMacrosAction::ExecuteAction() {
 void RewriteTestAction::ExecuteAction() {
   CompilerInstance &CI = getCompilerInstance();
   std::unique_ptr<raw_ostream> OS =
-      CI.createDefaultOutputFile(false, getCurrentFileOrBufferName());
+      CI.createDefaultOutputFile(/*Binary=*/true, getCurrentFileOrBufferName());
   if (!OS) return;
 
   DoRewriteTest(CI.getPreprocessor(), OS.get());
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index e7918f3cab9211d624b789b72a71309573f52b9a..6d80d66fa11d748fe0809610ffc0958cfcc7502e 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -211,8 +211,6 @@ clang_generate_header(-gen-arm-mve-header arm_mve.td arm_mve.h)
 clang_generate_header(-gen-arm-cde-header arm_cde.td arm_cde.h)
 # Generate riscv_vector.h
 clang_generate_header(-gen-riscv-vector-header riscv_vector.td riscv_vector.h)
-# Generate riscv_vector_generic.h
-clang_generate_header(-gen-riscv-vector-generic-header riscv_vector.td riscv_vector_generic.h)
 
 add_custom_target(clang-resource-headers ALL DEPENDS ${out_files})
 set_target_properties(clang-resource-headers PROPERTIES
diff --git a/clang/lib/Headers/altivec.h b/clang/lib/Headers/altivec.h
index ee39b521c0caf4f2b4fee2b3a1bc53800ebdc4bc..81e4cb686d8d2060ad2fc0c220a879c57d4b512d 100644
--- a/clang/lib/Headers/altivec.h
+++ b/clang/lib/Headers/altivec.h
@@ -41,9 +41,7 @@
 
 #define __ATTRS_o_ai __attribute__((__overloadable__, __always_inline__))
 
-#ifdef __POWER9_VECTOR__
 #include <stddef.h>
-#endif
 
 static __inline__ vector signed char __ATTRS_o_ai vec_perm(
     vector signed char __a, vector signed char __b, vector unsigned char __c);
@@ -295,6 +293,7 @@ vec_add(vector unsigned long long __a, vector unsigned long long __b) {
   return __a + __b;
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_add(vector signed __int128 __a, vector signed __int128 __b) {
   return __a + __b;
@@ -304,6 +303,12 @@ static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_add(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return __a + __b;
 }
+#endif
+
+static __inline__ vector unsigned char __attribute__((__always_inline__))
+vec_add_u128(vector unsigned char __a, vector unsigned char __b) {
+  return __builtin_altivec_vadduqm(__a, __b);
+}
 #endif // defined(__POWER8_VECTOR__) && defined(__powerpc64__)
 
 static __inline__ vector float __ATTRS_o_ai vec_add(vector float __a,
@@ -321,6 +326,7 @@ static __inline__ vector double __ATTRS_o_ai vec_add(vector double __a,
 /* vec_adde */
 
 #if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_adde(vector signed __int128 __a, vector signed __int128 __b,
          vector signed __int128 __c) {
@@ -334,6 +340,13 @@ vec_adde(vector unsigned __int128 __a, vector unsigned __int128 __b,
 }
 #endif
 
+static __inline__ vector unsigned char __attribute__((__always_inline__))
+vec_adde_u128(vector unsigned char __a, vector unsigned char __b,
+              vector unsigned char __c) {
+  return (vector unsigned char)__builtin_altivec_vaddeuqm(__a, __b, __c);
+}
+#endif
+
 static __inline__ vector signed int __ATTRS_o_ai
 vec_adde(vector signed int __a, vector signed int __b,
          vector signed int __c) {
@@ -353,6 +366,7 @@ vec_adde(vector unsigned int __a, vector unsigned int __b,
 /* vec_addec */
 
 #if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_addec(vector signed __int128 __a, vector signed __int128 __b,
           vector signed __int128 __c) {
@@ -364,6 +378,13 @@ vec_addec(vector unsigned __int128 __a, vector unsigned __int128 __b,
           vector unsigned __int128 __c) {
   return __builtin_altivec_vaddecuq(__a, __b, __c);
 }
+#endif
+
+static __inline__ vector unsigned char __attribute__((__always_inline__))
+vec_addec_u128(vector unsigned char __a, vector unsigned char __b,
+               vector unsigned char __c) {
+  return (vector unsigned char)__builtin_altivec_vaddecuq(__a, __b, __c);
+}
 
 static __inline__ vector signed int __ATTRS_o_ai
 vec_addec(vector signed int __a, vector signed int __b,
@@ -535,6 +556,7 @@ vec_addc(vector unsigned int __a, vector unsigned int __b) {
 }
 
 #if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_addc(vector signed __int128 __a, vector signed __int128 __b) {
   return (vector signed __int128)__builtin_altivec_vaddcuq(
@@ -545,6 +567,12 @@ static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_addc(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return __builtin_altivec_vaddcuq(__a, __b);
 }
+#endif
+
+static __inline__ vector unsigned char __attribute__((__always_inline__))
+vec_addc_u128(vector unsigned char __a, vector unsigned char __b) {
+  return (vector unsigned char)__builtin_altivec_vaddcuq(__a, __b);
+}
 #endif // defined(__POWER8_VECTOR__) && defined(__powerpc64__)
 
 /* vec_vaddcuw */
@@ -748,7 +776,8 @@ vec_vadduws(vector unsigned int __a, vector bool int __b) {
   return __builtin_altivec_vadduws(__a, (vector unsigned int)__b);
 }
 
-#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) &&                    \
+    defined(__SIZEOF_INT128__)
 /* vec_vadduqm */
 
 static __inline__ vector signed __int128 __ATTRS_o_ai
@@ -1709,7 +1738,7 @@ vec_cmpeq(vector double __a, vector double __b) {
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector bool __int128 __ATTRS_o_ai
 vec_cmpeq(vector signed __int128 __a, vector signed __int128 __b) {
   return (vector bool __int128)__builtin_altivec_vcmpequq(
@@ -1786,7 +1815,7 @@ vec_cmpne(vector float __a, vector float __b) {
                                                     (vector int)__b);
 }
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector bool __int128 __ATTRS_o_ai
 vec_cmpne(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return (vector bool __int128) ~(__builtin_altivec_vcmpequq(
@@ -1884,6 +1913,7 @@ vec_parity_lsbb(vector signed int __a) {
   return __builtin_altivec_vprtybw(__a);
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_parity_lsbb(vector unsigned __int128 __a) {
   return __builtin_altivec_vprtybq(__a);
@@ -1893,6 +1923,7 @@ static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_parity_lsbb(vector signed __int128 __a) {
   return __builtin_altivec_vprtybq(__a);
 }
+#endif
 
 static __inline__ vector unsigned long long __ATTRS_o_ai
 vec_parity_lsbb(vector unsigned long long __a) {
@@ -2046,7 +2077,7 @@ vec_cmpgt(vector double __a, vector double __b) {
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector bool __int128 __ATTRS_o_ai
 vec_cmpgt(vector signed __int128 __a, vector signed __int128 __b) {
   return (vector bool __int128)__builtin_altivec_vcmpgtsq(
@@ -2120,7 +2151,7 @@ vec_cmpge(vector unsigned long long __a, vector unsigned long long __b) {
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector bool __int128 __ATTRS_o_ai
 vec_cmpge(vector signed __int128 __a, vector signed __int128 __b) {
   return ~(vec_cmpgt(__b, __a));
@@ -2244,7 +2275,7 @@ vec_cmple(vector unsigned long long __a, vector unsigned long long __b) {
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector bool __int128 __ATTRS_o_ai
 vec_cmple(vector signed __int128 __a, vector signed __int128 __b) {
   return vec_cmpge(__b, __a);
@@ -2300,7 +2331,7 @@ vec_cmplt(vector double __a, vector double __b) {
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector bool __int128 __ATTRS_o_ai
 vec_cmplt(vector signed __int128 __a, vector signed __int128 __b) {
   return vec_cmpgt(__b, __a);
@@ -2870,6 +2901,7 @@ static __inline__ vector float __ATTRS_o_ai vec_xl_len(const float *__a, size_t
   return (vector float)__builtin_vsx_lxvl(__a, (__b << 56));
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_xl_len(const signed __int128 *__a, size_t __b) {
   return (vector signed __int128)__builtin_vsx_lxvl(__a, (__b << 56));
@@ -2879,6 +2911,7 @@ static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_xl_len(const unsigned __int128 *__a, size_t __b) {
   return (vector unsigned __int128)__builtin_vsx_lxvl(__a, (__b << 56));
 }
+#endif
 
 static __inline__ vector signed long long __ATTRS_o_ai
 vec_xl_len(const signed long long *__a, size_t __b) {
@@ -2946,6 +2979,7 @@ static __inline__ void __ATTRS_o_ai vec_xst_len(vector float __a, float *__b,
   return __builtin_vsx_stxvl((vector int)__a, __b, (__c << 56));
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ void __ATTRS_o_ai vec_xst_len(vector signed __int128 __a,
                                                 signed __int128 *__b,
                                                 size_t __c) {
@@ -2957,6 +2991,7 @@ static __inline__ void __ATTRS_o_ai vec_xst_len(vector unsigned __int128 __a,
                                                 size_t __c) {
   return __builtin_vsx_stxvl((vector int)__a, __b, (__c << 56));
 }
+#endif
 
 static __inline__ void __ATTRS_o_ai vec_xst_len(vector signed long long __a,
                                                 signed long long *__b,
@@ -3033,10 +3068,32 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
                                                    (__b)))
 #endif
 
+/* vec_ctd */
+#ifdef __VSX__
+#define vec_ctd(__a, __b)                                                      \
+  _Generic((__a), vector signed int                                            \
+           : (vec_doublee((vector signed int)(__a)) *                          \
+              (vector double)(vector unsigned long long)((0x3ffULL - (__b))    \
+                                                         << 52)),              \
+             vector unsigned int                                               \
+           : (vec_doublee((vector unsigned int)(__a)) *                        \
+              (vector double)(vector unsigned long long)((0x3ffULL - (__b))    \
+                                                         << 52)),              \
+             vector unsigned long long                                         \
+           : (__builtin_convertvector((vector unsigned long long)(__a),        \
+                                      vector double) *                         \
+              (vector double)(vector unsigned long long)((0x3ffULL - (__b))    \
+                                                         << 52)),              \
+             vector signed long long                                           \
+           : (__builtin_convertvector((vector signed long long)(__a),          \
+                                      vector double) *                         \
+              (vector double)(vector unsigned long long)((0x3ffULL - (__b))    \
+                                                         << 52)))
+#endif // __VSX__
+
 /* vec_vcfsx */
 
 #define vec_vcfux __builtin_altivec_vcfux
-
 /* vec_vcfux */
 
 #define vec_vcfsx(__a, __b) __builtin_altivec_vcfsx((vector int)(__a), (__b))
@@ -3114,7 +3171,7 @@ vec_signextll(vector signed int __a) {
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_signextq(vector signed long long __a) {
   return __builtin_altivec_vextsd2q(__a);
@@ -3399,6 +3456,15 @@ vec_doubleo(vector float __a) {
   return __builtin_vsx_xvcvspdp(vec_sld(__a, __a, 4));
 #endif
 }
+
+/* vec_cvf */
+static __inline__ vector double __ATTRS_o_ai vec_cvf(vector float __a) {
+  return vec_doublee(__a);
+}
+
+static __inline__ vector float __ATTRS_o_ai vec_cvf(vector double __a) {
+  return vec_floate(__a);
+}
 #endif
 
 /* vec_div */
@@ -3481,6 +3547,7 @@ vec_dive(vector unsigned long long __a, vector unsigned long long __b) {
   return __builtin_altivec_vdiveud(__a, __b);
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_dive(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return __builtin_altivec_vdiveuq(__a, __b);
@@ -3491,8 +3558,9 @@ vec_dive(vector signed __int128 __a, vector signed __int128 __b) {
   return __builtin_altivec_vdivesq(__a, __b);
 }
 #endif
+#endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_div(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return __a / __b;
@@ -5695,7 +5763,7 @@ vec_msum(vector unsigned short __a, vector unsigned short __b,
 
 /* vec_msumc */
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_msumc(vector unsigned long long __a, vector unsigned long long __b,
           vector unsigned __int128 __c) {
@@ -5929,7 +5997,7 @@ vec_mule(vector unsigned int __a, vector unsigned int __b) {
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_mule(vector signed long long __a, vector signed long long __b) {
 #ifdef __LITTLE_ENDIAN__
@@ -6075,7 +6143,7 @@ vec_mulo(vector unsigned int __a, vector unsigned int __b) {
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_mulo(vector signed long long __a, vector signed long long __b) {
 #ifdef __LITTLE_ENDIAN__
@@ -7927,7 +7995,7 @@ vec_rl(vector unsigned long long __a, vector unsigned long long __b) {
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_rl(vector signed __int128 __a, vector unsigned __int128 __b) {
   return (__b << __a)|(__b >> ((__CHAR_BIT__ * sizeof(vector signed __int128)) - __a));
@@ -7954,7 +8022,7 @@ vec_rlmi(vector unsigned long long __a, vector unsigned long long __b,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_rlmi(vector unsigned __int128 __a, vector unsigned __int128 __b,
          vector unsigned __int128 __c) {
@@ -7985,7 +8053,7 @@ vec_rlnm(vector unsigned long long __a, vector unsigned long long __b,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_rlnm(vector unsigned __int128 __a, vector unsigned __int128 __b,
          vector unsigned __int128 __c) {
@@ -11409,7 +11477,8 @@ vec_sub(vector unsigned int __a, vector bool int __b) {
   return __a - (vector unsigned int)__b;
 }
 
-#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) &&                    \
+    defined(__SIZEOF_INT128__)
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_sub(vector signed __int128 __a, vector signed __int128 __b) {
   return __a - __b;
@@ -11419,7 +11488,8 @@ static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_sub(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return __a - __b;
 }
-#endif // defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#endif // defined(__POWER8_VECTOR__) && defined(__powerpc64__) &&
+       // defined(__SIZEOF_INT128__)
 
 #ifdef __VSX__
 static __inline__ vector signed long long __ATTRS_o_ai
@@ -11568,6 +11638,7 @@ vec_subc(vector unsigned int __a, vector unsigned int __b) {
 }
 
 #if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#ifdef __SIZEOF_INT128__
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_subc(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return __builtin_altivec_vsubcuq(__a, __b);
@@ -11577,6 +11648,12 @@ static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_subc(vector signed __int128 __a, vector signed __int128 __b) {
   return __builtin_altivec_vsubcuq(__a, __b);
 }
+#endif
+
+static __inline__ vector unsigned char __attribute__((__always_inline__))
+vec_subc_u128(vector unsigned char __a, vector unsigned char __b) {
+  return (vector unsigned char)__builtin_altivec_vsubcuq(__a, __b);
+}
 #endif // defined(__POWER8_VECTOR__) && defined(__powerpc64__)
 
 /* vec_vsubcuw */
@@ -11783,6 +11860,7 @@ vec_vsubuws(vector unsigned int __a, vector bool int __b) {
 #if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
 /* vec_vsubuqm */
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_vsubuqm(vector signed __int128 __a, vector signed __int128 __b) {
   return __a - __b;
@@ -11792,10 +11870,16 @@ static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_vsubuqm(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return __a - __b;
 }
+#endif
 
-/* vec_vsubeuqm */
+static __inline__ vector unsigned char __attribute__((__always_inline__))
+vec_sub_u128(vector unsigned char __a, vector unsigned char __b) {
+  return __builtin_altivec_vsubuqm(__a, __b);
+}
 
+/* vec_vsubeuqm */
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_vsubeuqm(vector signed __int128 __a, vector signed __int128 __b,
              vector signed __int128 __c) {
@@ -11819,9 +11903,17 @@ vec_sube(vector unsigned __int128 __a, vector unsigned __int128 __b,
              vector unsigned __int128 __c) {
   return __builtin_altivec_vsubeuqm(__a, __b, __c);
 }
+#endif
+
+static __inline__ vector unsigned char __attribute__((__always_inline__))
+vec_sube_u128(vector unsigned char __a, vector unsigned char __b,
+              vector unsigned char __c) {
+  return (vector unsigned char)__builtin_altivec_vsubeuqm(__a, __b, __c);
+}
 
 /* vec_vsubcuq */
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_vsubcuq(vector signed __int128 __a, vector signed __int128 __b) {
   return __builtin_altivec_vsubcuq(__a, __b);
@@ -11845,6 +11937,7 @@ vec_vsubecuq(vector unsigned __int128 __a, vector unsigned __int128 __b,
              vector unsigned __int128 __c) {
   return __builtin_altivec_vsubecuq(__a, __b, __c);
 }
+#endif
 
 static __inline__ vector signed int __ATTRS_o_ai
 vec_subec(vector signed int __a, vector signed int __b,
@@ -11858,6 +11951,7 @@ vec_subec(vector unsigned int __a, vector unsigned int __b,
   return vec_addec(__a, ~__b, __c);
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_subec(vector signed __int128 __a, vector signed __int128 __b,
              vector signed __int128 __c) {
@@ -11869,6 +11963,13 @@ vec_subec(vector unsigned __int128 __a, vector unsigned __int128 __b,
              vector unsigned __int128 __c) {
   return __builtin_altivec_vsubecuq(__a, __b, __c);
 }
+#endif
+
+static __inline__ vector unsigned char __attribute__((__always_inline__))
+vec_subec_u128(vector unsigned char __a, vector unsigned char __b,
+               vector unsigned char __c) {
+  return (vector unsigned char)__builtin_altivec_vsubecuq(__a, __b, __c);
+}
 #endif // defined(__POWER8_VECTOR__) && defined(__powerpc64__)
 
 static __inline__ vector signed int __ATTRS_o_ai
@@ -12558,6 +12659,16 @@ static __inline__ void __ATTRS_o_ai vec_vsx_st(vector unsigned char __a,
 #ifdef __VSX__
 #define vec_xxpermdi __builtin_vsx_xxpermdi
 #define vec_xxsldwi __builtin_vsx_xxsldwi
+#define vec_permi(__a, __b, __c)                                               \
+  _Generic((__a), vector signed long long                                      \
+           : __builtin_shufflevector((__a), (__b), (((__c) >> 1) & 0x1),       \
+                                     (((__c)&0x1) + 2)),                       \
+             vector unsigned long long                                         \
+           : __builtin_shufflevector((__a), (__b), (((__c) >> 1) & 0x1),       \
+                                     (((__c)&0x1) + 2)),                       \
+             vector double                                                     \
+           : __builtin_shufflevector((__a), (__b), (((__c) >> 1) & 0x1),       \
+                                     (((__c)&0x1) + 2)))
 #endif
 
 /* vec_xor */
@@ -14129,7 +14240,8 @@ vec_splats(unsigned long long __a) {
   return (vector unsigned long long)(__a);
 }
 
-#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) &&                    \
+    defined(__SIZEOF_INT128__)
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_splats(signed __int128 __a) {
   return (vector signed __int128)(__a);
@@ -14340,7 +14452,7 @@ static __inline__ int __ATTRS_o_ai vec_all_eq(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_all_eq(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpequq_p(__CR6_LT, __a, __b);
@@ -14523,7 +14635,7 @@ static __inline__ int __ATTRS_o_ai vec_all_ge(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_all_ge(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpgtsq_p(__CR6_EQ, __b, __a);
@@ -14706,7 +14818,7 @@ static __inline__ int __ATTRS_o_ai vec_all_gt(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_all_gt(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpgtsq_p(__CR6_LT, __a, __b);
@@ -14897,7 +15009,7 @@ static __inline__ int __ATTRS_o_ai vec_all_le(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_all_le(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpgtsq_p(__CR6_EQ, __a, __b);
@@ -15081,7 +15193,7 @@ static __inline__ int __ATTRS_o_ai vec_all_lt(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_all_lt(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpgtsq_p(__CR6_LT, __b, __a);
@@ -15297,7 +15409,7 @@ static __inline__ int __ATTRS_o_ai vec_all_ne(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_all_ne(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpequq_p(__CR6_EQ, __a, __b);
@@ -15586,7 +15698,7 @@ static __inline__ int __ATTRS_o_ai vec_any_eq(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_any_eq(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpequq_p(__CR6_EQ_REV, __a, __b);
@@ -15777,7 +15889,7 @@ static __inline__ int __ATTRS_o_ai vec_any_ge(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_any_ge(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpgtsq_p(__CR6_LT_REV, __b, __a);
@@ -15968,7 +16080,7 @@ static __inline__ int __ATTRS_o_ai vec_any_gt(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_any_gt(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpgtsq_p(__CR6_EQ_REV, __a, __b);
@@ -16159,7 +16271,7 @@ static __inline__ int __ATTRS_o_ai vec_any_le(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_any_le(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpgtsq_p(__CR6_LT_REV, __a, __b);
@@ -16350,7 +16462,7 @@ static __inline__ int __ATTRS_o_ai vec_any_lt(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_any_lt(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpgtsq_p(__CR6_EQ_REV, __b, __a);
@@ -16565,7 +16677,7 @@ static __inline__ int __ATTRS_o_ai vec_any_ne(vector double __a,
 }
 #endif
 
-#ifdef __POWER10_VECTOR__
+#if defined(__POWER10_VECTOR__) && defined(__SIZEOF_INT128__)
 static __inline__ int __ATTRS_o_ai vec_any_ne(vector signed __int128 __a,
                                               vector signed __int128 __b) {
   return __builtin_altivec_vcmpequq_p(__CR6_LT_REV, __a, __b);
@@ -16813,6 +16925,16 @@ vec_vgbbd(vector unsigned char __a) {
   return __builtin_altivec_vgbbd(__a);
 }
 
+static __inline__ vector signed long long __ATTRS_o_ai
+vec_gbb(vector signed long long __a) {
+  return __builtin_altivec_vgbbd((vector unsigned char)__a);
+}
+
+static __inline__ vector unsigned long long __ATTRS_o_ai
+vec_gbb(vector unsigned long long __a) {
+  return __builtin_altivec_vgbbd((vector unsigned char)__a);
+}
+
 static __inline__ vector long long __ATTRS_o_ai
 vec_vbpermq(vector signed char __a, vector signed char __b) {
   return __builtin_altivec_vbpermq((vector unsigned char)__a,
@@ -16824,7 +16946,7 @@ vec_vbpermq(vector unsigned char __a, vector unsigned char __b) {
   return __builtin_altivec_vbpermq(__a, __b);
 }
 
-#ifdef __powerpc64__
+#if defined(__powerpc64__) && defined(__SIZEOF_INT128__)
 static __inline__ vector unsigned long long __attribute__((__always_inline__))
 vec_bperm(vector unsigned __int128 __a, vector unsigned char __b) {
   return __builtin_altivec_vbpermq((vector unsigned char)__a,
@@ -16999,7 +17121,8 @@ vec_revb(vector double __a) {
 }
 #endif /* End __VSX__ */
 
-#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) &&                    \
+    defined(__SIZEOF_INT128__)
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_revb(vector signed __int128 __a) {
   vector unsigned char __indices =
@@ -17021,6 +17144,8 @@ vec_revb(vector unsigned __int128 __a) {
 
 /* vec_xl */
 
+#define vec_xld2 vec_xl
+#define vec_xlw4 vec_xl
 typedef vector signed char unaligned_vec_schar __attribute__((aligned(1)));
 typedef vector unsigned char unaligned_vec_uchar __attribute__((aligned(1)));
 typedef vector signed short unaligned_vec_sshort __attribute__((aligned(1)));
@@ -17029,41 +17154,41 @@ typedef vector signed int unaligned_vec_sint __attribute__((aligned(1)));
 typedef vector unsigned int unaligned_vec_uint __attribute__((aligned(1)));
 typedef vector float unaligned_vec_float __attribute__((aligned(1)));
 
-static inline __ATTRS_o_ai vector signed char vec_xl(signed long long __offset,
+static inline __ATTRS_o_ai vector signed char vec_xl(ptrdiff_t __offset,
                                                      const signed char *__ptr) {
   return *(unaligned_vec_schar *)(__ptr + __offset);
 }
 
 static inline __ATTRS_o_ai vector unsigned char
-vec_xl(signed long long __offset, const unsigned char *__ptr) {
+vec_xl(ptrdiff_t __offset, const unsigned char *__ptr) {
   return *(unaligned_vec_uchar*)(__ptr + __offset);
 }
 
-static inline __ATTRS_o_ai vector signed short vec_xl(signed long long __offset,
-                                                      const signed short *__ptr) {
+static inline __ATTRS_o_ai vector signed short
+vec_xl(ptrdiff_t __offset, const signed short *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_sshort *)__addr;
 }
 
 static inline __ATTRS_o_ai vector unsigned short
-vec_xl(signed long long __offset, const unsigned short *__ptr) {
+vec_xl(ptrdiff_t __offset, const unsigned short *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_ushort *)__addr;
 }
 
-static inline __ATTRS_o_ai vector signed int vec_xl(signed long long __offset,
+static inline __ATTRS_o_ai vector signed int vec_xl(ptrdiff_t __offset,
                                                     const signed int *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_sint *)__addr;
 }
 
-static inline __ATTRS_o_ai vector unsigned int vec_xl(signed long long __offset,
-                                                      const unsigned int *__ptr) {
+static inline __ATTRS_o_ai vector unsigned int
+vec_xl(ptrdiff_t __offset, const unsigned int *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_uint *)__addr;
 }
 
-static inline __ATTRS_o_ai vector float vec_xl(signed long long __offset,
+static inline __ATTRS_o_ai vector float vec_xl(ptrdiff_t __offset,
                                                const float *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_float *)__addr;
@@ -17075,36 +17200,37 @@ typedef vector unsigned long long unaligned_vec_ull __attribute__((aligned(1)));
 typedef vector double unaligned_vec_double __attribute__((aligned(1)));
 
 static inline __ATTRS_o_ai vector signed long long
-vec_xl(signed long long __offset, const signed long long *__ptr) {
+vec_xl(ptrdiff_t __offset, const signed long long *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_sll *)__addr;
 }
 
 static inline __ATTRS_o_ai vector unsigned long long
-vec_xl(signed long long __offset, const unsigned long long *__ptr) {
+vec_xl(ptrdiff_t __offset, const unsigned long long *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_ull *)__addr;
 }
 
-static inline __ATTRS_o_ai vector double vec_xl(signed long long __offset,
+static inline __ATTRS_o_ai vector double vec_xl(ptrdiff_t __offset,
                                                 const double *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_double *)__addr;
 }
 #endif
 
-#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) &&                    \
+    defined(__SIZEOF_INT128__)
 typedef vector signed __int128 unaligned_vec_si128 __attribute__((aligned(1)));
 typedef vector unsigned __int128 unaligned_vec_ui128
     __attribute__((aligned(1)));
 static inline __ATTRS_o_ai vector signed __int128
-vec_xl(signed long long __offset, const signed __int128 *__ptr) {
+vec_xl(ptrdiff_t __offset, const signed __int128 *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_si128 *)__addr;
 }
 
 static inline __ATTRS_o_ai vector unsigned __int128
-vec_xl(signed long long __offset, const unsigned __int128 *__ptr) {
+vec_xl(ptrdiff_t __offset, const unsigned __int128 *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   return *(unaligned_vec_ui128 *)__addr;
 }
@@ -17114,27 +17240,27 @@ vec_xl(signed long long __offset, const unsigned __int128 *__ptr) {
 
 #ifdef __LITTLE_ENDIAN__
 static __inline__ vector signed char __ATTRS_o_ai
-vec_xl_be(signed long long __offset, const signed char *__ptr) {
+vec_xl_be(ptrdiff_t __offset, const signed char *__ptr) {
   vector signed char __vec = (vector signed char)__builtin_vsx_lxvd2x_be(__offset, __ptr);
   return __builtin_shufflevector(__vec, __vec, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14,
                                  13, 12, 11, 10, 9, 8);
 }
 
 static __inline__ vector unsigned char __ATTRS_o_ai
-vec_xl_be(signed long long __offset, const unsigned char *__ptr) {
+vec_xl_be(ptrdiff_t __offset, const unsigned char *__ptr) {
   vector unsigned char __vec = (vector unsigned char)__builtin_vsx_lxvd2x_be(__offset, __ptr);
   return __builtin_shufflevector(__vec, __vec, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14,
                                  13, 12, 11, 10, 9, 8);
 }
 
-static __inline__ vector signed short  __ATTRS_o_ai
-vec_xl_be(signed long long __offset, const signed short *__ptr) {
+static __inline__ vector signed short __ATTRS_o_ai
+vec_xl_be(ptrdiff_t __offset, const signed short *__ptr) {
   vector signed short __vec = (vector signed short)__builtin_vsx_lxvd2x_be(__offset, __ptr);
   return __builtin_shufflevector(__vec, __vec, 3, 2, 1, 0, 7, 6, 5, 4);
 }
 
 static __inline__ vector unsigned short __ATTRS_o_ai
-vec_xl_be(signed long long __offset, const unsigned short *__ptr) {
+vec_xl_be(ptrdiff_t __offset, const unsigned short *__ptr) {
   vector unsigned short __vec = (vector unsigned short)__builtin_vsx_lxvd2x_be(__offset, __ptr);
   return __builtin_shufflevector(__vec, __vec, 3, 2, 1, 0, 7, 6, 5, 4);
 }
@@ -17171,7 +17297,8 @@ vec_xl_be(signed long long  __offset, const double *__ptr) {
 }
 #endif
 
-#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) &&                    \
+    defined(__SIZEOF_INT128__)
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_xl_be(signed long long  __offset, const signed __int128 *__ptr) {
   return vec_xl(__offset, __ptr);
@@ -17186,49 +17313,50 @@ vec_xl_be(signed long long  __offset, const unsigned __int128 *__ptr) {
   #define vec_xl_be vec_xl
 #endif
 
-#if defined(__POWER10_VECTOR__) && defined(__VSX__)
+#if defined(__POWER10_VECTOR__) && defined(__VSX__) &&                         \
+    defined(__SIZEOF_INT128__)
 
 /* vect_xl_sext */
 
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
-vec_xl_sext(signed long long __offset, const signed char *__pointer) {
+vec_xl_sext(ptrdiff_t __offset, const signed char *__pointer) {
   return (vector unsigned __int128)*(__pointer + __offset);
 }
 
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
-vec_xl_sext(signed long long __offset, const signed short *__pointer) {
+vec_xl_sext(ptrdiff_t __offset, const signed short *__pointer) {
   return (vector unsigned __int128)*(__pointer + __offset);
 }
 
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
-vec_xl_sext(signed long long __offset, const signed int *__pointer) {
+vec_xl_sext(ptrdiff_t __offset, const signed int *__pointer) {
   return (vector unsigned __int128)*(__pointer + __offset);
 }
 
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
-vec_xl_sext(signed long long __offset, const signed long long *__pointer) {
+vec_xl_sext(ptrdiff_t __offset, const signed long long *__pointer) {
   return (vector unsigned __int128)*(__pointer + __offset);
 }
 
 /* vec_xl_zext */
 
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
-vec_xl_zext(signed long long __offset, const unsigned char *__pointer) {
+vec_xl_zext(ptrdiff_t __offset, const unsigned char *__pointer) {
   return (vector unsigned __int128)*(__pointer + __offset);
 }
 
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
-vec_xl_zext(signed long long __offset, const unsigned short *__pointer) {
+vec_xl_zext(ptrdiff_t __offset, const unsigned short *__pointer) {
   return (vector unsigned __int128)*(__pointer + __offset);
 }
 
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
-vec_xl_zext(signed long long __offset, const unsigned int *__pointer) {
+vec_xl_zext(ptrdiff_t __offset, const unsigned int *__pointer) {
   return (vector unsigned __int128)*(__pointer + __offset);
 }
 
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
-vec_xl_zext(signed long long __offset, const unsigned long long *__pointer) {
+vec_xl_zext(ptrdiff_t __offset, const unsigned long long *__pointer) {
   return (vector unsigned __int128)*(__pointer + __offset);
 }
 
@@ -17236,48 +17364,44 @@ vec_xl_zext(signed long long __offset, const unsigned long long *__pointer) {
 
 /* vec_xst */
 
-static inline __ATTRS_o_ai void vec_xst(vector signed char __vec,
-                                        signed long long __offset,
-                                        signed char *__ptr) {
+#define vec_xstd2 vec_xst
+#define vec_xstw4 vec_xst
+static inline __ATTRS_o_ai void
+vec_xst(vector signed char __vec, ptrdiff_t __offset, signed char *__ptr) {
   *(unaligned_vec_schar *)(__ptr + __offset) = __vec;
 }
 
-static inline __ATTRS_o_ai void vec_xst(vector unsigned char __vec,
-                                        signed long long __offset,
-                                        unsigned char *__ptr) {
+static inline __ATTRS_o_ai void
+vec_xst(vector unsigned char __vec, ptrdiff_t __offset, unsigned char *__ptr) {
   *(unaligned_vec_uchar *)(__ptr + __offset) = __vec;
 }
 
-static inline __ATTRS_o_ai void vec_xst(vector signed short __vec,
-                                        signed long long __offset,
-                                        signed short *__ptr) {
+static inline __ATTRS_o_ai void
+vec_xst(vector signed short __vec, ptrdiff_t __offset, signed short *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_sshort *)__addr = __vec;
 }
 
 static inline __ATTRS_o_ai void vec_xst(vector unsigned short __vec,
-                                        signed long long __offset,
+                                        ptrdiff_t __offset,
                                         unsigned short *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_ushort *)__addr = __vec;
 }
 
 static inline __ATTRS_o_ai void vec_xst(vector signed int __vec,
-                                        signed long long __offset,
-                                        signed int *__ptr) {
+                                        ptrdiff_t __offset, signed int *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_sint *)__addr = __vec;
 }
 
-static inline __ATTRS_o_ai void vec_xst(vector unsigned int __vec,
-                                        signed long long __offset,
-                                        unsigned int *__ptr) {
+static inline __ATTRS_o_ai void
+vec_xst(vector unsigned int __vec, ptrdiff_t __offset, unsigned int *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_uint *)__addr = __vec;
 }
 
-static inline __ATTRS_o_ai void vec_xst(vector float __vec,
-                                        signed long long __offset,
+static inline __ATTRS_o_ai void vec_xst(vector float __vec, ptrdiff_t __offset,
                                         float *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_float *)__addr = __vec;
@@ -17285,37 +17409,37 @@ static inline __ATTRS_o_ai void vec_xst(vector float __vec,
 
 #ifdef __VSX__
 static inline __ATTRS_o_ai void vec_xst(vector signed long long __vec,
-                                        signed long long __offset,
+                                        ptrdiff_t __offset,
                                         signed long long *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_sll *)__addr = __vec;
 }
 
 static inline __ATTRS_o_ai void vec_xst(vector unsigned long long __vec,
-                                        signed long long __offset,
+                                        ptrdiff_t __offset,
                                         unsigned long long *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_ull *)__addr = __vec;
 }
 
-static inline __ATTRS_o_ai void vec_xst(vector double __vec,
-                                        signed long long __offset,
+static inline __ATTRS_o_ai void vec_xst(vector double __vec, ptrdiff_t __offset,
                                         double *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_double *)__addr = __vec;
 }
 #endif
 
-#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) &&                    \
+    defined(__SIZEOF_INT128__)
 static inline __ATTRS_o_ai void vec_xst(vector signed __int128 __vec,
-                                        signed long long __offset,
+                                        ptrdiff_t __offset,
                                         signed __int128 *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_si128 *)__addr = __vec;
 }
 
 static inline __ATTRS_o_ai void vec_xst(vector unsigned __int128 __vec,
-                                        signed long long __offset,
+                                        ptrdiff_t __offset,
                                         unsigned __int128 *__ptr) {
   signed char *__addr = (signed char *)__ptr + __offset;
   *(unaligned_vec_ui128 *)__addr = __vec;
@@ -17324,51 +17448,52 @@ static inline __ATTRS_o_ai void vec_xst(vector unsigned __int128 __vec,
 
 /* vec_xst_trunc */
 
-#if defined(__POWER10_VECTOR__) && defined(__VSX__)
+#if defined(__POWER10_VECTOR__) && defined(__VSX__) &&                         \
+    defined(__SIZEOF_INT128__)
 static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
-                                              signed long long __offset,
+                                              ptrdiff_t __offset,
                                               signed char *__ptr) {
   *(__ptr + __offset) = (signed char)__vec[0];
 }
 
 static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
-                                              signed long long __offset,
+                                              ptrdiff_t __offset,
                                               unsigned char *__ptr) {
   *(__ptr + __offset) = (unsigned char)__vec[0];
 }
 
 static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
-                                              signed long long __offset,
+                                              ptrdiff_t __offset,
                                               signed short *__ptr) {
   *(__ptr + __offset) = (signed short)__vec[0];
 }
 
 static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
-                                              signed long long __offset,
+                                              ptrdiff_t __offset,
                                               unsigned short *__ptr) {
   *(__ptr + __offset) = (unsigned short)__vec[0];
 }
 
 static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
-                                              signed long long __offset,
+                                              ptrdiff_t __offset,
                                               signed int *__ptr) {
   *(__ptr + __offset) = (signed int)__vec[0];
 }
 
 static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
-                                              signed long long __offset,
+                                              ptrdiff_t __offset,
                                               unsigned int *__ptr) {
   *(__ptr + __offset) = (unsigned int)__vec[0];
 }
 
 static inline __ATTRS_o_ai void vec_xst_trunc(vector signed __int128 __vec,
-                                              signed long long __offset,
+                                              ptrdiff_t __offset,
                                               signed long long *__ptr) {
   *(__ptr + __offset) = (signed long long)__vec[0];
 }
 
 static inline __ATTRS_o_ai void vec_xst_trunc(vector unsigned __int128 __vec,
-                                              signed long long __offset,
+                                              ptrdiff_t __offset,
                                               unsigned long long *__ptr) {
   *(__ptr + __offset) = (unsigned long long)__vec[0];
 }
@@ -17453,7 +17578,8 @@ static __inline__ void __ATTRS_o_ai vec_xst_be(vector double __vec,
 }
 #endif
 
-#if defined(__POWER8_VECTOR__) && defined(__powerpc64__)
+#if defined(__POWER8_VECTOR__) && defined(__powerpc64__) &&                    \
+    defined(__SIZEOF_INT128__)
 static __inline__ void __ATTRS_o_ai vec_xst_be(vector signed __int128 __vec,
                                                signed long long  __offset,
                                                signed __int128 *__ptr) {
@@ -17563,10 +17689,12 @@ vec_extractm(vector unsigned long long __a) {
   return __builtin_altivec_vextractdm(__a);
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ unsigned int __ATTRS_o_ai
 vec_extractm(vector unsigned __int128 __a) {
   return __builtin_altivec_vextractqm(__a);
 }
+#endif
 
 /* vec_expandm */
 
@@ -17590,10 +17718,12 @@ vec_expandm(vector unsigned long long __a) {
   return __builtin_altivec_vexpanddm(__a);
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_expandm(vector unsigned __int128 __a) {
   return __builtin_altivec_vexpandqm(__a);
 }
+#endif
 
 /* vec_cntm */
 
@@ -17629,10 +17759,12 @@ vec_gendm(unsigned long long __bm) {
   return __builtin_altivec_mtvsrdm(__bm);
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_genqm(unsigned long long __bm) {
   return __builtin_altivec_mtvsrqm(__bm);
 }
+#endif
 
 /* vec_pdep */
 
@@ -17661,6 +17793,7 @@ vec_cfuge(vector unsigned long long __a, vector unsigned long long __b) {
 
 /* vec_ternarylogic */
 #ifdef __VSX__
+#ifdef __SIZEOF_INT128__
 #define vec_ternarylogic(__a, __b, __c, __imm)                                 \
   _Generic((__a), vector unsigned char                                         \
            : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
@@ -17682,6 +17815,25 @@ vec_cfuge(vector unsigned long long __a, vector unsigned long long __b) {
            : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
                                   (vector unsigned long long)(__b),            \
                                   (vector unsigned long long)(__c), (__imm)))
+#else
+#define vec_ternarylogic(__a, __b, __c, __imm)                                 \
+  _Generic((__a), vector unsigned char                                         \
+           : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
+                                  (vector unsigned long long)(__b),            \
+                                  (vector unsigned long long)(__c), (__imm)),  \
+             vector unsigned short                                             \
+           : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
+                                  (vector unsigned long long)(__b),            \
+                                  (vector unsigned long long)(__c), (__imm)),  \
+             vector unsigned int                                               \
+           : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
+                                  (vector unsigned long long)(__b),            \
+                                  (vector unsigned long long)(__c), (__imm)),  \
+             vector unsigned long long                                         \
+           : __builtin_vsx_xxeval((vector unsigned long long)(__a),            \
+                                  (vector unsigned long long)(__b),            \
+                                  (vector unsigned long long)(__c), (__imm)))
+#endif /* __SIZEOF_INT128__ */
 #endif /* __VSX__ */
 
 /* vec_genpcvm */
@@ -17774,6 +17926,7 @@ vec_mod(vector unsigned long long __a, vector unsigned long long __b) {
   return __a % __b;
 }
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_mod(vector signed __int128 __a, vector signed __int128 __b) {
   return __a % __b;
@@ -17783,6 +17936,7 @@ static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_mod(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return  __a % __b;
 }
+#endif
 
 /* vec_sldbi */
 
@@ -18305,6 +18459,7 @@ static __inline__ int __ATTRS_o_ai vec_strir_p(vector signed short __a) {
 
 /* vs[l | r | ra] */
 
+#ifdef __SIZEOF_INT128__
 static __inline__ vector unsigned __int128 __ATTRS_o_ai
 vec_sl(vector unsigned __int128 __a, vector unsigned __int128 __b) {
   return __a << (__b % (vector unsigned __int128)(sizeof(unsigned __int128) *
@@ -18349,6 +18504,7 @@ vec_sra(vector signed __int128 __a, vector unsigned __int128 __b) {
                                                   __CHAR_BIT__));
 }
 
+#endif /* __SIZEOF_INT128__ */
 #endif /* __POWER10_VECTOR__ */
 
 #undef __ATTRS_o_ai
diff --git a/clang/lib/Headers/amxintrin.h b/clang/lib/Headers/amxintrin.h
index 8c276519e3625a19015f8e0b58bbd1348fe2e2f5..12d21d40bcff809023f6c56e4a8a6478b873935c 100644
--- a/clang/lib/Headers/amxintrin.h
+++ b/clang/lib/Headers/amxintrin.h
@@ -267,8 +267,8 @@ _tile_stored_internal(unsigned short m, unsigned short n, void *base,
 }
 
 static __inline__ _tile1024i __DEFAULT_FN_ATTRS_BF16
-_tile_tdpbf16ps_internal(unsigned short m, unsigned short n, unsigned short k,
-                         _tile1024i dst, _tile1024i src1, _tile1024i src2) {
+_tile_dpbf16ps_internal(unsigned short m, unsigned short n, unsigned short k,
+                        _tile1024i dst, _tile1024i src1, _tile1024i src2) {
   return __builtin_ia32_tdpbf16ps_internal(m, n, k, dst, src1, src2);
 }
 
@@ -323,10 +323,10 @@ static void __tile_zero(__tile1024i *dst) {
 }
 
 __DEFAULT_FN_ATTRS_BF16
-static void __tile_tdpbf16ps(__tile1024i *dst, __tile1024i src1,
-                             __tile1024i src2) {
-  dst->tile = _tile_tdpbf16ps_internal(src1.row, src2.col, src1.col, dst->tile,
-                                       src1.tile, src2.tile);
+static void __tile_dpbf16ps(__tile1024i *dst, __tile1024i src1,
+                            __tile1024i src2) {
+  dst->tile = _tile_dpbf16ps_internal(src1.row, src2.col, src1.col, dst->tile,
+                                      src1.tile, src2.tile);
 }
 
 #undef __DEFAULT_FN_ATTRS_TILE
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index 22f7a520c92943dd65ae202c3805fadabcc28273..56d3dadf6a334b3898449785f38e571a315da58e 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -72,11 +72,6 @@
 #include <f16cintrin.h>
 #endif
 
-#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
-    defined(__VPCLMULQDQ__)
-#include <vpclmulqdqintrin.h>
-#endif
-
 /* No feature check desired due to internal checks */
 #include <bmiintrin.h>
 
@@ -230,6 +225,11 @@
 #include <pkuintrin.h>
 #endif
 
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__VPCLMULQDQ__)
+#include <vpclmulqdqintrin.h>
+#endif
+
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__VAES__)
 #include <vaesintrin.h>
diff --git a/clang/lib/Headers/opencl-c.h b/clang/lib/Headers/opencl-c.h
index 8bc669214f4f156722dbeaa670aa0f27bf603b5b..c0b7072d5b0aefbaf13d207901f68f36e13f6a03 100644
--- a/clang/lib/Headers/opencl-c.h
+++ b/clang/lib/Headers/opencl-c.h
@@ -13393,7 +13393,6 @@ ulong __ovld atomic_fetch_max_explicit(volatile atomic_ulong *object, ulong oper
 
 // OpenCL v2.0 s6.13.11.7.5:
 // add/sub: atomic type argument can be uintptr_t/intptr_t, value type argument can be ptrdiff_t.
-// or/xor/and/min/max: atomic type argument can be intptr_t/uintptr_t, value type argument can be intptr_t/uintptr_t.
 
 #if defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)
 uintptr_t __ovld atomic_fetch_add(volatile atomic_uintptr_t *object, ptrdiff_t operand);
@@ -13402,38 +13401,6 @@ uintptr_t __ovld atomic_fetch_add_explicit(volatile atomic_uintptr_t *object, pt
 uintptr_t __ovld atomic_fetch_sub(volatile atomic_uintptr_t *object, ptrdiff_t operand);
 uintptr_t __ovld atomic_fetch_sub_explicit(volatile atomic_uintptr_t *object, ptrdiff_t operand, memory_order order);
 uintptr_t __ovld atomic_fetch_sub_explicit(volatile atomic_uintptr_t *object, ptrdiff_t operand, memory_order order, memory_scope scope);
-
-uintptr_t __ovld atomic_fetch_or(volatile atomic_uintptr_t *object, intptr_t operand);
-uintptr_t __ovld atomic_fetch_or_explicit(volatile atomic_uintptr_t *object, intptr_t operand, memory_order order);
-uintptr_t __ovld atomic_fetch_or_explicit(volatile atomic_uintptr_t *object, intptr_t operand, memory_order order, memory_scope scope);
-uintptr_t __ovld atomic_fetch_xor(volatile atomic_uintptr_t *object, intptr_t operand);
-uintptr_t __ovld atomic_fetch_xor_explicit(volatile atomic_uintptr_t *object, intptr_t operand, memory_order order);
-uintptr_t __ovld atomic_fetch_xor_explicit(volatile atomic_uintptr_t *object, intptr_t operand, memory_order order, memory_scope scope);
-uintptr_t __ovld atomic_fetch_and(volatile atomic_uintptr_t *object, intptr_t operand);
-uintptr_t __ovld atomic_fetch_and_explicit(volatile atomic_uintptr_t *object, intptr_t operand, memory_order order);
-uintptr_t __ovld atomic_fetch_and_explicit(volatile atomic_uintptr_t *object, intptr_t operand, memory_order order, memory_scope scope);
-uintptr_t __ovld atomic_fetch_min(volatile atomic_uintptr_t *object, intptr_t opermax);
-uintptr_t __ovld atomic_fetch_min_explicit(volatile atomic_uintptr_t *object, intptr_t opermax, memory_order minder);
-uintptr_t __ovld atomic_fetch_min_explicit(volatile atomic_uintptr_t *object, intptr_t opermax, memory_order minder, memory_scope scope);
-uintptr_t __ovld atomic_fetch_max(volatile atomic_uintptr_t *object, intptr_t opermax);
-uintptr_t __ovld atomic_fetch_max_explicit(volatile atomic_uintptr_t *object, intptr_t opermax, memory_order minder);
-uintptr_t __ovld atomic_fetch_max_explicit(volatile atomic_uintptr_t *object, intptr_t opermax, memory_order minder, memory_scope scope);
-
-intptr_t __ovld atomic_fetch_or(volatile atomic_intptr_t *object, uintptr_t operand);
-intptr_t __ovld atomic_fetch_or_explicit(volatile atomic_intptr_t *object, uintptr_t operand, memory_order order);
-intptr_t __ovld atomic_fetch_or_explicit(volatile atomic_intptr_t *object, uintptr_t operand, memory_order order, memory_scope scope);
-intptr_t __ovld atomic_fetch_xor(volatile atomic_intptr_t *object, uintptr_t operand);
-intptr_t __ovld atomic_fetch_xor_explicit(volatile atomic_intptr_t *object, uintptr_t operand, memory_order order);
-intptr_t __ovld atomic_fetch_xor_explicit(volatile atomic_intptr_t *object, uintptr_t operand, memory_order order, memory_scope scope);
-intptr_t __ovld atomic_fetch_and(volatile atomic_intptr_t *object, uintptr_t operand);
-intptr_t __ovld atomic_fetch_and_explicit(volatile atomic_intptr_t *object, uintptr_t operand, memory_order order);
-intptr_t __ovld atomic_fetch_and_explicit(volatile atomic_intptr_t *object, uintptr_t operand, memory_order order, memory_scope scope);
-intptr_t __ovld atomic_fetch_min(volatile atomic_intptr_t *object, uintptr_t opermax);
-intptr_t __ovld atomic_fetch_min_explicit(volatile atomic_intptr_t *object, uintptr_t opermax, memory_order minder);
-intptr_t __ovld atomic_fetch_min_explicit(volatile atomic_intptr_t *object, uintptr_t opermax, memory_order minder, memory_scope scope);
-intptr_t __ovld atomic_fetch_max(volatile atomic_intptr_t *object, uintptr_t opermax);
-intptr_t __ovld atomic_fetch_max_explicit(volatile atomic_intptr_t *object, uintptr_t opermax, memory_order minder);
-intptr_t __ovld atomic_fetch_max_explicit(volatile atomic_intptr_t *object, uintptr_t opermax, memory_order minder, memory_scope scope);
 #endif
 
 // atomic_store()
diff --git a/clang/lib/Headers/vaesintrin.h b/clang/lib/Headers/vaesintrin.h
index c4d5c3e75140d4ff970ddd24d554302da34963c2..f3c0807bb94ab9977a27431ad31a8ef2429d22db 100644
--- a/clang/lib/Headers/vaesintrin.h
+++ b/clang/lib/Headers/vaesintrin.h
@@ -28,13 +28,6 @@ static __inline__ __m256i __DEFAULT_FN_ATTRS
               (__v4di) __B);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS_F
- _mm512_aesenc_epi128(__m512i __A, __m512i __B)
-{
-  return (__m512i) __builtin_ia32_aesenc512((__v8di) __A,
-              (__v8di) __B);
-}
-
 static __inline__ __m256i __DEFAULT_FN_ATTRS
  _mm256_aesdec_epi128(__m256i __A, __m256i __B)
 {
@@ -42,32 +35,40 @@ static __inline__ __m256i __DEFAULT_FN_ATTRS
               (__v4di) __B);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS_F
- _mm512_aesdec_epi128(__m512i __A, __m512i __B)
+static __inline__ __m256i __DEFAULT_FN_ATTRS
+ _mm256_aesenclast_epi128(__m256i __A, __m256i __B)
 {
-  return (__m512i) __builtin_ia32_aesdec512((__v8di) __A,
-              (__v8di) __B);
+  return (__m256i) __builtin_ia32_aesenclast256((__v4di) __A,
+              (__v4di) __B);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS
- _mm256_aesenclast_epi128(__m256i __A, __m256i __B)
+ _mm256_aesdeclast_epi128(__m256i __A, __m256i __B)
 {
-  return (__m256i) __builtin_ia32_aesenclast256((__v4di) __A,
+  return (__m256i) __builtin_ia32_aesdeclast256((__v4di) __A,
               (__v4di) __B);
 }
 
+#ifdef __AVX512FINTRIN_H
 static __inline__ __m512i __DEFAULT_FN_ATTRS_F
- _mm512_aesenclast_epi128(__m512i __A, __m512i __B)
+ _mm512_aesenc_epi128(__m512i __A, __m512i __B)
 {
-  return (__m512i) __builtin_ia32_aesenclast512((__v8di) __A,
+  return (__m512i) __builtin_ia32_aesenc512((__v8di) __A,
               (__v8di) __B);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS
- _mm256_aesdeclast_epi128(__m256i __A, __m256i __B)
+static __inline__ __m512i __DEFAULT_FN_ATTRS_F
+ _mm512_aesdec_epi128(__m512i __A, __m512i __B)
 {
-  return (__m256i) __builtin_ia32_aesdeclast256((__v4di) __A,
-              (__v4di) __B);
+  return (__m512i) __builtin_ia32_aesdec512((__v8di) __A,
+              (__v8di) __B);
+}
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS_F
+ _mm512_aesenclast_epi128(__m512i __A, __m512i __B)
+{
+  return (__m512i) __builtin_ia32_aesenclast512((__v8di) __A,
+              (__v8di) __B);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS_F
@@ -76,7 +77,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_F
   return (__m512i) __builtin_ia32_aesdeclast512((__v8di) __A,
               (__v8di) __B);
 }
-
+#endif // __AVX512FINTRIN_H
 
 #undef __DEFAULT_FN_ATTRS
 #undef __DEFAULT_FN_ATTRS_F
diff --git a/clang/lib/Headers/vpclmulqdqintrin.h b/clang/lib/Headers/vpclmulqdqintrin.h
index 470d832549058d8cd45eaf83bcc9b714c62cd1f2..44daadb07d57c1f11742d0d2c06cbbbf223f2d3a 100644
--- a/clang/lib/Headers/vpclmulqdqintrin.h
+++ b/clang/lib/Headers/vpclmulqdqintrin.h
@@ -19,10 +19,12 @@
                                        (__v4di)(__m256i)(B),  \
                                        (char)(I))
 
+#ifdef __AVX512FINTRIN_H
 #define _mm512_clmulepi64_epi128(A, B, I) \
   (__m512i)__builtin_ia32_pclmulqdq512((__v8di)(__m512i)(A),  \
                                        (__v8di)(__m512i)(B),  \
                                        (char)(I))
+#endif // __AVX512FINTRIN_H
 
 #endif /* __VPCLMULQDQINTRIN_H */
 
diff --git a/clang/lib/Headers/wasm_simd128.h b/clang/lib/Headers/wasm_simd128.h
index ac88516ac92489701f13c48f822374fab0033236..eb2a42f303b6177bdc2aac95d3762430c778047d 100644
--- a/clang/lib/Headers/wasm_simd128.h
+++ b/clang/lib/Headers/wasm_simd128.h
@@ -616,14 +616,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add(v128_t __a,
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i8x16_add_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_add_saturate_s_i8x16((__i8x16)__a,
-                                                     (__i8x16)__b);
+  return (v128_t)__builtin_wasm_add_sat_s_i8x16((__i8x16)__a, (__i8x16)__b);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u8x16_add_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_add_saturate_u_i8x16((__u8x16)__a,
-                                                     (__u8x16)__b);
+  return (v128_t)__builtin_wasm_add_sat_u_i8x16((__u8x16)__a, (__u8x16)__b);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,
@@ -633,14 +631,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i8x16_sub_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_sub_saturate_s_i8x16((__i8x16)__a,
-                                                     (__i8x16)__b);
+  return (v128_t)__builtin_wasm_sub_sat_s_i8x16((__i8x16)__a, (__i8x16)__b);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u8x16_sub_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_sub_saturate_u_i8x16((__u8x16)__a,
-                                                     (__u8x16)__b);
+  return (v128_t)__builtin_wasm_sub_sat_u_i8x16((__u8x16)__a, (__u8x16)__b);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_min(v128_t __a,
@@ -706,14 +702,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add(v128_t __a,
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_add_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_add_saturate_s_i16x8((__i16x8)__a,
-                                                     (__i16x8)__b);
+  return (v128_t)__builtin_wasm_add_sat_s_i16x8((__i16x8)__a, (__i16x8)__b);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u16x8_add_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_add_saturate_u_i16x8((__u16x8)__a,
-                                                     (__u16x8)__b);
+  return (v128_t)__builtin_wasm_add_sat_u_i16x8((__u16x8)__a, (__u16x8)__b);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,
@@ -723,14 +717,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_i16x8_sub_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_sub_saturate_s_i16x8((__i16x8)__a,
-                                                     (__i16x8)__b);
+  return (v128_t)__builtin_wasm_sub_sat_s_i16x8((__i16x8)__a, (__i16x8)__b);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS
 wasm_u16x8_sub_saturate(v128_t __a, v128_t __b) {
-  return (v128_t)__builtin_wasm_sub_saturate_u_i16x8((__u16x8)__a,
-                                                     (__u16x8)__b);
+  return (v128_t)__builtin_wasm_sub_sat_u_i16x8((__u16x8)__a, (__u16x8)__b);
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_mul(v128_t __a,
@@ -833,18 +825,6 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_neg(v128_t __a) {
   return (v128_t)(-(__u64x2)__a);
 }
 
-#ifdef __wasm_unimplemented_simd128__
-
-static __inline__ bool __DEFAULT_FN_ATTRS wasm_i64x2_any_true(v128_t __a) {
-  return __builtin_wasm_any_true_i64x2((__i64x2)__a);
-}
-
-static __inline__ bool __DEFAULT_FN_ATTRS wasm_i64x2_all_true(v128_t __a) {
-  return __builtin_wasm_all_true_i64x2((__i64x2)__a);
-}
-
-#endif // __wasm_unimplemented_simd128__
-
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shl(v128_t __a,
                                                            int32_t __b) {
   return (v128_t)((__i64x2)__a << (int64_t)__b);
@@ -887,24 +867,6 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_sqrt(v128_t __a) {
   return (v128_t)__builtin_wasm_sqrt_f32x4((__f32x4)__a);
 }
 
-#ifdef __wasm_unimplemented_simd128__
-
-static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_qfma(v128_t __a,
-                                                            v128_t __b,
-                                                            v128_t __c) {
-  return (v128_t)__builtin_wasm_qfma_f32x4((__f32x4)__a, (__f32x4)__b,
-                                           (__f32x4)__c);
-}
-
-static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_qfms(v128_t __a,
-                                                            v128_t __b,
-                                                            v128_t __c) {
-  return (v128_t)__builtin_wasm_qfms_f32x4((__f32x4)__a, (__f32x4)__b,
-                                           (__f32x4)__c);
-}
-
-#endif // __wasm_unimplemented_simd128__
-
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_add(v128_t __a,
                                                            v128_t __b) {
   return (v128_t)((__f32x4)__a + (__f32x4)__b);
@@ -957,24 +919,6 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_sqrt(v128_t __a) {
   return (v128_t)__builtin_wasm_sqrt_f64x2((__f64x2)__a);
 }
 
-#ifdef __wasm_unimplemented_simd128__
-
-static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_qfma(v128_t __a,
-                                                            v128_t __b,
-                                                            v128_t __c) {
-  return (v128_t)__builtin_wasm_qfma_f64x2((__f64x2)__a, (__f64x2)__b,
-                                           (__f64x2)__c);
-}
-
-static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_qfms(v128_t __a,
-                                                            v128_t __b,
-                                                            v128_t __c) {
-  return (v128_t)__builtin_wasm_qfms_f64x2((__f64x2)__a, (__f64x2)__b,
-                                           (__f64x2)__c);
-}
-
-#endif // __wasm_unimplemented_simd128__
-
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_add(v128_t __a,
                                                            v128_t __b) {
   return (v128_t)((__f64x2)__a + (__f64x2)__b);
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index c854d3e9c02b48bd80bd28ff85cd84b11f2776d2..a771b7c5d122dc7db6d2aca0c336a81f6324ff1f 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -124,7 +124,7 @@ static bool isReservedId(StringRef Text, const LangOptions &Lang) {
 // the specified module, meaning clang won't build the specified module. This is
 // useful in a number of situations, for instance, when building a library that
 // vends a module map, one might want to avoid hitting intermediate build
-// products containimg the the module map or avoid finding the system installed
+// products containing the the module map or avoid finding the system installed
 // modulemap for that library.
 static bool isForModuleBuilding(Module *M, StringRef CurrentModule,
                                 StringRef ModuleName) {
@@ -441,9 +441,9 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
     CurLexer->Lex(Tok);
 
     if (Tok.is(tok::code_completion)) {
+      setCodeCompletionReached();
       if (CodeComplete)
         CodeComplete->CodeCompleteInConditionalExclusion();
-      setCodeCompletionReached();
       continue;
     }
 
@@ -966,10 +966,10 @@ void Preprocessor::HandleDirective(Token &Result) {
   case tok::eod:
     return;   // null directive.
   case tok::code_completion:
+    setCodeCompletionReached();
     if (CodeComplete)
       CodeComplete->CodeCompleteDirective(
                                     CurPPLexer->getConditionalStackDepth() > 0);
-    setCodeCompletionReached();
     return;
   case tok::numeric_constant:  // # 7  GNU line marker directive.
     if (getLangOpts().AsmPreprocessor)
@@ -1045,12 +1045,12 @@ void Preprocessor::HandleDirective(Token &Result) {
       break;
 
     case tok::pp___public_macro:
-      if (getLangOpts().Modules)
+      if (getLangOpts().Modules || getLangOpts().ModulesLocalVisibility)
         return HandleMacroPublicDirective(Result);
       break;
 
     case tok::pp___private_macro:
-      if (getLangOpts().Modules)
+      if (getLangOpts().Modules || getLangOpts().ModulesLocalVisibility)
         return HandleMacroPrivateDirective();
       break;
     }
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index 43d31d6c5732e572e6dfd9db1c6095d4076cdc26..9528a8f575f0c0b55db3fdcfe635665568d009c5 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -170,7 +170,8 @@ ModuleMacro *Preprocessor::addModuleMacro(Module *Mod, IdentifierInfo *II,
   return MM;
 }
 
-ModuleMacro *Preprocessor::getModuleMacro(Module *Mod, IdentifierInfo *II) {
+ModuleMacro *Preprocessor::getModuleMacro(Module *Mod,
+                                          const IdentifierInfo *II) {
   llvm::FoldingSetNodeID ID;
   ModuleMacro::Profile(ID, Mod, II);
 
diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp
index a05df060813e711bb5449411b75b9149fcc08edc..5b42241a32c2e54274da8a25c78ac54843596345 100644
--- a/clang/lib/Lex/Pragma.cpp
+++ b/clang/lib/Lex/Pragma.cpp
@@ -1122,6 +1122,57 @@ struct PragmaDebugHandler : public PragmaHandler {
         DebugOverflowStack();
     } else if (II->isStr("captured")) {
       HandleCaptured(PP);
+    } else if (II->isStr("modules")) {
+      struct ModuleVisitor {
+        Preprocessor &PP;
+        void visit(Module *M, bool VisibleOnly) {
+          SourceLocation ImportLoc = PP.getModuleImportLoc(M);
+          if (!VisibleOnly || ImportLoc.isValid()) {
+            llvm::errs() << M->getFullModuleName() << " ";
+            if (ImportLoc.isValid()) {
+              llvm::errs() << M << " visible ";
+              ImportLoc.print(llvm::errs(), PP.getSourceManager());
+            }
+            llvm::errs() << "\n";
+          }
+          for (Module *Sub : M->submodules()) {
+            if (!VisibleOnly || ImportLoc.isInvalid() || Sub->IsExplicit)
+              visit(Sub, VisibleOnly);
+          }
+        }
+        void visitAll(bool VisibleOnly) {
+          for (auto &NameAndMod :
+               PP.getHeaderSearchInfo().getModuleMap().modules())
+            visit(NameAndMod.second, VisibleOnly);
+        }
+      } Visitor{PP};
+
+      Token Kind;
+      PP.LexUnexpandedToken(Kind);
+      auto *DumpII = Kind.getIdentifierInfo();
+      if (!DumpII) {
+        PP.Diag(Kind, diag::warn_pragma_debug_missing_argument)
+            << II->getName();
+      } else if (DumpII->isStr("all")) {
+        Visitor.visitAll(false);
+      } else if (DumpII->isStr("visible")) {
+        Visitor.visitAll(true);
+      } else if (DumpII->isStr("building")) {
+        for (auto &Building : PP.getBuildingSubmodules()) {
+          llvm::errs() << "in " << Building.M->getFullModuleName();
+          if (Building.ImportLoc.isValid()) {
+            llvm::errs() << " imported ";
+            if (Building.IsPragma)
+              llvm::errs() << "via pragma ";
+            llvm::errs() << "at ";
+            Building.ImportLoc.print(llvm::errs(), PP.getSourceManager());
+            llvm::errs() << "\n";
+          }
+        }
+      } else {
+        PP.Diag(Tok, diag::warn_pragma_debug_unexpected_command)
+          << DumpII->getName();
+      }
     } else {
       PP.Diag(Tok, diag::warn_pragma_debug_unexpected_command)
         << II->getName();
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 177786d903907aabca30d2994f83fc806af65eb7..e39b78d5ffec28d84c7637069fcfa08de1ef2001 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -442,15 +442,15 @@ bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
 
 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
                                             bool IsAngled) {
+  setCodeCompletionReached();
   if (CodeComplete)
     CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
-  setCodeCompletionReached();
 }
 
 void Preprocessor::CodeCompleteNaturalLanguage() {
+  setCodeCompletionReached();
   if (CodeComplete)
     CodeComplete->CodeCompleteNaturalLanguage();
-  setCodeCompletionReached();
 }
 
 /// getSpelling - This method is used to get the spelling of a token into a
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index f1f165187d47fd95b7f7698a077b031358937378..49298ef019f4668385c4f17ea7e9c756e8f6486e 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -1970,8 +1970,8 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
   // Check to see if we have a function *definition* which must have a body.
   if (D.isFunctionDeclarator()) {
     if (Tok.is(tok::equal) && NextToken().is(tok::code_completion)) {
-      Actions.CodeCompleteAfterFunctionEquals(D);
       cutOffParsing();
+      Actions.CodeCompleteAfterFunctionEquals(D);
       return nullptr;
     }
     // Look at the next token to make sure that this isn't a function
@@ -2310,9 +2310,9 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
       InitializerScopeRAII InitScope(*this, D, ThisDecl);
 
       if (Tok.is(tok::code_completion)) {
+        cutOffParsing();
         Actions.CodeCompleteInitializer(getCurScope(), ThisDecl);
         Actions.FinalizeDeclaration(ThisDecl);
-        cutOffParsing();
         return nullptr;
       }
 
@@ -3090,10 +3090,11 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
           = DSContext == DeclSpecContext::DSC_top_level ||
             (DSContext == DeclSpecContext::DSC_class && DS.isFriendSpecified());
 
+        cutOffParsing();
         Actions.CodeCompleteDeclSpec(getCurScope(), DS,
                                      AllowNonIdentifiers,
                                      AllowNestedNameSpecifiers);
-        return cutOffParsing();
+        return;
       }
 
       if (getCurScope()->getFnParent() || getCurScope()->getBlockParent())
@@ -3106,8 +3107,9 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       else if (CurParsedObjCImpl)
         CCC = Sema::PCC_ObjCImplementation;
 
+      cutOffParsing();
       Actions.CodeCompleteOrdinaryName(getCurScope(), CCC);
-      return cutOffParsing();
+      return;
     }
 
     case tok::coloncolon: // ::foo::bar
@@ -4374,8 +4376,9 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   // Parse the tag portion of this.
   if (Tok.is(tok::code_completion)) {
     // Code completion for an enum name.
+    cutOffParsing();
     Actions.CodeCompleteTag(getCurScope(), DeclSpec::TST_enum);
-    return cutOffParsing();
+    return;
   }
 
   // If attributes exist after tag, parse them.
@@ -5484,11 +5487,12 @@ void Parser::ParseTypeQualifierListOpt(
 
     switch (Tok.getKind()) {
     case tok::code_completion:
+      cutOffParsing();
       if (CodeCompletionHandler)
         (*CodeCompletionHandler)();
       else
         Actions.CodeCompleteTypeQualifiers(DS);
-      return cutOffParsing();
+      return;
 
     case tok::kw_const:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_const   , Loc, PrevSpec, DiagID,
@@ -7025,8 +7029,9 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
                   std::move(attrs), T.getCloseLocation());
     return;
   } else if (Tok.getKind() == tok::code_completion) {
+    cutOffParsing();
     Actions.CodeCompleteBracketDeclarator(getCurScope());
-    return cutOffParsing();
+    return;
   }
 
   // If valid, this location is the position where we read the 'static' keyword.
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index dd1cccf72668d90c2b9f8731b71bbed9061ffd0b..0e9bc42bfcb8f66dec4f5490acb30f31180a075c 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -63,8 +63,8 @@ Parser::DeclGroupPtrTy Parser::ParseNamespace(DeclaratorContext Context,
   ObjCDeclContextSwitch ObjCDC(*this);
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteNamespaceDecl(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteNamespaceDecl(getCurScope());
     return nullptr;
   }
 
@@ -283,8 +283,8 @@ Decl *Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc,
   ConsumeToken(); // eat the '='.
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteNamespaceAliasDecl(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteNamespaceAliasDecl(getCurScope());
     return nullptr;
   }
 
@@ -471,8 +471,8 @@ Parser::ParseUsingDirectiveOrDeclaration(DeclaratorContext Context,
   SourceLocation UsingLoc = ConsumeToken();
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteUsing(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteUsing(getCurScope());
     return nullptr;
   }
 
@@ -525,8 +525,8 @@ Decl *Parser::ParseUsingDirective(DeclaratorContext Context,
   SourceLocation NamespcLoc = ConsumeToken();
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteUsingDirective(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteUsingDirective(getCurScope());
     return nullptr;
   }
 
@@ -1433,8 +1433,9 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
 
   if (Tok.is(tok::code_completion)) {
     // Code completion for a struct, class, or union name.
+    cutOffParsing();
     Actions.CodeCompleteTag(getCurScope(), TagType);
-    return cutOffParsing();
+    return;
   }
 
   // C++03 [temp.explicit] 14.7.2/8:
@@ -2749,8 +2750,8 @@ Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
         else if (KW.is(tok::kw_delete))
           DefinitionKind = FunctionDefinitionKind::Deleted;
         else if (KW.is(tok::code_completion)) {
-          Actions.CodeCompleteAfterFunctionEquals(DeclaratorInfo);
           cutOffParsing();
+          Actions.CodeCompleteAfterFunctionEquals(DeclaratorInfo);
           return nullptr;
         }
       }
@@ -3498,9 +3499,10 @@ void Parser::ParseConstructorInitializer(Decl *ConstructorDecl) {
 
   do {
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       Actions.CodeCompleteConstructorInitializer(ConstructorDecl,
                                                  MemInitializers);
-      return cutOffParsing();
+      return;
     }
 
     MemInitResult MemInit = ParseMemInitializer(ConstructorDecl);
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index c6d34478e9b0c686e1b1335938dbfd7009d0546d..3a1b5d20803c3149803021991721ae1123c54fb2 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -159,9 +159,9 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
 /// Parse an expr that doesn't include (top-level) commas.
 ExprResult Parser::ParseAssignmentExpression(TypeCastState isTypeCast) {
   if (Tok.is(tok::code_completion)) {
+    cutOffParsing();
     Actions.CodeCompleteExpression(getCurScope(),
                                    PreferredType.get(Tok.getLocation()));
-    cutOffParsing();
     return ExprError();
   }
 
@@ -1156,9 +1156,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
       ConsumeToken();
 
       if (Tok.is(tok::code_completion) && &II != Ident_super) {
+        cutOffParsing();
         Actions.CodeCompleteObjCClassPropertyRefExpr(
             getCurScope(), II, ILoc, ExprStatementTokLoc == ILoc);
-        cutOffParsing();
         return ExprError();
       }
       // Allow either an identifier or the keyword 'class' (in C++).
@@ -1728,9 +1728,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     Res = ParseBlockLiteralExpression();
     break;
   case tok::code_completion: {
+    cutOffParsing();
     Actions.CodeCompleteExpression(getCurScope(),
                                    PreferredType.get(Tok.getLocation()));
-    cutOffParsing();
     return ExprError();
   }
   case tok::l_square:
@@ -1860,9 +1860,9 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       if (InMessageExpression)
         return LHS;
 
+      cutOffParsing();
       Actions.CodeCompletePostfixExpression(
           getCurScope(), LHS, PreferredType.get(Tok.getLocation()));
-      cutOffParsing();
       return ExprError();
 
     case tok::identifier:
@@ -2144,12 +2144,12 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
           CorrectedBase = Base;
 
         // Code completion for a member access expression.
+        cutOffParsing();
         Actions.CodeCompleteMemberReferenceExpr(
             getCurScope(), Base, CorrectedBase, OpLoc, OpKind == tok::arrow,
             Base && ExprStatementTokLoc == Base->getBeginLoc(),
             PreferredType.get(Tok.getLocation()));
 
-        cutOffParsing();
         return ExprError();
       }
 
@@ -2782,10 +2782,10 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
   CastTy = nullptr;
 
   if (Tok.is(tok::code_completion)) {
+    cutOffParsing();
     Actions.CodeCompleteExpression(
         getCurScope(), PreferredType.get(Tok.getLocation()),
         /*IsParenthesized=*/ExprType >= CompoundLiteral);
-    cutOffParsing();
     return ExprError();
   }
 
@@ -3416,8 +3416,9 @@ Parser::ParseSimpleExpressionList(SmallVectorImpl<Expr*> &Exprs,
 /// \endverbatim
 void Parser::ParseBlockId(SourceLocation CaretLoc) {
   if (Tok.is(tok::code_completion)) {
+    cutOffParsing();
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Type);
-    return cutOffParsing();
+    return;
   }
 
   // Parse the specifier-qualifier-list piece.
@@ -3602,8 +3603,8 @@ Optional<AvailabilitySpec> Parser::ParseAvailabilitySpec() {
   } else {
     // Parse the platform name.
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteAvailabilityPlatformName();
       cutOffParsing();
+      Actions.CodeCompleteAvailabilityPlatformName();
       return None;
     }
     if (Tok.isNot(tok::identifier)) {
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 9292541d7ede96c27783f595400bdd72c4638012..c2e74b5a7bbdfd0db762a0266841103eff7573ea 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -235,6 +235,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(
   while (true) {
     if (HasScopeSpecifier) {
       if (Tok.is(tok::code_completion)) {
+        cutOffParsing();
         // Code completion for a nested-name-specifier, where the code
         // completion token follows the '::'.
         Actions.CodeCompleteQualifiedId(getCurScope(), SS, EnteringContext,
@@ -245,7 +246,6 @@ bool Parser::ParseOptionalCXXScopeSpecifier(
         // token will cause assertion in
         // Preprocessor::AnnotatePreviousCachedTokens.
         SS.setEndLoc(Tok.getLocation());
-        cutOffParsing();
         return true;
       }
 
@@ -688,9 +688,9 @@ ExprResult Parser::ParseCXXIdExpression(bool isAddressOfOperand) {
 /// ParseLambdaExpression - Parse a C++11 lambda expression.
 ///
 ///       lambda-expression:
-///         lambda-introducer lambda-declarator[opt] compound-statement
+///         lambda-introducer lambda-declarator compound-statement
 ///         lambda-introducer '<' template-parameter-list '>'
-///             lambda-declarator[opt] compound-statement
+///             lambda-declarator compound-statement
 ///
 ///       lambda-introducer:
 ///         '[' lambda-capture[opt] ']'
@@ -722,9 +722,13 @@ ExprResult Parser::ParseCXXIdExpression(bool isAddressOfOperand) {
 ///         '&' identifier initializer
 ///
 ///       lambda-declarator:
-///         '(' parameter-declaration-clause ')' attribute-specifier[opt]
-///           'mutable'[opt] exception-specification[opt]
-///           trailing-return-type[opt]
+///         lambda-specifiers     [C++2b]
+///         '(' parameter-declaration-clause ')' lambda-specifiers
+///             requires-clause[opt]
+///
+///       lambda-specifiers:
+///         decl-specifier-seq[opt] noexcept-specifier[opt]
+///             attribute-specifier-seq[opt] trailing-return-type[opt]
 ///
 ExprResult Parser::ParseLambdaExpression() {
   // Parse lambda-introducer.
@@ -877,9 +881,9 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro,
         // expression parser perform the completion.
         if (Tok.is(tok::code_completion) &&
             !(getLangOpts().ObjC && Tentative)) {
+          cutOffParsing();
           Actions.CodeCompleteLambdaIntroducer(getCurScope(), Intro,
                                                /*AfterAmpersand=*/false);
-          cutOffParsing();
           break;
         }
 
@@ -891,6 +895,7 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro,
     }
 
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       // If we're in Objective-C++ and we have a bare '[', then this is more
       // likely to be a message receiver.
       if (getLangOpts().ObjC && Tentative && First)
@@ -898,7 +903,6 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro,
       else
         Actions.CodeCompleteLambdaIntroducer(getCurScope(), Intro,
                                              /*AfterAmpersand=*/false);
-      cutOffParsing();
       break;
     }
 
@@ -943,9 +947,9 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro,
         ConsumeToken();
 
         if (Tok.is(tok::code_completion)) {
+          cutOffParsing();
           Actions.CodeCompleteLambdaIntroducer(getCurScope(), Intro,
                                                /*AfterAmpersand=*/true);
-          cutOffParsing();
           break;
         }
       }
@@ -1249,7 +1253,6 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
   Actions.PushLambdaScope();
 
   ParsedAttributes Attr(AttrFactory);
-  SourceLocation DeclLoc = Tok.getLocation();
   if (getLangOpts().CUDA) {
     // In CUDA code, GNU attributes are allowed to appear immediately after the
     // "[...]", even if there is no "(...)" before the lambda body.
@@ -1315,11 +1318,92 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
 
   TypeResult TrailingReturnType;
   SourceLocation TrailingReturnTypeLoc;
+
+  auto ParseLambdaSpecifiers =
+      [&](SourceLocation LParenLoc, SourceLocation RParenLoc,
+          MutableArrayRef<DeclaratorChunk::ParamInfo> ParamInfo,
+          SourceLocation EllipsisLoc) {
+        SourceLocation DeclEndLoc = RParenLoc;
+
+        // GNU-style attributes must be parsed before the mutable specifier to
+        // be compatible with GCC. MSVC-style attributes must be parsed before
+        // the mutable specifier to be compatible with MSVC.
+        MaybeParseAttributes(PAKM_GNU | PAKM_Declspec, Attr);
+
+        // Parse mutable-opt and/or constexpr-opt or consteval-opt, and update
+        // the DeclEndLoc.
+        SourceLocation MutableLoc;
+        SourceLocation ConstexprLoc;
+        SourceLocation ConstevalLoc;
+        tryConsumeLambdaSpecifierToken(*this, MutableLoc, ConstexprLoc,
+                                       ConstevalLoc, DeclEndLoc);
+
+        addConstexprToLambdaDeclSpecifier(*this, ConstexprLoc, DS);
+        addConstevalToLambdaDeclSpecifier(*this, ConstevalLoc, DS);
+        // Parse exception-specification[opt].
+        ExceptionSpecificationType ESpecType = EST_None;
+        SourceRange ESpecRange;
+        SmallVector<ParsedType, 2> DynamicExceptions;
+        SmallVector<SourceRange, 2> DynamicExceptionRanges;
+        ExprResult NoexceptExpr;
+        CachedTokens *ExceptionSpecTokens;
+        ESpecType = tryParseExceptionSpecification(
+            /*Delayed=*/false, ESpecRange, DynamicExceptions,
+            DynamicExceptionRanges, NoexceptExpr, ExceptionSpecTokens);
+
+        if (ESpecType != EST_None)
+          DeclEndLoc = ESpecRange.getEnd();
+
+        // Parse attribute-specifier[opt].
+        MaybeParseCXX11Attributes(Attr, &DeclEndLoc);
+
+        // Parse OpenCL addr space attribute.
+        if (Tok.isOneOf(tok::kw___private, tok::kw___global, tok::kw___local,
+                        tok::kw___constant, tok::kw___generic)) {
+          ParseOpenCLQualifiers(DS.getAttributes());
+          ConsumeToken();
+        }
+
+        SourceLocation FunLocalRangeEnd = DeclEndLoc;
+
+        // Parse trailing-return-type[opt].
+        if (Tok.is(tok::arrow)) {
+          FunLocalRangeEnd = Tok.getLocation();
+          SourceRange Range;
+          TrailingReturnType = ParseTrailingReturnType(
+              Range, /*MayBeFollowedByDirectInit*/ false);
+          TrailingReturnTypeLoc = Range.getBegin();
+          if (Range.getEnd().isValid())
+            DeclEndLoc = Range.getEnd();
+        }
+
+        SourceLocation NoLoc;
+        D.AddTypeInfo(
+            DeclaratorChunk::getFunction(
+                /*HasProto=*/true,
+                /*IsAmbiguous=*/false, LParenLoc, ParamInfo.data(),
+                ParamInfo.size(), EllipsisLoc, RParenLoc,
+                /*RefQualifierIsLvalueRef=*/true,
+                /*RefQualifierLoc=*/NoLoc, MutableLoc, ESpecType, ESpecRange,
+                DynamicExceptions.data(), DynamicExceptionRanges.data(),
+                DynamicExceptions.size(),
+                NoexceptExpr.isUsable() ? NoexceptExpr.get() : nullptr,
+                /*ExceptionSpecTokens*/ nullptr,
+                /*DeclsInPrototype=*/None, LParenLoc, FunLocalRangeEnd, D,
+                TrailingReturnType, TrailingReturnTypeLoc, &DS),
+            std::move(Attr), DeclEndLoc);
+
+        // Parse requires-clause[opt].
+        if (Tok.is(tok::kw_requires))
+          ParseTrailingRequiresClause(D);
+
+        WarnIfHasCUDATargetAttr();
+      };
+
   if (Tok.is(tok::l_paren)) {
-    ParseScope PrototypeScope(this,
-                              Scope::FunctionPrototypeScope |
-                              Scope::FunctionDeclarationScope |
-                              Scope::DeclScope);
+    ParseScope PrototypeScope(this, Scope::FunctionPrototypeScope |
+                                        Scope::FunctionDeclarationScope |
+                                        Scope::DeclScope);
 
     BalancedDelimiterTracker T(*this, tok::l_paren);
     T.consumeOpen();
@@ -1345,165 +1429,28 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
     }
 
     T.consumeClose();
-    SourceLocation RParenLoc = T.getCloseLocation();
-    SourceLocation DeclEndLoc = RParenLoc;
-
-    // GNU-style attributes must be parsed before the mutable specifier to be
-    // compatible with GCC. MSVC-style attributes must be parsed before the
-    // mutable specifier to be compatible with MSVC.
-    MaybeParseAttributes(PAKM_GNU | PAKM_Declspec, Attr);
-
-    // Parse mutable-opt and/or constexpr-opt or consteval-opt, and update the
-    // DeclEndLoc.
-    SourceLocation MutableLoc;
-    SourceLocation ConstexprLoc;
-    SourceLocation ConstevalLoc;
-    tryConsumeLambdaSpecifierToken(*this, MutableLoc, ConstexprLoc,
-                                   ConstevalLoc, DeclEndLoc);
-
-    addConstexprToLambdaDeclSpecifier(*this, ConstexprLoc, DS);
-    addConstevalToLambdaDeclSpecifier(*this, ConstevalLoc, DS);
-    // Parse exception-specification[opt].
-    ExceptionSpecificationType ESpecType = EST_None;
-    SourceRange ESpecRange;
-    SmallVector<ParsedType, 2> DynamicExceptions;
-    SmallVector<SourceRange, 2> DynamicExceptionRanges;
-    ExprResult NoexceptExpr;
-    CachedTokens *ExceptionSpecTokens;
-    ESpecType = tryParseExceptionSpecification(/*Delayed=*/false,
-                                               ESpecRange,
-                                               DynamicExceptions,
-                                               DynamicExceptionRanges,
-                                               NoexceptExpr,
-                                               ExceptionSpecTokens);
-
-    if (ESpecType != EST_None)
-      DeclEndLoc = ESpecRange.getEnd();
-
-    // Parse attribute-specifier[opt].
-    MaybeParseCXX11Attributes(Attr, &DeclEndLoc);
-
-    // Parse OpenCL addr space attribute.
-    if (Tok.isOneOf(tok::kw___private, tok::kw___global, tok::kw___local,
-                    tok::kw___constant, tok::kw___generic)) {
-      ParseOpenCLQualifiers(DS.getAttributes());
-      ConsumeToken();
-    }
-
-    SourceLocation FunLocalRangeEnd = DeclEndLoc;
-
-    // Parse trailing-return-type[opt].
-    if (Tok.is(tok::arrow)) {
-      FunLocalRangeEnd = Tok.getLocation();
-      SourceRange Range;
-      TrailingReturnType =
-          ParseTrailingReturnType(Range, /*MayBeFollowedByDirectInit*/ false);
-      TrailingReturnTypeLoc = Range.getBegin();
-      if (Range.getEnd().isValid())
-        DeclEndLoc = Range.getEnd();
-    }
 
-    SourceLocation NoLoc;
-    D.AddTypeInfo(DeclaratorChunk::getFunction(
-                      /*HasProto=*/true,
-                      /*IsAmbiguous=*/false, LParenLoc, ParamInfo.data(),
-                      ParamInfo.size(), EllipsisLoc, RParenLoc,
-                      /*RefQualifierIsLvalueRef=*/true,
-                      /*RefQualifierLoc=*/NoLoc, MutableLoc, ESpecType,
-                      ESpecRange, DynamicExceptions.data(),
-                      DynamicExceptionRanges.data(), DynamicExceptions.size(),
-                      NoexceptExpr.isUsable() ? NoexceptExpr.get() : nullptr,
-                      /*ExceptionSpecTokens*/ nullptr,
-                      /*DeclsInPrototype=*/None, LParenLoc, FunLocalRangeEnd, D,
-                      TrailingReturnType, TrailingReturnTypeLoc, &DS),
-                  std::move(Attr), DeclEndLoc);
-
-    // Parse requires-clause[opt].
-    if (Tok.is(tok::kw_requires))
-      ParseTrailingRequiresClause(D);
-
-    PrototypeScope.Exit();
-
-    WarnIfHasCUDATargetAttr();
+    // Parse lambda-specifiers.
+    ParseLambdaSpecifiers(LParenLoc, /*DeclEndLoc=*/T.getCloseLocation(),
+                          ParamInfo, EllipsisLoc);
   } else if (Tok.isOneOf(tok::kw_mutable, tok::arrow, tok::kw___attribute,
                          tok::kw_constexpr, tok::kw_consteval,
                          tok::kw___private, tok::kw___global, tok::kw___local,
                          tok::kw___constant, tok::kw___generic,
-                         tok::kw_requires) ||
+                         tok::kw_requires, tok::kw_noexcept) ||
              (Tok.is(tok::l_square) && NextToken().is(tok::l_square))) {
-    // It's common to forget that one needs '()' before 'mutable', an attribute
-    // specifier, the result type, or the requires clause. Deal with this.
-    unsigned TokKind = 0;
-    switch (Tok.getKind()) {
-    case tok::kw_mutable: TokKind = 0; break;
-    case tok::arrow: TokKind = 1; break;
-    case tok::kw___attribute:
-    case tok::kw___private:
-    case tok::kw___global:
-    case tok::kw___local:
-    case tok::kw___constant:
-    case tok::kw___generic:
-    case tok::l_square: TokKind = 2; break;
-    case tok::kw_constexpr: TokKind = 3; break;
-    case tok::kw_consteval: TokKind = 4; break;
-    case tok::kw_requires: TokKind = 5; break;
-    default: llvm_unreachable("Unknown token kind");
-    }
-
-    Diag(Tok, diag::err_lambda_missing_parens)
-      << TokKind
-      << FixItHint::CreateInsertion(Tok.getLocation(), "() ");
-    SourceLocation DeclEndLoc = DeclLoc;
-
-    // GNU-style attributes must be parsed before the mutable specifier to be
-    // compatible with GCC.
-    MaybeParseGNUAttributes(Attr, &DeclEndLoc);
-
-    // Parse 'mutable', if it's there.
-    SourceLocation MutableLoc;
-    if (Tok.is(tok::kw_mutable)) {
-      MutableLoc = ConsumeToken();
-      DeclEndLoc = MutableLoc;
-    }
-
-    // Parse attribute-specifier[opt].
-    MaybeParseCXX11Attributes(Attr, &DeclEndLoc);
-
-    // Parse the return type, if there is one.
-    if (Tok.is(tok::arrow)) {
-      SourceRange Range;
-      TrailingReturnType =
-          ParseTrailingReturnType(Range, /*MayBeFollowedByDirectInit*/ false);
-      if (Range.getEnd().isValid())
-        DeclEndLoc = Range.getEnd();
-    }
+    if (!getLangOpts().CPlusPlus2b)
+      // It's common to forget that one needs '()' before 'mutable', an
+      // attribute specifier, the result type, or the requires clause. Deal with
+      // this.
+      Diag(Tok, diag::ext_lambda_missing_parens)
+          << FixItHint::CreateInsertion(Tok.getLocation(), "() ");
 
     SourceLocation NoLoc;
-    D.AddTypeInfo(DeclaratorChunk::getFunction(
-                      /*HasProto=*/true,
-                      /*IsAmbiguous=*/false,
-                      /*LParenLoc=*/NoLoc,
-                      /*Params=*/nullptr,
-                      /*NumParams=*/0,
-                      /*EllipsisLoc=*/NoLoc,
-                      /*RParenLoc=*/NoLoc,
-                      /*RefQualifierIsLvalueRef=*/true,
-                      /*RefQualifierLoc=*/NoLoc, MutableLoc, EST_None,
-                      /*ESpecRange=*/SourceRange(),
-                      /*Exceptions=*/nullptr,
-                      /*ExceptionRanges=*/nullptr,
-                      /*NumExceptions=*/0,
-                      /*NoexceptExpr=*/nullptr,
-                      /*ExceptionSpecTokens=*/nullptr,
-                      /*DeclsInPrototype=*/None, DeclLoc, DeclEndLoc, D,
-                      TrailingReturnType),
-                  std::move(Attr), DeclEndLoc);
-
-    // Parse the requires-clause, if present.
-    if (Tok.is(tok::kw_requires))
-      ParseTrailingRequiresClause(D);
-
-    WarnIfHasCUDATargetAttr();
+    // Parse lambda-specifiers.
+    std::vector<DeclaratorChunk::ParamInfo> EmptyParamInfo;
+    ParseLambdaSpecifiers(/*LParenLoc=*/NoLoc, /*RParenLoc=*/NoLoc,
+                          EmptyParamInfo, /*EllipsisLoc=*/NoLoc);
   }
 
   // FIXME: Rename BlockScope -> ClosureScope if we decide to continue using
@@ -1987,17 +1934,36 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
 /// \param FRI If non-null, a for range declaration is permitted, and if
 /// present will be parsed and stored here, and a null result will be returned.
 ///
+/// \param EnterForConditionScope If true, enter a continue/break scope at the
+/// appropriate moment for a 'for' loop.
+///
 /// \returns The parsed condition.
 Sema::ConditionResult Parser::ParseCXXCondition(StmtResult *InitStmt,
                                                 SourceLocation Loc,
                                                 Sema::ConditionKind CK,
-                                                ForRangeInfo *FRI) {
+                                                ForRangeInfo *FRI,
+                                                bool EnterForConditionScope) {
+  // Helper to ensure we always enter a continue/break scope if requested.
+  struct ForConditionScopeRAII {
+    Scope *S;
+    void enter(bool IsConditionVariable) {
+      if (S) {
+        S->AddFlags(Scope::BreakScope | Scope::ContinueScope);
+        S->setIsConditionVarScope(IsConditionVariable);
+      }
+    }
+    ~ForConditionScopeRAII() {
+      if (S)
+        S->setIsConditionVarScope(false);
+    }
+  } ForConditionScope{EnterForConditionScope ? getCurScope() : nullptr};
+
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
   PreferredType.enterCondition(Actions, Tok.getLocation());
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Condition);
     cutOffParsing();
+    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Condition);
     return Sema::ConditionError();
   }
 
@@ -2014,6 +1980,9 @@ Sema::ConditionResult Parser::ParseCXXCondition(StmtResult *InitStmt,
   // Determine what kind of thing we have.
   switch (isCXXConditionDeclarationOrInitStatement(InitStmt, FRI)) {
   case ConditionOrInitStatement::Expression: {
+    // If this is a for loop, we're entering its condition.
+    ForConditionScope.enter(/*IsConditionVariable=*/false);
+
     ProhibitAttributes(attrs);
 
     // We can have an empty expression here.
@@ -2056,6 +2025,9 @@ Sema::ConditionResult Parser::ParseCXXCondition(StmtResult *InitStmt,
   }
 
   case ConditionOrInitStatement::ForRangeDecl: {
+    // This is 'for (init-stmt; for-range-decl : range-expr)'.
+    // We're not actually in a for loop yet, so 'break' and 'continue' aren't
+    // permitted here.
     assert(FRI && "should not parse a for range declaration here");
     SourceLocation DeclStart = Tok.getLocation(), DeclEnd;
     DeclGroupPtrTy DG = ParseSimpleDeclaration(DeclaratorContext::ForInit,
@@ -2069,6 +2041,9 @@ Sema::ConditionResult Parser::ParseCXXCondition(StmtResult *InitStmt,
     break;
   }
 
+  // If this is a for loop, we're entering its condition.
+  ForConditionScope.enter(/*IsConditionVariable=*/true);
+
   // type-specifier-seq
   DeclSpec DS(AttrFactory);
   DS.takeAttributesFrom(attrs);
@@ -2608,10 +2583,10 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
     }
 
     case tok::code_completion: {
+      // Don't try to parse any further.
+      cutOffParsing();
       // Code completion for the operator name.
       Actions.CodeCompleteOperatorName(getCurScope());
-      cutOffParsing();
-      // Don't try to parse any further.
       return true;
     }
 
diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp
index 50e1f1eaba4d23c0a74595044edccae62d1952b6..9d9c03d28a97c3c2babbfacc239cb000f8e124f5 100644
--- a/clang/lib/Parse/ParseInit.cpp
+++ b/clang/lib/Parse/ParseInit.cpp
@@ -160,9 +160,6 @@ static void CheckArrayDesignatorSyntax(Parser &P, SourceLocation Loc,
 /// \p CodeCompleteCB is called with Designation parsed so far.
 ExprResult Parser::ParseInitializerWithPotentialDesignator(
     DesignatorCompletionInfo DesignatorCompletion) {
-  if (!getPreprocessor().isCodeCompletionEnabled())
-    DesignatorCompletion.PreferredBaseType = QualType(); // skip field lookup
-
   // If this is the old-style GNU extension:
   //   designation ::= identifier ':'
   // Handle it as a field designator.  Otherwise, this must be the start of a
@@ -203,9 +200,9 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator(
       SourceLocation DotLoc = ConsumeToken();
 
       if (Tok.is(tok::code_completion)) {
+        cutOffParsing();
         Actions.CodeCompleteDesignator(DesignatorCompletion.PreferredBaseType,
                                        DesignatorCompletion.InitExprs, Desig);
-        cutOffParsing();
         return ExprError();
       }
       if (Tok.isNot(tok::identifier)) {
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 223b36d7a0e69acfafb1a154a085be6719b52435..9e145f57d61f2142b29a26a7114427d9658510e7 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -50,8 +50,8 @@ Parser::ParseObjCAtDirectives(ParsedAttributesWithRange &Attrs) {
   SourceLocation AtLoc = ConsumeToken(); // the "@"
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteObjCAtDirective(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteObjCAtDirective(getCurScope());
     return nullptr;
   }
 
@@ -219,8 +219,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
 
   // Code completion after '@interface'.
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteObjCInterfaceDecl(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteObjCInterfaceDecl(getCurScope());
     return nullptr;
   }
 
@@ -253,8 +253,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
     SourceLocation categoryLoc;
     IdentifierInfo *categoryId = nullptr;
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCInterfaceCategory(getCurScope(), nameId, nameLoc);
       cutOffParsing();
+      Actions.CodeCompleteObjCInterfaceCategory(getCurScope(), nameId, nameLoc);
       return nullptr;
     }
 
@@ -308,8 +308,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
 
     // Code completion of superclass names.
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCSuperclass(getCurScope(), nameId, nameLoc);
       cutOffParsing();
+      Actions.CodeCompleteObjCSuperclass(getCurScope(), nameId, nameLoc);
       return nullptr;
     }
 
@@ -472,8 +472,8 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
       if (Tok.is(tok::code_completion)) {
         // FIXME: If these aren't protocol references, we'll need different
         // completions.
-        Actions.CodeCompleteObjCProtocolReferences(protocolIdents);
         cutOffParsing();
+        Actions.CodeCompleteObjCProtocolReferences(protocolIdents);
 
         // FIXME: Better recovery here?.
         return nullptr;
@@ -635,10 +635,11 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
 
     // Code completion within an Objective-C interface.
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       Actions.CodeCompleteOrdinaryName(getCurScope(),
                             CurParsedObjCImpl? Sema::PCC_ObjCImplementation
                                              : Sema::PCC_ObjCInterface);
-      return cutOffParsing();
+      return;
     }
 
     // If we don't have an @ directive, parse it as a function definition.
@@ -668,8 +669,9 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
     // Otherwise, we have an @ directive, eat the @.
     SourceLocation AtLoc = ConsumeToken(); // the "@"
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       Actions.CodeCompleteObjCAtDirective(getCurScope());
-      return cutOffParsing();
+      return;
     }
 
     tok::ObjCKeywordKind DirectiveKind = Tok.getObjCKeywordID();
@@ -778,8 +780,9 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
   // We break out of the big loop in two cases: when we see @end or when we see
   // EOF.  In the former case, eat the @end.  In the later case, emit an error.
   if (Tok.is(tok::code_completion)) {
+    cutOffParsing();
     Actions.CodeCompleteObjCAtDirective(getCurScope());
-    return cutOffParsing();
+    return;
   } else if (Tok.isObjCAtKeyword(tok::objc_end)) {
     ConsumeToken(); // the "end" identifier
   } else {
@@ -847,8 +850,9 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS) {
 
   while (1) {
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       Actions.CodeCompleteObjCPropertyFlags(getCurScope(), DS);
-      return cutOffParsing();
+      return;
     }
     const IdentifierInfo *II = Tok.getIdentifierInfo();
 
@@ -893,11 +897,12 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS) {
       }
 
       if (Tok.is(tok::code_completion)) {
+        cutOffParsing();
         if (IsSetter)
           Actions.CodeCompleteObjCPropertySetter(getCurScope());
         else
           Actions.CodeCompleteObjCPropertyGetter(getCurScope());
-        return cutOffParsing();
+        return;
       }
 
       SourceLocation SelLoc;
@@ -1146,9 +1151,10 @@ void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS,
 
   while (1) {
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       Actions.CodeCompleteObjCPassingType(
           getCurScope(), DS, Context == DeclaratorContext::ObjCParameter);
-      return cutOffParsing();
+      return;
     }
 
     if (Tok.isNot(tok::identifier))
@@ -1335,9 +1341,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
   ParsingDeclRAIIObject PD(*this, ParsingDeclRAIIObject::NoParent);
 
   if (Tok.is(tok::code_completion)) {
+    cutOffParsing();
     Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus,
                                        /*ReturnType=*/nullptr);
-    cutOffParsing();
     return nullptr;
   }
 
@@ -1354,9 +1360,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
                        methodAttrs);
 
   if (Tok.is(tok::code_completion)) {
+    cutOffParsing();
     Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus,
                                        ReturnType);
-    cutOffParsing();
     return nullptr;
   }
 
@@ -1416,12 +1422,12 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
     // Code completion for the next piece of the selector.
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       KeyIdents.push_back(SelIdent);
       Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(),
                                                  mType == tok::minus,
                                                  /*AtParameterName=*/true,
                                                  ReturnType, KeyIdents);
-      cutOffParsing();
       return nullptr;
     }
 
@@ -1441,11 +1447,11 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
     // Code completion for the next piece of the selector.
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(),
                                                  mType == tok::minus,
                                                  /*AtParameterName=*/false,
                                                  ReturnType, KeyIdents);
-      cutOffParsing();
       return nullptr;
     }
 
@@ -1527,8 +1533,8 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
 
   while (1) {
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCProtocolReferences(ProtocolIdents);
       cutOffParsing();
+      Actions.CodeCompleteObjCProtocolReferences(ProtocolIdents);
       return true;
     }
 
@@ -1626,12 +1632,12 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
       }
 
       QualType BaseT = Actions.GetTypeFromParser(baseType);
+      cutOffParsing();
       if (!BaseT.isNull() && BaseT->acceptsObjCTypeParams()) {
         Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Type);
       } else {
         Actions.CodeCompleteObjCProtocolReferences(identifierLocPairs);
       }
-      cutOffParsing();
       return;
     }
 
@@ -1920,8 +1926,9 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
     // Set the default visibility to private.
     if (TryConsumeToken(tok::at)) { // parse objc-visibility-spec
       if (Tok.is(tok::code_completion)) {
+        cutOffParsing();
         Actions.CodeCompleteObjCAtVisibility(getCurScope());
-        return cutOffParsing();
+        return;
       }
 
       switch (Tok.getObjCKeywordID()) {
@@ -1950,9 +1957,10 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
     }
 
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       Actions.CodeCompleteOrdinaryName(getCurScope(),
                                        Sema::PCC_ObjCInstanceVariableList);
-      return cutOffParsing();
+      return;
     }
 
     // This needs to duplicate a small amount of code from
@@ -2017,8 +2025,8 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   ConsumeToken(); // the "protocol" identifier
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteObjCProtocolDecl(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteObjCProtocolDecl(getCurScope());
     return nullptr;
   }
 
@@ -2101,8 +2109,8 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
 
   // Code completion after '@implementation'.
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteObjCImplementationDecl(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteObjCImplementationDecl(getCurScope());
     return nullptr;
   }
 
@@ -2139,8 +2147,8 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
     IdentifierInfo *categoryId = nullptr;
 
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCImplementationCategory(getCurScope(), nameId, nameLoc);
       cutOffParsing();
+      Actions.CodeCompleteObjCImplementationCategory(getCurScope(), nameId, nameLoc);
       return nullptr;
     }
 
@@ -2309,8 +2317,8 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
 
   while (true) {
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
       cutOffParsing();
+      Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
       return nullptr;
     }
 
@@ -2327,8 +2335,8 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
     if (TryConsumeToken(tok::equal)) {
       // property '=' ivar-name
       if (Tok.is(tok::code_completion)) {
-        Actions.CodeCompleteObjCPropertySynthesizeIvar(getCurScope(), propertyId);
         cutOffParsing();
+        Actions.CodeCompleteObjCPropertySynthesizeIvar(getCurScope(), propertyId);
         return nullptr;
       }
 
@@ -2387,8 +2395,8 @@ Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
 
   while (true) {
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
       cutOffParsing();
+      Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
       return nullptr;
     }
 
@@ -2724,8 +2732,8 @@ Decl *Parser::ParseObjCMethodDefinition() {
 StmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc,
                                         ParsedStmtContext StmtCtx) {
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteObjCAtStatement(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteObjCAtStatement(getCurScope());
     return StmtError();
   }
 
@@ -2765,8 +2773,8 @@ StmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc,
 ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
   switch (Tok.getKind()) {
   case tok::code_completion:
-    Actions.CodeCompleteObjCAtExpression(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteObjCAtExpression(getCurScope());
     return ExprError();
 
   case tok::minus:
@@ -3012,8 +3020,8 @@ ExprResult Parser::ParseObjCMessageExpression() {
   SourceLocation LBracLoc = ConsumeBracket(); // consume '['
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteObjCMessageReceiver(getCurScope());
     cutOffParsing();
+    Actions.CodeCompleteObjCMessageReceiver(getCurScope());
     return ExprError();
   }
 
@@ -3149,6 +3157,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
   InMessageExpressionRAIIObject InMessage(*this, true);
 
   if (Tok.is(tok::code_completion)) {
+    cutOffParsing();
     if (SuperLoc.isValid())
       Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc, None,
                                            false);
@@ -3158,7 +3167,6 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
     else
       Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
                                               None, false);
-    cutOffParsing();
     return ExprError();
   }
 
@@ -3187,6 +3195,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
       ///  Parse the expression after ':'
 
       if (Tok.is(tok::code_completion)) {
+        cutOffParsing();
         if (SuperLoc.isValid())
           Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc,
                                                KeyIdents,
@@ -3200,7 +3209,6 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
                                                   KeyIdents,
                                                   /*AtArgumentExpression=*/true);
 
-        cutOffParsing();
         return ExprError();
       }
 
@@ -3225,6 +3233,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
 
       // Code completion after each argument.
       if (Tok.is(tok::code_completion)) {
+        cutOffParsing();
         if (SuperLoc.isValid())
           Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc,
                                                KeyIdents,
@@ -3237,7 +3246,6 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
           Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
                                                   KeyIdents,
                                                 /*AtArgumentExpression=*/false);
-        cutOffParsing();
         return ExprError();
       }
 
@@ -3577,8 +3585,8 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
     ConsumeParen();
 
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents);
     cutOffParsing();
+    Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents);
     return ExprError();
   }
 
@@ -3603,8 +3611,8 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
         break;
 
       if (Tok.is(tok::code_completion)) {
-        Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents);
         cutOffParsing();
+        Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents);
         return ExprError();
       }
 
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 3de01be0db8785046c1f9181dd71102999b18a16..1ea01409d3d36342e9fe68a7b0e1954519a0486e 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -441,9 +441,9 @@ void Parser::ParseOpenMPReductionInitializerForDecl(VarDecl *OmpPrivParm) {
     ConsumeToken();
 
     if (Tok.is(tok::code_completion)) {
+      cutOffParsing();
       Actions.CodeCompleteInitializer(getCurScope(), OmpPrivParm);
       Actions.FinalizeDeclaration(OmpPrivParm);
-      cutOffParsing();
       return;
     }
 
@@ -2115,9 +2115,18 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
 
     ParsingOpenMPDirectiveRAII NormalScope(*this, /*Value=*/false);
     llvm::SmallVector<Decl *, 4> Decls;
-    DKind = parseOpenMPDirectiveKind(*this);
-    while (DKind != OMPD_end_declare_target && Tok.isNot(tok::eof) &&
-           Tok.isNot(tok::r_brace)) {
+    while (Tok.isNot(tok::eof) && Tok.isNot(tok::r_brace)) {
+      if (Tok.isAnnotation() && Tok.is(tok::annot_pragma_openmp)) {
+        TentativeParsingAction TPA(*this);
+        ConsumeAnnotationToken();
+        DKind = parseOpenMPDirectiveKind(*this);
+        if (DKind != OMPD_end_declare_target)
+          TPA.Revert();
+        else
+          TPA.Commit();
+      }
+      if (DKind == OMPD_end_declare_target)
+        break;
       DeclGroupPtrTy Ptr;
       // Here we expect to see some function declaration.
       if (AS == AS_none) {
@@ -2133,15 +2142,6 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
         DeclGroupRef Ref = Ptr.get();
         Decls.append(Ref.begin(), Ref.end());
       }
-      if (Tok.isAnnotation() && Tok.is(tok::annot_pragma_openmp)) {
-        TentativeParsingAction TPA(*this);
-        ConsumeAnnotationToken();
-        DKind = parseOpenMPDirectiveKind(*this);
-        if (DKind != OMPD_end_declare_target)
-          TPA.Revert();
-        else
-          TPA.Commit();
-      }
     }
 
     ParseOMPEndDeclareTargetDirective(DKind, DTLoc);
@@ -2378,6 +2378,7 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
   case OMPD_target_enter_data:
   case OMPD_target_exit_data:
   case OMPD_target_update:
+  case OMPD_interop:
     if ((StmtCtx & ParsedStmtContext::AllowStandaloneOpenMPDirectives) ==
         ParsedStmtContext()) {
       Diag(Tok, diag::err_omp_immediate_directive)
@@ -2864,7 +2865,6 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_unified_shared_memory:
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
-  case OMPC_destroy:
     // OpenMP [2.7.1, Restrictions, p. 9]
     //  Only one ordered clause can appear on a loop directive.
     // OpenMP [2.7.1, Restrictions, C/C++, p. 4]
@@ -2928,6 +2928,21 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_uses_allocators:
     Clause = ParseOpenMPUsesAllocatorClause(DKind);
     break;
+  case OMPC_destroy:
+    if (DKind != OMPD_interop) {
+      if (!FirstClause) {
+        Diag(Tok, diag::err_omp_more_one_clause)
+            << getOpenMPDirectiveName(DKind) << getOpenMPClauseName(CKind) << 0;
+        ErrorFound = true;
+      }
+      Clause = ParseOpenMPClause(CKind, WrongDirective);
+      break;
+    }
+    LLVM_FALLTHROUGH;
+  case OMPC_init:
+  case OMPC_use:
+    Clause = ParseOpenMPInteropClause(CKind, WrongDirective);
+    break;
   case OMPC_device_type:
   case OMPC_unknown:
     skipUntilPragmaOpenMPEnd(DKind);
@@ -3024,6 +3039,144 @@ OMPClause *Parser::ParseOpenMPSingleExprClause(OpenMPClauseKind Kind,
   return Actions.ActOnOpenMPSingleExprClause(Kind, Val.get(), Loc, LLoc, RLoc);
 }
 
+/// Parsing of OpenMP clauses that use an interop-var.
+///
+/// init-clause:
+///   init([interop-modifier, ]interop-type[[, interop-type] ... ]:interop-var)
+///
+/// destroy-clause:
+///   destroy(interop-var)
+///
+/// use-clause:
+///   use(interop-var)
+///
+/// interop-modifier:
+///   prefer_type(preference-list)
+///
+/// preference-list:
+///   foreign-runtime-id [, foreign-runtime-id]...
+///
+/// foreign-runtime-id:
+///   <string-literal> | <constant-integral-expression>
+///
+/// interop-type:
+///   target | targetsync
+///
+OMPClause *Parser::ParseOpenMPInteropClause(OpenMPClauseKind Kind,
+                                            bool ParseOnly) {
+  SourceLocation Loc = ConsumeToken();
+  // Parse '('.
+  BalancedDelimiterTracker T(*this, tok::l_paren, tok::annot_pragma_openmp_end);
+  if (T.expectAndConsume(diag::err_expected_lparen_after,
+                         getOpenMPClauseName(Kind).data()))
+    return nullptr;
+
+  bool IsTarget = false;
+  bool IsTargetSync = false;
+  SmallVector<Expr *, 4> Prefs;
+
+  if (Kind == OMPC_init) {
+
+    // Parse optional interop-modifier.
+    if (Tok.is(tok::identifier) && PP.getSpelling(Tok) == "prefer_type") {
+      ConsumeToken();
+      BalancedDelimiterTracker PT(*this, tok::l_paren,
+                                  tok::annot_pragma_openmp_end);
+      if (PT.expectAndConsume(diag::err_expected_lparen_after, "prefer_type"))
+        return nullptr;
+
+      while (Tok.isNot(tok::r_paren)) {
+        SourceLocation Loc = Tok.getLocation();
+        ExprResult LHS = ParseCastExpression(AnyCastExpr);
+        ExprResult PTExpr = Actions.CorrectDelayedTyposInExpr(
+            ParseRHSOfBinaryExpression(LHS, prec::Conditional));
+        PTExpr = Actions.ActOnFinishFullExpr(PTExpr.get(), Loc,
+                                             /*DiscardedValue=*/false);
+        if (PTExpr.isUsable())
+          Prefs.push_back(PTExpr.get());
+        else
+          SkipUntil(tok::comma, tok::r_paren, tok::annot_pragma_openmp_end,
+                    StopBeforeMatch);
+
+        if (Tok.is(tok::comma))
+          ConsumeToken();
+      }
+      PT.consumeClose();
+    }
+
+    if (!Prefs.empty()) {
+      if (Tok.is(tok::comma))
+        ConsumeToken();
+      else
+        Diag(Tok, diag::err_omp_expected_punc_after_interop_mod);
+    }
+
+    // Parse the interop-types.
+    bool HasError = false;
+    while (Tok.is(tok::identifier)) {
+      if (PP.getSpelling(Tok) == "target") {
+        // OpenMP 5.1 [2.15.1, interop Construct, Restrictions]
+        // Each interop-type may be specified on an action-clause at most
+        // once.
+        if (IsTarget)
+          Diag(Tok, diag::warn_omp_more_one_interop_type) << "target";
+        IsTarget = true;
+      } else if (PP.getSpelling(Tok) == "targetsync") {
+        if (IsTargetSync)
+          Diag(Tok, diag::warn_omp_more_one_interop_type) << "targetsync";
+        IsTargetSync = true;
+      } else {
+        HasError = true;
+        Diag(Tok, diag::err_omp_expected_interop_type);
+      }
+      ConsumeToken();
+
+      if (!Tok.is(tok::comma))
+        break;
+      ConsumeToken();
+    }
+    if (!HasError && !IsTarget && !IsTargetSync)
+      Diag(Tok, diag::err_omp_expected_interop_type);
+
+    if (Tok.is(tok::colon))
+      ConsumeToken();
+    else if (IsTarget || IsTargetSync)
+      Diag(Tok, diag::warn_pragma_expected_colon) << "interop types";
+  }
+
+  // Parse the variable.
+  SourceLocation VarLoc = Tok.getLocation();
+  ExprResult InteropVarExpr =
+      Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+  if (!InteropVarExpr.isUsable()) {
+    SkipUntil(tok::comma, tok::r_paren, tok::annot_pragma_openmp_end,
+              StopBeforeMatch);
+  }
+
+  // Parse ')'.
+  SourceLocation RLoc = Tok.getLocation();
+  if (!T.consumeClose())
+    RLoc = T.getCloseLocation();
+
+  if (ParseOnly || !InteropVarExpr.isUsable() ||
+      (Kind == OMPC_init && !IsTarget && !IsTargetSync))
+    return nullptr;
+
+  if (Kind == OMPC_init)
+    return Actions.ActOnOpenMPInitClause(InteropVarExpr.get(), Prefs, IsTarget,
+                                         IsTargetSync, Loc, T.getOpenLocation(),
+                                         VarLoc, RLoc);
+  if (Kind == OMPC_use)
+    return Actions.ActOnOpenMPUseClause(InteropVarExpr.get(), Loc,
+                                        T.getOpenLocation(), VarLoc, RLoc);
+
+  if (Kind == OMPC_destroy)
+    return Actions.ActOnOpenMPDestroyClause(InteropVarExpr.get(), Loc,
+                                            T.getOpenLocation(), VarLoc, RLoc);
+
+  llvm_unreachable("Unexpected interop variable clause.");
+}
+
 /// Parsing of simple OpenMP clauses like 'default' or 'proc_bind'.
 ///
 ///    default-clause:
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index f59271c458488d27bf4a9be6984cace7636860c1..bcda3560ce63e54e0d66fe5565eb6efa98f03a5f 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -172,14 +172,13 @@ Retry:
   switch (Kind) {
   case tok::at: // May be a @try or @throw statement
     {
-      ProhibitAttributes(Attrs); // TODO: is it correct?
       AtLoc = ConsumeToken();  // consume @
       return ParseObjCAtStatement(AtLoc, StmtCtx);
     }
 
   case tok::code_completion:
-    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Statement);
     cutOffParsing();
+    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Statement);
     return StmtError();
 
   case tok::identifier: {
@@ -726,8 +725,8 @@ StmtResult Parser::ParseCaseStatement(ParsedStmtContext StmtCtx,
     ColonLoc = SourceLocation();
 
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteCase(getCurScope());
       cutOffParsing();
+      Actions.CodeCompleteCase(getCurScope());
       return StmtError();
     }
 
@@ -1472,8 +1471,8 @@ StmtResult Parser::ParseIfStatement(SourceLocation *TrailingElseLoc) {
     // Pop the 'else' scope if needed.
     InnerScope.Exit();
   } else if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteAfterIf(getCurScope(), IsBracedThen);
     cutOffParsing();
+    Actions.CodeCompleteAfterIf(getCurScope(), IsBracedThen);
     return StmtError();
   } else if (InnerStatementTrailingElseLoc.isValid()) {
     Diag(InnerStatementTrailingElseLoc, diag::warn_dangling_else);
@@ -1827,10 +1826,10 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
   FullExprArg ThirdPart(Actions);
 
   if (Tok.is(tok::code_completion)) {
+    cutOffParsing();
     Actions.CodeCompleteOrdinaryName(getCurScope(),
                                      C99orCXXorObjC? Sema::PCC_ForInit
                                                    : Sema::PCC_Expression);
-    cutOffParsing();
     return StmtError();
   }
 
@@ -1898,8 +1897,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
       ConsumeToken(); // consume 'in'
 
       if (Tok.is(tok::code_completion)) {
-        Actions.CodeCompleteObjCForCollection(getCurScope(), DG);
         cutOffParsing();
+        Actions.CodeCompleteObjCForCollection(getCurScope(), DG);
         return StmtError();
       }
       Collection = ParseExpression();
@@ -1934,8 +1933,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
       ConsumeToken(); // consume 'in'
 
       if (Tok.is(tok::code_completion)) {
-        Actions.CodeCompleteObjCForCollection(getCurScope(), nullptr);
         cutOffParsing();
+        Actions.CodeCompleteObjCForCollection(getCurScope(), nullptr);
         return StmtError();
       }
       Collection = ParseExpression();
@@ -1959,7 +1958,6 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
   }
 
   // Parse the second part of the for specifier.
-  getCurScope()->AddFlags(Scope::BreakScope | Scope::ContinueScope);
   if (!ForEach && !ForRangeInfo.ParsedForRangeDecl() &&
       !SecondPart.isInvalid()) {
     // Parse the second part of the for specifier.
@@ -1975,7 +1973,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
         ColonProtectionRAIIObject ColonProtection(*this, MightBeForRangeStmt);
         SecondPart =
             ParseCXXCondition(nullptr, ForLoc, Sema::ConditionKind::Boolean,
-                              MightBeForRangeStmt ? &ForRangeInfo : nullptr);
+                              MightBeForRangeStmt ? &ForRangeInfo : nullptr,
+                              /*EnterForConditionScope*/ true);
 
         if (ForRangeInfo.ParsedForRangeDecl()) {
           Diag(FirstPart.get() ? FirstPart.get()->getBeginLoc()
@@ -1992,6 +1991,9 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
           }
         }
       } else {
+        // We permit 'continue' and 'break' in the condition of a for loop.
+        getCurScope()->AddFlags(Scope::BreakScope | Scope::ContinueScope);
+
         ExprResult SecondExpr = ParseExpression();
         if (SecondExpr.isInvalid())
           SecondPart = Sema::ConditionError();
@@ -2003,6 +2005,11 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
     }
   }
 
+  // Enter a break / continue scope, if we didn't already enter one while
+  // parsing the second part.
+  if (!(getCurScope()->getFlags() & Scope::ContinueScope))
+    getCurScope()->AddFlags(Scope::BreakScope | Scope::ContinueScope);
+
   // Parse the third part of the for statement.
   if (!ForEach && !ForRangeInfo.ParsedForRangeDecl()) {
     if (Tok.isNot(tok::semi)) {
@@ -2188,9 +2195,9 @@ StmtResult Parser::ParseReturnStatement() {
       PreferredType.enterReturn(Actions, Tok.getLocation());
     // FIXME: Code completion for co_return.
     if (Tok.is(tok::code_completion) && !IsCoreturn) {
+      cutOffParsing();
       Actions.CodeCompleteExpression(getCurScope(),
                                      PreferredType.get(Tok.getLocation()));
-      cutOffParsing();
       return StmtError();
     }
 
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 9b0f921b426928ab13728859a7e0473a68ca3898..b178b56e967c63ae2b2ecbc86b47cc0f9e741da0 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -49,10 +49,10 @@ IdentifierInfo *Parser::getSEHExceptKeyword() {
 }
 
 Parser::Parser(Preprocessor &pp, Sema &actions, bool skipFunctionBodies)
-  : PP(pp), Actions(actions), Diags(PP.getDiagnostics()),
-    GreaterThanIsOperator(true), ColonIsSacred(false),
-    InMessageExpression(false), TemplateParameterDepth(0),
-    ParsingInObjCContainer(false) {
+    : PP(pp), PreferredType(pp.isCodeCompletionEnabled()), Actions(actions),
+      Diags(PP.getDiagnostics()), GreaterThanIsOperator(true),
+      ColonIsSacred(false), InMessageExpression(false),
+      TemplateParameterDepth(0), ParsingInObjCContainer(false) {
   SkipFunctionBodies = pp.isCodeCompletionEnabled() || skipFunctionBodies;
   Tok.startToken();
   Tok.setKind(tok::eof);
@@ -870,6 +870,7 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
     SingleDecl = ParseObjCMethodDefinition();
     break;
   case tok::code_completion:
+    cutOffParsing();
     if (CurParsedObjCImpl) {
       // Code-complete Objective-C methods even without leading '-'/'+' prefix.
       Actions.CodeCompleteObjCMethodDecl(getCurScope(),
@@ -879,7 +880,6 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
     Actions.CodeCompleteOrdinaryName(
         getCurScope(),
         CurParsedObjCImpl ? Sema::PCC_ObjCImplementation : Sema::PCC_Namespace);
-    cutOffParsing();
     return nullptr;
   case tok::kw_import:
     SingleDecl = ParseModuleImport(SourceLocation());
@@ -2114,21 +2114,21 @@ SourceLocation Parser::handleUnexpectedCodeCompletionToken() {
 
   for (Scope *S = getCurScope(); S; S = S->getParent()) {
     if (S->getFlags() & Scope::FnScope) {
+      cutOffParsing();
       Actions.CodeCompleteOrdinaryName(getCurScope(),
                                        Sema::PCC_RecoveryInFunction);
-      cutOffParsing();
       return PrevTokLocation;
     }
 
     if (S->getFlags() & Scope::ClassScope) {
-      Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Class);
       cutOffParsing();
+      Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Class);
       return PrevTokLocation;
     }
   }
 
-  Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Namespace);
   cutOffParsing();
+  Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Namespace);
   return PrevTokLocation;
 }
 
@@ -2452,8 +2452,8 @@ bool Parser::ParseModuleName(
   while (true) {
     if (!Tok.is(tok::identifier)) {
       if (Tok.is(tok::code_completion)) {
-        Actions.CodeCompleteModuleImport(UseLoc, Path);
         cutOffParsing();
+        Actions.CodeCompleteModuleImport(UseLoc, Path);
         return true;
       }
 
diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp
index edd9742ed20715f7a9fd44cda2c56a7934fb37e9..bcd6a00d7ba5a0d087750d87180074037a903c00 100644
--- a/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -1506,6 +1506,25 @@ static void diagnoseRepeatedUseOfWeak(Sema &S,
   }
 }
 
+namespace clang {
+namespace {
+typedef SmallVector<PartialDiagnosticAt, 1> OptionalNotes;
+typedef std::pair<PartialDiagnosticAt, OptionalNotes> DelayedDiag;
+typedef std::list<DelayedDiag> DiagList;
+
+struct SortDiagBySourceLocation {
+  SourceManager &SM;
+  SortDiagBySourceLocation(SourceManager &SM) : SM(SM) {}
+
+  bool operator()(const DelayedDiag &left, const DelayedDiag &right) {
+    // Although this call will be slow, this is only called when outputting
+    // multiple warnings.
+    return SM.isBeforeInTranslationUnit(left.first.first, right.first.first);
+  }
+};
+} // anonymous namespace
+} // namespace clang
+
 namespace {
 class UninitValsDiagReporter : public UninitVariablesHandler {
   Sema &S;
@@ -1626,9 +1645,35 @@ private:
   }
 };
 
+/// Inter-procedural data for the called-once checker.
+class CalledOnceInterProceduralData {
+public:
+  // Add the delayed warning for the given block.
+  void addDelayedWarning(const BlockDecl *Block,
+                         PartialDiagnosticAt &&Warning) {
+    DelayedBlockWarnings[Block].emplace_back(std::move(Warning));
+  }
+  // Report all of the warnings we've gathered for the given block.
+  void flushWarnings(const BlockDecl *Block, Sema &S) {
+    for (const PartialDiagnosticAt &Delayed : DelayedBlockWarnings[Block])
+      S.Diag(Delayed.first, Delayed.second);
+
+    discardWarnings(Block);
+  }
+  // Discard all of the warnings we've gathered for the given block.
+  void discardWarnings(const BlockDecl *Block) {
+    DelayedBlockWarnings.erase(Block);
+  }
+
+private:
+  using DelayedDiagnostics = SmallVector<PartialDiagnosticAt, 2>;
+  llvm::DenseMap<const BlockDecl *, DelayedDiagnostics> DelayedBlockWarnings;
+};
+
 class CalledOnceCheckReporter : public CalledOnceCheckHandler {
 public:
-  CalledOnceCheckReporter(Sema &S) : S(S) {}
+  CalledOnceCheckReporter(Sema &S, CalledOnceInterProceduralData &Data)
+      : S(S), Data(Data) {}
   void handleDoubleCall(const ParmVarDecl *Parameter, const Expr *Call,
                         const Expr *PrevCall, bool IsCompletionHandler,
                         bool Poised) override {
@@ -1649,14 +1694,24 @@ public:
         << Parameter << /* Captured */ false;
   }
 
-  void handleNeverCalled(const ParmVarDecl *Parameter, const Stmt *Where,
-                         NeverCalledReason Reason, bool IsCalledDirectly,
+  void handleNeverCalled(const ParmVarDecl *Parameter, const Decl *Function,
+                         const Stmt *Where, NeverCalledReason Reason,
+                         bool IsCalledDirectly,
                          bool IsCompletionHandler) override {
     auto DiagToReport = IsCompletionHandler
                             ? diag::warn_completion_handler_never_called_when
                             : diag::warn_called_once_never_called_when;
-    S.Diag(Where->getBeginLoc(), DiagToReport)
-        << Parameter << IsCalledDirectly << (unsigned)Reason;
+    PartialDiagnosticAt Warning(Where->getBeginLoc(), S.PDiag(DiagToReport)
+                                                          << Parameter
+                                                          << IsCalledDirectly
+                                                          << (unsigned)Reason);
+
+    if (const auto *Block = dyn_cast<BlockDecl>(Function)) {
+      // We shouldn't report these warnings on blocks immediately
+      Data.addDelayedWarning(Block, std::move(Warning));
+    } else {
+      S.Diag(Warning.first, Warning.second);
+    }
   }
 
   void handleCapturedNeverCalled(const ParmVarDecl *Parameter,
@@ -1669,8 +1724,18 @@ public:
         << Parameter << /* Captured */ true;
   }
 
+  void
+  handleBlockThatIsGuaranteedToBeCalledOnce(const BlockDecl *Block) override {
+    Data.flushWarnings(Block, S);
+  }
+
+  void handleBlockWithNoGuarantees(const BlockDecl *Block) override {
+    Data.discardWarnings(Block);
+  }
+
 private:
   Sema &S;
+  CalledOnceInterProceduralData &Data;
 };
 
 constexpr unsigned CalledOnceWarnings[] = {
@@ -1703,25 +1768,6 @@ bool shouldAnalyzeCalledOnceParameters(const DiagnosticsEngine &Diags,
 }
 } // anonymous namespace
 
-namespace clang {
-namespace {
-typedef SmallVector<PartialDiagnosticAt, 1> OptionalNotes;
-typedef std::pair<PartialDiagnosticAt, OptionalNotes> DelayedDiag;
-typedef std::list<DelayedDiag> DiagList;
-
-struct SortDiagBySourceLocation {
-  SourceManager &SM;
-  SortDiagBySourceLocation(SourceManager &SM) : SM(SM) {}
-
-  bool operator()(const DelayedDiag &left, const DelayedDiag &right) {
-    // Although this call will be slow, this is only called when outputting
-    // multiple warnings.
-    return SM.isBeforeInTranslationUnit(left.first.first, right.first.first);
-  }
-};
-} // anonymous namespace
-} // namespace clang
-
 //===----------------------------------------------------------------------===//
 // -Wthread-safety
 //===----------------------------------------------------------------------===//
@@ -2107,54 +2153,68 @@ public:
 //  warnings on a function, method, or block.
 //===----------------------------------------------------------------------===//
 
-clang::sema::AnalysisBasedWarnings::Policy::Policy() {
+sema::AnalysisBasedWarnings::Policy::Policy() {
   enableCheckFallThrough = 1;
   enableCheckUnreachable = 0;
   enableThreadSafetyAnalysis = 0;
   enableConsumedAnalysis = 0;
 }
 
+/// InterProceduralData aims to be a storage of whatever data should be passed
+/// between analyses of different functions.
+///
+/// At the moment, its primary goal is to make the information gathered during
+/// the analysis of the blocks available during the analysis of the enclosing
+/// function.  This is important due to the fact that blocks are analyzed before
+/// the enclosed function is even parsed fully, so it is not viable to access
+/// anything in the outer scope while analyzing the block.  On the other hand,
+/// re-building CFG for blocks and re-analyzing them when we do have all the
+/// information (i.e. during the analysis of the enclosing function) seems to be
+/// ill-designed.
+class sema::AnalysisBasedWarnings::InterProceduralData {
+public:
+  // It is important to analyze blocks within functions because it's a very
+  // common pattern to capture completion handler parameters by blocks.
+  CalledOnceInterProceduralData CalledOnceData;
+};
+
 static unsigned isEnabled(DiagnosticsEngine &D, unsigned diag) {
   return (unsigned)!D.isIgnored(diag, SourceLocation());
 }
 
-clang::sema::AnalysisBasedWarnings::AnalysisBasedWarnings(Sema &s)
-  : S(s),
-    NumFunctionsAnalyzed(0),
-    NumFunctionsWithBadCFGs(0),
-    NumCFGBlocks(0),
-    MaxCFGBlocksPerFunction(0),
-    NumUninitAnalysisFunctions(0),
-    NumUninitAnalysisVariables(0),
-    MaxUninitAnalysisVariablesPerFunction(0),
-    NumUninitAnalysisBlockVisits(0),
-    MaxUninitAnalysisBlockVisitsPerFunction(0) {
+sema::AnalysisBasedWarnings::AnalysisBasedWarnings(Sema &s)
+    : S(s), IPData(std::make_unique<InterProceduralData>()),
+      NumFunctionsAnalyzed(0), NumFunctionsWithBadCFGs(0), NumCFGBlocks(0),
+      MaxCFGBlocksPerFunction(0), NumUninitAnalysisFunctions(0),
+      NumUninitAnalysisVariables(0), MaxUninitAnalysisVariablesPerFunction(0),
+      NumUninitAnalysisBlockVisits(0),
+      MaxUninitAnalysisBlockVisitsPerFunction(0) {
 
   using namespace diag;
   DiagnosticsEngine &D = S.getDiagnostics();
 
   DefaultPolicy.enableCheckUnreachable =
-    isEnabled(D, warn_unreachable) ||
-    isEnabled(D, warn_unreachable_break) ||
-    isEnabled(D, warn_unreachable_return) ||
-    isEnabled(D, warn_unreachable_loop_increment);
+      isEnabled(D, warn_unreachable) || isEnabled(D, warn_unreachable_break) ||
+      isEnabled(D, warn_unreachable_return) ||
+      isEnabled(D, warn_unreachable_loop_increment);
 
-  DefaultPolicy.enableThreadSafetyAnalysis =
-    isEnabled(D, warn_double_lock);
+  DefaultPolicy.enableThreadSafetyAnalysis = isEnabled(D, warn_double_lock);
 
   DefaultPolicy.enableConsumedAnalysis =
-    isEnabled(D, warn_use_in_invalid_state);
+      isEnabled(D, warn_use_in_invalid_state);
 }
 
+// We need this here for unique_ptr with forward declared class.
+sema::AnalysisBasedWarnings::~AnalysisBasedWarnings() = default;
+
 static void flushDiagnostics(Sema &S, const sema::FunctionScopeInfo *fscope) {
   for (const auto &D : fscope->PossiblyUnreachableDiags)
     S.Diag(D.Loc, D.PD);
 }
 
-void clang::sema::
-AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
-                                     sema::FunctionScopeInfo *fscope,
-                                     const Decl *D, QualType BlockType) {
+void clang::sema::AnalysisBasedWarnings::IssueWarnings(
+    sema::AnalysisBasedWarnings::Policy P, sema::FunctionScopeInfo *fscope,
+    const Decl *D, QualType BlockType) {
 
   // We avoid doing analysis-based warnings when there are errors for
   // two reasons:
@@ -2346,7 +2406,7 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
   if (S.getLangOpts().ObjC &&
       shouldAnalyzeCalledOnceParameters(Diags, D->getBeginLoc())) {
     if (AC.getCFG()) {
-      CalledOnceCheckReporter Reporter(S);
+      CalledOnceCheckReporter Reporter(S, IPData->CalledOnceData);
       checkCalledOnceParameters(
           AC, Reporter,
           shouldAnalyzeCalledOnceConventions(Diags, D->getBeginLoc()));
diff --git a/clang/lib/Sema/MultiplexExternalSemaSource.cpp b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
index 252008cda15d20633fc2e3dccec47af003122a1e..3a993e24b134de3f41e35672589e6627f7ecd968 100644
--- a/clang/lib/Sema/MultiplexExternalSemaSource.cpp
+++ b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 #include "clang/Sema/MultiplexExternalSemaSource.h"
-#include "clang/AST/DeclContextInternals.h"
 #include "clang/Sema/Lookup.h"
 
 using namespace clang;
diff --git a/clang/lib/Sema/OpenCLBuiltins.td b/clang/lib/Sema/OpenCLBuiltins.td
index d6d77dc90d30281ae9d509306946e29a9cc806ee..1ff658e567b848ec04c85047df3193519435b2d4 100644
--- a/clang/lib/Sema/OpenCLBuiltins.td
+++ b/clang/lib/Sema/OpenCLBuiltins.td
@@ -1100,7 +1100,6 @@ let MinVersion = CL20 in {
 
   foreach TypePair = [[AtomicInt, Int, Int], [AtomicUInt, UInt, UInt],
                       [AtomicLong, Long, Long], [AtomicULong, ULong, ULong],
-                      [AtomicIntPtr, IntPtr, PtrDiff],
                       [AtomicUIntPtr, UIntPtr, PtrDiff]] in {
     foreach ModOp = ["add", "sub"] in {
       def : Builtin<"atomic_fetch_" # ModOp,
@@ -1112,9 +1111,7 @@ let MinVersion = CL20 in {
     }
   }
   foreach TypePair = [[AtomicInt, Int, Int], [AtomicUInt, UInt, UInt],
-                      [AtomicLong, Long, Long], [AtomicULong, ULong, ULong],
-                      [AtomicIntPtr, IntPtr, IntPtr],
-                      [AtomicUIntPtr, UIntPtr, UIntPtr]] in {
+                      [AtomicLong, Long, Long], [AtomicULong, ULong, ULong]] in {
     foreach ModOp = ["or", "xor", "and", "min", "max"] in {
       def : Builtin<"atomic_fetch_" # ModOp,
           [TypePair[1], PointerType<VolatileType<TypePair[0]>, GenericAS>, TypePair[2]]>;
diff --git a/clang/lib/Sema/ParsedAttr.cpp b/clang/lib/Sema/ParsedAttr.cpp
index c6a3d7c4342c9bda872b93da45fb296a74645590..1ac7ed1afc4e599e0fe41d1113753c3dc85bc930 100644
--- a/clang/lib/Sema/ParsedAttr.cpp
+++ b/clang/lib/Sema/ParsedAttr.cpp
@@ -159,6 +159,10 @@ bool ParsedAttr::diagnoseAppertainsTo(Sema &S, const Decl *D) const {
   return getInfo().diagAppertainsToDecl(S, *this, D);
 }
 
+bool ParsedAttr::diagnoseAppertainsTo(Sema &S, const Stmt *St) const {
+  return getInfo().diagAppertainsToStmt(S, *this, St);
+}
+
 bool ParsedAttr::appliesToDecl(const Decl *D,
                                attr::SubjectMatchRule MatchRule) const {
   return checkAttributeMatchRuleAppliesTo(D, MatchRule);
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index 9df2b7f84b579bb4c09d981a8356e5e1ef0d797b..2c37ccee16161255f83ab6c05898ce4e58e2610a 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -1188,3 +1188,51 @@ void Sema::PopPragmaVisibility(bool IsNamespaceEnd, SourceLocation EndLoc) {
   if (Stack->empty())
     FreeVisContext();
 }
+
+template <typename Ty>
+static bool checkCommonAttributeFeatures(Sema& S, const Ty *Node,
+                                         const ParsedAttr& A) {
+  // Several attributes carry different semantics than the parsing requires, so
+  // those are opted out of the common argument checks.
+  //
+  // We also bail on unknown and ignored attributes because those are handled
+  // as part of the target-specific handling logic.
+  if (A.getKind() == ParsedAttr::UnknownAttribute)
+    return false;
+  // Check whether the attribute requires specific language extensions to be
+  // enabled.
+  if (!A.diagnoseLangOpts(S))
+    return true;
+  // Check whether the attribute appertains to the given subject.
+  if (!A.diagnoseAppertainsTo(S, Node))
+    return true;
+  // Check whether the attribute exists in the target architecture.
+  if (S.CheckAttrTarget(A))
+    return true;
+
+  if (A.hasCustomParsing())
+    return false;
+
+  if (A.getMinArgs() == A.getMaxArgs()) {
+    // If there are no optional arguments, then checking for the argument count
+    // is trivial.
+    if (!A.checkExactlyNumArgs(S, A.getMinArgs()))
+      return true;
+  } else {
+    // There are optional arguments, so checking is slightly more involved.
+    if (A.getMinArgs() && !A.checkAtLeastNumArgs(S, A.getMinArgs()))
+      return true;
+    else if (!A.hasVariadicArg() && A.getMaxArgs() &&
+             !A.checkAtMostNumArgs(S, A.getMaxArgs()))
+      return true;
+  }
+
+  return false;
+}
+
+bool Sema::checkCommonAttributeFeatures(const Decl *D, const ParsedAttr &A) {
+  return ::checkCommonAttributeFeatures(*this, D, A);
+}
+bool Sema::checkCommonAttributeFeatures(const Stmt *S, const ParsedAttr &A) {
+  return ::checkCommonAttributeFeatures(*this, S, A);
+}
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 22ec2c7ed8bbfdca207fb5c5290a2b39f459a9f9..719cbf46bd5c26041183ffbdcc8a17d46d8bbd92 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -13,8 +13,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Sema/SemaInternal.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/ASTStructuralEquivalence.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
@@ -23,6 +23,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Initialization.h"
+#include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/SmallVector.h"
 #include <set>
 using namespace clang;
@@ -1035,6 +1036,90 @@ static void DiagnoseReinterpretUpDownCast(Sema &Self, const Expr *SrcExpr,
     << FixItHint::CreateReplacement(BeginLoc, "static_cast");
 }
 
+static bool argTypeIsABIEquivalent(QualType SrcType, QualType DestType,
+                                   ASTContext &Context) {
+  if (SrcType->isPointerType() && DestType->isPointerType())
+    return true;
+
+  // Allow integral type mismatch if their size are equal.
+  if (SrcType->isIntegralType(Context) && DestType->isIntegralType(Context))
+    if (Context.getTypeInfoInChars(SrcType).Width ==
+        Context.getTypeInfoInChars(DestType).Width)
+      return true;
+
+  return Context.hasSameUnqualifiedType(SrcType, DestType);
+}
+
+static bool checkCastFunctionType(Sema &Self, const ExprResult &SrcExpr,
+                                  QualType DestType) {
+  if (Self.Diags.isIgnored(diag::warn_cast_function_type,
+                           SrcExpr.get()->getExprLoc()))
+    return true;
+
+  QualType SrcType = SrcExpr.get()->getType();
+  const FunctionType *SrcFTy = nullptr;
+  const FunctionType *DstFTy = nullptr;
+  if (((SrcType->isBlockPointerType() || SrcType->isFunctionPointerType()) &&
+       DestType->isFunctionPointerType()) ||
+      (SrcType->isMemberFunctionPointerType() &&
+       DestType->isMemberFunctionPointerType())) {
+    SrcFTy = SrcType->getPointeeType()->castAs<FunctionType>();
+    DstFTy = DestType->getPointeeType()->castAs<FunctionType>();
+  } else if (SrcType->isFunctionType() && DestType->isFunctionReferenceType()) {
+    SrcFTy = SrcType->castAs<FunctionType>();
+    DstFTy = DestType.getNonReferenceType()->castAs<FunctionType>();
+  } else {
+    return true;
+  }
+  assert(SrcFTy && DstFTy);
+
+  auto IsVoidVoid = [](const FunctionType *T) {
+    if (!T->getReturnType()->isVoidType())
+      return false;
+    if (const auto *PT = T->getAs<FunctionProtoType>())
+      return !PT->isVariadic() && PT->getNumParams() == 0;
+    return false;
+  };
+
+  // Skip if either function type is void(*)(void)
+  if (IsVoidVoid(SrcFTy) || IsVoidVoid(DstFTy))
+    return true;
+
+  // Check return type.
+  if (!argTypeIsABIEquivalent(SrcFTy->getReturnType(), DstFTy->getReturnType(),
+                              Self.Context))
+    return false;
+
+  // Check if either has unspecified number of parameters
+  if (SrcFTy->isFunctionNoProtoType() || DstFTy->isFunctionNoProtoType())
+    return true;
+
+  // Check parameter types.
+
+  const auto *SrcFPTy = cast<FunctionProtoType>(SrcFTy);
+  const auto *DstFPTy = cast<FunctionProtoType>(DstFTy);
+
+  // In a cast involving function types with a variable argument list only the
+  // types of initial arguments that are provided are considered.
+  unsigned NumParams = SrcFPTy->getNumParams();
+  unsigned DstNumParams = DstFPTy->getNumParams();
+  if (NumParams > DstNumParams) {
+    if (!DstFPTy->isVariadic())
+      return false;
+    NumParams = DstNumParams;
+  } else if (NumParams < DstNumParams) {
+    if (!SrcFPTy->isVariadic())
+      return false;
+  }
+
+  for (unsigned i = 0; i < NumParams; ++i)
+    if (!argTypeIsABIEquivalent(SrcFPTy->getParamType(i),
+                                DstFPTy->getParamType(i), Self.Context))
+      return false;
+
+  return true;
+}
+
 /// CheckReinterpretCast - Check that a reinterpret_cast\<DestType\>(SrcExpr) is
 /// valid.
 /// Refer to C++ 5.2.10 for details. reinterpret_cast is typically used in code
@@ -1072,6 +1157,10 @@ void CastOperation::CheckReinterpretCast() {
     if (Self.getLangOpts().allowsNonTrivialObjCLifetimeQualifiers())
       checkObjCConversion(Sema::CCK_OtherCast);
     DiagnoseReinterpretUpDownCast(Self, SrcExpr.get(), DestType, OpRange);
+
+    if (!checkCastFunctionType(Self, SrcExpr, DestType))
+      Self.Diag(OpRange.getBegin(), diag::warn_cast_function_type)
+          << SrcExpr.get()->getType() << DestType << OpRange;
   } else {
     SrcExpr = ExprError();
   }
@@ -2645,6 +2734,11 @@ void CastOperation::CheckCXXCStyleCast(bool FunctionalStyle,
   if (isValidCast(tcr)) {
     if (Kind == CK_BitCast)
       checkCastAlign();
+
+    if (!checkCastFunctionType(Self, SrcExpr, DestType))
+      Self.Diag(OpRange.getBegin(), diag::warn_cast_function_type)
+          << SrcExpr.get()->getType() << DestType << OpRange;
+
   } else {
     SrcExpr = ExprError();
   }
@@ -2957,6 +3051,10 @@ void CastOperation::CheckCStyleCast() {
     }
   }
 
+  if (!checkCastFunctionType(Self, SrcExpr, DestType))
+    Self.Diag(OpRange.getBegin(), diag::warn_cast_function_type)
+        << SrcType << DestType << OpRange;
+
   DiagnoseCastOfObjCSEL(Self, SrcExpr, DestType);
   DiagnoseCallingConvCast(Self, SrcExpr, DestType, OpRange);
   DiagnoseBadFunctionCast(Self, SrcExpr, DestType);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 4f6cc7ba9122a9ddd4726134575be09d4cba9645..8e7db23a33bc9500218a6e94ad5456cca8f4f09f 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1966,6 +1966,26 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
 
   case Builtin::BI__builtin_matrix_column_major_store:
     return SemaBuiltinMatrixColumnMajorStore(TheCall, TheCallResult);
+
+  case Builtin::BI__builtin_get_device_side_mangled_name: {
+    auto Check = [](CallExpr *TheCall) {
+      if (TheCall->getNumArgs() != 1)
+        return false;
+      auto *DRE = dyn_cast<DeclRefExpr>(TheCall->getArg(0)->IgnoreImpCasts());
+      if (!DRE)
+        return false;
+      auto *D = DRE->getDecl();
+      if (!isa<FunctionDecl>(D) && !isa<VarDecl>(D))
+        return false;
+      return D->hasAttr<CUDAGlobalAttr>() || D->hasAttr<CUDADeviceAttr>() ||
+             D->hasAttr<CUDAConstantAttr>() || D->hasAttr<HIPManagedAttr>();
+    };
+    if (!Check(TheCall)) {
+      Diag(TheCall->getBeginLoc(),
+           diag::err_hip_invalid_args_builtin_mangled_name);
+      return ExprError();
+    }
+  }
   }
 
   // Since the target specific builtins for each arch overlap, only check those
@@ -4507,7 +4527,8 @@ void Sema::CheckArgAlignment(SourceLocation Loc, NamedDecl *FDecl,
 
   // Find expected alignment, and the actual alignment of the passed object.
   // getTypeAlignInChars requires complete types
-  if (ParamTy->isIncompleteType() || ArgTy->isIncompleteType())
+  if (ParamTy->isIncompleteType() || ArgTy->isIncompleteType() ||
+      ParamTy->isUndeducedType() || ArgTy->isUndeducedType())
     return;
 
   CharUnits ParamAlign = Context.getTypeAlignInChars(ParamTy);
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 2feb02bbe4ed5d59a71868353d514dfb274b1893..dc7a67e92827f652d28324d551e335dec2475620 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -381,6 +381,8 @@ public:
 } // namespace
 
 void PreferredTypeBuilder::enterReturn(Sema &S, SourceLocation Tok) {
+  if (!Enabled)
+    return;
   if (isa<BlockDecl>(S.CurContext)) {
     if (sema::BlockScopeInfo *BSI = S.getCurBlock()) {
       ComputeType = nullptr;
@@ -399,6 +401,8 @@ void PreferredTypeBuilder::enterReturn(Sema &S, SourceLocation Tok) {
 }
 
 void PreferredTypeBuilder::enterVariableInit(SourceLocation Tok, Decl *D) {
+  if (!Enabled)
+    return;
   auto *VD = llvm::dyn_cast_or_null<ValueDecl>(D);
   ComputeType = nullptr;
   Type = VD ? VD->getType() : QualType();
@@ -410,6 +414,8 @@ static QualType getDesignatedType(QualType BaseType, const Designation &Desig);
 void PreferredTypeBuilder::enterDesignatedInitializer(SourceLocation Tok,
                                                       QualType BaseType,
                                                       const Designation &D) {
+  if (!Enabled)
+    return;
   ComputeType = nullptr;
   Type = getDesignatedType(BaseType, D);
   ExpectedLoc = Tok;
@@ -417,6 +423,8 @@ void PreferredTypeBuilder::enterDesignatedInitializer(SourceLocation Tok,
 
 void PreferredTypeBuilder::enterFunctionArgument(
     SourceLocation Tok, llvm::function_ref<QualType()> ComputeType) {
+  if (!Enabled)
+    return;
   this->ComputeType = ComputeType;
   Type = QualType();
   ExpectedLoc = Tok;
@@ -424,6 +432,8 @@ void PreferredTypeBuilder::enterFunctionArgument(
 
 void PreferredTypeBuilder::enterParenExpr(SourceLocation Tok,
                                           SourceLocation LParLoc) {
+  if (!Enabled)
+    return;
   // expected type for parenthesized expression does not change.
   if (ExpectedLoc == LParLoc)
     ExpectedLoc = Tok;
@@ -541,6 +551,8 @@ static QualType getPreferredTypeOfUnaryArg(Sema &S, QualType ContextType,
 
 void PreferredTypeBuilder::enterBinary(Sema &S, SourceLocation Tok, Expr *LHS,
                                        tok::TokenKind Op) {
+  if (!Enabled)
+    return;
   ComputeType = nullptr;
   Type = getPreferredTypeOfBinaryRHS(S, LHS, Op);
   ExpectedLoc = Tok;
@@ -548,7 +560,7 @@ void PreferredTypeBuilder::enterBinary(Sema &S, SourceLocation Tok, Expr *LHS,
 
 void PreferredTypeBuilder::enterMemAccess(Sema &S, SourceLocation Tok,
                                           Expr *Base) {
-  if (!Base)
+  if (!Enabled || !Base)
     return;
   // Do we have expected type for Base?
   if (ExpectedLoc != Base->getBeginLoc())
@@ -561,6 +573,8 @@ void PreferredTypeBuilder::enterMemAccess(Sema &S, SourceLocation Tok,
 void PreferredTypeBuilder::enterUnary(Sema &S, SourceLocation Tok,
                                       tok::TokenKind OpKind,
                                       SourceLocation OpLoc) {
+  if (!Enabled)
+    return;
   ComputeType = nullptr;
   Type = getPreferredTypeOfUnaryArg(S, this->get(OpLoc), OpKind);
   ExpectedLoc = Tok;
@@ -568,6 +582,8 @@ void PreferredTypeBuilder::enterUnary(Sema &S, SourceLocation Tok,
 
 void PreferredTypeBuilder::enterSubscript(Sema &S, SourceLocation Tok,
                                           Expr *LHS) {
+  if (!Enabled)
+    return;
   ComputeType = nullptr;
   Type = S.getASTContext().IntTy;
   ExpectedLoc = Tok;
@@ -575,12 +591,16 @@ void PreferredTypeBuilder::enterSubscript(Sema &S, SourceLocation Tok,
 
 void PreferredTypeBuilder::enterTypeCast(SourceLocation Tok,
                                          QualType CastType) {
+  if (!Enabled)
+    return;
   ComputeType = nullptr;
   Type = !CastType.isNull() ? CastType.getCanonicalType() : QualType();
   ExpectedLoc = Tok;
 }
 
 void PreferredTypeBuilder::enterCondition(Sema &S, SourceLocation Tok) {
+  if (!Enabled)
+    return;
   ComputeType = nullptr;
   Type = S.getASTContext().BoolTy;
   ExpectedLoc = Tok;
@@ -5691,8 +5711,9 @@ ProduceSignatureHelp(Sema &SemaRef, Scope *S,
                      unsigned CurrentArg, SourceLocation OpenParLoc) {
   if (Candidates.empty())
     return QualType();
-  SemaRef.CodeCompleter->ProcessOverloadCandidates(
-      SemaRef, CurrentArg, Candidates.data(), Candidates.size(), OpenParLoc);
+  if (SemaRef.getPreprocessor().isCodeCompletionReached())
+    SemaRef.CodeCompleter->ProcessOverloadCandidates(
+        SemaRef, CurrentArg, Candidates.data(), Candidates.size(), OpenParLoc);
   return getParamType(SemaRef, Candidates, CurrentArg);
 }
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 942376971d66c50ef0814e81e58b9ae3726c9579..ceedd8036780e715d3711b1502d2939c14dcaf75 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -11187,6 +11187,25 @@ void Sema::CheckMain(FunctionDecl* FD, const DeclSpec& DS) {
   }
 }
 
+static bool isDefaultStdCall(FunctionDecl *FD, Sema &S) {
+
+  // Default calling convention for main and wmain is __cdecl
+  if (FD->getName() == "main" || FD->getName() == "wmain")
+    return false;
+
+  // Default calling convention for MinGW is __cdecl
+  const llvm::Triple &T = S.Context.getTargetInfo().getTriple();
+  if (T.isWindowsGNUEnvironment())
+    return false;
+
+  // Default calling convention for WinMain, wWinMain and DllMain
+  // is __stdcall on 32 bit Windows
+  if (T.isOSWindows() && T.getArch() == llvm::Triple::x86)
+    return true;
+
+  return false;
+}
+
 void Sema::CheckMSVCRTEntryPoint(FunctionDecl *FD) {
   QualType T = FD->getType();
   assert(T->isFunctionType() && "function decl is not of function type");
@@ -11201,6 +11220,21 @@ void Sema::CheckMSVCRTEntryPoint(FunctionDecl *FD) {
     if (FD->getName() != "DllMain")
       FD->setHasImplicitReturnZero(true);
 
+  // Explicity specified calling conventions are applied to MSVC entry points
+  if (!hasExplicitCallingConv(T)) {
+    if (isDefaultStdCall(FD, *this)) {
+      if (FT->getCallConv() != CC_X86StdCall) {
+        FT = Context.adjustFunctionType(
+            FT, FT->getExtInfo().withCallingConv(CC_X86StdCall));
+        FD->setType(QualType(FT, 0));
+      }
+    } else if (FT->getCallConv() != CC_C) {
+      FT = Context.adjustFunctionType(FT,
+                                      FT->getExtInfo().withCallingConv(CC_C));
+      FD->setType(QualType(FT, 0));
+    }
+  }
+
   if (!FD->isInvalidDecl() && FD->getDescribedFunctionTemplate()) {
     Diag(FD->getLocation(), diag::err_mainlike_template_decl) << FD;
     FD->setInvalidDecl();
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 459343637318d09eeff863d6854a4331cfaaa73a..b39460d33214a08d14693aa81723d03c6ccaaae9 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -390,10 +390,10 @@ appendDiagnostics(const Sema::SemaDiagnosticBuilder &Bldr, T &&ExtraArg,
                            std::forward<DiagnosticArgs>(ExtraArgs)...);
 }
 
-/// Add an attribute {@code AttrType} to declaration {@code D}, provided that
-/// {@code PassesCheck} is true.
-/// Otherwise, emit diagnostic {@code DiagID}, passing in all parameters
-/// specified in {@code ExtraArgs}.
+/// Add an attribute @c AttrType to declaration @c D, provided that
+/// @c PassesCheck is true.
+/// Otherwise, emit diagnostic @c DiagID, passing in all parameters
+/// specified in @c ExtraArgs.
 template <typename AttrType, typename... DiagnosticArgs>
 static void handleSimpleAttributeOrDiagnose(Sema &S, Decl *D,
                                             const AttributeCommonInfo &CI,
@@ -513,16 +513,9 @@ static bool checkRecordDeclForAttr(const RecordDecl *RD) {
 
   // Else check if any base classes have the attribute.
   if (const auto *CRD = dyn_cast<CXXRecordDecl>(RD)) {
-    CXXBasePaths BPaths(false, false);
-    if (CRD->lookupInBases(
-            [](const CXXBaseSpecifier *BS, CXXBasePath &) {
-              const auto &Ty = *BS->getType();
-              // If it's type-dependent, we assume it could have the attribute.
-              if (Ty.isDependentType())
-                return true;
-              return Ty.castAs<RecordType>()->getDecl()->hasAttr<AttrType>();
-            },
-            BPaths, true))
+    if (!CRD->forallBases([](const CXXRecordDecl *Base) {
+          return !Base->hasAttr<AttrType>();
+        }))
       return true;
   }
   return false;
@@ -7371,48 +7364,6 @@ static void handleOpenCLNoSVMAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
                                                                    << "2.0";
 }
 
-/// Handles semantic checking for features that are common to all attributes,
-/// such as checking whether a parameter was properly specified, or the correct
-/// number of arguments were passed, etc.
-static bool handleCommonAttributeFeatures(Sema &S, Decl *D,
-                                          const ParsedAttr &AL) {
-  // Several attributes carry different semantics than the parsing requires, so
-  // those are opted out of the common argument checks.
-  //
-  // We also bail on unknown and ignored attributes because those are handled
-  // as part of the target-specific handling logic.
-  if (AL.getKind() == ParsedAttr::UnknownAttribute)
-    return false;
-  // Check whether the attribute requires specific language extensions to be
-  // enabled.
-  if (!AL.diagnoseLangOpts(S))
-    return true;
-  // Check whether the attribute appertains to the given subject.
-  if (!AL.diagnoseAppertainsTo(S, D))
-    return true;
-  if (AL.hasCustomParsing())
-    return false;
-
-  if (AL.getMinArgs() == AL.getMaxArgs()) {
-    // If there are no optional arguments, then checking for the argument count
-    // is trivial.
-    if (!AL.checkExactlyNumArgs(S, AL.getMinArgs()))
-      return true;
-  } else {
-    // There are optional arguments, so checking is slightly more involved.
-    if (AL.getMinArgs() && !AL.checkAtLeastNumArgs(S, AL.getMinArgs()))
-      return true;
-    else if (!AL.hasVariadicArg() && AL.getMaxArgs() &&
-             !AL.checkAtMostNumArgs(S, AL.getMaxArgs()))
-      return true;
-  }
-
-  if (S.CheckAttrTarget(AL))
-    return true;
-
-  return false;
-}
-
 static void handleOpenCLAccessAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (D->isInvalidDecl())
     return;
@@ -7766,7 +7717,7 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D,
     return;
   }
 
-  if (handleCommonAttributeFeatures(S, D, AL))
+  if (S.checkCommonAttributeFeatures(D, AL))
     return;
 
   switch (AL.getKind()) {
@@ -7778,6 +7729,9 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D,
       assert(AL.isTypeAttr() && "Non-type attribute not handled");
       break;
     }
+    // N.B., ClangAttrEmitter.cpp emits a diagnostic helper that ensures a
+    // statement attribute is not written on a declaration, but this code is
+    // needed for attributes in Attr.td that do not list any subjects.
     S.Diag(AL.getLoc(), diag::err_stmt_attribute_invalid_on_decl)
         << AL << D->getLocation();
     break;
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 65648747723b6f3af51374b00be7328166db0d79..80084f4209b735f788fd938c8dfd98da4a4e3e95 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -254,8 +254,7 @@ void Sema::ImplicitExceptionSpecification::CalledStmt(Stmt *S) {
     ComputedEST = EST_None;
 }
 
-ExprResult Sema::ConvertParamDefaultArgument(const ParmVarDecl *Param,
-                                             Expr *Arg,
+ExprResult Sema::ConvertParamDefaultArgument(ParmVarDecl *Param, Expr *Arg,
                                              SourceLocation EqualLoc) {
   if (RequireCompleteType(Param->getLocation(), Param->getType(),
                           diag::err_typecheck_decl_incomplete_type))
@@ -4140,13 +4139,9 @@ ValueDecl *Sema::tryLookupCtorInitMemberDecl(CXXRecordDecl *ClassDecl,
                                              IdentifierInfo *MemberOrBase) {
   if (SS.getScopeRep() || TemplateTypeTy)
     return nullptr;
-  DeclContext::lookup_result Result = ClassDecl->lookup(MemberOrBase);
-  if (Result.empty())
-    return nullptr;
-  ValueDecl *Member;
-  if ((Member = dyn_cast<FieldDecl>(Result.front())) ||
-      (Member = dyn_cast<IndirectFieldDecl>(Result.front())))
-    return Member;
+  for (auto *D : ClassDecl->lookup(MemberOrBase))
+    if (isa<FieldDecl>(D) || isa<IndirectFieldDecl>(D))
+      return cast<ValueDecl>(D);
   return nullptr;
 }
 
@@ -5249,6 +5244,20 @@ static const void *GetKeyForMember(ASTContext &Context,
   return Member->getAnyMember()->getCanonicalDecl();
 }
 
+static void AddInitializerToDiag(const Sema::SemaDiagnosticBuilder &Diag,
+                                 const CXXCtorInitializer *Previous,
+                                 const CXXCtorInitializer *Current) {
+  if (Previous->isAnyMemberInitializer())
+    Diag << 0 << Previous->getAnyMember();
+  else
+    Diag << 1 << Previous->getTypeSourceInfo()->getType();
+
+  if (Current->isAnyMemberInitializer())
+    Diag << 0 << Current->getAnyMember();
+  else
+    Diag << 1 << Current->getTypeSourceInfo()->getType();
+}
+
 static void DiagnoseBaseOrMemInitializerOrder(
     Sema &SemaRef, const CXXConstructorDecl *Constructor,
     ArrayRef<CXXCtorInitializer *> Inits) {
@@ -5298,10 +5307,15 @@ static void DiagnoseBaseOrMemInitializerOrder(
   unsigned NumIdealInits = IdealInitKeys.size();
   unsigned IdealIndex = 0;
 
-  CXXCtorInitializer *PrevInit = nullptr;
+  // Track initializers that are in an incorrect order for either a warning or
+  // note if multiple ones occur.
+  SmallVector<unsigned> WarnIndexes;
+  // Correlates the index of an initializer in the init-list to the index of
+  // the field/base in the class.
+  SmallVector<std::pair<unsigned, unsigned>, 32> CorrelatedInitOrder;
+
   for (unsigned InitIndex = 0; InitIndex != Inits.size(); ++InitIndex) {
-    CXXCtorInitializer *Init = Inits[InitIndex];
-    const void *InitKey = GetKeyForMember(SemaRef.Context, Init);
+    const void *InitKey = GetKeyForMember(SemaRef.Context, Inits[InitIndex]);
 
     // Scan forward to try to find this initializer in the idealized
     // initializers list.
@@ -5312,20 +5326,8 @@ static void DiagnoseBaseOrMemInitializerOrder(
     // If we didn't find this initializer, it must be because we
     // scanned past it on a previous iteration.  That can only
     // happen if we're out of order;  emit a warning.
-    if (IdealIndex == NumIdealInits && PrevInit) {
-      Sema::SemaDiagnosticBuilder D =
-        SemaRef.Diag(PrevInit->getSourceLocation(),
-                     diag::warn_initializer_out_of_order);
-
-      if (PrevInit->isAnyMemberInitializer())
-        D << 0 << PrevInit->getAnyMember()->getDeclName();
-      else
-        D << 1 << PrevInit->getTypeSourceInfo()->getType();
-
-      if (Init->isAnyMemberInitializer())
-        D << 0 << Init->getAnyMember()->getDeclName();
-      else
-        D << 1 << Init->getTypeSourceInfo()->getType();
+    if (IdealIndex == NumIdealInits && InitIndex) {
+      WarnIndexes.push_back(InitIndex);
 
       // Move back to the initializer's location in the ideal list.
       for (IdealIndex = 0; IdealIndex != NumIdealInits; ++IdealIndex)
@@ -5335,8 +5337,54 @@ static void DiagnoseBaseOrMemInitializerOrder(
       assert(IdealIndex < NumIdealInits &&
              "initializer not found in initializer list");
     }
+    CorrelatedInitOrder.emplace_back(IdealIndex, InitIndex);
+  }
 
-    PrevInit = Init;
+  if (WarnIndexes.empty())
+    return;
+
+  // Sort based on the ideal order, first in the pair.
+  llvm::sort(CorrelatedInitOrder,
+             [](auto &LHS, auto &RHS) { return LHS.first < RHS.first; });
+
+  // Introduce a new scope as SemaDiagnosticBuilder needs to be destroyed to
+  // emit the diagnostic before we can try adding notes.
+  {
+    Sema::SemaDiagnosticBuilder D = SemaRef.Diag(
+        Inits[WarnIndexes.front() - 1]->getSourceLocation(),
+        WarnIndexes.size() == 1 ? diag::warn_initializer_out_of_order
+                                : diag::warn_some_initializers_out_of_order);
+
+    for (unsigned I = 0; I < CorrelatedInitOrder.size(); ++I) {
+      if (CorrelatedInitOrder[I].second == I)
+        continue;
+      // Ideally we would be using InsertFromRange here, but clang doesn't
+      // appear to handle InsertFromRange correctly when the source range is
+      // modified by another fix-it.
+      D << FixItHint::CreateReplacement(
+          Inits[I]->getSourceRange(),
+          Lexer::getSourceText(
+              CharSourceRange::getTokenRange(
+                  Inits[CorrelatedInitOrder[I].second]->getSourceRange()),
+              SemaRef.getSourceManager(), SemaRef.getLangOpts()));
+    }
+
+    // If there is only 1 item out of order, the warning expects the name and
+    // type of each being added to it.
+    if (WarnIndexes.size() == 1) {
+      AddInitializerToDiag(D, Inits[WarnIndexes.front() - 1],
+                           Inits[WarnIndexes.front()]);
+      return;
+    }
+  }
+  // More than 1 item to warn, create notes letting the user know which ones
+  // are bad.
+  for (unsigned WarnIndex : WarnIndexes) {
+    const clang::CXXCtorInitializer *PrevInit = Inits[WarnIndex - 1];
+    auto D = SemaRef.Diag(PrevInit->getSourceLocation(),
+                          diag::note_initializer_out_of_order);
+    AddInitializerToDiag(D, PrevInit, Inits[WarnIndex]);
+    D << PrevInit->getSourceRange();
   }
 }
 
@@ -5404,7 +5452,7 @@ bool CheckRedundantUnionInit(Sema &S,
 
   return false;
 }
-}
+} // namespace
 
 /// ActOnMemInitializers - Handle the member initializers for a constructor.
 void Sema::ActOnMemInitializers(Decl *ConstructorDecl,
@@ -9682,9 +9730,9 @@ public:
 
     bool foundSameNameMethod = false;
     SmallVector<CXXMethodDecl *, 8> overloadedMethods;
-    for (Path.Decls = BaseRecord->lookup(Name); !Path.Decls.empty();
-         Path.Decls = Path.Decls.slice(1)) {
-      NamedDecl *D = Path.Decls.front();
+    for (Path.Decls = BaseRecord->lookup(Name).begin();
+         Path.Decls != DeclContext::lookup_iterator(); ++Path.Decls) {
+      NamedDecl *D = *Path.Decls;
       if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(D)) {
         MD = MD->getCanonicalDecl();
         foundSameNameMethod = true;
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 3ada5729c27a2a532678c85f887a4b8f4033fcf9..a9a487bd47a38e59811311667cc8ca7b231c066d 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1486,6 +1486,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
   case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
   case Stmt::OMPTeamsDistributeSimdDirectiveClass:
+  case Stmt::OMPInteropDirectiveClass:
   case Stmt::ReturnStmtClass:
   case Stmt::SEHExceptStmtClass:
   case Stmt::SEHFinallyStmtClass:
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 185f3080458b8857f34a09f648f981465b500e2c..33b7409e927ad058ea430a791cf98a185255e901 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -6523,24 +6523,30 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
                                ExecConfig, IsExecConfig);
 }
 
-/// ActOnAsTypeExpr - create a new asType (bitcast) from the arguments.
+/// Parse a __builtin_astype expression.
 ///
 /// __builtin_astype( value, dst type )
 ///
 ExprResult Sema::ActOnAsTypeExpr(Expr *E, ParsedType ParsedDestTy,
                                  SourceLocation BuiltinLoc,
                                  SourceLocation RParenLoc) {
+  QualType DstTy = GetTypeFromParser(ParsedDestTy);
+  return BuildAsTypeExpr(E, DstTy, BuiltinLoc, RParenLoc);
+}
+
+/// Create a new AsTypeExpr node (bitcast) from the arguments.
+ExprResult Sema::BuildAsTypeExpr(Expr *E, QualType DestTy,
+                                 SourceLocation BuiltinLoc,
+                                 SourceLocation RParenLoc) {
   ExprValueKind VK = VK_RValue;
   ExprObjectKind OK = OK_Ordinary;
-  QualType DstTy = GetTypeFromParser(ParsedDestTy);
   QualType SrcTy = E->getType();
-  if (Context.getTypeSize(DstTy) != Context.getTypeSize(SrcTy))
-    return ExprError(Diag(BuiltinLoc,
-                          diag::err_invalid_astype_of_different_size)
-                     << DstTy
-                     << SrcTy
-                     << E->getSourceRange());
-  return new (Context) AsTypeExpr(E, DstTy, VK, OK, BuiltinLoc, RParenLoc);
+  if (!SrcTy->isDependentType() &&
+      Context.getTypeSize(DestTy) != Context.getTypeSize(SrcTy))
+    return ExprError(
+        Diag(BuiltinLoc, diag::err_invalid_astype_of_different_size)
+        << DestTy << SrcTy << E->getSourceRange());
+  return new (Context) AsTypeExpr(E, DestTy, VK, OK, BuiltinLoc, RParenLoc);
 }
 
 /// ActOnConvertVectorExpr - create a new convert-vector expression from the
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index faf71baf3d1d27d6866a14d1783213353ff01d0d..ec7e4722ea4e565fa00b0f7b5f167da4a8a45afc 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -24,6 +24,7 @@
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/APInt.h"
+#include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
@@ -3281,10 +3282,7 @@ InitializedEntity::InitializeBase(ASTContext &Context,
   InitializedEntity Result;
   Result.Kind = EK_Base;
   Result.Parent = Parent;
-  Result.Base = reinterpret_cast<uintptr_t>(Base);
-  if (IsInheritedVirtualBase)
-    Result.Base |= 0x01;
-
+  Result.Base = {Base, IsInheritedVirtualBase};
   Result.Type = Base->getType();
   return Result;
 }
@@ -3293,7 +3291,7 @@ DeclarationName InitializedEntity::getName() const {
   switch (getKind()) {
   case EK_Parameter:
   case EK_Parameter_CF_Audited: {
-    ParmVarDecl *D = reinterpret_cast<ParmVarDecl*>(Parameter & ~0x1);
+    ParmVarDecl *D = Parameter.getPointer();
     return (D ? D->getDeclName() : DeclarationName());
   }
 
@@ -3336,7 +3334,7 @@ ValueDecl *InitializedEntity::getDecl() const {
 
   case EK_Parameter:
   case EK_Parameter_CF_Audited:
-    return reinterpret_cast<ParmVarDecl*>(Parameter & ~0x1);
+    return Parameter.getPointer();
 
   case EK_Result:
   case EK_StmtExprResult:
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index b6bf88203745b45671162704cbd641dbb9c9b59a..fef96b2eb11fd4dc39859100f714dff9bc594866 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -638,8 +638,8 @@ void LookupResult::resolveKind() {
 void LookupResult::addDeclsFromBasePaths(const CXXBasePaths &P) {
   CXXBasePaths::const_paths_iterator I, E;
   for (I = P.begin(), E = P.end(); I != E; ++I)
-    for (DeclContext::lookup_iterator DI = I->Decls.begin(),
-         DE = I->Decls.end(); DI != DE; ++DI)
+    for (DeclContext::lookup_iterator DI = I->Decls, DE = DI.end(); DI != DE;
+         ++DI)
       addDecl(*DI);
 }
 
@@ -2230,9 +2230,9 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     CXXRecordDecl *BaseRecord = Specifier->getType()->getAsCXXRecordDecl();
     // Drop leading non-matching lookup results from the declaration list so
     // we don't need to consider them again below.
-    for (Path.Decls = BaseRecord->lookup(Name); !Path.Decls.empty();
-         Path.Decls = Path.Decls.slice(1)) {
-      if (Path.Decls.front()->isInIdentifierNamespace(IDNS))
+    for (Path.Decls = BaseRecord->lookup(Name).begin();
+         Path.Decls != Path.Decls.end(); ++Path.Decls) {
+      if ((*Path.Decls)->isInIdentifierNamespace(IDNS))
         return true;
     }
     return false;
@@ -2256,9 +2256,9 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
   AccessSpecifier SubobjectAccess = AS_none;
 
   // Check whether the given lookup result contains only static members.
-  auto HasOnlyStaticMembers = [&](DeclContextLookupResult Result) {
-    for (NamedDecl *ND : Result)
-      if (ND->isInIdentifierNamespace(IDNS) && ND->isCXXInstanceMember())
+  auto HasOnlyStaticMembers = [&](DeclContext::lookup_iterator Result) {
+    for (DeclContext::lookup_iterator I = Result, E = I.end(); I != E; ++I)
+      if ((*I)->isInIdentifierNamespace(IDNS) && (*I)->isCXXInstanceMember())
         return false;
     return true;
   };
@@ -2267,8 +2267,8 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
 
   // Determine whether two sets of members contain the same members, as
   // required by C++ [class.member.lookup]p6.
-  auto HasSameDeclarations = [&](DeclContextLookupResult A,
-                                 DeclContextLookupResult B) {
+  auto HasSameDeclarations = [&](DeclContext::lookup_iterator A,
+                                 DeclContext::lookup_iterator B) {
     using Iterator = DeclContextLookupResult::iterator;
     using Result = const void *;
 
@@ -2305,7 +2305,7 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
 
     // We'll often find the declarations are in the same order. Handle this
     // case (and the special case of only one declaration) efficiently.
-    Iterator AIt = A.begin(), BIt = B.begin(), AEnd = A.end(), BEnd = B.end();
+    Iterator AIt = A, BIt = B, AEnd, BEnd;
     while (true) {
       Result AResult = Next(AIt, AEnd);
       Result BResult = Next(BIt, BEnd);
@@ -2388,10 +2388,11 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
 
   // Lookup in a base class succeeded; return these results.
 
-  for (auto *D : Paths.front().Decls) {
+  for (DeclContext::lookup_iterator I = Paths.front().Decls, E = I.end();
+       I != E; ++I) {
     AccessSpecifier AS = CXXRecordDecl::MergeAccess(SubobjectAccess,
-                                                    D->getAccess());
-    if (NamedDecl *ND = R.getAcceptableDecl(D))
+                                                    (*I)->getAccess());
+    if (NamedDecl *ND = R.getAcceptableDecl(*I))
       R.addDecl(ND, AS);
   }
   R.resolveKind();
@@ -2534,7 +2535,7 @@ void Sema::DiagnoseAmbiguousLookup(LookupResult &Result) {
       << Name << SubobjectType << getAmbiguousPathsDisplayString(*Paths)
       << LookupRange;
 
-    DeclContext::lookup_iterator Found = Paths->front().Decls.begin();
+    DeclContext::lookup_iterator Found = Paths->front().Decls;
     while (isa<CXXMethodDecl>(*Found) &&
            cast<CXXMethodDecl>(*Found)->isStatic())
       ++Found;
@@ -2552,7 +2553,7 @@ void Sema::DiagnoseAmbiguousLookup(LookupResult &Result) {
     for (CXXBasePaths::paths_iterator Path = Paths->begin(),
                                       PathEnd = Paths->end();
          Path != PathEnd; ++Path) {
-      const NamedDecl *D = Path->Decls.front();
+      const NamedDecl *D = *Path->Decls;
       if (!D->isInIdentifierNamespace(Result.getIdentifierNamespace()))
         continue;
       if (DeclsPrinted.insert(D).second) {
diff --git a/clang/lib/Sema/SemaObjCProperty.cpp b/clang/lib/Sema/SemaObjCProperty.cpp
index fdc30fe6f6576f631b3507351862885524dc6003..db999270219cb8cb7a316c95ec6ca112c84d8568 100644
--- a/clang/lib/Sema/SemaObjCProperty.cpp
+++ b/clang/lib/Sema/SemaObjCProperty.cpp
@@ -112,12 +112,10 @@ CheckPropertyAgainstProtocol(Sema &S, ObjCPropertyDecl *Prop,
     return;
 
   // Look for a property with the same name.
-  DeclContext::lookup_result R = Proto->lookup(Prop->getDeclName());
-  for (unsigned I = 0, N = R.size(); I != N; ++I) {
-    if (ObjCPropertyDecl *ProtoProp = dyn_cast<ObjCPropertyDecl>(R[I])) {
-      S.DiagnosePropertyMismatch(Prop, ProtoProp, Proto->getIdentifier(), true);
-      return;
-    }
+  if (ObjCPropertyDecl *ProtoProp =
+      Proto->lookup(Prop->getDeclName()).find_first<ObjCPropertyDecl>()) {
+    S.DiagnosePropertyMismatch(Prop, ProtoProp, Proto->getIdentifier(), true);
+    return;
   }
 
   // Check this property against any protocols we inherit.
@@ -233,18 +231,13 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
     bool FoundInSuper = false;
     ObjCInterfaceDecl *CurrentInterfaceDecl = IFace;
     while (ObjCInterfaceDecl *Super = CurrentInterfaceDecl->getSuperClass()) {
-      DeclContext::lookup_result R = Super->lookup(Res->getDeclName());
-      for (unsigned I = 0, N = R.size(); I != N; ++I) {
-        if (ObjCPropertyDecl *SuperProp = dyn_cast<ObjCPropertyDecl>(R[I])) {
-          DiagnosePropertyMismatch(Res, SuperProp, Super->getIdentifier(), false);
-          FoundInSuper = true;
-          break;
-        }
-      }
-      if (FoundInSuper)
+      if (ObjCPropertyDecl *SuperProp =
+          Super->lookup(Res->getDeclName()).find_first<ObjCPropertyDecl>()) {
+        DiagnosePropertyMismatch(Res, SuperProp, Super->getIdentifier(), false);
+        FoundInSuper = true;
         break;
-      else
-        CurrentInterfaceDecl = Super;
+      }
+      CurrentInterfaceDecl = Super;
     }
 
     if (FoundInSuper) {
@@ -1149,14 +1142,13 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       // redeclared 'readwrite', then no warning is to be issued.
       for (auto *Ext : IDecl->known_extensions()) {
         DeclContext::lookup_result R = Ext->lookup(property->getDeclName());
-        if (!R.empty())
-          if (ObjCPropertyDecl *ExtProp = dyn_cast<ObjCPropertyDecl>(R[0])) {
-            PIkind = ExtProp->getPropertyAttributesAsWritten();
-            if (PIkind & ObjCPropertyAttribute::kind_readwrite) {
-              ReadWriteProperty = true;
-              break;
-            }
+        if (auto *ExtProp = R.find_first<ObjCPropertyDecl>()) {
+          PIkind = ExtProp->getPropertyAttributesAsWritten();
+          if (PIkind & ObjCPropertyAttribute::kind_readwrite) {
+            ReadWriteProperty = true;
+            break;
           }
+        }
       }
 
       if (!ReadWriteProperty) {
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 2da772d3f41a359f4697d28b049a8ffdaf66ec0d..54c824c4a7599587201a0172a37ab6a66dfe0dd7 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6115,6 +6115,11 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
     if (LangOpts.OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
+  case OMPD_interop:
+    assert(AStmt == nullptr &&
+           "No associated statement allowed for 'omp interop' directive");
+    Res = ActOnOpenMPInteropDirective(ClausesWithImplicit, StartLoc, EndLoc);
+    break;
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_threadprivate:
@@ -13347,6 +13352,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
       CaptureRegion = OMPD_task;
       break;
     case OMPD_target_data:
+    case OMPD_interop:
       // Do not capture device-clause expressions.
       break;
     case OMPD_teams_distribute_parallel_for:
@@ -14435,7 +14441,9 @@ OMPClause *Sema::ActOnOpenMPClause(OpenMPClauseKind Kind,
     Res = ActOnOpenMPDynamicAllocatorsClause(StartLoc, EndLoc);
     break;
   case OMPC_destroy:
-    Res = ActOnOpenMPDestroyClause(StartLoc, EndLoc);
+    Res = ActOnOpenMPDestroyClause(/*InteropVar=*/nullptr, StartLoc,
+                                   /*LParenLoc=*/SourceLocation(),
+                                   /*VarLoc=*/SourceLocation(), EndLoc);
     break;
   case OMPC_if:
   case OMPC_final:
@@ -14593,9 +14601,187 @@ OMPClause *Sema::ActOnOpenMPDynamicAllocatorsClause(SourceLocation StartLoc,
   return new (Context) OMPDynamicAllocatorsClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPDestroyClause(SourceLocation StartLoc,
+StmtResult Sema::ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
+                                             SourceLocation StartLoc,
+                                             SourceLocation EndLoc) {
+
+  // OpenMP 5.1 [2.15.1, interop Construct, Restrictions]
+  // At least one action-clause must appear on a directive.
+  if (!hasClauses(Clauses, OMPC_init, OMPC_use, OMPC_destroy, OMPC_nowait)) {
+    StringRef Expected = "'init', 'use', 'destroy', or 'nowait'";
+    Diag(StartLoc, diag::err_omp_no_clause_for_directive)
+        << Expected << getOpenMPDirectiveName(OMPD_interop);
+    return StmtError();
+  }
+
+  // OpenMP 5.1 [2.15.1, interop Construct, Restrictions]
+  // A depend clause can only appear on the directive if a targetsync
+  // interop-type is present or the interop-var was initialized with
+  // the targetsync interop-type.
+
+  // If there is any 'init' clause diagnose if there is no 'init' clause with
+  // interop-type of 'targetsync'. Cases involving other directives cannot be
+  // diagnosed.
+  const OMPDependClause *DependClause = nullptr;
+  bool HasInitClause = false;
+  bool IsTargetSync = false;
+  for (const OMPClause *C : Clauses) {
+    if (IsTargetSync)
+      break;
+    if (const auto *InitClause = dyn_cast<OMPInitClause>(C)) {
+      HasInitClause = true;
+      if (InitClause->getIsTargetSync())
+        IsTargetSync = true;
+    } else if (const auto *DC = dyn_cast<OMPDependClause>(C)) {
+      DependClause = DC;
+    }
+  }
+  if (DependClause && HasInitClause && !IsTargetSync) {
+    Diag(DependClause->getBeginLoc(), diag::err_omp_interop_bad_depend_clause);
+    return StmtError();
+  }
+
+  // OpenMP 5.1 [2.15.1, interop Construct, Restrictions]
+  // Each interop-var may be specified for at most one action-clause of each
+  // interop construct.
+  llvm::SmallPtrSet<const VarDecl *, 4> InteropVars;
+  for (const OMPClause *C : Clauses) {
+    OpenMPClauseKind ClauseKind = C->getClauseKind();
+    const DeclRefExpr *DRE = nullptr;
+    SourceLocation VarLoc;
+
+    if (ClauseKind == OMPC_init) {
+      const auto *IC = cast<OMPInitClause>(C);
+      VarLoc = IC->getVarLoc();
+      DRE = dyn_cast_or_null<DeclRefExpr>(IC->getInteropVar());
+    } else if (ClauseKind == OMPC_use) {
+      const auto *UC = cast<OMPUseClause>(C);
+      VarLoc = UC->getVarLoc();
+      DRE = dyn_cast_or_null<DeclRefExpr>(UC->getInteropVar());
+    } else if (ClauseKind == OMPC_destroy) {
+      const auto *DC = cast<OMPDestroyClause>(C);
+      VarLoc = DC->getVarLoc();
+      DRE = dyn_cast_or_null<DeclRefExpr>(DC->getInteropVar());
+    }
+
+    if (!DRE)
+      continue;
+
+    if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
+      if (!InteropVars.insert(VD->getCanonicalDecl()).second) {
+        Diag(VarLoc, diag::err_omp_interop_var_multiple_actions) << VD;
+        return StmtError();
+      }
+    }
+  }
+
+  return OMPInteropDirective::Create(Context, StartLoc, EndLoc, Clauses);
+}
+
+static bool isValidInteropVariable(Sema &SemaRef, Expr *InteropVarExpr,
+                                   SourceLocation VarLoc,
+                                   OpenMPClauseKind Kind) {
+  if (InteropVarExpr->isValueDependent() || InteropVarExpr->isTypeDependent() ||
+      InteropVarExpr->isInstantiationDependent() ||
+      InteropVarExpr->containsUnexpandedParameterPack())
+    return true;
+
+  const auto *DRE = dyn_cast<DeclRefExpr>(InteropVarExpr);
+  if (!DRE || !isa<VarDecl>(DRE->getDecl())) {
+    SemaRef.Diag(VarLoc, diag::err_omp_interop_variable_expected) << 0;
+    return false;
+  }
+
+  // Interop variable should be of type omp_interop_t.
+  bool HasError = false;
+  QualType InteropType;
+  LookupResult Result(SemaRef, &SemaRef.Context.Idents.get("omp_interop_t"),
+                      VarLoc, Sema::LookupOrdinaryName);
+  if (SemaRef.LookupName(Result, SemaRef.getCurScope())) {
+    NamedDecl *ND = Result.getFoundDecl();
+    if (const auto *TD = dyn_cast<TypeDecl>(ND)) {
+      InteropType = QualType(TD->getTypeForDecl(), 0);
+    } else {
+      HasError = true;
+    }
+  } else {
+    HasError = true;
+  }
+
+  if (HasError) {
+    SemaRef.Diag(VarLoc, diag::err_omp_implied_type_not_found)
+        << "omp_interop_t";
+    return false;
+  }
+
+  QualType VarType = InteropVarExpr->getType().getUnqualifiedType();
+  if (!SemaRef.Context.hasSameType(InteropType, VarType)) {
+    SemaRef.Diag(VarLoc, diag::err_omp_interop_variable_wrong_type);
+    return false;
+  }
+
+  // OpenMP 5.1 [2.15.1, interop Construct, Restrictions]
+  // The interop-var passed to init or destroy must be non-const.
+  if ((Kind == OMPC_init || Kind == OMPC_destroy) &&
+      isConstNotMutableType(SemaRef, InteropVarExpr->getType())) {
+    SemaRef.Diag(VarLoc, diag::err_omp_interop_variable_expected)
+        << /*non-const*/ 1;
+    return false;
+  }
+  return true;
+}
+
+OMPClause *
+Sema::ActOnOpenMPInitClause(Expr *InteropVar, ArrayRef<Expr *> PrefExprs,
+                            bool IsTarget, bool IsTargetSync,
+                            SourceLocation StartLoc, SourceLocation LParenLoc,
+                            SourceLocation VarLoc, SourceLocation EndLoc) {
+
+  if (!isValidInteropVariable(*this, InteropVar, VarLoc, OMPC_init))
+    return nullptr;
+
+  // Check prefer_type values.  These foreign-runtime-id values are either
+  // string literals or constant integral expressions.
+  for (const Expr *E : PrefExprs) {
+    if (E->isValueDependent() || E->isTypeDependent() ||
+        E->isInstantiationDependent() || E->containsUnexpandedParameterPack())
+      continue;
+    if (E->isIntegerConstantExpr(Context))
+      continue;
+    if (isa<StringLiteral>(E))
+      continue;
+    Diag(E->getExprLoc(), diag::err_omp_interop_prefer_type);
+    return nullptr;
+  }
+
+  return OMPInitClause::Create(Context, InteropVar, PrefExprs, IsTarget,
+                               IsTargetSync, StartLoc, LParenLoc, VarLoc,
+                               EndLoc);
+}
+
+OMPClause *Sema::ActOnOpenMPUseClause(Expr *InteropVar, SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation VarLoc,
+                                      SourceLocation EndLoc) {
+
+  if (!isValidInteropVariable(*this, InteropVar, VarLoc, OMPC_use))
+    return nullptr;
+
+  return new (Context)
+      OMPUseClause(InteropVar, StartLoc, LParenLoc, VarLoc, EndLoc);
+}
+
+OMPClause *Sema::ActOnOpenMPDestroyClause(Expr *InteropVar,
+                                          SourceLocation StartLoc,
+                                          SourceLocation LParenLoc,
+                                          SourceLocation VarLoc,
                                           SourceLocation EndLoc) {
-  return new (Context) OMPDestroyClause(StartLoc, EndLoc);
+  if (InteropVar &&
+      !isValidInteropVariable(*this, InteropVar, VarLoc, OMPC_destroy))
+    return nullptr;
+
+  return new (Context)
+      OMPDestroyClause(InteropVar, StartLoc, LParenLoc, VarLoc, EndLoc);
 }
 
 OMPClause *Sema::ActOnOpenMPVarListClause(
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index e25d699315386a549ff40e3bc9d7e81f07b219b5..ceba83bcd814e1c70a35457340a3e48d6be9bbf8 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -3000,6 +3000,12 @@ Sema::ActOnContinueStmt(SourceLocation ContinueLoc, Scope *CurScope) {
     // C99 6.8.6.2p1: A break shall appear only in or as a loop body.
     return StmtError(Diag(ContinueLoc, diag::err_continue_not_in_loop));
   }
+  if (S->getFlags() & Scope::ConditionVarScope) {
+    // We cannot 'continue;' from within a statement expression in the
+    // initializer of a condition variable because we would jump past the
+    // initialization of that variable.
+    return StmtError(Diag(ContinueLoc, diag::err_continue_from_cond_var_init));
+  }
   CheckJumpOutOfSEHFinally(*this, ContinueLoc, *S);
 
   return new (Context) ContinueStmt(ContinueLoc);
@@ -3086,24 +3092,30 @@ bool Sema::isCopyElisionCandidate(QualType ReturnType, const VarDecl *VD,
   if (VD->hasAttr<BlocksAttr>())
     return false;
 
-  // ...non-volatile...
-  if (VD->getType().isVolatileQualified())
-    return false;
-
-  // C++20 [class.copy.elision]p3:
-  // ...rvalue reference to a non-volatile...
-  if (VD->getType()->isRValueReferenceType() &&
-      (!(CESK & CES_AllowRValueReferenceType) ||
-       VD->getType().getNonReferenceType().isVolatileQualified()))
+  if (VDType->isObjectType()) {
+    // C++17 [class.copy.elision]p3:
+    // ...non-volatile automatic object...
+    if (VDType.isVolatileQualified())
+      return false;
+  } else if (VDType->isRValueReferenceType()) {
+    // C++20 [class.copy.elision]p3:
+    // ...either a non-volatile object or an rvalue reference to a non-volatile object type...
+    if (!(CESK & CES_AllowRValueReferenceType))
+      return false;
+    QualType VDReferencedType = VDType.getNonReferenceType();
+    if (VDReferencedType.isVolatileQualified() || !VDReferencedType->isObjectType())
+      return false;
+  } else {
     return false;
+  }
 
   if (CESK & CES_AllowDifferentTypes)
     return true;
 
   // Variables with higher required alignment than their type's ABI
   // alignment cannot use NRVO.
-  if (!VD->getType()->isDependentType() && VD->hasAttr<AlignedAttr>() &&
-      Context.getDeclAlign(VD) > Context.getTypeAlignInChars(VD->getType()))
+  if (!VDType->isDependentType() && VD->hasAttr<AlignedAttr>() &&
+      Context.getDeclAlign(VD) > Context.getTypeAlignInChars(VDType))
     return false;
 
   return true;
diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp
index 86a09c42863f2f817384be93370cd9074e09ec85..cb90a03aa20e0ca0620a21063dd40f3d120e95d1 100644
--- a/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/clang/lib/Sema/SemaStmtAttr.cpp
@@ -26,14 +26,12 @@ using namespace sema;
 static Attr *handleFallThroughAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                                    SourceRange Range) {
   FallThroughAttr Attr(S.Context, A);
-  if (!isa<NullStmt>(St)) {
+  if (isa<SwitchCase>(St)) {
     S.Diag(A.getRange().getBegin(), diag::err_fallthrough_attr_wrong_target)
-        << Attr.getSpelling() << St->getBeginLoc();
-    if (isa<SwitchCase>(St)) {
-      SourceLocation L = S.getLocForEndOfToken(Range.getEnd());
-      S.Diag(L, diag::note_fallthrough_insert_semi_fixit)
-          << FixItHint::CreateInsertion(L, ";");
-    }
+        << A << St->getBeginLoc();
+    SourceLocation L = S.getLocForEndOfToken(Range.getEnd());
+    S.Diag(L, diag::note_fallthrough_insert_semi_fixit)
+        << FixItHint::CreateInsertion(L, ";");
     return nullptr;
   }
   auto *FnScope = S.getCurFunction();
@@ -54,11 +52,6 @@ static Attr *handleFallThroughAttr(Sema &S, Stmt *St, const ParsedAttr &A,
 
 static Attr *handleSuppressAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                                 SourceRange Range) {
-  if (A.getNumArgs() < 1) {
-    S.Diag(A.getLoc(), diag::err_attribute_too_few_arguments) << A << 1;
-    return nullptr;
-  }
-
   std::vector<StringRef> DiagnosticIdentifiers;
   for (unsigned I = 0, E = A.getNumArgs(); I != E; ++I) {
     StringRef RuleName;
@@ -88,10 +81,10 @@ static Attr *handleLoopHintAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                  PragmaNameLoc->Ident->getName())
           .Default("clang loop");
 
-  if (St->getStmtClass() != Stmt::DoStmtClass &&
-      St->getStmtClass() != Stmt::ForStmtClass &&
-      St->getStmtClass() != Stmt::CXXForRangeStmtClass &&
-      St->getStmtClass() != Stmt::WhileStmtClass) {
+  // This could be handled automatically by adding a Subjects definition in
+  // Attr.td, but that would make the diagnostic behavior worse in this case
+  // because the user spells this attribute as a pragma.
+  if (!isa<DoStmt, ForStmt, CXXForRangeStmt, WhileStmt>(St)) {
     std::string Pragma = "#pragma " + std::string(PragmaName);
     S.Diag(St->getBeginLoc(), diag::err_pragma_loop_precedes_nonloop) << Pragma;
     return nullptr;
@@ -205,9 +198,6 @@ public:
 static Attr *handleNoMergeAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                                SourceRange Range) {
   NoMergeAttr NMA(S.Context, A);
-  if (S.CheckAttrNoArgs(A))
-    return nullptr;
-
   CallExprFinder CEF(S, St);
 
   if (!CEF.foundCallExpr()) {
@@ -377,23 +367,8 @@ static Attr *handleOpenCLUnrollHint(Sema &S, Stmt *St, const ParsedAttr &A,
   // opencl_unroll_hint can have 0 arguments (compiler
   // determines unrolling factor) or 1 argument (the unroll factor provided
   // by the user).
-
-  if (!isa<ForStmt, CXXForRangeStmt, DoStmt, WhileStmt>(St)) {
-    S.Diag(A.getLoc(), diag::err_attribute_wrong_decl_type_str)
-        << A << "'for', 'while', and 'do' statements";
-    return nullptr;
-  }
-
-  unsigned NumArgs = A.getNumArgs();
-
-  if (NumArgs > 1) {
-    S.Diag(A.getLoc(), diag::err_attribute_too_many_arguments) << A << 1;
-    return nullptr;
-  }
-
   unsigned UnrollFactor = 0;
-
-  if (NumArgs == 1) {
+  if (A.getNumArgs() == 1) {
     Expr *E = A.getArgAsExpr(0);
     Optional<llvm::APSInt> ArgVal;
 
@@ -404,28 +379,42 @@ static Attr *handleOpenCLUnrollHint(Sema &S, Stmt *St, const ParsedAttr &A,
     }
 
     int Val = ArgVal->getSExtValue();
-
     if (Val <= 0) {
       S.Diag(A.getRange().getBegin(),
              diag::err_attribute_requires_positive_integer)
           << A << /* positive */ 0;
       return nullptr;
     }
-    UnrollFactor = Val;
+    UnrollFactor = static_cast<unsigned>(Val);
   }
 
-  return OpenCLUnrollHintAttr::CreateImplicit(S.Context, UnrollFactor);
+  return ::new (S.Context) OpenCLUnrollHintAttr(S.Context, A, UnrollFactor);
 }
 
 static Attr *ProcessStmtAttribute(Sema &S, Stmt *St, const ParsedAttr &A,
                                   SourceRange Range) {
-  switch (A.getKind()) {
-  case ParsedAttr::UnknownAttribute:
+  if (A.isInvalid() || A.getKind() == ParsedAttr::IgnoredAttribute)
+    return nullptr;
+
+  // Unknown attributes are automatically warned on. Target-specific attributes
+  // which do not apply to the current target architecture are treated as
+  // though they were unknown attributes.
+  const TargetInfo *Aux = S.Context.getAuxTargetInfo();
+  if (A.getKind() == ParsedAttr::UnknownAttribute ||
+      !(A.existsInTarget(S.Context.getTargetInfo()) ||
+        (S.Context.getLangOpts().SYCLIsDevice && Aux &&
+         A.existsInTarget(*Aux)))) {
     S.Diag(A.getLoc(), A.isDeclspecAttribute()
                            ? (unsigned)diag::warn_unhandled_ms_attribute_ignored
                            : (unsigned)diag::warn_unknown_attribute_ignored)
         << A << A.getRange();
     return nullptr;
+  }
+
+  if (S.checkCommonAttributeFeatures(St, A))
+    return nullptr;
+
+  switch (A.getKind()) {
   case ParsedAttr::AT_FallThrough:
     return handleFallThroughAttr(S, St, A, Range);
   case ParsedAttr::AT_LoopHint:
@@ -441,8 +430,9 @@ static Attr *ProcessStmtAttribute(Sema &S, Stmt *St, const ParsedAttr &A,
   case ParsedAttr::AT_Unlikely:
     return handleUnlikely(S, St, A, Range);
   default:
-    // if we're here, then we parsed a known attribute, but didn't recognize
-    // it as a statement attribute => it is declaration attribute
+    // N.B., ClangAttrEmitter.cpp emits a diagnostic helper that ensures a
+    // declaration attribute is not written on a statement, but this code is
+    // needed for attributes in Attr.td that do not list any subjects.
     S.Diag(A.getRange().getBegin(), diag::err_decl_attribute_invalid_on_stmt)
         << A << St->getBeginLoc();
     return nullptr;
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 6336f3b99452c0c54139f912c9eca9a1fd396039..ecf0c442ad46e198dda73978d4960cb4d1552f17 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -455,11 +455,13 @@ static Sema::TemplateDeductionResult DeduceNullPtrTemplateArgument(
     const NonTypeTemplateParmDecl *NTTP, QualType NullPtrType,
     TemplateDeductionInfo &Info,
     SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
-  Expr *Value =
-      S.ImpCastExprToType(new (S.Context) CXXNullPtrLiteralExpr(
-                              S.Context.NullPtrTy, NTTP->getLocation()),
-                          NullPtrType, CK_NullToPointer)
-          .get();
+  Expr *Value = S.ImpCastExprToType(
+                     new (S.Context) CXXNullPtrLiteralExpr(S.Context.NullPtrTy,
+                                                           NTTP->getLocation()),
+                     NullPtrType,
+                     NullPtrType->isMemberPointerType() ? CK_NullToMemberPointer
+                                                        : CK_NullToPointer)
+                    .get();
   return DeduceNonTypeTemplateArgument(S, TemplateParams, NTTP,
                                        DeducedTemplateArgument(Value),
                                        Value->getType(), Info, Deduced);
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 8bd812b39de4d32f86b547362b874432ef260693..578a77aceeda537b85140503704a2b10de1416ec 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -3420,7 +3420,8 @@ Sema::InstantiateClassMembers(SourceLocation PointOfInstantiation,
             Instantiation->getTemplateInstantiationPattern();
         DeclContext::lookup_result Lookup =
             ClassPattern->lookup(Field->getDeclName());
-        FieldDecl *Pattern = cast<FieldDecl>(Lookup.front());
+        FieldDecl *Pattern = Lookup.find_first<FieldDecl>();
+        assert(Pattern);
         InstantiateInClassInitializer(PointOfInstantiation, Field, Pattern,
                                       TemplateArgs);
       }
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 85480b282ac472d37bdb780f5b2d480b8cd8397f..e08e8570b605dec818681a165cec3531055ce57a 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -8036,8 +8036,6 @@ static void HandleLifetimeBoundAttr(TypeProcessingState &State,
     CurType = State.getAttributedType(
         createSimpleAttr<LifetimeBoundAttr>(State.getSema().Context, Attr),
         CurType, CurType);
-  } else {
-    Attr.diagnoseAppertainsTo(State.getSema(), nullptr);
   }
 }
 
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index e9dfbe0abe2bc2a88a1863a58bac6384694e51ee..6fa91035a0c486b545ccb01ff77c84aa32c68059 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -2170,6 +2170,44 @@ public:
                                             LParenLoc, EndLoc);
   }
 
+  /// Build a new OpenMP 'init' clause.
+  ///
+  /// By default, performs semantic analysis to build the new OpenMP clause.
+  /// Subclasses may override this routine to provide different behavior.
+  OMPClause *RebuildOMPInitClause(Expr *InteropVar, ArrayRef<Expr *> PrefExprs,
+                                  bool IsTarget, bool IsTargetSync,
+                                  SourceLocation StartLoc,
+                                  SourceLocation LParenLoc,
+                                  SourceLocation VarLoc,
+                                  SourceLocation EndLoc) {
+    return getSema().ActOnOpenMPInitClause(InteropVar, PrefExprs, IsTarget,
+                                           IsTargetSync, StartLoc, LParenLoc,
+                                           VarLoc, EndLoc);
+  }
+
+  /// Build a new OpenMP 'use' clause.
+  ///
+  /// By default, performs semantic analysis to build the new OpenMP clause.
+  /// Subclasses may override this routine to provide different behavior.
+  OMPClause *RebuildOMPUseClause(Expr *InteropVar, SourceLocation StartLoc,
+                                 SourceLocation LParenLoc,
+                                 SourceLocation VarLoc, SourceLocation EndLoc) {
+    return getSema().ActOnOpenMPUseClause(InteropVar, StartLoc, LParenLoc,
+                                          VarLoc, EndLoc);
+  }
+
+  /// Build a new OpenMP 'destroy' clause.
+  ///
+  /// By default, performs semantic analysis to build the new OpenMP clause.
+  /// Subclasses may override this routine to provide different behavior.
+  OMPClause *RebuildOMPDestroyClause(Expr *InteropVar, SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation VarLoc,
+                                     SourceLocation EndLoc) {
+    return getSema().ActOnOpenMPDestroyClause(InteropVar, StartLoc, LParenLoc,
+                                              VarLoc, EndLoc);
+  }
+
   /// Rebuild the operand to an Objective-C \@synchronized statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
@@ -9020,6 +9058,16 @@ TreeTransform<Derived>::TransformOMPTargetTeamsDistributeSimdDirective(
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPInteropDirective(OMPInteropDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().StartOpenMPDSABlock(OMPD_interop, DirName, nullptr,
+                                             D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
 
 //===----------------------------------------------------------------------===//
 // OpenMP clause transformation
@@ -9275,11 +9323,47 @@ TreeTransform<Derived>::TransformOMPNogroupClause(OMPNogroupClause *C) {
   return C;
 }
 
+template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPInitClause(OMPInitClause *C) {
+  ExprResult IVR = getDerived().TransformExpr(C->getInteropVar());
+  if (IVR.isInvalid())
+    return nullptr;
+
+  llvm::SmallVector<Expr *, 8> PrefExprs;
+  PrefExprs.reserve(C->varlist_size() - 1);
+  for (Expr *E : llvm::drop_begin(C->varlists())) {
+    ExprResult ER = getDerived().TransformExpr(cast<Expr>(E));
+    if (ER.isInvalid())
+      return nullptr;
+    PrefExprs.push_back(ER.get());
+  }
+  return getDerived().RebuildOMPInitClause(
+      IVR.get(), PrefExprs, C->getIsTarget(), C->getIsTargetSync(),
+      C->getBeginLoc(), C->getLParenLoc(), C->getVarLoc(), C->getEndLoc());
+}
+
+template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPUseClause(OMPUseClause *C) {
+  ExprResult ER = getDerived().TransformExpr(C->getInteropVar());
+  if (ER.isInvalid())
+    return nullptr;
+  return getDerived().RebuildOMPUseClause(ER.get(), C->getBeginLoc(),
+                                          C->getLParenLoc(), C->getVarLoc(),
+                                          C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPDestroyClause(OMPDestroyClause *C) {
-  // No need to rebuild this clause, no template-dependent parameters.
-  return C;
+  ExprResult ER;
+  if (Expr *IV = C->getInteropVar()) {
+    ER = getDerived().TransformExpr(IV);
+    if (ER.isInvalid())
+      return nullptr;
+  }
+  return getDerived().RebuildOMPDestroyClause(ER.get(), C->getBeginLoc(),
+                                              C->getLParenLoc(), C->getVarLoc(),
+                                              C->getEndLoc());
 }
 
 template <typename Derived>
@@ -13887,7 +13971,14 @@ TreeTransform<Derived>::TransformBlockExpr(BlockExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformAsTypeExpr(AsTypeExpr *E) {
-  llvm_unreachable("Cannot transform asType expressions yet");
+  ExprResult SrcExpr = getDerived().TransformExpr(E->getSrcExpr());
+  if (SrcExpr.isInvalid())
+    return ExprError();
+
+  QualType Type = getDerived().TransformType(E->getType());
+
+  return SemaRef.BuildAsTypeExpr(SrcExpr.get(), Type, E->getBuiltinLoc(),
+                                 E->getRParenLoc());
 }
 
 template<typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index a4a0a5ced90be3627064e315c86442bd37c88b22..a76bda15076be6f0c9a6e229718ae344ac0750f7 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7649,9 +7649,10 @@ ASTReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
 
   // Load the list of declarations.
   SmallVector<NamedDecl *, 64> Decls;
+  llvm::SmallPtrSet<NamedDecl *, 8> Found;
   for (DeclID ID : It->second.Table.find(Name)) {
     NamedDecl *ND = cast<NamedDecl>(GetDecl(ID));
-    if (ND->getDeclName() == Name)
+    if (ND->getDeclName() == Name && Found.insert(ND).second)
       Decls.push_back(ND);
   }
 
@@ -11967,6 +11968,12 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_order:
     C = new (Context) OMPOrderClause();
     break;
+  case llvm::omp::OMPC_init:
+    C = OMPInitClause::CreateEmpty(Context, Record.readInt());
+    break;
+  case llvm::omp::OMPC_use:
+    C = new (Context) OMPUseClause();
+    break;
   case llvm::omp::OMPC_destroy:
     C = new (Context) OMPDestroyClause();
     break;
@@ -12130,7 +12137,30 @@ void OMPClauseReader::VisitOMPSIMDClause(OMPSIMDClause *) {}
 
 void OMPClauseReader::VisitOMPNogroupClause(OMPNogroupClause *) {}
 
-void OMPClauseReader::VisitOMPDestroyClause(OMPDestroyClause *) {}
+void OMPClauseReader::VisitOMPInitClause(OMPInitClause *C) {
+  unsigned NumVars = C->varlist_size();
+  SmallVector<Expr *, 16> Vars;
+  Vars.reserve(NumVars);
+  for (unsigned I = 0; I != NumVars; ++I)
+    Vars.push_back(Record.readSubExpr());
+  C->setVarRefs(Vars);
+  C->setIsTarget(Record.readBool());
+  C->setIsTargetSync(Record.readBool());
+  C->setLParenLoc(Record.readSourceLocation());
+  C->setVarLoc(Record.readSourceLocation());
+}
+
+void OMPClauseReader::VisitOMPUseClause(OMPUseClause *C) {
+  C->setInteropVar(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+  C->setVarLoc(Record.readSourceLocation());
+}
+
+void OMPClauseReader::VisitOMPDestroyClause(OMPDestroyClause *C) {
+  C->setInteropVar(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+  C->setVarLoc(Record.readSourceLocation());
+}
 
 void OMPClauseReader::VisitOMPUnifiedAddressClause(OMPUnifiedAddressClause *) {}
 
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 24513b70edd7b59c494b254db70560802dd33a04..22e1f57e3313ef9be5f7cd476d62b3dab8db7f40 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2588,6 +2588,11 @@ void ASTStmtReader::VisitOMPTargetTeamsDistributeSimdDirective(
   VisitOMPLoopDirective(D);
 }
 
+void ASTStmtReader::VisitOMPInteropDirective(OMPInteropDirective *D) {
+  VisitStmt(D);
+  VisitOMPExecutableDirective(D);
+}
+
 //===----------------------------------------------------------------------===//
 // ASTReader Implementation
 //===----------------------------------------------------------------------===//
@@ -3503,6 +3508,11 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       break;
     }
 
+    case STMT_OMP_INTEROP_DIRECTIVE:
+      S = OMPInteropDirective::CreateEmpty(
+          Context, Record[ASTStmtReader::NumStmtFields], Empty);
+      break;
+
     case EXPR_CXX_OPERATOR_CALL:
       S = CXXOperatorCallExpr::CreateEmpty(
           Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields],
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 68ea7955121d7693652525e732c84785ae7efb71..18decd9e6bc1603b5daa123da624ed14e0d3e252 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6215,7 +6215,27 @@ void OMPClauseWriter::VisitOMPSIMDClause(OMPSIMDClause *) {}
 
 void OMPClauseWriter::VisitOMPNogroupClause(OMPNogroupClause *) {}
 
-void OMPClauseWriter::VisitOMPDestroyClause(OMPDestroyClause *) {}
+void OMPClauseWriter::VisitOMPInitClause(OMPInitClause *C) {
+  Record.push_back(C->varlist_size());
+  for (Expr *VE : C->varlists())
+    Record.AddStmt(VE);
+  Record.writeBool(C->getIsTarget());
+  Record.writeBool(C->getIsTargetSync());
+  Record.AddSourceLocation(C->getLParenLoc());
+  Record.AddSourceLocation(C->getVarLoc());
+}
+
+void OMPClauseWriter::VisitOMPUseClause(OMPUseClause *C) {
+  Record.AddStmt(C->getInteropVar());
+  Record.AddSourceLocation(C->getLParenLoc());
+  Record.AddSourceLocation(C->getVarLoc());
+}
+
+void OMPClauseWriter::VisitOMPDestroyClause(OMPDestroyClause *C) {
+  Record.AddStmt(C->getInteropVar());
+  Record.AddSourceLocation(C->getLParenLoc());
+  Record.AddSourceLocation(C->getVarLoc());
+}
 
 void OMPClauseWriter::VisitOMPPrivateClause(OMPPrivateClause *C) {
   Record.push_back(C->varlist_size());
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 2cb44bf9038b524b5635bbb49fcc2aa79a476664..2b8278090b058cfc6db13cb7953ade6da3fb19cd 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -13,7 +13,6 @@
 #include "ASTCommon.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclCXX.h"
-#include "clang/AST/DeclContextInternals.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclVisitor.h"
 #include "clang/AST/Expr.h"
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 58fb11e70d14f66a977c19bede16c01d0f7e0719..ed14058aeb0d602ef72386b44288b9a75ef9f315 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2541,6 +2541,12 @@ void ASTStmtWriter::VisitOMPTargetTeamsDistributeSimdDirective(
   Code = serialization::STMT_OMP_TARGET_TEAMS_DISTRIBUTE_SIMD_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPInteropDirective(OMPInteropDirective *D) {
+  VisitStmt(D);
+  VisitOMPExecutableDirective(D);
+  Code = serialization::STMT_OMP_INTEROP_DIRECTIVE;
+}
+
 //===----------------------------------------------------------------------===//
 // ASTWriter Implementation
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
index 6bc186aa275581e63ccfa07de683ad9f9297e649..8c86e83608b1ef36ac0a7c9a9d89b498a29db8d0 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
@@ -260,8 +260,8 @@ public:
         break;
     }
 
-    BR.EmitBasicReport(AC->getDecl(), Checker, BugType, "Dead store", os.str(),
-                       L, R, Fixits);
+    BR.EmitBasicReport(AC->getDecl(), Checker, BugType, categories::UnusedCode,
+                       os.str(), L, R, Fixits);
   }
 
   void CheckVarDecl(const VarDecl *VD, const Expr *Ex, const Expr *Val,
diff --git a/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp
index 80b705fb73926c7cde8143f5ce0d7e9fd60ae467..c5437b16c6886cfc6d2196515378fa60b0034f9c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp
@@ -89,7 +89,7 @@ void NonnullGlobalConstantsChecker::checkLocation(SVal location, bool isLoad,
 }
 
 /// \param V loaded lvalue.
-/// \return whether {@code val} is a string-like const global.
+/// \return whether @c val is a string-like const global.
 bool NonnullGlobalConstantsChecker::isGlobalConstString(SVal V) const {
   Optional<loc::MemRegionVal> RegionVal = V.getAs<loc::MemRegionVal>();
   if (!RegionVal)
@@ -127,7 +127,7 @@ bool NonnullGlobalConstantsChecker::isGlobalConstString(SVal V) const {
   return false;
 }
 
-/// \return whether {@code type} is extremely unlikely to be null
+/// \return whether @c type is extremely unlikely to be null
 bool NonnullGlobalConstantsChecker::isNonnullType(QualType Ty) const {
 
   if (Ty->isPointerType() && Ty->getPointeeType()->isCharType())
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp
index 7fd6e2abef4c5ac7ca16455f72f6c9f38e3ab578..c8eab32880943612c28a38f25a05a56fe08af445 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp
@@ -8,7 +8,7 @@
 //
 // This file defines ObjCAutoreleaseWriteChecker which warns against writes
 // into autoreleased out parameters which cause crashes.
-// An example of a problematic write is a write to {@code error} in the example
+// An example of a problematic write is a write to @c error in the example
 // below:
 //
 // - (BOOL) mymethod:(NSError *__autoreleasing *)error list:(NSArray*) list {
diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
index 1d903530201f84c0a3ab782e8cbc9ea315f58c7e..1fc3ee03d2e1008bbbf5d22da9773359e86d49df 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
@@ -89,7 +89,7 @@ static std::string getPrettyTypeName(QualType QT) {
   return QT.getAsString();
 }
 
-/// Write information about the type state change to {@code os},
+/// Write information about the type state change to @c os,
 /// return whether the note should be generated.
 static bool shouldGenerateNote(llvm::raw_string_ostream &os,
                                const RefVal *PrevT,
@@ -164,8 +164,8 @@ static bool shouldGenerateNote(llvm::raw_string_ostream &os,
   return true;
 }
 
-/// Finds argument index of the out paramter in the call {@code S}
-/// corresponding to the symbol {@code Sym}.
+/// Finds argument index of the out paramter in the call @c S
+/// corresponding to the symbol @c Sym.
 /// If none found, returns None.
 static Optional<unsigned> findArgIdxOfSymbol(ProgramStateRef CurrSt,
                                              const LocationContext *LCtx,
diff --git a/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp
index d9dc72ddaa218d9e07d76c40510bc34c47d03f0d..2cf6c6ff47f1d2440247b43502d92376d4492005 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp
@@ -57,8 +57,8 @@ public:
 
 } // end anonymous namespace
 
-/// \return Whether {@code A} occurs before {@code B} in traversal of
-/// {@code Parent}.
+/// \return Whether @c A occurs before @c B in traversal of
+/// @c Parent.
 /// Conceptually a very incomplete/unsound approximation of happens-before
 /// relationship (A is likely to be evaluated before B),
 /// but useful enough in this case.
diff --git a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
index 38a9d4ba65b6bf7e41523104bba0a820871aaaa4..c8b2e3d915207684f911c89f3c795155a42dc229 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
@@ -842,7 +842,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     llvm::Optional<QualType> operator()(StringRef Name) {
       IdentifierInfo &II = ACtx.Idents.get(Name);
       auto LookupRes = ACtx.getTranslationUnitDecl()->lookup(&II);
-      if (LookupRes.size() == 0)
+      if (LookupRes.empty())
         return None;
 
       // Prioritze typedef declarations.
@@ -994,7 +994,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         return false;
       IdentifierInfo &II = ACtx.Idents.get(Name);
       auto LookupRes = ACtx.getTranslationUnitDecl()->lookup(&II);
-      if (LookupRes.size() == 0)
+      if (LookupRes.empty())
         return false;
       for (Decl *D : LookupRes) {
         if (auto *FD = dyn_cast<FunctionDecl>(D)) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
index 74eec81ffb3e5ec1170b202706153036f4f6c946..d231be64c2e19468fb286fee353740944821aca5 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
@@ -169,7 +169,7 @@ void UnreachableCodeChecker::checkEndAnalysis(ExplodedGraph &G,
     if (SM.isInSystemHeader(SL) || SM.isInExternCSystemHeader(SL))
       continue;
 
-    B.EmitBasicReport(D, this, "Unreachable code", "Dead code",
+    B.EmitBasicReport(D, this, "Unreachable code", categories::UnusedCode,
                       "This statement is never executed", DL, SR);
   }
 }
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index bf38891b370a977fd17a917009263faddd204cdc..b64c0798d7e2f77ed01fbf2ea2838ce1f7babb36 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -2738,8 +2738,8 @@ static void CompactMacroExpandedPieces(PathPieces &path,
 }
 
 /// Generate notes from all visitors.
-/// Notes associated with {@code ErrorNode} are generated using
-/// {@code getEndPath}, and the rest are generated with {@code VisitNode}.
+/// Notes associated with @c ErrorNode are generated using
+/// @c getEndPath, and the rest are generated with @c VisitNode.
 static std::unique_ptr<VisitorsDiagnosticsTy>
 generateVisitorsDiagnostics(PathSensitiveBugReport *R,
                             const ExplodedNode *ErrorNode,
@@ -2749,7 +2749,7 @@ generateVisitorsDiagnostics(PathSensitiveBugReport *R,
   PathSensitiveBugReport::VisitorList visitors;
 
   // Run visitors on all nodes starting from the node *before* the last one.
-  // The last node is reserved for notes generated with {@code getEndPath}.
+  // The last node is reserved for notes generated with @c getEndPath.
   const ExplodedNode *NextNode = ErrorNode->getFirstPred();
   while (NextNode) {
 
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index bc72f4f8c1e300848395cb2b5b82bbc593a59cd2..0edd6e3f731b6347484c6148ce2f48fff3f7e407 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -1846,7 +1846,7 @@ static const MemRegion *getLocationRegionIfReference(const Expr *E,
   return nullptr;
 }
 
-/// \return A subexpression of {@code Ex} which represents the
+/// \return A subexpression of @c Ex which represents the
 /// expression-of-interest.
 static const Expr *peelOffOuterExpr(const Expr *Ex,
                                     const ExplodedNode *N) {
@@ -1942,7 +1942,7 @@ bool bugreporter::trackExpressionValue(const ExplodedNode *InputNode,
   const StackFrameContext *SFC = LVNode->getStackFrame();
 
   // We only track expressions if we believe that they are important. Chances
-  // are good that control dependencies to the tracking point are also improtant
+  // are good that control dependencies to the tracking point are also important
   // because of this, let's explain why we believe control reached this point.
   // TODO: Shouldn't we track control dependencies of every bug location, rather
   // than only tracked expressions?
diff --git a/clang/lib/StaticAnalyzer/Core/CommonBugCategories.cpp b/clang/lib/StaticAnalyzer/Core/CommonBugCategories.cpp
index b61e3075f3baaeaee6b4d8d802d6651bb95240c9..d12c35ef156a04b222a311ef025f83e0cea092ea 100644
--- a/clang/lib/StaticAnalyzer/Core/CommonBugCategories.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CommonBugCategories.cpp
@@ -22,6 +22,7 @@ const char *const UnixAPI = "Unix API";
 const char *const CXXObjectLifecycle = "C++ object lifecycle";
 const char *const CXXMoveSemantics = "C++ move semantics";
 const char *const SecurityError = "Security error";
+const char *const UnusedCode = "Unused code";
 } // namespace categories
 } // namespace ento
 } // namespace clang
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index a388fc9e6e26c96c81b2dbdb0893d2f795bfe5ac..fbd5fd87fcf0e0a83a7cf94b249f2f3de70aa031 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1294,6 +1294,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
     case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
+    case Stmt::OMPInteropDirectiveClass:
     case Stmt::CapturedStmtClass: {
       const ExplodedNode *node = Bldr.generateSink(S, Pred, Pred->getState());
       Engine.addAbortedBlock(node, currBldrCtx->getBlock());
@@ -3139,8 +3140,8 @@ struct DOTGraphTraits<ExplodedGraph*> : public DefaultDOTGraphTraits {
 
   /// \p PreCallback: callback before break.
   /// \p PostCallback: callback after break.
-  /// \p Stop: stop iteration if returns {@code true}
-  /// \return Whether {@code Stop} ever returned {@code true}.
+  /// \p Stop: stop iteration if returns @c true
+  /// \return Whether @c Stop ever returned @c true.
   static bool traverseHiddenNodes(
       const ExplodedNode *N,
       llvm::function_ref<void(const ExplodedNode *)> PreCallback,
diff --git a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
index fe530bce4a3e4cb820fb8367312cb4acc4a00bd8..64fc32ea75541bfa55cef310e43e9d0194207196 100644
--- a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
@@ -275,11 +275,11 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
                        << "' absolute: " << EC.message() << '\n';
         return;
       }
-      if (std::error_code EC =
-          llvm::sys::fs::createUniqueFile(Model, FD, ResultPath)) {
-          llvm::errs() << "warning: could not create file in '" << Directory
-                       << "': " << EC.message() << '\n';
-          return;
+      if (std::error_code EC = llvm::sys::fs::createUniqueFile(
+              Model, FD, ResultPath, llvm::sys::fs::OF_Text)) {
+        llvm::errs() << "warning: could not create file in '" << Directory
+                     << "': " << EC.message() << '\n';
+        return;
       }
   } else {
       int i = 1;
diff --git a/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
index 9842f3ace484ea5ac726a6ec57a154d783b5b939..7e7fe75082bbb80fbbe78df363217f95b45954f0 100644
--- a/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
@@ -157,8 +157,8 @@ private:
 
 } // end of anonymous namespace
 
-/// Print coverage information to output stream {@code o}.
-/// May modify the used list of files {@code Fids} by inserting new ones.
+/// Print coverage information to output stream @c o.
+/// May modify the used list of files @c Fids by inserting new ones.
 static void printCoverage(const PathDiagnostic *D,
                           unsigned InputIndentLevel,
                           SmallVectorImpl<FileID> &Fids,
@@ -484,8 +484,8 @@ void PlistPrinter::ReportPopUp(raw_ostream &o,
 // Static function definitions.
 //===----------------------------------------------------------------------===//
 
-/// Print coverage information to output stream {@code o}.
-/// May modify the used list of files {@code Fids} by inserting new ones.
+/// Print coverage information to output stream @c o.
+/// May modify the used list of files @c Fids by inserting new ones.
 static void printCoverage(const PathDiagnostic *D,
                           unsigned InputIndentLevel,
                           SmallVectorImpl<FileID> &Fids,
diff --git a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
index 95e61f9c8c61cb3b2dc5c2afb1dd58989229614f..974535952d0f0ec8a9a5d1f9150a7ff3621fb9ee 100644
--- a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
@@ -22,6 +22,8 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <iterator>
 
 using namespace clang;
 using namespace ento;
@@ -99,47 +101,63 @@ public:
     return CmpOpTable[getIndexFromOp(CurrentOP)][CmpOpCount];
   }
 };
+
 //===----------------------------------------------------------------------===//
 //                           RangeSet implementation
 //===----------------------------------------------------------------------===//
 
-void RangeSet::IntersectInRange(BasicValueFactory &BV, Factory &F,
-                                const llvm::APSInt &Lower,
-                                const llvm::APSInt &Upper,
-                                PrimRangeSet &newRanges,
-                                PrimRangeSet::iterator &i,
-                                PrimRangeSet::iterator &e) const {
-  // There are six cases for each range R in the set:
-  //   1. R is entirely before the intersection range.
-  //   2. R is entirely after the intersection range.
-  //   3. R contains the entire intersection range.
-  //   4. R starts before the intersection range and ends in the middle.
-  //   5. R starts in the middle of the intersection range and ends after it.
-  //   6. R is entirely contained in the intersection range.
-  // These correspond to each of the conditions below.
-  for (/* i = begin(), e = end() */; i != e; ++i) {
-    if (i->To() < Lower) {
-      continue;
-    }
-    if (i->From() > Upper) {
-      break;
-    }
+RangeSet::ContainerType RangeSet::Factory::EmptySet{};
 
-    if (i->Includes(Lower)) {
-      if (i->Includes(Upper)) {
-        newRanges =
-            F.add(newRanges, Range(BV.getValue(Lower), BV.getValue(Upper)));
-        break;
-      } else
-        newRanges = F.add(newRanges, Range(BV.getValue(Lower), i->To()));
-    } else {
-      if (i->Includes(Upper)) {
-        newRanges = F.add(newRanges, Range(i->From(), BV.getValue(Upper)));
-        break;
-      } else
-        newRanges = F.add(newRanges, *i);
-    }
+RangeSet RangeSet::Factory::add(RangeSet Original, Range Element) {
+  ContainerType Result;
+  Result.reserve(Original.size() + 1);
+
+  const_iterator Lower = llvm::lower_bound(Original, Element);
+  Result.insert(Result.end(), Original.begin(), Lower);
+  Result.push_back(Element);
+  Result.insert(Result.end(), Lower, Original.end());
+
+  return makePersistent(std::move(Result));
+}
+
+RangeSet RangeSet::Factory::add(RangeSet Original, const llvm::APSInt &Point) {
+  return add(Original, Range(Point));
+}
+
+RangeSet RangeSet::Factory::getRangeSet(Range From) {
+  ContainerType Result;
+  Result.push_back(From);
+  return makePersistent(std::move(Result));
+}
+
+RangeSet RangeSet::Factory::makePersistent(ContainerType &&From) {
+  llvm::FoldingSetNodeID ID;
+  void *InsertPos;
+
+  From.Profile(ID);
+  ContainerType *Result = Cache.FindNodeOrInsertPos(ID, InsertPos);
+
+  if (!Result) {
+    // It is cheaper to fully construct the resulting range on stack
+    // and move it to the freshly allocated buffer if we don't have
+    // a set like this already.
+    Result = construct(std::move(From));
+    Cache.InsertNode(Result, InsertPos);
   }
+
+  return Result;
+}
+
+RangeSet::ContainerType *RangeSet::Factory::construct(ContainerType &&From) {
+  void *Buffer = Arena.Allocate();
+  return new (Buffer) ContainerType(std::move(From));
+}
+
+RangeSet RangeSet::Factory::add(RangeSet LHS, RangeSet RHS) {
+  ContainerType Result;
+  std::merge(LHS.begin(), LHS.end(), RHS.begin(), RHS.end(),
+             std::back_inserter(Result));
+  return makePersistent(std::move(Result));
 }
 
 const llvm::APSInt &RangeSet::getMinValue() const {
@@ -149,22 +167,31 @@ const llvm::APSInt &RangeSet::getMinValue() const {
 
 const llvm::APSInt &RangeSet::getMaxValue() const {
   assert(!isEmpty());
-  // NOTE: It's a shame that we can't implement 'getMaxValue' without scanning
-  //       the whole tree to get to the last element.
-  //       llvm::ImmutableSet should support decrement for 'end' iterators
-  //       or reverse order iteration.
-  auto It = begin();
-  for (auto End = end(); std::next(It) != End; ++It) {
-  }
-  return It->To();
+  return std::prev(end())->To();
 }
 
-bool RangeSet::pin(llvm::APSInt &Lower, llvm::APSInt &Upper) const {
-  if (isEmpty()) {
-    // This range is already infeasible.
+bool RangeSet::containsImpl(llvm::APSInt &Point) const {
+  if (isEmpty() || !pin(Point))
     return false;
-  }
 
+  Range Dummy(Point);
+  const_iterator It = llvm::upper_bound(*this, Dummy);
+  if (It == begin())
+    return false;
+
+  return std::prev(It)->Includes(Point);
+}
+
+bool RangeSet::pin(llvm::APSInt &Point) const {
+  APSIntType Type(getMinValue());
+  if (Type.testInRange(Point, true) != APSIntType::RTR_Within)
+    return false;
+
+  Type.apply(Point);
+  return true;
+}
+
+bool RangeSet::pin(llvm::APSInt &Lower, llvm::APSInt &Upper) const {
   // This function has nine cases, the cartesian product of range-testing
   // both the upper and lower bounds against the symbol's type.
   // Each case requires a different pinning operation.
@@ -245,129 +272,216 @@ bool RangeSet::pin(llvm::APSInt &Lower, llvm::APSInt &Upper) const {
   return true;
 }
 
-// Returns a set containing the values in the receiving set, intersected with
-// the closed range [Lower, Upper]. Unlike the Range type, this range uses
-// modular arithmetic, corresponding to the common treatment of C integer
-// overflow. Thus, if the Lower bound is greater than the Upper bound, the
-// range is taken to wrap around. This is equivalent to taking the
-// intersection with the two ranges [Min, Upper] and [Lower, Max],
-// or, alternatively, /removing/ all integers between Upper and Lower.
-RangeSet RangeSet::Intersect(BasicValueFactory &BV, Factory &F,
-                             llvm::APSInt Lower, llvm::APSInt Upper) const {
-  PrimRangeSet newRanges = F.getEmptySet();
-
-  if (isEmpty() || !pin(Lower, Upper))
-    return newRanges;
-
-  PrimRangeSet::iterator i = begin(), e = end();
-  if (Lower <= Upper)
-    IntersectInRange(BV, F, Lower, Upper, newRanges, i, e);
-  else {
-    // The order of the next two statements is important!
-    // IntersectInRange() does not reset the iteration state for i and e.
-    // Therefore, the lower range most be handled first.
-    IntersectInRange(BV, F, BV.getMinValue(Upper), Upper, newRanges, i, e);
-    IntersectInRange(BV, F, Lower, BV.getMaxValue(Lower), newRanges, i, e);
-  }
-
-  return newRanges;
-}
-
-// Returns a set containing the values in the receiving set, intersected with
-// the range set passed as parameter.
-RangeSet RangeSet::Intersect(BasicValueFactory &BV, Factory &F,
-                             const RangeSet &Other) const {
-  PrimRangeSet newRanges = F.getEmptySet();
-
-  for (iterator i = Other.begin(), e = Other.end(); i != e; ++i) {
-    RangeSet newPiece = Intersect(BV, F, i->From(), i->To());
-    for (iterator j = newPiece.begin(), ee = newPiece.end(); j != ee; ++j) {
-      newRanges = F.add(newRanges, *j);
-    }
+RangeSet RangeSet::Factory::intersect(RangeSet What, llvm::APSInt Lower,
+                                      llvm::APSInt Upper) {
+  if (What.isEmpty() || !What.pin(Lower, Upper))
+    return getEmptySet();
+
+  ContainerType DummyContainer;
+
+  if (Lower <= Upper) {
+    // [Lower, Upper] is a regular range.
+    //
+    // Shortcut: check that there is even a possibility of the intersection
+    //           by checking the two following situations:
+    //
+    //               <---[  What  ]---[------]------>
+    //                              Lower  Upper
+    //                            -or-
+    //               <----[------]----[  What  ]---->
+    //                  Lower  Upper
+    if (What.getMaxValue() < Lower || Upper < What.getMinValue())
+      return getEmptySet();
+
+    DummyContainer.push_back(
+        Range(ValueFactory.getValue(Lower), ValueFactory.getValue(Upper)));
+  } else {
+    // [Lower, Upper] is an inverted range, i.e. [MIN, Upper] U [Lower, MAX]
+    //
+    // Shortcut: check that there is even a possibility of the intersection
+    //           by checking the following situation:
+    //
+    //               <------]---[  What  ]---[------>
+    //                    Upper             Lower
+    if (What.getMaxValue() < Lower && Upper < What.getMinValue())
+      return getEmptySet();
+
+    DummyContainer.push_back(
+        Range(ValueFactory.getMinValue(Upper), ValueFactory.getValue(Upper)));
+    DummyContainer.push_back(
+        Range(ValueFactory.getValue(Lower), ValueFactory.getMaxValue(Lower)));
   }
 
-  return newRanges;
+  return intersect(*What.Impl, DummyContainer);
 }
 
-// Turn all [A, B] ranges to [-B, -A], when "-" is a C-like unary minus
-// operation under the values of the type.
-//
-// We also handle MIN because applying unary minus to MIN does not change it.
-// Example 1:
-// char x = -128;        // -128 is a MIN value in a range of 'char'
-// char y = -x;          // y: -128
-// Example 2:
-// unsigned char x = 0;  // 0 is a MIN value in a range of 'unsigned char'
-// unsigned char y = -x; // y: 0
-//
-// And it makes us to separate the range
-// like [MIN, N] to [MIN, MIN] U [-N,MAX].
-// For instance, whole range is {-128..127} and subrange is [-128,-126],
-// thus [-128,-127,-126,.....] negates to [-128,.....,126,127].
-//
-// Negate restores disrupted ranges on bounds,
-// e.g. [MIN, B] => [MIN, MIN] U [-B, MAX] => [MIN, B].
-RangeSet RangeSet::Negate(BasicValueFactory &BV, Factory &F) const {
-  PrimRangeSet newRanges = F.getEmptySet();
+RangeSet RangeSet::Factory::intersect(const RangeSet::ContainerType &LHS,
+                                      const RangeSet::ContainerType &RHS) {
+  ContainerType Result;
+  Result.reserve(std::max(LHS.size(), RHS.size()));
+
+  const_iterator First = LHS.begin(), Second = RHS.begin(),
+                 FirstEnd = LHS.end(), SecondEnd = RHS.end();
+
+  const auto SwapIterators = [&First, &FirstEnd, &Second, &SecondEnd]() {
+    std::swap(First, Second);
+    std::swap(FirstEnd, SecondEnd);
+  };
 
-  if (isEmpty())
-    return newRanges;
+  // If we ran out of ranges in one set, but not in the other,
+  // it means that those elements are definitely not in the
+  // intersection.
+  while (First != FirstEnd && Second != SecondEnd) {
+    // We want to keep the following invariant at all times:
+    //
+    //    ----[ First ---------------------->
+    //    --------[ Second ----------------->
+    if (Second->From() < First->From())
+      SwapIterators();
+
+    // Loop where the invariant holds:
+    do {
+      // Check for the following situation:
+      //
+      //    ----[ First ]--------------------->
+      //    ---------------[ Second ]--------->
+      //
+      // which means that...
+      if (Second->From() > First->To()) {
+        // ...First is not in the intersection.
+        //
+        // We should move on to the next range after First and break out of the
+        // loop because the invariant might not be true.
+        ++First;
+        break;
+      }
 
-  const llvm::APSInt sampleValue = getMinValue();
-  const llvm::APSInt &MIN = BV.getMinValue(sampleValue);
-  const llvm::APSInt &MAX = BV.getMaxValue(sampleValue);
+      // We have a guaranteed intersection at this point!
+      // And this is the current situation:
+      //
+      //    ----[   First   ]----------------->
+      //    -------[ Second ------------------>
+      //
+      // Additionally, it definitely starts with Second->From().
+      const llvm::APSInt &IntersectionStart = Second->From();
+
+      // It is important to know which of the two ranges' ends
+      // is greater.  That "longer" range might have some other
+      // intersections, while the "shorter" range might not.
+      if (Second->To() > First->To()) {
+        // Here we make a decision to keep First as the "longer"
+        // range.
+        SwapIterators();
+      }
+
+      // At this point, we have the following situation:
+      //
+      //    ---- First      ]-------------------->
+      //    ---- Second ]--[  Second+1 ---------->
+      //
+      // We don't know the relationship between First->From and
+      // Second->From and we don't know whether Second+1 intersects
+      // with First.
+      //
+      // However, we know that [IntersectionStart, Second->To] is
+      // a part of the intersection...
+      Result.push_back(Range(IntersectionStart, Second->To()));
+      ++Second;
+      // ...and that the invariant will hold for a valid Second+1
+      // because First->From <= Second->To < (Second+1)->From.
+    } while (Second != SecondEnd);
+  }
+
+  if (Result.empty())
+    return getEmptySet();
+
+  return makePersistent(std::move(Result));
+}
+
+RangeSet RangeSet::Factory::intersect(RangeSet LHS, RangeSet RHS) {
+  // Shortcut: let's see if the intersection is even possible.
+  if (LHS.isEmpty() || RHS.isEmpty() || LHS.getMaxValue() < RHS.getMinValue() ||
+      RHS.getMaxValue() < LHS.getMinValue())
+    return getEmptySet();
+
+  return intersect(*LHS.Impl, *RHS.Impl);
+}
+
+RangeSet RangeSet::Factory::intersect(RangeSet LHS, llvm::APSInt Point) {
+  if (LHS.containsImpl(Point))
+    return getRangeSet(ValueFactory.getValue(Point));
+
+  return getEmptySet();
+}
+
+RangeSet RangeSet::Factory::negate(RangeSet What) {
+  if (What.isEmpty())
+    return getEmptySet();
+
+  const llvm::APSInt SampleValue = What.getMinValue();
+  const llvm::APSInt &MIN = ValueFactory.getMinValue(SampleValue);
+  const llvm::APSInt &MAX = ValueFactory.getMaxValue(SampleValue);
+
+  ContainerType Result;
+  Result.reserve(What.size() + (SampleValue == MIN));
 
   // Handle a special case for MIN value.
-  iterator i = begin();
-  const llvm::APSInt &from = i->From();
-  const llvm::APSInt &to = i->To();
-  if (from == MIN) {
-    // If [from, to] are [MIN, MAX], then just return the same [MIN, MAX].
-    if (to == MAX) {
-      newRanges = ranges;
+  const_iterator It = What.begin();
+  const_iterator End = What.end();
+
+  const llvm::APSInt &From = It->From();
+  const llvm::APSInt &To = It->To();
+
+  if (From == MIN) {
+    // If the range [From, To] is [MIN, MAX], then result is also [MIN, MAX].
+    if (To == MAX) {
+      return What;
+    }
+
+    const_iterator Last = std::prev(End);
+
+    // Try to find and unite the following ranges:
+    // [MIN, MIN] & [MIN + 1, N] => [MIN, N].
+    if (Last->To() == MAX) {
+      // It means that in the original range we have ranges
+      //   [MIN, A], ... , [B, MAX]
+      // And the result should be [MIN, -B], ..., [-A, MAX]
+      Result.emplace_back(MIN, ValueFactory.getValue(-Last->From()));
+      // We already negated Last, so we can skip it.
+      End = Last;
     } else {
-      // Add separate range for the lowest value.
-      newRanges = F.add(newRanges, Range(MIN, MIN));
-      // Skip adding the second range in case when [from, to] are [MIN, MIN].
-      if (to != MIN) {
-        newRanges = F.add(newRanges, Range(BV.getValue(-to), MAX));
-      }
+      // Add a separate range for the lowest value.
+      Result.emplace_back(MIN, MIN);
+    }
+
+    // Skip adding the second range in case when [From, To] are [MIN, MIN].
+    if (To != MIN) {
+      Result.emplace_back(ValueFactory.getValue(-To), MAX);
     }
+
     // Skip the first range in the loop.
-    ++i;
+    ++It;
   }
 
   // Negate all other ranges.
-  for (iterator e = end(); i != e; ++i) {
+  for (; It != End; ++It) {
     // Negate int values.
-    const llvm::APSInt &newFrom = BV.getValue(-i->To());
-    const llvm::APSInt &newTo = BV.getValue(-i->From());
-    // Add a negated range.
-    newRanges = F.add(newRanges, Range(newFrom, newTo));
-  }
+    const llvm::APSInt &NewFrom = ValueFactory.getValue(-It->To());
+    const llvm::APSInt &NewTo = ValueFactory.getValue(-It->From());
 
-  if (newRanges.isSingleton())
-    return newRanges;
-
-  // Try to find and unite next ranges:
-  // [MIN, MIN] & [MIN + 1, N] => [MIN, N].
-  iterator iter1 = newRanges.begin();
-  iterator iter2 = std::next(iter1);
-
-  if (iter1->To() == MIN && (iter2->From() - 1) == MIN) {
-    const llvm::APSInt &to = iter2->To();
-    // remove adjacent ranges
-    newRanges = F.remove(newRanges, *iter1);
-    newRanges = F.remove(newRanges, *newRanges.begin());
-    // add united range
-    newRanges = F.add(newRanges, Range(MIN, to));
+    // Add a negated range.
+    Result.emplace_back(NewFrom, NewTo);
   }
 
-  return newRanges;
+  llvm::sort(Result);
+  return makePersistent(std::move(Result));
 }
 
-RangeSet RangeSet::Delete(BasicValueFactory &BV, Factory &F,
-                          const llvm::APSInt &Point) const {
+RangeSet RangeSet::Factory::deletePoint(RangeSet From,
+                                        const llvm::APSInt &Point) {
+  if (!From.contains(Point))
+    return From;
+
   llvm::APSInt Upper = Point;
   llvm::APSInt Lower = Point;
 
@@ -375,22 +489,17 @@ RangeSet RangeSet::Delete(BasicValueFactory &BV, Factory &F,
   --Lower;
 
   // Notice that the lower bound is greater than the upper bound.
-  return Intersect(BV, F, Upper, Lower);
+  return intersect(From, Upper, Lower);
 }
 
-void RangeSet::print(raw_ostream &os) const {
-  bool isFirst = true;
-  os << "{ ";
-  for (iterator i = begin(), e = end(); i != e; ++i) {
-    if (isFirst)
-      isFirst = false;
-    else
-      os << ", ";
+void Range::dump(raw_ostream &OS) const {
+  OS << '[' << From().toString(10) << ", " << To().toString(10) << ']';
+}
 
-    os << '[' << i->From().toString(10) << ", " << i->To().toString(10)
-       << ']';
-  }
-  os << " }";
+void RangeSet::dump(raw_ostream &OS) const {
+  OS << "{ ";
+  llvm::interleaveComma(*this, OS, [&OS](const Range &R) { R.dump(OS); });
+  OS << " }";
 }
 
 REGISTER_SET_FACTORY_WITH_PROGRAMSTATE(SymbolSet, SymbolRef)
@@ -448,12 +557,12 @@ public:
                                               EquivalenceClass Other);
 
   /// Return a set of class members for the given state.
-  LLVM_NODISCARD inline SymbolSet getClassMembers(ProgramStateRef State);
+  LLVM_NODISCARD inline SymbolSet getClassMembers(ProgramStateRef State) const;
   /// Return true if the current class is trivial in the given state.
-  LLVM_NODISCARD inline bool isTrivial(ProgramStateRef State);
+  LLVM_NODISCARD inline bool isTrivial(ProgramStateRef State) const;
   /// Return true if the current class is trivial and its only member is dead.
   LLVM_NODISCARD inline bool isTriviallyDead(ProgramStateRef State,
-                                             SymbolReaper &Reaper);
+                                             SymbolReaper &Reaper) const;
 
   LLVM_NODISCARD static inline ProgramStateRef
   markDisequal(BasicValueFactory &BV, RangeSet::Factory &F,
@@ -521,7 +630,7 @@ private:
                                    ProgramStateRef State, SymbolSet Members,
                                    EquivalenceClass Other,
                                    SymbolSet OtherMembers);
-  static inline void
+  static inline bool
   addToDisequalityInfo(DisequalityMapTy &Info, ConstraintRangeTy &Constraints,
                        BasicValueFactory &BV, RangeSet::Factory &F,
                        ProgramStateRef State, EquivalenceClass First,
@@ -535,6 +644,15 @@ private:
 //                             Constraint functions
 //===----------------------------------------------------------------------===//
 
+LLVM_NODISCARD LLVM_ATTRIBUTE_UNUSED bool
+areFeasible(ConstraintRangeTy Constraints) {
+  return llvm::none_of(
+      Constraints,
+      [](const std::pair<EquivalenceClass, RangeSet> &ClassConstraint) {
+        return ClassConstraint.second.isEmpty();
+      });
+}
+
 LLVM_NODISCARD inline const RangeSet *getConstraint(ProgramStateRef State,
                                                     EquivalenceClass Class) {
   return State->get<ConstraintRange>(Class);
@@ -653,7 +771,7 @@ LLVM_NODISCARD inline RangeSet intersect(BasicValueFactory &BV,
                                          RangeSet Second, RestTy... Tail) {
   // Here we call either the <RangeSet,RangeSet,...> or <RangeSet,...> version
   // of the function and can be sure that the result is RangeSet.
-  return intersect(BV, F, Head.Intersect(BV, F, Second), Tail...);
+  return intersect(BV, F, F.intersect(Head, Second), Tail...);
 }
 
 template <class SecondTy, class... RestTy>
@@ -942,7 +1060,7 @@ private:
   /// Return a range set subtracting zero from \p Domain.
   RangeSet assumeNonZero(RangeSet Domain, QualType T) {
     APSIntType IntType = ValueFactory.getAPSIntType(T);
-    return Domain.Delete(ValueFactory, RangeFactory, IntType.getZeroValue());
+    return RangeFactory.deletePoint(Domain, IntType.getZeroValue());
   }
 
   // FIXME: Once SValBuilder supports unary minus, we should use SValBuilder to
@@ -965,7 +1083,7 @@ private:
             SymMgr.getSymSymExpr(SSE->getRHS(), BO_Sub, SSE->getLHS(), T);
 
         if (const RangeSet *NegatedRange = getConstraint(State, NegatedSym)) {
-          return NegatedRange->Negate(ValueFactory, RangeFactory);
+          return RangeFactory.negate(*NegatedRange);
         }
       }
     }
@@ -1259,7 +1377,7 @@ RangeSet SymbolicRangeInferrer::VisitBinaryOperator<BO_Rem>(Range LHS,
 class RangeConstraintManager : public RangedConstraintManager {
 public:
   RangeConstraintManager(ExprEngine *EE, SValBuilder &SVB)
-      : RangedConstraintManager(EE, SVB) {}
+      : RangedConstraintManager(EE, SVB), F(getBasicVals()) {}
 
   //===------------------------------------------------------------------===//
   // Implementation for interface from ConstraintManager.
@@ -1397,15 +1515,6 @@ private:
     return EquivalenceClass::merge(getBasicVals(), F, State, LHS, RHS);
   }
 
-  LLVM_NODISCARD LLVM_ATTRIBUTE_UNUSED static bool
-  areFeasible(ConstraintRangeTy Constraints) {
-    return llvm::none_of(
-        Constraints,
-        [](const std::pair<EquivalenceClass, RangeSet> &ClassConstraint) {
-          return ClassConstraint.second.isEmpty();
-        });
-  }
-
   LLVM_NODISCARD ProgramStateRef setConstraint(ProgramStateRef State,
                                                EquivalenceClass Class,
                                                RangeSet Constraint) {
@@ -1424,11 +1533,11 @@ private:
     // be simply a constant because we can't reason about range disequalities.
     if (const llvm::APSInt *Point = Constraint.getConcreteValue())
       for (EquivalenceClass DisequalClass : Class.getDisequalClasses(State)) {
-        RangeSet UpdatedConstraint =
-            getRange(State, DisequalClass).Delete(getBasicVals(), F, *Point);
+        RangeSet UpdatedConstraint = getRange(State, DisequalClass);
+        UpdatedConstraint = F.deletePoint(UpdatedConstraint, *Point);
 
         // If we end up with at least one of the disequal classes to be
-        // constrainted with an empty range-set, the state is infeasible.
+        // constrained with an empty range-set, the state is infeasible.
         if (UpdatedConstraint.isEmpty())
           return nullptr;
 
@@ -1574,6 +1683,9 @@ EquivalenceClass::mergeImpl(BasicValueFactory &ValueFactory,
     // Assign new constraints for this class.
     Constraints = CRF.add(Constraints, *this, *NewClassConstraint);
 
+    assert(areFeasible(Constraints) && "Constraint manager shouldn't produce "
+                                       "a state with infeasible constraints");
+
     State = State->set<ConstraintRange>(Constraints);
   }
 
@@ -1644,7 +1756,7 @@ EquivalenceClass::getMembersFactory(ProgramStateRef State) {
   return State->get_context<SymbolSet>();
 }
 
-SymbolSet EquivalenceClass::getClassMembers(ProgramStateRef State) {
+SymbolSet EquivalenceClass::getClassMembers(ProgramStateRef State) const {
   if (const SymbolSet *Members = State->get<ClassMembers>(*this))
     return *Members;
 
@@ -1654,12 +1766,12 @@ SymbolSet EquivalenceClass::getClassMembers(ProgramStateRef State) {
   return F.add(F.getEmptySet(), getRepresentativeSymbol());
 }
 
-bool EquivalenceClass::isTrivial(ProgramStateRef State) {
+bool EquivalenceClass::isTrivial(ProgramStateRef State) const {
   return State->get<ClassMembers>(*this) == nullptr;
 }
 
 bool EquivalenceClass::isTriviallyDead(ProgramStateRef State,
-                                       SymbolReaper &Reaper) {
+                                       SymbolReaper &Reaper) const {
   return isTrivial(State) && Reaper.isDead(getRepresentativeSymbol());
 }
 
@@ -1694,10 +1806,14 @@ EquivalenceClass::markDisequal(BasicValueFactory &VF, RangeSet::Factory &RF,
 
   // Disequality is a symmetric relation, so if we mark A as disequal to B,
   // we should also mark B as disequalt to A.
-  addToDisequalityInfo(DisequalityInfo, Constraints, VF, RF, State, *this,
-                       Other);
-  addToDisequalityInfo(DisequalityInfo, Constraints, VF, RF, State, Other,
-                       *this);
+  if (!addToDisequalityInfo(DisequalityInfo, Constraints, VF, RF, State, *this,
+                            Other) ||
+      !addToDisequalityInfo(DisequalityInfo, Constraints, VF, RF, State, Other,
+                            *this))
+    return nullptr;
+
+  assert(areFeasible(Constraints) && "Constraint manager shouldn't produce "
+                                     "a state with infeasible constraints");
 
   State = State->set<DisequalityMap>(DisequalityInfo);
   State = State->set<ConstraintRange>(Constraints);
@@ -1705,7 +1821,7 @@ EquivalenceClass::markDisequal(BasicValueFactory &VF, RangeSet::Factory &RF,
   return State;
 }
 
-inline void EquivalenceClass::addToDisequalityInfo(
+inline bool EquivalenceClass::addToDisequalityInfo(
     DisequalityMapTy &Info, ConstraintRangeTy &Constraints,
     BasicValueFactory &VF, RangeSet::Factory &RF, ProgramStateRef State,
     EquivalenceClass First, EquivalenceClass Second) {
@@ -1733,9 +1849,17 @@ inline void EquivalenceClass::addToDisequalityInfo(
       RangeSet FirstConstraint = SymbolicRangeInferrer::inferRange(
           VF, RF, State, First.getRepresentativeSymbol());
 
-      FirstConstraint = FirstConstraint.Delete(VF, RF, *Point);
+      FirstConstraint = RF.deletePoint(FirstConstraint, *Point);
+
+      // If the First class is about to be constrained with an empty
+      // range-set, the state is infeasible.
+      if (FirstConstraint.isEmpty())
+        return false;
+
       Constraints = CRF.add(Constraints, First, FirstConstraint);
     }
+
+  return true;
 }
 
 inline Optional<bool> EquivalenceClass::areEqual(ProgramStateRef State,
@@ -1884,7 +2008,7 @@ ConditionTruthVal RangeConstraintManager::checkNull(ProgramStateRef State,
   llvm::APSInt Zero = IntType.getZeroValue();
 
   // Check if zero is in the set of possible values.
-  if (Ranges->Intersect(BV, F, Zero, Zero).isEmpty())
+  if (!Ranges->contains(Zero))
     return false;
 
   // Zero is a possible value, but it is not the /only/ possible value.
@@ -2070,7 +2194,8 @@ RangeConstraintManager::assumeSymNE(ProgramStateRef St, SymbolRef Sym,
 
   llvm::APSInt Point = AdjustmentType.convert(Int) - Adjustment;
 
-  RangeSet New = getRange(St, Sym).Delete(getBasicVals(), F, Point);
+  RangeSet New = getRange(St, Sym);
+  New = F.deletePoint(New, Point);
 
   return trackNE(New, St, Sym, Int, Adjustment);
 }
@@ -2086,7 +2211,8 @@ RangeConstraintManager::assumeSymEQ(ProgramStateRef St, SymbolRef Sym,
 
   // [Int-Adjustment, Int-Adjustment]
   llvm::APSInt AdjInt = AdjustmentType.convert(Int) - Adjustment;
-  RangeSet New = getRange(St, Sym).Intersect(getBasicVals(), F, AdjInt, AdjInt);
+  RangeSet New = getRange(St, Sym);
+  New = F.intersect(New, AdjInt);
 
   return trackEQ(New, St, Sym, Int, Adjustment);
 }
@@ -2116,7 +2242,8 @@ RangeSet RangeConstraintManager::getSymLTRange(ProgramStateRef St,
   llvm::APSInt Upper = ComparisonVal - Adjustment;
   --Upper;
 
-  return getRange(St, Sym).Intersect(getBasicVals(), F, Lower, Upper);
+  RangeSet Result = getRange(St, Sym);
+  return F.intersect(Result, Lower, Upper);
 }
 
 ProgramStateRef
@@ -2152,7 +2279,8 @@ RangeSet RangeConstraintManager::getSymGTRange(ProgramStateRef St,
   llvm::APSInt Upper = Max - Adjustment;
   ++Lower;
 
-  return getRange(St, Sym).Intersect(getBasicVals(), F, Lower, Upper);
+  RangeSet SymRange = getRange(St, Sym);
+  return F.intersect(SymRange, Lower, Upper);
 }
 
 ProgramStateRef
@@ -2188,7 +2316,8 @@ RangeSet RangeConstraintManager::getSymGERange(ProgramStateRef St,
   llvm::APSInt Lower = ComparisonVal - Adjustment;
   llvm::APSInt Upper = Max - Adjustment;
 
-  return getRange(St, Sym).Intersect(getBasicVals(), F, Lower, Upper);
+  RangeSet SymRange = getRange(St, Sym);
+  return F.intersect(SymRange, Lower, Upper);
 }
 
 ProgramStateRef
@@ -2224,7 +2353,8 @@ RangeConstraintManager::getSymLERange(llvm::function_ref<RangeSet()> RS,
   llvm::APSInt Lower = Min - Adjustment;
   llvm::APSInt Upper = ComparisonVal - Adjustment;
 
-  return RS().Intersect(getBasicVals(), F, Lower, Upper);
+  RangeSet Default = RS();
+  return F.intersect(Default, Lower, Upper);
 }
 
 RangeSet RangeConstraintManager::getSymLERange(ProgramStateRef St,
@@ -2257,7 +2387,7 @@ ProgramStateRef RangeConstraintManager::assumeSymOutsideInclusiveRange(
     const llvm::APSInt &To, const llvm::APSInt &Adjustment) {
   RangeSet RangeLT = getSymLTRange(State, Sym, From, Adjustment);
   RangeSet RangeGT = getSymGTRange(State, Sym, To, Adjustment);
-  RangeSet New(RangeLT.addRange(F, RangeGT));
+  RangeSet New(F.add(RangeLT, RangeGT));
   return New.isEmpty() ? nullptr : setConstraint(State, Sym, New);
 }
 
@@ -2292,7 +2422,7 @@ void RangeConstraintManager::printJson(raw_ostream &Out, ProgramStateRef State,
       }
       Indent(Out, Space, IsDot)
           << "{ \"symbol\": \"" << ClassMember << "\", \"range\": \"";
-      P.second.print(Out);
+      P.second.dump(Out);
       Out << "\" }";
     }
   }
diff --git a/clang/lib/StaticAnalyzer/Core/RangedConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/RangedConstraintManager.cpp
index e7a03e6ed582e2d2beb76a45635a094f272e1259..1b8945fb66af064b28b2e4add7b8e239967ec993 100644
--- a/clang/lib/StaticAnalyzer/Core/RangedConstraintManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RangedConstraintManager.cpp
@@ -220,5 +220,4 @@ void RangedConstraintManager::computeAdjustment(SymbolRef &Sym,
 }
 
 } // end of namespace ento
-
 } // end of namespace clang
diff --git a/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp b/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
index facadaf1225f8ae4f6803c3aa9108a2850acc401..872616fedb4eb26f4b912f444cfe39eadddba79a 100644
--- a/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
+++ b/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
@@ -652,6 +652,8 @@ SVal SimpleSValBuilder::evalBinOpNN(ProgramStateRef state,
         if (LHSValue == 0)
           return evalCastFromNonLoc(lhs, resultTy);
         return makeSymExprValNN(op, InputLHS, InputRHS, resultTy);
+      case BO_Div:
+        // 0 / x == 0
       case BO_Rem:
         // 0 % x == 0
         if (LHSValue == 0)
diff --git a/clang/lib/Tooling/ArgumentsAdjusters.cpp b/clang/lib/Tooling/ArgumentsAdjusters.cpp
index bcfb5b39a0770635a78f0c3ce3582e1a3236e064..d94673bd2ab9099e17499e90f4b787d3da378231 100644
--- a/clang/lib/Tooling/ArgumentsAdjusters.cpp
+++ b/clang/lib/Tooling/ArgumentsAdjusters.cpp
@@ -62,7 +62,8 @@ ArgumentsAdjuster getClangSyntaxOnlyAdjuster() {
         HasSyntaxOnly = true;
     }
     if (!HasSyntaxOnly)
-      AdjustedArgs.push_back("-fsyntax-only");
+      AdjustedArgs =
+          getInsertArgumentAdjuster("-fsyntax-only")(AdjustedArgs, "");
     return AdjustedArgs;
   };
 }
@@ -137,7 +138,7 @@ ArgumentsAdjuster getInsertArgumentAdjuster(const CommandLineArguments &Extra,
 
     CommandLineArguments::iterator I;
     if (Pos == ArgumentInsertPosition::END) {
-      I = Return.end();
+      I = std::find(Return.begin(), Return.end(), "--");
     } else {
       I = Return.begin();
       ++I; // To leave the program name in place
diff --git a/clang/lib/Tooling/CMakeLists.txt b/clang/lib/Tooling/CMakeLists.txt
index e820f63d5b3d43f636b54f0e1e61e09ea1cccf32..0a6fb99152dc79b3f7257f1e9e5beb20f74190be 100644
--- a/clang/lib/Tooling/CMakeLists.txt
+++ b/clang/lib/Tooling/CMakeLists.txt
@@ -40,6 +40,9 @@ namespace tooling {
 NodeLocationAccessors NodeIntrospection::GetLocations(clang::Stmt const *) {
   return {};
 }
+NodeLocationAccessors NodeIntrospection::GetLocations(clang::Decl const *) {
+  return {};
+}
 NodeLocationAccessors
 NodeIntrospection::GetLocations(clang::DynTypedNode const &) {
   return {};
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
index 16040c2f4626022151f4a587eb0642d2b340dcff..3c61242da575deae5b72e3fbcc79f5b4ec544c64 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
@@ -14,8 +14,8 @@ namespace tooling{
 namespace dependencies{
 
 std::vector<std::string> FullDependencies::getAdditionalCommandLine(
-    std::function<StringRef(ClangModuleDep)> LookupPCMPath,
-    std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const {
+    std::function<StringRef(ModuleID)> LookupPCMPath,
+    std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const {
   std::vector<std::string> Ret = AdditionalNonPathCommandLine;
 
   dependencies::detail::appendCommonModuleArguments(
@@ -109,7 +109,7 @@ DependencyScanningTool::getFullDependencies(
     }
 
     void handleModuleDependency(ModuleDeps MD) override {
-      ClangModuleDeps[MD.ContextHash + MD.ModuleName] = std::move(MD);
+      ClangModuleDeps[MD.ID.ContextHash + MD.ID.ModuleName] = std::move(MD);
     }
 
     void handleContextHash(std::string Hash) override {
@@ -119,14 +119,14 @@ DependencyScanningTool::getFullDependencies(
     FullDependenciesResult getFullDependencies() const {
       FullDependencies FD;
 
-      FD.ContextHash = std::move(ContextHash);
+      FD.ID.ContextHash = std::move(ContextHash);
 
       FD.FileDeps.assign(Dependencies.begin(), Dependencies.end());
 
       for (auto &&M : ClangModuleDeps) {
         auto &MD = M.second;
         if (MD.ImportedByMainFile)
-          FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash});
+          FD.ClangModuleDeps.push_back({MD.ID.ModuleName, ContextHash});
       }
 
       FullDependenciesResult FDR;
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index f74ce7304df5ab01f7be8e6712c4e0fddf48cdfc..1fee831143e660e0f4217f0bbbc59d0d5dc43627 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -18,8 +18,8 @@ using namespace tooling;
 using namespace dependencies;
 
 std::vector<std::string> ModuleDeps::getFullCommandLine(
-    std::function<StringRef(ClangModuleDep)> LookupPCMPath,
-    std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const {
+    std::function<StringRef(ModuleID)> LookupPCMPath,
+    std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const {
   std::vector<std::string> Ret = NonPathCommandLine;
 
   // TODO: Build full command line. That also means capturing the original
@@ -32,21 +32,21 @@ std::vector<std::string> ModuleDeps::getFullCommandLine(
 }
 
 void dependencies::detail::appendCommonModuleArguments(
-    llvm::ArrayRef<ClangModuleDep> Modules,
-    std::function<StringRef(ClangModuleDep)> LookupPCMPath,
-    std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps,
+    llvm::ArrayRef<ModuleID> Modules,
+    std::function<StringRef(ModuleID)> LookupPCMPath,
+    std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps,
     std::vector<std::string> &Result) {
   llvm::StringSet<> AlreadyAdded;
 
-  std::function<void(llvm::ArrayRef<ClangModuleDep>)> AddArgs =
-      [&](llvm::ArrayRef<ClangModuleDep> Modules) {
-        for (const ClangModuleDep &CMD : Modules) {
-          if (!AlreadyAdded.insert(CMD.ModuleName + CMD.ContextHash).second)
+  std::function<void(llvm::ArrayRef<ModuleID>)> AddArgs =
+      [&](llvm::ArrayRef<ModuleID> Modules) {
+        for (const ModuleID &MID : Modules) {
+          if (!AlreadyAdded.insert(MID.ModuleName + MID.ContextHash).second)
             continue;
-          const ModuleDeps &M = LookupModuleDeps(CMD);
+          const ModuleDeps &M = LookupModuleDeps(MID);
           // Depth first traversal.
           AddArgs(M.ClangModuleDeps);
-          Result.push_back(("-fmodule-file=" + LookupPCMPath(CMD)).str());
+          Result.push_back(("-fmodule-file=" + LookupPCMPath(MID)).str());
           if (!M.ClangModuleMapFile.empty()) {
             Result.push_back("-fmodule-map-file=" + M.ClangModuleMapFile);
           }
@@ -79,7 +79,7 @@ void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
   // We do not want #line markers to affect dependency generation!
   if (Optional<StringRef> Filename =
           SM.getNonBuiltinFilenameForID(SM.getFileID(SM.getExpansionLoc(Loc))))
-    MDC.MainDeps.push_back(
+    MDC.FileDeps.push_back(
         std::string(llvm::sys::path::remove_leading_dotslash(*Filename)));
 }
 
@@ -91,7 +91,7 @@ void ModuleDepCollectorPP::InclusionDirective(
   if (!File && !Imported) {
     // This is a non-modular include that HeaderSearch failed to find. Add it
     // here as `FileChanged` will never see it.
-    MDC.MainDeps.push_back(std::string(FileName));
+    MDC.FileDeps.push_back(std::string(FileName));
   }
   handleImport(Imported);
 }
@@ -106,9 +106,10 @@ void ModuleDepCollectorPP::handleImport(const Module *Imported) {
   if (!Imported)
     return;
 
-  MDC.Deps[MDC.ContextHash + Imported->getTopLevelModule()->getFullModuleName()]
+  const Module *TopLevelModule = Imported->getTopLevelModule();
+  MDC.ModularDeps[MDC.ContextHash + TopLevelModule->getFullModuleName()]
       .ImportedByMainFile = true;
-  DirectDeps.insert(Imported->getTopLevelModule());
+  DirectModularDeps.insert(TopLevelModule);
 }
 
 void ModuleDepCollectorPP::EndOfMainFile() {
@@ -116,24 +117,23 @@ void ModuleDepCollectorPP::EndOfMainFile() {
   MDC.MainFile = std::string(
       Instance.getSourceManager().getFileEntryForID(MainFileID)->getName());
 
-  for (const Module *M : DirectDeps) {
+  for (const Module *M : DirectModularDeps)
     handleTopLevelModule(M);
-  }
 
-  for (auto &&I : MDC.Deps)
+  for (auto &&I : MDC.ModularDeps)
     MDC.Consumer.handleModuleDependency(I.second);
 
-  for (auto &&I : MDC.MainDeps)
+  for (auto &&I : MDC.FileDeps)
     MDC.Consumer.handleFileDependency(*MDC.Opts, I);
 }
 
 void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   assert(M == M->getTopLevelModule() && "Expected top level module!");
 
-  auto ModI = MDC.Deps.insert(
+  auto ModI = MDC.ModularDeps.insert(
       std::make_pair(MDC.ContextHash + M->getFullModuleName(), ModuleDeps{}));
 
-  if (!ModI.first->second.ModuleName.empty())
+  if (!ModI.first->second.ID.ModuleName.empty())
     return;
 
   ModuleDeps &MD = ModI.first->second;
@@ -144,9 +144,9 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
                                    .getContainingModuleMapFile(M);
 
   MD.ClangModuleMapFile = std::string(ModuleMap ? ModuleMap->getName() : "");
-  MD.ModuleName = M->getFullModuleName();
+  MD.ID.ModuleName = M->getFullModuleName();
   MD.ImplicitModulePCMPath = std::string(M->getASTFile()->getName());
-  MD.ContextHash = MDC.ContextHash;
+  MD.ID.ContextHash = MDC.ContextHash;
   serialization::ModuleFile *MF =
       MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile());
   MDC.Instance.getASTReader()->visitInputFiles(
diff --git a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp
index ff279d9425d81ce6233c84df762eea2730a5cdf2..d611261dd1a74d5feef0d043ab6f003fb879dba5 100644
--- a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp
+++ b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp
@@ -26,7 +26,8 @@ ASTSrcLocProcessor::ASTSrcLocProcessor(StringRef JsonPath)
           isDefinition(),
           isSameOrDerivedFrom(
               // TODO: Extend this with other clades
-              namedDecl(hasName("clang::Stmt")).bind("nodeClade")),
+              namedDecl(hasAnyName("clang::Stmt", "clang::Decl"))
+                  .bind("nodeClade")),
           optionally(isDerivedFrom(cxxRecordDecl().bind("derivedFrom"))))
           .bind("className"),
       this);
@@ -79,17 +80,16 @@ llvm::json::Object toJSON(llvm::StringMap<ClassData> const &Obj) {
   return JsonObj;
 }
 
-void WriteJSON(std::string JsonPath,
-               llvm::StringMap<StringRef> const &ClassInheritance,
-               llvm::StringMap<std::vector<StringRef>> const &ClassesInClade,
-               llvm::StringMap<ClassData> const &ClassEntries) {
+void WriteJSON(std::string JsonPath, llvm::json::Object &&ClassInheritance,
+               llvm::json::Object &&ClassesInClade,
+               llvm::json::Object &&ClassEntries) {
   llvm::json::Object JsonObj;
 
   using llvm::json::toJSON;
 
-  JsonObj["classInheritance"] = ::toJSON(ClassInheritance);
-  JsonObj["classesInClade"] = ::toJSON(ClassesInClade);
-  JsonObj["classEntries"] = ::toJSON(ClassEntries);
+  JsonObj["classInheritance"] = std::move(ClassInheritance);
+  JsonObj["classesInClade"] = std::move(ClassesInClade);
+  JsonObj["classEntries"] = std::move(ClassEntries);
 
   std::error_code EC;
   llvm::raw_fd_ostream JsonOut(JsonPath, EC, llvm::sys::fs::F_Text);
@@ -101,9 +101,12 @@ void WriteJSON(std::string JsonPath,
 }
 
 void ASTSrcLocProcessor::generate() {
-  WriteJSON(JsonPath, ClassInheritance, ClassesInClade, ClassEntries);
+  WriteJSON(JsonPath, ::toJSON(ClassInheritance), ::toJSON(ClassesInClade),
+            ::toJSON(ClassEntries));
 }
 
+void ASTSrcLocProcessor::generateEmpty() { WriteJSON(JsonPath, {}, {}, {}); }
+
 std::vector<std::string>
 CaptureMethods(std::string TypeString, const clang::CXXRecordDecl *ASTClass,
                const MatchFinder::MatchResult &Result) {
diff --git a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h
index 00994758e03ccf406571238f05c289d94fb0cf19..5d848f48ed54804b7f9b879dcda56e34f56f0b70 100644
--- a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h
+++ b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h
@@ -30,6 +30,7 @@ public:
                                                  StringRef File);
 
   void generate();
+  void generateEmpty();
 
 private:
   void run(const ast_matchers::MatchFinder::MatchResult &Result) override;
diff --git a/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp b/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp
index 6615f865221d98a7f717da5802419d58f8d15313..8328977178ccfcc5e2b266fbc917fa5b7338aad8 100644
--- a/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp
+++ b/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp
@@ -48,7 +48,13 @@ class ASTSrcLocGenerationAction : public clang::ASTFrontendAction {
 public:
   ASTSrcLocGenerationAction() : Processor(JsonOutputPath) {}
 
-  ~ASTSrcLocGenerationAction() { Processor.generate(); }
+  void ExecuteAction() override {
+    clang::ASTFrontendAction::ExecuteAction();
+    if (getCompilerInstance().getDiagnostics().getNumErrors() > 0)
+      Processor.generateEmpty();
+    else
+      Processor.generate();
+  }
 
   std::unique_ptr<clang::ASTConsumer>
   CreateASTConsumer(clang::CompilerInstance &Compiler,
@@ -92,7 +98,13 @@ int main(int argc, const char **argv) {
   auto ParsedArgs = Opts.ParseArgs(llvm::makeArrayRef(Argv).slice(1),
                                    MissingArgIndex, MissingArgCount);
   ParseDiagnosticArgs(*DiagOpts, ParsedArgs);
-  TextDiagnosticPrinter DiagnosticPrinter(llvm::errs(), &*DiagOpts);
+
+  // Don't output diagnostics, because common scenarios such as
+  // cross-compiling fail with diagnostics.  This is not fatal, but
+  // just causes attempts to use the introspection API to return no data.
+  std::string Str;
+  llvm::raw_string_ostream OS(Str);
+  TextDiagnosticPrinter DiagnosticPrinter(OS, &*DiagOpts);
   DiagnosticsEngine Diagnostics(
       IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts,
       &DiagnosticPrinter, false);
@@ -129,6 +141,8 @@ int main(int argc, const char **argv) {
   if (!Compiler.hasDiagnostics())
     return 1;
 
+  // Suppress "2 errors generated" or similar messages
+  Compiler.getDiagnosticOpts().ShowCarets = false;
   Compiler.createSourceManager(Files);
 
   ASTSrcLocGenerationAction ScopedToolAction;
diff --git a/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py b/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py
index b4d31d3e49a944b4a4e364c68878b84e9779c3a0..15a373e5248049ea3176a4fbca5c9594b5792898 100755
--- a/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py
+++ b/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py
@@ -177,6 +177,9 @@ namespace tooling {
 NodeLocationAccessors NodeIntrospection::GetLocations(clang::Stmt const *) {
   return {};
 }
+NodeLocationAccessors NodeIntrospection::GetLocations(clang::Decl const *) {
+  return {};
+}
 NodeLocationAccessors
 NodeIntrospection::GetLocations(clang::DynTypedNode const &) {
   return {};
diff --git a/clang/lib/Tooling/InterpolatingCompilationDatabase.cpp b/clang/lib/Tooling/InterpolatingCompilationDatabase.cpp
index 6f97d2867ae5d4a46546e6379b8789ee1c65fe82..3b65504b98ea3abc5d2a8cfae0c63e1f64b5147a 100644
--- a/clang/lib/Tooling/InterpolatingCompilationDatabase.cpp
+++ b/clang/lib/Tooling/InterpolatingCompilationDatabase.cpp
@@ -177,6 +177,10 @@ struct TransferableCommand {
                            Opt.matches(OPT__SLASH_Fo))))
         continue;
 
+      // ...including when the inputs are passed after --.
+      if (Opt.matches(OPT__DASH_DASH))
+        break;
+
       // Strip -x, but record the overridden language.
       if (const auto GivenType = tryParseTypeArg(*Arg)) {
         Type = *GivenType;
@@ -235,6 +239,8 @@ struct TransferableCommand {
           llvm::Twine(ClangCLMode ? "/std:" : "-std=") +
           LangStandard::getLangStandardForKind(Std).getName()).str());
     }
+    if (Filename.startswith("-") || (ClangCLMode && Filename.startswith("/")))
+      Result.CommandLine.push_back("--");
     Result.CommandLine.push_back(std::string(Filename));
     return Result;
   }
diff --git a/clang/lib/Tooling/JSONCompilationDatabase.cpp b/clang/lib/Tooling/JSONCompilationDatabase.cpp
index 2d8847a7a327f91099876bd4ff8bb0550d504fae..97ba7e411fbb37d59049525c088729a17ca90734 100644
--- a/clang/lib/Tooling/JSONCompilationDatabase.cpp
+++ b/clang/lib/Tooling/JSONCompilationDatabase.cpp
@@ -198,7 +198,7 @@ JSONCompilationDatabase::loadFromFile(StringRef FilePath,
                                       JSONCommandLineSyntax Syntax) {
   // Don't mmap: if we're a long-lived process, the build system may overwrite.
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
-      llvm::MemoryBuffer::getFile(FilePath, /*FileSize=*/-1,
+      llvm::MemoryBuffer::getFile(FilePath, /*IsText=*/false,
                                   /*RequiresNullTerminator=*/true,
                                   /*IsVolatile=*/true);
   if (std::error_code Result = DatabaseBuffer.getError()) {
diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp
index 234df9cb7182bc738c5bfc98990d85e20d4db099..2326e89ea48bae5824ecf8843ea98041f6a36025 100644
--- a/clang/lib/Tooling/Syntax/Tokens.cpp
+++ b/clang/lib/Tooling/Syntax/Tokens.cpp
@@ -183,7 +183,31 @@ llvm::StringRef FileRange::text(const SourceManager &SM) const {
   return Text.substr(Begin, length());
 }
 
+void TokenBuffer::indexExpandedTokens() {
+  // No-op if the index is already created.
+  if (!ExpandedTokIndex.empty())
+    return;
+  ExpandedTokIndex.reserve(ExpandedTokens.size());
+  // Index ExpandedTokens for faster lookups by SourceLocation.
+  for (size_t I = 0, E = ExpandedTokens.size(); I != E; ++I)
+    ExpandedTokIndex[ExpandedTokens[I].location()] = I;
+}
+
 llvm::ArrayRef<syntax::Token> TokenBuffer::expandedTokens(SourceRange R) const {
+  if (!ExpandedTokIndex.empty()) {
+    // Quick lookup if `R` is a token range.
+    // This is a huge win since majority of the users use ranges provided by an
+    // AST. Ranges in AST are token ranges from expanded token stream.
+    const auto B = ExpandedTokIndex.find(R.getBegin());
+    const auto E = ExpandedTokIndex.find(R.getEnd());
+    if (B != ExpandedTokIndex.end() && E != ExpandedTokIndex.end()) {
+      // Add 1 to End to make a half-open range.
+      return {ExpandedTokens.data() + B->getSecond(),
+              ExpandedTokens.data() + E->getSecond() + 1};
+    }
+  }
+  // Slow case. Use `isBeforeInTranslationUnit` to binary search for the
+  // required range.
   return getTokensCovering(expandedTokens(), R, *SourceMgr);
 }
 
diff --git a/clang/lib/Tooling/Tooling.cpp b/clang/lib/Tooling/Tooling.cpp
index 79851ac723da0f93e421af6b4f8b185679dd0172..b28e8f6a7c967fdd2f18bff06613d28690d2b66e 100644
--- a/clang/lib/Tooling/Tooling.cpp
+++ b/clang/lib/Tooling/Tooling.cpp
@@ -440,8 +440,9 @@ static void injectResourceDir(CommandLineArguments &Args, const char *Argv0,
       return;
 
   // If there's no override in place add our resource dir.
-  Args.push_back("-resource-dir=" +
-                 CompilerInvocation::GetResourcesPath(Argv0, MainAddr));
+  Args = getInsertArgumentAdjuster(
+      ("-resource-dir=" + CompilerInvocation::GetResourcesPath(Argv0, MainAddr))
+          .c_str())(Args, "");
 }
 
 int ClangTool::run(ToolAction *Action) {
diff --git a/clang/test/AST/alignas_maybe_odr_cleanup.cpp b/clang/test/AST/alignas_maybe_odr_cleanup.cpp
index 3adef4cba14403f871b79e862be32f3ec2a6e630..ed34930e98a05a9f645077d9213c73b3889f6123 100644
--- a/clang/test/AST/alignas_maybe_odr_cleanup.cpp
+++ b/clang/test/AST/alignas_maybe_odr_cleanup.cpp
@@ -8,7 +8,7 @@
 // RUN: | FileCheck %s
 
 struct FOO {
-  static const int vec_align_bytes = 32;
+  static const int vec_align_bytes = 16;
   void foo() {
     double a alignas(vec_align_bytes);
     ;
@@ -17,7 +17,7 @@ struct FOO {
 
 // CHECK:      |   `-AlignedAttr {{.*}} <col:14> alignas
 // CHECK-NEXT:      |     `-ConstantExpr {{.*}} <col:22> 'int'
-// CHECK-NEXT:      |       |-value: Int 32
+// CHECK-NEXT:      |       |-value: Int 16
 // CHECK-NEXT:      |       `-ImplicitCastExpr {{.*}} <col:22> 'int' <LValueToRValue>
 // CHECK-NEXT:      |         `-DeclRefExpr {{.*}} <col:22> 'const int' lvalue Var {{.*}} 'vec_align_bytes' 'const int' non_odr_use_constant
 // CHECK-NEXT:      `-NullStmt {{.*}} <line:14:5>
diff --git a/clang/test/AST/ast-print-int128.cpp b/clang/test/AST/ast-print-int128.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..51d15b609f0bfd7528a3bb7569a633d23cfba169
--- /dev/null
+++ b/clang/test/AST/ast-print-int128.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -ast-print -std=c++20 %s -o - -triple x86_64-linux | FileCheck %s
+
+template <bool>
+struct enable_if {
+};
+
+template <__uint128_t x, typename = typename enable_if<x != 0>::type>
+void f();
+
+template <__int128_t>
+void f();
+
+using T = decltype(f<0>());
+
+// CHECK: using T = decltype(f<0>());
diff --git a/clang/test/Analysis/Inputs/expected-plists/edges-new.mm.plist b/clang/test/Analysis/Inputs/expected-plists/edges-new.mm.plist
index 74e11075fe3d4feac21a81203264b8193f62f59b..1d82f3cd84242b272f2c96bce90d71ed92b3cb5c 100644
--- a/clang/test/Analysis/Inputs/expected-plists/edges-new.mm.plist
+++ b/clang/test/Analysis/Inputs/expected-plists/edges-new.mm.plist
@@ -2368,7 +2368,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;x&apos; is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead increment</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -11409,7 +11409,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;foo&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
diff --git a/clang/test/Analysis/Inputs/expected-plists/objc-arc.m.plist b/clang/test/Analysis/Inputs/expected-plists/objc-arc.m.plist
index d3a1a5c6c47fd6485a89a7d46e3ef171fd898ce4..b8bc73611111083a3088f4ab2d83964a61e61869 100644
--- a/clang/test/Analysis/Inputs/expected-plists/objc-arc.m.plist
+++ b/clang/test/Analysis/Inputs/expected-plists/objc-arc.m.plist
@@ -382,7 +382,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;x&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -450,7 +450,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;obj1&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -518,7 +518,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;obj4&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -586,7 +586,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;obj5&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -654,7 +654,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;obj6&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -1064,7 +1064,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;cf1&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -1132,7 +1132,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;cf2&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -1200,7 +1200,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;cf3&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -1268,7 +1268,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;cf4&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
diff --git a/clang/test/Analysis/Inputs/expected-plists/plist-output.m.plist b/clang/test/Analysis/Inputs/expected-plists/plist-output.m.plist
index 76fec546267cdbd41c768a294ff671babed30055..b7ffbf5b5fee24b284fbea682081c6e9ac523729 100644
--- a/clang/test/Analysis/Inputs/expected-plists/plist-output.m.plist
+++ b/clang/test/Analysis/Inputs/expected-plists/plist-output.m.plist
@@ -2169,7 +2169,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;foo&apos; during its initialization is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead initialization</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
@@ -5654,7 +5654,7 @@
     </dict>
    </array>
    <key>description</key><string>Value stored to &apos;x&apos; is never read</string>
-   <key>category</key><string>Dead store</string>
+   <key>category</key><string>Unused code</string>
    <key>type</key><string>Dead increment</string>
    <key>check_name</key><string>deadcode.DeadStores</string>
    <!-- This hash is experimental and going to change! -->
diff --git a/clang/test/Analysis/PR49642.c b/clang/test/Analysis/PR49642.c
new file mode 100644
index 0000000000000000000000000000000000000000..af691d6afd6fd3b0d3da781eb3fd812184b90078
--- /dev/null
+++ b/clang/test/Analysis/PR49642.c
@@ -0,0 +1,24 @@
+// RUN: %clang_analyze_cc1 -w -verify %s \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctions
+
+// expected-no-diagnostics
+
+typedef ssize_t;
+b;
+
+unsigned c;
+int write(int, const void *, unsigned long);
+
+a() {
+  d();
+  while (c > 0) {
+    b = write(0, d, c);
+    if (b)
+      c -= b;
+    b < 1;
+  }
+  if (c && c) {
+    //     ^ no-crash
+  }
+}
diff --git a/clang/test/Analysis/zero-operands.c b/clang/test/Analysis/zero-operands.c
new file mode 100644
index 0000000000000000000000000000000000000000..3311c524f81460da67b1b478d633aff7c585b88e
--- /dev/null
+++ b/clang/test/Analysis/zero-operands.c
@@ -0,0 +1,53 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core \
+// RUN:   -analyzer-checker=debug.ExprInspection \
+// RUN:   -verify %s
+
+void clang_analyzer_dump(int);
+
+void test_0_multiplier1(int x, int y) {
+  int a = x < 0; // Eagerly bifurcate.
+  clang_analyzer_dump(a);
+  // expected-warning@-1{{0 S32b}}
+  // expected-warning@-2{{1 S32b}}
+
+  int b = a * y;
+  clang_analyzer_dump(b);
+  // expected-warning@-1{{0 S32b}}
+  // expected-warning-re@-2{{reg_${{[[:digit:]]+}}<int y>}}
+}
+
+void test_0_multiplier2(int x, int y) {
+  int a = x < 0; // Eagerly bifurcate.
+  clang_analyzer_dump(a);
+  // expected-warning@-1{{0 S32b}}
+  // expected-warning@-2{{1 S32b}}
+
+  int b = y * a;
+  clang_analyzer_dump(b);
+  // expected-warning@-1{{0 S32b}}
+  // expected-warning-re@-2{{reg_${{[[:digit:]]+}}<int y>}}
+}
+
+void test_0_modulo(int x, int y) {
+  int a = x < 0; // Eagerly bifurcate.
+  clang_analyzer_dump(a);
+  // expected-warning@-1{{0 S32b}}
+  // expected-warning@-2{{1 S32b}}
+
+  int b = a % y;
+  clang_analyzer_dump(b);
+  // expected-warning@-1{{0 S32b}}
+  // expected-warning-re@-2{{1 % (reg_${{[[:digit:]]+}}<int y>)}}
+}
+
+void test_0_divisible(int x, int y) {
+  int a = x < 0; // Eagerly bifurcate.
+  clang_analyzer_dump(a);
+  // expected-warning@-1{{0 S32b}}
+  // expected-warning@-2{{1 S32b}}
+
+  int b = a / y;
+  clang_analyzer_dump(b);
+  // expected-warning@-1{{0 S32b}}
+  // expected-warning-re@-2{{1 / (reg_${{[[:digit:]]+}}<int y>)}}
+}
diff --git a/clang/test/CXX/class/class.init/class.copy.elision/p3.cpp b/clang/test/CXX/class/class.init/class.copy.elision/p3.cpp
index 29d818602537da2746eecbd0cac5c6be3d8d2dbf..e4056221b4f3ce466fbfaf966a6833b827842d44 100644
--- a/clang/test/CXX/class/class.init/class.copy.elision/p3.cpp
+++ b/clang/test/CXX/class/class.init/class.copy.elision/p3.cpp
@@ -292,3 +292,108 @@ NeedValue test_4_3() {
   return b; // cxx20-error {{calling a private constructor of class 'test_ctor_param_rvalue_ref::B2'}}
 }
 } // namespace test_ctor_param_rvalue_ref
+
+namespace test_lvalue_ref_is_not_moved_from {
+
+struct Target {};
+  // expected-note@-1 {{candidate constructor (the implicit copy constructor) not viable}}
+  // expected-note@-2 {{candidate constructor (the implicit move constructor) not viable}}
+  // cxx11_14_17-note@-3 {{candidate constructor (the implicit copy constructor) not viable}}
+  // cxx11_14_17-note@-4 {{candidate constructor (the implicit move constructor) not viable}}
+
+struct CopyOnly {
+  CopyOnly(CopyOnly&&) = delete; // cxx20-note {{has been explicitly marked deleted here}}
+  CopyOnly(CopyOnly&);
+  operator Target() && = delete; // cxx20-note {{has been explicitly marked deleted here}}
+  operator Target() &;
+};
+
+struct MoveOnly {
+  MoveOnly(MoveOnly&&); // expected-note {{copy constructor is implicitly deleted because}}
+    // cxx11_14_17-note@-1 {{copy constructor is implicitly deleted because}}
+  operator Target() &&; // expected-note {{candidate function not viable}}
+    // cxx11_14_17-note@-1 {{candidate function not viable}}
+};
+
+extern CopyOnly copyonly;
+extern MoveOnly moveonly;
+
+CopyOnly t1() {
+    CopyOnly& r = copyonly;
+    return r;
+}
+
+CopyOnly t2() {
+    CopyOnly&& r = static_cast<CopyOnly&&>(copyonly);
+    return r; // cxx20-error {{call to deleted constructor}}
+}
+
+MoveOnly t3() {
+    MoveOnly& r = moveonly;
+    return r; // expected-error {{call to implicitly-deleted copy constructor}}
+}
+
+MoveOnly t4() {
+    MoveOnly&& r = static_cast<MoveOnly&&>(moveonly);
+    return r; // cxx11_14_17-error {{call to implicitly-deleted copy constructor}}
+}
+
+Target t5() {
+    CopyOnly& r = copyonly;
+    return r;
+}
+
+Target t6() {
+    CopyOnly&& r = static_cast<CopyOnly&&>(copyonly);
+    return r; // cxx20-error {{invokes a deleted function}}
+}
+
+Target t7() {
+    MoveOnly& r = moveonly;
+    return r; // expected-error {{no viable conversion}}
+}
+
+Target t8() {
+    MoveOnly&& r = static_cast<MoveOnly&&>(moveonly);
+    return r; // cxx11_14_17-error {{no viable conversion}}
+}
+
+} // namespace test_lvalue_ref_is_not_moved_from
+
+namespace test_rvalue_ref_to_nonobject {
+
+struct CopyOnly {};
+struct MoveOnly {};
+
+struct Target {
+    Target(CopyOnly (&)());
+    Target(CopyOnly (&&)()) = delete;
+    Target(MoveOnly (&)()) = delete; // expected-note {{has been explicitly marked deleted here}}
+      // expected-note@-1 {{has been explicitly marked deleted here}}
+    Target(MoveOnly (&&)());
+};
+
+CopyOnly make_copyonly();
+MoveOnly make_moveonly();
+
+Target t1() {
+    CopyOnly (&r)() = make_copyonly;
+    return r;
+}
+
+Target t2() {
+    CopyOnly (&&r)() = static_cast<CopyOnly(&&)()>(make_copyonly);
+    return r; // OK in all modes; not subject to implicit move
+}
+
+Target t3() {
+    MoveOnly (&r)() = make_moveonly;
+    return r; // expected-error {{invokes a deleted function}}
+}
+
+Target t4() {
+    MoveOnly (&&r)() = static_cast<MoveOnly(&&)()>(make_moveonly);
+    return r; // expected-error {{invokes a deleted function}}
+}
+
+} // namespace test_rvalue_ref_to_nonobject
diff --git a/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.fallthrough/p1.cpp b/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.fallthrough/p1.cpp
index f267d9067bcc31cd7ac701421ded792743ba5d4a..22815bbde9db4067336d32cf4b56212662a173c5 100644
--- a/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.fallthrough/p1.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.fallthrough/p1.cpp
@@ -53,7 +53,7 @@ class [[fallthrough]] C {}; // expected-error {{'fallthrough' attribute cannot b
 [[fallthrough]] // expected-error {{'fallthrough' attribute cannot be applied to a declaration}}
 void g() {
   [[fallthrough]] int n; // expected-error {{'fallthrough' attribute cannot be applied to a declaration}}
-  [[fallthrough]] ++n; // expected-error-re {{{{^}}fallthrough attribute is only allowed on empty statements}}
+  [[fallthrough]] ++n; // expected-error {{'fallthrough' attribute only applies to empty statements}}
 
   switch (n) {
     // FIXME: This should be an error.
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4-1y.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4-1y.cpp
index f8461335b76896ecfee15a240cf3a8926caa8ed5..f2b0e26e29f946eec8c4049727f6c163b242c2e0 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4-1y.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4-1y.cpp
@@ -52,7 +52,10 @@ int test_no_parameter_list()
 {
   static int si = 0;
     auto M = [] { return 5; }; // OK
-    auto M2 = [] -> auto&& { return si; }; // expected-error{{lambda requires '()'}}
+    auto M2 = [] -> auto && { return si; };
+#if __cplusplus <= 202002L
+      // expected-warning@-2{{is a C++2b extension}}
+#endif
     M();
 }
 
diff --git a/clang/test/CodeCompletion/desig-init.cpp b/clang/test/CodeCompletion/desig-init.cpp
index 8a66f4554217f9a54aae5acca5ee8ef578364afd..999f368ba563462d466aa3f9a16e837b53613982 100644
--- a/clang/test/CodeCompletion/desig-init.cpp
+++ b/clang/test/CodeCompletion/desig-init.cpp
@@ -62,3 +62,18 @@ void aux() {
   Test<T> X{.x = T(2)};
   // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:62:14 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC3 %s
 }
+
+namespace signature_regression {
+  // Verify that an old bug is gone: passing an init-list as a constructor arg
+  // would emit overloads as a side-effect.
+  struct S{int x;};
+  int wrongFunction(S);
+  int rightFunction();
+  int dummy = wrongFunction({1});
+  int x = rightFunction();
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:73:25 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-SIGNATURE-REGRESSION %s
+  // CHECK-SIGNATURE-REGRESSION-NOT: OVERLOAD: [#int#]wrongFunction
+  // CHECK-SIGNATURE-REGRESSION:     OVERLOAD: [#int#]rightFunction
+  // CHECK-SIGNATURE-REGRESSION-NOT: OVERLOAD: [#int#]wrongFunction
+}
+
diff --git a/clang/test/CodeGen/PR5060-align.c b/clang/test/CodeGen/PR5060-align.c
index 34293a933aa9e2ba86095ebdbacc34972212498d..6e65175a7115517c91bdcd7ef4bc942073551a2e 100644
--- a/clang/test/CodeGen/PR5060-align.c
+++ b/clang/test/CodeGen/PR5060-align.c
@@ -1,13 +1,13 @@
 // RUN: %clang_cc1 -emit-llvm %s -o - | FileCheck %s
 
-// CHECK: @foo.p = internal global i8 0, align 32
+// CHECK: @foo.p = internal global i8 0, align 16
 char *foo(void) {
-  static char p __attribute__((aligned(32)));
+  static char p __attribute__((aligned(16)));
   return &p;
 }
 
 void bar(long n) {
-  // CHECK: align 32
-  char p[n] __attribute__((aligned(32)));
+  // CHECK: align 16
+  char p[n] __attribute__((aligned(16)));
 }
 
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-generic/vadd.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vadd.c
similarity index 96%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-generic/vadd.c
rename to clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vadd.c
index 33073587e74c6c6543cf2462cc75818d1a7112fb..fc783e7bdac795ab9534ee6e4b66ba4b4b843608 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-generic/vadd.c
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vadd.c
@@ -1,14 +1,14 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
 // RUN:   -target-feature +experimental-zfh -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
 // RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
 // RUN:   -target-feature +experimental-zfh -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
-// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
-// RUN:   -target-feature +experimental-zfh -Werror -Wall -o - %s >/dev/null 2>%t
-// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
 
 // ASM-NOT: warning
-#include <riscv_vector_generic.h>
+#include <riscv_vector.h>
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i8mf8(
 // CHECK-RV32-NEXT:  entry:
@@ -1253,7 +1253,7 @@ vuint64m8_t test_vadd_vx_u64m8(vuint64m8_t op1, uint64_t op2, size_t vl) {
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
 vint8mf8_t test_vadd_vv_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, vint8mf8_t op1, vint8mf8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i8mf8_m(
@@ -1267,7 +1267,7 @@ vint8mf8_t test_vadd_vv_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, vint8mf8_t
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
 vint8mf8_t test_vadd_vx_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, vint8mf8_t op1, int8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i8mf4_m(
@@ -1281,7 +1281,7 @@ vint8mf8_t test_vadd_vx_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, vint8mf8_t
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
 vint8mf4_t test_vadd_vv_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, vint8mf4_t op1, vint8mf4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i8mf4_m(
@@ -1295,7 +1295,7 @@ vint8mf4_t test_vadd_vv_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, vint8mf4_t
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
 vint8mf4_t test_vadd_vx_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, vint8mf4_t op1, int8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i8mf2_m(
@@ -1309,7 +1309,7 @@ vint8mf4_t test_vadd_vx_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, vint8mf4_t
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
 vint8mf2_t test_vadd_vv_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, vint8mf2_t op1, vint8mf2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i8mf2_m(
@@ -1323,7 +1323,7 @@ vint8mf2_t test_vadd_vv_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, vint8mf2_t
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
 vint8mf2_t test_vadd_vx_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, vint8mf2_t op1, int8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i8m1_m(
@@ -1337,7 +1337,7 @@ vint8mf2_t test_vadd_vx_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, vint8mf2_t
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
 vint8m1_t test_vadd_vv_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1, vint8m1_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i8m1_m(
@@ -1351,7 +1351,7 @@ vint8m1_t test_vadd_vv_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1,
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
 vint8m1_t test_vadd_vx_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1, int8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i8m2_m(
@@ -1365,7 +1365,7 @@ vint8m1_t test_vadd_vx_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1,
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 vint8m2_t test_vadd_vv_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, vint8m2_t op1, vint8m2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i8m2_m(
@@ -1379,7 +1379,7 @@ vint8m2_t test_vadd_vv_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, vint8m2_t op1,
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 vint8m2_t test_vadd_vx_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, vint8m2_t op1, int8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i8m4_m(
@@ -1393,7 +1393,7 @@ vint8m2_t test_vadd_vx_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, vint8m2_t op1,
 // CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
 vint8m4_t test_vadd_vv_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, vint8m4_t op1, vint8m4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i8m4_m(
@@ -1407,7 +1407,7 @@ vint8m4_t test_vadd_vv_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, vint8m4_t op1,
 // CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
 vint8m4_t test_vadd_vx_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, vint8m4_t op1, int8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i8m8_m(
@@ -1421,7 +1421,7 @@ vint8m4_t test_vadd_vx_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, vint8m4_t op1,
 // CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
 vint8m8_t test_vadd_vv_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, vint8m8_t op1, vint8m8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i8m8_m(
@@ -1435,7 +1435,7 @@ vint8m8_t test_vadd_vv_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, vint8m8_t op1,
 // CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
 vint8m8_t test_vadd_vx_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, vint8m8_t op1, int8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i16mf4_m(
@@ -1449,7 +1449,7 @@ vint8m8_t test_vadd_vx_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, vint8m8_t op1,
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
 vint16mf4_t test_vadd_vv_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, vint16mf4_t op1, vint16mf4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i16mf4_m(
@@ -1463,7 +1463,7 @@ vint16mf4_t test_vadd_vv_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, vint16m
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
 vint16mf4_t test_vadd_vx_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, vint16mf4_t op1, int16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i16mf2_m(
@@ -1477,7 +1477,7 @@ vint16mf4_t test_vadd_vx_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, vint16m
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
 vint16mf2_t test_vadd_vv_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, vint16mf2_t op1, vint16mf2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i16mf2_m(
@@ -1491,7 +1491,7 @@ vint16mf2_t test_vadd_vv_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, vint16m
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
 vint16mf2_t test_vadd_vx_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, vint16mf2_t op1, int16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i16m1_m(
@@ -1505,7 +1505,7 @@ vint16mf2_t test_vadd_vx_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, vint16m
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
 vint16m1_t test_vadd_vv_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, vint16m1_t op1, vint16m1_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i16m1_m(
@@ -1519,7 +1519,7 @@ vint16m1_t test_vadd_vv_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, vint16m1_t
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
 vint16m1_t test_vadd_vx_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, vint16m1_t op1, int16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i16m2_m(
@@ -1533,7 +1533,7 @@ vint16m1_t test_vadd_vx_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, vint16m1_t
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
 vint16m2_t test_vadd_vv_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, vint16m2_t op1, vint16m2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i16m2_m(
@@ -1547,7 +1547,7 @@ vint16m2_t test_vadd_vv_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, vint16m2_t
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
 vint16m2_t test_vadd_vx_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, vint16m2_t op1, int16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i16m4_m(
@@ -1561,7 +1561,7 @@ vint16m2_t test_vadd_vx_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, vint16m2_t
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
 vint16m4_t test_vadd_vv_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, vint16m4_t op1, vint16m4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i16m4_m(
@@ -1575,7 +1575,7 @@ vint16m4_t test_vadd_vv_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, vint16m4_t
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
 vint16m4_t test_vadd_vx_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, vint16m4_t op1, int16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i16m8_m(
@@ -1589,7 +1589,7 @@ vint16m4_t test_vadd_vx_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, vint16m4_t
 // CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
 vint16m8_t test_vadd_vv_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, vint16m8_t op1, vint16m8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i16m8_m(
@@ -1603,7 +1603,7 @@ vint16m8_t test_vadd_vv_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, vint16m8_t
 // CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
 vint16m8_t test_vadd_vx_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, vint16m8_t op1, int16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i32mf2_m(
@@ -1617,7 +1617,7 @@ vint16m8_t test_vadd_vx_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, vint16m8_t
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
 vint32mf2_t test_vadd_vv_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, vint32mf2_t op1, vint32mf2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i32mf2_m(
@@ -1631,7 +1631,7 @@ vint32mf2_t test_vadd_vv_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, vint32m
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
 vint32mf2_t test_vadd_vx_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, vint32mf2_t op1, int32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i32m1_m(
@@ -1645,7 +1645,7 @@ vint32mf2_t test_vadd_vx_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, vint32m
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
 vint32m1_t test_vadd_vv_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, vint32m1_t op1, vint32m1_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i32m1_m(
@@ -1659,7 +1659,7 @@ vint32m1_t test_vadd_vv_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, vint32m1_t
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
 vint32m1_t test_vadd_vx_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, vint32m1_t op1, int32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i32m2_m(
@@ -1673,7 +1673,7 @@ vint32m1_t test_vadd_vx_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, vint32m1_t
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
 vint32m2_t test_vadd_vv_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, vint32m2_t op1, vint32m2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i32m2_m(
@@ -1687,7 +1687,7 @@ vint32m2_t test_vadd_vv_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, vint32m2_t
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
 vint32m2_t test_vadd_vx_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, vint32m2_t op1, int32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i32m4_m(
@@ -1701,7 +1701,7 @@ vint32m2_t test_vadd_vx_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, vint32m2_t
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
 vint32m4_t test_vadd_vv_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, vint32m4_t op1, vint32m4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i32m4_m(
@@ -1715,7 +1715,7 @@ vint32m4_t test_vadd_vv_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, vint32m4_t
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
 vint32m4_t test_vadd_vx_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, vint32m4_t op1, int32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i32m8_m(
@@ -1729,7 +1729,7 @@ vint32m4_t test_vadd_vx_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, vint32m4_t
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
 vint32m8_t test_vadd_vv_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, vint32m8_t op1, vint32m8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i32m8_m(
@@ -1743,7 +1743,7 @@ vint32m8_t test_vadd_vv_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, vint32m8_t
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
 vint32m8_t test_vadd_vx_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, vint32m8_t op1, int32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i64m1_m(
@@ -1757,7 +1757,7 @@ vint32m8_t test_vadd_vx_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, vint32m8_t
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
 vint64m1_t test_vadd_vv_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, vint64m1_t op1, vint64m1_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i64m1_m(
@@ -1771,7 +1771,7 @@ vint64m1_t test_vadd_vv_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, vint64m1_t
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
 vint64m1_t test_vadd_vx_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, vint64m1_t op1, int64_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i64m2_m(
@@ -1785,7 +1785,7 @@ vint64m1_t test_vadd_vx_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, vint64m1_t
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
 vint64m2_t test_vadd_vv_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, vint64m2_t op1, vint64m2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i64m2_m(
@@ -1799,7 +1799,7 @@ vint64m2_t test_vadd_vv_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, vint64m2_t
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
 vint64m2_t test_vadd_vx_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, vint64m2_t op1, int64_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i64m4_m(
@@ -1813,7 +1813,7 @@ vint64m2_t test_vadd_vx_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, vint64m2_t
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
 vint64m4_t test_vadd_vv_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, vint64m4_t op1, vint64m4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i64m4_m(
@@ -1827,7 +1827,7 @@ vint64m4_t test_vadd_vv_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, vint64m4_t
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
 vint64m4_t test_vadd_vx_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, vint64m4_t op1, int64_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_i64m8_m(
@@ -1841,7 +1841,7 @@ vint64m4_t test_vadd_vx_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, vint64m4_t
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
 vint64m8_t test_vadd_vv_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, vint64m8_t op1, vint64m8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_i64m8_m(
@@ -1855,7 +1855,7 @@ vint64m8_t test_vadd_vv_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, vint64m8_t
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
 vint64m8_t test_vadd_vx_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, vint64m8_t op1, int64_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u8mf8_m(
@@ -1869,7 +1869,7 @@ vint64m8_t test_vadd_vx_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, vint64m8_t
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
 vuint8mf8_t test_vadd_vv_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t op1, vuint8mf8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u8mf8_m(
@@ -1883,7 +1883,7 @@ vuint8mf8_t test_vadd_vv_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
 //
 vuint8mf8_t test_vadd_vx_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf8_t op1, uint8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u8mf4_m(
@@ -1897,7 +1897,7 @@ vuint8mf8_t test_vadd_vx_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, vuint8mf
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
 vuint8mf4_t test_vadd_vv_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t op1, vuint8mf4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u8mf4_m(
@@ -1911,7 +1911,7 @@ vuint8mf4_t test_vadd_vv_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
 //
 vuint8mf4_t test_vadd_vx_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf4_t op1, uint8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u8mf2_m(
@@ -1925,7 +1925,7 @@ vuint8mf4_t test_vadd_vx_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, vuint8mf
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
 vuint8mf2_t test_vadd_vv_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t op1, vuint8mf2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u8mf2_m(
@@ -1939,7 +1939,7 @@ vuint8mf2_t test_vadd_vv_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
 //
 vuint8mf2_t test_vadd_vx_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf2_t op1, uint8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u8m1_m(
@@ -1953,7 +1953,7 @@ vuint8mf2_t test_vadd_vx_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, vuint8mf
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
 vuint8m1_t test_vadd_vv_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t op1, vuint8m1_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u8m1_m(
@@ -1967,7 +1967,7 @@ vuint8m1_t test_vadd_vv_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t o
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
 //
 vuint8m1_t test_vadd_vx_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t op1, uint8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u8m2_m(
@@ -1981,7 +1981,7 @@ vuint8m1_t test_vadd_vx_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, vuint8m1_t o
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 vuint8m2_t test_vadd_vv_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t op1, vuint8m2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u8m2_m(
@@ -1995,7 +1995,7 @@ vuint8m2_t test_vadd_vv_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t o
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 vuint8m2_t test_vadd_vx_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t op1, uint8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u8m4_m(
@@ -2009,7 +2009,7 @@ vuint8m2_t test_vadd_vx_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, vuint8m2_t o
 // CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
 vuint8m4_t test_vadd_vv_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t op1, vuint8m4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u8m4_m(
@@ -2023,7 +2023,7 @@ vuint8m4_t test_vadd_vv_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t o
 // CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
 //
 vuint8m4_t test_vadd_vx_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t op1, uint8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u8m8_m(
@@ -2037,7 +2037,7 @@ vuint8m4_t test_vadd_vx_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, vuint8m4_t o
 // CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
 vuint8m8_t test_vadd_vv_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t op1, vuint8m8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u8m8_m(
@@ -2051,7 +2051,7 @@ vuint8m8_t test_vadd_vv_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t o
 // CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
 //
 vuint8m8_t test_vadd_vx_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t op1, uint8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u16mf4_m(
@@ -2065,7 +2065,7 @@ vuint8m8_t test_vadd_vx_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, vuint8m8_t o
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
 vuint16mf4_t test_vadd_vv_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t op1, vuint16mf4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u16mf4_m(
@@ -2079,7 +2079,7 @@ vuint16mf4_t test_vadd_vv_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, vuint
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
 //
 vuint16mf4_t test_vadd_vx_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, vuint16mf4_t op1, uint16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u16mf2_m(
@@ -2093,7 +2093,7 @@ vuint16mf4_t test_vadd_vx_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, vuint
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
 vuint16mf2_t test_vadd_vv_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t op1, vuint16mf2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u16mf2_m(
@@ -2107,7 +2107,7 @@ vuint16mf2_t test_vadd_vv_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, vuint
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
 //
 vuint16mf2_t test_vadd_vx_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, vuint16mf2_t op1, uint16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u16m1_m(
@@ -2121,7 +2121,7 @@ vuint16mf2_t test_vadd_vx_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, vuint
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
 vuint16m1_t test_vadd_vv_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t op1, vuint16m1_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u16m1_m(
@@ -2135,7 +2135,7 @@ vuint16m1_t test_vadd_vv_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, vuint16m
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
 //
 vuint16m1_t test_vadd_vx_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, vuint16m1_t op1, uint16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u16m2_m(
@@ -2149,7 +2149,7 @@ vuint16m1_t test_vadd_vx_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, vuint16m
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
 vuint16m2_t test_vadd_vv_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t op1, vuint16m2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u16m2_m(
@@ -2163,7 +2163,7 @@ vuint16m2_t test_vadd_vv_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
 //
 vuint16m2_t test_vadd_vx_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2_t op1, uint16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u16m4_m(
@@ -2177,7 +2177,7 @@ vuint16m2_t test_vadd_vx_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, vuint16m2
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
 vuint16m4_t test_vadd_vv_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t op1, vuint16m4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u16m4_m(
@@ -2191,7 +2191,7 @@ vuint16m4_t test_vadd_vv_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
 //
 vuint16m4_t test_vadd_vx_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4_t op1, uint16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u16m8_m(
@@ -2205,7 +2205,7 @@ vuint16m4_t test_vadd_vx_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, vuint16m4
 // CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
 vuint16m8_t test_vadd_vv_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t op1, vuint16m8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u16m8_m(
@@ -2219,7 +2219,7 @@ vuint16m8_t test_vadd_vv_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8
 // CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
 //
 vuint16m8_t test_vadd_vx_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8_t op1, uint16_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u32mf2_m(
@@ -2233,7 +2233,7 @@ vuint16m8_t test_vadd_vx_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, vuint16m8
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
 vuint32mf2_t test_vadd_vv_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t op1, vuint32mf2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u32mf2_m(
@@ -2247,7 +2247,7 @@ vuint32mf2_t test_vadd_vv_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, vuint
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
 //
 vuint32mf2_t test_vadd_vx_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, vuint32mf2_t op1, uint32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u32m1_m(
@@ -2261,7 +2261,7 @@ vuint32mf2_t test_vadd_vx_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, vuint
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
 vuint32m1_t test_vadd_vv_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t op1, vuint32m1_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u32m1_m(
@@ -2275,7 +2275,7 @@ vuint32m1_t test_vadd_vv_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, vuint32m
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
 //
 vuint32m1_t test_vadd_vx_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, vuint32m1_t op1, uint32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u32m2_m(
@@ -2289,7 +2289,7 @@ vuint32m1_t test_vadd_vx_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, vuint32m
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
 vuint32m2_t test_vadd_vv_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t op1, vuint32m2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u32m2_m(
@@ -2303,7 +2303,7 @@ vuint32m2_t test_vadd_vv_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, vuint32m
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
 //
 vuint32m2_t test_vadd_vx_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, vuint32m2_t op1, uint32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u32m4_m(
@@ -2317,7 +2317,7 @@ vuint32m2_t test_vadd_vx_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, vuint32m
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
 vuint32m4_t test_vadd_vv_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t op1, vuint32m4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u32m4_m(
@@ -2331,7 +2331,7 @@ vuint32m4_t test_vadd_vv_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
 //
 vuint32m4_t test_vadd_vx_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4_t op1, uint32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u32m8_m(
@@ -2345,7 +2345,7 @@ vuint32m4_t test_vadd_vx_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, vuint32m4
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
 vuint32m8_t test_vadd_vv_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t op1, vuint32m8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u32m8_m(
@@ -2359,7 +2359,7 @@ vuint32m8_t test_vadd_vv_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8
 // CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
 //
 vuint32m8_t test_vadd_vx_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8_t op1, uint32_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u64m1_m(
@@ -2373,7 +2373,7 @@ vuint32m8_t test_vadd_vx_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, vuint32m8
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
 vuint64m1_t test_vadd_vv_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t op1, vuint64m1_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u64m1_m(
@@ -2387,7 +2387,7 @@ vuint64m1_t test_vadd_vv_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, vuint64m
 // CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
 //
 vuint64m1_t test_vadd_vx_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, vuint64m1_t op1, uint64_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u64m2_m(
@@ -2401,7 +2401,7 @@ vuint64m1_t test_vadd_vx_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, vuint64m
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
 vuint64m2_t test_vadd_vv_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t op1, vuint64m2_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u64m2_m(
@@ -2415,7 +2415,7 @@ vuint64m2_t test_vadd_vv_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, vuint64m
 // CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
 //
 vuint64m2_t test_vadd_vx_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, vuint64m2_t op1, uint64_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u64m4_m(
@@ -2429,7 +2429,7 @@ vuint64m2_t test_vadd_vx_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, vuint64m
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
 vuint64m4_t test_vadd_vv_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t op1, vuint64m4_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u64m4_m(
@@ -2443,7 +2443,7 @@ vuint64m4_t test_vadd_vv_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, vuint64m
 // CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
 //
 vuint64m4_t test_vadd_vx_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, vuint64m4_t op1, uint64_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vv_u64m8_m(
@@ -2457,7 +2457,7 @@ vuint64m4_t test_vadd_vx_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, vuint64m
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
 vuint64m8_t test_vadd_vv_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t op1, vuint64m8_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vadd_vx_u64m8_m(
@@ -2471,6 +2471,6 @@ vuint64m8_t test_vadd_vv_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8
 // CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
 //
 vuint64m8_t test_vadd_vx_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, vuint64m8_t op1, uint64_t op2, size_t vl) {
-  return vadd_m(mask, maskedoff, op1, op2, vl);
+  return vadd(mask, maskedoff, op1, op2, vl);
 }
 
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-generic/vfadd.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vfadd.c
similarity index 96%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-generic/vfadd.c
rename to clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vfadd.c
index d0389a333686a42ad0687c10c3277a83f8ac5b42..25fc2fe4e10134836bf865fed5240b328892722f 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-generic/vfadd.c
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vfadd.c
@@ -1,14 +1,14 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
 // RUN:   -target-feature +experimental-zfh -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
 // RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
 // RUN:   -target-feature +experimental-zfh -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
-// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
-// RUN:   -target-feature +experimental-zfh -Werror -Wall -o - %s >/dev/null 2>%t
-// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
 
 // ASM-NOT: warning
-#include <riscv_vector_generic.h>
+#include <riscv_vector.h>
 
 // CHECK-RV32-LABEL: @test_vfadd_vv_f32mf2(
 // CHECK-RV32-NEXT:  entry:
@@ -273,7 +273,7 @@ vfloat64m8_t test_vfadd_vf_f64m8(vfloat64m8_t op1, double op2, size_t vl) {
 // CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP0]]
 //
 vfloat32mf2_t test_vfadd_vv_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, vfloat32mf2_t op1, vfloat32mf2_t op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vf_f32mf2_m(
@@ -287,7 +287,7 @@ vfloat32mf2_t test_vfadd_vv_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, vf
 // CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP0]]
 //
 vfloat32mf2_t test_vfadd_vf_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, vfloat32mf2_t op1, float op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vv_f32m1_m(
@@ -301,7 +301,7 @@ vfloat32mf2_t test_vfadd_vf_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, vf
 // CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP0]]
 //
 vfloat32m1_t test_vfadd_vv_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, vfloat32m1_t op1, vfloat32m1_t op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vf_f32m1_m(
@@ -315,7 +315,7 @@ vfloat32m1_t test_vfadd_vv_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP0]]
 //
 vfloat32m1_t test_vfadd_vf_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, vfloat32m1_t op1, float op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vv_f32m2_m(
@@ -329,7 +329,7 @@ vfloat32m1_t test_vfadd_vf_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP0]]
 //
 vfloat32m2_t test_vfadd_vv_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, vfloat32m2_t op1, vfloat32m2_t op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vf_f32m2_m(
@@ -343,7 +343,7 @@ vfloat32m2_t test_vfadd_vv_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP0]]
 //
 vfloat32m2_t test_vfadd_vf_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, vfloat32m2_t op1, float op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vv_f32m4_m(
@@ -357,7 +357,7 @@ vfloat32m2_t test_vfadd_vf_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP0]]
 //
 vfloat32m4_t test_vfadd_vv_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, vfloat32m4_t op1, vfloat32m4_t op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vf_f32m4_m(
@@ -371,7 +371,7 @@ vfloat32m4_t test_vfadd_vv_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, vfloat
 // CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP0]]
 //
 vfloat32m4_t test_vfadd_vf_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, vfloat32m4_t op1, float op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vv_f32m8_m(
@@ -385,7 +385,7 @@ vfloat32m4_t test_vfadd_vf_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, vfloat
 // CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP0]]
 //
 vfloat32m8_t test_vfadd_vv_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, vfloat32m8_t op1, vfloat32m8_t op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vf_f32m8_m(
@@ -399,7 +399,7 @@ vfloat32m8_t test_vfadd_vv_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, vfloat
 // CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP0]]
 //
 vfloat32m8_t test_vfadd_vf_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, vfloat32m8_t op1, float op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vv_f64m1_m(
@@ -413,7 +413,7 @@ vfloat32m8_t test_vfadd_vf_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, vfloat
 // CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP0]]
 //
 vfloat64m1_t test_vfadd_vv_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, vfloat64m1_t op1, vfloat64m1_t op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vf_f64m1_m(
@@ -427,7 +427,7 @@ vfloat64m1_t test_vfadd_vv_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP0]]
 //
 vfloat64m1_t test_vfadd_vf_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, vfloat64m1_t op1, double op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vv_f64m2_m(
@@ -441,7 +441,7 @@ vfloat64m1_t test_vfadd_vf_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP0]]
 //
 vfloat64m2_t test_vfadd_vv_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, vfloat64m2_t op1, vfloat64m2_t op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vf_f64m2_m(
@@ -455,7 +455,7 @@ vfloat64m2_t test_vfadd_vv_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP0]]
 //
 vfloat64m2_t test_vfadd_vf_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, vfloat64m2_t op1, double op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vv_f64m4_m(
@@ -469,7 +469,7 @@ vfloat64m2_t test_vfadd_vf_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP0]]
 //
 vfloat64m4_t test_vfadd_vv_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, vfloat64m4_t op1, vfloat64m4_t op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vf_f64m4_m(
@@ -483,7 +483,7 @@ vfloat64m4_t test_vfadd_vv_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP0]]
 //
 vfloat64m4_t test_vfadd_vf_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, vfloat64m4_t op1, double op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vv_f64m8_m(
@@ -497,7 +497,7 @@ vfloat64m4_t test_vfadd_vf_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, vfloa
 // CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP0]]
 //
 vfloat64m8_t test_vfadd_vv_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, vfloat64m8_t op1, vfloat64m8_t op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
 // CHECK-RV32-LABEL: @test_vfadd_vf_f64m8_m(
@@ -511,6 +511,6 @@ vfloat64m8_t test_vfadd_vv_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, vfloat
 // CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP0]]
 //
 vfloat64m8_t test_vfadd_vf_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, vfloat64m8_t op1, double op2, size_t vl) {
-  return vfadd_m(mask, maskedoff, op1, op2, vl);
+  return vfadd(mask, maskedoff, op1, op2, vl);
 }
 
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vle.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vle.c
new file mode 100644
index 0000000000000000000000000000000000000000..0b313a0f5c30140bd1d109d50148cbb3d336b437
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vle.c
@@ -0,0 +1,859 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.mask.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8:#.*]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8:#.*]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vle8_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.mask.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vle8_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.mask.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vle8_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.mask.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vle8_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.mask.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vle8_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.mask.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vle8_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.mask.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t test_vle8_v_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.mask.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vle16_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.mask.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vle16_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.mask.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vle16_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.mask.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vle16_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.mask.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vle16_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.mask.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vle16_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.mask.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vle32_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.mask.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vle32_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.mask.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vle32_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.mask.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vle32_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.mask.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vle32_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vle64_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.mask.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vle64_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.mask.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vle64_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.mask.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vle64_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.mask.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vle8_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.mask.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vle8_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.mask.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vle8_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.mask.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vle8_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.mask.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vle8_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.mask.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vle8_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.mask.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t test_vle8_v_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.mask.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vle16_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.mask.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vle16_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.mask.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vle16_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.mask.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vle16_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.mask.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vle16_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.mask.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vle16_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.mask.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vle32_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.mask.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vle32_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.mask.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vle32_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.mask.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vle32_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.mask.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vle32_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vle64_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.mask.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vle64_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.mask.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vle64_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.mask.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vle64_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vle.mask.nxv1f32.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vle.mask.nxv1f32.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vle32_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vle.mask.nxv2f32.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vle.mask.nxv2f32.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vle32_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vle.mask.nxv4f32.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vle.mask.nxv4f32.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vle32_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vle.mask.nxv8f32.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vle.mask.nxv8f32.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vle32_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vle.mask.nxv16f32.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vle.mask.nxv16f32.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vle32_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, size_t vl) {
+  return vle32(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vle.mask.nxv1f64.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vle.mask.nxv1f64.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vle64_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vle.mask.nxv2f64.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vle.mask.nxv2f64.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vle64_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vle.mask.nxv4f64.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vle.mask.nxv4f64.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vle64_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vle.mask.nxv8f64.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vle.mask.nxv8f64.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vle64_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, size_t vl) {
+  return vle64(mask, maskedoff, base, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vloxei.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vloxei.c
new file mode 100644
index 0000000000000000000000000000000000000000..5652eda62b2521d4638812a8343e8a9a5af2698f
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vloxei.c
@@ -0,0 +1,6125 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8:#.*]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8:#.*]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vloxei8_v_i8mf8(const int8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vloxei8_v_i8mf4(const int8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vloxei8_v_i8mf2(const int8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vloxei8_v_i8m1(const int8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vloxei8_v_i8m2(const int8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vloxei8_v_i8m4(const int8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.nxv64i8.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.nxv64i8.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t test_vloxei8_v_i8m8(const int8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vloxei16_v_i8mf8(const int8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vloxei16_v_i8mf4(const int8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vloxei16_v_i8mf2(const int8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vloxei16_v_i8m1(const int8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vloxei16_v_i8m2(const int8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vloxei16_v_i8m4(const int8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vloxei32_v_i8mf8(const int8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vloxei32_v_i8mf4(const int8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vloxei32_v_i8mf2(const int8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vloxei32_v_i8m1(const int8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vloxei32_v_i8m2(const int8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i64.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i64.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vloxei64_v_i8mf8(const int8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i64.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i64.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vloxei64_v_i8mf4(const int8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i64.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i64.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vloxei64_v_i8mf2(const int8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i64.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i64.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vloxei64_v_i8m1(const int8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vloxei8_v_i16mf4(const int16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vloxei8_v_i16mf2(const int16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vloxei8_v_i16m1(const int16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vloxei8_v_i16m2(const int16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vloxei8_v_i16m4(const int16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vloxei8_v_i16m8(const int16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vloxei16_v_i16mf4(const int16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vloxei16_v_i16mf2(const int16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vloxei16_v_i16m1(const int16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vloxei16_v_i16m2(const int16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vloxei16_v_i16m4(const int16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vloxei16_v_i16m8(const int16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vloxei32_v_i16mf4(const int16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vloxei32_v_i16mf2(const int16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vloxei32_v_i16m1(const int16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vloxei32_v_i16m2(const int16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vloxei32_v_i16m4(const int16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i64.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i64.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vloxei64_v_i16mf4(const int16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i64.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i64.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vloxei64_v_i16mf2(const int16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i64.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i64.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vloxei64_v_i16m1(const int16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i64.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i64.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vloxei64_v_i16m2(const int16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vloxei8_v_i32mf2(const int32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vloxei8_v_i32m1(const int32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vloxei8_v_i32m2(const int32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vloxei8_v_i32m4(const int32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vloxei8_v_i32m8(const int32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vloxei16_v_i32mf2(const int32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vloxei16_v_i32m1(const int32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vloxei16_v_i32m2(const int32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vloxei16_v_i32m4(const int32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vloxei16_v_i32m8(const int32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vloxei32_v_i32mf2(const int32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vloxei32_v_i32m1(const int32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vloxei32_v_i32m2(const int32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vloxei32_v_i32m4(const int32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vloxei32_v_i32m8(const int32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i64.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i64.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vloxei64_v_i32mf2(const int32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i64.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i64.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vloxei64_v_i32m1(const int32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i64.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i64.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vloxei64_v_i32m2(const int32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i64.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i64.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vloxei64_v_i32m4(const int32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i8.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i8.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vloxei8_v_i64m1(const int64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i8.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i8.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vloxei8_v_i64m2(const int64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i8.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i8.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vloxei8_v_i64m4(const int64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i8.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i8.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vloxei8_v_i64m8(const int64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i16.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i16.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vloxei16_v_i64m1(const int64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i16.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i16.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vloxei16_v_i64m2(const int64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i16.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i16.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vloxei16_v_i64m4(const int64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i16.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i16.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vloxei16_v_i64m8(const int64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i32.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i32.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vloxei32_v_i64m1(const int64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i32.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i32.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vloxei32_v_i64m2(const int64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i32.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i32.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vloxei32_v_i64m4(const int64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i32.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i32.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vloxei32_v_i64m8(const int64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vloxei64_v_i64m1(const int64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vloxei64_v_i64m2(const int64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vloxei64_v_i64m4(const int64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vloxei64_v_i64m8(const int64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vloxei8_v_u8mf8(const uint8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vloxei8_v_u8mf4(const uint8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vloxei8_v_u8mf2(const uint8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vloxei8_v_u8m1(const uint8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vloxei8_v_u8m2(const uint8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vloxei8_v_u8m4(const uint8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.nxv64i8.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.nxv64i8.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t test_vloxei8_v_u8m8(const uint8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vloxei16_v_u8mf8(const uint8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vloxei16_v_u8mf4(const uint8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vloxei16_v_u8mf2(const uint8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vloxei16_v_u8m1(const uint8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vloxei16_v_u8m2(const uint8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vloxei16_v_u8m4(const uint8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vloxei32_v_u8mf8(const uint8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vloxei32_v_u8mf4(const uint8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vloxei32_v_u8mf2(const uint8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vloxei32_v_u8m1(const uint8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vloxei32_v_u8m2(const uint8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i64.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i64.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vloxei64_v_u8mf8(const uint8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i64.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i64.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vloxei64_v_u8mf4(const uint8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i64.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i64.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vloxei64_v_u8mf2(const uint8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i64.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i64.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vloxei64_v_u8m1(const uint8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vloxei8_v_u16mf4(const uint16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vloxei8_v_u16mf2(const uint16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vloxei8_v_u16m1(const uint16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vloxei8_v_u16m2(const uint16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vloxei8_v_u16m4(const uint16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vloxei8_v_u16m8(const uint16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vloxei16_v_u16mf4(const uint16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vloxei16_v_u16mf2(const uint16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vloxei16_v_u16m1(const uint16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vloxei16_v_u16m2(const uint16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vloxei16_v_u16m4(const uint16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vloxei16_v_u16m8(const uint16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vloxei32_v_u16mf4(const uint16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vloxei32_v_u16mf2(const uint16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vloxei32_v_u16m1(const uint16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vloxei32_v_u16m2(const uint16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vloxei32_v_u16m4(const uint16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i64.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i64.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vloxei64_v_u16mf4(const uint16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i64.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i64.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vloxei64_v_u16mf2(const uint16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i64.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i64.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vloxei64_v_u16m1(const uint16_t *base, vuint64m4_t bindex, size_t vl) {
+  //
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i64.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i64.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vloxei64_v_u16m2(const uint16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vloxei8_v_u32mf2(const uint32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vloxei8_v_u32m1(const uint32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vloxei8_v_u32m2(const uint32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vloxei8_v_u32m4(const uint32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vloxei8_v_u32m8(const uint32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vloxei16_v_u32mf2(const uint32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vloxei16_v_u32m1(const uint32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vloxei16_v_u32m2(const uint32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vloxei16_v_u32m4(const uint32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vloxei16_v_u32m8(const uint32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vloxei32_v_u32mf2(const uint32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vloxei32_v_u32m1(const uint32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vloxei32_v_u32m2(const uint32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vloxei32_v_u32m4(const uint32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vloxei32_v_u32m8(const uint32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i64.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i64.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vloxei64_v_u32mf2(const uint32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i64.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i64.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vloxei64_v_u32m1(const uint32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i64.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i64.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vloxei64_v_u32m2(const uint32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i64.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i64.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vloxei64_v_u32m4(const uint32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i8.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i8.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vloxei8_v_u64m1(const uint64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i8.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i8.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vloxei8_v_u64m2(const uint64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i8.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i8.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vloxei8_v_u64m4(const uint64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i8.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i8.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vloxei8_v_u64m8(const uint64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i16.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i16.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vloxei16_v_u64m1(const uint64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i16.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i16.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vloxei16_v_u64m2(const uint64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i16.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i16.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vloxei16_v_u64m4(const uint64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i16.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i16.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vloxei16_v_u64m8(const uint64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i32.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i32.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vloxei32_v_u64m1(const uint64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i32.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i32.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vloxei32_v_u64m2(const uint64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i32.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i32.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vloxei32_v_u64m4(const uint64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i32.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i32.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vloxei32_v_u64m8(const uint64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vloxei64_v_u64m1(const uint64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vloxei64_v_u64m2(const uint64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vloxei64_v_u64m4(const uint64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vloxei64_v_u64m8(const uint64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i8.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i8.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vloxei8_v_f32mf2(const float *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i8.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i8.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vloxei8_v_f32m1(const float *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i8.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i8.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vloxei8_v_f32m2(const float *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i8.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i8.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vloxei8_v_f32m4(const float *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i8.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i8.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vloxei8_v_f32m8(const float *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i16.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i16.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vloxei16_v_f32mf2(const float *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i16.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i16.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vloxei16_v_f32m1(const float *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i16.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i16.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vloxei16_v_f32m2(const float *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i16.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i16.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vloxei16_v_f32m4(const float *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i16.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i16.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vloxei16_v_f32m8(const float *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i32.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i32.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vloxei32_v_f32mf2(const float *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i32.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i32.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vloxei32_v_f32m1(const float *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i32.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i32.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vloxei32_v_f32m2(const float *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i32.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i32.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vloxei32_v_f32m4(const float *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i32.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i32.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vloxei32_v_f32m8(const float *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i64.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i64.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vloxei64_v_f32mf2(const float *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i64.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i64.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vloxei64_v_f32m1(const float *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i64.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i64.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vloxei64_v_f32m2(const float *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i64.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i64.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vloxei64_v_f32m4(const float *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i8.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i8.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vloxei8_v_f64m1(const double *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i8.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i8.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vloxei8_v_f64m2(const double *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i8.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i8.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vloxei8_v_f64m4(const double *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i8.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i8.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vloxei8_v_f64m8(const double *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i16.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i16.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vloxei16_v_f64m1(const double *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i16.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i16.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vloxei16_v_f64m2(const double *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i16.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i16.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vloxei16_v_f64m4(const double *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i16.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i16.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vloxei16_v_f64m8(const double *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i32.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i32.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vloxei32_v_f64m1(const double *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i32.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i32.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vloxei32_v_f64m2(const double *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i32.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i32.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vloxei32_v_f64m4(const double *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i32.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i32.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vloxei32_v_f64m8(const double *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i64.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i64.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vloxei64_v_f64m1(const double *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i64.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i64.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vloxei64_v_f64m2(const double *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i64.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i64.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vloxei64_v_f64m4(const double *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i64.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i64.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vloxei64_v_f64m8(const double *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vloxei8_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vloxei8_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vloxei8_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vloxei8_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vloxei8_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vloxei8_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.mask.nxv64i8.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.mask.nxv64i8.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t test_vloxei8_v_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, const int8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i16.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i16.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vloxei16_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i16.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i16.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vloxei16_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i16.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i16.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vloxei16_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i16.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i16.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vloxei16_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i16.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i16.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vloxei16_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i16.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i16.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vloxei16_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i32.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i32.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vloxei32_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i32.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i32.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vloxei32_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i32.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i32.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vloxei32_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i32.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i32.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vloxei32_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i32.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i32.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vloxei32_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i64.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i64.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vloxei64_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i64.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i64.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vloxei64_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i64.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i64.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vloxei64_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i64.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i64.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vloxei64_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i8.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i8.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vloxei8_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i8.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i8.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vloxei8_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i8.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i8.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vloxei8_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i8.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i8.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vloxei8_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i8.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i8.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vloxei8_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i8.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i8.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vloxei8_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vloxei16_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vloxei16_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vloxei16_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vloxei16_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vloxei16_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vloxei16_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i32.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i32.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vloxei32_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i32.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i32.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vloxei32_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i32.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i32.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vloxei32_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i32.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i32.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vloxei32_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i32.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i32.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vloxei32_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i64.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i64.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vloxei64_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i64.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i64.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vloxei64_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i64.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i64.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vloxei64_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i64.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i64.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vloxei64_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i8.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i8.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vloxei8_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i8.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i8.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vloxei8_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i8.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i8.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vloxei8_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i8.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i8.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vloxei8_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i8.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i8.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vloxei8_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i16.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i16.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vloxei16_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i16.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i16.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vloxei16_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i16.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i16.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vloxei16_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i16.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i16.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vloxei16_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i16.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i16.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vloxei16_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vloxei32_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vloxei32_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vloxei32_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vloxei32_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vloxei32_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i64.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i64.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vloxei64_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i64.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i64.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vloxei64_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i64.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i64.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vloxei64_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i64.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i64.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vloxei64_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i8.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i8.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vloxei8_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i8.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i8.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vloxei8_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i8.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i8.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vloxei8_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i8.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i8.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vloxei8_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i16.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i16.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vloxei16_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i16.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i16.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vloxei16_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i16.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i16.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vloxei16_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i16.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i16.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vloxei16_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i32.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i32.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vloxei32_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i32.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i32.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vloxei32_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i32.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vloxei32_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i32.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i32.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vloxei32_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vloxei64_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vloxei64_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vloxei64_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vloxei64_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vloxei8_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vloxei8_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vloxei8_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vloxei8_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vloxei8_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vloxei8_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.mask.nxv64i8.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.mask.nxv64i8.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t test_vloxei8_v_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, const uint8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i16.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i16.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vloxei16_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i16.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i16.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vloxei16_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i16.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i16.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vloxei16_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i16.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i16.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vloxei16_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i16.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i16.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vloxei16_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i16.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i16.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vloxei16_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i32.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i32.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vloxei32_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i32.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i32.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vloxei32_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i32.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i32.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vloxei32_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i32.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i32.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vloxei32_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i32.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i32.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vloxei32_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i64.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i64.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vloxei64_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i64.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i64.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vloxei64_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i64.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i64.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vloxei64_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i64.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i64.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vloxei64_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i8.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i8.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vloxei8_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i8.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i8.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vloxei8_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i8.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i8.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vloxei8_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i8.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i8.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vloxei8_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i8.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i8.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vloxei8_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i8.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i8.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vloxei8_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vloxei16_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vloxei16_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vloxei16_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vloxei16_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vloxei16_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vloxei16_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i32.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i32.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vloxei32_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i32.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i32.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vloxei32_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i32.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i32.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vloxei32_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i32.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i32.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vloxei32_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i32.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i32.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vloxei32_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i64.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i64.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vloxei64_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i64.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i64.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vloxei64_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i64.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i64.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vloxei64_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i64.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i64.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vloxei64_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i8.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i8.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vloxei8_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i8.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i8.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vloxei8_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i8.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i8.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vloxei8_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i8.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i8.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vloxei8_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i8.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i8.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vloxei8_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i16.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i16.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vloxei16_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i16.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i16.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vloxei16_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i16.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i16.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vloxei16_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i16.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i16.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vloxei16_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i16.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i16.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vloxei16_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vloxei32_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vloxei32_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vloxei32_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vloxei32_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vloxei32_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i64.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i64.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vloxei64_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i64.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i64.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vloxei64_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i64.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i64.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vloxei64_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i64.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i64.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vloxei64_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i8.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i8.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vloxei8_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i8.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i8.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vloxei8_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i8.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i8.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vloxei8_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i8.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i8.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vloxei8_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i16.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i16.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vloxei16_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i16.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i16.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vloxei16_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i16.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i16.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vloxei16_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i16.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i16.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vloxei16_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i32.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i32.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vloxei32_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i32.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i32.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vloxei32_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i32.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vloxei32_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i32.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i32.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vloxei32_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vloxei64_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vloxei64_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vloxei64_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vloxei64_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i8.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i8.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vloxei8_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i8.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i8.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vloxei8_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i8.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i8.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vloxei8_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i8.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i8.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vloxei8_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i8.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i8.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vloxei8_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i16.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i16.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vloxei16_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i16.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i16.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vloxei16_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i16.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i16.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vloxei16_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i16.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i16.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vloxei16_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i16.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i16.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vloxei16_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i32.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i32.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vloxei32_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i32.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i32.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vloxei32_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i32.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i32.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vloxei32_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i32.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i32.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vloxei32_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i32.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i32.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vloxei32_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i64.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i64.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vloxei64_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i64.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i64.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vloxei64_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i64.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i64.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vloxei64_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i64.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i64.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vloxei64_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i8.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i8.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vloxei8_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i8.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i8.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vloxei8_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i8.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i8.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vloxei8_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei8_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i8.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei8_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i8.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vloxei8_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i16.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i16.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vloxei16_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i16.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i16.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vloxei16_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i16.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i16.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vloxei16_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei16_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i16.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei16_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i16.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vloxei16_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i32.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i32.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vloxei32_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i32.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i32.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vloxei32_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i32.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i32.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vloxei32_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei32_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i32.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei32_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i32.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vloxei32_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i64.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i64.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vloxei64_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i64.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i64.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vloxei64_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i64.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i64.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vloxei64_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vloxei64_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i64.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vloxei64_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i64.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vloxei64_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64(mask, maskedoff, base, bindex, vl);
+}
+
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vluxei.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vluxei.c
new file mode 100644
index 0000000000000000000000000000000000000000..15fa817c9e9043d792a2d510cdd848e6fdef2144
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vluxei.c
@@ -0,0 +1,6123 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8:#.*]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8:#.*]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vluxei8_v_i8mf8(const int8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vluxei8_v_i8mf4(const int8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vluxei8_v_i8mf2(const int8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vluxei8_v_i8m1(const int8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vluxei8_v_i8m2(const int8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vluxei8_v_i8m4(const int8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.nxv64i8.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.nxv64i8.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t test_vluxei8_v_i8m8(const int8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vluxei16_v_i8mf8(const int8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vluxei16_v_i8mf4(const int8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vluxei16_v_i8mf2(const int8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vluxei16_v_i8m1(const int8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vluxei16_v_i8m2(const int8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vluxei16_v_i8m4(const int8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vluxei32_v_i8mf8(const int8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vluxei32_v_i8mf4(const int8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vluxei32_v_i8mf2(const int8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vluxei32_v_i8m1(const int8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vluxei32_v_i8m2(const int8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i64.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i64.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vluxei64_v_i8mf8(const int8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i64.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i64.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vluxei64_v_i8mf4(const int8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i64.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i64.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vluxei64_v_i8mf2(const int8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i64.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i64.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vluxei64_v_i8m1(const int8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vluxei8_v_i16mf4(const int16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vluxei8_v_i16mf2(const int16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vluxei8_v_i16m1(const int16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vluxei8_v_i16m2(const int16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vluxei8_v_i16m4(const int16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vluxei8_v_i16m8(const int16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vluxei16_v_i16mf4(const int16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vluxei16_v_i16mf2(const int16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vluxei16_v_i16m1(const int16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vluxei16_v_i16m2(const int16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vluxei16_v_i16m4(const int16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vluxei16_v_i16m8(const int16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vluxei32_v_i16mf4(const int16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vluxei32_v_i16mf2(const int16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vluxei32_v_i16m1(const int16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vluxei32_v_i16m2(const int16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vluxei32_v_i16m4(const int16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i64.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i64.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vluxei64_v_i16mf4(const int16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i64.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i64.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vluxei64_v_i16mf2(const int16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i64.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i64.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vluxei64_v_i16m1(const int16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i64.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i64.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vluxei64_v_i16m2(const int16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vluxei8_v_i32mf2(const int32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vluxei8_v_i32m1(const int32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vluxei8_v_i32m2(const int32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vluxei8_v_i32m4(const int32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vluxei8_v_i32m8(const int32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vluxei16_v_i32mf2(const int32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vluxei16_v_i32m1(const int32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vluxei16_v_i32m2(const int32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vluxei16_v_i32m4(const int32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vluxei16_v_i32m8(const int32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vluxei32_v_i32mf2(const int32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vluxei32_v_i32m1(const int32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vluxei32_v_i32m2(const int32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vluxei32_v_i32m4(const int32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vluxei32_v_i32m8(const int32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i64.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i64.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vluxei64_v_i32mf2(const int32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i64.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i64.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vluxei64_v_i32m1(const int32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i64.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i64.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vluxei64_v_i32m2(const int32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i64.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i64.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vluxei64_v_i32m4(const int32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i8.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i8.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vluxei8_v_i64m1(const int64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i8.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i8.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vluxei8_v_i64m2(const int64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i8.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i8.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vluxei8_v_i64m4(const int64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i8.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i8.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vluxei8_v_i64m8(const int64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i16.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i16.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vluxei16_v_i64m1(const int64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i16.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i16.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vluxei16_v_i64m2(const int64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i16.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i16.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vluxei16_v_i64m4(const int64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i16.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i16.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vluxei16_v_i64m8(const int64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i32.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i32.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vluxei32_v_i64m1(const int64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i32.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i32.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vluxei32_v_i64m2(const int64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i32.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i32.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vluxei32_v_i64m4(const int64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i32.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i32.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vluxei32_v_i64m8(const int64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vluxei64_v_i64m1(const int64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vluxei64_v_i64m2(const int64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vluxei64_v_i64m4(const int64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vluxei64_v_i64m8(const int64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vluxei8_v_u8mf8(const uint8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vluxei8_v_u8mf4(const uint8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vluxei8_v_u8mf2(const uint8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vluxei8_v_u8m1(const uint8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vluxei8_v_u8m2(const uint8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vluxei8_v_u8m4(const uint8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.nxv64i8.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.nxv64i8.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t test_vluxei8_v_u8m8(const uint8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vluxei16_v_u8mf8(const uint8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vluxei16_v_u8mf4(const uint8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vluxei16_v_u8mf2(const uint8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vluxei16_v_u8m1(const uint8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vluxei16_v_u8m2(const uint8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vluxei16_v_u8m4(const uint8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vluxei32_v_u8mf8(const uint8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vluxei32_v_u8mf4(const uint8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vluxei32_v_u8mf2(const uint8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vluxei32_v_u8m1(const uint8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vluxei32_v_u8m2(const uint8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i64.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i64.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vluxei64_v_u8mf8(const uint8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i64.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i64.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vluxei64_v_u8mf4(const uint8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i64.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i64.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vluxei64_v_u8mf2(const uint8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i64.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i64.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vluxei64_v_u8m1(const uint8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vluxei8_v_u16mf4(const uint16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vluxei8_v_u16mf2(const uint16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vluxei8_v_u16m1(const uint16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vluxei8_v_u16m2(const uint16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vluxei8_v_u16m4(const uint16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vluxei8_v_u16m8(const uint16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vluxei16_v_u16mf4(const uint16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vluxei16_v_u16mf2(const uint16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vluxei16_v_u16m1(const uint16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vluxei16_v_u16m2(const uint16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vluxei16_v_u16m4(const uint16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vluxei16_v_u16m8(const uint16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vluxei32_v_u16mf4(const uint16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vluxei32_v_u16mf2(const uint16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vluxei32_v_u16m1(const uint16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vluxei32_v_u16m2(const uint16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vluxei32_v_u16m4(const uint16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i64.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i64.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vluxei64_v_u16mf4(const uint16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i64.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i64.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vluxei64_v_u16mf2(const uint16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i64.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i64.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vluxei64_v_u16m1(const uint16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i64.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i64.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vluxei64_v_u16m2(const uint16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vluxei8_v_u32mf2(const uint32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vluxei8_v_u32m1(const uint32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vluxei8_v_u32m2(const uint32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vluxei8_v_u32m4(const uint32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vluxei8_v_u32m8(const uint32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vluxei16_v_u32mf2(const uint32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vluxei16_v_u32m1(const uint32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vluxei16_v_u32m2(const uint32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vluxei16_v_u32m4(const uint32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vluxei16_v_u32m8(const uint32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vluxei32_v_u32mf2(const uint32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vluxei32_v_u32m1(const uint32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vluxei32_v_u32m2(const uint32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vluxei32_v_u32m4(const uint32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vluxei32_v_u32m8(const uint32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i64.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i64.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vluxei64_v_u32mf2(const uint32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i64.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i64.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vluxei64_v_u32m1(const uint32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i64.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i64.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vluxei64_v_u32m2(const uint32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i64.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i64.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vluxei64_v_u32m4(const uint32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i8.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i8.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vluxei8_v_u64m1(const uint64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i8.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i8.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vluxei8_v_u64m2(const uint64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i8.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i8.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vluxei8_v_u64m4(const uint64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i8.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i8.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vluxei8_v_u64m8(const uint64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i16.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i16.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vluxei16_v_u64m1(const uint64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i16.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i16.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vluxei16_v_u64m2(const uint64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i16.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i16.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vluxei16_v_u64m4(const uint64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i16.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i16.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vluxei16_v_u64m8(const uint64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i32.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i32.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vluxei32_v_u64m1(const uint64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i32.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i32.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vluxei32_v_u64m2(const uint64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i32.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i32.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vluxei32_v_u64m4(const uint64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i32.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i32.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vluxei32_v_u64m8(const uint64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vluxei64_v_u64m1(const uint64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vluxei64_v_u64m2(const uint64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vluxei64_v_u64m4(const uint64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vluxei64_v_u64m8(const uint64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i8.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i8.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vluxei8_v_f32mf2(const float *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i8.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i8.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vluxei8_v_f32m1(const float *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i8.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i8.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vluxei8_v_f32m2(const float *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i8.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i8.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vluxei8_v_f32m4(const float *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i8.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i8.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vluxei8_v_f32m8(const float *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i16.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i16.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vluxei16_v_f32mf2(const float *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i16.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i16.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vluxei16_v_f32m1(const float *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i16.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i16.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vluxei16_v_f32m2(const float *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i16.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i16.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vluxei16_v_f32m4(const float *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i16.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i16.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vluxei16_v_f32m8(const float *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i32.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i32.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vluxei32_v_f32mf2(const float *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i32.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i32.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vluxei32_v_f32m1(const float *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i32.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i32.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vluxei32_v_f32m2(const float *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i32.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i32.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vluxei32_v_f32m4(const float *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i32.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i32.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vluxei32_v_f32m8(const float *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i64.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i64.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vluxei64_v_f32mf2(const float *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i64.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i64.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vluxei64_v_f32m1(const float *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i64.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i64.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vluxei64_v_f32m2(const float *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i64.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i64.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vluxei64_v_f32m4(const float *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i8.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i8.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vluxei8_v_f64m1(const double *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i8.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i8.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vluxei8_v_f64m2(const double *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i8.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i8.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vluxei8_v_f64m4(const double *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i8.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i8.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vluxei8_v_f64m8(const double *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i16.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i16.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vluxei16_v_f64m1(const double *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i16.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i16.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vluxei16_v_f64m2(const double *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i16.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i16.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vluxei16_v_f64m4(const double *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i16.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i16.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vluxei16_v_f64m8(const double *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i32.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i32.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vluxei32_v_f64m1(const double *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i32.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i32.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vluxei32_v_f64m2(const double *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i32.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i32.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vluxei32_v_f64m4(const double *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i32.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i32.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vluxei32_v_f64m8(const double *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i64.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i64.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vluxei64_v_f64m1(const double *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i64.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i64.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vluxei64_v_f64m2(const double *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i64.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i64.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vluxei64_v_f64m4(const double *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i64.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i64.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vluxei64_v_f64m8(const double *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vluxei8_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vluxei8_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vluxei8_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vluxei8_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vluxei8_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vluxei8_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.mask.nxv64i8.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.mask.nxv64i8.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t test_vluxei8_v_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, const int8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i16.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i16.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vluxei16_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i16.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i16.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vluxei16_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i16.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i16.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vluxei16_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i16.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i16.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vluxei16_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i16.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i16.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vluxei16_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i16.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i16.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vluxei16_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i32.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i32.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vluxei32_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i32.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i32.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vluxei32_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i32.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i32.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vluxei32_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i32.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i32.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vluxei32_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i32.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i32.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vluxei32_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i64.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i64.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vluxei64_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i64.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i64.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vluxei64_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i64.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i64.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vluxei64_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i64.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i64.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vluxei64_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i8.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i8.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vluxei8_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i8.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i8.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vluxei8_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i8.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i8.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vluxei8_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i8.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i8.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vluxei8_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i8.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i8.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vluxei8_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i8.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i8.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vluxei8_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vluxei16_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vluxei16_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vluxei16_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vluxei16_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vluxei16_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vluxei16_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i32.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i32.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vluxei32_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i32.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i32.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vluxei32_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i32.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i32.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vluxei32_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i32.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i32.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vluxei32_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i32.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i32.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vluxei32_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i64.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i64.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vluxei64_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i64.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i64.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vluxei64_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i64.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i64.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vluxei64_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i64.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i64.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vluxei64_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i8.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i8.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vluxei8_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i8.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i8.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vluxei8_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i8.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i8.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vluxei8_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i8.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i8.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vluxei8_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i8.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i8.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vluxei8_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i16.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i16.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vluxei16_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i16.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i16.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vluxei16_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i16.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i16.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vluxei16_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i16.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i16.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vluxei16_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i16.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i16.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vluxei16_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vluxei32_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vluxei32_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vluxei32_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vluxei32_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vluxei32_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i64.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i64.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vluxei64_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i64.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i64.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vluxei64_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i64.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i64.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vluxei64_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i64.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i64.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vluxei64_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i8.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i8.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vluxei8_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i8.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i8.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vluxei8_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i8.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i8.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vluxei8_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i8.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i8.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vluxei8_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i16.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i16.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vluxei16_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i16.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i16.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vluxei16_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i16.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i16.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vluxei16_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i16.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i16.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vluxei16_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i32.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i32.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vluxei32_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i32.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i32.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vluxei32_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i32.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vluxei32_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i32.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i32.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vluxei32_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vluxei64_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vluxei64_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vluxei64_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vluxei64_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vluxei8_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vluxei8_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vluxei8_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vluxei8_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vluxei8_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vluxei8_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.mask.nxv64i8.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.mask.nxv64i8.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t test_vluxei8_v_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, const uint8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i16.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i16.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vluxei16_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i16.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i16.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vluxei16_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i16.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i16.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vluxei16_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i16.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i16.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vluxei16_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i16.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i16.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vluxei16_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i16.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i16.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vluxei16_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i32.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i32.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vluxei32_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i32.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i32.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vluxei32_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i32.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i32.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vluxei32_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i32.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i32.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vluxei32_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i32.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i32.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vluxei32_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i64.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i64.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vluxei64_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i64.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i64.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vluxei64_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i64.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i64.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vluxei64_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i64.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i64.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vluxei64_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i8.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i8.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vluxei8_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i8.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i8.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vluxei8_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i8.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i8.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vluxei8_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i8.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i8.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vluxei8_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i8.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i8.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vluxei8_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i8.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i8.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vluxei8_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vluxei16_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vluxei16_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vluxei16_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vluxei16_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vluxei16_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vluxei16_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i32.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i32.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vluxei32_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i32.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i32.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vluxei32_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i32.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i32.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vluxei32_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i32.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i32.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vluxei32_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i32.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i32.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vluxei32_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i64.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i64.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vluxei64_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i64.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i64.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vluxei64_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i64.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i64.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vluxei64_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i64.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i64.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vluxei64_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i8.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i8.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vluxei8_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i8.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i8.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vluxei8_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i8.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i8.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vluxei8_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i8.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i8.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vluxei8_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i8.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i8.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vluxei8_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i16.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i16.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vluxei16_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i16.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i16.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vluxei16_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i16.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i16.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vluxei16_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i16.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i16.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vluxei16_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i16.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i16.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vluxei16_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vluxei32_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vluxei32_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vluxei32_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vluxei32_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vluxei32_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i64.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i64.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vluxei64_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i64.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i64.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vluxei64_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i64.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i64.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vluxei64_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i64.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i64.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vluxei64_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i8.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i8.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vluxei8_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i8.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i8.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vluxei8_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i8.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i8.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vluxei8_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i8.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i8.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vluxei8_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i16.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i16.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vluxei16_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i16.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i16.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vluxei16_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i16.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i16.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vluxei16_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i16.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i16.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vluxei16_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i32.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i32.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vluxei32_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i32.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i32.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vluxei32_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i32.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vluxei32_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i32.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i32.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vluxei32_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vluxei64_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vluxei64_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vluxei64_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vluxei64_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i8.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i8.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vluxei8_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i8.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i8.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vluxei8_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i8.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i8.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vluxei8_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i8.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i8.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vluxei8_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i8.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i8.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vluxei8_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i16.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i16.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vluxei16_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i16.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i16.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vluxei16_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i16.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i16.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vluxei16_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i16.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i16.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vluxei16_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i16.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i16.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vluxei16_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i32.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i32.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vluxei32_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i32.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i32.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vluxei32_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i32.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i32.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vluxei32_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i32.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i32.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vluxei32_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i32.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i32.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vluxei32_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i64.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i64.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vluxei64_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i64.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i64.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vluxei64_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i64.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i64.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vluxei64_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i64.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i64.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vluxei64_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i8.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i8.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vluxei8_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i8.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i8.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vluxei8_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i8.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i8.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vluxei8_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei8_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i8.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei8_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i8.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vluxei8_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i16.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i16.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vluxei16_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i16.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i16.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vluxei16_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i16.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i16.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vluxei16_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei16_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i16.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei16_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i16.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vluxei16_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i32.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i32.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vluxei32_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i32.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i32.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vluxei32_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i32.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i32.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vluxei32_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei32_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i32.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei32_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i32.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vluxei32_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i64.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i64.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vluxei64_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i64.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i64.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vluxei64_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i64.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i64.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vluxei64_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vluxei64_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i64.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vluxei64_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i64.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) [[ATTR8]]
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vluxei64_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64(mask, maskedoff, base, bindex, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vse.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vse.c
new file mode 100644
index 0000000000000000000000000000000000000000..a8242403b83d4366b96f8488d144359fcf955e5d
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-overloaded/vse.c
@@ -0,0 +1,1707 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i8.i32(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11:[0-9]+]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i8.i64(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11:[0-9]+]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf8(int8_t *base, vint8mf8_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i8.i32(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i8.i64(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf4(int8_t *base, vint8mf4_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i8.i32(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i8.i64(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf2(int8_t *base, vint8mf2_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i8.i32(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i8.i64(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m1(int8_t *base, vint8m1_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i8.i32(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i8.i64(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m2(int8_t *base, vint8m2_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv32i8.i32(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv32i8.i64(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m4(int8_t *base, vint8m4_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv64i8.i32(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv64i8.i64(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m8(int8_t *base, vint8m8_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i16.i32(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i16.i64(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16mf4(int16_t *base, vint16mf4_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i16.i32(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i16.i64(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16mf2(int16_t *base, vint16mf2_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i16.i32(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i16.i64(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m1(int16_t *base, vint16m1_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i16.i32(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i16.i64(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m2(int16_t *base, vint16m2_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i16.i32(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i16.i64(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m4(int16_t *base, vint16m4_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv32i16.i32(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv32i16.i64(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m8(int16_t *base, vint16m8_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i32.i32(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i32.i64(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32mf2(int32_t *base, vint32mf2_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i32.i32(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i32.i64(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m1(int32_t *base, vint32m1_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i32.i32(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i32.i64(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m2(int32_t *base, vint32m2_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i32.i32(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i32.i64(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m4(int32_t *base, vint32m4_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i32.i32(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i32.i64(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m8(int32_t *base, vint32m8_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i64.i32(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i64.i64(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m1(int64_t *base, vint64m1_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i64.i32(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i64.i64(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m2(int64_t *base, vint64m2_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i64.i32(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i64.i64(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m4(int64_t *base, vint64m4_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i64.i32(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i64.i64(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m8(int64_t *base, vint64m8_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i8.i32(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i8.i64(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf8(uint8_t *base, vuint8mf8_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i8.i32(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i8.i64(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf4(uint8_t *base, vuint8mf4_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i8.i32(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i8.i64(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf2(uint8_t *base, vuint8mf2_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i8.i32(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i8.i64(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m1(uint8_t *base, vuint8m1_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i8.i32(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i8.i64(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m2(uint8_t *base, vuint8m2_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv32i8.i32(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv32i8.i64(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m4(uint8_t *base, vuint8m4_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv64i8.i32(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv64i8.i64(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m8(uint8_t *base, vuint8m8_t value, size_t vl) {
+  return vse8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i16.i32(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i16.i64(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16mf4(uint16_t *base, vuint16mf4_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i16.i32(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i16.i64(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16mf2(uint16_t *base, vuint16mf2_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i16.i32(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i16.i64(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m1(uint16_t *base, vuint16m1_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i16.i32(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i16.i64(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m2(uint16_t *base, vuint16m2_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i16.i32(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i16.i64(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m4(uint16_t *base, vuint16m4_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv32i16.i32(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv32i16.i64(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m8(uint16_t *base, vuint16m8_t value, size_t vl) {
+  return vse16(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i32.i32(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i32.i64(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32mf2(uint32_t *base, vuint32mf2_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i32.i32(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i32.i64(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m1(uint32_t *base, vuint32m1_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i32.i32(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i32.i64(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m2(uint32_t *base, vuint32m2_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i32.i32(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i32.i64(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m4(uint32_t *base, vuint32m4_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i32.i32(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i32.i64(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m8(uint32_t *base, vuint32m8_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i64.i32(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i64.i64(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m1(uint64_t *base, vuint64m1_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i64.i32(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i64.i64(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m2(uint64_t *base, vuint64m2_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i64.i32(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i64.i64(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m4(uint64_t *base, vuint64m4_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i64.i32(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i64.i64(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m8(uint64_t *base, vuint64m8_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1f32.i32(<vscale x 1 x float> [[VALUE:%.*]], <vscale x 1 x float>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1f32.i64(<vscale x 1 x float> [[VALUE:%.*]], <vscale x 1 x float>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32mf2(float *base, vfloat32mf2_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2f32.i32(<vscale x 2 x float> [[VALUE:%.*]], <vscale x 2 x float>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2f32.i64(<vscale x 2 x float> [[VALUE:%.*]], <vscale x 2 x float>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m1(float *base, vfloat32m1_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4f32.i32(<vscale x 4 x float> [[VALUE:%.*]], <vscale x 4 x float>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4f32.i64(<vscale x 4 x float> [[VALUE:%.*]], <vscale x 4 x float>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m2(float *base, vfloat32m2_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8f32.i32(<vscale x 8 x float> [[VALUE:%.*]], <vscale x 8 x float>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8f32.i64(<vscale x 8 x float> [[VALUE:%.*]], <vscale x 8 x float>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m4(float *base, vfloat32m4_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16f32.i32(<vscale x 16 x float> [[VALUE:%.*]], <vscale x 16 x float>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16f32.i64(<vscale x 16 x float> [[VALUE:%.*]], <vscale x 16 x float>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m8(float *base, vfloat32m8_t value, size_t vl) {
+  return vse32(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1f64.i32(<vscale x 1 x double> [[VALUE:%.*]], <vscale x 1 x double>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1f64.i64(<vscale x 1 x double> [[VALUE:%.*]], <vscale x 1 x double>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m1(double *base, vfloat64m1_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2f64.i32(<vscale x 2 x double> [[VALUE:%.*]], <vscale x 2 x double>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2f64.i64(<vscale x 2 x double> [[VALUE:%.*]], <vscale x 2 x double>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m2(double *base, vfloat64m2_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4f64.i32(<vscale x 4 x double> [[VALUE:%.*]], <vscale x 4 x double>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4f64.i64(<vscale x 4 x double> [[VALUE:%.*]], <vscale x 4 x double>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m4(double *base, vfloat64m4_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8f64.i32(<vscale x 8 x double> [[VALUE:%.*]], <vscale x 8 x double>* [[TMP0]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8f64.i64(<vscale x 8 x double> [[VALUE:%.*]], <vscale x 8 x double>* [[TMP0]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m8(double *base, vfloat64m8_t value, size_t vl) {
+  return vse64(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i8.i32(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i8.i64(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf8_m(vbool64_t mask, int8_t *base, vint8mf8_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i8.i32(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i8.i64(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf4_m(vbool32_t mask, int8_t *base, vint8mf4_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i8.i32(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i8.i64(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf2_m(vbool16_t mask, int8_t *base, vint8mf2_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i8.i32(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i8.i64(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m1_m(vbool8_t mask, int8_t *base, vint8m1_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i8.i32(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i8.i64(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m2_m(vbool4_t mask, int8_t *base, vint8m2_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv32i8.i32(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv32i8.i64(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m4_m(vbool2_t mask, int8_t *base, vint8m4_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv64i8.i32(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv64i8.i64(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m8_m(vbool1_t mask, int8_t *base, vint8m8_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i16.i32(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i16.i64(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16mf4_m(vbool64_t mask, int16_t *base, vint16mf4_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i16.i32(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i16.i64(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16mf2_m(vbool32_t mask, int16_t *base, vint16mf2_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i16.i32(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i16.i64(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m1_m(vbool16_t mask, int16_t *base, vint16m1_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i16.i32(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i16.i64(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m2_m(vbool8_t mask, int16_t *base, vint16m2_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i16.i32(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i16.i64(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m4_m(vbool4_t mask, int16_t *base, vint16m4_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv32i16.i32(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv32i16.i64(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m8_m(vbool2_t mask, int16_t *base, vint16m8_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i32.i32(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i32.i64(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32mf2_m(vbool64_t mask, int32_t *base, vint32mf2_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i32.i32(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i32.i64(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m1_m(vbool32_t mask, int32_t *base, vint32m1_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i32.i32(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i32.i64(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m2_m(vbool16_t mask, int32_t *base, vint32m2_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i32.i32(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i32.i64(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m4_m(vbool8_t mask, int32_t *base, vint32m4_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i32.i32(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i32.i64(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m8_m(vbool4_t mask, int32_t *base, vint32m8_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i64.i32(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i64.i64(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m1_m(vbool64_t mask, int64_t *base, vint64m1_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i64.i32(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i64.i64(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m2_m(vbool32_t mask, int64_t *base, vint64m2_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i64.i32(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i64.i64(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m4_m(vbool16_t mask, int64_t *base, vint64m4_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i64.i32(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i64.i64(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m8_m(vbool8_t mask, int64_t *base, vint64m8_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i8.i32(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i8.i64(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf8_m(vbool64_t mask, uint8_t *base, vuint8mf8_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i8.i32(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i8.i64(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf4_m(vbool32_t mask, uint8_t *base, vuint8mf4_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i8.i32(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i8.i64(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf2_m(vbool16_t mask, uint8_t *base, vuint8mf2_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i8.i32(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i8.i64(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m1_m(vbool8_t mask, uint8_t *base, vuint8m1_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i8.i32(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i8.i64(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m2_m(vbool4_t mask, uint8_t *base, vuint8m2_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv32i8.i32(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv32i8.i64(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m4_m(vbool2_t mask, uint8_t *base, vuint8m4_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv64i8.i32(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv64i8.i64(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m8_m(vbool1_t mask, uint8_t *base, vuint8m8_t value, size_t vl) {
+  return vse8(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i16.i32(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i16.i64(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16mf4_m(vbool64_t mask, uint16_t *base, vuint16mf4_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i16.i32(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i16.i64(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16mf2_m(vbool32_t mask, uint16_t *base, vuint16mf2_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i16.i32(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i16.i64(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m1_m(vbool16_t mask, uint16_t *base, vuint16m1_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i16.i32(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i16.i64(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m2_m(vbool8_t mask, uint16_t *base, vuint16m2_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i16.i32(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i16.i64(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m4_m(vbool4_t mask, uint16_t *base, vuint16m4_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv32i16.i32(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv32i16.i64(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m8_m(vbool2_t mask, uint16_t *base, vuint16m8_t value, size_t vl) {
+  return vse16(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i32.i32(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i32.i64(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32mf2_m(vbool64_t mask, uint32_t *base, vuint32mf2_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i32.i32(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i32.i64(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m1_m(vbool32_t mask, uint32_t *base, vuint32m1_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i32.i32(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i32.i64(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m2_m(vbool16_t mask, uint32_t *base, vuint32m2_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i32.i32(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i32.i64(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m4_m(vbool8_t mask, uint32_t *base, vuint32m4_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i32.i32(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i32.i64(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m8_m(vbool4_t mask, uint32_t *base, vuint32m8_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i64.i32(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i64.i64(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m1_m(vbool64_t mask, uint64_t *base, vuint64m1_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i64.i32(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i64.i64(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m2_m(vbool32_t mask, uint64_t *base, vuint64m2_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i64.i32(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i64.i64(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m4_m(vbool16_t mask, uint64_t *base, vuint64m4_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i64.i32(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i64.i64(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m8_m(vbool8_t mask, uint64_t *base, vuint64m8_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1f32.i32(<vscale x 1 x float> [[VALUE:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1f32.i64(<vscale x 1 x float> [[VALUE:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32mf2_m(vbool64_t mask, float *base, vfloat32mf2_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2f32.i32(<vscale x 2 x float> [[VALUE:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2f32.i64(<vscale x 2 x float> [[VALUE:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m1_m(vbool32_t mask, float *base, vfloat32m1_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4f32.i32(<vscale x 4 x float> [[VALUE:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4f32.i64(<vscale x 4 x float> [[VALUE:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m2_m(vbool16_t mask, float *base, vfloat32m2_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8f32.i32(<vscale x 8 x float> [[VALUE:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8f32.i64(<vscale x 8 x float> [[VALUE:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m4_m(vbool8_t mask, float *base, vfloat32m4_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16f32.i32(<vscale x 16 x float> [[VALUE:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16f32.i64(<vscale x 16 x float> [[VALUE:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m8_m(vbool4_t mask, float *base, vfloat32m8_t value, size_t vl) {
+  return vse32(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1f64.i32(<vscale x 1 x double> [[VALUE:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1f64.i64(<vscale x 1 x double> [[VALUE:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m1_m(vbool64_t mask, double *base, vfloat64m1_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2f64.i32(<vscale x 2 x double> [[VALUE:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2f64.i64(<vscale x 2 x double> [[VALUE:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m2_m(vbool32_t mask, double *base, vfloat64m2_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4f64.i32(<vscale x 4 x double> [[VALUE:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4f64.i64(<vscale x 4 x double> [[VALUE:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m4_m(vbool16_t mask, double *base, vfloat64m4_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8f64.i32(<vscale x 8 x double> [[VALUE:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8f64.i64(<vscale x 8 x double> [[VALUE:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]]) #[[ATTR11]]
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m8_m(vbool8_t mask, double *base, vfloat64m8_t value, size_t vl) {
+  return vse64(mask, base, value, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics/vadd.c b/clang/test/CodeGen/RISCV/rvv-intrinsics/vadd.c
index d3d73e8d7bba3963dfc3369e1a32fe7fc5c01083..902f5e0bf2ed74744db1d4200b8b25d7bd37ecf7 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics/vadd.c
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics/vadd.c
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
 // RUN:   -target-feature +experimental-zfh -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
 // RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
 // RUN:   -target-feature +experimental-zfh -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
-// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
-// RUN:   -target-feature +experimental-zfh -Werror -Wall -o - %s >/dev/null 2>%t
-// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
 
 // ASM-NOT: warning
 #include <riscv_vector.h>
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics/vfadd.c b/clang/test/CodeGen/RISCV/rvv-intrinsics/vfadd.c
index 2c8f0e54afcfe20a6d3683457a494fb2860ae533..43be374794d404bd84fa1f858112e5b64395b4c7 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics/vfadd.c
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics/vfadd.c
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
 // RUN:   -target-feature +experimental-zfh -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
 // RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
 // RUN:   -target-feature +experimental-zfh -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
-// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
-// RUN:   -target-feature +experimental-zfh -Werror -Wall -o - %s >/dev/null 2>%t
-// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
 
 // ASM-NOT: warning
 #include <riscv_vector.h>
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics/vle.c b/clang/test/CodeGen/RISCV/rvv-intrinsics/vle.c
new file mode 100644
index 0000000000000000000000000000000000000000..aca7f314b88197192ec8b4b022cf3ff0b3928719
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics/vle.c
@@ -0,0 +1,1707 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone  -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone  -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vle8_v_i8mf8(const int8_t *base, size_t vl) {
+  return vle8_v_i8mf8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vle8_v_i8mf4(const int8_t *base, size_t vl) {
+  return vle8_v_i8mf4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vle8_v_i8mf2(const int8_t *base, size_t vl) {
+  return vle8_v_i8mf2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vle8_v_i8m1(const int8_t *base, size_t vl) {
+  return vle8_v_i8m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vle8_v_i8m2(const int8_t *base, size_t vl) {
+  return vle8_v_i8m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vle8_v_i8m4(const int8_t *base, size_t vl) {
+  return vle8_v_i8m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t test_vle8_v_i8m8(const int8_t *base, size_t vl) {
+  return vle8_v_i8m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vle16_v_i16mf4(const int16_t *base, size_t vl) {
+  return vle16_v_i16mf4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vle16_v_i16mf2(const int16_t *base, size_t vl) {
+  return vle16_v_i16mf2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vle16_v_i16m1(const int16_t *base, size_t vl) {
+  return vle16_v_i16m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vle16_v_i16m2(const int16_t *base, size_t vl) {
+  return vle16_v_i16m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vle16_v_i16m4(const int16_t *base, size_t vl) {
+  return vle16_v_i16m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vle16_v_i16m8(const int16_t *base, size_t vl) {
+  return vle16_v_i16m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vle32_v_i32mf2(const int32_t *base, size_t vl) {
+  return vle32_v_i32mf2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vle32_v_i32m1(const int32_t *base, size_t vl) {
+  return vle32_v_i32m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vle32_v_i32m2(const int32_t *base, size_t vl) {
+  return vle32_v_i32m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vle32_v_i32m4(const int32_t *base, size_t vl) {
+  return vle32_v_i32m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vle32_v_i32m8(const int32_t *base, size_t vl) {
+  return vle32_v_i32m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vle64_v_i64m1(const int64_t *base, size_t vl) {
+  return vle64_v_i64m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vle64_v_i64m2(const int64_t *base, size_t vl) {
+  return vle64_v_i64m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vle64_v_i64m4(const int64_t *base, size_t vl) {
+  return vle64_v_i64m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vle64_v_i64m8(const int64_t *base, size_t vl) {
+  return vle64_v_i64m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vle8_v_u8mf8(const uint8_t *base, size_t vl) {
+  return vle8_v_u8mf8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vle8_v_u8mf4(const uint8_t *base, size_t vl) {
+  return vle8_v_u8mf4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vle8_v_u8mf2(const uint8_t *base, size_t vl) {
+  return vle8_v_u8mf2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vle8_v_u8m1(const uint8_t *base, size_t vl) {
+  return vle8_v_u8m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vle8_v_u8m2(const uint8_t *base, size_t vl) {
+  return vle8_v_u8m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vle8_v_u8m4(const uint8_t *base, size_t vl) {
+  return vle8_v_u8m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t test_vle8_v_u8m8(const uint8_t *base, size_t vl) {
+  return vle8_v_u8m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vle16_v_u16mf4(const uint16_t *base, size_t vl) {
+  return vle16_v_u16mf4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vle16_v_u16mf2(const uint16_t *base, size_t vl) {
+  return vle16_v_u16mf2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vle16_v_u16m1(const uint16_t *base, size_t vl) {
+  return vle16_v_u16m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vle16_v_u16m2(const uint16_t *base, size_t vl) {
+  return vle16_v_u16m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vle16_v_u16m4(const uint16_t *base, size_t vl) {
+  return vle16_v_u16m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vle16_v_u16m8(const uint16_t *base, size_t vl) {
+  return vle16_v_u16m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vle32_v_u32mf2(const uint32_t *base, size_t vl) {
+  return vle32_v_u32mf2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vle32_v_u32m1(const uint32_t *base, size_t vl) {
+  return vle32_v_u32m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vle32_v_u32m2(const uint32_t *base, size_t vl) {
+  return vle32_v_u32m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vle32_v_u32m4(const uint32_t *base, size_t vl) {
+  return vle32_v_u32m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vle32_v_u32m8(const uint32_t *base, size_t vl) {
+  return vle32_v_u32m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vle64_v_u64m1(const uint64_t *base, size_t vl) {
+  return vle64_v_u64m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vle64_v_u64m2(const uint64_t *base, size_t vl) {
+  return vle64_v_u64m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vle64_v_u64m4(const uint64_t *base, size_t vl) {
+  return vle64_v_u64m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vle64_v_u64m8(const uint64_t *base, size_t vl) {
+  return vle64_v_u64m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vle.nxv1f32.i32(<vscale x 1 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vle.nxv1f32.i64(<vscale x 1 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vle32_v_f32mf2(const float *base, size_t vl) {
+  return vle32_v_f32mf2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vle.nxv2f32.i32(<vscale x 2 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vle.nxv2f32.i64(<vscale x 2 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vle32_v_f32m1(const float *base, size_t vl) {
+  return vle32_v_f32m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vle.nxv4f32.i32(<vscale x 4 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vle.nxv4f32.i64(<vscale x 4 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vle32_v_f32m2(const float *base, size_t vl) {
+  return vle32_v_f32m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vle.nxv8f32.i32(<vscale x 8 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vle.nxv8f32.i64(<vscale x 8 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vle32_v_f32m4(const float *base, size_t vl) {
+  return vle32_v_f32m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vle.nxv16f32.i32(<vscale x 16 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vle.nxv16f32.i64(<vscale x 16 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vle32_v_f32m8(const float *base, size_t vl) {
+  return vle32_v_f32m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vle.nxv1f64.i32(<vscale x 1 x double>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vle.nxv1f64.i64(<vscale x 1 x double>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vle64_v_f64m1(const double *base, size_t vl) {
+  return vle64_v_f64m1(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vle.nxv2f64.i32(<vscale x 2 x double>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vle.nxv2f64.i64(<vscale x 2 x double>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vle64_v_f64m2(const double *base, size_t vl) {
+  return vle64_v_f64m2(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vle.nxv4f64.i32(<vscale x 4 x double>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vle.nxv4f64.i64(<vscale x 4 x double>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vle64_v_f64m4(const double *base, size_t vl) {
+  return vle64_v_f64m4(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vle.nxv8f64.i32(<vscale x 8 x double>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vle.nxv8f64.i64(<vscale x 8 x double>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vle64_v_f64m8(const double *base, size_t vl) {
+  return vle64_v_f64m8(base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.mask.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t test_vle8_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8_v_i8mf8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.mask.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t test_vle8_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8_v_i8mf4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.mask.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t test_vle8_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8_v_i8mf2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.mask.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t test_vle8_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8_v_i8m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.mask.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t test_vle8_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8_v_i8m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.mask.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t test_vle8_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8_v_i8m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_i8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.mask.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_i8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t test_vle8_v_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, const int8_t *base, size_t vl) {
+  return vle8_v_i8m8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.mask.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t test_vle16_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16_v_i16mf4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.mask.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t test_vle16_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16_v_i16mf2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.mask.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t test_vle16_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16_v_i16m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.mask.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t test_vle16_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16_v_i16m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.mask.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t test_vle16_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16_v_i16m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.mask.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t test_vle16_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, size_t vl) {
+  return vle16_v_i16m8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.mask.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t test_vle32_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32_v_i32mf2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.mask.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t test_vle32_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32_v_i32m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.mask.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t test_vle32_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32_v_i32m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.mask.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t test_vle32_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32_v_i32m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.mask.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t test_vle32_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, size_t vl) {
+  return vle32_v_i32m8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t test_vle64_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, size_t vl) {
+  return vle64_v_i64m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.mask.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t test_vle64_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, size_t vl) {
+  return vle64_v_i64m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.mask.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t test_vle64_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, size_t vl) {
+  return vle64_v_i64m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.mask.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t test_vle64_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, size_t vl) {
+  return vle64_v_i64m8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.mask.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vle.mask.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t test_vle8_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8_v_u8mf8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.mask.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vle.mask.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t test_vle8_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8_v_u8mf4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.mask.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vle.mask.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t test_vle8_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8_v_u8mf2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.mask.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vle.mask.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t test_vle8_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8_v_u8m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.mask.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vle.mask.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t test_vle8_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8_v_u8m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.mask.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vle.mask.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t test_vle8_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8_v_u8m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle8_v_u8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.mask.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle8_v_u8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vle.mask.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t test_vle8_v_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, const uint8_t *base, size_t vl) {
+  return vle8_v_u8m8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.mask.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vle.mask.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t test_vle16_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16_v_u16mf4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.mask.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vle.mask.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t test_vle16_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16_v_u16mf2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.mask.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vle.mask.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t test_vle16_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16_v_u16m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.mask.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vle.mask.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t test_vle16_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16_v_u16m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.mask.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vle.mask.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t test_vle16_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16_v_u16m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle16_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.mask.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle16_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vle.mask.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t test_vle16_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, size_t vl) {
+  return vle16_v_u16m8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.mask.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vle.mask.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t test_vle32_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32_v_u32mf2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.mask.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vle.mask.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t test_vle32_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32_v_u32m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.mask.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vle.mask.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t test_vle32_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32_v_u32m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.mask.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vle.mask.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t test_vle32_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32_v_u32m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.mask.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vle.mask.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t test_vle32_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, size_t vl) {
+  return vle32_v_u32m8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vle.mask.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t test_vle64_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, size_t vl) {
+  return vle64_v_u64m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.mask.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vle.mask.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t test_vle64_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, size_t vl) {
+  return vle64_v_u64m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.mask.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vle.mask.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t test_vle64_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, size_t vl) {
+  return vle64_v_u64m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.mask.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vle.mask.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t test_vle64_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, size_t vl) {
+  return vle64_v_u64m8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vle.mask.nxv1f32.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vle.mask.nxv1f32.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t test_vle32_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, size_t vl) {
+  return vle32_v_f32mf2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vle.mask.nxv2f32.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vle.mask.nxv2f32.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t test_vle32_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, size_t vl) {
+  return vle32_v_f32m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vle.mask.nxv4f32.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vle.mask.nxv4f32.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t test_vle32_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, size_t vl) {
+  return vle32_v_f32m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vle.mask.nxv8f32.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vle.mask.nxv8f32.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t test_vle32_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, size_t vl) {
+  return vle32_v_f32m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle32_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vle.mask.nxv16f32.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle32_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vle.mask.nxv16f32.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t test_vle32_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, size_t vl) {
+  return vle32_v_f32m8_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vle.mask.nxv1f64.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vle.mask.nxv1f64.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t test_vle64_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, size_t vl) {
+  return vle64_v_f64m1_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vle.mask.nxv2f64.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vle.mask.nxv2f64.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t test_vle64_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, size_t vl) {
+  return vle64_v_f64m2_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vle.mask.nxv4f64.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vle.mask.nxv4f64.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t test_vle64_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, size_t vl) {
+  return vle64_v_f64m4_m(mask, maskedoff, base, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vle64_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vle.mask.nxv8f64.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vle64_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vle.mask.nxv8f64.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t test_vle64_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, size_t vl) {
+  return vle64_v_f64m8_m(mask, maskedoff, base, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics/vloxei.c b/clang/test/CodeGen/RISCV/rvv-intrinsics/vloxei.c
new file mode 100644
index 0000000000000000000000000000000000000000..21ca49ded56b336dc5e572c010c0b37cc97970f8
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics/vloxei.c
@@ -0,0 +1,6123 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S > /dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei8_v_i8mf8(const int8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_i8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei8_v_i8mf4(const int8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_i8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei8_v_i8mf2(const int8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_i8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei8_v_i8m1(const int8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_i8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei8_v_i8m2(const int8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_i8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t testuxei8_v_i8m4(const int8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8_v_i8m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.nxv64i8.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.nxv64i8.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t testuxei8_v_i8m8(const int8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vloxei8_v_i8m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei16_v_i8mf8(const int8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_i8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei16_v_i8mf4(const int8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_i8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei16_v_i8mf2(const int8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_i8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei16_v_i8m1(const int8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_i8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei16_v_i8m2(const int8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_i8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t testuxei16_v_i8m4(const int8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16_v_i8m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei32_v_i8mf8(const int8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_i8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei32_v_i8mf4(const int8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_i8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei32_v_i8mf2(const int8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_i8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei32_v_i8m1(const int8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_i8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei32_v_i8m2(const int8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_i8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i64.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i64.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei64_v_i8mf8(const int8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_i8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i64.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i64.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei64_v_i8mf4(const int8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_i8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i64.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i64.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei64_v_i8mf2(const int8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_i8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i64.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i64.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei64_v_i8m1(const int8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_i8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei8_v_i16mf4(const int16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_i16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei8_v_i16mf2(const int16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_i16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei8_v_i16m1(const int16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_i16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei8_v_i16m2(const int16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_i16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei8_v_i16m4(const int16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_i16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t testuxei8_v_i16m8(const int16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8_v_i16m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei16_v_i16mf4(const int16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_i16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei16_v_i16mf2(const int16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_i16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei16_v_i16m1(const int16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_i16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei16_v_i16m2(const int16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_i16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei16_v_i16m4(const int16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_i16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t testuxei16_v_i16m8(const int16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16_v_i16m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei32_v_i16mf4(const int16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_i16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei32_v_i16mf2(const int16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_i16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei32_v_i16m1(const int16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_i16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei32_v_i16m2(const int16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_i16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei32_v_i16m4(const int16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_i16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i64.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i64.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei64_v_i16mf4(const int16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_i16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i64.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i64.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei64_v_i16mf2(const int16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_i16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i64.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i64.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei64_v_i16m1(const int16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_i16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i64.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i64.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei64_v_i16m2(const int16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_i16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei8_v_i32mf2(const int32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_i32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei8_v_i32m1(const int32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_i32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei8_v_i32m2(const int32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_i32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei8_v_i32m4(const int32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_i32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei8_v_i32m8(const int32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_i32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei16_v_i32mf2(const int32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_i32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei16_v_i32m1(const int32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_i32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei16_v_i32m2(const int32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_i32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei16_v_i32m4(const int32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_i32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei16_v_i32m8(const int32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_i32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei32_v_i32mf2(const int32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_i32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei32_v_i32m1(const int32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_i32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei32_v_i32m2(const int32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_i32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei32_v_i32m4(const int32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_i32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei32_v_i32m8(const int32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_i32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i64.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i64.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei64_v_i32mf2(const int32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_i32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i64.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i64.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei64_v_i32m1(const int32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_i32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i64.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i64.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei64_v_i32m2(const int32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_i32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i64.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i64.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei64_v_i32m4(const int32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_i32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i8.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i8.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei8_v_i64m1(const int64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_i64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i8.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i8.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei8_v_i64m2(const int64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_i64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i8.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i8.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei8_v_i64m4(const int64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_i64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i8.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i8.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei8_v_i64m8(const int64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_i64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i16.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i16.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei16_v_i64m1(const int64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_i64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i16.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i16.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei16_v_i64m2(const int64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_i64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i16.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i16.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei16_v_i64m4(const int64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_i64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i16.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i16.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei16_v_i64m8(const int64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_i64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i32.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i32.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei32_v_i64m1(const int64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_i64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i32.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i32.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei32_v_i64m2(const int64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_i64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i32.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i32.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei32_v_i64m4(const int64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_i64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i32.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i32.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei32_v_i64m8(const int64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_i64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei64_v_i64m1(const int64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_i64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei64_v_i64m2(const int64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_i64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei64_v_i64m4(const int64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_i64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei64_v_i64m8(const int64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_i64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei8_v_u8mf8(const uint8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_u8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei8_v_u8mf4(const uint8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_u8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei8_v_u8mf2(const uint8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_u8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei8_v_u8m1(const uint8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_u8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei8_v_u8m2(const uint8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_u8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t testuxei8_v_u8m4(const uint8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8_v_u8m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.nxv64i8.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.nxv64i8.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t testuxei8_v_u8m8(const uint8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vloxei8_v_u8m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei16_v_u8mf8(const uint8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_u8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei16_v_u8mf4(const uint8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_u8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei16_v_u8mf2(const uint8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_u8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei16_v_u8m1(const uint8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_u8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei16_v_u8m2(const uint8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_u8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t testuxei16_v_u8m4(const uint8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16_v_u8m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei32_v_u8mf8(const uint8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_u8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei32_v_u8mf4(const uint8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_u8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei32_v_u8mf2(const uint8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_u8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei32_v_u8m1(const uint8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_u8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei32_v_u8m2(const uint8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_u8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i64.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i64.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei64_v_u8mf8(const uint8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_u8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i64.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i64.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei64_v_u8mf4(const uint8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_u8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i64.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i64.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei64_v_u8mf2(const uint8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_u8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i64.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i64.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei64_v_u8m1(const uint8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_u8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei8_v_u16mf4(const uint16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_u16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei8_v_u16mf2(const uint16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_u16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei8_v_u16m1(const uint16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_u16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei8_v_u16m2(const uint16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_u16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei8_v_u16m4(const uint16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_u16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t testuxei8_v_u16m8(const uint16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8_v_u16m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei16_v_u16mf4(const uint16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_u16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei16_v_u16mf2(const uint16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_u16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei16_v_u16m1(const uint16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_u16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei16_v_u16m2(const uint16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_u16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei16_v_u16m4(const uint16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_u16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t testuxei16_v_u16m8(const uint16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16_v_u16m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei32_v_u16mf4(const uint16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_u16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei32_v_u16mf2(const uint16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_u16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei32_v_u16m1(const uint16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_u16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei32_v_u16m2(const uint16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_u16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei32_v_u16m4(const uint16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_u16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i64.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i64.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei64_v_u16mf4(const uint16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_u16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i64.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i64.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei64_v_u16mf2(const uint16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_u16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i64.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i64.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei64_v_u16m1(const uint16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_u16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i64.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i64.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei64_v_u16m2(const uint16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_u16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei8_v_u32mf2(const uint32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_u32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei8_v_u32m1(const uint32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_u32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei8_v_u32m2(const uint32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_u32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei8_v_u32m4(const uint32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_u32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei8_v_u32m8(const uint32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_u32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei16_v_u32mf2(const uint32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_u32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei16_v_u32m1(const uint32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_u32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei16_v_u32m2(const uint32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_u32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei16_v_u32m4(const uint32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_u32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei16_v_u32m8(const uint32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_u32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei32_v_u32mf2(const uint32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_u32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei32_v_u32m1(const uint32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_u32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei32_v_u32m2(const uint32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_u32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei32_v_u32m4(const uint32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_u32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei32_v_u32m8(const uint32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_u32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i64.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i64.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei64_v_u32mf2(const uint32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_u32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i64.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i64.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei64_v_u32m1(const uint32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_u32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i64.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i64.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei64_v_u32m2(const uint32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_u32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i64.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i64.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei64_v_u32m4(const uint32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_u32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i8.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i8.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei8_v_u64m1(const uint64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_u64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i8.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i8.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei8_v_u64m2(const uint64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_u64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i8.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i8.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei8_v_u64m4(const uint64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_u64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i8.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i8.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei8_v_u64m8(const uint64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_u64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i16.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i16.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei16_v_u64m1(const uint64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_u64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i16.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i16.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei16_v_u64m2(const uint64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_u64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i16.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i16.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei16_v_u64m4(const uint64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_u64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i16.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i16.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei16_v_u64m8(const uint64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_u64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i32.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i32.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei32_v_u64m1(const uint64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_u64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i32.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i32.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei32_v_u64m2(const uint64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_u64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i32.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i32.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei32_v_u64m4(const uint64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_u64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i32.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i32.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei32_v_u64m8(const uint64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_u64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei64_v_u64m1(const uint64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_u64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei64_v_u64m2(const uint64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_u64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei64_v_u64m4(const uint64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_u64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei64_v_u64m8(const uint64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_u64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i8.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i8.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei8_v_f32mf2(const float *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_f32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i8.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i8.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei8_v_f32m1(const float *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_f32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i8.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i8.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei8_v_f32m2(const float *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_f32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i8.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i8.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei8_v_f32m4(const float *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_f32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i8.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i8.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei8_v_f32m8(const float *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_f32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i16.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i16.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei16_v_f32mf2(const float *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_f32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i16.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i16.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei16_v_f32m1(const float *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_f32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i16.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i16.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei16_v_f32m2(const float *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_f32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i16.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i16.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei16_v_f32m4(const float *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_f32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i16.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i16.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei16_v_f32m8(const float *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_f32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i32.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i32.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei32_v_f32mf2(const float *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_f32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i32.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i32.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei32_v_f32m1(const float *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_f32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i32.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i32.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei32_v_f32m2(const float *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_f32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i32.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i32.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei32_v_f32m4(const float *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_f32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i32.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i32.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei32_v_f32m8(const float *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_f32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i64.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i64.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei64_v_f32mf2(const float *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_f32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i64.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i64.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei64_v_f32m1(const float *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_f32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i64.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i64.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei64_v_f32m2(const float *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_f32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i64.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i64.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei64_v_f32m4(const float *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_f32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i8.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i8.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei8_v_f64m1(const double *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_f64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i8.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i8.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei8_v_f64m2(const double *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_f64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i8.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i8.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei8_v_f64m4(const double *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_f64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i8.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i8.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei8_v_f64m8(const double *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_f64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i16.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i16.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei16_v_f64m1(const double *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_f64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i16.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i16.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei16_v_f64m2(const double *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_f64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i16.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i16.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei16_v_f64m4(const double *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_f64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i16.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i16.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei16_v_f64m8(const double *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_f64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i32.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i32.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei32_v_f64m1(const double *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_f64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i32.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i32.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei32_v_f64m2(const double *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_f64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i32.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i32.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei32_v_f64m4(const double *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_f64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i32.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i32.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei32_v_f64m8(const double *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_f64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i64.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i64.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei64_v_f64m1(const double *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_f64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i64.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i64.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei64_v_f64m2(const double *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_f64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i64.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i64.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei64_v_f64m4(const double *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_f64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i64.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i64.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei64_v_f64m8(const double *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_f64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei8_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_i8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei8_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_i8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei8_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_i8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei8_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_i8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei8_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_i8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t testuxei8_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8_v_i8m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.mask.nxv64i8.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.mask.nxv64i8.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t testuxei8_v_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, const int8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vloxei8_v_i8m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i16.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i16.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei16_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_i8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i16.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i16.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei16_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_i8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i16.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i16.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei16_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_i8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i16.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i16.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei16_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_i8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i16.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i16.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei16_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_i8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i16.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i16.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t testuxei16_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16_v_i8m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i32.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i32.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei32_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_i8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i32.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i32.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei32_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_i8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i32.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i32.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei32_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_i8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i32.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i32.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei32_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_i8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i32.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i32.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei32_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_i8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i64.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i64.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei64_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_i8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i64.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i64.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei64_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_i8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i64.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i64.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei64_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_i8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i64.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i64.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei64_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_i8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i8.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i8.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei8_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_i16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i8.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i8.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei8_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_i16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i8.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i8.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei8_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_i16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i8.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i8.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei8_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_i16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i8.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i8.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei8_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_i16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i8.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i8.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t testuxei8_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8_v_i16m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei16_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_i16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei16_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_i16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei16_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_i16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei16_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_i16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei16_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_i16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t testuxei16_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16_v_i16m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i32.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i32.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei32_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_i16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i32.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i32.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei32_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_i16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i32.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i32.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei32_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_i16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i32.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i32.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei32_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_i16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i32.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i32.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei32_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_i16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i64.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i64.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei64_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_i16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i64.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i64.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei64_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_i16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i64.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i64.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei64_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_i16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i64.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i64.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei64_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_i16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i8.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i8.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei8_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_i32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i8.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i8.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei8_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_i32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i8.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i8.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei8_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_i32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i8.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i8.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei8_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_i32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i8.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i8.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei8_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_i32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i16.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i16.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei16_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_i32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i16.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i16.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei16_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_i32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i16.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i16.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei16_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_i32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i16.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i16.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei16_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_i32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i16.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i16.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei16_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_i32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei32_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_i32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei32_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_i32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei32_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_i32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei32_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_i32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei32_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_i32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i64.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i64.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei64_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_i32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i64.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i64.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei64_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_i32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i64.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i64.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei64_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_i32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i64.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i64.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei64_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_i32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i8.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i8.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei8_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_i64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i8.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i8.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei8_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_i64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i8.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i8.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei8_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_i64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i8.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i8.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei8_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_i64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i16.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i16.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei16_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_i64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i16.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i16.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei16_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_i64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i16.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i16.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei16_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_i64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i16.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i16.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei16_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_i64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i32.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i32.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei32_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_i64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i32.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i32.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei32_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_i64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i32.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei32_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_i64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i32.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i32.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei32_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_i64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei64_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_i64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei64_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_i64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei64_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_i64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei64_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_i64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei8_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_u8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei8_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_u8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei8_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_u8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei8_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_u8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei8_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_u8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t testuxei8_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8_v_u8m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.mask.nxv64i8.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vloxei.mask.nxv64i8.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t testuxei8_v_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, const uint8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vloxei8_v_u8m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i16.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i16.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei16_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_u8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i16.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i16.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei16_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_u8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i16.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i16.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei16_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_u8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i16.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i16.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei16_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_u8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i16.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i16.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei16_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_u8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i16.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vloxei.mask.nxv32i8.nxv32i16.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t testuxei16_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16_v_u8m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i32.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i32.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei32_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_u8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i32.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i32.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei32_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_u8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i32.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i32.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei32_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_u8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i32.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i32.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei32_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_u8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i32.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vloxei.mask.nxv16i8.nxv16i32.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei32_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_u8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i64.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vloxei.mask.nxv1i8.nxv1i64.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei64_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_u8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i64.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vloxei.mask.nxv2i8.nxv2i64.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei64_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_u8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i64.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vloxei.mask.nxv4i8.nxv4i64.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei64_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_u8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i64.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vloxei.mask.nxv8i8.nxv8i64.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei64_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_u8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i8.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i8.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei8_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_u16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i8.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i8.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei8_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_u16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i8.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i8.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei8_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_u16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i8.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i8.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei8_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_u16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i8.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i8.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei8_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_u16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i8.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i8.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t testuxei8_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vloxei8_v_u16m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei16_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_u16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei16_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_u16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei16_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_u16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei16_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_u16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei16_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_u16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vloxei.mask.nxv32i16.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t testuxei16_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vloxei16_v_u16m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i32.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i32.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei32_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_u16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i32.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i32.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei32_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_u16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i32.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i32.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei32_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_u16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i32.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i32.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei32_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_u16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i32.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vloxei.mask.nxv16i16.nxv16i32.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei32_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_u16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i64.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vloxei.mask.nxv1i16.nxv1i64.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei64_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_u16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i64.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vloxei.mask.nxv2i16.nxv2i64.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei64_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_u16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i64.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vloxei.mask.nxv4i16.nxv4i64.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei64_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_u16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i64.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vloxei.mask.nxv8i16.nxv8i64.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei64_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_u16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i8.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i8.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei8_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_u32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i8.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i8.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei8_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_u32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i8.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i8.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei8_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_u32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i8.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i8.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei8_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_u32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i8.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i8.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei8_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_u32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i16.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i16.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei16_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_u32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i16.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i16.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei16_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_u32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i16.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i16.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei16_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_u32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i16.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i16.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei16_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_u32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i16.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i16.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei16_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_u32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei32_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_u32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei32_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_u32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei32_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_u32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei32_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_u32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vloxei.mask.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei32_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_u32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i64.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vloxei.mask.nxv1i32.nxv1i64.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei64_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_u32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i64.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vloxei.mask.nxv2i32.nxv2i64.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei64_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_u32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i64.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vloxei.mask.nxv4i32.nxv4i64.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei64_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_u32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i64.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vloxei.mask.nxv8i32.nxv8i64.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei64_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_u32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i8.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i8.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei8_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_u64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i8.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i8.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei8_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_u64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i8.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i8.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei8_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_u64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i8.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i8.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei8_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_u64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i16.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i16.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei16_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_u64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i16.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i16.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei16_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_u64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i16.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i16.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei16_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_u64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i16.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i16.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei16_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_u64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i32.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i32.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei32_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_u64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i32.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i32.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei32_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_u64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i32.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei32_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_u64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i32.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i32.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei32_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_u64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vloxei.mask.nxv1i64.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei64_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_u64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vloxei.mask.nxv2i64.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei64_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_u64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vloxei.mask.nxv4i64.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei64_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_u64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vloxei.mask.nxv8i64.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei64_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_u64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i8.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i8.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei8_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_f32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i8.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i8.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei8_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_f32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i8.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i8.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei8_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_f32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i8.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i8.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei8_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_f32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i8.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i8.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei8_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint8m2_t bindex, size_t vl) {
+  return vloxei8_v_f32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i16.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i16.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei16_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_f32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i16.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i16.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei16_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_f32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i16.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i16.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei16_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_f32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i16.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i16.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei16_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_f32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i16.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i16.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei16_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint16m4_t bindex, size_t vl) {
+  return vloxei16_v_f32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i32.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i32.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei32_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_f32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i32.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i32.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei32_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_f32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i32.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i32.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei32_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_f32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i32.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i32.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei32_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_f32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i32.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vloxei.mask.nxv16f32.nxv16i32.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei32_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint32m8_t bindex, size_t vl) {
+  return vloxei32_v_f32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i64.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vloxei.mask.nxv1f32.nxv1i64.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei64_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_f32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i64.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vloxei.mask.nxv2f32.nxv2i64.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei64_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_f32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i64.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vloxei.mask.nxv4f32.nxv4i64.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei64_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_f32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i64.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vloxei.mask.nxv8f32.nxv8i64.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei64_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_f32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i8.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i8.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei8_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint8mf8_t bindex, size_t vl) {
+  return vloxei8_v_f64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i8.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i8.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei8_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint8mf4_t bindex, size_t vl) {
+  return vloxei8_v_f64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i8.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i8.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei8_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint8mf2_t bindex, size_t vl) {
+  return vloxei8_v_f64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i8.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i8.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei8_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint8m1_t bindex, size_t vl) {
+  return vloxei8_v_f64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i16.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i16.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei16_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint16mf4_t bindex, size_t vl) {
+  return vloxei16_v_f64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i16.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i16.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei16_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint16mf2_t bindex, size_t vl) {
+  return vloxei16_v_f64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i16.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i16.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei16_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint16m1_t bindex, size_t vl) {
+  return vloxei16_v_f64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i16.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i16.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei16_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint16m2_t bindex, size_t vl) {
+  return vloxei16_v_f64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i32.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i32.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei32_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint32mf2_t bindex, size_t vl) {
+  return vloxei32_v_f64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i32.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i32.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei32_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint32m1_t bindex, size_t vl) {
+  return vloxei32_v_f64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i32.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i32.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei32_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint32m2_t bindex, size_t vl) {
+  return vloxei32_v_f64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i32.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i32.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei32_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint32m4_t bindex, size_t vl) {
+  return vloxei32_v_f64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i64.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vloxei.mask.nxv1f64.nxv1i64.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei64_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint64m1_t bindex, size_t vl) {
+  return vloxei64_v_f64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i64.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vloxei.mask.nxv2f64.nxv2i64.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei64_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint64m2_t bindex, size_t vl) {
+  return vloxei64_v_f64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i64.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vloxei.mask.nxv4f64.nxv4i64.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei64_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint64m4_t bindex, size_t vl) {
+  return vloxei64_v_f64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i64.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vloxei.mask.nxv8f64.nxv8i64.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei64_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint64m8_t bindex, size_t vl) {
+  return vloxei64_v_f64m8_m(mask, maskedoff, base, bindex, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics/vluxei.c b/clang/test/CodeGen/RISCV/rvv-intrinsics/vluxei.c
new file mode 100644
index 0000000000000000000000000000000000000000..ae34fbe1909ab412861760e2b2090bc78843d569
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics/vluxei.c
@@ -0,0 +1,6123 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S > /dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei8_v_i8mf8(const int8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_i8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei8_v_i8mf4(const int8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_i8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei8_v_i8mf2(const int8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_i8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei8_v_i8m1(const int8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_i8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei8_v_i8m2(const int8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_i8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t testuxei8_v_i8m4(const int8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8_v_i8m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.nxv64i8.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.nxv64i8.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t testuxei8_v_i8m8(const int8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vluxei8_v_i8m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei16_v_i8mf8(const int8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_i8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei16_v_i8mf4(const int8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_i8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei16_v_i8mf2(const int8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_i8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei16_v_i8m1(const int8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_i8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei16_v_i8m2(const int8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_i8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t testuxei16_v_i8m4(const int8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16_v_i8m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei32_v_i8mf8(const int8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_i8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei32_v_i8mf4(const int8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_i8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei32_v_i8mf2(const int8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_i8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei32_v_i8m1(const int8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_i8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei32_v_i8m2(const int8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_i8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i64.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i64.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei64_v_i8mf8(const int8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_i8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i64.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i64.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei64_v_i8mf4(const int8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_i8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i64.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i64.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei64_v_i8mf2(const int8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_i8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i64.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i64.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei64_v_i8m1(const int8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_i8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei8_v_i16mf4(const int16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_i16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei8_v_i16mf2(const int16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_i16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei8_v_i16m1(const int16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_i16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei8_v_i16m2(const int16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_i16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei8_v_i16m4(const int16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_i16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t testuxei8_v_i16m8(const int16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8_v_i16m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei16_v_i16mf4(const int16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_i16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei16_v_i16mf2(const int16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_i16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei16_v_i16m1(const int16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_i16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei16_v_i16m2(const int16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_i16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei16_v_i16m4(const int16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_i16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t testuxei16_v_i16m8(const int16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16_v_i16m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei32_v_i16mf4(const int16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_i16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei32_v_i16mf2(const int16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_i16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei32_v_i16m1(const int16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_i16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei32_v_i16m2(const int16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_i16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei32_v_i16m4(const int16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_i16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i64.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i64.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei64_v_i16mf4(const int16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_i16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i64.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i64.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei64_v_i16mf2(const int16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_i16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i64.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i64.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei64_v_i16m1(const int16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_i16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i64.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i64.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei64_v_i16m2(const int16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_i16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei8_v_i32mf2(const int32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_i32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei8_v_i32m1(const int32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_i32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei8_v_i32m2(const int32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_i32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei8_v_i32m4(const int32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_i32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei8_v_i32m8(const int32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_i32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei16_v_i32mf2(const int32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_i32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei16_v_i32m1(const int32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_i32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei16_v_i32m2(const int32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_i32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei16_v_i32m4(const int32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_i32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei16_v_i32m8(const int32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_i32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei32_v_i32mf2(const int32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_i32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei32_v_i32m1(const int32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_i32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei32_v_i32m2(const int32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_i32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei32_v_i32m4(const int32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_i32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei32_v_i32m8(const int32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_i32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i64.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i64.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei64_v_i32mf2(const int32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_i32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i64.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i64.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei64_v_i32m1(const int32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_i32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i64.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i64.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei64_v_i32m2(const int32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_i32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i64.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i64.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei64_v_i32m4(const int32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_i32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i8.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i8.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei8_v_i64m1(const int64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_i64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i8.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i8.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei8_v_i64m2(const int64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_i64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i8.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i8.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei8_v_i64m4(const int64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_i64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i8.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i8.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei8_v_i64m8(const int64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_i64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i16.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i16.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei16_v_i64m1(const int64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_i64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i16.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i16.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei16_v_i64m2(const int64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_i64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i16.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i16.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei16_v_i64m4(const int64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_i64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i16.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i16.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei16_v_i64m8(const int64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_i64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i32.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i32.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei32_v_i64m1(const int64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_i64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i32.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i32.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei32_v_i64m2(const int64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_i64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i32.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i32.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei32_v_i64m4(const int64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_i64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i32.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i32.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei32_v_i64m8(const int64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_i64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei64_v_i64m1(const int64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_i64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei64_v_i64m2(const int64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_i64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei64_v_i64m4(const int64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_i64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei64_v_i64m8(const int64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_i64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei8_v_u8mf8(const uint8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_u8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei8_v_u8mf4(const uint8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_u8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei8_v_u8mf2(const uint8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_u8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei8_v_u8m1(const uint8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_u8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i8.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i8.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei8_v_u8m2(const uint8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_u8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i8.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i8.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t testuxei8_v_u8m4(const uint8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8_v_u8m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.nxv64i8.nxv64i8.i32(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.nxv64i8.nxv64i8.i64(<vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t testuxei8_v_u8m8(const uint8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vluxei8_v_u8m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei16_v_u8mf8(const uint8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_u8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei16_v_u8mf4(const uint8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_u8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei16_v_u8mf2(const uint8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_u8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei16_v_u8m1(const uint8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_u8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei16_v_u8m2(const uint8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_u8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16.i32(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16.i64(<vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t testuxei16_v_u8m4(const uint8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16_v_u8m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei32_v_u8mf8(const uint8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_u8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei32_v_u8mf4(const uint8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_u8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei32_v_u8mf2(const uint8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_u8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei32_v_u8m1(const uint8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_u8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32.i32(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32.i64(<vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei32_v_u8m2(const uint8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_u8m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i64.i32(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i64.i64(<vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei64_v_u8mf8(const uint8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_u8mf8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i64.i32(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i64.i64(<vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei64_v_u8mf4(const uint8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_u8mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i64.i32(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i64.i64(<vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei64_v_u8mf2(const uint8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_u8mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i64.i32(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i64.i64(<vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei64_v_u8m1(const uint8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_u8m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei8_v_u16mf4(const uint16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_u16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei8_v_u16mf2(const uint16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_u16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei8_v_u16m1(const uint16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_u16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei8_v_u16m2(const uint16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_u16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei8_v_u16m4(const uint16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_u16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t testuxei8_v_u16m8(const uint16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8_v_u16m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i16.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i16.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei16_v_u16mf4(const uint16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_u16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i16.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i16.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei16_v_u16mf2(const uint16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_u16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i16.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i16.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei16_v_u16m1(const uint16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_u16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i16.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i16.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei16_v_u16m2(const uint16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_u16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i16.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i16.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei16_v_u16m4(const uint16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_u16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i16.i32(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i16.i64(<vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t testuxei16_v_u16m8(const uint16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16_v_u16m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei32_v_u16mf4(const uint16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_u16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei32_v_u16mf2(const uint16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_u16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei32_v_u16m1(const uint16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_u16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei32_v_u16m2(const uint16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_u16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32.i32(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32.i64(<vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei32_v_u16m4(const uint16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_u16m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i64.i32(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i64.i64(<vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei64_v_u16mf4(const uint16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_u16mf4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i64.i32(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i64.i64(<vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei64_v_u16mf2(const uint16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_u16mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i64.i32(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i64.i64(<vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei64_v_u16m1(const uint16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_u16m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i64.i32(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i64.i64(<vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei64_v_u16m2(const uint16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_u16m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei8_v_u32mf2(const uint32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_u32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei8_v_u32m1(const uint32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_u32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei8_v_u32m2(const uint32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_u32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei8_v_u32m4(const uint32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_u32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei8_v_u32m8(const uint32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_u32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei16_v_u32mf2(const uint32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_u32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei16_v_u32m1(const uint32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_u32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei16_v_u32m2(const uint32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_u32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei16_v_u32m4(const uint32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_u32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei16_v_u32m8(const uint32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_u32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i32.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i32.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei32_v_u32mf2(const uint32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_u32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i32.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei32_v_u32m1(const uint32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_u32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i32.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i32.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei32_v_u32m2(const uint32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_u32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i32.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i32.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei32_v_u32m4(const uint32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_u32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i32.i32(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i32.i64(<vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei32_v_u32m8(const uint32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_u32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i64.i32(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i64.i64(<vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei64_v_u32mf2(const uint32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_u32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i64.i32(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i64.i64(<vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei64_v_u32m1(const uint32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_u32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i64.i32(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i64.i64(<vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei64_v_u32m2(const uint32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_u32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i64.i32(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i64.i64(<vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei64_v_u32m4(const uint32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_u32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i8.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i8.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei8_v_u64m1(const uint64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_u64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i8.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i8.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei8_v_u64m2(const uint64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_u64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i8.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i8.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei8_v_u64m4(const uint64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_u64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i8.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i8.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei8_v_u64m8(const uint64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_u64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i16.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i16.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei16_v_u64m1(const uint64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_u64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i16.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i16.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei16_v_u64m2(const uint64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_u64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i16.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i16.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei16_v_u64m4(const uint64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_u64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i16.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i16.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei16_v_u64m8(const uint64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_u64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i32.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i32.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei32_v_u64m1(const uint64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_u64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i32.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i32.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei32_v_u64m2(const uint64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_u64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i32.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i32.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei32_v_u64m4(const uint64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_u64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i32.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i32.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei32_v_u64m8(const uint64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_u64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i64.i32(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei64_v_u64m1(const uint64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_u64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i64.i32(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i64.i64(<vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei64_v_u64m2(const uint64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_u64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i64.i32(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i64.i64(<vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei64_v_u64m4(const uint64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_u64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i64.i32(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i64.i64(<vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei64_v_u64m8(const uint64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_u64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i8.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i8.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei8_v_f32mf2(const float *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_f32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i8.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i8.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei8_v_f32m1(const float *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_f32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i8.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i8.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei8_v_f32m2(const float *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_f32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i8.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i8.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei8_v_f32m4(const float *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_f32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i8.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i8.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei8_v_f32m8(const float *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_f32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i16.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i16.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei16_v_f32mf2(const float *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_f32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i16.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i16.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei16_v_f32m1(const float *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_f32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i16.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i16.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei16_v_f32m2(const float *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_f32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i16.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i16.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei16_v_f32m4(const float *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_f32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i16.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i16.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei16_v_f32m8(const float *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_f32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i32.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i32.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei32_v_f32mf2(const float *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_f32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i32.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i32.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei32_v_f32m1(const float *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_f32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i32.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i32.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei32_v_f32m2(const float *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_f32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i32.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i32.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei32_v_f32m4(const float *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_f32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i32.i32(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i32.i64(<vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei32_v_f32m8(const float *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_f32m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i64.i32(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i64.i64(<vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei64_v_f32mf2(const float *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_f32mf2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i64.i32(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i64.i64(<vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei64_v_f32m1(const float *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_f32m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i64.i32(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i64.i64(<vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei64_v_f32m2(const float *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_f32m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i64.i32(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i64.i64(<vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei64_v_f32m4(const float *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_f32m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i8.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i8.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei8_v_f64m1(const double *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_f64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i8.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i8.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei8_v_f64m2(const double *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_f64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i8.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i8.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei8_v_f64m4(const double *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_f64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i8.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i8.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei8_v_f64m8(const double *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_f64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i16.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i16.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei16_v_f64m1(const double *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_f64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i16.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i16.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei16_v_f64m2(const double *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_f64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i16.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i16.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei16_v_f64m4(const double *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_f64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i16.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i16.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei16_v_f64m8(const double *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_f64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i32.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i32.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei32_v_f64m1(const double *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_f64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i32.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i32.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei32_v_f64m2(const double *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_f64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i32.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i32.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei32_v_f64m4(const double *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_f64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i32.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i32.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei32_v_f64m8(const double *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_f64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i64.i32(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i64.i64(<vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei64_v_f64m1(const double *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_f64m1(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i64.i32(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i64.i64(<vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei64_v_f64m2(const double *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_f64m2(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i64.i32(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i64.i64(<vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei64_v_f64m4(const double *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_f64m4(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i64.i32(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i64.i64(<vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei64_v_f64m8(const double *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_f64m8(base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei8_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_i8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei8_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_i8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei8_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_i8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei8_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_i8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei8_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_i8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t testuxei8_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8_v_i8m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.mask.nxv64i8.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.mask.nxv64i8.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vint8m8_t testuxei8_v_i8m8_m(vbool1_t mask, vint8m8_t maskedoff, const int8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vluxei8_v_i8m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i16.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i16.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei16_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_i8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i16.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i16.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei16_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_i8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i16.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i16.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei16_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_i8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i16.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i16.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei16_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_i8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i16.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i16.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei16_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_i8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i16.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i16.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vint8m4_t testuxei16_v_i8m4_m(vbool2_t mask, vint8m4_t maskedoff, const int8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16_v_i8m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i32.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i32.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei32_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_i8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i32.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i32.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei32_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_i8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i32.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i32.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei32_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_i8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i32.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i32.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei32_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_i8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i32.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i32.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vint8m2_t testuxei32_v_i8m2_m(vbool4_t mask, vint8m2_t maskedoff, const int8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_i8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i64.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i64.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vint8mf8_t testuxei64_v_i8mf8_m(vbool64_t mask, vint8mf8_t maskedoff, const int8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_i8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i64.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i64.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vint8mf4_t testuxei64_v_i8mf4_m(vbool32_t mask, vint8mf4_t maskedoff, const int8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_i8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i64.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i64.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vint8mf2_t testuxei64_v_i8mf2_m(vbool16_t mask, vint8mf2_t maskedoff, const int8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_i8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i64.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i64.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vint8m1_t testuxei64_v_i8m1_m(vbool8_t mask, vint8m1_t maskedoff, const int8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_i8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i8.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i8.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei8_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_i16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i8.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i8.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei8_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_i16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i8.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i8.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei8_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_i16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i8.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i8.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei8_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_i16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i8.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i8.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei8_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_i16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i8.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i8.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t testuxei8_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8_v_i16m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei16_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_i16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei16_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_i16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei16_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_i16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei16_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_i16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei16_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_i16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vint16m8_t testuxei16_v_i16m8_m(vbool2_t mask, vint16m8_t maskedoff, const int16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16_v_i16m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i32.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i32.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei32_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_i16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i32.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i32.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei32_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_i16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i32.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i32.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei32_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_i16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i32.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i32.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei32_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_i16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i32.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i32.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vint16m4_t testuxei32_v_i16m4_m(vbool4_t mask, vint16m4_t maskedoff, const int16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_i16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i64.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i64.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vint16mf4_t testuxei64_v_i16mf4_m(vbool64_t mask, vint16mf4_t maskedoff, const int16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_i16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i64.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i64.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vint16mf2_t testuxei64_v_i16mf2_m(vbool32_t mask, vint16mf2_t maskedoff, const int16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_i16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i64.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i64.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vint16m1_t testuxei64_v_i16m1_m(vbool16_t mask, vint16m1_t maskedoff, const int16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_i16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i64.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i64.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vint16m2_t testuxei64_v_i16m2_m(vbool8_t mask, vint16m2_t maskedoff, const int16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_i16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i8.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i8.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei8_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_i32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i8.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i8.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei8_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_i32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i8.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i8.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei8_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_i32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i8.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i8.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei8_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_i32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i8.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i8.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei8_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_i32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i16.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i16.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei16_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_i32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i16.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i16.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei16_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_i32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i16.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i16.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei16_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_i32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i16.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i16.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei16_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_i32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i16.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i16.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei16_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_i32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei32_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_i32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei32_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_i32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei32_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_i32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei32_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_i32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vint32m8_t testuxei32_v_i32m8_m(vbool4_t mask, vint32m8_t maskedoff, const int32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_i32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i64.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i64.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vint32mf2_t testuxei64_v_i32mf2_m(vbool64_t mask, vint32mf2_t maskedoff, const int32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_i32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i64.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i64.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vint32m1_t testuxei64_v_i32m1_m(vbool32_t mask, vint32m1_t maskedoff, const int32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_i32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i64.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i64.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vint32m2_t testuxei64_v_i32m2_m(vbool16_t mask, vint32m2_t maskedoff, const int32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_i32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i64.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i64.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vint32m4_t testuxei64_v_i32m4_m(vbool8_t mask, vint32m4_t maskedoff, const int32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_i32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i8.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i8.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei8_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_i64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i8.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i8.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei8_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_i64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i8.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i8.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei8_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_i64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i8.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i8.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei8_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_i64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i16.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i16.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei16_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_i64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i16.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i16.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei16_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_i64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i16.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i16.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei16_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_i64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i16.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i16.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei16_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_i64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i32.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i32.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei32_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_i64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i32.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i32.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei32_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_i64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i32.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei32_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_i64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i32.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i32.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei32_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_i64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vint64m1_t testuxei64_v_i64m1_m(vbool64_t mask, vint64m1_t maskedoff, const int64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_i64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vint64m2_t testuxei64_v_i64m2_m(vbool32_t mask, vint64m2_t maskedoff, const int64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_i64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vint64m4_t testuxei64_v_i64m4_m(vbool16_t mask, vint64m4_t maskedoff, const int64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_i64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vint64m8_t testuxei64_v_i64m8_m(vbool8_t mask, vint64m8_t maskedoff, const int64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_i64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei8_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_u8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei8_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_u8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei8_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_u8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei8_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_u8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i8.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i8.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei8_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_u8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i8.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i8.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t testuxei8_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8_v_u8m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.mask.nxv64i8.nxv64i8.i32(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 64 x i8> @llvm.riscv.vluxei.mask.nxv64i8.nxv64i8.i64(<vscale x 64 x i8> [[MASKEDOFF:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i8> [[BINDEX:%.*]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP1]]
+//
+vuint8m8_t testuxei8_v_u8m8_m(vbool1_t mask, vuint8m8_t maskedoff, const uint8_t *base, vuint8m8_t bindex, size_t vl) {
+  return vluxei8_v_u8m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i16.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i16.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei16_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_u8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i16.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i16.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei16_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_u8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i16.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i16.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei16_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_u8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i16.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i16.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei16_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_u8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i16.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i16.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei16_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_u8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i16.i32(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i8> @llvm.riscv.vluxei.mask.nxv32i8.nxv32i16.i64(<vscale x 32 x i8> [[MASKEDOFF:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP1]]
+//
+vuint8m4_t testuxei16_v_u8m4_m(vbool2_t mask, vuint8m4_t maskedoff, const uint8_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16_v_u8m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i32.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i32.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei32_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_u8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i32.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i32.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei32_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_u8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i32.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i32.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei32_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_u8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i32.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i32.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei32_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_u8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i32.i32(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i8> @llvm.riscv.vluxei.mask.nxv16i8.nxv16i32.i64(<vscale x 16 x i8> [[MASKEDOFF:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+vuint8m2_t testuxei32_v_u8m2_m(vbool4_t mask, vuint8m2_t maskedoff, const uint8_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_u8m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i64.i32(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i8> @llvm.riscv.vluxei.mask.nxv1i8.nxv1i64.i64(<vscale x 1 x i8> [[MASKEDOFF:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP1]]
+//
+vuint8mf8_t testuxei64_v_u8mf8_m(vbool64_t mask, vuint8mf8_t maskedoff, const uint8_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_u8mf8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i64.i32(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i8> @llvm.riscv.vluxei.mask.nxv2i8.nxv2i64.i64(<vscale x 2 x i8> [[MASKEDOFF:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP1]]
+//
+vuint8mf4_t testuxei64_v_u8mf4_m(vbool32_t mask, vuint8mf4_t maskedoff, const uint8_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_u8mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i64.i32(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.riscv.vluxei.mask.nxv4i8.nxv4i64.i64(<vscale x 4 x i8> [[MASKEDOFF:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP1]]
+//
+vuint8mf2_t testuxei64_v_u8mf2_m(vbool16_t mask, vuint8mf2_t maskedoff, const uint8_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_u8mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i64.i32(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vluxei.mask.nxv8i8.nxv8i64.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP1]]
+//
+vuint8m1_t testuxei64_v_u8m1_m(vbool8_t mask, vuint8m1_t maskedoff, const uint8_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_u8m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i8.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i8.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei8_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_u16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i8.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i8.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei8_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_u16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i8.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i8.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei8_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_u16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i8.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i8.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei8_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_u16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i8.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i8.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei8_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_u16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i8.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i8.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i8> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t testuxei8_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, vuint8m4_t bindex, size_t vl) {
+  return vluxei8_v_u16m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i16.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i16.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei16_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_u16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i16.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i16.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei16_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_u16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i16.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i16.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei16_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_u16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i16.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i16.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei16_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_u16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i16.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i16.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei16_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_u16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i16.i32(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 32 x i16> @llvm.riscv.vluxei.mask.nxv32i16.nxv32i16.i64(<vscale x 32 x i16> [[MASKEDOFF:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i16> [[BINDEX:%.*]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP1]]
+//
+vuint16m8_t testuxei16_v_u16m8_m(vbool2_t mask, vuint16m8_t maskedoff, const uint16_t *base, vuint16m8_t bindex, size_t vl) {
+  return vluxei16_v_u16m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i32.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i32.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei32_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_u16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i32.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i32.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei32_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_u16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i32.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i32.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei32_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_u16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i32.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i32.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei32_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_u16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i32.i32(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i16> @llvm.riscv.vluxei.mask.nxv16i16.nxv16i32.i64(<vscale x 16 x i16> [[MASKEDOFF:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP1]]
+//
+vuint16m4_t testuxei32_v_u16m4_m(vbool4_t mask, vuint16m4_t maskedoff, const uint16_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_u16m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i64.i32(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i16> @llvm.riscv.vluxei.mask.nxv1i16.nxv1i64.i64(<vscale x 1 x i16> [[MASKEDOFF:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP1]]
+//
+vuint16mf4_t testuxei64_v_u16mf4_m(vbool64_t mask, vuint16mf4_t maskedoff, const uint16_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_u16mf4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i64.i32(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i16> @llvm.riscv.vluxei.mask.nxv2i16.nxv2i64.i64(<vscale x 2 x i16> [[MASKEDOFF:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP1]]
+//
+vuint16mf2_t testuxei64_v_u16mf2_m(vbool32_t mask, vuint16mf2_t maskedoff, const uint16_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_u16mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i64.i32(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i16> @llvm.riscv.vluxei.mask.nxv4i16.nxv4i64.i64(<vscale x 4 x i16> [[MASKEDOFF:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP1]]
+//
+vuint16m1_t testuxei64_v_u16m1_m(vbool16_t mask, vuint16m1_t maskedoff, const uint16_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_u16m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i64.i32(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.riscv.vluxei.mask.nxv8i16.nxv8i64.i64(<vscale x 8 x i16> [[MASKEDOFF:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+vuint16m2_t testuxei64_v_u16m2_m(vbool8_t mask, vuint16m2_t maskedoff, const uint16_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_u16m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i8.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i8.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei8_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_u32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i8.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i8.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei8_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_u32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i8.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i8.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei8_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_u32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i8.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i8.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei8_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_u32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i8.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i8.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei8_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_u32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i16.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i16.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei16_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_u32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i16.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i16.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei16_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_u32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i16.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i16.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei16_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_u32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i16.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i16.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei16_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_u32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i16.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i16.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei16_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_u32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i32.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i32.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei32_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_u32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i32.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei32_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_u32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i32.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i32.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei32_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_u32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i32.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei32_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_u32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i32.i32(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i32> @llvm.riscv.vluxei.mask.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> [[MASKEDOFF:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP1]]
+//
+vuint32m8_t testuxei32_v_u32m8_m(vbool4_t mask, vuint32m8_t maskedoff, const uint32_t *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_u32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i64.i32(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i32> @llvm.riscv.vluxei.mask.nxv1i32.nxv1i64.i64(<vscale x 1 x i32> [[MASKEDOFF:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP1]]
+//
+vuint32mf2_t testuxei64_v_u32mf2_m(vbool64_t mask, vuint32mf2_t maskedoff, const uint32_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_u32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i64.i32(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vluxei.mask.nxv2i32.nxv2i64.i64(<vscale x 2 x i32> [[MASKEDOFF:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP1]]
+//
+vuint32m1_t testuxei64_v_u32m1_m(vbool32_t mask, vuint32m1_t maskedoff, const uint32_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_u32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i64.i32(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.riscv.vluxei.mask.nxv4i32.nxv4i64.i64(<vscale x 4 x i32> [[MASKEDOFF:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+vuint32m2_t testuxei64_v_u32m2_m(vbool16_t mask, vuint32m2_t maskedoff, const uint32_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_u32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i64.i32(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i32> @llvm.riscv.vluxei.mask.nxv8i32.nxv8i64.i64(<vscale x 8 x i32> [[MASKEDOFF:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP1]]
+//
+vuint32m4_t testuxei64_v_u32m4_m(vbool8_t mask, vuint32m4_t maskedoff, const uint32_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_u32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i8.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i8.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei8_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_u64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i8.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i8.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei8_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_u64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i8.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i8.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei8_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_u64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i8.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i8.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei8_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_u64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i16.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i16.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei16_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_u64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i16.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i16.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei16_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_u64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i16.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i16.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei16_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_u64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i16.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i16.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei16_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_u64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i32.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i32.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei32_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_u64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i32.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i32.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei32_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_u64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i32.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i32.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei32_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_u64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i32.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i32.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei32_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_u64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i64.i32(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x i64> @llvm.riscv.vluxei.mask.nxv1i64.nxv1i64.i64(<vscale x 1 x i64> [[MASKEDOFF:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP1]]
+//
+vuint64m1_t testuxei64_v_u64m1_m(vbool64_t mask, vuint64m1_t maskedoff, const uint64_t *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_u64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i64.i32(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.riscv.vluxei.mask.nxv2i64.nxv2i64.i64(<vscale x 2 x i64> [[MASKEDOFF:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+vuint64m2_t testuxei64_v_u64m2_m(vbool32_t mask, vuint64m2_t maskedoff, const uint64_t *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_u64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i64.i32(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i64> @llvm.riscv.vluxei.mask.nxv4i64.nxv4i64.i64(<vscale x 4 x i64> [[MASKEDOFF:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP1]]
+//
+vuint64m4_t testuxei64_v_u64m4_m(vbool16_t mask, vuint64m4_t maskedoff, const uint64_t *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_u64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i64.i32(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.riscv.vluxei.mask.nxv8i64.nxv8i64.i64(<vscale x 8 x i64> [[MASKEDOFF:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP1]]
+//
+vuint64m8_t testuxei64_v_u64m8_m(vbool8_t mask, vuint64m8_t maskedoff, const uint64_t *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_u64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i8.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i8.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei8_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_f32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i8.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i8.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei8_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_f32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i8.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i8.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei8_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_f32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i8.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i8.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei8_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_f32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i8.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i8.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i8> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei8_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint8m2_t bindex, size_t vl) {
+  return vluxei8_v_f32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i16.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i16.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei16_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_f32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i16.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i16.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei16_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_f32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i16.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i16.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei16_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_f32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i16.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i16.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei16_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_f32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i16.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i16.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i16> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei16_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint16m4_t bindex, size_t vl) {
+  return vluxei16_v_f32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i32.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i32.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei32_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_f32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i32.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i32.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei32_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_f32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i32.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i32.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei32_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_f32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i32.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i32.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei32_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_f32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i32.i32(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x float> @llvm.riscv.vluxei.mask.nxv16f32.nxv16i32.i64(<vscale x 16 x float> [[MASKEDOFF:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i32> [[BINDEX:%.*]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP1]]
+//
+vfloat32m8_t testuxei32_v_f32m8_m(vbool4_t mask, vfloat32m8_t maskedoff, const float *base, vuint32m8_t bindex, size_t vl) {
+  return vluxei32_v_f32m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i64.i32(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x float> @llvm.riscv.vluxei.mask.nxv1f32.nxv1i64.i64(<vscale x 1 x float> [[MASKEDOFF:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP1]]
+//
+vfloat32mf2_t testuxei64_v_f32mf2_m(vbool64_t mask, vfloat32mf2_t maskedoff, const float *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_f32mf2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i64.i32(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x float> @llvm.riscv.vluxei.mask.nxv2f32.nxv2i64.i64(<vscale x 2 x float> [[MASKEDOFF:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP1]]
+//
+vfloat32m1_t testuxei64_v_f32m1_m(vbool32_t mask, vfloat32m1_t maskedoff, const float *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_f32m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i64.i32(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.riscv.vluxei.mask.nxv4f32.nxv4i64.i64(<vscale x 4 x float> [[MASKEDOFF:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+vfloat32m2_t testuxei64_v_f32m2_m(vbool16_t mask, vfloat32m2_t maskedoff, const float *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_f32m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i64.i32(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x float> @llvm.riscv.vluxei.mask.nxv8f32.nxv8i64.i64(<vscale x 8 x float> [[MASKEDOFF:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP1]]
+//
+vfloat32m4_t testuxei64_v_f32m4_m(vbool8_t mask, vfloat32m4_t maskedoff, const float *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_f32m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i8.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i8.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i8> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei8_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint8mf8_t bindex, size_t vl) {
+  return vluxei8_v_f64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i8.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i8.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i8> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei8_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint8mf4_t bindex, size_t vl) {
+  return vluxei8_v_f64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i8.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i8.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i8> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei8_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint8mf2_t bindex, size_t vl) {
+  return vluxei8_v_f64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei8_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i8.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei8_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i8.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i8> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei8_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint8m1_t bindex, size_t vl) {
+  return vluxei8_v_f64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i16.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i16.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i16> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei16_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint16mf4_t bindex, size_t vl) {
+  return vluxei16_v_f64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i16.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i16.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i16> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei16_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint16mf2_t bindex, size_t vl) {
+  return vluxei16_v_f64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i16.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i16.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i16> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei16_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint16m1_t bindex, size_t vl) {
+  return vluxei16_v_f64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei16_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i16.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei16_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i16.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i16> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei16_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint16m2_t bindex, size_t vl) {
+  return vluxei16_v_f64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i32.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i32.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i32> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei32_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint32mf2_t bindex, size_t vl) {
+  return vluxei32_v_f64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i32.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i32.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i32> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei32_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint32m1_t bindex, size_t vl) {
+  return vluxei32_v_f64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i32.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i32.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i32> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei32_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint32m2_t bindex, size_t vl) {
+  return vluxei32_v_f64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei32_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i32.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei32_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i32.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i32> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei32_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint32m4_t bindex, size_t vl) {
+  return vluxei32_v_f64m8_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i64.i32(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 1 x double> @llvm.riscv.vluxei.mask.nxv1f64.nxv1i64.i64(<vscale x 1 x double> [[MASKEDOFF:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i64> [[BINDEX:%.*]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x double> [[TMP1]]
+//
+vfloat64m1_t testuxei64_v_f64m1_m(vbool64_t mask, vfloat64m1_t maskedoff, const double *base, vuint64m1_t bindex, size_t vl) {
+  return vluxei64_v_f64m1_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i64.i32(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.riscv.vluxei.mask.nxv2f64.nxv2i64.i64(<vscale x 2 x double> [[MASKEDOFF:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i64> [[BINDEX:%.*]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+vfloat64m2_t testuxei64_v_f64m2_m(vbool32_t mask, vfloat64m2_t maskedoff, const double *base, vuint64m2_t bindex, size_t vl) {
+  return vluxei64_v_f64m2_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i64.i32(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x double> @llvm.riscv.vluxei.mask.nxv4f64.nxv4i64.i64(<vscale x 4 x double> [[MASKEDOFF:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i64> [[BINDEX:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x double> [[TMP1]]
+//
+vfloat64m4_t testuxei64_v_f64m4_m(vbool16_t mask, vfloat64m4_t maskedoff, const double *base, vuint64m4_t bindex, size_t vl) {
+  return vluxei64_v_f64m4_m(mask, maskedoff, base, bindex, vl);
+}
+
+// CHECK-RV32-LABEL: @testuxei64_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i64.i32(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+// CHECK-RV64-LABEL: @testuxei64_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x double> @llvm.riscv.vluxei.mask.nxv8f64.nxv8i64.i64(<vscale x 8 x double> [[MASKEDOFF:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i64> [[BINDEX:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x double> [[TMP1]]
+//
+vfloat64m8_t testuxei64_v_f64m8_m(vbool8_t mask, vfloat64m8_t maskedoff, const double *base, vuint64m8_t bindex, size_t vl) {
+  return vluxei64_v_f64m8_m(mask, maskedoff, base, bindex, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics/vse.c b/clang/test/CodeGen/RISCV/rvv-intrinsics/vse.c
new file mode 100644
index 0000000000000000000000000000000000000000..0f5d3170298bfc4718de13ee8bb48be5f051825e
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics/vse.c
@@ -0,0 +1,1707 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +f -target-feature +d -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i8.i32(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i8.i64(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf8(int8_t *base, vint8mf8_t value, size_t vl) {
+  return vse8_v_i8mf8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i8.i32(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i8.i64(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf4(int8_t *base, vint8mf4_t value, size_t vl) {
+  return vse8_v_i8mf4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i8.i32(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i8.i64(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf2(int8_t *base, vint8mf2_t value, size_t vl) {
+  return vse8_v_i8mf2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i8.i32(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i8.i64(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m1(int8_t *base, vint8m1_t value, size_t vl) {
+  return vse8_v_i8m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i8.i32(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i8.i64(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m2(int8_t *base, vint8m2_t value, size_t vl) {
+  return vse8_v_i8m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv32i8.i32(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv32i8.i64(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m4(int8_t *base, vint8m4_t value, size_t vl) {
+  return vse8_v_i8m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv64i8.i32(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv64i8.i64(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m8(int8_t *base, vint8m8_t value, size_t vl) {
+  return vse8_v_i8m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i16.i32(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i16.i64(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16mf4(int16_t *base, vint16mf4_t value, size_t vl) {
+  return vse16_v_i16mf4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i16.i32(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i16.i64(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16mf2(int16_t *base, vint16mf2_t value, size_t vl) {
+  return vse16_v_i16mf2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i16.i32(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i16.i64(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m1(int16_t *base, vint16m1_t value, size_t vl) {
+  return vse16_v_i16m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i16.i32(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i16.i64(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m2(int16_t *base, vint16m2_t value, size_t vl) {
+  return vse16_v_i16m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i16.i32(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i16.i64(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m4(int16_t *base, vint16m4_t value, size_t vl) {
+  return vse16_v_i16m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv32i16.i32(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv32i16.i64(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m8(int16_t *base, vint16m8_t value, size_t vl) {
+  return vse16_v_i16m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i32.i32(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i32.i64(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32mf2(int32_t *base, vint32mf2_t value, size_t vl) {
+  return vse32_v_i32mf2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i32.i32(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i32.i64(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m1(int32_t *base, vint32m1_t value, size_t vl) {
+  return vse32_v_i32m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i32.i32(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i32.i64(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m2(int32_t *base, vint32m2_t value, size_t vl) {
+  return vse32_v_i32m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i32.i32(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i32.i64(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m4(int32_t *base, vint32m4_t value, size_t vl) {
+  return vse32_v_i32m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i32.i32(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i32.i64(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m8(int32_t *base, vint32m8_t value, size_t vl) {
+  return vse32_v_i32m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i64.i32(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i64.i64(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m1(int64_t *base, vint64m1_t value, size_t vl) {
+  return vse64_v_i64m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i64.i32(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i64.i64(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m2(int64_t *base, vint64m2_t value, size_t vl) {
+  return vse64_v_i64m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i64.i32(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i64.i64(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m4(int64_t *base, vint64m4_t value, size_t vl) {
+  return vse64_v_i64m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i64.i32(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i64.i64(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m8(int64_t *base, vint64m8_t value, size_t vl) {
+  return vse64_v_i64m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i8.i32(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i8.i64(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf8(uint8_t *base, vuint8mf8_t value, size_t vl) {
+  return vse8_v_u8mf8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i8.i32(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i8.i64(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf4(uint8_t *base, vuint8mf4_t value, size_t vl) {
+  return vse8_v_u8mf4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i8.i32(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i8.i64(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf2(uint8_t *base, vuint8mf2_t value, size_t vl) {
+  return vse8_v_u8mf2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i8.i32(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i8.i64(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m1(uint8_t *base, vuint8m1_t value, size_t vl) {
+  return vse8_v_u8m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i8.i32(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i8.i64(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m2(uint8_t *base, vuint8m2_t value, size_t vl) {
+  return vse8_v_u8m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv32i8.i32(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv32i8.i64(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m4(uint8_t *base, vuint8m4_t value, size_t vl) {
+  return vse8_v_u8m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv64i8.i32(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv64i8.i64(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m8(uint8_t *base, vuint8m8_t value, size_t vl) {
+  return vse8_v_u8m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i16.i32(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i16.i64(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16mf4(uint16_t *base, vuint16mf4_t value, size_t vl) {
+  return vse16_v_u16mf4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i16.i32(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i16.i64(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16mf2(uint16_t *base, vuint16mf2_t value, size_t vl) {
+  return vse16_v_u16mf2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i16.i32(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i16.i64(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m1(uint16_t *base, vuint16m1_t value, size_t vl) {
+  return vse16_v_u16m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i16.i32(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i16.i64(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m2(uint16_t *base, vuint16m2_t value, size_t vl) {
+  return vse16_v_u16m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i16.i32(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i16.i64(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m4(uint16_t *base, vuint16m4_t value, size_t vl) {
+  return vse16_v_u16m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv32i16.i32(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv32i16.i64(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m8(uint16_t *base, vuint16m8_t value, size_t vl) {
+  return vse16_v_u16m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i32.i32(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i32.i64(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32mf2(uint32_t *base, vuint32mf2_t value, size_t vl) {
+  return vse32_v_u32mf2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i32.i32(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i32.i64(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m1(uint32_t *base, vuint32m1_t value, size_t vl) {
+  return vse32_v_u32m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i32.i32(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i32.i64(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m2(uint32_t *base, vuint32m2_t value, size_t vl) {
+  return vse32_v_u32m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i32.i32(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i32.i64(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m4(uint32_t *base, vuint32m4_t value, size_t vl) {
+  return vse32_v_u32m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16i32.i32(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16i32.i64(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m8(uint32_t *base, vuint32m8_t value, size_t vl) {
+  return vse32_v_u32m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1i64.i32(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1i64.i64(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m1(uint64_t *base, vuint64m1_t value, size_t vl) {
+  return vse64_v_u64m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2i64.i32(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2i64.i64(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m2(uint64_t *base, vuint64m2_t value, size_t vl) {
+  return vse64_v_u64m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4i64.i32(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4i64.i64(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m4(uint64_t *base, vuint64m4_t value, size_t vl) {
+  return vse64_v_u64m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8i64.i32(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8i64.i64(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m8(uint64_t *base, vuint64m8_t value, size_t vl) {
+  return vse64_v_u64m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1f32.i32(<vscale x 1 x float> [[VALUE:%.*]], <vscale x 1 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1f32.i64(<vscale x 1 x float> [[VALUE:%.*]], <vscale x 1 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32mf2(float *base, vfloat32mf2_t value, size_t vl) {
+  return vse32_v_f32mf2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2f32.i32(<vscale x 2 x float> [[VALUE:%.*]], <vscale x 2 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2f32.i64(<vscale x 2 x float> [[VALUE:%.*]], <vscale x 2 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m1(float *base, vfloat32m1_t value, size_t vl) {
+  return vse32_v_f32m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4f32.i32(<vscale x 4 x float> [[VALUE:%.*]], <vscale x 4 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4f32.i64(<vscale x 4 x float> [[VALUE:%.*]], <vscale x 4 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m2(float *base, vfloat32m2_t value, size_t vl) {
+  return vse32_v_f32m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8f32.i32(<vscale x 8 x float> [[VALUE:%.*]], <vscale x 8 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8f32.i64(<vscale x 8 x float> [[VALUE:%.*]], <vscale x 8 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m4(float *base, vfloat32m4_t value, size_t vl) {
+  return vse32_v_f32m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv16f32.i32(<vscale x 16 x float> [[VALUE:%.*]], <vscale x 16 x float>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv16f32.i64(<vscale x 16 x float> [[VALUE:%.*]], <vscale x 16 x float>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m8(float *base, vfloat32m8_t value, size_t vl) {
+  return vse32_v_f32m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv1f64.i32(<vscale x 1 x double> [[VALUE:%.*]], <vscale x 1 x double>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv1f64.i64(<vscale x 1 x double> [[VALUE:%.*]], <vscale x 1 x double>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m1(double *base, vfloat64m1_t value, size_t vl) {
+  return vse64_v_f64m1(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv2f64.i32(<vscale x 2 x double> [[VALUE:%.*]], <vscale x 2 x double>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv2f64.i64(<vscale x 2 x double> [[VALUE:%.*]], <vscale x 2 x double>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m2(double *base, vfloat64m2_t value, size_t vl) {
+  return vse64_v_f64m2(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv4f64.i32(<vscale x 4 x double> [[VALUE:%.*]], <vscale x 4 x double>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv4f64.i64(<vscale x 4 x double> [[VALUE:%.*]], <vscale x 4 x double>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m4(double *base, vfloat64m4_t value, size_t vl) {
+  return vse64_v_f64m4(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.nxv8f64.i32(<vscale x 8 x double> [[VALUE:%.*]], <vscale x 8 x double>* [[TMP0]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.nxv8f64.i64(<vscale x 8 x double> [[VALUE:%.*]], <vscale x 8 x double>* [[TMP0]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m8(double *base, vfloat64m8_t value, size_t vl) {
+  return vse64_v_f64m8(base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i8.i32(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i8.i64(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf8_m(vbool64_t mask, int8_t *base, vint8mf8_t value, size_t vl) {
+  return vse8_v_i8mf8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i8.i32(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i8.i64(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf4_m(vbool32_t mask, int8_t *base, vint8mf4_t value, size_t vl) {
+  return vse8_v_i8mf4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i8.i32(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i8.i64(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8mf2_m(vbool16_t mask, int8_t *base, vint8mf2_t value, size_t vl) {
+  return vse8_v_i8mf2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i8.i32(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i8.i64(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m1_m(vbool8_t mask, int8_t *base, vint8m1_t value, size_t vl) {
+  return vse8_v_i8m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i8.i32(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i8.i64(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m2_m(vbool4_t mask, int8_t *base, vint8m2_t value, size_t vl) {
+  return vse8_v_i8m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv32i8.i32(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv32i8.i64(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m4_m(vbool2_t mask, int8_t *base, vint8m4_t value, size_t vl) {
+  return vse8_v_i8m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_i8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv64i8.i32(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_i8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv64i8.i64(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_i8m8_m(vbool1_t mask, int8_t *base, vint8m8_t value, size_t vl) {
+  return vse8_v_i8m8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i16.i32(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i16.i64(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16mf4_m(vbool64_t mask, int16_t *base, vint16mf4_t value, size_t vl) {
+  return vse16_v_i16mf4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i16.i32(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i16.i64(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16mf2_m(vbool32_t mask, int16_t *base, vint16mf2_t value, size_t vl) {
+  return vse16_v_i16mf2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i16.i32(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i16.i64(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m1_m(vbool16_t mask, int16_t *base, vint16m1_t value, size_t vl) {
+  return vse16_v_i16m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i16.i32(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i16.i64(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m2_m(vbool8_t mask, int16_t *base, vint16m2_t value, size_t vl) {
+  return vse16_v_i16m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i16.i32(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i16.i64(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m4_m(vbool4_t mask, int16_t *base, vint16m4_t value, size_t vl) {
+  return vse16_v_i16m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_i16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv32i16.i32(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_i16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv32i16.i64(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_i16m8_m(vbool2_t mask, int16_t *base, vint16m8_t value, size_t vl) {
+  return vse16_v_i16m8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i32.i32(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i32.i64(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32mf2_m(vbool64_t mask, int32_t *base, vint32mf2_t value, size_t vl) {
+  return vse32_v_i32mf2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i32.i32(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i32.i64(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m1_m(vbool32_t mask, int32_t *base, vint32m1_t value, size_t vl) {
+  return vse32_v_i32m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i32.i32(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i32.i64(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m2_m(vbool16_t mask, int32_t *base, vint32m2_t value, size_t vl) {
+  return vse32_v_i32m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i32.i32(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i32.i64(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m4_m(vbool8_t mask, int32_t *base, vint32m4_t value, size_t vl) {
+  return vse32_v_i32m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_i32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i32.i32(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_i32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i32.i64(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_i32m8_m(vbool4_t mask, int32_t *base, vint32m8_t value, size_t vl) {
+  return vse32_v_i32m8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i64.i32(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i64.i64(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m1_m(vbool64_t mask, int64_t *base, vint64m1_t value, size_t vl) {
+  return vse64_v_i64m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i64.i32(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i64.i64(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m2_m(vbool32_t mask, int64_t *base, vint64m2_t value, size_t vl) {
+  return vse64_v_i64m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i64.i32(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i64.i64(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m4_m(vbool16_t mask, int64_t *base, vint64m4_t value, size_t vl) {
+  return vse64_v_i64m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_i64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i64.i32(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_i64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i64.i64(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_i64m8_m(vbool8_t mask, int64_t *base, vint64m8_t value, size_t vl) {
+  return vse64_v_i64m8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i8.i32(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 1 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i8.i64(<vscale x 1 x i8> [[VALUE:%.*]], <vscale x 1 x i8>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf8_m(vbool64_t mask, uint8_t *base, vuint8mf8_t value, size_t vl) {
+  return vse8_v_u8mf8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i8.i32(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 2 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i8.i64(<vscale x 2 x i8> [[VALUE:%.*]], <vscale x 2 x i8>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf4_m(vbool32_t mask, uint8_t *base, vuint8mf4_t value, size_t vl) {
+  return vse8_v_u8mf4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i8.i32(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 4 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i8.i64(<vscale x 4 x i8> [[VALUE:%.*]], <vscale x 4 x i8>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8mf2_m(vbool16_t mask, uint8_t *base, vuint8mf2_t value, size_t vl) {
+  return vse8_v_u8mf2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i8.i32(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 8 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i8.i64(<vscale x 8 x i8> [[VALUE:%.*]], <vscale x 8 x i8>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m1_m(vbool8_t mask, uint8_t *base, vuint8m1_t value, size_t vl) {
+  return vse8_v_u8m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i8.i32(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 16 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i8.i64(<vscale x 16 x i8> [[VALUE:%.*]], <vscale x 16 x i8>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m2_m(vbool4_t mask, uint8_t *base, vuint8m2_t value, size_t vl) {
+  return vse8_v_u8m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv32i8.i32(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 32 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv32i8.i64(<vscale x 32 x i8> [[VALUE:%.*]], <vscale x 32 x i8>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m4_m(vbool2_t mask, uint8_t *base, vuint8m4_t value, size_t vl) {
+  return vse8_v_u8m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse8_v_u8m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv64i8.i32(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse8_v_u8m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[BASE:%.*]] to <vscale x 64 x i8>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv64i8.i64(<vscale x 64 x i8> [[VALUE:%.*]], <vscale x 64 x i8>* [[TMP0]], <vscale x 64 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse8_v_u8m8_m(vbool1_t mask, uint8_t *base, vuint8m8_t value, size_t vl) {
+  return vse8_v_u8m8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16mf4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i16.i32(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16mf4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 1 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i16.i64(<vscale x 1 x i16> [[VALUE:%.*]], <vscale x 1 x i16>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16mf4_m(vbool64_t mask, uint16_t *base, vuint16mf4_t value, size_t vl) {
+  return vse16_v_u16mf4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i16.i32(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 2 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i16.i64(<vscale x 2 x i16> [[VALUE:%.*]], <vscale x 2 x i16>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16mf2_m(vbool32_t mask, uint16_t *base, vuint16mf2_t value, size_t vl) {
+  return vse16_v_u16mf2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i16.i32(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 4 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i16.i64(<vscale x 4 x i16> [[VALUE:%.*]], <vscale x 4 x i16>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m1_m(vbool16_t mask, uint16_t *base, vuint16m1_t value, size_t vl) {
+  return vse16_v_u16m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i16.i32(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 8 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i16.i64(<vscale x 8 x i16> [[VALUE:%.*]], <vscale x 8 x i16>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m2_m(vbool8_t mask, uint16_t *base, vuint16m2_t value, size_t vl) {
+  return vse16_v_u16m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i16.i32(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 16 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i16.i64(<vscale x 16 x i16> [[VALUE:%.*]], <vscale x 16 x i16>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m4_m(vbool4_t mask, uint16_t *base, vuint16m4_t value, size_t vl) {
+  return vse16_v_u16m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse16_v_u16m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv32i16.i32(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse16_v_u16m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i16* [[BASE:%.*]] to <vscale x 32 x i16>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv32i16.i64(<vscale x 32 x i16> [[VALUE:%.*]], <vscale x 32 x i16>* [[TMP0]], <vscale x 32 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse16_v_u16m8_m(vbool2_t mask, uint16_t *base, vuint16m8_t value, size_t vl) {
+  return vse16_v_u16m8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i32.i32(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 1 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i32.i64(<vscale x 1 x i32> [[VALUE:%.*]], <vscale x 1 x i32>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32mf2_m(vbool64_t mask, uint32_t *base, vuint32mf2_t value, size_t vl) {
+  return vse32_v_u32mf2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i32.i32(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 2 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i32.i64(<vscale x 2 x i32> [[VALUE:%.*]], <vscale x 2 x i32>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m1_m(vbool32_t mask, uint32_t *base, vuint32m1_t value, size_t vl) {
+  return vse32_v_u32m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i32.i32(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 4 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i32.i64(<vscale x 4 x i32> [[VALUE:%.*]], <vscale x 4 x i32>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m2_m(vbool16_t mask, uint32_t *base, vuint32m2_t value, size_t vl) {
+  return vse32_v_u32m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i32.i32(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 8 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i32.i64(<vscale x 8 x i32> [[VALUE:%.*]], <vscale x 8 x i32>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m4_m(vbool8_t mask, uint32_t *base, vuint32m4_t value, size_t vl) {
+  return vse32_v_u32m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_u32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16i32.i32(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_u32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i32* [[BASE:%.*]] to <vscale x 16 x i32>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16i32.i64(<vscale x 16 x i32> [[VALUE:%.*]], <vscale x 16 x i32>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_u32m8_m(vbool4_t mask, uint32_t *base, vuint32m8_t value, size_t vl) {
+  return vse32_v_u32m8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1i64.i32(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 1 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1i64.i64(<vscale x 1 x i64> [[VALUE:%.*]], <vscale x 1 x i64>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m1_m(vbool64_t mask, uint64_t *base, vuint64m1_t value, size_t vl) {
+  return vse64_v_u64m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2i64.i32(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 2 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2i64.i64(<vscale x 2 x i64> [[VALUE:%.*]], <vscale x 2 x i64>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m2_m(vbool32_t mask, uint64_t *base, vuint64m2_t value, size_t vl) {
+  return vse64_v_u64m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4i64.i32(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 4 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4i64.i64(<vscale x 4 x i64> [[VALUE:%.*]], <vscale x 4 x i64>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m4_m(vbool16_t mask, uint64_t *base, vuint64m4_t value, size_t vl) {
+  return vse64_v_u64m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_u64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8i64.i32(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_u64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast i64* [[BASE:%.*]] to <vscale x 8 x i64>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8i64.i64(<vscale x 8 x i64> [[VALUE:%.*]], <vscale x 8 x i64>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_u64m8_m(vbool8_t mask, uint64_t *base, vuint64m8_t value, size_t vl) {
+  return vse64_v_u64m8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32mf2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1f32.i32(<vscale x 1 x float> [[VALUE:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32mf2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 1 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1f32.i64(<vscale x 1 x float> [[VALUE:%.*]], <vscale x 1 x float>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32mf2_m(vbool64_t mask, float *base, vfloat32mf2_t value, size_t vl) {
+  return vse32_v_f32mf2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2f32.i32(<vscale x 2 x float> [[VALUE:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 2 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2f32.i64(<vscale x 2 x float> [[VALUE:%.*]], <vscale x 2 x float>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m1_m(vbool32_t mask, float *base, vfloat32m1_t value, size_t vl) {
+  return vse32_v_f32m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4f32.i32(<vscale x 4 x float> [[VALUE:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 4 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4f32.i64(<vscale x 4 x float> [[VALUE:%.*]], <vscale x 4 x float>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m2_m(vbool16_t mask, float *base, vfloat32m2_t value, size_t vl) {
+  return vse32_v_f32m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8f32.i32(<vscale x 8 x float> [[VALUE:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 8 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8f32.i64(<vscale x 8 x float> [[VALUE:%.*]], <vscale x 8 x float>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m4_m(vbool8_t mask, float *base, vfloat32m4_t value, size_t vl) {
+  return vse32_v_f32m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse32_v_f32m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv16f32.i32(<vscale x 16 x float> [[VALUE:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse32_v_f32m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast float* [[BASE:%.*]] to <vscale x 16 x float>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv16f32.i64(<vscale x 16 x float> [[VALUE:%.*]], <vscale x 16 x float>* [[TMP0]], <vscale x 16 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse32_v_f32m8_m(vbool4_t mask, float *base, vfloat32m8_t value, size_t vl) {
+  return vse32_v_f32m8_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m1_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv1f64.i32(<vscale x 1 x double> [[VALUE:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m1_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 1 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv1f64.i64(<vscale x 1 x double> [[VALUE:%.*]], <vscale x 1 x double>* [[TMP0]], <vscale x 1 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m1_m(vbool64_t mask, double *base, vfloat64m1_t value, size_t vl) {
+  return vse64_v_f64m1_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m2_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv2f64.i32(<vscale x 2 x double> [[VALUE:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m2_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 2 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv2f64.i64(<vscale x 2 x double> [[VALUE:%.*]], <vscale x 2 x double>* [[TMP0]], <vscale x 2 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m2_m(vbool32_t mask, double *base, vfloat64m2_t value, size_t vl) {
+  return vse64_v_f64m2_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m4_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv4f64.i32(<vscale x 4 x double> [[VALUE:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m4_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 4 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv4f64.i64(<vscale x 4 x double> [[VALUE:%.*]], <vscale x 4 x double>* [[TMP0]], <vscale x 4 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m4_m(vbool16_t mask, double *base, vfloat64m4_t value, size_t vl) {
+  return vse64_v_f64m4_m(mask, base, value, vl);
+}
+
+// CHECK-RV32-LABEL: @test_vse64_v_f64m8_m(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV32-NEXT:    call void @llvm.riscv.vse.mask.nxv8f64.i32(<vscale x 8 x double> [[VALUE:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_vse64_v_f64m8_m(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = bitcast double* [[BASE:%.*]] to <vscale x 8 x double>*
+// CHECK-RV64-NEXT:    call void @llvm.riscv.vse.mask.nxv8f64.i64(<vscale x 8 x double> [[VALUE:%.*]], <vscale x 8 x double>* [[TMP0]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_vse64_v_f64m8_m(vbool8_t mask, double *base, vfloat64m8_t value, size_t vl) {
+  return vse64_v_f64m8_m(mask, base, value, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics/vsetvl.c b/clang/test/CodeGen/RISCV/rvv-intrinsics/vsetvl.c
new file mode 100644
index 0000000000000000000000000000000000000000..5d68209d88a7f9050d527a34958109ce8092de01
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics/vsetvl.c
@@ -0,0 +1,451 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +experimental-v -emit-llvm -o - %s \
+// RUN:       | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +experimental-v -emit-llvm -o - %s \
+// RUN:       | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @test_vsetvl_e8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 0, i32 0)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 0, i64 0)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e8m1(size_t avl) {
+  return vsetvl_e8m1(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 0, i32 1)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 0, i64 1)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e8m2(size_t avl) {
+  return vsetvl_e8m2(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 0, i32 2)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 0, i64 2)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e8m4(size_t avl) {
+  return vsetvl_e8m4(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 0, i32 3)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 0, i64 3)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e8m8(size_t avl) {
+  return vsetvl_e8m8(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 0, i32 7)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 0, i64 7)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e8mf2(size_t avl) {
+  return vsetvl_e8mf2(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 0, i32 6)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 0, i64 6)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e8mf4(size_t avl) {
+  return vsetvl_e8mf4(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 0, i32 5)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 0, i64 5)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e8mf8(size_t avl) {
+  return vsetvl_e8mf8(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 1, i32 0)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 1, i64 0)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e16m1(size_t avl) {
+  return vsetvl_e16m1(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 1, i32 1)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 1, i64 1)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e16m2(size_t avl) {
+  return vsetvl_e16m2(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 1, i32 2)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 1, i64 2)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e16m4(size_t avl) {
+  return vsetvl_e16m4(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 1, i32 3)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 1, i64 3)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e16m8(size_t avl) {
+  return vsetvl_e16m8(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 1, i32 7)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 1, i64 7)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e16mf2(size_t avl) {
+  return vsetvl_e16mf2(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 1, i32 6)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 1, i64 6)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e16mf4(size_t avl) {
+  return vsetvl_e16mf4(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 2, i32 0)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 2, i64 0)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e32m1(size_t avl) {
+  return vsetvl_e32m1(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 2, i32 1)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 2, i64 1)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e32m2(size_t avl) {
+  return vsetvl_e32m2(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 2, i32 2)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 2, i64 2)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e32m4(size_t avl) {
+  return vsetvl_e32m4(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 2, i32 3)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 2, i64 3)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e32m8(size_t avl) {
+  return vsetvl_e32m8(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 2, i32 7)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 2, i64 7)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e32mf2(size_t avl) {
+  return vsetvl_e32mf2(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 3, i32 0)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 3, i64 0)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e64m1(size_t avl) {
+  return vsetvl_e64m1(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 3, i32 1)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 3, i64 1)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e64m2(size_t avl) {
+  return vsetvl_e64m2(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 3, i32 2)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 3, i64 2)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e64m4(size_t avl) {
+  return vsetvl_e64m4(avl);
+}
+
+// CHECK-RV32-LABEL: @test_vsetvl_e64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[AVL_ADDR:%.*]] = alloca i32, align 4
+// CHECK-RV32-NEXT:    store i32 [[AVL:%.*]], i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = load i32, i32* [[AVL_ADDR]], align 4
+// CHECK-RV32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 [[TMP0]], i32 3, i32 3)
+// CHECK-RV32-NEXT:    ret i32 [[TMP1]]
+//
+// CHECK-RV64-LABEL: @test_vsetvl_e64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[AVL_ADDR:%.*]] = alloca i64, align 8
+// CHECK-RV64-NEXT:    store i64 [[AVL:%.*]], i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = load i64, i64* [[AVL_ADDR]], align 8
+// CHECK-RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[TMP0]], i64 3, i64 3)
+// CHECK-RV64-NEXT:    ret i64 [[TMP1]]
+//
+size_t test_vsetvl_e64m8(size_t avl) {
+  return vsetvl_e64m8(avl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics/vsetvlmax.c b/clang/test/CodeGen/RISCV/rvv-intrinsics/vsetvlmax.c
new file mode 100644
index 0000000000000000000000000000000000000000..447ca0595f0bbd35c041f21e68a0c8cb9e415b13
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics/vsetvlmax.c
@@ -0,0 +1,319 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +experimental-v -emit-llvm -o - %s \
+// RUN:       | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +experimental-v -emit-llvm -o - %s \
+// RUN:       | FileCheck --check-prefix=CHECK-RV64 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +m -target-feature +experimental-v \
+// RUN:   -Werror -Wall -o - %s -S >/dev/null 2>&1 | FileCheck --check-prefix=ASM --allow-empty %s
+
+// ASM-NOT: warning
+#include <riscv_vector.h>
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 0, i32 0)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 0)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e8m1() {
+  return vsetvlmax_e8m1();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 0, i32 1)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 1)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e8m2() {
+  return vsetvlmax_e8m2();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 0, i32 2)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 2)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e8m4() {
+  return vsetvlmax_e8m4();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 0, i32 3)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 3)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e8m8() {
+  return vsetvlmax_e8m8();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 0, i32 7)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 7)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e8mf2() {
+  return vsetvlmax_e8mf2();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 0, i32 6)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 6)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e8mf4() {
+  return vsetvlmax_e8mf4();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 0, i32 5)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 5)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e8mf8() {
+  return vsetvlmax_e8mf8();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 0)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 0)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e16m1() {
+  return vsetvlmax_e16m1();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 1)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e16m2() {
+  return vsetvlmax_e16m2();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 2)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 2)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e16m4() {
+  return vsetvlmax_e16m4();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 3)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 3)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e16m8() {
+  return vsetvlmax_e16m8();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 7)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 7)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e16mf2() {
+  return vsetvlmax_e16mf2();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 6)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 6)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e16mf4() {
+  return vsetvlmax_e16mf4();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 2, i32 0)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 0)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e32m1() {
+  return vsetvlmax_e32m1();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 2, i32 1)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 1)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e32m2() {
+  return vsetvlmax_e32m2();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 2, i32 2)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 2)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e32m4() {
+  return vsetvlmax_e32m4();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 2, i32 3)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 3)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e32m8() {
+  return vsetvlmax_e32m8();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 2, i32 7)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 7)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e32mf2() {
+  return vsetvlmax_e32mf2();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 3, i32 0)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 0)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e64m1() {
+  return vsetvlmax_e64m1();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 3, i32 1)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 1)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e64m2() {
+  return vsetvlmax_e64m2();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 3, i32 2)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 2)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e64m4() {
+  return vsetvlmax_e64m4();
+}
+
+// CHECK-RV32-LABEL: @test_vsetvlmax_e64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 3, i32 3)
+// CHECK-RV32-NEXT:    ret i32 [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_vsetvlmax_e64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 3)
+// CHECK-RV64-NEXT:    ret i64 [[TMP0]]
+//
+size_t test_vsetvlmax_e64m8() {
+  return vsetvlmax_e64m8();
+}
diff --git a/clang/test/CodeGen/SystemZ/zos-alignment.c b/clang/test/CodeGen/SystemZ/zos-alignment.c
index 4b572fcac5a97ee48bd672450c3df03964683e6e..703fd1a46c3b9618e03f5d729a722c388e202f74 100644
--- a/clang/test/CodeGen/SystemZ/zos-alignment.c
+++ b/clang/test/CodeGen/SystemZ/zos-alignment.c
@@ -83,6 +83,35 @@ struct s6 {
 // CHECK-NEXT:         8 |   char *[] b
 // CHECK-NEXT:           | [sizeof=8, align=8]
 
+struct s7 {
+  long  :0;
+  short a;
+} S7;
+// CHECK:              0 | struct s7
+// CHECK-NEXT:       0:- |   long
+// CHECK-NEXT:         0 |   short a
+// CHECK-NEXT:           | [sizeof=2, align=2]
+
+#pragma pack(2)
+struct s8 {
+  unsigned long       :0;
+  long long           a;
+} S8;
+#pragma pack()
+// CHECK:              0 | struct s8
+// CHECK-NEXT:       0:- |   unsigned long
+// CHECK-NEXT:         0 |   long long a
+// CHECK-NEXT:           | [sizeof=8, align=2]
+
+struct s9 {
+  unsigned int   :0;
+  unsigned short :0;
+} S9;
+// CHECK:              0 | struct s9
+// CHECK-NEXT:       0:- |   unsigned int
+// CHECK-NEXT:       0:- |   unsigned short
+// CHECK-NEXT:           | [sizeof=0, align=1]
+
 struct s10 {
  unsigned int __attribute__((aligned)) a;
 } S10;
diff --git a/clang/test/CodeGen/X86/amx_api.c b/clang/test/CodeGen/X86/amx_api.c
index 824a3aec20ecf85aa23d352bdb9c327d0fac1ac1..3bfe887c0445b29454b126d964334c297af97d4b 100644
--- a/clang/test/CodeGen/X86/amx_api.c
+++ b/clang/test/CodeGen/X86/amx_api.c
@@ -81,9 +81,9 @@ void test_tile_zero(__tile1024i c) {
   __tile_zero(&c);
 }
 
-void test_tile_tdpbf16ps(__tile1024i a, __tile1024i b, __tile1024i c) {
-  //CHECK-LABEL: @test_tile_tdpbf16ps
+void test_tile_dpbf16ps(__tile1024i a, __tile1024i b, __tile1024i c) {
+  //CHECK-LABEL: @test_tile_dpbf16ps
   //CHECK: call x86_amx @llvm.x86.tdpbf16ps.internal
   //CHECK-NEXT: {{%.*}} = bitcast x86_amx {{%.*}} to <256 x i32>
-  __tile_tdpbf16ps(&a, b, c);
+  __tile_dpbf16ps(&a, b, c);
 }
diff --git a/clang/test/CodeGen/aarch64-neon-intrinsics.c b/clang/test/CodeGen/aarch64-neon-intrinsics.c
index a56080bace0fdce75dea3b7e7af536cbcef6003e..76f5cfd3aaa8a8f59e276960c4e9229825fda31a 100644
--- a/clang/test/CodeGen/aarch64-neon-intrinsics.c
+++ b/clang/test/CodeGen/aarch64-neon-intrinsics.c
@@ -18155,7 +18155,7 @@ float64x1_t test_vcvt_n_f64_u64(uint64x1_t a) {
 
 // CHECK-LABEL: @test_vrndn_f64(
 // CHECK:   [[TMP0:%.*]] = bitcast <1 x double> %a to <8 x i8>
-// CHECK:   [[VRNDN1_I:%.*]] = call <1 x double> @llvm.aarch64.neon.frintn.v1f64(<1 x double> %a)
+// CHECK:   [[VRNDN1_I:%.*]] = call <1 x double> @llvm.roundeven.v1f64(<1 x double> %a)
 // CHECK:   ret <1 x double> [[VRNDN1_I]]
 float64x1_t test_vrndn_f64(float64x1_t a) {
   return vrndn_f64(a);
diff --git a/clang/test/CodeGen/aarch64-neon-misc.c b/clang/test/CodeGen/aarch64-neon-misc.c
index 4f85f67cdaecfff36e01b55b8bba05f4f33683f9..ed9af88b56c14d011bcf8b12118d3052ef4ac535 100644
--- a/clang/test/CodeGen/aarch64-neon-misc.c
+++ b/clang/test/CodeGen/aarch64-neon-misc.c
@@ -2287,7 +2287,7 @@ float64x2_t test_vcvt_high_f64_f32(float32x4_t a) {
 
 // CHECK-LABEL: @test_vrndnq_f64(
 // CHECK:   [[TMP0:%.*]] = bitcast <2 x double> %a to <16 x i8>
-// CHECK:   [[VRNDN1_I:%.*]] = call <2 x double> @llvm.aarch64.neon.frintn.v2f64(<2 x double> %a)
+// CHECK:   [[VRNDN1_I:%.*]] = call <2 x double> @llvm.roundeven.v2f64(<2 x double> %a)
 // CHECK:   ret <2 x double> [[VRNDN1_I]]
 float64x2_t test_vrndnq_f64(float64x2_t a) {
   return vrndnq_f64(a);
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c
index 6475b19ab65330ba93cf077467d3b633d17876d9..6e3b32e1cc19fd2660d70a2fcf78b502759ea8a7 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c
@@ -114,7 +114,7 @@ svint32_t test_svld1sh_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
 
 svint64_t test_svld1sh_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   // CHECK-LABEL: test_svld1sh_gather_u64base_s64
-  // CHECK: %[[PG.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> %pg)
+  // CHECK: %[[PG:.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> %pg)
   // CHECK: %[[LOAD:.*]] = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> %[[PG]], <vscale x 2 x i64> %bases, i64 0)
   // CHECK: %[[SEXT:.*]] = sext <vscale x 2 x i16> %[[LOAD]] to <vscale x 2 x i64>
   // CHECK: ret <vscale x 2 x i64> %[[SEXT]]
diff --git a/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics.c b/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics.c
index 32161146ef450d6f6539a02bed8d00b4a1c516dd..01df5b0d1930d04ce759ef13ff5d74f83f41ef26 100644
--- a/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics.c
+++ b/clang/test/CodeGen/aarch64-v8.2a-fp16-intrinsics.c
@@ -366,7 +366,7 @@ float16_t test_vrndmh_f16(float16_t a) {
 }
 
 // CHECK-LABEL: test_vrndnh_f16
-// CHECK:  [[RND:%.*]] =  call half @llvm.aarch64.neon.frintn.f16(half %a)
+// CHECK:  [[RND:%.*]] =  call half @llvm.roundeven.f16(half %a)
 // CHECK:  ret half [[RND]]
 float16_t test_vrndnh_f16(float16_t a) {
   return vrndnh_f16(a);
diff --git a/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics.c b/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics.c
index 5c4f9053a9aec71b47ec214e5652e2dfd236d48a..401aa4da8d5c343dd1971eb2c647f73438ab24be 100644
--- a/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics.c
+++ b/clang/test/CodeGen/aarch64-v8.2a-neon-intrinsics.c
@@ -348,14 +348,14 @@ float16x8_t test_vrndmq_f16(float16x8_t a) {
 }
 
 // CHECK-LABEL: test_vrndn_f16
-// CHECK:  [[RND:%.*]] =  call <4 x half> @llvm.aarch64.neon.frintn.v4f16(<4 x half> %a)
+// CHECK:  [[RND:%.*]] =  call <4 x half> @llvm.roundeven.v4f16(<4 x half> %a)
 // CHECK:  ret <4 x half> [[RND]]
 float16x4_t test_vrndn_f16(float16x4_t a) {
   return vrndn_f16(a);
 }
 
 // CHECK-LABEL: test_vrndnq_f16
-// CHECK:  [[RND:%.*]] =  call <8 x half> @llvm.aarch64.neon.frintn.v8f16(<8 x half> %a)
+// CHECK:  [[RND:%.*]] =  call <8 x half> @llvm.roundeven.v8f16(<8 x half> %a)
 // CHECK:  ret <8 x half> [[RND]]
 float16x8_t test_vrndnq_f16(float16x8_t a) {
   return vrndnq_f16(a);
diff --git a/clang/test/CodeGen/arm-neon-directed-rounding.c b/clang/test/CodeGen/arm-neon-directed-rounding.c
index f329c669ba56113bea19fd9871603b93c51de3ea..c493e3897ab6ab4a8540b1cc4e43a882655e2fbc 100644
--- a/clang/test/CodeGen/arm-neon-directed-rounding.c
+++ b/clang/test/CodeGen/arm-neon-directed-rounding.c
@@ -41,7 +41,7 @@ float32x4_t test_vrndmq_f32(float32x4_t a) {
 
 // CHECK-LABEL: define{{.*}} <2 x float> @test_vrndn_f32(<2 x float> %a)
 // CHECK-A32: [[VRNDN_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintn.v2f32(<2 x float> %a)
-// CHECK-A64: [[VRNDN_V1_I:%.*]] = call <2 x float> @llvm.aarch64.neon.frintn.v2f32(<2 x float> %a)
+// CHECK-A64: [[VRNDN_V1_I:%.*]] = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %a)
 // CHECK: ret <2 x float> [[VRNDN_V1_I]]
 float32x2_t test_vrndn_f32(float32x2_t a) {
   return vrndn_f32(a);
@@ -49,7 +49,7 @@ float32x2_t test_vrndn_f32(float32x2_t a) {
 
 // CHECK-LABEL: define{{.*}} <4 x float> @test_vrndnq_f32(<4 x float> %a)
 // CHECK-A32: [[VRNDNQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintn.v4f32(<4 x float> %a)
-// CHECK-A64: [[VRNDNQ_V1_I:%.*]] = call <4 x float> @llvm.aarch64.neon.frintn.v4f32(<4 x float> %a)
+// CHECK-A64: [[VRNDNQ_V1_I:%.*]] = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %a)
 // CHECK: ret <4 x float> [[VRNDNQ_V1_I]]
 float32x4_t test_vrndnq_f32(float32x4_t a) {
   return vrndnq_f32(a);
@@ -105,7 +105,7 @@ float32x4_t test_vrndq_f32(float32x4_t a) {
 
 // CHECK-LABEL: define{{.*}} float @test_vrndns_f32(float %a)
 // CHECK-A32: [[VRNDN_I:%.*]] = call float @llvm.arm.neon.vrintn.f32(float %a)
-// CHECK-A64: [[VRNDN_I:%.*]] = call float @llvm.aarch64.neon.frintn.f32(float %a)
+// CHECK-A64: [[VRNDN_I:%.*]] = call float @llvm.roundeven.f32(float %a)
 // CHECK: ret float [[VRNDN_I]]
 float32_t test_vrndns_f32(float32_t a) {
   return vrndns_f32(a);
diff --git a/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c b/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c
new file mode 100644
index 0000000000000000000000000000000000000000..84541f9cb12dbb1b0ae7e1e72e1a529a9f2146e1
--- /dev/null
+++ b/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -msve-vector-bits=128 -S -emit-llvm -o - %s | FileCheck %s -D#VBITS=128
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -msve-vector-bits=256 -S -emit-llvm -o - %s | FileCheck %s -D#VBITS=256
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -msve-vector-bits=512 -S -emit-llvm -o - %s | FileCheck %s -D#VBITS=512
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -msve-vector-bits=1024 -S -emit-llvm -o - %s | FileCheck %s -D#VBITS=1024
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -msve-vector-bits=2048 -S -emit-llvm -o - %s | FileCheck %s -D#VBITS=2048
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -msve-vector-bits=scalable -S -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-NONE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -S -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-NONE
+
+// CHECK-LABEL: @func() #0
+// CHECK: attributes #0 = { {{.*}} vscale_range([[#div(VBITS,128)]],[[#div(VBITS,128)]]) {{.*}} }
+// CHECK-NONE-NOT: vscale_range
+void func() {}
diff --git a/clang/test/CodeGen/arm64-vrnd.c b/clang/test/CodeGen/arm64-vrnd.c
index c710caedf181b9b0b2b304506f280e3701a2522e..24298f896d319c6be7e1d8e090cfa03c13f89889 100644
--- a/clang/test/CodeGen/arm64-vrnd.c
+++ b/clang/test/CodeGen/arm64-vrnd.c
@@ -6,7 +6,7 @@ float64x2_t rnd5(float64x2_t a) { return vrndq_f64(a); }
 // CHECK: call <2 x double> @llvm.trunc.v2f64(<2 x double>
 
 float64x2_t rnd9(float64x2_t a) { return vrndnq_f64(a); }
-// CHECK: call <2 x double> @llvm.aarch64.neon.frintn.v2f64(<2 x double>
+// CHECK: call <2 x double> @llvm.roundeven.v2f64(<2 x double>
 
 float64x2_t rnd13(float64x2_t a) { return vrndmq_f64(a); }
 // CHECK: call <2 x double> @llvm.floor.v2f64(<2 x double>
diff --git a/clang/test/CodeGen/atomic-ops.c b/clang/test/CodeGen/atomic-ops.c
index 4deb1322e0ff27c2ccbf215fc9e1013177c9e10d..269406fc3c50fe5a81c70c347585a68992f6d1ba 100644
--- a/clang/test/CodeGen/atomic-ops.c
+++ b/clang/test/CodeGen/atomic-ops.c
@@ -500,6 +500,7 @@ void generalFailureOrder(_Atomic(int) *ptr, int *ptr2, int success, int fail) {
 
   // CHECK: [[RELEASE]]
   // CHECK: switch {{.*}}, label %[[RELEASE_MONOTONIC:[0-9a-zA-Z._]+]] [
+  // CHECK-NEXT: i32 2, label %[[RELEASE_ACQUIRE:[0-9a-zA-Z._]+]]
   // CHECK-NEXT: ]
 
   // CHECK: [[ACQREL]]
@@ -527,6 +528,14 @@ void generalFailureOrder(_Atomic(int) *ptr, int *ptr2, int success, int fail) {
   // CHECK: cmpxchg {{.*}} acquire acquire, align
   // CHECK: br
 
+  // CHECK: [[RELEASE_MONOTONIC]]
+  // CHECK: cmpxchg {{.*}} release monotonic, align
+  // CHECK: br
+
+  // CHECK: [[RELEASE_ACQUIRE]]
+  // CHECK: cmpxchg {{.*}} release acquire, align
+  // CHECK: br
+
   // CHECK: [[ACQREL_MONOTONIC]]
   // CHECK: cmpxchg {{.*}} acq_rel monotonic, align
   // CHECK: br
@@ -562,6 +571,20 @@ void generalWeakness(int *ptr, int *ptr2, _Bool weak) {
   // CHECK-NOT: br
   // CHECK: cmpxchg weak {{.*}} seq_cst seq_cst, align
   // CHECK: br
+
+  __atomic_compare_exchange_n(ptr, ptr2, 42, weak, memory_order_release, memory_order_acquire);
+  // CHECK: switch i1 {{.*}}, label %[[WEAK:[0-9a-zA-Z._]+]] [
+  // CHECK-NEXT: i1 false, label %[[STRONG:[0-9a-zA-Z._]+]]
+
+  // CHECK: [[STRONG]]
+  // CHECK-NOT: br
+  // CHECK: cmpxchg {{.*}} release acquire
+  // CHECK: br
+
+  // CHECK: [[WEAK]]
+  // CHECK-NOT: br
+  // CHECK: cmpxchg weak {{.*}} release acquire
+  // CHECK: br
 }
 
 // Having checked the flow in the previous two cases, we'll trust clang to
@@ -576,7 +599,9 @@ void EMIT_ALL_THE_THINGS(int *ptr, int *ptr2, int new, _Bool weak, int success,
   // CHECK: = cmpxchg weak {{.*}} acquire monotonic, align
   // CHECK: = cmpxchg weak {{.*}} acquire acquire, align
   // CHECK: = cmpxchg {{.*}} release monotonic, align
+  // CHECK: = cmpxchg {{.*}} release acquire, align
   // CHECK: = cmpxchg weak {{.*}} release monotonic, align
+  // CHECK: = cmpxchg weak {{.*}} release acquire, align
   // CHECK: = cmpxchg {{.*}} acq_rel monotonic, align
   // CHECK: = cmpxchg {{.*}} acq_rel acquire, align
   // CHECK: = cmpxchg weak {{.*}} acq_rel monotonic, align
diff --git a/clang/test/CodeGen/attr-speculative-load-hardening.c b/clang/test/CodeGen/attr-speculative-load-hardening.c
index 97bccd03585c3945f2a03516a3d77ce9fe4741ba..784640f93932ac06ff0fd05d3e1df586f9e7da42 100644
--- a/clang/test/CodeGen/attr-speculative-load-hardening.c
+++ b/clang/test/CodeGen/attr-speculative-load-hardening.c
@@ -12,4 +12,4 @@ int test1() {
 
 // NOSLH: @{{.*}}test1{{.*}}[[NOSLH:#[0-9]+]]
 
-// NOSLH-NOT: attributes [[SLH]] = { {{.*}}speculative_load_hardening{{.*}} }
+// NOSLH-NOT: attributes [[NOSLH]] = { {{.*}}speculative_load_hardening{{.*}} }
diff --git a/clang/test/CodeGen/attr-weak-import.c b/clang/test/CodeGen/attr-weak-import.c
index 85989f03a277e2620ba93e116ee913fa82df1aeb..f02d09e8150998ae1ef51534580da3d516532e12 100644
--- a/clang/test/CodeGen/attr-weak-import.c
+++ b/clang/test/CodeGen/attr-weak-import.c
@@ -18,9 +18,9 @@ extern int E __attribute__((weak_import));
 int E;
 extern int E __attribute__((weak_import));
 
-// CHECK: @A = dso_local global i32
+// CHECK: @A = global i32
 // CHECK-NOT: @B =
-// CHECK: @C = dso_local global i32
-// CHECK: @D = dso_local global i32
-// CHECK: @E = dso_local global i32
+// CHECK: @C = global i32
+// CHECK: @D = global i32
+// CHECK: @E = global i32
 
diff --git a/clang/test/CodeGen/builtins-ppc-p8vector.c b/clang/test/CodeGen/builtins-ppc-p8vector.c
index 8f9c62b875db15b24ba86a01ad00fe1572a70c3f..07494c22f23b1ac7a865509932449a3995c87539 100644
--- a/clang/test/CodeGen/builtins-ppc-p8vector.c
+++ b/clang/test/CodeGen/builtins-ppc-p8vector.c
@@ -84,6 +84,10 @@ void test1() {
 // CHECK-LE: add <2 x i64>
 // CHECK-PPC: error: call to 'vec_add' is ambiguous
 
+  res_vuc = vec_add_u128(vuc, vuc);
+// CHECK: add <1 x i128>
+// CHECK-LE: add <1 x i128>
+
   /* vec_addc */
   res_vsi = vec_addc(vsi, vsi);
 // CHECK: @llvm.ppc.altivec.vaddcuw
@@ -99,6 +103,10 @@ void test1() {
 
   res_vux = vec_addc(vux, vux);
 // CHECK: @llvm.ppc.altivec.vaddcuq
+// CHECK-LE: @llvm.ppc.altivec.vaddcuq
+
+  res_vuc = vec_addc_u128(vuc, vuc);
+// CHECK: @llvm.ppc.altivec.vaddcuq
 // CHECK-LE: @llvm.ppc.altivec.vaddcuq
 
   /* vec_adde */
@@ -108,11 +116,19 @@ void test1() {
 
   res_vux = vec_adde(vux, vux, vux);
 // CHECK: @llvm.ppc.altivec.vaddeuqm
+// CHECK-LE: @llvm.ppc.altivec.vaddeuqm
+
+  res_vuc = vec_adde_u128(vuc, vuc, vuc);
+// CHECK: @llvm.ppc.altivec.vaddeuqm
 // CHECK-LE: @llvm.ppc.altivec.vaddeuqm
 
   /* vec_addec */
   res_vsx = vec_addec(vsx, vsx, vsx);
 // CHECK: @llvm.ppc.altivec.vaddecuq
+// CHECK-LE: @llvm.ppc.altivec.vaddecuq
+
+  res_vuc = vec_addec_u128(vuc, vuc, vuc);
+// CHECK: @llvm.ppc.altivec.vaddecuq
 // CHECK-LE: @llvm.ppc.altivec.vaddecuq
 
   /* vec_mergee */  
@@ -1606,6 +1622,14 @@ void test1() {
 // CHECK-LE: llvm.ppc.altivec.vgbbd
 // CHECK-PPC: warning: implicit declaration of function 'vec_gb'
 
+  res_vsll = vec_gbb(vsll);
+// CHECK: llvm.ppc.altivec.vgbbd
+// CHECK-LE: llvm.ppc.altivec.vgbbd
+
+  res_vull = vec_gbb(vull);
+// CHECK: llvm.ppc.altivec.vgbbd
+// CHECK-LE: llvm.ppc.altivec.vgbbd
+
   res_vull = vec_bperm(vux, vux);
 // CHECK: llvm.ppc.altivec.vbpermq
 // CHECK-LE: llvm.ppc.altivec.vbpermq
diff --git a/clang/test/CodeGen/builtins-ppc-quadword-noi128.c b/clang/test/CodeGen/builtins-ppc-quadword-noi128.c
new file mode 100644
index 0000000000000000000000000000000000000000..bc97db2be1e9c8f3fe0ebd5eecbac975fb2ff0fc
--- /dev/null
+++ b/clang/test/CodeGen/builtins-ppc-quadword-noi128.c
@@ -0,0 +1,178 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: powerpc-registered-target
+// RUN: %clang_cc1 -O2 -target-feature +altivec -target-feature +power8-vector \
+// RUN:   -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck \
+// RUN:   %s -check-prefix=CHECK-LE
+// RUN: %clang_cc1 -O2 -target-feature +altivec -target-feature +power8-vector \
+// RUN:   -triple powerpc64-aix-unknown -emit-llvm %s -o - | FileCheck \
+// RUN:   %s -check-prefix=CHECK-AIX
+#include <altivec.h>
+// CHECK-LE-LABEL: @test_subc(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vsubcuq(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]]) #[[ATTR3:[0-9]+]]
+// CHECK-LE-NEXT:    [[TMP3:%.*]] = bitcast <1 x i128> [[TMP2]] to <16 x i8>
+// CHECK-LE-NEXT:    ret <16 x i8> [[TMP3]]
+//
+// CHECK-AIX-LABEL: @test_subc(
+// CHECK-AIX-NEXT:  entry:
+// CHECK-AIX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP2:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vsubcuq(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]]) #[[ATTR3:[0-9]+]]
+// CHECK-AIX-NEXT:    [[TMP3:%.*]] = bitcast <1 x i128> [[TMP2]] to <16 x i8>
+// CHECK-AIX-NEXT:    ret <16 x i8> [[TMP3]]
+//
+vector unsigned char test_subc(vector unsigned char a, vector unsigned char b) {
+  return vec_subc_u128(a, b);
+}
+// CHECK-LE-LABEL: @test_subec(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP3:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vsubecuq(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]], <1 x i128> [[TMP2]]) #[[ATTR3]]
+// CHECK-LE-NEXT:    [[TMP4:%.*]] = bitcast <1 x i128> [[TMP3]] to <16 x i8>
+// CHECK-LE-NEXT:    ret <16 x i8> [[TMP4]]
+//
+// CHECK-AIX-LABEL: @test_subec(
+// CHECK-AIX-NEXT:  entry:
+// CHECK-AIX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP3:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vsubecuq(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]], <1 x i128> [[TMP2]]) #[[ATTR3]]
+// CHECK-AIX-NEXT:    [[TMP4:%.*]] = bitcast <1 x i128> [[TMP3]] to <16 x i8>
+// CHECK-AIX-NEXT:    ret <16 x i8> [[TMP4]]
+//
+vector unsigned char test_subec(vector unsigned char a, vector unsigned char b,
+                                vector unsigned char c) {
+  return vec_subec_u128(a, b, c);
+}
+// CHECK-LE-LABEL: @test_sube(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP3:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vsubeuqm(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]], <1 x i128> [[TMP2]]) #[[ATTR3]]
+// CHECK-LE-NEXT:    [[TMP4:%.*]] = bitcast <1 x i128> [[TMP3]] to <16 x i8>
+// CHECK-LE-NEXT:    ret <16 x i8> [[TMP4]]
+//
+// CHECK-AIX-LABEL: @test_sube(
+// CHECK-AIX-NEXT:  entry:
+// CHECK-AIX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP3:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vsubeuqm(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]], <1 x i128> [[TMP2]]) #[[ATTR3]]
+// CHECK-AIX-NEXT:    [[TMP4:%.*]] = bitcast <1 x i128> [[TMP3]] to <16 x i8>
+// CHECK-AIX-NEXT:    ret <16 x i8> [[TMP4]]
+//
+vector unsigned char test_sube(vector unsigned char a, vector unsigned char b,
+                               vector unsigned char c) {
+  return vec_sube_u128(a, b, c);
+}
+// CHECK-LE-LABEL: @test_sub(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[VADDUQM_I_NEG:%.*]] = sub <1 x i128> [[TMP2]], [[TMP0]]
+// CHECK-LE-NEXT:    [[VSUBUQM_I:%.*]] = sub <1 x i128> [[VADDUQM_I_NEG]], [[TMP1]]
+// CHECK-LE-NEXT:    [[TMP3:%.*]] = bitcast <1 x i128> [[VSUBUQM_I]] to <16 x i8>
+// CHECK-LE-NEXT:    ret <16 x i8> [[TMP3]]
+//
+// CHECK-AIX-LABEL: @test_sub(
+// CHECK-AIX-NEXT:  entry:
+// CHECK-AIX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[VADDUQM_I_NEG:%.*]] = sub <1 x i128> [[TMP2]], [[TMP0]]
+// CHECK-AIX-NEXT:    [[VSUBUQM_I:%.*]] = sub <1 x i128> [[VADDUQM_I_NEG]], [[TMP1]]
+// CHECK-AIX-NEXT:    [[TMP3:%.*]] = bitcast <1 x i128> [[VSUBUQM_I]] to <16 x i8>
+// CHECK-AIX-NEXT:    ret <16 x i8> [[TMP3]]
+//
+vector unsigned char test_sub(vector unsigned char a, vector unsigned char b,
+                              vector unsigned char c) {
+  return vec_sub_u128(a, vec_add_u128(b, c));
+}
+// CHECK-LE-LABEL: @test_addc(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vaddcuq(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]]) #[[ATTR3]]
+// CHECK-LE-NEXT:    [[TMP3:%.*]] = bitcast <1 x i128> [[TMP2]] to <16 x i8>
+// CHECK-LE-NEXT:    ret <16 x i8> [[TMP3]]
+//
+// CHECK-AIX-LABEL: @test_addc(
+// CHECK-AIX-NEXT:  entry:
+// CHECK-AIX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP2:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vaddcuq(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]]) #[[ATTR3]]
+// CHECK-AIX-NEXT:    [[TMP3:%.*]] = bitcast <1 x i128> [[TMP2]] to <16 x i8>
+// CHECK-AIX-NEXT:    ret <16 x i8> [[TMP3]]
+//
+vector unsigned char test_addc(vector unsigned char a, vector unsigned char b) {
+  return vec_addc_u128(a, b);
+}
+// CHECK-LE-LABEL: @test_addec(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP3:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vaddecuq(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]], <1 x i128> [[TMP2]]) #[[ATTR3]]
+// CHECK-LE-NEXT:    [[TMP4:%.*]] = bitcast <1 x i128> [[TMP3]] to <16 x i8>
+// CHECK-LE-NEXT:    ret <16 x i8> [[TMP4]]
+//
+// CHECK-AIX-LABEL: @test_addec(
+// CHECK-AIX-NEXT:  entry:
+// CHECK-AIX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP3:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vaddecuq(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]], <1 x i128> [[TMP2]]) #[[ATTR3]]
+// CHECK-AIX-NEXT:    [[TMP4:%.*]] = bitcast <1 x i128> [[TMP3]] to <16 x i8>
+// CHECK-AIX-NEXT:    ret <16 x i8> [[TMP4]]
+//
+vector unsigned char test_addec(vector unsigned char a, vector unsigned char b,
+                                vector unsigned char c) {
+  return vec_addec_u128(a, b, c);
+}
+// CHECK-LE-LABEL: @test_adde(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP3:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vaddeuqm(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]], <1 x i128> [[TMP2]]) #[[ATTR3]]
+// CHECK-LE-NEXT:    [[TMP4:%.*]] = bitcast <1 x i128> [[TMP3]] to <16 x i8>
+// CHECK-LE-NEXT:    ret <16 x i8> [[TMP4]]
+//
+// CHECK-AIX-LABEL: @test_adde(
+// CHECK-AIX-NEXT:  entry:
+// CHECK-AIX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[C:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP3:%.*]] = tail call <1 x i128> @llvm.ppc.altivec.vaddeuqm(<1 x i128> [[TMP0]], <1 x i128> [[TMP1]], <1 x i128> [[TMP2]]) #[[ATTR3]]
+// CHECK-AIX-NEXT:    [[TMP4:%.*]] = bitcast <1 x i128> [[TMP3]] to <16 x i8>
+// CHECK-AIX-NEXT:    ret <16 x i8> [[TMP4]]
+//
+vector unsigned char test_adde(vector unsigned char a, vector unsigned char b,
+                               vector unsigned char c) {
+  return vec_adde_u128(a, b, c);
+}
+// CHECK-LE-LABEL: @test_add(
+// CHECK-LE-NEXT:  entry:
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-LE-NEXT:    [[VADDUQM_I:%.*]] = add <1 x i128> [[TMP1]], [[TMP0]]
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = bitcast <1 x i128> [[VADDUQM_I]] to <16 x i8>
+// CHECK-LE-NEXT:    ret <16 x i8> [[TMP2]]
+//
+// CHECK-AIX-LABEL: @test_add(
+// CHECK-AIX-NEXT:  entry:
+// CHECK-AIX-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[A:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[B:%.*]] to <1 x i128>
+// CHECK-AIX-NEXT:    [[VADDUQM_I:%.*]] = add <1 x i128> [[TMP1]], [[TMP0]]
+// CHECK-AIX-NEXT:    [[TMP2:%.*]] = bitcast <1 x i128> [[VADDUQM_I]] to <16 x i8>
+// CHECK-AIX-NEXT:    ret <16 x i8> [[TMP2]]
+//
+vector unsigned char test_add(vector unsigned char a, vector unsigned char b) {
+  return vec_add_u128(a, b);
+}
diff --git a/clang/test/CodeGen/builtins-ppc-quadword.c b/clang/test/CodeGen/builtins-ppc-quadword.c
index 66cc9e8c7a88a6f0a7f975b34674213686116a6b..561f0c28f22502245cb11d7a55b9e7544785d783 100644
--- a/clang/test/CodeGen/builtins-ppc-quadword.c
+++ b/clang/test/CodeGen/builtins-ppc-quadword.c
@@ -14,6 +14,7 @@
 vector signed __int128 vlll = { -1 };
 // CHECK-PPC: error: __int128 is not supported on this target
 vector unsigned __int128 vulll = { 1 };
+vector unsigned char vuc;
 
 signed long long param_sll;
 // CHECK-PPC: error: __int128 is not supported on this target
@@ -25,6 +26,7 @@ unsigned __int128 param_ulll;
 vector signed __int128 res_vlll;
 // CHECK-PPC: error: __int128 is not supported on this target
 vector unsigned __int128 res_vulll;
+vector unsigned char res_vuc;
 
 
 // CHECK-LABEL: define{{.*}} void @test1
@@ -119,6 +121,10 @@ void test1() {
 // CHECK-LE: sub <1 x i128> 
 // CHECK-PPC: error: assigning to '__vector unsigned __int128' (vector of 1 'unsigned __int128' value) from incompatible type 'int'
   
+  res_vuc = vec_sub_u128(vuc, vuc);
+// CHECK: sub <1 x i128>
+// CHECK-LE: sub <1 x i128>
+
   /* vec_vsubeuqm */
   res_vlll = vec_vsubeuqm(vlll, vlll, vlll);
 // CHECK: @llvm.ppc.altivec.vsubeuqm
@@ -151,6 +157,10 @@ void test1() {
 // CHECK-LE: @llvm.ppc.altivec.vsubeuqm
 // CHECK-PPC: error: call to 'vec_sube' is ambiguous
 
+  res_vuc = vec_sube_u128(vuc, vuc, vuc);
+// CHECK: @llvm.ppc.altivec.vsubeuqm
+// CHECK-LE: @llvm.ppc.altivec.vsubeuqm
+
   /* vec_subc */
   res_vlll = vec_subc(vlll, vlll);
 // CHECK: @llvm.ppc.altivec.vsubcuq
@@ -162,6 +172,10 @@ void test1() {
 // CHECK-LE: @llvm.ppc.altivec.vsubcuq
 // KCHECK-PPC: error: call to 'vec_subc' is ambiguous
 
+  res_vuc = vec_subc_u128(vuc, vuc);
+// CHECK: @llvm.ppc.altivec.vsubcuq
+// CHECK-LE: @llvm.ppc.altivec.vsubcuq
+
   /* vec_vsubcuq */
   res_vlll = vec_vsubcuq(vlll, vlll);
 // CHECK: @llvm.ppc.altivec.vsubcuq
@@ -194,6 +208,10 @@ void test1() {
 // CHECK-LE: @llvm.ppc.altivec.vsubecuq
 // CHECK-PPC: error: assigning to '__vector unsigned __int128' (vector of 1 'unsigned __int128' value) from incompatible type 'int'  
 
+  res_vuc = vec_subec_u128(vuc, vuc, vuc);
+// CHECK: @llvm.ppc.altivec.vsubecuq
+// CHECK-LE: @llvm.ppc.altivec.vsubecuq
+
   res_vulll = vec_revb(vulll);
 // CHECK: store <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <16 x i8>* {{%.+}}, align 16
 // CHECK: call <4 x i32> @llvm.ppc.altivec.vperm(<4 x i32> {{%.+}}, <4 x i32> {{%.+}}, <16 x i8> {{%.+}})
diff --git a/clang/test/CodeGen/builtins-ppc-vsx.c b/clang/test/CodeGen/builtins-ppc-vsx.c
index e13f1ee1c058f62cd54cdf3e020a3231f3c08d0e..f6157cf46778c6be92f5091f11b7f01b692c7db1 100644
--- a/clang/test/CodeGen/builtins-ppc-vsx.c
+++ b/clang/test/CodeGen/builtins-ppc-vsx.c
@@ -1294,6 +1294,32 @@ void test1() {
 // CHECK-LE: uitofp <2 x i64> %{{.*}} to <2 x double>
 // CHECK-LE: fmul <2 x double>
 
+  res_vd = vec_ctd(vsll, 2);
+// CHECK: sitofp <2 x i64> %{{.*}} to <2 x double>
+// CHECK: fmul <2 x double> {{.*}} <double 2.500000e-01, double 2.500000e-01>
+// CHECK-LE: sitofp <2 x i64> %{{.*}} to <2 x double>
+// CHECK-LE: fmul <2 x double> {{.*}} <double 2.500000e-01, double 2.500000e-01>
+
+  res_vd = vec_ctd(vull, 2);
+// CHECK: uitofp <2 x i64> %{{.*}} to <2 x double>
+// CHECK: fmul <2 x double> {{.*}} <double 2.500000e-01, double 2.500000e-01>
+// CHECK-LE: uitofp <2 x i64> %{{.*}} to <2 x double>
+// CHECK-LE: fmul <2 x double> {{.*}} <double 2.500000e-01, double 2.500000e-01>
+
+  res_vd = vec_ctd(vsi, 2);
+// CHECK: call <2 x double> @llvm.ppc.vsx.xvcvsxwdp(<4 x i32>
+// CHECK: fmul <2 x double> {{.*}} <double 2.500000e-01, double 2.500000e-01>
+// CHECK-LE: vperm
+// CHECK-LE: call <2 x double> @llvm.ppc.vsx.xvcvsxwdp(<4 x i32>
+// CHECK-LE: fmul <2 x double> {{.*}} <double 2.500000e-01, double 2.500000e-01>
+
+  res_vd = vec_ctd(vui, 2);
+// CHECK: call <2 x double> @llvm.ppc.vsx.xvcvuxwdp(<4 x i32>
+// CHECK: fmul <2 x double> {{.*}} <double 2.500000e-01, double 2.500000e-01>
+// CHECK-LE: vperm
+// CHECK-LE: call <2 x double> @llvm.ppc.vsx.xvcvuxwdp(<4 x i32>
+// CHECK-LE: fmul <2 x double> {{.*}} <double 2.500000e-01, double 2.500000e-01>
+
   res_vsll = vec_signed(vd);
 // CHECK: fptosi <2 x double>
 // CHECK-LE: fptosi <2 x double>
@@ -1493,6 +1519,15 @@ void test1() {
 // CHECK-LE: sub nsw i32 17
 // CHECK-LE: sub nsw i32 18
 // CHECK-LE: sub nsw i32 31
+// CHECK-LE: @llvm.ppc.altivec.vperm
+
+  res_vf = vec_cvf(vd);
+// CHECK: @llvm.ppc.vsx.xvcvdpsp
+// CHECK-LE: @llvm.ppc.vsx.xvcvdpsp
+// CHECK-LE: sub nsw i32 16
+// CHECK-LE: sub nsw i32 17
+// CHECK-LE: sub nsw i32 18
+// CHECK-LE: sub nsw i32 31
 // CHECK-LE: @llvm.ppc.altivec.vperm
 
   res_vf = vec_floato(vsll);
@@ -1555,6 +1590,15 @@ void test1() {
 // CHECK-LE: sub nsw i32 18
 // CHECK-LE: sub nsw i32 31
 // CHECK-LE: @llvm.ppc.altivec.vperm
+// CHECK-LE: @llvm.ppc.vsx.xvcvspdp(<4 x float
+
+  res_vd = vec_cvf(vf);
+// CHECK: @llvm.ppc.vsx.xvcvspdp(<4 x float
+// CHECK-LE: sub nsw i32 16
+// CHECK-LE: sub nsw i32 17
+// CHECK-LE: sub nsw i32 18
+// CHECK-LE: sub nsw i32 31
+// CHECK-LE: @llvm.ppc.altivec.vperm
 // CHECK-LE: @llvm.ppc.vsx.xvcvspdp(<4 x float
 
   res_vd = vec_doubleh(vsi);
@@ -1932,6 +1976,22 @@ res_vuc = vec_xxpermdi(vuc, vuc, 1);
 // CHECK: shufflevector <2 x i64> %{{[0-9]+}}, <2 x i64> %{{[0-9]+}}, <2 x i32> <i32 0, i32 3>
 // CHECK-LE: shufflevector <2 x i64> %{{[0-9]+}}, <2 x i64> %{{[0-9]+}}, <2 x i32> <i32 0, i32 3>
 
+res_vd = vec_permi(vd, vd, 0);
+// CHECK: shufflevector <2 x double> %{{[0-9]+}}, <2 x double> %{{[0-9]+}}, <2 x i32> <i32 0, i32 2>
+// CHECK-LE: shufflevector <2 x double> %{{[0-9]+}}, <2 x double> %{{[0-9]+}}, <2 x i32> <i32 0, i32 2>
+
+res_vsll = vec_permi(vsll, vsll, 2);
+// CHECK: shufflevector <2 x i64> %{{[0-9]+}}, <2 x i64> %{{[0-9]+}}, <2 x i32> <i32 1, i32 2>
+// CHECK-LE: shufflevector <2 x i64> %{{[0-9]+}}, <2 x i64> %{{[0-9]+}}, <2 x i32> <i32 1, i32 2>
+
+res_vull = vec_permi(vull, vull, 3);
+// CHECK: shufflevector <2 x i64> %{{[0-9]+}}, <2 x i64> %{{[0-9]+}}, <2 x i32> <i32 1, i32 3>
+// CHECK-LE: shufflevector <2 x i64> %{{[0-9]+}}, <2 x i64> %{{[0-9]+}}, <2 x i32> <i32 1, i32 3>
+
+res_vull = vec_permi(vbll, vbll, 3);
+// CHECK: shufflevector <2 x i64> %{{[0-9]+}}, <2 x i64> %{{[0-9]+}}, <2 x i32> <i32 1, i32 3>
+// CHECK-LE: shufflevector <2 x i64> %{{[0-9]+}}, <2 x i64> %{{[0-9]+}}, <2 x i32> <i32 1, i32 3>
+
 res_vd = vec_xxsldwi(vd, vd, 0);
 // CHECK: shufflevector <4 x i32> %{{[0-9]+}}, <4 x i32> %{{[0-9]+}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 // CHECK-LE: shufflevector <4 x i32> %{{[0-9]+}}, <4 x i32> %{{[0-9]+}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/clang/test/CodeGen/builtins-wasm.c b/clang/test/CodeGen/builtins-wasm.c
index 771764c85d6bda2bdee1924fa27dea0771f989e5..f635e68258963e025e46619cb8835f1464f80156 100644
--- a/clang/test/CodeGen/builtins-wasm.c
+++ b/clang/test/CodeGen/builtins-wasm.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple wasm32-unknown-unknown -target-feature +unimplemented-simd128 -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -flax-vector-conversions=none -O3 -emit-llvm -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY32
-// RUN: %clang_cc1 -triple wasm64-unknown-unknown -target-feature +unimplemented-simd128 -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -flax-vector-conversions=none -O3 -emit-llvm -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY64
+// RUN: %clang_cc1 -triple wasm32-unknown-unknown -target-feature +simd128 -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -flax-vector-conversions=none -O3 -emit-llvm -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY32
+// RUN: %clang_cc1 -triple wasm64-unknown-unknown -target-feature +simd128 -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -flax-vector-conversions=none -O3 -emit-llvm -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY64
 // RUN: not %clang_cc1 -triple wasm64-unknown-unknown -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -flax-vector-conversions=none -O3 -emit-llvm -o - %s 2>&1 | FileCheck %s -check-prefixes MISSING-SIMD
 
 // SIMD convenience types
@@ -340,44 +340,44 @@ void store64_lane(long long *p, i64x2 v) {
   // WEBASSEMBLY-NEXT: ret
 }
 
-i8x16 add_saturate_s_i8x16(i8x16 x, i8x16 y) {
-  return __builtin_wasm_add_saturate_s_i8x16(x, y);
+i8x16 add_sat_s_i8x16(i8x16 x, i8x16 y) {
+  return __builtin_wasm_add_sat_s_i8x16(x, y);
   // WEBASSEMBLY: call <16 x i8> @llvm.sadd.sat.v16i8(
   // WEBASSEMBLY-SAME: <16 x i8> %x, <16 x i8> %y)
   // WEBASSEMBLY-NEXT: ret
 }
 
-u8x16 add_saturate_u_i8x16(u8x16 x, u8x16 y) {
-  return __builtin_wasm_add_saturate_u_i8x16(x, y);
+u8x16 add_sat_u_i8x16(u8x16 x, u8x16 y) {
+  return __builtin_wasm_add_sat_u_i8x16(x, y);
   // WEBASSEMBLY: call <16 x i8> @llvm.uadd.sat.v16i8(
   // WEBASSEMBLY-SAME: <16 x i8> %x, <16 x i8> %y)
   // WEBASSEMBLY-NEXT: ret
 }
 
-i16x8 add_saturate_s_i16x8(i16x8 x, i16x8 y) {
-  return __builtin_wasm_add_saturate_s_i16x8(x, y);
+i16x8 add_sat_s_i16x8(i16x8 x, i16x8 y) {
+  return __builtin_wasm_add_sat_s_i16x8(x, y);
   // WEBASSEMBLY: call <8 x i16> @llvm.sadd.sat.v8i16(
   // WEBASSEMBLY-SAME: <8 x i16> %x, <8 x i16> %y)
   // WEBASSEMBLY-NEXT: ret
 }
 
-u16x8 add_saturate_u_i16x8(u16x8 x, u16x8 y) {
-  return __builtin_wasm_add_saturate_u_i16x8(x, y);
+u16x8 add_sat_u_i16x8(u16x8 x, u16x8 y) {
+  return __builtin_wasm_add_sat_u_i16x8(x, y);
   // WEBASSEMBLY: call <8 x i16> @llvm.uadd.sat.v8i16(
   // WEBASSEMBLY-SAME: <8 x i16> %x, <8 x i16> %y)
   // WEBASSEMBLY-NEXT: ret
 }
 
-i8x16 sub_saturate_s_i8x16(i8x16 x, i8x16 y) {
-  return __builtin_wasm_sub_saturate_s_i8x16(x, y);
-  // WEBASSEMBLY: call <16 x i8> @llvm.wasm.sub.saturate.signed.v16i8(
+i8x16 sub_sat_s_i8x16(i8x16 x, i8x16 y) {
+  return __builtin_wasm_sub_sat_s_i8x16(x, y);
+  // WEBASSEMBLY: call <16 x i8> @llvm.wasm.sub.sat.signed.v16i8(
   // WEBASSEMBLY-SAME: <16 x i8> %x, <16 x i8> %y)
   // WEBASSEMBLY-NEXT: ret
 }
 
-u8x16 sub_saturate_u_i8x16(u8x16 x, u8x16 y) {
-  return __builtin_wasm_sub_saturate_u_i8x16(x, y);
-  // WEBASSEMBLY: call <16 x i8> @llvm.wasm.sub.saturate.unsigned.v16i8(
+u8x16 sub_sat_u_i8x16(u8x16 x, u8x16 y) {
+  return __builtin_wasm_sub_sat_u_i8x16(x, y);
+  // WEBASSEMBLY: call <16 x i8> @llvm.wasm.sub.sat.unsigned.v16i8(
   // WEBASSEMBLY-SAME: <16 x i8> %x, <16 x i8> %y)
   // WEBASSEMBLY-NEXT: ret
 }
@@ -484,16 +484,16 @@ u32x4 max_u_i32x4(u32x4 x, u32x4 y) {
   // WEBASSEMBLY-NEXT: ret <4 x i32> %1
 }
 
-i16x8 sub_saturate_s_i16x8(i16x8 x, i16x8 y) {
-  return __builtin_wasm_sub_saturate_s_i16x8(x, y);
-  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.sub.saturate.signed.v8i16(
+i16x8 sub_sat_s_i16x8(i16x8 x, i16x8 y) {
+  return __builtin_wasm_sub_sat_s_i16x8(x, y);
+  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.sub.sat.signed.v8i16(
   // WEBASSEMBLY-SAME: <8 x i16> %x, <8 x i16> %y)
   // WEBASSEMBLY-NEXT: ret
 }
 
-u16x8 sub_saturate_u_i16x8(u16x8 x, u16x8 y) {
-  return __builtin_wasm_sub_saturate_u_i16x8(x, y);
-  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.sub.saturate.unsigned.v8i16(
+u16x8 sub_sat_u_i16x8(u16x8 x, u16x8 y) {
+  return __builtin_wasm_sub_sat_u_i16x8(x, y);
+  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.sub.sat.unsigned.v8i16(
   // WEBASSEMBLY-SAME: <8 x i16> %x, <8 x i16> %y)
   // WEBASSEMBLY-NEXT: ret
 }
@@ -512,9 +512,9 @@ u16x8 avgr_u_i16x8(u16x8 x, u16x8 y) {
   // WEBASSEMBLY-NEXT: ret
 }
 
-i16x8 q15mulr_saturate_s_i16x8(i16x8 x, i16x8 y) {
-  return __builtin_wasm_q15mulr_saturate_s_i16x8(x, y);
-  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.q15mulr.saturate.signed(
+i16x8 q15mulr_sat_s_i16x8(i16x8 x, i16x8 y) {
+  return __builtin_wasm_q15mulr_sat_s_i16x8(x, y);
+  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.q15mulr.sat.signed(
   // WEBASSEMBLY-SAME: <8 x i16> %x, <8 x i16> %y)
   // WEBASSEMBLY-NEXT: ret
 }
@@ -644,34 +644,6 @@ i32x4 bitselect(i32x4 x, i32x4 y, i32x4 c) {
   // WEBASSEMBLY-NEXT: ret
 }
 
-i8x16 signselect_i8x16(i8x16 x, i8x16 y, i8x16 c) {
-  return __builtin_wasm_signselect_i8x16(x, y, c);
-  // WEBASSEMBLY: call <16 x i8> @llvm.wasm.signselect.v16i8(
-  // WEBASSEMBLY-SAME: <16 x i8> %x, <16 x i8> %y, <16 x i8> %c)
-  // WEBASSEMBLY-NEXT: ret
-}
-
-i16x8 signselect_i16x8(i16x8 x, i16x8 y, i16x8 c) {
-  return __builtin_wasm_signselect_i16x8(x, y, c);
-  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.signselect.v8i16(
-  // WEBASSEMBLY-SAME: <8 x i16> %x, <8 x i16> %y, <8 x i16> %c)
-  // WEBASSEMBLY-NEXT: ret
-}
-
-i32x4 signselect_i32x4(i32x4 x, i32x4 y, i32x4 c) {
-  return __builtin_wasm_signselect_i32x4(x, y, c);
-  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.signselect.v4i32(
-  // WEBASSEMBLY-SAME: <4 x i32> %x, <4 x i32> %y, <4 x i32> %c)
-  // WEBASSEMBLY-NEXT: ret
-}
-
-i64x2 signselect_i64x2(i64x2 x, i64x2 y, i64x2 c) {
-  return __builtin_wasm_signselect_i64x2(x, y, c);
-  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.signselect.v2i64(
-  // WEBASSEMBLY-SAME: <2 x i64> %x, <2 x i64> %y, <2 x i64> %c)
-  // WEBASSEMBLY-NEXT: ret
-}
-
 i8x16 popcnt(i8x16 x) {
   return __builtin_wasm_popcnt_i8x16(x);
   // WEBASSEMBLY: call <16 x i8> @llvm.wasm.popcnt(<16 x i8> %x)
@@ -884,34 +856,6 @@ f64x2 sqrt_f64x2(f64x2 x) {
   // WEBASSEMBLY: ret
 }
 
-f32x4 qfma_f32x4(f32x4 a, f32x4 b, f32x4 c) {
-  return __builtin_wasm_qfma_f32x4(a, b, c);
-  // WEBASSEMBLY: call <4 x float> @llvm.wasm.qfma.v4f32(
-  // WEBASSEMBLY-SAME: <4 x float> %a, <4 x float> %b, <4 x float> %c)
-  // WEBASSEMBLY-NEXT: ret
-}
-
-f32x4 qfms_f32x4(f32x4 a, f32x4 b, f32x4 c) {
-  return __builtin_wasm_qfms_f32x4(a, b, c);
-  // WEBASSEMBLY: call <4 x float> @llvm.wasm.qfms.v4f32(
-  // WEBASSEMBLY-SAME: <4 x float> %a, <4 x float> %b, <4 x float> %c)
-  // WEBASSEMBLY-NEXT: ret
-}
-
-f64x2 qfma_f64x2(f64x2 a, f64x2 b, f64x2 c) {
-  return __builtin_wasm_qfma_f64x2(a, b, c);
-  // WEBASSEMBLY: call <2 x double> @llvm.wasm.qfma.v2f64(
-  // WEBASSEMBLY-SAME: <2 x double> %a, <2 x double> %b, <2 x double> %c)
-  // WEBASSEMBLY-NEXT: ret
-}
-
-f64x2 qfms_f64x2(f64x2 a, f64x2 b, f64x2 c) {
-  return __builtin_wasm_qfms_f64x2(a, b, c);
-  // WEBASSEMBLY: call <2 x double> @llvm.wasm.qfms.v2f64(
-  // WEBASSEMBLY-SAME: <2 x double> %a, <2 x double> %b, <2 x double> %c)
-  // WEBASSEMBLY-NEXT: ret
-}
-
 i32x4 trunc_saturate_s_i32x4_f32x4(f32x4 f) {
   return __builtin_wasm_trunc_saturate_s_i32x4_f32x4(f);
   // WEBASSEMBLY: call <4 x i32> @llvm.wasm.trunc.saturate.signed.v4i32.v4f32(<4 x float> %f)
@@ -952,39 +896,27 @@ u16x8 narrow_u_i16x8_i32x4(u32x4 low, u32x4 high) {
   // WEBASSEMBLY: ret
 }
 
-i64x2 widen_low_s_i32x4_i64x2(i32x4 x) {
-  return __builtin_wasm_widen_low_s_i32x4_i64x2(x);
-  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.widen.low.signed(<4 x i32> %x)
+i64x2 extend_low_s_i32x4_i64x2(i32x4 x) {
+  return __builtin_wasm_extend_low_s_i32x4_i64x2(x);
+  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.extend.low.signed(<4 x i32> %x)
   // WEBASSEMBLY: ret
 }
 
-i64x2 widen_high_s_i32x4_i64x2(i32x4 x) {
-  return __builtin_wasm_widen_high_s_i32x4_i64x2(x);
-  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.widen.high.signed(<4 x i32> %x)
+i64x2 extend_high_s_i32x4_i64x2(i32x4 x) {
+  return __builtin_wasm_extend_high_s_i32x4_i64x2(x);
+  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.extend.high.signed(<4 x i32> %x)
   // WEBASSEMBLY: ret
 }
 
-u64x2 widen_low_u_i32x4_i64x2(u32x4 x) {
-  return __builtin_wasm_widen_low_u_i32x4_i64x2(x);
-  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.widen.low.unsigned(<4 x i32> %x)
+u64x2 extend_low_u_i32x4_i64x2(u32x4 x) {
+  return __builtin_wasm_extend_low_u_i32x4_i64x2(x);
+  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.extend.low.unsigned(<4 x i32> %x)
   // WEBASSEMBLY: ret
 }
 
-u64x2 widen_high_u_i32x4_i64x2(u32x4 x) {
-  return __builtin_wasm_widen_high_u_i32x4_i64x2(x);
-  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.widen.high.unsigned(<4 x i32> %x)
-  // WEBASSEMBLY: ret
-}
-
-i32x4 widen_s_i8x16_i32x4(i8x16 x) {
-  return __builtin_wasm_widen_s_i8x16_i32x4(x, 3);
-  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.signed(<16 x i8> %x, i32 3)
-  // WEBASSEMBLY: ret
-}
-
-u32x4 widen_u_i8x16_i32x4(u8x16 x) {
-  return __builtin_wasm_widen_u_i8x16_i32x4(x, 3);
-  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.unsigned(<16 x i8> %x, i32 3)
+u64x2 extend_high_u_i32x4_i64x2(u32x4 x) {
+  return __builtin_wasm_extend_high_u_i32x4_i64x2(x);
+  // WEBASSEMBLY: call <2 x i64> @llvm.wasm.extend.high.unsigned(<4 x i32> %x)
   // WEBASSEMBLY: ret
 }
 
@@ -1000,15 +932,15 @@ f64x2 convert_low_u_i32x4_f64x2(u32x4 x) {
   // WEBASSEMBLY: ret
 }
 
-i32x4 trunc_saturate_zero_s_f64x2_i32x4(f64x2 x) {
-  return __builtin_wasm_trunc_saturate_zero_s_f64x2_i32x4(x);
-  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.trunc.saturate.zero.signed(<2 x double> %x)
+i32x4 trunc_sat_zero_s_f64x2_i32x4(f64x2 x) {
+  return __builtin_wasm_trunc_sat_zero_s_f64x2_i32x4(x);
+  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.trunc.sat.zero.signed(<2 x double> %x)
   // WEBASSEMBLY: ret
 }
 
-u32x4 trunc_saturate_zero_u_f64x2_i32x4(f64x2 x) {
-  return __builtin_wasm_trunc_saturate_zero_u_f64x2_i32x4(x);
-  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.trunc.saturate.zero.unsigned(<2 x double> %x)
+u32x4 trunc_sat_zero_u_f64x2_i32x4(f64x2 x) {
+  return __builtin_wasm_trunc_sat_zero_u_f64x2_i32x4(x);
+  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.trunc.sat.zero.unsigned(<2 x double> %x)
   // WEBASSEMBLY: ret
 }
 
@@ -1050,13 +982,3 @@ i8x16 shuffle(i8x16 x, i8x16 y) {
   // WEBASSEMBLY-SAME: i32 15
   // WEBASSEMBLY-NEXT: ret
 }
-
-void prefetch_t(void *p) {
-  return __builtin_wasm_prefetch_t(p);
-  // WEBASSEMBLY: call void @llvm.wasm.prefetch.t(i8* %p)
-}
-
-void prefetch_nt(void *p) {
-  return __builtin_wasm_prefetch_nt(p);
-  // WEBASSEMBLY: call void @llvm.wasm.prefetch.nt(i8* %p)
-}
diff --git a/clang/test/CodeGen/mcount.c b/clang/test/CodeGen/mcount.c
index 649f0b56949d013191d955a0ff014872427653d4..8f994ab4e75443a64d2c2336f9fbcaf4750916b1 100644
--- a/clang/test/CodeGen/mcount.c
+++ b/clang/test/CodeGen/mcount.c
@@ -12,6 +12,13 @@
 // RUN: %clang_cc1 -pg -triple mipsel-unknown-gnu-linux -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
 // RUN: %clang_cc1 -pg -triple mips64-unknown-gnu-linux -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
 // RUN: %clang_cc1 -pg -triple mips64el-unknown-gnu-linux -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
+// RUN: %clang_cc1 -pg -triple riscv32-elf -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
+// RUN: %clang_cc1 -pg -triple riscv64-elf -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
+// RUN: %clang_cc1 -pg -triple riscv32-linux -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
+// RUN: %clang_cc1 -pg -triple riscv64-linux -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
+// RUN: %clang_cc1 -pg -triple riscv64-freebsd -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
+// RUN: %clang_cc1 -pg -triple riscv64-freebsd -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
+// RUN: %clang_cc1 -pg -triple riscv64-openbsd -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-PREFIXED,NO-MCOUNT1 %s
 // RUN: %clang_cc1 -pg -triple powerpc-netbsd -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-DOUBLE-PREFIXED,NO-MCOUNT1 %s
 // RUN: %clang_cc1 -pg -triple powerpc64-netbsd -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-DOUBLE-PREFIXED,NO-MCOUNT1 %s
 // RUN: %clang_cc1 -pg -triple powerpc64le-netbsd -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK-DOUBLE-PREFIXED,NO-MCOUNT1 %s
diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll
index 867203417754fb270eee8bee2d0b7903946d5a9e..1e9d5d4d2629ce1ce5c299917a1eaafb52f6e0ef 100644
--- a/clang/test/CodeGen/thinlto-distributed-newpm.ll
+++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll
@@ -12,7 +12,7 @@
 ; RUN: %clang -target x86_64-grtev4-linux-gnu \
 ; RUN:   -O2 -fexperimental-new-pass-manager -Xclang -fdebug-pass-manager \
 ; RUN:   -c -fthinlto-index=%t.o.thinlto.bc \
-; RUN:   -o %t.native.o -x ir %t.o 2>&1 | FileCheck -check-prefixes=CHECK-O,CHECK-O2 %s --dump-input=fail
+; RUN:   -o %t.native.o -x ir %t.o 2>&1 | FileCheck -check-prefix=CHECK-O %s --dump-input=fail
 
 ; RUN: %clang -target x86_64-grtev4-linux-gnu \
 ; RUN:   -O3 -fexperimental-new-pass-manager -Xclang -fdebug-pass-manager \
@@ -70,24 +70,19 @@
 ; CHECK-O: Starting CGSCC pass manager run.
 ; CHECK-O: Running pass: InlinerPass on (main)
 ; CHECK-O: Running pass: PostOrderFunctionAttrsPass on (main)
-; CHECK-O: Clearing all analysis results for: main
+; CHECK-O: Invalidating analysis: DominatorTreeAnalysis on main
+; CHECK-O: Invalidating analysis: BasicAA on main
+; CHECK-O: Invalidating analysis: AAManager on main
 ; CHECK-O3: Running pass: ArgumentPromotionPass on (main)
-; CHECK-O3: Running analysis: TargetIRAnalysis on main
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: SROA on main
 ; These next two can appear in any order since they are accessed as parameters
 ; on the same call to SROA::runImpl
 ; CHECK-O-DAG: Running analysis: DominatorTreeAnalysis on main
-; CHECK-O-DAG: Running analysis: AssumptionAnalysis on main
 ; CHECK-O: Running pass: EarlyCSEPass on main
-; CHECK-O: Running analysis: TargetLibraryAnalysis on main
-; CHECK-O2: Running analysis: TargetIRAnalysis on main
 ; CHECK-O: Running analysis: MemorySSAAnalysis on main
 ; CHECK-O: Running analysis: AAManager on main
 ; CHECK-O: Running analysis: BasicAA on main
-; CHECK-O: Running analysis: ScopedNoAliasAA on main
-; CHECK-O: Running analysis: TypeBasedAA on main
-; CHECK-O: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-O: Running pass: SpeculativeExecutionPass on main
 ; CHECK-O: Running pass: JumpThreadingPass on main
 ; CHECK-O: Running analysis: LazyValueAnalysis on main
@@ -96,7 +91,6 @@
 ; CHECK-O: Running pass: SimplifyCFGPass on main
 ; CHECK-O3: Running pass: AggressiveInstCombinePass on main
 ; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O: Running analysis: OptimizationRemarkEmitterAnalysis on main
 ; CHECK-O: Running pass: LibCallsShrinkWrapPass on main
 ; CHECK-O: Running pass: TailCallElimPass on main
 ; CHECK-O: Running pass: SimplifyCFGPass on main
diff --git a/clang/test/CodeGen/unique-internal-linkage-names-dwarf.c b/clang/test/CodeGen/unique-internal-linkage-names-dwarf.c
index a3583426de7902de44f5ff64416f4e3c5a6ed9a4..e5d507e154aebf59dede719f6718d2ff062bdb4d 100644
--- a/clang/test/CodeGen/unique-internal-linkage-names-dwarf.c
+++ b/clang/test/CodeGen/unique-internal-linkage-names-dwarf.c
@@ -8,21 +8,48 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux -debug-info-kind=limited -dwarf-version=5 -funique-internal-linkage-names -emit-llvm -o -  %s | FileCheck %s --check-prefix=UNIQUE
 
 static int glob;
+// foo should be given a uniquefied name under -funique-internal-linkage-names.
 static int foo(void) {
   return glob;
 }
 
+// bar should not be given a uniquefied name under -funique-internal-linkage-names, 
+// since it doesn't come with valid prototype.
+static int bar(a) int a;
+{
+  return glob + a;
+}
+
+// go should be given a uniquefied name under -funique-internal-linkage-names, even 
+// if its definition doesn't come with a valid prototype, but the declaration here
+// has a prototype.
+static int go(int);
+
 void baz() {
   foo();
+  bar(1);
+  go(2);
 }
 
+static int go(a) int a;
+{
+  return glob + a;
+}
+
+
 // PLAIN: @glob = internal global i32
 // PLAIN: define internal i32 @foo()
+// PLAIN: define internal i32 @bar(i32 %a)
 // PLAIN: distinct !DIGlobalVariable(name: "glob"{{.*}})
 // PLAIN: distinct !DISubprogram(name: "foo"{{.*}})
+// PLAIN: distinct !DISubprogram(name: "bar"{{.*}})
+// PLAIN: distinct !DISubprogram(name: "go"{{.*}})
 // PLAIN-NOT: linkageName:
 //
 // UNIQUE: @glob = internal global i32
 // UNIQUE: define internal i32 @_ZL3foov.[[MODHASH:__uniq.[0-9]+]]()
+// UNIQUE: define internal i32 @bar(i32 %a)
+// UNIQUE: define internal i32 @_ZL2goi.[[MODHASH]](i32 %a)
 // UNIQUE: distinct !DIGlobalVariable(name: "glob"{{.*}})
 // UNIQUE: distinct !DISubprogram(name: "foo", linkageName: "_ZL3foov.[[MODHASH]]"{{.*}})
+// UNIQUE: distinct !DISubprogram(name: "go", linkageName: "_ZL2goi.[[MODHASH]]"{{.*}})
diff --git a/clang/test/CodeGenCUDA/builtin-mangled-name.cu b/clang/test/CodeGenCUDA/builtin-mangled-name.cu
new file mode 100644
index 0000000000000000000000000000000000000000..e9dca5680155b5d329a02ddeef5313ee123acb4a
--- /dev/null
+++ b/clang/test/CodeGenCUDA/builtin-mangled-name.cu
@@ -0,0 +1,28 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-gnu-linux -aux-triple amdgcn-amd-amdhsa \
+// RUN:   -emit-llvm -o - -x hip %s | FileCheck -check-prefixes=CHECK,LNX %s
+// RUN: %clang_cc1 -triple x86_64-unknown-windows-msvc -aux-triple amdgcn-amd-amdhsa \
+// RUN:   -emit-llvm -o - -x hip %s | FileCheck -check-prefixes=CHECK,MSVC %s
+
+#include "Inputs/cuda.h"
+
+namespace X {
+  __global__ void kern1(int *x);
+  __device__ int var1;
+}
+
+// CHECK: @[[STR1:.*]] = {{.*}} c"_ZN1X5kern1EPi\00"
+// CHECK: @[[STR2:.*]] = {{.*}} c"_ZN1X4var1E\00"
+
+// LNX-LABEL: define {{.*}}@_Z4fun1v()
+// MSVC-LABEL: define {{.*}} @"?fun1@@YAPEBDXZ"()
+// CHECK: ret i8* getelementptr inbounds ({{.*}} @[[STR1]], i64 0, i64 0)
+const char *fun1() {
+  return __builtin_get_device_side_mangled_name(X::kern1);
+}
+
+// LNX-LABEL: define {{.*}}@_Z4fun2v()
+// MSVC-LABEL: define {{.*}}@"?fun2@@YAPEBDXZ"()
+// CHECK: ret i8* getelementptr inbounds ({{.*}} @[[STR2]], i64 0, i64 0)
+__host__ __device__ const char *fun2() {
+  return __builtin_get_device_side_mangled_name(X::var1);
+}
diff --git a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
index ec2ee37f97c07590b6bbcb181cdeeae96c793cf3..27e8a4999c328f08b52a110858f9e898b91184ee 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
@@ -1,60 +1,263 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu -verify
 // RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s
 
+// CHECK-LABEL: @_Z2wli(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+// CHECK:       while.cond:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+// CHECK-NEXT:    [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 true)
+// CHECK-NEXT:    br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+// CHECK:       while.body:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK:       while.end:
+// CHECK-NEXT:    ret void
+//
 void wl(int e){
-  // CHECK-LABEL: define{{.*}}wl
-  // CHECK: br {{.*}} !prof !6
   while(e) [[likely]] ++e;
 }
 
+// CHECK-LABEL: @_Z2wui(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+// CHECK:       while.cond:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+// CHECK-NEXT:    [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 false)
+// CHECK-NEXT:    br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+// CHECK:       while.body:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_COND]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK:       while.end:
+// CHECK-NEXT:    ret void
+//
 void wu(int e){
-  // CHECK-LABEL: define{{.*}}wu
-  // CHECK: br {{.*}} !prof !10
   while(e) [[unlikely]] ++e;
 }
 
+// CHECK-LABEL: @_Z15w_branch_elidedj(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+// CHECK:       while.body:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add i32 [[TMP0]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+//
 void w_branch_elided(unsigned e){
-  // CHECK-LABEL: define{{.*}}w_branch_elided
-  // CHECK-NOT: br {{.*}} !prof
   // expected-warning@+2 {{attribute 'likely' has no effect when annotating an infinite loop}}
   // expected-note@+1 {{annotating the infinite loop here}}
   while(1) [[likely]] ++e;
 }
 
+// CHECK-LABEL: @_Z2flj(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4:[0-9]+]]
+// CHECK-NEXT:    store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
+// CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
+// CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK:       for.cond.cleanup:
+// CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add i32 [[TMP4]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    ret void
+//
 void fl(unsigned e)
 {
-  // CHECK-LABEL: define{{.*}}fl
-  // CHECK: br {{.*}} !prof !6
   for(int i = 0; i != e; ++e) [[likely]];
 }
 
+// CHECK-LABEL: @_Z2fui(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4]]
+// CHECK-NEXT:    store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
+// CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
+// CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK:       for.cond.cleanup:
+// CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    ret void
+//
 void fu(int e)
 {
-  // CHECK-LABEL: define{{.*}}fu
-  // CHECK: br {{.*}} !prof !10
   for(int i = 0; i != e; ++e) [[unlikely]];
 }
 
+// CHECK-LABEL: @_Z15f_branch_elidedv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP13:![0-9]+]]
+//
 void f_branch_elided()
 {
-  // CHECK-LABEL: define{{.*}}f_branch_elided
-  // CHECK-NOT: br {{.*}} !prof
   for(;;) [[likely]];
 }
 
+// CHECK-LABEL: @_Z3frlOA4_i(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT:    [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT:    [[__BEGIN1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[__END1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14:![0-9]+]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0
+// CHECK-NEXT:    store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4
+// CHECK-NEXT:    store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]]
+// CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
+// CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK:       for.cond.cleanup:
+// CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1
+// CHECK-NEXT:    store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    ret void
+//
 void frl(int (&&e) [4])
 {
-  // CHECK-LABEL: define{{.*}}frl
-  // CHECK: br {{.*}} !prof !6
   for(int i : e) [[likely]];
 }
 
+// CHECK-LABEL: @_Z3fruOA4_i(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT:    [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT:    [[__BEGIN1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[__END1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0
+// CHECK-NEXT:    store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4
+// CHECK-NEXT:    store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]]
+// CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
+// CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK:       for.cond.cleanup:
+// CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1
+// CHECK-NEXT:    store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    ret void
+//
 void fru(int (&&e) [4])
 {
-  // CHECK-LABEL: define{{.*}}fru
-  // CHECK: br {{.*}} !prof !10
   for(int i : e) [[unlikely]];
 }
-
-// CHECK: !6 = !{!"branch_weights", i32 2000, i32 1}
-// CHECK: !10 = !{!"branch_weights", i32 1, i32 2000}
diff --git a/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp b/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
index 5fb7a67a7d9e83c57696e8217ab6a31645049495..05729eb5a789e90a6131db2dcda60a7c906e3e2b 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
@@ -1,59 +1,111 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s
 
 extern volatile int i;
 
+// CHECK-LABEL: @_Z8OneCaseLv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !6
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void OneCaseL() {
-  // CHECK-LABEL: define{{.*}}OneCaseL
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !6
   switch (i) {
     [[likely]] case 1: break;
   }
 }
 
+// CHECK-LABEL: @_Z8OneCaseUv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    ], !prof !7
+// CHECK:       sw.bb:
+// CHECK-NEXT:    [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void OneCaseU() {
-  // CHECK-LABEL: define{{.*}}OneCaseU
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !7
   switch (i) {
     [[unlikely]] case 1: ++i; break;
   }
 }
 
+// CHECK-LABEL: @_Z10TwoCasesLNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !8
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesLN() {
-  // CHECK-LABEL: define{{.*}}TwoCasesLN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !8
   switch (i) {
     [[likely]] case 1: break;
     case 2: break;
   }
 }
 
+// CHECK-LABEL: @_Z10TwoCasesUNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !9
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesUN() {
-  // CHECK-LABEL: define{{.*}}TwoCasesUN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !9
   switch (i) {
     [[unlikely]] case 1: break;
     case 2: break;
   }
 }
 
+// CHECK-LABEL: @_Z10TwoCasesLUv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !10
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesLU() {
-  // CHECK-LABEL: define{{.*}}TwoCasesLU
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !10
   switch (i) {
     [[likely]] case 1: break;
     [[unlikely]] case 2: break;
   }
 }
 
+// CHECK-LABEL: @_Z20CasesFallthroughNNLNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_BB]]
+// CHECK-NEXT:    i32 3, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 4, label [[SW_BB1]]
+// CHECK-NEXT:    ], !prof !11
+// CHECK:       sw.bb:
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughNNLN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughNNLN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !11
   switch (i) {
     case 1:
     case 2:
@@ -62,10 +114,23 @@ void CasesFallthroughNNLN() {
   }
 }
 
+// CHECK-LABEL: @_Z20CasesFallthroughNNUNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_BB]]
+// CHECK-NEXT:    i32 3, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 4, label [[SW_BB1]]
+// CHECK-NEXT:    ], !prof !12
+// CHECK:       sw.bb:
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughNNUN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughNNUN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !12
   switch (i) {
     case 1:
     case 2:
@@ -74,10 +139,32 @@ void CasesFallthroughNNUN() {
   }
 }
 
+// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallLNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_BB]]
+// CHECK-NEXT:    i32 3, label [[SW_BB]]
+// CHECK-NEXT:    i32 4, label [[SW_BB]]
+// CHECK-NEXT:    i32 5, label [[SW_BB]]
+// CHECK-NEXT:    i32 102, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 103, label [[SW_BB2:%.*]]
+// CHECK-NEXT:    i32 104, label [[SW_BB2]]
+// CHECK-NEXT:    ], !prof !13
+// CHECK:       sw.bb:
+// CHECK-NEXT:    [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_BB2]]
+// CHECK:       sw.bb2:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughRangeSmallLN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallLN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !13
   switch (i) {
     case 1 ... 5: ++i;
     case 102:
@@ -86,10 +173,32 @@ void CasesFallthroughRangeSmallLN() {
   }
 }
 
+// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallUNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_BB]]
+// CHECK-NEXT:    i32 3, label [[SW_BB]]
+// CHECK-NEXT:    i32 4, label [[SW_BB]]
+// CHECK-NEXT:    i32 5, label [[SW_BB]]
+// CHECK-NEXT:    i32 102, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 103, label [[SW_BB2:%.*]]
+// CHECK-NEXT:    i32 104, label [[SW_BB2]]
+// CHECK-NEXT:    ], !prof !14
+// CHECK:       sw.bb:
+// CHECK-NEXT:    [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_BB2]]
+// CHECK:       sw.bb2:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughRangeSmallUN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallUN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !14
   switch (i) {
     case 1 ... 5: ++i;
     case 102:
@@ -98,12 +207,26 @@ void CasesFallthroughRangeSmallUN() {
   }
 }
 
+// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeLLNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [
+// CHECK-NEXT:    i32 1003, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 104, label [[SW_BB1]]
+// CHECK-NEXT:    ], !prof !8
+// CHECK:       sw.bb:
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_EPILOG:%.*]]
+// CHECK:       sw.caserange:
+// CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], 0
+// CHECK-NEXT:    [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64
+// CHECK-NEXT:    [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 true)
+// CHECK-NEXT:    br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughRangeLargeLLN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeLLN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !8
-  // CHECK: caserange
-  // CHECK: br{{.*}} !prof !15
   switch (i) {
     [[likely]] case 0 ... 64:
     [[likely]] case 1003:
@@ -111,12 +234,26 @@ void CasesFallthroughRangeLargeLLN() {
   }
 }
 
+// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeUUNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [
+// CHECK-NEXT:    i32 1003, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 104, label [[SW_BB1]]
+// CHECK-NEXT:    ], !prof !9
+// CHECK:       sw.bb:
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_EPILOG:%.*]]
+// CHECK:       sw.caserange:
+// CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], 0
+// CHECK-NEXT:    [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64
+// CHECK-NEXT:    [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 false)
+// CHECK-NEXT:    br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughRangeLargeUUN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeUUN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !9
-  // CHECK: caserange
-  // CHECK: br{{.*}} !prof !16
   switch (i) {
     [[unlikely]] case 0 ... 64:
     [[unlikely]] case 1003:
@@ -124,30 +261,55 @@ void CasesFallthroughRangeLargeUUN() {
   }
 }
 
+// CHECK-LABEL: @_Z15OneCaseDefaultLv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    ], !prof !15
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void OneCaseDefaultL() {
-  // CHECK-LABEL: define{{.*}}OneCaseDefaultL
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !17
   switch (i) {
     case 1: break;
     [[likely]] default: break;
   }
 }
 
+// CHECK-LABEL: @_Z15OneCaseDefaultUv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    ], !prof !16
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void OneCaseDefaultU() {
-  // CHECK-LABEL: define{{.*}}OneCaseDefaultU
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !18
   switch (i) {
     case 1: break;
     [[unlikely]] default: break;
   }
 }
 
+// CHECK-LABEL: @_Z18TwoCasesDefaultLNLv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !17
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesDefaultLNL() {
-  // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNL
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !19
   switch (i) {
     [[likely]] case 1: break;
     case 2: break;
@@ -155,10 +317,19 @@ void TwoCasesDefaultLNL() {
   }
 }
 
+// CHECK-LABEL: @_Z18TwoCasesDefaultLNNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !8
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesDefaultLNN() {
-  // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !8
   switch (i) {
     [[likely]] case 1: break;
     case 2: break;
@@ -166,29 +337,22 @@ void TwoCasesDefaultLNN() {
   }
 }
 
+// CHECK-LABEL: @_Z18TwoCasesDefaultLNUv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !18
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesDefaultLNU() {
-  // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNU
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !20
   switch (i) {
     [[likely]] case 1: break;
     case 2: break;
     [[unlikely]] default: break;
   }
 }
-
-// CHECK: !6 = !{!"branch_weights", i32 357913942, i32 715827883}
-// CHECK: !7 = !{!"branch_weights", i32 536870912, i32 1}
-// CHECK: !8 = !{!"branch_weights", i32 238609295, i32 715827883, i32 238609295}
-// CHECK: !9 = !{!"branch_weights", i32 357913942, i32 1, i32 357913942}
-// CHECK: !10 = !{!"branch_weights", i32 357913942, i32 715827883, i32 1}
-// CHECK: !11 = !{!"branch_weights", i32 143165577, i32 143165577, i32 143165577, i32 715827883, i32 143165577}
-// CHECK: !12 = !{!"branch_weights", i32 214748365, i32 214748365, i32 214748365, i32 1, i32 214748365}
-// CHECK: !13 = !{!"branch_weights", i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 715827883, i32 79536432}
-// CHECK: !14 = !{!"branch_weights", i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 1, i32 119304648}
-// CHECK: !15 = !{!"branch_weights", i32 2000, i32 1}
-// CHECK: !16 = !{!"branch_weights", i32 1, i32 2000}
-// CHECK: !17 = !{!"branch_weights", i32 715827883, i32 357913942}
-// CHECK: !18 = !{!"branch_weights", i32 1, i32 536870912}
-// CHECK: !19 = !{!"branch_weights", i32 536870912, i32 536870912, i32 268435456}
-// CHECK: !20 = !{!"branch_weights", i32 1, i32 715827883, i32 357913942}
diff --git a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
index a611587b56f72728d58a24413efb5585b2c69df1..be9fc941c4809e13fb0de9f1de3cbdcc39ad117e 100644
--- a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
+++ b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
@@ -16,20 +16,20 @@ void usage() {
   bar(f);
 }
 
-// WINDOWS: define dso_local i32 @"?bar@@YAHUFoo@@@Z"(<{ %struct.Foo }>* inalloca %0)
+// WINDOWS: define dso_local i32 @"?bar@@YAHUFoo@@@Z"(<{ %struct.Foo }>* inalloca(<{ %struct.Foo }>) %0)
 // WINDOWS: %[[O:[0-9a-zA-Z]+]] = getelementptr inbounds <{ %struct.Foo }>, <{ %struct.Foo }>* %0, i32 0, i32 0
 // WINDOWS: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds %struct.Foo, %struct.Foo* %[[O]], i32 0, i32 0
 // WINDOWS: %[[LOAD:[0-9a-zA-Z]+]] = load i32, i32* %[[X]]
 // WINDOWS: ret i32 %[[LOAD]]
 
-// WINDOWS: define dso_local i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(<{ %struct.Foo }>* inalloca %0)
+// WINDOWS: define dso_local i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(<{ %struct.Foo }>* inalloca(<{ %struct.Foo }>) %0)
 // WINDOWS: %[[O:[0-9a-zA-Z]+]] = getelementptr inbounds <{ %struct.Foo }>, <{ %struct.Foo }>* %0, i32 0, i32 0
 // WINDOWS: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds %struct.Foo, %struct.Foo* %[[O]], i32 0, i32 0
 // WINDOWS: %[[LOAD:[0-9a-zA-Z]+]] = load i32, i32* %[[X]]
 // WINDOWS: %[[ADD:[0-9a-zA-Z]+]] = add nsw i32 %[[LOAD]], 1
 // WINDOWS: ret i32 %[[ADD]]
 
-// WINDOWS: define dso_local i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(<{ %struct.Foo }>* inalloca %0)
+// WINDOWS: define dso_local i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(<{ %struct.Foo }>* inalloca(<{ %struct.Foo }>) %0)
 // WINDOWS: %[[O:[0-9a-zA-Z]+]] = getelementptr inbounds <{ %struct.Foo }>, <{ %struct.Foo }>* %0, i32 0, i32 0
 // WINDOWS: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds %struct.Foo, %struct.Foo* %[[O]], i32 0, i32 0
 // WINDOWS: %[[LOAD:[0-9a-zA-Z]+]] = load i32, i32* %[[X]]
@@ -39,7 +39,7 @@ void usage() {
 // WINDOWS: define dso_local void @"?usage@@YAXXZ"()
 // WINDOWS: %[[F:[0-9a-zA-Z]+]] = alloca %struct.Foo
 // WINDOWS: %[[ARGMEM:[0-9a-zA-Z]+]] = alloca inalloca <{ %struct.Foo }>
-// WINDOWS: %[[CALL:[0-9a-zA-Z]+]] = call i32 @"?bar@@YAHUFoo@@@Z.resolver"(<{ %struct.Foo }>* inalloca %[[ARGMEM]])
+// WINDOWS: %[[CALL:[0-9a-zA-Z]+]] = call i32 @"?bar@@YAHUFoo@@@Z.resolver"(<{ %struct.Foo }>* inalloca(<{ %struct.Foo }>) %[[ARGMEM]])
 
 // WINDOWS: define weak_odr dso_local i32 @"?bar@@YAHUFoo@@@Z.resolver"(<{ %struct.Foo }>* %0)
 // WINDOWS: %[[RET:[0-9a-zA-Z]+]] = musttail call i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(<{ %struct.Foo }>* %0)
diff --git a/clang/test/CodeGenCXX/bitfield-layout.cpp b/clang/test/CodeGenCXX/bitfield-layout.cpp
index 79dbf9c691c459f5bb914b701b8d6a68845734e1..d570b8f33e343553efc0cfc3c18c8866af5d3b4a 100644
--- a/clang/test/CodeGenCXX/bitfield-layout.cpp
+++ b/clang/test/CodeGenCXX/bitfield-layout.cpp
@@ -93,7 +93,7 @@ int test_trunc_int() {
   } const U = {15};  // 0b00001111
   return U.i;
 }
-// CHECK: define dso_local i32 @test_trunc_int()
+// CHECK: define{{.*}} i32 @test_trunc_int()
 // CHECK: ret i32 -1
 
 int test_trunc_three_bits() {
@@ -102,7 +102,7 @@ int test_trunc_three_bits() {
   } const U = {15};  // 0b00001111
   return U.i;
 }
-// CHECK: define dso_local i32 @test_trunc_three_bits()
+// CHECK: define{{.*}} i32 @test_trunc_three_bits()
 // CHECK: ret i32 -1
 
 int test_trunc_1() {
@@ -111,7 +111,7 @@ int test_trunc_1() {
   } const U = {15};  // 0b00001111
   return U.i;
 }
-// CHECK: define dso_local i32 @test_trunc_1()
+// CHECK: define{{.*}} i32 @test_trunc_1()
 // CHECK: ret i32 -1
 
 int test_trunc_zero() {
@@ -120,7 +120,7 @@ int test_trunc_zero() {
   } const U = {80};  // 0b01010000
   return U.i;
 }
-// CHECK: define dso_local i32 @test_trunc_zero()
+// CHECK: define{{.*}} i32 @test_trunc_zero()
 // CHECK: ret i32 0
 
 int test_constexpr() {
@@ -129,7 +129,7 @@ int test_constexpr() {
   } const U = {1 + 2 + 4 + 8}; // 0b00001111
   return U.i;
 }
-// CHECK: define dso_local i32 @test_constexpr()
+// CHECK: define{{.*}} i32 @test_constexpr()
 // CHECK: ret i32 -1
 
 int test_notrunc() {
@@ -138,7 +138,7 @@ int test_notrunc() {
   } const U = {1 + 2 + 4 + 8}; // 0b00001111
   return U.i;
 }
-// CHECK: define dso_local i32 @test_notrunc()
+// CHECK: define{{.*}} i32 @test_notrunc()
 // CHECK: ret i32 15
 
 long long test_trunc_long_long() {
@@ -147,6 +147,6 @@ long long test_trunc_long_long() {
   } const U = {0b0100111101001101};
   return U.i;
 }
-// CHECK: define dso_local i64 @test_trunc_long_long()
+// CHECK: define{{.*}} i64 @test_trunc_long_long()
 // CHECK: ret i64 3917
 }
diff --git a/clang/test/CodeGenCXX/const-init.cpp b/clang/test/CodeGenCXX/const-init.cpp
index 5b305bc5e4d62dc785f3e1ca7696742586b67b3b..f5c9dae7ba4b6321a2b049fc173a352359f7fff0 100644
--- a/clang/test/CodeGenCXX/const-init.cpp
+++ b/clang/test/CodeGenCXX/const-init.cpp
@@ -2,17 +2,17 @@
 // RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-llvm -std=c++98 -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-llvm -std=c++11 -o - %s | FileCheck %s
 
-// CHECK: @a = dso_local global i32 10
+// CHECK: @a = global i32 10
 int a = 10;
-// CHECK: @ar = dso_local constant i32* @a
+// CHECK: @ar = constant i32* @a
 int &ar = a;
 
 void f();
-// CHECK: @fr = dso_local constant void ()* @_Z1fv
+// CHECK: @fr = constant void ()* @_Z1fv
 void (&fr)() = f;
 
 struct S { int& a; };
-// CHECK: @s = dso_local global %struct.S { i32* @a }
+// CHECK: @s = global %struct.S { i32* @a }
 S s = { a };
 
 // PR5581
@@ -23,7 +23,7 @@ public:
   unsigned f;
 };
 
-// CHECK: @_ZN6PR55812g0E = dso_local global %"class.PR5581::C" { i32 1 }
+// CHECK: @_ZN6PR55812g0E = global %"class.PR5581::C" { i32 1 }
 C g0 = { C::e1 };
 }
 
@@ -39,10 +39,10 @@ namespace test2 {
     static int g();
   } a;
 
-  // CHECK: @_ZN5test22t0E = dso_local global double {{1\.0+e\+0+}}, align 8
-  // CHECK: @_ZN5test22t1E = dso_local global [2 x double] [double {{1\.0+e\+0+}}, double {{5\.0+e-0*}}1], align 16
-  // CHECK: @_ZN5test22t2E = dso_local global double* @_ZN5test21A1d
-  // CHECK: @_ZN5test22t3E = dso_local global {{.*}} @_ZN5test21A1g
+  // CHECK: @_ZN5test22t0E = global double {{1\.0+e\+0+}}, align 8
+  // CHECK: @_ZN5test22t1E = global [2 x double] [double {{1\.0+e\+0+}}, double {{5\.0+e-0*}}1], align 16
+  // CHECK: @_ZN5test22t2E = global double* @_ZN5test21A1d
+  // CHECK: @_ZN5test22t3E = global {{.*}} @_ZN5test21A1g
   double t0 = A::d;
   double t1[] = { A::d, A::f };
   const double *t2 = &a.d;
@@ -50,7 +50,7 @@ namespace test2 {
 }
 
 // We don't expect to fold this in the frontend, but make sure it doesn't crash.
-// CHECK: @PR9558 = dso_local global float 0.000000e+0
+// CHECK: @PR9558 = global float 0.000000e+0
 float PR9558 = reinterpret_cast<const float&>("asd");
 
 // An initialized const automatic variable cannot be promoted to a constant
@@ -66,7 +66,7 @@ int writeToMutable() {
 
 // Make sure we don't try to fold this in the frontend; the backend can't
 // handle it.
-// CHECK: @PR11705 = dso_local global i128 0
+// CHECK: @PR11705 = global i128 0
 __int128_t PR11705 = (__int128_t)&PR11705;
 
 // Make sure we don't try to fold this either.
@@ -77,11 +77,11 @@ void UnfoldableAddrLabelDiff() { static __int128_t x = (long)&&a-(long)&&b; a:b:
 // CHECK: @_ZZ21FoldableAddrLabelDiffvE1x = internal global i64 sub (i64 ptrtoint (i8* blockaddress(@_Z21FoldableAddrLabelDiffv
 void FoldableAddrLabelDiff() { static long x = (long)&&a-(long)&&b; a:b:return;}
 
-// CHECK: @i = dso_local constant i32* bitcast (float* @PR9558 to i32*)
+// CHECK: @i = constant i32* bitcast (float* @PR9558 to i32*)
 int &i = reinterpret_cast<int&>(PR9558);
 
 int arr[2];
-// CHECK: @pastEnd = dso_local constant i32* bitcast (i8* getelementptr (i8, i8* bitcast ([2 x i32]* @arr to i8*), i64 8) to i32*)
+// CHECK: @pastEnd = constant i32* bitcast (i8* getelementptr (i8, i8* bitcast ([2 x i32]* @arr to i8*), i64 8) to i32*)
 int &pastEnd = arr[2];
 
 struct X {
diff --git a/clang/test/CodeGenCXX/default_calling_conv.cpp b/clang/test/CodeGenCXX/default_calling_conv.cpp
index e3d7ac429a60bfa4ec82482dac2a8eae0bbd4ed9..83d1200e0ab109f9540b1e6ca4154e99188dedbf 100644
--- a/clang/test/CodeGenCXX/default_calling_conv.cpp
+++ b/clang/test/CodeGenCXX/default_calling_conv.cpp
@@ -4,6 +4,9 @@
 // RUN: %clang_cc1 -triple i486-unknown-linux-gnu -mrtd -emit-llvm -o - %s | FileCheck %s --check-prefix=STDCALL --check-prefix=ALL
 // RUN: %clang_cc1 -triple i986-unknown-linux-gnu -fdefault-calling-conv=vectorcall -emit-llvm -o - %s | FileCheck %s --check-prefix=VECTORCALL --check-prefix=ALL
 // RUN: %clang_cc1 -triple i986-unknown-linux-gnu -fdefault-calling-conv=regcall -emit-llvm -o - %s | FileCheck %s --check-prefix=REGCALL --check-prefix=ALL
+// RUN: %clang_cc1 -triple i686-pc-win32 -fdefault-calling-conv=vectorcall -emit-llvm -o - %s -DWINDOWS | FileCheck %s --check-prefix=WIN32
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fdefault-calling-conv=vectorcall -emit-llvm -o - %s -DWINDOWS | FileCheck %s --check-prefix=WIN64
+// RUN: %clang_cc1 -triple i686-pc-win32 -emit-llvm -o - %s -DEXPLICITCC | FileCheck %s --check-prefix=EXPLICITCC
 
 // CDECL: define{{.*}} void @_Z5test1v
 // FASTCALL: define{{.*}} x86_fastcallcc void @_Z5test1v
@@ -50,3 +53,45 @@ void test() {
 int main() {
   return 1;
 }
+
+#ifdef WINDOWS
+// WIN32: define dso_local i32 @wmain
+// WIN64: define dso_local i32 @wmain
+int wmain() {
+  return 1;
+}
+// WIN32: define dso_local x86_stdcallcc i32 @WinMain
+// WIN64: define dso_local i32 @WinMain
+int WinMain() {
+  return 1;
+}
+// WIN32: define dso_local x86_stdcallcc i32 @wWinMain
+// WIN64: define dso_local i32 @wWinMain
+int wWinMain() {
+  return 1;
+}
+// WIN32: define dso_local x86_stdcallcc i32 @DllMain
+// WIN64: define dso_local i32 @DllMain
+int DllMain() {
+  return 1;
+}
+#endif // Windows
+
+#ifdef EXPLICITCC
+// EXPLICITCC: define dso_local x86_fastcallcc i32 @wmain
+int __fastcall wmain() {
+  return 1;
+}
+// EXPLICITCC: define dso_local x86_fastcallcc i32 @WinMain
+int __fastcall WinMain() {
+  return 1;
+}
+// EXPLICITCC: define dso_local x86_fastcallcc i32 @wWinMain
+int __fastcall wWinMain() {
+  return 1;
+}
+// EXPLICITCC: define dso_local x86_fastcallcc i32 @DllMain
+int __fastcall DllMain() {
+  return 1;
+}
+#endif // ExplicitCC
diff --git a/clang/test/CodeGenCXX/for-cond-var.cpp b/clang/test/CodeGenCXX/for-cond-var.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..60e54d4141f729d5a44f29379378ed3aa43339a0
--- /dev/null
+++ b/clang/test/CodeGenCXX/for-cond-var.cpp
@@ -0,0 +1,138 @@
+// RUN: %clang_cc1 -emit-llvm -o - %s -triple x86_64-linux | FileCheck %s
+
+struct A {
+  A();
+  A(const A &);
+  ~A();
+  operator bool();
+  void *data;
+};
+
+A make();
+bool cond();
+void f(int);
+
+// PR49585: Ensure that 'continue' performs the proper cleanups in the presence
+// of a for loop condition variable.
+//
+// CHECK: define {{.*}} void @_Z7PR49585v(
+void PR49585() {
+  for (
+      // CHECK: call void @_Z1fi(i32 1)
+      // CHECK: br label %[[for_cond:.*]]
+      f(1);
+
+      // CHECK: [[for_cond]]:
+      // CHECK: call {{.*}} @_Z4makev(
+      // CHECK: call {{.*}} @_ZN1AcvbEv(
+      // CHECK: br i1 {{.*}}, label %[[for_body:.*]], label %[[for_cond_cleanup:.*]]
+      A a = make();
+
+      // CHECK: [[for_cond_cleanup]]:
+      // CHECK: store
+      // CHECK: br label %[[cleanup:.*]]
+
+      f(2)) {
+    // CHECK: [[for_body]]:
+    // CHECK: call {{.*}} @_Z4condv(
+    // CHECK: br i1 {{.*}}, label %[[if_then:.*]], label %[[if_end:.*]]
+    if (cond()) {
+      // CHECK: [[if_then]]:
+      // CHECK: call {{.*}} @_Z1fi(i32 3)
+      // CHECK: br label %[[for_inc:.*]]
+      f(3);
+      continue;
+    }
+
+    // CHECK: [[if_end]]:
+    // CHECK: call {{.*}} @_Z1fi(i32 4)
+    // CHECK: br label %[[for_inc]]
+    f(4);
+  }
+
+  // CHECK: [[for_inc]]:
+  // CHECK: call void @_Z1fi(i32 2)
+  // CHECK: store
+  // CHECK: br label %[[cleanup]]
+
+  // CHECK: [[cleanup]]:
+  // CHECK: call void @_ZN1AD1Ev(
+  // CHECK: load
+  // CHECK: switch {{.*}} label
+  // CHECK-NEXT: label %[[cleanup_cont:.*]]
+  // CHECK-NEXT: label %[[for_end:.*]]
+
+  // CHECK: [[cleanup_cont]]:
+  // CHECK: br label %[[for_cond]]
+
+  // CHECK [[for_end]]:
+  // CHECK: ret void
+}
+
+// CHECK: define {{.*}} void @_Z13PR49585_breakv(
+void PR49585_break() {
+  for (
+      // CHECK: call void @_Z1fi(i32 1)
+      // CHECK: br label %[[for_cond:.*]]
+      f(1);
+
+      // CHECK: [[for_cond]]:
+      // CHECK: call {{.*}} @_Z4makev(
+      // CHECK: call {{.*}} @_ZN1AcvbEv(
+      // CHECK: br i1 {{.*}}, label %[[for_body:.*]], label %[[for_cond_cleanup:.*]]
+      A a = make();
+
+      // CHECK: [[for_cond_cleanup]]:
+      // CHECK: store
+      // CHECK: br label %[[cleanup:.*]]
+
+      f(2)) {
+    // CHECK: [[for_body]]:
+    // CHECK: call {{.*}} @_Z4condv(
+    // CHECK: br i1 {{.*}}, label %[[if_then:.*]], label %[[if_end:.*]]
+    if (cond()) {
+      // CHECK: [[if_then]]:
+      // CHECK: call {{.*}} @_Z1fi(i32 3)
+      // CHECK: store
+      // CHECK: br label %[[cleanup:.*]]
+      f(3);
+      break;
+    }
+
+    // CHECK: [[if_end]]:
+    // CHECK: call {{.*}} @_Z1fi(i32 4)
+    // CHECK: br label %[[for_inc]]
+    f(4);
+  }
+
+  // CHECK: [[for_inc]]:
+  // CHECK: call void @_Z1fi(i32 2)
+  // CHECK: store
+  // CHECK: br label %[[cleanup]]
+
+  // CHECK: [[cleanup]]:
+  // CHECK: call void @_ZN1AD1Ev(
+  // CHECK: load
+  // CHECK: switch {{.*}} label
+  // CHECK-NEXT: label %[[cleanup_cont:.*]]
+  // CHECK-NEXT: label %[[for_end:.*]]
+
+  // CHECK: [[cleanup_cont]]:
+  // CHECK: br label %[[for_cond]]
+
+  // CHECK [[for_end]]:
+  // CHECK: ret void
+}
+
+// CHECK: define {{.*}} void @_Z16incless_for_loopv(
+void incless_for_loop() {
+  // CHECK: br label %[[for_cond:.*]]
+  // CHECK: [[for_cond]]:
+  // CHECK:   br i1 {{.*}}, label %[[for_body:.*]], label %[[for_end:.*]]
+  // CHECK: [[for_body]]:
+  // CHECK:   br label %[[for_cond]]
+  // CHECK: [[for_end]]:
+  // CHECK:   ret void
+  // CHECK: }
+  for (; int b = 0;) continue;
+}
diff --git a/clang/test/CodeGenCXX/inalloca-overaligned.cpp b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
index 48a6183db8eb055c4aff205f82ce6b168554173d..0a51875bb592234db45a051ef483f869100ec853 100644
--- a/clang/test/CodeGenCXX/inalloca-overaligned.cpp
+++ b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
@@ -28,7 +28,7 @@ int receive_inalloca_overaligned(NonTrivial nt, OverAligned o) {
 }
 
 // CHECK-LABEL: define dso_local i32 @"?receive_inalloca_overaligned@@Y{{.*}}"
-// CHECK-SAME: (<{ %struct.NonTrivial, %struct.OverAligned* }>* inalloca %0)
+// CHECK-SAME: (<{ %struct.NonTrivial, %struct.OverAligned* }>* inalloca(<{ %struct.NonTrivial, %struct.OverAligned* }>) %0)
 
 int pass_inalloca_overaligned() {
   gvi32 = receive_inalloca_overaligned(NonTrivial(), OverAligned());
@@ -50,7 +50,7 @@ int pass_inalloca_overaligned() {
 // Store the address of an OverAligned temporary into the struct.
 // CHECK: getelementptr inbounds <{ %struct.NonTrivial, %struct.OverAligned* }>, <{ %struct.NonTrivial, %struct.OverAligned* }>* %{{.*}}, i32 0, i32 1
 // CHECK: store %struct.OverAligned* [[TMP]], %struct.OverAligned** %{{.*}}, align 4
-// CHECK: call i32 @"?receive_inalloca_overaligned@@Y{{.*}}"(<{ %struct.NonTrivial, %struct.OverAligned* }>* inalloca %argmem)
+// CHECK: call i32 @"?receive_inalloca_overaligned@@Y{{.*}}"(<{ %struct.NonTrivial, %struct.OverAligned* }>* inalloca(<{ %struct.NonTrivial, %struct.OverAligned* }>) %argmem)
 
 int receive_both(Both o) {
   return o.x + o.y;
@@ -74,7 +74,7 @@ int receive_inalloca_both(NonTrivial nt, Both o) {
 }
 
 // CHECK-LABEL: define dso_local i32 @"?receive_inalloca_both@@Y{{.*}}"
-// CHECK-SAME: (<{ %struct.NonTrivial, %struct.Both* }>* inalloca %0)
+// CHECK-SAME: (<{ %struct.NonTrivial, %struct.Both* }>* inalloca(<{ %struct.NonTrivial, %struct.Both* }>) %0)
 
 int pass_inalloca_both() {
   gvi32 = receive_inalloca_both(NonTrivial(), Both());
@@ -84,7 +84,7 @@ int pass_inalloca_both() {
 // CHECK-LABEL: define dso_local i32 @"?pass_inalloca_both@@Y{{.*}}"
 // CHECK: [[TMP:%[^ ]*]] = alloca %struct.Both, align 8
 // CHECK: call x86_thiscallcc %struct.Both* @"??0Both@@QAE@XZ"(%struct.Both* {{[^,]*}} [[TMP]])
-// CHECK: call i32 @"?receive_inalloca_both@@Y{{.*}}"(<{ %struct.NonTrivial, %struct.Both* }>* inalloca %argmem)
+// CHECK: call i32 @"?receive_inalloca_both@@Y{{.*}}"(<{ %struct.NonTrivial, %struct.Both* }>* inalloca(<{ %struct.NonTrivial, %struct.Both* }>) %argmem)
 
 // Here we have a type that is:
 // - overaligned
diff --git a/clang/test/CodeGenCXX/inalloca-stmtexpr.cpp b/clang/test/CodeGenCXX/inalloca-stmtexpr.cpp
index e7ae2cb4e703067275913a3b934bb6b1a2bd9e69..090953ae3b1d9656ebe5ce28b3b732942a15d698 100644
--- a/clang/test/CodeGenCXX/inalloca-stmtexpr.cpp
+++ b/clang/test/CodeGenCXX/inalloca-stmtexpr.cpp
@@ -46,6 +46,6 @@ out:;
 // CHECK: call zeroext i1 @"?cond@@YA_NXZ"()
 // CHECK: br i1
 // CHECK: br label %out
-// CHECK: call void @"?inalloca@@YAXUFoo@@0@Z"(<{ %struct.Foo, %struct.Foo }>* inalloca %{{.*}})
+// CHECK: call void @"?inalloca@@YAXUFoo@@0@Z"(<{ %struct.Foo, %struct.Foo }>* inalloca(<{ %struct.Foo, %struct.Foo }>) %{{.*}})
 // CHECK: call void @llvm.stackrestore(i8* %inalloca.save)
 // CHECK: out:
diff --git a/clang/test/CodeGenCXX/inalloca-vector.cpp b/clang/test/CodeGenCXX/inalloca-vector.cpp
index bf71fac37b6a8355321e16e0912e32a8039d5c88..e052d2e6728d677397f2283ee43e9124c3e466fd 100644
--- a/clang/test/CodeGenCXX/inalloca-vector.cpp
+++ b/clang/test/CodeGenCXX/inalloca-vector.cpp
@@ -21,7 +21,7 @@ void receive_vec_128(NonTrivial nt, __m128 x, __m128 y, __m128 z, __m128 w, __m1
 // CHECK-SAME: (<4 x float> inreg %x,
 // CHECK-SAME: <4 x float> inreg %y,
 // CHECK-SAME: <4 x float> inreg %z,
-// CHECK-SAME: <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>* inalloca %0)
+// CHECK-SAME: <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>* inalloca(<{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>) %0)
 
 void pass_vec_128() {
   __m128 z = {0};
@@ -45,7 +45,7 @@ void pass_vec_128() {
 // CHECK-SAME: (<4 x float> inreg %{{[^,]*}},
 // CHECK-SAME: <4 x float> inreg %{{[^,]*}},
 // CHECK-SAME: <4 x float> inreg %{{[^,]*}},
-// CHECK-SAME: <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>* inalloca %{{[^,]*}})
+// CHECK-SAME: <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>* inalloca(<{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>) %{{[^,]*}})
 
 // w will be passed indirectly by register, and q will be passed indirectly, but
 // the pointer will be in memory.
@@ -58,7 +58,7 @@ void __fastcall fastcall_receive_vec(__m128 x, __m128 y, __m128 z, __m128 w, int
 // CHECK-SAME: <4 x float> inreg %z,
 // CHECK-SAME: <4 x float>* inreg %0,
 // CHECK-SAME: i32 inreg %edx,
-// CHECK-SAME: <{ <4 x float>*, %struct.NonTrivial }>* inalloca %1)
+// CHECK-SAME: <{ <4 x float>*, %struct.NonTrivial }>* inalloca(<{ <4 x float>*, %struct.NonTrivial }>) %1)
 
 
 void __vectorcall vectorcall_receive_vec(double xmm0, double xmm1, double xmm2,
@@ -75,4 +75,4 @@ void __vectorcall vectorcall_receive_vec(double xmm0, double xmm1, double xmm2,
 // CHECK-SAME: <4 x float> inreg %z,
 // CHECK-SAME: <4 x float>* inreg %0,
 // CHECK-SAME: i32 inreg %edx,
-// CHECK-SAME: <{ <4 x float>*, %struct.NonTrivial }>* inalloca %1)
+// CHECK-SAME: <{ <4 x float>*, %struct.NonTrivial }>* inalloca(<{ <4 x float>*, %struct.NonTrivial }>) %1)
diff --git a/clang/test/CodeGenCXX/inheriting-constructor.cpp b/clang/test/CodeGenCXX/inheriting-constructor.cpp
index 6de8e92186dd76bdcd3ddca398609b39dc33669d..c338edcc76ae433231659a6d9648d2cccdb36fc2 100644
--- a/clang/test/CodeGenCXX/inheriting-constructor.cpp
+++ b/clang/test/CodeGenCXX/inheriting-constructor.cpp
@@ -134,7 +134,7 @@ namespace inalloca_nonvirt {
   // WIN32: store i32 2, i32* %[[ARG2]]
   // WIN32: %[[ARG4:.*]] = getelementptr {{.*}} %[[ARGMEM]]
   // WIN32: store {{.*}}* %[[TMP]], {{.*}}** %[[ARG4]]
-  // WIN32: call {{.*}} @"??0A@inalloca_nonvirt@@QAE@UQ@@H0$$QAU2@@Z"(%{{[^,]*}}, <{{.*}}>* inalloca %[[ARGMEM]])
+  // WIN32: call {{.*}} @"??0A@inalloca_nonvirt@@QAE@UQ@@H0$$QAU2@@Z"(%{{[^,]*}}, <{{.*}}>* inalloca(<{{.*}}>) %[[ARGMEM]])
   // WIN32: call void @llvm.stackrestore(
   // WIN32: call {{.*}} @"??0Z@@QAE@XZ"(
   // WIN32: call {{.*}} @"??1Q@@QAE@XZ"(
@@ -170,7 +170,7 @@ namespace inalloca_nonvirt {
   // WIN32: store i32 2, i32* %[[ARG2]]
   // WIN32: %[[ARG4:.*]] = getelementptr {{.*}} %[[ARGMEM]]
   // WIN32: store {{.*}}* %[[TMP]], {{.*}}** %[[ARG4]]
-  // WIN32: call {{.*}} @"??0A@inalloca_nonvirt@@QAE@UQ@@H0$$QAU2@@Z"(%{{[^,]*}}, <{{.*}}>* inalloca %[[ARGMEM]])
+  // WIN32: call {{.*}} @"??0A@inalloca_nonvirt@@QAE@UQ@@H0$$QAU2@@Z"(%{{[^,]*}}, <{{.*}}>* inalloca(<{{.*}}>) %[[ARGMEM]])
   // WIN32: call void @llvm.stackrestore(
   // WIN32: call {{.*}} @"??0Z@@QAE@XZ"(
   // WIN32: call {{.*}} @"??1Q@@QAE@XZ"(
@@ -216,7 +216,7 @@ namespace inalloca_virt {
   // WIN32: store i32 2, i32* %[[ARG2]]
   // WIN32: %[[ARG4:.*]] = getelementptr {{.*}} %[[ARGMEM]]
   // WIN32: store {{.*}}* %[[TMP]], {{.*}}** %[[ARG4]]
-  // WIN32: call {{.*}} @"??0A@inalloca_virt@@QAE@UQ@@H0$$QAU2@@Z"(%{{[^,]*}}, <{{.*}}>* inalloca %[[ARGMEM]])
+  // WIN32: call {{.*}} @"??0A@inalloca_virt@@QAE@UQ@@H0$$QAU2@@Z"(%{{[^,]*}}, <{{.*}}>* inalloca(<{{.*}}>) %[[ARGMEM]])
   // WIN32: call void @llvm.stackrestore(
   // WIN32: br
   //
@@ -266,7 +266,7 @@ namespace inalloca_virt {
   // WIN32: store i32 2, i32* %[[ARG2]]
   // WIN32: %[[ARG4:.*]] = getelementptr {{.*}} %[[ARGMEM]]
   // WIN32: store {{.*}}* %[[TMP]], {{.*}}** %[[ARG4]]
-  // WIN32: call {{.*}} @"??0A@inalloca_virt@@QAE@UQ@@H0$$QAU2@@Z"(%{{[^,]*}}, <{{.*}}>* inalloca %[[ARGMEM]])
+  // WIN32: call {{.*}} @"??0A@inalloca_virt@@QAE@UQ@@H0$$QAU2@@Z"(%{{[^,]*}}, <{{.*}}>* inalloca(<{{.*}}>) %[[ARGMEM]])
   // WIN32: call void @llvm.stackrestore(
   // WIN32: br
   //
diff --git a/clang/test/CodeGenCXX/linkage.cpp b/clang/test/CodeGenCXX/linkage.cpp
index d6c45cefd37832a9dbfc754e44ba182e0a5aa5c7..69b426269ccdd88cc2e6adcd95f62c73499150f8 100644
--- a/clang/test/CodeGenCXX/linkage.cpp
+++ b/clang/test/CodeGenCXX/linkage.cpp
@@ -226,5 +226,5 @@ namespace test18 {
   template<template<typename> class> struct A {};
   struct B { template<typename> struct C; };
   void f(A<B::C>) {}
-  // CHECK-DAG: define dso_local void @_ZN6test181fENS_1AINS_1B1CEEE(
+  // CHECK-DAG: define void @_ZN6test181fENS_1AINS_1B1CEEE(
 }
diff --git a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
index 4da04a43ff610329fbb3367d069ec251ce2bef84..215a39ec7d489ca16969157e7208caaa5ea1ebd8 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
@@ -14,7 +14,7 @@ void foo(A a, A b, A c) {
 // Order of destruction should be left to right.
 //
 // X86-LABEL: define dso_local void @"?foo@@YAXUA@@00@Z"
-// X86:          ([[argmem_ty:<{ %struct.A, %struct.A, %struct.A }>]]* inalloca %0)
+// X86:          ([[argmem_ty:<{ %struct.A, %struct.A, %struct.A }>]]* inalloca([[argmem_ty]]) %0)
 // X86: %[[a:[^ ]*]] = getelementptr inbounds [[argmem_ty]], [[argmem_ty]]* %0, i32 0, i32 0
 // X86: %[[b:[^ ]*]] = getelementptr inbounds [[argmem_ty]], [[argmem_ty]]* %0, i32 0, i32 1
 // X86: %[[c:[^ ]*]] = getelementptr inbounds [[argmem_ty]], [[argmem_ty]]* %0, i32 0, i32 2
@@ -47,7 +47,7 @@ void call_foo() {
 // X86: invoke x86_thiscallcc %struct.A* @"??0A@@QAE@H@Z"(%struct.A* {{[^,]*}} %[[arg2]], i32 2)
 // X86: %[[arg1:[^ ]*]] = getelementptr inbounds [[argmem_ty]], [[argmem_ty]]* %[[argmem]], i32 0, i32 0
 // X86: invoke x86_thiscallcc %struct.A* @"??0A@@QAE@H@Z"(%struct.A* {{[^,]*}} %[[arg1]], i32 1)
-// X86: call void @"?foo@@YAXUA@@00@Z"([[argmem_ty]]* inalloca %[[argmem]])
+// X86: call void @"?foo@@YAXUA@@00@Z"([[argmem_ty]]* inalloca([[argmem_ty]]) %[[argmem]])
 // X86: call void @llvm.stackrestore
 // X86: ret void
 //
diff --git a/clang/test/CodeGenCXX/microsoft-abi-byval-sret.cpp b/clang/test/CodeGenCXX/microsoft-abi-byval-sret.cpp
index 7f8730080a099039dc18904b9e6485233f58a9a4..adf3921f71156984b9f4c722202e6411e397ce92 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-byval-sret.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-byval-sret.cpp
@@ -19,7 +19,7 @@ A B::foo(A x) {
 }
 
 // CHECK-LABEL: define dso_local x86_thiscallcc %struct.A* @"?foo@B@@QAE?AUA@@U2@@Z"
-// CHECK:       (%struct.B* %this, <{ %struct.A*, %struct.A }>* inalloca %0)
+// CHECK:       (%struct.B* %this, <{ %struct.A*, %struct.A }>* inalloca(<{ %struct.A*, %struct.A }>) %0)
 // CHECK:   getelementptr inbounds <{ %struct.A*, %struct.A }>, <{ %struct.A*, %struct.A }>* %{{.*}}, i32 0, i32 0
 // CHECK:   load %struct.A*, %struct.A**
 // CHECK:   ret %struct.A*
@@ -29,7 +29,7 @@ A B::bar(A x) {
 }
 
 // CHECK-LABEL: define dso_local %struct.A* @"?bar@B@@QAA?AUA@@U2@@Z"
-// CHECK:       (<{ %struct.B*, %struct.A*, %struct.A }>* inalloca %0)
+// CHECK:       (<{ %struct.B*, %struct.A*, %struct.A }>* inalloca(<{ %struct.B*, %struct.A*, %struct.A }>) %0)
 // CHECK:   getelementptr inbounds <{ %struct.B*, %struct.A*, %struct.A }>, <{ %struct.B*, %struct.A*, %struct.A }>* %{{.*}}, i32 0, i32 1
 // CHECK:   load %struct.A*, %struct.A**
 // CHECK:   ret %struct.A*
@@ -39,7 +39,7 @@ A B::baz(A x) {
 }
 
 // CHECK-LABEL: define dso_local x86_stdcallcc %struct.A* @"?baz@B@@QAG?AUA@@U2@@Z"
-// CHECK:       (<{ %struct.B*, %struct.A*, %struct.A }>* inalloca %0)
+// CHECK:       (<{ %struct.B*, %struct.A*, %struct.A }>* inalloca(<{ %struct.B*, %struct.A*, %struct.A }>) %0)
 // CHECK:   getelementptr inbounds <{ %struct.B*, %struct.A*, %struct.A }>, <{ %struct.B*, %struct.A*, %struct.A }>* %{{.*}}, i32 0, i32 1
 // CHECK:   load %struct.A*, %struct.A**
 // CHECK:   ret %struct.A*
@@ -49,7 +49,7 @@ A B::qux(A x) {
 }
 
 // CHECK-LABEL: define dso_local x86_fastcallcc void @"?qux@B@@QAI?AUA@@U2@@Z"
-// CHECK:       (%struct.B* inreg %this, %struct.A* inreg noalias sret(%struct.A) align 4 %agg.result, <{ %struct.A }>* inalloca %0)
+// CHECK:       (%struct.B* inreg %this, %struct.A* inreg noalias sret(%struct.A) align 4 %agg.result, <{ %struct.A }>* inalloca(<{ %struct.A }>) %0)
 // CHECK:   ret void
 
 int main() {
@@ -61,10 +61,10 @@ int main() {
 }
 
 // CHECK: call x86_thiscallcc %struct.A* @"?foo@B@@QAE?AUA@@U2@@Z"
-// CHECK:       (%struct.B* %{{[^,]*}}, <{ %struct.A*, %struct.A }>* inalloca %{{[^,]*}})
+// CHECK:       (%struct.B* %{{[^,]*}}, <{ %struct.A*, %struct.A }>* inalloca(<{ %struct.A*, %struct.A }>) %{{[^,]*}})
 // CHECK: call %struct.A* @"?bar@B@@QAA?AUA@@U2@@Z"
-// CHECK:       (<{ %struct.B*, %struct.A*, %struct.A }>* inalloca %{{[^,]*}})
+// CHECK:       (<{ %struct.B*, %struct.A*, %struct.A }>* inalloca(<{ %struct.B*, %struct.A*, %struct.A }>) %{{[^,]*}})
 // CHECK: call x86_stdcallcc %struct.A* @"?baz@B@@QAG?AUA@@U2@@Z"
-// CHECK:       (<{ %struct.B*, %struct.A*, %struct.A }>* inalloca %{{[^,]*}})
+// CHECK:       (<{ %struct.B*, %struct.A*, %struct.A }>* inalloca(<{ %struct.B*, %struct.A*, %struct.A }>) %{{[^,]*}})
 // CHECK: call x86_fastcallcc void @"?qux@B@@QAI?AUA@@U2@@Z"
-// CHECK:       (%struct.B* inreg %{{[^,]*}}, %struct.A* inreg sret(%struct.A) align 4 %{{.*}}, <{ %struct.A }>* inalloca %{{[^,]*}})
+// CHECK:       (%struct.B* inreg %{{[^,]*}}, %struct.A* inreg sret(%struct.A) align 4 %{{.*}}, <{ %struct.A }>* inalloca(<{ %struct.A }>) %{{[^,]*}})
diff --git a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
index 917a7677c41e45b6a7df7e379636e5e5c8407363..65e789ce5c637ddab262d106e44110c1c9517d66 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
@@ -15,10 +15,10 @@ struct C : A, B { C(); virtual void foo(Agg x); };
 C::C() {} // force emission
 
 // CHECK32-LABEL: define linkonce_odr dso_local x86_thiscallcc void @"?foo@C@byval_thunk@@W3AEXUAgg@2@@Z"
-// CHECK32:             (%"struct.byval_thunk::C"* %this, <{ %"struct.byval_thunk::Agg" }>* inalloca %0)
+// CHECK32:             (%"struct.byval_thunk::C"* %this, <{ %"struct.byval_thunk::Agg" }>* inalloca(<{ %"struct.byval_thunk::Agg" }>) %0)
 // CHECK32:   getelementptr i8, i8* %{{.*}}, i32 -4
 // CHECK32:   musttail call x86_thiscallcc void @"?foo@C@byval_thunk@@UAEXUAgg@2@@Z"
-// CHECK32:       (%"struct.byval_thunk::C"* %{{.*}}, <{ %"struct.byval_thunk::Agg" }>* inalloca %0)
+// CHECK32:       (%"struct.byval_thunk::C"* %{{.*}}, <{ %"struct.byval_thunk::Agg" }>* inalloca(<{ %"struct.byval_thunk::Agg" }>) %0)
 // CHECK32-NEXT: ret void
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo@C@byval_thunk@@W7EAAXUAgg@2@@Z"
@@ -44,13 +44,13 @@ struct C : A, B { C(); virtual void __stdcall foo(Agg x); };
 C::C() {} // force emission
 
 // CHECK32-LABEL: define linkonce_odr dso_local x86_stdcallcc void @"?foo@C@stdcall_thunk@@W3AGXUAgg@2@@Z"
-// CHECK32:             (<{ %"struct.stdcall_thunk::C"*, %"struct.stdcall_thunk::Agg" }>* inalloca %0)
+// CHECK32:             (<{ %"struct.stdcall_thunk::C"*, %"struct.stdcall_thunk::Agg" }>* inalloca(<{ %"struct.stdcall_thunk::C"*, %"struct.stdcall_thunk::Agg" }>) %0)
 // CHECK32:   %[[this_slot:[^ ]*]] = getelementptr inbounds <{ %"struct.stdcall_thunk::C"*, %"struct.stdcall_thunk::Agg" }>, <{ %"struct.stdcall_thunk::C"*, %"struct.stdcall_thunk::Agg" }>* %0, i32 0, i32 0
 // CHECK32:   load %"struct.stdcall_thunk::C"*, %"struct.stdcall_thunk::C"** %[[this_slot]]
 // CHECK32:   getelementptr i8, i8* %{{.*}}, i32 -4
 // CHECK32:   store %"struct.stdcall_thunk::C"* %{{.*}}, %"struct.stdcall_thunk::C"** %[[this_slot]]
 // CHECK32:   musttail call x86_stdcallcc void @"?foo@C@stdcall_thunk@@UAGXUAgg@2@@Z"
-// CHECK32:       (<{ %"struct.stdcall_thunk::C"*, %"struct.stdcall_thunk::Agg" }>*  inalloca %0)
+// CHECK32:       (<{ %"struct.stdcall_thunk::C"*, %"struct.stdcall_thunk::Agg" }>*  inalloca(<{ %"struct.stdcall_thunk::C"*, %"struct.stdcall_thunk::Agg" }>) %0)
 // CHECK32-NEXT: ret void
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo@C@stdcall_thunk@@W7EAAXUAgg@2@@Z"
@@ -76,13 +76,13 @@ struct C : A, B { C(); virtual Agg __cdecl foo(Agg x); };
 C::C() {} // force emission
 
 // CHECK32-LABEL: define linkonce_odr dso_local %"struct.sret_thunk::Agg"* @"?foo@C@sret_thunk@@W3AA?AUAgg@2@U32@@Z"
-// CHECK32:             (<{ %"struct.sret_thunk::C"*, %"struct.sret_thunk::Agg"*, %"struct.sret_thunk::Agg" }>* inalloca %0)
+// CHECK32:             (<{ %"struct.sret_thunk::C"*, %"struct.sret_thunk::Agg"*, %"struct.sret_thunk::Agg" }>* inalloca(<{ %"struct.sret_thunk::C"*, %"struct.sret_thunk::Agg"*, %"struct.sret_thunk::Agg" }>) %0)
 // CHECK32:   %[[this_slot:[^ ]*]] = getelementptr inbounds <{ %"struct.sret_thunk::C"*, %"struct.sret_thunk::Agg"*, %"struct.sret_thunk::Agg" }>, <{ %"struct.sret_thunk::C"*, %"struct.sret_thunk::Agg"*, %"struct.sret_thunk::Agg" }>* %0, i32 0, i32 0
 // CHECK32:   load %"struct.sret_thunk::C"*, %"struct.sret_thunk::C"** %[[this_slot]]
 // CHECK32:   getelementptr i8, i8* %{{.*}}, i32 -4
 // CHECK32:   store %"struct.sret_thunk::C"* %{{.*}}, %"struct.sret_thunk::C"** %[[this_slot]]
 // CHECK32:   %[[rv:[^ ]*]] = musttail call %"struct.sret_thunk::Agg"* @"?foo@C@sret_thunk@@UAA?AUAgg@2@U32@@Z"
-// CHECK32:       (<{ %"struct.sret_thunk::C"*, %"struct.sret_thunk::Agg"*, %"struct.sret_thunk::Agg" }>*  inalloca %0)
+// CHECK32:       (<{ %"struct.sret_thunk::C"*, %"struct.sret_thunk::Agg"*, %"struct.sret_thunk::Agg" }>*  inalloca(<{ %"struct.sret_thunk::C"*, %"struct.sret_thunk::Agg"*, %"struct.sret_thunk::Agg" }>) %0)
 // CHECK32-NEXT: ret %"struct.sret_thunk::Agg"* %[[rv]]
 
 // CHECK64-LABEL: define linkonce_odr dso_local void @"?foo@C@sret_thunk@@W7EAA?AUAgg@2@U32@@Z"
diff --git a/clang/test/CodeGenCXX/microsoft-abi-byval-vararg.cpp b/clang/test/CodeGenCXX/microsoft-abi-byval-vararg.cpp
index 26f6814cc1d4d959d829d69a52dc28e748181c4b..18333f36c239d4f5950ce5158e0d570d60d4ad80 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-byval-vararg.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-byval-vararg.cpp
@@ -19,14 +19,14 @@ int foo(A a, ...) {
   return sum;
 }
 
-// CHECK-LABEL: define dso_local i32 @"?foo@@YAHUA@@ZZ"(<{ %struct.A }>* inalloca %0, ...)
+// CHECK-LABEL: define dso_local i32 @"?foo@@YAHUA@@ZZ"(<{ %struct.A }>* inalloca(<{ %struct.A }>) %0, ...)
 
 int main() {
   return foo(A(3), 1, 2, 3);
 }
 // CHECK-LABEL: define dso_local i32 @main()
 // CHECK: %[[argmem:[^ ]*]] = alloca inalloca <{ %struct.A, i32, i32, i32 }>
-// CHECK: call i32 {{.*bitcast.*}}@"?foo@@YAHUA@@ZZ"{{.*}}(<{ %struct.A, i32, i32, i32 }>* inalloca %[[argmem]])
+// CHECK: call i32 {{.*bitcast.*}}@"?foo@@YAHUA@@ZZ"{{.*}}(<{ %struct.A, i32, i32, i32 }>* inalloca(<{ %struct.A, i32, i32, i32 }>) %[[argmem]])
 
 void varargs_zero(...);
 void varargs_one(int, ...);
@@ -41,10 +41,10 @@ void call_var_args() {
 }
 
 // CHECK-LABEL: define dso_local void @"?call_var_args@@YAXXZ"()
-// CHECK: call void {{.*bitcast.*varargs_zero.*}}(<{ %struct.A }>* inalloca %{{.*}})
-// CHECK: call void {{.*bitcast.*varargs_one.*}}(<{ i32, %struct.A }>* inalloca %{{.*}})
-// CHECK: call void {{.*bitcast.*varargs_two.*}}(<{ i32, i32, %struct.A }>* inalloca %{{.*}})
-// CHECK: call void {{.*bitcast.*varargs_three.*}}(<{ i32, i32, i32, %struct.A }>* inalloca %{{.*}})
+// CHECK: call void {{.*bitcast.*varargs_zero.*}}(<{ %struct.A }>* inalloca(<{ %struct.A }>) %{{.*}})
+// CHECK: call void {{.*bitcast.*varargs_one.*}}(<{ i32, %struct.A }>* inalloca(<{ i32, %struct.A }>) %{{.*}})
+// CHECK: call void {{.*bitcast.*varargs_two.*}}(<{ i32, i32, %struct.A }>* inalloca(<{ i32, i32, %struct.A }>) %{{.*}})
+// CHECK: call void {{.*bitcast.*varargs_three.*}}(<{ i32, i32, i32, %struct.A }>* inalloca(<{ i32, i32, i32, %struct.A }>) %{{.*}})
 
 // CHECK-LABEL: declare dso_local void @"?varargs_zero@@YAXZZ"(...)
 // CHECK-LABEL: declare dso_local void @"?varargs_one@@YAXHZZ"(i32, ...)
diff --git a/clang/test/CodeGenCXX/microsoft-abi-eh-cleanups.cpp b/clang/test/CodeGenCXX/microsoft-abi-eh-cleanups.cpp
index 7e173668f26ff3c7e6fd967f0e923c9a3035ccf9..0b6b4385a35217ea1a84d39039c2bd9d3426ed32 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-eh-cleanups.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-eh-cleanups.cpp
@@ -54,7 +54,7 @@ int HasDeactivatedCleanups() {
 // WIN32:   invoke x86_thiscallcc %struct.A* @"??0A@@QAE@XZ"
 // WIN32:   store i1 false, i1* %[[isactive]]
 //
-// WIN32:   invoke i32 @"?TakesTwo@@YAHUA@@0@Z"([[argmem_ty]]* inalloca %[[argmem]])
+// WIN32:   invoke i32 @"?TakesTwo@@YAHUA@@0@Z"([[argmem_ty]]* inalloca([[argmem_ty]]) %[[argmem]])
 //        Destroy the two const ref temporaries.
 // WIN32:   call x86_thiscallcc void @"??1A@@QAE@XZ"({{.*}})
 // WIN32:   call x86_thiscallcc void @"??1A@@QAE@XZ"({{.*}})
diff --git a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
index e0e4ba9e41b5f41aae529b914399e2ac51ff7cde..b36ea9ccd9f015d2fb057115ff20c78c415ccf21 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
@@ -76,7 +76,7 @@ private:
 
 // WIN32: declare dso_local void @"{{.*take_bools_and_chars.*}}"
 // WIN32:       (<{ i8, [3 x i8], i8, [3 x i8], %struct.SmallWithDtor,
-// WIN32:           i8, [3 x i8], i8, [3 x i8], i32, i8, [3 x i8] }>* inalloca)
+// WIN32:           i8, [3 x i8], i8, [3 x i8], i32, i8, [3 x i8] }>* inalloca(<{ i8, [3 x i8], i8, [3 x i8], %struct.SmallWithDtor, i8, [3 x i8], i8, [3 x i8], i32, i8, [3 x i8] }>)
 void take_bools_and_chars(char a, char b, SmallWithDtor c, char d, bool e, int f, bool g);
 void call_bools_and_chars() {
   take_bools_and_chars('A', 'B', SmallWithDtor(), 'D', true, 13, false);
@@ -176,7 +176,7 @@ void packed_arg(Packed s) {}
 
 // Test that dtors are invoked in the callee.
 void small_arg_with_dtor(SmallWithDtor s) {}
-// WIN32: define dso_local void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(<{ %struct.SmallWithDtor }>* inalloca %0) {{.*}} {
+// WIN32: define dso_local void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(<{ %struct.SmallWithDtor }>* inalloca(<{ %struct.SmallWithDtor }>) %0) {{.*}} {
 // WIN32:   call x86_thiscallcc void @"??1SmallWithDtor@@QAE@XZ"
 // WIN32: }
 // WIN64: define dso_local void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(i32 %s.coerce) {{.*}} {
@@ -253,13 +253,13 @@ void eh_cleanup_arg_with_dtor() {
 
 void small_arg_with_vftable(SmallWithVftable s) {}
 // LINUX-LABEL: define{{.*}} void @_Z22small_arg_with_vftable16SmallWithVftable(%struct.SmallWithVftable* %s)
-// WIN32: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(<{ %struct.SmallWithVftable }>* inalloca %0)
+// WIN32: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(<{ %struct.SmallWithVftable }>* inalloca(<{ %struct.SmallWithVftable }>) %0)
 // WIN64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(%struct.SmallWithVftable* %s)
 // WOA64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(%struct.SmallWithVftable* %s)
 
 void medium_arg_with_copy_ctor(MediumWithCopyCtor s) {}
 // LINUX-LABEL: define{{.*}} void @_Z25medium_arg_with_copy_ctor18MediumWithCopyCtor(%struct.MediumWithCopyCtor* %s)
-// WIN32: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(<{ %struct.MediumWithCopyCtor }>* inalloca %0)
+// WIN32: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(<{ %struct.MediumWithCopyCtor }>* inalloca(<{ %struct.MediumWithCopyCtor }>) %0)
 // WIN64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(%struct.MediumWithCopyCtor* %s)
 // WOA: define dso_local arm_aapcs_vfpcc void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(%struct.MediumWithCopyCtor* %s)
 // WOA64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(%struct.MediumWithCopyCtor* %s)
@@ -363,7 +363,7 @@ struct X {
 };
 void g(X) {
 }
-// WIN32: define dso_local void @"?g@@YAXUX@@@Z"(<{ %struct.X, [3 x i8] }>* inalloca %0) {{.*}} {
+// WIN32: define dso_local void @"?g@@YAXUX@@@Z"(<{ %struct.X, [3 x i8] }>* inalloca(<{ %struct.X, [3 x i8] }>) %0) {{.*}} {
 // WIN32:   call x86_thiscallcc void @"??1X@@QAE@XZ"(%struct.X* {{.*}})
 // WIN32: }
 void f() {
@@ -398,7 +398,7 @@ void bar() {
 // WIN32:   call void @llvm.memcpy
 // WIN32:   getelementptr inbounds [[argmem_ty]], [[argmem_ty]]* %[[argmem]], i32 0, i32 0
 // WIN32:   call x86_thiscallcc %"struct.test2::NonTrivial"* @"??0NonTrivial@test2@@QAE@XZ"
-// WIN32:   call i32 @"?foo@test2@@YAHUNonTrivial@1@UPOD@1@@Z"([[argmem_ty]]* inalloca %argmem)
+// WIN32:   call i32 @"?foo@test2@@YAHUNonTrivial@1@UPOD@1@@Z"([[argmem_ty]]* inalloca([[argmem_ty]]) %argmem)
 // WIN32:   ret void
 // WIN32: }
 
@@ -414,7 +414,7 @@ struct NonTrivial {
   int a;
 };
 void foo(NonTrivial a, bool b) { }
-// WIN32-LABEL: define dso_local void @"?foo@test3@@YAXUNonTrivial@1@_N@Z"(<{ %"struct.test3::NonTrivial", i8, [3 x i8] }>* inalloca %0)
+// WIN32-LABEL: define dso_local void @"?foo@test3@@YAXUNonTrivial@1@_N@Z"(<{ %"struct.test3::NonTrivial", i8, [3 x i8] }>* inalloca(<{ %"struct.test3::NonTrivial", i8, [3 x i8] }>) %0)
 
 }
 
@@ -440,7 +440,7 @@ void fn2(FnPtr1 a, SmallWithDtor b) { fn1(a, b); };
 // WIN32:   %[[gep2:[^ ]*]] = getelementptr inbounds [[argmem_ty]], [[argmem_ty]]* %[[argmem]], i32 0, i32 0
 // WIN32:   %[[addr:[^ ]*]] = bitcast {}** %[[gep2]] to void [[dst_ty]]*
 // WIN32:   store void [[dst_ty]] %[[a2]], void [[dst_ty]]* %[[addr]], align 4
-// WIN32:   call void @"?fn1@@YAXP6AXUForwardDeclare1@@@ZUSmallWithDtor@@@Z"([[argmem_ty]]* inalloca %[[argmem]])
+// WIN32:   call void @"?fn1@@YAXP6AXUForwardDeclare1@@@ZUSmallWithDtor@@@Z"([[argmem_ty]]* inalloca([[argmem_ty]]) %[[argmem]])
 
 namespace pr30293 {
 // Virtual methods living in a secondary vtable take i8* as their 'this'
@@ -462,8 +462,8 @@ void C::g() { return h(SmallWithDtor()); }
 
 // WIN32-LABEL: define dso_local x86_thiscallcc void @"?g@C@pr30293@@QAEXXZ"(%"struct.pr30293::C"* {{[^,]*}} %this)
 // WIN32: call x86_thiscallcc %struct.SmallWithDtor* @"??0SmallWithDtor@@QAE@XZ"
-// WIN32: call void @"?h@C@pr30293@@UAAXUSmallWithDtor@@@Z"(<{ i8*, %struct.SmallWithDtor }>* inalloca %{{[^,)]*}})
-// WIN32: declare dso_local void @"?h@C@pr30293@@UAAXUSmallWithDtor@@@Z"(<{ i8*, %struct.SmallWithDtor }>* inalloca)
+// WIN32: call void @"?h@C@pr30293@@UAAXUSmallWithDtor@@@Z"(<{ i8*, %struct.SmallWithDtor }>* inalloca(<{ i8*, %struct.SmallWithDtor }>) %{{[^,)]*}})
+// WIN32: declare dso_local void @"?h@C@pr30293@@UAAXUSmallWithDtor@@@Z"(<{ i8*, %struct.SmallWithDtor }>* inalloca(<{ i8*, %struct.SmallWithDtor }>))
 
 // WIN64-LABEL: define dso_local void @"?g@C@pr30293@@QEAAXXZ"(%"struct.pr30293::C"* {{[^,]*}} %this)
 // WIN64: declare dso_local void @"?h@C@pr30293@@UEAAXUSmallWithDtor@@@Z"(i8*, i32)
diff --git a/clang/test/CodeGenCXX/microsoft-abi-vmemptr-conflicts.cpp b/clang/test/CodeGenCXX/microsoft-abi-vmemptr-conflicts.cpp
index 6082228d36b6e9b3be4541dbb6549919c5745eff..e71d6238c53add791bc6332c0552a2a5c348990b 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-vmemptr-conflicts.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-vmemptr-conflicts.cpp
@@ -94,7 +94,7 @@ void f(C *c) {
 
 // CHECK-LABEL: define dso_local void @"?f@cdecl_inalloca@@YAXPAUC@1@@Z"(%"struct.cdecl_inalloca::C"* %c)
 // CHECK: call void bitcast (void (%"struct.cdecl_inalloca::C"*, ...)* @"??_9C@cdecl_inalloca@@$BA@AA" to void (%"struct.cdecl_inalloca::C"*)*)(%"struct.cdecl_inalloca::C"* {{[^,]*}} %{{.*}})
-// CHECK: call void bitcast (void (%"struct.cdecl_inalloca::C"*, ...)* @"??_9C@cdecl_inalloca@@$BA@AA" to void (<{ %"struct.cdecl_inalloca::C"*, %"struct.cdecl_inalloca::Big" }>*)*)(<{ %"struct.cdecl_inalloca::C"*, %"struct.cdecl_inalloca::Big" }>* inalloca %{{.*}})
+// CHECK: call void bitcast (void (%"struct.cdecl_inalloca::C"*, ...)* @"??_9C@cdecl_inalloca@@$BA@AA" to void (<{ %"struct.cdecl_inalloca::C"*, %"struct.cdecl_inalloca::Big" }>*)*)(<{ %"struct.cdecl_inalloca::C"*, %"struct.cdecl_inalloca::Big" }>* inalloca(<{ %"struct.cdecl_inalloca::C"*, %"struct.cdecl_inalloca::Big" }>) %{{.*}})
 
 // CHECK-LABEL: define linkonce_odr void @"??_9C@cdecl_inalloca@@$BA@AA"(%"struct.cdecl_inalloca::C"* %this, ...) {{.*}} comdat
 // CHECK: musttail call void (%"struct.cdecl_inalloca::C"*, ...) %{{.*}}(%"struct.cdecl_inalloca::C"* %{{.*}}, ...)
diff --git a/clang/test/CodeGenCXX/microsoft-abi-vtables-multiple-nonvirtual-inheritance-this-adjustment.cpp b/clang/test/CodeGenCXX/microsoft-abi-vtables-multiple-nonvirtual-inheritance-this-adjustment.cpp
index 93a7d4602223d886b8809a1a53eea2b07bec5825..5cced42834e1b177f2ea1b645063550cc3f9f9ec 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-vtables-multiple-nonvirtual-inheritance-this-adjustment.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-vtables-multiple-nonvirtual-inheritance-this-adjustment.cpp
@@ -189,7 +189,7 @@ void C::g(NonTrivial o) {
   whatsthis = this;
 }
 
-// BITCODE-LABEL: define dso_local void @"?g@C@pr30293@@UAAXUNonTrivial@2@@Z"(<{ i8*, %"struct.pr30293::NonTrivial" }>* inalloca %0)
+// BITCODE-LABEL: define dso_local void @"?g@C@pr30293@@UAAXUNonTrivial@2@@Z"(<{ i8*, %"struct.pr30293::NonTrivial" }>* inalloca(<{ i8*, %"struct.pr30293::NonTrivial" }>) %0)
 // BITCODE: %[[thisaddr:[^ ]*]] = getelementptr inbounds <{ i8*, %"struct.pr30293::NonTrivial" }>, <{ i8*, %"struct.pr30293::NonTrivial" }>* {{.*}}, i32 0, i32 0
 // BITCODE: %[[thisaddr1:[^ ]*]] = bitcast i8** %[[thisaddr]] to %"struct.pr30293::C"**
 // BITCODE: %[[this1:[^ ]*]] = load %"struct.pr30293::C"*, %"struct.pr30293::C"** %[[thisaddr1]], align 4
diff --git a/clang/test/CodeGenCXX/ms-thunks-ehspec.cpp b/clang/test/CodeGenCXX/ms-thunks-ehspec.cpp
index 256f7123ee517df16237ad10afe0fcbeb0e99a82..b8ebe2dd9f39996f2f54e4dbe9a94075e456b572 100644
--- a/clang/test/CodeGenCXX/ms-thunks-ehspec.cpp
+++ b/clang/test/CodeGenCXX/ms-thunks-ehspec.cpp
@@ -20,8 +20,8 @@ class C : A, B {
 };
 C c;
 
-// CHECK-LABEL: define linkonce_odr dso_local x86_thiscallcc void @"?f@C@@G3AEXUNonTrivial@@@Z"(%class.C* %this, <{ %struct.NonTrivial }>* inalloca %0)
+// CHECK-LABEL: define linkonce_odr dso_local x86_thiscallcc void @"?f@C@@G3AEXUNonTrivial@@@Z"(%class.C* %this, <{ %struct.NonTrivial }>* inalloca(<{ %struct.NonTrivial }>) %0)
 // CHECK-NOT: invoke
-// CHECK: musttail call x86_thiscallcc void @"?f@C@@EAEXUNonTrivial@@@Z"(%class.C* %{{.*}}, <{ %struct.NonTrivial }>* inalloca %0)
+// CHECK: musttail call x86_thiscallcc void @"?f@C@@EAEXUNonTrivial@@@Z"(%class.C* %{{.*}}, <{ %struct.NonTrivial }>* inalloca(<{ %struct.NonTrivial }>) %0)
 // CHECK-NEXT:  ret void
 
diff --git a/clang/test/CodeGenCXX/temporaries.cpp b/clang/test/CodeGenCXX/temporaries.cpp
index 3ce350d03f4827f90e696b8022407c974faacb3b..b769411c9518748e626cf09dbbc13f76c45acb6f 100644
--- a/clang/test/CodeGenCXX/temporaries.cpp
+++ b/clang/test/CodeGenCXX/temporaries.cpp
@@ -60,7 +60,7 @@ namespace RefTempSubobject {
   };
 
   // CHECK: @_ZGRN16RefTempSubobject2srE_ = internal global { i32*, [3 x i32] } { {{.*}} getelementptr {{.*}} @_ZGRN16RefTempSubobject2srE_ {{.*}}, [3 x i32] [i32 1, i32 2, i32 3] }
-  // CHECK: @_ZN16RefTempSubobject2srE = {{.*}} constant {{.*}} @_ZGRN16RefTempSubobject2srE_
+  // CHECK: @_ZN16RefTempSubobject2srE = constant {{.*}} @_ZGRN16RefTempSubobject2srE_
   constexpr const SelfReferential &sr = SelfReferential();
 }
 
diff --git a/clang/test/CodeGenCXX/type_visibility.cpp b/clang/test/CodeGenCXX/type_visibility.cpp
index 0096525cd7726ff9ac7615879bc574e7d65f4173..a7b7198a23fab0930c7756b2f5415c6a2352c8e7 100644
--- a/clang/test/CodeGenCXX/type_visibility.cpp
+++ b/clang/test/CodeGenCXX/type_visibility.cpp
@@ -110,14 +110,14 @@ namespace type0 {
   };
 
   void A::foo() {}
-  // FUNS-LABEL:        define{{.*}} void @_ZN5type01A3fooEv(
-  // VARS:        @_ZTVN5type01AE = dso_local unnamed_addr constant
-  // VARS:        @_ZTSN5type01AE = dso_local constant
-  // VARS:        @_ZTIN5type01AE = dso_local constant
+  // FUNS-LABEL:        define void @_ZN5type01A3fooEv(
+  // VARS:        @_ZTVN5type01AE = unnamed_addr constant
+  // VARS:        @_ZTSN5type01AE = constant
+  // VARS:        @_ZTIN5type01AE = constant
   // FUNS-HIDDEN-LABEL: define hidden void @_ZN5type01A3fooEv(
-  // VARS-HIDDEN: @_ZTVN5type01AE = dso_local unnamed_addr constant
-  // VARS-HIDDEN: @_ZTSN5type01AE = dso_local constant
-  // VARS-HIDDEN: @_ZTIN5type01AE = dso_local constant
+  // VARS-HIDDEN: @_ZTVN5type01AE = unnamed_addr constant
+  // VARS-HIDDEN: @_ZTSN5type01AE = constant
+  // VARS-HIDDEN: @_ZTIN5type01AE = constant
 }
 
 namespace type1 {
@@ -127,13 +127,13 @@ namespace type1 {
 
   void A::foo() {}
   // FUNS-LABEL:        define hidden void @_ZN5type11A3fooEv(
-  // VARS:        @_ZTVN5type11AE = dso_local unnamed_addr constant
-  // VARS:        @_ZTSN5type11AE = dso_local constant
-  // VARS:        @_ZTIN5type11AE = dso_local constant
+  // VARS:        @_ZTVN5type11AE = unnamed_addr constant
+  // VARS:        @_ZTSN5type11AE = constant
+  // VARS:        @_ZTIN5type11AE = constant
   // FUNS-HIDDEN-LABEL: define hidden void @_ZN5type11A3fooEv(
-  // VARS-HIDDEN: @_ZTVN5type11AE = dso_local unnamed_addr constant
-  // VARS-HIDDEN: @_ZTSN5type11AE = dso_local constant
-  // VARS-HIDDEN: @_ZTIN5type11AE = dso_local constant
+  // VARS-HIDDEN: @_ZTVN5type11AE = unnamed_addr constant
+  // VARS-HIDDEN: @_ZTSN5type11AE = constant
+  // VARS-HIDDEN: @_ZTIN5type11AE = constant
 }
 
 namespace type2 {
@@ -142,7 +142,7 @@ namespace type2 {
   };
 
   void A::foo() {}
-  // FUNS-LABEL:        define dso_local void @_ZN5type21A3fooEv(
+  // FUNS-LABEL:        define void @_ZN5type21A3fooEv(
   // VARS:        @_ZTVN5type21AE = hidden unnamed_addr constant
   // VARS:        @_ZTSN5type21AE = hidden constant
   // VARS:        @_ZTIN5type21AE = hidden constant
@@ -158,11 +158,11 @@ namespace type3 {
   };
 
   void A::foo() {}
-  // FUNS-LABEL:        define dso_local void @_ZN5type31A3fooEv(
+  // FUNS-LABEL:        define void @_ZN5type31A3fooEv(
   // VARS:        @_ZTVN5type31AE = hidden unnamed_addr constant
   // VARS:        @_ZTSN5type31AE = hidden constant
   // VARS:        @_ZTIN5type31AE = hidden constant
-  // FUNS-HIDDEN-LABEL: define dso_local void @_ZN5type31A3fooEv(
+  // FUNS-HIDDEN-LABEL: define void @_ZN5type31A3fooEv(
   // VARS-HIDDEN: @_ZTVN5type31AE = hidden unnamed_addr constant
   // VARS-HIDDEN: @_ZTSN5type31AE = hidden constant
   // VARS-HIDDEN: @_ZTIN5type31AE = hidden constant
diff --git a/clang/test/CodeGenCXX/vararg-non-pod-ms-compat.cpp b/clang/test/CodeGenCXX/vararg-non-pod-ms-compat.cpp
index 8f413021b3d0ca2fbcb18062857e1eb143b9adbb..dd1c88a653343d06adeaead185c8467461c685f0 100644
--- a/clang/test/CodeGenCXX/vararg-non-pod-ms-compat.cpp
+++ b/clang/test/CodeGenCXX/vararg-non-pod-ms-compat.cpp
@@ -13,7 +13,7 @@ void test(X x) {
   // CHECK-LABEL: define dso_local void @"?test@@YAXUX@@@Z"
 
   // X86: %[[argmem:[^ ]*]] = alloca inalloca <{ %struct.X }>
-  // X86: call void (<{ %struct.X }>*, ...) bitcast (void (...)* @"?vararg@@YAXZZ" to void (<{ %struct.X }>*, ...)*)(<{ %struct.X }>* inalloca %[[argmem]])
+  // X86: call void (<{ %struct.X }>*, ...) bitcast (void (...)* @"?vararg@@YAXZZ" to void (<{ %struct.X }>*, ...)*)(<{ %struct.X }>* inalloca(<{ %struct.X }>) %[[argmem]])
 
   // X64: alloca %struct.X
 
diff --git a/clang/test/CodeGenCXX/visibility.cpp b/clang/test/CodeGenCXX/visibility.cpp
index 48ec1b8b712fda4c436f4f2a36725c600696246b..aff6554282caf57f0de5b5db9b0d6fee8a58371f 100644
--- a/clang/test/CodeGenCXX/visibility.cpp
+++ b/clang/test/CodeGenCXX/visibility.cpp
@@ -18,7 +18,7 @@ namespace test30 {
   };
   H DEFAULT a;
   X<&a> b;
-  // CHECK: _ZN6test301bE = dso_local global
+  // CHECK: _ZN6test301bE = global
   // CHECK-HIDDEN: _ZN6test301bE = hidden global
 }
 
@@ -33,7 +33,7 @@ namespace test25 {
   class DEFAULT A { };
 
   X<int>::definition<A> a;
-  // CHECK: @_ZN6test251aE = dso_local global
+  // CHECK: @_ZN6test251aE = global
   // CHECK-HIDDEN: @_ZN6test251aE = hidden global
 }
 
@@ -41,7 +41,7 @@ namespace test28 {
   class DEFAULT foo {
   };
   foo myvec;
-  // CHECK: @_ZN6test285myvecE = dso_local global
+  // CHECK: @_ZN6test285myvecE = global
   // CHECK-HIDDEN: @_ZN6test285myvecE = hidden global
 }
 
@@ -53,8 +53,8 @@ namespace test29 {
   DEFAULT extern RECT data_rect;
   RECT data_rect = { -1};
 #pragma GCC visibility pop
-  // CHECK: @_ZN6test299data_rectE = dso_local global
-  // CHECK-HIDDEN: @_ZN6test299data_rectE = dso_local global
+  // CHECK: @_ZN6test299data_rectE = global
+  // CHECK-HIDDEN: @_ZN6test299data_rectE = global
 }
 
 namespace test40 {
@@ -103,17 +103,17 @@ namespace test48 {
 
 // CHECK: @_ZN5Test425VariableInHiddenNamespaceE = hidden global i32 10
 // CHECK: @_ZN5Test71aE = hidden global
-// CHECK: @_ZN5Test71bE = dso_local global
-// CHECK: @test9_var = dso_local global
-// CHECK-HIDDEN: @test9_var = dso_local global
+// CHECK: @_ZN5Test71bE = global
+// CHECK: @test9_var = global
+// CHECK-HIDDEN: @test9_var = global
 // CHECK: @_ZN6Test121A6hiddenE = external hidden global
 // CHECK: @_ZN6Test121A7visibleE = external global
 // CHECK-HIDDEN: @_ZN6Test121A6hiddenE = external hidden global
 // CHECK-HIDDEN: @_ZN6Test121A7visibleE = external global
 // CHECK: @_ZN6Test131B1aE = hidden global
-// CHECK: @_ZN6Test131C1aE = dso_local global
+// CHECK: @_ZN6Test131C1aE = global
 // CHECK-HIDDEN: @_ZN6Test131B1aE = hidden global
-// CHECK-HIDDEN: @_ZN6Test131C1aE = dso_local global
+// CHECK-HIDDEN: @_ZN6Test131C1aE = global
 // CHECK: @_ZN6Test143varE = external global
 // CHECK-HIDDEN: @_ZN6Test143varE = external global
 // CHECK: @_ZN6Test154TempINS_1AEE5Inner6bufferE = external global [0 x i8]
@@ -134,8 +134,8 @@ namespace test27 {
 
   void C<int>::D::g() {
   }
-  // CHECK: _ZTVN6test271CIiE1DE = dso_local unnamed_addr constant
-  // CHECK-HIDDEN: _ZTVN6test271CIiE1DE = dso_local unnamed_addr constant
+  // CHECK: _ZTVN6test271CIiE1DE = unnamed_addr constant
+  // CHECK-HIDDEN: _ZTVN6test271CIiE1DE = unnamed_addr constant
 }
 
 // CHECK: @_ZTVN5Test63fooE = linkonce_odr hidden unnamed_addr constant
@@ -195,7 +195,7 @@ namespace Test4 HIDDEN {
   };
   
   // A has default visibility.
-  // CHECK-LABEL: define dso_local void @_ZN5Test41A1fEv
+  // CHECK-LABEL: define void @_ZN5Test41A1fEv
   void A::f() { } 
 }
 
@@ -209,7 +209,7 @@ namespace Test5 {
   
   namespace NS {
     // g is in NS, but this NS decl is not hidden.
-    // CHECK-LABEL: define dso_local void @_ZN5Test52NS1gEv
+    // CHECK-LABEL: define void @_ZN5Test52NS1gEv
     void g() { }
   }
 }
@@ -268,8 +268,8 @@ namespace Test9 {
     void DEFAULT test9_fun(struct A *a) { }
     struct A DEFAULT test9_var; // above
   }
-  // CHECK-LABEL: define dso_local void @test9_fun(
-  // CHECK-HIDDEN-LABEL: define dso_local void @test9_fun(
+  // CHECK-LABEL: define void @test9_fun(
+  // CHECK-HIDDEN-LABEL: define void @test9_fun(
 
   void test() {
     A a = test9_var;
@@ -285,8 +285,8 @@ namespace Test10 {
     void foo(A*);
   };
 
-  // CHECK-LABEL: define dso_local void @_ZN6Test101B3fooEPNS_1AE(
-  // CHECK-HIDDEN-LABEL: define dso_local void @_ZN6Test101B3fooEPNS_1AE(
+  // CHECK-LABEL: define void @_ZN6Test101B3fooEPNS_1AE(
+  // CHECK-HIDDEN-LABEL: define void @_ZN6Test101B3fooEPNS_1AE(
   void B::foo(A*) {}
 }
 
@@ -507,7 +507,7 @@ namespace Test20 {
     static void test3();
   };
 
-  // CHECK-LABEL: define dso_local void @_ZN6Test201AILj1EE5test2Ev()
+  // CHECK-LABEL: define void @_ZN6Test201AILj1EE5test2Ev()
   void A<1>::test2() {}
 
   // CHECK: declare void @_ZN6Test201AILj1EE5test3Ev()
@@ -684,8 +684,8 @@ namespace test26 {
   template<>
   void C<int>::f() { }
 
-  // CHECK-LABEL: define dso_local void @_ZN6test261CIiE1fEv
-  // CHECK-HIDDEN-LABEL: define dso_local void @_ZN6test261CIiE1fEv
+  // CHECK-LABEL: define void @_ZN6test261CIiE1fEv
+  // CHECK-HIDDEN-LABEL: define void @_ZN6test261CIiE1fEv
 }
 
 namespace test31 {
@@ -709,8 +709,8 @@ namespace test32 {
   };
   void A::B::baz() {
   }
-  // CHECK-LABEL: define dso_local void @_ZN6test321A1B3bazEv
-  // CHECK-HIDDEN-LABEL: define dso_local void @_ZN6test321A1B3bazEv
+  // CHECK-LABEL: define void @_ZN6test321A1B3bazEv
+  // CHECK-HIDDEN-LABEL: define void @_ZN6test321A1B3bazEv
 }
 
 namespace test33 {
@@ -829,8 +829,8 @@ namespace test42 {
   };
   void bar<foo>::zed() {
   }
-  // CHECK-LABEL: define dso_local void @_ZN6test423barINS_3fooEE3zedEv
-  // CHECK-HIDDEN-LABEL: define dso_local void @_ZN6test423barINS_3fooEE3zedEv
+  // CHECK-LABEL: define void @_ZN6test423barINS_3fooEE3zedEv
+  // CHECK-HIDDEN-LABEL: define void @_ZN6test423barINS_3fooEE3zedEv
 }
 
 namespace test43 {
@@ -842,8 +842,8 @@ namespace test43 {
   template <>
   DEFAULT void bar<foo>() {
   }
-  // CHECK-LABEL: define dso_local void @_ZN6test433barINS_3fooEEEvv
-  // CHECK-HIDDEN-LABEL: define dso_local void @_ZN6test433barINS_3fooEEEvv
+  // CHECK-LABEL: define void @_ZN6test433barINS_3fooEEEvv
+  // CHECK-HIDDEN-LABEL: define void @_ZN6test433barINS_3fooEEEvv
 }
 
 namespace test44 {
@@ -1208,10 +1208,10 @@ namespace test65 {
     static void foo() {}
   };
 
-  // CHECK-LABEL: define dso_local void @_ZN6test651BINS_1AEE4funcEv()
+  // CHECK-LABEL: define void @_ZN6test651BINS_1AEE4funcEv()
   template <> DEFAULT void B<A>::func() {}
 
-  // CHECK-LABEL: define dso_local void @_ZN6test651BINS_1AEE6funcT2IS1_EEvv()
+  // CHECK-LABEL: define void @_ZN6test651BINS_1AEE6funcT2IS1_EEvv()
   template <> template <> DEFAULT void B<A>::funcT2<A>() {}
 
   // CHECK-LABEL: define linkonce_odr void @_ZN6test651BINS_1AEE6funcT1IiEEvv()
@@ -1314,6 +1314,6 @@ namespace test69 {
   }
   namespace foo __attribute__((visibility("hidden"))) {
   }
-  // CHECK-LABEL: define dso_local void @_ZN6test693foo1fEv
+  // CHECK-LABEL: define void @_ZN6test693foo1fEv
   // CHECK-HIDDEN-LABEL: define hidden void @_ZN6test693foo1fEv
 }
diff --git a/clang/test/CodeGenCoroutines/coro-alloc.cpp b/clang/test/CodeGenCoroutines/coro-alloc.cpp
index bf8edf012a33fbbc3a372238ea0d37d8d66cf4a9..c60ca5a83d484d35e9e833a91256d8efa7d5e17d 100644
--- a/clang/test/CodeGenCoroutines/coro-alloc.cpp
+++ b/clang/test/CodeGenCoroutines/coro-alloc.cpp
@@ -245,6 +245,8 @@ extern "C" int f4(promise_on_alloc_failure_tag) {
 
   // CHECK: %[[Tmp1:.*]] = load i32, i32* %[[Gro]]
   // CHECK-NEXT: store i32 %[[Tmp1]], i32* %[[RetVal]]
+  // CHECK-NEXT: %[[Gro_CAST:.+]] = bitcast i32* %[[Gro]] to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* %[[Gro_CAST]]) #2
   // CHECK-NEXT: br label %[[RetBB]]
 
   // CHECK: [[RetBB]]:
diff --git a/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp b/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp
index f8173d8f8df083c4eac645408807f2bd2e77eb16..1b0c3a1c5c5779a15b8693535a0ddba313d9d15c 100644
--- a/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp
+++ b/clang/test/CodeGenCoroutines/coro-await-resume-eh.cpp
@@ -57,12 +57,18 @@ throwing_task f() {
   // CHECK-NEXT: to label %[[RESUMEENDCATCHCONT:.+]] unwind label
   // CHECK: [[RESUMEENDCATCHCONT]]:
   // CHECK-NEXT: br label %[[RESUMETRYCONT]]
+  // CHECK: [[RESUMETRYCONT]]:
+  // CHECK-NEXT: br label %[[CLEANUP:.+]]
+  // CHECK: [[CLEANUP]]:
+  // CHECK: switch i32 %{{.+}}, label %{{.+}} [
+  // CHECK-NEXT: i32 0, label %[[CLEANUPCONT:.+]]
+  // CHECK-NEXT: ]
 
   // The variable RESUMETHREW is loaded and if true, then 'await_resume'
   // threw an exception and the coroutine body is skipped, and the final
   // suspend is executed immediately. Otherwise, the coroutine body is
   // executed, and then the final suspend.
-  // CHECK: [[RESUMETRYCONT]]:
+  // CHECK: [[CLEANUPCONT]]:
   // CHECK-NEXT: %[[RESUMETHREWLOAD:.+]] = load i1, i1* %[[RESUMETHREW]]
   // CHECK-NEXT: br i1 %[[RESUMETHREWLOAD]], label %[[RESUMEDCONT:.+]], label %[[RESUMEDBODY:.+]]
 
@@ -76,7 +82,7 @@ throwing_task f() {
   // CHECK-NEXT: br label %[[COROFINAL]]
 
   // CHECK: [[COROFINAL]]:
-  // CHECK-NEXT: call void @_ZN13throwing_task12promise_type13final_suspendEv
+  // CHECK: call void @_ZN13throwing_task12promise_type13final_suspendEv
   co_return;
 }
 
diff --git a/clang/test/CodeGenCoroutines/coro-await.cpp b/clang/test/CodeGenCoroutines/coro-await.cpp
index 4344cf3db4d8e5fcde53fc114d4edd75bc6365bd..3fa45d5f9ab68c620ac6d42514116d29bdd88c4f 100644
--- a/clang/test/CodeGenCoroutines/coro-await.cpp
+++ b/clang/test/CodeGenCoroutines/coro-await.cpp
@@ -231,7 +231,9 @@ extern "C" void TestScalar() {
 
   int Val = co_await ScalarAwaiter{};
   // CHECK: %[[Result2:.+]] = call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter*
-  // CHECK: store i32 %[[Result2]], i32* %Val
+  // CHECK: store i32 %[[Result2]], i32* %[[TMP_EXPRCLEANUP:.+]],
+  // CHECK: %[[TMP:.+]] = load i32, i32* %[[TMP_EXPRCLEANUP]],
+  // CHECK: store i32 %[[TMP]], i32* %Val,
 
   co_await ScalarAwaiter{};
   // CHECK: call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter*
@@ -312,19 +314,25 @@ void AwaitReturnsLValue(double) {
   // CHECK: %[[YVAR:.+]] = alloca %struct.RefTag*,
   // CHECK-NEXT: %[[TMP1:.+]] = alloca %struct.AwaitResumeReturnsLValue,
 
+  // CHECK: %[[TMP_EXPRCLEANUP1:.+]] = alloca %struct.RefTag*,
   // CHECK: %[[ZVAR:.+]] = alloca %struct.RefTag*,
   // CHECK-NEXT: %[[TMP2:.+]] = alloca %struct.AwaitResumeReturnsLValue,
+  // CHECK: %[[TMP_EXPRCLEANUP2:.+]] = alloca %struct.RefTag*,
 
   // CHECK: %[[RES1:.+]] = call nonnull align 1 dereferenceable({{.*}}) %struct.RefTag* @_ZN24AwaitResumeReturnsLValue12await_resumeEv(%struct.AwaitResumeReturnsLValue* {{[^,]*}} %[[AVAR]])
   // CHECK-NEXT: store %struct.RefTag* %[[RES1]], %struct.RefTag** %[[XVAR]],
   RefTag& x = co_await a;
 
   // CHECK: %[[RES2:.+]] = call nonnull align 1 dereferenceable({{.*}}) %struct.RefTag* @_ZN24AwaitResumeReturnsLValue12await_resumeEv(%struct.AwaitResumeReturnsLValue* {{[^,]*}} %[[TMP1]])
-  // CHECK-NEXT: store %struct.RefTag* %[[RES2]], %struct.RefTag** %[[YVAR]],
+  // CHECK-NEXT: store %struct.RefTag* %[[RES2]], %struct.RefTag** %[[TMP_EXPRCLEANUP1]],
+  // CHECK: %[[LOAD_TMP1:.+]] = load %struct.RefTag*, %struct.RefTag** %[[TMP_EXPRCLEANUP1]],
+  // CHECK: store %struct.RefTag* %[[LOAD_TMP1]], %struct.RefTag** %[[YVAR]],
 
   RefTag& y = co_await AwaitResumeReturnsLValue{};
   // CHECK: %[[RES3:.+]] = call nonnull align 1 dereferenceable({{.*}}) %struct.RefTag* @_ZN24AwaitResumeReturnsLValue12await_resumeEv(%struct.AwaitResumeReturnsLValue* {{[^,]*}} %[[TMP2]])
-  // CHECK-NEXT: store %struct.RefTag* %[[RES3]], %struct.RefTag** %[[ZVAR]],
+  // CHECK-NEXT: store %struct.RefTag* %[[RES3]], %struct.RefTag** %[[TMP_EXPRCLEANUP2]],
+  // CHECK: %[[LOAD_TMP2:.+]] = load %struct.RefTag*, %struct.RefTag** %[[TMP_EXPRCLEANUP2]],
+  // CHECK: store %struct.RefTag* %[[LOAD_TMP2]], %struct.RefTag** %[[ZVAR]],
   RefTag& z = co_yield 42;
 }
 
diff --git a/clang/test/CodeGenCoroutines/coro-dest-slot.cpp b/clang/test/CodeGenCoroutines/coro-dest-slot.cpp
index 0c8ef6b045821fda6cccc67cfd7e813fede4f31a..762fa1aede3c83bf7b4f67a13270699879400abd 100644
--- a/clang/test/CodeGenCoroutines/coro-dest-slot.cpp
+++ b/clang/test/CodeGenCoroutines/coro-dest-slot.cpp
@@ -17,10 +17,24 @@ struct coro {
 extern "C" coro f(int) { co_return; }
 // Verify that cleanup.dest.slot is eliminated in a coroutine.
 // CHECK-LABEL: f(
+// CHECK: %[[INIT_SUSPEND:.+]] = call i8 @llvm.coro.suspend(
+// CHECK-NEXT: switch i8 %[[INIT_SUSPEND]], label
+// CHECK-NEXT:   i8 0, label %[[INIT_READY:.+]]
+// CHECK-NEXT:   i8 1, label %[[INIT_CLEANUP:.+]]
+// CHECK-NEXT: ]
+// CHECK: %[[CLEANUP_DEST0:.+]] = phi i32 [ 0, %[[INIT_READY]] ], [ 2, %[[INIT_CLEANUP]] ]
+
+// CHECK: %[[FINAL_SUSPEND:.+]] = call i8 @llvm.coro.suspend(
+// CHECK-NEXT: switch i8 %29, label %coro.ret [
+// CHECK-NEXT:   i8 0, label %[[FINAL_READY:.+]]
+// CHECK-NEXT:   i8 1, label %[[FINAL_CLEANUP:.+]]
+// CHECK-NEXT: ]
+
 // CHECK: call void @_ZNSt12experimental13coroutines_v113suspend_never12await_resumeEv(
-// CHECK: %[[CLEANUP_DEST:.+]] = phi i32 [ 0, %{{.+}} ], [ 2, %{{.+}} ], [ 2, %{{.+}} ]
+// CHECK: %[[CLEANUP_DEST1:.+]] = phi i32 [ 0, %[[FINAL_READY]] ], [ 2, %[[FINAL_CLEANUP]] ]
+// CHECK: %[[CLEANUP_DEST2:.+]] = phi i32 [ %[[CLEANUP_DEST0]], %{{.+}} ], [ %[[CLEANUP_DEST1]], %{{.+}} ], [ 0, %{{.+}} ]
 // CHECK: call i8* @llvm.coro.free(
-// CHECK: switch i32 %cleanup.dest.slot.0, label %{{.+}} [
+// CHECK: switch i32 %[[CLEANUP_DEST2]], label %{{.+}} [
 // CHECK-NEXT: i32 0
 // CHECK-NEXT: i32 2
 // CHECK-NEXT: ]
diff --git a/clang/test/CodeGenCoroutines/coro-params.cpp b/clang/test/CodeGenCoroutines/coro-params.cpp
index d69d3103e39a401475f8d8e52b70bfe2459ece50..12332cd793c4fedfe7680ee677dd2a598b2b0fcc 100644
--- a/clang/test/CodeGenCoroutines/coro-params.cpp
+++ b/clang/test/CodeGenCoroutines/coro-params.cpp
@@ -70,7 +70,11 @@ void f(int val, MoveOnly moParam, MoveAndCopy mcParam) {
 
   // CHECK: call i8* @llvm.coro.begin(
   // CHECK: call void @_ZN8MoveOnlyC1EOS_(%struct.MoveOnly* {{[^,]*}} %[[MoCopy]], %struct.MoveOnly* nonnull align 4 dereferenceable(4) %[[MoParam]])
+  // CHECK-NEXT: bitcast %struct.MoveAndCopy* %[[McCopy]] to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0i8(
   // CHECK-NEXT: call void @_ZN11MoveAndCopyC1EOS_(%struct.MoveAndCopy* {{[^,]*}} %[[McCopy]], %struct.MoveAndCopy* nonnull align 4 dereferenceable(4) %[[McParam]]) #
+  // CHECK-NEXT: bitcast %"struct.std::experimental::coroutine_traits<void, int, MoveOnly, MoveAndCopy>::promise_type"* %__promise to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0i8(
   // CHECK-NEXT: invoke void @_ZNSt12experimental16coroutine_traitsIJvi8MoveOnly11MoveAndCopyEE12promise_typeC1Ev(
 
   // CHECK: call void @_ZN14suspend_always12await_resumeEv(
@@ -89,9 +93,17 @@ void f(int val, MoveOnly moParam, MoveAndCopy mcParam) {
   // CHECK: call void @_ZN14suspend_always12await_resumeEv(
 
   // Destroy promise, then parameter copies:
-  // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJvi8MoveOnly11MoveAndCopyEE12promise_typeD1Ev(%"struct.std::experimental::coroutine_traits<void, int, MoveOnly, MoveAndCopy>::promise_type"* {{[^,]*}} %__promise) #2
+  // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJvi8MoveOnly11MoveAndCopyEE12promise_typeD1Ev(%"struct.std::experimental::coroutine_traits<void, int, MoveOnly, MoveAndCopy>::promise_type"* {{[^,]*}} %__promise)
+  // CHECK-NEXT: bitcast %"struct.std::experimental::coroutine_traits<void, int, MoveOnly, MoveAndCopy>::promise_type"* %__promise to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0i8(
   // CHECK-NEXT: call void @_ZN11MoveAndCopyD1Ev(%struct.MoveAndCopy* {{[^,]*}} %[[McCopy]])
+  // CHECK-NEXT: bitcast %struct.MoveAndCopy* %[[McCopy]] to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0i8(
   // CHECK-NEXT: call void @_ZN8MoveOnlyD1Ev(%struct.MoveOnly* {{[^,]*}} %[[MoCopy]]
+  // CHECK-NEXT: bitcast %struct.MoveOnly* %[[MoCopy]] to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0i8(
+  // CHECK-NEXT: bitcast i32* %{{.+}} to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0i8(
   // CHECK-NEXT: call i8* @llvm.coro.free(
 }
 
@@ -103,9 +115,17 @@ void dependent_params(T x, U, U y) {
   // CHECK-NEXT: %[[y_copy:.+]] = alloca %struct.B
 
   // CHECK: call i8* @llvm.coro.begin
+  // CHECK-NEXT: bitcast %struct.A* %[[x_copy]] to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0i8(
   // CHECK-NEXT: call void @_ZN1AC1EOS_(%struct.A* {{[^,]*}} %[[x_copy]], %struct.A* nonnull align 4 dereferenceable(512) %x)
+  // CHECK-NEXT: bitcast %struct.B* %[[unnamed_copy]] to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0i8(
   // CHECK-NEXT: call void @_ZN1BC1EOS_(%struct.B* {{[^,]*}} %[[unnamed_copy]], %struct.B* nonnull align 4 dereferenceable(512) %0)
+  // CHECK-NEXT: %10 = bitcast %struct.B* %[[y_copy]] to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0i8(
   // CHECK-NEXT: call void @_ZN1BC1EOS_(%struct.B* {{[^,]*}} %[[y_copy]], %struct.B* nonnull align 4 dereferenceable(512) %y)
+  // CHECK-NEXT: bitcast %"struct.std::experimental::coroutine_traits<void, A, B, B>::promise_type"* %__promise to i8*
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0i8(
   // CHECK-NEXT: invoke void @_ZNSt12experimental16coroutine_traitsIJv1A1BS2_EE12promise_typeC1Ev(
 
   co_return;
diff --git a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp
index 0725abc91c029f0ad1c3cd79836558427e0c40a2..a5592b066f7a2fb4a038697d495e8c6422ecdb85 100644
--- a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp
+++ b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -O1 -emit-llvm %s -o - -disable-llvm-passes | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -O0 -emit-llvm %s -o - -disable-llvm-passes | FileCheck %s
 
 #include "Inputs/coroutine.h"
 
@@ -50,8 +50,13 @@ detached_task foo() {
 
 // check that the lifetime of the coroutine handle used to obtain the address is contained within single basic block, and hence does not live across suspension points.
 // CHECK-LABEL: final.suspend:
-// CHECK:         %[[PTR1:.+]] = bitcast %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[ADDR_TMP:.+]] to i8*
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* %[[PTR1]])
-// CHECK:         call i8* @{{.*address.*}}(%"struct.std::experimental::coroutines_v1::coroutine_handle.0"* {{[^,]*}} %[[ADDR_TMP]])
-// CHECK-NEXT:    %[[PTR2:.+]] = bitcast %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[ADDR_TMP]] to i8*
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* %[[PTR2]])
+// CHECK:         %{{.+}} = call token @llvm.coro.save(i8* null)
+// CHECK:         %[[HDL_CAST1:.+]] = bitcast %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[HDL:.+]] to i8*
+// CHECK:         call void @llvm.lifetime.start.p0i8(i64 8, i8* %[[HDL_CAST1]])
+// CHECK:         %[[CALL:.+]] = call i8* @_ZN13detached_task12promise_type13final_awaiter13await_suspendENSt12experimental13coroutines_v116coroutine_handleIS0_EE(
+// CHECK:         %[[HDL_CAST2:.+]] = getelementptr inbounds %"struct.std::experimental::coroutines_v1::coroutine_handle.0", %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[HDL]], i32 0, i32 0
+// CHECK:         store i8* %[[CALL]], i8** %[[HDL_CAST2]], align 8
+// CHECK:         %[[HDL_TRANSFER:.+]] = call i8* @_ZNKSt12experimental13coroutines_v116coroutine_handleIvE7addressEv(%"struct.std::experimental::coroutines_v1::coroutine_handle.0"* nonnull dereferenceable(8) %[[HDL]])
+// CHECK:         %[[HDL_CAST3:.+]] = bitcast %"struct.std::experimental::coroutines_v1::coroutine_handle.0"* %[[HDL]] to i8*
+// CHECK:         call void @llvm.lifetime.end.p0i8(i64 8, i8* %[[HDL_CAST3]])
+// CHECK:         call void @llvm.coro.resume(i8* %[[HDL_TRANSFER]])
diff --git a/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp b/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp
index 30cec4e5000bb9d24becca95c2b93ea561375c3a..f038c5b3a9138c0b1de4ee5365fca27bde577800 100644
--- a/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp
+++ b/clang/test/CodeGenCoroutines/coro-unhandled-exception.cpp
@@ -50,6 +50,8 @@ coro_t f() {
 // CHECK: [[TRYCONT]]:
 // CHECK-NEXT: br label %[[COROFIN:.+]]
 // CHECK: [[COROFIN]]:
+// CHECK-NEXT: bitcast %"struct.std::experimental::coroutines_v1::suspend_never"* %{{.+}} to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(
 // CHECK-NEXT: call void @"?final_suspend@promise_type@coro_t@@QEAA?AUsuspend_never@coroutines_v1@experimental@std@@XZ"(
 
 // CHECK-LPAD: @_Z1fv(
@@ -69,4 +71,6 @@ coro_t f() {
 // CHECK-LPAD: [[TRYCONT]]:
 // CHECK-LPAD: br label %[[COROFIN:.+]]
 // CHECK-LPAD: [[COROFIN]]:
+// CHECK-LPAD-NEXT: bitcast %"struct.std::experimental::coroutines_v1::suspend_never"* %{{.+}} to i8*
+// CHECK-LPAD-NEXT: call void @llvm.lifetime.start.p0i8(
 // CHECK-LPAD-NEXT: call void @_ZN6coro_t12promise_type13final_suspendEv(
diff --git a/clang/test/CodeGenObjC/attr-nomerge.m b/clang/test/CodeGenObjC/attr-nomerge.m
new file mode 100644
index 0000000000000000000000000000000000000000..7d053d0b1d69f1a579a9e62a570eb7738b8cb29a
--- /dev/null
+++ b/clang/test/CodeGenObjC/attr-nomerge.m
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -emit-llvm -fobjc-exceptions -triple x86_64-unknown-linux -o - %s | FileCheck %s
+
+// Test that the nomerge attribute is applied to function calls
+// in @try, @catch and @finally
+void opaque(void);
+void opaque2(void);
+void opaque3(void);
+
+int main(int argc, const char * argv[]) {
+  __attribute__((nomerge)) @try {
+    opaque();
+  } @catch(...) {
+    opaque2();
+  } @finally {
+    opaque3();
+  }
+
+  return 0;
+}
+
+// CHECK: call void @opaque() #[[ATTR0:[0-9]+]]
+// CHECK-DAG: call void @opaque2() #[[ATTR0]]
+// CHECK-DAG: call void @opaque3() #[[ATTR0]]
+// CHECK-DAG: attributes #[[ATTR0]] = {{{.*}}nomerge{{.*}}}
diff --git a/clang/test/CodeGenObjCXX/arc-indirect.mm b/clang/test/CodeGenObjCXX/arc-indirect.mm
index de7566fcf987e5ea7ebe14b0d849982024720db0..40543c054ea5551cbd7d11213d49c710717b6dff 100644
--- a/clang/test/CodeGenObjCXX/arc-indirect.mm
+++ b/clang/test/CodeGenObjCXX/arc-indirect.mm
@@ -15,8 +15,8 @@ struct S {
 }
 @end
 
-// CHECK-GNUSTEP: define internal void @_i_C__object_struct_(<{ %0*, i8*, i8*, %struct.S, [3 x i8] }>* inalloca %0)
-// CHECK-DARWIN: define internal void @"\01-[C object:struct:]"(<{ %0*, i8*, i8*, %struct.S, [3 x i8] }>* inalloca %0)
+// CHECK-GNUSTEP: define internal void @_i_C__object_struct_(<{ %0*, i8*, i8*, %struct.S, [3 x i8] }>* inalloca(<{ %0*, i8*, i8*, %struct.S, [3 x i8] }>) %0)
+// CHECK-DARWIN: define internal void @"\01-[C object:struct:]"(<{ %0*, i8*, i8*, %struct.S, [3 x i8] }>* inalloca(<{ %0*, i8*, i8*, %struct.S, [3 x i8] }>) %0)
 // CHECK: %obj = getelementptr inbounds <{ %0*, i8*, i8*, %struct.S, [3 x i8] }>, <{ %0*, i8*, i8*, %struct.S, [3 x i8] }>* %0, i32 0, i32 2
 // CHECK: %[[INSTANCE:[0-9]+]] = load i8*, i8** %obj, align 4
 // CHECK: call void @llvm.objc.storeStrong(i8** %obj, i8* %[[INSTANCE]])
diff --git a/clang/test/CodeGenObjCXX/microsoft-abi-arc-param-order.mm b/clang/test/CodeGenObjCXX/microsoft-abi-arc-param-order.mm
index 6be7995f5f01908c49738257e0691c71e1dcd43c..26c13acf8959ad687987a77bbbac15cea88689a9 100644
--- a/clang/test/CodeGenObjCXX/microsoft-abi-arc-param-order.mm
+++ b/clang/test/CodeGenObjCXX/microsoft-abi-arc-param-order.mm
@@ -10,7 +10,7 @@ struct A {
 // Verify that we destruct things from left to right in the MS C++ ABI: a, b, c, d.
 //
 // CHECK-LABEL: define dso_local void @"?test_arc_order@@YAXUA@@PAUobjc_object@@01@Z"
-// CHECK:                       (<{ %struct.A, i8*, %struct.A, i8* }>* inalloca %0)
+// CHECK:                       (<{ %struct.A, i8*, %struct.A, i8* }>* inalloca(<{ %struct.A, i8*, %struct.A, i8* }>) %0)
 void test_arc_order(A a, id __attribute__((ns_consumed)) b , A c, id __attribute__((ns_consumed)) d) {
   // CHECK: call x86_thiscallcc void @"??1A@@QAE@XZ"(%struct.A* {{[^,]*}} %{{.*}})
   // CHECK: call void @llvm.objc.storeStrong(i8** %{{.*}}, i8* null)
diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl
index 930c53705d84b939b99230b0d987ead77c029a26..5c9059866e88e4224fab1a618480dfd618fd63a5 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-features.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl
@@ -50,17 +50,17 @@
 // GFX900: "target-features"="+16-bit-insts,+ci-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
 // GFX902: "target-features"="+16-bit-insts,+ci-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
 // GFX904: "target-features"="+16-bit-insts,+ci-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
-// GFX906: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
-// GFX908: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+mai-insts,+s-memrealtime,+s-memtime-inst"
+// GFX906: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot7-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
+// GFX908: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+mai-insts,+s-memrealtime,+s-memtime-inst"
 // GFX909: "target-features"="+16-bit-insts,+ci-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
-// GFX90A: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+gfx90a-insts,+mai-insts,+s-memrealtime,+s-memtime-inst"
+// GFX90A: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+gfx90a-insts,+mai-insts,+s-memrealtime,+s-memtime-inst"
 // GFX90C: "target-features"="+16-bit-insts,+ci-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
 // GFX1010: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dpp,+flat-address-space,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
-// GFX1011: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
-// GFX1012: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
-// GFX1030: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx10-3-insts,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
-// GFX1031: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx10-3-insts,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
-// GFX1032: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx10-3-insts,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
-// GFX1033: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx10-3-insts,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
+// GFX1011: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dpp,+flat-address-space,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
+// GFX1012: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dpp,+flat-address-space,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
+// GFX1030: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dpp,+flat-address-space,+gfx10-3-insts,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
+// GFX1031: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dpp,+flat-address-space,+gfx10-3-insts,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
+// GFX1032: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dpp,+flat-address-space,+gfx10-3-insts,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
+// GFX1033: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dpp,+flat-address-space,+gfx10-3-insts,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst"
 
 kernel void test() {}
diff --git a/clang/test/CodeGenOpenCL/arm-integer-dot-product.cl b/clang/test/CodeGenOpenCL/arm-integer-dot-product.cl
index d1ab6aceac5cf3789c68646dbef83946508ca03b..a4d28c7e6cf8f0a8938140decb31a5590149db2e 100644
--- a/clang/test/CodeGenOpenCL/arm-integer-dot-product.cl
+++ b/clang/test/CodeGenOpenCL/arm-integer-dot-product.cl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -finclude-default-header -cl-std=CL1.2 -emit-llvm -o - -O0 | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -finclude-default-header -fdeclare-opencl-builtins -cl-std=CL1.2 -emit-llvm -o - -O0 | FileCheck %s
 
 #pragma OPENCL EXTENSION cl_arm_integer_dot_product_int8 : enable
 void test_int8(uchar4 ua, uchar4 ub, char4 sa, char4 sb) {
diff --git a/clang/test/CodeGenOpenCL/atomic-ops.cl b/clang/test/CodeGenOpenCL/atomic-ops.cl
index bd5a01c5434a9d9984d76d70c2623ddfefd36006..e5da50883c65d7490a03f49ea3012e9428bfe601 100644
--- a/clang/test/CodeGenOpenCL/atomic-ops.cl
+++ b/clang/test/CodeGenOpenCL/atomic-ops.cl
@@ -227,6 +227,7 @@ void generalFailureOrder(atomic_int *ptr, int *ptr2, int success, int fail) {
 
   // CHECK: [[RELEASE]]
   // CHECK: switch {{.*}}, label %[[RELEASE_MONOTONIC:[0-9a-zA-Z._]+]] [
+  // CHECK-NEXT: i32 2, label %[[RELEASE_ACQUIRE:[0-9a-zA-Z._]+]]
   // CHECK-NEXT: ]
 
   // CHECK: [[ACQREL]]
@@ -254,6 +255,14 @@ void generalFailureOrder(atomic_int *ptr, int *ptr2, int success, int fail) {
   // CHECK: cmpxchg {{.*}} acquire acquire, align 4
   // CHECK: br
 
+  // CHECK: [[RELEASE_MONOTONIC]]
+  // CHECK: cmpxchg {{.*}} release monotonic, align 4
+  // CHECK: br
+
+  // CHECK: [[RELEASE_ACQUIRE]]
+  // CHECK: cmpxchg {{.*}} release acquire, align 4
+  // CHECK: br
+
   // CHECK: [[ACQREL_MONOTONIC]]
   // CHECK: cmpxchg {{.*}} acq_rel monotonic, align 4
   // CHECK: br
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-dl-insts-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-dl-insts-err.cl
index cb83c0b48c653c48aa9fe67928a1b11ced10ef93..e7a71b5158859d5475cd0a8f94b4ae132e94f48d 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-dl-insts-err.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-dl-insts-err.cl
@@ -13,8 +13,8 @@ kernel void builtins_amdgcn_dl_insts_err(
     half2 v2hA, half2 v2hB, float fC,
     short2 v2ssA, short2 v2ssB, int siA, int siB, int siC,
     ushort2 v2usA, ushort2 v2usB, uint uiA, uint uiB, uint uiC) {
-  fOut[0] = __builtin_amdgcn_fdot2(v2hA, v2hB, fC, false);     // expected-error {{'__builtin_amdgcn_fdot2' needs target feature dot2-insts}}
-  fOut[1] = __builtin_amdgcn_fdot2(v2hA, v2hB, fC, true);      // expected-error {{'__builtin_amdgcn_fdot2' needs target feature dot2-insts}}
+  fOut[0] = __builtin_amdgcn_fdot2(v2hA, v2hB, fC, false);     // expected-error {{'__builtin_amdgcn_fdot2' needs target feature dot7-insts}}
+  fOut[1] = __builtin_amdgcn_fdot2(v2hA, v2hB, fC, true);      // expected-error {{'__builtin_amdgcn_fdot2' needs target feature dot7-insts}}
 
   siOut[0] = __builtin_amdgcn_sdot2(v2ssA, v2ssB, siC, false); // expected-error {{'__builtin_amdgcn_sdot2' needs target feature dot2-insts}}
   siOut[1] = __builtin_amdgcn_sdot2(v2ssA, v2ssB, siC, true);  // expected-error {{'__builtin_amdgcn_sdot2' needs target feature dot2-insts}}
@@ -25,12 +25,12 @@ kernel void builtins_amdgcn_dl_insts_err(
   siOut[2] = __builtin_amdgcn_sdot4(siA, siB, siC, false);     // expected-error {{'__builtin_amdgcn_sdot4' needs target feature dot1-insts}}
   siOut[3] = __builtin_amdgcn_sdot4(siA, siB, siC, true);      // expected-error {{'__builtin_amdgcn_sdot4' needs target feature dot1-insts}}
 
-  uiOut[2] = __builtin_amdgcn_udot4(uiA, uiB, uiC, false);     // expected-error {{'__builtin_amdgcn_udot4' needs target feature dot2-insts}}
-  uiOut[3] = __builtin_amdgcn_udot4(uiA, uiB, uiC, true);      // expected-error {{'__builtin_amdgcn_udot4' needs target feature dot2-insts}}
+  uiOut[2] = __builtin_amdgcn_udot4(uiA, uiB, uiC, false);     // expected-error {{'__builtin_amdgcn_udot4' needs target feature dot7-insts}}
+  uiOut[3] = __builtin_amdgcn_udot4(uiA, uiB, uiC, true);      // expected-error {{'__builtin_amdgcn_udot4' needs target feature dot7-insts}}
 
   siOut[4] = __builtin_amdgcn_sdot8(siA, siB, siC, false);     // expected-error {{'__builtin_amdgcn_sdot8' needs target feature dot1-insts}}
   siOut[5] = __builtin_amdgcn_sdot8(siA, siB, siC, true);      // expected-error {{'__builtin_amdgcn_sdot8' needs target feature dot1-insts}}
 
-  uiOut[4] = __builtin_amdgcn_udot8(uiA, uiB, uiC, false);     // expected-error {{'__builtin_amdgcn_udot8' needs target feature dot2-insts}}
-  uiOut[5] = __builtin_amdgcn_udot8(uiA, uiB, uiC, true);      // expected-error {{'__builtin_amdgcn_udot8' needs target feature dot2-insts}}
+  uiOut[4] = __builtin_amdgcn_udot8(uiA, uiB, uiC, false);     // expected-error {{'__builtin_amdgcn_udot8' needs target feature dot7-insts}}
+  uiOut[5] = __builtin_amdgcn_udot8(uiA, uiB, uiC, true);      // expected-error {{'__builtin_amdgcn_udot8' needs target feature dot7-insts}}
 }
diff --git a/clang/test/CodeGenOpenCL/builtins.cl b/clang/test/CodeGenOpenCL/builtins.cl
index 2ab6f12a9f4baa2ea86f76572e7b1714a5c1e4b3..1d566698d045bff297bb3e0cf9e516f1a1037706 100644
--- a/clang/test/CodeGenOpenCL/builtins.cl
+++ b/clang/test/CodeGenOpenCL/builtins.cl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -finclude-default-header -cl-std=clc++ -fblocks -O0 -emit-llvm -o - -triple "spir-unknown-unknown" | FileCheck %s
+// RUN: %clang_cc1 %s -finclude-default-header -fdeclare-opencl-builtins -cl-std=clc++ -fblocks -O0 -emit-llvm -o - -triple "spir-unknown-unknown" | FileCheck %s
 
 void testBranchingOnEnqueueKernel(queue_t default_queue, unsigned flags, ndrange_t ndrange) {
     // Ensure `enqueue_kernel` can be branched upon.
diff --git a/clang/test/CodeGenOpenCL/size_t.cl b/clang/test/CodeGenOpenCL/size_t.cl
index 63a062268da3198ba87bcea10481f788b3e9dda8..b9bb72ac45a0e22946feef0975cb422e527d2b05 100644
--- a/clang/test/CodeGenOpenCL/size_t.cl
+++ b/clang/test/CodeGenOpenCL/size_t.cl
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 %s -cl-std=CL2.0 -finclude-default-header -emit-llvm -O0 -triple spir-unknown-unknown -o - | FileCheck --check-prefix=SZ32 %s
-// RUN: %clang_cc1 %s -cl-std=CL2.0 -finclude-default-header -emit-llvm -O0 -triple spir64-unknown-unknown -o - | FileCheck --check-prefix=SZ64 --check-prefix=SZ64ONLY %s
-// RUN: %clang_cc1 %s -cl-std=CL2.0 -finclude-default-header -emit-llvm -O0 -triple amdgcn -o - | FileCheck --check-prefix=SZ64 --check-prefix=AMDGCN %s
-// RUN: %clang_cc1 %s -cl-std=CL2.0 -finclude-default-header -emit-llvm -O0 -triple amdgcn---opencl -o - | FileCheck --check-prefix=SZ64 --check-prefix=AMDGCN %s
+// RUN: %clang_cc1 %s -cl-std=CL2.0 -finclude-default-header -fdeclare-opencl-builtins -emit-llvm -O0 -triple spir-unknown-unknown -o - | FileCheck --check-prefix=SZ32 %s
+// RUN: %clang_cc1 %s -cl-std=CL2.0 -finclude-default-header -fdeclare-opencl-builtins -emit-llvm -O0 -triple spir64-unknown-unknown -o - | FileCheck --check-prefix=SZ64 --check-prefix=SZ64ONLY %s
+// RUN: %clang_cc1 %s -cl-std=CL2.0 -finclude-default-header -fdeclare-opencl-builtins -emit-llvm -O0 -triple amdgcn -o - | FileCheck --check-prefix=SZ64 --check-prefix=AMDGCN %s
+// RUN: %clang_cc1 %s -cl-std=CL2.0 -finclude-default-header -fdeclare-opencl-builtins -emit-llvm -O0 -triple amdgcn---opencl -o - | FileCheck --check-prefix=SZ64 --check-prefix=AMDGCN %s
 
 //SZ32: define{{.*}} i32 @test_ptrtoint_private(i8* %x)
 //SZ32: ptrtoint i8* %{{.*}} to i32
diff --git a/clang/test/CodeGenOpenCLCXX/address-space-deduction.cl b/clang/test/CodeGenOpenCLCXX/address-space-deduction.clcpp
similarity index 86%
rename from clang/test/CodeGenOpenCLCXX/address-space-deduction.cl
rename to clang/test/CodeGenOpenCLCXX/address-space-deduction.clcpp
index c0f30d291c95c466c1f7761990ff503b4dddefd1..e7c883bb8bcb8d16c0353f11337eaf38a9707a0b 100644
--- a/clang/test/CodeGenOpenCLCXX/address-space-deduction.cl
+++ b/clang/test/CodeGenOpenCLCXX/address-space-deduction.clcpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -O0 -emit-llvm -o - | FileCheck %s -check-prefixes=COMMON,PTR
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -O0 -emit-llvm -o - -DREF | FileCheck %s -check-prefixes=COMMON,REF
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -O0 -emit-llvm -o - | FileCheck %s -check-prefixes=COMMON,PTR
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -O0 -emit-llvm -o - -DREF | FileCheck %s -check-prefixes=COMMON,REF
 
 #ifdef REF
 #define PTR &
diff --git a/clang/test/CodeGenOpenCLCXX/address-space-deduction2.cl b/clang/test/CodeGenOpenCLCXX/address-space-deduction2.clcpp
similarity index 77%
rename from clang/test/CodeGenOpenCLCXX/address-space-deduction2.cl
rename to clang/test/CodeGenOpenCLCXX/address-space-deduction2.clcpp
index 36e89499c954844a1994dfba2527435158b798d2..b454f5a2de98b64abfe9ce2682ed621e198e4da0 100644
--- a/clang/test/CodeGenOpenCLCXX/address-space-deduction2.cl
+++ b/clang/test/CodeGenOpenCLCXX/address-space-deduction2.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -O0 -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -O0 -emit-llvm -o - | FileCheck %s
 
 class P {
 public:
diff --git a/clang/test/CodeGenOpenCLCXX/addrspace-conversion.cl b/clang/test/CodeGenOpenCLCXX/addrspace-conversion.clcpp
similarity index 63%
rename from clang/test/CodeGenOpenCLCXX/addrspace-conversion.cl
rename to clang/test/CodeGenOpenCLCXX/addrspace-conversion.clcpp
index a80662f72334ec9672e72d5b62fc04b87ab8d056..0608117dd660eac6e502ed03165a73b4c8d7f35e 100644
--- a/clang/test/CodeGenOpenCLCXX/addrspace-conversion.cl
+++ b/clang/test/CodeGenOpenCLCXX/addrspace-conversion.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -emit-llvm -O0 -o - | FileCheck %s
 
 void bar(__generic volatile unsigned int* ptr)
 {
diff --git a/clang/test/CodeGenOpenCLCXX/addrspace-derived-base.cl b/clang/test/CodeGenOpenCLCXX/addrspace-derived-base.clcpp
similarity index 95%
rename from clang/test/CodeGenOpenCLCXX/addrspace-derived-base.cl
rename to clang/test/CodeGenOpenCLCXX/addrspace-derived-base.clcpp
index 954536a944483f9b856af88cb9681cbb5c4de248..6b087e75c4861f0b3673bc0f4096e98df8e3ce88 100644
--- a/clang/test/CodeGenOpenCLCXX/addrspace-derived-base.cl
+++ b/clang/test/CodeGenOpenCLCXX/addrspace-derived-base.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir -emit-llvm -O0 -o - | FileCheck %s
 
 struct B {
   int mb;
diff --git a/clang/test/CodeGenOpenCLCXX/addrspace-new-delete.cl b/clang/test/CodeGenOpenCLCXX/addrspace-new-delete.clcpp
similarity index 80%
rename from clang/test/CodeGenOpenCLCXX/addrspace-new-delete.cl
rename to clang/test/CodeGenOpenCLCXX/addrspace-new-delete.clcpp
index f3a397419def2f4ec49dc5f5d0a5d462d4c72332..a78c0373ed1859917212d7ec32455116cb3ebb5a 100644
--- a/clang/test/CodeGenOpenCLCXX/addrspace-new-delete.cl
+++ b/clang/test/CodeGenOpenCLCXX/addrspace-new-delete.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir -emit-llvm -O0 -o - | FileCheck %s
 
 typedef __SIZE_TYPE__ size_t;
 
diff --git a/clang/test/CodeGenOpenCLCXX/addrspace-of-this.cl b/clang/test/CodeGenOpenCLCXX/addrspace-of-this.clcpp
similarity index 95%
rename from clang/test/CodeGenOpenCLCXX/addrspace-of-this.cl
rename to clang/test/CodeGenOpenCLCXX/addrspace-of-this.clcpp
index 764df17d1f51c8bada7cc53ceb88c2e036a6692e..50c146bae6ebbb1a016e216c6f1ea39e193c3e64 100644
--- a/clang/test/CodeGenOpenCLCXX/addrspace-of-this.cl
+++ b/clang/test/CodeGenOpenCLCXX/addrspace-of-this.clcpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -emit-llvm -pedantic -verify -O0 -o - -DDECL | FileCheck %s --check-prefixes="COMMON,EXPL"
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -emit-llvm -pedantic -verify -O0 -o - -DDECL -DUSE_DEFLT | FileCheck %s --check-prefixes="COMMON,IMPL"
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -emit-llvm -pedantic -verify -O0 -o - | FileCheck %s --check-prefixes="COMMON,IMPL"
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -emit-llvm -pedantic -verify -O0 -o - -DDECL | FileCheck %s --check-prefixes="COMMON,EXPL"
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -emit-llvm -pedantic -verify -O0 -o - -DDECL -DUSE_DEFLT | FileCheck %s --check-prefixes="COMMON,IMPL"
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -emit-llvm -pedantic -verify -O0 -o - | FileCheck %s --check-prefixes="COMMON,IMPL"
 // expected-no-diagnostics
 
 // Test that the 'this' pointer is in the __generic address space.
diff --git a/clang/test/CodeGenOpenCLCXX/addrspace-operators.cl b/clang/test/CodeGenOpenCLCXX/addrspace-operators.clcpp
similarity index 97%
rename from clang/test/CodeGenOpenCLCXX/addrspace-operators.cl
rename to clang/test/CodeGenOpenCLCXX/addrspace-operators.clcpp
index cad98c6072fa30e8e278dc466a672b7ec6669ea6..bd3832635d9b1d872135d0b62aa7bf0b5478b9c1 100644
--- a/clang/test/CodeGenOpenCLCXX/addrspace-operators.cl
+++ b/clang/test/CodeGenOpenCLCXX/addrspace-operators.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -triple spir -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
+//RUN: %clang_cc1 %s -triple spir -emit-llvm -O0 -o - | FileCheck %s
 
 enum E {
   a,
diff --git a/clang/test/CodeGenOpenCLCXX/addrspace-references.cl b/clang/test/CodeGenOpenCLCXX/addrspace-references.clcpp
similarity index 93%
rename from clang/test/CodeGenOpenCLCXX/addrspace-references.cl
rename to clang/test/CodeGenOpenCLCXX/addrspace-references.clcpp
index 6d4bece1a62409f79d11bbb02f20a82b77b2bac8..d8e83450a9e57e9baef5663acbc9f9d530853ce9 100644
--- a/clang/test/CodeGenOpenCLCXX/addrspace-references.cl
+++ b/clang/test/CodeGenOpenCLCXX/addrspace-references.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -cl-std=clc++ -triple spir -emit-llvm -o - -O0 | FileCheck %s
+//RUN: %clang_cc1 %s -triple spir -emit-llvm -o - -O0 | FileCheck %s
 
 typedef short short2 __attribute__((ext_vector_type(2)));
 
diff --git a/clang/test/CodeGenOpenCLCXX/addrspace-with-class.cl b/clang/test/CodeGenOpenCLCXX/addrspace-with-class.clcpp
similarity index 93%
rename from clang/test/CodeGenOpenCLCXX/addrspace-with-class.cl
rename to clang/test/CodeGenOpenCLCXX/addrspace-with-class.clcpp
index 7cff76a04dff98d480203fff542850f6a73548c8..c4051f93442e92bc5fe5bb50dc7884cca4325d66 100644
--- a/clang/test/CodeGenOpenCLCXX/addrspace-with-class.cl
+++ b/clang/test/CodeGenOpenCLCXX/addrspace-with-class.clcpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=CLC++ -emit-llvm -O0 -o - | FileCheck %s --check-prefix=CHECK-DEFINITIONS
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -emit-llvm -O0 -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -emit-llvm -O0 -o - | FileCheck %s --check-prefix=CHECK-DEFINITIONS
 
 // This test ensures the proper address spaces and address space cast are used
 // for constructors, member functions and destructors.
diff --git a/clang/test/CodeGenOpenCLCXX/addrspace_cast.cl b/clang/test/CodeGenOpenCLCXX/addrspace_cast.clcpp
similarity index 71%
rename from clang/test/CodeGenOpenCLCXX/addrspace_cast.cl
rename to clang/test/CodeGenOpenCLCXX/addrspace_cast.clcpp
index 1ea0fdfb2040ff835b50a6cd7dce31aa60b8368c..78656cb32aa7e780167666d0c38d62b7375ad2ab 100644
--- a/clang/test/CodeGenOpenCLCXX/addrspace_cast.cl
+++ b/clang/test/CodeGenOpenCLCXX/addrspace_cast.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -triple spir -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
+//RUN: %clang_cc1 %s -triple spir -emit-llvm -O0 -o - | FileCheck %s
 
 //CHECK-LABEL: define{{.*}} spir_func void @_Z3barPU3AS1i
 void bar(global int *gl) {
diff --git a/clang/test/CodeGenOpenCLCXX/atexit.cl b/clang/test/CodeGenOpenCLCXX/atexit.clcpp
similarity index 81%
rename from clang/test/CodeGenOpenCLCXX/atexit.cl
rename to clang/test/CodeGenOpenCLCXX/atexit.clcpp
index 2b28aeaacf45c79d315f3d163a9d7c2ae14f6403..e987790348fc471e9a3297034844293419fea9a2 100644
--- a/clang/test/CodeGenOpenCLCXX/atexit.cl
+++ b/clang/test/CodeGenOpenCLCXX/atexit.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -triple spir -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
+//RUN: %clang_cc1 %s -triple spir -emit-llvm -O0 -o - | FileCheck %s
 
 struct S {
   ~S(){};
diff --git a/clang/test/CodeGenOpenCLCXX/constexpr.cl b/clang/test/CodeGenOpenCLCXX/constexpr.clcpp
similarity index 94%
rename from clang/test/CodeGenOpenCLCXX/constexpr.cl
rename to clang/test/CodeGenOpenCLCXX/constexpr.clcpp
index 4f2ed7d06f0643f2b000bd60239c6722fbd0c19c..7a04d1671274d6806ca08f4aa32a3d991d40c90d 100644
--- a/clang/test/CodeGenOpenCLCXX/constexpr.cl
+++ b/clang/test/CodeGenOpenCLCXX/constexpr.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -O0 -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -O0 -emit-llvm -o - | FileCheck %s
 
 typedef int int2 __attribute__((ext_vector_type(2)));
 typedef int int4 __attribute__((ext_vector_type(4)));
diff --git a/clang/test/CodeGenOpenCLCXX/global_init.cl b/clang/test/CodeGenOpenCLCXX/global_init.clcpp
similarity index 79%
rename from clang/test/CodeGenOpenCLCXX/global_init.cl
rename to clang/test/CodeGenOpenCLCXX/global_init.clcpp
index 9f602beda5bab48cdf022619f08dc3057c507ac2..4e02d4accb62f5da7be1745d56f7982a30ffb340 100644
--- a/clang/test/CodeGenOpenCLCXX/global_init.cl
+++ b/clang/test/CodeGenOpenCLCXX/global_init.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir -emit-llvm -O0 -o - | FileCheck %s
 
 struct S {
   S() {}
diff --git a/clang/test/CodeGenOpenCLCXX/local_addrspace_init.cl b/clang/test/CodeGenOpenCLCXX/local_addrspace_init.clcpp
similarity index 89%
rename from clang/test/CodeGenOpenCLCXX/local_addrspace_init.cl
rename to clang/test/CodeGenOpenCLCXX/local_addrspace_init.clcpp
index e892e674ad14b99f413117a03e71ff69b40602eb..bb9acf09d120be1ec55355b20b6cf887caad56b4 100644
--- a/clang/test/CodeGenOpenCLCXX/local_addrspace_init.cl
+++ b/clang/test/CodeGenOpenCLCXX/local_addrspace_init.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir -emit-llvm -O0 -o - | FileCheck %s
 
 // Test that we don't initialize local address space objects.
 //CHECK: @_ZZ4testE1i = internal addrspace(3) global i32 undef
diff --git a/clang/test/CodeGenOpenCLCXX/method-overload-address-space.cl b/clang/test/CodeGenOpenCLCXX/method-overload-address-space.clcpp
similarity index 91%
rename from clang/test/CodeGenOpenCLCXX/method-overload-address-space.cl
rename to clang/test/CodeGenOpenCLCXX/method-overload-address-space.clcpp
index b170ead38179c063be648479d4fad830ce3ffb28..3cca5c2dc8c5119e8e8360df63b077d259aebbc1 100644
--- a/clang/test/CodeGenOpenCLCXX/method-overload-address-space.cl
+++ b/clang/test/CodeGenOpenCLCXX/method-overload-address-space.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -emit-llvm -O0 -o - | FileCheck %s
+//RUN: %clang_cc1 %s -triple spir-unknown-unknown -emit-llvm -O0 -o - | FileCheck %s
 
 struct C {
   void foo() __local;
diff --git a/clang/test/CodeGenOpenCLCXX/template-address-spaces.cl b/clang/test/CodeGenOpenCLCXX/template-address-spaces.clcpp
similarity index 91%
rename from clang/test/CodeGenOpenCLCXX/template-address-spaces.cl
rename to clang/test/CodeGenOpenCLCXX/template-address-spaces.clcpp
index 2142d387d5580995231272dc4c06ae7dd8dc4a69..d3814204ed99ce73274f9bec785551937caa49d3 100644
--- a/clang/test/CodeGenOpenCLCXX/template-address-spaces.cl
+++ b/clang/test/CodeGenOpenCLCXX/template-address-spaces.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -cl-std=clc++ %s -emit-llvm -o - -O0 -triple spir-unknown-unknown | FileCheck %s
+// RUN: %clang_cc1 %s -emit-llvm -o - -O0 -triple spir-unknown-unknown | FileCheck %s
 
 template <typename T>
 struct S{
diff --git a/clang/test/CodeGenSYCL/convergent.cpp b/clang/test/CodeGenSYCL/convergent.cpp
index 784fb8976c27197741d5bd48d0600621210165b5..58be1b153c93755cfd6b6fbfbae8592fe2b2c21d 100644
--- a/clang/test/CodeGenSYCL/convergent.cpp
+++ b/clang/test/CodeGenSYCL/convergent.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsycl -fsycl-is-device -emit-llvm -disable-llvm-passes \
+// RUN: %clang_cc1 -fsycl-is-device -emit-llvm -disable-llvm-passes \
 // RUN:  -triple spir64-unknown-unknown-sycldevice -emit-llvm %s -o - | \
 // RUN:   FileCheck %s
 
diff --git a/clang/test/CodeGenSYCL/filescope_asm.c b/clang/test/CodeGenSYCL/filescope_asm.c
index 5f4f6709a0e18393be1ac12d73094c7a9708bb5e..3c1c12fd589a6aaba86788e3422af07a3c113ceb 100644
--- a/clang/test/CodeGenSYCL/filescope_asm.c
+++ b/clang/test/CodeGenSYCL/filescope_asm.c
@@ -1,4 +1,4 @@
-// RUN:  %clang_cc1 -fsycl -fsycl-is-device -triple spir64-unknown-unknown-sycldevice -emit-llvm %s -o - | FileCheck %s
+// RUN:  %clang_cc1 -fsycl-is-device -triple spir64-unknown-unknown-sycldevice -emit-llvm %s -o - | FileCheck %s
 //
 // Check that file-scope asm is ignored during device-side SYCL compilation.
 //
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/lib/.keep b/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/hwasan+noexcept/libc++.so
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/lib/.keep
rename to clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/hwasan+noexcept/libc++.so
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/lib/i386-linux-gnu/libtest.so b/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/hwasan/libc++.so
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/lib/i386-linux-gnu/libtest.so
rename to clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/hwasan/libc++.so
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/lib/mips-linux-gnu/.keep b/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/relative-vtables+hwasan+noexcept/libc++.so
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/lib/mips-linux-gnu/.keep
rename to clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/relative-vtables+hwasan+noexcept/libc++.so
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/lib/mipsel-linux-gnu/.keep b/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/relative-vtables+hwasan/libc++.so
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/lib/mipsel-linux-gnu/.keep
rename to clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/relative-vtables+hwasan/libc++.so
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/lib/powerpc-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/lib/aarch64-linux-gnu/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/lib/powerpc-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/lib/aarch64-linux-gnu/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/lib/powerpc64-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/lib32/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/lib/powerpc64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/lib32/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/lib/powerpc64le-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/lib64/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/lib/powerpc64le-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/lib64/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/lib/sparc-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/include/c++/10/aarch64-linux-gnu/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/lib/sparc-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/include/c++/10/aarch64-linux-gnu/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/lib/sparc64-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/include/c++/10/backward/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/lib/sparc64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/include/c++/10/backward/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/lib/crt1.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/lib/crt1.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/lib/crti.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/lib/crti.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/backward/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/lib/crtn.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/backward/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/aarch64-linux-gnu/lib/crtn.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/i686-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/10/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/i686-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/10/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/mips-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/10/backward/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/mips-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/10/backward/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/mipsel-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/x86_64-linux-gnu/c++/10/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/mipsel-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/x86_64-linux-gnu/c++/10/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/powerpc-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/x86_64-linux-gnu/c++/10/32/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/powerpc-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/x86_64-linux-gnu/c++/10/32/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/powerpc64-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/aarch64-linux-gnu/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/powerpc64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/aarch64-linux-gnu/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/i686-linux-gnu/4.5/crtbegin.o b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc-cross/aarch64-linux-gnu/10/crtbegin.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/i686-linux-gnu/4.5/crtbegin.o
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc-cross/aarch64-linux-gnu/10/crtbegin.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/sparc-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc-cross/aarch64-linux-gnu/10/crtbeginT.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/sparc-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc-cross/aarch64-linux-gnu/10/crtbeginT.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/sparc64-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc-cross/aarch64-linux-gnu/10/crtend.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/sparc64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc-cross/aarch64-linux-gnu/10/crtend.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mips-linux-gnu/4.5/64/crtbegin.o b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/10/32/crtbegin.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mips-linux-gnu/4.5/64/crtbegin.o
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/10/32/crtbegin.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mips-linux-gnu/4.5/crtbegin.o b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/10/crtbegin.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mips-linux-gnu/4.5/crtbegin.o
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/10/crtbegin.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/x86_64-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/10/crtend.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/c++/4.5/x86_64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/10/crtend.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/mips-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/i386-linux-gnu/crt1.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/mips-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/i386-linux-gnu/crt1.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/mipsel-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/i386-linux-gnu/crti.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/mipsel-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/i386-linux-gnu/crti.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/powerpc-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/i386-linux-gnu/crtn.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/powerpc-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/i386-linux-gnu/crtn.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/powerpc64-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/x86_64-linux-gnu/crt1.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/powerpc64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/x86_64-linux-gnu/crt1.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/sparc-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/x86_64-linux-gnu/crti.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/sparc-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/x86_64-linux-gnu/crti.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/sparc64-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/x86_64-linux-gnu/crtn.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/sparc64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/x86_64-linux-gnu/crtn.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/x86_64-linux-gnu/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib32/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/include/x86_64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib32/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/.keep b/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib64/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/.keep
rename to clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib64/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mips-linux-gnu/4.5/n32/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabi/crt1.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mips-linux-gnu/4.5/n32/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabi/crt1.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mipsel-linux-gnu/4.5/64/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabi/crti.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mipsel-linux-gnu/4.5/64/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabi/crti.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mipsel-linux-gnu/4.5/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabi/crtn.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mipsel-linux-gnu/4.5/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabi/crtn.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mipsel-linux-gnu/4.5/n32/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabihf/crt1.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/mipsel-linux-gnu/4.5/n32/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabihf/crt1.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/powerpc-linux-gnu/4.5/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabihf/crti.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/powerpc-linux-gnu/4.5/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabihf/crti.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/powerpc64-linux-gnu/4.5/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabihf/crtn.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/powerpc64-linux-gnu/4.5/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/arm-linux-gnueabihf/crtn.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/powerpc64le-linux-gnu/4.5/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabi/crt1.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/powerpc64le-linux-gnu/4.5/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabi/crt1.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/sparc-linux-gnu/4.5/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabi/crti.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/sparc-linux-gnu/4.5/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabi/crti.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/sparc64-linux-gnu/4.5/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabi/crtn.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/sparc64-linux-gnu/4.5/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabi/crtn.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/4.5/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabihf/crt1.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/4.5/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabihf/crt1.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/i386-linux-gnu/.keep b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabihf/crti.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/i386-linux-gnu/.keep
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabihf/crti.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/mips-linux-gnu/.keep b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabihf/crtn.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/mips-linux-gnu/.keep
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/armeb-linux-gnueabihf/crtn.o
diff --git a/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/4.7/32/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/gcc/arm-linux-gnueabi/10/crtbegin.o
similarity index 100%
rename from clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/4.7/32/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/gcc/arm-linux-gnueabi/10/crtbegin.o
diff --git a/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/4.7/crtbegin.o b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/gcc/arm-linux-gnueabihf/10/crtbegin.o
similarity index 100%
rename from clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/4.7/crtbegin.o
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/gcc/arm-linux-gnueabihf/10/crtbegin.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/mipsel-linux-gnu/.keep b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/gcc/armeb-linux-gnueabi/10/crtbegin.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/mipsel-linux-gnu/.keep
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/gcc/armeb-linux-gnueabi/10/crtbegin.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/powerpc-linux-gnu/.keep b/clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/gcc/armeb-linux-gnueabihf/10/crtbegin.o
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/powerpc-linux-gnu/.keep
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/lib/gcc/armeb-linux-gnueabihf/10/crtbegin.o
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/powerpc64-linux-gnu/.keep b/clang/test/Driver/Inputs/multilib_arm_linux_tree/usr/include/arm-linux-gnueabihf/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/powerpc64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/usr/include/arm-linux-gnueabihf/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/powerpc64le-linux-gnu/.keep b/clang/test/Driver/Inputs/multilib_arm_linux_tree/usr/include/armeb-linux-gnueabi/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/powerpc64le-linux-gnu/.keep
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/usr/include/armeb-linux-gnueabi/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/sparc-linux-gnu/.keep b/clang/test/Driver/Inputs/multilib_arm_linux_tree/usr/include/armeb-linux-gnueabihf/.keep
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/sparc-linux-gnu/.keep
rename to clang/test/Driver/Inputs/multilib_arm_linux_tree/usr/include/armeb-linux-gnueabihf/.keep
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/sparc64-linux-gnu/.keep b/clang/test/Driver/Inputs/resource_dir/lib/windows/clang_rt.builtins-x86_64.lib
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/sparc64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/resource_dir/lib/windows/clang_rt.builtins-x86_64.lib
diff --git a/clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/x86_64-linux-gnu/.keep b/clang/test/Driver/Inputs/resource_dir_with_per_target_subdir/lib/x86_64-pc-windows-msvc/clang_rt.builtins.lib
similarity index 100%
rename from clang/test/Driver/Inputs/debian_multiarch_tree/usr/lib/x86_64-linux-gnu/.keep
rename to clang/test/Driver/Inputs/resource_dir_with_per_target_subdir/lib/x86_64-pc-windows-msvc/clang_rt.builtins.lib
diff --git a/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/lib/x86_64-linux-gnu/.keep b/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/lib/x86_64-linux-gnu/.keep
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/include/c++/4.7/backward/.keep b/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/include/c++/4.7/backward/.keep
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/include/x86_64-linux-gnu/c++/4.7/.keep b/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/include/x86_64-linux-gnu/c++/4.7/.keep
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/include/x86_64-linux-gnu/c++/4.7/32/.keep b/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/include/x86_64-linux-gnu/c++/4.7/32/.keep
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/lib/gcc-cross/arm-linux-gnueabihf/4.7/crtbegin.o b/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/lib/gcc-cross/arm-linux-gnueabihf/4.7/crtbegin.o
deleted file mode 100644
index c6cac69265af1e1684d2e3038f8fc90b84c87e9c..0000000000000000000000000000000000000000
--- a/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/lib/gcc-cross/arm-linux-gnueabihf/4.7/crtbegin.o
+++ /dev/null
@@ -1 +0,0 @@
-empty
diff --git a/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/4.7/32/.keep b/clang/test/Driver/Inputs/ubuntu_13.04_multiarch_tree/usr/lib/gcc/x86_64-linux-gnu/4.7/32/.keep
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/clang/test/Driver/Xlinker-args.c b/clang/test/Driver/Xlinker-args.c
index a44957cd8aef11311351cc218d736ffa21b0100b..cb045a1d40ac172e3c7bf1de039d31c170d79812 100644
--- a/clang/test/Driver/Xlinker-args.c
+++ b/clang/test/Driver/Xlinker-args.c
@@ -17,7 +17,7 @@
 // LINUX: "--no-demangle" "-e" "_start" "one" "two" "three" "four" "-z" "five" "-r" {{.*}} "-T" "a.lds"
 
 // Check that we forward '-Xlinker' and '-Wl,' on Windows.
-// RUN: %clang -target i686-pc-win32 -### \
+// RUN: %clang -target i686-pc-win32 -fuse-ld=link -### \
 // RUN:   -Xlinker one -Wl,two %s 2>&1 | \
 // RUN:   FileCheck -check-prefix=WIN %s
 // WIN: link.exe
diff --git a/clang/test/Driver/android-ndk-standalone.cpp b/clang/test/Driver/android-ndk-standalone.cpp
index f60f6f9863cc2120b8dca4b13cc0c52b8431aafc..4fe75c6b0f09617dee10858fde35e6c607306cf0 100644
--- a/clang/test/Driver/android-ndk-standalone.cpp
+++ b/clang/test/Driver/android-ndk-standalone.cpp
@@ -4,14 +4,14 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi21 \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck  %s
 // CHECK: {{.*}}clang{{.*}}" "-cc1"
 // CHECK: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK: "-internal-isystem" "{{.*}}/include/c++/v1"
-// CHECK: "-internal-isystem" "{{.*}}/sysroot/usr/local/include"
 // CHECK: "-internal-isystem" "[[RESOURCE_DIR]]{{(/|\\\\)}}include"
+// CHECK: "-internal-isystem" "{{.*}}/sysroot/usr/local/include"
 // CHECK: "-internal-externc-isystem" "{{.*}}/sysroot/usr/include/arm-linux-androideabi"
 // CHECK: "-internal-externc-isystem" "{{.*}}/sysroot/include"
 // CHECK: "-internal-externc-isystem" "{{.*}}/sysroot/usr/include"
@@ -35,7 +35,7 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi14 \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-14 %s
 // CHECK-14: "-L{{.*}}/sysroot/usr/lib/arm-linux-androideabi/14"
@@ -43,7 +43,7 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi21 -stdlib=libstdc++ \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-STDCXX %s
 // CHECK-STDCXX: {{.*}}clang{{.*}}" "-cc1"
@@ -77,14 +77,14 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target armv7a-none-linux-androideabi21 \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck  --check-prefix=CHECK-ARMV7 %s
 // CHECK-ARMV7: {{.*}}clang{{.*}}" "-cc1"
 // CHECK-ARMV7: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-ARMV7: "-internal-isystem" "{{.*}}/include/c++/v1"
-// CHECK-ARMV7: "-internal-isystem" "{{.*}}/sysroot/usr/local/include"
 // CHECK-ARMV7: "-internal-isystem" "[[RESOURCE_DIR]]{{(/|\\\\)}}include"
+// CHECK-ARMV7: "-internal-isystem" "{{.*}}/sysroot/usr/local/include"
 // CHECK-ARMV7: "-internal-externc-isystem" "{{.*}}/sysroot/usr/include/arm-linux-androideabi"
 // CHECK-ARMV7: "-internal-externc-isystem" "{{.*}}/sysroot/include"
 // CHECK-ARMV7: "-internal-externc-isystem" "{{.*}}/sysroot/usr/include"
@@ -110,19 +110,19 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi21 \
 // RUN:     -march=armv7 \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck  --check-prefix=CHECK-ARMV7 %s
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi21 \
 // RUN:     -march=armv7a \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck  --check-prefix=CHECK-ARMV7 %s
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi21 \
 // RUN:     -march=armv7-a \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck  --check-prefix=CHECK-ARMV7 %s
 //
@@ -130,14 +130,14 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi21 \
 // RUN:     -mthumb \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck  --check-prefix=CHECK-THUMB %s
 // CHECK-THUMB: {{.*}}clang{{.*}}" "-cc1"
 // CHECK-THUMB: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-THUMB: "-internal-isystem" "{{.*}}/include/c++/v1"
-// CHECK-THUMB: "-internal-isystem" "{{.*}}/sysroot/usr/local/include"
 // CHECK-THUMB: "-internal-isystem" "[[RESOURCE_DIR]]{{(/|\\\\)}}include"
+// CHECK-THUMB: "-internal-isystem" "{{.*}}/sysroot/usr/local/include"
 // CHECK-THUMB: "-internal-externc-isystem" "{{.*}}/sysroot/usr/include/arm-linux-androideabi"
 // CHECK-THUMB: "-internal-externc-isystem" "{{.*}}/sysroot/include"
 // CHECK-THUMB: "-internal-externc-isystem" "{{.*}}/sysroot/usr/include"
@@ -164,14 +164,14 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi21 \
 // RUN:     -march=armv7-a -mthumb \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck  --check-prefix=CHECK-ARMV7THUMB %s
 // CHECK-ARMV7THUMB: {{.*}}clang{{.*}}" "-cc1"
 // CHECK-ARMV7THUMB: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-ARMV7THUMB: "-internal-isystem" "{{.*}}/include/c++/v1"
-// CHECK-ARMV7THUMB: "-internal-isystem" "{{.*}}/sysroot/usr/local/include"
 // CHECK-ARMV7THUMB: "-internal-isystem" "[[RESOURCE_DIR]]{{(/|\\\\)}}include"
+// CHECK-ARMV7THUMB: "-internal-isystem" "{{.*}}/sysroot/usr/local/include"
 // CHECK-ARMV7THUMB: "-internal-externc-isystem" "{{.*}}/sysroot/usr/include/arm-linux-androideabi"
 // CHECK-ARMV7THUMB: "-internal-externc-isystem" "{{.*}}/sysroot/include"
 // CHECK-ARMV7THUMB: "-internal-externc-isystem" "{{.*}}/sysroot/usr/include"
@@ -196,7 +196,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi21 \
 // RUN:     -march=armv7-a -mthumb \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:     -print-multi-lib \
 // RUN:   | FileCheck  --check-prefix=CHECK-ARM-MULTILIBS %s
@@ -210,13 +210,13 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target armv7a-none-linux-androideabi21 \
 // RUN:     -mthumb \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck  --check-prefix=CHECK-ARMV7THUMB %s
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target aarch64-linux-android21 \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-AARCH64 %s
 // CHECK-AARCH64: {{.*}}clang{{.*}}" "-cc1"
@@ -232,7 +232,7 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm64-linux-android21 \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-ARM64 %s
 // CHECK-ARM64: {{.*}}clang{{.*}}" "-cc1"
@@ -249,7 +249,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target mipsel-linux-android21 \
 // RUN:     -mips32 \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-MIPS %s
 // CHECK-MIPS: {{.*}}clang{{.*}}" "-cc1"
@@ -264,7 +264,7 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target i686-linux-android21 \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-I686 %s
 // CHECK-I686: {{.*}}clang{{.*}}" "-cc1"
@@ -280,7 +280,7 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target x86_64-linux-android21 \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-X86_64 %s
 // CHECK-X86_64: {{.*}}clang{{.*}}" "-cc1"
diff --git a/clang/test/Driver/android-standalone.cpp b/clang/test/Driver/android-standalone.cpp
index 0f8cf0b1355e0aaaf3edd4d1d569d29a8a00514c..c238fc734716930d37cb9adc3a892b380d5aee70 100644
--- a/clang/test/Driver/android-standalone.cpp
+++ b/clang/test/Driver/android-standalone.cpp
@@ -3,7 +3,7 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm-linux-androideabi -stdlib=libstdc++ \
-// RUN:     -B%S/Inputs/basic_android_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
 // RUN:   | FileCheck  %s
 // CHECK: {{.*}}clang{{.*}}" "-cc1"
@@ -18,7 +18,7 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target aarch64-linux-android -stdlib=libstdc++ \
-// RUN:     -B%S/Inputs/basic_android_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-AARCH64 %s
 // CHECK-AARCH64: {{.*}}clang{{.*}}" "-cc1"
@@ -33,7 +33,7 @@
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target arm64-linux-android -stdlib=libstdc++ \
-// RUN:     -B%S/Inputs/basic_android_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-ARM64 %s
 // CHECK-ARM64: {{.*}}clang{{.*}}" "-cc1"
@@ -49,7 +49,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target mipsel-linux-android \
 // RUN:     -mips32 -stdlib=libstdc++ \
-// RUN:     -B%S/Inputs/basic_android_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-MIPS %s
 // CHECK-MIPS: {{.*}}clang{{.*}}" "-cc1"
@@ -65,7 +65,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target mipsel-linux-android \
 // RUN:     -march=mips32 -mips32r2 -stdlib=libstdc++ \
-// RUN:     -B%S/Inputs/basic_android_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-MIPSR2 %s
 // CHECK-MIPSR2: {{.*}}clang{{.*}}" "-cc1"
@@ -81,7 +81,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target mipsel-linux-android \
 // RUN:     -mips32 -march=mips32r2 -stdlib=libstdc++ \
-// RUN:     -B%S/Inputs/basic_android_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
 // RUN:   | FileCheck --check-prefix=CHECK-MIPSR2-A %s
 // CHECK-MIPSR2-A: {{.*}}clang{{.*}}" "-cc1"
diff --git a/clang/test/Driver/arm-multilibs.c b/clang/test/Driver/arm-multilibs.c
index bd9c80e8b16ac910f52dc4a0b833ea91f5e4694c..3ec9ea0b97c52727949a165de49c260aeeb82f8f 100644
--- a/clang/test/Driver/arm-multilibs.c
+++ b/clang/test/Driver/arm-multilibs.c
@@ -1,14 +1,14 @@
-// RUN: %clang -target armv7-linux-gnueabi --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s -o /dev/null 2>&1 | FileCheck -check-prefix CHECK-ARM %s
-// RUN: %clang -target thumbv7-linux-gnueabi --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s -o /dev/null 2>&1 | FileCheck -check-prefix CHECK-ARM %s
+// RUN: %clang --target=armv7-linux-gnueabi --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARM %s
+// RUN: %clang --target=thumbv7-linux-gnueabi --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARM %s
 
-// RUN: %clang -target armv7-linux-gnueabihf --sysroot=%S/Inputs/multilib_armhf_linux_tree -### -c %s -o /dev/null 2>&1 | FileCheck -check-prefix CHECK-ARMHF %s
-// RUN: %clang -target thumbv7-linux-gnueabihf --sysroot=%S/Inputs/multilib_armhf_linux_tree -### -c %s -o /dev/null 2>&1 | FileCheck -check-prefix CHECK-ARMHF %s
+// RUN: %clang --target=armv7-linux-gnueabihf --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARMHF %s
+// RUN: %clang --target=thumbv7-linux-gnueabihf --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARMHF %s
 
-// RUN: %clang -target armv7eb-linux-gnueabi --sysroot=%S/Inputs/multilib_armeb_linux_tree -### -c %s -o /dev/null 2>&1 | FileCheck -check-prefix CHECK-ARMEB %s
-// RUN: %clang -target thumbv7eb-linux-gnueabi --sysroot=%S/Inputs/multilib_armeb_linux_tree -### -c %s -o /dev/null 2>&1 | FileCheck -check-prefix CHECK-ARMEB %s
+// RUN: %clang --target=armv7eb-linux-gnueabi --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARMEB %s
+// RUN: %clang --target=thumbv7eb-linux-gnueabi --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARMEB %s
 
-// RUN: %clang -target armv7eb-linux-gnueabihf --sysroot=%S/Inputs/multilib_armebhf_linux_tree -### -c %s -o /dev/null 2>&1 | FileCheck -check-prefix CHECK-ARMEBHF %s
-// RUN: %clang -target thumbv7eb-linux-gnueabihf --sysroot=%S/Inputs/multilib_armebhf_linux_tree -### -c %s -o /dev/null 2>&1 | FileCheck -check-prefix CHECK-ARMEBHF %s
+// RUN: %clang --target=armv7eb-linux-gnueabihf --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARMEBHF %s
+// RUN: %clang --target=thumbv7eb-linux-gnueabihf --sysroot=%S/Inputs/multilib_arm_linux_tree -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARMEBHF %s
 
 // CHECK-ARM: "-internal-externc-isystem" "{{.*}}/usr/include/arm-linux-gnueabi"
 // CHECK-ARMHF: "-internal-externc-isystem" "{{.*}}/usr/include/arm-linux-gnueabihf"
diff --git a/clang/test/Driver/cl-inputs.c b/clang/test/Driver/cl-inputs.c
index 59455a0aa5e5c7e06a4f1bb673c9ce6fd43d6f09..8eb44517ee167a8bfcc087c3121fea1744a3912c 100644
--- a/clang/test/Driver/cl-inputs.c
+++ b/clang/test/Driver/cl-inputs.c
@@ -50,16 +50,16 @@
 // RUN: %clang_cl -### /Tc - 2>&1 | FileCheck -check-prefix=STDINTc %s
 // STDINTc: "-x" "c"
 
-// RUN: env LIB=%S/Inputs/cl-libs %clang_cl -### -- %s cl-test.lib 2>&1 | FileCheck -check-prefix=LIBINPUT %s
+// RUN: env LIB=%S/Inputs/cl-libs %clang_cl -fuse-ld=link -### -- %s cl-test.lib 2>&1 | FileCheck -check-prefix=LIBINPUT %s
 // LIBINPUT: link.exe"
 // LIBINPUT: "cl-test.lib"
 
-// RUN: env LIB=%S/Inputs/cl-libs %clang_cl -### -- %s cl-test2.lib 2>&1 | FileCheck -check-prefix=LIBINPUT2 %s
+// RUN: env LIB=%S/Inputs/cl-libs %clang_cl -fuse-ld=link -### -- %s cl-test2.lib 2>&1 | FileCheck -check-prefix=LIBINPUT2 %s
 // LIBINPUT2: error: no such file or directory: 'cl-test2.lib'
 // LIBINPUT2: link.exe"
 // LIBINPUT2-NOT: "cl-test2.lib"
 
-// RUN: %clang_cl -### -- %s /nonexisting.lib 2>&1 | FileCheck -check-prefix=LIBINPUT3 %s
+// RUN: %clang_cl -fuse-ld=link -### -- %s /nonexisting.lib 2>&1 | FileCheck -check-prefix=LIBINPUT3 %s
 // LIBINPUT3: error: no such file or directory: '/nonexisting.lib'
 // LIBINPUT3: link.exe"
 // LIBINPUT3-NOT: "/nonexisting.lib"
diff --git a/clang/test/Driver/cl-link-at-file.c b/clang/test/Driver/cl-link-at-file.c
index 50ae07fadf5bf0beb11c4eff09b5d6fc2fd65e7c..4e665f89b74e1d0e5d3ffeac8f73fbaf8f533f53 100644
--- a/clang/test/Driver/cl-link-at-file.c
+++ b/clang/test/Driver/cl-link-at-file.c
@@ -7,7 +7,7 @@
 
 // RUN: echo /link bar.lib baz.lib > %t.args
 // RUN: touch %t.obj
-// RUN: %clang_cl -### @%t.args -- %t.obj 2>&1 | FileCheck %s -check-prefix=ARGS
+// RUN: %clang_cl -fuse-ld=link -### @%t.args -- %t.obj 2>&1 | FileCheck %s -check-prefix=ARGS
 // If the "/link" option captures all remaining args beyond its response file,
 // it will also capture "--" and our input argument. In this case, Clang will
 // be clueless and will emit "argument unused" warnings. If PR17239 is properly
diff --git a/clang/test/Driver/cl-link.c b/clang/test/Driver/cl-link.c
index 142725fed8eb2ee1b0265460495a4eb3fd632224..e2f5397e913393bd10fafbfba9b66c518b732be9 100644
--- a/clang/test/Driver/cl-link.c
+++ b/clang/test/Driver/cl-link.c
@@ -2,14 +2,14 @@
 // be interpreted as a command-line option, e.g. on Mac where %s is commonly
 // under /Users.
 
-// RUN: %clang_cl /Tc%s -### /link foo bar baz 2>&1 | FileCheck --check-prefix=LINK %s
-// RUN: %clang_cl /Tc%s -### /linkfoo bar baz 2>&1 | FileCheck --check-prefix=LINK %s
+// RUN: %clang_cl /Tc%s -fuse-ld=link -### /link foo bar baz 2>&1 | FileCheck --check-prefix=LINK %s
+// RUN: %clang_cl /Tc%s -fuse-ld=link -### /linkfoo bar baz 2>&1 | FileCheck --check-prefix=LINK %s
 // LINK: link.exe
 // LINK: "foo"
 // LINK: "bar"
 // LINK: "baz"
 
-// RUN: %clang_cl -m32 -arch:IA32 --target=i386-pc-win32 /Tc%s -### -fsanitize=address 2>&1 | FileCheck --check-prefix=ASAN %s
+// RUN: %clang_cl -m32 -arch:IA32 --target=i386-pc-win32 /Tc%s -fuse-ld=link -### -fsanitize=address 2>&1 | FileCheck --check-prefix=ASAN %s
 // ASAN: link.exe
 // ASAN: "-debug"
 // ASAN: "-incremental:no"
@@ -19,7 +19,7 @@
 // ASAN: "-wholearchive:{{.*}}clang_rt.asan_cxx-i386.lib"
 // ASAN: "{{.*}}cl-link{{.*}}.obj"
 
-// RUN: %clang_cl -m32 -arch:IA32 --target=i386-pc-win32 /MD /Tc%s -### -fsanitize=address 2>&1 | FileCheck --check-prefix=ASAN-MD %s
+// RUN: %clang_cl -m32 -arch:IA32 --target=i386-pc-win32 /MD /Tc%s -fuse-ld=link -### -fsanitize=address 2>&1 | FileCheck --check-prefix=ASAN-MD %s
 // ASAN-MD: link.exe
 // ASAN-MD: "-debug"
 // ASAN-MD: "-incremental:no"
@@ -29,13 +29,13 @@
 // ASAN-MD: "-wholearchive:{{.*}}clang_rt.asan_dynamic_runtime_thunk-i386.lib"
 // ASAN-MD: "{{.*}}cl-link{{.*}}.obj"
 
-// RUN: %clang_cl /LD -### /Tc%s 2>&1 | FileCheck --check-prefix=DLL %s
-// RUN: %clang_cl /LDd -### /Tc%s 2>&1 | FileCheck --check-prefix=DLL %s
+// RUN: %clang_cl /LD -fuse-ld=link -### /Tc%s 2>&1 | FileCheck --check-prefix=DLL %s
+// RUN: %clang_cl /LDd -fuse-ld=link -### /Tc%s 2>&1 | FileCheck --check-prefix=DLL %s
 // DLL: link.exe
 // "-dll"
 
-// RUN: %clang_cl -m32 -arch:IA32 --target=i386-pc-win32 /LD /Tc%s -### -fsanitize=address 2>&1 | FileCheck --check-prefix=ASAN-DLL %s
-// RUN: %clang_cl -m32 -arch:IA32 --target=i386-pc-win32 /LDd /Tc%s -### -fsanitize=address 2>&1 | FileCheck --check-prefix=ASAN-DLL %s
+// RUN: %clang_cl -m32 -arch:IA32 --target=i386-pc-win32 /LD /Tc%s -fuse-ld=link -### -fsanitize=address 2>&1 | FileCheck --check-prefix=ASAN-DLL %s
+// RUN: %clang_cl -m32 -arch:IA32 --target=i386-pc-win32 /LDd /Tc%s -fuse-ld=link -### -fsanitize=address 2>&1 | FileCheck --check-prefix=ASAN-DLL %s
 // ASAN-DLL: link.exe
 // ASAN-DLL: "-dll"
 // ASAN-DLL: "-debug"
@@ -43,13 +43,13 @@
 // ASAN-DLL: "{{.*}}clang_rt.asan_dll_thunk-i386.lib"
 // ASAN-DLL: "{{.*}}cl-link{{.*}}.obj"
 
-// RUN: %clang_cl /Zi /Tc%s -### 2>&1 | FileCheck --check-prefix=DEBUG %s
+// RUN: %clang_cl /Zi /Tc%s -fuse-ld=link -### 2>&1 | FileCheck --check-prefix=DEBUG %s
 // DEBUG: link.exe
 // DEBUG: "-debug"
 
 // PR27234
-// RUN: %clang_cl /Tc%s nonexistent.obj -### /link /libpath:somepath 2>&1 | FileCheck --check-prefix=NONEXISTENT %s
-// RUN: %clang_cl /Tc%s nonexistent.lib -### /link /libpath:somepath 2>&1 | FileCheck --check-prefix=NONEXISTENT %s
+// RUN: %clang_cl /Tc%s nonexistent.obj -fuse-ld=link -### /link /libpath:somepath 2>&1 | FileCheck --check-prefix=NONEXISTENT %s
+// RUN: %clang_cl /Tc%s nonexistent.lib -fuse-ld=link -### /link /libpath:somepath 2>&1 | FileCheck --check-prefix=NONEXISTENT %s
 // NONEXISTENT-NOT: no such file
 // NONEXISTENT: link.exe
 // NONEXISTENT: "/libpath:somepath"
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index 7d83b3d60b1ecfd4fc2e59876e761b9936161f79..90f865d9c7c03eaefc4afdff5b3fb90d5820ec76 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -643,6 +643,7 @@
 // RUN:     -fno-diagnostics-color \
 // RUN:     -fdebug-compilation-dir . \
 // RUN:     -fdebug-compilation-dir=. \
+// RUN:     -ffile-compilation-dir=. \
 // RUN:     -fdiagnostics-parseable-fixits \
 // RUN:     -fdiagnostics-absolute-paths \
 // RUN:     -ferror-limit=10 \
diff --git a/clang/test/Driver/clang_f_opts.c b/clang/test/Driver/clang_f_opts.c
index b383579f107937a93554bae98c912792b296b9de..12369361f8115c3b7526093898bee59acbe537bc 100644
--- a/clang/test/Driver/clang_f_opts.c
+++ b/clang/test/Driver/clang_f_opts.c
@@ -3,7 +3,7 @@
 // RUN: %clang -### -S -fasm -fblocks -fbuiltin -fno-math-errno -fcommon -fpascal-strings -fno-blocks -fno-builtin -fmath-errno -fno-common -fno-pascal-strings -fblocks -fbuiltin -fmath-errno -fcommon -fpascal-strings -fsplit-stack %s 2>&1 | FileCheck -check-prefix=CHECK-OPTIONS1 %s
 // RUN: %clang -### -S -fasm -fblocks -fbuiltin -fno-math-errno -fcommon -fpascal-strings -fno-asm -fno-blocks -fno-builtin -fmath-errno -fno-common -fno-pascal-strings -fno-show-source-location -fshort-enums %s 2>&1 | FileCheck -check-prefix=CHECK-OPTIONS2 %s
 
-// CHECK-OPTIONS1: -split-stacks
+// CHECK-OPTIONS1: -fsplit-stack
 // CHECK-OPTIONS1: -fgnu-keywords
 // CHECK-OPTIONS1: -fblocks
 // CHECK-OPTIONS1: -fpascal-strings
diff --git a/clang/test/Driver/cross-linux.c b/clang/test/Driver/cross-linux.c
index 6c2dab2606958af6e40adc677b9b8180a420c898..f54df697b159d074267f77c690a9a2ef2d178c1e 100644
--- a/clang/test/Driver/cross-linux.c
+++ b/clang/test/Driver/cross-linux.c
@@ -63,7 +63,6 @@
 // CHECK-MULTI32-X86-64: "crti.o" "[[gcc_install:.*/Inputs/multilib_32bit_linux_tree/usr/lib/gcc/i386-unknown-linux/4.6.0]]/64{{/|\\\\}}crtbegin.o"
 // CHECK-MULTI32-X86-64: "-L[[gcc_install]]/64"
 // CHECK-MULTI32-X86-64: "-L[[gcc_install]]/../../../../i386-unknown-linux/lib/../lib64"
-// CHECK-MULTI32-X86-64: "-L[[gcc_install]]"
 // CHECK-MULTI32-X86-64: "-L[[gcc_install]]/../../../../i386-unknown-linux/lib"
 // CHECK-MULTI32-X86-64: "-L[[sysroot]]/lib"
 // CHECK-MULTI32-X86-64: "-L[[sysroot]]/usr/lib"
@@ -82,7 +81,6 @@
 // CHECK-MULTI64-I386: "crti.o" "[[gcc_install:.*/Inputs/multilib_64bit_linux_tree/usr/lib/gcc/x86_64-unknown-linux/4.6.0]]/32{{/|\\\\}}crtbegin.o"
 // CHECK-MULTI64-I386: "-L[[gcc_install]]/32"
 // CHECK-MULTI64-I386: "-L[[gcc_install]]/../../../../x86_64-unknown-linux/lib/../lib32"
-// CHECK-MULTI64-I386: "-L[[gcc_install]]"
 // CHECK-MULTI64-I386: "-L[[gcc_install]]/../../../../x86_64-unknown-linux/lib"
 // CHECK-MULTI64-I386: "-L[[sysroot]]/lib"
 // CHECK-MULTI64-I386: "-L[[sysroot]]/usr/lib"
diff --git a/clang/test/Driver/cxx_for_opencl.clcpp b/clang/test/Driver/cxx_for_opencl.clcpp
new file mode 100644
index 0000000000000000000000000000000000000000..e3e89c53b888646e5a1d3f14bbc0031ec3b2280b
--- /dev/null
+++ b/clang/test/Driver/cxx_for_opencl.clcpp
@@ -0,0 +1,18 @@
+// RUN: %clang %s -Xclang -verify -fsyntax-only
+// RUN: %clang %s -cl-std=clc++ -Xclang -verify -fsyntax-only
+// RUN: %clang %s -cl-std=cl2.0 -Xclang -verify -fsyntax-only
+// RUN: %clang %s -### 2>&1 | FileCheck %s
+
+// CHECK: "-x" "clcpp"
+
+#ifdef __OPENCL_CPP_VERSION__
+//expected-no-diagnostics
+#endif
+
+kernel void k(){
+  auto a = get_local_id(1);
+#ifndef __OPENCL_CPP_VERSION__
+//expected-error@-2{{OpenCL C version 2.0 does not support the 'auto' storage class specifier}}
+//expected-warning@-3{{type specifier missing, defaults to 'int'}}
+#endif
+}
diff --git a/clang/test/Driver/darwin-ld-lto.c b/clang/test/Driver/darwin-ld-lto.c
index 05e6bccd00f0899e7564c695ff5d86f6868beac9..252ca148c52003613f8ff7ae4948d82f1d2fb8dc 100644
--- a/clang/test/Driver/darwin-ld-lto.c
+++ b/clang/test/Driver/darwin-ld-lto.c
@@ -30,3 +30,11 @@
 // THIN_LTO_OBJECT_PATH: {{ld(.exe)?"}}
 // THIN_LTO_OBJECT_PATH-SAME: "-object_path_lto"
 // THIN_LTO_OBJECT_PATH-SAME: {{thinlto\-[a-zA-Z0-9_]+}}
+
+
+// Check that we pass through -fglobal-isel flags to libLTO.
+// RUN: %clang -target arm64-apple-darwin %s -flto -fglobal-isel -### 2>&1 | \
+// RUN:   FileCheck --check-prefix=GISEL %s
+// GISEL: {{ld(.exe)?"}}
+// GISEL: "-mllvm" "-global-isel"
+// GISEL: "-mllvm" "-global-isel-abort=0"
diff --git a/clang/test/Driver/env.c b/clang/test/Driver/env.c
index 0371bc91c4a30ac0122f99981ee83f42af57b2b2..eef3d0de5a2da6d3c20ba5d9fcbf2204036971af 100644
--- a/clang/test/Driver/env.c
+++ b/clang/test/Driver/env.c
@@ -23,6 +23,5 @@
 // CHECK-LD-32: "{{.*}}/usr/lib/gcc/i386-unknown-linux/4.6.0{{/|\\\\}}crtbegin.o"
 // CHECK-LD-32: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0"
 // CHECK-LD-32: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0/../../../../i386-unknown-linux/lib"
-// CHECK-LD-32: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0/../../.."
 // CHECK-LD-32: "-L[[SYSROOT]]/lib"
 // CHECK-LD-32: "-L[[SYSROOT]]/usr/lib"
diff --git a/clang/test/Driver/fdirect-access-external-data.c b/clang/test/Driver/fdirect-access-external-data.c
index c3fc93064179c0af583f8ad2e83532e9f238cedd..f132b1b088af35dc4e11657ed2f4c3fff41989b9 100644
--- a/clang/test/Driver/fdirect-access-external-data.c
+++ b/clang/test/Driver/fdirect-access-external-data.c
@@ -9,10 +9,6 @@
 // RUN: %clang -### -c -target aarch64 %s -fpic 2>&1 | FileCheck %s --check-prefix=DEFAULT
 // RUN: %clang -### -c -target aarch64 %s -fpic -fdirect-access-external-data 2>&1 | FileCheck %s --check-prefix=DIRECT
 
-/// -m[no-]pie-copy-relocations are aliases for compatibility.
-// RUN: %clang -### -c -target riscv64 %s -mno-pie-copy-relocations 2>&1 | FileCheck %s --check-prefix=INDIRECT
-// RUN: %clang -### -c -target riscv64 %s -fpic -mpie-copy-relocations 2>&1 | FileCheck %s --check-prefix=DIRECT
-
 // DEFAULT-NOT: direct-access-external-data"
 // DIRECT:      "-fdirect-access-external-data"
 // INDIRECT:    "-fno-direct-access-external-data"
diff --git a/clang/test/Driver/fpatchable-function-entry.c b/clang/test/Driver/fpatchable-function-entry.c
index 5ac262c1a46dc14cae8f3ea241dc6b7a423b05b6..da7370a4d87a7dfaf6e85504ea1340fa0c2981dd 100644
--- a/clang/test/Driver/fpatchable-function-entry.c
+++ b/clang/test/Driver/fpatchable-function-entry.c
@@ -2,6 +2,8 @@
 // RUN: %clang -target x86_64 %s -fpatchable-function-entry=1 -c -### 2>&1 | FileCheck %s
 // RUN: %clang -target aarch64 %s -fpatchable-function-entry=1 -c -### 2>&1 | FileCheck %s
 // RUN: %clang -target aarch64 %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s
+// RUN: %clang -target riscv32 %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s
+// RUN: %clang -target riscv64 %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s
 // CHECK: "-fpatchable-function-entry=1"
 
 // RUN: %clang -target aarch64 -fsyntax-only %s -fpatchable-function-entry=1,1 -c -### 2>&1 | FileCheck --check-prefix=11 %s
diff --git a/clang/test/Driver/fuchsia.cpp b/clang/test/Driver/fuchsia.cpp
index 6b288170a1fcabbe9b72c720183c1f3f2005a354..9177f48fc9e29eb1903c5eaaf402a11ccf720e9c 100644
--- a/clang/test/Driver/fuchsia.cpp
+++ b/clang/test/Driver/fuchsia.cpp
@@ -121,6 +121,26 @@
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
 // RUN:     -fuse-ld=lld 2>&1\
 // RUN:     | FileCheck %s -check-prefixes=CHECK-MULTILIB-X86
+// RUN: %clangxx %s -### --target=x86_64-fuchsia -fsanitize=hwaddress \
+// RUN:     -ccc-install-dir %S/Inputs/basic_fuchsia_tree/bin \
+// RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:     -fuse-ld=lld 2>&1\
+// RUN:     | FileCheck %s -check-prefixes=CHECK-MULTILIB-X86,CHECK-MULTILIB-HWASAN-X86
+// RUN: %clangxx %s -### --target=x86_64-fuchsia -fsanitize=hwaddress -fno-exceptions \
+// RUN:     -ccc-install-dir %S/Inputs/basic_fuchsia_tree/bin \
+// RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:     -fuse-ld=lld 2>&1\
+// RUN:     | FileCheck %s -check-prefixes=CHECK-MULTILIB-X86,CHECK-MULTILIB-HWASAN-NOEXCEPT-X86
+// RUN: %clangxx %s -### --target=x86_64-fuchsia -fexperimental-relative-c++-abi-vtables -fsanitize=hwaddress \
+// RUN:     -ccc-install-dir %S/Inputs/basic_fuchsia_tree/bin \
+// RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:     -fuse-ld=lld 2>&1\
+// RUN:     | FileCheck %s -check-prefixes=CHECK-MULTILIB-X86,CHECK-MULTILIB-RELATIVE-VTABLES-HWASAN-X86
+// RUN: %clangxx %s -### --target=x86_64-fuchsia -fexperimental-relative-c++-abi-vtables -fno-exceptions -fsanitize=hwaddress \
+// RUN:     -ccc-install-dir %S/Inputs/basic_fuchsia_tree/bin \
+// RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:     -fuse-ld=lld 2>&1\
+// RUN:     | FileCheck %s -check-prefixes=CHECK-MULTILIB-X86,CHECK-MULTILIB-RELATIVE-VTABLES-HWASAN-NOEXCEPT-X86
 // CHECK-MULTILIB-X86: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-MULTILIB-ASAN-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}asan"
 // CHECK-MULTILIB-NOEXCEPT-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}noexcept"
@@ -129,4 +149,8 @@
 // CHECK-MULTILIB-RELATIVE-VTABLES-NOEXCEPT-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}relative-vtables+noexcept"
 // CHECK-MULTILIB-RELATIVE-VTABLES-ASAN-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}relative-vtables+asan"
 // CHECK-MULTILIB-RELATIVE-VTABLES-ASAN-NOEXCEPT-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}relative-vtables+asan+noexcept"
+// CHECK-MULTILIB-HWASAN-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}hwasan"
+// CHECK-MULTILIB-HWASAN-NOEXCEPT-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}hwasan+noexcept"
+// CHECK-MULTILIB-RELATIVE-VTABLES-HWASAN-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}relative-vtables+hwasan"
+// CHECK-MULTILIB-RELATIVE-VTABLES-HWASAN-NOEXCEPT-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}relative-vtables+hwasan+noexcept"
 // CHECK-MULTILIB-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++"
diff --git a/clang/test/Driver/gcc-toolchain.cpp b/clang/test/Driver/gcc-toolchain.cpp
index 6c872f4255c34b5c451cc90647a36f364b02e1fc..7cdba0841b8cd1704d52d22ee3febb89e55624eb 100644
--- a/clang/test/Driver/gcc-toolchain.cpp
+++ b/clang/test/Driver/gcc-toolchain.cpp
@@ -1,31 +1,39 @@
 // Test that gcc-toolchain option is working correctly
 //
-// RUN: %clangxx -no-canonical-prefixes %s -### -o %t 2>&1 \
-// RUN:     --target=i386-unknown-linux -stdlib=libstdc++ \
-// RUN:     --gcc-toolchain=%S/Inputs/ubuntu_11.04_multiarch_tree/usr \
-// RUN:     --sysroot="" \
-// RUN:   | FileCheck %s
+/// Without --rtlib=libgcc the driver may pick clang_rt.crtbegin.o if
+/// -DCLANG_DEFAULT_RTLIB=compiler-rt.
+// RUN: %clangxx %s -### --target=x86_64-linux-gnu --sysroot= \
+// RUN:   --gcc-toolchain=%S/Inputs/ubuntu_14.04_multiarch_tree/usr -stdlib=libstdc++ --rtlib=libgcc 2>&1 | \
+// RUN:   FileCheck %s
 //
 // Additionally check that the legacy spelling of the flag works.
-// RUN: %clangxx -no-canonical-prefixes %s -### -o %t 2>&1 \
-// RUN:     --target=i386-unknown-linux -stdlib=libstdc++ \
-// RUN:     -gcc-toolchain %S/Inputs/ubuntu_11.04_multiarch_tree/usr \
-// RUN:     --sysroot="" \
-// RUN:   | FileCheck %s
+// RUN: %clangxx %s -### --target=x86_64-linux-gnu --sysroot= \
+// RUN:   -gcc-toolchain %S/Inputs/ubuntu_14.04_multiarch_tree/usr -stdlib=libstdc++ --rtlib=libgcc 2>&1 | \
+// RUN:   FileCheck %s
 //
 // Test for header search toolchain detection.
 // CHECK: "-internal-isystem"
-// CHECK: "[[TOOLCHAIN:[^"]+]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5/../../../../../include/c++/4.5"
+// CHECK: "[[TOOLCHAIN:[^"]+]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8"
 // CHECK: "-internal-isystem"
-// CHECK: "[[TOOLCHAIN]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5/../../../../../include/c++/4.5/i686-linux-gnu"
+// CHECK: "[[TOOLCHAIN]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/x86_64-linux-gnu/c++/4.8"
 // CHECK: "-internal-isystem"
-// CHECK: "[[TOOLCHAIN]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5/../../../../../include/c++/4.5/backward"
-// CHECK: "-internal-isystem" "/usr/local/include"
+// CHECK: "[[TOOLCHAIN]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8/backward"
 //
 // Test for linker toolchain detection. Note that only the '-L' flags will use
 // the same precise formatting of the path as the '-internal-system' flags
 // above, so we just blanket wildcard match the 'crtbegin.o'.
 // CHECK: "{{[^"]*}}ld{{(.exe)?}}"
-// CHECK: "{{[^"]*}}/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK: "-L[[TOOLCHAIN]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5"
-// CHECK: "-L[[TOOLCHAIN]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5/../../../.."
+// CHECK-SAME: "{{[^"]*}}/usr/lib/gcc/x86_64-linux-gnu/4.8{{/|\\\\}}crtbegin.o"
+// CHECK-SAME: "-L[[TOOLCHAIN]]/usr/lib/gcc/x86_64-linux-gnu/4.8"
+/// On x86_64, there is an extra usr/lib/gcc/x86_64-linux-gnu/4.8/../../../x86_64-linux-gnu but we should not test it.
+
+/// Test we don't detect GCC installation under -B.
+// RUN: %clangxx %s -### --sysroot= 2>&1 \
+// RUN:   --target=aarch64-suse-linux --gcc-toolchain=%S/Inputs/opensuse_42.2_aarch64_tree/usr | \
+// RUN:   FileCheck %s --check-prefix=AARCH64
+// RUN: %clangxx %s -### --sysroot= 2>&1 \
+// RUN:   --target=aarch64-suse-linux -B%S/Inputs/opensuse_42.2_aarch64_tree/usr | \
+// RUN:   FileCheck %s --check-prefix=NO_AARCH64
+
+// AARCH64:        Inputs{{[^"]+}}aarch64-suse-linux/{{[^"]+}}crt1.o"
+// NO_AARCH64-NOT: Inputs{{[^"]+}}aarch64-suse-linux/{{[^"]+}}crt1.o"
diff --git a/clang/test/Driver/gcc-version-debug.c b/clang/test/Driver/gcc-version-debug.c
deleted file mode 100644
index daa9606ef7a5c01f84d5d595d7c624f11b847084..0000000000000000000000000000000000000000
--- a/clang/test/Driver/gcc-version-debug.c
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: %clang -v --target=i386-unknown-linux \
-// RUN:           --gcc-toolchain="" \
-// RUN:           --sysroot=%S/Inputs/debian_multiarch_tree 2>&1 | FileCheck %s
-
-// CHECK: Found candidate GCC installation: {{.*}}Inputs{{.}}debian_multiarch_tree{{.}}usr{{.}}lib{{.}}gcc{{.}}i686-linux-gnu{{.}}4.5
-// CHECK-NEXT: Found candidate GCC installation: {{.*}}Inputs{{.}}debian_multiarch_tree{{.}}usr{{.}}lib{{.}}gcc{{.}}x86_64-linux-gnu{{.}}4.5
-// CHECK-NEXT: Selected GCC installation: {{.*}}Inputs{{.}}debian_multiarch_tree{{.}}usr{{.}}lib{{.}}gcc{{.}}i686-linux-gnu{{.}}4.5
diff --git a/clang/test/Driver/immediate-options.c b/clang/test/Driver/immediate-options.c
index 723a6fa302f88b9cf4a7f1f851edd6556711c8a8..c398e0d41c6ef69013d4e25cfcb6982ca2a2d959 100644
--- a/clang/test/Driver/immediate-options.c
+++ b/clang/test/Driver/immediate-options.c
@@ -17,3 +17,15 @@
 // Allow unspecified output because the value of CLANG_RESOURCE_DIR is unknown.
 // RUN: %clang -print-resource-dir | FileCheck %s -check-prefix=PRINT-RESOURCE-DIR
 // PRINT-RESOURCE-DIR: {{.+}}
+
+// Default resource-dir layout
+// RUN: %clang -print-runtime-dir --target=x86_64-pc-windows-msvc \
+// RUN:        -resource-dir=%S/Inputs/resource_dir \
+// RUN:      | FileCheck --check-prefix=PRINT-RUNTIME-DIR %s
+// PRINT-RUNTIME-DIR: lib{{/|\\}}windows
+
+// Per target dir layout
+// RUN: %clang -print-runtime-dir --target=x86_64-pc-windows-msvc \
+// RUN:        -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:      | FileCheck --check-prefix=PRINT-RUNTIME-DIR-PER-TARGET %s
+// PRINT-RUNTIME-DIR-PER-TARGET: lib{{/|\\}}x86_64-pc-windows-msvc
diff --git a/clang/test/Driver/linux-cross.cpp b/clang/test/Driver/linux-cross.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bcc858d7804dd384245f3cd176910461df702db0
--- /dev/null
+++ b/clang/test/Driver/linux-cross.cpp
@@ -0,0 +1,83 @@
+// UNSUPPORTED: system-windows
+
+/// Test native x86-64 in the tree.
+// RUN: %clang -### %s --target=x86_64-linux-gnu --sysroot=%S/Inputs/debian_multiarch_tree \
+// RUN:   -resource-dir=%S/Inputs/resource_dir --stdlib=platform --rtlib=platform 2>&1 | FileCheck %s --check-prefix=DEBIAN_X86_64
+// DEBIAN_X86_64:      "-resource-dir" "[[RESOURCE:[^"]+]]"
+// DEBIAN_X86_64:      "-internal-isystem"
+// DEBIAN_X86_64-SAME: {{^}} "[[SYSROOT:[^"]+]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10"
+// DEBIAN_X86_64-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10"
+// DEBIAN_X86_64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward"
+// DEBIAN_X86_64-SAME: {{^}} "-internal-isystem" "[[RESOURCE]]/include"
+// DEBIAN_X86_64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/local/include"
+// DEBIAN_X86_64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include"
+// DEBIAN_X86_64:      "-L
+// DEBIAN_X86_64-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/10"
+/// Debian patches MULTILIB_OSDIRNAMES (../lib64 -> ../lib), so gcc uses 'lib' instead of 'lib64'.
+/// This difference does not matter in practice.
+// DEBIAN_X86_64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../lib64"
+// DEBIAN_X86_64-SAME: {{^}} "-L[[SYSROOT]]/lib/x86_64-linux-gnu"
+// DEBIAN_X86_64-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// DEBIAN_X86_64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/x86_64-linux-gnu"
+// DEBIAN_X86_64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/../lib64"
+/// /usr/x86_64-linux-gnu does not exist, so there is no /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/lib.
+/// -ccc-install-dir is not within sysroot. No bin/../lib.
+/// $sysroot/lib and $sysroot/usr/lib. Fallback when GCC installation is unavailable.
+// DEBIAN_X86_64-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// DEBIAN_X86_64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+/// Test -m32.
+// RUN: %clang -### %s --target=x86_64-linux-gnu -m32 --sysroot=%S/Inputs/debian_multiarch_tree \
+// RUN:   -resource-dir=%S/Inputs/resource_dir --stdlib=platform --rtlib=platform 2>&1 | FileCheck %s --check-prefix=DEBIAN_X86_64_M32
+// DEBIAN_X86_64_M32:      "-resource-dir" "[[RESOURCE:[^"]+]]"
+// DEBIAN_X86_64_M32:      "-internal-isystem"
+// DEBIAN_X86_64_M32-SAME: {{^}} "[[SYSROOT:[^"]+]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10/32"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-internal-isystem" "[[RESOURCE]]/include"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/local/include"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include"
+// DEBIAN_X86_64_M32:      "-internal-externc-isystem"
+// DEBIAN_X86_64_M32-SAME: {{^}} "[[SYSROOT]]/usr/include/i386-linux-gnu"
+// DEBIAN_X86_64_M32:      "-L
+// DEBIAN_X86_64_M32-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/10/32"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/10/../../../../lib32"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-L[[SYSROOT]]/lib/i386-linux-gnu"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib32"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/i386-linux-gnu"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/../lib32"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// DEBIAN_X86_64_M32-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+/// Test a cross compiler.
+// RUN: %clang -### %s --target=aarch64-linux-gnu --sysroot=%S/Inputs/debian_multiarch_tree \
+// RUN:   -resource-dir=%S/Inputs/resource_dir --stdlib=platform --rtlib=platform 2>&1 | FileCheck %s --check-prefix=DEBIAN_AARCH64
+// DEBIAN_AARCH64:      "-resource-dir" "[[RESOURCE:[^"]+]]"
+// DEBIAN_AARCH64:      "-internal-isystem"
+// DEBIAN_AARCH64-SAME: {{^}} "[[SYSROOT:[^"]+]]/usr/lib/gcc-cross/aarch64-linux-gnu/10/../../../../aarch64-linux-gnu/include/c++/10"
+// DEBIAN_AARCH64-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/lib/gcc-cross/aarch64-linux-gnu/10/../../../../aarch64-linux-gnu/include/c++/10/aarch64-linux-gnu"
+// DEBIAN_AARCH64-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/lib/gcc-cross/aarch64-linux-gnu/10/../../../../aarch64-linux-gnu/include/c++/10/backward"
+// DEBIAN_AARCH64-SAME: {{^}} "-internal-isystem" "[[RESOURCE]]/include"
+// DEBIAN_AARCH64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/local/include"
+// DEBIAN_AARCH64-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/lib/gcc-cross/aarch64-linux-gnu/10/../../../../aarch64-linux-gnu/include"
+// DEBIAN_AARCH64:      "-L
+// DEBIAN_AARCH64-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc-cross/aarch64-linux-gnu/10"
+/// Debian patches MULTILIB_OSDIRNAMES (../lib64 -> ../lib), so aarch64-linux-gnu-gcc uses 'lib' instead of 'lib64'.
+/// This difference does not matter in practice.
+// DEBIAN_AARCH64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc-cross/aarch64-linux-gnu/10/../../../../lib64"
+// DEBIAN_AARCH64-SAME: {{^}} "-L[[SYSROOT]]/lib/aarch64-linux-gnu"
+// DEBIAN_AARCH64-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// DEBIAN_AARCH64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/aarch64-linux-gnu"
+// DEBIAN_AARCH64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/../lib64"
+// DEBIAN_AARCH64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc-cross/aarch64-linux-gnu/10/../../../../aarch64-linux-gnu/lib"
+// DEBIAN_AARCH64-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// DEBIAN_AARCH64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+/// LDSO_ARCH is i386 for all x86-32 variants.
+// RUN: %clang -### %s --target=i686-linux-musl --sysroot= \
+// RUN:   --stdlib=platform --rtlib=platform 2>&1 | FileCheck %s --check-prefix=MUSL_I686
+// MUSL_I686: "-dynamic-linker" "/lib/ld-musl-i386.so.1"
+
+// RUN: %clang -### %s --target=x86_64-linux-muslx32 --sysroot= \
+// RUN:   --stdlib=platform --rtlib=platform 2>&1 | FileCheck %s --check-prefix=MUSL_X32
+// MUSL_X32: "-dynamic-linker" "/lib/ld-musl-x32.so.1"
diff --git a/clang/test/Driver/linux-header-search.cpp b/clang/test/Driver/linux-header-search.cpp
index 8c1fc99d79f355ef3772a72a6652135e4ea21f70..9044ac2065a13f9532b528b6ccb8b35aa111774f 100644
--- a/clang/test/Driver/linux-header-search.cpp
+++ b/clang/test/Driver/linux-header-search.cpp
@@ -66,230 +66,7 @@
 // CHECK-BASIC-LIBSTDCXX-LIBCXXV2-SYSROOT: "-isysroot" "[[SYSROOT:[^"]+]]"
 // CHECK-BASIC-LIBSTDCXX-LIBCXXV2-SYSROOT: "-internal-isystem" "[[SYSROOT]]/usr/include/c++/v2"
 // CHECK-BASIC-LIBSTDCXX-LIBCXXV2-SYSROOT: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-//
-// Test a very broken version of multiarch that shipped in Ubuntu 11.04.
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target i386-unknown-linux -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/ubuntu_11.04_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-11-04 %s
-// CHECK-UBUNTU-11-04: "{{.*}}clang{{.*}}" "-cc1"
-// CHECK-UBUNTU-11-04: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-UBUNTU-11-04: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-11-04: "-internal-isystem" "[[SYSROOT]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5/../../../../../include/c++/4.5"
-// CHECK-UBUNTU-11-04: "-internal-isystem" "[[SYSROOT]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5/../../../../../include/c++/4.5/i686-linux-gnu"
-// CHECK-UBUNTU-11-04: "-internal-isystem" "[[SYSROOT]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5/../../../../../include/c++/4.5/backward"
-// CHECK-UBUNTU-11-04: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-UBUNTU-11-04: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-UBUNTU-11-04: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-UBUNTU-11-04: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-//
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target x86_64-unknown-linux-gnu -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/ubuntu_13.04_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-13-04 %s
-// CHECK-UBUNTU-13-04: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-UBUNTU-13-04: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-UBUNTU-13-04: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-13-04: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.7/../../../../include/c++/4.7"
-// CHECK-UBUNTU-13-04: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.7/../../../../include/x86_64-linux-gnu/c++/4.7"
-// CHECK-UBUNTU-13-04: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.7/../../../../include/c++/4.7/backward"
-// CHECK-UBUNTU-13-04: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-UBUNTU-13-04: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-UBUNTU-13-04: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/x86_64-linux-gnu"
-// CHECK-UBUNTU-13-04: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-UBUNTU-13-04: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-//
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target x86_64-unknown-linux-gnux32 -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/ubuntu_14.04_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-14-04 %s
-// CHECK-UBUNTU-14-04: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-UBUNTU-14-04: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-UBUNTU-14-04: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-14-04: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8"
-// CHECK-UBUNTU-14-04: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/x86_64-linux-gnu/c++/4.8/x32"
-// CHECK-UBUNTU-14-04: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8/backward"
-// CHECK-UBUNTU-14-04: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-UBUNTU-14-04: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-UBUNTU-14-04: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/x86_64-linux-gnu"
-// CHECK-UBUNTU-14-04: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-UBUNTU-14-04: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-///
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target arm-linux-gnueabihf -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/ubuntu_13.04_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-13-04-CROSS %s
-// CHECK-UBUNTU-13-04-CROSS: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-UBUNTU-13-04-CROSS: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-UBUNTU-13-04-CROSS: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-13-04-CROSS: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc-cross/arm-linux-gnueabihf/4.7/../../../../include/c++/4.7"
-// CHECK-UBUNTU-13-04-CROSS: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc-cross/arm-linux-gnueabihf/4.7/../../../../include/arm-linux-gnueabihf/c++/4.7"
-// CHECK-UBUNTU-13-04-CROSS: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc-cross/arm-linux-gnueabihf/4.7/../../../../include/c++/4.7/backward"
-// CHECK-UBUNTU-13-04-CROSS: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-UBUNTU-13-04-CROSS: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-UBUNTU-13-04-CROSS: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-UBUNTU-13-04-CROSS: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-//
-// Test Ubuntu/Debian's new version of multiarch, with -m32.
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target x86_64-unknown-linux-gnu -m32 -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/ubuntu_13.04_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-13-04-M32 %s
-// CHECK-UBUNTU-13-04-M32: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-UBUNTU-13-04-M32: "-triple" "i386-unknown-linux-gnu"
-// CHECK-UBUNTU-13-04-M32: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-13-04-M32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.7/../../../../include/c++/4.7"
-// CHECK-UBUNTU-13-04-M32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.7/../../../../include/x86_64-linux-gnu/c++/4.7/32"
-// CHECK-UBUNTU-13-04-M32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.7/../../../../include/c++/4.7/backward"
-//
-// Test Ubuntu/Debian's Ubuntu 14.04 config variant, with -m32
-// and an empty 4.9 directory.
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target x86_64-unknown-linux-gnu -m32 -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/ubuntu_14.04_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-14-04-M32 %s
-// CHECK-UBUNTU-14-04-M32: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-UBUNTU-14-04-M32: "-triple" "i386-unknown-linux-gnu"
-// CHECK-UBUNTU-14-04-M32: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-14-04-M32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8"
-// CHECK-UBUNTU-14-04-M32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/x86_64-linux-gnu/c++/4.8/32"
-// CHECK-UBUNTU-14-04-M32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8/backward"
-//
-// Test Ubuntu/Debian's Ubuntu 14.04 with -m32 and an i686 cross compiler
-// installed rather than relying on multilib. Also happens to look like an
-// actual i686 Ubuntu system.
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target x86_64-unknown-linux-gnu -m32 -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/ubuntu_14.04_multiarch_tree2 \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-14-04-I686 %s
-// CHECK-UBUNTU-14-04-I686: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-UBUNTU-14-04-I686: "-triple" "i386-unknown-linux-gnu"
-// CHECK-UBUNTU-14-04-I686: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-14-04-I686: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/i686-linux-gnu/4.8/../../../../include/c++/4.8"
-// CHECK-UBUNTU-14-04-I686: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/i686-linux-gnu/4.8/../../../../include/i386-linux-gnu/c++/4.8"
-// CHECK-UBUNTU-14-04-I686: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/i686-linux-gnu/4.8/../../../../include/c++/4.8/backward"
-//
-// Test Ubuntu/Debian's Ubuntu 14.04 for powerpc64le
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target powerpc64le-unknown-linux-gnu -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/ubuntu_14.04_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-14-04-PPC64LE %s
-// CHECK-UBUNTU-14-04-PPC64LE: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-UBUNTU-14-04-PPC64LE: "-triple" "powerpc64le-unknown-linux-gnu"
-// CHECK-UBUNTU-14-04-PPC64LE: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-14-04-PPC64LE: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/powerpc64le-linux-gnu/4.8/../../../../include/c++/4.8"
-// CHECK-UBUNTU-14-04-PPC64LE: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/powerpc64le-linux-gnu/4.8/../../../../include/powerpc64le-linux-gnu/c++/4.8"
-// CHECK-UBUNTU-14-04-PPC64LE: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/powerpc64le-linux-gnu/4.8/../../../../include/c++/4.8/backward"
-// CHECK-UBUNTU-14-04-PPC64LE: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/powerpc64le-linux-gnu"
-// CHECK-UBUNTU-14-04-PPC64LE: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-UBUNTU-14-04-PPC64LE: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-//
-// Thoroughly exercise the Debian multiarch environment.
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target i686-linux-gnu -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-X86 %s
-// CHECK-DEBIAN-X86: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-DEBIAN-X86: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-DEBIAN-X86: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-X86: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/i686-linux-gnu/4.5/../../../../include/c++/4.5"
-// CHECK-DEBIAN-X86: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/i686-linux-gnu/4.5/../../../../include/c++/4.5/i686-linux-gnu"
-// CHECK-DEBIAN-X86: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/i686-linux-gnu/4.5/../../../../include/c++/4.5/backward"
-// CHECK-DEBIAN-X86: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-DEBIAN-X86: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-DEBIAN-X86: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/i386-linux-gnu"
-// CHECK-DEBIAN-X86: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-DEBIAN-X86: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target x86_64-linux-gnu -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-X86-64 %s
-// CHECK-DEBIAN-X86-64: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-DEBIAN-X86-64: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-DEBIAN-X86-64: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-X86-64: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.5/../../../../include/c++/4.5"
-// CHECK-DEBIAN-X86-64: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.5/../../../../include/c++/4.5/x86_64-linux-gnu"
-// CHECK-DEBIAN-X86-64: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.5/../../../../include/c++/4.5/backward"
-// CHECK-DEBIAN-X86-64: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-DEBIAN-X86-64: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-DEBIAN-X86-64: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/x86_64-linux-gnu"
-// CHECK-DEBIAN-X86-64: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-DEBIAN-X86-64: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target powerpc-linux-gnu -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-PPC %s
-// CHECK-DEBIAN-PPC: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-DEBIAN-PPC: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-DEBIAN-PPC: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-PPC: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/powerpc-linux-gnu/4.5/../../../../include/c++/4.5"
-// CHECK-DEBIAN-PPC: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/powerpc-linux-gnu/4.5/../../../../include/c++/4.5/powerpc-linux-gnu"
-// CHECK-DEBIAN-PPC: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/powerpc-linux-gnu/4.5/../../../../include/c++/4.5/backward"
-// CHECK-DEBIAN-PPC: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-DEBIAN-PPC: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-DEBIAN-PPC: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/powerpc-linux-gnu"
-// CHECK-DEBIAN-PPC: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-DEBIAN-PPC: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target powerpc64-linux-gnu -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-PPC64 %s
-// CHECK-DEBIAN-PPC64: "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-DEBIAN-PPC64: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-DEBIAN-PPC64: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-PPC64: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/powerpc64-linux-gnu/4.5/../../../../include/c++/4.5"
-// CHECK-DEBIAN-PPC64: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/powerpc64-linux-gnu/4.5/../../../../include/c++/4.5/powerpc64-linux-gnu"
-// CHECK-DEBIAN-PPC64: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/powerpc64-linux-gnu/4.5/../../../../include/c++/4.5/backward"
-// CHECK-DEBIAN-PPC64: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-DEBIAN-PPC64: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-DEBIAN-PPC64: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/powerpc64-linux-gnu"
-// CHECK-DEBIAN-PPC64: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-DEBIAN-PPC64: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target sparc-linux-gnu -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-SPARC %s
-// CHECK-DEBIAN-SPARC:      "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-DEBIAN-SPARC-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-DEBIAN-SPARC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-SPARC-SAME: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/sparc-linux-gnu/4.5/../../../../include/c++/4.5"
-// CHECK-DEBIAN-SPARC-SAME: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/sparc-linux-gnu/4.5/../../../../include/c++/4.5/sparc-linux-gnu"
-// CHECK-DEBIAN-SPARC-SAME: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/sparc-linux-gnu/4.5/../../../../include/c++/4.5/backward"
-// CHECK-DEBIAN-SPARC-SAME: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-DEBIAN-SPARC-SAME: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-DEBIAN-SPARC-SAME: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/sparc-linux-gnu"
-// CHECK-DEBIAN-SPARC-SAME: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-DEBIAN-SPARC-SAME: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-// RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
-// RUN:     -target sparc64-linux-gnu -stdlib=libstdc++ \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:     --gcc-toolchain="" \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-SPARC64 %s
-// CHECK-DEBIAN-SPARC64:      "{{[^"]*}}clang{{[^"]*}}" "-cc1"
-// CHECK-DEBIAN-SPARC64-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-DEBIAN-SPARC64-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-SPARC64-SAME: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/sparc64-linux-gnu/4.5/../../../../include/c++/4.5"
-// CHECK-DEBIAN-SPARC64-SAME: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/sparc64-linux-gnu/4.5/../../../../include/c++/4.5/sparc64-linux-gnu"
-// CHECK-DEBIAN-SPARC64-SAME: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/sparc64-linux-gnu/4.5/../../../../include/c++/4.5/backward"
-// CHECK-DEBIAN-SPARC64-SAME: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
-// CHECK-DEBIAN-SPARC64-SAME: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
-// CHECK-DEBIAN-SPARC64-SAME: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/sparc64-linux-gnu"
-// CHECK-DEBIAN-SPARC64-SAME: "-internal-externc-isystem" "[[SYSROOT]]/include"
-// CHECK-DEBIAN-SPARC64-SAME: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
-//
+
 // Test Gentoo's weirdness both before and after they changed it in their GCC
 // 4.6.4 release.
 // RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
@@ -303,8 +80,8 @@
 // CHECK-GENTOO-4-6-2: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.6.2/include/g++-v4"
 // CHECK-GENTOO-4-6-2: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.6.2/include/g++-v4/x86_64-pc-linux-gnu"
 // CHECK-GENTOO-4-6-2: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.6.2/include/g++-v4/backward"
-// CHECK-GENTOO-4-6-2: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-6-2: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-GENTOO-4-6-2: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-6-2: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-GENTOO-4-6-2: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
 // RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
@@ -318,8 +95,8 @@
 // CHECK-GENTOO-4-6-4: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.6.4/include/g++-v4.6"
 // CHECK-GENTOO-4-6-4: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.6.4/include/g++-v4.6/x86_64-pc-linux-gnu"
 // CHECK-GENTOO-4-6-4: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.6.4/include/g++-v4.6/backward"
-// CHECK-GENTOO-4-6-4: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-6-4: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-GENTOO-4-6-4: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-6-4: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-GENTOO-4-6-4: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
 // RUN: %clang -no-canonical-prefixes %s -### -fsyntax-only 2>&1 \
@@ -333,8 +110,8 @@
 // CHECK-GENTOO-4-9-3: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/include/g++-v4.9.3"
 // CHECK-GENTOO-4-9-3: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/include/g++-v4.9.3/x86_64-pc-linux-gnu"
 // CHECK-GENTOO-4-9-3: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/include/g++-v4.9.3/backward"
-// CHECK-GENTOO-4-9-3: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-3: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-GENTOO-4-9-3: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-3: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-GENTOO-4-9-3: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
 //
@@ -359,8 +136,8 @@
 // CHECK-GENTOO-4-9-3-X32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/include/g++-v4.9.3"
 // CHECK-GENTOO-4-9-3-X32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/include/g++-v4.9.3/x86_64-pc-linux-gnu/x32"
 // CHECK-GENTOO-4-9-3-X32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/include/g++-v4.9.3/backward"
-// CHECK-GENTOO-4-9-3-X32: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-3-X32: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-GENTOO-4-9-3-X32: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-3-X32: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-GENTOO-4-9-3-X32: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
 //
@@ -375,8 +152,8 @@
 // CHECK-GENTOO-4-9-3-32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/include/g++-v4.9.3"
 // CHECK-GENTOO-4-9-3-32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/include/g++-v4.9.3/x86_64-pc-linux-gnu/32"
 // CHECK-GENTOO-4-9-3-32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/include/g++-v4.9.3/backward"
-// CHECK-GENTOO-4-9-3-32: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-3-32: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-GENTOO-4-9-3-32: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-3-32: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-GENTOO-4-9-3-32: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
 //
@@ -396,8 +173,8 @@
 // CHECK-GENTOO-4-9-X: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.x/include/g++-v4.9.3"
 // CHECK-GENTOO-4-9-X: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.x/include/g++-v4.9.3/x86_64-pc-linux-gnu"
 // CHECK-GENTOO-4-9-X: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.x/include/g++-v4.9.3/backward"
-// CHECK-GENTOO-4-9-X: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-X: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-GENTOO-4-9-X: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-X: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-GENTOO-4-9-X: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
 //
@@ -412,8 +189,8 @@
 // CHECK-GENTOO-4-9-X-X32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.x/include/g++-v4.9.3"
 // CHECK-GENTOO-4-9-X-X32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.x/include/g++-v4.9.3/x86_64-pc-linux-gnu/x32"
 // CHECK-GENTOO-4-9-X-X32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.x/include/g++-v4.9.3/backward"
-// CHECK-GENTOO-4-9-X-X32: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-X-X32: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-GENTOO-4-9-X-X32: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-X-X32: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-GENTOO-4-9-X-X32: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
 //
@@ -428,8 +205,8 @@
 // CHECK-GENTOO-4-9-X-32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.x/include/g++-v4.9.3"
 // CHECK-GENTOO-4-9-X-32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.x/include/g++-v4.9.3/x86_64-pc-linux-gnu/32"
 // CHECK-GENTOO-4-9-X-32: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.x/include/g++-v4.9.3/backward"
-// CHECK-GENTOO-4-9-X-32: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-X-32: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-GENTOO-4-9-X-32: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-GENTOO-4-9-X-32: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-GENTOO-4-9-X-32: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
 //
@@ -445,8 +222,8 @@
 // CHECK-MIPS64-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/mips64-linux-gnuabi64/4.9/../../../../include/c++/4.9"
 // CHECK-MIPS64-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/mips64-linux-gnuabi64/4.9/../../../../include/c++/4.9/mips64-linux-gnuabi64"
 // CHECK-MIPS64-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/mips64-linux-gnuabi64/4.9/../../../../include/c++/4.9/backward"
-// CHECK-MIPS64-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-MIPS64-GNUABI: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-MIPS64-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-MIPS64-GNUABI: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/mips64-linux-gnuabi64"
 // CHECK-MIPS64-GNUABI: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-MIPS64-GNUABI: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
@@ -463,8 +240,8 @@
 // CHECK-MIPS64EL-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/mips64el-linux-gnuabi64/4.9/../../../../include/c++/4.9"
 // CHECK-MIPS64EL-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/mips64el-linux-gnuabi64/4.9/../../../../include/c++/4.9/mips64el-linux-gnuabi64"
 // CHECK-MIPS64EL-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/mips64el-linux-gnuabi64/4.9/../../../../include/c++/4.9/backward"
-// CHECK-MIPS64EL-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-MIPS64EL-GNUABI: "-internal-isystem" "[[RESOURCE_DIR]]{{/|\\\\}}include"
+// CHECK-MIPS64EL-GNUABI: "-internal-isystem" "[[SYSROOT]]/usr/local/include"
 // CHECK-MIPS64EL-GNUABI: "-internal-externc-isystem" "[[SYSROOT]]/usr/include/mips64el-linux-gnuabi64"
 // CHECK-MIPS64EL-GNUABI: "-internal-externc-isystem" "[[SYSROOT]]/include"
 // CHECK-MIPS64EL-GNUABI: "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
diff --git a/clang/test/Driver/linux-ld.c b/clang/test/Driver/linux-ld.c
index eba09d2970ccc19f0307dbc1110072ae6247a7e1..7630d5f1324ee68b06e893d405187ad7be9a6a20 100644
--- a/clang/test/Driver/linux-ld.c
+++ b/clang/test/Driver/linux-ld.c
@@ -1,3 +1,4 @@
+// UNSUPPORTED: system-windows
 // General tests that ld invocations on Linux targets sane. Note that we use
 // sysroot to make these tests independent of the host system.
 //
@@ -11,7 +12,6 @@
 // CHECK-LD-32: "{{.*}}/usr/lib/gcc/i386-unknown-linux/4.6.0{{/|\\\\}}crtbegin.o"
 // CHECK-LD-32: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0"
 // CHECK-LD-32: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0/../../../../i386-unknown-linux/lib"
-// CHECK-LD-32: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0/../../.."
 // CHECK-LD-32: "-L[[SYSROOT]]/lib"
 // CHECK-LD-32: "-L[[SYSROOT]]/usr/lib"
 //
@@ -28,7 +28,6 @@
 // CHECK-LD-64: "{{.*}}/usr/lib/gcc/x86_64-unknown-linux/4.6.0{{/|\\\\}}crtbegin.o"
 // CHECK-LD-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-LD-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-LD-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-LD-64: "-L[[SYSROOT]]/lib"
 // CHECK-LD-64: "-L[[SYSROOT]]/usr/lib"
 // CHECK-LD-64: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
@@ -65,7 +64,6 @@
 // CHECK-LD-RT: "[[RESDIR]]{{/|\\\\}}lib{{/|\\\\}}linux{{/|\\\\}}clang_rt.crtbegin-x86_64.o"
 // CHECK-LD-RT: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-LD-RT: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-LD-RT: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-LD-RT: "-L[[SYSROOT]]/lib"
 // CHECK-LD-RT: "-L[[SYSROOT]]/usr/lib"
 // CHECK-LD-RT: libclang_rt.builtins-x86_64.a"
@@ -89,7 +87,6 @@
 // CHECK-LD-RT-I686: "[[RESDIR]]{{/|\\\\}}lib{{/|\\\\}}linux{{/|\\\\}}clang_rt.crtbegin-i386.o"
 // CHECK-LD-RT-I686: "-L[[SYSROOT]]/usr/lib/gcc/i686-unknown-linux/4.6.0"
 // CHECK-LD-RT-I686: "-L[[SYSROOT]]/usr/lib/gcc/i686-unknown-linux/4.6.0/../../../../i686-unknown-linux/lib"
-// CHECK-LD-RT-I686: "-L[[SYSROOT]]/usr/lib/gcc/i686-unknown-linux/4.6.0/../../.."
 // CHECK-LD-RT-I686: "-L[[SYSROOT]]/lib"
 // CHECK-LD-RT-I686: "-L[[SYSROOT]]/usr/lib"
 // CHECK-LD-RT-I686: libclang_rt.builtins-i386.a"
@@ -125,7 +122,6 @@
 // CHECK-LD-GCC: "{{.*}}/usr/lib/gcc/x86_64-unknown-linux/4.6.0{{/|\\\\}}crtbegin.o"
 // CHECK-LD-GCC: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-LD-GCC: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-LD-GCC: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-LD-GCC: "-L[[SYSROOT]]/lib"
 // CHECK-LD-GCC: "-L[[SYSROOT]]/usr/lib"
 // CHECK-LD-GCC: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
@@ -146,7 +142,6 @@
 // CHECK-LD-64-STATIC-LIBGCC: "{{.*}}/usr/lib/gcc/x86_64-unknown-linux/4.6.0{{/|\\\\}}crtbegin.o"
 // CHECK-LD-64-STATIC-LIBGCC: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-LD-64-STATIC-LIBGCC: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-LD-64-STATIC-LIBGCC: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-LD-64-STATIC-LIBGCC: "-L[[SYSROOT]]/lib"
 // CHECK-LD-64-STATIC-LIBGCC: "-L[[SYSROOT]]/usr/lib"
 // CHECK-LD-64-STATIC-LIBGCC: "-lgcc" "-lgcc_eh"
@@ -321,7 +316,6 @@
 // CHECK-LD-64-STATIC: "{{.*}}/usr/lib/gcc/x86_64-unknown-linux/4.6.0{{/|\\\\}}crtbeginT.o"
 // CHECK-LD-64-STATIC: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-LD-64-STATIC: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-LD-64-STATIC: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-LD-64-STATIC: "-L[[SYSROOT]]/lib"
 // CHECK-LD-64-STATIC: "-L[[SYSROOT]]/usr/lib"
 // CHECK-LD-64-STATIC: "--start-group" "-lgcc" "-lgcc_eh" "-lc" "--end-group"
@@ -347,7 +341,6 @@
 // CHECK-32-TO-32: "-L[[SYSROOT]]/lib/../lib32"
 // CHECK-32-TO-32: "-L[[SYSROOT]]/usr/lib/../lib32"
 // CHECK-32-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0/../../../../i386-unknown-linux/lib"
-// CHECK-32-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0/../../.."
 // CHECK-32-TO-32: "-L[[SYSROOT]]/lib"
 // CHECK-32-TO-32: "-L[[SYSROOT]]/usr/lib"
 //
@@ -363,9 +356,7 @@
 // CHECK-32-TO-64: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0/../../../../lib64"
 // CHECK-32-TO-64: "-L[[SYSROOT]]/lib/../lib64"
 // CHECK-32-TO-64: "-L[[SYSROOT]]/usr/lib/../lib64"
-// CHECK-32-TO-64: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0"
 // CHECK-32-TO-64: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0/../../../../i386-unknown-linux/lib"
-// CHECK-32-TO-64: "-L[[SYSROOT]]/usr/lib/gcc/i386-unknown-linux/4.6.0/../../.."
 // CHECK-32-TO-64: "-L[[SYSROOT]]/lib"
 // CHECK-32-TO-64: "-L[[SYSROOT]]/usr/lib"
 //
@@ -382,7 +373,6 @@
 // CHECK-64-TO-64: "-L[[SYSROOT]]/lib/../lib64"
 // CHECK-64-TO-64: "-L[[SYSROOT]]/usr/lib/../lib64"
 // CHECK-64-TO-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-64-TO-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-64-TO-64: "-L[[SYSROOT]]/lib"
 // CHECK-64-TO-64: "-L[[SYSROOT]]/usr/lib"
 //
@@ -398,9 +388,7 @@
 // CHECK-64-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../lib32"
 // CHECK-64-TO-32: "-L[[SYSROOT]]/lib/../lib32"
 // CHECK-64-TO-32: "-L[[SYSROOT]]/usr/lib/../lib32"
-// CHECK-64-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-64-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-64-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-64-TO-32: "-L[[SYSROOT]]/lib"
 // CHECK-64-TO-32: "-L[[SYSROOT]]/usr/lib"
 //
@@ -416,9 +404,7 @@
 // CHECK-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../libx32"
 // CHECK-X32: "-L[[SYSROOT]]/lib/../libx32"
 // CHECK-X32: "-L[[SYSROOT]]/usr/lib/../libx32"
-// CHECK-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-X32: "-L[[SYSROOT]]/lib"
 // CHECK-X32: "-L[[SYSROOT]]/usr/lib"
 //
@@ -434,9 +420,7 @@
 // CHECK-64-TO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../libx32"
 // CHECK-64-TO-X32: "-L[[SYSROOT]]/lib/../libx32"
 // CHECK-64-TO-X32: "-L[[SYSROOT]]/usr/lib/../libx32"
-// CHECK-64-TO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-64-TO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-64-TO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-64-TO-X32: "-L[[SYSROOT]]/lib"
 // CHECK-64-TO-X32: "-L[[SYSROOT]]/usr/lib"
 //
@@ -452,9 +436,7 @@
 // CHECK-32-TO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../libx32"
 // CHECK-32-TO-X32: "-L[[SYSROOT]]/lib/../libx32"
 // CHECK-32-TO-X32: "-L[[SYSROOT]]/usr/lib/../libx32"
-// CHECK-32-TO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-32-TO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-32-TO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-32-TO-X32: "-L[[SYSROOT]]/lib"
 // CHECK-32-TO-X32: "-L[[SYSROOT]]/usr/lib"
 //
@@ -471,7 +453,6 @@
 // CHECK-X32-TO-64: "-L[[SYSROOT]]/lib/../lib64"
 // CHECK-X32-TO-64: "-L[[SYSROOT]]/usr/lib/../lib64"
 // CHECK-X32-TO-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-X32-TO-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-X32-TO-64: "-L[[SYSROOT]]/lib"
 // CHECK-X32-TO-64: "-L[[SYSROOT]]/usr/lib"
 //
@@ -487,9 +468,7 @@
 // CHECK-X32-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../lib32"
 // CHECK-X32-TO-32: "-L[[SYSROOT]]/lib/../lib32"
 // CHECK-X32-TO-32: "-L[[SYSROOT]]/usr/lib/../lib32"
-// CHECK-X32-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-X32-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../../../x86_64-unknown-linux/lib"
-// CHECK-X32-TO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0/../../.."
 // CHECK-X32-TO-32: "-L[[SYSROOT]]/lib"
 // CHECK-X32-TO-32: "-L[[SYSROOT]]/usr/lib"
 //
@@ -503,32 +482,9 @@
 // CHECK-64-TO-32-SYSROOT: "-L{{[^"]*}}/Inputs/multilib_64bit_linux_tree/usr/lib/gcc/x86_64-unknown-linux/4.6.0/32"
 // CHECK-64-TO-32-SYSROOT: "-L[[SYSROOT]]/lib/../lib32"
 // CHECK-64-TO-32-SYSROOT: "-L[[SYSROOT]]/usr/lib/../lib32"
-// CHECK-64-TO-32-SYSROOT: "-L{{[^"]*}}/Inputs/multilib_64bit_linux_tree/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
 // CHECK-64-TO-32-SYSROOT: "-L[[SYSROOT]]/lib"
 // CHECK-64-TO-32-SYSROOT: "-L[[SYSROOT]]/usr/lib"
 //
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=i386-unknown-linux -rtlib=platform -m32 \
-// RUN:     -ccc-install-dir %S/Inputs/fake_install_tree/bin \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/basic_linux_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-INSTALL-DIR-32 %s
-// CHECK-INSTALL-DIR-32: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-INSTALL-DIR-32: "{{.*}}/Inputs/fake_install_tree/bin/../lib/gcc/i386-unknown-linux/4.7.0{{/|\\\\}}crtbegin.o"
-// CHECK-INSTALL-DIR-32: "-L{{.*}}/Inputs/fake_install_tree/bin/../lib/gcc/i386-unknown-linux/4.7.0"
-//
-// Check that with 64-bit builds, we don't actually use the install directory
-// as its version of GCC is lower than our sysrooted version.
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=x86_64-unknown-linux -rtlib=platform -m64 \
-// RUN:     -ccc-install-dir %S/Inputs/fake_install_tree/bin \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/basic_linux_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-INSTALL-DIR-64 %s
-// CHECK-INSTALL-DIR-64: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-INSTALL-DIR-64: "{{.*}}/usr/lib/gcc/x86_64-unknown-linux/4.6.0{{/|\\\\}}crtbegin.o"
-// CHECK-INSTALL-DIR-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-unknown-linux/4.6.0"
-//
 // Check that we support unusual patch version formats, including missing that
 // component.
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
@@ -538,45 +494,8 @@
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
 // RUN:   | FileCheck --check-prefix=CHECK-GCC-VERSION1 %s
 // CHECK-GCC-VERSION1: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-GCC-VERSION1: "{{.*}}/Inputs/gcc_version_parsing1/bin/../lib/gcc/i386-unknown-linux/4.7{{/|\\\\}}crtbegin.o"
-// CHECK-GCC-VERSION1: "-L{{.*}}/Inputs/gcc_version_parsing1/bin/../lib/gcc/i386-unknown-linux/4.7"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=i386-unknown-linux -rtlib=platform -m32 \
-// RUN:     -ccc-install-dir %S/Inputs/gcc_version_parsing2/bin \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/basic_linux_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-GCC-VERSION2 %s
-// CHECK-GCC-VERSION2: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-GCC-VERSION2: "{{.*}}/Inputs/gcc_version_parsing2/bin/../lib/gcc/i386-unknown-linux/4.7.x{{/|\\\\}}crtbegin.o"
-// CHECK-GCC-VERSION2: "-L{{.*}}/Inputs/gcc_version_parsing2/bin/../lib/gcc/i386-unknown-linux/4.7.x"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=i386-unknown-linux -rtlib=platform -m32 \
-// RUN:     -ccc-install-dir %S/Inputs/gcc_version_parsing3/bin \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/basic_linux_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-GCC-VERSION3 %s
-// CHECK-GCC-VERSION3: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-GCC-VERSION3: "{{.*}}/Inputs/gcc_version_parsing3/bin/../lib/gcc/i386-unknown-linux/4.7.99-rc5{{/|\\\\}}crtbegin.o"
-// CHECK-GCC-VERSION3: "-L{{.*}}/Inputs/gcc_version_parsing3/bin/../lib/gcc/i386-unknown-linux/4.7.99-rc5"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=i386-unknown-linux -rtlib=platform -m32 \
-// RUN:     -ccc-install-dir %S/Inputs/gcc_version_parsing4/bin \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/basic_linux_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-GCC-VERSION4 %s
-// CHECK-GCC-VERSION4: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-GCC-VERSION4: "{{.*}}/Inputs/gcc_version_parsing4/bin/../lib/gcc/i386-unknown-linux/4.7.99{{/|\\\\}}crtbegin.o"
-// CHECK-GCC-VERSION4: "-L{{.*}}/Inputs/gcc_version_parsing4/bin/../lib/gcc/i386-unknown-linux/4.7.99"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=i386-unknown-linux -rtlib=platform -m32 \
-// RUN:     -ccc-install-dir %S/Inputs/gcc_version_parsing5/bin \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/basic_linux_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-GCC-VERSION5 %s
-// CHECK-GCC-VERSION5: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-GCC-VERSION5: "{{.*}}/Inputs/gcc_version_parsing5/bin/../lib/gcc/i386-unknown-linux/5{{/|\\\\}}crtbegin.o"
-// CHECK-GCC-VERSION5: "-L{{.*}}/Inputs/gcc_version_parsing5/bin/../lib/gcc/i386-unknown-linux/5"
-//
+// CHECK-GCC-VERSION1: "{{.*}}/Inputs/basic_linux_tree/usr/lib/gcc/i386-unknown-linux/4.6.0{{/|\\\\}}crtbegin.o"
+
 // Test a simulated installation of libc++ on Linux, both through sysroot and
 // the installation path of Clang.
 // RUN: %clangxx -no-canonical-prefixes -x c++ %s -### -o %t.o 2>&1 \
@@ -624,21 +543,6 @@
 // CHECK-BASIC-LIBCXX-C-LINK: "--sysroot=[[SYSROOT]]"
 // CHECK-BASIC-LIBCXX-C-LINK: "-L[[SYSROOT]]/usr/bin/../lib"
 //
-// Test a very broken version of multiarch that shipped in Ubuntu 11.04.
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=i386-unknown-linux -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/ubuntu_11.04_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-11-04 %s
-// CHECK-UBUNTU-11-04: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-11-04: "{{.*}}/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-UBUNTU-11-04: "-L[[SYSROOT]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5"
-// CHECK-UBUNTU-11-04: "-L[[SYSROOT]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5/../../../../i386-linux-gnu"
-// CHECK-UBUNTU-11-04: "-L[[SYSROOT]]/usr/lib/i386-linux-gnu"
-// CHECK-UBUNTU-11-04: "-L[[SYSROOT]]/usr/lib/i386-linux-gnu/gcc/i686-linux-gnu/4.5/../../../.."
-// CHECK-UBUNTU-11-04: "-L[[SYSROOT]]/lib"
-// CHECK-UBUNTU-11-04: "-L[[SYSROOT]]/usr/lib"
-//
 // Check multi arch support on Ubuntu 12.04 LTS.
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     --target=arm-unknown-linux-gnueabihf -rtlib=platform \
@@ -646,16 +550,14 @@
 // RUN:     --sysroot=%S/Inputs/ubuntu_12.04_LTS_multiarch_tree \
 // RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-12-04-ARM-HF %s
 // CHECK-UBUNTU-12-04-ARM-HF: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-12-04-ARM-HF: "{{.*}}/usr/lib/gcc/arm-linux-gnueabihf/4.6.3/../../../arm-linux-gnueabihf{{/|\\\\}}crt1.o"
-// CHECK-UBUNTU-12-04-ARM-HF: "{{.*}}/usr/lib/gcc/arm-linux-gnueabihf/4.6.3/../../../arm-linux-gnueabihf{{/|\\\\}}crti.o"
+// CHECK-UBUNTU-12-04-ARM-HF: "{{.*}}/usr/lib/arm-linux-gnueabihf{{/|\\\\}}crt1.o"
+// CHECK-UBUNTU-12-04-ARM-HF: "{{.*}}/usr/lib/arm-linux-gnueabihf{{/|\\\\}}crti.o"
 // CHECK-UBUNTU-12-04-ARM-HF: "{{.*}}/usr/lib/gcc/arm-linux-gnueabihf/4.6.3{{/|\\\\}}crtbegin.o"
 // CHECK-UBUNTU-12-04-ARM-HF: "-L[[SYSROOT]]/usr/lib/gcc/arm-linux-gnueabihf/4.6.3"
-// CHECK-UBUNTU-12-04-ARM-HF: "-L[[SYSROOT]]/usr/lib/gcc/arm-linux-gnueabihf/4.6.3/../../../arm-linux-gnueabihf"
 // CHECK-UBUNTU-12-04-ARM-HF: "-L[[SYSROOT]]/lib/arm-linux-gnueabihf"
 // CHECK-UBUNTU-12-04-ARM-HF: "-L[[SYSROOT]]/usr/lib/arm-linux-gnueabihf"
-// CHECK-UBUNTU-12-04-ARM-HF: "-L[[SYSROOT]]/usr/lib/gcc/arm-linux-gnueabihf/4.6.3/../../.."
 // CHECK-UBUNTU-12-04-ARM-HF: "{{.*}}/usr/lib/gcc/arm-linux-gnueabihf/4.6.3{{/|\\\\}}crtend.o"
-// CHECK-UBUNTU-12-04-ARM-HF: "{{.*}}/usr/lib/gcc/arm-linux-gnueabihf/4.6.3/../../../arm-linux-gnueabihf{{/|\\\\}}crtn.o"
+// CHECK-UBUNTU-12-04-ARM-HF: "{{.*}}/usr/lib/arm-linux-gnueabihf{{/|\\\\}}crtn.o"
 //
 // Check Ubuntu 13.10 on x86-64 targeting arm-linux-gnueabihf.
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
@@ -702,16 +604,14 @@
 // RUN:     --sysroot=%S/Inputs/ubuntu_14.04_multiarch_tree \
 // RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-14-04-PPC64LE %s
 // CHECK-UBUNTU-14-04-PPC64LE: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-14-04-PPC64LE: "{{.*}}/usr/lib/gcc/powerpc64le-linux-gnu/4.8/../../../powerpc64le-linux-gnu{{/|\\\\}}crt1.o"
-// CHECK-UBUNTU-14-04-PPC64LE: "{{.*}}/usr/lib/gcc/powerpc64le-linux-gnu/4.8/../../../powerpc64le-linux-gnu{{/|\\\\}}crti.o"
+// CHECK-UBUNTU-14-04-PPC64LE: "{{.*}}/usr/lib/powerpc64le-linux-gnu{{/|\\\\}}crt1.o"
+// CHECK-UBUNTU-14-04-PPC64LE: "{{.*}}/usr/lib/powerpc64le-linux-gnu{{/|\\\\}}crti.o"
 // CHECK-UBUNTU-14-04-PPC64LE: "{{.*}}/usr/lib/gcc/powerpc64le-linux-gnu/4.8{{/|\\\\}}crtbegin.o"
 // CHECK-UBUNTU-14-04-PPC64LE: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64le-linux-gnu/4.8"
-// CHECK-UBUNTU-14-04-PPC64LE: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64le-linux-gnu/4.8/../../../powerpc64le-linux-gnu"
 // CHECK-UBUNTU-14-04-PPC64LE: "-L[[SYSROOT]]/lib/powerpc64le-linux-gnu"
 // CHECK-UBUNTU-14-04-PPC64LE: "-L[[SYSROOT]]/usr/lib/powerpc64le-linux-gnu"
-// CHECK-UBUNTU-14-04-PPC64LE: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64le-linux-gnu/4.8/../../.."
 // CHECK-UBUNTU-14-04-PPC64LE: "{{.*}}/usr/lib/gcc/powerpc64le-linux-gnu/4.8{{/|\\\\}}crtend.o"
-// CHECK-UBUNTU-14-04-PPC64LE: "{{.*}}/usr/lib/gcc/powerpc64le-linux-gnu/4.8/../../../powerpc64le-linux-gnu{{/|\\\\}}crtn.o"
+// CHECK-UBUNTU-14-04-PPC64LE: "{{.*}}/usr/lib/powerpc64le-linux-gnu{{/|\\\\}}crtn.o"
 //
 // Check Ubuntu 14.04 on x32.
 // "/usr/lib/gcc/x86_64-linux-gnu/4.8/x32/crtend.o" "/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../libx32/crtn.o"
@@ -725,12 +625,9 @@
 // CHECK-UBUNTU-14-04-X32: "{{.*}}/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../libx32{{/|\\\\}}crti.o"
 // CHECK-UBUNTU-14-04-X32: "{{.*}}/usr/lib/gcc/x86_64-linux-gnu/4.8/x32{{/|\\\\}}crtbegin.o"
 // CHECK-UBUNTU-14-04-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/x32"
-// CHECK-UBUNTU-14-04-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../libx32"
-// CHECK-UBUNTU-14-04-X32: "-L[[SYSROOT]]/lib/../libx32"
-// CHECK-UBUNTU-14-04-X32: "-L[[SYSROOT]]/usr/lib/../libx32"
-// CHECK-UBUNTU-14-04-X32: "-L[[SYSROOT]]/usr/lib/x86_64-linux-gnu/../../libx32"
-// CHECK-UBUNTU-14-04-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8"
-// CHECK-UBUNTU-14-04-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../.."
+// CHECK-UBUNTU-14-04-X32-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../libx32"
+// CHECK-UBUNTU-14-04-X32-SAME: {{^}} "-L[[SYSROOT]]/lib/../libx32"
+// CHECK-UBUNTU-14-04-X32-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/../libx32"
 // CHECK-UBUNTU-14-04-X32: "{{.*}}/usr/lib/gcc/x86_64-linux-gnu/4.8/x32{{/|\\\\}}crtend.o"
 // CHECK-UBUNTU-14-04-X32: "{{.*}}/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../libx32{{/|\\\\}}crtn.o"
 //
@@ -790,16 +687,14 @@
 // RUN:     --sysroot=%S/Inputs/ubuntu_12.04_LTS_multiarch_tree \
 // RUN:   | FileCheck --check-prefix=CHECK-UBUNTU-12-04-ARM %s
 // CHECK-UBUNTU-12-04-ARM: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-UBUNTU-12-04-ARM: "{{.*}}/usr/lib/gcc/arm-linux-gnueabi/4.6.1/../../../arm-linux-gnueabi{{/|\\\\}}crt1.o"
-// CHECK-UBUNTU-12-04-ARM: "{{.*}}/usr/lib/gcc/arm-linux-gnueabi/4.6.1/../../../arm-linux-gnueabi{{/|\\\\}}crti.o"
+// CHECK-UBUNTU-12-04-ARM: "{{.*}}/usr/lib/arm-linux-gnueabi{{/|\\\\}}crt1.o"
+// CHECK-UBUNTU-12-04-ARM: "{{.*}}/usr/lib/arm-linux-gnueabi{{/|\\\\}}crti.o"
 // CHECK-UBUNTU-12-04-ARM: "{{.*}}/usr/lib/gcc/arm-linux-gnueabi/4.6.1{{/|\\\\}}crtbegin.o"
 // CHECK-UBUNTU-12-04-ARM: "-L[[SYSROOT]]/usr/lib/gcc/arm-linux-gnueabi/4.6.1"
-// CHECK-UBUNTU-12-04-ARM: "-L[[SYSROOT]]/usr/lib/gcc/arm-linux-gnueabi/4.6.1/../../../arm-linux-gnueabi"
 // CHECK-UBUNTU-12-04-ARM: "-L[[SYSROOT]]/lib/arm-linux-gnueabi"
 // CHECK-UBUNTU-12-04-ARM: "-L[[SYSROOT]]/usr/lib/arm-linux-gnueabi"
-// CHECK-UBUNTU-12-04-ARM: "-L[[SYSROOT]]/usr/lib/gcc/arm-linux-gnueabi/4.6.1/../../.."
 // CHECK-UBUNTU-12-04-ARM: "{{.*}}/usr/lib/gcc/arm-linux-gnueabi/4.6.1{{/|\\\\}}crtend.o"
-// CHECK-UBUNTU-12-04-ARM: "{{.*}}/usr/lib/gcc/arm-linux-gnueabi/4.6.1/../../../arm-linux-gnueabi{{/|\\\\}}crtn.o"
+// CHECK-UBUNTU-12-04-ARM: "{{.*}}/usr/lib/arm-linux-gnueabi{{/|\\\\}}crtn.o"
 //
 // Test the setup that shipped in SUSE 10.3 on ppc64.
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
@@ -901,13 +796,12 @@
 // RUN:     --sysroot=%S/Inputs/opensuse_tumbleweed_ppc_tree \
 // RUN:   | FileCheck --check-prefix=CHECK-OPENSUSE-TW-PPC %s
 // CHECK-OPENSUSE-TW-PPC: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-OPENSUSE-TW-PPC: "{{.*}}/usr/lib/gcc/powerpc64-suse-linux/9/../../..{{/|\\\\}}crt1.o"
-// CHECK-OPENSUSE-TW-PPC: "{{.*}}/usr/lib/gcc/powerpc64-suse-linux/9/../../..{{/|\\\\}}crti.o"
+// CHECK-OPENSUSE-TW-PPC: "{{.*}}/usr/lib{{/|\\\\}}crt1.o"
+// CHECK-OPENSUSE-TW-PPC: "{{.*}}/usr/lib{{/|\\\\}}crti.o"
 // CHECK-OPENSUSE-TW-PPC: "{{.*}}/usr/lib/gcc/powerpc64-suse-linux/9{{/|\\\\}}crtbegin.o"
 // CHECK-OPENSUSE-TW-PPC: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64-suse-linux/9"
-// CHECK-OPENSUSE-TW-PPC: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64-suse-linux/9/../../.."
 // CHECK-OPENSUSE-TW-PPC: "{{.*}}/usr/lib/gcc/powerpc64-suse-linux/9{{/|\\\\}}crtend.o"
-// CHECK-OPENSUSE-TW-PPC: "{{.*}}/usr/lib/gcc/powerpc64-suse-linux/9/../../..{{/|\\\\}}crtn.o"
+// CHECK-OPENSUSE-TW-PPC: "{{.*}}/usr/lib/crtn.o"
 //
 // Check dynamic-linker for different archs
 // RUN: %clang %s -### -o %t.o 2>&1 \
@@ -1098,11 +992,11 @@
 // CHECK-ANDROID-NOEXECSTACK-NOT: "-z,execstack"
 // CHECK-ANDROID-NOEXECSTACK-NOT: "-zexecstack"
 
-+// RUN: %clang %s -### -o %t.o 2>&1 \
-+// RUN:     --target=armv7-linux-android21 \
-+// RUN:   | FileCheck --check-prefix=CHECK-ANDROID-WARN-SHARED-TEXTREL %s
-+// CHECK-ANDROID-WARN-SHARED-TEXTREL: "{{.*}}ld{{(.exe)?}}"
-+// CHECK-ANDROID-WARN-SHARED-TEXTREL: "--warn-shared-textrel"
+// RUN: %clang %s -### -o %t.o 2>&1 \
+// RUN:     --target=armv7-linux-android21 \
+// RUN:   | FileCheck --check-prefix=CHECK-ANDROID-WARN-SHARED-TEXTREL %s
+// CHECK-ANDROID-WARN-SHARED-TEXTREL: "{{.*}}ld{{(.exe)?}}"
+// CHECK-ANDROID-WARN-SHARED-TEXTREL: "--warn-shared-textrel"
 
 // RUN: %clang %s -### -o %t.o 2>&1 --target=mips64-linux-gnuabin32 \
 // RUN:   | FileCheck --check-prefix=CHECK-MIPS64EL-GNUABIN32 %s
@@ -1138,174 +1032,8 @@
 // CHECK-SPARCV9: "{{.*}}ld{{(.exe)?}}"
 // CHECK-SPARCV9: "-m" "elf64_sparc"
 // CHECK-SPARCV9: "-dynamic-linker" "{{(/usr/sparcv9-unknown-linux-gnu)?}}/lib{{(64)?}}/ld-linux.so.2"
-//
-// Thoroughly exercise the Debian multiarch environment.
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=i686-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-X86 %s
-// CHECK-DEBIAN-X86: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-X86: "{{.*}}/usr/lib/gcc/i686-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-X86: "-L[[SYSROOT]]/usr/lib/gcc/i686-linux-gnu/4.5"
-// CHECK-DEBIAN-X86: "-L[[SYSROOT]]/usr/lib/gcc/i686-linux-gnu/4.5/../../../i386-linux-gnu"
-// CHECK-DEBIAN-X86: "-L[[SYSROOT]]/usr/lib/i386-linux-gnu"
-// CHECK-DEBIAN-X86: "-L[[SYSROOT]]/usr/lib/gcc/i686-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-X86: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-X86: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=x86_64-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-X86-64 %s
-// CHECK-DEBIAN-X86-64: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-X86-64: "{{.*}}/usr/lib/gcc/x86_64-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-X86-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.5"
-// CHECK-DEBIAN-X86-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.5/../../../x86_64-linux-gnu"
-// CHECK-DEBIAN-X86-64: "-L[[SYSROOT]]/usr/lib/x86_64-linux-gnu"
-// CHECK-DEBIAN-X86-64: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-X86-64: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-X86-64: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=powerpc-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-PPC %s
-// CHECK-DEBIAN-PPC: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-PPC: "{{.*}}/usr/lib/gcc/powerpc-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-PPC: "-L[[SYSROOT]]/usr/lib/gcc/powerpc-linux-gnu/4.5"
-// CHECK-DEBIAN-PPC: "-L[[SYSROOT]]/usr/lib/gcc/powerpc-linux-gnu/4.5/../../../powerpc-linux-gnu"
-// CHECK-DEBIAN-PPC: "-L[[SYSROOT]]/usr/lib/powerpc-linux-gnu"
-// CHECK-DEBIAN-PPC: "-L[[SYSROOT]]/usr/lib/gcc/powerpc-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-PPC: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-PPC: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=powerpc64le-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-PPC64LE %s
-// CHECK-DEBIAN-PPC64LE: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-PPC64LE: "{{.*}}/usr/lib/gcc/powerpc64le-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-PPC64LE: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64le-linux-gnu/4.5"
-// CHECK-DEBIAN-PPC64LE: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64le-linux-gnu/4.5/../../../powerpc64le-linux-gnu"
-// CHECK-DEBIAN-PPC64LE: "-L[[SYSROOT]]/usr/lib/powerpc64le-linux-gnu"
-// CHECK-DEBIAN-PPC64LE: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64le-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-PPC64LE: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-PPC64LE: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=powerpc64-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-PPC64 %s
-// CHECK-DEBIAN-PPC64: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-PPC64: "{{.*}}/usr/lib/gcc/powerpc64-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-PPC64: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64-linux-gnu/4.5"
-// CHECK-DEBIAN-PPC64: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64-linux-gnu/4.5/../../../powerpc64-linux-gnu"
-// CHECK-DEBIAN-PPC64: "-L[[SYSROOT]]/usr/lib/powerpc64-linux-gnu"
-// CHECK-DEBIAN-PPC64: "-L[[SYSROOT]]/usr/lib/gcc/powerpc64-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-PPC64: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-PPC64: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=mips-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-MIPS %s
-// CHECK-DEBIAN-MIPS: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-MIPS: "{{.*}}/usr/lib/gcc/mips-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.5"
-// CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.5/../../../mips-linux-gnu"
-// CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib/mips-linux-gnu"
-// CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=mipsel-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-MIPSEL %s
-// CHECK-DEBIAN-MIPSEL: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-MIPSEL: "{{.*}}/usr/lib/gcc/mipsel-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.5"
-// CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.5/../../../mipsel-linux-gnu"
-// CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib/mipsel-linux-gnu"
-// CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=mips64-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-MIPS64 %s
-// CHECK-DEBIAN-MIPS64: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-MIPS64: "{{.*}}/usr/lib/gcc/mips-linux-gnu/4.5/64{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-MIPS64: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.5/64"
-// CHECK-DEBIAN-MIPS64: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.5"
-// CHECK-DEBIAN-MIPS64: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-MIPS64: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-MIPS64: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=mips64el-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-MIPS64EL %s
-// CHECK-DEBIAN-MIPS64EL: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-MIPS64EL: "{{.*}}/usr/lib/gcc/mipsel-linux-gnu/4.5/64{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-MIPS64EL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.5/64"
-// CHECK-DEBIAN-MIPS64EL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.5"
-// CHECK-DEBIAN-MIPS64EL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-MIPS64EL: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-MIPS64EL: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=mips64-linux-gnu -rtlib=platform -mabi=n32 \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-MIPS64-N32 %s
-// CHECK-DEBIAN-MIPS64-N32: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-MIPS64-N32: "{{.*}}/usr/lib/gcc/mips-linux-gnu/4.5/n32{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-MIPS64-N32: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.5/n32"
-// CHECK-DEBIAN-MIPS64-N32: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.5"
-// CHECK-DEBIAN-MIPS64-N32: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-MIPS64-N32: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-MIPS64-N32: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=mips64el-linux-gnu -rtlib=platform -mabi=n32 \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-MIPS64EL-N32 %s
-// CHECK-DEBIAN-MIPS64EL-N32: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-MIPS64EL-N32: "{{.*}}/usr/lib/gcc/mipsel-linux-gnu/4.5/n32{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-MIPS64EL-N32: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.5/n32"
-// CHECK-DEBIAN-MIPS64EL-N32: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.5"
-// CHECK-DEBIAN-MIPS64EL-N32: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-MIPS64EL-N32: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-MIPS64EL-N32: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=sparc-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-SPARC %s
-// CHECK-DEBIAN-SPARC:      "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-SPARC-SAME: "{{.*}}/usr/lib/gcc/sparc-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-SPARC-SAME: "-L[[SYSROOT]]/usr/lib/gcc/sparc-linux-gnu/4.5"
-// CHECK-DEBIAN-SPARC-SAME: "-L[[SYSROOT]]/usr/lib/gcc/sparc-linux-gnu/4.5/../../../sparc-linux-gnu"
-// CHECK-DEBIAN-SPARC-SAME: "-L[[SYSROOT]]/usr/lib/sparc-linux-gnu"
-// CHECK-DEBIAN-SPARC-SAME: "-L[[SYSROOT]]/usr/lib/gcc/sparc-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-SPARC-SAME: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-SPARC-SAME: "-L[[SYSROOT]]/usr/lib"
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
-// RUN:     --target=sparc64-linux-gnu -rtlib=platform \
-// RUN:     --gcc-toolchain="" \
-// RUN:     --sysroot=%S/Inputs/debian_multiarch_tree \
-// RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-SPARC64 %s
-// CHECK-DEBIAN-SPARC64:      "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-SPARC64-SAME: "{{.*}}/usr/lib/gcc/sparc64-linux-gnu/4.5{{/|\\\\}}crtbegin.o"
-// CHECK-DEBIAN-SPARC64-SAME: "-L[[SYSROOT]]/usr/lib/gcc/sparc64-linux-gnu/4.5"
-// CHECK-DEBIAN-SPARC64-SAME: "-L[[SYSROOT]]/usr/lib/gcc/sparc64-linux-gnu/4.5/../../../sparc64-linux-gnu"
-// CHECK-DEBIAN-SPARC64-SAME: "-L[[SYSROOT]]/usr/lib/sparc64-linux-gnu"
-// CHECK-DEBIAN-SPARC64-SAME: "-L[[SYSROOT]]/usr/lib/gcc/sparc64-linux-gnu/4.5/../../.."
-// CHECK-DEBIAN-SPARC64-SAME: "-L[[SYSROOT]]/lib"
-// CHECK-DEBIAN-SPARC64-SAME: "-L[[SYSROOT]]/usr/lib"
-//
+
+
 // Test linker invocation on Android.
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     --target=arm-linux-androideabi -rtlib=platform --unwindlib=platform \
@@ -1685,7 +1413,6 @@
 // CHECK-DEBIAN-ML-MIPSEL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.4/../../../../lib"
 // CHECK-DEBIAN-ML-MIPSEL: "-L[[SYSROOT]]/lib/../lib"
 // CHECK-DEBIAN-ML-MIPSEL: "-L[[SYSROOT]]/usr/lib/../lib"
-// CHECK-DEBIAN-ML-MIPSEL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.4/../../.."
 // CHECK-DEBIAN-ML-MIPSEL: "-L[[SYSROOT]]/lib"
 // CHECK-DEBIAN-ML-MIPSEL: "-L[[SYSROOT]]/usr/lib"
 //
@@ -1702,7 +1429,6 @@
 // CHECK-DEBIAN-ML-MIPS64EL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.4/../../../../lib64"
 // CHECK-DEBIAN-ML-MIPS64EL: "-L[[SYSROOT]]/lib/../lib64"
 // CHECK-DEBIAN-ML-MIPS64EL: "-L[[SYSROOT]]/usr/lib/../lib64"
-// CHECK-DEBIAN-ML-MIPS64EL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.4/../../.."
 // CHECK-DEBIAN-ML-MIPS64EL: "-L[[SYSROOT]]/lib"
 // CHECK-DEBIAN-ML-MIPS64EL: "-L[[SYSROOT]]/usr/lib"
 //
@@ -1719,7 +1445,6 @@
 // CHECK-DEBIAN-ML-MIPS64EL-N32: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.4/../../../../lib32"
 // CHECK-DEBIAN-ML-MIPS64EL-N32: "-L[[SYSROOT]]/lib/../lib32"
 // CHECK-DEBIAN-ML-MIPS64EL-N32: "-L[[SYSROOT]]/usr/lib/../lib32"
-// CHECK-DEBIAN-ML-MIPS64EL-N32: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.4/../../.."
 // CHECK-DEBIAN-ML-MIPS64EL-N32: "-L[[SYSROOT]]/lib"
 // CHECK-DEBIAN-ML-MIPS64EL-N32: "-L[[SYSROOT]]/usr/lib"
 //
@@ -1734,19 +1459,16 @@
 // RUN:     --sysroot=%S/Inputs/debian_6_mips64_tree \
 // RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-ML-MIPS64-GNUABI %s
 // CHECK-DEBIAN-ML-MIPS64-GNUABI: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-ML-MIPS64-GNUABI: "{{.*}}/usr/lib/gcc/mips64-linux-gnuabi64/4.9/../../../mips64-linux-gnuabi64{{/|\\\\}}crt1.o"
-// CHECK-DEBIAN-ML-MIPS64-GNUABI: "{{.*}}/usr/lib/gcc/mips64-linux-gnuabi64/4.9/../../../mips64-linux-gnuabi64{{/|\\\\}}crti.o"
+// CHECK-DEBIAN-ML-MIPS64-GNUABI: "{{.*}}/usr/lib/mips64-linux-gnuabi64{{/|\\\\}}crt1.o"
+// CHECK-DEBIAN-ML-MIPS64-GNUABI: "{{.*}}/usr/lib/mips64-linux-gnuabi64{{/|\\\\}}crti.o"
 // CHECK-DEBIAN-ML-MIPS64-GNUABI: "{{.*}}/usr/lib/gcc/mips64-linux-gnuabi64/4.9{{/|\\\\}}crtbegin.o"
 // CHECK-DEBIAN-ML-MIPS64-GNUABI: "-L[[SYSROOT]]/usr/lib/gcc/mips64-linux-gnuabi64/4.9"
-// CHECK-DEBIAN-ML-MIPS64-GNUABI: "-L[[SYSROOT]]/usr/lib/gcc/mips64-linux-gnuabi64/4.9/../../../mips64-linux-gnuabi64"
 // CHECK-DEBIAN-ML-MIPS64-GNUABI: "-L[[SYSROOT]]/lib/mips64-linux-gnuabi64"
 // CHECK-DEBIAN-ML-MIPS64-GNUABI: "-L[[SYSROOT]]/usr/lib/mips64-linux-gnuabi64"
-// CHECK-DEBIAN-ML-MIPS64-GNUABI: "-L[[SYSROOT]]/usr/lib/gcc/mips64-linux-gnuabi64/4.9"
-// CHECK-DEBIAN-ML-MIPS64-GNUABI: "-L[[SYSROOT]]/usr/lib/gcc/mips64-linux-gnuabi64/4.9/../../.."
 // CHECK-DEBIAN-ML-MIPS64-GNUABI: "-L[[SYSROOT]]/lib"
 // CHECK-DEBIAN-ML-MIPS64-GNUABI: "-L[[SYSROOT]]/usr/lib"
 // CHECK-DEBIAN-ML-MIPS64-GNUABI: "{{.*}}/usr/lib/gcc/mips64-linux-gnuabi64/4.9{{/|\\\\}}crtend.o"
-// CHECK-DEBIAN-ML-MIPS64-GNUABI: "{{.*}}/usr/lib/gcc/mips64-linux-gnuabi64/4.9/../../../mips64-linux-gnuabi64{{/|\\\\}}crtn.o"
+// CHECK-DEBIAN-ML-MIPS64-GNUABI: "{{.*}}/usr/lib/mips64-linux-gnuabi64{{/|\\\\}}crtn.o"
 //
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     --target=mips64el-unknown-linux-gnu -rtlib=platform \
@@ -1759,19 +1481,16 @@
 // RUN:     --sysroot=%S/Inputs/debian_6_mips64_tree \
 // RUN:   | FileCheck --check-prefix=CHECK-DEBIAN-ML-MIPS64EL-GNUABI %s
 // CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "{{.*}}/usr/lib/gcc/mips64el-linux-gnuabi64/4.9/../../../mips64el-linux-gnuabi64{{/|\\\\}}crt1.o"
-// CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "{{.*}}/usr/lib/gcc/mips64el-linux-gnuabi64/4.9/../../../mips64el-linux-gnuabi64{{/|\\\\}}crti.o"
+// CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "{{.*}}/usr/lib/mips64el-linux-gnuabi64{{/|\\\\}}crt1.o"
+// CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "{{.*}}/usr/lib/mips64el-linux-gnuabi64{{/|\\\\}}crti.o"
 // CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "{{.*}}/usr/lib/gcc/mips64el-linux-gnuabi64/4.9{{/|\\\\}}crtbegin.o"
 // CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "-L[[SYSROOT]]/usr/lib/gcc/mips64el-linux-gnuabi64/4.9"
-// CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "-L[[SYSROOT]]/usr/lib/gcc/mips64el-linux-gnuabi64/4.9/../../../mips64el-linux-gnuabi64"
 // CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "-L[[SYSROOT]]/lib/mips64el-linux-gnuabi64"
 // CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "-L[[SYSROOT]]/usr/lib/mips64el-linux-gnuabi64"
-// CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "-L[[SYSROOT]]/usr/lib/gcc/mips64el-linux-gnuabi64/4.9"
-// CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "-L[[SYSROOT]]/usr/lib/gcc/mips64el-linux-gnuabi64/4.9/../../.."
 // CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "-L[[SYSROOT]]/lib"
 // CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "-L[[SYSROOT]]/usr/lib"
 // CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "{{.*}}/usr/lib/gcc/mips64el-linux-gnuabi64/4.9{{/|\\\\}}crtend.o"
-// CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "{{.*}}/usr/lib/gcc/mips64el-linux-gnuabi64/4.9/../../../mips64el-linux-gnuabi64{{/|\\\\}}crtn.o"
+// CHECK-DEBIAN-ML-MIPS64EL-GNUABI: "{{.*}}/usr/lib/mips64el-linux-gnuabi64{{/|\\\\}}crtn.o"
 //
 // Test linker invocation for Freescale SDK (OpenEmbedded).
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
@@ -1793,7 +1512,6 @@
 // CHECK-FSL-PPC64: "-m" "elf64ppc"
 // CHECK-FSL-PPC64: "{{.*}}{{/|\\\\}}crt1.o"
 // CHECK-FSL-PPC64: "{{.*}}{{/|\\\\}}crtbegin.o"
-// CHECK-FSL-PPC64: "-L[[SYSROOT]]/usr/lib64/powerpc64-fsl-linux/4.6.2/../.."
 //
 // Check that crtfastmath.o is linked with -ffast-math and with -Ofast.
 // RUN: %clang --target=x86_64-unknown-linux -### %s \
@@ -2058,7 +1776,6 @@
 // CHECK-LD-GENTOO: "{{.*}}/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3{{/|\\\\}}crtbegin.o"
 // CHECK-LD-GENTOO: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3"
 // CHECK-LD-GENTOO: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/../../../../x86_64-pc-linux-gnu/lib"
-// CHECK-LD-GENTOO: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/../../.."
 // CHECK-LD-GENTOO: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
 // CHECK-LD-GENTOO: "-lc"
 // CHECK-LD-GENTOO: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
@@ -2075,7 +1792,6 @@
 // CHECK-LD-GENTOO-32: "{{.*}}/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/32{{/|\\\\}}crtbegin.o"
 // CHECK-LD-GENTOO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/32"
 // CHECK-LD-GENTOO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/../../../../x86_64-pc-linux-gnu/lib"
-// CHECK-LD-GENTOO-32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/../../.."
 // CHECK-LD-GENTOO-32: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
 // CHECK-LD-GENTOO-32: "-lc"
 // CHECK-LD-GENTOO-32: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
@@ -2092,7 +1808,6 @@
 // CHECK-LD-GENTOO-X32: "{{.*}}/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/x32{{/|\\\\}}crtbegin.o"
 // CHECK-LD-GENTOO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/x32"
 // CHECK-LD-GENTOO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/../../../../x86_64-pc-linux-gnu/lib"
-// CHECK-LD-GENTOO-X32: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-pc-linux-gnu/4.9.3/../../.."
 // CHECK-LD-GENTOO-X32: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
 // CHECK-LD-GENTOO-X32: "-lc"
 // CHECK-LD-GENTOO-X32: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
@@ -2121,7 +1836,6 @@
 // CHECK-LD-AMI: "{{.*}}/usr/lib/gcc/x86_64-amazon-linux/7{{/|\\\\}}crtbegin.o"
 // CHECK-LD-AMI: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-amazon-linux/7"
 // CHECK-LD-AMI: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-amazon-linux/7/../../../../lib64"
-// CHECK-LD-AMI: "-L[[SYSROOT]]/usr/lib/gcc/x86_64-amazon-linux/7/../../.."
 // CHECK-LD-AMI: "-L[[SYSROOT]]/lib"
 // CHECK-LD-AMI: "-L[[SYSROOT]]/usr/lib"
 // CHECK-LD-AMI: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
@@ -2142,7 +1856,6 @@
 // CHECK-OE-ARM: "[[SYSROOT]]/usr/lib/arm-oe-linux-gnueabi/6.3.0/../../../lib{{/|\\\\}}crti.o"
 // CHECK-OE-ARM: "[[SYSROOT]]/usr/lib/arm-oe-linux-gnueabi/6.3.0{{/|\\\\}}crtbegin.o"
 // CHECK-OE-ARM: "-L[[SYSROOT]]/usr/lib/arm-oe-linux-gnueabi/6.3.0"
-// CHECK-OE-ARM: "-L[[SYSROOT]]/usr/lib/arm-oe-linux-gnueabi"
 // CHECK-OE-ARM: "-L[[SYSROOT]]/usr/lib"
 // CHECK-OE-ARM: "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed"
 // CHECK-OE-ARM: "[[SYSROOT]]/usr/lib/arm-oe-linux-gnueabi/6.3.0{{/|\\\\}}crtend.o"
diff --git a/clang/test/Driver/lit.local.cfg b/clang/test/Driver/lit.local.cfg
index 7ab6c29ec9482051cf24835834d32ea30aac3fab..fe5d67a0f45bf902808f480a904e6e113ed6040f 100644
--- a/clang/test/Driver/lit.local.cfg
+++ b/clang/test/Driver/lit.local.cfg
@@ -1,5 +1,5 @@
 config.suffixes = ['.c', '.cpp', '.h', '.m', '.mm', '.S', '.s', '.f90', '.F90', '.f95',
-                   '.cu', '.rs', '.cl', '.hip']
+                   '.cu', '.rs', '.cl', '.clcpp', '.hip']
 config.substitutions = list(config.substitutions)
 config.substitutions.insert(0,
     ('%clang_cc1',
diff --git a/clang/test/Driver/mips-reduced-toolchain.cpp b/clang/test/Driver/mips-reduced-toolchain.cpp
index 894bdb5a756b750cff0e3a91cf1b84f8e0308d51..13e30a5687f14bc6d3f25f99d64b185e452e1c6e 100644
--- a/clang/test/Driver/mips-reduced-toolchain.cpp
+++ b/clang/test/Driver/mips-reduced-toolchain.cpp
@@ -9,9 +9,7 @@
 // CHECK-DEBIAN-MIPS: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
 // CHECK-DEBIAN-MIPS: "{{.*}}/usr/lib/gcc/mips-linux-gnu/4.7{{/|\\\\}}crtbegin.o"
 // CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.7"
-// CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.7/../../../mips-linux-gnu"
 // CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib/mips-linux-gnu"
-// CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib/gcc/mips-linux-gnu/4.7/../../.."
 // CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/lib"
 // CHECK-DEBIAN-MIPS: "-L[[SYSROOT]]/usr/lib"
 
@@ -23,8 +21,6 @@
 // CHECK-DEBIAN-MIPSEL: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
 // CHECK-DEBIAN-MIPSEL: "{{.*}}/usr/lib/gcc/mipsel-linux-gnu/4.7{{/|\\\\}}crtbegin.o"
 // CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.7"
-// CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.7/../../../mipsel-linux-gnu"
 // CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib/mipsel-linux-gnu"
-// CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib/gcc/mipsel-linux-gnu/4.7/../../.."
 // CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/lib"
 // CHECK-DEBIAN-MIPSEL: "-L[[SYSROOT]]/usr/lib"
diff --git a/clang/test/Driver/msvc-link.c b/clang/test/Driver/msvc-link.c
index 13dccd21bfd8d1c0ed7ec61c28670fe189d2f3ad..1ee17fc63c321359edade2c847b2c04282bb00d7 100644
--- a/clang/test/Driver/msvc-link.c
+++ b/clang/test/Driver/msvc-link.c
@@ -1,4 +1,4 @@
-// RUN: %clang -target i686-pc-windows-msvc -### %s 2>&1 | FileCheck --check-prefix=BASIC %s
+// RUN: %clang -target i686-pc-windows-msvc -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=BASIC %s
 // BASIC: link.exe"
 // BASIC: "-out:a.exe"
 // BASIC: "-defaultlib:libcmt"
@@ -6,7 +6,7 @@
 // BASIC: "-nologo"
 // BASIC-NOT: "-Brepro"
 
-// RUN: %clang -target i686-pc-windows-msvc -shared -o a.dll -### %s 2>&1 | FileCheck --check-prefix=DLL %s
+// RUN: %clang -target i686-pc-windows-msvc -shared -o a.dll -fuse-ld=link -### %s 2>&1 | FileCheck --check-prefix=DLL %s
 // DLL: link.exe"
 // DLL: "-out:a.dll"
 // DLL: "-defaultlib:libcmt"
@@ -19,13 +19,13 @@
 // LIBPATH: "-libpath:/usr/lib"
 // LIBPATH: "-nologo"
 
-// RUN: %clang_cl /Brepro -### -- %s 2>&1 | FileCheck --check-prefix=REPRO %s
+// RUN: %clang_cl /Brepro -fuse-ld=link -### -- %s 2>&1 | FileCheck --check-prefix=REPRO %s
 // REPRO: link.exe"
 // REPRO: "-out:msvc-link.exe"
 // REPRO: "-nologo"
 // REPRO: "-Brepro"
 
-// RUN: %clang_cl /Brepro- -### -- %s 2>&1 | FileCheck --check-prefix=NOREPRO %s
+// RUN: %clang_cl /Brepro- -fuse-ld=link -### -- %s 2>&1 | FileCheck --check-prefix=NOREPRO %s
 // NOREPRO: link.exe"
 // NOREPRO: "-out:msvc-link.exe"
 // NOREPRO: "-nologo"
diff --git a/clang/test/Driver/nostdincxx.cpp b/clang/test/Driver/nostdincxx.cpp
index dc87336ece54c26b119caeaa744c859d29011154..3fc7a5e76842423322211098c684e7591cba9ec9 100644
--- a/clang/test/Driver/nostdincxx.cpp
+++ b/clang/test/Driver/nostdincxx.cpp
@@ -1,4 +1,6 @@
+// RUN: not %clangxx -nostdinc %s 2>&1 | FileCheck %s
 // RUN: not %clangxx -nostdinc++ %s 2>&1 | FileCheck %s
+// RUN: not %clangxx -nostdlibinc %s 2>&1 | FileCheck %s
 // CHECK: file not found
 #include <vector> 
 
diff --git a/clang/test/Driver/ppc-mprivileged-support-check.c b/clang/test/Driver/ppc-mprivileged-support-check.c
new file mode 100644
index 0000000000000000000000000000000000000000..164b4d9483d321fb4ff883998a17bbaeec8d40a3
--- /dev/null
+++ b/clang/test/Driver/ppc-mprivileged-support-check.c
@@ -0,0 +1,26 @@
+// RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
+// RUN:   -mcpu=pwr10 -mprivileged %s 2>&1 | FileCheck %s --check-prefix=HASPRIV
+// RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
+// RUN:   -mcpu=power10 -mprivileged %s 2>&1 | FileCheck %s --check-prefix=HASPRIV
+// RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
+// RUN:   -mcpu=pwr9 -mprivileged %s 2>&1 | FileCheck %s --check-prefix=HASPRIV
+// RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
+// RUN:   -mcpu=power9 -mprivileged %s 2>&1 | FileCheck %s --check-prefix=HASPRIV
+// RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
+// RUN:   -mcpu=pwr8 -mprivileged %s 2>&1 | FileCheck %s --check-prefix=HASPRIV
+// RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
+// RUN:   -mcpu=power8 -mprivileged %s 2>&1 | FileCheck %s --check-prefix=HASPRIV
+
+// RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
+// RUN:   -mcpu=pwr7 -mprivileged %s 2>&1 | FileCheck %s --check-prefix=NOPRIV
+// RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
+// RUN:   -mcpu=power7 -mprivileged %s 2>&1 | FileCheck %s --check-prefix=NOPRIV
+
+#ifdef __PRIVILEGED__
+static_assert(false, "Privileged instructions enabled");
+#endif
+
+// HASPRIV: Privileged instructions enabled
+// HASPRIV-NOT: option '-mprivileged' cannot be specified with
+// NOPRIV: option '-mprivileged' cannot be specified with
+
diff --git a/clang/test/Driver/ppc-mrop-protection-support-check.c b/clang/test/Driver/ppc-mrop-protection-support-check.c
index c2761d21c9d1c324ac8a1ddc052aab9852bbfd69..50eaef3ed770b83c36fbbf479eabadf4ede6c9d4 100644
--- a/clang/test/Driver/ppc-mrop-protection-support-check.c
+++ b/clang/test/Driver/ppc-mrop-protection-support-check.c
@@ -1,26 +1,26 @@
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=pwr10 -mrop-protection %s 2>&1 | FileCheck %s --check-prefix=HASROP
+// RUN:   -mcpu=pwr10 -mrop-protect %s 2>&1 | FileCheck %s --check-prefix=HASROP
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=power10 -mrop-protection %s 2>&1 | FileCheck %s --check-prefix=HASROP
+// RUN:   -mcpu=power10 -mrop-protect %s 2>&1 | FileCheck %s --check-prefix=HASROP
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=pwr9 -mrop-protection %s 2>&1 | FileCheck %s --check-prefix=HASROP
+// RUN:   -mcpu=pwr9 -mrop-protect %s 2>&1 | FileCheck %s --check-prefix=HASROP
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=power9 -mrop-protection %s 2>&1 | FileCheck %s --check-prefix=HASROP
+// RUN:   -mcpu=power9 -mrop-protect %s 2>&1 | FileCheck %s --check-prefix=HASROP
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=pwr8 -mrop-protection %s 2>&1 | FileCheck %s --check-prefix=HASROP
+// RUN:   -mcpu=pwr8 -mrop-protect %s 2>&1 | FileCheck %s --check-prefix=HASROP
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=power8 -mrop-protection %s 2>&1 | FileCheck %s --check-prefix=HASROP
+// RUN:   -mcpu=power8 -mrop-protect %s 2>&1 | FileCheck %s --check-prefix=HASROP
 
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=pwr7 -mrop-protection %s 2>&1 | FileCheck %s --check-prefix=NOROP
+// RUN:   -mcpu=pwr7 -mrop-protect %s 2>&1 | FileCheck %s --check-prefix=NOROP
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=power7 -mrop-protection %s 2>&1 | FileCheck %s --check-prefix=NOROP
+// RUN:   -mcpu=power7 -mrop-protect %s 2>&1 | FileCheck %s --check-prefix=NOROP
 
-#ifdef __ROP_PROTECTION__
-static_assert(false, "ROP Protection enabled");
+#ifdef __ROP_PROTECT__
+static_assert(false, "ROP Protect enabled");
 #endif
 
-// HASROP: ROP Protection enabled
-// HASROP-NOT: option '-mrop-protection' cannot be specified with
-// NOROP: option '-mrop-protection' cannot be specified with
+// HASROP: ROP Protect enabled
+// HASROP-NOT: option '-mrop-protect' cannot be specified with
+// NOROP: option '-mrop-protect' cannot be specified with
 
diff --git a/clang/test/Driver/print-multi-directory.c b/clang/test/Driver/print-multi-directory.c
index ee927e857f2e538975cbcb67db000015f84083d6..e0bd6e4fe8a1ce8df4973d4561747bde949368f2 100644
--- a/clang/test/Driver/print-multi-directory.c
+++ b/clang/test/Driver/print-multi-directory.c
@@ -20,7 +20,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>/dev/null \
 // RUN:     -target arm-linux-androideabi21 \
 // RUN:     -mthumb \
-// RUN:     -B%S/Inputs/basic_android_ndk_tree \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_android_ndk_tree \
 // RUN:     --sysroot=%S/Inputs/basic_android_ndk_tree/sysroot \
 // RUN:     -print-multi-directory \
 // RUN:   | FileCheck --match-full-lines --check-prefix=CHECK-ARM-MULTILIBS %s
diff --git a/clang/test/Driver/rocm-detect.hip b/clang/test/Driver/rocm-detect.hip
index cd3c2d90fe88cc046b64030db9ecbb88e011f2ed..1e21323ba218082e48902f9de1e1c93d9bb1b41b 100644
--- a/clang/test/Driver/rocm-detect.hip
+++ b/clang/test/Driver/rocm-detect.hip
@@ -21,6 +21,20 @@
 // RUN:   --rocm-path=%S/Inputs/rocm %s 2>&1 \
 // RUN:   | FileCheck -check-prefixes=COMMON,NODEFAULTLIBS %s
 
+// Test environment variable ROCM_PATH.
+// RUN: env ROCM_PATH=%S/Inputs/rocm %clang -### -target x86_64-linux-gnu \
+// RUN:   --print-rocm-search-dirs %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=ROCM-ENV %s
+
+// Test detecting latest /opt/rocm-{release} directory.
+// RUN: rm -rf %T/opt
+// RUN: mkdir -p %T/opt
+// RUN: cp -r %S/Inputs/rocm %T/opt/rocm-3.9.0-1234
+// RUN: cp -r %S/Inputs/rocm %T/opt/rocm-3.10.0
+// RUN: %clang -### -target x86_64-linux-gnu --sysroot=%T \
+// RUN:   --print-rocm-search-dirs %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=ROCM-REL %s
+
 // Test ROCm installation built by SPACK by invoke clang at %T/rocm-spack/llvm-amdgpu-*
 // directory through a soft link.
 
@@ -60,6 +74,11 @@
 
 // COMMON: "-triple" "amdgcn-amd-amdhsa"
 
+// ROCM-ENV: ROCm installation search path: {{.*}}/Inputs/rocm
+
+// ROCM-REL: ROCm installation search path: {{.*}}/opt/rocm
+// ROCM-REL: ROCm installation search path: {{.*}}/opt/rocm-3.10.0
+
 // SPACK: ROCm installation search path (Spack 4.0.0): [[DIR:.*]]
 // SPACK: ROCm installation search path: [[CLANG:.*]]
 // SPACK: ROCm installation search path: [[CLANG]]/lib/clang/{{[0-9.]+}}
diff --git a/clang/test/Driver/stdlibxx-isystem.cpp b/clang/test/Driver/stdlibxx-isystem.cpp
index 827cdf9a7c71f08444afbbb8e555e24a7f52ff87..cadfa25c0db93641107218faa68aec97e6032622 100644
--- a/clang/test/Driver/stdlibxx-isystem.cpp
+++ b/clang/test/Driver/stdlibxx-isystem.cpp
@@ -43,11 +43,19 @@
 // RUN:   FileCheck -check-prefix=NOCC1 %s
 // NOCC1-NOT: "-stdlib++-isystem" "/tmp"
 
-// It should respect -nostdinc++.
+// It should respect -nostdinc++
 // RUN: %clang -target aarch64-linux-gnu -ccc-install-dir %t/bin \
 // RUN:   -stdlib++-isystem /tmp/foo -stdlib++-isystem /tmp/bar -nostdinc++ \
 // RUN:   -fsyntax-only %s -### 2>&1 | FileCheck -check-prefix=NOSTDINCXX %s
 // RUN: %clang -target x86_64-apple-darwin -ccc-install-dir %t/bin \
 // RUN:   -stdlib++-isystem /tmp/foo -stdlib++-isystem /tmp/bar -nostdinc++ \
 // RUN:   -fsyntax-only %s -### 2>&1 | FileCheck -check-prefix=NOSTDINCXX %s
+
+// ... and -nostdinc and -nostdlibinc.
+// RUN: %clang -target aarch64-linux-gnu -ccc-install-dir %t/bin \
+// RUN:   -stdlib++-isystem /tmp/foo -stdlib++-isystem /tmp/bar -nostdinc \
+// RUN:   -fsyntax-only %s -### 2>&1 | FileCheck --check-prefix=NOSTDINCXX %s
+// RUN: %clang -target aarch64-linux-gnu -ccc-install-dir %t/bin \
+// RUN:   -stdlib++-isystem /tmp/foo -stdlib++-isystem /tmp/bar -nostdlibinc \
+// RUN:   -fsyntax-only %s -### 2>&1 | FileCheck --check-prefix=NOSTDINCXX %s
 // NOSTDINCXX-NOT: "-internal-isystem" "/tmp/foo" "-internal-isystem" "/tmp/bar"
diff --git a/clang/test/Driver/verify-debug-info-preservation.c b/clang/test/Driver/verify-debug-info-preservation.c
new file mode 100644
index 0000000000000000000000000000000000000000..81c1ff7d02a24735666d31883335121bd1f30983
--- /dev/null
+++ b/clang/test/Driver/verify-debug-info-preservation.c
@@ -0,0 +1,19 @@
+// We support the CC1 options for testing whether each LLVM pass preserves
+// original debug info.
+
+// RUN: %clang -g -Xclang -fverify-debuginfo-preserve -### %s 2>&1 \
+// RUN:     | FileCheck --check-prefix=VERIFYDIPRESERVE %s
+
+// VERIFYDIPRESERVE: "-fverify-debuginfo-preserve"
+
+// RUN: %clang -g -Xclang -fverify-debuginfo-preserve \
+// RUN:     -Xclang -fverify-debuginfo-preserve-export=%t.json -### %s 2>&1 \
+// RUN:     | FileCheck --check-prefix=VERIFYDIPRESERVE-JSON-EXPORT %s
+
+// VERIFYDIPRESERVE-JSON-EXPORT: "-fverify-debuginfo-preserve"
+// VERIFYDIPRESERVE-JSON-EXPORT: "-fverify-debuginfo-preserve-export={{.*}}"
+
+// RUN: %clang -g -Xclang -fverify-debuginfo-preserve-export=%t.json %s -S -o /dev/null 2>&1 \
+// RUN:     | FileCheck --check-prefix=WARN %s
+
+// WARN: warning: ignoring -fverify-debuginfo-preserve-export
diff --git a/clang/test/Driver/wasm-toolchain.c b/clang/test/Driver/wasm-toolchain.c
index 17037819cfda54070a2528911bf4fee2ae1333a1..d2e6de4667ac84617d5a6284e83ba4eb1f91e18d 100644
--- a/clang/test/Driver/wasm-toolchain.c
+++ b/clang/test/Driver/wasm-toolchain.c
@@ -79,6 +79,21 @@
 // RUN:   | FileCheck -check-prefix=PTHREAD_NO_SIGN_EXT %s
 // PTHREAD_NO_SIGN_EXT: invalid argument '-pthread' not allowed with '-mno-sign-ext'
 
+// '-mllvm -emscripten-cxx-exceptions-allowed=foo,bar' sets
+// '-mllvm --force-attribute=foo:noinline -mllvm --force-attribute=bar:noinline'
+// RUN: %clang -### -no-canonical-prefixes -target wasm32-unknown-unknown \
+// RUN:    --sysroot=/foo %s -mllvm -enable-emscripten-cxx-exceptions \
+// RUN:    -mllvm -emscripten-cxx-exceptions-allowed=foo,bar 2>&1 \
+// RUN:  | FileCheck -check-prefix=EMSCRIPTEN_EH_ALLOWED_NOINLINE %s
+// EMSCRIPTEN_EH_ALLOWED_NOINLINE: clang{{.*}}" "-cc1" {{.*}} "-mllvm" "--force-attribute=foo:noinline" "-mllvm" "--force-attribute=bar:noinline"
+
+// '-mllvm -emscripten-cxx-exceptions-allowed' only allowed with
+// '-mllvm -enable-emscripten-cxx-exceptions'
+// RUN: %clang -### -no-canonical-prefixes -target wasm32-unknown-unknown \
+// RUN:     --sysroot=/foo %s -mllvm -emscripten-cxx-exceptions-allowed 2>&1 \
+// RUN:   | FileCheck -check-prefix=EMSCRIPTEN_EH_ALLOWED_WO_ENABLE %s
+// EMSCRIPTEN_EH_ALLOWED_WO_ENABLE: invalid argument '-mllvm -emscripten-cxx-exceptions-allowed' only allowed with '-mllvm -enable-emscripten-cxx-exceptions'
+
 // '-fwasm-exceptions' sets +exception-handling
 // RUN: %clang -### -no-canonical-prefixes -target wasm32-unknown-unknown \
 // RUN:    --sysroot=/foo %s -fwasm-exceptions 2>&1 \
diff --git a/clang/test/FixIt/fixit-c++11.cpp b/clang/test/FixIt/fixit-c++11.cpp
index 70342075a15e44a3cc56ee558e1771afb1035348..9a2020c41870ac110feae30909227604ae100541 100644
--- a/clang/test/FixIt/fixit-c++11.cpp
+++ b/clang/test/FixIt/fixit-c++11.cpp
@@ -56,8 +56,12 @@ void S2::f(int i) {
   (void)[&, &i, &i]{}; // expected-error 2{{'&' cannot precede a capture when the capture default is '&'}}
   (void)[i, i]{ }; // expected-error{{'i' can appear only once in a capture list}}
   (void)[&, i, i]{ }; // expected-error{{'i' can appear only once in a capture list}}
-  (void)[] mutable { }; // expected-error{{lambda requires '()' before 'mutable'}}
-  (void)[] -> int { }; // expected-error{{lambda requires '()' before return type}}
+  (void)[] mutable {};
+  (void)[]->int{};
+#if __cplusplus <= 202002L
+  // expected-warning@-3{{is a C++2b extension}}
+  // expected-warning@-3{{is a C++2b extension}}
+#endif
 
   delete []() { return new int; }(); // expected-error{{'[]' after delete interpreted as 'delete[]'}}
   delete [] { return new int; }(); // expected-error{{'[]' after delete interpreted as 'delete[]'}}
diff --git a/clang/test/FixIt/fixit-cxx-init-order.cpp b/clang/test/FixIt/fixit-cxx-init-order.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f39c8bf2a15c87bbb8ae5e8e43c5622c7f59f7c9
--- /dev/null
+++ b/clang/test/FixIt/fixit-cxx-init-order.cpp
@@ -0,0 +1,22 @@
+// Due to the fix having multiple edits we can't use
+// '-fdiagnostics-parseable-fixits' to determine if fixes are correct. However,
+// running fixit recompile with 'Werror' should fail if the fixes are invalid.
+
+// RUN: %clang_cc1 %s -Werror=reorder-ctor -fixit-recompile -fixit-to-temporary
+// RUN: %clang_cc1 %s -Wreorder-ctor -verify -verify-ignore-unexpected=note
+
+struct Foo {
+  int A, B, C;
+
+  Foo() : A(1), B(3), C(2) {}
+  Foo(int) : A(1), C(2), B(3) {}      // expected-warning {{field 'C' will be initialized after field 'B'}}
+  Foo(unsigned) : C(2), B(3), A(1) {} // expected-warning {{initializer order does not match the declaration order}}
+};
+
+struct Bar : Foo {
+  int D, E, F;
+
+  Bar() : Foo(), D(1), E(2), F(3) {}
+  Bar(int) : D(1), E(2), F(3), Foo(4) {}      // expected-warning {{field 'F' will be initialized after base 'Foo'}}
+  Bar(unsigned) : F(3), E(2), D(1), Foo(4) {} // expected-warning {{initializer order does not match the declaration order}}
+};
diff --git a/clang/test/Frontend/sycl-aux-triple.cpp b/clang/test/Frontend/sycl-aux-triple.cpp
index ae36b53c28b7188a4ae17e99e8a451d8ab588378..38b6a24fb3ce99b7c924fd6e9f8efae7259c0d17 100644
--- a/clang/test/Frontend/sycl-aux-triple.cpp
+++ b/clang/test/Frontend/sycl-aux-triple.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 %s -triple spir -aux-triple x86_64-unknown-linux-gnu -E -dM | FileCheck %s
-// RUN: %clang_cc1 %s -fsycl -fsycl-is-device -triple spir -aux-triple x86_64-unknown-linux-gnu -E -dM | FileCheck --check-prefix=CHECK-SYCL %s
+// RUN: %clang_cc1 %s -fsycl-is-device -triple spir -aux-triple x86_64-unknown-linux-gnu -E -dM | FileCheck --check-prefix=CHECK-SYCL %s
 
 // CHECK-NOT:#define __x86_64__ 1
 // CHECK-SYCL:#define __x86_64__ 1
diff --git a/clang/test/Frontend/sycl.cpp b/clang/test/Frontend/sycl.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..61a30d561d8301f5e5c669e7da2b57c94bb526a4
--- /dev/null
+++ b/clang/test/Frontend/sycl.cpp
@@ -0,0 +1,13 @@
+// Test that we disallow -cc1 -fsycl, even when specifying device or host mode.
+
+// RUN: not %clang_cc1 -fsycl %s 2>&1 | FileCheck --check-prefix=ERROR %s
+// RUN: not %clang_cc1 -fsycl -fsycl-is-device %s 2>&1 | FileCheck --check-prefix=ERROR %s
+// RUN: not %clang_cc1 -fsycl -fsycl-is-host %s 2>&1 | FileCheck --check-prefix=ERROR %s
+
+// ERROR: error: unknown argument: '-fsycl'
+
+// Test that you cannot specify -fsycl-is-device and -fsycl-is-host at the same time.
+// RUN: not %clang_cc1 -fsycl-is-device -fsycl-is-host %s 2>&1 | FileCheck --check-prefix=ERROR-BOTH %s
+// RUN: not %clang_cc1 -fsycl-is-host -fsycl-is-device %s 2>&1 | FileCheck --check-prefix=ERROR-BOTH %s
+
+// ERROR-BOTH: error: invalid argument '-fsycl-is-device' not allowed with '-fsycl-is-host'
diff --git a/clang/test/Layout/itanium-union-bitfield.cpp b/clang/test/Layout/itanium-union-bitfield.cpp
index 961bf5b6f3b44cc89d42a66b8a68b2f31a850701..febfc46dfee54c155325c7450a981810985bd20d 100644
--- a/clang/test/Layout/itanium-union-bitfield.cpp
+++ b/clang/test/Layout/itanium-union-bitfield.cpp
@@ -1,15 +1,23 @@
 // RUN: %clang_cc1 -emit-llvm-only -triple %itanium_abi_triple -fdump-record-layouts %s 2>/dev/null \
 // RUN:            | FileCheck %s
 
+// On z/OS, a bit-field has single byte alignment.  Add aligned(4) on z/OS so the union has
+// the same size & alignment as expected.
+#ifdef __MVS__
+#define ALIGN4 __attribute__((aligned(4)))
+#else
+#define ALIGN4
+#endif
+
 union A {
-  int f1: 3;
+  int f1 : 3 ALIGN4;
   A();
 };
 
 A::A() {}
 
 union B {
-  char f1: 35;
+  char f1 : 35 ALIGN4;
   B();
 };
 
diff --git a/clang/test/Modules/Inputs/lsv-private-macro/mod.map b/clang/test/Modules/Inputs/lsv-private-macro/mod.map
new file mode 100644
index 0000000000000000000000000000000000000000..62b92fb63e79450e07f1add92496de089ebf768b
--- /dev/null
+++ b/clang/test/Modules/Inputs/lsv-private-macro/mod.map
@@ -0,0 +1,7 @@
+module self {
+  header "self.h"
+}
+
+module other {
+  header "other.h"
+}
diff --git a/clang/test/Modules/Inputs/lsv-private-macro/other.h b/clang/test/Modules/Inputs/lsv-private-macro/other.h
new file mode 100644
index 0000000000000000000000000000000000000000..356eccaec27f94d969f72eadf13ee80a2f8bc530
--- /dev/null
+++ b/clang/test/Modules/Inputs/lsv-private-macro/other.h
@@ -0,0 +1,7 @@
+#define OTHER_PRIVATE
+#__private_macro OTHER_PRIVATE
+
+#define OTHER_PUBLIC
+#__public_macro OTHER_PUBLIC
+
+#define OTHER_DEFAULT
diff --git a/clang/test/Modules/Inputs/lsv-private-macro/self.h b/clang/test/Modules/Inputs/lsv-private-macro/self.h
new file mode 100644
index 0000000000000000000000000000000000000000..5a361308a10d0ffb8f66948cbcc386abf384c3a7
--- /dev/null
+++ b/clang/test/Modules/Inputs/lsv-private-macro/self.h
@@ -0,0 +1,7 @@
+#define SELF_PRIVATE
+#__private_macro SELF_PRIVATE
+
+#define SELF_PUBLIC
+#__public_macro SELF_PUBLIC
+
+#define SELF_DEFAULT
diff --git a/clang/test/Modules/lsv-private-macro.cpp b/clang/test/Modules/lsv-private-macro.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6564558453e3ab2e8734a75628505ff85a3a4695
--- /dev/null
+++ b/clang/test/Modules/lsv-private-macro.cpp
@@ -0,0 +1,53 @@
+// RUN: rm -rf %t
+//
+// RUN: %clang_cc1 %s \
+// RUN:   -fmodules-local-submodule-visibility \
+// RUN:   -fmodule-map-file=%S/Inputs/lsv-private-macro/mod.map \
+// RUN:   -I%S/Inputs/lsv-private-macro -fmodule-name=self \
+// RUN:   -verify=expected-lsv
+//
+// RUN: %clang_cc1 %s \
+// RUN:   -fmodules -fmodules-cache-path=%t \
+// RUN:   -fmodule-map-file=%S/Inputs/lsv-private-macro/mod.map \
+// RUN:   -I%S/Inputs/lsv-private-macro -fmodule-name=self \
+// RUN:   -verify=expected-nolsv
+//
+// RUN: %clang_cc1 %s \
+// RUN:   -fmodules -fmodules-cache-path=%t \
+// RUN:   -fmodules-local-submodule-visibility \
+// RUN:   -fmodule-map-file=%S/Inputs/lsv-private-macro/mod.map \
+// RUN:   -I%S/Inputs/lsv-private-macro -fmodule-name=self \
+// RUN:   -verify=expected-lsv
+
+#include "self.h"
+
+// With local submodule visibility enabled, private macros don't leak out of
+// their respective submodules, even within the same top-level module.
+// expected-lsv-no-diagnostics
+
+// expected-nolsv-error@+2 {{SELF_PRIVATE defined}}
+#ifdef SELF_PRIVATE
+#error SELF_PRIVATE defined
+#endif
+
+#ifndef SELF_PUBLIC
+#error SELF_PUBLIC not defined
+#endif
+
+#ifndef SELF_DEFAULT
+#error SELF_DEFAULT not defined
+#endif
+
+#include "other.h"
+
+#ifdef OTHER_PRIVATE
+#error OTHER_PRIVATE defined
+#endif
+
+#ifndef OTHER_PUBLIC
+#error OTHER_PUBLIC not defined
+#endif
+
+#ifndef OTHER_DEFAULT
+#error OTHER_DEFAULT not defined
+#endif
diff --git a/clang/test/OpenMP/critical_codegen.cpp b/clang/test/OpenMP/critical_codegen.cpp
index 46fad63b3bd88db2c1773bfd83db5829d8dec1cd..d84f2b2af22bbe85e5bbf2b014b2f36f650bfa30 100644
--- a/clang/test/OpenMP/critical_codegen.cpp
+++ b/clang/test/OpenMP/critical_codegen.cpp
@@ -68,6 +68,31 @@ int main() {
   return a;
 }
 
+// ALL-LABEL:        lambda_critical
+// TERM_DEBUG-LABEL: lambda_critical
+void lambda_critical(int a, int b) {
+  auto l = [=]() {
+#pragma omp critical
+    {
+      // ALL: call void @__kmpc_critical(
+      int c = a + b;
+    }
+  };
+
+  l();
+
+  auto l1 = [=]() {
+#pragma omp parallel
+#pragma omp critical
+    {
+      // ALL: call void @__kmpc_critical(
+      int c = a + b;
+    }
+  };
+
+  l1();
+}
+
 struct S {
   int a;
 };
diff --git a/clang/test/OpenMP/declare_target_ast_print.cpp b/clang/test/OpenMP/declare_target_ast_print.cpp
index c086f85261476174f8a3be3de88ea370e26eaaef..dbd0b923a7d6c4fa61ea5845770dfb842b9af7df 100644
--- a/clang/test/OpenMP/declare_target_ast_print.cpp
+++ b/clang/test/OpenMP/declare_target_ast_print.cpp
@@ -277,4 +277,8 @@ int main (int argc, char **argv) {
 // CHECK-NEXT: int ts = 1;
 // CHECK-NEXT: #pragma omp end declare target
 
+// Do not expect anything here since the region is empty.
+#pragma omp declare target
+#pragma omp end declare target
+
 #endif
diff --git a/clang/test/OpenMP/interop_ast_print.cpp b/clang/test/OpenMP/interop_ast_print.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8f8ddc839c7225b96f42a52a827dd91c886861f7
--- /dev/null
+++ b/clang/test/OpenMP/interop_ast_print.cpp
@@ -0,0 +1,279 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 \
+// RUN:   -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 \
+// RUN:   -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 \
+// RUN:   -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 \
+// RUN:   -emit-pch -o %t %s
+
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 \
+// RUN:   -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 \
+// RUN:   -include-pch %t -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+typedef void *omp_interop_t;
+
+//PRINT-LABEL: void foo1(
+//DUMP-LABEL:  FunctionDecl {{.*}} foo1
+void foo1(int *ap, int dev) {
+  omp_interop_t I;
+  omp_interop_t &IRef = I;
+
+  //PRINT: #pragma omp interop init(target : I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  #pragma omp interop init(target:I)
+
+  //PRINT: #pragma omp interop use(I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  #pragma omp interop use(I)
+
+  //PRINT: #pragma omp interop destroy(I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPDestroyClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  #pragma omp interop destroy(I)
+
+  //PRINT: #pragma omp interop init(target : IRef)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'IRef'
+  #pragma omp interop init(target:IRef)
+
+  //PRINT: #pragma omp interop use(IRef)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'IRef'
+  #pragma omp interop use(IRef)
+
+  //PRINT: #pragma omp interop destroy(IRef)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPDestroyClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'IRef'
+  #pragma omp interop destroy(IRef)
+
+  const omp_interop_t CI = (omp_interop_t)0;
+  //PRINT: #pragma omp interop use(CI)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}'const omp_interop_t'{{.*}}Var{{.*}}'CI'
+  #pragma omp interop use(CI)
+
+  //PRINT: #pragma omp interop device(dev) depend(inout : ap) init(targetsync : I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPDeviceClause
+  //DUMP: DeclRefExpr{{.*}}'dev' 'int'
+  //DUMP: OMPDependClause
+  //DUMP: DeclRefExpr{{.*}}'ap' 'int *'
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  #pragma omp interop device(dev) depend(inout:ap) init(targetsync:I)
+
+  //PRINT: #pragma omp interop device(dev) depend(inout : ap) use(I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPDeviceClause
+  //DUMP: DeclRefExpr{{.*}}'dev' 'int'
+  //DUMP: OMPDependClause
+  //DUMP: DeclRefExpr{{.*}}'ap' 'int *'
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  #pragma omp interop device(dev) depend(inout:ap) use(I)
+
+  //PRINT: #pragma omp interop device(dev) depend(inout : ap) destroy(I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPDeviceClause
+  //DUMP: DeclRefExpr{{.*}}'dev' 'int'
+  //DUMP: OMPDependClause
+  //DUMP: DeclRefExpr{{.*}}'ap' 'int *'
+  //DUMP: OMPDestroyClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  #pragma omp interop device(dev) depend(inout:ap) destroy(I)
+
+  //PRINT: #pragma omp interop init(prefer_type(1,2,3,4,5,6), targetsync : I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: IntegerLiteral{{.*}}1
+  //DUMP: IntegerLiteral{{.*}}2
+  //DUMP: IntegerLiteral{{.*}}3
+  //DUMP: IntegerLiteral{{.*}}4
+  //DUMP: IntegerLiteral{{.*}}5
+  //DUMP: IntegerLiteral{{.*}}6
+  #pragma omp interop init(prefer_type(1,2,3,4,5,6),targetsync:I)
+
+  //PRINT: #pragma omp interop init(prefer_type(2,4,6,1), targetsync : I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: IntegerLiteral{{.*}}2
+  //DUMP: IntegerLiteral{{.*}}4
+  //DUMP: IntegerLiteral{{.*}}6
+  //DUMP: IntegerLiteral{{.*}}1
+  #pragma omp interop init(prefer_type(2,4,6,1),targetsync:I)
+
+  //PRINT: #pragma omp interop init(prefer_type("cuda","cuda_driver","opencl","sycl","hip","level_zero"), targetsync : I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: StringLiteral{{.*}}"cuda"
+  //DUMP: StringLiteral{{.*}}"cuda_driver"
+  //DUMP: StringLiteral{{.*}}"opencl"
+  //DUMP: StringLiteral{{.*}}"sycl"
+  //DUMP: StringLiteral{{.*}}"hip"
+  //DUMP: StringLiteral{{.*}}"level_zero"
+  #pragma omp interop init( \
+    prefer_type("cuda","cuda_driver","opencl","sycl","hip","level_zero"), \
+    targetsync:I)
+
+  //PRINT: #pragma omp interop init(prefer_type("level_zero",2,4), targetsync : I)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: StringLiteral{{.*}}"level_zero"
+  //DUMP: IntegerLiteral{{.*}}2
+  //DUMP: IntegerLiteral{{.*}}4
+  #pragma omp interop init(prefer_type("level_zero",2,4),targetsync:I)
+
+  omp_interop_t J;
+
+  //PRINT: #pragma omp interop init(target : I) init(targetsync : J)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'J'
+  #pragma omp interop init(target:I) init(targetsync:J)
+
+  //PRINT: #pragma omp interop init(target : I) use(J)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'J'
+  #pragma omp interop init(target:I) use(J)
+
+  //PRINT: #pragma omp interop use(I) use(J)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'J'
+  #pragma omp interop use(I) use(J)
+
+  //PRINT: #pragma omp interop destroy(I) destroy(J)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPDestroyClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: OMPDestroyClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'J'
+  #pragma omp interop destroy(I) destroy(J)
+
+  //PRINT: #pragma omp interop init(target : I) destroy(J)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: OMPDestroyClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'J'
+  #pragma omp interop init(target:I) destroy(J)
+
+  //PRINT: #pragma omp interop destroy(I) use(J)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPDestroyClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'I'
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}Var{{.*}}'J'
+  #pragma omp interop destroy(I) use(J)
+}
+
+//DUMP: FunctionTemplateDecl{{.*}}fooTemp
+//DUMP-NEXT: NonTypeTemplateParmDecl{{.*}}'int{{.*}}I
+template <int I>
+void fooTemp() {
+  omp_interop_t interop_var;
+  //PRINT: #pragma omp interop init(prefer_type(I,4,"level_one"), target : interop_var)
+  //DUMP: FunctionDecl{{.*}}fooTemp
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}'interop_var'
+  //DUMP: DeclRefExpr{{.*}}NonTypeTemplateParm{{.*}}'I' 'int'
+  //DUMP: IntegerLiteral{{.*}}'int' 4
+  //DUMP: StringLiteral{{.*}}"level_one"
+
+  //PRINT: #pragma omp interop init(prefer_type(3,4,"level_one"), target : interop_var)
+  //DUMP: FunctionDecl{{.*}}fooTemp
+  //DUMP: TemplateArgument integral 3
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}'omp_interop_t'{{.*}}'interop_var'
+  //DUMP: SubstNonTypeTemplateParmExpr{{.*}}'int'
+  //DUMP: NonTypeTemplateParmDecl{{.*}}'int'{{.*}}I
+  //DUMP: IntegerLiteral{{.*}}'int' 3
+  //DUMP: IntegerLiteral{{.*}}'int' 4
+  //DUMP: StringLiteral{{.*}}"level_one"
+  #pragma omp interop init(prefer_type(I,4,"level_one"), target: interop_var)
+}
+
+//DUMP: FunctionTemplateDecl{{.*}}barTemp
+//DUMP-NEXT: TemplateTypeParmDecl{{.*}}typename{{.*}}T
+template <typename T>
+void barTemp(T t) {
+  //PRINT: #pragma omp interop init(prefer_type(4,"level_one"), target : t)
+  //DUMP: FunctionDecl{{.*}}barTemp 'void (T)'
+  //DUMP: ParmVarDecl{{.*}}t 'T'
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'t' 'T'
+  //DUMP: IntegerLiteral{{.*}}'int' 4
+  //DUMP: StringLiteral{{.*}}"level_one"
+  #pragma omp interop init(prefer_type(4,"level_one"), target: t)
+
+  //PRINT: #pragma omp interop use(t)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'t' 'T'
+  #pragma omp interop use(t)
+
+  //PRINT: #pragma omp interop destroy(t)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPDestroyClause
+  //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'t' 'T'
+  #pragma omp interop destroy(t)
+
+  //DUMP: FunctionDecl{{.*}}barTemp 'void (void *)'
+  //DUMP: TemplateArgument type 'void *'
+  //DUMP: ParmVarDecl{{.*}}t 'void *'
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPInitClause
+  //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'t' 'void *'
+  //PRINT: #pragma omp interop init(prefer_type(4,"level_one"), target : t)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPUseClause
+  //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'t' 'void *'
+  //PRINT: #pragma omp interop use(t)
+  //DUMP: OMPInteropDirective
+  //DUMP: OMPDestroyClause
+  //DUMP: DeclRefExpr{{.*}}ParmVar{{.*}}'t' 'void *'
+  //PRINT: #pragma omp interop destroy(t)
+}
+
+void bar()
+{
+  fooTemp<3>();
+  omp_interop_t Ivar;
+  barTemp(Ivar);
+}
+
+#endif // HEADER
diff --git a/clang/test/OpenMP/interop_messages.cpp b/clang/test/OpenMP/interop_messages.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..50f1efb5a6a9b385fb33db2f1c601cae35c28e63
--- /dev/null
+++ b/clang/test/OpenMP/interop_messages.cpp
@@ -0,0 +1,129 @@
+// RUN: %clang_cc1 -verify -fopenmp -std=c++11 -o - -DWITHDEF %s
+// RUN: %clang_cc1 -verify -fopenmp -std=c++11 -o - -DWITHOUTDEF %s
+
+#ifdef WITHDEF
+typedef void *omp_interop_t;
+
+void foo(int *Ap) {
+  omp_interop_t InteropVar;
+  omp_interop_t Another;
+
+  //expected-error@+1 {{expected interop type: 'target' and/or 'targetsync'}}
+  #pragma omp interop init(target,foo:InteropVar) init(target:Another)
+
+  //expected-error@+1 {{use of undeclared identifier 'NoDeclVar'}}
+  #pragma omp interop init(target:NoDeclVar) init(target:Another)
+
+  //expected-error@+1 {{use of undeclared identifier 'NoDeclVar'}}
+  #pragma omp interop use(NoDeclVar) use(Another)
+
+  //expected-error@+1 {{use of undeclared identifier 'NoDeclVar'}}
+  #pragma omp interop destroy(NoDeclVar) destroy(Another)
+
+  //expected-error@+2 {{expected interop type: 'target' and/or 'targetsync'}}
+  //expected-error@+1 {{expected expression}}
+  #pragma omp interop init(InteropVar) init(target:Another)
+
+  //expected-warning@+1 {{missing ':' after interop types}}
+  #pragma omp interop init(target InteropVar)
+
+  //expected-error@+1 {{expected expression}}
+  #pragma omp interop init(prefer_type(1,+,3),target:InteropVar) \
+                      init(target:Another)
+
+  int IntVar;
+  struct S { int I; } SVar;
+
+  //expected-error@+1 {{interop variable must be of type 'omp_interop_t'}}
+  #pragma omp interop init(prefer_type(1,"sycl",3),target:IntVar) \
+                      init(target:Another)
+
+  //expected-error@+1 {{interop variable must be of type 'omp_interop_t'}}
+  #pragma omp interop use(IntVar) use(Another)
+
+  //expected-error@+1 {{interop variable must be of type 'omp_interop_t'}}
+  #pragma omp interop destroy(IntVar) destroy(Another)
+
+  //expected-error@+1 {{interop variable must be of type 'omp_interop_t'}}
+  #pragma omp interop init(prefer_type(1,"sycl",3),target:SVar) \
+                      init(target:Another)
+
+  //expected-error@+1 {{interop variable must be of type 'omp_interop_t'}}
+  #pragma omp interop use(SVar) use(Another)
+
+  //expected-error@+1 {{interop variable must be of type 'omp_interop_t'}}
+  #pragma omp interop destroy(SVar) destroy(Another)
+
+  int a, b;
+  //expected-error@+1 {{expected variable of type 'omp_interop_t'}}
+  #pragma omp interop init(target:a+b) init(target:Another)
+
+  //expected-error@+1 {{expected variable of type 'omp_interop_t'}}
+  #pragma omp interop use(a+b) use(Another)
+
+  //expected-error@+1 {{expected variable of type 'omp_interop_t'}}
+  #pragma omp interop destroy(a+b) destroy(Another)
+
+  const omp_interop_t C = (omp_interop_t)5;
+  //expected-error@+1 {{expected non-const variable of type 'omp_interop_t'}}
+  #pragma omp interop init(target:C) init(target:Another)
+
+  //expected-error@+1 {{expected non-const variable of type 'omp_interop_t'}}
+  #pragma omp interop destroy(C) destroy(Another)
+
+  //expected-error@+1 {{prefer_list item must be a string literal or constant integral expression}}
+  #pragma omp interop init(prefer_type(1.0),target:InteropVar) \
+                      init(target:Another)
+
+  //expected-error@+1 {{prefer_list item must be a string literal or constant integral expression}}
+  #pragma omp interop init(prefer_type(a),target:InteropVar) \
+                      init(target:Another)
+
+  //expected-error@+1 {{expected at least one 'init', 'use', 'destroy', or 'nowait' clause for '#pragma omp interop'}}
+  #pragma omp interop device(0)
+
+  //expected-warning@+1 {{interop type 'target' cannot be specified more than once}}
+  #pragma omp interop init(target,targetsync,target:InteropVar)
+
+  //expected-error@+1 {{'depend' clause requires the 'targetsync' interop type}}
+  #pragma omp interop init(target:InteropVar) depend(inout:Ap)
+
+  //expected-error@+1 {{interop variable 'InteropVar' used in multiple action clauses}}
+  #pragma omp interop init(target:InteropVar) init(target:InteropVar)
+
+  //expected-error@+1 {{interop variable 'InteropVar' used in multiple action clauses}}
+  #pragma omp interop use(InteropVar) use(InteropVar)
+
+  //expected-error@+1 {{interop variable 'InteropVar' used in multiple action clauses}}
+  #pragma omp interop destroy(InteropVar) destroy(InteropVar)
+
+  //expected-error@+1 {{interop variable 'InteropVar' used in multiple action clauses}}
+  #pragma omp interop init(target:InteropVar) use(InteropVar)
+
+  //expected-error@+1 {{interop variable 'InteropVar' used in multiple action clauses}}
+  #pragma omp interop init(target:InteropVar) destroy(InteropVar)
+
+  //expected-error@+1 {{interop variable 'InteropVar' used in multiple action clauses}}
+  #pragma omp interop use(InteropVar) destroy(InteropVar)
+
+  //expected-error@+1 {{directive '#pragma omp interop' cannot contain more than one 'device' clause}}
+  #pragma omp interop init(target:InteropVar) device(0) device(1)
+
+  //expected-error@+1 {{argument to 'device' clause must be a non-negative integer value}}
+  #pragma omp interop init(target:InteropVar) device(-4)
+
+  //expected-error@+1 {{directive '#pragma omp interop' cannot contain more than one 'nowait' clause}}
+  #pragma omp interop nowait init(target:InteropVar) nowait
+}
+#endif
+#ifdef WITHOUTDEF
+void foo() {
+  int InteropVar;
+  //expected-error@+1 {{'omp_interop_t' type not found; include <omp.h>}}
+  #pragma omp interop init(prefer_type(1,"sycl",3),target:InteropVar) nowait
+  //expected-error@+1 {{'omp_interop_t' type not found; include <omp.h>}}
+  #pragma omp interop use(InteropVar) nowait
+  //expected-error@+1 {{'omp_interop_t' type not found; include <omp.h>}}
+  #pragma omp interop destroy(InteropVar) nowait
+}
+#endif
diff --git a/clang/test/OpenMP/linking.c b/clang/test/OpenMP/linking.c
index 802553c1be7525de7aeb8c5253a125a7f9e85f1d..1c44396264705bb6638ed289cd83849b07f6bb01 100644
--- a/clang/test/OpenMP/linking.c
+++ b/clang/test/OpenMP/linking.c
@@ -81,7 +81,7 @@
 // CHECK-LD-OVERRIDE-64: "-lgomp" "-lrt"
 // CHECK-LD-OVERRIDE-64: "-lpthread" "-lc"
 //
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
+// RUN: %clang -no-canonical-prefixes -fuse-ld=link %s -### -o %t.o 2>&1 \
 // RUN:     -fopenmp=libomp -target x86_64-msvc-win32 -rtlib=platform \
 // RUN:   | FileCheck --check-prefix=CHECK-MSVC-LINK-64 %s
 // CHECK-MSVC-LINK-64: link.exe
@@ -95,7 +95,7 @@
 // SIMD-ONLY11-NOT: libomp
 // SIMD-ONLY11-NOT: libgomp
 //
-// RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
+// RUN: %clang -no-canonical-prefixes %s -fuse-ld=link -### -o %t.o 2>&1 \
 // RUN:     -fopenmp=libiomp5 -target x86_64-msvc-win32 -rtlib=platform \
 // RUN:   | FileCheck --check-prefix=CHECK-MSVC-ILINK-64 %s
 
diff --git a/clang/test/OpenMP/master_codegen.cpp b/clang/test/OpenMP/master_codegen.cpp
index 8554ad8e7deca941ece74a83da2bdbbce2c7b392..353284ea85411bc15dbf48bb809e0bfc86427fff 100644
--- a/clang/test/OpenMP/master_codegen.cpp
+++ b/clang/test/OpenMP/master_codegen.cpp
@@ -55,6 +55,41 @@ int main() {
   return a;
 }
 
+// ALL-LABEL:        lambda_master
+// TERM_DEBUG-LABEL: lambda_master
+void lambda_master(int a, int b) {
+  auto l = [=]() {
+#pragma omp master
+    {
+      // ALL: call i32 @__kmpc_master(
+      int c = a + b;
+    }
+  };
+
+  l();
+
+  auto l1 = [=]() {
+#pragma omp parallel
+#pragma omp master
+    {
+      // ALL: call i32 @__kmpc_master(
+      int c = a + b;
+    }
+  };
+
+  l1();
+
+  auto l2 = [=]() {
+#pragma omp parallel master
+    {
+      // ALL: call i32 @__kmpc_master(
+      int c = a + b;
+    }
+  };
+
+  l2();
+}
+
 // ALL-LABEL:      parallel_master
 // TERM_DEBUG-LABEL: parallel_master
 void parallel_master() {
diff --git a/clang/test/OpenMP/omp_with_loop_pragma_instr_profile.c b/clang/test/OpenMP/omp_with_loop_pragma_instr_profile.c
new file mode 100644
index 0000000000000000000000000000000000000000..9667f9cc549d3a9c7a02bfc7e1c3e589c6f0aab7
--- /dev/null
+++ b/clang/test/OpenMP/omp_with_loop_pragma_instr_profile.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -verify -fopenmp -x c -emit-llvm %s -triple x86_64-unknown-linux -o - -femit-all-decls -disable-llvm-passes -fprofile-instrument=clang | FileCheck %s
+// expected-no-diagnostics
+
+void sub(double *restrict a, double *restrict b, int n) {
+  int i;
+
+#pragma omp parallel for
+#pragma clang loop vectorize(disable)
+  for (i = 0; i < n; i++) {
+    a[i] = a[i] + b[i];
+  }
+}
+
+// CHECK-LABEL: @.omp_outlined.(
+// CHECK-NEXT:  entry:
+// CHECK:         call void @llvm.instrprof.increment(
+// CHECK:       omp.precond.then:
+// CHECK-NEXT:    call void @llvm.instrprof.increment(
+// CHECK:       cond.true:
+// CEHCK-NEXT:    call void @llvm.instrprof.increment(
+// CHECK:       omp.inner.for.body:
+// CHECK-NEXT:    call void @llvm.instrprof.increment(
diff --git a/clang/test/OpenMP/simd_null_pointer_access.cpp b/clang/test/OpenMP/simd_null_pointer_access.cpp
index 11a828501d1ffbacc747e3bbbfc5f75c9c4db3f3..3b539fd3aa6419e977f79143efec2c786bed084e 100644
--- a/clang/test/OpenMP/simd_null_pointer_access.cpp
+++ b/clang/test/OpenMP/simd_null_pointer_access.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fopenmp-simd -fsycl -fsycl-is-device -triple spir64 -verify -fsyntax-only %s
+// RUN: %clang_cc1 -fopenmp-simd -fsycl-is-device -triple spir64 -verify -fsyntax-only %s
 
 // Test that in the presence of SYCL options, that null function
 // declarations are accounted for when checking to emit diagnostics.
diff --git a/clang/test/OpenMP/taskgroup_messages.cpp b/clang/test/OpenMP/taskgroup_messages.cpp
index 6bac1614b6a977a488124cc8e62f76dbad01e857..fb774e03b5305eda0f3c7e7fa11c6e29a7e283be 100644
--- a/clang/test/OpenMP/taskgroup_messages.cpp
+++ b/clang/test/OpenMP/taskgroup_messages.cpp
@@ -71,7 +71,7 @@ int foo() {
     foo();
   }
 
-#pragma omp taskgroup init // expected-warning {{extra tokens at the end of '#pragma omp taskgroup' are ignored}}
+#pragma omp taskgroup initi // expected-warning {{extra tokens at the end of '#pragma omp taskgroup' are ignored}}
   ;
   return 0;
 }
diff --git a/clang/test/PCH/cxx-explicit-specifier.cpp b/clang/test/PCH/cxx-explicit-specifier.cpp
index ede0730f7c13996c4e5b4047904c277740ca01ea..331c2e84c78158b41ecee256717ad917903a58cf 100644
--- a/clang/test/PCH/cxx-explicit-specifier.cpp
+++ b/clang/test/PCH/cxx-explicit-specifier.cpp
@@ -1,8 +1,10 @@
+// RUN: %clang_cc1 -std=c++2a -include %s %s -ast-print -verify | FileCheck %s
+//
 // RUN: %clang_cc1 -std=c++2a -emit-pch %s -o %t-cxx2a
-// RUN: %clang_cc1 -std=c++2a -DUSE_PCH -include-pch %t-cxx2a %s -ast-print -verify | FileCheck %s
+// RUN: %clang_cc1 -std=c++2a -include-pch %t-cxx2a %s -ast-print -verify | FileCheck %s
 
 // RUN: %clang_cc1 -std=c++2a -emit-pch -fpch-instantiate-templates %s -o %t-cxx2a
-// RUN: %clang_cc1 -std=c++2a -DUSE_PCH -include-pch %t-cxx2a %s -ast-print -verify | FileCheck %s
+// RUN: %clang_cc1 -std=c++2a -include-pch %t-cxx2a %s -ast-print -verify | FileCheck %s
 
 #ifndef USE_PCH
 namespace inheriting_constructor {
@@ -125,3 +127,5 @@ A a1 = { 0 };
 #endif
 
 }
+
+#define USE_PCH
diff --git a/clang/test/Parser/cxx-concepts-requires-clause.cpp b/clang/test/Parser/cxx-concepts-requires-clause.cpp
index 9cef4c64ee96ac027f5ca06babc7ef6cb987eb40..4d6d166d6fef026fcbe9606d5ce99d3c7c8e80ac 100644
--- a/clang/test/Parser/cxx-concepts-requires-clause.cpp
+++ b/clang/test/Parser/cxx-concepts-requires-clause.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -std=c++2a -x c++ %s -verify
+// RUN: %clang_cc1 -std=c++2b -x c++ %s -verify
 
 // Test parsing of the optional requires-clause in a template-declaration.
 
@@ -154,4 +155,6 @@ auto lambda1 = [] (auto x) requires (sizeof(decltype(x)) == 1) { };
 auto lambda2 = [] (auto x) constexpr -> int requires (sizeof(decltype(x)) == 1) { return 0; };
 
 auto lambda3 = [] requires (sizeof(char) == 1) { };
-// expected-error@-1{{lambda requires '()' before 'requires' clause}}
\ No newline at end of file
+#if __cplusplus <= 202002L
+// expected-warning@-2{{is a C++2b extension}}
+#endif
diff --git a/clang/test/Parser/cxx0x-lambda-expressions.cpp b/clang/test/Parser/cxx0x-lambda-expressions.cpp
index 3148c73a4069fbc236ca9642a8cc6fb0445d2ca6..00a9b70ccd016e8ca9cf705eb39c913fe191ceb5 100644
--- a/clang/test/Parser/cxx0x-lambda-expressions.cpp
+++ b/clang/test/Parser/cxx0x-lambda-expressions.cpp
@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++11 -Wno-c99-designator %s
 // RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++2a -Wno-c99-designator %s
+// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++2b -Wno-c99-designator %s
 
 enum E { e };
 
@@ -17,7 +18,7 @@ class C {
     [&this] {}; // expected-error {{'this' cannot be captured by reference}}
     [&,] {}; // expected-error {{expected variable name or 'this' in lambda capture list}}
     [=,] {}; // expected-error {{expected variable name or 'this' in lambda capture list}}
-    [] {}; 
+    [] {};
     [=] (int i) {}; 
     [&] (int) mutable -> void {}; 
     [foo,bar] () { return 3; }; 
@@ -27,8 +28,12 @@ class C {
     [] () -> class C { return C(); };
     [] () -> enum E { return e; };
 
-    [] -> int { return 0; }; // expected-error{{lambda requires '()' before return type}}
-    [] mutable -> int { return 0; }; // expected-error{{lambda requires '()' before 'mutable'}}
+    [] -> int { return 0; };
+    [] mutable -> int { return 0; };
+#if __cplusplus <= 202002L
+    // expected-warning@-3 {{lambda without a parameter clause is a C++2b extension}}
+    // expected-warning@-3 {{is a C++2b extension}}
+#endif
     [](int) -> {}; // PR13652 expected-error {{expected a type}}
     return 1;
   }
@@ -101,7 +106,10 @@ class C {
   }
 
   void attributes() {
-    [] __attribute__((noreturn)) {}; // expected-error {{lambda requires '()' before attribute specifier}}
+    [] __attribute__((noreturn)){};
+#if __cplusplus <= 202002L
+    // expected-warning@-2 {{is a C++2b extension}}
+#endif
     []() [[]]
       mutable {}; // expected-error {{expected body of lambda expression}}
 
@@ -118,11 +126,29 @@ class C {
 
     // Testing support for P2173 on adding attributes to the declaration
     // rather than the type.
-    [] [[]] () {}; // expected-warning {{an attribute specifier sequence in this position is a C++2b extension}}
+    [][[]](){};
+#if __cplusplus <= 202002L
+    // expected-warning@-2 {{an attribute specifier sequence in this position is a C++2b extension}}
+#endif
 #if __cplusplus > 201703L
-    [] <typename> [[]] () {};  // expected-warning {{an attribute specifier sequence in this position is a C++2b extension}}
+    []<typename>[[]](){};
+#if __cplusplus <= 202002L
+    // expected-warning@-2 {{an attribute specifier sequence in this position is a C++2b extension}}
+#endif
+#endif
+    [][[]]{};
+#if __cplusplus <= 202002L
+    // expected-warning@-2 {{an attribute specifier sequence in this position is a C++2b extension}}
+#endif
+  }
+
+  void missing_parens() {
+    [] mutable {};
+    [] noexcept {};
+#if __cplusplus <= 202002L
+    // expected-warning@-3 {{is a C++2b extension}}
+    // expected-warning@-3 {{is a C++2b extension}}
 #endif
-    [] [[]] {}; // expected-warning {{an attribute specifier sequence in this position is a C++2b extension}}
   }
 };
 
diff --git a/clang/test/Parser/cxx1z-constexpr-lambdas.cpp b/clang/test/Parser/cxx1z-constexpr-lambdas.cpp
index 4cf3d12211676ffa1bbe09853580279afa347494..0c2f81d318a143d16a0b26c49b4444f6b56dd5d3 100644
--- a/clang/test/Parser/cxx1z-constexpr-lambdas.cpp
+++ b/clang/test/Parser/cxx1z-constexpr-lambdas.cpp
@@ -1,12 +1,19 @@
-// RUN: %clang_cc1 -std=c++17 %s -verify 
-// RUN: %clang_cc1 -std=c++14 %s -verify 
-// RUN: %clang_cc1 -std=c++11 %s -verify 
+// RUN: %clang_cc1 -std=c++2b %s -verify
+// RUN: %clang_cc1 -std=c++20 %s -verify
+// RUN: %clang_cc1 -std=c++17 %s -verify
+// RUN: %clang_cc1 -std=c++14 %s -verify
+// RUN: %clang_cc1 -std=c++11 %s -verify
 
-
-auto XL0 = [] constexpr { }; //expected-error{{requires '()'}} expected-error{{expected body}}
-auto XL1 = [] () mutable 
-                 mutable     //expected-error{{cannot appear multiple times}}
-                 mutable { }; //expected-error{{cannot appear multiple times}}
+auto XL0 = [] constexpr { return true; };
+#if __cplusplus <= 201402L
+// expected-warning@-2 {{is a C++17 extension}}
+#endif
+#if __cplusplus <= 202002L
+// expected-warning@-5 {{lambda without a parameter clause is a C++2b extension}}
+#endif
+auto XL1 = []() mutable //
+    mutable             // expected-error{{cannot appear multiple times}}
+    mutable {};         // expected-error{{cannot appear multiple times}}
 
 #if __cplusplus > 201402L
 auto XL2 = [] () constexpr mutable constexpr { }; //expected-error{{cannot appear multiple times}}
diff --git a/clang/test/Parser/cxx2a-init-statement.cpp b/clang/test/Parser/cxx2a-init-statement.cpp
index 3b1862f1d3c5e58eeb2ebebff37656f3521dcd49..727ee63d1b92762c248e2985d3427f0bd18fcee0 100644
--- a/clang/test/Parser/cxx2a-init-statement.cpp
+++ b/clang/test/Parser/cxx2a-init-statement.cpp
@@ -31,4 +31,12 @@ void f() {
 
   for (int n = 0; static int m = 0; ++n) {} // expected-error {{type name does not allow storage class}}
   for (int n = 0; static int m : arr1) {} // expected-error {{loop variable 'm' may not be declared 'static'}}
+
+  // The init-statement and range are not break / continue scopes. (But the body is.)
+  for (int n = ({ break; 0; }); int m : arr1) {} // expected-error {{not in loop}}
+  for (int n = ({ continue; 0; }); int m : arr1) {} // expected-error {{not in loop}}
+  for (int arr[3]; int n : *({ break; &arr; })) {} // expected-error {{not in loop}}
+  for (int arr[3]; int n : *({ continue; &arr; })) {} // expected-error {{not in loop}}
+  for (int n = 0; int m : arr1) { break; }
+  for (int n = 0; int m : arr1) { continue; }
 }
diff --git a/clang/test/Parser/cxx2a-template-lambdas.cpp b/clang/test/Parser/cxx2a-template-lambdas.cpp
index 034a3b157d80ef487b791dd247e5f8262cd2a916..2a2305b5530cd2f4d3682ad47f5179831d4ce36a 100644
--- a/clang/test/Parser/cxx2a-template-lambdas.cpp
+++ b/clang/test/Parser/cxx2a-template-lambdas.cpp
@@ -1,3 +1,4 @@
+// RUN: %clang_cc1 -std=c++2b %s -verify
 // RUN: %clang_cc1 -std=c++2a %s -verify
 
 auto L0 = []<> { }; //expected-error {{cannot be empty}}
diff --git a/clang/test/Parser/cxx2b-lambdas.cpp b/clang/test/Parser/cxx2b-lambdas.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e008dc8f2d368970a06d3b50553d475d6282e009
--- /dev/null
+++ b/clang/test/Parser/cxx2b-lambdas.cpp
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -std=c++2b %s -verify
+
+auto LL0 = [] {};
+auto LL1 = []() {};
+auto LL2 = []() mutable {};
+auto LL3 = []() constexpr {};
+
+auto L0 = [] constexpr {};
+auto L1 = [] mutable {};
+auto L2 = [] noexcept {};
+auto L3 = [] constexpr mutable {};
+auto L4 = [] mutable constexpr {};
+auto L5 = [] constexpr mutable noexcept {};
+auto L6 = [s = 1] mutable {};
+auto L7 = [s = 1] constexpr mutable noexcept {};
+auto L8 = [] -> bool { return true; };
+auto L9 = []<typename T> { return true; };
+auto L10 = []<typename T> noexcept { return true; };
+auto L11 = []<typename T> -> bool { return true; };
+auto L12 = [] consteval {};
+auto L13 = [] requires requires() { true; }
+{};
+auto L15 = [] [[maybe_unused]]{};
+
+auto XL0 = [] mutable constexpr mutable {};    // expected-error{{cannot appear multiple times}}
+auto XL1 = [] constexpr mutable constexpr {};  // expected-error{{cannot appear multiple times}}
+auto XL2 = []) constexpr mutable constexpr {}; // expected-error{{expected body}}
+auto XL3 = []( constexpr mutable constexpr {}; // expected-error{{invalid storage class specifier}} \
+                                               // expected-error{{function parameter cannot be constexpr}} \
+                                               // expected-error{{C++ requires}} \
+                                               // expected-error{{expected ')'}} \
+                                               // expected-note{{to match this '('}} \
+                                               // expected-error{{expected body}} \
+                                               // expected-warning{{duplicate 'constexpr'}}
diff --git a/clang/test/Parser/static_assert.c b/clang/test/Parser/static_assert.c
index eaae2adbe3808f30b0911648e48c75e47cc947c9..452c404a79ce9dc0c2852d7146cbbca658a278f8 100644
--- a/clang/test/Parser/static_assert.c
+++ b/clang/test/Parser/static_assert.c
@@ -7,7 +7,7 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++17 -pedantic -verify=cxx17-pedantic -x c++ %s
 // RUN: %clang_cc1 -fsyntax-only -std=c++98 -verify=cxx98 -x c++ %s
 // RUN: %clang_cc1 -fsyntax-only -std=c++98 -pedantic -verify=cxx98-pedantic -x c++ %s
-// RUN: %clang_cc1 -fsyntax-only -std=c++17 -Wc++98-c++11-c++14-compat -verify=cxx17-compat -x c++ %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++17 -Wpre-c++17-compat -verify=cxx17-compat -x c++ %s
 
 // cxx17-no-diagnostics
 
diff --git a/clang/test/Parser/stmt-attributes.c b/clang/test/Parser/stmt-attributes.c
index d142ce1b5b9540afee53b2bec65591004181074b..86adc56f40ca2b71c4311887fcb4309b2edd9d76 100644
--- a/clang/test/Parser/stmt-attributes.c
+++ b/clang/test/Parser/stmt-attributes.c
@@ -40,7 +40,7 @@ void foo(int i) {
 
   __attribute__((unused)) switch (i) {         // expected-error {{'unused' attribute cannot be applied to a statement}}
   __attribute__((uuid)) case 0:                // expected-warning {{unknown attribute 'uuid' ignored}}
-  __attribute__((visibility)) default:         // expected-error {{'visibility' attribute cannot be applied to a statement}}
+  __attribute__((visibility(""))) default:         // expected-error {{'visibility' attribute cannot be applied to a statement}}
     __attribute__((carries_dependency)) break; // expected-error {{'carries_dependency' attribute cannot be applied to a statement}}
   }
 
diff --git a/clang/test/Preprocessor/init-ppc64.c b/clang/test/Preprocessor/init-ppc64.c
index 08368f323d44807afd376748324249ba59f531d5..6aed0e7daa96b761ea1f7c1cc315a19d04d5fc55 100644
--- a/clang/test/Preprocessor/init-ppc64.c
+++ b/clang/test/Preprocessor/init-ppc64.c
@@ -566,7 +566,8 @@
 // PPCPWR8-NOT:#define _ARCH_PWR6X 1
 // PPCPWR8:#define _ARCH_PWR7 1
 // PPCPWR8:#define _ARCH_PWR8 1
-// PPCPWR8-NOT:#define __ROP_PROTECTION__ 1
+// PPCPWR8-NOT:#define __ROP_PROTECT__ 1
+// PPCPWR8-NOT:#define __PRIVILEGED__ 1
 //
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu power8 -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCPOWER8 %s
 //
@@ -584,7 +585,8 @@
 // PPCPOWER8-NOT:#define _ARCH_PWR6X 1
 // PPCPOWER8:#define _ARCH_PWR7 1
 // PPCPOWER8:#define _ARCH_PWR8 1
-// PPCPOWER8-NOT:#define __ROP_PROTECTION__ 1
+// PPCPOWER8-NOT:#define __ROP_PROTECT__ 1
+// PPCPOWER8-NOT:#define __PRIVILEGED__ 1
 //
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu pwr9 -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCPWR9 %s
 //
@@ -599,7 +601,8 @@
 // PPCPWR9-NOT:#define _ARCH_PWR6X 1
 // PPCPWR9:#define _ARCH_PWR7 1
 // PPCPWR9:#define _ARCH_PWR9 1
-// PPCPWR9-NOT:#define __ROP_PROTECTION__ 1
+// PPCPWR9-NOT:#define __ROP_PROTECT__ 1
+// PPCPWR9-NOT:#define __PRIVILEGED__ 1
 //
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu power9 -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCPOWER9 %s
 //
@@ -614,7 +617,8 @@
 // PPCPOWER9-NOT:#define _ARCH_PWR6X 1
 // PPCPOWER9:#define _ARCH_PWR7 1
 // PPCPOWER9:#define _ARCH_PWR9 1
-// PPCPOWER9-NOT:#define __ROP_PROTECTION__ 1
+// PPCPOWER9-NOT:#define __ROP_PROTECT__ 1
+// PPCPOWER9-NOT:#define __PRIVILEGED__ 1
 //
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu pwr10 -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCPOWER10 %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu power10 -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCPOWER10 %s
@@ -634,7 +638,8 @@
 // PPCPOWER10:#define _ARCH_PWR9 1
 // PPCPOWER10:#define __MMA__ 1
 // PPCPOWER10:#define __PCREL__ 1
-// PPCPOWER10-NOT:#define __ROP_PROTECTION__ 1
+// PPCPOWER10-NOT:#define __ROP_PROTECT__ 1
+// PPCPOWER10-NOT:#define __PRIVILEGED__ 1
 //
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-cpu future -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix PPCFUTURE %s
 //
@@ -654,15 +659,21 @@
 // PPCFUTURE:#define _ARCH_PWR_FUTURE 1
 // PPCFUTURE:#define __MMA__ 1
 // PPCFUTURE:#define __PCREL__ 1
-// PPCFUTURE-NOT:#define __ROP_PROTECTION__ 1
+// PPCFUTURE-NOT:#define __ROP_PROTECT__ 1
+// PPCFUTURE-NOT:#define __PRIVILEGED__ 1
 //
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +mma -target-cpu power10 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-MMA %s
 // PPC-MMA:#define __MMA__ 1
 //
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +rop-protection -target-cpu power10 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-ROP %s
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +rop-protection -target-cpu power9 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-ROP %s
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +rop-protection -target-cpu power8 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-ROP %s
-// PPC-ROP:#define __ROP_PROTECTION__ 1
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +rop-protect -target-cpu power10 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-ROP %s
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +rop-protect -target-cpu power9 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-ROP %s
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +rop-protect -target-cpu power8 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-ROP %s
+// PPC-ROP:#define __ROP_PROTECT__ 1
+//
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +privileged -target-cpu power10 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-PRIV %s
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +privileged -target-cpu power9 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-PRIV %s
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +privileged -target-cpu power8 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-PRIV %s
+// PPC-PRIV:#define __PRIVILEGED__ 1
 //
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=powerpc64-none-none -target-feature +float128 -target-cpu power9 -fno-signed-char < /dev/null | FileCheck -check-prefix PPC-FLOAT128 %s
 // PPC-FLOAT128:#define __FLOAT128__ 1
diff --git a/clang/test/Preprocessor/sycl-macro.cpp b/clang/test/Preprocessor/sycl-macro.cpp
index 408dc200ad88b4eac9977bc85d236826ea3b3fb9..d1d830210529c68edc6ce4a3a960d9330e5cf4fe 100644
--- a/clang/test/Preprocessor/sycl-macro.cpp
+++ b/clang/test/Preprocessor/sycl-macro.cpp
@@ -1,7 +1,8 @@
 // RUN: %clang_cc1 %s -E -dM | FileCheck %s
-// RUN: %clang_cc1 %s -fsycl -sycl-std=2017 -E -dM | FileCheck --check-prefix=CHECK-SYCL-STD %s
-// RUN: %clang_cc1 %s -fsycl -fsycl-is-device -sycl-std=1.2.1 -E -dM | FileCheck --check-prefix=CHECK-SYCL-STD %s
-// RUN: %clang_cc1 %s -fsycl -fsycl-is-device -E -dM | FileCheck --check-prefixes=CHECK-SYCL %s
+// RUN: %clang_cc1 %s -fsycl-is-device -sycl-std=2017 -E -dM | FileCheck --check-prefix=CHECK-SYCL-STD %s
+// RUN: %clang_cc1 %s -fsycl-is-host -sycl-std=2017 -E -dM | FileCheck --check-prefix=CHECK-SYCL-STD %s
+// RUN: %clang_cc1 %s -fsycl-is-device -sycl-std=1.2.1 -E -dM | FileCheck --check-prefix=CHECK-SYCL-STD %s
+// RUN: %clang_cc1 %s -fsycl-is-device -E -dM | FileCheck --check-prefixes=CHECK-SYCL %s
 
 // CHECK-NOT:#define __SYCL_DEVICE_ONLY__ 1
 // CHECK-NOT:#define CL_SYCL_LANGUAGE_VERSION 121
diff --git a/clang/test/Preprocessor/wasm-target-features.c b/clang/test/Preprocessor/wasm-target-features.c
index 05b4bb49d73b4700e6be99798c6d47fb159e0578..29cc3071a235a3487dd3242e10c4a6b365520148 100644
--- a/clang/test/Preprocessor/wasm-target-features.c
+++ b/clang/test/Preprocessor/wasm-target-features.c
@@ -7,15 +7,6 @@
 //
 // SIMD128:#define __wasm_simd128__ 1{{$}}
 
-// RUN: %clang -E -dM %s -o - 2>&1 \
-// RUN:     -target wasm32-unknown-unknown -munimplemented-simd128 \
-// RUN:   | FileCheck %s -check-prefix=SIMD128-UNIMPLEMENTED
-// RUN: %clang -E -dM %s -o - 2>&1 \
-// RUN:     -target wasm64-unknown-unknown -munimplemented-simd128 \
-// RUN:   | FileCheck %s -check-prefix=SIMD128-UNIMPLEMENTED
-//
-// SIMD128-UNIMPLEMENTED:#define __wasm_unimplemented_simd128__ 1{{$}}
-
 // RUN: %clang -E -dM %s -o - 2>&1 \
 // RUN:     -target wasm32-unknown-unknown -mnontrapping-fptoint \
 // RUN:   | FileCheck %s -check-prefix=NONTRAPPING-FPTOINT
@@ -114,7 +105,6 @@
 // RUN:   | FileCheck %s -check-prefix=MVP
 //
 // MVP-NOT:#define __wasm_simd128__
-// MVP-NOT:#define __wasm_unimplemented_simd128__
 // MVP-NOT:#define __wasm_nontrapping_fptoint__
 // MVP-NOT:#define __wasm_sign_ext__
 // MVP-NOT:#define __wasm_exception_handling__
diff --git a/clang/test/Sema/c2x-fallthrough.c b/clang/test/Sema/c2x-fallthrough.c
index 2fd69c4da0f2551f145e55f2cfd87e9b856d936a..e5508e0a10f1dde51beb321c64dc343281ceda13 100644
--- a/clang/test/Sema/c2x-fallthrough.c
+++ b/clang/test/Sema/c2x-fallthrough.c
@@ -57,7 +57,7 @@ struct [[fallthrough]] S { // expected-error {{'fallthrough' attribute cannot be
 [[fallthrough]] // expected-error {{'fallthrough' attribute cannot be applied to a declaration}}
 void g(void) {
   [[fallthrough]] int n; // expected-error {{'fallthrough' attribute cannot be applied to a declaration}}
-  [[fallthrough]] ++n; // expected-error-re {{{{^}}fallthrough attribute is only allowed on empty statements}}
+  [[fallthrough]] ++n; // expected-error {{'fallthrough' attribute only applies to empty statements}}
 
   switch (n) {
     // FIXME: This should be an error.
diff --git a/clang/test/Sema/integer-overflow.c b/clang/test/Sema/integer-overflow.c
index 39395d9bc1fd4766a5c9a86c685efb2dcd94e1e5..79e9294067deef0740897947ba1fe0bd980c0356 100644
--- a/clang/test/Sema/integer-overflow.c
+++ b/clang/test/Sema/integer-overflow.c
@@ -203,3 +203,12 @@ struct s2 {
     }
   }
 };
+
+void PR49619() {
+  int n;
+  n = ({
+    while (1)
+      ;
+    0;
+  });
+}
diff --git a/clang/test/Sema/patchable-function-entry-attr.cpp b/clang/test/Sema/patchable-function-entry-attr.cpp
index 63de5a2abf70ad568ff16feeeed1db9ac60fb0aa..3dd0504987306e9986ab6f7160eb0fd367171db1 100644
--- a/clang/test/Sema/patchable-function-entry-attr.cpp
+++ b/clang/test/Sema/patchable-function-entry-attr.cpp
@@ -2,6 +2,8 @@
 // RUN: %clang_cc1 -triple aarch64_be -fsyntax-only -verify=silence %s
 // RUN: %clang_cc1 -triple i386 -fsyntax-only -verify=silence %s
 // RUN: %clang_cc1 -triple x86_64 -fsyntax-only -verify=silence %s
+// RUN: %clang_cc1 -triple riscv32 -fsyntax-only -verify=silence %s
+// RUN: %clang_cc1 -triple riscv64 -fsyntax-only -verify=silence %s
 // RUN: %clang_cc1 -triple ppc64le -fsyntax-only -verify %s
 
 // silence-no-diagnostics
diff --git a/clang/test/Sema/warn-cast-function-type.c b/clang/test/Sema/warn-cast-function-type.c
new file mode 100644
index 0000000000000000000000000000000000000000..e2572210c1370e8d88a87e2382e0fb81cd8e464d
--- /dev/null
+++ b/clang/test/Sema/warn-cast-function-type.c
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -x c %s -fsyntax-only -Wcast-function-type -triple x86_64-- -verify
+
+int x(long);
+
+typedef int (f1)(long);
+typedef int (f2)(void*);
+typedef int (f3)();
+typedef void (f4)();
+typedef void (f5)(void);
+typedef int (f6)(long, int);
+typedef int (f7)(long,...);
+
+f1 *a;
+f2 *b;
+f3 *c;
+f4 *d;
+f5 *e;
+f6 *f;
+f7 *g;
+
+void foo(void) {
+  a = (f1 *)x;
+  b = (f2 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f2 *' (aka 'int (*)(void *)') converts to incompatible function types}} */
+  c = (f3 *)x;
+  d = (f4 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f4 *' (aka 'void (*)()') converts to incompatible function types}} */
+  e = (f5 *)x;
+  f = (f6 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f6 *' (aka 'int (*)(long, int)') converts to incompatible function types}} */
+  g = (f7 *)x;
+}
diff --git a/clang/test/Sema/warn-cast-function-type.cpp b/clang/test/Sema/warn-cast-function-type.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d70657dad1793ec1ff00779fbdf91e9393a708d0
--- /dev/null
+++ b/clang/test/Sema/warn-cast-function-type.cpp
@@ -0,0 +1,47 @@
+// RUN: %clang_cc1 -x c++ %s -fblocks -fsyntax-only -Wcast-function-type -triple x86_64-- -verify
+
+int x(long);
+
+typedef int (f1)(long);
+typedef int (f2)(void*);
+typedef int (f3)(...);
+typedef void (f4)(...);
+typedef void (f5)(void);
+typedef int (f6)(long, int);
+typedef int (f7)(long,...);
+typedef int (&f8)(long, int);
+
+f1 *a;
+f2 *b;
+f3 *c;
+f4 *d;
+f5 *e;
+f6 *f;
+f7 *g;
+
+struct S
+{
+  void foo (int*);
+  void bar (int);
+};
+
+typedef void (S::*mf)(int);
+
+void foo() {
+  a = (f1 *)x;
+  b = (f2 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f2 *' (aka 'int (*)(void *)') converts to incompatible function types}} */
+  b = reinterpret_cast<f2 *>(x); /* expected-warning {{cast from 'int (*)(long)' to 'f2 *' (aka 'int (*)(void *)') converts to incompatible function types}} */
+  c = (f3 *)x;
+  d = (f4 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f4 *' (aka 'void (*)(...)') converts to incompatible function types}} */
+  e = (f5 *)x;
+  f = (f6 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f6 *' (aka 'int (*)(long, int)') converts to incompatible function types}} */
+  g = (f7 *)x;
+
+  mf p1 = (mf)&S::foo; /* expected-warning {{cast from 'void (S::*)(int *)' to 'mf' (aka 'void (S::*)(int)') converts to incompatible function types}} */
+
+  f8 f2 = (f8)x; /* expected-warning {{cast from 'int (long)' to 'f8' (aka 'int (&)(long, int)') converts to incompatible function types}} */
+  (void)f2;
+
+  int (^y)(long);
+  f = (f6 *)y; /* expected-warning {{cast from 'int (^)(long)' to 'f6 *' (aka 'int (*)(long, int)') converts to incompatible function types}} */
+}
diff --git a/clang/test/SemaCUDA/builtin-mangled-name.cu b/clang/test/SemaCUDA/builtin-mangled-name.cu
new file mode 100644
index 0000000000000000000000000000000000000000..6ca85083d7171d655f96654618af498e9e25a471
--- /dev/null
+++ b/clang/test/SemaCUDA/builtin-mangled-name.cu
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-gnu-linux -aux-triple amdgcn-amd-amdhsa \
+// RUN:   -verify -fsyntax-only -x hip %s
+
+#include "Inputs/cuda.h"
+
+__global__ void kern1();
+int y;
+
+void fun1() {
+  int x;
+  const char *p;
+  p = __builtin_get_device_side_mangled_name();
+  // expected-error@-1 {{invalid argument: symbol must be a device-side function or global variable}}
+  p = __builtin_get_device_side_mangled_name(kern1, kern1);
+  // expected-error@-1 {{invalid argument: symbol must be a device-side function or global variable}}
+  p = __builtin_get_device_side_mangled_name(1);
+  // expected-error@-1 {{invalid argument: symbol must be a device-side function or global variable}}
+  p = __builtin_get_device_side_mangled_name(x);
+  // expected-error@-1 {{invalid argument: symbol must be a device-side function or global variable}}
+  p = __builtin_get_device_side_mangled_name(fun1);
+  // expected-error@-1 {{invalid argument: symbol must be a device-side function or global variable}}
+  p = __builtin_get_device_side_mangled_name(y);
+  // expected-error@-1 {{invalid argument: symbol must be a device-side function or global variable}}
+}
diff --git a/clang/test/SemaCXX/constructor-initializer.cpp b/clang/test/SemaCXX/constructor-initializer.cpp
index df899141671234dd9333f81f7ad40348cd1dd08f..874682944bb857580d8dafd140ca1df452a61afc 100644
--- a/clang/test/SemaCXX/constructor-initializer.cpp
+++ b/clang/test/SemaCXX/constructor-initializer.cpp
@@ -91,13 +91,14 @@ struct Derived : Base, Base1, virtual V {
 
 struct Current : Derived {
   int Derived;
-  Current() : Derived(1), ::Derived(), // expected-warning {{field 'Derived' will be initialized after base '::Derived'}} \
-                                       // expected-warning {{base class '::Derived' will be initialized after base 'Derived::V'}}
-                          ::Derived::Base(), // expected-error {{type '::Derived::Base' is not a direct or virtual base of 'Current'}}
-                           Derived::Base1(), // expected-error {{type 'Derived::Base1' is not a direct or virtual base of 'Current'}}
-                           Derived::V(),
-                           ::NonExisting(), // expected-error {{member initializer 'NonExisting' does not name a non-static data member or}}
-                           INT::NonExisting()  {} // expected-error {{'INT' (aka 'int') is not a class, namespace, or enumeration}} \
+  Current() : Derived(1), ::Derived(), // expected-warning {{initializer order does not match the declaration order}} \
+                                       // expected-note {{field 'Derived' will be initialized after base '::Derived'}} \
+                                       // expected-note {{base class '::Derived' will be initialized after base 'Derived::V'}}
+              ::Derived::Base(),       // expected-error {{type '::Derived::Base' is not a direct or virtual base of 'Current'}}
+              Derived::Base1(),        // expected-error {{type 'Derived::Base1' is not a direct or virtual base of 'Current'}}
+              Derived::V(),
+              ::NonExisting(),      // expected-error {{member initializer 'NonExisting' does not name a non-static data member or}}
+              INT::NonExisting() {} // expected-error {{'INT' (aka 'int') is not a class, namespace, or enumeration}} \
                                                   // expected-error {{member initializer 'NonExisting' does not name a non-static data member or}}
 };
 
diff --git a/clang/test/SemaCXX/cxx0x-initializer-aggregates.cpp b/clang/test/SemaCXX/cxx0x-initializer-aggregates.cpp
index 7a7b92b7d04f12dacf80a3e0c40db3325557c272..94fee530aea6a8c154a7ae6095f741567e93c21a 100644
--- a/clang/test/SemaCXX/cxx0x-initializer-aggregates.cpp
+++ b/clang/test/SemaCXX/cxx0x-initializer-aggregates.cpp
@@ -133,3 +133,10 @@ namespace array_addressof {
 namespace PR24816 {
   struct { int i; } ne = {{0, 1}}; // expected-error{{excess elements in scalar initializer}}
 }
+
+namespace no_crash {
+class Foo; // expected-note {{forward declaration}}
+void test(int size) {
+  Foo array[size] = {0}; // expected-error {{variable has incomplete type}}
+}
+}
diff --git a/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp b/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
index 161944f9e64f6a66bcf3444860d84a620547a120..62b1c166e954cc6bc3b5d831bc5b99a9b2b27bd6 100644
--- a/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
+++ b/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
@@ -543,6 +543,18 @@ namespace PR47175 {
   int m = n<int>;
 }
 
+// Ensure we don't crash when CTAD fails.
+template <typename T1, typename T2>
+struct Foo {   // expected-note{{candidate function template not viable}}
+  Foo(T1, T2); // expected-note{{candidate function template not viable}}
+};
+
+template <typename... Args>
+void insert(Args &&...args);
+
+void foo() {
+  insert(Foo(2, 2, 2)); // expected-error{{no viable constructor or deduction guide}}
+}
 #else
 
 // expected-no-diagnostics
diff --git a/clang/test/SemaCXX/cxx98-compat-pedantic.cpp b/clang/test/SemaCXX/cxx98-compat-pedantic.cpp
index c72c44ad5feb68d967969fb36c7933465ab976b7..74aa890b8c62d45ba35cc1f3c5a8cc5b43e2ce4f 100644
--- a/clang/test/SemaCXX/cxx98-compat-pedantic.cpp
+++ b/clang/test/SemaCXX/cxx98-compat-pedantic.cpp
@@ -4,7 +4,7 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++11 -Wc++98-compat -Werror %s
 // RUN: %clang_cc1 -fsyntax-only -std=c++98 -Werror %s -DCXX98
 
-// RUN: %clang_cc1 -fsyntax-only -std=c++1y -Wc++98-compat-pedantic -verify %s -Wno-c++98-c++11-compat-pedantic -DCXX1Y2
+// RUN: %clang_cc1 -fsyntax-only -std=c++1y -Wc++98-compat-pedantic -verify %s -Wno-pre-c++14-compat-pedantic -DCXX1Y2
 
 // -Wc++98-compat-pedantic warns on C++11 features which we accept without a
 // warning in C++98 mode.
diff --git a/clang/test/SemaCXX/inline.cpp b/clang/test/SemaCXX/inline.cpp
index ba29521ce504aaa31662820984295fc3bd1fa5db..1a13609eb5783275268a870aaa678a5d016d8f8f 100644
--- a/clang/test/SemaCXX/inline.cpp
+++ b/clang/test/SemaCXX/inline.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++14 %s
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++17 %s -Wc++98-c++11-c++14-compat
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++17 %s -Wpre-c++17-compat
 
 // Check that we don't allow illegal uses of inline
 // (checking C++-only constructs here)
diff --git a/clang/test/SemaCXX/scope-check.cpp b/clang/test/SemaCXX/scope-check.cpp
index 518a1e9f2606c782a6f0adbc9fe4179307262af0..7eec58ca057a9ec06968b535215b7fcab14809ab 100644
--- a/clang/test/SemaCXX/scope-check.cpp
+++ b/clang/test/SemaCXX/scope-check.cpp
@@ -629,3 +629,19 @@ l: // expected-note 4 {{possible target of indirect goto statement}}
 }
 
 } // namespace seh
+
+void continue_scope_check() {
+  // These are OK.
+  for (; ({break; true;});) {}
+  for (; ({continue; true;});) {}
+  for (; int n = ({break; 0;});) {}
+  for (; int n = 0; ({break;})) {}
+  for (; int n = 0; ({continue;})) {}
+
+  // This would jump past the initialization of 'n' to the increment (where 'n'
+  // is in scope).
+  for (; int n = ({continue; 0;});) {} // expected-error {{cannot jump from this continue statement to the loop increment; jump bypasses initialization of loop condition variable}}
+
+  // An intervening loop makes it OK again.
+  for (; int n = ({while (true) continue; 0;});) {}
+}
diff --git a/clang/test/SemaCXX/switch-implicit-fallthrough.cpp b/clang/test/SemaCXX/switch-implicit-fallthrough.cpp
index a67f6bef1f49a02490ec3b611b003e7f7f5c563e..e6ae0d55b5884ee3528f17f0f8a0dc8ee9753873 100644
--- a/clang/test/SemaCXX/switch-implicit-fallthrough.cpp
+++ b/clang/test/SemaCXX/switch-implicit-fallthrough.cpp
@@ -299,16 +299,16 @@ int fallthrough_placement_error(int n) {
 int fallthrough_targets(int n) {
   [[clang::fallthrough]]; // expected-error{{fallthrough annotation is outside switch statement}}
 
-  [[clang::fallthrough]]  // expected-error{{fallthrough attribute is only allowed on empty statements}}
+  [[clang::fallthrough]]  // expected-error{{'fallthrough' attribute only applies to empty statements}}
   switch (n) {
     case 121:
       n += 400;
       [[clang::fallthrough]]; // no warning here, correct target
     case 123:
-      [[clang::fallthrough]]  // expected-error{{fallthrough attribute is only allowed on empty statements}}
+      [[clang::fallthrough]]  // expected-error{{'fallthrough' attribute only applies to empty statements}}
       n += 800;
       break;
-    [[clang::fallthrough]]    // expected-error{{fallthrough attribute is only allowed on empty statements}} expected-note{{did you forget ';'?}}
+    [[clang::fallthrough]]    // expected-error{{'fallthrough' attribute is only allowed on empty statements}} expected-note{{did you forget ';'?}}
     case 125:
       n += 1600;
   }
diff --git a/clang/test/SemaCXX/warn-reorder-ctor-initialization.cpp b/clang/test/SemaCXX/warn-reorder-ctor-initialization.cpp
index 6d38ec95fbfb744b8309b3d2b490fb560a4fc945..47588660d2049cd6179d250c25ef8c5856d2032e 100644
--- a/clang/test/SemaCXX/warn-reorder-ctor-initialization.cpp
+++ b/clang/test/SemaCXX/warn-reorder-ctor-initialization.cpp
@@ -4,15 +4,14 @@ struct BB {};
 
 struct BB1 {};
 
-class complex : public BB, BB1 { 
-public: 
+class complex : public BB, BB1 {
+public:
   complex()
-    : s2(1), // expected-warning {{field 's2' will be initialized after field 's1'}}
-      s1(1),
-      s3(3), // expected-warning {{field 's3' will be initialized after base 'BB1'}} 
-      BB1(), // expected-warning {{base class 'BB1' will be initialized after base 'BB'}}
-      BB()
-  {}
+      : s2(1), // expected-warning {{initializer order does not match the declaration order}} expected-note {{field 's2' will be initialized after field 's1'}}
+        s1(1),
+        s3(3), // expected-note {{field 's3' will be initialized after base 'BB1'}}
+        BB1(), // expected-note {{base class 'BB1' will be initialized after base 'BB'}}
+        BB() {}
   int s1;
   int s2;
   int s3;
diff --git a/clang/test/SemaCXX/warn-thread-safety-parsing.cpp b/clang/test/SemaCXX/warn-thread-safety-parsing.cpp
index 6ad0f877a11d3183bc39bf58763725fb1eca7605..b6e9c052a241ac79f8cf963b5dbaf13677213e8f 100644
--- a/clang/test/SemaCXX/warn-thread-safety-parsing.cpp
+++ b/clang/test/SemaCXX/warn-thread-safety-parsing.cpp
@@ -1295,6 +1295,11 @@ struct SLDerived2 : public SLTemplateClass<int> {
     // expected-warning{{'unlock_function' attribute without capability arguments refers to 'this', but 'SLDerived2' isn't annotated with 'capability' or 'scoped_lockable' attribute}}
 };
 
+struct SLDerived3 : public SLTemplateDerived<int> {
+  ~SLDerived3() UNLOCK_FUNCTION(); // \
+    // expected-warning{{'unlock_function' attribute without capability arguments refers to 'this', but 'SLDerived3' isn't annotated with 'capability' or 'scoped_lockable' attribute}}
+};
+
 //-----------------------------------------------------
 // Parsing of member variables and function parameters
 //------------------------------------------------------
diff --git a/clang/test/SemaObjC/warn-called-once.m b/clang/test/SemaObjC/warn-called-once.m
index 3d846deca9211a1f2cf6b02c556354f00f62b0dd..ff2778d4bd0a4a2d9dae1e8c7fa02dd9e4516f1d 100644
--- a/clang/test/SemaObjC/warn-called-once.m
+++ b/clang/test/SemaObjC/warn-called-once.m
@@ -31,6 +31,16 @@ typedef struct {
 @class NSString, Protocol;
 extern void NSLog(NSString *format, ...);
 
+typedef int group_t;
+typedef struct dispatch_queue_s *dispatch_queue_t;
+typedef void (^dispatch_block_t)(void);
+extern dispatch_queue_t queue;
+
+void dispatch_group_async(dispatch_queue_t queue,
+                          group_t group,
+                          dispatch_block_t block);
+void dispatch_async(dispatch_queue_t queue, dispatch_block_t block);
+
 void escape(void (^callback)(void));
 void escape_void(void *);
 void indirect_call(void (^callback)(void) CALLED_ONCE);
@@ -225,11 +235,11 @@ void indirect_call_within_direct_call(void (^callback)(void) CALLED_ONCE,
 }
 
 void block_call_1(void (^callback)(void) CALLED_ONCE) {
-  indirect_call(^{
-    callback();
-  });
-  callback();
-  // no-warning
+  indirect_call( // expected-note{{previous call is here}}
+      ^{
+        callback();
+      });
+  callback(); // expected-warning{{'callback' parameter marked 'called_once' is called twice}}
 }
 
 void block_call_2(void (^callback)(void) CALLED_ONCE) {
@@ -255,7 +265,7 @@ void block_call_4(int cond, void (^callback)(void) CALLED_ONCE) {
       // expected-warning@-1{{'callback' parameter marked 'called_once' is never used when taking false branch}}
       escape(callback);
     }
-  }();
+  }(); // no-warning
 }
 
 void block_call_5(void (^outer)(void) CALLED_ONCE) {
@@ -273,6 +283,32 @@ void block_with_called_once(void (^outer)(void) CALLED_ONCE) {
   outer(); // expected-warning{{'outer' parameter marked 'called_once' is called twice}}
 }
 
+void block_dispatch_call(int cond, void (^callback)(void) CALLED_ONCE) {
+  dispatch_async(queue, ^{
+    if (cond) // expected-warning{{'callback' parameter marked 'called_once' is never called when taking false branch}}
+      callback();
+  });
+}
+
+void block_escape_call_1(int cond, void (^callback)(void) CALLED_ONCE) {
+  escape_void((__bridge void *)^{
+    if (cond) {
+      // no-warning
+      callback();
+    }
+  });
+}
+
+void block_escape_call_2(int cond, void (^callback)(void) CALLED_ONCE) {
+  escape_void((__bridge void *)^{
+    if (cond) {
+      callback(); // expected-note{{previous call is here}}
+    }
+    // Double call can still be reported.
+    callback(); // expected-warning{{'callback' parameter marked 'called_once' is called twice}}
+  });
+}
+
 void never_called_one_exit(int cond, void (^callback)(void) CALLED_ONCE) {
   if (!cond) // expected-warning{{'callback' parameter marked 'called_once' is never called when taking true branch}}
     return;
@@ -822,11 +858,10 @@ void suppression_3(int cond, void (^callback)(void) CALLED_ONCE) {
 
 - (void)block_call_1:(void (^)(void))CALLED_ONCE callback {
   // We consider captures by blocks as escapes
-  [self indirect_call:(^{
+  [self indirect_call:(^{ // expected-note{{previous call is here}}
           callback();
         })];
-  callback();
-  // no-warning
+  callback(); // expected-warning{{'callback' parameter marked 'called_once' is called twice}}
 }
 
 - (void)block_call_2:(int)cond callback:(void (^)(void))CALLED_ONCE callback {
@@ -1130,4 +1165,74 @@ void suppression_3(int cond, void (^callback)(void) CALLED_ONCE) {
   }
 }
 
+- (void)test_escape_before_branch:(int)cond
+                   withCompletion:(void (^)(void))handler {
+  if (cond) {
+    filler();
+  }
+
+  void (^copiedHandler)(void) = ^{
+    handler();
+  };
+
+  if (cond) {
+    // no-warning
+    handler();
+  } else {
+    copiedHandler();
+  }
+}
+
+- (void)test_escape_after_branch:(int)cond
+                  withCompletion:(void (^)(void))handler {
+  if (cond) {
+    // no-warning
+    handler();
+  }
+
+  escape(handler);
+}
+
+// rdar://74441906
+typedef void (^DeferredBlock)(void);
+static inline void DefferedCallback(DeferredBlock *inBlock) { (*inBlock)(); }
+#define _DEFERCONCAT(a, b) a##b
+#define _DEFERNAME(a) _DEFERCONCAT(__DeferredVar_, a)
+#define DEFER __extension__ __attribute__((cleanup(DefferedCallback), unused)) \
+                  DeferredBlock _DEFERNAME(__COUNTER__) = ^
+
+- (void)test_cleanup_1:(int)cond
+        withCompletion:(void (^)(void))handler {
+  int error = 0;
+  DEFER {
+    if (error)
+      handler();
+  };
+
+  if (cond) {
+    error = 1;
+  } else {
+    // no-warning
+    handler();
+  }
+}
+
+- (void)test_cleanup_2:(int)cond
+        withCompletion:(void (^)(void))handler {
+  int error = 0;
+  DEFER {
+    if (error)
+      handler();
+  };
+
+  if (cond) {
+    error = 1;
+  } else {
+    handler(); // expected-note{{previous call is here}}
+  }
+
+  // We still can warn about double call even in this case.
+  handler(); // expected-warning{{completion handler is called twice}}
+}
+
 @end
diff --git a/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl b/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl
index 825dd3a935d07f5586b4726755dd3a6e22e3ced1..103d1d8b262b55ca619c9a83a909b34eb53b0123 100644
--- a/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl
+++ b/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl
@@ -39,6 +39,9 @@ typedef unsigned int uint;
 typedef unsigned long ulong;
 typedef unsigned short ushort;
 typedef __SIZE_TYPE__ size_t;
+typedef __PTRDIFF_TYPE__ ptrdiff_t;
+typedef __INTPTR_TYPE__ intptr_t;
+typedef __UINTPTR_TYPE__ uintptr_t;
 typedef char char2 __attribute__((ext_vector_type(2)));
 typedef char char4 __attribute__((ext_vector_type(4)));
 typedef uchar uchar4 __attribute__((ext_vector_type(4)));
@@ -98,6 +101,24 @@ void test_typedef_args(clk_event_t evt, volatile atomic_flag *flg, global unsign
   size_t ws[2] = {2, 8};
   ndrange_t r = ndrange_2D(ws);
 }
+
+// Check that atomic_fetch_ functions can be called with (u)intptr_t arguments,
+// despite OpenCLBuiltins.td not providing explicit overloads for those types.
+void test_atomic_fetch(volatile __generic atomic_int *a_int,
+                       volatile __generic atomic_intptr_t *a_intptr,
+                       volatile __generic atomic_uintptr_t *a_uintptr) {
+  int i;
+  intptr_t ip;
+  uintptr_t uip;
+  ptrdiff_t ptrdiff;
+
+  i = atomic_fetch_add(a_int, i);
+  ip = atomic_fetch_add(a_intptr, ptrdiff);
+  uip = atomic_fetch_add(a_uintptr, ptrdiff);
+
+  ip = atomic_fetch_or(a_intptr, ip);
+  uip = atomic_fetch_or(a_uintptr, uip);
+}
 #endif
 
 kernel void basic_conversion() {
diff --git a/clang/test/SemaOpenCL/fp-options.cl b/clang/test/SemaOpenCL/fp-options.cl
index 413afd61819d3899b0f1721aaa264b3afaa16237..3b7cb89cf41b77de113f51841324b49a2c8e5625 100644
--- a/clang/test/SemaOpenCL/fp-options.cl
+++ b/clang/test/SemaOpenCL/fp-options.cl
@@ -1,4 +1,3 @@
-// RUN: %clang_cc1 %s -finclude-default-header -triple spir-unknown-unknown -emit-pch -o %t.pch
-// RUN: %clang_cc1 %s -finclude-default-header -cl-no-signed-zeros -triple spir-unknown-unknown -include-pch %t.pch -fsyntax-only -verify
+// RUN: %clang_cc1 %s -finclude-default-header -fdeclare-opencl-builtins -triple spir-unknown-unknown -emit-pch -o %t.pch
+// RUN: %clang_cc1 %s -finclude-default-header -fdeclare-opencl-builtins -cl-no-signed-zeros -triple spir-unknown-unknown -include-pch %t.pch -fsyntax-only -verify
 // expected-no-diagnostics
-
diff --git a/clang/test/SemaOpenCL/printf-format-string-warnings.cl b/clang/test/SemaOpenCL/printf-format-string-warnings.cl
index d08c95b6d8abdb12cf54d32545653e6acb960104..af490510855711f32ded56cd9df4b2bfc2bb7c8a 100644
--- a/clang/test/SemaOpenCL/printf-format-string-warnings.cl
+++ b/clang/test/SemaOpenCL/printf-format-string-warnings.cl
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -verify -pedantic -fsyntax-only -cl-std=CL2.0 -finclude-default-header
+// RUN: %clang_cc1 %s -verify -pedantic -fsyntax-only -cl-std=CL2.0 -finclude-default-header -fdeclare-opencl-builtins
 
 // Make sure warnings are produced based on printf format strings.
 
diff --git a/clang/test/SemaOpenCLCXX/address-space-castoperators.cl b/clang/test/SemaOpenCLCXX/address-space-castoperators.clcpp
similarity index 93%
rename from clang/test/SemaOpenCLCXX/address-space-castoperators.cl
rename to clang/test/SemaOpenCLCXX/address-space-castoperators.clcpp
index 7fd7f728fda3989fb4c8e8f0151697fe0da0c2f9..f0ec86cb0afc688cdd74e42e8054a4a1cafbf71c 100644
--- a/clang/test/SemaOpenCLCXX/address-space-castoperators.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-castoperators.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -cl-std=clc++ -pedantic -ast-dump -verify | FileCheck %s
+//RUN: %clang_cc1 %s -pedantic -ast-dump -verify | FileCheck %s
 
 void nester_ptr() {
   local int * * locgen;
diff --git a/clang/test/SemaOpenCLCXX/address-space-cond.cl b/clang/test/SemaOpenCLCXX/address-space-cond.clcpp
similarity index 91%
rename from clang/test/SemaOpenCLCXX/address-space-cond.cl
rename to clang/test/SemaOpenCLCXX/address-space-cond.clcpp
index 8090598920009fd76f2124bdf8366ab7ea9a3102..1b45515b1a150a765d5968b481b67284aa99e4e3 100644
--- a/clang/test/SemaOpenCLCXX/address-space-cond.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-cond.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -cl-std=clc++ -pedantic -verify
+// RUN: %clang_cc1 %s -pedantic -verify
 
 namespace PointerRvalues {
 
diff --git a/clang/test/SemaOpenCLCXX/address-space-deduction.cl b/clang/test/SemaOpenCLCXX/address-space-deduction.clcpp
similarity index 97%
rename from clang/test/SemaOpenCLCXX/address-space-deduction.cl
rename to clang/test/SemaOpenCLCXX/address-space-deduction.clcpp
index ddfdb6da4347c8aa5b0b78e9db9f5d027cec496b..1b757ca43687192d150d6869492e256dd76f3aa5 100644
--- a/clang/test/SemaOpenCLCXX/address-space-deduction.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-deduction.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -cl-std=clc++ -pedantic -ast-dump -verify | FileCheck %s
+//RUN: %clang_cc1 %s -pedantic -ast-dump -verify | FileCheck %s
 
 //expected-no-diagnostics
 
diff --git a/clang/test/SemaOpenCLCXX/address-space-lambda.cl b/clang/test/SemaOpenCLCXX/address-space-lambda.clcpp
similarity index 88%
rename from clang/test/SemaOpenCLCXX/address-space-lambda.cl
rename to clang/test/SemaOpenCLCXX/address-space-lambda.clcpp
index 571ea903587765b50399ec8e4265afa160d99962..66dc4da13d2c3e3c4e12e4d6bef379d862a41686 100644
--- a/clang/test/SemaOpenCLCXX/address-space-lambda.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-lambda.clcpp
@@ -1,5 +1,5 @@
-//RUN: %clang_cc1 %s -cl-std=clc++ -pedantic -ast-dump -verify | FileCheck %s
-//RUN: %clang_cc1 %s -cl-std=clc++ -pedantic -ast-dump -verify -triple i386-windows | FileCheck %s
+//RUN: %clang_cc1 %s -pedantic -ast-dump -verify | FileCheck %s
+//RUN: %clang_cc1 %s -pedantic -ast-dump -verify -triple i386-windows | FileCheck %s
 
 //CHECK: CXXMethodDecl {{.*}} constexpr operator() 'int (__private int){{.*}} const __generic'
 auto glambda = [](auto a) { return a; };
@@ -61,7 +61,10 @@ __kernel void test_qual() {
   [&] () __global {} (); //expected-error{{no matching function for call to object of type '(lambda at}} expected-note{{candidate function not viable: 'this' object is in default address space, but method expects object in address space '__global'}}
   [&] () __private {} (); //expected-error{{no matching function for call to object of type '(lambda at}} expected-note{{candidate function not viable: 'this' object is in default address space, but method expects object in address space '__private'}}
 
-  [&] __private {} (); //expected-error{{lambda requires '()' before attribute specifier}} expected-error{{expected body of lambda expression}}
+  [&] __private {} (); // expected-error{{no matching function for call to object of type '(lambda at}} expected-note{{candidate function not viable: 'this' object is in default address space, but method expects object in address space '__private'}}
+#if __cplusplus <= 202002L
+// expected-warning@-2{{lambda without a parameter clause is a C++2b extension}}
+#endif
 
   [&] () mutable __private {} ();
   [&] () __private mutable {} (); //expected-error{{expected body of lambda expression}}
diff --git a/clang/test/SemaOpenCLCXX/address-space-of-this-class-scope.cl b/clang/test/SemaOpenCLCXX/address-space-of-this-class-scope.clcpp
similarity index 90%
rename from clang/test/SemaOpenCLCXX/address-space-of-this-class-scope.cl
rename to clang/test/SemaOpenCLCXX/address-space-of-this-class-scope.clcpp
index 86d839d9701b4403aa18fcbda080d26962a2369c..55cfef3ceee4bf35bc9c8f94b7e64e6cd024323c 100644
--- a/clang/test/SemaOpenCLCXX/address-space-of-this-class-scope.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-of-this-class-scope.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify
+//RUN: %clang_cc1 %s -triple spir-unknown-unknown -pedantic -verify
 
 struct C {
   auto fGlob() __global -> decltype(this);
diff --git a/clang/test/SemaOpenCLCXX/address-space-of-this.cl b/clang/test/SemaOpenCLCXX/address-space-of-this.clcpp
similarity index 56%
rename from clang/test/SemaOpenCLCXX/address-space-of-this.cl
rename to clang/test/SemaOpenCLCXX/address-space-of-this.clcpp
index ac79b34119284b4d0d301dad62e575421da59c09..d77469b197c28d10a300d7a0d102901b173103c5 100644
--- a/clang/test/SemaOpenCLCXX/address-space-of-this.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-of-this.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify -fsyntax-only
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -pedantic -verify -fsyntax-only
 // expected-no-diagnostics
 
 // Extract from PR38614
diff --git a/clang/test/SemaOpenCLCXX/address-space-references.cl b/clang/test/SemaOpenCLCXX/address-space-references.clcpp
similarity index 95%
rename from clang/test/SemaOpenCLCXX/address-space-references.cl
rename to clang/test/SemaOpenCLCXX/address-space-references.clcpp
index 05e789a7d4fddbc2bd7646601a1bf92f266ceba6..76426ea65c2811da3ce337c0cfbde0b469f1d88c 100644
--- a/clang/test/SemaOpenCLCXX/address-space-references.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-references.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify -fsyntax-only
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -pedantic -verify -fsyntax-only
 
 __global const int& f(__global float &ref) {
   return ref; // expected-error{{reference of type 'const __global int &' cannot bind to a temporary object because of address space mismatch}}
diff --git a/clang/test/SemaOpenCLCXX/address-space-templates.cl b/clang/test/SemaOpenCLCXX/address-space-templates.clcpp
similarity index 93%
rename from clang/test/SemaOpenCLCXX/address-space-templates.cl
rename to clang/test/SemaOpenCLCXX/address-space-templates.clcpp
index b7db0e6de3d2d9a5a20a4c1d548d808b975042ac..0ea1a2a1e4df1e1bff3b8fd3c7272bf8930d37a9 100644
--- a/clang/test/SemaOpenCLCXX/address-space-templates.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-templates.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -cl-std=clc++ -pedantic -verify -ast-dump  | FileCheck %s
+//RUN: %clang_cc1 %s -pedantic -verify -ast-dump  | FileCheck %s
 
 template <typename T>
 struct S {
@@ -31,7 +31,7 @@ template <class _Tp> struct as_pointer {
 // Check address space deduction in template parameter deduction.
 struct rep {
   // When there is no address space on a reference use __generic.
-  // CHECK: |-CXXConstructorDecl {{.*}} rep 'void (const __generic rep &__private) __generic'
+  // CHECK: |-CXXConstructorDecl {{.*}} rep 'void (const __generic rep &__private){{.*}} __generic'
   template<class U, class = typename as_pointer<U>::type>
   rep(U&& v) {}
 };
@@ -57,6 +57,6 @@ void bar() {
   rep_outer r;
   int i;
   // Preserve the address space of the type in forwarding reference.
-  // CHECK: CXXMethodDecl {{.*}} operator() 'void (__private int &__private) const __generic'
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (__private int &__private){{.*}} const __generic'
   foo4(i, [](auto&& x){;});
 }
diff --git a/clang/test/SemaOpenCLCXX/address_space_overloading.cl b/clang/test/SemaOpenCLCXX/address_space_overloading.clcpp
similarity index 83%
rename from clang/test/SemaOpenCLCXX/address_space_overloading.cl
rename to clang/test/SemaOpenCLCXX/address_space_overloading.clcpp
index 33337ef461df2d288a9b2e6115b62093daaad4e8..01f5010b58bfad6858b7e3551be3cf4c9d94cabf 100644
--- a/clang/test/SemaOpenCLCXX/address_space_overloading.cl
+++ b/clang/test/SemaOpenCLCXX/address_space_overloading.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -verify -pedantic -fsyntax-only -cl-std=clc++
+// RUN: %clang_cc1 %s -verify -pedantic -fsyntax-only
 // expected-no-diagnostics
 
 struct RetGlob {
diff --git a/clang/test/SemaOpenCLCXX/addrspace-auto.cl b/clang/test/SemaOpenCLCXX/addrspace-auto.clcpp
similarity index 94%
rename from clang/test/SemaOpenCLCXX/addrspace-auto.cl
rename to clang/test/SemaOpenCLCXX/addrspace-auto.clcpp
index 2860237ddef73836d0a4da75d1431f135215445a..7862564d1b4ef032b61183dda1a23c5e588b3e4a 100644
--- a/clang/test/SemaOpenCLCXX/addrspace-auto.cl
+++ b/clang/test/SemaOpenCLCXX/addrspace-auto.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -cl-std=clc++ -pedantic -ast-dump -verify | FileCheck %s
+//RUN: %clang_cc1 %s -pedantic -ast-dump -verify | FileCheck %s
 
 __constant int i = 1;
 //CHECK: |-VarDecl {{.*}} ai '__global int':'__global int'
diff --git a/clang/test/SemaOpenCLCXX/addrspace_cast.cl b/clang/test/SemaOpenCLCXX/addrspace_cast.clcpp
similarity index 95%
rename from clang/test/SemaOpenCLCXX/addrspace_cast.cl
rename to clang/test/SemaOpenCLCXX/addrspace_cast.clcpp
index 3bf01757accdc9923acef6c6a0589d4b7cb0198d..37cf1bc1d296bcac4e8b70cebe0ef2efd1cecfcc 100644
--- a/clang/test/SemaOpenCLCXX/addrspace_cast.cl
+++ b/clang/test/SemaOpenCLCXX/addrspace_cast.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify -fsyntax-only
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -pedantic -verify -fsyntax-only
 
 void foo(global int *gl, const global int *gl_const, global int &gl_ref) {
   //FIXME: Diagnostics can be improved to be more specific in some cases.
diff --git a/clang/test/SemaOpenCLCXX/addrspace_cast_ast_dump.cl b/clang/test/SemaOpenCLCXX/addrspace_cast_ast_dump.clcpp
similarity index 83%
rename from clang/test/SemaOpenCLCXX/addrspace_cast_ast_dump.cl
rename to clang/test/SemaOpenCLCXX/addrspace_cast_ast_dump.clcpp
index cdc3e2785774339e50730dc63a0e9591a27f5f13..87a7f669ea9ea87e8e99bd5b2341c0d09b1bad68 100644
--- a/clang/test/SemaOpenCLCXX/addrspace_cast_ast_dump.cl
+++ b/clang/test/SemaOpenCLCXX/addrspace_cast_ast_dump.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify -ast-dump | FileCheck %s
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -pedantic -verify -ast-dump | FileCheck %s
 
 // expected-no-diagnostics
 
diff --git a/clang/test/SemaOpenCLCXX/invalid-kernel.cl b/clang/test/SemaOpenCLCXX/invalid-kernel.clcpp
similarity index 89%
rename from clang/test/SemaOpenCLCXX/invalid-kernel.cl
rename to clang/test/SemaOpenCLCXX/invalid-kernel.clcpp
index 2efdb756446dcab390925c946db62070489b68e9..2cbfffd5a00e959622eb7679ded9bb2f08144266 100644
--- a/clang/test/SemaOpenCLCXX/invalid-kernel.cl
+++ b/clang/test/SemaOpenCLCXX/invalid-kernel.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -cl-std=clc++ -pedantic -verify -fsyntax-only
+// RUN: %clang_cc1 %s -pedantic -verify -fsyntax-only
 
 struct C {
   kernel void m(); //expected-error{{kernel functions cannot be class members}}
diff --git a/clang/test/SemaOpenCLCXX/members.cl b/clang/test/SemaOpenCLCXX/members.clcpp
similarity index 76%
rename from clang/test/SemaOpenCLCXX/members.cl
rename to clang/test/SemaOpenCLCXX/members.clcpp
index 855948f0615ebfe2ef05978d6e01d225441c131e..c0a885cc4bd1d745f34cab46afc334330ad775fe 100644
--- a/clang/test/SemaOpenCLCXX/members.cl
+++ b/clang/test/SemaOpenCLCXX/members.clcpp
@@ -1,5 +1,5 @@
-//RUN: %clang_cc1 %s -triple spir -cl-std=clc++ -verify -fsyntax-only
-//RUN: %clang_cc1 %s -triple spir -cl-std=clc++ -verify -fsyntax-only -DFUNCPTREXT
+//RUN: %clang_cc1 %s -triple spir -verify -fsyntax-only
+//RUN: %clang_cc1 %s -triple spir -verify -fsyntax-only -DFUNCPTREXT
 
 #ifdef FUNCPTREXT
 #pragma OPENCL EXTENSION __cl_clang_function_pointers : enable
diff --git a/clang/test/SemaOpenCLCXX/method-overload-address-space.cl b/clang/test/SemaOpenCLCXX/method-overload-address-space.clcpp
similarity index 86%
rename from clang/test/SemaOpenCLCXX/method-overload-address-space.cl
rename to clang/test/SemaOpenCLCXX/method-overload-address-space.clcpp
index 7c428a570c2ce4ac0eb433d57382486b13cafea9..3164901d38cf44f0de346a86998b660a7e79a106 100644
--- a/clang/test/SemaOpenCLCXX/method-overload-address-space.cl
+++ b/clang/test/SemaOpenCLCXX/method-overload-address-space.clcpp
@@ -1,4 +1,4 @@
-//RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify
+//RUN: %clang_cc1 %s -triple spir-unknown-unknown -pedantic -verify
 
 struct C {
   void m1() __local __local; //expected-warning{{multiple identical address spaces specified for type}}
diff --git a/clang/test/SemaOpenCLCXX/newdelete.cl b/clang/test/SemaOpenCLCXX/newdelete.clcpp
similarity index 94%
rename from clang/test/SemaOpenCLCXX/newdelete.cl
rename to clang/test/SemaOpenCLCXX/newdelete.clcpp
index 2ef27843d5ce319240f81d9ff567bc8c6dbc73c5..127efbedded88f7833d0ad9bd692220609f6fb9f 100644
--- a/clang/test/SemaOpenCLCXX/newdelete.cl
+++ b/clang/test/SemaOpenCLCXX/newdelete.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify -fsyntax-only
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -pedantic -verify -fsyntax-only
 
 class A {
   public:
diff --git a/clang/test/SemaOpenCLCXX/references.cl b/clang/test/SemaOpenCLCXX/references.clcpp
similarity index 89%
rename from clang/test/SemaOpenCLCXX/references.cl
rename to clang/test/SemaOpenCLCXX/references.clcpp
index 42acb12729273fc7537490d4330599981db1008d..cffcc2cfbc29fe2a7603821dc4fd8bb20c3d316a 100644
--- a/clang/test/SemaOpenCLCXX/references.cl
+++ b/clang/test/SemaOpenCLCXX/references.clcpp
@@ -1,5 +1,5 @@
-//RUN: %clang_cc1 %s -cl-std=clc++ -verify -fsyntax-only -triple spir
-//RUN: %clang_cc1 %s -cl-std=clc++ -verify -fsyntax-only -DFPTREXT -triple spir
+//RUN: %clang_cc1 %s -verify -fsyntax-only -triple spir
+//RUN: %clang_cc1 %s -verify -fsyntax-only -DFPTREXT -triple spir
 
 #ifdef FPTREXT
 #pragma OPENCL EXTENSION __cl_clang_function_pointers : enable
diff --git a/clang/test/SemaOpenCLCXX/restricted.cl b/clang/test/SemaOpenCLCXX/restricted.clcpp
similarity index 94%
rename from clang/test/SemaOpenCLCXX/restricted.cl
rename to clang/test/SemaOpenCLCXX/restricted.clcpp
index c00c634073fe774b679292e5665692f4b0282111..2a84b4138b460bab275b228c1da9fac250c70acf 100644
--- a/clang/test/SemaOpenCLCXX/restricted.cl
+++ b/clang/test/SemaOpenCLCXX/restricted.clcpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -triple spir-unknown-unknown -cl-std=clc++ -pedantic -verify -fsyntax-only
+// RUN: %clang_cc1 %s -triple spir-unknown-unknown -pedantic -verify -fsyntax-only
 
 // This test checks that various C/C++/OpenCL C constructs are not available in
 // C++ for OpenCL.
diff --git a/clang/test/SemaOpenCLCXX/template-astype.cl b/clang/test/SemaOpenCLCXX/template-astype.cl
new file mode 100644
index 0000000000000000000000000000000000000000..efb93412cb201c91c5f765880537033c34f08238
--- /dev/null
+++ b/clang/test/SemaOpenCLCXX/template-astype.cl
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -fdeclare-opencl-builtins -finclude-default-header %s -cl-std=clc++ -verify
+
+// Test as_type, which is defined in terms of __builtin_astype.
+template <typename T>
+auto templated_astype(T x) {
+  return as_int2(x);
+  // expected-error@-1{{invalid reinterpretation: sizes of 'int2' (vector of 2 'int' values) and '__private int' must match}}
+}
+
+auto test_long(long x) { return templated_astype(x); }
+
+auto neg_test_int(int x) { return templated_astype(x); }
+// expected-note@-1{{in instantiation of function template specialization 'templated_astype<int>' requested here}}
+
+auto test_short4(short4 x) { return templated_astype(x); }
+
+// Test __builtin_astype.
+template <typename T>
+auto templated_builtin_astype(T x) {
+  return __builtin_astype(x, int2);
+}
+
+auto test_builtin(char8 x) { return templated_builtin_astype(x); }
diff --git a/clang/test/SemaSYCL/float128.cpp b/clang/test/SemaSYCL/float128.cpp
index d2d520b5b12dc6e01173b018c4a9a9df1f4e2afc..b91535eda4897c1af4743125c212a5ff2730090e 100644
--- a/clang/test/SemaSYCL/float128.cpp
+++ b/clang/test/SemaSYCL/float128.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple spir64 -fsycl -fsycl-is-device -verify -fsyntax-only %s
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsycl -fsycl-is-device -fsyntax-only %s
+// RUN: %clang_cc1 -triple spir64 -fsycl-is-device -verify -fsyntax-only %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsycl-is-device -fsyntax-only %s
 
 typedef __float128 BIGTY;
 
diff --git a/clang/test/SemaSYCL/int128.cpp b/clang/test/SemaSYCL/int128.cpp
index 38271bc020d34ac4e23ebb8409c84f114c0e904a..f6f92c237a9c5e2e1b35a47c2af3296793e74bbb 100644
--- a/clang/test/SemaSYCL/int128.cpp
+++ b/clang/test/SemaSYCL/int128.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple spir64 -aux-triple x86_64-unknown-linux-gnu \
-// RUN:    -fsycl -fsycl-is-device -verify -fsyntax-only %s
+// RUN:    -fsycl-is-device -verify -fsyntax-only %s
 
 typedef __uint128_t BIGTY;
 
diff --git a/clang/test/SemaSYCL/kernel-attribute.cpp b/clang/test/SemaSYCL/kernel-attribute.cpp
index ae9589e7b099ffffe5eb1db080dd3e117a69641a..84ba69fd46f9a56dc4b5c292a02e09f44a38643a 100644
--- a/clang/test/SemaSYCL/kernel-attribute.cpp
+++ b/clang/test/SemaSYCL/kernel-attribute.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++11 -fsyntax-only -fsycl -fsycl-is-device -verify %s
+// RUN: %clang_cc1 -std=c++11 -fsyntax-only -fsycl-is-device -verify %s
 
 // Only function templates
 [[clang::sycl_kernel]] int gv2 = 0; // expected-warning {{'sycl_kernel' attribute only applies to function templates}}
diff --git a/clang/test/SemaSYCL/prohibit-thread-local.cpp b/clang/test/SemaSYCL/prohibit-thread-local.cpp
index 4fd113626ea7b7e75edac659d14adb6dc0f8ef86..e507489695f8acba79508977277b9e9e2c3d9451 100644
--- a/clang/test/SemaSYCL/prohibit-thread-local.cpp
+++ b/clang/test/SemaSYCL/prohibit-thread-local.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsycl -fsycl-is-device -triple spir64 -verify -fsyntax-only %s
+// RUN: %clang_cc1 -fsycl-is-device -triple spir64 -verify -fsyntax-only %s
 
 thread_local const int prohobit_ns_scope = 0;
 thread_local int prohobit_ns_scope2 = 0;
diff --git a/clang/test/SemaTemplate/deduction.cpp b/clang/test/SemaTemplate/deduction.cpp
index b9a1f0dccb243427d8d89ccf0c55d7742f0fa87b..0f48a4dc1095cb136f9388e60e19b4b44a5be9f0 100644
--- a/clang/test/SemaTemplate/deduction.cpp
+++ b/clang/test/SemaTemplate/deduction.cpp
@@ -604,3 +604,14 @@ namespace merge_size_only_deductions {
   int b = f(X<char [1], char [2]>(), Y<1, 2>(), X<id<int>, id<int>>());
 #endif
 }
+
+namespace PR49724 {
+  struct A;
+  template<int A::*> class X {};
+  template<int A::*P> void f(X<P>);
+  void g(X<nullptr> x) { f(x); }
+
+  template<void (A::*)()> class Y {};
+  template<void (A::*P)()> void f(Y<P>);
+  void g(Y<nullptr> y) { f(y); }
+}
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index 863ab444fb023766fc27438c46ef99555fb0d31a..21b674539a30cf19be7eb98ded27188430c92c2d 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -26,7 +26,7 @@ config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.c', '.cpp', '.i', '.cppm', '.m', '.mm', '.cu',
-                   '.ll', '.cl', '.s', '.S', '.modulemap', '.test', '.rs', '.ifs']
+                   '.ll', '.cl', '.clcpp', '.s', '.S', '.modulemap', '.test', '.rs', '.ifs']
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
@@ -92,6 +92,9 @@ config.substitutions.append(
     ('%hmaptool', "'%s' %s" % (config.python_executable,
                              os.path.join(config.clang_tools_dir, 'hmaptool'))))
 
+config.substitutions.append(('%host_cc', config.host_cc))
+config.substitutions.append(('%host_cxx', config.host_cxx))
+
 
 # Plugins (loadable modules)
 if config.has_plugins and config.llvm_plugin_ext:
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index c3382e2c1c42bd6ae73c2870161b5225efdbf6b4..85526b9d30d6acf83890c59d08c1d3f81955b513 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -15,6 +15,7 @@ config.clang_src_dir = path(r"@CLANG_SOURCE_DIR@")
 config.clang_tools_dir = path(r"@CLANG_TOOLS_DIR@")
 config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@TARGET_TRIPLE@"
+config.host_cc = "@CMAKE_C_COMPILER@"
 config.host_cxx = "@CMAKE_CXX_COMPILER@"
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c b/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c
index 1626eb540841574038f3eb4c898b5f9137692e18..e0dfc42c4bd62eeda1fb0b61dfdd7c185399e84d 100644
--- a/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c
+++ b/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c
@@ -1,5 +1,6 @@
 // Check that the non-clang/non-filechecked runlines execute
-// RUN: cp %s %s.copy.c
+// RUN: cp %s %S/Output/execute-all-runlines.copy.c
+// RUN: cp %S/Output/execute-all-runlines.copy.c %s.copy.c
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp %s.copy.c -emit-llvm-bc -o %t-host.bc
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -fopenmp-host-ir-file-path %t-host.bc %s.copy.c -emit-llvm -o - | FileCheck %s
 
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c.expected b/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c.expected
index 5edf11e668e4c24aa4742d89bf85be18626a7492..ae9745fa9b1effeebf4b67e1ba144ad2c2b5b4e0 100644
--- a/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c.expected
+++ b/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c.expected
@@ -1,6 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // Check that the non-clang/non-filechecked runlines execute
-// RUN: cp %s %s.copy.c
+// RUN: cp %s %S/Output/execute-all-runlines.copy.c
+// RUN: cp %S/Output/execute-all-runlines.copy.c %s.copy.c
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp %s.copy.c -emit-llvm-bc -o %t-host.bc
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -fopenmp-host-ir-file-path %t-host.bc %s.copy.c -emit-llvm -o - | FileCheck %s
 
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/resolve-tmp-conflict.cpp b/clang/test/utils/update_cc_test_checks/Inputs/resolve-tmp-conflict.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d82490ea3c886a1cc06942d155b16d4f1b5040a1
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/Inputs/resolve-tmp-conflict.cpp
@@ -0,0 +1,8 @@
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+
+void foo(int a) {
+  int &tmp0 = a;
+  int &&tmp1 = 1;
+  tmp1 = a;
+  return;
+}
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/resolve-tmp-conflict.cpp.expected b/clang/test/utils/update_cc_test_checks/Inputs/resolve-tmp-conflict.cpp.expected
new file mode 100644
index 0000000000000000000000000000000000000000..9a3c4580f4c1b44d5f339319fa8b66cfee09b585
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/Inputs/resolve-tmp-conflict.cpp.expected
@@ -0,0 +1,25 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --prefix-filecheck-ir-name _
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+
+// CHECK-LABEL: define {{[^@]+}}@_Z3fooi
+// CHECK-SAME: (i32 [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[_TMP0:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[_TMP1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[REF_TMP:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[A]], i32* [[A_ADDR]], align 4
+// CHECK-NEXT:    store i32* [[A_ADDR]], i32** [[_TMP0]], align 8
+// CHECK-NEXT:    store i32 1, i32* [[REF_TMP]], align 4
+// CHECK-NEXT:    store i32* [[REF_TMP]], i32** [[_TMP1]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A_ADDR]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32*, i32** [[_TMP1]], align 8
+// CHECK-NEXT:    store i32 [[TMP0]], i32* [[TMP1]], align 4
+// CHECK-NEXT:    ret void
+//
+void foo(int a) {
+  int &tmp0 = a;
+  int &&tmp1 = 1;
+  tmp1 = a;
+  return;
+}
diff --git a/clang/test/utils/update_cc_test_checks/resolve-tmp-conflict.test b/clang/test/utils/update_cc_test_checks/resolve-tmp-conflict.test
new file mode 100644
index 0000000000000000000000000000000000000000..a802e1aeecd82c2155f0484d17e2d4c96fe3ebd5
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/resolve-tmp-conflict.test
@@ -0,0 +1,8 @@
+## Test that CHECK lines generated avoid naming conflicts with FileCheck IR variables
+
+# RUN: cp %S/Inputs/resolve-tmp-conflict.cpp %t.cpp && %update_cc_test_checks --function-signature --prefix-filecheck-ir-name _ %t.cpp
+# RUN: diff -u %S/Inputs/resolve-tmp-conflict.cpp.expected %t.cpp
+
+## Check that re-running update_cc_test_checks doesn't change the output
+# RUN: %update_cc_test_checks %t.cpp
+# RUN: diff -u %S/Inputs/resolve-tmp-conflict.cpp.expected %t.cpp
diff --git a/clang/tools/arcmt-test/arcmt-test.cpp b/clang/tools/arcmt-test/arcmt-test.cpp
index 940e622b8a68f75da84d77f9640979bc28c5b703..26e123c59d59daaf3806b3516927aa6deab94e1b 100644
--- a/clang/tools/arcmt-test/arcmt-test.cpp
+++ b/clang/tools/arcmt-test/arcmt-test.cpp
@@ -207,11 +207,13 @@ static bool performTransformations(StringRef resourcesPath,
 static bool filesCompareEqual(StringRef fname1, StringRef fname2) {
   using namespace llvm;
 
-  ErrorOr<std::unique_ptr<MemoryBuffer>> file1 = MemoryBuffer::getFile(fname1);
+  ErrorOr<std::unique_ptr<MemoryBuffer>> file1 =
+      MemoryBuffer::getFile(fname1, /*IsText=*/true);
   if (!file1)
     return false;
 
-  ErrorOr<std::unique_ptr<MemoryBuffer>> file2 = MemoryBuffer::getFile(fname2);
+  ErrorOr<std::unique_ptr<MemoryBuffer>> file2 =
+      MemoryBuffer::getFile(fname2, /*IsText=*/true);
   if (!file2)
     return false;
 
@@ -240,7 +242,7 @@ static bool verifyTransformedFiles(ArrayRef<std::string> resultFiles) {
   if (RemappingsFile.empty())
     inputBuf = MemoryBuffer::getSTDIN();
   else
-    inputBuf = MemoryBuffer::getFile(RemappingsFile);
+    inputBuf = MemoryBuffer::getFile(RemappingsFile, /*IsText=*/true);
   if (!inputBuf) {
     errs() << "error: could not read remappings input\n";
     return true;
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 1bb65ef12d6b9ebd4a982cf21299b76bc0068182..a8ff42ab104cadb09c7da913c1caa2929f16bbd4 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -222,16 +222,16 @@ static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
   return llvm::json::Array(Strings);
 }
 
-static llvm::json::Array toJSONSorted(std::vector<ClangModuleDep> V) {
-  llvm::sort(V, [](const ClangModuleDep &A, const ClangModuleDep &B) {
+static llvm::json::Array toJSONSorted(std::vector<ModuleID> V) {
+  llvm::sort(V, [](const ModuleID &A, const ModuleID &B) {
     return std::tie(A.ModuleName, A.ContextHash) <
            std::tie(B.ModuleName, B.ContextHash);
   });
 
   llvm::json::Array Ret;
-  for (const ClangModuleDep &CMD : V)
+  for (const ModuleID &MID : V)
     Ret.push_back(llvm::json::Object(
-        {{"module-name", CMD.ModuleName}, {"context-hash", CMD.ContextHash}}));
+        {{"module-name", MID.ModuleName}, {"context-hash", MID.ContextHash}}));
   return Ret;
 }
 
@@ -244,26 +244,25 @@ public:
 
     InputDeps ID;
     ID.FileName = std::string(Input);
-    ID.ContextHash = std::move(FD.ContextHash);
+    ID.ContextHash = std::move(FD.ID.ContextHash);
     ID.FileDeps = std::move(FD.FileDeps);
     ID.ModuleDeps = std::move(FD.ClangModuleDeps);
 
     std::unique_lock<std::mutex> ul(Lock);
     for (const ModuleDeps &MD : FDR.DiscoveredModules) {
-      auto I = Modules.find({MD.ContextHash, MD.ModuleName, 0});
+      auto I = Modules.find({MD.ID, 0});
       if (I != Modules.end()) {
         I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
         continue;
       }
-      Modules.insert(
-          I, {{MD.ContextHash, MD.ModuleName, InputIndex}, std::move(MD)});
+      Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});
     }
 
     if (FullCommandLine)
       ID.AdditonalCommandLine = FD.getAdditionalCommandLine(
-          [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); },
-          [&](ClangModuleDep CMD) -> const ModuleDeps & {
-            return lookupModuleDeps(CMD);
+          [&](ModuleID MID) { return lookupPCMPath(MID); },
+          [&](ModuleID MID) -> const ModuleDeps & {
+            return lookupModuleDeps(MID);
           });
 
     Inputs.push_back(std::move(ID));
@@ -271,13 +270,13 @@ public:
 
   void printFullOutput(raw_ostream &OS) {
     // Sort the modules by name to get a deterministic order.
-    std::vector<ContextModulePair> ModuleNames;
+    std::vector<IndexedModuleID> ModuleIDs;
     for (auto &&M : Modules)
-      ModuleNames.push_back(M.first);
-    llvm::sort(ModuleNames,
-               [](const ContextModulePair &A, const ContextModulePair &B) {
-                 return std::tie(A.ModuleName, A.InputIndex) <
-                        std::tie(B.ModuleName, B.InputIndex);
+      ModuleIDs.push_back(M.first);
+    llvm::sort(ModuleIDs,
+               [](const IndexedModuleID &A, const IndexedModuleID &B) {
+                 return std::tie(A.ID.ModuleName, A.InputIndex) <
+                        std::tie(B.ID.ModuleName, B.InputIndex);
                });
 
     llvm::sort(Inputs, [](const InputDeps &A, const InputDeps &B) {
@@ -287,20 +286,20 @@ public:
     using namespace llvm::json;
 
     Array OutModules;
-    for (auto &&ModName : ModuleNames) {
-      auto &MD = Modules[ModName];
+    for (auto &&ModID : ModuleIDs) {
+      auto &MD = Modules[ModID];
       Object O{
-          {"name", MD.ModuleName},
-          {"context-hash", MD.ContextHash},
+          {"name", MD.ID.ModuleName},
+          {"context-hash", MD.ID.ContextHash},
           {"file-deps", toJSONSorted(MD.FileDeps)},
           {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
           {"clang-modulemap-file", MD.ClangModuleMapFile},
           {"command-line",
            FullCommandLine
                ? MD.getFullCommandLine(
-                     [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); },
-                     [&](ClangModuleDep CMD) -> const ModuleDeps & {
-                       return lookupModuleDeps(CMD);
+                     [&](ModuleID MID) { return lookupPCMPath(MID); },
+                     [&](ModuleID MID) -> const ModuleDeps & {
+                       return lookupModuleDeps(MID);
                      })
                : MD.NonPathCommandLine},
       };
@@ -328,33 +327,31 @@ public:
   }
 
 private:
-  StringRef lookupPCMPath(ClangModuleDep CMD) {
-    return Modules[ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}]
-        .ImplicitModulePCMPath;
+  StringRef lookupPCMPath(ModuleID MID) {
+    return Modules[IndexedModuleID{MID, 0}].ImplicitModulePCMPath;
   }
 
-  const ModuleDeps &lookupModuleDeps(ClangModuleDep CMD) {
-    auto I =
-        Modules.find(ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0});
+  const ModuleDeps &lookupModuleDeps(ModuleID MID) {
+    auto I = Modules.find(IndexedModuleID{MID, 0});
     assert(I != Modules.end());
     return I->second;
   };
 
-  struct ContextModulePair {
-    std::string ContextHash;
-    std::string ModuleName;
+  struct IndexedModuleID {
+    ModuleID ID;
     mutable size_t InputIndex;
 
-    bool operator==(const ContextModulePair &Other) const {
-      return ContextHash == Other.ContextHash && ModuleName == Other.ModuleName;
+    bool operator==(const IndexedModuleID &Other) const {
+      return ID.ModuleName == Other.ID.ModuleName &&
+             ID.ContextHash == Other.ID.ContextHash;
     }
   };
 
-  struct ContextModulePairHasher {
-    std::size_t operator()(const ContextModulePair &CMP) const {
+  struct IndexedModuleIDHasher {
+    std::size_t operator()(const IndexedModuleID &IMID) const {
       using llvm::hash_combine;
 
-      return hash_combine(CMP.ContextHash, CMP.ModuleName);
+      return hash_combine(IMID.ID.ModuleName, IMID.ID.ContextHash);
     }
   };
 
@@ -362,12 +359,12 @@ private:
     std::string FileName;
     std::string ContextHash;
     std::vector<std::string> FileDeps;
-    std::vector<ClangModuleDep> ModuleDeps;
+    std::vector<ModuleID> ModuleDeps;
     std::vector<std::string> AdditonalCommandLine;
   };
 
   std::mutex Lock;
-  std::unordered_map<ContextModulePair, ModuleDeps, ContextModulePairHasher>
+  std::unordered_map<IndexedModuleID, ModuleDeps, IndexedModuleIDHasher>
       Modules;
   std::vector<InputDeps> Inputs;
 };
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index c3a3aab066ace5c6f2bd39d9ccec4b71e32880d2..ee3ffe3012d1e6a0d3fb6525c141b16ad7e43cd8 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -244,25 +244,28 @@ static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
 }
 
 static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
-  // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
-  TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
-  if (TheDriver.CCPrintOptions)
-    TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
-
-  // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
-  TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
-  if (TheDriver.CCPrintHeaders)
-    TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
-
-  // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
-  TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
-  if (TheDriver.CCLogDiagnostics)
-    TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
-
-  // Handle CC_PRINT_PROC_STAT and CC_PRINT_PROC_STAT_FILE.
-  TheDriver.CCPrintProcessStats = !!::getenv("CC_PRINT_PROC_STAT");
-  if (TheDriver.CCPrintProcessStats)
-    TheDriver.CCPrintStatReportFilename = ::getenv("CC_PRINT_PROC_STAT_FILE");
+  auto CheckEnvVar = [](const char *EnvOptSet, const char *EnvOptFile,
+                        std::string &OptFile) {
+    bool OptSet = !!::getenv(EnvOptSet);
+    if (OptSet) {
+      if (const char *Var = ::getenv(EnvOptFile))
+        OptFile = Var;
+    }
+    return OptSet;
+  };
+
+  TheDriver.CCPrintOptions =
+      CheckEnvVar("CC_PRINT_OPTIONS", "CC_PRINT_OPTIONS_FILE",
+                  TheDriver.CCPrintOptionsFilename);
+  TheDriver.CCPrintHeaders =
+      CheckEnvVar("CC_PRINT_HEADERS", "CC_PRINT_HEADERS_FILE",
+                  TheDriver.CCPrintHeadersFilename);
+  TheDriver.CCLogDiagnostics =
+      CheckEnvVar("CC_LOG_DIAGNOSTICS", "CC_LOG_DIAGNOSTICS_FILE",
+                  TheDriver.CCLogDiagnosticsFilename);
+  TheDriver.CCPrintProcessStats =
+      CheckEnvVar("CC_PRINT_PROC_STAT", "CC_PRINT_PROC_STAT_FILE",
+                  TheDriver.CCPrintStatReportFilename);
 }
 
 static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index e949b8513535471005c613cfddd0d81cb2ca2e44..841b36a6036c93a9bdccf844f56d5b937befc99d 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2278,7 +2278,18 @@ void OMPClauseEnqueue::VisitOMPSIMDClause(const OMPSIMDClause *) {}
 
 void OMPClauseEnqueue::VisitOMPNogroupClause(const OMPNogroupClause *) {}
 
-void OMPClauseEnqueue::VisitOMPDestroyClause(const OMPDestroyClause *) {}
+void OMPClauseEnqueue::VisitOMPInitClause(const OMPInitClause *C) {
+  VisitOMPClauseList(C);
+}
+
+void OMPClauseEnqueue::VisitOMPUseClause(const OMPUseClause *C) {
+  Visitor->AddStmt(C->getInteropVar());
+}
+
+void OMPClauseEnqueue::VisitOMPDestroyClause(const OMPDestroyClause *C) {
+  if (C->getInteropVar())
+    Visitor->AddStmt(C->getInteropVar());
+}
 
 void OMPClauseEnqueue::VisitOMPUnifiedAddressClause(
     const OMPUnifiedAddressClause *) {}
@@ -5655,6 +5666,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
         "OMPTargetTeamsDistributeParallelForSimdDirective");
   case CXCursor_OMPTargetTeamsDistributeSimdDirective:
     return cxstring::createRef("OMPTargetTeamsDistributeSimdDirective");
+  case CXCursor_OMPInteropDirective:
+    return cxstring::createRef("OMPInteropDirective");
   case CXCursor_OverloadCandidate:
     return cxstring::createRef("OverloadCandidate");
   case CXCursor_TypeAliasTemplateDecl:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index 0811b0bcdb886bc7cecc2df9fee5fc19be921ecf..2f8d1e35936ec8b44ad1d9dab3789e768e21a27d 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -807,6 +807,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
     K = CXCursor_OMPTargetTeamsDistributeSimdDirective;
     break;
+  case Stmt::OMPInteropDirectiveClass:
+    K = CXCursor_OMPInteropDirective;
+    break;
   case Stmt::BuiltinBitCastExprClass:
     K = CXCursor_BuiltinBitCastExpr;
   }
diff --git a/clang/tools/libclang/CXType.cpp b/clang/tools/libclang/CXType.cpp
index 4b9620827002c9f63549515b43a47dc4776afbb1..f8c36df35607e370716336a9cb95c94a74160cf9 100644
--- a/clang/tools/libclang/CXType.cpp
+++ b/clang/tools/libclang/CXType.cpp
@@ -1030,7 +1030,7 @@ long long clang_Type_getOffsetOf(CXType PT, const char *S) {
   // and we would return InvalidFieldName instead of Incomplete.
   // But this erroneous results does protects again a hidden assertion failure
   // in the RecordLayoutBuilder
-  if (Res.size() != 1)
+  if (!Res.isSingleResult())
     return CXTypeLayoutError_InvalidFieldName;
   if (const FieldDecl *FD = dyn_cast<FieldDecl>(Res.front()))
     return Ctx.getFieldOffset(FD);
diff --git a/clang/unittests/AST/ASTImporterObjCTest.cpp b/clang/unittests/AST/ASTImporterObjCTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2d848dcf754edde13a7a0f9d46640e6a94948956
--- /dev/null
+++ b/clang/unittests/AST/ASTImporterObjCTest.cpp
@@ -0,0 +1,89 @@
+//===- unittest/AST/ASTImporterObjCTest.cpp -============================--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tests for the correct import of AST nodes related to Objective-C and
+// Objective-C++.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/DeclContextInternals.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "gtest/gtest.h"
+
+#include "ASTImporterFixtures.h"
+
+using namespace clang::ast_matchers;
+using namespace clang;
+
+namespace {
+struct ImportObjCDecl : ASTImporterOptionSpecificTestBase {};
+} // namespace
+
+TEST_P(ImportObjCDecl, ImplicitlyDeclareSelf) {
+  Decl *FromTU = getTuDecl(R"(
+                           __attribute__((objc_root_class))
+                           @interface Root
+                           @end
+                           @interface C : Root
+                             -(void)method;
+                           @end
+                           @implementation C
+                             -(void)method {}
+                           @end
+                           )",
+                           Lang_OBJCXX, "input.mm");
+  auto *FromMethod = LastDeclMatcher<ObjCMethodDecl>().match(
+      FromTU, namedDecl(hasName("method")));
+  ASSERT_TRUE(FromMethod);
+  auto ToMethod = Import(FromMethod, Lang_OBJCXX);
+  ASSERT_TRUE(ToMethod);
+
+  // Both methods should have their implicit parameters.
+  EXPECT_TRUE(FromMethod->getSelfDecl() != nullptr);
+  EXPECT_TRUE(ToMethod->getSelfDecl() != nullptr);
+}
+
+TEST_P(ImportObjCDecl, ObjPropertyNameConflict) {
+  // Tests that properties that share the same name are correctly imported.
+  // This is only possible with one instance and one class property.
+  Decl *FromTU = getTuDecl(R"(
+                           @interface DupProp{}
+                           @property (class) int prop;
+                           @property int prop;
+                           @end
+                           )",
+                           Lang_OBJCXX, "input.mm");
+  auto *FromClass = FirstDeclMatcher<ObjCInterfaceDecl>().match(
+      FromTU, namedDecl(hasName("DupProp")));
+  auto ToClass = Import(FromClass, Lang_OBJCXX);
+  ASSERT_TRUE(ToClass);
+  // We should have one class and one instance property.
+  ASSERT_EQ(
+      1, std::distance(ToClass->classprop_begin(), ToClass->classprop_end()));
+  ASSERT_EQ(1,
+            std::distance(ToClass->instprop_begin(), ToClass->instprop_end()));
+  for (clang::ObjCPropertyDecl *prop : ToClass->properties()) {
+    // All properties should have a getter and a setter.
+    ASSERT_TRUE(prop->getGetterMethodDecl());
+    ASSERT_TRUE(prop->getSetterMethodDecl());
+    // The getters/setters should be able to find the right associated property.
+    ASSERT_EQ(prop->getGetterMethodDecl()->findPropertyDecl(), prop);
+    ASSERT_EQ(prop->getSetterMethodDecl()->findPropertyDecl(), prop);
+  }
+}
+
+static const auto ObjCTestArrayForRunOptions =
+    std::array<std::vector<std::string>, 2>{
+        {std::vector<std::string>{"-fno-objc-arc"},
+         std::vector<std::string>{"-fobjc-arc"}}};
+
+const auto ObjCTestValuesForRunOptions =
+    ::testing::ValuesIn(ObjCTestArrayForRunOptions);
+
+INSTANTIATE_TEST_CASE_P(ParameterizedTests, ImportObjCDecl,
+                        ObjCTestValuesForRunOptions, );
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index 193523f2fc516cecd392b300a33744bedf4bff16..fdbf811c94dc176a1f680e272a0288d78889cb81 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -246,6 +246,24 @@ TEST_P(ImportPath, CycleAfterCycle) {
   EXPECT_FALSE(path.hasCycleAtBack());
 }
 
+const internal::VariadicDynCastAllOfMatcher<Stmt, SourceLocExpr> sourceLocExpr;
+
+AST_MATCHER_P(SourceLocExpr, hasBuiltinStr, StringRef, Str) {
+  return Node.getBuiltinStr() == Str;
+}
+
+TEST_P(ImportExpr, ImportSourceLocExpr) {
+  MatchVerifier<Decl> Verifier;
+  testImport("void declToImport() { (void)__builtin_FILE(); }", Lang_CXX03, "",
+             Lang_CXX03, Verifier,
+             functionDecl(hasDescendant(
+                 sourceLocExpr(hasBuiltinStr("__builtin_FILE")))));
+  testImport("void declToImport() { (void)__builtin_COLUMN(); }", Lang_CXX03,
+             "", Lang_CXX03, Verifier,
+             functionDecl(hasDescendant(
+                 sourceLocExpr(hasBuiltinStr("__builtin_COLUMN")))));
+}
+
 TEST_P(ImportExpr, ImportStringLiteral) {
   MatchVerifier<Decl> Verifier;
   testImport("void declToImport() { (void)\"foo\"; }", Lang_CXX03, "",
@@ -613,6 +631,15 @@ TEST_P(ImportType, ImportDependentTemplateSpecialization) {
                  fieldDecl(hasType(dependentTemplateSpecializationType())))))));
 }
 
+TEST_P(ImportType, ImportDeducedTemplateSpecialization) {
+  MatchVerifier<Decl> Verifier;
+  testImport("template <typename T>"
+             "class C { public: C(T); };"
+             "C declToImport(123);",
+             Lang_CXX17, "", Lang_CXX17, Verifier,
+             varDecl(hasType(deducedTemplateSpecializationType())));
+}
+
 const internal::VariadicDynCastAllOfMatcher<Stmt, SizeOfPackExpr>
     sizeOfPackExpr;
 
@@ -735,6 +762,12 @@ TEST_P(ImportDecl, ImportRecordDeclInFunc) {
                  has(declStmt(hasSingleDecl(varDecl(hasName("d")))))))));
 }
 
+TEST_P(ImportDecl, ImportedVarDeclPreservesThreadLocalStorage) {
+  MatchVerifier<Decl> Verifier;
+  testImport("thread_local int declToImport;", Lang_CXX11, "", Lang_CXX11,
+             Verifier, varDecl(hasThreadStorageDuration()));
+}
+
 TEST_P(ASTImporterOptionSpecificTestBase, ImportRecordTypeInFunc) {
   Decl *FromTU = getTuDecl("int declToImport() { "
                            "  struct data_t {int a;int b;};"
@@ -2561,9 +2594,9 @@ TEST_P(ImportFriendFunctions, Lookup) {
     auto FromName = FromD->getDeclName();
     auto *Class = FirstDeclMatcher<CXXRecordDecl>().match(FromTU, ClassPattern);
     auto LookupRes = Class->noload_lookup(FromName);
-    ASSERT_EQ(LookupRes.size(), 0u);
+    ASSERT_TRUE(LookupRes.empty());
     LookupRes = FromTU->noload_lookup(FromName);
-    ASSERT_EQ(LookupRes.size(), 1u);
+    ASSERT_TRUE(LookupRes.isSingleResult());
   }
 
   auto *ToD = cast<FunctionDecl>(Import(FromD, Lang_CXX03));
@@ -2572,9 +2605,9 @@ TEST_P(ImportFriendFunctions, Lookup) {
   TranslationUnitDecl *ToTU = ToAST->getASTContext().getTranslationUnitDecl();
   auto *Class = FirstDeclMatcher<CXXRecordDecl>().match(ToTU, ClassPattern);
   auto LookupRes = Class->noload_lookup(ToName);
-  EXPECT_EQ(LookupRes.size(), 0u);
+  EXPECT_TRUE(LookupRes.empty());
   LookupRes = ToTU->noload_lookup(ToName);
-  EXPECT_EQ(LookupRes.size(), 1u);
+  EXPECT_TRUE(LookupRes.isSingleResult());
 
   EXPECT_EQ(DeclCounter<FunctionDecl>().match(ToTU, FunctionPattern), 1u);
   auto *To0 = FirstDeclMatcher<FunctionDecl>().match(ToTU, FunctionPattern);
@@ -2608,9 +2641,9 @@ TEST_P(ImportFriendFunctions, LookupWithProtoAfter) {
   auto *FromClass =
       FirstDeclMatcher<CXXRecordDecl>().match(FromTU, ClassPattern);
   auto LookupRes = FromClass->noload_lookup(FromName);
-  ASSERT_EQ(LookupRes.size(), 0u);
+  ASSERT_TRUE(LookupRes.empty());
   LookupRes = FromTU->noload_lookup(FromName);
-  ASSERT_EQ(LookupRes.size(), 1u);
+  ASSERT_TRUE(LookupRes.isSingleResult());
 
   auto *ToFriend = cast<FunctionDecl>(Import(FromFriend, Lang_CXX03));
   auto ToName = ToFriend->getDeclName();
@@ -2618,10 +2651,10 @@ TEST_P(ImportFriendFunctions, LookupWithProtoAfter) {
   TranslationUnitDecl *ToTU = ToAST->getASTContext().getTranslationUnitDecl();
   auto *ToClass = FirstDeclMatcher<CXXRecordDecl>().match(ToTU, ClassPattern);
   LookupRes = ToClass->noload_lookup(ToName);
-  EXPECT_EQ(LookupRes.size(), 0u);
+  EXPECT_TRUE(LookupRes.empty());
   LookupRes = ToTU->noload_lookup(ToName);
   // Test is disabled because this result is 2.
-  EXPECT_EQ(LookupRes.size(), 1u);
+  EXPECT_TRUE(LookupRes.isSingleResult());
 
   ASSERT_EQ(DeclCounter<FunctionDecl>().match(ToTU, FunctionPattern), 2u);
   ToFriend = FirstDeclMatcher<FunctionDecl>().match(ToTU, FunctionPattern);
@@ -2652,9 +2685,9 @@ TEST_P(ImportFriendFunctions, LookupWithProtoBefore) {
   auto *FromClass =
       FirstDeclMatcher<CXXRecordDecl>().match(FromTU, ClassPattern);
   auto LookupRes = FromClass->noload_lookup(FromName);
-  ASSERT_EQ(LookupRes.size(), 0u);
+  ASSERT_TRUE(LookupRes.empty());
   LookupRes = FromTU->noload_lookup(FromName);
-  ASSERT_EQ(LookupRes.size(), 1u);
+  ASSERT_TRUE(LookupRes.isSingleResult());
 
   auto *ToNormal = cast<FunctionDecl>(Import(FromNormal, Lang_CXX03));
   auto ToName = ToNormal->getDeclName();
@@ -2662,9 +2695,9 @@ TEST_P(ImportFriendFunctions, LookupWithProtoBefore) {
 
   auto *ToClass = FirstDeclMatcher<CXXRecordDecl>().match(ToTU, ClassPattern);
   LookupRes = ToClass->noload_lookup(ToName);
-  EXPECT_EQ(LookupRes.size(), 0u);
+  EXPECT_TRUE(LookupRes.empty());
   LookupRes = ToTU->noload_lookup(ToName);
-  EXPECT_EQ(LookupRes.size(), 1u);
+  EXPECT_TRUE(LookupRes.isSingleResult());
 
   EXPECT_EQ(DeclCounter<FunctionDecl>().match(ToTU, FunctionPattern), 2u);
   ToNormal = FirstDeclMatcher<FunctionDecl>().match(ToTU, FunctionPattern);
@@ -2694,9 +2727,9 @@ TEST_P(ImportFriendFunctions, ImportFriendChangesLookup) {
   ASSERT_FALSE(FromFriendF->isInIdentifierNamespace(Decl::IDNS_Ordinary));
   ASSERT_TRUE(FromFriendF->isInIdentifierNamespace(Decl::IDNS_OrdinaryFriend));
   auto LookupRes = FromNormalTU->noload_lookup(FromNormalName);
-  ASSERT_EQ(LookupRes.size(), 1u);
+  ASSERT_TRUE(LookupRes.isSingleResult());
   LookupRes = FromFriendTU->noload_lookup(FromFriendName);
-  ASSERT_EQ(LookupRes.size(), 1u);
+  ASSERT_TRUE(LookupRes.isSingleResult());
 
   auto *ToNormalF = cast<FunctionDecl>(Import(FromNormalF, Lang_CXX03));
   TranslationUnitDecl *ToTU = ToAST->getASTContext().getTranslationUnitDecl();
@@ -2704,12 +2737,12 @@ TEST_P(ImportFriendFunctions, ImportFriendChangesLookup) {
   EXPECT_TRUE(ToNormalF->isInIdentifierNamespace(Decl::IDNS_Ordinary));
   EXPECT_FALSE(ToNormalF->isInIdentifierNamespace(Decl::IDNS_OrdinaryFriend));
   LookupRes = ToTU->noload_lookup(ToName);
-  EXPECT_EQ(LookupRes.size(), 1u);
+  EXPECT_TRUE(LookupRes.isSingleResult());
   EXPECT_EQ(DeclCounter<FunctionDecl>().match(ToTU, Pattern), 1u);
 
   auto *ToFriendF = cast<FunctionDecl>(Import(FromFriendF, Lang_CXX03));
   LookupRes = ToTU->noload_lookup(ToName);
-  EXPECT_EQ(LookupRes.size(), 1u);
+  EXPECT_TRUE(LookupRes.isSingleResult());
   EXPECT_EQ(DeclCounter<FunctionDecl>().match(ToTU, Pattern), 2u);
 
   EXPECT_TRUE(ToNormalF->isInIdentifierNamespace(Decl::IDNS_Ordinary));
@@ -4031,11 +4064,11 @@ TEST_P(DeclContextTest,
 
   ASSERT_TRUE(L.getAsDecl());
   // Simulate the private function DeclContext::reconcileExternalVisibleStorage.
-  // The point here is to have a Vec with only one element, which is not the
-  // one we are going to delete from the DC later.
+  // We do not have a list with one element.
   L.setHasExternalDecls();
-  ASSERT_TRUE(L.getAsVector());
-  ASSERT_EQ(1u, L.getAsVector()->size());
+  ASSERT_FALSE(L.getAsList());
+  auto Results = L.getLookupResult();
+  ASSERT_EQ(1u, std::distance(Results.begin(), Results.end()));
 
   // This asserts in the old implementation.
   DC->removeDecl(A0);
@@ -5591,30 +5624,6 @@ TEST_P(ASTImporterOptionSpecificTestBase, ImportDefaultConstructibleLambdas) {
             2u);
 }
 
-TEST_P(ASTImporterOptionSpecificTestBase, ImplicitlyDeclareSelf) {
-  Decl *FromTU = getTuDecl(R"(
-                           __attribute__((objc_root_class))
-                           @interface Root
-                           @end
-                           @interface C : Root
-                             -(void)method;
-                           @end
-                           @implementation C
-                             -(void)method {}
-                           @end
-                           )",
-                           Lang_OBJCXX, "input.mm");
-  auto *FromMethod = LastDeclMatcher<ObjCMethodDecl>().match(
-      FromTU, namedDecl(hasName("method")));
-  ASSERT_TRUE(FromMethod);
-  auto ToMethod = Import(FromMethod, Lang_OBJCXX);
-  ASSERT_TRUE(ToMethod);
-
-  // Both methods should have their implicit parameters.
-  EXPECT_TRUE(FromMethod->getSelfDecl() != nullptr);
-  EXPECT_TRUE(ToMethod->getSelfDecl() != nullptr);
-}
-
 struct ImportAutoFunctions : ASTImporterOptionSpecificTestBase {};
 
 TEST_P(ImportAutoFunctions, ReturnWithTypedefDeclaredInside) {
@@ -6208,6 +6217,25 @@ TEST_P(ASTImporterOptionSpecificTestBase,
   EXPECT_TRUE(To2);
 }
 
+TEST_P(ASTImporterOptionSpecificTestBase, ImportOfCapturedVLAType) {
+  Decl *FromTU = getTuDecl(
+      R"(
+      void declToImport(int N) {
+        int VLA[N];
+        [&VLA] {}; // FieldDecl inside the lambda.
+      }
+      )",
+      Lang_CXX14);
+  auto *FromFD = FirstDeclMatcher<FieldDecl>().match(FromTU, fieldDecl());
+  ASSERT_TRUE(FromFD);
+  ASSERT_TRUE(FromFD->hasCapturedVLAType());
+
+  auto *ToFD = Import(FromFD, Lang_CXX14);
+  EXPECT_TRUE(ToFD);
+  EXPECT_TRUE(ToFD->hasCapturedVLAType());
+  EXPECT_NE(FromFD->getCapturedVLAType(), ToFD->getCapturedVLAType());
+}
+
 INSTANTIATE_TEST_CASE_P(ParameterizedTests, ASTImporterLookupTableTest,
                         DefaultTestValuesForRunOptions, );
 
diff --git a/clang/unittests/AST/CMakeLists.txt b/clang/unittests/AST/CMakeLists.txt
index 2d5d0172afedc284370b25d4a3c7a12fc4efedb7..105bfd77df9059dfe45c38e242ab8b396ebf0553 100644
--- a/clang/unittests/AST/CMakeLists.txt
+++ b/clang/unittests/AST/CMakeLists.txt
@@ -8,12 +8,12 @@ add_clang_unittest(ASTTests
   ASTContextParentMapTest.cpp
   ASTImporterFixtures.cpp
   ASTImporterTest.cpp
+  ASTImporterObjCTest.cpp
   ASTImporterGenericRedeclTest.cpp
   ASTImporterODRStrategiesTest.cpp
   ASTImporterVisibilityTest.cpp
   ASTTraverserTest.cpp
   ASTTypeTraitsTest.cpp
-  ASTTraverserTest.cpp
   ASTVectorTest.cpp
   CommentLexer.cpp
   CommentParser.cpp
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index 9909cec2065cca916b529a0ec845c41d1e212618..bf4b0912c66167cfd73d17b175b94858ed1a4500 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -510,6 +510,83 @@ void foo()
                                   varDecl(hasName("lPtrDecay"))))))));
 }
 
+TEST(Matcher, MatchesCoroutine) {
+  FileContentMappings M;
+  M.push_back(std::make_pair("/coro_header", R"cpp(
+namespace std {
+namespace experimental {
+
+template <class... Args>
+struct void_t_imp {
+  using type = void;
+};
+template <class... Args>
+using void_t = typename void_t_imp<Args...>::type;
+
+template <class T, class = void>
+struct traits_sfinae_base {};
+
+template <class T>
+struct traits_sfinae_base<T, void_t<typename T::promise_type>> {
+  using promise_type = typename T::promise_type;
+};
+
+template <class Ret, class... Args>
+struct coroutine_traits : public traits_sfinae_base<Ret> {};
+}}  // namespace std::experimental
+struct awaitable {
+  bool await_ready() noexcept;
+  template <typename F>
+  void await_suspend(F) noexcept;
+  void await_resume() noexcept;
+} a;
+struct promise {
+  void get_return_object();
+  awaitable initial_suspend();
+  awaitable final_suspend() noexcept;
+  awaitable yield_value(int); // expected-note 2{{candidate}}
+  void return_value(int); // expected-note 2{{here}}
+  void unhandled_exception();
+};
+template <typename... T>
+struct std::experimental::coroutine_traits<void, T...> { using promise_type = promise; };
+namespace std {
+namespace experimental {
+template <class PromiseType = void>
+struct coroutine_handle {
+  static coroutine_handle from_address(void *) noexcept;
+};
+}} // namespace std::experimental
+)cpp"));
+  StringRef CoReturnCode = R"cpp(
+#include <coro_header>
+void check_match_co_return() {
+  co_return 1;
+}
+)cpp";
+  EXPECT_TRUE(matchesConditionally(CoReturnCode, 
+                                   coreturnStmt(isExpansionInMainFile()), 
+                                   true, {"-std=c++20", "-I/"}, M));
+  StringRef CoAwaitCode = R"cpp(
+#include <coro_header>
+void check_match_co_await() {
+  co_await a;
+}
+)cpp";
+  EXPECT_TRUE(matchesConditionally(CoAwaitCode, 
+                                   coawaitExpr(isExpansionInMainFile()), 
+                                   true, {"-std=c++20", "-I/"}, M));
+  StringRef CoYieldCode = R"cpp(
+#include <coro_header>
+void check_match_co_yield() {
+  co_yield 1.0;
+}
+)cpp";
+  EXPECT_TRUE(matchesConditionally(CoYieldCode, 
+                                   coyieldExpr(isExpansionInMainFile()), 
+                                   true, {"-std=c++20", "-I/"}, M));
+}
+
 TEST(Matcher, isClassMessage) {
   EXPECT_TRUE(matchesObjC(
       "@interface NSString +(NSString *) stringWithFormat; @end "
diff --git a/clang/unittests/Driver/ToolChainTest.cpp b/clang/unittests/Driver/ToolChainTest.cpp
index b3ab8984d6e191a0b422cead8fa756fd3585930f..8542dd6b829b3a601f48462d89e4fea40fff199f 100644
--- a/clang/unittests/Driver/ToolChainTest.cpp
+++ b/clang/unittests/Driver/ToolChainTest.cpp
@@ -33,11 +33,8 @@ TEST(ToolChainTest, VFSGCCInstallation) {
 
   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
   struct TestDiagnosticConsumer : public DiagnosticConsumer {};
-  DiagnosticsEngine Diags(DiagID, &*DiagOpts, new TestDiagnosticConsumer);
   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem(
       new llvm::vfs::InMemoryFileSystem);
-  Driver TheDriver("/bin/clang", "arm-linux-gnueabihf", Diags,
-                   "clang LLVM compiler", InMemoryFileSystem);
 
   const char *EmptyFiles[] = {
       "foo.cpp",
@@ -55,31 +52,78 @@ TEST(ToolChainTest, VFSGCCInstallation) {
       "/usr/include/arm-linux-gnueabi/.keep",
       "/usr/include/arm-linux-gnueabihf/.keep",
       "/lib/arm-linux-gnueabi/.keep",
-      "/lib/arm-linux-gnueabihf/.keep"};
+      "/lib/arm-linux-gnueabihf/.keep",
+
+      "/sysroot/usr/lib/gcc/arm-linux-gnueabi/4.5.1/crtbegin.o",
+      "/sysroot/usr/lib/gcc/arm-linux-gnueabi/4.5.1/crtend.o",
+      "/sysroot/usr/lib/gcc/arm-linux-gnueabihf/4.5.3/crtbegin.o",
+      "/sysroot/usr/lib/gcc/arm-linux-gnueabihf/4.5.3/crtend.o",
+      "/sysroot/usr/lib/arm-linux-gnueabi/crt1.o",
+      "/sysroot/usr/lib/arm-linux-gnueabi/crti.o",
+      "/sysroot/usr/lib/arm-linux-gnueabi/crtn.o",
+      "/sysroot/usr/lib/arm-linux-gnueabihf/crt1.o",
+      "/sysroot/usr/lib/arm-linux-gnueabihf/crti.o",
+      "/sysroot/usr/lib/arm-linux-gnueabihf/crtn.o",
+      "/sysroot/usr/include/arm-linux-gnueabi/.keep",
+      "/sysroot/usr/include/arm-linux-gnueabihf/.keep",
+      "/sysroot/lib/arm-linux-gnueabi/.keep",
+      "/sysroot/lib/arm-linux-gnueabihf/.keep",
+  };
 
   for (const char *Path : EmptyFiles)
     InMemoryFileSystem->addFile(Path, 0,
                                 llvm::MemoryBuffer::getMemBuffer("\n"));
 
-  std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(
-      {"-fsyntax-only", "--gcc-toolchain=", "--sysroot=", "foo.cpp"}));
-  EXPECT_TRUE(C);
-
-  std::string S;
   {
-    llvm::raw_string_ostream OS(S);
-    C->getDefaultToolChain().printVerboseInfo(OS);
+    DiagnosticsEngine Diags(DiagID, &*DiagOpts, new TestDiagnosticConsumer);
+    Driver TheDriver("/bin/clang", "arm-linux-gnueabihf", Diags,
+                     "clang LLVM compiler", InMemoryFileSystem);
+    std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(
+        {"-fsyntax-only", "--gcc-toolchain=", "--sysroot=", "foo.cpp"}));
+    ASSERT_TRUE(C);
+    std::string S;
+    {
+      llvm::raw_string_ostream OS(S);
+      C->getDefaultToolChain().printVerboseInfo(OS);
+    }
+#if _WIN32
+    std::replace(S.begin(), S.end(), '\\', '/');
+#endif
+    EXPECT_EQ(
+        "Found candidate GCC installation: "
+        "/usr/lib/gcc/arm-linux-gnueabihf/4.6.3\n"
+        "Selected GCC installation: /usr/lib/gcc/arm-linux-gnueabihf/4.6.3\n"
+        "Candidate multilib: .;@m32\n"
+        "Selected multilib: .;@m32\n",
+        S);
   }
+
+  {
+    DiagnosticsEngine Diags(DiagID, &*DiagOpts, new TestDiagnosticConsumer);
+    Driver TheDriver("/bin/clang", "arm-linux-gnueabihf", Diags,
+                     "clang LLVM compiler", InMemoryFileSystem);
+    std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(
+        {"-fsyntax-only", "--gcc-toolchain=", "--sysroot=/sysroot",
+         "foo.cpp"}));
+    ASSERT_TRUE(C);
+    std::string S;
+    {
+      llvm::raw_string_ostream OS(S);
+      C->getDefaultToolChain().printVerboseInfo(OS);
+    }
 #if _WIN32
-  std::replace(S.begin(), S.end(), '\\', '/');
+    std::replace(S.begin(), S.end(), '\\', '/');
 #endif
-  EXPECT_EQ(
-      "Found candidate GCC installation: "
-      "/usr/lib/gcc/arm-linux-gnueabihf/4.6.3\n"
-      "Selected GCC installation: /usr/lib/gcc/arm-linux-gnueabihf/4.6.3\n"
-      "Candidate multilib: .;@m32\n"
-      "Selected multilib: .;@m32\n",
-      S);
+    // Test that 4.5.3 from --sysroot is not overridden by 4.6.3 (larger
+    // version) from /usr.
+    EXPECT_EQ("Found candidate GCC installation: "
+              "/sysroot/usr/lib/gcc/arm-linux-gnueabihf/4.5.3\n"
+              "Selected GCC installation: "
+              "/sysroot/usr/lib/gcc/arm-linux-gnueabihf/4.5.3\n"
+              "Candidate multilib: .;@m32\n"
+              "Selected multilib: .;@m32\n",
+              S);
+  }
 }
 #endif
 
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 429621e0ca1c65b30043dcaec1de707d0805658c..1dd287f71ce1c782e55e45de8ea2b7ff7987dad0 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -14297,6 +14297,102 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
                Alignment);
 }
 
+TEST_F(FormatTest, AlignWithLineBreaks) {
+  auto Style = getLLVMStyleWithColumns(120);
+
+  EXPECT_EQ(Style.AlignConsecutiveAssignments, FormatStyle::ACS_None);
+  EXPECT_EQ(Style.AlignConsecutiveDeclarations, FormatStyle::ACS_None);
+  verifyFormat("void foo() {\n"
+               "  int myVar = 5;\n"
+               "  double x = 3.14;\n"
+               "  auto str = \"Hello \"\n"
+               "             \"World\";\n"
+               "  auto s = \"Hello \"\n"
+               "           \"Again\";\n"
+               "}",
+               Style);
+
+  // clang-format off
+  verifyFormat("void foo() {\n"
+               "  const int capacityBefore = Entries.capacity();\n"
+               "  const auto newEntry = Entries.emplaceHint(std::piecewise_construct, std::forward_as_tuple(uniqueId),\n"
+               "                                            std::forward_as_tuple(id, uniqueId, name, threadCreation));\n"
+               "  const X newEntry2 = Entries.emplaceHint(std::piecewise_construct, std::forward_as_tuple(uniqueId),\n"
+               "                                          std::forward_as_tuple(id, uniqueId, name, threadCreation));\n"
+               "}",
+               Style);
+  // clang-format on
+
+  Style.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
+  verifyFormat("void foo() {\n"
+               "  int myVar = 5;\n"
+               "  double x  = 3.14;\n"
+               "  auto str  = \"Hello \"\n"
+               "              \"World\";\n"
+               "  auto s    = \"Hello \"\n"
+               "              \"Again\";\n"
+               "}",
+               Style);
+
+  // clang-format off
+  verifyFormat("void foo() {\n"
+               "  const int capacityBefore = Entries.capacity();\n"
+               "  const auto newEntry      = Entries.emplaceHint(std::piecewise_construct, std::forward_as_tuple(uniqueId),\n"
+               "                                                 std::forward_as_tuple(id, uniqueId, name, threadCreation));\n"
+               "  const X newEntry2        = Entries.emplaceHint(std::piecewise_construct, std::forward_as_tuple(uniqueId),\n"
+               "                                                 std::forward_as_tuple(id, uniqueId, name, threadCreation));\n"
+               "}",
+               Style);
+  // clang-format on
+
+  Style.AlignConsecutiveAssignments = FormatStyle::ACS_None;
+  Style.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
+  verifyFormat("void foo() {\n"
+               "  int    myVar = 5;\n"
+               "  double x = 3.14;\n"
+               "  auto   str = \"Hello \"\n"
+               "               \"World\";\n"
+               "  auto   s = \"Hello \"\n"
+               "             \"Again\";\n"
+               "}",
+               Style);
+
+  // clang-format off
+  verifyFormat("void foo() {\n"
+               "  const int  capacityBefore = Entries.capacity();\n"
+               "  const auto newEntry = Entries.emplaceHint(std::piecewise_construct, std::forward_as_tuple(uniqueId),\n"
+               "                                            std::forward_as_tuple(id, uniqueId, name, threadCreation));\n"
+               "  const X    newEntry2 = Entries.emplaceHint(std::piecewise_construct, std::forward_as_tuple(uniqueId),\n"
+               "                                             std::forward_as_tuple(id, uniqueId, name, threadCreation));\n"
+               "}",
+               Style);
+  // clang-format on
+
+  Style.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
+  Style.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
+
+  verifyFormat("void foo() {\n"
+               "  int    myVar = 5;\n"
+               "  double x     = 3.14;\n"
+               "  auto   str   = \"Hello \"\n"
+               "                 \"World\";\n"
+               "  auto   s     = \"Hello \"\n"
+               "                 \"Again\";\n"
+               "}",
+               Style);
+
+  // clang-format off
+  verifyFormat("void foo() {\n"
+               "  const int  capacityBefore = Entries.capacity();\n"
+               "  const auto newEntry       = Entries.emplaceHint(std::piecewise_construct, std::forward_as_tuple(uniqueId),\n"
+               "                                                  std::forward_as_tuple(id, uniqueId, name, threadCreation));\n"
+               "  const X    newEntry2      = Entries.emplaceHint(std::piecewise_construct, std::forward_as_tuple(uniqueId),\n"
+               "                                                  std::forward_as_tuple(id, uniqueId, name, threadCreation));\n"
+               "}",
+               Style);
+  // clang-format on
+}
+
 TEST_F(FormatTest, LinuxBraceBreaking) {
   FormatStyle LinuxBraceStyle = getLLVMStyle();
   LinuxBraceStyle.BreakBeforeBraces = FormatStyle::BS_Linux;
diff --git a/clang/unittests/Format/FormatTestObjC.cpp b/clang/unittests/Format/FormatTestObjC.cpp
index c33f93bcf99df34124fa28d1c1f70f71dddd3de0..aa27f6ecf93b6bc3b65c3633fa91423e0690d9f8 100644
--- a/clang/unittests/Format/FormatTestObjC.cpp
+++ b/clang/unittests/Format/FormatTestObjC.cpp
@@ -977,6 +977,19 @@ TEST_F(FormatTestObjC, FormatObjCMethodExpr) {
       "        performSelectorOnMainThread:@selector(loadAccessories)\n"
       "                         withObject:nil\n"
       "                      waitUntilDone:false];");
+
+  // The appropriate indentation is used after a block statement.
+  Style.ContinuationIndentWidth = 4;
+  verifyFormat(
+      "void aaaaaaaaaaaaaaaaaaaaa(int c) {\n"
+      "  if (c) {\n"
+      "    f();\n"
+      "  }\n"
+      "  [dddddddddddddddddddddddddddddddddddddddddddddddddddddddd\n"
+      "      eeeeeeeeeeeeeeeeeeeeeeeeeeeee:^(fffffffffffffff gggggggg) {\n"
+      "        f(SSSSS, c);\n"
+      "      }];\n"
+      "}");
 }
 
 TEST_F(FormatTestObjC, ObjCAt) {
diff --git a/clang/unittests/Frontend/CompilerInvocationTest.cpp b/clang/unittests/Frontend/CompilerInvocationTest.cpp
index 89e024dcb78b607cabd30dd835145d34b9825b52..b846e6ead28c91bcab86cf34f8a54f0509704c79 100644
--- a/clang/unittests/Frontend/CompilerInvocationTest.cpp
+++ b/clang/unittests/Frontend/CompilerInvocationTest.cpp
@@ -516,7 +516,8 @@ TEST_F(CommandLineTest, ConditionalParsingIfFalseFlagNotPresent) {
   CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
   ASSERT_FALSE(Diags->hasErrorOccurred());
-  ASSERT_FALSE(Invocation.getLangOpts()->SYCL);
+  ASSERT_FALSE(Invocation.getLangOpts()->SYCLIsDevice);
+  ASSERT_FALSE(Invocation.getLangOpts()->SYCLIsHost);
   ASSERT_EQ(Invocation.getLangOpts()->getSYCLVersion(), LangOptions::SYCL_None);
 
   Invocation.generateCC1CommandLine(GeneratedArgs, *this);
@@ -531,42 +532,42 @@ TEST_F(CommandLineTest, ConditionalParsingIfFalseFlagPresent) {
   CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
   ASSERT_FALSE(Diags->hasErrorOccurred());
-  ASSERT_FALSE(Invocation.getLangOpts()->SYCL);
+  ASSERT_FALSE(Invocation.getLangOpts()->SYCLIsDevice);
+  ASSERT_FALSE(Invocation.getLangOpts()->SYCLIsHost);
   ASSERT_EQ(Invocation.getLangOpts()->getSYCLVersion(), LangOptions::SYCL_None);
 
   Invocation.generateCC1CommandLine(GeneratedArgs, *this);
 
-  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-fsycl"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-fsycl-is-device"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-fsycl-is-host"))));
   ASSERT_THAT(GeneratedArgs, Not(Contains(HasSubstr("-sycl-std="))));
 }
 
 TEST_F(CommandLineTest, ConditionalParsingIfTrueFlagNotPresent) {
-  const char *Args[] = {"-fsycl"};
+  const char *Args[] = {"-fsycl-is-host"};
 
   CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
   ASSERT_FALSE(Diags->hasErrorOccurred());
-  ASSERT_TRUE(Invocation.getLangOpts()->SYCL);
   ASSERT_EQ(Invocation.getLangOpts()->getSYCLVersion(), LangOptions::SYCL_None);
 
   Invocation.generateCC1CommandLine(GeneratedArgs, *this);
 
-  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-fsycl")));
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-fsycl-is-host")));
   ASSERT_THAT(GeneratedArgs, Not(Contains(HasSubstr("-sycl-std="))));
 }
 
 TEST_F(CommandLineTest, ConditionalParsingIfTrueFlagPresent) {
-  const char *Args[] = {"-fsycl", "-sycl-std=2017"};
+  const char *Args[] = {"-fsycl-is-device", "-sycl-std=2017"};
 
   CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
   ASSERT_FALSE(Diags->hasErrorOccurred());
-  ASSERT_TRUE(Invocation.getLangOpts()->SYCL);
   ASSERT_EQ(Invocation.getLangOpts()->getSYCLVersion(), LangOptions::SYCL_2017);
 
   Invocation.generateCC1CommandLine(GeneratedArgs, *this);
 
-  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-fsycl")));
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-fsycl-is-device")));
   ASSERT_THAT(GeneratedArgs, Contains(StrEq("-sycl-std=2017")));
 }
 
diff --git a/clang/unittests/Introspection/IntrospectionTest.cpp b/clang/unittests/Introspection/IntrospectionTest.cpp
index 7977e870101fe5f5ab5d1b56f36f9338cdba7cda..5e32d7d593c74f7b38c060df36a37d70c6915973 100644
--- a/clang/unittests/Introspection/IntrospectionTest.cpp
+++ b/clang/unittests/Introspection/IntrospectionTest.cpp
@@ -40,7 +40,9 @@ FormatExpected(const MapType &Accessors) {
   return Result;
 }
 
-TEST(Introspection, SourceLocations) {
+#define STRING_LOCATION_PAIR(INSTANCE, LOC) Pair(#LOC, INSTANCE->LOC)
+
+TEST(Introspection, SourceLocations_Stmt) {
   auto AST = buildASTFromCode("void foo() {} void bar() { foo(); }", "foo.cpp",
                               std::make_shared<PCHContainerOperations>());
   auto &Ctx = AST->getASTContext();
@@ -67,14 +69,79 @@ TEST(Introspection, SourceLocations) {
 
   EXPECT_THAT(
       ExpectedLocations,
-      UnorderedElementsAre(Pair("getBeginLoc()", FooCall->getBeginLoc()),
-                           Pair("getEndLoc()", FooCall->getEndLoc()),
-                           Pair("getExprLoc()", FooCall->getExprLoc()),
-                           Pair("getRParenLoc()", FooCall->getRParenLoc())));
+      UnorderedElementsAre(STRING_LOCATION_PAIR(FooCall, getBeginLoc()),
+                           STRING_LOCATION_PAIR(FooCall, getEndLoc()),
+                           STRING_LOCATION_PAIR(FooCall, getExprLoc()),
+                           STRING_LOCATION_PAIR(FooCall, getRParenLoc())));
 
   auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
 
-  EXPECT_THAT(ExpectedRanges,
+  EXPECT_THAT(ExpectedRanges, UnorderedElementsAre(STRING_LOCATION_PAIR(
+                                  FooCall, getSourceRange())));
+}
+
+TEST(Introspection, SourceLocations_Decl) {
+  auto AST =
+      buildASTFromCode(R"cpp(
+namespace ns1 {
+namespace ns2 {
+template <typename T, typename U> struct Foo {};
+template <typename T, typename U> struct Bar {
+  struct Nested {
+    template <typename A, typename B>
+    Foo<A, B> method(int i, bool b) const noexcept(true);
+  };
+};
+} // namespace ns2
+} // namespace ns1
+
+template <typename T, typename U>
+template <typename A, typename B>
+ns1::ns2::Foo<A, B> ns1::ns2::Bar<T, U>::Nested::method(int i, bool b) const
+    noexcept(true) {}
+)cpp",
+                       "foo.cpp", std::make_shared<PCHContainerOperations>());
+  auto &Ctx = AST->getASTContext();
+  auto &TU = *Ctx.getTranslationUnitDecl();
+
+  auto BoundNodes = ast_matchers::match(
+      decl(hasDescendant(
+          cxxMethodDecl(hasName("method")).bind("method"))),
+      TU, Ctx);
+
+  EXPECT_EQ(BoundNodes.size(), 1u);
+
+  const auto *MethodDecl = BoundNodes[0].getNodeAs<CXXMethodDecl>("method");
+
+  auto Result = NodeIntrospection::GetLocations(MethodDecl);
+
+  if (Result.LocationAccessors.empty() && Result.RangeAccessors.empty()) {
+    return;
+  }
+
+  auto ExpectedLocations =
+      FormatExpected<SourceLocation>(Result.LocationAccessors);
+
+  EXPECT_THAT(ExpectedLocations,
               UnorderedElementsAre(
-                  Pair("getSourceRange()", FooCall->getSourceRange())));
+                  STRING_LOCATION_PAIR(MethodDecl, getBeginLoc()),
+                  STRING_LOCATION_PAIR(MethodDecl, getBodyRBrace()),
+                  STRING_LOCATION_PAIR(MethodDecl, getEllipsisLoc()),
+                  STRING_LOCATION_PAIR(MethodDecl, getInnerLocStart()),
+                  STRING_LOCATION_PAIR(MethodDecl, getLocation()),
+                  STRING_LOCATION_PAIR(MethodDecl, getOuterLocStart()),
+                  STRING_LOCATION_PAIR(MethodDecl, getPointOfInstantiation()),
+                  STRING_LOCATION_PAIR(MethodDecl, getTypeSpecEndLoc()),
+                  STRING_LOCATION_PAIR(MethodDecl, getTypeSpecStartLoc()),
+                  STRING_LOCATION_PAIR(MethodDecl, getEndLoc())));
+
+  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
+
+  EXPECT_THAT(
+      ExpectedRanges,
+      UnorderedElementsAre(
+          STRING_LOCATION_PAIR(MethodDecl, getExceptionSpecSourceRange()),
+          STRING_LOCATION_PAIR(MethodDecl, getParametersSourceRange()),
+          STRING_LOCATION_PAIR(MethodDecl, getReturnTypeSourceRange()),
+          STRING_LOCATION_PAIR(MethodDecl, getSourceRange())));
 }
diff --git a/clang/unittests/StaticAnalyzer/RangeSetTest.cpp b/clang/unittests/StaticAnalyzer/RangeSetTest.cpp
index 83b4fac15a19869c2a1f6b6d1d2fde4fcf9f71d6..5be2ee3fc520b5d369842d2c43a5a19895e0228b 100644
--- a/clang/unittests/StaticAnalyzer/RangeSetTest.cpp
+++ b/clang/unittests/StaticAnalyzer/RangeSetTest.cpp
@@ -11,120 +11,339 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h"
 #include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/APSInt.h"
+#include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 
+using namespace clang;
+using namespace ento;
+
 namespace clang {
 namespace ento {
-namespace {
 
-// TestCase contains to lists of ranges.
-// Original one has to be negated.
-// Expected one has to be compared to negated original range.
-template <typename T> struct TestCase {
-  RangeSet original;
-  RangeSet expected;
-
-  TestCase(BasicValueFactory &BVF, RangeSet::Factory &F,
-           const std::initializer_list<T> &originalList,
-           const std::initializer_list<T> &expectedList)
-      : original(createRangeSetFromList(BVF, F, originalList)),
-        expected(createRangeSetFromList(BVF, F, expectedList)) {}
-
-private:
-  RangeSet createRangeSetFromList(BasicValueFactory &BVF, RangeSet::Factory &F,
-                                  const std::initializer_list<T> rangeList) {
-    llvm::APSInt from(sizeof(T) * 8, std::is_unsigned<T>::value);
-    llvm::APSInt to = from;
-    RangeSet rangeSet = F.getEmptySet();
-    for (auto it = rangeList.begin(); it != rangeList.end(); it += 2) {
-      from = *it;
-      to = *(it + 1);
-      rangeSet = rangeSet.addRange(
-          F, RangeSet(F, BVF.getValue(from), BVF.getValue(to)));
-    }
-    return rangeSet;
-  }
+template <class RangeOrSet> static std::string toString(const RangeOrSet &Obj) {
+  std::string ObjRepresentation;
+  llvm::raw_string_ostream SS(ObjRepresentation);
+  Obj.dump(SS);
+  return SS.str();
+}
+LLVM_ATTRIBUTE_UNUSED static std::string toString(const llvm::APSInt &Point) {
+  return Point.toString(10);
+}
+// We need it here for better fail diagnostics from gtest.
+LLVM_ATTRIBUTE_UNUSED static std::ostream &operator<<(std::ostream &OS,
+                                                      const RangeSet &Set) {
+  return OS << toString(Set);
+}
 
-  void printNegate(const TestCase &TestCase) {
-    TestCase.original.print(llvm::dbgs());
-    llvm::dbgs() << " => ";
-    TestCase.expected.print(llvm::dbgs());
-  }
-};
+} // namespace ento
+} // namespace clang
 
-class RangeSetTest : public testing::Test {
-protected:
+namespace {
+
+template <typename BaseType> class RangeSetTest : public testing::Test {
+public:
   // Init block
   std::unique_ptr<ASTUnit> AST = tooling::buildASTFromCode("struct foo;");
-  ASTContext &context = AST->getASTContext();
-  llvm::BumpPtrAllocator alloc;
-  BasicValueFactory BVF{context, alloc};
-  RangeSet::Factory F;
+  ASTContext &Context = AST->getASTContext();
+  llvm::BumpPtrAllocator Arena;
+  BasicValueFactory BVF{Context, Arena};
+  RangeSet::Factory F{BVF};
   // End init block
 
-  template <typename T> void checkNegate() {
-    using type = T;
-
-    // Use next values of the range {MIN, A, B, MID, C, D, MAX}.
-
-    // MID is a value in the middle of the range
-    // which unary minus does not affect on,
-    // e.g. int8/int32(0), uint8(128), uint32(2147483648).
-
-    constexpr type MIN = std::numeric_limits<type>::min();
-    constexpr type MAX = std::numeric_limits<type>::max();
-    constexpr type MID = std::is_signed<type>::value
-                             ? 0
-                             : ~(static_cast<type>(-1) / static_cast<type>(2));
-    constexpr type A = MID - static_cast<type>(42 + 42);
-    constexpr type B = MID - static_cast<type>(42);
-    constexpr type C = -B;
-    constexpr type D = -A;
-
-    static_assert(MIN < A && A < B && B < MID && MID < C && C < D && D < MAX,
-                  "Values shall be in an ascending order");
-
-    // Left {[x, y], [x, y]} is what shall be negated.
-    // Right {[x, y], [x, y]} is what shall be compared to a negation result.
-    TestCase<type> cases[] = {
-        {BVF, F, {MIN, A}, {MIN, MIN, D, MAX}},
-        {BVF, F, {MIN, C}, {MIN, MIN, B, MAX}},
-        {BVF, F, {MIN, MID}, {MIN, MIN, MID, MAX}},
-        {BVF, F, {MIN, MAX}, {MIN, MAX}},
-        {BVF, F, {A, D}, {A, D}},
-        {BVF, F, {A, B}, {C, D}},
-        {BVF, F, {MIN, A, D, MAX}, {MIN, A, D, MAX}},
-        {BVF, F, {MIN, B, MID, D}, {MIN, MIN, A, MID, C, MAX}},
-        {BVF, F, {MIN, MID, C, D}, {MIN, MIN, A, B, MID, MAX}},
-        {BVF, F, {MIN, MID, C, MAX}, {MIN, B, MID, MAX}},
-        {BVF, F, {A, MID, D, MAX}, {MIN + 1, A, MID, D}},
-        {BVF, F, {A, A}, {D, D}},
-        {BVF, F, {MID, MID}, {MID, MID}},
-        {BVF, F, {MAX, MAX}, {MIN + 1, MIN + 1}},
-    };
-
-    for (const auto &c : cases) {
-      // Negate original and check with expected.
-      RangeSet negatedFromOriginal = c.original.Negate(BVF, F);
-      EXPECT_EQ(negatedFromOriginal, c.expected);
-      // Negate negated back and check with original.
-      RangeSet negatedBackward = negatedFromOriginal.Negate(BVF, F);
-      EXPECT_EQ(negatedBackward, c.original);
+  using Self = RangeSetTest<BaseType>;
+  using RawRange = std::pair<BaseType, BaseType>;
+  using RawRangeSet = std::initializer_list<RawRange>;
+
+  static constexpr BaseType getMin() {
+    return std::numeric_limits<BaseType>::min();
+  }
+  static constexpr BaseType getMax() {
+    return std::numeric_limits<BaseType>::max();
+  }
+  static constexpr BaseType getMid() {
+    return isSigned() ? 0 : ~(fromInt(-1) / fromInt(2));
+  }
+
+  static constexpr bool isSigned() { return std::is_signed<BaseType>::value; }
+  static constexpr BaseType fromInt(int X) { return static_cast<BaseType>(X); }
+
+  static llvm::APSInt Base;
+  const llvm::APSInt &from(BaseType X) {
+    llvm::APSInt Dummy = Base;
+    Dummy = X;
+    return BVF.getValue(Dummy);
+  }
+
+  Range from(const RawRange &Init) {
+    return Range(from(Init.first), from(Init.second));
+  }
+
+  RangeSet from(const RawRangeSet &Init) {
+    RangeSet RangeSet = F.getEmptySet();
+    for (const auto &Raw : Init) {
+      RangeSet = F.add(RangeSet, from(Raw));
     }
+    return RangeSet;
+  }
+
+  template <class F, class... RawArgTypes>
+  void wrap(F ActualFunction, RawArgTypes &&... Args) {
+    (this->*ActualFunction)(from(std::forward<RawArgTypes>(Args))...);
+  }
+
+  void checkNegateImpl(RangeSet Original, RangeSet Expected) {
+    RangeSet NegatedFromOriginal = F.negate(Original);
+    EXPECT_EQ(NegatedFromOriginal, Expected);
+    // Negate negated back and check with original.
+    RangeSet NegatedBackward = F.negate(NegatedFromOriginal);
+    EXPECT_EQ(NegatedBackward, Original);
+  }
+
+  void checkNegate(RawRangeSet RawOriginal, RawRangeSet RawExpected) {
+    wrap(&Self::checkNegateImpl, RawOriginal, RawExpected);
+  }
+
+  template <class PointOrSet>
+  void checkIntersectImpl(RangeSet LHS, PointOrSet RHS, RangeSet Expected) {
+    RangeSet Result = F.intersect(LHS, RHS);
+    EXPECT_EQ(Result, Expected)
+        << "while intersecting " << toString(LHS) << " and " << toString(RHS);
+  }
+
+  void checkIntersectRangeImpl(RangeSet LHS, const llvm::APSInt &Lower,
+                               const llvm::APSInt &Upper, RangeSet Expected) {
+    RangeSet Result = F.intersect(LHS, Lower, Upper);
+    EXPECT_EQ(Result, Expected)
+        << "while intersecting " << toString(LHS) << " and [" << toString(Lower)
+        << ", " << toString(Upper) << "]";
+  }
+
+  void checkIntersect(RawRangeSet RawLHS, RawRangeSet RawRHS,
+                      RawRangeSet RawExpected) {
+    wrap(&Self::checkIntersectImpl<RangeSet>, RawLHS, RawRHS, RawExpected);
+  }
+
+  void checkIntersect(RawRangeSet RawLHS, BaseType RawRHS,
+                      RawRangeSet RawExpected) {
+    wrap(&Self::checkIntersectImpl<const llvm::APSInt &>, RawLHS, RawRHS,
+         RawExpected);
+  }
+
+  void checkIntersect(RawRangeSet RawLHS, BaseType RawLower, BaseType RawUpper,
+                      RawRangeSet RawExpected) {
+    wrap(&Self::checkIntersectRangeImpl, RawLHS, RawLower, RawUpper,
+         RawExpected);
+  }
+
+  void checkContainsImpl(RangeSet LHS, const llvm::APSInt &RHS, bool Expected) {
+    bool Result = LHS.contains(RHS);
+    EXPECT_EQ(Result, Expected)
+        << toString(LHS) << (Result ? " contains " : " doesn't contain ")
+        << toString(RHS);
+  }
+
+  void checkContains(RawRangeSet RawLHS, BaseType RawRHS, bool Expected) {
+    checkContainsImpl(from(RawLHS), from(RawRHS), Expected);
+  }
+
+  template <class RHSType>
+  void checkAddImpl(RangeSet LHS, RHSType RHS, RangeSet Expected) {
+    RangeSet Result = F.add(LHS, RHS);
+    EXPECT_EQ(Result, Expected)
+        << "while adding " << toString(LHS) << " and " << toString(RHS);
+  }
+
+  void checkAdd(RawRangeSet RawLHS, RawRange RawRHS, RawRangeSet RawExpected) {
+    wrap(&Self::checkAddImpl<Range>, RawLHS, RawRHS, RawExpected);
+  }
+
+  void checkAdd(RawRangeSet RawLHS, RawRangeSet RawRHS,
+                RawRangeSet RawExpected) {
+    wrap(&Self::checkAddImpl<RangeSet>, RawRHS, RawLHS, RawExpected);
+  }
+
+  void checkAdd(RawRangeSet RawLHS, BaseType RawRHS, RawRangeSet RawExpected) {
+    wrap(&Self::checkAddImpl<const llvm::APSInt &>, RawLHS, RawRHS,
+         RawExpected);
+  }
+
+  void checkDeleteImpl(const llvm::APSInt &Point, RangeSet From,
+                       RangeSet Expected) {
+    RangeSet Result = F.deletePoint(From, Point);
+    EXPECT_EQ(Result, Expected)
+        << "while deleting " << toString(Point) << " from " << toString(From);
+  }
+
+  void checkDelete(BaseType Point, RawRangeSet RawFrom,
+                   RawRangeSet RawExpected) {
+    wrap(&Self::checkDeleteImpl, Point, RawFrom, RawExpected);
   }
 };
 
-TEST_F(RangeSetTest, RangeSetNegateTest) {
-  checkNegate<int8_t>();
-  checkNegate<uint8_t>();
-  checkNegate<int16_t>();
-  checkNegate<uint16_t>();
-  checkNegate<int32_t>();
-  checkNegate<uint32_t>();
-  checkNegate<int64_t>();
-  checkNegate<uint64_t>();
+} // namespace
+
+template <typename BaseType>
+llvm::APSInt RangeSetTest<BaseType>::Base{sizeof(BaseType) * 8, !isSigned()};
+
+using IntTypes = ::testing::Types<int8_t, uint8_t, int16_t, uint16_t, int32_t,
+                                  uint32_t, int64_t, uint64_t>;
+TYPED_TEST_CASE(RangeSetTest, IntTypes);
+
+TYPED_TEST(RangeSetTest, RangeSetNegateTest) {
+  // Use next values of the range {MIN, A, B, MID, C, D, MAX}.
+
+  constexpr TypeParam MIN = TestFixture::getMin();
+  constexpr TypeParam MAX = TestFixture::getMax();
+  // MID is a value in the middle of the range
+  // which unary minus does not affect on,
+  // e.g. int8/int32(0), uint8(128), uint32(2147483648).
+  constexpr TypeParam MID = TestFixture::getMid();
+  constexpr TypeParam A = MID - TestFixture::fromInt(42 + 42);
+  constexpr TypeParam B = MID - TestFixture::fromInt(42);
+  constexpr TypeParam C = -B;
+  constexpr TypeParam D = -A;
+
+  static_assert(MIN < A && A < B && B < MID && MID < C && C < D && D < MAX,
+                "Values shall be in an ascending order");
+
+  this->checkNegate({{MIN, A}}, {{MIN, MIN}, {D, MAX}});
+  this->checkNegate({{MIN, C}}, {{MIN, MIN}, {B, MAX}});
+  this->checkNegate({{MIN, MID}}, {{MIN, MIN}, {MID, MAX}});
+  this->checkNegate({{MIN, MAX}}, {{MIN, MAX}});
+  this->checkNegate({{A, D}}, {{A, D}});
+  this->checkNegate({{A, B}}, {{C, D}});
+  this->checkNegate({{MIN, A}, {D, MAX}}, {{MIN, A}, {D, MAX}});
+  this->checkNegate({{MIN, B}, {MID, D}}, {{MIN, MIN}, {A, MID}, {C, MAX}});
+  this->checkNegate({{MIN, MID}, {C, D}}, {{MIN, MIN}, {A, B}, {MID, MAX}});
+  this->checkNegate({{MIN, MID}, {C, MAX}}, {{MIN, B}, {MID, MAX}});
+  this->checkNegate({{A, MID}, {D, MAX}}, {{MIN + 1, A}, {MID, D}});
+  this->checkNegate({{A, A}}, {{D, D}});
+  this->checkNegate({{MID, MID}}, {{MID, MID}});
+  this->checkNegate({{MAX, MAX}}, {{MIN + 1, MIN + 1}});
 }
 
-} // namespace
-} // namespace ento
-} // namespace clang
+TYPED_TEST(RangeSetTest, RangeSetPointIntersectTest) {
+  // Check that we can correctly intersect empty sets.
+  this->checkIntersect({}, 42, {});
+  // Check that intersection with itself produces the same set.
+  this->checkIntersect({{42, 42}}, 42, {{42, 42}});
+  // Check more general cases.
+  this->checkIntersect({{0, 10}, {20, 30}, {30, 40}, {50, 60}}, 42, {});
+  this->checkIntersect({{0, 10}, {20, 30}, {30, 60}}, 42, {{42, 42}});
+}
+
+TYPED_TEST(RangeSetTest, RangeSetRangeIntersectTest) {
+  constexpr TypeParam MIN = TestFixture::getMin();
+  constexpr TypeParam MAX = TestFixture::getMax();
+
+  // Check that we can correctly intersect empty sets.
+  this->checkIntersect({}, 10, 20, {});
+  this->checkIntersect({}, 20, 10, {});
+  // Check that intersection with itself produces the same set.
+  this->checkIntersect({{10, 20}}, 10, 20, {{10, 20}});
+  this->checkIntersect({{MIN, 10}, {20, MAX}}, 20, 10, {{MIN, 10}, {20, MAX}});
+  // Check non-overlapping range intersections.
+  this->checkIntersect({{10, 20}}, 21, 9, {});
+  this->checkIntersect({{MIN, 9}, {21, MAX}}, 10, 20, {});
+  // Check more general cases.
+  this->checkIntersect({{0, 10}, {20, 30}, {30, 40}, {50, 60}}, 10, 35,
+                       {{10, 10}, {20, 30}, {30, 35}});
+  this->checkIntersect({{0, 10}, {20, 30}, {30, 40}, {50, 60}}, 35, 10,
+                       {{0, 10}, {35, 40}, {50, 60}});
+}
+
+TYPED_TEST(RangeSetTest, RangeSetGenericIntersectTest) {
+  // Check that we can correctly intersect empty sets.
+  this->checkIntersect({}, {}, {});
+  this->checkIntersect({}, {{0, 10}}, {});
+  this->checkIntersect({{0, 10}}, {}, {});
+
+  this->checkIntersect({{0, 10}}, {{4, 6}}, {{4, 6}});
+  this->checkIntersect({{0, 10}}, {{4, 20}}, {{4, 10}});
+  // Check that intersection with points works as expected.
+  this->checkIntersect({{0, 10}}, {{4, 4}}, {{4, 4}});
+  // All ranges are closed, check that intersection with edge points works as
+  // expected.
+  this->checkIntersect({{0, 10}}, {{10, 10}}, {{10, 10}});
+
+  // Let's check that we can skip some intervals and partially intersect
+  // other intervals.
+  this->checkIntersect({{0, 2}, {4, 5}, {6, 9}, {10, 11}, {12, 12}, {13, 15}},
+                       {{8, 14}, {20, 30}},
+                       {{8, 9}, {10, 11}, {12, 12}, {13, 14}});
+  // Check more generic case.
+  this->checkIntersect(
+      {{0, 1}, {2, 3}, {5, 6}, {7, 15}, {25, 30}},
+      {{4, 10}, {11, 11}, {12, 16}, {17, 17}, {19, 20}, {21, 23}, {24, 27}},
+      {{5, 6}, {7, 10}, {11, 11}, {12, 15}, {25, 27}});
+}
+
+TYPED_TEST(RangeSetTest, RangeSetContainsTest) {
+  // Check with an empty set.
+  this->checkContains({}, 10, false);
+  // Check contains with sets of size one:
+  //   * when the whole range is less
+  this->checkContains({{0, 5}}, 10, false);
+  //   * when the whole range is greater
+  this->checkContains({{20, 25}}, 10, false);
+  //   * when the range is just the point we are looking for
+  this->checkContains({{10, 10}}, 10, true);
+  //   * when the range starts with the point
+  this->checkContains({{10, 15}}, 10, true);
+  //   * when the range ends with the point
+  this->checkContains({{5, 10}}, 10, true);
+  //   * when the range has the point somewhere in the middle
+  this->checkContains({{0, 25}}, 10, true);
+  // Check similar cases, but with larger sets.
+  this->checkContains({{0, 5}, {10, 10}, {15, 20}}, 10, true);
+  this->checkContains({{0, 5}, {10, 12}, {15, 20}}, 10, true);
+  this->checkContains({{0, 5}, {5, 7}, {8, 10}, {12, 41}}, 10, true);
+
+  constexpr TypeParam MIN = TestFixture::getMin();
+  constexpr TypeParam MAX = TestFixture::getMax();
+  constexpr TypeParam MID = TestFixture::getMid();
+  this->checkContains({{MIN, MAX}}, 0, true);
+  this->checkContains({{MIN, MAX}}, MID, true);
+  this->checkContains({{MIN, MAX}}, -10, true);
+  this->checkContains({{MIN, MAX}}, 10, true);
+}
+
+TYPED_TEST(RangeSetTest, RangeSetAddTest) {
+  // Check adding single points
+  this->checkAdd({}, 10, {{10, 10}});
+  this->checkAdd({{0, 5}}, 10, {{0, 5}, {10, 10}});
+  this->checkAdd({{0, 5}, {30, 40}}, 10, {{0, 5}, {10, 10}, {30, 40}});
+
+  // Check adding single ranges.
+  this->checkAdd({}, {10, 20}, {{10, 20}});
+  this->checkAdd({{0, 5}}, {10, 20}, {{0, 5}, {10, 20}});
+  this->checkAdd({{0, 5}, {30, 40}}, {10, 20}, {{0, 5}, {10, 20}, {30, 40}});
+
+  // Check adding whole sets of ranges.
+  this->checkAdd({{0, 5}}, {{10, 20}}, {{0, 5}, {10, 20}});
+  // Check that ordering of ranges is as expected.
+  this->checkAdd({{0, 5}, {30, 40}}, {{10, 20}}, {{0, 5}, {10, 20}, {30, 40}});
+  this->checkAdd({{0, 5}, {30, 40}}, {{10, 20}, {50, 60}},
+                 {{0, 5}, {10, 20}, {30, 40}, {50, 60}});
+  this->checkAdd({{10, 20}, {50, 60}}, {{0, 5}, {30, 40}, {70, 80}},
+                 {{0, 5}, {10, 20}, {30, 40}, {50, 60}, {70, 80}});
+}
+
+TYPED_TEST(RangeSetTest, RangeSetDeletePointTest) {
+  constexpr TypeParam MIN = TestFixture::getMin();
+  constexpr TypeParam MAX = TestFixture::getMax();
+  constexpr TypeParam MID = TestFixture::getMid();
+
+  this->checkDelete(MID, {{MIN, MAX}}, {{MIN, MID - 1}, {MID + 1, MAX}});
+  // Check that delete works with an empty set.
+  this->checkDelete(10, {}, {});
+  // Check that delete can remove entire ranges.
+  this->checkDelete(10, {{10, 10}}, {});
+  this->checkDelete(10, {{0, 5}, {10, 10}, {20, 30}}, {{0, 5}, {20, 30}});
+  // Check that delete can split existing ranges into two.
+  this->checkDelete(10, {{0, 5}, {7, 15}, {20, 30}},
+                    {{0, 5}, {7, 9}, {11, 15}, {20, 30}});
+  // Check that delete of the point not from the range set works as expected.
+  this->checkDelete(10, {{0, 5}, {20, 30}}, {{0, 5}, {20, 30}});
+}
diff --git a/clang/unittests/Tooling/CompilationDatabaseTest.cpp b/clang/unittests/Tooling/CompilationDatabaseTest.cpp
index ba40a7a643c421c2882f47aee5cc08b7a4fc93a3..8ff3387d3c1854c878ffbced58401e1e9a5f3fc3 100644
--- a/clang/unittests/Tooling/CompilationDatabaseTest.cpp
+++ b/clang/unittests/Tooling/CompilationDatabaseTest.cpp
@@ -725,14 +725,14 @@ class InterpolateTest : public MemDBTest {
 protected:
   // Look up the command from a relative path, and return it in string form.
   // The input file is not included in the returned command.
-  std::string getCommand(llvm::StringRef F) {
+  std::string getCommand(llvm::StringRef F, bool MakeNative = true) {
     auto Results =
         inferMissingCompileCommands(std::make_unique<MemCDB>(Entries))
-            ->getCompileCommands(path(F));
+            ->getCompileCommands(MakeNative ? path(F) : F);
     if (Results.empty())
       return "none";
     // drop the input file argument, so tests don't have to deal with path().
-    EXPECT_EQ(Results[0].CommandLine.back(), path(F))
+    EXPECT_EQ(Results[0].CommandLine.back(), MakeNative ? path(F) : F)
         << "Last arg should be the file";
     Results[0].CommandLine.pop_back();
     return llvm::join(Results[0].CommandLine, " ");
@@ -812,6 +812,28 @@ TEST_F(InterpolateTest, Strip) {
   EXPECT_EQ(getCommand("dir/bar.cpp"), "clang -D dir/foo.cpp -Wall");
 }
 
+TEST_F(InterpolateTest, StripDoubleDash) {
+  add("dir/foo.cpp", "-o foo.o -std=c++14 -Wall -- dir/foo.cpp");
+  // input file and output option are removed
+  // -Wall flag isn't
+  // -std option gets re-added as the last argument before the input file
+  // -- is removed as it's not necessary - the new input file doesn't start with
+  // a dash
+  EXPECT_EQ(getCommand("dir/bar.cpp"), "clang -D dir/foo.cpp -Wall -std=c++14");
+}
+
+TEST_F(InterpolateTest, InsertDoubleDash) {
+  add("dir/foo.cpp", "-o foo.o -std=c++14 -Wall");
+  EXPECT_EQ(getCommand("-dir/bar.cpp", false),
+            "clang -D dir/foo.cpp -Wall -std=c++14 --");
+}
+
+TEST_F(InterpolateTest, InsertDoubleDashForClangCL) {
+  add("dir/foo.cpp", "clang-cl", "/std:c++14 /W4");
+  EXPECT_EQ(getCommand("/dir/bar.cpp", false),
+            "clang-cl -D dir/foo.cpp /W4 /std:c++14 --");
+}
+
 TEST_F(InterpolateTest, Case) {
   add("FOO/BAR/BAZ/SHOUT.cc");
   add("foo/bar/baz/quiet.cc");
@@ -831,7 +853,7 @@ TEST_F(InterpolateTest, ClangCL) {
   add("foo.cpp", "clang-cl", "/W4");
 
   // Language flags should be added with CL syntax.
-  EXPECT_EQ(getCommand("foo.h"), "clang-cl -D foo.cpp /W4 /TP");
+  EXPECT_EQ(getCommand("foo.h", false), "clang-cl -D foo.cpp /W4 /TP");
 }
 
 TEST_F(InterpolateTest, DriverModes) {
@@ -839,8 +861,10 @@ TEST_F(InterpolateTest, DriverModes) {
   add("bar.cpp", "clang", "--driver-mode=cl");
 
   // --driver-mode overrides should be respected.
-  EXPECT_EQ(getCommand("foo.h"), "clang-cl -D foo.cpp --driver-mode=gcc -x c++-header");
-  EXPECT_EQ(getCommand("bar.h"), "clang -D bar.cpp --driver-mode=cl /TP");
+  EXPECT_EQ(getCommand("foo.h"),
+            "clang-cl -D foo.cpp --driver-mode=gcc -x c++-header");
+  EXPECT_EQ(getCommand("bar.h", false),
+            "clang -D bar.cpp --driver-mode=cl /TP");
 }
 
 TEST(TransferCompileCommandTest, Smoke) {
diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp
index 6a21be632d426b1273ddceb07f9f0ee08ddd3cc1..1768529e0a6207f72b93871dc55a990d04d19ce7 100644
--- a/clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -106,6 +106,7 @@ public:
       void EndSourceFileAction() override {
         assert(Collector && "BeginSourceFileAction was never called");
         Result = std::move(*Collector).consume();
+        Result.indexExpandedTokens();
       }
 
       std::unique_ptr<ASTConsumer>
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index aaef538e9bf983900b916deef7b531ddd343a23f..e74df36899d43564bb51c3e0d5b8fc304e561417 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -1828,6 +1828,22 @@ struct PragmaClangAttributeSupport {
 
 } // end anonymous namespace
 
+static bool isSupportedPragmaClangAttributeSubject(const Record &Subject) {
+  // FIXME: #pragma clang attribute does not currently support statement
+  // attributes, so test whether the subject is one that appertains to a
+  // declaration node. However, it may be reasonable for support for statement
+  // attributes to be added.
+  if (Subject.isSubClassOf("DeclNode") || Subject.isSubClassOf("DeclBase") ||
+      Subject.getName() == "DeclBase")
+    return true;
+
+  if (Subject.isSubClassOf("SubsetSubject"))
+    return isSupportedPragmaClangAttributeSubject(
+        *Subject.getValueAsDef("Base"));
+
+  return false;
+}
+
 static bool doesDeclDeriveFrom(const Record *D, const Record *Base) {
   const Record *CurrentBase = D->getValueAsOptionalDef(BaseFieldName);
   if (!CurrentBase)
@@ -1949,13 +1965,15 @@ bool PragmaClangAttributeSupport::isAttributedSupported(
     return false;
   const Record *SubjectObj = Attribute.getValueAsDef("Subjects");
   std::vector<Record *> Subjects = SubjectObj->getValueAsListOfDefs("Subjects");
-  if (Subjects.empty())
-    return false;
+  bool HasAtLeastOneValidSubject = false;
   for (const auto *Subject : Subjects) {
+    if (!isSupportedPragmaClangAttributeSubject(*Subject))
+      continue;
     if (SubjectsToRules.find(Subject) == SubjectsToRules.end())
       return false;
+    HasAtLeastOneValidSubject = true;
   }
-  return true;
+  return HasAtLeastOneValidSubject;
 }
 
 static std::string GenerateTestExpression(ArrayRef<Record *> LangOpts) {
@@ -2001,6 +2019,8 @@ PragmaClangAttributeSupport::generateStrictConformsTo(const Record &Attr,
   const Record *SubjectObj = Attr.getValueAsDef("Subjects");
   std::vector<Record *> Subjects = SubjectObj->getValueAsListOfDefs("Subjects");
   for (const auto *Subject : Subjects) {
+    if (!isSupportedPragmaClangAttributeSubject(*Subject))
+      continue;
     auto It = SubjectsToRules.find(Subject);
     assert(It != SubjectsToRules.end() &&
            "This attribute is unsupported by #pragma clang attribute");
@@ -3503,7 +3523,7 @@ static void GenerateAppertainsTo(const Record &Attr, raw_ostream &OS) {
     return;
 
   const Record *SubjectObj = Attr.getValueAsDef("Subjects");
-  std::vector<Record*> Subjects = SubjectObj->getValueAsListOfDefs("Subjects");
+  std::vector<Record *> Subjects = SubjectObj->getValueAsListOfDefs("Subjects");
 
   // If the list of subjects is empty, it is assumed that the attribute
   // appertains to everything.
@@ -3512,42 +3532,99 @@ static void GenerateAppertainsTo(const Record &Attr, raw_ostream &OS) {
 
   bool Warn = SubjectObj->getValueAsDef("Diag")->getValueAsBit("Warn");
 
-  // Otherwise, generate an appertainsTo check specific to this attribute which
-  // checks all of the given subjects against the Decl passed in.
-  //
-  // If D is null, that means the attribute was not applied to a declaration
-  // at all (for instance because it was applied to a type), or that the caller
-  // has determined that the check should fail (perhaps prior to the creation
-  // of the declaration).
-  OS << "bool diagAppertainsToDecl(Sema &S, ";
-  OS << "const ParsedAttr &Attr, const Decl *D) const override {\n";
-  OS << "  if (";
-  for (auto I = Subjects.begin(), E = Subjects.end(); I != E; ++I) {
-    // If the subject has custom code associated with it, use the generated
-    // function for it. The function cannot be inlined into this check (yet)
-    // because it requires the subject to be of a specific type, and were that
-    // information inlined here, it would not support an attribute with multiple
-    // custom subjects.
-    if ((*I)->isSubClassOf("SubsetSubject")) {
-      OS << "!" << functionNameForCustomAppertainsTo(**I) << "(D)";
-    } else {
-      OS << "!isa<" << GetSubjectWithSuffix(*I) << ">(D)";
+  // Split the subjects into declaration subjects and statement subjects.
+  // FIXME: subset subjects are added to the declaration list until there are
+  // enough statement attributes with custom subject needs to warrant
+  // the implementation effort.
+  std::vector<Record *> DeclSubjects, StmtSubjects;
+  llvm::copy_if(
+      Subjects, std::back_inserter(DeclSubjects), [](const Record *R) {
+        return R->isSubClassOf("SubsetSubject") || !R->isSubClassOf("StmtNode");
+      });
+  llvm::copy_if(Subjects, std::back_inserter(StmtSubjects),
+                [](const Record *R) { return R->isSubClassOf("StmtNode"); });
+
+  // We should have sorted all of the subjects into two lists.
+  // FIXME: this assertion will be wrong if we ever add type attribute subjects.
+  assert(DeclSubjects.size() + StmtSubjects.size() == Subjects.size());
+
+  if (DeclSubjects.empty()) {
+    // If there are no decl subjects but there are stmt subjects, diagnose
+    // trying to apply a statement attribute to a declaration.
+    if (!StmtSubjects.empty()) {
+      OS << "bool diagAppertainsToDecl(Sema &S, const ParsedAttr &AL, ";
+      OS << "const Decl *D) const override {\n";
+      OS << "  S.Diag(AL.getLoc(), diag::err_stmt_attribute_invalid_on_decl)\n";
+      OS << "    << AL << D->getLocation();\n";
+      OS << "  return false;\n";
+      OS << "}\n\n";
     }
+  } else {
+    // Otherwise, generate an appertainsTo check specific to this attribute
+    // which checks all of the given subjects against the Decl passed in.
+    OS << "bool diagAppertainsToDecl(Sema &S, ";
+    OS << "const ParsedAttr &Attr, const Decl *D) const override {\n";
+    OS << "  if (";
+    for (auto I = DeclSubjects.begin(), E = DeclSubjects.end(); I != E; ++I) {
+      // If the subject has custom code associated with it, use the generated
+      // function for it. The function cannot be inlined into this check (yet)
+      // because it requires the subject to be of a specific type, and were that
+      // information inlined here, it would not support an attribute with
+      // multiple custom subjects.
+      if ((*I)->isSubClassOf("SubsetSubject"))
+        OS << "!" << functionNameForCustomAppertainsTo(**I) << "(D)";
+      else
+        OS << "!isa<" << GetSubjectWithSuffix(*I) << ">(D)";
 
-    if (I + 1 != E)
-      OS << " && ";
+      if (I + 1 != E)
+        OS << " && ";
+    }
+    OS << ") {\n";
+    OS << "    S.Diag(Attr.getLoc(), diag::";
+    OS << (Warn ? "warn_attribute_wrong_decl_type_str"
+                : "err_attribute_wrong_decl_type_str");
+    OS << ")\n";
+    OS << "      << Attr << ";
+    OS << CalculateDiagnostic(*SubjectObj) << ";\n";
+    OS << "    return false;\n";
+    OS << "  }\n";
+    OS << "  return true;\n";
+    OS << "}\n\n";
+  }
+
+  if (StmtSubjects.empty()) {
+    // If there are no stmt subjects but there are decl subjects, diagnose
+    // trying to apply a declaration attribute to a statement.
+    if (!DeclSubjects.empty()) {
+      OS << "bool diagAppertainsToStmt(Sema &S, const ParsedAttr &AL, ";
+      OS << "const Stmt *St) const override {\n";
+      OS << "  S.Diag(AL.getLoc(), diag::err_decl_attribute_invalid_on_stmt)\n";
+      OS << "    << AL << St->getBeginLoc();\n";
+      OS << "  return false;\n";
+      OS << "}\n\n";
+    }
+  } else {
+    // Now, do the same for statements.
+    OS << "bool diagAppertainsToStmt(Sema &S, ";
+    OS << "const ParsedAttr &Attr, const Stmt *St) const override {\n";
+    OS << "  if (";
+    for (auto I = StmtSubjects.begin(), E = StmtSubjects.end(); I != E; ++I) {
+      OS << "!isa<" << (*I)->getName() << ">(St)";
+      if (I + 1 != E)
+        OS << " && ";
+    }
+    OS << ") {\n";
+    OS << "    S.Diag(Attr.getLoc(), diag::";
+    OS << (Warn ? "warn_attribute_wrong_decl_type_str"
+                : "err_attribute_wrong_decl_type_str");
+    OS << ")\n";
+    OS << "      << Attr << ";
+    OS << CalculateDiagnostic(*SubjectObj) << ";\n";
+    OS << "    return false;\n";
+    OS << "  }\n";
+    OS << "  return true;\n";
+    OS << "}\n\n";
   }
-  OS << ") {\n";
-  OS << "    S.Diag(Attr.getLoc(), diag::";
-  OS << (Warn ? "warn_attribute_wrong_decl_type_str" :
-               "err_attribute_wrong_decl_type_str");
-  OS << ")\n";
-  OS << "      << Attr << ";
-  OS << CalculateDiagnostic(*SubjectObj) << ";\n";
-  OS << "    return false;\n";
-  OS << "  }\n";
-  OS << "  return true;\n";
-  OS << "}\n\n";
 }
 
 static void
@@ -4214,9 +4291,13 @@ void EmitTestPragmaAttributeSupportedAttributes(RecordKeeper &Records,
     std::vector<Record *> Subjects =
         SubjectObj->getValueAsListOfDefs("Subjects");
     OS << " (";
+    bool PrintComma = false;
     for (const auto &Subject : llvm::enumerate(Subjects)) {
-      if (Subject.index())
+      if (!isSupportedPragmaClangAttributeSubject(*Subject.value()))
+        continue;
+      if (PrintComma)
         OS << ", ";
+      PrintComma = true;
       PragmaClangAttributeSupport::RuleOrAggregateRuleSet &RuleSet =
           Support.SubjectsToRules.find(Subject.value())->getSecond();
       if (RuleSet.isRule()) {
diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp
index e9ae08ac4c051a52ff61dfc56715595fa6fe1144..091af2dc52a1d997ef91ea075562131de8f60857 100644
--- a/clang/utils/TableGen/MveEmitter.cpp
+++ b/clang/utils/TableGen/MveEmitter.cpp
@@ -1272,6 +1272,13 @@ Result::Ptr EmitterBase::getCodeForDagArg(DagInit *D, unsigned ArgNum,
     return it->second;
   }
 
+  // Sometimes the Arg is a bit. Prior to multiclass template argument
+  // checking, integers would sneak through the bit declaration,
+  // but now they really are bits.
+  if (auto *BI = dyn_cast<BitInit>(Arg))
+    return std::make_shared<IntLiteralResult>(getScalarType("u32"),
+                                              BI->getValue());
+
   if (auto *II = dyn_cast<IntInit>(Arg))
     return std::make_shared<IntLiteralResult>(getScalarType("u32"),
                                               II->getValue());
@@ -1287,7 +1294,11 @@ Result::Ptr EmitterBase::getCodeForDagArg(DagInit *D, unsigned ArgNum,
     }
   }
 
-  PrintFatalError("bad dag argument type for code generation");
+  PrintError("bad DAG argument type for code generation");
+  PrintNote("DAG: " + D->getAsString());
+  if (TypedInit *Typed = dyn_cast<TypedInit>(Arg))
+    PrintNote("argument type: " + Typed->getType()->getAsString());
+  PrintFatalNote("argument number " + Twine(ArgNum) + ": " + Arg->getAsString());
 }
 
 Result::Ptr EmitterBase::getCodeForArg(unsigned ArgNum, const Type *ArgType,
diff --git a/clang/utils/TableGen/RISCVVEmitter.cpp b/clang/utils/TableGen/RISCVVEmitter.cpp
index ba96396c780d70ad668d2251c9611326e37f4a0d..4e4cf442e336fd60f797669ab94e51c769dd8f64 100644
--- a/clang/utils/TableGen/RISCVVEmitter.cpp
+++ b/clang/utils/TableGen/RISCVVEmitter.cpp
@@ -6,15 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This tablegen backend is responsible for emitting riscv_vector.h and
-// riscv_vector_generic.h, which includes a declaration and definition of each
-// intrinsic fucntions specified in https://github.com/riscv/rvv-intrinsic-doc.
+// This tablegen backend is responsible for emitting riscv_vector.h which
+// includes a declaration and definition of each intrinsic functions specified
+// in https://github.com/riscv/rvv-intrinsic-doc.
 //
 // See also the documentation in include/clang/Basic/riscv_vector.td.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
@@ -39,7 +40,8 @@ public:
   // Return the C/C++ string representation of LMUL
   std::string str() const;
   Optional<unsigned> getScale(unsigned ElementBitwidth) const;
-  LMULType &operator*=(unsigned RHS);
+  void MulLog2LMUL(int Log2LMUL);
+  LMULType &operator*=(uint32_t RHS);
 };
 
 // This class is compact representation of a valid and invalid RVVType.
@@ -88,7 +90,13 @@ public:
   const std::string &getTypeStr() const { return Str; }
 
   // Return the short name of a type for C/C++ name suffix.
-  const std::string &getShortStr() const { return ShortStr; }
+  const std::string &getShortStr() {
+    // Not all types are used in short name, so compute the short name by
+    // demanded.
+    if (ShortStr.empty())
+      initShortStr();
+    return ShortStr;
+  }
 
   bool isValid() const { return Valid; }
   bool isScalar() const { return Scale.hasValue() && Scale.getValue() == 0; }
@@ -142,9 +150,12 @@ private:
   std::string MangledName;
   std::string IRName;
   bool HasSideEffects;
+  bool IsMask;
   bool HasMaskedOffOperand;
   bool HasVL;
-  bool HasGeneric;
+  bool HasNoMaskedOverloaded;
+  bool HasAutoDef; // There is automiatic definition in header
+  std::string ManualCodegen;
   RVVTypePtr OutputType; // Builtin output type
   RVVTypes InputTypes;   // Builtin input types
   // The types we use to obtain the specific LLVM intrinsic. They are index of
@@ -158,9 +169,10 @@ private:
 public:
   RVVIntrinsic(StringRef Name, StringRef Suffix, StringRef MangledName,
                StringRef IRName, bool HasSideEffects, bool IsMask,
-               bool HasMaskedOffOperand, bool HasVL, bool HasGeneric,
-               const RVVTypes &Types,
-               const std::vector<int64_t> &RVVIntrinsicTypes);
+               bool HasMaskedOffOperand, bool HasVL, bool HasNoMaskedOverloaded,
+               bool HasAutoDef, StringRef ManualCodegen, const RVVTypes &Types,
+               const std::vector<int64_t> &IntrinsicTypes,
+               const std::vector<int64_t> &PermuteOperands);
   ~RVVIntrinsic() = default;
 
   StringRef getName() const { return Name; }
@@ -168,7 +180,10 @@ public:
   bool hasSideEffects() const { return HasSideEffects; }
   bool hasMaskedOffOperand() const { return HasMaskedOffOperand; }
   bool hasVL() const { return HasVL; }
-  bool hasGeneric() const { return HasGeneric; }
+  bool hasNoMaskedOverloaded() const { return HasNoMaskedOverloaded; }
+  bool hasManualCodegen() const { return !ManualCodegen.empty(); }
+  bool hasAutoDef() const { return HasAutoDef; }
+  bool isMask() const { return IsMask; }
   size_t getNumOperand() const { return InputTypes.size(); }
   StringRef getIRName() const { return IRName; }
   uint8_t getRISCVExtensions() const { return RISCVExtensions; }
@@ -190,6 +205,7 @@ public:
 class RVVEmitter {
 private:
   RecordKeeper &Records;
+  std::string HeaderCode;
   // Concat BasicType, LMUL and Proto as key
   StringMap<RVVType> LegalTypes;
   StringSet<> IllegalTypes;
@@ -200,15 +216,14 @@ public:
   /// Emit riscv_vector.h
   void createHeader(raw_ostream &o);
 
-  /// Emit riscv_generic.h
-  void createGenericHeader(raw_ostream &o);
-
   /// Emit all the __builtin prototypes and code needed by Sema.
   void createBuiltins(raw_ostream &o);
 
   /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
   void createCodeGen(raw_ostream &o);
 
+  std::string getSuffixStr(char Type, int Log2LMUL, StringRef Prototypes);
+
 private:
   /// Create all intrinsics and add them to \p Out
   void createRVVIntrinsics(std::vector<std::unique_ptr<RVVIntrinsic>> &Out);
@@ -220,7 +235,8 @@ private:
                                   ArrayRef<std::string> PrototypeSeq);
   Optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL, StringRef Proto);
 
-  /// Emit Acrh predecessor definitions and body
+  /// Emit Acrh predecessor definitions and body, assume the element of Defs are
+  /// sorted by extension.
   void emitArchMacroAndBody(
       std::vector<std::unique_ptr<RVVIntrinsic>> &Defs, raw_ostream &o,
       std::function<void(raw_ostream &, const RVVIntrinsic &)>);
@@ -228,6 +244,10 @@ private:
   // Emit the architecture preprocessor definitions. Return true when emits
   // non-empty string.
   bool emitExtDefStr(uint8_t Extensions, raw_ostream &o);
+  // Slice Prototypes string into sub prototype string and process each sub
+  // prototype string individually in the Handler.
+  void parsePrototypes(StringRef Prototypes,
+                       std::function<void(StringRef)> Handler);
 };
 
 } // namespace
@@ -272,6 +292,8 @@ VScaleVal LMULType::getScale(unsigned ElementBitwidth) const {
   return 1 << Log2ScaleResult;
 }
 
+void LMULType::MulLog2LMUL(int log2LMUL) { Log2LMUL += log2LMUL; }
+
 LMULType &LMULType::operator*=(uint32_t RHS) {
   assert(isPowerOf2_32(RHS));
   this->Log2LMUL = this->Log2LMUL + Log2_32(RHS);
@@ -288,7 +310,6 @@ RVVType::RVVType(BasicType BT, int Log2LMUL, StringRef prototype)
     initTypeStr();
     if (isVector()) {
       initClangBuiltinStr();
-      initShortStr();
     }
   }
 }
@@ -311,6 +332,8 @@ RVVType::RVVType(BasicType BT, int Log2LMUL, StringRef prototype)
 // clang-format on
 
 bool RVVType::verifyType() const {
+  if (ScalarType == Invalid)
+    return false;
   if (isScalar())
     return true;
   if (!Scale.hasValue())
@@ -546,7 +569,8 @@ void RVVType::applyModifier(StringRef Transformer) {
   if (Transformer.empty())
     return;
   // Handle primitive type transformer
-  switch (Transformer.back()) {
+  auto PType = Transformer.back();
+  switch (PType) {
   case 'e':
     Scale = 0;
     break;
@@ -592,7 +616,40 @@ void RVVType::applyModifier(StringRef Transformer) {
   }
   Transformer = Transformer.drop_back();
 
-  // Compute type transformers
+  // Extract and compute complex type transformer. It can only appear one time.
+  if (Transformer.startswith("(")) {
+    size_t Idx = Transformer.find(')');
+    assert(Idx != StringRef::npos);
+    StringRef ComplexType = Transformer.slice(1, Idx);
+    Transformer = Transformer.drop_front(Idx + 1);
+    assert(Transformer.find('(') == StringRef::npos &&
+           "Only allow one complex type transformer");
+
+    auto UpdateAndCheckComplexProto = [&]() {
+      Scale = LMUL.getScale(ElementBitwidth);
+      const StringRef VectorPrototypes("vwqom");
+      if (!VectorPrototypes.contains(PType))
+        PrintFatalError("Complex type transformer only supports vector type!");
+      if (Transformer.find_first_of("PCKWS") != StringRef::npos)
+        PrintFatalError(
+            "Illegal type transformer for Complex type transformer");
+    };
+    auto ComplexTT = ComplexType.split(":");
+    if (ComplexTT.first == "Log2EEW") {
+      uint32_t Log2EEW;
+      ComplexTT.second.getAsInteger(10, Log2EEW);
+      // update new elmul = (eew/sew) * lmul
+      LMUL.MulLog2LMUL(Log2EEW - Log2_32(ElementBitwidth));
+      // update new eew
+      ElementBitwidth = 1 << Log2EEW;
+      ScalarType = ScalarTypeKind::SignedInteger;
+      UpdateAndCheckComplexProto();
+    } else {
+      PrintFatalError("Illegal complex type transformers!");
+    }
+  }
+
+  // Compute the remain type transformers
   for (char I : Transformer) {
     switch (I) {
     case 'P':
@@ -637,11 +694,14 @@ RVVIntrinsic::RVVIntrinsic(StringRef NewName, StringRef Suffix,
                            StringRef NewMangledName, StringRef IRName,
                            bool HasSideEffects, bool IsMask,
                            bool HasMaskedOffOperand, bool HasVL,
-                           bool HasGeneric, const RVVTypes &OutInTypes,
-                           const std::vector<int64_t> &NewIntrinsicTypes)
-    : IRName(IRName), HasSideEffects(HasSideEffects),
+                           bool HasNoMaskedOverloaded, bool HasAutoDef,
+                           StringRef ManualCodegen, const RVVTypes &OutInTypes,
+                           const std::vector<int64_t> &NewIntrinsicTypes,
+                           const std::vector<int64_t> &PermuteOperands)
+    : IRName(IRName), HasSideEffects(HasSideEffects), IsMask(IsMask),
       HasMaskedOffOperand(HasMaskedOffOperand), HasVL(HasVL),
-      HasGeneric(HasGeneric) {
+      HasNoMaskedOverloaded(HasNoMaskedOverloaded), HasAutoDef(HasAutoDef),
+      ManualCodegen(ManualCodegen.str()) {
 
   // Init Name and MangledName
   Name = NewName.str();
@@ -653,7 +713,6 @@ RVVIntrinsic::RVVIntrinsic(StringRef NewName, StringRef Suffix,
     Name += "_" + Suffix.str();
   if (IsMask) {
     Name += "_m";
-    MangledName += "_m";
   }
   // Init RISC-V extensions
   for (const auto &T : OutInTypes) {
@@ -670,6 +729,29 @@ RVVIntrinsic::RVVIntrinsic(StringRef NewName, StringRef Suffix,
   InputTypes.assign(OutInTypes.begin() + 1, OutInTypes.end());
   CTypeOrder.resize(InputTypes.size());
   std::iota(CTypeOrder.begin(), CTypeOrder.end(), 0);
+  // Update default order if we need permutate.
+  if (!PermuteOperands.empty()) {
+    // PermuteOperands is nonmasked version index. Update index when there is
+    // maskedoff operand which is always in first operand.
+
+    unsigned Skew = HasMaskedOffOperand ? 1 : 0;
+    for (unsigned i = 0; i < PermuteOperands.size(); ++i) {
+      if (i != PermuteOperands[i])
+        CTypeOrder[i] = PermuteOperands[i] + Skew;
+    }
+    // Verify the result of CTypeOrder has legal value.
+    if (*std::max_element(CTypeOrder.begin(), CTypeOrder.end()) >=
+        CTypeOrder.size())
+      PrintFatalError(
+          "The index of PermuteOperand is bigger than the operand number");
+    SmallSet<unsigned, 8> Seen;
+    for (auto Idx : CTypeOrder) {
+      if (!Seen.insert(Idx).second)
+        PrintFatalError(
+            "The different element in PermuteOperand could not be equal");
+    }
+  }
+
   if (IsMask) {
     if (HasVL)
       // Builtin type order: op0, op1, ..., mask, vl
@@ -681,6 +763,7 @@ RVVIntrinsic::RVVIntrinsic(StringRef NewName, StringRef Suffix,
       // C type order: mask, op0, op1, ...,
       std::rotate(CTypeOrder.begin(), CTypeOrder.end() - 1, CTypeOrder.end());
   }
+
   // IntrinsicTypes is nonmasked version index. Need to update it
   // if there is maskedoff operand (It is always in first operand).
   IntrinsicTypes = NewIntrinsicTypes;
@@ -702,7 +785,13 @@ std::string RVVIntrinsic::getBuiltinTypeStr() const {
 }
 
 void RVVIntrinsic::emitCodeGenSwitchBody(raw_ostream &OS) const {
+
   OS << "  ID = Intrinsic::riscv_" + getIRName() + ";\n";
+  if (hasManualCodegen()) {
+    OS << ManualCodegen;
+    OS << "break;\n";
+    return;
+  }
   OS << "  IntrinsicTypes = {";
   ListSeparator LS;
   for (const auto &Idx : IntrinsicTypes) {
@@ -792,6 +881,11 @@ void RVVEmitter::createHeader(raw_ostream &OS) {
   std::vector<std::unique_ptr<RVVIntrinsic>> Defs;
   createRVVIntrinsics(Defs);
 
+  // Print header code
+  if (!HeaderCode.empty()) {
+    OS << HeaderCode;
+  }
+
   auto printType = [&](auto T) {
     OS << "typedef " << T->getClangBuiltinStr() << " " << T->getTypeStr()
        << ";\n";
@@ -832,37 +926,42 @@ void RVVEmitter::createHeader(raw_ostream &OS) {
   OS << "#endif\n";
 
   OS << "#if defined(__riscv_d)\n";
-  for (int ELMul : Log2LMULs) {
-    auto T = computeType('d', ELMul, "v");
+  for (int Log2LMUL : Log2LMULs) {
+    auto T = computeType('d', Log2LMUL, "v");
     if (T.hasValue())
       printType(T.getValue());
   }
   OS << "#endif\n\n";
 
+  // The same extension include in the same arch guard marco.
+  std::stable_sort(Defs.begin(), Defs.end(),
+                   [](const std::unique_ptr<RVVIntrinsic> &A,
+                      const std::unique_ptr<RVVIntrinsic> &B) {
+                     return A->getRISCVExtensions() < B->getRISCVExtensions();
+                   });
+
   // Print intrinsic functions with macro
   emitArchMacroAndBody(Defs, OS, [](raw_ostream &OS, const RVVIntrinsic &Inst) {
     Inst.emitIntrinsicMacro(OS);
   });
 
-  OS << "\n#ifdef __cplusplus\n";
-  OS << "}\n";
-  OS << "#endif // __riscv_vector\n";
-  OS << "#endif // __RISCV_VECTOR_H\n";
-}
+  OS << "#define __riscv_v_intrinsic_overloading 1\n";
 
-void RVVEmitter::createGenericHeader(raw_ostream &OS) {
-  std::vector<std::unique_ptr<RVVIntrinsic>> Defs;
-  createRVVIntrinsics(Defs);
+  // Print Overloaded APIs
+  OS << "#define __rvv_overloaded static inline "
+        "__attribute__((__always_inline__, __nodebug__, __overloadable__))\n";
 
-  OS << "#include <riscv_vector.h>\n\n";
-  // Print intrinsic functions macro
   emitArchMacroAndBody(Defs, OS, [](raw_ostream &OS, const RVVIntrinsic &Inst) {
-    if (!Inst.hasGeneric())
+    if (!Inst.isMask() && !Inst.hasNoMaskedOverloaded())
       return;
-    OS << "static inline __attribute__((__always_inline__, __nodebug__, "
-          "__overloadable__))\n";
+    OS << "__rvv_overloaded ";
     Inst.emitMangledFuncDef(OS);
   });
+
+  OS << "\n#ifdef __cplusplus\n";
+  OS << "}\n";
+  OS << "#endif // __riscv_vector\n";
+  OS << "#endif // __RISCV_VECTOR_H\n";
 }
 
 void RVVEmitter::createBuiltins(raw_ostream &OS) {
@@ -881,7 +980,6 @@ void RVVEmitter::createBuiltins(raw_ostream &OS) {
     else
       OS << "\"\")\n";
   }
-  OS << "\n#undef BUILTIN\n";
   OS << "#undef RISCVV_BUILTIN\n";
 }
 
@@ -909,40 +1007,70 @@ void RVVEmitter::createCodeGen(raw_ostream &OS) {
   OS << "\n";
 }
 
+void RVVEmitter::parsePrototypes(StringRef Prototypes,
+                                 std::function<void(StringRef)> Handler) {
+  const StringRef Primaries("evwqom0ztc");
+  while (!Prototypes.empty()) {
+    size_t Idx = 0;
+    // Skip over complex prototype because it could contain primitive type
+    // character.
+    if (Prototypes[0] == '(')
+      Idx = Prototypes.find_first_of(')');
+    Idx = Prototypes.find_first_of(Primaries, Idx);
+    assert(Idx != StringRef::npos);
+    Handler(Prototypes.slice(0, Idx + 1));
+    Prototypes = Prototypes.drop_front(Idx + 1);
+  }
+}
+
+std::string RVVEmitter::getSuffixStr(char Type, int Log2LMUL,
+                                     StringRef Prototypes) {
+  SmallVector<std::string> SuffixStrs;
+  parsePrototypes(Prototypes, [&](StringRef Proto) {
+    auto T = computeType(Type, Log2LMUL, Proto);
+    SuffixStrs.push_back(T.getValue()->getShortStr());
+  });
+  return join(SuffixStrs, "_");
+}
+
 void RVVEmitter::createRVVIntrinsics(
     std::vector<std::unique_ptr<RVVIntrinsic>> &Out) {
-
   std::vector<Record *> RV = Records.getAllDerivedDefinitions("RVVBuiltin");
   for (auto *R : RV) {
     StringRef Name = R->getValueAsString("Name");
-    StringRef Suffix = R->getValueAsString("Suffix");
+    StringRef SuffixProto = R->getValueAsString("Suffix");
     StringRef MangledName = R->getValueAsString("MangledName");
     StringRef Prototypes = R->getValueAsString("Prototype");
     StringRef TypeRange = R->getValueAsString("TypeRange");
     bool HasMask = R->getValueAsBit("HasMask");
     bool HasMaskedOffOperand = R->getValueAsBit("HasMaskedOffOperand");
     bool HasVL = R->getValueAsBit("HasVL");
-    bool HasGeneric = R->getValueAsBit("HasGeneric");
+    bool HasNoMaskedOverloaded = R->getValueAsBit("HasNoMaskedOverloaded");
     bool HasSideEffects = R->getValueAsBit("HasSideEffects");
     std::vector<int64_t> Log2LMULList = R->getValueAsListOfInts("Log2LMUL");
+    StringRef ManualCodegen = R->getValueAsString("ManualCodegen");
+    StringRef ManualCodegenMask = R->getValueAsString("ManualCodegenMask");
     std::vector<int64_t> IntrinsicTypes =
         R->getValueAsListOfInts("IntrinsicTypes");
+    std::vector<int64_t> PermuteOperands =
+        R->getValueAsListOfInts("PermuteOperands");
     StringRef IRName = R->getValueAsString("IRName");
     StringRef IRNameMask = R->getValueAsString("IRNameMask");
 
+    StringRef HeaderCodeStr = R->getValueAsString("HeaderCode");
+    bool HasAutoDef = HeaderCodeStr.empty();
+    if (!HeaderCodeStr.empty()) {
+      HeaderCode += HeaderCodeStr.str();
+    }
     // Parse prototype and create a list of primitive type with transformers
     // (operand) in ProtoSeq. ProtoSeq[0] is output operand.
-    SmallVector<std::string, 8> ProtoSeq;
-    const StringRef Primaries("evwqom0ztc");
-    while (!Prototypes.empty()) {
-      auto Idx = Prototypes.find_first_of(Primaries);
-      assert(Idx != StringRef::npos);
-      ProtoSeq.push_back(Prototypes.slice(0, Idx + 1).str());
-      Prototypes = Prototypes.drop_front(Idx + 1);
-    }
+    SmallVector<std::string> ProtoSeq;
+    parsePrototypes(Prototypes, [&ProtoSeq](StringRef Proto) {
+      ProtoSeq.push_back(Proto.str());
+    });
 
     // Compute Builtin types
-    SmallVector<std::string, 8> ProtoMaskSeq = ProtoSeq;
+    SmallVector<std::string> ProtoMaskSeq = ProtoSeq;
     if (HasMask) {
       // If HasMask, append 'm' to last operand.
       ProtoMaskSeq.push_back("m");
@@ -956,7 +1084,7 @@ void RVVEmitter::createRVVIntrinsics(
       ProtoMaskSeq.push_back("z");
     }
 
-    // Create intrinsics for each type and LMUL.
+    // Create Intrinsics for each type and LMUL.
     for (char I : TypeRange) {
       for (int Log2LMUL : Log2LMULList) {
         Optional<RVVTypes> Types = computeTypes(I, Log2LMUL, ProtoSeq);
@@ -964,23 +1092,24 @@ void RVVEmitter::createRVVIntrinsics(
         if (!Types.hasValue())
           continue;
 
-        auto SuffixStr =
-            computeType(I, Log2LMUL, Suffix).getValue()->getShortStr();
-        // Create a non-mask intrinsic.
+        auto SuffixStr = getSuffixStr(I, Log2LMUL, SuffixProto);
+        // Create a non-mask intrinsic
         Out.push_back(std::make_unique<RVVIntrinsic>(
             Name, SuffixStr, MangledName, IRName, HasSideEffects,
-            /*IsMask=*/false, /*HasMaskedOffOperand=*/false, HasVL, HasGeneric,
-            Types.getValue(), IntrinsicTypes));
+            /*IsMask=*/false, /*HasMaskedOffOperand=*/false, HasVL,
+            HasNoMaskedOverloaded, HasAutoDef, ManualCodegen, Types.getValue(),
+            IntrinsicTypes, PermuteOperands));
         if (HasMask) {
           // Create a mask intrinsic
           Optional<RVVTypes> MaskTypes =
               computeTypes(I, Log2LMUL, ProtoMaskSeq);
           Out.push_back(std::make_unique<RVVIntrinsic>(
               Name, SuffixStr, MangledName, IRNameMask, HasSideEffects,
-              /*IsMask=*/true, HasMaskedOffOperand, HasVL, HasGeneric,
-              MaskTypes.getValue(), IntrinsicTypes));
+              /*IsMask=*/true, HasMaskedOffOperand, HasVL,
+              HasNoMaskedOverloaded, HasAutoDef, ManualCodegenMask,
+              MaskTypes.getValue(), IntrinsicTypes, PermuteOperands));
         }
-      } // end for Log2LMUL
+      } // end for Log2LMULList
     }   // end for TypeRange
   }
 }
@@ -1023,13 +1152,6 @@ Optional<RVVTypePtr> RVVEmitter::computeType(BasicType BT, int Log2LMUL,
 void RVVEmitter::emitArchMacroAndBody(
     std::vector<std::unique_ptr<RVVIntrinsic>> &Defs, raw_ostream &OS,
     std::function<void(raw_ostream &, const RVVIntrinsic &)> PrintBody) {
-
-  // The same extension include in the same arch guard marco.
-  std::stable_sort(Defs.begin(), Defs.end(),
-                   [](const std::unique_ptr<RVVIntrinsic> &A,
-                      const std::unique_ptr<RVVIntrinsic> &B) {
-                     return A->getRISCVExtensions() < B->getRISCVExtensions();
-                   });
   uint8_t PrevExt = (*Defs.begin())->getRISCVExtensions();
   bool NeedEndif = emitExtDefStr(PrevExt, OS);
   for (auto &Def : Defs) {
@@ -1040,7 +1162,8 @@ void RVVEmitter::emitArchMacroAndBody(
       NeedEndif = emitExtDefStr(CurExt, OS);
       PrevExt = CurExt;
     }
-    PrintBody(OS, *Def);
+    if (Def->hasAutoDef())
+      PrintBody(OS, *Def);
   }
   if (NeedEndif)
     OS << "#endif\n\n";
@@ -1066,10 +1189,6 @@ void EmitRVVHeader(RecordKeeper &Records, raw_ostream &OS) {
   RVVEmitter(Records).createHeader(OS);
 }
 
-void EmitRVVGenericHeader(RecordKeeper &Records, raw_ostream &OS) {
-  RVVEmitter(Records).createGenericHeader(OS);
-}
-
 void EmitRVVBuiltins(RecordKeeper &Records, raw_ostream &OS) {
   RVVEmitter(Records).createBuiltins(OS);
 }
diff --git a/clang/utils/TableGen/TableGen.cpp b/clang/utils/TableGen/TableGen.cpp
index 645157aef419888669c7a0c3e3d9db16956bd653..55d62fa9f9bdc0495e664f3609a311fd8ecf61ba 100644
--- a/clang/utils/TableGen/TableGen.cpp
+++ b/clang/utils/TableGen/TableGen.cpp
@@ -84,7 +84,6 @@ enum ActionType {
   GenArmCdeBuiltinCG,
   GenArmCdeBuiltinAliases,
   GenRISCVVectorHeader,
-  GenRISCVVectorGenericHeader,
   GenRISCVVectorBuiltins,
   GenRISCVVectorBuiltinCG,
   GenAttrDocs,
@@ -234,9 +233,6 @@ cl::opt<ActionType> Action(
                    "Generate list of valid ARM CDE builtin aliases for clang"),
         clEnumValN(GenRISCVVectorHeader, "gen-riscv-vector-header",
                    "Generate riscv_vector.h for clang"),
-        clEnumValN(GenRISCVVectorGenericHeader,
-                   "gen-riscv-vector-generic-header",
-                   "Generate riscv_vector_generic.h for clang"),
         clEnumValN(GenRISCVVectorBuiltins, "gen-riscv-vector-builtins",
                    "Generate riscv_vector_builtins.inc for clang"),
         clEnumValN(GenRISCVVectorBuiltinCG, "gen-riscv-vector-builtin-codegen",
@@ -444,9 +440,6 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenRISCVVectorHeader:
     EmitRVVHeader(Records, OS);
     break;
-  case GenRISCVVectorGenericHeader:
-    EmitRVVGenericHeader(Records, OS);
-    break;
   case GenRISCVVectorBuiltins:
     EmitRVVBuiltins(Records, OS);
     break;
diff --git a/clang/utils/TableGen/TableGenBackends.h b/clang/utils/TableGen/TableGenBackends.h
index 5df149ebda191e9b7f1d03cffdfe05020c152794..6930f242681f2cf6edea03ea62e53f5ce39f7bd1 100644
--- a/clang/utils/TableGen/TableGenBackends.h
+++ b/clang/utils/TableGen/TableGenBackends.h
@@ -107,7 +107,6 @@ void EmitMveBuiltinCG(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitMveBuiltinAliases(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 
 void EmitRVVHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
-void EmitRVVGenericHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitRVVBuiltins(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitRVVBuiltinCG(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index d7baa595d6ee7d2cb3eb4cead09192be9c359023..473316005d37079fda1b0175cdc3879343a5f0f0 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -63,7 +63,7 @@
 <tr>
  <td><a href="#cxx23">C++2b (tentatively C++23)</a></td>
  <td><tt>-std=c++2b</tt></td>
- <td class="none" align="center">No</td>
+ <td class="partial" align="center">Partial</td>
 </tr>
 </table>
 
@@ -1276,7 +1276,7 @@ C++20, informally referred to as C++2b.
     <tr>
       <td>Make <tt>()</tt> in lambdas optional in all cases</td>
       <td><a href="https://wg21.link/p1102r2">P1102R2</a></td>
-      <td class="none" align="center">No</td>
+      <td class="full" align="center">Clang 13</td>
     </tr>
 </table>
 </details>
diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index fe4c61abd4037f6bd0a824d4ea40ac21b473e33c..ca2f34e618abaab25802261796158b7700a491ce 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -124,6 +124,21 @@ macro(set_output_name output name arch)
   else()
     if(ANDROID AND ${arch} STREQUAL "i386")
       set(${output} "${name}-i686${COMPILER_RT_OS_SUFFIX}")
+    elseif("${arch}" MATCHES "^arm")
+      if(COMPILER_RT_DEFAULT_TARGET_ONLY)
+        set(triple "${COMPILER_RT_DEFAULT_TARGET_TRIPLE}")
+      else()
+        set(triple "${TARGET_TRIPLE}")
+      endif()
+      # When using arch-suffixed runtime library names, clang only looks for
+      # libraries named "arm" or "armhf", see getArchNameForCompilerRTLib in
+      # clang. Therefore, try to inspect both the arch name and the triple
+      # if it seems like we're building an armhf target.
+      if ("${arch}" MATCHES "hf$" OR "${triple}" MATCHES "hf$")
+        set(${output} "${name}-armhf${COMPILER_RT_OS_SUFFIX}")
+      else()
+        set(${output} "${name}-arm${COMPILER_RT_OS_SUFFIX}")
+      endif()
     else()
       set(${output} "${name}-${arch}${COMPILER_RT_OS_SUFFIX}")
     endif()
diff --git a/compiler-rt/include/fuzzer/FuzzedDataProvider.h b/compiler-rt/include/fuzzer/FuzzedDataProvider.h
index 6cbfc39bc20b99997941b56bf7bd578ee609e0b6..71cb427ec4a9764da328e3d7da0cd177ccb579e9 100644
--- a/compiler-rt/include/fuzzer/FuzzedDataProvider.h
+++ b/compiler-rt/include/fuzzer/FuzzedDataProvider.h
@@ -390,7 +390,7 @@ TS FuzzedDataProvider::ConvertUnsignedToSigned(TU value) {
     return static_cast<TS>(value);
   } else {
     constexpr auto TS_min = std::numeric_limits<TS>::min();
-    return TS_min + static_cast<char>(value - TS_min);
+    return TS_min + static_cast<TS>(value - TS_min);
   }
 }
 
diff --git a/compiler-rt/include/sanitizer/tsan_interface.h b/compiler-rt/include/sanitizer/tsan_interface.h
index 96b8ad58541cbb1c050dbf4f85dadbb3b44a98fe..cfa9d3b5f632d905b7eccdf06d2ecc8df436096e 100644
--- a/compiler-rt/include/sanitizer/tsan_interface.h
+++ b/compiler-rt/include/sanitizer/tsan_interface.h
@@ -141,7 +141,7 @@ void __tsan_external_write(void *addr, void *caller_pc, void *tag);
 //     and freed by __tsan_destroy_fiber.
 //   - TSAN context of current fiber or thread can be obtained
 //     by calling __tsan_get_current_fiber.
-//   - __tsan_switch_to_fiber should be called immediatly before switch
+//   - __tsan_switch_to_fiber should be called immediately before switch
 //     to fiber, such as call of swapcontext.
 //   - Fiber name can be set by __tsan_set_fiber_name.
 void *__tsan_get_current_fiber(void);
@@ -154,6 +154,15 @@ void __tsan_set_fiber_name(void *fiber, const char *name);
 // Do not establish a happens-before relation between fibers
 static const unsigned __tsan_switch_to_fiber_no_sync = 1 << 0;
 
+// User-provided callback invoked on TSan initialization.
+void __tsan_on_initialize();
+
+// User-provided callback invoked on TSan shutdown.
+// `failed` - Nonzero if TSan did detect issues, zero otherwise.
+// Return `0` if TSan should exit as if no issues were detected.  Return nonzero
+// if TSan should exit as if issues were detected.
+int __tsan_on_finalize(int failed);
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif
diff --git a/compiler-rt/lib/asan/asan_descriptions.cpp b/compiler-rt/lib/asan/asan_descriptions.cpp
index 347eaa4a824f346f4ed268cddd531a12828ac4df..42f2215c14d6126c77d48266b9dec4489d4ae3d8 100644
--- a/compiler-rt/lib/asan/asan_descriptions.cpp
+++ b/compiler-rt/lib/asan/asan_descriptions.cpp
@@ -48,7 +48,7 @@ void DescribeThread(AsanThreadContext *context) {
     return;
   }
   context->announced = true;
-  InternalScopedString str(1024);
+  InternalScopedString str;
   str.append("Thread %s", AsanThreadIdAndName(context).c_str());
   if (context->parent_tid == kInvalidTid) {
     str.append(" created by unknown thread\n");
@@ -125,7 +125,7 @@ static void GetAccessToHeapChunkInformation(ChunkAccess *descr,
 
 static void PrintHeapChunkAccess(uptr addr, const ChunkAccess &descr) {
   Decorator d;
-  InternalScopedString str(4096);
+  InternalScopedString str;
   str.append("%s", d.Location());
   switch (descr.access_type) {
     case kAccessTypeLeft:
@@ -242,7 +242,7 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr,
     else if (addr >= prev_var_end && addr - prev_var_end >= var.beg - addr_end)
       pos_descr = "underflows";
   }
-  InternalScopedString str(1024);
+  InternalScopedString str;
   str.append("    [%zd, %zd)", var.beg, var_end);
   // Render variable name.
   str.append(" '");
@@ -275,7 +275,7 @@ bool DescribeAddressIfStack(uptr addr, uptr access_size) {
 // Global descriptions
 static void DescribeAddressRelativeToGlobal(uptr addr, uptr access_size,
                                             const __asan_global &g) {
-  InternalScopedString str(4096);
+  InternalScopedString str;
   Decorator d;
   str.append("%s", d.Location());
   if (addr < g.beg) {
diff --git a/compiler-rt/lib/asan/asan_errors.cpp b/compiler-rt/lib/asan/asan_errors.cpp
index 541c6e0353b57ac08a7147c658eed11f94540335..e68e6971f963a3797f907ebd1763c1e8ae492477 100644
--- a/compiler-rt/lib/asan/asan_errors.cpp
+++ b/compiler-rt/lib/asan/asan_errors.cpp
@@ -343,7 +343,8 @@ void ErrorODRViolation::Print() {
   Report("ERROR: AddressSanitizer: %s (%p):\n", scariness.GetDescription(),
          global1.beg);
   Printf("%s", d.Default());
-  InternalScopedString g1_loc(256), g2_loc(256);
+  InternalScopedString g1_loc;
+  InternalScopedString g2_loc;
   PrintGlobalLocation(&g1_loc, global1);
   PrintGlobalLocation(&g2_loc, global2);
   Printf("  [1] size=%zd '%s' %s\n", global1.size,
@@ -360,7 +361,7 @@ void ErrorODRViolation::Print() {
   Report(
       "HINT: if you don't care about these errors you may set "
       "ASAN_OPTIONS=detect_odr_violation=0\n");
-  InternalScopedString error_msg(256);
+  InternalScopedString error_msg;
   error_msg.append("%s: global '%s' at %s", scariness.GetDescription(),
                    MaybeDemangleGlobalName(global1.name), g1_loc.data());
   ReportErrorSummary(error_msg.data());
@@ -554,7 +555,7 @@ static void PrintShadowMemoryForAddress(uptr addr) {
   uptr shadow_addr = MemToShadow(addr);
   const uptr n_bytes_per_row = 16;
   uptr aligned_shadow = shadow_addr & ~(n_bytes_per_row - 1);
-  InternalScopedString str(4096 * 8);
+  InternalScopedString str;
   str.append("Shadow bytes around the buggy address:\n");
   for (int i = -5; i <= 5; i++) {
     uptr row_shadow_addr = aligned_shadow + i * n_bytes_per_row;
diff --git a/compiler-rt/lib/asan/asan_fake_stack.cpp b/compiler-rt/lib/asan/asan_fake_stack.cpp
index 295e6debc96c2d72bc399efa6c11a25ed0461b8f..1f873fec7d7cc18f7ebbbcf5091827fb47497dbb 100644
--- a/compiler-rt/lib/asan/asan_fake_stack.cpp
+++ b/compiler-rt/lib/asan/asan_fake_stack.cpp
@@ -65,7 +65,7 @@ FakeStack *FakeStack::Create(uptr stack_size_log) {
 void FakeStack::Destroy(int tid) {
   PoisonAll(0);
   if (Verbosity() >= 2) {
-    InternalScopedString str(kNumberOfSizeClasses * 50);
+    InternalScopedString str;
     for (uptr class_id = 0; class_id < kNumberOfSizeClasses; class_id++)
       str.append("%zd: %zd/%zd; ", class_id, hint_position_[class_id],
                  NumberOfFrames(stack_size_log(), class_id));
diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp
index 7b5a929963c6ad471f3907acaa49325716750721..106a526076319b73089b597ea26132d698c84121 100644
--- a/compiler-rt/lib/asan/asan_rtl.cpp
+++ b/compiler-rt/lib/asan/asan_rtl.cpp
@@ -490,9 +490,6 @@ static void AsanInitInternal() {
   if (flags()->start_deactivated)
     AsanDeactivate();
 
-  // interceptors
-  InitTlsSize();
-
   // Create main thread.
   AsanThread *main_thread = CreateMainThread();
   CHECK_EQ(0, main_thread->tid());
@@ -568,7 +565,7 @@ void UnpoisonStack(uptr bottom, uptr top, const char *type) {
         type, top, bottom, top - bottom, top - bottom);
     return;
   }
-  PoisonShadow(bottom, top - bottom, 0);
+  PoisonShadow(bottom, RoundUpTo(top - bottom, SHADOW_GRANULARITY), 0);
 }
 
 static void UnpoisonDefaultStack() {
diff --git a/compiler-rt/lib/asan/asan_thread.cpp b/compiler-rt/lib/asan/asan_thread.cpp
index ae3bcba204c63948e33e33d146cb77184e5d33e8..f7778c0f1e3402769fc5f1716f359138218c7f2c 100644
--- a/compiler-rt/lib/asan/asan_thread.cpp
+++ b/compiler-rt/lib/asan/asan_thread.cpp
@@ -307,7 +307,7 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) {
   uptr stack_size = 0;
   GetThreadStackAndTls(tid() == 0, &stack_bottom_, &stack_size, &tls_begin_,
                        &tls_size);
-  stack_top_ = stack_bottom_ + stack_size;
+  stack_top_ = RoundDownTo(stack_bottom_ + stack_size, SHADOW_GRANULARITY);
   tls_end_ = tls_begin_ + tls_size;
   dtls_ = DTLS_Get();
 
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 4e81093219c07fd3cd760b8e0f608f0c1560e94e..c81c535c8538452039561eb6cd796c05de0d5cac 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -707,10 +707,11 @@ else ()
           list(REMOVE_ITEM ${arch}_SOURCES ${arm_Thumb1_VFPv2_DP_SOURCES} ${arm_Thumb1_VFPv2_SP_SOURCES} ${arm_Thumb1_SjLj_EH_SOURCES})
         else()
           # Exclude any double-precision builtins if VFP is single-precision-only
-          check_c_source_compiles("#if !(__ARM_FP & 0x8)
+          try_compile_only(COMPILER_RT_HAS_${arch}_VFP_DP
+                           SOURCE "#if !(__ARM_FP & 0x8)
                                    #error No double-precision support!
                                    #endif
-                                   int main() { return 0; }" COMPILER_RT_HAS_${arch}_VFP_DP)
+                                   int main() { return 0; }")
           if(NOT COMPILER_RT_HAS_${arch}_VFP_DP)
             list(REMOVE_ITEM ${arch}_SOURCES ${arm_Thumb1_VFPv2_DP_SOURCES})
           endif()
diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp
index 5a9620aa417e5f7d81d3f22803d0de285ede54ef..2aff8869d2cf9e55d75e7a8a6d191e014cdde612 100644
--- a/compiler-rt/lib/dfsan/dfsan.cpp
+++ b/compiler-rt/lib/dfsan/dfsan.cpp
@@ -736,6 +736,13 @@ dfsan_read_origin_of_first_taint(const void *addr, uptr size) {
   return GetOriginIfTainted((uptr)addr, size);
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE void dfsan_set_label_origin(dfsan_label label,
+                                                          dfsan_origin origin,
+                                                          void *addr,
+                                                          uptr size) {
+  __dfsan_set_label(label, origin, addr, size);
+}
+
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label) {
   return &__dfsan_label_info[label];
diff --git a/compiler-rt/lib/dfsan/dfsan.h b/compiler-rt/lib/dfsan/dfsan.h
index c2f173f079ff42a251a1cc746e64f184f15eb9a8..73b4e4dcd29773804a17ef117343b3ebb3858785 100644
--- a/compiler-rt/lib/dfsan/dfsan.h
+++ b/compiler-rt/lib/dfsan/dfsan.h
@@ -48,6 +48,10 @@ void dfsan_clear_thread_local_state();
 // from the address addr.
 dfsan_origin dfsan_read_origin_of_first_taint(const void *addr, uptr size);
 
+// Set the data within [addr, addr+size) with label and origin.
+void dfsan_set_label_origin(dfsan_label label, dfsan_origin origin, void *addr,
+                            uptr size);
+
 // Copy or move the origins of the len bytes from src to dst.
 void dfsan_mem_origin_transfer(const void *dst, const void *src, uptr len);
 }  // extern "C"
diff --git a/compiler-rt/lib/dfsan/dfsan_custom.cpp b/compiler-rt/lib/dfsan/dfsan_custom.cpp
index ae0c46ac9b71b1e5f16da09d43a84ea08bad0ba2..8c3af25036d7b3f5da7da0f999b494d683ab1a74 100644
--- a/compiler-rt/lib/dfsan/dfsan_custom.cpp
+++ b/compiler-rt/lib/dfsan/dfsan_custom.cpp
@@ -1,4 +1,4 @@
-//===-- dfsan.cpp ---------------------------------------------------------===//
+//===-- dfsan_custom.cpp --------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -89,6 +89,14 @@ __dfsw_stat(const char *path, struct stat *buf, dfsan_label path_label,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_stat(
+    const char *path, struct stat *buf, dfsan_label path_label,
+    dfsan_label buf_label, dfsan_label *ret_label, dfsan_origin path_origin,
+    dfsan_origin buf_origin, dfsan_origin *ret_origin) {
+  int ret = __dfsw_stat(path, buf, path_label, buf_label, ret_label);
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_fstat(int fd, struct stat *buf,
                                                dfsan_label fd_label,
                                                dfsan_label buf_label,
@@ -100,27 +108,58 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_fstat(int fd, struct stat *buf,
   return ret;
 }
 
-SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strchr(const char *s, int c,
-                                                  dfsan_label s_label,
-                                                  dfsan_label c_label,
-                                                  dfsan_label *ret_label) {
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_fstat(
+    int fd, struct stat *buf, dfsan_label fd_label, dfsan_label buf_label,
+    dfsan_label *ret_label, dfsan_origin fd_origin, dfsan_origin buf_origin,
+    dfsan_origin *ret_origin) {
+  int ret = __dfsw_fstat(fd, buf, fd_label, buf_label, ret_label);
+  return ret;
+}
+
+static char *dfsan_strchr_with_label(const char *s, int c, size_t *bytes_read,
+                                     dfsan_label s_label, dfsan_label c_label,
+                                     dfsan_label *ret_label) {
+  char *match_pos = nullptr;
   for (size_t i = 0;; ++i) {
     if (s[i] == c || s[i] == 0) {
-      if (flags().strict_data_dependencies) {
-        *ret_label = s_label;
-      } else {
-        *ret_label = dfsan_union(dfsan_read_label(s, i + 1),
-                                 dfsan_union(s_label, c_label));
-      }
-
       // If s[i] is the \0 at the end of the string, and \0 is not the
       // character we are searching for, then return null.
-      if (s[i] == 0 && c != 0) {
-        return nullptr;
-      }
-      return const_cast<char *>(s + i);
+      *bytes_read = i + 1;
+      match_pos = s[i] == 0 && c != 0 ? nullptr : const_cast<char *>(s + i);
+      break;
     }
   }
+  if (flags().strict_data_dependencies)
+    *ret_label = s_label;
+  else
+    *ret_label = dfsan_union(dfsan_read_label(s, *bytes_read),
+                             dfsan_union(s_label, c_label));
+  return match_pos;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strchr(const char *s, int c,
+                                                  dfsan_label s_label,
+                                                  dfsan_label c_label,
+                                                  dfsan_label *ret_label) {
+  size_t bytes_read;
+  return dfsan_strchr_with_label(s, c, &bytes_read, s_label, c_label,
+                                 ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strchr(
+    const char *s, int c, dfsan_label s_label, dfsan_label c_label,
+    dfsan_label *ret_label, dfsan_origin s_origin, dfsan_origin c_origin,
+    dfsan_origin *ret_origin) {
+  size_t bytes_read;
+  char *r =
+      dfsan_strchr_with_label(s, c, &bytes_read, s_label, c_label, ret_label);
+  if (flags().strict_data_dependencies) {
+    *ret_origin = s_origin;
+  } else if (*ret_label) {
+    dfsan_origin o = dfsan_read_origin_of_first_taint(s, bytes_read);
+    *ret_origin = o ? o : (s_label ? s_origin : c_origin);
+  }
+  return r;
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strpbrk(const char *s,
@@ -141,36 +180,87 @@ SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strpbrk(const char *s,
   return const_cast<char *>(ret);
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strpbrk(
+    const char *s, const char *accept, dfsan_label s_label,
+    dfsan_label accept_label, dfsan_label *ret_label, dfsan_origin s_origin,
+    dfsan_origin accept_origin, dfsan_origin *ret_origin) {
+  const char *ret = __dfsw_strpbrk(s, accept, s_label, accept_label, ret_label);
+  if (flags().strict_data_dependencies) {
+    if (ret)
+      *ret_origin = s_origin;
+  } else {
+    if (*ret_label) {
+      size_t s_bytes_read = (ret ? ret - s : strlen(s)) + 1;
+      dfsan_origin o = dfsan_read_origin_of_first_taint(s, s_bytes_read);
+      if (o) {
+        *ret_origin = o;
+      } else {
+        o = dfsan_read_origin_of_first_taint(accept, strlen(accept) + 1);
+        *ret_origin = o ? o : (s_label ? s_origin : accept_origin);
+      }
+    }
+  }
+  return const_cast<char *>(ret);
+}
+
 static int dfsan_memcmp_bcmp(const void *s1, const void *s2, size_t n,
-                             dfsan_label s1_label, dfsan_label s2_label,
-                             dfsan_label n_label, dfsan_label *ret_label) {
+                             size_t *bytes_read) {
   const char *cs1 = (const char *) s1, *cs2 = (const char *) s2;
   for (size_t i = 0; i != n; ++i) {
     if (cs1[i] != cs2[i]) {
-      if (flags().strict_data_dependencies) {
-        *ret_label = 0;
-      } else {
-        *ret_label = dfsan_union(dfsan_read_label(cs1, i + 1),
-                                 dfsan_read_label(cs2, i + 1));
-      }
+      *bytes_read = i + 1;
       return cs1[i] - cs2[i];
     }
   }
-
-  if (flags().strict_data_dependencies) {
-    *ret_label = 0;
-  } else {
-    *ret_label = dfsan_union(dfsan_read_label(cs1, n),
-                             dfsan_read_label(cs2, n));
-  }
+  *bytes_read = n;
   return 0;
 }
 
+static dfsan_label dfsan_get_memcmp_label(const void *s1, const void *s2,
+                                          size_t pos) {
+  if (flags().strict_data_dependencies)
+    return 0;
+  return dfsan_union(dfsan_read_label(s1, pos), dfsan_read_label(s2, pos));
+}
+
+static void dfsan_get_memcmp_origin(const void *s1, const void *s2, size_t pos,
+                                    dfsan_label *ret_label,
+                                    dfsan_origin *ret_origin) {
+  *ret_label = dfsan_get_memcmp_label(s1, s2, pos);
+  if (*ret_label == 0)
+    return;
+  dfsan_origin o = dfsan_read_origin_of_first_taint(s1, pos);
+  *ret_origin = o ? o : dfsan_read_origin_of_first_taint(s2, pos);
+}
+
+static int dfsan_memcmp_bcmp_label(const void *s1, const void *s2, size_t n,
+                                   dfsan_label *ret_label) {
+  size_t bytes_read;
+  int r = dfsan_memcmp_bcmp(s1, s2, n, &bytes_read);
+  *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+  return r;
+}
+
+static int dfsan_memcmp_bcmp_origin(const void *s1, const void *s2, size_t n,
+                                    dfsan_label *ret_label,
+                                    dfsan_origin *ret_origin) {
+  size_t bytes_read;
+  int r = dfsan_memcmp_bcmp(s1, s2, n, &bytes_read);
+  dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+  return r;
+}
+
 DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_memcmp, uptr caller_pc,
                               const void *s1, const void *s2, size_t n,
                               dfsan_label s1_label, dfsan_label s2_label,
                               dfsan_label n_label)
 
+DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_memcmp, uptr caller_pc,
+                              const void *s1, const void *s2, size_t n,
+                              dfsan_label s1_label, dfsan_label s2_label,
+                              dfsan_label n_label, dfsan_origin s1_origin,
+                              dfsan_origin s2_origin, dfsan_origin n_origin)
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_memcmp(const void *s1, const void *s2,
                                                 size_t n, dfsan_label s1_label,
                                                 dfsan_label s2_label,
@@ -178,7 +268,18 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_memcmp(const void *s1, const void *s2,
                                                 dfsan_label *ret_label) {
   CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_memcmp, GET_CALLER_PC(), s1, s2, n,
                              s1_label, s2_label, n_label);
-  return dfsan_memcmp_bcmp(s1, s2, n, s1_label, s2_label, n_label, ret_label);
+  return dfsan_memcmp_bcmp_label(s1, s2, n, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_memcmp(
+    const void *s1, const void *s2, size_t n, dfsan_label s1_label,
+    dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+    dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+    dfsan_origin *ret_origin) {
+  CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_memcmp, GET_CALLER_PC(), s1,
+                             s2, n, s1_label, s2_label, n_label, s1_origin,
+                             s2_origin, n_origin);
+  return dfsan_memcmp_bcmp_origin(s1, s2, n, ret_label, ret_origin);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_bcmp(const void *s1, const void *s2,
@@ -186,51 +287,97 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_bcmp(const void *s1, const void *s2,
                                               dfsan_label s2_label,
                                               dfsan_label n_label,
                                               dfsan_label *ret_label) {
-  return dfsan_memcmp_bcmp(s1, s2, n, s1_label, s2_label, n_label, ret_label);
+  return dfsan_memcmp_bcmp_label(s1, s2, n, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_bcmp(
+    const void *s1, const void *s2, size_t n, dfsan_label s1_label,
+    dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+    dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+    dfsan_origin *ret_origin) {
+  return dfsan_memcmp_bcmp_origin(s1, s2, n, ret_label, ret_origin);
+}
+
+// When n == 0, compare strings without byte limit.
+// When n > 0, compare the first (at most) n bytes of s1 and s2.
+static int dfsan_strncmp(const char *s1, const char *s2, size_t n,
+                         size_t *bytes_read) {
+  for (size_t i = 0;; ++i) {
+    if (s1[i] != s2[i] || s1[i] == 0 || s2[i] == 0 || (n > 0 && i == n - 1)) {
+      *bytes_read = i + 1;
+      return s1[i] - s2[i];
+    }
+  }
 }
 
 DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strcmp, uptr caller_pc,
                               const char *s1, const char *s2,
                               dfsan_label s1_label, dfsan_label s2_label)
 
+DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_strcmp, uptr caller_pc,
+                              const char *s1, const char *s2,
+                              dfsan_label s1_label, dfsan_label s2_label,
+                              dfsan_origin s1_origin, dfsan_origin s2_origin)
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strcmp(const char *s1, const char *s2,
                                                 dfsan_label s1_label,
                                                 dfsan_label s2_label,
                                                 dfsan_label *ret_label) {
   CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strcmp, GET_CALLER_PC(), s1, s2,
                              s1_label, s2_label);
-  for (size_t i = 0;; ++i) {
-    if (s1[i] != s2[i] || s1[i] == 0 || s2[i] == 0) {
-      if (flags().strict_data_dependencies) {
-        *ret_label = 0;
-      } else {
-        *ret_label = dfsan_union(dfsan_read_label(s1, i + 1),
-                                 dfsan_read_label(s2, i + 1));
-      }
-      return s1[i] - s2[i];
-    }
-  }
-  return 0;
+  size_t bytes_read;
+  int r = dfsan_strncmp(s1, s2, 0, &bytes_read);
+  *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+  return r;
 }
 
-SANITIZER_INTERFACE_ATTRIBUTE int
-__dfsw_strcasecmp(const char *s1, const char *s2, dfsan_label s1_label,
-                  dfsan_label s2_label, dfsan_label *ret_label) {
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_strcmp(
+    const char *s1, const char *s2, dfsan_label s1_label, dfsan_label s2_label,
+    dfsan_label *ret_label, dfsan_origin s1_origin, dfsan_origin s2_origin,
+    dfsan_origin *ret_origin) {
+  CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_strcmp, GET_CALLER_PC(), s1,
+                             s2, s1_label, s2_label, s1_origin, s2_origin);
+  size_t bytes_read;
+  int r = dfsan_strncmp(s1, s2, 0, &bytes_read);
+  dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+  return r;
+}
+
+// When n == 0, compare strings without byte limit.
+// When n > 0, compare the first (at most) n bytes of s1 and s2.
+static int dfsan_strncasecmp(const char *s1, const char *s2, size_t n,
+                             size_t *bytes_read) {
   for (size_t i = 0;; ++i) {
     char s1_lower = tolower(s1[i]);
     char s2_lower = tolower(s2[i]);
 
-    if (s1_lower != s2_lower || s1[i] == 0 || s2[i] == 0) {
-      if (flags().strict_data_dependencies) {
-        *ret_label = 0;
-      } else {
-        *ret_label = dfsan_union(dfsan_read_label(s1, i + 1),
-                                 dfsan_read_label(s2, i + 1));
-      }
+    if (s1_lower != s2_lower || s1[i] == 0 || s2[i] == 0 ||
+        (n > 0 && i == n - 1)) {
+      *bytes_read = i + 1;
       return s1_lower - s2_lower;
     }
   }
-  return 0;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strcasecmp(const char *s1,
+                                                    const char *s2,
+                                                    dfsan_label s1_label,
+                                                    dfsan_label s2_label,
+                                                    dfsan_label *ret_label) {
+  size_t bytes_read;
+  int r = dfsan_strncasecmp(s1, s2, 0, &bytes_read);
+  *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+  return r;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_strcasecmp(
+    const char *s1, const char *s2, dfsan_label s1_label, dfsan_label s2_label,
+    dfsan_label *ret_label, dfsan_origin s1_origin, dfsan_origin s2_origin,
+    dfsan_origin *ret_origin) {
+  size_t bytes_read;
+  int r = dfsan_strncasecmp(s1, s2, 0, &bytes_read);
+  dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+  return r;
 }
 
 DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strncmp, uptr caller_pc,
@@ -238,6 +385,12 @@ DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strncmp, uptr caller_pc,
                               dfsan_label s1_label, dfsan_label s2_label,
                               dfsan_label n_label)
 
+DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_strncmp, uptr caller_pc,
+                              const char *s1, const char *s2, size_t n,
+                              dfsan_label s1_label, dfsan_label s2_label,
+                              dfsan_label n_label, dfsan_origin s1_origin,
+                              dfsan_origin s2_origin, dfsan_origin n_origin)
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strncmp(const char *s1, const char *s2,
                                                  size_t n, dfsan_label s1_label,
                                                  dfsan_label s2_label,
@@ -251,44 +404,60 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strncmp(const char *s1, const char *s2,
   CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strncmp, GET_CALLER_PC(), s1, s2,
                              n, s1_label, s2_label, n_label);
 
-  for (size_t i = 0;; ++i) {
-    if (s1[i] != s2[i] || s1[i] == 0 || s2[i] == 0 || i == n - 1) {
-      if (flags().strict_data_dependencies) {
-        *ret_label = 0;
-      } else {
-        *ret_label = dfsan_union(dfsan_read_label(s1, i + 1),
-                                 dfsan_read_label(s2, i + 1));
-      }
-      return s1[i] - s2[i];
-    }
+  size_t bytes_read;
+  int r = dfsan_strncmp(s1, s2, n, &bytes_read);
+  *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+  return r;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_strncmp(
+    const char *s1, const char *s2, size_t n, dfsan_label s1_label,
+    dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+    dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+    dfsan_origin *ret_origin) {
+  if (n == 0) {
+    *ret_label = 0;
+    return 0;
   }
-  return 0;
+
+  CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_strncmp, GET_CALLER_PC(),
+                             s1, s2, n, s1_label, s2_label, n_label, s1_origin,
+                             s2_origin, n_origin);
+
+  size_t bytes_read;
+  int r = dfsan_strncmp(s1, s2, n, &bytes_read);
+  dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+  return r;
 }
 
-SANITIZER_INTERFACE_ATTRIBUTE int
-__dfsw_strncasecmp(const char *s1, const char *s2, size_t n,
-                   dfsan_label s1_label, dfsan_label s2_label,
-                   dfsan_label n_label, dfsan_label *ret_label) {
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strncasecmp(
+    const char *s1, const char *s2, size_t n, dfsan_label s1_label,
+    dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label) {
   if (n == 0) {
     *ret_label = 0;
     return 0;
   }
 
-  for (size_t i = 0;; ++i) {
-    char s1_lower = tolower(s1[i]);
-    char s2_lower = tolower(s2[i]);
+  size_t bytes_read;
+  int r = dfsan_strncasecmp(s1, s2, n, &bytes_read);
+  *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+  return r;
+}
 
-    if (s1_lower != s2_lower || s1[i] == 0 || s2[i] == 0 || i == n - 1) {
-      if (flags().strict_data_dependencies) {
-        *ret_label = 0;
-      } else {
-        *ret_label = dfsan_union(dfsan_read_label(s1, i + 1),
-                                 dfsan_read_label(s2, i + 1));
-      }
-      return s1_lower - s2_lower;
-    }
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_strncasecmp(
+    const char *s1, const char *s2, size_t n, dfsan_label s1_label,
+    dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+    dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+    dfsan_origin *ret_origin) {
+  if (n == 0) {
+    *ret_label = 0;
+    return 0;
   }
-  return 0;
+
+  size_t bytes_read;
+  int r = dfsan_strncasecmp(s1, s2, n, &bytes_read);
+  dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+  return r;
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE void *__dfsw_calloc(size_t nmemb, size_t size,
@@ -301,6 +470,15 @@ SANITIZER_INTERFACE_ATTRIBUTE void *__dfsw_calloc(size_t nmemb, size_t size,
   return p;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE void *__dfso_calloc(
+    size_t nmemb, size_t size, dfsan_label nmemb_label, dfsan_label size_label,
+    dfsan_label *ret_label, dfsan_origin nmemb_origin, dfsan_origin size_origin,
+    dfsan_origin *ret_origin) {
+  void *p = __dfsw_calloc(nmemb, size, nmemb_label, size_label, ret_label);
+  *ret_origin = 0;
+  return p;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE size_t
 __dfsw_strlen(const char *s, dfsan_label s_label, dfsan_label *ret_label) {
   size_t ret = strlen(s);
@@ -330,6 +508,11 @@ static void *dfsan_memmove(void *dest, const void *src, size_t n) {
   return internal_memmove(dest, src, n);
 }
 
+static void *dfsan_memmove_with_origin(void *dest, const void *src, size_t n) {
+  dfsan_mem_origin_transfer(dest, src, n);
+  return dfsan_memmove(dest, src, n);
+}
+
 static void *dfsan_memcpy(void *dest, const void *src, size_t n) {
   dfsan_label *sdest = shadow_for(dest);
   const dfsan_label *ssrc = shadow_for(src);
@@ -337,11 +520,22 @@ static void *dfsan_memcpy(void *dest, const void *src, size_t n) {
   return internal_memcpy(dest, src, n);
 }
 
+static void *dfsan_memcpy_with_origin(void *dest, const void *src, size_t n) {
+  dfsan_mem_origin_transfer(dest, src, n);
+  return dfsan_memcpy(dest, src, n);
+}
+
 static void dfsan_memset(void *s, int c, dfsan_label c_label, size_t n) {
   internal_memset(s, c, n);
   dfsan_set_label(c_label, s, n);
 }
 
+static void dfsan_memset_with_origin(void *s, int c, dfsan_label c_label,
+                                     dfsan_origin c_origin, size_t n) {
+  internal_memset(s, c, n);
+  dfsan_set_label_origin(c_label, c_origin, s, n);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__dfsw_memcpy(void *dest, const void *src, size_t n,
                     dfsan_label dest_label, dfsan_label src_label,
@@ -350,6 +544,17 @@ void *__dfsw_memcpy(void *dest, const void *src, size_t n,
   return dfsan_memcpy(dest, src, n);
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__dfso_memcpy(void *dest, const void *src, size_t n,
+                    dfsan_label dest_label, dfsan_label src_label,
+                    dfsan_label n_label, dfsan_label *ret_label,
+                    dfsan_origin dest_origin, dfsan_origin src_origin,
+                    dfsan_origin n_origin, dfsan_origin *ret_origin) {
+  *ret_label = dest_label;
+  *ret_origin = dest_origin;
+  return dfsan_memcpy_with_origin(dest, src, n);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__dfsw_memmove(void *dest, const void *src, size_t n,
                      dfsan_label dest_label, dfsan_label src_label,
@@ -358,6 +563,17 @@ void *__dfsw_memmove(void *dest, const void *src, size_t n,
   return dfsan_memmove(dest, src, n);
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__dfso_memmove(void *dest, const void *src, size_t n,
+                     dfsan_label dest_label, dfsan_label src_label,
+                     dfsan_label n_label, dfsan_label *ret_label,
+                     dfsan_origin dest_origin, dfsan_origin src_origin,
+                     dfsan_origin n_origin, dfsan_origin *ret_origin) {
+  *ret_label = dest_label;
+  *ret_origin = dest_origin;
+  return dfsan_memmove_with_origin(dest, src, n);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__dfsw_memset(void *s, int c, size_t n,
                     dfsan_label s_label, dfsan_label c_label,
@@ -367,6 +583,18 @@ void *__dfsw_memset(void *s, int c, size_t n,
   return s;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__dfso_memset(void *s, int c, size_t n, dfsan_label s_label,
+                    dfsan_label c_label, dfsan_label n_label,
+                    dfsan_label *ret_label, dfsan_origin s_origin,
+                    dfsan_origin c_origin, dfsan_origin n_origin,
+                    dfsan_origin *ret_origin) {
+  dfsan_memset_with_origin(s, c, c_label, c_origin, n);
+  *ret_label = s_label;
+  *ret_origin = s_origin;
+  return s;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strcat(char *dest, const char *src,
                                                   dfsan_label dest_label,
                                                   dfsan_label src_label,
@@ -381,6 +609,23 @@ SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strcat(char *dest, const char *src,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strcat(
+    char *dest, const char *src, dfsan_label dest_label, dfsan_label src_label,
+    dfsan_label *ret_label, dfsan_origin dest_origin, dfsan_origin src_origin,
+    dfsan_origin *ret_origin) {
+  size_t dest_len = strlen(dest);
+  char *ret = strcat(dest, src);  // NOLINT
+  dfsan_label *sdest = shadow_for(dest + dest_len);
+  const dfsan_label *ssrc = shadow_for(src);
+  size_t src_len = strlen(src);
+  dfsan_mem_origin_transfer(dest + dest_len, src, src_len);
+  internal_memcpy((void *)sdest, (const void *)ssrc,
+                  src_len * sizeof(dfsan_label));
+  *ret_label = dest_label;
+  *ret_origin = dest_origin;
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE char *
 __dfsw_strdup(const char *s, dfsan_label s_label, dfsan_label *ret_label) {
   size_t len = strlen(s);
@@ -390,6 +635,18 @@ __dfsw_strdup(const char *s, dfsan_label s_label, dfsan_label *ret_label) {
   return static_cast<char *>(p);
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strdup(const char *s,
+                                                  dfsan_label s_label,
+                                                  dfsan_label *ret_label,
+                                                  dfsan_origin s_origin,
+                                                  dfsan_origin *ret_origin) {
+  size_t len = strlen(s);
+  void *p = malloc(len + 1);
+  dfsan_memcpy_with_origin(p, s, len + 1);
+  *ret_label = 0;
+  return static_cast<char *>(p);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE char *
 __dfsw_strncpy(char *s1, const char *s2, size_t n, dfsan_label s1_label,
                dfsan_label s2_label, dfsan_label n_label,
@@ -406,6 +663,24 @@ __dfsw_strncpy(char *s1, const char *s2, size_t n, dfsan_label s1_label,
   return s1;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strncpy(
+    char *s1, const char *s2, size_t n, dfsan_label s1_label,
+    dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+    dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+    dfsan_origin *ret_origin) {
+  size_t len = strlen(s2);
+  if (len < n) {
+    dfsan_memcpy_with_origin(s1, s2, len + 1);
+    dfsan_memset_with_origin(s1 + len + 1, 0, 0, 0, n - len - 1);
+  } else {
+    dfsan_memcpy_with_origin(s1, s2, n);
+  }
+
+  *ret_label = s1_label;
+  *ret_origin = s1_origin;
+  return s1;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE ssize_t
 __dfsw_pread(int fd, void *buf, size_t count, off_t offset,
              dfsan_label fd_label, dfsan_label buf_label,
@@ -418,6 +693,16 @@ __dfsw_pread(int fd, void *buf, size_t count, off_t offset,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfso_pread(
+    int fd, void *buf, size_t count, off_t offset, dfsan_label fd_label,
+    dfsan_label buf_label, dfsan_label count_label, dfsan_label offset_label,
+    dfsan_label *ret_label, dfsan_origin fd_origin, dfsan_origin buf_origin,
+    dfsan_origin count_origin, dfsan_label offset_origin,
+    dfsan_origin *ret_origin) {
+  return __dfsw_pread(fd, buf, count, offset, fd_label, buf_label, count_label,
+                      offset_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE ssize_t
 __dfsw_read(int fd, void *buf, size_t count,
              dfsan_label fd_label, dfsan_label buf_label,
@@ -430,6 +715,15 @@ __dfsw_read(int fd, void *buf, size_t count,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfso_read(
+    int fd, void *buf, size_t count, dfsan_label fd_label,
+    dfsan_label buf_label, dfsan_label count_label, dfsan_label *ret_label,
+    dfsan_origin fd_origin, dfsan_origin buf_origin, dfsan_origin count_origin,
+    dfsan_origin *ret_origin) {
+  return __dfsw_read(fd, buf, count, fd_label, buf_label, count_label,
+                     ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_clock_gettime(clockid_t clk_id,
                                                        struct timespec *tp,
                                                        dfsan_label clk_id_label,
@@ -442,7 +736,14 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_clock_gettime(clockid_t clk_id,
   return ret;
 }
 
-static void unpoison(const void *ptr, uptr size) {
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_clock_gettime(
+    clockid_t clk_id, struct timespec *tp, dfsan_label clk_id_label,
+    dfsan_label tp_label, dfsan_label *ret_label, dfsan_origin clk_id_origin,
+    dfsan_origin tp_origin, dfsan_origin *ret_origin) {
+  return __dfsw_clock_gettime(clk_id, tp, clk_id_label, tp_label, ret_label);
+}
+
+static void dfsan_set_zero_label(const void *ptr, uptr size) {
   dfsan_set_label(0, const_cast<void *>(ptr), size);
 }
 
@@ -455,11 +756,19 @@ __dfsw_dlopen(const char *filename, int flag, dfsan_label filename_label,
   void *handle = dlopen(filename, flag);
   link_map *map = GET_LINK_MAP_BY_DLOPEN_HANDLE(handle);
   if (map)
-    ForEachMappedRegion(map, unpoison);
+    ForEachMappedRegion(map, dfsan_set_zero_label);
   *ret_label = 0;
   return handle;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE void *__dfso_dlopen(
+    const char *filename, int flag, dfsan_label filename_label,
+    dfsan_label flag_label, dfsan_label *ret_label,
+    dfsan_origin filename_origin, dfsan_origin flag_origin,
+    dfsan_origin *ret_origin) {
+  return __dfsw_dlopen(filename, flag, filename_label, flag_label, ret_label);
+}
+
 static void *DFsanThreadStartFunc(void *arg) {
   DFsanThread *t = (DFsanThread *)arg;
   SetCurrentThread(t);
@@ -546,6 +855,17 @@ struct dl_iterate_phdr_info {
   void *data;
 };
 
+struct dl_iterate_phdr_origin_info {
+  int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
+                             size_t size, void *data, dfsan_label info_label,
+                             dfsan_label size_label, dfsan_label data_label,
+                             dfsan_label *ret_label, dfsan_origin info_origin,
+                             dfsan_origin size_origin, dfsan_origin data_origin,
+                             dfsan_origin *ret_origin);
+  void *callback;
+  void *data;
+};
+
 int dl_iterate_phdr_cb(struct dl_phdr_info *info, size_t size, void *data) {
   dl_iterate_phdr_info *dipi = (dl_iterate_phdr_info *)data;
   dfsan_set_label(0, *info);
@@ -559,6 +879,21 @@ int dl_iterate_phdr_cb(struct dl_phdr_info *info, size_t size, void *data) {
                                    0, &ret_label);
 }
 
+int dl_iterate_phdr_origin_cb(struct dl_phdr_info *info, size_t size,
+                              void *data) {
+  dl_iterate_phdr_origin_info *dipi = (dl_iterate_phdr_origin_info *)data;
+  dfsan_set_label(0, *info);
+  dfsan_set_label(0, const_cast<char *>(info->dlpi_name),
+                  strlen(info->dlpi_name) + 1);
+  dfsan_set_label(
+      0, const_cast<char *>(reinterpret_cast<const char *>(info->dlpi_phdr)),
+      sizeof(*info->dlpi_phdr) * info->dlpi_phnum);
+  dfsan_label ret_label;
+  dfsan_origin ret_origin;
+  return dipi->callback_trampoline(dipi->callback, info, size, dipi->data, 0, 0,
+                                   0, &ret_label, 0, 0, 0, &ret_origin);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_dl_iterate_phdr(
     int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
                                size_t size, void *data, dfsan_label info_label,
@@ -571,6 +906,23 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_dl_iterate_phdr(
   return dl_iterate_phdr(dl_iterate_phdr_cb, &dipi);
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_dl_iterate_phdr(
+    int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
+                               size_t size, void *data, dfsan_label info_label,
+                               dfsan_label size_label, dfsan_label data_label,
+                               dfsan_label *ret_label, dfsan_origin info_origin,
+                               dfsan_origin size_origin,
+                               dfsan_origin data_origin,
+                               dfsan_origin *ret_origin),
+    void *callback, void *data, dfsan_label callback_label,
+    dfsan_label data_label, dfsan_label *ret_label,
+    dfsan_origin callback_origin, dfsan_origin data_origin,
+    dfsan_origin *ret_origin) {
+  dl_iterate_phdr_origin_info dipi = {callback_trampoline, callback, data};
+  *ret_label = 0;
+  return dl_iterate_phdr(dl_iterate_phdr_origin_cb, &dipi);
+}
+
 // This function is only available for glibc 2.27 or newer.  Mark it weak so
 // linking succeeds with older glibcs.
 SANITIZER_WEAK_ATTRIBUTE void _dl_get_tls_static_info(size_t *sizep,
@@ -585,6 +937,13 @@ SANITIZER_INTERFACE_ATTRIBUTE void __dfsw__dl_get_tls_static_info(
   dfsan_set_label(0, alignp, sizeof(*alignp));
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE void __dfso__dl_get_tls_static_info(
+    size_t *sizep, size_t *alignp, dfsan_label sizep_label,
+    dfsan_label alignp_label, dfsan_origin sizep_origin,
+    dfsan_origin alignp_origin) {
+  __dfsw__dl_get_tls_static_info(sizep, alignp, sizep_label, alignp_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 char *__dfsw_ctime_r(const time_t *timep, char *buf, dfsan_label timep_label,
                      dfsan_label buf_label, dfsan_label *ret_label) {
@@ -599,6 +958,25 @@ char *__dfsw_ctime_r(const time_t *timep, char *buf, dfsan_label timep_label,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+char *__dfso_ctime_r(const time_t *timep, char *buf, dfsan_label timep_label,
+                     dfsan_label buf_label, dfsan_label *ret_label,
+                     dfsan_origin timep_origin, dfsan_origin buf_origin,
+                     dfsan_origin *ret_origin) {
+  char *ret = ctime_r(timep, buf);
+  if (ret) {
+    dfsan_set_label_origin(
+        dfsan_read_label(timep, sizeof(time_t)),
+        dfsan_read_origin_of_first_taint(timep, sizeof(time_t)), buf,
+        strlen(buf) + 1);
+    *ret_label = buf_label;
+    *ret_origin = buf_origin;
+  } else {
+    *ret_label = 0;
+  }
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 char *__dfsw_fgets(char *s, int size, FILE *stream, dfsan_label s_label,
                    dfsan_label size_label, dfsan_label stream_label,
@@ -613,6 +991,19 @@ char *__dfsw_fgets(char *s, int size, FILE *stream, dfsan_label s_label,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+char *__dfso_fgets(char *s, int size, FILE *stream, dfsan_label s_label,
+                   dfsan_label size_label, dfsan_label stream_label,
+                   dfsan_label *ret_label, dfsan_origin s_origin,
+                   dfsan_origin size_origin, dfsan_origin stream_origin,
+                   dfsan_origin *ret_origin) {
+  char *ret = __dfsw_fgets(s, size, stream, s_label, size_label, stream_label,
+                           ret_label);
+  if (ret)
+    *ret_origin = s_origin;
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 char *__dfsw_getcwd(char *buf, size_t size, dfsan_label buf_label,
                     dfsan_label size_label, dfsan_label *ret_label) {
@@ -626,16 +1017,32 @@ char *__dfsw_getcwd(char *buf, size_t size, dfsan_label buf_label,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+char *__dfso_getcwd(char *buf, size_t size, dfsan_label buf_label,
+                    dfsan_label size_label, dfsan_label *ret_label,
+                    dfsan_origin buf_origin, dfsan_origin size_origin,
+                    dfsan_origin *ret_origin) {
+  char *ret = __dfsw_getcwd(buf, size, buf_label, size_label, ret_label);
+  if (ret)
+    *ret_origin = buf_origin;
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 char *__dfsw_get_current_dir_name(dfsan_label *ret_label) {
   char *ret = get_current_dir_name();
-  if (ret) {
+  if (ret)
     dfsan_set_label(0, ret, strlen(ret) + 1);
-  }
   *ret_label = 0;
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+char *__dfso_get_current_dir_name(dfsan_label *ret_label,
+                                  dfsan_origin *ret_origin) {
+  return __dfsw_get_current_dir_name(ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_gethostname(char *name, size_t len, dfsan_label name_label,
                        dfsan_label len_label, dfsan_label *ret_label) {
@@ -647,6 +1054,14 @@ int __dfsw_gethostname(char *name, size_t len, dfsan_label name_label,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_gethostname(char *name, size_t len, dfsan_label name_label,
+                       dfsan_label len_label, dfsan_label *ret_label,
+                       dfsan_origin name_origin, dfsan_origin len_origin,
+                       dfsan_label *ret_origin) {
+  return __dfsw_gethostname(name, len, name_label, len_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_getrlimit(int resource, struct rlimit *rlim,
                      dfsan_label resource_label, dfsan_label rlim_label,
@@ -659,6 +1074,15 @@ int __dfsw_getrlimit(int resource, struct rlimit *rlim,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_getrlimit(int resource, struct rlimit *rlim,
+                     dfsan_label resource_label, dfsan_label rlim_label,
+                     dfsan_label *ret_label, dfsan_origin resource_origin,
+                     dfsan_origin rlim_origin, dfsan_origin *ret_origin) {
+  return __dfsw_getrlimit(resource, rlim, resource_label, rlim_label,
+                          ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_getrusage(int who, struct rusage *usage, dfsan_label who_label,
                      dfsan_label usage_label, dfsan_label *ret_label) {
@@ -670,6 +1094,14 @@ int __dfsw_getrusage(int who, struct rusage *usage, dfsan_label who_label,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_getrusage(int who, struct rusage *usage, dfsan_label who_label,
+                     dfsan_label usage_label, dfsan_label *ret_label,
+                     dfsan_origin who_origin, dfsan_origin usage_origin,
+                     dfsan_label *ret_origin) {
+  return __dfsw_getrusage(who, usage, who_label, usage_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 char *__dfsw_strcpy(char *dest, const char *src, dfsan_label dst_label,
                     dfsan_label src_label, dfsan_label *ret_label) {
@@ -683,14 +1115,34 @@ char *__dfsw_strcpy(char *dest, const char *src, dfsan_label dst_label,
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
-long int __dfsw_strtol(const char *nptr, char **endptr, int base,
-                       dfsan_label nptr_label, dfsan_label endptr_label,
-                       dfsan_label base_label, dfsan_label *ret_label) {
-  char *tmp_endptr;
-  long int ret = strtol(nptr, &tmp_endptr, base);
-  if (endptr) {
-    *endptr = tmp_endptr;
+char *__dfso_strcpy(char *dest, const char *src, dfsan_label dst_label,
+                    dfsan_label src_label, dfsan_label *ret_label,
+                    dfsan_origin dst_origin, dfsan_origin src_origin,
+                    dfsan_origin *ret_origin) {
+  char *ret = strcpy(dest, src);  // NOLINT
+  if (ret) {
+    size_t str_len = strlen(src) + 1;
+    dfsan_mem_origin_transfer(dest, src, str_len);
+    internal_memcpy(shadow_for(dest), shadow_for(src),
+                    sizeof(dfsan_label) * str_len);
   }
+  *ret_label = dst_label;
+  *ret_origin = dst_origin;
+  return ret;
+}
+
+static long int dfsan_strtol(const char *nptr, char **endptr, int base,
+                             char **tmp_endptr) {
+  assert(tmp_endptr);
+  long int ret = strtol(nptr, tmp_endptr, base);
+  if (endptr)
+    *endptr = *tmp_endptr;
+  return ret;
+}
+
+static void dfsan_strtolong_label(const char *nptr, const char *tmp_endptr,
+                                  dfsan_label base_label,
+                                  dfsan_label *ret_label) {
   if (tmp_endptr > nptr) {
     // If *tmp_endptr is '\0' include its label as well.
     *ret_label = dfsan_union(
@@ -699,18 +1151,58 @@ long int __dfsw_strtol(const char *nptr, char **endptr, int base,
   } else {
     *ret_label = 0;
   }
+}
+
+static void dfsan_strtolong_origin(const char *nptr, const char *tmp_endptr,
+                                   dfsan_label base_label,
+                                   dfsan_label *ret_label,
+                                   dfsan_origin base_origin,
+                                   dfsan_origin *ret_origin) {
+  if (tmp_endptr > nptr) {
+    // When multiple inputs are tainted, we propagate one of its origins.
+    // Because checking if base_label is tainted does not need additional
+    // computation, we prefer to propagating base_origin.
+    *ret_origin = base_label
+                      ? base_origin
+                      : dfsan_read_origin_of_first_taint(
+                            nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1));
+  }
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+long int __dfsw_strtol(const char *nptr, char **endptr, int base,
+                       dfsan_label nptr_label, dfsan_label endptr_label,
+                       dfsan_label base_label, dfsan_label *ret_label) {
+  char *tmp_endptr;
+  long int ret = dfsan_strtol(nptr, endptr, base, &tmp_endptr);
+  dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
   return ret;
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
-double __dfsw_strtod(const char *nptr, char **endptr,
+long int __dfso_strtol(const char *nptr, char **endptr, int base,
                        dfsan_label nptr_label, dfsan_label endptr_label,
-                       dfsan_label *ret_label) {
+                       dfsan_label base_label, dfsan_label *ret_label,
+                       dfsan_origin nptr_origin, dfsan_origin endptr_origin,
+                       dfsan_origin base_origin, dfsan_origin *ret_origin) {
   char *tmp_endptr;
-  double ret = strtod(nptr, &tmp_endptr);
-  if (endptr) {
-    *endptr = tmp_endptr;
-  }
+  long int ret = dfsan_strtol(nptr, endptr, base, &tmp_endptr);
+  dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+  dfsan_strtolong_origin(nptr, tmp_endptr, base_label, ret_label, base_origin,
+                         ret_origin);
+  return ret;
+}
+
+static double dfsan_strtod(const char *nptr, char **endptr, char **tmp_endptr) {
+  assert(tmp_endptr);
+  double ret = strtod(nptr, tmp_endptr);
+  if (endptr)
+    *endptr = *tmp_endptr;
+  return ret;
+}
+
+static void dfsan_strtod_label(const char *nptr, const char *tmp_endptr,
+                               dfsan_label *ret_label) {
   if (tmp_endptr > nptr) {
     // If *tmp_endptr is '\0' include its label as well.
     *ret_label = dfsan_read_label(
@@ -719,46 +1211,109 @@ double __dfsw_strtod(const char *nptr, char **endptr,
   } else {
     *ret_label = 0;
   }
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+double __dfsw_strtod(const char *nptr, char **endptr, dfsan_label nptr_label,
+                     dfsan_label endptr_label, dfsan_label *ret_label) {
+  char *tmp_endptr;
+  double ret = dfsan_strtod(nptr, endptr, &tmp_endptr);
+  dfsan_strtod_label(nptr, tmp_endptr, ret_label);
   return ret;
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
-long long int __dfsw_strtoll(const char *nptr, char **endptr, int base,
-                       dfsan_label nptr_label, dfsan_label endptr_label,
-                       dfsan_label base_label, dfsan_label *ret_label) {
+double __dfso_strtod(const char *nptr, char **endptr, dfsan_label nptr_label,
+                     dfsan_label endptr_label, dfsan_label *ret_label,
+                     dfsan_origin nptr_origin, dfsan_origin endptr_origin,
+                     dfsan_origin *ret_origin) {
   char *tmp_endptr;
-  long long int ret = strtoll(nptr, &tmp_endptr, base);
-  if (endptr) {
-    *endptr = tmp_endptr;
-  }
+  double ret = dfsan_strtod(nptr, endptr, &tmp_endptr);
+  dfsan_strtod_label(nptr, tmp_endptr, ret_label);
   if (tmp_endptr > nptr) {
     // If *tmp_endptr is '\0' include its label as well.
-    *ret_label = dfsan_union(
-        base_label,
-        dfsan_read_label(nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1)));
+    *ret_origin = dfsan_read_origin_of_first_taint(
+        nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1));
   } else {
-    *ret_label = 0;
+    *ret_origin = 0;
   }
   return ret;
 }
 
+static long long int dfsan_strtoll(const char *nptr, char **endptr, int base,
+                                   char **tmp_endptr) {
+  assert(tmp_endptr);
+  long long int ret = strtoll(nptr, tmp_endptr, base);
+  if (endptr)
+    *endptr = *tmp_endptr;
+  return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+long long int __dfsw_strtoll(const char *nptr, char **endptr, int base,
+                             dfsan_label nptr_label, dfsan_label endptr_label,
+                             dfsan_label base_label, dfsan_label *ret_label) {
+  char *tmp_endptr;
+  long long int ret = dfsan_strtoll(nptr, endptr, base, &tmp_endptr);
+  dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+  return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+long long int __dfso_strtoll(const char *nptr, char **endptr, int base,
+                             dfsan_label nptr_label, dfsan_label endptr_label,
+                             dfsan_label base_label, dfsan_label *ret_label,
+                             dfsan_origin nptr_origin,
+                             dfsan_origin endptr_origin,
+                             dfsan_origin base_origin,
+                             dfsan_origin *ret_origin) {
+  char *tmp_endptr;
+  long long int ret = dfsan_strtoll(nptr, endptr, base, &tmp_endptr);
+  dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+  dfsan_strtolong_origin(nptr, tmp_endptr, base_label, ret_label, base_origin,
+                         ret_origin);
+  return ret;
+}
+
+static unsigned long int dfsan_strtoul(const char *nptr, char **endptr,
+                                       int base, char **tmp_endptr) {
+  assert(tmp_endptr);
+  unsigned long int ret = strtoul(nptr, tmp_endptr, base);
+  if (endptr)
+    *endptr = *tmp_endptr;
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 unsigned long int __dfsw_strtoul(const char *nptr, char **endptr, int base,
                        dfsan_label nptr_label, dfsan_label endptr_label,
                        dfsan_label base_label, dfsan_label *ret_label) {
   char *tmp_endptr;
-  unsigned long int ret = strtoul(nptr, &tmp_endptr, base);
-  if (endptr) {
-    *endptr = tmp_endptr;
-  }
-  if (tmp_endptr > nptr) {
-    // If *tmp_endptr is '\0' include its label as well.
-    *ret_label = dfsan_union(
-        base_label,
-        dfsan_read_label(nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1)));
-  } else {
-    *ret_label = 0;
-  }
+  unsigned long int ret = dfsan_strtoul(nptr, endptr, base, &tmp_endptr);
+  dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+  return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+unsigned long int __dfso_strtoul(
+    const char *nptr, char **endptr, int base, dfsan_label nptr_label,
+    dfsan_label endptr_label, dfsan_label base_label, dfsan_label *ret_label,
+    dfsan_origin nptr_origin, dfsan_origin endptr_origin,
+    dfsan_origin base_origin, dfsan_origin *ret_origin) {
+  char *tmp_endptr;
+  unsigned long int ret = dfsan_strtoul(nptr, endptr, base, &tmp_endptr);
+  dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+  dfsan_strtolong_origin(nptr, tmp_endptr, base_label, ret_label, base_origin,
+                         ret_origin);
+  return ret;
+}
+
+static long long unsigned int dfsan_strtoull(const char *nptr, char **endptr,
+                                             int base, char **tmp_endptr) {
+  assert(tmp_endptr);
+  long long unsigned int ret = strtoull(nptr, tmp_endptr, base);
+  if (endptr)
+    *endptr = *tmp_endptr;
   return ret;
 }
 
@@ -769,18 +1324,22 @@ long long unsigned int __dfsw_strtoull(const char *nptr, char **endptr,
                                        dfsan_label base_label,
                                        dfsan_label *ret_label) {
   char *tmp_endptr;
-  long long unsigned int ret = strtoull(nptr, &tmp_endptr, base);
-  if (endptr) {
-    *endptr = tmp_endptr;
-  }
-  if (tmp_endptr > nptr) {
-    // If *tmp_endptr is '\0' include its label as well.
-    *ret_label = dfsan_union(
-        base_label,
-        dfsan_read_label(nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1)));
-  } else {
-    *ret_label = 0;
-  }
+  long long unsigned int ret = dfsan_strtoull(nptr, endptr, base, &tmp_endptr);
+  dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+  return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+long long unsigned int __dfso_strtoull(
+    const char *nptr, char **endptr, int base, dfsan_label nptr_label,
+    dfsan_label endptr_label, dfsan_label base_label, dfsan_label *ret_label,
+    dfsan_origin nptr_origin, dfsan_origin endptr_origin,
+    dfsan_origin base_origin, dfsan_origin *ret_origin) {
+  char *tmp_endptr;
+  long long unsigned int ret = dfsan_strtoull(nptr, endptr, base, &tmp_endptr);
+  dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+  dfsan_strtolong_origin(nptr, tmp_endptr, base_label, ret_label, base_origin,
+                         ret_origin);
   return ret;
 }
 
@@ -794,6 +1353,12 @@ time_t __dfsw_time(time_t *t, dfsan_label t_label, dfsan_label *ret_label) {
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+time_t __dfso_time(time_t *t, dfsan_label t_label, dfsan_label *ret_label,
+                   dfsan_origin t_origin, dfsan_origin *ret_origin) {
+  return __dfsw_time(t, t_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_inet_pton(int af, const char *src, void *dst, dfsan_label af_label,
                      dfsan_label src_label, dfsan_label dst_label,
@@ -807,6 +1372,24 @@ int __dfsw_inet_pton(int af, const char *src, void *dst, dfsan_label af_label,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_inet_pton(int af, const char *src, void *dst, dfsan_label af_label,
+                     dfsan_label src_label, dfsan_label dst_label,
+                     dfsan_label *ret_label, dfsan_origin af_origin,
+                     dfsan_origin src_origin, dfsan_origin dst_origin,
+                     dfsan_origin *ret_origin) {
+  int ret = inet_pton(af, src, dst);
+  if (ret == 1) {
+    int src_len = strlen(src) + 1;
+    dfsan_set_label_origin(
+        dfsan_read_label(src, src_len),
+        dfsan_read_origin_of_first_taint(src, src_len), dst,
+        af == AF_INET ? sizeof(struct in_addr) : sizeof(in6_addr));
+  }
+  *ret_label = 0;
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 struct tm *__dfsw_localtime_r(const time_t *timep, struct tm *result,
                               dfsan_label timep_label, dfsan_label result_label,
@@ -822,6 +1405,26 @@ struct tm *__dfsw_localtime_r(const time_t *timep, struct tm *result,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+struct tm *__dfso_localtime_r(const time_t *timep, struct tm *result,
+                              dfsan_label timep_label, dfsan_label result_label,
+                              dfsan_label *ret_label, dfsan_origin timep_origin,
+                              dfsan_origin result_origin,
+                              dfsan_origin *ret_origin) {
+  struct tm *ret = localtime_r(timep, result);
+  if (ret) {
+    dfsan_set_label_origin(
+        dfsan_read_label(timep, sizeof(time_t)),
+        dfsan_read_origin_of_first_taint(timep, sizeof(time_t)), result,
+        sizeof(struct tm));
+    *ret_label = result_label;
+    *ret_origin = result_origin;
+  } else {
+    *ret_label = 0;
+  }
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_getpwuid_r(id_t uid, struct passwd *pwd,
                       char *buf, size_t buflen, struct passwd **result,
@@ -840,6 +1443,19 @@ int __dfsw_getpwuid_r(id_t uid, struct passwd *pwd,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_getpwuid_r(id_t uid, struct passwd *pwd, char *buf, size_t buflen,
+                      struct passwd **result, dfsan_label uid_label,
+                      dfsan_label pwd_label, dfsan_label buf_label,
+                      dfsan_label buflen_label, dfsan_label result_label,
+                      dfsan_label *ret_label, dfsan_origin uid_origin,
+                      dfsan_origin pwd_origin, dfsan_origin buf_origin,
+                      dfsan_origin buflen_origin, dfsan_origin result_origin,
+                      dfsan_origin *ret_origin) {
+  return __dfsw_getpwuid_r(uid, pwd, buf, buflen, result, uid_label, pwd_label,
+                           buf_label, buflen_label, result_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_epoll_wait(int epfd, struct epoll_event *events, int maxevents,
                       int timeout, dfsan_label epfd_label,
@@ -852,6 +1468,19 @@ int __dfsw_epoll_wait(int epfd, struct epoll_event *events, int maxevents,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_epoll_wait(int epfd, struct epoll_event *events, int maxevents,
+                      int timeout, dfsan_label epfd_label,
+                      dfsan_label events_label, dfsan_label maxevents_label,
+                      dfsan_label timeout_label, dfsan_label *ret_label,
+                      dfsan_origin epfd_origin, dfsan_origin events_origin,
+                      dfsan_origin maxevents_origin,
+                      dfsan_origin timeout_origin, dfsan_origin *ret_origin) {
+  return __dfsw_epoll_wait(epfd, events, maxevents, timeout, epfd_label,
+                           events_label, maxevents_label, timeout_label,
+                           ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_poll(struct pollfd *fds, nfds_t nfds, int timeout,
                 dfsan_label dfs_label, dfsan_label nfds_label,
@@ -866,6 +1495,16 @@ int __dfsw_poll(struct pollfd *fds, nfds_t nfds, int timeout,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_poll(struct pollfd *fds, nfds_t nfds, int timeout,
+                dfsan_label dfs_label, dfsan_label nfds_label,
+                dfsan_label timeout_label, dfsan_label *ret_label,
+                dfsan_origin dfs_origin, dfsan_origin nfds_origin,
+                dfsan_origin timeout_origin, dfsan_origin *ret_origin) {
+  return __dfsw_poll(fds, nfds, timeout, dfs_label, nfds_label, timeout_label,
+                     ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_select(int nfds, fd_set *readfds, fd_set *writefds,
                   fd_set *exceptfds, struct timeval *timeout,
@@ -889,6 +1528,20 @@ int __dfsw_select(int nfds, fd_set *readfds, fd_set *writefds,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_select(int nfds, fd_set *readfds, fd_set *writefds,
+                  fd_set *exceptfds, struct timeval *timeout,
+                  dfsan_label nfds_label, dfsan_label readfds_label,
+                  dfsan_label writefds_label, dfsan_label exceptfds_label,
+                  dfsan_label timeout_label, dfsan_label *ret_label,
+                  dfsan_origin nfds_origin, dfsan_origin readfds_origin,
+                  dfsan_origin writefds_origin, dfsan_origin exceptfds_origin,
+                  dfsan_origin timeout_origin, dfsan_origin *ret_origin) {
+  return __dfsw_select(nfds, readfds, writefds, exceptfds, timeout, nfds_label,
+                       readfds_label, writefds_label, exceptfds_label,
+                       timeout_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t *mask,
                              dfsan_label pid_label,
@@ -902,6 +1555,19 @@ int __dfsw_sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t *mask,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t *mask,
+                             dfsan_label pid_label,
+                             dfsan_label cpusetsize_label,
+                             dfsan_label mask_label, dfsan_label *ret_label,
+                             dfsan_origin pid_origin,
+                             dfsan_origin cpusetsize_origin,
+                             dfsan_origin mask_origin,
+                             dfsan_origin *ret_origin) {
+  return __dfsw_sched_getaffinity(pid, cpusetsize, mask, pid_label,
+                                  cpusetsize_label, mask_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_sigemptyset(sigset_t *set, dfsan_label set_label,
                        dfsan_label *ret_label) {
@@ -1115,6 +1781,14 @@ int __dfsw_gettimeofday(struct timeval *tv, struct timezone *tz,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_gettimeofday(struct timeval *tv, struct timezone *tz,
+                        dfsan_label tv_label, dfsan_label tz_label,
+                        dfsan_label *ret_label, dfsan_origin tv_origin,
+                        dfsan_origin tz_origin, dfsan_origin *ret_origin) {
+  return __dfsw_gettimeofday(tv, tz, tv_label, tz_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE void *__dfsw_memchr(void *s, int c, size_t n,
                                                   dfsan_label s_label,
                                                   dfsan_label c_label,
@@ -1133,6 +1807,24 @@ SANITIZER_INTERFACE_ATTRIBUTE void *__dfsw_memchr(void *s, int c, size_t n,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE void *__dfso_memchr(
+    void *s, int c, size_t n, dfsan_label s_label, dfsan_label c_label,
+    dfsan_label n_label, dfsan_label *ret_label, dfsan_origin s_origin,
+    dfsan_origin c_origin, dfsan_origin n_origin, dfsan_origin *ret_origin) {
+  void *ret = __dfsw_memchr(s, c, n, s_label, c_label, n_label, ret_label);
+  if (flags().strict_data_dependencies) {
+    if (ret)
+      *ret_origin = s_origin;
+  } else {
+    size_t len =
+        ret ? reinterpret_cast<char *>(ret) - reinterpret_cast<char *>(s) + 1
+            : n;
+    dfsan_origin o = dfsan_read_origin_of_first_taint(s, len);
+    *ret_origin = o ? o : (s_label ? s_origin : c_origin);
+  }
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strrchr(char *s, int c,
                                                    dfsan_label s_label,
                                                    dfsan_label c_label,
@@ -1149,6 +1841,23 @@ SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strrchr(char *s, int c,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strrchr(
+    char *s, int c, dfsan_label s_label, dfsan_label c_label,
+    dfsan_label *ret_label, dfsan_origin s_origin, dfsan_origin c_origin,
+    dfsan_origin *ret_origin) {
+  char *ret = __dfsw_strrchr(s, c, s_label, c_label, ret_label);
+  if (flags().strict_data_dependencies) {
+    if (ret)
+      *ret_origin = s_origin;
+  } else {
+    size_t s_len = strlen(s) + 1;
+    dfsan_origin o = dfsan_read_origin_of_first_taint(s, s_len);
+    *ret_origin = o ? o : (s_label ? s_origin : c_origin);
+  }
+
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strstr(char *haystack, char *needle,
                                                   dfsan_label haystack_label,
                                                   dfsan_label needle_label,
@@ -1167,6 +1876,33 @@ SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strstr(char *haystack, char *needle,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strstr(char *haystack, char *needle,
+                                                  dfsan_label haystack_label,
+                                                  dfsan_label needle_label,
+                                                  dfsan_label *ret_label,
+                                                  dfsan_origin haystack_origin,
+                                                  dfsan_origin needle_origin,
+                                                  dfsan_origin *ret_origin) {
+  char *ret =
+      __dfsw_strstr(haystack, needle, haystack_label, needle_label, ret_label);
+  if (flags().strict_data_dependencies) {
+    if (ret)
+      *ret_origin = haystack_origin;
+  } else {
+    size_t needle_len = strlen(needle);
+    size_t len = ret ? ret + needle_len - haystack : strlen(haystack) + 1;
+    dfsan_origin o = dfsan_read_origin_of_first_taint(haystack, len);
+    if (o) {
+      *ret_origin = o;
+    } else {
+      o = dfsan_read_origin_of_first_taint(needle, needle_len + 1);
+      *ret_origin = o ? o : (haystack_label ? haystack_origin : needle_origin);
+    }
+  }
+
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_nanosleep(const struct timespec *req,
                                                    struct timespec *rem,
                                                    dfsan_label req_label,
@@ -1181,6 +1917,13 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_nanosleep(const struct timespec *req,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_nanosleep(
+    const struct timespec *req, struct timespec *rem, dfsan_label req_label,
+    dfsan_label rem_label, dfsan_label *ret_label, dfsan_origin req_origin,
+    dfsan_origin rem_origin, dfsan_origin *ret_origin) {
+  return __dfsw_nanosleep(req, rem, req_label, rem_label, ret_label);
+}
+
 static void clear_msghdr_labels(size_t bytes_written, struct msghdr *msg) {
   dfsan_set_label(0, msg, sizeof(*msg));
   dfsan_set_label(0, msg->msg_name, msg->msg_namelen);
@@ -1209,6 +1952,19 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_recvmmsg(
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_recvmmsg(
+    int sockfd, struct mmsghdr *msgvec, unsigned int vlen, int flags,
+    struct timespec *timeout, dfsan_label sockfd_label,
+    dfsan_label msgvec_label, dfsan_label vlen_label, dfsan_label flags_label,
+    dfsan_label timeout_label, dfsan_label *ret_label,
+    dfsan_origin sockfd_origin, dfsan_origin msgvec_origin,
+    dfsan_origin vlen_origin, dfsan_origin flags_origin,
+    dfsan_origin timeout_origin, dfsan_origin *ret_origin) {
+  return __dfsw_recvmmsg(sockfd, msgvec, vlen, flags, timeout, sockfd_label,
+                         msgvec_label, vlen_label, flags_label, timeout_label,
+                         ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfsw_recvmsg(
     int sockfd, struct msghdr *msg, int flags, dfsan_label sockfd_label,
     dfsan_label msg_label, dfsan_label flags_label, dfsan_label *ret_label) {
@@ -1219,6 +1975,15 @@ SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfsw_recvmsg(
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfso_recvmsg(
+    int sockfd, struct msghdr *msg, int flags, dfsan_label sockfd_label,
+    dfsan_label msg_label, dfsan_label flags_label, dfsan_label *ret_label,
+    dfsan_origin sockfd_origin, dfsan_origin msg_origin,
+    dfsan_origin flags_origin, dfsan_origin *ret_origin) {
+  return __dfsw_recvmsg(sockfd, msg, flags, sockfd_label, msg_label,
+                        flags_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE int
 __dfsw_socketpair(int domain, int type, int protocol, int sv[2],
                   dfsan_label domain_label, dfsan_label type_label,
@@ -1232,6 +1997,16 @@ __dfsw_socketpair(int domain, int type, int protocol, int sv[2],
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_socketpair(
+    int domain, int type, int protocol, int sv[2], dfsan_label domain_label,
+    dfsan_label type_label, dfsan_label protocol_label, dfsan_label sv_label,
+    dfsan_label *ret_label, dfsan_origin domain_origin,
+    dfsan_origin type_origin, dfsan_origin protocol_origin,
+    dfsan_origin sv_origin, dfsan_origin *ret_origin) {
+  return __dfsw_socketpair(domain, type, protocol, sv, domain_label, type_label,
+                           protocol_label, sv_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getsockopt(
     int sockfd, int level, int optname, void *optval, socklen_t *optlen,
     dfsan_label sockfd_label, dfsan_label level_label,
@@ -1246,6 +2021,19 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getsockopt(
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_getsockopt(
+    int sockfd, int level, int optname, void *optval, socklen_t *optlen,
+    dfsan_label sockfd_label, dfsan_label level_label,
+    dfsan_label optname_label, dfsan_label optval_label,
+    dfsan_label optlen_label, dfsan_label *ret_label,
+    dfsan_origin sockfd_origin, dfsan_origin level_origin,
+    dfsan_origin optname_origin, dfsan_origin optval_origin,
+    dfsan_origin optlen_origin, dfsan_origin *ret_origin) {
+  return __dfsw_getsockopt(sockfd, level, optname, optval, optlen, sockfd_label,
+                           level_label, optname_label, optval_label,
+                           optlen_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getsockname(
     int sockfd, struct sockaddr *addr, socklen_t *addrlen,
     dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
@@ -1261,6 +2049,16 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getsockname(
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_getsockname(
+    int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+    dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
+    dfsan_label *ret_label, dfsan_origin sockfd_origin,
+    dfsan_origin addr_origin, dfsan_origin addrlen_origin,
+    dfsan_origin *ret_origin) {
+  return __dfsw_getsockname(sockfd, addr, addrlen, sockfd_label, addr_label,
+                            addrlen_label, ret_label);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getpeername(
     int sockfd, struct sockaddr *addr, socklen_t *addrlen,
     dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
@@ -1276,6 +2074,16 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getpeername(
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_getpeername(
+    int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+    dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
+    dfsan_label *ret_label, dfsan_origin sockfd_origin,
+    dfsan_origin addr_origin, dfsan_origin addrlen_origin,
+    dfsan_origin *ret_origin) {
+  return __dfsw_getpeername(sockfd, addr, addrlen, sockfd_label, addr_label,
+                            addrlen_label, ret_label);
+}
+
 // Type of the trampoline function passed to the custom version of
 // dfsan_set_write_callback.
 typedef void (*write_trampoline_t)(
@@ -1437,6 +2245,7 @@ struct Formatter {
 // positional arguments.
 static int format_buffer(char *str, size_t size, const char *fmt,
                          dfsan_label *va_labels, dfsan_label *ret_label,
+                         dfsan_origin *va_origins, dfsan_origin *ret_origin,
                          va_list ap) {
   Formatter formatter(str, fmt, size);
 
@@ -1492,8 +2301,13 @@ static int format_buffer(char *str, size_t size, const char *fmt,
           default:
             retval = formatter.format(va_arg(ap, int));
           }
-          dfsan_set_label(*va_labels++, formatter.str_cur(),
-                          formatter.num_written_bytes(retval));
+          if (va_origins == nullptr)
+            dfsan_set_label(*va_labels++, formatter.str_cur(),
+                            formatter.num_written_bytes(retval));
+          else
+            dfsan_set_label_origin(*va_labels++, *va_origins++,
+                                   formatter.str_cur(),
+                                   formatter.num_written_bytes(retval));
           end_fmt = true;
           break;
 
@@ -1510,21 +2324,36 @@ static int format_buffer(char *str, size_t size, const char *fmt,
           } else {
             retval = formatter.format(va_arg(ap, double));
           }
-          dfsan_set_label(*va_labels++, formatter.str_cur(),
-                          formatter.num_written_bytes(retval));
+          if (va_origins == nullptr)
+            dfsan_set_label(*va_labels++, formatter.str_cur(),
+                            formatter.num_written_bytes(retval));
+          else
+            dfsan_set_label_origin(*va_labels++, *va_origins++,
+                                   formatter.str_cur(),
+                                   formatter.num_written_bytes(retval));
           end_fmt = true;
           break;
 
         case 'c':
           retval = formatter.format(va_arg(ap, int));
-          dfsan_set_label(*va_labels++, formatter.str_cur(),
-                          formatter.num_written_bytes(retval));
+          if (va_origins == nullptr)
+            dfsan_set_label(*va_labels++, formatter.str_cur(),
+                            formatter.num_written_bytes(retval));
+          else
+            dfsan_set_label_origin(*va_labels++, *va_origins++,
+                                   formatter.str_cur(),
+                                   formatter.num_written_bytes(retval));
           end_fmt = true;
           break;
 
         case 's': {
           char *arg = va_arg(ap, char *);
           retval = formatter.format(arg);
+          if (va_origins) {
+            va_origins++;
+            dfsan_mem_origin_transfer(formatter.str_cur(), arg,
+                                      formatter.num_written_bytes(retval));
+          }
           va_labels++;
           internal_memcpy(shadow_for(formatter.str_cur()), shadow_for(arg),
                           sizeof(dfsan_label) *
@@ -1535,8 +2364,13 @@ static int format_buffer(char *str, size_t size, const char *fmt,
 
         case 'p':
           retval = formatter.format(va_arg(ap, void *));
-          dfsan_set_label(*va_labels++, formatter.str_cur(),
-                          formatter.num_written_bytes(retval));
+          if (va_origins == nullptr)
+            dfsan_set_label(*va_labels++, formatter.str_cur(),
+                            formatter.num_written_bytes(retval));
+          else
+            dfsan_set_label_origin(*va_labels++, *va_origins++,
+                                   formatter.str_cur(),
+                                   formatter.num_written_bytes(retval));
           end_fmt = true;
           break;
 
@@ -1544,6 +2378,8 @@ static int format_buffer(char *str, size_t size, const char *fmt,
           int *ptr = va_arg(ap, int *);
           *ptr = (int)formatter.str_off;
           va_labels++;
+          if (va_origins)
+            va_origins++;
           dfsan_set_label(0, ptr, sizeof(ptr));
           end_fmt = true;
           break;
@@ -1559,6 +2395,8 @@ static int format_buffer(char *str, size_t size, const char *fmt,
         case '*':
           formatter.width = va_arg(ap, int);
           va_labels++;
+          if (va_origins)
+            va_origins++;
           break;
 
         default:
@@ -1576,6 +2414,8 @@ static int format_buffer(char *str, size_t size, const char *fmt,
   }
 
   *ret_label = 0;
+  if (ret_origin)
+    *ret_origin = 0;
 
   // Number of bytes written in total.
   return formatter.str_off;
@@ -1588,7 +2428,22 @@ int __dfsw_sprintf(char *str, const char *format, dfsan_label str_label,
                    dfsan_label *ret_label, ...) {
   va_list ap;
   va_start(ap, ret_label);
-  int ret = format_buffer(str, ~0ul, format, va_labels, ret_label, ap);
+  int ret = format_buffer(str, ~0ul, format, va_labels, ret_label, nullptr,
+                          nullptr, ap);
+  va_end(ap);
+  return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_sprintf(char *str, const char *format, dfsan_label str_label,
+                   dfsan_label format_label, dfsan_label *va_labels,
+                   dfsan_label *ret_label, dfsan_origin str_origin,
+                   dfsan_origin format_origin, dfsan_origin *va_origins,
+                   dfsan_origin *ret_origin, ...) {
+  va_list ap;
+  va_start(ap, ret_origin);
+  int ret = format_buffer(str, ~0ul, format, va_labels, ret_label, va_origins,
+                          ret_origin, ap);
   va_end(ap);
   return ret;
 }
@@ -1600,7 +2455,23 @@ int __dfsw_snprintf(char *str, size_t size, const char *format,
                     dfsan_label *ret_label, ...) {
   va_list ap;
   va_start(ap, ret_label);
-  int ret = format_buffer(str, size, format, va_labels, ret_label, ap);
+  int ret = format_buffer(str, size, format, va_labels, ret_label, nullptr,
+                          nullptr, ap);
+  va_end(ap);
+  return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_snprintf(char *str, size_t size, const char *format,
+                    dfsan_label str_label, dfsan_label size_label,
+                    dfsan_label format_label, dfsan_label *va_labels,
+                    dfsan_label *ret_label, dfsan_origin str_origin,
+                    dfsan_origin size_origin, dfsan_origin format_origin,
+                    dfsan_origin *va_origins, dfsan_origin *ret_origin, ...) {
+  va_list ap;
+  va_start(ap, ret_origin);
+  int ret = format_buffer(str, size, format, va_labels, ret_label, va_origins,
+                          ret_origin, ap);
   va_end(ap);
   return ret;
 }
diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt
index 7b392fad2e1d8fb550757a21eb83720fcca3f912..d41ee33203c9165c7fc1b9b52c61d85fcee351f4 100644
--- a/compiler-rt/lib/dfsan/done_abilist.txt
+++ b/compiler-rt/lib/dfsan/done_abilist.txt
@@ -402,3 +402,5 @@ fun:__sanitizer_cov_pcs_init=discard
 # Ignores the dfsan wrappers.
 fun:__dfsw_*=uninstrumented
 fun:__dfsw_*=discard
+fun:__dfso_*=uninstrumented
+fun:__dfso_*=discard
diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp
index c5322110cb662a55092d94d85b66767bf932f538..ce08ec3508c4a01b5800fc62de269ebfec198bff 100644
--- a/compiler-rt/lib/hwasan/hwasan.cpp
+++ b/compiler-rt/lib/hwasan/hwasan.cpp
@@ -136,8 +136,6 @@ static void HWAsanCheckFailed(const char *file, int line, const char *cond,
   Die();
 }
 
-static constexpr uptr kMemoryUsageBufferSize = 4096;
-
 static void HwasanFormatMemoryUsage(InternalScopedString &s) {
   HwasanThreadList &thread_list = hwasanThreadList();
   auto thread_stats = thread_list.GetThreadStats();
@@ -155,6 +153,8 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) {
 }
 
 #if SANITIZER_ANDROID
+static constexpr uptr kMemoryUsageBufferSize = 4096;
+
 static char *memory_usage_buffer = nullptr;
 
 static void InitMemoryUsage() {
@@ -171,7 +171,7 @@ void UpdateMemoryUsage() {
     return;
   if (!memory_usage_buffer)
     InitMemoryUsage();
-  InternalScopedString s(kMemoryUsageBufferSize);
+  InternalScopedString s;
   HwasanFormatMemoryUsage(s);
   internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1);
   memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0';
@@ -265,8 +265,6 @@ void __hwasan_init() {
   hwasan_init_is_running = 1;
   SanitizerToolName = "HWAddressSanitizer";
 
-  InitTlsSize();
-
   CacheBinaryName();
   InitializeFlags();
 
@@ -493,7 +491,7 @@ extern "C" void *__hwasan_extra_spill_area() {
 }
 
 void __hwasan_print_memory_usage() {
-  InternalScopedString s(kMemoryUsageBufferSize);
+  InternalScopedString s;
   HwasanFormatMemoryUsage(s);
   Printf("%s\n", s.data());
 }
diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h
index d4521efd089a7b2b4731334fe3849424d5642f31..24d96cedc044605a7b144da587198b36edf51656 100644
--- a/compiler-rt/lib/hwasan/hwasan.h
+++ b/compiler-rt/lib/hwasan/hwasan.h
@@ -14,11 +14,12 @@
 #ifndef HWASAN_H
 #define HWASAN_H
 
+#include "hwasan_flags.h"
+#include "hwasan_interface_internal.h"
+#include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
-#include "hwasan_interface_internal.h"
-#include "hwasan_flags.h"
 #include "ubsan/ubsan_platform.h"
 
 #ifndef HWASAN_CONTAINS_UBSAN
@@ -35,10 +36,31 @@
 
 typedef u8 tag_t;
 
+#if defined(__x86_64__)
+// Tags are done in middle bits using userspace aliasing.
+constexpr unsigned kAddressTagShift = 39;
+constexpr unsigned kTagBits = 3;
+
+// The alias region is placed next to the shadow so the upper bits of all
+// taggable addresses matches the upper bits of the shadow base.  This shift
+// value determines which upper bits must match.  It has a floor of 44 since the
+// shadow is always 8TB.
+// TODO(morehouse): In alias mode we can shrink the shadow and use a
+// simpler/faster shadow calculation.
+constexpr unsigned kTaggableRegionCheckShift =
+    __sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U);
+#else
 // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
 // translation and can be used to store a tag.
-const unsigned kAddressTagShift = 56;
-const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
+constexpr unsigned kAddressTagShift = 56;
+constexpr unsigned kTagBits = 8;
+#endif  // defined(__x86_64__)
+
+// Mask for extracting tag bits from the lower 8 bits.
+constexpr uptr kTagMask = (1UL << kTagBits) - 1;
+
+// Mask for extracting tag bits from full pointers.
+constexpr uptr kAddressTagMask = kTagMask << kAddressTagShift;
 
 // Minimal alignment of the shadow base address. Determines the space available
 // for threads and stack histories. This is an ABI constant.
@@ -50,7 +72,7 @@ const unsigned kRecordFPLShift = 4;
 const unsigned kRecordFPModulus = 1 << (64 - kRecordFPShift + kRecordFPLShift);
 
 static inline tag_t GetTagFromPointer(uptr p) {
-  return p >> kAddressTagShift;
+  return (p >> kAddressTagShift) & kTagMask;
 }
 
 static inline uptr UntagAddr(uptr tagged_addr) {
diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.cpp b/compiler-rt/lib/hwasan/hwasan_allocator.cpp
index 0b6b7347892ed3807c8c9087071bc818e6c956d8..a6fc794082a58c963e22935ad37032042a08fbd8 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocator.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_allocator.cpp
@@ -29,8 +29,8 @@ static AllocatorCache fallback_allocator_cache;
 static SpinMutex fallback_mutex;
 static atomic_uint8_t hwasan_allocator_tagging_enabled;
 
-static const tag_t kFallbackAllocTag = 0xBB;
-static const tag_t kFallbackFreeTag = 0xBC;
+static constexpr tag_t kFallbackAllocTag = 0xBB & kTagMask;
+static constexpr tag_t kFallbackFreeTag = 0xBC;
 
 enum RightAlignMode {
   kRightAlignNever,
@@ -84,7 +84,8 @@ void HwasanAllocatorInit() {
   atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
                        !flags()->disable_allocator_tagging);
   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
-  allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+  allocator.Init(common_flags()->allocator_release_to_os_interval_ms,
+                 kAliasRegionStart);
   for (uptr i = 0; i < sizeof(tail_magic); i++)
     tail_magic[i] = GetCurrentThread()->GenerateRandomTag();
 }
@@ -148,7 +149,8 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
   // Tagging can only be skipped when both tag_in_malloc and tag_in_free are
   // false. When tag_in_malloc = false and tag_in_free = true malloc needs to
   // retag to 0.
-  if ((flags()->tag_in_malloc || flags()->tag_in_free) &&
+  if (InTaggableRegion(reinterpret_cast<uptr>(user_ptr)) &&
+      (flags()->tag_in_malloc || flags()->tag_in_free) &&
       atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
     if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) {
       tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag;
@@ -175,6 +177,8 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
 static bool PointerAndMemoryTagsMatch(void *tagged_ptr) {
   CHECK(tagged_ptr);
   uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr);
+  if (!InTaggableRegion(tagged_uptr))
+    return true;
   tag_t mem_tag = *reinterpret_cast<tag_t *>(
       MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr))));
   return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1);
@@ -187,7 +191,9 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   if (!PointerAndMemoryTagsMatch(tagged_ptr))
     ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
 
-  void *untagged_ptr = UntagPtr(tagged_ptr);
+  void *untagged_ptr = InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr))
+                           ? UntagPtr(tagged_ptr)
+                           : tagged_ptr;
   void *aligned_ptr = reinterpret_cast<void *>(
       RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment));
   Metadata *meta =
@@ -219,10 +225,14 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
         Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size);
     internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size);
   }
-  if (flags()->tag_in_free && malloc_bisect(stack, 0) &&
-      atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+  if (InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr)) &&
+      flags()->tag_in_free && malloc_bisect(stack, 0) &&
+      atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
+    // Always store full 8-bit tags on free to maximize UAF detection.
+    tag_t tag = t ? t->GenerateRandomTag(/*num_bits=*/8) : kFallbackFreeTag;
     TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size),
-                     t ? t->GenerateRandomTag() : kFallbackFreeTag);
+                     tag);
+  }
   if (t) {
     allocator.Deallocate(t->allocator_cache(), aligned_ptr);
     if (auto *ha = t->heap_allocations())
@@ -365,7 +375,7 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
     // OOM error is already taken care of by HwasanAllocate.
     return errno_ENOMEM;
   CHECK(IsAligned((uptr)ptr, alignment));
-  *(void **)UntagPtr(memptr) = ptr;
+  *memptr = ptr;
   return 0;
 }
 
diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.h b/compiler-rt/lib/hwasan/hwasan_allocator.h
index 43670a6a3fb7eb5eefec3367f1df00f353e0ad3e..03bbcff3f0f2ee2e4b20dc7b80c6352e34c20fb6 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocator.h
+++ b/compiler-rt/lib/hwasan/hwasan_allocator.h
@@ -13,13 +13,15 @@
 #ifndef HWASAN_ALLOCATOR_H
 #define HWASAN_ALLOCATOR_H
 
+#include "hwasan.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_poisoning.h"
 #include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_allocator_report.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_ring_buffer.h"
-#include "hwasan_poisoning.h"
 
 #if !defined(__aarch64__) && !defined(__x86_64__)
 #error Unsupported platform
@@ -55,7 +57,12 @@ static const uptr kMaxAllowedMallocSize = 1UL << 40;  // 1T
 
 struct AP64 {
   static const uptr kSpaceBeg = ~0ULL;
+
+#if defined(__x86_64__)
+  static const uptr kSpaceSize = 1ULL << kAddressTagShift;
+#else
   static const uptr kSpaceSize = 0x2000000000ULL;
+#endif
   static const uptr kMetadataSize = sizeof(Metadata);
   typedef __sanitizer::VeryDenseSizeClassMap SizeClassMap;
   using AddressSpaceView = LocalAddressSpaceView;
@@ -102,6 +109,16 @@ typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer;
 
 void GetAllocatorStats(AllocatorStatCounters s);
 
+inline bool InTaggableRegion(uptr addr) {
+#if defined(__x86_64__)
+  // Aliases are mapped next to shadow so that the upper bits match the shadow
+  // base.
+  return (addr >> kTaggableRegionCheckShift) ==
+         (__hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+#endif
+  return true;
+}
+
 } // namespace __hwasan
 
 #endif // HWASAN_ALLOCATOR_H
diff --git a/compiler-rt/lib/hwasan/hwasan_checks.h b/compiler-rt/lib/hwasan/hwasan_checks.h
index a8de0fef20f0bd8522e02e05c9d2f11c38f3b2f5..ab543ea88bebf24a6c7065e90d1a808e713fa25f 100644
--- a/compiler-rt/lib/hwasan/hwasan_checks.h
+++ b/compiler-rt/lib/hwasan/hwasan_checks.h
@@ -13,6 +13,7 @@
 #ifndef HWASAN_CHECKS_H
 #define HWASAN_CHECKS_H
 
+#include "hwasan_allocator.h"
 #include "hwasan_mapping.h"
 #include "sanitizer_common/sanitizer_common.h"
 
@@ -81,6 +82,8 @@ enum class AccessType { Load, Store };
 
 template <ErrorAction EA, AccessType AT, unsigned LogSize>
 __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
+  if (!InTaggableRegion(p))
+    return;
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw);
   if (UNLIKELY(!PossiblyShortTagMatches(mem_tag, p, 1 << LogSize))) {
@@ -94,7 +97,7 @@ __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
 template <ErrorAction EA, AccessType AT>
 __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
                                                                       uptr sz) {
-  if (sz == 0)
+  if (sz == 0 || !InTaggableRegion(p))
     return;
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
diff --git a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
index 12730b29bae3675a22b08749c2664ca0e59e1da3..f53276e330d39ac0664cea346f60e518928541de 100644
--- a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
@@ -12,15 +12,17 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "hwasan.h"
 #include "hwasan_dynamic_shadow.h"
-#include "hwasan_mapping.h"
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_posix.h"
 
 #include <elf.h>
 #include <link.h>
 
+#include "hwasan.h"
+#include "hwasan_mapping.h"
+#include "hwasan_thread_list.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_posix.h"
+
 // The code in this file needs to run in an unrelocated binary. It should not
 // access any external symbol, including its own non-hidden globals.
 
@@ -117,6 +119,12 @@ namespace __hwasan {
 void InitShadowGOT() {}
 
 uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
+#if defined(__x86_64__)
+  constexpr uptr kAliasSize = 1ULL << kAddressTagShift;
+  constexpr uptr kNumAliases = 1ULL << kTagBits;
+  return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases,
+                                    RingBufferSize());
+#endif
   return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment,
                           kHighMemEnd);
 }
diff --git a/compiler-rt/lib/hwasan/hwasan_flags.h b/compiler-rt/lib/hwasan/hwasan_flags.h
index 0a6998f675d69b0c75b6b5cab207f59d1c41ab32..b17750158d021cb0079182ca54917dc8d3f8d0bf 100644
--- a/compiler-rt/lib/hwasan/hwasan_flags.h
+++ b/compiler-rt/lib/hwasan/hwasan_flags.h
@@ -12,6 +12,8 @@
 #ifndef HWASAN_FLAGS_H
 #define HWASAN_FLAGS_H
 
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
 namespace __hwasan {
 
 struct Flags {
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
index 44e569ee6d721a99a3333a21ebf1a51fb33b6e7a..ad67e2787d3123ef0c1988f2797568f0e77e58bf 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
@@ -221,8 +221,7 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
   ThreadStartArg *A = reinterpret_cast<ThreadStartArg *> (MmapOrDie(
       GetPageSizeCached(), "pthread_create"));
   *A = {callback, param};
-  int res = REAL(pthread_create)(UntagPtr(th), UntagPtr(attr),
-                                 &HwasanThreadStartFunc, A);
+  int res = REAL(pthread_create)(th, attr, &HwasanThreadStartFunc, A);
   return res;
 }
 
diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp
index 354bfe3e55f91ba3bb51443b9ca6ade0376f65f2..8ce0ff7da95695bc634e1c08b425141e22b9b38e 100644
--- a/compiler-rt/lib/hwasan/hwasan_linux.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp
@@ -76,6 +76,8 @@ uptr kHighShadowEnd;
 uptr kHighMemStart;
 uptr kHighMemEnd;
 
+uptr kAliasRegionStart;  // Always 0 on non-x86.
+
 static void PrintRange(uptr start, uptr end, const char *name) {
   Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
 }
@@ -123,7 +125,7 @@ void InitPrctl() {
   if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0),
                        &local_errno) &&
       local_errno == EINVAL) {
-#if SANITIZER_ANDROID
+#if SANITIZER_ANDROID || defined(__x86_64__)
     // Some older Android kernels have the tagged pointer ABI on
     // unconditionally, and hence don't have the tagged-addr prctl while still
     // allow the ABI.
@@ -179,6 +181,18 @@ bool InitShadow() {
   // High memory starts where allocated shadow allows.
   kHighMemStart = ShadowToMem(kHighShadowStart);
 
+#if defined(__x86_64__)
+  constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
+  kAliasRegionStart =
+      __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset;
+
+  CHECK_EQ(kAliasRegionStart >> kTaggableRegionCheckShift,
+           __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+  CHECK_EQ(
+      (kAliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift,
+      __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+#endif
+
   // Check the sanity of the defined memory ranges (there might be gaps).
   CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
   CHECK_GT(kHighMemStart, kHighShadowEnd);
@@ -222,7 +236,9 @@ void InitThreads() {
 }
 
 bool MemIsApp(uptr p) {
+#if !defined(__x86_64__)  // Memory outside the alias range has non-zero tags.
   CHECK(GetTagFromPointer(p) == 0);
+#endif
   return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
 }
 
diff --git a/compiler-rt/lib/hwasan/hwasan_mapping.h b/compiler-rt/lib/hwasan/hwasan_mapping.h
index c149687bdfa60e409ad6b2c8d3cdc6687c1afc1c..8243d1ec7ed50a9274459fdc28f9b4f8a4b6c4cd 100644
--- a/compiler-rt/lib/hwasan/hwasan_mapping.h
+++ b/compiler-rt/lib/hwasan/hwasan_mapping.h
@@ -48,6 +48,8 @@ extern uptr kHighShadowEnd;
 extern uptr kHighMemStart;
 extern uptr kHighMemEnd;
 
+extern uptr kAliasRegionStart;
+
 inline uptr MemToShadow(uptr untagged_addr) {
   return (untagged_addr >> kShadowScale) +
          __hwasan_shadow_memory_dynamic_address;
diff --git a/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp b/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp
index e82d77a1bc16f7c7303c8fd5a533509becdf5cdf..fab017aae60bed2bac6c02742a5f57a91eb888e6 100644
--- a/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp
@@ -24,7 +24,7 @@ using namespace __hwasan;
 void *__hwasan_memset(void *block, int c, uptr size) {
   CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
       reinterpret_cast<uptr>(block), size);
-  return memset(UntagPtr(block), c, size);
+  return memset(block, c, size);
 }
 
 void *__hwasan_memcpy(void *to, const void *from, uptr size) {
@@ -32,7 +32,7 @@ void *__hwasan_memcpy(void *to, const void *from, uptr size) {
       reinterpret_cast<uptr>(to), size);
   CheckAddressSized<ErrorAction::Recover, AccessType::Load>(
       reinterpret_cast<uptr>(from), size);
-  return memcpy(UntagPtr(to), UntagPtr(from), size);
+  return memcpy(to, from, size);
 }
 
 void *__hwasan_memmove(void *to, const void *from, uptr size) {
diff --git a/compiler-rt/lib/hwasan/hwasan_report.cpp b/compiler-rt/lib/hwasan/hwasan_report.cpp
index 4448d9243767a8eab4bd8cec92c96be76f3c2b55..c02177993918d88183ad229074f2decc05228a84 100644
--- a/compiler-rt/lib/hwasan/hwasan_report.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_report.cpp
@@ -224,7 +224,7 @@ static void PrintStackAllocations(StackAllocationsRingBuffer *sa,
 
   // We didn't find any locals. Most likely we don't have symbols, so dump
   // the information that we have for offline analysis.
-  InternalScopedString frame_desc(GetPageSizeCached() * 2);
+  InternalScopedString frame_desc;
   Printf("Previously allocated frames:\n");
   for (uptr i = 0; i < frames; i++) {
     const uptr *record_addr = &(*sa)[i];
@@ -459,7 +459,7 @@ static void PrintTagInfoAroundAddr(tag_t *tag_ptr, uptr num_rows,
       RoundDownTo(reinterpret_cast<uptr>(tag_ptr), row_len));
   tag_t *beg_row = center_row_beg - row_len * (num_rows / 2);
   tag_t *end_row = center_row_beg + row_len * ((num_rows + 1) / 2);
-  InternalScopedString s(GetPageSizeCached() * 8);
+  InternalScopedString s;
   for (tag_t *row = beg_row; row < end_row; row += row_len) {
     s.append("%s", row == center_row_beg ? "=>" : "  ");
     s.append("%p:", row);
@@ -547,7 +547,7 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size,
     GetStackTraceFromId(chunk.GetAllocStackId()).Print();
   }
 
-  InternalScopedString s(GetPageSizeCached() * 8);
+  InternalScopedString s;
   CHECK_GT(tail_size, 0U);
   CHECK_LT(tail_size, kShadowAlignment);
   u8 *tail = reinterpret_cast<u8*>(untagged_addr + orig_size);
diff --git a/compiler-rt/lib/hwasan/hwasan_thread.cpp b/compiler-rt/lib/hwasan/hwasan_thread.cpp
index b81a6350c05c0d854c744ed986ed8b587bcc4ed9..c1f0e013b49f5fe8981b92871b289246d9ca54c4 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_thread.cpp
@@ -113,18 +113,21 @@ static u32 xorshift(u32 state) {
 }
 
 // Generate a (pseudo-)random non-zero tag.
-tag_t Thread::GenerateRandomTag() {
+tag_t Thread::GenerateRandomTag(uptr num_bits) {
+  DCHECK_GT(num_bits, 0);
   if (tagging_disabled_) return 0;
   tag_t tag;
+  const uptr tag_mask = (1ULL << num_bits) - 1;
   do {
     if (flags()->random_tags) {
       if (!random_buffer_)
         random_buffer_ = random_state_ = xorshift(random_state_);
       CHECK(random_buffer_);
-      tag = random_buffer_ & 0xFF;
-      random_buffer_ >>= 8;
+      tag = random_buffer_ & tag_mask;
+      random_buffer_ >>= num_bits;
     } else {
-      tag = random_state_ = (random_state_ + 1) & 0xFF;
+      random_state_ += 1;
+      tag = random_state_ & tag_mask;
     }
   } while (!tag);
   return tag;
diff --git a/compiler-rt/lib/hwasan/hwasan_thread.h b/compiler-rt/lib/hwasan/hwasan_thread.h
index 88958daf767c107d3883c34baa45a248f1237fb9..1c71cab41c429af00414d3d68fcb976120ad67b2 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread.h
+++ b/compiler-rt/lib/hwasan/hwasan_thread.h
@@ -42,7 +42,7 @@ class Thread {
   HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; }
   StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; }
 
-  tag_t GenerateRandomTag();
+  tag_t GenerateRandomTag(uptr num_bits = kTagBits);
 
   void DisableTagging() { tagging_disabled_++; }
   void EnableTagging() { tagging_disabled_--; }
diff --git a/compiler-rt/lib/lsan/lsan.cpp b/compiler-rt/lib/lsan/lsan.cpp
index 2c0a3bf0787c2a313795668dfd35debc138d96c4..b264be0ba7922460b8908bb60e60812e5578e756 100644
--- a/compiler-rt/lib/lsan/lsan.cpp
+++ b/compiler-rt/lib/lsan/lsan.cpp
@@ -98,7 +98,6 @@ extern "C" void __lsan_init() {
   InitCommonLsan();
   InitializeAllocator();
   ReplaceSystemMalloc();
-  InitTlsSize();
   InitializeInterceptors();
   InitializeThreadRegistry();
   InstallDeadlySignalHandlers(LsanOnDeadlySignal);
diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp
index d5b4132b24d58a2ed8f2f801b1aa99f8f887b117..74400d2e8426d01e99a7832e7bd0c2bc1aec49e3 100644
--- a/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/compiler-rt/lib/lsan/lsan_common.cpp
@@ -895,7 +895,7 @@ void LeakReport::PrintSummary() {
       bytes += leaks_[i].total_size;
       allocations += leaks_[i].hit_count;
   }
-  InternalScopedString summary(kMaxSummaryLength);
+  InternalScopedString summary;
   summary.append("%zu byte(s) leaked in %zu allocation(s).", bytes,
                  allocations);
   ReportErrorSummary(summary.data());
diff --git a/compiler-rt/lib/memprof/memprof_descriptions.cpp b/compiler-rt/lib/memprof/memprof_descriptions.cpp
index ebd81d6f2f2306e801cd8722fdcee3b40b9748fd..aa7ac5d971f859dd5335a27bdc11c3512e94de27 100644
--- a/compiler-rt/lib/memprof/memprof_descriptions.cpp
+++ b/compiler-rt/lib/memprof/memprof_descriptions.cpp
@@ -48,7 +48,7 @@ void DescribeThread(MemprofThreadContext *context) {
     return;
   }
   context->announced = true;
-  InternalScopedString str(1024);
+  InternalScopedString str;
   str.append("Thread %s", MemprofThreadIdAndName(context).c_str());
   if (context->parent_tid == kInvalidTid) {
     str.append(" created by unknown thread\n");
diff --git a/compiler-rt/lib/memprof/memprof_rtl.cpp b/compiler-rt/lib/memprof/memprof_rtl.cpp
index d6d606f666ee54a6a8b5788003c53ab967e609a9..05759e406f7a3a59518359453005bde6bcfa4f91 100644
--- a/compiler-rt/lib/memprof/memprof_rtl.cpp
+++ b/compiler-rt/lib/memprof/memprof_rtl.cpp
@@ -214,9 +214,6 @@ static void MemprofInitInternal() {
 
   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
 
-  // interceptors
-  InitTlsSize();
-
   // Create main thread.
   MemprofThread *main_thread = CreateMainThread();
   CHECK_EQ(0, main_thread->tid());
diff --git a/compiler-rt/lib/msan/msan.cpp b/compiler-rt/lib/msan/msan.cpp
index 4be1630cd30273da86fec98f072cdc3088c56110..4ee7e2ec4dd65121d78a1999e5d9c9b3cc51c962 100644
--- a/compiler-rt/lib/msan/msan.cpp
+++ b/compiler-rt/lib/msan/msan.cpp
@@ -436,7 +436,6 @@ void __msan_init() {
 
   InitializeInterceptors();
   CheckASLR();
-  InitTlsSize();
   InstallDeadlySignalHandlers(MsanOnDeadlySignal);
   InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
index 33f89d6d49928c1bfb4d8f1ed0286b51a745cac9..eb836bc47876398e7fff50def697ed5a7d55aa44 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
@@ -35,9 +35,9 @@ class CombinedAllocator {
     secondary_.InitLinkerInitialized();
   }
 
-  void Init(s32 release_to_os_interval_ms) {
+  void Init(s32 release_to_os_interval_ms, uptr heap_start = 0) {
     stats_.Init();
-    primary_.Init(release_to_os_interval_ms);
+    primary_.Init(release_to_os_interval_ms, heap_start);
     secondary_.Init();
   }
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
index b90dabbf776929bc75a4f86ed56e935dd24c553c..fb5394cd39c4cc4f95d40daa5f5505c79d741053 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
@@ -119,7 +119,8 @@ class SizeClassAllocator32 {
   typedef SizeClassAllocator32<Params> ThisT;
   typedef SizeClassAllocator32LocalCache<ThisT> AllocatorCache;
 
-  void Init(s32 release_to_os_interval_ms) {
+  void Init(s32 release_to_os_interval_ms, uptr heap_start = 0) {
+    CHECK(!heap_start);
     possible_regions.Init();
     internal_memset(size_class_info_array, 0, sizeof(size_class_info_array));
   }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
index 26753b6c8aeba1e386d24f3af792149ce01599f3..db30e138154ac7113bef3722f74bf00ead653c2d 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -69,25 +69,45 @@ class SizeClassAllocator64 {
     return base + (static_cast<uptr>(ptr32) << kCompactPtrScale);
   }
 
-  void Init(s32 release_to_os_interval_ms) {
+  // If heap_start is nonzero, assumes kSpaceSize bytes are already mapped R/W
+  // at heap_start and places the heap there.  This mode requires kSpaceBeg ==
+  // ~(uptr)0.
+  void Init(s32 release_to_os_interval_ms, uptr heap_start = 0) {
     uptr TotalSpaceSize = kSpaceSize + AdditionalSize();
-    if (kUsingConstantSpaceBeg) {
-      CHECK(IsAligned(kSpaceBeg, SizeClassMap::kMaxSize));
-      CHECK_EQ(kSpaceBeg, address_range.Init(TotalSpaceSize,
-                                             PrimaryAllocatorName, kSpaceBeg));
+    PremappedHeap = heap_start != 0;
+    if (PremappedHeap) {
+      CHECK(!kUsingConstantSpaceBeg);
+      NonConstSpaceBeg = heap_start;
+      uptr RegionInfoSize = AdditionalSize();
+      RegionInfoSpace =
+          address_range.Init(RegionInfoSize, PrimaryAllocatorName);
+      CHECK_NE(RegionInfoSpace, ~(uptr)0);
+      CHECK_EQ(RegionInfoSpace,
+               address_range.MapOrDie(RegionInfoSpace, RegionInfoSize,
+                                      "SizeClassAllocator: region info"));
+      MapUnmapCallback().OnMap(RegionInfoSpace, RegionInfoSize);
     } else {
-      // Combined allocator expects that an 2^N allocation is always aligned to
-      // 2^N. For this to work, the start of the space needs to be aligned as
-      // high as the largest size class (which also needs to be a power of 2).
-      NonConstSpaceBeg = address_range.InitAligned(
-          TotalSpaceSize, SizeClassMap::kMaxSize, PrimaryAllocatorName);
-      CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
+      if (kUsingConstantSpaceBeg) {
+        CHECK(IsAligned(kSpaceBeg, SizeClassMap::kMaxSize));
+        CHECK_EQ(kSpaceBeg,
+                 address_range.Init(TotalSpaceSize, PrimaryAllocatorName,
+                                    kSpaceBeg));
+      } else {
+        // Combined allocator expects that an 2^N allocation is always aligned
+        // to 2^N. For this to work, the start of the space needs to be aligned
+        // as high as the largest size class (which also needs to be a power of
+        // 2).
+        NonConstSpaceBeg = address_range.InitAligned(
+            TotalSpaceSize, SizeClassMap::kMaxSize, PrimaryAllocatorName);
+        CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
+      }
+      RegionInfoSpace = SpaceEnd();
+      MapWithCallbackOrDie(RegionInfoSpace, AdditionalSize(),
+                           "SizeClassAllocator: region info");
     }
     SetReleaseToOSIntervalMs(release_to_os_interval_ms);
-    MapWithCallbackOrDie(SpaceEnd(), AdditionalSize(),
-                         "SizeClassAllocator: region info");
     // Check that the RegionInfo array is aligned on the CacheLine size.
-    DCHECK_EQ(SpaceEnd() % kCacheLineSize, 0);
+    DCHECK_EQ(RegionInfoSpace % kCacheLineSize, 0);
   }
 
   s32 ReleaseToOSIntervalMs() const {
@@ -596,6 +616,11 @@ class SizeClassAllocator64 {
 
   atomic_sint32_t release_to_os_interval_ms_;
 
+  uptr RegionInfoSpace;
+
+  // True if the user has already mapped the entire heap R/W.
+  bool PremappedHeap;
+
   struct Stats {
     uptr n_allocated;
     uptr n_freed;
@@ -625,7 +650,7 @@ class SizeClassAllocator64 {
 
   RegionInfo *GetRegionInfo(uptr class_id) const {
     DCHECK_LT(class_id, kNumClasses);
-    RegionInfo *regions = reinterpret_cast<RegionInfo *>(SpaceEnd());
+    RegionInfo *regions = reinterpret_cast<RegionInfo *>(RegionInfoSpace);
     return &regions[class_id];
   }
 
@@ -650,6 +675,9 @@ class SizeClassAllocator64 {
   }
 
   bool MapWithCallback(uptr beg, uptr size, const char *name) {
+    if (PremappedHeap)
+      return beg >= NonConstSpaceBeg &&
+             beg + size <= NonConstSpaceBeg + kSpaceSize;
     uptr mapped = address_range.Map(beg, size, name);
     if (UNLIKELY(!mapped))
       return false;
@@ -659,11 +687,18 @@ class SizeClassAllocator64 {
   }
 
   void MapWithCallbackOrDie(uptr beg, uptr size, const char *name) {
+    if (PremappedHeap) {
+      CHECK_GE(beg, NonConstSpaceBeg);
+      CHECK_LE(beg + size, NonConstSpaceBeg + kSpaceSize);
+      return;
+    }
     CHECK_EQ(beg, address_range.MapOrDie(beg, size, name));
     MapUnmapCallback().OnMap(beg, size);
   }
 
   void UnmapWithCallbackOrDie(uptr beg, uptr size) {
+    if (PremappedHeap)
+      return;
     MapUnmapCallback().OnUnmap(beg, size);
     address_range.Unmap(beg, size);
   }
@@ -832,6 +867,9 @@ class SizeClassAllocator64 {
 
   // Attempts to release RAM occupied by freed chunks back to OS. The region is
   // expected to be locked.
+  //
+  // TODO(morehouse): Support a callback on memory release so HWASan can release
+  // aliases as well.
   void MaybeReleaseToOS(uptr class_id, bool force) {
     RegionInfo *region = GetRegionInfo(class_id);
     const uptr chunk_size = ClassIdToSize(class_id);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
index 10b5c82f180bf7208ddf2a6322ea35abb7c8aec5..33960d94a2f4af07cba32ea99423cac1a5e1d2d2 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
@@ -87,7 +87,7 @@ const char *StripModuleName(const char *module) {
 void ReportErrorSummary(const char *error_message, const char *alt_tool_name) {
   if (!common_flags()->print_summary)
     return;
-  InternalScopedString buff(kMaxSummaryLength);
+  InternalScopedString buff;
   buff.append("SUMMARY: %s: %s",
               alt_tool_name ? alt_tool_name : SanitizerToolName, error_message);
   __sanitizer_report_error_summary(buff.data());
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
index 2fecc3b4bf7ca36cf10ae4a26ef920abc5323668..2b2629fc12dd3d9e19835dec961858b013e2374c 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
@@ -44,7 +44,7 @@ const uptr kMaxPathLength = 4096;
 
 const uptr kMaxThreadStackSize = 1 << 30;  // 1Gb
 
-static const uptr kErrorMessageBufferSize = 1 << 16;
+const uptr kErrorMessageBufferSize = 1 << 16;
 
 // Denotes fake PC values that come from JIT/JAVA/etc.
 // For such PC values __tsan_symbolize_external_ex() will be called.
@@ -135,6 +135,15 @@ void UnmapFromTo(uptr from, uptr to);
 uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
                       uptr min_shadow_base_alignment, uptr &high_mem_end);
 
+// Let S = max(shadow_size, num_aliases * alias_size, ring_buffer_size).
+// Reserves 2*S bytes of address space to the right of the returned address and
+// ring_buffer_size bytes to the left.  The returned address is aligned to 2*S.
+// Also creates num_aliases regions of accessible memory starting at offset S
+// from the returned address.  Each region has size alias_size and is backed by
+// the same physical memory.
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+                                uptr num_aliases, uptr ring_buffer_size);
+
 // Reserve memory range [beg, end]. If madvise_shadow is true then apply
 // madvise (e.g. hugepages, core dumping) requested by options.
 void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
@@ -275,7 +284,6 @@ void SetSandboxingCallback(void (*f)());
 
 void InitializeCoverage(bool enabled, const char *coverage_dir);
 
-void InitTlsSize();
 uptr GetTlsSize();
 
 // Other
@@ -344,8 +352,6 @@ void ReportDeadlySignal(const SignalContext &sig, u32 tid,
 void SetAlternateSignalStack();
 void UnsetAlternateSignalStack();
 
-// We don't want a summary too long.
-const int kMaxSummaryLength = 1024;
 // Construct a one-line string:
 //   SUMMARY: SanitizerToolName: error_message
 // and pass it to __sanitizer_report_error_summary.
@@ -442,8 +448,14 @@ inline uptr Log2(uptr x) {
 
 // Don't use std::min, std::max or std::swap, to minimize dependency
 // on libstdc++.
-template<class T> T Min(T a, T b) { return a < b ? a : b; }
-template<class T> T Max(T a, T b) { return a > b ? a : b; }
+template <class T>
+constexpr T Min(T a, T b) {
+  return a < b ? a : b;
+}
+template <class T>
+constexpr T Max(T a, T b) {
+  return a > b ? a : b;
+}
 template<class T> void Swap(T& a, T& b) {
   T tmp = a;
   a = b;
@@ -592,21 +604,21 @@ class InternalMmapVector : public InternalMmapVectorNoCtor<T> {
   InternalMmapVector &operator=(InternalMmapVector &&) = delete;
 };
 
-class InternalScopedString : public InternalMmapVector<char> {
+class InternalScopedString {
  public:
-  explicit InternalScopedString(uptr max_length)
-      : InternalMmapVector<char>(max_length), length_(0) {
-    (*this)[0] = '\0';
-  }
-  uptr length() { return length_; }
+  InternalScopedString() : buffer_(1) { buffer_[0] = '\0'; }
+
+  uptr length() const { return buffer_.size() - 1; }
   void clear() {
-    (*this)[0] = '\0';
-    length_ = 0;
+    buffer_.resize(1);
+    buffer_[0] = '\0';
   }
   void append(const char *format, ...);
+  const char *data() const { return buffer_.data(); }
+  char *data() { return buffer_.data(); }
 
  private:
-  uptr length_;
+  InternalMmapVector<char> buffer_;
 };
 
 template <class T>
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
index 047c5a17ea6e77dedc763482dc6b60dfad29a219..1037938f3d336f1c3ba2886265cba0bf910f801c 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
@@ -92,14 +92,13 @@ void *BackgroundThread(void *arg) {
 #endif
 
 void WriteToSyslog(const char *msg) {
-  InternalScopedString msg_copy(kErrorMessageBufferSize);
+  InternalScopedString msg_copy;
   msg_copy.append("%s", msg);
-  char *p = msg_copy.data();
-  char *q;
+  const char *p = msg_copy.data();
 
   // Print one line at a time.
   // syslog, at least on Android, has an implicit message length limit.
-  while ((q = internal_strchr(p, '\n'))) {
+  while (char* q = internal_strchr(p, '\n')) {
     *q = '\0';
     WriteOneLineToSyslog(p);
     p = q + 1;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
index 4f692f99c207cda08c6b6c23f7b35c457b3722c2..5d68ad8ee8e40dfa253a5df5b0cd3af47f591c9c 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
@@ -103,7 +103,6 @@ void DisableCoreDumperIfNecessary() {}
 void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
 void SetAlternateSignalStack() {}
 void UnsetAlternateSignalStack() {}
-void InitTlsSize() {}
 
 bool SignalContext::IsStackOverflow() const { return false; }
 void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp
index 9ea19bc21fa3c390b26f53ececd84f196656d52f..a65d3d896e33e180a2acf0d4c1b2ad1211f7c2a9 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cpp
@@ -38,7 +38,7 @@ void LibIgnore::AddIgnoredLibrary(const char *name_templ) {
 void LibIgnore::OnLibraryLoaded(const char *name) {
   BlockingMutexLock lock(&mutex_);
   // Try to match suppressions with symlink target.
-  InternalScopedString buf(kMaxPathLength);
+  InternalMmapVector<char> buf(kMaxPathLength);
   if (name && internal_readlink(name, buf.data(), buf.size() - 1) > 0 &&
       buf[0]) {
     for (uptr i = 0; i < count_; i++) {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index b58c6ecd124c6a524769f6acb1e344304add5d49..b371477755fd3ba29eb5499ff156bd9f716b1d90 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -183,6 +183,14 @@ uptr internal_munmap(void *addr, uptr length) {
   return internal_syscall(SYSCALL(munmap), (uptr)addr, length);
 }
 
+#if SANITIZER_LINUX
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+                     void *new_address) {
+  return internal_syscall(SYSCALL(mremap), (uptr)old_address, old_size,
+                          new_size, flags, (uptr)new_address);
+}
+#endif
+
 int internal_mprotect(void *addr, uptr length, int prot) {
   return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot);
 }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
index 41ae072d6cac59c99ab4bf7b4b0a8261a9788217..9a23fcfb3b93d9e09aa5e932816f4eb35b55ab04 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
@@ -98,7 +98,6 @@ class ThreadLister {
 // Exposed for testing.
 uptr ThreadDescriptorSize();
 uptr ThreadSelf();
-uptr ThreadSelfOffset();
 
 // Matches a library's file name against a base name (stripping path and version
 // information).
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index 2b1224babf18adb0268613db236d6bbb83333cf7..18441e4ab1a0e4ed97c5b1fd2933e641801735c1 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -36,6 +36,7 @@
 #include <link.h>
 #include <pthread.h>
 #include <signal.h>
+#include <sys/mman.h>
 #include <sys/resource.h>
 #include <syslog.h>
 
@@ -48,6 +49,10 @@
 #include <osreldate.h>
 #include <sys/sysctl.h>
 #define pthread_getattr_np pthread_attr_get_np
+// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
+// that, it was never implemented. So just define it to zero.
+#undef MAP_NORESERVE
+#define MAP_NORESERVE 0
 #endif
 
 #if SANITIZER_NETBSD
@@ -183,80 +188,8 @@ __attribute__((unused)) static bool GetLibcVersion(int *major, int *minor,
 #endif
 }
 
-#if SANITIZER_GLIBC && !SANITIZER_GO
-static uptr g_tls_size;
-
-#ifdef __i386__
-#define CHECK_GET_TLS_STATIC_INFO_VERSION (!__GLIBC_PREREQ(2, 27))
-#else
-#define CHECK_GET_TLS_STATIC_INFO_VERSION 0
-#endif
-
-#if CHECK_GET_TLS_STATIC_INFO_VERSION
-#define DL_INTERNAL_FUNCTION __attribute__((regparm(3), stdcall))
-#else
-#define DL_INTERNAL_FUNCTION
-#endif
-
-namespace {
-struct GetTlsStaticInfoCall {
-  typedef void (*get_tls_func)(size_t*, size_t*);
-};
-struct GetTlsStaticInfoRegparmCall {
-  typedef void (*get_tls_func)(size_t*, size_t*) DL_INTERNAL_FUNCTION;
-};
-
-template <typename T>
-void CallGetTls(void* ptr, size_t* size, size_t* align) {
-  typename T::get_tls_func get_tls;
-  CHECK_EQ(sizeof(get_tls), sizeof(ptr));
-  internal_memcpy(&get_tls, &ptr, sizeof(ptr));
-  CHECK_NE(get_tls, 0);
-  get_tls(size, align);
-}
-
-bool CmpLibcVersion(int major, int minor, int patch) {
-  int ma;
-  int mi;
-  int pa;
-  if (!GetLibcVersion(&ma, &mi, &pa))
-    return false;
-  if (ma > major)
-    return true;
-  if (ma < major)
-    return false;
-  if (mi > minor)
-    return true;
-  if (mi < minor)
-    return false;
-  return pa >= patch;
-}
-
-}  // namespace
-
-void InitTlsSize() {
-  // all current supported platforms have 16 bytes stack alignment
-  const size_t kStackAlign = 16;
-  void *get_tls_static_info_ptr = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
-  size_t tls_size = 0;
-  size_t tls_align = 0;
-  // On i?86, _dl_get_tls_static_info used to be internal_function, i.e.
-  // __attribute__((regparm(3), stdcall)) before glibc 2.27 and is normal
-  // function in 2.27 and later.
-  if (CHECK_GET_TLS_STATIC_INFO_VERSION && !CmpLibcVersion(2, 27, 0))
-    CallGetTls<GetTlsStaticInfoRegparmCall>(get_tls_static_info_ptr,
-                                            &tls_size, &tls_align);
-  else
-    CallGetTls<GetTlsStaticInfoCall>(get_tls_static_info_ptr,
-                                     &tls_size, &tls_align);
-  if (tls_align < kStackAlign)
-    tls_align = kStackAlign;
-  g_tls_size = RoundUpTo(tls_size, tls_align);
-}
-#else
-void InitTlsSize() { }
-#endif  // SANITIZER_GLIBC && !SANITIZER_GO
-
+// ThreadDescriptorSize() is only used by lsan to get the pointer to
+// thread-specific data keys in the thread control block.
 #if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) ||       \
      defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__) || \
      defined(__arm__) || SANITIZER_RISCV64) &&                              \
@@ -329,13 +262,6 @@ uptr ThreadDescriptorSize() {
   return val;
 }
 
-// The offset at which pointer to self is located in the thread descriptor.
-const uptr kThreadSelfOffset = FIRST_32_SECOND_64(8, 16);
-
-uptr ThreadSelfOffset() {
-  return kThreadSelfOffset;
-}
-
 #if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
 // TlsPreTcbSize includes size of struct pthread_descr and size of tcb
 // head structure. It lies before the static tls blocks.
@@ -354,48 +280,61 @@ static uptr TlsPreTcbSize() {
 }
 #endif
 
-uptr ThreadSelf() {
-  uptr descr_addr;
-#if defined(__i386__)
-  asm("mov %%gs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
-#elif defined(__x86_64__)
-  asm("mov %%fs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
-#elif defined(__mips__)
-  // MIPS uses TLS variant I. The thread pointer (in hardware register $29)
-  // points to the end of the TCB + 0x7000. The pthread_descr structure is
-  // immediately in front of the TCB. TlsPreTcbSize() includes the size of the
-  // TCB and the size of pthread_descr.
-  const uptr kTlsTcbOffset = 0x7000;
-  uptr thread_pointer;
-  asm volatile(".set push;\
-                .set mips64r2;\
-                rdhwr %0,$29;\
-                .set pop" : "=r" (thread_pointer));
-  descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize();
-#elif defined(__aarch64__) || defined(__arm__)
-  descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
-                                      ThreadDescriptorSize();
-#elif SANITIZER_RISCV64
-  // https://github.com/riscv/riscv-elf-psabi-doc/issues/53
-  uptr thread_pointer = reinterpret_cast<uptr>(__builtin_thread_pointer());
-  descr_addr = thread_pointer - TlsPreTcbSize();
-#elif defined(__s390__)
-  descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer());
-#elif defined(__powerpc64__)
-  // PPC64LE uses TLS variant I. The thread pointer (in GPR 13)
-  // points to the end of the TCB + 0x7000. The pthread_descr structure is
-  // immediately in front of the TCB. TlsPreTcbSize() includes the size of the
-  // TCB and the size of pthread_descr.
-  const uptr kTlsTcbOffset = 0x7000;
-  uptr thread_pointer;
-  asm("addi %0,13,%1" : "=r"(thread_pointer) : "I"(-kTlsTcbOffset));
-  descr_addr = thread_pointer - TlsPreTcbSize();
-#else
-#error "unsupported CPU arch"
-#endif
-  return descr_addr;
+#if !SANITIZER_GO
+namespace {
+struct TlsRange {
+  uptr begin, end, align;
+  size_t tls_modid;
+  bool operator<(const TlsRange &rhs) const { return begin < rhs.begin; }
+};
+}  // namespace
+
+static int CollectStaticTlsRanges(struct dl_phdr_info *info, size_t size,
+                                  void *data) {
+  if (!info->dlpi_tls_data)
+    return 0;
+  const uptr begin = (uptr)info->dlpi_tls_data;
+  for (unsigned i = 0; i != info->dlpi_phnum; ++i)
+    if (info->dlpi_phdr[i].p_type == PT_TLS) {
+      static_cast<InternalMmapVector<TlsRange> *>(data)->push_back(
+          TlsRange{begin, begin + info->dlpi_phdr[i].p_memsz,
+                   info->dlpi_phdr[i].p_align, info->dlpi_tls_modid});
+      break;
+    }
+  return 0;
 }
-#endif  // (x86_64 || i386 || MIPS) && SANITIZER_LINUX
+
+static void GetStaticTlsRange(uptr *addr, uptr *size) {
+  InternalMmapVector<TlsRange> ranges;
+  dl_iterate_phdr(CollectStaticTlsRanges, &ranges);
+  uptr len = ranges.size();
+  Sort(ranges.begin(), len);
+  // Find the range with tls_modid=1. For glibc, because libc.so uses PT_TLS,
+  // this module is guaranteed to exist and is one of the initially loaded
+  // modules.
+  uptr one = 0;
+  while (one != len && ranges[one].tls_modid != 1) ++one;
+  if (one == len) {
+    // This may happen with musl if no module uses PT_TLS.
+    *addr = 0;
+    *size = 0;
+    return;
+  }
+  // Find the maximum consecutive ranges. We consider two modules consecutive if
+  // the gap is smaller than the alignment. The dynamic loader places static TLS
+  // blocks this way not to waste space.
+  uptr l = one;
+  while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l - 1].align)
+    --l;
+  uptr r = one + 1;
+  while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r - 1].align)
+    ++r;
+  *addr = ranges[l].begin;
+  *size = ranges[r - 1].end - ranges[l].begin;
+}
+#endif  // !SANITIZER_GO
+#endif  // (x86_64 || i386 || mips || ...) && SANITIZER_LINUX &&
+        // !SANITIZER_ANDROID
 
 #if SANITIZER_FREEBSD
 static void **ThreadSelfSegbase() {
@@ -467,18 +406,44 @@ static void GetTls(uptr *addr, uptr *size) {
     *size = 0;
   }
 #elif SANITIZER_LINUX
+  GetStaticTlsRange(addr, size);
 #if defined(__x86_64__) || defined(__i386__) || defined(__s390__)
-  *addr = ThreadSelf();
-  *size = GetTlsSize();
-  *addr -= *size;
-  *addr += ThreadDescriptorSize();
-#elif defined(__mips__) || defined(__aarch64__) || defined(__powerpc64__) || \
-    defined(__arm__) || SANITIZER_RISCV64
-  *addr = ThreadSelf();
-  *size = GetTlsSize();
+  // lsan requires the range to additionally cover the static TLS surplus
+  // (elf/dl-tls.c defines 1664). Otherwise there may be false positives for
+  // allocations only referenced by tls in dynamically loaded modules.
+  if (SANITIZER_GLIBC) {
+    *addr -= 1664;
+    *size += 1664;
+  }
+  // Extend the range to include the thread control block. On glibc, lsan needs
+  // the range to include pthread::{specific_1stblock,specific} so that
+  // allocations only referenced by pthread_setspecific can be scanned. This may
+  // underestimate by at most TLS_TCB_ALIGN-1 bytes but it should be fine
+  // because the number of bytes after pthread::specific is larger.
+  *size += ThreadDescriptorSize();
 #else
-  *addr = 0;
-  *size = 0;
+  if (SANITIZER_GLIBC)
+    *size += 1664;
+#if defined(__powerpc64__)
+  // TODO Figure out why *addr may be zero and use TlsPreTcbSize.
+  void *ptr = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
+  uptr tls_size, tls_align;
+  ((void (*)(size_t *, size_t *))ptr)(&tls_size, &tls_align);
+  asm("addi %0,13,-0x7000" : "=r"(*addr));
+  *addr -= TlsPreTcbSize();
+  *size = RoundUpTo(tls_size + TlsPreTcbSize(), 16);
+#elif defined(__mips__) || SANITIZER_RISCV64
+  const uptr pre_tcb_size = TlsPreTcbSize();
+  *addr -= pre_tcb_size;
+  *size += pre_tcb_size;
+#else
+  // arm and aarch64 reserve two words at TP, so this underestimates the range.
+  // However, this is sufficient for the purpose of finding the pointers to
+  // thread-specific data keys.
+  const uptr tcb_size = ThreadDescriptorSize();
+  *addr -= tcb_size;
+  *size += tcb_size;
+#endif
 #endif
 #elif SANITIZER_FREEBSD
   void** segbase = ThreadSelfSegbase();
@@ -519,17 +484,11 @@ static void GetTls(uptr *addr, uptr *size) {
 
 #if !SANITIZER_GO
 uptr GetTlsSize() {
-#if SANITIZER_FREEBSD || SANITIZER_ANDROID || SANITIZER_NETBSD || \
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
     SANITIZER_SOLARIS
   uptr addr, size;
   GetTls(&addr, &size);
   return size;
-#elif SANITIZER_GLIBC
-#if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
-  return RoundUpTo(g_tls_size + TlsPreTcbSize(), 16);
-#else
-  return g_tls_size;
-#endif
 #else
   return 0;
 #endif
@@ -552,10 +511,9 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
   if (!main) {
     // If stack and tls intersect, make them non-intersecting.
     if (*tls_addr > *stk_addr && *tls_addr < *stk_addr + *stk_size) {
-      CHECK_GT(*tls_addr + *tls_size, *stk_addr);
-      CHECK_LE(*tls_addr + *tls_size, *stk_addr + *stk_size);
-      *stk_size -= *tls_size;
-      *tls_addr = *stk_addr + *stk_size;
+      if (*stk_addr + *stk_size < *tls_addr + *tls_size)
+        *tls_size = *stk_addr + *stk_size - *tls_addr;
+      *stk_size = *tls_addr - *stk_addr;
     }
   }
 #endif
@@ -574,20 +532,12 @@ struct DlIteratePhdrData {
   bool first;
 };
 
-static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
-  DlIteratePhdrData *data = (DlIteratePhdrData*)arg;
-  InternalScopedString module_name(kMaxPathLength);
-  if (data->first) {
-    data->first = false;
-    // First module is the binary itself.
-    ReadBinaryNameCached(module_name.data(), module_name.size());
-  } else if (info->dlpi_name) {
-    module_name.append("%s", info->dlpi_name);
-  }
+static int AddModuleSegments(const char *module_name, dl_phdr_info *info,
+                             InternalMmapVectorNoCtor<LoadedModule> *modules) {
   if (module_name[0] == '\0')
     return 0;
   LoadedModule cur_module;
-  cur_module.set(module_name.data(), info->dlpi_addr);
+  cur_module.set(module_name, info->dlpi_addr);
   for (int i = 0; i < (int)info->dlpi_phnum; i++) {
     const Elf_Phdr *phdr = &info->dlpi_phdr[i];
     if (phdr->p_type == PT_LOAD) {
@@ -599,7 +549,26 @@ static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
                                  writable);
     }
   }
-  data->modules->push_back(cur_module);
+  modules->push_back(cur_module);
+  return 0;
+}
+
+static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
+  DlIteratePhdrData *data = (DlIteratePhdrData *)arg;
+  if (data->first) {
+    InternalMmapVector<char> module_name(kMaxPathLength);
+    data->first = false;
+    // First module is the binary itself.
+    ReadBinaryNameCached(module_name.data(), module_name.size());
+    return AddModuleSegments(module_name.data(), info, data->modules);
+  }
+
+  if (info->dlpi_name) {
+    InternalScopedString module_name;
+    module_name.append("%s", info->dlpi_name);
+    return AddModuleSegments(module_name.data(), info, data->modules);
+  }
+
   return 0;
 }
 
@@ -904,6 +873,65 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
   return shadow_start;
 }
 
+static uptr MmapSharedNoReserve(uptr addr, uptr size) {
+  return internal_mmap(
+      reinterpret_cast<void *>(addr), size, PROT_READ | PROT_WRITE,
+      MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+}
+
+static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr,
+                              uptr alias_size) {
+#if SANITIZER_LINUX
+  return internal_mremap(reinterpret_cast<void *>(base_addr), 0, alias_size,
+                         MREMAP_MAYMOVE | MREMAP_FIXED,
+                         reinterpret_cast<void *>(alias_addr));
+#else
+  CHECK(false && "mremap is not supported outside of Linux");
+  return 0;
+#endif
+}
+
+static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) {
+  uptr total_size = alias_size * num_aliases;
+  uptr mapped = MmapSharedNoReserve(start_addr, total_size);
+  CHECK_EQ(mapped, start_addr);
+
+  for (uptr i = 1; i < num_aliases; ++i) {
+    uptr alias_addr = start_addr + i * alias_size;
+    CHECK_EQ(MremapCreateAlias(start_addr, alias_addr, alias_size), alias_addr);
+  }
+}
+
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+                                uptr num_aliases, uptr ring_buffer_size) {
+  CHECK_EQ(alias_size & (alias_size - 1), 0);
+  CHECK_EQ(num_aliases & (num_aliases - 1), 0);
+  CHECK_EQ(ring_buffer_size & (ring_buffer_size - 1), 0);
+
+  const uptr granularity = GetMmapGranularity();
+  shadow_size = RoundUpTo(shadow_size, granularity);
+  CHECK_EQ(shadow_size & (shadow_size - 1), 0);
+
+  const uptr alias_region_size = alias_size * num_aliases;
+  const uptr alignment =
+      2 * Max(Max(shadow_size, alias_region_size), ring_buffer_size);
+  const uptr left_padding = ring_buffer_size;
+
+  const uptr right_size = alignment;
+  const uptr map_size = left_padding + 2 * alignment;
+
+  const uptr map_start = reinterpret_cast<uptr>(MmapNoAccess(map_size));
+  CHECK_NE(map_start, static_cast<uptr>(-1));
+  const uptr right_start = RoundUpTo(map_start + left_padding, alignment);
+
+  UnmapFromTo(map_start, right_start - left_padding);
+  UnmapFromTo(right_start + right_size, map_start + map_size);
+
+  CreateAliases(right_start + right_size / 2, alias_size, num_aliases);
+
+  return right_start;
+}
+
 void InitializePlatformCommonFlags(CommonFlags *cf) {
 #if SANITIZER_ANDROID
   if (&__libc_get_static_tls_bounds == nullptr)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index b0d7bcc645fafa1f77deb84f0605bb207db8da87..5055df1ec29aaa7a9f403daf6081a2a70e153e6e 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -142,6 +142,12 @@ uptr internal_munmap(void *addr, uptr length) {
   return munmap(addr, length);
 }
 
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+                     void *new_address) {
+  CHECK(false && "internal_mremap is unimplemented on Mac");
+  return 0;
+}
+
 int internal_mprotect(void *addr, uptr length, int prot) {
   return mprotect(addr, length, prot);
 }
@@ -453,7 +459,7 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
   // On OS X the executable path is saved to the stack by dyld. Reading it
   // from there is much faster than calling dladdr, especially for large
   // binaries with symbols.
-  InternalScopedString exe_path(kMaxPathLength);
+  InternalMmapVector<char> exe_path(kMaxPathLength);
   uint32_t size = exe_path.size();
   if (_NSGetExecutablePath(exe_path.data(), &size) == 0 &&
       realpath(exe_path.data(), buf) != 0) {
@@ -542,9 +548,6 @@ uptr GetTlsSize() {
   return 0;
 }
 
-void InitTlsSize() {
-}
-
 uptr TlsBaseAddr() {
   uptr segbase = 0;
 #if defined(__x86_64__)
@@ -1019,7 +1022,7 @@ void MaybeReexec() {
   if (DyldNeedsEnvVariable() && !lib_is_in_env) {
     // DYLD_INSERT_LIBRARIES is not set or does not contain the runtime
     // library.
-    InternalScopedString program_name(1024);
+    InternalMmapVector<char> program_name(1024);
     uint32_t buf_size = program_name.size();
     _NSGetExecutablePath(program_name.data(), &buf_size);
     char *new_env = const_cast<char*>(info.dli_fname);
@@ -1252,6 +1255,12 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
   return shadow_start;
 }
 
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+                                uptr num_aliases, uptr ring_buffer_size) {
+  CHECK(false && "HWASan aliasing is unimplemented on Mac");
+  return 0;
+}
+
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
                               uptr *largest_gap_found,
                               uptr *max_occupied_addr) {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
index 647bcdfe105e66633aed488b20b54904d1dd01b7..e3b664f68b618cc48746cf3ce8e578d9e661a995 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
@@ -120,11 +120,7 @@ INTERCEPTOR(int, malloc_make_nonpurgeable, void *ptr) {
 
 INTERCEPTOR(void, malloc_set_zone_name, malloc_zone_t *zone, const char *name) {
   COMMON_MALLOC_ENTER();
-  // Allocate |sizeof(COMMON_MALLOC_ZONE_NAME "-") + internal_strlen(name)|
-  // bytes.
-  size_t buflen =
-      sizeof(COMMON_MALLOC_ZONE_NAME "-") + (name ? internal_strlen(name) : 0);
-  InternalScopedString new_name(buflen);
+  InternalScopedString new_name;
   if (name && zone->introspect == sanitizer_zone.introspect) {
     new_name.append(COMMON_MALLOC_ZONE_NAME "-%s", name);
     name = new_name.data();
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cpp
index 98ac7365da051d13a4169dfc68e0f635101d2e12..ac20f915fefe577195edf19d6c2c3f71495813a3 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cpp
@@ -105,6 +105,12 @@ uptr internal_munmap(void *addr, uptr length) {
   return _REAL(munmap, addr, length);
 }
 
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+                     void *new_address) {
+  CHECK(false && "internal_mremap is unimplemented on NetBSD");
+  return 0;
+}
+
 int internal_mprotect(void *addr, uptr length, int prot) {
   DEFINE__REAL(int, mprotect, void *a, uptr b, int c);
   return _REAL(mprotect, addr, length, prot);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp
index 2e080098283fde0dc5e525a388d8d65a2f6078e7..f8457a6aac413849593fbaf88902b9b4e9bab67a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp
@@ -275,8 +275,8 @@ void ReportFile::Write(const char *buffer, uptr length) {
 
 bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) {
   MemoryMappingLayout proc_maps(/*cache_enabled*/false);
-  InternalScopedString buff(kMaxPathLength);
-  MemoryMappedSegment segment(buff.data(), kMaxPathLength);
+  InternalMmapVector<char> buff(kMaxPathLength);
+  MemoryMappedSegment segment(buff.data(), buff.size());
   while (proc_maps.Next(&segment)) {
     if (segment.IsExecutable() &&
         internal_strcmp(module, segment.filename) == 0) {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_posix.h
index e1a2b48e5cd810a73fd21307b9220a3875783e34..b65dae644767ade1bd35440405f8fffb669acb0a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix.h
@@ -40,6 +40,10 @@ uptr internal_write(fd_t fd, const void *buf, uptr count);
 uptr internal_mmap(void *addr, uptr length, int prot, int flags,
                    int fd, u64 offset);
 uptr internal_munmap(void *addr, uptr length);
+#if SANITIZER_LINUX
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+                     void *new_address);
+#endif
 int internal_mprotect(void *addr, uptr length, int prot);
 int internal_madvise(uptr addr, uptr length, int advice);
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp
index a032787114bb9824dbcb803406db59545df13a16..5d16dfde678654984defe8cb22aea9725803c9cc 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp
@@ -249,26 +249,21 @@ static void NOINLINE SharedPrintfCodeNoBuffer(bool append_pid,
                                               va_list args) {
   va_list args2;
   va_copy(args2, args);
-  const int kLen = 16 * 1024;
-  int needed_length;
+  InternalMmapVector<char> v;
+  int needed_length = 0;
   char *buffer = local_buffer;
   // First try to print a message using a local buffer, and then fall back to
   // mmaped buffer.
-  for (int use_mmap = 0; use_mmap < 2; use_mmap++) {
+  for (int use_mmap = 0;; use_mmap++) {
     if (use_mmap) {
       va_end(args);
       va_copy(args, args2);
-      buffer = (char*)MmapOrDie(kLen, "Report");
-      buffer_size = kLen;
+      v.resize(needed_length + 1);
+      buffer_size = v.capacity();
+      v.resize(buffer_size);
+      buffer = &v[0];
     }
     needed_length = 0;
-    // Check that data fits into the current buffer.
-#   define CHECK_NEEDED_LENGTH \
-      if (needed_length >= buffer_size) { \
-        if (!use_mmap) continue; \
-        RAW_CHECK_MSG(needed_length < kLen, \
-                      "Buffer in Report is too short!\n"); \
-      }
     // Fuchsia's logging infrastructure always keeps track of the logging
     // process, thread, and timestamp, so never prepend such information.
     if (!SANITIZER_FUCHSIA && append_pid) {
@@ -277,18 +272,20 @@ static void NOINLINE SharedPrintfCodeNoBuffer(bool append_pid,
       if (common_flags()->log_exe_name && exe_name) {
         needed_length += internal_snprintf(buffer, buffer_size,
                                            "==%s", exe_name);
-        CHECK_NEEDED_LENGTH
+        if (needed_length >= buffer_size)
+          continue;
       }
       needed_length += internal_snprintf(
           buffer + needed_length, buffer_size - needed_length, "==%d==", pid);
-      CHECK_NEEDED_LENGTH
+      if (needed_length >= buffer_size)
+        continue;
     }
     needed_length += VSNPrintf(buffer + needed_length,
                                buffer_size - needed_length, format, args);
-    CHECK_NEEDED_LENGTH
+    if (needed_length >= buffer_size)
+      continue;
     // If the message fit into the buffer, print it and exit.
     break;
-#   undef CHECK_NEEDED_LENGTH
   }
   RawWrite(buffer);
 
@@ -297,9 +294,6 @@ static void NOINLINE SharedPrintfCodeNoBuffer(bool append_pid,
   CallPrintfAndReportCallback(buffer);
   LogMessageOnPrintf(buffer);
 
-  // If we had mapped any memory, clean up.
-  if (buffer != local_buffer)
-    UnmapOrDie((void *)buffer, buffer_size);
   va_end(args2);
 }
 
@@ -346,13 +340,24 @@ int internal_snprintf(char *buffer, uptr length, const char *format, ...) {
 
 FORMAT(2, 3)
 void InternalScopedString::append(const char *format, ...) {
-  CHECK_LT(length_, size());
-  va_list args;
-  va_start(args, format);
-  VSNPrintf(data() + length_, size() - length_, format, args);
-  va_end(args);
-  length_ += internal_strlen(data() + length_);
-  CHECK_LT(length_, size());
+  uptr prev_len = length();
+
+  while (true) {
+    buffer_.resize(buffer_.capacity());
+
+    va_list args;
+    va_start(args, format);
+    uptr sz = VSNPrintf(buffer_.data() + prev_len, buffer_.size() - prev_len,
+                        format, args);
+    va_end(args);
+    if (sz < buffer_.size() - prev_len) {
+      buffer_.resize(prev_len + sz + 1);
+      break;
+    }
+
+    buffer_.reserve(buffer_.capacity() * 2);
+  }
+  CHECK_EQ(buffer_[length()], '\0');
 }
 
 } // namespace __sanitizer
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cpp
index f2cfcffaf4764b394c99569f443c38851cd7d914..1b7dd46d8de4f793822b66abc7eadb5129884bcf 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cpp
@@ -120,7 +120,7 @@ void MemoryMappingLayout::LoadFromCache() {
 void MemoryMappingLayout::DumpListOfModules(
     InternalMmapVectorNoCtor<LoadedModule> *modules) {
   Reset();
-  InternalScopedString module_name(kMaxPathLength);
+  InternalMmapVector<char> module_name(kMaxPathLength);
   MemoryMappedSegment segment(module_name.data(), module_name.size());
   for (uptr i = 0; Next(&segment); i++) {
     const char *cur_name = segment.filename;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
index d02afcfe87ae0879034d1f9a8446e3e7df52787b..1f53e3e46d8fea509e882992b4e17187b2090f4f 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
@@ -354,8 +354,8 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
 void MemoryMappingLayout::DumpListOfModules(
     InternalMmapVectorNoCtor<LoadedModule> *modules) {
   Reset();
-  InternalScopedString module_name(kMaxPathLength);
-  MemoryMappedSegment segment(module_name.data(), kMaxPathLength);
+  InternalMmapVector<char> module_name(kMaxPathLength);
+  MemoryMappedSegment segment(module_name.data(), module_name.size());
   MemoryMappedSegmentData data;
   segment.data_ = &data;
   while (Next(&segment)) {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h b/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h
index b6a8bef06ee4ad33c0a7f0a8e472037562ed4cb5..5200354694851eba77f0eb9dbe2a58d62a06889d 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h
@@ -12,7 +12,7 @@
 #if __has_feature(ptrauth_calls)
 #include <ptrauth.h>
 #elif defined(__ARM_FEATURE_PAC_DEFAULT) && !defined(__APPLE__)
-inline unsigned long ptrauth_strip(unsigned long __value, unsigned int __key) {
+inline unsigned long ptrauth_strip(void* __value, unsigned int __key) {
   // On the stack the link register is protected with Pointer
   // Authentication Code when compiled with -mbranch-protection.
   // Let's stripping the PAC unconditionally because xpaclri is in
@@ -36,6 +36,6 @@ inline unsigned long ptrauth_strip(unsigned long __value, unsigned int __key) {
 #define ptrauth_string_discriminator(__string) ((int)0)
 #endif
 
-#define STRIP_PAC_PC(pc) ((uptr)ptrauth_strip((uptr)pc, 0))
+#define STRIP_PAC_PC(pc) ((uptr)ptrauth_strip(pc, 0))
 
 #endif // SANITIZER_PTRAUTH_H
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cpp
index d58bd08fb1a899ea5652eda39a6fdbb6f29c8ce8..01554349cc044d0563f36ccd43fa8699415474ea 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cpp
@@ -106,7 +106,6 @@ void DisableCoreDumperIfNecessary() {}
 void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
 void SetAlternateSignalStack() {}
 void UnsetAlternateSignalStack() {}
-void InitTlsSize() {}
 
 void SignalContext::DumpAllRegisters(void *context) {}
 const char *DescribeSignalOrException(int signo) { UNIMPLEMENTED(); }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cpp
index ea0d49ac2e8f9224c92ebf506504a5c983c86f1b..07e4409f4a5d6680db353a449d6b861f48d8b7ac 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cpp
@@ -123,7 +123,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
     // frame[-1] contains the return address
     uhwptr pc1 = frame[-1];
 #else
-    uhwptr pc1 = STRIP_PAC_PC(frame[1]);
+    uhwptr pc1 = STRIP_PAC_PC((void *)frame[1]);
 #endif
     // Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and
     // x86_64) is invalid and stop unwinding here.  If we're adding support for
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp
index 7808ba9b0f572f15cf8f6d2f474369203f36b4c9..738633209f087b9411eea7c8b0618e7e0cf1fdb0 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp
@@ -23,8 +23,8 @@ void StackTrace::Print() const {
     Printf("    <empty stack>\n\n");
     return;
   }
-  InternalScopedString frame_desc(GetPageSizeCached() * 2);
-  InternalScopedString dedup_token(GetPageSizeCached());
+  InternalScopedString frame_desc;
+  InternalScopedString dedup_token;
   int dedup_frames = common_flags()->dedup_token_length;
   bool symbolize = RenderNeedsSymbolization(common_flags()->stack_trace_format);
   uptr frame_num = 0;
@@ -125,7 +125,7 @@ void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf,
     out_buf[out_buf_size - 1] = 0;
     return;
   }
-  InternalScopedString frame_desc(GetPageSizeCached());
+  InternalScopedString frame_desc;
   uptr frame_num = 0;
   // Reserve one byte for the final 0.
   char *out_end = out_buf + out_buf_size - 1;
@@ -156,7 +156,7 @@ void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,
   out_buf[0] = 0;
   DataInfo DI;
   if (!Symbolizer::GetOrInit()->SymbolizeData(data_addr, &DI)) return;
-  InternalScopedString data_desc(GetPageSizeCached());
+  InternalScopedString data_desc;
   RenderData(&data_desc, fmt, &DI, common_flags()->strip_path_prefix);
   internal_strncpy(out_buf, data_desc.data(), out_buf_size);
   out_buf[out_buf_size - 1] = 0;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp
index 44c83a66c5fe19fc1d6ba356413b2fd0cd4aef4c..a674034b8e291c9c9b862bb241996e59c1761172 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp
@@ -34,7 +34,7 @@ SuppressionContext::SuppressionContext(const char *suppression_types[],
 static bool GetPathAssumingFileIsRelativeToExec(const char *file_path,
                                                 /*out*/char *new_file_path,
                                                 uptr new_file_path_size) {
-  InternalScopedString exec(kMaxPathLength);
+  InternalMmapVector<char> exec(kMaxPathLength);
   if (ReadBinaryNameCached(exec.data(), exec.size())) {
     const char *file_name_pos = StripModuleName(exec.data());
     uptr path_to_exec_len = file_name_pos - exec.data();
@@ -69,7 +69,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
   if (filename[0] == '\0')
     return;
 
-  InternalScopedString new_file_path(kMaxPathLength);
+  InternalMmapVector<char> new_file_path(kMaxPathLength);
   filename = FindFile(filename, new_file_path.data(), new_file_path.size());
 
   // Read the file.
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp
index c99a6ceaa5623f200611b5a3543eac5f0c395efa..9287993e665f697270b8d10c344ad486a644a2fe 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp
@@ -31,7 +31,7 @@ namespace __sanitizer {
 void ReportErrorSummary(const char *error_type, const AddressInfo &info,
                         const char *alt_tool_name) {
   if (!common_flags()->print_summary) return;
-  InternalScopedString buff(kMaxSummaryLength);
+  InternalScopedString buff;
   buff.append("%s ", error_type);
   RenderFrame(&buff, "%L %F", 0, info.address, &info,
               common_flags()->symbolize_vs_style,
@@ -150,7 +150,7 @@ static void PrintMemoryByte(InternalScopedString *str, const char *before,
 static void MaybeDumpInstructionBytes(uptr pc) {
   if (!common_flags()->dump_instruction_bytes || (pc < GetPageSizeCached()))
     return;
-  InternalScopedString str(1024);
+  InternalScopedString str;
   str.append("First 16 instruction bytes at pc: ");
   if (IsAccessibleMemoryRange(pc, 16)) {
     for (int i = 0; i < 16; ++i) {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp
index dc611a01a500c1a923bef34a0af612a1b974a9e7..702d901353dba142dc0f746fc16587961bb07a94 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp
@@ -133,16 +133,13 @@ void InitializeDbgHelpIfNeeded() {
   }
 }
 
-#ifdef __clang__
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wframe-larger-than="
-#endif
 bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) {
   InitializeDbgHelpIfNeeded();
 
   // See https://docs.microsoft.com/en-us/windows/win32/debug/retrieving-symbol-information-by-address
-  char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(CHAR)];
-  PSYMBOL_INFO symbol = (PSYMBOL_INFO)buffer;
+  InternalMmapVector<char> buffer(sizeof(SYMBOL_INFO) +
+                                  MAX_SYM_NAME * sizeof(CHAR));
+  PSYMBOL_INFO symbol = (PSYMBOL_INFO)&buffer[0];
   symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
   symbol->MaxNameLen = MAX_SYM_NAME;
   DWORD64 offset = 0;
@@ -166,9 +163,6 @@ bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) {
   // Otherwise, try llvm-symbolizer.
   return got_fileline;
 }
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
 
 const char *WinSymbolizerTool::Demangle(const char *name) {
   CHECK(is_dbghelp_initialized);
@@ -230,7 +224,7 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
   // Compute the command line. Wrap double quotes around everything.
   const char *argv[kArgVMax];
   GetArgV(path_, argv);
-  InternalScopedString command_line(kMaxPathLength * 3);
+  InternalScopedString command_line;
   for (int i = 0; argv[i]; i++) {
     const char *arg = argv[i];
     int arglen = internal_strlen(arg);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
index 99ecfd040c6a1c25d9c8e8e825be064a53e077dd..d47ccad1764dd366db88c06f0d05a07bcbac51ab 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
@@ -390,6 +390,12 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
   return 0;
 }
 
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+                                uptr num_aliases, uptr ring_buffer_size) {
+  CHECK(false && "HWASan aliasing is unimplemented on Windows");
+  return 0;
+}
+
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
   MEMORY_BASIC_INFORMATION mbi;
   CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi)));
@@ -568,7 +574,7 @@ void Abort() {
 // load the image at this address. Therefore, we call it the preferred base. Any
 // addresses in the DWARF typically assume that the object has been loaded at
 // this address.
-static uptr GetPreferredBase(const char *modname) {
+static uptr GetPreferredBase(const char *modname, char *buf, size_t buf_size) {
   fd_t fd = OpenFile(modname, RdOnly, nullptr);
   if (fd == kInvalidFd)
     return 0;
@@ -590,12 +596,10 @@ static uptr GetPreferredBase(const char *modname) {
   // IMAGE_FILE_HEADER
   // IMAGE_OPTIONAL_HEADER
   // Seek to e_lfanew and read all that data.
-  char buf[4 + sizeof(IMAGE_FILE_HEADER) + sizeof(IMAGE_OPTIONAL_HEADER)];
   if (::SetFilePointer(fd, dos_header.e_lfanew, nullptr, FILE_BEGIN) ==
       INVALID_SET_FILE_POINTER)
     return 0;
-  if (!ReadFromFile(fd, &buf[0], sizeof(buf), &bytes_read) ||
-      bytes_read != sizeof(buf))
+  if (!ReadFromFile(fd, buf, buf_size, &bytes_read) || bytes_read != buf_size)
     return 0;
 
   // Check for "PE\0\0" before the PE header.
@@ -615,10 +619,6 @@ static uptr GetPreferredBase(const char *modname) {
   return (uptr)pe_header->ImageBase;
 }
 
-#ifdef __clang__
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wframe-larger-than="
-#endif
 void ListOfModules::init() {
   clearOrInit();
   HANDLE cur_process = GetCurrentProcess();
@@ -641,6 +641,10 @@ void ListOfModules::init() {
     }
   }
 
+  InternalMmapVector<char> buf(4 + sizeof(IMAGE_FILE_HEADER) +
+                               sizeof(IMAGE_OPTIONAL_HEADER));
+  InternalMmapVector<wchar_t> modname_utf16(kMaxPathLength);
+  InternalMmapVector<char> module_name(kMaxPathLength);
   // |num_modules| is the number of modules actually present,
   size_t num_modules = bytes_required / sizeof(HMODULE);
   for (size_t i = 0; i < num_modules; ++i) {
@@ -650,15 +654,13 @@ void ListOfModules::init() {
       continue;
 
     // Get the UTF-16 path and convert to UTF-8.
-    wchar_t modname_utf16[kMaxPathLength];
     int modname_utf16_len =
-        GetModuleFileNameW(handle, modname_utf16, kMaxPathLength);
+        GetModuleFileNameW(handle, &modname_utf16[0], kMaxPathLength);
     if (modname_utf16_len == 0)
       modname_utf16[0] = '\0';
-    char module_name[kMaxPathLength];
-    int module_name_len =
-        ::WideCharToMultiByte(CP_UTF8, 0, modname_utf16, modname_utf16_len + 1,
-                              &module_name[0], kMaxPathLength, NULL, NULL);
+    int module_name_len = ::WideCharToMultiByte(
+        CP_UTF8, 0, &modname_utf16[0], modname_utf16_len + 1, &module_name[0],
+        kMaxPathLength, NULL, NULL);
     module_name[module_name_len] = '\0';
 
     uptr base_address = (uptr)mi.lpBaseOfDll;
@@ -668,21 +670,19 @@ void ListOfModules::init() {
     // RVA when computing the module offset. This helps llvm-symbolizer find the
     // right DWARF CU. In the common case that the image is loaded at it's
     // preferred address, we will now print normal virtual addresses.
-    uptr preferred_base = GetPreferredBase(&module_name[0]);
+    uptr preferred_base =
+        GetPreferredBase(&module_name[0], &buf[0], buf.size());
     uptr adjusted_base = base_address - preferred_base;
 
-    LoadedModule cur_module;
-    cur_module.set(module_name, adjusted_base);
+    modules_.push_back(LoadedModule());
+    LoadedModule &cur_module = modules_.back();
+    cur_module.set(&module_name[0], adjusted_base);
     // We add the whole module as one single address range.
     cur_module.addAddressRange(base_address, end_address, /*executable*/ true,
                                /*writable*/ true);
-    modules_.push_back(cur_module);
   }
   UnmapOrDie(hmodules, modules_buffer_size);
 }
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
 
 void ListOfModules::fallbackInit() { clear(); }
 
@@ -846,9 +846,6 @@ uptr GetTlsSize() {
   return 0;
 }
 
-void InitTlsSize() {
-}
-
 void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
                           uptr *tls_addr, uptr *tls_size) {
 #if SANITIZER_GO
@@ -1057,15 +1054,16 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
     return 0;
 
   // Get the UTF-16 path and convert to UTF-8.
-  wchar_t binname_utf16[kMaxPathLength];
+  InternalMmapVector<wchar_t> binname_utf16(kMaxPathLength);
   int binname_utf16_len =
-      GetModuleFileNameW(NULL, binname_utf16, ARRAY_SIZE(binname_utf16));
+      GetModuleFileNameW(NULL, &binname_utf16[0], kMaxPathLength);
   if (binname_utf16_len == 0) {
     buf[0] = '\0';
     return 0;
   }
-  int binary_name_len = ::WideCharToMultiByte(
-      CP_UTF8, 0, binname_utf16, binname_utf16_len, buf, buf_len, NULL, NULL);
+  int binary_name_len =
+      ::WideCharToMultiByte(CP_UTF8, 0, &binname_utf16[0], binname_utf16_len,
+                            buf, buf_len, NULL, NULL);
   if ((unsigned)binary_name_len == buf_len)
     --binary_name_len;
   buf[binary_name_len] = '\0';
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp
index 590e477678ea7981249f51a5723f556d0ae9fc96..38da7f0184c068cd200bec274b71f88be9410c66 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cpp
@@ -196,9 +196,9 @@ TEST(SanitizerCommon, DenseSizeClassMap) {
 }
 
 template <class Allocator>
-void TestSizeClassAllocator() {
+void TestSizeClassAllocator(uptr premapped_heap = 0) {
   Allocator *a = new Allocator;
-  a->Init(kReleaseToOSIntervalNever);
+  a->Init(kReleaseToOSIntervalNever, premapped_heap);
   typename Allocator::AllocatorCache cache;
   memset(&cache, 0, sizeof(cache));
   cache.Init(0);
@@ -265,6 +265,25 @@ void TestSizeClassAllocator() {
 }
 
 #if SANITIZER_CAN_USE_ALLOCATOR64
+
+// Allocates kAllocatorSize aligned bytes on construction and frees it on
+// destruction.
+class ScopedPremappedHeap {
+ public:
+  ScopedPremappedHeap() {
+    BasePtr = MmapNoReserveOrDie(2 * kAllocatorSize, "preallocated heap");
+    AlignedAddr = RoundUpTo(reinterpret_cast<uptr>(BasePtr), kAllocatorSize);
+  }
+
+  ~ScopedPremappedHeap() { UnmapOrDie(BasePtr, kAllocatorSize); }
+
+  uptr Addr() { return AlignedAddr; }
+
+ private:
+  void *BasePtr;
+  uptr AlignedAddr;
+};
+
 // These tests can fail on Windows if memory is somewhat full and lit happens
 // to run them all at the same time. FIXME: Make them not flaky and reenable.
 #if !SANITIZER_WINDOWS
@@ -277,6 +296,13 @@ TEST(SanitizerCommon, SizeClassAllocator64Dynamic) {
 }
 
 #if !SANITIZER_ANDROID
+// Android only has 39-bit address space, so mapping 2 * kAllocatorSize
+// sometimes fails.
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremapped) {
+  ScopedPremappedHeap h;
+  TestSizeClassAllocator<Allocator64Dynamic>(h.Addr());
+}
+
 //FIXME(kostyak): find values so that those work on Android as well.
 TEST(SanitizerCommon, SizeClassAllocator64Compact) {
   TestSizeClassAllocator<Allocator64Compact>();
@@ -320,9 +346,9 @@ TEST(SanitizerCommon, SizeClassAllocator32SeparateBatches) {
 }
 
 template <class Allocator>
-void SizeClassAllocatorMetadataStress() {
+void SizeClassAllocatorMetadataStress(uptr premapped_heap = 0) {
   Allocator *a = new Allocator;
-  a->Init(kReleaseToOSIntervalNever);
+  a->Init(kReleaseToOSIntervalNever, premapped_heap);
   typename Allocator::AllocatorCache cache;
   memset(&cache, 0, sizeof(cache));
   cache.Init(0);
@@ -362,6 +388,11 @@ TEST(SanitizerCommon, SizeClassAllocator64DynamicMetadataStress) {
 }
 
 #if !SANITIZER_ANDROID
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremappedMetadataStress) {
+  ScopedPremappedHeap h;
+  SizeClassAllocatorMetadataStress<Allocator64Dynamic>(h.Addr());
+}
+
 TEST(SanitizerCommon, SizeClassAllocator64CompactMetadataStress) {
   SizeClassAllocatorMetadataStress<Allocator64Compact>();
 }
@@ -374,9 +405,10 @@ TEST(SanitizerCommon, SizeClassAllocator32CompactMetadataStress) {
 }
 
 template <class Allocator>
-void SizeClassAllocatorGetBlockBeginStress(u64 TotalSize) {
+void SizeClassAllocatorGetBlockBeginStress(u64 TotalSize,
+                                           uptr premapped_heap = 0) {
   Allocator *a = new Allocator;
-  a->Init(kReleaseToOSIntervalNever);
+  a->Init(kReleaseToOSIntervalNever, premapped_heap);
   typename Allocator::AllocatorCache cache;
   memset(&cache, 0, sizeof(cache));
   cache.Init(0);
@@ -409,6 +441,11 @@ TEST(SanitizerCommon, SizeClassAllocator64DynamicGetBlockBegin) {
       1ULL << (SANITIZER_ANDROID ? 31 : 33));
 }
 #if !SANITIZER_ANDROID
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremappedGetBlockBegin) {
+  ScopedPremappedHeap h;
+  SizeClassAllocatorGetBlockBeginStress<Allocator64Dynamic>(
+      1ULL << (SANITIZER_ANDROID ? 31 : 33), h.Addr());
+}
 TEST(SanitizerCommon, SizeClassAllocator64CompactGetBlockBegin) {
   SizeClassAllocatorGetBlockBeginStress<Allocator64Compact>(1ULL << 33);
 }
@@ -624,10 +661,10 @@ TEST(SanitizerCommon, LargeMmapAllocator) {
 }
 
 template <class PrimaryAllocator>
-void TestCombinedAllocator() {
+void TestCombinedAllocator(uptr premapped_heap = 0) {
   typedef CombinedAllocator<PrimaryAllocator> Allocator;
   Allocator *a = new Allocator;
-  a->Init(kReleaseToOSIntervalNever);
+  a->Init(kReleaseToOSIntervalNever, premapped_heap);
   std::mt19937 r;
 
   typename Allocator::AllocatorCache cache;
@@ -699,6 +736,14 @@ TEST(SanitizerCommon, CombinedAllocator64Dynamic) {
 }
 
 #if !SANITIZER_ANDROID
+#if !SANITIZER_WINDOWS
+// Windows fails to map 1TB, so disable this test.
+TEST(SanitizerCommon, CombinedAllocator64DynamicPremapped) {
+  ScopedPremappedHeap h;
+  TestCombinedAllocator<Allocator64Dynamic>(h.Addr());
+}
+#endif
+
 TEST(SanitizerCommon, CombinedAllocator64Compact) {
   TestCombinedAllocator<Allocator64Compact>();
 }
@@ -714,12 +759,12 @@ TEST(SanitizerCommon, SKIP_ON_SOLARIS_SPARCV9(CombinedAllocator32Compact)) {
 }
 
 template <class Allocator>
-void TestSizeClassAllocatorLocalCache() {
+void TestSizeClassAllocatorLocalCache(uptr premapped_heap = 0) {
   using AllocatorCache = typename Allocator::AllocatorCache;
   AllocatorCache cache;
   Allocator *a = new Allocator();
 
-  a->Init(kReleaseToOSIntervalNever);
+  a->Init(kReleaseToOSIntervalNever, premapped_heap);
   memset(&cache, 0, sizeof(cache));
   cache.Init(0);
 
@@ -760,6 +805,11 @@ TEST(SanitizerCommon, SizeClassAllocator64DynamicLocalCache) {
 }
 
 #if !SANITIZER_ANDROID
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremappedLocalCache) {
+  ScopedPremappedHeap h;
+  TestSizeClassAllocatorLocalCache<Allocator64Dynamic>(h.Addr());
+}
+
 TEST(SanitizerCommon, SizeClassAllocator64CompactLocalCache) {
   TestSizeClassAllocatorLocalCache<Allocator64Compact>();
 }
@@ -891,9 +941,9 @@ void IterationTestCallback(uptr chunk, void *arg) {
 }
 
 template <class Allocator>
-void TestSizeClassAllocatorIteration() {
+void TestSizeClassAllocatorIteration(uptr premapped_heap = 0) {
   Allocator *a = new Allocator;
-  a->Init(kReleaseToOSIntervalNever);
+  a->Init(kReleaseToOSIntervalNever, premapped_heap);
   typename Allocator::AllocatorCache cache;
   memset(&cache, 0, sizeof(cache));
   cache.Init(0);
@@ -942,6 +992,12 @@ TEST(SanitizerCommon, SizeClassAllocator64Iteration) {
 TEST(SanitizerCommon, SizeClassAllocator64DynamicIteration) {
   TestSizeClassAllocatorIteration<Allocator64Dynamic>();
 }
+#if !SANITIZER_ANDROID
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremappedIteration) {
+  ScopedPremappedHeap h;
+  TestSizeClassAllocatorIteration<Allocator64Dynamic>(h.Addr());
+}
+#endif
 #endif
 #endif
 
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_common_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_common_test.cpp
index fcf8b14aeba57625704bbca16bf80ebbdf0537a3..21c6b036b95605a51b03e919a53f1c8a196b402d 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_common_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_common_test.cpp
@@ -210,12 +210,10 @@ static void *WorkerThread(void *arg) {
 }
 
 TEST(SanitizerCommon, ThreadStackTlsMain) {
-  InitTlsSize();
   TestThreadInfo(true);
 }
 
 TEST(SanitizerCommon, ThreadStackTlsWorker) {
-  InitTlsSize();
   pthread_t t;
   PTHREAD_CREATE(&t, 0, WorkerThread, 0);
   PTHREAD_JOIN(t, 0);
@@ -350,7 +348,7 @@ TEST(SanitizerCommon, RemoveANSIEscapeSequencesFromString) {
 }
 
 TEST(SanitizerCommon, InternalScopedString) {
-  InternalScopedString str(10);
+  InternalScopedString str;
   EXPECT_EQ(0U, str.length());
   EXPECT_STREQ("", str.data());
 
@@ -364,20 +362,37 @@ TEST(SanitizerCommon, InternalScopedString) {
   EXPECT_STREQ("foo1234", str.data());
 
   str.append("%d", x);
-  EXPECT_EQ(9U, str.length());
-  EXPECT_STREQ("foo123412", str.data());
+  EXPECT_EQ(11U, str.length());
+  EXPECT_STREQ("foo12341234", str.data());
 
   str.clear();
   EXPECT_EQ(0U, str.length());
   EXPECT_STREQ("", str.data());
+}
 
-  str.append("0123456789");
-  EXPECT_EQ(9U, str.length());
-  EXPECT_STREQ("012345678", str.data());
+TEST(SanitizerCommon, InternalScopedStringLarge) {
+  InternalScopedString str;
+  std::string expected;
+  for (int i = 0; i < 1000; ++i) {
+    std::string append(i, 'a' + i % 26);
+    expected += append;
+    str.append(append.c_str());
+    EXPECT_EQ(expected, str.data());
+  }
+}
+
+TEST(SanitizerCommon, InternalScopedStringLargeFormat) {
+  InternalScopedString str;
+  std::string expected;
+  for (int i = 0; i < 1000; ++i) {
+    std::string append(i, 'a' + i % 26);
+    expected += append;
+    str.append("%s", append.c_str());
+    EXPECT_EQ(expected, str.data());
+  }
 }
 
-#if SANITIZER_LINUX || SANITIZER_FREEBSD || \
-  SANITIZER_MAC || SANITIZER_IOS
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_IOS
 TEST(SanitizerCommon, GetRandom) {
   u8 buffer_1[32], buffer_2[32];
   for (bool blocking : { false, true }) {
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_linux_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_linux_test.cpp
index cb6c0724ac884b0ae935a9ba1406695ae201faad..025cba922d2dfacaaf2152b9d22a51b737c8ac2a 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_linux_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_linux_test.cpp
@@ -188,24 +188,9 @@ TEST(SanitizerCommon, SetEnvTest) {
 }
 
 #if (defined(__x86_64__) || defined(__i386__)) && !SANITIZER_ANDROID
-void *thread_self_offset_test_func(void *arg) {
-  bool result =
-      *(uptr *)((char *)ThreadSelf() + ThreadSelfOffset()) == ThreadSelf();
-  return (void *)result;
-}
-
-TEST(SanitizerLinux, ThreadSelfOffset) {
-  EXPECT_TRUE((bool)thread_self_offset_test_func(0));
-  pthread_t tid;
-  void *result;
-  ASSERT_EQ(0, pthread_create(&tid, 0, thread_self_offset_test_func, 0));
-  ASSERT_EQ(0, pthread_join(tid, &result));
-  EXPECT_TRUE((bool)result);
-}
-
 // libpthread puts the thread descriptor at the end of stack space.
 void *thread_descriptor_size_test_func(void *arg) {
-  uptr descr_addr = ThreadSelf();
+  uptr descr_addr = (uptr)pthread_self();
   pthread_attr_t attr;
   pthread_getattr_np(pthread_self(), &attr);
   void *stackaddr;
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cpp
index a98e47ab6c53b13e7791086ce4898dde7b022a10..4b379ba3d5925448eb502acb7245c0742a3bbf81 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_printer_test.cpp
@@ -16,7 +16,7 @@
 namespace __sanitizer {
 
 TEST(SanitizerStacktracePrinter, RenderSourceLocation) {
-  InternalScopedString str(128);
+  InternalScopedString str;
   RenderSourceLocation(&str, "/dir/file.cc", 10, 5, false, "");
   EXPECT_STREQ("/dir/file.cc:10:5", str.data());
 
@@ -50,7 +50,7 @@ TEST(SanitizerStacktracePrinter, RenderSourceLocation) {
 }
 
 TEST(SanitizerStacktracePrinter, RenderModuleLocation) {
-  InternalScopedString str(128);
+  InternalScopedString str;
   RenderModuleLocation(&str, "/dir/exe", 0x123, kModuleArchUnknown, "");
   EXPECT_STREQ("(/dir/exe+0x123)", str.data());
 
@@ -76,7 +76,7 @@ TEST(SanitizerStacktracePrinter, RenderFrame) {
   info.file = internal_strdup("/path/to/my/source");
   info.line = 10;
   info.column = 5;
-  InternalScopedString str(256);
+  InternalScopedString str;
 
   // Dump all the AddressInfo fields.
   RenderFrame(&str,
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cpp
index af19da0c5b6cb752751d70011510ee45406b1b54..58f92fcf9eea4c6402fb3dc967d6e50eb77eac28 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cpp
@@ -175,4 +175,11 @@ TEST(SlowUnwindTest, ShortStackTrace) {
   EXPECT_EQ(bp, stack.top_frame_bp);
 }
 
+// Dummy implementation. This should never be called, but is required to link
+// non-optimized builds of this test.
+void BufferedStackTrace::UnwindImpl(uptr pc, uptr bp, void *context,
+                                    bool request_fast, u32 max_depth) {
+  UNIMPLEMENTED();
+}
+
 }  // namespace __sanitizer
diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h
index ea504bbbf7a185187568b0587b0d28bf1cc5e583..0a8a0b52173ac17fb5f9dd5d6c9ad8019401c1f1 100644
--- a/compiler-rt/lib/scudo/standalone/memtag.h
+++ b/compiler-rt/lib/scudo/standalone/memtag.h
@@ -26,7 +26,7 @@ void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask, uptr *TaggedBegin,
 // We assume that Top-Byte Ignore is enabled if the architecture supports memory
 // tagging. Not all operating systems enable TBI, so we only claim architectural
 // support for memory tagging if the operating system enables TBI.
-#if SCUDO_LINUX
+#if SCUDO_LINUX && !defined(SCUDO_DISABLE_TBI)
 inline constexpr bool archSupportsMemoryTagging() { return true; }
 #else
 inline constexpr bool archSupportsMemoryTagging() { return false; }
diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h
index 366e0b1faea0ad1bb3c1033adaf1d044e8cf41bc..da96eb0ea56b63bc27ab1c5db65f90d605e1f1cb 100644
--- a/compiler-rt/lib/scudo/standalone/secondary.h
+++ b/compiler-rt/lib/scudo/standalone/secondary.h
@@ -86,6 +86,25 @@ public:
 
 static const uptr MaxUnusedCachePages = 4U;
 
+template <typename Config>
+void mapSecondary(Options Options, uptr CommitBase, uptr CommitSize,
+                  uptr AllocPos, uptr Flags, MapPlatformData *Data) {
+  const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * getPageSizeCached();
+  if (useMemoryTagging<Config>(Options) && CommitSize > MaxUnusedCacheBytes) {
+    const uptr UntaggedPos = Max(AllocPos, CommitBase + MaxUnusedCacheBytes);
+    map(reinterpret_cast<void *>(CommitBase), UntaggedPos - CommitBase,
+        "scudo:secondary", MAP_RESIZABLE | MAP_MEMTAG | Flags, Data);
+    map(reinterpret_cast<void *>(UntaggedPos),
+        CommitBase + CommitSize - UntaggedPos, "scudo:secondary",
+        MAP_RESIZABLE | Flags, Data);
+  } else {
+    map(reinterpret_cast<void *>(CommitBase), CommitSize, "scudo:secondary",
+        MAP_RESIZABLE | (useMemoryTagging<Config>(Options) ? MAP_MEMTAG : 0) |
+            Flags,
+        Data);
+  }
+}
+
 template <typename Config> class MapAllocatorCache {
 public:
   // Ensure the default maximum specified fits the array.
@@ -129,9 +148,8 @@ public:
         // Fuchsia does not support replacing mappings by creating a new mapping
         // on top so we just do the two syscalls there.
         Entry.Time = 0;
-        map(reinterpret_cast<void *>(Entry.CommitBase), Entry.CommitSize,
-            "scudo:secondary", MAP_RESIZABLE | MAP_NOACCESS | MAP_MEMTAG,
-            &Entry.Data);
+        mapSecondary<Config>(Options, Entry.CommitBase, Entry.CommitSize,
+                             Entry.CommitBase, MAP_NOACCESS, &Entry.Data);
       } else {
         setMemoryPermission(Entry.CommitBase, Entry.CommitSize, MAP_NOACCESS,
                             &Entry.Data);
@@ -530,19 +548,7 @@ void *MapAllocator<Config>::allocate(Options Options, uptr Size, uptr Alignment,
 
   const uptr CommitSize = MapEnd - PageSize - CommitBase;
   const uptr AllocPos = roundDownTo(CommitBase + CommitSize - Size, Alignment);
-  const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * getPageSizeCached();
-  if (useMemoryTagging<Config>(Options) && CommitSize > MaxUnusedCacheBytes) {
-    const uptr UntaggedPos = Max(AllocPos, CommitBase + MaxUnusedCacheBytes);
-    map(reinterpret_cast<void *>(CommitBase), UntaggedPos - CommitBase,
-        "scudo:secondary", MAP_RESIZABLE | MAP_MEMTAG, &Data);
-    map(reinterpret_cast<void *>(UntaggedPos),
-        CommitBase + CommitSize - UntaggedPos, "scudo:secondary", MAP_RESIZABLE,
-        &Data);
-  } else {
-    map(reinterpret_cast<void *>(CommitBase), CommitSize, "scudo:secondary",
-        MAP_RESIZABLE | (useMemoryTagging<Config>(Options) ? MAP_MEMTAG : 0),
-        &Data);
-  }
+  mapSecondary<Config>(Options, CommitBase, CommitSize, AllocPos, 0, &Data);
   const uptr HeaderPos =
       AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize();
   LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface.h b/compiler-rt/lib/tsan/rtl/tsan_interface.h
index e7131f498b50f5ec073f07990874024cffd53e45..6e022b56850cfafc25eae850cca165f5e15ee34f 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface.h
@@ -415,6 +415,13 @@ void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
                                          u8 *a);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_on_initialize();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_on_finalize(int failed);
+
 }  // extern "C"
 
 }  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
index 45acfe66ff3f962efd563c05efc6ee07e4eff6d4..0d26f497f2bd26e09711e644abe15dbe7551d9e7 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
@@ -318,7 +318,6 @@ void InitializePlatform() {
   }
 
   CheckAndProtect();
-  InitTlsSize();
 #endif  // !SANITIZER_GO
 }
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_report.cpp
index 968c7b97553c6f73172ea964497da275b5275881..ca96afe9817bc2deb2ca752f7c2e0cf0c1d12090 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_report.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_report.cpp
@@ -127,7 +127,7 @@ void PrintStack(const ReportStack *ent) {
   }
   SymbolizedStack *frame = ent->frames;
   for (int i = 0; frame && frame->info.address; frame = frame->next, i++) {
-    InternalScopedString res(2 * GetPageSizeCached());
+    InternalScopedString res;
     RenderFrame(&res, common_flags()->stack_trace_format, i,
                 frame->info.address, &frame->info,
                 common_flags()->symbolize_vs_style,
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
index 4dda62054d8d0a049f5927d793e35ca8596072f1..ed6cc83450d90382e760bc35d6a9e3489d155e1a 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
@@ -11,17 +11,19 @@
 // Main file (entry points) for the TSan run-time.
 //===----------------------------------------------------------------------===//
 
+#include "tsan_rtl.h"
+
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_file.h"
 #include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_symbolizer.h"
 #include "tsan_defs.h"
-#include "tsan_platform.h"
-#include "tsan_rtl.h"
+#include "tsan_interface.h"
 #include "tsan_mman.h"
+#include "tsan_platform.h"
 #include "tsan_suppressions.h"
 #include "tsan_symbolize.h"
 #include "ubsan/ubsan_init.h"
@@ -56,12 +58,23 @@ Context *ctx;
 bool OnFinalize(bool failed);
 void OnInitialize();
 #else
+#include <dlfcn.h>
 SANITIZER_WEAK_CXX_DEFAULT_IMPL
 bool OnFinalize(bool failed) {
+#if !SANITIZER_GO
+  if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_finalize"))
+    return reinterpret_cast<decltype(&__tsan_on_finalize)>(ptr)(failed);
+#endif
   return failed;
 }
 SANITIZER_WEAK_CXX_DEFAULT_IMPL
-void OnInitialize() {}
+void OnInitialize() {
+#if !SANITIZER_GO
+  if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_initialize")) {
+    return reinterpret_cast<decltype(&__tsan_on_initialize)>(ptr)();
+  }
+#endif
+}
 #endif
 
 static char thread_registry_placeholder[sizeof(ThreadRegistry)];
@@ -167,12 +180,12 @@ static void *BackgroundThread(void *arg) {
     } else if (internal_strcmp(flags()->profile_memory, "stderr") == 0) {
       mprof_fd = 2;
     } else {
-      InternalScopedString filename(kMaxPathLength);
+      InternalScopedString filename;
       filename.append("%s.%d", flags()->profile_memory, (int)internal_getpid());
       fd_t fd = OpenFile(filename.data(), WrOnly);
       if (fd == kInvalidFd) {
         Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
-            &filename[0]);
+               filename.data());
       } else {
         mprof_fd = fd;
       }
diff --git a/compiler-rt/lib/ubsan/ubsan_diag.cpp b/compiler-rt/lib/ubsan/ubsan_diag.cpp
index c081ed16981148040c8f82e89335cc138aa31caf..ef2e495cac8edc7d3e956e28768ac9f5c125ad17 100644
--- a/compiler-rt/lib/ubsan/ubsan_diag.cpp
+++ b/compiler-rt/lib/ubsan/ubsan_diag.cpp
@@ -278,7 +278,7 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc,
   }
 
   // Emit data.
-  InternalScopedString Buffer(1024);
+  InternalScopedString Buffer;
   for (uptr P = Min; P != Max; ++P) {
     unsigned char C = *reinterpret_cast<const unsigned char*>(P);
     Buffer.append("%s%02x", (P % 8 == 0) ? "  " : " ", C);
@@ -346,7 +346,7 @@ Diag::~Diag() {
   // All diagnostics should be printed under report mutex.
   ScopedReport::CheckLocked();
   Decorator Decor;
-  InternalScopedString Buffer(1024);
+  InternalScopedString Buffer;
 
   // Prepare a report that a monitor process can inspect.
   if (Level == DL_Error) {
diff --git a/compiler-rt/lib/ubsan/ubsan_monitor.cpp b/compiler-rt/lib/ubsan/ubsan_monitor.cpp
index d064e95f76f728db173117b422bdd0a84b21433c..69dd986f9bdfc52111ff2ff460036baa39b893c5 100644
--- a/compiler-rt/lib/ubsan/ubsan_monitor.cpp
+++ b/compiler-rt/lib/ubsan/ubsan_monitor.cpp
@@ -17,7 +17,7 @@ using namespace __ubsan;
 UndefinedBehaviorReport::UndefinedBehaviorReport(const char *IssueKind,
                                                  Location &Loc,
                                                  InternalScopedString &Msg)
-    : IssueKind(IssueKind), Loc(Loc), Buffer(Msg.length() + 1) {
+    : IssueKind(IssueKind), Loc(Loc) {
   // We have the common sanitizer reporting lock, so it's safe to register a
   // new UB report.
   RegisterUndefinedBehaviorReport(this);
@@ -52,9 +52,9 @@ void __ubsan::__ubsan_get_current_report_data(const char **OutIssueKind,
 
   // Ensure that the first character of the diagnostic text can't start with a
   // lowercase letter.
-  char FirstChar = Buf.data()[0];
+  char FirstChar = *Buf.data();
   if (FirstChar >= 'a' && FirstChar <= 'z')
-    Buf.data()[0] = FirstChar - 'a' + 'A';
+    *Buf.data() += 'A' - 'a';
 
   *OutIssueKind = CurrentUBR->IssueKind;
   *OutMessage = Buf.data();
diff --git a/compiler-rt/test/asan/TestCases/Posix/fgets_fputs.cpp b/compiler-rt/test/asan/TestCases/Posix/fgets_fputs.cpp
index 4688f84a246fa6019cdfc899942fcca210be2d49..34c952f2e02efbd4e3b462ca49f8859b41edc6bf 100644
--- a/compiler-rt/test/asan/TestCases/Posix/fgets_fputs.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/fgets_fputs.cpp
@@ -3,8 +3,6 @@
 // RUN: not %run %t 2 2>&1 | FileCheck %s --check-prefix=CHECK-FPUTS
 // RUN: not %run %t 3 2>&1 | FileCheck %s --check-prefix=CHECK-PUTS
 
-// UNSUPPORTED: aarch64
-
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/asan/TestCases/Posix/no-fd.cpp b/compiler-rt/test/asan/TestCases/Posix/no-fd.cpp
index a39f57c833bb2c6c3c7c0614ebf6a61a835d4941..3130eda2e426acf15078f3d483f3ee8612805d8d 100644
--- a/compiler-rt/test/asan/TestCases/Posix/no-fd.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/no-fd.cpp
@@ -1,6 +1,11 @@
 // RUN: %clangxx_asan -std=c++11 -O0 %s -o %t
-// RUN: %run %t 2>&1 | FileCheck %s
-// RUN: %env_asan_opts=debug=1,verbosity=2 %run %t 2>&1 | FileCheck %s
+
+// MallocNanoZone=0 disables initialization of the Nano MallocZone on Darwin.
+// Initialization of this zone can interfere with this test because the zone
+// might log which opens another file descriptor,
+// e.g. failing to setup the zone due to ASan taking the memory region it wants.
+// RUN: env MallocNanoZone=0 %run %t 2>&1 | FileCheck %s
+// RUN: env MallocNanoZone=0 %env_asan_opts=debug=1,verbosity=2 %run %t 2>&1 | FileCheck %s
 
 // Test ASan initialization
 
@@ -10,9 +15,6 @@
 // lld - see https://bugs.llvm.org/show_bug.cgi?id=45076.
 // UNSUPPORTED: android, powerpc
 
-// Deflake this test before reinabling for darwin (rdar://74992832)
-// UNSUPPORTED: darwin
-
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/asan/TestCases/asan_update_allocation.cpp b/compiler-rt/test/asan/TestCases/asan_update_allocation.cpp
index 19f8073e0509bc1fcfad74005d830b416463b57d..f6b54373af6f7235ec866b62b2ea0985dddb4401 100644
--- a/compiler-rt/test/asan/TestCases/asan_update_allocation.cpp
+++ b/compiler-rt/test/asan/TestCases/asan_update_allocation.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %clangxx_asan -O0 %s --std=c++14 -o %t
 
 // RUN: not %run %t 10 0 2>&1 | FileCheck %s --check-prefixes=CHECK,T0
 // RUN: not %run %t 10000000 0 2>&1 | FileCheck %s --check-prefixes=CHECK,T0
diff --git a/compiler-rt/test/asan/TestCases/lsan_crash.cpp b/compiler-rt/test/asan/TestCases/lsan_crash.cpp
index 23c2569a0b73c88555aa4717a9ec31f851a44076..8ea9e74c8cb25ba0c3370443ebcc77c4b4eac9e2 100644
--- a/compiler-rt/test/asan/TestCases/lsan_crash.cpp
+++ b/compiler-rt/test/asan/TestCases/lsan_crash.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx_asan -O2 %s -o %t && %run %t
+// RUN: %clangxx_asan -O2 %s --std=c++14 -o %t && %run %t
 
 #include <atomic>
 #include <memory>
diff --git a/compiler-rt/test/cfi/lit.cfg.py b/compiler-rt/test/cfi/lit.cfg.py
index cbffe6ea8a65a85cd46fc277662027cc179d7657..2f2d1ddcaa7920e8de1dc298b605d7fc331177a6 100644
--- a/compiler-rt/test/cfi/lit.cfg.py
+++ b/compiler-rt/test/cfi/lit.cfg.py
@@ -13,7 +13,11 @@ clangxx = build_invocation([config.target_cflags] + config.cxx_mode_flags)
 
 config.substitutions.append((r"%clang ", clang + ' '))
 config.substitutions.append((r"%clangxx ", clangxx + ' '))
-if config.lto_supported:
+
+if 'darwin' in config.available_features:
+  # -fsanitize=cfi is not supported on Darwin hosts
+  config.unsupported = True
+elif config.lto_supported:
   clang_cfi = clang + '-fsanitize=cfi '
 
   if config.cfi_lit_test_mode == "Devirt":
diff --git a/compiler-rt/test/dfsan/custom.cpp b/compiler-rt/test/dfsan/custom.cpp
index b95d74446acdd88cbb9f70a89b0cfdb4e257929f..63fa4389e6eaae558e69c4c5ba455570dfb954b7 100644
--- a/compiler-rt/test/dfsan/custom.cpp
+++ b/compiler-rt/test/dfsan/custom.cpp
@@ -83,8 +83,8 @@ dfsan_label i_j_label = 0;
   for (int i = 0; i < size; ++i) {                                \
     assert(origin == dfsan_get_origin((long)(((char *)ptr)[i]))); \
   }
-#define ASSERT_ORIGINS(ptr, size, origin)
 #else
+#define ASSERT_ORIGINS(ptr, size, origin)
 #endif
 
 #ifdef ORIGIN_TRACKING
@@ -142,6 +142,15 @@ dfsan_label i_j_label = 0;
 #define ASSERT_SAVED_ORIGINS(val)
 #endif
 
+#ifdef ORIGIN_TRACKING
+#define DEFINE_AND_SAVE_N_ORIGINS(val, n) \
+  dfsan_origin val##_o[n];                \
+  for (int i = 0; i < n; ++i)             \
+    val##_o[i] = dfsan_get_origin((long)(val[i]));
+#else
+#define DEFINE_AND_SAVE_N_ORIGINS(val, n)
+#endif
+
 #ifdef ORIGIN_TRACKING
 #define ASSERT_SAVED_N_ORIGINS(val, n) \
   for (int i = 0; i < n; ++i)          \
@@ -150,19 +159,26 @@ dfsan_label i_j_label = 0;
 #define ASSERT_SAVED_N_ORIGINS(val, n)
 #endif
 
-#if !defined(ORIGIN_TRACKING)
 void test_stat() {
   int i = 1;
   dfsan_set_label(i_label, &i, sizeof(i));
 
   struct stat s;
   s.st_dev = i;
-  assert(0 == stat("/", &s));
+  DEFINE_AND_SAVE_ORIGINS(s)
+  int ret = stat("/", &s);
+  assert(0 == ret);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_ZERO_LABEL(s.st_dev);
+  ASSERT_SAVED_ORIGINS(s)
 
   s.st_dev = i;
-  assert(-1 == stat("/nonexistent", &s));
+  SAVE_ORIGINS(s)
+  ret = stat("/nonexistent", &s);
+  assert(-1 == ret);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_LABEL(s.st_dev, i_label);
+  ASSERT_SAVED_ORIGINS(s)
 }
 
 void test_fstat() {
@@ -172,9 +188,12 @@ void test_fstat() {
   struct stat s;
   int fd = open("/dev/zero", O_RDONLY);
   s.st_dev = i;
+  DEFINE_AND_SAVE_ORIGINS(s)
   int rv = fstat(fd, &s);
   assert(0 == rv);
+  ASSERT_ZERO_LABEL(rv);
   ASSERT_ZERO_LABEL(s.st_dev);
+  ASSERT_SAVED_ORIGINS(s)
 }
 
 void test_memcmp() {
@@ -188,7 +207,12 @@ void test_memcmp() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, i_j_label);
+  ASSERT_EQ_ORIGIN(rv, str1[3]);
 #endif
+
+  rv = memcmp(str1, str2, sizeof(str1) - 2);
+  assert(rv == 0);
+  ASSERT_ZERO_LABEL(rv);
 }
 
 void test_bcmp() {
@@ -202,6 +226,7 @@ void test_bcmp() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, i_j_label);
+  ASSERT_EQ_ORIGIN(rv, str1[3]);
 #endif
 
   rv = bcmp(str1, str2, sizeof(str1) - 2);
@@ -214,31 +239,60 @@ void test_memcpy() {
   char str2[sizeof(str1)];
   dfsan_set_label(i_label, &str1[3], 1);
 
-  ASSERT_ZERO_LABEL(memcpy(str2, str1, sizeof(str1)));
+  DEFINE_AND_SAVE_ORIGINS(str1)
+
+  char *ptr2 = str2;
+  dfsan_set_label(j_label, &ptr2, sizeof(ptr2));
+
+  void *r = memcpy(ptr2, str1, sizeof(str1));
+  ASSERT_LABEL(r, j_label);
+  ASSERT_EQ_ORIGIN(r, ptr2);
   assert(0 == memcmp(str2, str1, sizeof(str1)));
   ASSERT_ZERO_LABEL(str2[0]);
   ASSERT_LABEL(str2[3], i_label);
+
+  for (int i = 0; i < sizeof(str2); ++i) {
+    if (!dfsan_get_label(str2[i]))
+      continue;
+    ASSERT_INIT_ORIGIN(&(str2[i]), str1_o[i]);
+  }
 }
 
 void test_memmove() {
   char str[] = "str1xx";
   dfsan_set_label(i_label, &str[3], 1);
 
-  ASSERT_ZERO_LABEL(memmove(str + 2, str, 4));
+  DEFINE_AND_SAVE_ORIGINS(str)
+
+  char *ptr = str + 2;
+  dfsan_set_label(j_label, &ptr, sizeof(ptr));
+
+  void *r = memmove(ptr, str, 4);
+  ASSERT_LABEL(r, j_label);
+  ASSERT_EQ_ORIGIN(r, ptr);
   assert(0 == memcmp(str + 2, "str1", 4));
-  for (int i = 0; i <= 4; ++i)
-    ASSERT_ZERO_LABEL(str[i]);
+  ASSERT_ZERO_LABEL(str[4]);
   ASSERT_LABEL(str[5], i_label);
+
+  for (int i = 0; i < 4; ++i) {
+    if (!dfsan_get_label(ptr[i]))
+      continue;
+    ASSERT_INIT_ORIGIN(&(ptr[i]), str_o[i]);
+  }
 }
 
 void test_memset() {
   char buf[8];
   int j = 'a';
+  char *ptr = buf;
   dfsan_set_label(j_label, &j, sizeof(j));
-
-  ASSERT_ZERO_LABEL(memset(&buf, j, sizeof(buf)));
+  dfsan_set_label(k_label, &ptr, sizeof(ptr));
+  void *ret = memset(ptr, j, sizeof(buf));
+  ASSERT_LABEL(ret, k_label);
+  ASSERT_EQ_ORIGIN(ret, ptr);
   for (int i = 0; i < 8; ++i) {
     ASSERT_LABEL(buf[i], j_label);
+    ASSERT_EQ_ORIGIN(buf[i], j);
     assert(buf[i] == 'a');
   }
 }
@@ -254,20 +308,48 @@ void test_strcmp() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, i_j_label);
+  ASSERT_EQ_ORIGIN(rv, str1[3]);
+#endif
+
+  rv = strcmp(str1, str1);
+  assert(rv == 0);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_ZERO_LABEL(rv);
+  ASSERT_ZERO_ORIGIN(rv);
+#else
+  ASSERT_LABEL(rv, i_label);
+  ASSERT_EQ_ORIGIN(rv, str1[3]);
 #endif
 }
 
 void test_strcat() {
   char src[] = "world";
+  int volatile x = 0; // buffer to ensure src and dst do not share origins
   char dst[] = "hello \0    ";
+  int volatile y = 0; // buffer to ensure dst and p do not share origins
   char *p = dst;
   dfsan_set_label(k_label, &p, sizeof(p));
   dfsan_set_label(i_label, src, sizeof(src));
   dfsan_set_label(j_label, dst, sizeof(dst));
+  dfsan_origin dst_o = dfsan_get_origin((long)dst[0]);
   char *ret = strcat(p, src);
   ASSERT_LABEL(ret, k_label);
+  ASSERT_EQ_ORIGIN(ret, p);
   assert(ret == dst);
   assert(strcmp(src, dst + 6) == 0);
+  // Origins are assigned for every 4 contiguous 4-aligned bytes. After
+  // appending src to dst, origins of src can overwrite origins of dst if their
+  // application adddresses are within [start_aligned_down, end_aligned_up).
+  // Other origins are not changed.
+  char *start_aligned_down = (char *)(((size_t)(dst + 6)) & ~3UL);
+  char *end_aligned_up = (char *)(((size_t)(dst + 11 + 4)) & ~3UL);
+  for (int i = 0; i < 12; ++i) {
+    if (dst + i < start_aligned_down || dst + i >= end_aligned_up) {
+      ASSERT_INIT_ORIGIN(&dst[i], dst_o);
+    } else {
+      ASSERT_INIT_ORIGIN_EQ_ORIGIN(&dst[i], src[0]);
+    }
+  }
   for (int i = 0; i < 6; ++i) {
     ASSERT_LABEL(dst[i], j_label);
   }
@@ -277,7 +359,6 @@ void test_strcat() {
   }
   ASSERT_LABEL(dst[11], j_label);
 }
-#endif // !defined(ORIGIN_TRACKING)
 
 void test_strlen() {
   char str1[] = "str1";
@@ -293,14 +374,22 @@ void test_strlen() {
 #endif
 }
 
-#if !defined(ORIGIN_TRACKING)
 void test_strdup() {
   char str1[] = "str1";
   dfsan_set_label(i_label, &str1[3], 1);
+  DEFINE_AND_SAVE_ORIGINS(str1)
 
   char *strd = strdup(str1);
+  ASSERT_ZERO_LABEL(strd);
   ASSERT_ZERO_LABEL(strd[0]);
   ASSERT_LABEL(strd[3], i_label);
+
+  for (int i = 0; i < strlen(strd); ++i) {
+    if (!dfsan_get_label(strd[i]))
+      continue;
+    ASSERT_INIT_ORIGIN(&(strd[i]), str1_o[i]);
+  }
+
   free(strd);
 }
 
@@ -317,14 +406,28 @@ void test_strncpy() {
   ASSERT_ZERO_LABEL(strd[1]);
   ASSERT_ZERO_LABEL(strd[2]);
   ASSERT_LABEL(strd[3], i_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&(strd[3]), str1[3]);
 
-  strd = strncpy(str2, str1, 3);
+  char *p2 = str2;
+  dfsan_set_label(j_label, &p2, sizeof(p2));
+  strd = strncpy(p2, str1, 3);
   assert(strd == str2);
   assert(strncmp(str1, str2, 3) == 0);
-  ASSERT_ZERO_LABEL(strd);
+  ASSERT_LABEL(strd, j_label);
+  ASSERT_EQ_ORIGIN(strd, p2);
+  // When -dfsan-combine-pointer-labels-on-load is on, strd's label propagates
+  // to strd[i]'s label. When ORIGIN_TRACKING is defined,
+  // -dfsan-combine-pointer-labels-on-load is always off, otherwise the flag
+  // is on by default.
+#if defined(ORIGIN_TRACKING)
   ASSERT_ZERO_LABEL(strd[0]);
   ASSERT_ZERO_LABEL(strd[1]);
   ASSERT_ZERO_LABEL(strd[2]);
+#else
+  ASSERT_LABEL(strd[0], j_label);
+  ASSERT_LABEL(strd[1], j_label);
+  ASSERT_LABEL(strd[2], j_label);
+#endif
 }
 
 void test_strncmp() {
@@ -338,11 +441,25 @@ void test_strncmp() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, dfsan_union(i_label, j_label));
+  ASSERT_EQ_ORIGIN(rv, str1[3]);
 #endif
 
+  rv = strncmp(str1, str2, 0);
+  assert(rv == 0);
+  ASSERT_ZERO_LABEL(rv);
+
   rv = strncmp(str1, str2, 3);
   assert(rv == 0);
   ASSERT_ZERO_LABEL(rv);
+
+  rv = strncmp(str1, str1, 4);
+  assert(rv == 0);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_ZERO_LABEL(rv);
+#else
+  ASSERT_LABEL(rv, i_label);
+  ASSERT_EQ_ORIGIN(rv, str1[3]);
+#endif
 }
 
 void test_strcasecmp() {
@@ -357,6 +474,7 @@ void test_strcasecmp() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, dfsan_union(i_label, j_label));
+  ASSERT_EQ_ORIGIN(rv, str1[3]);
 #endif
 
   rv = strcasecmp(str1, str3);
@@ -365,6 +483,7 @@ void test_strcasecmp() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, dfsan_union(i_label, j_label));
+  ASSERT_EQ_ORIGIN(rv, str1[3]);
 #endif
 
   char s1[] = "AbZ";
@@ -378,6 +497,7 @@ void test_strcasecmp() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, dfsan_union(i_label, j_label));
+  ASSERT_EQ_ORIGIN(rv, s1[2]);
 #endif
 }
 
@@ -392,6 +512,7 @@ void test_strncasecmp() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, dfsan_union(i_label, j_label));
+  ASSERT_EQ_ORIGIN(rv, str1[3]);
 #endif
 
   rv = strncasecmp(str1, str2, 3);
@@ -421,6 +542,7 @@ void test_strncasecmp() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, dfsan_union(i_label, j_label));
+  ASSERT_EQ_ORIGIN(rv, s1[2]);
 #endif
 }
 
@@ -428,35 +550,56 @@ void test_strchr() {
   char str1[] = "str1";
   dfsan_set_label(i_label, &str1[3], 1);
 
-  char *crv = strchr(str1, 'r');
+  char *p1 = str1;
+  char c = 'r';
+  dfsan_set_label(k_label, &c, sizeof(c));
+
+  char *crv = strchr(p1, c);
   assert(crv == &str1[2]);
+#ifdef STRICT_DATA_DEPENDENCIES
   ASSERT_ZERO_LABEL(crv);
+#else
+  ASSERT_LABEL(crv, k_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&crv, c);
+#endif
 
-  crv = strchr(str1, '1');
+  dfsan_set_label(j_label, &p1, sizeof(p1));
+  crv = strchr(p1, 'r');
+  assert(crv == &str1[2]);
+  ASSERT_LABEL(crv, j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&crv, p1);
+
+  crv = strchr(p1, '1');
   assert(crv == &str1[3]);
 #ifdef STRICT_DATA_DEPENDENCIES
-  ASSERT_ZERO_LABEL(crv);
+  ASSERT_LABEL(crv, j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&crv, p1);
 #else
-  ASSERT_LABEL(crv, i_label);
+  ASSERT_LABEL(crv, i_j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&crv, str1[3]);
 #endif
 
-  crv = strchr(str1, 'x');
+  crv = strchr(p1, 'x');
   assert(!crv);
 #ifdef STRICT_DATA_DEPENDENCIES
-  ASSERT_ZERO_LABEL(crv);
+  ASSERT_LABEL(crv, j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&crv, p1);
 #else
-  ASSERT_LABEL(crv, i_label);
+  ASSERT_LABEL(crv, i_j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&crv, str1[3]);
 #endif
 
   // `man strchr` says:
   // The terminating null byte is considered part of the string, so that if c
   // is specified as '\0', these functions return a pointer to the terminator.
-  crv = strchr(str1, '\0');
+  crv = strchr(p1, '\0');
   assert(crv == &str1[4]);
 #ifdef STRICT_DATA_DEPENDENCIES
-  ASSERT_ZERO_LABEL(crv);
+  ASSERT_LABEL(crv, j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&crv, p1);
 #else
-  ASSERT_LABEL(crv, i_label);
+  ASSERT_LABEL(crv, i_j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&crv, str1[3]);
 #endif
 }
 
@@ -511,6 +654,9 @@ void test_recvmmsg() {
   dfsan_set_label(i_label, &rmmsg[1].msg_len, sizeof(rmmsg[1].msg_len));
   dfsan_set_label(i_label, &timeout, sizeof(timeout));
 
+  dfsan_origin msg_len0_o = dfsan_get_origin((long)(rmmsg[0].msg_len));
+  dfsan_origin msg_len1_o = dfsan_get_origin((long)(rmmsg[1].msg_len));
+
   // Receive messages and check labels.
   int received_msgs = recvmmsg(sockfds[1], rmmsg, 2, 0, &timeout);
   assert(received_msgs == sent_msgs);
@@ -528,6 +674,9 @@ void test_recvmmsg() {
   ASSERT_LABEL(timeout.tv_sec, i_label);
   ASSERT_LABEL(timeout.tv_nsec, i_label);
 
+  ASSERT_ORIGIN((long)(rmmsg[0].msg_len), msg_len0_o);
+  ASSERT_ORIGIN((long)(rmmsg[1].msg_len), msg_len1_o);
+
   close(sockfds[0]);
   close(sockfds[1]);
 }
@@ -555,6 +704,8 @@ void test_recvmsg() {
   dfsan_set_label(i_label, rbuf, sizeof(rbuf));
   dfsan_set_label(i_label, &rmsg, sizeof(rmsg));
 
+  DEFINE_AND_SAVE_ORIGINS(rmsg)
+
   ssize_t received = recvmsg(sockfds[1], &rmsg, 0);
   assert(received == sent);
   assert(memcmp(sbuf, rbuf, 8) == 0);
@@ -563,6 +714,8 @@ void test_recvmsg() {
   ASSERT_READ_ZERO_LABEL(&rbuf[0], 8);
   ASSERT_READ_LABEL(&rbuf[8], 1, i_label);
 
+  ASSERT_SAVED_ORIGINS(rmsg)
+
   close(sockfds[0]);
   close(sockfds[1]);
 }
@@ -572,6 +725,7 @@ void test_read() {
   dfsan_set_label(i_label, buf, 1);
   dfsan_set_label(j_label, buf + 15, 1);
 
+  DEFINE_AND_SAVE_ORIGINS(buf)
   ASSERT_LABEL(buf[0], i_label);
   ASSERT_LABEL(buf[15], j_label);
 
@@ -581,6 +735,7 @@ void test_read() {
   ASSERT_ZERO_LABEL(rv);
   ASSERT_ZERO_LABEL(buf[0]);
   ASSERT_ZERO_LABEL(buf[15]);
+  ASSERT_SAVED_ORIGINS(buf)
   close(fd);
 }
 
@@ -589,6 +744,7 @@ void test_pread() {
   dfsan_set_label(i_label, buf, 1);
   dfsan_set_label(j_label, buf + 15, 1);
 
+  DEFINE_AND_SAVE_ORIGINS(buf)
   ASSERT_LABEL(buf[0], i_label);
   ASSERT_LABEL(buf[15], j_label);
 
@@ -598,6 +754,7 @@ void test_pread() {
   ASSERT_ZERO_LABEL(rv);
   ASSERT_ZERO_LABEL(buf[0]);
   ASSERT_ZERO_LABEL(buf[15]);
+  ASSERT_SAVED_ORIGINS(buf)
   close(fd);
 }
 
@@ -614,31 +771,44 @@ void test_dlopen() {
 void test_clock_gettime() {
   struct timespec tp;
   dfsan_set_label(j_label, ((char *)&tp) + 3, 1);
+  dfsan_origin origin = dfsan_get_origin((long)(((char *)&tp)[3]));
   int t = clock_gettime(CLOCK_REALTIME, &tp);
   assert(t == 0);
   ASSERT_ZERO_LABEL(t);
   ASSERT_ZERO_LABEL(((char *)&tp)[3]);
+  ASSERT_ORIGIN(((char *)&tp)[3], origin);
 }
 
 void test_ctime_r() {
   char *buf = (char*) malloc(64);
   time_t t = 0;
 
+  DEFINE_AND_SAVE_ORIGINS(buf)
+  dfsan_origin t_o = dfsan_get_origin((long)t);
+
   char *ret = ctime_r(&t, buf);
   ASSERT_ZERO_LABEL(ret);
   assert(buf == ret);
   ASSERT_READ_ZERO_LABEL(buf, strlen(buf) + 1);
+  ASSERT_SAVED_ORIGINS(buf)
 
   dfsan_set_label(i_label, &t, sizeof(t));
+  t_o = dfsan_get_origin((long)t);
   ret = ctime_r(&t, buf);
   ASSERT_ZERO_LABEL(ret);
   ASSERT_READ_LABEL(buf, strlen(buf) + 1, i_label);
+  for (int i = 0; i < strlen(buf) + 1; ++i)
+    ASSERT_ORIGIN(buf[i], t_o);
 
   t = 0;
   dfsan_set_label(j_label, &buf, sizeof(&buf));
+  dfsan_origin buf_ptr_o = dfsan_get_origin((long)buf);
   ret = ctime_r(&t, buf);
   ASSERT_LABEL(ret, j_label);
+  ASSERT_ORIGIN(ret, buf_ptr_o);
   ASSERT_READ_ZERO_LABEL(buf, strlen(buf) + 1);
+  for (int i = 0; i < strlen(buf) + 1; ++i)
+    ASSERT_ORIGIN(buf[i], t_o);
 }
 
 static int write_callback_count = 0;
@@ -664,6 +834,8 @@ void test_dfsan_set_write_callback() {
 
   write_callback_count = 0;
 
+  DEFINE_AND_SAVE_ORIGINS(buf)
+
   // Callback should be invoked on every call to write().
   int res = write(fd, buf, buf_len);
   assert(write_callback_count == 1);
@@ -672,12 +844,21 @@ void test_dfsan_set_write_callback() {
   ASSERT_READ_ZERO_LABEL(last_buf, sizeof(last_buf));
   ASSERT_READ_ZERO_LABEL(&last_count, sizeof(last_count));
 
+  for (int i = 0; i < buf_len; ++i)
+    ASSERT_ORIGIN(last_buf[i], buf_o[i]);
+
+  ASSERT_ZERO_ORIGINS(&last_count, sizeof(last_count));
+
   // Add a label to write() arguments.  Check that the labels are readable from
   // the values passed to the callback.
   dfsan_set_label(i_label, &fd, sizeof(fd));
   dfsan_set_label(j_label, &(buf[3]), 1);
   dfsan_set_label(k_label, &buf_len, sizeof(buf_len));
 
+  dfsan_origin fd_o = dfsan_get_origin((long)fd);
+  dfsan_origin buf3_o = dfsan_get_origin((long)(buf[3]));
+  dfsan_origin buf_len_o = dfsan_get_origin((long)buf_len);
+
   res = write(fd, buf, buf_len);
   assert(write_callback_count == 2);
   ASSERT_READ_ZERO_LABEL(&res, sizeof(res));
@@ -685,6 +866,22 @@ void test_dfsan_set_write_callback() {
   ASSERT_READ_LABEL(&last_buf[3], sizeof(last_buf[3]), j_label);
   ASSERT_READ_LABEL(last_buf, sizeof(last_buf), j_label);
   ASSERT_READ_LABEL(&last_count, sizeof(last_count), k_label);
+  ASSERT_ZERO_ORIGINS(&res, sizeof(res));
+  ASSERT_INIT_ORIGINS(&last_fd, sizeof(last_fd), fd_o);
+  ASSERT_INIT_ORIGINS(&last_buf[3], sizeof(last_buf[3]), buf3_o);
+
+  // Origins are assigned for every 4 contiguous 4-aligned bytes. After
+  // appending src to dst, origins of src can overwrite origins of dst if their
+  // application adddresses are within an aligned range. Other origins are not
+  // changed.
+  for (int i = 0; i < buf_len; ++i) {
+    size_t i_addr = size_t(&last_buf[i]);
+    if (((size_t(&last_buf[3]) & ~3UL) > i_addr) ||
+        (((size_t(&last_buf[3]) + 4) & ~3UL) <= i_addr))
+      ASSERT_ORIGIN(last_buf[i], buf_o[i]);
+  }
+
+  ASSERT_INIT_ORIGINS(&last_count, sizeof(last_count), buf_len_o);
 
   dfsan_set_write_callback(NULL);
 }
@@ -693,27 +890,44 @@ void test_fgets() {
   char *buf = (char*) malloc(128);
   FILE *f = fopen("/etc/passwd", "r");
   dfsan_set_label(j_label, buf, 1);
+  DEFINE_AND_SAVE_N_ORIGINS(buf, 128)
+
   char *ret = fgets(buf, sizeof(buf), f);
   assert(ret == buf);
   ASSERT_ZERO_LABEL(ret);
+  ASSERT_EQ_ORIGIN(ret, buf);
   ASSERT_READ_ZERO_LABEL(buf, 128);
+  ASSERT_SAVED_N_ORIGINS(buf, 128)
+
   dfsan_set_label(j_label, &buf, sizeof(&buf));
   ret = fgets(buf, sizeof(buf), f);
   ASSERT_LABEL(ret, j_label);
+  ASSERT_EQ_ORIGIN(ret, buf);
+  ASSERT_SAVED_N_ORIGINS(buf, 128)
+
   fclose(f);
+  free(buf);
 }
 
 void test_getcwd() {
   char buf[1024];
   char *ptr = buf;
   dfsan_set_label(i_label, buf + 2, 2);
+  DEFINE_AND_SAVE_ORIGINS(buf)
+
   char* ret = getcwd(buf, sizeof(buf));
   assert(ret == buf);
   assert(ret[0] == '/');
+  ASSERT_ZERO_LABEL(ret);
+  ASSERT_EQ_ORIGIN(ret, buf);
   ASSERT_READ_ZERO_LABEL(buf + 2, 2);
+  ASSERT_SAVED_ORIGINS(buf)
+
   dfsan_set_label(i_label, &ptr, sizeof(ptr));
   ret = getcwd(ptr, sizeof(buf));
   ASSERT_LABEL(ret, i_label);
+  ASSERT_EQ_ORIGIN(ret, ptr);
+  ASSERT_SAVED_ORIGINS(buf)
 }
 
 void test_get_current_dir_name() {
@@ -721,43 +935,62 @@ void test_get_current_dir_name() {
   assert(ret);
   assert(ret[0] == '/');
   ASSERT_READ_ZERO_LABEL(ret, strlen(ret) + 1);
+  ASSERT_ZERO_LABEL(ret);
 }
 
 void test_gethostname() {
   char buf[1024];
   dfsan_set_label(i_label, buf + 2, 2);
-  assert(gethostname(buf, sizeof(buf)) == 0);
+  DEFINE_AND_SAVE_ORIGINS(buf)
+  int ret = gethostname(buf, sizeof(buf));
+  assert(ret == 0);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_READ_ZERO_LABEL(buf + 2, 2);
+  ASSERT_SAVED_ORIGINS(buf)
 }
 
 void test_getrlimit() {
   struct rlimit rlim;
   dfsan_set_label(i_label, &rlim, sizeof(rlim));
-  assert(getrlimit(RLIMIT_CPU, &rlim) == 0);
+  DEFINE_AND_SAVE_ORIGINS(rlim);
+  int ret = getrlimit(RLIMIT_CPU, &rlim);
+  assert(ret == 0);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_READ_ZERO_LABEL(&rlim, sizeof(rlim));
+  ASSERT_SAVED_ORIGINS(rlim)
 }
 
 void test_getrusage() {
   struct rusage usage;
   dfsan_set_label(i_label, &usage, sizeof(usage));
-  assert(getrusage(RUSAGE_SELF, &usage) == 0);
+  DEFINE_AND_SAVE_ORIGINS(usage);
+  int ret = getrusage(RUSAGE_SELF, &usage);
+  assert(ret == 0);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_READ_ZERO_LABEL(&usage, sizeof(usage));
+  ASSERT_SAVED_ORIGINS(usage)
 }
 
 void test_strcpy() {
   char src[] = "hello world";
   char dst[sizeof(src) + 2];
+  char *p_dst = dst;
   dfsan_set_label(0, src, sizeof(src));
   dfsan_set_label(0, dst, sizeof(dst));
+  dfsan_set_label(k_label, &p_dst, sizeof(p_dst));
   dfsan_set_label(i_label, src + 2, 1);
   dfsan_set_label(j_label, src + 3, 1);
   dfsan_set_label(j_label, dst + 4, 1);
   dfsan_set_label(i_label, dst + 12, 1);
-  char *ret = strcpy(dst, src);
+  char *ret = strcpy(p_dst, src);
   assert(ret == dst);
   assert(strcmp(src, dst) == 0);
+  ASSERT_LABEL(ret, k_label);
+  ASSERT_EQ_ORIGIN(ret, p_dst);
   for (int i = 0; i < strlen(src) + 1; ++i) {
     assert(dfsan_get_label(dst[i]) == dfsan_get_label(src[i]));
+    if (dfsan_get_label(dst[i]))
+      assert(dfsan_get_init_origin(&dst[i]) == dfsan_get_origin(src[i]));
   }
   // Note: if strlen(src) + 1 were used instead to compute the first untouched
   // byte of dest, the label would be I|J. This is because strlen() might
@@ -767,67 +1000,137 @@ void test_strcpy() {
 }
 
 void test_strtol() {
-  char buf[] = "1234578910";
+  char non_number_buf[] = "ab ";
   char *endptr = NULL;
+  long int ret = strtol(non_number_buf, &endptr, 10);
+  assert(ret == 0);
+  assert(endptr == non_number_buf);
+  ASSERT_ZERO_LABEL(ret);
+
+  char buf[] = "1234578910";
+  int base = 10;
+  dfsan_set_label(k_label, &base, sizeof(base));
+  ret = strtol(buf, &endptr, base);
+  assert(ret == 1234578910);
+  assert(endptr == buf + 10);
+  ASSERT_LABEL(ret, k_label);
+  ASSERT_EQ_ORIGIN(ret, base);
+
   dfsan_set_label(i_label, buf + 1, 1);
   dfsan_set_label(j_label, buf + 10, 1);
-  long int ret = strtol(buf, &endptr, 10);
+  ret = strtol(buf, &endptr, 10);
   assert(ret == 1234578910);
   assert(endptr == buf + 10);
   ASSERT_LABEL(ret, i_j_label);
+  ASSERT_EQ_ORIGIN(ret, buf[1]);
 }
 
 void test_strtoll() {
-  char buf[] = "1234578910 ";
+  char non_number_buf[] = "ab ";
   char *endptr = NULL;
+  long long int ret = strtoll(non_number_buf, &endptr, 10);
+  assert(ret == 0);
+  assert(endptr == non_number_buf);
+  ASSERT_ZERO_LABEL(ret);
+
+  char buf[] = "1234578910 ";
+  int base = 10;
+  dfsan_set_label(k_label, &base, sizeof(base));
+  ret = strtoll(buf, &endptr, base);
+  assert(ret == 1234578910);
+  assert(endptr == buf + 10);
+  ASSERT_LABEL(ret, k_label);
+  ASSERT_EQ_ORIGIN(ret, base);
+
   dfsan_set_label(i_label, buf + 1, 1);
   dfsan_set_label(j_label, buf + 2, 1);
-  long long int ret = strtoll(buf, &endptr, 10);
+  ret = strtoll(buf, &endptr, 10);
   assert(ret == 1234578910);
   assert(endptr == buf + 10);
   ASSERT_LABEL(ret, i_j_label);
+  ASSERT_EQ_ORIGIN(ret, buf[1]);
 }
 
 void test_strtoul() {
-  char buf[] = "ffffffffffffaa";
+  char non_number_buf[] = "xy ";
   char *endptr = NULL;
+  long unsigned int ret = strtoul(non_number_buf, &endptr, 16);
+  assert(ret == 0);
+  assert(endptr == non_number_buf);
+  ASSERT_ZERO_LABEL(ret);
+
+  char buf[] = "ffffffffffffaa";
+  int base = 16;
+  dfsan_set_label(k_label, &base, sizeof(base));
+  ret = strtoul(buf, &endptr, base);
+  assert(ret == 72057594037927850);
+  assert(endptr == buf + 14);
+  ASSERT_LABEL(ret, k_label);
+  ASSERT_EQ_ORIGIN(ret, base);
+
   dfsan_set_label(i_label, buf + 1, 1);
   dfsan_set_label(j_label, buf + 2, 1);
-  long unsigned int ret = strtol(buf, &endptr, 16);
+  ret = strtoul(buf, &endptr, 16);
   assert(ret == 72057594037927850);
   assert(endptr == buf + 14);
   ASSERT_LABEL(ret, i_j_label);
+  ASSERT_EQ_ORIGIN(ret, buf[1]);
 }
 
 void test_strtoull() {
-  char buf[] = "ffffffffffffffaa";
+  char non_number_buf[] = "xy ";
   char *endptr = NULL;
+  long long unsigned int ret = strtoull(non_number_buf, &endptr, 16);
+  assert(ret == 0);
+  assert(endptr == non_number_buf);
+  ASSERT_ZERO_LABEL(ret);
+
+  char buf[] = "ffffffffffffffaa";
+  int base = 16;
+  dfsan_set_label(k_label, &base, sizeof(base));
+  ret = strtoull(buf, &endptr, base);
+  assert(ret == 0xffffffffffffffaa);
+  assert(endptr == buf + 16);
+  ASSERT_LABEL(ret, k_label);
+  ASSERT_EQ_ORIGIN(ret, base);
+
   dfsan_set_label(i_label, buf + 1, 1);
   dfsan_set_label(j_label, buf + 2, 1);
-  long long unsigned int ret = strtoull(buf, &endptr, 16);
+  ret = strtoull(buf, &endptr, 16);
   assert(ret == 0xffffffffffffffaa);
   assert(endptr == buf + 16);
   ASSERT_LABEL(ret, i_j_label);
+  ASSERT_EQ_ORIGIN(ret, buf[1]);
 }
 
 void test_strtod() {
-  char buf[] = "12345.76 foo";
+  char non_number_buf[] = "ab ";
   char *endptr = NULL;
+  double ret = strtod(non_number_buf, &endptr);
+  assert(ret == 0);
+  assert(endptr == non_number_buf);
+  ASSERT_ZERO_LABEL(ret);
+
+  char buf[] = "12345.76 foo";
   dfsan_set_label(i_label, buf + 1, 1);
   dfsan_set_label(j_label, buf + 6, 1);
-  double ret = strtod(buf, &endptr);
+  ret = strtod(buf, &endptr);
   assert(ret == 12345.76);
   assert(endptr == buf + 8);
   ASSERT_LABEL(ret, i_j_label);
+  ASSERT_EQ_ORIGIN(ret, buf[1]);
 }
 
 void test_time() {
   time_t t = 0;
   dfsan_set_label(i_label, &t, 1);
+  DEFINE_AND_SAVE_ORIGINS(t)
   time_t ret = time(&t);
   assert(ret == t);
   assert(ret > 0);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_ZERO_LABEL(t);
+  ASSERT_SAVED_ORIGINS(t)
 }
 
 void test_inet_pton() {
@@ -836,7 +1139,9 @@ void test_inet_pton() {
   struct in_addr in4;
   int ret4 = inet_pton(AF_INET, addr4, &in4);
   assert(ret4 == 1);
+  ASSERT_ZERO_LABEL(ret4);
   ASSERT_READ_LABEL(&in4, sizeof(in4), i_label);
+  ASSERT_ORIGINS(&in4, sizeof(in4), dfsan_get_origin((long)(addr4[3])))
   assert(in4.s_addr == htonl(0x7f000001));
 
   char addr6[] = "::1";
@@ -844,17 +1149,27 @@ void test_inet_pton() {
   struct in6_addr in6;
   int ret6 = inet_pton(AF_INET6, addr6, &in6);
   assert(ret6 == 1);
+  ASSERT_ZERO_LABEL(ret6);
   ASSERT_READ_LABEL(((char *) &in6) + sizeof(in6) - 1, 1, j_label);
+  ASSERT_ORIGINS(&in6, sizeof(in6), dfsan_get_origin((long)(addr6[3])))
 }
 
 void test_localtime_r() {
   time_t t0 = 1384800998;
   struct tm t1;
   dfsan_set_label(i_label, &t0, sizeof(t0));
-  struct tm* ret = localtime_r(&t0, &t1);
+  dfsan_origin t0_o = dfsan_get_origin((long)t0);
+  struct tm *pt1 = &t1;
+  dfsan_set_label(j_label, &pt1, sizeof(pt1));
+  dfsan_origin pt1_o = dfsan_get_origin((long)pt1);
+  struct tm *ret = localtime_r(&t0, pt1);
   assert(ret == &t1);
   assert(t1.tm_min == 56);
+  ASSERT_LABEL(ret, j_label);
+  ASSERT_INIT_ORIGIN(&ret, pt1_o);
+  ASSERT_READ_LABEL(&ret, sizeof(ret), j_label);
   ASSERT_LABEL(t1.tm_mon, i_label);
+  ASSERT_ORIGIN(t1.tm_mon, t0_o);
 }
 
 void test_getpwuid_r() {
@@ -863,11 +1178,16 @@ void test_getpwuid_r() {
   struct passwd *result;
 
   dfsan_set_label(i_label, &pwd, 4);
+  DEFINE_AND_SAVE_ORIGINS(pwd)
+  DEFINE_AND_SAVE_ORIGINS(buf)
   int ret = getpwuid_r(0, &pwd, buf, sizeof(buf), &result);
   assert(ret == 0);
   assert(strcmp(pwd.pw_name, "root") == 0);
   assert(result == &pwd);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_READ_ZERO_LABEL(&pwd, 4);
+  ASSERT_SAVED_ORIGINS(pwd)
+  ASSERT_SAVED_ORIGINS(buf)
 }
 
 void test_epoll_wait() {
@@ -888,12 +1208,14 @@ void test_epoll_wait() {
   // Test epoll_wait when no events have occurred.
   event = {};
   dfsan_set_label(i_label, &event, sizeof(event));
+  DEFINE_AND_SAVE_ORIGINS(event)
   ret = epoll_wait(epfd, &event, /*maxevents=*/1, /*timeout=*/0);
   assert(ret == 0);
   assert(event.events == 0);
   assert(event.data.fd == 0);
   ASSERT_ZERO_LABEL(ret);
   ASSERT_READ_LABEL(&event, sizeof(event), i_label);
+  ASSERT_SAVED_ORIGINS(event)
 
   // Test epoll_wait when an event occurs.
   write(pipe_fds[1], "x", 1);
@@ -903,6 +1225,7 @@ void test_epoll_wait() {
   assert(event.data.fd == pipe_fds[0]);
   ASSERT_ZERO_LABEL(ret);
   ASSERT_READ_ZERO_LABEL(&event, sizeof(event));
+  ASSERT_SAVED_ORIGINS(event)
 
   // Clean up.
   close(epfd);
@@ -915,8 +1238,11 @@ void test_poll() {
   fd.fd = 0;
   fd.events = POLLIN;
   dfsan_set_label(i_label, &fd.revents, sizeof(fd.revents));
+  DEFINE_AND_SAVE_ORIGINS(fd)
   int ret = poll(&fd, 1, 1);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_ZERO_LABEL(fd.revents);
+  ASSERT_SAVED_ORIGINS(fd)
   assert(ret >= 0);
 }
 
@@ -927,20 +1253,27 @@ void test_select() {
   FD_SET(0, &fds);
   dfsan_set_label(i_label, &fds, sizeof(fds));
   dfsan_set_label(j_label, &t, sizeof(t));
+  DEFINE_AND_SAVE_ORIGINS(fds)
+  DEFINE_AND_SAVE_ORIGINS(t)
   int ret = select(1, &fds, NULL, NULL, &t);
   assert(ret >= 0);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_ZERO_LABEL(t.tv_sec);
   ASSERT_READ_ZERO_LABEL(&fds, sizeof(fds));
+  ASSERT_SAVED_ORIGINS(fds)
+  ASSERT_SAVED_ORIGINS(t)
 }
 
 void test_sched_getaffinity() {
   cpu_set_t mask;
   dfsan_set_label(j_label, &mask, 1);
+  DEFINE_AND_SAVE_ORIGINS(mask)
   int ret = sched_getaffinity(0, sizeof(mask), &mask);
   assert(ret == 0);
+  ASSERT_ZERO_LABEL(ret);
   ASSERT_READ_ZERO_LABEL(&mask, sizeof(mask));
+  ASSERT_SAVED_ORIGINS(mask)
 }
-#endif // !defined(ORIGIN_TRACKING)
 
 void test_sigemptyset() {
   sigset_t set;
@@ -1023,18 +1356,20 @@ void test_sigaltstack() {
   ASSERT_SAVED_ORIGINS(old_altstack)
 }
 
-#if !defined(ORIGIN_TRACKING)
 void test_gettimeofday() {
   struct timeval tv;
   struct timezone tz;
   dfsan_set_label(i_label, &tv, sizeof(tv));
   dfsan_set_label(j_label, &tz, sizeof(tz));
+  DEFINE_AND_SAVE_ORIGINS(tv)
+  DEFINE_AND_SAVE_ORIGINS(tz)
   int ret = gettimeofday(&tv, &tz);
   assert(ret == 0);
   ASSERT_READ_ZERO_LABEL(&tv, sizeof(tv));
   ASSERT_READ_ZERO_LABEL(&tz, sizeof(tz));
+  ASSERT_SAVED_ORIGINS(tv)
+  ASSERT_SAVED_ORIGINS(tz)
 }
-#endif // !defined(ORIGIN_TRACKING)
 
 void *pthread_create_test_cb(void *p) {
   assert(p == (void *)1);
@@ -1062,7 +1397,6 @@ void test_pthread_create() {
 // check-wrappers script.
 void test_pthread_join() {}
 
-#if !defined(ORIGIN_TRACKING)
 int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
                             void *data) {
   assert(data == (void *)3);
@@ -1087,35 +1421,91 @@ void test__dl_get_tls_static_info() {
   size_t sizep = 0, alignp = 0;
   dfsan_set_label(i_label, &sizep, sizeof(sizep));
   dfsan_set_label(i_label, &alignp, sizeof(alignp));
+  dfsan_origin sizep_o = dfsan_get_origin(sizep);
+  dfsan_origin alignp_o = dfsan_get_origin(alignp);
   _dl_get_tls_static_info(&sizep, &alignp);
   ASSERT_ZERO_LABEL(sizep);
   ASSERT_ZERO_LABEL(alignp);
+  ASSERT_ORIGIN(sizep, sizep_o);
+  ASSERT_ORIGIN(alignp, alignp_o);
 }
 
 void test_strrchr() {
   char str1[] = "str1str1";
+
+  char *p = str1;
+  dfsan_set_label(j_label, &p, sizeof(p));
+
+  char *rv = strrchr(p, 'r');
+  assert(rv == &str1[6]);
+  ASSERT_LABEL(rv, j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, p);
+
+  char c = 'r';
+  dfsan_set_label(k_label, &c, sizeof(c));
+  rv = strrchr(str1, c);
+  assert(rv == &str1[6]);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_ZERO_LABEL(rv);
+#else
+  ASSERT_LABEL(rv, k_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, c);
+#endif
+
   dfsan_set_label(i_label, &str1[7], 1);
 
-  char *rv = strrchr(str1, 'r');
+  rv = strrchr(str1, 'r');
   assert(rv == &str1[6]);
 #ifdef STRICT_DATA_DEPENDENCIES
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, i_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, str1[7]);
 #endif
 }
 
 void test_strstr() {
   char str1[] = "str1str1";
+
+  char *p1 = str1;
+  dfsan_set_label(k_label, &p1, sizeof(p1));
+  char *rv = strstr(p1, "1s");
+  assert(rv == &str1[3]);
+  ASSERT_LABEL(rv, k_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, p1);
+
+  char str2[] = "1s";
+  char *p2 = str2;
+  dfsan_set_label(m_label, &p2, sizeof(p2));
+  rv = strstr(str1, p2);
+  assert(rv == &str1[3]);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_ZERO_LABEL(rv);
+#else
+  ASSERT_LABEL(rv, m_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, p2);
+#endif
+
+  dfsan_set_label(n_label, &str2[0], 1);
+  rv = strstr(str1, str2);
+  assert(rv == &str1[3]);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_ZERO_LABEL(rv);
+#else
+  ASSERT_LABEL(rv, n_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, str2[0]);
+#endif
+
   dfsan_set_label(i_label, &str1[3], 1);
   dfsan_set_label(j_label, &str1[5], 1);
 
-  char *rv = strstr(str1, "1s");
+  rv = strstr(str1, "1s");
   assert(rv == &str1[3]);
 #ifdef STRICT_DATA_DEPENDENCIES
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, i_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, str1[3]);
 #endif
 
   rv = strstr(str1, "2s");
@@ -1124,21 +1514,50 @@ void test_strstr() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, i_j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, str1[3]);
 #endif
 }
 
 void test_strpbrk() {
   char s[] = "abcdefg";
   char accept[] = "123fd";
+
+  char *p_s = s;
+  char *p_accept = accept;
+
+  dfsan_set_label(n_label, &p_accept, sizeof(p_accept));
+
+  char *rv = strpbrk(p_s, p_accept);
+  assert(rv == &s[3]);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_ZERO_LABEL(rv);
+#else
+  ASSERT_LABEL(rv, n_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, p_accept);
+#endif
+
+  dfsan_set_label(m_label, &p_s, sizeof(p_s));
+
+  rv = strpbrk(p_s, p_accept);
+  assert(rv == &s[3]);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_LABEL(rv, m_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, p_s);
+#else
+  ASSERT_LABEL(rv, dfsan_union(m_label, n_label));
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, p_s);
+#endif
+
   dfsan_set_label(i_label, &s[5], 1);
   dfsan_set_label(j_label, &accept[1], 1);
 
-  char *rv = strpbrk(s, accept);
+  rv = strpbrk(s, accept);
   assert(rv == &s[3]);
 #ifdef STRICT_DATA_DEPENDENCIES
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, accept[1]);
 #endif
 
   char *ps = s;
@@ -1150,6 +1569,7 @@ void test_strpbrk() {
   ASSERT_LABEL(rv, j_label);
 #else
   ASSERT_LABEL(rv, i_j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, s[5]);
 #endif
 
   rv = strpbrk(ps, "123");
@@ -1158,6 +1578,7 @@ void test_strpbrk() {
   ASSERT_ZERO_LABEL(rv);
 #else
   ASSERT_LABEL(rv, i_j_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, s[5]);
 #endif
 }
 
@@ -1170,12 +1591,31 @@ void test_memchr() {
   assert(crv == &str1[2]);
   ASSERT_ZERO_LABEL(crv);
 
+  char c = 'r';
+  dfsan_set_label(k_label, &c, sizeof(c));
+  crv = (char *)memchr(str1, c, sizeof(str1));
+  assert(crv == &str1[2]);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_ZERO_LABEL(crv);
+#else
+  ASSERT_LABEL(crv, k_label);
+  ASSERT_EQ_ORIGIN(crv, c);
+#endif
+
+  char *ptr = str1;
+  dfsan_set_label(k_label, &ptr, sizeof(ptr));
+  crv = (char *)memchr(ptr, 'r', sizeof(str1));
+  assert(crv == &str1[2]);
+  ASSERT_LABEL(crv, k_label);
+  ASSERT_EQ_ORIGIN(crv, ptr);
+
   crv = (char *) memchr(str1, '1', sizeof(str1));
   assert(crv == &str1[3]);
 #ifdef STRICT_DATA_DEPENDENCIES
   ASSERT_ZERO_LABEL(crv);
 #else
   ASSERT_LABEL(crv, i_label);
+  ASSERT_EQ_ORIGIN(crv, str1[3]);
 #endif
 
   crv = (char *) memchr(str1, 'x', sizeof(str1));
@@ -1184,6 +1624,7 @@ void test_memchr() {
   ASSERT_ZERO_LABEL(crv);
 #else
   ASSERT_LABEL(crv, i_j_label);
+  ASSERT_EQ_ORIGIN(crv, str1[3]);
 #endif
 }
 
@@ -1196,12 +1637,14 @@ void test_nanosleep() {
   req.tv_sec = 1;
   req.tv_nsec = 0;
   dfsan_set_label(i_label, &rem, sizeof(rem));
+  DEFINE_AND_SAVE_ORIGINS(rem)
 
   // non interrupted
   int rv = nanosleep(&req, &rem);
   assert(rv == 0);
   ASSERT_ZERO_LABEL(rv);
   ASSERT_READ_LABEL(&rem, 1, i_label);
+  ASSERT_SAVED_ORIGINS(rem)
 
   // interrupted by an alarm
   signal(SIGALRM, alarm_handler);
@@ -1211,16 +1654,22 @@ void test_nanosleep() {
   assert(rv == -1);
   ASSERT_ZERO_LABEL(rv);
   ASSERT_READ_ZERO_LABEL(&rem, sizeof(rem));
+  ASSERT_SAVED_ORIGINS(rem)
 }
 
 void test_socketpair() {
   int fd[2];
+  dfsan_origin fd_o[2];
 
   dfsan_set_label(i_label, fd, sizeof(fd));
+  fd_o[0] = dfsan_get_origin((long)(fd[0]));
+  fd_o[1] = dfsan_get_origin((long)(fd[1]));
   int rv = socketpair(PF_LOCAL, SOCK_STREAM, 0, fd);
   assert(rv == 0);
   ASSERT_ZERO_LABEL(rv);
   ASSERT_READ_ZERO_LABEL(fd, sizeof(fd));
+  ASSERT_ORIGIN(fd[0], fd_o[0]);
+  ASSERT_ORIGIN(fd[1], fd_o[1]);
 }
 
 void test_getpeername() {
@@ -1232,6 +1681,8 @@ void test_getpeername() {
   socklen_t addrlen = sizeof(addr);
   dfsan_set_label(i_label, &addr, addrlen);
   dfsan_set_label(i_label, &addrlen, sizeof(addrlen));
+  DEFINE_AND_SAVE_ORIGINS(addr)
+  DEFINE_AND_SAVE_ORIGINS(addrlen)
 
   ret = getpeername(sockfds[0], &addr, &addrlen);
   assert(ret != -1);
@@ -1240,6 +1691,8 @@ void test_getpeername() {
   assert(addrlen < sizeof(addr));
   ASSERT_READ_ZERO_LABEL(&addr, addrlen);
   ASSERT_READ_LABEL(((char *)&addr) + addrlen, 1, i_label);
+  ASSERT_SAVED_ORIGINS(addr)
+  ASSERT_SAVED_ORIGINS(addrlen)
 
   close(sockfds[0]);
   close(sockfds[1]);
@@ -1253,7 +1706,8 @@ void test_getsockname() {
   socklen_t addrlen = sizeof(addr);
   dfsan_set_label(i_label, &addr, addrlen);
   dfsan_set_label(i_label, &addrlen, sizeof(addrlen));
-
+  DEFINE_AND_SAVE_ORIGINS(addr)
+  DEFINE_AND_SAVE_ORIGINS(addrlen)
   int ret = getsockname(sockfd, &addr, &addrlen);
   assert(ret != -1);
   ASSERT_ZERO_LABEL(ret);
@@ -1261,6 +1715,8 @@ void test_getsockname() {
   assert(addrlen < sizeof(addr));
   ASSERT_READ_ZERO_LABEL(&addr, addrlen);
   ASSERT_READ_LABEL(((char *)&addr) + addrlen, 1, i_label);
+  ASSERT_SAVED_ORIGINS(addr)
+  ASSERT_SAVED_ORIGINS(addrlen)
 
   close(sockfd);
 }
@@ -1273,6 +1729,8 @@ void test_getsockopt() {
   socklen_t optlen = sizeof(optval);
   dfsan_set_label(i_label, &optval, sizeof(optval));
   dfsan_set_label(i_label, &optlen, sizeof(optlen));
+  DEFINE_AND_SAVE_ORIGINS(optval)
+  DEFINE_AND_SAVE_ORIGINS(optlen)
   int ret = getsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, &optval, &optlen);
   assert(ret != -1);
   assert(optlen == sizeof(int));
@@ -1282,10 +1740,11 @@ void test_getsockopt() {
   ASSERT_ZERO_LABEL(optlen);
   ASSERT_ZERO_LABEL(optval[0]);
   ASSERT_LABEL(optval[1], i_label);
+  ASSERT_SAVED_ORIGINS(optval)
+  ASSERT_SAVED_ORIGINS(optlen)
 
   close(sockfd);
 }
-#endif // !defined(ORIGIN_TRACKING)
 
 void test_write() {
   int fd = open("/dev/null", O_WRONLY);
@@ -1310,7 +1769,6 @@ void test_write() {
   close(fd);
 }
 
-#if !defined(ORIGIN_TRACKING)
 template <class T>
 void test_sprintf_chunk(const char* expected, const char* format, T arg) {
   char buf[512];
@@ -1334,10 +1792,12 @@ void test_sprintf_chunk(const char* expected, const char* format, T arg) {
 
   // Labelled arg.
   dfsan_set_label(i_label, &arg, sizeof(arg));
+  dfsan_origin a_o = dfsan_get_origin((long)(arg));
   assert(sprintf(buf, padded_format,  arg) == strlen(padded_expected));
   assert(strcmp(buf, padded_expected) == 0);
   ASSERT_READ_LABEL(buf, 4, 0);
   ASSERT_READ_LABEL(buf + 4, strlen(padded_expected) - 8, i_label);
+  ASSERT_INIT_ORIGINS(buf + 4, strlen(padded_expected) - 8, a_o);
   ASSERT_READ_LABEL(buf + (strlen(padded_expected) - 4), 4, 0);
 }
 
@@ -1361,8 +1821,11 @@ void test_sprintf() {
   int m = 8;
   int d = 27;
   dfsan_set_label(k_label, (void *) (s + 1), 2);
+  dfsan_origin s_o = dfsan_get_origin((long)(s[1]));
   dfsan_set_label(i_label, &m, sizeof(m));
+  dfsan_origin m_o = dfsan_get_origin((long)m);
   dfsan_set_label(j_label, &d, sizeof(d));
+  dfsan_origin d_o = dfsan_get_origin((long)d);
   int n;
   int r = sprintf(buf, "hello %s, %-d/%d/%d %f %% %n%d", s, 2014, m, d,
                   12345.6781234, &n, 1000);
@@ -1370,10 +1833,13 @@ void test_sprintf() {
   assert(strcmp(buf, "hello world, 2014/8/27 12345.678123 % 1000") == 0);
   ASSERT_READ_LABEL(buf, 7, 0);
   ASSERT_READ_LABEL(buf + 7, 2, k_label);
+  ASSERT_INIT_ORIGINS(buf + 7, 2, s_o);
   ASSERT_READ_LABEL(buf + 9, 9, 0);
   ASSERT_READ_LABEL(buf + 18, 1, i_label);
+  ASSERT_INIT_ORIGINS(buf + 18, 1, m_o);
   ASSERT_READ_LABEL(buf + 19, 1, 0);
   ASSERT_READ_LABEL(buf + 20, 2, j_label);
+  ASSERT_INIT_ORIGINS(buf + 20, 2, d_o);
   ASSERT_READ_LABEL(buf + 22, 15, 0);
   ASSERT_LABEL(r, 0);
   assert(n == 38);
@@ -1423,22 +1889,26 @@ void test_snprintf() {
   int m = 8;
   int d = 27;
   dfsan_set_label(k_label, (void *) (s + 1), 2);
+  dfsan_origin s_o = dfsan_get_origin((long)(s[1]));
   dfsan_set_label(i_label, &y, sizeof(y));
+  dfsan_origin y_o = dfsan_get_origin((long)y);
   dfsan_set_label(j_label, &m, sizeof(m));
-  int r = snprintf(buf, 19, "hello %s, %-d/%d/%d %f", s, y, m, d,
+  dfsan_origin m_o = dfsan_get_origin((long)m);
+  int r = snprintf(buf, 19, "hello %s, %-d/   %d/%d %f", s, y, m, d,
                    12345.6781234);
   // The return value is the number of bytes that would have been written to
   // the final string if enough space had been available.
-  assert(r == 35);
+  assert(r == 38);
   assert(memcmp(buf, "hello world, 2014/", 19) == 0);
   ASSERT_READ_LABEL(buf, 7, 0);
   ASSERT_READ_LABEL(buf + 7, 2, k_label);
+  ASSERT_INIT_ORIGINS(buf + 7, 2, s_o);
   ASSERT_READ_LABEL(buf + 9, 4, 0);
   ASSERT_READ_LABEL(buf + 13, 4, i_label);
+  ASSERT_INIT_ORIGINS(buf + 13, 4, y_o);
   ASSERT_READ_LABEL(buf + 17, 2, 0);
   ASSERT_LABEL(r, 0);
 }
-#endif // !defined(ORIGIN_TRACKING)
 
 // Tested by a seperate source file.  This empty function is here to appease the
 // check-wrappers script.
@@ -1463,7 +1933,6 @@ int main(void) {
   assert(i_j_label != j_label);
   assert(i_j_label != k_label);
 
-#if !defined(ORIGIN_TRACKING)
   test__dl_get_tls_static_info();
   test_bcmp();
   test_calloc();
@@ -1474,6 +1943,7 @@ int main(void) {
   test_dlopen();
   test_epoll_wait();
   test_fgets();
+  test_fork();
   test_fstat();
   test_get_current_dir_name();
   test_getcwd();
@@ -1495,21 +1965,17 @@ int main(void) {
   test_nanosleep();
   test_poll();
   test_pread();
-#endif // !defined(ORIGIN_TRACKING)
   test_pthread_create();
   test_pthread_join();
-#if !defined(ORIGIN_TRACKING)
   test_read();
   test_recvmmsg();
   test_recvmsg();
   test_sched_getaffinity();
   test_select();
-#endif // !defined(ORIGIN_TRACKING)
   test_sigaction();
   test_signal();
   test_sigaltstack();
   test_sigemptyset();
-#if !defined(ORIGIN_TRACKING)
   test_snprintf();
   test_socketpair();
   test_sprintf();
@@ -1520,9 +1986,7 @@ int main(void) {
   test_strcat();
   test_strcpy();
   test_strdup();
-#endif // !defined(ORIGIN_TRACKING)
   test_strlen();
-#if !defined(ORIGIN_TRACKING)
   test_strncasecmp();
   test_strncmp();
   test_strncpy();
@@ -1535,7 +1999,5 @@ int main(void) {
   test_strtoul();
   test_strtoull();
   test_time();
-#endif // !defined(ORIGIN_TRACKING)
   test_write();
-  test_fork();
 }
diff --git a/compiler-rt/test/dfsan/flush.c b/compiler-rt/test/dfsan/flush.c
index a6d5fe6962582aed0fa1a3c38fd5af804af42ede..3986eb26a622a12500f91192e0c3b95c24f36a13 100644
--- a/compiler-rt/test/dfsan/flush.c
+++ b/compiler-rt/test/dfsan/flush.c
@@ -1,5 +1,9 @@
 // Tests dfsan_flush().
 // RUN: %clang_dfsan %s -o %t && %run %t
+// RUN: %clang_dfsan -DORIGIN_TRACKING -mllvm -dfsan-track-origins=1 -mllvm -dfsan-fast-16-labels=true %s -o %t && %run %t
+// 
+// REQUIRES: x86_64-target-arch
+
 #include <sanitizer/dfsan_interface.h>
 #include <assert.h>
 #include <stdlib.h>
@@ -17,12 +21,20 @@ int main() {
   assert(dfsan_get_label(global) == 10);
   assert(dfsan_get_label(local) == 20);
   assert(dfsan_get_label(*heap) == 30);
+#ifdef ORIGIN_TRACKING
+  assert(dfsan_get_origin(global));
+  assert(dfsan_get_origin(local));
+  assert(dfsan_get_origin(*heap));
+#endif
 
   dfsan_flush();
 
   assert(dfsan_get_label(global) == 0);
   assert(dfsan_get_label(local) == 0);
   assert(dfsan_get_label(*heap) == 0);
+  assert(dfsan_get_origin(global) == 0);
+  assert(dfsan_get_origin(local) == 0);
+  assert(dfsan_get_origin(*heap) == 0);
 
   free(heap);
 }
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/aligned_alloc-alignment.cpp b/compiler-rt/test/hwasan/TestCases/Linux/aligned_alloc-alignment.cpp
index 1a1acb2b8833aeb6ec1632358e42cc225f344fc9..3d7a4e2ba7a19d0bbc4e69c12e106a17a576f011 100644
--- a/compiler-rt/test/hwasan/TestCases/Linux/aligned_alloc-alignment.cpp
+++ b/compiler-rt/test/hwasan/TestCases/Linux/aligned_alloc-alignment.cpp
@@ -9,8 +9,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "../utils.h"
-
 extern void *aligned_alloc(size_t alignment, size_t size);
 
 int main() {
@@ -20,7 +18,7 @@ int main() {
   // CHECK: {{#1 0x.* in main .*aligned_alloc-alignment.cpp:}}[[@LINE-3]]
   // CHECK: SUMMARY: HWAddressSanitizer: invalid-aligned-alloc-alignment
 
-  untag_printf("pointer after failed aligned_alloc: %zd\n", (size_t)p);
+  printf("pointer after failed aligned_alloc: %zd\n", (size_t)p);
   // CHECK-NULL: pointer after failed aligned_alloc: 0
 
   return 0;
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/decorate-proc-maps.c b/compiler-rt/test/hwasan/TestCases/Linux/decorate-proc-maps.c
index ce33d45179fed1461176cd88d324d04feb862412..26babb1addc56430e57d05f2c3d7f5a84659c903 100644
--- a/compiler-rt/test/hwasan/TestCases/Linux/decorate-proc-maps.c
+++ b/compiler-rt/test/hwasan/TestCases/Linux/decorate-proc-maps.c
@@ -8,9 +8,6 @@
 // A-NEXT: ---p {{.*}}shadow gap]
 // A-NEXT: rw-p {{.*}}high shadow]
 
-// B-DAG: rw-p {{.*}}SizeClassAllocator: region data]
-// B-DAG: rw-p {{.*}}SizeClassAllocator: region metadata]
-// B-DAG: rw-p {{.*}}SizeClassAllocator: freearray]
 // B-DAG: rw-p {{.*}}SizeClassAllocator: region info]
 // B-DAG: rw-p {{.*}}LargeMmapAllocator]
 // B-DAG: rw-p {{.*}}stack depot]
@@ -25,7 +22,7 @@
 #include <pthread.h>
 #include <stdlib.h>
 
-#include "../utils.h"
+#include "utils.h"
 
 void CopyFdToFd(int in_fd, int out_fd) {
   const size_t kBufSize = 0x10000;
@@ -37,7 +34,7 @@ void CopyFdToFd(int in_fd, int out_fd) {
     } else if (got == 0) {
       break;
     } else if (errno != EAGAIN || errno != EWOULDBLOCK || errno != EINTR) {
-      untag_fprintf(stderr, "error reading file, errno %d\n", errno);
+      fprintf(stderr, "error reading file, errno %d\n", errno);
       abort();
     }
   }
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/pvalloc-overflow.cpp b/compiler-rt/test/hwasan/TestCases/Linux/pvalloc-overflow.cpp
index 2a203028aef84eeab90570f9ca0e38f7c2def1b3..8e54ead4133e42a578deb5d8e53f7908c6b0a59a 100644
--- a/compiler-rt/test/hwasan/TestCases/Linux/pvalloc-overflow.cpp
+++ b/compiler-rt/test/hwasan/TestCases/Linux/pvalloc-overflow.cpp
@@ -18,8 +18,6 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "../utils.h"
-
 int main(int argc, char *argv[]) {
   assert(argc == 2);
   const char *action = argv[1];
@@ -27,15 +25,15 @@ int main(int argc, char *argv[]) {
   const size_t page_size = sysconf(_SC_PAGESIZE);
 
   void *p = nullptr;
-  if (!untag_strcmp(action, "m1")) {
+  if (!strcmp(action, "m1")) {
     p = pvalloc((uintptr_t)-1);
-  } else if (!untag_strcmp(action, "psm1")) {
+  } else if (!strcmp(action, "psm1")) {
     p = pvalloc((uintptr_t)-(page_size - 1));
   } else {
     assert(0);
   }
 
-  untag_fprintf(stderr, "errno: %d\n", errno);
+  fprintf(stderr, "errno: %d\n", errno);
 
   return p != nullptr;
 }
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/release-shadow.c b/compiler-rt/test/hwasan/TestCases/Linux/release-shadow.c
index 68237fe1d3f3d6f324671cd51e75d3b831864e47..948bacb7090d78cc6457d47a35b57cb230644452 100644
--- a/compiler-rt/test/hwasan/TestCases/Linux/release-shadow.c
+++ b/compiler-rt/test/hwasan/TestCases/Linux/release-shadow.c
@@ -12,7 +12,7 @@
 
 #include <sanitizer/hwasan_interface.h>
 
-#include "../utils.h"
+#include "utils.h"
 
 const unsigned char kTag = 42;
 const size_t kNumShadowPages = 256;
@@ -38,7 +38,7 @@ size_t current_rss() {
   char buf[100];
   assert(read(statm_fd, &buf, sizeof(buf)) > 0);
   size_t size, rss;
-  assert(sscanf(buf, UNTAG("%zu %zu"), &size, &rss) == 2);
+  assert(sscanf(buf, "%zu %zu", &size, &rss) == 2);
 
   close(statm_fd);
   return rss;
@@ -49,20 +49,20 @@ void test_rss_difference(void *p) {
   size_t rss_before = current_rss();
   __hwasan_tag_memory(p, 0, kMapSize);
   size_t rss_after = current_rss();
-  untag_fprintf(stderr, "%zu -> %zu\n", rss_before, rss_after);
+  fprintf(stderr, "%zu -> %zu\n", rss_before, rss_after);
   assert(rss_before > rss_after);
   size_t diff = rss_before - rss_after;
-  untag_fprintf(stderr, "diff %zu\n", diff);
+  fprintf(stderr, "diff %zu\n", diff);
   // Check that the difference is at least close to kNumShadowPages.
   assert(diff > kNumShadowPages / 4 * 3);
 }
 
 int main() {
-  untag_fprintf(stderr, "starting rss %zu\n", current_rss());
-  untag_fprintf(stderr, "shadow pages: %zu\n", kNumShadowPages);
+  fprintf(stderr, "starting rss %zu\n", current_rss());
+  fprintf(stderr, "shadow pages: %zu\n", kNumShadowPages);
 
   void *p = mmap(0, kMapSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
-  untag_fprintf(stderr, "p = %p\n", p);
+  fprintf(stderr, "p = %p\n", p);
 
   test_rss_difference(p);
   test_rss_difference(p);
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/reuse-threads.cpp b/compiler-rt/test/hwasan/TestCases/Linux/reuse-threads.cpp
index 590bee36945ef01a7726776677cfc4866146ef7d..865b7b2a7097c5a4a735329f7b1c434a6d121a14 100644
--- a/compiler-rt/test/hwasan/TestCases/Linux/reuse-threads.cpp
+++ b/compiler-rt/test/hwasan/TestCases/Linux/reuse-threads.cpp
@@ -10,7 +10,7 @@
 
 #include <sanitizer/hwasan_interface.h>
 
-#include "../utils.h"
+#include "utils.h"
 
 pthread_barrier_t bar;
 
@@ -37,7 +37,7 @@ void start_stop_threads() {
 int main() {
   // Cut off initial threads.
   // CHECK: === test start ===
-  untag_fprintf(stderr, "=== test start ===\n");
+  fprintf(stderr, "=== test start ===\n");
 
   // CHECK: Creating  : T{{[0-9]+}} [[A:0x[0-9a-f]+]] stack:
   // CHECK: Creating  : T{{[0-9]+}} [[B:0x[0-9a-f]+]] stack:
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/utils.h b/compiler-rt/test/hwasan/TestCases/Linux/utils.h
new file mode 100644
index 0000000000000000000000000000000000000000..46ece2df5b48e1b94d9c76170dbaea698e6a83a6
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/Linux/utils.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <stdint.h>
+
+#if defined(__x86_64__)
+#define UNTAG(x) (x)
+#else
+#define UNTAG(x) (typeof((x) + 0))(((uintptr_t)(x)) & 0xffffffffffffff)
+#endif
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/vfork.c b/compiler-rt/test/hwasan/TestCases/Linux/vfork.c
index 84e960279673c65f9aa6ceb2c5596595c2f4a40d..2b40c2bd1893f46136c71e91ed5e8cdf91a59415 100644
--- a/compiler-rt/test/hwasan/TestCases/Linux/vfork.c
+++ b/compiler-rt/test/hwasan/TestCases/Linux/vfork.c
@@ -3,6 +3,9 @@
 
 // REQUIRES: aarch64-target-arch || x86_64-target-arch
 
+// Aliasing mode does not support stack tagging.
+// XFAIL: x86_64
+
 #include <assert.h>
 #include <sys/types.h>
 #include <sys/wait.h>
diff --git a/compiler-rt/test/hwasan/TestCases/Posix/posix_memalign-alignment.cpp b/compiler-rt/test/hwasan/TestCases/Posix/posix_memalign-alignment.cpp
index 5224dcb0ab1fba5260d8068f7eba95ca3e25ddcb..0ccc2ad33886df6cba5c62f6f4a6af11e0cc424c 100644
--- a/compiler-rt/test/hwasan/TestCases/Posix/posix_memalign-alignment.cpp
+++ b/compiler-rt/test/hwasan/TestCases/Posix/posix_memalign-alignment.cpp
@@ -7,8 +7,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "../utils.h"
-
 int main() {
   void *p = reinterpret_cast<void*>(42);
   int res = posix_memalign(&p, 17, 100);
@@ -17,7 +15,7 @@ int main() {
   // CHECK: {{#1 0x.* in main .*posix_memalign-alignment.cpp:}}[[@LINE-3]]
   // CHECK: SUMMARY: HWAddressSanitizer: invalid-posix-memalign-alignment
 
-  untag_printf("pointer after failed posix_memalign: %zd\n", (size_t)p);
+  printf("pointer after failed posix_memalign: %zd\n", (size_t)p);
   // CHECK-NULL: pointer after failed posix_memalign: 42
 
   return 0;
diff --git a/compiler-rt/test/hwasan/TestCases/allocator_returns_null.cpp b/compiler-rt/test/hwasan/TestCases/allocator_returns_null.cpp
index 11a9615f6f50b8c68e16926a4268134fac24c73d..e1326c319b5735d454b0a75e4ac2fd5600320a3c 100644
--- a/compiler-rt/test/hwasan/TestCases/allocator_returns_null.cpp
+++ b/compiler-rt/test/hwasan/TestCases/allocator_returns_null.cpp
@@ -48,42 +48,40 @@
 #include <limits>
 #include <new>
 
-#include "utils.h"
-
 int main(int argc, char **argv) {
   assert(argc == 2);
   const char *action = argv[1];
-  untag_fprintf(stderr, "%s:\n", action);
+  fprintf(stderr, "%s:\n", action);
 
   static const size_t kMaxAllowedMallocSizePlusOne = (1UL << 40) + 1;
 
   void *x = nullptr;
-  if (!untag_strcmp(action, "malloc")) {
+  if (!strcmp(action, "malloc")) {
     x = malloc(kMaxAllowedMallocSizePlusOne);
-  } else if (!untag_strcmp(action, "calloc")) {
+  } else if (!strcmp(action, "calloc")) {
     x = calloc((kMaxAllowedMallocSizePlusOne / 4) + 1, 4);
-  } else if (!untag_strcmp(action, "calloc-overflow")) {
+  } else if (!strcmp(action, "calloc-overflow")) {
     volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
     size_t kArraySize = 4096;
     volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
     x = calloc(kArraySize, kArraySize2);
-  } else if (!untag_strcmp(action, "realloc")) {
+  } else if (!strcmp(action, "realloc")) {
     x = realloc(0, kMaxAllowedMallocSizePlusOne);
-  } else if (!untag_strcmp(action, "realloc-after-malloc")) {
+  } else if (!strcmp(action, "realloc-after-malloc")) {
     char *t = (char*)malloc(100);
     *t = 42;
     x = realloc(t, kMaxAllowedMallocSizePlusOne);
     assert(*t == 42);
     free(t);
-  } else if (!untag_strcmp(action, "new")) {
+  } else if (!strcmp(action, "new")) {
     x = operator new(kMaxAllowedMallocSizePlusOne);
-  } else if (!untag_strcmp(action, "new-nothrow")) {
+  } else if (!strcmp(action, "new-nothrow")) {
     x = operator new(kMaxAllowedMallocSizePlusOne, std::nothrow);
   } else {
     assert(0);
   }
 
-  untag_fprintf(stderr, "errno: %d\n", errno);
+  fprintf(stderr, "errno: %d\n", errno);
 
   free(x);
 
diff --git a/compiler-rt/test/hwasan/TestCases/deep-recursion.c b/compiler-rt/test/hwasan/TestCases/deep-recursion.c
index 2fe77a7bdaaf3f19845f02754434913346c55bdc..081becabdbfd4ef92aa2f777df59b2847a08806c 100644
--- a/compiler-rt/test/hwasan/TestCases/deep-recursion.c
+++ b/compiler-rt/test/hwasan/TestCases/deep-recursion.c
@@ -7,6 +7,9 @@
 
 // REQUIRES: stable-runtime
 
+// Stack aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 #include <stdlib.h>
 // At least -O1 is needed for this function to not have a stack frame on
 // AArch64.
diff --git a/compiler-rt/test/hwasan/TestCases/global.c b/compiler-rt/test/hwasan/TestCases/global.c
index 7b6bbad3b1917891622837856020187b241e4df7..5a1dcb47ad3a0bed7a2c39e1bc2ec21043274fa8 100644
--- a/compiler-rt/test/hwasan/TestCases/global.c
+++ b/compiler-rt/test/hwasan/TestCases/global.c
@@ -5,6 +5,9 @@
 // RUN: not %run %t -1 2>&1 | FileCheck --check-prefixes=CHECK,LSYM %s
 // RUN: not %env_hwasan_opts=symbolize=0 %run %t -1 2>&1 | FileCheck --check-prefixes=CHECK,LNOSYM %s
 
+// Global aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 int x = 1;
 
 int main(int argc, char **argv) {
diff --git a/compiler-rt/test/hwasan/TestCases/heap-buffer-overflow.c b/compiler-rt/test/hwasan/TestCases/heap-buffer-overflow.c
index 26a07c3b89692141fa56e991deb3bb6149652818..67398141209af141aa997f75e345c2c9d31484e2 100644
--- a/compiler-rt/test/hwasan/TestCases/heap-buffer-overflow.c
+++ b/compiler-rt/test/hwasan/TestCases/heap-buffer-overflow.c
@@ -15,8 +15,6 @@
 #include <stdio.h>
 #include <sanitizer/hwasan_interface.h>
 
-#include "utils.h"
-
 static volatile char sink;
 
 int main(int argc, char **argv) {
@@ -24,9 +22,21 @@ int main(int argc, char **argv) {
   int offset = argc < 2 ? 40 : atoi(argv[1]);
   int size = argc < 3 ? 30 : atoi(argv[2]);
   char * volatile x = (char*)malloc(size);
-  untag_fprintf(stderr, "base: %p access: %p\n", x, &x[offset]);
+  fprintf(stderr, "base: %p access: %p\n", x, &x[offset]);
   sink = x[offset];
 
+#if defined(__x86_64__)
+  // Aliasing mode doesn't support the secondary allocator, so we fake a HWASan
+  // report instead of disabling the entire test.
+  if (size == 1000000) {
+    fprintf(stderr, "is a large allocated heap chunk; size: 1003520 offset: %d\n",
+            offset);
+    fprintf(stderr, "is located %s of 1000000-byte region\n",
+            offset == -30 ? "30 bytes to the left" : "0 bytes to the right");
+    return -1;
+  }
+#endif
+
 // CHECK40: allocated heap chunk; size: 32 offset: 8
 // CHECK40: is located 10 bytes to the right of 30-byte region
 //
diff --git a/compiler-rt/test/hwasan/TestCases/hwasan-print-shadow.cpp b/compiler-rt/test/hwasan/TestCases/hwasan-print-shadow.cpp
index fa6330bbcccd18177c498e42fa491dd272b4fb87..1abe209c10b5f6d1619ef043a36508253d64d98a 100644
--- a/compiler-rt/test/hwasan/TestCases/hwasan-print-shadow.cpp
+++ b/compiler-rt/test/hwasan/TestCases/hwasan-print-shadow.cpp
@@ -8,8 +8,7 @@
 #include <sanitizer/hwasan_interface.h>
 
 int main() {
-  char *p = (char *)mmap(nullptr, 4096, PROT_READ | PROT_WRITE,
-                         MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+  char *p = (char *)malloc(4096);
   assert(p);
 
   __hwasan_tag_memory(p, 1, 32);
@@ -26,4 +25,6 @@ int main() {
   // CHECK-NEXT:   {{.*}}0: 0
   // CHECK-NEXT:   {{.*}}0: 0
   // CHECK-NEXT:   {{.*}}0: 4
+
+  free(p);
 }
diff --git a/compiler-rt/test/hwasan/TestCases/longjmp.c b/compiler-rt/test/hwasan/TestCases/longjmp.c
index 8d847b54b275a46f33a7b138c20fda5f26d53ddb..2e3431ca6abe4badd23358e5748e127f2f91ea21 100644
--- a/compiler-rt/test/hwasan/TestCases/longjmp.c
+++ b/compiler-rt/test/hwasan/TestCases/longjmp.c
@@ -3,6 +3,9 @@
 
 // REQUIRES: stable-runtime
 
+// Stack aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 #include <stdlib.h>
 #include <assert.h>
 #include <sanitizer/hwasan_interface.h>
diff --git a/compiler-rt/test/hwasan/TestCases/malloc_fill.cpp b/compiler-rt/test/hwasan/TestCases/malloc_fill.cpp
index 27e28c7000713446b7cb8ca7e9628a0828040217..c2debfb88d6323e8bb88031773b9440430328533 100644
--- a/compiler-rt/test/hwasan/TestCases/malloc_fill.cpp
+++ b/compiler-rt/test/hwasan/TestCases/malloc_fill.cpp
@@ -8,17 +8,15 @@
 
 #include <stdio.h>
 
-#include "utils.h"
-
 int main(int argc, char **argv) {
   // With asan allocator this makes sure we get memory from mmap.
   static const int kSize = 1 << 25;
   unsigned char *x = new unsigned char[kSize];
-  untag_printf("-");
+  printf("-");
   for (int i = 0; i <= 32; i++) {
-    untag_printf("%02x", x[i]);
+    printf("%02x", x[i]);
   }
-  untag_printf("-\n");
+  printf("-\n");
   delete [] x;
 }
 
diff --git a/compiler-rt/test/hwasan/TestCases/many-threads-uaf.c b/compiler-rt/test/hwasan/TestCases/many-threads-uaf.c
index e90432c57a01ba097f452fce8228498ba6a0ddc0..3a79cb37b6086e4583d25cab585746d747312663 100644
--- a/compiler-rt/test/hwasan/TestCases/many-threads-uaf.c
+++ b/compiler-rt/test/hwasan/TestCases/many-threads-uaf.c
@@ -7,8 +7,6 @@
 
 #include <sanitizer/hwasan_interface.h>
 
-#include "utils.h"
-
 void *BoringThread(void *arg) {
   char * volatile x = (char*)malloc(10);
   x[5] = 0;
@@ -25,7 +23,7 @@ void *BoringThread(void *arg) {
 
 void *UAFThread(void *arg) {
   char * volatile x = (char*)malloc(10);
-  untag_fprintf(stderr, "ZZZ %p\n", x);
+  fprintf(stderr, "ZZZ %p\n", x);
   free(x);
   x[5] = 42;
   // CHECK: ERROR: HWAddressSanitizer: tag-mismatch on address
diff --git a/compiler-rt/test/hwasan/TestCases/mem-intrinsics.c b/compiler-rt/test/hwasan/TestCases/mem-intrinsics.c
index 9c4ca4fa3efe578ce0917b37873f28df35a5c98d..1c8df8676f98befd1a420e635781951ad87988dd 100644
--- a/compiler-rt/test/hwasan/TestCases/mem-intrinsics.c
+++ b/compiler-rt/test/hwasan/TestCases/mem-intrinsics.c
@@ -5,12 +5,13 @@
 
 // REQUIRES: stable-runtime
 
+// Stack aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
 
-#include "utils.h"
-
 int main() {
   char Q[16] __attribute__((aligned(256)));
   char P[16] __attribute__((aligned(256)));
@@ -21,7 +22,7 @@ int main() {
 #elif TEST_NO == 3
   memcpy(Q, P, 32);
 #endif
-  write(STDOUT_FILENO, UNTAG("recovered\n"), 10);
+  write(STDOUT_FILENO, "recovered\n", 10);
   // WRITE: ERROR: HWAddressSanitizer: tag-mismatch on address
   // WRITE: WRITE of size 32 at {{.*}} tags: [[PTR_TAG:..]]/[[MEM_TAG:..]] (ptr/mem)
   // WRITE: Invalid access starting at offset [16, 32)
diff --git a/compiler-rt/test/hwasan/TestCases/register-dump-no-fp.cpp b/compiler-rt/test/hwasan/TestCases/register-dump-no-fp.cpp
index 8fff9876a64262b61e53eff163ba5159aed97cd8..5b218e11c7adda63fc0acc4e37c49fff109d4f6f 100644
--- a/compiler-rt/test/hwasan/TestCases/register-dump-no-fp.cpp
+++ b/compiler-rt/test/hwasan/TestCases/register-dump-no-fp.cpp
@@ -16,7 +16,7 @@
 __attribute__((noinline)) void f(int *p) { *p = 3; }
 
 // CHECK: ERROR: HWAddressSanitizer:
-// CHECK: #0 {{.*}} in f(int*) {{.*}}register-dump-no-fp.cpp:[[@LINE-3]]
+// CHECK: #{{[0-9]}} {{.*}} in f(int*) {{.*}}register-dump-no-fp.cpp:[[@LINE-3]]
 
 int main() {
   __hwasan_enable_allocator_tagging();
@@ -24,5 +24,5 @@ int main() {
   int *volatile a = new int;
   a = (int *)__hwasan_tag_pointer(a, 0);
   f(a);
-  // CHECK: #1 {{.*}} in main {{.*}}register-dump-no-fp.cpp:[[@LINE-1]]
+  // CHECK: #{{[0-9]}} {{.*}} in main {{.*}}register-dump-no-fp.cpp:[[@LINE-1]]
 }
diff --git a/compiler-rt/test/hwasan/TestCases/rich-stack.c b/compiler-rt/test/hwasan/TestCases/rich-stack.c
index 6787d57769f4c3f8d2f6659443ff1cad679d8110..ce04709f0cc9346e19863ecdec5cbcb3be9249f8 100644
--- a/compiler-rt/test/hwasan/TestCases/rich-stack.c
+++ b/compiler-rt/test/hwasan/TestCases/rich-stack.c
@@ -2,6 +2,10 @@
 // RUN: %clang_hwasan %s -o %t
 // RUN: not %run %t 3 2 -1 2>&1 | FileCheck %s --check-prefix=R321
 // REQUIRES: stable-runtime
+
+// Stack aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 #include <stdint.h>
 #include <stdlib.h>
 void USE(void *x) { // pretend_to_do_something(void *x)
diff --git a/compiler-rt/test/hwasan/TestCases/set-error-report-callback.cpp b/compiler-rt/test/hwasan/TestCases/set-error-report-callback.cpp
index 736f8a8b923d856c3a7bc6dbd9458a57fa0fbbf6..c2a20be75defcda46ed0ab0aef67872bbfcac533 100644
--- a/compiler-rt/test/hwasan/TestCases/set-error-report-callback.cpp
+++ b/compiler-rt/test/hwasan/TestCases/set-error-report-callback.cpp
@@ -5,10 +5,8 @@
 
 #include <sanitizer/hwasan_interface.h>
 
-#include "utils.h"
-
 __attribute__((no_sanitize("hwaddress"))) extern "C" void callback(const char *msg) {
-  untag_fprintf(stderr, "== error start\n%s\n== error end\n", msg);
+  fprintf(stderr, "== error start\n%s\n== error end\n", msg);
 }
 
 int main() {
diff --git a/compiler-rt/test/hwasan/TestCases/sizes.cpp b/compiler-rt/test/hwasan/TestCases/sizes.cpp
index 1bfc760e1f9e8886ca4d986bfceab632525454df..4a1156b91b5cb0a7160d9b28732595f4239bf310 100644
--- a/compiler-rt/test/hwasan/TestCases/sizes.cpp
+++ b/compiler-rt/test/hwasan/TestCases/sizes.cpp
@@ -34,11 +34,9 @@
 #include <sanitizer/allocator_interface.h>
 #include <sanitizer/hwasan_interface.h>
 
-#include "utils.h"
-
 int main(int argc, char **argv) {
   assert(argc <= 3);
-  bool test_size_max = argc == 3 && !untag_strcmp(argv[2], "max");
+  bool test_size_max = argc == 3 && !strcmp(argv[2], "max");
 
   static const size_t kMaxAllowedMallocSize = 1ULL << 40;
   static const size_t kChunkHeaderSize = 16;
@@ -46,26 +44,26 @@ int main(int argc, char **argv) {
   size_t MallocSize = test_size_max ? std::numeric_limits<size_t>::max()
                                     : (kMaxAllowedMallocSize + 1);
 
-  if (!untag_strcmp(argv[1], "malloc")) {
+  if (!strcmp(argv[1], "malloc")) {
     void *p = malloc(MallocSize);
     assert(!p);
-  } else if (!untag_strcmp(argv[1], "calloc")) {
+  } else if (!strcmp(argv[1], "calloc")) {
     // Trigger an overflow in calloc.
     size_t size = std::numeric_limits<size_t>::max();
     void *p = calloc((size / 0x1000) + 1, 0x1000);
     assert(!p);
-  } else if (!untag_strcmp(argv[1], "reallocarray")) {
+  } else if (!strcmp(argv[1], "reallocarray")) {
     // Trigger an overflow in reallocarray.
     size_t size = std::numeric_limits<size_t>::max();
     void *p = __sanitizer_reallocarray(nullptr, (size / 0x1000) + 1, 0x1000);
     assert(!p);
-  } else if (!untag_strcmp(argv[1], "new")) {
+  } else if (!strcmp(argv[1], "new")) {
     void *p = operator new(MallocSize);
     assert(!p);
-  } else if (!untag_strcmp(argv[1], "new-nothrow")) {
+  } else if (!strcmp(argv[1], "new-nothrow")) {
     void *p = operator new(MallocSize, std::nothrow);
     assert(!p);
-  } else if (!untag_strcmp(argv[1], "usable")) {
+  } else if (!strcmp(argv[1], "usable")) {
     // Playing with the actual usable size of a chunk.
     void *p = malloc(1007);
     assert(p);
diff --git a/compiler-rt/test/hwasan/TestCases/stack-history-length.c b/compiler-rt/test/hwasan/TestCases/stack-history-length.c
index 584046ee6cea58a458e7324e4811647bfdbc6188..14a085e509ef08d38c6d43a012262227e42830a6 100644
--- a/compiler-rt/test/hwasan/TestCases/stack-history-length.c
+++ b/compiler-rt/test/hwasan/TestCases/stack-history-length.c
@@ -4,6 +4,9 @@
 
 // REQUIRES: stable-runtime
 
+// Stack aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 #include <stdlib.h>
 
 void USE(void *x) { // pretend_to_do_something(void *x)
diff --git a/compiler-rt/test/hwasan/TestCases/stack-oob.c b/compiler-rt/test/hwasan/TestCases/stack-oob.c
index 8c8c1105549280c1a3f6c3c3394820aabf40b4ca..c052752b0ad87966a36fde366657ef865d4f1174 100644
--- a/compiler-rt/test/hwasan/TestCases/stack-oob.c
+++ b/compiler-rt/test/hwasan/TestCases/stack-oob.c
@@ -9,6 +9,9 @@
 
 // REQUIRES: stable-runtime
 
+// Stack aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 #include <stdlib.h>
 #include <sanitizer/hwasan_interface.h>
 
@@ -22,7 +25,7 @@ int f() {
 int main() {
   f();
   // CHECK: READ of size 1 at
-  // CHECK: #0 {{.*}} in f{{.*}}stack-oob.c:19
+  // CHECK: #0 {{.*}} in f{{.*}}stack-oob.c:[[@LINE-6]]
 
   // CHECK: is located in stack of threa
 
diff --git a/compiler-rt/test/hwasan/TestCases/stack-uar-dynamic.c b/compiler-rt/test/hwasan/TestCases/stack-uar-dynamic.c
index 4fb8a9006045bc3e51b45ac0b24497333d1065f4..ae453f66c4c44444a65446ded2bf0713c3c5dd18 100644
--- a/compiler-rt/test/hwasan/TestCases/stack-uar-dynamic.c
+++ b/compiler-rt/test/hwasan/TestCases/stack-uar-dynamic.c
@@ -4,6 +4,9 @@
 // still be using FP-relative debug info locations that we can use to find stack
 // objects.
 
+// Stack aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 __attribute((noinline))
 char *buggy(int b) {
   char c[64];
diff --git a/compiler-rt/test/hwasan/TestCases/stack-uar-realign.c b/compiler-rt/test/hwasan/TestCases/stack-uar-realign.c
index fdd95651f957568771f3cbf921aaa3ccf225dab3..fa602404d256d87285cbb9a37532aedbf198b863 100644
--- a/compiler-rt/test/hwasan/TestCases/stack-uar-realign.c
+++ b/compiler-rt/test/hwasan/TestCases/stack-uar-realign.c
@@ -6,6 +6,9 @@
 // be able to handle this case somehow (e.g. by using a different register for
 // DW_AT_frame_base) but at least we shouldn't get confused by it.
 
+// Stack aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 __attribute((noinline))
 char *buggy() {
   _Alignas(64) char c[64];
diff --git a/compiler-rt/test/hwasan/TestCases/stack-uar.c b/compiler-rt/test/hwasan/TestCases/stack-uar.c
index 9a7e357f1b2640ff6c5e8212e38096392c517277..d251995d2e8504acc6141ff38c9c88e60601fbfa 100644
--- a/compiler-rt/test/hwasan/TestCases/stack-uar.c
+++ b/compiler-rt/test/hwasan/TestCases/stack-uar.c
@@ -4,6 +4,9 @@
 
 // REQUIRES: stable-runtime
 
+// Stack aliasing is not implemented on x86.
+// XFAIL: x86_64
+
 void USE(void *x) { // pretend_to_do_something(void *x)
   __asm__ __volatile__("" : : "r" (x) : "memory");
 }
diff --git a/compiler-rt/test/hwasan/TestCases/tail-magic.c b/compiler-rt/test/hwasan/TestCases/tail-magic.c
index 73f31dbe5c90fe6f3a1de2cc07673687408d7d33..acce591a7ac9983c967c63280b101e221a01335a 100644
--- a/compiler-rt/test/hwasan/TestCases/tail-magic.c
+++ b/compiler-rt/test/hwasan/TestCases/tail-magic.c
@@ -10,22 +10,20 @@
 #include <stdio.h>
 #include <sanitizer/hwasan_interface.h>
 
-#include "utils.h"
-
 static volatile char *sink;
 
 // Overwrite the tail in a non-hwasan function so that we don't detect the
 // stores as OOB.
 __attribute__((no_sanitize("hwaddress"))) void overwrite_tail() {
-  (*UNTAG(&sink))[20] = 0x42;
-  (*UNTAG(&sink))[24] = 0x66;
+  sink[20] = 0x42;
+  sink[24] = 0x66;
 }
 
 int main(int argc, char **argv) {
   __hwasan_enable_allocator_tagging();
 
   char *p = (char*)malloc(20);
-  sink = UNTAG(p);
+  sink = p;
   overwrite_tail();
   free(p);
 // CHECK: ERROR: HWAddressSanitizer: allocation-tail-overwritten; heap object [{{.*}}) of size 20
diff --git a/compiler-rt/test/hwasan/TestCases/use-after-free.c b/compiler-rt/test/hwasan/TestCases/use-after-free.c
index 2b2562d601223c7abd6d64ed193d8459e5e09a2f..05ea7f4d7137ae7c52b2a07933b8900e22be2044 100644
--- a/compiler-rt/test/hwasan/TestCases/use-after-free.c
+++ b/compiler-rt/test/hwasan/TestCases/use-after-free.c
@@ -11,19 +11,17 @@
 #include <stdio.h>
 #include <sanitizer/hwasan_interface.h>
 
-#include "utils.h"
-
 int main() {
   __hwasan_enable_allocator_tagging();
   char * volatile x = (char*)malloc(10);
   free(x);
   __hwasan_disable_allocator_tagging();
-  untag_fprintf(stderr, ISREAD ? "Going to do a READ\n" : "Going to do a WRITE\n");
+  fprintf(stderr, ISREAD ? "Going to do a READ\n" : "Going to do a WRITE\n");
   // CHECK: Going to do a [[TYPE:[A-Z]*]]
   int r = 0;
   if (ISREAD) r = x[5]; else x[5] = 42;  // should be on the same line.
   // CHECK: [[TYPE]] of size 1 at {{.*}} tags: [[PTR_TAG:[0-9a-f][0-9a-f]]]/[[MEM_TAG:[0-9a-f][0-9a-f]]] (ptr/mem)
-  // CHECK: #0 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-2]]
+  // CHECK: #{{[0-9]}} {{.*}} in main {{.*}}use-after-free.c:[[@LINE-2]]
   // Offset is 5 or 11 depending on left/right alignment.
   // CHECK: is a small unallocated heap chunk; size: 32 offset: {{5|11}}
   // CHECK: is located 5 bytes inside of 10-byte region
@@ -37,6 +35,6 @@ int main() {
   // CHECK: #1 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-19]]
   // CHECK: Memory tags around the buggy address (one tag corresponds to 16 bytes):
   // CHECK: =>{{.*}}[[MEM_TAG]]
-  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
+  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch
   return r;
 }
diff --git a/compiler-rt/test/hwasan/TestCases/utils.h b/compiler-rt/test/hwasan/TestCases/utils.h
deleted file mode 100644
index 7c9f8852d23c6e4831d32faa0ae873c53d427d00..0000000000000000000000000000000000000000
--- a/compiler-rt/test/hwasan/TestCases/utils.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#pragma once
-
-#include <stdarg.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <string.h>
-
-#define UNTAG(x) (typeof((x) + 0))(((uintptr_t)(x)) & 0xffffffffffffff)
-
-__attribute__((no_sanitize("hwaddress")))
-int untag_printf(const char *fmt, ...) {
-  va_list ap;
-  va_start(ap, fmt);
-  int ret = vprintf(UNTAG(fmt), ap);
-  va_end(ap);
-  return ret;
-}
-
-__attribute__((no_sanitize("hwaddress")))
-int untag_fprintf(FILE *stream, const char *fmt, ...) {
-  va_list ap;
-  va_start(ap, fmt);
-  int ret = vfprintf(stream, UNTAG(fmt), ap);
-  va_end(ap);
-  return ret;
-}
-
-int untag_strcmp(const char *s1, const char *s2) {
-  return strcmp(UNTAG(s1), UNTAG(s2));
-}
diff --git a/compiler-rt/test/lsan/TestCases/Linux/fork_and_leak.cpp b/compiler-rt/test/lsan/TestCases/Linux/fork_and_leak.cpp
index d7427ce3ed04f7d44febece1d53d1ece1b4350d3..758cd819f16e8096e2725c2126d2ce3689fc8073 100644
--- a/compiler-rt/test/lsan/TestCases/Linux/fork_and_leak.cpp
+++ b/compiler-rt/test/lsan/TestCases/Linux/fork_and_leak.cpp
@@ -1,6 +1,9 @@
 // Test that leaks detected after forking without exec().
 // RUN: %clangxx_lsan %s -o %t && not %run %t 2>&1 | FileCheck %s
 
+/// Fails on clang-cmake-aarch64-full (glibc 2.27-3ubuntu1.4).
+// UNSUPPORTED: aarch64
+
 #include <assert.h>
 #include <stdlib.h>
 #include <sys/wait.h>
diff --git a/compiler-rt/test/lsan/TestCases/many_tls_keys_pthread.cpp b/compiler-rt/test/lsan/TestCases/many_tls_keys_pthread.cpp
index ecc577ac4c684b58a7f60e3d9eb86bc88e4db3e4..8c4970c9b13702f0e7f7a8e58e8a96be9ec895d7 100644
--- a/compiler-rt/test/lsan/TestCases/many_tls_keys_pthread.cpp
+++ b/compiler-rt/test/lsan/TestCases/many_tls_keys_pthread.cpp
@@ -5,8 +5,9 @@
 // RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=1" %run %t 2>&1
 // RUN: %env_lsan_opts="" %run %t 2>&1
 
-// Patch r303906 did not fix all the problems.
-// UNSUPPORTED: arm-linux,armhf-linux
+// On glibc, this requires the range returned by GetTLS to include
+// specific_1stblock and specific in `struct pthread`.
+// UNSUPPORTED: arm-linux, armhf-linux
 
 // TSD on NetBSD does not use TLS
 // UNSUPPORTED: netbsd
diff --git a/compiler-rt/test/lsan/TestCases/swapcontext.cpp b/compiler-rt/test/lsan/TestCases/swapcontext.cpp
index d0999598ad4c2207c95f712994b4cf6284a7da0e..f78867cc06959f69cbe742a4a1b7d1dcdb2b8e6d 100644
--- a/compiler-rt/test/lsan/TestCases/swapcontext.cpp
+++ b/compiler-rt/test/lsan/TestCases/swapcontext.cpp
@@ -5,7 +5,7 @@
 // RUN: %env_lsan_opts= %run %t 2>&1
 // RUN: %env_lsan_opts= not %run %t foo 2>&1 | FileCheck %s
 // Missing 'getcontext' and 'makecontext' on Android.
-// UNSUPPORTED: arm,powerpc64,android
+// UNSUPPORTED: arm,aarch64,powerpc64,android
 
 #include "sanitizer_common/sanitizer_ucontext.h"
 #include <stdio.h>
diff --git a/compiler-rt/test/lsan/lit.common.cfg.py b/compiler-rt/test/lsan/lit.common.cfg.py
index 6e012c06ab11d09ddb245287f95d5ff69a13d14f..88c557549b38b6238c612d7dd015ca2ba2fea20d 100644
--- a/compiler-rt/test/lsan/lit.common.cfg.py
+++ b/compiler-rt/test/lsan/lit.common.cfg.py
@@ -76,7 +76,7 @@ config.substitutions.append( ("%clangxx_lsan ", build_invocation(clang_lsan_cxxf
 # LeakSanitizer tests are currently supported on
 # Android{aarch64, x86, x86_64}, x86-64 Linux, PowerPC64 Linux, arm Linux, mips64 Linux, s390x Linux and x86_64 Darwin.
 supported_android = config.android and config.target_arch in ['x86_64', 'i386', 'aarch64'] and 'android-thread-properties-api' in config.available_features
-supported_linux = (not config.android) and config.host_os == 'Linux' and config.host_arch in ['x86_64', 'ppc64', 'ppc64le', 'mips64', 'riscv64', 'arm', 'armhf', 'armv7l', 's390x']
+supported_linux = (not config.android) and config.host_os == 'Linux' and config.host_arch in ['aarch64', 'x86_64', 'ppc64', 'ppc64le', 'mips64', 'riscv64', 'arm', 'armhf', 'armv7l', 's390x']
 supported_darwin = config.host_os == 'Darwin' and config.target_arch in ['x86_64']
 supported_netbsd = config.host_os == 'NetBSD' and config.target_arch in ['x86_64', 'i386']
 if not (supported_android or supported_linux or supported_darwin or supported_netbsd):
diff --git a/compiler-rt/test/memprof/TestCases/test_terse.cpp b/compiler-rt/test/memprof/TestCases/test_terse.cpp
index 4ac0d5e77278785b8b096cedc7f866a99e902af2..04763008016e584b0fe4c922b62a754e51b6e915 100644
--- a/compiler-rt/test/memprof/TestCases/test_terse.cpp
+++ b/compiler-rt/test/memprof/TestCases/test_terse.cpp
@@ -8,7 +8,7 @@
 // RUN: %clangxx_memprof -DFREE -O0 %s -o %t
 // RUN: %env_memprof_opts=log_path=stderr:print_terse=1 %run %t 2>&1 | FileCheck %s
 
-// CHECK: MIB:[[STACKID:[0-9]+]]/1/40.00/40/40/20.00/20/20/[[AVELIFETIME:[0-9]+]].00/[[AVELIFETIME]]/[[AVELIFETIME]]/0/0/0/0
+// CHECK: MIB:[[STACKID:[0-9]+]]/1/40.00/40/40/20.00/20/20/[[AVELIFETIME:[0-9]+]].00/[[AVELIFETIME]]/[[AVELIFETIME]]/{{[01]}}/0/0/0
 // CHECK: Stack for id [[STACKID]]:
 // CHECK-NEXT: #0 {{.*}} in operator new
 // CHECK-NEXT: #1 {{.*}} in main {{.*}}:[[@LINE+6]]
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/regex_startend.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/regex_startend.cpp
index 1a445783bcbece12df4fc573a47b8359448b955e..e143c6c9b5e2045fc70b90af2263d76a7b46c8d6 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/regex_startend.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/regex_startend.cpp
@@ -14,6 +14,8 @@
 #include <stdlib.h>
 #include <string.h>
 
+/// REG_STARTEND is a BSD extension not supported everywhere.
+#ifdef REG_STARTEND
 void test_matched(const regex_t *preg, const char *string, size_t start,
                   size_t end, const char *expected) {
   regmatch_t match[1];
@@ -59,3 +61,8 @@ int main(void) {
   printf("Successful test\n");
   return 0;
 }
+#else
+int main(void) {
+  return 0;
+}
+#endif
diff --git a/compiler-rt/test/tsan/on_initialize_finalize_hooks.cpp b/compiler-rt/test/tsan/on_initialize_finalize_hooks.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0b82ec1761794cdd4c7ced21699cc9643f8aad09
--- /dev/null
+++ b/compiler-rt/test/tsan/on_initialize_finalize_hooks.cpp
@@ -0,0 +1,33 @@
+// RUN: %clang_tsan -O1 %s -o %t.lib -fno-sanitize=thread -shared -fPIC -DBUILD_LIB=1
+// RUN: %clang_tsan -O1 %s %t.lib -o %t
+// RUN: %run %t | FileCheck %s
+
+// Test that initialization/finalization hooks are called, even when they are
+// not defined in the main executable, but by another another library that
+// doesn't directly link against the TSan runtime.
+
+#include <stdio.h>
+
+#if BUILD_LIB
+
+extern "C" void __tsan_on_initialize() {
+  printf("__tsan_on_initialize()\n");
+}
+
+extern "C" int __tsan_on_finalize(int failed) {
+  printf("__tsan_on_finalize()\n");
+  return failed;
+}
+
+#else // BUILD_LIB
+
+int main() {
+  printf("main()\n");
+  return 0;
+}
+
+#endif // BUILD_LIB
+
+// CHECK: __tsan_on_initialize()
+// CHECK: main()
+// CHECK: __tsan_on_finalize()
diff --git a/debuginfo-tests/dexter/dex/debugger/DebuggerBase.py b/debuginfo-tests/dexter/dex/debugger/DebuggerBase.py
index 37aaffe4889899f6f06a697fb4bb676c61dbc313..5b97974674a5c1446bbcda91616ad054e0909817 100644
--- a/debuginfo-tests/dexter/dex/debugger/DebuggerBase.py
+++ b/debuginfo-tests/dexter/dex/debugger/DebuggerBase.py
@@ -125,26 +125,46 @@ class DebuggerBase(object, metaclass=abc.ABCMeta):
         pass
 
     def add_breakpoint(self, file_, line):
+        """Returns a unique opaque breakpoint id.
+
+        The ID type depends on the debugger being used, but will probably be
+        an int.
+        """
         return self._add_breakpoint(self._external_to_debug_path(file_), line)
 
     @abc.abstractmethod
     def _add_breakpoint(self, file_, line):
+        """Returns a unique opaque breakpoint id.
+        """
         pass
 
     def add_conditional_breakpoint(self, file_, line, condition):
+        """Returns a unique opaque breakpoint id.
+
+        The ID type depends on the debugger being used, but will probably be
+        an int.
+        """
         return self._add_conditional_breakpoint(
             self._external_to_debug_path(file_), line, condition)
 
     @abc.abstractmethod
     def _add_conditional_breakpoint(self, file_, line, condition):
+        """Returns a unique opaque breakpoint id.
+        """
         pass
 
-    def delete_conditional_breakpoint(self, file_, line, condition):
-        return self._delete_conditional_breakpoint(
-            self._external_to_debug_path(file_), line, condition)
+    @abc.abstractmethod
+    def delete_breakpoint(self, id):
+        """Delete a breakpoint by id.
+
+        Raises a KeyError if no breakpoint with this id exists.
+        """
+        pass
 
     @abc.abstractmethod
-    def _delete_conditional_breakpoint(self, file_, line, condition):
+    def get_triggered_breakpoint_ids(self):
+        """Returns a set of opaque ids for just-triggered breakpoints.
+        """
         pass
 
     @abc.abstractmethod
diff --git a/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ConditionalController.py b/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ConditionalController.py
index 4e4327b53f826b7fe8f1b061002734c603c3b781..e225a48bcb66a6acfe8df1a5f058c0842d9903c6 100644
--- a/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ConditionalController.py
+++ b/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ConditionalController.py
@@ -42,17 +42,18 @@ class ConditionalController(DebuggerControllerBase):
     def __init__(self, context, step_collection):
       self.context = context
       self.step_collection = step_collection
-      self._conditional_bps = None
+      self._conditional_bp_ranges = None
+      self._build_conditional_bp_ranges()
       self._watches = set()
       self._step_index = 0
-      self._build_conditional_bps()
-      self._path_and_line_to_conditional_bp = defaultdict(list)
       self._pause_between_steps = context.options.pause_between_steps
       self._max_steps = context.options.max_steps
+      # Map {id: ConditionalBpRange}
+      self._conditional_bp_handles = {}
 
-    def _build_conditional_bps(self):
+    def _build_conditional_bp_ranges(self):
         commands = self.step_collection.commands
-        self._conditional_bps = []
+        self._conditional_bp_ranges = []
         try:
             limit_commands = commands['DexLimitSteps']
             for lc in limit_commands:
@@ -62,22 +63,19 @@ class ConditionalController(DebuggerControllerBase):
                   lc.from_line,
                   lc.to_line,
                   lc.values)
-                self._conditional_bps.append(conditional_bp)
+                self._conditional_bp_ranges.append(conditional_bp)
         except KeyError:
             raise DebuggerException('Missing DexLimitSteps commands, cannot conditionally step.')
 
     def _set_conditional_bps(self):
-        # When we break in the debugger we need a quick and easy way to look up
-        # which conditional bp we've breaked on.
-        for cbp in self._conditional_bps:
-            conditional_bp_list = self._path_and_line_to_conditional_bp[(cbp.path, cbp.range_from)]
-            conditional_bp_list.append(cbp)
-
-        # Set break points only on the first line of any conditional range, we'll set
-        # more break points for a range when the condition is satisfied.
-        for cbp in self._conditional_bps:
+        # Set a conditional breakpoint for each ConditionalBpRange and build a
+        # map of {id: ConditionalBpRange}.
+        for cbp in self._conditional_bp_ranges:
             for cond_expr in cbp.get_conditional_expression_list():
-                self.debugger.add_conditional_breakpoint(cbp.path, cbp.range_from, cond_expr)
+                id = self.debugger.add_conditional_breakpoint(cbp.path,
+                                                              cbp.range_from,
+                                                              cond_expr)
+                self._conditional_bp_handles[id] = cbp
 
     def _conditional_met(self, cbp):
         for cond_expr in cbp.get_conditional_expression_list():
@@ -98,7 +96,7 @@ class ConditionalController(DebuggerControllerBase):
             self._watches.update(command_obj.get_watches())
 
         self.debugger.launch()
-        time.sleep(self._pause_between_steps) 
+        time.sleep(self._pause_between_steps)
         while not self.debugger.is_finished:
             while self.debugger.is_running:
                 pass
@@ -109,19 +107,28 @@ class ConditionalController(DebuggerControllerBase):
                 update_step_watches(step_info, self._watches, self.step_collection.commands)
                 self.step_collection.new_step(self.context, step_info)
 
-                loc = step_info.current_location
-                conditional_bp_key = (loc.path, loc.lineno)
-                if conditional_bp_key in self._path_and_line_to_conditional_bp:
+            bp_to_delete = []
+            for bp_id in self.debugger.get_triggered_breakpoint_ids():
+                try:
+                    # See if this is one of our conditional breakpoints.
+                    cbp = self._conditional_bp_handles[bp_id]
+                except KeyError:
+                    # This is an unconditional bp. Mark it for removal.
+                    bp_to_delete.append(bp_id)
+                    continue
+                # We have triggered a breakpoint with a condition. Check that
+                # the condition has been met.
+                if self._conditional_met(cbp):
+                    # Add a range of unconditional breakpoints covering the
+                    # lines requested in the DexLimitSteps command. Ignore
+                    # first line as that's the conditional bp we just hit and
+                    # include the final line.
+                    for line in range(cbp.range_from + 1, cbp.range_to + 1):
+                        self.debugger.add_breakpoint(cbp.path, line)
+
+            # Remove any unconditional breakpoints we just hit.
+            for bp_id in bp_to_delete:
+                self.debugger.delete_breakpoint(bp_id)
 
-                    conditional_bps = self._path_and_line_to_conditional_bp[conditional_bp_key]
-                    for cbp in conditional_bps:
-                        if self._conditional_met(cbp):
-                            # Unconditional range should ignore first line as that's the
-                            # conditional bp we just hit and should be inclusive of final line
-                            for line in range(cbp.range_from + 1, cbp.range_to + 1):
-                                self.debugger.add_conditional_breakpoint(cbp.path, line, condition='')
-
-            # Clear any uncondtional break points at this loc.
-            self.debugger.delete_conditional_breakpoint(file_=loc.path, line=loc.lineno, condition='')
             self.debugger.go()
             time.sleep(self._pause_between_steps)
diff --git a/debuginfo-tests/dexter/dex/debugger/dbgeng/dbgeng.py b/debuginfo-tests/dexter/dex/debugger/dbgeng/dbgeng.py
index 5105b4afa706cc80f075ee42e84f1768dcc722ba..c95aa54f7e6b38ee4a303e3fd7074f641348980a 100644
--- a/debuginfo-tests/dexter/dex/debugger/dbgeng/dbgeng.py
+++ b/debuginfo-tests/dexter/dex/debugger/dbgeng/dbgeng.py
@@ -87,7 +87,10 @@ class DbgEng(DebuggerBase):
         # but is something that should be considered in the future.
         raise NotImplementedError('add_conditional_breakpoint is not yet implemented by dbgeng')
 
-    def _delete_conditional_breakpoint(self, file_, line, condition):
+    def get_triggered_breakpoint_ids(self):
+      raise NotImplementedError('get_triggered_breakpoint_ids is not yet implemented by dbgeng')
+
+    def delete_breakpoint(self, id):
         # breakpoint setting/deleting is not supported by dbgeng at this moment
         # but is something that should be considered in the future.
         raise NotImplementedError('delete_conditional_breakpoint is not yet implemented by dbgeng')
diff --git a/debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py b/debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py
index 5fc8fd3e95f861f1a64ca14897fd46753d388521..324467dd0819af7e64d78f395561f5a89c3b9a9c 100644
--- a/debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py
+++ b/debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py
@@ -104,9 +104,11 @@ class LLDB(DebuggerBase):
         self._target.DeleteAllBreakpoints()
 
     def _add_breakpoint(self, file_, line):
-        if not self._target.BreakpointCreateByLocation(file_, line):
+        bp = self._target.BreakpointCreateByLocation(file_, line)
+        if not bp:
             raise DebuggerException(
                 'could not add breakpoint [{}:{}]'.format(file_, line))
+        return bp.GetID()
 
     def _add_conditional_breakpoint(self, file_, line, condition):
         bp = self._target.BreakpointCreateByLocation(file_, line)
@@ -115,37 +117,24 @@ class LLDB(DebuggerBase):
         else:
             raise DebuggerException(
                   'could not add breakpoint [{}:{}]'.format(file_, line))
-
-    def _delete_conditional_breakpoint(self, file_, line, condition):
-        bp_count = self._target.GetNumBreakpoints()
-        bps = [self._target.GetBreakpointAtIndex(ix) for ix in range(0, bp_count)]
-
-        for bp in bps:
-            bp_cond = bp.GetCondition()
-            bp_cond = bp_cond if bp_cond is not None else ''
-
-            if bp_cond != condition:
-                continue
-
-            # If one of the bound bp locations for this bp is bound to the same
-            # line in file_ above, then delete the entire parent bp and all
-            # bp locs.
-            # https://lldb.llvm.org/python_reference/lldb.SBBreakpoint-class.html
-            for breakpoint_location in bp:
-                sb_address = breakpoint_location.GetAddress()
-
-                sb_line_entry = sb_address.GetLineEntry()
-                bl_line = sb_line_entry.GetLine()
-
-                sb_file_entry = sb_line_entry.GetFileSpec()
-                bl_dir = sb_file_entry.GetDirectory()
-                bl_file_name = sb_file_entry.GetFilename()
-
-                bl_file_path = os.path.join(bl_dir, bl_file_name)
-
-                if bl_file_path == file_ and bl_line == line:
-                    self._target.BreakpointDelete(bp.GetID())
-                    break
+        return bp.GetID()
+
+    def get_triggered_breakpoint_ids(self):
+        # Breakpoints can only have been triggered if we've hit one.
+        stop_reason = self._translate_stop_reason(self._thread.GetStopReason())
+        if stop_reason != StopReason.BREAKPOINT:
+            return []
+        # Breakpoints have two data parts: Breakpoint ID, Location ID. We're
+        # only interested in the Breakpoint ID so we skip every other item.
+        return set([self._thread.GetStopReasonDataAtIndex(i)
+                    for i in range(0, self._thread.GetStopReasonDataCount(), 2)])
+
+    def delete_breakpoint(self, id):
+        bp = self._target.FindBreakpointByID(id)
+        if not bp:
+            # The ID is not valid.
+            raise KeyError
+        self._target.BreakpointDelete(bp.GetID())
 
     def launch(self):
         self._process = self._target.LaunchSimple(None, None, os.getcwd())
diff --git a/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py b/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
index 6585a4938c12ce8849cb24d78cd3195488e6a0e2..b4558e2d8a50bcd45fe54adaaf10d05c36ff0155 100644
--- a/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
+++ b/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
@@ -10,6 +10,9 @@ import abc
 import imp
 import os
 import sys
+from pathlib import PurePath
+from collections import namedtuple
+from collections import defaultdict
 
 from dex.debugger.DebuggerBase import DebuggerBase
 from dex.dextIR import FrameIR, LocIR, StepIR, StopReason, ValueIR
@@ -28,6 +31,11 @@ def _load_com_module():
         raise LoadDebuggerException(e, sys.exc_info())
 
 
+# VSBreakpoint(path: PurePath, line: int, col: int, cond: str).  This is enough
+# info to identify breakpoint equivalence in visual studio based on the
+# properties we set through dexter currently.
+VSBreakpoint = namedtuple('VSBreakpoint', 'path, line, col, cond')
+
 class VisualStudio(DebuggerBase, metaclass=abc.ABCMeta):  # pylint: disable=abstract-method
 
     # Constants for results of Debugger.CurrentMode
@@ -42,6 +50,21 @@ class VisualStudio(DebuggerBase, metaclass=abc.ABCMeta):  # pylint: disable=abst
         self._solution = None
         self._fn_step = None
         self._fn_go = None
+        # The next available unique breakpoint id. Use self._get_next_id().
+        self._next_bp_id = 0
+        # VisualStudio appears to common identical breakpoints. That is, if you
+        # ask for a breakpoint that already exists the Breakpoints list will
+        # not grow. DebuggerBase requires all breakpoints have a unique id,
+        # even for duplicates, so we'll need to do some bookkeeping.  Map
+        # {VSBreakpoint: list(id)} where id is the unique dexter-side id for
+        # the requested breakpoint.
+        self._vs_to_dex_ids = defaultdict(list)
+        # Map {id: VSBreakpoint} where id is unique and VSBreakpoint identifies
+        # a breakpoint in Visual Studio. There may be many ids mapped to a
+        # single VSBreakpoint. Use self._vs_to_dex_ids to find (dexter)
+        # breakpoints mapped to the same visual studio breakpoint.
+        self._dex_id_to_vs = {}
+
         super(VisualStudio, self).__init__(*args)
 
     def _custom_init(self):
@@ -110,21 +133,88 @@ class VisualStudio(DebuggerBase, metaclass=abc.ABCMeta):  # pylint: disable=abst
     def clear_breakpoints(self):
         for bp in self._debugger.Breakpoints:
             bp.Delete()
+        self._vs_to_dex_ids.clear()
+        self._dex_id_to_vs.clear()
 
     def _add_breakpoint(self, file_, line):
-        self._debugger.Breakpoints.Add('', file_, line)
+        return self._add_conditional_breakpoint(file_, line, '')
 
-    def _add_conditional_breakpoint(self, file_, line, condition):
-        column = 1
-        self._debugger.Breakpoints.Add('', file_, line, column, condition)
+    def _get_next_id(self):
+        # "Generate" a new unique id for the breakpoint.
+        id = self._next_bp_id
+        self._next_bp_id += 1
+        return id
 
-    def _delete_conditional_breakpoint(self, file_, line, condition):
+    def _add_conditional_breakpoint(self, file_, line, condition):
+        col = 1
+        vsbp = VSBreakpoint(PurePath(file_), line, col, condition)
+        new_id = self._get_next_id()
+
+        # Do we have an exact matching breakpoint already?
+        if vsbp in self._vs_to_dex_ids:
+            self._vs_to_dex_ids[vsbp].append(new_id)
+            self._dex_id_to_vs[new_id] = vsbp
+            return new_id
+
+        # Breakpoint doesn't exist already. Add it now.
+        count_before = self._debugger.Breakpoints.Count
+        self._debugger.Breakpoints.Add('', file_, line, col, condition)
+        # Our internal representation of VS says that the breakpoint doesn't
+        # already exist so we do not expect this operation to fail here.
+        assert count_before < self._debugger.Breakpoints.Count
+        # We've added a new breakpoint, record its id.
+        self._vs_to_dex_ids[vsbp].append(new_id)
+        self._dex_id_to_vs[new_id] = vsbp
+        return new_id
+
+    def get_triggered_breakpoint_ids(self):
+        """Returns a set of opaque ids for just-triggered breakpoints.
+        """
+        bps_hit = self._debugger.AllBreakpointsLastHit
+        bp_id_list = []
+        # Intuitively, AllBreakpointsLastHit breakpoints are the last hit
+        # _bound_ breakpoints. A bound breakpoint's parent holds the info of
+        # the breakpoint the user requested. Our internal state tracks the user
+        # requested breakpoints so we look at the Parent of these triggered
+        # breakpoints to determine which have been hit.
+        for bp in bps_hit:
+            # All bound breakpoints should have the user-defined breakpoint as
+            # a parent.
+            assert bp.Parent
+            vsbp = VSBreakpoint(PurePath(bp.Parent.File), bp.Parent.FileLine,
+                                bp.Parent.FileColumn, bp.Parent.Condition)
+            try:
+                ids = self._vs_to_dex_ids[vsbp]
+            except KeyError:
+                pass
+            else:
+                bp_id_list += ids
+        return set(bp_id_list)
+
+    def delete_breakpoint(self, id):
+        """Delete a breakpoint by id.
+
+        Raises a KeyError if no breakpoint with this id exists.
+        """
+        vsbp = self._dex_id_to_vs[id]
+
+        # Remove our id from the associated list of dex ids.
+        self._vs_to_dex_ids[vsbp].remove(id)
+        del self._dex_id_to_vs[id]
+
+        # Bail if there are other uses of this vsbp.
+        if len(self._vs_to_dex_ids[vsbp]) > 0:
+            return
+        # Otherwise find and delete it.
         for bp in self._debugger.Breakpoints:
-            for bound_bp in bp.Children:
-                if (bound_bp.File == file_ and bound_bp.FileLine == line and
-                    bound_bp.Condition == condition):
-                    bp.Delete()
-                    break
+            # We're looking at the user-set breakpoints so there shouild be no
+            # Parent.
+            assert bp.Parent == None
+            this_vsbp = VSBreakpoint(PurePath(bp.File), bp.FileLine,
+                                     bp.FileColumn, bp.Condition)
+            if vsbp == this_vsbp:
+                bp.Delete()
+                break
 
     def launch(self):
         self._fn_go()
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/limit_steps/limit_steps_line_mismatch.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/limit_steps/limit_steps_line_mismatch.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..68ae4766653ec15bc38605c10fb30e1dc3c7f661
--- /dev/null
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/limit_steps/limit_steps_line_mismatch.cpp
@@ -0,0 +1,25 @@
+// Purpose:
+//      Check that \DexLimitSteps works even if the opening breakpoint line
+//      doesn't exist. This can happen due to optimisations or label is on an
+//      empty line.
+//
+// FIXME: Windows regression tests run with dbgeng. \DexLimitSteps isn't yet
+// supported with dbgeng.
+//
+// REQUIRES: system-linux
+//
+// RUN: %dexter_regression_test -- %s | FileCheck %s
+// CHECK: limit_steps_line_mismatch.cpp
+
+int main() {
+  int i = 0;
+  for (; i < 2; i++) {
+    // DexLabel('from')
+    int x = i;
+  }
+  int ret = 0;
+  return ret; // DexLabel('to')
+}
+
+// DexLimitSteps('1', '1', from_line='from', to_line='to')
+// DexExpectWatchValue('i', 0, 1, 2, from_line='from', to_line='to')
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 33e5521f6d5b118b2f59b65ac963ea9e2934faf6..812a794fe3e748b20373a2eed7456201c2140eb4 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -48,9 +48,23 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   list(APPEND CMAKE_MODULE_PATH ${LLVM_DIR_ABSOLUTE})
 
   if(FLANG_BUILD_NEW_DRIVER)
+    # Users might specify a path to CLANG_DIR that's:
+    #   * a full path, or
+    #   * a path relative to the path of this script.
+    # Append the absolute path to CLANG_DIR so that find_package works in both
+    # cases.
+    get_filename_component(
+      CLANG_DIR_ABSOLUTE
+      ${CLANG_DIR}
+      REALPATH
+      ${CMAKE_CURRENT_SOURCE_DIR})
+    list(APPEND CMAKE_MODULE_PATH ${CLANG_DIR_ABSOLUTE})
+
     # TODO: Remove when libclangDriver is lifted out of Clang
-    list(APPEND CMAKE_MODULE_PATH ${CLANG_DIR})
-    find_package(Clang REQUIRED HINTS "${CLANG_DIR}")
+    find_package(Clang REQUIRED PATHS "${CLANG_DIR_ABSOLUTE}" NO_DEFAULT_PATH)
+    if (NOT Clang_FOUND)
+      message(FATAL_ERROR "Failed to find Clang")
+    endif()
   endif()
 
   # If LLVM links to zlib we need the imported targets so we can too.
diff --git a/flang/include/flang/Evaluate/constant.h b/flang/include/flang/Evaluate/constant.h
index 89a5867722f72f003fbbc6705b14d1871920a430..d5f7db8be45d13a5be3a0e3b8893d045f7c45bb4 100644
--- a/flang/include/flang/Evaluate/constant.h
+++ b/flang/include/flang/Evaluate/constant.h
@@ -195,8 +195,11 @@ private:
 };
 
 class StructureConstructor;
-using StructureConstructorValues =
-    std::map<SymbolRef, common::CopyableIndirection<Expr<SomeType>>>;
+struct ComponentCompare {
+  bool operator()(SymbolRef x, SymbolRef y) const;
+};
+using StructureConstructorValues = std::map<SymbolRef,
+    common::CopyableIndirection<Expr<SomeType>>, ComponentCompare>;
 
 template <>
 class Constant<SomeDerived>
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index afa70fd0099a218d1489999cdd590e4ffb3faf4d..4a0a4dcf4041e34cbaf35f4aa1594dec9c874265 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -839,10 +839,12 @@ template <typename A> SymbolVector GetSymbolVector(const A &x) {
 const Symbol *GetLastTarget(const SymbolVector &);
 
 // Collects all of the Symbols in an expression
-template <typename A> semantics::SymbolSet CollectSymbols(const A &);
-extern template semantics::SymbolSet CollectSymbols(const Expr<SomeType> &);
-extern template semantics::SymbolSet CollectSymbols(const Expr<SomeInteger> &);
-extern template semantics::SymbolSet CollectSymbols(
+template <typename A> semantics::UnorderedSymbolSet CollectSymbols(const A &);
+extern template semantics::UnorderedSymbolSet CollectSymbols(
+    const Expr<SomeType> &);
+extern template semantics::UnorderedSymbolSet CollectSymbols(
+    const Expr<SomeInteger> &);
+extern template semantics::UnorderedSymbolSet CollectSymbols(
     const Expr<SubscriptInteger> &);
 
 // Predicate: does a variable contain a vector-valued subscript (not a triplet)?
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index 3be6f3fc4d402bfce10e8c083752e461d63d69ec..99050dcdbd7bcdb44dc1192170789ad63ab0e61c 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -74,6 +74,8 @@ class CompilerInvocation : public CompilerInvocationBase {
   // Fortran Dialect options
   Fortran::common::IntrinsicTypeDefaultKinds defaultKinds_;
 
+  bool EnableConformanceChecks_ = false;
+
 public:
   CompilerInvocation() = default;
 
@@ -96,6 +98,11 @@ public:
   bool &debugModuleDir() { return debugModuleDir_; }
   const bool &debugModuleDir() const { return debugModuleDir_; }
 
+  bool &enableConformanceChecks() { return EnableConformanceChecks_; }
+  const bool &enableConformanceChecks() const {
+    return EnableConformanceChecks_;
+  }
+
   Fortran::common::IntrinsicTypeDefaultKinds &defaultKinds() {
     return defaultKinds_;
   }
@@ -111,6 +118,9 @@ public:
       llvm::ArrayRef<const char *> commandLineArgs,
       clang::DiagnosticsEngine &diags);
 
+  // Enables the std=f2018 conformance check
+  void set_EnableConformanceChecks() { EnableConformanceChecks_ = true; }
+
   /// Useful setters
   void SetModuleDir(std::string &moduleDir) { moduleDir_ = moduleDir; }
 
diff --git a/flang/include/flang/Frontend/FrontendActions.h b/flang/include/flang/Frontend/FrontendActions.h
index 35d1e6f29b0fba69d707ce86c97524c70d4e8f86..f49f9f4714b53b45733300a07c32f0c84cc1cfec 100644
--- a/flang/include/flang/Frontend/FrontendActions.h
+++ b/flang/include/flang/Frontend/FrontendActions.h
@@ -100,6 +100,10 @@ class DebugPreFIRTreeAction : public PrescanAndSemaAction {
   void ExecuteAction() override;
 };
 
+class GetSymbolsSourcesAction : public PrescanAndSemaAction {
+  void ExecuteAction() override;
+};
+
 class ParseSyntaxOnlyAction : public PrescanAndSemaAction {
   void ExecuteAction() override;
 };
diff --git a/flang/include/flang/Frontend/FrontendOptions.h b/flang/include/flang/Frontend/FrontendOptions.h
index 48182f488466167f2e0becd87ae5e540f5db8b64..1d9002335c3ceffff5e2c3bfd52ba4c712f65b27 100644
--- a/flang/include/flang/Frontend/FrontendOptions.h
+++ b/flang/include/flang/Frontend/FrontendOptions.h
@@ -58,7 +58,10 @@ enum ActionKind {
   DebugMeasureParseTree,
 
   /// Parse, run semantics and then output the pre-FIR tree
-  DebugPreFIRTree
+  DebugPreFIRTree,
+
+  /// Parse, run semantics and then dump symbol sources map
+  GetSymbolsSources
 
   /// TODO: RunPreprocessor, EmitLLVM, EmitLLVMOnly,
   /// EmitCodeGenOnly, EmitAssembly, (...)
diff --git a/flang/include/flang/Frontend/PreprocessorOptions.h b/flang/include/flang/Frontend/PreprocessorOptions.h
index 39ea4d3d3c6cf0cad56373b02fbdc5a73c8dba51..3a3877bf0b28691e2848e046e8d126c34914ea3c 100644
--- a/flang/include/flang/Frontend/PreprocessorOptions.h
+++ b/flang/include/flang/Frontend/PreprocessorOptions.h
@@ -29,6 +29,8 @@ public:
   // consider collecting them in a separate aggregate. For now we keep it here
   // as there is no point creating a class for just one field.
   std::vector<std::string> searchDirectoriesFromDashI;
+  // Search directories specified by the user with -fintrinsic-modules-path
+  std::vector<std::string> searchDirectoriesFromIntrModPath;
 
 public:
   PreprocessorOptions() {}
@@ -44,4 +46,4 @@ public:
 
 } // namespace Fortran::frontend
 
-#endif // LLVM_FLANG_PREPROCESSOROPTIONS_H
\ No newline at end of file
+#endif // LLVM_FLANG_PREPROCESSOROPTIONS_H
diff --git a/flang/include/flang/Optimizer/CodeGen/CGOps.td b/flang/include/flang/Optimizer/CodeGen/CGOps.td
new file mode 100644
index 0000000000000000000000000000000000000000..9ebda32825a635aa71aca82bf9bdd931c8e9b2d9
--- /dev/null
+++ b/flang/include/flang/Optimizer/CodeGen/CGOps.td
@@ -0,0 +1,177 @@
+//===-- CGOps.td - FIR operation definitions ---------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Definition of the FIRCG dialect operations
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_DIALECT_FIRCG_OPS
+#define FORTRAN_DIALECT_FIRCG_OPS
+
+include "mlir/IR/SymbolInterfaces.td"
+include "flang/Optimizer/Dialect/FIRTypes.td"
+
+def fircg_Dialect : Dialect {
+  let name = "fircg";
+  let cppNamespace = "::fir::cg";
+}
+
+// Base class for FIR CG operations.
+// All operations automatically get a prefix of "fircg.".
+class fircg_Op<string mnemonic, list<OpTrait> traits>
+  : Op<fircg_Dialect, mnemonic, traits>;
+
+// Extended embox operation.
+def fircg_XEmboxOp : fircg_Op<"ext_embox", [AttrSizedOperandSegments]> {
+  let summary = "for internal conversion only";
+
+  let description = [{
+    Prior to lowering to LLVM IR dialect, a non-scalar non-trivial embox op will
+    be converted to an extended embox. This op will have the following sets of
+    arguments.
+
+       - memref: The memory reference being emboxed.
+       - shape: A vector that is the runtime shape of the underlying array.
+       - shift: A vector that is the runtime origin of the first element.
+         The default is a vector of the value 1.
+       - slice: A vector of triples that describe an array slice.
+       - subcomponent: A vector of indices for subobject slicing.
+       - LEN type parameters: A vector of runtime LEN type parameters that
+         describe an correspond to the elemental derived type.
+
+    The memref and shape arguments are mandatory. The rest are optional.
+  }];
+
+  let arguments = (ins
+    AnyReferenceLike:$memref,
+    Variadic<AnyIntegerType>:$shape,
+    Variadic<AnyIntegerType>:$shift,
+    Variadic<AnyIntegerType>:$slice,
+    Variadic<AnyCoordinateType>:$subcomponent,
+    Variadic<AnyIntegerType>:$lenParams
+  );
+  let results = (outs fir_BoxType);
+
+  let assemblyFormat = [{
+    $memref (`(`$shape^`)`)? (`origin` $shift^)? (`[`$slice^`]`)?
+      (`path` $subcomponent^)? (`typeparams` $lenParams^)? attr-dict
+      `:` functional-type(operands, results)
+  }];
+
+  let extraClassDeclaration = [{
+    // The rank of the entity being emboxed
+    unsigned getRank() { return shape().size(); }
+
+    // The rank of the result. A slice op can reduce the rank.
+    unsigned getOutRank();
+
+    // The shape operands are mandatory and always start at 1.
+    unsigned shapeOffset() { return 1; }
+    unsigned shiftOffset() { return shapeOffset() + shape().size(); }
+    unsigned sliceOffset() { return shiftOffset() + shift().size(); }
+    unsigned subcomponentOffset() { return sliceOffset() + slice().size(); }
+    unsigned lenParamOffset() {
+      return subcomponentOffset() + subcomponent().size();
+    }
+  }];
+}
+
+// Extended rebox operation.
+def fircg_XReboxOp : fircg_Op<"ext_rebox", [AttrSizedOperandSegments]> {
+  let summary = "for internal conversion only";
+
+  let description = [{
+    Prior to lowering to LLVM IR dialect, a non-scalar non-trivial rebox op will
+    be converted to an extended rebox. This op will have the following sets of
+    arguments.
+
+       - box: The box being reboxed.
+       - shape: A vector that is the new runtime shape for the array
+       - shift: A vector that is the new runtime origin of the first element.
+         The default is a vector of the value 1.
+       - slice: A vector of triples that describe an array slice.
+       - subcomponent: A vector of indices for subobject slicing.
+
+    The box argument is mandatory, the other arguments are optional.
+    There must not both be a shape and slice/subcomponent arguments
+  }];
+
+  let arguments = (ins
+    fir_BoxType:$box,
+    Variadic<AnyIntegerType>:$shape,
+    Variadic<AnyIntegerType>:$shift,
+    Variadic<AnyIntegerType>:$slice,
+    Variadic<AnyCoordinateType>:$subcomponent
+  );
+  let results = (outs fir_BoxType);
+
+  let assemblyFormat = [{
+    $box (`(`$shape^`)`)? (`origin` $shift^)? (`[`$slice^`]`)?
+      (`path` $subcomponent^) ? attr-dict
+      `:` functional-type(operands, results)
+  }];
+
+  let extraClassDeclaration = [{
+    // The rank of the entity being reboxed
+    unsigned getRank();
+    // The rank of the result box
+    unsigned getOutRank();
+  }];
+}
+
+
+// Extended array coordinate operation.
+def fircg_XArrayCoorOp : fircg_Op<"ext_array_coor", [AttrSizedOperandSegments]> {
+  let summary = "for internal conversion only";
+
+  let description = [{
+    Prior to lowering to LLVM IR dialect, a non-scalar non-trivial embox op will
+    be converted to an extended embox. This op will have the following sets of
+    arguments.
+
+       - memref: The memory reference of the array's data. It can be a fir.box if
+         the underlying data is not contiguous.
+       - shape: A vector that is the runtime shape of the underlying array.
+       - shift: A vector that is the runtime origin of the first element.
+         The default is a vector of the value 1.
+       - slice: A vector of triples that describe an array slice.
+       - subcomponent: A vector of indices that describe subobject slicing.
+       - indices: A vector of runtime values that describe the coordinate of
+         the element of the array to be computed.
+       - LEN type parameters: A vector of runtime LEN type parameters that
+         describe an correspond to the elemental derived type.
+
+    The memref and indices arguments are mandatory.
+    The shape argument is mandatory if the memref is not a box, and should be
+    omitted otherwise. The rest of the arguments are optional.
+  }];
+
+  let arguments = (ins
+    AnyRefOrBox:$memref,
+    Variadic<AnyIntegerType>:$shape,
+    Variadic<AnyIntegerType>:$shift,
+    Variadic<AnyIntegerType>:$slice,
+    Variadic<AnyCoordinateType>:$subcomponent,
+    Variadic<AnyCoordinateType>:$indices,
+    Variadic<AnyIntegerType>:$lenParams
+  );
+  let results = (outs fir_ReferenceType);
+
+  let assemblyFormat = [{
+    $memref (`(`$shape^`)`)? (`origin` $shift^)? (`[`$slice^`]`)?
+      (`path` $subcomponent^)? `<`$indices`>` (`typeparams` $lenParams^)?
+      attr-dict `:` functional-type(operands, results)
+  }];
+
+  let extraClassDeclaration = [{
+    unsigned getRank();
+  }];
+}
+
+#endif
diff --git a/flang/include/flang/Optimizer/CodeGen/CGPasses.td b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
index 46442a281606b326eb9dfffefe62d89634e74e1e..ffe829644d1aab709cf88d1c6c7d94de4297eb17 100644
--- a/flang/include/flang/Optimizer/CodeGen/CGPasses.td
+++ b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
@@ -11,18 +11,24 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef FLANG_OPTIMIZER_CODEGEN_PASSES
-#define FLANG_OPTIMIZER_CODEGEN_PASSES
+#ifndef FORTRAN_OPTIMIZER_CODEGEN_FIR_PASSES
+#define FORTRAN_OPTIMIZER_CODEGEN_FIR_PASSES
 
 include "mlir/Pass/PassBase.td"
 
-def CodeGenRewrite : Pass<"cg-rewrite", "mlir::ModuleOp"> {
+def CodeGenRewrite : Pass<"cg-rewrite"> {
   let summary = "Rewrite some FIR ops into their code-gen forms.";
   let description = [{
     Fuse specific subgraphs into single Ops for code generation.
   }];
   let constructor = "fir::createFirCodeGenRewritePass()";
-  let dependentDialects = ["fir::FIROpsDialect"];
+  let dependentDialects = [
+    "fir::FIROpsDialect", "fir::FIRCodeGenDialect", "mlir::BuiltinDialect",
+    "mlir::LLVM::LLVMDialect", "mlir::omp::OpenMPDialect"
+  ];
+  let statistics = [
+    Statistic<"numDCE", "num-dce'd", "Number of operations eliminated">
+  ];
 }
 
-#endif // FLANG_OPTIMIZER_CODEGEN_PASSES
+#endif // FORTRAN_OPTIMIZER_CODEGEN_FIR_PASSES
diff --git a/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt b/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt
index 8cbd772b30ab94bd2034e78d44559ee857aadc67..3eda75190ba20f95c1d5e9879155832ffebfcd2b 100644
--- a/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/include/flang/Optimizer/CodeGen/CMakeLists.txt
@@ -1,3 +1,7 @@
+set(LLVM_TARGET_DEFINITIONS CGOps.td)
+mlir_tablegen(CGOps.h.inc -gen-op-decls)
+mlir_tablegen(CGOps.cpp.inc -gen-op-defs)
+add_public_tablegen_target(CGOpsIncGen)
 
 set(LLVM_TARGET_DEFINITIONS CGPasses.td)
 mlir_tablegen(CGPasses.h.inc -gen-pass-decls -name OptCodeGen)
diff --git a/flang/include/flang/Optimizer/Dialect/FIRDialect.h b/flang/include/flang/Optimizer/Dialect/FIRDialect.h
index 4bafb4ab7fb68a2cabfee79639cc90c9776d47c8..fb828716d45a1aa4dcb9c3a303bc6d9fcf816865 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRDialect.h
+++ b/flang/include/flang/Optimizer/Dialect/FIRDialect.h
@@ -40,6 +40,16 @@ private:
   void registerTypes();
 };
 
+/// The FIR codegen dialect is a dialect containing a small set of transient
+/// operations used exclusively during code generation.
+class FIRCodeGenDialect final : public mlir::Dialect {
+public:
+  explicit FIRCodeGenDialect(mlir::MLIRContext *ctx);
+  virtual ~FIRCodeGenDialect();
+
+  static llvm::StringRef getDialectNamespace() { return "fircg"; }
+};
+
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_DIALECT_FIRDIALECT_H
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index a039001cb079cd6f2221ef9a711121df76c55ccd..a38630b2a04f0cef5d77c2d8786b9cef46934fa5 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -2795,18 +2795,7 @@ class RealArithmeticOp<string mnemonic, list<OpTrait> traits = []> :
       fir_ArithmeticOp<mnemonic, traits>,
       Arguments<(ins AnyRealLike:$lhs, AnyRealLike:$rhs)>;
 
-def fir_AddfOp : RealArithmeticOp<"addf", [Commutative]> {
-  let hasFolder = 1;
-}
-def fir_SubfOp : RealArithmeticOp<"subf"> {
-  let hasFolder = 1;
-}
-def fir_MulfOp : RealArithmeticOp<"mulf", [Commutative]> {
-  let hasFolder = 1;
-}
-def fir_DivfOp : RealArithmeticOp<"divf">;
 def fir_ModfOp : RealArithmeticOp<"modf">;
-// Pow is a builtin call and not a primitive
 
 def fir_CmpfOp : fir_Op<"cmpf",
     [NoSideEffect, SameTypeOperands, SameOperandsAndResultShape]> {
diff --git a/flang/include/flang/Optimizer/Support/InitFIR.h b/flang/include/flang/Optimizer/Support/InitFIR.h
index cb2dd4f4776bf5d4b18b04f040130c085f2d1b9e..194d42a41a1c3cbc63071caacdbc0db830dd9ba3 100644
--- a/flang/include/flang/Optimizer/Support/InitFIR.h
+++ b/flang/include/flang/Optimizer/Support/InitFIR.h
@@ -21,15 +21,16 @@
 #include "mlir/Pass/PassRegistry.h"
 #include "mlir/Transforms/LocationSnapshot.h"
 #include "mlir/Transforms/Passes.h"
+#include "flang/Optimizer/CodeGen/CodeGen.h"
 
 namespace fir::support {
 
 // The definitive list of dialects used by flang.
 #define FLANG_DIALECT_LIST                                                     \
-  mlir::AffineDialect, FIROpsDialect, mlir::LLVM::LLVMDialect,                 \
-      mlir::acc::OpenACCDialect, mlir::omp::OpenMPDialect,                     \
-      mlir::scf::SCFDialect, mlir::StandardOpsDialect,                         \
-      mlir::vector::VectorDialect
+  mlir::AffineDialect, FIROpsDialect, FIRCodeGenDialect,                       \
+      mlir::LLVM::LLVMDialect, mlir::acc::OpenACCDialect,                      \
+      mlir::omp::OpenMPDialect, mlir::scf::SCFDialect,                         \
+      mlir::StandardOpsDialect, mlir::vector::VectorDialect
 
 /// Register all the dialects used by flang.
 inline void registerDialects(mlir::DialectRegistry &registry) {
@@ -45,7 +46,7 @@ inline void loadDialects(mlir::MLIRContext &context) {
 
 /// Register the standard passes we use. This comes from registerAllPasses(),
 /// but is a smaller set since we aren't using many of the passes found there.
-inline void registerFIRPasses() {
+inline void registerMLIRPassesForFortranTools() {
   mlir::registerCanonicalizerPass();
   mlir::registerCSEPass();
   mlir::registerAffineLoopFusionPass();
@@ -69,6 +70,9 @@ inline void registerFIRPasses() {
   mlir::registerAffineDataCopyGenerationPass();
 
   mlir::registerConvertAffineToStandardPass();
+
+  // Flang passes
+  fir::registerOptCodeGenPasses();
 }
 
 } // namespace fir::support
diff --git a/flang/include/flang/Parser/char-block.h b/flang/include/flang/Parser/char-block.h
index 7c29c9aa41a0fe3accf3196ef010c315f81558c0..0f5758f8c552cee122f5892314c576b52d0a88c3 100644
--- a/flang/include/flang/Parser/char-block.h
+++ b/flang/include/flang/Parser/char-block.h
@@ -138,6 +138,13 @@ inline bool operator>(const char *left, const CharBlock &right) {
   return right < left;
 }
 
+// An alternative comparator based on pointer values; use with care!
+struct CharBlockPointerComparator {
+  bool operator()(CharBlock x, CharBlock y) const {
+    return x.end() < y.begin();
+  }
+};
+
 llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const CharBlock &x);
 
 } // namespace Fortran::parser
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index bc0fd388b11cdcb093c680b05c6cfd7350619562..150b011ad8ba0748d3138dc1c6dfcc58061242ee 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -793,7 +793,7 @@ protected:
   template <typename T> std::string AsFortran(const T &x) {
     std::string buf;
     llvm::raw_string_ostream ss{buf};
-    if constexpr (std::is_same_v<T, Expr>) {
+    if constexpr (HasTypedExpr<T>::value) {
       if (asFortran_ && x.typedExpr) {
         asFortran_->expr(ss, *x.typedExpr);
       }
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index dcc38090a3a18eb1b06178762bfd147990014c2a..152c2c8c9076b9543b1e26f2a7ac5c7957aae25b 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -1836,6 +1836,7 @@ struct ArrayElement {
 // R933 allocate-object -> variable-name | structure-component
 struct AllocateObject {
   UNION_CLASS_BOILERPLATE(AllocateObject);
+  mutable TypedExpr typedExpr;
   std::variant<Name, StructureComponent> u;
 };
 
@@ -1907,6 +1908,7 @@ struct AllocateStmt {
 //        variable-name | structure-component | proc-pointer-name
 struct PointerObject {
   UNION_CLASS_BOILERPLATE(PointerObject);
+  mutable TypedExpr typedExpr;
   std::variant<Name, StructureComponent> u;
 };
 
diff --git a/flang/include/flang/Parser/provenance.h b/flang/include/flang/Parser/provenance.h
index bce79809c7664a183eb7c251acb180781a701542..7ff475a2316ab030c3b7d4d884ae77de19f1a843 100644
--- a/flang/include/flang/Parser/provenance.h
+++ b/flang/include/flang/Parser/provenance.h
@@ -47,6 +47,7 @@ namespace Fortran::parser {
 // necessary.)
 
 class AllSources;
+class AllCookedSources;
 
 class Provenance {
 public:
@@ -219,6 +220,9 @@ private:
 // single instances of CookedSource.
 class CookedSource {
 public:
+  int number() const { return number_; }
+  void set_number(int n) { number_ = n; }
+
   CharBlock AsCharBlock() const { return CharBlock{data_}; }
   std::optional<ProvenanceRange> GetProvenanceRange(CharBlock) const;
   std::optional<CharBlock> GetCharBlock(ProvenanceRange) const;
@@ -242,11 +246,12 @@ public:
   }
 
   std::size_t BufferedBytes() const;
-  void Marshal(AllSources &); // marshals text into one contiguous block
+  void Marshal(AllCookedSources &); // marshals text into one contiguous block
   void CompileProvenanceRangeToOffsetMappings(AllSources &);
   llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
 
 private:
+  int number_{0}; // for sorting purposes
   CharBuffer buffer_; // before Marshal()
   std::string data_; // all of it, prescanned and preprocessed
   OffsetToProvenanceMappings provenanceMap_;
@@ -263,15 +268,8 @@ public:
 
   CookedSource &NewCookedSource();
 
-  template <typename A> // const char * or CharBlock
-  const CookedSource *Find(A x) const {
-    for (const auto &c : cooked_) {
-      if (c.AsCharBlock().Contains(x)) {
-        return &c;
-      }
-    }
-    return nullptr;
-  }
+  const CookedSource *Find(CharBlock) const;
+  const CookedSource *Find(const char *p) const { return Find(CharBlock{p}); }
 
   bool IsValid(ProvenanceRange r) const { return allSources_.IsValid(r); }
 
@@ -283,9 +281,29 @@ public:
   std::optional<CharBlock> GetCharBlock(ProvenanceRange) const;
   void Dump(llvm::raw_ostream &) const;
 
+  // For sorting symbol names without being dependent on pointer values
+  bool Precedes(CharBlock, CharBlock) const;
+
+  // Once a CookedSource is complete, add it to index_ and assign its number_
+  void Register(CookedSource &);
+
 private:
   AllSources &allSources_;
   std::list<CookedSource> cooked_; // owns all CookedSource instances
+  std::map<CharBlock, const CookedSource &, CharBlockPointerComparator> index_;
 };
+
+// For use as a Comparator for maps, sets, sorting, &c.
+class CharBlockComparator {
+public:
+  explicit CharBlockComparator(const AllCookedSources &all) : all_{all} {}
+  bool operator()(CharBlock x, CharBlock y) const {
+    return all_.Precedes(x, y);
+  }
+
+private:
+  const AllCookedSources &all_;
+};
+
 } // namespace Fortran::parser
 #endif // FORTRAN_PARSER_PROVENANCE_H_
diff --git a/flang/include/flang/Parser/tools.h b/flang/include/flang/Parser/tools.h
index 66c8793399c938e9d36b3a920aec445a41151315..ccd49d2a790e27cd7b27052378d5ac4c149f7161 100644
--- a/flang/include/flang/Parser/tools.h
+++ b/flang/include/flang/Parser/tools.h
@@ -117,5 +117,10 @@ template <typename A>
 struct HasSource<A, decltype(static_cast<void>(A::source), 0)>
     : std::true_type {};
 
+// Detects parse tree nodes with "typedExpr" members.
+template <typename A, typename = int> struct HasTypedExpr : std::false_type {};
+template <typename A>
+struct HasTypedExpr<A, decltype(static_cast<void>(A::typedExpr), 0)>
+    : std::true_type {};
 } // namespace Fortran::parser
 #endif // FORTRAN_PARSER_TOOLS_H_
diff --git a/flang/include/flang/Semantics/expression.h b/flang/include/flang/Semantics/expression.h
index f81d5199dc209a07b6a5d671703ed794f983206b..2f89820f4b0b154d9d1ed4618d732ef7c05bc203 100644
--- a/flang/include/flang/Semantics/expression.h
+++ b/flang/include/flang/Semantics/expression.h
@@ -74,14 +74,13 @@ struct SetExprHelper {
     x.Reset(new GenericExprWrapper{std::move(expr_)},
         evaluate::GenericExprWrapper::Deleter);
   }
-  void Set(const parser::Expr &x) { Set(x.typedExpr); }
-  void Set(const parser::Variable &x) { Set(x.typedExpr); }
-  void Set(const parser::DataStmtConstant &x) { Set(x.typedExpr); }
   template <typename T> void Set(const common::Indirection<T> &x) {
     Set(x.value());
   }
   template <typename T> void Set(const T &x) {
-    if constexpr (ConstraintTrait<T>) {
+    if constexpr (parser::HasTypedExpr<T>::value) {
+      Set(x.typedExpr);
+    } else if constexpr (ConstraintTrait<T>) {
       Set(x.thing);
     } else if constexpr (WrapperTrait<T>) {
       Set(x.v);
@@ -157,6 +156,8 @@ public:
   MaybeExpr Analyze(const parser::Variable &);
   MaybeExpr Analyze(const parser::Designator &);
   MaybeExpr Analyze(const parser::DataStmtValue &);
+  MaybeExpr Analyze(const parser::AllocateObject &);
+  MaybeExpr Analyze(const parser::PointerObject &);
 
   template <typename A> MaybeExpr Analyze(const common::Indirection<A> &x) {
     return Analyze(x.value());
@@ -451,6 +452,14 @@ public:
     exprAnalyzer_.Analyze(x);
     return false;
   }
+  bool Pre(const parser::AllocateObject &x) {
+    exprAnalyzer_.Analyze(x);
+    return false;
+  }
+  bool Pre(const parser::PointerObject &x) {
+    exprAnalyzer_.Analyze(x);
+    return false;
+  }
   bool Pre(const parser::DataImpliedDo &);
 
   bool Pre(const parser::CallStmt &x) {
diff --git a/flang/include/flang/Semantics/scope.h b/flang/include/flang/Semantics/scope.h
index 81840bdb01223c2c9ac256b2bcd7603d21a2229a..f1d5b0c87d48adc7c63e5bcba8cfff0c069f47aa 100644
--- a/flang/include/flang/Semantics/scope.h
+++ b/flang/include/flang/Semantics/scope.h
@@ -62,9 +62,10 @@ public:
   using ImportKind = common::ImportKind;
 
   // Create the Global scope -- the root of the scope tree
-  Scope() : Scope{*this, Kind::Global, nullptr} {}
-  Scope(Scope &parent, Kind kind, Symbol *symbol)
-      : parent_{parent}, kind_{kind}, symbol_{symbol} {
+  explicit Scope(SemanticsContext &context)
+      : Scope{*this, Kind::Global, nullptr, context} {}
+  Scope(Scope &parent, Kind kind, Symbol *symbol, SemanticsContext &context)
+      : parent_{parent}, kind_{kind}, symbol_{symbol}, context_{context} {
     if (symbol) {
       symbol->set_scope(this);
     }
@@ -99,6 +100,7 @@ public:
   }
   Symbol *symbol() { return symbol_; }
   const Symbol *symbol() const { return symbol_; }
+  SemanticsContext &context() const { return context_; }
 
   inline const Symbol *GetSymbol() const;
   const Scope *GetDerivedTypeParent() const;
@@ -107,6 +109,9 @@ public:
   bool Contains(const Scope &) const;
   /// Make a scope nested in this one
   Scope &MakeScope(Kind kind, Symbol *symbol = nullptr);
+  SemanticsContext &GetMutableSemanticsContext() const {
+    return const_cast<SemanticsContext &>(context());
+  }
 
   using size_type = mapType::size_type;
   using iterator = mapType::iterator;
@@ -244,7 +249,7 @@ public:
         symbol_->test(Symbol::Flag::ModFile);
   }
 
-  void InstantiateDerivedTypes(SemanticsContext &);
+  void InstantiateDerivedTypes();
 
   const Symbol *runtimeDerivedTypeDescription() const {
     return runtimeDerivedTypeDescription_;
@@ -273,8 +278,9 @@ private:
   parser::Message::Reference instantiationContext_;
   bool hasSAVE_{false}; // scope has a bare SAVE statement
   const Symbol *runtimeDerivedTypeDescription_{nullptr};
+  SemanticsContext &context_;
   // When additional data members are added to Scope, remember to
-  // copy them, if appropriate, in InstantiateDerivedType().
+  // copy them, if appropriate, in FindOrInstantiateDerivedType().
 
   // Storage for all Symbols. Every Symbol is in allSymbols and every Symbol*
   // or Symbol& points to one in there.
diff --git a/flang/include/flang/Semantics/semantics.h b/flang/include/flang/Semantics/semantics.h
index e6202c666429beeb7aada09671989b78c9636a76..3ef0cafa872aaf0cd3408522dccaab8c9e52ed83 100644
--- a/flang/include/flang/Semantics/semantics.h
+++ b/flang/include/flang/Semantics/semantics.h
@@ -198,8 +198,9 @@ private:
     parser::CharBlock location;
     IndexVarKind kind;
   };
-  std::map<SymbolRef, const IndexVarInfo> activeIndexVars_;
-  SymbolSet errorSymbols_;
+  std::map<SymbolRef, const IndexVarInfo, SymbolAddressCompare>
+      activeIndexVars_;
+  UnorderedSymbolSet errorSymbols_;
   std::set<std::string> tempNames_;
 };
 
diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index fb53c61c7d23cf785aa5a700253544b99a7f354e..4586ad9f864d830b0c0d02ece2853c36849c8123 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -18,7 +18,7 @@
 #include <functional>
 #include <list>
 #include <optional>
-#include <unordered_set>
+#include <set>
 #include <vector>
 
 namespace llvm {
@@ -60,7 +60,18 @@ public:
 private:
 };
 
-class SubprogramDetails {
+class WithBindName {
+public:
+  const std::string *bindName() const {
+    return bindName_ ? &*bindName_ : nullptr;
+  }
+  void set_bindName(std::string &&name) { bindName_ = std::move(name); }
+
+private:
+  std::optional<std::string> bindName_;
+};
+
+class SubprogramDetails : public WithBindName {
 public:
   bool isFunction() const { return result_ != nullptr; }
   bool isInterface() const { return isInterface_; }
@@ -68,8 +79,6 @@ public:
   Scope *entryScope() { return entryScope_; }
   const Scope *entryScope() const { return entryScope_; }
   void set_entryScope(Scope &scope) { entryScope_ = &scope; }
-  MaybeExpr bindName() const { return bindName_; }
-  void set_bindName(MaybeExpr &&expr) { bindName_ = std::move(expr); }
   const Symbol &result() const {
     CHECK(isFunction());
     return *result_;
@@ -86,7 +95,6 @@ public:
 
 private:
   bool isInterface_{false}; // true if this represents an interface-body
-  MaybeExpr bindName_;
   std::vector<Symbol *> dummyArgs_; // nullptr -> alternate return indicator
   Symbol *result_{nullptr};
   Scope *entryScope_{nullptr}; // if ENTRY, points to subprogram's scope
@@ -117,7 +125,7 @@ private:
 };
 
 // A name from an entity-decl -- could be object or function.
-class EntityDetails {
+class EntityDetails : public WithBindName {
 public:
   explicit EntityDetails(bool isDummy = false) : isDummy_{isDummy} {}
   const DeclTypeSpec *type() const { return type_; }
@@ -127,14 +135,11 @@ public:
   void set_isDummy(bool value = true) { isDummy_ = value; }
   bool isFuncResult() const { return isFuncResult_; }
   void set_funcResult(bool x) { isFuncResult_ = x; }
-  MaybeExpr bindName() const { return bindName_; }
-  void set_bindName(MaybeExpr &&expr) { bindName_ = std::move(expr); }
 
 private:
   bool isDummy_{false};
   bool isFuncResult_{false};
   const DeclTypeSpec *type_{nullptr};
-  MaybeExpr bindName_;
   friend llvm::raw_ostream &operator<<(
       llvm::raw_ostream &, const EntityDetails &);
 };
@@ -310,19 +315,16 @@ private:
   SymbolVector objects_;
 };
 
-class CommonBlockDetails {
+class CommonBlockDetails : public WithBindName {
 public:
   MutableSymbolVector &objects() { return objects_; }
   const MutableSymbolVector &objects() const { return objects_; }
   void add_object(Symbol &object) { objects_.emplace_back(object); }
-  MaybeExpr bindName() const { return bindName_; }
-  void set_bindName(MaybeExpr &&expr) { bindName_ = std::move(expr); }
   std::size_t alignment() const { return alignment_; }
   void set_alignment(std::size_t alignment) { alignment_ = alignment; }
 
 private:
   MutableSymbolVector objects_;
-  MaybeExpr bindName_;
   std::size_t alignment_{0}; // required alignment in bytes
 };
 
@@ -565,8 +567,10 @@ public:
 
   inline DeclTypeSpec *GetType();
   inline const DeclTypeSpec *GetType() const;
-
   void SetType(const DeclTypeSpec &);
+
+  const std::string *GetBindName() const;
+  void SetBindName(std::string &&);
   bool IsFuncResult() const;
   bool IsObjectArray() const;
   bool IsSubprogram() const;
@@ -595,10 +599,6 @@ public:
 
   bool operator==(const Symbol &that) const { return this == &that; }
   bool operator!=(const Symbol &that) const { return !(*this == that); }
-  bool operator<(const Symbol &that) const {
-    // Used to collate symbols by creation order
-    return sortIndex_ < that.sortIndex_;
-  }
 
   int Rank() const {
     return std::visit(
@@ -651,10 +651,11 @@ public:
   // for a parameterized derived type instantiation with the instance's scope.
   const DerivedTypeSpec *GetParentTypeSpec(const Scope * = nullptr) const;
 
+  SemanticsContext &GetSemanticsContext() const;
+
 private:
   const Scope *owner_;
   SourceName name_;
-  std::size_t sortIndex_; // to implement "operator<" platform independently
   Attrs attrs_;
   Flags flags_;
   Scope *scope_{nullptr};
@@ -689,7 +690,6 @@ public:
     Symbol &symbol = Get();
     symbol.owner_ = &owner;
     symbol.name_ = name;
-    symbol.sortIndex_ = ++symbolCount_;
     symbol.attrs_ = attrs;
     symbol.details_ = std::move(details);
     return symbol;
@@ -700,7 +700,6 @@ private:
   std::list<blockType *> blocks_;
   std::size_t nextIndex_{0};
   blockType *currBlock_{nullptr};
-  static inline std::size_t symbolCount_ = 0;
 
   Symbol &Get() {
     if (nextIndex_ == 0) {
@@ -765,17 +764,40 @@ inline const DeclTypeSpec *Symbol::GetType() const {
       details_);
 }
 
-inline bool operator<(SymbolRef x, SymbolRef y) { return *x < *y; }
-inline bool operator<(MutableSymbolRef x, MutableSymbolRef y) {
-  return *x < *y;
-}
-struct SymbolHash {
-  std::size_t operator()(SymbolRef symRef) const {
-    std::hash<std::string> hasher;
-    return hasher(symRef->name().ToString());
+// Sets and maps keyed by Symbols
+
+struct SymbolAddressCompare {
+  bool operator()(const SymbolRef &x, const SymbolRef &y) const {
+    return &*x < &*y;
+  }
+  bool operator()(const MutableSymbolRef &x, const MutableSymbolRef &y) const {
+    return &*x < &*y;
   }
 };
-using SymbolSet = std::unordered_set<SymbolRef, SymbolHash>;
+
+// Symbol comparison is based on the order of cooked source
+// stream creation and, when both are from the same cooked source,
+// their positions in that cooked source stream.
+// Don't use this comparator or OrderedSymbolSet to hold
+// Symbols that might be subject to ReplaceName().
+struct SymbolSourcePositionCompare {
+  // These functions are implemented in Evaluate/tools.cpp to
+  // satisfy complicated shared library interdependency.
+  bool operator()(const SymbolRef &, const SymbolRef &) const;
+  bool operator()(const MutableSymbolRef &, const MutableSymbolRef &) const;
+};
+
+using UnorderedSymbolSet = std::set<SymbolRef, SymbolAddressCompare>;
+using OrderedSymbolSet = std::set<SymbolRef, SymbolSourcePositionCompare>;
+
+template <typename A>
+OrderedSymbolSet OrderBySourcePosition(const A &container) {
+  OrderedSymbolSet result;
+  for (SymbolRef x : container) {
+    result.emplace(x);
+  }
+  return result;
+}
 
 } // namespace Fortran::semantics
 
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index 3e8d1993f9a04cb8bed1a94f43e12324bf79fce8..550cc99f85efe1cdbb829cffafec4d3f0b1792d2 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -257,9 +257,13 @@ bool ExprTypeKindIsDefault(
     const SomeExpr &expr, const SemanticsContext &context);
 
 struct GetExprHelper {
+  // Specializations for parse tree nodes that have a typedExpr member.
   static const SomeExpr *Get(const parser::Expr &);
   static const SomeExpr *Get(const parser::Variable &);
   static const SomeExpr *Get(const parser::DataStmtConstant &);
+  static const SomeExpr *Get(const parser::AllocateObject &);
+  static const SomeExpr *Get(const parser::PointerObject &);
+
   template <typename T>
   static const SomeExpr *Get(const common::Indirection<T> &x) {
     return Get(x.value());
@@ -268,6 +272,8 @@ struct GetExprHelper {
     return x ? Get(*x) : nullptr;
   }
   template <typename T> static const SomeExpr *Get(const T &x) {
+    static_assert(
+        !parser::HasTypedExpr<T>::value, "explicit Get overload must be added");
     if constexpr (ConstraintTrait<T>) {
       return Get(x.thing);
     } else if constexpr (WrapperTrait<T>) {
diff --git a/flang/lib/Evaluate/characteristics.cpp b/flang/lib/Evaluate/characteristics.cpp
index 4d5436a9776caaa867f82bea3b7a41ff680822d0..c6d6afeb81d18bbf69bdef2a5e1b989fed2b9729 100644
--- a/flang/lib/Evaluate/characteristics.cpp
+++ b/flang/lib/Evaluate/characteristics.cpp
@@ -343,30 +343,29 @@ bool DummyProcedure::operator==(const DummyProcedure &that) const {
       procedure.value() == that.procedure.value();
 }
 
-static std::string GetSeenProcs(const semantics::SymbolSet &seenProcs) {
+static std::string GetSeenProcs(
+    const semantics::UnorderedSymbolSet &seenProcs) {
   // Sort the symbols so that they appear in the same order on all platforms
-  std::vector<SymbolRef> sorter{seenProcs.begin(), seenProcs.end()};
-  std::sort(sorter.begin(), sorter.end());
-
+  auto ordered{semantics::OrderBySourcePosition(seenProcs)};
   std::string result;
   llvm::interleave(
-      sorter,
+      ordered,
       [&](const SymbolRef p) { result += '\'' + p->name().ToString() + '\''; },
       [&]() { result += ", "; });
   return result;
 }
 
-// These functions with arguments of type SymbolSet are used with mutually
-// recursive calls when characterizing a Procedure, a DummyArgument, or a
-// DummyProcedure to detect circularly defined procedures as required by
+// These functions with arguments of type UnorderedSymbolSet are used with
+// mutually recursive calls when characterizing a Procedure, a DummyArgument,
+// or a DummyProcedure to detect circularly defined procedures as required by
 // 15.4.3.6, paragraph 2.
 static std::optional<DummyArgument> CharacterizeDummyArgument(
     const semantics::Symbol &symbol, FoldingContext &context,
-    semantics::SymbolSet &seenProcs);
+    semantics::UnorderedSymbolSet &seenProcs);
 
 static std::optional<Procedure> CharacterizeProcedure(
     const semantics::Symbol &original, FoldingContext &context,
-    semantics::SymbolSet &seenProcs) {
+    semantics::UnorderedSymbolSet &seenProcs) {
   Procedure result;
   const auto &symbol{original.GetUltimate()};
   if (seenProcs.find(symbol) != seenProcs.end()) {
@@ -475,7 +474,7 @@ static std::optional<Procedure> CharacterizeProcedure(
 
 static std::optional<DummyProcedure> CharacterizeDummyProcedure(
     const semantics::Symbol &symbol, FoldingContext &context,
-    semantics::SymbolSet &seenProcs) {
+    semantics::UnorderedSymbolSet &seenProcs) {
   if (auto procedure{CharacterizeProcedure(symbol, context, seenProcs)}) {
     // Dummy procedures may not be elemental.  Elemental dummy procedure
     // interfaces are errors when the interface is not intrinsic, and that
@@ -516,7 +515,7 @@ bool DummyArgument::operator==(const DummyArgument &that) const {
 
 static std::optional<DummyArgument> CharacterizeDummyArgument(
     const semantics::Symbol &symbol, FoldingContext &context,
-    semantics::SymbolSet &seenProcs) {
+    semantics::UnorderedSymbolSet &seenProcs) {
   auto name{symbol.name().ToString()};
   if (symbol.has<semantics::ObjectEntityDetails>()) {
     if (auto obj{DummyDataObject::Characterize(symbol, context)}) {
@@ -779,7 +778,7 @@ bool Procedure::CanOverride(
 
 std::optional<Procedure> Procedure::Characterize(
     const semantics::Symbol &original, FoldingContext &context) {
-  semantics::SymbolSet seenProcs;
+  semantics::UnorderedSymbolSet seenProcs;
   return CharacterizeProcedure(original, context, seenProcs);
 }
 
diff --git a/flang/lib/Evaluate/constant.cpp b/flang/lib/Evaluate/constant.cpp
index 5b73979f1e2c2c2497b53306670ecae33cc64e02..8f30ca08116266b4a65249be4a12ff20589e00ef 100644
--- a/flang/lib/Evaluate/constant.cpp
+++ b/flang/lib/Evaluate/constant.cpp
@@ -315,5 +315,9 @@ std::size_t Constant<SomeDerived>::CopyFrom(const Constant<SomeDerived> &source,
   return Base::CopyFrom(source, count, resultSubscripts, dimOrder);
 }
 
+bool ComponentCompare::operator()(SymbolRef x, SymbolRef y) const {
+  return semantics::SymbolSourcePositionCompare{}(x, y);
+}
+
 INSTANTIATE_CONSTANT_TEMPLATES
 } // namespace Fortran::evaluate
diff --git a/flang/lib/Evaluate/fold-logical.cpp b/flang/lib/Evaluate/fold-logical.cpp
index 64e4bd8c8bd91ace2e1ab55888c7f3b767bc06a7..455b3c2605c31466e97bd8108c5bffcb1dc19fc0 100644
--- a/flang/lib/Evaluate/fold-logical.cpp
+++ b/flang/lib/Evaluate/fold-logical.cpp
@@ -106,6 +106,10 @@ Expr<Type<TypeCategory::Logical, KIND>> FoldIntrinsicFunction(
         }
       }
     }
+  } else if (name == "logical") {
+    if (auto *expr{UnwrapExpr<Expr<SomeLogical>>(args[0])}) {
+      return Fold(context, ConvertToType<T>(std::move(*expr)));
+    }
   } else if (name == "merge") {
     return FoldMerge<T>(context, std::move(funcRef));
   } else if (name == "__builtin_ieee_support_datatype" ||
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 0685f14088a61207353f0be2888dae35117ca5a0..9fbf21e43b72dca9daea2c6430476b1a71d16f9e 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -782,20 +782,22 @@ const Symbol *GetLastTarget(const SymbolVector &symbols) {
 }
 
 struct CollectSymbolsHelper
-    : public SetTraverse<CollectSymbolsHelper, semantics::SymbolSet> {
-  using Base = SetTraverse<CollectSymbolsHelper, semantics::SymbolSet>;
+    : public SetTraverse<CollectSymbolsHelper, semantics::UnorderedSymbolSet> {
+  using Base = SetTraverse<CollectSymbolsHelper, semantics::UnorderedSymbolSet>;
   CollectSymbolsHelper() : Base{*this} {}
   using Base::operator();
-  semantics::SymbolSet operator()(const Symbol &symbol) const {
+  semantics::UnorderedSymbolSet operator()(const Symbol &symbol) const {
     return {symbol};
   }
 };
-template <typename A> semantics::SymbolSet CollectSymbols(const A &x) {
+template <typename A> semantics::UnorderedSymbolSet CollectSymbols(const A &x) {
   return CollectSymbolsHelper{}(x);
 }
-template semantics::SymbolSet CollectSymbols(const Expr<SomeType> &);
-template semantics::SymbolSet CollectSymbols(const Expr<SomeInteger> &);
-template semantics::SymbolSet CollectSymbols(const Expr<SubscriptInteger> &);
+template semantics::UnorderedSymbolSet CollectSymbols(const Expr<SomeType> &);
+template semantics::UnorderedSymbolSet CollectSymbols(
+    const Expr<SomeInteger> &);
+template semantics::UnorderedSymbolSet CollectSymbols(
+    const Expr<SubscriptInteger> &);
 
 // HasVectorSubscript()
 struct HasVectorSubscriptHelper : public AnyTraverse<HasVectorSubscriptHelper> {
@@ -1177,7 +1179,7 @@ const Symbol &GetUsedModule(const UseDetails &details) {
 }
 
 static const Symbol *FindFunctionResult(
-    const Symbol &original, SymbolSet &seen) {
+    const Symbol &original, UnorderedSymbolSet &seen) {
   const Symbol &root{GetAssociationRoot(original)};
   ;
   if (!seen.insert(root).second) {
@@ -1199,8 +1201,27 @@ static const Symbol *FindFunctionResult(
 }
 
 const Symbol *FindFunctionResult(const Symbol &symbol) {
-  SymbolSet seen;
+  UnorderedSymbolSet seen;
   return FindFunctionResult(symbol, seen);
 }
 
+// These are here in Evaluate/tools.cpp so that Evaluate can use
+// them; they cannot be defined in symbol.h due to the dependence
+// on Scope.
+
+bool SymbolSourcePositionCompare::operator()(
+    const SymbolRef &x, const SymbolRef &y) const {
+  return x->GetSemanticsContext().allCookedSources().Precedes(
+      x->name(), y->name());
+}
+bool SymbolSourcePositionCompare::operator()(
+    const MutableSymbolRef &x, const MutableSymbolRef &y) const {
+  return x->GetSemanticsContext().allCookedSources().Precedes(
+      x->name(), y->name());
+}
+
+SemanticsContext &Symbol::GetSemanticsContext() const {
+  return DEREF(owner_).context();
+}
+
 } // namespace Fortran::semantics
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 1271cd314831d9d82012e551b0d6824a1af05ea9..3bd541e40c0fb5453ba8e7db6d2faaf41dcf2fad 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -21,6 +21,8 @@
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/OptTable.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/raw_ostream.h"
 #include <memory>
@@ -143,6 +145,9 @@ static InputKind ParseFrontendArgs(FrontendOptions &opts,
     case clang::driver::options::OPT_fdebug_pre_fir_tree:
       opts.programAction_ = DebugPreFIRTree;
       break;
+    case clang::driver::options::OPT_fget_symbols_sources:
+      opts.programAction_ = GetSymbolsSources;
+      break;
 
       // TODO:
       // case calng::driver::options::OPT_emit_llvm:
@@ -282,6 +287,16 @@ static InputKind ParseFrontendArgs(FrontendOptions &opts,
   return dashX;
 }
 
+// Generate the path to look for intrinsic modules
+static std::string getIntrinsicDir() {
+  // TODO: Find a system independent API
+  llvm::SmallString<128> driverPath;
+  driverPath.assign(llvm::sys::fs::getMainExecutable(nullptr, nullptr));
+  llvm::sys::path::remove_filename(driverPath);
+  driverPath.append("/../include/flang/");
+  return std::string(driverPath);
+}
+
 /// Parses all preprocessor input arguments and populates the preprocessor
 /// options accordingly.
 ///
@@ -302,6 +317,12 @@ static void parsePreprocessorArgs(
   // Add the ordered list of -I's.
   for (const auto *currentArg : args.filtered(clang::driver::options::OPT_I))
     opts.searchDirectoriesFromDashI.emplace_back(currentArg->getValue());
+
+  // Prepend the ordered list of -intrinsic-modules-path
+  // to the default location to search.
+  for (const auto *currentArg :
+      args.filtered(clang::driver::options::OPT_fintrinsic_modules_path))
+    opts.searchDirectoriesFromIntrModPath.emplace_back(currentArg->getValue());
 }
 
 /// Parses all semantic related arguments and populates the variables
@@ -368,6 +389,26 @@ static void parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
     res.frontendOpts().features_.Enable(
         Fortran::common::LanguageFeature::OpenMP);
   }
+
+  // -pedantic
+  if (args.hasArg(clang::driver::options::OPT_pedantic)) {
+    res.set_EnableConformanceChecks();
+  }
+  // -std=f2018 (currently this implies -pedantic)
+  // TODO: Set proper options when more fortran standards
+  // are supported.
+  if (args.hasArg(clang::driver::options::OPT_std_EQ)) {
+    auto standard = args.getLastArgValue(clang::driver::options::OPT_std_EQ);
+    // We only allow f2018 as the given standard
+    if (standard.equals("f2018")) {
+      res.set_EnableConformanceChecks();
+    } else {
+      const unsigned diagID =
+          diags.getCustomDiagID(clang::DiagnosticsEngine::Error,
+              "Only -std=f2018 is allowed currently.");
+      diags.Report(diagID);
+    }
+  }
   return;
 }
 
@@ -496,11 +537,21 @@ void CompilerInvocation::setFortranOpts() {
 
   collectMacroDefinitions(preprocessorOptions, fortranOptions);
 
+  // Adding search directories specified by -I
   fortranOptions.searchDirectories.insert(
       fortranOptions.searchDirectories.end(),
       preprocessorOptions.searchDirectoriesFromDashI.begin(),
       preprocessorOptions.searchDirectoriesFromDashI.end());
 
+  // Add the ordered list of -intrinsic-modules-path
+  fortranOptions.searchDirectories.insert(
+      fortranOptions.searchDirectories.end(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.begin(),
+      preprocessorOptions.searchDirectoriesFromIntrModPath.end());
+
+  //  Add the default intrinsic module directory at the end
+  fortranOptions.searchDirectories.emplace_back(getIntrinsicDir());
+
   // Add the directory supplied through -J/-module-dir to the list of search
   // directories
   if (moduleDirJ.compare(".") != 0)
@@ -508,6 +559,10 @@ void CompilerInvocation::setFortranOpts() {
 
   if (frontendOptions.instrumentedParse_)
     fortranOptions.instrumentedParse = true;
+
+  if (enableConformanceChecks()) {
+    fortranOptions.features.WarnOnAllNonstandard();
+  }
 }
 
 void CompilerInvocation::setSemanticsOpts(
@@ -518,5 +573,6 @@ void CompilerInvocation::setSemanticsOpts(
       defaultKinds(), fortranOptions.features, allCookedSources);
 
   semanticsContext_->set_moduleDirectory(moduleDir())
-      .set_searchDirectories(fortranOptions.searchDirectories);
+      .set_searchDirectories(fortranOptions.searchDirectories)
+      .set_warnOnNonstandardUsage(enableConformanceChecks());
 }
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index ea283fe7a0c93bd5173da19d648387f9b48fafb7..1871a35444db5ad7160e6286b43cf24d0eb2e8bc 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -16,6 +16,7 @@
 #include "flang/Parser/provenance.h"
 #include "flang/Parser/source.h"
 #include "flang/Parser/unparse.h"
+#include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/unparse-with-symbols.h"
 #include "llvm/ADT/StringRef.h"
@@ -314,6 +315,15 @@ void DebugDumpParsingLogAction::ExecuteAction() {
   ci.parsing().DumpParsingLog(llvm::outs());
 }
 
+void GetSymbolsSourcesAction::ExecuteAction() {
+  // Report and exit if fatal semantic errors are present
+  if (reportFatalSemanticErrors(semantics(), this->instance().diagnostics(),
+          GetCurrentFileOrBufferName()))
+    return;
+
+  semantics().DumpSymbolsSources(llvm::outs());
+}
+
 void EmitObjAction::ExecuteAction() {
   CompilerInstance &ci = this->instance();
   unsigned DiagID = ci.diagnostics().getCustomDiagID(
diff --git a/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 041e79b946f5fc5322eee097ae79aff9b047fe33..2a08e388a9d81d94bfbe03d2587d640c68d8d443 100644
--- a/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -61,6 +61,9 @@ static std::unique_ptr<FrontendAction> CreateFrontendBaseAction(
   case DebugPreFIRTree:
     return std::make_unique<DebugPreFIRTreeAction>();
     break;
+  case GetSymbolsSources:
+    return std::make_unique<GetSymbolsSourcesAction>();
+    break;
   default:
     break;
     // TODO:
diff --git a/flang/lib/Lower/IntrinsicCall.cpp b/flang/lib/Lower/IntrinsicCall.cpp
index 15fedf55cbcb1736799878c51636733e7e820bdc..b9c2bba03631c2f08883e40c53bbdd20a4afea08 100644
--- a/flang/lib/Lower/IntrinsicCall.cpp
+++ b/flang/lib/Lower/IntrinsicCall.cpp
@@ -1039,7 +1039,7 @@ mlir::Value IntrinsicLibrary::genDim(mlir::Type resultType,
   }
   assert(fir::isa_real(resultType) && "Only expects real and integer in DIM");
   auto zero = builder.createRealZeroConstant(loc, resultType);
-  auto diff = builder.create<fir::SubfOp>(loc, args[0], args[1]);
+  auto diff = builder.create<mlir::SubFOp>(loc, args[0], args[1]);
   auto cmp =
       builder.create<fir::CmpfOp>(loc, mlir::CmpFPredicate::OGT, diff, zero);
   return builder.create<mlir::SelectOp>(loc, cmp, diff, zero);
@@ -1053,7 +1053,7 @@ mlir::Value IntrinsicLibrary::genDprod(mlir::Type resultType,
          "Result must be double precision in DPROD");
   auto a = builder.createConvert(loc, resultType, args[0]);
   auto b = builder.createConvert(loc, resultType, args[1]);
-  return builder.create<fir::MulfOp>(loc, a, b);
+  return builder.create<mlir::MulFOp>(loc, a, b);
 }
 
 // FLOOR
diff --git a/flang/lib/Optimizer/CMakeLists.txt b/flang/lib/Optimizer/CMakeLists.txt
index 0a7286339e2ed88988546836217c3572a6fae6f2..b83d6a079db6302ce3eae67c0868230db55fca28 100644
--- a/flang/lib/Optimizer/CMakeLists.txt
+++ b/flang/lib/Optimizer/CMakeLists.txt
@@ -10,11 +10,16 @@ add_flang_library(FIROptimizer
   Support/InternalNames.cpp
   Support/KindMapping.cpp
 
+  CodeGen/CGOps.cpp
+  CodeGen/PreCGRewrite.cpp
+
   Transforms/Inliner.cpp
 
   DEPENDS
   FIROpsIncGen
+  FIROptCodeGenPassIncGen
   FIROptTransformsPassIncGen
+  CGOpsIncGen
   ${dialect_libs}
 
   LINK_LIBS
diff --git a/flang/lib/Optimizer/CodeGen/CGOps.cpp b/flang/lib/Optimizer/CodeGen/CGOps.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..527066ec5ccd22ca72a8c0c526c04859aa3e0c7d
--- /dev/null
+++ b/flang/lib/Optimizer/CodeGen/CGOps.cpp
@@ -0,0 +1,64 @@
+//===-- CGOps.cpp -- FIR codegen operations -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "CGOps.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+
+/// FIR codegen dialect constructor.
+fir::FIRCodeGenDialect::FIRCodeGenDialect(mlir::MLIRContext *ctx)
+    : mlir::Dialect("fircg", ctx, mlir::TypeID::get<FIRCodeGenDialect>()) {
+  addOperations<
+#define GET_OP_LIST
+#include "flang/Optimizer/CodeGen/CGOps.cpp.inc"
+      >();
+}
+
+// anchor the class vtable to this compilation unit
+fir::FIRCodeGenDialect::~FIRCodeGenDialect() {
+  // do nothing
+}
+
+#define GET_OP_CLASSES
+#include "flang/Optimizer/CodeGen/CGOps.cpp.inc"
+
+unsigned fir::cg::XEmboxOp::getOutRank() {
+  if (slice().empty())
+    return getRank();
+  auto outRank = fir::SliceOp::getOutputRank(slice());
+  assert(outRank >= 1);
+  return outRank;
+}
+
+unsigned fir::cg::XReboxOp::getOutRank() {
+  if (auto seqTy =
+          fir::dyn_cast_ptrOrBoxEleTy(getType()).dyn_cast<fir::SequenceType>())
+    return seqTy.getDimension();
+  return 0;
+}
+
+unsigned fir::cg::XReboxOp::getRank() {
+  if (auto seqTy = fir::dyn_cast_ptrOrBoxEleTy(box().getType())
+                       .dyn_cast<fir::SequenceType>())
+    return seqTy.getDimension();
+  return 0;
+}
+
+unsigned fir::cg::XArrayCoorOp::getRank() {
+  auto memrefTy = memref().getType();
+  if (memrefTy.isa<fir::BoxType>())
+    if (auto seqty =
+            fir::dyn_cast_ptrOrBoxEleTy(memrefTy).dyn_cast<fir::SequenceType>())
+      return seqty.getDimension();
+  return shape().size();
+}
diff --git a/flang/lib/Optimizer/CodeGen/CGOps.h b/flang/lib/Optimizer/CodeGen/CGOps.h
new file mode 100644
index 0000000000000000000000000000000000000000..f5f552c633762f8ab3bb0646860a3024dbd739e4
--- /dev/null
+++ b/flang/lib/Optimizer/CodeGen/CGOps.h
@@ -0,0 +1,24 @@
+//===-- CGOps.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPTIMIZER_CODEGEN_CGOPS_H
+#define OPTIMIZER_CODEGEN_CGOPS_H
+
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
+
+using namespace mlir;
+
+#define GET_OP_CLASSES
+#include "flang/Optimizer/CodeGen/CGOps.h.inc"
+
+#endif
diff --git a/flang/lib/Optimizer/CodeGen/PassDetail.h b/flang/lib/Optimizer/CodeGen/PassDetail.h
new file mode 100644
index 0000000000000000000000000000000000000000..f7030131beff901951961f7b28837a6884ff9c3d
--- /dev/null
+++ b/flang/lib/Optimizer/CodeGen/PassDetail.h
@@ -0,0 +1,26 @@
+//===- PassDetail.h - Optimizer code gen Pass class details -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPTMIZER_CODEGEN_PASSDETAIL_H
+#define OPTMIZER_CODEGEN_PASSDETAIL_H
+
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/IR/BuiltinDialect.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassRegistry.h"
+
+namespace fir {
+
+#define GEN_PASS_CLASSES
+#include "flang/Optimizer/CodeGen/CGPasses.h.inc"
+
+} // namespace fir
+
+#endif // OPTMIZER_CODEGEN_PASSDETAIL_H
diff --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..37c6e43cb7acd01c1be9597d4e934985f1f92ed0
--- /dev/null
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -0,0 +1,263 @@
+//===-- PreCGRewrite.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "CGOps.h"
+#include "PassDetail.h"
+#include "flang/Optimizer/CodeGen/CodeGen.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Optimizer/Support/FIRContext.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "llvm/ADT/STLExtras.h"
+
+//===----------------------------------------------------------------------===//
+// Codegen rewrite: rewriting of subgraphs of ops
+//===----------------------------------------------------------------------===//
+
+using namespace fir;
+
+#define DEBUG_TYPE "flang-codegen-rewrite"
+
+static void populateShape(llvm::SmallVectorImpl<mlir::Value> &vec,
+                          ShapeOp shape) {
+  vec.append(shape.extents().begin(), shape.extents().end());
+}
+
+// Operands of fir.shape_shift split into two vectors.
+static void populateShapeAndShift(llvm::SmallVectorImpl<mlir::Value> &shapeVec,
+                                  llvm::SmallVectorImpl<mlir::Value> &shiftVec,
+                                  ShapeShiftOp shift) {
+  auto endIter = shift.pairs().end();
+  for (auto i = shift.pairs().begin(); i != endIter;) {
+    shiftVec.push_back(*i++);
+    shapeVec.push_back(*i++);
+  }
+}
+
+static void populateShift(llvm::SmallVectorImpl<mlir::Value> &vec,
+                          ShiftOp shift) {
+  vec.append(shift.origins().begin(), shift.origins().end());
+}
+
+namespace {
+
+/// Convert fir.embox to the extended form where necessary.
+///
+/// The embox operation can take arguments that specify multidimensional array
+/// properties at runtime. These properties may be shared between distinct
+/// objects that have the same properties. Before we lower these small DAGs to
+/// LLVM-IR, we gather all the information into a single extended operation. For
+/// example,
+/// ```
+/// %1 = fir.shape_shift %4, %5 : (index, index) -> !fir.shapeshift<1>
+/// %2 = fir.slice %6, %7, %8 : (index, index, index) -> !fir.slice<1>
+/// %3 = fir.embox %0 (%1) [%2] : (!fir.ref<!fir.array<?xi32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xi32>>
+/// ```
+/// can be rewritten as
+/// ```
+/// %1 = fircg.ext_embox %0(%5) origin %4[%6, %7, %8] : (!fir.ref<!fir.array<?xi32>>, index, index, index, index, index) -> !fir.box<!fir.array<?xi32>>
+/// ```
+class EmboxConversion : public mlir::OpRewritePattern<EmboxOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(EmboxOp embox,
+                  mlir::PatternRewriter &rewriter) const override {
+    auto shapeVal = embox.getShape();
+    // If the embox does not include a shape, then do not convert it
+    if (shapeVal)
+      return rewriteDynamicShape(embox, rewriter, shapeVal);
+    if (auto boxTy = embox.getType().dyn_cast<BoxType>())
+      if (auto seqTy = boxTy.getEleTy().dyn_cast<SequenceType>())
+        if (seqTy.hasConstantShape())
+          return rewriteStaticShape(embox, rewriter, seqTy);
+    return mlir::failure();
+  }
+
+  mlir::LogicalResult rewriteStaticShape(EmboxOp embox,
+                                         mlir::PatternRewriter &rewriter,
+                                         SequenceType seqTy) const {
+    auto loc = embox.getLoc();
+    llvm::SmallVector<mlir::Value> shapeOpers;
+    auto idxTy = rewriter.getIndexType();
+    for (auto ext : seqTy.getShape()) {
+      auto iAttr = rewriter.getIndexAttr(ext);
+      auto extVal = rewriter.create<mlir::ConstantOp>(loc, idxTy, iAttr);
+      shapeOpers.push_back(extVal);
+    }
+    auto xbox = rewriter.create<cg::XEmboxOp>(
+        loc, embox.getType(), embox.memref(), shapeOpers, llvm::None,
+        llvm::None, llvm::None, embox.lenParams());
+    LLVM_DEBUG(llvm::dbgs() << "rewriting " << embox << " to " << xbox << '\n');
+    rewriter.replaceOp(embox, xbox.getOperation()->getResults());
+    return mlir::success();
+  }
+
+  mlir::LogicalResult rewriteDynamicShape(EmboxOp embox,
+                                          mlir::PatternRewriter &rewriter,
+                                          mlir::Value shapeVal) const {
+    auto loc = embox.getLoc();
+    auto shapeOp = dyn_cast<ShapeOp>(shapeVal.getDefiningOp());
+    llvm::SmallVector<mlir::Value> shapeOpers;
+    llvm::SmallVector<mlir::Value> shiftOpers;
+    if (shapeOp) {
+      populateShape(shapeOpers, shapeOp);
+    } else {
+      auto shiftOp = dyn_cast<ShapeShiftOp>(shapeVal.getDefiningOp());
+      assert(shiftOp && "shape is neither fir.shape nor fir.shape_shift");
+      populateShapeAndShift(shapeOpers, shiftOpers, shiftOp);
+    }
+    llvm::SmallVector<mlir::Value> sliceOpers;
+    llvm::SmallVector<mlir::Value> subcompOpers;
+    if (auto s = embox.getSlice())
+      if (auto sliceOp = dyn_cast_or_null<SliceOp>(s.getDefiningOp())) {
+        sliceOpers.append(sliceOp.triples().begin(), sliceOp.triples().end());
+        subcompOpers.append(sliceOp.fields().begin(), sliceOp.fields().end());
+      }
+    auto xbox = rewriter.create<cg::XEmboxOp>(
+        loc, embox.getType(), embox.memref(), shapeOpers, shiftOpers,
+        sliceOpers, subcompOpers, embox.lenParams());
+    LLVM_DEBUG(llvm::dbgs() << "rewriting " << embox << " to " << xbox << '\n');
+    rewriter.replaceOp(embox, xbox.getOperation()->getResults());
+    return mlir::success();
+  }
+};
+
+/// Convert fir.rebox to the extended form where necessary.
+///
+/// For example,
+/// ```
+/// %5 = fir.rebox %3(%1) : (!fir.box<!fir.array<?xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<?xi32>>
+/// ```
+/// converted to
+/// ```
+/// %5 = fircg.ext_rebox %3(%13) origin %12 : (!fir.box<!fir.array<?xi32>>, index, index) -> !fir.box<!fir.array<?xi32>>
+/// ```
+class ReboxConversion : public mlir::OpRewritePattern<ReboxOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(ReboxOp rebox,
+                  mlir::PatternRewriter &rewriter) const override {
+    auto loc = rebox.getLoc();
+    llvm::SmallVector<mlir::Value> shapeOpers;
+    llvm::SmallVector<mlir::Value> shiftOpers;
+    if (auto shapeVal = rebox.shape()) {
+      if (auto shapeOp = dyn_cast<ShapeOp>(shapeVal.getDefiningOp()))
+        populateShape(shapeOpers, shapeOp);
+      else if (auto shiftOp = dyn_cast<ShapeShiftOp>(shapeVal.getDefiningOp()))
+        populateShapeAndShift(shapeOpers, shiftOpers, shiftOp);
+      else if (auto shiftOp = dyn_cast<ShiftOp>(shapeVal.getDefiningOp()))
+        populateShift(shiftOpers, shiftOp);
+      else
+        return mlir::failure();
+    }
+    llvm::SmallVector<mlir::Value> sliceOpers;
+    llvm::SmallVector<mlir::Value> subcompOpers;
+    if (auto s = rebox.slice())
+      if (auto sliceOp = dyn_cast_or_null<SliceOp>(s.getDefiningOp())) {
+        sliceOpers.append(sliceOp.triples().begin(), sliceOp.triples().end());
+        subcompOpers.append(sliceOp.fields().begin(), sliceOp.fields().end());
+      }
+
+    auto xRebox = rewriter.create<cg::XReboxOp>(
+        loc, rebox.getType(), rebox.box(), shapeOpers, shiftOpers, sliceOpers,
+        subcompOpers);
+    LLVM_DEBUG(llvm::dbgs()
+               << "rewriting " << rebox << " to " << xRebox << '\n');
+    rewriter.replaceOp(rebox, xRebox.getOperation()->getResults());
+    return mlir::success();
+  }
+};
+
+/// Convert all fir.array_coor to the extended form.
+///
+/// For example,
+/// ```
+///  %4 = fir.array_coor %addr (%1) [%2] %0 : (!fir.ref<!fir.array<?xi32>>, !fir.shapeshift<1>, !fir.slice<1>, index) -> !fir.ref<i32>
+/// ```
+/// converted to
+/// ```
+/// %40 = fircg.ext_array_coor %addr(%9) origin %8[%4, %5, %6<%39> : (!fir.ref<!fir.array<?xi32>>, index, index, index, index, index, index) -> !fir.ref<i32>
+/// ```
+class ArrayCoorConversion : public mlir::OpRewritePattern<ArrayCoorOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(ArrayCoorOp arrCoor,
+                  mlir::PatternRewriter &rewriter) const override {
+    auto loc = arrCoor.getLoc();
+    llvm::SmallVector<mlir::Value> shapeOpers;
+    llvm::SmallVector<mlir::Value> shiftOpers;
+    if (auto shapeVal = arrCoor.shape()) {
+      if (auto shapeOp = dyn_cast<ShapeOp>(shapeVal.getDefiningOp()))
+        populateShape(shapeOpers, shapeOp);
+      else if (auto shiftOp = dyn_cast<ShapeShiftOp>(shapeVal.getDefiningOp()))
+        populateShapeAndShift(shapeOpers, shiftOpers, shiftOp);
+      else if (auto shiftOp = dyn_cast<ShiftOp>(shapeVal.getDefiningOp()))
+        populateShift(shiftOpers, shiftOp);
+      else
+        return mlir::failure();
+    }
+    llvm::SmallVector<mlir::Value> sliceOpers;
+    llvm::SmallVector<mlir::Value> subcompOpers;
+    if (auto s = arrCoor.slice())
+      if (auto sliceOp = dyn_cast_or_null<SliceOp>(s.getDefiningOp())) {
+        sliceOpers.append(sliceOp.triples().begin(), sliceOp.triples().end());
+        subcompOpers.append(sliceOp.fields().begin(), sliceOp.fields().end());
+      }
+    auto xArrCoor = rewriter.create<cg::XArrayCoorOp>(
+        loc, arrCoor.getType(), arrCoor.memref(), shapeOpers, shiftOpers,
+        sliceOpers, subcompOpers, arrCoor.indices(), arrCoor.lenParams());
+    LLVM_DEBUG(llvm::dbgs()
+               << "rewriting " << arrCoor << " to " << xArrCoor << '\n');
+    rewriter.replaceOp(arrCoor, xArrCoor.getOperation()->getResults());
+    return mlir::success();
+  }
+};
+
+class CodeGenRewrite : public CodeGenRewriteBase<CodeGenRewrite> {
+public:
+  void runOnOperation() override final {
+    auto op = getOperation();
+    auto &context = getContext();
+    mlir::OpBuilder rewriter(&context);
+    mlir::ConversionTarget target(context);
+    target.addLegalDialect<FIROpsDialect, FIRCodeGenDialect,
+                           mlir::StandardOpsDialect>();
+    target.addIllegalOp<ArrayCoorOp>();
+    target.addIllegalOp<ReboxOp>();
+    target.addDynamicallyLegalOp<EmboxOp>([](EmboxOp embox) {
+      return !(embox.getShape() ||
+               embox.getType().cast<BoxType>().getEleTy().isa<SequenceType>());
+    });
+    mlir::OwningRewritePatternList patterns(&context);
+    patterns.insert<EmboxConversion, ArrayCoorConversion, ReboxConversion>(
+        &context);
+    if (mlir::failed(
+            mlir::applyPartialConversion(op, target, std::move(patterns)))) {
+      mlir::emitError(mlir::UnknownLoc::get(&context),
+                      "error in running the pre-codegen conversions");
+      signalPassFailure();
+    }
+  }
+};
+
+} // namespace
+
+std::unique_ptr<mlir::Pass> fir::createFirCodeGenRewritePass() {
+  return std::make_unique<CodeGenRewrite>();
+}
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index f6ec7bb2cd999a92a9e63461fbd16b2235426736..38390d8011347571376fc616e50814e4442f1281 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -68,15 +68,6 @@ static bool verifyRecordLenParams(mlir::Type inType, unsigned numLenParams) {
   return false;
 }
 
-//===----------------------------------------------------------------------===//
-// AddfOp
-//===----------------------------------------------------------------------===//
-
-mlir::OpFoldResult fir::AddfOp::fold(llvm::ArrayRef<mlir::Attribute> opnds) {
-  return mlir::constFoldBinaryOp<FloatAttr>(
-      opnds, [](APFloat a, APFloat b) { return a + b; });
-}
-
 //===----------------------------------------------------------------------===//
 // AllocaOp
 //===----------------------------------------------------------------------===//
@@ -706,7 +697,7 @@ static bool isOne(mlir::Value v) { return checkIsIntegerConstant(v, 1); }
 template <typename FltOp, typename CpxOp>
 struct UndoComplexPattern : public mlir::RewritePattern {
   UndoComplexPattern(mlir::MLIRContext *ctx)
-      : mlir::RewritePattern("fir.insert_value", {}, 2, ctx) {}
+      : mlir::RewritePattern("fir.insert_value", 2, ctx) {}
 
   mlir::LogicalResult
   matchAndRewrite(mlir::Operation *op,
@@ -746,8 +737,8 @@ struct UndoComplexPattern : public mlir::RewritePattern {
 
 void fir::InsertValueOp::getCanonicalizationPatterns(
     mlir::OwningRewritePatternList &results, mlir::MLIRContext *context) {
-  results.insert<UndoComplexPattern<fir::AddfOp, fir::AddcOp>,
-                 UndoComplexPattern<fir::SubfOp, fir::SubcOp>>(context);
+  results.insert<UndoComplexPattern<mlir::AddFOp, fir::AddcOp>,
+                 UndoComplexPattern<mlir::SubFOp, fir::SubcOp>>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1227,15 +1218,6 @@ mlir::Value fir::DoLoopOp::blockArgToSourceOp(unsigned blockArgNum) {
   return {};
 }
 
-//===----------------------------------------------------------------------===//
-// MulfOp
-//===----------------------------------------------------------------------===//
-
-mlir::OpFoldResult fir::MulfOp::fold(llvm::ArrayRef<mlir::Attribute> opnds) {
-  return mlir::constFoldBinaryOp<FloatAttr>(
-      opnds, [](APFloat a, APFloat b) { return a * b; });
-}
-
 //===----------------------------------------------------------------------===//
 // ReboxOp
 //===----------------------------------------------------------------------===//
@@ -1761,15 +1743,6 @@ bool fir::StringLitOp::isWideValue() {
   return eleTy.cast<fir::CharacterType>().getFKind() != 1;
 }
 
-//===----------------------------------------------------------------------===//
-// SubfOp
-//===----------------------------------------------------------------------===//
-
-mlir::OpFoldResult fir::SubfOp::fold(llvm::ArrayRef<mlir::Attribute> opnds) {
-  return mlir::constFoldBinaryOp<FloatAttr>(
-      opnds, [](APFloat a, APFloat b) { return a - b; });
-}
-
 //===----------------------------------------------------------------------===//
 // IfOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Parser/parsing.cpp b/flang/lib/Parser/parsing.cpp
index 81097b2d08d1d6f35049ca904dc2f1ca619e8484..0afa2a94ac40c7fb36735beedd47b36e5cfac0db 100644
--- a/flang/lib/Parser/parsing.cpp
+++ b/flang/lib/Parser/parsing.cpp
@@ -88,7 +88,7 @@ const SourceFile *Parsing::Prescan(const std::string &path, Options options) {
     // message about nonstandard usage will have provenance.
     currentCooked_->Put('\n', range.start());
   }
-  currentCooked_->Marshal(allSources);
+  currentCooked_->Marshal(allCooked_);
   if (options.needProvenanceRangeToCharBlockMappings) {
     currentCooked_->CompileProvenanceRangeToOffsetMappings(allSources);
   }
diff --git a/flang/lib/Parser/provenance.cpp b/flang/lib/Parser/provenance.cpp
index 14124a546d1e39ba6fd4c133ddf9921595a20dfe..2aa1a97ce5570db579df9c8c6c3c79f10753b88c 100644
--- a/flang/lib/Parser/provenance.cpp
+++ b/flang/lib/Parser/provenance.cpp
@@ -442,11 +442,13 @@ std::optional<CharBlock> CookedSource::GetCharBlock(
 
 std::size_t CookedSource::BufferedBytes() const { return buffer_.bytes(); }
 
-void CookedSource::Marshal(AllSources &allSources) {
+void CookedSource::Marshal(AllCookedSources &allCookedSources) {
   CHECK(provenanceMap_.SizeInBytes() == buffer_.bytes());
-  provenanceMap_.Put(allSources.AddCompilerInsertion("(after end of source)"));
+  provenanceMap_.Put(allCookedSources.allSources().AddCompilerInsertion(
+      "(after end of source)"));
   data_ = buffer_.Marshal();
   buffer_.clear();
+  allCookedSources.Register(*this);
 }
 
 void CookedSource::CompileProvenanceRangeToOffsetMappings(
@@ -534,6 +536,16 @@ CookedSource &AllCookedSources::NewCookedSource() {
   return cooked_.emplace_back();
 }
 
+const CookedSource *AllCookedSources::Find(CharBlock x) const {
+  auto pair{index_.equal_range(x)};
+  for (auto iter{pair.first}; iter != pair.second; ++iter) {
+    if (iter->second.AsCharBlock().Contains(x)) {
+      return &iter->second;
+    }
+  }
+  return nullptr;
+}
+
 std::optional<ProvenanceRange> AllCookedSources::GetProvenanceRange(
     CharBlock cb) const {
   if (const CookedSource * c{Find(cb)}) {
@@ -589,4 +601,26 @@ void AllCookedSources::Dump(llvm::raw_ostream &o) const {
   }
 }
 
+bool AllCookedSources::Precedes(CharBlock x, CharBlock y) const {
+  if (const CookedSource * xSource{Find(x)}) {
+    if (xSource->AsCharBlock().Contains(y)) {
+      return x.begin() < y.begin();
+    } else if (const CookedSource * ySource{Find(y)}) {
+      return xSource->number() < ySource->number();
+    } else {
+      return true; // by fiat, all cooked source < anything outside
+    }
+  } else if (Find(y)) {
+    return false;
+  } else {
+    // Both names are compiler-created (SaveTempName).
+    return x < y;
+  }
+}
+
+void AllCookedSources::Register(CookedSource &cooked) {
+  index_.emplace(cooked.AsCharBlock(), cooked);
+  cooked.set_number(static_cast<int>(index_.size()));
+}
+
 } // namespace Fortran::parser
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 8adcc32b87afc47cd51c27e62093c61952864b73..eaa4c926068c3dc9d643310619f1f98c30cf016e 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -16,6 +16,7 @@
 #include "flang/Parser/characters.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cinttypes>
@@ -48,6 +49,14 @@ public:
       Unparse(x);
       Post(x);
       return false; // Walk() does not visit descendents
+    } else if constexpr (HasTypedExpr<T>::value) {
+      // Format the expression representation from semantics
+      if (asFortran_ && x.typedExpr) {
+        asFortran_->expr(out_, *x.typedExpr);
+        return false;
+      } else {
+        return true;
+      }
     } else {
       Before(x);
       return true; // there's no Unparse() defined here, Walk() the descendents
@@ -816,15 +825,6 @@ public:
   }
 
   // R1001 - R1022
-  bool Pre(const Expr &x) {
-    if (asFortran_ && x.typedExpr) {
-      // Format the expression representation from semantics
-      asFortran_->expr(out_, *x.typedExpr);
-      return false;
-    } else {
-      return true;
-    }
-  }
   void Unparse(const Expr::Parentheses &x) { Put('('), Walk(x.v), Put(')'); }
   void Before(const Expr::UnaryPlus &) { Put("+"); }
   void Before(const Expr::Negate &) { Put("-"); }
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index 4bab4b16149db8f452a7cc24bc0a71ff92813eef..9e7c07b9c55faf4f88fa3712125bef670c9bfc19 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -1,4 +1,3 @@
-
 add_flang_library(FortranSemantics
   assignment.cpp
   attr.cpp
diff --git a/flang/lib/Semantics/check-deallocate.cpp b/flang/lib/Semantics/check-deallocate.cpp
index 92e197bbb322e63a55e255a284f7f068a866cd2b..03c2d6ebdddacd53b59de043a152fdc7c0949ca8 100644
--- a/flang/lib/Semantics/check-deallocate.cpp
+++ b/flang/lib/Semantics/check-deallocate.cpp
@@ -34,8 +34,9 @@ void DeallocateChecker::Leave(const parser::DeallocateStmt &deallocateStmt) {
               }
             },
             [&](const parser::StructureComponent &structureComponent) {
-              evaluate::ExpressionAnalyzer analyzer{context_};
-              if (MaybeExpr checked{analyzer.Analyze(structureComponent)}) {
+              // Only perform structureComponent checks it was successfully
+              // analyzed in expression analysis.
+              if (GetExpr(allocateObject)) {
                 if (!IsAllocatableOrPointer(
                         *structureComponent.component.symbol)) { // C932
                   context_.Say(structureComponent.component.source,
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index ebc0bcf606b52b8e53faafad0dbc9e4d22c5ff6a..69607c466e162b39af2660dd07640330448452b5 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -110,7 +110,8 @@ private:
   // that has a symbol.
   const Symbol *innermostSymbol_{nullptr};
   // Cache of calls to Procedure::Characterize(Symbol)
-  std::map<SymbolRef, std::optional<Procedure>> characterizeCache_;
+  std::map<SymbolRef, std::optional<Procedure>, SymbolAddressCompare>
+      characterizeCache_;
 };
 
 class DistinguishabilityHelper {
@@ -1686,24 +1687,23 @@ void SubprogramMatchHelper::Check(
             : "Module subprogram '%s' does not have NON_RECURSIVE prefix but "
               "the corresponding interface body does"_err_en_US);
   }
-  MaybeExpr bindName1{details1.bindName()};
-  MaybeExpr bindName2{details2.bindName()};
-  if (bindName1.has_value() != bindName2.has_value()) {
+  const std::string *bindName1{details1.bindName()};
+  const std::string *bindName2{details2.bindName()};
+  if (!bindName1 && !bindName2) {
+    // OK - neither has a binding label
+  } else if (!bindName1) {
     Say(symbol1, symbol2,
-        bindName1.has_value()
-            ? "Module subprogram '%s' has a binding label but the corresponding"
-              " interface body does not"_err_en_US
-            : "Module subprogram '%s' does not have a binding label but the"
-              " corresponding interface body does"_err_en_US);
-  } else if (bindName1) {
-    std::string string1{bindName1->AsFortran()};
-    std::string string2{bindName2->AsFortran()};
-    if (string1 != string2) {
-      Say(symbol1, symbol2,
-          "Module subprogram '%s' has binding label %s but the corresponding"
-          " interface body has %s"_err_en_US,
-          string1, string2);
-    }
+        "Module subprogram '%s' does not have a binding label but the"
+        " corresponding interface body does"_err_en_US);
+  } else if (!bindName2) {
+    Say(symbol1, symbol2,
+        "Module subprogram '%s' has a binding label but the"
+        " corresponding interface body does not"_err_en_US);
+  } else if (*bindName1 != *bindName2) {
+    Say(symbol1, symbol2,
+        "Module subprogram '%s' has binding label '%s' but the corresponding"
+        " interface body has '%s'"_err_en_US,
+        *details1.bindName(), *details2.bindName());
   }
   const Procedure *proc1{checkHelper.Characterize(symbol1)};
   const Procedure *proc2{checkHelper.Characterize(symbol2)};
diff --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index d2f55eed539cf15b513a50dc1534936e9ecbaf95..1532dea61ac58604ff466971b0e40bde83149844 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -548,9 +548,9 @@ private:
   // the names up in the scope that encloses the DO construct to avoid getting
   // the local versions of them.  Then follow the host-, use-, and
   // construct-associations to get the root symbols
-  SymbolSet GatherLocals(
+  UnorderedSymbolSet GatherLocals(
       const std::list<parser::LocalitySpec> &localitySpecs) const {
-    SymbolSet symbols;
+    UnorderedSymbolSet symbols;
     const Scope &parentScope{
         context_.FindScope(currentStatementSourcePosition_).parent()};
     // Loop through the LocalitySpec::Local locality-specs
@@ -568,8 +568,9 @@ private:
     return symbols;
   }
 
-  static SymbolSet GatherSymbolsFromExpression(const parser::Expr &expression) {
-    SymbolSet result;
+  static UnorderedSymbolSet GatherSymbolsFromExpression(
+      const parser::Expr &expression) {
+    UnorderedSymbolSet result;
     if (const auto *expr{GetExpr(expression)}) {
       for (const Symbol &symbol : evaluate::CollectSymbols(*expr)) {
         result.insert(ResolveAssociations(symbol));
@@ -580,8 +581,9 @@ private:
 
   // C1121 - procedures in mask must be pure
   void CheckMaskIsPure(const parser::ScalarLogicalExpr &mask) const {
-    SymbolSet references{GatherSymbolsFromExpression(mask.thing.thing.value())};
-    for (const Symbol &ref : references) {
+    UnorderedSymbolSet references{
+        GatherSymbolsFromExpression(mask.thing.thing.value())};
+    for (const Symbol &ref : OrderBySourcePosition(references)) {
       if (IsProcedure(ref) && !IsPureProcedure(ref)) {
         context_.SayWithDecl(ref, parser::Unwrap<parser::Expr>(mask)->source,
             "%s mask expression may not reference impure procedure '%s'"_err_en_US,
@@ -591,10 +593,10 @@ private:
     }
   }
 
-  void CheckNoCollisions(const SymbolSet &refs, const SymbolSet &uses,
-      parser::MessageFixedText &&errorMessage,
+  void CheckNoCollisions(const UnorderedSymbolSet &refs,
+      const UnorderedSymbolSet &uses, parser::MessageFixedText &&errorMessage,
       const parser::CharBlock &refPosition) const {
-    for (const Symbol &ref : refs) {
+    for (const Symbol &ref : OrderBySourcePosition(refs)) {
       if (uses.find(ref) != uses.end()) {
         context_.SayWithDecl(ref, refPosition, std::move(errorMessage),
             LoopKindName(), ref.name());
@@ -603,8 +605,8 @@ private:
     }
   }
 
-  void HasNoReferences(
-      const SymbolSet &indexNames, const parser::ScalarIntExpr &expr) const {
+  void HasNoReferences(const UnorderedSymbolSet &indexNames,
+      const parser::ScalarIntExpr &expr) const {
     CheckNoCollisions(GatherSymbolsFromExpression(expr.thing.thing.value()),
         indexNames,
         "%s limit expression may not reference index variable '%s'"_err_en_US,
@@ -612,8 +614,8 @@ private:
   }
 
   // C1129, names in local locality-specs can't be in mask expressions
-  void CheckMaskDoesNotReferenceLocal(
-      const parser::ScalarLogicalExpr &mask, const SymbolSet &localVars) const {
+  void CheckMaskDoesNotReferenceLocal(const parser::ScalarLogicalExpr &mask,
+      const UnorderedSymbolSet &localVars) const {
     CheckNoCollisions(GatherSymbolsFromExpression(mask.thing.thing.value()),
         localVars,
         "%s mask expression references variable '%s'"
@@ -623,8 +625,8 @@ private:
 
   // C1129, names in local locality-specs can't be in limit or step
   // expressions
-  void CheckExprDoesNotReferenceLocal(
-      const parser::ScalarIntExpr &expr, const SymbolSet &localVars) const {
+  void CheckExprDoesNotReferenceLocal(const parser::ScalarIntExpr &expr,
+      const UnorderedSymbolSet &localVars) const {
     CheckNoCollisions(GatherSymbolsFromExpression(expr.thing.thing.value()),
         localVars,
         "%s expression references variable '%s'"
@@ -663,7 +665,7 @@ private:
       CheckMaskIsPure(*mask);
     }
     auto &controls{std::get<std::list<parser::ConcurrentControl>>(header.t)};
-    SymbolSet indexNames;
+    UnorderedSymbolSet indexNames;
     for (const parser::ConcurrentControl &control : controls) {
       const auto &indexName{std::get<parser::Name>(control.t)};
       if (indexName.symbol) {
@@ -697,7 +699,7 @@ private:
     const auto &localitySpecs{
         std::get<std::list<parser::LocalitySpec>>(concurrent.t)};
     if (!localitySpecs.empty()) {
-      const SymbolSet &localVars{GatherLocals(localitySpecs)};
+      const UnorderedSymbolSet &localVars{GatherLocals(localitySpecs)};
       for (const auto &c : GetControls(control)) {
         CheckExprDoesNotReferenceLocal(std::get<1>(c.t), localVars);
         CheckExprDoesNotReferenceLocal(std::get<2>(c.t), localVars);
@@ -733,7 +735,7 @@ private:
   void CheckForallIndexesUsed(const evaluate::Assignment &assignment) {
     SymbolVector indexVars{context_.GetIndexVars(IndexVarKind::FORALL)};
     if (!indexVars.empty()) {
-      SymbolSet symbols{evaluate::CollectSymbols(assignment.lhs)};
+      UnorderedSymbolSet symbols{evaluate::CollectSymbols(assignment.lhs)};
       std::visit(
           common::visitors{
               [&](const evaluate::Assignment::BoundsSpec &spec) {
diff --git a/flang/lib/Semantics/check-io.cpp b/flang/lib/Semantics/check-io.cpp
index c6b67a5046e0a9036bc1574e92a817d0c9ef84af..897b7fd6570a962a39e8d537c95f5fc6e4d48a87 100644
--- a/flang/lib/Semantics/check-io.cpp
+++ b/flang/lib/Semantics/check-io.cpp
@@ -930,7 +930,8 @@ void IoChecker::CheckForDefinableVariable(
   if (const auto *var{parser::Unwrap<parser::Variable>(variable)}) {
     if (auto expr{AnalyzeExpr(context_, *var)}) {
       auto at{var->GetSource()};
-      if (auto whyNot{WhyNotModifiable(at, *expr, context_.FindScope(at))}) {
+      if (auto whyNot{WhyNotModifiable(at, *expr, context_.FindScope(at),
+              true /*vectorSubscriptIsOk*/)}) {
         const Symbol *base{GetFirstSymbol(*expr)};
         context_
             .Say(at, "%s variable '%s' must be definable"_err_en_US, s,
diff --git a/flang/lib/Semantics/check-nullify.cpp b/flang/lib/Semantics/check-nullify.cpp
index ff49a661e206e058faaf3d349e8a9e2085dfbf6a..4c6e78e7f7e3e789574de59832a2024c48df8910 100644
--- a/flang/lib/Semantics/check-nullify.cpp
+++ b/flang/lib/Semantics/check-nullify.cpp
@@ -40,13 +40,12 @@ void NullifyChecker::Leave(const parser::NullifyStmt &nullifyStmt) {
               }
             },
             [&](const parser::StructureComponent &structureComponent) {
-              evaluate::ExpressionAnalyzer analyzer{context_};
-              if (MaybeExpr checked{analyzer.Analyze(structureComponent)}) {
+              if (const auto *checkedExpr{GetExpr(pointerObject)}) {
                 if (!IsPointer(*structureComponent.component.symbol)) { // C951
                   messages.Say(structureComponent.component.source,
                       "component in NULLIFY statement must have the POINTER attribute"_err_en_US);
                 } else if (pure) {
-                  if (const Symbol * symbol{GetFirstSymbol(checked)}) {
+                  if (const Symbol * symbol{GetFirstSymbol(*checkedExpr)}) {
                     CheckDefinabilityInPureScope(
                         messages, *symbol, scope, *pure);
                   }
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index b23ae51b509481aa6b289e52c1bd3b8badb6e46b..3ed86132cbea20637dd06d7e0c1f1bb6d2f9606b 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -343,12 +343,22 @@ void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
 void OmpStructureChecker::CheckIfDoOrderedClause(
     const parser::OmpBlockDirective &blkDirective) {
   if (blkDirective.v == llvm::omp::OMPD_ordered) {
-    if (!FindClauseParent(llvm::omp::Clause::OMPC_ordered)) {
+    // Loops
+    if (llvm::omp::doSet.test(GetContextParent().directive) &&
+        !FindClauseParent(llvm::omp::Clause::OMPC_ordered)) {
       context_.Say(blkDirective.source,
           "The ORDERED clause must be present on the loop"
           " construct if any ORDERED region ever binds"
           " to a loop region arising from the loop construct."_err_en_US);
     }
+    // Other disallowed nestings, these directives do not support
+    // ordered clause in them, so no need to check
+    else if (llvm::omp::nestedOrderedErrSet.test(
+                 GetContextParent().directive)) {
+      context_.Say(blkDirective.source,
+          "`ORDERED` region may not be closely nested inside of "
+          "`CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region."_err_en_US);
+    }
   }
 }
 
@@ -630,7 +640,7 @@ void OmpStructureChecker::Leave(const parser::OmpClauseList &) {
       }
     }
     // A list-item cannot appear in more than one aligned clause
-    semantics::SymbolSet alignedVars;
+    semantics::UnorderedSymbolSet alignedVars;
     auto clauseAll = FindClauses(llvm::omp::Clause::OMPC_aligned);
     for (auto itr = clauseAll.first; itr != clauseAll.second; ++itr) {
       const auto &alignedClause{
@@ -717,6 +727,8 @@ CHECK_SIMPLE_CLAUSE(UsesAllocators, OMPC_uses_allocators)
 CHECK_SIMPLE_CLAUSE(Update, OMPC_update)
 CHECK_SIMPLE_CLAUSE(UseDeviceAddr, OMPC_use_device_addr)
 CHECK_SIMPLE_CLAUSE(Write, OMPC_write)
+CHECK_SIMPLE_CLAUSE(Init, OMPC_init)
+CHECK_SIMPLE_CLAUSE(Use, OMPC_use)
 
 CHECK_REQ_SCALAR_INT_CLAUSE(Allocator, OMPC_allocator)
 CHECK_REQ_SCALAR_INT_CLAUSE(Grainsize, OMPC_grainsize)
@@ -840,7 +852,6 @@ void OmpStructureChecker::CheckReductionArraySection(
 
 void OmpStructureChecker::CheckMultipleAppearanceAcrossContext(
     const parser::OmpObjectList &redObjectList) {
-  const parser::OmpObjectList *objList{nullptr};
   //  TODO: Verify the assumption here that the immediately enclosing region is
   //  the parallel region to which the worksharing construct having reduction
   //  binds to.
@@ -848,43 +859,29 @@ void OmpStructureChecker::CheckMultipleAppearanceAcrossContext(
     for (auto it : enclosingContext->clauseInfo) {
       llvmOmpClause type = it.first;
       const auto *clause = it.second;
-      if (type == llvm::omp::Clause::OMPC_private) {
-        const auto &pClause{std::get<parser::OmpClause::Private>(clause->u)};
-        objList = &pClause.v;
-      } else if (type == llvm::omp::Clause::OMPC_firstprivate) {
-        const auto &fpClause{
-            std::get<parser::OmpClause::Firstprivate>(clause->u)};
-        objList = &fpClause.v;
-      } else if (type == llvm::omp::Clause::OMPC_lastprivate) {
-        const auto &lpClause{
-            std::get<parser::OmpClause::Lastprivate>(clause->u)};
-        objList = &lpClause.v;
-      } else if (type == llvm::omp::Clause::OMPC_reduction) {
-        const auto &rClause{std::get<parser::OmpClause::Reduction>(clause->u)};
-        const auto &olist{std::get<1>(rClause.v.t)};
-        objList = &olist;
-      }
-      if (objList) {
-        for (const auto &ompObject : objList->v) {
-          if (const auto *name{parser::Unwrap<parser::Name>(ompObject)}) {
-            if (const auto *symbol{name->symbol}) {
-              for (const auto &redOmpObject : redObjectList.v) {
-                if (const auto *rname{
-                        parser::Unwrap<parser::Name>(redOmpObject)}) {
-                  if (const auto *rsymbol{rname->symbol}) {
-                    if (rsymbol->name() == symbol->name()) {
-                      context_.Say(GetContext().clauseSource,
-                          "%s variable '%s' is %s in outer context must"
-                          " be shared in the parallel regions to which any"
-                          " of the worksharing regions arising from the "
-                          "worksharing"
-                          " construct bind."_err_en_US,
-                          parser::ToUpperCaseLetters(
-                              getClauseName(llvm::omp::Clause::OMPC_reduction)
-                                  .str()),
-                          symbol->name(),
-                          parser::ToUpperCaseLetters(
-                              getClauseName(type).str()));
+      if (llvm::omp::privateReductionSet.test(type)) {
+        if (const auto *objList{GetOmpObjectList(*clause)}) {
+          for (const auto &ompObject : objList->v) {
+            if (const auto *name{parser::Unwrap<parser::Name>(ompObject)}) {
+              if (const auto *symbol{name->symbol}) {
+                for (const auto &redOmpObject : redObjectList.v) {
+                  if (const auto *rname{
+                          parser::Unwrap<parser::Name>(redOmpObject)}) {
+                    if (const auto *rsymbol{rname->symbol}) {
+                      if (rsymbol->name() == symbol->name()) {
+                        context_.Say(GetContext().clauseSource,
+                            "%s variable '%s' is %s in outer context must"
+                            " be shared in the parallel regions to which any"
+                            " of the worksharing regions arising from the "
+                            "worksharing"
+                            " construct bind."_err_en_US,
+                            parser::ToUpperCaseLetters(
+                                getClauseName(llvm::omp::Clause::OMPC_reduction)
+                                    .str()),
+                            symbol->name(),
+                            parser::ToUpperCaseLetters(
+                                getClauseName(type).str()));
+                      }
                     }
                   }
                 }
@@ -1213,7 +1210,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
   DirectivesClauseTriple dirClauseTriple;
   SymbolSourceMap currSymbols;
   GetSymbolsInObjectList(x.v, currSymbols);
-  CheckDefinableObjects(currSymbols, llvm::omp::Clause::OMPC_lastprivate);
+  CheckDefinableObjects(currSymbols, GetClauseKindForParserClass(x));
 
   // Check lastprivate variables in worksharing constructs
   dirClauseTriple.emplace(llvm::omp::Directive::OMPD_do,
@@ -1224,7 +1221,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
           llvm::omp::Directive::OMPD_parallel, llvm::omp::privateReductionSet));
 
   CheckPrivateSymbolsInOuterCxt(
-      currSymbols, dirClauseTriple, llvm::omp::Clause::OMPC_lastprivate);
+      currSymbols, dirClauseTriple, GetClauseKindForParserClass(x));
 }
 
 llvm::StringRef OmpStructureChecker::getClauseName(llvm::omp::Clause clause) {
@@ -1368,40 +1365,11 @@ void OmpStructureChecker::CheckPrivateSymbolsInOuterCxt(
     if (auto *enclosingContext{GetEnclosingContextWithDir(enclosingDir)}) {
       for (auto it{enclosingContext->clauseInfo.begin()};
            it != enclosingContext->clauseInfo.end(); ++it) {
-        // TODO: Replace the hard-coded clause names by using autogen checks or
-        // a function which maps parser::OmpClause::<name> to the corresponding
-        // llvm::omp::Clause::OMPC_<name>
-        std::visit(common::visitors{
-                       [&](const parser::OmpClause::Private &x) {
-                         if (enclosingClauseSet.test(
-                                 llvm::omp::Clause::OMPC_private)) {
-                           GetSymbolsInObjectList(x.v, enclosingSymbols);
-                         }
-                       },
-                       [&](const parser::OmpClause::Firstprivate &x) {
-                         if (enclosingClauseSet.test(
-                                 llvm::omp::Clause::OMPC_firstprivate)) {
-                           GetSymbolsInObjectList(x.v, enclosingSymbols);
-                         }
-                       },
-                       [&](const parser::OmpClause::Lastprivate &x) {
-                         if (enclosingClauseSet.test(
-                                 llvm::omp::Clause::OMPC_lastprivate)) {
-                           GetSymbolsInObjectList(x.v, enclosingSymbols);
-                         }
-                       },
-                       [&](const parser::OmpClause::Reduction &x) {
-                         if (enclosingClauseSet.test(
-                                 llvm::omp::Clause::OMPC_reduction)) {
-                           const auto &ompObjectList{
-                               std::get<parser::OmpObjectList>(x.v.t)};
-                           GetSymbolsInObjectList(
-                               ompObjectList, enclosingSymbols);
-                         }
-                       },
-                       [&](const auto &) {},
-                   },
-            it->second->u);
+        if (enclosingClauseSet.test(it->first)) {
+          if (const auto *ompObjectList{GetOmpObjectList(*it->second)}) {
+            GetSymbolsInObjectList(*ompObjectList, enclosingSymbols);
+          }
+        }
       }
 
       // Check if the symbols in current context are private in outer context
@@ -1497,4 +1465,37 @@ void OmpStructureChecker::CheckWorkshareBlockStmts(
   }
 }
 
+const parser::OmpObjectList *OmpStructureChecker::GetOmpObjectList(
+    const parser::OmpClause &clause) {
+
+  // Clauses with OmpObjectList as its data member
+  using MemberObjectListClauses = std::tuple<parser::OmpClause::Copyprivate,
+      parser::OmpClause::Copyin, parser::OmpClause::Firstprivate,
+      parser::OmpClause::From, parser::OmpClause::Lastprivate,
+      parser::OmpClause::Link, parser::OmpClause::Private,
+      parser::OmpClause::Shared, parser::OmpClause::To>;
+
+  // Clauses with OmpObjectList in the tuple
+  using TupleObjectListClauses = std::tuple<parser::OmpClause::Allocate,
+      parser::OmpClause::Map, parser::OmpClause::Reduction>;
+
+  // TODO:: Generate the tuples using TableGen.
+  // Handle other constructs with OmpObjectList such as OpenMPThreadprivate.
+  return std::visit(
+      common::visitors{
+          [&](const auto &x) -> const parser::OmpObjectList * {
+            using Ty = std::decay_t<decltype(x)>;
+            if constexpr (common::HasMember<Ty, MemberObjectListClauses>) {
+              return &x.v;
+            } else if constexpr (common::HasMember<Ty,
+                                     TupleObjectListClauses>) {
+              return &(std::get<parser::OmpObjectList>(x.v.t));
+            } else {
+              return nullptr;
+            }
+          },
+      },
+      clause.u);
+}
+
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index cd560dd1cd796ac94009e6bfadfc06dfc3376c2d..0d11f72b5bc853d3041933ca3ad51530cb2b60a5 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -73,6 +73,9 @@ static OmpDirectiveSet simdSet{Directive::OMPD_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd};
 static OmpDirectiveSet taskGeneratingSet{
     OmpDirectiveSet{Directive::OMPD_task} | taskloopSet};
+static OmpDirectiveSet nestedOrderedErrSet{Directive::OMPD_critical,
+    Directive::OMPD_ordered, Directive::OMPD_atomic, Directive::OMPD_task,
+    Directive::OMPD_taskloop};
 static OmpClauseSet privateSet{
     Clause::OMPC_private, Clause::OMPC_firstprivate, Clause::OMPC_lastprivate};
 static OmpClauseSet privateReductionSet{
@@ -153,6 +156,13 @@ public:
 #define GEN_FLANG_CLAUSE_CHECK_ENTER
 #include "llvm/Frontend/OpenMP/OMP.inc"
 
+  // Get the OpenMP Clause Kind for the corresponding Parser class
+  template <typename A>
+  llvm::omp::Clause GetClauseKindForParserClass(const A &) {
+#define GEN_FLANG_CLAUSE_PARSER_KIND_MAP
+#include "llvm/Frontend/OpenMP/OMP.inc"
+  }
+
 private:
   bool HasInvalidWorksharingNesting(
       const parser::CharBlock &, const OmpDirectiveSet &);
@@ -197,6 +207,7 @@ private:
       const parser::Name &name, const llvm::omp::Clause clause);
   void CheckMultipleAppearanceAcrossContext(
       const parser::OmpObjectList &ompObjectList);
+  const parser::OmpObjectList *GetOmpObjectList(const parser::OmpClause &);
 };
 } // namespace Fortran::semantics
 #endif // FORTRAN_SEMANTICS_CHECK_OMP_STRUCTURE_H_
diff --git a/flang/lib/Semantics/compute-offsets.cpp b/flang/lib/Semantics/compute-offsets.cpp
index bb2f4d98a17dded220c77973c2344e4ed1f54491..4b1538ca785f2c6fb5f1788437d22417886b5323 100644
--- a/flang/lib/Semantics/compute-offsets.cpp
+++ b/flang/lib/Semantics/compute-offsets.cpp
@@ -58,9 +58,10 @@ private:
   std::size_t offset_{0};
   std::size_t alignment_{1};
   // symbol -> symbol+offset that determines its location, from EQUIVALENCE
-  std::map<MutableSymbolRef, SymbolAndOffset> dependents_;
+  std::map<MutableSymbolRef, SymbolAndOffset, SymbolAddressCompare> dependents_;
   // base symbol -> SizeAndAlignment for each distinct EQUIVALENCE block
-  std::map<MutableSymbolRef, SizeAndAlignment> equivalenceBlock_;
+  std::map<MutableSymbolRef, SizeAndAlignment, SymbolAddressCompare>
+      equivalenceBlock_;
 };
 
 void ComputeOffsetsHelper::Compute(Scope &scope) {
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 3413a7531759eb944ccddb60cef43ecd1dc0fd19..0b36de464129ed2e6b041ae597d5dba27824ae09 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -2139,18 +2139,48 @@ template <typename A> static const Symbol *AssumedTypeDummy(const A &x) {
     if (const auto *dataRef{
             std::get_if<parser::DataRef>(&designator->value().u)}) {
       if (const auto *name{std::get_if<parser::Name>(&dataRef->u)}) {
-        if (const Symbol * symbol{name->symbol}) {
-          if (const auto *type{symbol->GetType()}) {
-            if (type->category() == semantics::DeclTypeSpec::TypeStar) {
-              return symbol;
-            }
-          }
-        }
+        return AssumedTypeDummy(*name);
       }
     }
   }
   return nullptr;
 }
+template <>
+const Symbol *AssumedTypeDummy<parser::Name>(const parser::Name &name) {
+  if (const Symbol * symbol{name.symbol}) {
+    if (const auto *type{symbol->GetType()}) {
+      if (type->category() == semantics::DeclTypeSpec::TypeStar) {
+        return symbol;
+      }
+    }
+  }
+  return nullptr;
+}
+template <typename A>
+static const Symbol *AssumedTypePointerOrAllocatableDummy(const A &object) {
+  // It is illegal for allocatable of pointer objects to be TYPE(*), but at that
+  // point it is is not guaranteed that it has been checked the object has
+  // POINTER or ALLOCATABLE attribute, so do not assume nullptr can be directly
+  // returned.
+  return std::visit(
+      common::visitors{
+          [&](const parser::StructureComponent &x) {
+            return AssumedTypeDummy(x.component);
+          },
+          [&](const parser::Name &x) { return AssumedTypeDummy(x); },
+      },
+      object.u);
+}
+template <>
+const Symbol *AssumedTypeDummy<parser::AllocateObject>(
+    const parser::AllocateObject &x) {
+  return AssumedTypePointerOrAllocatableDummy(x);
+}
+template <>
+const Symbol *AssumedTypeDummy<parser::PointerObject>(
+    const parser::PointerObject &x) {
+  return AssumedTypePointerOrAllocatableDummy(x);
+}
 
 MaybeExpr ExpressionAnalyzer::Analyze(const parser::FunctionReference &funcRef,
     std::optional<parser::StructureConstructor> *structureConstructor) {
@@ -2737,6 +2767,18 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::DataStmtConstant &x) {
   return ExprOrVariable(x, x.source);
 }
 
+MaybeExpr ExpressionAnalyzer::Analyze(const parser::AllocateObject &x) {
+  parser::CharBlock source{parser::FindSourceLocation(x)};
+  auto restorer{GetContextualMessages().SetLocation(source)};
+  return ExprOrVariable(x, source);
+}
+
+MaybeExpr ExpressionAnalyzer::Analyze(const parser::PointerObject &x) {
+  parser::CharBlock source{parser::FindSourceLocation(x)};
+  auto restorer{GetContextualMessages().SetLocation(source)};
+  return ExprOrVariable(x, source);
+}
+
 Expr<SubscriptInteger> ExpressionAnalyzer::AnalyzeKindSelector(
     TypeCategory category,
     const std::optional<parser::KindSelector> &selector) {
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index c3b95a7836a8deaf65a7a4c5f2fafac68d034983..a60c8dd1cd0202e5142225f94f33848ef6514ece 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -54,8 +54,8 @@ static void PutEntity(
 static void PutInit(llvm::raw_ostream &, const Symbol &, const MaybeExpr &);
 static void PutInit(llvm::raw_ostream &, const MaybeIntExpr &);
 static void PutBound(llvm::raw_ostream &, const Bound &);
-static llvm::raw_ostream &PutAttrs(llvm::raw_ostream &, Attrs,
-    const MaybeExpr & = std::nullopt, std::string before = ","s,
+llvm::raw_ostream &PutAttrs(llvm::raw_ostream &, Attrs,
+    const std::string * = nullptr, std::string before = ","s,
     std::string after = ""s);
 
 static llvm::raw_ostream &PutAttr(llvm::raw_ostream &, Attr);
@@ -81,8 +81,8 @@ private:
   const Scope &scope_;
   bool isInterface_{false};
   SymbolVector need_; // symbols that are needed
-  SymbolSet needSet_; // symbols already in need_
-  SymbolSet useSet_; // use-associations that might be needed
+  UnorderedSymbolSet needSet_; // symbols already in need_
+  UnorderedSymbolSet useSet_; // use-associations that might be needed
   std::set<SourceName> imports_; // imports from host that are needed
 
   void DoSymbol(const Symbol &);
@@ -346,7 +346,7 @@ void ModFileWriter::PutSubprogram(const Symbol &symbol) {
   if (isInterface) {
     os << (isAbstract ? "abstract " : "") << "interface\n";
   }
-  PutAttrs(os, prefixAttrs, std::nullopt, ""s, " "s);
+  PutAttrs(os, prefixAttrs, nullptr, ""s, " "s);
   os << (details.isFunction() ? "function " : "subroutine ");
   os << symbol.name() << '(';
   int n = 0;
@@ -498,7 +498,8 @@ void CollectSymbols(
   for (const auto &pair : scope.commonBlocks()) {
     sorted.push_back(*pair.second);
   }
-  std::sort(sorted.end() - commonSize, sorted.end());
+  std::sort(
+      sorted.end() - commonSize, sorted.end(), SymbolSourcePositionCompare{});
 }
 
 void PutEntity(llvm::raw_ostream &os, const Symbol &symbol) {
@@ -635,26 +636,18 @@ void PutBound(llvm::raw_ostream &os, const Bound &x) {
 void PutEntity(llvm::raw_ostream &os, const Symbol &symbol,
     std::function<void()> writeType, Attrs attrs) {
   writeType();
-  MaybeExpr bindName;
-  std::visit(common::visitors{
-                 [&](const SubprogramDetails &x) { bindName = x.bindName(); },
-                 [&](const ObjectEntityDetails &x) { bindName = x.bindName(); },
-                 [&](const ProcEntityDetails &x) { bindName = x.bindName(); },
-                 [&](const auto &) {},
-             },
-      symbol.details());
-  PutAttrs(os, attrs, bindName);
+  PutAttrs(os, attrs, symbol.GetBindName());
   os << "::" << symbol.name();
 }
 
 // Put out each attribute to os, surrounded by `before` and `after` and
 // mapped to lower case.
 llvm::raw_ostream &PutAttrs(llvm::raw_ostream &os, Attrs attrs,
-    const MaybeExpr &bindName, std::string before, std::string after) {
+    const std::string *bindName, std::string before, std::string after) {
   attrs.set(Attr::PUBLIC, false); // no need to write PUBLIC
   attrs.set(Attr::EXTERNAL, false); // no need to write EXTERNAL
   if (bindName) {
-    bindName->AsFortran(os << before << "bind(c, name=") << ')' << after;
+    os << before << "bind(c, name=\"" << *bindName << "\")" << after;
     attrs.set(Attr::BIND_C, false);
   }
   for (std::size_t i{0}; i < Attr_enumSize; ++i) {
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 8f12278f8559ab7904536999587d0f77eb37905f..d5ba6a12995d931db25f0407c737924458cb4e65 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -105,7 +105,7 @@ protected:
   Symbol *DeclarePrivateAccessEntity(Symbol &, Symbol::Flag, Scope &);
   Symbol *DeclareOrMarkOtherAccessEntity(const parser::Name &, Symbol::Flag);
 
-  SymbolSet dataSharingAttributeObjects_; // on one directive
+  UnorderedSymbolSet dataSharingAttributeObjects_; // on one directive
   SemanticsContext &context_;
   std::vector<DirContext> dirContext_; // used as a stack
 };
@@ -452,8 +452,8 @@ private:
       Symbol::Flag::OmpCopyIn, Symbol::Flag::OmpCopyPrivate};
 
   std::vector<const parser::Name *> allocateNames_; // on one directive
-  SymbolSet privateDataSharingAttributeObjects_; // on one directive
-  SymbolSet stmtFunctionExprSymbols_;
+  UnorderedSymbolSet privateDataSharingAttributeObjects_; // on one directive
+  UnorderedSymbolSet stmtFunctionExprSymbols_;
   std::multimap<const parser::Label,
       std::pair<parser::CharBlock, std::optional<DirContext>>>
       sourceLabels_;
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index df358d8804453615dbf44fd5c6ef837de9f6c7c2..6818686f43e7fd2ec01dc113578c0341a5f90602 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -745,7 +745,7 @@ private:
   } funcInfo_;
 
   // Create a subprogram symbol in the current scope and push a new scope.
-  void CheckExtantExternal(const parser::Name &, Symbol::Flag);
+  void CheckExtantProc(const parser::Name &, Symbol::Flag);
   Symbol &PushSubprogramScope(const parser::Name &, Symbol::Flag);
   Symbol *GetSpecificFromGeneric(const parser::Name &);
   SubprogramDetails &PostSubprogramStmt(const parser::Name &);
@@ -1528,19 +1528,26 @@ bool AttrsVisitor::SetPassNameOn(Symbol &symbol) {
 }
 
 bool AttrsVisitor::SetBindNameOn(Symbol &symbol) {
-  if (!bindName_) {
+  if (!attrs_ || !attrs_->test(Attr::BIND_C)) {
     return false;
   }
-  std::visit(
-      common::visitors{
-          [&](EntityDetails &x) { x.set_bindName(std::move(bindName_)); },
-          [&](ObjectEntityDetails &x) { x.set_bindName(std::move(bindName_)); },
-          [&](ProcEntityDetails &x) { x.set_bindName(std::move(bindName_)); },
-          [&](SubprogramDetails &x) { x.set_bindName(std::move(bindName_)); },
-          [&](CommonBlockDetails &x) { x.set_bindName(std::move(bindName_)); },
-          [](auto &) { common::die("unexpected bind name"); },
-      },
-      symbol.details());
+  std::optional<std::string> label{evaluate::GetScalarConstantValue<
+      evaluate::Type<TypeCategory::Character, 1>>(bindName_)};
+  // 18.9.2(2): discard leading and trailing blanks, ignore if all blank
+  if (label) {
+    auto first{label->find_first_not_of(" ")};
+    auto last{label->find_last_not_of(" ")};
+    if (first == std::string::npos) {
+      Say(currStmtSource().value(), "Blank binding label ignored"_en_US);
+      label.reset();
+    } else {
+      label = label->substr(first, last - first + 1);
+    }
+  }
+  if (!label) {
+    label = parser::ToLowerCaseLetters(symbol.name().ToString());
+  }
+  symbol.SetBindName(std::move(*label));
   return true;
 }
 
@@ -2690,7 +2697,7 @@ void InterfaceVisitor::AddSpecificProcs(
 // this generic interface. Resolve those names to symbols.
 void InterfaceVisitor::ResolveSpecificsInGeneric(Symbol &generic) {
   auto &details{generic.get<GenericDetails>()};
-  SymbolSet symbolsSeen;
+  UnorderedSymbolSet symbolsSeen;
   for (const Symbol &symbol : details.specificProcs()) {
     symbolsSeen.insert(symbol);
   }
@@ -3084,7 +3091,6 @@ void SubprogramVisitor::Post(const parser::EntryStmt &stmt) {
 
   Symbol::Flag subpFlag{
       inFunction ? Symbol::Flag::Function : Symbol::Flag::Subroutine};
-  CheckExtantExternal(name, subpFlag);
   Scope &outer{inclusiveScope.parent()}; // global or module scope
   if (Symbol * extant{FindSymbol(outer, name)}) {
     if (extant->has<ProcEntityDetails>()) {
@@ -3176,7 +3182,7 @@ bool SubprogramVisitor::BeginSubprogram(
 
 void SubprogramVisitor::EndSubprogram() { PopScope(); }
 
-void SubprogramVisitor::CheckExtantExternal(
+void SubprogramVisitor::CheckExtantProc(
     const parser::Name &name, Symbol::Flag subpFlag) {
   if (auto *prev{FindSymbol(name)}) {
     if (prev->attrs().test(Attr::EXTERNAL) && prev->has<ProcEntityDetails>()) {
@@ -3189,6 +3195,11 @@ void SubprogramVisitor::CheckExtantExternal(
             *prev, "Previous call of '%s'"_en_US);
       }
       EraseSymbol(name);
+    } else if (const auto *details{prev->detailsIf<EntityDetails>()}) {
+      if (!details->isDummy()) {
+        Say2(name, "Procedure '%s' was previously declared"_err_en_US, *prev,
+            "Previous declaration of '%s'"_en_US);
+      }
     }
   }
 }
@@ -3197,7 +3208,7 @@ Symbol &SubprogramVisitor::PushSubprogramScope(
     const parser::Name &name, Symbol::Flag subpFlag) {
   auto *symbol{GetSpecificFromGeneric(name)};
   if (!symbol) {
-    CheckExtantExternal(name, subpFlag);
+    CheckExtantProc(name, subpFlag);
     symbol = &MakeSymbol(name, SubprogramDetails{});
   }
   symbol->set(subpFlag);
@@ -3248,7 +3259,12 @@ Symbol *SubprogramVisitor::GetSpecificFromGeneric(const parser::Name &name) {
       if (!specific) {
         specific =
             &currScope().MakeSymbol(name.source, Attrs{}, SubprogramDetails{});
-        details->set_specific(Resolve(name, *specific));
+        if (details->derivedType()) {
+          // A specific procedure with the same name as a derived type
+          SayAlreadyDeclared(name, *details->derivedType());
+        } else {
+          details->set_specific(Resolve(name, *specific));
+        }
       } else if (isGeneric()) {
         SayAlreadyDeclared(name, *specific);
       }
@@ -3651,7 +3667,7 @@ Symbol &DeclarationVisitor::DeclareUnknownEntity(
 
 bool DeclarationVisitor::HasCycle(
     const Symbol &procSymbol, const ProcInterface &interface) {
-  SymbolSet procsInCycle;
+  OrderedSymbolSet procsInCycle;
   procsInCycle.insert(procSymbol);
   const ProcInterface *thisInterface{&interface};
   bool haveInterface{true};
@@ -6393,7 +6409,7 @@ void ResolveNamesVisitor::FinishSpecificationPart(
       CheckPossibleBadForwardRef(symbol);
     }
   }
-  currScope().InstantiateDerivedTypes(context());
+  currScope().InstantiateDerivedTypes();
   for (const auto &decl : decls) {
     if (const auto *statement{std::get_if<
             parser::Statement<common::Indirection<parser::StmtFunctionStmt>>>(
diff --git a/flang/lib/Semantics/scope.cpp b/flang/lib/Semantics/scope.cpp
index 597f554abcb92b416b38999590a8d91a86f6e5e3..4faec3bd00cd2995613912eab2f616d176b6ead0 100644
--- a/flang/lib/Semantics/scope.cpp
+++ b/flang/lib/Semantics/scope.cpp
@@ -50,7 +50,7 @@ std::string EquivalenceObject::AsFortran() const {
 }
 
 Scope &Scope::MakeScope(Kind kind, Symbol *symbol) {
-  return children_.emplace_back(*this, kind, symbol);
+  return children_.emplace_back(*this, kind, symbol, context_);
 }
 
 template <typename T>
@@ -61,7 +61,7 @@ static std::vector<common::Reference<T>> GetSortedSymbols(
   for (auto &pair : symbols) {
     result.push_back(*pair.second);
   }
-  std::sort(result.begin(), result.end());
+  std::sort(result.begin(), result.end(), SymbolSourcePositionCompare{});
   return result;
 }
 
@@ -404,11 +404,11 @@ const Scope &Scope::GetDerivedTypeBase() const {
   return *child;
 }
 
-void Scope::InstantiateDerivedTypes(SemanticsContext &context) {
+void Scope::InstantiateDerivedTypes() {
   for (DeclTypeSpec &type : declTypeSpecs_) {
     if (type.category() == DeclTypeSpec::TypeDerived ||
         type.category() == DeclTypeSpec::ClassDerived) {
-      type.derivedTypeSpec().Instantiate(*this, context);
+      type.derivedTypeSpec().Instantiate(*this, context_);
     }
   }
 }
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index f299897603d92546ffe62e0dc096027a338ce9da..24bc5e3ea816093e656fc1f0370e6678af623a5d 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -185,8 +185,9 @@ SemanticsContext::SemanticsContext(
     : defaultKinds_{defaultKinds}, languageFeatures_{languageFeatures},
       allCookedSources_{allCookedSources},
       intrinsics_{evaluate::IntrinsicProcTable::Configure(defaultKinds_)},
-      foldingContext_{
-          parser::ContextualMessages{&messages_}, defaultKinds_, intrinsics_} {}
+      globalScope_{*this}, foldingContext_{
+                               parser::ContextualMessages{&messages_},
+                               defaultKinds_, intrinsics_} {}
 
 SemanticsContext::~SemanticsContext() {}
 
diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index edd2c84218c1dee520915a873c64a28a41cc8eda..7d439df75c2efc73fa0ae8fc4b97564334fe75f3 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -14,6 +14,7 @@
 #include "flang/Semantics/tools.h"
 #include "llvm/Support/raw_ostream.h"
 #include <string>
+#include <type_traits>
 
 namespace Fortran::semantics {
 
@@ -84,7 +85,7 @@ void ModuleDetails::set_scope(const Scope *scope) {
 llvm::raw_ostream &operator<<(
     llvm::raw_ostream &os, const SubprogramDetails &x) {
   DumpBool(os, "isInterface", x.isInterface_);
-  DumpExpr(os, "bindName", x.bindName_);
+  DumpOptional(os, "bindName", x.bindName());
   if (x.result_) {
     DumpType(os << " result:", x.result());
     os << x.result_->name();
@@ -290,6 +291,33 @@ void Symbol::SetType(const DeclTypeSpec &type) {
       details_);
 }
 
+template <typename T>
+constexpr bool HasBindName{std::is_convertible_v<T, const WithBindName *>};
+
+const std::string *Symbol::GetBindName() const {
+  return std::visit(
+      [&](auto &x) -> const std::string * {
+        if constexpr (HasBindName<decltype(&x)>) {
+          return x.bindName();
+        } else {
+          return nullptr;
+        }
+      },
+      details_);
+}
+
+void Symbol::SetBindName(std::string &&name) {
+  std::visit(
+      [&](auto &x) {
+        if constexpr (HasBindName<decltype(&x)>) {
+          x.set_bindName(std::move(name));
+        } else {
+          DIE("bind name not allowed on this kind of symbol");
+        }
+      },
+      details_);
+}
+
 bool Symbol::IsFuncResult() const {
   return std::visit(
       common::visitors{[](const EntityDetails &x) { return x.isFuncResult(); },
@@ -331,7 +359,7 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const EntityDetails &x) {
   if (x.type()) {
     os << " type: " << *x.type();
   }
-  DumpExpr(os, "bindName", x.bindName_);
+  DumpOptional(os, "bindName", x.bindName());
   return os;
 }
 
@@ -361,7 +389,7 @@ llvm::raw_ostream &operator<<(
   } else {
     DumpType(os, x.interface_.type());
   }
-  DumpExpr(os, "bindName", x.bindName());
+  DumpOptional(os, "bindName", x.bindName());
   DumpOptional(os, "passName", x.passName());
   if (x.init()) {
     if (const Symbol * target{*x.init()}) {
@@ -448,6 +476,7 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Details &details) {
             DumpSymbolVector(os, x.objects());
           },
           [&](const CommonBlockDetails &x) {
+            DumpOptional(os, "bindName", x.bindName());
             if (x.alignment()) {
               os << " alignment=" << x.alignment();
             }
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 2d7fa9de939227fdc4daebee099fd204d5aa6d26..256a5cc1d317f2a65b08b9d365790d683e8123e6 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -374,17 +374,24 @@ static void CheckMissingAnalysis(bool absent, const T &x) {
   }
 }
 
-const SomeExpr *GetExprHelper::Get(const parser::Expr &x) {
+template <typename T> static const SomeExpr *GetTypedExpr(const T &x) {
   CheckMissingAnalysis(!x.typedExpr, x);
   return common::GetPtrFromOptional(x.typedExpr->v);
 }
+const SomeExpr *GetExprHelper::Get(const parser::Expr &x) {
+  return GetTypedExpr(x);
+}
 const SomeExpr *GetExprHelper::Get(const parser::Variable &x) {
-  CheckMissingAnalysis(!x.typedExpr, x);
-  return common::GetPtrFromOptional(x.typedExpr->v);
+  return GetTypedExpr(x);
 }
 const SomeExpr *GetExprHelper::Get(const parser::DataStmtConstant &x) {
-  CheckMissingAnalysis(!x.typedExpr, x);
-  return common::GetPtrFromOptional(x.typedExpr->v);
+  return GetTypedExpr(x);
+}
+const SomeExpr *GetExprHelper::Get(const parser::AllocateObject &x) {
+  return GetTypedExpr(x);
+}
+const SomeExpr *GetExprHelper::Get(const parser::PointerObject &x) {
+  return GetTypedExpr(x);
 }
 
 const evaluate::Assignment *GetAssignment(const parser::AssignmentStmt &x) {
diff --git a/flang/runtime/buffer.h b/flang/runtime/buffer.h
index c5bd5aedaaee8b445960cac5a40bdda493db31e0..c601ee7c4be0652b1888c3f3c63f980512ec2064 100644
--- a/flang/runtime/buffer.h
+++ b/flang/runtime/buffer.h
@@ -27,7 +27,24 @@ void LeftShiftBufferCircularly(char *, std::size_t bytes, std::size_t shift);
 // preserve read data that may be reused by means of Tn/TLn edit descriptors
 // without needing to position the file (which may not always be possible,
 // e.g. a socket) and a general desire to reduce system call counts.
-template <typename STORE> class FileFrame {
+//
+// Possible scenario with a tiny 32-byte buffer after a ReadFrame or
+// WriteFrame with a file offset of 103 to access "DEF":
+//
+//    fileOffset_ 100 --+  +-+ frame of interest (103:105)
+//   file:  ............ABCDEFGHIJKLMNOPQRSTUVWXYZ....
+// buffer: [NOPQRSTUVWXYZ......ABCDEFGHIJKLM]   (size_ == 32)
+//                             |  +-- frame_ == 3
+//                             +----- start_ == 19, length_ == 26
+//
+// The buffer holds length_ == 26 bytes from file offsets 100:125.
+// Those 26 bytes "wrap around" the end of the circular buffer,
+// so file offsets 100:112 map to buffer offsets 19:31 ("A..M") and
+//    file offsets 113:125 map to buffer offsets  0:12 ("N..Z")
+// The 3-byte frame of file offsets 103:105 is contiguous in the buffer
+// at buffer offset (start_ + frame_) == 22 ("DEF").
+
+template <typename STORE, std::size_t minBuffer = 65536> class FileFrame {
 public:
   using FileOffset = std::int64_t;
 
@@ -50,28 +67,17 @@ public:
       FileOffset at, std::size_t bytes, IoErrorHandler &handler) {
     Flush(handler);
     Reallocate(bytes, handler);
-    if (at < fileOffset_ || at > fileOffset_ + length_) {
+    std::int64_t newFrame{at - fileOffset_};
+    if (newFrame < 0 || newFrame > length_) {
       Reset(at);
+    } else {
+      frame_ = newFrame;
     }
-    frame_ = at - fileOffset_;
+    RUNTIME_CHECK(handler, at == fileOffset_ + frame_);
     if (static_cast<std::int64_t>(start_ + frame_ + bytes) > size_) {
       DiscardLeadingBytes(frame_, handler);
-      if (static_cast<std::int64_t>(start_ + bytes) > size_) {
-        // Frame would wrap around; shift current data (if any) to force
-        // contiguity.
-        RUNTIME_CHECK(handler, length_ < size_);
-        if (start_ + length_ <= size_) {
-          // [......abcde..] -> [abcde........]
-          std::memmove(buffer_, buffer_ + start_, length_);
-        } else {
-          // [cde........ab] -> [abcde........]
-          auto n{start_ + length_ - size_}; // 3 for cde
-          RUNTIME_CHECK(handler, length_ >= n);
-          std::memmove(buffer_ + n, buffer_ + start_, length_ - n); // cdeab
-          LeftShiftBufferCircularly(buffer_, length_, n); // abcde
-        }
-        start_ = 0;
-      }
+      MakeDataContiguous(handler, bytes);
+      RUNTIME_CHECK(handler, at == fileOffset_ + frame_);
     }
     while (FrameLength() < bytes) {
       auto next{start_ + length_};
@@ -81,7 +87,7 @@ public:
       auto got{Store().Read(
           fileOffset_ + length_, buffer_ + next, minBytes, maxBytes, handler)};
       length_ += got;
-      RUNTIME_CHECK(handler, length_ < size_);
+      RUNTIME_CHECK(handler, length_ <= size_);
       if (got < minBytes) {
         break; // error or EOF & program can handle it
       }
@@ -90,32 +96,38 @@ public:
   }
 
   void WriteFrame(FileOffset at, std::size_t bytes, IoErrorHandler &handler) {
-    if (!dirty_ || at < fileOffset_ || at > fileOffset_ + length_ ||
-        start_ + (at - fileOffset_) + static_cast<std::int64_t>(bytes) >
-            size_) {
+    Reallocate(bytes, handler);
+    std::int64_t newFrame{at - fileOffset_};
+    if (!dirty_ || newFrame < 0 || newFrame > length_) {
       Flush(handler);
       Reset(at);
-      Reallocate(bytes, handler);
+    } else if (start_ + newFrame + static_cast<std::int64_t>(bytes) > size_) {
+      // Flush leading data before "at", retain from "at" onward
+      Flush(handler, length_ - newFrame);
+      MakeDataContiguous(handler, bytes);
+    } else {
+      frame_ = newFrame;
     }
+    RUNTIME_CHECK(handler, at == fileOffset_ + frame_);
     dirty_ = true;
-    frame_ = at - fileOffset_;
     length_ = std::max<std::int64_t>(length_, frame_ + bytes);
   }
 
-  void Flush(IoErrorHandler &handler) {
+  void Flush(IoErrorHandler &handler, std::int64_t keep = 0) {
     if (dirty_) {
-      while (length_ > 0) {
-        std::size_t chunk{std::min<std::size_t>(length_, size_ - start_)};
+      while (length_ > keep) {
+        std::size_t chunk{
+            std::min<std::size_t>(length_ - keep, size_ - start_)};
         std::size_t put{
             Store().Write(fileOffset_, buffer_ + start_, chunk, handler)};
-        length_ -= put;
-        start_ += put;
-        fileOffset_ += put;
+        DiscardLeadingBytes(put, handler);
         if (put < chunk) {
           break;
         }
       }
-      Reset(fileOffset_);
+      if (length_ == 0) {
+        Reset(fileOffset_);
+      }
     }
   }
 
@@ -162,7 +174,24 @@ private:
     fileOffset_ += n;
   }
 
-  static constexpr std::size_t minBuffer{64 << 10};
+  void MakeDataContiguous(IoErrorHandler &handler, std::size_t bytes) {
+    if (static_cast<std::int64_t>(start_ + bytes) > size_) {
+      // Frame would wrap around; shift current data (if any) to force
+      // contiguity.
+      RUNTIME_CHECK(handler, length_ < size_);
+      if (start_ + length_ <= size_) {
+        // [......abcde..] -> [abcde........]
+        std::memmove(buffer_, buffer_ + start_, length_);
+      } else {
+        // [cde........ab] -> [abcde........]
+        auto n{start_ + length_ - size_}; // 3 for cde
+        RUNTIME_CHECK(handler, length_ >= n);
+        std::memmove(buffer_ + n, buffer_ + start_, length_ - n); // cdeab
+        LeftShiftBufferCircularly(buffer_, length_, n); // abcde
+      }
+      start_ = 0;
+    }
+  }
 
   char *buffer_{nullptr};
   std::int64_t size_{0}; // current allocated buffer size
diff --git a/flang/runtime/descriptor.cpp b/flang/runtime/descriptor.cpp
index efcd61b50c4f4c9167b9efe6ad088d7618fcbbb9..b66874b924e6eca3460e5180033519e28afe6cd6 100644
--- a/flang/runtime/descriptor.cpp
+++ b/flang/runtime/descriptor.cpp
@@ -31,9 +31,23 @@ void Descriptor::Establish(TypeCode t, std::size_t elementBytes, void *p,
     int rank, const SubscriptValue *extent, ISO::CFI_attribute_t attribute,
     bool addendum) {
   Terminator terminator{__FILE__, __LINE__};
+  // Subtle: the standard CFI_establish() function doesn't allow a zero
+  // elem_len argument in cases where elem_len is not ignored; and when it
+  // returns an error code (CFI_INVALID_ELEM_LEN in this case), it must not
+  // modify the descriptor.  That design makes sense, maybe, for actual
+  // C interoperability, but we need to work around it here.  A zero
+  // incoming element length is replaced by 4 so that it will be valid
+  // for all CHARACTER kinds.
+  std::size_t workaroundElemLen{elementBytes ? elementBytes : 4};
   RUNTIME_CHECK(terminator,
-      ISO::CFI_establish(&raw_, p, attribute, t.raw(), elementBytes, rank,
+      ISO::CFI_establish(&raw_, p, attribute, t.raw(), workaroundElemLen, rank,
           extent) == CFI_SUCCESS);
+  if (elementBytes == 0) {
+    raw_.elem_len = 0;
+    for (int j{0}; j < rank; ++j) {
+      GetDimension(j).SetByteStride(0);
+    }
+  }
   raw_.f18Addendum = addendum;
   DescriptorAddendum *a{Addendum()};
   RUNTIME_CHECK(terminator, addendum == (a != nullptr));
diff --git a/flang/test/Driver/Inputs/ieee_arithmetic.mod b/flang/test/Driver/Inputs/ieee_arithmetic.mod
new file mode 100644
index 0000000000000000000000000000000000000000..30fd57801970b48ac0b01bd3369a1b5553df8541
--- /dev/null
+++ b/flang/test/Driver/Inputs/ieee_arithmetic.mod
@@ -0,0 +1,7 @@
+! DUMMY module
+! Added for testing purposes. The contents of this file are currently not relevant.
+module ieee_arithmetic
+type::ieee_round_type
+integer(1),private::mode=0_1
+end type
+end
diff --git a/flang/test/Driver/Inputs/iso_fortran_env.mod b/flang/test/Driver/Inputs/iso_fortran_env.mod
new file mode 100644
index 0000000000000000000000000000000000000000..689297d52027b576f8f3b67c1733c65232bd7c3c
--- /dev/null
+++ b/flang/test/Driver/Inputs/iso_fortran_env.mod
@@ -0,0 +1,7 @@
+! DUMMY module
+! Added for testing purposes. The contents of this file are currently not relevant.
+module iso_fortran_env
+use __fortran_builtins,only:event_type=>__builtin_event_type
+use __fortran_builtins,only:lock_type=>__builtin_lock_type
+use __fortran_builtins,only:team_type=>__builtin_team_type
+end
diff --git a/flang/test/Driver/driver-help-hidden.f90 b/flang/test/Driver/driver-help-hidden.f90
index e6235037b1e71f50de5dc1c2799c835405d1860a..e8b1141d00f4c1f4aa67a5b019f6cbf0ef70755e 100644
--- a/flang/test/Driver/driver-help-hidden.f90
+++ b/flang/test/Driver/driver-help-hidden.f90
@@ -35,6 +35,8 @@
 ! CHECK-NEXT: -ffree-form            Process source files in free form
 ! CHECK-NEXT: -fimplicit-none        No implicit typing allowed unless overridden by IMPLICIT statements
 ! CHECK-NEXT: -finput-charset=<value> Specify the default character set for source files
+! CHECK-NEXT: -fintrinsic-modules-path <dir>
+! CHECK-NEXT:                        Specify where to find the compiled intrinsic modules
 ! CHECK-NEXT: -flarge-sizes          Use INTEGER(KIND=8) for the result type in size-related intrinsics
 ! CHECK-NEXT: -flogical-abbreviations Enable logical abbreviations
 ! CHECK-NEXT: -fno-color-diagnostics Disable colors in diagnostics
@@ -45,6 +47,8 @@
 ! CHECK-NEXT: -I <dir>               Add directory to the end of the list of include search paths
 ! CHECK-NEXT: -module-dir <dir>      Put MODULE files in <dir>
 ! CHECK-NEXT: -o <file> Write output to <file>
+! CHECK-NEXT: -pedantic              Warn on language extensions
+! CHECK-NEXT: -std=<value>           Language standard to compile for
 ! CHECK-NEXT: -U <macro>             Undefine macro <macro>
 ! CHECK-NEXT: --version Print version information
 ! CHECK-NEXT: -Xflang <arg>          Pass <arg> to the flang compiler
diff --git a/flang/test/Driver/driver-help.f90 b/flang/test/Driver/driver-help.f90
index c32975416f2f5614ce6066c176eb5fec7002faaa..855d6f2b58d6249762bc9aaa157061a1efc2953f 100644
--- a/flang/test/Driver/driver-help.f90
+++ b/flang/test/Driver/driver-help.f90
@@ -35,6 +35,8 @@
 ! HELP-NEXT: -ffree-form            Process source files in free form
 ! HELP-NEXT: -fimplicit-none        No implicit typing allowed unless overridden by IMPLICIT statements
 ! HELP-NEXT: -finput-charset=<value> Specify the default character set for source files
+! HELP-NEXT: -fintrinsic-modules-path <dir>
+! HELP-NEXT:                        Specify where to find the compiled intrinsic modules
 ! HELP-NEXT: -flarge-sizes          Use INTEGER(KIND=8) for the result type in size-related intrinsics
 ! HELP-NEXT: -flogical-abbreviations Enable logical abbreviations
 ! HELP-NEXT: -fno-color-diagnostics Disable colors in diagnostics
@@ -45,6 +47,8 @@
 ! HELP-NEXT: -I <dir>               Add directory to the end of the list of include search paths
 ! HELP-NEXT: -module-dir <dir>      Put MODULE files in <dir>
 ! HELP-NEXT: -o <file>              Write output to <file>
+! HELP-NEXT: -pedantic              Warn on language extensions
+! HELP-NEXT: -std=<value>           Language standard to compile for
 ! HELP-NEXT: -U <macro>             Undefine macro <macro>
 ! HELP-NEXT: --version              Print version information
 ! HELP-NEXT: -Xflang <arg>          Pass <arg> to the flang compiler
@@ -80,8 +84,11 @@
 ! HELP-FC1-NEXT: -ffixed-line-length=<value>
 ! HELP-FC1-NEXT: Use <value> as character line width in fixed mode
 ! HELP-FC1-NEXT: -ffree-form            Process source files in free form
+! HELP-FC1-NEXT: -fget-symbols-sources   Dump symbols and their source code locations
 ! HELP-FC1-NEXT: -fimplicit-none        No implicit typing allowed unless overridden by IMPLICIT statements
 ! HELP-FC1-NEXT: -finput-charset=<value> Specify the default character set for source files
+! HELP-FC1-NEXT: -fintrinsic-modules-path <dir>
+! HELP-FC1-NEXT:                        Specify where to find the compiled intrinsic modules
 ! HELP-FC1-NEXT: -flarge-sizes          Use INTEGER(KIND=8) for the result type in size-related intrinsics
 ! HELP-FC1-NEXT: -flogical-abbreviations Enable logical abbreviations
 ! HELP-FC1-NEXT: -fopenacc              Enable OpenACC
@@ -91,6 +98,8 @@
 ! HELP-FC1-NEXT: -I <dir>               Add directory to the end of the list of include search paths
 ! HELP-FC1-NEXT: -module-dir <dir>      Put MODULE files in <dir>
 ! HELP-FC1-NEXT: -o <file>              Write output to <file>
+! HELP-FC1-NEXT: -pedantic              Warn on language extensions
+! HELP-FC1-NEXT: -std=<value>           Language standard to compile for
 ! HELP-FC1-NEXT: -test-io               Run the InputOuputTest action. Use for development and testing only.
 ! HELP-FC1-NEXT: -U <macro>             Undefine macro <macro>
 ! HELP-FC1-NEXT: --version              Print version information
diff --git a/flang/test/Driver/intrinsic_module_path.f90 b/flang/test/Driver/intrinsic_module_path.f90
new file mode 100644
index 0000000000000000000000000000000000000000..3f11512289a63bde37adddc7237ff8f56a38938f
--- /dev/null
+++ b/flang/test/Driver/intrinsic_module_path.f90
@@ -0,0 +1,37 @@
+! Ensure argument -fintrinsic-modules-path works as expected.
+! WITHOUT the option, the default location for the module is checked and no error generated.
+! With the option GIVEN, the module with the same name is PREPENDED, and considered over the
+! default one, causing a CHECKSUM error.
+
+! REQUIRES: new-flang-driver
+
+
+!--------------------------
+! FLANG DRIVER (flang-new)
+!--------------------------
+! RUN: %flang-new -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
+! RUN: not %flang-new -fsyntax-only -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+
+!-----------------------------------------
+! FRONTEND FLANG DRIVER (flang-new -fc1)
+!-----------------------------------------
+! RUN: %flang-new -fc1 %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
+! RUN: not %flang-new -fc1 -fintrinsic-modules-path %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+
+!-----------------------------------------
+! EXPECTED OUTPUT WITHOUT
+!-----------------------------------------
+! WITHOUT-NOT: 'ieee_arithmetic.mod' was not found
+! WITHOUT-NOT: 'iso_fortran_env.mod' was not found
+
+!-----------------------------------------
+! EXPECTED OUTPUT WITH
+!-----------------------------------------
+! GIVEN: error: Cannot read module file for module 'ieee_arithmetic': File has invalid checksum
+! GIVEN: error: Cannot read module file for module 'iso_fortran_env': File has invalid checksum
+
+
+program test_intrinsic_module_path
+   use ieee_arithmetic, only: ieee_round_type
+   use iso_fortran_env, only: team_type, event_type, lock_type
+end program
diff --git a/flang/test/Driver/std2018.f90 b/flang/test/Driver/std2018.f90
new file mode 100644
index 0000000000000000000000000000000000000000..acc063e5fe4b56bbce940f589a9c13f7ddea5b7e
--- /dev/null
+++ b/flang/test/Driver/std2018.f90
@@ -0,0 +1,28 @@
+! Ensure argument -std=f2018 works as expected.
+
+!-----------------------------------------
+! FRONTEND FLANG DRIVER (flang-new -fc1)
+!-----------------------------------------
+! RUN: %flang_fc1 -fsyntax-only %s  2>&1 | FileCheck %s --allow-empty --check-prefix=WITHOUT
+! RUN: %flang_fc1 -fsyntax-only -std=f2018 %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+! RUN: %flang_fc1 -fsyntax-only -pedantic %s  2>&1 | FileCheck %s --check-prefix=GIVEN
+
+!-----------------------------------------
+! EXPECTED OUTPUT WITHOUT
+!-----------------------------------------
+! WITHOUT-NOT: A DO loop should terminate with an END DO or CONTINUE
+
+!-----------------------------------------
+! EXPECTED OUTPUT WITH
+!-----------------------------------------
+! GIVEN: A DO loop should terminate with an END DO or CONTINUE
+
+subroutine foo2()
+    do 01 m=1,2
+      select case (m)
+      case default
+        print*, "default", m
+      case (1)
+        print*, "start"
+01    end select
+end subroutine
diff --git a/flang/test/Driver/std2018_wrong.f90 b/flang/test/Driver/std2018_wrong.f90
new file mode 100644
index 0000000000000000000000000000000000000000..867d014c4c113c44bfe27b263c6215faf72bbdc1
--- /dev/null
+++ b/flang/test/Driver/std2018_wrong.f90
@@ -0,0 +1,12 @@
+! REQUIRES: new-flang-driver
+! Ensure argument -std=f2018 works as expected.
+
+!-----------------------------------------
+! FRONTEND FLANG DRIVER (flang-new -fc1)
+!-----------------------------------------
+! RUN: not %flang_fc1 -std=90 %s  2>&1 | FileCheck %s --check-prefix=WRONG
+
+!-----------------------------------------
+! EXPECTED OUTPUT WITH WRONG
+!-----------------------------------------
+! WRONG: Only -std=f2018 is allowed currently.
diff --git a/flang/test/Evaluate/folding01.f90 b/flang/test/Evaluate/folding01.f90
index 465b22752cecb2e126768201233792c4825b5de0..b12c6a0e9aedeccc9ec4ca43a67029603d7678c4 100644
--- a/flang/test/Evaluate/folding01.f90
+++ b/flang/test/Evaluate/folding01.f90
@@ -30,6 +30,9 @@ module m
   logical, parameter :: test_neqv3 = .NOT.(.false..NEQV..false.)
   logical, parameter :: test_neqv4 = .NOT.(.true..NEQV..true.)
 
+  logical, parameter :: test_logical1 = logical(logical(.true., 2))
+  logical, parameter :: test_logical2 = .NOT.logical(logical(.false., 2))
+
 ! Check integer intrinsic operator folding
 
 ! Check integer relational intrinsic operation folding
diff --git a/flang/test/Fir/cg-ops.fir b/flang/test/Fir/cg-ops.fir
new file mode 100644
index 0000000000000000000000000000000000000000..a138313eef94f399ff4c674e57895cc6e88a3442
--- /dev/null
+++ b/flang/test/Fir/cg-ops.fir
@@ -0,0 +1,30 @@
+// RUN: fir-opt --pass-pipeline="func(cg-rewrite),fir.global(cg-rewrite),cse" %s | FileCheck %s
+
+// CHECK-LABEL: func @codegen(
+// CHECK-SAME: %[[arg:.*]]: !fir
+func @codegen(%addr : !fir.ref<!fir.array<?xi32>>) {
+  // CHECK: %[[zero:.*]] = constant 0 : index
+  %0 = constant 0 : index
+  %1 = fir.shape_shift %0, %0 : (index, index) -> !fir.shapeshift<1>
+  %2 = fir.slice %0, %0, %0 : (index, index, index) -> !fir.slice<1>
+  // CHECK: %[[box:.*]] = fircg.ext_embox %[[arg]](%[[zero]]) origin %[[zero]][%[[zero]], %[[zero]], %[[zero]]] : (!fir.ref<!fir.array<?xi32>>, index, index, index, index, index) -> !fir.box<!fir.array<?xi32>>
+  %3 = fir.embox %addr (%1) [%2] : (!fir.ref<!fir.array<?xi32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xi32>>
+  // CHECK: fircg.ext_array_coor %[[arg]](%[[zero]]) origin %[[zero]][%[[zero]], %[[zero]], %[[zero]]]<%[[zero]]> : (!fir.ref<!fir.array<?xi32>>, index, index, index, index, index, index) -> !fir.ref<i32>
+  %4 = fir.array_coor %addr (%1) [%2] %0 : (!fir.ref<!fir.array<?xi32>>, !fir.shapeshift<1>, !fir.slice<1>, index) -> !fir.ref<i32>
+  // CHECK: fircg.ext_rebox %[[box]](%[[zero]]) origin %[[zero]] : (!fir.box<!fir.array<?xi32>>, index, index) -> !fir.box<!fir.array<?xi32>>
+  %5 = fir.rebox %3(%1) : (!fir.box<!fir.array<?xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<?xi32>>
+  return
+}
+
+// CHECK-LABEL: fir.global @box_global
+fir.global @box_global : !fir.box<!fir.array<?xi32>> {
+  // CHECK: %[[arr:.*]] = fir.zero_bits !fir.ref
+  %arr = fir.zero_bits !fir.ref<!fir.array<?xi32>>
+  // CHECK: %[[zero:.*]] = constant 0 : index
+  %0 = constant 0 : index
+  %1 = fir.shape_shift %0, %0 : (index, index) -> !fir.shapeshift<1>
+  %2 = fir.slice %0, %0, %0 : (index, index, index) -> !fir.slice<1>
+  // CHECK: fircg.ext_embox %[[arr]](%[[zero]]) origin %[[zero]][%[[zero]], %[[zero]], %[[zero]]] : (!fir.ref<!fir.array<?xi32>>, index, index, index, index, index) -> !fir.box<!fir.array<?xi32>>
+  %3 = fir.embox %arr (%1) [%2] : (!fir.ref<!fir.array<?xi32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<?xi32>>
+  fir.has_value %3 : !fir.box<!fir.array<?xi32>>
+}
diff --git a/flang/test/Fir/fir-ops.fir b/flang/test/Fir/fir-ops.fir
index 6b7602513124a020e5135b36812b13639ba13cb6..775b09eb209fe662bf31339749e4f05fd5a38bda 100644
--- a/flang/test/Fir/fir-ops.fir
+++ b/flang/test/Fir/fir-ops.fir
@@ -442,44 +442,44 @@ fir.dispatch_table @dispatch_tbl {
 }
 
 // CHECK-LABEL: func @compare_real(
-// CHECK-SAME: [[VAL_133:%.*]]: !fir.real<16>, [[VAL_134:%.*]]: !fir.real<16>) {
-func @compare_real(%a : !fir.real<16>, %b : !fir.real<16>) {
-
-// CHECK: [[VAL_135:%.*]] = fir.cmpf "false", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_136:%.*]] = fir.cmpf "oeq", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_137:%.*]] = fir.cmpf "ogt", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_138:%.*]] = fir.cmpf "oge", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-  %d0 = fir.cmpf "false", %a, %b : !fir.real<16>
-  %d1 = fir.cmpf "oeq", %a, %b : !fir.real<16>
-  %d2 = fir.cmpf "ogt", %a, %b : !fir.real<16>
-  %d3 = fir.cmpf "oge", %a, %b : !fir.real<16>
-
-// CHECK: [[VAL_139:%.*]] = fir.cmpf "olt", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_140:%.*]] = fir.cmpf "ole", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_141:%.*]] = fir.cmpf "one", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_142:%.*]] = fir.cmpf "ord", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-  %a0 = fir.cmpf "olt", %a, %b : !fir.real<16>
-  %a1 = fir.cmpf "ole", %a, %b : !fir.real<16>
-  %a2 = fir.cmpf "one", %a, %b : !fir.real<16>
-  %a3 = fir.cmpf "ord", %a, %b : !fir.real<16>
-
-// CHECK: [[VAL_143:%.*]] = fir.cmpf "ueq", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_144:%.*]] = fir.cmpf "ugt", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_145:%.*]] = fir.cmpf "uge", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_146:%.*]] = fir.cmpf "ult", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-  %b0 = fir.cmpf "ueq", %a, %b : !fir.real<16>
-  %b1 = fir.cmpf "ugt", %a, %b : !fir.real<16>
-  %b2 = fir.cmpf "uge", %a, %b : !fir.real<16>
-  %b3 = fir.cmpf "ult", %a, %b : !fir.real<16>
-
-// CHECK: [[VAL_147:%.*]] = fir.cmpf "ule", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_148:%.*]] = fir.cmpf "une", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_149:%.*]] = fir.cmpf "uno", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-// CHECK: [[VAL_150:%.*]] = fir.cmpf "true", [[VAL_133]], [[VAL_134]] : !fir.real<16>
-  %c0 = fir.cmpf "ule", %a, %b : !fir.real<16>
-  %c1 = fir.cmpf "une", %a, %b : !fir.real<16>
-  %c2 = fir.cmpf "uno", %a, %b : !fir.real<16>
-  %c3 = fir.cmpf "true", %a, %b : !fir.real<16>
+// CHECK-SAME: [[VAL_133:%.*]]: f128, [[VAL_134:%.*]]: f128) {
+func @compare_real(%a : f128, %b : f128) {
+
+// CHECK: [[VAL_135:%.*]] = fir.cmpf "false", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_136:%.*]] = fir.cmpf "oeq", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_137:%.*]] = fir.cmpf "ogt", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_138:%.*]] = fir.cmpf "oge", [[VAL_133]], [[VAL_134]] : f128
+  %d0 = fir.cmpf "false", %a, %b : f128
+  %d1 = fir.cmpf "oeq", %a, %b : f128
+  %d2 = fir.cmpf "ogt", %a, %b : f128
+  %d3 = fir.cmpf "oge", %a, %b : f128
+
+// CHECK: [[VAL_139:%.*]] = fir.cmpf "olt", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_140:%.*]] = fir.cmpf "ole", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_141:%.*]] = fir.cmpf "one", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_142:%.*]] = fir.cmpf "ord", [[VAL_133]], [[VAL_134]] : f128
+  %a0 = fir.cmpf "olt", %a, %b : f128
+  %a1 = fir.cmpf "ole", %a, %b : f128
+  %a2 = fir.cmpf "one", %a, %b : f128
+  %a3 = fir.cmpf "ord", %a, %b : f128
+
+// CHECK: [[VAL_143:%.*]] = fir.cmpf "ueq", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_144:%.*]] = fir.cmpf "ugt", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_145:%.*]] = fir.cmpf "uge", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_146:%.*]] = fir.cmpf "ult", [[VAL_133]], [[VAL_134]] : f128
+  %b0 = fir.cmpf "ueq", %a, %b : f128
+  %b1 = fir.cmpf "ugt", %a, %b : f128
+  %b2 = fir.cmpf "uge", %a, %b : f128
+  %b3 = fir.cmpf "ult", %a, %b : f128
+
+// CHECK: [[VAL_147:%.*]] = fir.cmpf "ule", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_148:%.*]] = fir.cmpf "une", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_149:%.*]] = fir.cmpf "uno", [[VAL_133]], [[VAL_134]] : f128
+// CHECK: [[VAL_150:%.*]] = fir.cmpf "true", [[VAL_133]], [[VAL_134]] : f128
+  %c0 = fir.cmpf "ule", %a, %b : f128
+  %c1 = fir.cmpf "une", %a, %b : f128
+  %c2 = fir.cmpf "uno", %a, %b : f128
+  %c3 = fir.cmpf "true", %a, %b : f128
 
 // CHECK: return
 // CHECK: }
@@ -531,28 +531,28 @@ func @compare_complex(%a : !fir.complex<16>, %b : !fir.complex<16>) {
 }
 
 // CHECK-LABEL: func @arith_real(
-// CHECK-SAME: [[VAL_169:%.*]]: !fir.real<16>, [[VAL_170:%.*]]: !fir.real<16>) -> !fir.real<16> {
-func @arith_real(%a : !fir.real<16>, %b : !fir.real<16>) -> !fir.real<16> {
+// CHECK-SAME: [[VAL_169:%.*]]: f128, [[VAL_170:%.*]]: f128) -> f128 {
+func @arith_real(%a : f128, %b : f128) -> f128 {
 
 // CHECK: [[VAL_171:%.*]] = constant 1.0
-// CHECK: [[VAL_172:%.*]] = fir.convert [[VAL_171]] : (f32) -> !fir.real<16>
-// CHECK: [[VAL_173:%.*]] = fir.negf [[VAL_169]] : !fir.real<16>
-// CHECK: [[VAL_174:%.*]] = fir.addf [[VAL_172]], [[VAL_173]] : !fir.real<16>
-// CHECK: [[VAL_175:%.*]] = fir.subf [[VAL_174]], [[VAL_170]] : !fir.real<16>
-// CHECK: [[VAL_176:%.*]] = fir.mulf [[VAL_173]], [[VAL_175]] : !fir.real<16>
-// CHECK: [[VAL_177:%.*]] = fir.divf [[VAL_176]], [[VAL_169]] : !fir.real<16>
-// CHECK: [[VAL_178:%.*]] = fir.modf [[VAL_177]], [[VAL_170]] : !fir.real<16>
+// CHECK: [[VAL_172:%.*]] = fir.convert [[VAL_171]] : (f32) -> f128
+// CHECK: [[VAL_173:%.*]] = fir.negf [[VAL_169]] : f128
+// CHECK: [[VAL_174:%.*]] = addf [[VAL_172]], [[VAL_173]] : f128
+// CHECK: [[VAL_175:%.*]] = subf [[VAL_174]], [[VAL_170]] : f128
+// CHECK: [[VAL_176:%.*]] = mulf [[VAL_173]], [[VAL_175]] : f128
+// CHECK: [[VAL_177:%.*]] = divf [[VAL_176]], [[VAL_169]] : f128
+// CHECK: [[VAL_178:%.*]] = fir.modf [[VAL_177]], [[VAL_170]] : f128
   %c1 = constant 1.0 : f32
-  %0 = fir.convert %c1 : (f32) -> !fir.real<16>
-  %1 = fir.negf %a : !fir.real<16>
-  %2 = fir.addf %0, %1 : !fir.real<16>
-  %3 = fir.subf %2, %b : !fir.real<16>
-  %4 = fir.mulf %1, %3 : !fir.real<16>
-  %5 = fir.divf %4, %a : !fir.real<16>
-  %6 = fir.modf %5, %b : !fir.real<16>
-// CHECK: return [[VAL_178]] : !fir.real<16>
+  %0 = fir.convert %c1 : (f32) -> f128
+  %1 = fir.negf %a : f128
+  %2 = addf %0, %1 : f128
+  %3 = subf %2, %b : f128
+  %4 = mulf %1, %3 : f128
+  %5 = divf %4, %a : f128
+  %6 = fir.modf %5, %b : f128
+// CHECK: return [[VAL_178]] : f128
 // CHECK: }
-  return %6 : !fir.real<16>
+  return %6 : f128
 }
 
 // CHECK-LABEL: func @arith_complex(
diff --git a/flang/test/Semantics/data05.f90 b/flang/test/Semantics/data05.f90
index ff4d0686ffc7be96b63befbf2f7433c6aefb6546..8e059c2c06527e898833bb7ed9d4ec286b5cb57e 100644
--- a/flang/test/Semantics/data05.f90
+++ b/flang/test/Semantics/data05.f90
@@ -73,15 +73,15 @@ module m
   end function
   subroutine s11
     real, target, save :: arr(3,4) ! CHECK: arr, SAVE, TARGET size=48 offset=0: ObjectEntity type: REAL(4) shape: 1_8:3_8,1_8:4_8
-    type(t1) :: d1 = t1(1,reshape([1,2,3,4],[2,2]),(6.,7.),.false.,'ab',arr,ifunc2,rfunc,extrfunc) ! CHECK: d1 size=192 offset=48: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(c=[CHARACTER(KIND=1,LEN=1)::"a","a"],ifptr=ifunc2,j=1_4,rp=rfunc,t=.false._4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),xp=arr,xrp=extrfunc,z=(6._4,7._4))
+    type(t1) :: d1 = t1(1,reshape([1,2,3,4],[2,2]),(6.,7.),.false.,'ab',arr,ifunc2,rfunc,extrfunc) ! CHECK: d1 size=184 offset=48: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(j=1_4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),z=(6._4,7._4),t=.false._4,c=[CHARACTER(KIND=1,LEN=1)::"a","a"],xp=arr,ifptr=ifunc2,rp=rfunc,xrp=extrfunc)
     type(t1(4,len=1)) :: d2 = t1(4)(xrp=extrfunc,rp=rfunc,ifptr=ifunc2,xp=arr,c='a&
-      &b',t=.false.,z=(6.,7.),x=reshape([1,2,3,4],[2,2]),j=1) ! CHECK: d2 size=192 offset=240: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(c=[CHARACTER(KIND=1,LEN=1)::"a","a"],ifptr=ifunc2,j=1_4,rp=rfunc,t=.false._4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),xp=arr,xrp=extrfunc,z=(6._4,7._4))
-    type(t1(2+2)) :: d3 ! CHECK: d3 (InDataStmt) size=192 offset=432: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(c=[CHARACTER(KIND=1,LEN=1)::"a","a"],ifptr=ifunc2,j=1_4,rp=rfunc,t=.false._4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),xp=arr,xrp=extrfunc,z=(6._4,7._4))
+      &b',t=.false.,z=(6.,7.),x=reshape([1,2,3,4],[2,2]),j=1) ! CHECK: d2 size=184 offset=232: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(j=1_4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),z=(6._4,7._4),t=.false._4,c=[CHARACTER(KIND=1,LEN=1)::"a","a"],xp=arr,ifptr=ifunc2,rp=rfunc,xrp=extrfunc)
+    type(t1(2+2)) :: d3 ! CHECK: d3 (InDataStmt) size=184 offset=416: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(j=1_4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),z=(6._4,7._4),t=.false._4,c=[CHARACTER(KIND=1,LEN=1)::"a","a"],xp=arr,ifptr=ifunc2,rp=rfunc,xrp=extrfunc)
     data d3/t1(1,reshape([1,2,3,4],[2,2]),(6.,7.),.false.,'ab',arr,ifunc2,rfunc,extrfunc)/
-    type(t1) :: d4 ! CHECK: d4 (InDataStmt) size=192 offset=624: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(c=[CHARACTER(KIND=1,LEN=1)::"a","a"],ifptr=ifunc2,j=1_4,rp=rfunc,t=.false._4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),xp=arr,xrp=extrfunc,z=(6._4,7._4))
+    type(t1) :: d4 ! CHECK: d4 (InDataStmt) size=184 offset=600: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(j=1_4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),z=(6._4,7._4),t=.false._4,c=[CHARACTER(KIND=1,LEN=1)::"a","a"],xp=arr,ifptr=ifunc2,rp=rfunc,xrp=extrfunc)
     data d4/t1(4)(xrp=extrfunc,rp=rfunc,ifptr=ifunc2,xp=arr,c='ab',t=.false.,z=(6&
       &.,7.),x=reshape([1,2,3,4],[2,2]),j=1)/
-    type(t1) :: d5 ! CHECK: d5 (InDataStmt) size=192 offset=816: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(c=[CHARACTER(KIND=1,LEN=1)::"a","b"],ifptr=ifunc2,j=1_4,rp=rfunc,t=.false._4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),xp=arr,xrp=extrfunc,z=(6._4,7._4))
+    type(t1) :: d5 ! CHECK: d5 (InDataStmt) size=184 offset=784: ObjectEntity type: TYPE(t1(kind=4_1,len=1_2)) init:t1(kind=4_1,len=1_2)(j=1_4,x=reshape([REAL(4)::1._4,2._4,3._4,4._4],shape=[2,2]),z=(6._4,7._4),t=.false._4,c=[CHARACTER(KIND=1,LEN=1)::"a","b"],xp=arr,ifptr=ifunc2,rp=rfunc,xrp=extrfunc)
     data d5%j/1/,d5%x/1,2,3,4/,d5%z%re/6./,d5%z%im/7./,d5%t/.false./,d5%c(1:1)/'a'/,d5%c(2:&
       &2)/'b'/,d5%xp/arr/,d5%ifptr/ifunc2/,d5%rp/rfunc/,d5%xrp/extrfunc/
   end subroutine
diff --git a/flang/test/Semantics/getsymbols01.f90 b/flang/test/Semantics/getsymbols01.f90
index d26aa774ace40993e93a9e7dd5850feed10fb8ad..9a52ee7cbf2ab99dfaba61e4bfd3ed4b916e6e10 100644
--- a/flang/test/Semantics/getsymbols01.f90
+++ b/flang/test/Semantics/getsymbols01.f90
@@ -15,7 +15,7 @@ contains
  end function
 end module
 
-! RUN: %f18 -fget-symbols-sources -fsyntax-only %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fsyntax-only -fget-symbols-sources %s 2>&1 | FileCheck %s
 ! CHECK-COUNT-1:f:{{.*}}getsymbols01.f90, 12, 26-27
 ! CHECK-COUNT-1:mm1:{{.*}}getsymbols01.f90, 2, 8-11
 ! CHECK-COUNT-1:s:{{.*}}getsymbols01.f90, 5, 18-19
diff --git a/flang/test/Semantics/getsymbols02.f90 b/flang/test/Semantics/getsymbols02.f90
index 1667548f81c3bae5b5b03b3c4d4637300b25e44e..32929904fb7a4d9d4cfa27893cec26d4382bd08f 100644
--- a/flang/test/Semantics/getsymbols02.f90
+++ b/flang/test/Semantics/getsymbols02.f90
@@ -7,8 +7,8 @@ PROGRAM helloworld
     i = callget5()
 ENDPROGRAM
 
-! RUN: %f18 -fsyntax-only %S/Inputs/getsymbols02-a.f90
-! RUN: %f18 -fsyntax-only %S/Inputs/getsymbols02-b.f90
-! RUN: %f18 -fget-symbols-sources -fsyntax-only %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fsyntax-only %S/Inputs/getsymbols02-a.f90
+! RUN: %flang_fc1 -fsyntax-only %S/Inputs/getsymbols02-b.f90
+! RUN: %flang_fc1 -fsyntax-only -fget-symbols-sources %s 2>&1 | FileCheck %s
 ! CHECK: callget5: .{{[/\\]}}mm2b.mod,
 ! CHECK: get5: .{{[/\\]}}mm2a.mod,
diff --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index fddf513bcc5112fe0eeacfb11dbefd22c74d9825..0bc19b4fe8d08b3162530997a2c86866f6c3969e 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -7,7 +7,7 @@ program main
  x = f
 end program
 
-! RUN: %f18 -fget-symbols-sources -fsyntax-only %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fsyntax-only -fget-symbols-sources %s 2>&1 | FileCheck %s
 ! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
 ! CHECK:main:{{.*}}getsymbols03-a.f90, 4, 9-13
 ! CHECK:mm3:{{.*}}getsymbols03-a.f90, 5, 6-9
diff --git a/flang/test/Semantics/getsymbols04.f90 b/flang/test/Semantics/getsymbols04.f90
index ac8f2d0a7e44c91a8e1a8ed261c035ce35150389..28027ea759b64c01821334ad7735be685d23bd86 100644
--- a/flang/test/Semantics/getsymbols04.f90
+++ b/flang/test/Semantics/getsymbols04.f90
@@ -6,7 +6,7 @@ program main
   x = y
 end program
 
-! RUN: %f18 -fget-symbols-sources -fsyntax-only %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fsyntax-only -fget-symbols-sources %s 2>&1 | FileCheck %s
 ! CHECK:x:{{.*}}getsymbols04.f90, 3, 14-15
 ! CHECK:x:{{.*}}getsymbols04.f90, 5, 11-12
 ! CHECK:y:{{.*}}getsymbols04.f90, 4, 14-15
diff --git a/flang/test/Semantics/getsymbols05.f90 b/flang/test/Semantics/getsymbols05.f90
index 6b07678e42d0e61eb34e8bad2d73a21d3735afde..99771e227c3fe9250189ac3631e03b58c6170078 100644
--- a/flang/test/Semantics/getsymbols05.f90
+++ b/flang/test/Semantics/getsymbols05.f90
@@ -9,7 +9,7 @@ program main
   x = y
 end program
 
-! RUN: %f18 -fget-symbols-sources -fsyntax-only %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fsyntax-only -fget-symbols-sources %s 2>&1 | FileCheck %s
 ! CHECK:x:{{.*}}getsymbols05.f90, 3, 14-15
 ! CHECK:x:{{.*}}getsymbols05.f90, 6, 16-17
 ! CHECK:y:{{.*}}getsymbols05.f90, 4, 14-15
diff --git a/flang/test/Semantics/modfile04.f90 b/flang/test/Semantics/modfile04.f90
index bc4d8d4895ad296ae2e30840563b54484f82fcb5..9312b756513c52ccad235ab3b3a33570fa9c7478 100644
--- a/flang/test/Semantics/modfile04.f90
+++ b/flang/test/Semantics/modfile04.f90
@@ -6,7 +6,7 @@ module m1
   end type
 contains
 
-  pure subroutine s(x, y) bind(c)
+  pure subroutine Ss(x, y) bind(c)
     logical x
     intent(inout) y
     intent(in) x
@@ -53,7 +53,7 @@ end module m3
 !type::t
 !end type
 !contains
-!pure subroutine s(x,y) bind(c)
+!pure subroutine ss(x,y) bind(c, name="ss")
 !logical(4),intent(in)::x
 !real(4),intent(inout)::y
 !end
diff --git a/flang/test/Semantics/modfile21.f90 b/flang/test/Semantics/modfile21.f90
index d7b45f70c00d8c2ffd98ae188e514e5aac2b6803..73cf59f827a20b481e4f8a8d6b5653a1db040838 100644
--- a/flang/test/Semantics/modfile21.f90
+++ b/flang/test/Semantics/modfile21.f90
@@ -26,10 +26,10 @@ end
 !  real(4)::v
 !  complex(4)::w
 !  real(4)::cb
-!  common//t,w,u,v
 !  common/cb/x,y,z
 !  bind(c, name="CB")::/cb/
 !  common/cb2/a,b,c
-!  bind(c)::/cb2/
+!  bind(c, name="cb2")::/cb2/
 !  common/b/cb
+!  common//t,w,u,v
 !end
diff --git a/flang/test/Semantics/omp-ordered-simd.f90 b/flang/test/Semantics/omp-ordered-simd.f90
new file mode 100644
index 0000000000000000000000000000000000000000..d597191650e7845f31e2ec2c292f03083b0b3941
--- /dev/null
+++ b/flang/test/Semantics/omp-ordered-simd.f90
@@ -0,0 +1,95 @@
+! RUN: %S/test_errors.sh %s %t %flang -fopenmp
+! OpenMP Version 4.5
+! Various checks with the ordered construct
+
+SUBROUTINE WORK(I)
+  INTEGER I
+END SUBROUTINE WORK
+
+SUBROUTINE ORDERED_GOOD(N)
+  INTEGER N, I, A(10), B(10), C(10) 
+  !$OMP SIMD
+  DO I = 1,N
+    IF (I <= 10) THEN
+      !$OMP ORDERED SIMD
+      CALL WORK(I)
+      !$OMP END ORDERED
+    ENDIF
+  END DO
+  !$OMP END SIMD
+END SUBROUTINE ORDERED_GOOD
+
+SUBROUTINE ORDERED_BAD(N)
+  INTEGER N, I, A(10), B(10), C(10)
+
+  !$OMP DO SIMD
+  DO I = 1,N
+    IF (I <= 10) THEN
+      !ERROR: The ORDERED clause must be present on the loop construct if any ORDERED region ever binds to a loop region arising from the loop construct.
+      !$OMP ORDERED 
+      CALL WORK(I)
+      !$OMP END ORDERED
+    ENDIF
+  END DO
+  !$OMP END DO SIMD
+
+  !$OMP PARALLEL DO
+  DO I = 1,N
+    IF (I <= 10) THEN
+      !ERROR: The ORDERED clause must be present on the loop construct if any ORDERED region ever binds to a loop region arising from the loop construct.
+      !$OMP ORDERED 
+      CALL WORK(I)
+      !$OMP END ORDERED
+    ENDIF
+  END DO
+  !$OMP END PARALLEL DO
+
+  !$OMP CRITICAL  
+  DO I = 1,N
+    IF (I <= 10) THEN
+      !ERROR: `ORDERED` region may not be closely nested inside of `CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region.
+      !$OMP ORDERED 
+      CALL WORK(I)
+      !$OMP END ORDERED
+    ENDIF
+  END DO
+  !$OMP END CRITICAL
+
+  !$OMP CRITICAL
+    WRITE(*,*) I
+    !ERROR: `ORDERED` region may not be closely nested inside of `CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region.
+    !$OMP ORDERED 
+    CALL WORK(I)
+    !$OMP END ORDERED
+  !$OMP END CRITICAL
+
+  !$OMP ORDERED 
+    WRITE(*,*) I
+    IF (I <= 10) THEN
+      !ERROR: `ORDERED` region may not be closely nested inside of `CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region.
+      !$OMP ORDERED 
+      CALL WORK(I)
+      !$OMP END ORDERED
+    ENDIF
+  !$OMP END ORDERED
+
+  !$OMP TASK  
+    C =  C - A * B
+    !ERROR: `ORDERED` region may not be closely nested inside of `CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region.
+    !$OMP ORDERED 
+    CALL WORK(I)
+    !$OMP END ORDERED
+  !$OMP END TASK
+
+  !$OMP TASKLOOP 
+  DO I = 1,N
+    IF (I <= 10) THEN
+      !ERROR: `ORDERED` region may not be closely nested inside of `CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region.
+      !$OMP ORDERED 
+      CALL WORK(I)
+      !$OMP END ORDERED
+    ENDIF
+  END DO
+  !$OMP END TASKLOOP
+
+END SUBROUTINE ORDERED_BAD
diff --git a/flang/test/Semantics/resolve102.f90 b/flang/test/Semantics/resolve102.f90
index fec8314641e45772ad82c8044410f59c102840ea..c5b3f53bbdc44b9de9214b78066645a4dbb1d068 100644
--- a/flang/test/Semantics/resolve102.f90
+++ b/flang/test/Semantics/resolve102.f90
@@ -9,7 +9,7 @@ subroutine sub(p2)
 end subroutine
 
 subroutine circular
-  !ERROR: Procedure 'p' is recursively defined.  Procedures in the cycle: 'sub', 'p', 'p2'
+  !ERROR: Procedure 'p' is recursively defined.  Procedures in the cycle: 'p', 'sub', 'p2'
   procedure(sub) :: p
 
   call p(sub)
@@ -21,7 +21,7 @@ subroutine circular
 end subroutine circular
 
 program iface
-  !ERROR: Procedure 'p' is recursively defined.  Procedures in the cycle: 'sub', 'p', 'p2'
+  !ERROR: Procedure 'p' is recursively defined.  Procedures in the cycle: 'p', 'sub', 'p2'
   procedure(sub) :: p
   interface
     subroutine sub(p2)
@@ -38,7 +38,7 @@ Program mutual
   Call p(sub)
 
   contains
-    !ERROR: Procedure 'sub1' is recursively defined.  Procedures in the cycle: 'sub1', 'p', 'arg'
+    !ERROR: Procedure 'sub1' is recursively defined.  Procedures in the cycle: 'p', 'sub1', 'arg'
     Subroutine sub1(arg)
       procedure(sub1) :: arg
     End Subroutine
@@ -54,7 +54,7 @@ Program mutual1
   Call p(sub)
 
   contains
-    !ERROR: Procedure 'sub1' is recursively defined.  Procedures in the cycle: 'sub1', 'sub', 'p', 'arg', 'p2'
+    !ERROR: Procedure 'sub1' is recursively defined.  Procedures in the cycle: 'p', 'sub1', 'arg', 'sub', 'p2'
     Subroutine sub1(arg)
       procedure(sub) :: arg
     End Subroutine
diff --git a/flang/test/Semantics/resolve18.f90 b/flang/test/Semantics/resolve18.f90
index 94b217e248f031656b7554267d0533902e737dda..204f6e7b383d6c921188ad7874c66a63ef9e2eca 100644
--- a/flang/test/Semantics/resolve18.f90
+++ b/flang/test/Semantics/resolve18.f90
@@ -63,6 +63,15 @@ contains
   function foo(x)
   end
 end
+module m4c
+  type :: foo
+  end type
+  interface foo
+    !ERROR: 'foo' is already declared in this scoping unit
+    real function foo()
+    end function foo
+  end interface foo
+end
 
 ! Use associating a name that is a generic and a derived type
 module m5a
@@ -85,3 +94,30 @@ subroutine s5
   use m5b
   type(g) :: y
 end
+
+module m6
+  real :: f6
+  interface g6
+  !ERROR: Procedure 'f6' was previously declared
+    real function f6()
+    end function f6
+  end interface g6
+end module m6
+
+module m7
+  integer :: f7
+  interface g7
+    !ERROR: Procedure 'f7' was previously declared
+    real function f7()
+    end function f7
+  end interface g7
+end module m7
+
+module m8
+  real :: f8
+  interface g8
+    !ERROR: Procedure 'f8' was previously declared
+    subroutine f8()
+    end subroutine f8
+  end interface g8
+end module m8
diff --git a/flang/test/Semantics/separate-mp02.f90 b/flang/test/Semantics/separate-mp02.f90
index 6d620e71118b160dffe3a32ffd5077d525edf01f..3dd717dbc90a9ddd679102788d958489d2268cc6 100644
--- a/flang/test/Semantics/separate-mp02.f90
+++ b/flang/test/Semantics/separate-mp02.f90
@@ -136,6 +136,12 @@ module m2b
     end
     module subroutine s3() bind(c, name="s3")
     end
+    module subroutine s4() bind(c, name=" s4")
+    end
+    module subroutine s5() bind(c)
+    end
+    module subroutine s6() bind(c)
+    end
   end interface
 end
 
@@ -148,9 +154,16 @@ contains
   !ERROR: Module subprogram 's2' does not have a binding label but the corresponding interface body does
   module subroutine s2()
   end
-  !ERROR: Module subprogram 's3' has binding label "s3_xxx" but the corresponding interface body has "s3"
+  !ERROR: Module subprogram 's3' has binding label 's3_xxx' but the corresponding interface body has 's3'
   module subroutine s3() bind(c, name="s3" // suffix)
   end
+  module subroutine s4() bind(c, name="s4  ")
+  end
+  module subroutine s5() bind(c, name=" s5")
+  end
+  !ERROR: Module subprogram 's6' has binding label 'not_s6' but the corresponding interface body has 's6'
+  module subroutine s6() bind(c, name="not_s6")
+  end
 end
 
 
diff --git a/flang/test/Semantics/typeinfo01.f90 b/flang/test/Semantics/typeinfo01.f90
index 3e8b8181fe4a905572002df82707c7037bda1869..3575aca6e7a1ffd8eb4226db50cc54874a2e6d45 100644
--- a/flang/test/Semantics/typeinfo01.f90
+++ b/flang/test/Semantics/typeinfo01.f90
@@ -231,7 +231,7 @@ module m11
   subroutine s1(x)
 !CHECK: .b.t.1.allocatable, SAVE, TARGET: ObjectEntity type: TYPE(value) shape: 0_8:1_8,0_8:0_8 init:reshape([value::value(genre=1_1,value=0_8),value(genre=1_1,value=0_8)],shape=[2,1])
 !CHECK: .b.t.1.automatic, SAVE, TARGET: ObjectEntity type: TYPE(value) shape: 0_8:1_8,0_8:0_8 init:reshape([value::value(genre=2_1,value=1_8),value(genre=3_1,value=0_8)],shape=[2,1])
-!CHECK: .c.t.1, SAVE, TARGET: ObjectEntity type: TYPE(component) shape: 0_8:3_8 init:[component::component(name=.n.allocatable,genre=3_1,category=1_1,kind=4_1,rank=1_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=.b.t.1.allocatable,initialization=NULL()),component(name=.n.automatic,genre=4_1,category=1_1,kind=4_1,rank=1_1,offset=48_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=.b.t.1.automatic,initialization=NULL()),component(name=.n.chauto,genre=4_1,category=3_1,kind=1_1,rank=0_1,offset=96_8,characterlen=value(genre=3_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=NULL()),component(name=.n.pointer,genre=2_1,category=1_1,kind=4_1,rank=0_1,offset=120_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=target)]
+!CHECK: .c.t.1, SAVE, TARGET: ObjectEntity type: TYPE(component) shape: 0_8:3_8 init:[component::component(name=.n.allocatable,genre=3_1,category=1_1,kind=4_1,rank=1_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=.b.t.1.allocatable,initialization=NULL()),component(name=.n.automatic,genre=4_1,category=1_1,kind=4_1,rank=1_1,offset=96_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=.b.t.1.automatic,initialization=NULL()),component(name=.n.chauto,genre=4_1,category=3_1,kind=1_1,rank=0_1,offset=72_8,characterlen=value(genre=3_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=NULL()),component(name=.n.pointer,genre=2_1,category=1_1,kind=4_1,rank=0_1,offset=48_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=target)]
 !CHECK: .dt.t.1, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=144_8,parent=NULL(),uninstantiated=.dt.t,kindparameter=NULL(),lenparameterkind=.lpk.t.1,component=.c.t.1,procptr=NULL(),special=NULL())
 !CHECK: .lpk.t.1, SAVE, TARGET: ObjectEntity type: INTEGER(1) shape: 0_8:0_8 init:[INTEGER(1)::8_1]
     type(t(*)), intent(in) :: x
diff --git a/flang/tools/f18/f18.cpp b/flang/tools/f18/f18.cpp
index 5bc069cb5859ce4f7498689ec82774ed2b1d3917..e22905a86a9265dd52897fbfa4fa85800d485908 100644
--- a/flang/tools/f18/f18.cpp
+++ b/flang/tools/f18/f18.cpp
@@ -487,7 +487,8 @@ int main(int argc, char *const argv[]) {
       options.features.Enable(
           Fortran::common::LanguageFeature::BackslashEscapes, true);
     } else if (arg == "-Mstandard" || arg == "-std=f95" ||
-        arg == "-std=f2003" || arg == "-std=f2008" || arg == "-std=legacy") {
+        arg == "-std=f2003" || arg == "-std=f2008" || arg == "-std=legacy" ||
+        arg == "-std=f2018" || arg == "-pedantic") {
       driver.warnOnNonstandardUsage = true;
     } else if (arg == "-fopenacc") {
       options.features.Enable(Fortran::common::LanguageFeature::OpenACC);
diff --git a/flang/tools/fir-opt/fir-opt.cpp b/flang/tools/fir-opt/fir-opt.cpp
index b2d383c066821c40ebe86a24fd572051ddc5620f..b66294339f1a7857b7664941e9c09f56f9fb22b4 100644
--- a/flang/tools/fir-opt/fir-opt.cpp
+++ b/flang/tools/fir-opt/fir-opt.cpp
@@ -17,9 +17,9 @@
 using namespace mlir;
 
 int main(int argc, char **argv) {
-  fir::support::registerFIRPasses();
+  fir::support::registerMLIRPassesForFortranTools();
   DialectRegistry registry;
   fir::support::registerDialects(registry);
   return failed(MlirOptMain(argc, argv, "FIR modular optimizer driver\n",
-      registry, /*preloadDialectsInContext*/ false));
+      registry, /*preloadDialectsInContext=*/false));
 }
diff --git a/flang/tools/tco/tco.cpp b/flang/tools/tco/tco.cpp
index a67b1453fc28ddf0a9ba185b658176ee459f521b..62e31fe47ed14e0aa90545ab84924cd9fe897013 100644
--- a/flang/tools/tco/tco.cpp
+++ b/flang/tools/tco/tco.cpp
@@ -106,7 +106,7 @@ compileFIR(const mlir::PassPipelineCLParser &passPipeline) {
 }
 
 int main(int argc, char **argv) {
-  fir::support::registerFIRPasses();
+  fir::support::registerMLIRPassesForFortranTools();
   [[maybe_unused]] InitLLVM y(argc, argv);
   mlir::registerPassManagerCLOptions();
   mlir::PassPipelineCLParser passPipe("", "Compiler passes to run");
diff --git a/flang/unittests/Evaluate/intrinsics.cpp b/flang/unittests/Evaluate/intrinsics.cpp
index 52507b8ef8b675aae5797449829bcd1f399b4840..a36dbf5818220eecb0198b6c2b2bec8bdd4d15a6 100644
--- a/flang/unittests/Evaluate/intrinsics.cpp
+++ b/flang/unittests/Evaluate/intrinsics.cpp
@@ -24,7 +24,7 @@ public:
     offsets_[s] = cooked_.Put(s);
     cooked_.PutProvenance(allSources_.AddCompilerInsertion(s));
   }
-  void Marshal() { cooked_.Marshal(allSources_); }
+  void Marshal() { cooked_.Marshal(allCookedSources_); }
   parser::CharBlock operator()(const std::string &s) {
     return {cooked_.AsCharBlock().begin() + offsets_[s], s.size()};
   }
diff --git a/flang/unittests/Runtime/CMakeLists.txt b/flang/unittests/Runtime/CMakeLists.txt
index b80eceac74466352bd5d5ac7687980ac74513543..616c86f279fe735573a81d5332084ad64e5a7752 100644
--- a/flang/unittests/Runtime/CMakeLists.txt
+++ b/flang/unittests/Runtime/CMakeLists.txt
@@ -46,3 +46,8 @@ add_flang_nongtest_unittest(list-input
   RuntimeTesting
   FortranRuntime
 )
+
+add_flang_nongtest_unittest(buffer
+  RuntimeTesting
+  FortranRuntime
+)
diff --git a/flang/unittests/Runtime/buffer.cpp b/flang/unittests/Runtime/buffer.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f5eca0338939f728749ab7771a2fd40e65d22f95
--- /dev/null
+++ b/flang/unittests/Runtime/buffer.cpp
@@ -0,0 +1,115 @@
+#include "../../runtime/buffer.h"
+#include "testing.h"
+#include <algorithm>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+
+static constexpr std::size_t tinyBuffer{32};
+using FileOffset = std::int64_t;
+using namespace Fortran::runtime;
+using namespace Fortran::runtime::io;
+
+class Store : public FileFrame<Store, tinyBuffer> {
+public:
+  explicit Store(std::size_t bytes = 65536) : bytes_{bytes} {
+    data_.reset(new char[bytes]);
+    std::memset(&data_[0], 0, bytes);
+  }
+  std::size_t bytes() const { return bytes_; }
+  void set_enforceSequence(bool yes = true) { enforceSequence_ = yes; }
+  void set_expect(FileOffset to) { expect_ = to; }
+
+  std::size_t Read(FileOffset at, char *to, std::size_t minBytes,
+      std::size_t maxBytes, IoErrorHandler &handler) {
+    if (enforceSequence_ && at != expect_) {
+      handler.SignalError("Read(%d,%d,%d) not at expected %d",
+          static_cast<int>(at), static_cast<int>(minBytes),
+          static_cast<int>(maxBytes), static_cast<int>(expect_));
+    } else if (at < 0 || at + minBytes > bytes_) {
+      handler.SignalError("Read(%d,%d,%d) is out of bounds",
+          static_cast<int>(at), static_cast<int>(minBytes),
+          static_cast<int>(maxBytes));
+    }
+    auto result{std::min<std::size_t>(maxBytes, bytes_ - at)};
+    std::memcpy(to, &data_[at], result);
+    expect_ = at + result;
+    return result;
+  }
+  std::size_t Write(FileOffset at, const char *from, std::size_t bytes,
+      IoErrorHandler &handler) {
+    if (enforceSequence_ && at != expect_) {
+      handler.SignalError("Write(%d,%d) not at expected %d",
+          static_cast<int>(at), static_cast<int>(bytes),
+          static_cast<int>(expect_));
+    } else if (at < 0 || at + bytes > bytes_) {
+      handler.SignalError("Write(%d,%d) is out of bounds", static_cast<int>(at),
+          static_cast<int>(bytes));
+    }
+    std::memcpy(&data_[at], from, bytes);
+    expect_ = at + bytes;
+    return bytes;
+  }
+
+private:
+  std::size_t bytes_;
+  std::unique_ptr<char[]> data_;
+  bool enforceSequence_{false};
+  FileOffset expect_{0};
+};
+
+inline int ChunkSize(int j, int most) {
+  // 31, 1, 29, 3, 27, ...
+  j %= tinyBuffer;
+  auto chunk{
+      static_cast<int>(((j % 2) ? j : (tinyBuffer - 1 - j)) % tinyBuffer)};
+  return std::min(chunk, most);
+}
+
+inline int ValueFor(int at) { return (at ^ (at >> 8)) & 0xff; }
+
+int main() {
+  StartTests();
+  Terminator terminator{__FILE__, __LINE__};
+  IoErrorHandler handler{terminator};
+  Store store;
+  store.set_enforceSequence(true);
+  const auto bytes{static_cast<FileOffset>(store.bytes())};
+  // Fill with an assortment of chunks
+  int at{0}, j{0};
+  while (at < bytes) {
+    auto chunk{ChunkSize(j, static_cast<int>(bytes - at))};
+    store.WriteFrame(at, chunk, handler);
+    char *to{store.Frame()};
+    for (int k{0}; k < chunk; ++k) {
+      to[k] = ValueFor(at + k);
+    }
+    at += chunk;
+    ++j;
+  }
+  store.Flush(handler);
+  // Validate
+  store.set_expect(0);
+  at = 0;
+  while (at < bytes) {
+    auto chunk{ChunkSize(j, static_cast<int>(bytes - at))};
+    std::size_t frame{store.ReadFrame(at, chunk, handler)};
+    if (frame < static_cast<std::size_t>(chunk)) {
+      Fail() << "Badly-sized ReadFrame at " << at << ", chunk=" << chunk
+             << ", got " << frame << '\n';
+      break;
+    }
+    const char *from{store.Frame()};
+    for (int k{0}; k < chunk; ++k) {
+      auto expect{static_cast<char>(ValueFor(at + k))};
+      if (from[k] != expect) {
+        Fail() << "At " << at << '+' << k << '(' << (at + k) << "), read "
+               << (from[k] & 0xff) << ", expected " << static_cast<int>(expect)
+               << '\n';
+      }
+    }
+    at += chunk;
+    ++j;
+  }
+  return EndTests();
+}
diff --git a/flang/unittests/RuntimeGTest/CMakeLists.txt b/flang/unittests/RuntimeGTest/CMakeLists.txt
index 77aff3069f14bcaeb6c8e8620467a44304b6f466..f26cb44be5fee1201943c1d9cc9cb75a18c3af42 100644
--- a/flang/unittests/RuntimeGTest/CMakeLists.txt
+++ b/flang/unittests/RuntimeGTest/CMakeLists.txt
@@ -1,5 +1,7 @@
 add_flang_unittest(FlangRuntimeTests
   CharacterTest.cpp
+  RuntimeCrashTest.cpp
+  CrashHandlerFixture.cpp
 )
 
 target_link_libraries(FlangRuntimeTests
diff --git a/flang/unittests/RuntimeGTest/CrashHandlerFixture.cpp b/flang/unittests/RuntimeGTest/CrashHandlerFixture.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..315a555789e74d7ce49d83f73e03f4b3c5df0e16
--- /dev/null
+++ b/flang/unittests/RuntimeGTest/CrashHandlerFixture.cpp
@@ -0,0 +1,34 @@
+//===-- flang/unittests/RuntimeGTest/CrashHandlerFixture.cpp ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "CrashHandlerFixture.h"
+#include "../../runtime/terminator.h"
+#include <cstdarg>
+#include <cstdlib>
+
+// Replaces Fortran runtime's crash handler so we can verify the crash message
+[[noreturn]] static void CatchCrash(
+    const char *sourceFile, int sourceLine, const char *message, va_list &ap) {
+  char buffer[1000];
+  std::vsnprintf(buffer, sizeof buffer, message, ap);
+  va_end(ap);
+  llvm::errs()
+      << "Test "
+      << ::testing::UnitTest::GetInstance()->current_test_info()->name()
+      << " crashed in file "
+      << (sourceFile ? sourceFile : "unknown source file") << '(' << sourceLine
+      << "): " << buffer << '\n';
+  std::exit(EXIT_FAILURE);
+}
+
+// Register the crash handler above when creating each unit test in this suite
+void CrashHandlerFixture::SetUp() {
+  static bool isCrashHanlderRegistered{false};
+  if (not isCrashHanlderRegistered)
+    Fortran::runtime::Terminator::RegisterCrashHandler(CatchCrash);
+  isCrashHanlderRegistered = true;
+}
diff --git a/flang/unittests/RuntimeGTest/CrashHandlerFixture.h b/flang/unittests/RuntimeGTest/CrashHandlerFixture.h
new file mode 100644
index 0000000000000000000000000000000000000000..d368c6fb55ba4f4b6f4e436ba8ef5b69d3f26618
--- /dev/null
+++ b/flang/unittests/RuntimeGTest/CrashHandlerFixture.h
@@ -0,0 +1,21 @@
+//===-- flang/unittests/RuntimeGTest/CrashHandlerFixture.h ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Test fixture registers a custom crash handler to ensure death tests fail
+/// with expected message.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_FLANG_UNITTESTS_RUNTIMEGTEST_CRASHHANDLERFIXTURE_H
+#define LLVM_FLANG_UNITTESTS_RUNTIMEGTEST_CRASHHANDLERFIXTURE_H
+#include <gtest/gtest.h>
+
+struct CrashHandlerFixture : testing::Test {
+  void SetUp();
+};
+
+#endif
diff --git a/flang/unittests/RuntimeGTest/RuntimeCrashTest.cpp b/flang/unittests/RuntimeGTest/RuntimeCrashTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c8945409c8c792c1c6251d3baedcdac0cd0fc5a6
--- /dev/null
+++ b/flang/unittests/RuntimeGTest/RuntimeCrashTest.cpp
@@ -0,0 +1,157 @@
+//===-- flang/unittests/RuntimeGTest/CrashHandlerFixture.cpp ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Selected APIs are tested here to support development of unit tests for other
+/// runtime components and ensure the test fixture handles crashes as we expect.
+//
+//===----------------------------------------------------------------------===//
+#include "CrashHandlerFixture.h"
+#include "../../runtime/io-api.h"
+#include "../../runtime/terminator.h"
+#include <gtest/gtest.h>
+
+using namespace Fortran::runtime;
+using namespace Fortran::runtime::io;
+
+//------------------------------------------------------------------------------
+/// Test crashes through direct calls to terminator methods
+//------------------------------------------------------------------------------
+struct TestTerminator : CrashHandlerFixture {};
+
+#define TEST_CRASH_HANDLER_MESSAGE \
+  "Intentionally crashing runtime for unit test"
+
+TEST(TestTerminator, CrashTest) {
+  static Fortran::runtime::Terminator t;
+  ASSERT_DEATH(t.Crash(TEST_CRASH_HANDLER_MESSAGE), TEST_CRASH_HANDLER_MESSAGE);
+}
+
+#undef TEST_CRASH_HANDLER_MESSAGE
+
+TEST(TestTerminator, CheckFailedLocationTest) {
+  static Fortran::runtime::Terminator t;
+  ASSERT_DEATH(t.CheckFailed("predicate", "someFileName", 789),
+      "RUNTIME_CHECK\\(predicate\\) failed at someFileName\\(789\\)");
+}
+
+TEST(TestTerminator, CheckFailedTest) {
+  static Fortran::runtime::Terminator t;
+  ASSERT_DEATH(t.CheckFailed("predicate"),
+      "RUNTIME_CHECK\\(predicate\\) failed at \\(null\\)\\(0\\)");
+}
+
+//------------------------------------------------------------------------------
+/// Test misuse of io api
+//------------------------------------------------------------------------------
+struct TestIOCrash : CrashHandlerFixture {};
+
+TEST(TestIOCrash, FormatDescriptorWriteMismatchTest) {
+  static constexpr int bufferSize{4};
+  static char buffer[bufferSize];
+  static const char *format{"(A4)"};
+  auto *cookie{IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format))};
+  ASSERT_DEATH(IONAME(OutputInteger64)(cookie, 0xfeedface),
+      "Data edit descriptor 'A' may not be used with an INTEGER data item");
+}
+
+TEST(TestIOCrash, InvalidFormatCharacterTest) {
+  static constexpr int bufferSize{1};
+  static char buffer[bufferSize];
+  static const char *format{"(C1)"};
+  auto *cookie{IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format))};
+  ASSERT_DEATH(IONAME(OutputInteger64)(cookie, 0xfeedface),
+      "Unknown 'C' edit descriptor in FORMAT");
+}
+
+//------------------------------------------------------------------------------
+/// Test buffer overwrites with Output* functions
+/// Each test performs the tested IO operation correctly first, before causing
+/// an overwrite to demonstrate that the failure is caused by the overwrite and
+/// not a misuse of the API.
+//------------------------------------------------------------------------------
+TEST(TestIOCrash, OverwriteBufferAsciiTest) {
+  static constexpr int bufferSize{4};
+  static char buffer[bufferSize];
+  static const char *format{"(A4)"};
+  auto *cookie{IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format))};
+  IONAME(OutputAscii)(cookie, "four", bufferSize);
+  ASSERT_DEATH(IONAME(OutputAscii)(cookie, "Too many characters!", 20),
+      "Internal write overran available records");
+}
+
+TEST(TestIOCrash, OverwriteBufferCharacterTest) {
+  static constexpr int bufferSize{1};
+  static char buffer[bufferSize];
+  static const char *format{"(A1)"};
+  auto *cookie{IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format))};
+  IONAME(OutputCharacter)(cookie, "a", 1);
+  ASSERT_DEATH(IONAME(OutputCharacter)(cookie, "a", 1),
+      "Internal write overran available records");
+}
+
+TEST(TestIOCrash, OverwriteBufferLogicalTest) {
+  static constexpr int bufferSize{1};
+  static char buffer[bufferSize];
+  static const char *format{"(L1)"};
+  auto *cookie{IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format))};
+  IONAME(OutputLogical)(cookie, true);
+  ASSERT_DEATH(IONAME(OutputLogical)(cookie, true),
+      "Internal write overran available records");
+}
+
+TEST(TestIOCrash, OverwriteBufferRealTest) {
+  static constexpr int bufferSize{1};
+  static char buffer[bufferSize];
+  static const char *format{"(F1)"};
+  auto *cookie{IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format))};
+  IONAME(OutputReal32)(cookie, 1.);
+  EXPECT_DEATH(IONAME(OutputReal32)(cookie, 1.),
+      "Internal write overran available records");
+
+  std::memset(buffer, '\0', bufferSize);
+  cookie = IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format));
+  IONAME(OutputReal64)(cookie, 1.);
+  EXPECT_DEATH(IONAME(OutputReal64)(cookie, 1.),
+      "Internal write overran available records");
+}
+
+TEST(TestIOCrash, OverwriteBufferComplexTest) {
+  static constexpr int bufferSize{8};
+  static char buffer[bufferSize];
+  static const char *format{"(Z1,Z1)"};
+  auto *cookie{IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format))};
+  IONAME(OutputComplex32)(cookie, 1., 1.);
+  EXPECT_DEATH(IONAME(OutputComplex32)(cookie, 1., 1.),
+      "Internal write overran available records");
+
+  std::memset(buffer, '\0', bufferSize);
+  cookie = IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format));
+  IONAME(OutputComplex64)(cookie, 1., 1.);
+  EXPECT_DEATH(IONAME(OutputComplex64)(cookie, 1., 1.),
+      "Internal write overran available records");
+}
+
+TEST(TestIOCrash, OverwriteBufferIntegerTest) {
+  static constexpr int bufferSize{1};
+  static char buffer[bufferSize];
+  static const char *format{"(I1)"};
+  auto *cookie{IONAME(BeginInternalFormattedOutput)(
+      buffer, bufferSize, format, std::strlen(format))};
+  IONAME(OutputInteger64)(cookie, 0xdeadbeef);
+  ASSERT_DEATH(IONAME(OutputInteger64)(cookie, 0xdeadbeef),
+      "Internal write overran available records");
+}
diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt
index 95c3b6c66b5970fccc24f6d3147b24a38870ae22..7054bfc5d1484c35242f9cf3cc07d4351e2c9aa8 100644
--- a/libc/CMakeLists.txt
+++ b/libc/CMakeLists.txt
@@ -5,14 +5,16 @@ cmake_minimum_required(VERSION 3.13.4)
 cmake_policy(SET CMP0076 OLD)
 list(APPEND CMAKE_MODULE_PATH  "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
 
-# The top-level source directory of libc.
+# The top-level sourse and binary directories.
 set(LIBC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(LIBC_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 # The top-level directory in which libc is being built.
 set(LIBC_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
 # Path libc/scripts directory.
 set(LIBC_BUILD_SCRIPTS_DIR "${LIBC_SOURCE_DIR}/utils/build_scripts")
 
+set(LIBC_INSTALL_PREFIX "" CACHE STRING "Define libc destination prefix.")
 
 set(LIBC_TARGET_OS ${CMAKE_SYSTEM_NAME})
 string(TOLOWER ${LIBC_TARGET_OS} LIBC_TARGET_OS)
diff --git a/libc/cmake/modules/LLVMLibCLibraryRules.cmake b/libc/cmake/modules/LLVMLibCLibraryRules.cmake
index 21a99a0dd0a9276f42aec1aee223e917702867dc..bdc361a719de88327e160c9be8c796eb3d1fa009 100644
--- a/libc/cmake/modules/LLVMLibCLibraryRules.cmake
+++ b/libc/cmake/modules/LLVMLibCLibraryRules.cmake
@@ -1,70 +1,42 @@
-# This is a helper function and not a build rule. It is to be used by the
-# the "add_entrypoint_library" rule to generate the full list of object files
-# recursively produced by "add_object_library" targets upstream in the
-# dependency tree. This function traverses up through the
-# "add_entrypoint_object" targets but does not collect the object files
-# produced by them.
-# Usage:
-#   get_object_files_for_test(<result var> <target0> [<target1> ...])
-#
-#   targetN is either an "add_entrypoint_target" target or an
-#   "add_object_library" target.
-function(get_object_files_for_entrypoint_library result)
-  set(object_files "")
-  foreach(dep IN LISTS ARGN)
-    get_target_property(dep_type ${dep} "TARGET_TYPE")
-    if (NOT dep_type)
-      continue()
-    endif()
-
-    if(${dep_type} STREQUAL ${OBJECT_LIBRARY_TARGET_TYPE})
-      get_target_property(dep_object_files ${dep} "OBJECT_FILES")
-      if(dep_object_files)
-        list(APPEND object_files ${dep_object_files})
-      endif()
-    endif()
-
-    get_target_property(indirect_deps ${dep} "DEPS")
-    get_object_files_for_entrypoint_library(indirect_objfiles ${indirect_deps})
-    list(APPEND object_files ${indirect_objfiles})
-  endforeach(dep)
-  list(REMOVE_DUPLICATES object_files)
-  set(${result} ${object_files} PARENT_SCOPE)
-endfunction()
-
-# This is a helper function and not a build rule. Given an entrypoint object
-# target, it returns the object file produced by this target in |result|.
-# If the given entrypoint target is an alias, then it traverses up to the
-# aliasee to get the object file.
-function(get_entrypoint_object_file entrypoint_target result)
-  get_target_property(target_type ${entrypoint_target} "TARGET_TYPE")
-  if(NOT (${target_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE}))
-    message(FATAL_ERROR
-            "Expected an target added using `add_entrypoint_object` rule.")
+function(collect_object_file_deps target result)
+  set(all_deps "")
+  get_target_property(target_type ${target} "TARGET_TYPE")
+  if(NOT target_type)
+    return()
   endif()
 
-  get_target_property(objfile ${entrypoint_target} "OBJECT_FILE")
-  if(objfile)
-    set(${result} ${objfile} PARENT_SCOPE)
+  if(${target_type} STREQUAL ${OBJECT_LIBRARY_TARGET_TYPE})
+    list(APPEND all_deps ${target})
+    get_target_property(deps ${target} "DEPS")
+    foreach(dep IN LISTS deps)
+      collect_object_file_deps(${dep} dep_targets)
+      list(APPEND all_deps ${dep_targets})
+    endforeach(dep)
+    set(${result} ${all_deps} PARENT_SCOPE)
     return()
   endif()
 
-  # If the entrypoint is an alias, fetch the object file from the aliasee.
-  get_target_property(is_alias ${entrypoint_target} "IS_ALIAS")
-  if(is_alias)
-    get_target_property(aliasee ${entrypoint_target} "DEPS")
-    if(NOT aliasee)
-      message(FATAL_ERROR
-              "Entrypoint alias ${entrypoint_target} does not have an aliasee.")
+  if(${target_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE})
+    set(entrypoint_target ${target})
+    get_target_property(is_alias ${entrypoint_target} "IS_ALIAS")
+    if(is_alias)
+      get_target_property(aliasee ${entrypoint_target} "DEPS")
+      if(NOT aliasee)
+        message(FATAL_ERROR
+                "Entrypoint alias ${entrypoint_target} does not have an aliasee.")
+      endif()
+      set(entrypoint_target ${aliasee})
     endif()
-    get_entrypoint_object_file(${aliasee} objfile)
-    set(${result} ${objfile} PARENT_SCOPE)
+    list(APPEND all_deps ${entrypoint_target})
+    get_target_property(deps ${target} "DEPS")
+    foreach(dep IN LISTS deps)
+      collect_object_file_deps(${dep} dep_targets)
+      list(APPEND all_deps ${dep_targets})
+    endforeach(dep)
+    set(${result} ${all_deps} PARENT_SCOPE)
     return()
   endif()
-
-  message(FATAL_ERROR
-          "Entrypoint ${entrypoint_target} does not produce an object file.")
-endfunction(get_entrypoint_object_file)
+endfunction(collect_object_file_deps)
 
 # A rule to build a library from a collection of entrypoint objects.
 # Usage:
@@ -89,28 +61,30 @@ function(add_entrypoint_library target_name)
   endif()
 
   get_fq_deps_list(fq_deps_list ${ENTRYPOINT_LIBRARY_DEPENDS})
-  get_object_files_for_entrypoint_library(obj_list ${fq_deps_list})
+  set(all_deps "")
   foreach(dep IN LISTS fq_deps_list)
     get_target_property(dep_type ${dep} "TARGET_TYPE")
     if(NOT (${dep_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE}))
       message(FATAL_ERROR "Dependency '${dep}' of 'add_entrypoint_collection' is "
                           "not an 'add_entrypoint_object' target.")
     endif()
-    get_entrypoint_object_file(${dep} objfile)
-    list(APPEND obj_list ${objfile})
+    collect_object_file_deps(${dep} recursive_deps)
+    list(APPEND all_deps ${recursive_deps})
   endforeach(dep)
-  list(REMOVE_DUPLICATES obj_list)
-
-  set(library_file "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${target_name}${CMAKE_STATIC_LIBRARY_SUFFIX}")
-  add_custom_command(
-    OUTPUT ${library_file}
-    COMMAND ${CMAKE_AR} -r ${library_file} ${obj_list}
-    DEPENDS ${obj_list}
+  list(REMOVE_DUPLICATES all_deps)
+  set(objects "")
+  foreach(dep IN LISTS all_deps)
+    list(APPEND objects $<TARGET_OBJECTS:${dep}>)
+  endforeach(dep)
+  add_library(
+    ${target_name}
+    STATIC
+    ${objects}
   )
-  add_custom_target(
+  set_target_properties(
     ${target_name}
-    ALL
-    DEPENDS ${library_file}
+    PROPERTIES
+      ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   )
 endfunction(add_entrypoint_library)
 
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index 6816e0e19038221e3b887bc8146b66fdd099ade3..267db1d6ffc56626f9559cb5981130c8b11b6ea7 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -161,7 +161,7 @@ endfunction(add_libc_unittest)
 
 function(add_libc_testsuite suite_name)
   add_custom_target(${suite_name})
-  add_dependencies(check-libc ${suite_name})
+  add_dependencies(check-llvmlibc ${suite_name})
 endfunction(add_libc_testsuite)
 
 function(add_libc_exhaustive_testsuite suite_name)
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index fa7db68466de4b7ae3313789ec98501158d88554..36067fbbf22fe11a73b5bdf6750f0c1815c3c194 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -249,6 +249,8 @@ def TimeAPI : PublicAPI<"time.h"> {
   ];
 
   let Functions = [
+    "gmtime",
+    "gmtime_r",
     "mktime",
   ];
 }
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index a3fffae6e578d6597c68d2938d7ac375b6e1abe6..28ce217dc5134968cfaecab7a8a338eda94ec4dc 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -176,6 +176,8 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.threads.thrd_join
 
     # time.h entrypoints
+    libc.src.time.gmtime
+    libc.src.time.gmtime_r
     libc.src.time.mktime
 
     # unistd.h entrypoints
diff --git a/libc/lib/CMakeLists.txt b/libc/lib/CMakeLists.txt
index b8ca13b271248869decec72d5f2c5a858f43f90c..7d99bd9cebf8ced318a0ad8699597b210f3abe59 100644
--- a/libc/lib/CMakeLists.txt
+++ b/libc/lib/CMakeLists.txt
@@ -3,3 +3,22 @@ add_entrypoint_library(
   DEPENDS
   ${TARGET_LLVMLIBC_ENTRYPOINTS}
 )
+
+if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR)
+  set(LIBC_INSTALL_LIBRARY_DIR lib${LLVM_LIBDIR_SUFFIX}/${LLVM_DEFAULT_TARGET_TRIPLE})
+else()
+  set(LIBC_INSTALL_LIBRARY_DIR lib${LLVM_LIBDIR_SUFFIX})
+endif()
+
+install(
+  TARGETS llvmlibc
+  ARCHIVE DESTINATION ${LIBC_INSTALL_PREFIX}/${LIBC_INSTALL_LIBRARY_DIR}
+  COMPONENT llvmlibc
+)
+
+add_custom_target(
+    install-llvmlibc
+    DEPENDS llvmlibc
+    COMMAND "${CMAKE_COMMAND}" -DCMAKE_INSTALL_COMPONENT=llvmlibc
+            -P "${LIBC_BINARY_DIR}/cmake_install.cmake"
+)
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 9b332a8c1bbc2ebc8792bf1628ca40529dc6b835..2b3602b2e67ccb2de5603509608d7a47978d5c97 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -5,6 +5,7 @@ def StdC : StandardSpec<"stdc"> {
   RestrictedPtrType FILERestrictedPtr = RestrictedPtrType<FILE>;
   NamedType StructTmType = NamedType<"struct tm">;
   PtrType StructTmPtr = PtrType<StructTmType>;
+  PtrType TimeTTypePtr = PtrType<TimeTType>;
 
   HeaderSpec Assert = HeaderSpec<
       "assert.h",
@@ -597,6 +598,19 @@ def StdC : StandardSpec<"stdc"> {
       ],
       [], // Enumerations
       [
+          FunctionSpec<
+              "gmtime",
+              RetValSpec<StructTmPtr>,
+              [ArgSpec<TimeTTypePtr>]
+          >,
+          FunctionSpec<
+              "gmtime_r",
+              RetValSpec<StructTmPtr>,
+              [
+                  ArgSpec<TimeTTypePtr>,
+                  ArgSpec<StructTmPtr>,
+              ]
+          >,
           FunctionSpec<
               "mktime",
               RetValSpec<TimeTType>,
diff --git a/libc/src/time/CMakeLists.txt b/libc/src/time/CMakeLists.txt
index a4aa97f12f5f2b77b27738f06b595f0b82e00126..03343cf036c53aa2510d43cbd1fde2495448783d 100644
--- a/libc/src/time/CMakeLists.txt
+++ b/libc/src/time/CMakeLists.txt
@@ -1,11 +1,45 @@
+add_object_library(
+  time_utils
+  SRCS
+    time_utils.cpp
+  HDRS
+    time_utils.h
+  DEPENDS
+    libc.include.errno
+    libc.include.time
+    libc.src.errno.__errno_location
+)
+
+add_entrypoint_object(
+  gmtime
+  SRCS
+    gmtime.cpp
+  HDRS
+    gmtime.h
+  DEPENDS
+    .time_utils
+    libc.include.time
+)
+
+add_entrypoint_object(
+  gmtime_r
+  SRCS
+    gmtime_r.cpp
+  HDRS
+    gmtime_r.h
+  DEPENDS
+    .time_utils
+    libc.include.time
+)
+
 add_entrypoint_object(
   mktime
   SRCS
     mktime.cpp
   HDRS
     mktime.h
-    time_utils.h
   DEPENDS
+    .time_utils
     libc.include.errno
     libc.include.time
     libc.src.errno.__errno_location
diff --git a/libc/src/time/gmtime.cpp b/libc/src/time/gmtime.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..75ce85941b184f40388ddb8f49bcd7cf84744ca8
--- /dev/null
+++ b/libc/src/time/gmtime.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of gmtime function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/gmtime.h"
+#include "src/__support/common.h"
+#include "src/time/time_utils.h"
+
+namespace __llvm_libc {
+
+LLVM_LIBC_FUNCTION(struct tm *, gmtime, (const time_t *timer)) {
+  static struct tm tm_out;
+  return time_utils::gmtime_internal(timer, &tm_out);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/time/gmtime.h b/libc/src/time/gmtime.h
new file mode 100644
index 0000000000000000000000000000000000000000..8891a8c917ac55f079640a96ef5748fa92ddf525
--- /dev/null
+++ b/libc/src/time/gmtime.h
@@ -0,0 +1,22 @@
+//===-- Implementation header of gmtime -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_TIME_GMTIME_H
+#define LLVM_LIBC_SRC_TIME_GMTIME_H
+
+#include <time.h>
+
+namespace __llvm_libc {
+
+struct tm *gmtime(const time_t *timer);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_TIME_GMTIME_H
+
+#include "include/time.h"
diff --git a/libc/src/time/gmtime_r.cpp b/libc/src/time/gmtime_r.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..67bf12696e2eeb8fb0ec4d7462745f1f659565e9
--- /dev/null
+++ b/libc/src/time/gmtime_r.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of gmtime_r function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/gmtime_r.h"
+#include "src/__support/common.h"
+#include "src/time/time_utils.h"
+
+namespace __llvm_libc {
+
+LLVM_LIBC_FUNCTION(struct tm *, gmtime_r,
+                   (const time_t *timer, struct tm *result)) {
+  return time_utils::gmtime_internal(timer, result);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/time/gmtime_r.h b/libc/src/time/gmtime_r.h
new file mode 100644
index 0000000000000000000000000000000000000000..8e9fc94b5ceeddec7f219aa5a928cd60ccfc12bd
--- /dev/null
+++ b/libc/src/time/gmtime_r.h
@@ -0,0 +1,22 @@
+//===-- Implementation header of gmtime_r -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_TIME_GMTIME_R_H
+#define LLVM_LIBC_SRC_TIME_GMTIME_R_H
+
+#include <time.h>
+
+namespace __llvm_libc {
+
+struct tm *gmtime_r(const time_t *timer, struct tm *result);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_TIME_GMTIME_R_H
+
+#include "include/time.h"
diff --git a/libc/src/time/mktime.cpp b/libc/src/time/mktime.cpp
index a352cc60dbe9cd41acb85742098d5eef24c4d147..35970b1c910b2d3deede2dff6febed79160e6979 100644
--- a/libc/src/time/mktime.cpp
+++ b/libc/src/time/mktime.cpp
@@ -29,134 +29,6 @@ static constexpr bool isLeapYear(const int64_t year) {
   return (((year) % 4) == 0 && (((year) % 100) != 0 || ((year) % 400) == 0));
 }
 
-static int64_t computeRemainingYears(int64_t daysPerYears,
-                                     int64_t quotientYears,
-                                     int64_t *remainingDays) {
-  int64_t years = *remainingDays / daysPerYears;
-  if (years == quotientYears)
-    years--;
-  *remainingDays -= years * daysPerYears;
-  return years;
-}
-
-// Update the "tm" structure's year, month, etc. members from seconds.
-// "total_seconds" is the number of seconds since January 1st, 1970.
-//
-// First, divide "total_seconds" by the number of seconds in a day to get the
-// number of days since Jan 1 1970. The remainder will be used to calculate the
-// number of Hours, Minutes and Seconds.
-//
-// Then, adjust that number of days by a constant to be the number of days
-// since Mar 1 2000. Year 2000 is a multiple of 400, the leap year cycle. This
-// makes it easier to count how many leap years have passed using division.
-//
-// While calculating numbers of years in the days, the following algorithm
-// subdivides the days into the number of 400 years, the number of 100 years and
-// the number of 4 years. These numbers of cycle years are used in calculating
-// leap day. This is similar to the algorithm used in  getNumOfLeapYearsBefore()
-// and isLeapYear(). Then compute the total number of years in days from these
-// subdivided units.
-//
-// Compute the number of months from the remaining days. Finally, adjust years
-// to be 1900 and months to be from January.
-static int64_t updateFromSeconds(int64_t total_seconds, struct tm *tm) {
-  // Days in month starting from March in the year 2000.
-  static const char daysInMonth[] = {31 /* Mar */, 30, 31, 30, 31, 31,
-                                     30,           31, 30, 31, 31, 29};
-
-  if (sizeof(time_t) == 4) {
-    if (total_seconds < 0x80000000)
-      return time_utils::OutOfRange();
-    if (total_seconds > 0x7FFFFFFF)
-      return time_utils::OutOfRange();
-  } else {
-    if (total_seconds <
-            INT_MIN * static_cast<int64_t>(
-                          TimeConstants::NumberOfSecondsInLeapYear) ||
-        total_seconds > INT_MAX * static_cast<int64_t>(
-                                      TimeConstants::NumberOfSecondsInLeapYear))
-      return time_utils::OutOfRange();
-  }
-
-  int64_t seconds = total_seconds - TimeConstants::SecondsUntil2000MarchFirst;
-  int64_t days = seconds / TimeConstants::SecondsPerDay;
-  int64_t remainingSeconds = seconds % TimeConstants::SecondsPerDay;
-  if (remainingSeconds < 0) {
-    remainingSeconds += TimeConstants::SecondsPerDay;
-    days--;
-  }
-
-  int64_t wday = (TimeConstants::WeekDayOf2000MarchFirst + days) %
-                 TimeConstants::DaysPerWeek;
-  if (wday < 0)
-    wday += TimeConstants::DaysPerWeek;
-
-  // Compute the number of 400 year cycles.
-  int64_t numOfFourHundredYearCycles = days / TimeConstants::DaysPer400Years;
-  int64_t remainingDays = days % TimeConstants::DaysPer400Years;
-  if (remainingDays < 0) {
-    remainingDays += TimeConstants::DaysPer400Years;
-    numOfFourHundredYearCycles--;
-  }
-
-  // The reminder number of years after computing number of
-  // "four hundred year cycles" will be 4 hundred year cycles or less in 400
-  // years.
-  int64_t numOfHundredYearCycles =
-      computeRemainingYears(TimeConstants::DaysPer100Years, 4, &remainingDays);
-
-  // The reminder number of years after computing number of
-  // "hundred year cycles" will be 25 four year cycles or less in 100 years.
-  int64_t numOfFourYearCycles =
-      computeRemainingYears(TimeConstants::DaysPer4Years, 25, &remainingDays);
-
-  // The reminder number of years after computing number of "four year cycles"
-  // will be 4 one year cycles or less in 4 years.
-  int64_t remainingYears = computeRemainingYears(
-      TimeConstants::DaysPerNonLeapYear, 4, &remainingDays);
-
-  // Calculate number of years from year 2000.
-  int64_t years = remainingYears + 4 * numOfFourYearCycles +
-                  100 * numOfHundredYearCycles +
-                  400LL * numOfFourHundredYearCycles;
-
-  int leapDay =
-      !remainingYears && (numOfFourYearCycles || !numOfHundredYearCycles);
-
-  int64_t yday = remainingDays + 31 + 28 + leapDay;
-  if (yday >= TimeConstants::DaysPerNonLeapYear + leapDay)
-    yday -= TimeConstants::DaysPerNonLeapYear + leapDay;
-
-  int64_t months = 0;
-  while (daysInMonth[months] <= remainingDays) {
-    remainingDays -= daysInMonth[months];
-    months++;
-  }
-
-  if (months >= TimeConstants::MonthsPerYear - 2) {
-    months -= TimeConstants::MonthsPerYear;
-    years++;
-  }
-
-  if (years > INT_MAX || years < INT_MIN)
-    return time_utils::OutOfRange();
-
-  // All the data (years, month and remaining days) was calculated from
-  // March, 2000. Thus adjust the data to be from January, 1900.
-  tm->tm_year = years + 2000 - TimeConstants::TimeYearBase;
-  tm->tm_mon = months + 2;
-  tm->tm_mday = remainingDays + 1;
-  tm->tm_wday = wday;
-  tm->tm_yday = yday;
-
-  tm->tm_hour = remainingSeconds / TimeConstants::SecondsPerHour;
-  tm->tm_min = remainingSeconds / TimeConstants::SecondsPerMin %
-               TimeConstants::SecondsPerMin;
-  tm->tm_sec = remainingSeconds % TimeConstants::SecondsPerMin;
-
-  return 0;
-}
-
 LLVM_LIBC_FUNCTION(time_t, mktime, (struct tm * tm_out)) {
   // Unlike most C Library functions, mktime doesn't just die on bad input.
   // TODO(rtenneti); Handle leap seconds.
@@ -236,7 +108,7 @@ LLVM_LIBC_FUNCTION(time_t, mktime, (struct tm * tm_out)) {
                     totalDays * TimeConstants::SecondsPerDay;
 
   // Update the tm structure's year, month, day, etc. from seconds.
-  if (updateFromSeconds(seconds, tm_out) < 0)
+  if (time_utils::UpdateFromSeconds(seconds, tm_out) < 0)
     return time_utils::OutOfRange();
 
   return static_cast<time_t>(seconds);
diff --git a/libc/src/time/time_utils.cpp b/libc/src/time/time_utils.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..11ac2fb0d9a0b7dfa3733026e01a7a9df9113335
--- /dev/null
+++ b/libc/src/time/time_utils.cpp
@@ -0,0 +1,147 @@
+//===-- Implementation of mktime function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/time_utils.h"
+#include "src/__support/common.h"
+
+#include <limits.h>
+
+namespace __llvm_libc {
+namespace time_utils {
+
+using __llvm_libc::time_utils::TimeConstants;
+
+static int64_t computeRemainingYears(int64_t daysPerYears,
+                                     int64_t quotientYears,
+                                     int64_t *remainingDays) {
+  int64_t years = *remainingDays / daysPerYears;
+  if (years == quotientYears)
+    years--;
+  *remainingDays -= years * daysPerYears;
+  return years;
+}
+
+// First, divide "total_seconds" by the number of seconds in a day to get the
+// number of days since Jan 1 1970. The remainder will be used to calculate the
+// number of Hours, Minutes and Seconds.
+//
+// Then, adjust that number of days by a constant to be the number of days
+// since Mar 1 2000. Year 2000 is a multiple of 400, the leap year cycle. This
+// makes it easier to count how many leap years have passed using division.
+//
+// While calculating numbers of years in the days, the following algorithm
+// subdivides the days into the number of 400 years, the number of 100 years and
+// the number of 4 years. These numbers of cycle years are used in calculating
+// leap day. This is similar to the algorithm used in  getNumOfLeapYearsBefore()
+// and isLeapYear(). Then compute the total number of years in days from these
+// subdivided units.
+//
+// Compute the number of months from the remaining days. Finally, adjust years
+// to be 1900 and months to be from January.
+int64_t UpdateFromSeconds(int64_t total_seconds, struct tm *tm) {
+  // Days in month starting from March in the year 2000.
+  static const char daysInMonth[] = {31 /* Mar */, 30, 31, 30, 31, 31,
+                                     30,           31, 30, 31, 31, 29};
+
+  if (sizeof(time_t) == 4) {
+    if (total_seconds < 0x80000000)
+      return time_utils::OutOfRange();
+    if (total_seconds > 0x7FFFFFFF)
+      return time_utils::OutOfRange();
+  } else {
+    if (total_seconds <
+            INT_MIN * static_cast<int64_t>(
+                          TimeConstants::NumberOfSecondsInLeapYear) ||
+        total_seconds > INT_MAX * static_cast<int64_t>(
+                                      TimeConstants::NumberOfSecondsInLeapYear))
+      return time_utils::OutOfRange();
+  }
+
+  int64_t seconds = total_seconds - TimeConstants::SecondsUntil2000MarchFirst;
+  int64_t days = seconds / TimeConstants::SecondsPerDay;
+  int64_t remainingSeconds = seconds % TimeConstants::SecondsPerDay;
+  if (remainingSeconds < 0) {
+    remainingSeconds += TimeConstants::SecondsPerDay;
+    days--;
+  }
+
+  int64_t wday = (TimeConstants::WeekDayOf2000MarchFirst + days) %
+                 TimeConstants::DaysPerWeek;
+  if (wday < 0)
+    wday += TimeConstants::DaysPerWeek;
+
+  // Compute the number of 400 year cycles.
+  int64_t numOfFourHundredYearCycles = days / TimeConstants::DaysPer400Years;
+  int64_t remainingDays = days % TimeConstants::DaysPer400Years;
+  if (remainingDays < 0) {
+    remainingDays += TimeConstants::DaysPer400Years;
+    numOfFourHundredYearCycles--;
+  }
+
+  // The reminder number of years after computing number of
+  // "four hundred year cycles" will be 4 hundred year cycles or less in 400
+  // years.
+  int64_t numOfHundredYearCycles =
+      computeRemainingYears(TimeConstants::DaysPer100Years, 4, &remainingDays);
+
+  // The reminder number of years after computing number of
+  // "hundred year cycles" will be 25 four year cycles or less in 100 years.
+  int64_t numOfFourYearCycles =
+      computeRemainingYears(TimeConstants::DaysPer4Years, 25, &remainingDays);
+
+  // The reminder number of years after computing number of "four year cycles"
+  // will be 4 one year cycles or less in 4 years.
+  int64_t remainingYears = computeRemainingYears(
+      TimeConstants::DaysPerNonLeapYear, 4, &remainingDays);
+
+  // Calculate number of years from year 2000.
+  int64_t years = remainingYears + 4 * numOfFourYearCycles +
+                  100 * numOfHundredYearCycles +
+                  400LL * numOfFourHundredYearCycles;
+
+  int leapDay =
+      !remainingYears && (numOfFourYearCycles || !numOfHundredYearCycles);
+
+  int64_t yday = remainingDays + 31 + 28 + leapDay;
+  if (yday >= TimeConstants::DaysPerNonLeapYear + leapDay)
+    yday -= TimeConstants::DaysPerNonLeapYear + leapDay;
+
+  int64_t months = 0;
+  while (daysInMonth[months] <= remainingDays) {
+    remainingDays -= daysInMonth[months];
+    months++;
+  }
+
+  if (months >= TimeConstants::MonthsPerYear - 2) {
+    months -= TimeConstants::MonthsPerYear;
+    years++;
+  }
+
+  if (years > INT_MAX || years < INT_MIN)
+    return time_utils::OutOfRange();
+
+  // All the data (years, month and remaining days) was calculated from
+  // March, 2000. Thus adjust the data to be from January, 1900.
+  tm->tm_year = years + 2000 - TimeConstants::TimeYearBase;
+  tm->tm_mon = months + 2;
+  tm->tm_mday = remainingDays + 1;
+  tm->tm_wday = wday;
+  tm->tm_yday = yday;
+
+  tm->tm_hour = remainingSeconds / TimeConstants::SecondsPerHour;
+  tm->tm_min = remainingSeconds / TimeConstants::SecondsPerMin %
+               TimeConstants::SecondsPerMin;
+  tm->tm_sec = remainingSeconds % TimeConstants::SecondsPerMin;
+  // TODO(rtenneti): Need to handle timezone and update of tm_isdst.
+  tm->tm_isdst = 0;
+
+  return 0;
+}
+
+} // namespace time_utils
+} // namespace __llvm_libc
diff --git a/libc/src/time/time_utils.h b/libc/src/time/time_utils.h
index 48bbf7afca19efe10043175c2d37c8eab567b2ca..cca03007084c9df68ff97ea7ccabb32346b9c339 100644
--- a/libc/src/time/time_utils.h
+++ b/libc/src/time/time_utils.h
@@ -53,12 +53,28 @@ struct TimeConstants {
   static constexpr time_t OutOfRangeReturnValue = -1;
 };
 
+// Update the "tm" structure's year, month, etc. members from seconds.
+// "total_seconds" is the number of seconds since January 1st, 1970.
+extern int64_t UpdateFromSeconds(int64_t total_seconds, struct tm *tm);
+
 // POSIX.1-2017 requires this.
 static inline time_t OutOfRange() {
   llvmlibc_errno = EOVERFLOW;
   return static_cast<time_t>(-1);
 }
 
+static inline struct tm *gmtime_internal(const time_t *timer,
+                                         struct tm *result) {
+  int64_t seconds = *timer;
+  // Update the tm structure's year, month, day, etc. from seconds.
+  if (UpdateFromSeconds(seconds, result) < 0) {
+    OutOfRange();
+    return nullptr;
+  }
+
+  return result;
+}
+
 } // namespace time_utils
 } // namespace __llvm_libc
 
diff --git a/libc/test/CMakeLists.txt b/libc/test/CMakeLists.txt
index 1187ba553d5a6711b73e3ffb2201a321da6f8e85..370b28bfaa6f14587ba917aa869e5c336dbfbfbc 100644
--- a/libc/test/CMakeLists.txt
+++ b/libc/test/CMakeLists.txt
@@ -5,6 +5,9 @@ add_header_library(
 )
 
 add_custom_target(check-libc)
+add_custom_target(check-llvmlibc)
+add_dependencies(check-libc check-llvmlibc)
+
 add_custom_target(exhaustive-check-libc)
 
 add_subdirectory(src)
diff --git a/libc/test/src/time/CMakeLists.txt b/libc/test/src/time/CMakeLists.txt
index 2e34ac6d6c53310d9464e49ee51e324d4c766aa1..e3bf5e228c6535cd7f8c10c7f4dd21836282cda6 100644
--- a/libc/test/src/time/CMakeLists.txt
+++ b/libc/test/src/time/CMakeLists.txt
@@ -1,5 +1,29 @@
 add_libc_testsuite(libc_time_unittests)
 
+add_libc_unittest(
+  gmtime
+  SUITE
+    libc_time_unittests
+  SRCS
+    gmtime_test.cpp
+  HDRS
+    TmMatcher.h
+  DEPENDS
+    libc.src.time.gmtime
+)
+
+add_libc_unittest(
+  gmtime_r
+  SUITE
+    libc_time_unittests
+  SRCS
+    gmtime_r_test.cpp
+  HDRS
+    TmMatcher.h
+  DEPENDS
+    libc.src.time.gmtime_r
+)
+
 add_libc_unittest(
   mktime
   SUITE
diff --git a/libc/test/src/time/gmtime_r_test.cpp b/libc/test/src/time/gmtime_r_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..037460c050c749ceab1ca45b6c2acde94bd08996
--- /dev/null
+++ b/libc/test/src/time/gmtime_r_test.cpp
@@ -0,0 +1,57 @@
+//===-- Unittests for gmtime_r --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/gmtime_r.h"
+#include "src/time/time_utils.h"
+#include "test/src/time/TmMatcher.h"
+#include "utils/UnitTest/Test.h"
+
+using __llvm_libc::time_utils::TimeConstants;
+
+// gmtime and gmtime_r share the same code and thus didn't repeat all the tests
+// from gmtime. Added couple of validation tests.
+TEST(LlvmLibcGmTimeR, EndOf32BitEpochYear) {
+  // Test for maximum value of a signed 32-bit integer.
+  // Test implementation can encode time for Tue 19 January 2038 03:14:07 UTC.
+  time_t seconds = 0x7FFFFFFF;
+  struct tm tm_data;
+  struct tm *tm_data_ptr;
+  tm_data_ptr = __llvm_libc::gmtime_r(&seconds, &tm_data);
+  EXPECT_TM_EQ((tm{7,  // sec
+                   14, // min
+                   3,  // hr
+                   19, // day
+                   0,  // tm_mon starts with 0 for Jan
+                   2038 - TimeConstants::TimeYearBase, // year
+                   2,                                  // wday
+                   7,                                  // yday
+                   0}),
+               *tm_data_ptr);
+  EXPECT_TM_EQ(*tm_data_ptr, tm_data);
+}
+
+TEST(LlvmLibcGmTimeR, Max64BitYear) {
+  if (sizeof(time_t) == 4)
+    return;
+  // Test for Tue Jan 1 12:50:50 in 2,147,483,647th year.
+  time_t seconds = 67767976202043050;
+  struct tm tm_data;
+  struct tm *tm_data_ptr;
+  tm_data_ptr = __llvm_libc::gmtime_r(&seconds, &tm_data);
+  EXPECT_TM_EQ((tm{50, // sec
+                   50, // min
+                   12, // hr
+                   1,  // day
+                   0,  // tm_mon starts with 0 for Jan
+                   2147483647 - TimeConstants::TimeYearBase, // year
+                   2,                                        // wday
+                   50,                                       // yday
+                   0}),
+               *tm_data_ptr);
+  EXPECT_TM_EQ(*tm_data_ptr, tm_data);
+}
diff --git a/libc/test/src/time/gmtime_test.cpp b/libc/test/src/time/gmtime_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ba24d1f82aca6b52048829bbd166554e27f8a163
--- /dev/null
+++ b/libc/test/src/time/gmtime_test.cpp
@@ -0,0 +1,288 @@
+//===-- Unittests for gmtime ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/gmtime.h"
+#include "src/time/time_utils.h"
+#include "test/ErrnoSetterMatcher.h"
+#include "test/src/time/TmMatcher.h"
+#include "utils/UnitTest/Test.h"
+
+#include <errno.h>
+#include <limits.h>
+#include <string.h>
+
+using __llvm_libc::testing::ErrnoSetterMatcher::Fails;
+using __llvm_libc::testing::ErrnoSetterMatcher::Succeeds;
+using __llvm_libc::time_utils::TimeConstants;
+
+TEST(LlvmLibcGmTime, OutOfRange) {
+  time_t seconds = 1 + INT_MAX * static_cast<int64_t>(
+                                     TimeConstants::NumberOfSecondsInLeapYear);
+  struct tm *tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TRUE(tm_data == nullptr);
+  EXPECT_EQ(llvmlibc_errno, EOVERFLOW);
+
+  llvmlibc_errno = 0;
+  seconds =
+      INT_MIN * static_cast<int64_t>(TimeConstants::NumberOfSecondsInLeapYear) -
+      1;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TRUE(tm_data == nullptr);
+  EXPECT_EQ(llvmlibc_errno, EOVERFLOW);
+}
+
+TEST(LlvmLibcGmTime, InvalidSeconds) {
+  time_t seconds = 0;
+  struct tm *tm_data = nullptr;
+  // -1 second from 1970-01-01 00:00:00 returns 1969-12-31 23:59:59.
+  seconds = -1;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{59,     // sec
+                   59,     // min
+                   23,     // hr
+                   31,     // day
+                   12 - 1, // tm_mon starts with 0 for Jan
+                   1969 - TimeConstants::TimeYearBase, // year
+                   3,                                  // wday
+                   364,                                // yday
+                   0}),
+               *tm_data);
+  // 60 seconds from 1970-01-01 00:00:00 returns 1970-01-01 00:01:00.
+  seconds = 60;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0, // sec
+                   1, // min
+                   0, // hr
+                   1, // day
+                   0, // tm_mon starts with 0 for Jan
+                   1970 - TimeConstants::TimeYearBase, // year
+                   4,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+}
+
+TEST(LlvmLibcGmTime, InvalidMinutes) {
+  time_t seconds = 0;
+  struct tm *tm_data = nullptr;
+  // -1 minute from 1970-01-01 00:00:00 returns 1969-12-31 23:59:00.
+  seconds = -TimeConstants::SecondsPerMin;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0,  // sec
+                   59, // min
+                   23, // hr
+                   31, // day
+                   11, // tm_mon starts with 0 for Jan
+                   1969 - TimeConstants::TimeYearBase, // year
+                   3,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+  // 60 minutes from 1970-01-01 00:00:00 returns 1970-01-01 01:00:00.
+  seconds = 60 * TimeConstants::SecondsPerMin;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0, // sec
+                   0, // min
+                   1, // hr
+                   1, // day
+                   0, // tm_mon starts with 0 for Jan
+                   1970 - TimeConstants::TimeYearBase, // year
+                   4,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+}
+
+TEST(LlvmLibcGmTime, InvalidHours) {
+  time_t seconds = 0;
+  struct tm *tm_data = nullptr;
+  // -1 hour from 1970-01-01 00:00:00 returns 1969-12-31 23:00:00.
+  seconds = -TimeConstants::SecondsPerHour;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0,  // sec
+                   0,  // min
+                   23, // hr
+                   31, // day
+                   11, // tm_mon starts with 0 for Jan
+                   1969 - TimeConstants::TimeYearBase, // year
+                   3,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+  // 24 hours from 1970-01-01 00:00:00 returns 1970-01-02 00:00:00.
+  seconds = 24 * TimeConstants::SecondsPerHour;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0, // sec
+                   0, // min
+                   0, // hr
+                   2, // day
+                   0, // tm_mon starts with 0 for Jan
+                   1970 - TimeConstants::TimeYearBase, // year
+                   5,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+}
+
+TEST(LlvmLibcGmTime, InvalidYear) {
+  // -1 year from 1970-01-01 00:00:00 returns 1969-01-01 00:00:00.
+  time_t seconds =
+      -TimeConstants::DaysPerNonLeapYear * TimeConstants::SecondsPerDay;
+  struct tm *tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0, // sec
+                   0, // min
+                   0, // hr
+                   1, // day
+                   0, // tm_mon starts with 0 for Jan
+                   1969 - TimeConstants::TimeYearBase, // year
+                   3,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+}
+
+TEST(LlvmLibcGmTime, InvalidMonths) {
+  time_t seconds = 0;
+  struct tm *tm_data = nullptr;
+  // -1 month from 1970-01-01 00:00:00 returns 1969-12-01 00:00:00.
+  seconds = -31 * TimeConstants::SecondsPerDay;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0,      // sec
+                   0,      // min
+                   0,      // hr
+                   1,      // day
+                   12 - 1, // tm_mon starts with 0 for Jan
+                   1969 - TimeConstants::TimeYearBase, // year
+                   1,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+  // 1970-13-01 00:00:00 returns 1971-01-01 00:00:00.
+  seconds = TimeConstants::DaysPerNonLeapYear * TimeConstants::SecondsPerDay;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0, // sec
+                   0, // min
+                   0, // hr
+                   1, // day
+                   0, // tm_mon starts with 0 for Jan
+                   1971 - TimeConstants::TimeYearBase, // year
+                   5,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+}
+
+TEST(LlvmLibcGmTime, InvalidDays) {
+  time_t seconds = 0;
+  struct tm *tm_data = nullptr;
+  // -1 day from 1970-01-01 00:00:00 returns 1969-12-31 00:00:00.
+  seconds = -1 * TimeConstants::SecondsPerDay;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0,  // sec
+                   0,  // min
+                   0,  // hr
+                   31, // day
+                   11, // tm_mon starts with 0 for Jan
+                   1969 - TimeConstants::TimeYearBase, // year
+                   3,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+
+  // 1970-01-32 00:00:00 returns 1970-02-01 00:00:00.
+  seconds = 31 * TimeConstants::SecondsPerDay;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0, // sec
+                   0, // min
+                   0, // hr
+                   1, // day
+                   0, // tm_mon starts with 0 for Jan
+                   1970 - TimeConstants::TimeYearBase, // year
+                   0,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+
+  // 1970-02-29 00:00:00 returns 1970-03-01 00:00:00.
+  seconds = 59 * TimeConstants::SecondsPerDay;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0, // sec
+                   0, // min
+                   0, // hr
+                   1, // day
+                   2, // tm_mon starts with 0 for Jan
+                   1970 - TimeConstants::TimeYearBase, // year
+                   0,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+
+  // 1972-02-30 00:00:00 returns 1972-03-01 00:00:00.
+  seconds = ((2 * TimeConstants::DaysPerNonLeapYear) + 60) *
+            TimeConstants::SecondsPerDay;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{0, // sec
+                   0, // min
+                   0, // hr
+                   1, // day
+                   2, // tm_mon starts with 0 for Jan
+                   1972 - TimeConstants::TimeYearBase, // year
+                   3,                                  // wday
+                   0,                                  // yday
+                   0}),
+               *tm_data);
+}
+
+TEST(LlvmLibcGmTime, EndOf32BitEpochYear) {
+  // Test for maximum value of a signed 32-bit integer.
+  // Test implementation can encode time for Tue 19 January 2038 03:14:07 UTC.
+  time_t seconds = 0x7FFFFFFF;
+  struct tm *tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{7,  // sec
+                   14, // min
+                   3,  // hr
+                   19, // day
+                   0,  // tm_mon starts with 0 for Jan
+                   2038 - TimeConstants::TimeYearBase, // year
+                   2,                                  // wday
+                   7,                                  // yday
+                   0}),
+               *tm_data);
+}
+
+TEST(LlvmLibcGmTime, Max64BitYear) {
+  if (sizeof(time_t) == 4)
+    return;
+  // Mon Jan 1 12:50:50 2170 (200 years from 1970),
+  time_t seconds = 6311479850;
+  struct tm *tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{50, // sec
+                   50, // min
+                   12, // hr
+                   1,  // day
+                   0,  // tm_mon starts with 0 for Jan
+                   2170 - TimeConstants::TimeYearBase, // year
+                   1,                                  // wday
+                   50,                                 // yday
+                   0}),
+               *tm_data);
+
+  // Test for Tue Jan 1 12:50:50 in 2,147,483,647th year.
+  seconds = 67767976202043050;
+  tm_data = __llvm_libc::gmtime(&seconds);
+  EXPECT_TM_EQ((tm{50, // sec
+                   50, // min
+                   12, // hr
+                   1,  // day
+                   0,  // tm_mon starts with 0 for Jan
+                   2147483647 - TimeConstants::TimeYearBase, // year
+                   2,                                        // wday
+                   50,                                       // yday
+                   0}),
+               *tm_data);
+}
diff --git a/libc/test/src/time/mktime_test.cpp b/libc/test/src/time/mktime_test.cpp
index 3e76c586cba7e8c8e5b0fad85f4d6616b7757752..93290d3ec0a4ed9117b53299df4a2ac5e478d96e 100644
--- a/libc/test/src/time/mktime_test.cpp
+++ b/libc/test/src/time/mktime_test.cpp
@@ -50,7 +50,7 @@ TEST(LlvmLibcMkTime, FailureSetsErrno) {
   EXPECT_THAT(__llvm_libc::mktime(&tm_data), Fails(EOVERFLOW));
 }
 
-TEST(LlvmLibcMkTime, MkTimesInvalidSeconds) {
+TEST(LlvmLibcMkTime, InvalidSeconds) {
   struct tm tm_data;
   // -1 second from 1970-01-01 00:00:00 returns 1969-12-31 23:59:59.
   EXPECT_THAT(call_mktime(&tm_data,
@@ -96,7 +96,7 @@ TEST(LlvmLibcMkTime, MkTimesInvalidSeconds) {
                tm_data);
 }
 
-TEST(LlvmLibcMkTime, MktimeTestsInvalidMinutes) {
+TEST(LlvmLibcMkTime, InvalidMinutes) {
   struct tm tm_data;
   // -1 minute from 1970-01-01 00:00:00 returns 1969-12-31 23:59:00.
   EXPECT_THAT(call_mktime(&tm_data,
@@ -142,7 +142,7 @@ TEST(LlvmLibcMkTime, MktimeTestsInvalidMinutes) {
                tm_data);
 }
 
-TEST(LlvmLibcMkTime, MktimeTestsInvalidHours) {
+TEST(LlvmLibcMkTime, InvalidHours) {
   struct tm tm_data;
   // -1 hour from 1970-01-01 00:00:00 returns 1969-12-31 23:00:00.
   EXPECT_THAT(call_mktime(&tm_data,
@@ -188,7 +188,7 @@ TEST(LlvmLibcMkTime, MktimeTestsInvalidHours) {
                tm_data);
 }
 
-TEST(LlvmLibcMkTime, MktimeTestsInvalidYear) {
+TEST(LlvmLibcMkTime, InvalidYear) {
   struct tm tm_data;
   // -1 year from 1970-01-01 00:00:00 returns 1969-01-01 00:00:00.
   EXPECT_THAT(call_mktime(&tm_data,
@@ -214,7 +214,7 @@ TEST(LlvmLibcMkTime, MktimeTestsInvalidYear) {
                tm_data);
 }
 
-TEST(LlvmLibcMkTime, MktimeTestsInvalidEndOf32BitEpochYear) {
+TEST(LlvmLibcMkTime, InvalidEndOf32BitEpochYear) {
   if (sizeof(size_t) != 4)
     return;
   struct tm tm_data;
@@ -238,7 +238,7 @@ TEST(LlvmLibcMkTime, MktimeTestsInvalidEndOf32BitEpochYear) {
               Succeeds(TimeConstants::OutOfRangeReturnValue));
 }
 
-TEST(LlvmLibcMkTime, MktimeTestsInvalidMonths) {
+TEST(LlvmLibcMkTime, InvalidMonths) {
   struct tm tm_data;
   // -1 month from 1970-01-01 00:00:00 returns 1969-12-01 00:00:00.
   EXPECT_THAT(call_mktime(&tm_data,
@@ -285,7 +285,7 @@ TEST(LlvmLibcMkTime, MktimeTestsInvalidMonths) {
                tm_data);
 }
 
-TEST(LlvmLibcMkTime, MktimeTestsInvalidDays) {
+TEST(LlvmLibcMkTime, InvalidDays) {
   struct tm tm_data;
   // -1 day from 1970-01-01 00:00:00 returns 1969-12-31 00:00:00.
   EXPECT_THAT(call_mktime(&tm_data,
@@ -377,7 +377,7 @@ TEST(LlvmLibcMkTime, MktimeTestsInvalidDays) {
                tm_data);
 }
 
-TEST(LlvmLibcMkTime, MktimeTestsEndOf32BitEpochYear) {
+TEST(LlvmLibcMkTime, EndOf32BitEpochYear) {
   struct tm tm_data;
   // Test for maximum value of a signed 32-bit integer.
   // Test implementation can encode time for Tue 19 January 2038 03:14:07 UTC.
@@ -403,7 +403,7 @@ TEST(LlvmLibcMkTime, MktimeTestsEndOf32BitEpochYear) {
                tm_data);
 }
 
-TEST(LlvmLibcMkTime, MktimeTests64BitYear) {
+TEST(LlvmLibcMkTime, Max64BitYear) {
   if (sizeof(time_t) == 4)
     return;
   // Mon Jan 1 12:50:50 2170 (200 years from 1970),
diff --git a/libcxx/cmake/caches/Armv7.cmake b/libcxx/cmake/caches/Armv7Arm.cmake
similarity index 56%
rename from libcxx/cmake/caches/Armv7.cmake
rename to libcxx/cmake/caches/Armv7Arm.cmake
index 34b90083bd7dddaf76f7da5954a92feff0add38b..8b2b54eba13ce2e34f2ae98eb8acc512cacf2ca2 100644
--- a/libcxx/cmake/caches/Armv7.cmake
+++ b/libcxx/cmake/caches/Armv7Arm.cmake
@@ -1,2 +1,4 @@
 set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "")
 set(LIBCXX_TARGET_TRIPLE "armv7-linux-gnueabihf" CACHE STRING "")
+set(CMAKE_CXX_FLAGS "-marm" CACHE STRING "")
+set(CMAKE_C_FLAGS "-marm" CACHE STRING "")
diff --git a/libcxx/cmake/caches/Armv7Thumb-noexceptions.cmake b/libcxx/cmake/caches/Armv7Thumb-noexceptions.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..67ec43b93f2070cff85c874dfe09673ef3edc16b
--- /dev/null
+++ b/libcxx/cmake/caches/Armv7Thumb-noexceptions.cmake
@@ -0,0 +1,6 @@
+set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "")
+set(LIBCXX_TARGET_TRIPLE "armv7-linux-gnueabihf" CACHE STRING "")
+set(CMAKE_CXX_FLAGS "-mthumb" CACHE STRING "")
+set(CMAKE_C_FLAGS "-mthumb" CACHE STRING "")
+set(LIBCXX_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
+set(LIBCXXABI_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
diff --git a/libcxx/cmake/caches/Armv8.cmake b/libcxx/cmake/caches/Armv8Arm.cmake
similarity index 56%
rename from libcxx/cmake/caches/Armv8.cmake
rename to libcxx/cmake/caches/Armv8Arm.cmake
index 85da66cbea54c78b73b7ce85cb0781c07513b201..55dfa908b3d01134a6aa182674d689bcf8afde34 100644
--- a/libcxx/cmake/caches/Armv8.cmake
+++ b/libcxx/cmake/caches/Armv8Arm.cmake
@@ -1,2 +1,4 @@
 set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "")
 set(LIBCXX_TARGET_TRIPLE "armv8-linux-gnueabihf" CACHE STRING "")
+set(CMAKE_CXX_FLAGS "-marm" CACHE STRING "")
+set(CMAKE_C_FLAGS "-marm" CACHE STRING "")
diff --git a/libcxx/cmake/caches/Armv8Thumb-noexceptions.cmake b/libcxx/cmake/caches/Armv8Thumb-noexceptions.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..fb1d10efaddce828d1a7612b0d624f5db863f63a
--- /dev/null
+++ b/libcxx/cmake/caches/Armv8Thumb-noexceptions.cmake
@@ -0,0 +1,6 @@
+set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "")
+set(LIBCXX_TARGET_TRIPLE "armv8-linux-gnueabihf" CACHE STRING "")
+set(CMAKE_CXX_FLAGS "-mthumb" CACHE STRING "")
+set(CMAKE_C_FLAGS "-mthumb" CACHE STRING "")
+set(LIBCXX_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
+set(LIBCXXABI_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
diff --git a/libcxx/cmake/caches/Generic-nodebug.cmake b/libcxx/cmake/caches/Generic-no-debug.cmake
similarity index 100%
rename from libcxx/cmake/caches/Generic-nodebug.cmake
rename to libcxx/cmake/caches/Generic-no-debug.cmake
diff --git a/libcxx/cmake/caches/Generic-static.cmake b/libcxx/cmake/caches/Generic-static.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..4fe910ce4d801e9fc59a7884b50360f48a08d59f
--- /dev/null
+++ b/libcxx/cmake/caches/Generic-static.cmake
@@ -0,0 +1,10 @@
+set(LIBCXX_ENABLE_SHARED OFF CACHE BOOL "")
+set(LIBCXXABI_ENABLE_SHARED OFF CACHE BOOL "")
+set(LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "")
+
+# TODO: We should switch this to a from-sratch config with static libraries
+# instead and get rid of these options.
+set(LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXXABI OFF CACHE BOOL "")
+set(LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXX OFF CACHE BOOL "")
+set(LIBCXXABI_LINK_TESTS_WITH_SHARED_LIBCXXABI OFF CACHE BOOL "")
+set(LIBCXXABI_LINK_TESTS_WITH_SHARED_LIBCXX OFF CACHE BOOL "")
diff --git a/libcxx/docs/BuildingLibcxx.rst b/libcxx/docs/BuildingLibcxx.rst
index b1fd19a481a52458d47f1270f5efbbac8a8ef114..bf80c6de54821401b206f679afb1ab5f3a31bd49 100644
--- a/libcxx/docs/BuildingLibcxx.rst
+++ b/libcxx/docs/BuildingLibcxx.rst
@@ -73,7 +73,7 @@ Support for Windows
 -------------------
 
 libcxx supports being built with clang-cl, but not with MSVC's cl.exe, as
-cl doesn't support the `#include_next` extension. Furthermore, VS 2017 or
+cl doesn't support the ``#include_next`` extension. Furthermore, VS 2017 or
 newer (19.14) is required.
 
 libcxx also supports being built with clang targeting MinGW environments.
@@ -102,14 +102,14 @@ Running the tests also requires a Bash shell and Python to be available.
 
 If Git for Windows is available, that can be used to provide the bash
 shell by adding the right bin directory to the path, e.g.
-`set PATH=%PATH%;C:\Program Files\Git\usr\bin`.
+``set PATH=%PATH%;C:\Program Files\Git\usr\bin``.
 
 Alternatively, one can also choose to run the whole build in a MSYS2
 shell. That can be set up e.g. by starting a Visual Studio Tools Command
 Prompt (for getting the environment variables pointing to the headers and
 import libraries), and making sure that clang-cl is available in the
 path. From there, launch an MSYS2 shell via e.g.
-`C:\msys64\msys2_shell.cmd -full-path -mingw64` (preserving the earlier
+``C:\msys64\msys2_shell.cmd -full-path -mingw64`` (preserving the earlier
 environment, allowing the MSVC headers/libraries and clang-cl to be found).
 
 In either case, then run:
@@ -127,10 +127,10 @@ In either case, then run:
 
 If you are running in an MSYS2 shell and you have installed the
 MSYS2-provided clang package (which defaults to a non-MSVC target), you
-should add e.g. `-DLIBCXX_TARGET_TRIPLE=x86_64-windows-msvc` (replacing
-`x86_64` with the architecture you're targeting) to the `cmake` command
-line above. This will instruct `check-cxx` to use the right target triple
-when invoking `clang++`.
+should add e.g. ``-DLIBCXX_TARGET_TRIPLE=x86_64-windows-msvc`` (replacing
+``x86_64`` with the architecture you're targeting) to the ``cmake`` command
+line above. This will instruct ``check-cxx`` to use the right target triple
+when invoking ``clang++``.
 
 Also note that if not building in Release mode, a failed assert in the tests
 pops up a blocking dialog box, making it hard to run a larger number of tests.
@@ -140,7 +140,7 @@ CMake + ninja (MinGW)
 
 libcxx can also be built in MinGW environments, e.g. with the MinGW
 compilers in MSYS2. This requires clang to be available (installed with
-e.g. the `mingw-w64-x86_64-clang` package), together with CMake and ninja.
+e.g. the ``mingw-w64-x86_64-clang`` package), together with CMake and ninja.
 
 .. code-block:: bash
 
diff --git a/libcxx/docs/Cxx2aStatusPaperStatus.csv b/libcxx/docs/Cxx2aStatusPaperStatus.csv
index fd73d8978fc11623511465be06830edabdc93c7b..fa93be072c1b80a3b39a0e7b238a4b567a999a14 100644
--- a/libcxx/docs/Cxx2aStatusPaperStatus.csv
+++ b/libcxx/docs/Cxx2aStatusPaperStatus.csv
@@ -68,7 +68,7 @@
 "`P1006R1 <https://wg21.link/P1006R1>`__","LWG","Constexpr in std::pointer_traits","San Diego","|Complete|","8.0"
 "`P1007R3 <https://wg21.link/P1007R3>`__","LWG","``std::assume_aligned``\ ","San Diego","* *",""
 "`P1020R1 <https://wg21.link/P1020R1>`__","LWG","Smart pointer creation with default initialization","San Diego","* *",""
-"`P1032R1 <https://wg21.link/P1032R1>`__","LWG","Misc constexpr bits","San Diego","* *",""
+"`P1032R1 <https://wg21.link/P1032R1>`__","LWG","Misc constexpr bits","San Diego","|Complete|","13.0"
 "`P1085R2 <https://wg21.link/P1085R2>`__","LWG","Should Span be Regular?","San Diego","|Complete|","8.0"
 "`P1123R0 <https://wg21.link/P1123R0>`__","LWG","Editorial Guidance for merging P0019r8 and P0528r3","San Diego","* *",""
 "`P1148R0 <https://wg21.link/P1148R0>`__","LWG","Cleaning up Clause 20","San Diego","* *",""
diff --git a/libcxx/docs/Cxx2bStatusPaperStatus.csv b/libcxx/docs/Cxx2bStatusPaperStatus.csv
index 79b5db07693751203f31e46f928835fdd929d3ef..9cbd990b0bb94b899aa198ae6aa14b965dffee7b 100644
--- a/libcxx/docs/Cxx2bStatusPaperStatus.csv
+++ b/libcxx/docs/Cxx2bStatusPaperStatus.csv
@@ -7,7 +7,7 @@
 "`P1682R3 <https://wg21.link/P1682R3>`__","LWG","std::to_underlying for enumerations","February 2021","|Complete|","13.0"
 "`P2017R1 <https://wg21.link/P2017R1>`__","LWG","Conditionally borrowed ranges","February 2021","",""
 "`P2160R1 <https://wg21.link/P2160R1>`__","LWG","Locks lock lockables","February 2021","",""
-"`P2162R2 <https://wg21.link/P2162R2>`__","LWG","Inheriting from std::variant","February 2021","",""
+"`P2162R2 <https://wg21.link/P2162R2>`__","LWG","Inheriting from std::variant","February 2021","|Complete|","13.0"
 "`P2212R2 <https://wg21.link/P2212R2>`__","LWG","Relax Requirements for time_point::clock","February 2021","",""
 "`P2259R1 <https://wg21.link/P2259R1>`__","LWG","Repairing input range adaptors and counted_iterator","February 2021","",""
 "","","","","",""
diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 693a6683f7026dd6e7f0cc665bf0cb60b88d3083..806b30f22d74f2b787797f95495483d2eff2cb47 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -162,7 +162,7 @@ Status
     ------------------------------------------------- -----------------
     ``__cpp_lib_unordered_map_try_emplace``           ``201411L``
     ------------------------------------------------- -----------------
-    ``__cpp_lib_variant``                             ``201606L``
+    ``__cpp_lib_variant``                             ``202102L``
     ------------------------------------------------- -----------------
     ``__cpp_lib_void_t``                              ``201411L``
     ------------------------------------------------- -----------------
@@ -208,17 +208,17 @@ Status
     ------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_functional``                ``201907L``
     ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_iterator``                  *unimplemented*
+    ``__cpp_lib_constexpr_iterator``                  ``201811L``
     ------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_memory``                    ``201811L``
     ------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_numeric``                   ``201911L``
     ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_string``                    *unimplemented*
+    ``__cpp_lib_constexpr_string``                    ``201811L``
     ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_string_view``               *unimplemented*
+    ``__cpp_lib_constexpr_string_view``               ``201811L``
     ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_tuple``                     *unimplemented*
+    ``__cpp_lib_constexpr_tuple``                     ``201811L``
     ------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_utility``                   ``201811L``
     ------------------------------------------------- -----------------
diff --git a/libcxx/include/__config b/libcxx/include/__config
index f2874e6d3f65491897ff22dfd4dfa82c0f826ba7..cd748fde25863d3eae2c764757cafffef4e71534 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -1145,6 +1145,7 @@ extern "C" _LIBCPP_FUNC_VIS void __sanitizer_annotate_contiguous_container(
       defined(__CloudABI__) || \
       defined(__sun__) || \
       defined(__MVS__) || \
+      defined(_AIX) || \
       (defined(__MINGW32__) && __has_include(<pthread.h>))
 #    define _LIBCPP_HAS_THREAD_API_PTHREAD
 #  elif defined(__Fuchsia__)
@@ -1458,6 +1459,13 @@ extern "C" _LIBCPP_FUNC_VIS void __sanitizer_annotate_contiguous_container(
 # define _LIBCPP_INIT_PRIORITY_MAX
 #endif
 
+#if defined(__GNUC__) || defined(__clang__)
+#define _LIBCPP_FORMAT_PRINTF(a, b)                                            \
+  __attribute__((__format__(__printf__, a, b)))
+#else
+#define _LIBCPP_FORMAT_PRINTF(a, b)
+#endif
+
 #endif // __cplusplus
 
 #endif // _LIBCPP_CONFIG
diff --git a/libcxx/include/__string b/libcxx/include/__string
index 5b340d03ad866710e74fdb5d588633bd4c048796..01f583d27d7e4f52aec5f58994b38219bd294901 100644
--- a/libcxx/include/__string
+++ b/libcxx/include/__string
@@ -47,7 +47,9 @@ struct char_traits
 
 template <> struct char_traits<char>;
 template <> struct char_traits<wchar_t>;
-template <> struct char_traits<char8_t>;  // c++20
+template <> struct char_traits<char8_t>;  // C++20
+template <> struct char_traits<char16_t>;
+template <> struct char_traits<char32_t>;
 
 }  // std
 
diff --git a/libcxx/include/__support/ibm/xlocale.h b/libcxx/include/__support/ibm/xlocale.h
index b4d21172bcfabd6f5e6ebb9438bb0d69a373d2ed..ff14d2e60b0d423390025c5a7a9d56b264e6288a 100644
--- a/libcxx/include/__support/ibm/xlocale.h
+++ b/libcxx/include/__support/ibm/xlocale.h
@@ -10,6 +10,7 @@
 #ifndef _LIBCPP_SUPPORT_IBM_XLOCALE_H
 #define _LIBCPP_SUPPORT_IBM_XLOCALE_H
 
+#include <stdarg.h>
 #include <__support/ibm/locale_mgmt_aix.h>
 #include <__support/ibm/locale_mgmt_zos.h>
 
@@ -211,11 +212,13 @@ size_t wcsxfrm_l(wchar_t *__ws1, const wchar_t *__ws2, size_t __n,
 
 // strftime_l() is defined by POSIX. However, AIX 7.1 and z/OS do not have it
 // implemented yet. z/OS retrieves it from the POSIX fallbacks.
+#if !defined(_AIX72)
 static inline
 size_t strftime_l(char *__s, size_t __size, const char *__fmt,
                   const struct tm *__tm, locale_t locale) {
   return __xstrftime(locale, __s, __size, __fmt, __tm);
 }
+#endif
 
 #elif defined(__MVS__)
 #include <wctype.h>
@@ -223,63 +226,96 @@ size_t strftime_l(char *__s, size_t __size, const char *__fmt,
 #include <__support/xlocale/__posix_l_fallback.h>
 #endif // defined(__MVS__)
 
+namespace {
+
+struct __setAndRestore {
+  explicit __setAndRestore(locale_t locale) {
+    if (locale == (locale_t)0) {
+      __cloc = newlocale(LC_ALL_MASK, "C", /* base */ (locale_t)0);
+      __stored = uselocale(__cloc);
+    } else {
+      __stored = uselocale(locale);
+    }
+  }
+
+  ~__setAndRestore() {
+    uselocale(__stored);
+    if (__cloc)
+      freelocale(__cloc);
+  }
+
+private:
+  locale_t __stored = (locale_t)0;
+  locale_t __cloc = (locale_t)0;
+};
+
+} // namespace
+
 // The following are not POSIX routines.  These are quick-and-dirty hacks
 // to make things pretend to work
 static inline
 long long strtoll_l(const char *__nptr, char **__endptr,
     int __base, locale_t locale) {
+  __setAndRestore __newloc(locale);
   return strtoll(__nptr, __endptr, __base);
 }
 
 static inline
 long strtol_l(const char *__nptr, char **__endptr,
     int __base, locale_t locale) {
+  __setAndRestore __newloc(locale);
   return strtol(__nptr, __endptr, __base);
 }
 
 static inline
 double strtod_l(const char *__nptr, char **__endptr,
     locale_t locale) {
+  __setAndRestore __newloc(locale);
   return strtod(__nptr, __endptr);
 }
 
 static inline
 float strtof_l(const char *__nptr, char **__endptr,
     locale_t locale) {
+  __setAndRestore __newloc(locale);
   return strtof(__nptr, __endptr);
 }
 
 static inline
 long double strtold_l(const char *__nptr, char **__endptr,
     locale_t locale) {
+  __setAndRestore __newloc(locale);
   return strtold(__nptr, __endptr);
 }
 
 static inline
 unsigned long long strtoull_l(const char *__nptr, char **__endptr,
     int __base, locale_t locale) {
+  __setAndRestore __newloc(locale);
   return strtoull(__nptr, __endptr, __base);
 }
 
 static inline
 unsigned long strtoul_l(const char *__nptr, char **__endptr,
     int __base, locale_t locale) {
+  __setAndRestore __newloc(locale);
   return strtoul(__nptr, __endptr, __base);
 }
 
 static inline
-int vasprintf(char **strp, const char *fmt, va_list ap)
-{
+int vasprintf(char **strp, const char *fmt, va_list ap) {
   const size_t buff_size = 256;
-  int str_size;
-  if ((*strp = (char *)malloc(buff_size)) == NULL)
-  {
+  if ((*strp = (char *)malloc(buff_size)) == NULL) {
     return -1;
   }
-  if ((str_size = vsnprintf(*strp, buff_size, fmt,  ap)) >= buff_size)
-  {
-    if ((*strp = (char *)realloc(*strp, str_size + 1)) == NULL)
-    {
+
+  va_list ap_copy;
+  va_copy(ap_copy, ap);
+  int str_size = vsnprintf(*strp, buff_size, fmt,  ap_copy);
+  va_end(ap_copy);
+
+  if ((size_t) str_size >= buff_size) {
+    if ((*strp = (char *)realloc(*strp, str_size + 1)) == NULL) {
       return -1;
     }
     str_size = vsnprintf(*strp, str_size + 1, fmt,  ap);
diff --git a/libcxx/include/__support/win32/locale_win32.h b/libcxx/include/__support/win32/locale_win32.h
index d32a7a8ad3042f765722a4df165e5e1c0bb767b5..3c5c87ae592d4875d02a49ac9b6bdf6df30aaa58 100644
--- a/libcxx/include/__support/win32/locale_win32.h
+++ b/libcxx/include/__support/win32/locale_win32.h
@@ -12,9 +12,28 @@
 
 #include <__config>
 #include <stdio.h>
-#include <xlocinfo.h> // _locale_t
+#include <locale.h> // _locale_t
 #include <__nullptr>
 
+#define _X_ALL LC_ALL
+#define _X_COLLATE LC_COLLATE
+#define _X_CTYPE LC_CTYPE
+#define _X_MONETARY LC_MONETARY
+#define _X_NUMERIC LC_NUMERIC
+#define _X_TIME LC_TIME
+#define _X_MAX LC_MAX
+#define _X_MESSAGES 6
+#define _NCAT (_X_MESSAGES + 1)
+
+#define _CATMASK(n) ((1 << (n)) >> 1)
+#define _M_COLLATE _CATMASK(_X_COLLATE)
+#define _M_CTYPE _CATMASK(_X_CTYPE)
+#define _M_MONETARY _CATMASK(_X_MONETARY)
+#define _M_NUMERIC _CATMASK(_X_NUMERIC)
+#define _M_TIME _CATMASK(_X_TIME)
+#define _M_MESSAGES _CATMASK(_X_MESSAGES)
+#define _M_ALL (_CATMASK(_NCAT) - 1)
+
 #define LC_COLLATE_MASK _M_COLLATE
 #define LC_CTYPE_MASK _M_CTYPE
 #define LC_MONETARY_MASK _M_MONETARY
diff --git a/libcxx/include/any b/libcxx/include/any
index 968c9769ee36fd23aae697586fea988040e3aef2..aaeaab6c8f74e4c1eb5e3e2be39cf470ffcb9a8e 100644
--- a/libcxx/include/any
+++ b/libcxx/include/any
@@ -80,7 +80,7 @@ namespace std {
 
 */
 
-#include <experimental/__config>
+#include <__config>
 #include <__availability>
 #include <memory>
 #include <typeinfo>
diff --git a/libcxx/include/compare b/libcxx/include/compare
index 048f4821dd4eac23f1657ee559bbb3762187ed14..12c7aa0a740efe322680e773c26184453f7f99fe 100644
--- a/libcxx/include/compare
+++ b/libcxx/include/compare
@@ -126,7 +126,6 @@ namespace std {
 
 #include <__config>
 #include <type_traits>
-#include <array>
 
 #ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
 #pragma GCC system_header
@@ -701,8 +700,8 @@ constexpr _ClassifyCompCategory __type_to_enum() noexcept {
 
 template <size_t _Size>
 constexpr _ClassifyCompCategory
-__compute_comp_type(array<_ClassifyCompCategory, _Size> __types) {
-  array<int, _CCC_Size> __seen = {};
+__compute_comp_type(const _ClassifyCompCategory (&__types)[_Size]) {
+  int __seen[_CCC_Size] = {};
   for (auto __type : __types)
     ++__seen[__type];
   if (__seen[_None])
@@ -723,9 +722,8 @@ __compute_comp_type(array<_ClassifyCompCategory, _Size> __types) {
 template <class ..._Ts>
 constexpr auto __get_comp_type() {
   using _CCC = _ClassifyCompCategory;
-  constexpr array<_CCC, sizeof...(_Ts)> __type_kinds{{__comp_detail::__type_to_enum<_Ts>()...}};
-  constexpr _CCC _Cat = sizeof...(_Ts) == 0 ? _StrongOrd
-      : __compute_comp_type(__type_kinds);
+  constexpr _CCC __type_kinds[] = {_StrongOrd, __type_to_enum<_Ts>()...};
+  constexpr _CCC _Cat = __compute_comp_type(__type_kinds);
   if constexpr (_Cat == _None)
     return void();
   else if constexpr (_Cat == _WeakEq)
diff --git a/libcxx/include/functional b/libcxx/include/functional
index 747636f195d232a9409c184785c808ccaf6f6163..8a724f64c9429dd585c6484956d5d01bbec6ce5b 100644
--- a/libcxx/include/functional
+++ b/libcxx/include/functional
@@ -3215,22 +3215,6 @@ template <class _Tp>
 using unwrap_ref_decay_t = typename unwrap_ref_decay<_Tp>::type;
 #endif // > C++17
 
-template <class _Container, class _Predicate>
-inline typename _Container::size_type
-__libcpp_erase_if_container(_Container& __c, _Predicate __pred) {
-  typename _Container::size_type __old_size = __c.size();
-
-  const typename _Container::iterator __last = __c.end();
-  for (typename _Container::iterator __iter = __c.begin(); __iter != __last;) {
-    if (__pred(*__iter))
-      __iter = __c.erase(__iter);
-    else
-      ++__iter;
-  }
-
-  return __old_size - __c.size();
-}
-
 _LIBCPP_END_NAMESPACE_STD
 
 #endif  // _LIBCPP_FUNCTIONAL
diff --git a/libcxx/include/future b/libcxx/include/future
index db60ab69ecadbe49b49e73a96cdace3be74ad1fc..40beab18004aff49c2adbb11828e41f6625b8e48 100644
--- a/libcxx/include/future
+++ b/libcxx/include/future
@@ -2126,10 +2126,10 @@ packaged_task<void(_ArgTypes...)>::reset()
     __p_ = promise<result_type>();
 }
 
-template <class _Callable>
+template <class _Rp, class... _ArgTypes>
 inline _LIBCPP_INLINE_VISIBILITY
 void
-swap(packaged_task<_Callable>& __x, packaged_task<_Callable>& __y) _NOEXCEPT
+swap(packaged_task<_Rp(_ArgTypes...)>& __x, packaged_task<_Rp(_ArgTypes...)>& __y) _NOEXCEPT
 {
     __x.swap(__y);
 }
diff --git a/libcxx/include/iosfwd b/libcxx/include/iosfwd
index 0a0de99ff3a7b667d591f089e3b0a44b2fe66be2..f60437c952cfb33c71107c0356366bf16ee1dd2a 100644
--- a/libcxx/include/iosfwd
+++ b/libcxx/include/iosfwd
@@ -84,8 +84,11 @@ typedef basic_ofstream<wchar_t>      wofstream;
 typedef basic_fstream<wchar_t>       wfstream;
 
 template <class state> class fpos;
-typedef fpos<char_traits<char>::state_type>    streampos;
-typedef fpos<char_traits<wchar_t>::state_type> wstreampos;
+using streampos  = fpos<char_traits<char>::state_type>;
+using wstreampos = fpos<char_traits<wchar_t>::state_type>;
+using u8streampos = fpos<char_traits<char8_t>::state_type>; // C++20
+using u16streampos = fpos<char_traits<char16_t>::state_type>;
+using u32streampos = fpos<char_traits<char32_t>::state_type>;
 
 }  // std
 
diff --git a/libcxx/include/iterator b/libcxx/include/iterator
index d2db7de0cabe443bad56207ec07c3fe5462275bf..c02f5232880d3e2168ee2c8203673befec693e84 100644
--- a/libcxx/include/iterator
+++ b/libcxx/include/iterator
@@ -152,14 +152,14 @@ public:
     typedef void                        reference;
     typedef void                        pointer;
 
-    explicit back_insert_iterator(Container& x);
-    back_insert_iterator& operator=(const typename Container::value_type& value);
-    back_insert_iterator& operator*();
-    back_insert_iterator& operator++();
-    back_insert_iterator  operator++(int);
+    explicit back_insert_iterator(Container& x);  // constexpr in C++20
+    back_insert_iterator& operator=(const typename Container::value_type& value);  // constexpr in C++20
+    back_insert_iterator& operator*();  // constexpr in C++20
+    back_insert_iterator& operator++();  // constexpr in C++20
+    back_insert_iterator  operator++(int);  // constexpr in C++20
 };
 
-template <class Container> back_insert_iterator<Container> back_inserter(Container& x);
+template <class Container> back_insert_iterator<Container> back_inserter(Container& x);  // constexpr in C++20
 
 template <class Container>
 class front_insert_iterator
@@ -173,14 +173,14 @@ public:
     typedef void                         reference;
     typedef void                         pointer;
 
-    explicit front_insert_iterator(Container& x);
-    front_insert_iterator& operator=(const typename Container::value_type& value);
-    front_insert_iterator& operator*();
-    front_insert_iterator& operator++();
-    front_insert_iterator  operator++(int);
+    explicit front_insert_iterator(Container& x);  // constexpr in C++20
+    front_insert_iterator& operator=(const typename Container::value_type& value);  // constexpr in C++20
+    front_insert_iterator& operator*();  // constexpr in C++20
+    front_insert_iterator& operator++();  // constexpr in C++20
+    front_insert_iterator  operator++(int);  // constexpr in C++20
 };
 
-template <class Container> front_insert_iterator<Container> front_inserter(Container& x);
+template <class Container> front_insert_iterator<Container> front_inserter(Container& x);  // constexpr in C++20
 
 template <class Container>
 class insert_iterator
@@ -195,15 +195,15 @@ public:
     typedef void                   reference;
     typedef void                   pointer;
 
-    insert_iterator(Container& x, typename Container::iterator i);
-    insert_iterator& operator=(const typename Container::value_type& value);
-    insert_iterator& operator*();
-    insert_iterator& operator++();
-    insert_iterator& operator++(int);
+    insert_iterator(Container& x, typename Container::iterator i);  // constexpr in C++20
+    insert_iterator& operator=(const typename Container::value_type& value);  // constexpr in C++20
+    insert_iterator& operator*();  // constexpr in C++20
+    insert_iterator& operator++();  // constexpr in C++20
+    insert_iterator& operator++(int);  // constexpr in C++20
 };
 
 template <class Container, class Iterator>
-insert_iterator<Container> inserter(Container& x, Iterator i);
+insert_iterator<Container> inserter(Container& x, Iterator i);  // constexpr in C++20
 
 template <class Iterator>
 class move_iterator {
@@ -932,20 +932,20 @@ protected:
 public:
     typedef _Container container_type;
 
-    _LIBCPP_INLINE_VISIBILITY explicit back_insert_iterator(_Container& __x) : container(_VSTD::addressof(__x)) {}
-    _LIBCPP_INLINE_VISIBILITY back_insert_iterator& operator=(const typename _Container::value_type& __value_)
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 explicit back_insert_iterator(_Container& __x) : container(_VSTD::addressof(__x)) {}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 back_insert_iterator& operator=(const typename _Container::value_type& __value_)
         {container->push_back(__value_); return *this;}
 #ifndef _LIBCPP_CXX03_LANG
-    _LIBCPP_INLINE_VISIBILITY back_insert_iterator& operator=(typename _Container::value_type&& __value_)
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 back_insert_iterator& operator=(typename _Container::value_type&& __value_)
         {container->push_back(_VSTD::move(__value_)); return *this;}
 #endif  // _LIBCPP_CXX03_LANG
-    _LIBCPP_INLINE_VISIBILITY back_insert_iterator& operator*()     {return *this;}
-    _LIBCPP_INLINE_VISIBILITY back_insert_iterator& operator++()    {return *this;}
-    _LIBCPP_INLINE_VISIBILITY back_insert_iterator  operator++(int) {return *this;}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 back_insert_iterator& operator*()     {return *this;}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 back_insert_iterator& operator++()    {return *this;}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 back_insert_iterator  operator++(int) {return *this;}
 };
 
 template <class _Container>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 back_insert_iterator<_Container>
 back_inserter(_Container& __x)
 {
@@ -965,20 +965,20 @@ protected:
 public:
     typedef _Container container_type;
 
-    _LIBCPP_INLINE_VISIBILITY explicit front_insert_iterator(_Container& __x) : container(_VSTD::addressof(__x)) {}
-    _LIBCPP_INLINE_VISIBILITY front_insert_iterator& operator=(const typename _Container::value_type& __value_)
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 explicit front_insert_iterator(_Container& __x) : container(_VSTD::addressof(__x)) {}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 front_insert_iterator& operator=(const typename _Container::value_type& __value_)
         {container->push_front(__value_); return *this;}
 #ifndef _LIBCPP_CXX03_LANG
-    _LIBCPP_INLINE_VISIBILITY front_insert_iterator& operator=(typename _Container::value_type&& __value_)
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 front_insert_iterator& operator=(typename _Container::value_type&& __value_)
         {container->push_front(_VSTD::move(__value_)); return *this;}
 #endif  // _LIBCPP_CXX03_LANG
-    _LIBCPP_INLINE_VISIBILITY front_insert_iterator& operator*()     {return *this;}
-    _LIBCPP_INLINE_VISIBILITY front_insert_iterator& operator++()    {return *this;}
-    _LIBCPP_INLINE_VISIBILITY front_insert_iterator  operator++(int) {return *this;}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 front_insert_iterator& operator*()     {return *this;}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 front_insert_iterator& operator++()    {return *this;}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 front_insert_iterator  operator++(int) {return *this;}
 };
 
 template <class _Container>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 front_insert_iterator<_Container>
 front_inserter(_Container& __x)
 {
@@ -999,21 +999,21 @@ protected:
 public:
     typedef _Container container_type;
 
-    _LIBCPP_INLINE_VISIBILITY insert_iterator(_Container& __x, typename _Container::iterator __i)
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 insert_iterator(_Container& __x, typename _Container::iterator __i)
         : container(_VSTD::addressof(__x)), iter(__i) {}
-    _LIBCPP_INLINE_VISIBILITY insert_iterator& operator=(const typename _Container::value_type& __value_)
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 insert_iterator& operator=(const typename _Container::value_type& __value_)
         {iter = container->insert(iter, __value_); ++iter; return *this;}
 #ifndef _LIBCPP_CXX03_LANG
-    _LIBCPP_INLINE_VISIBILITY insert_iterator& operator=(typename _Container::value_type&& __value_)
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 insert_iterator& operator=(typename _Container::value_type&& __value_)
         {iter = container->insert(iter, _VSTD::move(__value_)); ++iter; return *this;}
 #endif  // _LIBCPP_CXX03_LANG
-    _LIBCPP_INLINE_VISIBILITY insert_iterator& operator*()        {return *this;}
-    _LIBCPP_INLINE_VISIBILITY insert_iterator& operator++()       {return *this;}
-    _LIBCPP_INLINE_VISIBILITY insert_iterator& operator++(int)    {return *this;}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 insert_iterator& operator*()        {return *this;}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 insert_iterator& operator++()       {return *this;}
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 insert_iterator& operator++(int)    {return *this;}
 };
 
 template <class _Container>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 insert_iterator<_Container>
 inserter(_Container& __x, typename _Container::iterator __i)
 {
@@ -2033,6 +2033,21 @@ _LIBCPP_INLINE_VISIBILITY
 constexpr const _Ep* data(initializer_list<_Ep> __il) noexcept { return __il.begin(); }
 #endif
 
+template <class _Container, class _Predicate>
+typename _Container::size_type
+__libcpp_erase_if_container(_Container& __c, _Predicate& __pred) {
+  typename _Container::size_type __old_size = __c.size();
+
+  const typename _Container::iterator __last = __c.end();
+  for (typename _Container::iterator __iter = __c.begin(); __iter != __last;) {
+    if (__pred(*__iter))
+      __iter = __c.erase(__iter);
+    else
+      ++__iter;
+  }
+
+  return __old_size - __c.size();
+}
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/map b/libcxx/include/map
index 6b2d419095796ee8eb4635362189667d2b874d35..9c3e5e64a098505c03bb51d62a3e6d7ee47f2cc8 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -480,7 +480,7 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
 #include <__config>
 #include <__tree>
 #include <__node_handle>
-#include <iterator>
+#include <iterator> // __libcpp_erase_if_container
 #include <memory>
 #include <utility>
 #include <functional>
@@ -1660,7 +1660,7 @@ template <class _Key, class _Tp, class _Compare, class _Allocator,
 inline _LIBCPP_INLINE_VISIBILITY
     typename map<_Key, _Tp, _Compare, _Allocator>::size_type
     erase_if(map<_Key, _Tp, _Compare, _Allocator>& __c, _Predicate __pred) {
-  return __libcpp_erase_if_container(__c, __pred);
+  return _VSTD::__libcpp_erase_if_container(__c, __pred);
 }
 #endif
 
@@ -2246,7 +2246,7 @@ inline _LIBCPP_INLINE_VISIBILITY
     typename multimap<_Key, _Tp, _Compare, _Allocator>::size_type
     erase_if(multimap<_Key, _Tp, _Compare, _Allocator>& __c,
              _Predicate __pred) {
-  return __libcpp_erase_if_container(__c, __pred);
+  return _VSTD::__libcpp_erase_if_container(__c, __pred);
 }
 #endif
 
diff --git a/libcxx/include/set b/libcxx/include/set
index 3a83827711f1780b36ee46614e6d52fde90fa083..506a5d5af395f9272870194b4abba1b2f09f073b 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -429,6 +429,7 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred);  // C++20
 #include <__tree>
 #include <__node_handle>
 #include <functional>
+#include <iterator> // __libcpp_erase_if_container
 #include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -964,7 +965,7 @@ template <class _Key, class _Compare, class _Allocator, class _Predicate>
 inline _LIBCPP_INLINE_VISIBILITY
     typename set<_Key, _Compare, _Allocator>::size_type
     erase_if(set<_Key, _Compare, _Allocator>& __c, _Predicate __pred) {
-  return __libcpp_erase_if_container(__c, __pred);
+  return _VSTD::__libcpp_erase_if_container(__c, __pred);
 }
 #endif
 
@@ -1490,7 +1491,7 @@ template <class _Key, class _Compare, class _Allocator, class _Predicate>
 inline _LIBCPP_INLINE_VISIBILITY
     typename multiset<_Key, _Compare, _Allocator>::size_type
     erase_if(multiset<_Key, _Compare, _Allocator>& __c, _Predicate __pred) {
-  return __libcpp_erase_if_container(__c, __pred);
+  return _VSTD::__libcpp_erase_if_container(__c, __pred);
 }
 #endif
 
diff --git a/libcxx/include/string b/libcxx/include/string
index 19bd22b3acf5cc080886151ceea391fb1e5530dc..a5fe62572349f2162ef8ec46ad5b4f99365cc341 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -69,6 +69,9 @@ struct char_traits
 
 template <> struct char_traits<char>;
 template <> struct char_traits<wchar_t>;
+template <> struct char_traits<char8_t>;  // C++20
+template <> struct char_traits<char16_t>;
+template <> struct char_traits<char32_t>;
 
 template<class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
 class basic_string
@@ -450,6 +453,7 @@ erase_if(basic_string<charT, traits, Allocator>& c, Predicate pred); // C++20
 
 typedef basic_string<char>    string;
 typedef basic_string<wchar_t> wstring;
+typedef basic_string<char8_t> u8string; // C++20
 typedef basic_string<char16_t> u16string;
 typedef basic_string<char32_t> u32string;
 
@@ -494,12 +498,14 @@ wstring to_wstring(double val);
 wstring to_wstring(long double val);
 
 template <> struct hash<string>;
+template <> struct hash<u8string>; // C++20
 template <> struct hash<u16string>;
 template <> struct hash<u32string>;
 template <> struct hash<wstring>;
 
 basic_string<char>     operator "" s( const char *str,     size_t len ); // C++14
 basic_string<wchar_t>  operator "" s( const wchar_t *str,  size_t len ); // C++14
+basic_string<char8_t>  operator "" s( const char8_t *str,  size_t len ); // C++20
 basic_string<char16_t> operator "" s( const char16_t *str, size_t len ); // C++14
 basic_string<char32_t> operator "" s( const char32_t *str, size_t len ); // C++14
 
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index f8c63571f3c42e685f90221e4361adc25b20100c..bc92dd5b1cb2e187b4ae28e797f0eea69c06c68a 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -48,6 +48,7 @@ namespace std {
 
     // basic_string_view typedef names
     typedef basic_string_view<char> string_view;
+    typedef basic_string_view<char8_t> u8string_view; // C++20
     typedef basic_string_view<char16_t> u16string_view;
     typedef basic_string_view<char32_t> u32string_view;
     typedef basic_string_view<wchar_t> wstring_view;
@@ -106,7 +107,7 @@ namespace std {
       constexpr void remove_suffix(size_type n);
       constexpr void swap(basic_string_view& s) noexcept;
 
-      size_type copy(charT* s, size_type n, size_type pos = 0) const;
+      size_type copy(charT* s, size_type n, size_type pos = 0) const;  // constexpr in C++20
 
       constexpr basic_string_view substr(size_type pos = 0, size_type n = npos) const;
       constexpr int compare(basic_string_view s) const noexcept;
@@ -161,12 +162,14 @@ namespace std {
   // 7.11, Hash support
   template <class T> struct hash;
   template <> struct hash<string_view>;
+  template <> struct hash<u8string_view>; // C++20
   template <> struct hash<u16string_view>;
   template <> struct hash<u32string_view>;
   template <> struct hash<wstring_view>;
 
   constexpr basic_string_view<char>     operator "" sv( const char *str,     size_t len ) noexcept;
   constexpr basic_string_view<wchar_t>  operator "" sv( const wchar_t *str,  size_t len ) noexcept;
+  constexpr basic_string_view<char8_t>  operator "" sv( const char8_t *str,  size_t len ) noexcept; // C++20
   constexpr basic_string_view<char16_t> operator "" sv( const char16_t *str, size_t len ) noexcept;
   constexpr basic_string_view<char32_t> operator "" sv( const char32_t *str, size_t len ) noexcept;
 
@@ -356,7 +359,7 @@ public:
         __other.__size = __sz;
     }
 
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     size_type copy(_CharT* __s, size_type __n, size_type __pos = 0) const
     {
         if (__pos > size())
diff --git a/libcxx/include/system_error b/libcxx/include/system_error
index b714e1d4263d1db9a08067fb777e3e172d4d5561..784adccbec023ad644ab30f8983d99b0691f23cc 100644
--- a/libcxx/include/system_error
+++ b/libcxx/include/system_error
@@ -142,11 +142,12 @@ template <> struct hash<std::error_condition>;
 
 */
 
+#include <__config>
 #include <__errc>
-#include <type_traits>
+#include <__functional_base> // unary_function
 #include <stdexcept>
-#include <__functional_base>
 #include <string>
+#include <type_traits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #pragma GCC system_header
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index b3d3bae6901917a72db7d8db8de4b8d141ec2830..6e07892f94791ec02f70ad42d7e983e4e703a46e 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -38,39 +38,39 @@ public:
     template <class Alloc>
         tuple(allocator_arg_t, const Alloc& a);
     template <class Alloc>
-        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, const T&...);
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, const T&...);          // constexpr in C++20
     template <class Alloc, class... U>
-        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, U&&...);
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, U&&...);               // constexpr in C++20
     template <class Alloc>
-        tuple(allocator_arg_t, const Alloc& a, const tuple&);
+        tuple(allocator_arg_t, const Alloc& a, const tuple&);                             // constexpr in C++20
     template <class Alloc>
-        tuple(allocator_arg_t, const Alloc& a, tuple&&);
+        tuple(allocator_arg_t, const Alloc& a, tuple&&);                                  // constexpr in C++20
     template <class Alloc, class... U>
-        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, const tuple<U...>&);
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, const tuple<U...>&);   // constexpr in C++20
     template <class Alloc, class... U>
-        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, tuple<U...>&&);
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, tuple<U...>&&);        // constexpr in C++20
     template <class Alloc, class U1, class U2>
-        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, const pair<U1, U2>&);
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, const pair<U1, U2>&);  // constexpr in C++20
     template <class Alloc, class U1, class U2>
-        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, pair<U1, U2>&&);
+        explicit(see-below) tuple(allocator_arg_t, const Alloc& a, pair<U1, U2>&&);       // constexpr in C++20
 
-    tuple& operator=(const tuple&);
-    tuple& operator=(tuple&&) noexcept(is_nothrow_move_assignable_v<T> && ...);
+    tuple& operator=(const tuple&);                                                       // constexpr in C++20
+    tuple& operator=(tuple&&) noexcept(is_nothrow_move_assignable_v<T> && ...);           // constexpr in C++20
     template <class... U>
-        tuple& operator=(const tuple<U...>&);
+        tuple& operator=(const tuple<U...>&);                                             // constexpr in C++20
     template <class... U>
-        tuple& operator=(tuple<U...>&&);
+        tuple& operator=(tuple<U...>&&);                                                  // constexpr in C++20
     template <class U1, class U2>
-        tuple& operator=(const pair<U1, U2>&); // iff sizeof...(T) == 2
+        tuple& operator=(const pair<U1, U2>&); // iff sizeof...(T) == 2                   // constexpr in C++20
     template <class U1, class U2>
-        tuple& operator=(pair<U1, U2>&&); // iff sizeof...(T) == 2
+        tuple& operator=(pair<U1, U2>&&); // iff sizeof...(T) == 2                        // constexpr in C++20
 
     template<class U, size_t N>
         tuple& operator=(array<U, N> const&) // iff sizeof...(T) == N, EXTENSION
     template<class U, size_t N>
         tuple& operator=(array<U, N>&&) // iff sizeof...(T) == N, EXTENSION
 
-    void swap(tuple&) noexcept(AND(swap(declval<T&>(), declval<T&>())...));
+    void swap(tuple&) noexcept(AND(swap(declval<T&>(), declval<T&>())...));               // constexpr in C++20
 };
 
 template <class ...T>
@@ -174,7 +174,7 @@ template <size_t _Ip, class _Hp,
 class __tuple_leaf;
 
 template <size_t _Ip, class _Hp, bool _Ep>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
 void swap(__tuple_leaf<_Ip, _Hp, _Ep>& __x, __tuple_leaf<_Ip, _Hp, _Ep>& __y)
     _NOEXCEPT_(__is_nothrow_swappable<_Hp>::value)
 {
@@ -195,29 +195,30 @@ class __tuple_leaf
 #endif
     }
 
+    _LIBCPP_CONSTEXPR_AFTER_CXX11
     __tuple_leaf& operator=(const __tuple_leaf&);
 public:
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR __tuple_leaf()
+    _LIBCPP_INLINE_VISIBILITY constexpr __tuple_leaf()
              _NOEXCEPT_(is_nothrow_default_constructible<_Hp>::value) : __value_()
        {static_assert(!is_reference<_Hp>::value,
               "Attempted to default construct a reference element in a tuple");}
 
     template <class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY constexpr
         __tuple_leaf(integral_constant<int, 0>, const _Alloc&)
             : __value_()
         {static_assert(!is_reference<_Hp>::value,
               "Attempted to default construct a reference element in a tuple");}
 
     template <class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY constexpr
         __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a)
             : __value_(allocator_arg_t(), __a)
         {static_assert(!is_reference<_Hp>::value,
               "Attempted to default construct a reference element in a tuple");}
 
     template <class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY constexpr
         __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a)
             : __value_(__a)
         {static_assert(!is_reference<_Hp>::value,
@@ -238,21 +239,21 @@ public:
        "Attempted construction of reference element binds to a temporary whose lifetime has ended");}
 
     template <class _Tp, class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
         explicit __tuple_leaf(integral_constant<int, 0>, const _Alloc&, _Tp&& __t)
             : __value_(_VSTD::forward<_Tp>(__t))
         {static_assert(__can_bind_reference<_Tp&&>(),
        "Attempted construction of reference element binds to a temporary whose lifetime has ended");}
 
     template <class _Tp, class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
         explicit __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a, _Tp&& __t)
             : __value_(allocator_arg_t(), __a, _VSTD::forward<_Tp>(__t))
         {static_assert(!is_reference<_Hp>::value,
             "Attempted to uses-allocator construct a reference element in a tuple");}
 
     template <class _Tp, class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
         explicit __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a, _Tp&& __t)
             : __value_(_VSTD::forward<_Tp>(__t), __a)
         {static_assert(!is_reference<_Hp>::value,
@@ -261,7 +262,7 @@ public:
     __tuple_leaf(const __tuple_leaf& __t) = default;
     __tuple_leaf(__tuple_leaf&& __t) = default;
 
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
     int swap(__tuple_leaf& __t) _NOEXCEPT_(__is_nothrow_swappable<__tuple_leaf>::value)
     {
         _VSTD::swap(*this, __t);
@@ -276,23 +277,23 @@ template <size_t _Ip, class _Hp>
 class __tuple_leaf<_Ip, _Hp, true>
     : private _Hp
 {
-
+    _LIBCPP_CONSTEXPR_AFTER_CXX11
     __tuple_leaf& operator=(const __tuple_leaf&);
 public:
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR __tuple_leaf()
+    _LIBCPP_INLINE_VISIBILITY constexpr __tuple_leaf()
              _NOEXCEPT_(is_nothrow_default_constructible<_Hp>::value) {}
 
     template <class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY constexpr
         __tuple_leaf(integral_constant<int, 0>, const _Alloc&) {}
 
     template <class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY constexpr
         __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a)
             : _Hp(allocator_arg_t(), __a) {}
 
     template <class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY constexpr
         __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a)
             : _Hp(__a) {}
 
@@ -309,24 +310,24 @@ public:
             : _Hp(_VSTD::forward<_Tp>(__t)) {}
 
     template <class _Tp, class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY constexpr
         explicit __tuple_leaf(integral_constant<int, 0>, const _Alloc&, _Tp&& __t)
             : _Hp(_VSTD::forward<_Tp>(__t)) {}
 
     template <class _Tp, class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY constexpr
         explicit __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a, _Tp&& __t)
             : _Hp(allocator_arg_t(), __a, _VSTD::forward<_Tp>(__t)) {}
 
     template <class _Tp, class _Alloc>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY constexpr
         explicit __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a, _Tp&& __t)
             : _Hp(_VSTD::forward<_Tp>(__t), __a) {}
 
     __tuple_leaf(__tuple_leaf const &) = default;
     __tuple_leaf(__tuple_leaf &&) = default;
 
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
     int
     swap(__tuple_leaf& __t) _NOEXCEPT_(__is_nothrow_swappable<__tuple_leaf>::value)
     {
@@ -339,7 +340,7 @@ public:
 };
 
 template <class ..._Tp>
-_LIBCPP_INLINE_VISIBILITY
+_LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
 void __swallow(_Tp&&...) _NOEXCEPT {}
 
 template <class _Tp>
@@ -359,7 +360,7 @@ struct _LIBCPP_DECLSPEC_EMPTY_BASES __tuple_impl<__tuple_indices<_Indx...>, _Tp.
     : public __tuple_leaf<_Indx, _Tp>...
 {
     _LIBCPP_INLINE_VISIBILITY
-    _LIBCPP_CONSTEXPR __tuple_impl()
+    constexpr __tuple_impl()
         _NOEXCEPT_(__all<is_nothrow_default_constructible<_Tp>::value...>::value) {}
 
     template <size_t ..._Uf, class ..._Tf,
@@ -377,7 +378,7 @@ struct _LIBCPP_DECLSPEC_EMPTY_BASES __tuple_impl<__tuple_indices<_Indx...>, _Tp.
 
     template <class _Alloc, size_t ..._Uf, class ..._Tf,
               size_t ..._Ul, class ..._Tl, class ..._Up>
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
         explicit
         __tuple_impl(allocator_arg_t, const _Alloc& __a,
                      __tuple_indices<_Uf...>, __tuple_types<_Tf...>,
@@ -407,7 +408,7 @@ struct _LIBCPP_DECLSPEC_EMPTY_BASES __tuple_impl<__tuple_indices<_Indx...>, _Tp.
                          __tuple_constructible<_Tuple, tuple<_Tp...> >::value
                       >::type
              >
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
         __tuple_impl(allocator_arg_t, const _Alloc& __a, _Tuple&& __t)
             : __tuple_leaf<_Indx, _Tp>(__uses_alloc_ctor<_Tp, _Alloc, typename tuple_element<_Indx,
                                        typename __make_tuple_types<_Tuple>::type>::type>(), __a,
@@ -418,7 +419,7 @@ struct _LIBCPP_DECLSPEC_EMPTY_BASES __tuple_impl<__tuple_indices<_Indx...>, _Tp.
     __tuple_impl(const __tuple_impl&) = default;
     __tuple_impl(__tuple_impl&&) = default;
 
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
     void swap(__tuple_impl& __t)
         _NOEXCEPT_(__all<__is_nothrow_swappable<_Tp>::value...>::value)
     {
@@ -427,13 +428,13 @@ struct _LIBCPP_DECLSPEC_EMPTY_BASES __tuple_impl<__tuple_indices<_Indx...>, _Tp.
 };
 
 template<class _Dest, class _Source, size_t ..._Np>
-_LIBCPP_INLINE_VISIBILITY
+_LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
 void __memberwise_copy_assign(_Dest& __dest, _Source const& __source, __tuple_indices<_Np...>) {
     _VSTD::__swallow(((_VSTD::get<_Np>(__dest) = _VSTD::get<_Np>(__source)), void(), 0)...);
 }
 
 template<class _Dest, class _Source, class ..._Up, size_t ..._Np>
-_LIBCPP_INLINE_VISIBILITY
+_LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
 void __memberwise_forward_assign(_Dest& __dest, _Source&& __source, __tuple_types<_Up...>, __tuple_indices<_Np...>) {
     _VSTD::__swallow(((
         _VSTD::get<_Np>(__dest) = _VSTD::forward<_Up>(_VSTD::get<_Np>(__source))
@@ -619,14 +620,14 @@ public:
     template <bool _Dummy = true, _EnableIf<
         _CheckArgsConstructor<_Dummy>::__enable_implicit_default()
     , void*> = nullptr>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+    _LIBCPP_INLINE_VISIBILITY constexpr
     tuple()
         _NOEXCEPT_(__all<is_nothrow_default_constructible<_Tp>::value...>::value) {}
 
     template <bool _Dummy = true, _EnableIf<
         _CheckArgsConstructor<_Dummy>::__enable_explicit_default()
     , void*> = nullptr>
-    explicit _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+    explicit _LIBCPP_INLINE_VISIBILITY constexpr
     tuple()
         _NOEXCEPT_(__all<is_nothrow_default_constructible<_Tp>::value...>::value) {}
 
@@ -637,7 +638,7 @@ public:
              _CheckArgsConstructor<_IsSame<allocator_arg_t, _AllocArgT>::value >::__enable_implicit_default()
       , void*> = nullptr
     >
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple(_AllocArgT, _Alloc const& __a)
       : __base_(allocator_arg_t(), __a,
                     __tuple_indices<>(), __tuple_types<>(),
@@ -648,7 +649,7 @@ public:
              _CheckArgsConstructor<_IsSame<allocator_arg_t, _AllocArgT>::value>::__enable_explicit_default()
       , void*> = nullptr
     >
-    explicit _LIBCPP_INLINE_VISIBILITY
+    explicit _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple(_AllocArgT, _Alloc const& __a)
       : __base_(allocator_arg_t(), __a,
                     __tuple_indices<>(), __tuple_types<>(),
@@ -700,7 +701,7 @@ public:
                          bool
                       >::type = false
         >
-      _LIBCPP_INLINE_VISIBILITY
+      _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
       tuple(allocator_arg_t, const _Alloc& __a, const _Tp& ... __t)
         : __base_(allocator_arg_t(), __a,
                 typename __make_tuple_indices<sizeof...(_Tp)>::type(),
@@ -719,7 +720,7 @@ public:
                          bool
                       >::type = false
         >
-      _LIBCPP_INLINE_VISIBILITY
+      _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
       explicit
       tuple(allocator_arg_t, const _Alloc& __a, const _Tp& ... __t)
         : __base_(allocator_arg_t(), __a,
@@ -806,7 +807,7 @@ public:
                          bool
                       >::type = false
              >
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
         tuple(allocator_arg_t, const _Alloc& __a, _Up&&... __u)
             : __base_(allocator_arg_t(), __a,
                     typename __make_tuple_indices<sizeof...(_Up)>::type(),
@@ -825,7 +826,7 @@ public:
                          bool
                       >::type = false
              >
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
         explicit
         tuple(allocator_arg_t, const _Alloc& __a, _Up&&... __u)
             : __base_(allocator_arg_t(), __a,
@@ -865,7 +866,7 @@ public:
                          bool
                       >::type = false
              >
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
         tuple(allocator_arg_t, const _Alloc& __a, _Tuple&& __t)
             : __base_(allocator_arg_t(), __a, _VSTD::forward<_Tuple>(__t)) {}
 
@@ -878,13 +879,13 @@ public:
                          bool
                       >::type = false
              >
-        _LIBCPP_INLINE_VISIBILITY
+        _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
         explicit
         tuple(allocator_arg_t, const _Alloc& __a, _Tuple&& __t)
             : __base_(allocator_arg_t(), __a, _VSTD::forward<_Tuple>(__t)) {}
 
     // [tuple.assign]
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple& operator=(_If<_And<is_copy_assignable<_Tp>...>::value, tuple, __nat> const& __tuple)
         _NOEXCEPT_((_And<is_nothrow_copy_assignable<_Tp>...>::value))
     {
@@ -893,7 +894,7 @@ public:
         return *this;
     }
 
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple& operator=(_If<_And<is_move_assignable<_Tp>...>::value, tuple, __nat>&& __tuple)
         _NOEXCEPT_((_And<is_nothrow_move_assignable<_Tp>...>::value))
     {
@@ -909,7 +910,7 @@ public:
             is_assignable<_Tp&, _Up const&>...
         >::value
     ,int> = 0>
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple& operator=(tuple<_Up...> const& __tuple)
         _NOEXCEPT_((_And<is_nothrow_assignable<_Tp&, _Up const&>...>::value))
     {
@@ -924,7 +925,7 @@ public:
             is_assignable<_Tp&, _Up>...
         >::value
     ,int> = 0>
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple& operator=(tuple<_Up...>&& __tuple)
         _NOEXCEPT_((_And<is_nothrow_assignable<_Tp&, _Up>...>::value))
     {
@@ -941,7 +942,7 @@ public:
             is_assignable<_SecondType<_Tp..., _Dep>&, _Up2 const&>
         >::value
     ,int> = 0>
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple& operator=(pair<_Up1, _Up2> const& __pair)
         _NOEXCEPT_((_And<
             is_nothrow_assignable<_FirstType<_Tp...>&, _Up1 const&>,
@@ -960,7 +961,7 @@ public:
             is_assignable<_SecondType<_Tp..., _Dep>&, _Up2>
         >::value
     ,int> = 0>
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple& operator=(pair<_Up1, _Up2>&& __pair)
         _NOEXCEPT_((_And<
             is_nothrow_assignable<_FirstType<_Tp...>&, _Up1>,
@@ -979,7 +980,7 @@ public:
             is_assignable<_Tp&, _Up const&>...
         >::value
     > >
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple& operator=(array<_Up, _Np> const& __array)
         _NOEXCEPT_((_And<is_nothrow_assignable<_Tp&, _Up const&>...>::value))
     {
@@ -995,7 +996,7 @@ public:
             is_assignable<_Tp&, _Up>...
         >::value
     > >
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     tuple& operator=(array<_Up, _Np>&& __array)
         _NOEXCEPT_((_And<is_nothrow_assignable<_Tp&, _Up>...>::value))
     {
@@ -1006,7 +1007,7 @@ public:
     }
 
     // [tuple.swap]
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     void swap(tuple& __t) _NOEXCEPT_(__all<__is_nothrow_swappable<_Tp>::value...>::value)
         {__base_.swap(__t.__base_);}
 };
@@ -1015,21 +1016,21 @@ template <>
 class _LIBCPP_TEMPLATE_VIS tuple<>
 {
 public:
-    _LIBCPP_INLINE_VISIBILITY
-    _LIBCPP_CONSTEXPR tuple() _NOEXCEPT = default;
+    _LIBCPP_INLINE_VISIBILITY constexpr
+        tuple() _NOEXCEPT = default;
     template <class _Alloc>
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
         tuple(allocator_arg_t, const _Alloc&) _NOEXCEPT {}
     template <class _Alloc>
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
         tuple(allocator_arg_t, const _Alloc&, const tuple&) _NOEXCEPT {}
     template <class _Up>
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
         tuple(array<_Up, 0>) _NOEXCEPT {}
     template <class _Alloc, class _Up>
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
         tuple(allocator_arg_t, const _Alloc&, array<_Up, 0>) _NOEXCEPT {}
-    _LIBCPP_INLINE_VISIBILITY
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     void swap(tuple&) _NOEXCEPT {}
 };
 
@@ -1047,7 +1048,7 @@ tuple(allocator_arg_t, _Alloc, tuple<_Tp...>) -> tuple<_Tp...>;
 #endif
 
 template <class ..._Tp>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 typename enable_if
 <
     __all<__is_swappable<_Tp>::value...>::value,
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index efc5c41c5f9b923b32c51d663c307549a1a7cea2..d028ca22fac0312e9ccc8a366f924c832ece5795 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -834,8 +834,8 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_array_v
 
 // is_pointer
 
-// In clang 10.0.0 and earlier __is_pointer didn't work with Objective-C types.
-#if __has_keyword(__is_pointer) && _LIBCPP_CLANG_VER > 1000
+// Before Clang 11, __is_pointer didn't work for Objective-C types.
+#if __has_keyword(__is_pointer) && !(defined(_LIBCPP_COMPILER_CLANG) && _LIBCPP_CLANG_VER < 1100)
 
 template<class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_pointer : _BoolConstant<__is_pointer(_Tp)> { };
@@ -1129,9 +1129,9 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_arithmetic_v
 
 // is_fundamental
 
-// In clang 9 and lower, this builtin did not work for nullptr_t. Additionally, in C++03 mode,
-// nullptr isn't defined by the compiler so, this builtin won't work.
-#if __has_keyword(__is_fundamental) && _LIBCPP_CLANG_VER > 900 && !defined(_LIBCPP_CXX03_LANG)
+// Before Clang 10, __is_fundamental didn't work for nullptr_t.
+// In C++03 nullptr_t is library-provided but must still count as "fundamental."
+#if __has_keyword(__is_fundamental) && !(defined(_LIBCPP_COMPILER_CLANG) && _LIBCPP_CLANG_VER < 1000) && !defined(_LIBCPP_CXX03_LANG)
 
 template<class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_fundamental : _BoolConstant<__is_fundamental(_Tp)> { };
@@ -1158,7 +1158,7 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_fundamental_v
 
 // is_scalar
 
-// >= 11 because in C++03 nullptr isn't actually nullptr
+// In C++03 nullptr_t is library-provided but must still count as "scalar."
 #if __has_keyword(__is_scalar) && !defined(_LIBCPP_CXX03_LANG)
 
 template<class _Tp>
@@ -1415,8 +1415,8 @@ template<class _Tp> using type_identity_t = typename type_identity<_Tp>::type;
 
 // is_signed
 
-// In clang 9 and earlier, this builtin did not work for floating points or enums
-#if __has_keyword(__is_signed) && _LIBCPP_CLANG_VER > 900
+// Before Clang 10, __is_signed didn't work for floating-point types or enums.
+#if __has_keyword(__is_signed) && !(defined(_LIBCPP_COMPILER_CLANG) && _LIBCPP_CLANG_VER < 1000)
 
 template<class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_signed : _BoolConstant<__is_signed(_Tp)> { };
@@ -1451,7 +1451,8 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_signed_v
 
 // is_unsigned
 
-#if __has_keyword(__is_unsigned)
+// Before Clang 13, __is_unsigned returned true for enums with signed underlying type.
+#if __has_keyword(__is_unsigned) && !(defined(_LIBCPP_COMPILER_CLANG) && _LIBCPP_CLANG_VER < 1300)
 
 template<class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_unsigned : _BoolConstant<__is_unsigned(_Tp)> { };
diff --git a/libcxx/include/unordered_map b/libcxx/include/unordered_map
index 83a41148b6813365872927d9466957448433b16d..fc10c8b9c2a2b8cbca0feca6f1b52305f92716c4 100644
--- a/libcxx/include/unordered_map
+++ b/libcxx/include/unordered_map
@@ -435,6 +435,7 @@ template <class Key, class T, class Hash, class Pred, class Alloc>
 #include <__hash_table>
 #include <__node_handle>
 #include <functional>
+#include <iterator> // __libcpp_erase_if_container
 #include <stdexcept>
 #include <tuple>
 #include <version>
@@ -1817,7 +1818,7 @@ inline _LIBCPP_INLINE_VISIBILITY
     typename unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::size_type
     erase_if(unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>& __c,
              _Predicate __pred) {
-  return __libcpp_erase_if_container(__c, __pred);
+  return _VSTD::__libcpp_erase_if_container(__c, __pred);
 }
 #endif
 
@@ -2550,7 +2551,7 @@ inline _LIBCPP_INLINE_VISIBILITY
     typename unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::size_type
     erase_if(unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __c,
              _Predicate __pred) {
-  return __libcpp_erase_if_container(__c, __pred);
+  return _VSTD::__libcpp_erase_if_container(__c, __pred);
 }
 #endif
 
diff --git a/libcxx/include/unordered_set b/libcxx/include/unordered_set
index 8a0f3aacbf305e60881690fd1d37786d5055eb6a..726e37389ff125cb77ad4c5bca68f7879bb9ee83 100644
--- a/libcxx/include/unordered_set
+++ b/libcxx/include/unordered_set
@@ -390,6 +390,7 @@ template <class Value, class Hash, class Pred, class Alloc>
 #include <__hash_table>
 #include <__node_handle>
 #include <functional>
+#include <iterator> // __libcpp_erase_if_container
 #include <version>
 
 #include <__debug>
@@ -1069,7 +1070,7 @@ inline _LIBCPP_INLINE_VISIBILITY
     typename unordered_set<_Value, _Hash, _Pred, _Alloc>::size_type
     erase_if(unordered_set<_Value, _Hash, _Pred, _Alloc>& __c,
              _Predicate __pred) {
-  return __libcpp_erase_if_container(__c, __pred);
+  return _VSTD::__libcpp_erase_if_container(__c, __pred);
 }
 #endif
 
@@ -1735,7 +1736,7 @@ inline _LIBCPP_INLINE_VISIBILITY
     typename unordered_multiset<_Value, _Hash, _Pred, _Alloc>::size_type
     erase_if(unordered_multiset<_Value, _Hash, _Pred, _Alloc>& __c,
              _Predicate __pred) {
-  return __libcpp_erase_if_container(__c, __pred);
+  return _VSTD::__libcpp_erase_if_container(__c, __pred);
 }
 #endif
 
diff --git a/libcxx/include/utility b/libcxx/include/utility
index ca3d3fe7dbaac4d67559c1869128b7cc1d8caa8f..181919c15ef2e6e7a8c9f76da8b86fce5ba51aad 100644
--- a/libcxx/include/utility
+++ b/libcxx/include/utility
@@ -76,15 +76,15 @@ struct pair
     template <class U, class V> explicit(see-below) pair(pair<U, V>&& p);        // constexpr in C++14
     template <class... Args1, class... Args2>
         pair(piecewise_construct_t, tuple<Args1...> first_args,
-             tuple<Args2...> second_args);
+             tuple<Args2...> second_args);                                       // constexpr in C++20
 
-    template <class U, class V> pair& operator=(const pair<U, V>& p);
+    template <class U, class V> pair& operator=(const pair<U, V>& p);            // constexpr in C++20
     pair& operator=(pair&& p) noexcept(is_nothrow_move_assignable<T1>::value &&
-                                       is_nothrow_move_assignable<T2>::value);
-    template <class U, class V> pair& operator=(pair<U, V>&& p);
+                                       is_nothrow_move_assignable<T2>::value);   // constexpr in C++20
+    template <class U, class V> pair& operator=(pair<U, V>&& p);                 // constexpr in C++20
 
     void swap(pair& p) noexcept(is_nothrow_swappable_v<T1> &&
-                                is_nothrow_swappable_v<T2>);
+                                is_nothrow_swappable_v<T2>);                     // constexpr in C++20
 };
 
 template <class T1, class T2> bool operator==(const pair<T1,T2>&, const pair<T1,T2>&); // constexpr in C++14
@@ -94,10 +94,10 @@ template <class T1, class T2> bool operator> (const pair<T1,T2>&, const pair<T1,
 template <class T1, class T2> bool operator>=(const pair<T1,T2>&, const pair<T1,T2>&); // constexpr in C++14
 template <class T1, class T2> bool operator<=(const pair<T1,T2>&, const pair<T1,T2>&); // constexpr in C++14
 
-template <class T1, class T2> pair<V1, V2> make_pair(T1&&, T2&&);   // constexpr in C++14
+template <class T1, class T2> pair<V1, V2> make_pair(T1&&, T2&&);                // constexpr in C++14
 template <class T1, class T2>
 void
-swap(pair<T1, T2>& x, pair<T1, T2>& y) noexcept(noexcept(x.swap(y)));
+swap(pair<T1, T2>& x, pair<T1, T2>& y) noexcept(noexcept(x.swap(y)));            // constexpr in C++20
 
 struct piecewise_construct_t { explicit piecewise_construct_t() = default; };
 inline constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
diff --git a/libcxx/include/variant b/libcxx/include/variant
index 770dd335bae21e94d49bdb40b1427b7cb361e354..4e3db1f7afbc7b08c16afe45b39a496c55b5b9fc 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -318,6 +318,33 @@ using __variant_index_t =
 template <class _IndexType>
 constexpr _IndexType __variant_npos = static_cast<_IndexType>(-1);
 
+template <class... _Types>
+class _LIBCPP_TEMPLATE_VIS variant;
+
+template <class... _Types>
+_LIBCPP_INLINE_VISIBILITY constexpr variant<_Types...>&
+__as_variant(variant<_Types...>& __vs) noexcept {
+  return __vs;
+}
+
+template <class... _Types>
+_LIBCPP_INLINE_VISIBILITY constexpr const variant<_Types...>&
+__as_variant(const variant<_Types...>& __vs) noexcept {
+  return __vs;
+}
+
+template <class... _Types>
+_LIBCPP_INLINE_VISIBILITY constexpr variant<_Types...>&&
+__as_variant(variant<_Types...>&& __vs) noexcept {
+  return _VSTD::move(__vs);
+}
+
+template <class... _Types>
+_LIBCPP_INLINE_VISIBILITY constexpr const variant<_Types...>&&
+__as_variant(const variant<_Types...>&& __vs) noexcept {
+  return _VSTD::move(__vs);
+}
+
 namespace __find_detail {
 
 template <class _Tp, class... _Types>
@@ -564,8 +591,9 @@ struct __variant {
   inline _LIBCPP_INLINE_VISIBILITY
   static constexpr decltype(auto) __visit_alt(_Visitor&& __visitor,
                                               _Vs&&... __vs) {
-    return __base::__visit_alt(_VSTD::forward<_Visitor>(__visitor),
-                               _VSTD::forward<_Vs>(__vs).__impl...);
+    return __base::__visit_alt(
+        _VSTD::forward<_Visitor>(__visitor),
+        _VSTD::__as_variant(_VSTD::forward<_Vs>(__vs)).__impl...);
   }
 
   template <class _Visitor, class... _Vs>
@@ -586,6 +614,7 @@ struct __variant {
         __make_value_visitor(_VSTD::forward<_Visitor>(__visitor)),
         _VSTD::forward<_Vs>(__vs)...);
   }
+
 #if _LIBCPP_STD_VER > 17
   template <class _Rp, class _Visitor, class... _Vs>
   inline _LIBCPP_INLINE_VISIBILITY
@@ -1637,18 +1666,21 @@ constexpr bool operator>=(const variant<_Types...>& __lhs,
 
 template <class... _Vs>
 inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS
-constexpr void __throw_if_valueless(_Vs&&... __vs) {
-  const bool __valueless = (... || __vs.valueless_by_exception());
+    _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr void
+    __throw_if_valueless(_Vs&&... __vs) {
+  const bool __valueless =
+      (... || _VSTD::__as_variant(__vs).valueless_by_exception());
   if (__valueless) {
-      __throw_bad_variant_access();
+    __throw_bad_variant_access();
   }
 }
 
-template <class _Visitor, class... _Vs>
+template <
+    class _Visitor, class... _Vs,
+    typename = void_t<decltype(_VSTD::__as_variant(_VSTD::declval<_Vs>()))...> >
 inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS
-constexpr decltype(auto) visit(_Visitor&& __visitor, _Vs&&... __vs) {
+    _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr
+    decltype(auto) visit(_Visitor&& __visitor, _Vs&&... __vs) {
   using __variant_detail::__visitation::__variant;
   _VSTD::__throw_if_valueless(_VSTD::forward<_Vs>(__vs)...);
   return __variant::__visit_value(_VSTD::forward<_Visitor>(__visitor),
@@ -1656,10 +1688,12 @@ constexpr decltype(auto) visit(_Visitor&& __visitor, _Vs&&... __vs) {
 }
 
 #if _LIBCPP_STD_VER > 17
-template <class _Rp, class _Visitor, class... _Vs>
+template <
+    class _Rp, class _Visitor, class... _Vs,
+    typename = void_t<decltype(_VSTD::__as_variant(_VSTD::declval<_Vs>()))...> >
 inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS
-constexpr _Rp visit(_Visitor&& __visitor, _Vs&&... __vs) {
+    _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS constexpr _Rp
+    visit(_Visitor&& __visitor, _Vs&&... __vs) {
   using __variant_detail::__visitation::__variant;
   _VSTD::__throw_if_valueless(_VSTD::forward<_Vs>(__vs)...);
   return __variant::__visit_value<_Rp>(_VSTD::forward<_Visitor>(__visitor),
diff --git a/libcxx/include/version b/libcxx/include/version
index becbfa5c2cdbe4416614efab600dd8bbdd8c878f..7ff5f914d76caf8eb52d1e50d39ff811ff0d9099 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -56,7 +56,7 @@ __cpp_lib_constexpr_functional                          201907L <functional>
 __cpp_lib_constexpr_iterator                            201811L <iterator>
 __cpp_lib_constexpr_memory                              201811L <memory>
 __cpp_lib_constexpr_numeric                             201911L <numeric>
-__cpp_lib_constexpr_string                              201907L <string>
+__cpp_lib_constexpr_string                              201811L <string>
 __cpp_lib_constexpr_string_view                         201811L <string_view>
 __cpp_lib_constexpr_tuple                               201811L <tuple>
 __cpp_lib_constexpr_utility                             201811L <utility>
@@ -160,7 +160,7 @@ __cpp_lib_type_trait_variable_templates                 201510L <type_traits>
 __cpp_lib_uncaught_exceptions                           201411L <exception>
 __cpp_lib_unordered_map_try_emplace                     201411L <unordered_map>
 __cpp_lib_unwrap_ref                                    201811L <functional>
-__cpp_lib_variant                                       201606L <variant>
+__cpp_lib_variant                                       202102L <variant>
 __cpp_lib_void_t                                        201411L <type_traits>
 
 */
@@ -259,7 +259,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # define __cpp_lib_type_trait_variable_templates        201510L
 # define __cpp_lib_uncaught_exceptions                  201411L
 # define __cpp_lib_unordered_map_try_emplace            201411L
-# define __cpp_lib_variant                              201606L
+# define __cpp_lib_variant                              202102L
 # define __cpp_lib_void_t                               201411L
 #endif
 
@@ -303,12 +303,12 @@ __cpp_lib_void_t                                        201411L <type_traits>
 // # define __cpp_lib_constexpr_complex                    201711L
 # define __cpp_lib_constexpr_dynamic_alloc              201907L
 # define __cpp_lib_constexpr_functional                 201907L
-// # define __cpp_lib_constexpr_iterator                   201811L
+# define __cpp_lib_constexpr_iterator                   201811L
 # define __cpp_lib_constexpr_memory                     201811L
 # define __cpp_lib_constexpr_numeric                    201911L
-// # define __cpp_lib_constexpr_string                     201907L
-// # define __cpp_lib_constexpr_string_view                201811L
-// # define __cpp_lib_constexpr_tuple                      201811L
+# define __cpp_lib_constexpr_string                     201811L
+# define __cpp_lib_constexpr_string_view                201811L
+# define __cpp_lib_constexpr_tuple                      201811L
 # define __cpp_lib_constexpr_utility                    201811L
 // # define __cpp_lib_constexpr_vector                     201907L
 // # define __cpp_lib_coroutine                            201902L
@@ -337,7 +337,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 #   define __cpp_lib_latch                              201907L
 # endif
 # define __cpp_lib_list_remove_return_type              201806L
-# ifndef _LIBCPP_HAS_NO_CONCEPTS
+# if !defined(_LIBCPP_HAS_NO_CONCEPTS)
 #   define __cpp_lib_math_constants                     201907L
 # endif
 // # define __cpp_lib_polymorphic_allocator                201902L
diff --git a/libcxx/include/wchar.h b/libcxx/include/wchar.h
index e4ba5004629417826e86e09230711facaa602727..b21a78968e23a1e1c78e51f0a34095d4976a0ebd 100644
--- a/libcxx/include/wchar.h
+++ b/libcxx/include/wchar.h
@@ -170,13 +170,13 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_PREFERRED_OVERLOAD
 }
 #endif
 
-#if defined(__cplusplus) && (defined(_LIBCPP_MSVCRT_LIKE) || defined(__MVS__))
+#if defined(__cplusplus) && defined(_LIBCPP_MSVCRT_LIKE)
 extern "C" {
 size_t mbsnrtowcs(wchar_t *__restrict dst, const char **__restrict src,
                   size_t nmc, size_t len, mbstate_t *__restrict ps);
 size_t wcsnrtombs(char *__restrict dst, const wchar_t **__restrict src,
                   size_t nwc, size_t len, mbstate_t *__restrict ps);
-}  // extern "C"
-#endif  // __cplusplus && (_LIBCPP_MSVCRT || __MVS__)
+}  // extern "C++"
+#endif  // __cplusplus && _LIBCPP_MSVCRT
 
 #endif  // _LIBCPP_WCHAR_H
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index 5a59b58d4363a04bfb340b6bc1b7540b3f1c0eca..015f2a9e22a83d0addc05e232913c5f820bb0b5e 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -92,8 +92,6 @@ elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "SunOS")
     )
 elseif(ZOS)
   list(APPEND LIBCXX_SOURCES
-    support/ibm/mbsnrtowcs.inc
-    support/ibm/wcsnrtombs.inc
     support/ibm/xlocale_zos.cpp
     )
 endif()
@@ -261,12 +259,13 @@ if (LIBCXX_ENABLE_SHARED)
   endif()
 endif()
 
+set(CMAKE_STATIC_LIBRARY_PREFIX "lib")
+
 # Build the static library.
 if (LIBCXX_ENABLE_STATIC)
   add_library(cxx_static STATIC ${exclude_from_all} ${LIBCXX_SOURCES} ${LIBCXX_HEADERS})
   target_link_libraries(cxx_static PUBLIC cxx-headers
                                    PRIVATE ${LIBCXX_LIBRARIES})
-  set(CMAKE_STATIC_LIBRARY_PREFIX "lib")
   set_target_properties(cxx_static
     PROPERTIES
       COMPILE_FLAGS "${LIBCXX_COMPILE_FLAGS}"
@@ -299,6 +298,9 @@ if (LIBCXX_ENABLE_STATIC)
     else()
       set(MERGE_ARCHIVES_ABI_TARGET
         "${CMAKE_STATIC_LIBRARY_PREFIX}${LIBCXX_CXX_STATIC_ABI_LIBRARY}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+      if (LIBCXX_CXX_ABI_LIBRARY_PATH)
+        set(MERGE_ARCHIVES_ABI_TARGET "${LIBCXX_CXX_ABI_LIBRARY_PATH}/${MERGE_ARCHIVES_ABI_TARGET}")
+      endif ()
     endif()
     if (APPLE)
       set(MERGE_ARCHIVES_LIBTOOL "--use-libtool" "--libtool" "${CMAKE_LIBTOOL}")
@@ -314,6 +316,7 @@ if (LIBCXX_ENABLE_STATIC)
       "${MERGE_ARCHIVES_ABI_TARGET}"
       "${MERGE_ARCHIVES_SEARCH_PATHS}"
     WORKING_DIRECTORY ${LIBCXX_BUILD_DIR}
+    DEPENDS ${MERGE_ARCHIVES_ABI_TARGET}
     )
   endif()
 endif()
diff --git a/libcxx/src/filesystem/directory_iterator.cpp b/libcxx/src/filesystem/directory_iterator.cpp
index bb3653076bfcf95839c4f4d7bb1ac6d590774337..7b83ba9ff12393cf7354afd6565ba4c0c5bdb7a1 100644
--- a/libcxx/src/filesystem/directory_iterator.cpp
+++ b/libcxx/src/filesystem/directory_iterator.cpp
@@ -273,7 +273,7 @@ directory_iterator& directory_iterator::__increment(error_code* ec) {
     path root = move(__imp_->__root_);
     __imp_.reset();
     if (m_ec)
-      err.report(m_ec, "at root \"%s\"", root);
+      err.report(m_ec, "at root " PATH_CSTR_FMT, root.c_str());
   }
   return *this;
 }
@@ -360,7 +360,7 @@ void recursive_directory_iterator::__advance(error_code* ec) {
   if (m_ec) {
     path root = move(stack.top().__root_);
     __imp_.reset();
-    err.report(m_ec, "at root \"%s\"", root);
+    err.report(m_ec, "at root " PATH_CSTR_FMT, root.c_str());
   } else {
     __imp_.reset();
   }
@@ -405,7 +405,8 @@ bool recursive_directory_iterator::__try_recursion(error_code* ec) {
     } else {
       path at_ent = move(curr_it.__entry_.__p_);
       __imp_.reset();
-      err.report(m_ec, "attempting recursion into \"%s\"", at_ent);
+      err.report(m_ec, "attempting recursion into " PATH_CSTR_FMT,
+                 at_ent.c_str());
     }
   }
   return false;
diff --git a/libcxx/src/filesystem/filesystem_common.h b/libcxx/src/filesystem/filesystem_common.h
index 22bf8404860ab9f6bf1a728500eee3cc8c38e66f..f7f9013edb738ac31faca6949a2415a68a085e40 100644
--- a/libcxx/src/filesystem/filesystem_common.h
+++ b/libcxx/src/filesystem/filesystem_common.h
@@ -42,8 +42,10 @@
 
 #if defined(_LIBCPP_WIN32API)
 #define PS(x) (L##x)
+#define PATH_CSTR_FMT "\"%ls\""
 #else
 #define PS(x) (x)
+#define PATH_CSTR_FMT "\"%s\""
 #endif
 
 _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
@@ -57,68 +59,47 @@ errc __win_err_to_errc(int err);
 
 namespace {
 
-static string format_string_imp(const char* msg, ...) {
-  // we might need a second shot at this, so pre-emptivly make a copy
-  struct GuardVAList {
-    va_list& target;
-    bool active = true;
-    GuardVAList(va_list& tgt) : target(tgt), active(true) {}
-    void clear() {
-      if (active)
-        va_end(target);
-      active = false;
-    }
-    ~GuardVAList() {
-      if (active)
-        va_end(target);
-    }
-  };
-  va_list args;
-  va_start(args, msg);
-  GuardVAList args_guard(args);
-
-  va_list args_cp;
-  va_copy(args_cp, args);
-  GuardVAList args_copy_guard(args_cp);
-
-  std::string result;
-
-  array<char, 256> local_buff;
-  size_t size_with_null = local_buff.size();
-  auto ret = ::vsnprintf(local_buff.data(), size_with_null, msg, args_cp);
-
-  args_copy_guard.clear();
-
-  // handle empty expansion
-  if (ret == 0)
-    return result;
-  if (static_cast<size_t>(ret) < size_with_null) {
-    result.assign(local_buff.data(), static_cast<size_t>(ret));
-    return result;
+static _LIBCPP_FORMAT_PRINTF(1, 0) string
+format_string_impl(const char* msg, va_list ap) {
+  array<char, 256> buf;
+
+  va_list apcopy;
+  va_copy(apcopy, ap);
+  int ret = ::vsnprintf(buf.data(), buf.size(), msg, apcopy);
+  va_end(apcopy);
+
+  string result;
+  if (static_cast<size_t>(ret) < buf.size()) {
+    result.assign(buf.data(), static_cast<size_t>(ret));
+  } else {
+    // we did not provide a long enough buffer on our first attempt. The
+    // return value is the number of bytes (excluding the null byte) that are
+    // needed for formatting.
+    size_t size_with_null = static_cast<size_t>(ret) + 1;
+    result.__resize_default_init(size_with_null - 1);
+    ret = ::vsnprintf(&result[0], size_with_null, msg, ap);
+    _LIBCPP_ASSERT(static_cast<size_t>(ret) == (size_with_null - 1), "TODO");
   }
-
-  // we did not provide a long enough buffer on our first attempt. The
-  // return value is the number of bytes (excluding the null byte) that are
-  // needed for formatting.
-  size_with_null = static_cast<size_t>(ret) + 1;
-  result.__resize_default_init(size_with_null - 1);
-  ret = ::vsnprintf(&result[0], size_with_null, msg, args);
-  _LIBCPP_ASSERT(static_cast<size_t>(ret) == (size_with_null - 1), "TODO");
-
   return result;
 }
 
-const path::value_type* unwrap(path::string_type const& s) { return s.c_str(); }
-const path::value_type* unwrap(path const& p) { return p.native().c_str(); }
-template <class Arg>
-Arg const& unwrap(Arg const& a) {
-  static_assert(!is_class<Arg>::value, "cannot pass class here");
-  return a;
-}
-
-template <class... Args>
-string format_string(const char* fmt, Args const&... args) {
-  return format_string_imp(fmt, unwrap(args)...);
+static _LIBCPP_FORMAT_PRINTF(1, 2) string
+format_string(const char* msg, ...) {
+  string ret;
+  va_list ap;
+  va_start(ap, msg);
+#ifndef _LIBCPP_NO_EXCEPTIONS
+  try {
+#endif  // _LIBCPP_NO_EXCEPTIONS
+    ret = format_string_impl(msg, ap);
+#ifndef _LIBCPP_NO_EXCEPTIONS
+  } catch (...) {
+    va_end(ap);
+    throw;
+  }
+#endif  // _LIBCPP_NO_EXCEPTIONS
+  va_end(ap);
+  return ret;
 }
 
 error_code capture_errno() {
@@ -190,14 +171,14 @@ struct ErrorHandler {
     _LIBCPP_UNREACHABLE();
   }
 
-  template <class... Args>
-  T report(const error_code& ec, const char* msg, Args const&... args) const {
+  _LIBCPP_FORMAT_PRINTF(3, 0)
+  void report_impl(const error_code& ec, const char* msg, va_list ap) const {
     if (ec_) {
       *ec_ = ec;
-      return error_value<T>();
+      return;
     }
     string what =
-        string("in ") + func_name_ + ": " + format_string(msg, args...);
+        string("in ") + func_name_ + ": " + format_string_impl(msg, ap);
     switch (bool(p1_) + bool(p2_)) {
     case 0:
       __throw_filesystem_error(what, ec);
@@ -209,11 +190,44 @@ struct ErrorHandler {
     _LIBCPP_UNREACHABLE();
   }
 
-  T report(errc const& err) const { return report(make_error_code(err)); }
+  _LIBCPP_FORMAT_PRINTF(3, 4)
+  T report(const error_code& ec, const char* msg, ...) const {
+    va_list ap;
+    va_start(ap, msg);
+#ifndef _LIBCPP_NO_EXCEPTIONS
+    try {
+#endif  // _LIBCPP_NO_EXCEPTIONS
+      report_impl(ec, msg, ap);
+#ifndef _LIBCPP_NO_EXCEPTIONS
+    } catch (...) {
+      va_end(ap);
+      throw;
+    }
+#endif  // _LIBCPP_NO_EXCEPTIONS
+    va_end(ap);
+    return error_value<T>();
+  }
+
+  T report(errc const& err) const {
+    return report(make_error_code(err));
+  }
 
-  template <class... Args>
-  T report(errc const& err, const char* msg, Args const&... args) const {
-    return report(make_error_code(err), msg, args...);
+  _LIBCPP_FORMAT_PRINTF(3, 4)
+  T report(errc const& err, const char* msg, ...) const {
+    va_list ap;
+    va_start(ap, msg);
+#ifndef _LIBCPP_NO_EXCEPTIONS
+    try {
+#endif  // _LIBCPP_NO_EXCEPTIONS
+      report_impl(make_error_code(err), msg, ap);
+#ifndef _LIBCPP_NO_EXCEPTIONS
+    } catch (...) {
+      va_end(ap);
+      throw;
+    }
+#endif  // _LIBCPP_NO_EXCEPTIONS
+    va_end(ap);
+    return error_value<T>();
   }
 
 private:
@@ -453,6 +467,15 @@ inline TimeSpec extract_atime(StatT const& st) {
   TimeSpec TS = {st.st_atime, 0};
   return TS;
 }
+#elif defined(_AIX)
+inline TimeSpec extract_mtime(StatT const& st) {
+  TimeSpec TS = {st.st_mtime, st.st_mtime_n};
+  return TS;
+}
+inline TimeSpec extract_atime(StatT const& st) {
+  TimeSpec TS = {st.st_atime, st.st_atime_n};
+  return TS;
+}
 #else
 inline TimeSpec extract_mtime(StatT const& st) { return st.st_mtim; }
 inline TimeSpec extract_atime(StatT const& st) { return st.st_atim; }
diff --git a/libcxx/src/filesystem/operations.cpp b/libcxx/src/filesystem/operations.cpp
index a002d0a5c93e2b4c7c9472d72fb1e7f0e13bdd4b..e604cc6c57c05037d66a3b42cc43441719fdfced 100644
--- a/libcxx/src/filesystem/operations.cpp
+++ b/libcxx/src/filesystem/operations.cpp
@@ -667,27 +667,20 @@ _FilesystemClock::time_point _FilesystemClock::now() noexcept {
 
 filesystem_error::~filesystem_error() {}
 
-#if defined(_LIBCPP_WIN32API)
-#define PS_FMT "%ls"
-#else
-#define PS_FMT "%s"
-#endif
-
 void filesystem_error::__create_what(int __num_paths) {
   const char* derived_what = system_error::what();
   __storage_->__what_ = [&]() -> string {
-    const path::value_type* p1 = path1().native().empty() ? PS("\"\"") : path1().c_str();
-    const path::value_type* p2 = path2().native().empty() ? PS("\"\"") : path2().c_str();
     switch (__num_paths) {
-    default:
+    case 0:
       return detail::format_string("filesystem error: %s", derived_what);
     case 1:
-      return detail::format_string("filesystem error: %s [" PS_FMT "]", derived_what,
-                                   p1);
+      return detail::format_string("filesystem error: %s [" PATH_CSTR_FMT "]",
+                                   derived_what, path1().c_str());
     case 2:
-      return detail::format_string("filesystem error: %s [" PS_FMT "] [" PS_FMT "]",
-                                   derived_what, p1, p2);
+      return detail::format_string("filesystem error: %s [" PATH_CSTR_FMT "] [" PATH_CSTR_FMT "]",
+                                   derived_what, path1().c_str(), path2().c_str());
     }
+    _LIBCPP_UNREACHABLE();
   }();
 }
 
@@ -1455,11 +1448,11 @@ path __temp_directory_path(error_code* ec) {
   error_code m_ec;
   file_status st = detail::posix_stat(p, &m_ec);
   if (!status_known(st))
-    return err.report(m_ec, "cannot access path \"" PS_FMT "\"", p);
+    return err.report(m_ec, "cannot access path " PATH_CSTR_FMT, p.c_str());
 
   if (!exists(st) || !is_directory(st))
-    return err.report(errc::not_a_directory, "path \"" PS_FMT "\" is not a directory",
-                      p);
+    return err.report(errc::not_a_directory,
+                      "path " PATH_CSTR_FMT " is not a directory", p.c_str());
 
   return p;
 }
diff --git a/libcxx/src/support/ibm/mbsnrtowcs.inc b/libcxx/src/support/ibm/mbsnrtowcs.inc
deleted file mode 100644
index aad3057df7924bf7db8b6ffc098098fba136cc84..0000000000000000000000000000000000000000
--- a/libcxx/src/support/ibm/mbsnrtowcs.inc
+++ /dev/null
@@ -1,72 +0,0 @@
-/*-
- * Some portions of this implementation are copied from
- * FreeBSD libc.  These are covered by the following copyright:
- *
- * Copyright (c) 2002-2004 Tim J. Robbins.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-size_t
-mbsnrtowcs(wchar_t * __restrict dst, const char ** __restrict src,
-    size_t nms, size_t len, mbstate_t * __restrict ps) {
-  const char *s;
-  size_t nchr;
-  wchar_t wc;
-  size_t nb;
-
-  s = *src;
-  nchr = 0;
-
-  if (dst == NULL) {
-    for (;;) {
-      if ((nb = mbrtowc(&wc, s, nms, ps)) == (size_t)-1)
-        /* Invalid sequence - mbrtowc() sets errno. */
-        return ((size_t)-1);
-      else if (nb == 0 || nb == (size_t)-2)
-        return (nchr);
-      s += nb;
-      nms -= nb;
-      nchr++;
-    }
-    /*NOTREACHED*/
-  }
-
-  while (len-- > 0) {
-    if ((nb = mbrtowc(dst, s, nms, ps)) == (size_t)-1) {
-      *src = s;
-      return ((size_t)-1);
-    } else if (nb == (size_t)-2) {
-      *src = s + nms;
-      return (nchr);
-    } else if (nb == 0) {
-      *src = NULL;
-      return (nchr);
-    }
-    s += nb;
-    nms -= nb;
-    nchr++;
-    dst++;
-  }
-  *src = s;
-  return (nchr);
-}
diff --git a/libcxx/src/support/ibm/wcsnrtombs.inc b/libcxx/src/support/ibm/wcsnrtombs.inc
deleted file mode 100644
index 89c435e5cc4adb1419ca70632a273543dba4e5ce..0000000000000000000000000000000000000000
--- a/libcxx/src/support/ibm/wcsnrtombs.inc
+++ /dev/null
@@ -1,90 +0,0 @@
-/*-
- * Copyright (c) 2002-2004 Tim J. Robbins.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-size_t
-wcsnrtombs(char * __restrict dst, const wchar_t ** __restrict src,
-    size_t nwc, size_t len, mbstate_t * __restrict ps) {
-  mbstate_t mbsbak;
-  char buf[MB_CUR_MAX];
-  const wchar_t *s;
-  size_t nbytes;
-  size_t nb;
-
-  s = *src;
-  nbytes = 0;
-
-  if (dst == NULL) {
-    while (nwc-- > 0) {
-      if ((nb = wcrtomb(buf, *s, ps)) == (size_t)-1)
-        /* Invalid character - wcrtomb() sets errno. */
-        return ((size_t)-1);
-      else if (*s == L'\0')
-        return (nbytes + nb - 1);
-      s++;
-      nbytes += nb;
-    }
-    return (nbytes);
-  }
-
-  while (len > 0 && nwc-- > 0) {
-    if (len > (size_t)MB_CUR_MAX) {
-      /* Enough space to translate in-place. */
-      if ((nb = wcrtomb(dst, *s, ps)) == (size_t)-1) {
-        *src = s;
-        return ((size_t)-1);
-      }
-    } else {
-      /*
-       * May not be enough space; use temp. buffer.
-       *
-       * We need to save a copy of the conversion state
-       * here so we can restore it if the multibyte
-       * character is too long for the buffer.
-       */
-      mbsbak = *ps;
-      if ((nb = wcrtomb(buf, *s, ps)) == (size_t)-1) {
-        *src = s;
-        return ((size_t)-1);
-      }
-      if (nb > len) {
-        /* MB sequence for character won't fit. */
-        *ps = mbsbak;
-        break;
-      }
-      memcpy(dst, buf, nb);
-    }
-    if (*s == L'\0') {
-      *src = NULL;
-      return (nbytes + nb - 1);
-    }
-    s++;
-    dst += nb;
-    len -= nb;
-    nbytes += nb;
-  }
-  *src = s;
-  return (nbytes);
-}
-
diff --git a/libcxx/src/support/win32/support.cpp b/libcxx/src/support/win32/support.cpp
index 52453f5479266aa2ffc487e650a183b8684bc5aa..5890e669a34e48d5e462cc6c28927bd388952cd2 100644
--- a/libcxx/src/support/win32/support.cpp
+++ b/libcxx/src/support/win32/support.cpp
@@ -22,7 +22,10 @@ int __libcpp_vasprintf( char **sptr, const char *__restrict format, va_list ap )
 {
     *sptr = NULL;
     // Query the count required.
-    int count = _vsnprintf( NULL, 0, format, ap );
+    va_list ap_copy;
+    va_copy(ap_copy, ap);
+    int count = _vsnprintf( NULL, 0, format, ap_copy );
+    va_end(ap_copy);
     if (count < 0)
         return count;
     size_t buffer_size = static_cast<size_t>(count) + 1;
diff --git a/libcxx/src/support/win32/thread_win32.cpp b/libcxx/src/support/win32/thread_win32.cpp
index 2b1aa5635ac798fa7c49c826108214db996d59af..63c5aa65374fd9b3666d4118342d6ee8b07bea86 100644
--- a/libcxx/src/support/win32/thread_win32.cpp
+++ b/libcxx/src/support/win32/thread_win32.cpp
@@ -246,10 +246,8 @@ void __libcpp_thread_yield()
 
 void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns)
 {
-  using namespace chrono;
-  // round-up to the nearest milisecond
-  milliseconds __ms =
-      duration_cast<milliseconds>(__ns + chrono::nanoseconds(999999));
+  // round-up to the nearest millisecond
+  chrono::milliseconds __ms = chrono::ceil<chrono::milliseconds>(__ns);
   // FIXME(compnerd) this should be an alertable sleep (WFSO or SleepEx)
   Sleep(__ms.count());
 }
@@ -305,7 +303,7 @@ bool __libcpp_semaphore_wait(__libcpp_semaphore_t* __sem)
 bool __libcpp_semaphore_wait_timed(__libcpp_semaphore_t* __sem,
                                    chrono::nanoseconds const& __ns)
 {
-  chrono::milliseconds __ms = std::chrono::ceil<chrono::milliseconds>(__ns);
+  chrono::milliseconds __ms = chrono::ceil<chrono::milliseconds>(__ns);
   return WaitForSingleObjectEx(*(PHANDLE)__sem, __ms.count(), false) ==
          WAIT_OBJECT_0;
 }
diff --git a/libcxx/test/CMakeLists.txt b/libcxx/test/CMakeLists.txt
index 61ef4aea3c7f9eb9636ec66a676cc00123af71e8..0892822b943ae76af281f75c2a2fe5c4b2eaf8e6 100644
--- a/libcxx/test/CMakeLists.txt
+++ b/libcxx/test/CMakeLists.txt
@@ -103,7 +103,7 @@ if (LIBCXX_INCLUDE_TESTS)
     DEPENDS cxx ${LIBCXX_TEST_DEPS}
     COMMENT "Builds dependencies required to run the test suite.")
 
-  add_lit_target(check-cxx
+  add_lit_testsuite(check-cxx
     "Running libcxx tests"
     ${CMAKE_CURRENT_BINARY_DIR}
     DEPENDS cxx-test-depends
diff --git a/libcxx/test/libcxx/debug/extern-templates.sh.cpp b/libcxx/test/libcxx/debug/extern-templates.sh.cpp
index 0e19895ba8f0a390484c624c607bb3fbb11a88a3..3e8e0b766be3261435550276b30a9048ddfb62fc 100644
--- a/libcxx/test/libcxx/debug/extern-templates.sh.cpp
+++ b/libcxx/test/libcxx/debug/extern-templates.sh.cpp
@@ -14,6 +14,8 @@
 // UNSUPPORTED: libcxx-no-debug-mode
 // UNSUPPORTED: libcpp-has-no-localization
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // RUN: %{cxx} %{flags} %{compile_flags} %s %{link_flags} -fPIC -DTU1 -D_LIBCPP_DEBUG=1 -fvisibility=hidden -shared -o %t.lib
 // RUN: cd %T && %{cxx} %{flags} %{compile_flags} %s ./%basename_t.tmp.lib %{link_flags} -fPIC -DTU2 -D_LIBCPP_DEBUG=1 -fvisibility=hidden -o %t.exe
 // RUN: %{exec} %t.exe
diff --git a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.pass.cpp b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.pass.cpp
index cfb7054d3e2d40f4756459a87d83515b7cb9c76e..db8eb3357966ab4b8ff81e957679e83e6820d4c5 100644
--- a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.pass.cpp
+++ b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.pass.cpp
@@ -18,6 +18,8 @@
 // UNSUPPORTED: apple-clang-9
 // UNSUPPORTED: gcc-5
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // All entities to which libc++ applies [[nodiscard]] as an extension should
 // be tested here and in nodiscard_extensions.fail.cpp. They should also
 // be listed in `UsingLibcxx.rst` in the documentation for the extension.
diff --git a/libcxx/test/libcxx/include_as_c.sh.cpp b/libcxx/test/libcxx/include_as_c.sh.cpp
index 60e0a26ed576895dcf6700e3afbe84a4c31e9b1e..bbdcd022940b4ffe7b901c73d318e5d38004ac2f 100644
--- a/libcxx/test/libcxx/include_as_c.sh.cpp
+++ b/libcxx/test/libcxx/include_as_c.sh.cpp
@@ -14,6 +14,8 @@
 // file as C, but we're passing C++ flags on the command-line.
 // UNSUPPORTED: gcc
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // Test that the C wrapper headers can be included when compiling them as C.
 
 // NOTE: It's not common or recommended to have libc++ in the header search
diff --git a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
index 5a5735934e95f41d87ef0ce22c7aa3e8f4ae253d..44f90b019223a74fa5a42d35a7a5318568a4d094 100644
--- a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
@@ -9,6 +9,8 @@
 // UNSUPPORTED: c++03
 // ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../../../src/filesystem
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/libcxx/language.support/has_aligned_alloc.compile.pass.cpp b/libcxx/test/libcxx/language.support/has_aligned_alloc.compile.pass.cpp
index d1b41de5dc1a9a734b9f3f386d70add13d0db28e..5e863717ea39fa89eabcbc574310576bafc22bf8 100644
--- a/libcxx/test/libcxx/language.support/has_aligned_alloc.compile.pass.cpp
+++ b/libcxx/test/libcxx/language.support/has_aligned_alloc.compile.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // Make sure TEST_HAS_ALIGNED_ALLOC (defined by the test suite) and
 // _LIBCPP_HAS_ALIGNED_ALLOC (defined by libc++) stay in sync.
 
diff --git a/libcxx/test/libcxx/language.support/has_timespec_get.compile.pass.cpp b/libcxx/test/libcxx/language.support/has_timespec_get.compile.pass.cpp
index 8b86a5ef97195bcac007cd0ef8bc1e12cb4779e3..a68a7eaf85253251a5252bb35e1f9c2ff0701abb 100644
--- a/libcxx/test/libcxx/language.support/has_timespec_get.compile.pass.cpp
+++ b/libcxx/test/libcxx/language.support/has_timespec_get.compile.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // Make sure TEST_HAS_TIMESPEC_GET (defined by the test suite) and
 // _LIBCPP_HAS_TIMESPEC_GET (defined by libc++) stay in sync.
 
diff --git a/libcxx/test/libcxx/language.support/support.rtti/type.info/type_info.comparison.merged.sh.cpp b/libcxx/test/libcxx/language.support/support.rtti/type.info/type_info.comparison.merged.sh.cpp
index da80337f90bdadd6a6d62d48dade03badae7dcdb..cd742a6f09075efc86751e59bedfbec8a1c0314a 100644
--- a/libcxx/test/libcxx/language.support/support.rtti/type.info/type_info.comparison.merged.sh.cpp
+++ b/libcxx/test/libcxx/language.support/support.rtti/type.info/type_info.comparison.merged.sh.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: no-rtti
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // RUN: %{cxx} %s %{flags} %{compile_flags} -c -o %t.tu1.o -DTU1 -D_LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION=1
 // RUN: %{cxx} %s %{flags} %{compile_flags} -c -o %t.tu2.o -DTU2 -D_LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION=1
 // RUN: %{cxx} %s %{flags} %{compile_flags} -c -o %t.main.o -DMAIN -D_LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION=1
diff --git a/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_destruction_order.pass.cpp b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_destruction_order.pass.cpp
index 980349866a98784b4d102f09bdd492823aa6503c..47bc940b27e244d53f4107fcfcbcbf98d2168a86 100644
--- a/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_destruction_order.pass.cpp
+++ b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_destruction_order.pass.cpp
@@ -17,6 +17,8 @@
 // There were assertion failures in both parse and codegen, which are fixed in clang 11.
 // UNSUPPORTED: gcc, clang-4, clang-5, clang-6, clang-7, clang-8, clang-9, clang-10
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <memory>
 #include <cassert>
 
diff --git a/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_ret.pass.cpp b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_ret.pass.cpp
index 73fe18becc3a96cf7aa539e50b5c110bb7ccc269..b4bc4f8f9349d4a59164695969d94edf5a79a97a 100644
--- a/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_ret.pass.cpp
+++ b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_ret.pass.cpp
@@ -15,6 +15,8 @@
 // There were assertion failures in both parse and codegen, which are fixed in clang 11.
 // UNSUPPORTED: gcc, clang-4, clang-5, clang-6, clang-7, clang-8, clang-9, clang-10
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <memory>
 #include <cassert>
 
diff --git a/libcxx/test/libcxx/memory/trivial_abi/weak_ptr_ret.pass.cpp b/libcxx/test/libcxx/memory/trivial_abi/weak_ptr_ret.pass.cpp
index e69c94506f2aff0e19b18cbbec3d8576e0739007..b61b0f6b468a7ca29d15f9ba33567f0b280be42b 100644
--- a/libcxx/test/libcxx/memory/trivial_abi/weak_ptr_ret.pass.cpp
+++ b/libcxx/test/libcxx/memory/trivial_abi/weak_ptr_ret.pass.cpp
@@ -15,6 +15,8 @@
 // There were assertion failures in both parse and codegen, which are fixed in clang 11.
 // UNSUPPORTED: gcc, clang-4, clang-5, clang-6, clang-7, clang-8, clang-9, clang-10
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <memory>
 #include <cassert>
 
diff --git a/libcxx/test/libcxx/modules/stdint_h_exports.compile.pass.cpp b/libcxx/test/libcxx/modules/stdint_h_exports.compile.pass.cpp
index 2b1997a24e6d3b3559cc03e97679eb306a0c6938..ad59b3ce82cd72eb0c30d9aa2d8e556ab278cefd 100644
--- a/libcxx/test/libcxx/modules/stdint_h_exports.compile.pass.cpp
+++ b/libcxx/test/libcxx/modules/stdint_h_exports.compile.pass.cpp
@@ -17,6 +17,8 @@
 // REQUIRES: modules-support
 // ADDITIONAL_COMPILE_FLAGS: -fmodules
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <stdint.h>
 
 int main(int, char**) {
diff --git a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py
index 0393190df53c1facdc8e76a4a7de63ba071693fa..2cbb0616523c665da8edb841c68c7e591d4678eb 100644
--- a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py
+++ b/libcxx/test/libcxx/selftest/dsl/dsl.sh.py
@@ -6,6 +6,8 @@
 #
 #===----------------------------------------------------------------------===##
 
+# XFAIL: LIBCXX-WINDOWS-FIXME
+
 # Note: We prepend arguments with 'x' to avoid thinking there are too few
 #       arguments in case an argument is an empty string.
 # RUN: %{python} %s x%S \
diff --git a/libcxx/test/std/concepts/callable/invocable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.callable/concept.invocable/invocable.compile.pass.cpp
similarity index 92%
rename from libcxx/test/std/concepts/callable/invocable.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.callable/concept.invocable/invocable.compile.pass.cpp
index cd415d8d09987cf25047780918f36508c3e71630..f201910d503105deae368d76654804962e8465e0 100644
--- a/libcxx/test/std/concepts/callable/invocable.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.callable/concept.invocable/invocable.compile.pass.cpp
@@ -18,15 +18,17 @@
 #include <random>
 #include <type_traits>
 
-#include "functions.h"
+#include "../functions.h"
 
+// clang-format off
 template <class F, class... Args>
-requires std::invocable<F, Args...> constexpr void
-ModelsInvocable(F, Args&&...) noexcept{};
+requires std::invocable<F, Args...>
+constexpr void ModelsInvocable(F, Args&&...) noexcept{}
 
 template <class F, class... Args>
-requires(!std::invocable<F, Args...>) constexpr
-    void NotInvocable(F, Args&&...) noexcept {}
+requires(!std::invocable<F, Args...>)
+constexpr void NotInvocable(F, Args&&...) noexcept {}
+// clang-format on
 
 static_assert(!std::invocable<void>);
 static_assert(!std::invocable<void*>);
diff --git a/libcxx/test/std/concepts/callable/regularinvocable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.callable/concept.regularinvocable/regular_invocable.pass.cpp
similarity index 85%
rename from libcxx/test/std/concepts/callable/regularinvocable.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.callable/concept.regularinvocable/regular_invocable.pass.cpp
index b085b7e500223266478afbde957349d21ec833cb..3912504d506620ab8f456291e22ded0117304f84 100644
--- a/libcxx/test/std/concepts/callable/regularinvocable.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.callable/concept.regularinvocable/regular_invocable.pass.cpp
@@ -17,21 +17,23 @@
 #include <random>
 #include <type_traits>
 
-#include "functions.h"
+#include "../functions.h"
 
+// clang-format off
 template <class F, class... Args>
-requires std::invocable<F, Args...> constexpr void
-ModelsRegularInvocable(F, Args&&...) noexcept{};
+requires std::regular_invocable<F, Args...>
+constexpr void ModelsRegularInvocable(F, Args&&...) noexcept {}
 
 template <class F, class... Args>
-requires(!std::invocable<F, Args...>) constexpr
-    void NotRegularInvocable(F, Args&&...) noexcept {}
+requires (!std::regular_invocable<F, Args...>)
+constexpr void NotRegularInvocable(F, Args&&...) noexcept {}
+// clang-format on
 
-static_assert(!std::invocable<void>);
-static_assert(!std::invocable<void*>);
-static_assert(!std::invocable<int>);
-static_assert(!std::invocable<int&>);
-static_assert(!std::invocable<int&&>);
+static_assert(!std::regular_invocable<void>);
+static_assert(!std::regular_invocable<void*>);
+static_assert(!std::regular_invocable<int>);
+static_assert(!std::regular_invocable<int&>);
+static_assert(!std::regular_invocable<int&&>);
 
 int main(int, char**) {
   {
diff --git a/libcxx/test/std/concepts/callable/functions.h b/libcxx/test/std/concepts/concepts.callable/functions.h
similarity index 100%
rename from libcxx/test/std/concepts/callable/functions.h
rename to libcxx/test/std/concepts/concepts.callable/functions.h
diff --git a/libcxx/test/std/concepts/comparison/concepts.equalitycomparable/equality_comparable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/comparison/concepts.equalitycomparable/equality_comparable.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/comparison/concepts.equalitycomparable/equality_comparable_with.compile.pass.cpp b/libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/comparison/concepts.equalitycomparable/equality_comparable_with.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/comparison/types.h b/libcxx/test/std/concepts/concepts.compare/types.h
similarity index 100%
rename from libcxx/test/std/concepts/comparison/types.h
rename to libcxx/test/std/concepts/concepts.compare/types.h
diff --git a/libcxx/test/std/concepts/lang/assignable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.assignable/assignable_from.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/assignable.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.assignable/assignable_from.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/lang/common.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.common/common_with.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/common.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.common/common_with.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/lang/commonreference.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.commonref/common_reference.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/commonreference.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.commonref/common_reference.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/concept.constructible/constructible_from.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.constructible/constructible_from.compile.pass.cpp
similarity index 99%
rename from libcxx/test/std/concepts/concept.constructible/constructible_from.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.constructible/constructible_from.compile.pass.cpp
index 2e59b42c3da1bc13b2b73e54d3423ffa2bd4595c..4db2f9299192a46b0f81ed87db30bae6fad0f4c2 100644
--- a/libcxx/test/std/concepts/concept.constructible/constructible_from.compile.pass.cpp
+++ b/libcxx/test/std/concepts/concepts.lang/concept.constructible/constructible_from.compile.pass.cpp
@@ -149,3 +149,5 @@ void test() {
   test<std::array<int, 1>, int>();
   test<std::array<int, 1>, int, int>();
 }
+
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/concepts/lang/convertible.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.convertible/convertible_to.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/convertible.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.convertible/convertible_to.pass.cpp
diff --git a/libcxx/test/std/concepts/lang/copyconstructible.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.copyconstructible/copy_constructible.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/copyconstructible.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.copyconstructible/copy_constructible.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/concept.default.init/default_initializable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.default.init/default_initializable.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/concept.default.init/default_initializable.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.default.init/default_initializable.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/concept.default.init/default_initializable.verify.cpp b/libcxx/test/std/concepts/concepts.lang/concept.default.init/default_initializable.verify.cpp
similarity index 100%
rename from libcxx/test/std/concepts/concept.default.init/default_initializable.verify.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.default.init/default_initializable.verify.cpp
diff --git a/libcxx/test/std/concepts/lang/derived.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.derived/derived_from.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/derived.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.derived/derived_from.pass.cpp
diff --git a/libcxx/test/std/concepts/concept.destructible/destructible.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.destructible/destructible.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/concept.destructible/destructible.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.destructible/destructible.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/lang/moveconstructible.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.moveconstructible/move_constructible.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/moveconstructible.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.moveconstructible/move_constructible.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/lang/same_as.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.same/same_as.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/same_as.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.same/same_as.pass.cpp
diff --git a/libcxx/test/std/concepts/lang/swappable.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.swappable/swappable.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/swappable.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.swappable/swappable.pass.cpp
diff --git a/libcxx/test/std/concepts/lang/swappable_with.compile.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concept.swappable/swappable_with.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/lang/swappable_with.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.lang/concept.swappable/swappable_with.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/arithmetic.h b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/arithmetic.h
new file mode 100644
index 0000000000000000000000000000000000000000..bb7016508ccabe82a60e1f42337e7e8517cc0cc4
--- /dev/null
+++ b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/arithmetic.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LIBCXX_TEST_CONCEPTS_LANG_CONCEPTS_ARITHMETIC_H_
+#define LIBCXX_TEST_CONCEPTS_LANG_CONCEPTS_ARITHMETIC_H_
+
+#include <concepts>
+
+// This overload should never be called. It exists solely to force subsumption.
+template <std::integral I>
+[[nodiscard]] constexpr bool CheckSubsumption(I) {
+  return false;
+}
+
+// clang-format off
+template <std::integral I>
+requires std::signed_integral<I> && (!std::unsigned_integral<I>)
+[[nodiscard]] constexpr bool CheckSubsumption(I) {
+  return std::is_signed_v<I>;
+}
+
+template <std::integral I>
+requires std::unsigned_integral<I> && (!std::signed_integral<I>)
+[[nodiscard]] constexpr bool CheckSubsumption(I) {
+  return std::is_unsigned_v<I>;
+}
+// clang-format on
+
+enum ClassicEnum { a, b, c };
+enum class ScopedEnum { x, y, z };
+struct EmptyStruct {};
+
+#endif // LIBCXX_TEST_CONCEPTS_LANG_CONCEPTS_ARITHMETIC_H_
diff --git a/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/floating_point.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/floating_point.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b540ceb4a06bd89739fe5ef66fa239e691779b30
--- /dev/null
+++ b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/floating_point.pass.cpp
@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-no-concepts
+
+// template<class T>
+// concept floating_point = // see below
+
+#include <concepts>
+#include <type_traits>
+
+#include "arithmetic.h"
+
+template <typename T>
+constexpr bool CheckFloatingPointQualifiers() {
+  constexpr bool result = std::floating_point<T>;
+  static_assert(std::floating_point<const T> == result);
+  static_assert(std::floating_point<volatile T> == result);
+  static_assert(std::floating_point<const volatile T> == result);
+
+  static_assert(!std::floating_point<T&>);
+  static_assert(!std::floating_point<const T&>);
+  static_assert(!std::floating_point<volatile T&>);
+  static_assert(!std::floating_point<const volatile T&>);
+
+  static_assert(!std::floating_point<T&&>);
+  static_assert(!std::floating_point<const T&&>);
+  static_assert(!std::floating_point<volatile T&&>);
+  static_assert(!std::floating_point<const volatile T&&>);
+
+  static_assert(!std::floating_point<T*>);
+  static_assert(!std::floating_point<const T*>);
+  static_assert(!std::floating_point<volatile T*>);
+  static_assert(!std::floating_point<const volatile T*>);
+
+  static_assert(!std::floating_point<T (*)()>);
+  static_assert(!std::floating_point<T (&)()>);
+  static_assert(!std::floating_point<T(&&)()>);
+
+  return result;
+}
+
+// floating-point types
+static_assert(CheckFloatingPointQualifiers<float>());
+static_assert(CheckFloatingPointQualifiers<double>());
+static_assert(CheckFloatingPointQualifiers<long double>());
+
+// types that aren't floating-point
+static_assert(!CheckFloatingPointQualifiers<signed char>());
+static_assert(!CheckFloatingPointQualifiers<unsigned char>());
+static_assert(!CheckFloatingPointQualifiers<short>());
+static_assert(!CheckFloatingPointQualifiers<unsigned short>());
+static_assert(!CheckFloatingPointQualifiers<int>());
+static_assert(!CheckFloatingPointQualifiers<unsigned int>());
+static_assert(!CheckFloatingPointQualifiers<long>());
+static_assert(!CheckFloatingPointQualifiers<unsigned long>());
+static_assert(!CheckFloatingPointQualifiers<long long>());
+static_assert(!CheckFloatingPointQualifiers<unsigned long long>());
+static_assert(!CheckFloatingPointQualifiers<wchar_t>());
+static_assert(!CheckFloatingPointQualifiers<bool>());
+static_assert(!CheckFloatingPointQualifiers<char>());
+static_assert(!CheckFloatingPointQualifiers<char8_t>());
+static_assert(!CheckFloatingPointQualifiers<char16_t>());
+static_assert(!CheckFloatingPointQualifiers<char32_t>());
+static_assert(!std::floating_point<void>);
+
+static_assert(!CheckFloatingPointQualifiers<ClassicEnum>());
+static_assert(!CheckFloatingPointQualifiers<ScopedEnum>());
+static_assert(!CheckFloatingPointQualifiers<EmptyStruct>());
+static_assert(!CheckFloatingPointQualifiers<int EmptyStruct::*>());
+static_assert(!CheckFloatingPointQualifiers<int (EmptyStruct::*)()>());
+
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/integral.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/integral.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..42b85a1c020bbc6436a00ee14a8e093099f8650a
--- /dev/null
+++ b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/integral.pass.cpp
@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-no-concepts
+
+// template<class T>
+// concept integral = // see below
+
+#include <concepts>
+#include <type_traits>
+
+#include "arithmetic.h"
+
+template <typename T>
+constexpr bool CheckIntegralQualifiers() {
+  constexpr bool result = std::integral<T>;
+  static_assert(std::integral<const T> == result);
+  static_assert(std::integral<volatile T> == result);
+  static_assert(std::integral<const volatile T> == result);
+
+  static_assert(!std::integral<T&>);
+  static_assert(!std::integral<const T&>);
+  static_assert(!std::integral<volatile T&>);
+  static_assert(!std::integral<const volatile T&>);
+
+  static_assert(!std::integral<T&&>);
+  static_assert(!std::integral<const T&&>);
+  static_assert(!std::integral<volatile T&&>);
+  static_assert(!std::integral<const volatile T&&>);
+
+  static_assert(!std::integral<T*>);
+  static_assert(!std::integral<const T*>);
+  static_assert(!std::integral<volatile T*>);
+  static_assert(!std::integral<const volatile T*>);
+
+  static_assert(!std::integral<T (*)()>);
+  static_assert(!std::integral<T (&)()>);
+  static_assert(!std::integral<T(&&)()>);
+
+  return result;
+}
+
+// standard signed and unsigned integers
+static_assert(CheckIntegralQualifiers<signed char>());
+static_assert(CheckIntegralQualifiers<unsigned char>());
+static_assert(CheckIntegralQualifiers<short>());
+static_assert(CheckIntegralQualifiers<unsigned short>());
+static_assert(CheckIntegralQualifiers<int>());
+static_assert(CheckIntegralQualifiers<unsigned int>());
+static_assert(CheckIntegralQualifiers<long>());
+static_assert(CheckIntegralQualifiers<unsigned long>());
+static_assert(CheckIntegralQualifiers<long long>());
+static_assert(CheckIntegralQualifiers<unsigned long long>());
+
+// extended integers
+#ifndef _LIBCPP_HAS_NO_INT128
+static_assert(CheckIntegralQualifiers<__int128_t>());
+static_assert(CheckIntegralQualifiers<__uint128_t>());
+#endif
+
+// bool and char types are also integral
+static_assert(CheckIntegralQualifiers<wchar_t>());
+static_assert(CheckIntegralQualifiers<bool>());
+static_assert(CheckIntegralQualifiers<char>());
+static_assert(CheckIntegralQualifiers<char8_t>());
+static_assert(CheckIntegralQualifiers<char16_t>());
+static_assert(CheckIntegralQualifiers<char32_t>());
+
+// types that aren't integral
+static_assert(!std::integral<void>);
+static_assert(!CheckIntegralQualifiers<float>());
+static_assert(!CheckIntegralQualifiers<double>());
+static_assert(!CheckIntegralQualifiers<long double>());
+
+static_assert(!CheckIntegralQualifiers<ClassicEnum>());
+
+static_assert(!CheckIntegralQualifiers<ScopedEnum>());
+
+static_assert(!CheckIntegralQualifiers<EmptyStruct>());
+static_assert(!CheckIntegralQualifiers<int EmptyStruct::*>());
+static_assert(!CheckIntegralQualifiers<int (EmptyStruct::*)()>());
+
+static_assert(CheckSubsumption(0));
+static_assert(CheckSubsumption(0U));
+
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/signed_integral.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/signed_integral.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..df97b39709dc99e7e16859f3aaec4cba53cf843f
--- /dev/null
+++ b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/signed_integral.pass.cpp
@@ -0,0 +1,96 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-no-concepts
+
+// template<class T>
+// concept signed_integral = // see below
+
+#include <concepts>
+#include <type_traits>
+
+#include "arithmetic.h"
+
+template <typename T>
+constexpr bool CheckSignedIntegralQualifiers() {
+  constexpr bool result = std::signed_integral<T>;
+  static_assert(std::signed_integral<const T> == result);
+  static_assert(std::signed_integral<volatile T> == result);
+  static_assert(std::signed_integral<const volatile T> == result);
+
+  static_assert(!std::signed_integral<T&>);
+  static_assert(!std::signed_integral<const T&>);
+  static_assert(!std::signed_integral<volatile T&>);
+  static_assert(!std::signed_integral<const volatile T&>);
+
+  static_assert(!std::signed_integral<T&&>);
+  static_assert(!std::signed_integral<const T&&>);
+  static_assert(!std::signed_integral<volatile T&&>);
+  static_assert(!std::signed_integral<const volatile T&&>);
+
+  static_assert(!std::signed_integral<T*>);
+  static_assert(!std::signed_integral<const T*>);
+  static_assert(!std::signed_integral<volatile T*>);
+  static_assert(!std::signed_integral<const volatile T*>);
+
+  static_assert(!std::signed_integral<T (*)()>);
+  static_assert(!std::signed_integral<T (&)()>);
+  static_assert(!std::signed_integral<T(&&)()>);
+
+  return result;
+}
+
+// standard signed integers
+static_assert(CheckSignedIntegralQualifiers<signed char>());
+static_assert(CheckSignedIntegralQualifiers<short>());
+static_assert(CheckSignedIntegralQualifiers<int>());
+static_assert(CheckSignedIntegralQualifiers<long>());
+static_assert(CheckSignedIntegralQualifiers<long long>());
+
+// bool and character *may* be signed
+static_assert(CheckSignedIntegralQualifiers<wchar_t>() ==
+              std::is_signed_v<wchar_t>);
+static_assert(CheckSignedIntegralQualifiers<bool>() == std::is_signed_v<bool>);
+static_assert(CheckSignedIntegralQualifiers<char>() == std::is_signed_v<char>);
+static_assert(CheckSignedIntegralQualifiers<char8_t>() ==
+              std::is_signed_v<char8_t>);
+static_assert(CheckSignedIntegralQualifiers<char16_t>() ==
+              std::is_signed_v<char16_t>);
+static_assert(CheckSignedIntegralQualifiers<char32_t>() ==
+              std::is_signed_v<char32_t>);
+
+// integers that aren't signed integrals
+static_assert(!CheckSignedIntegralQualifiers<unsigned char>());
+static_assert(!CheckSignedIntegralQualifiers<unsigned short>());
+static_assert(!CheckSignedIntegralQualifiers<unsigned int>());
+static_assert(!CheckSignedIntegralQualifiers<unsigned long>());
+static_assert(!CheckSignedIntegralQualifiers<unsigned long long>());
+
+// extended integers
+#ifndef _LIBCPP_HAS_NO_INT128
+static_assert(CheckSignedIntegralQualifiers<__int128_t>());
+static_assert(!CheckSignedIntegralQualifiers<__uint128_t>());
+#endif
+
+// types that aren't even integers shouldn't be signed integers!
+static_assert(!std::signed_integral<void>);
+static_assert(!CheckSignedIntegralQualifiers<float>());
+static_assert(!CheckSignedIntegralQualifiers<double>());
+static_assert(!CheckSignedIntegralQualifiers<long double>());
+
+static_assert(!CheckSignedIntegralQualifiers<ClassicEnum>());
+static_assert(!CheckSignedIntegralQualifiers<ScopedEnum>());
+static_assert(!CheckSignedIntegralQualifiers<EmptyStruct>());
+static_assert(!CheckSignedIntegralQualifiers<int EmptyStruct::*>());
+static_assert(!CheckSignedIntegralQualifiers<int (EmptyStruct::*)()>());
+
+static_assert(CheckSubsumption(0));
+static_assert(CheckSubsumption(0U));
+
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/unsigned_integral.pass.cpp b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/unsigned_integral.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..02a62865f2ddf6c5980ac00c49c70042eca6db42
--- /dev/null
+++ b/libcxx/test/std/concepts/concepts.lang/concepts.arithmetic/unsigned_integral.pass.cpp
@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-no-concepts
+
+// template<class T>
+// concept unsigned_integral = // see below
+
+#include <concepts>
+#include <type_traits>
+
+#include "arithmetic.h"
+
+template <typename T>
+constexpr bool CheckUnsignedIntegralQualifiers() {
+  constexpr bool result = std::unsigned_integral<T>;
+  static_assert(std::unsigned_integral<const T> == result);
+  static_assert(std::unsigned_integral<volatile T> == result);
+  static_assert(std::unsigned_integral<const volatile T> == result);
+
+  static_assert(!std::unsigned_integral<T&>);
+  static_assert(!std::unsigned_integral<const T&>);
+  static_assert(!std::unsigned_integral<volatile T&>);
+  static_assert(!std::unsigned_integral<const volatile T&>);
+
+  static_assert(!std::unsigned_integral<T&&>);
+  static_assert(!std::unsigned_integral<const T&&>);
+  static_assert(!std::unsigned_integral<volatile T&&>);
+  static_assert(!std::unsigned_integral<const volatile T&&>);
+
+  static_assert(!std::unsigned_integral<T*>);
+  static_assert(!std::unsigned_integral<const T*>);
+  static_assert(!std::unsigned_integral<volatile T*>);
+  static_assert(!std::unsigned_integral<const volatile T*>);
+
+  static_assert(!std::unsigned_integral<T (*)()>);
+  static_assert(!std::unsigned_integral<T (&)()>);
+  static_assert(!std::unsigned_integral<T(&&)()>);
+
+  return result;
+}
+
+// standard unsigned types
+static_assert(CheckUnsignedIntegralQualifiers<unsigned char>());
+static_assert(CheckUnsignedIntegralQualifiers<unsigned short>());
+static_assert(CheckUnsignedIntegralQualifiers<unsigned int>());
+static_assert(CheckUnsignedIntegralQualifiers<unsigned long>());
+static_assert(CheckUnsignedIntegralQualifiers<unsigned long long>());
+
+// Whether bool and character types are signed or unsigned is impl-defined
+static_assert(CheckUnsignedIntegralQualifiers<wchar_t>() ==
+              !std::is_signed_v<wchar_t>);
+static_assert(CheckUnsignedIntegralQualifiers<bool>() ==
+              !std::is_signed_v<bool>);
+static_assert(CheckUnsignedIntegralQualifiers<char>() ==
+              !std::is_signed_v<char>);
+static_assert(CheckUnsignedIntegralQualifiers<char8_t>() ==
+              !std::is_signed_v<char8_t>);
+static_assert(CheckUnsignedIntegralQualifiers<char16_t>() ==
+              !std::is_signed_v<char16_t>);
+static_assert(CheckUnsignedIntegralQualifiers<char32_t>() ==
+              !std::is_signed_v<char32_t>);
+
+// extended integers
+#ifndef _LIBCPP_HAS_NO_INT128
+static_assert(CheckUnsignedIntegralQualifiers<__uint128_t>());
+static_assert(!CheckUnsignedIntegralQualifiers<__int128_t>());
+#endif
+
+// integer types that aren't unsigned integrals
+static_assert(!CheckUnsignedIntegralQualifiers<signed char>());
+static_assert(!CheckUnsignedIntegralQualifiers<short>());
+static_assert(!CheckUnsignedIntegralQualifiers<int>());
+static_assert(!CheckUnsignedIntegralQualifiers<long>());
+static_assert(!CheckUnsignedIntegralQualifiers<long long>());
+
+static_assert(!std::unsigned_integral<void>);
+static_assert(!CheckUnsignedIntegralQualifiers<float>());
+static_assert(!CheckUnsignedIntegralQualifiers<double>());
+static_assert(!CheckUnsignedIntegralQualifiers<long double>());
+
+static_assert(!CheckUnsignedIntegralQualifiers<ClassicEnum>());
+static_assert(!CheckUnsignedIntegralQualifiers<ScopedEnum>());
+static_assert(!CheckUnsignedIntegralQualifiers<EmptyStruct>());
+static_assert(!CheckUnsignedIntegralQualifiers<int EmptyStruct::*>());
+static_assert(!CheckUnsignedIntegralQualifiers<int (EmptyStruct::*)()>());
+
+static_assert(CheckSubsumption(0));
+static_assert(CheckSubsumption(0U));
+
+int main(int, char**) { return 0; }
diff --git a/libcxx/test/std/concepts/object/copyable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.object/copyable.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/object/copyable.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.object/copyable.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/object/movable.compile.pass.cpp b/libcxx/test/std/concepts/concepts.object/movable.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/object/movable.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.object/movable.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/object/regular.compile.pass.cpp b/libcxx/test/std/concepts/concepts.object/regular.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/object/regular.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.object/regular.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/object/semiregular.compile.pass.cpp b/libcxx/test/std/concepts/concepts.object/semiregular.compile.pass.cpp
similarity index 100%
rename from libcxx/test/std/concepts/object/semiregular.compile.pass.cpp
rename to libcxx/test/std/concepts/concepts.object/semiregular.compile.pass.cpp
diff --git a/libcxx/test/std/concepts/lang/arithmetic.pass.cpp b/libcxx/test/std/concepts/lang/arithmetic.pass.cpp
deleted file mode 100644
index 9b7b75b27cc1e08cee4b26b4b61ae0e4c17d3f44..0000000000000000000000000000000000000000
--- a/libcxx/test/std/concepts/lang/arithmetic.pass.cpp
+++ /dev/null
@@ -1,346 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-no-concepts
-
-// template<class T>
-// concept integral = // see below
-
-// template<class T>
-// concept signed_integral = // see below
-
-// template<class T>
-// concept unsigned_integral = // see below
-
-// template<class T>
-// concept floating_point = // see below
-
-#include <concepts>
-#include <type_traits>
-
-namespace {
-template <typename T>
-constexpr bool CheckIntegralQualifiers() {
-  constexpr bool result = std::integral<T>;
-  static_assert(std::integral<const T> == result);
-  static_assert(std::integral<volatile T> == result);
-  static_assert(std::integral<const volatile T> == result);
-
-  static_assert(!std::integral<T&>);
-  static_assert(!std::integral<const T&>);
-  static_assert(!std::integral<volatile T&>);
-  static_assert(!std::integral<const volatile T&>);
-
-  static_assert(!std::integral<T&&>);
-  static_assert(!std::integral<const T&&>);
-  static_assert(!std::integral<volatile T&&>);
-  static_assert(!std::integral<const volatile T&&>);
-
-  static_assert(!std::integral<T*>);
-  static_assert(!std::integral<const T*>);
-  static_assert(!std::integral<volatile T*>);
-  static_assert(!std::integral<const volatile T*>);
-
-  static_assert(!std::integral<T (*)()>);
-  static_assert(!std::integral<T (&)()>);
-  static_assert(!std::integral<T(&&)()>);
-
-  return result;
-}
-
-enum ClassicEnum { a, b, c };
-enum class ScopedEnum { x, y, z };
-struct EmptyStruct {};
-
-constexpr void CheckIntegral() {
-  // standard signed and unsigned integers
-  static_assert(CheckIntegralQualifiers<signed char>());
-  static_assert(CheckIntegralQualifiers<unsigned char>());
-  static_assert(CheckIntegralQualifiers<short>());
-  static_assert(CheckIntegralQualifiers<unsigned short>());
-  static_assert(CheckIntegralQualifiers<int>());
-  static_assert(CheckIntegralQualifiers<unsigned int>());
-  static_assert(CheckIntegralQualifiers<long>());
-  static_assert(CheckIntegralQualifiers<unsigned long>());
-  static_assert(CheckIntegralQualifiers<long long>());
-  static_assert(CheckIntegralQualifiers<unsigned long long>());
-
-  // extended integers
-#ifndef _LIBCPP_HAS_NO_INT128
-  static_assert(CheckIntegralQualifiers<__int128_t>());
-  static_assert(CheckIntegralQualifiers<__uint128_t>());
-#endif
-
-  // bool and char types are also integral
-  static_assert(CheckIntegralQualifiers<wchar_t>());
-  static_assert(CheckIntegralQualifiers<bool>());
-  static_assert(CheckIntegralQualifiers<char>());
-  static_assert(CheckIntegralQualifiers<char8_t>());
-  static_assert(CheckIntegralQualifiers<char16_t>());
-  static_assert(CheckIntegralQualifiers<char32_t>());
-
-  // types that aren't integral
-  static_assert(!std::integral<void>);
-  static_assert(!CheckIntegralQualifiers<float>());
-  static_assert(!CheckIntegralQualifiers<double>());
-  static_assert(!CheckIntegralQualifiers<long double>());
-
-  static_assert(!CheckIntegralQualifiers<ClassicEnum>());
-
-  static_assert(!CheckIntegralQualifiers<ScopedEnum>());
-
-  static_assert(!CheckIntegralQualifiers<EmptyStruct>());
-  static_assert(!CheckIntegralQualifiers<int EmptyStruct::*>());
-  static_assert(!CheckIntegralQualifiers<int (EmptyStruct::*)()>());
-}
-
-template <typename T>
-constexpr bool CheckSignedIntegralQualifiers() {
-  constexpr bool result = std::signed_integral<T>;
-  static_assert(std::signed_integral<const T> == result);
-  static_assert(std::signed_integral<volatile T> == result);
-  static_assert(std::signed_integral<const volatile T> == result);
-
-  static_assert(!std::signed_integral<T&>);
-  static_assert(!std::signed_integral<const T&>);
-  static_assert(!std::signed_integral<volatile T&>);
-  static_assert(!std::signed_integral<const volatile T&>);
-
-  static_assert(!std::signed_integral<T&&>);
-  static_assert(!std::signed_integral<const T&&>);
-  static_assert(!std::signed_integral<volatile T&&>);
-  static_assert(!std::signed_integral<const volatile T&&>);
-
-  static_assert(!std::signed_integral<T*>);
-  static_assert(!std::signed_integral<const T*>);
-  static_assert(!std::signed_integral<volatile T*>);
-  static_assert(!std::signed_integral<const volatile T*>);
-
-  static_assert(!std::signed_integral<T (*)()>);
-  static_assert(!std::signed_integral<T (&)()>);
-  static_assert(!std::signed_integral<T(&&)()>);
-
-  return result;
-}
-
-constexpr void CheckSignedIntegral() {
-  // standard signed integers
-  static_assert(CheckSignedIntegralQualifiers<signed char>());
-  static_assert(CheckSignedIntegralQualifiers<short>());
-  static_assert(CheckSignedIntegralQualifiers<int>());
-  static_assert(CheckSignedIntegralQualifiers<long>());
-  static_assert(CheckSignedIntegralQualifiers<long long>());
-
-  // bool and character *may* be signed
-  static_assert(CheckSignedIntegralQualifiers<wchar_t>() ==
-                std::is_signed_v<wchar_t>);
-  static_assert(CheckSignedIntegralQualifiers<bool>() ==
-                std::is_signed_v<bool>);
-  static_assert(CheckSignedIntegralQualifiers<char>() ==
-                std::is_signed_v<char>);
-  static_assert(CheckSignedIntegralQualifiers<char8_t>() ==
-                std::is_signed_v<char8_t>);
-  static_assert(CheckSignedIntegralQualifiers<char16_t>() ==
-                std::is_signed_v<char16_t>);
-  static_assert(CheckSignedIntegralQualifiers<char32_t>() ==
-                std::is_signed_v<char32_t>);
-
-  // integers that aren't signed integrals
-  static_assert(!CheckSignedIntegralQualifiers<unsigned char>());
-  static_assert(!CheckSignedIntegralQualifiers<unsigned short>());
-  static_assert(!CheckSignedIntegralQualifiers<unsigned int>());
-  static_assert(!CheckSignedIntegralQualifiers<unsigned long>());
-  static_assert(!CheckSignedIntegralQualifiers<unsigned long long>());
-
-  // extended integers
-#ifndef _LIBCPP_HAS_NO_INT128
-  static_assert(CheckSignedIntegralQualifiers<__int128_t>());
-  static_assert(!CheckSignedIntegralQualifiers<__uint128_t>());
-#endif
-
-  // types that aren't even integers shouldn't be signed integers!
-  static_assert(!std::signed_integral<void>);
-  static_assert(!CheckSignedIntegralQualifiers<float>());
-  static_assert(!CheckSignedIntegralQualifiers<double>());
-  static_assert(!CheckSignedIntegralQualifiers<long double>());
-
-  static_assert(!CheckSignedIntegralQualifiers<ClassicEnum>());
-  static_assert(!CheckSignedIntegralQualifiers<ScopedEnum>());
-  static_assert(!CheckSignedIntegralQualifiers<EmptyStruct>());
-  static_assert(!CheckSignedIntegralQualifiers<int EmptyStruct::*>());
-  static_assert(!CheckSignedIntegralQualifiers<int (EmptyStruct::*)()>());
-}
-
-template <typename T>
-constexpr bool CheckUnsignedIntegralQualifiers() {
-  constexpr bool result = std::unsigned_integral<T>;
-  static_assert(std::unsigned_integral<const T> == result);
-  static_assert(std::unsigned_integral<volatile T> == result);
-  static_assert(std::unsigned_integral<const volatile T> == result);
-
-  static_assert(!std::unsigned_integral<T&>);
-  static_assert(!std::unsigned_integral<const T&>);
-  static_assert(!std::unsigned_integral<volatile T&>);
-  static_assert(!std::unsigned_integral<const volatile T&>);
-
-  static_assert(!std::unsigned_integral<T&&>);
-  static_assert(!std::unsigned_integral<const T&&>);
-  static_assert(!std::unsigned_integral<volatile T&&>);
-  static_assert(!std::unsigned_integral<const volatile T&&>);
-
-  static_assert(!std::unsigned_integral<T*>);
-  static_assert(!std::unsigned_integral<const T*>);
-  static_assert(!std::unsigned_integral<volatile T*>);
-  static_assert(!std::unsigned_integral<const volatile T*>);
-
-  static_assert(!std::unsigned_integral<T (*)()>);
-  static_assert(!std::unsigned_integral<T (&)()>);
-  static_assert(!std::unsigned_integral<T(&&)()>);
-
-  return result;
-}
-
-constexpr void CheckUnsignedIntegral() {
-  // standard unsigned types
-  static_assert(CheckUnsignedIntegralQualifiers<unsigned char>());
-  static_assert(CheckUnsignedIntegralQualifiers<unsigned short>());
-  static_assert(CheckUnsignedIntegralQualifiers<unsigned int>());
-  static_assert(CheckUnsignedIntegralQualifiers<unsigned long>());
-  static_assert(CheckUnsignedIntegralQualifiers<unsigned long long>());
-
-  // Whether bool and character types are signed or unsigned is impl-defined
-  static_assert(CheckUnsignedIntegralQualifiers<wchar_t>() ==
-                !std::is_signed_v<wchar_t>);
-  static_assert(CheckUnsignedIntegralQualifiers<bool>() ==
-                !std::is_signed_v<bool>);
-  static_assert(CheckUnsignedIntegralQualifiers<char>() ==
-                !std::is_signed_v<char>);
-  static_assert(CheckUnsignedIntegralQualifiers<char8_t>() ==
-                !std::is_signed_v<char8_t>);
-  static_assert(CheckUnsignedIntegralQualifiers<char16_t>() ==
-                !std::is_signed_v<char16_t>);
-  static_assert(CheckUnsignedIntegralQualifiers<char32_t>() ==
-                !std::is_signed_v<char32_t>);
-
-  // extended integers
-#ifndef _LIBCPP_HAS_NO_INT128
-  static_assert(CheckUnsignedIntegralQualifiers<__uint128_t>());
-  static_assert(!CheckUnsignedIntegralQualifiers<__int128_t>());
-#endif
-
-  // integer types that aren't unsigned integrals
-  static_assert(!CheckUnsignedIntegralQualifiers<signed char>());
-  static_assert(!CheckUnsignedIntegralQualifiers<short>());
-  static_assert(!CheckUnsignedIntegralQualifiers<int>());
-  static_assert(!CheckUnsignedIntegralQualifiers<long>());
-  static_assert(!CheckUnsignedIntegralQualifiers<long long>());
-
-  static_assert(!std::unsigned_integral<void>);
-  static_assert(!CheckUnsignedIntegralQualifiers<float>());
-  static_assert(!CheckUnsignedIntegralQualifiers<double>());
-  static_assert(!CheckUnsignedIntegralQualifiers<long double>());
-
-  static_assert(!CheckUnsignedIntegralQualifiers<ClassicEnum>());
-  static_assert(!CheckUnsignedIntegralQualifiers<ScopedEnum>());
-  static_assert(!CheckUnsignedIntegralQualifiers<EmptyStruct>());
-  static_assert(!CheckUnsignedIntegralQualifiers<int EmptyStruct::*>());
-  static_assert(!CheckUnsignedIntegralQualifiers<int (EmptyStruct::*)()>());
-}
-
-// This overload should never be called. It exists solely to force subsumption.
-template <std::integral I>
-[[nodiscard]] constexpr bool CheckSubsumption(I) {
-  return false;
-}
-
-// clang-format off
-template <std::integral I>
-requires std::signed_integral<I> && (!std::unsigned_integral<I>)
-[[nodiscard]] constexpr bool CheckSubsumption(I) {
-  return std::is_signed_v<I>;
-}
-
-template <std::integral I>
-requires std::unsigned_integral<I> && (!std::signed_integral<I>)
-[[nodiscard]] constexpr bool CheckSubsumption(I) {
-  return std::is_unsigned_v<I>;
-}
-// clang-format on
-
-template <typename T>
-constexpr bool CheckFloatingPointQualifiers() {
-  constexpr bool result = std::floating_point<T>;
-  static_assert(std::floating_point<const T> == result);
-  static_assert(std::floating_point<volatile T> == result);
-  static_assert(std::floating_point<const volatile T> == result);
-
-  static_assert(!std::floating_point<T&>);
-  static_assert(!std::floating_point<const T&>);
-  static_assert(!std::floating_point<volatile T&>);
-  static_assert(!std::floating_point<const volatile T&>);
-
-  static_assert(!std::floating_point<T&&>);
-  static_assert(!std::floating_point<const T&&>);
-  static_assert(!std::floating_point<volatile T&&>);
-  static_assert(!std::floating_point<const volatile T&&>);
-
-  static_assert(!std::floating_point<T*>);
-  static_assert(!std::floating_point<const T*>);
-  static_assert(!std::floating_point<volatile T*>);
-  static_assert(!std::floating_point<const volatile T*>);
-
-  static_assert(!std::floating_point<T (*)()>);
-  static_assert(!std::floating_point<T (&)()>);
-  static_assert(!std::floating_point<T(&&)()>);
-
-  return result;
-}
-
-constexpr void CheckFloatingPoint() {
-  // floating-point types
-  static_assert(CheckFloatingPointQualifiers<float>());
-  static_assert(CheckFloatingPointQualifiers<double>());
-  static_assert(CheckFloatingPointQualifiers<long double>());
-
-  // types that aren't floating-point
-  static_assert(!CheckFloatingPointQualifiers<signed char>());
-  static_assert(!CheckFloatingPointQualifiers<unsigned char>());
-  static_assert(!CheckFloatingPointQualifiers<short>());
-  static_assert(!CheckFloatingPointQualifiers<unsigned short>());
-  static_assert(!CheckFloatingPointQualifiers<int>());
-  static_assert(!CheckFloatingPointQualifiers<unsigned int>());
-  static_assert(!CheckFloatingPointQualifiers<long>());
-  static_assert(!CheckFloatingPointQualifiers<unsigned long>());
-  static_assert(!CheckFloatingPointQualifiers<long long>());
-  static_assert(!CheckFloatingPointQualifiers<unsigned long long>());
-  static_assert(!CheckFloatingPointQualifiers<wchar_t>());
-  static_assert(!CheckFloatingPointQualifiers<bool>());
-  static_assert(!CheckFloatingPointQualifiers<char>());
-  static_assert(!CheckFloatingPointQualifiers<char8_t>());
-  static_assert(!CheckFloatingPointQualifiers<char16_t>());
-  static_assert(!CheckFloatingPointQualifiers<char32_t>());
-  static_assert(!std::floating_point<void>);
-
-  static_assert(!CheckFloatingPointQualifiers<ClassicEnum>());
-  static_assert(!CheckFloatingPointQualifiers<ScopedEnum>());
-  static_assert(!CheckFloatingPointQualifiers<EmptyStruct>());
-  static_assert(!CheckFloatingPointQualifiers<int EmptyStruct::*>());
-  static_assert(!CheckFloatingPointQualifiers<int (EmptyStruct::*)()>());
-}
-} // namespace
-
-int main(int, char**) {
-  CheckIntegral();
-  CheckSignedIntegral();
-  CheckUnsignedIntegral();
-  static_assert(CheckSubsumption(0));
-  static_assert(CheckSubsumption(0U));
-  CheckFloatingPoint();
-  return 0;
-}
diff --git a/libcxx/test/std/depr/depr.c.headers/stdlib_h.pass.cpp b/libcxx/test/std/depr/depr.c.headers/stdlib_h.pass.cpp
index 295f51807d41f29aec3253247a8594b486457beb..8dd15c9e27c3cb28c231da456e883e7f9586926c 100644
--- a/libcxx/test/std/depr/depr.c.headers/stdlib_h.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/stdlib_h.pass.cpp
@@ -8,6 +8,8 @@
 
 // test <stdlib.h>
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <stdlib.h>
 #include <type_traits>
 #include <cassert>
diff --git a/libcxx/test/std/experimental/memory/memory.resource.global/new_delete_resource.pass.cpp b/libcxx/test/std/experimental/memory/memory.resource.global/new_delete_resource.pass.cpp
index 8c7af9dff376ba02f6a3f3856a915b81427848f1..91bf1f5efd918e348ba22638ede22d654bc8507b 100644
--- a/libcxx/test/std/experimental/memory/memory.resource.global/new_delete_resource.pass.cpp
+++ b/libcxx/test/std/experimental/memory/memory.resource.global/new_delete_resource.pass.cpp
@@ -25,14 +25,14 @@ namespace ex = std::experimental::pmr;
 struct assert_on_compare : public ex::memory_resource
 {
 protected:
-    virtual void * do_allocate(size_t, size_t)
-    { assert(false); }
+    void * do_allocate(size_t, size_t) override
+    { assert(false); return nullptr; }
 
-    virtual void do_deallocate(void *, size_t, size_t)
+    void do_deallocate(void *, size_t, size_t) override
     { assert(false); }
 
-    virtual bool do_is_equal(ex::memory_resource const &) const noexcept
-    { assert(false); }
+    bool do_is_equal(ex::memory_resource const &) const noexcept override
+    { assert(false); return false; }
 };
 
 void test_return()
diff --git a/libcxx/test/std/experimental/memory/memory.resource.global/null_memory_resource.pass.cpp b/libcxx/test/std/experimental/memory/memory.resource.global/null_memory_resource.pass.cpp
index c7efeab186b0b08b55868561fb785a3a4f0f3622..bdd62484d0759b6ab87a18418e428f0561fdbec3 100644
--- a/libcxx/test/std/experimental/memory/memory.resource.global/null_memory_resource.pass.cpp
+++ b/libcxx/test/std/experimental/memory/memory.resource.global/null_memory_resource.pass.cpp
@@ -25,14 +25,14 @@ namespace ex = std::experimental::pmr;
 struct assert_on_compare : public ex::memory_resource
 {
 protected:
-    virtual void * do_allocate(size_t, size_t)
-    { assert(false); }
+    void * do_allocate(size_t, size_t) override
+    { assert(false); return nullptr; }
 
-    virtual void do_deallocate(void *, size_t, size_t)
+    void do_deallocate(void *, size_t, size_t) override
     { assert(false); }
 
-    virtual bool do_is_equal(ex::memory_resource const &) const noexcept
-    { assert(false); }
+    bool do_is_equal(ex::memory_resource const &) const noexcept override
+    { assert(false); return false; }
 };
 
 void test_return()
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/overflow.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/overflow.pass.cpp
index 72d85323893a24a3bab7d59567ed65c340451f01..69ab2788d77c2cb10c942cbee621ca5367272cdf 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/overflow.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/overflow.pass.cpp
@@ -8,6 +8,8 @@
 
 // REQUIRES: locale.en_US.UTF-8
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <fstream>
 
 // int_type overflow(int_type c = traits::eof());
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/underflow.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/underflow.pass.cpp
index 0d1f7a384a9e43f25def32eee2141c6c1708ba8d..1954ffa5aa290e5d7da952ca8497a5936cf4f6b4 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/underflow.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/underflow.pass.cpp
@@ -9,6 +9,8 @@
 // REQUIRES: locale.en_US.UTF-8
 // FILE_DEPENDENCIES: underflow.dat, underflow_utf8.dat
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <fstream>
 
 // int_type underflow();
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.cons/path.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.cons/path.pass.cpp
index a118fae4d54b14b481bb38a3fb90ffc92b706562..f91d87e9b6d4ab1e6119bdb836d0f438eb6ceb69 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.cons/path.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.cons/path.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/assign.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/assign.pass.cpp
index e35a77cc7ba0a9c382c1091027cb0fd413eabdf0..13eb082285826a02446ab23fba4f19365375ed12 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/assign.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/assign.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/refresh.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/refresh.pass.cpp
index 282d979fe02c532f9f8ca19eda32826292c1a903..89e30b500b3e51d6242badae9b0ab6458382e122 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/refresh.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/refresh.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/replace_filename.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/replace_filename.pass.cpp
index b545f465ce0e61532edf9396b8b9252c7ebdfb6f..13fe77787ccd3bd4076f060baa3ffc2981939317 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/replace_filename.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/replace_filename.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp
index 1d0e6f1b8bab6b268aa524891e78f37df5bd7778..740089c8e14966e734b10b044fcfe728d959ff19 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_type_obs.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_type_obs.pass.cpp
index 52c6afa39494f8627e343e28c4f4e0e3ee93eb7e..fbf1bec73e1113cddfeafa81b4d3a68e485c86f9 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_type_obs.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_type_obs.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp
index 078dd35a00d4ceaeaa86dc0209672e5fdcc85559..b1a6c8edcba201be585893703eeca2764cdb7511 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/last_write_time.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/last_write_time.pass.cpp
index 10050fb4f211407db437bfe3fc82a7318a660793..04dc6ab11e3c043ea6903ce3034d9faa4ee9ce1b 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/last_write_time.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/last_write_time.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/ctor.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/ctor.pass.cpp
index fae696613c75eb556651983cf119fe199332e6cc..306b65267aca08f5b301255b1c90381e6a25095b 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/ctor.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/ctor.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_iterator
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.append.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.append.pass.cpp
index b1290e301f6b777bae9d9b1cc6e885bfb860e1c9..201cb99e6481a02a3151ec459172f0332436c6ee 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.append.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.append.pass.cpp
@@ -11,6 +11,8 @@
 // These tests require locale for non-char paths
 // UNSUPPORTED: libcpp-has-no-localization
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class path
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/move.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/move.pass.cpp
index 82f5248e9f31af6301cefbbff664ab4ae20d6cca..144966c690a4ac2a9ccaad2d03ad5daacec5feec 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/move.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/move.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class path
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.concat.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.concat.pass.cpp
index 5446610af383b08cea5a69496aa66074dcd45eed..c09d60d28e2c23240dca72e8ee58b28430aa824b 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.concat.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.concat.pass.cpp
@@ -11,6 +11,8 @@
 // These tests require locale for non-char paths
 // UNSUPPORTED: libcpp-has-no-localization
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class path
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.construct/move.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.construct/move.pass.cpp
index a0418d8661a4bc1389f2b2283f5d3c254f633f27..bf63f702305edabad504c37a4518d720386b2f0b 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.construct/move.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.construct/move.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class path
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/ctor.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/ctor.pass.cpp
index 53ca5737ba31f98019cc5c39fe9f72794156a9ae..4b28130a85d07060041baa7304c47ab403c36fbd 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/ctor.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/ctor.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class directory_iterator
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp
index 795c3ce0b3029bb444509903d4e0c042098b4388..f2fde11ddb16066ca3d891f947cc9ba0d8ec153a 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // class recursive_directory_iterator
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp
index dfccfb28ea118b1cded0e8c7f1efae65f1d81be3..6a70685612308fad56b9ec62acb266df11f8514e 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool copy_file(const path& from, const path& to);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory_with_attributes.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory_with_attributes.pass.cpp
index cd436d9c6fb145ed41abb83b0ab823ed3646ba22..353ecf873f46faad7e55ee7880c72daae348d78a 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory_with_attributes.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory_with_attributes.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // This test requires the dylib support introduced in D92769.
 // XFAIL: with_system_cxx_lib=macosx10.15
 
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.current_path/current_path.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.current_path/current_path.pass.cpp
index 4b8d9d4ec1f8fe26c4c403c76e96de9bf9aaf5e6..5e90c4452a9db7e8e1013447ee77cfa3be389933 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.current_path/current_path.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.current_path/current_path.pass.cpp
@@ -51,8 +51,8 @@ TEST_CASE(current_path_test)
 
 TEST_CASE(current_path_after_change_test)
 {
-    CWDGuard guard;
     static_test_env static_env;
+    CWDGuard guard;
     const path new_path = static_env.Dir;
     current_path(new_path);
     TEST_CHECK(current_path() == new_path);
@@ -60,8 +60,8 @@ TEST_CASE(current_path_after_change_test)
 
 TEST_CASE(current_path_is_file_test)
 {
-    CWDGuard guard;
     static_test_env static_env;
+    CWDGuard guard;
     const path p = static_env.File;
     std::error_code ec;
     const path old_p = current_path();
@@ -72,8 +72,8 @@ TEST_CASE(current_path_is_file_test)
 
 TEST_CASE(set_to_non_absolute_path)
 {
-    CWDGuard guard;
     static_test_env static_env;
+    CWDGuard guard;
     const path base = static_env.Dir;
     current_path(base);
     const path p = static_env.Dir2.filename();
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.exists/exists.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.exists/exists.pass.cpp
index a116d0886dd4964b34e7dc2008646e8a1e5f35cb..fd3a63b0eaeaf1b8bff73ca699a83020e5c5dd12 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.exists/exists.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.exists/exists.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool exists(file_status s) noexcept
@@ -66,6 +68,17 @@ TEST_CASE(test_exist_not_found)
     const path p = static_env.DNE;
     TEST_CHECK(exists(p) == false);
 
+    TEST_CHECK(exists(static_env.Dir) == true);
+    TEST_CHECK(exists(static_env.Dir / "dne") == false);
+    // Whether <dir>/dne/.. is considered to exist or not is not necessarily
+    // something we need to define, but the platform specific behaviour
+    // does affect a few other tests, so clarify the root cause here.
+#ifdef _WIN32
+    TEST_CHECK(exists(static_env.Dir / "dne" / "..") == true);
+#else
+    TEST_CHECK(exists(static_env.Dir / "dne" / "..") == false);
+#endif
+
     std::error_code ec = GetTestEC();
     TEST_CHECK(exists(p, ec) == false);
     TEST_CHECK(!ec);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_block_file/is_block_file.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_block_file/is_block_file.pass.cpp
index 157e4f33acb0c1132d83ac09608014823acf19f9..3a5f9458b7d50a402b041063587c95959f5114ea 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_block_file/is_block_file.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_block_file/is_block_file.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool is_block_file(file_status s) noexcept
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_char_file/is_character_file.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_char_file/is_character_file.pass.cpp
index fcef134c9bba1cee04b1e621dda211876e678e5e..99ccf2b886ecd87da5e0face2658a9b408add377 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_char_file/is_character_file.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_char_file/is_character_file.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool is_character_file(file_status s) noexcept
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_directory/is_directory.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_directory/is_directory.pass.cpp
index 1508ee4dc6bafc622d708cc8702d29af487d9c31..e24f193385041fc51527e35a9fd9d944de0e1c69 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_directory/is_directory.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_directory/is_directory.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool is_directory(file_status s) noexcept
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp
index 8cbdf13b7b73b79cd12568d95603bc194f7c1e36..6b051d5d3ae46a96c11085736b7d98ac60d9c166 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool is_empty(path const& p);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_fifo/is_fifo.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_fifo/is_fifo.pass.cpp
index 585a9e201c70804477903525ccc5bb7e78369234..c7d11e8017b51b0a3094d1da238c66cc3e82efcc 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_fifo/is_fifo.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_fifo/is_fifo.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool is_fifo(file_status s) noexcept
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_other/is_other.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_other/is_other.pass.cpp
index 7475c86e67f29e057b34b4728f66f7019d72137b..a3e812d82399c27e5de29f7e83a0217a037fe034 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_other/is_other.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_other/is_other.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool is_other(file_status s) noexcept
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_regular_file/is_regular_file.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_regular_file/is_regular_file.pass.cpp
index bcff7f77c37ba32371fccab07672cec9d4cda04d..0635c98abb08edf93cdf78571380c7ef5aa7fa76 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_regular_file/is_regular_file.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_regular_file/is_regular_file.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool is_regular_file(file_status s) noexcept
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_socket/is_socket.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_socket/is_socket.pass.cpp
index 589c0e66151e6b8e9a2dc44d1748095e26a9ca50..b45e8f07086f4c253f1dc74d024489c7de33fc4e 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_socket/is_socket.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_socket/is_socket.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool is_socket(file_status s) noexcept
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_symlink/is_symlink.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_symlink/is_symlink.pass.cpp
index a4fa057fa1dc41fbeecff1c78729761cded1a688..a0f0468d2e23b9cd94a56d4fd3803886bcddf46d 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_symlink/is_symlink.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_symlink/is_symlink.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool is_symlink(file_status s) noexcept
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.last_write_time/last_write_time.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.last_write_time/last_write_time.pass.cpp
index c3b6c9af9598ffbc765829f429a94fdff6b0112b..fd7a5f2bc7cc8798307b014889decbf49e073973 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.last_write_time/last_write_time.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.last_write_time/last_write_time.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // file_time_type last_write_time(const path& p);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp
index d7a8c8a82a82c86a5b9177ffcbaec58c8d1b83d4..aa2e0920bc48ee7c379436e224a3915693f68dfc 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // void permissions(const path& p, perms prms,
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.proximate/proximate.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.proximate/proximate.pass.cpp
index 944092fa4ff0fe78f2843c7580d2de291e0b57fa..c87e3966c7b9e40b249e81401aaba990697c5e8b 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.proximate/proximate.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.proximate/proximate.pass.cpp
@@ -27,7 +27,7 @@
 static int count_path_elems(const fs::path& p) {
   int count = 0;
   for (auto& elem : p) {
-    if (elem != "/" && elem != "")
+    if (elem != p.root_name() && elem != "/" && elem != "")
       ++count;
   }
   return count;
@@ -57,7 +57,7 @@ TEST_CASE(basic_test) {
   path dot_dot_to_root;
   for (int i=0; i < cwd_depth; ++i)
     dot_dot_to_root /= "..";
-  path relative_cwd = cwd.native().substr(1);
+  path relative_cwd = cwd.native().substr(cwd.root_path().native().size());
   // clang-format off
   struct {
     fs::path input;
@@ -75,11 +75,51 @@ TEST_CASE(basic_test) {
       {"a", "/", relative_cwd / "a"},
       {"a/b", "/", relative_cwd / "a/b"},
       {"a", "/net", ".." / relative_cwd / "a"},
+#ifdef _WIN32
+      {"//foo/", "//foo", "//foo/"},
+      {"//foo", "//foo/", "//foo"},
+#else
       {"//foo/", "//foo", "."},
       {"//foo", "//foo/", "."},
+#endif
       {"//foo", "//foo", "."},
       {"//foo/", "//foo/", "."},
-      {"//base", "a", dot_dot_to_root / "../base"},
+#ifdef _WIN32
+      {"//foo", "a", "//foo"},
+      {"//foo/a", "//bar", "//foo/a"},
+      {"//foo/a", "//bar/", "//foo/a"},
+      {"//foo/a", "b", "//foo/a"},
+      {"//foo/a", "/b", "//foo/a"},
+      {"//foo/a", "//bar/b", "//foo/a"},
+      // Using X: instead of C: to avoid influence from the CWD being under C:
+      {"X:/a", "X:/b", "../a"},
+      {"X:/a", "X:b", "X:/a"},
+      {"X:/a", "Y:/a", "X:/a"},
+      {"X:/a", "Y:/b", "X:/a"},
+      {"X:/a", "Y:b", "X:/a"},
+      {"X:a", "X:/b", "X:a"},
+      {"X:a", "X:b", "../a"},
+      {"X:a", "Y:/a", "X:a"},
+      {"X:a", "Y:/b", "X:a"},
+      {"X:a", "Y:b", "X:a"},
+#else
+      {"//foo", "a", dot_dot_to_root / "../foo"},
+      {"//foo/a", "//bar", "../foo/a"},
+      {"//foo/a", "//bar/", "../foo/a"},
+      {"//foo/a", "b", dot_dot_to_root / "../foo/a"},
+      {"//foo/a", "/b", "../foo/a"},
+      {"//foo/a", "//bar/b", "../../foo/a"},
+      {"X:/a", "X:/b", "../a"},
+      {"X:/a", "X:b", "../X:/a"},
+      {"X:/a", "Y:/a", "../../X:/a"},
+      {"X:/a", "Y:/b", "../../X:/a"},
+      {"X:/a", "Y:b", "../X:/a"},
+      {"X:a", "X:/b", "../../X:a"},
+      {"X:a", "X:b", "../X:a"},
+      {"X:a", "Y:/a", "../../X:a"},
+      {"X:a", "Y:/b", "../../X:a"},
+      {"X:a", "Y:b", "../X:a"},
+#endif
       {"a", "a", "."},
       {"a/b", "a/b", "."},
       {"a/b/c/", "a/b/c/", "."},
@@ -96,8 +136,10 @@ TEST_CASE(basic_test) {
   for (auto& TC : TestCases) {
     ++ID;
     std::error_code ec = GetTestEC();
-    fs::path p(TC.input);
+    fs::path p = TC.input;
     const fs::path output = fs::proximate(p, TC.base, ec);
+    fs::path expect = TC.expect;
+    expect.make_preferred();
     if (ec) {
       TEST_CHECK(!ec);
       std::fprintf(stderr, "TEST CASE #%d FAILED:\n"
@@ -105,9 +147,9 @@ TEST_CASE(basic_test) {
                   "  Base: '%s'\n"
                   "  Expected: '%s'\n",
         ID, TC.input.string().c_str(), TC.base.string().c_str(),
-        TC.expect.string().c_str());
-    } else if (!PathEq(output, TC.expect)) {
-      TEST_CHECK(PathEq(output, TC.expect));
+        expect.string().c_str());
+    } else if (!PathEq(output, expect)) {
+      TEST_CHECK(PathEq(output, expect));
 
       const path canon_input = fs::weakly_canonical(TC.input);
       const path canon_base = fs::weakly_canonical(TC.base);
@@ -121,7 +163,7 @@ TEST_CASE(basic_test) {
                   "  Canon Input: '%s'\n"
                   "  Canon Base: '%s'\n",
         ID, TC.input.string().c_str(), TC.base.string().c_str(),
-        TC.expect.string().c_str(), output.string().c_str(),
+        expect.string().c_str(), output.string().c_str(),
         lexically_p.string().c_str(), canon_input.string().c_str(),
         canon_base.string().c_str());
     }
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.relative/relative.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.relative/relative.pass.cpp
index aba9023bf8b437f88a47e3760c956d62346bb35d..0c056057927d3b990d1b52dd38ca6dd1823ca2f7 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.relative/relative.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.relative/relative.pass.cpp
@@ -95,7 +95,17 @@ TEST_CASE(test_signature_9) {
   static_test_env static_env;
   fs::path p(static_env.SymlinkToDir / "dir2/../dir2/DNE/..");
   const fs::path output = fs::weakly_canonical(p);
+  // weakly_canonical has a quirk - if the path is considered to exist,
+  // it's returned without a trailing slash, otherwise it's returned with
+  // one (see a note in fs.op.weakly_canonical/weakly_canonical.pass.cpp).
+  // On Windows, a path like existent/nonexistentsubdir/.. is considered
+  // to exist, on posix it's considered to not exist. Therefore, the
+  // result here differs in the trailing slash.
+#ifdef _WIN32
+  TEST_CHECK(output == fs::path::string_type(static_env.Dir2));
+#else
   TEST_CHECK(output == fs::path::string_type(static_env.Dir2 / ""));
+#endif
 }
 
 TEST_CASE(test_signature_10) {
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.remove/remove.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.remove/remove.pass.cpp
index ef962bb98a7f6c0809cf270101db99b029d7526b..017da8f8c6117e6b061a37c501d5d961f1f9f738 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.remove/remove.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.remove/remove.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // bool remove(const path& p);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.remove_all/remove_all.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.remove_all/remove_all.pass.cpp
index 8dadf96883a2150facc536a47c7ea7b7bfc85ffa..c94726fff555e91d8a61990b51b53c7cfa05108e 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.remove_all/remove_all.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.remove_all/remove_all.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // uintmax_t remove_all(const path& p);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.rename/rename.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.rename/rename.pass.cpp
index c1491581b11c7ca651e41efdbbb4b8a0be1f26ee..b6930d8d5fa94da98afb2a69da0614d5f5a13ecc 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.rename/rename.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.rename/rename.pass.cpp
@@ -62,7 +62,13 @@ TEST_CASE(test_error_reporting)
     } cases[] = {
         {dne, dne},
         {file, dir},
-        {dir, file}
+#ifndef _WIN32
+        // The spec doesn't say that this case must be an error; fs.op.rename
+        // note 1.2.1 says that a file may be overwritten by a rename.
+        // On Windows, with rename() implemented with MoveFileExW, overwriting
+        // a file with a directory is not an error.
+        {dir, file},
+#endif
     };
     for (auto& TC : cases) {
         auto from_before = status(TC.from);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp
index 726e89cbb66e5a43131eef30c0a24c803ec7aeb5..881ea162e505acaaf84b9988a38e3a62c2f4ba2a 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // file_status status(const path& p);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp
index b9c072acdc2e9a82e40ce5ba75233b2d2c2311c5..3976e69d2c5ddd22f447de842738c9421578eb8a 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // file_status symlink_status(const path& p);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp
index 32748ded142868bbf69fcbfefdf831b6465f75b1..3cc60a26092f29a103261abeb136850077c8a3d9 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <filesystem>
 
 // path temp_directory_path();
@@ -64,6 +66,14 @@ TEST_CASE(basic_tests)
         {"TMP", env.create_dir("dir2")},
         {"TEMP", env.create_dir("dir3")},
         {"TEMPDIR", env.create_dir("dir4")}
+#endif
+    };
+    TestCase ignored_cases[] = {
+#ifdef _WIN32
+        {"TMPDIR", env.create_dir("dir5")},
+        {"TEMPDIR", env.create_dir("dir6")},
+#else
+        {"USERPROFILE", env.create_dir("dir5")},
 #endif
     };
     for (auto& TC : cases) {
@@ -114,6 +124,7 @@ TEST_CASE(basic_tests)
         UnsetEnv(TC.name);
     }
     // No env variables are defined
+    path fallback;
     {
         std::error_code ec = GetTestEC();
         path ret = temp_directory_path(ec);
@@ -123,6 +134,20 @@ TEST_CASE(basic_tests)
         TEST_CHECK(ret == "/tmp");
 #endif
         TEST_CHECK(is_directory(ret));
+        fallback = ret;
+    }
+    for (auto& TC : ignored_cases) {
+        // Check that certain variables are ignored
+        PutEnv(TC.name, TC.p);
+        std::error_code ec = GetTestEC();
+        path ret = temp_directory_path(ec);
+        TEST_CHECK(!ec);
+
+        // Check that we return the same as above when no vars were defined.
+        TEST_CHECK(ret == fallback);
+
+        // Finally erase this env variable
+        UnsetEnv(TC.name);
     }
 }
 
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.weakly_canonical/weakly_canonical.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.weakly_canonical/weakly_canonical.pass.cpp
index 08d963fe6652fae0a33c17fb017406f9f1a9281f..b0909da011710a3773e39631f9a942e2f0536600 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.weakly_canonical/weakly_canonical.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.weakly_canonical/weakly_canonical.pass.cpp
@@ -27,6 +27,7 @@ int main(int, char**) {
 
   static_test_env static_env;
 
+  fs::path root = fs::current_path().root_path();
   // clang-format off
   struct {
     fs::path input;
@@ -34,10 +35,10 @@ int main(int, char**) {
   } TestCases[] = {
       {"", fs::current_path()},
       {".", fs::current_path()},
-      {"/", "/"},
-      {"/foo", "/foo"},
-      {"/.", "/"},
-      {"/./", "/"},
+      {"/", root},
+      {"/foo", root / "foo"},
+      {"/.", root},
+      {"/./", root},
       {"a/b", fs::current_path() / "a/b"},
       {"a", fs::current_path() / "a"},
       {"a/b/", fs::current_path() / "a/b/"},
@@ -45,12 +46,20 @@ int main(int, char**) {
       {static_env.Dir, static_env.Dir},
       {static_env.SymlinkToDir, static_env.Dir},
       {static_env.SymlinkToDir / "dir2/.", static_env.Dir / "dir2"},
-      // FIXME? If the trailing separator occurs in a part of the path that exists,
+      // Note: If the trailing separator occurs in a part of the path that exists,
       // it is omitted. Otherwise it is added to the end of the result.
+      // MS STL and libstdc++ behave similarly.
       {static_env.SymlinkToDir / "dir2/./", static_env.Dir / "dir2"},
       {static_env.SymlinkToDir / "dir2/DNE/./", static_env.Dir / "dir2/DNE/"},
       {static_env.SymlinkToDir / "dir2", static_env.Dir2},
+#ifdef _WIN32
+      // On Windows, this path is considered to exist (even though it
+      // passes through a nonexistent directory), and thus is returned
+      // without a trailing slash, see the note above.
+      {static_env.SymlinkToDir / "dir2/../dir2/DNE/..", static_env.Dir2},
+#else
       {static_env.SymlinkToDir / "dir2/../dir2/DNE/..", static_env.Dir2 / ""},
+#endif
       {static_env.SymlinkToDir / "dir2/dir3/../DNE/DNE2", static_env.Dir2 / "DNE/DNE2"},
       {static_env.Dir / "../dir1", static_env.Dir},
       {static_env.Dir / "./.", static_env.Dir},
@@ -61,15 +70,17 @@ int main(int, char**) {
   bool Failed = false;
   for (auto& TC : TestCases) {
     ++ID;
-    fs::path p(TC.input);
+    fs::path p = TC.input;
+    fs::path expect = TC.expect;
+    expect.make_preferred();
     const fs::path output = fs::weakly_canonical(p);
-    if (!PathEq(output, TC.expect)) {
+    if (!PathEq(output, expect)) {
       Failed = true;
       std::fprintf(stderr, "TEST CASE #%d FAILED:\n"
                   "  Input: '%s'\n"
                   "  Expected: '%s'\n"
                   "  Output: '%s'\n",
-        ID, TC.input.string().c_str(), TC.expect.string().c_str(),
+        ID, TC.input.string().c_str(), expect.string().c_str(),
         output.string().c_str());
     }
   }
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
index 36868adabe2e78385a65658fb455efc9720b58d2..a2934b21826d8fdcf329d135c9b3f0f5a4e7bb66 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
@@ -12,6 +12,8 @@
 
 // REQUIRES: locale.en_US.UTF-8
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <iomanip>
 #include <istream>
 #include <cassert>
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/put_money.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/put_money.pass.cpp
index 5c896892b5a1451ed97833c093452e33e49e5d0f..acb4552e20e25e01cf00230c220c654fb1e4d982 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/put_money.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/put_money.pass.cpp
@@ -12,6 +12,8 @@
 
 // REQUIRES: locale.en_US.UTF-8
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <iomanip>
 #include <ostream>
 #include <cassert>
diff --git a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cerr.sh.cpp b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cerr.sh.cpp
index b14323d9fd1e33eca8d5ab804ca9a5b91ad600db..bb6c64fa4e0c1ce4cd7322ef31eb3acb99d0a58d 100644
--- a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cerr.sh.cpp
+++ b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cerr.sh.cpp
@@ -10,6 +10,8 @@
 
 // istream cerr;
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // FILE_DEPENDENCIES: ../check-stderr.sh
 // RUN: %{build}
 // RUN: %{exec} bash check-stderr.sh "%t.exe" "1234"
diff --git a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/clog.sh.cpp b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/clog.sh.cpp
index 6e69ae3426e80ee25daa0103c4a8b2df28243a72..884f34598ce88da518c44d894e558499e9c88854 100644
--- a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/clog.sh.cpp
+++ b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/clog.sh.cpp
@@ -10,6 +10,8 @@
 
 // istream clog;
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // FILE_DEPENDENCIES: ../check-stderr.sh
 // RUN: %{build}
 // RUN: %{exec} bash check-stderr.sh "%t.exe" "1234"
diff --git a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cout.sh.cpp b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cout.sh.cpp
index 171ac74593e8a1853fc0849d3c78d1f67b6cd143..809f2ce3783cffc73c1a1c860a1105bb065b4991 100644
--- a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cout.sh.cpp
+++ b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cout.sh.cpp
@@ -12,6 +12,8 @@
 
 // istream cout;
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // FILE_DEPENDENCIES: ../check-stdout.sh
 // RUN: %{build}
 // RUN: %{exec} bash check-stdout.sh "%t.exe" "1234"
diff --git a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcerr.sh.cpp b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcerr.sh.cpp
index 5d705e7a569c9667c928fe7d51abd7dddc8965cc..64ef28536dda699a05af102c8d134707c76a2a26 100644
--- a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcerr.sh.cpp
+++ b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcerr.sh.cpp
@@ -10,6 +10,8 @@
 
 // istream wcerr;
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // FILE_DEPENDENCIES: ../check-stderr.sh
 // RUN: %{build}
 // RUN: %{exec} bash check-stderr.sh "%t.exe" "1234"
diff --git a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wclog.sh.cpp b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wclog.sh.cpp
index ada106449c3b435e8804b54cc02f9184368bb1db..07c6a2540a17d2dafcf5372b2d17c3c3755f024a 100644
--- a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wclog.sh.cpp
+++ b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wclog.sh.cpp
@@ -10,6 +10,8 @@
 
 // istream wclog;
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // FILE_DEPENDENCIES: ../check-stderr.sh
 // RUN: %{build}
 // RUN: %{exec} bash check-stderr.sh "%t.exe" "1234"
diff --git a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcout.sh.cpp b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcout.sh.cpp
index 0d3306bf62da07068f7bf4c342ff822d3012866c..babdc3b510fd4073c80a2522421daaa13331e75e 100644
--- a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcout.sh.cpp
+++ b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcout.sh.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // XFAIL: libcpp-has-no-stdout
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 // <iostream>
 
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.cons/container.compile.fail.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.cons/container.compile.fail.cpp
index 9aad14992f567e7122f479ad41a720f187608f72..42aab385ce4b341184a27377cf9ac5d935fbee5c 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.cons/container.compile.fail.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.cons/container.compile.fail.cpp
@@ -19,7 +19,8 @@
 
 int main(int, char**)
 {
-    std::back_insert_iterator<std::vector<int> > i = std::vector<int>();
+    std::vector<int> v;
+    std::back_insert_iterator<std::vector<int> > i = v;
 
   return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op++/post.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op++/post.pass.cpp
index c6f93d222830423153883eec1f7299878804e813..4157f45c0bb3a816b8f6bbea4e4503cc986ed976 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op++/post.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op++/post.pass.cpp
@@ -15,12 +15,13 @@
 #include <iterator>
 #include <vector>
 #include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::back_insert_iterator<C> i(c);
@@ -28,12 +29,16 @@ test(C c)
     r = 0;
     assert(c.size() == 1);
     assert(c.back() == 0);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::vector<int>());
     test(nasty_vector<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op++/pre.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op++/pre.pass.cpp
index a104d46b4dd4eb2132b284b1055f520962887a2d..d0d043fe75b8ba389d511383c31f19d0f6c66fac 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op++/pre.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op++/pre.pass.cpp
@@ -12,26 +12,31 @@
 
 // back_insert_iterator<Cont>& operator++();
 
+#include <cassert>
 #include <iterator>
 #include <vector>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::back_insert_iterator<C> i(c);
     std::back_insert_iterator<C>& r = ++i;
     assert(&r == &i);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::vector<int>());
     test(nasty_vector<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op=/lv_value.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op=/lv_value.pass.cpp
index 9e8cad2bb387e5ce4b50761af03501f4ca81a234..d1e7c7897c8ec0562942ba55414c647af342eebe 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op=/lv_value.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op=/lv_value.pass.cpp
@@ -19,15 +19,17 @@
 #include <cassert>
 
 #include "test_macros.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX14 bool
 test(C c)
 {
     const typename C::value_type v = typename C::value_type();
     std::back_insert_iterator<C> i(c);
     i = v;
     assert(c.back() == v);
+    return true;
 }
 
 class Copyable
@@ -44,6 +46,9 @@ public:
 int main(int, char**)
 {
     test(std::vector<Copyable>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op=/rv_value.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op=/rv_value.pass.cpp
index 14d39775552111625644fc6a3d359e4c8ba73e12..8b0355cc6c15cfef16b11aa47b7cb7cc080ba41f 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op=/rv_value.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op=/rv_value.pass.cpp
@@ -23,19 +23,24 @@
 #include <cassert>
 
 #include "test_macros.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX14 bool
 test(C c)
 {
     std::back_insert_iterator<C> i(c);
     i = typename C::value_type();
     assert(c.back() == typename C::value_type());
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::vector<std::unique_ptr<int> >());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op_astrk/test.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op_astrk/test.pass.cpp
index 22180f5a6f153f81447762d25676323f37771fcf..35d399e0068ee68ffe858bf8aebf1c8e22f369dc 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op_astrk/test.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/back.insert.iter.op_astrk/test.pass.cpp
@@ -12,26 +12,31 @@
 
 // back_insert_iterator<Cont>& operator*();
 
+#include <cassert>
 #include <iterator>
 #include <vector>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::back_insert_iterator<C> i(c);
     std::back_insert_iterator<C>& r = *i;
     assert(&r == &i);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::vector<int>());
     test(nasty_vector<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.cons/container.compile.fail.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.cons/container.compile.fail.cpp
index eb3346b2e7aeabecbca56a6ba7ffdced8c453a3f..7ff5e6eb05fa14712c336e4d1a97155edac08470 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.cons/container.compile.fail.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.cons/container.compile.fail.cpp
@@ -19,7 +19,8 @@
 
 int main(int, char**)
 {
-    std::front_insert_iterator<std::list<int> > i = std::list<int>();
+    std::list<int> l;
+    std::front_insert_iterator<std::list<int> > i = l;
 
   return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.cons/container.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.cons/container.pass.cpp
index 7fac5f31a99e7dcb6e3055726cb19419590d425c..ccdb444f11b39122fb6b754214fdbea2b31d88a8 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.cons/container.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.cons/container.pass.cpp
@@ -14,21 +14,26 @@
 
 #include <iterator>
 #include <list>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::front_insert_iterator<C> i(c);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::list<int>());
     test(nasty_list<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op++/post.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op++/post.pass.cpp
index 44a42713a2b296d38f07b69c1055b028f4ce2047..f67e5d4629a5c7c5fb539ba9d7092503b329db8e 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op++/post.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op++/post.pass.cpp
@@ -12,15 +12,16 @@
 
 // front_insert_iterator<Cont> operator++(int);
 
+#include <cassert>
 #include <iterator>
 #include <list>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::front_insert_iterator<C> i(c);
@@ -28,12 +29,16 @@ test(C c)
     r = 0;
     assert(c.size() == 1);
     assert(c.back() == 0);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::list<int>());
     test(nasty_list<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op++/pre.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op++/pre.pass.cpp
index ac58374ae417b827c078343ae0fa5351d443ef88..e21f9078c450539044d37beb9ec73bb0d9d516bb 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op++/pre.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op++/pre.pass.cpp
@@ -12,26 +12,31 @@
 
 // front_insert_iterator<Cont>& operator++();
 
+#include <cassert>
 #include <iterator>
 #include <list>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::front_insert_iterator<C> i(c);
     std::front_insert_iterator<C>& r = ++i;
     assert(&r == &i);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::list<int>());
     test(nasty_list<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op=/lv_value.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op=/lv_value.pass.cpp
index ceeacb4bf452998e521ca15ea83c545dc31ed6d4..922a95b931d44ccd1579d9ebcdd9122a2cf3e797 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op=/lv_value.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op=/lv_value.pass.cpp
@@ -13,21 +13,23 @@
 // front_insert_iterator<Cont>&
 //   operator=(const Cont::value_type& value);
 
+#include <cassert>
 #include <iterator>
 #include <list>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     const typename C::value_type v = typename C::value_type();
     std::front_insert_iterator<C> i(c);
     i = v;
     assert(c.front() == v);
+    return true;
 }
 
 class Copyable
@@ -45,6 +47,9 @@ int main(int, char**)
 {
     test(std::list<Copyable>());
     test(nasty_list<Copyable>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op=/rv_value.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op=/rv_value.pass.cpp
index a2c7f4b1de6d44c42757ad800f412626ba5859e1..fe12cff7df562bf5f96965f7e54e280edde18b1e 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op=/rv_value.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op=/rv_value.pass.cpp
@@ -15,25 +15,30 @@
 // front_insert_iterator<Cont>&
 //   operator=(Cont::value_type&& value);
 
+#include <cassert>
 #include <iterator>
 #include <list>
 #include <memory>
-#include <cassert>
 
 #include "test_macros.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::front_insert_iterator<C> i(c);
     i = typename C::value_type();
     assert(c.front() == typename C::value_type());
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::list<std::unique_ptr<int> >());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op_astrk/test.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op_astrk/test.pass.cpp
index 1c2e62ebf3e4a9cb86e6debd868c68c7c6adcf21..d4d5351ac7b9bf8c6e85586e31d4d82fb5229a10 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op_astrk/test.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.insert.iter.op_astrk/test.pass.cpp
@@ -12,26 +12,31 @@
 
 // front_insert_iterator<Cont>& operator*();
 
+#include <cassert>
 #include <iterator>
 #include <list>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::front_insert_iterator<C> i(c);
     std::front_insert_iterator<C>& r = *i;
     assert(&r == &i);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::list<int>());
     test(nasty_list<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.inserter/test.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.inserter/test.pass.cpp
index e7e09c9998f89f1511d981eafac3c2b2615fd1dd..f27178941936c8eb1f0afc117d542f361df6ec3a 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.inserter/test.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/front.inserter/test.pass.cpp
@@ -12,27 +12,32 @@
 //   front_insert_iterator<Cont>
 //   front_inserter(Cont& x);
 
+#include <cassert>
 #include <iterator>
 #include <list>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::front_insert_iterator<C> i = std::front_inserter(c);
     i = 0;
     assert(c.size() == 1);
     assert(c.front() == 0);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::list<int>());
     test(nasty_list<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.cons/test.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.cons/test.pass.cpp
index c8650e400ee1187bfbab5a26d541e346d45d63e9..8def436a0cc914bb3678f875331829a005fcd864 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.cons/test.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.cons/test.pass.cpp
@@ -14,21 +14,26 @@
 
 #include <iterator>
 #include <vector>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::insert_iterator<C> i(c, c.begin());
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::vector<int>());
     test(nasty_vector<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op++/post.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op++/post.pass.cpp
index e8a1780a40b87e483ef2d3937f861fb3d0b37071..3447df97993aa7a27e6fdb73474e2e3fa957832b 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op++/post.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op++/post.pass.cpp
@@ -12,15 +12,16 @@
 
 // insert_iterator<Cont> operator++(int);
 
+#include <cassert>
 #include <iterator>
 #include <vector>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::insert_iterator<C> i(c, c.end());
@@ -28,12 +29,16 @@ test(C c)
     r = 0;
     assert(c.size() == 1);
     assert(c.back() == 0);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::vector<int>());
     test(nasty_vector<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op++/pre.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op++/pre.pass.cpp
index 5f6a359d2ed35f7005d03fd1968fba469a4048ae..0bfb5724e6eb7ff3aeede9ce25c6d7ec38de9812 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op++/pre.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op++/pre.pass.cpp
@@ -12,26 +12,31 @@
 
 // insert_iterator<Cont>& operator++();
 
+#include <cassert>
 #include <iterator>
 #include <vector>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::insert_iterator<C> i(c, c.end());
     std::insert_iterator<C>& r = ++i;
     assert(&r == &i);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::vector<int>());
     test(nasty_vector<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op_astrk/test.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op_astrk/test.pass.cpp
index 57080a26c8fa62a786a8100cc7b8df9519b67a34..d63497b6409e320b238559d370904be7fa441251 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op_astrk/test.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/insert.iter.op_astrk/test.pass.cpp
@@ -12,26 +12,31 @@
 
 // insert_iterator<Cont>& operator*();
 
+#include <cassert>
 #include <iterator>
 #include <vector>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::insert_iterator<C> i(c, c.end());
     std::insert_iterator<C>& r = *i;
     assert(&r == &i);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::vector<int>());
     test(nasty_vector<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/inserter/test.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/inserter/test.pass.cpp
index 77916a496fcf1c375f1fd5021b45cfa30cb77545..088989fe96c16127587e17d4ebaa4047bc312f20 100644
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/inserter/test.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/inserter/test.pass.cpp
@@ -12,27 +12,32 @@
 //   insert_iterator<Cont>
 //   inserter(Cont& x, Cont::iterator i);
 
+#include <cassert>
 #include <iterator>
 #include <vector>
-#include <cassert>
-#include "nasty_containers.h"
 
 #include "test_macros.h"
+#include "nasty_containers.h"
+#include "test_constexpr_container.h"
 
 template <class C>
-void
+TEST_CONSTEXPR_CXX20 bool
 test(C c)
 {
     std::insert_iterator<C> i = std::inserter(c, c.end());
     i = 0;
     assert(c.size() == 1);
     assert(c.back() == 0);
+    return true;
 }
 
 int main(int, char**)
 {
     test(std::vector<int>());
     test(nasty_vector<int>());
-
-  return 0;
+#if TEST_STD_VER >= 20
+    test(ConstexprFixedCapacityDeque<int, 10>());
+    static_assert(test(ConstexprFixedCapacityDeque<int, 10>()));
+#endif
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.dynamic/alloc.errors/set.new.handler/get_new_handler.pass.cpp b/libcxx/test/std/language.support/support.dynamic/alloc.errors/set.new.handler/get_new_handler.pass.cpp
index d20a65d8b6b489f2fded2128d4e6c932db05ba3b..b62d09c03e0a8285abb6d1444d4388a9fd2dbf7e 100644
--- a/libcxx/test/std/language.support/support.dynamic/alloc.errors/set.new.handler/get_new_handler.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/alloc.errors/set.new.handler/get_new_handler.pass.cpp
@@ -8,6 +8,8 @@
 
 // test get_new_handler
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <new>
 #include <cassert>
 
diff --git a/libcxx/test/std/language.support/support.dynamic/alloc.errors/set.new.handler/set_new_handler.pass.cpp b/libcxx/test/std/language.support/support.dynamic/alloc.errors/set.new.handler/set_new_handler.pass.cpp
index b0becc9efe8c42d7d5f5f23496c2e9cfa10eb506..ce27cec6b3149b508c7006fe2f12e4d3401f192f 100644
--- a/libcxx/test/std/language.support/support.dynamic/alloc.errors/set.new.handler/set_new_handler.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/alloc.errors/set.new.handler/set_new_handler.pass.cpp
@@ -8,6 +8,8 @@
 
 // test set_new_handler
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <new>
 #include <cassert>
 
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_align_val_t_nothrow_replace.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_align_val_t_nothrow_replace.pass.cpp
index e7a1e403d73dd0f97a10525dfe62a5f971dee893..8f87dfc5f8156d41cc7118d4193e45a19e8ddd3c 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_align_val_t_nothrow_replace.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_align_val_t_nothrow_replace.pass.cpp
@@ -20,11 +20,6 @@
 // XFAIL: with_system_cxx_lib=macosx10.10
 // XFAIL: with_system_cxx_lib=macosx10.9
 
-// On Windows libc++ doesn't provide its own definitions for new/delete
-// but instead depends on the ones in VCRuntime. However VCRuntime does not
-// yet provide aligned new/delete definitions so this test fails.
-// XFAIL: LIBCXX-WINDOWS-FIXME
-
 // test operator new nothrow by replacing only operator new
 
 #include <new>
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_array.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_array.pass.cpp
index 55dc5c7533403d3dd6f1a1424c86adeb9c1a0e67..543ee52063256688e720e0e44667d72810e1617b 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_array.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_array.pass.cpp
@@ -9,6 +9,7 @@
 // test operator new[]
 // NOTE: asan and msan will not call the new handler.
 // UNSUPPORTED: sanitizer-new-delete
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 
 #include <new>
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_array_nothrow.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_array_nothrow.pass.cpp
index ce8b18d0436415d7e97d7d585c35eccaab9dde1e..9f68b5d13832ed91ed85bc102a63948827e6e6f4 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_array_nothrow.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_array_nothrow.pass.cpp
@@ -9,6 +9,7 @@
 // test operator new [] (nothrow)
 // NOTE: asan and msan will not call the new handler.
 // UNSUPPORTED: sanitizer-new-delete
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 
 #include <new>
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.pass.cpp
index 448da1717900e209027601b512c8c53ce6e95fe0..906df9fc7a63972d26236b1834d14119669269ce 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.pass.cpp
@@ -10,6 +10,7 @@
 
 // asan and msan will not call the new handler.
 // UNSUPPORTED: sanitizer-new-delete
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 #include <new>
 #include <cstddef>
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_align_val_t_nothrow_replace.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_align_val_t_nothrow_replace.pass.cpp
index 62ceafb7644af6cfa217b89ae48451434720da14..efbb8fce9670ad3c98bdda7f73dff55a52e992a1 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_align_val_t_nothrow_replace.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_align_val_t_nothrow_replace.pass.cpp
@@ -20,11 +20,6 @@
 // XFAIL: with_system_cxx_lib=macosx10.10
 // XFAIL: with_system_cxx_lib=macosx10.9
 
-// On Windows libc++ doesn't provide its own definitions for new/delete
-// but instead depends on the ones in VCRuntime. However VCRuntime does not
-// yet provide aligned new/delete definitions so this test fails.
-// XFAIL: LIBCXX-WINDOWS-FIXME
-
 // test operator new nothrow by replacing only operator new
 
 #include <new>
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_nothrow.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_nothrow.pass.cpp
index dfdf7d77e801634de652e5781cd01c276bd5932b..4aaf52f4f7c11c79260c2bcfebb092405f3cbf70 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_nothrow.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_nothrow.pass.cpp
@@ -10,6 +10,7 @@
 
 // asan and msan will not call the new handler.
 // UNSUPPORTED: sanitizer-new-delete
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 #include <new>
 #include <cstddef>
diff --git a/libcxx/test/std/language.support/support.limits/limits/is_specialized.pass.cpp b/libcxx/test/std/language.support/support.limits/limits/is_specialized.pass.cpp
index e4546fee4bd910ef67dfbbb181387178953731ec..369e6ae10048fea4616ee719b00647f86e3c430e 100644
--- a/libcxx/test/std/language.support/support.limits/limits/is_specialized.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/limits/is_specialized.pass.cpp
@@ -46,6 +46,9 @@ int main(int, char**)
     test<bool>();
     test<char>();
     test<wchar_t>();
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+    test<char8_t>();
+#endif
 #ifndef _LIBCPP_HAS_NO_UNICODE_CHARS
     test<char16_t>();
     test<char32_t>();
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/iterator.version.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/iterator.version.pass.cpp
index 27318c51a0a0943c26c241e6b59af75b70ec1c66..4234cc9a142d4efca42d2aa89d9a23495585e038 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/iterator.version.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/iterator.version.pass.cpp
@@ -146,17 +146,11 @@
 #   error "__cpp_lib_array_constexpr should have the value 201811L in c++20"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_iterator
-#     error "__cpp_lib_constexpr_iterator should be defined in c++20"
-#   endif
-#   if __cpp_lib_constexpr_iterator != 201811L
-#     error "__cpp_lib_constexpr_iterator should have the value 201811L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_iterator
-#     error "__cpp_lib_constexpr_iterator should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_iterator
+#   error "__cpp_lib_constexpr_iterator should be defined in c++20"
+# endif
+# if __cpp_lib_constexpr_iterator != 201811L
+#   error "__cpp_lib_constexpr_iterator should have the value 201811L in c++20"
 # endif
 
 # ifndef __cpp_lib_make_reverse_iterator
@@ -209,17 +203,11 @@
 #   error "__cpp_lib_array_constexpr should have the value 201811L in c++2b"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_iterator
-#     error "__cpp_lib_constexpr_iterator should be defined in c++2b"
-#   endif
-#   if __cpp_lib_constexpr_iterator != 201811L
-#     error "__cpp_lib_constexpr_iterator should have the value 201811L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_iterator
-#     error "__cpp_lib_constexpr_iterator should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_iterator
+#   error "__cpp_lib_constexpr_iterator should be defined in c++2b"
+# endif
+# if __cpp_lib_constexpr_iterator != 201811L
+#   error "__cpp_lib_constexpr_iterator should have the value 201811L in c++2b"
 # endif
 
 # ifndef __cpp_lib_make_reverse_iterator
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/numbers.version.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/numbers.version.pass.cpp
index aaa64d1f7feb216ceea8170220dc15f8804beaa5..00752ded5cfbad0618dc208584733a6dd4072974 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/numbers.version.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/numbers.version.pass.cpp
@@ -42,7 +42,7 @@
 
 #elif TEST_STD_VER == 20
 
-# ifndef _LIBCPP_HAS_NO_CONCEPTS
+# if defined(__cpp_concepts) && __cpp_concepts >= 201907L
 #   ifndef __cpp_lib_math_constants
 #     error "__cpp_lib_math_constants should be defined in c++20"
 #   endif
@@ -53,11 +53,11 @@
 #   ifdef __cpp_lib_math_constants
 #     error "__cpp_lib_math_constants should not be defined when defined(__cpp_concepts) && __cpp_concepts >= 201907L is not defined!"
 #   endif
-# endif // _LIBCPP_HAS_NO_CONCEPTS
+# endif
 
 #elif TEST_STD_VER > 20
 
-# ifndef _LIBCPP_HAS_NO_CONCEPTS
+# if defined(__cpp_concepts) && __cpp_concepts >= 201907L
 #   ifndef __cpp_lib_math_constants
 #     error "__cpp_lib_math_constants should be defined in c++2b"
 #   endif
@@ -68,7 +68,7 @@
 #   ifdef __cpp_lib_math_constants
 #     error "__cpp_lib_math_constants should not be defined when defined(__cpp_concepts) && __cpp_concepts >= 201907L is not defined!"
 #   endif
-# endif // _LIBCPP_HAS_NO_CONCEPTS
+# endif
 
 #endif // TEST_STD_VER > 20
 
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/string.version.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/string.version.pass.cpp
index f65078736f8f6f990d04652468fd2ed191e75a89..1657440104999a643a5541ace84e7b77d0a6db9c 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/string.version.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/string.version.pass.cpp
@@ -18,7 +18,7 @@
 /*  Constant                                      Value
     __cpp_lib_allocator_traits_is_always_equal    201411L [C++17]
     __cpp_lib_char8_t                             201811L [C++20]
-    __cpp_lib_constexpr_string                    201907L [C++20]
+    __cpp_lib_constexpr_string                    201811L [C++20]
     __cpp_lib_erase_if                            202002L [C++20]
     __cpp_lib_nonmember_container_access          201411L [C++17]
     __cpp_lib_starts_ends_with                    201711L [C++20]
@@ -182,17 +182,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_string
-#     error "__cpp_lib_constexpr_string should be defined in c++20"
-#   endif
-#   if __cpp_lib_constexpr_string != 201907L
-#     error "__cpp_lib_constexpr_string should have the value 201907L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_string
-#     error "__cpp_lib_constexpr_string should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_string
+#   error "__cpp_lib_constexpr_string should be defined in c++20"
+# endif
+# if __cpp_lib_constexpr_string != 201811L
+#   error "__cpp_lib_constexpr_string should have the value 201811L in c++20"
 # endif
 
 # ifndef __cpp_lib_erase_if
@@ -256,17 +250,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_string
-#     error "__cpp_lib_constexpr_string should be defined in c++2b"
-#   endif
-#   if __cpp_lib_constexpr_string != 201907L
-#     error "__cpp_lib_constexpr_string should have the value 201907L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_string
-#     error "__cpp_lib_constexpr_string should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_string
+#   error "__cpp_lib_constexpr_string should be defined in c++2b"
+# endif
+# if __cpp_lib_constexpr_string != 201811L
+#   error "__cpp_lib_constexpr_string should have the value 201811L in c++2b"
 # endif
 
 # ifndef __cpp_lib_erase_if
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/string_view.version.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/string_view.version.pass.cpp
index 30c351888c08fe6cfcabafbda61895573fbe5dcd..22d9297cca515e5da28177b57c3df45875810490 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/string_view.version.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/string_view.version.pass.cpp
@@ -111,17 +111,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_string_view
-#     error "__cpp_lib_constexpr_string_view should be defined in c++20"
-#   endif
-#   if __cpp_lib_constexpr_string_view != 201811L
-#     error "__cpp_lib_constexpr_string_view should have the value 201811L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_string_view
-#     error "__cpp_lib_constexpr_string_view should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_string_view
+#   error "__cpp_lib_constexpr_string_view should be defined in c++20"
+# endif
+# if __cpp_lib_constexpr_string_view != 201811L
+#   error "__cpp_lib_constexpr_string_view should have the value 201811L in c++20"
 # endif
 
 # ifndef __cpp_lib_starts_ends_with
@@ -157,17 +151,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_string_view
-#     error "__cpp_lib_constexpr_string_view should be defined in c++2b"
-#   endif
-#   if __cpp_lib_constexpr_string_view != 201811L
-#     error "__cpp_lib_constexpr_string_view should have the value 201811L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_string_view
-#     error "__cpp_lib_constexpr_string_view should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_string_view
+#   error "__cpp_lib_constexpr_string_view should be defined in c++2b"
+# endif
+# if __cpp_lib_constexpr_string_view != 201811L
+#   error "__cpp_lib_constexpr_string_view should have the value 201811L in c++2b"
 # endif
 
 # ifndef __cpp_lib_starts_ends_with
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/tuple.version.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/tuple.version.pass.cpp
index ef9f61428782fe6357dd217f01024c6aa6c57b20..5d870a8cd0c102fbf203588ae5bd3e86de6c6594 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/tuple.version.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/tuple.version.pass.cpp
@@ -119,17 +119,11 @@
 #   error "__cpp_lib_apply should have the value 201603L in c++20"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_tuple
-#     error "__cpp_lib_constexpr_tuple should be defined in c++20"
-#   endif
-#   if __cpp_lib_constexpr_tuple != 201811L
-#     error "__cpp_lib_constexpr_tuple should have the value 201811L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_tuple
-#     error "__cpp_lib_constexpr_tuple should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_tuple
+#   error "__cpp_lib_constexpr_tuple should be defined in c++20"
+# endif
+# if __cpp_lib_constexpr_tuple != 201811L
+#   error "__cpp_lib_constexpr_tuple should have the value 201811L in c++20"
 # endif
 
 # ifndef __cpp_lib_make_from_tuple
@@ -162,17 +156,11 @@
 #   error "__cpp_lib_apply should have the value 201603L in c++2b"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_tuple
-#     error "__cpp_lib_constexpr_tuple should be defined in c++2b"
-#   endif
-#   if __cpp_lib_constexpr_tuple != 201811L
-#     error "__cpp_lib_constexpr_tuple should have the value 201811L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_tuple
-#     error "__cpp_lib_constexpr_tuple should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_tuple
+#   error "__cpp_lib_constexpr_tuple should be defined in c++2b"
+# endif
+# if __cpp_lib_constexpr_tuple != 201811L
+#   error "__cpp_lib_constexpr_tuple should have the value 201811L in c++2b"
 # endif
 
 # ifndef __cpp_lib_make_from_tuple
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.pass.cpp
index 6d198d1121f7449aa5a9d9b98fc663d758b66463..f34c68183c151f5b77bee9e3cd15842dd987c32b 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.pass.cpp
@@ -16,7 +16,7 @@
 // Test the feature test macros defined by <variant>
 
 /*  Constant             Value
-    __cpp_lib_variant    201606L [C++17]
+    __cpp_lib_variant    202102L [C++17]
 */
 
 #include <variant>
@@ -39,8 +39,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++17"
 # endif
-# if __cpp_lib_variant != 201606L
-#   error "__cpp_lib_variant should have the value 201606L in c++17"
+# if __cpp_lib_variant != 202102L
+#   error "__cpp_lib_variant should have the value 202102L in c++17"
 # endif
 
 #elif TEST_STD_VER == 20
@@ -48,8 +48,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++20"
 # endif
-# if __cpp_lib_variant != 201606L
-#   error "__cpp_lib_variant should have the value 201606L in c++20"
+# if __cpp_lib_variant != 202102L
+#   error "__cpp_lib_variant should have the value 202102L in c++20"
 # endif
 
 #elif TEST_STD_VER > 20
@@ -57,8 +57,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++2b"
 # endif
-# if __cpp_lib_variant != 201606L
-#   error "__cpp_lib_variant should have the value 201606L in c++2b"
+# if __cpp_lib_variant != 202102L
+#   error "__cpp_lib_variant should have the value 202102L in c++2b"
 # endif
 
 #endif // TEST_STD_VER > 20
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.pass.cpp
index 3a668768d5e571a68d84fd6cd14589d53b57afe7..593d63d3406cbe7d9b31097ff281b0d1e31f8a89 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.pass.cpp
@@ -53,7 +53,7 @@
     __cpp_lib_constexpr_iterator                   201811L [C++20]
     __cpp_lib_constexpr_memory                     201811L [C++20]
     __cpp_lib_constexpr_numeric                    201911L [C++20]
-    __cpp_lib_constexpr_string                     201907L [C++20]
+    __cpp_lib_constexpr_string                     201811L [C++20]
     __cpp_lib_constexpr_string_view                201811L [C++20]
     __cpp_lib_constexpr_tuple                      201811L [C++20]
     __cpp_lib_constexpr_utility                    201811L [C++20]
@@ -150,7 +150,7 @@
     __cpp_lib_uncaught_exceptions                  201411L [C++17]
     __cpp_lib_unordered_map_try_emplace            201411L [C++17]
     __cpp_lib_unwrap_ref                           201811L [C++20]
-    __cpp_lib_variant                              201606L [C++17]
+    __cpp_lib_variant                              202102L [C++17]
     __cpp_lib_void_t                               201411L [C++17]
 */
 
@@ -2090,8 +2090,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++17"
 # endif
-# if __cpp_lib_variant != 201606L
-#   error "__cpp_lib_variant should have the value 201606L in c++17"
+# if __cpp_lib_variant != 202102L
+#   error "__cpp_lib_variant should have the value 202102L in c++17"
 # endif
 
 # ifndef __cpp_lib_void_t
@@ -2436,17 +2436,11 @@
 #   error "__cpp_lib_constexpr_functional should have the value 201907L in c++20"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_iterator
-#     error "__cpp_lib_constexpr_iterator should be defined in c++20"
-#   endif
-#   if __cpp_lib_constexpr_iterator != 201811L
-#     error "__cpp_lib_constexpr_iterator should have the value 201811L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_iterator
-#     error "__cpp_lib_constexpr_iterator should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_iterator
+#   error "__cpp_lib_constexpr_iterator should be defined in c++20"
+# endif
+# if __cpp_lib_constexpr_iterator != 201811L
+#   error "__cpp_lib_constexpr_iterator should have the value 201811L in c++20"
 # endif
 
 # ifndef __cpp_lib_constexpr_memory
@@ -2463,43 +2457,25 @@
 #   error "__cpp_lib_constexpr_numeric should have the value 201911L in c++20"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_string
-#     error "__cpp_lib_constexpr_string should be defined in c++20"
-#   endif
-#   if __cpp_lib_constexpr_string != 201907L
-#     error "__cpp_lib_constexpr_string should have the value 201907L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_string
-#     error "__cpp_lib_constexpr_string should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_string
+#   error "__cpp_lib_constexpr_string should be defined in c++20"
+# endif
+# if __cpp_lib_constexpr_string != 201811L
+#   error "__cpp_lib_constexpr_string should have the value 201811L in c++20"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_string_view
-#     error "__cpp_lib_constexpr_string_view should be defined in c++20"
-#   endif
-#   if __cpp_lib_constexpr_string_view != 201811L
-#     error "__cpp_lib_constexpr_string_view should have the value 201811L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_string_view
-#     error "__cpp_lib_constexpr_string_view should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_string_view
+#   error "__cpp_lib_constexpr_string_view should be defined in c++20"
+# endif
+# if __cpp_lib_constexpr_string_view != 201811L
+#   error "__cpp_lib_constexpr_string_view should have the value 201811L in c++20"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_tuple
-#     error "__cpp_lib_constexpr_tuple should be defined in c++20"
-#   endif
-#   if __cpp_lib_constexpr_tuple != 201811L
-#     error "__cpp_lib_constexpr_tuple should have the value 201811L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_tuple
-#     error "__cpp_lib_constexpr_tuple should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_tuple
+#   error "__cpp_lib_constexpr_tuple should be defined in c++20"
+# endif
+# if __cpp_lib_constexpr_tuple != 201811L
+#   error "__cpp_lib_constexpr_tuple should have the value 201811L in c++20"
 # endif
 
 # ifndef __cpp_lib_constexpr_utility
@@ -2890,7 +2866,7 @@
 #   error "__cpp_lib_map_try_emplace should have the value 201411L in c++20"
 # endif
 
-# ifndef _LIBCPP_HAS_NO_CONCEPTS
+# if defined(__cpp_concepts) && __cpp_concepts >= 201907L
 #   ifndef __cpp_lib_math_constants
 #     error "__cpp_lib_math_constants should be defined in c++20"
 #   endif
@@ -2901,7 +2877,7 @@
 #   ifdef __cpp_lib_math_constants
 #     error "__cpp_lib_math_constants should not be defined when defined(__cpp_concepts) && __cpp_concepts >= 201907L is not defined!"
 #   endif
-# endif // _LIBCPP_HAS_NO_CONCEPTS
+# endif
 
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_math_special_functions
@@ -3301,8 +3277,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++20"
 # endif
-# if __cpp_lib_variant != 201606L
-#   error "__cpp_lib_variant should have the value 201606L in c++20"
+# if __cpp_lib_variant != 202102L
+#   error "__cpp_lib_variant should have the value 202102L in c++20"
 # endif
 
 # ifndef __cpp_lib_void_t
@@ -3647,17 +3623,11 @@
 #   error "__cpp_lib_constexpr_functional should have the value 201907L in c++2b"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_iterator
-#     error "__cpp_lib_constexpr_iterator should be defined in c++2b"
-#   endif
-#   if __cpp_lib_constexpr_iterator != 201811L
-#     error "__cpp_lib_constexpr_iterator should have the value 201811L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_iterator
-#     error "__cpp_lib_constexpr_iterator should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_iterator
+#   error "__cpp_lib_constexpr_iterator should be defined in c++2b"
+# endif
+# if __cpp_lib_constexpr_iterator != 201811L
+#   error "__cpp_lib_constexpr_iterator should have the value 201811L in c++2b"
 # endif
 
 # ifndef __cpp_lib_constexpr_memory
@@ -3674,43 +3644,25 @@
 #   error "__cpp_lib_constexpr_numeric should have the value 201911L in c++2b"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_string
-#     error "__cpp_lib_constexpr_string should be defined in c++2b"
-#   endif
-#   if __cpp_lib_constexpr_string != 201907L
-#     error "__cpp_lib_constexpr_string should have the value 201907L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_string
-#     error "__cpp_lib_constexpr_string should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_string
+#   error "__cpp_lib_constexpr_string should be defined in c++2b"
+# endif
+# if __cpp_lib_constexpr_string != 201811L
+#   error "__cpp_lib_constexpr_string should have the value 201811L in c++2b"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_string_view
-#     error "__cpp_lib_constexpr_string_view should be defined in c++2b"
-#   endif
-#   if __cpp_lib_constexpr_string_view != 201811L
-#     error "__cpp_lib_constexpr_string_view should have the value 201811L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_string_view
-#     error "__cpp_lib_constexpr_string_view should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_string_view
+#   error "__cpp_lib_constexpr_string_view should be defined in c++2b"
+# endif
+# if __cpp_lib_constexpr_string_view != 201811L
+#   error "__cpp_lib_constexpr_string_view should have the value 201811L in c++2b"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_constexpr_tuple
-#     error "__cpp_lib_constexpr_tuple should be defined in c++2b"
-#   endif
-#   if __cpp_lib_constexpr_tuple != 201811L
-#     error "__cpp_lib_constexpr_tuple should have the value 201811L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_constexpr_tuple
-#     error "__cpp_lib_constexpr_tuple should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_constexpr_tuple
+#   error "__cpp_lib_constexpr_tuple should be defined in c++2b"
+# endif
+# if __cpp_lib_constexpr_tuple != 201811L
+#   error "__cpp_lib_constexpr_tuple should have the value 201811L in c++2b"
 # endif
 
 # ifndef __cpp_lib_constexpr_utility
@@ -4104,7 +4056,7 @@
 #   error "__cpp_lib_map_try_emplace should have the value 201411L in c++2b"
 # endif
 
-# if !_LIBCPP_HAS_NO_CONCEPTS
+# if defined(__cpp_concepts) && __cpp_concepts >= 201907L
 #   ifndef __cpp_lib_math_constants
 #     error "__cpp_lib_math_constants should be defined in c++2b"
 #   endif
@@ -4115,7 +4067,7 @@
 #   ifdef __cpp_lib_math_constants
 #     error "__cpp_lib_math_constants should not be defined when defined(__cpp_concepts) && __cpp_concepts >= 201907L is not defined!"
 #   endif
-# endif // !_LIBCPP_HAS_NO_CONCEPTS
+# endif
 
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_math_special_functions
@@ -4539,8 +4491,8 @@
 # ifndef __cpp_lib_variant
 #   error "__cpp_lib_variant should be defined in c++2b"
 # endif
-# if __cpp_lib_variant != 201606L
-#   error "__cpp_lib_variant should have the value 201606L in c++2b"
+# if __cpp_lib_variant != 202102L
+#   error "__cpp_lib_variant should have the value 202102L in c++2b"
 # endif
 
 # ifndef __cpp_lib_void_t
diff --git a/libcxx/test/std/language.support/support.runtime/cstdlib.pass.cpp b/libcxx/test/std/language.support/support.runtime/cstdlib.pass.cpp
index 668213f84838c69e0b250f4c315a3ef7c24fe5d8..f2ff89db31c4c9a63642279829d3e3f7e72d45fd 100644
--- a/libcxx/test/std/language.support/support.runtime/cstdlib.pass.cpp
+++ b/libcxx/test/std/language.support/support.runtime/cstdlib.pass.cpp
@@ -8,6 +8,8 @@
 
 // test <cstdlib>
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <cstdlib>
 #include <type_traits>
 #include <cassert>
diff --git a/libcxx/test/std/language.support/support.runtime/ctime.pass.cpp b/libcxx/test/std/language.support/support.runtime/ctime.pass.cpp
index df827793a128273e0af7c0f5261a9aaf2625ecd3..d52581dd81bd2fdacd5bc13c8a9f03ba4886d2d4 100644
--- a/libcxx/test/std/language.support/support.runtime/ctime.pass.cpp
+++ b/libcxx/test/std/language.support/support.runtime/ctime.pass.cpp
@@ -8,6 +8,8 @@
 
 // test <ctime>
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <ctime>
 #include <type_traits>
 #include "test_macros.h"
diff --git a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/compare.pass.cpp b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/compare.pass.cpp
index 32c63633ae8bf200da6cb0468667f9ed0753b988..792292dc927a04f0e54856cda733dccc0c64a053 100644
--- a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/compare.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/compare.pass.cpp
@@ -24,6 +24,8 @@
 // it.
 // XFAIL: linux-gnu
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <locale>
 #include <string>
 #include <cassert>
diff --git a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/hash.pass.cpp b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/hash.pass.cpp
index eaad7a901bfdbc671340d56fa82fa4ea42fc4371..dbf50c12ff2ea0f39a857fb16d839b5e70bc8fbf 100644
--- a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/hash.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/hash.pass.cpp
@@ -8,6 +8,8 @@
 
 // REQUIRES: locale.en_US.UTF-8
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <locale>
 
 // template <class charT> class collate_byname
diff --git a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate/locale.collate.members/compare.pass.cpp b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate/locale.collate.members/compare.pass.cpp
index df657da29143cc5319a6eab3e4df9dabccdc9087..0a255a5b64f76d510d60a49da4d6beec837fe5b3 100644
--- a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate/locale.collate.members/compare.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate/locale.collate.members/compare.pass.cpp
@@ -13,6 +13,8 @@
 // int compare(const charT* low1, const charT* high1,
 //             const charT* low2, const charT* high2) const;
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate/locale.collate.members/hash.pass.cpp b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate/locale.collate.members/hash.pass.cpp
index b555805a078bec197273ce0586fe33d8c0aee645..2858ab02bffe27134df56ce44ba17476af574dd8 100644
--- a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate/locale.collate.members/hash.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate/locale.collate.members/hash.pass.cpp
@@ -14,6 +14,8 @@
 
 //   This test is not portable
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <locale>
 #include <string>
 #include <cassert>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/facet.ctype.special/facet.ctype.char.dtor/dtor.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/facet.ctype.special/facet.ctype.char.dtor/dtor.pass.cpp
index 4296212dacb16400c45c2689bca7a1cf14d7aa3d..b5fe59b0d9d6812de98b4ae79eac8ac97075a4a3 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/facet.ctype.special/facet.ctype.char.dtor/dtor.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/facet.ctype.special/facet.ctype.char.dtor/dtor.pass.cpp
@@ -12,6 +12,8 @@
 
 // ~ctype();
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/facet.ctype.special/facet.ctype.char.statics/classic_table.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/facet.ctype.special/facet.ctype.char.statics/classic_table.pass.cpp
index c824b1bd28ffe0a60e03e2d4f0e153ca23f732fa..8f8e02c2b2b5cbef72bb949838aeb121ddf4aff9 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/facet.ctype.special/facet.ctype.char.statics/classic_table.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/facet.ctype.special/facet.ctype.char.statics/classic_table.pass.cpp
@@ -12,6 +12,8 @@
 
 // static const mask* classic_table() throw();
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_1.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_1.pass.cpp
index 77ba7bd9f436d4b0809eb74d2d61922e442038ac..d033ec99870cdb47475474d2d0f7f011fac1b6e2 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_1.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_1.pass.cpp
@@ -13,6 +13,7 @@
 // bool is(mask m, charT c) const;
 
 // REQUIRES: locale.en_US.UTF-8
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 #include <locale>
 #include <type_traits>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_many.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_many.pass.cpp
index 31994fc5eb7006bd8d05c3b2db73e2bb5dc318ac..97ffeb2f7c60d4d3fec73d32aba99c7b81f825e3 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_many.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_many.pass.cpp
@@ -13,6 +13,7 @@
 // const charT* do_is(const charT* low, const charT* high, mask* vec) const;
 
 // REQUIRES: locale.en_US.UTF-8
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 #include <locale>
 #include <string>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_1.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_1.pass.cpp
index 6e4b15a48040e5a5de26320eaa21a33915095605..b819b62431956e41ae49b406f1943c8735741345 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_1.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_1.pass.cpp
@@ -8,6 +8,7 @@
 
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_CA.ISO8859-1
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_many.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_many.pass.cpp
index 0c2d477fbf253b210a9d638da31635e06decacc7..3f8e538782e3a9b3effc2bc773fce5ff57da0819 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_many.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/narrow_many.pass.cpp
@@ -8,6 +8,7 @@
 
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_CA.ISO8859-1
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_is.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_is.pass.cpp
index 02388b84bcfeb6935812d69cf6fef8ee31c110dc..2329d9bdadeb45d2e4c806862c121cdc5765dfda 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_is.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_is.pass.cpp
@@ -13,6 +13,7 @@
 // const charT* scan_is(mask m, const charT* low, const charT* high) const;
 
 // REQUIRES: locale.en_US.UTF-8
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 #include <locale>
 #include <string>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_not.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_not.pass.cpp
index 6bfb6b7a69f292074aece8efa2937fd59ad774e0..39e1bd5db30f28c6c6c03b94f093d3921119d6b0 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_not.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_not.pass.cpp
@@ -13,6 +13,7 @@
 // const charT* scan_not(mask m, const charT* low, const charT* high) const;
 
 // REQUIRES: locale.en_US.UTF-8
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 #include <locale>
 #include <string>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_1.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_1.pass.cpp
index 1327e7e01bc8f12a6083abd7069088e099da6db9..2d096e9f7f572057d05db7e9178ab45dc9122244 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_1.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_1.pass.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: locale.en_US.UTF-8
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_many.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_many.pass.cpp
index f1a30dd9605bf6bf12e8011d1ac50c22382b5629..3550c5474f5b308f0c6e6652babf0f305ecda088 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_many.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_many.pass.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: locale.en_US.UTF-8
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_en_US.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_en_US.pass.cpp
index 2fe66a5c6b790722f12051f4733509dde69346af..b9c96f2feb79f81a6d5f8675e018653bb83d5848 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_en_US.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_en_US.pass.cpp
@@ -14,6 +14,7 @@
 //               ios_base::iostate& err, long double& v) const;
 
 // REQUIRES: locale.en_US.UTF-8
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 #include <locale>
 #include <ios>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp
index f7041f96675d19aad0f92c1f2c3086fc87d8e066..77ee90eda0d44759793c25b1cb5d6f5f027c0b60 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp
@@ -11,6 +11,8 @@
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.fr_FR.UTF-8
 
 // <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_ru_RU.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_ru_RU.pass.cpp
index 2f734284a9051f952f61f434363daa1661587fa6..a6829d0ecbb7c36229ffe4e1f038bed24fa1caeb 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_ru_RU.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_ru_RU.pass.cpp
@@ -15,6 +15,8 @@
 // Possibly related to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=16006
 // XFAIL: linux
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.ru_RU.UTF-8
 
 // <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_zh_CN.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_zh_CN.pass.cpp
index f75a0bbb3df88faf4fa045293524b135f34b869a..632006c676e2de86d59a16f7503a4117175fd5bb 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_zh_CN.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_zh_CN.pass.cpp
@@ -9,6 +9,8 @@
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.zh_CN.UTF-8
 
 // <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_string_en_US.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_string_en_US.pass.cpp
index 5cb20dfe4f71e511bd3721ebadf7c1525b4cef62..2288cf5d1bd248d1c126a3897149f2f6bc42246c 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_string_en_US.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_string_en_US.pass.cpp
@@ -14,6 +14,7 @@
 //               ios_base::iostate& err, string_type& v) const;
 
 // REQUIRES: locale.en_US.UTF-8
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 #include <locale>
 #include <ios>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_en_US.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_en_US.pass.cpp
index 46135895bc07e497cb6f6fd368754a2453e130ff..bf47e1427364f6257a7f6c39e4ffd6978ebe7ae3 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_en_US.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_en_US.pass.cpp
@@ -14,6 +14,7 @@
 //               long double units) const;
 
 // REQUIRES: locale.en_US.UTF-8
+// XFAIL: LIBCXX-WINDOWS-FIXME
 
 #include <locale>
 #include <ios>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp
index 837f542a9e62124a4ec123e1013fc0743f8f3c4b..6b233430250aa55ea56a65888cc72994bf163b53 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp
@@ -11,6 +11,8 @@
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.fr_FR.UTF-8
 
 // <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_ru_RU.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_ru_RU.pass.cpp
index 2b82d61139c80161ccbb9ff975dfc463f092a253..1d73718709f605155f6a9cf485c1d8b3a81bae20 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_ru_RU.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_ru_RU.pass.cpp
@@ -15,6 +15,8 @@
 // Possibly related to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=16006
 // XFAIL: linux
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.ru_RU.UTF-8
 
 // <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_zh_CN.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_zh_CN.pass.cpp
index d33d40bcebef5a66fb6abb59192ea32d50f2e56a..b8cc831f15a5c68dac4c2f2f0fe6c10c95078b53 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_zh_CN.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_zh_CN.pass.cpp
@@ -9,6 +9,8 @@
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.zh_CN.UTF-8
 
 // <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_string_en_US.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_string_en_US.pass.cpp
index b0f5351746dcff4ef33ef98ac2ab14b2ae15bf29..a0602502463b99aff4a21037fbc9cd3dfefcf7b1 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_string_en_US.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_string_en_US.pass.cpp
@@ -15,6 +15,8 @@
 
 // REQUIRES: locale.en_US.UTF-8
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <locale>
 #include <ios>
 #include <streambuf>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/curr_symbol.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/curr_symbol.pass.cpp
index 983a3db6a197be35e72b0f6365359f179d9753a3..df0d53844215565b69c78604b6bd0ea4bb44e183 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/curr_symbol.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/curr_symbol.pass.cpp
@@ -11,6 +11,8 @@
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 // REQUIRES: locale.ru_RU.UTF-8
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp
index 77a9fbf798a8237d3d3ebca9be9503eb273c491b..53e54c2b89df1fc5f1a208cf2aefcf2a7bf89dcc 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp
@@ -11,6 +11,8 @@
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 // REQUIRES: locale.ru_RU.UTF-8
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/neg_format.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/neg_format.pass.cpp
index 9381a509228484bcab6d319bbbac251f303c1b04..5cb42164da10ac4f2620c53252ce7122795f83be 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/neg_format.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/neg_format.pass.cpp
@@ -11,6 +11,8 @@
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 // REQUIRES: locale.ru_RU.UTF-8
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/negative_sign.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/negative_sign.pass.cpp
index 8dfa00a7253e9d4717d81e13babaa79a28b900d0..19ae0015501446f22965054002fbc8df8fe7ab33 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/negative_sign.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/negative_sign.pass.cpp
@@ -11,6 +11,8 @@
 // REQUIRES: locale.ru_RU.UTF-8
 // REQUIRES: locale.zh_CN.UTF-8
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <locale>
 
 // class moneypunct_byname<charT, International>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/pos_format.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/pos_format.pass.cpp
index aab5fd99f3c83ab48923c3ae92117010169a6c1a..225c2464b70e33e0166ea926e93f6cc1bade95e6 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/pos_format.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/pos_format.pass.cpp
@@ -11,6 +11,8 @@
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 // REQUIRES: locale.ru_RU.UTF-8
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/thousands_sep.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/thousands_sep.pass.cpp
index 0a06657d0c65ebd1cc12df8604ed0613003f9f77..e360124c42f27f676209692d38f342119b44c80b 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/thousands_sep.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/thousands_sep.pass.cpp
@@ -9,6 +9,8 @@
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 // REQUIRES: locale.ru_RU.UTF-8
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_double.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_double.pass.cpp
index 474f9ce571877c4ff1af1c3b601c45559acb5683..c653d54f1c0fbb93089d0a1bf27e9e9ed3009fb8 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_double.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_double.pass.cpp
@@ -12,6 +12,8 @@
 
 // iter_type put(iter_type s, ios_base& iob, char_type fill, double v) const;
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <locale>
 #include <ios>
 #include <cassert>
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp
index fcb7aafa48ac996d6efae7332b09ba0992e854cd..a7bcacb7408f2c85dff45a91c5e7450fc8e83b9f 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp
@@ -15,6 +15,8 @@
 // TODO GLIBC uses a different string for positive and negative NAN numbers.
 // XFAIL: linux-gnu
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <locale>
 #include <ios>
 #include <cassert>
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp
index 526f4b8e69a7ea78b1310c645d54662da6e4fdb9..da872e69d8dc3cfc8b470d85c19de8d2b20f2931 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp
@@ -18,6 +18,8 @@
 // GLIBC also fails on the zh_CN test.
 // XFAIL: linux
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <locale>
 
 // class time_get_byname<charT, InputIterator>
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date_wide.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date_wide.pass.cpp
index c9f6a35016dec5a1926ed1305b10d38702fe07af..b8e04e19bd065511d8e5a60e9954fbd925166014 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date_wide.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date_wide.pass.cpp
@@ -18,6 +18,8 @@
 // GLIBC also fails on the zh_CN test.
 // XFAIL: linux
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <locale>
 
 // class time_get_byname<charT, InputIterator>
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp
index 648c079bdf3deb1a9a4bd63755473ec840bb36af..56f8d035b53a3451601e089c52a905c649222e53 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp
@@ -10,6 +10,8 @@
 // REQUIRES: locale.fr_FR.UTF-8
 // REQUIRES: locale.zh_CN.UTF-8
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <locale>
 
 // class time_get_byname<charT, InputIterator>
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp
index d851574f7631ec76f6832ca9727ea02458559953..a4c63754478136cd3e6625379f207d6378305c2b 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp
@@ -9,6 +9,8 @@
 // NetBSD does not support LC_TIME at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 // REQUIRES: locale.ru_RU.UTF-8
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one_wide.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one_wide.pass.cpp
index de81236503e4d0adb455f49ea203f42001f52efe..78960cb4cb26c8d44a5dda60ffda78293d462ee1 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one_wide.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one_wide.pass.cpp
@@ -9,6 +9,8 @@
 // NetBSD does not support LC_TIME at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 // REQUIRES: locale.ru_RU.UTF-8
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp
index 648a01b3cc14795fbf4aa1e7eb2127d762b7b401..9f2c30a32cdcca156cd6a79ccade6c249c6bdfd5 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp
@@ -11,6 +11,8 @@
 // REQUIRES: locale.ru_RU.UTF-8
 // REQUIRES: locale.zh_CN.UTF-8
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <locale>
 
 // class time_get_byname<charT, InputIterator>
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.put.byname/put1.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.put.byname/put1.pass.cpp
index 210a9aefa6754a5972186aaa21861d832e237a05..8daf3405d86b5f7c2fac134f3e39d7f1185519ee 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.put.byname/put1.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.put.byname/put1.pass.cpp
@@ -9,6 +9,8 @@
 // NetBSD does not support LC_TIME at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.put/locale.time.put.members/put2.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.put/locale.time.put.members/put2.pass.cpp
index 14bfeb7217698b1226d246c8ff7ad795a6a610d6..20b3f4337d68af557d72c5b6e158ac0f920734e3 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.put/locale.time.put.members/put2.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.put/locale.time.put.members/put2.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <locale>
 
 // class time_put<charT, OutputIterator>
diff --git a/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/grouping.pass.cpp b/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/grouping.pass.cpp
index 4dbc7e7b8f2a9d0fd318b9c4cec0ea28c3c8eb54..3bd1c855ea9165e1f2dab925a7cc84de6f2bcf4e 100644
--- a/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/grouping.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/grouping.pass.cpp
@@ -9,6 +9,8 @@
 // NetBSD does not support LC_NUMERIC at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 
diff --git a/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/thousands_sep.pass.cpp b/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/thousands_sep.pass.cpp
index 620408ef82d1e9786d72833faf4819f13cb6f037..627bc6a7fa8a68fc907a510b677aac9484d2ca79 100644
--- a/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/thousands_sep.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/thousands_sep.pass.cpp
@@ -9,6 +9,8 @@
 // NetBSD does not support LC_NUMERIC at the moment
 // XFAIL: netbsd
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 // REQUIRES: locale.fr_FR.UTF-8
 
diff --git a/libcxx/test/std/localization/locales/locale/locale.operators/compare.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.operators/compare.pass.cpp
index 0710e28c558fba6a70a8131be3b7695e16e1a5ef..001d834adc7517fec590b33d3be834e67d3ee956 100644
--- a/libcxx/test/std/localization/locales/locale/locale.operators/compare.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.operators/compare.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <locale>
 
 // template <class charT, class Traits, class Allocator>
diff --git a/libcxx/test/std/re/re.traits/isctype.pass.cpp b/libcxx/test/std/re/re.traits/isctype.pass.cpp
index 8c0b07988d6e730ddb7ed7f73751508704962faf..3563f8a37fd3a96ce318068db0ce77b23a40b6a7 100644
--- a/libcxx/test/std/re/re.traits/isctype.pass.cpp
+++ b/libcxx/test/std/re/re.traits/isctype.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <regex>
 
 // template <class charT> struct regex_traits;
diff --git a/libcxx/test/std/re/re.traits/lookup_classname.pass.cpp b/libcxx/test/std/re/re.traits/lookup_classname.pass.cpp
index 63ae0ca598f669f7bb61cf0fb70b80e89f46beae..31d93cbed8c78f6726f80e3a9e6bbdc38fed1afd 100644
--- a/libcxx/test/std/re/re.traits/lookup_classname.pass.cpp
+++ b/libcxx/test/std/re/re.traits/lookup_classname.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <regex>
 
 // template <class charT> struct regex_traits;
diff --git a/libcxx/test/std/strings/basic.string.hash/char_type_hash.fail.cpp b/libcxx/test/std/strings/basic.string.hash/char_type_hash.fail.cpp
index 94cd87b128f36b79c70652bc138a72bb57a2cbb0..91585d0fdae510bffcab4f9a481f6a2acbdc665b 100644
--- a/libcxx/test/std/strings/basic.string.hash/char_type_hash.fail.cpp
+++ b/libcxx/test/std/strings/basic.string.hash/char_type_hash.fail.cpp
@@ -17,6 +17,8 @@
 
 #include <string>
 
+#include "test_macros.h"
+
 template <class _CharT>
 struct trait // copied from <__string>
 {
@@ -57,12 +59,18 @@ void test() {
     typedef std::basic_string<CharT, trait<CharT> > str_t;
     std::hash<str_t>
         h; // expected-error-re 4 {{{{call to implicitly-deleted default constructor of 'std::hash<str_t>'|implicit instantiation of undefined template}} {{.+}}}}}}
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+    // expected-error-re@-2 {{{{call to implicitly-deleted default constructor of 'std::hash<str_t>'|implicit instantiation of undefined template}} {{.+}}}}}}
+#endif
     (void)h;
 }
 
 int main(int, char**) {
     test<char>();
     test<wchar_t>();
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+    test<char8_t>();
+#endif
     test<char16_t>();
     test<char32_t>();
 
diff --git a/libcxx/test/std/strings/char.traits/char.traits.specializations/char.traits.specializations.char8_t/types.pass.cpp b/libcxx/test/std/strings/char.traits/char.traits.specializations/char.traits.specializations.char8_t/types.pass.cpp
index c634a235d22d06cef9cc4aa4c95fe21f6883c2d9..53e10204fbb333daa9dd4c9f9b802934e4ead6f0 100644
--- a/libcxx/test/std/strings/char.traits/char.traits.specializations/char.traits.specializations.char8_t/types.pass.cpp
+++ b/libcxx/test/std/strings/char.traits/char.traits.specializations/char.traits.specializations.char8_t/types.pass.cpp
@@ -11,10 +11,10 @@
 
 // template<> struct char_traits<char8_t>
 
-// typedef char8_t       char_type;
+// typedef char8_t        char_type;
 // typedef unsigned int   int_type;
 // typedef streamoff      off_type;
-// typedef u16streampos   pos_type;
+// typedef u8streampos    pos_type;
 // typedef mbstate_t      state_type;
 
 #include <string>
@@ -29,7 +29,7 @@ int main(int, char**)
     static_assert((std::is_same<std::char_traits<char8_t>::char_type,  char8_t>::value), "");
     static_assert((std::is_same<std::char_traits<char8_t>::int_type,   unsigned int>::value), "");
     static_assert((std::is_same<std::char_traits<char8_t>::off_type,   std::streamoff>::value), "");
-    static_assert((std::is_same<std::char_traits<char8_t>::pos_type,   std::u16streampos>::value), "");
+    static_assert((std::is_same<std::char_traits<char8_t>::pos_type,   std::u8streampos>::value), "");
     static_assert((std::is_same<std::char_traits<char8_t>::state_type, std::mbstate_t>::value), "");
 #endif
 
diff --git a/libcxx/test/std/strings/string.view/string.view.hash/char_type.hash.fail.cpp b/libcxx/test/std/strings/string.view/string.view.hash/char_type.hash.fail.cpp
index 5d6d152b6c59e07599aedbad40f36793b871960b..4d645c5da166a9c5fcc6723aae1fd3d08925df41 100644
--- a/libcxx/test/std/strings/string.view/string.view.hash/char_type.hash.fail.cpp
+++ b/libcxx/test/std/strings/string.view/string.view.hash/char_type.hash.fail.cpp
@@ -18,6 +18,8 @@
 #include <string_view>
 #include <string> // for 'mbstate_t'
 
+#include "test_macros.h"
+
 template <class _CharT>
 struct trait // copied from <__string>
 {
@@ -58,12 +60,18 @@ void test() {
     typedef std::basic_string_view<CharT, trait<CharT> > strv_t;
     std::hash<strv_t>
         h; // expected-error-re 4 {{{{call to implicitly-deleted default constructor of 'std::hash<strv_t>'|implicit instantiation of undefined template}} {{.+}}}}}}
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+    // expected-error-re@-2 {{{{call to implicitly-deleted default constructor of 'std::hash<strv_t>'|implicit instantiation of undefined template}} {{.+}}}}}}
+#endif
     (void)h;
 }
 
 int main(int, char**) {
     test<char>();
     test<wchar_t>();
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+    test<char8_t>();
+#endif
     test<char16_t>();
     test<char32_t>();
 
diff --git a/libcxx/test/std/strings/string.view/string.view.ops/copy.pass.cpp b/libcxx/test/std/strings/string.view/string.view.ops/copy.pass.cpp
index e96650992cba608475a9bda5536d21bfd728da67..300650c5c4fa233f1e4eeca3dd2399d43948705a 100644
--- a/libcxx/test/std/strings/string.view/string.view.ops/copy.pass.cpp
+++ b/libcxx/test/std/strings/string.view/string.view.ops/copy.pass.cpp
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// XFAIL: gcc-10
+//     GCC's __builtin_strlen isn't constexpr yet
+
 // <string_view>
 
 // size_type copy(charT* s, size_type n, size_type pos = 0) const;
@@ -74,7 +77,16 @@ void test ( const CharT *s ) {
     test1(sv1, sv1.size() + 1, 0);
     test1(sv1, sv1.size() + 1, 1);
     test1(sv1, sv1.size() + 1, string_view_t::npos);
+}
 
+template<typename CharT>
+TEST_CONSTEXPR_CXX20 bool test_constexpr_copy(const CharT *abcde, const CharT *ghijk, const CharT *bcdjk)
+{
+    CharT buf[6] = {};
+    std::basic_string_view<CharT> lval(ghijk); lval.copy(buf, 6);
+    std::basic_string_view<CharT>(abcde).copy(buf, 3, 1);
+    assert(std::basic_string_view<CharT>(buf) == bcdjk);
+    return true;
 }
 
 int main(int, char**) {
@@ -100,5 +112,22 @@ int main(int, char**) {
     test ( U"" );
 #endif
 
+    test_constexpr_copy("ABCDE", "GHIJK", "BCDJK");
+    test_constexpr_copy(L"ABCDE", L"GHIJK", L"BCDJK");
+#if TEST_STD_VER >= 11
+    test_constexpr_copy(u"ABCDE", u"GHIJK", u"BCDJK");
+    test_constexpr_copy(U"ABCDE", U"GHIJK", U"BCDJK");
+#endif
+#if TEST_STD_VER >= 17
+    test_constexpr_copy(u8"ABCDE", u8"GHIJK", u8"BCDJK");
+#endif
+#if TEST_STD_VER >= 20
+    static_assert(test_constexpr_copy("ABCDE", "GHIJK", "BCDJK"));
+    static_assert(test_constexpr_copy(L"ABCDE", L"GHIJK", L"BCDJK"));
+    static_assert(test_constexpr_copy(u"ABCDE", u"GHIJK", u"BCDJK"));
+    static_assert(test_constexpr_copy(U"ABCDE", U"GHIJK", U"BCDJK"));
+    static_assert(test_constexpr_copy(u8"ABCDE", u8"GHIJK", u8"BCDJK"));
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp b/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp
index 36276b01b92fc14ddfdd2ab9a2e405565416eda3..64bbe1b941a7aeffc79baa4d594efae4c792576b 100644
--- a/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp
+++ b/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp
@@ -8,6 +8,8 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <thread>
 
 // class thread
diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.default/default.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.default/default.pass.cpp
index 9c51b3ac39f30741c1f4f9e8fe3d60cf6ca9088e..079dfaa57c96bac91f3e42eb27d811616ccb910e 100644
--- a/libcxx/test/std/utilities/function.objects/func.search/func.search.default/default.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.default/default.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <functional>
 
 // default searcher
diff --git a/libcxx/test/std/utilities/function.objects/func.search/func.search.default/default.pred.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/func.search.default/default.pred.pass.cpp
index 8ad0fe32d8d9b3895134b43f1bf4c8d9ffab5124..c24e792eb9b9e3069d5fdea297b782ed0f94b0c6 100644
--- a/libcxx/test/std/utilities/function.objects/func.search/func.search.default/default.pred.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.search/func.search.default/default.pred.pass.cpp
@@ -10,6 +10,8 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // default searcher
 // template<class _ForwardIterator, class _BinaryPredicate = equal_to<>>
 // class default_searcher {
diff --git a/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp b/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp
index 494b243759a5b77b7e1a48037c85c692d25a9ba2..b872915cc1da4d9b2b7d27bebd1f370642feec0f 100644
--- a/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <functional>
 
 // class function<R(ArgTypes...)>
diff --git a/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/noncopyable_return_type.pass.cpp b/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/noncopyable_return_type.pass.cpp
index ce84cc1fd8396ca796c8495284850b8a76cc5b18..ee8469a07d028b885b8d35415b2b2439af44ecb5 100644
--- a/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/noncopyable_return_type.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.func/noncopyable_return_type.pass.cpp
@@ -8,6 +8,8 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <functional>
 
 #include <functional>
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor2.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor2.pass.cpp
index debdc12c85886dde33f5fed45dca955deae9683c..b67667dc3fefbda043fbb8c21e1619265c27d8d3 100644
--- a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor2.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor2.pass.cpp
@@ -18,6 +18,8 @@
 #include <functional>
 #include <cassert>
 
+#include "test_macros.h"
+
 struct B {} b;
 
 struct A1 {
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/shared_ptr_copy_move.fail.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/shared_ptr_copy_move.fail.cpp
index 41452fd1af1254939e0a40a9d4741fc8d8088569..689d8ec85cea90c099295546dfd559581309e379 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/shared_ptr_copy_move.fail.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/shared_ptr_copy_move.fail.cpp
@@ -15,6 +15,8 @@
 #include <memory>
 #include <type_traits>
 
+#include "test_macros.h"
+
 struct A {
   int x = 42;
 };
diff --git a/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/aligned_storage.pass.cpp b/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/aligned_storage.pass.cpp
index 5aa9354e2ae0a4bd1ccbb95c97b0488ad875b9a8..592a34a513d26d9b127632b9a554294a25101e16 100644
--- a/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/aligned_storage.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/aligned_storage.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // type_traits
 
 // aligned_storage
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.comp/is_arithmetic.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.comp/is_arithmetic.pass.cpp
index 683e885e288d8cfe0b7d3c46f0bc566786364385..a6fc44384c3befd04d7743f32b759ee847a95335 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.comp/is_arithmetic.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.comp/is_arithmetic.pass.cpp
@@ -69,6 +69,12 @@ class Abstract
 
 enum Enum {zero, one};
 
+enum EnumSigned : int { two };
+
+enum EnumUnsigned : unsigned { three };
+
+enum class EnumClass { zero, one };
+
 typedef void (*FunctionPtr)();
 
 
@@ -97,6 +103,9 @@ int main(int, char**)
     test_is_not_arithmetic<char[]>();
     test_is_not_arithmetic<Union>();
     test_is_not_arithmetic<Enum>();
+    test_is_not_arithmetic<EnumSigned>();
+    test_is_not_arithmetic<EnumUnsigned>();
+    test_is_not_arithmetic<EnumClass>();
     test_is_not_arithmetic<FunctionPtr>();
     test_is_not_arithmetic<Empty>();
     test_is_not_arithmetic<incomplete_type>();
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_signed.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_signed.pass.cpp
index 4936cc788e250a7781f4eba951864cbacd8eb442..29a259fb2588ea2e58d168ba4dc2065264d1ca6c 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_signed.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_signed.pass.cpp
@@ -51,21 +51,73 @@ public:
 
 struct A; // incomplete
 
+class incomplete_type;
+
+class Empty {};
+
+class NotEmpty {
+  virtual ~NotEmpty();
+};
+
+union Union {};
+
+struct bit_zero {
+  int : 0;
+};
+
+class Abstract {
+  virtual ~Abstract() = 0;
+};
+
+enum Enum { zero, one };
+
+enum EnumSigned : int { two };
+
+enum EnumUnsigned : unsigned { three };
+
+enum class EnumClass { zero, one };
+
+typedef void (*FunctionPtr)();
+
 int main(int, char**)
 {
-    test_is_not_signed<void>();
-    test_is_not_signed<int&>();
-    test_is_not_signed<Class>();
-    test_is_not_signed<int*>();
-    test_is_not_signed<const int*>();
-    test_is_not_signed<char[3]>();
-    test_is_not_signed<char[]>();
-    test_is_not_signed<bool>();
-    test_is_not_signed<unsigned>();
-    test_is_not_signed<A>();
-
-    test_is_signed<int>();
-    test_is_signed<double>();
+  // Cases where !is_arithmetic implies !is_signed
+  test_is_not_signed<std::nullptr_t>();
+  test_is_not_signed<void>();
+  test_is_not_signed<int&>();
+  test_is_not_signed<int&&>();
+  test_is_not_signed<Class>();
+  test_is_not_signed<int*>();
+  test_is_not_signed<const int*>();
+  test_is_not_signed<char[3]>();
+  test_is_not_signed<char[]>();
+  test_is_not_signed<Union>();
+  test_is_not_signed<Enum>();
+  test_is_not_signed<EnumSigned>();
+  test_is_not_signed<EnumUnsigned>();
+  test_is_not_signed<EnumClass>();
+  test_is_not_signed<FunctionPtr>();
+  test_is_not_signed<Empty>();
+  test_is_not_signed<incomplete_type>();
+  test_is_not_signed<A>();
+  test_is_not_signed<bit_zero>();
+  test_is_not_signed<NotEmpty>();
+  test_is_not_signed<Abstract>();
+
+  test_is_signed<signed char>();
+  test_is_signed<short>();
+  test_is_signed<int>();
+  test_is_signed<long>();
+  test_is_signed<float>();
+  test_is_signed<double>();
+
+  test_is_not_signed<unsigned char>();
+  test_is_not_signed<unsigned short>();
+  test_is_not_signed<unsigned int>();
+  test_is_not_signed<unsigned long>();
+
+  test_is_not_signed<bool>();
+  test_is_not_signed<unsigned>();
 
 #ifndef _LIBCPP_HAS_NO_INT128
     test_is_signed<__int128_t>();
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_unsigned.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_unsigned.pass.cpp
index bc70a43b9bd505aa6c58931af0d7ea535bfe3135..3c200b8f390547b600ce91b36d9dd26f166e6897 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_unsigned.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_unsigned.pass.cpp
@@ -51,21 +51,73 @@ public:
 
 struct A; // incomplete
 
+class incomplete_type;
+
+class Empty {};
+
+class NotEmpty {
+  virtual ~NotEmpty();
+};
+
+union Union {};
+
+struct bit_zero {
+  int : 0;
+};
+
+class Abstract {
+  virtual ~Abstract() = 0;
+};
+
+enum Enum { zero, one };
+
+enum EnumSigned : int { two };
+
+enum EnumUnsigned : unsigned { three };
+
+enum class EnumClass { zero, one };
+
+typedef void (*FunctionPtr)();
+
 int main(int, char**)
 {
-    test_is_not_unsigned<void>();
-    test_is_not_unsigned<int&>();
-    test_is_not_unsigned<Class>();
-    test_is_not_unsigned<int*>();
-    test_is_not_unsigned<const int*>();
-    test_is_not_unsigned<char[3]>();
-    test_is_not_unsigned<char[]>();
-    test_is_not_unsigned<int>();
-    test_is_not_unsigned<double>();
-    test_is_not_unsigned<A>();
-
-    test_is_unsigned<bool>();
-    test_is_unsigned<unsigned>();
+  // Cases where !is_arithmetic implies !is_unsigned
+  test_is_not_unsigned<std::nullptr_t>();
+  test_is_not_unsigned<void>();
+  test_is_not_unsigned<int&>();
+  test_is_not_unsigned<int&&>();
+  test_is_not_unsigned<Class>();
+  test_is_not_unsigned<int*>();
+  test_is_not_unsigned<const int*>();
+  test_is_not_unsigned<char[3]>();
+  test_is_not_unsigned<char[]>();
+  test_is_not_unsigned<Union>();
+  test_is_not_unsigned<Enum>();
+  test_is_not_unsigned<EnumSigned>();
+  test_is_not_unsigned<EnumUnsigned>();
+  test_is_not_unsigned<EnumClass>();
+  test_is_not_unsigned<FunctionPtr>();
+  test_is_not_unsigned<Empty>();
+  test_is_not_unsigned<incomplete_type>();
+  test_is_not_unsigned<A>();
+  test_is_not_unsigned<bit_zero>();
+  test_is_not_unsigned<NotEmpty>();
+  test_is_not_unsigned<Abstract>();
+
+  test_is_not_unsigned<signed char>();
+  test_is_not_unsigned<short>();
+  test_is_not_unsigned<int>();
+  test_is_not_unsigned<long>();
+  test_is_not_unsigned<float>();
+  test_is_not_unsigned<double>();
+
+  test_is_unsigned<unsigned char>();
+  test_is_unsigned<unsigned short>();
+  test_is_unsigned<unsigned int>();
+  test_is_unsigned<unsigned long>();
+
+  test_is_unsigned<bool>();
+  test_is_unsigned<unsigned>();
 
 #ifndef _LIBCPP_HAS_NO_INT128
     test_is_unsigned<__uint128_t>();
diff --git a/libcxx/test/std/utilities/time/date.time/ctime.pass.cpp b/libcxx/test/std/utilities/time/date.time/ctime.pass.cpp
index 8ad3cca4758e088259b8ec401fc010cc4853dd3b..4a1fda01353bf8e05edd75de2910be6253f86000 100644
--- a/libcxx/test/std/utilities/time/date.time/ctime.pass.cpp
+++ b/libcxx/test/std/utilities/time/date.time/ctime.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 #include <ctime>
 #include <type_traits>
 
diff --git a/libcxx/test/std/utilities/time/time.clock/time.clock.file/now.pass.cpp b/libcxx/test/std/utilities/time/time.clock/time.clock.file/now.pass.cpp
index 9bc047771d6ddefd973cfb1b987130655b09d5bf..a8f8472064363b8bdddf9d7e3e8034cf25ab7016 100644
--- a/libcxx/test/std/utilities/time/time.clock/time.clock.file/now.pass.cpp
+++ b/libcxx/test/std/utilities/time/time.clock/time.clock.file/now.pass.cpp
@@ -17,6 +17,8 @@
 // UNSUPPORTED: with_system_cxx_lib=macosx10.10
 // UNSUPPORTED: with_system_cxx_lib=macosx10.9
 
+// XFAIL: LIBCXX-WINDOWS-FIXME
+
 // <chrono>
 
 // file_clock
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/const_pair.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/const_pair.pass.cpp
index 02c59948d7f81bce5fa9bc7a691be4fc25fa0a44..e922d70062b81847cfd4889e411492e905ee5219 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/const_pair.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/const_pair.pass.cpp
@@ -30,7 +30,8 @@ struct PotentiallyThrowingCopyAssignable {
 
 #include "test_macros.h"
 
-int main(int, char**)
+TEST_CONSTEXPR_CXX20
+bool test()
 {
     {
         typedef std::pair<long, char> T0;
@@ -41,6 +42,16 @@ int main(int, char**)
         assert(std::get<0>(t1) == 2);
         assert(std::get<1>(t1) == short('a'));
     }
+    return true;
+}
+
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER >= 20
+    static_assert(test());
+#endif
+
     {
         // test that the implicitly generated copy assignment operator
         // is properly deleted
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/convert_copy.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/convert_copy.pass.cpp
index b50cbdef8abf31cecc14f7018f104b1d08c5e79e..c04baef71ff47a1bdd4b9ff64795340cf0a09064 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/convert_copy.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/convert_copy.pass.cpp
@@ -21,35 +21,31 @@
 
 #include "test_macros.h"
 
-struct B
-{
+struct B {
     int id_;
 
-    explicit B(int i = 0) : id_(i) {}
+    constexpr explicit B(int i = 0) : id_(i) {}
 };
 
-struct D
-    : B
-{
-    explicit D(int i = 0) : B(i) {}
+struct D : B {
+    constexpr explicit D(int i = 0) : B(i) {}
 };
 
 struct NonAssignable {
-  NonAssignable& operator=(NonAssignable const&) = delete;
-  NonAssignable& operator=(NonAssignable&&) = delete;
+    NonAssignable& operator=(NonAssignable const&) = delete;
+    NonAssignable& operator=(NonAssignable&&) = delete;
 };
 
-struct NothrowCopyAssignable
-{
+struct NothrowCopyAssignable {
     NothrowCopyAssignable& operator=(NothrowCopyAssignable const&) noexcept { return *this; }
 };
 
-struct PotentiallyThrowingCopyAssignable
-{
+struct PotentiallyThrowingCopyAssignable {
     PotentiallyThrowingCopyAssignable& operator=(PotentiallyThrowingCopyAssignable const&) { return *this; }
 };
 
-int main(int, char**)
+TEST_CONSTEXPR_CXX20
+bool test()
 {
     {
         typedef std::tuple<long> T0;
@@ -102,6 +98,16 @@ int main(int, char**)
         assert(std::get<0>(t) == 43);
         assert(&std::get<0>(t) == &x);
     }
+    return true;
+}
+
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER >= 20
+    static_assert(test());
+#endif
+
     {
         using T = std::tuple<int, NonAssignable>;
         using U = std::tuple<NonAssignable, int>;
@@ -116,6 +122,7 @@ int main(int, char**)
     {
         typedef std::tuple<PotentiallyThrowingCopyAssignable, long> T0;
         typedef std::tuple<PotentiallyThrowingCopyAssignable, int> T1;
+        static_assert(std::is_assignable<T0&, T1 const&>::value, "");
         static_assert(!std::is_nothrow_assignable<T0&, T1 const&>::value, "");
     }
 
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/convert_move.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/convert_move.pass.cpp
index 0d09c3be0b6077919ba8b7987c2c2f493dbe06e6..d1c3cfb0479fb0e66e247042ca2a612f5a908a8d 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/convert_move.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/convert_move.pass.cpp
@@ -23,24 +23,19 @@
 
 #include "test_macros.h"
 
-struct B
-{
+struct B {
     int id_;
-
-    explicit B(int i= 0) : id_(i) {}
-
+    explicit B(int i = 0) : id_(i) {}
     virtual ~B() {}
 };
 
-struct D
-    : B
-{
+struct D : B {
     explicit D(int i) : B(i) {}
 };
 
 struct E {
-  E() = default;
-  E& operator=(int) {
+  constexpr E() = default;
+  TEST_CONSTEXPR_CXX14 E& operator=(int) {
       return *this;
   }
 };
@@ -92,7 +87,8 @@ struct TrackMove
     bool moved_from;
 };
 
-int main(int, char**)
+TEST_CONSTEXPR_CXX20
+bool test()
 {
     {
         typedef std::tuple<long> T0;
@@ -111,6 +107,29 @@ int main(int, char**)
         assert(std::get<0>(t1) == 2);
         assert(std::get<1>(t1) == int('a'));
     }
+    {
+        // Test that tuple evaluates correctly applies an lvalue reference
+        // before evaluating is_assignable (i.e. 'is_assignable<int&, int&&>')
+        // instead of evaluating 'is_assignable<int&&, int&&>' which is false.
+        int x = 42;
+        int y = 43;
+        std::tuple<int&&, E> t(std::move(x), E{});
+        std::tuple<int&&, int> t2(std::move(y), 44);
+        t = std::move(t2);
+        assert(std::get<0>(t) == 43);
+        assert(&std::get<0>(t) == &x);
+    }
+
+    return true;
+}
+
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER >= 20
+    static_assert(test());
+#endif
+
     {
         typedef std::tuple<long, char, D> T0;
         typedef std::tuple<long long, int, B> T1;
@@ -143,18 +162,7 @@ int main(int, char**)
         assert(std::get<1>(t1) == int('a'));
         assert(std::get<2>(t1)->id_ == 3);
     }
-    {
-        // Test that tuple evaluates correctly applies an lvalue reference
-        // before evaluating is_assignable (i.e. 'is_assignable<int&, int&&>')
-        // instead of evaluating 'is_assignable<int&&, int&&>' which is false.
-        int x = 42;
-        int y = 43;
-        std::tuple<int&&, E> t(std::move(x), E{});
-        std::tuple<int&&, int> t2(std::move(y), 44);
-        t = std::move(t2);
-        assert(std::get<0>(t) == 43);
-        assert(&std::get<0>(t) == &x);
-    }
+
     {
         using T = std::tuple<int, NonAssignable>;
         using U = std::tuple<NonAssignable, int>;
@@ -169,6 +177,7 @@ int main(int, char**)
     {
         typedef std::tuple<PotentiallyThrowingMoveAssignable, long> T0;
         typedef std::tuple<PotentiallyThrowingMoveAssignable, int> T1;
+        static_assert(std::is_assignable<T0&, T1&&>::value, "");
         static_assert(!std::is_nothrow_assignable<T0&, T1&&>::value, "");
     }
     {
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/copy.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/copy.pass.cpp
index a0356464adfce9fdd7ed47f4cea6529bc64b65d1..71a878a7a0d5df82c443eb15c98bb63a36a02d4f 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/copy.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/copy.pass.cpp
@@ -45,7 +45,8 @@ struct CopyAssignableInt {
   CopyAssignableInt& operator=(int&) { return *this; }
 };
 
-int main(int, char**)
+TEST_CONSTEXPR_CXX20
+bool test()
 {
     {
         typedef std::tuple<> T;
@@ -68,15 +69,6 @@ int main(int, char**)
         assert(std::get<0>(t) == 2);
         assert(std::get<1>(t) == 'a');
     }
-    {
-        typedef std::tuple<int, char, std::string> T;
-        const T t0(2, 'a', "some text");
-        T t;
-        t = t0;
-        assert(std::get<0>(t) == 2);
-        assert(std::get<1>(t) == 'a');
-        assert(std::get<2>(t) == "some text");
-    }
     {
         // test reference assignment.
         using T = std::tuple<int&, int&&>;
@@ -92,6 +84,27 @@ int main(int, char**)
         assert(std::get<1>(t) == y2);
         assert(&std::get<1>(t) == &y);
     }
+
+    return true;
+}
+
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER >= 20
+    static_assert(test());
+#endif
+
+    {
+        // cannot be constexpr because of std::string
+        typedef std::tuple<int, char, std::string> T;
+        const T t0(2, 'a', "some text");
+        T t;
+        t = t0;
+        assert(std::get<0>(t) == 2);
+        assert(std::get<1>(t) == 'a');
+        assert(std::get<2>(t) == "some text");
+    }
     {
         // test that the implicitly generated copy assignment operator
         // is properly deleted
@@ -99,8 +112,8 @@ int main(int, char**)
         static_assert(!std::is_copy_assignable<T>::value, "");
     }
     {
-      using T = std::tuple<int, NonAssignable>;
-      static_assert(!std::is_copy_assignable<T>::value, "");
+        using T = std::tuple<int, NonAssignable>;
+        static_assert(!std::is_copy_assignable<T>::value, "");
     }
     {
         using T = std::tuple<int, CopyAssignable>;
@@ -132,6 +145,7 @@ int main(int, char**)
     }
     {
         using T = std::tuple<PotentiallyThrowingCopyAssignable, int>;
+        static_assert(std::is_copy_assignable<T>::value, "");
         static_assert(!std::is_nothrow_copy_assignable<T>::value, "");
     }
 
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/move.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/move.pass.cpp
index a89c4eab900e7201c12f7e20de75f8f3dcf045b2..91e513909c30a441413cddf78f58a490e6e2b16b 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/move.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.assign/move.pass.cpp
@@ -53,7 +53,8 @@ struct CountAssign {
 int CountAssign::copied = 0;
 int CountAssign::moved = 0;
 
-int main(int, char**)
+TEST_CONSTEXPR_CXX20
+bool test()
 {
     {
         typedef std::tuple<> T;
@@ -100,6 +101,16 @@ int main(int, char**)
         assert(std::get<1>(t) == y2);
         assert(&std::get<1>(t) == &y);
     }
+    return true;
+}
+
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER >= 20
+    static_assert(test());
+#endif
+
     {
         // test that the implicitly generated move assignment operator
         // is properly deleted
@@ -108,8 +119,8 @@ int main(int, char**)
         static_assert(!std::is_copy_assignable<T>::value, "");
     }
     {
-      using T = std::tuple<int, NonAssignable>;
-      static_assert(!std::is_move_assignable<T>::value, "");
+        using T = std::tuple<int, NonAssignable>;
+        static_assert(!std::is_move_assignable<T>::value, "");
     }
     {
         using T = std::tuple<int, MoveAssignable>;
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.creation/make_tuple.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.creation/make_tuple.pass.cpp
index f67d285eab8012b25fe199ecf49cd37e137143f1..9a6526e9af3a5b013637982d4d25f6f023fcd060 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.creation/make_tuple.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.creation/make_tuple.pass.cpp
@@ -21,26 +21,36 @@
 
 #include "test_macros.h"
 
+TEST_CONSTEXPR_CXX20
+bool test()
+{
+    int i = 0;
+    float j = 0;
+    std::tuple<int, int&, float&> t =
+        std::make_tuple(1, std::ref(i), std::ref(j));
+    assert(std::get<0>(t) == 1);
+    assert(std::get<1>(t) == 0);
+    assert(std::get<2>(t) == 0);
+    i = 2;
+    j = 3.5;
+    assert(std::get<0>(t) == 1);
+    assert(std::get<1>(t) == 2);
+    assert(std::get<2>(t) == 3.5);
+    std::get<1>(t) = 0;
+    std::get<2>(t) = 0;
+    assert(i == 0);
+    assert(j == 0);
+
+    return true;
+}
+
 int main(int, char**)
 {
-    {
-        int i = 0;
-        float j = 0;
-        std::tuple<int, int&, float&> t = std::make_tuple(1, std::ref(i),
-                                                          std::ref(j));
-        assert(std::get<0>(t) == 1);
-        assert(std::get<1>(t) == 0);
-        assert(std::get<2>(t) == 0);
-        i = 2;
-        j = 3.5;
-        assert(std::get<0>(t) == 1);
-        assert(std::get<1>(t) == 2);
-        assert(std::get<2>(t) == 3.5);
-        std::get<1>(t) = 0;
-        std::get<2>(t) = 0;
-        assert(i == 0);
-        assert(j == 0);
-    }
+    test();
+#if TEST_STD_VER >= 20
+    static_assert(test());
+#endif
+
 #if TEST_STD_VER > 11
     {
         constexpr auto t1 = std::make_tuple(0, 1, 3.14);
@@ -51,5 +61,5 @@ int main(int, char**)
     }
 #endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.creation/tie.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.creation/tie.pass.cpp
index 4cd91be2bc16ab3fdf7fa34f28cad8c2e4810011..0c6e6291c8bf46f42fa22b4d0db326f39c4ffe01 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.creation/tie.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.creation/tie.pass.cpp
@@ -21,8 +21,9 @@
 
 #include "test_macros.h"
 
-#if TEST_STD_VER > 11
-constexpr bool test_tie_constexpr() {
+TEST_CONSTEXPR_CXX14
+bool test_tie()
+{
     {
         int i = 42;
         double f = 1.1;
@@ -32,15 +33,23 @@ constexpr bool test_tie_constexpr() {
         assert(&std::get<0>(res) == &i);
         assert(&std::get<1>(res) == &std::ignore);
         assert(&std::get<2>(res) == &f);
-        // FIXME: If/when tuple gets constexpr assignment
-        //res = std::make_tuple(101, nullptr, -1.0);
+
+#if TEST_STD_VER >= 20
+        res = std::make_tuple(101, nullptr, -1.0);
+        assert(i == 101);
+        assert(f == -1.0);
+#endif
     }
     return true;
 }
-#endif
 
 int main(int, char**)
 {
+    test_tie();
+#if TEST_STD_VER >= 14
+    static_assert(test_tie(), "");
+#endif
+
     {
         int i = 0;
         std::string s;
@@ -48,18 +57,6 @@ int main(int, char**)
         assert(i == 42);
         assert(s == "C++");
     }
-#if TEST_STD_VER > 11
-    {
-        static constexpr int i = 42;
-        static constexpr double f = 1.1;
-        constexpr std::tuple<const int &, const double &> t = std::tie(i, f);
-        static_assert ( std::get<0>(t) == 42, "" );
-        static_assert ( std::get<1>(t) == 1.1, "" );
-    }
-    {
-        static_assert(test_tie_constexpr(), "");
-    }
-#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.swap/member_swap.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.swap/member_swap.pass.cpp
index 767d2dd0abdd7b349bac7166a85de1502d695389..e479a61e49f21a765f32a4c19c6e1cdce641e3ae 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.swap/member_swap.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.swap/member_swap.pass.cpp
@@ -20,7 +20,8 @@
 #include "test_macros.h"
 #include "MoveOnly.h"
 
-int main(int, char**)
+TEST_CONSTEXPR_CXX20
+bool test()
 {
     {
         typedef std::tuple<> T;
@@ -58,6 +59,15 @@ int main(int, char**)
         assert(std::get<1>(t1) == 1);
         assert(std::get<2>(t1) == 2);
     }
+    return true;
+}
+
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER >= 20
+    static_assert(test());
+#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct.pass.cpp
index 0178f5dffd0b46d1ce0cbe07724d7da01c192627..65f6d62ec09a34d8cbb70c54a2daaa26a9c1094c 100644
--- a/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct.pass.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct.pass.cpp
@@ -26,9 +26,9 @@ class A
     int i_;
     char c_;
 public:
-    A(int i, char c) : i_(i), c_(c) {}
-    int get_i() const {return i_;}
-    char get_c() const {return c_;}
+    constexpr A(int i, char c) : i_(i), c_(c) {}
+    constexpr int get_i() const {return i_;}
+    constexpr char get_c() const {return c_;}
 };
 
 class B
@@ -37,13 +37,14 @@ class B
     unsigned u1_;
     unsigned u2_;
 public:
-    B(double d, unsigned u1, unsigned u2) : d_(d), u1_(u1), u2_(u2) {}
-    double get_d() const {return d_;}
-    unsigned get_u1() const {return u1_;}
-    unsigned get_u2() const {return u2_;}
+    constexpr explicit B(double d, unsigned u1, unsigned u2) : d_(d), u1_(u1), u2_(u2) {}
+    constexpr double get_d() const {return d_;}
+    constexpr unsigned get_u1() const {return u1_;}
+    constexpr unsigned get_u2() const {return u2_;}
 };
 
-int main(int, char**)
+TEST_CONSTEXPR_CXX20
+bool test()
 {
     std::pair<A, B> p(std::piecewise_construct,
                       std::make_tuple(4, 'a'),
@@ -54,5 +55,15 @@ int main(int, char**)
     assert(p.second.get_u1() == 6u);
     assert(p.second.get_u2() == 2u);
 
-  return 0;
+    return true;
+}
+
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER >= 20
+    static_assert(test());
+#endif
+
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp
index 676c5fc1f05b32cbcebf7d3928e53d1607a3cf6f..c3cc7eb77d0bbaaede02fd181960916d78c4e929 100644
--- a/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp
@@ -348,6 +348,86 @@ void test_caller_accepts_nonconst() {
   std::visit(Visitor{}, v);
 }
 
+struct MyVariant : std::variant<short, long, float> {};
+
+namespace std {
+template <size_t Index>
+void get(const MyVariant&) {
+  assert(false);
+}
+} // namespace std
+
+void test_derived_from_variant() {
+  auto v1 = MyVariant{42};
+  const auto cv1 = MyVariant{142};
+  std::visit([](auto x) { assert(x == 42); }, v1);
+  std::visit([](auto x) { assert(x == 142); }, cv1);
+  std::visit([](auto x) { assert(x == -1.25f); }, MyVariant{-1.25f});
+  std::visit([](auto x) { assert(x == 42); }, std::move(v1));
+  std::visit([](auto x) { assert(x == 142); }, std::move(cv1));
+
+  // Check that visit does not take index nor valueless_by_exception members from the base class.
+  struct EvilVariantBase {
+    int index;
+    char valueless_by_exception;
+  };
+
+  struct EvilVariant1 : std::variant<int, long, double>,
+                        std::tuple<int>,
+                        EvilVariantBase {
+    using std::variant<int, long, double>::variant;
+  };
+
+  std::visit([](auto x) { assert(x == 12); }, EvilVariant1{12});
+  std::visit([](auto x) { assert(x == 12.3); }, EvilVariant1{12.3});
+
+  // Check that visit unambiguously picks the variant, even if the other base has __impl member.
+  struct ImplVariantBase {
+    struct Callable {
+      bool operator()();
+    };
+
+    Callable __impl;
+  };
+
+  struct EvilVariant2 : std::variant<int, long, double>, ImplVariantBase {
+    using std::variant<int, long, double>::variant;
+  };
+
+  std::visit([](auto x) { assert(x == 12); }, EvilVariant2{12});
+  std::visit([](auto x) { assert(x == 12.3); }, EvilVariant2{12.3});
+}
+
+struct any_visitor {
+  template <typename T>
+  void operator()(const T&) const {}
+};
+
+template <typename T, typename = decltype(std::visit(
+                          std::declval<any_visitor&>(), std::declval<T>()))>
+constexpr bool has_visit(int) {
+  return true;
+}
+
+template <typename T>
+constexpr bool has_visit(...) {
+  return false;
+}
+
+void test_sfinae() {
+  struct BadVariant : std::variant<short>, std::variant<long, float> {};
+  struct BadVariant2 : private std::variant<long, float> {};
+  struct GoodVariant : std::variant<long, float> {};
+  struct GoodVariant2 : GoodVariant {};
+
+  static_assert(!has_visit<int>(0));
+  static_assert(!has_visit<BadVariant>(0));
+  static_assert(!has_visit<BadVariant2>(0));
+  static_assert(has_visit<std::variant<int>>(0));
+  static_assert(has_visit<GoodVariant>(0));
+  static_assert(has_visit<GoodVariant2>(0));
+}
+
 int main(int, char**) {
   test_call_operator_forwarding();
   test_argument_forwarding();
@@ -355,6 +435,8 @@ int main(int, char**) {
   test_constexpr();
   test_exceptions();
   test_caller_accepts_nonconst();
+  test_derived_from_variant();
+  test_sfinae();
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp
index 7e569e2e9e42bd0ef1a4c2a30b868866a1352c10..459b75f717f517e41b044fc281ae7044c21aad3e 100644
--- a/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp
@@ -411,6 +411,99 @@ void test_constexpr_explicit_side_effect() {
   static_assert(test_lambda(202) == 202, "");
 }
 
+void test_derived_from_variant() {
+  struct MyVariant : std::variant<short, long, float> {};
+
+  std::visit<bool>(
+      [](auto x) {
+        assert(x == 42);
+        return true;
+      },
+      MyVariant{42});
+  std::visit<bool>(
+      [](auto x) {
+        assert(x == -1.3f);
+        return true;
+      },
+      MyVariant{-1.3f});
+
+  // Check that visit does not take index nor valueless_by_exception members from the base class.
+  struct EvilVariantBase {
+    int index;
+    char valueless_by_exception;
+  };
+
+  struct EvilVariant1 : std::variant<int, long, double>,
+                        std::tuple<int>,
+                        EvilVariantBase {
+    using std::variant<int, long, double>::variant;
+  };
+
+  std::visit<bool>(
+      [](auto x) {
+        assert(x == 12);
+        return true;
+      },
+      EvilVariant1{12});
+  std::visit<bool>(
+      [](auto x) {
+        assert(x == 12.3);
+        return true;
+      },
+      EvilVariant1{12.3});
+
+  // Check that visit unambiguously picks the variant, even if the other base has __impl member.
+  struct ImplVariantBase {
+    struct Callable {
+      bool operator()();
+    };
+
+    Callable __impl;
+  };
+
+  struct EvilVariant2 : std::variant<int, long, double>, ImplVariantBase {
+    using std::variant<int, long, double>::variant;
+  };
+
+  std::visit<bool>(
+      [](auto x) {
+        assert(x == 12);
+        return true;
+      },
+      EvilVariant2{12});
+  std::visit<bool>(
+      [](auto x) {
+        assert(x == 12.3);
+        return true;
+      },
+      EvilVariant2{12.3});
+}
+
+struct any_visitor {
+  template <typename T>
+  bool operator()(const T&) {
+    return true;
+  }
+};
+
+template <typename T, typename = decltype(std::visit<bool>(
+                          std::declval<any_visitor&>(), std::declval<T>()))>
+constexpr bool has_visit(int) {
+  return true;
+}
+
+template <typename T>
+constexpr bool has_visit(...) {
+  return false;
+}
+
+void test_sfinae() {
+  struct BadVariant : std::variant<short>, std::variant<long, float> {};
+
+  static_assert(has_visit<std::variant<int> >(int()));
+  static_assert(!has_visit<BadVariant>(int()));
+}
+
 int main(int, char**) {
   test_call_operator_forwarding<void>();
   test_argument_forwarding<void>();
@@ -425,6 +518,8 @@ int main(int, char**) {
   test_exceptions<int>();
   test_caller_accepts_nonconst<int>();
   test_constexpr_explicit_side_effect();
+  test_derived_from_variant();
+  test_sfinae();
 
   return 0;
 }
diff --git a/libcxx/test/support/filesystem_test_helper.h b/libcxx/test/support/filesystem_test_helper.h
index 32d2b6ab6a8636b28df2fbc8ea7c049951e7c00d..e87c43b9ea09bfc5107b1bc1d5ebbf12ba6bb54d 100644
--- a/libcxx/test/support/filesystem_test_helper.h
+++ b/libcxx/test/support/filesystem_test_helper.h
@@ -296,14 +296,17 @@ private:
     // sharing the same cwd). However, it is fairly unlikely to happen as
     // we generally don't use scoped_test_env from multiple threads, so
     // this is deemed acceptable.
+    // The cwd.filename() itself isn't unique across all tests in the suite,
+    // so start the numbering from a hash of the full cwd, to avoid
+    // different tests interfering with each other.
     static inline fs::path available_cwd_path() {
         fs::path const cwd = utils::getcwd();
         fs::path const tmp = fs::temp_directory_path();
-        fs::path const base = tmp / cwd.filename();
-        int i = 0;
-        fs::path p = base / ("static_env." + std::to_string(i));
+        std::string base = cwd.filename().string();
+        size_t i = std::hash<std::string>()(cwd.string());
+        fs::path p = tmp / (base + "-static_env." + std::to_string(i));
         while (utils::exists(p.string())) {
-            p = fs::path(base) / ("static_env." + std::to_string(++i));
+            p = tmp / (base + "-static_env." + std::to_string(++i));
         }
         return p;
     }
@@ -634,9 +637,7 @@ struct ExceptionChecker {
       additional_msg = opt_message + ": ";
     }
     auto transform_path = [](const fs::path& p) {
-      if (p.native().empty())
-        return std::string("\"\"");
-      return p.string();
+      return "\"" + p.string() + "\"";
     };
     std::string format = [&]() -> std::string {
       switch (num_paths) {
diff --git a/libcxx/test/support/test_constexpr_container.h b/libcxx/test/support/test_constexpr_container.h
new file mode 100644
index 0000000000000000000000000000000000000000..3f466d517cccb73fa6df6b8b5b1aa918ef80388a
--- /dev/null
+++ b/libcxx/test/support/test_constexpr_container.h
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef SUPPORT_TEST_CONSTEXPR_CONTAINER_H
+#define SUPPORT_TEST_CONSTEXPR_CONTAINER_H
+
+// A dummy container with enough constexpr support to test the standard
+// insert iterators, such as `back_insert_iterator`.
+
+#include <algorithm>
+#include <cassert>
+#include <utility>
+
+#include "test_macros.h"
+
+#if TEST_STD_VER >= 14
+
+template<class T, int N>
+class ConstexprFixedCapacityDeque {
+    T data_[N];
+    int size_ = 0;
+public:
+    using value_type = T;
+    using iterator = T *;
+    using const_iterator = T const *;
+
+    constexpr ConstexprFixedCapacityDeque() = default;
+    constexpr iterator begin() { return data_; }
+    constexpr iterator end() { return data_ + size_; }
+    constexpr const_iterator begin() const { return data_; }
+    constexpr const_iterator end() const { return data_ + size_; }
+    constexpr size_t size() const { return size_; }
+    constexpr const T& front() const { assert(size_ >= 1); return data_[0]; }
+    constexpr const T& back() const { assert(size_ >= 1); return data_[size_-1]; }
+
+    constexpr iterator insert(const_iterator pos, T t) {
+        int i = (pos - data_);
+        if (i != size_) {
+            std::move_backward(data_ + i, data_ + size_, data_ + size_ + 1);
+        }
+        data_[i] = std::move(t);
+        size_ += 1;
+        return data_ + i;
+    }
+
+    constexpr void push_back(T t) { insert(end(), std::move(t)); }
+    constexpr void push_front(T t) { insert(begin(), std::move(t)); }
+};
+
+#endif // TEST_STD_VER >= 14
+
+#endif // SUPPORT_TEST_CONSTEXPR_CONTAINER_H
diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index 3eff6780133a60b4155a2ea6718d360e8cd39aaf..dfc06fd56455f7355e64d50ae79f8589f12cad9a 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -111,6 +111,17 @@ steps:
         - exit_status: -1  # Agent was lost
           limit: 2
 
+  - label: "Static libraries"
+    command: "libcxx/utils/ci/run-buildbot generic-static"
+    artifact_paths:
+      - "**/test-results.xml"
+    agents:
+      queue: "libcxx-builders"
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
   - label: "GCC/C++20"
     command: "libcxx/utils/ci/run-buildbot generic-gcc"
     artifact_paths:
@@ -178,7 +189,7 @@ steps:
           limit: 2
 
   - label: "No debug mode"
-    command: "libcxx/utils/ci/run-buildbot generic-nodebug"
+    command: "libcxx/utils/ci/run-buildbot generic-no-debug"
     artifact_paths:
       - "**/test-results.xml"
     agents:
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index f76a669f9f1de6b008939b1df9e99053ab0dd053..38d9bc4de0cfe6a5143da34f7312c0daec7ac1f8 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -86,30 +86,30 @@ function generate-cmake() {
 }
 
 function check-cxx-cxxabi() {
+    echo "--- Installing libc++ and libc++abi to a fake location"
+    ${NINJA} -vC "${BUILD_DIR}" install-cxx install-cxxabi
+
     echo "+++ Running the libc++ tests"
-    ${NINJA} -C "${BUILD_DIR}" check-cxx
+    ${NINJA} -vC "${BUILD_DIR}" check-cxx
 
     echo "+++ Running the libc++abi tests"
-    ${NINJA} -C "${BUILD_DIR}" check-cxxabi
-
-    echo "--- Installing libc++ and libc++abi to a fake location"
-    ${NINJA} -C "${BUILD_DIR}" install-cxx install-cxxabi
+    ${NINJA} -vC "${BUILD_DIR}" check-cxxabi
 }
 
 # TODO: The goal is to test this against all configurations. We should also move
 #       this to the Lit test suite instead of being a separate CMake target.
 function check-abi-list() {
     echo "+++ Running the libc++ ABI list test"
-    ${NINJA} -C "${BUILD_DIR}" check-cxx-abilist || (
+    ${NINJA} -vC "${BUILD_DIR}" check-cxx-abilist || (
         echo "+++ Generating the libc++ ABI list after failed check"
-        ${NINJA} -C "${BUILD_DIR}" generate-cxx-abilist
+        ${NINJA} -vC "${BUILD_DIR}" generate-cxx-abilist
         false
     )
 }
 
 function check-cxx-benchmarks() {
     echo "--- Running the benchmarks"
-    ${NINJA} -C "${BUILD_DIR}" check-cxx-benchmarks
+    ${NINJA} -vC "${BUILD_DIR}" check-cxx-benchmarks
 }
 
 case "${BUILDER}" in
@@ -183,6 +183,13 @@ generic-noexceptions)
     generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-noexceptions.cmake"
     check-cxx-cxxabi
 ;;
+generic-static)
+    export CC=clang
+    export CXX=clang++
+    clean
+    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-static.cmake"
+    check-cxx-cxxabi
+;;
 generic-32bit)
     export CC=clang
     export CXX=clang++
@@ -194,9 +201,7 @@ generic-gcc)
     export CC=gcc
     export CXX=g++
     clean
-    # FIXME: Re-enable experimental testing on GCC. GCC cares about the order
-    #        in which we link -lc++experimental, which causes issues.
-    generate-cmake -DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF
+    generate-cmake
     check-cxx-cxxabi
 ;;
 generic-asan)
@@ -241,11 +246,11 @@ generic-singlethreaded)
     generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-singlethreaded.cmake"
     check-cxx-cxxabi
 ;;
-generic-nodebug)
+generic-no-debug)
     export CC=clang
     export CXX=clang++
     clean
-    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-nodebug.cmake"
+    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-no-debug.cmake"
     check-cxx-cxxabi
 ;;
 generic-no-filesystem)
@@ -330,7 +335,7 @@ documentation)
     generate-cmake -DLLVM_ENABLE_SPHINX=ON
 
     echo "+++ Generating documentation"
-    ${NINJA} -C "${BUILD_DIR}" docs-libcxx-html
+    ${NINJA} -vC "${BUILD_DIR}" docs-libcxx-html
 ;;
 unified-standalone)
     export CC=clang
@@ -376,16 +381,16 @@ legacy-standalone)
           -DLIBCXXABI_LIBCXX_LIBRARY_PATH="${BUILD_DIR}/libcxx/lib"
 
     echo "+++ Building libc++abi"
-    ${NINJA} -C "${BUILD_DIR}/libcxxabi" cxxabi
+    ${NINJA} -vC "${BUILD_DIR}/libcxxabi" cxxabi
 
     echo "+++ Building libc++"
-    ${NINJA} -C "${BUILD_DIR}/libcxx" cxx
+    ${NINJA} -vC "${BUILD_DIR}/libcxx" cxx
 
     echo "+++ Running the libc++ tests"
-    ${NINJA} -C "${BUILD_DIR}/libcxx" check-cxx
+    ${NINJA} -vC "${BUILD_DIR}/libcxx" check-cxx
 
     echo "+++ Running the libc++abi tests"
-    ${NINJA} -C "${BUILD_DIR}/libcxxabi" check-cxxabi
+    ${NINJA} -vC "${BUILD_DIR}/libcxxabi" check-cxxabi
 ;;
 aarch64)
     clean
@@ -402,35 +407,23 @@ aarch64-noexceptions)
 # Aka Armv8 32 bit
 armv8)
     clean
-    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv8.cmake" \
-    -DCMAKE_CXX_FLAGS="-marm" \
-    -DCMAKE_C_FLAGS="-marm"
+    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv8Arm.cmake"
     check-cxx-cxxabi
 ;;
 armv8-noexceptions)
     clean
-    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv8.cmake" \
-    -DCMAKE_CXX_FLAGS="-mthumb" \
-    -DCMAKE_C_FLAGS="-mthumb" \
-    -DLIBCXX_ENABLE_EXCEPTIONS=OFF \
-    -DLIBCXXABI_ENABLE_EXCEPTIONS=OFF
+    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv8Thumb-noexceptions.cmake"
     check-cxx-cxxabi
 ;;
 # Armv7 32 bit. One building Arm only one Thumb only code.
 armv7)
     clean
-    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv7.cmake" \
-    -DCMAKE_CXX_FLAGS="-marm" \
-    -DCMAKE_C_FLAGS="-marm"
+    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv7Arm.cmake"
     check-cxx-cxxabi
 ;;
 armv7-noexceptions)
     clean
-    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv7.cmake" \
-    -DCMAKE_CXX_FLAGS="-mthumb" \
-    -DCMAKE_C_FLAGS="-mthumb" \
-    -DLIBCXX_ENABLE_EXCEPTIONS=OFF \
-    -DLIBCXXABI_ENABLE_EXCEPTIONS=OFF
+    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Armv7Thumb-noexceptions.cmake"
     check-cxx-cxxabi
 ;;
 *)
diff --git a/libcxx/utils/docker/README.txt b/libcxx/utils/docker/README.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/libcxx/utils/docker/debian9/buildbot/Dockerfile b/libcxx/utils/docker/debian9/buildbot/Dockerfile
deleted file mode 100644
index 7da50687b9527bb6414ea7d32756a33701e7f6c3..0000000000000000000000000000000000000000
--- a/libcxx/utils/docker/debian9/buildbot/Dockerfile
+++ /dev/null
@@ -1,39 +0,0 @@
-
-#===-------------------------------------------------------------------------------------------===//
-# buildslave
-#===-------------------------------------------------------------------------------------------===//
-ARG gcc_tot
-ARG llvm_tot
-
-FROM ${gcc_tot} AS gcc-tot
-FROM ${llvm_tot} AS llvm-tot
-
-FROM debian:stretch AS base-image
-
-ADD install-packages.sh /tmp/
-RUN /tmp/install-packages.sh && rm /tmp/install-packages.sh
-
-COPY --from=ericwf/gcc:5.5.0 /compiler /opt/gcc-5
-
-FROM base-image as worker-image
-
-COPY --from=gcc-tot /compiler /opt/gcc-tot
-COPY --from=llvm-tot /compiler /opt/llvm-tot
-
-ENV PATH /opt/llvm-tot/bin:$PATH
-
-RUN clang++ --version && echo hello
-RUN g++ --version
-
-
-RUN /opt/gcc-tot/bin/g++ --version
-RUN /opt/llvm-tot/bin/clang++ --version
-RUN /opt/llvm-tot/bin/clang --version
-
-# FIXME(EricWF): remove this once the buildbot's config doesn't clobber the path.
-RUN ln -s /opt/llvm-tot/bin/clang /usr/local/bin/clang
-RUN ln -s /opt/llvm-tot/bin/clang++ /usr/local/bin/clang++
-
-
-ADD run_buildbot.sh /
-CMD /run_buildbot.sh /run/secrets/buildbot-auth
diff --git a/libcxx/utils/docker/debian9/buildbot/buildbot-auth.json b/libcxx/utils/docker/debian9/buildbot/buildbot-auth.json
deleted file mode 100644
index 5e91e2d4158faf1cb2ff9d12a71beaa177031582..0000000000000000000000000000000000000000
--- a/libcxx/utils/docker/debian9/buildbot/buildbot-auth.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "login": "<login>",
-  "password": "<password>"
-}
diff --git a/libcxx/utils/docker/debian9/buildbot/docker-compose.yml b/libcxx/utils/docker/debian9/buildbot/docker-compose.yml
deleted file mode 100644
index bd61dea4871c6900c52205b15b58d474535fed31..0000000000000000000000000000000000000000
--- a/libcxx/utils/docker/debian9/buildbot/docker-compose.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-version: '3.7'
-services:
-  llvm-buildbot-worker:
-    build:
-      context: https://github.com/llvm/llvm-project.git#master:libcxx/utils/docker/debian9/buildbot
-      args:
-        gcc_tot: "ericwf/gcc:9.2.0"
-        llvm_tot: "ericwf/llvm:11.x"
-    image: llvm-buildbot-worker
-    volumes:
-    - /var/run/docker.sock:/var/run/docker.sock
-    secrets:
-    - buildbot-auth
-    logging:
-      driver: gcplogs
-
-secrets:
-  buildbot-auth:
-    file: buildbot-auth.json
diff --git a/libcxx/utils/docker/debian9/buildbot/install-gcloud-agents.sh b/libcxx/utils/docker/debian9/buildbot/install-gcloud-agents.sh
deleted file mode 100755
index d2656ca5092a04c056c488fdf5884a51f1cc6305..0000000000000000000000000000000000000000
--- a/libcxx/utils/docker/debian9/buildbot/install-gcloud-agents.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-cd /tmp/
-
-curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh
-sudo bash install-monitoring-agent.sh
-rm install-monitoring-agent.sh
-
-curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
-sudo bash install-logging-agent.sh
-rm install-logging-agent.sh
diff --git a/libcxx/utils/docker/debian9/buildbot/install-packages.sh b/libcxx/utils/docker/debian9/buildbot/install-packages.sh
deleted file mode 100755
index 56e7c00d493018b688fa671891aabf7d4666dc1b..0000000000000000000000000000000000000000
--- a/libcxx/utils/docker/debian9/buildbot/install-packages.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-
-set -x
-set -e
-
-apt-get update && \
-    apt-get install -y --no-install-recommends \
-      buildbot-slave \
-      ca-certificates \
-      gnupg \
-      build-essential \
-      wget \
-      unzip \
-      python \
-      ninja-build \
-      curl \
-      git \
-      gcc-multilib \
-      g++-multilib \
-      libc6-dev \
-      libtool \
-      locales-all \
-      binutils-dev \
-      binutils-gold \
-      software-properties-common \
-      gnupg \
-      apt-transport-https \
-      sudo \
-      bash-completion \
-      vim \
-      jq \
-      systemd \
-      sysvinit-utils \
-      systemd-sysv && \
-  rm -rf /var/lib/apt/lists/*
-
-# Install a recent CMake
-yes | apt-get purge cmake
-wget https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2-Linux-x86_64.sh -O /tmp/install-cmake.sh
-bash /tmp/install-cmake.sh --prefix=/usr --exclude-subdir --skip-license
diff --git a/libcxx/utils/docker/debian9/buildbot/run_buildbot.sh b/libcxx/utils/docker/debian9/buildbot/run_buildbot.sh
deleted file mode 100755
index e008a30558c9fb32241f7cb68dcb621615155917..0000000000000000000000000000000000000000
--- a/libcxx/utils/docker/debian9/buildbot/run_buildbot.sh
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/usr/bin/env bash
-set -x
-
-readonly BOT_ROOT=/b
-readonly AUTH_FILE=$1
-readonly BOT_ROOT_NAME=$(jq -r ".login" $AUTH_FILE)
-
-systemctl daemon-reload
-service buildslave stop
-mkdir -p /b
-rm -rf /b/*
-service buildslave stop
-
-pushd /tmp/
-
-curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh
-sudo bash install-monitoring-agent.sh
-rm install-monitoring-agent.sh
-
-curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
-sudo bash install-logging-agent.sh
-rm install-logging-agent.sh
-
-popd
-
-
-systemctl set-property buildslave.service TasksMax=100000
-
-function setup_numbered_bot() {
-  local BOT_NAME=$1
-  local BOT_DIR=$2
-  mkdir -p $BOT_DIR
-
-  buildslave stop $BOT_DIR
-  chown buildbot $BOT_DIR
-  rm -rf $BOT_DIR/*
-
-  buildslave create-slave --allow-shutdown=signal "$BOT_DIR" "lab.llvm.org:9990" "$BOT_NAME" $(jq -r ".password" $AUTH_FILE)
-
-  echo "Eric Fiselier <ericwf@google.com>" > $BOT_DIR/info/admin
-
-  echo "Connecting as $1"
-  {
-    uname -a | head -n1
-    cmake --version | head -n1
-    g++ --version | head -n1
-    clang++ --version | head -n1
-    ld --version | head -n1
-    date
-    lscpu
-  } > $BOT_DIR/info/host
-
-
-#echo "SLAVE_RUNNER=/usr/bin/buildslave
-#SLAVE_ENABLED[1]=\"1\"
-#SLAVE_NAME[1]=\"$BOT_NAME\"
-#SLAVE_USER[1]=\"buildbot\"
-#SLAVE_BASEDIR[1]=\"$BOT_DIR\"
-#SLAVE_OPTIONS[1]=\"\"
-#SLAVE_PREFIXCMD[1]=\"\"" > $BOT_DIR/buildslave.cfg
-
-  ls $BOT_DIR/
-  cat $BOT_DIR/buildbot.tac
-}
-
-function try_start_builder {
-  local N=$1
-  local BOT_DIR="$BOT_ROOT/b$N"
-  local BOT_NAME="$BOT_ROOT_NAME$N"
-
-  systemctl daemon-reload
-  service buildslave restart
-  setup_numbered_bot "$BOT_NAME" "$BOT_DIR"
-
-  systemctl daemon-reload
-  service buildslave restart
-
-  chown -R buildbot $BOT_DIR/
-  sudo -u buildbot /usr/bin/buildslave start $BOT_DIR/
-
-  sleep 30
-  cat $BOT_DIR/twistd.log
-  if grep --quiet "slave is ready" $BOT_DIR/twistd.log; then
-    return 0
-  fi
-  if grep --quiet "configuration update complete" $BOT_DIR/twistd.log; then
-    return 0
-  fi
-  if grep "rejecting duplicate slave" $BOT_DIR/twistd.log; then
-    return 1
-  fi
-  echo "Unknown error"
-  cat $BOT_DIR/twistd.log
-  exit 1
-}
-
-for N in `shuf -i 1-5`
-do
-  if try_start_builder $N; then
-    break
-  fi
-  echo "failed to start any buildbot"
-  shutdown now
-done
-
-# GCE can restart instance after 24h in the middle of the build.
-# Gracefully restart before that happen.
-sleep 72000
-while pkill -SIGHUP buildslave; do sleep 5; done;
-shutdown now
-
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index 7351da3b2a4d64aff918e6eb47fbc3adbd732812..c3db978f05f7807bd5c58cfe729db8d13d61d567 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -198,7 +198,6 @@ feature_test_macros = [ add_version_header(x) for x in [
     "name": "__cpp_lib_constexpr_iterator",
     "values": { "c++20": 201811 },
     "headers": ["iterator"],
-    "unimplemented": True,
   }, {
     "name": "__cpp_lib_constexpr_memory",
     "values": { "c++20": 201811 },
@@ -209,19 +208,16 @@ feature_test_macros = [ add_version_header(x) for x in [
     "headers": ["numeric"],
   }, {
     "name": "__cpp_lib_constexpr_string",
-    "values": { "c++20": 201907 },
+    "values": { "c++20": 201811 },  # because P1032R1 is implemented; but should become 201907 after P0980R1
     "headers": ["string"],
-    "unimplemented": True,
   }, {
     "name": "__cpp_lib_constexpr_string_view",
     "values": { "c++20": 201811 },
     "headers": ["string_view"],
-    "unimplemented": True,
   }, {
     "name": "__cpp_lib_constexpr_tuple",
     "values": { "c++20": 201811 },
     "headers": ["tuple"],
-    "unimplemented": True,
   }, {
     "name": "__cpp_lib_constexpr_utility",
     "values": { "c++20": 201811 },
@@ -422,7 +418,7 @@ feature_test_macros = [ add_version_header(x) for x in [
     "values": { "c++20": 201907 },
     "headers": ["numbers"],
     "depends": "defined(__cpp_concepts) && __cpp_concepts >= 201907L",
-    "internal_depends": "defined(__cpp_concepts) && __cpp_concepts >= 201907L",
+    "internal_depends": "!defined(_LIBCPP_HAS_NO_CONCEPTS)",
   }, {
     "name": "__cpp_lib_math_special_functions",
     "values": { "c++17": 201603 },
@@ -631,7 +627,7 @@ feature_test_macros = [ add_version_header(x) for x in [
     "headers": ["functional"],
   }, {
     "name": "__cpp_lib_variant",
-    "values": { "c++17": 201606 },
+    "values": { "c++17": 202102 },
     "headers": ["variant"],
   }, {
     "name": "__cpp_lib_void_t",
@@ -720,7 +716,7 @@ def get_std_number(std):
 
 def produce_macros_definition_for_std(std):
   result = ""
-  indent = 56
+  indent = 55
   for tc in feature_test_macros:
     if std not in tc["values"]:
       continue
@@ -734,7 +730,7 @@ def produce_macros_definition_for_std(std):
       result += "# undef  %s\n" % tc["name"]
     line = "#%sdefine %s" % ((" " * inner_indent), tc["name"])
     line += " " * (indent - len(line))
-    line += "%sL" % tc["values"][std]
+    line += " %sL" % tc["values"][std]
     if 'unimplemented' in tc.keys():
       line = "// " + line
     result += line
diff --git a/libcxx/utils/graph_header_deps.py b/libcxx/utils/graph_header_deps.py
index b6f0a250ccef730aaccd8e20d4463cf0cea116ce..8a4840383dc95d18d5465e393aad128f75fb7818 100755
--- a/libcxx/utils/graph_header_deps.py
+++ b/libcxx/utils/graph_header_deps.py
@@ -7,202 +7,205 @@
 #
 #===----------------------------------------------------------------------===##
 
-from argparse import ArgumentParser
+import argparse
 import os
-import shutil
-import sys
-import shlex
-import json
 import re
-import libcxx.graph as dot
-import libcxx.util
-
-def print_and_exit(msg):
-    sys.stderr.write(msg + '\n')
-    sys.exit(1)
-
-def libcxx_include_path():
-    curr_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-    include_dir = os.path.join(curr_dir, 'include')
-    return include_dir
-
-def get_libcxx_headers():
-    headers = []
-    include_dir = libcxx_include_path()
-    for fname in os.listdir(include_dir):
-        f = os.path.join(include_dir, fname)
-        if not os.path.isfile(f):
-            continue
-        base, ext = os.path.splitext(fname)
-        if (ext == '' or ext == '.h') and (not fname.startswith('__') or fname == '__config'):
-            headers += [f]
-    return headers
-
-
-def rename_headers_and_remove_test_root(graph):
-    inc_root = libcxx_include_path()
-    to_remove = set()
-    for n in graph.nodes:
-        assert 'label' in n.attributes
-        l = n.attributes['label']
-        if not l.startswith('/') and os.path.exists(os.path.join('/', l)):
-            l = '/' + l
-        if l.endswith('.tmp.cpp'):
-            to_remove.add(n)
-        if l.startswith(inc_root):
-            l = l[len(inc_root):]
-            if l.startswith('/'):
-                l = l[1:]
-        n.attributes['label'] = l
-    for n in to_remove:
-        graph.removeNode(n)
-
-def remove_non_std_headers(graph):
-    inc_root = libcxx_include_path()
-    to_remove = set()
-    for n in graph.nodes:
-        test_file = os.path.join(inc_root, n.attributes['label'])
-        if not test_file.startswith(inc_root):
-            to_remove.add(n)
-    for xn in to_remove:
-        graph.removeNode(xn)
-
-class DependencyCommand(object):
-    def __init__(self, compile_commands, output_dir, new_std=None):
-        output_dir = os.path.abspath(output_dir)
-        if not os.path.isdir(output_dir):
-            print_and_exit('"%s" must point to a directory' % output_dir)
-        self.output_dir = output_dir
-        self.new_std = new_std
-        cwd,bcmd =  self._get_base_command(compile_commands)
-        self.cwd = cwd
-        self.base_cmd = bcmd
-
-    def run_for_headers(self, header_list):
-        outputs = []
-        for header in header_list:
-            header_name = os.path.basename(header)
-            out = os.path.join(self.output_dir, ('%s.dot' % header_name))
-            outputs += [out]
-            cmd =  self.base_cmd + ["-fsyntax-only", "-Xclang", "-dependency-dot", "-Xclang", "%s" % out, '-xc++', '-']
-            libcxx.util.executeCommandOrDie(cmd, cwd=self.cwd, input='#include <%s>\n\n' % header_name)
-        return outputs
-
-    def _get_base_command(self, command_file):
-        commands = None
-        with open(command_file, 'r') as f:
-            commands = json.load(f)
-        for compile_cmd in commands:
-            file = compile_cmd['file']
-            if not file.endswith('src/algorithm.cpp'):
-                continue
-            wd = compile_cmd['directory']
-            cmd_str = compile_cmd['command']
-            cmd = shlex.split(cmd_str)
-            out_arg = cmd.index('-o')
-            del cmd[out_arg]
-            del cmd[out_arg]
-            in_arg = cmd.index('-c')
-            del cmd[in_arg]
-            del cmd[in_arg]
-            if self.new_std is not None:
-                for f in cmd:
-                    if f.startswith('-std='):
-                        del cmd[cmd.index(f)]
-                        cmd += [self.new_std]
-                        break
-            return wd, cmd
-        print_and_exit("failed to find command to build algorithm.cpp")
-
-def post_process_outputs(outputs, libcxx_only):
-    graphs = []
-    for dot_file in outputs:
-        g = dot.DirectedGraph.fromDotFile(dot_file)
-        rename_headers_and_remove_test_root(g)
-        if libcxx_only:
-            remove_non_std_headers(g)
-        graphs += [g]
-        g.toDotFile(dot_file)
-    return graphs
-
-def build_canonical_names(graphs):
-    canonical_names = {}
-    next_idx = 0
-    for g in graphs:
-        for n in g.nodes:
-            if n.attributes['label'] not in canonical_names:
-                name = 'header_%d' % next_idx
-                next_idx += 1
-                canonical_names[n.attributes['label']] = name
-    return canonical_names
-
-
-
-class CanonicalGraphBuilder(object):
-    def __init__(self, graphs):
-        self.graphs = list(graphs)
-        self.canonical_names = build_canonical_names(graphs)
-
-    def build(self):
-        self.canonical = dot.DirectedGraph('all_headers')
-        for k,v in self.canonical_names.iteritems():
-            n = dot.Node(v, edges=[], attributes={'shape': 'box', 'label': k})
-            self.canonical.addNode(n)
-        for g in self.graphs:
-            self._merge_graph(g)
-        return self.canonical
-
-    def _merge_graph(self, g):
-        for n in g.nodes:
-            new_name = self.canonical.getNodeByLabel(n.attributes['label']).id
-            for e in n.edges:
-                to_node = self.canonical.getNodeByLabel(e.attributes['label']).id
-                self.canonical.addEdge(new_name, to_node)
-
-
-def main():
-    parser = ArgumentParser(
-        description="Generate a graph of libc++ header dependencies")
-    parser.add_argument(
-        '-v', '--verbose', dest='verbose', action='store_true', default=False)
-    parser.add_argument(
-        '-o', '--output', dest='output', required=True,
-        help='The output file. stdout is used if not given',
-        type=str, action='store')
-    parser.add_argument(
-        '--no-compile', dest='no_compile', action='store_true', default=False)
-    parser.add_argument(
-        '--libcxx-only', dest='libcxx_only', action='store_true', default=False)
-    parser.add_argument(
-        'compile_commands', metavar='compile-commands-file',
-        help='the compile commands database')
-
-    args = parser.parse_args()
-    builder = DependencyCommand(args.compile_commands, args.output, new_std='-std=c++2a')
-    if not args.no_compile:
-        outputs = builder.run_for_headers(get_libcxx_headers())
-        graphs = post_process_outputs(outputs, args.libcxx_only)
-    else:
-        outputs = [os.path.join(args.output, l) for l in os.listdir(args.output) if not l.endswith('all_headers.dot')]
-        graphs = [dot.DirectedGraph.fromDotFile(o) for o in outputs]
-
-    canon = CanonicalGraphBuilder(graphs).build()
-    canon.toDotFile(os.path.join(args.output, 'all_headers.dot'))
-    all_graphs = graphs + [canon]
 
-    found_cycles = False
-    for g in all_graphs:
-        cycle_finder = dot.CycleFinder(g)
-        all_cycles = cycle_finder.findCyclesInGraph()
-        if len(all_cycles):
-            found_cycles = True
-            print("cycle in graph %s" % g.name)
-            for start, path in all_cycles:
-                print("Cycle for %s = %s" % (start, path))
-    if not found_cycles:
-        print("No cycles found")
 
+def is_config_header(h):
+    return os.path.basename(h) in ['__config', '__libcpp_version']
+
+
+def is_experimental_header(h):
+    return ('experimental/' in h) or ('ext/' in h)
+
+
+def is_support_header(h):
+    return '__support/' in h
+
+
+class FileEntry:
+    def __init__(self, includes, individual_linecount):
+        self.includes = includes
+        self.individual_linecount = individual_linecount
+        self.cumulative_linecount = None  # documentation: this gets filled in later
+        self.is_graph_root = None  # documentation: this gets filled in later
+
+
+def list_all_roots_under(root):
+    result = []
+    for root, _, files in os.walk(root):
+        for fname in files:
+            if '__support' in root:
+                pass
+            elif ('.' in fname and not fname.endswith('.h')):
+                pass
+            else:
+                result.append(root + '/' + fname)
+    return result
+
+
+def build_file_entry(fname, options):
+    assert os.path.exists(fname)
+
+    def locate_header_file(h, paths):
+        for p in paths:
+            fullname = p + '/' + h
+            if os.path.exists(fullname):
+                return fullname
+        if options.error_on_file_not_found:
+            raise RuntimeError('Header not found: %s, included by %s' % (h, fname))
+        return None
+
+    local_includes = []
+    system_includes = []
+    linecount = 0
+    with open(fname, 'r') as f:
+        for line in f.readlines():
+            linecount += 1
+            m = re.match(r'\s*#\s*include\s+"([^"]*)"', line)
+            if m is not None:
+                local_includes.append(m.group(1))
+            m = re.match(r'\s*#\s*include\s+<([^>]*)>', line)
+            if m is not None:
+                system_includes.append(m.group(1))
+
+    fully_qualified_includes = [
+        locate_header_file(h, options.search_dirs)
+        for h in system_includes
+    ] + [
+        locate_header_file(h, os.path.dirname(fname))
+        for h in local_includes
+    ]
+
+    return FileEntry(
+        # If file-not-found wasn't an error, then skip non-found files
+        includes = [h for h in fully_qualified_includes if h is not None],
+        individual_linecount = linecount,
+    )
+
+
+def transitive_closure_of_includes(graph, h1):
+    visited = set()
+    def explore(graph, h1):
+        if h1 not in visited:
+            visited.add(h1)
+            for h2 in graph[h1].includes:
+                explore(graph, h2)
+    explore(graph, h1)
+    return visited
+
+
+def transitively_includes(graph, h1, h2):
+    return (h1 != h2) and (h2 in transitive_closure_of_includes(graph, h1))
+
+
+def build_graph(roots, options):
+    original_roots = list(roots)
+    graph = {}
+    while roots:
+        frontier = roots
+        roots = []
+        for fname in frontier:
+            if fname not in graph:
+                graph[fname] = build_file_entry(fname, options)
+                graph[fname].is_graph_root = (fname in original_roots)
+                roots += graph[fname].includes
+    for fname, entry in graph.items():
+        entry.cumulative_linecount = sum(graph[h].individual_linecount for h in transitive_closure_of_includes(graph, fname))
+    return graph
+
+
+def get_graphviz(graph, options):
+
+    def get_friendly_id(fname):
+        i = fname.index('include/')
+        assert(i >= 0)
+        result = fname[i+8:]
+        return result
+
+    def get_decorators(fname, entry):
+        result = ''
+        if entry.is_graph_root:
+            result += ' [style=bold]'
+        if options.show_individual_line_counts and options.show_cumulative_line_counts:
+            result += ' [label="%s\\n%d indiv, %d cumul"]' % (
+                get_friendly_id(fname), entry.individual_linecount, entry.cumulative_linecount
+            )
+        elif options.show_individual_line_counts:
+            result += ' [label="%s\\n%d indiv"]' % (get_friendly_id(fname), entry.individual_linecount)
+        elif options.show_cumulative_line_counts:
+            result += ' [label="%s\\n%d cumul"]' % (get_friendly_id(fname), entry.cumulative_linecount)
+        return result
+
+    result = ''
+    result += 'strict digraph {\n'
+    result += '    rankdir=LR;\n'
+    result += '    layout=dot;\n\n'
+    for fname, entry in graph.items():
+        result += '    "%s"%s;\n' % (get_friendly_id(fname), get_decorators(fname, entry))
+        for h in entry.includes:
+            if any(transitively_includes(graph, i, h) for i in entry.includes) and not options.show_transitive_edges:
+                continue
+            result += '        "%s" -> "%s";\n' % (get_friendly_id(fname), get_friendly_id(h))
+    result += '}\n'
+    return result
 
 
 if __name__ == '__main__':
-    main()
+    parser = argparse.ArgumentParser(description='Produce a dependency graph of libc++ headers, in GraphViz dot format.')
+    parser.add_argument('--root', default=None, metavar='FILE', help='File or directory to be the root of the dependency graph')
+    parser.add_argument('-I', dest='search_dirs', default=[], action='append', metavar='DIR', help='Path(s) to search for local includes')
+    parser.add_argument('--show-transitive-edges', action='store_true', help='Show edges to headers that are transitively included anyway')
+    parser.add_argument('--show-config-headers', action='store_true', help='Show headers named __config')
+    parser.add_argument('--show-experimental-headers', action='store_true', help='Show headers in the experimental/ and ext/ directories')
+    parser.add_argument('--show-support-headers', action='store_true', help='Show headers in the __support/ directory')
+    parser.add_argument('--show-individual-line-counts', action='store_true', help='Include an individual line count in each node')
+    parser.add_argument('--show-cumulative-line-counts', action='store_true', help='Include a total line count in each node')
+    parser.add_argument('--error-on-file-not-found', action='store_true', help="Don't ignore failure to open an #included file")
+
+    options = parser.parse_args()
+
+    if options.root is None:
+        curr_dir = os.path.dirname(os.path.abspath(__file__))
+        options.root = os.path.join(curr_dir, '../include')
+
+    if options.search_dirs == [] and os.path.isdir(options.root):
+        options.search_dirs = [options.root]
+
+    options.root = os.path.abspath(options.root)
+    options.search_dirs = [os.path.abspath(p) for p in options.search_dirs]
+
+    if os.path.isdir(options.root):
+        roots = list_all_roots_under(options.root)
+    elif os.path.isfile(options.root):
+        roots = [options.root]
+    else:
+        raise RuntimeError('--root seems to be invalid')
+
+    graph = build_graph(roots, options)
+
+    # Eliminate certain kinds of "visual noise" headers, if asked for.
+    def should_keep(fname):
+        return all([
+            options.show_config_headers or not is_config_header(fname),
+            options.show_experimental_headers or not is_experimental_header(fname),
+            options.show_support_headers or not is_support_header(fname),
+        ])
+
+    for fname in list(graph.keys()):
+        if should_keep(fname):
+            graph[fname].includes = [h for h in graph[fname].includes if should_keep(h)]
+        else:
+            del graph[fname]
+
+    # Look for cycles.
+    no_cycles_detected = True
+    for fname, entry in graph.items():
+        for h in entry.includes:
+            if transitively_includes(graph, h, fname):
+                print('Cycle detected between %s and %s' % (fname, h))
+                no_cycles_detected = False
+    assert no_cycles_detected
+
+    print(get_graphviz(graph, options))
diff --git a/libcxx/utils/libcxx/graph.py b/libcxx/utils/libcxx/graph.py
deleted file mode 100644
index 681d3ad2568f86d20b9dee1c79123bf9b86e7bc8..0000000000000000000000000000000000000000
--- a/libcxx/utils/libcxx/graph.py
+++ /dev/null
@@ -1,298 +0,0 @@
-#===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===----------------------------------------------------------------------===##
-
-import platform
-import os
-from collections import defaultdict
-import re
-import libcxx.util
-
-
-class DotEmitter(object):
-  def __init__(self, name):
-    self.name = name
-    self.node_strings = {}
-    self.edge_strings = []
-
-  def addNode(self, node):
-    res = str(node.id)
-    if len(node.attributes):
-      attr_strs = []
-      for k,v in node.attributes.iteritems():
-        attr_strs += ['%s="%s"' % (k, v)]
-      res += ' [ %s ]' % (', '.join(attr_strs))
-    res += ';'
-    assert node.id not in self.node_strings
-    self.node_strings[node.id] = res
-
-  def addEdge(self, n1, n2):
-    res = '%s -> %s;' % (n1.id, n2.id)
-    self.edge_strings += [res]
-
-  def node_key(self, n):
-    id = n.id
-    assert id.startswith('\w*\d+')
-
-  def emit(self):
-    node_definitions_list = []
-    sorted_keys = self.node_strings.keys()
-    sorted_keys.sort()
-    for k in sorted_keys:
-      node_definitions_list += [self.node_strings[k]]
-    node_definitions = '\n  '.join(node_definitions_list)
-    edge_list = '\n  '.join(self.edge_strings)
-    return '''
-digraph "{name}" {{
-  {node_definitions}
-  {edge_list}
-}}    
-'''.format(name=self.name, node_definitions=node_definitions, edge_list=edge_list).strip()
-
-
-class DotReader(object):
-  def __init__(self):
-    self.graph = DirectedGraph(None)
-
-  def abortParse(self, msg="bad input"):
-    raise Exception(msg)
-
-  def parse(self, data):
-    lines = [l.strip() for l in data.splitlines() if l.strip()]
-    maxIdx = len(lines)
-    idx = 0
-    if not self.parseIntroducer(lines[idx]):
-      self.abortParse('failed to parse introducer')
-    idx += 1
-    while idx < maxIdx:
-      if self.parseNodeDefinition(lines[idx]) or self.parseEdgeDefinition(lines[idx]):
-        idx += 1
-        continue
-      else:
-        break
-    if idx == maxIdx or not self.parseCloser(lines[idx]):
-      self.abortParse("no closing } found")
-    return self.graph
-
-  def parseEdgeDefinition(self, l):
-    edge_re = re.compile('^\s*(\w+)\s+->\s+(\w+);\s*$')
-    m = edge_re.match(l)
-    if not m:
-      return False
-    n1 = m.group(1)
-    n2 = m.group(2)
-    self.graph.addEdge(n1, n2)
-    return True
-
-  def parseAttributes(self, raw_str):
-    attribute_re = re.compile('^\s*(\w+)="([^"]+)"')
-    parts = [l.strip() for l in raw_str.split(',') if l.strip()]
-    attribute_dict = {}
-    for a in parts:
-      m = attribute_re.match(a)
-      if not m:
-        self.abortParse('Bad attribute "%s"' % a)
-      attribute_dict[m.group(1)] = m.group(2)
-    return attribute_dict
-
-  def parseNodeDefinition(self, l):
-    node_definition_re = re.compile('^\s*(\w+)\s+\[([^\]]+)\]\s*;\s*$')
-    m = node_definition_re.match(l)
-    if not m:
-      return False
-    id = m.group(1)
-    attributes = self.parseAttributes(m.group(2))
-    n = Node(id, edges=[], attributes=attributes)
-    self.graph.addNode(n)
-    return True
-
-  def parseIntroducer(self, l):
-    introducer_re = re.compile('^\s*digraph "([^"]+)"\s+{\s*$')
-    m = introducer_re.match(l)
-    if not m:
-      return False
-    self.graph.setName(m.group(1))
-    return True
-
-  def parseCloser(self, l):
-    closer_re = re.compile('^\s*}\s*$')
-    m = closer_re.match(l)
-    if not m:
-      return False
-    return True
-
-class Node(object):
-  def __init__(self, id, edges=[], attributes={}):
-    self.id = id
-    self.edges = set(edges)
-    self.attributes = dict(attributes)
-
-  def addEdge(self, dest):
-    self.edges.add(dest)
-
-  def __eq__(self, another):
-    if isinstance(another, str):
-      return another == self.id
-    return hasattr(another, 'id') and self.id == another.id
-
-  def __hash__(self):
-    return hash(self.id)
-
-  def __str__(self):
-    return self.attributes["label"]
-
-  def __repr__(self):
-    return self.__str__()
-    res = self.id
-    if len(self.attributes):
-      attr = []
-      for k,v in self.attributes.iteritems():
-        attr += ['%s="%s"' % (k, v)]
-      res += ' [%s ]' % (', '.join(attr))
-    return res
-
-class DirectedGraph(object):
-  def __init__(self, name=None, nodes=None):
-    self.name = name
-    self.nodes = set() if nodes is None else set(nodes)
-
-  def setName(self, n):
-    self.name = n
-
-  def _getNode(self, n_or_id):
-    if isinstance(n_or_id, Node):
-      return n_or_id
-    return self.getNode(n_or_id)
-
-  def getNode(self, str_id):
-    assert isinstance(str_id, str) or isinstance(str_id, Node)
-    for s in self.nodes:
-      if s == str_id:
-        return s
-    return None
-
-  def getNodeByLabel(self, l):
-    found = None
-    for s in self.nodes:
-      if s.attributes['label'] == l:
-        assert found is None
-        found = s
-    return found
-
-  def addEdge(self, n1, n2):
-    n1 = self._getNode(n1)
-    n2 = self._getNode(n2)
-    assert n1 in self.nodes
-    assert n2 in self.nodes
-    n1.addEdge(n2)
-
-  def addNode(self, n):
-    self.nodes.add(n)
-
-  def removeNode(self, n):
-    n = self._getNode(n)
-    for other_n in self.nodes:
-      if other_n == n:
-        continue
-      new_edges = set()
-      for e in other_n.edges:
-        if e != n:
-          new_edges.add(e)
-      other_n.edges = new_edges
-    self.nodes.remove(n)
-
-  def toDot(self):
-    dot = DotEmitter(self.name)
-    for n in self.nodes:
-      dot.addNode(n)
-      for ndest in n.edges:
-        dot.addEdge(n, ndest)
-    return dot.emit()
-
-  @staticmethod
-  def fromDot(str):
-    reader = DotReader()
-    graph = reader.parse(str)
-    return graph
-
-  @staticmethod
-  def fromDotFile(fname):
-    with open(fname, 'r') as f:
-      return DirectedGraph.fromDot(f.read())
-
-  def toDotFile(self, fname):
-    with open(fname, 'w') as f:
-      f.write(self.toDot())
-
-  def __repr__(self):
-    return self.toDot()
-
-class BFS(object):
-  def __init__(self, start):
-    self.visited = set()
-    self.to_visit = []
-    self.start = start
-
-  def __nonzero__(self):
-    return len(self.to_visit) != 0
-
-  def empty(self):
-    return len(self.to_visit) == 0
-
-  def push_back(self, node):
-    assert node not in self.visited
-    self.visited.add(node)
-    self.to_visit += [node]
-
-  def maybe_push_back(self, node):
-    if node in self.visited:
-      return
-    self.push_back(node)
-
-  def pop_front(self):
-    assert len(self.to_visit)
-    elem = self.to_visit[0]
-    del self.to_visit[0]
-    return elem
-
-  def seen(self, n):
-    return n in self.visited
-
-
-
-class CycleFinder(object):
-  def __init__(self, graph):
-    self.graph = graph
-
-  def findCycleForNode(self, n):
-    assert n in self.graph.nodes
-    all_paths = {}
-    all_cycles = []
-    bfs = BFS(n)
-    bfs.push_back(n)
-    all_paths[n] = [n]
-    while bfs:
-      n = bfs.pop_front()
-      assert n in all_paths
-      for e in n.edges:
-        en = self.graph.getNode(e)
-        if not bfs.seen(en):
-          new_path = list(all_paths[n])
-          new_path.extend([en])
-          all_paths[en] = new_path
-          bfs.push_back(en)
-        if en == bfs.start:
-          all_cycles += [all_paths[n]]
-    return all_cycles
-
-  def findCyclesInGraph(self):
-    all_cycles = []
-    for n in self.graph.nodes:
-      cycle = self.findCycleForNode(n)
-      if cycle:
-        all_cycles += [(n, cycle)]
-    return all_cycles
diff --git a/libcxx/utils/libcxx/test/config.py b/libcxx/utils/libcxx/test/config.py
index 955ef7979c7e466ac9e9bb264ff0d3ba6cf34cc9..3262c37f215355c8777c5c6f235e231ff76aafc7 100644
--- a/libcxx/utils/libcxx/test/config.py
+++ b/libcxx/utils/libcxx/test/config.py
@@ -124,7 +124,7 @@ class Configuration(object):
         self.configure_obj_root()
         self.cxx_stdlib_under_test = self.get_lit_conf('cxx_stdlib_under_test', 'libc++')
         self.cxx_library_root = self.get_lit_conf('cxx_library_root', self.libcxx_obj_root)
-        self.abi_library_root = self.get_lit_conf('abi_library_root', None)
+        self.abi_library_root = self.get_lit_conf('abi_library_root') or self.cxx_library_root
         self.cxx_runtime_root = self.get_lit_conf('cxx_runtime_root', self.cxx_library_root)
         self.abi_runtime_root = self.get_lit_conf('abi_runtime_root', self.abi_library_root)
         self.configure_compile_flags()
@@ -336,7 +336,8 @@ class Configuration(object):
         support_path = os.path.join(self.libcxx_src_root, 'test', 'support')
         self.configure_config_site_header()
         if self.cxx_stdlib_under_test != 'libstdc++' and \
-           not self.target_info.is_windows():
+           not self.target_info.is_windows() and \
+           not self.target_info.is_zos():
             self.cxx.compile_flags += [
                 '-include', os.path.join(support_path, 'nasty_macros.h')]
         if self.cxx_stdlib_under_test == 'msvc':
diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index 012d13aa2d7772ed0e3663623d928a94b47343e8..7f4dcec8e91b61fa660ed0a6d606f7657317cf4e 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -211,9 +211,12 @@ def featureTestMacros(config, flags=''):
   return {m: int(v.rstrip('LlUu')) for (m, v) in allMacros.items() if m.startswith('__cpp_')}
 
 
-def _addToSubstitution(substitutions, key, value):
+def _appendToSubstitution(substitutions, key, value):
   return [(k, v + ' ' + value) if k == key else (k, v) for (k, v) in substitutions]
 
+def _prependToSubstitution(substitutions, key, value):
+  return [(k, value + ' ' + v) if k == key else (k, v) for (k, v) in substitutions]
+
 
 class ConfigAction(object):
   """
@@ -285,7 +288,7 @@ class AddFlag(ConfigAction):
   def applyTo(self, config):
     flag = self._getFlag(config)
     assert hasCompileFlag(config, flag), "Trying to enable flag {}, which is not supported".format(flag)
-    config.substitutions = _addToSubstitution(config.substitutions, '%{flags}', flag)
+    config.substitutions = _appendToSubstitution(config.substitutions, '%{flags}', flag)
 
   def pretty(self, config, litParams):
     return 'add {} to %{{flags}}'.format(self._getFlag(config))
@@ -304,7 +307,7 @@ class AddCompileFlag(ConfigAction):
   def applyTo(self, config):
     flag = self._getFlag(config)
     assert hasCompileFlag(config, flag), "Trying to enable compile flag {}, which is not supported".format(flag)
-    config.substitutions = _addToSubstitution(config.substitutions, '%{compile_flags}', flag)
+    config.substitutions = _appendToSubstitution(config.substitutions, '%{compile_flags}', flag)
 
   def pretty(self, config, litParams):
     return 'add {} to %{{compile_flags}}'.format(self._getFlag(config))
@@ -312,7 +315,26 @@ class AddCompileFlag(ConfigAction):
 
 class AddLinkFlag(ConfigAction):
   """
-  This action adds the given flag to the %{link_flags} substitution.
+  This action appends the given flag to the %{link_flags} substitution.
+
+  The flag can be a string or a callable, in which case it is called with the
+  configuration to produce the actual flag (as a string).
+  """
+  def __init__(self, flag):
+    self._getFlag = lambda config: flag(config) if callable(flag) else flag
+
+  def applyTo(self, config):
+    flag = self._getFlag(config)
+    assert hasCompileFlag(config, flag), "Trying to enable link flag {}, which is not supported".format(flag)
+    config.substitutions = _appendToSubstitution(config.substitutions, '%{link_flags}', flag)
+
+  def pretty(self, config, litParams):
+    return 'append {} to %{{link_flags}}'.format(self._getFlag(config))
+
+
+class PrependLinkFlag(ConfigAction):
+  """
+  This action prepends the given flag to the %{link_flags} substitution.
 
   The flag can be a string or a callable, in which case it is called with the
   configuration to produce the actual flag (as a string).
@@ -323,10 +345,10 @@ class AddLinkFlag(ConfigAction):
   def applyTo(self, config):
     flag = self._getFlag(config)
     assert hasCompileFlag(config, flag), "Trying to enable link flag {}, which is not supported".format(flag)
-    config.substitutions = _addToSubstitution(config.substitutions, '%{link_flags}', flag)
+    config.substitutions = _prependToSubstitution(config.substitutions, '%{link_flags}', flag)
 
   def pretty(self, config, litParams):
-    return 'add {} to %{{link_flags}}'.format(self._getFlag(config))
+    return 'prepend {} to %{{link_flags}}'.format(self._getFlag(config))
 
 
 class AddOptionalWarningFlag(ConfigAction):
@@ -344,7 +366,7 @@ class AddOptionalWarningFlag(ConfigAction):
     flag = self._getFlag(config)
     # Use -Werror to make sure we see an error about the flag being unsupported.
     if hasCompileFlag(config, '-Werror ' + flag):
-      config.substitutions = _addToSubstitution(config.substitutions, '%{compile_flags}', flag)
+      config.substitutions = _appendToSubstitution(config.substitutions, '%{compile_flags}', flag)
 
   def pretty(self, config, litParams):
     return 'add {} to %{{compile_flags}}'.format(self._getFlag(config))
diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index fef2543f0b6f5b7b45e97be838850bb0636c59a1..99bd1f45b196b8033e26898d0197e0d06cbd94da 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -90,7 +90,7 @@ DEFAULT_PARAMETERS = [
             help="Whether to enable tests for experimental C++ libraries (typically Library Fundamentals TSes).",
             actions=lambda experimental: [] if not experimental else [
               AddFeature('c++experimental'),
-              AddLinkFlag('-lc++experimental')
+              PrependLinkFlag('-lc++experimental')
             ]),
 
   Parameter(name='long_tests', choices=[True, False], type=bool, default=True,
diff --git a/libcxx/utils/libcxx/test/target_info.py b/libcxx/utils/libcxx/test/target_info.py
index 45e3c4ae4d73b0aa6d7d05641d945ee74de639a5..198d2eebe979e9ffeb1dfb49a1603f4c6d7bdb47 100644
--- a/libcxx/utils/libcxx/test/target_info.py
+++ b/libcxx/utils/libcxx/test/target_info.py
@@ -21,16 +21,13 @@ class DefaultTargetInfo(object):
         self.full_config = full_config
         self.executor = None
 
-    def platform(self):
-        return sys.platform.lower().strip()
-
     def is_windows(self):
         return False
 
-    def is_mingw(self):
+    def is_zos(self):
         return False
 
-    def is_darwin(self):
+    def is_mingw(self):
         return False
 
     def add_cxx_flags(self, flags): pass
@@ -53,33 +50,6 @@ class DarwinLocalTI(DefaultTargetInfo):
     def __init__(self, full_config):
         super(DarwinLocalTI, self).__init__(full_config)
 
-    def is_darwin(self):
-        return True
-
-    def is_host_macosx(self):
-        name = lit.util.to_string(subprocess.check_output(['sw_vers', '-productName'])).strip()
-        return name == "Mac OS X"
-
-    def get_macosx_version(self):
-        assert self.is_host_macosx()
-        version = lit.util.to_string(subprocess.check_output(['sw_vers', '-productVersion'])).strip()
-        version = re.sub(r'([0-9]+\.[0-9]+)(\..*)?', r'\1', version)
-        return version
-
-    def get_sdk_version(self, name):
-        assert self.is_host_macosx()
-        cmd = ['xcrun', '--sdk', name, '--show-sdk-path']
-        try:
-            out = subprocess.check_output(cmd).strip()
-        except OSError:
-            pass
-
-        if not out:
-            self.full_config.lit_config.fatal(
-                    "cannot infer sdk version with: %r" % cmd)
-
-        return re.sub(r'.*/[^0-9]+([0-9.]+)\.sdk', r'\1', out)
-
     def add_cxx_flags(self, flags):
         out, err, exit_code = executeCommand(['xcrun', '--show-sdk-path'])
         if exit_code != 0:
@@ -120,31 +90,6 @@ class LinuxLocalTI(DefaultTargetInfo):
     def __init__(self, full_config):
         super(LinuxLocalTI, self).__init__(full_config)
 
-    def platform(self):
-        return 'linux'
-
-    def _distribution(self):
-        try:
-            # linux_distribution is not available since Python 3.8
-            # However, this function is only used to detect SLES 11,
-            # which is quite an old distribution that doesn't have
-            # Python 3.8.
-            return platform.linux_distribution()
-        except AttributeError:
-            return '', '', ''
-
-    def platform_name(self):
-        name, _, _ = self._distribution()
-        # Some distros have spaces, e.g. 'SUSE Linux Enterprise Server'
-        # lit features can't have spaces
-        name = name.lower().strip().replace(' ', '-')
-        return name # Permitted to be None
-
-    def platform_ver(self):
-        _, ver, _ = self._distribution()
-        ver = ver.lower().strip().replace(' ', '-')
-        return ver # Permitted to be None.
-
     def add_cxx_compile_flags(self, flags):
         flags += ['-D__STDC_FORMAT_MACROS',
                   '-D__STDC_LIMIT_MACROS',
@@ -193,6 +138,13 @@ class WindowsLocalTI(DefaultTargetInfo):
     def is_windows(self):
         return True
 
+class ZOSLocalTI(DefaultTargetInfo):
+    def __init__(self, full_config):
+        super(ZOSLocalTI, self).__init__(full_config)
+
+    def is_zos(self):
+        return True
+
 class MingwLocalTI(WindowsLocalTI):
     def __init__(self, full_config):
         super(MingwLocalTI, self).__init__(full_config)
@@ -215,4 +167,5 @@ def make_target_info(full_config):
     if target_system == 'NetBSD':  return NetBSDLocalTI(full_config)
     if target_system == 'Linux':   return LinuxLocalTI(full_config)
     if target_system == 'Windows': return WindowsLocalTI(full_config)
+    if target_system == 'OS/390':  return ZOSLocalTI(full_config)
     return DefaultTargetInfo(full_config)
diff --git a/libcxx/utils/ssh.py b/libcxx/utils/ssh.py
index a434271265a4cd85fab55c9e9ea227583c5b4210..d6346fcb20be900b0adaac24ccafb372f6490999 100755
--- a/libcxx/utils/ssh.py
+++ b/libcxx/utils/ssh.py
@@ -23,6 +23,12 @@ import sys
 import tarfile
 import tempfile
 
+try:
+   from shlex import quote as cmd_quote
+except ImportError:
+   # for Python 2 compatibility
+   from pipes import quote as cmd_quote
+
 def ssh(args, command):
     cmd = ['ssh', '-oBatchMode=yes']
     if args.extra_ssh_args is not None:
@@ -107,7 +113,7 @@ def main():
         commandLine = (pathOnRemote(x) if isTestExe(x) else x for x in commandLine)
         remoteCommands.append('cd {}'.format(tmp))
         if args.env:
-            remoteCommands.append('export {}'.format(' '.join(args.env)))
+            remoteCommands.append('export {}'.format(cmd_quote(' '.join(args.env))))
         remoteCommands.append(subprocess.list2cmdline(commandLine))
 
         # Finally, SSH to the remote host and execute all the commands.
diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt
index 8b0c88689df32418ed8e91d4d234326f5b908655..2ffd8b6119303b2b440e3f97cbdfc441e25cc22d 100644
--- a/libcxxabi/CMakeLists.txt
+++ b/libcxxabi/CMakeLists.txt
@@ -156,9 +156,23 @@ if (NOT LIBCXXABI_ENABLE_SHARED AND NOT LIBCXXABI_ENABLE_STATIC)
   message(FATAL_ERROR "libc++abi must be built as either a shared or static library.")
 endif()
 
-set(LIBCXXABI_LIBCXX_INCLUDES "${LIBCXXABI_LIBCXX_PATH}/include" CACHE PATH
+# TODO: This is a workaround for the fact that Standalone builds can't use
+# targets from the other runtimes (so the cxx-headers target doesn't exist).
+set(LIBCXXABI_LIBCXX_INCLUDES "" CACHE PATH
     "Specify path to libc++ includes.")
-message(STATUS "Libc++abi will be using libc++ includes from ${LIBCXXABI_LIBCXX_INCLUDES}")
+if (LIBCXXABI_STANDALONE_BUILD)
+  if (NOT IS_DIRECTORY ${LIBCXXABI_LIBCXX_INCLUDES})
+    message(FATAL_ERROR
+      "LIBCXXABI_LIBCXX_INCLUDES=${LIBCXXABI_LIBCXX_INCLUDES} is not a valid directory. "
+      "Please provide the path to where the libc++ headers have been installed.")
+  endif()
+  add_library(cxx-headers INTERFACE)
+  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC" OR "${CMAKE_CXX_SIMULATE_ID}" STREQUAL "MSVC")
+    target_compile_options(cxx-headers INTERFACE /I "${LIBCXXABI_LIBCXX_INCLUDES}")
+  else()
+    target_compile_options(cxx-headers INTERFACE -I "${LIBCXXABI_LIBCXX_INCLUDES}")
+  endif()
+endif()
 
 option(LIBCXXABI_HERMETIC_STATIC_LIBRARY
   "Do not export any symbols from the static library." OFF)
@@ -180,6 +194,7 @@ set(CMAKE_MODULE_PATH
   )
 
 if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
+  set(LIBCXXABI_HEADER_DIR ${LLVM_BINARY_DIR})
   set(LIBCXXABI_LIBRARY_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/${LLVM_DEFAULT_TARGET_TRIPLE}/c++)
   set(LIBCXXABI_INSTALL_LIBRARY_DIR lib${LLVM_LIBDIR_SUFFIX}/${LLVM_DEFAULT_TARGET_TRIPLE}/c++)
   if(LIBCXX_LIBDIR_SUBDIR)
@@ -187,9 +202,11 @@ if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
     string(APPEND LIBCXXABI_INSTALL_LIBRARY_DIR /${LIBCXXABI_LIBDIR_SUBDIR})
   endif()
 elseif(LLVM_LIBRARY_OUTPUT_INTDIR)
+  set(LIBCXXABI_HEADER_DIR ${LLVM_BINARY_DIR})
   set(LIBCXXABI_LIBRARY_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR})
   set(LIBCXXABI_INSTALL_LIBRARY_DIR lib${LIBCXXABI_LIBDIR_SUFFIX})
 else()
+  set(LIBCXXABI_HEADER_DIR ${CMAKE_BINARY_DIR})
   set(LIBCXXABI_LIBRARY_DIR ${CMAKE_BINARY_DIR}/lib${LIBCXXABI_LIBDIR_SUFFIX})
   set(LIBCXXABI_INSTALL_LIBRARY_DIR lib${LIBCXXABI_LIBDIR_SUFFIX})
 endif()
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index 50afdf6890a3f772cc509b2467c6d53ce217ef38..32a998e02f4823951bdaaac56d8290114109a0f9 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -36,7 +36,8 @@ else()
   )
 endif()
 
-if (LIBCXXABI_ENABLE_THREADS AND (UNIX OR FUCHSIA) AND NOT (APPLE OR CYGWIN))
+if (LIBCXXABI_ENABLE_THREADS AND (UNIX OR FUCHSIA) AND NOT (APPLE OR CYGWIN)
+    AND NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX"))
   list(APPEND LIBCXXABI_SOURCES
     cxa_thread_atexit.cpp
   )
@@ -55,8 +56,6 @@ if (MSVC_IDE OR XCODE)
   endif()
 endif()
 
-include_directories("${LIBCXXABI_LIBCXX_INCLUDES}")
-
 # stdlib_stdexcept.cpp depends on libc++ internals.
 include_directories("${LIBCXXABI_LIBCXX_PATH}")
 
@@ -177,7 +176,7 @@ endif()
 # Build the shared library.
 if (LIBCXXABI_ENABLE_SHARED)
   add_library(cxxabi_shared SHARED ${LIBCXXABI_SOURCES} ${LIBCXXABI_HEADERS})
-  target_link_libraries(cxxabi_shared PRIVATE ${LIBCXXABI_SHARED_LIBRARIES} ${LIBCXXABI_LIBRARIES})
+  target_link_libraries(cxxabi_shared PRIVATE cxx-headers ${LIBCXXABI_SHARED_LIBRARIES} ${LIBCXXABI_LIBRARIES})
   if (TARGET pstl::ParallelSTL)
     target_link_libraries(cxxabi_shared PUBLIC pstl::ParallelSTL)
   endif()
@@ -244,7 +243,7 @@ endif()
 # Build the static library.
 if (LIBCXXABI_ENABLE_STATIC)
   add_library(cxxabi_static STATIC ${LIBCXXABI_SOURCES} ${LIBCXXABI_HEADERS})
-  target_link_libraries(cxxabi_static PRIVATE ${LIBCXXABI_STATIC_LIBRARIES} ${LIBCXXABI_LIBRARIES})
+  target_link_libraries(cxxabi_static PRIVATE cxx-headers ${LIBCXXABI_STATIC_LIBRARIES} ${LIBCXXABI_LIBRARIES})
   if (TARGET pstl::ParallelSTL)
     target_link_libraries(cxxabi_static PUBLIC pstl::ParallelSTL)
   endif()
@@ -302,6 +301,7 @@ if (LIBCXXABI_ENABLE_STATIC)
         "$<TARGET_LINKER_FILE:cxxabi_static>"
         "$<TARGET_LINKER_FILE:unwind_static>"
       WORKING_DIRECTORY ${LIBCXXABI_BUILD_DIR}
+      DEPENDS unwind_static
     )
   endif()
 endif()
diff --git a/libcxxabi/src/demangle/ItaniumDemangle.h b/libcxxabi/src/demangle/ItaniumDemangle.h
index e5fca98f9271798a3476ca5c6a5c10cacaa01ffd..202c959d40d207d5952292a86425336bd06ea1c9 100644
--- a/libcxxabi/src/demangle/ItaniumDemangle.h
+++ b/libcxxabi/src/demangle/ItaniumDemangle.h
@@ -280,17 +280,20 @@ public:
 class VendorExtQualType final : public Node {
   const Node *Ty;
   StringView Ext;
+  const Node *TA;
 
 public:
-  VendorExtQualType(const Node *Ty_, StringView Ext_)
-      : Node(KVendorExtQualType), Ty(Ty_), Ext(Ext_) {}
+  VendorExtQualType(const Node *Ty_, StringView Ext_, const Node *TA_)
+      : Node(KVendorExtQualType), Ty(Ty_), Ext(Ext_), TA(TA_) {}
 
-  template<typename Fn> void match(Fn F) const { F(Ty, Ext); }
+  template <typename Fn> void match(Fn F) const { F(Ty, Ext, TA); }
 
   void printLeft(OutputStream &S) const override {
     Ty->print(S);
     S += " ";
     S += Ext;
+    if (TA != nullptr)
+      TA->print(S);
   }
 };
 
@@ -3680,8 +3683,6 @@ Node *AbstractManglingParser<Derived, Alloc>::parseQualifiedType() {
     if (Qual.empty())
       return nullptr;
 
-    // FIXME parse the optional <template-args> here!
-
     // extension            ::= U <objc-name> <objc-type>  # objc-type<identifier>
     if (Qual.startsWith("objcproto")) {
       StringView ProtoSourceName = Qual.dropFront(std::strlen("objcproto"));
@@ -3699,10 +3700,17 @@ Node *AbstractManglingParser<Derived, Alloc>::parseQualifiedType() {
       return make<ObjCProtoName>(Child, Proto);
     }
 
+    Node *TA = nullptr;
+    if (look() == 'I') {
+      TA = getDerived().parseTemplateArgs();
+      if (TA == nullptr)
+        return nullptr;
+    }
+
     Node *Child = getDerived().parseQualifiedType();
     if (Child == nullptr)
       return nullptr;
-    return make<VendorExtQualType>(Child, Qual);
+    return make<VendorExtQualType>(Child, Qual, TA);
   }
 
   Qualifiers Quals = parseCVQualifiers();
diff --git a/libcxxabi/test/libcxxabi/test/config.py b/libcxxabi/test/libcxxabi/test/config.py
index 280a60a864bcb55f5dd9b35f171c6880f3b960aa..5357bb18b931c7a684f9be8fd6952d2ecbac9e43 100644
--- a/libcxxabi/test/libcxxabi/test/config.py
+++ b/libcxxabi/test/libcxxabi/test/config.py
@@ -16,12 +16,16 @@ class Configuration(LibcxxConfiguration):
     # pylint: disable=redefined-outer-name
     def __init__(self, lit_config, config):
         super(Configuration, self).__init__(lit_config, config)
+        self.libcxxabi_hdr_root = None
         self.libcxxabi_src_root = None
         self.libcxxabi_obj_root = None
         self.abi_library_root = None
         self.libcxx_src_root = None
 
     def configure_src_root(self):
+        self.libcxxabi_hdr_root = self.get_lit_conf(
+            'libcxxabi_hdr_root',
+            self.project_obj_root)
         self.libcxxabi_src_root = self.get_lit_conf(
             'libcxxabi_src_root',
             os.path.dirname(self.config.test_source_root))
@@ -57,9 +61,8 @@ class Configuration(LibcxxConfiguration):
 
     def configure_compile_flags_header_includes(self):
         self.configure_config_site_header()
-        cxx_headers = self.get_lit_conf(
-            'cxx_headers',
-            os.path.join(self.libcxx_src_root, '/include'))
+        cxx_headers = self.get_lit_conf('cxx_headers', None) or \
+            os.path.join(self.libcxxabi_hdr_root, 'include', 'c++', 'v1')
         if cxx_headers == '':
             self.lit_config.note('using the systems c++ headers')
         else:
diff --git a/libcxxabi/test/lit.site.cfg.in b/libcxxabi/test/lit.site.cfg.in
index 1b7713e3ef73c2c1849a1e02c21070f323508e34..76072187101c3d7054cfdb9a2052523811a5a971 100644
--- a/libcxxabi/test/lit.site.cfg.in
+++ b/libcxxabi/test/lit.site.cfg.in
@@ -5,6 +5,7 @@ import site
 
 config.cxx_under_test           = "@CMAKE_CXX_COMPILER@"
 config.project_obj_root         = "@CMAKE_BINARY_DIR@"
+config.libcxxabi_hdr_root       = "@LIBCXXABI_HEADER_DIR@"
 config.libcxxabi_src_root       = "@LIBCXXABI_SOURCE_DIR@"
 config.libcxxabi_obj_root       = "@LIBCXXABI_BINARY_DIR@"
 config.abi_library_root         = "@LIBCXXABI_LIBRARY_DIR@"
diff --git a/libcxxabi/test/test_demangle.pass.cpp b/libcxxabi/test/test_demangle.pass.cpp
index 1d7b10706e4390bfbedb6b4ab668941d5f3ca7f6..4f19dafd9e8263c236d80e81891c857ea35b6d8b 100644
--- a/libcxxabi/test/test_demangle.pass.cpp
+++ b/libcxxabi/test/test_demangle.pass.cpp
@@ -29843,6 +29843,10 @@ const char* cases[][2] =
     {"_Z1fIL4Enumn1EEvv", "void f<(Enum)-1>()"},
 
     {"_ZN1A1gIiEEDTcldtptfpT1b1fIT_EEEv", "decltype(this->b.f<int>()) A::g<int>()"},
+
+    // Optional template-args for vendor extended type qualifier.
+    // See https://bugs.llvm.org/show_bug.cgi?id=48009.
+    {"_Z3fooILi79EEbU7_ExtIntIXT_EEi", "bool foo<79>(int _ExtInt<79>)"},
 };
 
 const unsigned N = sizeof(cases) / sizeof(cases[0]);
diff --git a/lld/COFF/Chunks.h b/lld/COFF/Chunks.h
index e076d8e71109a80be1beadca5b489c9e0f7bc6a9..6e7af0babe585802868656f3359aa5944a4cb17b 100644
--- a/lld/COFF/Chunks.h
+++ b/lld/COFF/Chunks.h
@@ -293,8 +293,12 @@ public:
 
   // Allow iteration over the associated child chunks for this section.
   llvm::iterator_range<AssociatedIterator> children() const {
-    return llvm::make_range(AssociatedIterator(assocChildren),
-                            AssociatedIterator(nullptr));
+    // Associated sections do not have children. The assocChildren field is
+    // part of the parent's list of children.
+    bool isAssoc = selection == llvm::COFF::IMAGE_COMDAT_SELECT_ASSOCIATIVE;
+    return llvm::make_range(
+        AssociatedIterator(isAssoc ? nullptr : assocChildren),
+        AssociatedIterator(nullptr));
   }
 
   // The section ID this chunk belongs to in its Obj.
diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index ba1a5793ed5bd42a7e7602cf44e463130ce616da..bde7b5b473d98107a8fff7efd4dd86974a2f09dd 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -215,6 +215,12 @@ struct Configuration {
   // Used for /lto-obj-path:
   llvm::StringRef ltoObjPath;
 
+  // Used for /lto-cs-profile-generate:
+  bool ltoCSProfileGenerate = false;
+
+  // Used for /lto-cs-profile-path
+  llvm::StringRef ltoCSProfileFile;
+
   // Used for /call-graph-ordering-file:
   llvm::MapVector<std::pair<const SectionChunk *, const SectionChunk *>,
                   uint64_t>
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 9c13d4a078ec76be98b091d57f22b418d951aa44..cf96ecb731a2c4ad7164567f565e38710bb38a44 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -157,9 +157,8 @@ static std::future<MBErrPair> createFutureForFile(std::string path) {
   auto strategy = std::launch::deferred;
 #endif
   return std::async(strategy, [=]() {
-    auto mbOrErr = MemoryBuffer::getFile(path,
-                                         /*FileSize*/ -1,
-                                         /*RequiresNullTerminator*/ false);
+    auto mbOrErr = MemoryBuffer::getFile(path, /*IsText=*/false,
+                                         /*RequiresNullTerminator=*/false);
     if (!mbOrErr)
       return MBErrPair{nullptr, mbOrErr.getError()};
     return MBErrPair{std::move(*mbOrErr), std::error_code()};
@@ -829,7 +828,7 @@ static void createImportLibrary(bool asLib) {
   // If the import library already exists, replace it only if the contents
   // have changed.
   ErrorOr<std::unique_ptr<MemoryBuffer>> oldBuf = MemoryBuffer::getFile(
-      path, /*FileSize*/ -1, /*RequiresNullTerminator*/ false);
+      path, /*IsText=*/false, /*RequiresNullTerminator=*/false);
   if (!oldBuf) {
     handleError(writeImportLibrary(libName, path, exports, config->machine,
                                    config->mingw));
@@ -849,7 +848,7 @@ static void createImportLibrary(bool asLib) {
   }
 
   std::unique_ptr<MemoryBuffer> newBuf = check(MemoryBuffer::getFile(
-      tmpName, /*FileSize*/ -1, /*RequiresNullTerminator*/ false));
+      tmpName, /*IsText=*/false, /*RequiresNullTerminator=*/false));
   if ((*oldBuf)->getBuffer() != newBuf->getBuffer()) {
     oldBuf->reset();
     handleError(errorCodeToError(sys::fs::rename(tmpName, path)));
@@ -859,8 +858,11 @@ static void createImportLibrary(bool asLib) {
 }
 
 static void parseModuleDefs(StringRef path) {
-  std::unique_ptr<MemoryBuffer> mb = CHECK(
-      MemoryBuffer::getFile(path, -1, false, true), "could not open " + path);
+  std::unique_ptr<MemoryBuffer> mb =
+      CHECK(MemoryBuffer::getFile(path, /*IsText=*/false,
+                                  /*RequiresNullTerminator=*/false,
+                                  /*IsVolatile=*/true),
+            "could not open " + path);
   COFFModuleDefinition m = check(parseCOFFModuleDefinition(
       mb->getMemBufferRef(), config->machine, config->mingw));
 
@@ -948,8 +950,11 @@ static void parseOrderFile(StringRef arg) {
 
   // Open a file.
   StringRef path = arg.substr(1);
-  std::unique_ptr<MemoryBuffer> mb = CHECK(
-      MemoryBuffer::getFile(path, -1, false, true), "could not open " + path);
+  std::unique_ptr<MemoryBuffer> mb =
+      CHECK(MemoryBuffer::getFile(path, /*IsText=*/false,
+                                  /*RequiresNullTerminator=*/false,
+                                  /*IsVolatile=*/true),
+            "could not open " + path);
 
   // Parse a file. An order file contains one symbol per line.
   // All symbols that were not present in a given order file are
@@ -973,8 +978,11 @@ static void parseOrderFile(StringRef arg) {
 }
 
 static void parseCallGraphFile(StringRef path) {
-  std::unique_ptr<MemoryBuffer> mb = CHECK(
-      MemoryBuffer::getFile(path, -1, false, true), "could not open " + path);
+  std::unique_ptr<MemoryBuffer> mb =
+      CHECK(MemoryBuffer::getFile(path, /*IsText=*/false,
+                                  /*RequiresNullTerminator=*/false,
+                                  /*IsVolatile=*/true),
+            "could not open " + path);
 
   // Build a map from symbol name to section.
   DenseMap<StringRef, Symbol *> map;
@@ -1712,6 +1720,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   config->thinLTOObjectSuffixReplace =
       getOldNewOptions(args, OPT_thinlto_object_suffix_replace);
   config->ltoObjPath = args.getLastArgValue(OPT_lto_obj_path);
+  config->ltoCSProfileGenerate = args.hasArg(OPT_lto_cs_profile_generate);
+  config->ltoCSProfileFile = args.getLastArgValue(OPT_lto_cs_profile_file);
   // Handle miscellaneous boolean flags.
   config->allowBind = args.hasFlag(OPT_allowbind, OPT_allowbind_no, true);
   config->allowIsolation =
diff --git a/lld/COFF/DriverUtils.cpp b/lld/COFF/DriverUtils.cpp
index 19964428050b01cf45b0281085eddb6aadddaf1c..1e8cbe8e2b17205e65d6a543149eae6d821df716 100644
--- a/lld/COFF/DriverUtils.cpp
+++ b/lld/COFF/DriverUtils.cpp
@@ -350,7 +350,7 @@ public:
   // is called (you cannot remove an opened file on Windows.)
   std::unique_ptr<MemoryBuffer> getMemoryBuffer() {
     // IsVolatile=true forces MemoryBuffer to not use mmap().
-    return CHECK(MemoryBuffer::getFile(path, /*FileSize=*/-1,
+    return CHECK(MemoryBuffer::getFile(path, /*IsText=*/false,
                                        /*RequiresNullTerminator=*/false,
                                        /*IsVolatile=*/true),
                  "could not open " + path);
diff --git a/lld/COFF/ICF.cpp b/lld/COFF/ICF.cpp
index 73264696729682e7c659f86adba01afaa5cbd08d..8e6b1a0bfcbabbb6c606f5dd94f3318be6f45bbe 100644
--- a/lld/COFF/ICF.cpp
+++ b/lld/COFF/ICF.cpp
@@ -46,7 +46,7 @@ public:
 private:
   void segregate(size_t begin, size_t end, bool constant);
 
-  bool assocEquals(const SectionChunk *a, const SectionChunk *b);
+  bool equalsEHData(const SectionChunk *a, const SectionChunk *b);
 
   bool equalsConstant(const SectionChunk *a, const SectionChunk *b);
   bool equalsVariable(const SectionChunk *a, const SectionChunk *b);
@@ -127,21 +127,31 @@ void ICF::segregate(size_t begin, size_t end, bool constant) {
   }
 }
 
-// Returns true if two sections' associative children are equal.
-bool ICF::assocEquals(const SectionChunk *a, const SectionChunk *b) {
-  // Ignore associated metadata sections that don't participate in ICF, such as
-  // debug info and CFGuard metadata.
-  auto considerForICF = [](const SectionChunk &assoc) {
-    StringRef Name = assoc.getSectionName();
-    return !(Name.startswith(".debug") || Name == ".gfids$y" ||
-             Name == ".giats$y" || Name == ".gljmp$y");
+// Returns true if two sections have equivalent associated .pdata/.xdata
+// sections.
+bool ICF::equalsEHData(const SectionChunk *a, const SectionChunk *b) {
+  auto findEHData = [](const SectionChunk *s) {
+    const SectionChunk *pdata = nullptr;
+    const SectionChunk *xdata = nullptr;
+    for (const SectionChunk &assoc : s->children()) {
+      StringRef name = assoc.getSectionName();
+      if (name.startswith(".pdata") && (name.size() == 6 || name[6] == '$'))
+        pdata = &assoc;
+      else if (name.startswith(".xdata") &&
+               (name.size() == 6 || name[6] == '$'))
+        xdata = &assoc;
+    }
+    return std::make_pair(pdata, xdata);
+  };
+  auto aData = findEHData(a);
+  auto bData = findEHData(b);
+  auto considerEqual = [cnt = cnt](const SectionChunk *l,
+                                   const SectionChunk *r) {
+    return l == r || (l->getContents() == r->getContents() &&
+                      l->eqClass[cnt % 2] == r->eqClass[cnt % 2]);
   };
-  auto ra = make_filter_range(a->children(), considerForICF);
-  auto rb = make_filter_range(b->children(), considerForICF);
-  return std::equal(ra.begin(), ra.end(), rb.begin(), rb.end(),
-                    [&](const SectionChunk &ia, const SectionChunk &ib) {
-                      return ia.eqClass[cnt % 2] == ib.eqClass[cnt % 2];
-                    });
+  return considerEqual(aData.first, bData.first) &&
+         considerEqual(aData.second, bData.second);
 }
 
 // Compare "non-moving" part of two sections, namely everything
@@ -175,7 +185,7 @@ bool ICF::equalsConstant(const SectionChunk *a, const SectionChunk *b) {
          a->getSectionName() == b->getSectionName() &&
          a->header->SizeOfRawData == b->header->SizeOfRawData &&
          a->checksum == b->checksum && a->getContents() == b->getContents() &&
-         assocEquals(a, b);
+         equalsEHData(a, b);
 }
 
 // Compare "moving" part of two sections, namely relocation targets.
@@ -193,7 +203,7 @@ bool ICF::equalsVariable(const SectionChunk *a, const SectionChunk *b) {
   };
   return std::equal(a->getRelocs().begin(), a->getRelocs().end(),
                     b->getRelocs().begin(), eq) &&
-         assocEquals(a, b);
+         equalsEHData(a, b);
 }
 
 // Find the first Chunk after Begin that has a different class from Begin.
diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp
index 2fa3536db873d511592e125fe8194eac99048938..a47f66ec7cf0888101a30fe5eee749d867c00a7a 100644
--- a/lld/COFF/LTO.cpp
+++ b/lld/COFF/LTO.cpp
@@ -84,6 +84,8 @@ static lto::Config createConfig() {
   c.AlwaysEmitRegularLTOObj = !config->ltoObjPath.empty();
   c.UseNewPM = config->ltoNewPassManager;
   c.DebugPassManager = config->ltoDebugPassManager;
+  c.CSIRProfile = std::string(config->ltoCSProfileFile);
+  c.RunCSIRInstr = config->ltoCSProfileGenerate;
 
   if (config->saveTemps)
     checkError(c.addSaveTemps(std::string(config->outputFile) + ".",
diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td
index 73c3380df17c9c351a1bae9000c59edc68308610..33d560902f78d694b6d8b373cf104450d48556e1 100644
--- a/lld/COFF/Options.td
+++ b/lld/COFF/Options.td
@@ -239,6 +239,10 @@ def thinlto_prefix_replace: P<
 def lto_obj_path : P<
     "lto-obj-path",
     "output native object for merged LTO unit to this path">;
+def lto_cs_profile_generate: F<"lto-cs-profile-generate">,
+    HelpText<"Perform context sensitive PGO instrumentation">;
+def lto_cs_profile_file : P<"lto-cs-profile-file",
+    "Context sensitive profile file path">;
 def dash_dash_version : Flag<["--"], "version">,
   HelpText<"Display the version number and exit">;
 def threads
diff --git a/lld/ELF/Arch/AMDGPU.cpp b/lld/ELF/Arch/AMDGPU.cpp
index 4f4ce0094bbfd3a40e50c487b940a3359bfccbc8..c765d6e083002cc06e780851105134ded99ecc4f 100644
--- a/lld/ELF/Arch/AMDGPU.cpp
+++ b/lld/ELF/Arch/AMDGPU.cpp
@@ -22,6 +22,10 @@ using namespace lld::elf;
 
 namespace {
 class AMDGPU final : public TargetInfo {
+private:
+  uint32_t calcEFlagsV3() const;
+  uint32_t calcEFlagsV4() const;
+
 public:
   AMDGPU();
   uint32_t calcEFlags() const override;
@@ -44,8 +48,7 @@ static uint32_t getEFlags(InputFile *file) {
   return cast<ObjFile<ELF64LE>>(file)->getObj().getHeader().e_flags;
 }
 
-uint32_t AMDGPU::calcEFlags() const {
-  assert(!objectFiles.empty());
+uint32_t AMDGPU::calcEFlagsV3() const {
   uint32_t ret = getEFlags(objectFiles[0]);
 
   // Verify that all input files have the same e_flags.
@@ -58,6 +61,67 @@ uint32_t AMDGPU::calcEFlags() const {
   return ret;
 }
 
+uint32_t AMDGPU::calcEFlagsV4() const {
+  uint32_t retMach = getEFlags(objectFiles[0]) & EF_AMDGPU_MACH;
+  uint32_t retXnack = getEFlags(objectFiles[0]) & EF_AMDGPU_FEATURE_XNACK_V4;
+  uint32_t retSramEcc =
+      getEFlags(objectFiles[0]) & EF_AMDGPU_FEATURE_SRAMECC_V4;
+
+  // Verify that all input files have compatible e_flags (same mach, all
+  // features in the same category are either ANY, ANY and ON, or ANY and OFF).
+  for (InputFile *f : makeArrayRef(objectFiles).slice(1)) {
+    if (retMach != (getEFlags(f) & EF_AMDGPU_MACH)) {
+      error("incompatible mach: " + toString(f));
+      return 0;
+    }
+
+    if (retXnack == EF_AMDGPU_FEATURE_XNACK_UNSUPPORTED_V4 ||
+        (retXnack != EF_AMDGPU_FEATURE_XNACK_ANY_V4 &&
+            (getEFlags(f) & EF_AMDGPU_FEATURE_XNACK_V4)
+                != EF_AMDGPU_FEATURE_XNACK_ANY_V4)) {
+      if (retXnack != (getEFlags(f) & EF_AMDGPU_FEATURE_XNACK_V4)) {
+        error("incompatible xnack: " + toString(f));
+        return 0;
+      }
+    } else {
+      if (retXnack == EF_AMDGPU_FEATURE_XNACK_ANY_V4)
+        retXnack = getEFlags(f) & EF_AMDGPU_FEATURE_XNACK_V4;
+    }
+
+    if (retSramEcc == EF_AMDGPU_FEATURE_SRAMECC_UNSUPPORTED_V4 ||
+        (retSramEcc != EF_AMDGPU_FEATURE_SRAMECC_ANY_V4 &&
+            (getEFlags(f) & EF_AMDGPU_FEATURE_SRAMECC_V4) !=
+                EF_AMDGPU_FEATURE_SRAMECC_ANY_V4)) {
+      if (retSramEcc != (getEFlags(f) & EF_AMDGPU_FEATURE_SRAMECC_V4)) {
+        error("incompatible sramecc: " + toString(f));
+        return 0;
+      }
+    } else {
+      if (retSramEcc == EF_AMDGPU_FEATURE_SRAMECC_ANY_V4)
+        retSramEcc = getEFlags(f) & EF_AMDGPU_FEATURE_SRAMECC_V4;
+    }
+  }
+
+  return retMach | retXnack | retSramEcc;
+}
+
+uint32_t AMDGPU::calcEFlags() const {
+  assert(!objectFiles.empty());
+
+  uint8_t abiVersion = cast<ObjFile<ELF64LE>>(objectFiles[0])->getObj()
+      .getHeader().e_ident[EI_ABIVERSION];
+  switch (abiVersion) {
+  case ELFABIVERSION_AMDGPU_HSA_V2:
+  case ELFABIVERSION_AMDGPU_HSA_V3:
+    return calcEFlagsV3();
+  case ELFABIVERSION_AMDGPU_HSA_V4:
+    return calcEFlagsV4();
+  default:
+    error("unknown abi version: " + Twine(abiVersion));
+    return 0;
+  }
+}
+
 void AMDGPU::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   switch (rel.type) {
   case R_AMDGPU_ABS32:
diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index 03ecc811b2cf1dbb34f2c92ece3ccc7a01c45103..a0c2d1617caa2aede8ac1c90f488ce4993e34e31 100644
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -920,7 +920,15 @@ void PPC64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
       // that comes before it will already have computed the address of the
       // symbol.
       if (secondaryOp == 266) {
-        write32(loc - 1, NOP);
+        // Check if the add uses the same result register as the input register.
+        uint32_t rt = (tlsInstr & 0x03E00000) >> 21; // bits 6-10
+        uint32_t ra = (tlsInstr & 0x001F0000) >> 16; // bits 11-15
+        if (ra == rt) {
+          write32(loc - 1, NOP);
+        } else {
+          // mr rt, ra
+          write32(loc - 1, 0x7C000378 | (rt << 16) | (ra << 21) | (ra << 11));
+        }
       } else {
         uint32_t dFormOp = getPPCDFormOp(secondaryOp);
         if (dFormOp == 0)
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index fcfe5f64c32f054e86fcbb434c44e50be4144c62..ab55c60bb6f9938aa3076b29d0bc13ea9eed1323 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -198,7 +198,7 @@ struct Configuration {
   bool relocatable;
   bool relrPackDynRelocs;
   bool saveTemps;
-  llvm::Optional<uint32_t> shuffleSectionSeed;
+  std::vector<std::pair<llvm::GlobPattern, uint32_t>> shuffleSections;
   bool singleRoRx;
   bool shared;
   bool symbolic;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index df9925d74f8a294b9d05e00abb8b57716e99c764..3401c016dbe900d5e1ed89663e70eec0351b82ad 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1068,8 +1068,6 @@ static void readConfigs(opt::InputArgList &args) {
   config->rpath = getRpath(args);
   config->relocatable = args.hasArg(OPT_relocatable);
   config->saveTemps = args.hasArg(OPT_save_temps);
-  if (args.hasArg(OPT_shuffle_sections))
-    config->shuffleSectionSeed = args::getInteger(args, OPT_shuffle_sections, 0);
   config->searchPaths = args::getStrings(args, OPT_library_path);
   config->sectionStartMap = getSectionStartMap(args);
   config->shared = args.hasArg(OPT_shared);
@@ -1149,6 +1147,24 @@ static void readConfigs(opt::InputArgList &args) {
       config->optEL = true;
   }
 
+  for (opt::Arg *arg : args.filtered(OPT_shuffle_sections)) {
+    constexpr StringRef errPrefix = "--shuffle-sections=: ";
+    std::pair<StringRef, StringRef> kv = StringRef(arg->getValue()).split('=');
+    if (kv.first.empty() || kv.second.empty()) {
+      error(errPrefix + "expected <section_glob>=<seed>, but got '" +
+            arg->getValue() + "'");
+      continue;
+    }
+    // Signed so that <section_glob>=-1 is allowed.
+    int64_t v;
+    if (!to_integer(kv.second, v))
+      error(errPrefix + "expected an integer, but got '" + kv.second + "'");
+    else if (Expected<GlobPattern> pat = GlobPattern::create(kv.first))
+      config->shuffleSections.emplace_back(std::move(*pat), uint32_t(v));
+    else
+      error(errPrefix + toString(pat.takeError()));
+  }
+
   for (opt::Arg *arg : args.filtered(OPT_z)) {
     std::pair<StringRef, StringRef> option =
         StringRef(arg->getValue()).split('=');
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 6cf668d1b264adeaf07f2d03dfd1e3cadc830650..1f1ff22247ed0e88a2fd3b4c009220d661a16307 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -115,7 +115,8 @@ Optional<MemoryBufferRef> elf::readFile(StringRef path) {
   log(path);
   config->dependencyFiles.insert(llvm::CachedHashString(path));
 
-  auto mbOrErr = MemoryBuffer::getFile(path, -1, false);
+  auto mbOrErr = MemoryBuffer::getFile(path, /*IsText=*/false,
+                                       /*RequiresNullTerminator=*/false);
   if (auto ec = mbOrErr.getError()) {
     error("cannot open " + path + ": " + ec.message());
     return None;
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 302afd2a4374ac8f60552cebbd9ec4b7230d8045..55bde53cddcbb82529678d2e4fc441d3f9c74938 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -586,8 +586,10 @@ def lto_basic_block_sections: JJ<"lto-basic-block-sections=">,
 defm lto_unique_basic_block_section_names: BB<"lto-unique-basic-block-section-names",
     "Give unique names to every basic block section for LTO",
     "Do not give unique names to every basic block section for LTO (default)">;
-def shuffle_sections: JJ<"shuffle-sections=">, MetaVarName<"<seed>">,
-  HelpText<"Shuffle input sections using the given seed. If 0, use a random seed">;
+defm shuffle_sections: EEq<"shuffle-sections",
+  "Shuffle matched sections using the given seed before mapping them to the output sections. "
+  "If -1, reverse the section order. If 0, use a random seed">,
+  MetaVarName<"<section-glob>=<seed>">;
 def thinlto_cache_dir: JJ<"thinlto-cache-dir=">,
   HelpText<"Path to ThinLTO cached object file directory">;
 defm thinlto_cache_policy: EEq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">;
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 7a505ff8cfa13cdc5cad4321393dad3868f15d36..5f5f7ccb4d3502c1752ecca35f88f0d0ffd2d5cd 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -1291,22 +1291,39 @@ findOrphanPos(std::vector<BaseCommand *>::iterator b,
 
 // Adds random priorities to sections not already in the map.
 static void maybeShuffle(DenseMap<const InputSectionBase *, int> &order) {
-  if (!config->shuffleSectionSeed)
+  if (config->shuffleSections.empty())
     return;
 
-  std::vector<int> priorities(inputSections.size() - order.size());
+  std::vector<InputSectionBase *> matched, sections = inputSections;
+  matched.reserve(sections.size());
+  for (const auto &patAndSeed : config->shuffleSections) {
+    matched.clear();
+    for (InputSectionBase *sec : sections)
+      if (patAndSeed.first.match(sec->name))
+        matched.push_back(sec);
+    const uint32_t seed = patAndSeed.second;
+    if (seed == UINT32_MAX) {
+      // If --shuffle-sections <section-glob>=-1, reverse the section order. The
+      // section order is stable even if the number of sections changes. This is
+      // useful to catch issues like static initialization order fiasco
+      // reliably.
+      std::reverse(matched.begin(), matched.end());
+    } else {
+      std::mt19937 g(seed ? seed : std::random_device()());
+      llvm::shuffle(matched.begin(), matched.end(), g);
+    }
+    size_t i = 0;
+    for (InputSectionBase *&sec : sections)
+      if (patAndSeed.first.match(sec->name))
+        sec = matched[i++];
+  }
+
   // Existing priorities are < 0, so use priorities >= 0 for the missing
   // sections.
-  int curPrio = 0;
-  for (int &prio : priorities)
-    prio = curPrio++;
-  uint32_t seed = *config->shuffleSectionSeed;
-  std::mt19937 g(seed ? seed : std::random_device()());
-  llvm::shuffle(priorities.begin(), priorities.end(), g);
-  int prioIndex = 0;
-  for (InputSectionBase *sec : inputSections) {
-    if (order.try_emplace(sec, priorities[prioIndex]).second)
-      ++prioIndex;
+  int prio = 0;
+  for (InputSectionBase *sec : sections) {
+    if (order.try_emplace(sec, prio).second)
+      ++prio;
   }
 }
 
diff --git a/lld/MachO/CMakeLists.txt b/lld/MachO/CMakeLists.txt
index 8eb3371580b7aaf6e70c00dd4b5000dc73696c0a..16b372945d075be0905bf037fd8dc909514d6e5c 100644
--- a/lld/MachO/CMakeLists.txt
+++ b/lld/MachO/CMakeLists.txt
@@ -24,6 +24,7 @@ add_lld_library(lldMachO2
   Symbols.cpp
   SyntheticSections.cpp
   Target.cpp
+  MapFile.cpp
   Writer.cpp
 
   LINK_COMPONENTS
diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index 53c2c6ae157431201d4e8b1de958fb6fb314bbfa..810ae7b9a9c6a75883f5930d106101d6765e5528 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -51,12 +51,12 @@ enum class UndefinedSymbolTreatment {
 };
 
 class SymbolPatterns {
+public:
   // GlobPattern can also match literals,
   // but we prefer the O(1) lookup of DenseSet.
   llvm::DenseSet<llvm::CachedHashStringRef> literals;
   std::vector<llvm::GlobPattern> globs;
 
-public:
   bool empty() const { return literals.empty() && globs.empty(); }
   void clear();
   void insert(llvm::StringRef symbolName);
@@ -81,13 +81,15 @@ struct Configuration {
   bool searchDylibsFirst = false;
   bool saveTemps = false;
   bool adhocCodesign = false;
+  bool emitFunctionStarts = false;
   bool timeTraceEnabled = false;
   uint32_t headerPad;
   uint32_t dylibCompatibilityVersion = 0;
   uint32_t dylibCurrentVersion = 0;
-  uint32_t timeTraceGranularity;
+  uint32_t timeTraceGranularity = 500;
   std::string progName;
   llvm::StringRef installName;
+  llvm::StringRef mapFile;
   llvm::StringRef outputFile;
   llvm::StringRef ltoObjPath;
   bool demangle = false;
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 655d4cc555a57bb1dc9a30b008a72957e3e43126..d262ad568539d7481728949217dbc4c66bc7644f 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -38,6 +38,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Parallel.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TarWriter.h"
 #include "llvm/Support/TargetSelect.h"
@@ -54,7 +55,8 @@ using namespace llvm::sys;
 using namespace lld;
 using namespace lld::macho;
 
-Configuration *lld::macho::config;
+Configuration *macho::config;
+DependencyTracker *macho::depTracker;
 
 static HeaderFileType getOutputType(const InputArgList &args) {
   // TODO: -r, -dylinker, -preload...
@@ -84,6 +86,8 @@ findAlongPathsWithExtensions(StringRef name, ArrayRef<StringRef> extensions) {
       Twine location = base + ext;
       if (fs::exists(location))
         return location.str();
+      else
+        depTracker->logFileNotFound(location);
     }
   }
   return {};
@@ -493,6 +497,7 @@ static void initLLVM() {
 }
 
 static void compileBitcodeFiles() {
+  TimeTraceScope timeScope("LTO");
   auto *lto = make<BitcodeCompiler>();
   for (InputFile *file : inputFiles)
     if (auto *bitcodeFile = dyn_cast<BitcodeFile>(file))
@@ -507,6 +512,7 @@ static void compileBitcodeFiles() {
 // all InputFiles have been loaded.) As a result, later operations won't see
 // any CommonSymbols.
 static void replaceCommonSymbols() {
+  TimeTraceScope timeScope("Replace common symbols");
   for (macho::Symbol *sym : symtab->getSymbols()) {
     auto *common = dyn_cast<CommonSymbol>(sym);
     if (common == nullptr)
@@ -769,6 +775,44 @@ static void handleSymbolPatterns(InputArgList &args,
   }
 }
 
+void createFiles(const InputArgList &args) {
+  TimeTraceScope timeScope("Load input files");
+  // This loop should be reserved for options whose exact ordering matters.
+  // Other options should be handled via filtered() and/or getLastArg().
+  for (const Arg *arg : args) {
+    const Option &opt = arg->getOption();
+    warnIfDeprecatedOption(opt);
+    warnIfUnimplementedOption(opt);
+
+    switch (opt.getID()) {
+    case OPT_INPUT:
+      addFile(arg->getValue(), false);
+      break;
+    case OPT_weak_library:
+      if (auto *dylibFile =
+              dyn_cast_or_null<DylibFile>(addFile(arg->getValue(), false)))
+        dylibFile->forceWeakImport = true;
+      break;
+    case OPT_filelist:
+      addFileList(arg->getValue());
+      break;
+    case OPT_force_load:
+      addFile(arg->getValue(), true);
+      break;
+    case OPT_l:
+    case OPT_weak_l:
+      addLibrary(arg->getValue(), opt.getID() == OPT_weak_l);
+      break;
+    case OPT_framework:
+    case OPT_weak_framework:
+      addFramework(arg->getValue(), opt.getID() == OPT_weak_framework);
+      break;
+    default:
+      break;
+    }
+  }
+}
+
 bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
                  raw_ostream &stdoutOS, raw_ostream &stderrOS) {
   lld::stdoutOS = &stdoutOS;
@@ -815,6 +859,19 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
   symtab = make<SymbolTable>();
   target = createTargetInfo(args);
 
+  depTracker =
+      make<DependencyTracker>(args.getLastArgValue(OPT_dependency_info, ""));
+
+  if (auto *arg = args.getLastArg(OPT_threads_eq)) {
+    StringRef v(arg->getValue());
+    unsigned threads = 0;
+    if (!llvm::to_integer(v, threads, 0) || threads == 0)
+      error(arg->getSpelling() + ": expected a positive integer, but got '" +
+            arg->getValue() + "'");
+    parallel::strategy = hardware_concurrency(threads);
+    // FIXME: use this to configure ThinLTO concurrency too
+  }
+
   config->entry = symtab->addUndefined(args.getLastArgValue(OPT_e, "_main"),
                                        /*file=*/nullptr,
                                        /*isWeakRef=*/false);
@@ -826,6 +883,7 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
   for (const Arg *arg : args.filtered(OPT_U))
     symtab->addDynamicLookup(arg->getValue());
 
+  config->mapFile = args.getLastArgValue(OPT_map);
   config->outputFile = args.getLastArgValue(OPT_o, "a.out");
   config->headerPad = args::getHex(args, OPT_headerpad, /*Default=*/32);
   config->headerPadMaxInstallNames =
@@ -847,6 +905,7 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
   config->forceLoadObjC = args.hasArg(OPT_ObjC);
   config->demangle = args.hasArg(OPT_demangle);
   config->implicitDylibs = !args.hasArg(OPT_no_implicit_dylibs);
+  config->emitFunctionStarts = !args.hasArg(OPT_no_function_starts);
 
   if (const Arg *arg = args.getLastArg(OPT_install_name)) {
     if (config->outputType != MH_DYLIB)
@@ -939,50 +998,18 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
   config->progName = argsArr[0];
 
   config->timeTraceEnabled = args.hasArg(OPT_time_trace);
+  config->timeTraceGranularity =
+      args::getInteger(args, OPT_time_trace_granularity_eq, 500);
 
   // Initialize time trace profiler.
   if (config->timeTraceEnabled)
     timeTraceProfilerInitialize(config->timeTraceGranularity, config->progName);
 
   {
-    llvm::TimeTraceScope timeScope("Link", StringRef("ExecuteLinker"));
+    TimeTraceScope timeScope("ExecuteLinker");
 
     initLLVM(); // must be run before any call to addFile()
-
-    // This loop should be reserved for options whose exact ordering matters.
-    // Other options should be handled via filtered() and/or getLastArg().
-    for (const Arg *arg : args) {
-      const Option &opt = arg->getOption();
-      warnIfDeprecatedOption(opt);
-      warnIfUnimplementedOption(opt);
-
-      switch (opt.getID()) {
-      case OPT_INPUT:
-        addFile(arg->getValue(), false);
-        break;
-      case OPT_weak_library:
-        if (auto *dylibFile =
-                dyn_cast_or_null<DylibFile>(addFile(arg->getValue(), false)))
-          dylibFile->forceWeakImport = true;
-        break;
-      case OPT_filelist:
-        addFileList(arg->getValue());
-        break;
-      case OPT_force_load:
-        addFile(arg->getValue(), true);
-        break;
-      case OPT_l:
-      case OPT_weak_l:
-        addLibrary(arg->getValue(), opt.getID() == OPT_weak_l);
-        break;
-      case OPT_framework:
-      case OPT_weak_framework:
-        addFramework(arg->getValue(), opt.getID() == OPT_weak_framework);
-        break;
-      default:
-        break;
-      }
-    }
+    createFiles(args);
 
     config->isPic = config->outputType == MH_DYLIB ||
                     config->outputType == MH_BUNDLE || isPie(args);
@@ -1030,16 +1057,19 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
         return false;
       }
     }
+    // Literal exported-symbol names must be defined, but glob
+    // patterns need not match.
+    for (const CachedHashStringRef &cachedName :
+         config->exportedSymbols.literals) {
+      if (const Symbol *sym = symtab->find(cachedName))
+        if (isa<Defined>(sym))
+          continue;
+      error("undefined symbol " + cachedName.val() +
+            "\n>>> referenced from option -exported_symbol(s_list)");
+    }
 
     createSyntheticSections();
-
-    // The Itanium C++ ABI requires dylibs to pass a pointer to __cxa_atexit
-    // which does e.g. cleanup of static global variables. The ABI document says
-    // that the pointer can point to any address in one of the dylib's segments,
-    // but in practice ld64 seems to set it to point to the header, so that's
-    // what's implemented here.
-    symtab->addSynthetic("___dso_handle", in.header->isec, 0,
-                         /*privateExtern=*/true, /*linkerInternal=*/true);
+    createSyntheticSymbols();
 
     for (const Arg *arg : args.filtered(OPT_sectcreate)) {
       StringRef segName = arg->getValue(0);
@@ -1050,18 +1080,23 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
         inputFiles.insert(make<OpaqueFile>(*buffer, segName, sectName));
     }
 
-    // Initialize InputSections.
-    for (const InputFile *file : inputFiles) {
-      for (const SubsectionMap &map : file->subsections) {
-        for (const auto &p : map) {
-          InputSection *isec = p.second;
-          inputSections.push_back(isec);
+    {
+      TimeTraceScope timeScope("Gathering input sections");
+      // Gather all InputSections into one vector.
+      for (const InputFile *file : inputFiles) {
+        for (const SubsectionMap &map : file->subsections) {
+          for (const auto &p : map) {
+            InputSection *isec = p.second;
+            inputSections.push_back(isec);
+          }
         }
       }
     }
 
     // Write to an output file.
     writeResult();
+
+    depTracker->write(getLLDVersion(), inputFiles, config->outputFile);
   }
 
   if (config->timeTraceEnabled) {
diff --git a/lld/MachO/Driver.h b/lld/MachO/Driver.h
index 8176e9828035502bd629291d7208556268a63f3d..89ad82e0c9905039b7b7e3bef9d3bb31fece375a 100644
--- a/lld/MachO/Driver.h
+++ b/lld/MachO/Driver.h
@@ -11,10 +11,14 @@
 
 #include "lld/Common/LLVM.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Option/OptTable.h"
 #include "llvm/Support/MemoryBuffer.h"
 
+#include <set>
+#include <type_traits>
+
 namespace llvm {
 namespace MachO {
 class InterfaceFile;
@@ -61,6 +65,52 @@ uint32_t getModTime(llvm::StringRef path);
 
 void printArchiveMemberLoad(StringRef reason, const InputFile *);
 
+// Helper class to export dependency info.
+class DependencyTracker {
+public:
+  explicit DependencyTracker(llvm::StringRef path);
+
+  // Adds the given path to the set of not-found files.
+  inline void logFileNotFound(std::string path) {
+    if (active)
+      notFounds.insert(std::move(path));
+  }
+
+  inline void logFileNotFound(const Twine &path) {
+    if (active)
+      notFounds.insert(path.str());
+  }
+
+  // Writes the dependencies to specified path.
+  // The content is sorted by its Op Code, then within each section,
+  // alphabetical order.
+  void write(llvm::StringRef version,
+             const llvm::SetVector<InputFile *> &inputs,
+             llvm::StringRef output);
+
+private:
+  enum DepOpCode : uint8_t {
+    // Denotes the linker version.
+    Version = 0x00,
+    // Denotes the input files.
+    Input = 0x10,
+    // Denotes the files that do not exist(?)
+    NotFound = 0x11,
+    // Denotes the output files.
+    Output = 0x40,
+  };
+
+  const llvm::StringRef path;
+  bool active;
+
+  // The paths need to be alphabetically ordered.
+  // We need to own the paths because some of them are temporarily
+  // constructed.
+  std::set<std::string> notFounds;
+};
+
+extern DependencyTracker *depTracker;
+
 } // namespace macho
 } // namespace lld
 
diff --git a/lld/MachO/DriverUtils.cpp b/lld/MachO/DriverUtils.cpp
index faa9b760d904887e0ef1dfd889c9d977ecf655ce..49bd83ecf09a18aef47ef1c3cdff5eb8dbe55e14 100644
--- a/lld/MachO/DriverUtils.cpp
+++ b/lld/MachO/DriverUtils.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/TextAPI/MachO/InterfaceFile.h"
 #include "llvm/TextAPI/MachO/TextAPIReader.h"
@@ -164,12 +165,15 @@ Optional<std::string> macho::resolveDylibPath(StringRef path) {
   // they are consistent.
   if (fs::exists(path))
     return std::string(path);
+  else
+    depTracker->logFileNotFound(path);
 
   SmallString<261> location = path;
   path::replace_extension(location, ".tbd");
   if (fs::exists(location))
     return std::string(location);
-
+  else
+    depTracker->logFileNotFound(location);
   return {};
 }
 
@@ -240,3 +244,53 @@ void macho::printArchiveMemberLoad(StringRef reason, const InputFile *f) {
   if (config->printWhyLoad)
     message(reason + " forced load of " + toString(f));
 }
+
+macho::DependencyTracker::DependencyTracker(StringRef path)
+    : path(path), active(!path.empty()) {
+  if (active && fs::exists(path) && !fs::can_write(path)) {
+    warn("Ignoring dependency_info option since specified path is not "
+         "writeable.");
+    active = false;
+  }
+}
+
+void macho::DependencyTracker::write(llvm::StringRef version,
+                                     const llvm::SetVector<InputFile *> &inputs,
+                                     llvm::StringRef output) {
+  if (!active)
+    return;
+
+  std::error_code ec;
+  llvm::raw_fd_ostream os(path, ec, llvm::sys::fs::OF_None);
+  if (ec) {
+    warn("Error writing dependency info to file");
+    return;
+  }
+
+  auto addDep = [&os](DepOpCode opcode, const StringRef &path) {
+    // XXX: Even though DepOpCode's underlying type is uint8_t,
+    // this cast is still needed because Clang older than 10.x has a bug,
+    // where it doesn't know to cast the enum to its underlying type.
+    // Hence `<< DepOpCode` is ambiguous to it.
+    os << static_cast<uint8_t>(opcode);
+    os << path;
+    os << '\0';
+  };
+
+  addDep(DepOpCode::Version, version);
+
+  // Sort the input by its names.
+  std::vector<StringRef> inputNames;
+  inputNames.reserve(inputs.size());
+  for (InputFile *f : inputs)
+    inputNames.push_back(f->getName());
+  llvm::sort(inputNames);
+
+  for (const StringRef &in : inputNames)
+    addDep(DepOpCode::Input, in);
+
+  for (const std::string &f : notFounds)
+    addDep(DepOpCode::NotFound, f);
+
+  addDep(DepOpCode::Output, output);
+}
diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp
index 0cf5328e2d6c1760e7813a71764360d4d6de0b7b..bb42189a1f5e2ac633887e9dfc446fa9b9de2968 100644
--- a/lld/MachO/LTO.cpp
+++ b/lld/MachO/LTO.cpp
@@ -36,6 +36,8 @@ static lto::Config createConfig() {
   c.PreCodeGenPassesHook = [](legacy::PassManager &pm) {
     pm.add(createObjCARCContractPass());
   };
+  c.TimeTraceEnabled = config->timeTraceEnabled;
+  c.TimeTraceGranularity = config->timeTraceGranularity;
   return c;
 }
 
@@ -99,9 +101,8 @@ std::vector<ObjFile *> BitcodeCompiler::compile() {
 
   std::vector<ObjFile *> ret;
   for (unsigned i = 0; i != maxTasks; ++i) {
-    if (buf[i].empty()) {
+    if (buf[i].empty())
       continue;
-    }
     SmallString<261> filePath("/tmp/lto.tmp");
     uint32_t modTime = 0;
     if (!config->ltoObjPath.empty()) {
diff --git a/lld/MachO/MapFile.cpp b/lld/MachO/MapFile.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a10516f90d192697d6465ce423ca027e91ec9d52
--- /dev/null
+++ b/lld/MachO/MapFile.cpp
@@ -0,0 +1,154 @@
+//===- MapFile.cpp --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the -map option. It shows lists in order and
+// hierarchically the outputFile, arch, input files, output sections and
+// symbol:
+//
+// # Path: test
+// # Arch: x86_84
+// # Object files:
+// [  0] linker synthesized
+// [  1] a.o
+// # Sections:
+// # Address  Size      Segment  Section
+// 0x1000005C0  0x0000004C  __TEXT  __text
+// # Symbols:
+// # Address  File  Name
+// 0x1000005C0  [  1] _main
+//
+//===----------------------------------------------------------------------===//
+
+#include "MapFile.h"
+#include "Config.h"
+#include "InputFiles.h"
+#include "InputSection.h"
+#include "OutputSection.h"
+#include "OutputSegment.h"
+#include "Symbols.h"
+#include "Target.h"
+#include "llvm/Support/Parallel.h"
+#include "llvm/Support/TimeProfiler.h"
+
+using namespace llvm;
+using namespace llvm::sys;
+using namespace lld;
+using namespace lld::macho;
+
+using SymbolMapTy = DenseMap<const InputSection *, SmallVector<Defined *, 4>>;
+
+// Returns a map from sections to their symbols.
+static SymbolMapTy getSectionSyms(ArrayRef<Defined *> syms) {
+  SymbolMapTy ret;
+  for (Defined *dr : syms)
+    ret[dr->isec].push_back(dr);
+
+  // Sort symbols by address. We want to print out symbols in the
+  // order in the output file rather than the order they appeared
+  // in the input files.
+  for (auto &it : ret)
+    llvm::stable_sort(it.second, [](Defined *a, Defined *b) {
+      return a->getVA() < b->getVA();
+    });
+  return ret;
+}
+
+// Returns a list of all symbols that we want to print out.
+static std::vector<Defined *> getSymbols() {
+  std::vector<Defined *> v;
+  for (InputFile *file : inputFiles)
+    if (isa<ObjFile>(file))
+      for (Symbol *sym : file->symbols) {
+        if (sym == nullptr)
+          continue;
+        if (auto *d = dyn_cast<Defined>(sym))
+          if (d->isec && d->getFile() == file)
+            v.push_back(d);
+      }
+  return v;
+}
+
+// Construct a map from symbols to their stringified representations.
+// Demangling symbols (which is what toString() does) is slow, so
+// we do that in batch using parallel-for.
+static DenseMap<macho::Symbol *, std::string>
+getSymbolStrings(ArrayRef<Defined *> syms) {
+  std::vector<std::string> str(syms.size());
+  parallelForEachN(0, syms.size(), [&](size_t i) {
+    raw_string_ostream os(str[i]);
+    os << toString(*syms[i]);
+  });
+
+  DenseMap<macho::Symbol *, std::string> ret;
+  for (size_t i = 0, e = syms.size(); i < e; ++i)
+    ret[syms[i]] = std::move(str[i]);
+  return ret;
+}
+
+void macho::writeMapFile() {
+  if (config->mapFile.empty())
+    return;
+
+  TimeTraceScope timeScope("Write map file");
+
+  // Open a map file for writing.
+  std::error_code ec;
+  raw_fd_ostream os(config->mapFile, ec, sys::fs::OF_None);
+  if (ec) {
+    error("cannot open " + config->mapFile + ": " + ec.message());
+    return;
+  }
+
+  // Dump output path
+  os << format("# Path: %s\n", config->outputFile.str().c_str());
+
+  // Dump output architecure
+  os << format("# Arch: %s\n",
+               getArchitectureName(config->target.Arch).str().c_str());
+
+  // Dump table of object files
+  os << "# Object files:\n";
+  os << format("[%3u] %s\n", 0, (const char *)"linker synthesized");
+  uint32_t fileIndex = 1;
+  DenseMap<lld::macho::InputFile *, uint32_t> readerToFileOrdinal;
+  for (InputFile *file : inputFiles) {
+    if (isa<ObjFile>(file)) {
+      os << format("[%3u] %s\n", fileIndex, file->getName().str().c_str());
+      readerToFileOrdinal[file] = fileIndex++;
+    }
+  }
+
+  // Collect symbol info that we want to print out.
+  std::vector<Defined *> syms = getSymbols();
+  SymbolMapTy sectionSyms = getSectionSyms(syms);
+  DenseMap<lld::macho::Symbol *, std::string> symStr = getSymbolStrings(syms);
+
+  // Dump table of sections
+  os << "# Sections:\n";
+  os << "# Address\tSize    \tSegment\tSection\n";
+  for (OutputSegment *seg : outputSegments)
+    for (OutputSection *osec : seg->getSections()) {
+      if (osec->isHidden())
+        continue;
+
+      os << format("0x%08llX\t0x%08llX\t%s\t%s\n", osec->addr, osec->getSize(),
+                   seg->name.str().c_str(), osec->name.str().c_str());
+    }
+
+  // Dump table of symbols
+  os << "# Symbols:\n";
+  os << "# Address\t    File  Name\n";
+  for (InputSection *isec : inputSections) {
+    for (macho::Symbol *sym : sectionSyms[isec]) {
+      os << format("0x%08llX\t[%3u] %s\n", sym->getVA(),
+                   readerToFileOrdinal[sym->getFile()], symStr[sym].c_str());
+    }
+  }
+
+  // TODO: when we implement -dead_strip, we should dump dead stripped symbols
+}
diff --git a/lld/MachO/MapFile.h b/lld/MachO/MapFile.h
new file mode 100644
index 0000000000000000000000000000000000000000..bf16ffdd03823590a7ea4ec501d5981031ecfcb7
--- /dev/null
+++ b/lld/MachO/MapFile.h
@@ -0,0 +1,18 @@
+//===- MapFile.h ------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_MAPFILE_H
+#define LLD_MACHO_MAPFILE_H
+
+namespace lld {
+namespace macho {
+void writeMapFile();
+} // namespace macho
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/MergedOutputSection.cpp b/lld/MachO/MergedOutputSection.cpp
index 15e957250dd0ed27b395f4a3b7d1d156a229b5b1..9a74925ce9670d31ea55c5fc8f54b3ed22b5aff6 100644
--- a/lld/MachO/MergedOutputSection.cpp
+++ b/lld/MachO/MergedOutputSection.cpp
@@ -45,9 +45,8 @@ void MergedOutputSection::finalize() {
 }
 
 void MergedOutputSection::writeTo(uint8_t *buf) const {
-  for (InputSection *isec : inputs) {
+  for (InputSection *isec : inputs)
     isec->writeTo(buf + isec->outSecFileOff);
-  }
 }
 
 // TODO: this is most likely wrong; reconsider how section flags
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 6af0d2c4152fa916eb6efe7c119dd6fe127ba3b8..da2c53e2e492994c4cf449d5768f00bbb3b6fe1f 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -20,6 +20,9 @@ def color_diagnostics_eq: Joined<["--"], "color-diagnostics=">,
     HelpText<"Use colors in diagnostics (default: auto)">,
     MetaVarName<"[auto,always,never]">,
     Group<grp_lld>;
+def threads_eq : Joined<["--"], "threads=">,
+    HelpText<"Number of threads. '1' disables multi-threading. By default all available hardware threads are used">,
+    Group<grp_lld>;
 def reproduce: Separate<["--"], "reproduce">,
     Group<grp_lld>;
 def reproduce_eq: Joined<["--"], "reproduce=">,
@@ -36,9 +39,9 @@ def no_lto_legacy_pass_manager : Flag<["--"], "no-lto-legacy-pass-manager">,
     HelpText<"Use the new pass manager in LLVM">,
     Group<grp_lld>;
 def time_trace: Flag<["--"], "time-trace">, HelpText<"Record time trace">;
-def time_trace_granularity: Flag<["--"], "time-trace-granularity">,
-  HelpText<"Minimum time granularity (in microseconds) traced by time profiler">;
-def time_trace_file_eq: Flag<["--"], "time-trace-file=">, HelpText<"Specify time trace output file">;
+def time_trace_granularity_eq: Joined<["--"], "time-trace-granularity=">,
+    HelpText<"Minimum time granularity (in microseconds) traced by time profiler">;
+def time_trace_file_eq: Joined<["--"], "time-trace-file=">, HelpText<"Specify time trace output file">;
 
 // This is a complete Options.td compiled from Apple's ld(1) manpage
 // dated 2018-03-07 and cross checked with ld64 source code in repo
@@ -53,1265 +56,1262 @@ def time_trace_file_eq: Flag<["--"], "time-trace-file=">, HelpText<"Specify time
 def grp_kind : OptionGroup<"kind">, HelpText<"OUTPUT KIND">;
 
 def execute : Flag<["-"], "execute">,
-     HelpText<"Produce a main executable (default)">,
-     Group<grp_kind>;
+    HelpText<"Produce a main executable (default)">,
+    Group<grp_kind>;
 def dylib : Flag<["-"], "dylib">,
-     HelpText<"Produce a shared library">,
-     Group<grp_kind>;
+    HelpText<"Produce a shared library">,
+    Group<grp_kind>;
 def bundle : Flag<["-"], "bundle">,
-     HelpText<"Produce a bundle">,
-     Group<grp_kind>;
+    HelpText<"Produce a bundle">,
+    Group<grp_kind>;
 def r : Flag<["-"], "r">,
-     HelpText<"Merge multiple object files into one, retaining relocations">,
-     Flags<[HelpHidden]>,
-     Group<grp_kind>;
+    HelpText<"Merge multiple object files into one, retaining relocations">,
+    Flags<[HelpHidden]>,
+    Group<grp_kind>;
 def dylinker : Flag<["-"], "dylinker">,
-     HelpText<"Produce a dylinker only used when building dyld">,
-     Flags<[HelpHidden]>,
-     Group<grp_kind>;
+    HelpText<"Produce a dylinker only used when building dyld">,
+    Flags<[HelpHidden]>,
+    Group<grp_kind>;
 def dynamic : Flag<["-"], "dynamic">,
-     HelpText<"Link dynamically (default)">,
-     Group<grp_kind>;
+    HelpText<"Link dynamically (default)">,
+    Group<grp_kind>;
 def static : Flag<["-"], "static">,
-     HelpText<"Link statically">,
-     Flags<[HelpHidden]>,
-     Group<grp_kind>;
+    HelpText<"Link statically">,
+    Flags<[HelpHidden]>,
+    Group<grp_kind>;
 def preload : Flag<["-"], "preload">,
-     HelpText<"Produce an unsegmented binary for embedded systems">,
-     Flags<[HelpHidden]>,
-     Group<grp_kind>;
+    HelpText<"Produce an unsegmented binary for embedded systems">,
+    Flags<[HelpHidden]>,
+    Group<grp_kind>;
 def arch : Separate<["-"], "arch">,
-     MetaVarName<"<arch_name>">,
-     HelpText<"The architecture (e.g. ppc, ppc64, i386, x86_64)">,
-     Group<grp_kind>;
+    MetaVarName<"<arch_name>">,
+    HelpText<"The architecture (e.g. ppc, ppc64, i386, x86_64)">,
+    Group<grp_kind>;
 def o : Separate<["-"], "o">,
-     MetaVarName<"<path>">,
-     HelpText<"The name of the output file (default: `a.out')">,
-     Group<grp_kind>;
+    MetaVarName<"<path>">,
+    HelpText<"The name of the output file (default: `a.out')">,
+    Group<grp_kind>;
 
 def grp_libs : OptionGroup<"libs">, HelpText<"LIBRARIES">;
 
 def l : Joined<["-"], "l">,
-     MetaVarName<"<name>">,
-     HelpText<"Search for lib<name>.dylib or lib<name>.a on the library search path">,
-     Group<grp_libs>;
+    MetaVarName<"<name>">,
+    HelpText<"Search for lib<name>.dylib or lib<name>.a on the library search path">,
+    Group<grp_libs>;
 def weak_l : Joined<["-"], "weak-l">,
-     MetaVarName<"<name>">,
-     HelpText<"Like -l<name>, but mark library and its references as weak imports">,
-     Group<grp_libs>;
+    MetaVarName<"<name>">,
+    HelpText<"Like -l<name>, but mark library and its references as weak imports">,
+    Group<grp_libs>;
 def weak_library : Separate<["-"], "weak_library">,
-     MetaVarName<"<path>">,
-     HelpText<"Like bare <path>, but mark library and its references as weak imports">,
-     Group<grp_libs>;
+    MetaVarName<"<path>">,
+    HelpText<"Like bare <path>, but mark library and its references as weak imports">,
+    Group<grp_libs>;
 def reexport_l : Joined<["-"], "reexport-l">,
-     MetaVarName<"<name>">,
-     HelpText<"Like -l<name>, but export all symbols of <name> from newly created library">,
-     Flags<[HelpHidden]>,
-     Group<grp_libs>;
+    MetaVarName<"<name>">,
+    HelpText<"Like -l<name>, but export all symbols of <name> from newly created library">,
+    Flags<[HelpHidden]>,
+    Group<grp_libs>;
 def reexport_library : Separate<["-"], "reexport_library">,
-     MetaVarName<"<path>">,
-     HelpText<"Like bare <path>, but export all symbols of <path> from newly created library">,
-     Flags<[HelpHidden]>,
-     Group<grp_libs>;
+    MetaVarName<"<path>">,
+    HelpText<"Like bare <path>, but export all symbols of <path> from newly created library">,
+    Flags<[HelpHidden]>,
+    Group<grp_libs>;
 def upward_l : Joined<["-"], "upward-l">,
-     MetaVarName<"<name>">,
-     HelpText<"Like -l<name>, but specify dylib as an upward dependency">,
-     Flags<[HelpHidden]>,
-     Group<grp_libs>;
+    MetaVarName<"<name>">,
+    HelpText<"Like -l<name>, but specify dylib as an upward dependency">,
+    Flags<[HelpHidden]>,
+    Group<grp_libs>;
 def upward_library : Separate<["-"], "upward_library">,
-     MetaVarName<"<path>">,
-     HelpText<"Like bare <path>, but specify dylib as an upward dependency">,
-     Flags<[HelpHidden]>,
-     Group<grp_libs>;
+    MetaVarName<"<path>">,
+    HelpText<"Like bare <path>, but specify dylib as an upward dependency">,
+    Flags<[HelpHidden]>,
+    Group<grp_libs>;
 def L : JoinedOrSeparate<["-"], "L">,
-     MetaVarName<"<dir>">,
-     HelpText<"Add dir to the library search path">,
-     Group<grp_libs>;
+    MetaVarName<"<dir>">,
+    HelpText<"Add dir to the library search path">,
+    Group<grp_libs>;
 def Z : Flag<["-"], "Z">,
-     HelpText<"Remove standard directories from the library and framework search paths">,
-     Group<grp_libs>;
+    HelpText<"Remove standard directories from the library and framework search paths">,
+    Group<grp_libs>;
 def syslibroot : Separate<["-"], "syslibroot">,
-     MetaVarName<"<rootdir>">,
-     HelpText<"Prepend <rootdir> to all library and framework search paths">,
-     Group<grp_libs>;
+    MetaVarName<"<rootdir>">,
+    HelpText<"Prepend <rootdir> to all library and framework search paths">,
+    Group<grp_libs>;
 def search_paths_first : Flag<["-"], "search_paths_first">,
-     HelpText<"Search for lib<name>.dylib and lib<name>.a at each step in traversing search path (default for Xcode 4 and later)">,
-     Group<grp_libs>;
+    HelpText<"Search for lib<name>.dylib and lib<name>.a at each step in traversing search path (default for Xcode 4 and later)">,
+    Group<grp_libs>;
 def search_dylibs_first : Flag<["-"], "search_dylibs_first">,
-     HelpText<"Search for lib<name>.dylib on first pass, then for lib<name>.a on second pass through search path (default for Xcode 3 and earlier)">,
-     Group<grp_libs>;
+    HelpText<"Search for lib<name>.dylib on first pass, then for lib<name>.a on second pass through search path (default for Xcode 3 and earlier)">,
+    Group<grp_libs>;
 def framework : Separate<["-"], "framework">,
-     MetaVarName<"<name>">,
-     HelpText<"Search for <name>.framework/<name> on the framework search path">,
-     Group<grp_libs>;
+    MetaVarName<"<name>">,
+    HelpText<"Search for <name>.framework/<name> on the framework search path">,
+    Group<grp_libs>;
 def weak_framework : Separate<["-"], "weak_framework">,
-     MetaVarName<"<name>">,
-     HelpText<"Like -framework <name>, but mark framework and its references as weak imports">,
-     Group<grp_libs>;
+    MetaVarName<"<name>">,
+    HelpText<"Like -framework <name>, but mark framework and its references as weak imports">,
+    Group<grp_libs>;
 def reexport_framework : Separate<["-"], "reexport_framework">,
-     MetaVarName<"<name>">,
-     HelpText<"Like -framework <name>, but export all symbols of <name> from the newly created library">,
-     Flags<[HelpHidden]>,
-     Group<grp_libs>;
+    MetaVarName<"<name>">,
+    HelpText<"Like -framework <name>, but export all symbols of <name> from the newly created library">,
+    Flags<[HelpHidden]>,
+    Group<grp_libs>;
 def upward_framework : Separate<["-"], "upward_framework">,
-     MetaVarName<"<name>">,
-     HelpText<"Like -framework <name>, but specify the framework as an upward dependency">,
-     Flags<[HelpHidden]>,
-     Group<grp_libs>;
+    MetaVarName<"<name>">,
+    HelpText<"Like -framework <name>, but specify the framework as an upward dependency">,
+    Flags<[HelpHidden]>,
+    Group<grp_libs>;
 def F : JoinedOrSeparate<["-"], "F">,
-     MetaVarName<"<dir>">,
-     HelpText<"Add dir to the framework search path">,
-     Group<grp_libs>;
+    MetaVarName<"<dir>">,
+    HelpText<"Add dir to the framework search path">,
+    Group<grp_libs>;
 def all_load : Flag<["-"], "all_load">,
-     HelpText<"Load all members of all static archive libraries">,
-     Group<grp_libs>;
+    HelpText<"Load all members of all static archive libraries">,
+    Group<grp_libs>;
 def ObjC : Flag<["-"], "ObjC">,
-     HelpText<"Load all members of static archives that are an Objective-C class or category.">,
-     Group<grp_libs>;
+    HelpText<"Load all members of static archives that are an Objective-C class or category.">,
+    Group<grp_libs>;
 def force_load : Separate<["-"], "force_load">,
-     MetaVarName<"<path>">,
-     HelpText<"Load all members static archive library at <path>">,
-     Group<grp_libs>;
+    MetaVarName<"<path>">,
+    HelpText<"Load all members static archive library at <path>">,
+    Group<grp_libs>;
 
 def grp_content : OptionGroup<"content">, HelpText<"ADDITIONAL CONTENT">;
 
 def sectcreate : MultiArg<["-"], "sectcreate", 3>,
-     MetaVarName<"<segment> <section> <file>">,
-     HelpText<"Create <section> in <segment> from the contents of <file>">,
-     Group<grp_content>;
+    MetaVarName<"<segment> <section> <file>">,
+    HelpText<"Create <section> in <segment> from the contents of <file>">,
+    Group<grp_content>;
 def segcreate : MultiArg<["-"], "segcreate", 3>,
-     MetaVarName<"<segment> <section> <file>">,
-     Alias<sectcreate>,
-     HelpText<"Alias for -sectcreate">,
-     Flags<[HelpHidden]>,
-     Group<grp_content>;
+    MetaVarName<"<segment> <section> <file>">,
+    Alias<sectcreate>,
+    HelpText<"Alias for -sectcreate">,
+    Flags<[HelpHidden]>,
+    Group<grp_content>;
 def filelist : Separate<["-"], "filelist">,
-     MetaVarName<"<file>">,
-     HelpText<"Read names of files to link from <file>">,
-     Group<grp_content>;
+    MetaVarName<"<file>">,
+    HelpText<"Read names of files to link from <file>">,
+    Group<grp_content>;
 def dtrace : Separate<["-"], "dtrace">,
-     MetaVarName<"<script>">,
-     HelpText<"Enable DTrace static probes according to declarations in <script>">,
-     Flags<[HelpHidden]>,
-     Group<grp_content>;
+    MetaVarName<"<script>">,
+    HelpText<"Enable DTrace static probes according to declarations in <script>">,
+    Flags<[HelpHidden]>,
+    Group<grp_content>;
 
 def grp_opts : OptionGroup<"opts">, HelpText<"OPTIMIZATIONS">;
 
 def dead_strip : Flag<["-"], "dead_strip">,
-     HelpText<"Remove unreachable functions and data">,
-     Flags<[HelpHidden]>,
-     Group<grp_opts>;
+    HelpText<"Remove unreachable functions and data">,
+    Flags<[HelpHidden]>,
+    Group<grp_opts>;
 def order_file : Separate<["-"], "order_file">,
-     MetaVarName<"<file>">,
-     HelpText<"Layout functions and data according to specification in <file>">,
-     Group<grp_opts>;
+    MetaVarName<"<file>">,
+    HelpText<"Layout functions and data according to specification in <file>">,
+    Group<grp_opts>;
 def no_order_inits : Flag<["-"], "no_order_inits">,
-     HelpText<"Disable default reordering of initializer and terminator functions">,
-     Flags<[HelpHidden]>,
-     Group<grp_opts>;
+    HelpText<"Disable default reordering of initializer and terminator functions">,
+    Flags<[HelpHidden]>,
+    Group<grp_opts>;
 def no_order_data : Flag<["-"], "no_order_data">,
-     HelpText<"Disable default reordering of global data accessed at launch time">,
-     Flags<[HelpHidden]>,
-     Group<grp_opts>;
+    HelpText<"Disable default reordering of global data accessed at launch time">,
+    Flags<[HelpHidden]>,
+    Group<grp_opts>;
 def image_base : Separate<["-"], "image_base">,
-     MetaVarName<"<address>">,
-     HelpText<"Preferred hex load address for a dylib or bundle.">,
-     Flags<[HelpHidden]>,
-     Group<grp_opts>;
+    MetaVarName<"<address>">,
+    HelpText<"Preferred hex load address for a dylib or bundle.">,
+    Flags<[HelpHidden]>,
+    Group<grp_opts>;
 def seg1addr : Separate<["-"], "seg1addr">,
-     MetaVarName<"<address>">,
-     Alias<image_base>,
-     HelpText<"Alias for -image_base">,
-     Flags<[HelpHidden]>,
-     Group<grp_opts>;
+    MetaVarName<"<address>">,
+    Alias<image_base>,
+    HelpText<"Alias for -image_base">,
+    Flags<[HelpHidden]>,
+    Group<grp_opts>;
 def no_implicit_dylibs : Flag<["-"], "no_implicit_dylibs">,
-     HelpText<"Do not optimize public dylib transitive symbol references">,
-     Group<grp_opts>;
+    HelpText<"Do not optimize public dylib transitive symbol references">,
+    Group<grp_opts>;
 def exported_symbols_order : Separate<["-"], "exported_symbols_order">,
-     MetaVarName<"<file>">,
-     HelpText<"Specify frequently-used symbols in <file> to optimize symbol exports">,
-     Flags<[HelpHidden]>,
-     Group<grp_opts>;
+    MetaVarName<"<file>">,
+    HelpText<"Specify frequently-used symbols in <file> to optimize symbol exports">,
+    Flags<[HelpHidden]>,
+    Group<grp_opts>;
 def no_zero_fill_sections : Flag<["-"], "no_zero_fill_sections">,
-     HelpText<"Explicitly store zeroed data in the final image">,
-     Flags<[HelpHidden]>,
-     Group<grp_opts>;
+    HelpText<"Explicitly store zeroed data in the final image">,
+    Flags<[HelpHidden]>,
+    Group<grp_opts>;
 def merge_zero_fill_sections : Flag<["-"], "merge_zero_fill_sections">,
-     HelpText<"Merge all zeroed data into the __zerofill section">,
-     Flags<[HelpHidden]>,
-     Group<grp_opts>;
+    HelpText<"Merge all zeroed data into the __zerofill section">,
+    Flags<[HelpHidden]>,
+    Group<grp_opts>;
 def no_branch_islands : Flag<["-"], "no_branch_islands">,
-     HelpText<"Disable infra for branches beyond the maximum branch distance.">,
-     Flags<[HelpHidden]>,
-     Group<grp_opts>;
+    HelpText<"Disable infra for branches beyond the maximum branch distance.">,
+    Flags<[HelpHidden]>,
+    Group<grp_opts>;
 
 def grp_version : OptionGroup<"version">, HelpText<"VERSION TARGETING">;
 
 def platform_version : MultiArg<["-"], "platform_version", 3>,
-     MetaVarName<"<platform> <min_version> <sdk_version>">,
-     HelpText<"Platform (e.g., macos, ios, tvos, watchos, bridgeos, mac-catalyst, ios-sim, tvos-sim, watchos-sim, driverkit) and version numbers">,
-     Group<grp_version>;
+    MetaVarName<"<platform> <min_version> <sdk_version>">,
+    HelpText<"Platform (e.g., macos, ios, tvos, watchos, bridgeos, mac-catalyst, ios-sim, tvos-sim, watchos-sim, driverkit) and version numbers">,
+    Group<grp_version>;
 def sdk_version : Separate<["-"], "sdk_version">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def macos_version_min : Separate<["-"], "macos_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Oldest macOS version for which linked output is usable">,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    HelpText<"Oldest macOS version for which linked output is usable">,
+    Group<grp_version>;
 def macosx_version_min : Separate<["-"], "macosx_version_min">,
-     MetaVarName<"<version>">,
-     Alias<macos_version_min>,
-     HelpText<"Alias for -macos_version_min">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    Alias<macos_version_min>,
+    HelpText<"Alias for -macos_version_min">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def ios_version_min : Separate<["-"], "ios_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Oldest iOS version for which linked output is usable">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    HelpText<"Oldest iOS version for which linked output is usable">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def ios_simulator_version_min : Separate<["-"], "ios_simulator_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Oldest iOS simulator version for which linked output is usable">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    HelpText<"Oldest iOS simulator version for which linked output is usable">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def iphoneos_version_min : Separate<["-"], "iphoneos_version_min">,
-     MetaVarName<"<version>">,
-     Alias<ios_version_min>,
-     HelpText<"Alias for -ios_version_min">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    Alias<ios_version_min>,
+    HelpText<"Alias for -ios_version_min">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def maccatalyst_version_min : Separate<["-"], "maccatalyst_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Oldest MacCatalyst version for which linked output is usable">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    HelpText<"Oldest MacCatalyst version for which linked output is usable">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def iosmac_version_min : Separate<["-"], "iosmac_version_min">,
-     MetaVarName<"<version>">,
-     Alias<maccatalyst_version_min>,
-     HelpText<"Alias for -maccatalyst_version_min">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    Alias<maccatalyst_version_min>,
+    HelpText<"Alias for -maccatalyst_version_min">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def uikitformac_version_min : Separate<["-"], "uikitformac_version_min">,
-     MetaVarName<"<version>">,
-     Alias<maccatalyst_version_min>,
-     HelpText<"Alias for -maccatalyst_version_min">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    Alias<maccatalyst_version_min>,
+    HelpText<"Alias for -maccatalyst_version_min">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def tvos_version_min : Separate<["-"], "tvos_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Oldest tvOS version for which linked output is usable">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    HelpText<"Oldest tvOS version for which linked output is usable">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def watchos_version_min : Separate<["-"], "watchos_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Oldest watchOS version for which linked output is usable">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    HelpText<"Oldest watchOS version for which linked output is usable">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def bridgeos_version_min : Separate<["-"], "bridgeos_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Oldest bridgeOS version for which linked output is usable">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    HelpText<"Oldest bridgeOS version for which linked output is usable">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 def driverkit_version_min : Separate<["-"], "driverkit_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Oldest DriverKit version for which linked output is usable">,
-     Flags<[HelpHidden]>,
-     Group<grp_version>;
+    MetaVarName<"<version>">,
+    HelpText<"Oldest DriverKit version for which linked output is usable">,
+    Flags<[HelpHidden]>,
+    Group<grp_version>;
 
 def grp_dylib : OptionGroup<"dylib">, HelpText<"DYNAMIC LIBRARIES (DYLIB)">;
 
 def install_name : Separate<["-"], "install_name">,
-     MetaVarName<"<name>">,
-     HelpText<"Set an internal install path in a dylib">,
-     Group<grp_dylib>;
+    MetaVarName<"<name>">,
+    HelpText<"Set an internal install path in a dylib">,
+    Group<grp_dylib>;
 def dylib_install_name : Separate<["-"], "dylib_install_name">,
-     MetaVarName<"<name>">,
-     Alias<install_name>,
-     HelpText<"Alias for -install_name">,
-     Flags<[HelpHidden]>,
-     Group<grp_dylib>;
+    MetaVarName<"<name>">,
+    Alias<install_name>,
+    HelpText<"Alias for -install_name">,
+    Flags<[HelpHidden]>,
+    Group<grp_dylib>;
 def dylinker_install_name : Separate<["-"], "dylinker_install_name">,
-     MetaVarName<"<name>">,
-     Alias<install_name>,
-     HelpText<"Alias for -install_name">,
-     Flags<[HelpHidden]>,
-     Group<grp_dylib>;
+    MetaVarName<"<name>">,
+    Alias<install_name>,
+    HelpText<"Alias for -install_name">,
+    Flags<[HelpHidden]>,
+    Group<grp_dylib>;
 def mark_dead_strippable_dylib : Flag<["-"], "mark_dead_strippable_dylib">,
-     HelpText<"Mark output dylib as dead-strippable: When a client links against it but does not use any of its symbols, the dylib will not be added to the client's list of needed dylibs">,
-     Group<grp_dylib>;
+    HelpText<"Mark output dylib as dead-strippable: When a client links against it but does not use any of its symbols, the dylib will not be added to the client's list of needed dylibs">,
+    Group<grp_dylib>;
 def compatibility_version : Separate<["-"], "compatibility_version">,
-     MetaVarName<"<version>">,
-     HelpText<"Compatibility <version> of this library">,
-     Group<grp_dylib>;
+    MetaVarName<"<version>">,
+    HelpText<"Compatibility <version> of this library">,
+    Group<grp_dylib>;
 def dylib_compatibility_version : Separate<["-"], "dylib_compatibility_version">,
-     MetaVarName<"<version>">,
-     Alias<compatibility_version>,
-     HelpText<"Alias for -compatibility_version">,
-     Flags<[HelpHidden]>,
-     Group<grp_dylib>;
+    MetaVarName<"<version>">,
+    Alias<compatibility_version>,
+    HelpText<"Alias for -compatibility_version">,
+    Flags<[HelpHidden]>,
+    Group<grp_dylib>;
 def current_version : Separate<["-"], "current_version">,
-     MetaVarName<"<version>">,
-     HelpText<"Current <version> of this library">,
-     Group<grp_dylib>;
+    MetaVarName<"<version>">,
+    HelpText<"Current <version> of this library">,
+    Group<grp_dylib>;
 def dylib_current_version : Separate<["-"], "dylib_current_version">,
-     MetaVarName<"<version>">,
-     Alias<current_version>,
-     HelpText<"Alias for -current_version">,
-     Flags<[HelpHidden]>,
-     Group<grp_dylib>;
+    MetaVarName<"<version>">,
+    Alias<current_version>,
+    HelpText<"Alias for -current_version">,
+    Flags<[HelpHidden]>,
+    Group<grp_dylib>;
 
 def grp_main : OptionGroup<"main">, HelpText<"MAIN EXECUTABLE">;
 
 def pie : Flag<["-"], "pie">,
-     HelpText<"Build a position independent executable (default for macOS 10.7 and later)">,
-     Group<grp_main>;
+    HelpText<"Build a position independent executable (default for macOS 10.7 and later)">,
+    Group<grp_main>;
 def no_pie : Flag<["-"], "no_pie">,
-     HelpText<"Do not build a position independent executable (default for macOS 10.6 and earlier)">,
-     Group<grp_main>;
+    HelpText<"Do not build a position independent executable (default for macOS 10.6 and earlier)">,
+    Group<grp_main>;
 def pagezero_size : Separate<["-"], "pagezero_size">,
-     MetaVarName<"<size>">,
-     HelpText<"Size of unreadable segment at address zero is hex <size> (default is 4KB on 32-bit and 4GB on 64-bit)">,
-     Flags<[HelpHidden]>,
-     Group<grp_main>;
+    MetaVarName<"<size>">,
+    HelpText<"Size of unreadable segment at address zero is hex <size> (default is 4KB on 32-bit and 4GB on 64-bit)">,
+    Flags<[HelpHidden]>,
+    Group<grp_main>;
 def stack_size : Separate<["-"], "stack_size">,
-     MetaVarName<"<size>">,
-     HelpText<"Maximum hex stack size for the main thread in a program. (default is 8MB)">,
-     Flags<[HelpHidden]>,
-     Group<grp_main>;
+    MetaVarName<"<size>">,
+    HelpText<"Maximum hex stack size for the main thread in a program. (default is 8MB)">,
+    Flags<[HelpHidden]>,
+    Group<grp_main>;
 def allow_stack_execute : Flag<["-"], "allow_stack_execute">,
-     HelpText<"Mark stack segment as executable">,
-     Flags<[HelpHidden]>,
-     Group<grp_main>;
+    HelpText<"Mark stack segment as executable">,
+    Flags<[HelpHidden]>,
+    Group<grp_main>;
 def export_dynamic : Flag<["-"], "export_dynamic">,
-     HelpText<"Preserve all global symbols during LTO">,
-     Flags<[HelpHidden]>,
-     Group<grp_main>;
+    HelpText<"Preserve all global symbols during LTO">,
+    Flags<[HelpHidden]>,
+    Group<grp_main>;
 
 def grp_bundle : OptionGroup<"bundle">, HelpText<"CREATING A BUNDLE">;
 
 def bundle_loader : Separate<["-"], "bundle_loader">,
-     MetaVarName<"<executable>">,
-     HelpText<"Resolve undefined symbols from <executable>">,
-     Group<grp_bundle>;
+    MetaVarName<"<executable>">,
+    HelpText<"Resolve undefined symbols from <executable>">,
+    Group<grp_bundle>;
 
 def grp_object : OptionGroup<"object">, HelpText<"CREATING AN OBJECT FILE">;
 
 def keep_private_externs : Flag<["-"], "keep_private_externs">,
-     HelpText<"Do not convert private external symbols to static symbols (only valid with -r)">,
-     Flags<[HelpHidden]>,
-     Group<grp_object>;
+    HelpText<"Do not convert private external symbols to static symbols (only valid with -r)">,
+    Flags<[HelpHidden]>,
+    Group<grp_object>;
 def d : Flag<["-"], "d">,
-     HelpText<"Force tentative into real definitions for common symbols">,
-     Flags<[HelpHidden]>,
-     Group<grp_object>;
+    HelpText<"Force tentative into real definitions for common symbols">,
+    Flags<[HelpHidden]>,
+    Group<grp_object>;
 
 def grp_resolve : OptionGroup<"resolve">, HelpText<"SYMBOL RESOLUTION">;
 
 def exported_symbol : Separate<["-"], "exported_symbol">,
-     MetaVarName<"<symbol>">,
-     HelpText<"<symbol> remains global, while others become private externs">,
-     Group<grp_resolve>;
+    MetaVarName<"<symbol>">,
+    HelpText<"<symbol> remains global, while others become private externs">,
+    Group<grp_resolve>;
 def exported_symbols_list : Separate<["-"], "exported_symbols_list">,
-     MetaVarName<"<file>">,
-     HelpText<"Symbols specified in <file> remain global, while others become private externs">,
-     Group<grp_resolve>;
+    MetaVarName<"<file>">,
+    HelpText<"Symbols specified in <file> remain global, while others become private externs">,
+    Group<grp_resolve>;
 def unexported_symbol : Separate<["-"], "unexported_symbol">,
-     MetaVarName<"<symbol>">,
-     HelpText<"Global <symbol> becomes private extern">,
-     Group<grp_resolve>;
+    MetaVarName<"<symbol>">,
+    HelpText<"Global <symbol> becomes private extern">,
+    Group<grp_resolve>;
 def unexported_symbols_list : Separate<["-"], "unexported_symbols_list">,
-     MetaVarName<"<file>">,
-     HelpText<"Global symbols specified in <file> become private externs">,
-     Group<grp_resolve>;
+    MetaVarName<"<file>">,
+    HelpText<"Global symbols specified in <file> become private externs">,
+    Group<grp_resolve>;
 def reexported_symbols_list : Separate<["-"], "reexported_symbols_list">,
-     MetaVarName<"<file>">,
-     HelpText<"Symbols from dependent dylibs specified in <file> are reexported by this dylib">,
-     Flags<[HelpHidden]>,
-     Group<grp_resolve>;
+    MetaVarName<"<file>">,
+    HelpText<"Symbols from dependent dylibs specified in <file> are reexported by this dylib">,
+    Flags<[HelpHidden]>,
+    Group<grp_resolve>;
 def alias : MultiArg<["-"], "alias", 2>,
-     MetaVarName<"<symbol_name> <alternate_name>">,
-     HelpText<"Create a symbol alias with default global visibility">,
-     Flags<[HelpHidden]>,
-     Group<grp_resolve>;
+    MetaVarName<"<symbol_name> <alternate_name>">,
+    HelpText<"Create a symbol alias with default global visibility">,
+    Flags<[HelpHidden]>,
+    Group<grp_resolve>;
 def alias_list : Separate<["-"], "alias_list">,
-     MetaVarName<"<file>">,
-     HelpText<"Create symbol aliases specified in <file>">,
-     Flags<[HelpHidden]>,
-     Group<grp_resolve>;
+    MetaVarName<"<file>">,
+    HelpText<"Create symbol aliases specified in <file>">,
+    Flags<[HelpHidden]>,
+    Group<grp_resolve>;
 def flat_namespace : Flag<["-"], "flat_namespace">,
-     HelpText<"Resolve symbols from all dylibs, both direct and transitive. Do not record source libraries: dyld must re-search at runtime and use the first definition found">,
-     Group<grp_resolve>;
+    HelpText<"Resolve symbols from all dylibs, both direct and transitive. Do not record source libraries: dyld must re-search at runtime and use the first definition found">,
+    Group<grp_resolve>;
 def twolevel_namespace : Flag<["-"], "twolevel_namespace">,
-     HelpText<"Make dyld look up symbols by (dylib,name) pairs (default)">,
-     Group<grp_resolve>;
+    HelpText<"Make dyld look up symbols by (dylib,name) pairs (default)">,
+    Group<grp_resolve>;
 def u : Separate<["-"], "u">,
-     MetaVarName<"<symbol>">,
-     HelpText<"Require that <symbol> be defined for the link to succeed">,
-     Group<grp_resolve>;
+    MetaVarName<"<symbol>">,
+    HelpText<"Require that <symbol> be defined for the link to succeed">,
+    Group<grp_resolve>;
 def U : Separate<["-"], "U">,
-     MetaVarName<"<symbol>">,
-     HelpText<"Allow <symbol> to have no definition">,
-     Group<grp_resolve>;
+    MetaVarName<"<symbol>">,
+    HelpText<"Allow <symbol> to have no definition">,
+    Group<grp_resolve>;
 def undefined : Separate<["-"], "undefined">,
-     MetaVarName<"<treatment>">,
-     HelpText<"Handle undefined symbols according to <treatment>: error, warning, suppress, or dynamic_lookup (default is error)">,
-     Group<grp_resolve>;
+    MetaVarName<"<treatment>">,
+    HelpText<"Handle undefined symbols according to <treatment>: error, warning, suppress, or dynamic_lookup (default is error)">,
+    Group<grp_resolve>;
 def rpath : Separate<["-"], "rpath">,
-     MetaVarName<"<path>">,
-     HelpText<"Add <path> to dyld search list for dylibs with load path prefix `@rpath/'">,
-     Group<grp_resolve>;
+    MetaVarName<"<path>">,
+    HelpText<"Add <path> to dyld search list for dylibs with load path prefix `@rpath/'">,
+    Group<grp_resolve>;
 def commons : Separate<["-"], "commons">,
-     MetaVarName<"<treatment>">,
-     HelpText<"Resolve tentative definitions in dylibs according to <treatment>: ignore_dylibs, use_dylibs, error (default is ignore_dylibs)">,
-     Flags<[HelpHidden]>,
-     Group<grp_resolve>;
+    MetaVarName<"<treatment>">,
+    HelpText<"Resolve tentative definitions in dylibs according to <treatment>: ignore_dylibs, use_dylibs, error (default is ignore_dylibs)">,
+    Flags<[HelpHidden]>,
+    Group<grp_resolve>;
 
 def grp_introspect : OptionGroup<"introspect">, HelpText<"INTROSPECTING THE LINKER">;
 
 def why_load : Flag<["-"], "why_load">,
-     HelpText<"Log why each object file is loaded from a static library">,
-     Group<grp_introspect>;
+    HelpText<"Log why each object file is loaded from a static library">,
+    Group<grp_introspect>;
 def whyload : Flag<["-"], "whyload">,
-     Alias<why_load>,
-     HelpText<"Alias for -why_load">,
-     Flags<[HelpHidden]>,
-     Group<grp_introspect>;
+    Alias<why_load>,
+    HelpText<"Alias for -why_load">,
+    Flags<[HelpHidden]>,
+    Group<grp_introspect>;
 def why_live : Separate<["-"], "why_live">,
-     MetaVarName<"<symbol>">,
-     HelpText<"Log a chain of references to <symbol>, for use with -dead_strip">,
-     Flags<[HelpHidden]>,
-     Group<grp_introspect>;
+    MetaVarName<"<symbol>">,
+    HelpText<"Log a chain of references to <symbol>, for use with -dead_strip">,
+    Flags<[HelpHidden]>,
+    Group<grp_introspect>;
 def print_statistics : Flag<["-"], "print_statistics">,
-     HelpText<"Log the linker's memory and CPU usage">,
-     Flags<[HelpHidden]>,
-     Group<grp_introspect>;
+    HelpText<"Log the linker's memory and CPU usage">,
+    Flags<[HelpHidden]>,
+    Group<grp_introspect>;
 def t : Flag<["-"], "t">,
-     HelpText<"Log every file the linker loads: object, archive, and dylib">,
-     Group<grp_introspect>;
+    HelpText<"Log every file the linker loads: object, archive, and dylib">,
+    Group<grp_introspect>;
 def whatsloaded : Flag<["-"], "whatsloaded">,
-     HelpText<"Logs only the object files the linker loads">,
-     Flags<[HelpHidden]>,
-     Group<grp_introspect>;
+    HelpText<"Logs only the object files the linker loads">,
+    Flags<[HelpHidden]>,
+    Group<grp_introspect>;
 def order_file_statistics : Flag<["-"], "order_file_statistics">,
-     HelpText<"Logs information about -order_file">,
-     Flags<[HelpHidden]>,
-     Group<grp_introspect>;
+    HelpText<"Logs information about -order_file">,
+    Flags<[HelpHidden]>,
+    Group<grp_introspect>;
 def map : Separate<["-"], "map">,
-     MetaVarName<"<path>">,
-     HelpText<"Writes all symbols and their addresses to <path>">,
-     Flags<[HelpHidden]>,
-     Group<grp_introspect>;
+    MetaVarName<"<path>">,
+    HelpText<"Writes all symbols and their addresses to <path>">,
+    Group<grp_introspect>;
 def dependency_info : Separate<["-"], "dependency_info">,
-     MetaVarName<"<path>">,
-     HelpText<"Dump dependency info">,
-     Flags<[HelpHidden]>,
-     Group<grp_introspect>;
+    MetaVarName<"<path>">,
+    HelpText<"Dump dependency info">,
+    Group<grp_introspect>;
 def save_temps : Flag<["-"], "save-temps">,
-     HelpText<"Save intermediate LTO compilation results">,
-     Group<grp_introspect>;
+    HelpText<"Save intermediate LTO compilation results">,
+    Group<grp_introspect>;
 
 def grp_symtab : OptionGroup<"symtab">, HelpText<"SYMBOL TABLE OPTIMIZATIONS">;
 
 def S : Flag<["-"], "S">,
-     HelpText<"Strip debug information (STABS or DWARF) from the output">,
-     Flags<[HelpHidden]>,
-     Group<grp_symtab>;
+    HelpText<"Strip debug information (STABS or DWARF) from the output">,
+    Flags<[HelpHidden]>,
+    Group<grp_symtab>;
 def x : Flag<["-"], "x">,
-     HelpText<"Exclude non-global symbols from the output symbol table">,
-     Flags<[HelpHidden]>,
-     Group<grp_symtab>;
+    HelpText<"Exclude non-global symbols from the output symbol table">,
+    Flags<[HelpHidden]>,
+    Group<grp_symtab>;
 def non_global_symbols_strip_list : Separate<["-"], "non_global_symbols_strip_list">,
-     MetaVarName<"<path>">,
-     HelpText<"Specify in <path> the non-global symbols that should be removed from the output symbol table">,
-     Flags<[HelpHidden]>,
-     Group<grp_symtab>;
+    MetaVarName<"<path>">,
+    HelpText<"Specify in <path> the non-global symbols that should be removed from the output symbol table">,
+    Flags<[HelpHidden]>,
+    Group<grp_symtab>;
 def non_global_symbols_no_strip_list : Separate<["-"], "non_global_symbols_no_strip_list">,
-     MetaVarName<"<path>">,
-     HelpText<"Specify in <path> the non-global symbols that should remain in the output symbol table">,
-     Flags<[HelpHidden]>,
-     Group<grp_symtab>;
+    MetaVarName<"<path>">,
+    HelpText<"Specify in <path> the non-global symbols that should remain in the output symbol table">,
+    Flags<[HelpHidden]>,
+    Group<grp_symtab>;
 def oso_prefix : Separate<["-"], "oso_prefix">,
-     MetaVarName<"<path>">,
-     HelpText<"Remove the prefix <path> from OSO symbols in the debug map">,
-     Flags<[HelpHidden]>,
-     Group<grp_symtab>;
+    MetaVarName<"<path>">,
+    HelpText<"Remove the prefix <path> from OSO symbols in the debug map">,
+    Flags<[HelpHidden]>,
+    Group<grp_symtab>;
 
 def grp_bitcode : OptionGroup<"bitcode">, HelpText<"BITCODE BUILD FLOW">;
 
 def bitcode_bundle : Flag<["-"], "bitcode_bundle">,
-     HelpText<"Generate an embedded bitcode bundle in the __LLVM,__bundle section of the output">,
-     Flags<[HelpHidden]>,
-     Group<grp_bitcode>;
+    HelpText<"Generate an embedded bitcode bundle in the __LLVM,__bundle section of the output">,
+    Flags<[HelpHidden]>,
+    Group<grp_bitcode>;
 def bitcode_hide_symbols : Flag<["-"], "bitcode_hide_symbols">,
-     HelpText<"With -bitcode_bundle, hide all non-exported symbols from output bitcode bundle.">,
-     Flags<[HelpHidden]>,
-     Group<grp_bitcode>;
+    HelpText<"With -bitcode_bundle, hide all non-exported symbols from output bitcode bundle.">,
+    Flags<[HelpHidden]>,
+    Group<grp_bitcode>;
 def bitcode_symbol_map : Separate<["-"], "bitcode_symbol_map">,
-     MetaVarName<"<path>">,
-     HelpText<"Write the bitcode symbol reverse mapping to file <path>, or if a directory, to <path>/UUID.bcsymbolmap">,
-     Flags<[HelpHidden]>,
-     Group<grp_bitcode>;
+    MetaVarName<"<path>">,
+    HelpText<"Write the bitcode symbol reverse mapping to file <path>, or if a directory, to <path>/UUID.bcsymbolmap">,
+    Flags<[HelpHidden]>,
+    Group<grp_bitcode>;
 
 def grp_rare : OptionGroup<"rare">, HelpText<"RARELY USED">;
 
 def v : Flag<["-"], "v">,
-     HelpText<"Print the linker version and search paths in addition to linking">,
-     Group<grp_rare>;
+    HelpText<"Print the linker version and search paths in addition to linking">,
+    Group<grp_rare>;
 def adhoc_codesign : Flag<["-"], "adhoc_codesign">,
-     HelpText<"Write an ad-hoc code signature to the output file (default for arm64 binaries)">,
-     Group<grp_rare>;
+    HelpText<"Write an ad-hoc code signature to the output file (default for arm64 binaries)">,
+    Group<grp_rare>;
 def no_adhoc_codesign : Flag<["-"], "no_adhoc_codesign">,
-     HelpText<"Do not write an ad-hoc code signature to the output file (default for x86_64 binaries)">,
-     Group<grp_rare>;
+    HelpText<"Do not write an ad-hoc code signature to the output file (default for x86_64 binaries)">,
+    Group<grp_rare>;
 def version_details : Flag<["-"], "version_details">,
-     HelpText<"Print the linker version in JSON form">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Print the linker version in JSON form">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def no_weak_imports : Flag<["-"], "no_weak_imports">,
-     HelpText<"Fail if any symbols are weak imports, allowed to be NULL at runtime">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Fail if any symbols are weak imports, allowed to be NULL at runtime">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def no_deduplicate : Flag<["-"], "no_deduplicate">,
-     HelpText<"Omit the deduplication pass">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Omit the deduplication pass">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def verbose_deduplicate : Flag<["-"], "verbose_deduplicate">,
-     HelpText<"Print function names eliminated by deduplication and the total size of code savings">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Print function names eliminated by deduplication and the total size of code savings">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def no_inits : Flag<["-"], "no_inits">,
-     HelpText<"Fail if the output contains static initializers">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Fail if the output contains static initializers">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def no_warn_inits : Flag<["-"], "no_warn_inits">,
-     HelpText<"Suppress warnings for static initializers in the output">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Suppress warnings for static initializers in the output">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def debug_variant : Flag<["-"], "debug_variant">,
-     HelpText<"Suppress warnings germane to binaries shipping to customers">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Suppress warnings germane to binaries shipping to customers">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def unaligned_pointers : Separate<["-"], "unaligned_pointers">,
-     MetaVarName<"<treatment>">,
-     HelpText<"Handle unaligned pointers in __DATA segments according to <treatment>: warning, error, or suppress (default for arm64e is error, otherwise suppress)">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<treatment>">,
+    HelpText<"Handle unaligned pointers in __DATA segments according to <treatment>: warning, error, or suppress (default for arm64e is error, otherwise suppress)">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def dirty_data_list : Separate<["-"], "dirty_data_list">,
-     MetaVarName<"<path>">,
-     HelpText<"Specify data symbols in <path> destined for the __DATA_DIRTY segment">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<path>">,
+    HelpText<"Specify data symbols in <path> destined for the __DATA_DIRTY segment">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def max_default_common_align : Separate<["-"], "max_default_common_align">,
-     MetaVarName<"<boundary>">,
-     HelpText<"Reduce maximum alignment for common symbols to a hex power-of-2 <boundary>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<boundary>">,
+    HelpText<"Reduce maximum alignment for common symbols to a hex power-of-2 <boundary>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def move_to_rw_segment : MultiArg<["-"], "move_to_rw_segment", 2>,
-     MetaVarName<"<segment> <path>">,
-     HelpText<"Move data symbols listed in <path> to another <segment>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<segment> <path>">,
+    HelpText<"Move data symbols listed in <path> to another <segment>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def move_to_ro_segment : MultiArg<["-"], "move_to_ro_segment", 2>,
-     MetaVarName<"<segment> <path>">,
-     HelpText<"Move code symbols listed in <path> to another <segment>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<segment> <path>">,
+    HelpText<"Move code symbols listed in <path> to another <segment>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def rename_section : MultiArg<["-"], "rename_section", 4>,
-     MetaVarName<"<from_segment> <from_section> <to_segment> <to_section>">,
-     HelpText<"Rename <from_segment>/<from_section> as <to_segment>/<to_section>">,
-     Group<grp_rare>;
+    MetaVarName<"<from_segment> <from_section> <to_segment> <to_section>">,
+    HelpText<"Rename <from_segment>/<from_section> as <to_segment>/<to_section>">,
+    Group<grp_rare>;
 def rename_segment : MultiArg<["-"], "rename_segment", 2>,
-     MetaVarName<"<from_segment> <to_segment>">,
-     HelpText<"Rename <from_segment> as <to_segment>">,
-     Group<grp_rare>;
+    MetaVarName<"<from_segment> <to_segment>">,
+    HelpText<"Rename <from_segment> as <to_segment>">,
+    Group<grp_rare>;
 def trace_symbol_layout : Flag<["-"], "trace_symbol_layout">,
-     HelpText<"Show where and why symbols move, as specified by -move_to_ro_segment, -move_to_rw_segment, -rename_section, and -rename_segment">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Show where and why symbols move, as specified by -move_to_ro_segment, -move_to_rw_segment, -rename_section, and -rename_segment">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def section_order : MultiArg<["-"], "section_order", 2>,
-     MetaVarName<"<segment> <sections>">,
-     HelpText<"With -preload, specify layout sequence of colon-separated <sections> in <segment>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<segment> <sections>">,
+    HelpText<"With -preload, specify layout sequence of colon-separated <sections> in <segment>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def segment_order : Separate<["-"], "segment_order">,
-     MetaVarName<"<colon_separated_segment_list>">,
-     HelpText<"With -preload, specify layout sequence of colon-separated <segments>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<colon_separated_segment_list>">,
+    HelpText<"With -preload, specify layout sequence of colon-separated <segments>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def allow_heap_execute : Flag<["-"], "allow_heap_execute">,
-     HelpText<"On i386, allow any page to execute code">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"On i386, allow any page to execute code">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def application_extension : Flag<["-"], "application_extension">,
-     HelpText<"Designate the linker output as safe for use in an application extension">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Designate the linker output as safe for use in an application extension">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def no_application_extension : Flag<["-"], "no_application_extension">,
-     HelpText<"Designate the linker output as unsafe for use in an application extension">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Designate the linker output as unsafe for use in an application extension">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def fatal_warnings : Flag<["-"], "fatal_warnings">,
-     HelpText<"Treat warnings as errors">,
-     Group<grp_rare>;
+    HelpText<"Treat warnings as errors">,
+    Group<grp_rare>;
 def no_eh_labels : Flag<["-"], "no_eh_labels">,
-     HelpText<"In -r mode, suppress .eh labels in the __eh_frame section">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"In -r mode, suppress .eh labels in the __eh_frame section">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def warn_compact_unwind : Flag<["-"], "warn_compact_unwind">,
-     HelpText<"Warn for each FDE that cannot compact into the __unwind_info section and must remain in the __eh_frame section">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Warn for each FDE that cannot compact into the __unwind_info section and must remain in the __eh_frame section">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def warn_weak_exports : Flag<["-"], "warn_weak_exports">,
-     HelpText<"Warn if the linked image contains weak external symbols">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Warn if the linked image contains weak external symbols">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def no_weak_exports : Flag<["-"], "no_weak_exports">,
-     HelpText<"Fail if the linked image contains weak external symbols">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Fail if the linked image contains weak external symbols">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def objc_gc_compaction : Flag<["-"], "objc_gc_compaction">,
-     HelpText<"Mark the Objective-C image as compatible with compacting garbage collection">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Mark the Objective-C image as compatible with compacting garbage collection">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def objc_gc : Flag<["-"], "objc_gc">,
-     HelpText<"Verify that all code was compiled with -fobjc-gc or -fobjc-gc-only">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Verify that all code was compiled with -fobjc-gc or -fobjc-gc-only">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def objc_gc_only : Flag<["-"], "objc_gc_only">,
-     HelpText<"Verify that all code was compiled with -fobjc-gc-only">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Verify that all code was compiled with -fobjc-gc-only">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def dead_strip_dylibs : Flag<["-"], "dead_strip_dylibs">,
-     HelpText<"Remove dylibs that are unreachable by the entry point or exported symbols">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Remove dylibs that are unreachable by the entry point or exported symbols">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def allow_sub_type_mismatches : Flag<["-"], "allow_sub_type_mismatches">,
-     HelpText<"Permit mixing objects compiled for different ARM CPU subtypes">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Permit mixing objects compiled for different ARM CPU subtypes">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def no_uuid : Flag<["-"], "no_uuid">,
-     HelpText<"Do not generate the LC_UUID load command">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Do not generate the LC_UUID load command">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def root_safe : Flag<["-"], "root_safe">,
-     HelpText<"Set the MH_ROOT_SAFE bit in the mach-o header">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Set the MH_ROOT_SAFE bit in the mach-o header">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def setuid_safe : Flag<["-"], "setuid_safe">,
-     HelpText<"Set the MH_SETUID_SAFE bit in the mach-o header">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Set the MH_SETUID_SAFE bit in the mach-o header">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def interposable : Flag<["-"], "interposable">,
-     HelpText<"Indirects access to all to exported symbols in a dylib">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Indirects access to all to exported symbols in a dylib">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def multi_module : Flag<["-"], "multi_module">,
-     Alias<interposable>,
-     HelpText<"Alias for -interposable">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    Alias<interposable>,
+    HelpText<"Alias for -interposable">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def init : Separate<["-"], "init">,
-     MetaVarName<"<symbol>">,
-     HelpText<"Run <symbol> as the first initializer in a dylib">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<symbol>">,
+    HelpText<"Run <symbol> as the first initializer in a dylib">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def sub_library : Separate<["-"], "sub_library">,
-     MetaVarName<"<name>">,
-     HelpText<"Re-export the dylib as <name>">,
-     Group<grp_rare>;
+    MetaVarName<"<name>">,
+    HelpText<"Re-export the dylib as <name>">,
+    Group<grp_rare>;
 def sub_umbrella : Separate<["-"], "sub_umbrella">,
-     MetaVarName<"<name>">,
-     HelpText<"Re-export the framework as <name>">,
-     Group<grp_rare>;
+    MetaVarName<"<name>">,
+    HelpText<"Re-export the framework as <name>">,
+    Group<grp_rare>;
 def allowable_client : Separate<["-"], "allowable_client">,
-     MetaVarName<"<name>">,
-     HelpText<"Specify <name> of a dylib or framework that is allowed to link to this dylib">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<name>">,
+    HelpText<"Specify <name> of a dylib or framework that is allowed to link to this dylib">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def client_name : Separate<["-"], "client_name">,
-     MetaVarName<"<name>">,
-     HelpText<"Specifies a <name> this client should match with the -allowable_client <name> in a dependent dylib">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<name>">,
+    HelpText<"Specifies a <name> this client should match with the -allowable_client <name> in a dependent dylib">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def umbrella : Separate<["-"], "umbrella">,
-     MetaVarName<"<<name>>">,
-     HelpText<"Re-export this dylib through the umbrella framework <name>a">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<<name>>">,
+    HelpText<"Re-export this dylib through the umbrella framework <name>a">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def headerpad : Separate<["-"], "headerpad">,
-     MetaVarName<"<size>">,
-     HelpText<"Allocate hex <size> extra space for future expansion of the load commands via install_name_tool (default is 0x20)">,
-     Group<grp_rare>;
+    MetaVarName<"<size>">,
+    HelpText<"Allocate hex <size> extra space for future expansion of the load commands via install_name_tool (default is 0x20)">,
+    Group<grp_rare>;
 def headerpad_max_install_names : Flag<["-"], "headerpad_max_install_names">,
-     HelpText<"Allocate extra space so all load-command paths can expand to MAXPATHLEN via install_name_tool">,
-     Group<grp_rare>;
+    HelpText<"Allocate extra space so all load-command paths can expand to MAXPATHLEN via install_name_tool">,
+    Group<grp_rare>;
 def bind_at_load : Flag<["-"], "bind_at_load">,
-     HelpText<"Tell dyld to bind all symbols at load time, rather than lazily">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Tell dyld to bind all symbols at load time, rather than lazily">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def force_flat_namespace : Flag<["-"], "force_flat_namespace">,
-     HelpText<"Tell dyld to use a flat namespace on this executable and all its dependent dylibs & bundles">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Tell dyld to use a flat namespace on this executable and all its dependent dylibs & bundles">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def segalign : Separate<["-"], "segalign">,
-     MetaVarName<"<boundary>">,
-     HelpText<"Align all segments to hex power-of-2 <boundary>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<boundary>">,
+    HelpText<"Align all segments to hex power-of-2 <boundary>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def sectalign : MultiArg<["-"], "sectalign", 3>,
-     MetaVarName<"<segment> <section> <boundary>">,
-     HelpText<"Align <section> within <segment> to hex power-of-2 <boundary>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<segment> <section> <boundary>">,
+    HelpText<"Align <section> within <segment> to hex power-of-2 <boundary>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def stack_addr : Separate<["-"], "stack_addr">,
-     MetaVarName<"<address>">,
-     HelpText<"Initialize stack pointer to hex <address> rounded to a page boundary">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<address>">,
+    HelpText<"Initialize stack pointer to hex <address> rounded to a page boundary">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def segprot : MultiArg<["-"], "segprot", 3>,
-     MetaVarName<"<segment> <max> <init>">,
-     HelpText<"Specifies the <max> and <init> virtual memory protection of <segment> as r/w/x/-seg_addr_table path Specify hex base addresses and dylib install names on successive lines in <path>. This option is obsolete">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<segment> <max> <init>">,
+    HelpText<"Specifies the <max> and <init> virtual memory protection of <segment> as r/w/x/-seg_addr_table path Specify hex base addresses and dylib install names on successive lines in <path>. This option is obsolete">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def segs_read_write_addr : Separate<["-"], "segs_read_write_addr">,
-     MetaVarName<"<address>">,
-     HelpText<"This option is obsolete">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<address>">,
+    HelpText<"This option is obsolete">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def segs_read_only_addr : Separate<["-"], "segs_read_only_addr">,
-     MetaVarName<"<address>">,
-     HelpText<"This option is obsolete">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<address>">,
+    HelpText<"This option is obsolete">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def segaddr : MultiArg<["-"], "segaddr", 2>,
-     MetaVarName<"<segment> <address>">,
-     HelpText<"Specify the starting hex <address> at a 4KiB page boundary for <segment>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<segment> <address>">,
+    HelpText<"Specify the starting hex <address> at a 4KiB page boundary for <segment>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def seg_page_size : MultiArg<["-"], "seg_page_size", 2>,
-     MetaVarName<"<segment> <size>">,
-     HelpText<"Specifies the page <size> for <segment>. Segment size will be a multiple of its page size">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<segment> <size>">,
+    HelpText<"Specifies the page <size> for <segment>. Segment size will be a multiple of its page size">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def dylib_file : Separate<["-"], "dylib_file">,
-     MetaVarName<"<install_path:current_path>">,
-     HelpText<"Specify <current_path> as different from where a dylib normally resides at <install_path>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<install_path:current_path>">,
+    HelpText<"Specify <current_path> as different from where a dylib normally resides at <install_path>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def weak_reference_mismatches : Separate<["-"], "weak_reference_mismatches">,
-     MetaVarName<"<treatment>">,
-     HelpText<"Resolve symbol imports of conflicting weakness according to <treatment> as weak, non-weak, or error (default is non-weak)">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<treatment>">,
+    HelpText<"Resolve symbol imports of conflicting weakness according to <treatment> as weak, non-weak, or error (default is non-weak)">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def read_only_relocs : Separate<["-"], "read_only_relocs">,
-     MetaVarName<"<treatment>">,
-     HelpText<"Handle relocations that modify read-only pages according to <treatment> of warning, error, or suppress (i.e., allow)">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<treatment>">,
+    HelpText<"Handle relocations that modify read-only pages according to <treatment> of warning, error, or suppress (i.e., allow)">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def force_cpusubtype_ALL : Flag<["-"], "force_cpusubtype_ALL">,
-     HelpText<"Mark binary as runnable on any PowerPC, ignoring any PowerPC cpu requirements encoded in the object files">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Mark binary as runnable on any PowerPC, ignoring any PowerPC cpu requirements encoded in the object files">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def no_arch_warnings : Flag<["-"], "no_arch_warnings">,
-     HelpText<"Suppresses warnings about inputs whose architecture does not match the -arch option">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Suppresses warnings about inputs whose architecture does not match the -arch option">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def arch_errors_fatal : Flag<["-"], "arch_errors_fatal">,
-     HelpText<"Escalate to errors any warnings about inputs whose architecture does not match the -arch option">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Escalate to errors any warnings about inputs whose architecture does not match the -arch option">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def e : Separate<["-"], "e">,
-     MetaVarName<"<symbol>">,
-     HelpText<"Make <symbol> the entry point of an executable (default is \"start\" from crt1.o)">,
-     Group<grp_rare>;
+    MetaVarName<"<symbol>">,
+    HelpText<"Make <symbol> the entry point of an executable (default is \"start\" from crt1.o)">,
+    Group<grp_rare>;
 def w : Flag<["-"], "w">,
-     HelpText<"Suppress all warnings">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Suppress all warnings">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def final_output : Separate<["-"], "final_output">,
-     MetaVarName<"<name>">,
-     HelpText<"Specify the dylib install name if -install_name is not used--used by compiler driver for multiple -arch arguments">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<name>">,
+    HelpText<"Specify the dylib install name if -install_name is not used--used by compiler driver for multiple -arch arguments">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def arch_multiple : Flag<["-"], "arch_multiple">,
-     HelpText<"Augment error and warning messages with the architecture name">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Augment error and warning messages with the architecture name">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def dot : Separate<["-"], "dot">,
-     MetaVarName<"<path>">,
-     HelpText<"Write a graph of symbol dependencies to <path> as a .dot file viewable with GraphViz">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<path>">,
+    HelpText<"Write a graph of symbol dependencies to <path> as a .dot file viewable with GraphViz">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def keep_relocs : Flag<["-"], "keep_relocs">,
-     HelpText<"Retain section-based relocation records in the output, which are ignored at runtime by dyld">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Retain section-based relocation records in the output, which are ignored at runtime by dyld">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def warn_stabs : Flag<["-"], "warn_stabs">,
-     HelpText<"Warn when bad stab symbols inside a BINCL/EINCL prevent optimization">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Warn when bad stab symbols inside a BINCL/EINCL prevent optimization">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def warn_commons : Flag<["-"], "warn_commons">,
-     HelpText<"Warn when a tentative definition in an object file matches an external symbol in a dylib, which often means \"extern\" is missing from a variable declaration in a header file">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Warn when a tentative definition in an object file matches an external symbol in a dylib, which often means \"extern\" is missing from a variable declaration in a header file">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def read_only_stubs : Flag<["-"], "read_only_stubs">,
-     HelpText<"On i386, make the __IMPORT segment of a final linked image read-only">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"On i386, make the __IMPORT segment of a final linked image read-only">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def interposable_list : Separate<["-"], "interposable_list">,
-     MetaVarName<"<path>">,
-     HelpText<"Access global symbols listed in <path> indirectly">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<path>">,
+    HelpText<"Access global symbols listed in <path> indirectly">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def no_function_starts : Flag<["-"], "no_function_starts">,
-     HelpText<"Do not creates a compressed table of function start addresses">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Do not create a table of function start addresses">,
+    Group<grp_rare>;
 def no_objc_category_merging : Flag<["-"], "no_objc_category_merging">,
-     HelpText<"Do not merge Objective-C categories into their classes">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Do not merge Objective-C categories into their classes">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def object_path_lto : Separate<["-"], "object_path_lto">,
-     MetaVarName<"<path>">,
-     HelpText<"Retain any temporary mach-o file in <path> that would otherwise be deleted during LTO">,
-     Group<grp_rare>;
+    MetaVarName<"<path>">,
+    HelpText<"Retain any temporary mach-o file in <path> that would otherwise be deleted during LTO">,
+    Group<grp_rare>;
 def cache_path_lto : Separate<["-"], "cache_path_lto">,
-     MetaVarName<"<path>">,
-     HelpText<"Use <path> as a directory for the incremental LTO cache">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<path>">,
+    HelpText<"Use <path> as a directory for the incremental LTO cache">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def prune_interval_lto : Separate<["-"], "prune_interval_lto">,
-     MetaVarName<"<seconds>">,
-     HelpText<"Prune the incremental LTO cache after <seconds> (-1 disables pruning)">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<seconds>">,
+    HelpText<"Prune the incremental LTO cache after <seconds> (-1 disables pruning)">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def prune_after_lto : Separate<["-"], "prune_after_lto">,
-     MetaVarName<"<seconds>">,
-     HelpText<"Remove LTO cache entries after <seconds>">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<seconds>">,
+    HelpText<"Remove LTO cache entries after <seconds>">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def max_relative_cache_size_lto : Separate<["-"], "max_relative_cache_size_lto">,
-     MetaVarName<"<percent>">,
-     HelpText<"Limit the incremental LTO cache growth to <percent> of free disk, space">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    MetaVarName<"<percent>">,
+    HelpText<"Limit the incremental LTO cache growth to <percent> of free disk, space">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def page_align_data_atoms : Flag<["-"], "page_align_data_atoms">,
-     HelpText<"Distribute global variables on separate pages so page used/dirty status can guide creation of an order file to cluster commonly used/dirty globals">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Distribute global variables on separate pages so page used/dirty status can guide creation of an order file to cluster commonly used/dirty globals">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def not_for_dyld_shared_cache : Flag<["-"], "not_for_dyld_shared_cache">,
-     HelpText<"Prevent system dylibs from being placed into the dylib shared cache">,
-     Flags<[HelpHidden]>,
-     Group<grp_rare>;
+    HelpText<"Prevent system dylibs from being placed into the dylib shared cache">,
+    Flags<[HelpHidden]>,
+    Group<grp_rare>;
 def mllvm : Separate<["-"], "mllvm">,
-     HelpText<"Options to pass to LLVM">,
-     Group<grp_rare>;
+    HelpText<"Options to pass to LLVM">,
+    Group<grp_rare>;
 def mcpu : Separate<["-"], "mcpu">,
-     HelpText<"Processor family target for LTO code generation">,
-     Group<grp_rare>;
+    HelpText<"Processor family target for LTO code generation">,
+    Group<grp_rare>;
 
 def grp_deprecated : OptionGroup<"deprecated">, HelpText<"DEPRECATED">;
 
 def lazy_framework : Separate<["-"], "lazy_framework">,
-     MetaVarName<"<name>">,
-     HelpText<"This option is deprecated and is now an alias for -framework.">,
-     Flags<[HelpHidden]>,
-     Group<grp_deprecated>;
+    MetaVarName<"<name>">,
+    HelpText<"This option is deprecated and is now an alias for -framework.">,
+    Flags<[HelpHidden]>,
+    Group<grp_deprecated>;
 def lazy_library : Separate<["-"], "lazy_library">,
-     MetaVarName<"<path>">,
-     HelpText<"This option is deprecated and is now an alias for regular linking">,
-     Flags<[HelpHidden]>,
-     Group<grp_deprecated>;
+    MetaVarName<"<path>">,
+    HelpText<"This option is deprecated and is now an alias for regular linking">,
+    Flags<[HelpHidden]>,
+    Group<grp_deprecated>;
 def lazy_l : Joined<["-"], "lazy-l">,
-     MetaVarName<"<name>">,
-     HelpText<"This option is deprecated and is now an alias for -l<path>.">,
-     Flags<[HelpHidden]>,
-     Group<grp_deprecated>;
+    MetaVarName<"<name>">,
+    HelpText<"This option is deprecated and is now an alias for -l<path>.">,
+    Flags<[HelpHidden]>,
+    Group<grp_deprecated>;
 def single_module : Flag<["-"], "single_module">,
-     HelpText<"Unnecessary option: this is already the default">,
-     Flags<[HelpHidden]>,
-     Group<grp_deprecated>;
+    HelpText<"Unnecessary option: this is already the default">,
+    Flags<[HelpHidden]>,
+    Group<grp_deprecated>;
 def no_dead_strip_inits_and_terms : Flag<["-"], "no_dead_strip_inits_and_terms">,
-     HelpText<"Unnecessary option: initialization and termination are roots of the dead strip graph, so never dead stripped">,
-     Flags<[HelpHidden]>,
-     Group<grp_deprecated>;
+    HelpText<"Unnecessary option: initialization and termination are roots of the dead strip graph, so never dead stripped">,
+    Flags<[HelpHidden]>,
+    Group<grp_deprecated>;
 def noall_load : Flag<["-"], "noall_load">,
-     HelpText<"Unnecessary option: this is already the default">,
-     Flags<[HelpHidden]>,
-     Group<grp_deprecated>;
+    HelpText<"Unnecessary option: this is already the default">,
+    Flags<[HelpHidden]>,
+    Group<grp_deprecated>;
 
 def grp_obsolete : OptionGroup<"obsolete">, HelpText<"OBSOLETE">;
 
 def sectorder : MultiArg<["-"], "sectorder", 3>,
-     MetaVarName<"<segname> <sectname> <orderfile>">,
-     HelpText<"Obsolete. Replaced by more general -order_file option">,
-     Group<grp_obsolete>;
+    MetaVarName<"<segname> <sectname> <orderfile>">,
+    HelpText<"Obsolete. Replaced by more general -order_file option">,
+    Group<grp_obsolete>;
 def lto_library : Separate<["-"], "lto_library">,
-     MetaVarName<"<path>">,
-     HelpText<"Obsolete. LLD supports LTO directly, without using an external dylib.">,
-     Group<grp_obsolete>;
+    MetaVarName<"<path>">,
+    HelpText<"Obsolete. LLD supports LTO directly, without using an external dylib.">,
+    Group<grp_obsolete>;
 def y : Joined<["-"], "y">,
-     MetaVarName<"<symbol>">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    MetaVarName<"<symbol>">,
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def sectobjectsymbols : MultiArg<["-"], "sectobjectsymbols", 2>,
-     MetaVarName<"<segname> <sectname>">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    MetaVarName<"<segname> <sectname>">,
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def nofixprebinding : Flag<["-"], "nofixprebinding">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def noprebind_all_twolevel_modules : Flag<["-"], "noprebind_all_twolevel_modules">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def prebind_all_twolevel_modules : Flag<["-"], "prebind_all_twolevel_modules">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def prebind_allow_overlap : Flag<["-"], "prebind_allow_overlap">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def noprebind : Flag<["-"], "noprebind">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def sect_diff_relocs : Separate<["-"], "sect_diff_relocs">,
-     MetaVarName<"<treatment>">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    MetaVarName<"<treatment>">,
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def A : Separate<["-"], "A">,
-     MetaVarName<"<basefile>">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    MetaVarName<"<basefile>">,
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def b : Flag<["-"], "b">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def Sn : Flag<["-"], "Sn">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def Si : Flag<["-"], "Si">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def Sp : Flag<["-"], "Sp">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def X : Flag<["-"], "X">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def s : Flag<["-"], "s">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def m : Flag<["-"], "m">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def Y : Separate<["-"], "Y">,
-     MetaVarName<"<number>">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    MetaVarName<"<number>">,
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def nomultidefs : Flag<["-"], "nomultidefs">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def multiply_defined_unused : Separate<["-"], "multiply_defined_unused">,
-     MetaVarName<"<treatment>">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    MetaVarName<"<treatment>">,
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def multiply_defined : Separate<["-"], "multiply_defined">,
-     MetaVarName<"<treatment>">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    MetaVarName<"<treatment>">,
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def private_bundle : Flag<["-"], "private_bundle">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def seg_addr_table_filename : Separate<["-"], "seg_addr_table_filename">,
-     MetaVarName<"<path>">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    MetaVarName<"<path>">,
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def sectorder_detail : Flag<["-"], "sectorder_detail">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def no_compact_linkedit : Flag<["-"], "no_compact_linkedit">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def dependent_dr_info : Flag<["-"], "dependent_dr_info">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def no_dependent_dr_info : Flag<["-"], "no_dependent_dr_info">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def seglinkedit : Flag<["-"], "seglinkedit">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def noseglinkedit : Flag<["-"], "noseglinkedit">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def fvmlib : Flag<["-"], "fvmlib">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def run_init_lazily : Flag<["-"], "run_init_lazily">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def prebind : Flag<["-"], "prebind">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def twolevel_namespace_hints : Flag<["-"], "twolevel_namespace_hints">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 def slow_stubs : Flag<["-"], "slow_stubs">,
-     HelpText<"This option is obsolete in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_obsolete>;
+    HelpText<"This option is obsolete in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_obsolete>;
 
 def grp_undocumented : OptionGroup<"undocumented">, HelpText<"UNDOCUMENTED">;
 
 def add_ast_path : Flag<["-"], "add_ast_path">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def add_linker_option : Flag<["-"], "add_linker_option">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def add_source_version : Flag<["-"], "add_source_version">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def no_source_version : Flag<["-"], "no_source_version">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def add_split_seg_info : Flag<["-"], "add_split_seg_info">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def allow_dead_duplicates : Flag<["-"], "allow_dead_duplicates">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def allow_simulator_linking_to_macosx_dylibs : Flag<["-"], "allow_simulator_linking_to_macosx_dylibs">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def bitcode_process_mode : Flag<["-"], "bitcode_process_mode">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def bitcode_verify : Flag<["-"], "bitcode_verify">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def classic_linker : Flag<["-"], "classic_linker">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def data_const : Flag<["-"], "data_const">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def no_data_const : Flag<["-"], "no_data_const">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def data_in_code_info : Flag<["-"], "data_in_code_info">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def no_data_in_code_info : Flag<["-"], "no_data_in_code_info">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def debug_snapshot : Flag<["-"], "debug_snapshot">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def demangle : Flag<["-"], "demangle">,
-     HelpText<"Demangle symbol names in diagnostics">;
+    HelpText<"Demangle symbol names in diagnostics">;
 def dyld_env : Flag<["-"], "dyld_env">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def encryptable : Flag<["-"], "encryptable">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def executable_path : Flag<["-"], "executable_path">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def fixup_chains : Flag<["-"], "fixup_chains">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def fixup_chains_section : Flag<["-"], "fixup_chains_section">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def flto_codegen_only : Flag<["-"], "flto-codegen-only">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def force_load_swift_libs : Flag<["-"], "force_load_swift_libs">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def force_symbol_not_weak : Flag<["-"], "force_symbol_not_weak">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def force_symbols_coalesce_list : Flag<["-"], "force_symbols_coalesce_list">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def force_symbols_not_weak_list : Flag<["-"], "force_symbols_not_weak_list">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def force_symbols_weak_list : Flag<["-"], "force_symbols_weak_list">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def force_symbol_weak : Flag<["-"], "force_symbol_weak">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def function_starts : Flag<["-"], "function_starts">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def i : Flag<["-"], "i">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def ignore_auto_link : Flag<["-"], "ignore_auto_link">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def ignore_optimization_hints : Flag<["-"], "ignore_optimization_hints">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def init_offsets : Flag<["-"], "init_offsets">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def keep_dwarf_unwind : Flag<["-"], "keep_dwarf_unwind">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def no_keep_dwarf_unwind : Flag<["-"], "no_keep_dwarf_unwind">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def kext : Flag<["-"], "kext">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def kext_objects_dir : Flag<["-"], "kext_objects_dir">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def no_kext_objects : Flag<["-"], "no_kext_objects">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def kexts_use_stubs : Flag<["-"], "kexts_use_stubs">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def no_compact_unwind : Flag<["-"], "no_compact_unwind">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def no_dtrace_dof : Flag<["-"], "no_dtrace_dof">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def no_encryption : Flag<["-"], "no_encryption">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def no_new_main : Flag<["-"], "no_new_main">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def objc_abi_version : Separate<["-"], "objc_abi_version">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def pause : Flag<["-"], "pause">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def random_uuid : Flag<["-"], "random_uuid">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def simulator_support : Flag<["-"], "simulator_support">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def snapshot_dir : Flag<["-"], "snapshot_dir">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def source_version : Flag<["-"], "source_version">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def text_exec : Flag<["-"], "text_exec">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def threaded_starts_section : Flag<["-"], "threaded_starts_section">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def verbose_optimization_hints : Flag<["-"], "verbose_optimization_hints">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 def version_load_command : Flag<["-"], "version_load_command">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
+    HelpText<"This option is undocumented in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_undocumented>;
 
 def grp_ignored : OptionGroup<"ignored">, HelpText<"IGNORED">;
 
 def M : Flag<["-"], "M">,
-     HelpText<"This option is ignored in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_ignored>;
+    HelpText<"This option is ignored in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_ignored>;
 def new_linker : Flag<["-"], "new_linker">,
-     HelpText<"This option is ignored in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_ignored>;
+    HelpText<"This option is ignored in ld64">,
+    Flags<[HelpHidden]>,
+    Group<grp_ignored>;
diff --git a/lld/MachO/OutputSegment.cpp b/lld/MachO/OutputSegment.cpp
index 1ea5d49a32a1273ec1a3a75f9f2c9064c1dd48cf..edfc474d3df2b12bc377c4b23aa4dde59458a2d3 100644
--- a/lld/MachO/OutputSegment.cpp
+++ b/lld/MachO/OutputSegment.cpp
@@ -38,9 +38,8 @@ static uint32_t maxProt(StringRef name) {
 
 size_t OutputSegment::numNonHiddenSections() const {
   size_t count = 0;
-  for (const OutputSection *osec : sections) {
+  for (const OutputSection *osec : sections)
     count += (!osec->isHidden() ? 1 : 0);
-  }
   return count;
 }
 
diff --git a/lld/MachO/SymbolTable.cpp b/lld/MachO/SymbolTable.cpp
index 61a19b0af4278f02f121633ca7a571ffc33ae292..311c0018379d9e13395aceda66918e7295c3dbf2 100644
--- a/lld/MachO/SymbolTable.cpp
+++ b/lld/MachO/SymbolTable.cpp
@@ -17,8 +17,8 @@ using namespace llvm;
 using namespace lld;
 using namespace lld::macho;
 
-Symbol *SymbolTable::find(StringRef name) {
-  auto it = symMap.find(CachedHashStringRef(name));
+Symbol *SymbolTable::find(CachedHashStringRef cachedName) {
+  auto it = symMap.find(cachedName);
   if (it == symMap.end())
     return nullptr;
   return symVector[it->second];
@@ -54,11 +54,10 @@ Defined *SymbolTable::addDefined(StringRef name, InputFile *file,
           defined->privateExtern &= isPrivateExtern;
         return defined;
       }
-      if (!defined->isWeakDef()) {
+      if (!defined->isWeakDef())
         error("duplicate symbol: " + name + "\n>>> defined in " +
               toString(defined->getFile()) + "\n>>> defined in " +
               toString(file));
-      }
     } else if (auto *dysym = dyn_cast<DylibSymbol>(s)) {
       overridesWeakDef = !isWeakDef && dysym->isWeakDef();
     }
@@ -160,10 +159,10 @@ Symbol *SymbolTable::addLazy(StringRef name, ArchiveFile *file,
 
 Defined *SymbolTable::addSynthetic(StringRef name, InputSection *isec,
                                    uint32_t value, bool isPrivateExtern,
-                                   bool isLinkerInternal) {
-  Defined *s = addDefined(name, nullptr, isec, value, /*isWeakDef=*/false,
-                          isPrivateExtern);
-  s->linkerInternal = isLinkerInternal;
+                                   bool includeInSymtab) {
+  Defined *s = addDefined(name, nullptr, isec, value,
+                          /*isWeakDef=*/false, isPrivateExtern);
+  s->includeInSymtab = includeInSymtab;
   return s;
 }
 
diff --git a/lld/MachO/SymbolTable.h b/lld/MachO/SymbolTable.h
index 661e6545e7a6704285e97d4335f75c95620afb70..9aed8c90064f7eaa6c6f4f5dbeda62905ab369b4 100644
--- a/lld/MachO/SymbolTable.h
+++ b/lld/MachO/SymbolTable.h
@@ -9,6 +9,8 @@
 #ifndef LLD_MACHO_SYMBOL_TABLE_H
 #define LLD_MACHO_SYMBOL_TABLE_H
 
+#include "Symbols.h"
+
 #include "lld/Common/LLVM.h"
 #include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/DenseMap.h"
@@ -50,10 +52,11 @@ public:
                   const llvm::object::Archive::Symbol &sym);
 
   Defined *addSynthetic(StringRef name, InputSection *, uint32_t value,
-                        bool isPrivateExtern, bool isLinkerInternal);
+                        bool isPrivateExtern, bool includeInSymtab);
 
   ArrayRef<Symbol *> getSymbols() const { return symVector; }
-  Symbol *find(StringRef name);
+  Symbol *find(llvm::CachedHashStringRef name);
+  Symbol *find(StringRef name) { return find(llvm::CachedHashStringRef(name)); }
 
 private:
   std::pair<Symbol *, bool> insert(StringRef name);
diff --git a/lld/MachO/Symbols.h b/lld/MachO/Symbols.h
index ada5bc164c82c2d6352e274b7352212f9c1b111d..8a3004a061b361e1dd3c132e55526945cf771d74 100644
--- a/lld/MachO/Symbols.h
+++ b/lld/MachO/Symbols.h
@@ -43,7 +43,7 @@ public:
 
   virtual ~Symbol() {}
 
-  Kind kind() const { return static_cast<Kind>(symbolKind); }
+  Kind kind() const { return symbolKind; }
 
   StringRef getName() const {
     if (nameSize == (uint32_t)-1)
@@ -99,7 +99,7 @@ public:
           bool isWeakDef, bool isExternal, bool isPrivateExtern)
       : Symbol(DefinedKind, name, file), isec(isec), value(value),
         overridesWeakDef(false), privateExtern(isPrivateExtern),
-        linkerInternal(false), weakDef(isWeakDef), external(isExternal) {}
+        includeInSymtab(true), weakDef(isWeakDef), external(isExternal) {}
 
   bool isWeakDef() const override { return weakDef; }
   bool isExternalWeakDef() const {
@@ -124,8 +124,8 @@ public:
   bool overridesWeakDef : 1;
   // Whether this symbol should appear in the output binary's export trie.
   bool privateExtern : 1;
-  // Whether this symbol should appear in the output binary's symbol table.
-  bool linkerInternal : 1;
+  // Whether this symbol should appear in the output symbol table.
+  bool includeInSymtab : 1;
 
 private:
   const bool weakDef : 1;
diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index ffab1f3d44666da8524db3cb7bb7ad72f70e26ee..623307971ef7ab07f92e49d9df3b91a0fa39a9e7 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -53,7 +53,12 @@ SyntheticSection::SyntheticSection(const char *segname, const char *name)
 // dyld3's MachOLoaded::getSlide() assumes that the __TEXT segment starts
 // from the beginning of the file (i.e. the header).
 MachHeaderSection::MachHeaderSection()
-    : SyntheticSection(segment_names::text, section_names::header) {}
+    : SyntheticSection(segment_names::text, section_names::header) {
+  // XXX: This is a hack. (See D97007)
+  // Setting the index to 1 to pretend that this section is the text
+  // section.
+  index = 1;
+}
 
 void MachHeaderSection::addLoadCommand(LoadCommand *lc) {
   loadCommands.push_back(lc);
@@ -207,6 +212,24 @@ NonLazyPointerSectionBase::NonLazyPointerSectionBase(const char *segname,
   flags = S_NON_LAZY_SYMBOL_POINTERS;
 }
 
+void macho::addNonLazyBindingEntries(const Symbol *sym,
+                                     const InputSection *isec, uint64_t offset,
+                                     int64_t addend) {
+  if (const auto *dysym = dyn_cast<DylibSymbol>(sym)) {
+    in.binding->addEntry(dysym, isec, offset, addend);
+    if (dysym->isWeakDef())
+      in.weakBinding->addEntry(sym, isec, offset, addend);
+  } else if (const auto *defined = dyn_cast<Defined>(sym)) {
+    in.rebase->addEntry(isec, offset);
+    if (defined->isExternalWeakDef())
+      in.weakBinding->addEntry(sym, isec, offset, addend);
+  } else {
+    // Undefined symbols are filtered out in scanRelocations(); we should never
+    // get here
+    llvm_unreachable("cannot bind to an undefined symbol");
+  }
+}
+
 void NonLazyPointerSectionBase::addEntry(Symbol *sym) {
   if (entries.insert(sym)) {
     assert(!sym->isInGot());
@@ -370,32 +393,6 @@ void WeakBindingSection::writeTo(uint8_t *buf) const {
   memcpy(buf, contents.data(), contents.size());
 }
 
-bool macho::needsBinding(const Symbol *sym) {
-  if (isa<DylibSymbol>(sym))
-    return true;
-  if (const auto *defined = dyn_cast<Defined>(sym))
-    return defined->isExternalWeakDef();
-  return false;
-}
-
-void macho::addNonLazyBindingEntries(const Symbol *sym,
-                                     const InputSection *isec, uint64_t offset,
-                                     int64_t addend) {
-  if (auto *dysym = dyn_cast<DylibSymbol>(sym)) {
-    in.binding->addEntry(dysym, isec, offset, addend);
-    if (dysym->isWeakDef())
-      in.weakBinding->addEntry(sym, isec, offset, addend);
-  } else if (auto *defined = dyn_cast<Defined>(sym)) {
-    in.rebase->addEntry(isec, offset);
-    if (defined->isExternalWeakDef())
-      in.weakBinding->addEntry(sym, isec, offset, addend);
-  } else {
-    // Undefined symbols are filtered out in scanRelocations(); we should never
-    // get here
-    llvm_unreachable("cannot bind to an undefined symbol");
-  }
-}
-
 StubsSection::StubsSection()
     : SyntheticSection(segment_names::text, "__stubs") {
   flags = S_SYMBOL_STUBS | S_ATTR_SOME_INSTRUCTIONS | S_ATTR_PURE_INSTRUCTIONS;
@@ -549,44 +546,40 @@ uint32_t LazyBindingSection::encode(const DylibSymbol &sym) {
   return opstreamOffset;
 }
 
-void macho::prepareBranchTarget(Symbol *sym) {
-  if (auto *dysym = dyn_cast<DylibSymbol>(sym)) {
-    if (in.stubs->addEntry(dysym)) {
-      if (sym->isWeakDef()) {
-        in.binding->addEntry(dysym, in.lazyPointers->isec,
-                             sym->stubsIndex * WordSize);
-        in.weakBinding->addEntry(sym, in.lazyPointers->isec,
-                                 sym->stubsIndex * WordSize);
-      } else {
-        in.lazyBinding->addEntry(dysym);
-      }
-    }
-  } else if (auto *defined = dyn_cast<Defined>(sym)) {
-    if (defined->isExternalWeakDef()) {
-      if (in.stubs->addEntry(sym)) {
-        in.rebase->addEntry(in.lazyPointers->isec, sym->stubsIndex * WordSize);
-        in.weakBinding->addEntry(sym, in.lazyPointers->isec,
-                                 sym->stubsIndex * WordSize);
-      }
-    }
-  }
-}
-
 ExportSection::ExportSection()
     : LinkEditSection(segment_names::linkEdit, section_names::export_) {}
 
+static void validateExportSymbol(const Defined *defined) {
+  StringRef symbolName = defined->getName();
+  if (defined->privateExtern && config->exportedSymbols.match(symbolName))
+    error("cannot export hidden symbol " + symbolName + "\n>>> defined in " +
+          toString(defined->getFile()));
+}
+
+static bool shouldExportSymbol(const Defined *defined) {
+  if (defined->privateExtern)
+    return false;
+  // TODO: Is this a performance bottleneck? If a build has mostly
+  // global symbols in the input but uses -exported_symbols to filter
+  // out most of them, then it would be better to set the value of
+  // privateExtern at parse time instead of calling
+  // exportedSymbols.match() more than once.
+  //
+  // Measurements show that symbol ordering (which again looks up
+  // every symbol in a hashmap) is the biggest bottleneck when linking
+  // chromium_framework, so this will likely be worth optimizing.
+  return config->exportedSymbols.empty()
+             ? !config->unexportedSymbols.match(defined->getName())
+             : config->exportedSymbols.match(defined->getName());
+}
+
 void ExportSection::finalizeContents() {
   trieBuilder.setImageBase(in.header->addr);
   for (const Symbol *sym : symtab->getSymbols()) {
     if (const auto *defined = dyn_cast<Defined>(sym)) {
-      if (config->exportedSymbols.empty()) {
-        if (defined->privateExtern ||
-            config->unexportedSymbols.match(defined->getName()))
-          continue;
-      } else {
-        if (!config->exportedSymbols.match(defined->getName()))
-          continue;
-      }
+      validateExportSymbol(defined);
+      if (!shouldExportSymbol(defined))
+        continue;
       trieBuilder.addSymbol(*defined);
       hasWeakSymbol = hasWeakSymbol || sym->isWeakDef();
     }
@@ -766,7 +759,7 @@ void SymtabSection::finalizeContents() {
 
   for (Symbol *sym : symtab->getSymbols()) {
     if (auto *defined = dyn_cast<Defined>(sym)) {
-      if (defined->linkerInternal)
+      if (!defined->includeInSymtab)
         continue;
       assert(defined->isExternal());
       addSymbol(externalSymbols, defined);
@@ -808,7 +801,7 @@ void SymtabSection::writeTo(uint8_t *buf) const {
     // TODO populate n_desc with more flags
     if (auto *defined = dyn_cast<Defined>(entry.sym)) {
       uint8_t scope = 0;
-      if (defined->privateExtern) {
+      if (!shouldExportSymbol(defined)) {
         // Private external -- dylib scoped symbol.
         // Promote to non-external at link time.
         assert(defined->isExternal() && "invalid input file");
@@ -1005,3 +998,56 @@ void CodeSignatureSection::writeTo(uint8_t *buf) const {
   memcpy(id, fileName.begin(), fileName.size());
   memset(id + fileName.size(), 0, fileNamePad);
 }
+
+void macho::createSyntheticSymbols() {
+  auto addHeaderSymbol = [](const char *name) {
+    symtab->addSynthetic(name, in.header->isec, 0,
+                         /*privateExtern=*/true,
+                         /*includeInSymtab*/ false);
+  };
+
+  switch (config->outputType) {
+    // FIXME: Assign the right address value for these symbols
+    // (rather than 0). But we need to do that after assignAddresses().
+  case MH_EXECUTE:
+    // If linking PIE, __mh_execute_header is a defined symbol in
+    //  __TEXT, __text)
+    // Otherwise, it's an absolute symbol.
+    if (config->isPic)
+      symtab->addSynthetic("__mh_execute_header", in.header->isec, 0,
+                           /*privateExtern*/ false,
+                           /*includeInSymbtab*/ true);
+    else
+      symtab->addSynthetic("__mh_execute_header",
+                           /*isec*/ nullptr, 0,
+                           /*privateExtern*/ false,
+                           /*includeInSymbtab*/ true);
+    break;
+
+    // The following symbols are  N_SECT symbols, even though the header is not
+    // part of any section and that they are private to the bundle/dylib/object
+    // they are part of.
+  case MH_BUNDLE:
+    addHeaderSymbol("__mh_bundle_header");
+    break;
+  case MH_DYLIB:
+    addHeaderSymbol("__mh_dylib_header");
+    break;
+  case MH_DYLINKER:
+    addHeaderSymbol("__mh_dylinker_header");
+    break;
+  case MH_OBJECT:
+    addHeaderSymbol("__mh_object_header");
+    break;
+  default:
+    llvm_unreachable("unexpected outputType");
+    break;
+  }
+
+  // The Itanium C++ ABI requires dylibs to pass a pointer to __cxa_atexit
+  // which does e.g. cleanup of static global variables. The ABI document
+  // says that the pointer can point to any address in one of the dylib's
+  // segments, but in practice ld64 seems to set it to point to the header,
+  // so that's what's implemented here.
+  addHeaderSymbol("___dso_handle");
+}
diff --git a/lld/MachO/SyntheticSections.h b/lld/MachO/SyntheticSections.h
index 8a100fdbfb8c8abb72be2c18cb1e8939c74cac20..8674e5455afd09c65c3cd9ef9fe7cecdac89869d 100644
--- a/lld/MachO/SyntheticSections.h
+++ b/lld/MachO/SyntheticSections.h
@@ -243,13 +243,6 @@ private:
   SmallVector<char, 128> contents;
 };
 
-// Whether a given symbol's address can only be resolved at runtime.
-bool needsBinding(const Symbol *);
-
-// Add bindings for symbols that need weak or non-lazy bindings.
-void addNonLazyBindingEntries(const Symbol *, const InputSection *,
-                              uint64_t offset, int64_t addend = 0);
-
 // The following sections implement lazy symbol binding -- very similar to the
 // PLT mechanism in ELF.
 //
@@ -349,10 +342,6 @@ private:
   llvm::raw_svector_ostream os{contents};
 };
 
-// Adds stubs and bindings where necessary (e.g. if the symbol is a
-// DylibSymbol.)
-void prepareBranchTarget(Symbol *);
-
 // Stores a trie that describes the set of exported symbols.
 class ExportSection : public LinkEditSection {
 public:
@@ -499,7 +488,6 @@ struct InStruct {
   WeakBindingSection *weakBinding = nullptr;
   LazyBindingSection *lazyBinding = nullptr;
   ExportSection *exports = nullptr;
-  FunctionStartsSection *functionStarts = nullptr;
   GotSection *got = nullptr;
   TlvPointerSection *tlvPointers = nullptr;
   LazyPointerSection *lazyPointers = nullptr;
@@ -511,6 +499,8 @@ struct InStruct {
 extern InStruct in;
 extern std::vector<SyntheticSection *> syntheticSections;
 
+void createSyntheticSymbols();
+
 } // namespace macho
 } // namespace lld
 
diff --git a/lld/MachO/Target.h b/lld/MachO/Target.h
index 52a2c5b44376946b457ef1d6d4a18d90cc5ba9d0..2e4329bc00308a4e7a85eddda38a294d77710330 100644
--- a/lld/MachO/Target.h
+++ b/lld/MachO/Target.h
@@ -67,11 +67,6 @@ public:
     return getRelocAttrs(type).hasAttr(bit);
   }
 
-  bool validateRelocationInfo(llvm::MemoryBufferRef,
-                              const llvm::MachO::section_64 &sec,
-                              llvm::MachO::relocation_info);
-  void prepareSymbolRelocation(Symbol *, const InputSection *, const Reloc &);
-
   uint32_t cpuType;
   uint32_t cpuSubtype;
 
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index 1776fa01538db09ab2518c4ae0ae2df9caa66a3e..38a6627b63e9346c4aefd2fc3c06d7273734fffb 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -10,6 +10,7 @@
 #include "Config.h"
 #include "InputFiles.h"
 #include "InputSection.h"
+#include "MapFile.h"
 #include "MergedOutputSection.h"
 #include "OutputSection.h"
 #include "OutputSegment.h"
@@ -26,6 +27,7 @@
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/xxhash.h"
 
 #include <algorithm>
@@ -47,12 +49,15 @@ public:
   void scanSymbols();
   void createOutputSections();
   void createLoadCommands();
+  void finalizeAddressses();
+  void finalizeLinkEditSegment();
   void assignAddresses(OutputSegment *);
 
   void openFile();
   void writeSections();
   void writeUuid();
   void writeCodeSignature();
+  void writeOutputFile();
 
   void run();
 
@@ -65,7 +70,10 @@ public:
   IndirectSymtabSection *indirectSymtabSection = nullptr;
   CodeSignatureSection *codeSignatureSection = nullptr;
   UnwindInfoSection *unwindInfoSection = nullptr;
+  FunctionStartsSection *functionStartsSection = nullptr;
+
   LCUuid *uuidCommand = nullptr;
+  OutputSegment *linkEditSegment = nullptr;
 };
 
 // LC_DYLD_INFO_ONLY stores the offsets of symbol import/export information.
@@ -116,8 +124,8 @@ public:
 
 class LCFunctionStarts : public LoadCommand {
 public:
-  explicit LCFunctionStarts(FunctionStartsSection *functionStarts)
-      : functionStarts(functionStarts) {}
+  explicit LCFunctionStarts(FunctionStartsSection *functionStartsSection)
+      : functionStartsSection(functionStartsSection) {}
 
   uint32_t getSize() const override { return sizeof(linkedit_data_command); }
 
@@ -125,12 +133,12 @@ public:
     auto *c = reinterpret_cast<linkedit_data_command *>(buf);
     c->cmd = LC_FUNCTION_STARTS;
     c->cmdsize = getSize();
-    c->dataoff = functionStarts->fileOff;
-    c->datasize = functionStarts->getFileSize();
+    c->dataoff = functionStartsSection->fileOff;
+    c->datasize = functionStartsSection->getFileSize();
   }
 
 private:
-  FunctionStartsSection *functionStarts;
+  FunctionStartsSection *functionStartsSection;
 };
 
 class LCDysymtab : public LoadCommand {
@@ -442,6 +450,40 @@ public:
 
 } // namespace
 
+// Adds stubs and bindings where necessary (e.g. if the symbol is a
+// DylibSymbol.)
+static void prepareBranchTarget(Symbol *sym) {
+  if (auto *dysym = dyn_cast<DylibSymbol>(sym)) {
+    if (in.stubs->addEntry(dysym)) {
+      if (sym->isWeakDef()) {
+        in.binding->addEntry(dysym, in.lazyPointers->isec,
+                             sym->stubsIndex * WordSize);
+        in.weakBinding->addEntry(sym, in.lazyPointers->isec,
+                                 sym->stubsIndex * WordSize);
+      } else {
+        in.lazyBinding->addEntry(dysym);
+      }
+    }
+  } else if (auto *defined = dyn_cast<Defined>(sym)) {
+    if (defined->isExternalWeakDef()) {
+      if (in.stubs->addEntry(sym)) {
+        in.rebase->addEntry(in.lazyPointers->isec, sym->stubsIndex * WordSize);
+        in.weakBinding->addEntry(sym, in.lazyPointers->isec,
+                                 sym->stubsIndex * WordSize);
+      }
+    }
+  }
+}
+
+// Can a symbol's address can only be resolved at runtime?
+static bool needsBinding(const Symbol *sym) {
+  if (isa<DylibSymbol>(sym))
+    return true;
+  if (const auto *defined = dyn_cast<Defined>(sym))
+    return defined->isExternalWeakDef();
+  return false;
+}
+
 static void prepareSymbolRelocation(lld::macho::Symbol *sym,
                                     const InputSection *isec, const Reloc &r) {
   const RelocAttrs &relocAttrs = target->getRelocAttrs(r.type);
@@ -464,6 +506,7 @@ static void prepareSymbolRelocation(lld::macho::Symbol *sym,
 }
 
 void Writer::scanRelocations() {
+  TimeTraceScope timeScope("Scan relocations");
   for (InputSection *isec : inputSections) {
     if (isec->segname == segment_names::ld) {
       prepareCompactUnwind(isec);
@@ -496,6 +539,7 @@ void Writer::scanRelocations() {
 }
 
 void Writer::scanSymbols() {
+  TimeTraceScope timeScope("Scan symbols");
   for (const macho::Symbol *sym : symtab->getSymbols()) {
     if (const auto *defined = dyn_cast<Defined>(sym)) {
       if (defined->overridesWeakDef)
@@ -521,7 +565,8 @@ void Writer::createLoadCommands() {
   in.header->addLoadCommand(make<LCSymtab>(symtabSection, stringTableSection));
   in.header->addLoadCommand(
       make<LCDysymtab>(symtabSection, indirectSymtabSection));
-  in.header->addLoadCommand(make<LCFunctionStarts>(in.functionStarts));
+  if (functionStartsSection)
+    in.header->addLoadCommand(make<LCFunctionStarts>(functionStartsSection));
   for (StringRef path : config->runtimePaths)
     in.header->addLoadCommand(make<LCRPath>(path));
 
@@ -699,6 +744,8 @@ static std::function<bool(T, T)> compareByOrder(F ord) {
 // segments, output sections within each segment, and input sections within each
 // output segment.
 static void sortSegmentsAndSections() {
+  TimeTraceScope timeScope("Sort segments and sections");
+
   llvm::stable_sort(outputSegments,
                     compareByOrder<OutputSegment *>(segmentOrder));
 
@@ -713,7 +760,6 @@ static void sortSegmentsAndSections() {
       // output section indices.
       if (!osec->isHidden())
         osec->index = ++sectionIndex;
-
       if (!firstTLVDataSection && isThreadLocalData(osec->flags))
         firstTLVDataSection = osec;
 
@@ -740,6 +786,7 @@ static NamePair maybeRenameSection(NamePair key) {
 }
 
 void Writer::createOutputSections() {
+  TimeTraceScope timeScope("Create output sections");
   // First, create hidden sections
   stringTableSection = make<StringTableSection>();
   unwindInfoSection = make<UnwindInfoSection>(); // TODO(gkm): only when no -r
@@ -747,6 +794,8 @@ void Writer::createOutputSections() {
   indirectSymtabSection = make<IndirectSymtabSection>();
   if (config->adhocCodesign)
     codeSignatureSection = make<CodeSignatureSection>();
+  if (config->emitFunctionStarts)
+    functionStartsSection = make<FunctionStartsSection>();
 
   switch (config->outputType) {
   case MH_EXECUTE:
@@ -791,6 +840,43 @@ void Writer::createOutputSections() {
             ssec->name);
     }
   }
+
+  // dyld requires __LINKEDIT segment to always exist (even if empty).
+  linkEditSegment = getOrCreateOutputSegment(segment_names::linkEdit);
+}
+
+void Writer::finalizeAddressses() {
+  TimeTraceScope timeScope("Finalize addresses");
+  // Ensure that segments (and the sections they contain) are allocated
+  // addresses in ascending order, which dyld requires.
+  //
+  // Note that at this point, __LINKEDIT sections are empty, but we need to
+  // determine addresses of other segments/sections before generating its
+  // contents.
+  for (OutputSegment *seg : outputSegments)
+    if (seg != linkEditSegment)
+      assignAddresses(seg);
+
+  // FIXME(gkm): create branch-extension thunks here, then adjust addresses
+}
+
+void Writer::finalizeLinkEditSegment() {
+  TimeTraceScope timeScope("Finalize __LINKEDIT segment");
+  // Fill __LINKEDIT contents.
+  in.rebase->finalizeContents();
+  in.binding->finalizeContents();
+  in.weakBinding->finalizeContents();
+  in.lazyBinding->finalizeContents();
+  in.exports->finalizeContents();
+  symtabSection->finalizeContents();
+  indirectSymtabSection->finalizeContents();
+
+  if (functionStartsSection)
+    functionStartsSection->finalizeContents();
+
+  // Now that __LINKEDIT is filled out, do a proper calculation of its
+  // addresses and offsets.
+  assignAddresses(linkEditSegment);
 }
 
 void Writer::assignAddresses(OutputSegment *seg) {
@@ -834,6 +920,7 @@ void Writer::writeSections() {
 }
 
 void Writer::writeUuid() {
+  TimeTraceScope timeScope("Computing UUID");
   uint64_t digest =
       xxHash64({buffer->getBufferStart(), buffer->getBufferEnd()});
   uuidCommand->writeUuid(digest);
@@ -844,52 +931,11 @@ void Writer::writeCodeSignature() {
     codeSignatureSection->writeHashes(buffer->getBufferStart());
 }
 
-void Writer::run() {
-  // dyld requires __LINKEDIT segment to always exist (even if empty).
-  OutputSegment *linkEditSegment =
-      getOrCreateOutputSegment(segment_names::linkEdit);
-
-  prepareBranchTarget(config->entry);
-  scanRelocations();
-  if (in.stubHelper->isNeeded())
-    in.stubHelper->setup();
-  scanSymbols();
-
-  // Sort and assign sections to their respective segments. No more sections nor
-  // segments may be created after these methods run.
-  createOutputSections();
-  sortSegmentsAndSections();
-
-  createLoadCommands();
-
-  // Ensure that segments (and the sections they contain) are allocated
-  // addresses in ascending order, which dyld requires.
-  //
-  // Note that at this point, __LINKEDIT sections are empty, but we need to
-  // determine addresses of other segments/sections before generating its
-  // contents.
-  for (OutputSegment *seg : outputSegments)
-    if (seg != linkEditSegment)
-      assignAddresses(seg);
-
-  // Fill __LINKEDIT contents.
-  in.rebase->finalizeContents();
-  in.binding->finalizeContents();
-  in.weakBinding->finalizeContents();
-  in.lazyBinding->finalizeContents();
-  in.exports->finalizeContents();
-  in.functionStarts->finalizeContents();
-  symtabSection->finalizeContents();
-  indirectSymtabSection->finalizeContents();
-
-  // Now that __LINKEDIT is filled out, do a proper calculation of its
-  // addresses and offsets.
-  assignAddresses(linkEditSegment);
-
+void Writer::writeOutputFile() {
+  TimeTraceScope timeScope("Write output file");
   openFile();
   if (errorCount())
     return;
-
   writeSections();
   writeUuid();
   writeCodeSignature();
@@ -898,6 +944,22 @@ void Writer::run() {
     error("failed to write to the output file: " + toString(std::move(e)));
 }
 
+void Writer::run() {
+  prepareBranchTarget(config->entry);
+  scanRelocations();
+  if (in.stubHelper->isNeeded())
+    in.stubHelper->setup();
+  scanSymbols();
+  createOutputSections();
+  // No more sections nor segments are created beyond this point.
+  sortSegmentsAndSections();
+  createLoadCommands();
+  finalizeAddressses();
+  finalizeLinkEditSegment();
+  writeMapFile();
+  writeOutputFile();
+}
+
 void macho::writeResult() { Writer().run(); }
 
 void macho::createSyntheticSections() {
@@ -907,7 +969,6 @@ void macho::createSyntheticSections() {
   in.weakBinding = make<WeakBindingSection>();
   in.lazyBinding = make<LazyBindingSection>();
   in.exports = make<ExportSection>();
-  in.functionStarts = make<FunctionStartsSection>();
   in.got = make<GotSection>();
   in.tlvPointers = make<TlvPointerSection>();
   in.lazyPointers = make<LazyPointerSection>();
diff --git a/lld/MachO/Writer.h b/lld/MachO/Writer.h
index 88baa8a1e4bbcae310f04fe157d30249cc6a9aa8..a13b5ceebf5a4d8d74afa4ce9a97cf08cfbe8afc 100644
--- a/lld/MachO/Writer.h
+++ b/lld/MachO/Writer.h
@@ -15,6 +15,8 @@ namespace lld {
 namespace macho {
 
 class OutputSection;
+class InputSection;
+class Symbol;
 
 class LoadCommand {
 public:
@@ -27,6 +29,10 @@ void writeResult();
 
 void createSyntheticSections();
 
+// Add bindings for symbols that need weak or non-lazy bindings.
+void addNonLazyBindingEntries(const Symbol *, const InputSection *,
+                              uint64_t offset, int64_t addend = 0);
+
 extern OutputSection *firstTLVDataSection;
 
 } // namespace macho
diff --git a/lld/docs/ELF/warn_backrefs.rst b/lld/docs/ELF/warn_backrefs.rst
index d4388f9afbb42dba7b287107e9ec9d1522f28969..fac2145cc0c0ee92a14c0176411f0cdfcdebfe2d 100644
--- a/lld/docs/ELF/warn_backrefs.rst
+++ b/lld/docs/ELF/warn_backrefs.rst
@@ -37,11 +37,11 @@ symbols.
 The ``warn-backrefs`` option provides information that helps identify cases
 where LLD and GNU ld archive selection may differ.
 
-    % ld.lld --warn-backrefs ... -lB -lA
-    ld.lld: warning: backward reference detected: system in A.a(a.o) refers to B.a(b.o)
+    | % ld.lld --warn-backrefs ... -lB -lA
+    | ld.lld: warning: backward reference detected: system in A.a(a.o) refers to B.a(b.o)
 
-    % ld.lld --warn-backrefs ... --start-lib B/b.o --end-lib --start-lib A/a.o --end-lib
-    ld.lld: warning: backward reference detected: system in A/a.o refers to B/b.o
+    | % ld.lld --warn-backrefs ... --start-lib B/b.o --end-lib --start-lib A/a.o --end-lib
+    | ld.lld: warning: backward reference detected: system in A/a.o refers to B/b.o
 
     # To suppress the warning, you can specify --warn-backrefs-exclude=<glob> to match B/b.o or B.a(b.o)
 
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 3684e99cb80c6dfd0ade5b43027886a01b3ed717..a3b577e48fb12f1f29598d15071b962999537368 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -29,7 +29,8 @@ ELF Improvements
 Breaking changes
 ----------------
 
-* ...
+* ``--shuffle-sections=<seed>`` has been changed to ``--shuffle-sections=<section-glob>=<seed>``.
+  Specify ``*`` as ``<section-glob>`` to get the previous behavior.
 
 COFF Improvements
 -----------------
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 42525df76ada4a8ec37af4ef3754734386097fea..37c42a0eb51ff25e78c34b0e2d45511f54fb2a4d 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -487,7 +487,8 @@ Set address of section.
 .It Fl -shared , Fl -Bsharable
 Build a shared object.
 .It Fl -shuffle-sections Ns = Ns Ar seed
-Shuffle input sections using the given seed. If 0, use a random seed.
+Shuffle matched sections using the given seed before mapping them to the output sections.
+If -1, reverse the section order. If 0, use a random seed.
 .It Fl -soname Ns = Ns Ar value , Fl h Ar value
 Set
 .Dv DT_SONAME
diff --git a/lld/test/COFF/icf-assoc-order.s b/lld/test/COFF/icf-assoc-order.s
new file mode 100644
index 0000000000000000000000000000000000000000..45ddc0c6e7f70241901e91232e11530024c8c9a0
--- /dev/null
+++ b/lld/test/COFF/icf-assoc-order.s
@@ -0,0 +1,52 @@
+# REQUIRES: x86
+# RUN: llvm-mc %s -filetype=obj -triple=x86_64-windows-msvc -o %t.obj
+# RUN: lld-link %t.obj -export:foo -export:bar -dll -noentry -out:%t.dll -verbose 2>&1 | FileCheck %s
+# RUN: llvm-readobj --sections %t.dll | FileCheck %s --check-prefix=TEXT
+
+# The order of the pdata and xdata sections here shouldn't matter. We should
+# still replace bar with foo.
+
+# CHECK: ICF needed {{.*}} iterations
+# CHECK: Selected foo
+# CHECK: Removed bar
+
+# We should only have five bytes of text.
+# TEXT: Name: .text
+# TEXT-NEXT: Size: 0x5
+
+	.section	.text,"xr",discard,foo
+	.globl	foo
+foo:
+	pushq %rbx
+	pushq %rdi
+	popq %rdi
+	popq %rbx
+        retq
+
+
+.section .pdata,"r",associative,foo
+.long foo
+.long 5
+.long foo_xdata@IMGREL
+
+.section .xdata,"r",associative,foo
+foo_xdata:
+.long 42
+
+	.section	.text,"xr",discard,bar
+	.globl	bar
+bar:
+	pushq %rbx
+	pushq %rdi
+	popq %rdi
+	popq %rbx
+        retq
+
+.section .xdata,"r",associative,bar
+bar_xdata:
+.long 42
+
+.section .pdata,"r",associative,bar
+.long bar
+.long 5
+.long bar_xdata@IMGREL
diff --git a/lld/test/COFF/icf-xdata-last.s b/lld/test/COFF/icf-xdata-last.s
new file mode 100644
index 0000000000000000000000000000000000000000..d8ad7d394fb37f79fc555f805f6ee6a7be940a3a
--- /dev/null
+++ b/lld/test/COFF/icf-xdata-last.s
@@ -0,0 +1,48 @@
+# REQUIRES: x86
+# RUN: llvm-mc %s -filetype=obj -triple=x86_64-windows-msvc -o %t.obj
+# RUN: lld-link %t.obj -export:foo -export:bar -dll -noentry -out:%t.dll -merge:.xdata=.xdata -verbose 2>&1 | FileCheck %s
+# RUN: llvm-readobj --sections %t.dll | FileCheck %s --check-prefix=XDATA
+
+# Test xdata can be merged when text and pdata differ. This test is structured
+# so that xdata comes after pdata, which makes xdata come before pdata in the
+# assocChildren linked list.
+
+# CHECK: ICF needed {{.*}} iterations
+# CHECK: Selected
+# CHECK: Removed
+
+# XDATA:         Name: .xdata
+# XDATA-NEXT:    VirtualSize: 0x4
+
+	.section	.text,"xr",discard,foo
+	.globl	foo
+foo:
+	pushq %rax
+	popq %rax
+        retq
+
+.section .pdata,"r",associative,foo
+.long foo
+.long 5
+.long foo_xdata@IMGREL
+
+
+.section .xdata,"r",associative,foo
+foo_xdata:
+.long 42
+
+	.section	.text,"xr",discard,bar
+	.globl	bar
+bar:
+	pushq %rcx
+	popq %rcx
+        retq
+
+.section .pdata,"r",associative,bar
+.long bar
+.long 5
+.long bar_xdata@IMGREL
+
+.section .xdata,"r",associative,bar
+bar_xdata:
+.long 42
diff --git a/lld/test/COFF/pdb-invalid-func-type-2.yaml b/lld/test/COFF/pdb-invalid-func-type-2.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7d47e72640b7306cee6d4f9ef911ef6748017dc5
--- /dev/null
+++ b/lld/test/COFF/pdb-invalid-func-type-2.yaml
@@ -0,0 +1,151 @@
+# This test has an S_GPROC32_ID symbol that doesn't refer to an LF_FUNC_ID
+# record. Make sure we have reasonable behavior.
+
+# RUN: yaml2obj %s -o %t.obj
+# RUN: lld-link %t.obj -out:%t.exe -debug -pdb:%t.pdb -nodefaultlib -entry:main 2>&1 | FileCheck %s --check-prefix=WARNING
+# RUN: llvm-pdbutil dump -symbols %t.pdb | FileCheck %s
+# RUN: lld-link %t.obj -out:%t.exe -debug -debug:ghash -pdb:%t.pdb -nodefaultlib -entry:main 2>&1 | FileCheck %s --check-prefix=WARNING
+# RUN: llvm-pdbutil dump -symbols %t.pdb | FileCheck %s
+
+# WARNING: warning: procedure symbol record for `main` in {{.*}}.tmp.obj refers to PDB item index 0x{{.*}} which is not a valid function ID record
+
+# CHECK: Mod 0000 | `{{.*}}pdb-invalid-func-type{{.*}}.tmp.obj`:
+# CHECK:      4 | S_GPROC32 [size = 44] `main`
+# CHECK:          parent = 0, end = 80, addr = 0001:0000, code size = 3
+# CHECK:          type = `0x0007 (<not translated>)`, debug start = 0, debug end = 2, flags = none
+# CHECK:     48 | S_FRAMEPROC [size = 32]
+# CHECK:     80 | S_END [size = 4]
+
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:
+  - Name:            '.debug$S'
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    Alignment:       1
+    Subsections:
+      - !Symbols
+        Records:
+          - Kind:            S_GPROC32_ID
+            ProcSym:
+              CodeSize:        3
+              DbgStart:        0
+              DbgEnd:          2
+              # Refers to the wrong kind of type record (LF_PROCEDURE vs
+              # LF_FUNC_ID).
+              FunctionType:    4097
+              Flags:           [  ]
+              DisplayName:     main
+          - Kind:            S_FRAMEPROC
+            FrameProcSym:
+              TotalFrameBytes: 0
+              PaddingFrameBytes: 0
+              OffsetToPadding: 0
+              BytesOfCalleeSavedRegisters: 0
+              OffsetOfExceptionHandler: 0
+              SectionIdOfExceptionHandler: 0
+              Flags:           [ AsynchronousExceptionHandling, OptimizedForSpeed ]
+          - Kind:            S_PROC_ID_END
+            ScopeEndSym:
+      - !Lines
+        CodeSize:        3
+        Flags:           [  ]
+        RelocOffset:     0
+        RelocSegment:    0
+        Blocks:
+          - FileName:        'c:\src\llvm-project\build\t.c'
+            Lines:
+              - Offset:          0
+                LineStart:       1
+                IsStatement:     true
+                EndDelta:        0
+            Columns:
+      - !FileChecksums
+        Checksums:
+          - FileName:        'c:\src\llvm-project\build\t.c'
+            Kind:            MD5
+            Checksum:        270A878DCC1B845655B162F56C4F5020
+      - !StringTable
+        Strings:
+          - 'c:\src\llvm-project\build\t.c'
+    Relocations:
+      - VirtualAddress:  44
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_SECREL
+      - VirtualAddress:  48
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_SECTION
+      - VirtualAddress:  100
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_SECREL
+      - VirtualAddress:  104
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_SECTION
+  - Name:            '.debug$T'
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    Alignment:       1
+    Types:
+      - Kind:            LF_ARGLIST
+        ArgList:
+          ArgIndices:      [ 0 ]
+      - Kind:            LF_PROCEDURE
+        Procedure:
+          ReturnType:      116
+          CallConv:        NearC
+          Options:         [ None ]
+          ParameterCount:  0
+          ArgumentList:    4096
+      - Kind:            LF_FUNC_ID
+        FuncId:
+          ParentScope:     0
+          FunctionType:    4097
+          Name:            main
+  - Name:            '.text$mn'
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    Alignment:       16
+    SectionData:     33C0C3
+symbols:
+  - Name:            '.debug$S'
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          328
+      NumberOfRelocations: 4
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          0
+  - Name:            '.debug$T'
+    Value:           0
+    SectionNumber:   2
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          564
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          0
+  - Name:            '.text$mn'
+    Value:           0
+    SectionNumber:   3
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          3
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        4021952397
+      Number:          0
+  - Name:            main
+    Value:           0
+    SectionNumber:   3
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_FUNCTION
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...
diff --git a/lld/test/ELF/amdgpu-abi-version.s b/lld/test/ELF/amdgpu-abi-version.s
index 6fc95f1039aad3e03c86154d969969d740402425..777a0c3eba807e2bad9d3714ed506d1218a3ed49 100644
--- a/lld/test/ELF/amdgpu-abi-version.s
+++ b/lld/test/ELF/amdgpu-abi-version.s
@@ -1,5 +1,5 @@
 # REQUIRES: amdgpu
-# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj %s -o %t.o
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -filetype=obj %s -o %t.o
 # RUN: ld.lld -shared %t.o -o %t.so
 # RUN: llvm-readobj --file-headers %t.so | FileCheck %s
 
diff --git a/lld/test/ELF/amdgpu-tid.s b/lld/test/ELF/amdgpu-tid.s
new file mode 100644
index 0000000000000000000000000000000000000000..6623443a4541d7285d8adee44ae86faebae510cf
--- /dev/null
+++ b/lld/test/ELF/amdgpu-tid.s
@@ -0,0 +1,45 @@
+# REQUIRES: amdgpu
+
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-xnack --amdhsa-code-object-version=4 -filetype=obj %s -o %t-xnack-off0.o
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-xnack --amdhsa-code-object-version=4 -filetype=obj %s -o %t-xnack-off1.o
+# RUN: ld.lld -shared %t-xnack-off0.o %t-xnack-off1.o -o %t-xnack-off2.so
+# RUN: llvm-readobj --file-headers %t-xnack-off2.so | FileCheck --check-prefix=XNACK-OFF %s
+
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+xnack --amdhsa-code-object-version=4 -filetype=obj %s -o %t-xnack-on0.o
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+xnack --amdhsa-code-object-version=4 -filetype=obj %s -o %t-xnack-on1.o
+# RUN: ld.lld -shared %t-xnack-on0.o %t-xnack-on1.o -o %t-xnack-on2.so
+# RUN: llvm-readobj --file-headers %t-xnack-on2.so | FileCheck --check-prefix=XNACK-ON %s
+
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 --amdhsa-code-object-version=4 -filetype=obj %s -o %t-xnack-any.o
+# RUN: ld.lld -shared %t-xnack-off0.o %t-xnack-any.o -o %t-xnack-off3.so
+# RUN: llvm-readobj --file-headers %t-xnack-off3.so | FileCheck --check-prefix=XNACK-OFF %s
+# RUN: ld.lld -shared %t-xnack-on0.o %t-xnack-any.o -o %t-xnack-on3.so
+# RUN: llvm-readobj --file-headers %t-xnack-on3.so | FileCheck --check-prefix=XNACK-ON %s
+
+# RUN: not ld.lld -shared %t-xnack-off0.o %t-xnack-on0.o -o /dev/null 2>&1 | FileCheck --check-prefix=XNACK-INCOMPATIBLE %s
+
+# XNACK-OFF:          EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200)
+# XNACK-ON:           EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300)
+# XNACK-INCOMPATIBLE: incompatible xnack:
+
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-sramecc --amdhsa-code-object-version=4 -filetype=obj %s -o %t-sramecc-off0.o
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-sramecc --amdhsa-code-object-version=4 -filetype=obj %s -o %t-sramecc-off1.o
+# RUN: ld.lld -shared %t-sramecc-off0.o %t-sramecc-off1.o -o %t-sramecc-off2.so
+# RUN: llvm-readobj --file-headers %t-sramecc-off2.so | FileCheck --check-prefix=SRAMECC-OFF %s
+
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+sramecc --amdhsa-code-object-version=4 -filetype=obj %s -o %t-sramecc-on0.o
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+sramecc --amdhsa-code-object-version=4 -filetype=obj %s -o %t-sramecc-on1.o
+# RUN: ld.lld -shared %t-sramecc-on0.o %t-sramecc-on1.o -o %t-sramecc-on2.so
+# RUN: llvm-readobj --file-headers %t-sramecc-on2.so | FileCheck --check-prefix=SRAMECC-ON %s
+
+# RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx906 --amdhsa-code-object-version=4 -filetype=obj %s -o %t-sramecc-any.o
+# RUN: ld.lld -shared %t-sramecc-off0.o %t-sramecc-any.o -o %t-sramecc-off3.so
+# RUN: llvm-readobj --file-headers %t-sramecc-off3.so | FileCheck --check-prefix=SRAMECC-OFF %s
+# RUN: ld.lld -shared %t-sramecc-on0.o %t-sramecc-any.o -o %t-sramecc-on3.so
+# RUN: llvm-readobj --file-headers %t-sramecc-on3.so | FileCheck --check-prefix=SRAMECC-ON %s
+
+# RUN: not ld.lld -shared %t-sramecc-off0.o %t-sramecc-on0.o -o /dev/null 2>&1 | FileCheck --check-prefix=SRAMECC-INCOMPATIBLE %s
+
+# SRAMECC-OFF:          EF_AMDGPU_FEATURE_SRAMECC_OFF_V4 (0x800)
+# SRAMECC-ON:           EF_AMDGPU_FEATURE_SRAMECC_ON_V4 (0xC00)
+# SRAMECC-INCOMPATIBLE: incompatible sramecc:
diff --git a/lld/test/ELF/gnu-ifunc-plt.s b/lld/test/ELF/gnu-ifunc-plt.s
index 540bfbc5325c8b22bebdbe86f841794287099110..58fae803a0e52db4e67644d317f4564165fd6cbe 100644
--- a/lld/test/ELF/gnu-ifunc-plt.s
+++ b/lld/test/ELF/gnu-ifunc-plt.s
@@ -80,9 +80,9 @@
 // Test that --shuffle-sections does not affect the order of relocations and that
 // we still place IRELATIVE relocations last. Check both random seed (0) and an
 // arbitrary seed that was known to break the order of relocations previously (3).
-// RUN: ld.lld --shuffle-sections=3 %t.so %t.o -o %tout2
+// RUN: ld.lld --shuffle-sections='*=3' %t.so %t.o -o %tout2
 // RUN: llvm-readobj --relocations %tout2 | FileCheck %s --check-prefix=SHUFFLE
-// RUN: ld.lld --shuffle-sections=0 %t.so %t.o -o %tout3
+// RUN: ld.lld --shuffle-sections='*=0' %t.so %t.o -o %tout3
 // RUN: llvm-readobj --relocations %tout3 | FileCheck %s --check-prefix=SHUFFLE
 
 // SHUFFLE:      Section {{.*}} .rela.dyn {
diff --git a/lld/test/ELF/lto/amdgcn-oses.ll b/lld/test/ELF/lto/amdgcn-oses.ll
index 714fa5a94c594693f19004c762226a6723ba0885..a2f25cdd57d87b56c383e7267ff4a540c732894f 100644
--- a/lld/test/ELF/lto/amdgcn-oses.ll
+++ b/lld/test/ELF/lto/amdgcn-oses.ll
@@ -15,7 +15,7 @@
 ; RUN: llvm-readobj --file-headers %t/mesa3d.so | FileCheck %s --check-prefixes=GCN,NON-AMDHSA,MESA3D
 
 ; AMDHSA: OS/ABI: AMDGPU_HSA (0x40)
-; AMDHSA: ABIVersion: 1
+; AMDHSA: ABIVersion: 2
 
 ; AMDPAL: OS/ABI: AMDGPU_PAL (0x41)
 ; MESA3D: OS/ABI: AMDGPU_MESA3D (0x42)
diff --git a/lld/test/ELF/ppc64-tls-le-relax.s b/lld/test/ELF/ppc64-tls-le-relax.s
new file mode 100644
index 0000000000000000000000000000000000000000..3c3bc33a65b73e2cad5b8343877268f6a8496477
--- /dev/null
+++ b/lld/test/ELF/ppc64-tls-le-relax.s
@@ -0,0 +1,81 @@
+# REQUIRES: ppc
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %t/initexec -o %t/initexec.o
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %t/defs -o %t/defs.o
+# RUN: ld.lld %t/initexec.o %t/defs.o -o %t/out
+# RUN: llvm-objdump -d --mcpu=pwr10 --no-show-raw-insn %t/out | FileCheck %s
+
+# CHECK-LABEL: <GetAddrT>:
+# CHECK:         mflr 0
+# CHECK-NEXT:    std 30, -16(1)
+# CHECK-NEXT:    std 0, 16(1)
+# CHECK-NEXT:    stdu 1, -48(1)
+# CHECK-NEXT:    paddi 3, 13, -28672, 0
+# CHECK-NEXT:    mr 30, 3
+# CHECK-NEXT:    mr 3, 30
+# CHECK-NEXT:    bl
+# CHECK-NEXT:    mr 4, 30
+# CHECK-NEXT:    addi 1, 1, 48
+# CHECK-NEXT:    ld 0, 16(1)
+# CHECK-NEXT:    ld 30, -16(1)
+# CHECK-NEXT:    mtlr 0
+# CHECK-NEXT:    b
+
+## Generated From:
+## extern __thread unsigned TGlobal;
+## unsigned getConst(unsigned*);
+## unsigned addVal(unsigned, unsigned*);
+##
+## unsigned GetAddrT() {
+##   return addVal(getConst(&TGlobal), &TGlobal);
+## }
+
+//--- initexec
+GetAddrT:
+  mflr 0
+  std 30, -16(1)
+  std 0, 16(1)
+  stdu 1, -48(1)
+  pld 3, TGlobal@got@tprel@pcrel(0), 1
+  add 30, 3, TGlobal@tls@pcrel
+  mr      3, 30
+  bl getConst@notoc
+  mr      4, 30
+  addi 1, 1, 48
+  ld 0, 16(1)
+  ld 30, -16(1)
+  mtlr 0
+  b addVal@notoc
+
+## Generated From:
+## __thread unsigned TGlobal;
+##
+## unsigned getConst(unsigned* A) {
+##   return *A + 3;
+## }
+##
+## unsigned addVal(unsigned A, unsigned* B) {
+##   return A + *B;
+## }
+
+//--- defs
+.globl  getConst
+getConst:
+  lwz 3, 0(3)
+  addi 3, 3, 3
+  clrldi  3, 3, 32
+  blr
+
+.globl  addVal
+addVal:
+  lwz 4, 0(4)
+  add 3, 4, 3
+  clrldi  3, 3, 32
+  blr
+
+.section        .tbss,"awT",@nobits
+.globl  TGlobal
+.p2align        2
+TGlobal:
+  .long   0
+  .size   TGlobal, 4
diff --git a/lld/test/ELF/ppc64-tls-pcrel-ie.s b/lld/test/ELF/ppc64-tls-pcrel-ie.s
index 93a286a2423721d69e150628d7f8dd267aa228a2..eaa8619086018e9ce1d57d61daac7c3c3879da3b 100644
--- a/lld/test/ELF/ppc64-tls-pcrel-ie.s
+++ b/lld/test/ELF/ppc64-tls-pcrel-ie.s
@@ -54,9 +54,9 @@ y:
 
 # LE-RELOC: There are no relocations in this file.
 
-# LE-SYM: Symbol table '.symtab' contains 7 entries:
-# LE-SYM: 5: 0000000000000000     0 TLS     GLOBAL DEFAULT     6 x
-# LE-SYM: 6: 0000000000000004     0 TLS     GLOBAL DEFAULT     6 y
+# LE-SYM: Symbol table '.symtab' contains 8 entries:
+# LE-SYM: 6: 0000000000000000     0 TLS     GLOBAL DEFAULT     6 x
+# LE-SYM: 7: 0000000000000004     0 TLS     GLOBAL DEFAULT     6 y
 
 # LE-GOT: could not find section '.got'
 
@@ -74,6 +74,20 @@ IEAddr:
 	add 3, 3, x@tls@pcrel
 	blr
 
+# IE-LABEL: <IEAddrCopy>:
+# IE-NEXT:    pld 3, 12488(0), 1
+# IE-NEXT:    add 4, 3, 13
+# IE-NEXT:    blr
+# LE-LABEL: <IEAddrCopy>:
+# LE-NEXT:    paddi 3, 13, -28672, 0
+# LE-NEXT:    mr 4, 3
+# LE-NEXT:    blr
+.section .text_addr, "ax", %progbits
+IEAddrCopy:
+	pld 3, x@got@tprel@pcrel(0), 1
+	add 4, 3, x@tls@pcrel
+	blr
+
 # IE-LABEL: <IEVal>:
 # IE-NEXT:    pld 3, 8408(0), 1
 # IE-NEXT:    lwzx 3, 3, 13
diff --git a/lld/test/ELF/shuffle-sections-init-fini.s b/lld/test/ELF/shuffle-sections-init-fini.s
index d98ca8d359de8e54232ca3d5b4368797d2fdca5f..4ddbf6cb74835c0c8d2e8185e4aa5e5c1182c52f 100644
--- a/lld/test/ELF/shuffle-sections-init-fini.s
+++ b/lld/test/ELF/shuffle-sections-init-fini.s
@@ -5,7 +5,7 @@
 # RUN: llvm-readelf -x .init -x .fini -x .init_array -x .fini_array %t | \
 # RUN:   FileCheck --check-prefixes=CHECK,ORDERED %s
 
-# RUN: ld.lld %t.o --shuffle-sections=1 -o %t1
+# RUN: ld.lld %t.o --shuffle-sections '*=1' -o %t1
 # RUN: llvm-readelf -x .init -x .fini -x .init_array -x .fini_array %t1 | \
 # RUN:   FileCheck --check-prefixes=CHECK,SHUFFLED %s
 
@@ -21,12 +21,12 @@
 # CHECK:      Hex dump of section '.init_array'
 # CHECK-NEXT: 0x{{[0-9a-f]+}} ff
 # ORDERED-SAME: 000102 03040506 0708090a 0b
-# SHUFFLED-SAME: 04000b 06010a08 09070203 05
+# SHUFFLED-SAME: 080301 04050907 0b020a06 00
 
 # CHECK:      Hex dump of section '.fini_array'
 # CHECK-NEXT: 0x{{[0-9a-f]+}} ff
 # ORDERED-SAME:  000102 03040506 0708090a 0b
-# SHUFFLED-SAME: 090401 070b0003 080a0605 02
+# SHUFFLED-SAME: 0a0405 08070b02 03090006 01
 
 ## With a SECTIONS command, SHT_INIT_ARRAY prirotities are ignored.
 ## All .init_array* are shuffled together.
@@ -36,13 +36,13 @@
 # RUN: ld.lld -T %t.script %t.o -o %t2
 # RUN: llvm-readelf -x .init -x .fini -x .init_array -x .fini_array %t2 | \
 # RUN:   FileCheck --check-prefixes=CHECK2,ORDERED2 %s
-# RUN: ld.lld -T %t.script %t.o --shuffle-sections=1 -o %t3
+# RUN: ld.lld -T %t.script %t.o --shuffle-sections '*=1' -o %t3
 # RUN: llvm-readelf -x .init -x .fini -x .init_array -x .fini_array %t3 | \
 # RUN:   FileCheck --check-prefixes=CHECK2,SHUFFLED2 %s
 
 # CHECK2:       Hex dump of section '.init_array'
 # ORDERED2-NEXT:  0x{{[0-9a-f]+}} 00010203 04050607 08090a0b ff
-# SHUFFLED2-NEXT: 0x{{[0-9a-f]+}} 04000b06 010a0809 07ff0203 05
+# SHUFFLED2-NEXT: 0x{{[0-9a-f]+}} 08030104 0509070b 02ff0a06 00
 
 .irp i,0,1,2,3,4,5,6,7,8,9,10,11
   .section .init,"ax",@progbits,unique,\i
diff --git a/lld/test/ELF/shuffle-sections.s b/lld/test/ELF/shuffle-sections.s
index bc0f57b98d7a41c5ae534212d6d61e6f50a2f3cd..8211c482732b45c6c7d467cfa0dacf931983d8e4 100644
--- a/lld/test/ELF/shuffle-sections.s
+++ b/lld/test/ELF/shuffle-sections.s
@@ -7,25 +7,53 @@
 # CHECK-NEXT: 01020304
 
 ## --shuffle-sections= shuffles input sections.
-# RUN: ld.lld --shuffle-sections=1 %t.o -o %t1.out
+# RUN: ld.lld --shuffle-sections='*=1' %t.o -o %t1.out
 # RUN: llvm-readelf -x .text %t1.out | FileCheck %s --check-prefix=SHUFFLE1
 # SHUFFLE1: Hex dump of section '.text':
-# SHUFFLE1-NEXT: 0204cccc 0103
+# SHUFFLE1-NEXT: 0203cccc 0104
 
 ## Test that --shuffle-sections= can be used with --symbol-ordering-file
 # RUN: echo "foo" > %t_order.txt
 # RUN: echo "_start " >> %t_order.txt
 
-# RUN: ld.lld --symbol-ordering-file %t_order.txt --shuffle-sections=2 %t.o -o %t2.out
+# RUN: ld.lld --symbol-ordering-file %t_order.txt --shuffle-sections='*=2' %t.o -o %t2.out
 # RUN: llvm-readelf -x .text %t2.out | FileCheck %s --check-prefix=SHUFFLE2
 # SHUFFLE2: Hex dump of section '.text':
-# SHUFFLE2-NEXT: 02cccccc 010304
+# SHUFFLE2-NEXT: 02cccccc 010403
 
-# RUN: ld.lld --symbol-ordering-file %t_order.txt --shuffle-sections=3 %t.o -o %t3.out
+# RUN: ld.lld --symbol-ordering-file %t_order.txt --shuffle-sections='*=3' %t.o -o %t3.out
 # RUN: llvm-readelf -x .text %t3.out | FileCheck %s --check-prefix=SHUFFLE3
 # SHUFFLE3: Hex dump of section '.text':
 # SHUFFLE3-NEXT: 02cccccc 010403
 
+## As a special case, -1 reverses sections as a stable transform.
+# RUN: ld.lld --shuffle-sections '*=-1' %t.o -o %t-1.out
+# RUN: llvm-readelf -x .text %t-1.out | FileCheck %s --check-prefix=SHUFFLE-1
+# SHUFFLE-1: Hex dump of section '.text':
+# SHUFFLE-1-NEXT: 040302cc 01
+
+## .text does not change its order while .text.{foo,bar,zed} are reversed.
+# RUN: ld.lld --shuffle-sections '.text.*=-1' %t.o -o %t4.out
+# RUN: llvm-readelf -x .text %t4.out | FileCheck %s --check-prefix=SHUFFLE4
+# SHUFFLE4: Hex dump of section '.text':
+# SHUFFLE4-NEXT: 01040302
+
+## Reversing twice restores the original order.
+# RUN: ld.lld --shuffle-sections '.text.*=-1' --shuffle-sections '.text.*=-1' %t.o -o %t.out
+# RUN: llvm-readelf -x .text %t.out | FileCheck %s
+
+## Test all possible invalid cases.
+# RUN: not ld.lld --shuffle-sections= 2>&1 | FileCheck %s --check-prefix=USAGE -DV=
+# RUN: not ld.lld --shuffle-sections=a= 2>&1 | FileCheck %s --check-prefix=USAGE -DV=a=
+# RUN: not ld.lld --shuffle-sections==0 2>&1 | FileCheck %s --check-prefix=USAGE -DV==0
+# RUN: not ld.lld --shuffle-sections=a 2>&1 | FileCheck %s --check-prefix=USAGE -DV=a
+
+# USAGE: error: --shuffle-sections=: expected <section_glob>=<seed>, but got '[[V]]'
+
+# RUN: not ld.lld --shuffle-sections='['=0 2>&1 | FileCheck %s --check-prefix=INVALID
+
+# INVALID: error: --shuffle-sections=: invalid glob pattern: [
+
 ## .text has an alignment of 4.
 .global _start
 _start:
diff --git a/lld/test/MachO/Inputs/DependencyDump.py b/lld/test/MachO/Inputs/DependencyDump.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1c1151d33fab50f7600f753edb3f70385855b08
--- /dev/null
+++ b/lld/test/MachO/Inputs/DependencyDump.py
@@ -0,0 +1,26 @@
+#
+# Dump the dependency file (produced with -dependency_info) to text
+# format for testing purposes.
+#
+
+import sys
+
+f = open(sys.argv[1], "rb")
+byte = f.read(1)
+while byte != b'':
+    if byte == b'\x00':
+        sys.stdout.write("lld-version: ")
+    elif byte == b'\x10':
+        sys.stdout.write("input-file: ")
+    elif byte == b'\x11':
+        sys.stdout.write("not-found: ")
+    elif byte == b'\x40':
+        sys.stdout.write("output-file: ")
+    byte = f.read(1)
+    while byte != b'\x00':
+        sys.stdout.write(byte.decode("ascii"))
+        byte = f.read(1)
+    sys.stdout.write("\n")
+    byte = f.read(1)
+
+f.close()
diff --git a/lld/test/MachO/dependency-info.s b/lld/test/MachO/dependency-info.s
new file mode 100644
index 0000000000000000000000000000000000000000..9e69ea16980f3c67babea6516a0e0077387f0386
--- /dev/null
+++ b/lld/test/MachO/dependency-info.s
@@ -0,0 +1,44 @@
+# REQUIRES: x86
+#
+# RUN: rm -rf %t
+# RUN: split-file %s %t
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/foo.o %t/foo.s
+# RUN: %lld -dylib -o %t/libfoo.dylib %t/foo.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/bar.o %t/bar.s
+# RUN: llvm-ar csr  %t/bar.a %t/bar.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/main.o %t/main.s
+
+# RUN: %lld %t/main.o %t/bar.a %t/libfoo.dylib -lSystem -o %t/test.out -dependency_info %t/deps_info.out
+# RUN: %python %S/Inputs/DependencyDump.py %t/deps_info.out | FileCheck %s
+
+# CHECK: lld-version: {{.*}}LLD {{.*}}
+
+# CHECK-DAG: input-file: {{.*}}{{[/\]}}bar.a
+# CHECK-DAG: input-file: {{.*}}{{[/\]}}libfoo.dylib
+# CHECK-DAG: input-file: {{.*}}{{[/\]}}libSystem.tbd
+# CHECK-DAG: input-file: {{.*}}{{[/\]}}main.o
+# CHECK-DAG: input-file: {{.*}}bar.o
+
+# CHECK-NEXT: not-found: {{.*}}{{[/\]}}libdyld.{{.*}}
+## There could be more not-found here but we are not checking those because it's brittle.
+
+# CHECK: output-file: {{.*}}{{[/\]}}test.out
+
+#--- foo.s
+.globl __Z3foo
+__Z3foo:
+  ret
+
+#--- bar.s
+.globl _bar
+_bar:
+  callq __Z3foo
+  ret
+
+#--- main.s
+.globl _main
+_main:
+  callq _bar
+  callq __Z3foo
+  ret
diff --git a/lld/test/MachO/export-options.s b/lld/test/MachO/export-options.s
index 51fcdb12f60ffbd4bb2092126fb09a1877b8633d..419accf03ad395e25b881545e3cfa750526d0e43 100644
--- a/lld/test/MachO/export-options.s
+++ b/lld/test/MachO/export-options.s
@@ -11,6 +11,23 @@
 # CONFLICT: error: cannot use both -exported_symbol* and -unexported_symbol* options
 # CONFLICT-NEXT: >>> ignoring unexports
 
+## Check that exported literal symbol name is present in symbol table
+# RUN: not %lld -dylib %t/default.o -o /dev/null \
+# RUN:         -exported_symbol absent_literal \
+# RUN:         -exported_symbol absent_gl?b 2>&1 | \
+# RUN:     FileCheck --check-prefix=UNDEF %s
+
+# UNDEF: error: undefined symbol absent_literal
+# UNDEF-NEXT: >>> referenced from option -exported_symbol(s_list)
+# UNDEF-NOT: error: {{.*}} absent_gl{{.}}b
+
+## Check that exported symbol is global
+# RUN: not %lld -dylib %t/default.o -o /dev/null \
+# RUN:         -exported_symbol _private 2>&1 | \
+# RUN:     FileCheck --check-prefix=PRIVATE %s
+
+# PRIVATE: error: cannot export hidden symbol _private
+
 #--- default.s
 
 .macro DEFSYM, type, sym
@@ -21,8 +38,7 @@
 
 DEFSYM .globl, _keep_globl
 DEFSYM .globl, _hide_globl
-DEFSYM .private_extern, _keep_private
-DEFSYM .private_extern, _show_private
+DEFSYM .private_extern, _private
 
 ## Check that the export trie is unaltered
 # RUN: %lld -dylib %t/default.o -o %t/default
@@ -32,36 +48,35 @@ DEFSYM .private_extern, _show_private
 # DEFAULT-LABEL: Exports trie:
 # DEFAULT-DAG:   _hide_globl
 # DEFAULT-DAG:   _keep_globl
-# DEFAULT-NOT:   _hide_private
-# DEFAULT-NOT:   _show_private
-
-## Check that the export trie is properly augmented
-## Check that non-matching literal pattern has no effect
-# RUN: %lld -dylib %t/default.o -o %t/export \
-# RUN:     -exported_symbol _show_private \
-# RUN:     -exported_symbol _extra_cruft -exported_symbol '*xtra_cr?ft'
-# RUN: llvm-objdump --macho --exports-trie %t/export | \
-# RUN:     FileCheck --check-prefix=EXPORTED %s
-
-# EXPORTED-LABEL: Exports trie:
-# EXPORTED-DAG:   _show_private
-# EXPORTED-NOT:   _hide_globl
-# EXPORTED-NOT:   _keep_globl
-# EXPORTED-NOT:   _hide_private
-# EXPORTED-NOT:   {{.*}}xtra_cr{{.}}ft
-
-## Check that the export trie is properly diminished
-## Check that non-matching glob pattern has no effect
-# RUN: %lld -dylib %t/default.o -o %t/unexport \
-# RUN:     -unexported_symbol _hide_global
-# RUN: llvm-objdump --macho --exports-trie %t/unexport | \
-# RUN:     FileCheck --check-prefix=UNEXPORTED %s
-
-# UNEXPORTED-LABEL: Exports trie:
-# UNEXPORTED-DAG:   _keep_globl
-# UNEXPORTED-NOT:   _hide_globl
-# UNEXPORTED-NOT:   _show_private
-# UNEXPORTED-NOT:   _hide_private
+# DEFAULT-NOT:   _private
+
+## Check that the export trie is shaped by an allow list and then
+## by a deny list. Both lists are designed to yield the same result.
+
+## Check the allow list
+# RUN: %lld -dylib %t/default.o -o %t/allowed \
+# RUN:     -exported_symbol _keep_globl
+# RUN: llvm-objdump --macho --exports-trie %t/allowed | \
+# RUN:     FileCheck --check-prefix=TRIE %s
+# RUN: llvm-nm -m %t/allowed | \
+# RUN:     FileCheck --check-prefix=NM %s
+
+## Check the deny list
+# RUN: %lld -dylib %t/default.o -o %t/denied \
+# RUN:     -unexported_symbol _hide_globl
+# RUN: llvm-objdump --macho --exports-trie %t/denied | \
+# RUN:     FileCheck --check-prefix=TRIE %s
+# RUN: llvm-nm -m %t/denied | \
+# RUN:     FileCheck --check-prefix=NM %s
+
+# TRIE-LABEL: Exports trie:
+# TRIE-DAG:   _keep_globl
+# TRIE-NOT:   _hide_globl
+# TRIE-NOT:   _private
+
+# NM-DAG: external _keep_globl
+# NM-DAG: non-external (was a private external) _hide_globl
+# NM-DAG: non-external (was a private external) _private
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos \
 # RUN:     %t/symdefs.s -o %t/symdefs.o
@@ -69,7 +84,7 @@ DEFSYM .private_extern, _show_private
 #--- symdefs.s
 
 .macro DEFSYM, sym
-.private_extern \sym
+.globl \sym
 \sym:
   retq
 .endm
diff --git a/lld/test/MachO/export-trie.s b/lld/test/MachO/export-trie.s
index c479e166561ccc16797abdbb830443f33307d943..cab8312fe1d0c7e11370cce73d9ad9e3af136c9b 100644
--- a/lld/test/MachO/export-trie.s
+++ b/lld/test/MachO/export-trie.s
@@ -15,7 +15,9 @@
 # EXPORTS-DAG:   [[#%x, HELLO_WORLD_ADDR:]] {{.*}} _hello_world
 # EXPORTS-DAG:   [[#%x, HELLO_ITS_ME_ADDR:]] {{.*}} _hello_its_me
 # EXPORTS-DAG:   [[#%x, HELLO_ITS_YOU_ADDR:]] {{.*}} _hello_its_you
+# EXPORTS-DAG:   {{0+}} g *ABS* __mh_execute_header
 # EXPORTS-LABEL: Exports trie:
+# EXPORTS-DAG:   0x{{0+}} __mh_execute_header [absolute]
 # EXPORTS-DAG:   0x{{0*}}[[#%X, MAIN_ADDR]] _main
 # EXPORTS-DAG:   0x{{0*}}[[#%X, HELLO_ADDR]] _hello
 # EXPORTS-DAG:   0x{{0*}}[[#%X, HELLO_WORLD_ADDR]] _hello_world
@@ -27,13 +29,16 @@
 # CHECK-LABEL: ExportTrie:
 # CHECK: Name: ''
 # CHECK: Name: _
-# CHECK: Name: main
-# CHECK: Name: hello
+# CHECK-DAG: Name: _mh_execute_header
+# CHECK-DAG: Name: main
+# CHECK-DAG: Name: hello
 # CHECK: Name: _
 # CHECK: Name: world
 # CHECK: Name: its_
-# CHECK: Name: you
-# CHECK: Name: me
+# CHECK-DAG: Name: you
+# CHECK-DAG: Name: me
+
+
 
 .section __TEXT,__cstring
 .globl _hello, _hello_world, _hello_its_me, _hello_its_you, _main
diff --git a/lld/test/MachO/function-starts.s b/lld/test/MachO/function-starts.s
index ce2d4299cbdd7f81baa3aa3991d5f933dd1acabc..d5168c630b6176751153956120924834e9f93c18 100644
--- a/lld/test/MachO/function-starts.s
+++ b/lld/test/MachO/function-starts.s
@@ -1,33 +1,40 @@
 # REQUIRES: x86
 
-# RUN: split-file %s %t
-
-# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/basic.s -o %t.basic.o
-# RUN: %lld %t.basic.o -o %t.basic
-# RUN: llvm-objdump --syms %t.basic > %t.objdump
-# RUN: llvm-objdump --macho --function-starts %t.basic >> %t.objdump
-# RUN: FileCheck %s --check-prefix=BASIC < %t.objdump
-
-# BASIC:      SYMBOL TABLE:
-# BASIC-NEXT: [[#%x,MAIN:]] g F __TEXT,__text _main
-# BASIC-NEXT: [[#%x,F1:]] g F __TEXT,__text _f1
-# BASIC-NEXT: [[#%x,F2:]] g F __TEXT,__text _f2
-# BASIC:      [[#MAIN]]
-# BASIC:      [[#F1]]
-# BASIC:      [[#F2]]
-
-# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/alias.s -o %t.alias.o
-# RUN: %lld %t.alias.o -o %t.alias
-# RUN: llvm-objdump --syms  %t.alias > %t.objdump
-# RUN: llvm-objdump --macho --function-starts %t.alias >> %t.objdump
-# RUN: FileCheck %s --check-prefix=ALIAS < %t.objdump
-
-# ALIAS:      SYMBOL TABLE:
-# ALIAS-NEXT: [[#%x,F2:]] l F __TEXT,__text _f2
-# ALIAS-NEXT: [[#%x,MAIN:]] g F __TEXT,__text _main
-# ALIAS-NEXT: [[#%x,F1:]] g F __TEXT,__text _f1
-# ALIAS:      [[#MAIN]]
-# ALIAS:      [[#F1]]
+# RUN: rm -rf %t; split-file %s %t
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/basic.s -o %t/basic.o
+# RUN: %lld %t/basic.o -o %t/basic
+# RUN: llvm-objdump --syms %t/basic > %t/objdump
+# RUN: llvm-objdump --macho --function-starts %t/basic >> %t/objdump
+# RUN: FileCheck %s --check-prefix=BASIC < %t/objdump
+
+# BASIC-LABEL: SYMBOL TABLE:
+# BASIC-NEXT:  [[#%x,MAIN:]] g F __TEXT,__text _main
+# BASIC-NEXT:  [[#%x,F1:]] g F __TEXT,__text _f1
+# BASIC-NEXT:  [[#%x,F2:]] g F __TEXT,__text _f2
+# BASIC-LABEL: basic:
+# BASIC:       [[#MAIN]]
+# BASIC:       [[#F1]]
+# BASIC:       [[#F2]]
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/alias.s -o %t/alias.o
+# RUN: %lld %t/alias.o -o %t/alias
+# RUN: llvm-objdump --syms  %t/alias > %t/objdump
+# RUN: llvm-objdump --macho --function-starts %t/alias >> %t/objdump
+# RUN: FileCheck %s --check-prefix=ALIAS < %t/objdump
+
+# ALIAS-LABEL: SYMBOL TABLE:
+# ALIAS-NEXT:  [[#%x,F2:]] l F __TEXT,__text _f2
+# ALIAS-NEXT:  [[#%x,MAIN:]] g F __TEXT,__text _main
+# ALIAS-NEXT:  [[#%x,F1:]] g F __TEXT,__text _f1
+# ALIAS-LABEL: alias:
+# ALIAS:       [[#MAIN]]
+# ALIAS:       [[#F1]]
+
+# RUN: %lld %t/basic.o -no_function_starts -o %t/basic-no-function-starts
+# RUN: llvm-objdump --macho --function-starts %t/basic-no-function-starts | FileCheck %s --check-prefix=NO-FUNCTION-STARTS
+# NO-FUNCTION-STARTS: basic-no-function-starts:
+# NO-FUNCTION-STARTS-EMPTY:
 
 #--- basic.s
 .section  __TEXT,__text,regular,pure_instructions
diff --git a/lld/test/MachO/map-file.s b/lld/test/MachO/map-file.s
new file mode 100644
index 0000000000000000000000000000000000000000..215d193aa622fa1d9728f123d7614a6a9c016f3a
--- /dev/null
+++ b/lld/test/MachO/map-file.s
@@ -0,0 +1,51 @@
+# REQUIRES: x86
+# RUN: rm -rf %t; split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/foo.s -o %t/foo.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/test.s -o %t/test.o
+
+# RUN: %lld -map %t/map %t/test.o %t/foo.o -o %t/test-map
+# RUN: llvm-objdump --syms --section-headers %t/test-map > %t/objdump
+# RUN: cat %t/objdump %t/map > %t/out
+# RUN: FileCheck %s < %t/out
+
+#--- foo.s
+.section __TEXT,obj
+.globl _foo
+_foo:
+
+#--- test.s
+.comm _number, 1
+.globl _main
+_main:
+  ret
+
+# CHECK:      Sections:
+# CHECK-NEXT: Idx  Name          Size           VMA           Type
+# CHECK-NEXT: 0    __text        {{[0-9a-f]+}}  [[#%x,TEXT:]] TEXT
+# CHECK-NEXT: 1    obj           {{[0-9a-f]+}}  [[#%x,DATA:]] DATA
+# CHECK-NEXT: 2    __common      {{[0-9a-f]+}}  [[#%x,BSS:]]  BSS
+
+# CHECK: SYMBOL TABLE:
+# CHECK-NEXT: [[#%x,MAIN:]]   g     F __TEXT,__text _main
+# CHECK-NEXT: [[#%x,NUMBER:]] g     O __DATA,__common _number
+# CHECK-NEXT: [[#%x,FOO:]]    g     O __TEXT,obj _foo
+# CHECK-NEXT: {{0+}}          g       *ABS* __mh_execute_header
+
+# CHECK-NEXT: # Path: {{.*}}{{/|\\}}map-file.s.tmp/test-map
+# CHECK-NEXT: # Arch: x86_64
+# CHECK-NEXT: # Object files:
+# CHECK-NEXT: [  0] linker synthesized
+# CHECK-NEXT: [  1] {{.*}}{{/|\\}}map-file.s.tmp/test.o
+# CHECK-NEXT: [  2] {{.*}}{{/|\\}}map-file.s.tmp/foo.o
+
+# CHECK-NEXT: # Sections:
+# CHECK-NEXT: # Address    Size              Segment    Section
+# CHECK-NEXT: 0x[[#TEXT]]  0x{{[0-9a-f]+}}   __TEXT  __text
+# CHECK-NEXT: 0x[[#DATA]]  0x{{[0-9a-f]+}}   __TEXT  obj
+# CHECK-NEXT: 0x[[#BSS]]   0x{{[0-9a-f]+}}   __DATA  __common
+
+# CHECK-NEXT: # Symbols:
+# CHECK-NEXT: # Address        File  Name
+# CHECK-NEXT: 0x[[#NUMBER]]    [  1]  _number
+# CHECK-NEXT: 0x[[#MAIN]]      [  1]  _main
+# CHECK-NEXT: 0x[[#FOO]]       [  2]  _foo
diff --git a/lld/test/MachO/mh-execute-header.s b/lld/test/MachO/mh-execute-header.s
new file mode 100644
index 0000000000000000000000000000000000000000..4a62b27bb8fa869ba70d58c2b1f085c5c75a3669
--- /dev/null
+++ b/lld/test/MachO/mh-execute-header.s
@@ -0,0 +1,16 @@
+# REQUIRES: x86
+# RUN: rm -rf %t; mkdir %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t/test.o
+# RUN: %lld -o %t/test.pie %t/test.o -pie
+# RUN: llvm-objdump --macho --syms %t/test.pie | FileCheck %s --check-prefix=PIE
+
+# RUN: %lld -o %t/test.no_pie %t/test.o -no_pie
+# RUN: llvm-objdump --macho --syms %t/test.no_pie | FileCheck %s --check-prefix=NO-PIE
+
+# PIE:    0000000100000000 g     F __TEXT,__text __mh_execute_header
+# NO-PIE: 0000000000000000 g       *ABS* __mh_execute_header
+
+.text
+.global _main
+_main:
+  ret
diff --git a/lld/test/MachO/mh-header-link.s b/lld/test/MachO/mh-header-link.s
new file mode 100644
index 0000000000000000000000000000000000000000..0813435397b73ef380ddee893565dda668d862eb
--- /dev/null
+++ b/lld/test/MachO/mh-header-link.s
@@ -0,0 +1,43 @@
+# REQUIRES: x86
+
+## This tests that we can link against these synthetic symbols even
+## if they are not in the symbol table.
+
+# RUN: rm -rf %t; split-file %s %t
+
+## Test that in a dylib, we can link against __mh_dylib_header
+## (but not in other types of files)
+
+# RUN: llvm-mc %t/dylib.s -triple=x86_64-apple-macos11.0 -filetype=obj -o %t/dylib.o
+# RUN: %lld -pie -dylib %t/dylib.o -o %t/dylib.out
+# RUN: llvm-objdump -m --syms %t/dylib.out | FileCheck %s --check-prefix DYLIB
+
+# RUN: not %lld -pie -o /dev/null %t/dylib.o 2>&1 | FileCheck %s --check-prefix ERR-DYLIB
+
+# DYLIB:      SYMBOL TABLE:
+# DYLIB-NEXT: {{[0-9a-f]+}} g     F __TEXT,__text _main
+# DYLIB-NEXT-EMPTY:
+# ERR-DYLIB: error: undefined symbol: __mh_dylib_header
+
+## Test that in an executable, we can link against __mh_execute_header
+# RUN: llvm-mc %t/main.s -triple=x86_64-apple-macos11.0 -filetype=obj -o %t/exec.o
+# RUN: %lld -pie %t/exec.o -o %t/exec.out
+
+## But it would be an error trying to reference __mh_execute_header in a dylib
+# RUN: not %lld -pie -o /dev/null -dylib %t/exec.o 2>&1 | FileCheck %s --check-prefix ERR-EXEC
+
+# ERR-EXEC: error: undefined symbol: __mh_execute_header
+
+#--- main.s
+.text
+.globl _main
+_main:
+ mov __mh_execute_header@GOTPCREL(%rip), %rax
+ ret
+
+#--- dylib.s
+.text
+.globl _main
+_main:
+ mov __mh_dylib_header@GOTPCREL(%rip), %rax
+ ret
diff --git a/lld/test/MachO/objc.s b/lld/test/MachO/objc.s
index 53dd12e8f190912cc015e845c30d984ab40d6a25..dafee74796d0839157a0687f6c056fe0caf8225e 100644
--- a/lld/test/MachO/objc.s
+++ b/lld/test/MachO/objc.s
@@ -32,6 +32,7 @@
 # NO-OBJC-EMPTY:
 # NO-OBJC-NEXT:  SYMBOL TABLE:
 # NO-OBJC-NEXT:  g     F __TEXT,__text _main
+# NO-OBJC-NEXT:  g *ABS* __mh_execute_header
 # NO-OBJC-EMPTY:
 
 #--- has-objc-symbol.s
diff --git a/lld/test/MachO/stabs.s b/lld/test/MachO/stabs.s
index 09735feb5b263782317a01cbc1772215e78f0471..6a72ff246e1da5b03ec8523bf11c0e3b68a289d3 100644
--- a/lld/test/MachO/stabs.s
+++ b/lld/test/MachO/stabs.s
@@ -60,6 +60,7 @@
 # CHECK-NEXT:  [[#ZERO]]        S _zero
 # CHECK-NEXT:  [[#FOO]]         T _foo
 # CHECK-NEXT:  {{[0-9af]+}}     T _no_debug
+# CHECK-NEXT:  {{0+}}           A __mh_execute_header
 # CHECK-EMPTY:
 
 ## Check that we don't attempt to emit rebase opcodes for the debug sections
diff --git a/lld/test/MachO/symtab.s b/lld/test/MachO/symtab.s
index 44d42068ffb53f1cc60d9a19879d488cc9b96a95..35fc961d1b350fd00eb54598160818ce5be630fe 100644
--- a/lld/test/MachO/symtab.s
+++ b/lld/test/MachO/symtab.s
@@ -56,6 +56,16 @@
 # CHECK-NEXT:     ]
 # CHECK-NEXT:     Value: 0x1{{[0-9a-f]*}}
 # CHECK-NEXT:   }
+# CHECK-NEXT:  Symbol {
+# CHECK-NEXT:    Name: __mh_execute_header (81)
+# CHECK-NEXT:    Extern
+# CHECK-NEXT:    Type: Abs (0x2)
+# CHECK-NEXT:    Section:  (0x0)
+# CHECK-NEXT:    RefType: UndefinedNonLazy (0x0)
+# CHECK-NEXT:    Flags [ (0x0)
+# CHECK-NEXT:    ]
+# CHECK-NEXT:    Value: 0x0
+# CHECK-NEXT:  }
 # CHECK-NEXT:   Symbol {
 # CHECK-NEXT:     Name: dyld_stub_binder
 # CHECK-NEXT:     Extern
@@ -82,8 +92,8 @@
 # CHECK-NEXT:   ilocalsym: 0
 # CHECK-NEXT:   nlocalsym: 2
 # CHECK-NEXT:   iextdefsym: 2
-# CHECK-NEXT:   nextdefsym: 3
-# CHECK-NEXT:   iundefsym: 5
+# CHECK-NEXT:   nextdefsym: 4
+# CHECK-NEXT:   iundefsym: 6
 # CHECK-NEXT:   nundefsym: 2
 
 ## Verify that the first entry in the StringTable is a space, and that
diff --git a/lld/test/MachO/thinlto-time-trace.ll b/lld/test/MachO/thinlto-time-trace.ll
new file mode 100644
index 0000000000000000000000000000000000000000..de2116fa47f2bc69e7b5da49aebf5348c4bea2d5
--- /dev/null
+++ b/lld/test/MachO/thinlto-time-trace.ll
@@ -0,0 +1,45 @@
+; REQUIRES: x86
+; RUN: rm -rf %t; split-file %s %t
+
+; Test ThinLTO with time trace
+; RUN: opt -module-summary %t/f.s -o %t/f.o
+; RUN: opt -module-summary %t/g.s -o %t/g.o
+; RUN: %lld --time-trace --time-trace-granularity=0 -dylib %t/f.o %t/g.o -o %t/libTest.dylib
+; RUN: cat %t/libTest.dylib.time-trace \
+; RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
+; RUN:   | FileCheck %s
+
+; CHECK: "traceEvents": [
+; Check fields for an event are present
+; CHECK: "args":
+; CHECK-NEXT: "detail":
+; CHECK: "dur":
+; CHECK-NEXT: "name":
+; CHECK-NEXT: "ph":
+; CHECK-NEXT: "pid":
+; CHECK-NEXT: "tid":
+; CHECK-NEXT: "ts":
+
+; Check that an optimization event is present
+; CHECK: "name": "OptModule"
+
+;--- f.s
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @g(...)
+
+define void @f() {
+entry:
+  call void (...) @g()
+  ret void
+}
+
+;--- g.s
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @g() {
+entry:
+  ret void
+}
diff --git a/lld/test/MachO/threads.s b/lld/test/MachO/threads.s
new file mode 100644
index 0000000000000000000000000000000000000000..b3176388de1ec9b07e7604cb333e8a53ebabdd8d
--- /dev/null
+++ b/lld/test/MachO/threads.s
@@ -0,0 +1,16 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+
+## A positive integer is allowed.
+# RUN: %lld --threads=1 %t.o -o /dev/null
+# RUN: %lld --threads=2 %t.o -o /dev/null
+
+# RUN: not %lld --threads=all %t.o -o /dev/null 2>&1 | FileCheck %s -DN=all
+# RUN: not %lld --threads=0 %t.o -o /dev/null 2>&1 | FileCheck %s -DN=0
+# RUN: not %lld --threads=-1 %t.o -o /dev/null 2>&1 | FileCheck %s -DN=-1
+
+# CHECK: error: --threads=: expected a positive integer, but got '[[N]]'
+
+.globl _main
+_main:
+  ret
diff --git a/lld/test/MachO/time-trace.s b/lld/test/MachO/time-trace.s
index cb31dd706f9db423a66062260e969c7da74207e1..b203055a020aa964f7458ee609a41c14e699f202 100644
--- a/lld/test/MachO/time-trace.s
+++ b/lld/test/MachO/time-trace.s
@@ -1,20 +1,20 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
 
 # Test implicit trace file name
-# RUN: ld.lld --time-trace --time-trace-granularity=0 -o %t1.macho %t.o
+# RUN: %lld --time-trace --time-trace-granularity=0 -o %t1.macho %t.o
 # RUN: cat %t1.macho.time-trace \
 # RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 # RUN:   | FileCheck %s
 
 # Test specified trace file name
-# RUN: ld.lld --time-trace --time-trace-file=%t2.json --time-trace-granularity=0 -o %t2.macho %t.o
+# RUN: %lld --time-trace --time-trace-file=%t2.json --time-trace-granularity=0 -o %t2.macho %t.o
 # RUN: cat %t2.json \
 # RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 # RUN:   | FileCheck %s
 
 # Test trace requested to stdout
-# RUN: ld.lld --time-trace --time-trace-file=- --time-trace-granularity=0 -o %t3.macho %t.o \
+# RUN: %lld --time-trace --time-trace-file=- --time-trace-granularity=0 -o %t3.macho %t.o \
 # RUN:   | %python -c 'import json, sys; json.dump(json.loads(sys.stdin.read()), sys.stdout, sort_keys=True, indent=2)' \
 # RUN:   | FileCheck %s
 
@@ -33,10 +33,10 @@
 # CHECK: "name": "ExecuteLinker"
 
 # Check process_name entry field
-# CHECK: "name": "ld.lld{{(.exe)?}}"
+# CHECK: "name": "ld64.lld{{(.exe)?}}"
 # CHECK: "name": "process_name"
 # CHECK: "name": "thread_name"
 
-.globl _start
-_start:
+.globl _main
+_main:
   ret
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py
index 8e31fd3977f9a44071c147ef3c6d8109dfc1b4f4..670f41f0b63132e64e22e5e0458d6aa59a664cec 100644
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@@ -101,11 +101,13 @@ if config.sizeof_void_p == 8:
 
 tar_executable = lit.util.which('tar', config.environment['PATH'])
 if tar_executable:
+    env = os.environ
+    env['LANG'] = 'C'
     tar_version = subprocess.Popen(
         [tar_executable, '--version'],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
-        env={'LANG': 'C'})
+        env=env)
     sout, _ = tar_version.communicate()
     if 'GNU tar' in sout.decode():
         config.available_features.add('gnutar')
diff --git a/lld/test/wasm/alias.s b/lld/test/wasm/alias.s
index b2ab45e98d9598675b944338bcf11b1a9aef6dc8..0bb035b92f29c19e0376649861f1ea78bd22d966 100644
--- a/lld/test/wasm/alias.s
+++ b/lld/test/wasm/alias.s
@@ -24,7 +24,7 @@ _start:
 # CHECK-NEXT:     FunctionTypes:   [ 0 ]
 # CHECK-NEXT:   - Type:            MEMORY
 # CHECK-NEXT:     Memories:
-# CHECK-NEXT:       - Initial:         0x2
+# CHECK-NEXT:       - Minimum:         0x2
 # CHECK-NEXT:   - Type:            GLOBAL
 # CHECK-NEXT:     Globals:
 # CHECK-NEXT:       - Index:           0
diff --git a/lld/test/wasm/bsymbolic.s b/lld/test/wasm/bsymbolic.s
index bc2e4e777406921db7b160f22e4601c603f91cc2..79c4a8700fddaf7bfbdbd55cc0735edfb392ef74 100644
--- a/lld/test/wasm/bsymbolic.s
+++ b/lld/test/wasm/bsymbolic.s
@@ -14,7 +14,7 @@
 // NOOPTION-NEXT:        Field:           memory
 // NOOPTION-NEXT:        Kind:            MEMORY
 // NOOPTION-NEXT:        Memory:
-// NOOPTION-NEXT:          Initial:         0x1
+// NOOPTION-NEXT:          Minimum:         0x1
 // NOOPTION-NEXT:      - Module:          env
 // NOOPTION-NEXT:        Field:           __memory_base
 // NOOPTION-NEXT:        Kind:            GLOBAL
@@ -56,7 +56,7 @@
 // SYMBOLIC-NEXT:        Field:           memory
 // SYMBOLIC-NEXT:        Kind:            MEMORY
 // SYMBOLIC-NEXT:        Memory:
-// SYMBOLIC-NEXT:          Initial:         0x1
+// SYMBOLIC-NEXT:          Minimum:         0x1
 // SYMBOLIC-NEXT:      - Module:          env
 // SYMBOLIC-NEXT:        Field:           __memory_base
 // SYMBOLIC-NEXT:        Kind:            GLOBAL
@@ -74,7 +74,7 @@
 // SYMBOLIC-NEXT:          Index:           0
 // SYMBOLIC-NEXT:          ElemType:        FUNCREF
 // SYMBOLIC-NEXT:          Limits:
-// SYMBOLIC-NEXT:            Initial:         0x1
+// SYMBOLIC-NEXT:            Minimum:         0x1
 // SYMBOLIC-NEXT:  - Type:            FUNCTION
 
 // SYMBOLIC:       - Type:            GLOBAL
diff --git a/lld/test/wasm/call-indirect.ll b/lld/test/wasm/call-indirect.ll
index f6d3df4914bfa8296fd9050dee227e76c31ca370..31e2c75864554b28297af65e6fc2e8bd74828798 100644
--- a/lld/test/wasm/call-indirect.ll
+++ b/lld/test/wasm/call-indirect.ll
@@ -67,11 +67,11 @@ define void @call_ptr(i64 (i64)* %arg) {
 ; CHECK-NEXT:         ElemType:        FUNCREF
 ; CHECK-NEXT:         Limits:
 ; CHECK-NEXT:           Flags:           [ HAS_MAX ]
-; CHECK-NEXT:           Initial:         0x3
+; CHECK-NEXT:           Minimum:         0x3
 ; CHECK-NEXT:           Maximum:         0x3
 ; CHECK-NEXT:   - Type:            MEMORY
 ; CHECK-NEXT:     Memories:
-; CHECK-NEXT:       - Initial:         0x2
+; CHECK-NEXT:       - Minimum:         0x2
 ; CHECK-NEXT:   - Type:            GLOBAL
 ; CHECK-NEXT:     Globals:
 ; CHECK-NEXT:       - Index:           0
diff --git a/lld/test/wasm/data-layout.s b/lld/test/wasm/data-layout.s
index 72a859b8a0a10dc6fd97a7d4ea7b7cc5b9f93056..8399e9f43f1aa6422994e9ae62f0dbe79b28fa22 100644
--- a/lld/test/wasm/data-layout.s
+++ b/lld/test/wasm/data-layout.s
@@ -55,7 +55,7 @@ local_struct_internal_ptr:
 # CHECK:        - Type:            MEMORY
 # CHECK-NEXT:     Memories:
 # CHK64-NEXT:       - Flags:           [ IS_64 ]
-# CHECK-NEXT:         Initial:         0x2
+# CHECK-NEXT:         Minimum:         0x2
 # CHECK-NEXT:   - Type:            GLOBAL
 # CHECK-NEXT:     Globals:
 # CHECK-NEXT:       - Index:           0
@@ -104,7 +104,7 @@ local_struct_internal_ptr:
 # CHECK-MAX:        - Type:            MEMORY
 # CHECK-MAX-NEXT:     Memories:
 # CHECK-MAX-NEXT:       - Flags:           [ HAS_MAX ]
-# CHECK-MAX-NEXT:         Initial:         0x2
+# CHECK-MAX-NEXT:         Minimum:         0x2
 # CHECK-MAX-NEXT:         Maximum:         0x2
 
 # RUN: wasm-ld -no-gc-sections --allow-undefined --no-entry --shared-memory \
@@ -115,7 +115,7 @@ local_struct_internal_ptr:
 # CHECK-SHARED:        - Type:            MEMORY
 # CHECK-SHARED-NEXT:     Memories:
 # CHECK-SHARED-NEXT:       - Flags:           [ HAS_MAX, IS_SHARED ]
-# CHECK-SHARED-NEXT:         Initial:         0x2
+# CHECK-SHARED-NEXT:         Minimum:         0x2
 # CHECK-SHARED-NEXT:         Maximum:         0x2
 
 # XUN: wasm-ld --relocatable -o %t_reloc.wasm %t32.o %t.hello32.o
diff --git a/lld/test/wasm/export-table-explicit.test b/lld/test/wasm/export-table-explicit.test
index b6f03b99eb60726d393e20e3269d0f15a9230fae..58f0032ce081de48cbe0888a7dac8e97b98eb0dd 100644
--- a/lld/test/wasm/export-table-explicit.test
+++ b/lld/test/wasm/export-table-explicit.test
@@ -13,7 +13,7 @@
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
 # CHECK-NEXT:           Flags:           [ HAS_MAX ]
-# CHECK-NEXT:           Initial:         0x1
+# CHECK-NEXT:           Minimum:         0x1
 # CHECK-NEXT:           Maximum:         0x1
 # CHECK-NEXT:   - Type:
 
diff --git a/lld/test/wasm/export-table.test b/lld/test/wasm/export-table.test
index c0cb581e919f6fd07ec164693fe5f9ee10456a82..927eccef7ceddf6b366965afa8c44190b49d9b1a 100644
--- a/lld/test/wasm/export-table.test
+++ b/lld/test/wasm/export-table.test
@@ -10,7 +10,7 @@
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
 # CHECK-NEXT:           Flags:           [ HAS_MAX ]
-# CHECK-NEXT:           Initial:         0x1
+# CHECK-NEXT:           Minimum:         0x1
 # CHECK-NEXT:           Maximum:         0x1
 # CHECK-NEXT:   - Type:
 # CHECK:        - Type:            EXPORT
diff --git a/lld/test/wasm/growable-table.test b/lld/test/wasm/growable-table.test
index c5df1c3bc7e0988b45b6c0a55217f74dd2a536ee..675974ea45aaa0cdffe84c1430574d1199a7d849 100644
--- a/lld/test/wasm/growable-table.test
+++ b/lld/test/wasm/growable-table.test
@@ -9,7 +9,7 @@
 # CHECK-NEXT:       - Index:           0
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
-# CHECK-NEXT:           Initial:         0x1
+# CHECK-NEXT:           Minimum:       0x1
 # CHECK-NEXT:   - Type:
 # CHECK:        - Type:            EXPORT
 # CHECK-NEXT:     Exports:
diff --git a/lld/test/wasm/import-memory.test b/lld/test/wasm/import-memory.test
index e79775ca7198f1e93f5df068bfa491bc026fe5ec..dd7066dec00597ed0b6573fcf74da8288857ddb2 100644
--- a/lld/test/wasm/import-memory.test
+++ b/lld/test/wasm/import-memory.test
@@ -10,7 +10,7 @@
 # CHECK-NEXT:        Field:           memory
 # CHECK-NEXT:        Kind:            MEMORY
 # CHECK-NEXT:        Memory:
-# CHECK-NEXT:          Initial:         0x2
+# CHECK-NEXT:          Minimum:         0x2
 # CHECK-NEXT:  - Type:
 
 
@@ -28,7 +28,7 @@
 # CHECK-MAX-NEXT:        Kind:            MEMORY
 # CHECK-MAX-NEXT:        Memory:
 # CHECK-MAX-NEXT:          Flags:           [ HAS_MAX ]
-# CHECK-MAX-NEXT:          Initial:         0x4
+# CHECK-MAX-NEXT:          Minimum:         0x4
 # CHECK-MAX-NEXT:          Maximum:         0x5
 # CHECK-MAX-NEXT:  - Type:
 
@@ -45,6 +45,6 @@
 # CHECK-SHARED-NEXT:        Kind:            MEMORY
 # CHECK-SHARED-NEXT:        Memory:
 # CHECK-SHARED-NEXT:          Flags:           [ HAS_MAX, IS_SHARED ]
-# CHECK-SHARED-NEXT:          Initial:         0x4
+# CHECK-SHARED-NEXT:          Minimum:         0x4
 # CHECK-SHARED-NEXT:          Maximum:         0x5
 # CHECK-SHARED-NEXT:  - Type:
diff --git a/lld/test/wasm/import-table.test b/lld/test/wasm/import-table.test
index 088ff124e8ff2033d07ab2cfe4ae435da779e78a..73dc7189bbf28bed0532a72f21f87377df1e696f 100644
--- a/lld/test/wasm/import-table.test
+++ b/lld/test/wasm/import-table.test
@@ -21,5 +21,5 @@ require_function_table:
 # CHECK-NEXT:          Index:           0
 # CHECK-NEXT:          ElemType:        FUNCREF
 # CHECK-NEXT:          Limits:
-# CHECK-NEXT:            Initial:         0x1
+# CHECK-NEXT:            Minimum:         0x1
 
diff --git a/lld/test/wasm/large-memory.test b/lld/test/wasm/large-memory.test
index b82c4afdf1c01c96b6e8717a9c219279b369ae0c..5b737e4154963075052370a797c447d0db0ad62f 100644
--- a/lld/test/wasm/large-memory.test
+++ b/lld/test/wasm/large-memory.test
@@ -12,7 +12,7 @@ RUN: obj2yaml %t2.wasm | FileCheck %s --check-prefixes=CHECK,CHECK-4G
 CHECK:      - Type:            MEMORY
 CHECK-NEXT:   Memories:
 CHECK-NEXT:     - Flags:           [ HAS_MAX ]
-CHECK-NEXT:       Initial:         0x2
+CHECK-NEXT:       Minimum:         0x2
 CHECK-2G-NEXT:    Maximum:         0x8000
 CHECK-4G-NEXT:    Maximum:         0x10000
 
diff --git a/lld/test/wasm/local-symbols.ll b/lld/test/wasm/local-symbols.ll
index 216aced9cf0777b04f49e14e1814d47a079b6a56..f5046030458584a173fd7e35a5c10ba2d26e1d52 100644
--- a/lld/test/wasm/local-symbols.ll
+++ b/lld/test/wasm/local-symbols.ll
@@ -37,7 +37,7 @@ entry:
 ; CHECK-NEXT:     FunctionTypes:   [ 0, 1 ]
 ; CHECK-NEXT:   - Type:            MEMORY
 ; CHECK-NEXT:     Memories:
-; CHECK-NEXT:       - Initial:         0x2
+; CHECK-NEXT:       - Minimum:         0x2
 ; CHECK-NEXT:   - Type:            GLOBAL
 ; CHECK-NEXT:     Globals:
 ; CHECK-NEXT:       - Index:           0
diff --git a/lld/test/wasm/locals-duplicate.test b/lld/test/wasm/locals-duplicate.test
index df76d2f69126cffc8d0f645f1685d95ad807f3ab..7de8ef15b18408c56fe38d552de47c16da426dca 100644
--- a/lld/test/wasm/locals-duplicate.test
+++ b/lld/test/wasm/locals-duplicate.test
@@ -22,11 +22,11 @@
 ; CHECK-NEXT:         ElemType:        FUNCREF
 ; CHECK-NEXT:         Limits:
 ; CHECK-NEXT:           Flags:           [ HAS_MAX ]
-; CHECK-NEXT:           Initial:         0x7
+; CHECK-NEXT:           Minimum:         0x7
 ; CHECK-NEXT:           Maximum:         0x7
 ; CHECK-NEXT:   - Type:            MEMORY
 ; CHECK-NEXT:     Memories:
-; CHECK-NEXT:       - Initial:         0x2
+; CHECK-NEXT:       - Minimum:         0x2
 ; CHECK-NEXT:   - Type:            GLOBAL
 ; CHECK-NEXT:     Globals:
 ; CHECK-NEXT:       - Index:           0
@@ -240,13 +240,13 @@
 ; RELOC-NEXT:          Index:           0
 ; RELOC-NEXT:          ElemType:        FUNCREF
 ; RELOC-NEXT:          Limits:
-; RELOC-NEXT:            Initial:         0x7
+; RELOC-NEXT:            Minimum:         0x7
 ; RELOC-NEXT:   - Type:            FUNCTION
 ; RELOC-NEXT:     FunctionTypes:   [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 ; RELOC-NEXT:                        0, 0 ]
 ; RELOC-NEXT:   - Type:            MEMORY
 ; RELOC-NEXT:     Memories:
-; RELOC-NEXT:       - Initial:         0x1
+; RELOC-NEXT:       - Minimum:         0x1
 ; RELOC-NEXT:   - Type:            ELEM
 ; RELOC-NEXT:     Segments:
 ; RELOC-NEXT:       - Offset:
diff --git a/lld/test/wasm/multi-table.s b/lld/test/wasm/multi-table.s
index 2fdbf2547da50b83db397b6f68bce0a91c2e217d..bf905ac748f9fb7f3501b336d07bdc16903103ac 100644
--- a/lld/test/wasm/multi-table.s
+++ b/lld/test/wasm/multi-table.s
@@ -54,7 +54,7 @@ call_indirect_explicit_tables:
 # CHECK-NEXT:           Index:           0
 # CHECK-NEXT:           ElemType:        FUNCREF
 # CHECK-NEXT:           Limits:
-# CHECK-NEXT:             Initial:         0x0
+# CHECK-NEXT:             Minimum:         0x0
 # CHECK-NEXT:       - Module:          env
 # CHECK-NEXT:         Field:           foo
 # CHECK-NEXT:         Kind:            FUNCTION
@@ -66,20 +66,20 @@ call_indirect_explicit_tables:
 # CHECK-NEXT:       - Index:           1
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
-# CHECK-NEXT:           Initial:         0x0
+# CHECK-NEXT:           Minimum:         0x0
 # CHECK-NEXT:       - Index:           2
 # CHECK-NEXT:         ElemType:        EXTERNREF
 # CHECK-NEXT:         Limits:
-# CHECK-NEXT:           Initial:         0x0
+# CHECK-NEXT:           Minimum:         0x0
 # CHECK-NEXT:       - Index:           3
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
 # CHECK-NEXT:           Flags:           [ HAS_MAX ]
-# CHECK-NEXT:           Initial:         0x3
+# CHECK-NEXT:           Minimum:         0x3
 # CHECK-NEXT:           Maximum:         0x3
 # CHECK-NEXT:   - Type:            MEMORY
 # CHECK-NEXT:     Memories:
-# CHECK-NEXT:       - Initial:         0x2
+# CHECK-NEXT:       - Minimum:         0x2
 # CHECK-NEXT:   - Type:            GLOBAL
 # CHECK-NEXT:     Globals:
 # CHECK-NEXT:       - Index:           0
diff --git a/lld/test/wasm/pie.ll b/lld/test/wasm/pie.ll
index 1aca4df3a594dfc799c206c1ae0053aaf7c4796f..03e2c6cee1ce8afc95fb0959a8d650cd8168271f 100644
--- a/lld/test/wasm/pie.ll
+++ b/lld/test/wasm/pie.ll
@@ -62,7 +62,7 @@ define void @_start() {
 ; CHECK-NEXT:           Index:           0
 ; CHECK-NEXT:           ElemType:        FUNCREF
 ; CHECK-NEXT:           Limits:
-; CHECK-NEXT:             Initial:         0x1
+; CHECK-NEXT:             Minimum:         0x1
 
 ; CHECK:        - Type:            START
 ; CHECK-NEXT:     StartFunction:   2
diff --git a/lld/test/wasm/relocatable.ll b/lld/test/wasm/relocatable.ll
index b011813984dc402a6048785efb73fff11b773081..c71a9c2f9c612a22a15b803fce411d1202fd0465 100644
--- a/lld/test/wasm/relocatable.ll
+++ b/lld/test/wasm/relocatable.ll
@@ -66,7 +66,7 @@ define void @_start() {
 ; CHECK-NEXT:           Index:           0
 ; CHECK-NEXT:           ElemType:        FUNCREF
 ; CHECK-NEXT:           Limits:
-; CHECK-NEXT:             Initial:         0x4
+; CHECK-NEXT:             Minimum:         0x4
 ; CHECK-NEXT:       - Module:          env
 ; CHECK-NEXT:         Field:           puts
 ; CHECK-NEXT:         Kind:            FUNCTION
@@ -83,7 +83,7 @@ define void @_start() {
 ; CHECK-NEXT:     FunctionTypes:   [ 2, 1, 1, 2 ]
 ; CHECK-NEXT:   - Type:            MEMORY
 ; CHECK-NEXT:     Memories:
-; CHECK-NEXT:      - Initial:         0x1
+; CHECK-NEXT:      - Minimum:         0x1
 ; CHECK-NEXT:   - Type:            ELEM
 ; CHECK-NEXT:     Segments:
 ; CHECK-NEXT:       - Offset:
diff --git a/lld/test/wasm/responsefile.test b/lld/test/wasm/responsefile.test
index 36209d48edda55882859e6557cbd6d44de055070..47345175dc235fc2415cd0d504654b94b9f9e273 100644
--- a/lld/test/wasm/responsefile.test
+++ b/lld/test/wasm/responsefile.test
@@ -3,7 +3,7 @@ RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown %p/Inputs/ret32.s -o %
 RUN: echo "%/t.o -o %/t.wasm -e ret32" > %t.rsp
 RUN: wasm-ld @%t.rsp --initial-memory=655360
 RUN: llvm-readobj --sections %t.wasm | FileCheck %s
-CHECK: InitialPages: 10
+CHECK: MinPages: 10
 
 RUN: not wasm-ld --rsp-quoting=foobar @%t.rsp 2>&1 | \
 RUN:     FileCheck --check-prefix=INVRSP %s
diff --git a/lld/test/wasm/shared-memory-no-atomics.yaml b/lld/test/wasm/shared-memory-no-atomics.yaml
index e303e39625f2755ca4747075aca2e1e432f156aa..942c69053a4b2fb0ef2e6bceb33ff33d9a399804 100644
--- a/lld/test/wasm/shared-memory-no-atomics.yaml
+++ b/lld/test/wasm/shared-memory-no-atomics.yaml
@@ -14,7 +14,7 @@ Sections:
         Field:           __linear_memory
         Kind:            MEMORY
         Memory:
-          Initial:         0x00000001
+          Minimum:         0x00000001
       - Module:          env
         Field:           __indirect_function_table
         Kind:            TABLE
@@ -22,7 +22,7 @@ Sections:
           Index:           0
           ElemType:        FUNCREF
           Limits:
-            Initial:         0x00000000
+            Minimum:         0x00000000
   - Type:            DATA
     Segments:
       - SectionOffset:   6
@@ -55,7 +55,7 @@ Sections:
 
 # NO-SHARED:      - Type:            MEMORY
 # NO-SHARED-NEXT:   Memories:
-# NO-SHARED-NEXT:     - Initial:         0x2
+# NO-SHARED-NEXT:     - Minimum:         0x2
 # NO-SHARED-NOT:        Maximum:
 
 # SHARED: --shared-memory is disallowed by {{.*}}shared-memory-no-atomics.yaml.tmp1.o because it was not compiled with 'atomics' or 'bulk-memory' features.
diff --git a/lld/test/wasm/shared-memory.yaml b/lld/test/wasm/shared-memory.yaml
index 8710a06db3f7d352d58a91028e66a92eb75f2de3..f10ac6ad53194d817755d558aef44bd35d20caad 100644
--- a/lld/test/wasm/shared-memory.yaml
+++ b/lld/test/wasm/shared-memory.yaml
@@ -22,7 +22,7 @@ Sections:
         Field:           __linear_memory
         Kind:            MEMORY
         Memory:
-          Initial:         0x00000001
+          Minimum:         0x00000001
       - Module:          env
         Field:           __indirect_function_table
         Kind:            TABLE
@@ -30,7 +30,7 @@ Sections:
           Index:           0
           ElemType:        FUNCREF
           Limits:
-            Initial:         0x00000000
+            Minimum:         0x00000000
   - Type:            DATA
     Segments:
       - SectionOffset:   6
@@ -64,11 +64,11 @@ Sections:
 
 # ATOMICS-RELOCATABLE:        - Type:            MEMORY
 # ATOMICS-RELOCATABLE-NEXT:     Memories:
-# ATOMICS-RELOCATABLE-NEXT:         Initial:         0x1
+# ATOMICS-RELOCATABLE-NEXT:         Minimum:         0x1
 # ATOMICS-RELOCATABLE-NEXT:   - Type:
 
 # SHARED:        - Type:            MEMORY
 # SHARED-NEXT:     Memories:
 # SHARED-NEXT:       - Flags:           [ HAS_MAX, IS_SHARED ]
-# SHARED-NEXT:         Initial:         0x2
+# SHARED-NEXT:         Minimum:         0x2
 # SHARED-NEXT:         Maximum:         0x2
diff --git a/lld/test/wasm/shared.ll b/lld/test/wasm/shared.ll
index ecfc0e3062afe54e6453adf45fc03a9f79a45a78..5cf49fb50bf40fffc66e8810e892e6274d7532fa 100644
--- a/lld/test/wasm/shared.ll
+++ b/lld/test/wasm/shared.ll
@@ -67,7 +67,7 @@ declare void @func_external()
 ; CHECK-NEXT:         Field:           memory
 ; CHECK-NEXT:         Kind:            MEMORY
 ; CHECK-NEXT:         Memory:
-; CHECK-NEXT:           Initial:       0x1
+; CHECK-NEXT:           Minimum:       0x1
 ; CHECK-NEXT:       - Module:          env
 ; CHECK-NEXT:         Field:           __indirect_function_table
 ; CHECK-NEXT:         Kind:            TABLE
@@ -75,7 +75,7 @@ declare void @func_external()
 ; CHECK-NEXT:           Index:           0
 ; CHECK-NEXT:           ElemType:        FUNCREF
 ; CHECK-NEXT:           Limits:
-; CHECK-NEXT:             Initial:         0x2
+; CHECK-NEXT:             Minimum:         0x2
 ; CHECK-NEXT:       - Module:          env
 ; CHECK-NEXT:         Field:           __stack_pointer
 ; CHECK-NEXT:         Kind:            GLOBAL
diff --git a/lld/test/wasm/stack-pointer.ll b/lld/test/wasm/stack-pointer.ll
index 11ab62fab1ae9add1d391ba193449f18f14d5328..38693d252ea7f19a0187f2c098529fd8e3c08d62 100644
--- a/lld/test/wasm/stack-pointer.ll
+++ b/lld/test/wasm/stack-pointer.ll
@@ -32,7 +32,7 @@ entry:
 ; CHECK-NEXT:     FunctionTypes:   [ 0 ]
 ; CHECK-NEXT:   - Type:            MEMORY
 ; CHECK-NEXT:     Memories:
-; CHECK-NEXT:       - Initial:         0x0
+; CHECK-NEXT:       - Minimum:         0x0
 ; CHECK-NEXT:   - Type:            CODE
 ; CHECK-NEXT:     Relocations:
 ; CHECK-NEXT:       - Type:            R_WASM_GLOBAL_INDEX_LEB
diff --git a/lld/test/wasm/undefined-weak-call.s b/lld/test/wasm/undefined-weak-call.s
index 0c8abcd387b18ecca53ce02fcf17542a293e9134..8f50173e9c3ec5cef550fbf9e0933eb42fa8dcbe 100644
--- a/lld/test/wasm/undefined-weak-call.s
+++ b/lld/test/wasm/undefined-weak-call.s
@@ -56,11 +56,11 @@ callWeakFuncs:
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
 # CHECK-NEXT:           Flags:           [ HAS_MAX ]
-# CHECK-NEXT:           Initial:         0x1
+# CHECK-NEXT:           Minimum:         0x1
 # CHECK-NEXT:           Maximum:         0x1
 # CHECK-NEXT:   - Type:            MEMORY
 # CHECK-NEXT:     Memories:
-# CHECK-NEXT:       - Initial:         0x2
+# CHECK-NEXT:       - Minimum:         0x2
 # CHECK-NEXT:   - Type:            GLOBAL
 # CHECK-NEXT:     Globals:
 # CHECK-NEXT:       - Index:           0
diff --git a/lld/test/wasm/weak-alias-overide.ll b/lld/test/wasm/weak-alias-overide.ll
index 4d47341e226882373546e53c095097631fdc8c3b..fcf2293892910edc8676a12671d5d0f2eb66237f 100644
--- a/lld/test/wasm/weak-alias-overide.ll
+++ b/lld/test/wasm/weak-alias-overide.ll
@@ -40,11 +40,11 @@ entry:
 ; CHECK-NEXT:         ElemType:        FUNCREF
 ; CHECK-NEXT:         Limits:
 ; CHECK-NEXT:           Flags:           [ HAS_MAX ]
-; CHECK-NEXT:           Initial:         0x3
+; CHECK-NEXT:           Minimum:         0x3
 ; CHECK-NEXT:           Maximum:         0x3
 ; CHECK-NEXT:   - Type:            MEMORY
 ; CHECK-NEXT:     Memories:
-; CHECK-NEXT:       - Initial:         0x2
+; CHECK-NEXT:       - Minimum:         0x2
 ; CHECK-NEXT:   - Type:            GLOBAL
 ; CHECK-NEXT:     Globals:
 ; CHECK-NEXT:       - Index:           0
diff --git a/lld/test/wasm/weak-alias.ll b/lld/test/wasm/weak-alias.ll
index 8bdeda05dd3e8f7efca5941a5c61011f18d0aad5..aa0a271396d1a298c9d3243356e4ed8a9662e7dc 100644
--- a/lld/test/wasm/weak-alias.ll
+++ b/lld/test/wasm/weak-alias.ll
@@ -37,11 +37,11 @@ entry:
 ; CHECK-NEXT:         ElemType:        FUNCREF
 ; CHECK-NEXT:         Limits:
 ; CHECK-NEXT:           Flags:           [ HAS_MAX ]
-; CHECK-NEXT:           Initial:         0x2
+; CHECK-NEXT:           Minimum:         0x2
 ; CHECK-NEXT:           Maximum:         0x2
 ; CHECK-NEXT:   - Type:            MEMORY
 ; CHECK-NEXT:     Memories:
-; CHECK-NEXT:       - Initial:         0x2
+; CHECK-NEXT:       - Minimum:         0x2
 ; CHECK-NEXT:   - Type:            GLOBAL
 ; CHECK-NEXT:     Globals:
 ; CHECK-NEXT:       - Index:           0
@@ -151,7 +151,7 @@ entry:
 ; RELOC-NEXT:           Index:           0
 ; RELOC-NEXT:           ElemType:        FUNCREF
 ; RELOC-NEXT:           Limits:
-; RELOC-NEXT:             Initial:         0x2
+; RELOC-NEXT:             Minimum:         0x2
 ; RELOC-NEXT:       - Module:          env
 ; RELOC-NEXT:         Field:           __stack_pointer
 ; RELOC-NEXT:         Kind:            GLOBAL
@@ -161,7 +161,7 @@ entry:
 ; RELOC-NEXT:     FunctionTypes:   [ 0, 1, 1, 1, 1, 1 ]
 ; RELOC-NEXT:   - Type:            MEMORY
 ; RELOC-NEXT:     Memories:
-; RELOC-NEXT:       - Initial:         0x0
+; RELOC-NEXT:       - Minimum:         0x0
 ; RELOC-NEXT:   - Type:            ELEM
 ; RELOC-NEXT:     Segments:
 ; RELOC-NEXT:       - Offset:
diff --git a/lld/test/wasm/weak-symbols.s b/lld/test/wasm/weak-symbols.s
index 7557dfb5535b5f5e48348d8d56b7c9c98846990d..705b0b920d42fa68ff338d73c007246cc734e426 100644
--- a/lld/test/wasm/weak-symbols.s
+++ b/lld/test/wasm/weak-symbols.s
@@ -35,11 +35,11 @@ _start:
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
 # CHECK-NEXT:           Flags:           [ HAS_MAX ]
-# CHECK-NEXT:           Initial:         0x2
+# CHECK-NEXT:           Minimum:         0x2
 # CHECK-NEXT:           Maximum:         0x2
 # CHECK-NEXT:   - Type:            MEMORY
 # CHECK-NEXT:     Memories:
-# CHECK-NEXT:       - Initial:         0x2
+# CHECK-NEXT:       - Minimum:         0x2
 # CHECK-NEXT:   - Type:            GLOBAL
 # CHECK-NEXT:     Globals:
 # CHECK-NEXT:       - Index:           0
diff --git a/lld/test/wasm/weak-undefined.s b/lld/test/wasm/weak-undefined.s
index e7abebb2f8ce142657b5d1c38a2c246a214e7f25..44b0bc11eb65c7f019a8fe449c9dd4637df2f0c4 100644
--- a/lld/test/wasm/weak-undefined.s
+++ b/lld/test/wasm/weak-undefined.s
@@ -54,11 +54,11 @@ _start:
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
 # CHECK-NEXT:           Flags:           [ HAS_MAX ]
-# CHECK-NEXT:           Initial:         0x1
+# CHECK-NEXT:           Minimum:         0x1
 # CHECK-NEXT:           Maximum:         0x1
 # CHECK-NEXT:   - Type:            MEMORY
 # CHECK-NEXT:     Memories:
-# CHECK-NEXT:       - Initial:         0x2
+# CHECK-NEXT:       - Minimum:         0x2
 # CHECK-NEXT:   - Type:            GLOBAL
 # CHECK-NEXT:     Globals:
 # CHECK-NEXT:       - Index:           0
diff --git a/lld/wasm/SyntheticSections.cpp b/lld/wasm/SyntheticSections.cpp
index 6c834771b6cb5d3c2d4b3d0188fb83e3631ab369..e4da8dc5f2088788047cddebec97e2cf6b33b1d0 100644
--- a/lld/wasm/SyntheticSections.cpp
+++ b/lld/wasm/SyntheticSections.cpp
@@ -131,7 +131,7 @@ void ImportSection::writeBody() {
     import.Field = "memory";
     import.Kind = WASM_EXTERNAL_MEMORY;
     import.Memory.Flags = 0;
-    import.Memory.Initial = out.memorySec->numMemoryPages;
+    import.Memory.Minimum = out.memorySec->numMemoryPages;
     if (out.memorySec->maxMemoryPages != 0 || config->sharedMemory) {
       import.Memory.Flags |= WASM_LIMITS_FLAG_HAS_MAX;
       import.Memory.Maximum = out.memorySec->maxMemoryPages;
diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp
index 63e3dac5d298523fb53448758ce8458bf8f32fce..3a7104ed920bcdc9a5e0c4121e1cca9623fb204f 100644
--- a/lld/wasm/Writer.cpp
+++ b/lld/wasm/Writer.cpp
@@ -752,7 +752,7 @@ static void finalizeIndirectFunctionTable() {
   WasmLimits limits = {0, tableSize, 0};
   if (WasmSym::indirectFunctionTable->isDefined() && !config->growableTable) {
     limits.Flags |= WASM_LIMITS_FLAG_HAS_MAX;
-    limits.Maximum = limits.Initial;
+    limits.Maximum = limits.Minimum;
   }
   WasmSym::indirectFunctionTable->setLimits(limits);
 }
diff --git a/lld/wasm/WriterUtils.cpp b/lld/wasm/WriterUtils.cpp
index 88e89dff069ca4593eebced1d63567db01da2bcf..1870b3686341842d2cbb535d3f99acafac920fb2 100644
--- a/lld/wasm/WriterUtils.cpp
+++ b/lld/wasm/WriterUtils.cpp
@@ -67,7 +67,7 @@ std::string toString(const WasmEventType &type) {
 static std::string toString(const llvm::wasm::WasmLimits &limits) {
   std::string ret;
   ret += "flags=0x" + std::to_string(limits.Flags);
-  ret += "; initial=" + std::to_string(limits.Initial);
+  ret += "; min=" + std::to_string(limits.Minimum);
   if (limits.Flags & WASM_LIMITS_FLAG_HAS_MAX)
     ret += "; max=" + std::to_string(limits.Maximum);
   return ret;
@@ -191,7 +191,7 @@ void writeInitExpr(raw_ostream &os, const WasmInitExpr &initExpr) {
 
 void writeLimits(raw_ostream &os, const WasmLimits &limits) {
   writeU8(os, limits.Flags, "limits flags");
-  writeUleb128(os, limits.Initial, "limits initial");
+  writeUleb128(os, limits.Minimum, "limits min");
   if (limits.Flags & WASM_LIMITS_FLAG_HAS_MAX)
     writeUleb128(os, limits.Maximum, "limits max");
 }
diff --git a/lldb/bindings/interface/SBLaunchInfo.i b/lldb/bindings/interface/SBLaunchInfo.i
index d76656ddd4932172516baa9b1ab0f9b3476ecfe3..535ed3b7906027291ea4b78e672bed30c9aa1309 100644
--- a/lldb/bindings/interface/SBLaunchInfo.i
+++ b/lldb/bindings/interface/SBLaunchInfo.i
@@ -135,6 +135,16 @@ public:
 
     void
     SetDetachOnError(bool enable);
+
+    const char *
+    GetScriptedProcessClassName() const;
+
+    void SetScriptedProcessClassName(const char *class_name);
+
+    lldb::SBStructuredData
+    GetScriptedProcessDictionary() const;
+
+    void SetScriptedProcessDictionary(lldb::SBStructuredData dict);
 };
 
 } // namespace lldb
diff --git a/lldb/bindings/python/CMakeLists.txt b/lldb/bindings/python/CMakeLists.txt
index 2b14ee339d0deb634c72ad9797e8c92046cc588a..9422ee00cb5fc1d874251a7e7d4dade04689335e 100644
--- a/lldb/bindings/python/CMakeLists.txt
+++ b/lldb/bindings/python/CMakeLists.txt
@@ -104,6 +104,13 @@ function(finish_swig_python swig_target lldb_python_bindings_dir lldb_python_tar
     FILES "${LLDB_SOURCE_DIR}/examples/python/in_call_stack.py"
           "${LLDB_SOURCE_DIR}/examples/python/symbolication.py")
 
+  create_python_package(
+    ${swig_target}
+    ${lldb_python_target_dir}
+    "plugins"
+    FILES
+    "${LLDB_SOURCE_DIR}/examples/python/scripted_process/scripted_process.py")
+
   if(APPLE)
     create_python_package(
       ${swig_target}
diff --git a/lldb/bindings/python/python-wrapper.swig b/lldb/bindings/python/python-wrapper.swig
index b189bfd7b9ebfa6dfdfc4b077a2e52514fa0ba0e..4c39e9c2c776a63102478644c3843225b5e780db 100644
--- a/lldb/bindings/python/python-wrapper.swig
+++ b/lldb/bindings/python/python-wrapper.swig
@@ -275,6 +275,72 @@ LLDBSwigPythonCreateCommandObject
     Py_RETURN_NONE;
 }
 
+SWIGEXPORT void*
+LLDBSwigPythonCreateScriptedProcess
+(
+    const char *python_class_name,
+    const char *session_dictionary_name,
+    const lldb::TargetSP& target_sp,
+    lldb_private::StructuredDataImpl *args_impl,
+    std::string &error_string
+)
+{
+    if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
+        Py_RETURN_NONE;
+
+
+    PyErr_Cleaner py_err_cleaner(true);
+
+    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
+    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name, dict);
+
+    if (!pfunc.IsAllocated()) {
+        error_string.append("could not find script class: ");
+        error_string.append(python_class_name);
+        return nullptr;
+    }
+
+    // I do not want the SBTarget to be deallocated when going out of scope
+    // because python has ownership of it and will manage memory for this
+    // object by itself
+    PythonObject target_arg(PyRefType::Owned, SBTypeToSWIGWrapper(new lldb::SBTarget(target_sp)));
+
+    if (!target_arg.IsAllocated())
+        Py_RETURN_NONE;
+
+    llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
+    if (!arg_info) {
+        llvm::handleAllErrors(
+            arg_info.takeError(),
+            [&](PythonException &E) {
+                error_string.append(E.ReadBacktrace());
+            },
+            [&](const llvm::ErrorInfoBase &E) {
+                error_string.append(E.message());
+            });
+        Py_RETURN_NONE;
+    }
+
+    PythonObject result = {};
+    if (arg_info.get().max_positional_args == 2) {
+        if (args_impl != nullptr) {
+           error_string.assign("args passed, but __init__ does not take an args dictionary");
+           Py_RETURN_NONE;
+        }
+        result = pfunc(target_arg, dict);
+    } else if (arg_info.get().max_positional_args >= 3) {
+        PythonObject args_arg(PyRefType::Owned, SBTypeToSWIGWrapper(new lldb::SBStructuredData(args_impl)));
+        result = pfunc(target_arg, args_arg, dict);
+    } else {
+        error_string.assign("wrong number of arguments in __init__, should be 2 or 3 (not including self)");
+        Py_RETURN_NONE;
+    }
+
+    if (result.IsAllocated())
+        return result.release();
+    Py_RETURN_NONE;
+}
+
 SWIGEXPORT void*
 LLDBSwigPythonCreateScriptedThreadPlan
 (
@@ -802,6 +868,40 @@ LLDBSwigPython_GetValueSynthProviderInstance
     return ret_val;
 }
 
+SWIGEXPORT void*
+LLDBSWIGPython_CastPyObjectToSBData
+(
+    PyObject* data
+)
+{
+    lldb::SBData* sb_ptr = nullptr;
+
+    int valid_cast = SWIG_ConvertPtr(data, (void**)&sb_ptr, SWIGTYPE_p_lldb__SBData, 0);
+
+    if (valid_cast == -1)
+        return NULL;
+
+    return sb_ptr;
+}
+
+
+SWIGEXPORT void*
+LLDBSWIGPython_CastPyObjectToSBError
+(
+    PyObject* data
+)
+{
+    lldb::SBError* sb_ptr = nullptr;
+
+    int valid_cast = SWIG_ConvertPtr(data, (void**)&sb_ptr, SWIGTYPE_p_lldb__SBError, 0);
+
+    if (valid_cast == -1)
+        return NULL;
+
+    return sb_ptr;
+}
+
+
 SWIGEXPORT void*
 LLDBSWIGPython_CastPyObjectToSBValue
 (
diff --git a/lldb/docs/resources/bots.rst b/lldb/docs/resources/bots.rst
index f80a2333992bcd8eef67717778c0ef579aa3d12b..8a43f53bd2502748f9cd6e021473911bc371af7b 100644
--- a/lldb/docs/resources/bots.rst
+++ b/lldb/docs/resources/bots.rst
@@ -7,15 +7,15 @@ Buildbot
 LLVM Buildbot is the place where volunteers provide build machines. Everyone can
 `add a buildbot for LLDB <https://llvm.org/docs/HowToAddABuilder.html>`_.
 
-* `lldb-x64-windows-ninja <http://lab.llvm.org:8011/#/builders/83>`_
-* `lldb-x86_64-debian <http://lab.llvm.org:8011/#/builders/68>`_
-* `lldb-aarch64-ubuntu <http://lab.llvm.org:8011/#/builders/96>`_
-* `lldb-arm-ubuntu <http://lab.llvm.org:8011/#/builders/17>`_
-* `lldb-x86_64-fedora <http://lab.llvm.org:8014/#/builders/14>`_
+* `lldb-x64-windows-ninja <https://lab.llvm.org/buildbot/#/builders/83>`_
+* `lldb-x86_64-debian <https://lab.llvm.org/buildbot/#/builders/68>`_
+* `lldb-aarch64-ubuntu <https://lab.llvm.org/buildbot/#/builders/96>`_
+* `lldb-arm-ubuntu <https://lab.llvm.org/buildbot/#/builders/17>`_
+* `lldb-x86_64-fedora <https://lab.llvm.org/staging/#/builders/16>`_
 
-An overview of all LLDB builders can be found here:
+An overview of all LLDB builders (except Fedora) can be found here:
 
-`http://lab.llvm.org:8011/#/builders?tags=lldb <http://lab.llvm.org:8011/#/builders?tags=lldb>`_
+`https://lab.llvm.org/buildbot/#/builders?tags=lldb <https://lab.llvm.org/buildbot/#/builders?tags=lldb>`_
 
 GreenDragon
 -----------
diff --git a/lldb/examples/python/scripted_process/my_scripted_process.py b/lldb/examples/python/scripted_process/my_scripted_process.py
new file mode 100644
index 0000000000000000000000000000000000000000..d769e137d3f34bedf7b99e821f0481a7f6382ae3
--- /dev/null
+++ b/lldb/examples/python/scripted_process/my_scripted_process.py
@@ -0,0 +1,42 @@
+import os
+
+import lldb
+from lldb.plugins.scripted_process import ScriptedProcess
+
+class MyScriptedProcess(ScriptedProcess):
+    def __init__(self, target: lldb.SBTarget, args : lldb.SBStructuredData):
+        super().__init__(target, args)
+
+    def get_memory_region_containing_address(self, addr: int) -> lldb.SBMemoryRegionInfo:
+        return self.memory_regions[0]
+
+    def get_thread_with_id(self, tid: int):
+        return {}
+
+    def get_registers_for_thread(self, tid: int):
+        return {}
+
+    def read_memory_at_address(self, addr: int, size: int) -> lldb.SBData:
+        data = lldb.SBData().CreateDataFromCString(
+                                    self.target.GetByteOrder(),
+                                    self.target.GetCodeByteSize(),
+                                    "Hello, world!")
+        return data
+
+    def get_loaded_images(self):
+        return self.loaded_images
+
+    def get_process_id(self) -> int:
+        return 42
+
+    def is_alive(self) -> bool:
+        return True
+
+def __lldb_init_module(debugger, dict):
+    if not 'SKIP_SCRIPTED_PROCESS_LAUNCH' in os.environ:
+        debugger.HandleCommand(
+            "process launch -C %s.%s" % (__name__,
+                                     MyScriptedProcess.__name__))
+    else:
+        print("Name of the class that will manage the scripted process: '%s.%s'"
+                % (__name__, MyScriptedProcess.__name__))
\ No newline at end of file
diff --git a/lldb/examples/python/scripted_process/scripted_process.py b/lldb/examples/python/scripted_process/scripted_process.py
new file mode 100644
index 0000000000000000000000000000000000000000..354b20cfa7f90a2e50c35691ba9bc326fc75217b
--- /dev/null
+++ b/lldb/examples/python/scripted_process/scripted_process.py
@@ -0,0 +1,147 @@
+from abc import ABCMeta, abstractmethod
+import six
+
+import lldb
+
+@six.add_metaclass(ABCMeta)
+class ScriptedProcess:
+
+    """
+    The base class for a scripted process.
+
+    Most of the base class methods are `@abstractmethod` that need to be
+    overwritten by the inheriting class.
+
+    DISCLAIMER: THIS INTERFACE IS STILL UNDER DEVELOPMENT AND NOT STABLE.
+                THE METHODS EXPOSED MIGHT CHANGE IN THE FUTURE.
+    """
+
+    @abstractmethod
+    def __init__(self, target, args):
+        """ Construct a scripted process.
+
+        Args:
+            target (lldb.SBTarget): The target launching the scripted process.
+            args (lldb.SBStructuredData): A Dictionary holding arbitrary
+                key/value pairs used by the scripted process.
+        """
+        self.target = None
+        self.args = None
+        if isinstance(target, lldb.SBTarget) and target.IsValid():
+            self.target = target
+        if isinstance(args, lldb.SBStructuredData) and args.IsValid():
+            self.args = args
+
+    @abstractmethod
+    def get_memory_region_containing_address(addr):
+        """ Get the memory region for the scripted process, containing a
+            specific address.
+
+        Args:
+            addr (int): Address to look for in the scripted process memory
+                regions.
+
+        Returns:
+            lldb.SBMemoryRegionInfo: The memory region containing the address.
+                None if out of bounds.
+        """
+        pass
+
+    @abstractmethod
+    def get_thread_with_id(tid):
+        """ Get the scripted process thread with a specific ID.
+
+        Args:
+            tid (int): Thread ID to look for in the scripted process.
+
+        Returns:
+            Dict: The thread represented as a dictionary, withr the
+                tid thread ID. None if tid doesn't match any of the scripted
+                process threads.
+        """
+        pass
+
+    @abstractmethod
+    def get_registers_for_thread(tid):
+        """ Get the register context dictionary for a certain thread of
+            the scripted process.
+
+        Args:
+            tid (int): Thread ID for the thread's register context.
+
+        Returns:
+            Dict: The register context represented as a dictionary, for the
+                tid thread. None if tid doesn't match any of the scripted
+                process threads.
+        """
+        pass
+
+    @abstractmethod
+    def read_memory_at_address(addr, size):
+        """ Get a memory buffer from the scripted process at a certain address,
+            of a certain size.
+
+        Args:
+            addr (int): Address from which we should start reading.
+            size (int): Size of the memory to read.
+
+        Returns:
+            lldb.SBData: An `lldb.SBData` buffer with the target byte size and
+                byte order storing the memory read.
+        """
+        pass
+
+    @abstractmethod
+    def get_loaded_images(self):
+        """ Get the list of loaded images for the scripted process.
+
+        ```
+        class ScriptedProcessImage:
+            def __init__(name, file_spec, uuid, load_address):
+              self.name = name
+              self.file_spec = file_spec
+              self.uuid = uuid
+              self.load_address = load_address
+        ```
+
+        Returns:
+            List[ScriptedProcessImage]: A list of `ScriptedProcessImage`
+                containing for each entry, the name of the library, a UUID,
+                an `lldb.SBFileSpec` and a load address.
+                None if the list is empty.
+        """
+        pass
+
+    def get_process_id(self):
+        """ Get the scripted process identifier.
+
+        Returns:
+            int: The scripted process identifier.
+        """
+        return 0
+
+
+    def launch(self):
+        """ Simulate the scripted process launch.
+
+        Returns:
+            lldb.SBError: An `lldb.SBError` with error code 0.
+        """
+        return lldb.SBError()
+
+    def resume(self):
+        """ Simulate the scripted process resume.
+
+        Returns:
+            lldb.SBError: An `lldb.SBError` with error code 0.
+        """
+        return lldb.SBError()
+
+    @abstractmethod
+    def is_alive(self):
+        """ Check if the scripted process is alive.
+
+        Returns:
+            bool: True if scripted process is alive. False otherwise.
+        """
+        pass
diff --git a/lldb/include/lldb/API/SBBroadcaster.h b/lldb/include/lldb/API/SBBroadcaster.h
index 69a516a8bfb1b97b2ca7a6a3640907db7c36e388..20756113793ab5fe75f9180c6724c2c3e19d9aa7 100644
--- a/lldb/include/lldb/API/SBBroadcaster.h
+++ b/lldb/include/lldb/API/SBBroadcaster.h
@@ -63,6 +63,7 @@ public:
 protected:
   friend class SBCommandInterpreter;
   friend class SBCommunication;
+  friend class SBDebugger;
   friend class SBEvent;
   friend class SBListener;
   friend class SBProcess;
diff --git a/lldb/include/lldb/API/SBData.h b/lldb/include/lldb/API/SBData.h
index 95c8086d5d470e1cd3f89c2dbe3a090061d48a96..85c8110e181c21e4d3a823ff65f0ad8f6277eadc 100644
--- a/lldb/include/lldb/API/SBData.h
+++ b/lldb/include/lldb/API/SBData.h
@@ -11,6 +11,10 @@
 
 #include "lldb/API/SBDefines.h"
 
+namespace lldb_private {
+class ScriptInterpreter;
+} // namespace lldb_private
+
 namespace lldb {
 
 class LLDB_API SBData {
@@ -147,6 +151,8 @@ private:
   friend class SBTarget;
   friend class SBValue;
 
+  friend class lldb_private::ScriptInterpreter;
+
   lldb::DataExtractorSP m_opaque_sp;
 };
 
diff --git a/lldb/include/lldb/API/SBDebugger.h b/lldb/include/lldb/API/SBDebugger.h
index b3bfa230139c988cfe769d3636bc5867f7b4e77e..489ed4f2524857b197e7f172de5eea9156b1e2f8 100644
--- a/lldb/include/lldb/API/SBDebugger.h
+++ b/lldb/include/lldb/API/SBDebugger.h
@@ -33,6 +33,8 @@ public:
 
 class LLDB_API SBDebugger {
 public:
+  FLAGS_ANONYMOUS_ENUM(){eBroadcastBitProgress = (1 << 0)};
+
   SBDebugger();
 
   SBDebugger(const lldb::SBDebugger &rhs);
@@ -41,6 +43,42 @@ public:
 
   ~SBDebugger();
 
+  static const char *GetBroadcasterClass();
+
+  lldb::SBBroadcaster GetBroadcaster();
+
+  /// Get progress data from a SBEvent whose type is eBroadcastBitProgress.
+  ///
+  /// \param [in] event
+  ///   The event to extract the progress information from.
+  ///
+  /// \param [out] progress_id
+  ///   The unique integer identifier for the progress to report.
+  ///
+  /// \param [out] completed
+  ///   The amount of work completed. If \a completed is zero, then this event
+  ///   is a progress started event. If \a completed is equal to \a total, then
+  ///   this event is a progress end event. Otherwise completed indicates the
+  ///   current progress update.
+  ///
+  /// \param [out] total
+  ///   The total amount of work units that need to be completed. If this value
+  ///   is UINT64_MAX, then an indeterminate progress indicator should be
+  ///   displayed.
+  ///
+  /// \param [out] is_debugger_specific
+  ///   Set to true if this progress is specific to this debugger only. Many
+  ///   progress events are not specific to a debugger instance, like any
+  ///   progress events for loading information in modules since LLDB has a
+  ///   global module cache that all debuggers use.
+  ///
+  /// \return The message for the progress. If the returned value is NULL, then
+  ///   \a event was not a eBroadcastBitProgress event.
+  static const char *GetProgressFromEvent(const lldb::SBEvent &event,
+                                          uint64_t &progress_id,
+                                          uint64_t &completed, uint64_t &total,
+                                          bool &is_debugger_specific);
+
   lldb::SBDebugger &operator=(const lldb::SBDebugger &rhs);
 
   static void Initialize();
diff --git a/lldb/include/lldb/API/SBError.h b/lldb/include/lldb/API/SBError.h
index e1960ef9a98311020a37c0ecaf3aa4040649442c..f8289e2fcbb3a48916422326c4ccee01b8f99c0b 100644
--- a/lldb/include/lldb/API/SBError.h
+++ b/lldb/include/lldb/API/SBError.h
@@ -11,6 +11,10 @@
 
 #include "lldb/API/SBDefines.h"
 
+namespace lldb_private {
+class ScriptInterpreter;
+} // namespace lldb_private
+
 namespace lldb {
 
 class LLDB_API SBError {
@@ -72,6 +76,8 @@ protected:
   friend class SBWatchpoint;
   friend class SBFile;
 
+  friend class lldb_private::ScriptInterpreter;
+
   lldb_private::Status *get();
 
   lldb_private::Status *operator->();
diff --git a/lldb/include/lldb/API/SBLaunchInfo.h b/lldb/include/lldb/API/SBLaunchInfo.h
index 04ebb5707688df8b7d5a188051e1c7449b69b458..eb4f4a65833c89c15d386405f16330cd4bab0cf9 100644
--- a/lldb/include/lldb/API/SBLaunchInfo.h
+++ b/lldb/include/lldb/API/SBLaunchInfo.h
@@ -171,6 +171,14 @@ public:
 
   void SetDetachOnError(bool enable);
 
+  const char *GetScriptedProcessClassName() const;
+
+  void SetScriptedProcessClassName(const char *class_name);
+
+  lldb::SBStructuredData GetScriptedProcessDictionary() const;
+
+  void SetScriptedProcessDictionary(lldb::SBStructuredData dict);
+
 protected:
   friend class SBPlatform;
   friend class SBTarget;
diff --git a/lldb/include/lldb/API/SBStream.h b/lldb/include/lldb/API/SBStream.h
index f44b87bb4c98e3662f9858e7dd48d9b2f858fe48..f39bf13f2bf7801054d6ae95ba43fbf60fc84302 100644
--- a/lldb/include/lldb/API/SBStream.h
+++ b/lldb/include/lldb/API/SBStream.h
@@ -72,6 +72,7 @@ protected:
   friend class SBFunction;
   friend class SBInstruction;
   friend class SBInstructionList;
+  friend class SBLaunchInfo;
   friend class SBLineEntry;
   friend class SBMemoryRegionInfo;
   friend class SBModule;
diff --git a/lldb/include/lldb/API/SBStructuredData.h b/lldb/include/lldb/API/SBStructuredData.h
index 44a86bdabe259c1076236e009aa4dba7d43e5611..07763f2de52b66571646061407fb2a9473bce2b7 100644
--- a/lldb/include/lldb/API/SBStructuredData.h
+++ b/lldb/include/lldb/API/SBStructuredData.h
@@ -88,6 +88,7 @@ public:
   size_t GetStringValue(char *dst, size_t dst_len) const;
 
 protected:
+  friend class SBLaunchInfo;
   friend class SBTraceOptions;
   friend class SBDebugger;
   friend class SBTarget;
diff --git a/lldb/include/lldb/API/SBThreadPlan.h b/lldb/include/lldb/API/SBThreadPlan.h
index 0dc48437a6681b9c08d2e7da5a17c6b73ec19702..269cbc64a8efd4f0fcf8941f094388ad9a7258bc 100644
--- a/lldb/include/lldb/API/SBThreadPlan.h
+++ b/lldb/include/lldb/API/SBThreadPlan.h
@@ -17,8 +17,6 @@ namespace lldb {
 
 class LLDB_API SBThreadPlan {
 
-  friend class lldb_private::ThreadPlan;
-
 public:
   SBThreadPlan();
 
diff --git a/lldb/include/lldb/Breakpoint/BreakpointLocation.h b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
index 4e1c57a40435282e059a7d8e7c50176b9202879a..a12696be91e3843df2d4984d3ae986c5fed97080 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointLocation.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
@@ -230,6 +230,12 @@ public:
   ///     \b true if the target should stop at this breakpoint and \b
   ///     false not.
   bool InvokeCallback(StoppointCallbackContext *context);
+  
+  /// Report whether the callback for this location is synchronous or not.
+  ///
+  /// \return
+  ///     \b true if the callback is synchronous and \b false if not.
+  bool IsCallbackSynchronous();
 
   /// Returns whether we should resolve Indirect functions in setting the
   /// breakpoint site for this location.
diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h
index 02ff683a3b62638c428d4af95d9aaf47788d7034..7d7db3958944080ff449b58438be521b90035d5d 100644
--- a/lldb/include/lldb/Core/Debugger.h
+++ b/lldb/include/lldb/Core/Debugger.h
@@ -73,6 +73,50 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
   friend class SourceManager; // For GetSourceFileCache.
 
 public:
+  /// Broadcaster event bits definitions.
+  enum {
+    eBroadcastBitProgress = (1 << 0),
+  };
+
+  static ConstString GetStaticBroadcasterClass();
+
+  /// Get the public broadcaster for this debugger.
+  Broadcaster &GetBroadcaster() { return m_broadcaster; }
+  const Broadcaster &GetBroadcaster() const { return m_broadcaster; }
+
+  class ProgressEventData : public EventData {
+
+  public:
+    ProgressEventData(uint64_t progress_id, const std::string &message,
+                      uint64_t completed, uint64_t total,
+                      bool debugger_specific)
+        : m_message(message), m_id(progress_id), m_completed(completed),
+          m_total(total), m_debugger_specific(debugger_specific) {}
+
+    static ConstString GetFlavorString();
+
+    ConstString GetFlavor() const override;
+
+    void Dump(Stream *s) const override;
+
+    static const ProgressEventData *
+    GetEventDataFromEvent(const Event *event_ptr);
+    uint64_t GetID() const { return m_id; }
+    uint64_t GetCompleted() const { return m_completed; }
+    uint64_t GetTotal() const { return m_total; }
+    const std::string &GetMessage() const { return m_message; }
+    bool IsDebuggerSpecific() const { return m_debugger_specific; }
+
+  private:
+    std::string m_message;
+    const uint64_t m_id;
+    uint64_t m_completed;
+    const uint64_t m_total;
+    const bool m_debugger_specific;
+    ProgressEventData(const ProgressEventData &) = delete;
+    const ProgressEventData &operator=(const ProgressEventData &) = delete;
+  };
+
   ~Debugger() override;
 
   static lldb::DebuggerSP
@@ -346,6 +390,40 @@ public:
 protected:
   friend class CommandInterpreter;
   friend class REPL;
+  friend class Progress;
+
+  /// Report progress events.
+  ///
+  /// Progress events will be delivered to any debuggers that have listeners
+  /// for the eBroadcastBitProgress. This function is called by the
+  /// lldb_private::Progress class to deliver the events to any debuggers that
+  /// qualify.
+  ///
+  /// \param [in] progress_id
+  ///   The unique integer identifier for the progress to report.
+  ///
+  /// \param[in] message
+  ///   The title of the progress dialog to display in the UI.
+  ///
+  /// \param [in] completed
+  ///   The amount of work completed. If \a completed is zero, then this event
+  ///   is a progress started event. If \a completed is equal to \a total, then
+  ///   this event is a progress end event. Otherwise completed indicates the
+  ///   current progress compare to the total value.
+  ///
+  /// \param [in] total
+  ///   The total amount of work units that need to be completed. If this value
+  ///   is UINT64_MAX, then an indeterminate progress indicator should be
+  ///   displayed.
+  ///
+  /// \param [in] debugger_id
+  ///   If this optional parameter has a value, it indicates the unique
+  ///   debugger identifier that this progress should be delivered to. If this
+  ///   optional parameter does not have a value, the the progress will be
+  ///   delivered to all debuggers.
+  static void ReportProgress(uint64_t progress_id, const std::string &message,
+                             uint64_t completed, uint64_t total,
+                             llvm::Optional<lldb::user_id_t> debugger_id);
 
   bool StartEventHandlerThread();
 
@@ -432,7 +510,8 @@ protected:
   LoadedPluginsList m_loaded_plugins;
   HostThread m_event_handler_thread;
   HostThread m_io_handler_thread;
-  Broadcaster m_sync_broadcaster;
+  Broadcaster m_sync_broadcaster; ///< Private debugger synchronization.
+  Broadcaster m_broadcaster;      ///< Public Debugger event broadcaster.
   lldb::ListenerSP m_forward_listener_sp;
   llvm::once_flag m_clear_once;
   lldb::TargetSP m_dummy_target_sp;
diff --git a/lldb/include/lldb/Core/Progress.h b/lldb/include/lldb/Core/Progress.h
new file mode 100644
index 0000000000000000000000000000000000000000..f625d014f2684ea7078bad65532c59370a9ec252
--- /dev/null
+++ b/lldb/include/lldb/Core/Progress.h
@@ -0,0 +1,114 @@
+//===-- Progress.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_CORE_PROGRESS_H
+#define LLDB_CORE_PROGRESS_H
+
+#include "lldb/Utility/ConstString.h"
+#include "lldb/lldb-types.h"
+#include <atomic>
+#include <mutex>
+
+namespace lldb_private {
+
+/// A Progress indicator helper class.
+///
+/// Any potentially long running sections of code in LLDB should report
+/// progress so that clients are aware of delays that might appear during
+/// debugging. Delays commonly include indexing debug information, parsing
+/// symbol tables for object files, downloading symbols from remote
+/// repositories, and many more things.
+///
+/// The Progress class helps make sure that progress is correctly reported
+/// and will always send an initial progress update, updates when
+/// Progress::Increment() is called, and also will make sure that a progress
+/// completed update is reported even if the user doesn't explicitly cause one
+/// to be sent.
+///
+/// The progress is reported via a callback whose type is ProgressCallback:
+///
+///   typedef void (*ProgressCallback)(uint64_t progress_id,
+///                                    const char *message,
+///                                    uint64_t completed,
+///                                    uint64_t total,
+///                                    void *baton);
+///
+/// This callback will always initially be called with "completed" set to zero
+/// and "total" set to the total amount specified in the contructor. This is
+/// considered the progress start event. As Progress::Increment() is called,
+/// the callback will be called as long as the Progress::m_completed has not
+/// yet exceeded the Progress::m_total. When the callback is called with
+/// Progress::m_completed == Progress::m_total, that is considered a progress
+/// completed event. If Progress::m_completed is non-zero and less than
+/// Progress::m_total, then this is considered a progress update event.
+///
+/// This callback will be called in the destructor if Progress::m_completed is
+/// not equal to Progress::m_total with the "completed" set to
+/// Progress::m_total. This ensures we always send a progress completed update
+/// even if the user does not.
+
+class Progress {
+public:
+  /// Construct a progress object that will report information.
+  ///
+  /// The constructor will create a unique progress reporting object and
+  /// immediately send out a progress update by calling the installed callback
+  /// with completed set to zero out of the specified total.
+  ///
+  /// @param [in] title The title of this progress activity.
+  ///
+  /// @param [in] total The total units of work to be done if specified, if
+  /// set to UINT64_MAX then an indeterminate progress indicator should be
+  /// displayed.
+  ///
+  /// @param [in] debugger An optional debugger pointer to specify that this
+  /// progress is to be reported only to specific debuggers.
+  Progress(std::string title, uint64_t total = UINT64_MAX,
+           lldb_private::Debugger *debugger = nullptr);
+
+  /// Destroy the progress object.
+  ///
+  /// If the progress has not yet sent a completion update, the destructor
+  /// will send out a notification where the completed == m_total. This ensures
+  /// that we always send out a progress complete notification.
+  ~Progress();
+
+  /// Increment the progress and send a notification to the intalled callback.
+  ///
+  /// If incrementing ends up exceeding m_total, m_completed will be updated
+  /// to match m_total and no subsequent progress notifications will be sent.
+  /// If no total was specified in the constructor, this function will not do
+  /// anything nor send any progress updates.
+  ///
+  /// @param [in] amount The amount to increment m_completed by.
+  void Increment(uint64_t amount = 1);
+
+private:
+  void ReportProgress();
+  static std::atomic<uint64_t> g_id;
+  /// The title of the progress activity.
+  std::string m_title;
+  std::mutex m_mutex;
+  /// A unique integer identifier for progress reporting.
+  const uint64_t m_id;
+  /// How much work ([0...m_total]) that has been completed.
+  uint64_t m_completed;
+  /// Total amount of work, UINT64_MAX for non deterministic progress.
+  const uint64_t m_total;
+  /// The optional debugger ID to report progress to. If this has no value then
+  /// all debuggers will receive this event.
+  llvm::Optional<lldb::user_id_t> m_debugger_id;
+  /// Set to true when progress has been reported where m_completed == m_total
+  /// to ensure that we don't send progress updates after progress has
+  /// completed.
+  bool m_complete = false;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_CORE_PROGRESS_H
diff --git a/lldb/include/lldb/Host/ProcessLaunchInfo.h b/lldb/include/lldb/Host/ProcessLaunchInfo.h
index ee97555808251225f6861caf10e83bd39404b7da..883133776214d64457fb0d6c6d849978c3a1a80c 100644
--- a/lldb/include/lldb/Host/ProcessLaunchInfo.h
+++ b/lldb/include/lldb/Host/ProcessLaunchInfo.h
@@ -20,6 +20,7 @@
 #include "lldb/Host/PseudoTerminal.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/ProcessInfo.h"
+#include "lldb/Utility/StructuredData.h"
 
 namespace lldb_private {
 
@@ -146,6 +147,28 @@ public:
     return m_flags.Test(lldb::eLaunchFlagDetachOnError);
   }
 
+  bool IsScriptedProcess() const {
+    return !m_scripted_process_class_name.empty();
+  }
+
+  std::string GetScriptedProcessClassName() const {
+    return m_scripted_process_class_name;
+  }
+
+  void SetScriptedProcessClassName(std::string name) {
+    m_scripted_process_class_name = name;
+  }
+
+  lldb_private::StructuredData::DictionarySP
+  GetScriptedProcessDictionarySP() const {
+    return m_scripted_process_dictionary_sp;
+  }
+
+  void SetScriptedProcessDictionarySP(
+      lldb_private::StructuredData::DictionarySP dictionary_sp) {
+    m_scripted_process_dictionary_sp = dictionary_sp;
+  }
+
 protected:
   FileSpec m_working_dir;
   std::string m_plugin_name;
@@ -161,6 +184,11 @@ protected:
                             // meaning to the upper levels of lldb.
   lldb::ListenerSP m_listener_sp;
   lldb::ListenerSP m_hijack_listener_sp;
+  std::string m_scripted_process_class_name; // The name of the class that will
+                                             // manage a scripted process.
+  StructuredData::DictionarySP
+      m_scripted_process_dictionary_sp; // A dictionary that holds key/value
+                                        // pairs passed to the scripted process.
 };
 }
 
diff --git a/lldb/include/lldb/Host/SafeMachO.h b/lldb/include/lldb/Host/SafeMachO.h
index d7c376d23a4a7d204e3e135074003f7ba08b3f73..0540383b8c526e54cc84144416fbc73d0ac63cbe 100644
--- a/lldb/include/lldb/Host/SafeMachO.h
+++ b/lldb/include/lldb/Host/SafeMachO.h
@@ -23,6 +23,7 @@
 
 #undef CPU_ARCH_MASK
 #undef CPU_ARCH_ABI64
+#undef CPU_ARCH_ABI64_32
 
 #undef CPU_TYPE_ANY
 #undef CPU_TYPE_X86
@@ -31,12 +32,13 @@
 #undef CPU_TYPE_MC98000
 #undef CPU_TYPE_ARM
 #undef CPU_TYPE_ARM64
+#undef CPU_TYPE_ARM64_32
 #undef CPU_TYPE_SPARC
 #undef CPU_TYPE_POWERPC
 #undef CPU_TYPE_POWERPC64
 
-#undef CPU_SUB_TYPE_MASK
-#undef CPU_SUB_TYPE_LIB64
+#undef CPU_SUBTYPE_MASK
+#undef CPU_SUBTYPE_LIB64
 
 #undef CPU_SUBTYPE_MULTIPLE
 
@@ -88,6 +90,9 @@
 #undef CPU_SUBTYPE_ARM_V7M
 #undef CPU_SUBTYPE_ARM_V7EM
 
+#undef CPU_SUBTYPE_ARM64E
+#undef CPU_SUBTYPE_ARM64_32_V8
+#undef CPU_SUBTYPE_ARM64_V8
 #undef CPU_SUBTYPE_ARM64_ALL
 
 #undef CPU_SUBTYPE_SPARC_ALL
@@ -110,6 +115,47 @@
 #undef CPU_SUBTYPE_MC980000_ALL
 #undef CPU_SUBTYPE_MC98601
 
+#undef VM_PROT_READ
+#undef VM_PROT_WRITE
+#undef VM_PROT_EXECUTE
+
+#undef ARM_DEBUG_STATE
+#undef ARM_EXCEPTION_STATE
+#undef ARM_EXCEPTION_STATE64
+#undef ARM_EXCEPTION_STATE64_COUNT
+#undef ARM_THREAD_STATE
+#undef ARM_THREAD_STATE64
+#undef ARM_THREAD_STATE64_COUNT
+#undef ARM_THREAD_STATE_COUNT
+#undef ARM_VFP_STATE
+#undef ARN_THREAD_STATE_NONE
+#undef PPC_EXCEPTION_STATE
+#undef PPC_EXCEPTION_STATE64
+#undef PPC_FLOAT_STATE
+#undef PPC_THREAD_STATE
+#undef PPC_THREAD_STATE64
+#undef PPC_THREAD_STATE_NONE
+#undef PPC_VECTOR_STATE
+#undef x86_DEBUG_STATE
+#undef x86_DEBUG_STATE32
+#undef x86_DEBUG_STATE64
+#undef x86_EXCEPTION_STATE
+#undef x86_EXCEPTION_STATE32
+#undef x86_EXCEPTION_STATE64
+#undef x86_EXCEPTION_STATE64_COUNT
+#undef x86_EXCEPTION_STATE_COUNT
+#undef x86_FLOAT_STATE
+#undef x86_FLOAT_STATE32
+#undef x86_FLOAT_STATE64
+#undef x86_FLOAT_STATE64_COUNT
+#undef x86_FLOAT_STATE_COUNT
+#undef x86_THREAD_STATE
+#undef x86_THREAD_STATE32
+#undef x86_THREAD_STATE32_COUNT
+#undef x86_THREAD_STATE64
+#undef x86_THREAD_STATE64_COUNT
+#undef x86_THREAD_STATE_COUNT
+
 #include "llvm/BinaryFormat/MachO.h"
 
 #endif // LLDB_HOST_SAFEMACHO_H
diff --git a/lldb/include/lldb/Interpreter/ScriptInterpreter.h b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
index 275d61a63f056889d1d280a28dc15bcf27427db3..fa713e66d000cecd36c4a9eb1607da9bca8db0e4 100644
--- a/lldb/include/lldb/Interpreter/ScriptInterpreter.h
+++ b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
@@ -9,12 +9,15 @@
 #ifndef LLDB_INTERPRETER_SCRIPTINTERPRETER_H
 #define LLDB_INTERPRETER_SCRIPTINTERPRETER_H
 
+#include "lldb/API/SBData.h"
+#include "lldb/API/SBError.h"
 #include "lldb/Breakpoint/BreakpointOptions.h"
 #include "lldb/Core/Communication.h"
 #include "lldb/Core/PluginInterface.h"
 #include "lldb/Core/SearchFilter.h"
 #include "lldb/Core/StreamFile.h"
 #include "lldb/Host/PseudoTerminal.h"
+#include "lldb/Interpreter/ScriptedProcessInterface.h"
 #include "lldb/Utility/Broadcaster.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StructuredData.h"
@@ -83,7 +86,9 @@ public:
     eScriptReturnTypeOpaqueObject
   };
 
-  ScriptInterpreter(Debugger &debugger, lldb::ScriptLanguage script_lang);
+  ScriptInterpreter(
+      Debugger &debugger, lldb::ScriptLanguage script_lang,
+      lldb::ScriptedProcessInterfaceUP scripted_process_interface_up = {});
 
   ~ScriptInterpreter() override = default;
 
@@ -528,9 +533,19 @@ public:
 
   lldb::ScriptLanguage GetLanguage() { return m_script_lang; }
 
+  ScriptedProcessInterface &GetScriptedProcessInterface() {
+    return *m_scripted_process_interface_up;
+  }
+
+  lldb::DataExtractorSP
+  GetDataExtractorFromSBData(const lldb::SBData &data) const;
+
+  Status GetStatusFromSBError(const lldb::SBError &error) const;
+
 protected:
   Debugger &m_debugger;
   lldb::ScriptLanguage m_script_lang;
+  lldb::ScriptedProcessInterfaceUP m_scripted_process_interface_up;
 };
 
 } // namespace lldb_private
diff --git a/lldb/include/lldb/Interpreter/ScriptedProcessInterface.h b/lldb/include/lldb/Interpreter/ScriptedProcessInterface.h
new file mode 100644
index 0000000000000000000000000000000000000000..67fa8e3133cd50833dd3049d51ce15d485e43df6
--- /dev/null
+++ b/lldb/include/lldb/Interpreter/ScriptedProcessInterface.h
@@ -0,0 +1,64 @@
+//===-- ScriptedProcessInterface.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_INTERPRETER_SCRIPTEDPROCESSINTERFACE_H
+#define LLDB_INTERPRETER_SCRIPTEDPROCESSINTERFACE_H
+
+#include "lldb/Core/StructuredDataImpl.h"
+#include "lldb/Interpreter/ScriptInterpreter.h"
+#include "lldb/lldb-private.h"
+
+#include <string>
+
+namespace lldb_private {
+class ScriptedProcessInterface {
+public:
+  ScriptedProcessInterface() : m_object_instance_sp(nullptr) {}
+
+  virtual ~ScriptedProcessInterface() = default;
+
+  virtual StructuredData::GenericSP
+  CreatePluginObject(const llvm::StringRef class_name, lldb::TargetSP target_sp,
+                     StructuredData::DictionarySP args_sp) {
+    return nullptr;
+  }
+
+  virtual Status Launch() { return Status("ScriptedProcess did not launch"); }
+
+  virtual Status Resume() { return Status("ScriptedProcess did not resume"); }
+
+  virtual lldb::MemoryRegionInfoSP
+  GetMemoryRegionContainingAddress(lldb::addr_t address) {
+    return nullptr;
+  }
+
+  virtual StructuredData::DictionarySP GetThreadWithID(lldb::tid_t tid) {
+    return nullptr;
+  }
+
+  virtual StructuredData::DictionarySP GetRegistersForThread(lldb::tid_t tid) {
+    return nullptr;
+  }
+
+  virtual lldb::DataExtractorSP
+  ReadMemoryAtAddress(lldb::addr_t address, size_t size, Status &error) {
+    return nullptr;
+  }
+
+  virtual StructuredData::DictionarySP GetLoadedImages() { return nullptr; }
+
+  virtual lldb::pid_t GetProcessID() { return LLDB_INVALID_PROCESS_ID; }
+
+  virtual bool IsAlive() { return true; }
+
+private:
+  StructuredData::ObjectSP m_object_instance_sp;
+};
+} // namespace lldb_private
+
+#endif // LLDB_INTERPRETER_SCRIPTEDPROCESSINTERFACE_H
diff --git a/lldb/include/lldb/Target/DynamicLoader.h b/lldb/include/lldb/Target/DynamicLoader.h
index dead3eec7013348b75f74b1224cb02776368962b..1e4038bb208e93ee0d7f9b51d1fcbea597f064c7 100644
--- a/lldb/include/lldb/Target/DynamicLoader.h
+++ b/lldb/include/lldb/Target/DynamicLoader.h
@@ -251,6 +251,14 @@ public:
     return false;
   }
 
+  /// Return whether the dynamic loader is fully initialized and it's safe to
+  /// call its APIs.
+  ///
+  /// On some systems (e.g. Darwin based systems), lldb will get notified by
+  /// the dynamic loader before it itself finished initializing and it's not
+  /// safe to call certain APIs or SPIs.
+  virtual bool IsFullyInitialized() { return true; }
+
 protected:
   // Utility methods for derived classes
 
@@ -294,7 +302,7 @@ protected:
   // Read a pointer from memory at the given addr. Return LLDB_INVALID_ADDRESS
   // if the read fails.
   lldb::addr_t ReadPointer(lldb::addr_t addr);
-  
+
   // Calls into the Process protected method LoadOperatingSystemPlugin:
   void LoadOperatingSystemPlugin(bool flush);
 
diff --git a/lldb/include/lldb/Utility/LLDBAssert.h b/lldb/include/lldb/Utility/LLDBAssert.h
index 845af1d4cc2aa96819db746c74277cbef220ef48..471a2f7e824fcd531bdc9ad723c4d2427fffabc5 100644
--- a/lldb/include/lldb/Utility/LLDBAssert.h
+++ b/lldb/include/lldb/Utility/LLDBAssert.h
@@ -20,6 +20,6 @@
 namespace lldb_private {
 void lldb_assert(bool expression, const char *expr_text, const char *func,
                  const char *file, unsigned int line);
-}
+} // namespace lldb_private
 
 #endif // LLDB_UTILITY_LLDBASSERT_H
diff --git a/lldb/include/lldb/Utility/Timer.h b/lldb/include/lldb/Utility/Timer.h
index edc064b23b574d21aeeae3ee46920d48e298a382..32422b34b1adfdb975d93e3b1f32e203c5ce6c02 100644
--- a/lldb/include/lldb/Utility/Timer.h
+++ b/lldb/include/lldb/Utility/Timer.h
@@ -76,7 +76,7 @@ private:
 
 #define LLDB_SCOPED_TIMER()                                                    \
   static ::lldb_private::Timer::Category _cat(LLVM_PRETTY_FUNCTION);           \
-  ::lldb_private::Timer _scoped_timer(_cat, LLVM_PRETTY_FUNCTION)
+  ::lldb_private::Timer _scoped_timer(_cat, "%s", LLVM_PRETTY_FUNCTION)
 #define LLDB_SCOPED_TIMERF(...)                                                \
   static ::lldb_private::Timer::Category _cat(LLVM_PRETTY_FUNCTION);           \
   ::lldb_private::Timer _scoped_timer(_cat, __VA_ARGS__)
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index a297a928a3f4ca968f21ffd550f5692ea5397b89..3cf575486277679b90e5e57f47a8b6063755fa3f 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -174,6 +174,7 @@ class RichManglingContext;
 class Scalar;
 class ScriptInterpreter;
 class ScriptInterpreterLocker;
+class ScriptedProcessInterface;
 class ScriptedSyntheticChildren;
 class SearchFilter;
 class Section;
@@ -341,6 +342,7 @@ typedef std::shared_ptr<lldb_private::Listener> ListenerSP;
 typedef std::weak_ptr<lldb_private::Listener> ListenerWP;
 typedef std::shared_ptr<lldb_private::MemoryHistory> MemoryHistorySP;
 typedef std::unique_ptr<lldb_private::MemoryRegionInfo> MemoryRegionInfoUP;
+typedef std::shared_ptr<lldb_private::MemoryRegionInfo> MemoryRegionInfoSP;
 typedef std::shared_ptr<lldb_private::Module> ModuleSP;
 typedef std::weak_ptr<lldb_private::Module> ModuleWP;
 typedef std::shared_ptr<lldb_private::ObjectFile> ObjectFileSP;
@@ -391,6 +393,8 @@ typedef std::shared_ptr<lldb_private::ScriptSummaryFormat>
     ScriptSummaryFormatSP;
 typedef std::shared_ptr<lldb_private::ScriptInterpreter> ScriptInterpreterSP;
 typedef std::unique_ptr<lldb_private::ScriptInterpreter> ScriptInterpreterUP;
+typedef std::unique_ptr<lldb_private::ScriptedProcessInterface>
+    ScriptedProcessInterfaceUP;
 typedef std::shared_ptr<lldb_private::Section> SectionSP;
 typedef std::unique_ptr<lldb_private::SectionList> SectionListUP;
 typedef std::weak_ptr<lldb_private::Section> SectionWP;
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
index 5a433f2baac2ee5432b868031e0edabd34b37c59..926a63aeb239ee340fedec03c9e8d4dc114e8609 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
@@ -113,6 +113,7 @@ class DebugCommunication(object):
         self.initialize_body = None
         self.thread_stop_reasons = {}
         self.breakpoint_events = []
+        self.progress_events = []
         self.sequence = 1
         self.threads = None
         self.recv_thread.start()
@@ -225,6 +226,13 @@ class DebugCommunication(object):
                 self.breakpoint_events.append(packet)
                 # no need to add 'breakpoint' event packets to our packets list
                 return keepGoing
+            elif event.startswith('progress'):
+                # Progress events come in as 'progressStart', 'progressUpdate',
+                # and 'progressEnd' events. Keep these around in case test
+                # cases want to verify them.
+                self.progress_events.append(packet)
+                # No need to add 'progress' event packets to our packets list.
+                return keepGoing
 
         elif packet_type == 'response':
             if packet['command'] == 'disconnect':
diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp
index 6245b3a83565c1b0884e779933d0778dd65e3602..8bbb89516a0a95ebc6306a60849d67ea8e0029c9 100644
--- a/lldb/source/API/SBDebugger.cpp
+++ b/lldb/source/API/SBDebugger.cpp
@@ -38,6 +38,7 @@
 
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Core/Progress.h"
 #include "lldb/Core/StreamFile.h"
 #include "lldb/Core/StructuredDataImpl.h"
 #include "lldb/DataFormatters/DataVisualization.h"
@@ -149,6 +150,41 @@ SBDebugger &SBDebugger::operator=(const SBDebugger &rhs) {
   return LLDB_RECORD_RESULT(*this);
 }
 
+const char *SBDebugger::GetBroadcasterClass() {
+  LLDB_RECORD_STATIC_METHOD_NO_ARGS(const char *, SBDebugger,
+                                    GetBroadcasterClass);
+
+  return Debugger::GetStaticBroadcasterClass().AsCString();
+}
+
+const char *SBDebugger::GetProgressFromEvent(const lldb::SBEvent &event,
+                                             uint64_t &progress_id,
+                                             uint64_t &completed,
+                                             uint64_t &total,
+                                             bool &is_debugger_specific) {
+  const Debugger::ProgressEventData *progress_data =
+      Debugger::ProgressEventData::GetEventDataFromEvent(event.get());
+  if (progress_data == nullptr)
+    return nullptr;
+  progress_id = progress_data->GetID();
+  completed = progress_data->GetCompleted();
+  total = progress_data->GetTotal();
+  is_debugger_specific = progress_data->IsDebuggerSpecific();
+  // We must record the static method _after_ the out paramters have been
+  // filled in.
+  LLDB_RECORD_STATIC_METHOD(
+      const char *, SBDebugger, GetProgressFromEvent,
+      (const lldb::SBEvent &, uint64_t &, uint64_t &, uint64_t &, bool &),
+      event, progress_id, completed, total, is_debugger_specific);
+  return LLDB_RECORD_RESULT(progress_data->GetMessage().c_str())
+}
+
+SBBroadcaster SBDebugger::GetBroadcaster() {
+  LLDB_RECORD_METHOD_NO_ARGS(lldb::SBBroadcaster, SBDebugger, GetBroadcaster);
+  SBBroadcaster broadcaster(&m_opaque_sp->GetBroadcaster(), false);
+  return LLDB_RECORD_RESULT(broadcaster);
+}
+
 void SBDebugger::Initialize() {
   LLDB_RECORD_STATIC_METHOD_NO_ARGS(void, SBDebugger, Initialize);
   SBError ignored = SBDebugger::InitializeWithErrorHandling();
@@ -824,7 +860,7 @@ SBTarget SBDebugger::CreateTargetWithFileAndArch(const char *filename,
     if (error.Success())
       sb_target.SetSP(target_sp);
   }
-  
+
   LLDB_LOGF(log,
             "SBDebugger(%p)::CreateTargetWithFileAndArch (filename=\"%s\", "
             "arch=%s) => SBTarget(%p)",
@@ -1711,6 +1747,12 @@ template <> void RegisterMethods<SBDebugger>(Registry &R) {
   LLDB_REGISTER_METHOD(void, SBDebugger, Clear, ());
   LLDB_REGISTER_STATIC_METHOD(lldb::SBDebugger, SBDebugger, Create, ());
   LLDB_REGISTER_STATIC_METHOD(lldb::SBDebugger, SBDebugger, Create, (bool));
+  LLDB_REGISTER_STATIC_METHOD(
+      const char *, SBDebugger, GetProgressFromEvent,
+      (const lldb::SBEvent &, uint64_t &, uint64_t &, uint64_t &, bool &));
+  LLDB_REGISTER_STATIC_METHOD(const char *, SBDebugger, GetBroadcasterClass,
+                              ());
+  LLDB_REGISTER_METHOD(SBBroadcaster, SBDebugger, GetBroadcaster, ());
   LLDB_REGISTER_STATIC_METHOD(void, SBDebugger, Destroy, (lldb::SBDebugger &));
   LLDB_REGISTER_STATIC_METHOD(void, SBDebugger, MemoryPressureDetected, ());
   LLDB_REGISTER_METHOD_CONST(bool, SBDebugger, IsValid, ());
diff --git a/lldb/source/API/SBLaunchInfo.cpp b/lldb/source/API/SBLaunchInfo.cpp
index cda8134c9853b1dd09029485a036648983f59801..70cd1c6ecf744608d07901740d8e397745d093d0 100644
--- a/lldb/source/API/SBLaunchInfo.cpp
+++ b/lldb/source/API/SBLaunchInfo.cpp
@@ -10,8 +10,12 @@
 #include "SBReproducerPrivate.h"
 
 #include "lldb/API/SBEnvironment.h"
+#include "lldb/API/SBError.h"
 #include "lldb/API/SBFileSpec.h"
 #include "lldb/API/SBListener.h"
+#include "lldb/API/SBStream.h"
+#include "lldb/API/SBStructuredData.h"
+#include "lldb/Core/StructuredDataImpl.h"
 #include "lldb/Host/ProcessLaunchInfo.h"
 
 using namespace lldb;
@@ -343,6 +347,53 @@ bool SBLaunchInfo::GetDetachOnError() const {
   return m_opaque_sp->GetDetachOnError();
 }
 
+const char *SBLaunchInfo::GetScriptedProcessClassName() const {
+  LLDB_RECORD_METHOD_CONST_NO_ARGS(const char *, SBLaunchInfo,
+                                   GetScriptedProcessClassName);
+
+  // Constify this string so that it is saved in the string pool.  Otherwise it
+  // would be freed when this function goes out of scope.
+  ConstString class_name(m_opaque_sp->GetScriptedProcessClassName().c_str());
+  return class_name.AsCString();
+}
+
+void SBLaunchInfo::SetScriptedProcessClassName(const char *class_name) {
+  LLDB_RECORD_METHOD(void, SBLaunchInfo, SetScriptedProcessClassName,
+                     (const char *), class_name);
+
+  m_opaque_sp->SetScriptedProcessClassName(class_name);
+}
+
+lldb::SBStructuredData SBLaunchInfo::GetScriptedProcessDictionary() const {
+  LLDB_RECORD_METHOD_CONST_NO_ARGS(lldb::SBStructuredData, SBLaunchInfo,
+                                   GetScriptedProcessDictionary);
+
+  lldb_private::StructuredData::DictionarySP dict_sp =
+      m_opaque_sp->GetScriptedProcessDictionarySP();
+
+  SBStructuredData data;
+  data.m_impl_up->SetObjectSP(dict_sp);
+
+  return LLDB_RECORD_RESULT(data);
+}
+
+void SBLaunchInfo::SetScriptedProcessDictionary(lldb::SBStructuredData dict) {
+  LLDB_RECORD_METHOD(void, SBLaunchInfo, SetScriptedProcessDictionary,
+                     (lldb::SBStructuredData), dict);
+
+  SBStream stream;
+  SBError error = dict.GetAsJSON(stream);
+
+  if (error.Fail())
+    return;
+
+  StructuredData::DictionarySP dict_sp;
+  llvm::json::OStream s(stream.ref().AsRawOstream());
+  dict_sp->Serialize(s);
+
+  m_opaque_sp->SetScriptedProcessDictionarySP(dict_sp);
+}
+
 namespace lldb_private {
 namespace repro {
 
@@ -403,6 +454,14 @@ void RegisterMethods<SBLaunchInfo>(Registry &R) {
                              ());
   LLDB_REGISTER_METHOD(void, SBLaunchInfo, SetDetachOnError, (bool));
   LLDB_REGISTER_METHOD_CONST(bool, SBLaunchInfo, GetDetachOnError, ());
+  LLDB_REGISTER_METHOD_CONST(const char *, SBLaunchInfo,
+                             GetScriptedProcessClassName, ());
+  LLDB_REGISTER_METHOD(void, SBLaunchInfo, SetScriptedProcessClassName,
+                       (const char *));
+  LLDB_REGISTER_METHOD_CONST(lldb::SBStructuredData, SBLaunchInfo,
+                             GetScriptedProcessDictionary, ());
+  LLDB_REGISTER_METHOD(void, SBLaunchInfo, SetScriptedProcessDictionary,
+                       (lldb::SBStructuredData));
   LLDB_REGISTER_METHOD(void, SBLaunchInfo, SetEnvironment,
                        (const lldb::SBEnvironment &, bool));
   LLDB_REGISTER_METHOD(lldb::SBEnvironment, SBLaunchInfo, GetEnvironment, ());
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index d3d6ea08bdb385845d1a122e2e8388f951c68c93..617b89bf19643eff37d751ffad30cd3b4e1308ec 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -195,6 +195,13 @@ bool BreakpointLocation::InvokeCallback(StoppointCallbackContext *context) {
     return m_owner.InvokeCallback(context, GetID());
 }
 
+bool BreakpointLocation::IsCallbackSynchronous() {
+  if (m_options_up != nullptr && m_options_up->HasCallback())
+    return m_options_up->IsCallbackSynchronous();
+  else
+    return m_owner.GetOptions()->IsCallbackSynchronous();
+}
+
 void BreakpointLocation::SetCallback(BreakpointHitCallback callback,
                                      void *baton, bool is_synchronous) {
   // The default "Baton" class will keep a copy of "baton" and won't free or
diff --git a/lldb/source/Breakpoint/BreakpointOptions.cpp b/lldb/source/Breakpoint/BreakpointOptions.cpp
index 2fdb53e527237ecabd2a26d32a91e1f86aca1bac..920f843d9764164131216e4d5eb147748ab2c1bb 100644
--- a/lldb/source/Breakpoint/BreakpointOptions.cpp
+++ b/lldb/source/Breakpoint/BreakpointOptions.cpp
@@ -453,8 +453,6 @@ bool BreakpointOptions::InvokeCallback(StoppointCallbackContext *context,
                                           : nullptr,
                       context, break_id, break_loc_id);
     } else if (IsCallbackSynchronous()) {
-      // If a synchronous callback is called at async time, it should not say
-      // to stop.
       return false;
     }
   }
diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp
index 2aac4920a73bf0cea592ac6e580865daddc60c48..ca8487906fd0583aaf151818ab74902728d85854 100644
--- a/lldb/source/Commands/CommandObjectMemory.cpp
+++ b/lldb/source/Commands/CommandObjectMemory.cpp
@@ -608,7 +608,7 @@ protected:
       } else if (end_addr <= addr) {
         result.AppendErrorWithFormat(
             "end address (0x%" PRIx64
-            ") must be greater that the start address (0x%" PRIx64 ").\n",
+            ") must be greater than the start address (0x%" PRIx64 ").\n",
             end_addr, addr);
         result.SetStatus(eReturnStatusFailed);
         return false;
@@ -1505,7 +1505,7 @@ protected:
 
       case eFormatUnsigned:
 
-        if (!entry.ref().getAsInteger(0, uval64)) {
+        if (entry.ref().getAsInteger(0, uval64)) {
           result.AppendErrorWithFormat(
               "'%s' is not a valid unsigned decimal string value.\n",
               entry.c_str());
diff --git a/lldb/source/Commands/CommandObjectPlatform.cpp b/lldb/source/Commands/CommandObjectPlatform.cpp
index f306da3c8543e54ebaf40840739c584738ca5499..d0c58ac807ff56b17ff77fb78efc127cc66a3fa8 100644
--- a/lldb/source/Commands/CommandObjectPlatform.cpp
+++ b/lldb/source/Commands/CommandObjectPlatform.cpp
@@ -1009,11 +1009,14 @@ public:
                             "Launch a new process on a remote platform.",
                             "platform process launch program",
                             eCommandRequiresTarget | eCommandTryTargetAPILock),
-        m_options() {}
+        m_options(), m_all_options() {
+    m_all_options.Append(&m_options);
+    m_all_options.Finalize();
+  }
 
   ~CommandObjectPlatformProcessLaunch() override = default;
 
-  Options *GetOptions() override { return &m_options; }
+  Options *GetOptions() override { return &m_all_options; }
 
 protected:
   bool DoExecute(Args &args, CommandReturnObject &result) override {
@@ -1085,6 +1088,7 @@ protected:
   }
 
   CommandOptionsProcessLaunch m_options;
+  OptionGroupOptions m_all_options;
 };
 
 // "platform process list"
diff --git a/lldb/source/Commands/CommandObjectProcess.cpp b/lldb/source/Commands/CommandObjectProcess.cpp
index c21c499536f2b12884ff15f32f71730bfc0ddda8..91ae4c64ab0e79537578da6868d5610caabc01fd 100644
--- a/lldb/source/Commands/CommandObjectProcess.cpp
+++ b/lldb/source/Commands/CommandObjectProcess.cpp
@@ -17,6 +17,7 @@
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Interpreter/OptionArgParser.h"
+#include "lldb/Interpreter/OptionGroupPythonClassWithDict.h"
 #include "lldb/Interpreter/Options.h"
 #include "lldb/Target/Platform.h"
 #include "lldb/Target/Process.h"
@@ -108,7 +109,14 @@ public:
             interpreter, "process launch",
             "Launch the executable in the debugger.", nullptr,
             eCommandRequiresTarget, "restart"),
-        m_options() {
+        m_options(),
+        m_class_options("scripted process", true, 'C', 'k', 'v', 0),
+        m_all_options() {
+    m_all_options.Append(&m_options);
+    m_all_options.Append(&m_class_options, LLDB_OPT_SET_1 | LLDB_OPT_SET_2,
+                         LLDB_OPT_SET_ALL);
+    m_all_options.Finalize();
+
     CommandArgumentEntry arg;
     CommandArgumentData run_args_arg;
 
@@ -135,7 +143,7 @@ public:
         request, nullptr);
   }
 
-  Options *GetOptions() override { return &m_options; }
+  Options *GetOptions() override { return &m_all_options; }
 
   const char *GetRepeatCommand(Args &current_command_args,
                                uint32_t index) override {
@@ -180,6 +188,15 @@ protected:
       disable_aslr = target->GetDisableASLR();
     }
 
+    if (!m_class_options.GetName().empty()) {
+      m_options.launch_info.SetProcessPluginName("ScriptedProcess");
+      m_options.launch_info.SetScriptedProcessClassName(
+          m_class_options.GetName());
+      m_options.launch_info.SetScriptedProcessDictionarySP(
+          m_class_options.GetStructuredData());
+      target->SetProcessLaunchInfo(m_options.launch_info);
+    }
+
     if (disable_aslr)
       m_options.launch_info.GetFlags().Set(eLaunchFlagDisableASLR);
     else
@@ -253,6 +270,8 @@ protected:
   }
 
   CommandOptionsProcessLaunch m_options;
+  OptionGroupPythonClassWithDict m_class_options;
+  OptionGroupOptions m_all_options;
 };
 
 #define LLDB_OPTIONS_process_attach
diff --git a/lldb/source/Commands/CommandOptionsProcessLaunch.cpp b/lldb/source/Commands/CommandOptionsProcessLaunch.cpp
index dfc4887fec2612d145a1edff04a7af32326e3764..a618796156a6aeabf763a93bb649c98589ee288b 100644
--- a/lldb/source/Commands/CommandOptionsProcessLaunch.cpp
+++ b/lldb/source/Commands/CommandOptionsProcessLaunch.cpp
@@ -30,7 +30,7 @@ Status CommandOptionsProcessLaunch::SetOptionValue(
     uint32_t option_idx, llvm::StringRef option_arg,
     ExecutionContext *execution_context) {
   Status error;
-  const int short_option = m_getopt_table[option_idx].val;
+  const int short_option = g_process_launch_options[option_idx].short_option;
 
   switch (short_option) {
   case 's': // Stop at program entry point
diff --git a/lldb/source/Commands/CommandOptionsProcessLaunch.h b/lldb/source/Commands/CommandOptionsProcessLaunch.h
index e0ece8a5159ddea032c0f227ca2bf207b5a30a0c..d18a23245080df33c8d448f5258c087b4e7998cd 100644
--- a/lldb/source/Commands/CommandOptionsProcessLaunch.h
+++ b/lldb/source/Commands/CommandOptionsProcessLaunch.h
@@ -1,4 +1,4 @@
-//===-- CommandOptionsProcessLaunch.h -------------------------------------===//
+//===-- CommandOptionsProcessLaunch.h ---------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -16,9 +16,9 @@ namespace lldb_private {
 
 // CommandOptionsProcessLaunch
 
-class CommandOptionsProcessLaunch : public lldb_private::Options {
+class CommandOptionsProcessLaunch : public lldb_private::OptionGroup {
 public:
-  CommandOptionsProcessLaunch() : lldb_private::Options() {
+  CommandOptionsProcessLaunch() : lldb_private::OptionGroup() {
     // Keep default values of all options in one place: OptionParsingStarting
     // ()
     OptionParsingStarting(nullptr);
diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td
index c19874f731202f8269222a681efc71da19d15c9a..1cc190ebc211e42e1f49963fb0fc3e636a75dbb4 100644
--- a/lldb/source/Commands/Options.td
+++ b/lldb/source/Commands/Options.td
@@ -200,7 +200,7 @@ let Command = "breakpoint set" in {
   def breakpoint_set_move_to_nearest_code : Option<"move-to-nearest-code", "m">,
     Groups<[1,9,12]>, Arg<"Boolean">,
     Desc<"Move breakpoints to nearest code. If not set the "
-    "target.move-to-nearest-codesetting is used.">;
+    "target.move-to-nearest-code setting is used.">;
   def breakpoint_set_file_colon_line : Option<"joint-specifier", "y">, Group<12>, Arg<"FileLineColumn">,
     Required, Completion<"SourceFile">,
     Desc<"A specifier in the form filename:line[:column] for setting file & line breakpoints.">;
diff --git a/lldb/source/Core/CMakeLists.txt b/lldb/source/Core/CMakeLists.txt
index 3d6f3e2230f98aa8c26e6a08308f815ca426d859..ae07de6789c5795df35aee1f46dd511239035155 100644
--- a/lldb/source/Core/CMakeLists.txt
+++ b/lldb/source/Core/CMakeLists.txt
@@ -43,6 +43,7 @@ add_lldb_library(lldbCore
   ModuleList.cpp
   Opcode.cpp
   PluginManager.cpp
+  Progress.cpp
   RichManglingContext.cpp
   SearchFilter.cpp
   Section.cpp
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index a9a5c7445976739c95319faae1b73035010362e9..d3b2cc62b6767a9a18dcb60bfbda9afef473cd75 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -661,6 +661,11 @@ TargetSP Debugger::FindTargetWithProcess(Process *process) {
   return target_sp;
 }
 
+ConstString Debugger::GetStaticBroadcasterClass() {
+  static ConstString class_name("lldb.debugger");
+  return class_name;
+}
+
 Debugger::Debugger(lldb::LogOutputCallback log_callback, void *baton)
     : UserID(g_unique_id++),
       Properties(std::make_shared<OptionValueProperties>()),
@@ -677,6 +682,8 @@ Debugger::Debugger(lldb::LogOutputCallback log_callback, void *baton)
       m_io_handler_stack(), m_instance_name(), m_loaded_plugins(),
       m_event_handler_thread(), m_io_handler_thread(),
       m_sync_broadcaster(nullptr, "lldb.debugger.sync"),
+      m_broadcaster(m_broadcaster_manager_sp,
+                    GetStaticBroadcasterClass().AsCString()),
       m_forward_listener_sp(), m_clear_once() {
   m_instance_name.SetString(llvm::formatv("debugger_{0}", GetID()).str());
   if (log_callback)
@@ -1137,6 +1144,74 @@ void Debugger::SetLoggingCallback(lldb::LogOutputCallback log_callback,
       std::make_shared<StreamCallback>(log_callback, baton);
 }
 
+ConstString Debugger::ProgressEventData::GetFlavorString() {
+  static ConstString g_flavor("Debugger::ProgressEventData");
+  return g_flavor;
+}
+
+ConstString Debugger::ProgressEventData::GetFlavor() const {
+  return Debugger::ProgressEventData::GetFlavorString();
+}
+
+void Debugger::ProgressEventData::Dump(Stream *s) const {
+  s->Printf(" id = %" PRIu64 ", message = \"%s\"", m_id, m_message.c_str());
+  if (m_completed == 0 || m_completed == m_total)
+    s->Printf(", type = %s", m_completed == 0 ? "start" : "end");
+  else
+    s->PutCString(", type = update");
+  // If m_total is UINT64_MAX, there is no progress to report, just "start"
+  // and "end". If it isn't we will show the completed and total amounts.
+  if (m_total != UINT64_MAX)
+    s->Printf(", progress = %" PRIu64 " of %" PRIu64, m_completed, m_total);
+}
+
+const Debugger::ProgressEventData *
+Debugger::ProgressEventData::GetEventDataFromEvent(const Event *event_ptr) {
+  if (event_ptr)
+    if (const EventData *event_data = event_ptr->GetData())
+      if (event_data->GetFlavor() == ProgressEventData::GetFlavorString())
+        return static_cast<const ProgressEventData *>(event_ptr->GetData());
+  return nullptr;
+}
+
+static void PrivateReportProgress(Debugger &debugger, uint64_t progress_id,
+                                  const std::string &message,
+                                  uint64_t completed, uint64_t total,
+                                  bool is_debugger_specific) {
+  // Only deliver progress events if we have any progress listeners.
+  const uint32_t event_type = Debugger::eBroadcastBitProgress;
+  if (!debugger.GetBroadcaster().EventTypeHasListeners(event_type))
+    return;
+  EventSP event_sp(new Event(event_type, new Debugger::ProgressEventData(
+                                             progress_id, message, completed,
+                                             total, is_debugger_specific)));
+  debugger.GetBroadcaster().BroadcastEvent(event_sp);
+}
+
+void Debugger::ReportProgress(uint64_t progress_id, const std::string &message,
+                              uint64_t completed, uint64_t total,
+                              llvm::Optional<lldb::user_id_t> debugger_id) {
+  // Check if this progress is for a specific debugger.
+  if (debugger_id.hasValue()) {
+    // It is debugger specific, grab it and deliver the event if the debugger
+    // still exists.
+    DebuggerSP debugger_sp = FindDebuggerWithID(*debugger_id);
+    if (debugger_sp)
+      PrivateReportProgress(*debugger_sp, progress_id, message, completed,
+                            total, /*is_debugger_specific*/ true);
+    return;
+  }
+  // The progress event is not debugger specific, iterate over all debuggers
+  // and deliver a progress event to each one.
+  if (g_debugger_list_ptr && g_debugger_list_mutex_ptr) {
+    std::lock_guard<std::recursive_mutex> guard(*g_debugger_list_mutex_ptr);
+    DebuggerList::iterator pos, end = g_debugger_list_ptr->end();
+    for (pos = g_debugger_list_ptr->begin(); pos != end; ++pos)
+      PrivateReportProgress(*(*pos), progress_id, message, completed, total,
+                            /*is_debugger_specific*/ false);
+  }
+}
+
 bool Debugger::EnableLog(llvm::StringRef channel,
                          llvm::ArrayRef<const char *> categories,
                          llvm::StringRef log_file, uint32_t log_options,
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index 1f9987c216588eb0c70753174193e456c1daa082..3bd47b256b31f5ad17c8e56df6a4eedc2a04c6fc 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -1072,8 +1072,6 @@ std::string Module::GetSpecificationDescription() const {
 
 void Module::GetDescription(llvm::raw_ostream &s,
                             lldb::DescriptionLevel level) {
-  std::lock_guard<std::recursive_mutex> guard(m_mutex);
-
   if (level >= eDescriptionLevelFull) {
     if (m_arch.IsValid())
       s << llvm::formatv("({0}) ", m_arch.GetArchitectureName());
diff --git a/lldb/source/Core/Progress.cpp b/lldb/source/Core/Progress.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c54e7774adf3a30a6bcae3e6084f97d4cd232c10
--- /dev/null
+++ b/lldb/source/Core/Progress.cpp
@@ -0,0 +1,60 @@
+//===-- Progress.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Core/Progress.h"
+
+#include "lldb/Core/Debugger.h"
+#include "lldb/Utility/StreamString.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+std::atomic<uint64_t> Progress::g_id(0);
+
+Progress::Progress(std::string title, uint64_t total,
+                   lldb_private::Debugger *debugger)
+    : m_title(title), m_id(++g_id), m_completed(0), m_total(total) {
+  assert(total > 0);
+  if (debugger)
+    m_debugger_id = debugger->GetID();
+  std::lock_guard<std::mutex> guard(m_mutex);
+  ReportProgress();
+}
+
+Progress::~Progress() {
+  // Make sure to always report progress completed when this object is
+  // destructed so it indicates the progress dialog/activity should go away.
+  std::lock_guard<std::mutex> guard(m_mutex);
+  if (!m_completed) {
+    m_completed = m_total;
+    ReportProgress();
+  }
+}
+
+void Progress::Increment(uint64_t amount) {
+  if (amount > 0) {
+    std::lock_guard<std::mutex> guard(m_mutex);
+    // Watch out for unsigned overflow and make sure we don't increment too
+    // much and exceed m_total.
+    if (amount > (m_total - m_completed))
+      m_completed = m_total;
+    else
+      m_completed += amount;
+    ReportProgress();
+  }
+}
+
+void Progress::ReportProgress() {
+  if (!m_complete) {
+    // Make sure we only send one notification that indicates the progress is
+    // complete.
+    m_complete = m_completed == m_total;
+    Debugger::ReportProgress(m_id, m_title, m_completed, m_total,
+                             m_debugger_id);
+  }
+}
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index a8843ca0543b09f55484a6debf6503bc041e0364..12fe76ee95d2f768c5e527a632b280147a76721b 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -904,6 +904,52 @@ bool DWARFExpression::Evaluate(ExecutionContext *exe_ctx,
                                    object_address_ptr, result, error_ptr);
 }
 
+namespace {
+/// The location description kinds described by the DWARF v5
+/// specification.  Composite locations are handled out-of-band and
+/// thus aren't part of the enum.
+enum LocationDescriptionKind {
+  Empty,
+  Memory,
+  Register,
+  Implicit
+  /* Composite*/
+};
+/// Adjust value's ValueType according to the kind of location description.
+void UpdateValueTypeFromLocationDescription(Log *log, const DWARFUnit *dwarf_cu,
+                                            LocationDescriptionKind kind,
+                                            Value *value = nullptr) {
+  // Note that this function is conflating DWARF expressions with
+  // DWARF location descriptions. Perhaps it would be better to define
+  // a wrapper for DWARFExpresssion::Eval() that deals with DWARF
+  // location descriptions (which consist of one or more DWARF
+  // expressions). But doing this would mean we'd also need factor the
+  // handling of DW_OP_(bit_)piece out of this function.
+  if (dwarf_cu && dwarf_cu->GetVersion() >= 4) {
+    const char *log_msg = "DWARF location description kind: %s";
+    switch (kind) {
+    case Empty:
+      LLDB_LOGF(log, log_msg, "Empty");
+      break;
+    case Memory:
+      LLDB_LOGF(log, log_msg, "Memory");
+      if (value->GetValueType() == Value::ValueType::Scalar)
+        value->SetValueType(Value::ValueType::LoadAddress);
+      break;
+    case Register:
+      LLDB_LOGF(log, log_msg, "Register");
+      value->SetValueType(Value::ValueType::Scalar);
+      break;
+    case Implicit:
+      LLDB_LOGF(log, log_msg, "Implicit");
+      if (value->GetValueType() == Value::ValueType::LoadAddress)
+        value->SetValueType(Value::ValueType::Scalar);
+      break;
+    }
+  }
+}
+} // namespace
+
 bool DWARFExpression::Evaluate(
     ExecutionContext *exe_ctx, RegisterContext *reg_ctx,
     lldb::ModuleSP module_sp, const DataExtractor &opcodes,
@@ -952,6 +998,11 @@ bool DWARFExpression::Evaluate(
         !is_signed));
   };
 
+  // The default kind is a memory location. This is updated by any
+  // operation that changes this, such as DW_OP_stack_value, and reset
+  // by composition operations like DW_OP_piece.
+  LocationDescriptionKind dwarf4_location_description_kind = Memory;
+
   while (opcodes.ValidOffset(offset)) {
     const lldb::offset_t op_offset = offset;
     const uint8_t op = opcodes.GetU8(&offset);
@@ -1950,6 +2001,7 @@ bool DWARFExpression::Evaluate(
     case DW_OP_reg29:
     case DW_OP_reg30:
     case DW_OP_reg31: {
+      dwarf4_location_description_kind = Register;
       reg_num = op - DW_OP_reg0;
 
       if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, error_ptr, tmp))
@@ -1962,6 +2014,7 @@ bool DWARFExpression::Evaluate(
     //      ULEB128 literal operand that encodes the register.
     // DESCRIPTION: Push the value in register on the top of the stack.
     case DW_OP_regx: {
+      dwarf4_location_description_kind = Register;
       reg_num = opcodes.GetULEB128(&offset);
       if (ReadRegisterValueAsScalar(reg_ctx, reg_kind, reg_num, error_ptr, tmp))
         stack.push_back(tmp);
@@ -2085,12 +2138,18 @@ bool DWARFExpression::Evaluate(
     // provides a way of describing how large a part of a variable a particular
     // DWARF expression refers to.
     case DW_OP_piece: {
+      LocationDescriptionKind piece_locdesc = dwarf4_location_description_kind;
+      // Reset for the next piece.
+      dwarf4_location_description_kind = Memory;
+
       const uint64_t piece_byte_size = opcodes.GetULEB128(&offset);
 
       if (piece_byte_size > 0) {
         Value curr_piece;
 
         if (stack.empty()) {
+          UpdateValueTypeFromLocationDescription(
+              log, dwarf_cu, LocationDescriptionKind::Empty);
           // In a multi-piece expression, this means that the current piece is
           // not available. Fill with zeros for now by resizing the data and
           // appending it
@@ -2106,6 +2165,8 @@ bool DWARFExpression::Evaluate(
           // Extract the current piece into "curr_piece"
           Value curr_piece_source_value(stack.back());
           stack.pop_back();
+          UpdateValueTypeFromLocationDescription(log, dwarf_cu, piece_locdesc,
+                                                 &curr_piece_source_value);
 
           const Value::ValueType curr_piece_source_value_type =
               curr_piece_source_value.GetValueType();
@@ -2216,11 +2277,19 @@ bool DWARFExpression::Evaluate(
 
     case DW_OP_bit_piece: // 0x9d ULEB128 bit size, ULEB128 bit offset (DWARF3);
       if (stack.size() < 1) {
+        UpdateValueTypeFromLocationDescription(log, dwarf_cu,
+                                               LocationDescriptionKind::Empty);
+        // Reset for the next piece.
+        dwarf4_location_description_kind = Memory;
         if (error_ptr)
           error_ptr->SetErrorString(
               "Expression stack needs at least 1 item for DW_OP_bit_piece.");
         return false;
       } else {
+        UpdateValueTypeFromLocationDescription(
+            log, dwarf_cu, dwarf4_location_description_kind, &stack.back());
+        // Reset for the next piece.
+        dwarf4_location_description_kind = Memory;
         const uint64_t piece_bit_size = opcodes.GetULEB128(&offset);
         const uint64_t piece_bit_offset = opcodes.GetULEB128(&offset);
         switch (stack.back().GetValueType()) {
@@ -2261,6 +2330,8 @@ bool DWARFExpression::Evaluate(
     // DESCRIPTION: Value is immediately stored in block in the debug info with
     // the memory representation of the target.
     case DW_OP_implicit_value: {
+      dwarf4_location_description_kind = Implicit;
+
       const uint32_t len = opcodes.GetULEB128(&offset);
       const void *data = opcodes.GetData(&offset, len);
 
@@ -2276,6 +2347,12 @@ bool DWARFExpression::Evaluate(
       break;
     }
 
+    case DW_OP_implicit_pointer: {
+      dwarf4_location_description_kind = Implicit;
+      LLDB_ERRORF(error_ptr, "Could not evaluate %s.", DW_OP_value_to_name(op));
+      return false;
+    }
+
     // OPCODE: DW_OP_push_object_address
     // OPERANDS: none
     // DESCRIPTION: Pushes the address of the object currently being
@@ -2347,6 +2424,7 @@ bool DWARFExpression::Evaluate(
     // rather is a constant value.  The value from the top of the stack is the
     // value to be used.  This is the actual object value and not the location.
     case DW_OP_stack_value:
+      dwarf4_location_description_kind = Implicit;
       if (stack.empty()) {
         if (error_ptr)
           error_ptr->SetErrorString(
@@ -2567,25 +2645,28 @@ bool DWARFExpression::Evaluate(
     // or DW_OP_bit_piece opcodes
     if (pieces.GetBuffer().GetByteSize()) {
       result = pieces;
-    } else {
-      if (error_ptr)
-        error_ptr->SetErrorString("Stack empty after evaluation.");
-      return false;
+      return true;
     }
-  } else {
-    if (log && log->GetVerbose()) {
-      size_t count = stack.size();
-      LLDB_LOGF(log, "Stack after operation has %" PRIu64 " values:",
-                (uint64_t)count);
-      for (size_t i = 0; i < count; ++i) {
-        StreamString new_value;
-        new_value.Printf("[%" PRIu64 "]", (uint64_t)i);
-        stack[i].Dump(&new_value);
-        LLDB_LOGF(log, "  %s", new_value.GetData());
-      }
+    if (error_ptr)
+      error_ptr->SetErrorString("Stack empty after evaluation.");
+    return false;
+  }
+
+  UpdateValueTypeFromLocationDescription(
+      log, dwarf_cu, dwarf4_location_description_kind, &stack.back());
+
+  if (log && log->GetVerbose()) {
+    size_t count = stack.size();
+    LLDB_LOGF(log,
+              "Stack after operation has %" PRIu64 " values:", (uint64_t)count);
+    for (size_t i = 0; i < count; ++i) {
+      StreamString new_value;
+      new_value.Printf("[%" PRIu64 "]", (uint64_t)i);
+      stack[i].Dump(&new_value);
+      LLDB_LOGF(log, "  %s", new_value.GetData());
     }
-    result = stack.back();
   }
+  result = stack.back();
   return true; // Return true on success
 }
 
diff --git a/lldb/source/Host/common/FileSystem.cpp b/lldb/source/Host/common/FileSystem.cpp
index 64ecf27858abe91da5e5a70fdc88fba05f0cdb23..9076188168b6d986bbd8b2fa1c9aa9a2a78f6231 100644
--- a/lldb/source/Host/common/FileSystem.cpp
+++ b/lldb/source/Host/common/FileSystem.cpp
@@ -307,7 +307,7 @@ FileSystem::CreateDataBuffer(const llvm::Twine &path, uint64_t size,
   std::unique_ptr<llvm::WritableMemoryBuffer> buffer;
   if (size == 0) {
     auto buffer_or_error =
-        llvm::WritableMemoryBuffer::getFile(*external_path, -1, is_volatile);
+        llvm::WritableMemoryBuffer::getFile(*external_path, is_volatile);
     if (!buffer_or_error)
       return nullptr;
     buffer = std::move(*buffer_or_error);
diff --git a/lldb/source/Host/common/ProcessLaunchInfo.cpp b/lldb/source/Host/common/ProcessLaunchInfo.cpp
index 1b4b2c6c3ac293ca5e09a10f316d41e4a5acea42..d42687e9a4e326d61566a8b36a6b8dc4f3cb011b 100644
--- a/lldb/source/Host/common/ProcessLaunchInfo.cpp
+++ b/lldb/source/Host/common/ProcessLaunchInfo.cpp
@@ -32,7 +32,8 @@ ProcessLaunchInfo::ProcessLaunchInfo()
     : ProcessInfo(), m_working_dir(), m_plugin_name(), m_flags(0),
       m_file_actions(), m_pty(new PseudoTerminal), m_resume_count(0),
       m_monitor_callback(nullptr), m_monitor_callback_baton(nullptr),
-      m_monitor_signals(false), m_listener_sp(), m_hijack_listener_sp() {}
+      m_monitor_signals(false), m_listener_sp(), m_hijack_listener_sp(),
+      m_scripted_process_class_name(), m_scripted_process_dictionary_sp() {}
 
 ProcessLaunchInfo::ProcessLaunchInfo(const FileSpec &stdin_file_spec,
                                      const FileSpec &stdout_file_spec,
@@ -42,7 +43,8 @@ ProcessLaunchInfo::ProcessLaunchInfo(const FileSpec &stdin_file_spec,
     : ProcessInfo(), m_working_dir(), m_plugin_name(), m_flags(launch_flags),
       m_file_actions(), m_pty(new PseudoTerminal), m_resume_count(0),
       m_monitor_callback(nullptr), m_monitor_callback_baton(nullptr),
-      m_monitor_signals(false), m_listener_sp(), m_hijack_listener_sp() {
+      m_monitor_signals(false), m_listener_sp(), m_hijack_listener_sp(),
+      m_scripted_process_class_name(), m_scripted_process_dictionary_sp() {
   if (stdin_file_spec) {
     FileAction file_action;
     const bool read = true;
@@ -171,6 +173,8 @@ void ProcessLaunchInfo::Clear() {
   m_resume_count = 0;
   m_listener_sp.reset();
   m_hijack_listener_sp.reset();
+  m_scripted_process_class_name.clear();
+  m_scripted_process_dictionary_sp.reset();
 }
 
 void ProcessLaunchInfo::SetMonitorProcessCallback(
diff --git a/lldb/source/Interpreter/ScriptInterpreter.cpp b/lldb/source/Interpreter/ScriptInterpreter.cpp
index 2c55d08497abc332559d28ed7a86bff290ae1452..d822ea23a10a757730ee04c1a9faf7049fbce3c8 100644
--- a/lldb/source/Interpreter/ScriptInterpreter.cpp
+++ b/lldb/source/Interpreter/ScriptInterpreter.cpp
@@ -26,9 +26,12 @@
 using namespace lldb;
 using namespace lldb_private;
 
-ScriptInterpreter::ScriptInterpreter(Debugger &debugger,
-                                     lldb::ScriptLanguage script_lang)
-    : m_debugger(debugger), m_script_lang(script_lang) {}
+ScriptInterpreter::ScriptInterpreter(
+    Debugger &debugger, lldb::ScriptLanguage script_lang,
+    lldb::ScriptedProcessInterfaceUP scripted_process_interface_up)
+    : m_debugger(debugger), m_script_lang(script_lang),
+      m_scripted_process_interface_up(
+          std::move(scripted_process_interface_up)) {}
 
 void ScriptInterpreter::CollectDataForBreakpointCommandCallback(
     std::vector<BreakpointOptions *> &bp_options_vec,
@@ -69,6 +72,19 @@ std::string ScriptInterpreter::LanguageToString(lldb::ScriptLanguage language) {
   llvm_unreachable("Unhandled ScriptInterpreter!");
 }
 
+lldb::DataExtractorSP
+ScriptInterpreter::GetDataExtractorFromSBData(const lldb::SBData &data) const {
+  return data.m_opaque_sp;
+}
+
+Status
+ScriptInterpreter::GetStatusFromSBError(const lldb::SBError &error) const {
+  if (error.m_opaque_up)
+    return *error.m_opaque_up.get();
+
+  return Status();
+}
+
 lldb::ScriptLanguage
 ScriptInterpreter::StringToLanguage(const llvm::StringRef &language) {
   if (language.equals_lower(LanguageToString(eScriptLanguageNone)))
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
index 5cc6a6475d12ec255d05d64ff04c78ed9938f044..6872dd72efad8d81d2be731ae41d7c9d0f5c8f9a 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
@@ -1114,6 +1114,12 @@ bool DynamicLoaderMacOSXDYLD::GetSharedCacheInformation(
   return false;
 }
 
+bool DynamicLoaderMacOSXDYLD::IsFullyInitialized() {
+  if (ReadAllImageInfosStructure())
+    return m_dyld_all_image_infos.libSystemInitialized;
+  return false;
+}
+
 void DynamicLoaderMacOSXDYLD::Initialize() {
   PluginManager::RegisterPlugin(GetPluginNameStatic(),
                                 GetPluginDescriptionStatic(), CreateInstance);
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.h b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.h
index 21bf5875dc13ebd11955d2942ca36315e125687a..b3cb3e50fa30faf430bff8b46ec79c079e34a810 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.h
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.h
@@ -68,6 +68,8 @@ public:
 
   uint32_t GetPluginVersion() override;
 
+  bool IsFullyInitialized() override;
+
 protected:
   void PutToLog(lldb_private::Log *log) const;
 
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp
index 7b331307c0f783a748bb1e0da3453ad0221f6c6e..9bc40c16e5d05b13cceb4c578a5fd552bc62e573 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp
@@ -59,7 +59,7 @@ public:
       clang::DeclContext::lookup_result result =
           non_const_interface_decl->lookup(name);
 
-      return (result.size() != 0);
+      return (!result.empty());
     } while (false);
 
     SetNoExternalVisibleDeclsForName(decl_ctx, name);
@@ -555,7 +555,7 @@ uint32_t AppleObjCDeclVendor::FindDecls(ConstString name, bool append,
 
     if (!lookup_result.empty()) {
       if (clang::ObjCInterfaceDecl *result_iface_decl =
-              llvm::dyn_cast<clang::ObjCInterfaceDecl>(lookup_result[0])) {
+             llvm::dyn_cast<clang::ObjCInterfaceDecl>(*lookup_result.begin())) {
         if (log) {
           clang::QualType result_iface_type =
               ast_ctx.getObjCInterfaceType(result_iface_decl);
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index 30ee257f2b2902935425acb5005e04bda8ffa9a8..31ddf83ba9883195ec2f8917dd64ade5a1beba44 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -39,6 +39,7 @@
 #include "lldb/Symbol/TypeList.h"
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Target/ABI.h"
+#include "lldb/Target/DynamicLoader.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Target/Platform.h"
 #include "lldb/Target/Process.h"
@@ -75,8 +76,7 @@ char AppleObjCRuntimeV2::ID = 0;
 
 static const char *g_get_dynamic_class_info_name =
     "__lldb_apple_objc_v2_get_dynamic_class_info";
-// Testing using the new C++11 raw string literals. If this breaks GCC then we
-// will need to revert to the code above...
+
 static const char *g_get_dynamic_class_info_body = R"(
 
 extern "C"
@@ -164,6 +164,72 @@ __lldb_apple_objc_v2_get_dynamic_class_info (void *gdb_objc_realized_classes_ptr
 
 )";
 
+static const char *g_get_dynamic_class_info2_name =
+    "__lldb_apple_objc_v2_get_dynamic_class_info2";
+
+static const char *g_get_dynamic_class_info2_body = R"(
+
+extern "C" {
+    int printf(const char * format, ...);
+    void free(void *ptr);
+    Class* objc_copyRealizedClassList(unsigned int *outCount);
+    const char* objc_debug_class_getNameRaw(Class cls);
+}
+
+#define DEBUG_PRINTF(fmt, ...) if (should_log) printf(fmt, ## __VA_ARGS__)
+
+struct ClassInfo
+{
+    Class isa;
+    uint32_t hash;
+} __attribute__((__packed__));
+
+uint32_t
+__lldb_apple_objc_v2_get_dynamic_class_info2(void *gdb_objc_realized_classes_ptr,
+                                             void *class_infos_ptr,
+                                             uint32_t class_infos_byte_size,
+                                             uint32_t should_log)
+{
+    DEBUG_PRINTF ("class_infos_ptr = %p\n", class_infos_ptr);
+    DEBUG_PRINTF ("class_infos_byte_size = %u\n", class_infos_byte_size);
+
+    const size_t max_class_infos = class_infos_byte_size/sizeof(ClassInfo);
+    DEBUG_PRINTF ("max_class_infos = %u\n", max_class_infos);
+
+    ClassInfo *class_infos = (ClassInfo *)class_infos_ptr;
+
+    uint32_t count = 0;
+    Class* realized_class_list = objc_copyRealizedClassList(&count);
+
+    uint32_t idx = 0;
+    for (uint32_t i=0; i<=count; ++i)
+    {
+        if (idx < max_class_infos)
+        {
+            Class isa = realized_class_list[i];
+            const char *name_ptr = objc_debug_class_getNameRaw(isa);
+            const char *s = name_ptr;
+            uint32_t h = 5381;
+            for (unsigned char c = *s; c; c = *++s)
+                h = ((h << 5) + h) + c;
+            class_infos[idx].hash = h;
+            class_infos[idx].isa = isa;
+            DEBUG_PRINTF ("[%u] isa = %8p %s\n", idx, class_infos[idx].isa, name_ptr);
+        }
+        idx++;
+    }
+
+    if (idx < max_class_infos)
+    {
+        class_infos[idx].isa = NULL;
+        class_infos[idx].hash = 0;
+    }
+
+    free(realized_class_list);
+    return count;
+}
+)";
+
 // We'll substitute in class_getName or class_getNameRaw depending
 // on which is present.
 static const char *g_shared_cache_class_name_funcptr = R"(
@@ -176,8 +242,7 @@ extern "C"
 
 static const char *g_get_shared_cache_class_info_name =
     "__lldb_apple_objc_v2_get_shared_cache_class_info";
-// Testing using the new C++11 raw string literals. If this breaks GCC then we
-// will need to revert to the code above...
+
 static const char *g_get_shared_cache_class_info_body = R"(
 
 extern "C"
@@ -297,7 +362,7 @@ __lldb_apple_objc_v2_get_shared_cache_class_info (void *objc_opt_ro_ptr,
                     DEBUG_PRINTF("clsOffset == invalidEntryOffset\n");
                     continue; // invalid offset
                 }
-                
+
                 if (class_infos && idx < max_class_infos)
                 {
                     class_infos[idx].isa = (Class)((uint8_t *)clsopt + clsOffset);
@@ -327,7 +392,7 @@ __lldb_apple_objc_v2_get_shared_cache_class_info (void *objc_opt_ro_ptr,
                 }
                 ++idx;
             }
-            
+
             const uint32_t *duplicate_count_ptr = (uint32_t *)&classOffsets[clsopt->capacity];
             const uint32_t duplicate_count = *duplicate_count_ptr;
             const objc_classheader_t *duplicateClassOffsets = (const objc_classheader_t *)(&duplicate_count_ptr[1]);
@@ -340,7 +405,7 @@ __lldb_apple_objc_v2_get_shared_cache_class_info (void *objc_opt_ro_ptr,
                     continue; // duplicate
                 else if (clsOffset == invalidEntryOffset)
                     continue; // invalid offset
-                
+
                 if (class_infos && idx < max_class_infos)
                 {
                     class_infos[idx].isa = (Class)((uint8_t *)clsopt + clsOffset);
@@ -359,7 +424,7 @@ __lldb_apple_objc_v2_get_shared_cache_class_info (void *objc_opt_ro_ptr,
                         {
                             h = 0;
                             break;
-                        } 
+                        }
                         h = ((h << 5) + h) + c;
                     }
                     class_infos[idx].hash = h;
@@ -401,8 +466,7 @@ ExtractRuntimeGlobalSymbol(Process *process, ConstString name,
       if (read_value)
         return process->ReadUnsignedIntegerFromMemory(
             symbol_load_addr, byte_size, default_value, error);
-      else
-        return symbol_load_addr;
+      return symbol_load_addr;
     } else {
       error.SetErrorString("symbol address invalid");
       return default_value;
@@ -417,24 +481,23 @@ static void RegisterObjCExceptionRecognizer(Process *process);
 
 AppleObjCRuntimeV2::AppleObjCRuntimeV2(Process *process,
                                        const ModuleSP &objc_module_sp)
-    : AppleObjCRuntime(process), m_get_class_info_code(),
-      m_get_class_info_args(LLDB_INVALID_ADDRESS),
-      m_get_class_info_args_mutex(), m_get_shared_cache_class_info_code(),
-      m_get_shared_cache_class_info_args(LLDB_INVALID_ADDRESS),
-      m_get_shared_cache_class_info_args_mutex(), m_decl_vendor_up(),
+    : AppleObjCRuntime(process), m_objc_module_sp(objc_module_sp),
+      m_dynamic_class_info_extractor(*this),
+      m_shared_cache_class_info_extractor(*this), m_decl_vendor_up(),
       m_tagged_pointer_obfuscator(LLDB_INVALID_ADDRESS),
       m_isa_hash_table_ptr(LLDB_INVALID_ADDRESS), m_hash_signature(),
-      m_has_object_getClass(false), m_loaded_objc_opt(false),
-      m_non_pointer_isa_cache_up(
-          NonPointerISACache::CreateInstance(*this, objc_module_sp)),
+      m_has_object_getClass(false), m_has_objc_copyRealizedClassList(false),
+      m_loaded_objc_opt(false), m_non_pointer_isa_cache_up(),
       m_tagged_pointer_vendor_up(
           TaggedPointerVendorV2::CreateInstance(*this, objc_module_sp)),
       m_encoding_to_type_sp(), m_noclasses_warning_emitted(false),
-      m_CFBoolean_values() {
+      m_CFBoolean_values(), m_realized_class_generation_count(0) {
   static const ConstString g_gdb_object_getClass("gdb_object_getClass");
-  m_has_object_getClass =
-      (objc_module_sp->FindFirstSymbolWithNameAndType(
-           g_gdb_object_getClass, eSymbolTypeCode) != nullptr);
+  m_has_object_getClass = HasSymbol(g_gdb_object_getClass);
+  static const ConstString g_objc_copyRealizedClassList(
+      "objc_copyRealizedClassList");
+  m_has_objc_copyRealizedClassList = HasSymbol(g_objc_copyRealizedClassList);
+
   RegisterObjCExceptionRecognizer(process);
 }
 
@@ -504,15 +567,21 @@ LanguageRuntime *AppleObjCRuntimeV2::CreateInstance(Process *process,
     if (AppleObjCRuntime::GetObjCVersion(process, objc_module_sp) ==
         ObjCRuntimeVersions::eAppleObjC_V2)
       return new AppleObjCRuntimeV2(process, objc_module_sp);
-    else
-      return nullptr;
+    return nullptr;
   } else
     return nullptr;
 }
 
 static constexpr OptionDefinition g_objc_classtable_dump_options[] = {
-    {LLDB_OPT_SET_ALL, false, "verbose", 'v', OptionParser::eNoArgument,
-     nullptr, {}, 0, eArgTypeNone,
+    {LLDB_OPT_SET_ALL,
+     false,
+     "verbose",
+     'v',
+     OptionParser::eNoArgument,
+     nullptr,
+     {},
+     0,
+     eArgTypeNone,
      "Print ivar and method information in detail"}};
 
 class CommandObjectObjC_ClassTable_Dump : public CommandObjectParsed {
@@ -554,12 +623,13 @@ public:
   };
 
   CommandObjectObjC_ClassTable_Dump(CommandInterpreter &interpreter)
-      : CommandObjectParsed(
-            interpreter, "dump", "Dump information on Objective-C classes "
-                                 "known to the current process.",
-            "language objc class-table dump",
-            eCommandRequiresProcess | eCommandProcessMustBeLaunched |
-                eCommandProcessMustBePaused),
+      : CommandObjectParsed(interpreter, "dump",
+                            "Dump information on Objective-C classes "
+                            "known to the current process.",
+                            "language objc class-table dump",
+                            eCommandRequiresProcess |
+                                eCommandProcessMustBeLaunched |
+                                eCommandProcessMustBePaused),
         m_options() {
     CommandArgumentEntry arg;
     CommandArgumentData index_arg;
@@ -638,6 +708,7 @@ protected:
                   ivar.m_type.GetDisplayTypeName().AsCString("<unknown>"),
                   ivar.m_size, ivar.m_offset);
             }
+
             iterator->second->Describe(
                 nullptr,
                 [&std_out](const char *name, const char *type) -> bool {
@@ -661,11 +732,10 @@ protected:
       }
       result.SetStatus(lldb::eReturnStatusSuccessFinishResult);
       return true;
-    } else {
-      result.AppendError("current process has no Objective-C runtime loaded");
-      result.SetStatus(lldb::eReturnStatusFailed);
-      return false;
     }
+    result.AppendError("current process has no Objective-C runtime loaded");
+    result.SetStatus(lldb::eReturnStatusFailed);
+    return false;
   }
 
   CommandOptions m_options;
@@ -747,11 +817,10 @@ protected:
       }
       result.SetStatus(lldb::eReturnStatusSuccessFinishResult);
       return true;
-    } else {
-      result.AppendError("current process has no Objective-C runtime loaded");
-      result.SetStatus(lldb::eReturnStatusFailed);
-      return false;
     }
+    result.AppendError("current process has no Objective-C runtime loaded");
+    result.SetStatus(lldb::eReturnStatusFailed);
+    return false;
   }
 };
 
@@ -1177,8 +1246,8 @@ bool AppleObjCRuntimeV2::HashTableSignature::NeedsUpdate(
 ObjCLanguageRuntime::ClassDescriptorSP
 AppleObjCRuntimeV2::GetClassDescriptorFromISA(ObjCISA isa) {
   ObjCLanguageRuntime::ClassDescriptorSP class_descriptor_sp;
-  if (m_non_pointer_isa_cache_up)
-    class_descriptor_sp = m_non_pointer_isa_cache_up->GetClassDescriptor(isa);
+  if (auto *non_pointer_isa_cache = GetNonPointerIsaCache())
+    class_descriptor_sp = non_pointer_isa_cache->GetClassDescriptor(isa);
   if (!class_descriptor_sp)
     class_descriptor_sp = ObjCLanguageRuntime::GetClassDescriptorFromISA(isa);
   return class_descriptor_sp;
@@ -1220,11 +1289,9 @@ AppleObjCRuntimeV2::GetClassDescriptor(ValueObject &valobj) {
 
   objc_class_sp = GetClassDescriptorFromISA(isa);
   if (isa && !objc_class_sp) {
-    Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_PROCESS |
-                                      LIBLLDB_LOG_TYPES));
+    Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_PROCESS | LIBLLDB_LOG_TYPES));
     LLDB_LOGF(log,
-              "0x%" PRIx64
-              ": AppleObjCRuntimeV2::GetClassDescriptor() ISA was "
+              "0x%" PRIx64 ": AppleObjCRuntimeV2::GetClassDescriptor() ISA was "
               "not in class descriptor cache 0x%" PRIx64,
               isa_pointer, isa);
   }
@@ -1235,28 +1302,29 @@ lldb::addr_t AppleObjCRuntimeV2::GetTaggedPointerObfuscator() {
   if (m_tagged_pointer_obfuscator != LLDB_INVALID_ADDRESS)
     return m_tagged_pointer_obfuscator;
 
-
   Process *process = GetProcess();
   ModuleSP objc_module_sp(GetObjCModule());
 
   if (!objc_module_sp)
     return LLDB_INVALID_ADDRESS;
 
-  static ConstString g_gdb_objc_obfuscator("objc_debug_taggedpointer_obfuscator");
+  static ConstString g_gdb_objc_obfuscator(
+      "objc_debug_taggedpointer_obfuscator");
 
   const Symbol *symbol = objc_module_sp->FindFirstSymbolWithNameAndType(
-  g_gdb_objc_obfuscator, lldb::eSymbolTypeAny);
+      g_gdb_objc_obfuscator, lldb::eSymbolTypeAny);
   if (symbol) {
     lldb::addr_t g_gdb_obj_obfuscator_ptr =
-      symbol->GetLoadAddress(&process->GetTarget());
+        symbol->GetLoadAddress(&process->GetTarget());
 
     if (g_gdb_obj_obfuscator_ptr != LLDB_INVALID_ADDRESS) {
       Status error;
-      m_tagged_pointer_obfuscator = process->ReadPointerFromMemory(
-        g_gdb_obj_obfuscator_ptr, error);
+      m_tagged_pointer_obfuscator =
+          process->ReadPointerFromMemory(g_gdb_obj_obfuscator_ptr, error);
     }
   }
-  // If we don't have a correct value at this point, there must be no obfuscation.
+  // If we don't have a correct value at this point, there must be no
+  // obfuscation.
   if (m_tagged_pointer_obfuscator == LLDB_INVALID_ADDRESS)
     m_tagged_pointer_obfuscator = 0;
 
@@ -1290,6 +1358,198 @@ lldb::addr_t AppleObjCRuntimeV2::GetISAHashTablePointer() {
   return m_isa_hash_table_ptr;
 }
 
+std::unique_ptr<UtilityFunction>
+AppleObjCRuntimeV2::DynamicClassInfoExtractor::GetClassInfoUtilityFunctionImpl(
+    ExecutionContext &exe_ctx, std::string code, std::string name) {
+  Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_PROCESS | LIBLLDB_LOG_TYPES));
+
+  LLDB_LOG(log, "Creating utility function {0}", name);
+
+  TypeSystemClang *ast =
+      ScratchTypeSystemClang::GetForTarget(exe_ctx.GetTargetRef());
+  if (!ast)
+    return {};
+
+  auto utility_fn_or_error = exe_ctx.GetTargetRef().CreateUtilityFunction(
+      std::move(code), std::move(name), eLanguageTypeC, exe_ctx);
+  if (!utility_fn_or_error) {
+    LLDB_LOG_ERROR(
+        log, utility_fn_or_error.takeError(),
+        "Failed to get utility function for implementation lookup: {0}");
+    return {};
+  }
+
+  // Make some types for our arguments.
+  CompilerType clang_uint32_t_type =
+      ast->GetBuiltinTypeForEncodingAndBitSize(eEncodingUint, 32);
+  CompilerType clang_void_pointer_type =
+      ast->GetBasicType(eBasicTypeVoid).GetPointerType();
+
+  // Make the runner function for our implementation utility function.
+  ValueList arguments;
+  Value value;
+  value.SetValueType(Value::ValueType::Scalar);
+  value.SetCompilerType(clang_void_pointer_type);
+  arguments.PushValue(value);
+  arguments.PushValue(value);
+  value.SetValueType(Value::ValueType::Scalar);
+  value.SetCompilerType(clang_uint32_t_type);
+  arguments.PushValue(value);
+  arguments.PushValue(value);
+
+  std::unique_ptr<UtilityFunction> utility_fn = std::move(*utility_fn_or_error);
+
+  Status error;
+  utility_fn->MakeFunctionCaller(clang_uint32_t_type, arguments,
+                                 exe_ctx.GetThreadSP(), error);
+
+  if (error.Fail()) {
+    LLDB_LOG(log,
+             "Failed to make function caller for implementation lookup: {0}.",
+             error.AsCString());
+    return {};
+  }
+
+  return utility_fn;
+}
+
+UtilityFunction *
+AppleObjCRuntimeV2::DynamicClassInfoExtractor::GetClassInfoUtilityFunction(
+    ExecutionContext &exe_ctx, Helper helper) {
+  switch (helper) {
+  case gdb_objc_realized_classes: {
+    if (!m_gdb_objc_realized_classes_helper.utility_function)
+      m_gdb_objc_realized_classes_helper.utility_function =
+          GetClassInfoUtilityFunctionImpl(exe_ctx,
+                                          g_get_dynamic_class_info_body,
+                                          g_get_dynamic_class_info_name);
+    return m_gdb_objc_realized_classes_helper.utility_function.get();
+  }
+  case objc_copyRealizedClassList: {
+    if (!m_objc_copyRealizedClassList_helper.utility_function)
+      m_objc_copyRealizedClassList_helper.utility_function =
+          GetClassInfoUtilityFunctionImpl(exe_ctx,
+                                          g_get_dynamic_class_info2_body,
+                                          g_get_dynamic_class_info2_name);
+    return m_objc_copyRealizedClassList_helper.utility_function.get();
+  }
+  };
+}
+
+lldb::addr_t &
+AppleObjCRuntimeV2::DynamicClassInfoExtractor::GetClassInfoArgs(Helper helper) {
+  switch (helper) {
+  case gdb_objc_realized_classes:
+    return m_gdb_objc_realized_classes_helper.args;
+  case objc_copyRealizedClassList:
+    return m_objc_copyRealizedClassList_helper.args;
+  }
+}
+
+AppleObjCRuntimeV2::DynamicClassInfoExtractor::Helper
+AppleObjCRuntimeV2::DynamicClassInfoExtractor::ComputeHelper() const {
+  if (!m_runtime.m_has_objc_copyRealizedClassList)
+    return DynamicClassInfoExtractor::gdb_objc_realized_classes;
+
+  if (Process *process = m_runtime.GetProcess()) {
+    if (DynamicLoader *loader = process->GetDynamicLoader()) {
+      if (loader->IsFullyInitialized())
+        return DynamicClassInfoExtractor::objc_copyRealizedClassList;
+    }
+  }
+
+  return DynamicClassInfoExtractor::gdb_objc_realized_classes;
+}
+
+std::unique_ptr<UtilityFunction>
+AppleObjCRuntimeV2::SharedCacheClassInfoExtractor::
+    GetClassInfoUtilityFunctionImpl(ExecutionContext &exe_ctx) {
+  Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_PROCESS | LIBLLDB_LOG_TYPES));
+
+  LLDB_LOG(log, "Creating utility function {0}",
+           g_get_shared_cache_class_info_name);
+
+  TypeSystemClang *ast =
+      ScratchTypeSystemClang::GetForTarget(exe_ctx.GetTargetRef());
+  if (!ast)
+    return {};
+
+  // If the inferior objc.dylib has the class_getNameRaw function, use that in
+  // our jitted expression.  Else fall back to the old class_getName.
+  static ConstString g_class_getName_symbol_name("class_getName");
+  static ConstString g_class_getNameRaw_symbol_name(
+      "objc_debug_class_getNameRaw");
+
+  ConstString class_name_getter_function_name =
+      m_runtime.HasSymbol(g_class_getNameRaw_symbol_name)
+          ? g_class_getNameRaw_symbol_name
+          : g_class_getName_symbol_name;
+
+  // Substitute in the correct class_getName / class_getNameRaw function name,
+  // concatenate the two parts of our expression text.  The format string has
+  // two %s's, so provide the name twice.
+  std::string shared_class_expression;
+  llvm::raw_string_ostream(shared_class_expression)
+      << llvm::format(g_shared_cache_class_name_funcptr,
+                      class_name_getter_function_name.AsCString(),
+                      class_name_getter_function_name.AsCString());
+
+  shared_class_expression += g_get_shared_cache_class_info_body;
+
+  auto utility_fn_or_error = exe_ctx.GetTargetRef().CreateUtilityFunction(
+      std::move(shared_class_expression), g_get_shared_cache_class_info_name,
+      eLanguageTypeC, exe_ctx);
+
+  if (!utility_fn_or_error) {
+    LLDB_LOG_ERROR(
+        log, utility_fn_or_error.takeError(),
+        "Failed to get utility function for implementation lookup: {0}");
+    return nullptr;
+  }
+
+  // Make some types for our arguments.
+  CompilerType clang_uint32_t_type =
+      ast->GetBuiltinTypeForEncodingAndBitSize(eEncodingUint, 32);
+  CompilerType clang_void_pointer_type =
+      ast->GetBasicType(eBasicTypeVoid).GetPointerType();
+
+  // Next make the function caller for our implementation utility function.
+  ValueList arguments;
+  Value value;
+  value.SetValueType(Value::ValueType::Scalar);
+  value.SetCompilerType(clang_void_pointer_type);
+  arguments.PushValue(value);
+  arguments.PushValue(value);
+
+  value.SetValueType(Value::ValueType::Scalar);
+  value.SetCompilerType(clang_uint32_t_type);
+  arguments.PushValue(value);
+  arguments.PushValue(value);
+
+  std::unique_ptr<UtilityFunction> utility_fn = std::move(*utility_fn_or_error);
+
+  Status error;
+  utility_fn->MakeFunctionCaller(clang_uint32_t_type, arguments,
+                                 exe_ctx.GetThreadSP(), error);
+
+  if (error.Fail()) {
+    LLDB_LOG(log,
+             "Failed to make function caller for implementation lookup: {0}.",
+             error.AsCString());
+    return {};
+  }
+
+  return utility_fn;
+}
+
+UtilityFunction *
+AppleObjCRuntimeV2::SharedCacheClassInfoExtractor::GetClassInfoUtilityFunction(
+    ExecutionContext &exe_ctx) {
+  if (!m_utility_function)
+    m_utility_function = GetClassInfoUtilityFunctionImpl(exe_ctx);
+  return m_utility_function.get();
+}
+
 AppleObjCRuntimeV2::DescriptorMapUpdateResult
 AppleObjCRuntimeV2::UpdateISAToDescriptorMapDynamic(
     RemoteNXMapTable &hash_table) {
@@ -1322,65 +1582,38 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapDynamic(
 
   Status err;
 
+  // Compute which helper we're going to use for this update.
+  const DynamicClassInfoExtractor::Helper helper =
+      m_dynamic_class_info_extractor.ComputeHelper();
+
   // Read the total number of classes from the hash table
-  const uint32_t num_classes = hash_table.GetCount();
+  const uint32_t num_classes =
+      helper == DynamicClassInfoExtractor::gdb_objc_realized_classes
+          ? hash_table.GetCount()
+          : m_realized_class_generation_count;
   if (num_classes == 0) {
-    LLDB_LOGF(log, "No dynamic classes found in gdb_objc_realized_classes.");
+    LLDB_LOGF(log, "No dynamic classes found.");
     return DescriptorMapUpdateResult::Success(0);
   }
 
-  // Make some types for our arguments
-  CompilerType clang_uint32_t_type =
-      ast->GetBuiltinTypeForEncodingAndBitSize(eEncodingUint, 32);
-  CompilerType clang_void_pointer_type =
-      ast->GetBasicType(eBasicTypeVoid).GetPointerType();
-
-  ValueList arguments;
-  FunctionCaller *get_class_info_function = nullptr;
-
-  if (!m_get_class_info_code) {
-    auto utility_fn_or_error = GetTargetRef().CreateUtilityFunction(
-        g_get_dynamic_class_info_body, g_get_dynamic_class_info_name,
-        eLanguageTypeC, exe_ctx);
-    if (!utility_fn_or_error) {
-      LLDB_LOG_ERROR(
-          log, utility_fn_or_error.takeError(),
-          "Failed to get utility function for implementation lookup: {0}");
-      return DescriptorMapUpdateResult::Fail();
-    }
-    m_get_class_info_code = std::move(*utility_fn_or_error);
-
-    // Next make the runner function for our implementation utility function.
-    Value value;
-    value.SetValueType(Value::ValueType::Scalar);
-    value.SetCompilerType(clang_void_pointer_type);
-    arguments.PushValue(value);
-    arguments.PushValue(value);
-
-    value.SetValueType(Value::ValueType::Scalar);
-    value.SetCompilerType(clang_uint32_t_type);
-    arguments.PushValue(value);
-    arguments.PushValue(value);
+  UtilityFunction *get_class_info_code =
+      m_dynamic_class_info_extractor.GetClassInfoUtilityFunction(exe_ctx,
+                                                                 helper);
+  if (!get_class_info_code) {
+    // The callee will have already logged a useful error message.
+    return DescriptorMapUpdateResult::Fail();
+  }
 
-    Status error;
-    get_class_info_function = m_get_class_info_code->MakeFunctionCaller(
-        clang_uint32_t_type, arguments, thread_sp, error);
+  FunctionCaller *get_class_info_function =
+      get_class_info_code->GetFunctionCaller();
 
-    if (error.Fail()) {
-      LLDB_LOGF(log,
-                "Failed to make function caller for implementation lookup: %s.",
-                error.AsCString());
-      return DescriptorMapUpdateResult::Fail();
-    }
-  } else {
-    get_class_info_function = m_get_class_info_code->GetFunctionCaller();
-    if (!get_class_info_function) {
-      LLDB_LOGF(log, "Failed to get implementation lookup function caller.");
-      return DescriptorMapUpdateResult::Fail();
-    }
-    arguments = get_class_info_function->GetArgumentValues();
+  if (!get_class_info_function) {
+    LLDB_LOGF(log, "Failed to get implementation lookup function caller.");
+    return DescriptorMapUpdateResult::Fail();
   }
 
+  ValueList arguments = get_class_info_function->GetArgumentValues();
+
   DiagnosticManager diagnostics;
 
   const uint32_t class_info_byte_size = addr_size + 4;
@@ -1396,18 +1629,18 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapDynamic(
     return DescriptorMapUpdateResult::Fail();
   }
 
-  std::lock_guard<std::mutex> guard(m_get_class_info_args_mutex);
+  std::lock_guard<std::mutex> guard(m_dynamic_class_info_extractor.GetMutex());
 
   // Fill in our function argument values
   arguments.GetValueAtIndex(0)->GetScalar() = hash_table.GetTableLoadAddress();
   arguments.GetValueAtIndex(1)->GetScalar() = class_infos_addr;
   arguments.GetValueAtIndex(2)->GetScalar() = class_infos_byte_size;
-  
+
   // Only dump the runtime classes from the expression evaluation if the log is
   // verbose:
   Log *type_log = GetLogIfAllCategoriesSet(LIBLLDB_LOG_TYPES);
   bool dump_log = type_log && type_log->GetVerbose();
-  
+
   arguments.GetValueAtIndex(3)->GetScalar() = dump_log ? 1 : 0;
 
   bool success = false;
@@ -1416,7 +1649,8 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapDynamic(
 
   // Write our function arguments into the process so we can run our function
   if (get_class_info_function->WriteFunctionArguments(
-          exe_ctx, m_get_class_info_args, arguments, diagnostics)) {
+          exe_ctx, m_dynamic_class_info_extractor.GetClassInfoArgs(helper),
+          arguments, diagnostics)) {
     EvaluateExpressionOptions options;
     options.SetUnwindOnError(true);
     options.SetTryAllThreads(false);
@@ -1425,6 +1659,9 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapDynamic(
     options.SetTimeout(process->GetUtilityExpressionTimeout());
     options.SetIsForUtilityExpr(true);
 
+    CompilerType clang_uint32_t_type =
+        ast->GetBuiltinTypeForEncodingAndBitSize(eEncodingUint, 32);
+
     Value return_value;
     return_value.SetValueType(Value::ValueType::Scalar);
     return_value.SetCompilerType(clang_uint32_t_type);
@@ -1434,12 +1671,13 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapDynamic(
 
     // Run the function
     ExpressionResults results = get_class_info_function->ExecuteFunction(
-        exe_ctx, &m_get_class_info_args, options, diagnostics, return_value);
+        exe_ctx, &m_dynamic_class_info_extractor.GetClassInfoArgs(helper),
+        options, diagnostics, return_value);
 
     if (results == eExpressionCompleted) {
       // The result is the number of ClassInfo structures that were filled in
       num_class_infos = return_value.GetScalar().ULong();
-      LLDB_LOGF(log, "Discovered %u ObjC classes\n", num_class_infos);
+      LLDB_LOG(log, "Discovered {0} Objective-C classes", num_class_infos);
       if (num_class_infos > 0) {
         // Read the ClassInfo structures
         DataBufferHeap buffer(num_class_infos * class_info_byte_size, 0);
@@ -1513,15 +1751,16 @@ uint32_t AppleObjCRuntimeV2::ParseClassInfoArray(const DataExtractor &data,
       ClassDescriptorSP descriptor_sp(
           new ClassDescriptorV2(*this, isa, nullptr));
 
-      // The code in g_get_shared_cache_class_info_body sets the value of the hash
-      // to 0 to signal a demangled symbol. We use class_getName() in that code to
-      // find the class name, but this returns a demangled name for Swift symbols.
-      // For those symbols, recompute the hash here by reading their name from the
-      // runtime.
+      // The code in g_get_shared_cache_class_info_body sets the value of the
+      // hash to 0 to signal a demangled symbol. We use class_getName() in that
+      // code to find the class name, but this returns a demangled name for
+      // Swift symbols. For those symbols, recompute the hash here by reading
+      // their name from the runtime.
       if (name_hash)
         AddClass(isa, descriptor_sp, name_hash);
       else
-        AddClass(isa, descriptor_sp, descriptor_sp->GetClassName().AsCString(nullptr));
+        AddClass(isa, descriptor_sp,
+                 descriptor_sp->GetClassName().AsCString(nullptr));
       num_parsed++;
       if (should_log)
         LLDB_LOGF(log,
@@ -1537,6 +1776,17 @@ uint32_t AppleObjCRuntimeV2::ParseClassInfoArray(const DataExtractor &data,
   return num_parsed;
 }
 
+bool AppleObjCRuntimeV2::HasSymbol(ConstString Name) {
+  if (!m_objc_module_sp)
+    return false;
+  if (const Symbol *symbol = m_objc_module_sp->FindFirstSymbolWithNameAndType(
+          Name, lldb::eSymbolTypeCode)) {
+    if (symbol->ValueIsAddress() || symbol->GetAddressRef().IsValid())
+      return true;
+  }
+  return false;
+}
+
 AppleObjCRuntimeV2::DescriptorMapUpdateResult
 AppleObjCRuntimeV2::UpdateISAToDescriptorMapSharedCache() {
   Process *process = GetProcess();
@@ -1575,90 +1825,19 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapSharedCache() {
 
   const uint32_t num_classes = 128 * 1024;
 
-  // Make some types for our arguments
-  CompilerType clang_uint32_t_type =
-      ast->GetBuiltinTypeForEncodingAndBitSize(eEncodingUint, 32);
-  CompilerType clang_void_pointer_type =
-      ast->GetBasicType(eBasicTypeVoid).GetPointerType();
-
-  ValueList arguments;
-  FunctionCaller *get_shared_cache_class_info_function = nullptr;
-
-  if (!m_get_shared_cache_class_info_code) {
-    Status error;
-
-    // If the inferior objc.dylib has the class_getNameRaw function,
-    // use that in our jitted expression.  Else fall back to the old
-    // class_getName.
-    static ConstString g_class_getName_symbol_name("class_getName");
-    static ConstString g_class_getNameRaw_symbol_name("objc_debug_class_getNameRaw");
-    ConstString class_name_getter_function_name = g_class_getName_symbol_name;
-
-    ObjCLanguageRuntime *objc_runtime = ObjCLanguageRuntime::Get(*process);
-    if (objc_runtime) {
-      for (lldb::ModuleSP mod_sp : process->GetTarget().GetImages().Modules()) {
-        if (objc_runtime->IsModuleObjCLibrary(mod_sp)) {
-          const Symbol *symbol =
-              mod_sp->FindFirstSymbolWithNameAndType(g_class_getNameRaw_symbol_name, 
-                                                lldb::eSymbolTypeCode);
-          if (symbol && 
-              (symbol->ValueIsAddress() || symbol->GetAddressRef().IsValid())) {
-            class_name_getter_function_name = g_class_getNameRaw_symbol_name;
-          }
-        }
-      }
-    }
-
-    // Substitute in the correct class_getName / class_getNameRaw function name,
-    // concatenate the two parts of our expression text.  The format string
-    // has two %s's, so provide the name twice.
-    std::string shared_class_expression;
-    llvm::raw_string_ostream(shared_class_expression) << llvm::format(
-                               g_shared_cache_class_name_funcptr,
-                               class_name_getter_function_name.AsCString(),
-                               class_name_getter_function_name.AsCString());
-
-    shared_class_expression += g_get_shared_cache_class_info_body;
-
-    auto utility_fn_or_error = exe_ctx.GetTargetRef().CreateUtilityFunction(
-        std::move(shared_class_expression), g_get_shared_cache_class_info_name,
-        eLanguageTypeC, exe_ctx);
-    if (!utility_fn_or_error) {
-      LLDB_LOG_ERROR(
-          log, utility_fn_or_error.takeError(),
-          "Failed to get utility function for implementation lookup: {0}");
-      return DescriptorMapUpdateResult::Fail();
-    }
-
-    m_get_shared_cache_class_info_code = std::move(*utility_fn_or_error);
-
-    // Next make the function caller for our implementation utility function.
-    Value value;
-    value.SetValueType(Value::ValueType::Scalar);
-    value.SetCompilerType(clang_void_pointer_type);
-    arguments.PushValue(value);
-    arguments.PushValue(value);
-
-    value.SetValueType(Value::ValueType::Scalar);
-    value.SetCompilerType(clang_uint32_t_type);
-    arguments.PushValue(value);
-    arguments.PushValue(value);
+  UtilityFunction *get_class_info_code =
+      m_shared_cache_class_info_extractor.GetClassInfoUtilityFunction(exe_ctx);
+  FunctionCaller *get_shared_cache_class_info_function =
+      get_class_info_code->GetFunctionCaller();
 
-    get_shared_cache_class_info_function =
-        m_get_shared_cache_class_info_code->MakeFunctionCaller(
-            clang_uint32_t_type, arguments, thread_sp, error);
-
-    if (get_shared_cache_class_info_function == nullptr)
-      return DescriptorMapUpdateResult::Fail();
-
-  } else {
-    get_shared_cache_class_info_function =
-        m_get_shared_cache_class_info_code->GetFunctionCaller();
-    if (get_shared_cache_class_info_function == nullptr)
-      return DescriptorMapUpdateResult::Fail();
-    arguments = get_shared_cache_class_info_function->GetArgumentValues();
+  if (!get_shared_cache_class_info_function) {
+    LLDB_LOGF(log, "Failed to get implementation lookup function caller.");
+    return DescriptorMapUpdateResult::Fail();
   }
 
+  ValueList arguments =
+      get_shared_cache_class_info_function->GetArgumentValues();
+
   DiagnosticManager diagnostics;
 
   const uint32_t class_info_byte_size = addr_size + 4;
@@ -1674,7 +1853,8 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapSharedCache() {
     return DescriptorMapUpdateResult::Fail();
   }
 
-  std::lock_guard<std::mutex> guard(m_get_shared_cache_class_info_args_mutex);
+  std::lock_guard<std::mutex> guard(
+      m_shared_cache_class_info_extractor.GetMutex());
 
   // Fill in our function argument values
   arguments.GetValueAtIndex(0)->GetScalar() = objc_opt_ptr;
@@ -1684,7 +1864,7 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapSharedCache() {
   // verbose:
   Log *type_log = GetLogIfAllCategoriesSet(LIBLLDB_LOG_TYPES);
   bool dump_log = type_log && type_log->GetVerbose();
-  
+
   arguments.GetValueAtIndex(3)->GetScalar() = dump_log ? 1 : 0;
 
   bool success = false;
@@ -1693,8 +1873,8 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapSharedCache() {
 
   // Write our function arguments into the process so we can run our function
   if (get_shared_cache_class_info_function->WriteFunctionArguments(
-          exe_ctx, m_get_shared_cache_class_info_args, arguments,
-          diagnostics)) {
+          exe_ctx, m_shared_cache_class_info_extractor.GetClassInfoArgs(),
+          arguments, diagnostics)) {
     EvaluateExpressionOptions options;
     options.SetUnwindOnError(true);
     options.SetTryAllThreads(false);
@@ -1703,6 +1883,9 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapSharedCache() {
     options.SetTimeout(process->GetUtilityExpressionTimeout());
     options.SetIsForUtilityExpr(true);
 
+    CompilerType clang_uint32_t_type =
+        ast->GetBuiltinTypeForEncodingAndBitSize(eEncodingUint, 32);
+
     Value return_value;
     return_value.SetValueType(Value::ValueType::Scalar);
     return_value.SetCompilerType(clang_uint32_t_type);
@@ -1713,14 +1896,14 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapSharedCache() {
     // Run the function
     ExpressionResults results =
         get_shared_cache_class_info_function->ExecuteFunction(
-            exe_ctx, &m_get_shared_cache_class_info_args, options, diagnostics,
-            return_value);
+            exe_ctx, &m_shared_cache_class_info_extractor.GetClassInfoArgs(),
+            options, diagnostics, return_value);
 
     if (results == eExpressionCompleted) {
       // The result is the number of ClassInfo structures that were filled in
       num_class_infos = return_value.GetScalar().ULong();
-      LLDB_LOGF(log, "Discovered %u ObjC classes in shared cache\n",
-                num_class_infos);
+      LLDB_LOG(log, "Discovered {0} Objective-C classes in the shared cache",
+               num_class_infos);
       assert(num_class_infos <= num_classes);
       if (num_class_infos > 0) {
         if (num_class_infos > num_classes) {
@@ -1764,42 +1947,6 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapSharedCache() {
   return DescriptorMapUpdateResult(success, num_class_infos);
 }
 
-bool AppleObjCRuntimeV2::UpdateISAToDescriptorMapFromMemory(
-    RemoteNXMapTable &hash_table) {
-  Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_PROCESS | LIBLLDB_LOG_TYPES));
-
-  Process *process = GetProcess();
-
-  if (process == nullptr)
-    return false;
-
-  uint32_t num_map_table_isas = 0;
-
-  ModuleSP objc_module_sp(GetObjCModule());
-
-  if (objc_module_sp) {
-    for (RemoteNXMapTable::element elt : hash_table) {
-      ++num_map_table_isas;
-
-      if (ISAIsCached(elt.second))
-        continue;
-
-      ClassDescriptorSP descriptor_sp = ClassDescriptorSP(
-          new ClassDescriptorV2(*this, elt.second, elt.first.AsCString()));
-
-      if (log && log->GetVerbose())
-        LLDB_LOGF(log,
-                  "AppleObjCRuntimeV2 added (ObjCISA)0x%" PRIx64
-                  " (%s) from dynamic table to isa->descriptor cache",
-                  elt.second, elt.first.AsCString());
-
-      AddClass(elt.second, descriptor_sp, elt.first.AsCString());
-    }
-  }
-
-  return num_map_table_isas > 0;
-}
-
 lldb::addr_t AppleObjCRuntimeV2::GetSharedCacheReadOnlyAddress() {
   Process *process = GetProcess();
 
@@ -1848,12 +1995,17 @@ void AppleObjCRuntimeV2::UpdateISAToDescriptorMapIfNeeded() {
     // map, whether it was successful or not.
     m_isa_to_descriptor_stop_id = process->GetStopID();
 
-    if (!m_hash_signature.NeedsUpdate(process, this, hash_table))
+    // Ask the runtime is the realized class generation count changed. Unlike
+    // the hash table, this accounts for lazily named classes.
+    const bool class_count_changed = RealizedClassGenerationCountChanged();
+
+    if (!m_hash_signature.NeedsUpdate(process, this, hash_table) &&
+        !class_count_changed)
       return;
 
     m_hash_signature.UpdateSignature(hash_table);
 
-    // Grab the dynamically loaded objc classes from the hash table in memory
+    // Grab the dynamically loaded Objective-C classes from memory.
     DescriptorMapUpdateResult dynamic_update_result =
         UpdateISAToDescriptorMapDynamic(hash_table);
 
@@ -1901,6 +2053,35 @@ void AppleObjCRuntimeV2::UpdateISAToDescriptorMapIfNeeded() {
   }
 }
 
+bool AppleObjCRuntimeV2::RealizedClassGenerationCountChanged() {
+  Process *process = GetProcess();
+  if (!process)
+    return false;
+
+  Status error;
+  uint64_t objc_debug_realized_class_generation_count =
+      ExtractRuntimeGlobalSymbol(
+          process, ConstString("objc_debug_realized_class_generation_count"),
+          GetObjCModule(), error);
+  if (error.Fail())
+    return false;
+
+  if (m_realized_class_generation_count ==
+      objc_debug_realized_class_generation_count)
+    return false;
+
+  Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_PROCESS | LIBLLDB_LOG_TYPES));
+  LLDB_LOG(log,
+           "objc_debug_realized_class_generation_count changed from {0} to {1}",
+           m_realized_class_generation_count,
+           objc_debug_realized_class_generation_count);
+
+  m_realized_class_generation_count =
+      objc_debug_realized_class_generation_count;
+
+  return true;
+}
+
 static bool DoesProcessHaveSharedCache(Process &process) {
   PlatformSP platform_sp = process.GetTarget().GetPlatform();
   if (!platform_sp)
@@ -1981,9 +2162,10 @@ lldb::addr_t AppleObjCRuntimeV2::LookupRuntimeSymbol(ConstString name) {
           const ConstString ivar_name_cs(class_and_ivar.second);
           const char *ivar_name_cstr = ivar_name_cs.AsCString();
 
-          auto ivar_func = [&ret, ivar_name_cstr](
-              const char *name, const char *type, lldb::addr_t offset_addr,
-              uint64_t size) -> lldb::addr_t {
+          auto ivar_func = [&ret,
+                            ivar_name_cstr](const char *name, const char *type,
+                                            lldb::addr_t offset_addr,
+                                            uint64_t size) -> lldb::addr_t {
             if (!strcmp(name, ivar_name_cstr)) {
               ret = offset_addr;
               return true;
@@ -2401,9 +2583,9 @@ AppleObjCRuntimeV2::TaggedPointerVendorExtended::GetClassDescriptor(
     m_ext_cache[slot] = actual_class_descriptor_sp;
   }
 
-  data_payload =
-      (((uint64_t)unobfuscated << m_objc_debug_taggedpointer_ext_payload_lshift) >>
-       m_objc_debug_taggedpointer_ext_payload_rshift);
+  data_payload = (((uint64_t)unobfuscated
+                   << m_objc_debug_taggedpointer_ext_payload_lshift) >>
+                  m_objc_debug_taggedpointer_ext_payload_rshift);
 
   return ClassDescriptorSP(
       new ClassDescriptorV2Tagged(actual_class_descriptor_sp, data_payload));
@@ -2558,8 +2740,8 @@ lldb_private::AppleObjCRuntime::ObjCISA
 AppleObjCRuntimeV2::GetPointerISA(ObjCISA isa) {
   ObjCISA ret = isa;
 
-  if (m_non_pointer_isa_cache_up)
-    m_non_pointer_isa_cache_up->EvaluateNonPointerISA(isa, ret);
+  if (auto *non_pointer_isa_cache = GetNonPointerIsaCache())
+    non_pointer_isa_cache->EvaluateNonPointerISA(isa, ret);
 
   return ret;
 }
@@ -2604,13 +2786,14 @@ void AppleObjCRuntimeV2::GetValuesForGlobalCFBooleans(lldb::addr_t &cf_true,
 #pragma mark Frame recognizers
 
 class ObjCExceptionRecognizedStackFrame : public RecognizedStackFrame {
- public:
+public:
   ObjCExceptionRecognizedStackFrame(StackFrameSP frame_sp) {
     ThreadSP thread_sp = frame_sp->GetThread();
     ProcessSP process_sp = thread_sp->GetProcess();
 
     const lldb::ABISP &abi = process_sp->GetABI();
-    if (!abi) return;
+    if (!abi)
+      return;
 
     TypeSystemClang *clang_ast_context =
         ScratchTypeSystemClang::GetForTarget(process_sp->GetTarget());
@@ -2624,7 +2807,8 @@ class ObjCExceptionRecognizedStackFrame : public RecognizedStackFrame {
     input_value.SetCompilerType(voidstar);
     args.PushValue(input_value);
 
-    if (!abi->GetArgumentValues(*thread_sp, args)) return;
+    if (!abi->GetArgumentValues(*thread_sp, args))
+      return;
 
     addr_t exception_addr = args.GetValueAtIndex(0)->GetScalar().ULongLong();
 
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
index c6fb6ea26b98dc58e5fa88d6d24beedb7955aa29..b0c1584dd369217907d803eb459b68497b86c810 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
@@ -26,7 +26,6 @@ class AppleObjCRuntimeV2 : public AppleObjCRuntime {
 public:
   ~AppleObjCRuntimeV2() override = default;
 
-  // Static Functions
   static void Initialize();
 
   static void Terminate();
@@ -46,7 +45,6 @@ public:
     return runtime->isA(&ID);
   }
 
-  // These are generic runtime functions:
   bool GetDynamicTypeAndAddress(ValueObject &in_value,
                                 lldb::DynamicValueType use_dynamic,
                                 TypeAndOrName &class_type_or_name,
@@ -56,7 +54,6 @@ public:
   llvm::Expected<std::unique_ptr<UtilityFunction>>
   CreateObjectChecker(std::string name, ExecutionContext &exe_ctx) override;
 
-  // PluginInterface protocol
   ConstString GetPluginName() override;
 
   uint32_t GetPluginVersion() override;
@@ -105,8 +102,8 @@ public:
 
 protected:
   lldb::BreakpointResolverSP
-  CreateExceptionResolver(const lldb::BreakpointSP &bkpt,
-                          bool catch_bp, bool throw_bp) override;
+  CreateExceptionResolver(const lldb::BreakpointSP &bkpt, bool catch_bp,
+                          bool throw_bp) override;
 
 private:
   class HashTableSignature {
@@ -296,13 +293,93 @@ private:
     }
   };
 
+  /// Abstraction to read the Objective-C class info.
+  class ClassInfoExtractor {
+  public:
+    ClassInfoExtractor(AppleObjCRuntimeV2 &runtime) : m_runtime(runtime) {}
+    std::mutex &GetMutex() { return m_mutex; }
+
+  protected:
+    /// The lifetime of this object is tied to that of the runtime.
+    AppleObjCRuntimeV2 &m_runtime;
+    std::mutex m_mutex;
+  };
+
+  /// We can read the class info from the Objective-C runtime using
+  /// gdb_objc_realized_classes or objc_copyRealizedClassList. The latter is
+  /// preferred because it includes lazily named classes, but it's not always
+  /// available or safe to call.
+  ///
+  /// We potentially need both for the same process, because we may need to use
+  /// gdb_objc_realized_classes until dyld is initialized and then switch over
+  /// to objc_copyRealizedClassList for lazily named classes.
+  class DynamicClassInfoExtractor : public ClassInfoExtractor {
+  public:
+    DynamicClassInfoExtractor(AppleObjCRuntimeV2 &runtime)
+        : ClassInfoExtractor(runtime) {}
+
+    enum Helper { gdb_objc_realized_classes, objc_copyRealizedClassList };
+
+    /// Compute which helper to use. Prefer objc_copyRealizedClassList if it's
+    /// available and it's safe to call (i.e. dyld is fully initialized). Use
+    /// gdb_objc_realized_classes otherwise.
+    Helper ComputeHelper() const;
+
+    UtilityFunction *GetClassInfoUtilityFunction(ExecutionContext &exe_ctx,
+                                                 Helper helper);
+    lldb::addr_t &GetClassInfoArgs(Helper helper);
+
+  private:
+    std::unique_ptr<UtilityFunction>
+    GetClassInfoUtilityFunctionImpl(ExecutionContext &exe_ctx, std::string code,
+                                    std::string name);
+
+    /// Helper to read class info using the gdb_objc_realized_classes.
+    struct gdb_objc_realized_classes_helper {
+      std::unique_ptr<UtilityFunction> utility_function;
+      lldb::addr_t args = LLDB_INVALID_ADDRESS;
+    };
+
+    /// Helper to read class info using objc_copyRealizedClassList.
+    struct objc_copyRealizedClassList_helper {
+      std::unique_ptr<UtilityFunction> utility_function;
+      lldb::addr_t args = LLDB_INVALID_ADDRESS;
+    };
+
+    gdb_objc_realized_classes_helper m_gdb_objc_realized_classes_helper;
+    objc_copyRealizedClassList_helper m_objc_copyRealizedClassList_helper;
+  };
+
+  /// Abstraction to read the Objective-C class info from the shared cache.
+  class SharedCacheClassInfoExtractor : public ClassInfoExtractor {
+  public:
+    SharedCacheClassInfoExtractor(AppleObjCRuntimeV2 &runtime)
+        : ClassInfoExtractor(runtime) {}
+
+    UtilityFunction *GetClassInfoUtilityFunction(ExecutionContext &exe_ctx);
+    lldb::addr_t &GetClassInfoArgs() { return m_args; }
+    std::mutex &GetMutex() { return m_mutex; }
+
+  private:
+    std::unique_ptr<UtilityFunction>
+    GetClassInfoUtilityFunctionImpl(ExecutionContext &exe_ctx);
+
+    std::unique_ptr<UtilityFunction> m_utility_function;
+    lldb::addr_t m_args = LLDB_INVALID_ADDRESS;
+    std::mutex m_mutex;
+  };
+
   AppleObjCRuntimeV2(Process *process, const lldb::ModuleSP &objc_module_sp);
 
   ObjCISA GetPointerISA(ObjCISA isa);
 
   lldb::addr_t GetISAHashTablePointer();
 
-  bool UpdateISAToDescriptorMapFromMemory(RemoteNXMapTable &hash_table);
+  /// Update the generation count of realized classes. This is not an exact
+  /// count but rather a value that is incremented when new classes are realized
+  /// or destroyed. Unlike the count in gdb_objc_realized_classes, it will
+  /// change when lazily named classes get realized.
+  bool RealizedClassGenerationCountChanged();
 
   DescriptorMapUpdateResult
   UpdateISAToDescriptorMapDynamic(RemoteNXMapTable &hash_table);
@@ -323,27 +400,35 @@ private:
 
   bool GetCFBooleanValuesIfNeeded();
 
+  bool HasSymbol(ConstString Name);
+
+  NonPointerISACache *GetNonPointerIsaCache() {
+    if (!m_non_pointer_isa_cache_up)
+      m_non_pointer_isa_cache_up.reset(
+          NonPointerISACache::CreateInstance(*this, m_objc_module_sp));
+    return m_non_pointer_isa_cache_up.get();
+  }
+
   friend class ClassDescriptorV2;
 
-  std::unique_ptr<UtilityFunction> m_get_class_info_code;
-  lldb::addr_t m_get_class_info_args;
-  std::mutex m_get_class_info_args_mutex;
+  lldb::ModuleSP m_objc_module_sp;
 
-  std::unique_ptr<UtilityFunction> m_get_shared_cache_class_info_code;
-  lldb::addr_t m_get_shared_cache_class_info_args;
-  std::mutex m_get_shared_cache_class_info_args_mutex;
+  DynamicClassInfoExtractor m_dynamic_class_info_extractor;
+  SharedCacheClassInfoExtractor m_shared_cache_class_info_extractor;
 
   std::unique_ptr<DeclVendor> m_decl_vendor_up;
   lldb::addr_t m_tagged_pointer_obfuscator;
   lldb::addr_t m_isa_hash_table_ptr;
   HashTableSignature m_hash_signature;
   bool m_has_object_getClass;
+  bool m_has_objc_copyRealizedClassList;
   bool m_loaded_objc_opt;
   std::unique_ptr<NonPointerISACache> m_non_pointer_isa_cache_up;
   std::unique_ptr<TaggedPointerVendor> m_tagged_pointer_vendor_up;
   EncodingToTypeSP m_encoding_to_type_sp;
   bool m_noclasses_warning_emitted;
   llvm::Optional<std::pair<lldb::addr_t, lldb::addr_t>> m_CFBoolean_values;
+  uint64_t m_realized_class_generation_count;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index df058636783299c4935e101962c1c7726661bb58..ae432ac89eaacdf8fadec883acaef48c5ca8ac69 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -16,6 +16,7 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Core/Progress.h"
 #include "lldb/Core/Section.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/LZMA.h"
@@ -37,6 +38,7 @@
 #include "llvm/Object/Decompressor.h"
 #include "llvm/Support/ARMBuildAttributes.h"
 #include "llvm/Support/CRC.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/MipsABIFlags.h"
@@ -1861,7 +1863,7 @@ void ObjectFileELF::CreateSections(SectionList &unified_section_list) {
   // unified section list.
   if (GetType() != eTypeDebugInfo)
     unified_section_list = *m_sections_up;
-  
+
   // If there's a .gnu_debugdata section, we'll try to read the .symtab that's
   // embedded in there and replace the one in the original object file (if any).
   // If there's none in the orignal object file, we add it to it.
@@ -1923,7 +1925,7 @@ std::shared_ptr<ObjectFileELF> ObjectFileELF::GetGnuDebugDataObjectFile() {
   ArchSpec spec = m_gnu_debug_data_object_file->GetArchitecture();
   if (spec && m_gnu_debug_data_object_file->SetModulesArchitecture(spec))
     return m_gnu_debug_data_object_file;
-  
+
   return nullptr;
 }
 
@@ -2707,6 +2709,9 @@ Symtab *ObjectFileELF::GetSymtab() {
   if (!module_sp)
     return nullptr;
 
+  Progress progress(llvm::formatv("Parsing symbol table for {0}",
+                                  m_file.GetFilename().AsCString("<Unknown>")));
+
   // We always want to use the main object file so we (hopefully) only have one
   // cached copy of our symtab, dynamic sections, etc.
   ObjectFile *module_obj_file = module_sp->GetObjectFile();
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index 548e21ad9aea0cc65485ce8f9fb0f1ffb13474f9..b8f94129d4185c9bfae49a16d89bdaa53ab8d393 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -17,6 +17,7 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Core/Progress.h"
 #include "lldb/Core/Section.h"
 #include "lldb/Core/StreamFile.h"
 #include "lldb/Host/Host.h"
@@ -43,6 +44,7 @@
 
 #include "lldb/Host/SafeMachO.h"
 
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 #include "ObjectFileMachO.h"
@@ -2167,6 +2169,9 @@ size_t ObjectFileMachO::ParseSymtab() {
   if (!module_sp)
     return 0;
 
+  Progress progress(llvm::formatv("Parsing symbol table for {0}",
+                                  m_file.GetFilename().AsCString("<Unknown>")));
+
   struct symtab_command symtab_load_command = {0, 0, 0, 0, 0, 0};
   struct linkedit_data_command function_starts_load_command = {0, 0, 0, 0};
   struct dyld_info_command dyld_info = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
diff --git a/lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.cpp b/lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.cpp
index 35a823e9a28feb2d85ef5b2eae0c505f7b1f3ae7..cb7bbeeca054682efb91983326bce0c46aff3fd7 100644
--- a/lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.cpp
+++ b/lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.cpp
@@ -173,7 +173,7 @@ ObjectFilePDB::loadPDBFile(std::string PdbPath,
   if (ec || magic != llvm::file_magic::pdb)
     return nullptr;
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> ErrorOrBuffer =
-      llvm::MemoryBuffer::getFile(PdbPath, /*FileSize=*/-1,
+      llvm::MemoryBuffer::getFile(PdbPath, /*IsText=*/false,
                                   /*RequiresNullTerminator=*/false);
   if (!ErrorOrBuffer)
     return nullptr;
diff --git a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
index c8a006001fcbf28eb6d4a5e7983816d8f2a7ce43..3e5f1451ef5febeb9da8b38882435d9ff0e11179 100644
--- a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
+++ b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
@@ -578,7 +578,19 @@ PlatformPOSIX::MakeLoadImageUtilityFunction(ExecutionContext &exe_ctx,
   // __lldb_dlopen_result for consistency. The wrapper returns a void * but
   // doesn't use it because UtilityFunctions don't work with void returns at
   // present.
+  //
+  // Use lazy binding so as to not make dlopen()'s success conditional on
+  // forcing every symbol in the library.
+  //
+  // In general, the debugger should allow programs to load & run with
+  // libraries as far as they can, instead of defaulting to being super-picky
+  // about unavailable symbols.
+  //
+  // The value "1" appears to imply lazy binding (RTLD_LAZY) on both Darwin
+  // and other POSIX OSes.
   static const char *dlopen_wrapper_code = R"(
+  const int RTLD_LAZY = 1;
+
   struct __lldb_dlopen_result {
     void *image_ptr;
     const char *error_str;
@@ -595,7 +607,7 @@ PlatformPOSIX::MakeLoadImageUtilityFunction(ExecutionContext &exe_ctx,
   {
     // This is the case where the name is the full path:
     if (!path_strings) {
-      result_ptr->image_ptr = dlopen(name, 2);
+      result_ptr->image_ptr = dlopen(name, RTLD_LAZY);
       if (result_ptr->image_ptr)
         result_ptr->error_str = nullptr;
       return nullptr;
@@ -609,7 +621,7 @@ PlatformPOSIX::MakeLoadImageUtilityFunction(ExecutionContext &exe_ctx,
       buffer[path_len] = '/';
       char *target_ptr = buffer+path_len+1; 
       memcpy((void *) target_ptr, (void *) name, name_len + 1);
-      result_ptr->image_ptr = dlopen(buffer, 2);
+      result_ptr->image_ptr = dlopen(buffer, RTLD_LAZY);
       if (result_ptr->image_ptr) {
         result_ptr->error_str = nullptr;
         break;
diff --git a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp
index 0f771106401c1d06a6d015a1872fad174193eb66..82a946f9320690663cc42b5a7c99f8b4b18470c5 100644
--- a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp
+++ b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp
@@ -285,14 +285,16 @@ Status ProcessMachCore::DoLoadCore() {
   ObjectFile::BinaryType type;
   if (core_objfile->GetCorefileMainBinaryInfo(objfile_binary_addr,
                                               objfile_binary_uuid, type)) {
-    if (objfile_binary_addr != LLDB_INVALID_ADDRESS)
-    {
+    if (objfile_binary_addr != LLDB_INVALID_ADDRESS) {
+      if (type == ObjectFile::eBinaryTypeUser)
+        m_dyld_addr = objfile_binary_addr;
+      else
         m_mach_kernel_addr = objfile_binary_addr;
-        found_main_binary_definitively = true;
-        LLDB_LOGF(log,
-                  "ProcessMachCore::DoLoadCore: using kernel address 0x%" PRIx64
-                  " from LC_NOTE 'main bin spec' load command.",
-                  m_mach_kernel_addr);
+      found_main_binary_definitively = true;
+      LLDB_LOGF(log,
+                "ProcessMachCore::DoLoadCore: using kernel address 0x%" PRIx64
+                " from LC_NOTE 'main bin spec' load command.",
+                m_mach_kernel_addr);
     }
   }
 
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt b/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt
index 2cbf8bcbb229dd32e79f3781020b2eaff810a9f2..84115aae01a5cbec57389e8353976979f7858326 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/CMakeLists.txt
@@ -11,6 +11,8 @@ add_lldb_library(lldbPluginScriptInterpreterPython PLUGIN
   PythonDataObjects.cpp
   PythonReadline.cpp
   ScriptInterpreterPython.cpp
+  ScriptedProcessPythonInterface.cpp
+  SWIGPythonBridge.cpp
 
   LINK_LIBS
     lldbBreakpoint
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7c7c5d73680aed54248f0cb8de540943d5194684
--- /dev/null
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.cpp
@@ -0,0 +1,48 @@
+//===-- SWIGPythonBridge.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Host/Config.h"
+#include "lldb/lldb-enumerations.h"
+
+#if LLDB_ENABLE_PYTHON
+
+// LLDB Python header must be included first
+#include "lldb-python.h"
+
+#include "SWIGPythonBridge.h"
+
+using namespace lldb;
+
+namespace lldb_private {
+
+template <typename T> const char *GetPythonValueFormatString(T t);
+template <> const char *GetPythonValueFormatString(char *) { return "s"; }
+template <> const char *GetPythonValueFormatString(char) { return "b"; }
+template <> const char *GetPythonValueFormatString(unsigned char) {
+  return "B";
+}
+template <> const char *GetPythonValueFormatString(short) { return "h"; }
+template <> const char *GetPythonValueFormatString(unsigned short) {
+  return "H";
+}
+template <> const char *GetPythonValueFormatString(int) { return "i"; }
+template <> const char *GetPythonValueFormatString(unsigned int) { return "I"; }
+template <> const char *GetPythonValueFormatString(long) { return "l"; }
+template <> const char *GetPythonValueFormatString(unsigned long) {
+  return "k";
+}
+template <> const char *GetPythonValueFormatString(long long) { return "L"; }
+template <> const char *GetPythonValueFormatString(unsigned long long) {
+  return "K";
+}
+template <> const char *GetPythonValueFormatString(float) { return "f"; }
+template <> const char *GetPythonValueFormatString(double) { return "d"; }
+
+} // namespace lldb_private
+
+#endif // LLDB_ENABLE_PYTHON
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
new file mode 100644
index 0000000000000000000000000000000000000000..1ef792bcf303be465747eddcb21c8cff3d0541c7
--- /dev/null
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
@@ -0,0 +1,56 @@
+//===-- ScriptInterpreterPython.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SWIGPYTHONBRIDGE_H
+#define LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SWIGPYTHONBRIDGE_H
+
+#include <string>
+
+#include "lldb/Host/Config.h"
+
+#if LLDB_ENABLE_PYTHON
+
+#include "lldb/lldb-forward.h"
+#include "lldb/lldb-types.h"
+
+namespace lldb_private {
+
+// GetPythonValueFormatString provides a system independent type safe way to
+// convert a variable's type into a python value format. Python value formats
+// are defined in terms of builtin C types and could change from system to as
+// the underlying typedef for uint* types, size_t, off_t and other values
+// change.
+
+template <typename T> const char *GetPythonValueFormatString(T t);
+template <> const char *GetPythonValueFormatString(char *);
+template <> const char *GetPythonValueFormatString(char);
+template <> const char *GetPythonValueFormatString(unsigned char);
+template <> const char *GetPythonValueFormatString(short);
+template <> const char *GetPythonValueFormatString(unsigned short);
+template <> const char *GetPythonValueFormatString(int);
+template <> const char *GetPythonValueFormatString(unsigned int);
+template <> const char *GetPythonValueFormatString(long);
+template <> const char *GetPythonValueFormatString(unsigned long);
+template <> const char *GetPythonValueFormatString(long long);
+template <> const char *GetPythonValueFormatString(unsigned long long);
+template <> const char *GetPythonValueFormatString(float t);
+template <> const char *GetPythonValueFormatString(double t);
+
+extern "C" void *LLDBSwigPythonCreateScriptedProcess(
+    const char *python_class_name, const char *session_dictionary_name,
+    const lldb::TargetSP &target_sp, StructuredDataImpl *args_impl,
+    std::string &error_string);
+
+extern "C" void *LLDBSWIGPython_CastPyObjectToSBData(void *data);
+extern "C" void *LLDBSWIGPython_CastPyObjectToSBError(void *data);
+extern "C" void *LLDBSWIGPython_CastPyObjectToSBValue(void *data);
+
+} // namespace lldb_private
+
+#endif // LLDB_ENABLE_PYTHON
+#endif // LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SWIGPYTHONBRIDGE_H
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index c4cc67cf7ab388653e35c23c681e9579016dd1bd..5f3c0b27446ca3500c181d6c2d0a5add0a21e9fd 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -16,7 +16,11 @@
 
 #include "PythonDataObjects.h"
 #include "PythonReadline.h"
+#include "SWIGPythonBridge.h"
 #include "ScriptInterpreterPythonImpl.h"
+#include "ScriptedProcessPythonInterface.h"
+
+#include "lldb/API/SBError.h"
 #include "lldb/API/SBFrame.h"
 #include "lldb/API/SBValue.h"
 #include "lldb/Breakpoint/StoppointCallbackContext.h"
@@ -148,8 +152,6 @@ extern "C" void *LLDBSwigPython_GetChildAtIndex(void *implementor,
 extern "C" int LLDBSwigPython_GetIndexOfChildWithName(void *implementor,
                                                       const char *child_name);
 
-extern "C" void *LLDBSWIGPython_CastPyObjectToSBValue(void *data);
-
 extern lldb::ValueObjectSP
 LLDBSWIGPython_GetValueObjectSPFromSBValue(void *data);
 
@@ -532,6 +534,9 @@ ScriptInterpreterPythonImpl::ScriptInterpreterPythonImpl(Debugger &debugger)
       m_command_thread_state(nullptr) {
   InitializePrivate();
 
+  m_scripted_process_interface_up =
+      std::make_unique<ScriptedProcessPythonInterface>(*this);
+
   m_dictionary_name.append("_dict");
   StreamString run_string;
   run_string.Printf("%s = dict()", m_dictionary_name.c_str());
@@ -1704,35 +1709,6 @@ StructuredData::ArraySP ScriptInterpreterPythonImpl::OSPlugin_ThreadsInfo(
   return StructuredData::ArraySP();
 }
 
-// GetPythonValueFormatString provides a system independent type safe way to
-// convert a variable's type into a python value format. Python value formats
-// are defined in terms of builtin C types and could change from system to as
-// the underlying typedef for uint* types, size_t, off_t and other values
-// change.
-
-template <typename T> const char *GetPythonValueFormatString(T t);
-template <> const char *GetPythonValueFormatString(char *) { return "s"; }
-template <> const char *GetPythonValueFormatString(char) { return "b"; }
-template <> const char *GetPythonValueFormatString(unsigned char) {
-  return "B";
-}
-template <> const char *GetPythonValueFormatString(short) { return "h"; }
-template <> const char *GetPythonValueFormatString(unsigned short) {
-  return "H";
-}
-template <> const char *GetPythonValueFormatString(int) { return "i"; }
-template <> const char *GetPythonValueFormatString(unsigned int) { return "I"; }
-template <> const char *GetPythonValueFormatString(long) { return "l"; }
-template <> const char *GetPythonValueFormatString(unsigned long) {
-  return "k";
-}
-template <> const char *GetPythonValueFormatString(long long) { return "L"; }
-template <> const char *GetPythonValueFormatString(unsigned long long) {
-  return "K";
-}
-template <> const char *GetPythonValueFormatString(float t) { return "f"; }
-template <> const char *GetPythonValueFormatString(double t) { return "d"; }
-
 StructuredData::StringSP
 ScriptInterpreterPythonImpl::OSPlugin_RegisterContextData(
     StructuredData::ObjectSP os_plugin_object_sp, lldb::tid_t tid) {
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h
index 5a75c0a655e58dd7812e9f3b0e56993840e74b08..b8b97811821813e80fef4e93dac1a4d09ad29b51 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h
@@ -13,6 +13,8 @@
 
 #if LLDB_ENABLE_PYTHON
 
+#include "ScriptedProcessPythonInterface.h"
+
 #include "lldb/Breakpoint/BreakpointOptions.h"
 #include "lldb/Core/IOHandler.h"
 #include "lldb/Core/StructuredDataImpl.h"
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
index 45dad42170052496eeb889024bac9265a26b087a..93d5768d9279e4226275a4a363b61362a978ee14 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SCRIPTINTERPRETERPYTHONIMPL_H
+#define LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SCRIPTINTERPRETERPYTHONIMPL_H
+
 #include "lldb/Host/Config.h"
 
 #if LLDB_ENABLE_PYTHON
@@ -483,4 +486,5 @@ protected:
 
 } // namespace lldb_private
 
-#endif
+#endif // LLDB_ENABLE_PYTHON
+#endif // LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SCRIPTINTERPRETERPYTHONIMPL_H
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..03f745eacd827d4fa978c2d827a71deea710180f
--- /dev/null
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp
@@ -0,0 +1,287 @@
+//===-- ScriptedProcessPythonInterface.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Host/Config.h"
+#include "lldb/lldb-enumerations.h"
+
+#if LLDB_ENABLE_PYTHON
+
+// LLDB Python header must be included first
+#include "lldb-python.h"
+
+#include "SWIGPythonBridge.h"
+#include "ScriptInterpreterPythonImpl.h"
+#include "ScriptedProcessPythonInterface.h"
+
+using namespace lldb;
+using namespace lldb_private;
+using namespace lldb_private::python;
+using Locker = ScriptInterpreterPythonImpl::Locker;
+
+StructuredData::GenericSP ScriptedProcessPythonInterface::CreatePluginObject(
+    const llvm::StringRef class_name, lldb::TargetSP target_sp,
+    StructuredData::DictionarySP args_sp) {
+  if (class_name.empty())
+    return {};
+
+  std::string error_string;
+  StructuredDataImpl *args_impl = nullptr;
+  if (args_sp) {
+    args_impl = new StructuredDataImpl();
+    args_impl->SetObjectSP(args_sp);
+  }
+
+  void *ret_val;
+
+  {
+
+    Locker py_lock(&m_interpreter, Locker::AcquireLock | Locker::NoSTDIN,
+                   Locker::FreeLock);
+
+    ret_val = LLDBSwigPythonCreateScriptedProcess(
+        class_name.str().c_str(), m_interpreter.GetDictionaryName(), target_sp,
+        args_impl, error_string);
+  }
+
+  m_object_instance_sp =
+      StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+
+  return m_object_instance_sp;
+}
+
+Status ScriptedProcessPythonInterface::Launch() {
+  return LaunchOrResume("launch");
+}
+
+Status ScriptedProcessPythonInterface::Resume() {
+  return LaunchOrResume("resume");
+}
+
+Status
+ScriptedProcessPythonInterface::LaunchOrResume(llvm::StringRef method_name) {
+  Locker py_lock(&m_interpreter, Locker::AcquireLock | Locker::NoSTDIN,
+                 Locker::FreeLock);
+
+  if (!m_object_instance_sp)
+    return Status("Python object ill-formed.");
+
+  if (!m_object_instance_sp)
+    return Status("Cannot convert Python object to StructuredData::Generic.");
+  PythonObject implementor(PyRefType::Borrowed,
+                           (PyObject *)m_object_instance_sp->GetValue());
+
+  if (!implementor.IsAllocated())
+    return Status("Python implementor not allocated.");
+
+  PythonObject pmeth(
+      PyRefType::Owned,
+      PyObject_GetAttrString(implementor.get(), method_name.str().c_str()));
+
+  if (PyErr_Occurred())
+    PyErr_Clear();
+
+  if (!pmeth.IsAllocated())
+    return Status("Python method not allocated.");
+
+  if (PyCallable_Check(pmeth.get()) == 0) {
+    if (PyErr_Occurred())
+      PyErr_Clear();
+    return Status("Python method not callable.");
+  }
+
+  if (PyErr_Occurred())
+    PyErr_Clear();
+
+  PythonObject py_return(PyRefType::Owned,
+                         PyObject_CallMethod(implementor.get(),
+                                             method_name.str().c_str(),
+                                             nullptr));
+
+  if (PyErr_Occurred()) {
+    PyErr_Print();
+    PyErr_Clear();
+    return Status("Python method could not be called.");
+  }
+
+  if (PyObject *py_ret_ptr = py_return.get()) {
+    lldb::SBError *sb_error =
+        (lldb::SBError *)LLDBSWIGPython_CastPyObjectToSBError(py_ret_ptr);
+
+    if (!sb_error)
+      return Status("Couldn't cast lldb::SBError to lldb::Status.");
+
+    Status status = m_interpreter.GetStatusFromSBError(*sb_error);
+
+    if (status.Fail())
+      return Status("error: %s", status.AsCString());
+
+    return status;
+  }
+
+  return Status("Returned object is null.");
+}
+
+size_t
+ScriptedProcessPythonInterface::GetGenericInteger(llvm::StringRef method_name) {
+  Locker py_lock(&m_interpreter, Locker::AcquireLock | Locker::NoSTDIN,
+                 Locker::FreeLock);
+
+  if (!m_object_instance_sp)
+    return LLDB_INVALID_ADDRESS;
+
+  if (!m_object_instance_sp)
+    return LLDB_INVALID_ADDRESS;
+  PythonObject implementor(PyRefType::Borrowed,
+                           (PyObject *)m_object_instance_sp->GetValue());
+
+  if (!implementor.IsAllocated())
+    return LLDB_INVALID_ADDRESS;
+
+  PythonObject pmeth(
+      PyRefType::Owned,
+      PyObject_GetAttrString(implementor.get(), method_name.str().c_str()));
+
+  if (PyErr_Occurred())
+    PyErr_Clear();
+
+  if (!pmeth.IsAllocated())
+    return LLDB_INVALID_ADDRESS;
+
+  if (PyCallable_Check(pmeth.get()) == 0) {
+    if (PyErr_Occurred())
+      PyErr_Clear();
+    return LLDB_INVALID_ADDRESS;
+  }
+
+  if (PyErr_Occurred())
+    PyErr_Clear();
+
+  PythonObject py_return(PyRefType::Owned,
+                         PyObject_CallMethod(implementor.get(),
+                                             method_name.str().c_str(),
+                                             nullptr));
+
+  if (PyErr_Occurred()) {
+    PyErr_Print();
+    PyErr_Clear();
+  }
+
+  if (py_return.get()) {
+    auto size = py_return.AsUnsignedLongLong();
+    return (size) ? *size : LLDB_INVALID_ADDRESS;
+  }
+  return LLDB_INVALID_ADDRESS;
+}
+
+lldb::MemoryRegionInfoSP
+ScriptedProcessPythonInterface::GetMemoryRegionContainingAddress(
+    lldb::addr_t address) {
+  // TODO: Implement
+  return nullptr;
+}
+
+StructuredData::DictionarySP
+ScriptedProcessPythonInterface::GetThreadWithID(lldb::tid_t tid) {
+  // TODO: Implement
+  return nullptr;
+}
+
+StructuredData::DictionarySP
+ScriptedProcessPythonInterface::GetRegistersForThread(lldb::tid_t tid) {
+  // TODO: Implement
+  return nullptr;
+}
+
+lldb::DataExtractorSP ScriptedProcessPythonInterface::ReadMemoryAtAddress(
+    lldb::addr_t address, size_t size, Status &error) {
+  Locker py_lock(&m_interpreter, Locker::AcquireLock | Locker::NoSTDIN,
+                 Locker::FreeLock);
+
+  auto error_with_message = [&error](llvm::StringRef message) {
+    error.SetErrorString(message);
+    return nullptr;
+  };
+
+  static char callee_name[] = "read_memory_at_address";
+  std::string param_format = GetPythonValueFormatString(address);
+  param_format += GetPythonValueFormatString(size);
+
+  if (!m_object_instance_sp)
+    return error_with_message("Python object ill-formed.");
+
+  if (!m_object_instance_sp)
+    return error_with_message("Python method not callable.");
+
+  PythonObject implementor(PyRefType::Borrowed,
+                           (PyObject *)m_object_instance_sp->GetValue());
+
+  if (!implementor.IsAllocated())
+    return error_with_message("Python implementor not allocated.");
+
+  PythonObject pmeth(PyRefType::Owned,
+                     PyObject_GetAttrString(implementor.get(), callee_name));
+
+  if (PyErr_Occurred())
+    PyErr_Clear();
+
+  if (!pmeth.IsAllocated())
+    return error_with_message("Python method not allocated.");
+
+  if (PyCallable_Check(pmeth.get()) == 0) {
+    if (PyErr_Occurred())
+      PyErr_Clear();
+    return error_with_message("Python method not callable.");
+  }
+
+  if (PyErr_Occurred())
+    PyErr_Clear();
+
+  PythonObject py_return(PyRefType::Owned,
+                         PyObject_CallMethod(implementor.get(), callee_name,
+                                             param_format.c_str(), address,
+                                             size));
+
+  if (PyErr_Occurred()) {
+    PyErr_Print();
+    PyErr_Clear();
+    return error_with_message("Python method could not be called.");
+  }
+
+  if (PyObject *py_ret_ptr = py_return.get()) {
+    lldb::SBData *sb_data =
+        (lldb::SBData *)LLDBSWIGPython_CastPyObjectToSBData(py_ret_ptr);
+
+    if (!sb_data)
+      return error_with_message(
+          "Couldn't cast lldb::SBData to lldb::DataExtractor.");
+
+    return m_interpreter.GetDataExtractorFromSBData(*sb_data);
+  }
+
+  return error_with_message("Returned object is null.");
+}
+
+StructuredData::DictionarySP ScriptedProcessPythonInterface::GetLoadedImages() {
+  // TODO: Implement
+  return nullptr;
+}
+
+lldb::pid_t ScriptedProcessPythonInterface::GetProcessID() {
+  size_t pid = GetGenericInteger("get_process_id");
+
+  return (pid >= std::numeric_limits<lldb::pid_t>::max())
+             ? LLDB_INVALID_PROCESS_ID
+             : pid;
+}
+
+bool ScriptedProcessPythonInterface::IsAlive() {
+  return GetGenericInteger("is_alive");
+  ;
+}
+
+#endif
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.h
new file mode 100644
index 0000000000000000000000000000000000000000..fc07c927be2d1361baac5a73909f4edfc8b99953
--- /dev/null
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.h
@@ -0,0 +1,61 @@
+//===-- ScriptedProcessPythonInterface.h ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SCRIPTEDPROCESSPYTHONINTERFACE_H
+#define LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SCRIPTEDPROCESSPYTHONINTERFACE_H
+
+#include "lldb/Host/Config.h"
+
+#if LLDB_ENABLE_PYTHON
+
+#include "lldb/Interpreter/ScriptedProcessInterface.h"
+
+namespace lldb_private {
+class ScriptInterpreterPythonImpl;
+class ScriptedProcessPythonInterface : public ScriptedProcessInterface {
+public:
+  ScriptedProcessPythonInterface(ScriptInterpreterPythonImpl &interpreter)
+      : ScriptedProcessInterface(), m_interpreter(interpreter) {}
+
+  StructuredData::GenericSP
+  CreatePluginObject(const llvm::StringRef class_name, lldb::TargetSP target_sp,
+                     StructuredData::DictionarySP args_sp) override;
+
+  Status Launch() override;
+
+  Status Resume() override;
+
+  lldb::MemoryRegionInfoSP
+  GetMemoryRegionContainingAddress(lldb::addr_t address) override;
+
+  StructuredData::DictionarySP GetThreadWithID(lldb::tid_t tid) override;
+
+  StructuredData::DictionarySP GetRegistersForThread(lldb::tid_t tid) override;
+
+  lldb::DataExtractorSP ReadMemoryAtAddress(lldb::addr_t address, size_t size,
+                                            Status &error) override;
+
+  StructuredData::DictionarySP GetLoadedImages() override;
+
+  lldb::pid_t GetProcessID() override;
+
+  bool IsAlive() override;
+
+protected:
+  size_t GetGenericInteger(llvm::StringRef method_name);
+  Status LaunchOrResume(llvm::StringRef method_name);
+
+private:
+  // The lifetime is managed by the ScriptInterpreter
+  ScriptInterpreterPythonImpl &m_interpreter;
+  StructuredData::GenericSP m_object_instance_sp;
+};
+} // namespace lldb_private
+
+#endif // LLDB_ENABLE_PYTHON
+#endif // LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_SCRIPTEDPROCESSPYTHONINTERFACE_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
index dda599baffebf5fc5de07e4447eb1df4f4cdeea2..1f40d880ea34b52180e831ff77f634370f58faa1 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
@@ -13,9 +13,11 @@
 #include "Plugins/SymbolFile/DWARF/LogChannelDWARF.h"
 #include "Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h"
 #include "lldb/Core/Module.h"
+#include "lldb/Core/Progress.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/Timer.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/ThreadPool.h"
 
 using namespace lldb_private;
@@ -56,6 +58,17 @@ void ManualDWARFIndex::Index() {
   if (units_to_index.empty())
     return;
 
+  StreamString module_desc;
+  m_module.GetDescription(module_desc.AsRawOstream(),
+                          lldb::eDescriptionLevelBrief);
+
+  // Include 2 passes per unit to index for extracting DIEs from the unit and
+  // indexing the unit, and then 8 extra entries for finalizing each index set.
+  const uint64_t total_progress = units_to_index.size() * 2 + 8;
+  Progress progress(
+      llvm::formatv("Manually indexing DWARF for {0}", module_desc.GetData()),
+      total_progress);
+
   std::vector<IndexSet> sets(units_to_index.size());
 
   // Keep memory down by clearing DIEs for any units if indexing
@@ -64,10 +77,12 @@ void ManualDWARFIndex::Index() {
       units_to_index.size());
   auto parser_fn = [&](size_t cu_idx) {
     IndexUnit(*units_to_index[cu_idx], dwp_dwarf, sets[cu_idx]);
+    progress.Increment();
   };
 
-  auto extract_fn = [&units_to_index, &clear_cu_dies](size_t cu_idx) {
+  auto extract_fn = [&](size_t cu_idx) {
     clear_cu_dies[cu_idx] = units_to_index[cu_idx]->ExtractDIEsScoped();
+    progress.Increment();
   };
 
   // Share one thread pool across operations to avoid the overhead of
@@ -92,11 +107,12 @@ void ManualDWARFIndex::Index() {
     pool.async(parser_fn, i);
   pool.wait();
 
-  auto finalize_fn = [this, &sets](NameToDIE(IndexSet::*index)) {
+  auto finalize_fn = [this, &sets, &progress](NameToDIE(IndexSet::*index)) {
     NameToDIE &result = m_set.*index;
     for (auto &set : sets)
       result.Append(set.*index);
     result.Finalize();
+    progress.Increment();
   };
 
   pool.async(finalize_fn, &IndexSet::function_basenames);
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 587550961ec9d92a78f1c285a482db8b87b3e9de..24c44ac83e803dcc96d320225f0d9ef900afa07d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -16,6 +16,7 @@
 #include "lldb/Core/ModuleList.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Core/Progress.h"
 #include "lldb/Core/Section.h"
 #include "lldb/Core/StreamFile.h"
 #include "lldb/Core/Value.h"
@@ -74,6 +75,7 @@
 
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
 
 #include <algorithm>
 #include <map>
@@ -467,22 +469,32 @@ void SymbolFileDWARF::InitializeObject() {
   Log *log = LogChannelDWARF::GetLogIfAll(DWARF_LOG_DEBUG_INFO);
 
   if (!GetGlobalPluginProperties()->IgnoreFileIndexes()) {
+    StreamString module_desc;
+    GetObjectFile()->GetModule()->GetDescription(module_desc.AsRawOstream(),
+                                                 lldb::eDescriptionLevelBrief);
     DWARFDataExtractor apple_names, apple_namespaces, apple_types, apple_objc;
     LoadSectionData(eSectionTypeDWARFAppleNames, apple_names);
     LoadSectionData(eSectionTypeDWARFAppleNamespaces, apple_namespaces);
     LoadSectionData(eSectionTypeDWARFAppleTypes, apple_types);
     LoadSectionData(eSectionTypeDWARFAppleObjC, apple_objc);
 
-    m_index = AppleDWARFIndex::Create(
-        *GetObjectFile()->GetModule(), apple_names, apple_namespaces,
-        apple_types, apple_objc, m_context.getOrLoadStrData());
+    if (apple_names.GetByteSize() > 0 || apple_namespaces.GetByteSize() > 0 ||
+        apple_types.GetByteSize() > 0 || apple_objc.GetByteSize() > 0) {
+      Progress progress(llvm::formatv("Loading Apple DWARF index for {0}",
+                                      module_desc.GetData()));
+      m_index = AppleDWARFIndex::Create(
+          *GetObjectFile()->GetModule(), apple_names, apple_namespaces,
+          apple_types, apple_objc, m_context.getOrLoadStrData());
 
-    if (m_index)
-      return;
+      if (m_index)
+        return;
+    }
 
     DWARFDataExtractor debug_names;
     LoadSectionData(eSectionTypeDWARFDebugNames, debug_names);
     if (debug_names.GetByteSize() > 0) {
+      Progress progress(
+          llvm::formatv("Loading DWARF5 index for {0}", module_desc.GetData()));
       llvm::Expected<std::unique_ptr<DebugNamesDWARFIndex>> index_or =
           DebugNamesDWARFIndex::Create(*GetObjectFile()->GetModule(),
                                        debug_names,
@@ -1966,66 +1978,8 @@ uint32_t SymbolFileDWARF::ResolveSymbolContext(const FileSpec &file_spec,
       bool file_spec_matches_cu_file_spec =
           FileSpec::Match(file_spec, dc_cu->GetPrimaryFile());
       if (check_inlines || file_spec_matches_cu_file_spec) {
-        SymbolContext sc(m_objfile_sp->GetModule());
-        sc.comp_unit = dc_cu;
-        uint32_t file_idx = UINT32_MAX;
-
-        // If we are looking for inline functions only and we don't find it
-        // in the support files, we are done.
-        if (check_inlines) {
-          file_idx =
-              sc.comp_unit->GetSupportFiles().FindFileIndex(1, file_spec, true);
-          if (file_idx == UINT32_MAX)
-            continue;
-        }
-
-        if (line != 0) {
-          LineTable *line_table = sc.comp_unit->GetLineTable();
-
-          if (line_table != nullptr && line != 0) {
-            // We will have already looked up the file index if we are
-            // searching for inline entries.
-            if (!check_inlines)
-              file_idx = sc.comp_unit->GetSupportFiles().FindFileIndex(
-                  1, file_spec, true);
-
-            if (file_idx != UINT32_MAX) {
-              uint32_t found_line;
-              uint32_t line_idx = line_table->FindLineEntryIndexByFileIndex(
-                  0, file_idx, line, false, &sc.line_entry);
-              found_line = sc.line_entry.line;
-
-              while (line_idx != UINT32_MAX) {
-                sc.function = nullptr;
-                sc.block = nullptr;
-                if (resolve_scope &
-                    (eSymbolContextFunction | eSymbolContextBlock)) {
-                  const lldb::addr_t file_vm_addr =
-                      sc.line_entry.range.GetBaseAddress().GetFileAddress();
-                  if (file_vm_addr != LLDB_INVALID_ADDRESS) {
-                    ResolveFunctionAndBlock(
-                        file_vm_addr, resolve_scope & eSymbolContextBlock, sc);
-                  }
-                }
-
-                sc_list.Append(sc);
-                line_idx = line_table->FindLineEntryIndexByFileIndex(
-                    line_idx + 1, file_idx, found_line, true, &sc.line_entry);
-              }
-            }
-          } else if (file_spec_matches_cu_file_spec && !check_inlines) {
-            // only append the context if we aren't looking for inline call
-            // sites by file and line and if the file spec matches that of
-            // the compile unit
-            sc_list.Append(sc);
-          }
-        } else if (file_spec_matches_cu_file_spec && !check_inlines) {
-          // only append the context if we aren't looking for inline call
-          // sites by file and line and if the file spec matches that of
-          // the compile unit
-          sc_list.Append(sc);
-        }
-
+        dc_cu->ResolveSymbolContext(file_spec, line, check_inlines, false,
+                                    resolve_scope, sc_list);
         if (!check_inlines)
           break;
       }
diff --git a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
index f9c12e634140724931e25ffcdaf9a9c683698a29..ef80d764eb4956478e05a6bb33a7322cb3d1bee4 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
@@ -326,7 +326,7 @@ GetDeclFromContextByName(const clang::ASTContext &ast,
   if (result.empty())
     return nullptr;
 
-  return result[0];
+  return *result.begin();
 }
 
 static bool IsAnonymousNamespaceName(llvm::StringRef name) {
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index c94cb8d16ee9e73b596c73ceda95b416a8b8a4a8..d1add1111b864b6f4417e7101f4d372ff31e4ed0 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -163,7 +163,6 @@ void addOverridesForMethod(clang::CXXMethodDecl *decl) {
           if (name.getNameKind() == clang::DeclarationName::CXXDestructorName)
             if (auto *baseDtorDecl = base_record->getDestructor()) {
               if (baseDtorDecl->isVirtual()) {
-                path.Decls = baseDtorDecl;
                 decls.push_back(baseDtorDecl);
                 return true;
               } else
@@ -171,12 +170,11 @@ void addOverridesForMethod(clang::CXXMethodDecl *decl) {
             }
 
           // Otherwise, search for name in the base class.
-          for (path.Decls = base_record->lookup(name); !path.Decls.empty();
-               path.Decls = path.Decls.slice(1)) {
+          for (path.Decls = base_record->lookup(name).begin();
+               path.Decls != path.Decls.end(); ++path.Decls) {
             if (auto *method_decl =
-                    llvm::dyn_cast<clang::CXXMethodDecl>(path.Decls.front()))
+                    llvm::dyn_cast<clang::CXXMethodDecl>(*path.Decls))
               if (method_decl->isVirtual() && !isOverload(decl, method_decl)) {
-                path.Decls = method_decl;
                 decls.push_back(method_decl);
                 return true;
               }
@@ -479,6 +477,9 @@ static void ParseLangArgs(LangOptions &Opts, InputKind IK, const char *triple) {
     case clang::Language::OpenCL:
       LangStd = LangStandard::lang_opencl10;
       break;
+    case clang::Language::OpenCLCXX:
+      LangStd = LangStandard::lang_openclcpp;
+      break;
     case clang::Language::CUDA:
       LangStd = LangStandard::lang_cuda;
       break;
@@ -6605,10 +6606,11 @@ size_t TypeSystemClang::GetIndexOfChildMemberWithName(
           if (cxx_record_decl->lookupInBases(
                   [decl_name](const clang::CXXBaseSpecifier *specifier,
                               clang::CXXBasePath &path) {
-                    path.Decls =
-                        specifier->getType()->getAsCXXRecordDecl()->lookup(
-                            decl_name);
-                    return !path.Decls.empty();
+                    CXXRecordDecl *record =
+                      specifier->getType()->getAsCXXRecordDecl();
+                    auto r = record->lookup(decl_name);
+                    path.Decls = r.begin();
+                    return !r.empty();
                   },
                   paths)) {
             clang::CXXBasePaths::const_paths_iterator path,
@@ -6631,9 +6633,10 @@ size_t TypeSystemClang::GetIndexOfChildMemberWithName(
                           ->getDecl());
                 }
               }
-              for (clang::NamedDecl *path_decl : path->Decls) {
+              for (clang::DeclContext::lookup_iterator I = path->Decls, E;
+                   I != E; ++I) {
                 child_idx = GetIndexForRecordChild(
-                    parent_record_decl, path_decl, omit_empty_base_classes);
+                    parent_record_decl, *I, omit_empty_base_classes);
                 if (child_idx == UINT32_MAX) {
                   child_indexes.clear();
                   return 0;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index f37652ff477a6bfd74cb2dbbb408b1263b2422ec..45c84e7447acba4254d0e7cd557830097194c1b3 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -265,7 +265,7 @@ public:
       clang::DeclContext::lookup_result result = decl_context->lookup(myName);
 
       if (!result.empty()) {
-        clang::NamedDecl *named_decl = result[0];
+        clang::NamedDecl *named_decl = *result.begin();
         if (const RecordDeclType *record_decl =
                 llvm::dyn_cast<RecordDeclType>(named_decl))
           compiler_type.SetCompilerType(
diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp
index 822f0df4da37a8d55ee83119f3523ff62d76491c..22c22cc26bc7f4f00a6cfd4a53cf6cc021c07c9d 100644
--- a/lldb/source/Symbol/CompileUnit.cpp
+++ b/lldb/source/Symbol/CompileUnit.cpp
@@ -248,6 +248,18 @@ void CompileUnit::ResolveSymbolContext(const FileSpec &file_spec,
   if (!file_spec_matches_cu_file_spec && !check_inlines)
     return;
 
+  SymbolContext sc(GetModule());
+  sc.comp_unit = this;
+
+  if (line == 0) {
+    if (file_spec_matches_cu_file_spec && !check_inlines) {
+      // only append the context if we aren't looking for inline call sites by
+      // file and line and if the file spec matches that of the compile unit
+      sc_list.Append(sc);
+    }
+    return;
+  }
+
   uint32_t file_idx =
       GetSupportFiles().FindFileIndex(0, file_spec, true);
   while (file_idx != UINT32_MAX) {
@@ -259,23 +271,15 @@ void CompileUnit::ResolveSymbolContext(const FileSpec &file_spec,
   if (num_file_indexes == 0)
     return;
 
-  SymbolContext sc(GetModule());
-  sc.comp_unit = this;
+  LineTable *line_table = sc.comp_unit->GetLineTable();
 
-  if (line == 0) {
+  if (line_table == nullptr) {
     if (file_spec_matches_cu_file_spec && !check_inlines) {
-      // only append the context if we aren't looking for inline call sites by
-      // file and line and if the file spec matches that of the compile unit
       sc_list.Append(sc);
     }
     return;
   }
 
-  LineTable *line_table = sc.comp_unit->GetLineTable();
-
-  if (line_table == nullptr)
-    return;
-
   uint32_t line_idx;
   LineEntry line_entry;
 
diff --git a/lldb/source/Target/StopInfo.cpp b/lldb/source/Target/StopInfo.cpp
index 7e830c6e2bed15dd3922a77d06bcf8804dc7dda3..95efd0a17127b80fe1c3e4d2848e3bf0052c5821 100644
--- a/lldb/source/Target/StopInfo.cpp
+++ b/lldb/source/Target/StopInfo.cpp
@@ -305,6 +305,20 @@ protected:
           // location said we should stop. But that's better than not running
           // all the callbacks.
 
+          // There's one other complication here.  We may have run an async
+          // breakpoint callback that said we should stop.  We only want to
+          // override that if another breakpoint action says we shouldn't 
+          // stop.  If nobody else has an opinion, then we should stop if the
+          // async callback says we should.  An example of this is the async
+          // shared library load notification breakpoint and the setting
+          // stop-on-sharedlibrary-events.
+          // We'll keep the async value in async_should_stop, and track whether
+          // anyone said we should NOT stop in actually_said_continue.
+          bool async_should_stop = false;
+          if (m_should_stop_is_valid)
+            async_should_stop = m_should_stop;
+          bool actually_said_continue = false;
+
           m_should_stop = false;
 
           // We don't select threads as we go through them testing breakpoint
@@ -422,9 +436,10 @@ protected:
 
             bool precondition_result =
                 bp_loc_sp->GetBreakpoint().EvaluatePrecondition(context);
-            if (!precondition_result)
+            if (!precondition_result) {
+              actually_said_continue = true;
               continue;
-
+            }
             // Next run the condition for the breakpoint.  If that says we
             // should stop, then we'll run the callback for the breakpoint.  If
             // the callback says we shouldn't stop that will win.
@@ -462,6 +477,7 @@ protected:
                   // the condition fails. We've already bumped it by the time
                   // we get here, so undo the bump:
                   bp_loc_sp->UndoBumpHitCount();
+                  actually_said_continue = true;
                   continue;
                 }
               }
@@ -494,16 +510,22 @@ protected:
             // When we figure out how to nest breakpoint hits then this will
             // change.
 
-            Debugger &debugger = thread_sp->CalculateTarget()->GetDebugger();
-            bool old_async = debugger.GetAsyncExecution();
-            debugger.SetAsyncExecution(true);
+            // Don't run async callbacks in PerformAction.  They have already
+            // been taken into account with async_should_stop.
+            if (!bp_loc_sp->IsCallbackSynchronous()) {
+              Debugger &debugger = thread_sp->CalculateTarget()->GetDebugger();
+              bool old_async = debugger.GetAsyncExecution();
+              debugger.SetAsyncExecution(true);
 
-            callback_says_stop = bp_loc_sp->InvokeCallback(&context);
+              callback_says_stop = bp_loc_sp->InvokeCallback(&context);
 
-            debugger.SetAsyncExecution(old_async);
+              debugger.SetAsyncExecution(old_async);
 
-            if (callback_says_stop && auto_continue_says_stop)
-              m_should_stop = true;
+              if (callback_says_stop && auto_continue_says_stop)
+                m_should_stop = true;
+              else
+                actually_said_continue = true;
+            }
                   
             // If we are going to stop for this breakpoint, then remove the
             // breakpoint.
@@ -517,9 +539,15 @@ protected:
             // here.
             if (HasTargetRunSinceMe()) {
               m_should_stop = false;
+              actually_said_continue = true;
               break;
             }
           }
+          // At this point if nobody actually told us to continue, we should
+          // give the async breakpoint callback a chance to weigh in:
+          if (!actually_said_continue && !m_should_stop) {
+            m_should_stop = async_should_stop;
+          }
         }
         // We've figured out what this stop wants to do, so mark it as valid so
         // we don't compute it again.
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 98f63a81ea17c190fdaa2957fb6463a067ff5f9d..c6667ce942cd066e78bf0df5e364c82948c76b05 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -2925,6 +2925,28 @@ Status Target::Launch(ProcessLaunchInfo &launch_info, Stream *stream) {
 
   launch_info.GetFlags().Set(eLaunchFlagDebug);
 
+  if (launch_info.IsScriptedProcess()) {
+    TargetPropertiesSP properties_sp = GetGlobalProperties();
+
+    if (!properties_sp) {
+      LLDB_LOGF(log, "Target::%s Couldn't fetch target global properties.",
+                __FUNCTION__);
+      return error;
+    }
+
+    // Only copy scripted process launch options.
+    ProcessLaunchInfo &default_launch_info =
+        const_cast<ProcessLaunchInfo &>(properties_sp->GetProcessLaunchInfo());
+
+    default_launch_info.SetProcessPluginName("ScriptedProcess");
+    default_launch_info.SetScriptedProcessClassName(
+        launch_info.GetScriptedProcessClassName());
+    default_launch_info.SetScriptedProcessDictionarySP(
+        launch_info.GetScriptedProcessDictionarySP());
+
+    SetProcessLaunchInfo(launch_info);
+  }
+
   // Get the value of synchronous execution here.  If you wait till after you
   // have started to run, then you could have hit a breakpoint, whose command
   // might switch the value, and then you'll pick up that incorrect value.
diff --git a/lldb/source/Utility/LLDBAssert.cpp b/lldb/source/Utility/LLDBAssert.cpp
index 6ae0ee50ef149e274aaa8b31f6ab98b8b7df3a32..532b56b6f59e3f7545d9eacc303be8bceaa51369 100644
--- a/lldb/source/Utility/LLDBAssert.cpp
+++ b/lldb/source/Utility/LLDBAssert.cpp
@@ -7,11 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Utility/LLDBAssert.h"
-
+#include "llvm/Config/config.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 
+#if LLVM_SUPPORT_XCODE_SIGNPOSTS
+#include <os/log.h>
+#endif
+
 using namespace llvm;
 using namespace lldb_private;
 
@@ -24,6 +28,14 @@ void lldb_private::lldb_assert(bool expression, const char *expr_text,
   // If asserts are enabled abort here.
   assert(false && "lldb_assert failed");
 
+#if LLVM_SUPPORT_XCODE_SIGNPOSTS
+  if (__builtin_available(macos 10.12, iOS 10, tvOS 10, watchOS 3, *)) {
+    os_log_fault(OS_LOG_DEFAULT,
+                 "Assertion failed: (%s), function %s, file %s, line %u\n",
+                 expr_text, func, file, line);
+  }
+#endif
+
   // In a release configuration it will print a warning and encourage the user
   // to file a bug report, similar to LLVM’s crash handler, and then return
   // execution.
diff --git a/lldb/test/API/CMakeLists.txt b/lldb/test/API/CMakeLists.txt
index 001712fcfbceb11db2d3bc942462f8bb1f9b1a86..ce7531fd98665caf266fa25c40c7b8fe2deb24a1 100644
--- a/lldb/test/API/CMakeLists.txt
+++ b/lldb/test/API/CMakeLists.txt
@@ -1,3 +1,10 @@
+add_custom_target(lldb-api-test-deps)
+add_dependencies(lldb-api-test-deps lldb-test-deps)
+
+add_lit_testsuites(LLDB-API
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS lldb-api-test-deps)
+
 function(add_python_test_target name test_script args comment)
   set(PYTHON_TEST_COMMAND
     ${Python3_EXECUTABLE}
@@ -109,7 +116,7 @@ if(CMAKE_HOST_APPLE)
     list(APPEND LLDB_TEST_COMMON_ARGS --out-of-tree-debugserver)
     add_lldb_test_dependency(debugserver)
   else()
-    message(STATUS "LLDB Tests use just-built debug server")
+    message(STATUS "LLDB tests use just-built debug server")
   endif()
 endif()
 
@@ -153,17 +160,35 @@ string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_EXECUTAB
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_COMPILER "${LLDB_TEST_COMPILER}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_DSYMUTIL "${LLDB_TEST_DSYMUTIL}")
 
-# Configure the API test suite.
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
 
-if (CMAKE_GENERATOR STREQUAL "Xcode")
-  # Xcode does not get the auto-generated targets. We need to create
-  # check-lldb-api manually.
-  add_lit_testsuite(check-lldb-api "Running lldb api test suite"
-    ${CMAKE_CURRENT_BINARY_DIR}
-    DEPENDS lldb-test-deps)
-endif()
+# Targets for running the test suite on the different Apple simulators.
+add_lit_testsuite(check-lldb-simulator-ios
+  "Running lldb test suite on the iOS simulator"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  PARAMS "lldb-run-with-simulator=ios"
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS lldb-api-test-deps)
+
+add_lit_testsuite(check-lldb-simulator-watchos
+  "Running lldb test suite on the watchOS simulator"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  PARAMS "lldb-run-with-simulator=watchos"
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS lldb-api-test-deps)
+
+add_lit_testsuite(check-lldb-simulator-tvos
+  "Running lldb test suite on the tvOS simulator"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  PARAMS "lldb-run-with-simulator=tvos"
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS lldb-api-test-deps)
+
+add_lit_testsuite(check-lldb-api "Running lldb api test suite"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS lldb-api-test-deps)
diff --git a/lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py b/lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py
index 5afb57f3ac46c18347741c3d5582541c3c98add3..862943c41aa37bcabfa34dc3e74fa74634ba618e 100644
--- a/lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py
+++ b/lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py
@@ -2,61 +2,57 @@
 Test target commands: target.auto-install-main-executable.
 """
 
+import socket
 import time
-import gdbremote_testcase
+import lldbgdbserverutils
 
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
 
-class TestAutoInstallMainExecutable(gdbremote_testcase.GdbRemoteTestCaseBase):
+class TestAutoInstallMainExecutable(TestBase):
     mydir = TestBase.compute_mydir(__file__)
+    NO_DEBUG_INFO_TESTCASE = True
 
     @llgs_test
-    @no_debug_info_test
-    @skipIf(remote=False)
-    @expectedFailureAll(hostoslist=["windows"], triple='.*-android')
+    @skipIfRemote
+    @expectedFailureAll(oslist=["windows"]) # process modules not loaded
     def test_target_auto_install_main_executable(self):
         self.build()
 
-        # Manually install the modified binary.
-        working_dir = lldb.remote_platform.GetWorkingDirectory()
-        src_device = lldb.SBFileSpec(self.getBuildArtifact("a.device.out"))
-        dest = lldb.SBFileSpec(os.path.join(working_dir, "a.out"))
-        err = lldb.remote_platform.Put(src_device, dest)
-        if err.Fail():
-            raise RuntimeError(
-                "Unable copy '%s' to '%s'.\n>>> %s" %
-                (src_device.GetFilename(), working_dir, err.GetCString()))
-
-        m = re.search("^(.*)://([^/]*):(.*)$", configuration.lldb_platform_url)
-        protocol = m.group(1)
-        hostname = m.group(2)
-        hostport = int(m.group(3))
-        listen_url = "*:"+str(hostport+1)
+        hostname = socket.getaddrinfo("localhost", 0, proto=socket.IPPROTO_TCP)[0][4][0]
+        listen_url = "[%s]:0"%hostname
 
+        port_file = self.getBuildArtifact("port")
         commandline_args = [
             "platform",
             "--listen",
             listen_url,
-            "--server"
-            ]
-
+            "--socket-file",
+            port_file]
         self.spawnSubprocess(
-            self.debug_monitor_exe,
-            commandline_args,
-            install_remote=False)
+            lldbgdbserverutils.get_lldb_server_exe(),
+            commandline_args)
 
-        # Wait for the new process gets ready.
-        time.sleep(0.1)
+        socket_id = lldbutil.wait_for_file_on_target(self, port_file)
 
-        self.dbg.SetAsync(False)
-
-        new_platform = lldb.SBPlatform(lldb.remote_platform.GetName())
+        new_platform = lldb.SBPlatform("remote-" + self.getPlatform())
         self.dbg.SetSelectedPlatform(new_platform)
 
-        connect_url = "%s://%s:%s" % (protocol, hostname, str(hostport+1))
+        connect_url = "connect://[%s]:%s" % (hostname, socket_id)
+        connect_opts = lldb.SBPlatformConnectOptions(connect_url)
+        self.assertSuccess(new_platform.ConnectRemote(connect_opts))
+
+        wd = self.getBuildArtifact("wd")
+        os.mkdir(wd)
+        new_platform.SetWorkingDirectory(wd)
+
+
+        # Manually install the modified binary.
+        src_device = lldb.SBFileSpec(self.getBuildArtifact("a.device.out"))
+        dest = lldb.SBFileSpec(os.path.join(wd, "a.out"))
+        self.assertSuccess(new_platform.Put(src_device, dest))
 
         # Test the default setting.
         self.expect("settings show target.auto-install-main-executable",
@@ -68,12 +64,9 @@ class TestAutoInstallMainExecutable(gdbremote_testcase.GdbRemoteTestCaseBase):
         self.expect("settings show target.auto-install-main-executable",
             substrs=["target.auto-install-main-executable (boolean) = false"])
 
-        self.runCmd("platform select %s"%configuration.lldb_platform_name)
-        self.runCmd("platform connect %s" % (connect_url))
-
         # Create the target with the original file.
         self.runCmd("target create --remote-file %s %s "%
-                                        (os.path.join(working_dir,dest.GetFilename()),
+                                        (dest.fullpath,
                                             self.getBuildArtifact("a.out")))
 
         target = self.dbg.GetSelectedTarget()
diff --git a/lldb/test/API/functionalities/load_lazy/Makefile b/lldb/test/API/functionalities/load_lazy/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..7200114d03aebc0abbfa8f62e81a5169494ea7e2
--- /dev/null
+++ b/lldb/test/API/functionalities/load_lazy/Makefile
@@ -0,0 +1,18 @@
+CXX_SOURCES := main.cpp
+USE_LIBDL := 1
+
+all: t2_0 t2_1 t1 a.out
+
+include Makefile.rules
+
+t1: t2_0
+	$(MAKE) VPATH=$(SRCDIR) -f $(MAKEFILE_RULES) \
+		DYLIB_ONLY=YES DYLIB_C_SOURCES=t1.c DYLIB_NAME=t1 LD_EXTRAS="-L. -lt2_0"
+
+t2_0:
+	$(MAKE) VPATH=$(SRCDIR) -f $(MAKEFILE_RULES) \
+		DYLIB_ONLY=YES DYLIB_C_SOURCES=t2_0.c DYLIB_NAME=t2_0
+
+t2_1:
+	$(MAKE) VPATH=$(SRCDIR) -f $(MAKEFILE_RULES) \
+		DYLIB_ONLY=YES DYLIB_C_SOURCES=t2_1.c DYLIB_NAME=t2_1
diff --git a/lldb/test/API/functionalities/load_lazy/TestLoadUsingLazyBind.py b/lldb/test/API/functionalities/load_lazy/TestLoadUsingLazyBind.py
new file mode 100644
index 0000000000000000000000000000000000000000..a32e589884ce6f0fbd3f20e652c0f950ac3af788
--- /dev/null
+++ b/lldb/test/API/functionalities/load_lazy/TestLoadUsingLazyBind.py
@@ -0,0 +1,65 @@
+"""
+Test that SBProcess.LoadImageUsingPaths uses RTLD_LAZY
+"""
+
+
+
+import os
+import shutil
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class LoadUsingLazyBind(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @skipIfRemote
+    @skipIfWindows # The Windows platform doesn't implement DoLoadImage.
+    # Failing for unknown reasons on Linux, see
+    # https://bugs.llvm.org/show_bug.cgi?id=49656.
+    @skipUnlessDarwin
+    def test_load_using_lazy_bind(self):
+        """Test that we load using RTLD_LAZY"""
+
+        self.build()
+        wd = os.path.realpath(self.getBuildDir())
+
+        ext = '.so'
+        if self.platformIsDarwin():
+            ext = '.dylib'
+
+        def make_lib_path(name):
+            libpath = os.path.join(wd, name + ext)
+            self.assertTrue(os.path.exists(libpath))
+            return libpath
+
+        libt1 = make_lib_path('libt1')
+        libt2_0 = make_lib_path('libt2_0')
+        libt2_1 = make_lib_path('libt2_1')
+
+        # Overwrite t2_0 with t2_1 to delete the definition of `use`.
+        shutil.copy(libt2_1, libt2_0)
+
+        # Launch a process and break
+        (target, process, thread, _) = lldbutil.run_to_source_breakpoint(self,
+                                                "break here",
+                                                lldb.SBFileSpec("main.cpp"))
+
+        # Load libt1; should fail unless we use RTLD_LAZY
+        error = lldb.SBError()
+        lib_spec = lldb.SBFileSpec('libt1' + ext)
+        paths = lldb.SBStringList()
+        paths.AppendString(wd)
+        out_spec = lldb.SBFileSpec()
+        token = process.LoadImageUsingPaths(lib_spec, paths, out_spec, error)
+        self.assertNotEqual(token, lldb.LLDB_INVALID_IMAGE_TOKEN, "Got a valid token")
+
+        # Calling `f1()` should return 5.
+        frame = thread.GetFrameAtIndex(0)
+        val = frame.EvaluateExpression("f1()")
+        self.assertTrue(val.IsValid())
+        self.assertEquals(val.GetValueAsSigned(-1), 5)
diff --git a/lldb/test/API/functionalities/load_lazy/categories b/lldb/test/API/functionalities/load_lazy/categories
new file mode 100644
index 0000000000000000000000000000000000000000..c00c25822e4c240bac722d6dc2ae987682606dcf
--- /dev/null
+++ b/lldb/test/API/functionalities/load_lazy/categories
@@ -0,0 +1 @@
+basic_process
diff --git a/lldb/test/API/functionalities/load_lazy/main.cpp b/lldb/test/API/functionalities/load_lazy/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ba45ee316cd42b60cd5377225555f6c5bed83c98
--- /dev/null
+++ b/lldb/test/API/functionalities/load_lazy/main.cpp
@@ -0,0 +1,3 @@
+int main() {
+  return 0; // break here
+}
diff --git a/lldb/test/API/functionalities/load_lazy/t1.c b/lldb/test/API/functionalities/load_lazy/t1.c
new file mode 100644
index 0000000000000000000000000000000000000000..e2fc213270626392bac69b4dc09694ca6195d134
--- /dev/null
+++ b/lldb/test/API/functionalities/load_lazy/t1.c
@@ -0,0 +1,3 @@
+extern void use();
+int f1() { return 5; }
+void f2() { use(); }
diff --git a/lldb/test/API/functionalities/load_lazy/t2_0.c b/lldb/test/API/functionalities/load_lazy/t2_0.c
new file mode 100644
index 0000000000000000000000000000000000000000..9fc1edfbf460f6b7ac00769e44eb23f3b2b43d44
--- /dev/null
+++ b/lldb/test/API/functionalities/load_lazy/t2_0.c
@@ -0,0 +1 @@
+void use() {}
diff --git a/clang/test/Driver/Inputs/multilib_armeb_linux_tree/usr/include/armeb-linux-gnueabi/.keep b/lldb/test/API/functionalities/load_lazy/t2_1.c
similarity index 100%
rename from clang/test/Driver/Inputs/multilib_armeb_linux_tree/usr/include/armeb-linux-gnueabi/.keep
rename to lldb/test/API/functionalities/load_lazy/t2_1.c
diff --git a/lldb/test/API/functionalities/scripted_process/Makefile b/lldb/test/API/functionalities/scripted_process/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..692ba1732285987d68c0ba2698a25f6347f606c7
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_process/Makefile
@@ -0,0 +1,4 @@
+C_SOURCES := main.c
+
+include Makefile.rules
+
diff --git a/lldb/test/API/functionalities/scripted_process/TestScriptedProcess.py b/lldb/test/API/functionalities/scripted_process/TestScriptedProcess.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5da07027aaf5bde9e4637833f42582248b0eef9
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_process/TestScriptedProcess.py
@@ -0,0 +1,45 @@
+"""
+Test python scripted process in lldb
+"""
+
+import os
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+from lldbsuite.test import lldbtest
+
+
+class PlatformProcessCrashInfoTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    def setUp(self):
+        TestBase.setUp(self)
+        self.source = "main.c"
+
+    def tearDown(self):
+        TestBase.tearDown(self)
+
+    def test_python_plugin_package(self):
+        """Test that the lldb python module has a `plugins.scripted_process`
+        package."""
+        self.expect('script import lldb.plugins',
+                    substrs=["ModuleNotFoundError"], matching=False)
+
+        self.expect('script dir(lldb.plugins)',
+                    substrs=["scripted_process"])
+
+        self.expect('script import lldb.plugins.scripted_process',
+                    substrs=["ModuleNotFoundError"], matching=False)
+
+        self.expect('script dir(lldb.plugins.scripted_process)',
+                    substrs=["ScriptedProcess"])
+
+        self.expect('script from lldb.plugins.scripted_process import ScriptedProcess',
+                    substrs=["ImportError"], matching=False)
+
+        self.expect('script dir(ScriptedProcess)',
+                    substrs=["launch"])
+
diff --git a/lldb/test/API/functionalities/scripted_process/main.c b/lldb/test/API/functionalities/scripted_process/main.c
new file mode 100644
index 0000000000000000000000000000000000000000..3e82b70c8f4d3458ea6098b3b2a4f8ce2f74803f
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_process/main.c
@@ -0,0 +1,5 @@
+#include <stdlib.h>
+
+int main() {
+  return 0; // break here
+}
diff --git a/lldb/test/API/functionalities/stop-on-sharedlibrary-load/Makefile b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..4abcab84eac2922c2dbe19e15c05b3f8b84333c8
--- /dev/null
+++ b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/Makefile
@@ -0,0 +1,16 @@
+CXX_SOURCES := main.cpp
+USE_LIBDL := 1
+
+a.out: lib_a lib_b
+
+include Makefile.rules
+
+lib_a:
+	$(MAKE) -f $(MAKEFILE_RULES) \
+		DYLIB_ONLY=YES DYLIB_CXX_SOURCES=a.cpp DYLIB_NAME=load_a
+
+lib_b:
+	$(MAKE) -f $(MAKEFILE_RULES) \
+		DYLIB_ONLY=YES DYLIB_CXX_SOURCES=b.cpp DYLIB_NAME=load_b
+
+
diff --git a/lldb/test/API/functionalities/stop-on-sharedlibrary-load/TestStopOnSharedlibraryEvents.py b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/TestStopOnSharedlibraryEvents.py
new file mode 100644
index 0000000000000000000000000000000000000000..18962d1443e0d74679d9ba09e9cd5d7393aacf02
--- /dev/null
+++ b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/TestStopOnSharedlibraryEvents.py
@@ -0,0 +1,99 @@
+""" Test that stop-on-sharedlibrary-events works and cooperates with breakpoints. """
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestStopOnSharedlibraryEvents(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @skipIfRemote
+    @skipUnlessDarwin
+    @no_debug_info_test
+    def test_stopping_breakpoints(self):
+        self.do_test()
+
+    @skipIfRemote
+    @skipUnlessDarwin
+    @no_debug_info_test
+    def test_auto_continue(self):
+        def auto_continue(bkpt):
+            bkpt.SetAutoContinue(True)
+        self.do_test(auto_continue)
+
+    @skipIfRemote
+    @no_debug_info_test
+    @skipUnlessDarwin
+    def test_failing_condition(self):
+        def condition(bkpt):
+            bkpt.SetCondition("1 == 2")
+        self.do_test(condition)
+        
+    @skipIfRemote
+    @skipUnlessDarwin
+    @no_debug_info_test
+    def test_continue_callback(self):
+        def bkpt_callback(bkpt):
+            bkpt.SetScriptCallbackBody("return False")
+        self.do_test(bkpt_callback)
+
+    def do_test(self, bkpt_modifier = None):
+        self.build()
+        main_spec = lldb.SBFileSpec("main.cpp")
+        # Launch and stop before the dlopen call.
+        target, process, thread, _ = lldbutil.run_to_source_breakpoint(self,
+                                                                  "// Set a breakpoint here", main_spec)
+
+        # Now turn on shared library events, continue and make sure we stop for the event.
+        self.runCmd("settings set target.process.stop-on-sharedlibrary-events 1")
+        self.addTearDownHook(lambda: self.runCmd(
+            "settings set target.process.stop-on-sharedlibrary-events 0"))
+
+        # Since I don't know how to check that we are at the "right place" to stop for
+        # shared library events, make an breakpoint after the load is done and
+        # make sure we don't stop there:
+        backstop_bkpt_1 = target.BreakpointCreateBySourceRegex("Set another here - we should not hit this one", main_spec)
+        self.assertGreater(backstop_bkpt_1.GetNumLocations(), 0, "Set our second breakpoint")
+        
+        process.Continue() 
+        self.assertEqual(process.GetState(), lldb.eStateStopped, "We didn't stop for the load")
+        self.assertEqual(backstop_bkpt_1.GetHitCount(), 0, "Hit our backstop breakpoint")
+        
+        # We should be stopped after the library is loaded, check that:
+        found_it = False
+        for module in target.modules:
+            if module.file.basename.find("load_a") > -1:
+                found_it = True
+                break
+        self.assertTrue(found_it, "Found the loaded module.")
+
+        # Now capture the place where we stopped so we can set a breakpoint and make
+        # sure the breakpoint there works correctly:
+        load_address = process.GetSelectedThread().frames[0].addr
+        load_bkpt = target.BreakpointCreateBySBAddress(load_address)
+        self.assertGreater(load_bkpt.GetNumLocations(), 0, "Set the load breakpoint")
+
+        backstop_bkpt_1.SetEnabled(False)
+
+        backstop_bkpt_2 = target.BreakpointCreateBySourceRegex("Set a third here - we should not hit this one", main_spec)
+        self.assertGreater(backstop_bkpt_2.GetNumLocations(), 0, "Set our third breakpoint")
+            
+        if bkpt_modifier == None:
+            process.Continue()
+            self.assertEqual(process.GetState(), lldb.eStateStopped, "We didn't stop for the load")
+            self.assertEqual(backstop_bkpt_2.GetHitCount(), 0, "Hit our backstop breakpoint")
+            self.assertEqual(thread.stop_reason, lldb.eStopReasonBreakpoint, "We attributed the stop to the breakpoint")
+            self.assertEqual(load_bkpt.GetHitCount(), 1, "We hit our breakpoint at the load address")
+        else:
+            bkpt_modifier(load_bkpt)
+            process.Continue()
+            self.assertEqual(process.GetState(), lldb.eStateStopped, "We didn't stop")
+            self.assertTrue(thread.IsValid(), "Our thread was no longer valid.")
+            self.assertEqual(thread.stop_reason, lldb.eStopReasonBreakpoint, "We didn't hit some breakpoint")
+            self.assertEqual(backstop_bkpt_2.GetHitCount(), 1, "We continued to the right breakpoint")
+
+        
+        
+        
+        
diff --git a/lldb/test/API/functionalities/stop-on-sharedlibrary-load/a.cpp b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/a.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b7b702c5d62d52f654a1d2cdae4065df18e48a39
--- /dev/null
+++ b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/a.cpp
@@ -0,0 +1,6 @@
+extern int a_has_a_function();
+
+int
+a_has_a_function() {
+  return 10;
+}
diff --git a/lldb/test/API/functionalities/stop-on-sharedlibrary-load/b.cpp b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/b.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5a347e60db3af206ea945c28fcce4cab78a2310a
--- /dev/null
+++ b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/b.cpp
@@ -0,0 +1,6 @@
+extern int b_has_a_function();
+
+int
+b_has_a_function() {
+  return 100;
+}
diff --git a/lldb/test/API/functionalities/stop-on-sharedlibrary-load/main.cpp b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..96b1e1df445bec57697b47ca6d98e5c95631e3c5
--- /dev/null
+++ b/lldb/test/API/functionalities/stop-on-sharedlibrary-load/main.cpp
@@ -0,0 +1,27 @@
+#include "dylib.h"
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int main(int argc, char const *argv[]) {
+  const char *a_name = "load_a";
+  void *a_dylib_handle = NULL;
+
+  a_dylib_handle = dylib_open(a_name); // Set a breakpoint here.
+  if (a_dylib_handle == NULL) { // Set another here - we should not hit this one
+    fprintf(stderr, "%s\n", dylib_last_error());
+    exit(1);
+  }
+
+  const char *b_name = "load_b";
+  void *b_dylib_handle = NULL;
+
+  b_dylib_handle = dylib_open(b_name);
+  if (b_dylib_handle == NULL) { // Set a third here - we should not hit this one
+    fprintf(stderr, "%s\n", dylib_last_error());
+    exit(1);
+  }
+
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/thread/exit_during_expression/TestExitDuringExpression.py b/lldb/test/API/functionalities/thread/exit_during_expression/TestExitDuringExpression.py
index 4ee65c85e8f1e7eb2f1bbf1a2c593f505d9dbd4b..dafc0a967605ccfa56f59194a664a845d958b419 100644
--- a/lldb/test/API/functionalities/thread/exit_during_expression/TestExitDuringExpression.py
+++ b/lldb/test/API/functionalities/thread/exit_during_expression/TestExitDuringExpression.py
@@ -15,7 +15,7 @@ class TestExitDuringExpression(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
     @skipIfWindows
-    @skipIf(oslist=["linux"], archs=["arm"], bugnumber="llvm.org/pr48414")
+    @skipIf(oslist=["linux"], archs=["arm", "aarch64"], bugnumber="llvm.org/pr48414")
     @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48414")
     @expectedFailureNetBSD
     def test_exit_before_one_thread_unwind(self):
diff --git a/lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py b/lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py
index 2ab36b57eaee848ac634baf04f49df5552770d72..5b34e74b410dfadccbcee66be4c6be3379784474 100644
--- a/lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py
+++ b/lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py
@@ -62,50 +62,58 @@ class ThreadStepOutTestCase(TestBase):
         """Test thread step out on one thread via Python API (dwarf)."""
         self.build(dictionary=self.getBuildFlags())
         self.step_out_test(self.step_out_with_python)
-
+        
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
         # Find the line number for our breakpoint.
         self.bkpt_string = '// Set breakpoint here'
         self.breakpoint = line_number('main.cpp', self.bkpt_string)       
-
-        self.step_out_destination = line_number(
-             'main.cpp', '// Expect to stop here after step-out.')
-
+        self.step_in_line = line_number('main.cpp', '// But we might still be here')
+        self.step_out_dest = line_number('main.cpp', '// Expect to stop here after step-out.')
+
+    def check_stepping_thread(self):
+        zeroth_frame = self.step_out_thread.frames[0]
+        line_entry = zeroth_frame.line_entry
+        self.assertTrue(line_entry.IsValid(), "Stopped at a valid line entry")
+        self.assertEqual("main.cpp", line_entry.file.basename, "Still in main.cpp")
+        # We can't really tell whether we stay on our line
+        # or get to the next line, it depends on whether there are any
+        # instructions between the call and the return.
+        line = line_entry.line
+        self.assertTrue(line == self.step_out_dest or line == self.step_in_line, "Stepped to the wrong line: {0}".format(line))
+        
     def step_out_single_thread_with_cmd(self):
+        other_threads = {}
+        for thread in self.process.threads:
+            if thread.GetIndexID() == self.step_out_thread.GetIndexID():
+                continue
+            other_threads[thread.GetIndexID()] = thread.frames[0].line_entry
+
+        # There should be other threads...
+        self.assertNotEqual(len(other_threads), 0)
         self.step_out_with_cmd("this-thread")
-        self.expect(
-            "thread backtrace all",
-            "Thread location after step out is correct",
-            substrs=[
-                "main.cpp:%d" %
-                self.step_out_destination,
-                "main.cpp:%d" %
-                self.breakpoint])
+        # The other threads should not have made progress:
+        for thread in self.process.threads:
+            index_id = thread.GetIndexID()
+            line_entry = other_threads.get(index_id)
+            if line_entry:
+                self.assertEqual(thread.frames[0].line_entry.file.basename, line_entry.file.basename, "Thread {0} moved by file".format(index_id))
+                self.assertEqual(thread.frames[0].line_entry.line, line_entry.line, "Thread {0} moved by line".format(index_id))
 
     def step_out_all_threads_with_cmd(self):
         self.step_out_with_cmd("all-threads")
-        self.expect(
-            "thread backtrace all",
-            "Thread location after step out is correct",
-            substrs=[
-                "main.cpp:%d" %
-                self.step_out_destination])
-
+                                        
     def step_out_with_cmd(self, run_mode):
         self.runCmd("thread select %d" % self.step_out_thread.GetIndexID())
         self.runCmd("thread step-out -m %s" % run_mode)
         self.expect("process status", "Expected stop reason to be step-out",
                     substrs=["stop reason = step out"])
 
-        self.expect(
-            "thread list",
-            "Selected thread did not change during step-out",
-            substrs=[
-                "* thread #%d" %
-                self.step_out_thread.GetIndexID()])
-
+        selected_thread = self.process.GetSelectedThread()
+        self.assertEqual(selected_thread.GetIndexID(), self.step_out_thread.GetIndexID(), "Step out changed selected thread.")
+        self.check_stepping_thread()
+                                        
     def step_out_with_python(self):
         self.step_out_thread.StepOut()
 
@@ -115,18 +123,12 @@ class ThreadStepOutTestCase(TestBase):
             reason,
             "Expected thread stop reason 'plancomplete', but got '%s'" %
             lldbutil.stop_reason_to_str(reason))
-
-        # Verify location after stepping out
-        frame = self.step_out_thread.GetFrameAtIndex(0)
-        desc = lldbutil.get_description(frame.GetLineEntry())
-        expect = "main.cpp:%d" % self.step_out_destination
-        self.assertTrue(
-            expect in desc, "Expected %s but thread stopped at %s" %
-            (expect, desc))
+        self.check_stepping_thread()
+                            
 
     def step_out_test(self, step_out_func):
         """Test single thread step out of a function."""
-        (self.inferior_target, self.inferior_process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+        (self.inferior_target, self.process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
             self, self.bkpt_string, lldb.SBFileSpec('main.cpp'), only_one_thread = False)
 
         # We hit the breakpoint on at least one thread.  If we hit it on both threads
@@ -135,13 +137,13 @@ class ThreadStepOutTestCase(TestBase):
         # the breakpoint:
 
         (breakpoint_threads, other_threads) = ([], [])
-        lldbutil.sort_stopped_threads(self.inferior_process,
+        lldbutil.sort_stopped_threads(self.process,
                                       breakpoint_threads=breakpoint_threads,
                                       other_threads=other_threads)
         if len(breakpoint_threads) == 1:
             success = thread.Suspend()
             self.assertTrue(success, "Couldn't suspend a thread")
-            bkpt_threads = lldbutil.continue_to_breakpoint(self.inferior_process,
+            bkpt_threads = lldbutil.continue_to_breakpoint(self.process,
                                                            bkpt)
             self.assertEqual(len(bkpt_threads), 1, "Second thread stopped")
             success = thread.Resume()
diff --git a/lldb/test/API/functionalities/thread/step_out/main.cpp b/lldb/test/API/functionalities/thread/step_out/main.cpp
index e7dd230d239ce6d9f1148823c44c025291d199be..824f1b6c912d0f4ebe713e928096ecb8bad6b09f 100644
--- a/lldb/test/API/functionalities/thread/step_out/main.cpp
+++ b/lldb/test/API/functionalities/thread/step_out/main.cpp
@@ -19,7 +19,7 @@ thread_func ()
     pseudo_barrier_wait(g_barrier);
 
     // Do something
-    step_out_of_here();
+    step_out_of_here(); // But we might still be here 
 
     // Return
     return NULL;  // Expect to stop here after step-out.
diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index 54a02453b174b2c884cfc7e099148a0f6074fe30..cbb457e9328c605ed2fc470fcd231b81dbe15726 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -17,10 +17,10 @@ config.name = 'lldb-api'
 config.suffixes = ['.py']
 
 # test_source_root: The root path where tests are located.
-# test_exec_root: The root path where tests should be run.
 config.test_source_root = os.path.dirname(__file__)
-config.test_exec_root = config.test_source_root
 
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = os.path.join(config.lldb_obj_root, 'test', 'API')
 
 def mkdir_p(path):
   import errno
diff --git a/lldb/test/API/lit.site.cfg.py.in b/lldb/test/API/lit.site.cfg.py.in
index 2e368325a9f09cf968267e0b8ee41c1a39ac3539..49ea94aacd119fc8cd6f7a98421e72bbacb9b77a 100644
--- a/lldb/test/API/lit.site.cfg.py.in
+++ b/lldb/test/API/lit.site.cfg.py.in
@@ -1,6 +1,5 @@
 @LIT_SITE_CFG_IN_HEADER@
 
-config.test_exec_root = "@LLDB_BINARY_DIR@"
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
diff --git a/lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py b/lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py
index d25ac26cf5fb47f938e8ab1d3c56153c131ee495..f44955ac838a6dc8c944df714ff849ff670746ea 100644
--- a/lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py
+++ b/lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py
@@ -6,6 +6,7 @@ from lldbsuite.test import lldbutil
 
 import json
 import platform
+import re
 
 @skipIfReproducer
 class TestAppleSimulatorOSType(gdbremote_testcase.GdbRemoteTestCaseBase):
@@ -46,9 +47,17 @@ class TestAppleSimulatorOSType(gdbremote_testcase.GdbRemoteTestCaseBase):
 
         # Launch the process using simctl
         self.assertIsNotNone(deviceUDID)
+
         exe_name = 'test_simulator_platform_{}'.format(platform_name)
         sdkroot = lldbutil.get_xcode_sdk_root(sdk)
         vers = lldbutil.get_xcode_sdk_version(sdk)
+
+        # Older versions of watchOS (<7.0) only support i386
+        if platform_name == 'watchos':
+            from distutils.version import LooseVersion
+            if LooseVersion(vers) < LooseVersion("7.0"):
+                arch = 'i386'
+
         triple = '-'.join([arch, 'apple', platform_name + vers, 'simulator'])
         version_min = '-m{}-simulator-version-min={}'.format(platform_name, vers)
         self.build(
@@ -140,5 +149,4 @@ class TestAppleSimulatorOSType(gdbremote_testcase.GdbRemoteTestCaseBase):
     @skipIfRemote
     def test_simulator_ostype_watchos(self):
         self.check_simulator_ostype(sdk='watchsimulator',
-                                    platform_name='watchos',
-                                    arch='i386')
+                                    platform_name='watchos')
diff --git a/lldb/test/API/tools/lldb-server/TestGdbRemote_vContThreads.py b/lldb/test/API/tools/lldb-server/TestGdbRemote_vContThreads.py
index 5278d59fb06a28b08901548facd33f2c2187f324..fc65e67393764d864b1fbde85ccb672d8a827eeb 100644
--- a/lldb/test/API/tools/lldb-server/TestGdbRemote_vContThreads.py
+++ b/lldb/test/API/tools/lldb-server/TestGdbRemote_vContThreads.py
@@ -31,6 +31,36 @@ class TestGdbRemote_vContThreads(gdbremote_testcase.GdbRemoteTestCaseBase):
         self.reset_test_sequence()
         return threads
 
+    def send_and_check_signal(self, vCont_data, threads):
+        self.test_sequence.add_log_lines([
+            "read packet: $vCont;{0}#00".format(vCont_data),
+            {"type": "output_match",
+             "regex": self.maybe_strict_output_regex(
+                 len(threads) *
+                 r"received SIGUSR1 on thread id: ([0-9a-f]+)\r\n"),
+             "capture": dict((i, "tid{0}".format(i)) for i
+                             in range(1, len(threads)+1)),
+             },
+        ], True)
+
+        context = self.expect_gdbremote_sequence()
+        self.assertIsNotNone(context)
+        tids = sorted(int(context["tid{0}".format(x)], 16)
+                      for x in range(1, len(threads)+1))
+        self.assertEqual(tids, sorted(threads))
+
+    @skipIfWindows
+    @expectedFailureNetBSD
+    @expectedFailureDarwin # No signals delivered
+    def test_signal_process_without_tid(self):
+        self.build()
+        self.set_inferior_startup_launch()
+
+        threads = self.start_threads(1)
+        self.send_and_check_signal(
+            "C{0:x}".format(lldbutil.get_signal_number('SIGUSR1')),
+            threads)
+
     @skipUnlessPlatform(["netbsd"])
     @expectedFailureNetBSD
     def test_signal_one_thread(self):
@@ -39,32 +69,24 @@ class TestGdbRemote_vContThreads(gdbremote_testcase.GdbRemoteTestCaseBase):
 
         threads = self.start_threads(1)
         # try sending a signal to one of the two threads
-        self.test_sequence.add_log_lines([
-            "read packet: $vCont;C{0:x}:{1:x};c#00".format(
-                lldbutil.get_signal_number('SIGUSR1'), threads[0]),
-            {"direction": "send", "regex": r"^\$W00#b7$"},
-        ], True)
-
-        context = self.expect_gdbremote_sequence()
-        self.assertIsNotNone(context)
+        self.send_and_check_signal(
+            "C{0:x}:{1:x};c".format(lldbutil.get_signal_number('SIGUSR1')),
+            threads[:1])
 
-    @skipUnlessPlatform(["netbsd"])
+    @skipIfWindows
     @expectedFailureNetBSD
+    @expectedFailureDarwin # Only one signal delivered
     def test_signal_all_threads(self):
         self.build()
         self.set_inferior_startup_launch()
 
         threads = self.start_threads(1)
         # try sending a signal to two threads (= the process)
-        self.test_sequence.add_log_lines([
-            "read packet: $vCont;C{0:x}:{1:x};C{0:x}:{2:x}#00".format(
+        self.send_and_check_signal(
+            "C{0:x}:{1:x};C{0:x}:{2:x}".format(
                 lldbutil.get_signal_number('SIGUSR1'),
-                threads[0], threads[1]),
-            {"direction": "send", "regex": r"^\$W00#b7$"},
-        ], True)
-
-        context = self.expect_gdbremote_sequence()
-        self.assertIsNotNone(context)
+                *threads),
+            threads)
 
     @skipUnlessPlatform(["netbsd"])
     def test_signal_two_of_three_threads(self):
diff --git a/lldb/test/API/tools/lldb-server/main.cpp b/lldb/test/API/tools/lldb-server/main.cpp
index 4a877deaafbdca6563de9fa46d68284b132a6fff..8a14c11075f13f8f605fbfc241249215bfac6730 100644
--- a/lldb/test/API/tools/lldb-server/main.cpp
+++ b/lldb/test/API/tools/lldb-server/main.cpp
@@ -70,25 +70,23 @@ static void print_pid() {
 #endif
 }
 
-static void print_thread_id() {
+static uint64_t get_thread_id() {
 // Put in the right magic here for your platform to spit out the thread id (tid)
-// that debugserver/lldb-gdbserver would see as a TID. Otherwise, let the else
-// clause print out the unsupported text so that the unit test knows to skip
-// verifying thread ids.
+// that debugserver/lldb-gdbserver would see as a TID.
 #if defined(__APPLE__)
   __uint64_t tid = 0;
   pthread_threadid_np(pthread_self(), &tid);
-  printf("%" PRIx64, tid);
+  return tid;
 #elif defined(__linux__)
   // This is a call to gettid() via syscall.
-  printf("%" PRIx64, static_cast<uint64_t>(syscall(__NR_gettid)));
+  return syscall(__NR_gettid);
 #elif defined(__NetBSD__)
   // Technically lwpid_t is 32-bit signed integer
-  printf("%" PRIx64, static_cast<uint64_t>(_lwp_self()));
+  return static_cast<uint64_t>(_lwp_self());
 #elif defined(_WIN32)
-  printf("%" PRIx64, static_cast<uint64_t>(::GetCurrentThreadId()));
+  return static_cast<uint64_t>(::GetCurrentThreadId());
 #else
-  printf("{no-tid-support}");
+  return -1;
 #endif
 }
 
@@ -109,15 +107,12 @@ static void signal_handler(int signo) {
   }
 
   // Print notice that we received the signal on a given thread.
-  {
-    std::lock_guard<std::mutex> lock(g_print_mutex);
-    if (signal_name)
-      printf("received %s on thread id: ", signal_name);
-    else
-      printf("received signo %d (%s) on thread id: ", signo, strsignal(signo));
-    print_thread_id();
-    printf("\n");
-  }
+  char buf[100];
+  if (signal_name)
+    snprintf(buf, sizeof(buf), "received %s on thread id: %" PRIx64 "\n", signal_name, get_thread_id());
+  else
+    snprintf(buf, sizeof(buf), "received signo %d (%s) on thread id: %" PRIx64 "\n", signo, strsignal(signo), get_thread_id());
+  write(STDOUT_FILENO, buf, strlen(buf));
 
   // Reset the signal handler if we're one of the expected signal handlers.
   switch (signo) {
@@ -195,9 +190,7 @@ static void *thread_func(void *arg) {
   const int this_thread_index = s_thread_index++;
   if (g_print_thread_ids) {
     std::lock_guard<std::mutex> lock(g_print_mutex);
-    printf("thread %d id: ", this_thread_index);
-    print_thread_id();
-    printf("\n");
+    printf("thread %d id: %" PRIx64 "\n", this_thread_index, get_thread_id());
   }
 
   if (g_threads_do_segfault) {
@@ -229,9 +222,7 @@ static void *thread_func(void *arg) {
 
     {
       std::lock_guard<std::mutex> lock(g_print_mutex);
-      printf("thread ");
-      print_thread_id();
-      printf(": past SIGSEGV\n");
+      printf("thread %" PRIx64 ": past SIGSEGV\n", get_thread_id());
     }
   }
 
@@ -362,9 +353,7 @@ int main(int argc, char **argv) {
         // And announce us.
         {
           std::lock_guard<std::mutex> lock(g_print_mutex);
-          printf("thread 0 id: ");
-          print_thread_id();
-          printf("\n");
+          printf("thread 0 id: %" PRIx64 "\n", get_thread_id());
         }
       } else if (std::strstr(argv[i] + strlen(THREAD_PREFIX),
                              THREAD_COMMAND_SEGFAULT)) {
diff --git a/lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py b/lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py
index 2de3ed9e1e98214607fd67ec487271cce796bff5..91b2ae783048300aa1bf459e8597fe451e5f3dd8 100644
--- a/lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py
+++ b/lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py
@@ -29,6 +29,7 @@ class TestVSCode_launch(lldbvscode_testcase.VSCodeTestCaseBase):
     @skipIfDarwin
     @skipIfWindows
     @skipIfRemote
+    @skipIf(oslist=["linux"], archs=["arm"])
     def test_launch(self):
         """
             This test launches a process that would creates a file, but we disconnect
diff --git a/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py b/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
index 85837d51c49738fde0ee236eda67524dbb182134..aceed56fe249138a66f6f544ab3b0db3647cbca0 100644
--- a/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
+++ b/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
@@ -440,7 +440,7 @@ class TestVSCode_launch(lldbvscode_testcase.VSCodeTestCaseBase):
         '''
         self.build_and_create_debug_adaptor()
         program = self.getBuildArtifact("a.out")
-        
+
         terminateCommands = ['expr 4+2']
         self.launch(program=program,
                     terminateCommands=terminateCommands)
@@ -450,3 +450,63 @@ class TestVSCode_launch(lldbvscode_testcase.VSCodeTestCaseBase):
         self.vscode.request_disconnect(terminateDebuggee=True)
         output = self.collect_console(duration=1.0)
         self.verify_commands('terminateCommands', output, terminateCommands)
+
+
+    @skipIfWindows
+    @skipIfRemote
+    @skipIf(oslist=["linux"])
+    def test_progress_events(self):
+        '''
+            Tests the progress events to ensure we are receiving them.
+        '''
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        # Set a breakpoint at 'main'. This will cause all of the symbol tables
+        # for all modules in LLDB to be parsed and we should get a progress
+        # event for each shared library.
+        breakpoint_ids = self.set_function_breakpoints(['main'])
+        self.continue_to_breakpoints(breakpoint_ids)
+        # Make sure we at least got some progress events
+        self.assertTrue(len(self.vscode.progress_events) > 0)
+        # Track all 'progressStart' events by saving all 'progressId' values.
+        progressStart_ids = set()
+        # Track all 'progressEnd' events by saving all 'progressId' values.
+        progressEnd_ids = set()
+        # We will watch for events whose title starts with
+        # 'Parsing symbol table for ' and we will save the remainder of the
+        # line which will contain the shared library basename. Since we set a
+        # breakpoint by name for 'main', we will expect to see progress events
+        # for all shared libraries that say that the symbol table is being
+        # parsed.
+        symtab_progress_shlibs = set()
+        # Get a list of modules in the current target so we can verify that
+        # we do in fact get a progress event for each shared library.
+        target_shlibs = self.vscode.get_modules()
+
+        # Iterate over all progress events and save all start and end IDs, and
+        # remember any shared libraries that got symbol table parsing progress
+        # events.
+        for progress_event in self.vscode.progress_events:
+            event_type = progress_event['event']
+            if event_type == 'progressStart':
+                progressStart_ids.add(progress_event['body']['progressId'])
+                title = progress_event['body']['title']
+                if title.startswith('Parsing symbol table for '):
+                    symtab_progress_shlibs.add(title[25:])
+            if event_type == 'progressEnd':
+                progressEnd_ids.add(progress_event['body']['progressId'])
+        # Make sure for each 'progressStart' event, we got a matching
+        # 'progressEnd' event.
+        self.assertTrue(progressStart_ids == progressEnd_ids,
+                        ('Make sure we got a "progressEnd" for each '
+                         '"progressStart" event that we have.'))
+
+        ignored_libraries = {"[vdso]"}
+
+        # Verify we got a symbol table parsing progress event for each shared
+        # library in our target.
+        for target_shlib_basename in target_shlibs.keys():
+            if target_shlib_basename in ignored_libraries:
+                continue
+            self.assertIn(target_shlib_basename, symtab_progress_shlibs,
+                            'Make sure we got a symbol table progress event for "%s"' % (target_shlib_basename))
diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt
index 9944e37a46fc93b49be502b2397fe404457adee3..c6b01c66a0ef2a1dead957b7575689561ccdb84b 100644
--- a/lldb/test/CMakeLists.txt
+++ b/lldb/test/CMakeLists.txt
@@ -185,19 +185,13 @@ configure_lit_site_cfg(
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
 
-add_lit_testsuites(LLDB
-  ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS lldb-test-deps)
-
-add_lit_testsuite(check-lldb-lit "Running lldb lit test suite"
+add_lit_testsuite(check-lldb "Running lldb lit test suite"
   ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS lldb-test-deps)
-set_target_properties(check-lldb-lit PROPERTIES FOLDER "lldb tests")
-
-add_custom_target(check-lldb)
-add_dependencies(check-lldb lldb-test-deps)
-set_target_properties(check-lldb PROPERTIES FOLDER "lldb misc")
-add_dependencies(check-lldb check-lldb-lit)
+  DEPENDS
+    lldb-api-test-deps
+    lldb-shell-test-deps
+    lldb-unit-test-deps)
+set_target_properties(check-lldb PROPERTIES FOLDER "lldb tests")
 
 # Add a lit test suite that runs the API & shell test while capturing a
 # reproducer.
@@ -220,29 +214,6 @@ add_lit_testsuite(check-lldb-reproducers
   DEPENDS lldb-test-deps)
 add_dependencies(check-lldb-reproducers check-lldb-reproducers-capture)
 
-# Targets for running the test suite on the different Apple simulators.
-add_lit_testsuite(check-lldb-simulator-ios
-  "Running lldb test suite on the iOS simulator"
-  ${CMAKE_CURRENT_BINARY_DIR}/API
-  PARAMS "lldb-run-with-simulator=ios"
-  EXCLUDE_FROM_CHECK_ALL
-  DEPENDS lldb-test-deps)
-
-add_lit_testsuite(check-lldb-simulator-watchos
-  "Running lldb test suite on the watchOS simulator"
-  ${CMAKE_CURRENT_BINARY_DIR}/API
-  PARAMS "lldb-run-with-simulator=watchos"
-  EXCLUDE_FROM_CHECK_ALL
-  DEPENDS lldb-test-deps)
-
-add_lit_testsuite(check-lldb-simulator-tvos
-  "Running lldb test suite on the tvOS simulator"
-  ${CMAKE_CURRENT_BINARY_DIR}/API
-  PARAMS "lldb-run-with-simulator=tvos"
-  EXCLUDE_FROM_CHECK_ALL
-  DEPENDS lldb-test-deps)
-
-
 if(LLDB_BUILT_STANDALONE)
   # This has to happen *AFTER* add_lit_testsuite.
   if (EXISTS ${LLVM_MAIN_SRC_DIR}/utils/llvm-lit)
diff --git a/lldb/test/Shell/CMakeLists.txt b/lldb/test/Shell/CMakeLists.txt
index d203f1e093c74cb72a112fae36624c33298a4baf..f0d7b9a34651bd48af9fbe58072a69adc1f82cf9 100644
--- a/lldb/test/Shell/CMakeLists.txt
+++ b/lldb/test/Shell/CMakeLists.txt
@@ -1,4 +1,10 @@
-# Configure the Shell test suite.
+add_custom_target(lldb-shell-test-deps)
+add_dependencies(lldb-shell-test-deps lldb-test-deps)
+
+add_lit_testsuites(LLDB-SHELL
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS lldb-shell-test-deps)
+
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
@@ -8,10 +14,7 @@ configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit-lldb-init.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit-lldb-init)
 
-if (CMAKE_GENERATOR STREQUAL "Xcode")
-  # Xcode does not get the auto-generated targets. We need to create
-  # check-lldb-shell manually.
-  add_lit_testsuite(check-lldb-shell "Running lldb shell test suite"
-    ${CMAKE_CURRENT_BINARY_DIR}
-    DEPENDS lldb-test-deps)
-endif()
+add_lit_testsuite(check-lldb-shell "Running lldb shell test suite"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS lldb-shell-test-deps)
diff --git a/lldb/test/Shell/Error/assert.test b/lldb/test/Shell/Error/assert.test
new file mode 100644
index 0000000000000000000000000000000000000000..2585e64f39529c0a8e6f5a0613a6771fd5d3be12
--- /dev/null
+++ b/lldb/test/Shell/Error/assert.test
@@ -0,0 +1,4 @@
+# REQUIRES: asserts
+# RUN: not --crash lldb-test assert > %t.error 2>&1
+# RUN: cat %t.error | FileCheck %s
+# CHECK: "{{lldb_assert failed|lldb-test assert}}"' failed.
diff --git a/lldb/test/Shell/SymbolFile/DWARF/dwarf5-debug_line-file-index.s b/lldb/test/Shell/SymbolFile/DWARF/dwarf5-debug_line-file-index.s
new file mode 100644
index 0000000000000000000000000000000000000000..54d07df1043167fc4157d238a13a10b080378e39
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/dwarf5-debug_line-file-index.s
@@ -0,0 +1,86 @@
+# Test handling of DWARF5 file index 0.
+# REQUIRES: x86
+
+# RUN: llvm-mc -filetype=obj -o %t -triple x86_64-pc-linux %s
+# RUN: %lldb %t -o "image lookup -f hello.c -l 1" \
+# RUN:   -o exit | FileCheck %s
+
+# CHECK: 2 matches found in hello.c:1
+	.text
+.Lfunc_begin0:
+	.file	0 "." "hello.c"
+	.loc	0 1 0                           # hello.c:1:0
+		nop
+	.loc	0 1 13 prologue_end             # hello.c:1:13
+		nop
+.Lfunc_end0:
+	.section	.debug_abbrev,"",@progbits
+	.byte	1                               # Abbreviation Code
+	.byte	17                              # DW_TAG_compile_unit
+	.byte	0                               # DW_CHILDREN_no
+	.byte	37                              # DW_AT_producer
+	.byte	37                              # DW_FORM_strx1
+	.byte	19                              # DW_AT_language
+	.byte	5                               # DW_FORM_data2
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	114                             # DW_AT_str_offsets_base
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	16                              # DW_AT_stmt_list
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	27                              # DW_AT_comp_dir
+	.byte	37                              # DW_FORM_strx1
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	115                             # DW_AT_addr_base
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	0                               # EOM(3)
+	.section	.debug_info,"",@progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+	.short	5                               # DWARF version number
+	.byte	1                               # DWARF Unit Type
+	.byte	8                               # Address Size (in bytes)
+	.long	.debug_abbrev                   # Offset Into Abbrev. Section
+	.byte	1                               # Abbrev [1] 0xc:0x2b DW_TAG_compile_unit
+	.byte	0                               # DW_AT_producer
+	.short	12                              # DW_AT_language
+	.byte	1                               # DW_AT_name
+	.long	.Lstr_offsets_base0             # DW_AT_str_offsets_base
+	.long	.Lline_table_start0             # DW_AT_stmt_list
+	.byte	2                               # DW_AT_comp_dir
+	.byte	0                               # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.long	.Laddr_table_base0              # DW_AT_addr_base
+.Ldebug_info_end0:
+	.section	.debug_str_offsets,"",@progbits
+	.long	12                              # Length of String Offsets Set
+	.short	5
+	.short	0
+.Lstr_offsets_base0:
+	.section	.debug_str,"MS",@progbits,1
+.Linfo_string0:
+	.asciz	"" # string offset=0
+.Linfo_string1:
+	.asciz	"hello.c"                       # string offset=101
+.Linfo_string2:
+	.asciz	"."                             # string offset=109
+	.section	.debug_str_offsets,"",@progbits
+	.long	.Linfo_string0
+	.long	.Linfo_string1
+	.long	.Linfo_string2
+	.section	.debug_addr,"",@progbits
+.Ldebug_addr_start0:
+	.short	5                               # DWARF version number
+	.byte	8                               # Address size
+	.byte	0                               # Segment selector size
+.Laddr_table_base0:
+	.quad	.Lfunc_begin0
+.Ldebug_addr_end0:
+	.section	.debug_line,"",@progbits
+.Lline_table_start0:
diff --git a/lldb/test/Shell/helper/toolchain.py b/lldb/test/Shell/helper/toolchain.py
index 59b078411c1c492f471de03acaec1af2fcb50186..6e564f31dbd37199acadd4c3d1537e9938a7c102 100644
--- a/lldb/test/Shell/helper/toolchain.py
+++ b/lldb/test/Shell/helper/toolchain.py
@@ -11,7 +11,7 @@ from lit.llvm.subst import ToolSubst
 
 
 def _get_lldb_init_path(config):
-    return os.path.join(config.test_exec_root, 'Shell', 'lit-lldb-init')
+    return os.path.join(config.test_exec_root, 'lit-lldb-init')
 
 
 def _disallow(config, execName):
diff --git a/lldb/test/Shell/lit.cfg.py b/lldb/test/Shell/lit.cfg.py
index 1e793846023a427f410d3660895313709dcfe33f..c4730b284b23c8edb3a35800227ecb0a22f45389 100644
--- a/lldb/test/Shell/lit.cfg.py
+++ b/lldb/test/Shell/lit.cfg.py
@@ -36,7 +36,7 @@ config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']
 config.test_source_root = os.path.dirname(__file__)
 
 # test_exec_root: The root path where tests should be run.
-config.test_exec_root = os.path.join(config.lldb_obj_root, 'test')
+config.test_exec_root = os.path.join(config.lldb_obj_root, 'test', 'Shell')
 
 # Propagate environment vars.
 llvm_config.with_system_environment([
diff --git a/lldb/test/Unit/CMakeLists.txt b/lldb/test/Unit/CMakeLists.txt
index e9b3d9e35d74aca054a687298dda5de14a1bb558..3233c0873c1fb6cf944753293ae317ca8b8f690f 100644
--- a/lldb/test/Unit/CMakeLists.txt
+++ b/lldb/test/Unit/CMakeLists.txt
@@ -1,7 +1,17 @@
-# Configure the Unit test suite.
+add_custom_target(lldb-unit-test-deps)
+add_dependencies(lldb-unit-test-deps lldb-test-deps)
+
+add_lit_testsuites(LLDB-UNIT
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS lldb-unit-test-deps)
+
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
 
+add_lit_testsuite(check-lldb-unit "Running lldb unit test suite"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS lldb-unit-test-deps)
diff --git a/lldb/test/Unit/lit.site.cfg.py.in b/lldb/test/Unit/lit.site.cfg.py.in
index c0627b772362f6c178cb8699bc12200e99f99805..c0a881ce5c635facb6d11606baa99e3169462c65 100644
--- a/lldb/test/Unit/lit.site.cfg.py.in
+++ b/lldb/test/Unit/lit.site.cfg.py.in
@@ -1,6 +1,5 @@
 @LIT_SITE_CFG_IN_HEADER@
 
-config.test_exec_root = "@LLDB_BINARY_DIR@"
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
diff --git a/lldb/test/lit.site.cfg.py.in b/lldb/test/lit.site.cfg.py.in
index 24c0a4d5aa04c3f54139a847ae2bb67624360d8f..4bd4b52b91aa4629040a892e5feda264968e4279 100644
--- a/lldb/test/lit.site.cfg.py.in
+++ b/lldb/test/lit.site.cfg.py.in
@@ -1,6 +1,5 @@
 @LIT_SITE_CFG_IN_HEADER@
 
-config.test_exec_root = "@LLDB_BINARY_DIR@"
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
diff --git a/lldb/tools/debugserver/source/MacOSX/ThreadInfo.h b/lldb/tools/debugserver/source/MacOSX/ThreadInfo.h
index a114a47eaab839e9df318197d10b5233e5cf3569..592d50fd4284d95d0b3dfb4b1819537ee5ed39fc 100644
--- a/lldb/tools/debugserver/source/MacOSX/ThreadInfo.h
+++ b/lldb/tools/debugserver/source/MacOSX/ThreadInfo.h
@@ -20,6 +20,6 @@ public:
   std::string printable_name;
   uint32_t enum_value;
 };
-};
+}
 
 #endif // LLDB_TOOLS_DEBUGSERVER_SOURCE_MACOSX_THREADINFO_H
diff --git a/lldb/tools/lldb-test/lldb-test.cpp b/lldb/tools/lldb-test/lldb-test.cpp
index 842a951f384b8f9c72cedac35cb1e0ae2cf193b8..1109a6bb6558040fdb920f6e411e96eb0ab373b7 100644
--- a/lldb/tools/lldb-test/lldb-test.cpp
+++ b/lldb/tools/lldb-test/lldb-test.cpp
@@ -29,6 +29,7 @@
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/DataExtractor.h"
+#include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/StreamString.h"
 
@@ -57,6 +58,7 @@ cl::SubCommand ObjectFileSubcommand("object-file",
                                     "Display LLDB object file information");
 cl::SubCommand SymbolsSubcommand("symbols", "Dump symbols for an object file");
 cl::SubCommand IRMemoryMapSubcommand("ir-memory-map", "Test IRMemoryMap");
+cl::SubCommand AssertSubcommand("assert", "Test assert handling");
 
 cl::opt<std::string> Log("log", cl::desc("Path to a log file"), cl::init(""),
                          cl::sub(BreakpointSubcommand),
@@ -236,6 +238,9 @@ bool evalFree(StringRef Line, IRMemoryMapTestState &State);
 int evaluateMemoryMapCommands(Debugger &Dbg);
 } // namespace irmemorymap
 
+namespace assert {
+int lldb_assert(Debugger &Dbg);
+} // namespace assert
 } // namespace opts
 
 std::vector<CompilerContext> parseCompilerContext() {
@@ -1077,6 +1082,11 @@ int opts::irmemorymap::evaluateMemoryMapCommands(Debugger &Dbg) {
   return 0;
 }
 
+int opts::assert::lldb_assert(Debugger &Dbg) {
+  lldbassert(false && "lldb-test assert");
+  return 1;
+}
+
 int main(int argc, const char *argv[]) {
   StringRef ToolName = argv[0];
   sys::PrintStackTraceOnErrorSignal(ToolName);
@@ -1120,6 +1130,8 @@ int main(int argc, const char *argv[]) {
     return opts::symbols::dumpSymbols(*Dbg);
   if (opts::IRMemoryMapSubcommand)
     return opts::irmemorymap::evaluateMemoryMapCommands(*Dbg);
+  if (opts::AssertSubcommand)
+    return opts::assert::lldb_assert(*Dbg);
 
   WithColor::error() << "No command specified.\n";
   return 1;
diff --git a/lldb/tools/lldb-vscode/VSCode.cpp b/lldb/tools/lldb-vscode/VSCode.cpp
index e9fdc17f414743803ae76200ed61e2fb99c1c48f..8dc7d28a25002880d29dcca55bb5287a75525a87 100644
--- a/lldb/tools/lldb-vscode/VSCode.cpp
+++ b/lldb/tools/lldb-vscode/VSCode.cpp
@@ -6,8 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <chrono>
 #include <fstream>
 #include <mutex>
+#include <sstream>
 #include <stdarg.h>
 
 #include "LLDBUtils.h"
@@ -225,6 +227,146 @@ void VSCode::SendOutput(OutputType o, const llvm::StringRef output) {
   SendJSON(llvm::json::Value(std::move(event)));
 }
 
+// interface ProgressStartEvent extends Event {
+//   event: 'progressStart';
+//
+//   body: {
+//     /**
+//      * An ID that must be used in subsequent 'progressUpdate' and
+//      'progressEnd'
+//      * events to make them refer to the same progress reporting.
+//      * IDs must be unique within a debug session.
+//      */
+//     progressId: string;
+//
+//     /**
+//      * Mandatory (short) title of the progress reporting. Shown in the UI to
+//      * describe the long running operation.
+//      */
+//     title: string;
+//
+//     /**
+//      * The request ID that this progress report is related to. If specified a
+//      * debug adapter is expected to emit
+//      * progress events for the long running request until the request has
+//      been
+//      * either completed or cancelled.
+//      * If the request ID is omitted, the progress report is assumed to be
+//      * related to some general activity of the debug adapter.
+//      */
+//     requestId?: number;
+//
+//     /**
+//      * If true, the request that reports progress may be canceled with a
+//      * 'cancel' request.
+//      * So this property basically controls whether the client should use UX
+//      that
+//      * supports cancellation.
+//      * Clients that don't support cancellation are allowed to ignore the
+//      * setting.
+//      */
+//     cancellable?: boolean;
+//
+//     /**
+//      * Optional, more detailed progress message.
+//      */
+//     message?: string;
+//
+//     /**
+//      * Optional progress percentage to display (value range: 0 to 100). If
+//      * omitted no percentage will be shown.
+//      */
+//     percentage?: number;
+//   };
+// }
+//
+// interface ProgressUpdateEvent extends Event {
+//   event: 'progressUpdate';
+//
+//   body: {
+//     /**
+//      * The ID that was introduced in the initial 'progressStart' event.
+//      */
+//     progressId: string;
+//
+//     /**
+//      * Optional, more detailed progress message. If omitted, the previous
+//      * message (if any) is used.
+//      */
+//     message?: string;
+//
+//     /**
+//      * Optional progress percentage to display (value range: 0 to 100). If
+//      * omitted no percentage will be shown.
+//      */
+//     percentage?: number;
+//   };
+// }
+//
+// interface ProgressEndEvent extends Event {
+//   event: 'progressEnd';
+//
+//   body: {
+//     /**
+//      * The ID that was introduced in the initial 'ProgressStartEvent'.
+//      */
+//     progressId: string;
+//
+//     /**
+//      * Optional, more detailed progress message. If omitted, the previous
+//      * message (if any) is used.
+//      */
+//     message?: string;
+//   };
+// }
+
+void VSCode::SendProgressEvent(uint64_t progress_id, const char *message,
+                               uint64_t completed, uint64_t total) {
+  enum ProgressEventType {
+    progressInvalid,
+    progressStart,
+    progressUpdate,
+    progressEnd
+  };
+  const char *event_name = nullptr;
+  ProgressEventType event_type = progressInvalid;
+  if (completed == 0) {
+    event_type = progressStart;
+    event_name = "progressStart";
+  } else if (completed == total) {
+    event_type = progressEnd;
+    event_name = "progressEnd";
+  } else if (completed < total) {
+    event_type = progressUpdate;
+    event_name = "progressUpdate";
+  }
+  if (event_type == progressInvalid)
+    return;
+
+  llvm::json::Object event(CreateEventObject(event_name));
+  llvm::json::Object body;
+  std::string progress_id_str;
+  llvm::raw_string_ostream progress_id_strm(progress_id_str);
+  progress_id_strm << progress_id;
+  progress_id_strm.flush();
+  body.try_emplace("progressId", progress_id_str);
+  if (event_type == progressStart) {
+    EmplaceSafeString(body, "title", message);
+    body.try_emplace("cancellable", false);
+  }
+  auto now = std::chrono::duration<double>(
+      std::chrono::system_clock::now().time_since_epoch());
+  std::string timestamp(llvm::formatv("{0:f9}", now.count()));
+  EmplaceSafeString(body, "timestamp", timestamp);
+
+  if (0 < total && total < UINT64_MAX) {
+    uint32_t percentage = (uint32_t)(((float)completed / (float)total) * 100.0);
+    body.try_emplace("percentage", percentage);
+  }
+  event.try_emplace("body", std::move(body));
+  SendJSON(llvm::json::Value(std::move(event)));
+}
+
 void __attribute__((format(printf, 3, 4)))
 VSCode::SendFormattedOutput(OutputType o, const char *format, ...) {
   char buffer[1024];
diff --git a/lldb/tools/lldb-vscode/VSCode.h b/lldb/tools/lldb-vscode/VSCode.h
index a2e1cac8ecf9168fd6519ea63d0ea757f32dd389..0897e002238176ad978c83585e72f744bb09ff1d 100644
--- a/lldb/tools/lldb-vscode/VSCode.h
+++ b/lldb/tools/lldb-vscode/VSCode.h
@@ -68,7 +68,10 @@ typedef llvm::DenseMap<uint32_t, SourceBreakpoint> SourceBreakpointMap;
 typedef llvm::StringMap<FunctionBreakpoint> FunctionBreakpointMap;
 enum class OutputType { Console, Stdout, Stderr, Telemetry };
 
-enum VSCodeBroadcasterBits { eBroadcastBitStopEventThread = 1u << 0 };
+enum VSCodeBroadcasterBits {
+  eBroadcastBitStopEventThread = 1u << 0,
+  eBroadcastBitStopProgressThread = 1u << 1
+};
 
 typedef void (*RequestCallback)(const llvm::json::Object &command);
 
@@ -91,6 +94,7 @@ struct VSCode {
   int64_t num_locals;
   int64_t num_globals;
   std::thread event_thread;
+  std::thread progress_event_thread;
   std::unique_ptr<std::ofstream> log;
   llvm::DenseMap<lldb::addr_t, int64_t> addr_to_source_ref;
   llvm::DenseMap<int64_t, SourceReference> source_map;
@@ -132,6 +136,9 @@ struct VSCode {
 
   void SendOutput(OutputType o, const llvm::StringRef output);
 
+  void SendProgressEvent(uint64_t progress_id, const char *message,
+                         uint64_t completed, uint64_t total);
+
   void __attribute__((format(printf, 3, 4)))
   SendFormattedOutput(OutputType o, const char *format, ...);
 
diff --git a/lldb/tools/lldb-vscode/lldb-vscode.cpp b/lldb/tools/lldb-vscode/lldb-vscode.cpp
index 1097decbf55640596ae159b1bdc9b487776f7fcd..243461780cfd70f404268ee8c882db144ce22894 100644
--- a/lldb/tools/lldb-vscode/lldb-vscode.cpp
+++ b/lldb/tools/lldb-vscode/lldb-vscode.cpp
@@ -349,6 +349,34 @@ void SendStdOutStdErr(lldb::SBProcess &process) {
     g_vsc.SendOutput(OutputType::Stderr, llvm::StringRef(buffer, count));
 }
 
+void ProgressEventThreadFunction() {
+  lldb::SBListener listener("lldb-vscode.progress.listener");
+  g_vsc.debugger.GetBroadcaster().AddListener(
+      listener, lldb::SBDebugger::eBroadcastBitProgress);
+  g_vsc.broadcaster.AddListener(listener, eBroadcastBitStopProgressThread);
+  lldb::SBEvent event;
+  bool done = false;
+  while (!done) {
+    if (listener.WaitForEvent(1, event)) {
+      const auto event_mask = event.GetType();
+      if (event.BroadcasterMatchesRef(g_vsc.broadcaster)) {
+        if (event_mask & eBroadcastBitStopProgressThread) {
+          done = true;
+        }
+      } else {
+        uint64_t progress_id = 0;
+        uint64_t completed = 0;
+        uint64_t total = 0;
+        bool is_debugger_specific = false;
+        const char *message = lldb::SBDebugger::GetProgressFromEvent(
+            event, progress_id, completed, total, is_debugger_specific);
+        if (message)
+          g_vsc.SendProgressEvent(progress_id, message, completed, total);
+      }
+    }
+  }
+}
+
 // All events from the debugger, target, process, thread and frames are
 // received in this function that runs in its own thread. We are using a
 // "FILE *" to output packets back to VS Code and they have mutexes in them
@@ -806,6 +834,10 @@ void request_disconnect(const llvm::json::Object &request) {
     g_vsc.broadcaster.BroadcastEventByType(eBroadcastBitStopEventThread);
     g_vsc.event_thread.join();
   }
+  if (g_vsc.progress_event_thread.joinable()) {
+    g_vsc.broadcaster.BroadcastEventByType(eBroadcastBitStopProgressThread);
+    g_vsc.progress_event_thread.join();
+  }
 }
 
 void request_exceptionInfo(const llvm::json::Object &request) {
@@ -1357,6 +1389,8 @@ void request_modules(const llvm::json::Object &request) {
 // }
 void request_initialize(const llvm::json::Object &request) {
   g_vsc.debugger = lldb::SBDebugger::Create(true /*source_init_files*/);
+  g_vsc.progress_event_thread = std::thread(ProgressEventThreadFunction);
+
   // Create an empty target right away since we might get breakpoint requests
   // before we are given an executable to launch in a "launch" request, or a
   // executable when attaching to a process by process ID in a "attach"
@@ -1453,6 +1487,8 @@ void request_initialize(const llvm::json::Object &request) {
   body.try_emplace("supportsDelayedStackTraceLoading", true);
   // The debug adapter supports the 'loadedSources' request.
   body.try_emplace("supportsLoadedSourcesRequest", false);
+  // The debug adapter supports sending progress reporting events.
+  body.try_emplace("supportsProgressReporting", true);
 
   response.try_emplace("body", std::move(body));
   g_vsc.SendJSON(llvm::json::Value(std::move(response)));
diff --git a/lldb/unittests/CMakeLists.txt b/lldb/unittests/CMakeLists.txt
index 37a5f972cdec6f5e19bbe20f056da56fb574ee78..e7b0f1c17d6d1202aed019266a026eb34673a448 100644
--- a/lldb/unittests/CMakeLists.txt
+++ b/lldb/unittests/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_custom_target(LLDBUnitTests)
 set_target_properties(LLDBUnitTests PROPERTIES FOLDER "lldb tests")
-add_dependencies(lldb-test-deps LLDBUnitTests)
+
+add_dependencies(lldb-unit-test-deps LLDBUnitTests)
 
 include_directories(${LLDB_SOURCE_ROOT})
 include_directories(${LLDB_PROJECT_ROOT}/unittests)
diff --git a/lldb/unittests/Expression/DWARFExpressionTest.cpp b/lldb/unittests/Expression/DWARFExpressionTest.cpp
index 7fcd967990cefb8b3665ca22a52f4258594e829c..92101e913c22d3ed614e778f14d38317d2fee374 100644
--- a/lldb/unittests/Expression/DWARFExpressionTest.cpp
+++ b/lldb/unittests/Expression/DWARFExpressionTest.cpp
@@ -213,42 +213,45 @@ DWARF:
   //
 
   // Leave as is.
-  EXPECT_THAT_EXPECTED(t.Eval({DW_OP_const4u, 0x11, 0x22, 0x33, 0x44, //
-                               DW_OP_convert, offs_uint32_t}),
-                       llvm::HasValue(GetScalar(64, 0x44332211, not_signed)));
+  EXPECT_THAT_EXPECTED(
+      t.Eval({DW_OP_const4u, 0x11, 0x22, 0x33, 0x44, //
+              DW_OP_convert, offs_uint32_t, DW_OP_stack_value}),
+      llvm::HasValue(GetScalar(64, 0x44332211, not_signed)));
 
   // Zero-extend to 64 bits.
-  EXPECT_THAT_EXPECTED(t.Eval({DW_OP_const4u, 0x11, 0x22, 0x33, 0x44, //
-                               DW_OP_convert, offs_uint64_t}),
-                       llvm::HasValue(GetScalar(64, 0x44332211, not_signed)));
+  EXPECT_THAT_EXPECTED(
+      t.Eval({DW_OP_const4u, 0x11, 0x22, 0x33, 0x44, //
+              DW_OP_convert, offs_uint64_t, DW_OP_stack_value}),
+      llvm::HasValue(GetScalar(64, 0x44332211, not_signed)));
 
   // Sign-extend to 64 bits.
   EXPECT_THAT_EXPECTED(
       t.Eval({DW_OP_const4s, 0xcc, 0xdd, 0xee, 0xff, //
-              DW_OP_convert, offs_sint64_t}),
+              DW_OP_convert, offs_sint64_t, DW_OP_stack_value}),
       llvm::HasValue(GetScalar(64, 0xffffffffffeeddcc, is_signed)));
 
   // Sign-extend, then truncate.
-  EXPECT_THAT_EXPECTED(t.Eval({DW_OP_const4s, 0xcc, 0xdd, 0xee, 0xff, //
-                               DW_OP_convert, offs_sint64_t,          //
-                               DW_OP_convert, offs_uint32_t}),
-                       llvm::HasValue(GetScalar(32, 0xffeeddcc, not_signed)));
+  EXPECT_THAT_EXPECTED(
+      t.Eval({DW_OP_const4s, 0xcc, 0xdd, 0xee, 0xff, //
+              DW_OP_convert, offs_sint64_t,          //
+              DW_OP_convert, offs_uint32_t, DW_OP_stack_value}),
+      llvm::HasValue(GetScalar(32, 0xffeeddcc, not_signed)));
 
   // Truncate to default unspecified (pointer-sized) type.
   EXPECT_THAT_EXPECTED(t.Eval({DW_OP_const4s, 0xcc, 0xdd, 0xee, 0xff, //
                                DW_OP_convert, offs_sint64_t,          //
-                               DW_OP_convert, 0x00}),
+                               DW_OP_convert, 0x00, DW_OP_stack_value}),
                        llvm::HasValue(GetScalar(32, 0xffeeddcc, not_signed)));
 
   // Truncate to 8 bits.
-  EXPECT_THAT_EXPECTED(
-      t.Eval({DW_OP_const4s, 'A', 'B', 'C', 'D', DW_OP_convert, offs_uchar}),
-      llvm::HasValue(GetScalar(8, 'A', not_signed)));
+  EXPECT_THAT_EXPECTED(t.Eval({DW_OP_const4s, 'A', 'B', 'C', 'D', DW_OP_convert,
+                               offs_uchar, DW_OP_stack_value}),
+                       llvm::HasValue(GetScalar(8, 'A', not_signed)));
 
   // Also truncate to 8 bits.
-  EXPECT_THAT_EXPECTED(
-      t.Eval({DW_OP_const4s, 'A', 'B', 'C', 'D', DW_OP_convert, offs_schar}),
-      llvm::HasValue(GetScalar(8, 'A', is_signed)));
+  EXPECT_THAT_EXPECTED(t.Eval({DW_OP_const4s, 'A', 'B', 'C', 'D', DW_OP_convert,
+                               offs_schar, DW_OP_stack_value}),
+                       llvm::HasValue(GetScalar(8, 'A', is_signed)));
 
   //
   // Errors.
@@ -354,4 +357,13 @@ TEST_F(DWARFExpressionMockProcessTest, DW_OP_deref) {
   // Evaluate returns LLDB_INVALID_ADDRESS for all load addresses.
   EXPECT_THAT_EXPECTED(Evaluate({DW_OP_lit4, DW_OP_deref}, {}, {}, &exe_ctx),
                        llvm::HasValue(Scalar(LLDB_INVALID_ADDRESS)));
+  // Memory location: *0x4.
+  // Evaluate returns LLDB_INVALID_ADDRESS for all load addresses.
+  EXPECT_THAT_EXPECTED(Evaluate({DW_OP_lit4}, {}, {}, &exe_ctx),
+                       llvm::HasValue(Scalar(4)));
+  // Implicit location: *0x4.
+  // Evaluate returns LLDB_INVALID_ADDRESS for all load addresses.
+  EXPECT_THAT_EXPECTED(
+      Evaluate({DW_OP_lit4, DW_OP_deref, DW_OP_stack_value}, {}, {}, &exe_ctx),
+      llvm::HasValue(GetScalar(32, 0x07060504, false)));
 }
diff --git a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
index 58ddf0c40a267ab4311c7035efe6659031d85db6..f93733b3a5b783ee0bfda739fdda6fa31fca80ae 100644
--- a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
+++ b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
@@ -12,6 +12,7 @@
 
 #include "Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h"
 #include "Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h"
+#include "lldb/API/SBError.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
 
@@ -153,6 +154,14 @@ extern "C" int LLDBSwigPython_GetIndexOfChildWithName(void *implementor,
   return 0;
 }
 
+extern "C" void *LLDBSWIGPython_CastPyObjectToSBData(void *data) {
+  return nullptr;
+}
+
+extern "C" void *LLDBSWIGPython_CastPyObjectToSBError(void *data) {
+  return nullptr;
+}
+
 extern "C" void *LLDBSWIGPython_CastPyObjectToSBValue(void *data) {
   return nullptr;
 }
@@ -207,6 +216,13 @@ LLDBSWIGPythonCreateOSPlugin(const char *python_class_name,
   return nullptr;
 }
 
+extern "C" void *LLDBSwigPythonCreateScriptedProcess(
+    const char *python_class_name, const char *session_dictionary_name,
+    const lldb::TargetSP &target_sp, StructuredDataImpl *args_impl,
+    std::string &error_string) {
+  return nullptr;
+}
+
 extern "C" void *
 LLDBSWIGPython_CreateFrameRecognizer(const char *python_class_name,
                                      const char *session_dictionary_name) {
diff --git a/lldb/unittests/TestingSupport/TestUtilities.cpp b/lldb/unittests/TestingSupport/TestUtilities.cpp
index 34f49e5862a7a281f0df88b417efcd1598896a9a..86f3d1a7dfa75bb171ce9cde5aa40bde6a70ad26 100644
--- a/lldb/unittests/TestingSupport/TestUtilities.cpp
+++ b/lldb/unittests/TestingSupport/TestUtilities.cpp
@@ -38,8 +38,8 @@ llvm::Expected<TestFile> TestFile::fromYaml(llvm::StringRef Yaml) {
 
 llvm::Expected<TestFile> TestFile::fromYamlFile(const llvm::Twine &Name) {
   auto BufferOrError =
-      llvm::MemoryBuffer::getFile(GetInputFilePath(Name), /*FileSize*/ -1,
-                                  /*RequiresNullTerminator*/ false);
+      llvm::MemoryBuffer::getFile(GetInputFilePath(Name), /*IsText=*/false,
+                                  /*RequiresNullTerminator=*/false);
   if (!BufferOrError)
     return llvm::errorCodeToError(BufferOrError.getError());
   return fromYaml(BufferOrError.get()->getBuffer());
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 834813c8bce5e1dfc73a63ae310f6a1a890b1ba8..c5a81d1ebccffc556239ea4e33ce67ae3b560a6d 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -507,9 +507,6 @@ if (MSVC_IDE OR XCODE)
 endif()
 set(LLVM_LIT_ARGS "${LIT_ARGS_DEFAULT}" CACHE STRING "Default options for lit")
 
-include(GetErrcMessages)
-get_errc_messages(LLVM_LIT_ERRC_MESSAGES)
-
 # On Win32 hosts, provide an option to specify the path to the GnuWin32 tools.
 if( WIN32 AND NOT CYGWIN )
   set(LLVM_LIT_TOOLS_DIR "" CACHE PATH "Path to GnuWin32 tools")
@@ -762,6 +759,11 @@ set( CMAKE_RUNTIME_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/bin )
 set( CMAKE_LIBRARY_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX} )
 set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX} )
 
+if(LLVM_INCLUDE_TESTS)
+  include(GetErrcMessages)
+  get_errc_messages(LLVM_LIT_ERRC_MESSAGES)
+endif()
+
 # For up-to-date instructions for installing the Tensorflow dependency, refer to
 # the bot setup script: https://github.com/google/ml-compiler-opt/blob/master/buildbot/buildbot_init.sh
 # In this case, the latest C API library is available for download from
diff --git a/clang/test/Driver/Inputs/multilib_armebhf_linux_tree/usr/include/armeb-linux-gnueabihf/.keep b/llvm/bindings/ocaml/.ocamlformat
similarity index 100%
rename from clang/test/Driver/Inputs/multilib_armebhf_linux_tree/usr/include/armeb-linux-gnueabihf/.keep
rename to llvm/bindings/ocaml/.ocamlformat
diff --git a/llvm/bindings/ocaml/CMakeLists.txt b/llvm/bindings/ocaml/CMakeLists.txt
index 20583682c3d750b01df13876a34bb91299c0dcf9..7fe960b67f272b608c3c3fdf4f3affa6a03b6143 100644
--- a/llvm/bindings/ocaml/CMakeLists.txt
+++ b/llvm/bindings/ocaml/CMakeLists.txt
@@ -4,6 +4,7 @@ add_subdirectory(analysis)
 add_subdirectory(backends)
 add_subdirectory(bitreader)
 add_subdirectory(bitwriter)
+add_subdirectory(debuginfo)
 add_subdirectory(irreader)
 add_subdirectory(linker)
 add_subdirectory(target)
diff --git a/llvm/bindings/ocaml/README.txt b/llvm/bindings/ocaml/README.txt
index 68216b6792a77ec021a593d399072c8ba5a79d8f..a6a595e75bb5ff929c27743790e63f43b0b468a6 100644
--- a/llvm/bindings/ocaml/README.txt
+++ b/llvm/bindings/ocaml/README.txt
@@ -20,7 +20,8 @@ The bindings can also be built out-of-tree, i.e. targeting a preinstalled
 LLVM. To do this, configure the LLVM build tree as follows:
 
     $ cmake -DLLVM_OCAML_OUT_OF_TREE=TRUE \
-            -DCMAKE_INSTALL_PREFIX=[OCaml install prefix] \
+            -DCMAKE_INSTALL_PREFIX=[Preinstalled LLVM path] \
+            -DLLVM_OCAML_INSTALL_PATH=[OCaml install prefix] \
             [... any other options]
 
 then build and install it as:
diff --git a/llvm/bindings/ocaml/analysis/llvm_analysis.mli b/llvm/bindings/ocaml/analysis/llvm_analysis.mli
index cf323b547d9a68a5d40bfe6327b02eee162816c4..3dd397425a78ccdac14434db9c7bf79a4079d76d 100644
--- a/llvm/bindings/ocaml/analysis/llvm_analysis.mli
+++ b/llvm/bindings/ocaml/analysis/llvm_analysis.mli
@@ -16,9 +16,8 @@
     human-readable validation report. See [llvm::verifyModule]. *)
 external verify_module : Llvm.llmodule -> string option = "llvm_verify_module"
 
-(** [verify_function f] returns [None] if the function [f] is valid, and
-    [Some reason] if it is invalid. [reason] is a string containing a
-    human-readable validation report. See [llvm::verifyFunction]. *)
+(** [verify_function f] returns [true] if the function [f] is valid, and
+    [false] if it is invalid. See [llvm::verifyFunction]. *)
 external verify_function : Llvm.llvalue -> bool = "llvm_verify_function"
 
 (** [verify_module m] returns if the module [m] is valid, but prints a
diff --git a/llvm/bindings/ocaml/debuginfo/CMakeLists.txt b/llvm/bindings/ocaml/debuginfo/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..07f4956cccf2f3de4a6065134e3c77b2fff5bd46
--- /dev/null
+++ b/llvm/bindings/ocaml/debuginfo/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_ocaml_library(llvm_debuginfo
+  OCAML    llvm_debuginfo
+  OCAMLDEP llvm
+  C        debuginfo_ocaml
+  CFLAGS   "-I${CMAKE_CURRENT_SOURCE_DIR}/../llvm"
+  LLVM     Core)
diff --git a/llvm/bindings/ocaml/debuginfo/debuginfo_ocaml.c b/llvm/bindings/ocaml/debuginfo/debuginfo_ocaml.c
new file mode 100644
index 0000000000000000000000000000000000000000..11c260735c4ebdcd712074d32a51ad5a022bfd14
--- /dev/null
+++ b/llvm/bindings/ocaml/debuginfo/debuginfo_ocaml.c
@@ -0,0 +1,918 @@
+/*===-- debuginfo_ocaml.c - LLVM OCaml Glue ---------------------*- C++ -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* This file glues LLVM's OCaml interface to its C interface. These functions *|
+|* are by and large transparent wrappers to the corresponding C functions.    *|
+|*                                                                            *|
+|* Note that these functions intentionally take liberties with the CAMLparamX *|
+|* macros, since most of the parameters are not GC heap objects.              *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#include <string.h>
+
+#include "caml/memory.h"
+#include "caml/mlvalues.h"
+#include "llvm-c/Core.h"
+#include "llvm-c/DebugInfo.h"
+#include "llvm-c/Support.h"
+
+#include "llvm_ocaml.h"
+
+// This is identical to the definition in llvm_debuginfo.ml:DIFlag.t
+typedef enum {
+  i_DIFlagZero,
+  i_DIFlagPrivate,
+  i_DIFlagProtected,
+  i_DIFlagPublic,
+  i_DIFlagFwdDecl,
+  i_DIFlagAppleBlock,
+  i_DIFlagReservedBit4,
+  i_DIFlagVirtual,
+  i_DIFlagArtificial,
+  i_DIFlagExplicit,
+  i_DIFlagPrototyped,
+  i_DIFlagObjcClassComplete,
+  i_DIFlagObjectPointer,
+  i_DIFlagVector,
+  i_DIFlagStaticMember,
+  i_DIFlagLValueReference,
+  i_DIFlagRValueReference,
+  i_DIFlagReserved,
+  i_DIFlagSingleInheritance,
+  i_DIFlagMultipleInheritance,
+  i_DIFlagVirtualInheritance,
+  i_DIFlagIntroducedVirtual,
+  i_DIFlagBitField,
+  i_DIFlagNoReturn,
+  i_DIFlagTypePassByValue,
+  i_DIFlagTypePassByReference,
+  i_DIFlagEnumClass,
+  i_DIFlagFixedEnum,
+  i_DIFlagThunk,
+  i_DIFlagNonTrivial,
+  i_DIFlagBigEndian,
+  i_DIFlagLittleEndian,
+  i_DIFlagIndirectVirtualBase,
+  i_DIFlagAccessibility,
+  i_DIFlagPtrToMemberRep
+} LLVMDIFlag_i;
+
+static LLVMDIFlags map_DIFlag(LLVMDIFlag_i DIF) {
+  switch (DIF) {
+  case i_DIFlagZero:
+    return LLVMDIFlagZero;
+  case i_DIFlagPrivate:
+    return LLVMDIFlagPrivate;
+  case i_DIFlagProtected:
+    return LLVMDIFlagProtected;
+  case i_DIFlagPublic:
+    return LLVMDIFlagPublic;
+  case i_DIFlagFwdDecl:
+    return LLVMDIFlagFwdDecl;
+  case i_DIFlagAppleBlock:
+    return LLVMDIFlagAppleBlock;
+  case i_DIFlagReservedBit4:
+    return LLVMDIFlagReservedBit4;
+  case i_DIFlagVirtual:
+    return LLVMDIFlagVirtual;
+  case i_DIFlagArtificial:
+    return LLVMDIFlagArtificial;
+  case i_DIFlagExplicit:
+    return LLVMDIFlagExplicit;
+  case i_DIFlagPrototyped:
+    return LLVMDIFlagPrototyped;
+  case i_DIFlagObjcClassComplete:
+    return LLVMDIFlagObjcClassComplete;
+  case i_DIFlagObjectPointer:
+    return LLVMDIFlagObjectPointer;
+  case i_DIFlagVector:
+    return LLVMDIFlagVector;
+  case i_DIFlagStaticMember:
+    return LLVMDIFlagStaticMember;
+  case i_DIFlagLValueReference:
+    return LLVMDIFlagLValueReference;
+  case i_DIFlagRValueReference:
+    return LLVMDIFlagRValueReference;
+  case i_DIFlagReserved:
+    return LLVMDIFlagReserved;
+  case i_DIFlagSingleInheritance:
+    return LLVMDIFlagSingleInheritance;
+  case i_DIFlagMultipleInheritance:
+    return LLVMDIFlagMultipleInheritance;
+  case i_DIFlagVirtualInheritance:
+    return LLVMDIFlagVirtualInheritance;
+  case i_DIFlagIntroducedVirtual:
+    return LLVMDIFlagIntroducedVirtual;
+  case i_DIFlagBitField:
+    return LLVMDIFlagBitField;
+  case i_DIFlagNoReturn:
+    return LLVMDIFlagNoReturn;
+  case i_DIFlagTypePassByValue:
+    return LLVMDIFlagTypePassByValue;
+  case i_DIFlagTypePassByReference:
+    return LLVMDIFlagTypePassByReference;
+  case i_DIFlagEnumClass:
+    return LLVMDIFlagEnumClass;
+  case i_DIFlagFixedEnum:
+    return LLVMDIFlagFixedEnum;
+  case i_DIFlagThunk:
+    return LLVMDIFlagThunk;
+  case i_DIFlagNonTrivial:
+    return LLVMDIFlagNonTrivial;
+  case i_DIFlagBigEndian:
+    return LLVMDIFlagBigEndian;
+  case i_DIFlagLittleEndian:
+    return LLVMDIFlagLittleEndian;
+  case i_DIFlagIndirectVirtualBase:
+    return LLVMDIFlagIndirectVirtualBase;
+  case i_DIFlagAccessibility:
+    return LLVMDIFlagAccessibility;
+  case i_DIFlagPtrToMemberRep:
+    return LLVMDIFlagPtrToMemberRep;
+  }
+}
+
+CAMLprim value llvm_debug_metadata_version(value Unit) {
+  return Val_int(LLVMDebugMetadataVersion());
+}
+
+CAMLprim value llvm_get_module_debug_metadata_version(LLVMModuleRef Module) {
+  return Val_int(LLVMGetModuleDebugMetadataVersion(Module));
+}
+
+#define DIFlags_val(v) (*(LLVMDIFlags *)(Data_custom_val(v)))
+
+static struct custom_operations diflags_ops = {
+    (char *)"DebugInfo.lldiflags", custom_finalize_default,
+    custom_compare_default,        custom_hash_default,
+    custom_serialize_default,      custom_deserialize_default,
+    custom_compare_ext_default};
+
+static value alloc_diflags(LLVMDIFlags Flags) {
+  value V = alloc_custom(&diflags_ops, sizeof(LLVMDIFlags), 0, 1);
+  DIFlags_val(V) = Flags;
+  return V;
+}
+
+CAMLprim LLVMDIFlags llvm_diflags_get(value i_Flag) {
+  LLVMDIFlags Flags = map_DIFlag(Int_val(i_Flag));
+  return alloc_diflags(Flags);
+}
+
+CAMLprim LLVMDIFlags llvm_diflags_set(value Flags, value i_Flag) {
+  LLVMDIFlags FlagsNew = DIFlags_val(Flags) | map_DIFlag(Int_val(i_Flag));
+  return alloc_diflags(FlagsNew);
+}
+
+CAMLprim value llvm_diflags_test(value Flags, value i_Flag) {
+  LLVMDIFlags Flag = map_DIFlag(Int_val(i_Flag));
+  return Val_bool((DIFlags_val(Flags) & Flag) == Flag);
+}
+
+#define DIBuilder_val(v) (*(LLVMDIBuilderRef *)(Data_custom_val(v)))
+
+static void llvm_finalize_dibuilder(value B) {
+  LLVMDIBuilderFinalize(DIBuilder_val(B));
+  LLVMDisposeDIBuilder(DIBuilder_val(B));
+}
+
+static struct custom_operations dibuilder_ops = {
+    (char *)"DebugInfo.lldibuilder", llvm_finalize_dibuilder,
+    custom_compare_default,          custom_hash_default,
+    custom_serialize_default,        custom_deserialize_default,
+    custom_compare_ext_default};
+
+static value alloc_dibuilder(LLVMDIBuilderRef B) {
+  value V = alloc_custom(&dibuilder_ops, sizeof(LLVMDIBuilderRef), 0, 1);
+  DIBuilder_val(V) = B;
+  return V;
+}
+
+/* llmodule -> lldibuilder */
+CAMLprim value llvm_dibuilder(LLVMModuleRef M) {
+  return alloc_dibuilder(LLVMCreateDIBuilder(M));
+}
+
+CAMLprim value llvm_dibuild_finalize(value Builder) {
+  LLVMDIBuilderFinalize(DIBuilder_val(Builder));
+  return Val_unit;
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_compile_unit_native(
+    value Builder, value Lang, LLVMMetadataRef FileRef, value Producer,
+    value IsOptimized, value Flags, value RuntimeVer, value SplitName,
+    value Kind, value DWOId, value SplitDebugInline,
+    value DebugInfoForProfiling, value SysRoot, value SDK) {
+  return LLVMDIBuilderCreateCompileUnit(
+      DIBuilder_val(Builder), Int_val(Lang), FileRef, String_val(Producer),
+      caml_string_length(Producer), Bool_val(IsOptimized), String_val(Flags),
+      caml_string_length(Flags), Int_val(RuntimeVer), String_val(SplitName),
+      caml_string_length(SplitName), Int_val(Kind), Int_val(DWOId),
+      Bool_val(SplitDebugInline), Bool_val(DebugInfoForProfiling),
+      String_val(SysRoot), caml_string_length(SysRoot), String_val(SDK),
+      caml_string_length(SDK));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_compile_unit_bytecode(value *argv,
+                                                                   int argn) {
+  return llvm_dibuild_create_compile_unit_native(
+      argv[0],                  // Builder
+      argv[1],                  // Lang
+      (LLVMMetadataRef)argv[2], // FileRef
+      argv[3],                  // Producer
+      argv[4],                  // IsOptimized
+      argv[5],                  // Flags
+      argv[6],                  // RuntimeVer
+      argv[7],                  // SplitName
+      argv[8],                  // Kind
+      argv[9],                  // DWOId
+      argv[10],                 // SplitDebugInline
+      argv[11],                 // DebugInfoForProfiling
+      argv[12],                 // SysRoot
+      argv[13]                  // SDK
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_file(value Builder, value Filename,
+                                                  value Directory) {
+  return LLVMDIBuilderCreateFile(DIBuilder_val(Builder), String_val(Filename),
+                                 caml_string_length(Filename),
+                                 String_val(Directory),
+                                 caml_string_length(Directory));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_module_native(
+    value Builder, LLVMMetadataRef ParentScope, value Name, value ConfigMacros,
+    value IncludePath, value SysRoot) {
+  return LLVMDIBuilderCreateModule(
+      DIBuilder_val(Builder), ParentScope, String_val(Name),
+      caml_string_length(Name), String_val(ConfigMacros),
+      caml_string_length(ConfigMacros), String_val(IncludePath),
+      caml_string_length(IncludePath), String_val(SysRoot),
+      caml_string_length(SysRoot));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_module_bytecode(value *argv,
+                                                             int argn) {
+  return llvm_dibuild_create_module_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // ParentScope
+      argv[2],                  // Name
+      argv[3],                  // ConfigMacros
+      argv[4],                  // IncludePath
+      argv[5]                   // SysRoot
+  );
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_namespace(value Builder, LLVMMetadataRef ParentScope,
+                              value Name, value ExportSymbols) {
+  return LLVMDIBuilderCreateNameSpace(
+      DIBuilder_val(Builder), ParentScope, String_val(Name),
+      caml_string_length(Name), Bool_val(ExportSymbols));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_function_native(
+    value Builder, LLVMMetadataRef Scope, value Name, value LinkageName,
+    LLVMMetadataRef File, value LineNo, LLVMMetadataRef Ty, value IsLocalToUnit,
+    value IsDefinition, value ScopeLine, value Flags, value IsOptimized) {
+  return LLVMDIBuilderCreateFunction(
+      DIBuilder_val(Builder), Scope, String_val(Name), caml_string_length(Name),
+      String_val(LinkageName), caml_string_length(LinkageName), File,
+      Int_val(LineNo), Ty, Bool_val(IsLocalToUnit), Bool_val(IsDefinition),
+      Int_val(ScopeLine), DIFlags_val(Flags), Bool_val(IsOptimized));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_function_bytecode(value *argv,
+                                                               int argn) {
+  return llvm_dibuild_create_function_native(argv[0], // Builder,
+                                             (LLVMMetadataRef)argv[1], // Scope
+                                             argv[2],                  // Name
+                                             argv[3], // LinkageName
+                                             (LLVMMetadataRef)argv[4], // File
+                                             argv[5],                  // LineNo
+                                             (LLVMMetadataRef)argv[6], // Ty
+                                             argv[7],  // IsLocalUnit
+                                             argv[8],  // IsDefinition
+                                             argv[9],  // ScopeLine
+                                             argv[10], // Flags
+                                             argv[11]  // IsOptimized
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_lexical_block(
+    value Builder, LLVMMetadataRef Scope, LLVMMetadataRef File, value Line,
+    value Column) {
+  return LLVMDIBuilderCreateLexicalBlock(DIBuilder_val(Builder), Scope, File,
+                                         Int_val(Line), Int_val(Column));
+}
+
+CAMLprim LLVMMetadataRef llvm_metadata_null() { return (LLVMMetadataRef)NULL; }
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_debug_location(
+    LLVMContextRef Ctx, value Line, value Column, LLVMMetadataRef Scope,
+    LLVMMetadataRef InlinedAt) {
+  return LLVMDIBuilderCreateDebugLocation(Ctx, Int_val(Line), Int_val(Column),
+                                          Scope, InlinedAt);
+}
+
+CAMLprim value llvm_di_location_get_line(LLVMMetadataRef Location) {
+  return Val_int(LLVMDILocationGetLine(Location));
+}
+
+CAMLprim value llvm_di_location_get_column(LLVMMetadataRef Location) {
+  return Val_int(LLVMDILocationGetColumn(Location));
+}
+
+CAMLprim LLVMMetadataRef llvm_di_location_get_scope(LLVMMetadataRef Location) {
+  return LLVMDILocationGetScope(Location);
+}
+
+CAMLprim value llvm_di_location_get_inlined_at(LLVMMetadataRef Location) {
+  return (ptr_to_option(LLVMDILocationGetInlinedAt(Location)));
+}
+
+CAMLprim value llvm_di_scope_get_file(LLVMMetadataRef Scope) {
+  return (ptr_to_option(LLVMDIScopeGetFile(Scope)));
+}
+
+CAMLprim value llvm_di_file_get_directory(LLVMMetadataRef File) {
+  unsigned Len;
+  const char *Directory = LLVMDIFileGetDirectory(File, &Len);
+  return cstr_to_string(Directory, Len);
+}
+
+CAMLprim value llvm_di_file_get_filename(LLVMMetadataRef File) {
+  unsigned Len;
+  const char *Filename = LLVMDIFileGetFilename(File, &Len);
+  return cstr_to_string(Filename, Len);
+}
+
+CAMLprim value llvm_di_file_get_source(LLVMMetadataRef File) {
+  unsigned Len;
+  const char *Source = LLVMDIFileGetSource(File, &Len);
+  return cstr_to_string(Source, Len);
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_get_or_create_type_array(value Builder,
+                                                               value Data) {
+
+  return LLVMDIBuilderGetOrCreateTypeArray(DIBuilder_val(Builder),
+                                           (LLVMMetadataRef *)Op_val(Data),
+                                           Wosize_val(Data));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_get_or_create_array(value Builder,
+                                                          value Data) {
+
+  return LLVMDIBuilderGetOrCreateArray(DIBuilder_val(Builder),
+                                       (LLVMMetadataRef *)Op_val(Data),
+                                       Wosize_val(Data));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_subroutine_type(
+    value Builder, LLVMMetadataRef File, value ParameterTypes, value Flags) {
+
+  return LLVMDIBuilderCreateSubroutineType(
+      DIBuilder_val(Builder), File, (LLVMMetadataRef *)Op_val(ParameterTypes),
+      Wosize_val(ParameterTypes), DIFlags_val(Flags));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_enumerator(value Builder,
+                                                        value Name, value Value,
+                                                        value IsUnsigned) {
+  return LLVMDIBuilderCreateEnumerator(
+      DIBuilder_val(Builder), String_val(Name), caml_string_length(Name),
+      (int64_t)Int_val(Value), Bool_val(IsUnsigned));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_enumeration_type_native(
+    value Builder, LLVMMetadataRef Scope, value Name, LLVMMetadataRef File,
+    value LineNumber, value SizeInBits, value AlignInBits, value Elements,
+    LLVMMetadataRef ClassTy) {
+  return LLVMDIBuilderCreateEnumerationType(
+      DIBuilder_val(Builder), Scope, String_val(Name), caml_string_length(Name),
+      File, Int_val(LineNumber), (uint64_t)Int_val(SizeInBits),
+      Int_val(AlignInBits), (LLVMMetadataRef *)Op_val(Elements),
+      Wosize_val(Elements), ClassTy);
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_enumeration_type_bytecode(value *argv, int argn) {
+  return llvm_dibuild_create_enumeration_type_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // Scope
+      argv[2],                  // Name
+      (LLVMMetadataRef)argv[3], // File
+      argv[4],                  // LineNumber
+      argv[5],                  // SizeInBits
+      argv[6],                  // AlignInBits
+      argv[7],                  // Elements
+      (LLVMMetadataRef)argv[8]  // ClassTy
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_union_type_native(
+    value Builder, LLVMMetadataRef Scope, value Name, LLVMMetadataRef File,
+    value LineNumber, value SizeInBits, value AlignInBits, value Flags,
+    value Elements, value RunTimeLanguage, value UniqueId) {
+
+  return LLVMDIBuilderCreateUnionType(
+      DIBuilder_val(Builder), Scope, String_val(Name), caml_string_length(Name),
+      File, Int_val(LineNumber), (uint64_t)Int_val(SizeInBits),
+      Int_val(AlignInBits), DIFlags_val(Flags),
+      (LLVMMetadataRef *)Op_val(Elements), Wosize_val(Elements),
+      Int_val(RunTimeLanguage), String_val(UniqueId),
+      caml_string_length(UniqueId));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_union_type_bytecode(value *argv,
+                                                                 int argn) {
+  return llvm_dibuild_create_union_type_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // Scope
+      argv[2],                  // Name
+      (LLVMMetadataRef)argv[3], // File
+      argv[4],                  // LineNumber
+      argv[5],                  // SizeInBits
+      argv[6],                  // AlignInBits
+      argv[7],                  // Flags
+      argv[8],                  // Elements
+      argv[9],                  // RunTimeLanguage
+      argv[10]                  // UniqueId
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_array_type(value Builder,
+                                                        value Size,
+                                                        value AlignInBits,
+                                                        LLVMMetadataRef Ty,
+                                                        value Subscripts) {
+  return LLVMDIBuilderCreateArrayType(
+      DIBuilder_val(Builder), (uint64_t)Int_val(Size), Int_val(AlignInBits), Ty,
+      (LLVMMetadataRef *)Op_val(Subscripts), Wosize_val(Subscripts));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_vector_type(value Builder,
+                                                         value Size,
+                                                         value AlignInBits,
+                                                         LLVMMetadataRef Ty,
+                                                         value Subscripts) {
+  return LLVMDIBuilderCreateVectorType(
+      DIBuilder_val(Builder), (uint64_t)Int_val(Size), Int_val(AlignInBits), Ty,
+      (LLVMMetadataRef *)Op_val(Subscripts), Wosize_val(Subscripts));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_unspecified_type(value Builder,
+                                                              value Name) {
+  return LLVMDIBuilderCreateUnspecifiedType(
+      DIBuilder_val(Builder), String_val(Name), caml_string_length(Name));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_basic_type(
+    value Builder, value Name, value SizeInBits, value Encoding, value Flags) {
+
+  return LLVMDIBuilderCreateBasicType(
+      DIBuilder_val(Builder), String_val(Name), caml_string_length(Name),
+      (uint64_t)Int_val(SizeInBits), Int_val(Encoding), DIFlags_val(Flags));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_pointer_type_native(
+    value Builder, LLVMMetadataRef PointeeTy, value SizeInBits,
+    value AlignInBits, value AddressSpace, value Name) {
+  return LLVMDIBuilderCreatePointerType(
+      DIBuilder_val(Builder), PointeeTy, (uint64_t)Int_val(SizeInBits),
+      Int_val(AlignInBits), Int_val(AddressSpace), String_val(Name),
+      caml_string_length(Name));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_pointer_type_bytecode(value *argv,
+                                                                   int argn) {
+  return llvm_dibuild_create_pointer_type_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // PointeeTy
+      argv[2],                  // SizeInBits
+      argv[3],                  // AlignInBits
+      argv[4],                  // AddressSpace
+      argv[5]                   // Name
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_struct_type_native(
+    value Builder, LLVMMetadataRef Scope, value Name, LLVMMetadataRef File,
+    value LineNumber, value SizeInBits, value AlignInBits, value Flags,
+    LLVMMetadataRef DerivedFrom, value Elements, value RunTimeLanguage,
+    LLVMMetadataRef VTableHolder, value UniqueId) {
+
+  return LLVMDIBuilderCreateStructType(
+      DIBuilder_val(Builder), Scope, String_val(Name), caml_string_length(Name),
+      File, Int_val(LineNumber), (uint64_t)Int_val(SizeInBits),
+      Int_val(AlignInBits), DIFlags_val(Flags), DerivedFrom,
+      (LLVMMetadataRef *)Op_val(Elements), Wosize_val(Elements),
+      Int_val(RunTimeLanguage), VTableHolder, String_val(UniqueId),
+      caml_string_length(UniqueId));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_struct_type_bytecode(value *argv,
+                                                                  int argn) {
+  return llvm_dibuild_create_struct_type_native(
+      argv[0],                   // Builder
+      (LLVMMetadataRef)argv[1],  // Scope
+      argv[2],                   // Name
+      (LLVMMetadataRef)argv[3],  // File
+      argv[4],                   // LineNumber
+      argv[5],                   // SizeInBits
+      argv[6],                   // AlignInBits
+      argv[7],                   // Flags
+      (LLVMMetadataRef)argv[8],  // DeriviedFrom
+      argv[9],                   // Elements
+      argv[10],                  // RunTimeLanguage
+      (LLVMMetadataRef)argv[11], // VTableHolder
+      argv[12]                   // UniqueId
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_member_type_native(
+    value Builder, LLVMMetadataRef Scope, value Name, LLVMMetadataRef File,
+    value LineNumber, value SizeInBits, value AlignInBits, value OffsetInBits,
+    value Flags, LLVMMetadataRef Ty) {
+
+  return LLVMDIBuilderCreateMemberType(
+      DIBuilder_val(Builder), Scope, String_val(Name), caml_string_length(Name),
+      File, Int_val(LineNumber), (uint64_t)Int_val(SizeInBits),
+      Int_val(AlignInBits), (uint64_t)Int_val(OffsetInBits), DIFlags_val(Flags),
+      Ty);
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_member_type_bytecode(value *argv,
+                                                                  int argn) {
+  return llvm_dibuild_create_member_type_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // Scope
+      argv[2],                  // Name
+      (LLVMMetadataRef)argv[3], // File
+      argv[4],                  // LineNumber
+      argv[5],                  // SizeInBits
+      argv[6],                  // AlignInBits
+      argv[7],                  // OffsetInBits
+      argv[8],                  // Flags
+      (LLVMMetadataRef)argv[9]  // Ty
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_static_member_type_native(
+    value Builder, LLVMMetadataRef Scope, value Name, LLVMMetadataRef File,
+    value LineNumber, LLVMMetadataRef Type, value Flags,
+    LLVMValueRef ConstantVal, value AlignInBits) {
+
+  return LLVMDIBuilderCreateStaticMemberType(
+      DIBuilder_val(Builder), Scope, String_val(Name), caml_string_length(Name),
+      File, Int_val(LineNumber), Type, DIFlags_val(Flags), ConstantVal,
+      Int_val(AlignInBits));
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_static_member_type_bytecode(value *argv, int argn) {
+  return llvm_dibuild_create_static_member_type_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // Scope
+      argv[2],                  // Name
+      (LLVMMetadataRef)argv[3], // File
+      argv[4],                  // LineNumber
+      (LLVMMetadataRef)argv[5], // Type
+      argv[6],                  // Flags,
+      (LLVMValueRef)argv[7],    // ConstantVal
+      argv[8]                   // AlignInBits
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_member_pointer_type_native(
+    value Builder, LLVMMetadataRef PointeeType, LLVMMetadataRef ClassType,
+    uint64_t SizeInBits, uint32_t AlignInBits, LLVMDIFlags Flags) {
+
+  return LLVMDIBuilderCreateMemberPointerType(
+      DIBuilder_val(Builder), PointeeType, ClassType,
+      (uint64_t)Int_val(SizeInBits), Int_val(AlignInBits), Flags);
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_member_pointer_type_bytecode(value *argv, int argn) {
+  return llvm_dibuild_create_member_pointer_type_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // PointeeType
+      (LLVMMetadataRef)argv[2], // ClassType
+      argv[3],                  // SizeInBits
+      argv[4],                  // AlignInBits
+      argv[5]                   // Flags
+  );
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_object_pointer_type(value Builder, LLVMMetadataRef Type) {
+  return LLVMDIBuilderCreateObjectPointerType(DIBuilder_val(Builder), Type);
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_qualified_type(
+    value Builder, value Tag, LLVMMetadataRef Type) {
+
+  return LLVMDIBuilderCreateQualifiedType(DIBuilder_val(Builder), Int_val(Tag),
+                                          Type);
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_reference_type(
+    value Builder, value Tag, LLVMMetadataRef Type) {
+
+  return LLVMDIBuilderCreateReferenceType(DIBuilder_val(Builder), Int_val(Tag),
+                                          Type);
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_null_ptr_type(value Builder) {
+
+  return LLVMDIBuilderCreateNullPtrType(DIBuilder_val(Builder));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_typedef_native(
+    value Builder, LLVMMetadataRef Type, value Name, LLVMMetadataRef File,
+    value LineNo, LLVMMetadataRef Scope, value AlignInBits) {
+
+  return LLVMDIBuilderCreateTypedef(
+      DIBuilder_val(Builder), Type, String_val(Name), caml_string_length(Name),
+      File, Int_val(LineNo), Scope, Int_val(AlignInBits));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_typedef_bytecode(value *argv,
+                                                              int argn) {
+
+  return llvm_dibuild_create_typedef_native(argv[0],                  // Builder
+                                            (LLVMMetadataRef)argv[1], // Type
+                                            argv[2],                  // Name
+                                            (LLVMMetadataRef)argv[3], // File
+                                            argv[4],                  // LineNo
+                                            (LLVMMetadataRef)argv[5], // Scope
+                                            argv[6] // AlignInBits
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_inheritance_native(
+    value Builder, LLVMMetadataRef Ty, LLVMMetadataRef BaseTy, value BaseOffset,
+    value VBPtrOffset, value Flags) {
+
+  return LLVMDIBuilderCreateInheritance(DIBuilder_val(Builder), Ty, BaseTy,
+                                        (uint64_t)Int_val(BaseOffset),
+                                        Int_val(VBPtrOffset), Flags);
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_inheritance_bytecode(value *argv,
+                                                                  int arg) {
+
+  return llvm_dibuild_create_inheritance_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // Ty
+      (LLVMMetadataRef)argv[2], // BaseTy
+      argv[3],                  // BaseOffset
+      argv[4],                  // VBPtrOffset
+      argv[5]                   // Flags
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_forward_decl_native(
+    value Builder, value Tag, value Name, LLVMMetadataRef Scope,
+    LLVMMetadataRef File, value Line, value RuntimeLang, value SizeInBits,
+    value AlignInBits, value UniqueIdentifier) {
+  return LLVMDIBuilderCreateForwardDecl(
+      DIBuilder_val(Builder), Int_val(Tag), String_val(Name),
+      caml_string_length(Name), Scope, File, Int_val(Line),
+      Int_val(RuntimeLang), (uint64_t)Int_val(SizeInBits), Int_val(AlignInBits),
+      String_val(UniqueIdentifier), caml_string_length(UniqueIdentifier));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_forward_decl_bytecode(value *argv,
+                                                                   int arg) {
+
+  return llvm_dibuild_create_forward_decl_native(
+      argv[0],                  // Builder
+      argv[1],                  // Tag
+      argv[2],                  // Name
+      (LLVMMetadataRef)argv[3], // Scope
+      (LLVMMetadataRef)argv[4], // File
+      argv[5],                  // Line
+      argv[6],                  // RuntimeLang
+      argv[7],                  // SizeInBits
+      argv[8],                  // AlignInBits
+      argv[9]                   // UniqueIdentifier
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_replaceable_composite_type_native(
+    value Builder, value Tag, value Name, LLVMMetadataRef Scope,
+    LLVMMetadataRef File, value Line, value RuntimeLang, value SizeInBits,
+    value AlignInBits, value Flags, value UniqueIdentifier) {
+
+  return LLVMDIBuilderCreateReplaceableCompositeType(
+      DIBuilder_val(Builder), Int_val(Tag), String_val(Name),
+      caml_string_length(Name), Scope, File, Int_val(Line),
+      Int_val(RuntimeLang), (uint64_t)Int_val(SizeInBits), Int_val(AlignInBits),
+      DIFlags_val(Flags), String_val(UniqueIdentifier),
+      caml_string_length(UniqueIdentifier));
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_replaceable_composite_type_bytecode(value *argv, int arg) {
+
+  return llvm_dibuild_create_replaceable_composite_type_native(
+      argv[0],                  // Builder
+      argv[1],                  // Tag
+      argv[2],                  // Name
+      (LLVMMetadataRef)argv[3], // Scope
+      (LLVMMetadataRef)argv[4], // File
+      argv[5],                  // Line
+      argv[6],                  // RuntimeLang
+      argv[7],                  // SizeInBits
+      argv[8],                  // AlignInBits
+      argv[9],                  // Flags
+      argv[10]                  // UniqueIdentifier
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_bit_field_member_type_native(
+    value Builder, LLVMMetadataRef Scope, value Name, LLVMMetadataRef File,
+    value LineNum, value SizeInBits, value OffsetInBits,
+    value StorageOffsetInBits, value Flags, LLVMMetadataRef Ty) {
+
+  return LLVMDIBuilderCreateBitFieldMemberType(
+      DIBuilder_val(Builder), Scope, String_val(Name), caml_string_length(Name),
+      File, Int_val(LineNum), (uint64_t)Int_val(SizeInBits),
+      (uint64_t)Int_val(OffsetInBits), (uint64_t)Int_val(StorageOffsetInBits),
+      DIFlags_val(Flags), Ty);
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_bit_field_member_type_bytecode(value *argv, int arg) {
+
+  return llvm_dibuild_create_bit_field_member_type_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // Scope
+      argv[2],                  // Name
+      (LLVMMetadataRef)argv[3], // File
+      argv[4],                  // LineNum
+      argv[5],                  // SizeInBits
+      argv[6],                  // OffsetInBits
+      argv[7],                  // StorageOffsetInBits
+      argv[8],                  // Flags
+      (LLVMMetadataRef)argv[9]  // Ty
+  );
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_class_type_native(
+    value Builder, LLVMMetadataRef Scope, value Name, LLVMMetadataRef File,
+    value LineNumber, value SizeInBits, value AlignInBits, value OffsetInBits,
+    value Flags, LLVMMetadataRef DerivedFrom, value Elements,
+    LLVMMetadataRef VTableHolder, LLVMMetadataRef TemplateParamsNode,
+    value UniqueIdentifier) {
+
+  return LLVMDIBuilderCreateClassType(
+      DIBuilder_val(Builder), Scope, String_val(Name), caml_string_length(Name),
+      File, Int_val(LineNumber), (uint64_t)Int_val(SizeInBits),
+      Int_val(AlignInBits), (uint64_t)Int_val(OffsetInBits), DIFlags_val(Flags),
+      DerivedFrom, (LLVMMetadataRef *)Op_val(Elements), Wosize_val(Elements),
+      VTableHolder, TemplateParamsNode, String_val(UniqueIdentifier),
+      caml_string_length(UniqueIdentifier));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_class_type_bytecode(value *argv,
+                                                                 int arg) {
+
+  return llvm_dibuild_create_class_type_native(
+      argv[0],                   // Builder
+      (LLVMMetadataRef)argv[1],  // Scope
+      argv[2],                   // Name
+      (LLVMMetadataRef)argv[3],  // File
+      argv[4],                   // LineNumber
+      argv[5],                   // SizeInBits
+      argv[6],                   // AlignInBits
+      argv[7],                   // OffsetInBits
+      argv[8],                   // Flags
+      (LLVMMetadataRef)argv[9],  // DerivedFrom
+      argv[10],                  // Elements
+      (LLVMMetadataRef)argv[11], // VTableHolder
+      (LLVMMetadataRef)argv[12], // TemplateParamsNode
+      argv[13]                   // UniqueIdentifier
+  );
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_artificial_type(value Builder, LLVMMetadataRef Type) {
+  return LLVMDIBuilderCreateArtificialType(DIBuilder_val(Builder), Type);
+}
+
+CAMLprim value llvm_di_type_get_name(LLVMMetadataRef DType) {
+  size_t Len;
+  const char *Name = LLVMDITypeGetName(DType, &Len);
+  return cstr_to_string(Name, Len);
+}
+
+CAMLprim value llvm_di_type_get_size_in_bits(LLVMMetadataRef DType) {
+  uint64_t Size = LLVMDITypeGetSizeInBits(DType);
+  return Val_int((int)Size);
+}
+
+CAMLprim value llvm_di_type_get_offset_in_bits(LLVMMetadataRef DType) {
+  uint64_t Size = LLVMDITypeGetOffsetInBits(DType);
+  return Val_int((int)Size);
+}
+
+CAMLprim value llvm_di_type_get_align_in_bits(LLVMMetadataRef DType) {
+  uint32_t Size = LLVMDITypeGetAlignInBits(DType);
+  return Val_int(Size);
+}
+
+CAMLprim value llvm_di_type_get_line(LLVMMetadataRef DType) {
+  unsigned Line = LLVMDITypeGetLine(DType);
+  return Val_int(Line);
+}
+
+CAMLprim value llvm_di_type_get_flags(LLVMMetadataRef DType) {
+  LLVMDIFlags Flags = LLVMDITypeGetLine(DType);
+  return alloc_diflags(Flags);
+}
+
+CAMLprim value llvm_get_subprogram(LLVMValueRef Func) {
+  return (ptr_to_option(LLVMGetSubprogram(Func)));
+}
+
+CAMLprim value llvm_set_subprogram(LLVMValueRef Func, LLVMMetadataRef SP) {
+  LLVMSetSubprogram(Func, SP);
+  return Val_unit;
+}
+
+CAMLprim value llvm_di_subprogram_get_line(LLVMMetadataRef Subprogram) {
+  return Val_int(LLVMDISubprogramGetLine(Subprogram));
+}
+
+CAMLprim value llvm_instr_get_debug_loc(LLVMValueRef Inst) {
+  return (ptr_to_option(LLVMInstructionGetDebugLoc(Inst)));
+}
+
+CAMLprim value llvm_instr_set_debug_loc(LLVMValueRef Inst,
+                                        LLVMMetadataRef Loc) {
+  LLVMInstructionSetDebugLoc(Inst, Loc);
+  return Val_unit;
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_constant_value_expression(value Builder, value Value) {
+  return LLVMDIBuilderCreateConstantValueExpression(DIBuilder_val(Builder),
+                                                    (int64_t)Int_val(Value));
+}
+
+CAMLprim LLVMMetadataRef llvm_dibuild_create_global_variable_expression_native(
+    value Builder, LLVMMetadataRef Scope, value Name, value Linkage,
+    LLVMMetadataRef File, value Line, LLVMMetadataRef Ty, value LocalToUnit,
+    LLVMMetadataRef Expr, LLVMMetadataRef Decl, value AlignInBits) {
+  return LLVMDIBuilderCreateGlobalVariableExpression(
+      DIBuilder_val(Builder), Scope, String_val(Name), caml_string_length(Name),
+      String_val(Linkage), caml_string_length(Linkage), File, Int_val(Line), Ty,
+      Bool_val(LocalToUnit), Expr, Decl, Int_val(AlignInBits));
+}
+
+CAMLprim LLVMMetadataRef
+llvm_dibuild_create_global_variable_expression_bytecode(value *argv, int arg) {
+
+  return llvm_dibuild_create_global_variable_expression_native(
+      argv[0],                  // Builder
+      (LLVMMetadataRef)argv[1], // Scope
+      argv[2],                  // Name
+      argv[3],                  // Linkage
+      (LLVMMetadataRef)argv[4], // File
+      argv[5],                  // Line
+      (LLVMMetadataRef)argv[6], // Ty
+      argv[7],                  // LocalToUnit
+      (LLVMMetadataRef)argv[8], // Expr
+      (LLVMMetadataRef)argv[9], // Decl
+      argv[10]                  // AlignInBits
+  );
+}
+
+CAMLprim value
+llvm_di_global_variable_expression_get_variable(LLVMMetadataRef GVE) {
+  return (ptr_to_option(LLVMDIGlobalVariableExpressionGetVariable(GVE)));
+}
+
+CAMLprim value llvm_di_variable_get_line(LLVMMetadataRef Variable) {
+  return Val_int(LLVMDIVariableGetLine(Variable));
+}
+
+CAMLprim value llvm_di_variable_get_file(LLVMMetadataRef Variable) {
+  return (ptr_to_option(LLVMDIVariableGetFile(Variable)));
+}
+
+CAMLprim value llvm_get_metadata_kind(LLVMMetadataRef Metadata) {
+  return Val_int(LLVMGetMetadataKind(Metadata));
+}
diff --git a/llvm/bindings/ocaml/debuginfo/llvm_debuginfo.ml b/llvm/bindings/ocaml/debuginfo/llvm_debuginfo.ml
new file mode 100644
index 0000000000000000000000000000000000000000..232efd4cdd387399ee3f2545a4ce2a77ddd7a0b3
--- /dev/null
+++ b/llvm/bindings/ocaml/debuginfo/llvm_debuginfo.ml
@@ -0,0 +1,567 @@
+(*===-- llvm_debuginfo.ml - LLVM OCaml Interface --------------*- OCaml -*-===*
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===----------------------------------------------------------------------===*)
+
+type lldibuilder
+
+(** Source languages known by DWARF. *)
+module DWARFSourceLanguageKind = struct
+  type t =
+    | C89
+    | C
+    | Ada83
+    | C_plus_plus
+    | Cobol74
+    | Cobol85
+    | Fortran77
+    | Fortran90
+    | Pascal83
+    | Modula2
+    (*  New in DWARF v3: *)
+    | LLVMJava
+    | C99
+    | Ada95
+    | Fortran95
+    | PLI
+    | ObjC
+    | ObjC_plus_plus
+    | UPC
+    | D
+    (*  New in DWARF v4: *)
+    | LLVMPython
+    (*  New in DWARF v5: *)
+    | LLVMOpenCL
+    | Go
+    | Modula3
+    | Haskell
+    | C_plus_plus_03
+    | C_plus_plus_11
+    | OCaml
+    | Rust
+    | C11
+    | Swift
+    | Julia
+    | Dylan
+    | C_plus_plus_14
+    | Fortran03
+    | Fortran08
+    | RenderScript
+    | BLISS
+    (*  Vendor extensions: *)
+    | LLVMMips_Assembler
+    | GOOGLE_RenderScript
+    | BORLAND_Delphi
+end
+
+module DIFlag = struct
+  type t =
+    | Zero
+    | Private
+    | Protected
+    | Public
+    | FwdDecl
+    | AppleBlock
+    | ReservedBit4
+    | Virtual
+    | Artificial
+    | Explicit
+    | Prototyped
+    | ObjcClassComplete
+    | ObjectPointer
+    | Vector
+    | StaticMember
+    | LValueReference
+    | RValueReference
+    | Reserved
+    | SingleInheritance
+    | MultipleInheritance
+    | VirtualInheritance
+    | IntroducedVirtual
+    | BitField
+    | NoReturn
+    | TypePassByValue
+    | TypePassByReference
+    | EnumClass
+    | FixedEnum
+    | Thunk
+    | NonTrivial
+    | BigEndian
+    | LittleEndian
+    | IndirectVirtualBase
+    | Accessibility
+    | PtrToMemberRep
+end
+
+type lldiflags
+
+external diflags_get : DIFlag.t -> lldiflags = "llvm_diflags_get"
+
+external diflags_set : lldiflags -> DIFlag.t -> lldiflags = "llvm_diflags_set"
+
+external diflags_test : lldiflags -> DIFlag.t -> bool = "llvm_diflags_test"
+
+(** The kind of metadata nodes. *)
+module MetadataKind = struct
+  type t =
+    | MDStringMetadataKind
+    | ConstantAsMetadataMetadataKind
+    | LocalAsMetadataMetadataKind
+    | DistinctMDOperandPlaceholderMetadataKind
+    | MDTupleMetadataKind
+    | DILocationMetadataKind
+    | DIExpressionMetadataKind
+    | DIGlobalVariableExpressionMetadataKind
+    | GenericDINodeMetadataKind
+    | DISubrangeMetadataKind
+    | DIEnumeratorMetadataKind
+    | DIBasicTypeMetadataKind
+    | DIDerivedTypeMetadataKind
+    | DICompositeTypeMetadataKind
+    | DISubroutineTypeMetadataKind
+    | DIFileMetadataKind
+    | DICompileUnitMetadataKind
+    | DISubprogramMetadataKind
+    | DILexicalBlockMetadataKind
+    | DILexicalBlockFileMetadataKind
+    | DINamespaceMetadataKind
+    | DIModuleMetadataKind
+    | DITemplateTypeParameterMetadataKind
+    | DITemplateValueParameterMetadataKind
+    | DIGlobalVariableMetadataKind
+    | DILocalVariableMetadataKind
+    | DILabelMetadataKind
+    | DIObjCPropertyMetadataKind
+    | DIImportedEntityMetadataKind
+    | DIMacroMetadataKind
+    | DIMacroFileMetadataKind
+    | DICommonBlockMetadataKind
+end
+
+(** The amount of debug information to emit. *)
+module DWARFEmissionKind = struct
+  type t = None | Full | LineTablesOnly
+end
+
+external debug_metadata_version : unit -> int = "llvm_debug_metadata_version"
+
+external get_module_debug_metadata_version : Llvm.llmodule -> int
+  = "llvm_get_module_debug_metadata_version"
+
+external dibuilder : Llvm.llmodule -> lldibuilder = "llvm_dibuilder"
+
+external dibuild_finalize : lldibuilder -> unit = "llvm_dibuild_finalize"
+
+(* See LLVMDIBuilderCreateCompileUnit for argument details. *)
+external dibuild_create_compile_unit :
+  lldibuilder ->
+  DWARFSourceLanguageKind.t ->
+  file_ref:Llvm.llmetadata ->
+  producer:string ->
+  is_optimized:bool ->
+  flags:string ->
+  runtime_ver:int ->
+  split_name:string ->
+  DWARFEmissionKind.t ->
+  dwoid:int ->
+  di_inlining:bool ->
+  di_profiling:bool ->
+  sys_root:string ->
+  sdk:string ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_compile_unit_bytecode" "llvm_dibuild_create_compile_unit_native"
+
+external dibuild_create_file :
+  lldibuilder -> filename:string -> directory:string -> Llvm.llmetadata
+  = "llvm_dibuild_create_file"
+
+external dibuild_create_module :
+  lldibuilder ->
+  parent_ref:Llvm.llmetadata ->
+  name:string ->
+  config_macros:string ->
+  include_path:string ->
+  sys_root:string ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_module_bytecode" "llvm_dibuild_create_module_native"
+
+external dibuild_create_namespace :
+  lldibuilder ->
+  parent_ref:Llvm.llmetadata ->
+  name:string ->
+  export_symbols:bool ->
+  Llvm.llmetadata = "llvm_dibuild_create_namespace"
+
+external dibuild_create_function :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  linkage_name:string ->
+  file:Llvm.llmetadata ->
+  line_no:int ->
+  ty:Llvm.llmetadata ->
+  is_local_to_unit:bool ->
+  is_definition:bool ->
+  scope_line:int ->
+  flags:lldiflags ->
+  is_optimized:bool ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_function_bytecode" "llvm_dibuild_create_function_native"
+
+external dibuild_create_lexical_block :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  file:Llvm.llmetadata ->
+  line:int ->
+  column:int ->
+  Llvm.llmetadata = "llvm_dibuild_create_lexical_block"
+
+external dibuild_create_debug_location_helper :
+  Llvm.llcontext ->
+  line:int ->
+  column:int ->
+  scope:Llvm.llmetadata ->
+  inlined_at:Llvm.llmetadata ->
+  Llvm.llmetadata = "llvm_dibuild_create_debug_location"
+
+external llmetadata_null : unit -> Llvm.llmetadata = "llvm_metadata_null"
+
+let dibuild_create_debug_location ?(inlined_at = llmetadata_null ()) llctx ~line
+    ~column ~scope =
+  dibuild_create_debug_location_helper llctx line column scope inlined_at
+
+external di_location_get_line : location:Llvm.llmetadata -> int
+  = "llvm_di_location_get_line"
+
+external di_location_get_column : location:Llvm.llmetadata -> int
+  = "llvm_di_location_get_column"
+
+external di_location_get_scope : location:Llvm.llmetadata -> Llvm.llmetadata
+  = "llvm_di_location_get_scope"
+
+external di_location_get_inlined_at :
+  location:Llvm.llmetadata -> Llvm.llmetadata option
+  = "llvm_di_location_get_inlined_at"
+
+external di_scope_get_file : scope:Llvm.llmetadata -> Llvm.llmetadata option
+  = "llvm_di_scope_get_file"
+
+external di_file_get_directory : file:Llvm.llmetadata -> string
+  = "llvm_di_file_get_directory"
+
+external di_file_get_filename : file:Llvm.llmetadata -> string
+  = "llvm_di_file_get_filename"
+
+external di_file_get_source : file:Llvm.llmetadata -> string
+  = "llvm_di_file_get_source"
+
+external dibuild_get_or_create_type_array :
+  lldibuilder -> data:Llvm.llmetadata array -> Llvm.llmetadata
+  = "llvm_dibuild_get_or_create_type_array"
+
+external dibuild_get_or_create_array :
+  lldibuilder -> data:Llvm.llmetadata array -> Llvm.llmetadata
+  = "llvm_dibuild_get_or_create_array"
+
+external dibuild_create_subroutine_type :
+  lldibuilder ->
+  file:Llvm.llmetadata ->
+  param_types:Llvm.llmetadata array ->
+  lldiflags ->
+  Llvm.llmetadata = "llvm_dibuild_create_subroutine_type"
+
+external dibuild_create_enumerator :
+  lldibuilder -> name:string -> value:int -> is_unsigned:bool -> Llvm.llmetadata
+  = "llvm_dibuild_create_enumerator"
+
+external dibuild_create_enumeration_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  elements:Llvm.llmetadata array ->
+  class_ty:Llvm.llmetadata ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_enumeration_type_bytecode" "llvm_dibuild_create_enumeration_type_native"
+
+external dibuild_create_union_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  lldiflags ->
+  elements:Llvm.llmetadata array ->
+  run_time_language:int ->
+  unique_id:string ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_union_type_bytecode" "llvm_dibuild_create_union_type_native"
+
+external dibuild_create_array_type :
+  lldibuilder ->
+  size:int ->
+  align_in_bits:int ->
+  ty:Llvm.llmetadata ->
+  subscripts:Llvm.llmetadata array ->
+  Llvm.llmetadata = "llvm_dibuild_create_array_type"
+
+external dibuild_create_vector_type :
+  lldibuilder ->
+  size:int ->
+  align_in_bits:int ->
+  ty:Llvm.llmetadata ->
+  subscripts:Llvm.llmetadata array ->
+  Llvm.llmetadata = "llvm_dibuild_create_array_type"
+
+external dibuild_create_unspecified_type :
+  lldibuilder -> name:string -> Llvm.llmetadata
+  = "llvm_dibuild_create_unspecified_type"
+
+external dibuild_create_basic_type :
+  lldibuilder ->
+  name:string ->
+  size_in_bits:int ->
+  encoding:int ->
+  lldiflags ->
+  Llvm.llmetadata = "llvm_dibuild_create_basic_type"
+
+external dibuild_create_pointer_type :
+  lldibuilder ->
+  pointee_ty:Llvm.llmetadata ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  address_space:int ->
+  name:string ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_pointer_type_bytecode" "llvm_dibuild_create_pointer_type_native"
+
+external dibuild_create_struct_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  lldiflags ->
+  derived_from:Llvm.llmetadata ->
+  elements:Llvm.llmetadata array ->
+  DWARFSourceLanguageKind.t ->
+  vtable_holder:Llvm.llmetadata ->
+  unique_id:string ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_struct_type_bytecode" "llvm_dibuild_create_struct_type_native"
+
+external dibuild_create_member_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  offset_in_bits:int ->
+  lldiflags ->
+  ty:Llvm.llmetadata ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_member_type_bytecode" "llvm_dibuild_create_member_type_native"
+
+external dibuild_create_static_member_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  ty:Llvm.llmetadata ->
+  lldiflags ->
+  const_val:Llvm.llvalue ->
+  align_in_bits:int ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_static_member_type_bytecode" "llvm_dibuild_create_static_member_type_native"
+
+external dibuild_create_member_pointer_type :
+  lldibuilder ->
+  pointee_type:Llvm.llmetadata ->
+  class_type:Llvm.llmetadata ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  lldiflags ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_member_pointer_type_bytecode" "llvm_dibuild_create_member_pointer_type_native"
+
+external dibuild_create_object_pointer_type :
+  lldibuilder -> Llvm.llmetadata -> Llvm.llmetadata
+  = "llvm_dibuild_create_object_pointer_type"
+
+external dibuild_create_qualified_type :
+  lldibuilder -> tag:int -> Llvm.llmetadata -> Llvm.llmetadata
+  = "llvm_dibuild_create_qualified_type"
+
+external dibuild_create_reference_type :
+  lldibuilder -> tag:int -> Llvm.llmetadata -> Llvm.llmetadata
+  = "llvm_dibuild_create_reference_type"
+
+external dibuild_create_null_ptr_type : lldibuilder -> Llvm.llmetadata
+  = "llvm_dibuild_create_null_ptr_type"
+
+external dibuild_create_typedef :
+  lldibuilder ->
+  ty:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_no:int ->
+  scope:Llvm.llmetadata ->
+  align_in_bits:int ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_typedef_bytecode" "llvm_dibuild_create_typedef_native"
+
+external dibuild_create_inheritance :
+  lldibuilder ->
+  ty:Llvm.llmetadata ->
+  base_ty:Llvm.llmetadata ->
+  base_offset:int ->
+  vb_ptr_offset:int ->
+  lldiflags ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_inheritance_bytecode" "llvm_dibuild_create_inheritance_native"
+
+external dibuild_create_forward_decl :
+  lldibuilder ->
+  tag:int ->
+  name:string ->
+  scope:Llvm.llmetadata ->
+  file:Llvm.llmetadata ->
+  line:int ->
+  runtime_lang:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  unique_identifier:string ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_forward_decl_bytecode" "llvm_dibuild_create_forward_decl_native"
+
+external dibuild_create_replaceable_composite_type :
+  lldibuilder ->
+  tag:int ->
+  name:string ->
+  scope:Llvm.llmetadata ->
+  file:Llvm.llmetadata ->
+  line:int ->
+  runtime_lang:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  lldiflags ->
+  unique_identifier:string ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_replaceable_composite_type_bytecode" "llvm_dibuild_create_replaceable_composite_type_native"
+
+external dibuild_create_bit_field_member_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_num:int ->
+  size_in_bits:int ->
+  offset_in_bits:int ->
+  storage_offset_in_bits:int ->
+  lldiflags ->
+  ty:Llvm.llmetadata ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_bit_field_member_type_bytecode" "llvm_dibuild_create_bit_field_member_type_native"
+
+external dibuild_create_class_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  offset_in_bits:int ->
+  lldiflags ->
+  derived_from:Llvm.llmetadata ->
+  elements:Llvm.llmetadata array ->
+  vtable_holder:Llvm.llmetadata ->
+  template_params_node:Llvm.llmetadata ->
+  unique_identifier:string ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_class_type_bytecode" "llvm_dibuild_create_class_type_native"
+
+external dibuild_create_artificial_type :
+  lldibuilder -> ty:Llvm.llmetadata -> Llvm.llmetadata
+  = "llvm_dibuild_create_artificial_type"
+
+external di_type_get_name : Llvm.llmetadata -> string = "llvm_di_type_get_name"
+
+external di_type_get_size_in_bits : Llvm.llmetadata -> int
+  = "llvm_di_type_get_size_in_bits"
+
+external di_type_get_offset_in_bits : Llvm.llmetadata -> int
+  = "llvm_di_type_get_offset_in_bits"
+
+external di_type_get_align_in_bits : Llvm.llmetadata -> int
+  = "llvm_di_type_get_align_in_bits"
+
+external di_type_get_line : Llvm.llmetadata -> int = "llvm_di_type_get_line"
+
+external di_type_get_flags : Llvm.llmetadata -> lldiflags
+  = "llvm_di_type_get_flags"
+
+external get_subprogram : Llvm.llvalue -> Llvm.llmetadata option
+  = "llvm_get_subprogram"
+
+external set_subprogram : Llvm.llvalue -> Llvm.llmetadata -> unit
+  = "llvm_set_subprogram"
+
+external di_subprogram_get_line : Llvm.llmetadata -> int
+  = "llvm_di_subprogram_get_line"
+
+external instr_get_debug_loc : Llvm.llvalue -> Llvm.llmetadata option
+  = "llvm_instr_get_debug_loc"
+
+external instr_set_debug_loc_helper : Llvm.llvalue -> Llvm.llmetadata -> unit
+  = "llvm_instr_set_debug_loc"
+
+let instr_set_debug_loc i mopt =
+  match mopt with
+  | None -> instr_set_debug_loc_helper i (llmetadata_null ())
+  | Some m -> instr_set_debug_loc_helper i m
+
+external dibuild_create_constant_value_expression :
+  lldibuilder -> int -> Llvm.llmetadata
+  = "llvm_dibuild_create_constant_value_expression"
+
+external dibuild_create_global_variable_expression :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  linkage:string ->
+  file:Llvm.llmetadata ->
+  line:int ->
+  ty:Llvm.llmetadata ->
+  is_local_to_unit:bool ->
+  expr:Llvm.llmetadata ->
+  decl:Llvm.llmetadata ->
+  align_in_bits:int ->
+  Llvm.llmetadata
+  = "llvm_dibuild_create_global_variable_expression_bytecode" "llvm_dibuild_create_global_variable_expression_native"
+
+external di_global_variable_expression_get_variable :
+  Llvm.llmetadata -> Llvm.llmetadata option
+  = "llvm_di_global_variable_expression_get_variable"
+
+external di_variable_get_line : Llvm.llmetadata -> int
+  = "llvm_di_variable_get_line"
+
+external di_variable_get_file : Llvm.llmetadata -> Llvm.llmetadata option
+  = "llvm_di_variable_get_file"
+
+external get_metadata_kind : Llvm.llmetadata -> MetadataKind.t
+  = "llvm_get_metadata_kind"
diff --git a/llvm/bindings/ocaml/debuginfo/llvm_debuginfo.mli b/llvm/bindings/ocaml/debuginfo/llvm_debuginfo.mli
new file mode 100644
index 0000000000000000000000000000000000000000..7f2bb6dd53bf9a90048f7465e4e2534f8d218e09
--- /dev/null
+++ b/llvm/bindings/ocaml/debuginfo/llvm_debuginfo.mli
@@ -0,0 +1,625 @@
+(*===-- llvm_debuginfo.mli - LLVM OCaml Interface -------------*- OCaml -*-===*
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===----------------------------------------------------------------------===*)
+
+type lldibuilder
+
+(** Source languages known by DWARF. *)
+module DWARFSourceLanguageKind : sig
+  type t =
+    | C89
+    | C
+    | Ada83
+    | C_plus_plus
+    | Cobol74
+    | Cobol85
+    | Fortran77
+    | Fortran90
+    | Pascal83
+    | Modula2
+    (*  New in DWARF v3: *)
+    | LLVMJava
+    | C99
+    | Ada95
+    | Fortran95
+    | PLI
+    | ObjC
+    | ObjC_plus_plus
+    | UPC
+    | D
+    (*  New in DWARF v4: *)
+    | LLVMPython
+    (*  New in DWARF v5: *)
+    | LLVMOpenCL
+    | Go
+    | Modula3
+    | Haskell
+    | C_plus_plus_03
+    | C_plus_plus_11
+    | OCaml
+    | Rust
+    | C11
+    | Swift
+    | Julia
+    | Dylan
+    | C_plus_plus_14
+    | Fortran03
+    | Fortran08
+    | RenderScript
+    | BLISS
+    (*  Vendor extensions: *)
+    | LLVMMips_Assembler
+    | GOOGLE_RenderScript
+    | BORLAND_Delphi
+end
+
+module DIFlag : sig
+  type t =
+    | Zero
+    | Private
+    | Protected
+    | Public
+    | FwdDecl
+    | AppleBlock
+    | ReservedBit4
+    | Virtual
+    | Artificial
+    | Explicit
+    | Prototyped
+    | ObjcClassComplete
+    | ObjectPointer
+    | Vector
+    | StaticMember
+    | LValueReference
+    | RValueReference
+    | Reserved
+    | SingleInheritance
+    | MultipleInheritance
+    | VirtualInheritance
+    | IntroducedVirtual
+    | BitField
+    | NoReturn
+    | TypePassByValue
+    | TypePassByReference
+    | EnumClass
+    | FixedEnum
+    | Thunk
+    | NonTrivial
+    | BigEndian
+    | LittleEndian
+    | IndirectVirtualBase
+    | Accessibility
+    | PtrToMemberRep
+end
+
+type lldiflags
+(** An opaque type to represent OR of multiple DIFlag.t. *)
+
+val diflags_get : DIFlag.t -> lldiflags
+(** [diflags_set f] Construct an lldiflags value with a single flag [f]. *)
+
+val diflags_set : lldiflags -> DIFlag.t -> lldiflags
+(** [diflags_set fs f] Include flag [f] in [fs] and return the new value. *)
+
+val diflags_test : lldiflags -> DIFlag.t -> bool
+(** [diflags_test fs f] Does [fs] contain flag [f]? *)
+
+(** The kind of metadata nodes. *)
+module MetadataKind : sig
+  type t =
+    | MDStringMetadataKind
+    | ConstantAsMetadataMetadataKind
+    | LocalAsMetadataMetadataKind
+    | DistinctMDOperandPlaceholderMetadataKind
+    | MDTupleMetadataKind
+    | DILocationMetadataKind
+    | DIExpressionMetadataKind
+    | DIGlobalVariableExpressionMetadataKind
+    | GenericDINodeMetadataKind
+    | DISubrangeMetadataKind
+    | DIEnumeratorMetadataKind
+    | DIBasicTypeMetadataKind
+    | DIDerivedTypeMetadataKind
+    | DICompositeTypeMetadataKind
+    | DISubroutineTypeMetadataKind
+    | DIFileMetadataKind
+    | DICompileUnitMetadataKind
+    | DISubprogramMetadataKind
+    | DILexicalBlockMetadataKind
+    | DILexicalBlockFileMetadataKind
+    | DINamespaceMetadataKind
+    | DIModuleMetadataKind
+    | DITemplateTypeParameterMetadataKind
+    | DITemplateValueParameterMetadataKind
+    | DIGlobalVariableMetadataKind
+    | DILocalVariableMetadataKind
+    | DILabelMetadataKind
+    | DIObjCPropertyMetadataKind
+    | DIImportedEntityMetadataKind
+    | DIMacroMetadataKind
+    | DIMacroFileMetadataKind
+    | DICommonBlockMetadataKind
+end
+
+(** The amount of debug information to emit. *)
+module DWARFEmissionKind : sig
+  type t = None | Full | LineTablesOnly
+end
+
+val debug_metadata_version : unit -> int
+(** [debug_metadata_version ()] The current debug metadata version number *)
+
+val get_module_debug_metadata_version : Llvm.llmodule -> int
+(** [get_module_debug_metadata_version m] Version of metadata present in [m]. *)
+
+val dibuilder : Llvm.llmodule -> lldibuilder
+(** [dibuilder m] Create a debug info builder for [m]. *)
+
+val dibuild_finalize : lldibuilder -> unit
+(** [dibuild_finalize dib] Construct any deferred debug info descriptors. *)
+
+val dibuild_create_compile_unit :
+  lldibuilder ->
+  DWARFSourceLanguageKind.t ->
+  file_ref:Llvm.llmetadata ->
+  producer:string ->
+  is_optimized:bool ->
+  flags:string ->
+  runtime_ver:int ->
+  split_name:string ->
+  DWARFEmissionKind.t ->
+  dwoid:int ->
+  di_inlining:bool ->
+  di_profiling:bool ->
+  sys_root:string ->
+  sdk:string ->
+  Llvm.llmetadata
+(** [dibuild_create_compile_unit] A CompileUnit provides an anchor for all
+    debugging information generated during this instance of compilation.
+    See LLVMDIBuilderCreateCompileUnit. *)
+
+val dibuild_create_file :
+  lldibuilder -> filename:string -> directory:string -> Llvm.llmetadata
+(** [dibuild_create_file] Create a file descriptor to hold debugging information
+    for a file. See LLVMDIBuilderCreateFile. *)
+
+val dibuild_create_module :
+  lldibuilder ->
+  parent_ref:Llvm.llmetadata ->
+  name:string ->
+  config_macros:string ->
+  include_path:string ->
+  sys_root:string ->
+  Llvm.llmetadata
+(** [dibuild_create_module] Create a new descriptor for a module with the
+    specified parent scope. See LLVMDIBuilderCreateModule. *)
+
+val dibuild_create_namespace :
+  lldibuilder ->
+  parent_ref:Llvm.llmetadata ->
+  name:string ->
+  export_symbols:bool ->
+  Llvm.llmetadata
+(** [dibuild_create_namespace] Create a new descriptor for a namespace with
+    the specified parent scope. See LLVMDIBuilderCreateNameSpace *)
+
+val dibuild_create_function :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  linkage_name:string ->
+  file:Llvm.llmetadata ->
+  line_no:int ->
+  ty:Llvm.llmetadata ->
+  is_local_to_unit:bool ->
+  is_definition:bool ->
+  scope_line:int ->
+  flags:lldiflags ->
+  is_optimized:bool ->
+  Llvm.llmetadata
+(** [dibuild_create_function] Create a new descriptor for the specified
+    subprogram. See LLVMDIBuilderCreateFunction. *)
+
+val dibuild_create_lexical_block :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  file:Llvm.llmetadata ->
+  line:int ->
+  column:int ->
+  Llvm.llmetadata
+(** [dibuild_create_lexical_block] Create a descriptor for a lexical block with
+    the specified parent context. See LLVMDIBuilderCreateLexicalBlock *)
+
+val llmetadata_null : unit -> Llvm.llmetadata
+(** [llmetadata_null ()] llmetadata is a wrapper around "llvm::Metadata *".
+    This function returns a nullptr valued llmetadata. For example, it
+    can be used to convey an llmetadata for "void" type. *)
+
+val dibuild_create_debug_location :
+  ?inlined_at:Llvm.llmetadata ->
+  Llvm.llcontext ->
+  line:int ->
+  column:int ->
+  scope:Llvm.llmetadata ->
+  Llvm.llmetadata
+(** [dibuild_create] Create a new DebugLocation that describes a source
+    location. See LLVMDIBuilderCreateDebugLocation *)
+
+val di_location_get_line : location:Llvm.llmetadata -> int
+(** [di_location_get_line l] Get the line number of debug location [l]. *)
+
+val di_location_get_column : location:Llvm.llmetadata -> int
+(** [di_location_get_column l] Get the column number of debug location [l]. *)
+
+val di_location_get_scope : location:Llvm.llmetadata -> Llvm.llmetadata
+(** [di_location_get_scope l] Get the local scope associated with
+    debug location [l]. *)
+
+val di_location_get_inlined_at :
+  location:Llvm.llmetadata -> Llvm.llmetadata option
+(** [di_location_get_inlined_at l] Get the "inlined at" location associated with
+    debug location [l], if it exists. *)
+
+val di_scope_get_file : scope:Llvm.llmetadata -> Llvm.llmetadata option
+(** [di_scope_get_file l] Get the metadata of the file associated with scope [s]
+    if it exists. *)
+
+val di_file_get_directory : file:Llvm.llmetadata -> string
+(** [di_file_get_directory f] Get the directory of file [f]. *)
+
+val di_file_get_filename : file:Llvm.llmetadata -> string
+(** [di_file_get_filename f] Get the name of file [f]. *)
+
+val di_file_get_source : file:Llvm.llmetadata -> string
+(** [di_file_get_source f] Get the source of file [f]. *)
+
+val dibuild_get_or_create_type_array :
+  lldibuilder -> data:Llvm.llmetadata array -> Llvm.llmetadata
+(** [dibuild_get_or_create_type_array] Create a type array.
+    See LLVMDIBuilderGetOrCreateTypeArray. *)
+
+val dibuild_get_or_create_array :
+  lldibuilder -> data:Llvm.llmetadata array -> Llvm.llmetadata
+(** [dibuild_get_or_create_array] Create an array of DI Nodes.
+    See LLVMDIBuilderGetOrCreateArray. *)
+
+val dibuild_create_constant_value_expression :
+  lldibuilder -> int -> Llvm.llmetadata
+(** [dibuild_create_constant_value_expression] Create a new descriptor for
+    the specified variable that does not have an address, but does have
+    a constant value. See LLVMDIBuilderCreateConstantValueExpression. *)
+
+val dibuild_create_global_variable_expression :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  linkage:string ->
+  file:Llvm.llmetadata ->
+  line:int ->
+  ty:Llvm.llmetadata ->
+  is_local_to_unit:bool ->
+  expr:Llvm.llmetadata ->
+  decl:Llvm.llmetadata ->
+  align_in_bits:int ->
+  Llvm.llmetadata
+(** [dibuild_create_global_variable_expression] Create a new descriptor for
+    the specified variable. See LLVMDIBuilderCreateGlobalVariableExpression. *)
+
+val di_global_variable_expression_get_variable :
+  Llvm.llmetadata -> Llvm.llmetadata option
+(** [di_global_variable_expression_get_variable gve] returns the debug variable
+    of [gve], which must be a [DIGlobalVariableExpression].
+    See LLVMDIGlobalVariableExpressionGetVariable. *)
+
+val di_variable_get_line : Llvm.llmetadata -> int
+(** [di_variable_get_line v] returns the line number of the variable [v].
+    See LLVMDIVariableGetLine. *)
+
+val di_variable_get_file : Llvm.llmetadata -> Llvm.llmetadata option
+(** [di_variable_get_file v] returns the file of the variable [v].
+    See LLVMDIVariableGetFile. *)
+
+val dibuild_create_subroutine_type :
+  lldibuilder ->
+  file:Llvm.llmetadata ->
+  param_types:Llvm.llmetadata array ->
+  lldiflags ->
+  Llvm.llmetadata
+(** [dibuild_create_subroutine_type] Create subroutine type.
+    See LLVMDIBuilderCreateSubroutineType *)
+
+val dibuild_create_enumerator :
+  lldibuilder -> name:string -> value:int -> is_unsigned:bool -> Llvm.llmetadata
+(** [dibuild_create_enumerator] Create debugging information entry for an
+    enumerator. See LLVMDIBuilderCreateEnumerator *)
+
+val dibuild_create_enumeration_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  elements:Llvm.llmetadata array ->
+  class_ty:Llvm.llmetadata ->
+  Llvm.llmetadata
+(** [dibuild_create_enumeration_type] Create debugging information entry for
+    an enumeration. See LLVMDIBuilderCreateEnumerationType. *)
+
+val dibuild_create_union_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  lldiflags ->
+  elements:Llvm.llmetadata array ->
+  run_time_language:int ->
+  unique_id:string ->
+  Llvm.llmetadata
+(** [dibuild_create_union_type] Create debugging information entry for a union.
+    See LLVMDIBuilderCreateUnionType. *)
+
+val dibuild_create_array_type :
+  lldibuilder ->
+  size:int ->
+  align_in_bits:int ->
+  ty:Llvm.llmetadata ->
+  subscripts:Llvm.llmetadata array ->
+  Llvm.llmetadata
+(** [dibuild_create_array_type] Create debugging information entry for an array.
+    See LLVMDIBuilderCreateArrayType. *)
+
+val dibuild_create_vector_type :
+  lldibuilder ->
+  size:int ->
+  align_in_bits:int ->
+  ty:Llvm.llmetadata ->
+  subscripts:Llvm.llmetadata array ->
+  Llvm.llmetadata
+(** [dibuild_create_vector_type] Create debugging information entry for a
+    vector type. See LLVMDIBuilderCreateVectorType. *)
+
+val dibuild_create_unspecified_type :
+  lldibuilder -> name:string -> Llvm.llmetadata
+(** [dibuild_create_unspecified_type] Create a DWARF unspecified type. *)
+
+val dibuild_create_basic_type :
+  lldibuilder ->
+  name:string ->
+  size_in_bits:int ->
+  encoding:int ->
+  lldiflags ->
+  Llvm.llmetadata
+(** [dibuild_create_basic_type] Create debugging information entry for a basic
+    type. See LLVMDIBuilderCreateBasicType. *)
+
+val dibuild_create_pointer_type :
+  lldibuilder ->
+  pointee_ty:Llvm.llmetadata ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  address_space:int ->
+  name:string ->
+  Llvm.llmetadata
+(** [dibuild_create_pointer_type] Create debugging information entry for a
+    pointer. See LLVMDIBuilderCreatePointerType. *)
+
+val dibuild_create_struct_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  lldiflags ->
+  derived_from:Llvm.llmetadata ->
+  elements:Llvm.llmetadata array ->
+  DWARFSourceLanguageKind.t ->
+  vtable_holder:Llvm.llmetadata ->
+  unique_id:string ->
+  Llvm.llmetadata
+(** [dibuild_create_struct_type] Create debugging information entry for a
+    struct. See LLVMDIBuilderCreateStructType *)
+
+val dibuild_create_member_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  offset_in_bits:int ->
+  lldiflags ->
+  ty:Llvm.llmetadata ->
+  Llvm.llmetadata
+(** [dibuild_create_member_type] Create debugging information entry for a
+    member. See LLVMDIBuilderCreateMemberType. *)
+
+val dibuild_create_static_member_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  ty:Llvm.llmetadata ->
+  lldiflags ->
+  const_val:Llvm.llvalue ->
+  align_in_bits:int ->
+  Llvm.llmetadata
+(** [dibuild_create_static_member_type] Create debugging information entry for
+    a C++ static data member. See LLVMDIBuilderCreateStaticMemberType *)
+
+val dibuild_create_member_pointer_type :
+  lldibuilder ->
+  pointee_type:Llvm.llmetadata ->
+  class_type:Llvm.llmetadata ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  lldiflags ->
+  Llvm.llmetadata
+(** [dibuild_create_member_pointer_type] Create debugging information entry for
+    a pointer to member. See LLVMDIBuilderCreateMemberPointerType *)
+
+val dibuild_create_object_pointer_type :
+  lldibuilder -> Llvm.llmetadata -> Llvm.llmetadata
+(** [dibuild_create_object_pointer_type dib ty] Create a uniqued DIType* clone
+  with FlagObjectPointer and FlagArtificial set. [dib] is the dibuilder
+  value and [ty] the underlying type to which this pointer points. *)
+
+val dibuild_create_qualified_type :
+  lldibuilder -> tag:int -> Llvm.llmetadata -> Llvm.llmetadata
+(** [dibuild_create_qualified_type dib tag ty] Create debugging information
+    entry for a qualified type, e.g. 'const int'. [dib] is the dibuilder value,
+    [tag] identifyies the type and [ty] is the base type. *)
+
+val dibuild_create_reference_type :
+  lldibuilder -> tag:int -> Llvm.llmetadata -> Llvm.llmetadata
+(** [dibuild_create_reference_type dib tag ty] Create debugging information
+    entry for a reference type. [dib] is the dibuilder value, [tag] identifyies
+    the type and [ty] is the base type. *)
+
+val dibuild_create_null_ptr_type : lldibuilder -> Llvm.llmetadata
+(** [dibuild_create_null_ptr_type dib] Create C++11 nullptr type. *)
+
+val dibuild_create_typedef :
+  lldibuilder ->
+  ty:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_no:int ->
+  scope:Llvm.llmetadata ->
+  align_in_bits:int ->
+  Llvm.llmetadata
+(** [dibuild_create_typedef] Create debugging information entry for a typedef.
+    See LLVMDIBuilderCreateTypedef. *)
+
+val dibuild_create_inheritance :
+  lldibuilder ->
+  ty:Llvm.llmetadata ->
+  base_ty:Llvm.llmetadata ->
+  base_offset:int ->
+  vb_ptr_offset:int ->
+  lldiflags ->
+  Llvm.llmetadata
+(** [dibuild_create_inheritance] Create debugging information entry
+    to establish inheritance relationship between two types.
+    See LLVMDIBuilderCreateInheritance. *)
+
+val dibuild_create_forward_decl :
+  lldibuilder ->
+  tag:int ->
+  name:string ->
+  scope:Llvm.llmetadata ->
+  file:Llvm.llmetadata ->
+  line:int ->
+  runtime_lang:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  unique_identifier:string ->
+  Llvm.llmetadata
+(** [dibuild_create_forward_decl] Create a permanent forward-declared type.
+    See LLVMDIBuilderCreateForwardDecl. *)
+
+val dibuild_create_replaceable_composite_type :
+  lldibuilder ->
+  tag:int ->
+  name:string ->
+  scope:Llvm.llmetadata ->
+  file:Llvm.llmetadata ->
+  line:int ->
+  runtime_lang:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  lldiflags ->
+  unique_identifier:string ->
+  Llvm.llmetadata
+(** [dibuild_create_replaceable_composite_type] Create a temporary
+    forward-declared type. See LLVMDIBuilderCreateReplaceableCompositeType. *)
+
+val dibuild_create_bit_field_member_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_num:int ->
+  size_in_bits:int ->
+  offset_in_bits:int ->
+  storage_offset_in_bits:int ->
+  lldiflags ->
+  ty:Llvm.llmetadata ->
+  Llvm.llmetadata
+(** [dibuild_create_bit_field_member_type] Create debugging information entry
+    for a bit field member. See LLVMDIBuilderCreateBitFieldMemberType. *)
+
+val dibuild_create_class_type :
+  lldibuilder ->
+  scope:Llvm.llmetadata ->
+  name:string ->
+  file:Llvm.llmetadata ->
+  line_number:int ->
+  size_in_bits:int ->
+  align_in_bits:int ->
+  offset_in_bits:int ->
+  lldiflags ->
+  derived_from:Llvm.llmetadata ->
+  elements:Llvm.llmetadata array ->
+  vtable_holder:Llvm.llmetadata ->
+  template_params_node:Llvm.llmetadata ->
+  unique_identifier:string ->
+  Llvm.llmetadata
+(** [dibuild_create_class_type] Create debugging information entry for a class.
+    See LLVMDIBuilderCreateClassType. *)
+
+val dibuild_create_artificial_type :
+  lldibuilder -> ty:Llvm.llmetadata -> Llvm.llmetadata
+(** [dibuild_create_artificial_type dib ty] Create a uniqued DIType* clone with
+    FlagArtificial set.
+    [dib] is the dibuilder value and [ty] the underlying type. *)
+
+val di_type_get_name : Llvm.llmetadata -> string
+(** [di_type_get_name m] Get the name of DIType [m]. *)
+
+val di_type_get_size_in_bits : Llvm.llmetadata -> int
+(** [di_type_get_size_in_bits m] Get size in bits of DIType [m]. *)
+
+val di_type_get_offset_in_bits : Llvm.llmetadata -> int
+(** [di_type_get_offset_in_bits m] Get offset in bits of DIType [m]. *)
+
+val di_type_get_align_in_bits : Llvm.llmetadata -> int
+(** [di_type_get_align_in_bits m] Get alignment in bits of DIType [m]. *)
+
+val di_type_get_line : Llvm.llmetadata -> int
+(** [di_type_get_line m] Get source line where DIType [m] is declared. *)
+
+val di_type_get_flags : Llvm.llmetadata -> lldiflags
+(** [di_type_get_flags m] Get the flags associated with DIType [m]. *)
+
+val get_subprogram : Llvm.llvalue -> Llvm.llmetadata option
+(** [get_subprogram f] Get the metadata of the subprogram attached to
+    function [f]. *)
+
+val set_subprogram : Llvm.llvalue -> Llvm.llmetadata -> unit
+(** [set_subprogram f m] Set the subprogram [m] attached to function [f]. *)
+
+val di_subprogram_get_line : Llvm.llmetadata -> int
+(** [di_subprogram_get_line m] Get the line associated with subprogram [m]. *)
+
+val instr_get_debug_loc : Llvm.llvalue -> Llvm.llmetadata option
+(** [instr_get_debug_loc i] Get the debug location for instruction [i]. *)
+
+val instr_set_debug_loc : Llvm.llvalue -> Llvm.llmetadata option -> unit
+(** [instr_set_debug_loc i mopt] If [mopt] is None location metadata of [i]
+    is cleared, Otherwise location of [i] is set to the value in [mopt]. *)
+
+val get_metadata_kind : Llvm.llmetadata -> MetadataKind.t
+(** [get_metadata_kind] Obtain the enumerated type of a Metadata instance. *)
diff --git a/llvm/bindings/ocaml/llvm/CMakeLists.txt b/llvm/bindings/ocaml/llvm/CMakeLists.txt
index 99ef1cbff233bc03f352bc573ebebb9ce73772c7..5e6f74ec9c5945c99603cbb8ed6655df914adeaa 100644
--- a/llvm/bindings/ocaml/llvm/CMakeLists.txt
+++ b/llvm/bindings/ocaml/llvm/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_ocaml_library(llvm
   OCAML llvm
   C     llvm_ocaml
+  CFLAGS   "-I${CMAKE_CURRENT_SOURCE_DIR}/"
   LLVM  Core Support)
 
 configure_file(
diff --git a/llvm/bindings/ocaml/llvm/META.llvm.in b/llvm/bindings/ocaml/llvm/META.llvm.in
index adafd788ebf4ef9045d00937d45392d0d08b4beb..991bbc06009804b36fa543dd9f835816da8603bb 100644
--- a/llvm/bindings/ocaml/llvm/META.llvm.in
+++ b/llvm/bindings/ocaml/llvm/META.llvm.in
@@ -45,6 +45,14 @@ package "ipo" (
     archive(native) = "llvm_ipo.cmxa"
 )
 
+package "debuginfo" (
+    requires = "llvm"
+    version = "@PACKAGE_VERSION@"
+    description = "DebugInfo support for LLVM"
+    archive(byte) = "llvm_debuginfo.cma"
+    archive(native) = "llvm_debuginfo.cmxa"
+)
+
 package "irreader" (
     requires = "llvm"
     version  = "@PACKAGE_VERSION@"
diff --git a/llvm/bindings/ocaml/llvm/llvm.ml b/llvm/bindings/ocaml/llvm/llvm.ml
index 723ac66ffd05cac43890bb4a84b964192270d8e2..9e55ea8c436439acaec4811966586bb90840fb58 100644
--- a/llvm/bindings/ocaml/llvm/llvm.ml
+++ b/llvm/bindings/ocaml/llvm/llvm.ml
@@ -9,6 +9,7 @@
 
 type llcontext
 type llmodule
+type llmetadata
 type lltype
 type llvalue
 type lluse
@@ -42,6 +43,9 @@ module TypeKind = struct
   | Metadata
   | X86_mmx
   | Token
+  | ScalableVector
+  | BFloat
+  | X86_amx
 end
 
 module Linkage = struct
@@ -246,6 +250,7 @@ module Opcode  = struct
   | CatchSwitch
   | FNeg
   | CallBr
+  | Freeze
 end
 
 module LandingPadClauseTy = struct
@@ -288,6 +293,8 @@ module AtomicRMWBinOp = struct
   | Min
   | UMax
   | UMin
+  | FAdd
+  | FSub
 end
 
 module ValueKind = struct
@@ -326,6 +333,16 @@ module DiagnosticSeverity = struct
   | Note
 end
 
+module ModuleFlagBehavior = struct
+  type t =
+  | Error
+  | Warning
+  | Require
+  | Override
+  | Append
+  | AppendUnique
+end
+
 exception IoError of string
 
 let () = Callback.register_exception "Llvm.IoError" (IoError "")
@@ -426,6 +443,17 @@ external set_module_inline_asm : llmodule -> string -> unit
                                = "llvm_set_module_inline_asm"
 external module_context : llmodule -> llcontext = "LLVMGetModuleContext"
 
+external get_module_identifier : llmodule -> string
+                               = "llvm_get_module_identifier"
+
+external set_module_identifer : llmodule -> string -> unit
+                              = "llvm_set_module_identifier"
+
+external get_module_flag : llmodule -> string -> llmetadata option
+                         = "llvm_get_module_flag"
+external add_module_flag : llmodule -> ModuleFlagBehavior.t ->
+            string -> llmetadata -> unit = "llvm_add_module_flag"
+
 (*===-- Types -------------------------------------------------------------===*)
 external classify_type : lltype -> TypeKind.t = "llvm_classify_type"
 external type_context : lltype -> llcontext = "llvm_type_context"
@@ -571,6 +599,9 @@ external get_named_metadata : llmodule -> string -> llvalue array
                             = "llvm_get_namedmd"
 external add_named_metadata_operand : llmodule -> string -> llvalue -> unit
                                     = "llvm_append_namedmd"
+external value_as_metadata : llvalue -> llmetadata = "llvm_value_as_metadata"
+external metadata_as_value : llcontext -> llmetadata -> llvalue
+                        = "llvm_metadata_as_value"
 
 (*--... Operations on scalar constants .....................................--*)
 external const_int : lltype -> int -> llvalue = "llvm_const_int"
@@ -695,6 +726,8 @@ external dll_storage_class : llvalue -> DLLStorageClass.t = "llvm_dll_storage_cl
 external set_dll_storage_class : DLLStorageClass.t -> llvalue -> unit = "llvm_set_dll_storage_class"
 external alignment : llvalue -> int = "llvm_alignment"
 external set_alignment : int -> llvalue -> unit = "llvm_set_alignment"
+external global_copy_all_metadata : llvalue -> (llmdkind * llmetadata) array
+                                  = "llvm_global_copy_all_metadata"
 external is_global_constant : llvalue -> bool = "llvm_is_global_constant"
 external set_global_constant : bool -> llvalue -> unit
                              = "llvm_set_global_constant"
@@ -713,7 +746,7 @@ external define_qualified_global : string -> llvalue -> int -> llmodule ->
 external lookup_global : string -> llmodule -> llvalue option
                        = "llvm_lookup_global"
 external delete_global : llvalue -> unit = "llvm_delete_global"
-external global_initializer : llvalue -> llvalue = "LLVMGetInitializer"
+external global_initializer : llvalue -> llvalue option = "llvm_global_initializer"
 external set_initializer : llvalue -> llvalue -> unit = "llvm_set_initializer"
 external remove_initializer : llvalue -> unit = "llvm_remove_initializer"
 external is_thread_local : llvalue -> bool = "llvm_is_thread_local"
diff --git a/llvm/bindings/ocaml/llvm/llvm.mli b/llvm/bindings/ocaml/llvm/llvm.mli
index ba9e6c1f212023c559361e82066dd663dd06a9d4..c191382aee227beb953a4ac4df3a812ffadc708e 100644
--- a/llvm/bindings/ocaml/llvm/llvm.mli
+++ b/llvm/bindings/ocaml/llvm/llvm.mli
@@ -24,6 +24,9 @@ type llcontext
     objects. See the [llvm::Module] class. *)
 type llmodule
 
+(** Opaque representation of Metadata nodes. See the [llvm::Metadata] class. *)
+type llmetadata
+
 (** Each value in the LLVM IR has a type, an instance of [lltype]. See the
     [llvm::Type] class. *)
 type lltype
@@ -77,6 +80,9 @@ module TypeKind : sig
   | Metadata
   | X86_mmx
   | Token
+  | ScalableVector
+  | BFloat
+  | X86_amx
 end
 
 (** The linkage of a global value, accessed with {!linkage} and
@@ -268,6 +274,7 @@ module Opcode : sig
   | CatchSwitch
   | FNeg
   | CallBr
+  | Freeze
 end
 
 (** The type of a clause of a [landingpad] instruction.
@@ -319,6 +326,8 @@ module AtomicRMWBinOp : sig
   | Min
   | UMax
   | UMin
+  | FAdd
+  | FSub
 end
 
 (** The kind of an [llvalue], the result of [classify_value v].
@@ -361,6 +370,15 @@ module DiagnosticSeverity : sig
   | Note
 end
 
+module ModuleFlagBehavior :sig
+  type t =
+  | Error
+  | Warning
+  | Require
+  | Override
+  | Append
+  | AppendUnique
+end
 
 (** {6 Iteration} *)
 
@@ -525,7 +543,24 @@ val set_module_inline_asm : llmodule -> string -> unit
     See the method [llvm::Module::getContext] *)
 val module_context : llmodule -> llcontext
 
+(** [get_module_identifier m] returns the module identifier of the
+    specified module. See the method [llvm::Module::getModuleIdentifier] *)
+val get_module_identifier : llmodule -> string
+
+(** [set_module_identifier m id] sets the module identifier of [m]
+    to [id]. See the method [llvm::Module::setModuleIdentifier] *)
+val set_module_identifer : llmodule -> string -> unit
+
+(** [get_module_flag m k] Return the corresponding value if key [k] appears in
+    the module flags of [m], otherwise return None
+    See the method [llvm::Module::getModuleFlag] *)
+val get_module_flag : llmodule -> string -> llmetadata option
 
+(** [add_module_flag m b k v] Add a module-level flag b, with key [k] and
+    value [v] to the flags metadata of module [m]. It will create the 
+    module-level flags named metadata if it doesn't already exist. *)
+val add_module_flag : llmodule -> ModuleFlagBehavior.t ->
+                        string -> llmetadata -> unit
 (** {6 Types} *)
 
 (** [classify_type ty] returns the {!TypeKind.t} corresponding to the type [ty].
@@ -919,6 +954,13 @@ val get_named_metadata : llmodule -> string -> llvalue array
     [llvm::MDNode::addOperand()]. *)
 val add_named_metadata_operand : llmodule -> string -> llvalue -> unit
 
+(** Obtain a Metadata as a Value.
+    See the method [llvm::ValueAsMetadata::get()]. *)
+val value_as_metadata : llvalue -> llmetadata
+
+(** Obtain a Value as a Metadata.
+    See the method [llvm::MetadataAsValue::get()]. *)
+val metadata_as_value : llcontext -> llmetadata -> llvalue
 
 (** {7 Operations on scalar constants} *)
 
@@ -926,8 +968,9 @@ val add_named_metadata_operand : llmodule -> string -> llvalue -> unit
     See the method [llvm::ConstantInt::get]. *)
 val const_int : lltype -> int -> llvalue
 
-(** [const_of_int64 ty i] returns the integer constant of type [ty] and value
-    [i]. See the method [llvm::ConstantInt::get]. *)
+(** [const_of_int64 ty i s] returns the integer constant of type [ty] and value
+    [i]. [s] indicates whether the integer is signed or not.
+    See the method [llvm::ConstantInt::get]. *)
 val const_of_int64 : lltype -> Int64.t -> bool -> llvalue
 
 (** [int64_of_const c] returns the int64 value of the [c] constant integer.
@@ -1378,6 +1421,12 @@ val alignment : llvalue -> int
     [n] bytes. See the method [llvm::GlobalValue::setAlignment]. *)
 val set_alignment : int -> llvalue -> unit
 
+(** [global_copy_all_metadata g] returns all the metadata associated with [g],
+    which must be an [Instruction] or [GlobalObject].
+    See the [llvm::Instruction::getAllMetadata()] and
+    [llvm::GlobalObject::getAllMetadata()] methods. *)
+val global_copy_all_metadata : llvalue -> (llmdkind * llmetadata) array
+
 
 (** {7 Operations on global variables} *)
 
@@ -1463,9 +1512,9 @@ val is_global_constant : llvalue -> bool
     See the method [llvm::GlobalVariable::setConstant]. *)
 val set_global_constant : bool -> llvalue -> unit
 
-(** [global_initializer gv] returns the initializer for the global variable
-    [gv]. See the method [llvm::GlobalVariable::getInitializer]. *)
-val global_initializer : llvalue -> llvalue
+(** [global_initializer gv] If global variable [gv] has an initializer it is returned,
+    otherwise returns [None]. See the method [llvm::GlobalVariable::getInitializer]. *)
+val global_initializer : llvalue -> llvalue option
 
 (** [set_initializer c gv] sets the initializer for the global variable
     [gv] to the constant [c].
diff --git a/llvm/bindings/ocaml/llvm/llvm_ocaml.c b/llvm/bindings/ocaml/llvm/llvm_ocaml.c
index 5845783278d95e7c26a9730c076dc27e5f65efc2..04bc06e65b759809106e8835784427e841b7f9bd 100644
--- a/llvm/bindings/ocaml/llvm/llvm_ocaml.c
+++ b/llvm/bindings/ocaml/llvm/llvm_ocaml.c
@@ -21,12 +21,12 @@
 #include "llvm-c/Core.h"
 #include "llvm-c/Support.h"
 #include "llvm/Config/llvm-config.h"
-#include "caml/alloc.h"
-#include "caml/custom.h"
 #include "caml/memory.h"
 #include "caml/fail.h"
 #include "caml/callback.h"
 
+#include "llvm_ocaml.h"
+
 value llvm_string_of_message(char* Message) {
   value String = caml_copy_string(Message);
   LLVMDisposeMessage(Message);
@@ -34,6 +34,40 @@ value llvm_string_of_message(char* Message) {
   return String;
 }
 
+CAMLprim value ptr_to_option(void *Ptr) {
+  CAMLparam0();
+  CAMLlocal1(Option);
+  if (!Ptr)
+    CAMLreturn(Val_int(0));
+  Option = caml_alloc_small(1, 0);
+  Store_field(Option, 0, (value)Ptr);
+  CAMLreturn(Option);
+}
+
+CAMLprim value cstr_to_string(const char *Str, mlsize_t Len) {
+  CAMLparam0();
+  CAMLlocal1(String);
+  if (Str) {
+    String = caml_alloc_string(Len);
+    memcpy((char *)String_val(String), Str, Len);
+  } else {
+    String = caml_alloc_string(0);
+  }
+  CAMLreturn(String);
+}
+
+CAMLprim value cstr_to_string_option(const char *CStr, mlsize_t Len) {
+  CAMLparam0();
+  CAMLlocal2(Option, String);
+  if (!CStr)
+    CAMLreturn(Val_int(0));
+  String = caml_alloc_string(Len);
+  memcpy((char *)String_val(String), CStr, Len);
+  Option = caml_alloc_small(1, 0);
+  Store_field(Option, 0, (value)String);
+  CAMLreturn(Option);
+}
+
 void llvm_raise(value Prototype, char *Message) {
   CAMLparam1(Prototype);
   caml_raise_with_arg(Prototype, llvm_string_of_message(Message));
@@ -65,7 +99,7 @@ CAMLprim value llvm_enable_pretty_stacktrace(value Unit) {
 }
 
 CAMLprim value llvm_parse_command_line_options(value Overview, value Args) {
-  char *COverview;
+  const char *COverview;
   if (Overview == Val_int(0)) {
     COverview = NULL;
   } else {
@@ -236,7 +270,7 @@ CAMLprim value llvm_get_string_attr_kind(LLVMAttributeRef A) {
   unsigned Length;
   const char *String = LLVMGetStringAttributeKind(A, &Length);
   value Result = caml_alloc_string(Length);
-  memcpy(String_val(Result), String, Length);
+  memcpy((char *)String_val(Result), String, Length);
   return Result;
 }
 
@@ -245,7 +279,7 @@ CAMLprim value llvm_get_string_attr_value(LLVMAttributeRef A) {
   unsigned Length;
   const char *String = LLVMGetStringAttributeValue(A, &Length);
   value Result = caml_alloc_string(Length);
-  memcpy(String_val(Result), String, Length);
+  memcpy((char *)String_val(Result), String, Length);
   return Result;
 }
 
@@ -313,12 +347,39 @@ CAMLprim value llvm_string_of_llmodule(LLVMModuleRef M) {
   CAMLreturn(ModuleStr);
 }
 
+/* llmodule -> string */
+CAMLprim value llvm_get_module_identifier(LLVMModuleRef M) {
+  size_t Len;
+  const char *Name = LLVMGetModuleIdentifier(M, &Len);
+  return cstr_to_string(Name, (mlsize_t)Len);
+}
+
+/* llmodule -> string -> unit */
+CAMLprim value llvm_set_module_identifier(LLVMModuleRef M, value Id) {
+  LLVMSetModuleIdentifier(M, String_val(Id), caml_string_length(Id));
+  return Val_unit;
+}
+
 /* llmodule -> string -> unit */
 CAMLprim value llvm_set_module_inline_asm(LLVMModuleRef M, value Asm) {
   LLVMSetModuleInlineAsm(M, String_val(Asm));
   return Val_unit;
 }
 
+/* llmodule -> string -> llmetadata option */
+CAMLprim value llvm_get_module_flag(LLVMModuleRef M, value Key) {
+  return ptr_to_option(
+      LLVMGetModuleFlag(M, String_val(Key), caml_string_length(Key)));
+}
+
+CAMLprim value llvm_add_module_flag(LLVMModuleRef M,
+                                    LLVMModuleFlagBehavior Behaviour, value Key,
+                                    LLVMMetadataRef Val) {
+  LLVMAddModuleFlag(M, Int_val(Behaviour), String_val(Key),
+                    caml_string_length(Key), Val);
+  return Val_unit;
+}
+
 /*===-- Types -------------------------------------------------------------===*/
 
 /* lltype -> TypeKind.t */
@@ -480,17 +541,13 @@ CAMLprim value llvm_struct_set_body(LLVMTypeRef Ty,
 }
 
 /* lltype -> string option */
-CAMLprim value llvm_struct_name(LLVMTypeRef Ty)
-{
-  CAMLparam0();
-  CAMLlocal1(result);
-  const char *C = LLVMGetStructName(Ty);
-  if (C) {
-    result = caml_alloc_small(1, 0);
-    Store_field(result, 0, caml_copy_string(C));
-    CAMLreturn(result);
-  }
-  CAMLreturn(Val_int(0));
+CAMLprim value llvm_struct_name(LLVMTypeRef Ty) {
+  const char *CStr = LLVMGetStructName(Ty);
+  size_t Len;
+  if (!CStr)
+    return Val_int(0);
+  Len = strlen(CStr);
+  return cstr_to_string_option(CStr, Len);
 }
 
 /* lltype -> lltype array */
@@ -828,19 +885,9 @@ CAMLprim LLVMValueRef llvm_mdnull(LLVMContextRef C) {
 
 /* llvalue -> string option */
 CAMLprim value llvm_get_mdstring(LLVMValueRef V) {
-  CAMLparam0();
-  CAMLlocal2(Option, Str);
-  const char *S;
   unsigned Len;
-
-  if ((S = LLVMGetMDString(V, &Len))) {
-    Str = caml_alloc_string(Len);
-    memcpy(String_val(Str), S, Len);
-    Option = alloc(1,0);
-    Store_field(Option, 0, Str);
-    CAMLreturn(Option);
-  }
-  CAMLreturn(Val_int(0));
+  const char *CStr = LLVMGetMDString(V, &Len);
+  return cstr_to_string_option(CStr, Len);
 }
 
 CAMLprim value llvm_get_mdnode_operands(LLVMValueRef V) {
@@ -870,6 +917,17 @@ CAMLprim value llvm_append_namedmd(LLVMModuleRef M, value Name, LLVMValueRef Val
   return Val_unit;
 }
 
+/* llvalue -> llmetadata */
+CAMLprim LLVMMetadataRef llvm_value_as_metadata(LLVMValueRef Val) {
+  return LLVMValueAsMetadata(Val);
+}
+
+/* llcontext -> llmetadata -> llvalue */
+CAMLprim LLVMValueRef llvm_metadata_as_value(LLVMContextRef C,
+                                             LLVMMetadataRef MD) {
+  return LLVMMetadataAsValue(C, MD);
+}
+
 /*--... Operations on scalar constants .....................................--*/
 
 /* lltype -> int -> llvalue */
@@ -985,22 +1043,12 @@ CAMLprim LLVMValueRef llvm_const_vector(value ElementVals) {
 
 /* llvalue -> string option */
 CAMLprim value llvm_string_of_const(LLVMValueRef Const) {
-  const char *S;
   size_t Len;
-  CAMLparam0();
-  CAMLlocal2(Option, Str);
-
-  if(LLVMIsAConstantDataSequential(Const) && LLVMIsConstantString(Const)) {
-    S = LLVMGetAsString(Const, &Len);
-    Str = caml_alloc_string(Len);
-    memcpy(String_val(Str), S, Len);
-
-    Option = alloc(1, 0);
-    Field(Option, 0) = Str;
-    CAMLreturn(Option);
-  } else {
-    CAMLreturn(Val_int(0));
-  }
+  const char *CStr;
+  if (!LLVMIsAConstantDataSequential(Const) || !LLVMIsConstantString(Const))
+    return Val_int(0);
+  CStr = LLVMGetAsString(Const, &Len);
+  return cstr_to_string_option(CStr, Len);
 }
 
 /* llvalue -> int -> llvalue */
@@ -1160,6 +1208,25 @@ CAMLprim value llvm_set_alignment(value Bytes, LLVMValueRef Global) {
   return Val_unit;
 }
 
+/* llvalue -> (llmdkind * llmetadata) array */
+CAMLprim value llvm_global_copy_all_metadata(LLVMValueRef Global) {
+  CAMLparam0();
+  CAMLlocal2(Array, Pair);
+  size_t NumEntries;
+  LLVMValueMetadataEntry *Entries =
+      LLVMGlobalCopyAllMetadata(Global, &NumEntries);
+  Array = caml_alloc_tuple(NumEntries);
+  for (int i = 0; i < NumEntries; i++) {
+    Pair = caml_alloc_tuple(2);
+    Store_field(Pair, 0, Val_int(LLVMValueMetadataEntriesGetKind(Entries, i)));
+    Store_field(Pair, 1,
+                (value)LLVMValueMetadataEntriesGetMetadata(Entries, i));
+    Store_field(Array, i, Pair);
+  }
+  LLVMDisposeValueMetadataEntries(Entries);
+  CAMLreturn(Array);
+}
+
 /*--... Operations on uses .................................................--*/
 
 /* llvalue -> lluse option */
@@ -1268,6 +1335,11 @@ CAMLprim value llvm_delete_global(LLVMValueRef GlobalVar) {
   return Val_unit;
 }
 
+/* llvalue -> llvalue option */
+CAMLprim value llvm_global_initializer(LLVMValueRef GlobalVar) {
+  return ptr_to_option(LLVMGetInitializer(GlobalVar));
+}
+
 /* llvalue -> llvalue -> unit */
 CAMLprim value llvm_set_initializer(LLVMValueRef ConstantVal,
                                     LLVMValueRef GlobalVar) {
@@ -1541,7 +1613,7 @@ CAMLprim value llvm_instr_get_opcode(LLVMValueRef Inst) {
   if (!LLVMIsAInstruction(Inst))
       failwith("Not an instruction");
   o = LLVMGetInstructionOpcode(Inst);
-  assert (o <= LLVMCallBr);
+  assert(o <= LLVMFreeze);
   return Val_int(o);
 }
 
@@ -2511,7 +2583,7 @@ CAMLprim LLVMMemoryBufferRef llvm_memorybuffer_of_string(value Name, value Strin
 /* llmemorybuffer -> string */
 CAMLprim value llvm_memorybuffer_as_string(LLVMMemoryBufferRef MemBuf) {
   value String = caml_alloc_string(LLVMGetBufferSize(MemBuf));
-  memcpy(String_val(String), LLVMGetBufferStart(MemBuf),
+  memcpy((char *)String_val(String), LLVMGetBufferStart(MemBuf),
          LLVMGetBufferSize(MemBuf));
 
   return String;
diff --git a/llvm/bindings/ocaml/llvm/llvm_ocaml.h b/llvm/bindings/ocaml/llvm/llvm_ocaml.h
new file mode 100644
index 0000000000000000000000000000000000000000..1202cc79f2c2877ab681117d2cb9c86ca74b8ea4
--- /dev/null
+++ b/llvm/bindings/ocaml/llvm/llvm_ocaml.h
@@ -0,0 +1,30 @@
+/*===-- llvm_ocaml.h - LLVM OCaml Glue --------------------------*- C++ -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* This file glues LLVM's OCaml interface to its C interface. These functions *|
+|* are by and large transparent wrappers to the corresponding C functions.    *|
+|*                                                                            *|
+|* Note that these functions intentionally take liberties with the CAMLparamX *|
+|* macros, since most of the parameters are not GC heap objects.              *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef LLVM_LLVM_OCAML_H
+#define LLVM_LLVM_OCAML_H
+
+#include "caml/alloc.h"
+#include "caml/custom.h"
+
+/* Convert a C pointer to an OCaml option */
+CAMLprim value ptr_to_option(void *Ptr);
+
+/* Convert a C string into an OCaml string */
+CAMLprim value cstr_to_string(const char *Str, mlsize_t Len);
+
+#endif // LLVM_LLVM_OCAML_H
diff --git a/llvm/bindings/ocaml/target/target_ocaml.c b/llvm/bindings/ocaml/target/target_ocaml.c
index cf48fbe45730543337b86f8925b4da71414ccf2a..b19f6337e368c71b1e8a04b568914c23a7873917 100644
--- a/llvm/bindings/ocaml/target/target_ocaml.c
+++ b/llvm/bindings/ocaml/target/target_ocaml.c
@@ -314,8 +314,8 @@ CAMLprim value llvm_targetmachine_emit_to_file(LLVMModuleRef Module,
   char *ErrorMessage;
 
   if(LLVMTargetMachineEmitToFile(TargetMachine_val(Machine), Module,
-                                 String_val(FileName), Int_val(FileType),
-                                 &ErrorMessage)) {
+                                 (char *)String_val(FileName),
+                                 Int_val(FileType), &ErrorMessage)) {
     llvm_raise(*caml_named_value("Llvm_target.Error"), ErrorMessage);
   }
 
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index da1b78b4f530543094f8588c3bbfb9e716fc528f..2f055c779962418526d43ade565b25341d0de806 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -2105,6 +2105,12 @@ function(llvm_setup_rpath name)
   if (APPLE)
     set(_install_name_dir INSTALL_NAME_DIR "@rpath")
     set(_install_rpath "@loader_path/../lib${LLVM_LIBDIR_SUFFIX}" ${extra_libdir})
+  elseif(${CMAKE_SYSTEM_NAME} MATCHES "AIX" AND BUILD_SHARED_LIBS)
+    # $ORIGIN is not interpreted at link time by aix ld.
+    # Since BUILD_SHARED_LIBS is only recommended for use by developers,
+    # hardcode the rpath to build/install lib dir first in this mode.
+    # FIXME: update this when there is better solution.
+    set(_install_rpath "${LLVM_LIBRARY_OUTPUT_INTDIR}" "${CMAKE_INSTALL_PREFIX}/lib${LLVM_LIBDIR_SUFFIX}" ${extra_libdir})
   elseif(UNIX)
     set(_install_rpath "\$ORIGIN/../lib${LLVM_LIBDIR_SUFFIX}" ${extra_libdir})
     if(${CMAKE_SYSTEM_NAME} MATCHES "(FreeBSD|DragonFly)")
diff --git a/llvm/cmake/modules/CMakeLists.txt b/llvm/cmake/modules/CMakeLists.txt
index 505dc9a29d7002eed432ec69af3c677924906b10..8276126e9eda9885198870f9bfebc41f862cb3c1 100644
--- a/llvm/cmake/modules/CMakeLists.txt
+++ b/llvm/cmake/modules/CMakeLists.txt
@@ -127,11 +127,13 @@ configure_file(
   ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/LLVMConfig.cmake
   @ONLY)
 
-# Generate LLVMConfigVersion.cmake for build and install tree.
-configure_file(
-  LLVMConfigVersion.cmake.in
-  ${llvm_cmake_builddir}/LLVMConfigVersion.cmake
-  @ONLY)
+# Generate LLVMConfigVersion.cmake for build tree (later copied to install tree).
+include(CMakePackageConfigHelpers)
+write_basic_package_version_file(
+  "${llvm_cmake_builddir}/LLVMConfigVersion.cmake"
+  VERSION "${PACKAGE_VERSION}"
+  COMPATIBILITY SameMinorVersion
+)
 
 if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
   get_property(llvm_has_exports GLOBAL PROPERTY LLVM_HAS_EXPORTS)
diff --git a/llvm/cmake/modules/FindGRPC.cmake b/llvm/cmake/modules/FindGRPC.cmake
index 66e9d832ae944e86664a04383e97d1e779e241b9..e058f544cb290b1566f632fd9983d81ac13254b0 100644
--- a/llvm/cmake/modules/FindGRPC.cmake
+++ b/llvm/cmake/modules/FindGRPC.cmake
@@ -12,7 +12,7 @@ if (GRPC_INSTALL_PATH)
   # LLVM's BUILD_SHARED_LIBS has no effect).
   set(protobuf_MODULE_COMPATIBLE TRUE)
   find_package(Protobuf CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
-  message(STATUS "Using protobuf ${protobuf_VERSION}")
+  message(STATUS "Using protobuf ${Protobuf_VERSION}")
   find_package(gRPC CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
   message(STATUS "Using gRPC ${gRPC_VERSION}")
 
diff --git a/llvm/cmake/modules/GetErrcMessages.cmake b/llvm/cmake/modules/GetErrcMessages.cmake
index 79aa6456cc7e28b19286c13fac764c63def6e25e..b04f269d79d24ecb30a9c857d61d262071f35d9c 100644
--- a/llvm/cmake/modules/GetErrcMessages.cmake
+++ b/llvm/cmake/modules/GetErrcMessages.cmake
@@ -1,10 +1,14 @@
-
 # This function returns the messages of various POSIX error codes as they are returned by std::error_code.
-# The purpose of this function is to supply those error messages to llvm-lit using the errc_messages config
-# Currently supplied and needed error codes: ENOENT, EISDIR, EINVAL and EACCES
-# Messages are semi colon separated
-# Keep amount, order and tested error codes in sync with llvm/utils/lit/lit/llvm/config.py
+# The purpose of this function is to supply those error messages to llvm-lit using the errc_messages config.
+# Currently supplied and needed error codes: ENOENT, EISDIR, EINVAL and EACCES.
+# Messages are semi colon separated.
+# Keep amount, order and tested error codes in sync with llvm/utils/lit/lit/llvm/config.py.
 function(get_errc_messages outvar)
+    if(CMAKE_CROSSCOMPILING AND NOT CMAKE_CROSSCOMPILING_EMULATOR AND NOT DEFINED errc_exit_code)
+        set(${outvar} "" PARENT_SCOPE)
+        message(STATUS "Can't get errc messages in cross-compilation mode")
+        return()
+    endif()
 
     set(errc_test_code ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/getErrc.cpp)
 
@@ -30,7 +34,7 @@ function(get_errc_messages outvar)
             ${errc_test_code}
             RUN_OUTPUT_VARIABLE errc_result
             COMPILE_OUTPUT_VARIABLE errc_compile_errors)
-    if (errc_compiled)
+    if (errc_compiled AND "${errc_exit_code}" STREQUAL "0")
         set(${outvar} ${errc_result} PARENT_SCOPE)
     else()
         set(${outvar} "" PARENT_SCOPE)
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 2e088bd6e91687571725d7483e7968783288df26..8b64694757e9f46c45f877e51b5ce3fddd58a650 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -212,6 +212,11 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
     append("-bcdtors:mbr"
            CMAKE_EXE_LINKER_FLAGS CMAKE_MODULE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS)
   endif()
+  if(BUILD_SHARED_LIBS)
+    # See rpath handling in AddLLVM.cmake
+    # FIXME: Remove this warning if this rpath is no longer hardcoded.
+    message(WARNING "Build and install environment path info may be exposed; binaries will also be unrelocatable.")
+  endif()
 endif()
 
 # Pass -Wl,-z,defs. This makes sure all symbols are defined. Otherwise a DSO
@@ -668,6 +673,11 @@ if (LLVM_ENABLE_WARNINGS AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
   if (LLVM_ENABLE_PEDANTIC AND LLVM_COMPILER_IS_GCC_COMPATIBLE)
     append("-pedantic" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
     append("-Wno-long-long" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+
+    # GCC warns about redundant toplevel semicolons (enabled by -pedantic
+    # above), while Clang doesn't. Enable the corresponding Clang option to
+    # pick up on these even in builds with Clang.
+    add_flag_if_supported("-Wc++98-compat-extra-semi" CXX98_COMPAT_EXTRA_SEMI_FLAG)
   endif()
 
   add_flag_if_supported("-Wimplicit-fallthrough" IMPLICIT_FALLTHROUGH_FLAG)
@@ -680,14 +690,17 @@ if (LLVM_ENABLE_WARNINGS AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
   check_cxx_compiler_flag("-Wclass-memaccess" CXX_SUPPORTS_CLASS_MEMACCESS_FLAG)
   append_if(CXX_SUPPORTS_CLASS_MEMACCESS_FLAG "-Wno-class-memaccess" CMAKE_CXX_FLAGS)
 
-  # Disable -Wredundant-move on GCC>=9. GCC wants to remove std::move in code
-  # like "A foo(ConvertibleToA a) { return std::move(a); }", but this code does
-  # not compile (or uses the copy constructor instead) on clang<=3.8. Clang also
-  # has a -Wredundant-move, but it only fires when the types match exactly, so
-  # we can keep it here.
+  # Disable -Wredundant-move and -Wpessimizing-move on GCC>=9. GCC wants to
+  # remove std::move in code like "A foo(ConvertibleToA a) {
+  # return std::move(a); }", but this code does not compile (or uses the copy
+  # constructor instead) on clang<=3.8. Clang also has a -Wredundant-move and
+  # -Wpessimizing-move, but they only fire when the types match exactly, so we
+  # can keep them here.
   if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     check_cxx_compiler_flag("-Wredundant-move" CXX_SUPPORTS_REDUNDANT_MOVE_FLAG)
     append_if(CXX_SUPPORTS_REDUNDANT_MOVE_FLAG "-Wno-redundant-move" CMAKE_CXX_FLAGS)
+    check_cxx_compiler_flag("-Wpessimizing-move" CXX_SUPPORTS_PESSIMIZING_MOVE_FLAG)
+    append_if(CXX_SUPPORTS_PESSIMIZING_MOVE_FLAG "-Wno-pessimizing-move" CMAKE_CXX_FLAGS)
   endif()
 
   # The LLVM libraries have no stable C++ API, so -Wnoexcept-type is not useful.
@@ -982,8 +995,8 @@ endif()
 # linker directly, it isn't sufficient to pass -fsanitize=* to the linker.
 if (CLANG_CL AND (LLVM_BUILD_INSTRUMENTED OR LLVM_USE_SANITIZER))
   execute_process(
-    COMMAND ${CMAKE_CXX_COMPILER} /clang:-print-resource-dir
-    OUTPUT_VARIABLE clang_resource_dir
+    COMMAND ${CMAKE_CXX_COMPILER} /clang:-print-libgcc-file-name /clang:--rtlib=compiler-rt
+    OUTPUT_VARIABLE clang_compiler_rt_file
     ERROR_VARIABLE clang_cl_stderr
     OUTPUT_STRIP_TRAILING_WHITESPACE
     ERROR_STRIP_TRAILING_WHITESPACE
@@ -992,8 +1005,9 @@ if (CLANG_CL AND (LLVM_BUILD_INSTRUMENTED OR LLVM_USE_SANITIZER))
     message(FATAL_ERROR
       "Unable to invoke clang-cl to find resource dir: ${clang_cl_stderr}")
   endif()
-  file(TO_CMAKE_PATH "${clang_resource_dir}" clang_resource_dir)
-  append("/libpath:${clang_resource_dir}/lib/windows"
+  file(TO_CMAKE_PATH "${clang_compiler_rt_file}" clang_compiler_rt_file)
+  get_filename_component(clang_runtime_dir "${clang_compiler_rt_file}" DIRECTORY)
+  append("/libpath:${clang_runtime_dir}"
     CMAKE_EXE_LINKER_FLAGS
     CMAKE_MODULE_LINKER_FLAGS
     CMAKE_SHARED_LINKER_FLAGS)
diff --git a/llvm/cmake/modules/LLVMConfigVersion.cmake.in b/llvm/cmake/modules/LLVMConfigVersion.cmake.in
deleted file mode 100644
index e9ac4ed2da7865ea2c80a3e4fb628f599e3e2999..0000000000000000000000000000000000000000
--- a/llvm/cmake/modules/LLVMConfigVersion.cmake.in
+++ /dev/null
@@ -1,13 +0,0 @@
-set(PACKAGE_VERSION "@PACKAGE_VERSION@")
-
-# LLVM is API-compatible only with matching major.minor versions
-# and patch versions not less than that requested.
-if("@LLVM_VERSION_MAJOR@.@LLVM_VERSION_MINOR@" VERSION_EQUAL
-    "${PACKAGE_FIND_VERSION_MAJOR}.${PACKAGE_FIND_VERSION_MINOR}"
-   AND NOT "@LLVM_VERSION_PATCH@" VERSION_LESS "${PACKAGE_FIND_VERSION_PATCH}")
-  set(PACKAGE_VERSION_COMPATIBLE 1)
-  if("@LLVM_VERSION_PATCH@" VERSION_EQUAL
-      "${PACKAGE_FIND_VERSION_PATCH}")
-    set(PACKAGE_VERSION_EXACT 1)
-  endif()
-endif()
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 9205c79e7504f6710eeba75f9703f69260c72e3c..51fd90e058ab1e14655fb7d9f0ff20e60591ac96 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1155,6 +1155,8 @@ The AMDGPU backend uses the following ELF header:
      *reserved*                           0x03d      Reserved.
      *reserved*                           0x03e      Reserved.
      ``EF_AMDGPU_MACH_AMDGCN_GFX90A``     0x03f      ``gfx90a``
+     *reserved*                           0x040      Reserved.
+     *reserved*                           0x041      Reserved.
      ==================================== ========== =============================
 
 Sections
@@ -4104,9 +4106,9 @@ The fields used by CP for code objects before V3 also match those specified in
                                                      work-group. Granularity is
                                                      device specific:
 
-                                                     GFX6:
+                                                     GFX6
                                                        roundup(lds-size / (64 * 4))
-                                                     GFX7-GFX10:
+                                                     GFX7-GFX10
                                                        roundup(lds-size / (128 * 4))
 
      24      1 bit   ENABLE_EXCEPTION_IEEE_754_FP    Wavefront starts execution
@@ -4278,12 +4280,11 @@ SGPR register initial state is defined in
                 (enable_sgpr_dispatch_id)         dispatch packet being
                                                   executed.
      then       Flat Scratch Init          2      See
-                                                  :ref:`amdgpu-amdhsa-kernel-prolog-flat-scratch`.
+                (enable_sgpr_flat_scratch         :ref:`amdgpu-amdhsa-kernel-prolog-flat-scratch`.
+                _init)
      then       Private Segment Size       1      The 32-bit byte size of a
-                                                  (enable_sgpr_private single
-                                                  work-item's
-                                                  scratch_segment_size) memory
-                                                  allocation. This is the
+                (enable_sgpr_private              single work-item's memory
+                _segment_size)                    allocation. This is the
                                                   value from the kernel
                                                   dispatch packet Private
                                                   Segment Byte Size rounded up
@@ -4301,36 +4302,6 @@ SGPR register initial state is defined in
                                                   may be needed for GFX9-GFX10 which
                                                   changes the meaning of the
                                                   Flat Scratch Init value.
-     then       Grid Work-Group Count X    1      32-bit count of the number of
-                (enable_sgpr_grid                 work-groups in the X dimension
-                _workgroup_count_X)               for the grid being
-                                                  executed. Computed from the
-                                                  fields in the kernel dispatch
-                                                  packet as ((grid_size.x +
-                                                  workgroup_size.x - 1) /
-                                                  workgroup_size.x).
-     then       Grid Work-Group Count Y    1      32-bit count of the number of
-                (enable_sgpr_grid                 work-groups in the Y dimension
-                _workgroup_count_Y &&             for the grid being
-                less than 16 previous             executed. Computed from the
-                SGPRs)                            fields in the kernel dispatch
-                                                  packet as ((grid_size.y +
-                                                  workgroup_size.y - 1) /
-                                                  workgroupSize.y).
-
-                                                  Only initialized if <16
-                                                  previous SGPRs initialized.
-     then       Grid Work-Group Count Z    1      32-bit count of the number of
-                (enable_sgpr_grid                 work-groups in the Z dimension
-                _workgroup_count_Z &&             for the grid being
-                less than 16 previous             executed. Computed from the
-                SGPRs)                            fields in the kernel dispatch
-                                                  packet as ((grid_size.z +
-                                                  workgroup_size.z - 1) /
-                                                  workgroupSize.z).
-
-                                                  Only initialized if <16
-                                                  previous SGPRs initialized.
      then       Work-Group Id X            1      32-bit work-group id in X
                 (enable_sgpr_workgroup_id         dimension of grid for
                 _X)                               wavefront.
diff --git a/llvm/docs/BitCodeFormat.rst b/llvm/docs/BitCodeFormat.rst
index df0e195c6809b92a3719561f87b86cb6166bab75..eff9d2866a8f812679f20924be74640cffb75b44 100644
--- a/llvm/docs/BitCodeFormat.rst
+++ b/llvm/docs/BitCodeFormat.rst
@@ -1070,6 +1070,7 @@ The integer codes are mapped to well-known attributes as follows.
 * code 68: ``noundef``
 * code 69: ``byref``
 * code 70: ``mustprogress``
+* code 74: ``vscale_range(<Min>[, <Max>])``
 
 .. note::
   The ``allocsize`` attribute has a special encoding for its arguments. Its two
@@ -1077,6 +1078,12 @@ The integer codes are mapped to well-known attributes as follows.
   (i.e. ``(EltSizeParam << 32) | NumEltsParam``), with ``NumEltsParam`` taking on
   the sentinel value -1 if it is not specified.
 
+.. note::
+  The ``vscale_range`` attribute has a special encoding for its arguments. Its two
+  arguments, which are 32-bit integers, are packed into one 64-bit integer value
+  (i.e. ``(Min << 32) | Max``), with ``Max`` taking on the value of ``Min`` if
+  it is not specified.
+
 .. _TYPE_BLOCK:
 
 TYPE_BLOCK Contents
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 7e61a276765b18881bbc75eec7b1867c6b0436a9..413b64e950077deda4f5c4d1b4e66ce01d5ff297 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -20,7 +20,7 @@ user interface as possible.
 command line.  Tests can be either individual test files or directories to
 search for tests (see :ref:`test-discovery`).
 
-Each specified test will be executed (potentially in parallel) and once all
+Each specified test will be executed (potentially concurrently) and once all
 tests have been run :program:`lit` will print summary information on the number
 of tests which passed or failed (see :ref:`test-status-results`).  The
 :program:`lit` program will execute with a non-zero exit code if any tests
@@ -151,8 +151,7 @@ EXECUTION OPTIONS
 
  Track the wall time individual tests take to execute and includes the results
  in the summary output.  This is useful for determining which tests in a test
- suite take the most time to execute.  Note that this option is most useful
- with ``-j 1``.
+ suite take the most time to execute.
 
 .. option:: --ignore-fail
 
@@ -168,6 +167,17 @@ EXECUTION OPTIONS
 SELECTION OPTIONS
 -----------------
 
+By default, `lit` will run failing tests first, then run tests in descending
+execution time order to optimize concurrency.
+
+The timing data is stored in the `test_exec_root` in a file named
+`.lit_test_times.txt`. If this file does not exist, then `lit` checks the
+`test_source_root` for the file to optionally accelerate clean builds.
+
+.. option:: --shuffle
+
+ Run the tests in a random order, not failing/slowest first.
+
 .. option:: --max-failures N
 
  Stop execution after the given number ``N`` of failures.
@@ -201,10 +211,6 @@ SELECTION OPTIONS
  must be in the range ``1..M``. The environment variable
  ``LIT_RUN_SHARD`` can also be used in place of this option.
 
-.. option:: --shuffle
-
- Run the tests in a random order.
-
 .. option:: --timeout=N
 
  Spend at most ``N`` seconds (approximately) running each individual test.
@@ -416,13 +422,6 @@ executed, two important global variables are predefined:
  **root** The root configuration.  This is the top-most :program:`lit` configuration in
  the project.
 
- **is_early** Whether the test suite as a whole should be given a head start
- before other test suites run.
-
- **early_tests** An explicit set of '/' separated test paths that should be
- given a head start before other tests run. For example, the top five or so
- slowest tests. See also: `--time-tests`
-
  **pipefail** Normally a test using a shell pipe fails if any of the commands
  on the pipe fail. If this is not desired, setting this variable to false
  makes the test fail only if the last command in the pipe fails.
diff --git a/llvm/docs/CommandGuide/llvm-mca.rst b/llvm/docs/CommandGuide/llvm-mca.rst
index 1229fd934c80edd3535121e2dc2339f1aaf04547..9e40e5d9e4f55446dffe355fb0126ddc06b1c2a0 100644
--- a/llvm/docs/CommandGuide/llvm-mca.rst
+++ b/llvm/docs/CommandGuide/llvm-mca.rst
@@ -975,7 +975,6 @@ met. Multiple instructions can be issued in one cycle according to the value of
 the ``IssueWidth`` parameter in LLVM's scheduling model.
 
 Once issued, an instruction is moved to ``IssuedInst`` set until it is ready to
-retire. If ``RetireControlUnit`` is defined in the LLVM's scheduling model,
-:program:`llvm-mca` ensures that instructions are retired in-order. However, an
-instruction is allowed to retire out-of-order if ``RetireOOO`` property is true
-for at least one of its writes.
+retire. :program:`llvm-mca` ensures that writes are committed in-order. However,
+an instruction is allowed to commit writes and retire out-of-order if
+``RetireOOO`` property is true for at least one of its writes.
diff --git a/llvm/docs/CommandGuide/llvm-nm.rst b/llvm/docs/CommandGuide/llvm-nm.rst
index 747192a9d924dc4e45e928267db8ec79ae518688..20779b17fddec5a634b3cef4155ca806874be5ce 100644
--- a/llvm/docs/CommandGuide/llvm-nm.rst
+++ b/llvm/docs/CommandGuide/llvm-nm.rst
@@ -149,7 +149,8 @@ OPTIONS
 
 .. option:: --format=<format>, -f
 
- Select an output format; *format* may be *sysv*, *posix*, *darwin*, or *bsd*.
+ Select an output format; *format* may be *sysv*, *posix*, *darwin*, *bsd* or
+ *just-symbols*.
  The default is *bsd*.
 
 .. option:: --help, -h
@@ -162,7 +163,7 @@ OPTIONS
 
 .. option:: --just-symbol-name, -j
 
- Print just the symbol names.
+ Print just the symbol names. Alias for `--format=just-symbols``.
 
 .. option:: -m
 
diff --git a/llvm/docs/CommandGuide/llvm-objdump.rst b/llvm/docs/CommandGuide/llvm-objdump.rst
index e92654b654eac9ce7a49bee612920205f030705a..49edbcd91b072696acdbfe8807867463f2309d76 100644
--- a/llvm/docs/CommandGuide/llvm-objdump.rst
+++ b/llvm/docs/CommandGuide/llvm-objdump.rst
@@ -172,6 +172,12 @@ OPTIONS
   When disassembling with the :option:`--source` option, prepend ``prefix`` to
   absolute paths.
 
+.. option:: --prefix-strip=<level>
+
+  When disassembling with the :option:`--source` option, strip out ``level``
+  initial directories from absolute paths. This option has no effect without
+  :option:`--prefix`.
+
 .. option:: --print-imm-hex
 
   Use hex format when printing immediate values in disassembly output.
diff --git a/llvm/docs/CommandGuide/tblgen.rst b/llvm/docs/CommandGuide/tblgen.rst
index dfe93e04ccb575046ffec01a54866c9c517516dc..d279029f5e45aa50fc541d1baeea5248023da328 100644
--- a/llvm/docs/CommandGuide/tblgen.rst
+++ b/llvm/docs/CommandGuide/tblgen.rst
@@ -545,10 +545,6 @@ clang-tblgen Options
 
   Generate ``riscv_vector.h`` for Clang.
 
-.. option:: -gen-riscv-vector-generic-header
-
-  Generate ``riscv_vector_generic.h`` for Clang.
-
 .. option:: -gen-riscv-vector-builtins
 
   Generate ``riscv_vector_builtins.inc`` for Clang.
diff --git a/llvm/docs/DeveloperPolicy.rst b/llvm/docs/DeveloperPolicy.rst
index 6b6fe29a963e460412c84ffbfc89ec0576f4b09a..361aedb8e6c5961e9db6bc814d69f1d06832453f 100644
--- a/llvm/docs/DeveloperPolicy.rst
+++ b/llvm/docs/DeveloperPolicy.rst
@@ -291,6 +291,9 @@ Below are some guidelines about the format of the message itself:
   related commit. This could be as simple as "Revert commit NNNN because it
   caused PR#".
 
+* If the patch has been reviewed, add a link to its review page, as shown
+  `here <https://www.llvm.org/docs/Phabricator.html#committing-a-change>`_.
+
 For minor violations of these recommendations, the community normally favors
 reminding the contributor of this policy over reverting. Minor corrections and
 omissions can be handled by sending a reply to the commits mailing list.
diff --git a/llvm/docs/GettingInvolved.rst b/llvm/docs/GettingInvolved.rst
index 3b6e14e5840ef0da9554af3b71306def182cd2cb..157ac6aa12a486cfeca8c29baf96b4d2221a64a5 100644
--- a/llvm/docs/GettingInvolved.rst
+++ b/llvm/docs/GettingInvolved.rst
@@ -135,6 +135,61 @@ lists.
 
   .. __: http://lists.llvm.org/mailman/listinfo/llvm-announce
 
+Online Sync-Ups
+---------------
+
+A number of regular calls are organized on specific topics. It should be
+expected that the range of topics will change over time. At the time of
+writing, the following sync-ups are organized:
+
+.. list-table:: LLVM regular sync-up calls
+   :widths: 25 25 25 25
+   :header-rows: 1
+
+   * - Topic
+     - Frequency
+     - Calendar link
+     - Minutes/docs link
+   * - RISC-V
+     - Every 2 weeks on Thursday
+     - `ics <https://calendar.google.com/calendar/ical/lowrisc.org_0n5pkesfjcnp0bh5hps1p0bd80%40group.calendar.google.com/public/basic.ics>`__
+       `gcal <https://calendar.google.com/calendar/b/1?cid=bG93cmlzYy5vcmdfMG41cGtlc2ZqY25wMGJoNWhwczFwMGJkODBAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ>`__
+     - 
+   * - Scalable Vectors and Arm SVE
+     - Monthly, every 3rd Tuesday
+     - `ics <https://calendar.google.com/calendar/ical/bjms39pe6k6bo5egtsp7don414%40group.calendar.google.com/public/basic.ics>`__
+       `gcal <https://calendar.google.com/calendar/u/0/embed?src=bjms39pe6k6bo5egtsp7don414@group.calendar.google.com>`__
+     - `Minutes/docs <https://docs.google.com/document/d/1UPH2Hzou5RgGT8XfO39OmVXKEibWPfdYLELSaHr3xzo/edit>`__
+   * - ML Guided Compiler Optimizations
+     - Monthly
+     -
+     - `Minutes/docs <https://docs.google.com/document/d/1JecbplF09l3swTjze-UVeLh4L48svJxGVy4mz_e9Rhs/edit?usp=gmail#heading=h.ts9cmcjbir1j>`__
+   * - `LLVM security group <https://llvm.org/docs/Security.html>`__
+     - Monthly, every 3rd Tuesday
+     - `ics <https://calendar.google.com/calendar/ical/eoh3m9k1l6vqbd1fkp94fv5q74%40group.calendar.google.com/public/basic.ics>`__
+       `gcal <https://calendar.google.com/calendar/embed?src=eoh3m9k1l6vqbd1fkp94fv5q74%40group.calendar.google.com>`__
+     - `Minutes/docs <https://docs.google.com/document/d/1GLCE8cl7goCaLSiM9j1eIq5IqeXt6_YTY2UEcC4jmsg/edit?usp=sharing>`__
+   * - `CIRCT <https://github.com/llvm/circt>`__
+     - Weekly, on Wednesday
+     - 
+     - `Minutes/docs <https://docs.google.com/document/d/1fOSRdyZR2w75D87yU2Ma9h2-_lEPL4NxvhJGJd-s5pk/edit#heading=h.mulvhjtr8dk9>`__
+   * - `MLIR <https://mlir.llvm.org>`__ design meetings
+     - Weekly, on Thursdays
+     - 
+     - `Minutes/docs <https://docs.google.com/document/d/1y_9f1AbfgcoVdJh4_aM6-BaSHvrHl8zuA5G4jv_94K8/edit#heading=h.cite1kolful9>`__
+   * - flang
+     - Multiple meeting series, `documented here <https://github.com/llvm/llvm-project/blob/main/flang/docs/GettingInvolved.md#calls>`__
+     - 
+     - 
+   * - OpenMP
+     - Multiple meeting series, `documented here <https://openmp.llvm.org/docs/SupportAndFAQ.html>`__
+     - 
+     - 
+   * - LLVM Alias Analysis
+     - Every 4 weeks on Tuesdays
+     - `ics <http://lists.llvm.org/pipermail/llvm-dev/attachments/20201103/a3499a67/attachment-0001.ics>`__
+     - `Minutes/docs <https://docs.google.com/document/d/17U-WvX8qyKc3S36YUKr3xfF-GHunWyYowXbxEdpHscw>`__
+
 IRC
 ---
 
diff --git a/llvm/docs/GettingStartedVS.rst b/llvm/docs/GettingStartedVS.rst
index 2ed4397ac39f656c980a07b7e00f47024455eca8..2178c41824eae8b356844a88fa3a63c367df78be 100644
--- a/llvm/docs/GettingStartedVS.rst
+++ b/llvm/docs/GettingStartedVS.rst
@@ -57,6 +57,20 @@ need `GnuWin32 <http://gnuwin32.sourceforge.net/>`_ tools, too.
 Do not install the LLVM directory tree into a path containing spaces (e.g.
 ``C:\Documents and Settings\...``) as the configure step will fail.
 
+To simplify the installation procedure, you can also use 
+`Chocolatey <https://chocolatey.org/>`_ as package manager. After the
+`installation <https://chocolatey.org/install>`_ of Chocolatey, run these
+commands in an admin shell to install the required tools:
+
+.. code-block:: bat
+
+   choco install -y ninja git cmake gnuwin python3
+   pip3 install psutil
+
+There is also a Windows 
+`Dockerfile <https://github.com/llvm/llvm-zorg/blob/main/buildbot/google/docker/windows-base-vscode2019/Dockerfile>`_ 
+with the entire build tool chain. This can be used to test the build with a
+tool chain different from your host installation or to create build servers. 
 
 Getting Started
 ===============
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 15a28e5afd17cb4ff232d4691cfa4ab5f18eb487..331efd242caffa83a424df7eaaab286d8a8674a9 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -233,6 +233,39 @@ Reverse the order of the bits in a scalar.
 
   %1:_(s32) = G_BITREVERSE %0:_(s32)
 
+G_SBFX, G_UBFX
+^^^^^^^^^^^^^^
+
+Extract a range of bits from a register.
+
+The source operands are registers as follows:
+
+- Source
+- The least-significant bit for the extraction
+- The width of the extraction
+
+G_SBFX sign-extends the result, while G_UBFX zero-extends the result.
+
+.. code-block:: none
+
+  ; Extract 5 bits starting at bit 1 from %x and store them in %a.
+  ; Sign-extend the result.
+  ;
+  ; Example:
+  ; %x = 0...0000[10110]1 ---> %a = 1...111111[10110]
+  %lsb_one = G_CONSTANT i32 1
+  %width_five = G_CONSTANT i32 5
+  %a:_(s32) = G_SBFX %x, %lsb_one, %width_five
+
+  ; Extract 3 bits starting at bit 2 from %x and store them in %b. Zero-extend
+  ; the result.
+  ;
+  ; Example:
+  ; %x = 1...11111[100]11 ---> %b = 0...00000[100]
+  %lsb_two = G_CONSTANT i32 2
+  %width_three = G_CONSTANT i32 3
+  %b:_(s32) = G_UBFX %x, %lsb_two, %width_three
+
 Integer Operations
 -------------------
 
@@ -268,6 +301,11 @@ G_SHL, G_LSHR, G_ASHR
 
 Shift the bits of a scalar left or right inserting zeros (sign-bit for G_ASHR).
 
+G_ROTR, G_ROTL
+^^^^^^^^^^^^^^
+
+Rotate the bits right (G_ROTR) or left (G_ROTL).
+
 G_ICMP
 ^^^^^^
 
diff --git a/llvm/docs/HowToUpdateDebugInfo.rst b/llvm/docs/HowToUpdateDebugInfo.rst
index b9bdc485b4876334043aeecff89095f9f96b241c..58bdc111a90c843b8ff7d53a8bd4978b64225910 100644
--- a/llvm/docs/HowToUpdateDebugInfo.rst
+++ b/llvm/docs/HowToUpdateDebugInfo.rst
@@ -376,6 +376,17 @@ as follows:
 
   $ llvm-original-di-preservation.py sample.json sample.html
 
+Testing of original debug info preservation can be invoked from front-end level
+as follows:
+
+.. code-block:: bash
+
+  # Test each pass.
+  $ clang -Xclang -fverify-debuginfo-preserve -g -O2 sample.c
+
+  # Test each pass and export the issues report into the JSON file.
+  $ clang -Xclang -fverify-debuginfo-preserve -Xclang -fverify-debuginfo-preserve-export=sample.json -g -O2 sample.c
+
 Mutation testing for MIR-level transformations
 ----------------------------------------------
 
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index d53795ef5607b5866efb5a5880aa722c36ec4df8..794ca1092eafa303333cc631ac75e8ee954b9c00 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1119,7 +1119,7 @@ Currently, only the following parameter attributes are defined:
 
 .. _attr_inalloca:
 
-``inalloca``
+``inalloca(<ty>)``
 
     The ``inalloca`` argument attribute allows the caller to take the
     address of outgoing stack arguments. An ``inalloca`` argument must
@@ -1143,6 +1143,9 @@ Currently, only the following parameter attributes are defined:
     must be cleared off with :ref:`llvm.stackrestore
     <int_stackrestore>`.
 
+    The inalloca attribute requires a type argument, which must be the
+    same as the pointee type of the argument.
+
     See :doc:`InAlloca` for more information on how to use this
     attribute.
 
@@ -1992,6 +1995,11 @@ example:
     function does not satisfy this contract, the behavior is undefined.  This
     attribute does not apply transitively to callees, but does apply to call
     sites within the function. Note that `willreturn` implies `mustprogress`.
+``vscale_range(<min>[, <max>])``
+    This attribute indicates the minimum and maximum vscale value for the given
+    function. A value of 0 means unbounded. If the optional max value is omitted
+    then max is set to the value of min. If the attribute is not present, no
+    assumptions are made about the range of vscale.
 
 Call Site Attributes
 ----------------------
@@ -3200,6 +3208,63 @@ and a scalable property to represent vectors where the exact hardware
 vector length is unknown at compile time. Vector types are considered
 :ref:`first class <t_firstclass>`.
 
+:Memory Layout:
+
+In general vector elements are laid out in memory in the same way as
+:ref:`array types <t_array>`. Such an anology works fine as long as the vector
+elements are byte sized. However, when the elements of the vector aren't byte
+sized it gets a bit more complicated. One way to describe the layout is by
+describing what happens when a vector such as <N x iM> is bitcasted to an
+integer type with N*M bits, and then following the rules for storing such an
+integer to memory.
+
+A bitcast from a vector type to a scalar integer type will see the elements
+being packed together (without padding). The order in which elements are
+inserted in the integer depends on endianess. For little endian element zero
+is put in the least significant bits of the integer, and for big endian
+element zero is put in the most significant bits.
+
+Using a vector such as ``<i4 1, i4 2, i4 3, i4 5>`` as an example, together
+with the analogy that we can replace a vector store by a bitcast followed by
+an integer store, we get this for big endian:
+
+.. code-block:: llvm
+
+      %val = bitcast <4 x i4> <i4 1, i4 2, i4 3, i4 5> to i16
+
+      ; Bitcasting from a vector to an integral type can be seen as
+      ; concatenating the values:
+      ;   %val now has the hexadecimal value 0x1235.
+
+      store i16 %val, i16* %ptr
+
+      ; In memory the content will be (8-bit addressing):
+      ;
+      ;    [%ptr + 0]: 00010010  (0x12)
+      ;    [%ptr + 1]: 00110101  (0x35)
+
+The same example for little endian:
+
+.. code-block:: llvm
+
+      %val = bitcast <4 x i4> <i4 1, i4 2, i4 3, i4 5> to i16
+
+      ; Bitcasting from a vector to an integral type can be seen as
+      ; concatenating the values:
+      ;   %val now has the hexadecimal value 0x5321.
+
+      store i16 %val, i16* %ptr
+
+      ; In memory the content will be (8-bit addressing):
+      ;
+      ;    [%ptr + 0]: 01010011  (0x53)
+      ;    [%ptr + 1]: 00100001  (0x21)
+
+When ``<N*M>`` isn't evenly divisible by the byte size the exact memory layout
+is unspecified (just like it is for an integral type of the same size). This
+is because different targets could put the padding at different positions when
+the type size is smaller than the type's store size.
+
 :Syntax:
 
 ::
@@ -3392,9 +3457,10 @@ Opaque Structure Types
 
 :Overview:
 
-Opaque structure types are used to represent named structure types that
+Opaque structure types are used to represent structure types that
 do not have a body specified. This corresponds (for example) to the C
-notion of a forward declared structure.
+notion of a forward declared structure. They can be named (``%X``) or
+unnamed (``%52``).
 
 :Syntax:
 
@@ -9705,8 +9771,7 @@ this ``cmpxchg`` with other :ref:`volatile operations <volatile>`.
 
 The success and failure :ref:`ordering <ordering>` arguments specify how this
 ``cmpxchg`` synchronizes with other atomic operations. Both ordering parameters
-must be at least ``monotonic``, the ordering constraint on failure must be no
-stronger than that on success, and the failure ordering cannot be either
+must be at least ``monotonic``, the failure ordering cannot be either
 ``release`` or ``acq_rel``.
 
 A ``cmpxchg`` instruction can also take an optional
@@ -10603,14 +10668,19 @@ pointers) types with the same address space through this instruction.
 To convert pointers to other types, use the :ref:`inttoptr <i_inttoptr>`
 or :ref:`ptrtoint <i_ptrtoint>` instructions first.
 
+There is a caveat for bitcasts involving vector types in relation to
+endianess. For example ``bitcast <2 x i8> <value> to i16`` puts element zero
+of the vector in the least significant bits of the i16 for little-endian while
+element zero ends up in the most significant bits for big-endian.
+
 Example:
 """"""""
 
 .. code-block:: text
 
-      %X = bitcast i8 255 to i8              ; yields i8 :-1
-      %Y = bitcast i32* %x to sint*          ; yields sint*:%x
-      %Z = bitcast <2 x int> %V to i64;        ; yields i64: %V
+      %X = bitcast i8 255 to i8          ; yields i8 :-1
+      %Y = bitcast i32* %x to sint*      ; yields sint*:%x
+      %Z = bitcast <2 x int> %V to i64;  ; yields i64: %V (depends on endianess)
       %Z = bitcast <2 x i32*> %V to <2 x i64*> ; yields <2 x i64*>
 
 .. _i_addrspacecast:
@@ -11507,6 +11577,15 @@ overloaded, and only one type suffix is required. Because the argument's
 type is matched against the return type, it does not require its own
 name suffix.
 
+:ref:`Unnamed types <t_opaque>` are encoded as ``s_s``. Overloaded intrinsics
+that depend on an unnamed type in one of its overloaded argument types get an
+additional ``.<number>`` suffix. This allows differentiating intrinsics with
+different unnamed types as arguments. (For example:
+``llvm.ssa.copy.p0s_s.2(%42*)``) The number is tracked in the LLVM module and
+it ensures unique names in the module. While linking together two modules, it is
+still possible to get a name clash. In that case one of the names will be
+changed by getting a new number.
+
 For target developers who are defining intrinsics for back-end code
 generation, any intrinsic overloads based solely the distinction between
 integer or floating point types should not be relied upon for correct
@@ -16596,6 +16675,36 @@ The first two operands are vectors with the same type. The third argument
 the source/result vector. The ``imm`` is a signed integer constant in the range
 ``-VL <= imm < VL``. For values outside of this range the result is poison.
 
+
+'``llvm.experimental.stepvector``' Intrinsic
+
+This is an overloaded intrinsic. You can use ``llvm.experimental.stepvector``
+to generate a vector whose lane values comprise the linear sequence
+<0, 1, 2, ...>. It is primarily intended for scalable vectors.
+
+::
+
+      declare <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+      declare <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+
+The '``llvm.experimental.stepvector``' intrinsics are used to create vectors
+of integers whose elements contain a linear sequence of values starting from 0
+with a step of 1.  This experimental intrinsic can only be used for vectors
+with integer elements that are at least 8 bits in size. If the sequence value
+exceeds the allowed limit for the element type then the result for that lane is
+undefined.
+
+These intrinsics work for both fixed and scalable vectors. While this intrinsic
+is marked as experimental, the recommended way to express this operation for
+fixed-width vectors is still to generate a constant vector instead.
+
+
+Arguments:
+""""""""""
+
+None.
+
+
 Matrix Intrinsics
 -----------------
 
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index e35dfddbe0439abc4e21fd91a6fa62f60a0fec33..e751ed90db2a48d5ffa933fccba2e34cd58ff40e 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -58,7 +58,8 @@ Non-comprehensive list of changes in this release
 Changes to the LLVM IR
 ----------------------
 
-* ...
+* The ``inalloca`` attribute now has a mandatory type field, similar
+  to ``byval`` and ``sret``.
 
 
 Changes to building LLVM
diff --git a/llvm/docs/Security.rst b/llvm/docs/Security.rst
index 20760bcb9f01ae5718971e1bc9ad14ef8abe2b35..4f223c95fd4e01888221f7d5d6ab1b694f199a06 100644
--- a/llvm/docs/Security.rst
+++ b/llvm/docs/Security.rst
@@ -29,6 +29,7 @@ The members of the group represent a wide cross-section of the community, and me
 * Artur Pilipenko (Azul Systems Inc)
 * Dimitry Andric (individual; FreeBSD)
 * Ed Maste (individual; FreeBSD)
+* George Burgess IV (Google)
 * Josh Eads (Sony)
 * Kristof Beyls (ARM)
 * Matthew Riley (Google)
diff --git a/llvm/docs/TableGen/ProgRef.rst b/llvm/docs/TableGen/ProgRef.rst
index c60bffef3ed2bf25dc9c1d7fef8fae533dff5abb..9799e29a63e67e9b8afc4d3fc5abcd6d345dbeac 100644
--- a/llvm/docs/TableGen/ProgRef.rst
+++ b/llvm/docs/TableGen/ProgRef.rst
@@ -299,7 +299,7 @@ wide range of records conveniently and compactly.
 :token:`ClassID`
     Specifying a class name in a type context indicates
     that the type of the defined value must
-    be a subclass of the specified class.  This is useful in conjunction with
+    be a subclass of the specified class. This is useful in conjunction with
     the ``list`` type; for example, to constrain the elements of the list to a
     common base class (e.g., a ``list<Register>`` can only contain definitions
     derived from the ``Register`` class).
@@ -554,19 +554,22 @@ classes and records can inherit.
    TemplateArgDecl: `Type` `TokIdentifier` ["=" `Value`]
 
 A class can be parameterized by a list of "template arguments," whose values
-can be used in the class's record body.  These template arguments are
+can be used in the class's record body. These template arguments are
 specified each time the class is inherited by another class or record.
 
 If a template argument is not assigned a default value with ``=``, it is
 uninitialized (has the "value" ``?``) and must be specified in the template
-argument list when the class is inherited. If an argument is assigned a
-default value, then it need not be specified in the argument list. The
-template argument default values are evaluated from left to right.
+argument list when the class is inherited (required argument). If an
+argument is assigned a default value, then it need not be specified in the
+argument list (optional argument). In the declaration, all required template
+arguments must precede any optional arguments. The template argument default
+values are evaluated from left to right.
 
 The :token:`RecordBody` is defined below. It can include a list of
-superclasses from which the current class inherits, along with field definitions
-and other statements. When a class ``C`` inherits from another class ``D``,
-the fields of ``D`` are effectively merged into the fields of ``C``.
+superclasses from which the current class inherits, along with field
+definitions and other statements. When a class ``C`` inherits from another
+class ``D``, the fields of ``D`` are effectively merged into the fields of
+``C``.
 
 A given class can only be defined once. A ``class`` statement is
 considered to define the class if *any* of the following are true (the
@@ -605,7 +608,7 @@ of the fields of the class or record.
    RecordBody: `ParentClassList` `Body`
    ParentClassList: [":" `ParentClassListNE`]
    ParentClassListNE: `ClassRef` ("," `ClassRef`)*
-   ClassRef: (`ClassID` | `MultiClassID`) ["<" `ValueList` ">"]
+   ClassRef: (`ClassID` | `MultiClassID`) ["<" [`ValueList`] ">"]
 
 A :token:`ParentClassList` containing a :token:`MultiClassID` is valid only
 in the class list of a ``defm`` statement. In that case, the ID must be the
diff --git a/llvm/docs/llvm-objdump.1 b/llvm/docs/llvm-objdump.1
index 1cf557dc8b6411424d0eb7d6a275faaa18822ba8..4e9dbaea870437517d556a33d2c5defaac7749c7 100644
--- a/llvm/docs/llvm-objdump.1
+++ b/llvm/docs/llvm-objdump.1
@@ -109,6 +109,11 @@ When disassembling instructions, do not print the instruction bytes.
 When disassembling, add
 .Ar PREFIX
 to absolute paths.
+.It Fl -prefix-strip Ns = Ns Ar LEVEL
+When disassembling, strip out
+.Ar LEVEL
+initial directories from absolute paths. This option has no effect without
+.Fl -prefix Ns = Ns PREFIX .
 .It Fl -print-imm-hex
 Use hex format for immediate values.
 .It Fl -private-header
diff --git a/llvm/examples/OrcV2Examples/CMakeLists.txt b/llvm/examples/OrcV2Examples/CMakeLists.txt
index bed277e59e0bbffe986a6e01ad6de2c4d83f920a..f6a11d6f0ef209091db6ce9b60244164d792f9bf 100644
--- a/llvm/examples/OrcV2Examples/CMakeLists.txt
+++ b/llvm/examples/OrcV2Examples/CMakeLists.txt
@@ -12,3 +12,7 @@ add_subdirectory(OrcV2CBindingsAddObjectFile)
 add_subdirectory(OrcV2CBindingsBasicUsage)
 add_subdirectory(OrcV2CBindingsReflectProcessSymbols)
 add_subdirectory(OrcV2CBindingsRemovableCode)
+
+if(CMAKE_HOST_UNIX)
+  add_subdirectory(LLJITWithRemoteDebugging)
+endif()
diff --git a/llvm/examples/OrcV2Examples/ExampleModules.h b/llvm/examples/OrcV2Examples/ExampleModules.h
index 72b0cc767f11993ed857d847ea3c1ec4b847faa0..53da756e15f78d08425b110cc354e731936c1adc 100644
--- a/llvm/examples/OrcV2Examples/ExampleModules.h
+++ b/llvm/examples/OrcV2Examples/ExampleModules.h
@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_EXAMPLES_HOWTOUSELLJIT_EXAMPLEMODULES_H
-#define LLVM_EXAMPLES_HOWTOUSELLJIT_EXAMPLEMODULES_H
+#ifndef LLVM_EXAMPLES_ORCV2EXAMPLES_EXAMPLEMODULES_H
+#define LLVM_EXAMPLES_ORCV2EXAMPLES_EXAMPLEMODULES_H
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
@@ -31,25 +31,37 @@ const llvm::StringRef Add1Example =
   }
 )";
 
+inline llvm::Error createSMDiagnosticError(llvm::SMDiagnostic &Diag) {
+  using namespace llvm;
+  std::string Msg;
+  {
+    raw_string_ostream OS(Msg);
+    Diag.print("", OS);
+  }
+  return make_error<StringError>(std::move(Msg), inconvertibleErrorCode());
+}
+
 inline llvm::Expected<llvm::orc::ThreadSafeModule>
 parseExampleModule(llvm::StringRef Source, llvm::StringRef Name) {
   using namespace llvm;
-  using namespace llvm::orc;
+  auto Ctx = std::make_unique<LLVMContext>();
+  SMDiagnostic Err;
+  if (auto M = parseIR(MemoryBufferRef(Source, Name), Err, *Ctx))
+    return orc::ThreadSafeModule(std::move(M), std::move(Ctx));
 
+  return createSMDiagnosticError(Err);
+}
+
+inline llvm::Expected<llvm::orc::ThreadSafeModule>
+parseExampleModuleFromFile(llvm::StringRef FileName) {
+  using namespace llvm;
   auto Ctx = std::make_unique<LLVMContext>();
   SMDiagnostic Err;
-  auto M = parseIR(MemoryBufferRef(Source, Name), Err, *Ctx);
-
-  if (!M) {
-    std::string ErrMsg;
-    {
-      raw_string_ostream ErrStream(ErrMsg);
-      Err.print("", ErrStream);
-    }
-    return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode());
-  }
 
-  return ThreadSafeModule(std::move(M), std::move(Ctx));
+  if (auto M = parseIRFile(FileName, Err, *Ctx))
+    return orc::ThreadSafeModule(std::move(M), std::move(Ctx));
+
+  return createSMDiagnosticError(Err);
 }
 
-#endif // LLVM_EXAMPLES_HOWTOUSELLJIT_EXAMPLEMODULES_H
+#endif // LLVM_EXAMPLES_ORCV2EXAMPLES_EXAMPLEMODULES_H
diff --git a/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp b/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp
index 78a0c5b197db201b8e9c5be61b149889fa2fefa1..1923139925b7ccd6a5997df3e6456ccf021d9ae7 100644
--- a/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp
+++ b/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp
@@ -1,4 +1,4 @@
-//===--------------- LLJITWithCustomObjectLinkingLayer.cpp ----------------===//
+//===--------------- LLJITWithGDBRegistrationListener.cpp -----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -48,7 +48,7 @@ int main(int argc, char *argv[]) {
   InitializeNativeTarget();
   InitializeNativeTargetAsmPrinter();
 
-  cl::ParseCommandLineOptions(argc, argv, "LLJITWithCustomObjectLinkingLayer");
+  cl::ParseCommandLineOptions(argc, argv, "LLJITWithGDBRegistrationListener");
   ExitOnErr.setBanner(std::string(argv[0]) + ": ");
 
   // Detect the host and set code model to small.
diff --git a/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/CMakeLists.txt b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..38d447bc91e3f538d97b6da3917c52ecb846bddd
--- /dev/null
+++ b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/CMakeLists.txt
@@ -0,0 +1,19 @@
+set(LLVM_LINK_COMPONENTS
+  Core
+  ExecutionEngine
+  IRReader
+  JITLink
+  OrcJIT
+  OrcShared
+  OrcTargetProcess
+  Support
+  nativecodegen
+  )
+
+add_llvm_example(LLJITWithRemoteDebugging
+  LLJITWithRemoteDebugging.cpp
+  RemoteJITUtils.cpp
+
+  DEPENDS
+    llvm-jitlink-executor
+  )
diff --git a/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6da30f6082062c0b18c01a5a5fab0e9b423d5bc0
--- /dev/null
+++ b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp
@@ -0,0 +1,258 @@
+//===--- LLJITWithRemoteDebugging.cpp - LLJIT targeting a child process ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This example shows how to use LLJIT and JITLink for out-of-process execution
+// with debug support.  A few notes beforehand:
+//
+//  * Debuggers must implement the GDB JIT interface (gdb, udb, lldb 12+).
+//  * Debug support is currently limited to ELF on x86-64 platforms that run
+//    Unix-like systems.
+//  * There is a test for this example and it ships an IR file that is prepared
+//    for the instructions below.
+//
+//
+// The following command line session provides a complete walkthrough of the
+// feature using LLDB 12:
+//
+// [Terminal 1] Prepare a debuggable out-of-process JIT session:
+//
+//    > cd llvm-project/build
+//    > ninja LLJITWithRemoteDebugging llvm-jitlink-executor
+//    > cp ../llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll .
+//    > bin/LLJITWithRemoteDebugging --wait-for-debugger argc_sub1_elf.ll
+//    Found out-of-process executor: bin/llvm-jitlink-executor
+//    Launched executor in subprocess: 65535
+//    Attach a debugger and press any key to continue.
+//
+//
+// [Terminal 2] Attach a debugger to the child process:
+//
+//    (lldb) log enable lldb jit
+//    (lldb) settings set plugin.jit-loader.gdb.enable on
+//    (lldb) settings set target.source-map Inputs/ \
+//             /path/to/llvm-project/llvm/test/Examples/OrcV2Examples/Inputs/
+//    (lldb) attach -p 65535
+//     JITLoaderGDB::SetJITBreakpoint looking for JIT register hook
+//     JITLoaderGDB::SetJITBreakpoint setting JIT breakpoint
+//    Process 65535 stopped
+//    (lldb) b sub1
+//    Breakpoint 1: no locations (pending).
+//    WARNING:  Unable to resolve breakpoint to any actual locations.
+//    (lldb) c
+//    Process 65535 resuming
+//
+//
+// [Terminal 1] Press a key to start code generation and execution:
+//
+//    Parsed input IR code from: argc_sub1_elf.ll
+//    Initialized LLJIT for remote executor
+//    Running: argc_sub1_elf.ll
+//
+//
+// [Terminal 2] Breakpoint hits; we change the argc value from 1 to 42:
+//
+//    (lldb)  JITLoaderGDB::JITDebugBreakpointHit hit JIT breakpoint
+//     JITLoaderGDB::ReadJITDescriptorImpl registering JIT entry at 0x106b34000
+//    1 location added to breakpoint 1
+//    Process 65535 stopped
+//    * thread #1, queue = 'com.apple.main-thread', stop reason = breakpoint 1.1
+//        frame #0: JIT(0x106b34000)`sub1(x=1) at argc_sub1.c:1:28
+//    -> 1   	int sub1(int x) { return x - 1; }
+//       2   	int main(int argc, char **argv) { return sub1(argc); }
+//    (lldb) p x
+//    (int) $0 = 1
+//    (lldb) expr x = 42
+//    (int) $1 = 42
+//    (lldb) c
+//
+//
+// [Terminal 1] Example output reflects the modified value:
+//
+//    Exit code: 41
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include "../ExampleModules.h"
+#include "RemoteJITUtils.h"
+
+#include <memory>
+#include <string>
+
+using namespace llvm;
+using namespace llvm::orc;
+
+// The LLVM IR file to run.
+static cl::list<std::string> InputFiles(cl::Positional, cl::OneOrMore,
+                                        cl::desc("<input files>"));
+
+// Command line arguments to pass to the JITed main function.
+static cl::list<std::string> InputArgv("args", cl::Positional,
+                                       cl::desc("<program arguments>..."),
+                                       cl::ZeroOrMore, cl::PositionalEatsArgs);
+
+// Given paths must exist on the remote target.
+static cl::list<std::string>
+    Dylibs("dlopen", cl::desc("Dynamic libraries to load before linking"),
+           cl::value_desc("filename"), cl::ZeroOrMore);
+
+// File path of the executable to launch for execution in a child process.
+// Inter-process communication will go through stdin/stdout pipes.
+static cl::opt<std::string>
+    OOPExecutor("executor", cl::desc("Set the out-of-process executor"),
+                cl::value_desc("filename"));
+
+// Network address of a running executor process that we can connected through a
+// TCP socket. It may run locally or on a remote machine.
+static cl::opt<std::string> OOPExecutorConnect(
+    "connect",
+    cl::desc("Connect to an out-of-process executor through a TCP socket"),
+    cl::value_desc("<hostname>:<port>"));
+
+// Give the user a chance to connect a debugger. Once we connected the executor
+// process, wait for the user to press a key (and print out its PID if it's a
+// child process).
+static cl::opt<bool>
+    WaitForDebugger("wait-for-debugger",
+                    cl::desc("Wait for user input before entering JITed code"),
+                    cl::init(false));
+
+ExitOnError ExitOnErr;
+
+static std::unique_ptr<JITLinkExecutor> connectExecutor(const char *Argv0,
+                                                        ExecutionSession &ES) {
+  // Connect to a running out-of-process executor through a TCP socket.
+  if (!OOPExecutorConnect.empty()) {
+    std::unique_ptr<TCPSocketJITLinkExecutor> Exec =
+        ExitOnErr(JITLinkExecutor::ConnectTCPSocket(OOPExecutorConnect, ES));
+
+    outs() << "Connected to executor at " << OOPExecutorConnect << "\n";
+    if (WaitForDebugger) {
+      outs() << "Attach a debugger and press any key to continue.\n";
+      fflush(stdin);
+      getchar();
+    }
+
+    return std::move(Exec);
+  }
+
+  // Launch a out-of-process executor locally in a child process.
+  std::unique_ptr<ChildProcessJITLinkExecutor> Exec = ExitOnErr(
+      OOPExecutor.empty() ? JITLinkExecutor::FindLocal(Argv0)
+                          : JITLinkExecutor::CreateLocal(OOPExecutor));
+
+  outs() << "Found out-of-process executor: " << Exec->getPath() << "\n";
+
+  ExitOnErr(Exec->launch(ES));
+  if (WaitForDebugger) {
+    outs() << "Launched executor in subprocess: " << Exec->getPID() << "\n"
+           << "Attach a debugger and press any key to continue.\n";
+    fflush(stdin);
+    getchar();
+  }
+
+  return std::move(Exec);
+}
+
+int main(int argc, char *argv[]) {
+  InitLLVM X(argc, argv);
+
+  InitializeNativeTarget();
+  InitializeNativeTargetAsmPrinter();
+
+  ExitOnErr.setBanner(std::string(argv[0]) + ": ");
+  cl::ParseCommandLineOptions(argc, argv, "LLJITWithRemoteDebugging");
+
+  auto ES = std::make_unique<ExecutionSession>();
+  ES->setErrorReporter([&](Error Err) { ExitOnErr(std::move(Err)); });
+
+  // Launch/connect the out-of-process executor.
+  std::unique_ptr<JITLinkExecutor> Executor = connectExecutor(argv[0], *ES);
+
+  // Load the given IR files.
+  std::vector<ThreadSafeModule> TSMs;
+  for (const std::string &Path : InputFiles) {
+    outs() << "Parsing input IR code from: " << Path << "\n";
+    TSMs.push_back(ExitOnErr(parseExampleModuleFromFile(Path)));
+  }
+
+  StringRef TT;
+  StringRef MainModuleName;
+  TSMs.front().withModuleDo([&MainModuleName, &TT](Module &M) {
+    MainModuleName = M.getName();
+    TT = M.getTargetTriple();
+  });
+
+  for (const ThreadSafeModule &TSM : TSMs)
+    ExitOnErr(TSM.withModuleDo([TT, MainModuleName](Module &M) -> Error {
+      if (M.getTargetTriple() != TT)
+        return make_error<StringError>(
+            formatv("Different target triples in input files:\n"
+                    "  '{0}' in '{1}'\n  '{2}' in '{3}'",
+                    TT, MainModuleName, M.getTargetTriple(), M.getName()),
+            inconvertibleErrorCode());
+      return Error::success();
+    }));
+
+  // Create a target machine that matches the input triple.
+  JITTargetMachineBuilder JTMB((Triple(TT)));
+  JTMB.setCodeModel(CodeModel::Small);
+  JTMB.setRelocationModel(Reloc::PIC_);
+
+  // Create LLJIT and destroy it before disconnecting the target process.
+  {
+    outs() << "Initializing LLJIT for remote executor\n";
+    auto J = ExitOnErr(LLJITBuilder()
+                           .setExecutionSession(std::move(ES))
+                           .setJITTargetMachineBuilder(std::move(JTMB))
+                           .setObjectLinkingLayerCreator(std::ref(*Executor))
+                           .create());
+
+    // Add plugin for debug support.
+    ExitOnErr(Executor->addDebugSupport(J->getObjLinkingLayer()));
+
+    // Load required shared libraries on the remote target and add a generator
+    // for each of it, so the compiler can lookup their symbols.
+    for (const std::string &Path : Dylibs)
+      J->getMainJITDylib().addGenerator(ExitOnErr(Executor->loadDylib(Path)));
+
+    // Add the loaded IR module to the JIT. This will set up symbol tables and
+    // prepare for materialization.
+    for (ThreadSafeModule &TSM : TSMs)
+      ExitOnErr(J->addIRModule(std::move(TSM)));
+
+    // The example uses a non-lazy JIT for simplicity. Thus, looking up the main
+    // function will materialize all reachable code. It also triggers debug
+    // registration in the remote target process.
+    JITEvaluatedSymbol MainFn = ExitOnErr(J->lookup("main"));
+
+    outs() << "Running: main(";
+    int Pos = 0;
+    for (const std::string &Arg : InputArgv)
+      outs() << (Pos++ == 0 ? "" : ", ") << Arg;
+    outs() << ")\n";
+
+    // Execute the code in the remote target process and dump the result. With
+    // the debugger attached to the target, it should be possible to inspect the
+    // JITed code as if it was compiled statically.
+    int Result = ExitOnErr(Executor->runAsMain(MainFn, InputArgv));
+    outs() << "Exit code: " << Result << "\n";
+  }
+
+  ExitOnErr(Executor->disconnect());
+  return 0;
+}
diff --git a/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/RemoteJITUtils.cpp b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/RemoteJITUtils.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..abce14de0fe04b2c239bac67f32c81ce4c677d62
--- /dev/null
+++ b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/RemoteJITUtils.cpp
@@ -0,0 +1,347 @@
+//===-- RemoteJITUtils.cpp - Utilities for remote-JITing --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RemoteJITUtils.h"
+
+#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
+#include "llvm/ExecutionEngine/Orc/OrcRPCTargetProcessControl.h"
+#include "llvm/ExecutionEngine/Orc/Shared/RPCUtils.h"
+#include "llvm/ExecutionEngine/Orc/TPCDebugObjectRegistrar.h"
+#include "llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+#ifdef LLVM_ON_UNIX
+#include <netdb.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <unistd.h>
+#endif // LLVM_ON_UNIX
+
+using namespace llvm;
+using namespace llvm::orc;
+
+namespace llvm {
+namespace orc {
+
+class RemoteTargetProcessControl
+    : public OrcRPCTargetProcessControlBase<
+          shared::MultiThreadedRPCEndpoint<JITLinkExecutor::RPCChannel>> {
+public:
+  using RPCChannel = JITLinkExecutor::RPCChannel;
+  using RPCEndpoint = shared::MultiThreadedRPCEndpoint<RPCChannel>;
+
+private:
+  using ThisT = RemoteTargetProcessControl;
+  using BaseT = OrcRPCTargetProcessControlBase<RPCEndpoint>;
+  using MemoryAccess = OrcRPCTPCMemoryAccess<ThisT>;
+  using MemoryManager = OrcRPCTPCJITLinkMemoryManager<ThisT>;
+
+public:
+  using BaseT::initializeORCRPCTPCBase;
+
+  RemoteTargetProcessControl(ExecutionSession &ES,
+                             std::unique_ptr<RPCChannel> Channel,
+                             std::unique_ptr<RPCEndpoint> Endpoint);
+
+  void initializeMemoryManagement();
+  Error disconnect() override;
+
+private:
+  std::unique_ptr<RPCChannel> Channel;
+  std::unique_ptr<RPCEndpoint> Endpoint;
+  std::unique_ptr<MemoryAccess> OwnedMemAccess;
+  std::unique_ptr<MemoryManager> OwnedMemMgr;
+  std::atomic<bool> Finished{false};
+  std::thread ListenerThread;
+};
+
+RemoteTargetProcessControl::RemoteTargetProcessControl(
+    ExecutionSession &ES, std::unique_ptr<RPCChannel> Channel,
+    std::unique_ptr<RPCEndpoint> Endpoint)
+    : BaseT(ES.getSymbolStringPool(), *Endpoint,
+            [&ES](Error Err) { ES.reportError(std::move(Err)); }),
+      Channel(std::move(Channel)), Endpoint(std::move(Endpoint)) {
+
+  ListenerThread = std::thread([&]() {
+    while (!Finished) {
+      if (auto Err = this->Endpoint->handleOne()) {
+        reportError(std::move(Err));
+        return;
+      }
+    }
+  });
+}
+
+void RemoteTargetProcessControl::initializeMemoryManagement() {
+  OwnedMemAccess = std::make_unique<MemoryAccess>(*this);
+  OwnedMemMgr = std::make_unique<MemoryManager>(*this);
+
+  // Base class needs non-owning access.
+  MemAccess = OwnedMemAccess.get();
+  MemMgr = OwnedMemMgr.get();
+}
+
+Error RemoteTargetProcessControl::disconnect() {
+  std::promise<MSVCPError> P;
+  auto F = P.get_future();
+  auto Err = closeConnection([&](Error Err) -> Error {
+    P.set_value(std::move(Err));
+    Finished = true;
+    return Error::success();
+  });
+  ListenerThread.join();
+  return joinErrors(std::move(Err), F.get());
+}
+
+} // namespace orc
+} // namespace llvm
+
+JITLinkExecutor::JITLinkExecutor() = default;
+JITLinkExecutor::~JITLinkExecutor() = default;
+
+Expected<std::unique_ptr<ObjectLayer>>
+JITLinkExecutor::operator()(ExecutionSession &ES, const Triple &TT) {
+  return std::make_unique<ObjectLinkingLayer>(ES, TPC->getMemMgr());
+}
+
+Error JITLinkExecutor::addDebugSupport(ObjectLayer &ObjLayer) {
+  auto Registrar = createJITLoaderGDBRegistrar(*TPC);
+  if (!Registrar)
+    return Registrar.takeError();
+
+  cast<ObjectLinkingLayer>(&ObjLayer)->addPlugin(
+      std::make_unique<DebugObjectManagerPlugin>(ObjLayer.getExecutionSession(),
+                                                 std::move(*Registrar)));
+
+  return Error::success();
+}
+
+Expected<std::unique_ptr<DefinitionGenerator>>
+JITLinkExecutor::loadDylib(StringRef RemotePath) {
+  if (auto Handle = TPC->loadDylib(RemotePath.data()))
+    return std::make_unique<TPCDynamicLibrarySearchGenerator>(*TPC, *Handle);
+  else
+    return Handle.takeError();
+}
+
+Expected<int> JITLinkExecutor::runAsMain(JITEvaluatedSymbol MainSym,
+                                         ArrayRef<std::string> Args) {
+  return TPC->runAsMain(MainSym.getAddress(), Args);
+}
+
+Error JITLinkExecutor::disconnect() { return TPC->disconnect(); }
+
+static std::string defaultPath(const char *HostArgv0, StringRef ExecutorName) {
+  // This just needs to be some symbol in the binary; C++ doesn't
+  // allow taking the address of ::main however.
+  void *P = (void *)(intptr_t)defaultPath;
+  SmallString<256> FullName(sys::fs::getMainExecutable(HostArgv0, P));
+  sys::path::remove_filename(FullName);
+  sys::path::append(FullName, ExecutorName);
+  return FullName.str().str();
+}
+
+Expected<std::unique_ptr<ChildProcessJITLinkExecutor>>
+JITLinkExecutor::FindLocal(const char *HostArgv) {
+  std::string BestGuess = defaultPath(HostArgv, "llvm-jitlink-executor");
+  auto Executor = CreateLocal(BestGuess);
+  if (!Executor) {
+    consumeError(Executor.takeError());
+    return make_error<StringError>(
+        formatv("Unable to find usable executor: {0}", BestGuess),
+        inconvertibleErrorCode());
+  }
+  return Executor;
+}
+
+Expected<std::unique_ptr<ChildProcessJITLinkExecutor>>
+JITLinkExecutor::CreateLocal(std::string ExecutablePath) {
+  if (!sys::fs::can_execute(ExecutablePath))
+    return make_error<StringError>(
+        formatv("Specified executor invalid: {0}", ExecutablePath),
+        inconvertibleErrorCode());
+  return std::unique_ptr<ChildProcessJITLinkExecutor>(
+      new ChildProcessJITLinkExecutor(std::move(ExecutablePath)));
+}
+
+TCPSocketJITLinkExecutor::TCPSocketJITLinkExecutor(
+    std::unique_ptr<RemoteTargetProcessControl> TPC) {
+  this->TPC = std::move(TPC);
+}
+
+#ifndef LLVM_ON_UNIX
+
+// FIXME: Add support for Windows.
+Error ChildProcessJITLinkExecutor::launch(ExecutionSession &ES) {
+  return make_error<StringError>(
+      "Remote JITing not yet supported on non-unix platforms",
+      inconvertibleErrorCode());
+}
+
+// FIXME: Add support for Windows.
+Expected<std::unique_ptr<TCPSocketJITLinkExecutor>>
+JITLinkExecutor::ConnectTCPSocket(StringRef NetworkAddress,
+                                  ExecutionSession &ES) {
+  return make_error<StringError>(
+      "Remote JITing not yet supported on non-unix platforms",
+      inconvertibleErrorCode());
+}
+
+#else
+
+Error ChildProcessJITLinkExecutor::launch(ExecutionSession &ES) {
+  constexpr int ReadEnd = 0;
+  constexpr int WriteEnd = 1;
+
+  // Pipe FDs.
+  int ToExecutor[2];
+  int FromExecutor[2];
+
+  // Create pipes to/from the executor..
+  if (pipe(ToExecutor) != 0 || pipe(FromExecutor) != 0)
+    return make_error<StringError>("Unable to create pipe for executor",
+                                   inconvertibleErrorCode());
+
+  ProcessID = fork();
+  if (ProcessID == 0) {
+    // In the child...
+
+    // Close the parent ends of the pipes
+    close(ToExecutor[WriteEnd]);
+    close(FromExecutor[ReadEnd]);
+
+    // Execute the child process.
+    std::unique_ptr<char[]> ExecPath, FDSpecifier;
+    {
+      ExecPath = std::make_unique<char[]>(ExecutablePath.size() + 1);
+      strcpy(ExecPath.get(), ExecutablePath.data());
+
+      std::string FDSpecifierStr("filedescs=");
+      FDSpecifierStr += utostr(ToExecutor[ReadEnd]);
+      FDSpecifierStr += ',';
+      FDSpecifierStr += utostr(FromExecutor[WriteEnd]);
+      FDSpecifier = std::make_unique<char[]>(FDSpecifierStr.size() + 1);
+      strcpy(FDSpecifier.get(), FDSpecifierStr.c_str());
+    }
+
+    char *const Args[] = {ExecPath.get(), FDSpecifier.get(), nullptr};
+    int RC = execvp(ExecPath.get(), Args);
+    if (RC != 0)
+      return make_error<StringError>(
+          "Unable to launch out-of-process executor '" + ExecutablePath + "'\n",
+          inconvertibleErrorCode());
+
+    llvm_unreachable("Fork won't return in success case");
+  }
+  // else we're the parent...
+
+  // Close the child ends of the pipes
+  close(ToExecutor[ReadEnd]);
+  close(FromExecutor[WriteEnd]);
+
+  auto Channel =
+      std::make_unique<RPCChannel>(FromExecutor[ReadEnd], ToExecutor[WriteEnd]);
+  auto Endpoint =
+      std::make_unique<RemoteTargetProcessControl::RPCEndpoint>(*Channel, true);
+
+  TPC = std::make_unique<RemoteTargetProcessControl>(ES, std::move(Channel),
+                                                     std::move(Endpoint));
+
+  if (auto Err = TPC->initializeORCRPCTPCBase())
+    return joinErrors(std::move(Err), TPC->disconnect());
+
+  TPC->initializeMemoryManagement();
+
+  shared::registerStringError<RPCChannel>();
+  return Error::success();
+}
+
+static Expected<int> connectTCPSocketImpl(std::string Host,
+                                          std::string PortStr) {
+  addrinfo *AI;
+  addrinfo Hints{};
+  Hints.ai_family = AF_INET;
+  Hints.ai_socktype = SOCK_STREAM;
+  Hints.ai_flags = AI_NUMERICSERV;
+
+  if (int EC = getaddrinfo(Host.c_str(), PortStr.c_str(), &Hints, &AI))
+    return make_error<StringError>(
+        formatv("address resolution failed ({0})", gai_strerror(EC)),
+        inconvertibleErrorCode());
+
+  // Cycle through the returned addrinfo structures and connect to the first
+  // reachable endpoint.
+  int SockFD;
+  addrinfo *Server;
+  for (Server = AI; Server != nullptr; Server = Server->ai_next) {
+    // If socket fails, maybe it's because the address family is not supported.
+    // Skip to the next addrinfo structure.
+    if ((SockFD = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol)) < 0)
+      continue;
+
+    // If connect works, we exit the loop with a working socket.
+    if (connect(SockFD, Server->ai_addr, Server->ai_addrlen) == 0)
+      break;
+
+    close(SockFD);
+  }
+  freeaddrinfo(AI);
+
+  // Did we reach the end of the loop without connecting to a valid endpoint?
+  if (Server == nullptr)
+    return make_error<StringError>("invalid hostname",
+                                   inconvertibleErrorCode());
+
+  return SockFD;
+}
+
+Expected<std::unique_ptr<TCPSocketJITLinkExecutor>>
+JITLinkExecutor::ConnectTCPSocket(StringRef NetworkAddress,
+                                  ExecutionSession &ES) {
+  auto CreateErr = [NetworkAddress](StringRef Details) {
+    return make_error<StringError>(
+        formatv("Failed to connect TCP socket '{0}': {1}", NetworkAddress,
+                Details),
+        inconvertibleErrorCode());
+  };
+
+  StringRef Host, PortStr;
+  std::tie(Host, PortStr) = NetworkAddress.split(':');
+  if (Host.empty())
+    return CreateErr("host name cannot be empty");
+  if (PortStr.empty())
+    return CreateErr("port cannot be empty");
+  int Port = 0;
+  if (PortStr.getAsInteger(10, Port))
+    return CreateErr("port number is not a valid integer");
+
+  Expected<int> SockFD = connectTCPSocketImpl(Host.str(), PortStr.str());
+  if (!SockFD)
+    return CreateErr(toString(SockFD.takeError()));
+
+  auto Channel = std::make_unique<RPCChannel>(*SockFD, *SockFD);
+  auto Endpoint =
+      std::make_unique<RemoteTargetProcessControl::RPCEndpoint>(*Channel, true);
+
+  auto TPC = std::make_unique<RemoteTargetProcessControl>(
+      ES, std::move(Channel), std::move(Endpoint));
+
+  if (auto Err = TPC->initializeORCRPCTPCBase())
+    return joinErrors(std::move(Err), TPC->disconnect());
+
+  TPC->initializeMemoryManagement();
+  shared::registerStringError<RPCChannel>();
+
+  return std::unique_ptr<TCPSocketJITLinkExecutor>(
+      new TCPSocketJITLinkExecutor(std::move(TPC)));
+}
+
+#endif
diff --git a/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/RemoteJITUtils.h b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/RemoteJITUtils.h
new file mode 100644
index 0000000000000000000000000000000000000000..e629c0e036f507f80572904876e7b4489b75d3ed
--- /dev/null
+++ b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/RemoteJITUtils.h
@@ -0,0 +1,111 @@
+//===-- RemoteJITUtils.h - Utilities for remote-JITing ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for TargetProcessControl-based remote JITing with Orc and JITLink.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXAMPLES_ORCV2EXAMPLES_LLJITWITHREMOTEDEBUGGING_REMOTEJITUTILS_H
+#define LLVM_EXAMPLES_ORCV2EXAMPLES_LLJITWITHREMOTEDEBUGGING_REMOTEJITUTILS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
+#include "llvm/ExecutionEngine/Orc/Shared/FDRawByteChannel.h"
+#include "llvm/Support/Error.h"
+
+#include <memory>
+#include <string>
+
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <unistd.h>
+#else
+#include <io.h>
+#endif
+
+namespace llvm {
+namespace orc {
+
+class ChildProcessJITLinkExecutor;
+class RemoteTargetProcessControl;
+class TCPSocketJITLinkExecutor;
+
+class JITLinkExecutor {
+public:
+  using RPCChannel = shared::FDRawByteChannel;
+
+  /// Create a JITLinkExecutor for the given exectuable on disk.
+  static Expected<std::unique_ptr<ChildProcessJITLinkExecutor>>
+  CreateLocal(std::string ExecutablePath);
+
+  /// Find the default exectuable on disk and create a JITLinkExecutor for it.
+  static Expected<std::unique_ptr<ChildProcessJITLinkExecutor>>
+  FindLocal(const char *JITArgv0);
+
+  /// Create a JITLinkExecutor that connects to the given network address
+  /// through a TCP socket. A valid NetworkAddress provides hostname and port,
+  /// e.g. localhost:20000.
+  static Expected<std::unique_ptr<TCPSocketJITLinkExecutor>>
+  ConnectTCPSocket(StringRef NetworkAddress, ExecutionSession &ES);
+
+  // Implement ObjectLinkingLayerCreator
+  Expected<std::unique_ptr<ObjectLayer>> operator()(ExecutionSession &,
+                                                    const Triple &);
+
+  Error addDebugSupport(ObjectLayer &ObjLayer);
+
+  Expected<std::unique_ptr<DefinitionGenerator>>
+  loadDylib(StringRef RemotePath);
+
+  Expected<int> runAsMain(JITEvaluatedSymbol MainSym,
+                          ArrayRef<std::string> Args);
+  Error disconnect();
+
+  virtual ~JITLinkExecutor();
+
+protected:
+  std::unique_ptr<RemoteTargetProcessControl> TPC;
+
+  JITLinkExecutor();
+};
+
+/// JITLinkExecutor that runs in a child process on the local machine.
+class ChildProcessJITLinkExecutor : public JITLinkExecutor {
+public:
+  Error launch(ExecutionSession &ES);
+
+  pid_t getPID() const { return ProcessID; }
+  StringRef getPath() const { return ExecutablePath; }
+
+private:
+  std::string ExecutablePath;
+  pid_t ProcessID;
+
+  ChildProcessJITLinkExecutor(std::string ExecutablePath)
+      : ExecutablePath(std::move(ExecutablePath)) {}
+
+  static std::string defaultPath(const char *HostArgv0, StringRef ExecutorName);
+  friend class JITLinkExecutor;
+};
+
+/// JITLinkExecutor connected through a TCP socket.
+class TCPSocketJITLinkExecutor : public JITLinkExecutor {
+private:
+  TCPSocketJITLinkExecutor(std::unique_ptr<RemoteTargetProcessControl> TPC);
+
+  friend class JITLinkExecutor;
+};
+
+} // namespace orc
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 91c15323451fd97903fc00ed0e1fc7b0b0ac2f7e..b3200520b90a7df87e90cd7b2aa30a91c83d1662 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -604,6 +604,17 @@ unsigned LLVMGetEnumAttributeKind(LLVMAttributeRef A);
  */
 uint64_t LLVMGetEnumAttributeValue(LLVMAttributeRef A);
 
+/**
+ * Create a type attribute
+ */
+LLVMAttributeRef LLVMCreateTypeAttribute(LLVMContextRef C, unsigned KindID,
+                                         LLVMTypeRef type_ref);
+
+/**
+ * Get the type attribute's value.
+ */
+LLVMTypeRef LLVMGetTypeAttributeValue(LLVMAttributeRef A);
+
 /**
  * Create a string attribute.
  */
@@ -626,6 +637,7 @@ const char *LLVMGetStringAttributeValue(LLVMAttributeRef A, unsigned *Length);
  */
 LLVMBool LLVMIsEnumAttribute(LLVMAttributeRef A);
 LLVMBool LLVMIsStringAttribute(LLVMAttributeRef A);
+LLVMBool LLVMIsTypeAttribute(LLVMAttributeRef A);
 
 /**
  * Obtain a Type from a context by its registered name.
diff --git a/llvm/include/llvm-c/Orc.h b/llvm/include/llvm-c/Orc.h
index 183107c148a6d2e128d65867425fb97a1461e783..9beef44c89dd5ed2b49fbd320df437e42b6a5f0b 100644
--- a/llvm/include/llvm-c/Orc.h
+++ b/llvm/include/llvm-c/Orc.h
@@ -339,8 +339,7 @@ LLVMErrorRef LLVMOrcResourceTrackerRemove(LLVMOrcResourceTrackerRef RT);
  * ownership has not been passed to a JITDylib (e.g. because some error
  * prevented the client from calling LLVMOrcJITDylibAddGenerator).
  */
-void LLVMOrcDisposeDefinitionGenerator(
-    LLVMOrcDefinitionGeneratorRef DG);
+void LLVMOrcDisposeDefinitionGenerator(LLVMOrcDefinitionGeneratorRef DG);
 
 /**
  * Dispose of a MaterializationUnit.
@@ -388,7 +387,9 @@ LLVMOrcExecutionSessionCreateJITDylib(LLVMOrcExecutionSessionRef ES,
  * Returns the JITDylib with the given name, or NULL if no such JITDylib
  * exists.
  */
-LLVMOrcJITDylibRef LLVMOrcExecutionSessionGetJITDylibByName(const char *Name);
+LLVMOrcJITDylibRef
+LLVMOrcExecutionSessionGetJITDylibByName(LLVMOrcExecutionSessionRef ES,
+                                         const char *Name);
 
 /**
  * Return a reference to a newly created resource tracker associated with JD.
diff --git a/llvm/include/llvm/ADT/BitVector.h b/llvm/include/llvm/ADT/BitVector.h
index 2a857786f454ffbd7e57bf2de932c2c4a5a1cbff..080f9611678fcfcfc529f5c9faede792807e9769 100644
--- a/llvm/include/llvm/ADT/BitVector.h
+++ b/llvm/include/llvm/ADT/BitVector.h
@@ -568,6 +568,20 @@ public:
     return false;
   }
 
+  template <class F, class... ArgTys>
+  static BitVector &apply(F &&f, BitVector &Out, BitVector const &Arg,
+                          ArgTys const &...Args) {
+    assert(llvm::all_of(
+               std::initializer_list<unsigned>{Args.size()...},
+               [&Arg](auto const &BV) { return Arg.size() == BV; }) &&
+           "consistent sizes");
+    Out.resize(Arg.size());
+    for (size_t i = 0, e = Out.NumBitWords(Arg.size()); i != e; ++i)
+      Out.Bits[i] = f(Arg.Bits[i], Args.Bits[i]...);
+    Out.clear_unused_bits();
+    return Out;
+  }
+
   BitVector &operator|=(const BitVector &RHS) {
     if (size() < RHS.size())
       resize(RHS.size());
diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h
index 3b351b732e9af1da8e56514c898be7ca94cfab23..b6a6fe67b7af19bdbd651ef17760f40f28cd024c 100644
--- a/llvm/include/llvm/ADT/Triple.h
+++ b/llvm/include/llvm/ADT/Triple.h
@@ -218,6 +218,7 @@ public:
     Musl,
     MuslEABI,
     MuslEABIHF,
+    MuslX32,
 
     MSVC,
     Itanium,
@@ -688,7 +689,8 @@ public:
   bool isMusl() const {
     return getEnvironment() == Triple::Musl ||
            getEnvironment() == Triple::MuslEABI ||
-           getEnvironment() == Triple::MuslEABIHF;
+           getEnvironment() == Triple::MuslEABIHF ||
+           getEnvironment() == Triple::MuslX32;
   }
 
   /// Tests whether the target is SPIR (32- or 64-bit).
diff --git a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h
index 6b5e9d5b71ba74615b167def0174e9932052d762..f13cf0c313c1f3f708d2ed27118cad4a2e095f75 100644
--- a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h
@@ -35,7 +35,6 @@ class DataLayout;
 class DominatorTree;
 class Function;
 class GEPOperator;
-class LoopInfo;
 class PHINode;
 class SelectInst;
 class TargetLibraryInfo;
@@ -56,23 +55,20 @@ class BasicAAResult : public AAResultBase<BasicAAResult> {
   const TargetLibraryInfo &TLI;
   AssumptionCache &AC;
   DominatorTree *DT;
-  LoopInfo *LI;
   PhiValues *PV;
 
 public:
   BasicAAResult(const DataLayout &DL, const Function &F,
                 const TargetLibraryInfo &TLI, AssumptionCache &AC,
-                DominatorTree *DT = nullptr, LoopInfo *LI = nullptr,
-                PhiValues *PV = nullptr)
-      : AAResultBase(), DL(DL), F(F), TLI(TLI), AC(AC), DT(DT), LI(LI), PV(PV)
-        {}
+                DominatorTree *DT = nullptr, PhiValues *PV = nullptr)
+      : AAResultBase(), DL(DL), F(F), TLI(TLI), AC(AC), DT(DT), PV(PV) {}
 
   BasicAAResult(const BasicAAResult &Arg)
       : AAResultBase(Arg), DL(Arg.DL), F(Arg.F), TLI(Arg.TLI), AC(Arg.AC),
-        DT(Arg.DT),  LI(Arg.LI), PV(Arg.PV) {}
+        DT(Arg.DT), PV(Arg.PV) {}
   BasicAAResult(BasicAAResult &&Arg)
       : AAResultBase(std::move(Arg)), DL(Arg.DL), F(Arg.F), TLI(Arg.TLI),
-        AC(Arg.AC), DT(Arg.DT), LI(Arg.LI), PV(Arg.PV) {}
+        AC(Arg.AC), DT(Arg.DT), PV(Arg.PV) {}
 
   /// Handle invalidation events in the new pass manager.
   bool invalidate(Function &Fn, const PreservedAnalyses &PA,
@@ -184,12 +180,6 @@ private:
   /// Tracks instructions visited by pointsToConstantMemory.
   SmallPtrSet<const Value *, 16> Visited;
 
-  static const Value *
-  GetLinearExpression(const Value *V, APInt &Scale, APInt &Offset,
-                      unsigned &ZExtBits, unsigned &SExtBits,
-                      const DataLayout &DL, unsigned Depth, AssumptionCache *AC,
-                      DominatorTree *DT, bool &NSW, bool &NUW);
-
   static DecomposedGEP
   DecomposeGEPExpression(const Value *V, const DataLayout &DL,
                          AssumptionCache *AC, DominatorTree *DT);
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 41d353dcd57349521d9ca8b587a869caba033d11..0a8d5c0d2eaea33373163844e8d310c87e883014 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -305,10 +305,9 @@ public:
   /// Returns floating-point induction operator that does not allow
   /// reassociation (transforming the induction requires an override of normal
   /// floating-point rules).
-  /// TODO: This should not require the full 'fast' FMF, but caller code
-  ///       may need to be fixed to propagate FMF correctly.
   Instruction *getExactFPMathInst() {
-    if (IK == IK_FpInduction && InductionBinOp && !InductionBinOp->isFast())
+    if (IK == IK_FpInduction && InductionBinOp &&
+        !InductionBinOp->hasAllowReassoc())
       return InductionBinOp;
     return nullptr;
   }
diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h
index 17d6f30a35cbccce706791e40ed374e4bae4a690..dda90e826bbabdcf69b403feef21fad393760835 100644
--- a/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -294,8 +294,8 @@ Value *SimplifyInstruction(Instruction *I, const SimplifyQuery &Q,
 
 /// See if V simplifies when its operand Op is replaced with RepOp. If not,
 /// return null.
-/// AllowRefinement specifies whether the simplification can be a refinement,
-/// or whether it needs to be strictly identical.
+/// AllowRefinement specifies whether the simplification can be a refinement
+/// (e.g. 0 instead of poison), or whether it needs to be strictly identical.
 Value *SimplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
                               const SimplifyQuery &Q, bool AllowRefinement);
 
diff --git a/llvm/include/llvm/Analysis/Loads.h b/llvm/include/llvm/Analysis/Loads.h
index f41e5e0c6e95de3982d5d385274df4b7ffa64481..ced1943b81d9593cb1462babc73a52abec1c0182 100644
--- a/llvm/include/llvm/Analysis/Loads.h
+++ b/llvm/include/llvm/Analysis/Loads.h
@@ -25,6 +25,7 @@ class Instruction;
 class LoadInst;
 class Loop;
 class MDNode;
+class MemoryLocation;
 class ScalarEvolution;
 class TargetLibraryInfo;
 
@@ -147,7 +148,7 @@ Value *FindAvailableLoadedValue(LoadInst *Load, AAResults &AA, bool *IsLoadCSE,
 /// this function, if ScanFrom points at the beginning of the block, it's safe
 /// to continue scanning the predecessors.
 ///
-/// \param Ptr The pointer we want the load and store to originate from.
+/// \param Loc The location we want the load and store to originate from.
 /// \param AccessTy The access type of the pointer.
 /// \param AtLeastAtomic Are we looking for at-least an atomic load/store ? In
 /// case it is false, we can return an atomic or non-atomic load or store. In
@@ -163,8 +164,8 @@ Value *FindAvailableLoadedValue(LoadInst *Load, AAResults &AA, bool *IsLoadCSE,
 /// location in memory, as opposed to the value operand of a store.
 ///
 /// \returns The found value, or nullptr if no value is found.
-Value *FindAvailablePtrLoadStore(Value *Ptr, Type *AccessTy, bool AtLeastAtomic,
-                                 BasicBlock *ScanBB,
+Value *findAvailablePtrLoadStore(const MemoryLocation &Loc, Type *AccessTy,
+                                 bool AtLeastAtomic, BasicBlock *ScanBB,
                                  BasicBlock::iterator &ScanFrom,
                                  unsigned MaxInstsToScan, AAResults *AA,
                                  bool *IsLoadCSE, unsigned *NumScanedInst);
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 13fbe884eddf9fa02ecb5e35a7c53382f71dabd6..39acfd5bbbeeca453ba4f8f0c6d48a1e130d1335 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -679,6 +679,15 @@ int64_t getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, const Loop *Lp,
                      const ValueToValueMap &StridesMap = ValueToValueMap(),
                      bool Assume = false, bool ShouldCheckWrap = true);
 
+/// Returns the distance between the pointers \p PtrA and \p PtrB iff they are
+/// compatible and it is possible to calculate the distance between them. This
+/// is a simple API that does not depend on the analysis pass.
+/// \param StrictCheck Ensure that the calculated distance matches the
+/// type-based one after all the bitcasts removal in the provided pointers.
+Optional<int> getPointersDiff(Value *PtrA, Value *PtrB, const DataLayout &DL,
+                              ScalarEvolution &SE, bool StrictCheck = false,
+                              bool CheckType = true);
+
 /// Attempt to sort the pointers in \p VL and return the sorted indices
 /// in \p SortedIndices, if reordering is required.
 ///
diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h
index a5717bae12c3f65b69d412ed699631a95f7a001c..c2c68004a9f9cf50b80826f4252024931a5e11b7 100644
--- a/llvm/include/llvm/Analysis/LoopInfo.h
+++ b/llvm/include/llvm/Analysis/LoopInfo.h
@@ -1199,6 +1199,14 @@ public:
 
     return true;
   }
+
+  // Return true if a new use of V added in ExitBB would require an LCSSA PHI
+  // to be inserted at the begining of the block.  Note that V is assumed to
+  // dominate ExitBB, and ExitBB must be the exit block of some loop.  The
+  // IR is assumed to be in LCSSA form before the planned insertion.
+  bool wouldBeOutOfLoopUseRequiringLCSSA(const Value *V,
+                                         const BasicBlock *ExitBB) const;
+
 };
 
 // Allow clients to walk the list of nested loops...
diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index c5428726995e4b2b3cdd95cf2224e020eaa175c6..39ade20df53f6ba9bb61139499b3c6555a779d57 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -212,6 +212,10 @@ struct ObjectSizeOpts {
 /// object size in Size if successful, and false otherwise. In this context, by
 /// object we mean the region of memory starting at Ptr to the end of the
 /// underlying object pointed to by Ptr.
+///
+/// WARNING: The object size returned is the allocation size.  This does not
+/// imply dereferenceability at site of use since the object may be freeed in
+/// between.
 bool getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout &DL,
                    const TargetLibraryInfo *TLI, ObjectSizeOpts Opts = {});
 
diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
index c7967df12418fa376faa2560d9ed4de4a6405609..e3e6268004f8197e820b969a4dc3ef9e802a2325 100644
--- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
+++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
@@ -38,7 +38,7 @@ class Function;
 // units. This would require making this depend on BFI.
 class ProfileSummaryInfo {
 private:
-  const Module &M;
+  const Module *M;
   std::unique_ptr<ProfileSummary> Summary;
   void computeThresholds();
   // Count thresholds to answer isHotCount and isColdCount queries.
@@ -58,7 +58,9 @@ private:
   mutable DenseMap<int, uint64_t> ThresholdCache;
 
 public:
-  ProfileSummaryInfo(const Module &M) : M(M) { refresh(); }
+  ProfileSummaryInfo(const Module &M) : M(&M) { refresh(); }
+  ProfileSummaryInfo(std::unique_ptr<ProfileSummary> PSI)
+      : M(nullptr), Summary(std::move(PSI)) {}
 
   ProfileSummaryInfo(ProfileSummaryInfo &&Arg) = default;
 
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index c35c1db7dfe04d7152ac699c8b31cd4b823af13a..8407be99a82b148ba69921306a352c420fee1c00 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -938,11 +938,24 @@ public:
   bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS,
                         const SCEV *RHS);
 
+  /// Check whether the condition described by Pred, LHS, and RHS is true or
+  /// false. If we know it, return the evaluation of this condition. If neither
+  /// is proved, return None.
+  Optional<bool> evaluatePredicate(ICmpInst::Predicate Pred, const SCEV *LHS,
+                                   const SCEV *RHS);
+
   /// Test if the given expression is known to satisfy the condition described
   /// by Pred, LHS, and RHS in the given Context.
   bool isKnownPredicateAt(ICmpInst::Predicate Pred, const SCEV *LHS,
                         const SCEV *RHS, const Instruction *Context);
 
+  /// Check whether the condition described by Pred, LHS, and RHS is true or
+  /// false in the given \p Context. If we know it, return the evaluation of
+  /// this condition. If neither is proved, return None.
+  Optional<bool> evaluatePredicateAt(ICmpInst::Predicate Pred, const SCEV *LHS,
+                                     const SCEV *RHS,
+                                     const Instruction *Context);
+
   /// Test if the condition described by Pred, LHS, RHS is known to be true on
   /// every iteration of the loop of the recurrency LHS.
   bool isKnownOnEveryIteration(ICmpInst::Predicate Pred,
@@ -1561,6 +1574,12 @@ private:
   ConstantRange getRangeViaFactoring(const SCEV *Start, const SCEV *Stop,
                                      const SCEV *MaxBECount, unsigned BitWidth);
 
+  /// If the unknown expression U corresponds to a simple recurrence, return
+  /// a constant range which represents the entire recurrence.  Note that
+  /// *add* recurrences with loop invariant steps aren't represented by
+  /// SCEVUnknowns and thus don't use this mechanism.
+  ConstantRange getRangeForUnknownRecurrence(const SCEVUnknown *U);
+
   /// We know that there is no SCEV for the specified value.  Analyze the
   /// expression.
   const SCEV *createSCEV(Value *V);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index a7bc5e800716521be7667b7076792863a90eb8e7..ddece01daac15b618b5b92ab392f75b1a4cc5e7f 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -27,6 +27,7 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/AtomicOrdering.h"
+#include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/InstructionCost.h"
 #include "llvm/Support/TypeSize.h"
@@ -329,6 +330,10 @@ public:
     return getUserCost(U, Operands, CostKind);
   }
 
+  /// If a branch or a select condition is skewed in one direction by more than
+  /// this factor, it is very likely to be predicted correctly.
+  BranchProbability getPredictableBranchThreshold() const;
+
   /// Return true if branch divergence exists.
   ///
   /// Branch divergence has a significantly negative impact on GPU performance
@@ -916,8 +921,10 @@ public:
   /// \return the target-provided register class name
   const char *getRegisterClassName(unsigned ClassID) const;
 
+  enum RegisterKind { RGK_Scalar, RGK_FixedWidthVector, RGK_ScalableVector };
+
   /// \return The width of the largest scalar or vector register type.
-  unsigned getRegisterBitWidth(bool Vector) const;
+  TypeSize getRegisterBitWidth(RegisterKind K) const;
 
   /// \return The widest element supported by scalable vectors.
   unsigned getMaxElementWidth() const;
@@ -925,18 +932,13 @@ public:
   /// \return The width of the smallest vector register type.
   unsigned getMinVectorRegisterBitWidth() const;
 
-  /// \return the actual width of the vector register type. For targets with
-  /// variable width vector registers, \param WidthFactor is the factor by which
-  /// the vector register width is the multiple of min vector register width.
-  unsigned getVectorRegisterBitWidth(unsigned WidthFactor) const;
-
-  unsigned getVectorRegisterUsage(unsigned VFKnownMin, unsigned ElementTypeSize,
+  unsigned getVectorRegisterUsage(RegisterKind K, unsigned VFKnownMin,
+                                  unsigned ElementTypeSize,
                                   unsigned SafeDepDist = -1U) const;
 
-  std::pair<ElementCount, ElementCount>
-  getFeasibleMaxVFRange(unsigned SmallestType, unsigned WidestType,
-                        unsigned MaxSafeRegisterWidth = -1U,
-                        unsigned RegWidthFactor = 1) const;
+  std::pair<ElementCount, ElementCount> getFeasibleMaxVFRange(
+      RegisterKind K, unsigned SmallestType, unsigned WidestType,
+      unsigned MaxSafeRegisterWidth = -1U, unsigned RegWidthFactor = 1) const;
 
   /// \return The maximum value of vscale if the target specifies an
   ///  architectural maximum vector length, and None otherwise.
@@ -1056,11 +1058,13 @@ public:
       const Instruction *CxtI = nullptr) const;
 
   /// \return The cost of a shuffle instruction of kind Kind and of type Tp.
+  /// The exact mask may be passed as Mask, or else the array will be empty.
   /// The index and subtype parameters are used by the subvector insertion and
   /// extraction shuffle kinds to show the insert/extract point and the type of
   /// the subvector being inserted/extracted.
   /// NOTE: For subvector extractions Tp represents the source type.
-  int getShuffleCost(ShuffleKind Kind, VectorType *Tp, int Index = 0,
+  int getShuffleCost(ShuffleKind Kind, VectorType *Tp,
+                     ArrayRef<int> Mask = None, int Index = 0,
                      VectorType *SubTp = nullptr) const;
 
   /// Represents a hint about the context in which a cast is used.
@@ -1422,6 +1426,7 @@ public:
                                    BlockFrequencyInfo *BFI) = 0;
   virtual int getUserCost(const User *U, ArrayRef<const Value *> Operands,
                           TargetCostKind CostKind) = 0;
+  virtual BranchProbability getPredictableBranchThreshold() = 0;
   virtual bool hasBranchDivergence() = 0;
   virtual bool useGPUDivergenceAnalysis() = 0;
   virtual bool isSourceOfDivergence(const Value *V) = 0;
@@ -1534,15 +1539,15 @@ public:
   virtual unsigned getRegisterClassForType(bool Vector,
                                            Type *Ty = nullptr) const = 0;
   virtual const char *getRegisterClassName(unsigned ClassID) const = 0;
-  virtual unsigned getRegisterBitWidth(bool Vector) const = 0;
+  virtual TypeSize getRegisterBitWidth(RegisterKind K) const = 0;
   virtual unsigned getMaxElementWidth() const = 0;
   virtual unsigned getMinVectorRegisterBitWidth() = 0;
-  virtual unsigned getVectorRegisterBitWidth(unsigned WidthFactor) const = 0;
-  virtual unsigned getVectorRegisterUsage(unsigned VFKnownMin,
+  virtual unsigned getVectorRegisterUsage(RegisterKind K, unsigned VFKnownMin,
                                           unsigned ElementTypeSize,
                                           unsigned SafeDepDist = -1U) const = 0;
   virtual std::pair<ElementCount, ElementCount>
-  getFeasibleMaxVFRange(unsigned SmallestType, unsigned WidestType,
+  getFeasibleMaxVFRange(RegisterKind K, unsigned SmallestType,
+                        unsigned WidestType,
                         unsigned MaxSafeRegisterWidth = -1U,
                         unsigned RegWidthFactor = 1) const = 0;
   virtual Optional<unsigned> getMaxVScale() const = 0;
@@ -1588,7 +1593,8 @@ public:
       OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo,
       OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args,
       const Instruction *CxtI = nullptr) = 0;
-  virtual int getShuffleCost(ShuffleKind Kind, VectorType *Tp, int Index,
+  virtual int getShuffleCost(ShuffleKind Kind, VectorType *Tp,
+                             ArrayRef<int> Mask, int Index,
                              VectorType *SubTp) = 0;
   virtual int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                CastContextHint CCH,
@@ -1723,6 +1729,9 @@ public:
                   TargetCostKind CostKind) override {
     return Impl.getUserCost(U, Operands, CostKind);
   }
+  BranchProbability getPredictableBranchThreshold() override {
+    return Impl.getPredictableBranchThreshold();
+  }
   bool hasBranchDivergence() override { return Impl.hasBranchDivergence(); }
   bool useGPUDivergenceAnalysis() override {
     return Impl.useGPUDivergenceAnalysis();
@@ -1967,8 +1976,8 @@ public:
   const char *getRegisterClassName(unsigned ClassID) const override {
     return Impl.getRegisterClassName(ClassID);
   }
-  unsigned getRegisterBitWidth(bool Vector) const override {
-    return Impl.getRegisterBitWidth(Vector);
+  TypeSize getRegisterBitWidth(RegisterKind K) const override {
+    return Impl.getRegisterBitWidth(K);
   }
   unsigned getMaxElementWidth() const override {
     return Impl.getMaxElementWidth();
@@ -1976,19 +1985,18 @@ public:
   unsigned getMinVectorRegisterBitWidth() override {
     return Impl.getMinVectorRegisterBitWidth();
   }
-  unsigned getVectorRegisterBitWidth(unsigned WidthFactor) const override {
-    return Impl.getVectorRegisterBitWidth(WidthFactor);
-  }
-  unsigned getVectorRegisterUsage(unsigned VFKnownMin, unsigned ElementTypeSize,
+  unsigned getVectorRegisterUsage(RegisterKind K, unsigned VFKnownMin,
+                                  unsigned ElementTypeSize,
                                   unsigned SafeDepDist = -1U) const override {
-    return Impl.getVectorRegisterUsage(VFKnownMin, ElementTypeSize,
+    return Impl.getVectorRegisterUsage(K, VFKnownMin, ElementTypeSize,
                                        SafeDepDist);
   }
   std::pair<ElementCount, ElementCount>
-  getFeasibleMaxVFRange(unsigned SmallestType, unsigned WidestType,
+  getFeasibleMaxVFRange(RegisterKind K, unsigned SmallestType,
+                        unsigned WidestType,
                         unsigned MaxSafeRegisterWidth = -1U,
                         unsigned RegWidthFactor = 1) const override {
-    return Impl.getFeasibleMaxVFRange(SmallestType, WidestType,
+    return Impl.getFeasibleMaxVFRange(K, SmallestType, WidestType,
                                       MaxSafeRegisterWidth, RegWidthFactor);
   }
   Optional<unsigned> getMaxVScale() const override {
@@ -2066,9 +2074,9 @@ public:
     return Impl.getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, Opd2Info,
                                        Opd1PropInfo, Opd2PropInfo, Args, CxtI);
   }
-  int getShuffleCost(ShuffleKind Kind, VectorType *Tp, int Index,
-                     VectorType *SubTp) override {
-    return Impl.getShuffleCost(Kind, Tp, Index, SubTp);
+  int getShuffleCost(ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask,
+                     int Index, VectorType *SubTp) override {
+    return Impl.getShuffleCost(Kind, Tp, Mask, Index, SubTp);
   }
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                        CastContextHint CCH, TTI::TargetCostKind CostKind,
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 97e921a843c7b2068f228755484f0d585c3a256e..bf90768db83f82b856415837964153505ab145ca 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -79,6 +79,16 @@ public:
     return TTI::TCC_Expensive;
   }
 
+  // Although this default value is arbitrary, it is not random. It is assumed
+  // that a condition that evaluates the same way by a higher percentage than
+  // this is best represented as control flow. Therefore, the default value N
+  // should be set such that the win from N% correct executions is greater than
+  // the loss from (100 - N)% mispredicted executions for the majority of
+  //  intended targets.
+  BranchProbability getPredictableBranchThreshold() const {
+    return BranchProbability(99, 100);
+  }
+
   bool hasBranchDivergence() const { return false; }
 
   bool useGPUDivergenceAnalysis() const { return false; }
@@ -371,7 +381,9 @@ public:
     }
   }
 
-  unsigned getRegisterBitWidth(bool Vector) const { return 32; }
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+    return TypeSize::getFixed(32);
+  }
 
   unsigned getMaxElementWidth() const { return 64; }
 
@@ -379,22 +391,24 @@ public:
 
   Optional<unsigned> getMaxVScale() const { return None; }
 
-  unsigned getVectorRegisterBitWidth(unsigned WidthFactor) const {
-    return getRegisterBitWidth(true);
-  }
-
-  unsigned getVectorRegisterUsage(unsigned VFKnownMin, unsigned ElementTypeSize,
+  unsigned getVectorRegisterUsage(TargetTransformInfo::RegisterKind K,
+                                  unsigned VFKnownMin, unsigned ElementTypeSize,
                                   unsigned SafeDepDist = -1U) const {
-    unsigned WidestRegister = std::min(getRegisterBitWidth(true), SafeDepDist);
+    assert((K == TargetTransformInfo::RGK_FixedWidthVector ||
+            K == TargetTransformInfo::RGK_ScalableVector) &&
+           "This function is for vectors");
+    unsigned WidestRegister =
+        std::min<unsigned>(getRegisterBitWidth(K).getFixedSize(), SafeDepDist);
     return std::max<unsigned>(1, VFKnownMin * ElementTypeSize / WidestRegister);
   }
 
   std::pair<ElementCount, ElementCount>
-  getFeasibleMaxVFRange(unsigned SmallestType, unsigned WidestType,
+  getFeasibleMaxVFRange(TargetTransformInfo::RegisterKind K,
+                        unsigned SmallestType, unsigned WidestType,
                         unsigned MaxSafeRegisterWidth = -1U,
                         unsigned RegWidthFactor = 1) const {
-    unsigned WidestRegister = std::min(
-        getVectorRegisterBitWidth(RegWidthFactor), MaxSafeRegisterWidth);
+    unsigned WidestRegister = std::min<unsigned>(
+        getRegisterBitWidth(K).getFixedSize(), MaxSafeRegisterWidth);
     bool IsScalable = useScalableVectorType();
 
     unsigned LowerBoundVFKnownMin = PowerOf2Floor(WidestRegister / WidestType);
@@ -484,7 +498,8 @@ public:
     return 1;
   }
 
-  unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Ty, int Index,
+  unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Ty,
+                          ArrayRef<int> Mask, int Index,
                           VectorType *SubTp) const {
     return 1;
   }
@@ -1080,25 +1095,30 @@ public:
       int SubIndex;
       if (Shuffle->isExtractSubvectorMask(SubIndex))
         return TargetTTI->getShuffleCost(TTI::SK_ExtractSubvector, VecSrcTy,
-                                         SubIndex, VecTy);
+                                         Shuffle->getShuffleMask(), SubIndex,
+                                         VecTy);
       else if (Shuffle->changesLength())
         return CostKind == TTI::TCK_RecipThroughput ? -1 : 1;
       else if (Shuffle->isIdentity())
         return 0;
       else if (Shuffle->isReverse())
-        return TargetTTI->getShuffleCost(TTI::SK_Reverse, VecTy, 0, nullptr);
+        return TargetTTI->getShuffleCost(TTI::SK_Reverse, VecTy,
+                                         Shuffle->getShuffleMask(), 0, nullptr);
       else if (Shuffle->isSelect())
-        return TargetTTI->getShuffleCost(TTI::SK_Select, VecTy, 0, nullptr);
+        return TargetTTI->getShuffleCost(TTI::SK_Select, VecTy,
+                                         Shuffle->getShuffleMask(), 0, nullptr);
       else if (Shuffle->isTranspose())
-        return TargetTTI->getShuffleCost(TTI::SK_Transpose, VecTy, 0, nullptr);
+        return TargetTTI->getShuffleCost(TTI::SK_Transpose, VecTy,
+                                         Shuffle->getShuffleMask(), 0, nullptr);
       else if (Shuffle->isZeroEltSplat())
-        return TargetTTI->getShuffleCost(TTI::SK_Broadcast, VecTy, 0, nullptr);
+        return TargetTTI->getShuffleCost(TTI::SK_Broadcast, VecTy,
+                                         Shuffle->getShuffleMask(), 0, nullptr);
       else if (Shuffle->isSingleSource())
-        return TargetTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, VecTy, 0,
-                                         nullptr);
+        return TargetTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, VecTy,
+                                         Shuffle->getShuffleMask(), 0, nullptr);
 
-      return TargetTTI->getShuffleCost(TTI::SK_PermuteTwoSrc, VecTy, 0,
-                                       nullptr);
+      return TargetTTI->getShuffleCost(TTI::SK_PermuteTwoSrc, VecTy,
+                                       Shuffle->getShuffleMask(), 0, nullptr);
     }
     case Instruction::ExtractElement: {
       unsigned Idx = -1;
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index e2c5785f6b0528081a7fb403eace3690d9b97ff7..90ec742f18e6765ec04a40c68255deddd20d3b0e 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -461,7 +461,8 @@ constexpr unsigned MaxAnalysisRecursionDepth = 6;
   /// for such instructions, moving them may change the resulting value.
   bool isSafeToSpeculativelyExecute(const Value *V,
                                     const Instruction *CtxI = nullptr,
-                                    const DominatorTree *DT = nullptr);
+                                    const DominatorTree *DT = nullptr,
+                                    const TargetLibraryInfo *TLI = nullptr);
 
   /// Returns true if the result or effects of the given instructions \p I
   /// depend on or influence global memory.
@@ -758,9 +759,20 @@ constexpr unsigned MaxAnalysisRecursionDepth = 6;
   ///   %iv = phi Ty [%Start, %Entry], [%Inc, %backedge]
   ///   %inc = binop %step, %iv
   ///
-  /// WARNING: For non-commutative operators, we will match both forms.  This
-  /// results in some odd recurrence structures.  Callers may wish to filter
-  /// out recurrences where the phi is not the LHS of the returned operator.
+  /// A first order recurrence is a formula with the form: X_n = f(X_(n-1))
+  ///
+  /// A couple of notes on subtleties in that definition:
+  /// * The Step does not have to be loop invariant.  In math terms, it can
+  ///   be a free variable.  We allow recurrences with both constant and
+  ///   variable coefficients. Callers may wish to filter cases where Step
+  ///   does not dominate P.
+  /// * For non-commutative operators, we will match both forms.  This
+  ///   results in some odd recurrence structures.  Callers may wish to filter
+  ///   out recurrences where the phi is not the LHS of the returned operator.
+  /// * Because of the structure matched, the caller can assume as a post
+  ///   condition of the match the presence of a Loop with P's parent as it's
+  ///   header *except* in unreachable code.  (Dominance decays in unreachable
+  ///   code.)
   ///
   /// NOTE: This is intentional simple.  If you want the ability to analyze
   /// non-trivial loop conditons, see ScalarEvolution instead.
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 30209a59ca8996922c42baeb1c1dc27df0c7840c..0118b55524ce21d97a3d91770e50b27b3496f868 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -370,6 +370,7 @@ enum {
   // was never defined for V1.
   ELFABIVERSION_AMDGPU_HSA_V2 = 0,
   ELFABIVERSION_AMDGPU_HSA_V3 = 1,
+  ELFABIVERSION_AMDGPU_HSA_V4 = 2
 };
 
 #define ELF_RELOC(name, value) name = value,
@@ -733,6 +734,8 @@ enum : unsigned {
   EF_AMDGPU_MACH_AMDGCN_RESERVED_0X3D = 0x03d,
   EF_AMDGPU_MACH_AMDGCN_RESERVED_0X3E = 0x03e,
   EF_AMDGPU_MACH_AMDGCN_GFX90A        = 0x03f,
+  EF_AMDGPU_MACH_AMDGCN_RESERVED_0X40 = 0x040,
+  EF_AMDGPU_MACH_AMDGCN_RESERVED_0X41 = 0x041,
 
   // First/last AMDGCN-based processors.
   EF_AMDGPU_MACH_AMDGCN_FIRST = EF_AMDGPU_MACH_AMDGCN_GFX600,
@@ -740,10 +743,51 @@ enum : unsigned {
 
   // Indicates if the "xnack" target feature is enabled for all code contained
   // in the object.
-  EF_AMDGPU_XNACK = 0x100,
-  // Indicates if the "sram-ecc" target feature is enabled for all code
+  //
+  // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V2.
+  EF_AMDGPU_FEATURE_XNACK_V2 = 0x01,
+  // Indicates if the trap handler is enabled for all code contained
+  // in the object.
+  //
+  // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V2.
+  EF_AMDGPU_FEATURE_TRAP_HANDLER_V2 = 0x02,
+
+  // Indicates if the "xnack" target feature is enabled for all code contained
+  // in the object.
+  //
+  // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V3.
+  EF_AMDGPU_FEATURE_XNACK_V3 = 0x100,
+  // Indicates if the "sramecc" target feature is enabled for all code
   // contained in the object.
-  EF_AMDGPU_SRAM_ECC = 0x200,
+  //
+  // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V3.
+  EF_AMDGPU_FEATURE_SRAMECC_V3 = 0x200,
+
+  // XNACK selection mask for EF_AMDGPU_FEATURE_XNACK_* values.
+  //
+  // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V4.
+  EF_AMDGPU_FEATURE_XNACK_V4 = 0x300,
+  // XNACK is not supported.
+  EF_AMDGPU_FEATURE_XNACK_UNSUPPORTED_V4 = 0x000,
+  // XNACK is any/default/unspecified.
+  EF_AMDGPU_FEATURE_XNACK_ANY_V4 = 0x100,
+  // XNACK is off.
+  EF_AMDGPU_FEATURE_XNACK_OFF_V4 = 0x200,
+  // XNACK is on.
+  EF_AMDGPU_FEATURE_XNACK_ON_V4 = 0x300,
+
+  // SRAMECC selection mask for EF_AMDGPU_FEATURE_SRAMECC_* values.
+  //
+  // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V4.
+  EF_AMDGPU_FEATURE_SRAMECC_V4 = 0xc00,
+  // SRAMECC is not supported.
+  EF_AMDGPU_FEATURE_SRAMECC_UNSUPPORTED_V4 = 0x000,
+  // SRAMECC is any/default/unspecified.
+  EF_AMDGPU_FEATURE_SRAMECC_ANY_V4 = 0x400,
+  // SRAMECC is off.
+  EF_AMDGPU_FEATURE_SRAMECC_OFF_V4 = 0x800,
+  // SRAMECC is on.
+  EF_AMDGPU_FEATURE_SRAMECC_ON_V4 = 0xc00,
 };
 
 // ELF Relocation types for AMDGPU
@@ -1470,10 +1514,14 @@ enum : unsigned {
   GNU_PROPERTY_NO_COPY_ON_PROTECTED = 2,
   GNU_PROPERTY_AARCH64_FEATURE_1_AND = 0xc0000000,
   GNU_PROPERTY_X86_FEATURE_1_AND = 0xc0000002,
-  GNU_PROPERTY_X86_ISA_1_NEEDED = 0xc0008000,
-  GNU_PROPERTY_X86_FEATURE_2_NEEDED = 0xc0008001,
-  GNU_PROPERTY_X86_ISA_1_USED = 0xc0010000,
-  GNU_PROPERTY_X86_FEATURE_2_USED = 0xc0010001,
+
+  GNU_PROPERTY_X86_UINT32_OR_LO = 0xc0008000,
+  GNU_PROPERTY_X86_FEATURE_2_NEEDED = GNU_PROPERTY_X86_UINT32_OR_LO + 1,
+  GNU_PROPERTY_X86_ISA_1_NEEDED = GNU_PROPERTY_X86_UINT32_OR_LO + 2,
+
+  GNU_PROPERTY_X86_UINT32_OR_AND_LO = 0xc0010000,
+  GNU_PROPERTY_X86_FEATURE_2_USED = GNU_PROPERTY_X86_UINT32_OR_AND_LO + 1,
+  GNU_PROPERTY_X86_ISA_1_USED = GNU_PROPERTY_X86_UINT32_OR_AND_LO + 2,
 };
 
 // aarch64 processor feature bits.
@@ -1487,31 +1535,6 @@ enum : unsigned {
   GNU_PROPERTY_X86_FEATURE_1_IBT = 1 << 0,
   GNU_PROPERTY_X86_FEATURE_1_SHSTK = 1 << 1,
 
-  GNU_PROPERTY_X86_ISA_1_CMOV = 1 << 0,
-  GNU_PROPERTY_X86_ISA_1_SSE = 1 << 1,
-  GNU_PROPERTY_X86_ISA_1_SSE2 = 1 << 2,
-  GNU_PROPERTY_X86_ISA_1_SSE3 = 1 << 3,
-  GNU_PROPERTY_X86_ISA_1_SSSE3 = 1 << 4,
-  GNU_PROPERTY_X86_ISA_1_SSE4_1 = 1 << 5,
-  GNU_PROPERTY_X86_ISA_1_SSE4_2 = 1 << 6,
-  GNU_PROPERTY_X86_ISA_1_AVX = 1 << 7,
-  GNU_PROPERTY_X86_ISA_1_AVX2 = 1 << 8,
-  GNU_PROPERTY_X86_ISA_1_FMA = 1 << 9,
-  GNU_PROPERTY_X86_ISA_1_AVX512F = 1 << 10,
-  GNU_PROPERTY_X86_ISA_1_AVX512CD = 1 << 11,
-  GNU_PROPERTY_X86_ISA_1_AVX512ER = 1 << 12,
-  GNU_PROPERTY_X86_ISA_1_AVX512PF = 1 << 13,
-  GNU_PROPERTY_X86_ISA_1_AVX512VL = 1 << 14,
-  GNU_PROPERTY_X86_ISA_1_AVX512DQ = 1 << 15,
-  GNU_PROPERTY_X86_ISA_1_AVX512BW = 1 << 16,
-  GNU_PROPERTY_X86_ISA_1_AVX512_4FMAPS = 1 << 17,
-  GNU_PROPERTY_X86_ISA_1_AVX512_4VNNIW = 1 << 18,
-  GNU_PROPERTY_X86_ISA_1_AVX512_BITALG = 1 << 19,
-  GNU_PROPERTY_X86_ISA_1_AVX512_IFMA = 1 << 20,
-  GNU_PROPERTY_X86_ISA_1_AVX512_VBMI = 1 << 21,
-  GNU_PROPERTY_X86_ISA_1_AVX512_VBMI2 = 1 << 22,
-  GNU_PROPERTY_X86_ISA_1_AVX512_VNNI = 1 << 23,
-
   GNU_PROPERTY_X86_FEATURE_2_X86 = 1 << 0,
   GNU_PROPERTY_X86_FEATURE_2_X87 = 1 << 1,
   GNU_PROPERTY_X86_FEATURE_2_MMX = 1 << 2,
@@ -1522,6 +1545,11 @@ enum : unsigned {
   GNU_PROPERTY_X86_FEATURE_2_XSAVE = 1 << 7,
   GNU_PROPERTY_X86_FEATURE_2_XSAVEOPT = 1 << 8,
   GNU_PROPERTY_X86_FEATURE_2_XSAVEC = 1 << 9,
+
+  GNU_PROPERTY_X86_ISA_1_BASELINE = 1 << 0,
+  GNU_PROPERTY_X86_ISA_1_V2 = 1 << 1,
+  GNU_PROPERTY_X86_ISA_1_V3 = 1 << 2,
+  GNU_PROPERTY_X86_ISA_1_V4 = 1 << 3,
 };
 
 // FreeBSD note types.
@@ -1561,15 +1589,18 @@ enum {
   SHN_AMDGPU_LDS = 0xff00, // Variable in LDS; symbol encoded like SHN_COMMON
 };
 
-// AMD specific notes. (Code Object V2)
+// AMD vendor specific notes. (Code Object V2)
 enum {
-  // Note types with values between 0 and 9 (inclusive) are reserved.
-  NT_AMD_AMDGPU_HSA_METADATA = 10,
-  NT_AMD_AMDGPU_ISA = 11,
-  NT_AMD_AMDGPU_PAL_METADATA = 12
+  NT_AMD_HSA_CODE_OBJECT_VERSION = 1,
+  NT_AMD_HSA_HSAIL = 2,
+  NT_AMD_HSA_ISA_VERSION = 3,
+  // Note types with values between 4 and 9 (inclusive) are reserved.
+  NT_AMD_HSA_METADATA = 10,
+  NT_AMD_HSA_ISA_NAME = 11,
+  NT_AMD_PAL_METADATA = 12
 };
 
-// AMDGPU specific notes. (Code Object V3)
+// AMDGPU vendor specific notes. (Code Object V3)
 enum {
   // Note types with values between 0 and 31 (inclusive) are reserved.
   NT_AMDGPU_METADATA = 32
diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h
index 9c3f2569addc91dfd56c08aa5826760e2a351ef3..34be93b9112f0d3603e17f6b59a53f13322fba02 100644
--- a/llvm/include/llvm/BinaryFormat/Wasm.h
+++ b/llvm/include/llvm/BinaryFormat/Wasm.h
@@ -63,7 +63,7 @@ struct WasmExport {
 
 struct WasmLimits {
   uint8_t Flags;
-  uint64_t Initial;
+  uint64_t Minimum;
   uint64_t Maximum;
 };
 
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 88a629493f04bb7c088297ece7fc81cbd9840c32..bc1c5d4479f46447e1e82158d445fa9ba4d21bfb 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -659,6 +659,7 @@ enum AttributeKindCodes {
   ATTR_KIND_NO_CALLBACK = 71,
   ATTR_KIND_HOT = 72,
   ATTR_KIND_NO_PROFILE = 73,
+  ATTR_KIND_VSCALE_RANGE = 74,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index dfad5f6560e877884a0ac2866d0a139ecd332006..242cff73db504f64a55a31a9eb140335e0aab9b7 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -571,24 +571,26 @@ public:
   /// \name Vector TTI Implementations
   /// @{
 
-  unsigned getRegisterBitWidth(bool Vector) const { return 32; }
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+    return TypeSize::getFixed(32);
+  }
 
-  unsigned getVectorRegisterUsage(unsigned VFKnownMin, unsigned ElementTypeSize,
+  unsigned getVectorRegisterUsage(TargetTransformInfo::RegisterKind K,
+                                  unsigned VFKnownMin, unsigned ElementTypeSize,
                                   unsigned SafeDepDist) const {
-    unsigned WidestRegister = std::min(
-        static_cast<const T *>(this)->getRegisterBitWidth(true), SafeDepDist);
+    unsigned WidestRegister = std::min<unsigned>(
+        static_cast<const T *>(this)->getRegisterBitWidth(K).getFixedSize(),
+        SafeDepDist);
     return std::max<unsigned>(1, VFKnownMin * ElementTypeSize / WidestRegister);
   }
 
-  unsigned getVectorRegisterBitWidth(unsigned WidthFactor) const {
-    return static_cast<const T *>(this)->getRegisterBitWidth(true);
-  }
-
   std::pair<ElementCount, ElementCount>
-  getFeasibleMaxVFRange(unsigned SmallestType, unsigned WidestType,
+  getFeasibleMaxVFRange(TargetTransformInfo::RegisterKind K,
+                        unsigned SmallestType, unsigned WidestType,
                         unsigned MaxSafeRegisterWidth = -1U,
                         unsigned RegWidthFactor = 1) const {
-    unsigned WidestRegister = getVectorRegisterBitWidth(RegWidthFactor);
+    unsigned WidestRegister =
+        static_cast<const T *>(this)->getRegisterBitWidth(K).getFixedSize();
     WidestRegister = std::min(WidestRegister, MaxSafeRegisterWidth);
     bool IsScalable = static_cast<const T *>(this)->useScalableVectorType();
 
@@ -749,8 +751,8 @@ public:
     return OpCost;
   }
 
-  unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
-                          VectorType *SubTp) {
+  unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
+                          ArrayRef<int> Mask, int Index, VectorType *SubTp) {
 
     switch (Kind) {
     case TTI::SK_Broadcast:
@@ -1283,6 +1285,12 @@ public:
       return thisT()->getGatherScatterOpCost(Instruction::Load, RetTy, Args[0],
                                              VarMask, Alignment, CostKind, I);
     }
+    case Intrinsic::experimental_stepvector: {
+      if (isa<ScalableVectorType>(RetTy))
+        return BaseT::getIntrinsicInstrCost(ICA, CostKind);
+      // The cost of materialising a constant integer vector.
+      return TargetTransformInfo::TCC_Basic;
+    }
     case Intrinsic::experimental_vector_extract: {
       // FIXME: Handle case where a scalable vector is extracted from a scalable
       // vector
@@ -1290,7 +1298,7 @@ public:
         return BaseT::getIntrinsicInstrCost(ICA, CostKind);
       unsigned Index = cast<ConstantInt>(Args[1])->getZExtValue();
       return thisT()->getShuffleCost(TTI::SK_ExtractSubvector,
-                                     cast<VectorType>(Args[0]->getType()),
+                                     cast<VectorType>(Args[0]->getType()), None,
                                      Index, cast<VectorType>(RetTy));
     }
     case Intrinsic::experimental_vector_insert: {
@@ -1300,13 +1308,13 @@ public:
         return BaseT::getIntrinsicInstrCost(ICA, CostKind);
       unsigned Index = cast<ConstantInt>(Args[2])->getZExtValue();
       return thisT()->getShuffleCost(
-          TTI::SK_InsertSubvector, cast<VectorType>(Args[0]->getType()), Index,
-          cast<VectorType>(Args[1]->getType()));
+          TTI::SK_InsertSubvector, cast<VectorType>(Args[0]->getType()), None,
+          Index, cast<VectorType>(Args[1]->getType()));
     }
     case Intrinsic::experimental_vector_reverse: {
       return thisT()->getShuffleCost(TTI::SK_Reverse,
-                                     cast<VectorType>(Args[0]->getType()), 0,
-                                     cast<VectorType>(RetTy));
+                                     cast<VectorType>(Args[0]->getType()), None,
+                                     0, cast<VectorType>(RetTy));
     }
     case Intrinsic::vector_reduce_add:
     case Intrinsic::vector_reduce_mul:
@@ -1930,6 +1938,22 @@ public:
     Type *ScalarTy = Ty->getElementType();
     unsigned NumVecElts =
         cast<VectorType>(Ty)->getElementCount().getKnownMinValue();
+    if ((Opcode == Instruction::Or || Opcode == Instruction::And) &&
+        ScalarTy == IntegerType::getInt1Ty(Ty->getContext()) &&
+        NumVecElts >= 2) {
+      // Or reduction for i1 is represented as:
+      // %val = bitcast <ReduxWidth x i1> to iReduxWidth
+      // %res = cmp ne iReduxWidth %val, 0
+      // And reduction for i1 is represented as:
+      // %val = bitcast <ReduxWidth x i1> to iReduxWidth
+      // %res = cmp eq iReduxWidth %val, 11111
+      Type *ValTy = IntegerType::get(Ty->getContext(), NumVecElts);
+      return thisT()->getCastInstrCost(Instruction::BitCast, ValTy, Ty,
+                                       TTI::CastContextHint::None, CostKind) +
+             thisT()->getCmpSelInstrCost(Instruction::ICmp, ValTy,
+                                         CmpInst::makeCmpResultType(ValTy),
+                                         CmpInst::BAD_ICMP_PREDICATE, CostKind);
+    }
     unsigned NumReduxLevels = Log2_32(NumVecElts);
     unsigned ArithCost = 0;
     unsigned ShuffleCost = 0;
@@ -1942,9 +1966,9 @@ public:
       NumVecElts /= 2;
       VectorType *SubTy = FixedVectorType::get(ScalarTy, NumVecElts);
       // Assume the pairwise shuffles add a cost.
-      ShuffleCost +=
-          (IsPairwise + 1) * thisT()->getShuffleCost(TTI::SK_ExtractSubvector,
-                                                     Ty, NumVecElts, SubTy);
+      ShuffleCost += (IsPairwise + 1) *
+                     thisT()->getShuffleCost(TTI::SK_ExtractSubvector, Ty, None,
+                                             NumVecElts, SubTy);
       ArithCost += thisT()->getArithmeticInstrCost(Opcode, SubTy, CostKind);
       Ty = SubTy;
       ++LongVectorCount;
@@ -1963,8 +1987,8 @@ public:
     unsigned NumShuffles = NumReduxLevels;
     if (IsPairwise && NumReduxLevels >= 1)
       NumShuffles += NumReduxLevels - 1;
-    ShuffleCost += NumShuffles *
-                   thisT()->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, 0, Ty);
+    ShuffleCost += NumShuffles * thisT()->getShuffleCost(
+                                     TTI::SK_PermuteSingleSrc, Ty, None, 0, Ty);
     ArithCost += NumReduxLevels * thisT()->getArithmeticInstrCost(Opcode, Ty);
     return ShuffleCost + ArithCost +
            thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty, 0);
@@ -2001,9 +2025,9 @@ public:
       CondTy = FixedVectorType::get(ScalarCondTy, NumVecElts);
 
       // Assume the pairwise shuffles add a cost.
-      ShuffleCost +=
-          (IsPairwise + 1) * thisT()->getShuffleCost(TTI::SK_ExtractSubvector,
-                                                     Ty, NumVecElts, SubTy);
+      ShuffleCost += (IsPairwise + 1) *
+                     thisT()->getShuffleCost(TTI::SK_ExtractSubvector, Ty, None,
+                                             NumVecElts, SubTy);
       MinMaxCost +=
           thisT()->getCmpSelInstrCost(CmpOpcode, SubTy, CondTy,
                                       CmpInst::BAD_ICMP_PREDICATE, CostKind) +
@@ -2026,8 +2050,8 @@ public:
     unsigned NumShuffles = NumReduxLevels;
     if (IsPairwise && NumReduxLevels >= 1)
       NumShuffles += NumReduxLevels - 1;
-    ShuffleCost += NumShuffles *
-                   thisT()->getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, 0, Ty);
+    ShuffleCost += NumShuffles * thisT()->getShuffleCost(
+                                     TTI::SK_PermuteSingleSrc, Ty, None, 0, Ty);
     MinMaxCost +=
         NumReduxLevels *
         (thisT()->getCmpSelInstrCost(CmpOpcode, Ty, CondTy,
diff --git a/llvm/include/llvm/CodeGen/CalcSpillWeights.h b/llvm/include/llvm/CodeGen/CalcSpillWeights.h
index 78dae81f596e1ab01252d6de6df8cd1bfe09688f..0b6ed079b38e6421a832ad6ca4cfd463fe144325 100644
--- a/llvm/include/llvm/CodeGen/CalcSpillWeights.h
+++ b/llvm/include/llvm/CodeGen/CalcSpillWeights.h
@@ -50,6 +50,10 @@ class VirtRegMap;
     const MachineLoopInfo &Loops;
     const MachineBlockFrequencyInfo &MBFI;
 
+    /// Returns true if Reg of live interval LI is used in instruction with many
+    /// operands like STATEPOINT.
+    bool isLiveAtStatepointVarArg(LiveInterval &LI);
+
   public:
     VirtRegAuxInfo(MachineFunction &MF, LiveIntervals &LIS,
                    const VirtRegMap &VRM, const MachineLoopInfo &Loops,
diff --git a/llvm/include/llvm/CodeGen/EdgeBundles.h b/llvm/include/llvm/CodeGen/EdgeBundles.h
index b26956023971275ba67a24833161182714fd37e5..b6187fc6dcef59fb68327c6a1ebe5ea6d54147a8 100644
--- a/llvm/include/llvm/CodeGen/EdgeBundles.h
+++ b/llvm/include/llvm/CodeGen/EdgeBundles.h
@@ -8,7 +8,7 @@
 //
 // The EdgeBundles analysis forms equivalence classes of CFG edges such that all
 // edges leaving a machine basic block are in the same bundle, and all edges
-// leaving a basic block are in the same bundle.
+// entering a machine basic block are in the same bundle.
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 0d587c14a39352676fb7f5f4616aa22e4fa28aef..868980d24fc2f41985580e678dd48c89d2af4a77 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -17,11 +17,13 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/CallingConvLower.h"
+#include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MachineValueType.h"
 #include <cstdint>
@@ -37,7 +39,6 @@ class MachineIRBuilder;
 struct MachinePointerInfo;
 class MachineRegisterInfo;
 class TargetLowering;
-class Value;
 
 class CallLowering {
   const TargetLowering *TLI;
@@ -64,10 +65,17 @@ public:
     // if the argument was an incoming arg.
     SmallVector<Register, 2> OrigRegs;
 
+    /// Optionally track the original IR value for the argument. This may not be
+    /// meaningful in all contexts. This should only be used on for forwarding
+    /// through to use for aliasing information in MachinePointerInfo for memory
+    /// arguments.
+    const Value *OrigValue = nullptr;
+
     ArgInfo(ArrayRef<Register> Regs, Type *Ty,
             ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(),
-            bool IsFixed = true)
-        : BaseArgInfo(Ty, Flags, IsFixed), Regs(Regs.begin(), Regs.end()) {
+            bool IsFixed = true, const Value *OrigValue = nullptr)
+        : BaseArgInfo(Ty, Flags, IsFixed), Regs(Regs.begin(), Regs.end()),
+          OrigValue(OrigValue) {
       if (!Regs.empty() && Flags.empty())
         this->Flags.push_back(ISD::ArgFlagsTy());
       // FIXME: We should have just one way of saying "no register".
@@ -76,6 +84,11 @@ public:
              "only void types should have no register");
     }
 
+    ArgInfo(ArrayRef<Register> Regs, const Value &OrigValue,
+            ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(),
+            bool IsFixed = true)
+        : ArgInfo(Regs, OrigValue.getType(), Flags, IsFixed, &OrigValue) {}
+
     ArgInfo() : BaseArgInfo() {}
   };
 
@@ -186,6 +199,14 @@ public:
       llvm_unreachable("Custom values not supported");
     }
 
+    /// Do a memory copy of \p MemSize bytes from \p SrcPtr to \p DstPtr. This
+    /// is necessary for outgoing stack-passed byval arguments.
+    void
+    copyArgumentMemory(const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
+                       const MachinePointerInfo &DstPtrInfo, Align DstAlign,
+                       const MachinePointerInfo &SrcPtrInfo, Align SrcAlign,
+                       uint64_t MemSize, CCValAssign &VA) const;
+
     /// Extend a register to the location type given in VA, capped at extending
     /// to at most MaxSize bits. If MaxSizeBits is 0 then no maximum is set.
     Register extendRegister(Register ValReg, CCValAssign &VA,
@@ -407,7 +428,9 @@ public:
     return false;
   }
 
-  virtual bool fallBackToDAGISel(const Function &F) const { return false; }
+  virtual bool fallBackToDAGISel(const MachineFunction &MF) const {
+    return false;
+  }
 
   /// This hook must be implemented to lower the incoming (formal)
   /// arguments, described by \p VRegs, for GlobalISel. Each argument
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index d8e79d573ebc31f951615904f2447cb81f0764f0..9d09e8a9015f52f71a3cffce438d0e8680cc145e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -174,6 +174,8 @@ private:
                                            LLT WideTy);
   LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
                                          LLT WideTy);
+  LegalizeResult widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
+                                 LLT WideTy);
 
   /// Helper function to split a wide generic register into bitwise blocks with
   /// the given Type (which implies the number of blocks needed). The generic
@@ -295,6 +297,9 @@ public:
                                                            unsigned TypeIdx,
                                                            LLT NarrowTy);
 
+  LegalizeResult fewerElementsVectorMulo(MachineInstr &MI, unsigned TypeIdx,
+                                         LLT NarrowTy);
+
   LegalizeResult
   reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);
 
@@ -340,6 +345,9 @@ public:
   LegalizeResult lowerLoad(MachineInstr &MI);
   LegalizeResult lowerStore(MachineInstr &MI);
   LegalizeResult lowerBitCount(MachineInstr &MI);
+  LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI);
+  LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI);
+  LegalizeResult lowerFunnelShift(MachineInstr &MI);
 
   LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI);
   LegalizeResult lowerUITOFP(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 6c64cd5cb2082b3a72d892f33066b4ee376ca8e0..ccc44b46a3752f150f878c1e25036159ed553756 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -712,6 +712,12 @@ public:
   MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res,
                                       const SrcOp &Op);
 
+  /// Build and inserts \p Res = \p G_AND \p Op, \p LowBitsSet(ImmOp)
+  /// Since there is no G_ZEXT_INREG like G_SEXT_INREG, the instruction is
+  /// emulated using G_AND.
+  MachineInstrBuilder buildZExtInReg(const DstOp &Res, const SrcOp &Op,
+                                     int64_t ImmOp);
+
   /// Build and insert an appropriate cast between two registers of equal size.
   MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src);
 
@@ -1438,6 +1444,13 @@ public:
     return buildInstr(TargetOpcode::G_SMULH, {Dst}, {Src0, Src1}, Flags);
   }
 
+  /// Build and insert \p Res = G_UREM \p Op0, \p Op1
+  MachineInstrBuilder buildURem(const DstOp &Dst, const SrcOp &Src0,
+                                const SrcOp &Src1,
+                                Optional<unsigned> Flags = None) {
+    return buildInstr(TargetOpcode::G_UREM, {Dst}, {Src0, Src1}, Flags);
+  }
+
   MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0,
                                 const SrcOp &Src1,
                                 Optional<unsigned> Flags = None) {
@@ -1810,6 +1823,51 @@ public:
   MachineInstrBuilder buildVecReduceUMin(const DstOp &Dst, const SrcOp &Src) {
     return buildInstr(TargetOpcode::G_VECREDUCE_UMIN, {Dst}, {Src});
   }
+
+  /// Build and insert G_MEMCPY or G_MEMMOVE
+  MachineInstrBuilder buildMemTransferInst(unsigned Opcode, const SrcOp &DstPtr,
+                                           const SrcOp &SrcPtr,
+                                           const SrcOp &Size,
+                                           MachineMemOperand &DstMMO,
+                                           MachineMemOperand &SrcMMO) {
+    auto MIB = buildInstr(
+        Opcode, {}, {DstPtr, SrcPtr, Size, SrcOp(INT64_C(0) /*isTailCall*/)});
+    MIB.addMemOperand(&DstMMO);
+    MIB.addMemOperand(&SrcMMO);
+    return MIB;
+  }
+
+  MachineInstrBuilder buildMemCpy(const SrcOp &DstPtr, const SrcOp &SrcPtr,
+                                  const SrcOp &Size, MachineMemOperand &DstMMO,
+                                  MachineMemOperand &SrcMMO) {
+    return buildMemTransferInst(TargetOpcode::G_MEMCPY, DstPtr, SrcPtr, Size,
+                                DstMMO, SrcMMO);
+  }
+
+  /// Build and insert \p Dst = G_SBFX \p Src, \p LSB, \p Width.
+  MachineInstrBuilder buildSbfx(const DstOp &Dst, const SrcOp &Src,
+                                const SrcOp &LSB, const SrcOp &Width) {
+    return buildInstr(TargetOpcode::G_SBFX, {Dst}, {Src, LSB, Width});
+  }
+
+  /// Build and insert \p Dst = G_UBFX \p Src, \p LSB, \p Width.
+  MachineInstrBuilder buildUbfx(const DstOp &Dst, const SrcOp &Src,
+                                const SrcOp &LSB, const SrcOp &Width) {
+    return buildInstr(TargetOpcode::G_UBFX, {Dst}, {Src, LSB, Width});
+  }
+
+  /// Build and insert \p Dst = G_ROTR \p Src, \p Amt
+  MachineInstrBuilder buildRotateRight(const DstOp &Dst, const SrcOp &Src,
+                                       const SrcOp &Amt) {
+    return buildInstr(TargetOpcode::G_ROTR, {Dst}, {Src, Amt});
+  }
+
+  /// Build and insert \p Dst = G_ROTL \p Src, \p Amt
+  MachineInstrBuilder buildRotateLeft(const DstOp &Dst, const SrcOp &Src,
+                                      const SrcOp &Amt) {
+    return buildInstr(TargetOpcode::G_ROTL, {Dst}, {Src, Amt});
+  }
+
   virtual MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
                                          ArrayRef<SrcOp> SrcOps,
                                          Optional<unsigned> Flags = None);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 17826cc8d6cb2e82d9fe26c0ee45a041fae83331..ddf78356615d0c880b669c420686500d89f37e95 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -204,6 +204,9 @@ void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU);
 Optional<APInt> ConstantFoldBinOp(unsigned Opcode, const Register Op1,
                                   const Register Op2,
                                   const MachineRegisterInfo &MRI);
+Optional<APFloat> ConstantFoldFPBinOp(unsigned Opcode, const Register Op1,
+                                      const Register Op2,
+                                      const MachineRegisterInfo &MRI);
 
 Optional<APInt> ConstantFoldExtOp(unsigned Opcode, const Register Op1,
                                   uint64_t Imm, const MachineRegisterInfo &MRI);
@@ -325,6 +328,13 @@ bool isBuildVectorAllOnes(const MachineInstr &MI,
 Optional<RegOrConstant> getVectorSplat(const MachineInstr &MI,
                                        const MachineRegisterInfo &MRI);
 
+/// Attempt to match a unary predicate against a scalar/splat constant or every
+/// element of a constant G_BUILD_VECTOR. If \p ConstVal is null, the source
+/// value was undef.
+bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg,
+                         std::function<bool(const Constant *ConstVal)> Match,
+                         bool AllowUndefs = false);
+
 /// Returns true if given the TargetLowering's boolean contents information,
 /// the value \p Val contains a true value.
 bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector,
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 9d6c45f4e5c3e6a9b92d218dd25f2aa0d92e7e0f..a6337260f0f4573bd23bdece595747588ddd6be4 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -592,6 +592,14 @@ enum NodeType {
   /// scalars should have the same type.
   SPLAT_VECTOR_PARTS,
 
+  /// STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised
+  /// of a linear sequence of unsigned values starting from 0 with a step of
+  /// IMM, where IMM must be a constant positive integer value. The operation
+  /// does not support returning fixed-width vectors or non-constant operands.
+  /// If the sequence value exceeds the limit allowed for the element type then
+  /// the values for those lanes are undefined.
+  STEP_VECTOR,
+
   /// MULHU/MULHS - Multiply high - Multiply two integers of type iN,
   /// producing an unsigned/signed value of type i[2*N], then return the top
   /// part.
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index 4a7406473b11783cf4a38e7971b1bbc0f71005ec..4f3ac1db496b64cc03e2f8dd2a8e368829d627af 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -564,6 +564,7 @@ struct MachineFrameInfo {
   bool HasOpaqueSPAdjustment = false;
   bool HasVAStart = false;
   bool HasMustTailInVarArgFunc = false;
+  bool HasTailCall = false;
   unsigned LocalFrameSize = 0;
   StringValue SavePoint;
   StringValue RestorePoint;
@@ -584,6 +585,7 @@ struct MachineFrameInfo {
            HasOpaqueSPAdjustment == Other.HasOpaqueSPAdjustment &&
            HasVAStart == Other.HasVAStart &&
            HasMustTailInVarArgFunc == Other.HasMustTailInVarArgFunc &&
+           HasTailCall == Other.HasTailCall &&
            LocalFrameSize == Other.LocalFrameSize &&
            SavePoint == Other.SavePoint && RestorePoint == Other.RestorePoint;
   }
@@ -610,6 +612,7 @@ template <> struct MappingTraits<MachineFrameInfo> {
     YamlIO.mapOptional("hasVAStart", MFI.HasVAStart, false);
     YamlIO.mapOptional("hasMustTailInVarArgFunc", MFI.HasMustTailInVarArgFunc,
                        false);
+    YamlIO.mapOptional("hasTailCall", MFI.HasTailCall, false);
     YamlIO.mapOptional("localFrameSize", MFI.LocalFrameSize, (unsigned)0);
     YamlIO.mapOptional("savePoint", MFI.SavePoint,
                        StringValue()); // Don't print it out when it's empty.
diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
index 3a9d9efde94274b3f5ded9b96a87b1fb746d72dd..f7f9ee4cdac73901c837a57a32e9cdff781bb8bc 100644
--- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
@@ -626,7 +626,7 @@ public:
 
   /// Returns true if the function contains a tail call.
   bool hasTailCall() const { return HasTailCall; }
-  void setHasTailCall() { HasTailCall = true; }
+  void setHasTailCall(bool V = true) { HasTailCall = V; }
 
   /// Computes the maximum size of a callframe and the AdjustsStack property.
   /// This only works for targets defining
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 57e976f9d265eed33ee9c612b8ba060d1ed1dad6..7948bd3fbfdfc6fd9aea48710c83129afaf6dcc1 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -833,6 +833,10 @@ public:
     return getNode(ISD::SPLAT_VECTOR, DL, VT, Op);
   }
 
+  /// Returns a vector of type ResVT whose elements contain the linear sequence
+  ///   <0, Step, Step * 2, Step * 3, ...>
+  SDValue getStepVector(const SDLoc &DL, EVT ResVT, SDValue Step);
+
   /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to
   /// the shuffle node in input but with swapped operands.
   ///
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 4a8dd95837df37574c569a30c9035af546e24f0d..731fb9968c4d199ba9589d348cd53b3058b4450b 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1684,6 +1684,11 @@ bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
 /// Does not permit build vector implicit truncation.
 bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
 
+/// Return true if \p V is either a integer or FP constant.
+inline bool isIntOrFPConstant(SDValue V) {
+  return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
+}
+
 class GlobalAddressSDNode : public SDNode {
   friend class SelectionDAG;
 
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 0d2453a778a45db999a9070ecd76157ae18b1508..793acb6263d210794fa877e98770475f97222f92 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -528,10 +528,6 @@ public:
     return false;
   }
 
-  /// If a branch or a select condition is skewed in one direction by more than
-  /// this factor, it is very likely to be predicted correctly.
-  virtual BranchProbability getPredictableBranchThreshold() const;
-
   /// Return true if the following transform is beneficial:
   /// fold (conv (load x)) -> (load (conv*)x)
   /// On architectures that don't natively support some vector loads
@@ -2458,6 +2454,8 @@ public:
     case ISD::UDIV:
     case ISD::SREM:
     case ISD::UREM:
+    case ISD::SSUBSAT:
+    case ISD::USUBSAT:
     case ISD::FSUB:
     case ISD::FDIV:
     case ISD::FREM:
diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td
index 1ac9e47cab69a51b7c32395fc56afa8e99ac4d8a..775ce448226a04768144efc616c9ff728225c4d0 100644
--- a/llvm/include/llvm/CodeGen/ValueTypes.td
+++ b/llvm/include/llvm/CodeGen/ValueTypes.td
@@ -18,121 +18,121 @@ class ValueType<int size, int value> {
   int Value = value;
 }
 
-def OtherVT: ValueType<0  ,  1>;   // "Other" value
-def i1     : ValueType<1  ,  2>;   // One bit boolean value
-def i8     : ValueType<8  ,  3>;   // 8-bit integer value
-def i16    : ValueType<16 ,  4>;   // 16-bit integer value
-def i32    : ValueType<32 ,  5>;   // 32-bit integer value
-def i64    : ValueType<64 ,  6>;   // 64-bit integer value
-def i128   : ValueType<128,  7>;   // 128-bit integer value
-
-def bf16   : ValueType<16 ,  8>;   // 16-bit brain floating point value
-def f16    : ValueType<16 ,  9>;   // 16-bit floating point value
-def f32    : ValueType<32 , 10>;   // 32-bit floating point value
-def f64    : ValueType<64 , 11>;   // 64-bit floating point value
-def f80    : ValueType<80 , 12>;   // 80-bit floating point value
-def f128   : ValueType<128, 13>;   // 128-bit floating point value
-def ppcf128: ValueType<128, 14>;   // PPC 128-bit floating point value
-
-def v1i1   : ValueType<1 ,  15>;   //   1 x i1 vector value
-def v2i1   : ValueType<2 ,  16>;   //   2 x i1 vector value
-def v4i1   : ValueType<4 ,  17>;   //   4 x i1 vector value
-def v8i1   : ValueType<8 ,  18>;   //   8 x i1 vector value
-def v16i1  : ValueType<16,  19>;   //  16 x i1 vector value
-def v32i1  : ValueType<32 , 20>;   //  32 x i1 vector value
-def v64i1  : ValueType<64 , 21>;   //  64 x i1 vector value
-def v128i1 : ValueType<128, 22>;   // 128 x i1 vector value
-def v256i1 : ValueType<256, 23>;   // 256 x i1 vector value
-def v512i1 : ValueType<512, 24>;   // 512 x i1 vector value
-def v1024i1: ValueType<1024,25>;   //1024 x i1 vector value
-
-def v1i8   : ValueType<8,   26>;   //  1 x i8  vector value
-def v2i8   : ValueType<16 , 27>;   //  2 x i8  vector value
-def v4i8   : ValueType<32 , 28>;   //  4 x i8  vector value
-def v8i8   : ValueType<64 , 29>;   //  8 x i8  vector value
-def v16i8  : ValueType<128, 30>;   // 16 x i8  vector value
-def v32i8  : ValueType<256, 31>;   // 32 x i8  vector value
-def v64i8  : ValueType<512, 32>;   // 64 x i8  vector value
-def v128i8 : ValueType<1024,33>;   //128 x i8  vector value
-def v256i8 : ValueType<2048,34>;   //256 x i8  vector value
-
-def v1i16  : ValueType<16 , 35>;   //  1 x i16 vector value
-def v2i16  : ValueType<32 , 36>;   //  2 x i16 vector value
-def v3i16  : ValueType<48 , 37>;   //  3 x i16 vector value
-def v4i16  : ValueType<64 , 38>;   //  4 x i16 vector value
-def v8i16  : ValueType<128, 39>;   //  8 x i16 vector value
-def v16i16 : ValueType<256, 40>;   // 16 x i16 vector value
-def v32i16 : ValueType<512, 41>;   // 32 x i16 vector value
-def v64i16 : ValueType<1024,42>;   // 64 x i16 vector value
-def v128i16: ValueType<2048,43>;   //128 x i16 vector value
-
-def v1i32    : ValueType<32 , 44>;   //  1 x i32 vector value
-def v2i32    : ValueType<64 , 45>;   //  2 x i32 vector value
-def v3i32    : ValueType<96 , 46>;   //  3 x i32 vector value
-def v4i32    : ValueType<128, 47>;   //  4 x i32 vector value
-def v5i32    : ValueType<160, 48>;   //  5 x i32 vector value
-def v8i32    : ValueType<256, 49>;   //  8 x i32 vector value
-def v16i32   : ValueType<512, 50>;   // 16 x i32 vector value
-def v32i32   : ValueType<1024,51>;   // 32 x i32 vector value
-def v64i32   : ValueType<2048,52>;   // 64 x i32 vector value
-def v128i32  : ValueType<4096,53>;   // 128 x i32 vector value
-def v256i32  : ValueType<8182,54>;   // 256 x i32 vector value
-def v512i32  : ValueType<16384,55>;  // 512 x i32 vector value
-def v1024i32 : ValueType<32768,56>;  // 1024 x i32 vector value
-def v2048i32 : ValueType<65536,57>;  // 2048 x i32 vector value
-
-def v1i64  : ValueType<64 , 58>;   //  1 x i64 vector value
-def v2i64  : ValueType<128, 59>;   //  2 x i64 vector value
-def v4i64  : ValueType<256, 60>;   //  4 x i64 vector value
-def v8i64  : ValueType<512, 61>;   //  8 x i64 vector value
-def v16i64 : ValueType<1024,62>;   // 16 x i64 vector value
-def v32i64 : ValueType<2048,63>;   // 32 x i64 vector value
-def v64i64 : ValueType<4096,64>;   // 64 x i64 vector value
-def v128i64: ValueType<8192,65>;   // 128 x i64 vector value
-def v256i64: ValueType<16384,66>;  // 256 x i64 vector value
-
-def v1i128 : ValueType<128, 67>;   //  1 x i128 vector value
-
-def v1f16    : ValueType<16 , 68>;    //    1 x f16 vector value
-def v2f16    : ValueType<32 , 69>;    //    2 x f16 vector value
-def v3f16    : ValueType<48 , 70>;    //    3 x f16 vector value
-def v4f16    : ValueType<64 , 71>;    //    4 x f16 vector value
-def v8f16    : ValueType<128, 72>;    //    8 x f16 vector value
-def v16f16   : ValueType<256, 73>;    //   16 x f16 vector value
-def v32f16   : ValueType<512, 74>;    //   32 x f16 vector value
-def v64f16   : ValueType<1024, 75>;   //   64 x f16 vector value
-def v128f16  : ValueType<2048, 76>;   //  128 x f16 vector value
-def v2bf16   : ValueType<32 , 77>;    //    2 x bf16 vector value
-def v3bf16   : ValueType<48 , 78>;    //    3 x bf16 vector value
-def v4bf16   : ValueType<64 , 79>;    //    4 x bf16 vector value
-def v8bf16   : ValueType<128, 80>;    //    8 x bf16 vector value
-def v16bf16  : ValueType<256, 81>;    //   16 x bf16 vector value
-def v32bf16  : ValueType<512, 82>;    //   32 x bf16 vector value
-def v64bf16  : ValueType<1024, 83>;   //   64 x bf16 vector value
-def v128bf16 : ValueType<2048, 84>;   //  128 x bf16 vector value
-def v1f32    : ValueType<32 , 85>;    //    1 x f32 vector value
-def v2f32    : ValueType<64 , 86>;    //    2 x f32 vector value
-def v3f32    : ValueType<96 , 87>;    //    3 x f32 vector value
-def v4f32    : ValueType<128, 88>;    //    4 x f32 vector value
-def v5f32    : ValueType<160, 89>;    //    5 x f32 vector value
-def v8f32    : ValueType<256, 90>;    //    8 x f32 vector value
-def v16f32   : ValueType<512,  91>;   //   16 x f32 vector value
-def v32f32   : ValueType<1024, 92>;   //   32 x f32 vector value
-def v64f32   : ValueType<2048, 93>;   //   64 x f32 vector value
-def v128f32  : ValueType<4096, 94>;   //  128 x f32 vector value
-def v256f32  : ValueType<8182, 95>;   //  256 x f32 vector value
-def v512f32  : ValueType<16384, 96>;  //  512 x f32 vector value
-def v1024f32 : ValueType<32768, 97>;  // 1024 x f32 vector value
-def v2048f32 : ValueType<65536, 98>;  // 2048 x f32 vector value
-def v1f64    : ValueType<64, 99>;     //    1 x f64 vector value
-def v2f64    : ValueType<128, 100>;   //    2 x f64 vector value
-def v4f64    : ValueType<256, 101>;   //    4 x f64 vector value
-def v8f64    : ValueType<512, 102>;   //    8 x f64 vector value
-def v16f64   : ValueType<1024, 103>;  //   16 x f64 vector value
-def v32f64   : ValueType<2048, 104>;  //   32 x f64 vector value
-def v64f64   : ValueType<4096, 105>;  //   64 x f64 vector value
-def v128f64  : ValueType<8192, 106>;  //  128 x f64 vector value
-def v256f64  : ValueType<16384, 107>; //  256 x f64 vector value
+def OtherVT : ValueType<0,   1>;  // "Other" value
+def i1      : ValueType<1,   2>;  // One bit boolean value
+def i8      : ValueType<8,   3>;  // 8-bit integer value
+def i16     : ValueType<16,  4>;  // 16-bit integer value
+def i32     : ValueType<32,  5>;  // 32-bit integer value
+def i64     : ValueType<64,  6>;  // 64-bit integer value
+def i128    : ValueType<128, 7>;  // 128-bit integer value
+
+def bf16    : ValueType<16,   8>;  // 16-bit brain floating point value
+def f16     : ValueType<16,   9>;  // 16-bit floating point value
+def f32     : ValueType<32,  10>;  // 32-bit floating point value
+def f64     : ValueType<64,  11>;  // 64-bit floating point value
+def f80     : ValueType<80,  12>;  // 80-bit floating point value
+def f128    : ValueType<128, 13>;  // 128-bit floating point value
+def ppcf128 : ValueType<128, 14>;  // PPC 128-bit floating point value
+
+def v1i1    : ValueType<1,    15>;  //    1 x i1 vector value
+def v2i1    : ValueType<2,    16>;  //    2 x i1 vector value
+def v4i1    : ValueType<4,    17>;  //    4 x i1 vector value
+def v8i1    : ValueType<8,    18>;  //    8 x i1 vector value
+def v16i1   : ValueType<16,   19>;  //   16 x i1 vector value
+def v32i1   : ValueType<32,   20>;  //   32 x i1 vector value
+def v64i1   : ValueType<64,   21>;  //   64 x i1 vector value
+def v128i1  : ValueType<128,  22>;  //  128 x i1 vector value
+def v256i1  : ValueType<256,  23>;  //  256 x i1 vector value
+def v512i1  : ValueType<512,  24>;  //  512 x i1 vector value
+def v1024i1 : ValueType<1024, 25>;  // 1024 x i1 vector value
+
+def v1i8   : ValueType<8,    26>;  //   1 x i8  vector value
+def v2i8   : ValueType<16,   27>;  //   2 x i8  vector value
+def v4i8   : ValueType<32,   28>;  //   4 x i8  vector value
+def v8i8   : ValueType<64,   29>;  //   8 x i8  vector value
+def v16i8  : ValueType<128,  30>;  //  16 x i8  vector value
+def v32i8  : ValueType<256,  31>;  //  32 x i8  vector value
+def v64i8  : ValueType<512,  32>;  //  64 x i8  vector value
+def v128i8 : ValueType<1024, 33>;  // 128 x i8  vector value
+def v256i8 : ValueType<2048, 34>;  // 256 x i8  vector value
+
+def v1i16   : ValueType<16,   35>;  //   1 x i16 vector value
+def v2i16   : ValueType<32,   36>;  //   2 x i16 vector value
+def v3i16   : ValueType<48,   37>;  //   3 x i16 vector value
+def v4i16   : ValueType<64,   38>;  //   4 x i16 vector value
+def v8i16   : ValueType<128,  39>;  //   8 x i16 vector value
+def v16i16  : ValueType<256,  40>;  //  16 x i16 vector value
+def v32i16  : ValueType<512,  41>;  //  32 x i16 vector value
+def v64i16  : ValueType<1024, 42>;  //  64 x i16 vector value
+def v128i16 : ValueType<2048, 43>;  // 128 x i16 vector value
+
+def v1i32    : ValueType<32,    44>;  //    1 x i32 vector value
+def v2i32    : ValueType<64,    45>;  //    2 x i32 vector value
+def v3i32    : ValueType<96,    46>;  //    3 x i32 vector value
+def v4i32    : ValueType<128,   47>;  //    4 x i32 vector value
+def v5i32    : ValueType<160,   48>;  //    5 x i32 vector value
+def v8i32    : ValueType<256,   49>;  //    8 x i32 vector value
+def v16i32   : ValueType<512,   50>;  //   16 x i32 vector value
+def v32i32   : ValueType<1024,  51>;  //   32 x i32 vector value
+def v64i32   : ValueType<2048,  52>;  //   64 x i32 vector value
+def v128i32  : ValueType<4096,  53>;  //  128 x i32 vector value
+def v256i32  : ValueType<8182,  54>;  //  256 x i32 vector value
+def v512i32  : ValueType<16384, 55>;  //  512 x i32 vector value
+def v1024i32 : ValueType<32768, 56>;  // 1024 x i32 vector value
+def v2048i32 : ValueType<65536, 57>;  // 2048 x i32 vector value
+
+def v1i64   : ValueType<64,    58>;  //   1 x i64 vector value
+def v2i64   : ValueType<128,   59>;  //   2 x i64 vector value
+def v4i64   : ValueType<256,   60>;  //   4 x i64 vector value
+def v8i64   : ValueType<512,   61>;  //   8 x i64 vector value
+def v16i64  : ValueType<1024,  62>;  //  16 x i64 vector value
+def v32i64  : ValueType<2048,  63>;  //  32 x i64 vector value
+def v64i64  : ValueType<4096,  64>;  //  64 x i64 vector value
+def v128i64 : ValueType<8192,  65>;  // 128 x i64 vector value
+def v256i64 : ValueType<16384, 66>;  // 256 x i64 vector value
+
+def v1i128 : ValueType<128, 67>;  //  1 x i128 vector value
+
+def v1f16    : ValueType<16,     68>;  //    1 x f16 vector value
+def v2f16    : ValueType<32,     69>;  //    2 x f16 vector value
+def v3f16    : ValueType<48,     70>;  //    3 x f16 vector value
+def v4f16    : ValueType<64,     71>;  //    4 x f16 vector value
+def v8f16    : ValueType<128,    72>;  //    8 x f16 vector value
+def v16f16   : ValueType<256,    73>;  //   16 x f16 vector value
+def v32f16   : ValueType<512,    74>;  //   32 x f16 vector value
+def v64f16   : ValueType<1024,   75>;  //   64 x f16 vector value
+def v128f16  : ValueType<2048,   76>;  //  128 x f16 vector value
+def v2bf16   : ValueType<32,     77>;  //    2 x bf16 vector value
+def v3bf16   : ValueType<48,     78>;  //    3 x bf16 vector value
+def v4bf16   : ValueType<64,     79>;  //    4 x bf16 vector value
+def v8bf16   : ValueType<128,    80>;  //    8 x bf16 vector value
+def v16bf16  : ValueType<256,    81>;  //   16 x bf16 vector value
+def v32bf16  : ValueType<512,    82>;  //   32 x bf16 vector value
+def v64bf16  : ValueType<1024,   83>;  //   64 x bf16 vector value
+def v128bf16 : ValueType<2048,   84>;  //  128 x bf16 vector value
+def v1f32    : ValueType<32,     85>;  //    1 x f32 vector value
+def v2f32    : ValueType<64,     86>;  //    2 x f32 vector value
+def v3f32    : ValueType<96,     87>;  //    3 x f32 vector value
+def v4f32    : ValueType<128,    88>;  //    4 x f32 vector value
+def v5f32    : ValueType<160,    89>;  //    5 x f32 vector value
+def v8f32    : ValueType<256,    90>;  //    8 x f32 vector value
+def v16f32   : ValueType<512,    91>;  //   16 x f32 vector value
+def v32f32   : ValueType<1024,   92>;  //   32 x f32 vector value
+def v64f32   : ValueType<2048,   93>;  //   64 x f32 vector value
+def v128f32  : ValueType<4096,   94>;  //  128 x f32 vector value
+def v256f32  : ValueType<8182,   95>;  //  256 x f32 vector value
+def v512f32  : ValueType<16384,  96>;  //  512 x f32 vector value
+def v1024f32 : ValueType<32768,  97>;  // 1024 x f32 vector value
+def v2048f32 : ValueType<65536,  98>;  // 2048 x f32 vector value
+def v1f64    : ValueType<64,     99>;  //    1 x f64 vector value
+def v2f64    : ValueType<128,   100>;  //    2 x f64 vector value
+def v4f64    : ValueType<256,   101>;  //    4 x f64 vector value
+def v8f64    : ValueType<512,   102>;  //    8 x f64 vector value
+def v16f64   : ValueType<1024,  103>;  //   16 x f64 vector value
+def v32f64   : ValueType<2048,  104>;  //   32 x f64 vector value
+def v64f64   : ValueType<4096,  105>;  //   64 x f64 vector value
+def v128f64  : ValueType<8192,  106>;  //  128 x f64 vector value
+def v256f64  : ValueType<16384, 107>;  //  256 x f64 vector value
 
 def nxv1i1  : ValueType<1,  108>;  // n x  1 x i1  vector value
 def nxv2i1  : ValueType<2,  109>;  // n x  2 x i1  vector value
@@ -140,7 +140,7 @@ def nxv4i1  : ValueType<4,  110>;  // n x  4 x i1  vector value
 def nxv8i1  : ValueType<8,  111>;  // n x  8 x i1  vector value
 def nxv16i1 : ValueType<16, 112>;  // n x 16 x i1  vector value
 def nxv32i1 : ValueType<32, 113>;  // n x 32 x i1  vector value
-def nxv64i1  : ValueType<64,114>;  // n x  64 x i1  vector value
+def nxv64i1 : ValueType<64, 114>;  // n x 64 x i1  vector value
 
 def nxv1i8  : ValueType<8,   115>;  // n x  1 x i8  vector value
 def nxv2i8  : ValueType<16,  116>;  // n x  2 x i8  vector value
@@ -148,79 +148,79 @@ def nxv4i8  : ValueType<32,  117>;  // n x  4 x i8  vector value
 def nxv8i8  : ValueType<64,  118>;  // n x  8 x i8  vector value
 def nxv16i8 : ValueType<128, 119>;  // n x 16 x i8  vector value
 def nxv32i8 : ValueType<256, 120>;  // n x 32 x i8  vector value
-def nxv64i8  : ValueType<512, 121>; // n x  64 x i8  vector value
-
-def nxv1i16 : ValueType<16,  122>; // n x  1 x i16 vector value
-def nxv2i16 : ValueType<32,  123>; // n x  2 x i16 vector value
-def nxv4i16 : ValueType<64,  124>; // n x  4 x i16 vector value
-def nxv8i16 : ValueType<128, 125>; // n x  8 x i16 vector value
-def nxv16i16: ValueType<256, 126>; // n x 16 x i16 vector value
-def nxv32i16: ValueType<512, 127>; // n x 32 x i16 vector value
-
-def nxv1i32 : ValueType<32,  128>; // n x  1 x i32 vector value
-def nxv2i32 : ValueType<64,  129>; // n x  2 x i32 vector value
-def nxv4i32 : ValueType<128, 130>; // n x  4 x i32 vector value
-def nxv8i32 : ValueType<256, 131>; // n x  8 x i32 vector value
-def nxv16i32: ValueType<512, 132>; // n x 16 x i32 vector value
-def nxv32i32: ValueType<1024,133>; // n x 32 x i32 vector value
-
-def nxv1i64 : ValueType<64,  134>; // n x  1 x i64 vector value
-def nxv2i64 : ValueType<128, 135>; // n x  2 x i64 vector value
-def nxv4i64 : ValueType<256, 136>; // n x  4 x i64 vector value
-def nxv8i64 : ValueType<512, 137>; // n x  8 x i64 vector value
-def nxv16i64: ValueType<1024,138>; // n x 16 x i64 vector value
-def nxv32i64: ValueType<2048,139>; // n x 32 x i64 vector value
-
-def nxv1f16  : ValueType<16, 140>; // n x  1 x  f16 vector value
-def nxv2f16  : ValueType<32, 141>; // n x  2 x  f16 vector value
-def nxv4f16  : ValueType<64, 142>; // n x  4 x  f16 vector value
-def nxv8f16  : ValueType<128,143>; // n x  8 x  f16 vector value
-def nxv16f16 : ValueType<256,144>; // n x 16 x  f16 vector value
-def nxv32f16 : ValueType<512,145>; // n x 32 x  f16 vector value
-def nxv1bf16 : ValueType<16, 146>; // n x  1 x bf16 vector value
-def nxv2bf16 : ValueType<32, 147>; // n x  2 x bf16 vector value
-def nxv4bf16 : ValueType<64, 148>; // n x  4 x bf16 vector value
-def nxv8bf16 : ValueType<128,149>; // n x  8 x bf16 vector value
-def nxv1f32  : ValueType<32, 150>; // n x  1 x  f32 vector value
-def nxv2f32  : ValueType<64, 151>; // n x  2 x  f32 vector value
-def nxv4f32  : ValueType<128,152>; // n x  4 x  f32 vector value
-def nxv8f32  : ValueType<256,153>; // n x  8 x  f32 vector value
-def nxv16f32 : ValueType<512,154>; // n x 16 x  f32 vector value
-def nxv1f64  : ValueType<64, 155>; // n x  1 x  f64 vector value
-def nxv2f64  : ValueType<128,156>; // n x  2 x  f64 vector value
-def nxv4f64  : ValueType<256,157>; // n x  4 x  f64 vector value
-def nxv8f64  : ValueType<512,158>; // n x  8 x  f64 vector value
-
-def x86mmx : ValueType<64,   159>; // X86 MMX value
-def FlagVT : ValueType<0,    160>; // Pre-RA sched glue
-def isVoid : ValueType<0,    161>; // Produces no value
-def untyped: ValueType<8,    162>; // Produces an untyped value
-def funcref : ValueType<0,   163>; // WebAssembly's funcref type
-def externref : ValueType<0, 164>; // WebAssembly's externref type
-def x86amx : ValueType<8192, 165>; // X86 AMX value
-
-
-def token  : ValueType<0  , 248>;   // TokenTy
-def MetadataVT: ValueType<0,249>;   // Metadata
+def nxv64i8 : ValueType<512, 121>;  // n x 64 x i8  vector value
+
+def nxv1i16  : ValueType<16,  122>;  // n x  1 x i16 vector value
+def nxv2i16  : ValueType<32,  123>;  // n x  2 x i16 vector value
+def nxv4i16  : ValueType<64,  124>;  // n x  4 x i16 vector value
+def nxv8i16  : ValueType<128, 125>;  // n x  8 x i16 vector value
+def nxv16i16 : ValueType<256, 126>;  // n x 16 x i16 vector value
+def nxv32i16 : ValueType<512, 127>;  // n x 32 x i16 vector value
+
+def nxv1i32  : ValueType<32,   128>;  // n x  1 x i32 vector value
+def nxv2i32  : ValueType<64,   129>;  // n x  2 x i32 vector value
+def nxv4i32  : ValueType<128,  130>;  // n x  4 x i32 vector value
+def nxv8i32  : ValueType<256,  131>;  // n x  8 x i32 vector value
+def nxv16i32 : ValueType<512,  132>;  // n x 16 x i32 vector value
+def nxv32i32 : ValueType<1024, 133>;  // n x 32 x i32 vector value
+
+def nxv1i64  : ValueType<64,   134>;  // n x  1 x i64 vector value
+def nxv2i64  : ValueType<128,  135>;  // n x  2 x i64 vector value
+def nxv4i64  : ValueType<256,  136>;  // n x  4 x i64 vector value
+def nxv8i64  : ValueType<512,  137>;  // n x  8 x i64 vector value
+def nxv16i64 : ValueType<1024, 138>;  // n x 16 x i64 vector value
+def nxv32i64 : ValueType<2048, 139>;  // n x 32 x i64 vector value
+
+def nxv1f16  : ValueType<16,  140>;  // n x  1 x  f16 vector value
+def nxv2f16  : ValueType<32,  141>;  // n x  2 x  f16 vector value
+def nxv4f16  : ValueType<64,  142>;  // n x  4 x  f16 vector value
+def nxv8f16  : ValueType<128, 143>;  // n x  8 x  f16 vector value
+def nxv16f16 : ValueType<256, 144>;  // n x 16 x  f16 vector value
+def nxv32f16 : ValueType<512, 145>;  // n x 32 x  f16 vector value
+def nxv1bf16 : ValueType<16,  146>;  // n x  1 x bf16 vector value
+def nxv2bf16 : ValueType<32,  147>;  // n x  2 x bf16 vector value
+def nxv4bf16 : ValueType<64,  148>;  // n x  4 x bf16 vector value
+def nxv8bf16 : ValueType<128, 149>;  // n x  8 x bf16 vector value
+def nxv1f32  : ValueType<32,  150>;  // n x  1 x  f32 vector value
+def nxv2f32  : ValueType<64,  151>;  // n x  2 x  f32 vector value
+def nxv4f32  : ValueType<128, 152>;  // n x  4 x  f32 vector value
+def nxv8f32  : ValueType<256, 153>;  // n x  8 x  f32 vector value
+def nxv16f32 : ValueType<512, 154>;  // n x 16 x  f32 vector value
+def nxv1f64  : ValueType<64,  155>;  // n x  1 x  f64 vector value
+def nxv2f64  : ValueType<128, 156>;  // n x  2 x  f64 vector value
+def nxv4f64  : ValueType<256, 157>;  // n x  4 x  f64 vector value
+def nxv8f64  : ValueType<512, 158>;  // n x  8 x  f64 vector value
+
+def x86mmx    : ValueType<64,   159>;  // X86 MMX value
+def FlagVT    : ValueType<0,    160>;  // Pre-RA sched glue
+def isVoid    : ValueType<0,    161>;  // Produces no value
+def untyped   : ValueType<8,    162>;  // Produces an untyped value
+def funcref   : ValueType<0,    163>;  // WebAssembly's funcref type
+def externref : ValueType<0,    164>;  // WebAssembly's externref type
+def x86amx    : ValueType<8192, 165>;  // X86 AMX value
+
+
+def token      : ValueType<0, 248>;  // TokenTy
+def MetadataVT : ValueType<0, 249>;  // Metadata
 
 // Pseudo valuetype mapped to the current pointer size to any address space.
 // Should only be used in TableGen.
-def iPTRAny   : ValueType<0, 250>;
+def iPTRAny    : ValueType<0, 250>;
 
 // Pseudo valuetype to represent "vector of any size"
-def vAny   : ValueType<0  , 251>;
+def vAny       : ValueType<0, 251>;
 
 // Pseudo valuetype to represent "float of any format"
-def fAny   : ValueType<0  , 252>;
+def fAny       : ValueType<0, 252>;
 
 // Pseudo valuetype to represent "integer of any bit width"
-def iAny   : ValueType<0  , 253>;
+def iAny       : ValueType<0, 253>;
 
 // Pseudo valuetype mapped to the current pointer size.
-def iPTR   : ValueType<0  , 254>;
+def iPTR       : ValueType<0, 254>;
 
 // Pseudo valuetype to represent "any type of any size".
-def Any    : ValueType<0  , 255>;
+def Any        : ValueType<0, 255>;
 
 /// This class is for targets that want to use pointer types in patterns
 /// with the GlobalISelEmitter.  Targets must define their own pointer
diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index 112d5a489e56f5e2d613a7bd5c8f728d6862ba27..4e3af0cb26490b441811b48a6bc0d5940a956512 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -280,17 +280,20 @@ public:
 class VendorExtQualType final : public Node {
   const Node *Ty;
   StringView Ext;
+  const Node *TA;
 
 public:
-  VendorExtQualType(const Node *Ty_, StringView Ext_)
-      : Node(KVendorExtQualType), Ty(Ty_), Ext(Ext_) {}
+  VendorExtQualType(const Node *Ty_, StringView Ext_, const Node *TA_)
+      : Node(KVendorExtQualType), Ty(Ty_), Ext(Ext_), TA(TA_) {}
 
-  template<typename Fn> void match(Fn F) const { F(Ty, Ext); }
+  template <typename Fn> void match(Fn F) const { F(Ty, Ext, TA); }
 
   void printLeft(OutputStream &S) const override {
     Ty->print(S);
     S += " ";
     S += Ext;
+    if (TA != nullptr)
+      TA->print(S);
   }
 };
 
@@ -3680,8 +3683,6 @@ Node *AbstractManglingParser<Derived, Alloc>::parseQualifiedType() {
     if (Qual.empty())
       return nullptr;
 
-    // FIXME parse the optional <template-args> here!
-
     // extension            ::= U <objc-name> <objc-type>  # objc-type<identifier>
     if (Qual.startsWith("objcproto")) {
       StringView ProtoSourceName = Qual.dropFront(std::strlen("objcproto"));
@@ -3699,10 +3700,17 @@ Node *AbstractManglingParser<Derived, Alloc>::parseQualifiedType() {
       return make<ObjCProtoName>(Child, Proto);
     }
 
+    Node *TA = nullptr;
+    if (look() == 'I') {
+      TA = getDerived().parseTemplateArgs();
+      if (TA == nullptr)
+        return nullptr;
+    }
+
     Node *Child = getDerived().parseQualifiedType();
     if (Child == nullptr)
       return nullptr;
-    return make<VendorExtQualType>(Child, Qual);
+    return make<VendorExtQualType>(Child, Qual, TA);
   }
 
   Qualifiers Quals = parseCVQualifiers();
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
index e605ce4c704acfdd1ddf329f9c12ec0c7e629748..d8ed953363e68a1f48b2af4665a4e9dbfdca647d 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
@@ -26,6 +26,7 @@ enum ELFX86RelocationKind : Edge::Kind {
   Pointer64,
   Pointer64Anon,
   PCRel32,
+  PCRel64,
   PCRel32Minus1,
   PCRel32Minus2,
   PCRel32Minus4,
@@ -35,6 +36,9 @@ enum ELFX86RelocationKind : Edge::Kind {
   PCRel32Minus4Anon,
   PCRel32GOTLoad,
   PCRel32GOT,
+  PCRel64GOT,
+  GOTOFF64,
+  GOT64,
   PCRel32TLV,
   Delta32,
   Delta64,
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index 2ba63f6ce45819608f88a58672cd72de53f439ca..c2335b024198750a2ef83202b94631c39c48e085 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -122,11 +122,16 @@ public:
   void setAddress(JITTargetAddress Address) { this->Address = Address; }
 
   /// Returns true if this is a defined addressable, in which case you
-  /// can downcast this to a .
+  /// can downcast this to a Block.
   bool isDefined() const { return static_cast<bool>(IsDefined); }
   bool isAbsolute() const { return static_cast<bool>(IsAbsolute); }
 
 private:
+  void setAbsolute(bool IsAbsolute) {
+    assert(!IsDefined && "Cannot change the Absolute flag on a defined block");
+    this->IsAbsolute = IsAbsolute;
+  }
+
   JITTargetAddress Address = 0;
   uint64_t IsDefined : 1;
   uint64_t IsAbsolute : 1;
@@ -282,7 +287,11 @@ const char *getLinkageName(Linkage L);
 ///   Default -- Visible in the public interface of the linkage unit.
 ///   Hidden -- Visible within the linkage unit, but not exported from it.
 ///   Local -- Visible only within the LinkGraph.
-enum class Scope : uint8_t { Default, Hidden, Local };
+enum class Scope : uint8_t {
+  Default,
+  Hidden,
+  Local
+};
 
 /// For debugging output.
 const char *getScopeName(Scope S);
@@ -437,7 +446,7 @@ public:
   /// Returns true if the underlying addressable is an absolute symbol.
   bool isAbsolute() const {
     assert(Base && "Attempt to access null symbol");
-    return !Base->isDefined() && Base->isAbsolute();
+    return Base->isAbsolute();
   }
 
   /// Return the addressable that this symbol points to.
@@ -475,6 +484,16 @@ public:
   /// Returns the size of this symbol.
   JITTargetAddress getSize() const { return Size; }
 
+  /// Set the size of this symbol.
+  void setSize(JITTargetAddress Size) {
+    assert(Base && "Cannot set size for null Symbol");
+    assert((Size == 0 || Base->isDefined()) &&
+           "Non-zero size can only be set for defined symbols");
+    assert((Offset + Size <= static_cast<const Block &>(*Base).getSize()) &&
+           "Symbol size cannot extend past the end of its containing block");
+    this->Size = Size;
+  }
+
   /// Returns true if this symbol is backed by a zero-fill block.
   /// This method may only be called on defined symbols.
   bool isSymbolZeroFill() const { return getBlock().isZeroFill(); }
@@ -509,13 +528,20 @@ public:
 
 private:
   void makeExternal(Addressable &A) {
-    assert(!A.isDefined() && "Attempting to make external with defined block");
+    assert(!A.isDefined() && !A.isAbsolute() &&
+           "Attempting to make external with defined or absolute block");
     Base = &A;
     Offset = 0;
-    setLinkage(Linkage::Strong);
     setScope(Scope::Default);
     IsLive = 0;
-    // note: Size and IsCallable fields left unchanged.
+    // note: Size, Linkage and IsCallable fields left unchanged.
+  }
+
+  void makeAbsolute(Addressable &A) {
+    assert(!A.isDefined() && A.isAbsolute() &&
+           "Attempting to make absolute with defined or external block");
+    Base = &A;
+    Offset = 0;
   }
 
   void setBlock(Block &B) { Base = &B; }
@@ -648,7 +674,7 @@ public:
     assert((First || !Last) && "Last can not be null if start is non-null");
     return Last;
   }
-  bool isEmpty() const {
+  bool empty() const {
     assert((First || !Last) && "Last can not be null if start is non-null");
     return !First;
   }
@@ -902,6 +928,11 @@ public:
   /// an error will be emitted. Externals with weak linkage are permitted to
   /// be undefined, in which case they are assigned a value of 0.
   Symbol &addExternalSymbol(StringRef Name, uint64_t Size, Linkage L) {
+    assert(llvm::count_if(ExternalSymbols,
+                          [&](const Symbol *Sym) {
+                            return Sym->getName() == Name;
+                          }) == 0 &&
+           "Duplicate external symbol");
     auto &Sym =
         Symbol::constructExternal(Allocator.Allocate<Symbol>(),
                                   createAddressable(0, false), Name, Size, L);
@@ -912,6 +943,11 @@ public:
   /// Add an absolute symbol.
   Symbol &addAbsoluteSymbol(StringRef Name, JITTargetAddress Address,
                             uint64_t Size, Linkage L, Scope S, bool IsLive) {
+    assert(llvm::count_if(AbsoluteSymbols,
+                          [&](const Symbol *Sym) {
+                            return Sym->getName() == Name;
+                          }) == 0 &&
+           "Duplicate absolute symbol");
     auto &Sym = Symbol::constructAbsolute(Allocator.Allocate<Symbol>(),
                                           createAddressable(Address), Name,
                                           Size, L, S, IsLive);
@@ -923,6 +959,11 @@ public:
   Symbol &addCommonSymbol(StringRef Name, Scope S, Section &Section,
                           JITTargetAddress Address, uint64_t Size,
                           uint64_t Alignment, bool IsLive) {
+    assert(llvm::count_if(defined_symbols(),
+                          [&](const Symbol *Sym) {
+                            return Sym->getName() == Name;
+                          }) == 0 &&
+           "Duplicate defined symbol");
     auto &Sym = Symbol::constructCommon(
         Allocator.Allocate<Symbol>(),
         createBlock(Section, Size, Address, Alignment, 0), Name, Size, S,
@@ -945,6 +986,11 @@ public:
   Symbol &addDefinedSymbol(Block &Content, JITTargetAddress Offset,
                            StringRef Name, JITTargetAddress Size, Linkage L,
                            Scope S, bool IsCallable, bool IsLive) {
+    assert(llvm::count_if(defined_symbols(),
+                          [&](const Symbol *Sym) {
+                            return Sym->getName() == Name;
+                          }) == 0 &&
+           "Duplicate defined symbol");
     auto &Sym =
         Symbol::constructNamedDef(Allocator.Allocate<Symbol>(), Content, Offset,
                                   Name, Size, L, S, IsLive, IsCallable);
@@ -995,21 +1041,74 @@ public:
         const_defined_symbol_iterator(Sections.end(), Sections.end()));
   }
 
-  /// Turn a defined symbol into an external one.
+  /// Make the given symbol external (must not already be external).
+  ///
+  /// Symbol size, linkage and callability will be left unchanged. Symbol scope
+  /// will be set to Default, and offset will be reset to 0.
   void makeExternal(Symbol &Sym) {
-    if (Sym.getAddressable().isAbsolute()) {
+    assert(!Sym.isExternal() && "Symbol is already external");
+    if (Sym.isAbsolute()) {
       assert(AbsoluteSymbols.count(&Sym) &&
              "Sym is not in the absolute symbols set");
+      assert(Sym.getOffset() == 0 && "Absolute not at offset 0");
       AbsoluteSymbols.erase(&Sym);
+      Sym.getAddressable().setAbsolute(false);
     } else {
       assert(Sym.isDefined() && "Sym is not a defined symbol");
       Section &Sec = Sym.getBlock().getSection();
       Sec.removeSymbol(Sym);
+      Sym.makeExternal(createAddressable(0, false));
     }
-    Sym.makeExternal(createAddressable(0, false));
     ExternalSymbols.insert(&Sym);
   }
 
+  /// Make the given symbol an absolute with the given address (must not already
+  /// be absolute).
+  ///
+  /// Symbol size, linkage, scope, and callability, and liveness will be left
+  /// unchanged. Symbol offset will be reset to 0.
+  void makeAbsolute(Symbol &Sym, JITTargetAddress Address) {
+    assert(!Sym.isAbsolute() && "Symbol is already absolute");
+    if (Sym.isExternal()) {
+      assert(ExternalSymbols.count(&Sym) &&
+             "Sym is not in the absolute symbols set");
+      assert(Sym.getOffset() == 0 && "External is not at offset 0");
+      ExternalSymbols.erase(&Sym);
+      Sym.getAddressable().setAbsolute(true);
+    } else {
+      assert(Sym.isDefined() && "Sym is not a defined symbol");
+      Section &Sec = Sym.getBlock().getSection();
+      Sec.removeSymbol(Sym);
+      Sym.makeAbsolute(createAddressable(Address));
+    }
+    AbsoluteSymbols.insert(&Sym);
+  }
+
+  /// Turn an absolute or external symbol into a defined one by attaching it to
+  /// a block. Symbol must not already be defined.
+  void makeDefined(Symbol &Sym, Block &Content, JITTargetAddress Offset,
+                   JITTargetAddress Size, Linkage L, Scope S, bool IsLive) {
+    assert(!Sym.isDefined() && "Sym is already a defined symbol");
+    if (Sym.isAbsolute()) {
+      assert(AbsoluteSymbols.count(&Sym) &&
+             "Symbol is not in the absolutes set");
+      AbsoluteSymbols.erase(&Sym);
+    } else {
+      assert(ExternalSymbols.count(&Sym) &&
+             "Symbol is not in the externals set");
+      ExternalSymbols.erase(&Sym);
+    }
+    Addressable &OldBase = *Sym.Base;
+    Sym.setBlock(Content);
+    Sym.setOffset(Offset);
+    Sym.setSize(Size);
+    Sym.setLinkage(L);
+    Sym.setScope(S);
+    Sym.setLive(IsLive);
+    Content.getSection().addSymbol(Sym);
+    destroyAddressable(OldBase);
+  }
+
   /// Removes an external symbol. Also removes the underlying Addressable.
   void removeExternalSymbol(Symbol &Sym) {
     assert(!Sym.isDefined() && !Sym.isAbsolute() &&
@@ -1388,8 +1487,8 @@ private:
 Error markAllSymbolsLive(LinkGraph &G);
 
 /// Create an out of range error for the given edge in the given block.
-Error makeTargetOutOfRangeError(const Block &B, const Edge &E,
-                                const char *(*getEdgeKindName)(Edge::Kind));
+Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
+                                const Edge &E);
 
 /// Create a LinkGraph from the given object buffer.
 ///
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
index d310c30270ce46a60325e471cda6e7c875f37661..868a53552b9747c780398a4d07dd4d83f9171c85 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
@@ -15,6 +15,8 @@
 
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 
+#include <limits>
+
 namespace llvm {
 namespace jitlink {
 namespace x86_64 {
@@ -244,8 +246,20 @@ enum EdgeKind_x86_64 : Edge::Kind {
 /// only.
 const char *getEdgeKindName(Edge::Kind K);
 
+/// Returns true if the given uint64_t value is in range for a uint32_t.
+inline bool isInRangeForImmU32(uint64_t Value) {
+  return Value <= std::numeric_limits<uint32_t>::max();
+}
+
+/// Returns true if the given int64_t value is in range for an int32_t.
+inline bool isInRangeForImmS32(int64_t Value) {
+  return (Value >= std::numeric_limits<int32_t>::min() &&
+          Value <= std::numeric_limits<int32_t>::max());
+}
+
 /// Apply fixup expression for edge to block content.
-inline Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) {
+inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
+                        char *BlockWorkingMem) {
   using namespace support;
 
   char *FixupPtr = BlockWorkingMem + E.getOffset();
@@ -261,9 +275,10 @@ inline Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) {
 
   case Pointer32: {
     uint64_t Value = E.getTarget().getAddress() + E.getAddend();
-    if (Value > std::numeric_limits<uint32_t>::max())
-      return makeTargetOutOfRangeError(B, E, getEdgeKindName);
-    *(ulittle32_t *)FixupPtr = Value;
+    if (LLVM_LIKELY(isInRangeForImmU32(Value)))
+      *(ulittle32_t *)FixupPtr = Value;
+    else
+      return makeTargetOutOfRangeError(G, B, E);
     break;
   }
 
@@ -274,10 +289,10 @@ inline Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) {
   case PCRel32TLVPLoadRelaxable: {
     int64_t Value =
         E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend();
-    if (Value < std::numeric_limits<int32_t>::min() ||
-        Value > std::numeric_limits<int32_t>::max())
-      return makeTargetOutOfRangeError(B, E, getEdgeKindName);
-    *(little32_t *)FixupPtr = Value;
+    if (LLVM_LIKELY(isInRangeForImmS32(Value)))
+      *(little32_t *)FixupPtr = Value;
+    else
+      return makeTargetOutOfRangeError(G, B, E);
     break;
   }
 
@@ -289,10 +304,10 @@ inline Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) {
 
   case Delta32: {
     int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
-    if (Value < std::numeric_limits<int32_t>::min() ||
-        Value > std::numeric_limits<int32_t>::max())
-      return makeTargetOutOfRangeError(B, E, getEdgeKindName);
-    *(little32_t *)FixupPtr = Value;
+    if (LLVM_LIKELY(isInRangeForImmS32(Value)))
+      *(little32_t *)FixupPtr = Value;
+    else
+      return makeTargetOutOfRangeError(G, B, E);
     break;
   }
 
@@ -304,10 +319,10 @@ inline Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) {
 
   case NegDelta32: {
     int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
-    if (Value < std::numeric_limits<int32_t>::min() ||
-        Value > std::numeric_limits<int32_t>::max())
-      return makeTargetOutOfRangeError(B, E, getEdgeKindName);
-    *(little32_t *)FixupPtr = Value;
+    if (LLVM_LIKELY(isInRangeForImmS32(Value)))
+      *(little32_t *)FixupPtr = Value;
+    else
+      return makeTargetOutOfRangeError(G, B, E);
     break;
   }
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h
index 1b4ee7b5b1f212d7abe79aa926495be224d8a3b3..2e332130d262a87d508c0e2472fbf9798bdb3063 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h
@@ -70,7 +70,7 @@ private:
   ExecutionSession &ES;
 
   using OwnedDebugObject = std::unique_ptr<DebugObject>;
-  std::map<ResourceKey, OwnedDebugObject> PendingObjs;
+  std::map<MaterializationResponsibility *, OwnedDebugObject> PendingObjs;
   std::map<ResourceKey, std::vector<OwnedDebugObject>> RegisteredObjs;
 
   std::mutex PendingObjsLock;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/RPCUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/RPCUtils.h
index e0ac640ebcdd04a2eaac919421c486c9f4967a8b..1ff47ce427585e0ad1c0bbc8106c5d3e7bd3e180 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/RPCUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/RPCUtils.h
@@ -243,6 +243,8 @@ public:
   static void consumeAbandoned(ErrorReturnType RetOrErr) {
     consumeError(RetOrErr.takeError());
   }
+
+  static ErrorReturnType returnError(Error Err) { return std::move(Err); }
 };
 
 // ResultTraits specialization for void functions.
@@ -275,6 +277,8 @@ public:
   static void consumeAbandoned(ErrorReturnType Err) {
     consumeError(std::move(Err));
   }
+
+  static ErrorReturnType returnError(Error Err) { return Err; }
 };
 
 // ResultTraits<Error> is equivalent to ResultTraits<void>. This allows
@@ -1494,36 +1498,34 @@ public:
   typename detail::ResultTraits<AltRetT>::ErrorReturnType
   callB(const ArgTs &...Args) {
     bool ReceivedResponse = false;
-    using ResultType = typename detail::ResultTraits<AltRetT>::ErrorReturnType;
-    auto Result = detail::ResultTraits<AltRetT>::createBlankErrorReturnValue();
+    using AltRetTraits = detail::ResultTraits<AltRetT>;
+    using ResultType = typename AltRetTraits::ErrorReturnType;
+    ResultType Result = AltRetTraits::createBlankErrorReturnValue();
 
     // We have to 'Check' result (which we know is in a success state at this
     // point) so that it can be overwritten in the async handler.
     (void)!!Result;
 
-    if (auto Err = this->template appendCallAsync<Func>(
+    if (Error Err = this->template appendCallAsync<Func>(
             [&](ResultType R) {
               Result = std::move(R);
               ReceivedResponse = true;
               return Error::success();
             },
             Args...)) {
-      detail::ResultTraits<typename Func::ReturnType>::consumeAbandoned(
-          std::move(Result));
-      return std::move(Err);
+      AltRetTraits::consumeAbandoned(std::move(Result));
+      return AltRetTraits::returnError(std::move(Err));
     }
 
-    if (auto Err = this->C.send()) {
-      detail::ResultTraits<typename Func::ReturnType>::consumeAbandoned(
-          std::move(Result));
-      return std::move(Err);
+    if (Error Err = this->C.send()) {
+      AltRetTraits::consumeAbandoned(std::move(Result));
+      return AltRetTraits::returnError(std::move(Err));
     }
 
     while (!ReceivedResponse) {
-      if (auto Err = this->handleOne()) {
-        detail::ResultTraits<typename Func::ReturnType>::consumeAbandoned(
-            std::move(Result));
-        return std::move(Err);
+      if (Error Err = this->handleOne()) {
+        AltRetTraits::consumeAbandoned(std::move(Result));
+        return AltRetTraits::returnError(std::move(Err));
       }
     }
 
diff --git a/llvm/include/llvm/FileCheck/FileCheck.h b/llvm/include/llvm/FileCheck/FileCheck.h
index b44ab025694bed770344b8a97a930c2b183fb2f1..6ed75e14ccb65dd4ad1904d1dc00470017dc0e13 100644
--- a/llvm/include/llvm/FileCheck/FileCheck.h
+++ b/llvm/include/llvm/FileCheck/FileCheck.h
@@ -118,8 +118,6 @@ struct FileCheckDiag {
   /// depending on whether the pattern must have or must not have a match in
   /// order for the directive to succeed.  For example, a CHECK directive's
   /// pattern is expected, and a CHECK-NOT directive's pattern is excluded.
-  /// All match result types whose names end with "Excluded" are for excluded
-  /// patterns, and all others are for expected patterns.
   ///
   /// There might be more than one match result for a single pattern.  For
   /// example, there might be several discarded matches
@@ -136,18 +134,29 @@ struct FileCheckDiag {
     MatchFoundButWrongLine,
     /// Indicates a discarded match for an expected pattern.
     MatchFoundButDiscarded,
+    /// Indicates an error while processing a match after the match was found
+    /// for an expected or excluded pattern.  The error is specified by \c Note,
+    /// to which it should be appropriate to prepend "error: " later.  The full
+    /// match itself should be recorded in a preceding diagnostic of a different
+    /// \c MatchFound match type.
+    MatchFoundErrorNote,
     /// Indicates no match for an excluded pattern.
     MatchNoneAndExcluded,
     /// Indicates no match for an expected pattern, but this might follow good
     /// matches when multiple matches are expected for the pattern, or it might
     /// follow discarded matches for the pattern.
     MatchNoneButExpected,
+    /// Indicates no match due to an expected or excluded pattern that has
+    /// proven to be invalid at match time.  The exact problems are usually
+    /// reported in subsequent diagnostics of the same match type but with
+    /// \c Note set.
+    MatchNoneForInvalidPattern,
     /// Indicates a fuzzy match that serves as a suggestion for the next
     /// intended match for an expected pattern with too few or no good matches.
     MatchFuzzy,
   } MatchTy;
-  /// The search range if MatchTy is MatchNoneAndExcluded or
-  /// MatchNoneButExpected, or the match range otherwise.
+  /// The search range if MatchTy starts with MatchNone, or the match range
+  /// otherwise.
   unsigned InputStartLine;
   unsigned InputStartCol;
   unsigned InputEndLine;
diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td
index 44de0d8757303769cbf4d90c0cac9dacbc4efa09..c68330665b029643bec88429afd8a4068561ad3e 100644
--- a/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -23,7 +23,6 @@ def OpenACC : DirectiveLanguage {
   let clausePrefix = "ACCC_";
   let makeEnumAvailableInNamespace = true;
   let enableBitmaskEnumInNamespace = true;
-  let includeHeader = "llvm/Frontend/OpenACC/ACC.h.inc";
   let clauseEnumSetClass = "AccClauseSet";
   let flangClauseBaseClass = "AccClause";
 }
diff --git a/llvm/include/llvm/Frontend/OpenACC/CMakeLists.txt b/llvm/include/llvm/Frontend/OpenACC/CMakeLists.txt
index b8332e03ff8137d2c75ef878c086011f484deec3..558de3aa92204c5aca49476410eb7d3381c1b568 100644
--- a/llvm/include/llvm/Frontend/OpenACC/CMakeLists.txt
+++ b/llvm/include/llvm/Frontend/OpenACC/CMakeLists.txt
@@ -1,4 +1,4 @@
 set(LLVM_TARGET_DEFINITIONS ACC.td)
 tablegen(LLVM ACC.h.inc --gen-directive-decl)
-tablegen(LLVM ACC.inc --gen-directive-gen)
+tablegen(LLVM ACC.inc --gen-directive-impl)
 add_public_tablegen_target(acc_gen)
diff --git a/llvm/include/llvm/Frontend/OpenMP/CMakeLists.txt b/llvm/include/llvm/Frontend/OpenMP/CMakeLists.txt
index 1359929b67c7bbb047449da15e654c87f07a91a2..e983cf5ca56db3becf1ec3cee7498b87e8b9e765 100644
--- a/llvm/include/llvm/Frontend/OpenMP/CMakeLists.txt
+++ b/llvm/include/llvm/Frontend/OpenMP/CMakeLists.txt
@@ -1,4 +1,4 @@
 set(LLVM_TARGET_DEFINITIONS OMP.td)
 tablegen(LLVM OMP.h.inc --gen-directive-decl)
-tablegen(LLVM OMP.inc --gen-directive-gen)
+tablegen(LLVM OMP.inc --gen-directive-impl)
 add_public_tablegen_target(omp_gen)
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index f30799e9c0e05cae43de7555e9786fc35d40a535..7845e4bc98dbf7a935c029f759ff59b28b54d17c 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -23,7 +23,6 @@ def OpenMP : DirectiveLanguage {
   let clausePrefix = "OMPC_";
   let makeEnumAvailableInNamespace = true;
   let enableBitmaskEnumInNamespace = true;
-  let includeHeader = "llvm/Frontend/OpenMP/OMP.h.inc";
   let clauseEnumSetClass = "OmpClauseSet";
   let flangClauseBaseClass = "OmpClause";
 }
@@ -268,6 +267,12 @@ def OMPC_Order : Clause<"order"> {
     OMP_ORDER_concurrent
   ];
 }
+def OMPC_Init : Clause<"init"> {
+  let clangClass = "OMPInitClause";
+}
+def OMPC_Use : Clause<"use"> {
+  let clangClass = "OMPUseClause";
+}
 def OMPC_Destroy : Clause<"destroy"> {
   let clangClass = "OMPDestroyClause";
 }
@@ -1640,6 +1645,16 @@ def OMP_EndWorkshare : Directive<"end workshare"> {
     VersionedClause<OMPC_NoWait>
   ];
 }
+def OMP_interop : Directive<"interop"> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_Destroy>,
+    VersionedClause<OMPC_Init>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_Use>,
+  ];
+}
 def OMP_Unknown : Directive<"unknown"> {
   let isDefault = true;
 }
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 75d360bf4237a4b728863de84acb34fac25cf109..9a77dd7f098d68a489ace14da599a3d02ebb642e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -838,29 +838,41 @@ __OMP_RTL_ATTRS(__kmpc_doacross_wait, BarrierAttrs, AttributeSet(),
 __OMP_RTL_ATTRS(__kmpc_doacross_fini, BarrierAttrs, AttributeSet(),
                 ParamAttrs(ReadOnlyPtrAttrs))
 
-__OMP_RTL_ATTRS(__kmpc_alloc, DefaultAttrs, ReturnPtrAttrs, {})
-__OMP_RTL_ATTRS(__kmpc_free, AllocAttrs, AttributeSet(), {})
-
-__OMP_RTL_ATTRS(__kmpc_init_allocator, DefaultAttrs, ReturnPtrAttrs, {})
-__OMP_RTL_ATTRS(__kmpc_destroy_allocator, AllocAttrs, AttributeSet(), {})
-
-__OMP_RTL_ATTRS(__kmpc_push_target_tripcount_mapper, SetterAttrs, AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_target_mapper, ForkAttrs, AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_target_nowait_mapper, ForkAttrs, AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_target_teams_mapper, ForkAttrs, AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_target_teams_nowait_mapper, ForkAttrs, AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_register_requires, ForkAttrs, AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_target_data_begin_mapper, ForkAttrs, AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_alloc, DefaultAttrs, ReturnPtrAttrs, ParamAttrs())
+__OMP_RTL_ATTRS(__kmpc_free, AllocAttrs, AttributeSet(), ParamAttrs())
+
+__OMP_RTL_ATTRS(__kmpc_init_allocator, DefaultAttrs, ReturnPtrAttrs,
+                ParamAttrs())
+__OMP_RTL_ATTRS(__kmpc_destroy_allocator, AllocAttrs, AttributeSet(),
+                ParamAttrs())
+
+__OMP_RTL_ATTRS(__kmpc_push_target_tripcount_mapper, SetterAttrs,
+                AttributeSet(), ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_target_mapper, ForkAttrs, AttributeSet(), ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_target_nowait_mapper, ForkAttrs, AttributeSet(),
+                ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_target_teams_mapper, ForkAttrs, AttributeSet(),
+                ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_target_teams_nowait_mapper, ForkAttrs, AttributeSet(),
+                ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_register_requires, ForkAttrs, AttributeSet(),
+                ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_target_data_begin_mapper, ForkAttrs, AttributeSet(),
+                ParamAttrs())
 __OMP_RTL_ATTRS(__tgt_target_data_begin_nowait_mapper, ForkAttrs,
-        AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_target_data_end_mapper, ForkAttrs, AttributeSet(), {})
+                AttributeSet(), ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_target_data_end_mapper, ForkAttrs, AttributeSet(),
+                ParamAttrs())
 __OMP_RTL_ATTRS(__tgt_target_data_end_nowait_mapper, ForkAttrs,
-        AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_target_data_update_mapper, ForkAttrs, AttributeSet(), {})
+                AttributeSet(), ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_target_data_update_mapper, ForkAttrs, AttributeSet(),
+                ParamAttrs())
 __OMP_RTL_ATTRS(__tgt_target_data_update_nowait_mapper, ForkAttrs,
-        AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_mapper_num_components, ForkAttrs, AttributeSet(), {})
-__OMP_RTL_ATTRS(__tgt_push_mapper_component, ForkAttrs, AttributeSet(), {})
+                AttributeSet(), ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_mapper_num_components, ForkAttrs, AttributeSet(),
+                ParamAttrs())
+__OMP_RTL_ATTRS(__tgt_push_mapper_component, ForkAttrs, AttributeSet(),
+                ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_task_allow_completion_event, DefaultAttrs,
                 ReturnPtrAttrs, ParamAttrs(ReadOnlyPtrAttrs))
 
diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h
index 76d780485ea0b3d26fd26053fbc264fddba01c37..4b13e2d2a9e8201c7777c71772bf156797863873 100644
--- a/llvm/include/llvm/IR/Argument.h
+++ b/llvm/include/llvm/IR/Argument.h
@@ -111,6 +111,9 @@ public:
   /// If this is a byref argument, return its type.
   Type *getParamByRefType() const;
 
+  /// If this is an inalloca argument, return its type.
+  Type *getParamInAllocaType() const;
+
   /// Return true if this argument has the nest attribute.
   bool hasNestAttr() const;
 
@@ -120,6 +123,9 @@ public:
   /// Return true if this argument has the nocapture attribute.
   bool hasNoCaptureAttr() const;
 
+  /// Return true if this argument has the nofree attribute.
+  bool hasNoFreeAttr() const;
+
   /// Return true if this argument has the sret attribute.
   bool hasStructRetAttr() const;
 
diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h
index b4056540663f2b826ec108d4237560cf0ee1b9a9..a8c4017118580cfc85137a7f9970b7be7581005a 100644
--- a/llvm/include/llvm/IR/Attributes.h
+++ b/llvm/include/llvm/IR/Attributes.h
@@ -18,6 +18,7 @@
 #include "llvm-c/Types.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Config/llvm-config.h"
@@ -107,10 +108,13 @@ public:
   static Attribute getWithAllocSizeArgs(LLVMContext &Context,
                                         unsigned ElemSizeArg,
                                         const Optional<unsigned> &NumElemsArg);
+  static Attribute getWithVScaleRangeArgs(LLVMContext &Context,
+                                          unsigned MinValue, unsigned MaxValue);
   static Attribute getWithByValType(LLVMContext &Context, Type *Ty);
   static Attribute getWithStructRetType(LLVMContext &Context, Type *Ty);
   static Attribute getWithByRefType(LLVMContext &Context, Type *Ty);
   static Attribute getWithPreallocatedType(LLVMContext &Context, Type *Ty);
+  static Attribute getWithInAllocaType(LLVMContext &Context, Type *Ty);
 
   /// For a typed attribute, return the equivalent attribute with the type
   /// changed to \p ReplacementTy.
@@ -157,7 +161,7 @@ public:
   bool hasAttribute(StringRef Val) const;
 
   /// Return the attribute's kind as an enum (Attribute::AttrKind). This
-  /// requires the attribute to be an enum or integer attribute.
+  /// requires the attribute to be an enum, integer, or type attribute.
   Attribute::AttrKind getKindAsEnum() const;
 
   /// Return the attribute's value as an integer. This requires that the
@@ -196,6 +200,10 @@ public:
   /// if not known).
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
 
+  /// Returns the argument numbers for the vscale_range attribute (or pair(0, 0)
+  /// if not known).
+  std::pair<unsigned, unsigned> getVScaleRangeArgs() const;
+
   /// The Attribute is converted to a string of equivalent mnemonic. This
   /// is, presumably, for writing out the mnemonics for the assembly writer.
   std::string getAsString(bool InAttrGrp = false) const;
@@ -318,7 +326,9 @@ public:
   Type *getStructRetType() const;
   Type *getByRefType() const;
   Type *getPreallocatedType() const;
+  Type *getInAllocaType() const;
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
+  std::pair<unsigned, unsigned> getVScaleRangeArgs() const;
   std::string getAsString(bool InAttrGrp = false) const;
 
   using iterator = const Attribute *;
@@ -513,6 +523,12 @@ public:
     return removeAttributes(C, ArgNo + FirstArgIndex, AttrsToRemove);
   }
 
+  /// Remove noundef attribute and other attributes that imply undefined
+  /// behavior if a `undef` or `poison` value is passed from this attribute
+  /// list. Returns a new list because attribute lists are immutable.
+  LLVM_NODISCARD AttributeList
+  removeParamUndefImplyingAttributes(LLVMContext &C, unsigned ArgNo) const;
+
   /// Remove all attributes at the specified arg index from this
   /// attribute list. Returns a new list because attribute lists are immutable.
   LLVM_NODISCARD AttributeList removeParamAttributes(LLVMContext &C,
@@ -571,6 +587,13 @@ public:
     return addAllocSizeAttr(C, ArgNo + FirstArgIndex, ElemSizeArg, NumElemsArg);
   }
 
+  /// Add the vscale_range attribute to the attribute set at the given index.
+  /// Returns a new list because attribute lists are immutable.
+  LLVM_NODISCARD AttributeList addVScaleRangeAttr(LLVMContext &C,
+                                                  unsigned Index,
+                                                  unsigned MinValue,
+                                                  unsigned MaxValue);
+
   //===--------------------------------------------------------------------===//
   // AttributeList Accessors
   //===--------------------------------------------------------------------===//
@@ -663,6 +686,9 @@ public:
   /// Return the preallocated type for the specified function parameter.
   Type *getParamPreallocatedType(unsigned ArgNo) const;
 
+  /// Return the inalloca type for the specified function parameter.
+  Type *getParamInAllocaType(unsigned ArgNo) const;
+
   /// Get the stack alignment.
   MaybeAlign getStackAlignment(unsigned Index) const;
 
@@ -689,6 +715,9 @@ public:
   std::pair<unsigned, Optional<unsigned>>
   getAllocSizeArgs(unsigned Index) const;
 
+  /// Get the vscale_range argument numbers (or pair(0, 0) if unknown).
+  std::pair<unsigned, unsigned> getVScaleRangeArgs(unsigned Index) const;
+
   /// Return the attributes at the index as a string.
   std::string getAsString(unsigned Index, bool InAttrGrp = false) const;
 
@@ -756,16 +785,18 @@ template <> struct DenseMapInfo<AttributeList> {
 /// equality, presence of attributes, etc.
 class AttrBuilder {
   std::bitset<Attribute::EndAttrKinds> Attrs;
-  std::map<std::string, std::string, std::less<>> TargetDepAttrs;
+  std::map<SmallString<32>, SmallString<32>, std::less<>> TargetDepAttrs;
   MaybeAlign Alignment;
   MaybeAlign StackAlignment;
   uint64_t DerefBytes = 0;
   uint64_t DerefOrNullBytes = 0;
   uint64_t AllocSizeArgs = 0;
+  uint64_t VScaleRangeArgs = 0;
   Type *ByValType = nullptr;
   Type *StructRetType = nullptr;
   Type *ByRefType = nullptr;
   Type *PreallocatedType = nullptr;
+  Type *InAllocaType = nullptr;
 
 public:
   AttrBuilder() = default;
@@ -860,10 +891,17 @@ public:
   /// Retrieve the preallocated type.
   Type *getPreallocatedType() const { return PreallocatedType; }
 
+  /// Retrieve the inalloca type.
+  Type *getInAllocaType() const { return InAllocaType; }
+
   /// Retrieve the allocsize args, if the allocsize attribute exists.  If it
   /// doesn't exist, pair(0, 0) is returned.
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
 
+  /// Retrieve the vscale_range args, if the vscale_range attribute exists.  If
+  /// it doesn't exist, pair(0, 0) is returned.
+  std::pair<unsigned, unsigned> getVScaleRangeArgs() const;
+
   /// This turns an alignment into the form used internally in Attribute.
   /// This call has no effect if Align is not set.
   AttrBuilder &addAlignmentAttr(MaybeAlign Align);
@@ -900,6 +938,9 @@ public:
   AttrBuilder &addAllocSizeAttr(unsigned ElemSizeArg,
                                 const Optional<unsigned> &NumElemsArg);
 
+  /// This turns two ints into the form used internally in Attribute.
+  AttrBuilder &addVScaleRangeAttr(unsigned MinValue, unsigned MaxValue);
+
   /// This turns a byval type into the form used internally in Attribute.
   AttrBuilder &addByValAttr(Type *Ty);
 
@@ -912,16 +953,23 @@ public:
   /// This turns a preallocated type into the form used internally in Attribute.
   AttrBuilder &addPreallocatedAttr(Type *Ty);
 
+  /// This turns an inalloca type into the form used internally in Attribute.
+  AttrBuilder &addInAllocaAttr(Type *Ty);
+
   /// Add an allocsize attribute, using the representation returned by
   /// Attribute.getIntValue().
   AttrBuilder &addAllocSizeAttrFromRawRepr(uint64_t RawAllocSizeRepr);
 
+  /// Add a vscale_range attribute, using the representation returned by
+  /// Attribute.getIntValue().
+  AttrBuilder &addVScaleRangeAttrFromRawRepr(uint64_t RawVScaleRangeRepr);
+
   /// Return true if the builder contains no target-independent
   /// attributes.
   bool empty() const { return Attrs.none(); }
 
   // Iterators for target-dependent attributes.
-  using td_type = std::pair<std::string, std::string>;
+  using td_type = decltype(TargetDepAttrs)::value_type;
   using td_iterator = decltype(TargetDepAttrs)::iterator;
   using td_const_iterator = decltype(TargetDepAttrs)::const_iterator;
   using td_range = iterator_range<td_iterator>;
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 5515375d131ef8d86d2bf72b7bc474383b3e1215..9684ffa0009b1e3e6346d8122f66656c582511f9 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -80,7 +80,7 @@ def InaccessibleMemOnly : EnumAttr<"inaccessiblememonly">;
 def InaccessibleMemOrArgMemOnly : EnumAttr<"inaccessiblemem_or_argmemonly">;
 
 /// Pass structure in an alloca.
-def InAlloca : EnumAttr<"inalloca">;
+def InAlloca : TypeAttr<"inalloca">;
 
 /// Source said inlining was desirable.
 def InlineHint : EnumAttr<"inlinehint">;
@@ -247,6 +247,9 @@ def SwiftSelf : EnumAttr<"swiftself">;
 /// Function must be in a unwind table.
 def UWTable : EnumAttr<"uwtable">;
 
+/// Minimum/Maximum vscale value for function.
+def VScaleRange : IntAttr<"vscale_range">;
+
 /// Function always comes back to callsite.
 def WillReturn : EnumAttr<"willreturn">;
 
diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h
index 796029e0a782595524be69c744a8740696bc013b..e047acb8f5af03311f40e3d4a240ad52fe53f3c2 100644
--- a/llvm/include/llvm/IR/Constants.h
+++ b/llvm/include/llvm/IR/Constants.h
@@ -559,10 +559,10 @@ public:
 
 //===----------------------------------------------------------------------===//
 /// ConstantDataSequential - A vector or array constant whose element type is a
-/// simple 1/2/4/8-byte integer or float/double, and whose elements are just
-/// simple data values (i.e. ConstantInt/ConstantFP).  This Constant node has no
-/// operands because it stores all of the elements of the constant as densely
-/// packed data, instead of as Value*'s.
+/// simple 1/2/4/8-byte integer or half/bfloat/float/double, and whose elements
+/// are just simple data values (i.e. ConstantInt/ConstantFP).  This Constant
+/// node has no operands because it stores all of the elements of the constant
+/// as densely packed data, instead of as Value*'s.
 ///
 /// This is the common base class of ConstantDataArray and ConstantDataVector.
 ///
@@ -701,11 +701,11 @@ public:
     return ConstantDataArray::get(Context, makeArrayRef(Elts));
   }
 
-  /// get() constructor - Return a constant with array type with an element
+  /// getRaw() constructor - Return a constant with array type with an element
   /// count and element type matching the NumElements and ElementTy parameters
   /// passed in. Note that this can return a ConstantAggregateZero object.
-  /// ElementTy needs to be one of i8/i16/i32/i64/float/double. Data is the
-  /// buffer containing the elements. Be careful to make sure Data uses the
+  /// ElementTy must be one of i8/i16/i32/i64/half/bfloat/float/double. Data is
+  /// the buffer containing the elements. Be careful to make sure Data uses the
   /// right endianness, the buffer will be used as-is.
   static Constant *getRaw(StringRef Data, uint64_t NumElements,
                           Type *ElementTy) {
@@ -773,6 +773,18 @@ public:
   static Constant *get(LLVMContext &Context, ArrayRef<float> Elts);
   static Constant *get(LLVMContext &Context, ArrayRef<double> Elts);
 
+  /// getRaw() constructor - Return a constant with vector type with an element
+  /// count and element type matching the NumElements and ElementTy parameters
+  /// passed in. Note that this can return a ConstantAggregateZero object.
+  /// ElementTy must be one of i8/i16/i32/i64/half/bfloat/float/double. Data is
+  /// the buffer containing the elements. Be careful to make sure Data uses the
+  /// right endianness, the buffer will be used as-is.
+  static Constant *getRaw(StringRef Data, uint64_t NumElements,
+                          Type *ElementTy) {
+    Type *Ty = VectorType::get(ElementTy, ElementCount::getFixed(NumElements));
+    return getImpl(Data, Ty);
+  }
+
   /// getFP() constructors - Return a constant of vector type with a float
   /// element type taken from argument `ElementType', and count taken from
   /// argument `Elts'.  The amount of bits of the contained type must match the
@@ -785,7 +797,7 @@ public:
 
   /// Return a ConstantVector with the specified constant in each element.
   /// The specified constant has to be a of a compatible type (i8/i16/
-  /// i32/i64/float/double) and must be a ConstantFP or ConstantInt.
+  /// i32/i64/half/bfloat/float/double) and must be a ConstantFP or ConstantInt.
   static Constant *getSplat(unsigned NumElts, Constant *Elt);
 
   /// Returns true if this is a splat constant, meaning that all elements have
diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h
index 29816da3090ef62ee2bf1cf7634381eb868093c5..a1f84dc45f1f317722f6ae748efb162526db6db8 100644
--- a/llvm/include/llvm/IR/DerivedTypes.h
+++ b/llvm/include/llvm/IR/DerivedTypes.h
@@ -679,14 +679,17 @@ Type *Type::getExtendedType() const {
   return cast<IntegerType>(this)->getExtendedType();
 }
 
+Type *Type::getWithNewType(Type *EltTy) const {
+  if (auto *VTy = dyn_cast<VectorType>(this))
+    return VectorType::get(EltTy, VTy->getElementCount());
+  return EltTy;
+}
+
 Type *Type::getWithNewBitWidth(unsigned NewBitWidth) const {
   assert(
       isIntOrIntVectorTy() &&
       "Original type expected to be a vector of integers or a scalar integer.");
-  Type *NewType = getIntNTy(getContext(), NewBitWidth);
-  if (auto *VTy = dyn_cast<VectorType>(this))
-    NewType = VectorType::get(NewType, VTy->getElementCount());
-  return NewType;
+  return getWithNewType(getIntNTy(getContext(), NewBitWidth));
 }
 
 unsigned Type::getPointerAddressSpace() const {
diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h
index b1ef3b1131908b0f091e5dff78df9fea4ded48f7..a24b12c1a470a2127d963f81f91d25958277f06d 100644
--- a/llvm/include/llvm/IR/Function.h
+++ b/llvm/include/llvm/IR/Function.h
@@ -426,6 +426,10 @@ public:
   /// removes the attribute from the list of attributes.
   void removeParamAttrs(unsigned ArgNo, const AttrBuilder &Attrs);
 
+  /// removes noundef and other attributes that imply undefined behavior if a
+  /// `undef` or `poison` value is passed from the list of attributes.
+  void removeParamUndefImplyingAttrs(unsigned ArgNo);
+
   /// check if an attributes is in the list of attributes.
   bool hasAttribute(unsigned i, Attribute::AttrKind Kind) const {
     return getAttributes().hasAttribute(i, Kind);
@@ -489,6 +493,11 @@ public:
     return AttributeSets.getParamStructRetType(ArgNo);
   }
 
+  /// Extract the inalloca type for a parameter.
+  Type *getParamInAllocaType(unsigned ArgNo) const {
+    return AttributeSets.getParamInAllocaType(ArgNo);
+  }
+
   /// Extract the byref type for a parameter.
   Type *getParamByRefType(unsigned ArgNo) const {
     return AttributeSets.getParamByRefType(ArgNo);
@@ -624,6 +633,14 @@ public:
     addFnAttr(Attribute::NoFree);
   }
 
+  /// Determine if the call can synchroize with other threads
+  bool hasNoSync() const {
+    return hasFnAttribute(Attribute::NoSync);
+  }
+  void setNoSync() {
+    addFnAttr(Attribute::NoSync);
+  }
+
   /// Determine if the function is known not to recurse, directly or
   /// indirectly.
   bool doesNotRecurse() const {
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index b62de5978abc1e36cd5b4ace4cb3045bce26ed02..1ec70fa66582e0f156fef903873c8a412c332a04 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -856,6 +856,9 @@ public:
   /// will be the same type as that of \p Scaling.
   Value *CreateVScale(Constant *Scaling, const Twine &Name = "");
 
+  /// Creates a vector of type \p DstType with the linear sequence <0, 1, ...>
+  Value *CreateStepVector(Type *DstType, const Twine &Name = "");
+
   /// Create a call to intrinsic \p ID with 1 operand which is mangled on its
   /// type.
   CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
@@ -2501,6 +2504,9 @@ public:
   /// address space before call and casted back to Ptr type after call.
   Value *CreateStripInvariantGroup(Value *Ptr);
 
+  /// Return a vector value that contains the vector V reversed
+  Value *CreateVectorReverse(Value *V, const Twine &Name = "");
+
   /// Return a vector value that contains \arg V broadcasted to \p
   /// NumElts elements.
   Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "");
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 5c1d2bdd296ea48dd28a5db1af9951d550efd0c8..f218bc4cd36fc528694923f925d28fbbfd2edc4d 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1555,6 +1555,15 @@ public:
     setAttributes(PAL);
   }
 
+  /// Removes noundef and other attributes that imply undefined behavior if a
+  /// `undef` or `poison` value is passed from the given argument.
+  void removeParamUndefImplyingAttrs(unsigned ArgNo) {
+    assert(ArgNo < getNumArgOperands() && "Out of bounds");
+    AttributeList PAL = getAttributes();
+    PAL = PAL.removeParamUndefImplyingAttributes(getContext(), ArgNo);
+    setAttributes(PAL);
+  }
+
   /// adds the dereferenceable attribute to the list of attributes.
   void addDereferenceableAttr(unsigned i, uint64_t Bytes) {
     AttributeList PAL = getAttributes();
@@ -2013,12 +2022,7 @@ public:
 
   /// Return true if this operand bundle user has operand bundles that
   /// may read from the heap.
-  bool hasReadingOperandBundles() const {
-    // Implementation note: this is a conservative implementation of operand
-    // bundle semantics, where *any* operand bundle forces a callsite to be at
-    // least readonly.
-    return hasOperandBundles();
-  }
+  bool hasReadingOperandBundles() const;
 
   /// Return true if this operand bundle user has operand bundles that
   /// may write to the heap.
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index e77295c2c743633be622b7d0259c49f040c07eb5..9b7f956ca06e0c542b2037f887e3493387d218ce 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -56,11 +56,20 @@ namespace Intrinsic {
   StringRef getName(ID id);
 
   /// Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
-  /// Note, this version of getName supports overloads, but is less efficient
-  /// than the StringRef version of this function.  If no overloads are
-  /// requried, it is safe to use this version, but better to use the StringRef
-  /// version.
-  std::string getName(ID id, ArrayRef<Type*> Tys);
+  /// Note, this version of getName supports overloads, but not unnamed types.
+  /// It is less efficient than the StringRef version of this function. If no
+  /// overloads are required, it is safe to use this version, but better to use
+  /// the StringRef version.
+  std::string getName(ID Id, ArrayRef<Type *> Tys);
+
+  /// Return the LLVM name for an intrinsic, such as "llvm.ssa.copy.p0s_s.1".
+  /// Note, this version of getName supports overloads and unnamed types, but is
+  /// less efficient than the StringRef version of this function.  If no
+  /// overloads are required, it is safe to use this version, but better to use
+  /// the StringRef version. A function type FT can be provided to avoid
+  /// computing it. It is used (or computed) if one of the types is based on an
+  /// unnamed type.
+  std::string getName(ID Id, ArrayRef<Type *> Tys, Module *M, FunctionType *FT);
 
   /// Return the function type for an intrinsic.
   FunctionType *getType(LLVMContext &Context, ID id,
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 4ebb46236150b5f06f5a1151391509ffb630a778..380657db94aaf263c99f403a41db2598d4450f05 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -541,10 +541,10 @@ def int_pcmarker      : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>;
 
 def int_readcyclecounter : DefaultAttrsIntrinsic<[llvm_i64_ty]>;
 
-// The assume intrinsic is marked as arbitrarily writing so that proper
-// control dependencies will be maintained.
-def int_assume        : DefaultAttrsIntrinsic<[], [llvm_i1_ty], [IntrWillReturn,
-                                                     NoUndef<ArgIndex<0>>]>;
+// The assume intrinsic is marked InaccessibleMemOnly so that proper control
+// dependencies will be maintained.
+def int_assume : DefaultAttrsIntrinsic<
+    [], [llvm_i1_ty], [IntrInaccessibleMemOnly, NoUndef<ArgIndex<0>>]>;
 
 // 'llvm.experimental.noalias.scope.decl' intrinsic: Inserted at the location of
 // noalias scope declaration. Makes it possible to identify that a noalias scope
@@ -972,18 +972,19 @@ def int_eh_sjlj_setup_dispatch  : Intrinsic<[], []>;
 
 //===---------------- Generic Variable Attribute Intrinsics----------------===//
 //
-def int_var_annotation : DefaultAttrsIntrinsic<[],
-                                   [llvm_ptr_ty, llvm_ptr_ty,
-                                    llvm_ptr_ty, llvm_i32_ty, llvm_ptr_ty],
-                                   [IntrWillReturn], "llvm.var.annotation">;
-def int_ptr_annotation : DefaultAttrsIntrinsic<[LLVMAnyPointerType<llvm_anyint_ty>],
-                                   [LLVMMatchType<0>, llvm_ptr_ty, llvm_ptr_ty,
-                                    llvm_i32_ty, llvm_ptr_ty],
-                                   [IntrWillReturn], "llvm.ptr.annotation">;
-def int_annotation : DefaultAttrsIntrinsic<[llvm_anyint_ty],
-                               [LLVMMatchType<0>, llvm_ptr_ty,
-                                llvm_ptr_ty, llvm_i32_ty],
-                               [IntrWillReturn], "llvm.annotation">;
+def int_var_annotation : DefaultAttrsIntrinsic<
+    [], [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty, llvm_ptr_ty],
+    [IntrInaccessibleMemOnly], "llvm.var.annotation">;
+
+def int_ptr_annotation : DefaultAttrsIntrinsic<
+    [LLVMAnyPointerType<llvm_anyint_ty>],
+    [LLVMMatchType<0>, llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty, llvm_ptr_ty],
+    [IntrInaccessibleMemOnly], "llvm.ptr.annotation">;
+
+def int_annotation : DefaultAttrsIntrinsic<
+    [llvm_anyint_ty],
+    [LLVMMatchType<0>, llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty],
+    [IntrInaccessibleMemOnly], "llvm.annotation">;
 
 // Annotates the current program point with metadata strings which are emitted
 // as CodeView debug info records. This is expensive, as it disables inlining
@@ -1182,11 +1183,11 @@ def int_experimental_gc_statepoint : Intrinsic<[llvm_token_ty],
                                 ImmArg<ArgIndex<4>>]>;
 
 def int_experimental_gc_result   : Intrinsic<[llvm_any_ty], [llvm_token_ty],
-                                             [IntrReadMem]>;
+                                             [IntrNoMem]>;
 def int_experimental_gc_relocate : Intrinsic<[llvm_any_ty],
                                              [llvm_token_ty, llvm_i32_ty,
                                               llvm_i32_ty],
-                                             [IntrReadMem, ImmArg<ArgIndex<1>>,
+                                             [IntrNoMem, ImmArg<ArgIndex<1>>,
                                               ImmArg<ArgIndex<2>>]>;
 
 //===------------------------ Coroutine Intrinsics ---------------===//
@@ -1336,6 +1337,9 @@ def int_is_constant : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty],
 def int_ptrmask: DefaultAttrsIntrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_anyint_ty],
                            [IntrNoMem, IntrSpeculatable, IntrWillReturn]>;
 
+def int_experimental_stepvector : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                                                        [], [IntrNoMem]>;
+
 //===---------------- Vector Predication Intrinsics --------------===//
 
 // FIXME: TO BE REPLACED
@@ -1882,10 +1886,6 @@ def int_experimental_vector_reverse : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
 //===---------- Intrinsics to query properties of scalable vectors --------===//
 def int_vscale : DefaultAttrsIntrinsic<[llvm_anyint_ty], [], [IntrNoMem]>;
 
-def int_experimental_vector_stepvector : Intrinsic<[llvm_anyvector_ty],
-                                                   [],
-                                                   [IntrNoMem]>;
-
 //===---------- Intrinsics to modify scalable vectors ---------------------===//
 // This intrinsic takes two vector arguments %v1 and %v2, an integer number
 // %num, and and integer Explicit Vector Length values %evl1 and %evl2 such that
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 8f8f713fb5f08835264e5ffaa35906ce1ecf43d2..91b3c8fe2114b3a2a121e1e06d7b980cd2006017 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -471,10 +471,6 @@ let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
   def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
   def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
 
-  // Vector FP Rounding: only ties to even is unrepresented by a normal
-  // intrinsic.
-  def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
-
   // v8.5-A Vector FP Rounding
   def int_aarch64_neon_frint32x : AdvSIMD_1FloatArg_Intrinsic;
   def int_aarch64_neon_frint32z : AdvSIMD_1FloatArg_Intrinsic;
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 6ab044254eb4bc2830eeb2efe1d866b06164fe25..7b62b9de79b2210cecedd05e74539b9189c011e1 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1882,146 +1882,35 @@ def int_amdgcn_udot8 :
 
 def int_amdgcn_global_atomic_fadd : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>;
 
-// llvm.amdgcn.mfma.f32.* vdst, srcA, srcB, srcC, cbsz, abid, blgp
-def int_amdgcn_mfma_f32_32x32x1f32 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_32x32x1f32">,
-  Intrinsic<[llvm_v32f32_ty],
-            [llvm_float_ty, llvm_float_ty, llvm_v32f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+// llvm.amdgcn.mfma.*.* vdst, srcA, srcB, srcC, cbsz, abid, blgp
+class AMDGPUMfmaIntrinsic<LLVMType DestTy, LLVMType SrcABTy> :
+  GCCBuiltin<!subst("int", "__builtin", NAME)>,
+  Intrinsic<[DestTy],
+            [SrcABTy, SrcABTy, DestTy,
+             llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
             [IntrConvergent, IntrNoMem, IntrWillReturn,
              ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
 
-def int_amdgcn_mfma_f32_16x16x1f32 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_16x16x1f32">,
-  Intrinsic<[llvm_v16f32_ty],
-            [llvm_float_ty, llvm_float_ty, llvm_v16f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_4x4x1f32 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_4x4x1f32">,
-  Intrinsic<[llvm_v4f32_ty],
-            [llvm_float_ty, llvm_float_ty, llvm_v4f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_32x32x2f32 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_32x32x2f32">,
-  Intrinsic<[llvm_v16f32_ty],
-            [llvm_float_ty, llvm_float_ty, llvm_v16f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_16x16x4f32 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_16x16x4f32">,
-  Intrinsic<[llvm_v4f32_ty],
-            [llvm_float_ty, llvm_float_ty, llvm_v4f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_32x32x4f16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_32x32x4f16">,
-  Intrinsic<[llvm_v32f32_ty],
-            [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v32f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_16x16x4f16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_16x16x4f16">,
-  Intrinsic<[llvm_v16f32_ty],
-            [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v16f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_4x4x4f16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_4x4x4f16">,
-  Intrinsic<[llvm_v4f32_ty],
-            [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v4f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_32x32x8f16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_32x32x8f16">,
-  Intrinsic<[llvm_v16f32_ty],
-            [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v16f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_16x16x16f16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_16x16x16f16">,
-  Intrinsic<[llvm_v4f32_ty],
-            [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v4f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_i32_32x32x4i8 : GCCBuiltin<"__builtin_amdgcn_mfma_i32_32x32x4i8">,
-  Intrinsic<[llvm_v32i32_ty],
-            [llvm_i32_ty, llvm_i32_ty, llvm_v32i32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_i32_16x16x4i8 : GCCBuiltin<"__builtin_amdgcn_mfma_i32_16x16x4i8">,
-  Intrinsic<[llvm_v16i32_ty],
-            [llvm_i32_ty, llvm_i32_ty, llvm_v16i32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_i32_4x4x4i8 : GCCBuiltin<"__builtin_amdgcn_mfma_i32_4x4x4i8">,
-  Intrinsic<[llvm_v4i32_ty],
-            [llvm_i32_ty, llvm_i32_ty, llvm_v4i32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_i32_32x32x8i8 : GCCBuiltin<"__builtin_amdgcn_mfma_i32_32x32x8i8">,
-  Intrinsic<[llvm_v16i32_ty],
-            [llvm_i32_ty, llvm_i32_ty, llvm_v16i32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_i32_16x16x16i8 : GCCBuiltin<"__builtin_amdgcn_mfma_i32_16x16x16i8">,
-  Intrinsic<[llvm_v4i32_ty],
-            [llvm_i32_ty, llvm_i32_ty, llvm_v4i32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_32x32x2bf16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_32x32x2bf16">,
-  Intrinsic<[llvm_v32f32_ty],
-            [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v32f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_16x16x2bf16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_16x16x2bf16">,
-  Intrinsic<[llvm_v16f32_ty],
-            [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v16f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_4x4x2bf16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_4x4x2bf16">,
-  Intrinsic<[llvm_v4f32_ty],
-            [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v4f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_32x32x4bf16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_32x32x4bf16">,
-  Intrinsic<[llvm_v16f32_ty],
-            [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v16f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_16x16x8bf16 : GCCBuiltin<"__builtin_amdgcn_mfma_f32_16x16x8bf16">,
-  Intrinsic<[llvm_v4f32_ty],
-            [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v4f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
+def int_amdgcn_mfma_f32_32x32x1f32  : AMDGPUMfmaIntrinsic<llvm_v32f32_ty, llvm_float_ty>;
+def int_amdgcn_mfma_f32_16x16x1f32  : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_float_ty>;
+def int_amdgcn_mfma_f32_4x4x1f32    : AMDGPUMfmaIntrinsic<llvm_v4f32_ty,  llvm_float_ty>;
+def int_amdgcn_mfma_f32_32x32x2f32  : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_float_ty>;
+def int_amdgcn_mfma_f32_16x16x4f32  : AMDGPUMfmaIntrinsic<llvm_v4f32_ty,  llvm_float_ty>;
+def int_amdgcn_mfma_f32_32x32x4f16  : AMDGPUMfmaIntrinsic<llvm_v32f32_ty, llvm_v4f16_ty>;
+def int_amdgcn_mfma_f32_16x16x4f16  : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_v4f16_ty>;
+def int_amdgcn_mfma_f32_4x4x4f16    : AMDGPUMfmaIntrinsic<llvm_v4f32_ty,  llvm_v4f16_ty>;
+def int_amdgcn_mfma_f32_32x32x8f16  : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_v4f16_ty>;
+def int_amdgcn_mfma_f32_16x16x16f16 : AMDGPUMfmaIntrinsic<llvm_v4f32_ty,  llvm_v4f16_ty>;
+def int_amdgcn_mfma_i32_32x32x4i8   : AMDGPUMfmaIntrinsic<llvm_v32i32_ty, llvm_i32_ty>;
+def int_amdgcn_mfma_i32_16x16x4i8   : AMDGPUMfmaIntrinsic<llvm_v16i32_ty, llvm_i32_ty>;
+def int_amdgcn_mfma_i32_4x4x4i8     : AMDGPUMfmaIntrinsic<llvm_v4i32_ty,  llvm_i32_ty>;
+def int_amdgcn_mfma_i32_32x32x8i8   : AMDGPUMfmaIntrinsic<llvm_v16i32_ty, llvm_i32_ty>;
+def int_amdgcn_mfma_i32_16x16x16i8  : AMDGPUMfmaIntrinsic<llvm_v4i32_ty,  llvm_i32_ty>;
+def int_amdgcn_mfma_f32_32x32x2bf16 : AMDGPUMfmaIntrinsic<llvm_v32f32_ty, llvm_v2i16_ty>;
+def int_amdgcn_mfma_f32_16x16x2bf16 : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_v2i16_ty>;
+def int_amdgcn_mfma_f32_4x4x2bf16   : AMDGPUMfmaIntrinsic<llvm_v4f32_ty,  llvm_v2i16_ty>;
+def int_amdgcn_mfma_f32_32x32x4bf16 : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_v2i16_ty>;
+def int_amdgcn_mfma_f32_16x16x8bf16 : AMDGPUMfmaIntrinsic<llvm_v4f32_ty,  llvm_v2i16_ty>;
 
 //===----------------------------------------------------------------------===//
 // gfx90a intrinsics
@@ -2033,54 +1922,14 @@ def int_amdgcn_flat_atomic_fadd   : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>;
 def int_amdgcn_flat_atomic_fmin   : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>;
 def int_amdgcn_flat_atomic_fmax   : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>;
 
-def int_amdgcn_mfma_f32_32x32x4bf16_1k : GCCBuiltin<"__builtin_amdgcn_mfma_f32_32x32x4bf16_1k">,
-  Intrinsic<[llvm_v32f32_ty],
-            [llvm_v4i16_ty, llvm_v4i16_ty, llvm_v32f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_16x16x4bf16_1k : GCCBuiltin<"__builtin_amdgcn_mfma_f32_16x16x4bf16_1k">,
-  Intrinsic<[llvm_v16f32_ty],
-            [llvm_v4i16_ty, llvm_v4i16_ty, llvm_v16f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
+def int_amdgcn_mfma_f32_32x32x4bf16_1k  : AMDGPUMfmaIntrinsic<llvm_v32f32_ty, llvm_v4i16_ty>;
+def int_amdgcn_mfma_f32_16x16x4bf16_1k  : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_v4i16_ty>;
+def int_amdgcn_mfma_f32_4x4x4bf16_1k    : AMDGPUMfmaIntrinsic<llvm_v4f32_ty,  llvm_v4i16_ty>;
+def int_amdgcn_mfma_f32_32x32x8bf16_1k  : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_v4i16_ty>;
+def int_amdgcn_mfma_f32_16x16x16bf16_1k : AMDGPUMfmaIntrinsic<llvm_v4f32_ty,  llvm_v4i16_ty>;
 
-def int_amdgcn_mfma_f32_4x4x4bf16_1k : GCCBuiltin<"__builtin_amdgcn_mfma_f32_4x4x4bf16_1k">,
-  Intrinsic<[llvm_v4f32_ty],
-            [llvm_v4i16_ty, llvm_v4i16_ty, llvm_v4f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_32x32x8bf16_1k : GCCBuiltin<"__builtin_amdgcn_mfma_f32_32x32x8bf16_1k">,
-  Intrinsic<[llvm_v16f32_ty],
-            [llvm_v4i16_ty, llvm_v4i16_ty, llvm_v16f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f32_16x16x16bf16_1k : GCCBuiltin<"__builtin_amdgcn_mfma_f32_16x16x16bf16_1k">,
-  Intrinsic<[llvm_v4f32_ty],
-            [llvm_v4i16_ty, llvm_v4i16_ty, llvm_v4f32_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f64_16x16x4f64 : GCCBuiltin<"__builtin_amdgcn_mfma_f64_16x16x4f64">,
-  Intrinsic<[llvm_v4f64_ty],
-            [llvm_double_ty, llvm_double_ty, llvm_v4f64_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
-
-def int_amdgcn_mfma_f64_4x4x4f64 : GCCBuiltin<"__builtin_amdgcn_mfma_f64_4x4x4f64">,
-  Intrinsic<[llvm_double_ty],
-            [llvm_double_ty, llvm_double_ty, llvm_double_ty,
-            llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-            [IntrConvergent, IntrNoMem, IntrWillReturn,
-             ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, ImmArg<ArgIndex<5>>]>;
+def int_amdgcn_mfma_f64_16x16x4f64      : AMDGPUMfmaIntrinsic<llvm_v4f64_ty,  llvm_double_ty>;
+def int_amdgcn_mfma_f64_4x4x4f64        : AMDGPUMfmaIntrinsic<llvm_double_ty, llvm_double_ty>;
 
 //===----------------------------------------------------------------------===//
 // Special Intrinsics for backend internal use only. No frontend
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index 323b9a770c05f6d099042199249398da22adb20a..7e7d151d22fefab25a4f9aecd43b060634abba5e 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -115,11 +115,11 @@ def int_wasm_shuffle :
              llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty,
              llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
             [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_sub_saturate_signed :
+def int_wasm_sub_sat_signed :
   Intrinsic<[llvm_anyvector_ty],
             [LLVMMatchType<0>, LLVMMatchType<0>],
             [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_sub_saturate_unsigned :
+def int_wasm_sub_sat_unsigned :
   Intrinsic<[llvm_anyvector_ty],
             [LLVMMatchType<0>, LLVMMatchType<0>],
             [IntrNoMem, IntrSpeculatable]>;
@@ -143,14 +143,6 @@ def int_wasm_bitmask :
   Intrinsic<[llvm_i32_ty],
             [llvm_anyvector_ty],
             [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_qfma :
-  Intrinsic<[llvm_anyvector_ty],
-            [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
-            [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_qfms :
-  Intrinsic<[llvm_anyvector_ty],
-            [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
-            [IntrNoMem, IntrSpeculatable]>;
 def int_wasm_dot :
   Intrinsic<[llvm_v4i32_ty],
             [llvm_v8i16_ty, llvm_v8i16_ty],
@@ -166,17 +158,17 @@ def int_wasm_narrow_unsigned :
             [IntrNoMem, IntrSpeculatable]>;
 
 // TODO: Replace these intrinsics with normal ISel patterns once i32x4 to i64x2
-// widening is merged to the proposal.
-def int_wasm_widen_low_signed :
+// extending is merged to the proposal.
+def int_wasm_extend_low_signed :
   Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_widen_high_signed :
+def int_wasm_extend_high_signed :
   Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_widen_low_unsigned :
+def int_wasm_extend_low_unsigned :
   Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_widen_high_unsigned :
+def int_wasm_extend_high_unsigned :
   Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], [IntrNoMem, IntrSpeculatable]>;
 
-def int_wasm_q15mulr_saturate_signed :
+def int_wasm_q15mulr_sat_signed :
   Intrinsic<[llvm_v8i16_ty],
             [llvm_v8i16_ty, llvm_v8i16_ty],
             [IntrNoMem, IntrSpeculatable]>;
@@ -302,11 +294,6 @@ def int_wasm_extadd_pairwise_unsigned :
             [LLVMSubdivide2VectorType<0>],
             [IntrNoMem, IntrSpeculatable]>;
 
-def int_wasm_signselect :
-  Intrinsic<[llvm_anyvector_ty],
-            [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
-            [IntrNoMem, IntrSpeculatable]>;
-
 // TODO: Remove this intrinsic and the associated builtin if i64x2.eq gets
 // merged to the proposal.
 def int_wasm_eq :
@@ -314,20 +301,6 @@ def int_wasm_eq :
             [llvm_v2i64_ty, llvm_v2i64_ty],
             [IntrNoMem, IntrSpeculatable]>;
 
-// TODO: Remove this after experiments have been run. Use the target-agnostic
-// int_prefetch if this becomes specified at some point.
-def int_wasm_prefetch_t :
-  Intrinsic<[], [llvm_ptr_ty],
-            [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn,
-             ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
-            "", [SDNPMemOperand]>;
-
-def int_wasm_prefetch_nt :
-  Intrinsic<[], [llvm_ptr_ty],
-            [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn,
-             ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
-            "", [SDNPMemOperand]>;
-
 // TODO: Remove these if possible if they are merged to the spec.
 def int_wasm_convert_low_signed :
   Intrinsic<[llvm_v2f64_ty], [llvm_v4i32_ty],
@@ -335,10 +308,10 @@ def int_wasm_convert_low_signed :
 def int_wasm_convert_low_unsigned :
   Intrinsic<[llvm_v2f64_ty], [llvm_v4i32_ty],
             [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_trunc_saturate_zero_signed :
+def int_wasm_trunc_sat_zero_signed :
   Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty],
             [IntrNoMem, IntrSpeculatable]>;
-def int_wasm_trunc_saturate_zero_unsigned :
+def int_wasm_trunc_sat_zero_unsigned :
   Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty],
             [IntrNoMem, IntrSpeculatable]>;
 def int_wasm_demote_zero :
@@ -348,14 +321,6 @@ def int_wasm_promote_low :
   Intrinsic<[llvm_v2f64_ty], [llvm_v4f32_ty],
             [IntrNoMem, IntrSpeculatable]>;
 
-// TODO: Remove these if possible if they are merged to the spec.
-def int_wasm_widen_signed :
-  Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty, llvm_i32_ty],
-            [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<1>>]>;
-def int_wasm_widen_unsigned :
-  Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty, llvm_i32_ty],
-            [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<1>>]>;
-
 //===----------------------------------------------------------------------===//
 // Thread-local storage intrinsics
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index a27f44ed9d312a3f1aea41d0d1e109122973788c..ca8a49f10f535f29289f6fa261630c4353716928 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -197,6 +197,14 @@ private:
                                   ///< Format: (arch)(sub)-(vendor)-(sys0-(abi)
   NamedMDSymTabType NamedMDSymTab;  ///< NamedMDNode names.
   DataLayout DL;                  ///< DataLayout associated with the module
+  StringMap<unsigned>
+      CurrentIntrinsicIds; ///< Keep track of the current unique id count for
+                           ///< the specified intrinsic basename.
+  DenseMap<std::pair<Intrinsic::ID, const FunctionType *>, unsigned>
+      UniquedIntrinsicNames; ///< Keep track of uniqued names of intrinsics
+                             ///< based on unnamed types. The combination of
+                             ///< ID and FunctionType maps to the extension that
+                             ///< is used to make the intrinsic name unique.
 
   friend class Constant;
 
@@ -221,7 +229,7 @@ public:
   /// Returns the number of non-debug IR instructions in the module.
   /// This is equivalent to the sum of the IR instruction counts of each
   /// function contained in the module.
-  unsigned getInstructionCount();
+  unsigned getInstructionCount() const;
 
   /// Get the module's original source file name. When compiling from
   /// bitcode, this is taken from a bitcode record where it was recorded.
@@ -331,6 +339,11 @@ public:
 
   std::vector<StructType *> getIdentifiedStructTypes() const;
 
+  /// Return a unique name for an intrinsic whose mangling is based on an
+  /// unnamed type. The Proto represents the function prototype.
+  std::string getUniqueIntrinsicName(StringRef BaseName, Intrinsic::ID Id,
+                                     const FunctionType *Proto);
+
 /// @}
 /// @name Function Accessors
 /// @{
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 575eff4650e15e6ac24b8d04ad6c965e4a76388f..544d4187b3b276b71f8f46a0a58fcaebc87746fa 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -1735,10 +1735,10 @@ struct MaxMin_match {
       return false;
     // At this point we have a select conditioned on a comparison.  Check that
     // it is the values returned by the select that are being compared.
-    Value *TrueVal = SI->getTrueValue();
-    Value *FalseVal = SI->getFalseValue();
-    Value *LHS = Cmp->getOperand(0);
-    Value *RHS = Cmp->getOperand(1);
+    auto *TrueVal = SI->getTrueValue();
+    auto *FalseVal = SI->getFalseValue();
+    auto *LHS = Cmp->getOperand(0);
+    auto *RHS = Cmp->getOperand(1);
     if ((TrueVal != LHS || FalseVal != RHS) &&
         (TrueVal != RHS || FalseVal != LHS))
       return false;
diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h
index 5c78864d35bf5b29b6f79d185540e3cea539081a..60352710715107dff51db438656deefa6e50de6a 100644
--- a/llvm/include/llvm/IR/Type.h
+++ b/llvm/include/llvm/IR/Type.h
@@ -380,6 +380,11 @@ public:
     return ContainedTys[0];
   }
 
+  /// Given vector type, change the element type,
+  /// whilst keeping the old number of elements.
+  /// For non-vectors simply returns \p EltTy.
+  inline Type *getWithNewType(Type *EltTy) const;
+
   /// Given an integer or vector type, change the lane bitwidth to NewBitwidth,
   /// whilst keeping the old number of lanes.
   inline Type *getWithNewBitWidth(unsigned NewBitWidth) const;
diff --git a/llvm/include/llvm/IR/Value.h b/llvm/include/llvm/IR/Value.h
index 5a7e90aeb0f644595cdfcd93120b2a61ab6f68b1..e9a9acfd69baa567bb5948cb48046a0c29d3143c 100644
--- a/llvm/include/llvm/IR/Value.h
+++ b/llvm/include/llvm/IR/Value.h
@@ -743,8 +743,13 @@ public:
   ///
   /// If CanBeNull is set by this function the pointer can either be null or be
   /// dereferenceable up to the returned number of bytes.
+  ///
+  /// IF CanBeFreed is true, the pointer is known to be dereferenceable at
+  /// point of definition only.  Caller must prove that allocation is not
+  /// deallocated between point of definition and use.
   uint64_t getPointerDereferenceableBytes(const DataLayout &DL,
-                                          bool &CanBeNull) const;
+                                          bool &CanBeNull,
+                                          bool &CanBeFreed) const;
 
   /// Returns an alignment of the pointer value.
   ///
diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index 309932b29bb1393cc8746a219a5880ebf1710220..d97ebb6e476304bbb4ec57953094c52cb1098012 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -122,10 +122,14 @@ protected:
   /// other when on the same line.  Defaults to ';'
   const char *SeparatorString;
 
-  /// This indicates the comment character used by the assembler.  Defaults to
+  /// This indicates the comment string used by the assembler.  Defaults to
   /// "#"
   StringRef CommentString;
 
+  /// This indicates whether the comment string is only accepted as a comment
+  /// at the beginning of statements. Defaults to false.
+  bool RestrictCommentStringToStartOfStatement = false;
+
   /// This is appended to emitted labels.  Defaults to ":"
   const char *LabelSuffix;
 
@@ -557,6 +561,9 @@ public:
   unsigned getCommentColumn() const { return 40; }
 
   StringRef getCommentString() const { return CommentString; }
+  bool getRestrictCommentStringToStartOfStatement() const {
+    return RestrictCommentStringToStartOfStatement;
+  }
   const char *getLabelSuffix() const { return LabelSuffix; }
 
   bool useAssignmentForEHBegin() const { return UseAssignmentForEHBegin; }
diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 106763c5d7c23b74cb9ecbbfbb18a077237e2f7c..f07e5a89b101f89572dff55288f7775919d90c88 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -396,7 +396,6 @@ namespace llvm {
 
     void initInlineSourceManager();
     SourceMgr *getInlineSourceManager() {
-      assert(InlineSrcMgr);
       return InlineSrcMgr.get();
     }
     std::vector<const MDNode *> &getLocInfos() { return LocInfos; }
diff --git a/llvm/include/llvm/MC/MCInst.h b/llvm/include/llvm/MC/MCInst.h
index f22a452d06f9cda61a53f7c475ea2544f1203434..35f9d94dec749d9b1d918de2fd529a71b158c7aa 100644
--- a/llvm/include/llvm/MC/MCInst.h
+++ b/llvm/include/llvm/MC/MCInst.h
@@ -61,7 +61,6 @@ public:
   bool isImm() const { return Kind == kImmediate; }
   bool isSFPImm() const { return Kind == kSFPImmediate; }
   bool isDFPImm() const { return Kind == kDFPImmediate; }
-  bool isFPImm() const { return Kind == kDFPImmediate; }
   bool isExpr() const { return Kind == kExpr; }
   bool isInst() const { return Kind == kInst; }
 
@@ -101,10 +100,6 @@ public:
     assert(isDFPImm() && "This is not an FP immediate");
     return FPImmVal;
   }
-  double getFPImm() const {
-    assert(isDFPImm() && "This is not an FP immediate");
-    return bit_cast<double>(FPImmVal);
-  }
 
   void setDFPImm(uint64_t Val) {
     assert(isDFPImm() && "This is not an FP immediate");
@@ -162,12 +157,6 @@ public:
     Op.FPImmVal = Val;
     return Op;
   }
-  static MCOperand createFPImm(double Val) {
-    MCOperand Op;
-    Op.Kind = kDFPImmediate;
-    Op.FPImmVal = bit_cast<uint64_t>(Val);
-    return Op;
-  }
 
   static MCOperand createExpr(const MCExpr *Val) {
     MCOperand Op;
diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/llvm/include/llvm/MC/MCParser/MCAsmParser.h
index 02cc22009196deff8cfbe01a7e2c4c597940cded..24d4ada5fa0ba82f77364350c981df837f9178ac 100644
--- a/llvm/include/llvm/MC/MCParser/MCAsmParser.h
+++ b/llvm/include/llvm/MC/MCParser/MCAsmParser.h
@@ -182,6 +182,8 @@ public:
   virtual void setParsingMSInlineAsm(bool V) = 0;
   virtual bool isParsingMSInlineAsm() = 0;
 
+  virtual bool discardLTOSymbol(StringRef) const { return false; }
+
   virtual bool isParsingMasm() const { return false; }
 
   virtual bool defineMacro(StringRef Name, StringRef Value) { return true; }
diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
index 0a1e50d501e9380737500b5a71e51ac2d5ddc5bf..908ee30e40609661672b826a31eff48cc7f24392 100644
--- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
+++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
@@ -491,6 +491,9 @@ public:
     return nullptr;
   }
 
+  // For any initialization at the beginning of parsing.
+  virtual void onBeginOfFile() {}
+
   // For any checks or cleanups at the end of parsing.
   virtual void onEndOfFile() {}
 };
diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h
index e1baf5b0f5e5c39449e9a1db14c94ff93d47d230..5d39fe3ac0bd8b6774a78d17732b1a5292e16a9f 100644
--- a/llvm/include/llvm/MC/MCRegisterInfo.h
+++ b/llvm/include/llvm/MC/MCRegisterInfo.h
@@ -39,6 +39,7 @@ public:
   const uint16_t RegsSize;
   const uint16_t RegSetSize;
   const uint16_t ID;
+  const uint16_t RegSizeInBits;
   const int8_t CopyCost;
   const bool Allocatable;
   const bool AllowsNoRegister;
@@ -79,6 +80,12 @@ public:
     return contains(Reg1) && contains(Reg2);
   }
 
+  /// Return the size of the physical register in bits if we are able to
+  /// determine it. This always returns zero for registers of targets that use
+  /// HW modes, as we need more information to determine the size of registers
+  /// in such cases. Use TargetRegisterInfo to cover them.
+  unsigned getSizeInBits() const { return RegSizeInBits; }
+
   /// getCopyCost - Return the cost of copying a value between two registers in
   /// this class. A negative number means the register class is very expensive
   /// to copy e.g. status flag register classes.
diff --git a/llvm/include/llvm/MC/MCSubtargetInfo.h b/llvm/include/llvm/MC/MCSubtargetInfo.h
index 2c1072d833fba8c273feecf5138b2bea9d5923d0..839a3bd85829b2598a667502f533a940e34a76f2 100644
--- a/llvm/include/llvm/MC/MCSubtargetInfo.h
+++ b/llvm/include/llvm/MC/MCSubtargetInfo.h
@@ -89,6 +89,7 @@ class MCSubtargetInfo {
   const unsigned *OperandCycles;       // Itinerary operand cycles
   const unsigned *ForwardingPaths;
   FeatureBitset FeatureBits;           // Feature bits for current CPU + FS
+  std::string FeatureString;           // Feature string
 
 public:
   MCSubtargetInfo(const MCSubtargetInfo &) = default;
@@ -112,6 +113,8 @@ public:
     FeatureBits = FeatureBits_;
   }
 
+  StringRef getFeatureString() const { return FeatureString; }
+
   bool hasFeature(unsigned Feature) const {
     return FeatureBits[Feature];
   }
diff --git a/llvm/include/llvm/MC/MCSymbolWasm.h b/llvm/include/llvm/MC/MCSymbolWasm.h
index 49d57919f6f3e951e38619a6596bf87c10db1000..14b3a5d8e941db235270ca0cb8df6089f181fa88 100644
--- a/llvm/include/llvm/MC/MCSymbolWasm.h
+++ b/llvm/include/llvm/MC/MCSymbolWasm.h
@@ -26,7 +26,7 @@ class MCSymbolWasm : public MCSymbol {
   Optional<StringRef> ExportName;
   wasm::WasmSignature *Signature = nullptr;
   Optional<wasm::WasmGlobalType> GlobalType;
-  Optional<wasm::ValType> TableType;
+  Optional<wasm::WasmTableType> TableType;
   Optional<wasm::WasmEventType> EventType;
 
   /// An expression describing how to calculate the size of a symbol. If a
@@ -108,7 +108,7 @@ public:
 
   bool isFunctionTable() const {
     return isTable() && hasTableType() &&
-           getTableType() == wasm::ValType::FUNCREF;
+           getTableType().ElemType == wasm::WASM_TYPE_FUNCREF;
   }
   void setFunctionTable() {
     setType(wasm::WASM_SYMBOL_TYPE_TABLE);
@@ -131,11 +131,17 @@ public:
   void setGlobalType(wasm::WasmGlobalType GT) { GlobalType = GT; }
 
   bool hasTableType() const { return TableType.hasValue(); }
-  wasm::ValType getTableType() const {
+  const wasm::WasmTableType &getTableType() const {
     assert(hasTableType());
     return TableType.getValue();
   }
-  void setTableType(wasm::ValType TT) { TableType = TT; }
+  void setTableType(wasm::WasmTableType TT) { TableType = TT; }
+  void setTableType(wasm::ValType VT) {
+    // Declare a table with element type VT and no limits (min size 0, no max
+    // size).
+    wasm::WasmLimits Limits = {wasm::WASM_LIMITS_FLAG_NONE, 0, 0};
+    setTableType({uint8_t(VT), Limits});
+  }
 
   const wasm::WasmEventType &getEventType() const {
     assert(EventType.hasValue());
diff --git a/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h b/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
index 7884d01c707a33bd3fd38c8666cde012a297887f..510cff4b5934c694f895885f9f42fec60d0f5b12 100644
--- a/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
+++ b/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
@@ -28,7 +28,55 @@ namespace mca {
 
 class ReadState;
 class WriteState;
-class WriteRef;
+class Instruction;
+
+/// A reference to a register write.
+///
+/// This class is mainly used by the register file to describe register
+/// mappings. It correlates a register write to the source index of the
+/// defining instruction.
+class WriteRef {
+  unsigned IID;
+  unsigned WriteBackCycle;
+  unsigned WriteResID;
+  MCPhysReg RegisterID;
+  WriteState *Write;
+
+  static const unsigned INVALID_IID;
+
+public:
+  WriteRef()
+      : IID(INVALID_IID), WriteBackCycle(), WriteResID(), RegisterID(),
+        Write() {}
+  WriteRef(unsigned SourceIndex, WriteState *WS);
+
+  unsigned getSourceIndex() const { return IID; }
+  unsigned getWriteBackCycle() const;
+
+  const WriteState *getWriteState() const { return Write; }
+  WriteState *getWriteState() { return Write; }
+  unsigned getWriteResourceID() const;
+  MCPhysReg getRegisterID() const;
+
+  void commit();
+  void notifyExecuted(unsigned Cycle);
+
+  bool hasKnownWriteBackCycle() const;
+  bool isWriteZero() const;
+  bool isValid() const { return getSourceIndex() != INVALID_IID; }
+
+  /// Returns true if this register write has been executed, and the new
+  /// register value is therefore available to users.
+  bool isAvailable() const { return hasKnownWriteBackCycle(); }
+
+  bool operator==(const WriteRef &Other) const {
+    return Write && Other.Write && Write == Other.Write;
+  }
+
+#ifndef NDEBUG
+  void dump() const;
+#endif
+};
 
 /// Manages hardware register files, and tracks register definitions for
 /// register renaming purposes.
@@ -145,6 +193,8 @@ class RegisterFile : public HardwareUnit {
   // the target. Bits are set for registers that are known to be zero.
   APInt ZeroRegisters;
 
+  unsigned CurrentCycle;
+
   // This method creates a new register file descriptor.
   // The new register file owns all of the registers declared by register
   // classes in the 'RegisterClasses' set.
@@ -183,8 +233,9 @@ public:
                unsigned NumRegs = 0);
 
   // Collects writes that are in a RAW dependency with RS.
-  void collectWrites(const ReadState &RS,
-                     SmallVectorImpl<WriteRef> &Writes) const;
+  void collectWrites(const MCSubtargetInfo &STI, const ReadState &RS,
+                     SmallVectorImpl<WriteRef> &Writes,
+                     SmallVectorImpl<WriteRef> &CommittedWrites) const;
 
   // This method updates the register mappings inserting a new register
   // definition. This method is also responsible for updating the number of
@@ -223,9 +274,15 @@ public:
   // Returns the number of PRFs implemented by this processor.
   unsigned getNumRegisterFiles() const { return RegisterFiles.size(); }
 
+  unsigned getElapsedCyclesFromWriteBack(const WriteRef &WR) const;
+
+  void onInstructionExecuted(Instruction *IS);
+
   // Notify each PRF that a new cycle just started.
   void cycleStart();
 
+  void cycleEnd() { ++CurrentCycle; }
+
 #ifndef NDEBUG
   void dump() const;
 #endif
diff --git a/llvm/include/llvm/MCA/Instruction.h b/llvm/include/llvm/MCA/Instruction.h
index 2d3b0ab22e2cd9425a93d08ddaa3e7cd79eefad0..cc886a19025435ac4fc52cbfd4be20ff02bbd7f9 100644
--- a/llvm/include/llvm/MCA/Instruction.h
+++ b/llvm/include/llvm/MCA/Instruction.h
@@ -595,45 +595,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, const InstRef &IR) {
 }
 #endif
 
-/// A reference to a register write.
-///
-/// This class is mainly used by the register file to describe register
-/// mappings. It correlates a register write to the source index of the
-/// defining instruction.
-class WriteRef {
-  std::pair<unsigned, WriteState *> Data;
-  static const unsigned INVALID_IID;
-
-public:
-  WriteRef() : Data(INVALID_IID, nullptr) {}
-  WriteRef(unsigned SourceIndex, WriteState *WS) : Data(SourceIndex, WS) {}
-
-  unsigned getSourceIndex() const { return Data.first; }
-  const WriteState *getWriteState() const { return Data.second; }
-  WriteState *getWriteState() { return Data.second; }
-  void invalidate() { Data.second = nullptr; }
-  bool isWriteZero() const {
-    assert(isValid() && "Invalid null WriteState found!");
-    return getWriteState()->isWriteZero();
-  }
-
-  /// Returns true if this register write has been executed, and the new
-  /// register value is therefore available to users.
-  bool isAvailable() const {
-    if (getSourceIndex() == INVALID_IID)
-      return false;
-    const WriteState *WS = getWriteState();
-    return !WS || WS->isExecuted();
-  }
-
-  bool isValid() const { return Data.second && Data.first != INVALID_IID; }
-  bool operator==(const WriteRef &Other) const { return Data == Other.Data; }
-
-#ifndef NDEBUG
-  void dump() const;
-#endif
-};
-
 } // namespace mca
 } // namespace llvm
 
diff --git a/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h b/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h
index 0b4ea99d06dbd17f9e0cfba1dfe017e401ff8f5c..1a944243db60148932e53e42e1f4430b6c5016c6 100644
--- a/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h
+++ b/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h
@@ -27,12 +27,10 @@ class MCSubtargetInfo;
 namespace mca {
 class RegisterFile;
 class ResourceManager;
-struct RetireControlUnit;
 
 class InOrderIssueStage final : public Stage {
   const MCSchedModel &SM;
   const MCSubtargetInfo &STI;
-  RetireControlUnit &RCU;
   RegisterFile &PRF;
   std::unique_ptr<ResourceManager> RM;
 
@@ -47,9 +45,19 @@ class InOrderIssueStage final : public Stage {
   InstRef StalledInst;
   unsigned StallCyclesLeft;
 
+  /// Instruction that is issued in more than 1 cycle.
+  InstRef CarriedOver;
+  /// Number of CarriedOver uops left to issue.
+  unsigned CarryOver;
+
   /// Number of instructions that can be issued in the current cycle.
   unsigned Bandwidth;
 
+  /// Number of cycles (counted from the current cycle) until the last write is
+  /// committed. This is taken into account to ensure that writes commit in the
+  /// program order.
+  unsigned LastWriteBackCycle;
+
   InOrderIssueStage(const InOrderIssueStage &Other) = delete;
   InOrderIssueStage &operator=(const InOrderIssueStage &Other) = delete;
 
@@ -62,14 +70,20 @@ class InOrderIssueStage final : public Stage {
   Error tryIssue(InstRef &IR, unsigned *StallCycles);
 
   /// Update status of instructions from IssuedInst.
-  Error updateIssuedInst();
+  void updateIssuedInst();
+
+  /// Continue to issue the CarriedOver instruction.
+  void updateCarriedOver();
+
+  /// Retire instruction once it is executed.
+  void retireInstruction(InstRef &IR);
 
 public:
-  InOrderIssueStage(RetireControlUnit &RCU, RegisterFile &PRF,
-                    const MCSchedModel &SM, const MCSubtargetInfo &STI)
-      : SM(SM), STI(STI), RCU(RCU), PRF(PRF),
-        RM(std::make_unique<ResourceManager>(SM)), NumIssued(0),
-        StallCyclesLeft(0), Bandwidth(0) {}
+  InOrderIssueStage(RegisterFile &PRF, const MCSchedModel &SM,
+                    const MCSubtargetInfo &STI)
+      : SM(SM), STI(STI), PRF(PRF), RM(std::make_unique<ResourceManager>(SM)),
+        NumIssued(0), StallCyclesLeft(0), CarryOver(0), Bandwidth(0),
+        LastWriteBackCycle(0) {}
 
   bool isAvailable(const InstRef &) const override;
   bool hasWorkToComplete() const override;
diff --git a/llvm/include/llvm/MCA/Stages/RetireStage.h b/llvm/include/llvm/MCA/Stages/RetireStage.h
index 81d7cd86ca47c9b6034605fd5fb62fd24e294e04..b635a01db85e6b4900fbd50de0a16710e9f216d2 100644
--- a/llvm/include/llvm/MCA/Stages/RetireStage.h
+++ b/llvm/include/llvm/MCA/Stages/RetireStage.h
@@ -30,7 +30,6 @@ class RetireStage final : public Stage {
   RetireControlUnit &RCU;
   RegisterFile &PRF;
   LSUnitBase &LSU;
-  SmallVector<InstRef, 4> RetireInst;
 
   RetireStage(const RetireStage &Other) = delete;
   RetireStage &operator=(const RetireStage &Other) = delete;
@@ -39,10 +38,9 @@ public:
   RetireStage(RetireControlUnit &R, RegisterFile &F, LSUnitBase &LS)
       : Stage(), RCU(R), PRF(F), LSU(LS) {}
 
-  bool hasWorkToComplete() const override {
-    return !RCU.isEmpty() || !RetireInst.empty();
-  }
+  bool hasWorkToComplete() const override { return !RCU.isEmpty(); }
   Error cycleStart() override;
+  Error cycleEnd() override;
   Error execute(InstRef &IR) override;
   void notifyInstructionRetired(const InstRef &IR) const;
 };
diff --git a/llvm/include/llvm/Object/Binary.h b/llvm/include/llvm/Object/Binary.h
index dd98e1143e2593a52bd0d5572341c2a996ae64b2..a8f4437d5dbb3e4c99da5a513be81b38f87c3c71 100644
--- a/llvm/include/llvm/Object/Binary.h
+++ b/llvm/include/llvm/Object/Binary.h
@@ -147,7 +147,8 @@ public:
 
   bool isLittleEndian() const {
     return !(TypeID == ID_ELF32B || TypeID == ID_ELF64B ||
-             TypeID == ID_MachO32B || TypeID == ID_MachO64B);
+             TypeID == ID_MachO32B || TypeID == ID_MachO64B ||
+             TypeID == ID_XCOFF32 || TypeID == ID_XCOFF64);
   }
 
   bool isWinRes() const { return TypeID == ID_WinRes; }
diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h
index fd93ed42b81204f37ac66678b530ca1bc4ac0770..4736fd398854a080ec876237a972e6eefbd617a8 100644
--- a/llvm/include/llvm/Object/ELFObjectFile.h
+++ b/llvm/include/llvm/Object/ELFObjectFile.h
@@ -716,7 +716,16 @@ Expected<uint32_t> ELFObjectFile<ELFT>::getSymbolFlags(DataRefImpl Sym) const {
     // TODO: Test this error.
     return SymbolsOrErr.takeError();
 
-  if (EF.getHeader().e_machine == ELF::EM_ARM) {
+  if (EF.getHeader().e_machine == ELF::EM_AARCH64) {
+    if (Expected<StringRef> NameOrErr = getSymbolName(Sym)) {
+      StringRef Name = *NameOrErr;
+      if (Name.startswith("$d") || Name.startswith("$x"))
+        Result |= SymbolRef::SF_FormatSpecific;
+    } else {
+      // TODO: Actually report errors helpfully.
+      consumeError(NameOrErr.takeError());
+    }
+  } else if (EF.getHeader().e_machine == ELF::EM_ARM) {
     if (Expected<StringRef> NameOrErr = getSymbolName(Sym)) {
       StringRef Name = *NameOrErr;
       if (Name.startswith("$d") || Name.startswith("$t") ||
@@ -728,6 +737,15 @@ Expected<uint32_t> ELFObjectFile<ELFT>::getSymbolFlags(DataRefImpl Sym) const {
     }
     if (ESym->getType() == ELF::STT_FUNC && (ESym->st_value & 1) == 1)
       Result |= SymbolRef::SF_Thumb;
+  } else if (EF.getHeader().e_machine == ELF::EM_RISCV) {
+    if (Expected<StringRef> NameOrErr = getSymbolName(Sym)) {
+      // Mark empty name symbols used for label differences.
+      if (NameOrErr->empty())
+        Result |= SymbolRef::SF_FormatSpecific;
+    } else {
+      // TODO: Actually report errors helpfully.
+      consumeError(NameOrErr.takeError());
+    }
   }
 
   if (ESym->st_shndx == ELF::SHN_UNDEF)
diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h
index 32548f1435f7bd41e2aeedaee7e5cf7918e8929b..534deaabed8d156003413fd6f4df7b2d063a9636 100644
--- a/llvm/include/llvm/Object/ELFTypes.h
+++ b/llvm/include/llvm/Object/ELFTypes.h
@@ -802,13 +802,15 @@ template <class ELFT> struct Elf_BBAddrMap_Impl {
 
     // The following fields are decoded from the Metadata field. The encoding
     // happens in AsmPrinter.cpp:getBBAddrMapMetadata.
-    bool HasReturn;   // If this block ends with a return (or tail call).
-    bool HasTailCall; // If this block ends with a tail call.
-    bool IsEHPad;     // If this is an exception handling block.
+    bool HasReturn;      // If this block ends with a return (or tail call).
+    bool HasTailCall;    // If this block ends with a tail call.
+    bool IsEHPad;        // If this is an exception handling block.
+    bool CanFallThrough; // If this block can fall through to its next.
 
     BBEntry(uint32_t Offset, uint32_t Size, uint32_t Metadata)
         : Offset(Offset), Size(Size), HasReturn(Metadata & 1),
-          HasTailCall(Metadata & (1 << 1)), IsEHPad(Metadata & (1 << 2)){};
+          HasTailCall(Metadata & (1 << 1)), IsEHPad(Metadata & (1 << 2)),
+          CanFallThrough(Metadata & (1 << 3)){};
   };
   std::vector<BBEntry> BBEntries; // Basic block entries for this function.
 };
diff --git a/llvm/include/llvm/Object/XCOFFObjectFile.h b/llvm/include/llvm/Object/XCOFFObjectFile.h
index 1ac00ed5e2c757276c8d9d5fff5f4f5d6c7d7fca..dbc40c4ea88049cbf11cf8c05b466f9e011c01f6 100644
--- a/llvm/include/llvm/Object/XCOFFObjectFile.h
+++ b/llvm/include/llvm/Object/XCOFFObjectFile.h
@@ -317,6 +317,7 @@ public:
   Triple::ArchType getArch() const override;
   SubtargetFeatures getFeatures() const override;
   Expected<uint64_t> getStartAddress() const override;
+  StringRef mapDebugSectionName(StringRef Name) const override;
   bool isRelocatableObject() const override;
 
   // Below here is the non-inherited interface.
diff --git a/llvm/include/llvm/ObjectYAML/WasmYAML.h b/llvm/include/llvm/ObjectYAML/WasmYAML.h
index cc5df9674fb1ef586e88fcf58ca22b8d3b52cf8e..708e82c9b9f1d51f1e2acda31a4dbc26fdf15bec 100644
--- a/llvm/include/llvm/ObjectYAML/WasmYAML.h
+++ b/llvm/include/llvm/ObjectYAML/WasmYAML.h
@@ -46,7 +46,7 @@ struct FileHeader {
 
 struct Limits {
   LimitFlags Flags;
-  yaml::Hex32 Initial;
+  yaml::Hex32 Minimum;
   yaml::Hex32 Maximum;
 };
 
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index f023a2924d8f939edf2950029a9a5dd4e7f4104c..d24f3fbbf2e54c2e4878e606c1ca28514203c689 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -327,8 +327,8 @@ using ChangedIRData = OrderedChangedData<ChangedFuncData>;
 class ChangedIRComparer {
 public:
   ChangedIRComparer(raw_ostream &OS, const ChangedIRData &Before,
-                    const ChangedIRData &After)
-      : Before(Before), After(After), Out(OS) {}
+                    const ChangedIRData &After, bool ColourMode)
+      : Before(Before), After(After), Out(OS), UseColour(ColourMode) {}
 
   // Compare the 2 IRs.
   void compare(Any IR, StringRef Prefix, StringRef PassID, StringRef Name);
@@ -353,6 +353,7 @@ protected:
   const ChangedIRData &Before;
   const ChangedIRData &After;
   raw_ostream &Out;
+  bool UseColour;
 };
 
 // A change printer that prints out in-line differences in the basic
@@ -363,8 +364,8 @@ protected:
 // -print-module-scope does not affect this change reporter.
 class InLineChangePrinter : public TextChangeReporter<ChangedIRData> {
 public:
-  InLineChangePrinter(bool VerboseMode)
-      : TextChangeReporter<ChangedIRData>(VerboseMode) {}
+  InLineChangePrinter(bool VerboseMode, bool ColourMode)
+      : TextChangeReporter<ChangedIRData>(VerboseMode), UseColour(ColourMode) {}
   ~InLineChangePrinter() override;
   void registerCallbacks(PassInstrumentationCallbacks &PIC);
 
@@ -380,6 +381,8 @@ protected:
   // Called to compare the before and after representations of the IR.
   virtual bool same(const ChangedIRData &Before,
                     const ChangedIRData &After) override;
+
+  bool UseColour;
 };
 
 class VerifyInstrumentation {
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index 957dfe9f1503f7b3e71bf2f289a8cb218dcb5b6d..3e06de356af6aa8bdf06e42af0273411518e6c14 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -573,6 +573,11 @@ class CoverageMapping {
 
   CoverageMapping() = default;
 
+  // Load coverage records from readers.
+  static Error loadFromReaders(
+      ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders,
+      IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage);
+
   /// Add a function record corresponding to \p Record.
   Error loadFunctionRecord(const CoverageMappingRecord &Record,
                            IndexedInstrProfReader &ProfileReader);
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index 6126a61efb72688b5de77a10baed353284a53940..ffc7dee4ed6d00aabdc60474ec8905d73bc56376 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -873,7 +873,7 @@ InstrProfGetRangeRepValue(uint64_t Value) {
     return Value;
   else
     // Otherwise, take to the previous power of two + 1.
-    return (1 << (64 - InstProfClzll(Value) - 1)) + 1;
+    return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
 }
 
 /* Return true if the range that an (observed) memop size value belongs to has
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 70fdaff385044435545f22a229a2827670708100..8b590e84dd9be9d2a7314b5ffeb7e6c5917f19e7 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -196,6 +196,7 @@ enum class SecProfSummaryFlags : uint32_t {
 enum class SecFuncMetadataFlags : uint32_t {
   SecFlagInvalid = 0,
   SecFlagIsProbeBased = (1 << 0),
+  SecFlagHasAttribute = (1 << 1)
 };
 
 // Verify section specific flag is used for the correct section.
@@ -385,6 +386,13 @@ enum ContextStateMask {
   MergedContext = 0x8     // Profile for context merged into base profile
 };
 
+// Attribute of context associated with FunctionSamples
+enum ContextAttributeMask {
+  ContextNone = 0x0,
+  ContextWasInlined = 0x1,      // Leaf of context was inlined in previous build
+  ContextShouldBeInlined = 0x2, // Leaf of context should be inlined
+};
+
 // Sample context for FunctionSamples. It consists of the calling context,
 // the function name and context state. Internally sample context is represented
 // using StringRef, which is also the input for constructing a `SampleContext`.
@@ -396,9 +404,9 @@ enum ContextStateMask {
 //    `_Z8funcLeafi`
 class SampleContext {
 public:
-  SampleContext() : State(UnknownContext) {}
-  SampleContext(StringRef ContextStr,
-                ContextStateMask CState = UnknownContext) {
+  SampleContext() : State(UnknownContext), Attributes(ContextNone) {}
+  SampleContext(StringRef ContextStr, ContextStateMask CState = UnknownContext)
+      : Attributes(ContextNone) {
     setContext(ContextStr, CState);
   }
 
@@ -443,6 +451,10 @@ public:
   }
 
   operator StringRef() const { return FullContext; }
+  bool hasAttribute(ContextAttributeMask A) { return Attributes & (uint32_t)A; }
+  void setAttribute(ContextAttributeMask A) { Attributes |= (uint32_t)A; }
+  uint32_t getAllAttributes() { return Attributes; }
+  void setAllAttributes(uint32_t A) { Attributes = A; }
   bool hasState(ContextStateMask S) { return State & (uint32_t)S; }
   void setState(ContextStateMask S) { State |= (uint32_t)S; }
   void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
@@ -503,6 +515,8 @@ private:
   StringRef CallingContext;
   // State of the associated sample profile
   uint32_t State;
+  // Attribute of the associated sample profile
+  uint32_t Attributes;
 };
 
 class FunctionSamples;
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index db1ec686972425370fcde8aa29d0c8617e53f6b2..8203a1b8fb3b8e8e48e7554ddcfac8408c794176 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -28,6 +28,7 @@
 //       offsetA1[.discriminator]: number_of_samples [fn7:num fn8:num ... ]
 //       ...
 //      !CFGChecksum: num
+//      !Attribute: flags
 //
 // This is a nested tree in which the indentation represents the nesting level
 // of the inline stack. There are no blank lines in the file. And the spacing
@@ -127,6 +128,8 @@
 //
 // a. CFG Checksum (a.k.a. function hash):
 //   !CFGChecksum: 12345
+// b. CFG Checksum (see ContextAttributeMask):
+//   !Atribute: 1
 //
 //
 // Binary format
@@ -647,7 +650,7 @@ protected:
   std::error_code readSecHdrTableEntry(uint32_t Idx);
   std::error_code readSecHdrTable();
 
-  std::error_code readFuncMetadata();
+  std::error_code readFuncMetadata(bool ProfileHasAttribute);
   std::error_code readFuncOffsetTable();
   std::error_code readFuncProfiles();
   std::error_code readMD5NameTable();
diff --git a/llvm/include/llvm/Support/AMDGPUMetadata.h b/llvm/include/llvm/Support/AMDGPUMetadata.h
index eadc25870096a72e7be526c06c7be73fd3a52140..784a980fee24fcc3a0e68a5b14dbb712360cdfd7 100644
--- a/llvm/include/llvm/Support/AMDGPUMetadata.h
+++ b/llvm/include/llvm/Support/AMDGPUMetadata.h
@@ -29,10 +29,20 @@ namespace AMDGPU {
 //===----------------------------------------------------------------------===//
 namespace HSAMD {
 
-/// HSA metadata major version.
-constexpr uint32_t VersionMajor = 1;
-/// HSA metadata minor version.
-constexpr uint32_t VersionMinor = 0;
+/// HSA metadata major version for code object V2.
+constexpr uint32_t VersionMajorV2 = 1;
+/// HSA metadata minor version for code object V2.
+constexpr uint32_t VersionMinorV2 = 0;
+
+/// HSA metadata major version for code object V3.
+constexpr uint32_t VersionMajorV3 = 1;
+/// HSA metadata minor version for code object V3.
+constexpr uint32_t VersionMinorV3 = 0;
+
+/// HSA metadata major version for code object V4.
+constexpr uint32_t VersionMajorV4 = 1;
+/// HSA metadata minor version for code object V4.
+constexpr uint32_t VersionMinorV4 = 1;
 
 /// HSA metadata beginning assembler directive.
 constexpr char AssemblerDirectiveBegin[] = ".amd_amdgpu_hsa_metadata";
diff --git a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
index 7177557b6ff0a51385981cfbb7a3d2ec7208c5c4..aec80291f01fa31048465717185cc5723b71f777 100644
--- a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
+++ b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
@@ -165,7 +165,8 @@ enum : int32_t {
 struct kernel_descriptor_t {
   uint32_t group_segment_fixed_size;
   uint32_t private_segment_fixed_size;
-  uint8_t reserved0[8];
+  uint32_t kernarg_size;
+  uint8_t reserved0[4];
   int64_t kernel_code_entry_byte_offset;
   uint8_t reserved1[20];
   uint32_t compute_pgm_rsrc3; // GFX10+ and GFX90A+
@@ -178,7 +179,8 @@ struct kernel_descriptor_t {
 enum : uint32_t {
   GROUP_SEGMENT_FIXED_SIZE_OFFSET = 0,
   PRIVATE_SEGMENT_FIXED_SIZE_OFFSET = 4,
-  RESERVED0_OFFSET = 8,
+  KERNARG_SIZE_OFFSET = 8,
+  RESERVED0_OFFSET = 12,
   KERNEL_CODE_ENTRY_BYTE_OFFSET_OFFSET = 16,
   RESERVED1_OFFSET = 24,
   COMPUTE_PGM_RSRC3_OFFSET = 44,
@@ -197,6 +199,9 @@ static_assert(offsetof(kernel_descriptor_t, group_segment_fixed_size) ==
 static_assert(offsetof(kernel_descriptor_t, private_segment_fixed_size) ==
                   PRIVATE_SEGMENT_FIXED_SIZE_OFFSET,
               "invalid offset for private_segment_fixed_size");
+static_assert(offsetof(kernel_descriptor_t, kernarg_size) ==
+                  KERNARG_SIZE_OFFSET,
+              "invalid offset for kernarg_size");
 static_assert(offsetof(kernel_descriptor_t, reserved0) == RESERVED0_OFFSET,
               "invalid offset for reserved0");
 static_assert(offsetof(kernel_descriptor_t, kernel_code_entry_byte_offset) ==
diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h
index 9348ada91325096924c62a39eea0fb3d4dac4a3d..57052b596edbef24eb4e3fb0f61d3f230c0e9188 100644
--- a/llvm/include/llvm/Support/Compiler.h
+++ b/llvm/include/llvm/Support/Compiler.h
@@ -97,7 +97,8 @@
 /// Sadly, this is separate from just rvalue reference support because GCC
 /// and MSVC implemented this later than everything else. This appears to be
 /// corrected in MSVC 2019 but not MSVC 2017.
-#if __has_feature(cxx_rvalue_references) || LLVM_GNUC_PREREQ(4, 8, 1)
+#if __has_feature(cxx_rvalue_references) || LLVM_GNUC_PREREQ(4, 8, 1) ||       \
+    LLVM_MSC_PREREQ(1920)
 #define LLVM_HAS_RVALUE_REFERENCE_THIS 1
 #else
 #define LLVM_HAS_RVALUE_REFERENCE_THIS 0
diff --git a/llvm/include/llvm/Support/FileOutputBuffer.h b/llvm/include/llvm/Support/FileOutputBuffer.h
index d65997201ef3312910d8c10a97f88e556317b488..17b44380e9cd931c0fa3948daefc34eb593eff94 100644
--- a/llvm/include/llvm/Support/FileOutputBuffer.h
+++ b/llvm/include/llvm/Support/FileOutputBuffer.h
@@ -34,9 +34,6 @@ public:
     /// Don't use mmap and instead write an in-memory buffer to a file when this
     /// buffer is closed.
     F_no_mmap = 2,
-
-    /// Preserve ownership if the file already exists.
-    F_keep_ownership = 4,
   };
 
   /// Factory method to create an OutputBuffer object which manages a read/write
@@ -49,8 +46,7 @@ public:
   /// \p Size.  It is an error to specify F_modify and Size=-1 if \p FilePath
   /// does not exist.
   static Expected<std::unique_ptr<FileOutputBuffer>>
-  create(StringRef FilePath, size_t Size, unsigned Flags = 0,
-         unsigned UserID = 0, unsigned GroupID = 0);
+  create(StringRef FilePath, size_t Size, unsigned Flags = 0);
 
   /// Returns a pointer to the start of the buffer.
   virtual uint8_t *getBufferStart() const = 0;
diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h
index d82e966215dc7418bbc23bbdeecbe4213091daf0..1dc88243e83574403bedc02b927443fb58fdbb1e 100644
--- a/llvm/include/llvm/Support/FileSystem.h
+++ b/llvm/include/llvm/Support/FileSystem.h
@@ -802,10 +802,13 @@ void createUniquePath(const Twine &Model, SmallVectorImpl<char> &ResultPath,
 /// @param Model Name to base unique path off of.
 /// @param ResultFD Set to the opened file's file descriptor.
 /// @param ResultPath Set to the opened file's absolute path.
+/// @param Flags Set to the opened file's flags.
+/// @param Mode Set to the opened file's permissions.
 /// @returns errc::success if Result{FD,Path} have been successfully set,
 ///          otherwise a platform-specific error_code.
 std::error_code createUniqueFile(const Twine &Model, int &ResultFD,
                                  SmallVectorImpl<char> &ResultPath,
+                                 OpenFlags Flags = OF_None,
                                  unsigned Mode = all_read | all_write);
 
 /// Simpler version for clients that don't want an open file. An empty
@@ -862,12 +865,14 @@ public:
 /// running the assembler.
 std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
                                     int &ResultFD,
-                                    SmallVectorImpl<char> &ResultPath);
+                                    SmallVectorImpl<char> &ResultPath,
+                                    OpenFlags Flags = OF_None);
 
 /// Simpler version for clients that don't want an open file. An empty
 /// file will still be created.
 std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
-                                    SmallVectorImpl<char> &ResultPath);
+                                    SmallVectorImpl<char> &ResultPath,
+                                    OpenFlags Flags = OF_None);
 
 std::error_code createUniqueDirectory(const Twine &Prefix,
                                       SmallVectorImpl<char> &ResultPath);
diff --git a/llvm/include/llvm/Support/GraphWriter.h b/llvm/include/llvm/Support/GraphWriter.h
index 1f60fbc3512653d39bfd543af9f5aa46a4b222cf..4bb410d99f3f595349ba3ffd7b9945d6f6a07c41 100644
--- a/llvm/include/llvm/Support/GraphWriter.h
+++ b/llvm/include/llvm/Support/GraphWriter.h
@@ -318,8 +318,8 @@ raw_ostream &WriteGraph(raw_ostream &O, const GraphType &G,
 
 std::string createGraphFilename(const Twine &Name, int &FD);
 
-/// Writes graph into a provided {@code Filename}.
-/// If {@code Filename} is empty, generates a random one.
+/// Writes graph into a provided @c Filename.
+/// If @c Filename is empty, generates a random one.
 /// \return The resulting filename, or an empty string if writing
 /// failed.
 template <typename GraphType>
diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h
index d854aadbd430519cba04e8e76332f1cb611076f3..c27ddb0ce804f2a65926a003213d0d9a647a8614 100644
--- a/llvm/include/llvm/Support/KnownBits.h
+++ b/llvm/include/llvm/Support/KnownBits.h
@@ -296,6 +296,12 @@ public:
   /// Compute known bits resulting from multiplying LHS and RHS.
   static KnownBits computeForMul(const KnownBits &LHS, const KnownBits &RHS);
 
+  /// Compute known bits from sign-extended multiply-hi.
+  static KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS);
+
+  /// Compute known bits from zero-extended multiply-hi.
+  static KnownBits mulhu(const KnownBits &LHS, const KnownBits &RHS);
+
   /// Compute known bits for udiv(LHS, RHS).
   static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS);
 
diff --git a/llvm/include/llvm/Support/MachineValueType.h b/llvm/include/llvm/Support/MachineValueType.h
index e663953d0577b600fbc471b8f359909bb4232338..d01056887271c64c415d200a48ca66817bf75dde 100644
--- a/llvm/include/llvm/Support/MachineValueType.h
+++ b/llvm/include/llvm/Support/MachineValueType.h
@@ -70,25 +70,25 @@ namespace llvm {
       v512i1         =  24,   //  512 x i1
       v1024i1        =  25,   // 1024 x i1
 
-      v1i8           =  26,   //  1 x i8
-      v2i8           =  27,   //  2 x i8
-      v4i8           =  28,   //  4 x i8
-      v8i8           =  29,   //  8 x i8
-      v16i8          =  30,   // 16 x i8
-      v32i8          =  31,   // 32 x i8
-      v64i8          =  32,   // 64 x i8
-      v128i8         =  33,   //128 x i8
-      v256i8         =  34,   //256 x i8
-
-      v1i16          =  35,   //  1 x i16
-      v2i16          =  36,   //  2 x i16
-      v3i16          =  37,   //  3 x i16
-      v4i16          =  38,   //  4 x i16
-      v8i16          =  39,   //  8 x i16
-      v16i16         =  40,   // 16 x i16
-      v32i16         =  41,   // 32 x i16
-      v64i16         =  42,   // 64 x i16
-      v128i16        =  43,   //128 x i16
+      v1i8           =  26,   //   1 x i8
+      v2i8           =  27,   //   2 x i8
+      v4i8           =  28,   //   4 x i8
+      v8i8           =  29,   //   8 x i8
+      v16i8          =  30,   //  16 x i8
+      v32i8          =  31,   //  32 x i8
+      v64i8          =  32,   //  64 x i8
+      v128i8         =  33,   // 128 x i8
+      v256i8         =  34,   // 256 x i8
+
+      v1i16          =  35,   //   1 x i16
+      v2i16          =  36,   //   2 x i16
+      v3i16          =  37,   //   3 x i16
+      v4i16          =  38,   //   4 x i16
+      v8i16          =  39,   //   8 x i16
+      v16i16         =  40,   //  16 x i16
+      v32i16         =  41,   //  32 x i16
+      v64i16         =  42,   //  64 x i16
+      v128i16        =  43,   // 128 x i16
 
       v1i32          =  44,   //    1 x i32
       v2i32          =  45,   //    2 x i32
@@ -105,13 +105,13 @@ namespace llvm {
       v1024i32       =  56,   // 1024 x i32
       v2048i32       =  57,   // 2048 x i32
 
-      v1i64          =  58,   //  1 x i64
-      v2i64          =  59,   //  2 x i64
-      v4i64          =  60,   //  4 x i64
-      v8i64          =  61,   //  8 x i64
-      v16i64         =  62,   // 16 x i64
-      v32i64         =  63,   // 32 x i64
-      v64i64         =  64,   // 64 x i64
+      v1i64          =  58,   //   1 x i64
+      v2i64          =  59,   //   2 x i64
+      v4i64          =  60,   //   4 x i64
+      v8i64          =  61,   //   8 x i64
+      v16i64         =  62,   //  16 x i64
+      v32i64         =  63,   //  32 x i64
+      v64i64         =  64,   //  64 x i64
       v128i64        =  65,   // 128 x i64
       v256i64        =  66,   // 256 x i64
 
@@ -173,7 +173,7 @@ namespace llvm {
       nxv8i1         = 111,   // n x  8 x i1
       nxv16i1        = 112,   // n x 16 x i1
       nxv32i1        = 113,   // n x 32 x i1
-      nxv64i1        = 114,   // n x  64 x i1
+      nxv64i1        = 114,   // n x 64 x i1
 
       nxv1i8         = 115,   // n x  1 x i8
       nxv2i8         = 116,   // n x  2 x i8
@@ -181,7 +181,7 @@ namespace llvm {
       nxv8i8         = 118,   // n x  8 x i8
       nxv16i8        = 119,   // n x 16 x i8
       nxv32i8        = 120,   // n x 32 x i8
-      nxv64i8        = 121,   // n x  64 x i8
+      nxv64i8        = 121,   // n x 64 x i8
 
       nxv1i16        = 122,  // n x  1 x i16
       nxv2i16        = 123,  // n x  2 x i16
diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h
index ec9b5008792e172af7f01d279b373eadc2e8aeca..640b3a11ce6c24a93b484f398859009b9c78b989 100644
--- a/llvm/include/llvm/Support/MathExtras.h
+++ b/llvm/include/llvm/Support/MathExtras.h
@@ -766,7 +766,7 @@ template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
 }
 
 /// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
-/// Requires 0 < B < 32.
+/// Requires 0 < B <= 32.
 inline int32_t SignExtend32(uint32_t X, unsigned B) {
   assert(B > 0 && "Bit width can't be 0.");
   assert(B <= 32 && "Bit width out of range.");
@@ -774,7 +774,7 @@ inline int32_t SignExtend32(uint32_t X, unsigned B) {
 }
 
 /// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
-/// Requires 0 < B < 64.
+/// Requires 0 < B <= 64.
 template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
   static_assert(B > 0, "Bit width can't be 0.");
   static_assert(B <= 64, "Bit width out of range.");
@@ -782,7 +782,7 @@ template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
 }
 
 /// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
-/// Requires 0 < B < 64.
+/// Requires 0 < B <= 64.
 inline int64_t SignExtend64(uint64_t X, unsigned B) {
   assert(B > 0 && "Bit width can't be 0.");
   assert(B <= 64 && "Bit width out of range.");
diff --git a/llvm/include/llvm/Support/MemoryBuffer.h b/llvm/include/llvm/Support/MemoryBuffer.h
index 9e6ee2536c5e2d230f656baedefa43e25eab6b61..c9ceeedbf3dc53e6af96cba181af8f9d91ba2a7e 100644
--- a/llvm/include/llvm/Support/MemoryBuffer.h
+++ b/llvm/include/llvm/Support/MemoryBuffer.h
@@ -75,15 +75,16 @@ public:
   virtual StringRef getBufferIdentifier() const { return "Unknown buffer"; }
 
   /// Open the specified file as a MemoryBuffer, returning a new MemoryBuffer
-  /// if successful, otherwise returning null. If FileSize is specified, this
-  /// means that the client knows that the file exists and that it has the
-  /// specified size.
+  /// if successful, otherwise returning null.
+  ///
+  /// \param IsText Set to true to indicate that the file should be read in
+  /// text mode.
   ///
   /// \param IsVolatile Set to true to indicate that the contents of the file
   /// can change outside the user's control, e.g. when libclang tries to parse
   /// while the user is editing/updating the file or if the file is on an NFS.
   static ErrorOr<std::unique_ptr<MemoryBuffer>>
-  getFile(const Twine &Filename, int64_t FileSize = -1,
+  getFile(const Twine &Filename, bool IsText = false,
           bool RequiresNullTerminator = true, bool IsVolatile = false);
 
   /// Read all of the specified file into a MemoryBuffer as a stream
@@ -129,7 +130,7 @@ public:
   /// Open the specified file as a MemoryBuffer, or open stdin if the Filename
   /// is "-".
   static ErrorOr<std::unique_ptr<MemoryBuffer>>
-  getFileOrSTDIN(const Twine &Filename, int64_t FileSize = -1,
+  getFileOrSTDIN(const Twine &Filename, bool IsText = false,
                  bool RequiresNullTerminator = true);
 
   /// Map a subrange of the specified file as a MemoryBuffer.
@@ -180,8 +181,7 @@ public:
   }
 
   static ErrorOr<std::unique_ptr<WritableMemoryBuffer>>
-  getFile(const Twine &Filename, int64_t FileSize = -1,
-          bool IsVolatile = false);
+  getFile(const Twine &Filename, bool IsVolatile = false);
 
   /// Map a subrange of the specified file as a WritableMemoryBuffer.
   static ErrorOr<std::unique_ptr<WritableMemoryBuffer>>
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 2fc1de2d8551af0e482f079655fc67e9d10d2351..09f182c46b37449dc7fd34c2c3809adef965141f 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -432,6 +432,12 @@ HANDLE_TARGET_OPCODE(G_FSHL)
 // Generic funnel right shift
 HANDLE_TARGET_OPCODE(G_FSHR)
 
+// Generic right rotate
+HANDLE_TARGET_OPCODE(G_ROTR)
+
+// Generic left rotate
+HANDLE_TARGET_OPCODE(G_ROTL)
+
 /// Generic integer-base comparison, also applicable to vectors of integers.
 HANDLE_TARGET_OPCODE(G_ICMP)
 
@@ -731,6 +737,7 @@ HANDLE_TARGET_OPCODE(G_MEMMOVE)
 
 /// llvm.memset intrinsic
 HANDLE_TARGET_OPCODE(G_MEMSET)
+HANDLE_TARGET_OPCODE(G_BZERO)
 
 /// Vector reductions
 HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FADD)
@@ -749,10 +756,13 @@ HANDLE_TARGET_OPCODE(G_VECREDUCE_SMIN)
 HANDLE_TARGET_OPCODE(G_VECREDUCE_UMAX)
 HANDLE_TARGET_OPCODE(G_VECREDUCE_UMIN)
 
+HANDLE_TARGET_OPCODE(G_SBFX)
+HANDLE_TARGET_OPCODE(G_UBFX)
+
 /// Marker for the end of the generic opcode.
 /// This is used to check if an opcode is in the range of the
 /// generic opcodes.
-HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_VECREDUCE_UMIN)
+HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_UBFX)
 
 /// BUILTIN_OP_END - This must be the last enum value in this list.
 /// The target-specific post-isel opcode values start here.
diff --git a/llvm/include/llvm/Support/raw_ostream.h b/llvm/include/llvm/Support/raw_ostream.h
index 201a4a8579646d8b276af52501a09666ddad9209..86d6570dfb0f4c6180550560a74ec744e323b4a2 100644
--- a/llvm/include/llvm/Support/raw_ostream.h
+++ b/llvm/include/llvm/Support/raw_ostream.h
@@ -714,6 +714,17 @@ public:
   ~buffer_unique_ostream() override { *OS << str(); }
 };
 
+class Error;
+
+/// This helper creates an output stream and then passes it to \p Write.
+/// The stream created is based on the specified \p OutputFileName:
+/// llvm::outs for "-", raw_null_ostream for "/dev/null", and raw_fd_ostream
+/// for other names. For raw_fd_ostream instances, the stream writes to
+/// a temporary file. The final output file is atomically replaced with the
+/// temporary file after the \p Write function is finished.
+Error writeToOutput(StringRef OutputFileName,
+                    std::function<Error(raw_ostream &)> Write);
+
 } // end namespace llvm
 
 #endif // LLVM_SUPPORT_RAW_OSTREAM_H
diff --git a/llvm/include/llvm/TableGen/DirectiveEmitter.h b/llvm/include/llvm/TableGen/DirectiveEmitter.h
index b1757249670a039b03f691e8b57fc7781888e264..5c4a736eb10745166dd4cc75428fe53fcdf8e266 100644
--- a/llvm/include/llvm/TableGen/DirectiveEmitter.h
+++ b/llvm/include/llvm/TableGen/DirectiveEmitter.h
@@ -30,10 +30,6 @@ public:
     return Def->getValueAsString("clausePrefix");
   }
 
-  StringRef getIncludeHeader() const {
-    return Def->getValueAsString("includeHeader");
-  }
-
   StringRef getClauseEnumSetClass() const {
     return Def->getValueAsString("clauseEnumSetClass");
   }
diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index e75b7f01c868a45206b9cd2455968042fdd64541..ea47d6713026b4d3f66aa358ad9052dfc90059aa 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -2024,6 +2024,12 @@ public:
 
   void set(Init *Key, Init *Value) { Map[Key] = {Value, false}; }
 
+  bool isComplete(Init *VarName) const {
+    auto It = Map.find(VarName);
+    assert(It != Map.end() && "key must be present in map");
+    return It->second.V->isComplete();
+  }
+
   Init *resolve(Init *VarName) override;
 };
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 1732c9577a35cdf0d1ff08887f4c9230603d96b4..7eb9ffa2a930ca34ed592a27812b2f349babdcf5 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -362,6 +362,20 @@ def G_FSHR : GenericInstruction {
   let hasSideEffects = false;
 }
 
+/// Rotate bits right.
+def G_ROTR : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type1:$src2);
+  let hasSideEffects = false;
+}
+
+/// Rotate bits left.
+def G_ROTL : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type1:$src2);
+  let hasSideEffects = false;
+}
+
 // Generic integer comparison.
 def G_ICMP : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
@@ -1354,6 +1368,31 @@ def G_MEMSET : GenericInstruction {
   let mayStore = true;
 }
 
+def G_BZERO : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins ptype0:$dst_addr, type1:$size, untyped_imm_0:$tailcall);
+  let hasSideEffects = false;
+  let mayStore = true;
+}
+
+//------------------------------------------------------------------------------
+// Bitfield extraction.
+//------------------------------------------------------------------------------
+
+// Generic signed bitfield extraction.
+def G_SBFX : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src, type0:$lsb, type0:$width);
+  let hasSideEffects = false;
+}
+
+// Generic unsigned bitfield extraction.
+def G_UBFX : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src, type0:$lsb, type0:$width);
+  let hasSideEffects = false;
+}
+
 //------------------------------------------------------------------------------
 // Optimization hints
 //------------------------------------------------------------------------------
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 6fb8a6b15dd74ad8595b8407486500c4b6284626..4c947b5c6696ae01fd9496e60faeb8164ac0a145 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -141,6 +141,8 @@ def : GINodeEquiv<G_FMAXNUM, fmaxnum>;
 def : GINodeEquiv<G_FMINNUM_IEEE, fminnum_ieee>;
 def : GINodeEquiv<G_FMAXNUM_IEEE, fmaxnum_ieee>;
 def : GINodeEquiv<G_READCYCLECOUNTER, readcyclecounter>;
+def : GINodeEquiv<G_ROTR, rotr>;
+def : GINodeEquiv<G_ROTL, rotl>;
 
 def : GINodeEquiv<G_STRICT_FADD, strict_fadd>;
 def : GINodeEquiv<G_STRICT_FSUB, strict_fsub>;
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 247ac68034b20a5b2933c8b2d698b09b6146158c..99066f675eea8be304831286be215a516ada9ab5 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -152,10 +152,10 @@ def SDTIntTruncOp  : SDTypeProfile<1, 1, [  // trunc
 def SDTFPUnaryOp  : SDTypeProfile<1, 1, [   // fneg, fsqrt, etc
   SDTCisSameAs<0, 1>, SDTCisFP<0>
 ]>;
-def SDTFPRoundOp  : SDTypeProfile<1, 1, [   // fround
+def SDTFPRoundOp  : SDTypeProfile<1, 1, [   // fpround
   SDTCisFP<0>, SDTCisFP<1>, SDTCisOpSmallerThanOp<0, 1>, SDTCisSameNumEltsAs<0, 1>
 ]>;
-def SDTFPExtendOp  : SDTypeProfile<1, 1, [  // fextend
+def SDTFPExtendOp  : SDTypeProfile<1, 1, [  // fpextend
   SDTCisFP<0>, SDTCisFP<1>, SDTCisOpSmallerThanOp<1, 0>, SDTCisSameNumEltsAs<0, 1>
 ]>;
 def SDTIntToFPOp : SDTypeProfile<1, 1, [    // [su]int_to_fp
@@ -486,6 +486,7 @@ def fceil      : SDNode<"ISD::FCEIL"      , SDTFPUnaryOp>;
 def ffloor     : SDNode<"ISD::FFLOOR"     , SDTFPUnaryOp>;
 def fnearbyint : SDNode<"ISD::FNEARBYINT" , SDTFPUnaryOp>;
 def fround     : SDNode<"ISD::FROUND"     , SDTFPUnaryOp>;
+def froundeven : SDNode<"ISD::FROUNDEVEN" , SDTFPUnaryOp>;
 
 def lround     : SDNode<"ISD::LROUND"     , SDTFPToIntOp>;
 def llround    : SDNode<"ISD::LLROUND"    , SDTFPToIntOp>;
@@ -547,6 +548,8 @@ def strict_llround    : SDNode<"ISD::STRICT_LLROUND",
                                SDTFPToIntOp, [SDNPHasChain]>;
 def strict_fround     : SDNode<"ISD::STRICT_FROUND",
                                SDTFPUnaryOp, [SDNPHasChain]>;
+def strict_froundeven : SDNode<"ISD::STRICT_FROUNDEVEN",
+                               SDTFPUnaryOp, [SDNPHasChain]>;
 def strict_ftrunc     : SDNode<"ISD::STRICT_FTRUNC",
                                SDTFPUnaryOp, [SDNPHasChain]>;
 def strict_fminnum    : SDNode<"ISD::STRICT_FMINNUM",
@@ -661,6 +664,8 @@ def vector_reverse : SDNode<"ISD::VECTOR_REVERSE", SDTVecReverse>;
 def vector_splice : SDNode<"ISD::VECTOR_SPLICE", SDTVecSlice, []>;
 def build_vector : SDNode<"ISD::BUILD_VECTOR", SDTypeProfile<1, -1, []>, []>;
 def splat_vector : SDNode<"ISD::SPLAT_VECTOR", SDTypeProfile<1, 1, []>, []>;
+def step_vector : SDNode<"ISD::STEP_VECTOR", SDTypeProfile<1, 1,
+                       [SDTCisVec<0>, SDTCisInt<1>]>, []>;
 def scalar_to_vector : SDNode<"ISD::SCALAR_TO_VECTOR", SDTypeProfile<1, 1, []>,
                               []>;
 
@@ -1414,6 +1419,9 @@ def any_llround    : PatFrags<(ops node:$src),
 def any_fround     : PatFrags<(ops node:$src),
                               [(strict_fround node:$src),
                                (fround node:$src)]>;
+def any_froundeven : PatFrags<(ops node:$src),
+                              [(strict_froundeven node:$src),
+                               (froundeven node:$src)]>;
 def any_ftrunc     : PatFrags<(ops node:$src),
                               [(strict_ftrunc node:$src),
                                (ftrunc node:$src)]>;
diff --git a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
index 22b2e649e4d4870bff0faf295feee23dd5b0a5e6..4e47ff70d55747cfcec9b8442bfe1177d52f2a79 100644
--- a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
+++ b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
@@ -17,7 +17,6 @@
 
 #include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/Support/CommandLine.h"
 
 namespace llvm {
 
@@ -32,8 +31,6 @@ struct LowerExpectIntrinsicPass : PassInfoMixin<LowerExpectIntrinsicPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &);
 };
 
-extern cl::opt<uint32_t> LikelyBranchWeight;
-extern cl::opt<uint32_t> UnlikelyBranchWeight;
 }
 
 #endif
diff --git a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
index c922476ac79dacd8e1afa5595dc4475dfc92f57f..c4030735d965737bd6f5e1144d230caa1da2aff1 100644
--- a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
+++ b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
@@ -51,13 +51,11 @@
 #define LLVM_TRANSFORMS_UTILS_PREDICATEINFO_H
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Value.h"
-#include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
 
 namespace llvm {
@@ -178,7 +176,7 @@ public:
 class PredicateInfo {
 public:
   PredicateInfo(Function &, DominatorTree &, AssumptionCache &);
-  ~PredicateInfo();
+  ~PredicateInfo() = default;
 
   void verifyPredicateInfo() const;
 
@@ -205,8 +203,6 @@ private:
   // the Predicate Info, they belong to the ValueInfo structs in the ValueInfos
   // vector.
   DenseMap<const Value *, const PredicateBase *> PredicateMap;
-  // The set of ssa_copy declarations we created with our custom mangling.
-  SmallSet<AssertingVH<Function>, 20> CreatedDeclarations;
 };
 
 // This pass does eager building and then printing of PredicateInfo. It is used
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index c092c1da9b711440f06d7ca4a3e5cc436729e9ef..0fa4b0b77b9bb9e4f730372b921e2b51bc27609f 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -191,6 +191,11 @@ public:
 
   bool doesNotMeet(Function *F, Loop *L, const LoopVectorizeHints &Hints);
 
+  Instruction *getExactFPInst() { return ExactFPMathInst; }
+  bool canVectorizeFPMath(const LoopVectorizeHints &Hints) const {
+    return !ExactFPMathInst || Hints.allowReordering();
+  }
+
 private:
   unsigned NumRuntimePointerChecks = 0;
   Instruction *ExactFPMathInst = nullptr;
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index fa83f4648eeebf55ea97fd1e4817d8f19c9f0a05..6a69e71470b299bd105a3487f02e363c4c04958e 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -23,7 +23,6 @@
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/PhiValues.h"
@@ -104,7 +103,6 @@ bool BasicAAResult::invalidate(Function &Fn, const PreservedAnalyses &PA,
   // depend on them.
   if (Inv.invalidate<AssumptionAnalysis>(Fn, PA) ||
       (DT && Inv.invalidate<DominatorTreeAnalysis>(Fn, PA)) ||
-      (LI && Inv.invalidate<LoopAnalysis>(Fn, PA)) ||
       (PV && Inv.invalidate<PhiValuesAnalysis>(Fn, PA)))
     return true;
 
@@ -201,9 +199,11 @@ static uint64_t getMinimalExtentFrom(const Value &V,
   // If we have dereferenceability information we know a lower bound for the
   // extent as accesses for a lower offset would be valid. We need to exclude
   // the "or null" part if null is a valid pointer.
-  bool CanBeNull;
-  uint64_t DerefBytes = V.getPointerDereferenceableBytes(DL, CanBeNull);
+  bool CanBeNull, CanBeFreed;
+  uint64_t DerefBytes =
+    V.getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
   DerefBytes = (CanBeNull && NullIsValidLoc) ? 0 : DerefBytes;
+  DerefBytes = CanBeFreed ? 0 : DerefBytes;
   // If queried with a precise location size, we assume that location size to be
   // accessed, thus valid.
   if (LocSize.isPrecise())
@@ -222,167 +222,159 @@ static bool isObjectSize(const Value *V, uint64_t Size, const DataLayout &DL,
 // GetElementPtr Instruction Decomposition and Analysis
 //===----------------------------------------------------------------------===//
 
-/// Analyzes the specified value as a linear expression: "A*V + B", where A and
-/// B are constant integers.
-///
-/// Returns the scale and offset values as APInts and return V as a Value*, and
-/// return whether we looked through any sign or zero extends.  The incoming
-/// Value is known to have IntegerType, and it may already be sign or zero
-/// extended.
-///
-/// Note that this looks through extends, so the high bits may not be
-/// represented in the result.
-/*static*/ const Value *BasicAAResult::GetLinearExpression(
-    const Value *V, APInt &Scale, APInt &Offset, unsigned &ZExtBits,
-    unsigned &SExtBits, const DataLayout &DL, unsigned Depth,
-    AssumptionCache *AC, DominatorTree *DT, bool &NSW, bool &NUW) {
-  assert(V->getType()->isIntegerTy() && "Not an integer value");
+namespace {
+/// Represents zext(sext(V)).
+struct ExtendedValue {
+  const Value *V;
+  unsigned ZExtBits;
+  unsigned SExtBits;
 
-  // Limit our recursion depth.
-  if (Depth == 6) {
-    Scale = 1;
-    Offset = 0;
-    return V;
+  explicit ExtendedValue(const Value *V, unsigned ZExtBits = 0,
+                         unsigned SExtBits = 0)
+      : V(V), ZExtBits(ZExtBits), SExtBits(SExtBits) {}
+
+  unsigned getBitWidth() const {
+    return V->getType()->getPrimitiveSizeInBits() + ZExtBits + SExtBits;
   }
 
-  if (const ConstantInt *Const = dyn_cast<ConstantInt>(V)) {
-    // If it's a constant, just convert it to an offset and remove the variable.
-    // If we've been called recursively, the Offset bit width will be greater
-    // than the constant's (the Offset's always as wide as the outermost call),
-    // so we'll zext here and process any extension in the isa<SExtInst> &
-    // isa<ZExtInst> cases below.
-    Offset += Const->getValue().zextOrSelf(Offset.getBitWidth());
-    assert(Scale == 0 && "Constant values don't have a scale");
-    return V;
+  ExtendedValue withValue(const Value *NewV) const {
+    return ExtendedValue(NewV, ZExtBits, SExtBits);
   }
 
-  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(V)) {
+  ExtendedValue withZExtOfValue(const Value *NewV) const {
+    unsigned ExtendBy = V->getType()->getPrimitiveSizeInBits() -
+                        NewV->getType()->getPrimitiveSizeInBits();
+    // zext(sext(zext(NewV))) == zext(zext(zext(NewV)))
+    return ExtendedValue(NewV, ZExtBits + SExtBits + ExtendBy, 0);
+  }
+
+  ExtendedValue withSExtOfValue(const Value *NewV) const {
+    unsigned ExtendBy = V->getType()->getPrimitiveSizeInBits() -
+                        NewV->getType()->getPrimitiveSizeInBits();
+    // zext(sext(sext(NewV)))
+    return ExtendedValue(NewV, ZExtBits, SExtBits + ExtendBy);
+  }
+
+  APInt evaluateWith(APInt N) const {
+    assert(N.getBitWidth() == V->getType()->getPrimitiveSizeInBits() &&
+           "Incompatible bit width");
+    if (SExtBits) N = N.sext(N.getBitWidth() + SExtBits);
+    if (ZExtBits) N = N.zext(N.getBitWidth() + ZExtBits);
+    return N;
+  }
+
+  bool canDistributeOver(bool NUW, bool NSW) const {
+    // zext(x op<nuw> y) == zext(x) op<nuw> zext(y)
+    // sext(x op<nsw> y) == sext(x) op<nsw> sext(y)
+    return (!ZExtBits || NUW) && (!SExtBits || NSW);
+  }
+};
+
+/// Represents zext(sext(V)) * Scale + Offset.
+struct LinearExpression {
+  ExtendedValue Val;
+  APInt Scale;
+  APInt Offset;
+
+  LinearExpression(const ExtendedValue &Val, const APInt &Scale,
+                   const APInt &Offset)
+      : Val(Val), Scale(Scale), Offset(Offset) {}
+
+  LinearExpression(const ExtendedValue &Val) : Val(Val) {
+    unsigned BitWidth = Val.getBitWidth();
+    Scale = APInt(BitWidth, 1);
+    Offset = APInt(BitWidth, 0);
+  }
+};
+}
+
+/// Analyzes the specified value as a linear expression: "A*V + B", where A and
+/// B are constant integers.
+static LinearExpression GetLinearExpression(
+    const ExtendedValue &Val,  const DataLayout &DL, unsigned Depth,
+    AssumptionCache *AC, DominatorTree *DT) {
+  // Limit our recursion depth.
+  if (Depth == 6)
+    return Val;
+
+  if (const ConstantInt *Const = dyn_cast<ConstantInt>(Val.V))
+    return LinearExpression(Val, APInt(Val.getBitWidth(), 0),
+                            Val.evaluateWith(Const->getValue()));
+
+  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val.V)) {
     if (ConstantInt *RHSC = dyn_cast<ConstantInt>(BOp->getOperand(1))) {
-      // If we've been called recursively, then Offset and Scale will be wider
-      // than the BOp operands. We'll always zext it here as we'll process sign
-      // extensions below (see the isa<SExtInst> / isa<ZExtInst> cases).
-      APInt RHS = RHSC->getValue().zextOrSelf(Offset.getBitWidth());
+      APInt RHS = Val.evaluateWith(RHSC->getValue());
+      // The only non-OBO case we deal with is or, and only limited to the
+      // case where it is both nuw and nsw.
+      bool NUW = true, NSW = true;
+      if (isa<OverflowingBinaryOperator>(BOp)) {
+        NUW &= BOp->hasNoUnsignedWrap();
+        NSW &= BOp->hasNoSignedWrap();
+      }
+      if (!Val.canDistributeOver(NUW, NSW))
+        return Val;
 
       switch (BOp->getOpcode()) {
       default:
         // We don't understand this instruction, so we can't decompose it any
         // further.
-        Scale = 1;
-        Offset = 0;
-        return V;
+        return Val;
       case Instruction::Or:
         // X|C == X+C if all the bits in C are unset in X.  Otherwise we can't
         // analyze it.
         if (!MaskedValueIsZero(BOp->getOperand(0), RHSC->getValue(), DL, 0, AC,
-                               BOp, DT)) {
-          Scale = 1;
-          Offset = 0;
-          return V;
-        }
+                               BOp, DT))
+          return Val;
+
         LLVM_FALLTHROUGH;
-      case Instruction::Add:
-        V = GetLinearExpression(BOp->getOperand(0), Scale, Offset, ZExtBits,
-                                SExtBits, DL, Depth + 1, AC, DT, NSW, NUW);
-        Offset += RHS;
-        break;
-      case Instruction::Sub:
-        V = GetLinearExpression(BOp->getOperand(0), Scale, Offset, ZExtBits,
-                                SExtBits, DL, Depth + 1, AC, DT, NSW, NUW);
-        Offset -= RHS;
-        break;
-      case Instruction::Mul:
-        V = GetLinearExpression(BOp->getOperand(0), Scale, Offset, ZExtBits,
-                                SExtBits, DL, Depth + 1, AC, DT, NSW, NUW);
-        Offset *= RHS;
-        Scale *= RHS;
-        break;
+      case Instruction::Add: {
+        LinearExpression E = GetLinearExpression(
+            Val.withValue(BOp->getOperand(0)), DL, Depth + 1, AC, DT);
+        E.Offset += RHS;
+        return E;
+      }
+      case Instruction::Sub: {
+        LinearExpression E = GetLinearExpression(
+            Val.withValue(BOp->getOperand(0)), DL, Depth + 1, AC, DT);
+        E.Offset -= RHS;
+        return E;
+      }
+      case Instruction::Mul: {
+        LinearExpression E = GetLinearExpression(
+            Val.withValue(BOp->getOperand(0)), DL, Depth + 1, AC, DT);
+        E.Offset *= RHS;
+        E.Scale *= RHS;
+        return E;
+      }
       case Instruction::Shl:
-        V = GetLinearExpression(BOp->getOperand(0), Scale, Offset, ZExtBits,
-                                SExtBits, DL, Depth + 1, AC, DT, NSW, NUW);
-
         // We're trying to linearize an expression of the kind:
         //   shl i8 -128, 36
         // where the shift count exceeds the bitwidth of the type.
         // We can't decompose this further (the expression would return
         // a poison value).
-        if (Offset.getBitWidth() < RHS.getLimitedValue() ||
-            Scale.getBitWidth() < RHS.getLimitedValue()) {
-          Scale = 1;
-          Offset = 0;
-          return V;
-        }
-
-        Offset <<= RHS.getLimitedValue();
-        Scale <<= RHS.getLimitedValue();
-        // the semantics of nsw and nuw for left shifts don't match those of
-        // multiplications, so we won't propagate them.
-        NSW = NUW = false;
-        return V;
-      }
-
-      if (isa<OverflowingBinaryOperator>(BOp)) {
-        NUW &= BOp->hasNoUnsignedWrap();
-        NSW &= BOp->hasNoSignedWrap();
+        if (RHS.getLimitedValue() > Val.getBitWidth())
+          return Val;
+
+        LinearExpression E = GetLinearExpression(
+            Val.withValue(BOp->getOperand(0)), DL, Depth + 1, AC, DT);
+        E.Offset <<= RHS.getLimitedValue();
+        E.Scale <<= RHS.getLimitedValue();
+        return E;
       }
-      return V;
     }
   }
 
-  // Since GEP indices are sign extended anyway, we don't care about the high
-  // bits of a sign or zero extended value - just scales and offsets.  The
-  // extensions have to be consistent though.
-  if (isa<SExtInst>(V) || isa<ZExtInst>(V)) {
-    Value *CastOp = cast<CastInst>(V)->getOperand(0);
-    unsigned NewWidth = V->getType()->getPrimitiveSizeInBits();
-    unsigned SmallWidth = CastOp->getType()->getPrimitiveSizeInBits();
-    unsigned OldZExtBits = ZExtBits, OldSExtBits = SExtBits;
-    const Value *Result =
-        GetLinearExpression(CastOp, Scale, Offset, ZExtBits, SExtBits, DL,
-                            Depth + 1, AC, DT, NSW, NUW);
-
-    // zext(zext(%x)) == zext(%x), and similarly for sext; we'll handle this
-    // by just incrementing the number of bits we've extended by.
-    unsigned ExtendedBy = NewWidth - SmallWidth;
-
-    if (isa<SExtInst>(V) && ZExtBits == 0) {
-      // sext(sext(%x, a), b) == sext(%x, a + b)
-
-      if (NSW) {
-        // We haven't sign-wrapped, so it's valid to decompose sext(%x + c)
-        // into sext(%x) + sext(c). We'll sext the Offset ourselves:
-        unsigned OldWidth = Offset.getBitWidth();
-        Offset = Offset.trunc(SmallWidth).sext(NewWidth).zextOrSelf(OldWidth);
-      } else {
-        // We may have signed-wrapped, so don't decompose sext(%x + c) into
-        // sext(%x) + sext(c)
-        Scale = 1;
-        Offset = 0;
-        Result = CastOp;
-        ZExtBits = OldZExtBits;
-        SExtBits = OldSExtBits;
-      }
-      SExtBits += ExtendedBy;
-    } else {
-      // sext(zext(%x, a), b) = zext(zext(%x, a), b) = zext(%x, a + b)
-
-      if (!NUW) {
-        // We may have unsigned-wrapped, so don't decompose zext(%x + c) into
-        // zext(%x) + zext(c)
-        Scale = 1;
-        Offset = 0;
-        Result = CastOp;
-        ZExtBits = OldZExtBits;
-        SExtBits = OldSExtBits;
-      }
-      ZExtBits += ExtendedBy;
-    }
+  if (isa<ZExtInst>(Val.V))
+    return GetLinearExpression(
+        Val.withZExtOfValue(cast<CastInst>(Val.V)->getOperand(0)),
+        DL, Depth + 1, AC, DT);
 
-    return Result;
-  }
+  if (isa<SExtInst>(Val.V))
+    return GetLinearExpression(
+        Val.withSExtOfValue(cast<CastInst>(Val.V)->getOperand(0)),
+        DL, Depth + 1, AC, DT);
 
-  Scale = 1;
-  Offset = 0;
-  return V;
+  return Val;
 }
 
 /// To ensure a pointer offset fits in an integer of size PointerSize
@@ -532,20 +524,12 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL,
 
       APInt Scale(MaxPointerSize,
                   DL.getTypeAllocSize(GTI.getIndexedType()).getFixedSize());
-      unsigned ZExtBits = 0, SExtBits = 0;
-
       // If the integer type is smaller than the pointer size, it is implicitly
       // sign extended to pointer size.
       unsigned Width = Index->getType()->getIntegerBitWidth();
-      if (PointerSize > Width)
-        SExtBits += PointerSize - Width;
-
-      // Use GetLinearExpression to decompose the index into a C1*V+C2 form.
-      APInt IndexScale(Width, 0), IndexOffset(Width, 0);
-      bool NSW = true, NUW = true;
-      const Value *OrigIndex = Index;
-      Index = GetLinearExpression(Index, IndexScale, IndexOffset, ZExtBits,
-                                  SExtBits, DL, 0, AC, DT, NSW, NUW);
+      unsigned SExtBits = PointerSize > Width ? PointerSize - Width : 0;
+      LinearExpression LE = GetLinearExpression(
+          ExtendedValue(Index, 0, SExtBits), DL, 0, AC, DT);
 
       // The GEP index scale ("Scale") scales C1*V+C2, yielding (C1*V+C2)*Scale.
       // This gives us an aggregate computation of (C1*Scale)*V + C2*Scale.
@@ -558,19 +542,13 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL,
       // (C1*Scale)*V+C2*Scale can also overflow. We should check for this
       // possibility.
       bool Overflow;
-      APInt ScaledOffset = IndexOffset.sextOrTrunc(MaxPointerSize)
+      APInt ScaledOffset = LE.Offset.sextOrTrunc(MaxPointerSize)
                            .smul_ov(Scale, Overflow);
       if (Overflow) {
-        Index = OrigIndex;
-        IndexScale = 1;
-        IndexOffset = 0;
-
-        ZExtBits = SExtBits = 0;
-        if (PointerSize > Width)
-          SExtBits += PointerSize - Width;
+        LE = LinearExpression(ExtendedValue(Index, 0, SExtBits));
       } else {
         Decomposed.Offset += ScaledOffset;
-        Scale *= IndexScale.sextOrTrunc(MaxPointerSize);
+        Scale *= LE.Scale.sextOrTrunc(MaxPointerSize);
       }
 
       // If we already had an occurrence of this index variable, merge this
@@ -578,9 +556,9 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL,
       //   A[x][x] -> x*16 + x*4 -> x*20
       // This also ensures that 'x' only appears in the index list once.
       for (unsigned i = 0, e = Decomposed.VarIndices.size(); i != e; ++i) {
-        if (Decomposed.VarIndices[i].V == Index &&
-            Decomposed.VarIndices[i].ZExtBits == ZExtBits &&
-            Decomposed.VarIndices[i].SExtBits == SExtBits) {
+        if (Decomposed.VarIndices[i].V == LE.Val.V &&
+            Decomposed.VarIndices[i].ZExtBits == LE.Val.ZExtBits &&
+            Decomposed.VarIndices[i].SExtBits == LE.Val.SExtBits) {
           Scale += Decomposed.VarIndices[i].Scale;
           Decomposed.VarIndices.erase(Decomposed.VarIndices.begin() + i);
           break;
@@ -592,7 +570,8 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL,
       Scale = adjustToPointerSize(Scale, PointerSize);
 
       if (!!Scale) {
-        VariableGEPIndex Entry = {Index, ZExtBits, SExtBits, Scale, CxtI};
+        VariableGEPIndex Entry = {LE.Val.V, LE.Val.ZExtBits, LE.Val.SExtBits,
+                                  Scale, CxtI};
         Decomposed.VarIndices.push_back(Entry);
       }
     }
@@ -672,6 +651,11 @@ bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc,
   return Worklist.empty();
 }
 
+static bool isIntrinsicCall(const CallBase *Call, Intrinsic::ID IID) {
+  const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Call);
+  return II && II->getIntrinsicID() == IID;
+}
+
 /// Returns the behavior when calling the given call site.
 FunctionModRefBehavior BasicAAResult::getModRefBehavior(const CallBase *Call) {
   if (Call->doesNotAccessMemory())
@@ -771,11 +755,6 @@ ModRefInfo BasicAAResult::getArgModRefInfo(const CallBase *Call,
   return AAResultBase::getArgModRefInfo(Call, ArgIdx);
 }
 
-static bool isIntrinsicCall(const CallBase *Call, Intrinsic::ID IID) {
-  const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Call);
-  return II && II->getIntrinsicID() == IID;
-}
-
 #ifndef NDEBUG
 static const Function *getParent(const Value *V) {
   if (const Instruction *inst = dyn_cast<Instruction>(V)) {
@@ -939,15 +918,9 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call,
     return rv;
   }
 
-  // While the assume intrinsic is marked as arbitrarily writing so that
-  // proper control dependencies will be maintained, it never aliases any
-  // particular memory location.
-  if (isIntrinsicCall(Call, Intrinsic::assume))
-    return ModRefInfo::NoModRef;
-
-  // Like assumes, guard intrinsics are also marked as arbitrarily writing so
-  // that proper control dependencies are maintained but they never mods any
-  // particular memory location.
+  // Guard intrinsics are marked as arbitrarily writing so that proper control
+  // dependencies are maintained but they never mods any particular memory
+  // location.
   //
   // *Unlike* assumes, guard intrinsics are modeled as reading memory since the
   // heap state at the point the guard is issued needs to be consistent in case
@@ -991,16 +964,9 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call,
 ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call1,
                                         const CallBase *Call2,
                                         AAQueryInfo &AAQI) {
-  // While the assume intrinsic is marked as arbitrarily writing so that
-  // proper control dependencies will be maintained, it never aliases any
-  // particular memory location.
-  if (isIntrinsicCall(Call1, Intrinsic::assume) ||
-      isIntrinsicCall(Call2, Intrinsic::assume))
-    return ModRefInfo::NoModRef;
-
-  // Like assumes, guard intrinsics are also marked as arbitrarily writing so
-  // that proper control dependencies are maintained but they never mod any
-  // particular memory location.
+  // Guard intrinsics are marked as arbitrarily writing so that proper control
+  // dependencies are maintained but they never mods any particular memory
+  // location.
   //
   // *Unlike* assumes, guard intrinsics are modeled as reading memory since the
   // heap state at the point the guard is issued needs to be consistent in case
@@ -1046,6 +1012,60 @@ AliasResult BasicAAResult::aliasGEP(
     const GEPOperator *GEP1, LocationSize V1Size, const AAMDNodes &V1AAInfo,
     const Value *V2, LocationSize V2Size, const AAMDNodes &V2AAInfo,
     const Value *UnderlyingV1, const Value *UnderlyingV2, AAQueryInfo &AAQI) {
+  if (!V1Size.hasValue() && !V2Size.hasValue()) {
+    // Special case to avoid being too pessimistic in the specific case we want
+    // a struct of scalable vectors (which is used to implement intrinsics). In
+    // this case if the UnderlyingV1 is the same as V2, the type is a pointer
+    // to a struct and the indices are <0, X> where X != 0, then this is
+    // NoAlias.
+    auto IsPointerToStruct = [](Type *Ty) {
+      return isa<PointerType>(Ty) &&
+             isa<StructType>(cast<PointerType>(Ty)->getElementType());
+    };
+    auto GEPIndexesScalableVectorField = [](const GEPOperator *GEP) {
+      if (!GEP->isInBounds() || GEP->getNumIndices() != 2 ||
+          !GEP->hasAllConstantIndices())
+        return false;
+
+      if (!isa<ScalableVectorType>(GEP->getResultElementType()))
+        return false;
+
+      // This is unlikely but just in case.
+      if (GEP->hasAllZeroIndices())
+        return false;
+
+      auto I = GEP->idx_begin();
+      const ConstantInt *Idx0 = cast<const ConstantInt>(*I++);
+      const ConstantInt *Idx1 = cast<const ConstantInt>(*I);
+
+      if (!Idx0->isZero())
+        return false;
+      if (Idx1->isZero())
+        return false;
+
+      return true;
+    };
+    // FIXME: This is perhaps too specific.
+    if (UnderlyingV1 == V2 && IsPointerToStruct(UnderlyingV1->getType()) &&
+        GEPIndexesScalableVectorField(GEP1))
+      return NoAlias;
+
+    // TODO: This limitation exists for compile-time reasons. Relax it if we
+    // can avoid exponential pathological cases.
+    if (!isa<GEPOperator>(V2))
+      return MayAlias;
+
+    // If both accesses have unknown size, we can only check whether the base
+    // objects don't alias.
+    AliasResult BaseAlias = getBestAAResults().alias(
+        MemoryLocation::getBeforeOrAfter(UnderlyingV1),
+        MemoryLocation::getBeforeOrAfter(UnderlyingV2), AAQI);
+    if (BaseAlias == NoAlias)
+      return NoAlias;
+
+    return MayAlias;
+  }
+
   DecomposedGEP DecompGEP1 = DecomposeGEPExpression(GEP1, DL, &AC, DT);
   DecomposedGEP DecompGEP2 = DecomposeGEPExpression(V2, DL, &AC, DT);
 
@@ -1077,58 +1097,6 @@ AliasResult BasicAAResult::aliasGEP(
         V1Size.hasValue() && DecompGEP1.Offset.sle(-V1Size.getValue()) &&
         isBaseOfObject(DecompGEP1.Base))
     return NoAlias;
-  } else {
-    // Check to see if these two pointers are related by the getelementptr
-    // instruction.  If one pointer is a GEP with a non-zero index of the other
-    // pointer, we know they cannot alias.
-
-    // If both accesses are unknown size, we can't do anything useful here.
-    if (!V1Size.hasValue() && !V2Size.hasValue()) {
-      // However this is too pessimistic in the specific case we want a struct
-      // of scalable vectors (which is used to implement intrinsics). In this
-      // case if the UnderlyingV1 is the same as V2, the type is a pointer to a
-      // struct and the indices are <0, X> where X != 0, then this is NoAlias.
-      auto IsPointerToStruct = [](Type *Ty) {
-        return isa<PointerType>(Ty) &&
-               isa<StructType>(cast<PointerType>(Ty)->getElementType());
-      };
-
-      auto GEPIndexesScalableVectorField = [](const GEPOperator *GEP) {
-        if (!GEP->isInBounds() || GEP->getNumIndices() != 2 ||
-            !GEP->hasAllConstantIndices())
-          return false;
-
-        if (!isa<ScalableVectorType>(GEP->getResultElementType()))
-          return false;
-
-        // This is unlikely but just in case.
-        if (GEP->hasAllZeroIndices())
-          return false;
-
-        auto I = GEP->idx_begin();
-        const ConstantInt *Idx0 = cast<const ConstantInt>(*I++);
-        const ConstantInt *Idx1 = cast<const ConstantInt>(*I);
-
-        if (!Idx0->isZero())
-          return false;
-        if (Idx1->isZero())
-          return false;
-
-        return true;
-      };
-
-      // FIXME: This is perhaps too specific.
-      if (UnderlyingV1 == V2 && IsPointerToStruct(UnderlyingV1->getType()) &&
-          GEPIndexesScalableVectorField(GEP1))
-        return NoAlias;
-
-      return MayAlias;
-    }
-
-    // TODO: This limitation exists for compile-time reasons. Relax it if we
-    // can avoid exponential pathological cases.
-    if (!V1Size.hasValue() && !V2Size.hasValue())
-      return MayAlias;
   }
 
   // For GEPs with identical offsets, we can preserve the size and AAInfo
@@ -1737,7 +1705,7 @@ bool BasicAAResult::isValueEqualInPotentialCycles(const Value *V,
   // the Values cannot come from different iterations of a potential cycle the
   // phi nodes could be involved in.
   for (auto *P : VisitedPhiBBs)
-    if (isPotentiallyReachable(&P->front(), Inst, nullptr, DT, LI))
+    if (isPotentiallyReachable(&P->front(), Inst, nullptr, DT))
       return false;
 
   return true;
@@ -1797,28 +1765,20 @@ bool BasicAAResult::constantOffsetHeuristic(
   const VariableGEPIndex &Var0 = VarIndices[0], &Var1 = VarIndices[1];
 
   if (Var0.ZExtBits != Var1.ZExtBits || Var0.SExtBits != Var1.SExtBits ||
-      Var0.Scale != -Var1.Scale)
+      Var0.Scale != -Var1.Scale || Var0.V->getType() != Var1.V->getType())
     return false;
 
-  unsigned Width = Var1.V->getType()->getIntegerBitWidth();
-
   // We'll strip off the Extensions of Var0 and Var1 and do another round
   // of GetLinearExpression decomposition. In the example above, if Var0
   // is zext(%x + 1) we should get V1 == %x and V1Offset == 1.
 
-  APInt V0Scale(Width, 0), V0Offset(Width, 0), V1Scale(Width, 0),
-      V1Offset(Width, 0);
-  bool NSW = true, NUW = true;
-  unsigned V0ZExtBits = 0, V0SExtBits = 0, V1ZExtBits = 0, V1SExtBits = 0;
-  const Value *V0 = GetLinearExpression(Var0.V, V0Scale, V0Offset, V0ZExtBits,
-                                        V0SExtBits, DL, 0, AC, DT, NSW, NUW);
-  NSW = true;
-  NUW = true;
-  const Value *V1 = GetLinearExpression(Var1.V, V1Scale, V1Offset, V1ZExtBits,
-                                        V1SExtBits, DL, 0, AC, DT, NSW, NUW);
-
-  if (V0Scale != V1Scale || V0ZExtBits != V1ZExtBits ||
-      V0SExtBits != V1SExtBits || !isValueEqualInPotentialCycles(V0, V1))
+  LinearExpression E0 =
+      GetLinearExpression(ExtendedValue(Var0.V), DL, 0, AC, DT);
+  LinearExpression E1 =
+      GetLinearExpression(ExtendedValue(Var1.V), DL, 0, AC, DT);
+  if (E0.Scale != E1.Scale || E0.Val.ZExtBits != E1.Val.ZExtBits ||
+      E0.Val.SExtBits != E1.Val.SExtBits ||
+      !isValueEqualInPotentialCycles(E0.Val.V, E1.Val.V))
     return false;
 
   // We have a hit - Var0 and Var1 only differ by a constant offset!
@@ -1828,7 +1788,7 @@ bool BasicAAResult::constantOffsetHeuristic(
   // minimum difference between the two. The minimum distance may occur due to
   // wrapping; consider "add i3 %i, 5": if %i == 7 then 7 + 5 mod 8 == 4, and so
   // the minimum distance between %i and %i + 5 is 3.
-  APInt MinDiff = V0Offset - V1Offset, Wrapped = -MinDiff;
+  APInt MinDiff = E0.Offset - E1.Offset, Wrapped = -MinDiff;
   MinDiff = APIntOps::umin(MinDiff, Wrapped);
   APInt MinDiffBytes =
     MinDiff.zextOrTrunc(Var0.Scale.getBitWidth()) * Var0.Scale.abs();
@@ -1851,9 +1811,8 @@ BasicAAResult BasicAA::run(Function &F, FunctionAnalysisManager &AM) {
   auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
   auto &AC = AM.getResult<AssumptionAnalysis>(F);
   auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *LI = AM.getCachedResult<LoopAnalysis>(F);
   auto *PV = AM.getCachedResult<PhiValuesAnalysis>(F);
-  return BasicAAResult(F.getParent()->getDataLayout(), F, TLI, AC, DT, LI, PV);
+  return BasicAAResult(F.getParent()->getDataLayout(), F, TLI, AC, DT, PV);
 }
 
 BasicAAWrapperPass::BasicAAWrapperPass() : FunctionPass(ID) {
@@ -1881,13 +1840,11 @@ bool BasicAAWrapperPass::runOnFunction(Function &F) {
   auto &ACT = getAnalysis<AssumptionCacheTracker>();
   auto &TLIWP = getAnalysis<TargetLibraryInfoWrapperPass>();
   auto &DTWP = getAnalysis<DominatorTreeWrapperPass>();
-  auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>();
   auto *PVWP = getAnalysisIfAvailable<PhiValuesWrapperPass>();
 
   Result.reset(new BasicAAResult(F.getParent()->getDataLayout(), F,
                                  TLIWP.getTLI(F), ACT.getAssumptionCache(F),
                                  &DTWP.getDomTree(),
-                                 LIWP ? &LIWP->getLoopInfo() : nullptr,
                                  PVWP ? &PVWP->getResult() : nullptr));
 
   return false;
diff --git a/llvm/lib/Analysis/CFLGraph.h b/llvm/lib/Analysis/CFLGraph.h
index 21842ed364876e6cb9279dd0f9b0fceec1807cb7..02a13d673f40bee2b61e2b4f138784305ad5fba6 100644
--- a/llvm/lib/Analysis/CFLGraph.h
+++ b/llvm/lib/Analysis/CFLGraph.h
@@ -284,6 +284,13 @@ template <typename CFLAA> class CFLGraphBuilder {
       addAssignEdge(Src, &Inst);
     }
 
+    void visitFreezeInst(FreezeInst &Inst) {
+      // Accessing freeze(ptr) is equivalent to accessing ptr.
+      // The former raises UB iff latter raises UB.
+      auto *Src = Inst.getOperand(0);
+      addAssignEdge(Src, &Inst);
+    }
+
     void visitBinaryOperator(BinaryOperator &Inst) {
       auto *Op1 = Inst.getOperand(0);
       auto *Op2 = Inst.getOperand(1);
diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp
index 0a05c3c875e08a13654d11c8c7fee0ee10fd17f2..eaaa3d09a7f287bb0c91d3c0c5c8d041105277af 100644
--- a/llvm/lib/Analysis/CGSCCPassManager.cpp
+++ b/llvm/lib/Analysis/CGSCCPassManager.cpp
@@ -720,7 +720,7 @@ bool FunctionAnalysisManagerCGSCCProxy::Result::invalidate(
   auto PAC = PA.getChecker<FunctionAnalysisManagerCGSCCProxy>();
   if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<LazyCallGraph::SCC>>()) {
     for (LazyCallGraph::Node &N : C)
-      FAM->clear(N.getFunction(), N.getFunction().getName());
+      FAM->invalidate(N.getFunction(), PA);
 
     return false;
   }
@@ -1043,9 +1043,9 @@ static LazyCallGraph::SCC &updateCGAndAnalysisManagerForPass(
     if (&TargetRC == RC)
       return false;
 
-    RC->removeOutgoingEdge(N, *TargetN);
     LLVM_DEBUG(dbgs() << "Deleting outgoing edge from '" << N << "' to '"
-                      << TargetN << "'\n");
+                      << *TargetN << "'\n");
+    RC->removeOutgoingEdge(N, *TargetN);
     return true;
   });
 
diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp
index b2fc6e603f9ea8b480e5dfe25a44778180ece280..cf5e53b26b5d817b504d17cd52fbd00e2e284edd 100644
--- a/llvm/lib/Analysis/CaptureTracking.cpp
+++ b/llvm/lib/Analysis/CaptureTracking.cpp
@@ -68,8 +68,8 @@ bool CaptureTracker::isDereferenceableOrNull(Value *O, const DataLayout &DL) {
   if (auto *GEP = dyn_cast<GetElementPtrInst>(O))
     if (GEP->isInBounds())
       return true;
-  bool CanBeNull;
-  return O->getPointerDereferenceableBytes(DL, CanBeNull);
+  bool CanBeNull, CanBeFreed;
+  return O->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
 }
 
 namespace {
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 4d7237becbe66ef95388bf3615c4ce1516ebfeb5..be164a5a29f5a9071863e60a0e9cb53ad8b4d51e 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -387,6 +387,11 @@ Constant *llvm::ConstantFoldLoadThroughBitcast(Constant *C, Type *DestTy,
         return ConstantExpr::getCast(Cast, C, DestTy);
     }
 
+    // If this isn't an aggregate type, there is nothing we can do to drill down
+    // and find a bitcastable constant.
+    if (!SrcTy->isAggregateType() && !SrcTy->isVectorTy())
+      return nullptr;
+
     // We're simulating a load through a pointer that was bitcast to point to
     // a different type, so we can try to walk down through the initial
     // elements of an aggregate to see if some part of the aggregate is
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index b742259758d86665c855b6bcf330537e0f1d7be0..f4e22d47b98aa65f5e9585e99df8aaf56ca86608 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -489,6 +489,9 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
   // sense that it's not weighted by profile counts at all.
   int ColdSize = 0;
 
+  // Whether inlining is decided by cost-benefit analysis.
+  bool DecidedByCostBenefit = false;
+
   bool SingleBB = true;
 
   unsigned SROACostSavings = 0;
@@ -672,15 +675,22 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
   }
 
   bool isCostBenefitAnalysisEnabled() {
-    if (!InlineEnableCostBenefitAnalysis)
-      return false;
-
     if (!PSI || !PSI->hasProfileSummary())
       return false;
 
     if (!GetBFI)
       return false;
 
+    if (InlineEnableCostBenefitAnalysis.getNumOccurrences()) {
+      // Honor the explicit request from the user.
+      if (!InlineEnableCostBenefitAnalysis)
+        return false;
+    } else {
+      // Otherwise, require instrumentation profile.
+      if (!PSI->hasInstrumentationProfile())
+        return false;
+    }
+
     auto *Caller = CandidateCall.getParent()->getParent();
     if (!Caller->getEntryCount())
       return false;
@@ -693,7 +703,9 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
     if (!PSI->isHotCallSite(CandidateCall, CallerBFI))
       return false;
 
-    if (!F.getEntryCount())
+    // Make sure we have a nonzero entry count.
+    auto EntryCount = F.getEntryCount();
+    if (!EntryCount || !EntryCount.getCount())
       return false;
 
     BlockFrequencyInfo *CalleeBFI = &(GetBFI(F));
@@ -762,7 +774,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
 
     // Compute the cycle savings per call.
     auto EntryProfileCount = F.getEntryCount();
-    assert(EntryProfileCount.hasValue());
+    assert(EntryProfileCount.hasValue() && EntryProfileCount.getCount());
     auto EntryCount = EntryProfileCount.getCount();
     CycleSavings += EntryCount / 2;
     CycleSavings = CycleSavings.udiv(EntryCount);
@@ -825,6 +837,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
       Threshold -= VectorBonus / 2;
 
     if (auto Result = costBenefitAnalysis()) {
+      DecidedByCostBenefit = true;
       if (Result.getValue())
         return InlineResult::success();
       else
@@ -926,6 +939,7 @@ public:
   virtual ~InlineCostCallAnalyzer() {}
   int getThreshold() { return Threshold; }
   int getCost() { return Cost; }
+  bool wasDecidedByCostBenefit() { return DecidedByCostBenefit; }
 };
 } // namespace
 
@@ -2609,6 +2623,16 @@ InlineCost llvm::getInlineCost(
 
   LLVM_DEBUG(CA.dump());
 
+  // Always make cost benefit based decision explicit.
+  // We use always/never here since threshold is not meaningful,
+  // as it's not what drives cost-benefit analysis.
+  if (CA.wasDecidedByCostBenefit()) {
+    if (ShouldInline.isSuccess())
+      return InlineCost::getAlways("benefit over cost");
+    else
+      return InlineCost::getNever("cost over benefit");
+  }
+
   // Check if there was a reason to force inlining or no inlining.
   if (!ShouldInline.isSuccess() && CA.getCost() < CA.getThreshold())
     return InlineCost::getNever(ShouldInline.getFailureReason());
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index f63b4c7022433374f455e4b249d95ddec79aecc2..290d081b6e87e9d1dbf083ae56dfd904d10ae111 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -68,6 +68,8 @@ static Value *SimplifyCastInst(unsigned, Value *, Type *,
                                const SimplifyQuery &, unsigned);
 static Value *SimplifyGEPInst(Type *, ArrayRef<Value *>, const SimplifyQuery &,
                               unsigned);
+static Value *SimplifySelectInst(Value *, Value *, Value *,
+                                 const SimplifyQuery &, unsigned);
 
 static Value *foldSelectWithBinaryOp(Value *Cond, Value *TrueVal,
                                      Value *FalseVal) {
@@ -3928,103 +3930,104 @@ static Value *SimplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
     return nullptr;
 
   auto *I = dyn_cast<Instruction>(V);
-  if (!I)
-    return nullptr;
-
-  // Consider:
-  //   %cmp = icmp eq i32 %x, 2147483647
-  //   %add = add nsw i32 %x, 1
-  //   %sel = select i1 %cmp, i32 -2147483648, i32 %add
-  //
-  // We can't replace %sel with %add unless we strip away the flags (which will
-  // be done in InstCombine).
-  // TODO: This is unsound, because it only catches some forms of refinement.
-  if (!AllowRefinement && canCreatePoison(cast<Operator>(I)))
+  if (!I || !is_contained(I->operands(), Op))
     return nullptr;
 
-  // The simplification queries below may return the original value. Consider:
-  //   %div = udiv i32 %arg, %arg2
-  //   %mul = mul nsw i32 %div, %arg2
-  //   %cmp = icmp eq i32 %mul, %arg
-  //   %sel = select i1 %cmp, i32 %div, i32 undef
-  // Replacing %arg by %mul, %div becomes "udiv i32 %mul, %arg2", which
-  // simplifies back to %arg. This can only happen because %mul does not
-  // dominate %div. To ensure a consistent return value contract, we make sure
-  // that this case returns nullptr as well.
-  auto PreventSelfSimplify = [V](Value *Simplified) {
-    return Simplified != V ? Simplified : nullptr;
-  };
-
-  // If this is a binary operator, try to simplify it with the replaced op.
-  if (auto *B = dyn_cast<BinaryOperator>(I)) {
-    if (MaxRecurse) {
-      if (B->getOperand(0) == Op)
-        return PreventSelfSimplify(SimplifyBinOp(B->getOpcode(), RepOp,
-                                                 B->getOperand(1), Q,
-                                                 MaxRecurse - 1));
-      if (B->getOperand(1) == Op)
-        return PreventSelfSimplify(SimplifyBinOp(B->getOpcode(),
-                                                 B->getOperand(0), RepOp, Q,
-                                                 MaxRecurse - 1));
+  // Replace Op with RepOp in instruction operands.
+  SmallVector<Value *, 8> NewOps(I->getNumOperands());
+  transform(I->operands(), NewOps.begin(),
+            [&](Value *V) { return V == Op ? RepOp : V; });
+
+  if (!AllowRefinement) {
+    // General InstSimplify functions may refine the result, e.g. by returning
+    // a constant for a potentially poison value. To avoid this, implement only
+    // a few non-refining but profitable transforms here.
+
+    if (auto *BO = dyn_cast<BinaryOperator>(I)) {
+      unsigned Opcode = BO->getOpcode();
+      // id op x -> x, x op id -> x
+      if (NewOps[0] == ConstantExpr::getBinOpIdentity(Opcode, I->getType()))
+        return NewOps[1];
+      if (NewOps[1] == ConstantExpr::getBinOpIdentity(Opcode, I->getType(),
+                                                      /* RHS */ true))
+        return NewOps[0];
+
+      // x & x -> x, x | x -> x
+      if ((Opcode == Instruction::And || Opcode == Instruction::Or) &&
+          NewOps[0] == NewOps[1])
+        return NewOps[0];
     }
-  }
 
-  // Same for CmpInsts.
-  if (CmpInst *C = dyn_cast<CmpInst>(I)) {
-    if (MaxRecurse) {
-      if (C->getOperand(0) == Op)
-        return PreventSelfSimplify(SimplifyCmpInst(C->getPredicate(), RepOp,
-                                                   C->getOperand(1), Q,
-                                                   MaxRecurse - 1));
-      if (C->getOperand(1) == Op)
-        return PreventSelfSimplify(SimplifyCmpInst(C->getPredicate(),
-                                                   C->getOperand(0), RepOp, Q,
-                                                   MaxRecurse - 1));
+    if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
+      // getelementptr x, 0 -> x
+      if (NewOps.size() == 2 && match(NewOps[1], m_Zero()) &&
+          !GEP->isInBounds())
+        return NewOps[0];
     }
-  }
+  } else if (MaxRecurse) {
+    // The simplification queries below may return the original value. Consider:
+    //   %div = udiv i32 %arg, %arg2
+    //   %mul = mul nsw i32 %div, %arg2
+    //   %cmp = icmp eq i32 %mul, %arg
+    //   %sel = select i1 %cmp, i32 %div, i32 undef
+    // Replacing %arg by %mul, %div becomes "udiv i32 %mul, %arg2", which
+    // simplifies back to %arg. This can only happen because %mul does not
+    // dominate %div. To ensure a consistent return value contract, we make sure
+    // that this case returns nullptr as well.
+    auto PreventSelfSimplify = [V](Value *Simplified) {
+      return Simplified != V ? Simplified : nullptr;
+    };
+
+    if (auto *B = dyn_cast<BinaryOperator>(I))
+      return PreventSelfSimplify(SimplifyBinOp(B->getOpcode(), NewOps[0],
+                                               NewOps[1], Q, MaxRecurse - 1));
 
-  // Same for GEPs.
-  if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
-    if (MaxRecurse) {
-      SmallVector<Value *, 8> NewOps(GEP->getNumOperands());
-      transform(GEP->operands(), NewOps.begin(),
-                [&](Value *V) { return V == Op ? RepOp : V; });
+    if (CmpInst *C = dyn_cast<CmpInst>(I))
+      return PreventSelfSimplify(SimplifyCmpInst(C->getPredicate(), NewOps[0],
+                                                 NewOps[1], Q, MaxRecurse - 1));
+
+    if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
       return PreventSelfSimplify(SimplifyGEPInst(GEP->getSourceElementType(),
                                                  NewOps, Q, MaxRecurse - 1));
-    }
-  }
 
-  // TODO: We could hand off more cases to instsimplify here.
+    if (isa<SelectInst>(I))
+      return PreventSelfSimplify(
+          SimplifySelectInst(NewOps[0], NewOps[1], NewOps[2], Q,
+                             MaxRecurse - 1));
+    // TODO: We could hand off more cases to instsimplify here.
+  }
 
   // If all operands are constant after substituting Op for RepOp then we can
   // constant fold the instruction.
-  if (Constant *CRepOp = dyn_cast<Constant>(RepOp)) {
-    // Build a list of all constant operands.
-    SmallVector<Constant *, 8> ConstOps;
-    for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
-      if (I->getOperand(i) == Op)
-        ConstOps.push_back(CRepOp);
-      else if (Constant *COp = dyn_cast<Constant>(I->getOperand(i)))
-        ConstOps.push_back(COp);
-      else
-        break;
-    }
+  SmallVector<Constant *, 8> ConstOps;
+  for (Value *NewOp : NewOps) {
+    if (Constant *ConstOp = dyn_cast<Constant>(NewOp))
+      ConstOps.push_back(ConstOp);
+    else
+      return nullptr;
+  }
 
-    // All operands were constants, fold it.
-    if (ConstOps.size() == I->getNumOperands()) {
-      if (CmpInst *C = dyn_cast<CmpInst>(I))
-        return ConstantFoldCompareInstOperands(C->getPredicate(), ConstOps[0],
-                                               ConstOps[1], Q.DL, Q.TLI);
+  // Consider:
+  //   %cmp = icmp eq i32 %x, 2147483647
+  //   %add = add nsw i32 %x, 1
+  //   %sel = select i1 %cmp, i32 -2147483648, i32 %add
+  //
+  // We can't replace %sel with %add unless we strip away the flags (which
+  // will be done in InstCombine).
+  // TODO: This may be unsound, because it only catches some forms of
+  // refinement.
+  if (!AllowRefinement && canCreatePoison(cast<Operator>(I)))
+    return nullptr;
 
-      if (LoadInst *LI = dyn_cast<LoadInst>(I))
-        if (!LI->isVolatile())
-          return ConstantFoldLoadFromConstPtr(ConstOps[0], LI->getType(), Q.DL);
+  if (CmpInst *C = dyn_cast<CmpInst>(I))
+    return ConstantFoldCompareInstOperands(C->getPredicate(), ConstOps[0],
+                                           ConstOps[1], Q.DL, Q.TLI);
 
-      return ConstantFoldInstOperands(I, ConstOps, Q.DL, Q.TLI);
-    }
-  }
+  if (LoadInst *LI = dyn_cast<LoadInst>(I))
+    if (!LI->isVolatile())
+      return ConstantFoldLoadFromConstPtr(ConstOps[0], LI->getType(), Q.DL);
 
-  return nullptr;
+  return ConstantFoldInstOperands(I, ConstOps, Q.DL, Q.TLI);
 }
 
 Value *llvm::SimplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
@@ -4349,40 +4352,32 @@ static Value *SimplifyGEPInst(Type *SrcTy, ArrayRef<Value *> Ops,
       // doesn't truncate the pointers.
       if (Ops[1]->getType()->getScalarSizeInBits() ==
           Q.DL.getPointerSizeInBits(AS)) {
-        auto PtrToInt = [GEPTy](Value *P) -> Value * {
-          Value *Temp;
-          if (match(P, m_PtrToInt(m_Value(Temp))))
-            if (Temp->getType() == GEPTy)
-              return Temp;
-          return nullptr;
+        auto CanSimplify = [GEPTy, &P, V = Ops[0]]() -> bool {
+          return P->getType() == GEPTy &&
+                 getUnderlyingObject(P) == getUnderlyingObject(V);
         };
-
-        // FIXME: The following transforms are only legal if P and V have the
-        // same provenance (PR44403). Check whether getUnderlyingObject() is
-        // the same?
-
         // getelementptr V, (sub P, V) -> P if P points to a type of size 1.
         if (TyAllocSize == 1 &&
-            match(Ops[1], m_Sub(m_Value(P), m_PtrToInt(m_Specific(Ops[0])))))
-          if (Value *R = PtrToInt(P))
-            return R;
-
-        // getelementptr V, (ashr (sub P, V), C) -> Q
-        // if P points to a type of size 1 << C.
-        if (match(Ops[1],
-                  m_AShr(m_Sub(m_Value(P), m_PtrToInt(m_Specific(Ops[0]))),
-                         m_ConstantInt(C))) &&
-            TyAllocSize == 1ULL << C)
-          if (Value *R = PtrToInt(P))
-            return R;
-
-        // getelementptr V, (sdiv (sub P, V), C) -> Q
-        // if P points to a type of size C.
-        if (match(Ops[1],
-                  m_SDiv(m_Sub(m_Value(P), m_PtrToInt(m_Specific(Ops[0]))),
-                         m_SpecificInt(TyAllocSize))))
-          if (Value *R = PtrToInt(P))
-            return R;
+            match(Ops[1], m_Sub(m_PtrToInt(m_Value(P)),
+                                m_PtrToInt(m_Specific(Ops[0])))) &&
+            CanSimplify())
+          return P;
+
+        // getelementptr V, (ashr (sub P, V), C) -> P if P points to a type of
+        // size 1 << C.
+        if (match(Ops[1], m_AShr(m_Sub(m_PtrToInt(m_Value(P)),
+                                       m_PtrToInt(m_Specific(Ops[0]))),
+                                 m_ConstantInt(C))) &&
+            TyAllocSize == 1ULL << C && CanSimplify())
+          return P;
+
+        // getelementptr V, (sdiv (sub P, V), C) -> P if P points to a type of
+        // size C.
+        if (match(Ops[1], m_SDiv(m_Sub(m_PtrToInt(m_Value(P)),
+                                       m_PtrToInt(m_Specific(Ops[0]))),
+                                 m_SpecificInt(TyAllocSize))) &&
+            CanSimplify())
+          return P;
       }
     }
   }
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 266118bbb8342886a0dbddb386bc8b2f2b976166..8d9330ca86148f1208354e60d844bad85d20264c 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
@@ -67,10 +68,12 @@ static bool isDereferenceableAndAlignedPointer(
           Visited, MaxDepth);
   }
 
-  bool CheckForNonNull = false;
+  bool CheckForNonNull, CheckForFreed;
   APInt KnownDerefBytes(Size.getBitWidth(),
-                        V->getPointerDereferenceableBytes(DL, CheckForNonNull));
-  if (KnownDerefBytes.getBoolValue() && KnownDerefBytes.uge(Size))
+                        V->getPointerDereferenceableBytes(DL, CheckForNonNull,
+                                                          CheckForFreed));
+  if (KnownDerefBytes.getBoolValue() && KnownDerefBytes.uge(Size) &&
+      !CheckForFreed)
     if (!CheckForNonNull || isKnownNonZero(V, DL, 0, nullptr, CtxI, DT)) {
       // As we recursed through GEPs to get here, we've incrementally checked
       // that each step advanced by a multiple of the alignment. If our base is
@@ -435,21 +438,24 @@ Value *llvm::FindAvailableLoadedValue(LoadInst *Load,
   if (!Load->isUnordered())
     return nullptr;
 
-  return FindAvailablePtrLoadStore(
-      Load->getPointerOperand(), Load->getType(), Load->isAtomic(), ScanBB,
-      ScanFrom, MaxInstsToScan, AA, IsLoad, NumScanedInst);
+  MemoryLocation Loc = MemoryLocation::get(Load);
+  return findAvailablePtrLoadStore(Loc, Load->getType(), Load->isAtomic(),
+                                   ScanBB, ScanFrom, MaxInstsToScan, AA, IsLoad,
+                                   NumScanedInst);
 }
 
 // Check if the load and the store have the same base, constant offsets and
 // non-overlapping access ranges.
-static bool AreNonOverlapSameBaseLoadAndStore(
-    Value *LoadPtr, Type *LoadTy, Value *StorePtr, Type *StoreTy,
-    const DataLayout &DL) {
+static bool areNonOverlapSameBaseLoadAndStore(const Value *LoadPtr,
+                                              Type *LoadTy,
+                                              const Value *StorePtr,
+                                              Type *StoreTy,
+                                              const DataLayout &DL) {
   APInt LoadOffset(DL.getTypeSizeInBits(LoadPtr->getType()), 0);
   APInt StoreOffset(DL.getTypeSizeInBits(StorePtr->getType()), 0);
-  Value *LoadBase = LoadPtr->stripAndAccumulateConstantOffsets(
+  const Value *LoadBase = LoadPtr->stripAndAccumulateConstantOffsets(
       DL, LoadOffset, /* AllowNonInbounds */ false);
-  Value *StoreBase = StorePtr->stripAndAccumulateConstantOffsets(
+  const Value *StoreBase = StorePtr->stripAndAccumulateConstantOffsets(
       DL, StoreOffset, /* AllowNonInbounds */ false);
   if (LoadBase != StoreBase)
     return false;
@@ -462,7 +468,7 @@ static bool AreNonOverlapSameBaseLoadAndStore(
   return LoadRange.intersectWith(StoreRange).isEmptySet();
 }
 
-static Value *getAvailableLoadStore(Instruction *Inst, Value *Ptr,
+static Value *getAvailableLoadStore(Instruction *Inst, const Value *Ptr,
                                     Type *AccessTy, bool AtLeastAtomic,
                                     const DataLayout &DL, bool *IsLoadCSE) {
   // If this is a load of Ptr, the loaded value is available.
@@ -509,20 +515,15 @@ static Value *getAvailableLoadStore(Instruction *Inst, Value *Ptr,
   return nullptr;
 }
 
-Value *llvm::FindAvailablePtrLoadStore(Value *Ptr, Type *AccessTy,
-                                       bool AtLeastAtomic, BasicBlock *ScanBB,
-                                       BasicBlock::iterator &ScanFrom,
-                                       unsigned MaxInstsToScan,
-                                       AAResults *AA, bool *IsLoadCSE,
-                                       unsigned *NumScanedInst) {
+Value *llvm::findAvailablePtrLoadStore(
+    const MemoryLocation &Loc, Type *AccessTy, bool AtLeastAtomic,
+    BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan,
+    AAResults *AA, bool *IsLoadCSE, unsigned *NumScanedInst) {
   if (MaxInstsToScan == 0)
     MaxInstsToScan = ~0U;
 
   const DataLayout &DL = ScanBB->getModule()->getDataLayout();
-
-  // Try to get the store size for the type.
-
-  Value *StrippedPtr = Ptr->stripPointerCasts();
+  const Value *StrippedPtr = Loc.Ptr->stripPointerCasts();
 
   while (ScanFrom != ScanBB->begin()) {
     // We must ignore debug info directives when counting (otherwise they
@@ -548,8 +549,6 @@ Value *llvm::FindAvailablePtrLoadStore(Value *Ptr, Type *AccessTy,
       return Available;
 
     // Try to get the store size for the type.
-    auto AccessSize = LocationSize::precise(DL.getTypeStoreSize(AccessTy));
-
     if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
       Value *StorePtr = SI->getPointerOperand()->stripPointerCasts();
 
@@ -566,14 +565,14 @@ Value *llvm::FindAvailablePtrLoadStore(Value *Ptr, Type *AccessTy,
         // base, constant offsets and non-overlapping access ranges, ignore the
         // store. This is a simple form of alias analysis that is used by the
         // inliner. FIXME: use BasicAA if possible.
-        if (AreNonOverlapSameBaseLoadAndStore(
-                Ptr, AccessTy, SI->getPointerOperand(),
+        if (areNonOverlapSameBaseLoadAndStore(
+                Loc.Ptr, AccessTy, SI->getPointerOperand(),
                 SI->getValueOperand()->getType(), DL))
           continue;
       } else {
         // If we have alias analysis and it says the store won't modify the
         // loaded value, ignore the store.
-        if (!isModSet(AA->getModRefInfo(SI, StrippedPtr, AccessSize)))
+        if (!isModSet(AA->getModRefInfo(SI, Loc)))
           continue;
       }
 
@@ -586,7 +585,7 @@ Value *llvm::FindAvailablePtrLoadStore(Value *Ptr, Type *AccessTy,
     if (Inst->mayWriteToMemory()) {
       // If alias analysis claims that it really won't modify the load,
       // ignore it.
-      if (AA && !isModSet(AA->getModRefInfo(Inst, StrippedPtr, AccessSize)))
+      if (AA && !isModSet(AA->getModRefInfo(Inst, Loc)))
         continue;
 
       // May modify the pointer, bail out.
@@ -636,9 +635,9 @@ Value *llvm::FindAvailableLoadedValue(LoadInst *Load, AAResults &AA,
   // If we found an available value, ensure that the instructions in between
   // did not modify the memory location.
   if (Available) {
-    auto AccessSize = LocationSize::precise(DL.getTypeStoreSize(AccessTy));
+    MemoryLocation Loc = MemoryLocation::get(Load);
     for (Instruction *Inst : MustNotAliasInsts)
-      if (isModSet(AA.getModRefInfo(Inst, StrippedPtr, AccessSize)))
+      if (isModSet(AA.getModRefInfo(Inst, Loc)))
         return nullptr;
   }
 
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index e632fe25c24c855fbed367e27f380a3df66a505e..2a6987848ca5956d417a59f790844139fa805ad0 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1124,139 +1124,123 @@ int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr,
   return Stride;
 }
 
+Optional<int> llvm::getPointersDiff(Value *PtrA, Value *PtrB,
+                                    const DataLayout &DL, ScalarEvolution &SE,
+                                    bool StrictCheck, bool CheckType) {
+  assert(PtrA && PtrB && "Expected non-nullptr pointers.");
+  // Make sure that A and B are different pointers.
+  if (PtrA == PtrB)
+    return 0;
+
+  // Make sure that PtrA and PtrB have the same type if required
+  if (CheckType && PtrA->getType() != PtrB->getType())
+    return None;
+
+  unsigned ASA = PtrA->getType()->getPointerAddressSpace();
+  unsigned ASB = PtrB->getType()->getPointerAddressSpace();
+
+  // Check that the address spaces match.
+  if (ASA != ASB)
+    return None;
+  unsigned IdxWidth = DL.getIndexSizeInBits(ASA);
+
+  APInt OffsetA(IdxWidth, 0), OffsetB(IdxWidth, 0);
+  Value *PtrA1 = PtrA->stripAndAccumulateInBoundsConstantOffsets(DL, OffsetA);
+  Value *PtrB1 = PtrB->stripAndAccumulateInBoundsConstantOffsets(DL, OffsetB);
+
+  int Val;
+  if (PtrA1 == PtrB1) {
+    // Retrieve the address space again as pointer stripping now tracks through
+    // `addrspacecast`.
+    ASA = cast<PointerType>(PtrA1->getType())->getAddressSpace();
+    ASB = cast<PointerType>(PtrB1->getType())->getAddressSpace();
+    // Check that the address spaces match and that the pointers are valid.
+    if (ASA != ASB)
+      return None;
+
+    IdxWidth = DL.getIndexSizeInBits(ASA);
+    OffsetA = OffsetA.sextOrTrunc(IdxWidth);
+    OffsetB = OffsetB.sextOrTrunc(IdxWidth);
+
+    OffsetB -= OffsetA;
+    Val = OffsetB.getSExtValue();
+  } else {
+    // Otherwise compute the distance with SCEV between the base pointers.
+    const SCEV *PtrSCEVA = SE.getSCEV(PtrA);
+    const SCEV *PtrSCEVB = SE.getSCEV(PtrB);
+    const auto *Diff =
+        dyn_cast<SCEVConstant>(SE.getMinusSCEV(PtrSCEVB, PtrSCEVA));
+    if (!Diff)
+      return None;
+    Val = Diff->getAPInt().getSExtValue();
+  }
+  Type *Ty = cast<PointerType>(PtrA->getType())->getElementType();
+  int Size = DL.getTypeStoreSize(Ty);
+  int Dist = Val / Size;
+
+  // Ensure that the calculated distance matches the type-based one after all
+  // the bitcasts removal in the provided pointers.
+  if (!StrictCheck || Dist * Size == Val)
+    return Dist;
+  return None;
+}
+
 bool llvm::sortPtrAccesses(ArrayRef<Value *> VL, const DataLayout &DL,
                            ScalarEvolution &SE,
                            SmallVectorImpl<unsigned> &SortedIndices) {
   assert(llvm::all_of(
              VL, [](const Value *V) { return V->getType()->isPointerTy(); }) &&
          "Expected list of pointer operands.");
-  SmallVector<std::pair<int64_t, Value *>, 4> OffValPairs;
-  OffValPairs.reserve(VL.size());
-
   // Walk over the pointers, and map each of them to an offset relative to
   // first pointer in the array.
   Value *Ptr0 = VL[0];
-  const SCEV *Scev0 = SE.getSCEV(Ptr0);
-  Value *Obj0 = getUnderlyingObject(Ptr0);
-
-  llvm::SmallSet<int64_t, 4> Offsets;
-  for (auto *Ptr : VL) {
-    // TODO: Outline this code as a special, more time consuming, version of
-    // computeConstantDifference() function.
-    if (Ptr->getType()->getPointerAddressSpace() !=
-        Ptr0->getType()->getPointerAddressSpace())
-      return false;
-    // If a pointer refers to a different underlying object, bail - the
-    // pointers are by definition incomparable.
-    Value *CurrObj = getUnderlyingObject(Ptr);
-    if (CurrObj != Obj0)
-      return false;
 
-    const SCEV *Scev = SE.getSCEV(Ptr);
-    const auto *Diff = dyn_cast<SCEVConstant>(SE.getMinusSCEV(Scev, Scev0));
-    // The pointers may not have a constant offset from each other, or SCEV
-    // may just not be smart enough to figure out they do. Regardless,
-    // there's nothing we can do.
+  using DistOrdPair = std::pair<int64_t, int>;
+  auto Compare = [](const DistOrdPair &L, const DistOrdPair &R) {
+    return L.first < R.first;
+  };
+  std::set<DistOrdPair, decltype(Compare)> Offsets(Compare);
+  Offsets.emplace(0, 0);
+  int Cnt = 1;
+  bool IsConsecutive = true;
+  for (auto *Ptr : VL.drop_front()) {
+    Optional<int> Diff =
+        getPointersDiff(Ptr0, Ptr, DL, SE, /*StrictCheck=*/true);
     if (!Diff)
       return false;
 
     // Check if the pointer with the same offset is found.
-    int64_t Offset = Diff->getAPInt().getSExtValue();
-    if (!Offsets.insert(Offset).second)
+    int64_t Offset = *Diff;
+    auto Res = Offsets.emplace(Offset, Cnt);
+    if (!Res.second)
       return false;
-    OffValPairs.emplace_back(Offset, Ptr);
+    // Consecutive order if the inserted element is the last one.
+    IsConsecutive = IsConsecutive && std::next(Res.first) == Offsets.end();
+    ++Cnt;
   }
   SortedIndices.clear();
-  SortedIndices.resize(VL.size());
-  std::iota(SortedIndices.begin(), SortedIndices.end(), 0);
-
-  // Sort the memory accesses and keep the order of their uses in UseOrder.
-  llvm::stable_sort(SortedIndices, [&](unsigned Left, unsigned Right) {
-    return OffValPairs[Left].first < OffValPairs[Right].first;
-  });
-
-  // Check if the order is consecutive already.
-  if (llvm::all_of(SortedIndices, [&SortedIndices](const unsigned I) {
-        return I == SortedIndices[I];
-      }))
-    SortedIndices.clear();
-
+  if (!IsConsecutive) {
+    // Fill SortedIndices array only if it is non-consecutive.
+    SortedIndices.resize(VL.size());
+    Cnt = 0;
+    for (const std::pair<int64_t, int> &Pair : Offsets) {
+      SortedIndices[Cnt] = Pair.second;
+      ++Cnt;
+    }
+  }
   return true;
 }
 
-/// Take the address space operand from the Load/Store instruction.
-/// Returns -1 if this is not a valid Load/Store instruction.
-static unsigned getAddressSpaceOperand(Value *I) {
-  if (LoadInst *L = dyn_cast<LoadInst>(I))
-    return L->getPointerAddressSpace();
-  if (StoreInst *S = dyn_cast<StoreInst>(I))
-    return S->getPointerAddressSpace();
-  return -1;
-}
-
 /// Returns true if the memory operations \p A and \p B are consecutive.
 bool llvm::isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL,
                                ScalarEvolution &SE, bool CheckType) {
   Value *PtrA = getLoadStorePointerOperand(A);
   Value *PtrB = getLoadStorePointerOperand(B);
-  unsigned ASA = getAddressSpaceOperand(A);
-  unsigned ASB = getAddressSpaceOperand(B);
-
-  // Check that the address spaces match and that the pointers are valid.
-  if (!PtrA || !PtrB || (ASA != ASB))
-    return false;
-
-  // Make sure that A and B are different pointers.
-  if (PtrA == PtrB)
-    return false;
-
-  // Make sure that A and B have the same type if required.
-  if (CheckType && PtrA->getType() != PtrB->getType())
+  if (!PtrA || !PtrB)
     return false;
-
-  unsigned IdxWidth = DL.getIndexSizeInBits(ASA);
-  Type *Ty = cast<PointerType>(PtrA->getType())->getElementType();
-
-  APInt OffsetA(IdxWidth, 0), OffsetB(IdxWidth, 0);
-  PtrA = PtrA->stripAndAccumulateInBoundsConstantOffsets(DL, OffsetA);
-  PtrB = PtrB->stripAndAccumulateInBoundsConstantOffsets(DL, OffsetB);
-
-  // Retrieve the address space again as pointer stripping now tracks through
-  // `addrspacecast`.
-  ASA = cast<PointerType>(PtrA->getType())->getAddressSpace();
-  ASB = cast<PointerType>(PtrB->getType())->getAddressSpace();
-  // Check that the address spaces match and that the pointers are valid.
-  if (ASA != ASB)
-    return false;
-
-  IdxWidth = DL.getIndexSizeInBits(ASA);
-  OffsetA = OffsetA.sextOrTrunc(IdxWidth);
-  OffsetB = OffsetB.sextOrTrunc(IdxWidth);
-
-  APInt Size(IdxWidth, DL.getTypeStoreSize(Ty));
-
-  //  OffsetDelta = OffsetB - OffsetA;
-  const SCEV *OffsetSCEVA = SE.getConstant(OffsetA);
-  const SCEV *OffsetSCEVB = SE.getConstant(OffsetB);
-  const SCEV *OffsetDeltaSCEV = SE.getMinusSCEV(OffsetSCEVB, OffsetSCEVA);
-  const APInt &OffsetDelta = cast<SCEVConstant>(OffsetDeltaSCEV)->getAPInt();
-
-  // Check if they are based on the same pointer. That makes the offsets
-  // sufficient.
-  if (PtrA == PtrB)
-    return OffsetDelta == Size;
-
-  // Compute the necessary base pointer delta to have the necessary final delta
-  // equal to the size.
-  // BaseDelta = Size - OffsetDelta;
-  const SCEV *SizeSCEV = SE.getConstant(Size);
-  const SCEV *BaseDelta = SE.getMinusSCEV(SizeSCEV, OffsetDeltaSCEV);
-
-  // Otherwise compute the distance with SCEV between the base pointers.
-  const SCEV *PtrSCEVA = SE.getSCEV(PtrA);
-  const SCEV *PtrSCEVB = SE.getSCEV(PtrB);
-  const SCEV *X = SE.getAddExpr(PtrSCEVA, BaseDelta);
-  return X == PtrSCEVB;
+  Optional<int> Diff =
+      getPointersDiff(PtrA, PtrB, DL, SE, /*StrictCheck=*/true, CheckType);
+  return Diff && *Diff == 1;
 }
 
 MemoryDepChecker::VectorizationSafetyStatus
diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp
index 5d3621935b8a4facc184cf139df09458b9d25458..93a00215e34ae6a4f091ae63dcfabf206146850a 100644
--- a/llvm/lib/Analysis/LoopInfo.cpp
+++ b/llvm/lib/Analysis/LoopInfo.cpp
@@ -924,6 +924,31 @@ void LoopInfo::erase(Loop *Unloop) {
   }
 }
 
+bool
+LoopInfo::wouldBeOutOfLoopUseRequiringLCSSA(const Value *V,
+                                            const BasicBlock *ExitBB) const {
+  if (V->getType()->isTokenTy())
+    // We can't form PHIs of token type, so the definition of LCSSA excludes
+    // values of that type.
+    return false;
+
+  const Instruction *I = dyn_cast<Instruction>(V);
+  if (!I)
+    return false;
+  const Loop *L = getLoopFor(I->getParent());
+  if (!L)
+    return false;
+  if (L->contains(ExitBB))
+    // Could be an exit bb of a subloop and contained in defining loop
+    return false;
+
+  // We found a (new) out-of-loop use location, for a value defined in-loop.
+  // (Note that because of LCSSA, we don't have to account for values defined
+  // in sibling loops.  Such values will have LCSSA phis of their own in the
+  // common parent loop.)
+  return true;
+}
+
 AnalysisKey LoopAnalysis::Key;
 
 LoopInfo LoopAnalysis::run(Function &F, FunctionAnalysisManager &AM) {
diff --git a/llvm/lib/Analysis/MemDerefPrinter.cpp b/llvm/lib/Analysis/MemDerefPrinter.cpp
index 0078ceacbad24cd4a3f6b7b8bdf068252c242333..1b16e1a9bcb2915e59f8cc0af9fa64e6e4f1612b 100644
--- a/llvm/lib/Analysis/MemDerefPrinter.cpp
+++ b/llvm/lib/Analysis/MemDerefPrinter.cpp
@@ -70,12 +70,13 @@ bool MemDerefPrinter::runOnFunction(Function &F) {
 void MemDerefPrinter::print(raw_ostream &OS, const Module *M) const {
   OS << "The following are dereferenceable:\n";
   for (Value *V: Deref) {
+    OS << "  ";
     V->print(OS);
     if (DerefAndAligned.count(V))
       OS << "\t(aligned)";
     else
       OS << "\t(unaligned)";
-    OS << "\n\n";
+    OS << "\n";
   }
 }
 
@@ -101,12 +102,13 @@ PreservedAnalyses MemDerefPrinterPass::run(Function &F,
 
   OS << "The following are dereferenceable:\n";
   for (Value *V : Deref) {
+    OS << "  ";
     V->print(OS);
     if (DerefAndAligned.count(V))
       OS << "\t(aligned)";
     else
       OS << "\t(unaligned)";
-    OS << "\n\n";
+    OS << "\n";
   }
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
index 886b5bf4acd387fbb452d62f183b1ad685521c08..3131da2f8b0ab68f1e038b69b8f724494122142a 100644
--- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -1135,9 +1135,6 @@ bool MemoryDependenceResults::getNonLocalPointerDepFromBB(
   // investigating, just return it with no recomputation.
   // Don't use cached information for invariant loads since it is valid for
   // non-invariant loads only.
-  //
-  // Don't use cached information for invariant loads since it is valid for
-  // non-invariant loads only.
   if (!IsIncomplete && !isInvariantLoad &&
       CacheInfo->Pair == BBSkipFirstBlockPair(StartBB, SkipFirstBlock)) {
     // We have a fully cached result for this query then we can just return the
diff --git a/llvm/lib/Analysis/MemorySSA.cpp b/llvm/lib/Analysis/MemorySSA.cpp
index 83e63410340909982f6c896dfa3243dc36f820ec..bdceaa790ff1686ff40c6f003cd07f42f258a4d3 100644
--- a/llvm/lib/Analysis/MemorySSA.cpp
+++ b/llvm/lib/Analysis/MemorySSA.cpp
@@ -2550,6 +2550,11 @@ bool upward_defs_iterator::IsGuaranteedLoopInvariant(Value *Ptr) const {
   };
 
   Ptr = Ptr->stripPointerCasts();
+  if (auto *I = dyn_cast<Instruction>(Ptr)) {
+    if (I->getParent() == &I->getFunction()->getEntryBlock()) {
+      return true;
+    }
+  }
   if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
     return IsGuaranteedLoopInvariantBase(GEP->getPointerOperand()) &&
            GEP->hasAllConstantIndices();
diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
index c9671d4f5c2e12ec3b82a1d827599695d79f431b..67712c3d818e8641fd7a64b6d60e2b881efeaaca 100644
--- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp
+++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
@@ -95,13 +95,13 @@ void ProfileSummaryInfo::refresh() {
   if (hasProfileSummary())
     return;
   // First try to get context sensitive ProfileSummary.
-  auto *SummaryMD = M.getProfileSummary(/* IsCS */ true);
+  auto *SummaryMD = M->getProfileSummary(/* IsCS */ true);
   if (SummaryMD)
     Summary.reset(ProfileSummary::getFromMD(SummaryMD));
 
   if (!hasProfileSummary()) {
     // This will actually return PSK_Instr or PSK_Sample summary.
-    SummaryMD = M.getProfileSummary(/* IsCS */ false);
+    SummaryMD = M->getProfileSummary(/* IsCS */ false);
     if (SummaryMD)
       Summary.reset(ProfileSummary::getFromMD(SummaryMD));
   }
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index c94aca57628284c514276df72882c56c12c79bf5..d0700399e5066d2f933f2806bc405cb85d72e32f 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -5645,6 +5645,77 @@ void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec,
   }
 }
 
+ConstantRange ScalarEvolution::
+getRangeForUnknownRecurrence(const SCEVUnknown *U) {
+  const DataLayout &DL = getDataLayout();
+
+  unsigned BitWidth = getTypeSizeInBits(U->getType());
+  ConstantRange CR(BitWidth, /*isFullSet=*/true);
+
+  // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
+  // use information about the trip count to improve our available range.  Note
+  // that the trip count independent cases are already handled by known bits.
+  // WARNING: The definition of recurrence used here is subtly different than
+  // the one used by AddRec (and thus most of this file).  Step is allowed to
+  // be arbitrarily loop varying here, where AddRec allows only loop invariant
+  // and other addrecs in the same loop (for non-affine addrecs).  The code
+  // below intentionally handles the case where step is not loop invariant.
+  auto *P = dyn_cast<PHINode>(U->getValue());
+  if (!P)
+    return CR;
+
+  BinaryOperator *BO;
+  Value *Start, *Step;
+  if (!matchSimpleRecurrence(P, BO, Start, Step))
+    return CR;
+
+  // If we found a recurrence, we must be in a loop -- unless we're
+  // in unreachable code where dominance collapses.  Note that BO might
+  // be in some subloop of L, and that's completely okay.
+  auto *L = LI.getLoopFor(P->getParent());
+  if (!L)
+    return CR;
+  assert(L->getHeader() == P->getParent());
+  if (!L->contains(BO->getParent()))
+    // NOTE: This bailout should be an assert instead.  However, asserting
+    // the condition here exposes a case where LoopFusion is querying SCEV
+    // with malformed loop information during the midst of the transform.
+    // There doesn't appear to be an obvious fix, so for the moment bailout
+    // until the caller issue can be fixed.  PR49566 tracks the bug.
+    return CR;
+
+  // TODO: Handle ashr and lshr cases to increase minimum value reported
+  if (BO->getOpcode() != Instruction::Shl || BO->getOperand(0) != P)
+    return CR;
+
+  unsigned TC = getSmallConstantMaxTripCount(L);
+  if (!TC || TC >= BitWidth)
+    return CR;
+
+  auto KnownStart = computeKnownBits(Start, DL, 0, &AC, nullptr, &DT);
+  auto KnownStep = computeKnownBits(Step, DL, 0, &AC, nullptr, &DT);
+  assert(KnownStart.getBitWidth() == BitWidth &&
+         KnownStep.getBitWidth() == BitWidth);
+
+  // Compute total shift amount, being careful of overflow and bitwidths.
+  auto MaxShiftAmt = KnownStep.getMaxValue();
+  APInt TCAP(BitWidth, TC-1);
+  bool Overflow = false;
+  auto TotalShift = MaxShiftAmt.umul_ov(TCAP, Overflow);
+  if (Overflow)
+    return CR;
+
+  // Iff no bits are shifted out, value increases on every shift.
+  auto KnownEnd = KnownBits::shl(KnownStart,
+                                 KnownBits::makeConstant(TotalShift));
+  if (TotalShift.ult(KnownStart.countMinLeadingZeros()))
+    CR = CR.intersectWith(ConstantRange(KnownStart.getMinValue(),
+                                        KnownEnd.getMaxValue() + 1));
+  return CR;
+}
+
+
+
 /// Determine the range for a particular SCEV.  If SignHint is
 /// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
 /// with a "cleaner" unsigned (resp. signed) representation.
@@ -5845,12 +5916,19 @@ ScalarEvolution::getRangeRef(const SCEV *S,
   }
 
   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
+
     // Check if the IR explicitly contains !range metadata.
     Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue());
     if (MDRange.hasValue())
       ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue(),
                                                             RangeType);
 
+    // Use facts about recurrences in the underlying IR.  Note that add
+    // recurrences are AddRecExprs and thus don't hit this path.  This
+    // primarily handles shift recurrences.
+    auto CR = getRangeForUnknownRecurrence(U);
+    ConservativeResult = ConservativeResult.intersectWith(CR);
+
     // See if ValueTracking can give us a useful range.
     const DataLayout &DL = getDataLayout();
     KnownBits Known = computeKnownBits(U->getValue(), DL, 0, &AC, nullptr, &DT);
@@ -9534,6 +9612,16 @@ bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred,
   return isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
 }
 
+Optional<bool> ScalarEvolution::evaluatePredicate(ICmpInst::Predicate Pred,
+                                                  const SCEV *LHS,
+                                                  const SCEV *RHS) {
+  if (isKnownPredicate(Pred, LHS, RHS))
+    return true;
+  else if (isKnownPredicate(ICmpInst::getInversePredicate(Pred), LHS, RHS))
+    return false;
+  return None;
+}
+
 bool ScalarEvolution::isKnownPredicateAt(ICmpInst::Predicate Pred,
                                          const SCEV *LHS, const SCEV *RHS,
                                          const Instruction *Context) {
@@ -9542,6 +9630,23 @@ bool ScalarEvolution::isKnownPredicateAt(ICmpInst::Predicate Pred,
          isBasicBlockEntryGuardedByCond(Context->getParent(), Pred, LHS, RHS);
 }
 
+Optional<bool>
+ScalarEvolution::evaluatePredicateAt(ICmpInst::Predicate Pred, const SCEV *LHS,
+                                     const SCEV *RHS,
+                                     const Instruction *Context) {
+  Optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
+  if (KnownWithoutContext)
+    return KnownWithoutContext;
+
+  if (isBasicBlockEntryGuardedByCond(Context->getParent(), Pred, LHS, RHS))
+    return true;
+  else if (isBasicBlockEntryGuardedByCond(Context->getParent(),
+                                          ICmpInst::getInversePredicate(Pred),
+                                          LHS, RHS))
+    return false;
+  return None;
+}
+
 bool ScalarEvolution::isKnownOnEveryIteration(ICmpInst::Predicate Pred,
                                               const SCEVAddRecExpr *LHS,
                                               const SCEV *RHS) {
@@ -9991,9 +10096,6 @@ bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
     assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
            "This cannot be done on broken IR!");
 
-  if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
-    return true;
-
   // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
   // the facts (a >= b && a != b) separately. A typical situation is when the
   // non-strict comparison is known from ranges and non-equality is known from
@@ -10004,13 +10106,23 @@ bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
   bool ProvedNonStrictComparison = false;
   bool ProvedNonEquality = false;
 
-  if (ProvingStrictComparison) {
-    ProvedNonStrictComparison =
-        isKnownViaNonRecursiveReasoning(NonStrictPredicate, LHS, RHS);
-    ProvedNonEquality =
-        isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_NE, LHS, RHS);
+  auto SplitAndProve =
+    [&](std::function<bool(ICmpInst::Predicate)> Fn) -> bool {
+    if (!ProvedNonStrictComparison)
+      ProvedNonStrictComparison = Fn(NonStrictPredicate);
+    if (!ProvedNonEquality)
+      ProvedNonEquality = Fn(ICmpInst::ICMP_NE);
     if (ProvedNonStrictComparison && ProvedNonEquality)
       return true;
+    return false;
+  };
+
+  if (ProvingStrictComparison) {
+    auto ProofFn = [&](ICmpInst::Predicate P) {
+      return isKnownViaNonRecursiveReasoning(P, LHS, RHS);
+    };
+    if (SplitAndProve(ProofFn))
+      return true;
   }
 
   // Try to prove (Pred, LHS, RHS) using isImpliedViaGuard.
@@ -10018,13 +10130,10 @@ bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
     if (isImpliedViaGuard(Block, Pred, LHS, RHS))
       return true;
     if (ProvingStrictComparison) {
-      if (!ProvedNonStrictComparison)
-        ProvedNonStrictComparison =
-            isImpliedViaGuard(Block, NonStrictPredicate, LHS, RHS);
-      if (!ProvedNonEquality)
-        ProvedNonEquality =
-            isImpliedViaGuard(Block, ICmpInst::ICMP_NE, LHS, RHS);
-      if (ProvedNonStrictComparison && ProvedNonEquality)
+      auto ProofFn = [&](ICmpInst::Predicate P) {
+        return isImpliedViaGuard(Block, P, LHS, RHS);
+      };
+      if (SplitAndProve(ProofFn))
         return true;
     }
     return false;
@@ -10036,13 +10145,10 @@ bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
     if (isImpliedCond(Pred, LHS, RHS, Condition, Inverse, Context))
       return true;
     if (ProvingStrictComparison) {
-      if (!ProvedNonStrictComparison)
-        ProvedNonStrictComparison = isImpliedCond(NonStrictPredicate, LHS, RHS,
-                                                  Condition, Inverse, Context);
-      if (!ProvedNonEquality)
-        ProvedNonEquality = isImpliedCond(ICmpInst::ICMP_NE, LHS, RHS,
-                                          Condition, Inverse, Context);
-      if (ProvedNonStrictComparison && ProvedNonEquality)
+      auto ProofFn = [&](ICmpInst::Predicate P) {
+        return isImpliedCond(P, LHS, RHS, Condition, Inverse, Context);
+      };
+      if (SplitAndProve(ProofFn))
         return true;
     }
     return false;
@@ -10102,6 +10208,10 @@ bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L,
          "LHS is not available at Loop Entry");
   assert(isAvailableAtLoopEntry(RHS, L) &&
          "RHS is not available at Loop Entry");
+
+  if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
+    return true;
+
   return isBasicBlockEntryGuardedByCond(L->getHeader(), Pred, LHS, RHS);
 }
 
@@ -10109,6 +10219,11 @@ bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS,
                                     const SCEV *RHS,
                                     const Value *FoundCondValue, bool Inverse,
                                     const Instruction *Context) {
+  // False conditions implies anything. Do not bother analyzing it further.
+  if (FoundCondValue ==
+      ConstantInt::getBool(FoundCondValue->getContext(), Inverse))
+    return true;
+
   if (!PendingLoopPredicates.insert(FoundCondValue).second)
     return false;
 
@@ -10116,20 +10231,15 @@ bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS,
       make_scope_exit([&]() { PendingLoopPredicates.erase(FoundCondValue); });
 
   // Recursively handle And and Or conditions.
-  if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) {
-    if (BO->getOpcode() == Instruction::And) {
-      if (!Inverse)
-        return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse,
-                             Context) ||
-               isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse,
-                             Context);
-    } else if (BO->getOpcode() == Instruction::Or) {
-      if (Inverse)
-        return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse,
-                             Context) ||
-               isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse,
-                             Context);
-    }
+  const Value *Op0, *Op1;
+  if (match(FoundCondValue, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
+    if (!Inverse)
+      return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, Context) ||
+              isImpliedCond(Pred, LHS, RHS, Op1, Inverse, Context);
+  } else if (match(FoundCondValue, m_LogicalOr(m_Value(Op0), m_Value(Op1)))) {
+    if (Inverse)
+      return isImpliedCond(Pred, LHS, RHS, Op0, Inverse, Context) ||
+              isImpliedCond(Pred, LHS, RHS, Op1, Inverse, Context);
   }
 
   const ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue);
@@ -10231,11 +10341,26 @@ bool ScalarEvolution::isImpliedCondBalancedTypes(
   // Check whether swapping the found predicate makes it the same as the
   // desired predicate.
   if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) {
-    if (isa<SCEVConstant>(RHS))
+    // We can write the implication
+    // 0.  LHS Pred      RHS  <-   FoundLHS SwapPred  FoundRHS
+    // using one of the following ways:
+    // 1.  LHS Pred      RHS  <-   FoundRHS Pred      FoundLHS
+    // 2.  RHS SwapPred  LHS  <-   FoundLHS SwapPred  FoundRHS
+    // 3.  LHS Pred      RHS  <-  ~FoundLHS Pred     ~FoundRHS
+    // 4. ~LHS SwapPred ~RHS  <-   FoundLHS SwapPred  FoundRHS
+    // Forms 1. and 2. require swapping the operands of one condition. Don't
+    // do this if it would break canonical constant/addrec ordering.
+    if (!isa<SCEVConstant>(RHS) && !isa<SCEVAddRecExpr>(LHS))
+      return isImpliedCondOperands(FoundPred, RHS, LHS, FoundLHS, FoundRHS,
+                                   Context);
+    if (!isa<SCEVConstant>(FoundRHS) && !isa<SCEVAddRecExpr>(FoundLHS))
       return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS, Context);
-    else
-      return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred), RHS,
-                                   LHS, FoundLHS, FoundRHS, Context);
+
+    // There's no clear preference between forms 3. and 4., try both.
+    return isImpliedCondOperands(FoundPred, getNotSCEV(LHS), getNotSCEV(RHS),
+                                 FoundLHS, FoundRHS, Context) ||
+           isImpliedCondOperands(Pred, LHS, RHS, getNotSCEV(FoundLHS),
+                                 getNotSCEV(FoundRHS), Context);
   }
 
   // Unsigned comparison is the same as signed comparison when both the operands
@@ -10658,11 +10783,7 @@ bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred,
     return true;
 
   return isImpliedCondOperandsHelper(Pred, LHS, RHS,
-                                     FoundLHS, FoundRHS) ||
-         // ~x < ~y --> x > y
-         isImpliedCondOperandsHelper(Pred, LHS, RHS,
-                                     getNotSCEV(FoundRHS),
-                                     getNotSCEV(FoundLHS));
+                                     FoundLHS, FoundRHS);
 }
 
 /// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 47b4505da8331d44c8daef69406b2627fc63e3ac..1cf891c38f91e831cf9127d9b7b0e9bccc7128a8 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -229,6 +229,10 @@ int TargetTransformInfo::getUserCost(const User *U,
   return Cost;
 }
 
+BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const {
+  return TTIImpl->getPredictableBranchThreshold();
+}
+
 bool TargetTransformInfo::hasBranchDivergence() const {
   return TTIImpl->hasBranchDivergence();
 }
@@ -574,8 +578,9 @@ const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const {
   return TTIImpl->getRegisterClassName(ClassID);
 }
 
-unsigned TargetTransformInfo::getRegisterBitWidth(bool Vector) const {
-  return TTIImpl->getRegisterBitWidth(Vector);
+TypeSize TargetTransformInfo::getRegisterBitWidth(
+    TargetTransformInfo::RegisterKind K) const {
+  return TTIImpl->getRegisterBitWidth(K);
 }
 
 unsigned TargetTransformInfo::getMaxElementWidth() const {
@@ -586,23 +591,20 @@ unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
   return TTIImpl->getMinVectorRegisterBitWidth();
 }
 
-unsigned
-TargetTransformInfo::getVectorRegisterBitWidth(unsigned WidthFactor) const {
-  return TTIImpl->getVectorRegisterBitWidth(WidthFactor);
-}
-
 unsigned TargetTransformInfo::getVectorRegisterUsage(
-    unsigned VFKnownMin, unsigned ElementTypeSize, unsigned SafeDepDist) const {
-  return TTIImpl->getVectorRegisterUsage(VFKnownMin, ElementTypeSize,
+    RegisterKind K, unsigned VFKnownMin,
+    unsigned ElementTypeSize, unsigned SafeDepDist) const {
+  return TTIImpl->getVectorRegisterUsage(K, VFKnownMin, ElementTypeSize,
                                          SafeDepDist);
 }
 
 std::pair<ElementCount, ElementCount>
-TargetTransformInfo::getFeasibleMaxVFRange(unsigned SmallestType,
+TargetTransformInfo::getFeasibleMaxVFRange(RegisterKind K,
+                                           unsigned SmallestType,
                                            unsigned WidestType,
                                            unsigned MaxSafeRegisterWidth,
                                            unsigned RegWidthFactor) const {
-  return TTIImpl->getFeasibleMaxVFRange(SmallestType, WidestType,
+  return TTIImpl->getFeasibleMaxVFRange(K, SmallestType, WidestType,
                                         MaxSafeRegisterWidth, RegWidthFactor);
 }
 
@@ -731,8 +733,9 @@ int TargetTransformInfo::getArithmeticInstrCost(
 }
 
 int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, VectorType *Ty,
-                                        int Index, VectorType *SubTp) const {
-  int Cost = TTIImpl->getShuffleCost(Kind, Ty, Index, SubTp);
+                                        ArrayRef<int> Mask, int Index,
+                                        VectorType *SubTp) const {
+  int Cost = TTIImpl->getShuffleCost(Kind, Ty, Mask, Index, SubTp);
   assert(Cost >= 0 && "TTI should not produce negative costs!");
   return Cost;
 }
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 3e29eaf730a2f73223544e3a342819d773b99d7d..3bb57abc41c6cd08fbd81b339b3711d784ad7681 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1977,6 +1977,12 @@ bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
     return isKnownToBeAPowerOfTwo(SI->getTrueValue(), OrZero, Depth, Q) &&
            isKnownToBeAPowerOfTwo(SI->getFalseValue(), OrZero, Depth, Q);
 
+  // Peek through min/max.
+  if (match(V, m_MaxOrMin(m_Value(X), m_Value(Y)))) {
+    return isKnownToBeAPowerOfTwo(X, OrZero, Depth, Q) &&
+           isKnownToBeAPowerOfTwo(Y, OrZero, Depth, Q);
+  }
+
   if (OrZero && match(V, m_And(m_Value(X), m_Value(Y)))) {
     // A power of two and'd with anything is a power of two or zero.
     if (isKnownToBeAPowerOfTwo(X, /*OrZero*/ true, Depth, Q) ||
@@ -2207,6 +2213,35 @@ static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value)
   return true;
 }
 
+static bool isNonZeroRecurrence(const PHINode *PN) {
+  // Try and detect a recurrence that monotonically increases from a
+  // starting value, as these are common as induction variables.
+  BinaryOperator *BO = nullptr;
+  Value *Start = nullptr, *Step = nullptr;
+  const APInt *StartC, *StepC;
+  if (!matchSimpleRecurrence(PN, BO, Start, Step) ||
+      !match(Start, m_APInt(StartC)))
+    return false;
+
+  switch (BO->getOpcode()) {
+  case Instruction::Add:
+    return match(Step, m_APInt(StepC)) &&
+           ((BO->hasNoUnsignedWrap() && !StartC->isNullValue() &&
+             !StepC->isNullValue()) ||
+            (BO->hasNoSignedWrap() && StartC->isStrictlyPositive() &&
+             StepC->isNonNegative()));
+  case Instruction::Mul:
+    return !StartC->isNullValue() && match(Step, m_APInt(StepC)) &&
+           ((BO->hasNoUnsignedWrap() && !StepC->isNullValue()) ||
+            (BO->hasNoSignedWrap() && StepC->isStrictlyPositive()));
+  case Instruction::Shl:
+    return !StartC->isNullValue() &&
+           (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap());
+  default:
+    return false;
+  }
+}
+
 /// Return true if the given value is known to be non-zero when defined. For
 /// vectors, return true if every demanded element is known to be non-zero when
 /// defined. For pointers, if the context instruction and dominator tree are
@@ -2448,24 +2483,9 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
   }
   // PHI
   else if (const PHINode *PN = dyn_cast<PHINode>(V)) {
-    // Try and detect a recurrence that monotonically increases from a
-    // starting value, as these are common as induction variables.
-    if (PN->getNumIncomingValues() == 2) {
-      Value *Start = PN->getIncomingValue(0);
-      Value *Induction = PN->getIncomingValue(1);
-      if (isa<ConstantInt>(Induction) && !isa<ConstantInt>(Start))
-        std::swap(Start, Induction);
-      if (ConstantInt *C = dyn_cast<ConstantInt>(Start)) {
-        if (!C->isZero() && !C->isNegative()) {
-          ConstantInt *X;
-          if (Q.IIQ.UseInstrInfo &&
-              (match(Induction, m_NSWAdd(m_Specific(PN), m_ConstantInt(X))) ||
-               match(Induction, m_NUWAdd(m_Specific(PN), m_ConstantInt(X)))) &&
-              !X->isNegative())
-            return true;
-        }
-      }
-    }
+    if (Q.IIQ.UseInstrInfo && isNonZeroRecurrence(PN))
+      return true;
+
     // Check if all incoming values are non-zero using recursion.
     Query RecQ = Q;
     unsigned NewDepth = std::max(Depth, MaxAnalysisRecursionDepth - 1);
@@ -2535,6 +2555,62 @@ static bool isAddOfNonZero(const Value *V1, const Value *V2, unsigned Depth,
   return isKnownNonZero(Op, Depth + 1, Q);
 }
 
+/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
+/// the multiplication is nuw or nsw.
+static bool isNonEqualMul(const Value *V1, const Value *V2, unsigned Depth,
+                          const Query &Q) {
+  if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(V2)) {
+    const APInt *C;
+    return match(OBO, m_Mul(m_Specific(V1), m_APInt(C))) &&
+           (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
+           !C->isNullValue() && !C->isOneValue() &&
+           isKnownNonZero(V1, Depth + 1, Q);
+  }
+  return false;
+}
+
+/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
+/// the shift is nuw or nsw.
+static bool isNonEqualShl(const Value *V1, const Value *V2, unsigned Depth,
+                          const Query &Q) {
+  if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(V2)) {
+    const APInt *C;
+    return match(OBO, m_Shl(m_Specific(V1), m_APInt(C))) &&
+           (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
+           !C->isNullValue() && isKnownNonZero(V1, Depth + 1, Q);
+  }
+  return false;
+}
+
+static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
+                           unsigned Depth, const Query &Q) {
+  // Check two PHIs are in same block.
+  if (PN1->getParent() != PN2->getParent())
+    return false;
+
+  SmallPtrSet<const BasicBlock *, 8> VisitedBBs;
+  bool UsedFullRecursion = false;
+  for (const BasicBlock *IncomBB : PN1->blocks()) {
+    if (!VisitedBBs.insert(IncomBB).second)
+      continue; // Don't reprocess blocks that we have dealt with already.
+    const Value *IV1 = PN1->getIncomingValueForBlock(IncomBB);
+    const Value *IV2 = PN2->getIncomingValueForBlock(IncomBB);
+    const APInt *C1, *C2;
+    if (match(IV1, m_APInt(C1)) && match(IV2, m_APInt(C2)) && *C1 != *C2)
+      continue;
+
+    // Only one pair of phi operands is allowed for full recursion.
+    if (UsedFullRecursion)
+      return false;
+
+    Query RecQ = Q;
+    RecQ.CxtI = IncomBB->getTerminator();
+    if (!isKnownNonEqual(IV1, IV2, Depth + 1, RecQ))
+      return false;
+    UsedFullRecursion = true;
+  }
+  return true;
+}
 
 /// Return true if it is known that V1 != V2.
 static bool isKnownNonEqual(const Value *V1, const Value *V2, unsigned Depth,
@@ -2584,18 +2660,46 @@ static bool isKnownNonEqual(const Value *V1, const Value *V2, unsigned Depth,
                                Depth + 1, Q);
       break;
     }
+    case Instruction::Shl: {
+      // Same as multiplies, with the difference that we don't need to check
+      // for a non-zero multiply. Shifts always multiply by non-zero.
+      auto *OBO1 = cast<OverflowingBinaryOperator>(O1);
+      auto *OBO2 = cast<OverflowingBinaryOperator>(O2);
+      if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
+          (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
+        break;
+
+      if (O1->getOperand(1) == O2->getOperand(1))
+        return isKnownNonEqual(O1->getOperand(0), O2->getOperand(0),
+                               Depth + 1, Q);
+      break;
+    }
     case Instruction::SExt:
     case Instruction::ZExt:
       if (O1->getOperand(0)->getType() == O2->getOperand(0)->getType())
-        return isKnownNonEqual(O1->getOperand(0), O2->getOperand(0),
-                               Depth + 1, Q);
+        return isKnownNonEqual(O1->getOperand(0), O2->getOperand(0), Depth + 1,
+                               Q);
+      break;
+    case Instruction::PHI:
+      const PHINode *PN1 = cast<PHINode>(V1);
+      const PHINode *PN2 = cast<PHINode>(V2);
+      // FIXME: This is missing a generalization to handle the case where one is
+      // a PHI and another one isn't.
+      if (isNonEqualPHIs(PN1, PN2, Depth, Q))
+        return true;
       break;
     };
   }
-  
+
   if (isAddOfNonZero(V1, V2, Depth, Q) || isAddOfNonZero(V2, V1, Depth, Q))
     return true;
 
+  if (isNonEqualMul(V1, V2, Depth, Q) || isNonEqualMul(V2, V1, Depth, Q))
+    return true;
+
+  if (isNonEqualShl(V1, V2, Depth, Q) || isNonEqualShl(V2, V1, Depth, Q))
+    return true;
+
   if (V1->getType()->isIntOrIntVectorTy()) {
     // Are any known bits in V1 contradictory to known bits in V2? If V1
     // has a known zero where V2 has a known one, they must not be equal.
@@ -4402,7 +4506,8 @@ bool llvm::mustSuppressSpeculation(const LoadInst &LI) {
 
 bool llvm::isSafeToSpeculativelyExecute(const Value *V,
                                         const Instruction *CtxI,
-                                        const DominatorTree *DT) {
+                                        const DominatorTree *DT,
+                                        const TargetLibraryInfo *TLI) {
   const Operator *Inst = dyn_cast<Operator>(V);
   if (!Inst)
     return false;
@@ -4449,7 +4554,7 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V,
     const DataLayout &DL = LI->getModule()->getDataLayout();
     return isDereferenceableAndAlignedPointer(
         LI->getPointerOperand(), LI->getType(), MaybeAlign(LI->getAlignment()),
-        DL, CtxI, DT);
+        DL, CtxI, DT, TLI);
   }
   case Instruction::Call: {
     auto *CI = cast<const CallInst>(Inst);
@@ -4789,6 +4894,14 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly) {
     // destination type.
     return true;
   case Instruction::Call:
+    if (auto *II = dyn_cast<IntrinsicInst>(Op)) {
+      switch (II->getIntrinsicID()) {
+      // TODO: Add more intrinsics.
+      case Intrinsic::ctpop:
+        return false;
+      }
+    }
+    LLVM_FALLTHROUGH;
   case Instruction::CallBr:
   case Instruction::Invoke: {
     const auto *CB = cast<CallBase>(Op);
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index f1e2c0b39664ca614e6357ebf62ae4205e798d68..e9fb0820a839cc6714a1cd073713e25b3d8428ef 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -697,6 +697,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(swifterror);
   KEYWORD(swiftself);
   KEYWORD(uwtable);
+  KEYWORD(vscale_range);
   KEYWORD(willreturn);
   KEYWORD(writeonly);
   KEYWORD(zeroext);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 816dda6b8080664410569dc8723aabbe9e021b41..ee84424b31f63b08c3060f8378f9577e1f22fb5b 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -1341,6 +1341,14 @@ bool LLParser::parseFnAttributeValuePairs(AttrBuilder &B,
       B.addAllocSizeAttr(ElemSizeArg, NumElemsArg);
       continue;
     }
+    case lltok::kw_vscale_range: {
+      unsigned MinValue, MaxValue;
+      // inAttrGrp doesn't matter; we only support vscale_range(a[, b])
+      if (parseVScaleRangeArguments(MinValue, MaxValue))
+        return true;
+      B.addVScaleRangeAttr(MinValue, MaxValue);
+      continue;
+    }
     case lltok::kw_alwaysinline: B.addAttribute(Attribute::AlwaysInline); break;
     case lltok::kw_argmemonly: B.addAttribute(Attribute::ArgMemOnly); break;
     case lltok::kw_builtin: B.addAttribute(Attribute::Builtin); break;
@@ -1728,6 +1736,13 @@ bool LLParser::parseOptionalParamAttrs(AttrBuilder &B) {
       B.addPreallocatedAttr(Ty);
       continue;
     }
+    case lltok::kw_inalloca: {
+      Type *Ty;
+      if (parseInalloca(Ty))
+        return true;
+      B.addInAllocaAttr(Ty);
+      continue;
+    }
     case lltok::kw_dereferenceable: {
       uint64_t Bytes;
       if (parseOptionalDerefAttrBytes(lltok::kw_dereferenceable, Bytes))
@@ -1749,7 +1764,6 @@ bool LLParser::parseOptionalParamAttrs(AttrBuilder &B) {
       B.addByRefAttr(Ty);
       continue;
     }
-    case lltok::kw_inalloca:        B.addAttribute(Attribute::InAlloca); break;
     case lltok::kw_inreg:           B.addAttribute(Attribute::InReg); break;
     case lltok::kw_nest:            B.addAttribute(Attribute::Nest); break;
     case lltok::kw_noundef:
@@ -1806,6 +1820,7 @@ bool LLParser::parseOptionalParamAttrs(AttrBuilder &B) {
     case lltok::kw_shadowcallstack:
     case lltok::kw_strictfp:
     case lltok::kw_uwtable:
+    case lltok::kw_vscale_range:
       HaveError |=
           error(Lex.getLoc(), "invalid use of function-only attribute");
       break;
@@ -1915,6 +1930,7 @@ bool LLParser::parseOptionalReturnAttrs(AttrBuilder &B) {
     case lltok::kw_shadowcallstack:
     case lltok::kw_strictfp:
     case lltok::kw_uwtable:
+    case lltok::kw_vscale_range:
       HaveError |=
           error(Lex.getLoc(), "invalid use of function-only attribute");
       break;
@@ -2355,6 +2371,29 @@ bool LLParser::parseAllocSizeArguments(unsigned &BaseSizeArg,
   return false;
 }
 
+bool LLParser::parseVScaleRangeArguments(unsigned &MinValue,
+                                         unsigned &MaxValue) {
+  Lex.Lex();
+
+  auto StartParen = Lex.getLoc();
+  if (!EatIfPresent(lltok::lparen))
+    return error(StartParen, "expected '('");
+
+  if (parseUInt32(MinValue))
+    return true;
+
+  if (EatIfPresent(lltok::comma)) {
+    if (parseUInt32(MaxValue))
+      return true;
+  } else
+    MaxValue = MinValue;
+
+  auto EndParen = Lex.getLoc();
+  if (!EatIfPresent(lltok::rparen))
+    return error(EndParen, "expected ')'");
+  return false;
+}
+
 /// parseScopeAndOrdering
 ///   if isAtomic: ::= SyncScope? AtomicOrdering
 ///   else: ::=
@@ -2661,6 +2700,12 @@ bool LLParser::parsePreallocated(Type *&Result) {
   return parseRequiredTypeAttr(Result, lltok::kw_preallocated);
 }
 
+/// parseInalloca
+///   ::= inalloca(<ty>)
+bool LLParser::parseInalloca(Type *&Result) {
+  return parseRequiredTypeAttr(Result, lltok::kw_inalloca);
+}
+
 /// parseByRef
 ///   ::= byref(<type>)
 bool LLParser::parseByRef(Type *&Result) {
diff --git a/llvm/lib/AsmParser/LLParser.h b/llvm/lib/AsmParser/LLParser.h
index a891e136c59fb498a1e7f98e70a1f133ecccb6cf..3d9ffe6e90da93fb792a8ab1bb9eb8245dacb1e0 100644
--- a/llvm/lib/AsmParser/LLParser.h
+++ b/llvm/lib/AsmParser/LLParser.h
@@ -281,6 +281,7 @@ namespace llvm {
     bool parseOptionalCommaInAlloca(bool &IsInAlloca);
     bool parseAllocSizeArguments(unsigned &BaseSizeArg,
                                  Optional<unsigned> &HowManyArg);
+    bool parseVScaleRangeArguments(unsigned &MinValue, unsigned &MaxValue);
     bool parseIndexList(SmallVectorImpl<unsigned> &Indices,
                         bool &AteExtraComma);
     bool parseIndexList(SmallVectorImpl<unsigned> &Indices) {
@@ -330,6 +331,7 @@ namespace llvm {
                                     bool inAttrGrp, LocTy &BuiltinLoc);
     bool parseRequiredTypeAttr(Type *&Result, lltok::Kind AttrName);
     bool parsePreallocated(Type *&Result);
+    bool parseInalloca(Type *&Result);
     bool parseByRef(Type *&Result);
 
     // Module Summary Index Parsing.
diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h
index 0b09dc84eeb67ac4f972987ecd3d484f0792f89c..b4b10c919ec7437bafa14d7b3bb222c224451b04 100644
--- a/llvm/lib/AsmParser/LLToken.h
+++ b/llvm/lib/AsmParser/LLToken.h
@@ -240,6 +240,7 @@ enum Kind {
   kw_swifterror,
   kw_swiftself,
   kw_uwtable,
+  kw_vscale_range,
   kw_willreturn,
   kw_writeonly,
   kw_zeroext,
diff --git a/llvm/lib/BinaryFormat/Magic.cpp b/llvm/lib/BinaryFormat/Magic.cpp
index 61b1504e59b0af81728925da6081437dea44c204..591da4877479f3bf9490c6f47fa97b41d67d7097 100644
--- a/llvm/lib/BinaryFormat/Magic.cpp
+++ b/llvm/lib/BinaryFormat/Magic.cpp
@@ -223,7 +223,8 @@ file_magic llvm::identify_magic(StringRef Magic) {
 }
 
 std::error_code llvm::identify_magic(const Twine &Path, file_magic &Result) {
-  auto FileOrError = MemoryBuffer::getFile(Path, -1LL, false);
+  auto FileOrError = MemoryBuffer::getFile(Path, /*IsText=*/false,
+                                           /*RequiresNullTerminator=*/false);
   if (!FileOrError)
     return FileOrError.getError();
 
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index fd2432981acf4844df935d8ccff86765816d975d..46db3edcc3427ca671001b8a6cd3b5c7431c8a08 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1526,6 +1526,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::SwiftSelf;
   case bitc::ATTR_KIND_UW_TABLE:
     return Attribute::UWTable;
+  case bitc::ATTR_KIND_VSCALE_RANGE:
+    return Attribute::VScaleRange;
   case bitc::ATTR_KIND_WILLRETURN:
     return Attribute::WillReturn;
   case bitc::ATTR_KIND_WRITEONLY:
@@ -1622,6 +1624,8 @@ Error BitcodeReader::parseAttributeGroupBlock() {
             B.addByValAttr(nullptr);
           else if (Kind == Attribute::StructRet)
             B.addStructRetAttr(nullptr);
+          else if (Kind == Attribute::InAlloca)
+            B.addInAllocaAttr(nullptr);
 
           B.addAttribute(Kind);
         } else if (Record[i] == 1) { // Integer attribute
@@ -1638,6 +1642,8 @@ Error BitcodeReader::parseAttributeGroupBlock() {
             B.addDereferenceableOrNullAttr(Record[++i]);
           else if (Kind == Attribute::AllocSize)
             B.addAllocSizeAttrFromRawRepr(Record[++i]);
+          else if (Kind == Attribute::VScaleRange)
+            B.addVScaleRangeAttrFromRawRepr(Record[++i]);
         } else if (Record[i] == 3 || Record[i] == 4) { // String attribute
           bool HasValue = (Record[i++] == 4);
           SmallString<64> KindStr;
@@ -1671,6 +1677,8 @@ Error BitcodeReader::parseAttributeGroupBlock() {
             B.addByRefAttr(getTypeByID(Record[++i]));
           } else if (Kind == Attribute::Preallocated) {
             B.addPreallocatedAttr(getTypeByID(Record[++i]));
+          } else if (Kind == Attribute::InAlloca) {
+            B.addInAllocaAttr(HasType ? getTypeByID(Record[++i]) : nullptr);
           }
         }
       }
@@ -3324,7 +3332,8 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) {
   // argument's pointee type. There should be no opaque pointers where the byval
   // type is implicit.
   for (unsigned i = 0; i != Func->arg_size(); ++i) {
-    for (Attribute::AttrKind Kind : {Attribute::ByVal, Attribute::StructRet}) {
+    for (Attribute::AttrKind Kind : {Attribute::ByVal, Attribute::StructRet,
+                                     Attribute::InAlloca}) {
       if (!Func->hasParamAttribute(i, Kind))
         continue;
 
@@ -3332,10 +3341,21 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) {
 
       Type *PTy = cast<FunctionType>(FullFTy)->getParamType(i);
       Type *PtrEltTy = getPointerElementFlatType(PTy);
-      Attribute NewAttr =
-          Kind == Attribute::ByVal
-              ? Attribute::getWithByValType(Context, PtrEltTy)
-              : Attribute::getWithStructRetType(Context, PtrEltTy);
+      Attribute NewAttr;
+      switch (Kind) {
+      case Attribute::ByVal:
+        NewAttr = Attribute::getWithByValType(Context, PtrEltTy);
+        break;
+      case Attribute::StructRet:
+        NewAttr = Attribute::getWithStructRetType(Context, PtrEltTy);
+        break;
+      case Attribute::InAlloca:
+        NewAttr = Attribute::getWithInAllocaType(Context, PtrEltTy);
+        break;
+      default:
+        llvm_unreachable("not an upgraded type attribute");
+      }
+
       Func->addParamAttr(i, NewAttr);
     }
   }
@@ -3801,17 +3821,29 @@ Error BitcodeReader::typeCheckLoadStoreInst(Type *ValType, Type *PtrType) {
 void BitcodeReader::propagateByValSRetTypes(CallBase *CB,
                                             ArrayRef<Type *> ArgsFullTys) {
   for (unsigned i = 0; i != CB->arg_size(); ++i) {
-    for (Attribute::AttrKind Kind : {Attribute::ByVal, Attribute::StructRet}) {
+    for (Attribute::AttrKind Kind : {Attribute::ByVal, Attribute::StructRet,
+                                     Attribute::InAlloca}) {
       if (!CB->paramHasAttr(i, Kind))
         continue;
 
       CB->removeParamAttr(i, Kind);
 
       Type *PtrEltTy = getPointerElementFlatType(ArgsFullTys[i]);
-      Attribute NewAttr =
-          Kind == Attribute::ByVal
-              ? Attribute::getWithByValType(Context, PtrEltTy)
-              : Attribute::getWithStructRetType(Context, PtrEltTy);
+      Attribute NewAttr;
+      switch (Kind) {
+      case Attribute::ByVal:
+        NewAttr = Attribute::getWithByValType(Context, PtrEltTy);
+        break;
+      case Attribute::StructRet:
+        NewAttr = Attribute::getWithStructRetType(Context, PtrEltTy);
+        break;
+      case Attribute::InAlloca:
+        NewAttr = Attribute::getWithInAllocaType(Context, PtrEltTy);
+        break;
+      default:
+        llvm_unreachable("not an upgraded type attribute");
+      }
+
       CB->addParamAttr(i, NewAttr);
     }
   }
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index fd4b0a6a6a38ac1a288ebd6abf7e9ebb5b4ff8bb..e36ce87554f2dfa1fcbfb22dd0093100f2b1632c 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -738,6 +738,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_SWIFT_SELF;
   case Attribute::UWTable:
     return bitc::ATTR_KIND_UW_TABLE;
+  case Attribute::VScaleRange:
+    return bitc::ATTR_KIND_VSCALE_RANGE;
   case Attribute::WillReturn:
     return bitc::ATTR_KIND_WILLRETURN;
   case Attribute::WriteOnly:
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index fc0b278325d948d0e4ab0a3c27d8139078672145..160eaefeaf9070b35b2ea7c98a44d20618623f6d 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1078,17 +1078,19 @@ void AsmPrinter::emitFrameAlloc(const MachineInstr &MI) {
 
 /// Returns the BB metadata to be emitted in the .llvm_bb_addr_map section for a
 /// given basic block. This can be used to capture more precise profile
-/// information. We use the last 3 bits (LSBs) to ecnode the following
+/// information. We use the last 4 bits (LSBs) to encode the following
 /// information:
 ///  * (1): set if return block (ret or tail call).
 ///  * (2): set if ends with a tail call.
 ///  * (3): set if exception handling (EH) landing pad.
+///  * (4): set if the block can fall through to its next.
 /// The remaining bits are zero.
 static unsigned getBBAddrMapMetadata(const MachineBasicBlock &MBB) {
   const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo();
   return ((unsigned)MBB.isReturnBlock()) |
          ((!MBB.empty() && TII->isTailCall(MBB.back())) << 1) |
-         (MBB.isEHPad() << 2);
+         (MBB.isEHPad() << 2) |
+         (const_cast<MachineBasicBlock &>(MBB).canFallThrough() << 3);
 }
 
 void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
diff --git a/llvm/lib/CodeGen/CFIInstrInserter.cpp b/llvm/lib/CodeGen/CFIInstrInserter.cpp
index 31346c9356334a0b2e6eff73d591f6862bad6bee..c6244419d6d16bbd381857185097fb0102f6a50d 100644
--- a/llvm/lib/CodeGen/CFIInstrInserter.cpp
+++ b/llvm/lib/CodeGen/CFIInstrInserter.cpp
@@ -264,9 +264,9 @@ void CFIInstrInserter::calculateOutgoingCFAInfo(MBBCFAInfo &MBBInfo) {
   MBBInfo.OutgoingCFARegister = SetRegister;
 
   // Update outgoing CSR info.
-  MBBInfo.OutgoingCSRSaved = MBBInfo.IncomingCSRSaved;
-  MBBInfo.OutgoingCSRSaved |= CSRSaved;
-  MBBInfo.OutgoingCSRSaved.reset(CSRRestored);
+  BitVector::apply([](auto x, auto y, auto z) { return (x | y) & ~z; },
+                   MBBInfo.OutgoingCSRSaved, MBBInfo.IncomingCSRSaved, CSRSaved,
+                   CSRRestored);
 }
 
 void CFIInstrInserter::updateSuccCFAInfo(MBBCFAInfo &MBBInfo) {
@@ -294,6 +294,7 @@ bool CFIInstrInserter::insertCFIInstrs(MachineFunction &MF) {
   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
   bool InsertedCFIInstr = false;
 
+  BitVector SetDifference;
   for (MachineBasicBlock &MBB : MF) {
     // Skip the first MBB in a function
     if (MBB.getNumber() == MF.front().getNumber()) continue;
@@ -345,8 +346,8 @@ bool CFIInstrInserter::insertCFIInstrs(MachineFunction &MF) {
       continue;
     }
 
-    BitVector SetDifference = PrevMBBInfo->OutgoingCSRSaved;
-    SetDifference.reset(MBBInfo.IncomingCSRSaved);
+    BitVector::apply([](auto x, auto y) { return x & ~y; }, SetDifference,
+                     PrevMBBInfo->OutgoingCSRSaved, MBBInfo.IncomingCSRSaved);
     for (int Reg : SetDifference.set_bits()) {
       unsigned CFIIndex =
           MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, Reg));
@@ -355,8 +356,8 @@ bool CFIInstrInserter::insertCFIInstrs(MachineFunction &MF) {
       InsertedCFIInstr = true;
     }
 
-    SetDifference = MBBInfo.IncomingCSRSaved;
-    SetDifference.reset(PrevMBBInfo->OutgoingCSRSaved);
+    BitVector::apply([](auto x, auto y) { return x & ~y; }, SetDifference,
+                     MBBInfo.IncomingCSRSaved, PrevMBBInfo->OutgoingCSRSaved);
     for (int Reg : SetDifference.set_bits()) {
       auto it = CSRLocMap.find(Reg);
       assert(it != CSRLocMap.end() && "Reg should have an entry in CSRLocMap");
diff --git a/llvm/lib/CodeGen/CalcSpillWeights.cpp b/llvm/lib/CodeGen/CalcSpillWeights.cpp
index 16f380c1eb62cc58cf1b63a1532c411c69264552..863a0e1e0b56f58448b59cfd314e23c9526a3f48 100644
--- a/llvm/lib/CodeGen/CalcSpillWeights.cpp
+++ b/llvm/lib/CodeGen/CalcSpillWeights.cpp
@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/VirtRegMap.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/CodeGen/StackMaps.h"
 #include <cassert>
 #include <tuple>
 
@@ -125,6 +126,16 @@ static bool isRematerializable(const LiveInterval &LI, const LiveIntervals &LIS,
   return true;
 }
 
+bool VirtRegAuxInfo::isLiveAtStatepointVarArg(LiveInterval &LI) {
+  return any_of(VRM.getRegInfo().reg_operands(LI.reg()),
+                [](MachineOperand &MO) {
+    MachineInstr *MI = MO.getParent();
+    if (MI->getOpcode() != TargetOpcode::STATEPOINT)
+      return false;
+    return StatepointOpers(MI).getVarIdx() <= MI->getOperandNo(&MO);
+  });
+}
+
 void VirtRegAuxInfo::calculateSpillWeightAndHint(LiveInterval &LI) {
   float Weight = weightCalcHelper(LI);
   // Check if unspillable.
@@ -290,9 +301,15 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI, SlotIndex *Start,
 
   // Mark li as unspillable if all live ranges are tiny and the interval
   // is not live at any reg mask.  If the interval is live at a reg mask
-  // spilling may be required.
+  // spilling may be required. If li is live as use in statepoint instruction
+  // spilling may be required due to if we mark interval with use in statepoint
+  // as not spillable we are risky to end up with no register to allocate.
+  // At the same time STATEPOINT instruction is perfectly fine to have this
+  // operand on stack, so spilling such interval and folding its load from stack
+  // into instruction itself makes perfect sense.
   if (ShouldUpdateLI && LI.isZeroLength(LIS.getSlotIndexes()) &&
-      !LI.isLiveAtIndexes(LIS.getRegMaskSlots())) {
+      !LI.isLiveAtIndexes(LIS.getRegMaskSlots()) &&
+      !isLiveAtStatepointVarArg(LI)) {
     LI.markNotSpillable();
     return -1.0;
   }
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 8e24c86dfcf298c0ad16582289e2fff762b0f089..7e05b307e170fac1b3cb2b4acda50f2ef4539b66 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -6587,7 +6587,7 @@ static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
     uint64_t Sum = TrueWeight + FalseWeight;
     if (Sum != 0) {
       auto Probability = BranchProbability::getBranchProbability(Max, Sum);
-      if (Probability > TLI->getPredictableBranchThreshold())
+      if (Probability > TTI->getPredictableBranchThreshold())
         return true;
     }
   }
diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
index 2c86f06a602deecec4b1add67b713a8824d20b27..82942b9476dd13e2bf0b623633ac5dfc2cfc2077 100644
--- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
@@ -189,7 +189,7 @@ MachineInstrBuilder CSEMIRBuilder::buildInstr(unsigned Opc,
     assert(DstOps.size() == 1 && "Invalid dsts");
     if (Optional<APInt> Cst = ConstantFoldBinOp(Opc, SrcOps[0].getReg(),
                                                 SrcOps[1].getReg(), *getMRI()))
-      return buildConstant(DstOps[0], Cst->getSExtValue());
+      return buildConstant(DstOps[0], *Cst);
     break;
   }
   case TargetOpcode::G_SEXT_INREG: {
@@ -200,7 +200,7 @@ MachineInstrBuilder CSEMIRBuilder::buildInstr(unsigned Opc,
     const SrcOp &Src1 = SrcOps[1];
     if (auto MaybeCst =
             ConstantFoldExtOp(Opc, Src0.getReg(), Src1.getImm(), *getMRI()))
-      return buildConstant(Dst, MaybeCst->getSExtValue());
+      return buildConstant(Dst, *MaybeCst);
     break;
   }
   }
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index f689801fa30fc8eade5970c3fc414e8aa9d56244..808be0ff6381f68e715c1fe697542f33a3cd735b 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -112,7 +112,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
   unsigned i = 0;
   unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
   for (auto &Arg : CB.args()) {
-    ArgInfo OrigArg{ArgRegs[i], Arg->getType(), getAttributesForArgIdx(CB, i),
+    ArgInfo OrigArg{ArgRegs[i], *Arg.get(), getAttributesForArgIdx(CB, i),
                     i < NumFixedArgs};
     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);
 
@@ -204,7 +204,8 @@ void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
     // No splitting to do, but we want to replace the original type (e.g. [1 x
     // double] -> double).
     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
-                           OrigArg.Flags[0], OrigArg.IsFixed);
+                           OrigArg.Flags[0], OrigArg.IsFixed,
+                           OrigArg.OrigValue);
     return;
   }
 
@@ -647,17 +648,44 @@ bool CallLowering::handleAssignments(CCState &CCInfo,
       }
 
       if (VA.isMemLoc() && Flags.isByVal()) {
-        // FIXME: We should be inserting a memcpy from the source pointer to the
-        // result for outgoing byval parameters.
-        if (!Handler.isIncomingArgumentHandler())
-          continue;
-
-        MachinePointerInfo MPO;
-        Register StackAddr = Handler.getStackAddress(
-            Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags);
         assert(Args[i].Regs.size() == 1 &&
                "didn't expect split byval pointer");
-        MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr);
+
+        if (Handler.isIncomingArgumentHandler()) {
+          // We just need to copy the frame index value to the pointer.
+          MachinePointerInfo MPO;
+          Register StackAddr = Handler.getStackAddress(
+              Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags);
+          MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr);
+        } else {
+          // For outgoing byval arguments, insert the implicit copy byval
+          // implies, such that writes in the callee do not modify the caller's
+          // value.
+          uint64_t MemSize = Flags.getByValSize();
+          int64_t Offset = VA.getLocMemOffset();
+
+          MachinePointerInfo DstMPO;
+          Register StackAddr =
+              Handler.getStackAddress(MemSize, Offset, DstMPO, Flags);
+
+          MachinePointerInfo SrcMPO(Args[i].OrigValue);
+          if (!Args[i].OrigValue) {
+            // We still need to accurately track the stack address space if we
+            // don't know the underlying value.
+            const LLT PtrTy = MRI.getType(StackAddr);
+            SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
+          }
+
+          Align DstAlign = std::max(Flags.getNonZeroByValAlign(),
+                                    inferAlignFromPtrInfo(MF, DstMPO));
+
+          Align SrcAlign = std::max(Flags.getNonZeroByValAlign(),
+                                    inferAlignFromPtrInfo(MF, SrcMPO));
+
+          Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0],
+                                     DstMPO, DstAlign, SrcMPO, SrcAlign,
+                                     MemSize, VA);
+        }
         continue;
       }
 
@@ -963,6 +991,29 @@ bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
   return true;
 }
 
+void CallLowering::ValueHandler::copyArgumentMemory(
+    const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
+    const MachinePointerInfo &DstPtrInfo, Align DstAlign,
+    const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
+    CCValAssign &VA) const {
+  MachineFunction &MF = MIRBuilder.getMF();
+  MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
+      SrcPtrInfo,
+      MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize,
+      SrcAlign);
+
+  MachineMemOperand *DstMMO = MF.getMachineMemOperand(
+      DstPtrInfo,
+      MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
+      MemSize, DstAlign);
+
+  const LLT PtrTy = MRI.getType(DstPtr);
+  const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits());
+
+  auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize);
+  MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO);
+}
+
 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
                                                     CCValAssign &VA,
                                                     unsigned MaxSizeBits) {
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 86247a41bded3b62ec481132f65cc284ef690c92..035b757e3c16e09850e6a6f4369cf436a37e5673 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3110,7 +3110,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   // Make our arguments/constants entry block fallthrough to the IR entry block.
   EntryBB->addSuccessor(&getMBB(F.front()));
 
-  if (CLI->fallBackToDAGISel(F)) {
+  if (CLI->fallBackToDAGISel(*MF)) {
     OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to lower function: " << ore::NV("Prototype", F.getType());
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index f74ef8e7877aec39a13cebe27792a0f89c2df4be..21114daee24460ccb39d2ed251b6aa949223bcac 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -582,7 +582,11 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
   auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
   auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
   RTLIB::Libcall RTLibcall;
-  switch (MI.getOpcode()) {
+  unsigned Opc = MI.getOpcode();
+  switch (Opc) {
+  case TargetOpcode::G_BZERO:
+    RTLibcall = RTLIB::BZERO;
+    break;
   case TargetOpcode::G_MEMCPY:
     RTLibcall = RTLIB::MEMCPY;
     break;
@@ -597,6 +601,13 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
   }
   const char *Name = TLI.getLibcallName(RTLibcall);
 
+  // Unsupported libcall on the target.
+  if (!Name) {
+    LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
+                      << MIRBuilder.getTII().getName(Opc) << "\n");
+    return LegalizerHelper::UnableToLegalize;
+  }
+
   CallLowering::CallLoweringInfo Info;
   Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
   Info.Callee = MachineOperand::CreateES(Name);
@@ -748,10 +759,14 @@ LegalizerHelper::libcall(MachineInstr &MI) {
       return Status;
     break;
   }
+  case TargetOpcode::G_BZERO:
   case TargetOpcode::G_MEMCPY:
   case TargetOpcode::G_MEMMOVE:
   case TargetOpcode::G_MEMSET: {
-    LegalizeResult Result = createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI);
+    LegalizeResult Result =
+        createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI);
+    if (Result != Legalized)
+      return Result;
     MI.eraseFromParent();
     return Result;
   }
@@ -1823,6 +1838,62 @@ LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
   return Legalized;
 }
 
+LegalizerHelper::LegalizeResult
+LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
+                                 LLT WideTy) {
+  if (TypeIdx == 1)
+    return UnableToLegalize;
+
+  bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO;
+  Register Result = MI.getOperand(0).getReg();
+  Register OriginalOverflow = MI.getOperand(1).getReg();
+  Register LHS = MI.getOperand(2).getReg();
+  Register RHS = MI.getOperand(3).getReg();
+  LLT SrcTy = MRI.getType(LHS);
+  LLT OverflowTy = MRI.getType(OriginalOverflow);
+  unsigned SrcBitWidth = SrcTy.getScalarSizeInBits();
+
+  // To determine if the result overflowed in the larger type, we extend the
+  // input to the larger type, do the multiply (checking if it overflows),
+  // then also check the high bits of the result to see if overflow happened
+  // there.
+  unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
+  auto LeftOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {LHS});
+  auto RightOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {RHS});
+
+  auto Mulo = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy, OverflowTy},
+                                    {LeftOperand, RightOperand});
+  auto Mul = Mulo->getOperand(0);
+  MIRBuilder.buildTrunc(Result, Mul);
+
+  MachineInstrBuilder ExtResult;
+  // Overflow occurred if it occurred in the larger type, or if the high part
+  // of the result does not zero/sign-extend the low part.  Check this second
+  // possibility first.
+  if (IsSigned) {
+    // For signed, overflow occurred when the high part does not sign-extend
+    // the low part.
+    ExtResult = MIRBuilder.buildSExtInReg(WideTy, Mul, SrcBitWidth);
+  } else {
+    // Unsigned overflow occurred when the high part does not zero-extend the
+    // low part.
+    ExtResult = MIRBuilder.buildZExtInReg(WideTy, Mul, SrcBitWidth);
+  }
+
+  // Multiplication cannot overflow if the WideTy is >= 2 * original width,
+  // so we don't need to check the overflow result of larger type Mulo.
+  if (WideTy.getScalarSizeInBits() < 2 * SrcBitWidth) {
+    auto Overflow =
+        MIRBuilder.buildICmp(CmpInst::ICMP_NE, OverflowTy, Mul, ExtResult);
+    // Finally check if the multiplication in the larger type itself overflowed.
+    MIRBuilder.buildOr(OriginalOverflow, Mulo->getOperand(1), Overflow);
+  } else {
+    MIRBuilder.buildICmp(CmpInst::ICMP_NE, OriginalOverflow, Mul, ExtResult);
+  }
+  MI.eraseFromParent();
+  return Legalized;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   switch (MI.getOpcode()) {
@@ -1845,6 +1916,9 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_UADDE:
   case TargetOpcode::G_USUBE:
     return widenScalarAddSubOverflow(MI, TypeIdx, WideTy);
+  case TargetOpcode::G_UMULO:
+  case TargetOpcode::G_SMULO:
+    return widenScalarMulo(MI, TypeIdx, WideTy);
   case TargetOpcode::G_SADDSAT:
   case TargetOpcode::G_SSUBSAT:
   case TargetOpcode::G_SSHLSAT:
@@ -3151,6 +3225,9 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
   case G_SDIVREM:
   case G_UDIVREM:
     return lowerDIVREM(MI);
+  case G_FSHL:
+  case G_FSHR:
+    return lowerFunnelShift(MI);
   }
 }
 
@@ -3619,6 +3696,55 @@ LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
   return Legalized;
 }
 
+LegalizerHelper::LegalizeResult
+LegalizerHelper::fewerElementsVectorMulo(MachineInstr &MI, unsigned TypeIdx,
+                                         LLT NarrowTy) {
+  Register Result = MI.getOperand(0).getReg();
+  Register Overflow = MI.getOperand(1).getReg();
+  Register LHS = MI.getOperand(2).getReg();
+  Register RHS = MI.getOperand(3).getReg();
+
+  LLT SrcTy = MRI.getType(LHS);
+  if (!SrcTy.isVector())
+    return UnableToLegalize;
+
+  LLT ElementType = SrcTy.getElementType();
+  LLT OverflowElementTy = MRI.getType(Overflow).getElementType();
+  const int NumResult = SrcTy.getNumElements();
+  LLT GCDTy = getGCDType(SrcTy, NarrowTy);
+
+  // Unmerge the operands to smaller parts of GCD type.
+  auto UnmergeLHS = MIRBuilder.buildUnmerge(GCDTy, LHS);
+  auto UnmergeRHS = MIRBuilder.buildUnmerge(GCDTy, RHS);
+
+  const int NumOps = UnmergeLHS->getNumOperands() - 1;
+  const int PartsPerUnmerge = NumResult / NumOps;
+  LLT OverflowTy = LLT::scalarOrVector(PartsPerUnmerge, OverflowElementTy);
+  LLT ResultTy = LLT::scalarOrVector(PartsPerUnmerge, ElementType);
+
+  // Perform the operation over unmerged parts.
+  SmallVector<Register, 8> ResultParts;
+  SmallVector<Register, 8> OverflowParts;
+  for (int I = 0; I != NumOps; ++I) {
+    Register Operand1 = UnmergeLHS->getOperand(I).getReg();
+    Register Operand2 = UnmergeRHS->getOperand(I).getReg();
+    auto PartMul = MIRBuilder.buildInstr(MI.getOpcode(), {ResultTy, OverflowTy},
+                                         {Operand1, Operand2});
+    ResultParts.push_back(PartMul->getOperand(0).getReg());
+    OverflowParts.push_back(PartMul->getOperand(1).getReg());
+  }
+
+  LLT ResultLCMTy = buildLCMMergePieces(SrcTy, NarrowTy, GCDTy, ResultParts);
+  LLT OverflowLCMTy =
+      LLT::scalarOrVector(ResultLCMTy.getNumElements(), OverflowElementTy);
+
+  // Recombine the pieces to the original result and overflow registers.
+  buildWidenedRemergeToDst(Result, ResultLCMTy, ResultParts);
+  buildWidenedRemergeToDst(Overflow, OverflowLCMTy, OverflowParts);
+  MI.eraseFromParent();
+  return Legalized;
+}
+
 // Handle FewerElementsVector a G_BUILD_VECTOR or G_CONCAT_VECTORS that produces
 // a vector
 //
@@ -4026,6 +4152,9 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_UADDSAT:
   case G_USUBSAT:
     return reduceOperationWidth(MI, TypeIdx, NarrowTy);
+  case G_UMULO:
+  case G_SMULO:
+    return fewerElementsVectorMulo(MI, TypeIdx, NarrowTy);
   case G_SHL:
   case G_LSHR:
   case G_ASHR:
@@ -5096,6 +5225,132 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI) {
   }
 }
 
+// Check that (every element of) Reg is undef or not an exact multiple of BW.
+static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI,
+                                        Register Reg, unsigned BW) {
+  return matchUnaryPredicate(
+      MRI, Reg,
+      [=](const Constant *C) {
+        // Null constant here means an undef.
+        const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(C);
+        return !CI || CI->getValue().urem(BW) != 0;
+      },
+      /*AllowUndefs*/ true);
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerFunnelShiftWithInverse(MachineInstr &MI) {
+  Register Dst = MI.getOperand(0).getReg();
+  Register X = MI.getOperand(1).getReg();
+  Register Y = MI.getOperand(2).getReg();
+  Register Z = MI.getOperand(3).getReg();
+  LLT Ty = MRI.getType(Dst);
+  LLT ShTy = MRI.getType(Z);
+
+  unsigned BW = Ty.getScalarSizeInBits();
+  const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
+  unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
+
+  if (isNonZeroModBitWidthOrUndef(MRI, Z, BW)) {
+    // fshl X, Y, Z -> fshr X, Y, -Z
+    // fshr X, Y, Z -> fshl X, Y, -Z
+    auto Zero = MIRBuilder.buildConstant(ShTy, 0);
+    Z = MIRBuilder.buildSub(Ty, Zero, Z).getReg(0);
+  } else {
+    // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
+    // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
+    auto One = MIRBuilder.buildConstant(ShTy, 1);
+    if (IsFSHL) {
+      Y = MIRBuilder.buildInstr(RevOpcode, {Ty}, {X, Y, One}).getReg(0);
+      X = MIRBuilder.buildLShr(Ty, X, One).getReg(0);
+    } else {
+      X = MIRBuilder.buildInstr(RevOpcode, {Ty}, {X, Y, One}).getReg(0);
+      Y = MIRBuilder.buildShl(Ty, Y, One).getReg(0);
+    }
+
+    Z = MIRBuilder.buildNot(ShTy, Z).getReg(0);
+  }
+
+  MIRBuilder.buildInstr(RevOpcode, {Dst}, {X, Y, Z});
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) {
+  Register Dst = MI.getOperand(0).getReg();
+  Register X = MI.getOperand(1).getReg();
+  Register Y = MI.getOperand(2).getReg();
+  Register Z = MI.getOperand(3).getReg();
+  LLT Ty = MRI.getType(Dst);
+  LLT ShTy = MRI.getType(Z);
+
+  const unsigned BW = Ty.getScalarSizeInBits();
+  const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
+
+  Register ShX, ShY;
+  Register ShAmt, InvShAmt;
+
+  // FIXME: Emit optimized urem by constant instead of letting it expand later.
+  if (isNonZeroModBitWidthOrUndef(MRI, Z, BW)) {
+    // fshl: X << C | Y >> (BW - C)
+    // fshr: X << (BW - C) | Y >> C
+    // where C = Z % BW is not zero
+    auto BitWidthC = MIRBuilder.buildConstant(ShTy, BW);
+    ShAmt = MIRBuilder.buildURem(ShTy, Z, BitWidthC).getReg(0);
+    InvShAmt = MIRBuilder.buildSub(ShTy, BitWidthC, ShAmt).getReg(0);
+    ShX = MIRBuilder.buildShl(Ty, X, IsFSHL ? ShAmt : InvShAmt).getReg(0);
+    ShY = MIRBuilder.buildLShr(Ty, Y, IsFSHL ? InvShAmt : ShAmt).getReg(0);
+  } else {
+    // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
+    // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
+    auto Mask = MIRBuilder.buildConstant(ShTy, BW - 1);
+    if (isPowerOf2_32(BW)) {
+      // Z % BW -> Z & (BW - 1)
+      ShAmt = MIRBuilder.buildAnd(ShTy, Z, Mask).getReg(0);
+      // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
+      auto NotZ = MIRBuilder.buildNot(ShTy, Z);
+      InvShAmt = MIRBuilder.buildAnd(ShTy, NotZ, Mask).getReg(0);
+    } else {
+      auto BitWidthC = MIRBuilder.buildConstant(ShTy, BW);
+      ShAmt = MIRBuilder.buildURem(ShTy, Z, BitWidthC).getReg(0);
+      InvShAmt = MIRBuilder.buildSub(ShTy, Mask, ShAmt).getReg(0);
+    }
+
+    auto One = MIRBuilder.buildConstant(ShTy, 1);
+    if (IsFSHL) {
+      ShX = MIRBuilder.buildShl(Ty, X, ShAmt).getReg(0);
+      auto ShY1 = MIRBuilder.buildLShr(Ty, Y, One);
+      ShY = MIRBuilder.buildLShr(Ty, ShY1, InvShAmt).getReg(0);
+    } else {
+      auto ShX1 = MIRBuilder.buildShl(Ty, X, One);
+      ShX = MIRBuilder.buildShl(Ty, ShX1, InvShAmt).getReg(0);
+      ShY = MIRBuilder.buildLShr(Ty, Y, ShAmt).getReg(0);
+    }
+  }
+
+  MIRBuilder.buildOr(Dst, ShX, ShY);
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerFunnelShift(MachineInstr &MI) {
+  // These operations approximately do the following (while avoiding undefined
+  // shifts by BW):
+  // G_FSHL: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
+  // G_FSHR: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
+  Register Dst = MI.getOperand(0).getReg();
+  LLT Ty = MRI.getType(Dst);
+  LLT ShTy = MRI.getType(MI.getOperand(3).getReg());
+
+  bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
+  unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
+  if (LI.getAction({RevOpcode, {Ty, ShTy}}).Action == Lower)
+    return lowerFunnelShiftAsShifts(MI);
+  return lowerFunnelShiftWithInverse(MI);
+}
+
 // Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
 // representation.
 LegalizerHelper::LegalizeResult
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index df8b7995ac88f33ff4dfc1732b68d9a840d357be..bde442ae0429e54fcff82f01738d339ec24cdcc4 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -486,6 +486,15 @@ MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
   return buildExtOrTrunc(TargetOpcode::G_ANYEXT, Res, Op);
 }
 
+MachineInstrBuilder MachineIRBuilder::buildZExtInReg(const DstOp &Res,
+                                                     const SrcOp &Op,
+                                                     int64_t ImmOp) {
+  LLT ResTy = Res.getLLTTy(*getMRI());
+  auto Mask = buildConstant(
+      ResTy, APInt::getLowBitsSet(ResTy.getScalarSizeInBits(), ImmOp));
+  return buildAnd(ResTy, Op, Mask);
+}
+
 MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
                                                 const SrcOp &Src) {
   LLT SrcTy = Src.getLLTTy(*getMRI());
diff --git a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
index cb85998a2afdffbfe0ca2a32cf0856d8955fa0f3..644a81d8021ec7551118e225b6583ee18644096c 100644
--- a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
@@ -719,6 +719,10 @@ bool RegBankSelect::runOnMachineFunction(MachineFunction &MF) {
       if (MI.isDebugInstr())
         continue;
 
+      // Ignore IMPLICIT_DEF which must have a regclass.
+      if (MI.isImplicitDef())
+        continue;
+
       if (!assignInstr(MI)) {
         reportGISelFailure(MF, *TPC, *MORE, "gisel-regbankselect",
                            "unable to map instruction", MI);
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index c24ebcf38c5f7e4bf3c11e5b5ec317531c4dfc6e..68f51c3702e4ce516124469651964f612e7daf51 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -200,6 +200,10 @@ bool llvm::isTriviallyDead(const MachineInstr &MI,
   // Don't delete frame allocation labels.
   if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE)
     return false;
+  // LIFETIME markers should be preserved even if they seem dead.
+  if (MI.getOpcode() == TargetOpcode::LIFETIME_START ||
+      MI.getOpcode() == TargetOpcode::LIFETIME_END)
+    return false;
 
   // If we can move an instruction, we can remove it.  Otherwise, it has
   // a side-effect of some sort.
@@ -477,6 +481,60 @@ Optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode, const Register Op1,
   return None;
 }
 
+Optional<APFloat> llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1,
+                                            const Register Op2,
+                                            const MachineRegisterInfo &MRI) {
+  const ConstantFP *Op2Cst = getConstantFPVRegVal(Op2, MRI);
+  if (!Op2Cst)
+    return None;
+
+  const ConstantFP *Op1Cst = getConstantFPVRegVal(Op1, MRI);
+  if (!Op1Cst)
+    return None;
+
+  APFloat C1 = Op1Cst->getValueAPF();
+  const APFloat &C2 = Op2Cst->getValueAPF();
+  switch (Opcode) {
+  case TargetOpcode::G_FADD:
+    C1.add(C2, APFloat::rmNearestTiesToEven);
+    return C1;
+  case TargetOpcode::G_FSUB:
+    C1.subtract(C2, APFloat::rmNearestTiesToEven);
+    return C1;
+  case TargetOpcode::G_FMUL:
+    C1.multiply(C2, APFloat::rmNearestTiesToEven);
+    return C1;
+  case TargetOpcode::G_FDIV:
+    C1.divide(C2, APFloat::rmNearestTiesToEven);
+    return C1;
+  case TargetOpcode::G_FREM:
+    C1.mod(C2);
+    return C1;
+  case TargetOpcode::G_FCOPYSIGN:
+    C1.copySign(C2);
+    return C1;
+  case TargetOpcode::G_FMINNUM:
+    return minnum(C1, C2);
+  case TargetOpcode::G_FMAXNUM:
+    return maxnum(C1, C2);
+  case TargetOpcode::G_FMINIMUM:
+    return minimum(C1, C2);
+  case TargetOpcode::G_FMAXIMUM:
+    return maximum(C1, C2);
+  case TargetOpcode::G_FMINNUM_IEEE:
+  case TargetOpcode::G_FMAXNUM_IEEE:
+    // FIXME: These operations were unfortunately named. fminnum/fmaxnum do not
+    // follow the IEEE behavior for signaling nans and follow libm's fmin/fmax,
+    // and currently there isn't a nice wrapper in APFloat for the version with
+    // correct snan handling.
+    break;
+  default:
+    break;
+  }
+
+  return None;
+}
+
 bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI,
                            bool SNaN) {
   const MachineInstr *DefMI = MRI.getVRegDef(Val);
@@ -548,6 +606,11 @@ Align llvm::inferAlignFromPtrInfo(MachineFunction &MF,
                            MPO.Offset);
   }
 
+  if (const Value *V = MPO.V.dyn_cast<const Value *>()) {
+    const Module *M = MF.getFunction().getParent();
+    return V->getPointerAlignment(M->getDataLayout());
+  }
+
   return Align(1);
 }
 
@@ -638,11 +701,32 @@ bool llvm::isKnownToBeAPowerOfTwo(Register Reg, const MachineRegisterInfo &MRI,
 
     break;
   }
+  case TargetOpcode::G_BUILD_VECTOR: {
+    // TODO: Probably should have a recursion depth guard since you could have
+    // bitcasted vector elements.
+    for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
+      if (!isKnownToBeAPowerOfTwo(MI.getOperand(I).getReg(), MRI, KB))
+        return false;
+    }
+
+    return true;
+  }
+  case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
+    // Only handle constants since we would need to know if number of leading
+    // zeros is greater than the truncation amount.
+    const unsigned BitWidth = Ty.getScalarSizeInBits();
+    for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
+      auto Const = getConstantVRegVal(MI.getOperand(I).getReg(), MRI);
+      if (!Const || !Const->zextOrTrunc(BitWidth).isPowerOf2())
+        return false;
+    }
+
+    return true;
+  }
   default:
     break;
   }
 
-  // TODO: Are all operands of a build vector constant powers of two?
   if (!KB)
     return false;
 
@@ -842,6 +926,38 @@ Optional<RegOrConstant> llvm::getVectorSplat(const MachineInstr &MI,
   return RegOrConstant(Reg);
 }
 
+bool llvm::matchUnaryPredicate(
+    const MachineRegisterInfo &MRI, Register Reg,
+    std::function<bool(const Constant *ConstVal)> Match, bool AllowUndefs) {
+
+  const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
+  if (AllowUndefs && Def->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
+    return Match(nullptr);
+
+  // TODO: Also handle fconstant
+  if (Def->getOpcode() == TargetOpcode::G_CONSTANT)
+    return Match(Def->getOperand(1).getCImm());
+
+  if (Def->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
+    return false;
+
+  for (unsigned I = 1, E = Def->getNumOperands(); I != E; ++I) {
+    Register SrcElt = Def->getOperand(I).getReg();
+    const MachineInstr *SrcDef = getDefIgnoringCopies(SrcElt, MRI);
+    if (AllowUndefs && SrcDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) {
+      if (!Match(nullptr))
+        return false;
+      continue;
+    }
+
+    if (SrcDef->getOpcode() != TargetOpcode::G_CONSTANT ||
+        !Match(SrcDef->getOperand(1).getCImm()))
+      return false;
+  }
+
+  return true;
+}
+
 bool llvm::isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector,
                           bool IsFP) {
   switch (TLI.getBooleanContents(IsVector, IsFP)) {
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 7dccd5da9c9a9214a285bb97728e49b8bad734ef..e4e00b2057ce137f4f01210653399922f8539802 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -700,6 +700,7 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS,
   MFI.setHasOpaqueSPAdjustment(YamlMFI.HasOpaqueSPAdjustment);
   MFI.setHasVAStart(YamlMFI.HasVAStart);
   MFI.setHasMustTailInVarArgFunc(YamlMFI.HasMustTailInVarArgFunc);
+  MFI.setHasTailCall(YamlMFI.HasTailCall);
   MFI.setLocalFrameSize(YamlMFI.LocalFrameSize);
   if (!YamlMFI.SavePoint.Value.empty()) {
     MachineBasicBlock *MBB = nullptr;
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index eae174019b5627b4c41be2bd883da3b3d2477dbe..71c815538a3771c02cc246af7fdd8d89e867d482 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -351,6 +351,7 @@ void MIRPrinter::convert(ModuleSlotTracker &MST,
   YamlMFI.HasOpaqueSPAdjustment = MFI.hasOpaqueSPAdjustment();
   YamlMFI.HasVAStart = MFI.hasVAStart();
   YamlMFI.HasMustTailInVarArgFunc = MFI.hasMustTailInVarArgFunc();
+  YamlMFI.HasTailCall = MFI.hasTailCall();
   YamlMFI.LocalFrameSize = MFI.getLocalFrameSize();
   if (MFI.getSavePoint()) {
     raw_string_ostream StrOS(YamlMFI.SavePoint.Value);
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 57eb9443a8ddcd88b5ed0122ff0ece68d17105d9..503cd5250e51a7890f8e8caceaa302982c9599ec 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1509,26 +1509,28 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
 
     break;
   }
+  case TargetOpcode::G_BZERO:
   case TargetOpcode::G_MEMSET: {
     ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
+    std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset" : "bzero";
     if (MMOs.size() != 1) {
-      report("memset must have 1 memory operand", MI);
+      report(Twine(Name, " must have 1 memory operand"), MI);
       break;
     }
 
     if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) {
-      report("memset memory operand must be a store", MI);
+      report(Twine(Name, " memory operand must be a store"), MI);
       break;
     }
 
     LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg());
     if (!DstPtrTy.isPointer()) {
-      report("memset operand must be a pointer", MI);
+      report(Twine(Name, " operand must be a pointer"), MI);
       break;
     }
 
     if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
-      report("inconsistent memset address space", MI);
+      report("inconsistent " + Twine(Name, " address space"), MI);
 
     break;
   }
@@ -1566,6 +1568,28 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
       report("Vector reduction requires vector source=", MI);
     break;
   }
+
+  case TargetOpcode::G_SBFX:
+  case TargetOpcode::G_UBFX: {
+    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+    if (DstTy.isVector()) {
+      report("Bitfield extraction is not supported on vectors", MI);
+      break;
+    }
+    break;
+  }
+  case TargetOpcode::G_ROTR:
+  case TargetOpcode::G_ROTL: {
+    LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg());
+    LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg());
+    if (Src1Ty.isVector() != Src2Ty.isVector()) {
+      report("Rotate requires operands to be either all scalars or all vectors",
+             MI);
+      break;
+    }
+    break;
+  }
+
   default:
     break;
   }
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index d98a3c1bad9cdf644f3b4d1f7358e5e48cab24e7..4ec275fdc0bf8d014d69f8a1efbbcfbc89b18dc1 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -3023,7 +3023,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
       AllocationOrder::create(VirtReg.reg(), *VRM, RegClassInfo, Matrix);
   if (MCRegister PhysReg =
           tryAssign(VirtReg, Order, NewVRegs, FixedRegisters)) {
-    // If VirtReg got an assignment, the eviction info is no longre relevant.
+    // If VirtReg got an assignment, the eviction info is no longer relevant.
     LastEvicted.clearEvicteeInfo(VirtReg.reg());
     // When NewVRegs is not empty, we may have made decisions such as evicting
     // a virtual register, go with the earlier decisions and use the physical
@@ -3060,7 +3060,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
       if (Hint && Hint != PhysReg)
         SetOfBrokenHints.insert(&VirtReg);
       // If VirtReg eviction someone, the eviction info for it as an evictee is
-      // no longre relevant.
+      // no longer relevant.
       LastEvicted.clearEvicteeInfo(VirtReg.reg());
       return PhysReg;
     }
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 0b8e6222169e3a5600182b0d017d7e9d6a3cee70..0f7811caa17f5b74c35a43f28678729b12fd60da 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -9618,7 +9618,7 @@ SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
   SDLoc DL(N);
 
   // Zap scatters with a zero mask.
-  if (ISD::isBuildVectorAllZeros(Mask.getNode()))
+  if (ISD::isConstantSplatVectorAllZeros(Mask.getNode()))
     return Chain;
 
   if (refineUniformBase(BasePtr, Index, DAG)) {
@@ -9674,7 +9674,7 @@ SDValue DAGCombiner::visitMGATHER(SDNode *N) {
   SDLoc DL(N);
 
   // Zap gathers with a zero mask.
-  if (ISD::isBuildVectorAllZeros(Mask.getNode()))
+  if (ISD::isConstantSplatVectorAllZeros(Mask.getNode()))
     return CombineTo(N, PassThru, MGT->getChain());
 
   if (refineUniformBase(BasePtr, Index, DAG)) {
@@ -11787,14 +11787,24 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
   }
 
   // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x)
-  if ((N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
-       N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
-       N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
-      N0.getOperand(0).getScalarValueSizeInBits() == ExtVTBits) {
-    if (!LegalOperations ||
-        TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))
-      return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT,
-                         N0.getOperand(0));
+  // if x is small enough or if we know that x has more than 1 sign bit and the
+  // sign_extend_inreg is extending from one of them.
+  if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
+      N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
+      N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
+    SDValue N00 = N0.getOperand(0);
+    unsigned N00Bits = N00.getScalarValueSizeInBits();
+    unsigned DstElts = N0.getValueType().getVectorMinNumElements();
+    unsigned SrcElts = N00.getValueType().getVectorMinNumElements();
+    bool IsZext = N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
+    APInt DemandedSrcElts = APInt::getLowBitsSet(SrcElts, DstElts);
+    if ((N00Bits == ExtVTBits ||
+         (!IsZext && (N00Bits < ExtVTBits ||
+                      (N00Bits - DAG.ComputeNumSignBits(N00, DemandedSrcElts)) <
+                          ExtVTBits))) &&
+        (!LegalOperations ||
+         TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT)))
+      return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, N00);
   }
 
   // fold (sext_in_reg (zext x)) -> (sext x)
@@ -12403,7 +12413,7 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
                                              VT.getVectorElementType());
 
   // If the input is a constant, let getNode fold it.
-  if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
+  if (isIntOrFPConstant(N0)) {
     // If we can't allow illegal operations, we need to check that this is just
     // a fp -> int or int -> conversion and that the resulting operation will
     // be legal.
@@ -12641,7 +12651,7 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
     return N0;
 
   // If the input is a constant, return it.
-  if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0))
+  if (isIntOrFPConstant(N0))
     return N0;
 
   return SDValue();
@@ -16902,7 +16912,7 @@ void DAGCombiner::getStoreMergeCandidates(
     case StoreSource::Constant:
       if (NoTypeMatch)
         return false;
-      if (!(isa<ConstantSDNode>(OtherBC) || isa<ConstantFPSDNode>(OtherBC)))
+      if (!isIntOrFPConstant(OtherBC))
         return false;
       break;
     case StoreSource::Extract:
@@ -20486,7 +20496,7 @@ static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN,
     // generating a splat; semantically, this is fine, but it's likely to
     // generate low-quality code if the target can't reconstruct an appropriate
     // shuffle.
-    if (!Op.isUndef() && !isa<ConstantSDNode>(Op) && !isa<ConstantFPSDNode>(Op))
+    if (!Op.isUndef() && !isIntOrFPConstant(Op))
       if (!IsSplat && !DuplicateOps.insert(Op).second)
         return SDValue();
 
@@ -21255,14 +21265,17 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
 
     // Merge shuffles through binops if we are able to merge it with at least
     // one other shuffles.
+    // shuffle(bop(shuffle(x,y),shuffle(z,w)),undef)
     // shuffle(bop(shuffle(x,y),shuffle(z,w)),bop(shuffle(a,b),shuffle(c,d)))
     unsigned SrcOpcode = N0.getOpcode();
-    if (SrcOpcode == N1.getOpcode() && TLI.isBinOp(SrcOpcode) &&
-        N->isOnlyUserOf(N0.getNode()) && N->isOnlyUserOf(N1.getNode())) {
+    if (TLI.isBinOp(SrcOpcode) && N->isOnlyUserOf(N0.getNode()) &&
+        (N1.isUndef() ||
+         (SrcOpcode == N1.getOpcode() && N->isOnlyUserOf(N1.getNode())))) {
+      // Get binop source ops, or just pass on the undef.
       SDValue Op00 = N0.getOperand(0);
-      SDValue Op10 = N1.getOperand(0);
       SDValue Op01 = N0.getOperand(1);
-      SDValue Op11 = N1.getOperand(1);
+      SDValue Op10 = N1.isUndef() ? N1 : N1.getOperand(0);
+      SDValue Op11 = N1.isUndef() ? N1 : N1.getOperand(1);
       // TODO: We might be able to relax the VT check but we don't currently
       // have any isBinOp() that has different result/ops VTs so play safe until
       // we have test coverage.
diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index cc931df5c75c3657fabbca41d55553f79b0c1907..85c6eca5775e68cc1fd75e05e6db7f38c9785359 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -192,10 +192,8 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
           MF->getFrameInfo().CreateVariableSizedObject(
               Alignment <= StackAlign ? Align(1) : Alignment, AI);
         }
-      }
-
-      // Look for inline asm that clobbers the SP register.
-      if (auto *Call = dyn_cast<CallBase>(&I)) {
+      } else if (auto *Call = dyn_cast<CallBase>(&I)) {
+        // Look for inline asm that clobbers the SP register.
         if (Call->isInlineAsm()) {
           Register SP = TLI->getStackPointerRegisterToSaveRestore();
           const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
@@ -214,21 +212,20 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
             }
           }
         }
-      }
-
-      // Look for calls to the @llvm.va_start intrinsic. We can omit some
-      // prologue boilerplate for variadic functions that don't examine their
-      // arguments.
-      if (const auto *II = dyn_cast<IntrinsicInst>(&I)) {
-        if (II->getIntrinsicID() == Intrinsic::vastart)
-          MF->getFrameInfo().setHasVAStart(true);
-      }
+        // Look for calls to the @llvm.va_start intrinsic. We can omit some
+        // prologue boilerplate for variadic functions that don't examine their
+        // arguments.
+        if (const auto *II = dyn_cast<IntrinsicInst>(&I)) {
+          if (II->getIntrinsicID() == Intrinsic::vastart)
+            MF->getFrameInfo().setHasVAStart(true);
+        }
 
-      // If we have a musttail call in a variadic function, we need to ensure we
-      // forward implicit register parameters.
-      if (const auto *CI = dyn_cast<CallInst>(&I)) {
-        if (CI->isMustTailCall() && Fn->isVarArg())
-          MF->getFrameInfo().setHasMustTailInVarArgFunc(true);
+        // If we have a musttail call in a variadic function, we need to ensure
+        // we forward implicit register parameters.
+        if (const auto *CI = dyn_cast<CallInst>(&I)) {
+          if (CI->isMustTailCall() && Fn->isVarArg())
+            MF->getFrameInfo().setHasMustTailInVarArgFunc(true);
+        }
       }
 
       // Mark values used outside their block as exported, by allocating
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 5308bc983a3afe377be07629513cf57b76749940..1a131bf32ef1654dddc86491932308f9e4002448 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1259,6 +1259,11 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
           return;
 
         if (Node->getNumValues() == 1) {
+          // Verify the new types match the original. Glue is waived because
+          // ISD::ADDC can be legalized by replacing Glue with an integer type.
+          assert((Res.getValueType() == Node->getValueType(0) ||
+                  Node->getValueType(0) == MVT::Glue) &&
+                 "Type mismatch for custom legalized operation");
           LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
           // We can just directly replace this node with the lowered value.
           ReplaceNode(SDValue(Node, 0), Res);
@@ -1266,8 +1271,14 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
         }
 
         SmallVector<SDValue, 8> ResultVals;
-        for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
+        for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
+          // Verify the new types match the original. Glue is waived because
+          // ISD::ADDC can be legalized by replacing Glue with an integer type.
+          assert((Res->getValueType(i) == Node->getValueType(i) ||
+                  Node->getValueType(i) == MVT::Glue) &&
+                 "Type mismatch for custom legalized operation");
           ResultVals.push_back(Res.getValue(i));
+        }
         LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
         ReplaceNode(Node, ResultVals.data());
         return;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index cf970566a9d23aa0ab8c4908f9bea8ab28b670bc..88b48f247698d343339dd0c8eb5236a260d43ca7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -110,6 +110,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
                          Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
   case ISD::SPLAT_VECTOR:
                          Res = PromoteIntRes_SPLAT_VECTOR(N); break;
+  case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
   case ISD::CONCAT_VECTORS:
                          Res = PromoteIntRes_CONCAT_VECTORS(N); break;
 
@@ -4782,6 +4783,18 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
   return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
 }
 
+SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
+  SDLoc dl(N);
+  EVT OutVT = N->getValueType(0);
+  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
+  assert(NOutVT.isVector() && "Type must be promoted to a vector type");
+  EVT NOutElemVT = TLI.getTypeToTransformTo(*DAG.getContext(),
+                                            NOutVT.getVectorElementType());
+  APInt StepVal = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
+  SDValue Step = DAG.getConstant(StepVal.getZExtValue(), dl, NOutElemVT);
+  return DAG.getStepVector(dl, NOutVT, Step);
+}
+
 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
   SDLoc dl(N);
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index de916823463f53676646a724c8551193f529bc8f..a40dd888703351c30cd605c75631d39b65c20e31 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -304,6 +304,7 @@ private:
   SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
   SDValue PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N);
   SDValue PromoteIntRes_SPLAT_VECTOR(SDNode *N);
+  SDValue PromoteIntRes_STEP_VECTOR(SDNode *N);
   SDValue PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N);
   SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N);
   SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N);
@@ -836,6 +837,7 @@ private:
   void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, SDValue &Hi);
+  void SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 5843e7396818c7d0c837af8af4cd7960479dd4c2..190b4f32c4f88b0a6cc66b35c6b4b8a6dd4ddcb7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -928,6 +928,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::SCALAR_TO_VECTOR:
     SplitVecRes_ScalarOp(N, Lo, Hi);
     break;
+  case ISD::STEP_VECTOR:
+    SplitVecRes_STEP_VECTOR(N, Lo, Hi);
+    break;
   case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
   case ISD::LOAD:
     SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
@@ -1639,6 +1642,30 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
     Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
 }
 
+void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
+                                               SDValue &Hi) {
+  EVT LoVT, HiVT;
+  SDLoc dl(N);
+  assert(N->getValueType(0).isScalableVector() &&
+         "Only scalable vectors are supported for STEP_VECTOR");
+  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
+  SDValue Step = N->getOperand(0);
+
+  Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
+
+  // Hi = Lo + (EltCnt * Step)
+  EVT EltVT = Step.getValueType();
+  SDValue StartOfHi =
+      DAG.getVScale(dl, EltVT,
+                    cast<ConstantSDNode>(Step)->getAPIntValue() *
+                        LoVT.getVectorMinNumElements());
+  StartOfHi = DAG.getZExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
+  StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
+
+  Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
+  Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
+}
+
 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
                                             SDValue &Hi) {
   EVT LoVT, HiVT;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 4ea4219e54ed77a74d01fa9fc501bbb0a85fcc6e..fb4ae86a7e24089f7c04be3fa963dbdbf79cd4e7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1744,6 +1744,18 @@ SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
   return SDValue(CondCodeNodes[Cond], 0);
 }
 
+SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, SDValue Step) {
+  if (ResVT.isScalableVector())
+    return getNode(ISD::STEP_VECTOR, DL, ResVT, Step);
+
+  EVT OpVT = Step.getValueType();
+  APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
+  SmallVector<SDValue, 16> OpsStepConstants;
+  for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
+    OpsStepConstants.push_back(getConstant(StepVal * i, DL, OpVT));
+  return getBuildVector(ResVT, DL, OpsStepConstants);
+}
+
 /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
 /// point at N1 to point at N2 and indices that point at N2 to point at N1.
 static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
@@ -2470,6 +2482,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
     }
     break;
   }
+  case ISD::ABS:
   case ISD::TRUNCATE:
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
@@ -2978,6 +2991,38 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     Known = KnownBits::computeForMul(Known, Known2);
     break;
   }
+  case ISD::MULHU: {
+    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    Known = KnownBits::mulhu(Known, Known2);
+    break;
+  }
+  case ISD::MULHS: {
+    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    Known = KnownBits::mulhs(Known, Known2);
+    break;
+  }
+  case ISD::UMUL_LOHI: {
+    assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
+    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    if (Op.getResNo() == 0)
+      Known = KnownBits::computeForMul(Known, Known2);
+    else
+      Known = KnownBits::mulhu(Known, Known2);
+    break;
+  }
+  case ISD::SMUL_LOHI: {
+    assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
+    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    if (Op.getResNo() == 0)
+      Known = KnownBits::computeForMul(Known, Known2);
+    else
+      Known = KnownBits::mulhs(Known, Known2);
+    break;
+  }
   case ISD::UDIV: {
     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
@@ -4306,6 +4351,14 @@ bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
   return (computeKnownBits(A).Zero | computeKnownBits(B).Zero).isAllOnesValue();
 }
 
+static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
+                               SelectionDAG &DAG) {
+  if (cast<ConstantSDNode>(Step)->isNullValue())
+    return DAG.getConstant(0, DL, VT);
+
+  return SDValue();
+}
+
 static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
                                 ArrayRef<SDValue> Ops,
                                 SelectionDAG &DAG) {
@@ -4527,6 +4580,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                         APFloat::rmNearestTiesToEven, &Ignored);
       return getConstantFP(FPV, DL, VT);
     }
+    case ISD::STEP_VECTOR: {
+      if (SDValue V = FoldSTEP_VECTOR(DL, VT, Operand, *this))
+        return V;
+      break;
+    }
     }
   }
 
@@ -4636,6 +4694,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
 
   unsigned OpOpcode = Operand.getNode()->getOpcode();
   switch (Opcode) {
+  case ISD::STEP_VECTOR:
+    assert(VT.isScalableVector() &&
+           "STEP_VECTOR can only be used with scalable types");
+    assert(VT.getScalarSizeInBits() >= 8 &&
+           "STEP_VECTOR can only be used with vectors of integers that are at "
+           "least 8 bits wide");
+    assert(Operand.getValueType().bitsGE(VT.getScalarType()) &&
+           "Operand type should be at least as large as the element type");
+    assert(isa<ConstantSDNode>(Operand) &&
+           cast<ConstantSDNode>(Operand)->getAPIntValue().isNonNegative() &&
+           "Expected positive integer constant for STEP_VECTOR");
+    break;
   case ISD::FREEZE:
     assert(VT == Operand.getValueType() && "Unexpected VT!");
     break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 432ceccb12ddbfb0b789f158c8f3473fd1dc9904..ff7692a59eddc85f5210f1d8d87920c01ba9f765 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7008,7 +7008,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   case Intrinsic::experimental_deoptimize:
     LowerDeoptimizeCall(&I);
     return;
-
+  case Intrinsic::experimental_stepvector:
+    visitStepVector(I);
+    return;
   case Intrinsic::vector_reduce_fadd:
   case Intrinsic::vector_reduce_fmul:
   case Intrinsic::vector_reduce_add:
@@ -11005,6 +11007,16 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
   }
 }
 
+void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  auto DL = getCurSDLoc();
+  EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+  EVT OpVT =
+      TLI.getTypeToTransformTo(*DAG.getContext(), ResultVT.getScalarType());
+  SDValue Step = DAG.getConstant(1, DL, OpVT);
+  setValue(&I, DAG.getStepVector(DL, ResultVT, Step));
+}
+
 void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index a759f8babe33470c5679f6cada65021b2e2af4d4..156f4c00273dc1a3207c856cd6c6aedf9b790dfe 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -779,6 +779,7 @@ private:
   void visitVectorReduce(const CallInst &I, unsigned Intrinsic);
   void visitVectorReverse(const CallInst &I);
   void visitVectorSplice(const CallInst &I);
+  void visitStepVector(const CallInst &I);
 
   void visitUserOp1(const Instruction &I) {
     llvm_unreachable("UserOp1 should not exist at instruction selection time!");
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 8402163fd92bd561f5c0095fd4e0739388750c3d..7bba8d30eb3fa7ac8384b747f6567ee9a4948f58 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -292,6 +292,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::SPLAT_VECTOR:               return "splat_vector";
   case ISD::SPLAT_VECTOR_PARTS:         return "splat_vector_parts";
   case ISD::VECTOR_REVERSE:             return "vector_reverse";
+  case ISD::STEP_VECTOR:                return "step_vector";
   case ISD::CARRY_FALSE:                return "carry_false";
   case ISD::ADDC:                       return "addc";
   case ISD::ADDE:                       return "adde";
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 9ec94892c0d2a8aadd9112459f249ca1aa6eae59..48a2cdd86d08965c2d55ec61ad7ecdeaa3d544f0 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -45,7 +45,6 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
-#include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
@@ -114,17 +113,6 @@ static bool darwinHasSinCos(const Triple &TT) {
   return true;
 }
 
-// Although this default value is arbitrary, it is not random. It is assumed
-// that a condition that evaluates the same way by a higher percentage than this
-// is best represented as control flow. Therefore, the default value N should be
-// set such that the win from N% correct executions is greater than the loss
-// from (100 - N)% mispredicted executions for the majority of intended targets.
-static cl::opt<int> MinPercentageForPredictableBranch(
-    "min-predictable-branch", cl::init(99),
-    cl::desc("Minimum percentage (0-100) that a condition must be either true "
-             "or false to assume that the condition is predictable"),
-    cl::Hidden);
-
 void TargetLoweringBase::InitLibcalls(const Triple &TT) {
 #define HANDLE_LIBCALL(code, name) \
   setLibcallName(RTLIB::code, name);
@@ -1736,10 +1724,6 @@ bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
                             MMO.getAlign(), MMO.getFlags(), Fast);
 }
 
-BranchProbability TargetLoweringBase::getPredictableBranchThreshold() const {
-  return BranchProbability(MinPercentageForPredictableBranch, 100);
-}
-
 //===----------------------------------------------------------------------===//
 //  TargetTransformInfo Helpers
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index a011b03d747c983a0a50bb217a54d2aa76a51ee5..bd20f32ee2534d1e1ad68efb2b1fc44545926e85 100644
--- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -1357,11 +1357,9 @@ void
 TwoAddressInstructionPass::processTiedPairs(MachineInstr *MI,
                                             TiedPairList &TiedPairs,
                                             unsigned &Dist) {
-  bool IsEarlyClobber = false;
-  for (unsigned tpi = 0, tpe = TiedPairs.size(); tpi != tpe; ++tpi) {
-    const MachineOperand &DstMO = MI->getOperand(TiedPairs[tpi].second);
-    IsEarlyClobber |= DstMO.isEarlyClobber();
-  }
+  bool IsEarlyClobber = llvm::find_if(TiedPairs, [MI](auto const &TP) {
+                          return MI->getOperand(TP.second).isEarlyClobber();
+                        }) != TiedPairs.end();
 
   bool RemovedKillFlag = false;
   bool AllUsesCopied = true;
@@ -1369,9 +1367,9 @@ TwoAddressInstructionPass::processTiedPairs(MachineInstr *MI,
   SlotIndex LastCopyIdx;
   Register RegB = 0;
   unsigned SubRegB = 0;
-  for (unsigned tpi = 0, tpe = TiedPairs.size(); tpi != tpe; ++tpi) {
-    unsigned SrcIdx = TiedPairs[tpi].first;
-    unsigned DstIdx = TiedPairs[tpi].second;
+  for (auto &TP : TiedPairs) {
+    unsigned SrcIdx = TP.first;
+    unsigned DstIdx = TP.second;
 
     const MachineOperand &DstMO = MI->getOperand(DstIdx);
     Register RegA = DstMO.getReg();
diff --git a/llvm/lib/CodeGen/TypePromotion.cpp b/llvm/lib/CodeGen/TypePromotion.cpp
index a42095d8718a338406ef274135d873efc93f14a6..1a488863a8cd4c260ee06c62ac0516b0ec59e795 100644
--- a/llvm/lib/CodeGen/TypePromotion.cpp
+++ b/llvm/lib/CodeGen/TypePromotion.cpp
@@ -952,7 +952,8 @@ bool TypePromotion::runOnFunction(Function &F) {
   const TargetLowering *TLI = SubtargetInfo->getTargetLowering();
   const TargetTransformInfo &TII =
     getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
-  RegisterBitWidth = TII.getRegisterBitWidth(false);
+  RegisterBitWidth =
+      TII.getRegisterBitWidth(TargetTransformInfo::RGK_Scalar).getFixedSize();
   Ctx = &F.getParent()->getContext();
 
   // Search up from icmps to try to promote their operands.
diff --git a/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp b/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
index f946dd4860ace02d718a0ed78931a7e0d734a009..a02b9ea9e35bdae73683b931772bef9b61690307 100644
--- a/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
+++ b/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
@@ -341,7 +341,17 @@ Expected<FileBufferByteStream> MSFBuilder::commit(StringRef Path,
 
   Layout = std::move(*L);
 
-  uint64_t FileSize = Layout.SB->BlockSize * Layout.SB->NumBlocks;
+  uint64_t FileSize = uint64_t(Layout.SB->BlockSize) * Layout.SB->NumBlocks;
+  if (FileSize > UINT32_MAX) {
+    // FIXME: Changing the BinaryStream classes to use 64-bit numbers lets
+    // us create PDBs larger than 4 GiB successfully. The file format is
+    // block-based and as long as each stream is small enough, PDBs larger than
+    // 4 GiB might work. Check if tools can handle these large PDBs, and if so
+    // add support for writing them.
+    return make_error<MSFError>(msf_error_code::invalid_format,
+                                "Output larger than 4 GiB");
+  }
+
   auto OutFileOrError = FileOutputBuffer::create(Path, FileSize);
   if (auto EC = OutFileOrError.takeError())
     return std::move(EC);
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp b/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp
index 5d7946cdc2f718ef06cbcb3b936ad6a1e0b71811..7212a0e65035774ab75cf21a992a9d048d394e9f 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp
@@ -83,7 +83,7 @@ Error NativeSession::createFromPdb(std::unique_ptr<MemoryBuffer> Buffer,
 static Expected<std::unique_ptr<PDBFile>>
 loadPdbFile(StringRef PdbPath, std::unique_ptr<BumpPtrAllocator> &Allocator) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> ErrorOrBuffer =
-      MemoryBuffer::getFile(PdbPath, /*FileSize=*/-1,
+      MemoryBuffer::getFile(PdbPath, /*IsText=*/false,
                             /*RequiresNullTerminator=*/false);
   if (!ErrorOrBuffer)
     return make_error<RawError>(ErrorOrBuffer.getError());
diff --git a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h
deleted file mode 100644
index a67d8b25ef54daf48522a090bcd6be21fff6f31e..0000000000000000000000000000000000000000
--- a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h
+++ /dev/null
@@ -1,107 +0,0 @@
-//===--- BasicGOTAndStubsBuilder.h - Generic GOT/Stub creation --*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// A base for simple GOT and stub creation.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H
-#define LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H
-
-#include "llvm/ExecutionEngine/JITLink/JITLink.h"
-
-#define DEBUG_TYPE "jitlink"
-
-namespace llvm {
-namespace jitlink {
-
-template <typename BuilderImpl> class BasicGOTAndStubsBuilder {
-public:
-  BasicGOTAndStubsBuilder(LinkGraph &G) : G(G) {}
-
-  void run() {
-    // We're going to be adding new blocks, but we don't want to iterate over
-    // the newly added ones, so just copy the existing blocks out.
-    std::vector<Block *> Blocks(G.blocks().begin(), G.blocks().end());
-
-    LLVM_DEBUG(dbgs() << "Creating GOT entries and stubs:\n");
-
-    for (auto *B : Blocks)
-      for (auto &E : B->edges())
-        if (impl().isGOTEdgeToFix(E)) {
-          LLVM_DEBUG({
-            dbgs() << "  Updating GOT edge ";
-            printEdge(dbgs(), *B, E, "<target GOT>");
-            dbgs() << "\n";
-          });
-          impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget()));
-        } else if (impl().isExternalBranchEdge(E)) {
-          LLVM_DEBUG({
-            dbgs() << "  Updating external branch edge ";
-            printEdge(dbgs(), *B, E, "<target PC-rel>");
-            dbgs() << "\n";
-          });
-          impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget()));
-        }
-  }
-
-protected:
-  Symbol &getGOTEntrySymbol(Symbol &Target) {
-    assert(Target.hasName() && "GOT edge cannot point to anonymous target");
-
-    auto GOTEntryI = GOTEntries.find(Target.getName());
-
-    // Build the entry if it doesn't exist.
-    if (GOTEntryI == GOTEntries.end()) {
-      auto &GOTEntry = impl().createGOTEntry(Target);
-      LLVM_DEBUG({
-        dbgs() << "    Created GOT entry for " << Target.getName() << ": "
-               << GOTEntry << "\n";
-      });
-      GOTEntryI =
-          GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first;
-    }
-
-    assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol");
-    LLVM_DEBUG(
-        { dbgs() << "    Using GOT entry " << *GOTEntryI->second << "\n"; });
-    return *GOTEntryI->second;
-  }
-
-  Symbol &getStubSymbol(Symbol &Target) {
-    assert(Target.hasName() &&
-           "External branch edge can not point to an anonymous target");
-    auto StubI = Stubs.find(Target.getName());
-
-    if (StubI == Stubs.end()) {
-      auto &StubSymbol = impl().createStub(Target);
-      LLVM_DEBUG({
-        dbgs() << "    Created stub for " << Target.getName() << ": "
-               << StubSymbol << "\n";
-      });
-      StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first;
-    }
-
-    assert(StubI != Stubs.end() && "Count not get stub symbol");
-    LLVM_DEBUG({ dbgs() << "    Using stub " << *StubI->second << "\n"; });
-    return *StubI->second;
-  }
-
-  LinkGraph &G;
-
-private:
-  BuilderImpl &impl() { return static_cast<BuilderImpl &>(*this); }
-
-  DenseMap<StringRef, Symbol *> GOTEntries;
-  DenseMap<StringRef, Symbol *> Stubs;
-};
-
-} // end namespace jitlink
-} // end namespace llvm
-
-#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H
diff --git a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt
index f613b93135058a175a2c874d77165b932908d5de..bb3b622671067d81e04eb8ea7109f0d6a4bc66eb 100644
--- a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt
@@ -1,8 +1,8 @@
 add_llvm_component_library(LLVMJITLink
+  EHFrameSupport.cpp
   JITLink.cpp
   JITLinkGeneric.cpp
   JITLinkMemoryManager.cpp
-  EHFrameSupport.cpp
 
   # Formats:
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
new file mode 100644
index 0000000000000000000000000000000000000000..8ae3bc2bf61def39e39c71d1d0da849e0132987a
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
@@ -0,0 +1,116 @@
+//===--------- DefineExternalSectionStartAndEndSymbols.h --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utility class for recognizing external section start and end symbols and
+// transforming them into defined symbols for the start and end blocks of the
+// associated Section.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_JITLINK_DEFINEEXTERNALSECTIONSTARTANDENDSYMBOLS_H
+#define LLVM_EXECUTIONENGINE_JITLINK_DEFINEEXTERNALSECTIONSTARTANDENDSYMBOLS_H
+
+#include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "jitlink"
+
+namespace llvm {
+namespace jitlink {
+
+struct SectionRangeSymbolDesc {
+  SectionRangeSymbolDesc() = default;
+  SectionRangeSymbolDesc(Section &Sec, bool IsStart)
+      : Sec(&Sec), IsStart(IsStart) {}
+  Section *Sec = nullptr;
+  bool IsStart = false;
+};
+
+/// Pass implementation for the createDefineExternalSectionStartAndEndSymbols
+/// function.
+template <typename SymbolIdentifierFunction>
+class DefineExternalSectionStartAndEndSymbols {
+public:
+  DefineExternalSectionStartAndEndSymbols(SymbolIdentifierFunction F)
+      : F(std::move(F)) {}
+
+  Error operator()(LinkGraph &G) {
+
+    // This pass will affect the external symbols set, so copy them out into a
+    // vector and iterate over that.
+    std::vector<Symbol *> Externals(G.external_symbols().begin(),
+                                    G.external_symbols().end());
+
+    for (auto *Sym : Externals) {
+      SectionRangeSymbolDesc D = F(G, *Sym);
+      if (D.Sec) {
+        auto &SR = getSectionRange(*D.Sec);
+        if (D.IsStart) {
+          if (SR.empty())
+            G.makeAbsolute(*Sym, 0);
+          else
+            G.makeDefined(*Sym, *SR.getFirstBlock(), 0, 0, Linkage::Strong,
+                          Scope::Local, false);
+        } else {
+          if (SR.empty())
+            G.makeAbsolute(*Sym, 0);
+          else
+            G.makeDefined(*Sym, *SR.getLastBlock(),
+                          SR.getLastBlock()->getSize(), 0, Linkage::Strong,
+                          Scope::Local, false);
+        }
+      }
+    }
+    return Error::success();
+  }
+
+private:
+  SectionRange &getSectionRange(Section &Sec) {
+    auto I = SectionRanges.find(&Sec);
+    if (I == SectionRanges.end())
+      I = SectionRanges.insert(std::make_pair(&Sec, SectionRange(Sec))).first;
+    return I->second;
+  }
+
+  DenseMap<Section *, SectionRange> SectionRanges;
+  SymbolIdentifierFunction F;
+};
+
+/// Returns a JITLink pass (as a function class) that uses the given symbol
+/// identification function to identify external section start and end symbols
+/// (and their associated Section*s) and transform the identified externals
+/// into defined symbols pointing to the start of the first block in the
+/// section and the end of the last (start and end symbols for empty sections
+/// will be transformed into absolute symbols at address 0).
+///
+/// The identification function should be callable as
+///
+///   SectionRangeSymbolDesc (LinkGraph &G, Symbol &Sym)
+///
+/// If Sym is not a section range start or end symbol then a default
+/// constructed SectionRangeSymbolDesc should be returned. If Sym is a start
+/// symbol then SectionRangeSymbolDesc(Sec, true), where Sec is a reference to
+/// the target Section. If Sym is an end symbol then
+/// SectionRangeSymbolDesc(Sec, false) should be returned.
+///
+/// This pass should be run in the PostAllocationPass pipeline, at which point
+/// all blocks should have been assigned their final addresses.
+template <typename SymbolIdentifierFunction>
+DefineExternalSectionStartAndEndSymbols<SymbolIdentifierFunction>
+createDefineExternalSectionStartAndEndSymbolsPass(
+    SymbolIdentifierFunction &&F) {
+  return DefineExternalSectionStartAndEndSymbols<SymbolIdentifierFunction>(
+      std::forward<SymbolIdentifierFunction>(F));
+}
+
+} // end namespace jitlink
+} // end namespace llvm
+
+#undef DEBUG_TYPE
+
+#endif // LLVM_EXECUTIONENGINE_JITLINK_DEFINEEXTERNALSECTIONSTARTANDENDSYMBOLS_H
diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
index 93123ca743e998b4bffdbc511572b4defd84749a..1cf84b6b7c8a76c63e74abb2bda66f0615e31261 100644
--- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
@@ -774,7 +774,7 @@ createEHFrameRecorderPass(const Triple &TT,
                           StoreFrameRangeFunction StoreRangeAddress) {
   const char *EHFrameSectionName = nullptr;
   if (TT.getObjectFormat() == Triple::MachO)
-    EHFrameSectionName = "__eh_frame";
+    EHFrameSectionName = "__TEXT,__eh_frame";
   else
     EHFrameSectionName = ".eh_frame";
 
@@ -791,8 +791,9 @@ createEHFrameRecorderPass(const Triple &TT,
       Size = R.getSize();
     }
     if (Addr == 0 && Size != 0)
-      return make_error<JITLinkError>("__eh_frame section can not have zero "
-                                      "address with non-zero size");
+      return make_error<JITLinkError>(
+          StringRef(EHFrameSectionName) +
+          " section can not have zero address with non-zero size");
     StoreFrameRange(Addr, Size);
     return Error::success();
   };
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
index be104c3ad1453de778c2b8f117e859873eec1f19..17c537eea6b5cfad8b9a7e904c2f2cc2fbb98b33 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -12,12 +12,14 @@
 
 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/ExecutionEngine/JITLink/x86_64.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Support/Endian.h"
 
-#include "BasicGOTAndStubsBuilder.h"
+#include "DefineExternalSectionStartAndEndSymbols.h"
 #include "EHFrameSupportImpl.h"
 #include "JITLinkGeneric.h"
+#include "PerGraphGOTAndPLTStubsBuilder.h"
 
 #define DEBUG_TYPE "jitlink"
 
@@ -27,17 +29,29 @@ using namespace llvm::jitlink::ELF_x86_64_Edges;
 
 namespace {
 
-class ELF_x86_64_GOTAndStubsBuilder
-    : public BasicGOTAndStubsBuilder<ELF_x86_64_GOTAndStubsBuilder> {
+constexpr StringRef ELFGOTSectionName = "$__GOT";
+constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
+
+class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64
+    : public PerGraphGOTAndPLTStubsBuilder<
+          PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> {
 public:
   static const uint8_t NullGOTEntryContent[8];
   static const uint8_t StubContent[6];
 
-  ELF_x86_64_GOTAndStubsBuilder(LinkGraph &G)
-      : BasicGOTAndStubsBuilder<ELF_x86_64_GOTAndStubsBuilder>(G) {}
+  using PerGraphGOTAndPLTStubsBuilder<
+      PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder;
 
   bool isGOTEdgeToFix(Edge &E) const {
-    return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad;
+    if (E.getKind() == GOTOFF64) {
+      // We need to make sure that the GOT section exists, but don't otherwise
+      // need to fix up this edge.
+      getGOTSection();
+      return false;
+    }
+
+    return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad ||
+           E.getKind() == PCRel64GOT || E.getKind() == GOT64;
   }
 
   Symbol &createGOTEntry(Symbol &Target) {
@@ -48,14 +62,25 @@ public:
   }
 
   void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
-    assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) &&
-           "Not a GOT edge?");
-    // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is
-    // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to
-    // check for GOT optimization opportunities in the
+    // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary
+    // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now:
+    // We will use the kind to check for GOT optimization opportunities in the
     // optimizeMachO_x86_64_GOTAndStubs pass below.
-    if (E.getKind() == PCRel32GOT)
+    // If it's a GOT64 leave it as is.
+    switch (E.getKind()) {
+    case PCRel32GOT:
       E.setKind(PCRel32);
+      break;
+    case PCRel64GOT:
+      E.setKind(PCRel64);
+      break;
+    case GOT64:
+      break;
+    case PCRel32GOTLoad:
+      break;
+    default:
+      llvm_unreachable("Unexpected GOT edge kind");
+    }
 
     E.setTarget(GOTEntry);
     // Leave the edge addend as-is.
@@ -65,16 +90,16 @@ public:
     return E.getKind() == Branch32 && !E.getTarget().isDefined();
   }
 
-  Symbol &createStub(Symbol &Target) {
+  Symbol &createPLTStub(Symbol &Target) {
     auto &StubContentBlock =
         G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
     // Re-use GOT entries for stub targets.
-    auto &GOTEntrySymbol = getGOTEntrySymbol(Target);
+    auto &GOTEntrySymbol = getGOTEntry(Target);
     StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, -4);
     return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
   }
 
-  void fixExternalBranchEdge(Edge &E, Symbol &Stub) {
+  void fixPLTEdge(Edge &E, Symbol &Stub) {
     assert(E.getKind() == Branch32 && "Not a Branch32 edge?");
 
     // Set the edge kind to Branch32ToStub. We will use this to check for stub
@@ -85,13 +110,13 @@ public:
   }
 
 private:
-  Section &getGOTSection() {
+  Section &getGOTSection() const {
     if (!GOTSection)
-      GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ);
+      GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ);
     return *GOTSection;
   }
 
-  Section &getStubsSection() {
+  Section &getStubsSection() const {
     if (!StubsSection) {
       auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
           sys::Memory::MF_READ | sys::Memory::MF_EXEC);
@@ -110,8 +135,8 @@ private:
                      sizeof(StubContent));
   }
 
-  Section *GOTSection = nullptr;
-  Section *StubsSection = nullptr;
+  mutable Section *GOTSection = nullptr;
+  mutable Section *StubsSection = nullptr;
 };
 
 const char *const DwarfSectionNames[] = {
@@ -123,9 +148,9 @@ const char *const DwarfSectionNames[] = {
 
 } // namespace
 
-const uint8_t ELF_x86_64_GOTAndStubsBuilder::NullGOTEntryContent[8] = {
-    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
-const uint8_t ELF_x86_64_GOTAndStubsBuilder::StubContent[6] = {
+const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] =
+    {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = {
     0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
 
 static const char *CommonSectionName = "__common";
@@ -171,9 +196,10 @@ static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) {
         }
       } else if (E.getKind() == Branch32ToStub) {
         auto &StubBlock = E.getTarget().getBlock();
-        assert(StubBlock.getSize() ==
-                   sizeof(ELF_x86_64_GOTAndStubsBuilder::StubContent) &&
-               "Stub block should be stub sized");
+        assert(
+            StubBlock.getSize() ==
+                sizeof(PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent) &&
+            "Stub block should be stub sized");
         assert(StubBlock.edges_size() == 1 &&
                "Stub block should only have one outgoing edge");
 
@@ -216,6 +242,7 @@ class ELFLinkGraphBuilder_x86_64 {
 
 private:
   Section *CommonSection = nullptr;
+
   // TODO hack to get this working
   // Find a better way
   using SymbolTable = object::ELFFile<object::ELF64LE>::Elf_Shdr;
@@ -238,6 +265,7 @@ private:
     case ELF::R_X86_64_PC32:
       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
     case ELF::R_X86_64_PC64:
+    case ELF::R_X86_64_GOTPC64:
       return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
     case ELF::R_X86_64_64:
       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
@@ -245,6 +273,12 @@ private:
     case ELF::R_X86_64_GOTPCRELX:
     case ELF::R_X86_64_REX_GOTPCRELX:
       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
+    case ELF::R_X86_64_GOTPCREL64:
+      return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
+    case ELF::R_X86_64_GOT64:
+      return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
+    case ELF::R_X86_64_GOTOFF64:
+      return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
     case ELF::R_X86_64_PLT32:
       return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
     }
@@ -692,22 +726,63 @@ public:
   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
                       std::unique_ptr<LinkGraph> G,
                       PassConfiguration PassConfig)
-      : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}
+      : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
+    getPassConfig().PostAllocationPasses.push_back(
+        [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
+  }
 
 private:
+  Symbol *GOTSymbol = nullptr;
+
+  Error getOrCreateGOTSymbol(LinkGraph &G) {
+    auto DefineExternalGOTSymbolIfPresent =
+        createDefineExternalSectionStartAndEndSymbolsPass(
+            [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
+              if (Sym.getName() == ELFGOTSymbolName)
+                if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
+                  GOTSymbol = &Sym;
+                  return {*GOTSection, true};
+                }
+              return {};
+            });
+
+    // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
+    // external.
+    if (auto Err = DefineExternalGOTSymbolIfPresent(G))
+      return Err;
 
-  static Error targetOutOfRangeError(const Block &B, const Edge &E) {
-    std::string ErrMsg;
-    {
-      raw_string_ostream ErrStream(ErrMsg);
-      ErrStream << "Relocation target out of range: ";
-      printEdge(ErrStream, B, E, getELFX86RelocationKindName(E.getKind()));
-      ErrStream << "\n";
+    // If we succeeded then we're done.
+    if (GOTSymbol)
+      return Error::success();
+
+    // Otherwise look for a GOT section: If it already has a start symbol we'll
+    // record it, otherwise we'll create our own.
+    // If there's a GOT section but we didn't find an external GOT symbol...
+    if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) {
+
+      // Check for an existing defined symbol.
+      for (auto *Sym : GOTSection->symbols())
+        if (Sym->getName() == ELFGOTSymbolName) {
+          GOTSymbol = Sym;
+          return Error::success();
+        }
+
+      // If there's no defined symbol then create one.
+      SectionRange SR(*GOTSection);
+      if (SR.empty())
+        GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0,
+                                         Linkage::Strong, Scope::Local, true);
+      else
+        GOTSymbol =
+            &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
+                                Linkage::Strong, Scope::Local, false, true);
     }
-    return make_error<JITLinkError>(std::move(ErrMsg));
+
+    return Error::success();
   }
 
-  Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const {
+  Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
+                   char *BlockWorkingMem) const {
     using namespace ELF_x86_64_Edges;
     using namespace llvm::support;
     char *FixupPtr = BlockWorkingMem + E.getOffset();
@@ -718,10 +793,15 @@ private:
     case ELFX86RelocationKind::PCRel32:
     case ELFX86RelocationKind::PCRel32GOTLoad: {
       int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
-      if (Value < std::numeric_limits<int32_t>::min() ||
-          Value > std::numeric_limits<int32_t>::max())
-        return targetOutOfRangeError(B, E);
-      *(little32_t *)FixupPtr = Value;
+      if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value)))
+        *(little32_t *)FixupPtr = Value;
+      else
+        return makeTargetOutOfRangeError(G, B, E);
+      break;
+    }
+    case ELFX86RelocationKind::PCRel64: {
+      int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
+      *(little64_t *)FixupPtr = Value;
       break;
     }
     case ELFX86RelocationKind::Pointer64: {
@@ -729,11 +809,51 @@ private:
       *(ulittle64_t *)FixupPtr = Value;
       break;
     }
+    case ELFX86RelocationKind::Delta32: {
+      int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
+      if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value)))
+        *(little32_t *)FixupPtr = Value;
+      else
+        return makeTargetOutOfRangeError(G, B, E);
+      break;
+    }
     case ELFX86RelocationKind::Delta64: {
       int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
       *(little64_t *)FixupPtr = Value;
       break;
     }
+    case ELFX86RelocationKind::NegDelta32: {
+      int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
+      if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value)))
+        *(little32_t *)FixupPtr = Value;
+      else
+        return makeTargetOutOfRangeError(G, B, E);
+      break;
+    }
+    case ELFX86RelocationKind::NegDelta64: {
+      int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
+      *(little64_t *)FixupPtr = Value;
+      break;
+    }
+    case ELFX86RelocationKind::GOT64:
+    case ELFX86RelocationKind::GOTOFF64: {
+      // GOT64: Offset of GOT entry within GOT.
+      // GOTOFF64: Offset from GOT base to target.
+      // The expressions are the same in both cases, but in the GOT64 case the
+      // edge will have been fixed to point at the GOT entry, and in the
+      // GOTOFF64 case it will still point at the original target.
+      assert(GOTSymbol && "No GOT section symbol");
+      int64_t Value =
+          E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend();
+      *(little64_t *)FixupPtr = Value;
+      break;
+    }
+    default:
+      LLVM_DEBUG({
+        dbgs() << "Bad edge: " << getELFX86RelocationKindName(E.getKind())
+               << "\n";
+      });
+      llvm_unreachable("Unsupported relocation");
     }
     return Error::success();
   }
@@ -756,6 +876,24 @@ createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
       .buildGraph();
 }
 
+static SectionRangeSymbolDesc
+identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
+  constexpr StringRef StartSymbolPrefix = "__start";
+  constexpr StringRef EndSymbolPrefix = "__end";
+
+  auto SymName = Sym.getName();
+  if (SymName.startswith(StartSymbolPrefix)) {
+    if (auto *Sec =
+            G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
+      return {*Sec, true};
+  } else if (SymName.startswith(EndSymbolPrefix)) {
+    if (auto *Sec =
+            G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
+      return {*Sec, false};
+  }
+  return {};
+}
+
 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
                      std::unique_ptr<JITLinkContext> Ctx) {
   PassConfiguration Config;
@@ -775,10 +913,13 @@ void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 
     // Add an in-place GOT/Stubs pass.
-    Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error {
-      ELF_x86_64_GOTAndStubsBuilder(G).run();
-      return Error::success();
-    });
+    Config.PostPrunePasses.push_back(
+        PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass);
+
+    // Resolve any external section start / end symbols.
+    Config.PostAllocationPasses.push_back(
+        createDefineExternalSectionStartAndEndSymbolsPass(
+            identifyELFSectionStartAndEndSymbols));
 
     // Add GOT/Stubs optimizer pass.
     Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs);
@@ -791,16 +932,46 @@ void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
 }
 const char *getELFX86RelocationKindName(Edge::Kind R) {
   switch (R) {
-  case PCRel32:
-    return "PCRel32";
-  case Pointer64:
-    return "Pointer64";
-  case PCRel32GOTLoad:
-    return "PCRel32GOTLoad";
   case Branch32:
     return "Branch32";
   case Branch32ToStub:
     return "Branch32ToStub";
+  case Pointer32:
+    return "Pointer32";
+  case Pointer64:
+    return "Pointer64";
+  case Pointer64Anon:
+    return "Pointer64Anon";
+  case PCRel32:
+    return "PCRel32";
+  case PCRel32Minus1:
+    return "PCRel32Minus1";
+  case PCRel32Minus2:
+    return "PCRel32Minus2";
+  case PCRel32Minus4:
+    return "PCRel32Minus4";
+  case PCRel32Anon:
+    return "PCRel32Anon";
+  case PCRel32Minus1Anon:
+    return "PCRel32Minus1Anon";
+  case PCRel32Minus2Anon:
+    return "PCRel32Minus2Anon";
+  case PCRel32Minus4Anon:
+    return "PCRel32Minus4Anon";
+  case PCRel32GOTLoad:
+    return "PCRel32GOTLoad";
+  case PCRel32GOT:
+    return "PCRel32GOT";
+  case PCRel32TLV:
+    return "PCRel32TLV";
+  case Delta32:
+    return "Delta32";
+  case Delta64:
+    return "Delta64";
+  case NegDelta32:
+    return "NegDelta32";
+  case NegDelta64:
+    return "NegDelta64";
   }
   return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
 }
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
index 2619f31b6d85791ca1cf39ba62ca9974e3778dd9..954423bb7512359e028270979a25a21bf137c180 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
@@ -51,7 +51,7 @@ namespace jitlink {
 
 char JITLinkError::ID = 0;
 
-void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg << "\n"; }
+void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
 
 std::error_code JITLinkError::convertToErrorCode() const {
   return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory);
@@ -309,14 +309,35 @@ Error markAllSymbolsLive(LinkGraph &G) {
   return Error::success();
 }
 
-Error makeTargetOutOfRangeError(const Block &B, const Edge &E,
-                                const char *(*getEdgeKindName)(Edge::Kind)) {
+Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
+                                const Edge &E) {
   std::string ErrMsg;
   {
     raw_string_ostream ErrStream(ErrMsg);
-    ErrStream << "Relocation target out of range: ";
-    printEdge(ErrStream, B, E, getEdgeKindName(E.getKind()));
-    ErrStream << "\n";
+    Section &Sec = B.getSection();
+    ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
+              << ": relocation target ";
+    if (E.getTarget().hasName())
+      ErrStream << "\"" << E.getTarget().getName() << "\" ";
+    ErrStream << "at address " << formatv("{0:x}", E.getTarget().getAddress());
+    ErrStream << " is out of range of " << G.getEdgeKindName(E.getKind())
+              << " fixup at " << formatv("{0:x}", B.getFixupAddress(E)) << " (";
+
+    Symbol *BestSymbolForBlock = nullptr;
+    for (auto *Sym : Sec.symbols())
+      if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
+          (!BestSymbolForBlock ||
+           Sym->getScope() < BestSymbolForBlock->getScope() ||
+           Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
+        BestSymbolForBlock = Sym;
+
+    if (BestSymbolForBlock)
+      ErrStream << BestSymbolForBlock->getName() << ", ";
+    else
+      ErrStream << "<anonymous block> @ ";
+
+    ErrStream << formatv("{0:x}", B.getAddress()) << " + "
+              << formatv("{0:x}", E.getOffset()) << ")";
   }
   return make_error<JITLinkError>(std::move(ErrMsg));
 }
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
index 2e5b7cbc3745f8fdd3c3fbc90226966e7e552b9c..63f862b96325dc8bf4e903ca943dcb79fe31b484 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
@@ -66,14 +66,27 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
     return Ctx->notifyFailed(std::move(Err));
 
   // Notify client that the defined symbols have been assigned addresses.
-  LLVM_DEBUG(
-      { dbgs() << "Resolving symbols defined in " << G->getName() << "\n"; });
+  LLVM_DEBUG(dbgs() << "Resolving symbols defined in " << G->getName() << "\n");
 
   if (auto Err = Ctx->notifyResolved(*G))
     return Ctx->notifyFailed(std::move(Err));
 
   auto ExternalSymbols = getExternalSymbolNames();
 
+  // If there are no external symbols then proceed immediately with phase 2.
+  if (ExternalSymbols.empty()) {
+    LLVM_DEBUG({
+      dbgs() << "No external symbols for " << G->getName()
+             << ". Proceeding immediately with link phase 2.\n";
+    });
+    // FIXME: Once callee expressions are defined to be sequenced before
+    //        argument expressions (c++17) we can simplify this. See below.
+    auto &TmpSelf = *Self;
+    TmpSelf.linkPhase2(std::move(Self), AsyncLookupResult(), std::move(Layout));
+    return;
+  }
+
+  // Otherwise look up the externals.
   LLVM_DEBUG({
     dbgs() << "Issuing lookup for external symbols for " << G->getName()
            << " (may trigger materialization/linking of other graphs)...\n";
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
index c5d30cfb52b475c050b04588eb33ac9074e04d23..ef68f96149ab23ef0b169a8106bec4640d433dd3 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
@@ -51,6 +51,12 @@ protected:
 
   using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>;
 
+  // Returns the PassConfiguration for this instance. This can be used by
+  // JITLinkerBase implementations to add late passes that reference their
+  // own data structures (e.g. for ELF implementations to locate / construct
+  // a GOT start symbol prior to fixup).
+  PassConfiguration &getPassConfig() { return Passes; }
+
   // Phase 1:
   //   1.1: Run pre-prune passes
   //   1.2: Prune graph
@@ -154,7 +160,7 @@ private:
 
         // Dispatch to LinkerImpl for fixup.
         auto *BlockData = const_cast<char *>(B->getContent().data());
-        if (auto Err = impl().applyFixup(*B, E, BlockData))
+        if (auto Err = impl().applyFixup(G, *B, E, BlockData))
           return Err;
       }
     }
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
index fab510e63aaf2d3db182841b7e48906781ce4427..ccf286d8a1443b25f10e999ff5c255a7dc85433f 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
@@ -116,10 +116,6 @@ Error MachOLinkGraphBuilder::createNormalizedSections() {
 
     auto SecIndex = Obj.getSectionIndex(SecRef.getRawDataRefImpl());
 
-    auto Name = SecRef.getName();
-    if (!Name)
-      return Name.takeError();
-
     if (Obj.is64Bit()) {
       const MachO::section_64 &Sec64 =
           Obj.getSection64(SecRef.getRawDataRefImpl());
@@ -150,8 +146,9 @@ Error MachOLinkGraphBuilder::createNormalizedSections() {
     }
 
     LLVM_DEBUG({
-      dbgs() << "  " << *Name << ": " << formatv("{0:x16}", NSec.Address)
-             << " -- " << formatv("{0:x16}", NSec.Address + NSec.Size)
+      dbgs() << "  " << NSec.SegName << "," << NSec.SectName << ": "
+             << formatv("{0:x16}", NSec.Address) << " -- "
+             << formatv("{0:x16}", NSec.Address + NSec.Size)
              << ", align: " << NSec.Alignment << ", index: " << SecIndex
              << "\n";
     });
@@ -182,10 +179,12 @@ Error MachOLinkGraphBuilder::createNormalizedSections() {
                                                        sys::Memory::MF_WRITE);
 
     if (!isDebugSection(NSec))
-      NSec.GraphSection = &G->createSection(*Name, Prot);
+      NSec.GraphSection = &G->createSection(
+          G->allocateString(StringRef(NSec.SegName) + "," + NSec.SectName),
+          Prot);
     else
       LLVM_DEBUG({
-        dbgs() << "    " << *Name
+        dbgs() << "    " << NSec.SegName << "," << NSec.SectName
                << " is a debug section: No graph section will be created.\n";
       });
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
index 729320c6c0df8d397380ef7c5a96a83c0976480d..344334a61720e949b14d92d527e258c80e85d6dc 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -12,8 +12,8 @@
 
 #include "llvm/ExecutionEngine/JITLink/MachO_arm64.h"
 
-#include "BasicGOTAndStubsBuilder.h"
 #include "MachOLinkGraphBuilder.h"
+#include "PerGraphGOTAndPLTStubsBuilder.h"
 
 #define DEBUG_TYPE "jitlink"
 
@@ -405,11 +405,12 @@ private:
   unsigned NumSymbols = 0;
 };
 
-class MachO_arm64_GOTAndStubsBuilder
-    : public BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder> {
+class PerGraphGOTAndPLTStubsBuilder_MachO_arm64
+    : public PerGraphGOTAndPLTStubsBuilder<
+          PerGraphGOTAndPLTStubsBuilder_MachO_arm64> {
 public:
-  MachO_arm64_GOTAndStubsBuilder(LinkGraph &G)
-      : BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder>(G) {}
+  using PerGraphGOTAndPLTStubsBuilder<
+      PerGraphGOTAndPLTStubsBuilder_MachO_arm64>::PerGraphGOTAndPLTStubsBuilder;
 
   bool isGOTEdgeToFix(Edge &E) const {
     return (E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 ||
@@ -439,16 +440,16 @@ public:
     return E.getKind() == Branch26 && !E.getTarget().isDefined();
   }
 
-  Symbol &createStub(Symbol &Target) {
+  Symbol &createPLTStub(Symbol &Target) {
     auto &StubContentBlock =
         G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
     // Re-use GOT entries for stub targets.
-    auto &GOTEntrySymbol = getGOTEntrySymbol(Target);
+    auto &GOTEntrySymbol = getGOTEntry(Target);
     StubContentBlock.addEdge(LDRLiteral19, 0, GOTEntrySymbol, 0);
     return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false);
   }
 
-  void fixExternalBranchEdge(Edge &E, Symbol &Stub) {
+  void fixPLTEdge(Edge &E, Symbol &Stub) {
     assert(E.getKind() == Branch26 && "Not a Branch32 edge?");
     assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?");
     E.setTarget(Stub);
@@ -486,9 +487,10 @@ private:
   Section *StubsSection = nullptr;
 };
 
-const uint8_t MachO_arm64_GOTAndStubsBuilder::NullGOTEntryContent[8] = {
-    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
-const uint8_t MachO_arm64_GOTAndStubsBuilder::StubContent[8] = {
+const uint8_t
+    PerGraphGOTAndPLTStubsBuilder_MachO_arm64::NullGOTEntryContent[8] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+const uint8_t PerGraphGOTAndPLTStubsBuilder_MachO_arm64::StubContent[8] = {
     0x10, 0x00, 0x00, 0x58, // LDR x16, <literal>
     0x00, 0x02, 0x1f, 0xd6  // BR  x16
 };
@@ -509,17 +511,6 @@ public:
 
 private:
 
-  static Error targetOutOfRangeError(const Block &B, const Edge &E) {
-    std::string ErrMsg;
-    {
-      raw_string_ostream ErrStream(ErrMsg);
-      ErrStream << "Relocation target out of range: ";
-      printEdge(ErrStream, B, E, getMachOARM64RelocationKindName(E.getKind()));
-      ErrStream << "\n";
-    }
-    return make_error<JITLinkError>(std::move(ErrMsg));
-  }
-
   static unsigned getPageOffset12Shift(uint32_t Instr) {
     constexpr uint32_t LoadStoreImm12Mask = 0x3b000000;
     constexpr uint32_t Vec128Mask = 0x04800000;
@@ -536,7 +527,8 @@ private:
     return 0;
   }
 
-  Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const {
+  Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
+                   char *BlockWorkingMem) const {
     using namespace support;
 
     char *FixupPtr = BlockWorkingMem + E.getOffset();
@@ -553,7 +545,7 @@ private:
                                         "aligned");
 
       if (Value < -(1 << 27) || Value > ((1 << 27) - 1))
-        return targetOutOfRangeError(B, E);
+        return makeTargetOutOfRangeError(G, B, E);
 
       uint32_t RawInstr = *(little32_t *)FixupPtr;
       assert((RawInstr & 0x7fffffff) == 0x14000000 &&
@@ -566,7 +558,7 @@ private:
     case Pointer32: {
       uint64_t Value = E.getTarget().getAddress() + E.getAddend();
       if (Value > std::numeric_limits<uint32_t>::max())
-        return targetOutOfRangeError(B, E);
+        return makeTargetOutOfRangeError(G, B, E);
       *(ulittle32_t *)FixupPtr = Value;
       break;
     }
@@ -587,7 +579,7 @@ private:
 
       int64_t PageDelta = TargetPage - PCPage;
       if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1))
-        return targetOutOfRangeError(B, E);
+        return makeTargetOutOfRangeError(G, B, E);
 
       uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
       assert((RawInstr & 0xffffffe0) == 0x90000000 &&
@@ -637,7 +629,7 @@ private:
         return make_error<JITLinkError>("LDR literal target is not 32-bit "
                                         "aligned");
       if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1))
-        return targetOutOfRangeError(B, E);
+        return makeTargetOutOfRangeError(G, B, E);
 
       uint32_t EncodedImm = (static_cast<uint32_t>(Delta) >> 2) << 5;
       uint32_t FixedInstr = RawInstr | EncodedImm;
@@ -657,7 +649,7 @@ private:
       if (E.getKind() == Delta32 || E.getKind() == NegDelta32) {
         if (Value < std::numeric_limits<int32_t>::min() ||
             Value > std::numeric_limits<int32_t>::max())
-          return targetOutOfRangeError(B, E);
+          return makeTargetOutOfRangeError(G, B, E);
         *(little32_t *)FixupPtr = Value;
       } else
         *(little64_t *)FixupPtr = Value;
@@ -694,10 +686,8 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G,
       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 
     // Add an in-place GOT/Stubs pass.
-    Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error {
-      MachO_arm64_GOTAndStubsBuilder(G).run();
-      return Error::success();
-    });
+    Config.PostPrunePasses.push_back(
+        PerGraphGOTAndPLTStubsBuilder_MachO_arm64::asPass);
   }
 
   if (auto Err = Ctx->modifyPassConfig(*G, Config))
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
index 058cf9b036c263fdc10d2bdade4a0c4aad213542..bd35f2d358dd872bf940a569470342940e4fac77 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
@@ -13,8 +13,8 @@
 #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h"
 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
 
-#include "BasicGOTAndStubsBuilder.h"
 #include "MachOLinkGraphBuilder.h"
+#include "PerGraphGOTAndPLTStubsBuilder.h"
 
 #define DEBUG_TYPE "jitlink"
 
@@ -347,8 +347,6 @@ private:
           else
             return TargetSymbolOrErr.takeError();
           Addend = *(const little32_t *)FixupContent - 4;
-          // -
-          //   (1 << (*MachORelocKind - MachOPCRel32Minus1));
           Kind = x86_64::Delta32;
           break;
         case MachOPCRel32Anon: {
@@ -414,14 +412,16 @@ private:
   }
 };
 
-class MachO_x86_64_GOTAndStubsBuilder
-    : public BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder> {
+class PerGraphGOTAndPLTStubsBuilder_MachO_x86_64
+    : public PerGraphGOTAndPLTStubsBuilder<
+          PerGraphGOTAndPLTStubsBuilder_MachO_x86_64> {
 public:
   static const uint8_t NullGOTEntryContent[8];
   static const uint8_t StubContent[6];
 
-  MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G)
-      : BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder>(G) {}
+  using PerGraphGOTAndPLTStubsBuilder<
+      PerGraphGOTAndPLTStubsBuilder_MachO_x86_64>::
+      PerGraphGOTAndPLTStubsBuilder;
 
   bool isGOTEdgeToFix(Edge &E) const {
     return E.getKind() == x86_64::RequestGOTAndTransformToDelta32 ||
@@ -456,16 +456,16 @@ public:
     return E.getKind() == x86_64::BranchPCRel32 && E.getTarget().isExternal();
   }
 
-  Symbol &createStub(Symbol &Target) {
+  Symbol &createPLTStub(Symbol &Target) {
     auto &StubContentBlock =
         G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
     // Re-use GOT entries for stub targets.
-    auto &GOTEntrySymbol = getGOTEntrySymbol(Target);
+    auto &GOTEntrySymbol = getGOTEntry(Target);
     StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4);
     return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
   }
 
-  void fixExternalBranchEdge(Edge &E, Symbol &Stub) {
+  void fixPLTEdge(Edge &E, Symbol &Stub) {
     assert(E.getKind() == x86_64::BranchPCRel32 && "Not a Branch32 edge?");
     assert(E.getAddend() == 0 &&
            "BranchPCRel32 edge has unexpected addend value");
@@ -507,9 +507,10 @@ private:
   Section *StubsSection = nullptr;
 };
 
-const uint8_t MachO_x86_64_GOTAndStubsBuilder::NullGOTEntryContent[8] = {
-    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
-const uint8_t MachO_x86_64_GOTAndStubsBuilder::StubContent[6] = {
+const uint8_t
+    PerGraphGOTAndPLTStubsBuilder_MachO_x86_64::NullGOTEntryContent[8] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+const uint8_t PerGraphGOTAndPLTStubsBuilder_MachO_x86_64::StubContent[6] = {
     0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
 } // namespace
 
@@ -556,9 +557,11 @@ static Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) {
         }
       } else if (E.getKind() == x86_64::BranchPCRel32ToPtrJumpStubRelaxable) {
         auto &StubBlock = E.getTarget().getBlock();
-        assert(StubBlock.getSize() ==
-                   sizeof(MachO_x86_64_GOTAndStubsBuilder::StubContent) &&
-               "Stub block should be stub sized");
+        assert(
+            StubBlock.getSize() ==
+                sizeof(
+                    PerGraphGOTAndPLTStubsBuilder_MachO_x86_64::StubContent) &&
+            "Stub block should be stub sized");
         assert(StubBlock.edges_size() == 1 &&
                "Stub block should only have one outgoing edge");
 
@@ -601,9 +604,9 @@ public:
       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}
 
 private:
-
-  Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const {
-    return x86_64::applyFixup(B, E, BlockWorkingMem);
+  Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
+                   char *BlockWorkingMem) const {
+    return x86_64::applyFixup(G, B, E, BlockWorkingMem);
   }
 };
 
@@ -622,10 +625,11 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G,
 
   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
     // Add eh-frame passses.
-    Config.PrePrunePasses.push_back(EHFrameSplitter("__eh_frame"));
+    StringRef EHFrameSectionName = "__TEXT,__eh_frame";
+    Config.PrePrunePasses.push_back(EHFrameSplitter(EHFrameSectionName));
     Config.PrePrunePasses.push_back(
-        EHFrameEdgeFixer("__eh_frame", G->getPointerSize(), x86_64::Delta64,
-                         x86_64::Delta32, x86_64::NegDelta32));
+        EHFrameEdgeFixer(EHFrameSectionName, G->getPointerSize(),
+                         x86_64::Delta64, x86_64::Delta32, x86_64::NegDelta32));
 
     // Add a mark-live pass.
     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
@@ -634,10 +638,8 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G,
       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 
     // Add an in-place GOT/Stubs pass.
-    Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error {
-      MachO_x86_64_GOTAndStubsBuilder(G).run();
-      return Error::success();
-    });
+    Config.PostPrunePasses.push_back(
+        PerGraphGOTAndPLTStubsBuilder_MachO_x86_64::asPass);
 
     // Add GOT/Stubs optimizer pass.
     Config.PreFixupPasses.push_back(optimizeMachO_x86_64_GOTAndStubs);
diff --git a/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h b/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h
new file mode 100644
index 0000000000000000000000000000000000000000..6e9df9c75a652d84c3356ff681f717f704647b2c
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h
@@ -0,0 +1,126 @@
+//===--------------- PerGraphGOTAndPLTStubBuilder.h -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Construct GOT and PLT entries for each graph.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHGOTANDPLTSTUBSBUILDER_H
+#define LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHGOTANDPLTSTUBSBUILDER_H
+
+#include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "jitlink"
+
+namespace llvm {
+namespace jitlink {
+
+/// Per-object GOT and PLT Stub builder.
+///
+/// Constructs GOT entries and PLT stubs in every graph for referenced symbols.
+/// Building these blocks in every graph is likely to lead to duplicate entries
+/// in the JITLinkDylib, but allows graphs to be trivially removed independently
+/// without affecting other graphs (since those other graphs will have their own
+/// copies of any required entries).
+template <typename BuilderImplT>
+class PerGraphGOTAndPLTStubsBuilder {
+public:
+  PerGraphGOTAndPLTStubsBuilder(LinkGraph &G) : G(G) {}
+
+  static Error asPass(LinkGraph &G) { return BuilderImplT(G).run(); }
+
+  Error run() {
+    LLVM_DEBUG(dbgs() << "Running Per-Graph GOT and Stubs builder:\n");
+
+    // We're going to be adding new blocks, but we don't want to iterate over
+    // the new ones, so build a worklist.
+    std::vector<Block *> Worklist(G.blocks().begin(), G.blocks().end());
+
+    for (auto *B : Worklist)
+      for (auto &E : B->edges()) {
+        if (impl().isGOTEdgeToFix(E)) {
+          LLVM_DEBUG({
+            dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind())
+                   << " edge at " << formatv("{0:x}", B->getFixupAddress(E))
+                   << " (" << formatv("{0:x}", B->getAddress()) << " + "
+                   << formatv("{0:x}", E.getOffset()) << ")\n";
+          });
+          impl().fixGOTEdge(E, getGOTEntry(E.getTarget()));
+        } else if (impl().isExternalBranchEdge(E)) {
+          LLVM_DEBUG({
+            dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind())
+                   << " edge at " << formatv("{0:x}", B->getFixupAddress(E))
+                   << " (" << formatv("{0:x}", B->getAddress()) << " + "
+                   << formatv("{0:x}", E.getOffset()) << ")\n";
+          });
+          impl().fixPLTEdge(E, getPLTStub(E.getTarget()));
+        }
+      }
+
+    return Error::success();
+  }
+
+protected:
+  Symbol &getGOTEntry(Symbol &Target) {
+    assert(Target.hasName() && "GOT edge cannot point to anonymous target");
+
+    auto GOTEntryI = GOTEntries.find(Target.getName());
+
+    // Build the entry if it doesn't exist.
+    if (GOTEntryI == GOTEntries.end()) {
+      auto &GOTEntry = impl().createGOTEntry(Target);
+      LLVM_DEBUG({
+        dbgs() << "    Created GOT entry for " << Target.getName() << ": "
+               << GOTEntry << "\n";
+      });
+      GOTEntryI =
+          GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first;
+    }
+
+    assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol");
+    LLVM_DEBUG(
+        { dbgs() << "    Using GOT entry " << *GOTEntryI->second << "\n"; });
+    return *GOTEntryI->second;
+  }
+
+  Symbol &getPLTStub(Symbol &Target) {
+    assert(Target.hasName() &&
+           "External branch edge can not point to an anonymous target");
+    auto StubI = PLTStubs.find(Target.getName());
+
+    if (StubI == PLTStubs.end()) {
+      auto &StubSymbol = impl().createPLTStub(Target);
+      LLVM_DEBUG({
+        dbgs() << "    Created PLT stub for " << Target.getName() << ": "
+               << StubSymbol << "\n";
+      });
+      StubI =
+          PLTStubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first;
+    }
+
+    assert(StubI != PLTStubs.end() && "Count not get stub symbol");
+    LLVM_DEBUG({ dbgs() << "    Using PLT stub " << *StubI->second << "\n"; });
+    return *StubI->second;
+  }
+
+  LinkGraph &G;
+
+private:
+  BuilderImplT &impl() { return static_cast<BuilderImplT &>(*this); }
+
+  DenseMap<StringRef, Symbol *> GOTEntries;
+  DenseMap<StringRef, Symbol *> PLTStubs;
+};
+
+} // end namespace jitlink
+} // end namespace llvm
+
+#undef DEBUG_TYPE
+
+#endif // LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHGOTANDPLTSTUBSBUILDER_H
diff --git a/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
index 2959d37aafec07ad6821fb7068479f77364aa468..36efc744bf30761b5197fd2b3ae033467ad136ec 100644
--- a/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp
@@ -371,16 +371,6 @@ DebugObjectSection *ELFDebugObject::getSection(StringRef Name) {
   return It == Sections.end() ? nullptr : It->second.get();
 }
 
-static ResourceKey getResourceKey(MaterializationResponsibility &MR) {
-  ResourceKey Key;
-  if (auto Err = MR.withResourceKeyDo([&](ResourceKey K) { Key = K; })) {
-    MR.getExecutionSession().reportError(std::move(Err));
-    return ResourceKey{};
-  }
-  assert(Key && "Invalid key");
-  return Key;
-}
-
 /// Creates a debug object based on the input object file from
 /// ObjectLinkingLayerJITLinkContext.
 ///
@@ -407,17 +397,15 @@ DebugObjectManagerPlugin::~DebugObjectManagerPlugin() = default;
 void DebugObjectManagerPlugin::notifyMaterializing(
     MaterializationResponsibility &MR, LinkGraph &G, JITLinkContext &Ctx,
     MemoryBufferRef ObjBuffer) {
-  assert(PendingObjs.count(getResourceKey(MR)) == 0 &&
+  std::lock_guard<std::mutex> Lock(PendingObjsLock);
+  assert(PendingObjs.count(&MR) == 0 &&
          "Cannot have more than one pending debug object per "
          "MaterializationResponsibility");
 
-  std::lock_guard<std::mutex> Lock(PendingObjsLock);
   if (auto DebugObj = createDebugObjectFromBuffer(ES, G, Ctx, ObjBuffer)) {
     // Not all link artifacts allow debugging.
-    if (*DebugObj != nullptr) {
-      ResourceKey Key = getResourceKey(MR);
-      PendingObjs[Key] = std::move(*DebugObj);
-    }
+    if (*DebugObj != nullptr)
+      PendingObjs[&MR] = std::move(*DebugObj);
   } else {
     ES.reportError(DebugObj.takeError());
   }
@@ -428,7 +416,7 @@ void DebugObjectManagerPlugin::modifyPassConfig(
     PassConfiguration &PassConfig) {
   // Not all link artifacts have associated debug objects.
   std::lock_guard<std::mutex> Lock(PendingObjsLock);
-  auto It = PendingObjs.find(getResourceKey(MR));
+  auto It = PendingObjs.find(&MR);
   if (It == PendingObjs.end())
     return;
 
@@ -446,16 +434,11 @@ void DebugObjectManagerPlugin::modifyPassConfig(
 
 Error DebugObjectManagerPlugin::notifyEmitted(
     MaterializationResponsibility &MR) {
-  ResourceKey Key = getResourceKey(MR);
-
   std::lock_guard<std::mutex> Lock(PendingObjsLock);
-  auto It = PendingObjs.find(Key);
+  auto It = PendingObjs.find(&MR);
   if (It == PendingObjs.end())
     return Error::success();
 
-  DebugObject *UnownedDebugObj = It->second.release();
-  PendingObjs.erase(It);
-
   // During finalization the debug object is registered with the target.
   // Materialization must wait for this process to finish. Otherwise we might
   // start running code before the debugger processed the corresponding debug
@@ -463,29 +446,27 @@ Error DebugObjectManagerPlugin::notifyEmitted(
   std::promise<MSVCPError> FinalizePromise;
   std::future<MSVCPError> FinalizeErr = FinalizePromise.get_future();
 
-  // FIXME: We released ownership of the DebugObject, so we can easily capture
-  // the raw pointer in the continuation function, which re-owns it immediately.
-  if (UnownedDebugObj)
-    UnownedDebugObj->finalizeAsync(
-        [this, Key, UnownedDebugObj,
-         &FinalizePromise](Expected<sys::MemoryBlock> TargetMem) {
-          std::unique_ptr<DebugObject> ReownedDebugObj(UnownedDebugObj);
-          if (!TargetMem) {
-            FinalizePromise.set_value(TargetMem.takeError());
-            return;
-          }
-          if (Error Err = Target->registerDebugObject(*TargetMem)) {
-            FinalizePromise.set_value(std::move(Err));
-            return;
-          }
-
-          // Registration successful, notifyEmitted() can return now and
-          // materialization can finish.
-          FinalizePromise.set_value(Error::success());
-
+  It->second->finalizeAsync(
+      [this, &FinalizePromise, &MR](Expected<sys::MemoryBlock> TargetMem) {
+        // Any failure here will fail materialization.
+        if (!TargetMem) {
+          FinalizePromise.set_value(TargetMem.takeError());
+          return;
+        }
+        if (Error Err = Target->registerDebugObject(*TargetMem)) {
+          FinalizePromise.set_value(std::move(Err));
+          return;
+        }
+
+        // Once our tracking info is updated, notifyEmitted() can return and
+        // finish materialization.
+        FinalizePromise.set_value(MR.withResourceKeyDo([&](ResourceKey K) {
+          assert(PendingObjs.count(&MR) && "We still hold PendingObjsLock");
           std::lock_guard<std::mutex> Lock(RegisteredObjsLock);
-          RegisteredObjs[Key].push_back(std::move(ReownedDebugObj));
-        });
+          RegisteredObjs[K].push_back(std::move(PendingObjs[&MR]));
+          PendingObjs.erase(&MR);
+        }));
+      });
 
   return FinalizeErr.get();
 }
@@ -493,45 +474,32 @@ Error DebugObjectManagerPlugin::notifyEmitted(
 Error DebugObjectManagerPlugin::notifyFailed(
     MaterializationResponsibility &MR) {
   std::lock_guard<std::mutex> Lock(PendingObjsLock);
-  PendingObjs.erase(getResourceKey(MR));
+  PendingObjs.erase(&MR);
   return Error::success();
 }
 
 void DebugObjectManagerPlugin::notifyTransferringResources(ResourceKey DstKey,
                                                            ResourceKey SrcKey) {
-  {
-    std::lock_guard<std::mutex> Lock(RegisteredObjsLock);
-    auto SrcIt = RegisteredObjs.find(SrcKey);
-    if (SrcIt != RegisteredObjs.end()) {
-      // Resources from distinct MaterializationResponsibilitys can get merged
-      // after emission, so we can have multiple debug objects per resource key.
-      for (std::unique_ptr<DebugObject> &DebugObj : SrcIt->second)
-        RegisteredObjs[DstKey].push_back(std::move(DebugObj));
-      RegisteredObjs.erase(SrcIt);
-    }
-  }
-  {
-    std::lock_guard<std::mutex> Lock(PendingObjsLock);
-    auto SrcIt = PendingObjs.find(SrcKey);
-    if (SrcIt != PendingObjs.end()) {
-      assert(PendingObjs.count(DstKey) == 0 &&
-             "Cannot have more than one pending debug object per "
-             "MaterializationResponsibility");
-      PendingObjs[DstKey] = std::move(SrcIt->second);
-      PendingObjs.erase(SrcIt);
-    }
+  // Debug objects are stored by ResourceKey only after registration.
+  // Thus, pending objects don't need to be updated here.
+  std::lock_guard<std::mutex> Lock(RegisteredObjsLock);
+  auto SrcIt = RegisteredObjs.find(SrcKey);
+  if (SrcIt != RegisteredObjs.end()) {
+    // Resources from distinct MaterializationResponsibilitys can get merged
+    // after emission, so we can have multiple debug objects per resource key.
+    for (std::unique_ptr<DebugObject> &DebugObj : SrcIt->second)
+      RegisteredObjs[DstKey].push_back(std::move(DebugObj));
+    RegisteredObjs.erase(SrcIt);
   }
 }
 
-Error DebugObjectManagerPlugin::notifyRemovingResources(ResourceKey K) {
-  {
-    std::lock_guard<std::mutex> Lock(RegisteredObjsLock);
-    RegisteredObjs.erase(K);
-    // TODO: Implement unregister notifications.
-  }
-  std::lock_guard<std::mutex> Lock(PendingObjsLock);
-  PendingObjs.erase(K);
+Error DebugObjectManagerPlugin::notifyRemovingResources(ResourceKey Key) {
+  // Removing the resource for a pending object fails materialization, so they
+  // get cleaned up in the notifyFailed() handler.
+  std::lock_guard<std::mutex> Lock(RegisteredObjsLock);
+  RegisteredObjs.erase(Key);
 
+  // TODO: Implement unregister notifications.
   return Error::success();
 }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index b47b5e9d39696c95d467f756c067bfa45c55b571..80df097a07410ef3b92754cfb3624b605bdb8374 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -306,9 +306,12 @@ void MachOPlatform::InitScraperPlugin::modifyPassConfig(
 
   Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error {
     JITLinkSymbolVector InitSectionSymbols;
-    preserveInitSectionIfPresent(InitSectionSymbols, G, "__mod_init_func");
-    preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_selrefs");
-    preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_classlist");
+    preserveInitSectionIfPresent(InitSectionSymbols, G,
+                                 "__DATA,__mod_init_func");
+    preserveInitSectionIfPresent(InitSectionSymbols, G,
+                                 "__DATA,__objc_selrefs");
+    preserveInitSectionIfPresent(InitSectionSymbols, G,
+                                 "__DATA,__objc_classlist");
 
     if (!InitSectionSymbols.empty()) {
       std::lock_guard<std::mutex> Lock(InitScraperMutex);
@@ -327,25 +330,27 @@ void MachOPlatform::InitScraperPlugin::modifyPassConfig(
         ObjCClassList;
 
     JITTargetAddress ObjCImageInfoAddr = 0;
-    if (auto *ObjCImageInfoSec = G.findSectionByName("__objc_image_info")) {
+    if (auto *ObjCImageInfoSec =
+            G.findSectionByName("__DATA,__objc_image_info")) {
       if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart())
         ObjCImageInfoAddr = Addr;
     }
 
     // Record __mod_init_func.
-    if (auto ModInitsOrErr = getSectionExtent(G, "__mod_init_func"))
+    if (auto ModInitsOrErr = getSectionExtent(G, "__DATA,__mod_init_func"))
       ModInits = std::move(*ModInitsOrErr);
     else
       return ModInitsOrErr.takeError();
 
     // Record __objc_selrefs.
-    if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__objc_selrefs"))
+    if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__DATA,__objc_selrefs"))
       ObjCSelRefs = std::move(*ObjCSelRefsOrErr);
     else
       return ObjCSelRefsOrErr.takeError();
 
     // Record __objc_classlist.
-    if (auto ObjCClassListOrErr = getSectionExtent(G, "__objc_classlist"))
+    if (auto ObjCClassListOrErr =
+            getSectionExtent(G, "__DATA,__objc_classlist"))
       ObjCClassList = std::move(*ObjCClassListOrErr);
     else
       return ObjCClassListOrErr.takeError();
diff --git a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
index dfdd2c6c669f980af9172a89f86efd6ebdc4679d..834d4cc8f51455eae814dd0f13871b76d04ee582 100644
--- a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
@@ -393,7 +393,7 @@ void LLVMOrcDisposeJITTargetMachineBuilder(
   delete unwrap(JTMB);
 }
 
-void lLVMOrcDisposeObjectLayer(LLVMOrcObjectLayerRef ObjLayer) {
+void LLVMOrcDisposeObjectLayer(LLVMOrcObjectLayerRef ObjLayer) {
   delete unwrap(ObjLayer);
 }
 
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
index 2fbe707ce8df1b248eeaadf838eb8ca6db3b7640..a884863b63c054362610dd869479790ea2716d85 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
@@ -381,7 +381,9 @@ private:
     RemainingExpr = RemainingExpr.substr(1).ltrim();
 
     StringRef SectionName;
-    std::tie(SectionName, RemainingExpr) = parseSymbol(RemainingExpr);
+    size_t CloseParensIdx = RemainingExpr.find(')');
+    SectionName = RemainingExpr.substr(0, CloseParensIdx).rtrim();
+    RemainingExpr = RemainingExpr.substr(CloseParensIdx).ltrim();
 
     if (!RemainingExpr.startswith(")"))
       return std::make_pair(
diff --git a/llvm/lib/FileCheck/FileCheck.cpp b/llvm/lib/FileCheck/FileCheck.cpp
index 462f507886633d71ea147d0435c3b06413d99fc4..12a0ae63419f5039c0fa8a6ae83f1ae350a9a267 100644
--- a/llvm/lib/FileCheck/FileCheck.cpp
+++ b/llvm/lib/FileCheck/FileCheck.cpp
@@ -476,6 +476,7 @@ char OverflowError::ID = 0;
 char UndefVarError::ID = 0;
 char ErrorDiagnostic::ID = 0;
 char NotFoundError::ID = 0;
+char ErrorReported::ID = 0;
 
 Expected<NumericVariable *> Pattern::parseNumericVariableDefinition(
     StringRef &Expr, FileCheckPatternContext *Context,
@@ -1082,8 +1083,15 @@ bool Pattern::parsePattern(StringRef PatternStr, StringRef Prefix,
           if (IsPseudo) {
             MatchStr = OrigMatchStr;
             IsLegacyLineExpr = IsNumBlock = true;
-          } else
+          } else {
+            if (!MatchStr.empty()) {
+              SM.PrintMessage(SMLoc::getFromPointer(Name.data()),
+                              SourceMgr::DK_Error,
+                              "invalid name in string variable use");
+              return true;
+            }
             SubstStr = Name;
+          }
         }
       }
 
@@ -1212,22 +1220,19 @@ void Pattern::AddBackrefToRegEx(unsigned BackrefNum) {
   RegExStr += Backref;
 }
 
-Expected<size_t> Pattern::match(StringRef Buffer, size_t &MatchLen,
-                                const SourceMgr &SM) const {
+Pattern::MatchResult Pattern::match(StringRef Buffer,
+                                    const SourceMgr &SM) const {
   // If this is the EOF pattern, match it immediately.
-  if (CheckTy == Check::CheckEOF) {
-    MatchLen = 0;
-    return Buffer.size();
-  }
+  if (CheckTy == Check::CheckEOF)
+    return MatchResult(Buffer.size(), 0, Error::success());
 
   // If this is a fixed string pattern, just match it now.
   if (!FixedStr.empty()) {
-    MatchLen = FixedStr.size();
     size_t Pos =
         IgnoreCase ? Buffer.find_lower(FixedStr) : Buffer.find(FixedStr);
     if (Pos == StringRef::npos)
       return make_error<NotFoundError>();
-    return Pos;
+    return MatchResult(Pos, /*MatchLen=*/FixedStr.size(), Error::success());
   }
 
   // Regex match.
@@ -1250,7 +1255,7 @@ Expected<size_t> Pattern::match(StringRef Buffer, size_t &MatchLen,
       Expected<std::string> Value = Substitution->getResult();
       if (!Value) {
         // Convert to an ErrorDiagnostic to get location information. This is
-        // done here rather than PrintNoMatch since now we know which
+        // done here rather than printMatch/printNoMatch since now we know which
         // substitution block caused the overflow.
         Error Err =
             handleErrors(Value.takeError(), [&](const OverflowError &E) {
@@ -1289,6 +1294,14 @@ Expected<size_t> Pattern::match(StringRef Buffer, size_t &MatchLen,
         MatchInfo[VariableDef.second];
   }
 
+  // Like CHECK-NEXT, CHECK-EMPTY's match range is considered to start after
+  // the required preceding newline, which is consumed by the pattern in the
+  // case of CHECK-EMPTY but not CHECK-NEXT.
+  size_t MatchStartSkip = CheckTy == Check::CheckEmpty;
+  Match TheMatch;
+  TheMatch.Pos = FullMatch.data() - Buffer.data() + MatchStartSkip;
+  TheMatch.Len = FullMatch.size() - MatchStartSkip;
+
   // If this defines any numeric variables, remember their values.
   for (const auto &NumericVariableDef : NumericVariableDefs) {
     const NumericVariableMatch &NumericVariableMatch =
@@ -1303,16 +1316,11 @@ Expected<size_t> Pattern::match(StringRef Buffer, size_t &MatchLen,
     Expected<ExpressionValue> Value =
         Format.valueFromStringRepr(MatchedValue, SM);
     if (!Value)
-      return Value.takeError();
+      return MatchResult(TheMatch, Value.takeError());
     DefinedNumericVariable->setValue(*Value, MatchedValue);
   }
 
-  // Like CHECK-NEXT, CHECK-EMPTY's match range is considered to start after
-  // the required preceding newline, which is consumed by the pattern in the
-  // case of CHECK-EMPTY but not CHECK-NEXT.
-  size_t MatchStartSkip = CheckTy == Check::CheckEmpty;
-  MatchLen = FullMatch.size() - MatchStartSkip;
-  return FullMatch.data() - Buffer.data() + MatchStartSkip;
+  return MatchResult(TheMatch, Error::success());
 }
 
 unsigned Pattern::computeMatchDistance(StringRef Buffer) const {
@@ -1349,7 +1357,7 @@ void Pattern::printSubstitutions(const SourceMgr &SM, StringRef Buffer,
         bool UndefSeen = false;
         handleAllErrors(
             MatchedValue.takeError(), [](const NotFoundError &E) {},
-            // Handled in PrintNoMatch().
+            // Handled in printMatch and printNoMatch().
             [](const ErrorDiagnostic &E) {},
             // Handled in match().
             [](const OverflowError &E) {},
@@ -1361,6 +1369,8 @@ void Pattern::printSubstitutions(const SourceMgr &SM, StringRef Buffer,
               OS << " ";
               E.log(OS);
             });
+        if (!OS.tell())
+          continue;
       } else {
         // Substitution succeeded. Print substituted value.
         OS << "with \"";
@@ -1404,11 +1414,12 @@ void Pattern::printVariableDefs(const SourceMgr &SM,
   for (const auto &VariableDef : NumericVariableDefs) {
     VarCapture VC;
     VC.Name = VariableDef.getKey();
-    StringRef StrValue = VariableDef.getValue()
-                             .DefinedNumericVariable->getStringValue()
-                             .getValue();
-    SMLoc Start = SMLoc::getFromPointer(StrValue.data());
-    SMLoc End = SMLoc::getFromPointer(StrValue.data() + StrValue.size());
+    Optional<StringRef> StrValue =
+        VariableDef.getValue().DefinedNumericVariable->getStringValue();
+    if (!StrValue)
+      continue;
+    SMLoc Start = SMLoc::getFromPointer(StrValue->data());
+    SMLoc End = SMLoc::getFromPointer(StrValue->data() + StrValue->size());
     VC.Range = SMRange(Start, End);
     VarCaptures.push_back(VC);
   }
@@ -2036,123 +2047,179 @@ bool FileCheck::readCheckFile(
   return false;
 }
 
-static void PrintMatch(bool ExpectedMatch, const SourceMgr &SM,
-                       StringRef Prefix, SMLoc Loc, const Pattern &Pat,
-                       int MatchedCount, StringRef Buffer, size_t MatchPos,
-                       size_t MatchLen, const FileCheckRequest &Req,
-                       std::vector<FileCheckDiag> *Diags) {
+/// Returns either (1) \c ErrorSuccess if there was no error or (2)
+/// \c ErrorReported if an error was reported, such as an unexpected match.
+static Error printMatch(bool ExpectedMatch, const SourceMgr &SM,
+                        StringRef Prefix, SMLoc Loc, const Pattern &Pat,
+                        int MatchedCount, StringRef Buffer,
+                        Pattern::MatchResult MatchResult,
+                        const FileCheckRequest &Req,
+                        std::vector<FileCheckDiag> *Diags) {
+  // Suppress some verbosity if there's no error.
+  bool HasError = !ExpectedMatch || MatchResult.TheError;
   bool PrintDiag = true;
-  if (ExpectedMatch) {
+  if (!HasError) {
     if (!Req.Verbose)
-      return;
+      return ErrorReported::reportedOrSuccess(HasError);
     if (!Req.VerboseVerbose && Pat.getCheckTy() == Check::CheckEOF)
-      return;
+      return ErrorReported::reportedOrSuccess(HasError);
     // Due to their verbosity, we don't print verbose diagnostics here if we're
-    // gathering them for a different rendering, but we always print other
-    // diagnostics.
+    // gathering them for Diags to be rendered elsewhere, but we always print
+    // other diagnostics.
     PrintDiag = !Diags;
   }
+
+  // Add "found" diagnostic, substitutions, and variable definitions to Diags.
   FileCheckDiag::MatchType MatchTy = ExpectedMatch
                                          ? FileCheckDiag::MatchFoundAndExpected
                                          : FileCheckDiag::MatchFoundButExcluded;
   SMRange MatchRange = ProcessMatchResult(MatchTy, SM, Loc, Pat.getCheckTy(),
-                                          Buffer, MatchPos, MatchLen, Diags);
+                                          Buffer, MatchResult.TheMatch->Pos,
+                                          MatchResult.TheMatch->Len, Diags);
   if (Diags) {
     Pat.printSubstitutions(SM, Buffer, MatchRange, MatchTy, Diags);
     Pat.printVariableDefs(SM, MatchTy, Diags);
   }
-  if (!PrintDiag)
-    return;
+  if (!PrintDiag) {
+    assert(!HasError && "expected to report more diagnostics for error");
+    return ErrorReported::reportedOrSuccess(HasError);
+  }
 
+  // Print the match.
   std::string Message = formatv("{0}: {1} string found in input",
                                 Pat.getCheckTy().getDescription(Prefix),
                                 (ExpectedMatch ? "expected" : "excluded"))
                             .str();
   if (Pat.getCount() > 1)
     Message += formatv(" ({0} out of {1})", MatchedCount, Pat.getCount()).str();
-
   SM.PrintMessage(
       Loc, ExpectedMatch ? SourceMgr::DK_Remark : SourceMgr::DK_Error, Message);
   SM.PrintMessage(MatchRange.Start, SourceMgr::DK_Note, "found here",
                   {MatchRange});
+
+  // Print additional information, which can be useful even if there are errors.
   Pat.printSubstitutions(SM, Buffer, MatchRange, MatchTy, nullptr);
   Pat.printVariableDefs(SM, MatchTy, nullptr);
-}
-
-static void PrintMatch(bool ExpectedMatch, const SourceMgr &SM,
-                       const FileCheckString &CheckStr, int MatchedCount,
-                       StringRef Buffer, size_t MatchPos, size_t MatchLen,
-                       FileCheckRequest &Req,
-                       std::vector<FileCheckDiag> *Diags) {
-  PrintMatch(ExpectedMatch, SM, CheckStr.Prefix, CheckStr.Loc, CheckStr.Pat,
-             MatchedCount, Buffer, MatchPos, MatchLen, Req, Diags);
-}
 
-static void PrintNoMatch(bool ExpectedMatch, const SourceMgr &SM,
-                         StringRef Prefix, SMLoc Loc, const Pattern &Pat,
-                         int MatchedCount, StringRef Buffer,
-                         bool VerboseVerbose, std::vector<FileCheckDiag> *Diags,
-                         Error MatchErrors) {
-  assert(MatchErrors && "Called on successful match");
+  // Print errors and add them to Diags.  We report these errors after the match
+  // itself because we found them after the match.  If we had found them before
+  // the match, we'd be in printNoMatch.
+  handleAllErrors(std::move(MatchResult.TheError),
+                  [&](const ErrorDiagnostic &E) {
+                    E.log(errs());
+                    if (Diags) {
+                      Diags->emplace_back(SM, Pat.getCheckTy(), Loc,
+                                          FileCheckDiag::MatchFoundErrorNote,
+                                          E.getRange(), E.getMessage().str());
+                    }
+                  });
+  return ErrorReported::reportedOrSuccess(HasError);
+}
+
+/// Returns either (1) \c ErrorSuccess if there was no error, or (2)
+/// \c ErrorReported if an error was reported, such as an expected match not
+/// found.
+static Error printNoMatch(bool ExpectedMatch, const SourceMgr &SM,
+                          StringRef Prefix, SMLoc Loc, const Pattern &Pat,
+                          int MatchedCount, StringRef Buffer, Error MatchError,
+                          bool VerboseVerbose,
+                          std::vector<FileCheckDiag> *Diags) {
+  // Print any pattern errors, and record them to be added to Diags later.
+  bool HasError = ExpectedMatch;
+  bool HasPatternError = false;
+  FileCheckDiag::MatchType MatchTy = ExpectedMatch
+                                         ? FileCheckDiag::MatchNoneButExpected
+                                         : FileCheckDiag::MatchNoneAndExcluded;
+  SmallVector<std::string, 4> ErrorMsgs;
+  handleAllErrors(
+      std::move(MatchError),
+      [&](const ErrorDiagnostic &E) {
+        HasError = HasPatternError = true;
+        MatchTy = FileCheckDiag::MatchNoneForInvalidPattern;
+        E.log(errs());
+        if (Diags)
+          ErrorMsgs.push_back(E.getMessage().str());
+      },
+      // UndefVarError is reported in printSubstitutions below.
+      // FIXME: It probably should be handled as a pattern error and actually
+      // change the exit status to 1, even if !ExpectedMatch.  To do so, we
+      // could stop calling printSubstitutions and actually report the error
+      // here as we do ErrorDiagnostic above.
+      [](const UndefVarError &E) {},
+      // NotFoundError is why printNoMatch was invoked.
+      [](const NotFoundError &E) {});
+
+  // Suppress some verbosity if there's no error.
   bool PrintDiag = true;
-  if (!ExpectedMatch) {
-    if (!VerboseVerbose) {
-      consumeError(std::move(MatchErrors));
-      return;
-    }
+  if (!HasError) {
+    if (!VerboseVerbose)
+      return ErrorReported::reportedOrSuccess(HasError);
     // Due to their verbosity, we don't print verbose diagnostics here if we're
-    // gathering them for a different rendering, but we always print other
-    // diagnostics.
+    // gathering them for Diags to be rendered elsewhere, but we always print
+    // other diagnostics.
     PrintDiag = !Diags;
   }
 
-  FileCheckDiag::MatchType MatchTy = ExpectedMatch
-                                         ? FileCheckDiag::MatchNoneButExpected
-                                         : FileCheckDiag::MatchNoneAndExcluded;
+  // Add "not found" diagnostic, substitutions, and pattern errors to Diags.
+  //
+  // We handle Diags a little differently than the errors we print directly:
+  // we add the "not found" diagnostic to Diags even if there are pattern
+  // errors.  The reason is that we need to attach pattern errors as notes
+  // somewhere in the input, and the input search range from the "not found"
+  // diagnostic is all we have to anchor them.
   SMRange SearchRange = ProcessMatchResult(MatchTy, SM, Loc, Pat.getCheckTy(),
                                            Buffer, 0, Buffer.size(), Diags);
-  if (Diags)
+  if (Diags) {
+    SMRange NoteRange = SMRange(SearchRange.Start, SearchRange.Start);
+    for (StringRef ErrorMsg : ErrorMsgs)
+      Diags->emplace_back(SM, Pat.getCheckTy(), Loc, MatchTy, NoteRange,
+                          ErrorMsg);
     Pat.printSubstitutions(SM, Buffer, SearchRange, MatchTy, Diags);
-  if (!PrintDiag) {
-    consumeError(std::move(MatchErrors));
-    return;
   }
-
-  MatchErrors = handleErrors(std::move(MatchErrors),
-                             [](const ErrorDiagnostic &E) { E.log(errs()); });
-
-  // No problem matching the string per se.
-  if (!MatchErrors)
-    return;
-  consumeError(std::move(MatchErrors));
-
-  // Print "not found" diagnostic.
-  std::string Message = formatv("{0}: {1} string not found in input",
-                                Pat.getCheckTy().getDescription(Prefix),
-                                (ExpectedMatch ? "expected" : "excluded"))
-                            .str();
-  if (Pat.getCount() > 1)
-    Message += formatv(" ({0} out of {1})", MatchedCount, Pat.getCount()).str();
-  SM.PrintMessage(
-      Loc, ExpectedMatch ? SourceMgr::DK_Error : SourceMgr::DK_Remark, Message);
-
-  // Print the "scanning from here" line.
-  SM.PrintMessage(SearchRange.Start, SourceMgr::DK_Note, "scanning from here");
-
-  // Allow the pattern to print additional information if desired.
+  if (!PrintDiag) {
+    assert(!HasError && "expected to report more diagnostics for error");
+    return ErrorReported::reportedOrSuccess(HasError);
+  }
+
+  // Print "not found" diagnostic, except that's implied if we already printed a
+  // pattern error.
+  if (!HasPatternError) {
+    std::string Message = formatv("{0}: {1} string not found in input",
+                                  Pat.getCheckTy().getDescription(Prefix),
+                                  (ExpectedMatch ? "expected" : "excluded"))
+                              .str();
+    if (Pat.getCount() > 1)
+      Message +=
+          formatv(" ({0} out of {1})", MatchedCount, Pat.getCount()).str();
+    SM.PrintMessage(Loc,
+                    ExpectedMatch ? SourceMgr::DK_Error : SourceMgr::DK_Remark,
+                    Message);
+    SM.PrintMessage(SearchRange.Start, SourceMgr::DK_Note,
+                    "scanning from here");
+  }
+
+  // Print additional information, which can be useful even after a pattern
+  // error.
   Pat.printSubstitutions(SM, Buffer, SearchRange, MatchTy, nullptr);
-
   if (ExpectedMatch)
     Pat.printFuzzyMatch(SM, Buffer, Diags);
+  return ErrorReported::reportedOrSuccess(HasError);
 }
 
-static void PrintNoMatch(bool ExpectedMatch, const SourceMgr &SM,
-                         const FileCheckString &CheckStr, int MatchedCount,
-                         StringRef Buffer, bool VerboseVerbose,
-                         std::vector<FileCheckDiag> *Diags, Error MatchErrors) {
-  PrintNoMatch(ExpectedMatch, SM, CheckStr.Prefix, CheckStr.Loc, CheckStr.Pat,
-               MatchedCount, Buffer, VerboseVerbose, Diags,
-               std::move(MatchErrors));
+/// Returns either (1) \c ErrorSuccess if there was no error, or (2)
+/// \c ErrorReported if an error was reported.
+static Error reportMatchResult(bool ExpectedMatch, const SourceMgr &SM,
+                               StringRef Prefix, SMLoc Loc, const Pattern &Pat,
+                               int MatchedCount, StringRef Buffer,
+                               Pattern::MatchResult MatchResult,
+                               const FileCheckRequest &Req,
+                               std::vector<FileCheckDiag> *Diags) {
+  if (MatchResult.TheMatch)
+    return printMatch(ExpectedMatch, SM, Prefix, Loc, Pat, MatchedCount, Buffer,
+                      std::move(MatchResult), Req, Diags);
+  return printNoMatch(ExpectedMatch, SM, Prefix, Loc, Pat, MatchedCount, Buffer,
+                      std::move(MatchResult.TheError), Req.VerboseVerbose,
+                      Diags);
 }
 
 /// Counts the number of newlines in the specified range.
@@ -2204,24 +2271,23 @@ size_t FileCheckString::Check(const SourceMgr &SM, StringRef Buffer,
   assert(Pat.getCount() != 0 && "pattern count can not be zero");
   for (int i = 1; i <= Pat.getCount(); i++) {
     StringRef MatchBuffer = Buffer.substr(LastMatchEnd);
-    size_t CurrentMatchLen;
     // get a match at current start point
-    Expected<size_t> MatchResult = Pat.match(MatchBuffer, CurrentMatchLen, SM);
+    Pattern::MatchResult MatchResult = Pat.match(MatchBuffer, SM);
 
     // report
-    if (!MatchResult) {
-      PrintNoMatch(true, SM, *this, i, MatchBuffer, Req.VerboseVerbose, Diags,
-                   MatchResult.takeError());
+    if (Error Err = reportMatchResult(/*ExpectedMatch=*/true, SM, Prefix, Loc,
+                                      Pat, i, MatchBuffer,
+                                      std::move(MatchResult), Req, Diags)) {
+      cantFail(handleErrors(std::move(Err), [&](const ErrorReported &E) {}));
       return StringRef::npos;
     }
-    size_t MatchPos = *MatchResult;
-    PrintMatch(true, SM, *this, i, MatchBuffer, MatchPos, CurrentMatchLen, Req,
-               Diags);
+
+    size_t MatchPos = MatchResult.TheMatch->Pos;
     if (i == 1)
       FirstMatchPos = LastPos + MatchPos;
 
     // move start point after the match
-    LastMatchEnd += MatchPos + CurrentMatchLen;
+    LastMatchEnd += MatchPos + MatchResult.TheMatch->Len;
   }
   // Full match len counts from first match pos.
   MatchLen = LastMatchEnd - FirstMatchPos;
@@ -2328,22 +2394,15 @@ bool FileCheckString::CheckNot(const SourceMgr &SM, StringRef Buffer,
   bool DirectiveFail = false;
   for (const Pattern *Pat : NotStrings) {
     assert((Pat->getCheckTy() == Check::CheckNot) && "Expect CHECK-NOT!");
-
-    size_t MatchLen = 0;
-    Expected<size_t> MatchResult = Pat->match(Buffer, MatchLen, SM);
-
-    if (!MatchResult) {
-      PrintNoMatch(false, SM, Prefix, Pat->getLoc(), *Pat, 1, Buffer,
-                   Req.VerboseVerbose, Diags, MatchResult.takeError());
+    Pattern::MatchResult MatchResult = Pat->match(Buffer, SM);
+    if (Error Err = reportMatchResult(/*ExpectedMatch=*/false, SM, Prefix,
+                                      Pat->getLoc(), *Pat, 1, Buffer,
+                                      std::move(MatchResult), Req, Diags)) {
+      cantFail(handleErrors(std::move(Err), [&](const ErrorReported &E) {}));
+      DirectiveFail = true;
       continue;
     }
-    size_t Pos = *MatchResult;
-
-    PrintMatch(false, SM, Prefix, Pat->getLoc(), *Pat, 1, Buffer, Pos, MatchLen,
-               Req, Diags);
-    DirectiveFail = true;
   }
-
   return DirectiveFail;
 }
 
@@ -2389,20 +2448,22 @@ size_t FileCheckString::CheckDag(const SourceMgr &SM, StringRef Buffer,
     // CHECK-DAG group.
     for (auto MI = MatchRanges.begin(), ME = MatchRanges.end(); true; ++MI) {
       StringRef MatchBuffer = Buffer.substr(MatchPos);
-      Expected<size_t> MatchResult = Pat.match(MatchBuffer, MatchLen, SM);
+      Pattern::MatchResult MatchResult = Pat.match(MatchBuffer, SM);
       // With a group of CHECK-DAGs, a single mismatching means the match on
       // that group of CHECK-DAGs fails immediately.
-      if (!MatchResult) {
-        PrintNoMatch(true, SM, Prefix, Pat.getLoc(), Pat, 1, MatchBuffer,
-                     Req.VerboseVerbose, Diags, MatchResult.takeError());
-        return StringRef::npos;
+      if (MatchResult.TheError || Req.VerboseVerbose) {
+        if (Error Err = reportMatchResult(/*ExpectedMatch=*/true, SM, Prefix,
+                                          Pat.getLoc(), Pat, 1, MatchBuffer,
+                                          std::move(MatchResult), Req, Diags)) {
+          cantFail(
+              handleErrors(std::move(Err), [&](const ErrorReported &E) {}));
+          return StringRef::npos;
+        }
       }
-      size_t MatchPosBuf = *MatchResult;
-      // Re-calc it as the offset relative to the start of the original string.
-      MatchPos += MatchPosBuf;
-      if (Req.VerboseVerbose)
-        PrintMatch(true, SM, Prefix, Pat.getLoc(), Pat, 1, Buffer, MatchPos,
-                   MatchLen, Req, Diags);
+      MatchLen = MatchResult.TheMatch->Len;
+      // Re-calc it as the offset relative to the start of the original
+      // string.
+      MatchPos += MatchResult.TheMatch->Pos;
       MatchRange M{MatchPos, MatchPos + MatchLen};
       if (Req.AllowDeprecatedDagOverlap) {
         // We don't need to track all matches in this mode, so we just maintain
@@ -2453,8 +2514,10 @@ size_t FileCheckString::CheckDag(const SourceMgr &SM, StringRef Buffer,
       MatchPos = MI->End;
     }
     if (!Req.VerboseVerbose)
-      PrintMatch(true, SM, Prefix, Pat.getLoc(), Pat, 1, Buffer, MatchPos,
-                 MatchLen, Req, Diags);
+      cantFail(printMatch(
+          /*ExpectedMatch=*/true, SM, Prefix, Pat.getLoc(), Pat, 1, Buffer,
+          Pattern::MatchResult(MatchPos, MatchLen, Error::success()), Req,
+          Diags));
 
     // Handle the end of a CHECK-DAG group.
     if (std::next(PatItr) == PatEnd ||
diff --git a/llvm/lib/FileCheck/FileCheckImpl.h b/llvm/lib/FileCheck/FileCheckImpl.h
index 1966b0e7831cd57f4ea53e1eebbb541564f0b647..d4d891f7c1f0de3da554a840eccb4f8b79592dee 100644
--- a/llvm/lib/FileCheck/FileCheckImpl.h
+++ b/llvm/lib/FileCheck/FileCheckImpl.h
@@ -537,11 +537,13 @@ private:
 class ErrorDiagnostic : public ErrorInfo<ErrorDiagnostic> {
 private:
   SMDiagnostic Diagnostic;
+  SMRange Range;
 
 public:
   static char ID;
 
-  ErrorDiagnostic(SMDiagnostic &&Diag) : Diagnostic(Diag) {}
+  ErrorDiagnostic(SMDiagnostic &&Diag, SMRange Range)
+      : Diagnostic(Diag), Range(Range) {}
 
   std::error_code convertToErrorCode() const override {
     return inconvertibleErrorCode();
@@ -550,13 +552,19 @@ public:
   /// Print diagnostic associated with this error when printing the error.
   void log(raw_ostream &OS) const override { Diagnostic.print(nullptr, OS); }
 
-  static Error get(const SourceMgr &SM, SMLoc Loc, const Twine &ErrMsg) {
+  StringRef getMessage() const { return Diagnostic.getMessage(); }
+  SMRange getRange() const { return Range; }
+
+  static Error get(const SourceMgr &SM, SMLoc Loc, const Twine &ErrMsg,
+                   SMRange Range = None) {
     return make_error<ErrorDiagnostic>(
-        SM.GetMessage(Loc, SourceMgr::DK_Error, ErrMsg));
+        SM.GetMessage(Loc, SourceMgr::DK_Error, ErrMsg), Range);
   }
 
   static Error get(const SourceMgr &SM, StringRef Buffer, const Twine &ErrMsg) {
-    return get(SM, SMLoc::getFromPointer(Buffer.data()), ErrMsg);
+    SMLoc Start = SMLoc::getFromPointer(Buffer.data());
+    SMLoc End = SMLoc::getFromPointer(Buffer.data() + Buffer.size());
+    return get(SM, Start, ErrMsg, SMRange(Start, End));
   }
 };
 
@@ -574,6 +582,36 @@ public:
   }
 };
 
+/// An error that has already been reported.
+///
+/// This class is designed to support a function whose callers may need to know
+/// whether the function encountered and reported an error but never need to
+/// know the nature of that error.  For example, the function has a return type
+/// of \c Error and always returns either \c ErrorReported or \c ErrorSuccess.
+/// That interface is similar to that of a function returning bool to indicate
+/// an error except, in the former case, (1) there is no confusion over polarity
+/// and (2) the caller must either check the result or explicitly ignore it with
+/// a call like \c consumeError.
+class ErrorReported final : public ErrorInfo<ErrorReported> {
+public:
+  static char ID;
+
+  std::error_code convertToErrorCode() const override {
+    return inconvertibleErrorCode();
+  }
+
+  /// Print diagnostic associated with this error when printing the error.
+  void log(raw_ostream &OS) const override {
+    OS << "error previously reported";
+  }
+
+  static inline Error reportedOrSuccess(bool HasErrorReported) {
+    if (HasErrorReported)
+      return make_error<ErrorReported>();
+    return Error::success();
+  }
+};
+
 class Pattern {
   SMLoc PatternLoc;
 
@@ -694,11 +732,22 @@ public:
   /// \returns true in case of an error, false otherwise.
   bool parsePattern(StringRef PatternStr, StringRef Prefix, SourceMgr &SM,
                     const FileCheckRequest &Req);
-  /// Matches the pattern string against the input buffer \p Buffer
+  struct Match {
+    size_t Pos;
+    size_t Len;
+  };
+  struct MatchResult {
+    Optional<Match> TheMatch;
+    Error TheError;
+    MatchResult(size_t MatchPos, size_t MatchLen, Error E)
+        : TheMatch(Match{MatchPos, MatchLen}), TheError(std::move(E)) {}
+    MatchResult(Match M, Error E) : TheMatch(M), TheError(std::move(E)) {}
+    MatchResult(Error E) : TheError(std::move(E)) {}
+  };
+  /// Matches the pattern string against the input buffer \p Buffer.
   ///
-  /// \returns the position that is matched or an error indicating why matching
-  /// failed. If there is a match, updates \p MatchLen with the size of the
-  /// matched string.
+  /// \returns either (1) an error resulting in no match or (2) a match possibly
+  /// with an error encountered while processing the match.
   ///
   /// The GlobalVariableTable StringMap in the FileCheckPatternContext class
   /// instance provides the current values of FileCheck string variables and is
@@ -706,8 +755,7 @@ public:
   /// GlobalNumericVariableTable StringMap in the same class provides the
   /// current values of FileCheck numeric variables and is updated if this
   /// match defines new numeric values.
-  Expected<size_t> match(StringRef Buffer, size_t &MatchLen,
-                         const SourceMgr &SM) const;
+  MatchResult match(StringRef Buffer, const SourceMgr &SM) const;
   /// Prints the value of successful substitutions or the name of the undefined
   /// string or numeric variables preventing a successful substitution.
   void printSubstitutions(const SourceMgr &SM, StringRef Buffer,
diff --git a/llvm/lib/Frontend/OpenACC/ACC.cpp b/llvm/lib/Frontend/OpenACC/ACC.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1e0516021bc6d2cf48823878a5089557ffa0c8c8
--- /dev/null
+++ b/llvm/lib/Frontend/OpenACC/ACC.cpp
@@ -0,0 +1,19 @@
+//===- ACC.cpp ------ Collection of helpers for OpenACC -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/OpenACC/ACC.h.inc"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+using namespace acc;
+
+#define GEN_DIRECTIVES_IMPL
+#include "llvm/Frontend/OpenACC/ACC.inc"
diff --git a/llvm/lib/Frontend/OpenACC/CMakeLists.txt b/llvm/lib/Frontend/OpenACC/CMakeLists.txt
index ba340ab9c5619a62271aa3acc9bc533011c9bef9..f35201497869058d430ad258079f830c107515ce 100644
--- a/llvm/lib/Frontend/OpenACC/CMakeLists.txt
+++ b/llvm/lib/Frontend/OpenACC/CMakeLists.txt
@@ -1,9 +1,5 @@
-set(LLVM_TARGET_DEFINITIONS ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend/OpenACC/ACC.td)
-tablegen(LLVM ACC.cpp --gen-directive-impl)
-add_public_tablegen_target(acc_cpp)
-
 add_llvm_component_library(LLVMFrontendOpenACC
-  ACC.cpp # Generated by tablegen above
+  ACC.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend
@@ -11,7 +7,6 @@ add_llvm_component_library(LLVMFrontendOpenACC
 
   DEPENDS
   acc_gen
-  acc_cpp
 )
 
 target_link_libraries(LLVMFrontendOpenACC LLVMSupport)
diff --git a/llvm/lib/Frontend/OpenMP/CMakeLists.txt b/llvm/lib/Frontend/OpenMP/CMakeLists.txt
index cc5d3a7668706b74f9fee60bdcb04e42b9f16499..17ca2c63327cc1a85d895bc78ac86009f81d1eb6 100644
--- a/llvm/lib/Frontend/OpenMP/CMakeLists.txt
+++ b/llvm/lib/Frontend/OpenMP/CMakeLists.txt
@@ -1,9 +1,5 @@
-set(LLVM_TARGET_DEFINITIONS ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend/OpenMP/OMP.td)
-tablegen(LLVM OMP.cpp --gen-directive-impl)
-add_public_tablegen_target(omp_cpp)
-
 add_llvm_component_library(LLVMFrontendOpenMP
-  OMP.cpp # Generated by tablegen above
+  OMP.cpp
   OMPContext.cpp
   OMPIRBuilder.cpp
 
@@ -14,7 +10,6 @@ add_llvm_component_library(LLVMFrontendOpenMP
   DEPENDS
   intrinsics_gen
   omp_gen
-  omp_cpp
 
   LINK_COMPONENTS
   Core
diff --git a/llvm/lib/Frontend/OpenMP/OMP.cpp b/llvm/lib/Frontend/OpenMP/OMP.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..eac5d4168c5ca433c93dc6b7678ab549a5091fc8
--- /dev/null
+++ b/llvm/lib/Frontend/OpenMP/OMP.cpp
@@ -0,0 +1,19 @@
+//===- OMP.cpp ------ Collection of helpers for OpenMP --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+using namespace omp;
+
+#define GEN_DIRECTIVES_IMPL
+#include "llvm/Frontend/OpenMP/OMP.inc"
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 0827298654d58f309f3524314b20cde270aff6bc..26f5901d3440c6b5142a48c410638041f65fb610 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -41,15 +41,27 @@ static cl::opt<bool>
 void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
   LLVMContext &Ctx = Fn.getContext();
 
+  // Get the function's current attributes.
+  auto Attrs = Fn.getAttributes();
+  auto FnAttrs = Attrs.getFnAttributes();
+  auto RetAttrs = Attrs.getRetAttributes();
+  SmallVector<AttributeSet, 4> ArgAttrs;
+  for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
+    ArgAttrs.emplace_back(Attrs.getParamAttributes(ArgNo));
+
 #define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
 
-  // Add attributes to the new declaration.
+  // Add attributes to the function declaration.
   switch (FnID) {
 #define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets)                \
   case Enum:                                                                   \
-    Fn.setAttributes(                                                          \
-        AttributeList::get(Ctx, FnAttrSet, RetAttrSet, ArgAttrSets));          \
+    FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet);                           \
+    RetAttrs = RetAttrs.addAttributes(Ctx, RetAttrSet);                        \
+    for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo)                \
+      ArgAttrs[ArgNo] =                                                        \
+          ArgAttrs[ArgNo].addAttributes(Ctx, ArgAttrSets[ArgNo]);              \
+    Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs));    \
     break;
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
   default:
diff --git a/llvm/lib/FuzzMutate/FuzzerCLI.cpp b/llvm/lib/FuzzMutate/FuzzerCLI.cpp
index be0d5bfcab46db7ac601bd7ca04c4e1ca0f7a9f3..1527062cfd97441d49d080c50b315342d2af49b7 100644
--- a/llvm/lib/FuzzMutate/FuzzerCLI.cpp
+++ b/llvm/lib/FuzzMutate/FuzzerCLI.cpp
@@ -153,7 +153,7 @@ int llvm::runFuzzerOnInputs(int ArgC, char *ArgV[], FuzzerTestFun TestOne,
       continue;
     }
 
-    auto BufOrErr = MemoryBuffer::getFile(Arg, /*FileSize-*/ -1,
+    auto BufOrErr = MemoryBuffer::getFile(Arg, /*IsText=*/false,
                                           /*RequiresNullTerminator=*/false);
     if (std::error_code EC = BufOrErr.getError()) {
       errs() << "Error reading file: " << Arg << ": " << EC.message() << "\n";
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 09f21d26971d44cb1095ea81cf04cb023fbdeab4..91f8939a8ffa0dd52afbc6ee1b0e7884523e113a 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -4413,20 +4413,18 @@ void AssemblyWriter::writeAttribute(const Attribute &Attr, bool InAttrGroup) {
     return;
   }
 
-  assert((Attr.hasAttribute(Attribute::ByVal) ||
-          Attr.hasAttribute(Attribute::StructRet) ||
-          Attr.hasAttribute(Attribute::ByRef) ||
-          Attr.hasAttribute(Attribute::Preallocated)) &&
-         "unexpected type attr");
-
   if (Attr.hasAttribute(Attribute::ByVal)) {
     Out << "byval";
   } else if (Attr.hasAttribute(Attribute::StructRet)) {
     Out << "sret";
   } else if (Attr.hasAttribute(Attribute::ByRef)) {
     Out << "byref";
-  } else {
+  } else if (Attr.hasAttribute(Attribute::Preallocated)) {
     Out << "preallocated";
+  } else if (Attr.hasAttribute(Attribute::InAlloca)) {
+    Out << "inalloca";
+  } else {
+    llvm_unreachable("unexpected type attr");
   }
 
   if (Type *Ty = Attr.getValueAsType()) {
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index c69fe3fe08277aaad318808974bfb327c348bde6..60e2ec2c21be39320b74072e15c107e23dcf22b2 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -252,11 +252,13 @@ public:
   uint64_t getDereferenceableBytes() const;
   uint64_t getDereferenceableOrNullBytes() const;
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
+  std::pair<unsigned, unsigned> getVScaleRangeArgs() const;
   std::string getAsString(bool InAttrGrp) const;
   Type *getByValType() const;
   Type *getStructRetType() const;
   Type *getByRefType() const;
   Type *getPreallocatedType() const;
+  Type *getInAllocaType() const;
 
   using iterator = const Attribute *;
 
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index c98e5c36a5184b86a819284460d0496eb633aca0..c174e4f931969246759dda3ef85e03cb72a558dc 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -78,6 +78,17 @@ unpackAllocSizeArgs(uint64_t Num) {
   return std::make_pair(ElemSizeArg, NumElemsArg);
 }
 
+static uint64_t packVScaleRangeArgs(unsigned MinValue, unsigned MaxValue) {
+  return uint64_t(MinValue) << 32 | MaxValue;
+}
+
+static std::pair<unsigned, unsigned> unpackVScaleRangeArgs(uint64_t Value) {
+  unsigned MaxValue = Value & std::numeric_limits<unsigned>::max();
+  unsigned MinValue = Value >> 32;
+
+  return std::make_pair(MinValue, MaxValue);
+}
+
 Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
                          uint64_t Val) {
   LLVMContextImpl *pImpl = Context.pImpl;
@@ -184,6 +195,10 @@ Attribute Attribute::getWithPreallocatedType(LLVMContext &Context, Type *Ty) {
   return get(Context, Preallocated, Ty);
 }
 
+Attribute Attribute::getWithInAllocaType(LLVMContext &Context, Type *Ty) {
+  return get(Context, InAlloca, Ty);
+}
+
 Attribute
 Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg,
                                 const Optional<unsigned> &NumElemsArg) {
@@ -192,6 +207,12 @@ Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg,
   return get(Context, AllocSize, packAllocSizeArgs(ElemSizeArg, NumElemsArg));
 }
 
+Attribute Attribute::getWithVScaleRangeArgs(LLVMContext &Context,
+                                            unsigned MinValue,
+                                            unsigned MaxValue) {
+  return get(Context, VScaleRange, packVScaleRangeArgs(MinValue, MaxValue));
+}
+
 Attribute::AttrKind Attribute::getAttrKindFromName(StringRef AttrName) {
   return StringSwitch<Attribute::AttrKind>(AttrName)
 #define GET_ATTR_NAMES
@@ -220,7 +241,8 @@ bool Attribute::doesAttrKindHaveArgument(Attribute::AttrKind AttrKind) {
          AttrKind == Attribute::StackAlignment ||
          AttrKind == Attribute::Dereferenceable ||
          AttrKind == Attribute::AllocSize ||
-         AttrKind == Attribute::DereferenceableOrNull;
+         AttrKind == Attribute::DereferenceableOrNull ||
+         AttrKind == Attribute::VScaleRange;
 }
 
 bool Attribute::isExistingAttribute(StringRef Name) {
@@ -328,6 +350,12 @@ std::pair<unsigned, Optional<unsigned>> Attribute::getAllocSizeArgs() const {
   return unpackAllocSizeArgs(pImpl->getValueAsInt());
 }
 
+std::pair<unsigned, unsigned> Attribute::getVScaleRangeArgs() const {
+  assert(hasAttribute(Attribute::VScaleRange) &&
+         "Trying to get vscale args from non-vscale attribute");
+  return unpackVScaleRangeArgs(pImpl->getValueAsInt());
+}
+
 std::string Attribute::getAsString(bool InAttrGrp) const {
   if (!pImpl) return {};
 
@@ -353,8 +381,6 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return "inaccessiblememonly";
   if (hasAttribute(Attribute::InaccessibleMemOrArgMemOnly))
     return "inaccessiblemem_or_argmemonly";
-  if (hasAttribute(Attribute::InAlloca))
-    return "inalloca";
   if (hasAttribute(Attribute::InlineHint))
     return "inlinehint";
   if (hasAttribute(Attribute::InReg))
@@ -460,24 +486,30 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
   if (hasAttribute(Attribute::MustProgress))
     return "mustprogress";
 
-  const bool IsByVal = hasAttribute(Attribute::ByVal);
-  if (IsByVal || hasAttribute(Attribute::StructRet)) {
+  if (isTypeAttribute()) {
     std::string Result;
-    Result += IsByVal ? "byval" : "sret";
-    if (Type *Ty = getValueAsType()) {
-      raw_string_ostream OS(Result);
-      Result += '(';
-      Ty->print(OS, false, true);
-      OS.flush();
-      Result += ')';
+    raw_string_ostream OS(Result);
+
+    switch (getKindAsEnum()) {
+    case Attribute::ByVal:
+      Result += "byval";
+      break;
+    case Attribute::StructRet:
+      Result += "sret";
+      break;
+    case Attribute::ByRef:
+      Result += "byref";
+      break;
+    case Attribute::Preallocated:
+      Result += "preallocated";
+      break;
+    case Attribute::InAlloca:
+      Result += "inalloca";
+      break;
+    default:
+      llvm_unreachable("unhandled type attribute");
     }
-    return Result;
-  }
 
-  const bool IsByRef = hasAttribute(Attribute::ByRef);
-  if (IsByRef || hasAttribute(Attribute::Preallocated)) {
-    std::string Result = IsByRef ? "byref" : "preallocated";
-    raw_string_ostream OS(Result);
     Result += '(';
     getValueAsType()->print(OS, false, true);
     OS.flush();
@@ -536,6 +568,18 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return Result;
   }
 
+  if (hasAttribute(Attribute::VScaleRange)) {
+    unsigned MinValue, MaxValue;
+    std::tie(MinValue, MaxValue) = getVScaleRangeArgs();
+
+    std::string Result = "vscale_range(";
+    Result += utostr(MinValue);
+    Result += ',';
+    Result += utostr(MaxValue);
+    Result += ')';
+    return Result;
+  }
+
   // Convert target-dependent attributes to strings of the form:
   //
   //   "kind"
@@ -773,11 +817,20 @@ Type *AttributeSet::getPreallocatedType() const {
   return SetNode ? SetNode->getPreallocatedType() : nullptr;
 }
 
+Type *AttributeSet::getInAllocaType() const {
+  return SetNode ? SetNode->getInAllocaType() : nullptr;
+}
+
 std::pair<unsigned, Optional<unsigned>> AttributeSet::getAllocSizeArgs() const {
   return SetNode ? SetNode->getAllocSizeArgs()
                  : std::pair<unsigned, Optional<unsigned>>(0, 0);
 }
 
+std::pair<unsigned, unsigned> AttributeSet::getVScaleRangeArgs() const {
+  return SetNode ? SetNode->getVScaleRangeArgs()
+                 : std::pair<unsigned, unsigned>(0, 0);
+}
+
 std::string AttributeSet::getAsString(bool InAttrGrp) const {
   return SetNode ? SetNode->getAsString(InAttrGrp) : "";
 }
@@ -874,6 +927,9 @@ AttributeSetNode *AttributeSetNode::get(LLVMContext &C, const AttrBuilder &B) {
     case Attribute::Preallocated:
       Attr = Attribute::getWithPreallocatedType(C, B.getPreallocatedType());
       break;
+    case Attribute::InAlloca:
+      Attr = Attribute::getWithInAllocaType(C, B.getInAllocaType());
+      break;
     case Attribute::Alignment:
       assert(B.getAlignment() && "Alignment must be set");
       Attr = Attribute::getWithAlignment(C, *B.getAlignment());
@@ -895,6 +951,11 @@ AttributeSetNode *AttributeSetNode::get(LLVMContext &C, const AttrBuilder &B) {
       Attr = Attribute::getWithAllocSizeArgs(C, A.first, A.second);
       break;
     }
+    case Attribute::VScaleRange: {
+      auto A = B.getVScaleRangeArgs();
+      Attr = Attribute::getWithVScaleRangeArgs(C, A.first, A.second);
+      break;
+    }
     default:
       Attr = Attribute::get(C, Kind);
     }
@@ -975,6 +1036,12 @@ Type *AttributeSetNode::getPreallocatedType() const {
   return nullptr;
 }
 
+Type *AttributeSetNode::getInAllocaType() const {
+  if (auto A = findEnumAttribute(Attribute::InAlloca))
+    return A->getValueAsType();
+  return nullptr;
+}
+
 uint64_t AttributeSetNode::getDereferenceableBytes() const {
   if (auto A = findEnumAttribute(Attribute::Dereferenceable))
     return A->getDereferenceableBytes();
@@ -994,6 +1061,12 @@ AttributeSetNode::getAllocSizeArgs() const {
   return std::make_pair(0, 0);
 }
 
+std::pair<unsigned, unsigned> AttributeSetNode::getVScaleRangeArgs() const {
+  if (auto A = findEnumAttribute(Attribute::VScaleRange))
+    return A->getVScaleRangeArgs();
+  return std::make_pair(0, 0);
+}
+
 std::string AttributeSetNode::getAsString(bool InAttrGrp) const {
   std::string Str;
   for (iterator I = begin(), E = end(); I != E; ++I) {
@@ -1402,6 +1475,17 @@ AttributeList AttributeList::removeAttributes(LLVMContext &C,
   return getImpl(C, AttrSets);
 }
 
+AttributeList
+AttributeList::removeParamUndefImplyingAttributes(LLVMContext &C,
+                                                  unsigned ArgNo) const {
+  AttrBuilder B;
+  B.addAttribute(Attribute::NoUndef);
+  B.addAttribute(Attribute::NonNull);
+  B.addDereferenceableAttr(1);
+  B.addDereferenceableOrNullAttr(1);
+  return removeParamAttributes(C, ArgNo, B);
+}
+
 AttributeList AttributeList::addDereferenceableAttr(LLVMContext &C,
                                                     unsigned Index,
                                                     uint64_t Bytes) const {
@@ -1427,6 +1511,14 @@ AttributeList::addAllocSizeAttr(LLVMContext &C, unsigned Index,
   return addAttributes(C, Index, B);
 }
 
+AttributeList AttributeList::addVScaleRangeAttr(LLVMContext &C, unsigned Index,
+                                                unsigned MinValue,
+                                                unsigned MaxValue) {
+  AttrBuilder B;
+  B.addVScaleRangeAttr(MinValue, MaxValue);
+  return addAttributes(C, Index, B);
+}
+
 //===----------------------------------------------------------------------===//
 // AttributeList Accessor Methods
 //===----------------------------------------------------------------------===//
@@ -1507,6 +1599,10 @@ Type *AttributeList::getParamPreallocatedType(unsigned Index) const {
   return getAttributes(Index + FirstArgIndex).getPreallocatedType();
 }
 
+Type *AttributeList::getParamInAllocaType(unsigned Index) const {
+  return getAttributes(Index + FirstArgIndex).getInAllocaType();
+}
+
 MaybeAlign AttributeList::getStackAlignment(unsigned Index) const {
   return getAttributes(Index).getStackAlignment();
 }
@@ -1524,6 +1620,11 @@ AttributeList::getAllocSizeArgs(unsigned Index) const {
   return getAttributes(Index).getAllocSizeArgs();
 }
 
+std::pair<unsigned, unsigned>
+AttributeList::getVScaleRangeArgs(unsigned Index) const {
+  return getAttributes(Index).getVScaleRangeArgs();
+}
+
 std::string AttributeList::getAsString(unsigned Index, bool InAttrGrp) const {
   return getAttributes(Index).getAsString(InAttrGrp);
 }
@@ -1587,6 +1688,7 @@ void AttrBuilder::clear() {
   StackAlignment.reset();
   DerefBytes = DerefOrNullBytes = 0;
   AllocSizeArgs = 0;
+  VScaleRangeArgs = 0;
   ByValType = nullptr;
   StructRetType = nullptr;
   ByRefType = nullptr;
@@ -1620,11 +1722,16 @@ AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) {
     DerefOrNullBytes = Attr.getDereferenceableOrNullBytes();
   else if (Kind == Attribute::AllocSize)
     AllocSizeArgs = Attr.getValueAsInt();
+  else if (Kind == Attribute::VScaleRange)
+    VScaleRangeArgs = Attr.getValueAsInt();
+  else if (Kind == Attribute::InAlloca)
+    InAllocaType = Attr.getValueAsType();
+
   return *this;
 }
 
 AttrBuilder &AttrBuilder::addAttribute(StringRef A, StringRef V) {
-  TargetDepAttrs[std::string(A)] = std::string(V);
+  TargetDepAttrs[A] = V;
   return *this;
 }
 
@@ -1644,12 +1751,16 @@ AttrBuilder &AttrBuilder::removeAttribute(Attribute::AttrKind Val) {
     ByRefType = nullptr;
   else if (Val == Attribute::Preallocated)
     PreallocatedType = nullptr;
+  else if (Val == Attribute::InAlloca)
+    InAllocaType = nullptr;
   else if (Val == Attribute::Dereferenceable)
     DerefBytes = 0;
   else if (Val == Attribute::DereferenceableOrNull)
     DerefOrNullBytes = 0;
   else if (Val == Attribute::AllocSize)
     AllocSizeArgs = 0;
+  else if (Val == Attribute::VScaleRange)
+    VScaleRangeArgs = 0;
 
   return *this;
 }
@@ -1670,6 +1781,10 @@ std::pair<unsigned, Optional<unsigned>> AttrBuilder::getAllocSizeArgs() const {
   return unpackAllocSizeArgs(AllocSizeArgs);
 }
 
+std::pair<unsigned, unsigned> AttrBuilder::getVScaleRangeArgs() const {
+  return unpackVScaleRangeArgs(VScaleRangeArgs);
+}
+
 AttrBuilder &AttrBuilder::addAlignmentAttr(MaybeAlign Align) {
   if (!Align)
     return *this;
@@ -1726,6 +1841,23 @@ AttrBuilder &AttrBuilder::addAllocSizeAttrFromRawRepr(uint64_t RawArgs) {
   return *this;
 }
 
+AttrBuilder &AttrBuilder::addVScaleRangeAttr(unsigned MinValue,
+                                             unsigned MaxValue) {
+  return addVScaleRangeAttrFromRawRepr(packVScaleRangeArgs(MinValue, MaxValue));
+}
+
+AttrBuilder &AttrBuilder::addVScaleRangeAttrFromRawRepr(uint64_t RawArgs) {
+  // (0, 0) is not present hence ignore this case
+  if (RawArgs == 0)
+    return *this;
+
+  Attrs[Attribute::VScaleRange] = true;
+  // Reuse existing machinery to store this as a single 64-bit integer so we can
+  // save a few bytes over using a pair<unsigned, unsigned>.
+  VScaleRangeArgs = RawArgs;
+  return *this;
+}
+
 AttrBuilder &AttrBuilder::addByValAttr(Type *Ty) {
   Attrs[Attribute::ByVal] = true;
   ByValType = Ty;
@@ -1750,6 +1882,12 @@ AttrBuilder &AttrBuilder::addPreallocatedAttr(Type *Ty) {
   return *this;
 }
 
+AttrBuilder &AttrBuilder::addInAllocaAttr(Type *Ty) {
+  Attrs[Attribute::InAlloca] = true;
+  InAllocaType = Ty;
+  return *this;
+}
+
 AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
   // FIXME: What if both have alignments, but they don't match?!
   if (!Alignment)
@@ -1779,6 +1917,12 @@ AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
   if (!PreallocatedType)
     PreallocatedType = B.PreallocatedType;
 
+  if (!InAllocaType)
+    InAllocaType = B.InAllocaType;
+
+  if (!VScaleRangeArgs)
+    VScaleRangeArgs = B.VScaleRangeArgs;
+
   Attrs |= B.Attrs;
 
   for (const auto &I : B.td_attrs())
@@ -1816,6 +1960,12 @@ AttrBuilder &AttrBuilder::remove(const AttrBuilder &B) {
   if (B.PreallocatedType)
     PreallocatedType = nullptr;
 
+  if (B.InAllocaType)
+    InAllocaType = nullptr;
+
+  if (B.VScaleRangeArgs)
+    VScaleRangeArgs = 0;
+
   Attrs &= ~B.Attrs;
 
   for (const auto &I : B.td_attrs())
@@ -1876,7 +2026,9 @@ bool AttrBuilder::operator==(const AttrBuilder &B) const {
   return Alignment == B.Alignment && StackAlignment == B.StackAlignment &&
          DerefBytes == B.DerefBytes && ByValType == B.ByValType &&
          StructRetType == B.StructRetType && ByRefType == B.ByRefType &&
-         PreallocatedType == B.PreallocatedType;
+         PreallocatedType == B.PreallocatedType &&
+         InAllocaType == B.InAllocaType &&
+         VScaleRangeArgs == B.VScaleRangeArgs;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1905,6 +2057,7 @@ AttrBuilder AttributeFuncs::typeIncompatible(Type *Ty) {
         .addAttribute(Attribute::ReadOnly)
         .addAttribute(Attribute::InAlloca)
         .addPreallocatedAttr(Ty)
+        .addInAllocaAttr(Ty)
         .addByValAttr(Ty)
         .addStructRetAttr(Ty)
         .addByRefAttr(Ty);
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index e0d152b5ec219b88909bd3b5c965b6591af58daa..c5d085fafbc7a741b6e660392f9d4b117ec47f2c 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -548,6 +548,11 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
                                         F->arg_begin()->getType());
       return true;
     }
+    if (Name.startswith("aarch64.neon.frintn")) {
+      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::roundeven,
+                                        F->arg_begin()->getType());
+      return true;
+    }
     if (Name.startswith("arm.neon.vclz")) {
       Type* args[2] = {
         F->arg_begin()->getType(),
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index dc141659907830edee2304a0c59f3b4040159ef7..b53ab55575544366e7afa4185e8d59098e0b85be 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -421,6 +421,9 @@ Constant *Constant::getAllOnesValue(Type *Ty) {
 }
 
 Constant *Constant::getAggregateElement(unsigned Elt) const {
+  assert((getType()->isAggregateType() || getType()->isVectorTy()) &&
+         "Must be an aggregate/vector constant");
+
   if (const auto *CC = dyn_cast<ConstantAggregate>(this))
     return Elt < CC->getNumOperands() ? CC->getOperand(Elt) : nullptr;
 
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 7398a7efd8cd4275df8dabe737cc10da55a42d87..2d93d50b88995c22ae579f02d076ddff426f1adf 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -164,6 +164,18 @@ uint64_t LLVMGetEnumAttributeValue(LLVMAttributeRef A) {
   return Attr.getValueAsInt();
 }
 
+LLVMAttributeRef LLVMCreateTypeAttribute(LLVMContextRef C, unsigned KindID,
+                                         LLVMTypeRef type_ref) {
+  auto &Ctx = *unwrap(C);
+  auto AttrKind = (Attribute::AttrKind)KindID;
+  return wrap(Attribute::get(Ctx, AttrKind, unwrap(type_ref)));
+}
+
+LLVMTypeRef LLVMGetTypeAttributeValue(LLVMAttributeRef A) {
+  auto Attr = unwrap(A);
+  return wrap(Attr.getValueAsType());
+}
+
 LLVMAttributeRef LLVMCreateStringAttribute(LLVMContextRef C,
                                            const char *K, unsigned KLength,
                                            const char *V, unsigned VLength) {
@@ -194,6 +206,10 @@ LLVMBool LLVMIsStringAttribute(LLVMAttributeRef A) {
   return unwrap(A).isStringAttribute();
 }
 
+LLVMBool LLVMIsTypeAttribute(LLVMAttributeRef A) {
+  return unwrap(A).isTypeAttribute();
+}
+
 char *LLVMGetDiagInfoDescription(LLVMDiagnosticInfoRef DI) {
   std::string MsgStorage;
   raw_string_ostream Stream(MsgStorage);
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index fec98f5a36c092a88c5d97c7c3d37eb3c1a8e89d..99349a304e30c581cc82924d25e950e8ffb5ef1d 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -345,6 +345,9 @@ bool llvm::stripDebugInfo(Function &F) {
         if (NewLoopID != LoopID)
           I.setMetadata(LLVMContext::MD_loop, NewLoopID);
       }
+      // Strip heapallocsite attachments, they point into the DIType system.
+      if (I.hasMetadataOtherThanDebugLoc())
+        I.setMetadata("heapallocsite", nullptr);
     }
   }
   return Changed;
@@ -654,6 +657,10 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) {
         updateLoopMetadataDebugLocations(I, [&](const DILocation &Loc) {
           return remapDebugLoc(&Loc).get();
         });
+
+        // Strip heapallocsite attachments, they point into the DIType system.
+        if (I.hasMetadataOtherThanDebugLoc())
+          I.setMetadata("heapallocsite", nullptr);
       }
     }
   }
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index bae7960ddc35a85342db9944b22344c154b0f815..eba7316a0fe7470b88ebc186e5ed300761c016b3 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -163,6 +163,8 @@ static Type *getMemoryParamAllocType(AttributeSet ParamAttrs, Type *ArgTy) {
     return ByRefTy;
   if (Type *PreAllocTy = ParamAttrs.getPreallocatedType())
     return PreAllocTy;
+  if (Type *InAllocaTy = ParamAttrs.getInAllocaType())
+    return InAllocaTy;
 
   // FIXME: sret and inalloca always depends on pointee element type. It's also
   // possible for byval to miss it.
@@ -214,6 +216,11 @@ Type *Argument::getParamByRefType() const {
   return getParent()->getParamByRefType(getArgNo());
 }
 
+Type *Argument::getParamInAllocaType() const {
+  assert(getType()->isPointerTy() && "Only pointers have inalloca types");
+  return getParent()->getParamInAllocaType(getArgNo());
+}
+
 uint64_t Argument::getDereferenceableBytes() const {
   assert(getType()->isPointerTy() &&
          "Only pointers have dereferenceable bytes");
@@ -241,6 +248,11 @@ bool Argument::hasNoCaptureAttr() const {
   return hasAttribute(Attribute::NoCapture);
 }
 
+bool Argument::hasNoFreeAttr() const {
+  if (!getType()->isPointerTy()) return false;
+  return hasAttribute(Attribute::NoFree);
+}
+
 bool Argument::hasStructRetAttr() const {
   if (!getType()->isPointerTy()) return false;
   return hasAttribute(Attribute::StructRet);
@@ -558,6 +570,12 @@ void Function::removeParamAttrs(unsigned ArgNo, const AttrBuilder &Attrs) {
   setAttributes(PAL);
 }
 
+void Function::removeParamUndefImplyingAttrs(unsigned ArgNo) {
+  AttributeList PAL = getAttributes();
+  PAL = PAL.removeParamUndefImplyingAttributes(getContext(), ArgNo);
+  setAttributes(PAL);
+}
+
 void Function::addDereferenceableAttr(unsigned i, uint64_t Bytes) {
   AttributeList PAL = getAttributes();
   PAL = PAL.addDereferenceableAttr(getContext(), i, Bytes);
@@ -722,30 +740,34 @@ void Function::recalculateIntrinsicID() {
 /// which can't be confused with it's prefix.  This ensures we don't have
 /// collisions between two unrelated function types. Otherwise, you might
 /// parse ffXX as f(fXX) or f(fX)X.  (X is a placeholder for any other type.)
-///
-static std::string getMangledTypeStr(Type* Ty) {
+/// The HasUnnamedType boolean is set if an unnamed type was encountered,
+/// indicating that extra care must be taken to ensure a unique name.
+static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
   std::string Result;
   if (PointerType* PTyp = dyn_cast<PointerType>(Ty)) {
     Result += "p" + utostr(PTyp->getAddressSpace()) +
-      getMangledTypeStr(PTyp->getElementType());
+              getMangledTypeStr(PTyp->getElementType(), HasUnnamedType);
   } else if (ArrayType* ATyp = dyn_cast<ArrayType>(Ty)) {
     Result += "a" + utostr(ATyp->getNumElements()) +
-      getMangledTypeStr(ATyp->getElementType());
+              getMangledTypeStr(ATyp->getElementType(), HasUnnamedType);
   } else if (StructType *STyp = dyn_cast<StructType>(Ty)) {
     if (!STyp->isLiteral()) {
       Result += "s_";
-      Result += STyp->getName();
+      if (STyp->hasName())
+        Result += STyp->getName();
+      else
+        HasUnnamedType = true;
     } else {
       Result += "sl_";
       for (auto Elem : STyp->elements())
-        Result += getMangledTypeStr(Elem);
+        Result += getMangledTypeStr(Elem, HasUnnamedType);
     }
     // Ensure nested structs are distinguishable.
     Result += "s";
   } else if (FunctionType *FT = dyn_cast<FunctionType>(Ty)) {
-    Result += "f_" + getMangledTypeStr(FT->getReturnType());
+    Result += "f_" + getMangledTypeStr(FT->getReturnType(), HasUnnamedType);
     for (size_t i = 0; i < FT->getNumParams(); i++)
-      Result += getMangledTypeStr(FT->getParamType(i));
+      Result += getMangledTypeStr(FT->getParamType(i), HasUnnamedType);
     if (FT->isVarArg())
       Result += "vararg";
     // Ensure nested function types are distinguishable.
@@ -755,7 +777,7 @@ static std::string getMangledTypeStr(Type* Ty) {
     if (EC.isScalable())
       Result += "nx";
     Result += "v" + utostr(EC.getKnownMinValue()) +
-              getMangledTypeStr(VTy->getElementType());
+              getMangledTypeStr(VTy->getElementType(), HasUnnamedType);
   } else if (Ty) {
     switch (Ty->getTypeID()) {
     default: llvm_unreachable("Unhandled type");
@@ -785,17 +807,32 @@ StringRef Intrinsic::getName(ID id) {
   return IntrinsicNameTable[id];
 }
 
-std::string Intrinsic::getName(ID id, ArrayRef<Type*> Tys) {
-  assert(id < num_intrinsics && "Invalid intrinsic ID!");
-  assert((Tys.empty() || Intrinsic::isOverloaded(id)) &&
+std::string Intrinsic::getName(ID Id, ArrayRef<Type *> Tys, Module *M,
+                               FunctionType *FT) {
+  assert(Id < num_intrinsics && "Invalid intrinsic ID!");
+  assert((Tys.empty() || Intrinsic::isOverloaded(Id)) &&
          "This version of getName is for overloaded intrinsics only");
-  std::string Result(IntrinsicNameTable[id]);
+  bool HasUnnamedType = false;
+  std::string Result(IntrinsicNameTable[Id]);
   for (Type *Ty : Tys) {
-    Result += "." + getMangledTypeStr(Ty);
+    Result += "." + getMangledTypeStr(Ty, HasUnnamedType);
+  }
+  assert((M || !HasUnnamedType) && "unnamed types need a module");
+  if (M && HasUnnamedType) {
+    if (!FT)
+      FT = getType(M->getContext(), Id, Tys);
+    else
+      assert((FT == getType(M->getContext(), Id, Tys)) &&
+             "Provided FunctionType must match arguments");
+    return M->getUniqueIntrinsicName(Result, Id, FT);
   }
   return Result;
 }
 
+std::string Intrinsic::getName(ID Id, ArrayRef<Type *> Tys) {
+  return getName(Id, Tys, nullptr, nullptr);
+}
+
 /// IIT_Info - These are enumerators that describe the entries returned by the
 /// getIntrinsicInfoTableEntries function.
 ///
@@ -1255,8 +1292,10 @@ bool Intrinsic::isLeaf(ID id) {
 Function *Intrinsic::getDeclaration(Module *M, ID id, ArrayRef<Type*> Tys) {
   // There can never be multiple globals with the same name of different types,
   // because intrinsics must be a specific type.
+  auto *FT = getType(M->getContext(), id, Tys);
   return cast<Function>(
-      M->getOrInsertFunction(Tys.empty() ? getName(id) : getName(id, Tys),
+      M->getOrInsertFunction(Tys.empty() ? getName(id)
+                                         : getName(id, Tys, M, FT),
                              getType(M->getContext(), id, Tys))
           .getCallee());
 }
@@ -1569,7 +1608,8 @@ Optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
 
   Intrinsic::ID ID = F->getIntrinsicID();
   StringRef Name = F->getName();
-  if (Name == Intrinsic::getName(ID, ArgTys))
+  if (Name ==
+      Intrinsic::getName(ID, ArgTys, F->getParent(), F->getFunctionType()))
     return None;
 
   auto NewDecl = Intrinsic::getDeclaration(F->getParent(), ID, ArgTys);
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 45a7bc4c05c2f1b5e2792461740531034de25f05..3105d3920e82b7e93faee860216a5fe2455a8294 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -92,6 +92,23 @@ Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
              : CreateMul(CI, Scaling);
 }
 
+Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
+  if (isa<ScalableVectorType>(DstType))
+    return CreateIntrinsic(Intrinsic::experimental_stepvector, {DstType}, {},
+                           nullptr, Name);
+
+  Type *STy = DstType->getScalarType();
+  unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
+
+  // Create a vector of consecutive numbers from zero to VF.
+  SmallVector<Constant *, 8> Indices;
+  for (unsigned i = 0; i < NumEls; ++i)
+    Indices.push_back(ConstantInt::get(STy, i));
+
+  // Add the consecutive indices to the vector value.
+  return ConstantVector::get(Indices);
+}
+
 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
                                       MaybeAlign Align, bool isVolatile,
                                       MDNode *TBAATag, MDNode *ScopeTag,
@@ -1001,6 +1018,22 @@ Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
   return Fn;
 }
 
+Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
+  auto *Ty = cast<VectorType>(V->getType());
+  if (isa<ScalableVectorType>(Ty)) {
+    Module *M = BB->getParent()->getParent();
+    Function *F = Intrinsic::getDeclaration(
+        M, Intrinsic::experimental_vector_reverse, Ty);
+    return Insert(CallInst::Create(F, V), Name);
+  }
+  // Keep the original behaviour for fixed vector
+  SmallVector<int, 8> ShuffleMask;
+  int NumElts = Ty->getElementCount().getKnownMinValue();
+  for (int i = 0; i < NumElts; ++i)
+    ShuffleMask.push_back(NumElts - i - 1);
+  return CreateShuffleVector(V, ShuffleMask, Name);
+}
+
 Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
                                         const Twine &Name) {
   auto EC = ElementCount::getFixed(NumElts);
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index bf1a780d0cc75d6e8312010098923a293396ebb1..984edc66268a07ff18a6f0ce16357df6e1d51b5d 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -466,6 +466,13 @@ CallBase *CallBase::removeOperandBundle(CallBase *CB, uint32_t ID,
   return CreateNew ? Create(CB, Bundles, InsertPt) : CB;
 }
 
+bool CallBase::hasReadingOperandBundles() const {
+  // Implementation note: this is a conservative implementation of operand
+  // bundle semantics, where *any* non-assume operand bundle forces a callsite
+  // to be at least readonly.
+  return hasOperandBundles() && getIntrinsicID() != Intrinsic::assume;
+}
+
 //===----------------------------------------------------------------------===//
 //                        CallInst Implementation
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index 60056f142d8a65ac69b667f3898625e0a160f911..41666e8274d42e278fce9caf352943e5235f209d 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -473,6 +473,56 @@ std::vector<StructType *> Module::getIdentifiedStructTypes() const {
   return Ret;
 }
 
+std::string Module::getUniqueIntrinsicName(StringRef BaseName, Intrinsic::ID Id,
+                                           const FunctionType *Proto) {
+  auto Encode = [&BaseName](unsigned Suffix) {
+    return (Twine(BaseName) + "." + Twine(Suffix)).str();
+  };
+
+  {
+    // fast path - the prototype is already known
+    auto UinItInserted = UniquedIntrinsicNames.insert({{Id, Proto}, 0});
+    if (!UinItInserted.second)
+      return Encode(UinItInserted.first->second);
+  }
+
+  // Not known yet. A new entry was created with index 0. Check if there already
+  // exists a matching declaration, or select a new entry.
+
+  // Start looking for names with the current known maximum count (or 0).
+  auto NiidItInserted = CurrentIntrinsicIds.insert({BaseName, 0});
+  unsigned Count = NiidItInserted.first->second;
+
+  // This might be slow if a whole population of intrinsics already existed, but
+  // we cache the values for later usage.
+  std::string NewName;
+  while (true) {
+    NewName = Encode(Count);
+    GlobalValue *F = getNamedValue(NewName);
+    if (!F) {
+      // Reserve this entry for the new proto
+      UniquedIntrinsicNames[{Id, Proto}] = Count;
+      break;
+    }
+
+    // A declaration with this name already exists. Remember it.
+    FunctionType *FT = dyn_cast<FunctionType>(F->getType()->getElementType());
+    auto UinItInserted = UniquedIntrinsicNames.insert({{Id, FT}, Count});
+    if (FT == Proto) {
+      // It was a declaration for our prototype. This entry was allocated in the
+      // beginning. Update the count to match the existing declaration.
+      UinItInserted.first->second = Count;
+      break;
+    }
+
+    ++Count;
+  }
+
+  NiidItInserted.first->second = Count + 1;
+
+  return NewName;
+}
+
 // dropAllReferences() - This function causes all the subelements to "let go"
 // of all references that they are maintaining.  This allows one to 'delete' a
 // whole module at a time, even though there may be circular references... first
@@ -521,9 +571,9 @@ unsigned Module::getCodeViewFlag() const {
   return cast<ConstantInt>(Val->getValue())->getZExtValue();
 }
 
-unsigned Module::getInstructionCount() {
+unsigned Module::getInstructionCount() const {
   unsigned NumInstrs = 0;
-  for (Function &F : FunctionList)
+  for (const Function &F : FunctionList)
     NumInstrs += F.getInstructionCount();
   return NumInstrs;
 }
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index 92ffae18ae6fa69fab4efadfe6ce6db157ddbc42..fb4eb9a9e3a64bdea92d0f30f34e8653c478dd09 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -38,6 +38,11 @@
 
 using namespace llvm;
 
+static cl::opt<unsigned> UseDerefAtPointSemantics(
+    "use-dereferenceable-at-point-semantics", cl::Hidden, cl::init(false),
+    cl::desc("Deref attributes and metadata infer facts at definition only"));
+
+
 static cl::opt<unsigned> NonGlobalValueMaxNameSize(
     "non-global-value-max-name-size", cl::Hidden, cl::init(1024),
     cl::desc("Maximum size for the name of non-global values."));
@@ -723,12 +728,78 @@ Value::stripInBoundsOffsets(function_ref<void(const Value *)> Func) const {
   return stripPointerCastsAndOffsets<PSK_InBounds>(this, Func);
 }
 
+// Return true if the memory object referred to by V can by freed in the scope
+// for which the SSA value defining the allocation is statically defined.  E.g.
+// deallocation after the static scope of a value does not count.
+static bool canBeFreed(const Value *V) {
+  assert(V->getType()->isPointerTy());
+
+  // Cases that can simply never be deallocated
+  // *) Constants aren't allocated per se, thus not deallocated either.
+  if (isa<Constant>(V))
+    return false;
+
+  // Handle byval/byref/sret/inalloca/preallocated arguments.  The storage
+  // lifetime is guaranteed to be longer than the callee's lifetime.
+  if (auto *A = dyn_cast<Argument>(V))
+    if (A->hasPointeeInMemoryValueAttr())
+      return false;
+
+  const Function *F = nullptr;
+  if (auto *I = dyn_cast<Instruction>(V))
+    F = I->getFunction();
+  if (auto *A = dyn_cast<Argument>(V))
+    F = A->getParent();
+
+  if (!F)
+    return true;
+
+  // A pointer to an object in a function which neither frees, nor can arrange
+  // for another thread to free on its behalf, can not be freed in the scope
+  // of the function.
+  if (F->doesNotFreeMemory() && F->hasNoSync())
+    return false;
+
+  // With garbage collection, deallocation typically occurs solely at or after
+  // safepoints.  If we're compiling for a collector which uses the
+  // gc.statepoint infrastructure, safepoints aren't explicitly present
+  // in the IR until after lowering from abstract to physical machine model.
+  // The collector could chose to mix explicit deallocation and gc'd objects
+  // which is why we need the explicit opt in on a per collector basis.
+  if (!F->hasGC())
+    return true;
+  
+  const auto &GCName = F->getGC();
+  const StringRef StatepointExampleName("statepoint-example");
+  if (GCName != StatepointExampleName)
+    return true;
+
+  auto *PT = cast<PointerType>(V->getType());
+  if (PT->getAddressSpace() != 1)
+    // For the sake of this example GC, we arbitrarily pick addrspace(1) as our
+    // GC managed heap.  This must match the same check in
+    // RewriteStatepointsForGC (and probably needs better factored.)
+    return true;
+
+  // It is cheaper to scan for a declaration than to scan for a use in this
+  // function.  Note that gc.statepoint is a type overloaded function so the
+  // usual trick of requesting declaration of the intrinsic from the module
+  // doesn't work.
+  for (auto &Fn : *F->getParent())
+    if (Fn.getIntrinsicID() == Intrinsic::experimental_gc_statepoint)
+      return true;
+  return false;
+}
+
+
 uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL,
-                                               bool &CanBeNull) const {
+                                               bool &CanBeNull,
+                                               bool &CanBeFreed) const {
   assert(getType()->isPointerTy() && "must be pointer");
 
   uint64_t DerefBytes = 0;
   CanBeNull = false;
+  CanBeFreed = UseDerefAtPointSemantics && canBeFreed(this);
   if (const Argument *A = dyn_cast<Argument>(this)) {
     DerefBytes = A->getDereferenceableBytes();
     if (DerefBytes == 0) {
@@ -783,6 +854,7 @@ uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL,
       DerefBytes =
           DL.getTypeStoreSize(AI->getAllocatedType()).getKnownMinSize();
       CanBeNull = false;
+      CanBeFreed = false;
     }
   } else if (auto *GV = dyn_cast<GlobalVariable>(this)) {
     if (GV->getValueType()->isSized() && !GV->hasExternalWeakLinkage()) {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index b7a002b0573bbd588f2baf5c8942a96cb2bdb104..b6952f703041629a8e6f907da245610c80d807fd 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -813,6 +813,9 @@ void Verifier::visitMDNode(const MDNode &MD, AreDebugLocsAllowed AllowLocs) {
   if (!MDNodes.insert(&MD).second)
     return;
 
+  Assert(&MD.getContext() == &Context,
+         "MDNode context does not match Module context!", &MD);
+
   switch (MD.getMetadataID()) {
   default:
     llvm_unreachable("Invalid MDNode subclass");
@@ -1629,6 +1632,7 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) {
   case Attribute::InlineHint:
   case Attribute::StackAlignment:
   case Attribute::UWTable:
+  case Attribute::VScaleRange:
   case Attribute::NonLazyBind:
   case Attribute::ReturnsTwice:
   case Attribute::SanitizeAddress:
@@ -1809,6 +1813,11 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
       Assert(Attrs.getPreallocatedType() == PTy->getElementType(),
              "Attribute 'preallocated' type does not match parameter!", V);
     }
+
+    if (Attrs.hasAttribute(Attribute::InAlloca)) {
+      Assert(Attrs.getInAllocaType() == PTy->getElementType(),
+             "Attribute 'inalloca' type does not match parameter!", V);
+    }
   } else {
     Assert(!Attrs.hasAttribute(Attribute::ByVal),
            "Attribute 'byval' only applies to parameters with pointer type!",
@@ -1987,6 +1996,14 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
       return;
   }
 
+  if (Attrs.hasFnAttribute(Attribute::VScaleRange)) {
+    std::pair<unsigned, unsigned> Args =
+        Attrs.getVScaleRangeArgs(AttributeList::FunctionIndex);
+
+    if (Args.first > Args.second && Args.second != 0)
+      CheckFailed("'vscale_range' minimum cannot be greater than maximum", V);
+  }
+
   if (Attrs.hasFnAttribute("frame-pointer")) {
     StringRef FP = Attrs.getAttribute(AttributeList::FunctionIndex,
                                       "frame-pointer").getValueAsString();
@@ -4542,7 +4559,8 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
   // know they are legal for the intrinsic!) get the intrinsic name through the
   // usual means.  This allows us to verify the mangling of argument types into
   // the name.
-  const std::string ExpectedName = Intrinsic::getName(ID, ArgTys);
+  const std::string ExpectedName =
+      Intrinsic::getName(ID, ArgTys, IF->getParent(), IFTy);
   Assert(ExpectedName == IF->getName(),
          "Intrinsic name not mangled correctly for type arguments! "
          "Should be: " +
@@ -5185,6 +5203,15 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
 
     break;
   }
+  case Intrinsic::experimental_stepvector: {
+    VectorType *VecTy = dyn_cast<VectorType>(Call.getType());
+    Assert(VecTy && VecTy->getScalarType()->isIntegerTy() &&
+               VecTy->getScalarSizeInBits() >= 8,
+           "experimental_stepvector only supported for vectors of integers "
+           "with a bitwidth of at least 8.",
+           &Call);
+    break;
+  }
   case Intrinsic::experimental_vector_insert: {
     VectorType *VecTy = cast<VectorType>(Call.getArgOperand(0)->getType());
     VectorType *SubVecTy = cast<VectorType>(Call.getArgOperand(1)->getType());
diff --git a/llvm/lib/IRReader/IRReader.cpp b/llvm/lib/IRReader/IRReader.cpp
index e7fd835f8ad07c1357098004ce29687627db389e..b645e0b766a946d1fb5611239a7bd3abea2f7d01 100644
--- a/llvm/lib/IRReader/IRReader.cpp
+++ b/llvm/lib/IRReader/IRReader.cpp
@@ -92,7 +92,7 @@ std::unique_ptr<Module>
 llvm::parseIRFile(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context,
                   DataLayoutCallbackTy DataLayoutCallback) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
-      MemoryBuffer::getFileOrSTDIN(Filename);
+      MemoryBuffer::getFileOrSTDIN(Filename, /*IsText=*/true);
   if (std::error_code EC = FileOrErr.getError()) {
     Err = SMDiagnostic(Filename, SourceMgr::DK_Error,
                        "Could not open input file: " + EC.message());
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 8bcb1600925d5ee0edda1c893fe83c8bd543ee60..3cd8c78c42e6fa376d78c0a7ca220299236f7931 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -11,7 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/LTO/LTO.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/StackSafetyAnalysis.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
@@ -752,6 +754,7 @@ LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
   Skip();
 
   std::set<const Comdat *> NonPrevailingComdats;
+  SmallSet<StringRef, 2> NonPrevailingAsmSymbols;
   for (const InputFile::Symbol &Sym : Syms) {
     assert(ResI != ResE);
     SymbolResolution Res = *ResI++;
@@ -798,7 +801,14 @@ LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
           GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
                                  DefaultStorageClass);
       }
+    } else if (auto *AS = Msym.dyn_cast<ModuleSymbolTable::AsmSymbol *>()) {
+      // Collect non-prevailing symbols.
+      if (!Res.Prevailing)
+        NonPrevailingAsmSymbols.insert(AS->first);
+    } else {
+      llvm_unreachable("unknown symbol type");
     }
+
     // Common resolution: collect the maximum size/alignment over all commons.
     // We also record if we see an instance of a common as prevailing, so that
     // if none is prevailing we can ignore it later.
@@ -812,11 +822,29 @@ LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
         CommonRes.Align = max(*SymAlign, CommonRes.Align);
       CommonRes.Prevailing |= Res.Prevailing;
     }
-
   }
+
   if (!M.getComdatSymbolTable().empty())
     for (GlobalValue &GV : M.global_values())
       handleNonPrevailingComdat(GV, NonPrevailingComdats);
+
+  // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm
+  // block.
+  if (!M.getModuleInlineAsm().empty()) {
+    std::string NewIA = ".lto_discard";
+    if (!NonPrevailingAsmSymbols.empty()) {
+      // Don't dicard a symbol if there is a live .symver for it.
+      ModuleSymbolTable::CollectAsmSymvers(
+          M, [&](StringRef Name, StringRef Alias) {
+            if (!NonPrevailingAsmSymbols.count(Alias))
+              NonPrevailingAsmSymbols.erase(Name);
+          });
+      NewIA += " " + llvm::join(NonPrevailingAsmSymbols, ", ");
+    }
+    NewIA += "\n";
+    M.setModuleInlineAsm(NewIA + M.getModuleInlineAsm());
+  }
+
   assert(MsymI == MsymE);
   return std::move(Mod);
 }
diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp
index 9634014dad6a6f24e359b5c6c39f77f0c98c1f0c..8e5587051c26f19d49ddbb1147db203c5e1267d5 100644
--- a/llvm/lib/LTO/LTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/LTOCodeGenerator.cpp
@@ -285,8 +285,8 @@ LTOCodeGenerator::compileOptimized() {
     return nullptr;
 
   // read .o file into memory buffer
-  ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
-      MemoryBuffer::getFile(name, -1, false);
+  ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr = MemoryBuffer::getFile(
+      name, /*IsText=*/false, /*RequiresNullTerminator=*/false);
   if (std::error_code EC = BufferOrErr.getError()) {
     emitError(EC.message());
     sys::fs::remove(NativeObjectPath);
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index 1004e4e7d3347b81e5aa983e2ecc9e80eba8b34c..1433d074595bacdf8e70350da358a28fdaa7186e 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -460,6 +460,14 @@ class IRLinker {
     if (DGV->hasLocalLinkage())
       return nullptr;
 
+    // If we found an intrinsic declaration with mismatching prototypes, we
+    // probably had a nameclash. Don't use that version.
+    if (auto *FDGV = dyn_cast<Function>(DGV))
+      if (FDGV->isIntrinsic())
+        if (const auto *FSrcGV = dyn_cast<Function>(SrcGV))
+          if (FDGV->getFunctionType() != TypeMap.get(FSrcGV->getFunctionType()))
+            return nullptr;
+
     // Otherwise, we do in fact link to the destination global.
     return DGV;
   }
@@ -639,7 +647,8 @@ GlobalVariable *IRLinker::copyGlobalVariableProto(const GlobalVariable *SGVar) {
 AttributeList IRLinker::mapAttributeTypes(LLVMContext &C, AttributeList Attrs) {
   for (unsigned i = 0; i < Attrs.getNumAttrSets(); ++i) {
     for (Attribute::AttrKind TypedAttr :
-         {Attribute::ByVal, Attribute::StructRet, Attribute::ByRef}) {
+         {Attribute::ByVal, Attribute::StructRet, Attribute::ByRef,
+          Attribute::InAlloca}) {
       if (Attrs.hasAttribute(i, TypedAttr)) {
         if (Type *Ty = Attrs.getAttribute(i, TypedAttr).getValueAsType()) {
           Attrs = Attrs.replaceAttributeType(C, i, TypedAttr, TypeMap.get(Ty));
@@ -995,6 +1004,7 @@ Expected<Constant *> IRLinker::linkGlobalValueProto(GlobalValue *SGV,
     return linkAppendingVarProto(cast_or_null<GlobalVariable>(DGV),
                                  cast<GlobalVariable>(SGV));
 
+  bool NeedsRenaming = false;
   GlobalValue *NewGV;
   if (DGV && !ShouldLink) {
     NewGV = DGV;
@@ -1007,15 +1017,21 @@ Expected<Constant *> IRLinker::linkGlobalValueProto(GlobalValue *SGV,
 
     NewGV = copyGlobalValueProto(SGV, ShouldLink || ForIndirectSymbol);
     if (ShouldLink || !ForIndirectSymbol)
-      forceRenaming(NewGV, SGV->getName());
+      NeedsRenaming = true;
   }
 
   // Overloaded intrinsics have overloaded types names as part of their
   // names. If we renamed overloaded types we should rename the intrinsic
   // as well.
   if (Function *F = dyn_cast<Function>(NewGV))
-    if (auto Remangled = Intrinsic::remangleIntrinsicFunction(F))
+    if (auto Remangled = Intrinsic::remangleIntrinsicFunction(F)) {
+      NewGV->eraseFromParent();
       NewGV = Remangled.getValue();
+      NeedsRenaming = false;
+    }
+
+  if (NeedsRenaming)
+    forceRenaming(NewGV, SGV->getName());
 
   if (ShouldLink || ForIndirectSymbol) {
     if (const Comdat *SC = SGV->getComdat()) {
diff --git a/llvm/lib/MC/MCParser/AsmLexer.cpp b/llvm/lib/MC/MCParser/AsmLexer.cpp
index 1fa22ab000f08cba00a06cc99e6bd7ebba7a4894..dd481d46f7883c1b9a47e26bc9594cdef27c569d 100644
--- a/llvm/lib/MC/MCParser/AsmLexer.cpp
+++ b/llvm/lib/MC/MCParser/AsmLexer.cpp
@@ -659,6 +659,9 @@ size_t AsmLexer::peekTokens(MutableArrayRef<AsmToken> Buf,
 }
 
 bool AsmLexer::isAtStartOfComment(const char *Ptr) {
+  if (MAI.getRestrictCommentStringToStartOfStatement() && !IsAtStartOfStatement)
+    return false;
+
   StringRef CommentString = MAI.getCommentString();
 
   if (CommentString.size() == 1)
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 3ef51e69ab7e10c396d547790103b8c95c1a1c52..f2fb5714a6bc418b90bd0210d0475cf8b9773c84 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -168,6 +169,8 @@ private:
   /// List of forward directional labels for diagnosis at the end.
   SmallVector<std::tuple<SMLoc, CppHashInfoTy, MCSymbol *>, 4> DirLabels;
 
+  SmallSet<StringRef, 2> LTODiscardSymbols;
+
   /// AssemblerDialect. ~OU means unset value and use value provided by MAI.
   unsigned AssemblerDialect = ~0U;
 
@@ -235,6 +238,10 @@ public:
   }
   bool isParsingMSInlineAsm() override { return ParsingMSInlineAsm; }
 
+  bool discardLTOSymbol(StringRef Name) const override {
+    return LTODiscardSymbols.contains(Name);
+  }
+
   bool parseMSInlineAsm(void *AsmLoc, std::string &AsmString,
                         unsigned &NumOutputs, unsigned &NumInputs,
                         SmallVectorImpl<std::pair<void *,bool>> &OpDecls,
@@ -516,6 +523,7 @@ private:
     DK_ADDRSIG,
     DK_ADDRSIG_SYM,
     DK_PSEUDO_PROBE,
+    DK_LTO_DISCARD,
     DK_END
   };
 
@@ -682,6 +690,9 @@ private:
   // .pseudoprobe
   bool parseDirectivePseudoProbe();
 
+  // ".lto_discard"
+  bool parseDirectiveLTODiscard();
+
   // Directives to support address-significance tables.
   bool parseDirectiveAddrsig();
   bool parseDirectiveAddrsigSym();
@@ -892,6 +903,8 @@ bool AsmParser::enabledGenDwarfForAssembly() {
 }
 
 bool AsmParser::Run(bool NoInitialTextSection, bool NoFinalize) {
+  LTODiscardSymbols.clear();
+
   // Create the initial section, if requested.
   if (!NoInitialTextSection)
     Out.InitSections(false);
@@ -919,6 +932,8 @@ bool AsmParser::Run(bool NoInitialTextSection, bool NoFinalize) {
     (void)InsertResult;
   }
 
+  getTargetParser().onBeginOfFile();
+
   // While we have input, parse each statement.
   while (Lexer.isNot(AsmToken::Eof)) {
     ParseStatementInfo Info(&AsmStrRewrites);
@@ -1770,7 +1785,6 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
   StringMap<DirectiveKind>::const_iterator DirKindIt =
       DirectiveKindMap.find(IDVal.lower());
   DirectiveKind DirKind = (DirKindIt == DirectiveKindMap.end())
-
                               ? DK_NO_DIRECTIVE
                               : DirKindIt->getValue();
   switch (DirKind) {
@@ -1868,6 +1882,9 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
       Lex();
     }
 
+    if (discardLTOSymbol(IDVal))
+      return false;
+
     getTargetParser().doBeforeLabelEmit(Sym);
 
     // Emit the label.
@@ -2208,6 +2225,8 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
       return parseDirectiveAddrsigSym();
     case DK_PSEUDO_PROBE:
       return parseDirectivePseudoProbe();
+    case DK_LTO_DISCARD:
+      return parseDirectiveLTODiscard();
     }
 
     return Error(IDLoc, "unknown directive");
@@ -2852,6 +2871,9 @@ bool AsmParser::parseAssignment(StringRef Name, bool allow_redef,
     return false;
   }
 
+  if (discardLTOSymbol(Name))
+    return false;
+
   // Do the assignment.
   Out.emitAssignment(Sym, Value);
   if (NoDeadStrip)
@@ -4870,6 +4892,10 @@ bool AsmParser::parseDirectiveSymbolAttribute(MCSymbolAttr Attr) {
     SMLoc Loc = getTok().getLoc();
     if (parseIdentifier(Name))
       return Error(Loc, "expected identifier");
+
+    if (discardLTOSymbol(Name))
+      return false;
+
     MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
     // Assembler local symbols don't make any sense here. Complain loudly.
@@ -5493,6 +5519,7 @@ void AsmParser::initializeDirectiveKindMap() {
   DirectiveKindMap[".addrsig"] = DK_ADDRSIG;
   DirectiveKindMap[".addrsig_sym"] = DK_ADDRSIG_SYM;
   DirectiveKindMap[".pseudoprobe"] = DK_PSEUDO_PROBE;
+  DirectiveKindMap[".lto_discard"] = DK_LTO_DISCARD;
 }
 
 MCAsmMacro *AsmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
@@ -5806,6 +5833,27 @@ bool AsmParser::parseDirectivePseudoProbe() {
   return false;
 }
 
+/// parseDirectiveLTODiscard
+///  ::= ".lto_discard" [ identifier ( , identifier )* ]
+/// The LTO library emits this directive to discard non-prevailing symbols.
+/// We ignore symbol assignments and attribute changes for the specified
+/// symbols.
+bool AsmParser::parseDirectiveLTODiscard() {
+  auto ParseOp = [&]() -> bool {
+    StringRef Name;
+    SMLoc Loc = getTok().getLoc();
+    if (parseIdentifier(Name))
+      return Error(Loc, "expected identifier");
+    LTODiscardSymbols.insert(Name);
+    return false;
+  };
+
+  LTODiscardSymbols.clear();
+  if (parseMany(ParseOp))
+    return addErrorSuffix(" in directive");
+  return false;
+}
+
 // We are comparing pointers, but the pointers are relative to a single string.
 // Thus, this should always be deterministic.
 static int rewritesSort(const AsmRewrite *AsmRewriteA,
diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
index 5b3f0225bba977562031c9c471392e0189476652..70d69fc8dd32185d44a44c54582719e293da5a56 100644
--- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
@@ -182,6 +182,12 @@ bool ELFAsmParser::ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
       if (getParser().parseIdentifier(Name))
         return TokError("expected identifier in directive");
 
+      if (getParser().discardLTOSymbol(Name)) {
+        if (getLexer().is(AsmToken::EndOfStatement))
+          break;
+        continue;
+      }
+
       MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
 
       getStreamer().emitSymbolAttribute(Sym, Attr);
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index 4957ee7a0323eca613dee55d236fe9ce57fcda29..18114b10d5686ad9545458f55cda8d0ab3d5b0fa 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -1235,6 +1235,8 @@ bool MasmParser::Run(bool NoInitialTextSection, bool NoFinalize) {
     (void)InsertResult;
   }
 
+  getTargetParser().onBeginOfFile();
+
   // While we have input, parse each statement.
   while (Lexer.isNot(AsmToken::Eof) ||
          SrcMgr.getParentIncludeLoc(CurBuffer) != SMLoc()) {
diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp
index 55ada91857be4e1dea8249347e7732a3a8e2f77c..33971e5dc171b35022ce51872c12734dd3125f26 100644
--- a/llvm/lib/MC/MCSubtargetInfo.cpp
+++ b/llvm/lib/MC/MCSubtargetInfo.cpp
@@ -208,6 +208,8 @@ static FeatureBitset getFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS,
 void MCSubtargetInfo::InitMCProcessorInfo(StringRef CPU, StringRef TuneCPU,
                                           StringRef FS) {
   FeatureBits = getFeatures(CPU, TuneCPU, FS, ProcDesc, ProcFeatures);
+  FeatureString = std::string(FS);
+
   if (!TuneCPU.empty())
     CPUSchedModel = &getSchedModelForCPU(TuneCPU);
   else
@@ -217,6 +219,7 @@ void MCSubtargetInfo::InitMCProcessorInfo(StringRef CPU, StringRef TuneCPU,
 void MCSubtargetInfo::setDefaultFeatures(StringRef CPU, StringRef TuneCPU,
                                          StringRef FS) {
   FeatureBits = getFeatures(CPU, TuneCPU, FS, ProcDesc, ProcFeatures);
+  FeatureString = std::string(FS);
 }
 
 MCSubtargetInfo::MCSubtargetInfo(const Triple &TT, StringRef C, StringRef TC,
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index 25138d651b5bad2ce76095b595782d42417d112a..0eda8b5a3234aa657c77db2bc2dd4602b89ecdf3 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -899,7 +899,7 @@ void WasmObjectWriter::writeTableSection(ArrayRef<wasm::WasmTable> Tables) {
   for (const wasm::WasmTable &Table : Tables) {
     encodeULEB128(Table.Type.ElemType, W->OS);
     encodeULEB128(Table.Type.Limits.Flags, W->OS);
-    encodeULEB128(Table.Type.Limits.Initial, W->OS);
+    encodeULEB128(Table.Type.Limits.Minimum, W->OS);
     if (Table.Type.Limits.Flags & wasm::WASM_LIMITS_FLAG_HAS_MAX)
       encodeULEB128(Table.Type.Limits.Maximum, W->OS);
   }
@@ -1338,12 +1338,7 @@ void WasmObjectWriter::prepareImports(
         Import.Module = WS.getImportModule();
         Import.Field = WS.getImportName();
         Import.Kind = wasm::WASM_EXTERNAL_TABLE;
-        wasm::ValType ElemType = WS.getTableType();
-        Import.Table.ElemType = uint8_t(ElemType);
-        // FIXME: Extend table type to include limits? For now we don't specify
-        // a min or max which does not place any restrictions on the size of the
-        // imported table.
-        Import.Table.Limits = {wasm::WASM_LIMITS_FLAG_NONE, 0, 0};
+        Import.Table = WS.getTableType();
         Imports.push_back(Import);
         assert(WasmIndices.count(&WS) == 0);
         WasmIndices[&WS] = NumTableImports++;
@@ -1626,9 +1621,7 @@ uint64_t WasmObjectWriter::writeOneObject(MCAssembler &Asm,
         if (WS.isDefined()) {
           wasm::WasmTable Table;
           Table.Index = NumTableImports + Tables.size();
-          Table.Type.ElemType = static_cast<uint8_t>(WS.getTableType());
-          // FIXME: Work on custom limits is ongoing
-          Table.Type.Limits = {wasm::WASM_LIMITS_FLAG_NONE, 0, 0};
+          Table.Type = WS.getTableType();
           assert(WasmIndices.count(&WS) == 0);
           WasmIndices[&WS] = Table.Index;
           Tables.push_back(Table);
diff --git a/llvm/lib/MCA/Context.cpp b/llvm/lib/MCA/Context.cpp
index 250ebebefe7a38304cc9f303079cf2893bc436bb..8f5addbe671501b1a75bcfe0b2a6c699ed9ce94c 100644
--- a/llvm/lib/MCA/Context.cpp
+++ b/llvm/lib/MCA/Context.cpp
@@ -71,23 +71,16 @@ Context::createDefaultPipeline(const PipelineOptions &Opts, SourceMgr &SrcMgr) {
 std::unique_ptr<Pipeline>
 Context::createInOrderPipeline(const PipelineOptions &Opts, SourceMgr &SrcMgr) {
   const MCSchedModel &SM = STI.getSchedModel();
-  auto RCU = std::make_unique<RetireControlUnit>(SM);
   auto PRF = std::make_unique<RegisterFile>(SM, MRI, Opts.RegisterFileSize);
-  auto LSU = std::make_unique<LSUnit>(SM, Opts.LoadQueueSize,
-                                      Opts.StoreQueueSize, Opts.AssumeNoAlias);
 
   auto Entry = std::make_unique<EntryStage>(SrcMgr);
-  auto InOrderIssue = std::make_unique<InOrderIssueStage>(*RCU, *PRF, SM, STI);
-  auto Retire = std::make_unique<RetireStage>(*RCU, *PRF, *LSU);
+  auto InOrderIssue = std::make_unique<InOrderIssueStage>(*PRF, SM, STI);
 
   auto StagePipeline = std::make_unique<Pipeline>();
   StagePipeline->appendStage(std::move(Entry));
   StagePipeline->appendStage(std::move(InOrderIssue));
-  StagePipeline->appendStage(std::move(Retire));
 
-  addHardwareUnit(std::move(RCU));
   addHardwareUnit(std::move(PRF));
-  addHardwareUnit(std::move(LSU));
 
   return StagePipeline;
 }
diff --git a/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp b/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
index 11a24a6889f142ed3b44974be6bf48c5ead5b676..a48915dbed73530161f6df2394082572bc8cf66e 100644
--- a/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
+++ b/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
@@ -22,11 +22,48 @@
 namespace llvm {
 namespace mca {
 
+const unsigned WriteRef::INVALID_IID = std::numeric_limits<unsigned>::max();
+
+WriteRef::WriteRef(unsigned SourceIndex, WriteState *WS)
+    : IID(SourceIndex), WriteBackCycle(), WriteResID(), RegisterID(),
+      Write(WS) {}
+
+void WriteRef::commit() {
+  assert(Write && Write->isExecuted() && "Cannot commit before write back!");
+  Write = nullptr;
+}
+
+void WriteRef::notifyExecuted(unsigned Cycle) {
+  assert(Write && Write->isExecuted() && "Not executed!");
+  WriteBackCycle = Cycle;
+}
+
+bool WriteRef::hasKnownWriteBackCycle() const {
+  return isValid() && (!Write || Write->isExecuted());
+}
+
+bool WriteRef::isWriteZero() const {
+  assert(isValid() && "Invalid null WriteState found!");
+  return getWriteState()->isWriteZero();
+}
+
+unsigned WriteRef::getWriteResourceID() const {
+  if (Write)
+    return Write->getWriteResourceID();
+  return WriteResID;
+}
+
+MCPhysReg WriteRef::getRegisterID() const {
+  if (Write)
+    return Write->getRegisterID();
+  return RegisterID;
+}
+
 RegisterFile::RegisterFile(const MCSchedModel &SM, const MCRegisterInfo &mri,
                            unsigned NumRegs)
     : MRI(mri),
       RegisterMappings(mri.getNumRegs(), {WriteRef(), RegisterRenamingInfo()}),
-      ZeroRegisters(mri.getNumRegs(), false) {
+      ZeroRegisters(mri.getNumRegs(), false), CurrentCycle() {
   initialize(SM, NumRegs);
 }
 
@@ -63,6 +100,43 @@ void RegisterFile::cycleStart() {
     RMT.NumMoveEliminated = 0;
 }
 
+void RegisterFile::onInstructionExecuted(Instruction *IS) {
+  assert(IS && IS->isExecuted() && "Unexpected internal state found!");
+  for (WriteState &WS : IS->getDefs()) {
+    if (WS.isEliminated())
+      return;
+
+    MCPhysReg RegID = WS.getRegisterID();
+    assert(RegID != 0 && "A write of an invalid register?");
+    assert(WS.getCyclesLeft() != UNKNOWN_CYCLES &&
+           "The number of cycles should be known at this point!");
+    assert(WS.getCyclesLeft() <= 0 && "Invalid cycles left for this write!");
+
+    MCPhysReg RenameAs = RegisterMappings[RegID].second.RenameAs;
+    if (RenameAs && RenameAs != RegID)
+      RegID = RenameAs;
+
+    WriteRef &WR = RegisterMappings[RegID].first;
+    if (WR.getWriteState() == &WS)
+      WR.notifyExecuted(CurrentCycle);
+
+    for (MCSubRegIterator I(RegID, &MRI); I.isValid(); ++I) {
+      WriteRef &OtherWR = RegisterMappings[*I].first;
+      if (OtherWR.getWriteState() == &WS)
+        OtherWR.notifyExecuted(CurrentCycle);
+    }
+
+    if (!WS.clearsSuperRegisters())
+      continue;
+
+    for (MCSuperRegIterator I(RegID, &MRI); I.isValid(); ++I) {
+      WriteRef &OtherWR = RegisterMappings[*I].first;
+      if (OtherWR.getWriteState() == &WS)
+        OtherWR.notifyExecuted(CurrentCycle);
+    }
+  }
+}
+
 void RegisterFile::addRegisterFile(const MCRegisterFileDesc &RF,
                                    ArrayRef<MCRegisterCostEntry> Entries) {
   // A default register file is always allocated at index #0. That register file
@@ -261,12 +335,12 @@ void RegisterFile::removeRegisterWrite(
 
   WriteRef &WR = RegisterMappings[RegID].first;
   if (WR.getWriteState() == &WS)
-    WR.invalidate();
+    WR.commit();
 
   for (MCSubRegIterator I(RegID, &MRI); I.isValid(); ++I) {
     WriteRef &OtherWR = RegisterMappings[*I].first;
     if (OtherWR.getWriteState() == &WS)
-      OtherWR.invalidate();
+      OtherWR.commit();
   }
 
   if (!WS.clearsSuperRegisters())
@@ -275,7 +349,7 @@ void RegisterFile::removeRegisterWrite(
   for (MCSuperRegIterator I(RegID, &MRI); I.isValid(); ++I) {
     WriteRef &OtherWR = RegisterMappings[*I].first;
     if (OtherWR.getWriteState() == &WS)
-      OtherWR.invalidate();
+      OtherWR.commit();
   }
 }
 
@@ -344,8 +418,25 @@ bool RegisterFile::tryEliminateMove(WriteState &WS, ReadState &RS) {
   return true;
 }
 
-void RegisterFile::collectWrites(const ReadState &RS,
-                                 SmallVectorImpl<WriteRef> &Writes) const {
+unsigned WriteRef::getWriteBackCycle() const {
+  assert(hasKnownWriteBackCycle() && "Instruction not executed!");
+  assert((!Write || Write->getCyclesLeft() <= 0) &&
+         "Inconsistent state found!");
+  return WriteBackCycle;
+}
+
+unsigned RegisterFile::getElapsedCyclesFromWriteBack(const WriteRef &WR) const {
+  assert(WR.hasKnownWriteBackCycle() && "Write hasn't been committed yet!");
+  return CurrentCycle - WR.getWriteBackCycle();
+}
+
+void RegisterFile::collectWrites(
+    const MCSubtargetInfo &STI, const ReadState &RS,
+    SmallVectorImpl<WriteRef> &Writes,
+    SmallVectorImpl<WriteRef> &CommittedWrites) const {
+  const ReadDescriptor &RD = RS.getDescriptor();
+  const MCSchedModel &SM = STI.getSchedModel();
+  const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
   MCPhysReg RegID = RS.getRegisterID();
   assert(RegID && RegID < RegisterMappings.size());
   LLVM_DEBUG(dbgs() << "RegisterFile: collecting writes for register "
@@ -357,14 +448,32 @@ void RegisterFile::collectWrites(const ReadState &RS,
     RegID = RRI.AliasRegID;
 
   const WriteRef &WR = RegisterMappings[RegID].first;
-  if (WR.isValid())
+  if (WR.getWriteState()) {
     Writes.push_back(WR);
+  } else if (WR.hasKnownWriteBackCycle()) {
+    unsigned WriteResID = WR.getWriteResourceID();
+    int ReadAdvance = STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
+    if (ReadAdvance < 0) {
+      unsigned Elapsed = getElapsedCyclesFromWriteBack(WR);
+      if (Elapsed < static_cast<unsigned>(-ReadAdvance))
+        CommittedWrites.push_back(WR);
+    }
+  }
 
   // Handle potential partial register updates.
   for (MCSubRegIterator I(RegID, &MRI); I.isValid(); ++I) {
     const WriteRef &WR = RegisterMappings[*I].first;
-    if (WR.isValid())
+    if (WR.getWriteState()) {
       Writes.push_back(WR);
+    } else if (WR.hasKnownWriteBackCycle()) {
+      unsigned WriteResID = WR.getWriteResourceID();
+      int ReadAdvance = STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
+      if (ReadAdvance < 0) {
+        unsigned Elapsed = getElapsedCyclesFromWriteBack(WR);
+        if (Elapsed < static_cast<unsigned>(-ReadAdvance))
+          CommittedWrites.push_back(WR);
+      }
+    }
   }
 
   // Remove duplicate entries and resize the input vector.
@@ -398,21 +507,34 @@ void RegisterFile::addRegisterRead(ReadState &RS,
     RS.setReadZero();
 
   SmallVector<WriteRef, 4> DependentWrites;
-  collectWrites(RS, DependentWrites);
-  RS.setDependentWrites(DependentWrites.size());
+  SmallVector<WriteRef, 4> CompletedWrites;
+  collectWrites(STI, RS, DependentWrites, CompletedWrites);
+  RS.setDependentWrites(DependentWrites.size() + CompletedWrites.size());
 
   // We know that this read depends on all the writes in DependentWrites.
   // For each write, check if we have ReadAdvance information, and use it
-  // to figure out in how many cycles this read becomes available.
+  // to figure out in how many cycles this read will be available.
   const ReadDescriptor &RD = RS.getDescriptor();
   const MCSchedModel &SM = STI.getSchedModel();
   const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
   for (WriteRef &WR : DependentWrites) {
+    unsigned WriteResID = WR.getWriteResourceID();
     WriteState &WS = *WR.getWriteState();
-    unsigned WriteResID = WS.getWriteResourceID();
     int ReadAdvance = STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
     WS.addUser(WR.getSourceIndex(), &RS, ReadAdvance);
   }
+
+  for (WriteRef &WR : CompletedWrites) {
+    unsigned WriteResID = WR.getWriteResourceID();
+    assert(WR.hasKnownWriteBackCycle() && "Invalid write!");
+    assert(STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID) < 0);
+    unsigned ReadAdvance = static_cast<unsigned>(
+        -STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID));
+    unsigned Elapsed = getElapsedCyclesFromWriteBack(WR);
+    assert(Elapsed < ReadAdvance && "Should not have been added to the set!");
+    RS.writeStartEvent(WR.getSourceIndex(), WR.getRegisterID(),
+                       ReadAdvance - Elapsed);
+  }
 }
 
 unsigned RegisterFile::isAvailable(ArrayRef<MCPhysReg> Regs) const {
@@ -463,6 +585,14 @@ unsigned RegisterFile::isAvailable(ArrayRef<MCPhysReg> Regs) const {
 }
 
 #ifndef NDEBUG
+void WriteRef::dump() const {
+  dbgs() << "IID=" << getSourceIndex() << ' ';
+  if (isValid())
+    getWriteState()->dump();
+  else
+    dbgs() << "(null)";
+}
+
 void RegisterFile::dump() const {
   for (unsigned I = 0, E = MRI.getNumRegs(); I < E; ++I) {
     const RegisterMapping &RM = RegisterMappings[I];
diff --git a/llvm/lib/MCA/HardwareUnits/RetireControlUnit.cpp b/llvm/lib/MCA/HardwareUnits/RetireControlUnit.cpp
index 812109f26684f59f4ce21589f5476a6ef72885a3..9297f0c4fd7bf1b9cd23830ab9c17fa05619e91a 100644
--- a/llvm/lib/MCA/HardwareUnits/RetireControlUnit.cpp
+++ b/llvm/lib/MCA/HardwareUnits/RetireControlUnit.cpp
@@ -23,6 +23,8 @@ RetireControlUnit::RetireControlUnit(const MCSchedModel &SM)
     : NextAvailableSlotIdx(0), CurrentInstructionSlotIdx(0),
       AvailableEntries(SM.isOutOfOrder() ? SM.MicroOpBufferSize : 0),
       MaxRetirePerCycle(0) {
+  assert(SM.isOutOfOrder() &&
+         "RetireControlUnit is not available for in-order processors");
   // Check if the scheduling model provides extra information about the machine
   // processor. If so, then use that information to set the reorder buffer size
   // and the maximum number of instructions retired per cycle.
@@ -33,17 +35,12 @@ RetireControlUnit::RetireControlUnit(const MCSchedModel &SM)
     MaxRetirePerCycle = EPI.MaxRetirePerCycle;
   }
   NumROBEntries = AvailableEntries;
-  if (!SM.isOutOfOrder() && !NumROBEntries)
-    return;
   assert(NumROBEntries && "Invalid reorder buffer size!");
   Queue.resize(2 * NumROBEntries);
 }
 
 // Reserves a number of slots, and returns a new token.
 unsigned RetireControlUnit::dispatch(const InstRef &IR) {
-  if (!NumROBEntries)
-    return UnhandledTokenID;
-
   const Instruction &Inst = *IR.getInstruction();
   unsigned Entries = normalizeQuantity(Inst.getNumMicroOps());
   assert((AvailableEntries >= Entries) && "Reorder Buffer unavailable!");
diff --git a/llvm/lib/MCA/Instruction.cpp b/llvm/lib/MCA/Instruction.cpp
index e5f2c4fd1eec81801ad76fdd92d482edd416475d..e658b869a67eb2df5ef24db5391c1f43b54098da 100644
--- a/llvm/lib/MCA/Instruction.cpp
+++ b/llvm/lib/MCA/Instruction.cpp
@@ -27,7 +27,8 @@ void WriteState::writeStartEvent(unsigned IID, MCPhysReg RegID,
   DependentWrite = nullptr;
 }
 
-void ReadState::writeStartEvent(unsigned IID, MCPhysReg RegID, unsigned Cycles) {
+void ReadState::writeStartEvent(unsigned IID, MCPhysReg RegID,
+                                unsigned Cycles) {
   assert(DependentWrites);
   assert(CyclesLeft == UNKNOWN_CYCLES);
 
@@ -125,14 +126,6 @@ void WriteState::dump() const {
   dbgs() << "{ OpIdx=" << WD->OpIndex << ", Lat=" << getLatency() << ", RegID "
          << getRegisterID() << ", Cycles Left=" << getCyclesLeft() << " }";
 }
-
-void WriteRef::dump() const {
-  dbgs() << "IID=" << getSourceIndex() << ' ';
-  if (isValid())
-    getWriteState()->dump();
-  else
-    dbgs() << "(null)";
-}
 #endif
 
 const CriticalDependency &Instruction::computeCriticalRegDep() {
@@ -248,7 +241,5 @@ void Instruction::cycleEvent() {
     Stage = IS_EXECUTED;
 }
 
-const unsigned WriteRef::INVALID_IID = std::numeric_limits<unsigned>::max();
-
 } // namespace mca
 } // namespace llvm
diff --git a/llvm/lib/MCA/Stages/DispatchStage.cpp b/llvm/lib/MCA/Stages/DispatchStage.cpp
index 3a3d82259160a93dc370e72ad31b95253fe6a105..c44018dc30f46a551a315ba06435807175e2fe9a 100644
--- a/llvm/lib/MCA/Stages/DispatchStage.cpp
+++ b/llvm/lib/MCA/Stages/DispatchStage.cpp
@@ -136,8 +136,8 @@ Error DispatchStage::dispatch(InstRef IR) {
 }
 
 Error DispatchStage::cycleStart() {
-  PRF.cycleStart();
-
+  // The retire stage is responsible for calling method `cycleStart`
+  // on the PRF.
   if (!CarryOver) {
     AvailableEntries = DispatchWidth;
     return ErrorSuccess();
diff --git a/llvm/lib/MCA/Stages/InOrderIssueStage.cpp b/llvm/lib/MCA/Stages/InOrderIssueStage.cpp
index a675b92e106866422be7343bdf43f915431e7c3b..a32319b4b39015d3d75f15705dbb941c95a56b23 100644
--- a/llvm/lib/MCA/Stages/InOrderIssueStage.cpp
+++ b/llvm/lib/MCA/Stages/InOrderIssueStage.cpp
@@ -29,15 +29,19 @@ namespace llvm {
 namespace mca {
 
 bool InOrderIssueStage::hasWorkToComplete() const {
-  return !IssuedInst.empty() || StalledInst;
+  return !IssuedInst.empty() || StalledInst || CarriedOver;
 }
 
 bool InOrderIssueStage::isAvailable(const InstRef &IR) const {
+  if (StalledInst || CarriedOver)
+    return false;
+
   const Instruction &Inst = *IR.getInstruction();
   unsigned NumMicroOps = Inst.getNumMicroOps();
   const InstrDesc &Desc = Inst.getDesc();
 
-  if (Bandwidth < NumMicroOps)
+  bool ShouldCarryOver = NumMicroOps > SM.IssueWidth;
+  if (Bandwidth < NumMicroOps && !ShouldCarryOver)
     return false;
 
   // Instruction with BeginGroup must be the first instruction to be issued in a
@@ -57,6 +61,32 @@ static bool hasResourceHazard(const ResourceManager &RM, const InstRef &IR) {
   return false;
 }
 
+static unsigned findLastWriteBackCycle(const InstRef &IR) {
+  unsigned LastWBCycle = 0;
+  for (const WriteState &WS : IR.getInstruction()->getDefs()) {
+    int CyclesLeft = WS.getCyclesLeft();
+    if (CyclesLeft == UNKNOWN_CYCLES)
+      CyclesLeft = WS.getLatency();
+    if (CyclesLeft < 0)
+      CyclesLeft = 0;
+    LastWBCycle = std::max(LastWBCycle, (unsigned)CyclesLeft);
+  }
+  return LastWBCycle;
+}
+
+static unsigned findFirstWriteBackCycle(const InstRef &IR) {
+  unsigned FirstWBCycle = ~0U;
+  for (const WriteState &WS : IR.getInstruction()->getDefs()) {
+    int CyclesLeft = WS.getCyclesLeft();
+    if (CyclesLeft == UNKNOWN_CYCLES)
+      CyclesLeft = WS.getLatency();
+    if (CyclesLeft < 0)
+      CyclesLeft = 0;
+    FirstWBCycle = std::min(FirstWBCycle, (unsigned)CyclesLeft);
+  }
+  return FirstWBCycle;
+}
+
 /// Return a number of cycles left until register requirements of the
 /// instructions are met.
 static unsigned checkRegisterHazard(const RegisterFile &PRF,
@@ -65,12 +95,13 @@ static unsigned checkRegisterHazard(const RegisterFile &PRF,
                                     const InstRef &IR) {
   unsigned StallCycles = 0;
   SmallVector<WriteRef, 4> Writes;
+  SmallVector<WriteRef, 4> CommittedWrites;
 
   for (const ReadState &RS : IR.getInstruction()->getUses()) {
     const ReadDescriptor &RD = RS.getDescriptor();
     const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
 
-    PRF.collectWrites(RS, Writes);
+    PRF.collectWrites(STI, RS, Writes, CommittedWrites);
     for (const WriteRef &WR : Writes) {
       const WriteState *WS = WR.getWriteState();
       unsigned WriteResID = WS->getWriteResourceID();
@@ -92,6 +123,19 @@ static unsigned checkRegisterHazard(const RegisterFile &PRF,
       }
     }
     Writes.clear();
+
+    for (const WriteRef &WR : CommittedWrites) {
+      unsigned WriteResID = WR.getWriteResourceID();
+      assert(!WR.getWriteState() && "Should be already committed!");
+      assert(WR.hasKnownWriteBackCycle() && "Invalid write!");
+      assert(STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID) < 0);
+      unsigned ReadAdvance = static_cast<unsigned>(
+          -STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID));
+      unsigned Elapsed = PRF.getElapsedCyclesFromWriteBack(WR);
+      assert(Elapsed < ReadAdvance && "Should not have been added to the set!");
+      unsigned CyclesLeft = (ReadAdvance - Elapsed);
+      StallCycles = std::max(StallCycles, CyclesLeft);
+    }
   }
 
   return StallCycles;
@@ -116,6 +160,14 @@ bool InOrderIssueStage::canExecute(const InstRef &IR,
         HWStallEvent(HWStallEvent::DispatchGroupStall, IR));
     notifyEvent<HWPressureEvent>(
         HWPressureEvent(HWPressureEvent::RESOURCES, IR));
+  } else if (LastWriteBackCycle) {
+    if (!IR.getInstruction()->getDesc().RetireOOO) {
+      unsigned NextWriteBackCycle = findFirstWriteBackCycle(IR);
+      // Delay the instruction to ensure that writes occur in program order
+      if (NextWriteBackCycle < LastWriteBackCycle) {
+        *StallCycles = LastWriteBackCycle - NextWriteBackCycle;
+      }
+    }
   }
 
   return *StallCycles == 0;
@@ -134,7 +186,7 @@ static void addRegisterReadWrite(RegisterFile &PRF, Instruction &IS,
     PRF.addRegisterWrite(WriteRef(SourceIndex, &WS), UsedRegs);
 }
 
-static void notifyInstructionExecute(
+static void notifyInstructionIssue(
     const InstRef &IR,
     const SmallVectorImpl<std::pair<ResourceRef, ResourceCycles>> &UsedRes,
     const Stage &S) {
@@ -157,28 +209,11 @@ static void notifyInstructionDispatch(const InstRef &IR, unsigned Ops,
 }
 
 llvm::Error InOrderIssueStage::execute(InstRef &IR) {
-  Instruction &IS = *IR.getInstruction();
-  const InstrDesc &Desc = IS.getDesc();
-
-  unsigned RCUTokenID = RetireControlUnit::UnhandledTokenID;
-  if (!Desc.RetireOOO)
-    RCUTokenID = RCU.dispatch(IR);
-  IS.dispatch(RCUTokenID);
-
-  if (Desc.EndGroup) {
-    Bandwidth = 0;
-  } else {
-    unsigned NumMicroOps = IR.getInstruction()->getNumMicroOps();
-    assert(Bandwidth >= NumMicroOps);
-    Bandwidth -= NumMicroOps;
-  }
-
   if (llvm::Error E = tryIssue(IR, &StallCyclesLeft))
     return E;
 
   if (StallCyclesLeft) {
     StalledInst = IR;
-    Bandwidth = 0;
   }
 
   return llvm::ErrorSuccess();
@@ -187,20 +222,26 @@ llvm::Error InOrderIssueStage::execute(InstRef &IR) {
 llvm::Error InOrderIssueStage::tryIssue(InstRef &IR, unsigned *StallCycles) {
   Instruction &IS = *IR.getInstruction();
   unsigned SourceIndex = IR.getSourceIndex();
+  const InstrDesc &Desc = IS.getDesc();
 
   if (!canExecute(IR, StallCycles)) {
     LLVM_DEBUG(dbgs() << "[E] Stalled #" << IR << " for " << *StallCycles
                       << " cycles\n");
+    Bandwidth = 0;
     return llvm::ErrorSuccess();
   }
 
+  unsigned RCUTokenID = RetireControlUnit::UnhandledTokenID;
+  IS.dispatch(RCUTokenID);
+
   SmallVector<unsigned, 4> UsedRegs(PRF.getNumRegisterFiles());
   addRegisterReadWrite(PRF, IS, SourceIndex, STI, UsedRegs);
 
-  notifyInstructionDispatch(IR, IS.getDesc().NumMicroOps, UsedRegs, *this);
+  unsigned NumMicroOps = IS.getNumMicroOps();
+  notifyInstructionDispatch(IR, NumMicroOps, UsedRegs, *this);
 
   SmallVector<std::pair<ResourceRef, ResourceCycles>, 4> UsedResources;
-  RM->issueInstruction(IS.getDesc(), UsedResources);
+  RM->issueInstruction(Desc, UsedResources);
   IS.execute(SourceIndex);
 
   // Replace resource masks with valid resource processor IDs.
@@ -208,15 +249,29 @@ llvm::Error InOrderIssueStage::tryIssue(InstRef &IR, unsigned *StallCycles) {
     uint64_t Mask = Use.first.first;
     Use.first.first = RM->resolveResourceMask(Mask);
   }
-  notifyInstructionExecute(IR, UsedResources, *this);
+  notifyInstructionIssue(IR, UsedResources, *this);
+
+  bool ShouldCarryOver = NumMicroOps > Bandwidth;
+  if (ShouldCarryOver) {
+    CarryOver = NumMicroOps - Bandwidth;
+    CarriedOver = IR;
+    Bandwidth = 0;
+    NumIssued += Bandwidth;
+    LLVM_DEBUG(dbgs() << "[N] Carry over #" << IR << " \n");
+  } else {
+    NumIssued += NumMicroOps;
+    Bandwidth = Desc.EndGroup ? 0 : Bandwidth - NumMicroOps;
+  }
 
   IssuedInst.push_back(IR);
-  ++NumIssued;
+
+  if (!IR.getInstruction()->getDesc().RetireOOO)
+    LastWriteBackCycle = findLastWriteBackCycle(IR);
 
   return llvm::ErrorSuccess();
 }
 
-llvm::Error InOrderIssueStage::updateIssuedInst() {
+void InOrderIssueStage::updateIssuedInst() {
   // Update other instructions. Executed instructions will be retired during the
   // next cycle.
   unsigned NumExecuted = 0;
@@ -232,36 +287,74 @@ llvm::Error InOrderIssueStage::updateIssuedInst() {
       ++I;
       continue;
     }
+
+    PRF.onInstructionExecuted(&IS);
     notifyEvent<HWInstructionEvent>(
         HWInstructionEvent(HWInstructionEvent::Executed, IR));
-
     LLVM_DEBUG(dbgs() << "[E] Instruction #" << IR << " is executed\n");
     ++NumExecuted;
+
+    retireInstruction(*I);
+
     std::iter_swap(I, E - NumExecuted);
   }
 
-  // Retire instructions in the next cycle
-  if (NumExecuted) {
-    for (auto I = IssuedInst.end() - NumExecuted, E = IssuedInst.end(); I != E;
-         ++I) {
-      if (llvm::Error E = moveToTheNextStage(*I))
-        return E;
-    }
+  if (NumExecuted)
     IssuedInst.resize(IssuedInst.size() - NumExecuted);
+}
+
+void InOrderIssueStage::updateCarriedOver() {
+  if (!CarriedOver)
+    return;
+
+  assert(!StalledInst && "A stalled instruction cannot be carried over.");
+
+  if (CarryOver > Bandwidth) {
+    CarryOver -= Bandwidth;
+    Bandwidth = 0;
+    LLVM_DEBUG(dbgs() << "[N] Carry over (" << CarryOver << "uops left) #"
+               << CarriedOver << " \n");
+    return;
   }
 
-  return llvm::ErrorSuccess();
+  LLVM_DEBUG(dbgs() << "[N] Carry over (complete) #" << CarriedOver
+             << " \n");
+
+  if (CarriedOver.getInstruction()->getDesc().EndGroup)
+    Bandwidth = 0;
+  else
+    Bandwidth -= CarryOver;
+
+  CarriedOver = InstRef();
+  CarryOver = 0;
+}
+
+void InOrderIssueStage::retireInstruction(InstRef &IR) {
+  Instruction &IS = *IR.getInstruction();
+  IS.retire();
+
+  llvm::SmallVector<unsigned, 4> FreedRegs(PRF.getNumRegisterFiles());
+  for (const WriteState &WS : IS.getDefs())
+    PRF.removeRegisterWrite(WS, FreedRegs);
+
+  notifyEvent<HWInstructionEvent>(HWInstructionRetiredEvent(IR, FreedRegs));
+  LLVM_DEBUG(dbgs() << "[E] Retired #" << IR << " \n");
 }
 
 llvm::Error InOrderIssueStage::cycleStart() {
   NumIssued = 0;
+  Bandwidth = SM.IssueWidth;
+
+  PRF.cycleStart();
 
   // Release consumed resources.
   SmallVector<ResourceRef, 4> Freed;
   RM->cycleEvent(Freed);
 
-  if (llvm::Error E = updateIssuedInst())
-    return E;
+  updateIssuedInst();
+
+  // Continue to issue the instruction carried over from the previous cycle
+  updateCarriedOver();
 
   // Issue instructions scheduled for this cycle
   if (!StallCyclesLeft && StalledInst) {
@@ -272,7 +365,6 @@ llvm::Error InOrderIssueStage::cycleStart() {
   if (!StallCyclesLeft) {
     StalledInst.invalidate();
     assert(NumIssued <= SM.IssueWidth && "Overflow.");
-    Bandwidth = SM.IssueWidth - NumIssued;
   } else {
     // The instruction is still stalled, cannot issue any new instructions in
     // this cycle.
@@ -283,8 +375,14 @@ llvm::Error InOrderIssueStage::cycleStart() {
 }
 
 llvm::Error InOrderIssueStage::cycleEnd() {
+  PRF.cycleEnd();
+
   if (StallCyclesLeft > 0)
     --StallCyclesLeft;
+
+  if (LastWriteBackCycle > 0)
+    --LastWriteBackCycle;
+
   return llvm::ErrorSuccess();
 }
 
diff --git a/llvm/lib/MCA/Stages/RetireStage.cpp b/llvm/lib/MCA/Stages/RetireStage.cpp
index 6b24b48065d527ef88b1ef35d27f25878ca2d1e0..00dbb4b0347a9bbfda1bbc851fe3dca6608b10cf 100644
--- a/llvm/lib/MCA/Stages/RetireStage.cpp
+++ b/llvm/lib/MCA/Stages/RetireStage.cpp
@@ -23,6 +23,8 @@ namespace llvm {
 namespace mca {
 
 llvm::Error RetireStage::cycleStart() {
+  PRF.cycleStart();
+
   const unsigned MaxRetirePerCycle = RCU.getMaxRetirePerCycle();
   unsigned NumRetired = 0;
   while (!RCU.isEmpty()) {
@@ -36,26 +38,22 @@ llvm::Error RetireStage::cycleStart() {
     NumRetired++;
   }
 
-  // Retire instructions that are not controlled by the RCU
-  for (InstRef &IR : RetireInst) {
-    IR.getInstruction()->retire();
-    notifyInstructionRetired(IR);
-  }
-  RetireInst.resize(0);
+  return llvm::ErrorSuccess();
+}
 
+llvm::Error RetireStage::cycleEnd() {
+  PRF.cycleEnd();
   return llvm::ErrorSuccess();
 }
 
 llvm::Error RetireStage::execute(InstRef &IR) {
   Instruction &IS = *IR.getInstruction();
 
+  PRF.onInstructionExecuted(&IS);
   unsigned TokenID = IS.getRCUTokenID();
-  if (TokenID != RetireControlUnit::UnhandledTokenID) {
-    RCU.onInstructionExecuted(TokenID);
-    return llvm::ErrorSuccess();
-  }
+  assert(TokenID != RetireControlUnit::UnhandledTokenID);
+  RCU.onInstructionExecuted(TokenID);
 
-  RetireInst.push_back(IR);
   return llvm::ErrorSuccess();
 }
 
diff --git a/llvm/lib/Object/Binary.cpp b/llvm/lib/Object/Binary.cpp
index e741cbba2882a67830881c235c14305b2aae90aa..71c044630223569f32781c76054be91af950cda0 100644
--- a/llvm/lib/Object/Binary.cpp
+++ b/llvm/lib/Object/Binary.cpp
@@ -97,7 +97,7 @@ Expected<std::unique_ptr<Binary>> object::createBinary(MemoryBufferRef Buffer,
 Expected<OwningBinary<Binary>>
 object::createBinary(StringRef Path, LLVMContext *Context, bool InitContent) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
-      MemoryBuffer::getFileOrSTDIN(Path, /*FileSize=*/-1,
+      MemoryBuffer::getFileOrSTDIN(Path, /*IsText=*/false,
                                    /*RequiresNullTerminator=*/false);
   if (std::error_code EC = FileOrErr.getError())
     return errorCodeToError(EC);
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index 30225592628940ea114243a3c1e7a227138687fe..498e99bdb828d663c4293cdcb6d0c87fef50632a 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -1836,6 +1836,8 @@ MachOObjectFile::getSymbolType(DataRefImpl Symb) const {
       if (!SecOrError)
         return SecOrError.takeError();
       section_iterator Sec = *SecOrError;
+      if (Sec == section_end())
+        return SymbolRef::ST_Other;
       if (Sec->isData() || Sec->isBSS())
         return SymbolRef::ST_Data;
       return SymbolRef::ST_Function;
diff --git a/llvm/lib/Object/WasmObjectFile.cpp b/llvm/lib/Object/WasmObjectFile.cpp
index 1a0958ad039d4afaca65b46d896a50141473e876..f759a61161987ca59e43531f52caf25268267528 100644
--- a/llvm/lib/Object/WasmObjectFile.cpp
+++ b/llvm/lib/Object/WasmObjectFile.cpp
@@ -208,7 +208,7 @@ static Error readInitExpr(wasm::WasmInitExpr &Expr,
 static wasm::WasmLimits readLimits(WasmObjectFile::ReadContext &Ctx) {
   wasm::WasmLimits Result;
   Result.Flags = readVaruint32(Ctx);
-  Result.Initial = readVaruint64(Ctx);
+  Result.Minimum = readVaruint64(Ctx);
   if (Result.Flags & wasm::WASM_LIMITS_FLAG_HAS_MAX)
     Result.Maximum = readVaruint64(Ctx);
   return Result;
diff --git a/llvm/lib/Object/XCOFFObjectFile.cpp b/llvm/lib/Object/XCOFFObjectFile.cpp
index a16a458168d4289a15bee4ae6d60a6da6b21ff0d..74b10faad68f1d9df1a4286fdea7a9a94e9dbf89 100644
--- a/llvm/lib/Object/XCOFFObjectFile.cpp
+++ b/llvm/lib/Object/XCOFFObjectFile.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Object/XCOFFObjectFile.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Support/DataExtractor.h"
 #include <cstddef>
@@ -296,9 +297,7 @@ uint64_t XCOFFObjectFile::getSectionAlignment(DataRefImpl Sec) const {
 }
 
 bool XCOFFObjectFile::isSectionCompressed(DataRefImpl Sec) const {
-  bool Result = false;
-  llvm_unreachable("Not yet implemented!");
-  return Result;
+  return false;
 }
 
 bool XCOFFObjectFile::isSectionText(DataRefImpl Sec) const {
@@ -458,6 +457,22 @@ Expected<uint64_t> XCOFFObjectFile::getStartAddress() const {
   return 0;
 }
 
+StringRef XCOFFObjectFile::mapDebugSectionName(StringRef Name) const {
+  return StringSwitch<StringRef>(Name)
+      .Case("dwinfo", "debug_info")
+      .Case("dwline", "debug_line")
+      .Case("dwpbnms", "debug_pubnames")
+      .Case("dwpbtyp", "debug_pubtypes")
+      .Case("dwarnge", "debug_aranges")
+      .Case("dwabrev", "debug_abbrev")
+      .Case("dwstr", "debug_str")
+      .Case("dwrnges", "debug_ranges")
+      .Case("dwloc", "debug_loc")
+      .Case("dwframe", "debug_frame")
+      .Case("dwmac", "debug_macinfo")
+      .Default(Name);
+}
+
 size_t XCOFFObjectFile::getFileHeaderSize() const {
   return is64Bit() ? sizeof(XCOFFFileHeader64) : sizeof(XCOFFFileHeader32);
 }
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 1ae09647593c7ad294458543e129c67d438a2f40..cdeb37f9a6d26e5811fe4652bbe10aaebac035a8 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -155,9 +155,9 @@ void ScalarEnumerationTraits<ELFYAML::ELF_NT>::enumeration(
   ECase(NT_FREEBSD_PROCSTAT_PSSTRINGS);
   ECase(NT_FREEBSD_PROCSTAT_AUXV);
   // AMD specific notes. (Code Object V2)
-  ECase(NT_AMD_AMDGPU_HSA_METADATA);
-  ECase(NT_AMD_AMDGPU_ISA);
-  ECase(NT_AMD_AMDGPU_PAL_METADATA);
+  ECase(NT_AMD_HSA_METADATA);
+  ECase(NT_AMD_HSA_ISA_NAME);
+  ECase(NT_AMD_PAL_METADATA);
   // AMDGPU specific notes. (Code Object V3)
   ECase(NT_AMDGPU_METADATA);
 #undef ECase
@@ -459,6 +459,7 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
     BCase(EF_HEXAGON_MACH_V66);
     BCase(EF_HEXAGON_MACH_V67);
     BCase(EF_HEXAGON_MACH_V67T);
+    BCase(EF_HEXAGON_MACH_V68);
     BCase(EF_HEXAGON_ISA_V2);
     BCase(EF_HEXAGON_ISA_V3);
     BCase(EF_HEXAGON_ISA_V4);
@@ -469,6 +470,7 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
     BCase(EF_HEXAGON_ISA_V65);
     BCase(EF_HEXAGON_ISA_V66);
     BCase(EF_HEXAGON_ISA_V67);
+    BCase(EF_HEXAGON_ISA_V68);
     break;
   case ELF::EM_AVR:
     BCase(EF_AVR_ARCH_AVR1);
@@ -544,8 +546,33 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1031, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1032, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1033, EF_AMDGPU_MACH);
-    BCase(EF_AMDGPU_XNACK);
-    BCase(EF_AMDGPU_SRAM_ECC);
+    switch (Object->Header.ABIVersion) {
+    default:
+      // ELFOSABI_AMDGPU_PAL, ELFOSABI_AMDGPU_MESA3D support *_V3 flags.
+      LLVM_FALLTHROUGH;
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
+      BCase(EF_AMDGPU_FEATURE_XNACK_V3);
+      BCase(EF_AMDGPU_FEATURE_SRAMECC_V3);
+      break;
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
+      BCaseMask(EF_AMDGPU_FEATURE_XNACK_UNSUPPORTED_V4,
+                EF_AMDGPU_FEATURE_XNACK_V4);
+      BCaseMask(EF_AMDGPU_FEATURE_XNACK_ANY_V4,
+                EF_AMDGPU_FEATURE_XNACK_V4);
+      BCaseMask(EF_AMDGPU_FEATURE_XNACK_OFF_V4,
+                EF_AMDGPU_FEATURE_XNACK_V4);
+      BCaseMask(EF_AMDGPU_FEATURE_XNACK_ON_V4,
+                EF_AMDGPU_FEATURE_XNACK_V4);
+      BCaseMask(EF_AMDGPU_FEATURE_SRAMECC_UNSUPPORTED_V4,
+                EF_AMDGPU_FEATURE_SRAMECC_V4);
+      BCaseMask(EF_AMDGPU_FEATURE_SRAMECC_ANY_V4,
+                EF_AMDGPU_FEATURE_SRAMECC_V4);
+      BCaseMask(EF_AMDGPU_FEATURE_SRAMECC_OFF_V4,
+                EF_AMDGPU_FEATURE_SRAMECC_V4);
+      BCaseMask(EF_AMDGPU_FEATURE_SRAMECC_ON_V4,
+                EF_AMDGPU_FEATURE_SRAMECC_V4);
+      break;
+    }
     break;
   default:
     break;
diff --git a/llvm/lib/ObjectYAML/WasmEmitter.cpp b/llvm/lib/ObjectYAML/WasmEmitter.cpp
index ce5d624c7a0e1d2c1cfa87b556088740d012c7e0..acc5dff7369a5914853d70042421526cde9826f3 100644
--- a/llvm/lib/ObjectYAML/WasmEmitter.cpp
+++ b/llvm/lib/ObjectYAML/WasmEmitter.cpp
@@ -117,7 +117,7 @@ static int writeStringRef(const StringRef &Str, raw_ostream &OS) {
 
 static int writeLimits(const WasmYAML::Limits &Lim, raw_ostream &OS) {
   writeUint8(OS, Lim.Flags);
-  encodeULEB128(Lim.Initial, OS);
+  encodeULEB128(Lim.Minimum, OS);
   if (Lim.Flags & wasm::WASM_LIMITS_FLAG_HAS_MAX)
     encodeULEB128(Lim.Maximum, OS);
   return 0;
diff --git a/llvm/lib/ObjectYAML/WasmYAML.cpp b/llvm/lib/ObjectYAML/WasmYAML.cpp
index 7e57df0c370b2a89d9fd54216d5703b5f812cac2..67074b21298a7983b7129940dd8f6dbe653d8f80 100644
--- a/llvm/lib/ObjectYAML/WasmYAML.cpp
+++ b/llvm/lib/ObjectYAML/WasmYAML.cpp
@@ -367,7 +367,7 @@ void MappingTraits<WasmYAML::Limits>::mapping(IO &IO,
                                               WasmYAML::Limits &Limits) {
   if (!IO.outputting() || Limits.Flags)
     IO.mapOptional("Flags", Limits.Flags);
-  IO.mapRequired("Initial", Limits.Initial);
+  IO.mapRequired("Minimum", Limits.Minimum);
   if (!IO.outputting() || Limits.Flags & wasm::WASM_LIMITS_FLAG_HAS_MAX)
     IO.mapOptional("Maximum", Limits.Maximum);
 }
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 601957034489ee082f059982fbfc8adaaf200df8..1cf17972cc2fd761ce968819b926af035e92909c 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -82,19 +82,26 @@ enum class ChangePrinter {
   PrintChangedVerbose,
   PrintChangedQuiet,
   PrintChangedDiffVerbose,
-  PrintChangedDiffQuiet
+  PrintChangedDiffQuiet,
+  PrintChangedColourDiffVerbose,
+  PrintChangedColourDiffQuiet
 };
 static cl::opt<ChangePrinter> PrintChanged(
     "print-changed", cl::desc("Print changed IRs"), cl::Hidden,
     cl::ValueOptional, cl::init(ChangePrinter::NoChangePrinter),
-    cl::values(clEnumValN(ChangePrinter::PrintChangedQuiet, "quiet",
-                          "Run in quiet mode"),
-               clEnumValN(ChangePrinter::PrintChangedDiffVerbose, "diff",
-                          "Display patch-like changes"),
-               clEnumValN(ChangePrinter::PrintChangedDiffQuiet, "diff-quiet",
-                          "Display patch-like changes in quiet mode"),
-               // Sentinel value for unspecified option.
-               clEnumValN(ChangePrinter::PrintChangedVerbose, "", "")));
+    cl::values(
+        clEnumValN(ChangePrinter::PrintChangedQuiet, "quiet",
+                   "Run in quiet mode"),
+        clEnumValN(ChangePrinter::PrintChangedDiffVerbose, "diff",
+                   "Display patch-like changes"),
+        clEnumValN(ChangePrinter::PrintChangedDiffQuiet, "diff-quiet",
+                   "Display patch-like changes in quiet mode"),
+        clEnumValN(ChangePrinter::PrintChangedColourDiffVerbose, "cdiff",
+                   "Display patch-like changes with color"),
+        clEnumValN(ChangePrinter::PrintChangedColourDiffQuiet, "cdiff-quiet",
+                   "Display patch-like changes in quiet mode with color"),
+        // Sentinel value for unspecified option.
+        clEnumValN(ChangePrinter::PrintChangedVerbose, "", "")));
 
 // An option that supports the -print-changed option.  See
 // the description for -print-changed for an explanation of the use
@@ -1112,7 +1119,8 @@ void InLineChangePrinter::handleAfter(StringRef PassID, std::string &Name,
   SmallString<20> Banner =
       formatv("*** IR Dump After {0} ***{1}\n", PassID, Name);
   Out << Banner;
-  ChangedIRComparer(Out, Before, After).compare(IR, "", PassID, Name);
+  ChangedIRComparer(Out, Before, After, UseColour)
+      .compare(IR, "", PassID, Name);
   Out << "\n";
 }
 
@@ -1133,8 +1141,9 @@ void ChangedIRComparer::handleFunctionCompare(StringRef Name, StringRef Prefix,
       Before, After, [&](const ChangedBlockData *B, const ChangedBlockData *A) {
         StringRef BStr = B ? B->getBody() : "\n";
         StringRef AStr = A ? A->getBody() : "\n";
-        const std::string Removed = "\033[31m-%l\033[0m\n";
-        const std::string Added = "\033[32m+%l\033[0m\n";
+        const std::string Removed =
+            UseColour ? "\033[31m-%l\033[0m\n" : "-%l\n";
+        const std::string Added = UseColour ? "\033[32m+%l\033[0m\n" : "+%l\n";
         const std::string NoChange = " %l\n";
         Out << doSystemDiff(BStr, AStr, Removed, Added, NoChange);
       });
@@ -1142,7 +1151,9 @@ void ChangedIRComparer::handleFunctionCompare(StringRef Name, StringRef Prefix,
 
 void InLineChangePrinter::registerCallbacks(PassInstrumentationCallbacks &PIC) {
   if (PrintChanged == ChangePrinter::PrintChangedDiffVerbose ||
-      PrintChanged == ChangePrinter::PrintChangedDiffQuiet)
+      PrintChanged == ChangePrinter::PrintChangedDiffQuiet ||
+      PrintChanged == ChangePrinter::PrintChangedColourDiffVerbose ||
+      PrintChanged == ChangePrinter::PrintChangedColourDiffQuiet)
     TextChangeReporter<ChangedIRData>::registerRequiredCallbacks(PIC);
 }
 
@@ -1150,7 +1161,11 @@ StandardInstrumentations::StandardInstrumentations(bool DebugLogging,
                                                    bool VerifyEach)
     : PrintPass(DebugLogging), OptNone(DebugLogging),
       PrintChangedIR(PrintChanged == ChangePrinter::PrintChangedVerbose),
-      PrintChangedDiff(PrintChanged == ChangePrinter::PrintChangedDiffVerbose),
+      PrintChangedDiff(
+          PrintChanged == ChangePrinter::PrintChangedDiffVerbose ||
+              PrintChanged == ChangePrinter::PrintChangedColourDiffVerbose,
+          PrintChanged == ChangePrinter::PrintChangedColourDiffVerbose ||
+              PrintChanged == ChangePrinter::PrintChangedColourDiffQuiet),
       Verify(DebugLogging), VerifyEach(VerifyEach) {}
 
 void StandardInstrumentations::registerCallbacks(
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
index cdbcde50d33acc031cf4fbdd7af28e4868f96db0..f81394612190facf04761afb82e1240446f735c4 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -281,21 +281,29 @@ Error CoverageMapping::loadFunctionRecord(
   return Error::success();
 }
 
-Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(
+// This function is for memory optimization by shortening the lifetimes
+// of CoverageMappingReader instances.
+Error CoverageMapping::loadFromReaders(
     ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders,
-    IndexedInstrProfReader &ProfileReader) {
-  auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping());
-
+    IndexedInstrProfReader &ProfileReader, CoverageMapping &Coverage) {
   for (const auto &CoverageReader : CoverageReaders) {
     for (auto RecordOrErr : *CoverageReader) {
       if (Error E = RecordOrErr.takeError())
-        return std::move(E);
+        return E;
       const auto &Record = *RecordOrErr;
-      if (Error E = Coverage->loadFunctionRecord(Record, ProfileReader))
-        return std::move(E);
+      if (Error E = Coverage.loadFunctionRecord(Record, ProfileReader))
+        return E;
     }
   }
+  return Error::success();
+}
 
+Expected<std::unique_ptr<CoverageMapping>> CoverageMapping::load(
+    ArrayRef<std::unique_ptr<CoverageMappingReader>> CoverageReaders,
+    IndexedInstrProfReader &ProfileReader) {
+  auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping());
+  if (Error E = loadFromReaders(CoverageReaders, ProfileReader, *Coverage))
+    return std::move(E);
   return std::move(Coverage);
 }
 
@@ -316,16 +324,18 @@ CoverageMapping::load(ArrayRef<StringRef> ObjectFilenames,
   if (Error E = ProfileReaderOrErr.takeError())
     return std::move(E);
   auto ProfileReader = std::move(ProfileReaderOrErr.get());
+  auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping());
+  bool DataFound = false;
 
-  SmallVector<std::unique_ptr<CoverageMappingReader>, 4> Readers;
-  SmallVector<std::unique_ptr<MemoryBuffer>, 4> Buffers;
   for (const auto &File : llvm::enumerate(ObjectFilenames)) {
-    auto CovMappingBufOrErr = MemoryBuffer::getFileOrSTDIN(File.value());
+    auto CovMappingBufOrErr = MemoryBuffer::getFileOrSTDIN(
+        File.value(), /*IsText=*/false, /*RequiresNullTerminator=*/false);
     if (std::error_code EC = CovMappingBufOrErr.getError())
       return errorCodeToError(EC);
     StringRef Arch = Arches.empty() ? StringRef() : Arches[File.index()];
     MemoryBufferRef CovMappingBufRef =
         CovMappingBufOrErr.get()->getMemBufferRef();
+    SmallVector<std::unique_ptr<MemoryBuffer>, 4> Buffers;
     auto CoverageReadersOrErr =
         BinaryCoverageReader::create(CovMappingBufRef, Arch, Buffers);
     if (Error E = CoverageReadersOrErr.takeError()) {
@@ -335,15 +345,19 @@ CoverageMapping::load(ArrayRef<StringRef> ObjectFilenames,
       // E == success (originally a no_data_found error).
       continue;
     }
+
+    SmallVector<std::unique_ptr<CoverageMappingReader>, 4> Readers;
     for (auto &Reader : CoverageReadersOrErr.get())
       Readers.push_back(std::move(Reader));
-    Buffers.push_back(std::move(CovMappingBufOrErr.get()));
+    DataFound |= !Readers.empty();
+    if (Error E = loadFromReaders(Readers, *ProfileReader, *Coverage))
+      return std::move(E);
   }
   // If no readers were created, either no objects were provided or none of them
   // had coverage data. Return an error in the latter case.
-  if (Readers.empty() && !ObjectFilenames.empty())
+  if (!DataFound && !ObjectFilenames.empty())
     return make_error<CoverageMapError>(coveragemap_error::no_data_found);
-  return load(Readers, *ProfileReader);
+  return std::move(Coverage);
 }
 
 namespace {
diff --git a/llvm/lib/ProfileData/GCOV.cpp b/llvm/lib/ProfileData/GCOV.cpp
index 3332a898603b4a491d772dfb9ba711ff6dbfe50f..f24c4b0cf9b5bbc0823cd2928e23335058bfad63 100644
--- a/llvm/lib/ProfileData/GCOV.cpp
+++ b/llvm/lib/ProfileData/GCOV.cpp
@@ -564,7 +564,7 @@ public:
     // Open source files without requiring a NUL terminator. The concurrent
     // modification may nullify the NUL terminator condition.
     ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
-        MemoryBuffer::getFileOrSTDIN(Filename, -1,
+        MemoryBuffer::getFileOrSTDIN(Filename, /*IsText=*/false,
                                      /*RequiresNullTerminator=*/false);
     if (std::error_code EC = BufferOrErr.getError()) {
       errs() << Filename << ": " << EC.message() << "\n";
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index 697d29f6f412ce299e965fdb5f14331bd9abf81d..200a0afb01c614d3be2247da01a5155284b64c87 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -88,13 +88,22 @@ static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
 /// Possible metadata:
 /// - CFG Checksum information:
 ///     !CFGChecksum: 12345
+/// - CFG Checksum information:
+///     !Attributes: 1
 /// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
-static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash) {
-  if (!Input.startswith("!CFGChecksum:"))
-    return false;
+static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
+                          uint32_t &Attributes) {
+  if (Input.startswith("!CFGChecksum:")) {
+    StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim();
+    return !CFGInfo.getAsInteger(10, FunctionHash);
+  }
+
+  if (Input.startswith("!Attributes:")) {
+    StringRef Attrib = Input.substr(strlen("!Attributes:")).trim();
+    return !Attrib.getAsInteger(10, Attributes);
+  }
 
-  StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim();
-  return !CFGInfo.getAsInteger(10, FunctionHash);
+  return false;
 }
 
 enum class LineType {
@@ -119,7 +128,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
                       uint64_t &NumSamples, uint32_t &LineOffset,
                       uint32_t &Discriminator, StringRef &CalleeName,
                       DenseMap<StringRef, uint64_t> &TargetCountMap,
-                      uint64_t &FunctionHash) {
+                      uint64_t &FunctionHash, uint32_t &Attributes) {
   for (Depth = 0; Input[Depth] == ' '; Depth++)
     ;
   if (Depth == 0)
@@ -127,7 +136,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
 
   if (Depth == 1 && Input[Depth] == '!') {
     LineTy = LineType::Metadata;
-    return parseMetadata(Input.substr(Depth), FunctionHash);
+    return parseMetadata(Input.substr(Depth), FunctionHash, Attributes);
   }
 
   size_t n1 = Input.find(':');
@@ -270,9 +279,11 @@ std::error_code SampleProfileReaderText::readImpl() {
       DenseMap<StringRef, uint64_t> TargetCountMap;
       uint32_t Depth, LineOffset, Discriminator;
       LineType LineTy;
-      uint64_t FunctionHash;
+      uint64_t FunctionHash = 0;
+      uint32_t Attributes = 0;
       if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset,
-                     Discriminator, FName, TargetCountMap, FunctionHash)) {
+                     Discriminator, FName, TargetCountMap, FunctionHash,
+                     Attributes)) {
         reportError(LineIt.line_number(),
                     "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
                         *LineIt);
@@ -312,8 +323,12 @@ std::error_code SampleProfileReaderText::readImpl() {
       }
       case LineType::Metadata: {
         FunctionSamples &FProfile = *InlineStack.back();
-        FProfile.setFunctionHash(FunctionHash);
-        ++ProbeProfileCount;
+        if (FunctionHash) {
+          FProfile.setFunctionHash(FunctionHash);
+          ++ProbeProfileCount;
+        }
+        if (Attributes)
+          FProfile.getContext().setAllAttributes(Attributes);
         SeenMetadata = true;
         break;
       }
@@ -601,13 +616,16 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
     if (std::error_code EC = readFuncOffsetTable())
       return EC;
     break;
-  case SecFuncMetadata:
+  case SecFuncMetadata: {
     ProfileIsProbeBased =
         hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased);
     FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
-    if (std::error_code EC = readFuncMetadata())
+    bool HasAttribute =
+        hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute);
+    if (std::error_code EC = readFuncMetadata(HasAttribute))
       return EC;
     break;
+  }
   case SecProfileSymbolList:
     if (std::error_code EC = readProfileSymbolList())
       return EC;
@@ -941,23 +959,31 @@ std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5) {
   return SampleProfileReaderBinary::readNameTable();
 }
 
-std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() {
-  if (!ProfileIsProbeBased)
-    return sampleprof_error::success;
+std::error_code
+SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
   while (Data < End) {
     auto FName(readStringFromTable());
     if (std::error_code EC = FName.getError())
       return EC;
 
-    auto Checksum = readNumber<uint64_t>();
-    if (std::error_code EC = Checksum.getError())
-      return EC;
-
     SampleContext FContext(*FName);
-    // No need to load metadata for profiles that are not loaded in the current
-    // module.
-    if (Profiles.count(FContext))
-      Profiles[FContext].setFunctionHash(*Checksum);
+    bool ProfileInMap = Profiles.count(FContext);
+
+    if (ProfileIsProbeBased) {
+      auto Checksum = readNumber<uint64_t>();
+      if (std::error_code EC = Checksum.getError())
+        return EC;
+      if (ProfileInMap)
+        Profiles[FContext].setFunctionHash(*Checksum);
+    }
+
+    if (ProfileHasAttribute) {
+      auto Attributes = readNumber<uint32_t>();
+      if (std::error_code EC = Attributes.getError())
+        return EC;
+      if (ProfileInMap)
+        Profiles[FContext].getContext().setAllAttributes(*Attributes);
+    }
   }
 
   assert(Data == End && "More data is read than expected");
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index 7a00c3fec7c7b65f434bc3d2cf63cd32c03a8945..b9643480a8e4de43df2e3bdf16c7ff72fccfd055 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -170,12 +170,15 @@ std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
 
 std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
     const StringMap<FunctionSamples> &Profiles) {
-  if (!FunctionSamples::ProfileIsProbeBased)
+  if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS)
     return sampleprof_error::success;
   auto &OS = *OutputStream;
   for (const auto &Entry : Profiles) {
     writeNameIdx(Entry.first());
-    encodeULEB128(Entry.second.getFunctionHash(), OS);
+    if (FunctionSamples::ProfileIsProbeBased)
+      encodeULEB128(Entry.second.getFunctionHash(), OS);
+    if (FunctionSamples::ProfileIsCS)
+      encodeULEB128(Entry.second.getContext().getAllAttributes(), OS);
   }
   return sampleprof_error::success;
 }
@@ -239,6 +242,8 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
     addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased);
   if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
     addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFullContext);
+  if (Type == SecFuncMetadata && FunctionSamples::ProfileIsCS)
+    addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagHasAttribute);
 
   uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
   switch (Type) {
@@ -417,6 +422,10 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
       OS.indent(Indent + 1);
       OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
     }
+    if (FunctionSamples::ProfileIsCS) {
+      OS.indent(Indent + 1);
+      OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
+    }
   }
 
   return sampleprof_error::success;
diff --git a/llvm/lib/Support/FileOutputBuffer.cpp b/llvm/lib/Support/FileOutputBuffer.cpp
index 7b2a512bd475049b5686df004d79e8826aeac374..3342682270dcdc7a0d8e5110e4b6fb6a22e0dc55 100644
--- a/llvm/lib/Support/FileOutputBuffer.cpp
+++ b/llvm/lib/Support/FileOutputBuffer.cpp
@@ -125,8 +125,7 @@ createInMemoryBuffer(StringRef Path, size_t Size, unsigned Mode) {
 }
 
 static Expected<std::unique_ptr<FileOutputBuffer>>
-createOnDiskBuffer(StringRef Path, size_t Size, unsigned Mode,
-                   bool KeepOwnership, unsigned UserID, unsigned GroupID) {
+createOnDiskBuffer(StringRef Path, size_t Size, unsigned Mode) {
   Expected<fs::TempFile> FileOrErr =
       fs::TempFile::create(Path + ".tmp%%%%%%%", Mode);
   if (!FileOrErr)
@@ -134,13 +133,6 @@ createOnDiskBuffer(StringRef Path, size_t Size, unsigned Mode,
   fs::TempFile File = std::move(*FileOrErr);
 
 #ifndef _WIN32
-  // Try to preserve file ownership if requested.
-  if (KeepOwnership) {
-    fs::file_status Stat;
-    if (!fs::status(File.FD, Stat) && Stat.getUser() == 0)
-      fs::changeFileOwnership(File.FD, UserID, GroupID);
-  }
-
   // On Windows, CreateFileMapping (the mmap function on Windows)
   // automatically extends the underlying file. We don't need to
   // extend the file beforehand. _chsize (ftruncate on Windows) is
@@ -171,8 +163,7 @@ createOnDiskBuffer(StringRef Path, size_t Size, unsigned Mode,
 
 // Create an instance of FileOutputBuffer.
 Expected<std::unique_ptr<FileOutputBuffer>>
-FileOutputBuffer::create(StringRef Path, size_t Size, unsigned Flags,
-                         unsigned UserID, unsigned GroupID) {
+FileOutputBuffer::create(StringRef Path, size_t Size, unsigned Flags) {
   // Handle "-" as stdout just like llvm::raw_ostream does.
   if (Path == "-")
     return createInMemoryBuffer("-", Size, /*Mode=*/0);
@@ -205,8 +196,7 @@ FileOutputBuffer::create(StringRef Path, size_t Size, unsigned Flags,
     if (Flags & F_no_mmap)
       return createInMemoryBuffer(Path, Size, Mode);
     else
-      return createOnDiskBuffer(Path, Size, Mode, Flags & F_keep_ownership,
-                                UserID, GroupID);
+      return createOnDiskBuffer(Path, Size, Mode);
   default:
     return createInMemoryBuffer(Path, Size, Mode);
   }
diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp
index 6c7aaad968f5d1dfb119891e480d302bdb0aec1a..423a908eed57f6d61637a2670653f8b5f39b12a7 100644
--- a/llvm/lib/Support/KnownBits.cpp
+++ b/llvm/lib/Support/KnownBits.cpp
@@ -489,6 +489,24 @@ KnownBits KnownBits::computeForMul(const KnownBits &LHS, const KnownBits &RHS) {
   return Res;
 }
 
+KnownBits KnownBits::mulhs(const KnownBits &LHS, const KnownBits &RHS) {
+  unsigned BitWidth = LHS.getBitWidth();
+  assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() &&
+         !RHS.hasConflict() && "Operand mismatch");
+  KnownBits WideLHS = LHS.sext(2 * BitWidth);
+  KnownBits WideRHS = RHS.sext(2 * BitWidth);
+  return computeForMul(WideLHS, WideRHS).extractBits(BitWidth, BitWidth);
+}
+
+KnownBits KnownBits::mulhu(const KnownBits &LHS, const KnownBits &RHS) {
+  unsigned BitWidth = LHS.getBitWidth();
+  assert(BitWidth == RHS.getBitWidth() && !LHS.hasConflict() &&
+         !RHS.hasConflict() && "Operand mismatch");
+  KnownBits WideLHS = LHS.zext(2 * BitWidth);
+  KnownBits WideRHS = RHS.zext(2 * BitWidth);
+  return computeForMul(WideLHS, WideRHS).extractBits(BitWidth, BitWidth);
+}
+
 KnownBits KnownBits::udiv(const KnownBits &LHS, const KnownBits &RHS) {
   unsigned BitWidth = LHS.getBitWidth();
   assert(!LHS.hasConflict() && !RHS.hasConflict());
diff --git a/llvm/lib/Support/ManagedStatic.cpp b/llvm/lib/Support/ManagedStatic.cpp
index 053493f72fb5eeba9cc623201aba80ccc62a5f79..a6ae67066ea0423334e8ee52106f220cd456e25e 100644
--- a/llvm/lib/Support/ManagedStatic.cpp
+++ b/llvm/lib/Support/ManagedStatic.cpp
@@ -18,16 +18,10 @@
 using namespace llvm;
 
 static const ManagedStaticBase *StaticList = nullptr;
-static std::recursive_mutex *ManagedStaticMutex = nullptr;
-static llvm::once_flag mutex_init_flag;
-
-static void initializeMutex() {
-  ManagedStaticMutex = new std::recursive_mutex();
-}
 
 static std::recursive_mutex *getManagedStaticMutex() {
-  llvm::call_once(mutex_init_flag, initializeMutex);
-  return ManagedStaticMutex;
+  static std::recursive_mutex m;
+  return &m;
 }
 
 void ManagedStaticBase::RegisterManagedStatic(void *(*Creator)(),
@@ -75,9 +69,10 @@ void ManagedStaticBase::destroy() const {
 }
 
 /// llvm_shutdown - Deallocate and destroy all ManagedStatic variables.
+/// IMPORTANT: it's only safe to call llvm_shutdown() in single thread,
+/// without any other threads executing LLVM APIs.
+/// llvm_shutdown() should be the last use of LLVM APIs.
 void llvm::llvm_shutdown() {
-  std::lock_guard<std::recursive_mutex> Lock(*getManagedStaticMutex());
-
   while (StaticList)
     StaticList->destroy();
 }
diff --git a/llvm/lib/Support/MemoryBuffer.cpp b/llvm/lib/Support/MemoryBuffer.cpp
index a05b7d8ddd0e251e16769b851d9ddb1ac45584a0..1d75f0be1ca62552e0bf3bac143c5e9d8dcee549 100644
--- a/llvm/lib/Support/MemoryBuffer.cpp
+++ b/llvm/lib/Support/MemoryBuffer.cpp
@@ -105,8 +105,8 @@ public:
 
 template <typename MB>
 static ErrorOr<std::unique_ptr<MB>>
-getFileAux(const Twine &Filename, int64_t FileSize, uint64_t MapSize,
-           uint64_t Offset, bool RequiresNullTerminator, bool IsVolatile);
+getFileAux(const Twine &Filename, uint64_t MapSize, uint64_t Offset,
+           bool IsText, bool RequiresNullTerminator, bool IsVolatile);
 
 std::unique_ptr<MemoryBuffer>
 MemoryBuffer::getMemBuffer(StringRef InputData, StringRef BufferName,
@@ -140,21 +140,22 @@ MemoryBuffer::getMemBufferCopy(StringRef InputData, const Twine &BufferName) {
 }
 
 ErrorOr<std::unique_ptr<MemoryBuffer>>
-MemoryBuffer::getFileOrSTDIN(const Twine &Filename, int64_t FileSize,
+MemoryBuffer::getFileOrSTDIN(const Twine &Filename, bool IsText,
                              bool RequiresNullTerminator) {
   SmallString<256> NameBuf;
   StringRef NameRef = Filename.toStringRef(NameBuf);
 
   if (NameRef == "-")
     return getSTDIN();
-  return getFile(Filename, FileSize, RequiresNullTerminator);
+  return getFile(Filename, IsText, RequiresNullTerminator,
+                 /*IsVolatile=*/false);
 }
 
 ErrorOr<std::unique_ptr<MemoryBuffer>>
 MemoryBuffer::getFileSlice(const Twine &FilePath, uint64_t MapSize,
                            uint64_t Offset, bool IsVolatile) {
-  return getFileAux<MemoryBuffer>(FilePath, -1, MapSize, Offset, false,
-                                  IsVolatile);
+  return getFileAux<MemoryBuffer>(FilePath, MapSize, Offset, /*IsText=*/false,
+                                  /*RequiresNullTerminator=*/false, IsVolatile);
 }
 
 //===----------------------------------------------------------------------===//
@@ -240,12 +241,11 @@ getMemoryBufferForStream(sys::fs::file_t FD, const Twine &BufferName) {
   return getMemBufferCopyImpl(Buffer, BufferName);
 }
 
-
 ErrorOr<std::unique_ptr<MemoryBuffer>>
-MemoryBuffer::getFile(const Twine &Filename, int64_t FileSize,
+MemoryBuffer::getFile(const Twine &Filename, bool IsText,
                       bool RequiresNullTerminator, bool IsVolatile) {
-  return getFileAux<MemoryBuffer>(Filename, FileSize, FileSize, 0,
-                                  RequiresNullTerminator, IsVolatile);
+  return getFileAux<MemoryBuffer>(Filename, /*MapSize=*/-1, /*Offset=*/0,
+                                  IsText, RequiresNullTerminator, IsVolatile);
 }
 
 template <typename MB>
@@ -256,32 +256,32 @@ getOpenFileImpl(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize,
 
 template <typename MB>
 static ErrorOr<std::unique_ptr<MB>>
-getFileAux(const Twine &Filename, int64_t FileSize, uint64_t MapSize,
-           uint64_t Offset, bool RequiresNullTerminator, bool IsVolatile) {
-  Expected<sys::fs::file_t> FDOrErr =
-      sys::fs::openNativeFileForRead(Filename, sys::fs::OF_None);
+getFileAux(const Twine &Filename, uint64_t MapSize, uint64_t Offset,
+           bool IsText, bool RequiresNullTerminator, bool IsVolatile) {
+  Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
+      Filename, IsText ? sys::fs::OF_Text : sys::fs::OF_None);
   if (!FDOrErr)
     return errorToErrorCode(FDOrErr.takeError());
   sys::fs::file_t FD = *FDOrErr;
-  auto Ret = getOpenFileImpl<MB>(FD, Filename, FileSize, MapSize, Offset,
+  auto Ret = getOpenFileImpl<MB>(FD, Filename, /*FileSize=*/-1, MapSize, Offset,
                                  RequiresNullTerminator, IsVolatile);
   sys::fs::closeFile(FD);
   return Ret;
 }
 
 ErrorOr<std::unique_ptr<WritableMemoryBuffer>>
-WritableMemoryBuffer::getFile(const Twine &Filename, int64_t FileSize,
-                              bool IsVolatile) {
-  return getFileAux<WritableMemoryBuffer>(Filename, FileSize, FileSize, 0,
-                                          /*RequiresNullTerminator*/ false,
-                                          IsVolatile);
+WritableMemoryBuffer::getFile(const Twine &Filename, bool IsVolatile) {
+  return getFileAux<WritableMemoryBuffer>(
+      Filename, /*MapSize=*/-1, /*Offset=*/0, /*IsText=*/false,
+      /*RequiresNullTerminator=*/false, IsVolatile);
 }
 
 ErrorOr<std::unique_ptr<WritableMemoryBuffer>>
 WritableMemoryBuffer::getFileSlice(const Twine &Filename, uint64_t MapSize,
                                    uint64_t Offset, bool IsVolatile) {
-  return getFileAux<WritableMemoryBuffer>(Filename, -1, MapSize, Offset, false,
-                                          IsVolatile);
+  return getFileAux<WritableMemoryBuffer>(
+      Filename, MapSize, Offset, /*IsText=*/false,
+      /*RequiresNullTerminator=*/false, IsVolatile);
 }
 
 std::unique_ptr<WritableMemoryBuffer>
diff --git a/llvm/lib/Support/Path.cpp b/llvm/lib/Support/Path.cpp
index ef223ae5ac1d6bb9f1b81f90a6ccc6abf95dffe0..f49affb3fa99bd680d011b58b5b8578e9e480fde 100644
--- a/llvm/lib/Support/Path.cpp
+++ b/llvm/lib/Support/Path.cpp
@@ -167,8 +167,8 @@ enum FSEntity {
 static std::error_code
 createUniqueEntity(const Twine &Model, int &ResultFD,
                    SmallVectorImpl<char> &ResultPath, bool MakeAbsolute,
-                   unsigned Mode, FSEntity Type,
-                   sys::fs::OpenFlags Flags = sys::fs::OF_None) {
+                   FSEntity Type, sys::fs::OpenFlags Flags = sys::fs::OF_None,
+                   unsigned Mode = 0) {
 
   // Limit the number of attempts we make, so that we don't infinite loop. E.g.
   // "permission denied" could be for a specific file (so we retry with a
@@ -816,22 +816,16 @@ void createUniquePath(const Twine &Model, SmallVectorImpl<char> &ResultPath,
 
 std::error_code createUniqueFile(const Twine &Model, int &ResultFd,
                                  SmallVectorImpl<char> &ResultPath,
-                                 unsigned Mode) {
-  return createUniqueEntity(Model, ResultFd, ResultPath, false, Mode, FS_File);
-}
-
-static std::error_code createUniqueFile(const Twine &Model, int &ResultFd,
-                                        SmallVectorImpl<char> &ResultPath,
-                                        unsigned Mode, OpenFlags Flags) {
-  return createUniqueEntity(Model, ResultFd, ResultPath, false, Mode, FS_File,
-                            Flags);
+                                 OpenFlags Flags, unsigned Mode) {
+  return createUniqueEntity(Model, ResultFd, ResultPath, false, FS_File, Flags,
+                            Mode);
 }
 
 std::error_code createUniqueFile(const Twine &Model,
                                  SmallVectorImpl<char> &ResultPath,
                                  unsigned Mode) {
   int FD;
-  auto EC = createUniqueFile(Model, FD, ResultPath, Mode);
+  auto EC = createUniqueFile(Model, FD, ResultPath, OF_None, Mode);
   if (EC)
     return EC;
   // FD is only needed to avoid race conditions. Close it right away.
@@ -841,34 +835,39 @@ std::error_code createUniqueFile(const Twine &Model,
 
 static std::error_code
 createTemporaryFile(const Twine &Model, int &ResultFD,
-                    llvm::SmallVectorImpl<char> &ResultPath, FSEntity Type) {
+                    llvm::SmallVectorImpl<char> &ResultPath, FSEntity Type,
+                    sys::fs::OpenFlags Flags = sys::fs::OF_None) {
   SmallString<128> Storage;
   StringRef P = Model.toNullTerminatedStringRef(Storage);
   assert(P.find_first_of(separators(Style::native)) == StringRef::npos &&
          "Model must be a simple filename.");
   // Use P.begin() so that createUniqueEntity doesn't need to recreate Storage.
-  return createUniqueEntity(P.begin(), ResultFD, ResultPath, true,
-                            owner_read | owner_write, Type);
+  return createUniqueEntity(P.begin(), ResultFD, ResultPath, true, Type, Flags,
+                            owner_read | owner_write);
 }
 
 static std::error_code
 createTemporaryFile(const Twine &Prefix, StringRef Suffix, int &ResultFD,
-                    llvm::SmallVectorImpl<char> &ResultPath, FSEntity Type) {
+                    llvm::SmallVectorImpl<char> &ResultPath, FSEntity Type,
+                    sys::fs::OpenFlags Flags = sys::fs::OF_None) {
   const char *Middle = Suffix.empty() ? "-%%%%%%" : "-%%%%%%.";
   return createTemporaryFile(Prefix + Middle + Suffix, ResultFD, ResultPath,
-                             Type);
+                             Type, Flags);
 }
 
 std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
                                     int &ResultFD,
-                                    SmallVectorImpl<char> &ResultPath) {
-  return createTemporaryFile(Prefix, Suffix, ResultFD, ResultPath, FS_File);
+                                    SmallVectorImpl<char> &ResultPath,
+                                    sys::fs::OpenFlags Flags) {
+  return createTemporaryFile(Prefix, Suffix, ResultFD, ResultPath, FS_File,
+                             Flags);
 }
 
 std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
-                                    SmallVectorImpl<char> &ResultPath) {
+                                    SmallVectorImpl<char> &ResultPath,
+                                    sys::fs::OpenFlags Flags) {
   int FD;
-  auto EC = createTemporaryFile(Prefix, Suffix, FD, ResultPath);
+  auto EC = createTemporaryFile(Prefix, Suffix, FD, ResultPath, Flags);
   if (EC)
     return EC;
   // FD is only needed to avoid race conditions. Close it right away.
@@ -876,13 +875,12 @@ std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
   return EC;
 }
 
-
 // This is a mkdtemp with a different pattern. We use createUniqueEntity mostly
 // for consistency. We should try using mkdtemp.
 std::error_code createUniqueDirectory(const Twine &Prefix,
                                       SmallVectorImpl<char> &ResultPath) {
   int Dummy;
-  return createUniqueEntity(Prefix + "-%%%%%%", Dummy, ResultPath, true, 0,
+  return createUniqueEntity(Prefix + "-%%%%%%", Dummy, ResultPath, true,
                             FS_Dir);
 }
 
@@ -890,7 +888,7 @@ std::error_code
 getPotentiallyUniqueFileName(const Twine &Model,
                              SmallVectorImpl<char> &ResultPath) {
   int Dummy;
-  return createUniqueEntity(Model, Dummy, ResultPath, false, 0, FS_Name);
+  return createUniqueEntity(Model, Dummy, ResultPath, false, FS_Name);
 }
 
 std::error_code
@@ -1279,7 +1277,7 @@ Expected<TempFile> TempFile::create(const Twine &Model, unsigned Mode) {
   int FD;
   SmallString<128> ResultPath;
   if (std::error_code EC =
-          createUniqueFile(Model, FD, ResultPath, Mode, OF_Delete))
+          createUniqueFile(Model, FD, ResultPath, OF_Delete, Mode))
     return errorCodeToError(EC);
 
   TempFile Ret(ResultPath, FD);
diff --git a/llvm/lib/Support/ToolOutputFile.cpp b/llvm/lib/Support/ToolOutputFile.cpp
index c2ca97a59c62090fa7d8c0b3d8d6c483b54714da..3735aac79e2f40096607d61dc96e61d617b4c4ad 100644
--- a/llvm/lib/Support/ToolOutputFile.cpp
+++ b/llvm/lib/Support/ToolOutputFile.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/ToolOutputFile.h"
+#include "llvm/ADT/Triple.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Signals.h"
 using namespace llvm;
@@ -45,7 +46,12 @@ ToolOutputFile::ToolOutputFile(StringRef Filename, std::error_code &EC,
     EC = std::error_code();
     return;
   }
-  OSHolder.emplace(Filename, EC, Flags);
+
+  // On Windows, we set the OF_None flag even for text files to avoid
+  // CRLF translation.
+  OSHolder.emplace(
+      Filename, EC,
+      llvm::Triple(LLVM_HOST_TRIPLE).isOSWindows() ? sys::fs::OF_None : Flags);
   OS = OSHolder.getPointer();
   // If open fails, no cleanup is needed.
   if (EC)
diff --git a/llvm/lib/Support/Triple.cpp b/llvm/lib/Support/Triple.cpp
index e5dd32fb58278c55ba978ac286b37b16feed467e..3c2182ecb09a0afc700144b7208749ed3309cc91 100644
--- a/llvm/lib/Support/Triple.cpp
+++ b/llvm/lib/Support/Triple.cpp
@@ -250,6 +250,7 @@ StringRef Triple::getEnvironmentTypeName(EnvironmentType Kind) {
   case Musl: return "musl";
   case MuslEABI: return "musleabi";
   case MuslEABIHF: return "musleabihf";
+  case MuslX32: return "muslx32";
   case Simulator: return "simulator";
   }
 
@@ -555,6 +556,7 @@ static Triple::EnvironmentType parseEnvironment(StringRef EnvironmentName) {
       .StartsWith("android", Triple::Android)
       .StartsWith("musleabihf", Triple::MuslEABIHF)
       .StartsWith("musleabi", Triple::MuslEABI)
+      .StartsWith("muslx32", Triple::MuslX32)
       .StartsWith("musl", Triple::Musl)
       .StartsWith("msvc", Triple::MSVC)
       .StartsWith("itanium", Triple::Itanium)
diff --git a/llvm/lib/Support/Unix/Process.inc b/llvm/lib/Support/Unix/Process.inc
index c3d8f7001de367c0a3404e10095432d12a7b25bc..1ea36aae63f2422b899977e3291f8a00b75c2d47 100644
--- a/llvm/lib/Support/Unix/Process.inc
+++ b/llvm/lib/Support/Unix/Process.inc
@@ -236,11 +236,11 @@ std::error_code Process::FixupStandardFileDescriptors() {
 
 std::error_code Process::SafelyCloseFileDescriptor(int FD) {
   // Create a signal set filled with *all* signals.
-  sigset_t FullSet;
-  if (sigfillset(&FullSet) < 0)
+  sigset_t FullSet, SavedSet;
+  if (sigfillset(&FullSet) < 0 || sigfillset(&SavedSet) < 0)
     return std::error_code(errno, std::generic_category());
+
   // Atomically swap our current signal mask with a full mask.
-  sigset_t SavedSet;
 #if LLVM_ENABLE_THREADS
   if (int EC = pthread_sigmask(SIG_SETMASK, &FullSet, &SavedSet))
     return std::error_code(EC, std::generic_category());
diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc
index dc9bcf8683810a496b9eaa9d8a34994497e7537c..e2bd7da4f04bf68d9e2bee458ce99ce22c0920e4 100644
--- a/llvm/lib/Support/Windows/Path.inc
+++ b/llvm/lib/Support/Windows/Path.inc
@@ -402,8 +402,22 @@ std::error_code is_local(int FD, bool &Result) {
 }
 
 static std::error_code setDeleteDisposition(HANDLE Handle, bool Delete) {
-  // First, check if the file is on a network (non-local) drive. If so, don't
-  // set DeleteFile to true, since it prevents opening the file for writes.
+  // Clear the FILE_DISPOSITION_INFO flag first, before checking if it's a
+  // network file. On Windows 7 the function realPathFromHandle() below fails
+  // if the FILE_DISPOSITION_INFO flag was already set to 'DeleteFile = true' by
+  // a prior call.
+  FILE_DISPOSITION_INFO Disposition;
+  Disposition.DeleteFile = false;
+  if (!SetFileInformationByHandle(Handle, FileDispositionInfo, &Disposition,
+                                  sizeof(Disposition)))
+    return mapWindowsError(::GetLastError());
+  if (!Delete)
+    return std::error_code();
+
+  // Check if the file is on a network (non-local) drive. If so, don't
+  // continue when DeleteFile is true, since it prevents opening the file for
+  // writes. Note -- this will leak temporary files on disk, but only when the
+  // target file is on a network drive.
   SmallVector<wchar_t, 128> FinalPath;
   if (std::error_code EC = realPathFromHandle(Handle, FinalPath))
     return EC;
@@ -415,9 +429,9 @@ static std::error_code setDeleteDisposition(HANDLE Handle, bool Delete) {
   if (!IsLocal)
     return std::error_code();
 
-  // The file is on a local drive, set the DeleteFile to true.
-  FILE_DISPOSITION_INFO Disposition;
-  Disposition.DeleteFile = Delete;
+  // The file is on a local drive, we can safely set FILE_DISPOSITION_INFO's
+  // flag.
+  Disposition.DeleteFile = true;
   if (!SetFileInformationByHandle(Handle, FileDispositionInfo, &Disposition,
                                   sizeof(Disposition)))
     return mapWindowsError(::GetLastError());
@@ -609,6 +623,9 @@ std::error_code access(const Twine &Path, AccessMode Mode) {
   if (Mode == AccessMode::Write && (Attributes & FILE_ATTRIBUTE_READONLY))
     return errc::permission_denied;
 
+  if (Mode == AccessMode::Execute && (Attributes & FILE_ATTRIBUTE_DIRECTORY))
+    return errc::permission_denied;
+
   return std::error_code();
 }
 
diff --git a/llvm/lib/Support/Windows/Program.inc b/llvm/lib/Support/Windows/Program.inc
index f1d612cf3c98295f301dacacaada6690631375ab..687109c01355e758e05bc7d8be2efe3d36833737 100644
--- a/llvm/lib/Support/Windows/Program.inc
+++ b/llvm/lib/Support/Windows/Program.inc
@@ -67,13 +67,10 @@ ErrorOr<std::string> sys::findProgramByName(StringRef Name,
   if (const char *PathExtEnv = std::getenv("PATHEXT"))
     SplitString(PathExtEnv, PathExts, ";");
 
-  SmallVector<wchar_t, MAX_PATH> U16Result;
-  DWORD Len = MAX_PATH;
+  SmallVector<char, MAX_PATH> U8Result;
   for (StringRef Ext : PathExts) {
-    SmallVector<wchar_t, MAX_PATH> U16Ext;
-    if (std::error_code EC = windows::UTF8ToUTF16(Ext, U16Ext))
-      return EC;
-
+    SmallVector<wchar_t, MAX_PATH> U16Result;
+    DWORD Len = MAX_PATH;
     do {
       U16Result.reserve(Len);
       // Lets attach the extension manually. That is needed for files
@@ -88,20 +85,24 @@ ErrorOr<std::string> sys::findProgramByName(StringRef Name,
                           U16Result.capacity(), U16Result.data(), nullptr);
     } while (Len > U16Result.capacity());
 
-    if (Len != 0)
+    if (Len == 0)
+      continue;
+
+    U16Result.set_size(Len);
+
+    if (std::error_code EC =
+        windows::UTF16ToUTF8(U16Result.data(), U16Result.size(), U8Result))
+      return EC;
+
+    if (sys::fs::can_execute(U8Result))
       break; // Found it.
+
+    U8Result.clear();
   }
 
-  if (Len == 0)
+  if (U8Result.empty())
     return mapWindowsError(::GetLastError());
 
-  U16Result.set_size(Len);
-
-  SmallVector<char, MAX_PATH> U8Result;
-  if (std::error_code EC =
-          windows::UTF16ToUTF8(U16Result.data(), U16Result.size(), U8Result))
-    return EC;
-
   return std::string(U8Result.begin(), U8Result.end());
 }
 
diff --git a/llvm/lib/Support/raw_ostream.cpp b/llvm/lib/Support/raw_ostream.cpp
index 8f10d136bc388fba5252fa7f237d3696189b8042..440b49f8b8dd88a9a33833e16e61cecd7f76ea75 100644
--- a/llvm/lib/Support/raw_ostream.cpp
+++ b/llvm/lib/Support/raw_ostream.cpp
@@ -989,3 +989,31 @@ void raw_pwrite_stream::anchor() {}
 void buffer_ostream::anchor() {}
 
 void buffer_unique_ostream::anchor() {}
+
+Error llvm::writeToOutput(StringRef OutputFileName,
+                          std::function<Error(raw_ostream &)> Write) {
+  if (OutputFileName == "-")
+    return Write(outs());
+
+  if (OutputFileName == "/dev/null") {
+    raw_null_ostream Out;
+    return Write(Out);
+  }
+
+  unsigned Mode = sys::fs::all_read | sys::fs::all_write | sys::fs::all_exe;
+  Expected<sys::fs::TempFile> Temp =
+      sys::fs::TempFile::create(OutputFileName + ".temp-stream-%%%%%%", Mode);
+  if (!Temp)
+    return createFileError(OutputFileName, Temp.takeError());
+
+  raw_fd_ostream Out(Temp->FD, false);
+
+  if (Error E = Write(Out)) {
+    if (Error DiscardError = Temp->discard())
+      return joinErrors(std::move(E), std::move(DiscardError));
+    return E;
+  }
+  Out.flush();
+
+  return Temp->keep(OutputFileName);
+}
diff --git a/llvm/lib/TableGen/Main.cpp b/llvm/lib/TableGen/Main.cpp
index 0ace5363dd0517cb776b56de3c7819a9b921136a..a58defaf515f1ce6b7028dd380d2704e13605383 100644
--- a/llvm/lib/TableGen/Main.cpp
+++ b/llvm/lib/TableGen/Main.cpp
@@ -70,7 +70,7 @@ static int createDependencyFile(const TGParser &Parser, const char *argv0) {
     return reportError(argv0, "the option -d must be used together with -o\n");
 
   std::error_code EC;
-  ToolOutputFile DepOut(DependFilename, EC, sys::fs::OF_None);
+  ToolOutputFile DepOut(DependFilename, EC, sys::fs::OF_Text);
   if (EC)
     return reportError(argv0, "error opening " + DependFilename + ":" +
                                   EC.message() + "\n");
@@ -93,7 +93,7 @@ int llvm::TableGenMain(const char *argv0, TableGenMainFn *MainFn) {
 
   Records.startTimer("Parse, build records");
   ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
-      MemoryBuffer::getFileOrSTDIN(InputFilename);
+      MemoryBuffer::getFileOrSTDIN(InputFilename, /*IsText=*/false);
   if (std::error_code EC = FileOrErr.getError())
     return reportError(argv0, "Could not open input file '" + InputFilename +
                                   "': " + EC.message() + "\n");
@@ -137,13 +137,14 @@ int llvm::TableGenMain(const char *argv0, TableGenMainFn *MainFn) {
     // Only updates the real output file if there are any differences.
     // This prevents recompilation of all the files depending on it if there
     // aren't any.
-    if (auto ExistingOrErr = MemoryBuffer::getFile(OutputFilename))
+    if (auto ExistingOrErr =
+            MemoryBuffer::getFile(OutputFilename, /*IsText=*/false))
       if (std::move(ExistingOrErr.get())->getBuffer() == Out.str())
         WriteFile = false;
   }
   if (WriteFile) {
     std::error_code EC;
-    ToolOutputFile OutFile(OutputFilename, EC, sys::fs::OF_None);
+    ToolOutputFile OutFile(OutputFilename, EC, sys::fs::OF_Text);
     if (EC)
       return reportError(argv0, "error opening " + OutputFilename + ": " +
                                     EC.message() + "\n");
diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index 13212098514d33cfb3ba376cae4e1915e4f1193e..3172d711e7f63511b975582e011979989c7864bc 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -2344,13 +2344,13 @@ void Record::resolveReferences(Resolver &R, const RecordVal *SkipVal) {
         if (TypedInit *VRT = dyn_cast<TypedInit>(VR))
           Type =
               (Twine("of type '") + VRT->getType()->getAsString() + "' ").str();
-        PrintFatalError(getLoc(), Twine("Invalid value ") + Type +
-                                      "is found when setting '" +
-                                      Value.getNameInitAsString() +
-                                      "' of type '" +
-                                      Value.getType()->getAsString() +
-                                      "' after resolving references: " +
-                                      VR->getAsUnquotedString() + "\n");
+        PrintFatalError(
+            getLoc(),
+            Twine("Invalid value ") + Type + "found when setting field '" +
+                Value.getNameInitAsString() + "' of type '" +
+                Value.getType()->getAsString() +
+                "' after resolving references: " + VR->getAsUnquotedString() +
+                "\n");
       }
     }
   }
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index 87faf77671c6751b063f5ab086264c1eaabe7b9b..974df42de4c1cb2d41491cff1c21c6441741de21 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -229,38 +229,33 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, Init *ValName,
 /// args as SubClass's template arguments.
 bool TGParser::AddSubClass(Record *CurRec, SubClassReference &SubClass) {
   Record *SC = SubClass.Rec;
-  // Add all of the values in the subclass into the current class.
-  for (const RecordVal &Val : SC->getValues())
-    if (AddValue(CurRec, SubClass.RefRange.Start, Val))
-      return true;
-
-  ArrayRef<Init *> TArgs = SC->getTemplateArgs();
-
-  // Ensure that an appropriate number of template arguments are specified.
-  if (TArgs.size() < SubClass.TemplateArgs.size())
-    return Error(SubClass.RefRange.Start,
-                 "More template args specified than expected");
-
-  // Loop over all of the template arguments, setting them to the specified
-  // value or leaving them as the default if necessary.
   MapResolver R(CurRec);
 
-  for (unsigned i = 0, e = TArgs.size(); i != e; ++i) {
-    if (i < SubClass.TemplateArgs.size()) {
-      // If a value is specified for this template arg, set it now.
-      if (SetValue(CurRec, SubClass.RefRange.Start, TArgs[i],
-                   None, SubClass.TemplateArgs[i]))
+  // Loop over all the subclass record's fields. Add template arguments
+  // to the resolver map. Add regular fields to the new record.
+  for (const RecordVal &Field : SC->getValues()) {
+    if (Field.isTemplateArg()) {
+      R.set(Field.getNameInit(), Field.getValue());
+    } else {
+      if (AddValue(CurRec, SubClass.RefRange.Start, Field))
         return true;
-    } else if (!CurRec->getValue(TArgs[i])->getValue()->isComplete()) {
-      return Error(SubClass.RefRange.Start,
-                   "Value not specified for template argument #" +
-                   Twine(i) + " (" + TArgs[i]->getAsUnquotedString() +
-                   ") of subclass '" + SC->getNameInitAsString() + "'!");
     }
+  }
 
-    R.set(TArgs[i], CurRec->getValue(TArgs[i])->getValue());
-
-    CurRec->removeValue(TArgs[i]);
+  ArrayRef<Init *> TArgs = SC->getTemplateArgs();
+  assert(SubClass.TemplateArgs.size() <= TArgs.size() &&
+         "Too many template arguments allowed");
+
+  // Loop over the template argument names. If a value was specified,
+  // reset the map value. If not and there was no default, complain.
+  for (unsigned I = 0, E = TArgs.size(); I != E; ++I) {
+    if (I < SubClass.TemplateArgs.size())
+      R.set(TArgs[I], SubClass.TemplateArgs[I]);
+    else if (!R.isComplete(TArgs[I]))
+      return Error(SubClass.RefRange.Start,
+                   "Value not specified for template argument '" +
+                       TArgs[I]->getAsUnquotedString() + "' (#" + Twine(I) +
+                       ") of parent class '" + SC->getNameInitAsString() + "'");
   }
 
   Init *Name;
@@ -584,8 +579,8 @@ MultiClass *TGParser::ParseMultiClassID() {
   return Result;
 }
 
-/// ParseSubClassReference - Parse a reference to a subclass or to a templated
-/// subclass.  This returns a SubClassRefTy with a null Record* on error.
+/// ParseSubClassReference - Parse a reference to a subclass or a
+/// multiclass. This returns a SubClassRefTy with a null Record* on error.
 ///
 ///  SubClassRef ::= ClassID
 ///  SubClassRef ::= ClassID '<' ValueList '>'
@@ -609,25 +604,18 @@ ParseSubClassReference(Record *CurRec, bool isDefm) {
     return Result;
   }
 
-  if (Lex.getCode() == tgtok::greater) {
-    TokError("subclass reference requires a non-empty list of template values");
-    Result.Rec = nullptr;
+  if (ParseTemplateArgValueList(Result.TemplateArgs, CurRec, Result.Rec)) {
+    Result.Rec = nullptr; // Error parsing value list.
     return Result;
   }
 
-  ParseValueList(Result.TemplateArgs, CurRec, Result.Rec);
-  if (Result.TemplateArgs.empty()) {
-    Result.Rec = nullptr;   // Error parsing value list.
+  if (CheckTemplateArgValues(Result.TemplateArgs, Result.RefRange.Start,
+                             Result.Rec)) {
+    Result.Rec = nullptr; // Error checking value list.
     return Result;
   }
 
-  if (!consume(tgtok::greater)) {
-    TokError("expected '>' in template value list");
-    Result.Rec = nullptr;
-    return Result;
-  }
   Result.RefRange.End = Lex.getLoc();
-
   return Result;
 }
 
@@ -652,23 +640,12 @@ ParseSubMultiClassReference(MultiClass *CurMC) {
     return Result;
   }
 
-  if (Lex.getCode() == tgtok::greater) {
-    TokError("subclass reference requires a non-empty list of template values");
-    Result.MC = nullptr;
+  if (ParseTemplateArgValueList(Result.TemplateArgs, &CurMC->Rec,
+                                &Result.MC->Rec)) {
+    Result.MC = nullptr; // Error parsing value list.
     return Result;
   }
 
-  ParseValueList(Result.TemplateArgs, &CurMC->Rec, &Result.MC->Rec);
-  if (Result.TemplateArgs.empty()) {
-    Result.MC = nullptr;   // Error parsing value list.
-    return Result;
-  }
-
-  if (!consume(tgtok::greater)) {
-    TokError("expected '>' in template value list");
-    Result.MC = nullptr;
-    return Result;
-  }
   Result.RefRange.End = Lex.getLoc();
 
   return Result;
@@ -2032,15 +2009,9 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
     if (Lex.Lex() != tgtok::less)  // consume the Id.
       return ParseIDValue(CurRec, Name, NameLoc, Mode);    // Value ::= IDValue
 
-    // Value ::= ID '<' ValueListNE '>'
-    if (Lex.Lex() == tgtok::greater) {
-      TokError("expected non-empty value list");
-      return nullptr;
-    }
-
-    // This is a CLASS<initvalslist> expression.  This is supposed to synthesize
-    // a new anonymous definition, deriving from CLASS<initvalslist> with no
-    // body.
+    // Value ::= CLASSID '<' ValueListNE '>' (CLASSID has been consumed)
+    // This is supposed to synthesize a new anonymous definition, deriving
+    // from the class with the template arguments, but no body.
     Record *Class = Records.getClass(Name->getValue());
     if (!Class) {
       Error(NameLoc, "Expected a class name, got '" + Name->getValue() + "'");
@@ -2048,44 +2019,26 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
     }
 
     SmallVector<Init *, 8> Args;
-    ParseValueList(Args, CurRec, Class);
-    if (Args.empty()) return nullptr;
-
-    if (!consume(tgtok::greater)) {
-      TokError("expected '>' at end of value list");
-      return nullptr;
-    }
-
-    // Typecheck the template arguments list
-    ArrayRef<Init *> ExpectedArgs = Class->getTemplateArgs();
-    if (ExpectedArgs.size() < Args.size()) {
-      Error(NameLoc,
-            "More template args specified than expected");
-      return nullptr;
-    }
-
-    for (unsigned i = 0, e = ExpectedArgs.size(); i != e; ++i) {
-      RecordVal *ExpectedArg = Class->getValue(ExpectedArgs[i]);
-      if (i < Args.size()) {
-        if (TypedInit *TI = dyn_cast<TypedInit>(Args[i])) {
-          RecTy *ExpectedType = ExpectedArg->getType();
-          if (!TI->getType()->typeIsConvertibleTo(ExpectedType)) {
-            Error(NameLoc,
-                  "Value specified for template argument #" + Twine(i) + " (" +
-                  ExpectedArg->getNameInitAsString() + ") is of type '" +
-                  TI->getType()->getAsString() + "', expected '" +
-                  ExpectedType->getAsString() + "': " + TI->getAsString());
-            return nullptr;
-          }
-          continue;
-        }
-      } else if (ExpectedArg->getValue()->isComplete())
-        continue;
-
-      Error(NameLoc,
-            "Value not specified for template argument #" + Twine(i) + " (" +
-            ExpectedArgs[i]->getAsUnquotedString() + ")");
-      return nullptr;
+    Lex.Lex(); // consume the <
+    if (ParseTemplateArgValueList(Args, CurRec, Class))
+      return nullptr; // Error parsing value list.
+
+    if (CheckTemplateArgValues(Args, NameLoc, Class))
+      return nullptr; // Error checking template argument values.
+
+    // Loop through the arguments that were not specified and make sure
+    // they have a complete value.
+    // TODO: If we just keep a required argument count, we can do away
+    //       with this checking.
+    ArrayRef<Init *> TArgs = Class->getTemplateArgs();
+    for (unsigned I = Args.size(), E = TArgs.size(); I < E; ++I) {
+      RecordVal *Arg = Class->getValue(TArgs[I]);
+      if (!Arg->getValue()->isComplete()) 
+        Error(NameLoc, "Value not specified for template argument '" +
+                           TArgs[I]->getAsUnquotedString() + "' (#" + Twine(I) +
+                           ") of parent class '" +
+                           Class->getNameInitAsString() + "'");
+              
     }
 
     return VarDefInit::get(Class, Args)->Fold();
@@ -2158,7 +2111,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
     }
 
     if (Lex.getCode() != tgtok::r_square) {
-      ParseValueList(Vals, CurRec, nullptr,
+      ParseValueList(Vals, CurRec,
                      GivenListTy ? GivenListTy->getElementType() : nullptr);
       if (Vals.empty()) return nullptr;
     }
@@ -2522,32 +2475,15 @@ void TGParser::ParseDagArgList(
   }
 }
 
-/// ParseValueList - Parse a comma separated list of values, returning them as a
-/// vector.  Note that this always expects to be able to parse at least one
-/// value.  It returns an empty list if this is not possible.
+/// ParseValueList - Parse a comma separated list of values, returning them
+/// in a vector. Note that this always expects to be able to parse at least one
+/// value. It returns an empty list if this is not possible.
 ///
 ///   ValueList ::= Value (',' Value)
 ///
-void TGParser::ParseValueList(SmallVectorImpl<Init*> &Result, Record *CurRec,
-                              Record *ArgsRec, RecTy *EltTy) {
-  RecTy *ItemType = EltTy;
-  unsigned int ArgN = 0;
-  if (ArgsRec && !EltTy) {
-    ArrayRef<Init *> TArgs = ArgsRec->getTemplateArgs();
-    if (TArgs.empty()) {
-      TokError("template argument provided to non-template class");
-      Result.clear();
-      return;
-    }
-    const RecordVal *RV = ArgsRec->getValue(TArgs[ArgN]);
-    if (!RV) {
-      errs() << "Cannot find template arg " << ArgN << " (" << TArgs[ArgN]
-        << ")\n";
-    }
-    assert(RV && "Template argument record not found??");
-    ItemType = RV->getType();
-    ++ArgN;
-  }
+void TGParser::ParseValueList(SmallVectorImpl<Init *> &Result, Record *CurRec,
+                              RecTy *ItemType) {
+
   Result.push_back(ParseValue(CurRec, ItemType));
   if (!Result.back()) {
     Result.clear();
@@ -2558,19 +2494,6 @@ void TGParser::ParseValueList(SmallVectorImpl<Init*> &Result, Record *CurRec,
     // ignore trailing comma for lists
     if (Lex.getCode() == tgtok::r_square)
       return;
-
-    if (ArgsRec && !EltTy) {
-      ArrayRef<Init *> TArgs = ArgsRec->getTemplateArgs();
-      if (ArgN >= TArgs.size()) {
-        TokError("too many template arguments");
-        Result.clear();
-        return;
-      }
-      const RecordVal *RV = ArgsRec->getValue(TArgs[ArgN]);
-      assert(RV && "Template argument record not found??");
-      ItemType = RV->getType();
-      ++ArgN;
-    }
     Result.push_back(ParseValue(CurRec, ItemType));
     if (!Result.back()) {
       Result.clear();
@@ -2579,9 +2502,48 @@ void TGParser::ParseValueList(SmallVectorImpl<Init*> &Result, Record *CurRec,
   }
 }
 
+// ParseTemplateArgValueList - Parse a template argument list with the syntax
+// shown, filling in the Result vector. The open angle has been consumed.
+// An empty argument list is allowed. Return false if okay, true if an 
+// error was detected.
+//
+//   TemplateArgList ::= '<' [Value {',' Value}*] '>'
+bool TGParser::ParseTemplateArgValueList(SmallVectorImpl<Init *> &Result,
+                                         Record *CurRec, Record *ArgsRec) {
+
+  assert(Result.empty() && "Result vector is not empty");
+  ArrayRef<Init *> TArgs = ArgsRec->getTemplateArgs();
+  unsigned ArgIndex = 0;
+  RecTy *ItemType;
+
+  if (consume(tgtok::greater)) // empty value list
+    return false;
+
+  while (true) {
+    if (ArgIndex >= TArgs.size()) {
+      TokError("Too many template arguments: " + utostr(ArgIndex + 1));
+      return true;
+    }
+    const RecordVal *Arg = ArgsRec->getValue(TArgs[ArgIndex]);
+    assert(Arg && "Template argument record not found");
+
+    ItemType = Arg->getType();
+    Init *Value = ParseValue(CurRec, ItemType);
+    if (!Value)
+      return true;
+    Result.push_back(Value);
+
+    if (consume(tgtok::greater)) // end of argument list?
+      return false;
+    if (!consume(tgtok::comma)) // must be comma
+      return true;
+    ++ArgIndex; 
+  }
+}
+
 /// ParseDeclaration - Read a declaration, returning the name of field ID, or an
-/// empty string on error.  This can happen in a number of different context's,
-/// including within a def or in the template args for a def (which which case
+/// empty string on error.  This can happen in a number of different contexts,
+/// including within a def or in the template args for a class (in which case
 /// CurRec will be non-null) and within the template args for a multiclass (in
 /// which case CurRec will be null, but CurMultiClass will be set).  This can
 /// also happen within a def that is within a multiclass, which will set both
@@ -2612,23 +2574,28 @@ Init *TGParser::ParseDeclaration(Record *CurRec,
   Init *DeclName = StringInit::get(Str);
   Lex.Lex();
 
-  if (ParsingTemplateArgs) {
-    if (CurRec)
-      DeclName = QualifyName(*CurRec, CurMultiClass, DeclName, ":");
-    else
-      assert(CurMultiClass);
-    if (CurMultiClass)
-      DeclName = QualifyName(CurMultiClass->Rec, CurMultiClass, DeclName,
-                             "::");
-  }
-
-  // Add the field to the record.
-  if (AddValue(CurRec, IdLoc, RecordVal(DeclName, IdLoc, Type,
-                                        HasField ? RecordVal::FK_NonconcreteOK
-                                                 : RecordVal::FK_Normal)))
+  bool BadField;
+  if (!ParsingTemplateArgs) { // def, possibly in a multiclass
+    BadField = AddValue(CurRec, IdLoc,
+                        RecordVal(DeclName, IdLoc, Type,
+                                  HasField ? RecordVal::FK_NonconcreteOK
+                                           : RecordVal::FK_Normal));
+
+  } else if (CurRec) { // class template argument
+    DeclName = QualifyName(*CurRec, CurMultiClass, DeclName, ":");
+    BadField = AddValue(CurRec, IdLoc, RecordVal(DeclName, IdLoc, Type,
+                                                 RecordVal::FK_TemplateArg));
+
+  } else { // multiclass template argument
+    assert(CurMultiClass && "invalid context for template argument");
+    DeclName = QualifyName(CurMultiClass->Rec, CurMultiClass, DeclName, "::");
+    BadField = AddValue(CurRec, IdLoc, RecordVal(DeclName, IdLoc, Type,
+                                                 RecordVal::FK_TemplateArg));
+  }
+  if (BadField)
     return nullptr;
 
-  // If a value is present, parse it.
+  // If a value is present, parse it and set new field's value.
   if (consume(tgtok::equal)) {
     SMLoc ValLoc = Lex.getLoc();
     Init *Val = ParseValue(CurRec, Type);
@@ -2715,7 +2682,7 @@ VarInit *TGParser::ParseForeachDeclaration(Init *&ForeachListValue) {
   if (!Ranges.empty()) {
     assert(!IterType && "Type already initialized?");
     IterType = IntRecTy::get();
-    std::vector<Init*> Values;
+    std::vector<Init *> Values;
     for (unsigned R : Ranges)
       Values.push_back(IntInit::get(R));
     ForeachListValue = ListInit::get(Values, IterType);
@@ -2729,7 +2696,7 @@ VarInit *TGParser::ParseForeachDeclaration(Init *&ForeachListValue) {
 
 /// ParseTemplateArgList - Read a template argument list, which is a non-empty
 /// sequence of template-declarations in <>'s.  If CurRec is non-null, these are
-/// template args for a def, which may or may not be in a multiclass.  If null,
+/// template args for a class, which may or may not be in a multiclass. If null,
 /// these are the template args for a multiclass.
 ///
 ///    TemplateArgList ::= '<' Declaration (',' Declaration)* '>'
@@ -3493,32 +3460,28 @@ bool TGParser::ParseDefm(MultiClass *CurMultiClass) {
   while (true) {
     if (!Ref.Rec) return true;
 
-    // To instantiate a multiclass, we need to first get the multiclass, then
-    // instantiate each def contained in the multiclass with the SubClassRef
-    // template parameters.
+    // To instantiate a multiclass, we get the multiclass and then loop
+    // through its template argument names. Substs contains a substitution
+    // value for each argument, either the value specified or the default.
+    // Then we can resolve the template arguments.
     MultiClass *MC = MultiClasses[std::string(Ref.Rec->getName())].get();
     assert(MC && "Didn't lookup multiclass correctly?");
-    ArrayRef<Init*> TemplateVals = Ref.TemplateArgs;
 
-    // Verify that the correct number of template arguments were specified.
+    ArrayRef<Init *> TemplateVals = Ref.TemplateArgs;
     ArrayRef<Init *> TArgs = MC->Rec.getTemplateArgs();
-    if (TArgs.size() < TemplateVals.size())
-      return Error(SubClassLoc,
-                   "more template args specified than multiclass expects");
-
     SubstStack Substs;
+
     for (unsigned i = 0, e = TArgs.size(); i != e; ++i) {
       if (i < TemplateVals.size()) {
         Substs.emplace_back(TArgs[i], TemplateVals[i]);
       } else {
         Init *Default = MC->Rec.getValue(TArgs[i])->getValue();
-        if (!Default->isComplete()) {
+        if (!Default->isComplete())
           return Error(SubClassLoc,
-                       "value not specified for template argument #" +
-                           Twine(i) + " (" + TArgs[i]->getAsUnquotedString() +
-                           ") of multiclass '" + MC->Rec.getNameInitAsString() +
-                           "'");
-        }
+                       "value not specified for template argument '" +
+                           TArgs[i]->getAsUnquotedString() + "' (#" +
+                           Twine(i) + ") of multiclass '" +
+                           MC->Rec.getNameInitAsString() + "'");
         Substs.emplace_back(TArgs[i], Default);
       }
     }
@@ -3537,7 +3500,7 @@ bool TGParser::ParseDefm(MultiClass *CurMultiClass) {
 
     SubClassLoc = Lex.getLoc();
 
-    // A defm can inherit from regular classes (non-multiclass) as
+    // A defm can inherit from regular classes (non-multiclasses) as
     // long as they come in the end of the inheritance list.
     InheritFromClass = (Records.getClass(Lex.getCurStrVal()) != nullptr);
 
@@ -3642,6 +3605,41 @@ bool TGParser::ParseFile() {
   return TokError("Unexpected token at top level");
 }
 
+// Check the types of the template argument values for a class
+// inheritance, multiclass invocation, or anonymous class invocation.
+// If necessary, replace an argument with a cast to the required type.
+// The argument count has already been checked.
+bool TGParser::CheckTemplateArgValues(SmallVectorImpl<llvm::Init *> &Values,
+                                      SMLoc Loc, Record *ArgsRec) {
+
+  ArrayRef<Init *> TArgs = ArgsRec->getTemplateArgs();
+
+  for (unsigned I = 0, E = Values.size(); I < E; ++I) {
+    RecordVal *Arg = ArgsRec->getValue(TArgs[I]);
+    RecTy *ArgType = Arg->getType();
+    auto *Value = Values[I];
+
+    if (TypedInit *ArgValue = dyn_cast<TypedInit>(Value)) { 
+      auto *CastValue = ArgValue->getCastTo(ArgType);
+      if (CastValue) {
+        assert((!isa<TypedInit>(CastValue) ||
+                cast<TypedInit>(CastValue)->getType()->typeIsA(ArgType)) &&
+               "result of template arg value cast has wrong type");
+        Values[I] = CastValue;
+      } else {
+        PrintFatalError(Loc,
+                        "Value specified for template argument '" +
+                            Arg->getNameInitAsString() + "' (#" + Twine(I) +
+                            ") is of type " + ArgValue->getType()->getAsString() +
+                            "; expected type " + ArgType->getAsString() + ": " +
+                            ArgValue->getAsString());
+      }
+    }
+  }
+
+  return false;
+}
+
 // Check an assertion: Obtain the condition value and be sure it is true.
 // If not, print a nonfatal error along with the message.
 void TGParser::CheckAssert(SMLoc Loc, Init *Condition, Init *Message) {
diff --git a/llvm/lib/TableGen/TGParser.h b/llvm/lib/TableGen/TGParser.h
index 578a56c9d01c23b89b027787fb9a37d5f8d45b09..5b847ab7344f19e0569f150ad4d640e6dde286db 100644
--- a/llvm/lib/TableGen/TGParser.h
+++ b/llvm/lib/TableGen/TGParser.h
@@ -243,8 +243,10 @@ private:  // Parser methods.
                          IDParseMode Mode = ParseValueMode);
   Init *ParseValue(Record *CurRec, RecTy *ItemType = nullptr,
                    IDParseMode Mode = ParseValueMode);
-  void ParseValueList(SmallVectorImpl<llvm::Init*> &Result, Record *CurRec,
-                      Record *ArgsRec = nullptr, RecTy *EltTy = nullptr);
+  void ParseValueList(SmallVectorImpl<llvm::Init*> &Result,
+                      Record *CurRec, RecTy *ItemType = nullptr);
+  bool ParseTemplateArgValueList(SmallVectorImpl<llvm::Init *> &Result,
+                                 Record *CurRec, Record *ArgsRec);
   void ParseDagArgList(
       SmallVectorImpl<std::pair<llvm::Init*, StringInit*>> &Result,
       Record *CurRec);
@@ -264,6 +266,8 @@ private:  // Parser methods.
   MultiClass *ParseMultiClassID();
   bool ApplyLetStack(Record *CurRec);
   bool ApplyLetStack(RecordsEntry &Entry);
+  bool CheckTemplateArgValues(SmallVectorImpl<llvm::Init *> &Values,
+                              SMLoc Loc, Record *ArgsRec);
   void CheckAssert(SMLoc Loc, Init *Condition, Init *Message);
   void CheckRecordAsserts(Record &Rec);
 };
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index 144e6b747f514e24826dd6b7bbdc424293122747..d5ea2d3eee982fccf7208623e1791ce00abbf9ca 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -24,10 +24,20 @@ def icmp_redundant_trunc : GICombineRule<
          [{ return matchICmpRedundantTrunc(*${root}, MRI, Helper.getKnownBits(), ${matchinfo}); }]),
   (apply [{ applyICmpRedundantTrunc(*${root}, MRI, B, Observer, ${matchinfo}); }])>;
 
+// AArch64-specific offset folding for G_GLOBAL_VALUE.
+def fold_global_offset_matchdata : GIDefMatchData<"std::pair<uint64_t, uint64_t>">;
+def fold_global_offset : GICombineRule<
+  (defs root:$root, fold_global_offset_matchdata:$matchinfo),
+  (match (wip_match_opcode G_GLOBAL_VALUE):$root,
+          [{ return matchFoldGlobalOffset(*${root}, MRI, ${matchinfo}); }]),
+  (apply [{  return applyFoldGlobalOffset(*${root}, MRI, B, Observer, ${matchinfo});}])
+>;
+
 def AArch64PreLegalizerCombinerHelper: GICombinerHelper<
   "AArch64GenPreLegalizerCombinerHelper", [all_combines,
                                            fconstant_to_constant,
-                                           icmp_redundant_trunc]> {
+                                           icmp_redundant_trunc,
+                                           fold_global_offset]> {
   let DisableRuleOption = "aarch64prelegalizercombiner-disable-rule";
   let StateClass = "AArch64PreLegalizerCombinerHelperState";
   let AdditionalArguments = [];
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 613895dd3625a649d9e68f5d01792cd3f48f042f..6497d8e9f052abde9a2c69cca57e487891d32cc1 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -605,6 +605,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FNEARBYINT,  MVT::f16,  Promote);
     setOperationAction(ISD::FRINT,       MVT::f16,  Promote);
     setOperationAction(ISD::FROUND,      MVT::f16,  Promote);
+    setOperationAction(ISD::FROUNDEVEN,  MVT::f16,  Promote);
     setOperationAction(ISD::FTRUNC,      MVT::f16,  Promote);
     setOperationAction(ISD::FMINNUM,     MVT::f16,  Promote);
     setOperationAction(ISD::FMAXNUM,     MVT::f16,  Promote);
@@ -624,6 +625,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FABS,        MVT::v4f16, Expand);
     setOperationAction(ISD::FNEG,        MVT::v4f16, Expand);
     setOperationAction(ISD::FROUND,      MVT::v4f16, Expand);
+    setOperationAction(ISD::FROUNDEVEN,  MVT::v4f16, Expand);
     setOperationAction(ISD::FMA,         MVT::v4f16, Expand);
     setOperationAction(ISD::SETCC,       MVT::v4f16, Expand);
     setOperationAction(ISD::BR_CC,       MVT::v4f16, Expand);
@@ -648,6 +650,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FNEARBYINT,  MVT::v8f16, Expand);
     setOperationAction(ISD::FNEG,        MVT::v8f16, Expand);
     setOperationAction(ISD::FROUND,      MVT::v8f16, Expand);
+    setOperationAction(ISD::FROUNDEVEN,  MVT::v8f16, Expand);
     setOperationAction(ISD::FRINT,       MVT::v8f16, Expand);
     setOperationAction(ISD::FSQRT,       MVT::v8f16, Expand);
     setOperationAction(ISD::FSUB,        MVT::v8f16, Expand);
@@ -667,6 +670,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FRINT, Ty, Legal);
     setOperationAction(ISD::FTRUNC, Ty, Legal);
     setOperationAction(ISD::FROUND, Ty, Legal);
+    setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
     setOperationAction(ISD::FMINNUM, Ty, Legal);
     setOperationAction(ISD::FMAXNUM, Ty, Legal);
     setOperationAction(ISD::FMINIMUM, Ty, Legal);
@@ -684,6 +688,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FRINT,   MVT::f16, Legal);
     setOperationAction(ISD::FTRUNC,  MVT::f16, Legal);
     setOperationAction(ISD::FROUND,  MVT::f16, Legal);
+    setOperationAction(ISD::FROUNDEVEN,  MVT::f16, Legal);
     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
     setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
@@ -943,6 +948,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FPOW, MVT::v1f64, Expand);
     setOperationAction(ISD::FREM, MVT::v1f64, Expand);
     setOperationAction(ISD::FROUND, MVT::v1f64, Expand);
+    setOperationAction(ISD::FROUNDEVEN, MVT::v1f64, Expand);
     setOperationAction(ISD::FRINT, MVT::v1f64, Expand);
     setOperationAction(ISD::FSIN, MVT::v1f64, Expand);
     setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand);
@@ -1069,6 +1075,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::FRINT, Ty, Legal);
       setOperationAction(ISD::FTRUNC, Ty, Legal);
       setOperationAction(ISD::FROUND, Ty, Legal);
+      setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
     }
 
     if (Subtarget->hasFullFP16()) {
@@ -1079,6 +1086,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
         setOperationAction(ISD::FRINT, Ty, Legal);
         setOperationAction(ISD::FTRUNC, Ty, Legal);
         setOperationAction(ISD::FROUND, Ty, Legal);
+        setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
       }
     }
 
@@ -1127,6 +1135,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
+      setOperationAction(ISD::STEP_VECTOR, VT, Custom);
 
       setOperationAction(ISD::MULHU, VT, Expand);
       setOperationAction(ISD::MULHS, VT, Expand);
@@ -1389,6 +1398,7 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
   setOperationAction(ISD::CTLZ, VT, Custom);
   setOperationAction(ISD::CTPOP, VT, Custom);
   setOperationAction(ISD::CTTZ, VT, Custom);
+  setOperationAction(ISD::FABS, VT, Custom);
   setOperationAction(ISD::FADD, VT, Custom);
   setOperationAction(ISD::FCEIL, VT, Custom);
   setOperationAction(ISD::FDIV, VT, Custom);
@@ -1403,6 +1413,7 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
   setOperationAction(ISD::FNEG, VT, Custom);
   setOperationAction(ISD::FRINT, VT, Custom);
   setOperationAction(ISD::FROUND, VT, Custom);
+  setOperationAction(ISD::FROUNDEVEN, VT, Custom);
   setOperationAction(ISD::FSQRT, VT, Custom);
   setOperationAction(ISD::FSUB, VT, Custom);
   setOperationAction(ISD::FTRUNC, VT, Custom);
@@ -4108,7 +4119,7 @@ SDValue AArch64TargetLowering::LowerMGATHER(SDValue Op,
 
   if (VT.isFloatingPoint()) {
     SDValue Cast = getSVESafeBitCast(VT, Gather, DAG);
-    return DAG.getMergeValues({Cast, Gather}, DL);
+    return DAG.getMergeValues({Cast, Gather.getValue(1)}, DL);
   }
 
   return Gather;
@@ -4392,6 +4403,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::SPLAT_VECTOR:
     return LowerSPLAT_VECTOR(Op, DAG);
+  case ISD::STEP_VECTOR:
+    return LowerSTEP_VECTOR(Op, DAG);
   case ISD::EXTRACT_SUBVECTOR:
     return LowerEXTRACT_SUBVECTOR(Op, DAG);
   case ISD::INSERT_SUBVECTOR:
@@ -9039,6 +9052,20 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
   return GenerateTBL(Op, ShuffleMask, DAG);
 }
 
+SDValue AArch64TargetLowering::LowerSTEP_VECTOR(SDValue Op,
+                                                SelectionDAG &DAG) const {
+  SDLoc dl(Op);
+  EVT VT = Op.getValueType();
+  assert(VT.isScalableVector() &&
+         "Only expect scalable vectors for STEP_VECTOR");
+  assert(VT.getScalarType() != MVT::i1 &&
+         "Vectors of i1 types not supported for STEP_VECTOR");
+
+  SDValue StepVal = Op.getOperand(0);
+  SDValue Zero = DAG.getConstant(0, dl, StepVal.getValueType());
+  return DAG.getNode(AArch64ISD::INDEX_VECTOR, dl, VT, Zero, StepVal);
+}
+
 SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
                                                  SelectionDAG &DAG) const {
   SDLoc dl(Op);
@@ -9653,10 +9680,10 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
     }
     if (i > 0)
       isOnlyLowElement = false;
-    if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
+    if (!isIntOrFPConstant(V))
       isConstant = false;
 
-    if (isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V)) {
+    if (isIntOrFPConstant(V)) {
       ++NumConstantLanes;
       if (!ConstantValue.getNode())
         ConstantValue = V;
@@ -9822,7 +9849,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
     for (unsigned i = 0; i < NumElts; ++i) {
       SDValue V = Op.getOperand(i);
       SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
-      if (!isa<ConstantSDNode>(V) && !isa<ConstantFPSDNode>(V))
+      if (!isIntOrFPConstant(V))
         // Note that type legalization likely mucked about with the VT of the
         // source operand, so we may have to convert it here before inserting.
         Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx);
@@ -13181,6 +13208,13 @@ static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
   SDValue RHS = Op->getOperand(1);
   SetCCInfoAndKind InfoAndKind;
 
+  // If both operands are a SET_CC, then we don't want to perform this
+  // folding and create another csel as this results in more instructions
+  // (and higher register usage).
+  if (isSetCCOrZExtSetCC(LHS, InfoAndKind) &&
+      isSetCCOrZExtSetCC(RHS, InfoAndKind))
+    return SDValue();
+
   // If neither operand is a SET_CC, give up.
   if (!isSetCCOrZExtSetCC(LHS, InfoAndKind)) {
     std::swap(LHS, RHS);
@@ -14400,6 +14434,63 @@ static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG) {
   return SDValue();
 }
 
+static SDValue performGLD1Combine(SDNode *N, SelectionDAG &DAG) {
+  unsigned Opc = N->getOpcode();
+
+  assert(((Opc >= AArch64ISD::GLD1_MERGE_ZERO && // unsigned gather loads
+           Opc <= AArch64ISD::GLD1_IMM_MERGE_ZERO) ||
+          (Opc >= AArch64ISD::GLD1S_MERGE_ZERO && // signed gather loads
+           Opc <= AArch64ISD::GLD1S_IMM_MERGE_ZERO)) &&
+         "Invalid opcode.");
+
+  const bool Scaled = Opc == AArch64ISD::GLD1_SCALED_MERGE_ZERO ||
+                      Opc == AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
+  const bool Signed = Opc == AArch64ISD::GLD1S_MERGE_ZERO ||
+                      Opc == AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
+  const bool Extended = Opc == AArch64ISD::GLD1_SXTW_MERGE_ZERO ||
+                        Opc == AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO ||
+                        Opc == AArch64ISD::GLD1_UXTW_MERGE_ZERO ||
+                        Opc == AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO;
+
+  SDLoc DL(N);
+  SDValue Chain = N->getOperand(0);
+  SDValue Pg = N->getOperand(1);
+  SDValue Base = N->getOperand(2);
+  SDValue Offset = N->getOperand(3);
+  SDValue Ty = N->getOperand(4);
+
+  EVT ResVT = N->getValueType(0);
+
+  const auto OffsetOpc = Offset.getOpcode();
+  const bool OffsetIsZExt =
+      OffsetOpc == AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU;
+  const bool OffsetIsSExt =
+      OffsetOpc == AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU;
+
+  // Fold sign/zero extensions of vector offsets into GLD1 nodes where possible.
+  if (!Extended && (OffsetIsSExt || OffsetIsZExt)) {
+    SDValue ExtPg = Offset.getOperand(0);
+    VTSDNode *ExtFrom = cast<VTSDNode>(Offset.getOperand(2).getNode());
+    EVT ExtFromEVT = ExtFrom->getVT().getVectorElementType();
+
+    // If the predicate for the sign- or zero-extended offset is the
+    // same as the predicate used for this load and the sign-/zero-extension
+    // was from a 32-bits...
+    if (ExtPg == Pg && ExtFromEVT == MVT::i32) {
+      SDValue UnextendedOffset = Offset.getOperand(1);
+
+      unsigned NewOpc = getGatherVecOpcode(Scaled, OffsetIsSExt, true);
+      if (Signed)
+        NewOpc = getSignExtendedGatherOpcode(NewOpc);
+
+      return DAG.getNode(NewOpc, DL, {ResVT, MVT::Other},
+                         {Chain, Pg, Base, UnextendedOffset, Ty});
+    }
+  }
+
+  return SDValue();
+}
+
 /// Target-specific DAG combine function for post-increment LD1 (lane) and
 /// post-increment LD1R.
 static SDValue performPostLD1Combine(SDNode *N,
@@ -15139,7 +15230,8 @@ static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
     }
   }
 
-  if (N0.getOpcode() != ISD::SETCC || CCVT.getVectorNumElements() != 1 ||
+  if (N0.getOpcode() != ISD::SETCC ||
+      CCVT.getVectorElementCount() != ElementCount::getFixed(1) ||
       CCVT.getVectorElementType() != MVT::i1)
     return SDValue();
 
@@ -15777,6 +15869,21 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
     return performNVCASTCombine(N);
   case AArch64ISD::UZP1:
     return performUzpCombine(N, DAG);
+  case AArch64ISD::GLD1_MERGE_ZERO:
+  case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
+  case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
+  case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLD1_IMM_MERGE_ZERO:
+  case AArch64ISD::GLD1S_MERGE_ZERO:
+  case AArch64ISD::GLD1S_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLD1S_UXTW_MERGE_ZERO:
+  case AArch64ISD::GLD1S_SXTW_MERGE_ZERO:
+  case AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO:
+  case AArch64ISD::GLD1S_IMM_MERGE_ZERO:
+    return performGLD1Combine(N, DAG);
   case ISD::INSERT_VECTOR_ELT:
     return performPostLD1Combine(N, DCI, true);
   case ISD::EXTRACT_VECTOR_ELT:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 1264d67799246a8d0b44d27d57558204b81f4c3a..cd9185be40e49abdef318b1f73f50e4dae9707ba 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -936,6 +936,7 @@ private:
   SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerSTEP_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG, unsigned NewOp,
                               bool OverrideNEON = false) const;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f62ff547333ac0ecf416688423587926023f03ae..0fb4f0891688e10d18757f80f7b7d2d83fca819d 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -2162,6 +2162,11 @@ def : Pat<(AArch64csel (i32 -1), GPR32:$fval, (i32 imm:$cc), NZCV),
 def : Pat<(AArch64csel (i64 -1), GPR64:$fval, (i32 imm:$cc), NZCV),
           (CSINVXr GPR64:$fval, XZR, (i32 (inv_cond_XFORM imm:$cc)))>;
 
+def : Pat<(add GPR32:$val, (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV)),
+          (CSINCWr GPR32:$val, GPR32:$val, (i32 imm:$cc))>;
+def : Pat<(add GPR64:$val, (zext (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV))),
+          (CSINCXr GPR64:$val, GPR64:$val, (i32 imm:$cc))>;
+
 // The inverse of the condition code from the alias instruction is what is used
 // in the aliased instruction. The parser all ready inverts the condition code
 // for these aliases.
@@ -3796,12 +3801,9 @@ defm FNEG   : SingleOperandFPData<0b0010, "fneg", fneg>;
 defm FRINTA : SingleOperandFPData<0b1100, "frinta", fround>;
 defm FRINTI : SingleOperandFPData<0b1111, "frinti", fnearbyint>;
 defm FRINTM : SingleOperandFPData<0b1010, "frintm", ffloor>;
-defm FRINTN : SingleOperandFPData<0b1000, "frintn", int_aarch64_neon_frintn>;
+defm FRINTN : SingleOperandFPData<0b1000, "frintn", froundeven>;
 defm FRINTP : SingleOperandFPData<0b1001, "frintp", fceil>;
 
-def : Pat<(v1f64 (int_aarch64_neon_frintn (v1f64 FPR64:$Rn))),
-          (FRINTNDr FPR64:$Rn)>;
-
 defm FRINTX : SingleOperandFPData<0b1110, "frintx", frint>;
 defm FRINTZ : SingleOperandFPData<0b1011, "frintz", ftrunc>;
 
@@ -4090,7 +4092,7 @@ defm FRECPE : SIMDTwoVectorFP<0, 1, 0b11101, "frecpe", int_aarch64_neon_frecpe>;
 defm FRINTA : SIMDTwoVectorFP<1, 0, 0b11000, "frinta", fround>;
 defm FRINTI : SIMDTwoVectorFP<1, 1, 0b11001, "frinti", fnearbyint>;
 defm FRINTM : SIMDTwoVectorFP<0, 0, 0b11001, "frintm", ffloor>;
-defm FRINTN : SIMDTwoVectorFP<0, 0, 0b11000, "frintn", int_aarch64_neon_frintn>;
+defm FRINTN : SIMDTwoVectorFP<0, 0, 0b11000, "frintn", froundeven>;
 defm FRINTP : SIMDTwoVectorFP<0, 1, 0b11000, "frintp", fceil>;
 defm FRINTX : SIMDTwoVectorFP<1, 0, 0b11001, "frintx", frint>;
 defm FRINTZ : SIMDTwoVectorFP<0, 1, 0b11001, "frintz", ftrunc>;
@@ -4744,6 +4746,27 @@ def : Pat<(f64 (AArch64frsqrts (f64 FPR64:$Rn), (f64 FPR64:$Rm))),
 def : Pat<(v2f64 (AArch64frsqrts (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))),
           (FRSQRTSv2f64 FPR128:$Rn, FPR128:$Rm)>;
 
+// Some float -> int -> float conversion patterns for which we want to keep the
+// int values in FP registers using the corresponding NEON instructions to
+// avoid more costly int <-> fp register transfers.
+let Predicates = [HasNEON] in {
+def : Pat<(f64 (sint_to_fp (i64 (fp_to_sint f64:$Rn)))),
+          (SCVTFv1i64 (i64 (FCVTZSv1i64 f64:$Rn)))>;
+def : Pat<(f32 (sint_to_fp (i32 (fp_to_sint f32:$Rn)))),
+          (SCVTFv1i32 (i32 (FCVTZSv1i32 f32:$Rn)))>;
+def : Pat<(f64 (uint_to_fp (i64 (fp_to_uint f64:$Rn)))),
+          (UCVTFv1i64 (i64 (FCVTZUv1i64 f64:$Rn)))>;
+def : Pat<(f32 (uint_to_fp (i32 (fp_to_uint f32:$Rn)))),
+          (UCVTFv1i32 (i32 (FCVTZUv1i32 f32:$Rn)))>;
+
+let Predicates = [HasFullFP16] in {
+def : Pat<(f16 (sint_to_fp (i32 (fp_to_sint f16:$Rn)))),
+          (SCVTFv1i16 (f16 (FCVTZSv1f16 f16:$Rn)))>;
+def : Pat<(f16 (uint_to_fp (i32 (fp_to_uint f16:$Rn)))),
+          (UCVTFv1i16 (f16 (FCVTZUv1f16 f16:$Rn)))>;
+}
+}
+
 // If an integer is about to be converted to a floating point value,
 // just load it on the floating point unit.
 // Here are the patterns for 8 and 16-bits to float.
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA55.td b/llvm/lib/Target/AArch64/AArch64SchedA55.td
index ff7766f2caecf137b966534d8d2a757f2024d965..0015c27228f6897596593ae7ae3a5bbc01672e9b 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedA55.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedA55.td
@@ -339,5 +339,4 @@ def : InstRW<[CortexA55WriteFSqrtHP], (instregex "^.*SQRT.*16$")>;
 def : InstRW<[CortexA55WriteFSqrtSP], (instregex "^.*SQRT.*32$")>;
 def : InstRW<[CortexA55WriteFSqrtDP], (instregex "^.*SQRT.*64$")>;
 
-def A55RCU : RetireControlUnit<64, 0>;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 4851458e13e96c6d258c8745e5be88f373549383..d199213e140d0c9a3ba828c4eeb89bdb831d7736 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -260,6 +260,19 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
       return LT.first;
     break;
   }
+  case Intrinsic::experimental_stepvector: {
+    unsigned Cost = 1; // Cost of the `index' instruction
+    auto LT = TLI->getTypeLegalizationCost(DL, RetTy);
+    // Legalisation of illegal vectors involves an `index' instruction plus
+    // (LT.first - 1) vector adds.
+    if (LT.first > 1) {
+      Type *LegalVTy = EVT(LT.second).getTypeForEVT(RetTy->getContext());
+      unsigned AddCost =
+          getArithmeticInstrCost(Instruction::Add, LegalVTy, CostKind);
+      Cost += AddCost * (LT.first - 1);
+    }
+    return Cost;
+  }
   default:
     break;
   }
@@ -1217,7 +1230,8 @@ int AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode,
 }
 
 int AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
-                                   int Index, VectorType *SubTp) {
+                                   ArrayRef<int> Mask, int Index,
+                                   VectorType *SubTp) {
   if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose ||
       Kind == TTI::SK_Select || Kind == TTI::SK_PermuteSingleSrc ||
       Kind == TTI::SK_Reverse) {
@@ -1289,5 +1303,5 @@ int AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
       return LT.first * Entry->Cost;
   }
 
-  return BaseT::getShuffleCost(Kind, Tp, Index, SubTp);
+  return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
 }
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 89c9365d08479bbe66746c91a470ad6d6d1dc830..5dfa518828200cfe6e84d9390e83ca6393723e6b 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -100,15 +100,19 @@ public:
   unsigned getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                  TTI::TargetCostKind CostKind);
 
-  unsigned getRegisterBitWidth(bool Vector) const {
-    if (Vector) {
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+    switch (K) {
+    case TargetTransformInfo::RGK_Scalar:
+      return TypeSize::getFixed(64);
+    case TargetTransformInfo::RGK_FixedWidthVector:
       if (ST->hasSVE())
-        return std::max(ST->getMinSVEVectorSizeInBits(), 128u);
-      if (ST->hasNEON())
-        return 128;
-      return 0;
+        return TypeSize::getFixed(
+            std::max(ST->getMinSVEVectorSizeInBits(), 128u));
+      return TypeSize::getFixed(ST->hasNEON() ? 128 : 0);
+    case TargetTransformInfo::RGK_ScalableVector:
+      return TypeSize::getScalable(ST->hasSVE() ? 128 : 0);
     }
-    return 64;
+    llvm_unreachable("Unsupported register kind");
   }
 
   unsigned getMinVectorRegisterBitWidth() {
@@ -275,8 +279,8 @@ public:
                                  bool IsPairwiseForm,
                                  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput);
 
-  int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
-                     VectorType *SubTp);
+  int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask,
+                     int Index, VectorType *SubTp);
   /// @}
 };
 
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 037e24b1b486dd9bd143bd9902df6ed61f01f86b..9141e786977a44dfcf860079b4dd7979e11b7d2b 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -2906,9 +2906,11 @@ static const struct Extension {
     {"mte", {AArch64::FeatureMTE}},
     {"memtag", {AArch64::FeatureMTE}},
     {"tlb-rmi", {AArch64::FeatureTLB_RMI}},
+    {"pan", {AArch64::FeaturePAN}},
     {"pan-rwv", {AArch64::FeaturePAN_RWV}},
     {"ccpp", {AArch64::FeatureCCPP}},
     {"rcpc", {AArch64::FeatureRCPC}},
+    {"rng", {AArch64::FeatureRandGen}},
     {"sve", {AArch64::FeatureSVE}},
     {"sve2", {AArch64::FeatureSVE2}},
     {"sve2-aes", {AArch64::FeatureSVE2AES}},
@@ -2920,7 +2922,6 @@ static const struct Extension {
     {"pauth", {AArch64::FeaturePAuth}},
     {"flagm", {AArch64::FeatureFlagM}},
     // FIXME: Unsupported extensions
-    {"pan", {}},
     {"lor", {}},
     {"rdma", {}},
     {"profile", {}},
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index d16e2fd134752641d62e83971194e15a37a9697f..ef0d4c6ee93c94362a4037c7a2d85e4f666b235c 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -357,9 +357,14 @@ bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
               return false;
             }
           } else {
-            // A scalar extend.
-            CurVReg =
-                MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg}).getReg(0);
+            // If the split EVT was a <1 x T> vector, and NewVT is T, then we
+            // don't have to do anything since we don't distinguish between the
+            // two.
+            if (NewLLT != MRI.getType(CurVReg)) {
+              // A scalar extend.
+              CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
+                            .getReg(0);
+            }
           }
         }
       }
@@ -428,12 +433,19 @@ static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder,
   }
 }
 
-bool AArch64CallLowering::fallBackToDAGISel(const Function &F) const {
+bool AArch64CallLowering::fallBackToDAGISel(const MachineFunction &MF) const {
+  auto &F = MF.getFunction();
   if (isa<ScalableVectorType>(F.getReturnType()))
     return true;
-  return llvm::any_of(F.args(), [](const Argument &A) {
-    return isa<ScalableVectorType>(A.getType());
-  });
+  if (llvm::any_of(F.args(), [](const Argument &A) {
+        return isa<ScalableVectorType>(A.getType());
+      }))
+    return true;
+  const auto &ST = MF.getSubtarget<AArch64Subtarget>();
+  LLVM_DEBUG(dbgs() << "Falling back to SDAG because we don't support no-NEON");
+  if (!ST.hasNEON() || !ST.hasFPARMv8())
+    return true;
+  return false;
 }
 
 bool AArch64CallLowering::lowerFormalArguments(
@@ -450,7 +462,7 @@ bool AArch64CallLowering::lowerFormalArguments(
     if (DL.getTypeStoreSize(Arg.getType()).isZero())
       continue;
 
-    ArgInfo OrigArg{VRegs[i], Arg.getType()};
+    ArgInfo OrigArg{VRegs[i], Arg};
     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);
 
     splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv());
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.h b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.h
index 1b9de9f93a33c063b53ddb2e2e2bc1a6848c2a3d..add0342c90fdc252862e7411670f2e74addeae08 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.h
@@ -37,7 +37,7 @@ public:
                    ArrayRef<Register> VRegs, FunctionLoweringInfo &FLI,
                    Register SwiftErrorVReg) const override;
 
-  bool fallBackToDAGISel(const Function &F) const override;
+  bool fallBackToDAGISel(const MachineFunction &MF) const override;
 
   bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F,
                             ArrayRef<ArrayRef<Register>> VRegs,
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 3b099d0b91bf32979b1ae7ac9a4992d596873db0..236e8f1c83da95ed661dabe761369298d5d8289d 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2305,6 +2305,30 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
   MachineIRBuilder MIB(I);
 
   switch (Opcode) {
+  case TargetOpcode::G_SBFX:
+  case TargetOpcode::G_UBFX: {
+    static const unsigned OpcTable[2][2] = {
+        {AArch64::UBFMWri, AArch64::UBFMXri},
+        {AArch64::SBFMWri, AArch64::SBFMXri}};
+    bool IsSigned = Opcode == TargetOpcode::G_SBFX;
+    unsigned Size = Ty.getSizeInBits();
+    unsigned Opc = OpcTable[IsSigned][Size == 64];
+    auto Cst1 =
+        getConstantVRegValWithLookThrough(I.getOperand(2).getReg(), MRI);
+    assert(Cst1 && "Should have gotten a constant for src 1?");
+    auto Cst2 =
+        getConstantVRegValWithLookThrough(I.getOperand(3).getReg(), MRI);
+    assert(Cst2 && "Should have gotten a constant for src 2?");
+    auto LSB = Cst1->Value.getZExtValue();
+    auto Width = Cst2->Value.getZExtValue();
+    MachineIRBuilder MIB(I);
+    auto BitfieldInst =
+        MIB.buildInstr(Opc, {I.getOperand(0)}, {I.getOperand(1)})
+            .addImm(LSB)
+            .addImm(Width);
+    I.eraseFromParent();
+    return constrainSelectedInstRegOperands(*BitfieldInst, TII, TRI, RBI);
+  }
   case TargetOpcode::G_BRCOND:
     return selectCompareBranch(I, MF, MRI);
 
@@ -5654,8 +5678,10 @@ AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
     return None;
 
   // TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
-  // TODO: Need to check GV's offset % size if doing offset folding into globals.
-  assert(Adrp.getOperand(1).getOffset() == 0 && "Unexpected offset in global");
+  auto Offset = Adrp.getOperand(1).getOffset();
+  if (Offset % Size != 0)
+    return None;
+
   auto GV = Adrp.getOperand(1).getGlobal();
   if (GV->isThreadLocal())
     return None;
@@ -5669,7 +5695,7 @@ AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
   Register AdrpReg = Adrp.getOperand(0).getReg();
   return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(AdrpReg); },
            [=](MachineInstrBuilder &MIB) {
-             MIB.addGlobalAddress(GV, /* Offset */ 0,
+             MIB.addGlobalAddress(GV, Offset,
                                   OpFlags | AArch64II::MO_PAGEOFF |
                                       AArch64II::MO_NC);
            }}};
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index b01f1fee9ea3a835a37575552de1e5c187311354..b5ea4eb462364f943366e665e500a62b11a70c92 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -647,6 +647,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .legalForCartesianProduct(
           {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
       .scalarize(1);
+  getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).lower();
 
   getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)
       .legalIf([=](const LegalityQuery &Query) {
@@ -681,7 +682,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
   getActionDefinitionsBuilder(G_DYN_STACKALLOC).lower();
 
-  getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
+  getActionDefinitionsBuilder({G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
+      .libcall();
 
   getActionDefinitionsBuilder(G_ABS).lowerIf(
       [=](const LegalityQuery &Query) { return Query.Types[0].isScalar(); });
@@ -699,6 +701,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
   getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
       .lowerIf([=](const LegalityQuery &Q) { return Q.Types[0].isScalar(); });
 
+  getActionDefinitionsBuilder({G_FSHL, G_FSHR}).lower();
+
+  getActionDefinitionsBuilder({G_SBFX, G_UBFX}).customFor({s32, s64});
+
   computeTables();
   verify(*ST.getInstrInfo());
 }
@@ -725,6 +731,9 @@ bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
     return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
   case TargetOpcode::G_TRUNC:
     return legalizeVectorTrunc(MI, Helper);
+  case TargetOpcode::G_SBFX:
+  case TargetOpcode::G_UBFX:
+    return legalizeBitfieldExtract(MI, MRI, Helper);
   }
 
   llvm_unreachable("expected switch to return");
@@ -785,7 +794,8 @@ bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
   // G_ADD_LOW instructions.
   // By splitting this here, we can optimize accesses in the small code model by
   // folding in the G_ADD_LOW into the load/store offset.
-  auto GV = MI.getOperand(1).getGlobal();
+  auto &GlobalOp = MI.getOperand(1);
+  const auto* GV = GlobalOp.getGlobal();
   if (GV->isThreadLocal())
     return true; // Don't want to modify TLS vars.
 
@@ -795,9 +805,10 @@ bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
   if (OpFlags & AArch64II::MO_GOT)
     return true;
 
+  auto Offset = GlobalOp.getOffset();
   Register DstReg = MI.getOperand(0).getReg();
   auto ADRP = MIRBuilder.buildInstr(AArch64::ADRP, {LLT::pointer(0, 64)}, {})
-                  .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
+                  .addGlobalAddress(GV, Offset, OpFlags | AArch64II::MO_PAGE);
   // Set the regclass on the dest reg too.
   MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
 
@@ -815,6 +826,8 @@ bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
   // binary must also be loaded into address range [0, 2^48). Both of these
   // properties need to be ensured at runtime when using tagged addresses.
   if (OpFlags & AArch64II::MO_TAGGED) {
+    assert(!Offset &&
+           "Should not have folded in an offset for a tagged global!");
     ADRP = MIRBuilder.buildInstr(AArch64::MOVKXi, {LLT::pointer(0, 64)}, {ADRP})
                .addGlobalAddress(GV, 0x100000000,
                                  AArch64II::MO_PREL | AArch64II::MO_G3)
@@ -823,7 +836,7 @@ bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
   }
 
   MIRBuilder.buildInstr(AArch64::G_ADD_LOW, {DstReg}, {ADRP})
-      .addGlobalAddress(GV, 0,
+      .addGlobalAddress(GV, Offset,
                         OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
   MI.eraseFromParent();
   return true;
@@ -940,3 +953,11 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
   MI.eraseFromParent();
   return true;
 }
+
+bool AArch64LegalizerInfo::legalizeBitfieldExtract(
+    MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
+  // Only legal if we can select immediate forms.
+  // TODO: Lower this otherwise.
+  return getConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI) &&
+         getConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
+}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index 8217e37c85128c1d789cacd8a2b4d7cde06ff15a..24c63d7da2ac9f0034dc64eba025401320a162eb 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -47,6 +47,8 @@ private:
                                   MachineIRBuilder &MIRBuilder,
                                   GISelChangeObserver &Observer) const;
   bool legalizeVectorTrunc(MachineInstr &MI, LegalizerHelper &Helper) const;
+  bool legalizeBitfieldExtract(MachineInstr &MI, MachineRegisterInfo &MRI,
+                               LegalizerHelper &Helper) const;
   const AArch64Subtarget *ST;
 };
 } // End llvm namespace.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index 6e7fe7c98512930c4a1f52afab9456db3256e17a..4efc63ea68b74b4ca0c1248166d11e596219ecc0 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -107,6 +107,156 @@ static bool applyICmpRedundantTrunc(MachineInstr &MI, MachineRegisterInfo &MRI,
   return true;
 }
 
+/// \returns true if it is possible to fold a constant into a G_GLOBAL_VALUE.
+///
+/// e.g.
+///
+/// %g = G_GLOBAL_VALUE @x -> %g = G_GLOBAL_VALUE @x + cst
+static bool matchFoldGlobalOffset(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                  std::pair<uint64_t, uint64_t> &MatchInfo) {
+  assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
+  MachineFunction &MF = *MI.getMF();
+  auto &GlobalOp = MI.getOperand(1);
+  auto *GV = GlobalOp.getGlobal();
+
+  // Don't allow anything that could represent offsets etc.
+  if (MF.getSubtarget<AArch64Subtarget>().ClassifyGlobalReference(
+          GV, MF.getTarget()) != AArch64II::MO_NO_FLAG)
+    return false;
+
+  // Look for a G_GLOBAL_VALUE only used by G_PTR_ADDs against constants:
+  //
+  //  %g = G_GLOBAL_VALUE @x
+  //  %ptr1 = G_PTR_ADD %g, cst1
+  //  %ptr2 = G_PTR_ADD %g, cst2
+  //  ...
+  //  %ptrN = G_PTR_ADD %g, cstN
+  //
+  // Identify the *smallest* constant. We want to be able to form this:
+  //
+  //  %offset_g = G_GLOBAL_VALUE @x + min_cst
+  //  %g = G_PTR_ADD %offset_g, -min_cst
+  //  %ptr1 = G_PTR_ADD %g, cst1
+  //  ...
+  Register Dst = MI.getOperand(0).getReg();
+  uint64_t MinOffset = -1ull;
+  for (auto &UseInstr : MRI.use_nodbg_instructions(Dst)) {
+    if (UseInstr.getOpcode() != TargetOpcode::G_PTR_ADD)
+      return false;
+    auto Cst =
+        getConstantVRegValWithLookThrough(UseInstr.getOperand(2).getReg(), MRI);
+    if (!Cst)
+      return false;
+    MinOffset = std::min(MinOffset, Cst->Value.getZExtValue());
+  }
+
+  // Require that the new offset is larger than the existing one to avoid
+  // infinite loops.
+  uint64_t CurrOffset = GlobalOp.getOffset();
+  uint64_t NewOffset = MinOffset + CurrOffset;
+  if (NewOffset <= CurrOffset)
+    return false;
+
+  // Check whether folding this offset is legal. It must not go out of bounds of
+  // the referenced object to avoid violating the code model, and must be
+  // smaller than 2^21 because this is the largest offset expressible in all
+  // object formats.
+  //
+  // This check also prevents us from folding negative offsets, which will end
+  // up being treated in the same way as large positive ones. They could also
+  // cause code model violations, and aren't really common enough to matter.
+  if (NewOffset >= (1 << 21))
+    return false;
+
+  Type *T = GV->getValueType();
+  if (!T->isSized() ||
+      NewOffset > GV->getParent()->getDataLayout().getTypeAllocSize(T))
+    return false;
+  MatchInfo = std::make_pair(NewOffset, MinOffset);
+  return true;
+}
+
+static bool applyFoldGlobalOffset(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                  MachineIRBuilder &B,
+                                  GISelChangeObserver &Observer,
+                                  std::pair<uint64_t, uint64_t> &MatchInfo) {
+  // Change:
+  //
+  //  %g = G_GLOBAL_VALUE @x
+  //  %ptr1 = G_PTR_ADD %g, cst1
+  //  %ptr2 = G_PTR_ADD %g, cst2
+  //  ...
+  //  %ptrN = G_PTR_ADD %g, cstN
+  //
+  // To:
+  //
+  //  %offset_g = G_GLOBAL_VALUE @x + min_cst
+  //  %g = G_PTR_ADD %offset_g, -min_cst
+  //  %ptr1 = G_PTR_ADD %g, cst1
+  //  ...
+  //  %ptrN = G_PTR_ADD %g, cstN
+  //
+  // Then, the original G_PTR_ADDs should be folded later on so that they look
+  // like this:
+  //
+  //  %ptrN = G_PTR_ADD %offset_g, cstN - min_cst
+  uint64_t Offset, MinOffset;
+  std::tie(Offset, MinOffset) = MatchInfo;
+  B.setInstrAndDebugLoc(MI);
+  Observer.changingInstr(MI);
+  auto &GlobalOp = MI.getOperand(1);
+  auto *GV = GlobalOp.getGlobal();
+  GlobalOp.ChangeToGA(GV, Offset, GlobalOp.getTargetFlags());
+  Register Dst = MI.getOperand(0).getReg();
+  Register NewGVDst = MRI.cloneVirtualRegister(Dst);
+  MI.getOperand(0).setReg(NewGVDst);
+  Observer.changedInstr(MI);
+  B.buildPtrAdd(
+      Dst, NewGVDst,
+      B.buildConstant(LLT::scalar(64), -static_cast<int64_t>(MinOffset)));
+  return true;
+}
+
+/// Replace a G_MEMSET with a value of 0 with a G_BZERO instruction if it is
+/// supported and beneficial to do so.
+///
+/// \note This only applies on Darwin.
+///
+/// \returns true if \p MI was replaced with a G_BZERO.
+static bool tryEmitBZero(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
+                         bool MinSize) {
+  assert(MI.getOpcode() == TargetOpcode::G_MEMSET);
+  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
+  auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
+  if (!TLI.getLibcallName(RTLIB::BZERO))
+    return false;
+  auto Zero = getConstantVRegValWithLookThrough(MI.getOperand(1).getReg(), MRI);
+  if (!Zero || Zero->Value.getSExtValue() != 0)
+    return false;
+
+  // It's not faster to use bzero rather than memset for sizes <= 256.
+  // However, it *does* save us a mov from wzr, so if we're going for
+  // minsize, use bzero even if it's slower.
+  if (!MinSize) {
+    // If the size is known, check it. If it is not known, assume using bzero is
+    // better.
+    if (auto Size =
+            getConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI)) {
+      if (Size->Value.getSExtValue() <= 256)
+        return false;
+    }
+  }
+
+  MIRBuilder.setInstrAndDebugLoc(MI);
+  MIRBuilder
+      .buildInstr(TargetOpcode::G_BZERO, {},
+                  {MI.getOperand(0), MI.getOperand(2)})
+      .addImm(MI.getOperand(3).getImm())
+      .addMemOperand(*MI.memoperands_begin());
+  MI.eraseFromParent();
+  return true;
+}
+
 class AArch64PreLegalizerCombinerHelperState {
 protected:
   CombinerHelper &Helper;
@@ -153,7 +303,8 @@ bool AArch64PreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
   if (Generated.tryCombineAll(Observer, MI, B))
     return true;
 
-  switch (MI.getOpcode()) {
+  unsigned Opc = MI.getOpcode();
+  switch (Opc) {
   case TargetOpcode::G_CONCAT_VECTORS:
     return Helper.tryCombineConcatVectors(MI);
   case TargetOpcode::G_SHUFFLE_VECTOR:
@@ -165,7 +316,11 @@ bool AArch64PreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
     // heuristics decide.
     unsigned MaxLen = EnableOpt ? 0 : 32;
     // Try to inline memcpy type calls if optimizations are enabled.
-    return !EnableMinSize ? Helper.tryCombineMemCpyFamily(MI, MaxLen) : false;
+    if (!EnableMinSize && Helper.tryCombineMemCpyFamily(MI, MaxLen))
+      return true;
+    if (Opc == TargetOpcode::G_MEMSET)
+      return tryEmitBZero(MI, B, EnableMinSize);
+    return false;
   }
   }
 
diff --git a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
index 3d9080f7997d5798e2c5c357a5b4c22ce06b9c1f..6b8cb786bb6c5e6ad9a5e4c6c2770217a33f60d2 100644
--- a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
@@ -77,6 +77,7 @@ private:
 
   static bool optimizeConvertFromSVBool(IntrinsicInst *I);
   static bool optimizePTest(IntrinsicInst *I);
+  static bool optimizeVectorMul(IntrinsicInst *I);
 
   static bool processPhiNode(IntrinsicInst *I);
 };
@@ -366,6 +367,76 @@ bool SVEIntrinsicOpts::optimizePTest(IntrinsicInst *I) {
   return false;
 }
 
+bool SVEIntrinsicOpts::optimizeVectorMul(IntrinsicInst *I) {
+  assert((I->getIntrinsicID() == Intrinsic::aarch64_sve_mul ||
+          I->getIntrinsicID() == Intrinsic::aarch64_sve_fmul) &&
+         "Unexpected opcode");
+
+  auto *OpPredicate = I->getOperand(0);
+  auto *OpMultiplicand = I->getOperand(1);
+  auto *OpMultiplier = I->getOperand(2);
+
+  // Return true if a given instruction is an aarch64_sve_dup_x intrinsic call
+  // with a unit splat value, false otherwise.
+  auto IsUnitDupX = [](auto *I) {
+    auto *IntrI = dyn_cast<IntrinsicInst>(I);
+    if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup_x)
+      return false;
+
+    auto *SplatValue = IntrI->getOperand(0);
+    return match(SplatValue, m_FPOne()) || match(SplatValue, m_One());
+  };
+
+  // Return true if a given instruction is an aarch64_sve_dup intrinsic call
+  // with a unit splat value, false otherwise.
+  auto IsUnitDup = [](auto *I) {
+    auto *IntrI = dyn_cast<IntrinsicInst>(I);
+    if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup)
+      return false;
+
+    auto *SplatValue = IntrI->getOperand(2);
+    return match(SplatValue, m_FPOne()) || match(SplatValue, m_One());
+  };
+
+  bool Changed = true;
+
+  // The OpMultiplier variable should always point to the dup (if any), so
+  // swap if necessary.
+  if (IsUnitDup(OpMultiplicand) || IsUnitDupX(OpMultiplicand))
+    std::swap(OpMultiplier, OpMultiplicand);
+
+  if (IsUnitDupX(OpMultiplier)) {
+    // [f]mul pg (dupx 1) %n => %n
+    I->replaceAllUsesWith(OpMultiplicand);
+    I->eraseFromParent();
+    Changed = true;
+  } else if (IsUnitDup(OpMultiplier)) {
+    // [f]mul pg (dup pg 1) %n => %n
+    auto *DupInst = cast<IntrinsicInst>(OpMultiplier);
+    auto *DupPg = DupInst->getOperand(1);
+    // TODO: this is naive. The optimization is still valid if DupPg
+    // 'encompasses' OpPredicate, not only if they're the same predicate.
+    if (OpPredicate == DupPg) {
+      I->replaceAllUsesWith(OpMultiplicand);
+      I->eraseFromParent();
+      Changed = true;
+    }
+  }
+
+  // If an instruction was optimized out then it is possible that some dangling
+  // instructions are left.
+  if (Changed) {
+    auto *OpPredicateInst = dyn_cast<Instruction>(OpPredicate);
+    auto *OpMultiplierInst = dyn_cast<Instruction>(OpMultiplier);
+    if (OpMultiplierInst && OpMultiplierInst->use_empty())
+      OpMultiplierInst->eraseFromParent();
+    if (OpPredicateInst && OpPredicateInst->use_empty())
+      OpPredicateInst->eraseFromParent();
+  }
+
+  return Changed;
+}
+
 bool SVEIntrinsicOpts::optimizeConvertFromSVBool(IntrinsicInst *I) {
   assert(I->getIntrinsicID() == Intrinsic::aarch64_sve_convert_from_svbool &&
          "Unexpected opcode");
@@ -429,6 +500,9 @@ bool SVEIntrinsicOpts::optimizeIntrinsic(Instruction *I) {
   switch (IntrI->getIntrinsicID()) {
   case Intrinsic::aarch64_sve_convert_from_svbool:
     return optimizeConvertFromSVBool(IntrI);
+  case Intrinsic::aarch64_sve_fmul:
+  case Intrinsic::aarch64_sve_mul:
+    return optimizeVectorMul(IntrI);
   case Intrinsic::aarch64_sve_ptest_any:
   case Intrinsic::aarch64_sve_ptest_first:
   case Intrinsic::aarch64_sve_ptest_last:
@@ -484,6 +558,8 @@ bool SVEIntrinsicOpts::runOnModule(Module &M) {
     case Intrinsic::aarch64_sve_ptest_first:
     case Intrinsic::aarch64_sve_ptest_last:
     case Intrinsic::aarch64_sve_ptrue:
+    case Intrinsic::aarch64_sve_mul:
+    case Intrinsic::aarch64_sve_fmul:
       for (User *U : F.users())
         Functions.insert(cast<Instruction>(U)->getFunction());
       break;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index cdd59fe0b847b67311c5d640001497103b621ca2..4b0367501ae09a555691707c3d4368b4829db3d1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -204,14 +204,11 @@ extern char &SIWholeQuadModeID;
 void initializeSILowerControlFlowPass(PassRegistry &);
 extern char &SILowerControlFlowID;
 
-void initializeSIRemoveShortExecBranchesPass(PassRegistry &);
-extern char &SIRemoveShortExecBranchesID;
-
 void initializeSIPreEmitPeepholePass(PassRegistry &);
 extern char &SIPreEmitPeepholeID;
 
-void initializeSIInsertSkipsPass(PassRegistry &);
-extern char &SIInsertSkipsPassID;
+void initializeSILateBranchLoweringPass(PassRegistry &);
+extern char &SILateBranchLoweringPassID;
 
 void initializeSIOptimizeExecMaskingPass(PassRegistry &);
 extern char &SIOptimizeExecMaskingID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 121373eeffd7adb3401528d9a013ac820a557829..f061dd5fcbd5480f76f7c1eabbc90560032f4ae7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -480,7 +480,7 @@ def FeatureDot1Insts : SubtargetFeature<"dot1-insts",
 def FeatureDot2Insts : SubtargetFeature<"dot2-insts",
   "HasDot2Insts",
   "true",
-  "Has v_dot2_f32_f16, v_dot2_i32_i16, v_dot2_u32_u16, v_dot4_u32_u8, v_dot8_u32_u4 instructions"
+  "Has v_dot2_i32_i16, v_dot2_u32_u16 instructions"
 >;
 
 def FeatureDot3Insts : SubtargetFeature<"dot3-insts",
@@ -507,6 +507,12 @@ def FeatureDot6Insts : SubtargetFeature<"dot6-insts",
   "Has v_dot4c_i32_i8 instruction"
 >;
 
+def FeatureDot7Insts : SubtargetFeature<"dot7-insts",
+  "HasDot7Insts",
+  "true",
+  "Has v_dot2_f32_f16, v_dot4_u32_u8, v_dot8_u32_u4 instructions"
+>;
+
 def FeatureMAIInsts : SubtargetFeature<"mai-insts",
   "HasMAIInsts",
   "true",
@@ -902,6 +908,7 @@ def FeatureISAVersion9_0_6 : FeatureSet<
    FeatureDLInsts,
    FeatureDot1Insts,
    FeatureDot2Insts,
+   FeatureDot7Insts,
    FeatureSupportsSRAMECC,
    FeatureImageGather4D16Bug]>;
 
@@ -920,6 +927,7 @@ def FeatureISAVersion9_0_8 : FeatureSet<
    FeatureDot4Insts,
    FeatureDot5Insts,
    FeatureDot6Insts,
+   FeatureDot7Insts,
    FeatureMAIInsts,
    FeaturePkFmacF16Inst,
    FeatureAtomicFaddInsts,
@@ -948,6 +956,7 @@ def FeatureISAVersion9_0_A : FeatureSet<
    FeatureDot4Insts,
    FeatureDot5Insts,
    FeatureDot6Insts,
+   FeatureDot7Insts,
    Feature64BitDPP,
    FeaturePackedFP32Ops,
    FeatureMAIInsts,
@@ -1008,6 +1017,7 @@ def FeatureISAVersion10_1_1 : FeatureSet<
      FeatureDot2Insts,
      FeatureDot5Insts,
      FeatureDot6Insts,
+     FeatureDot7Insts,
      FeatureNSAEncoding,
      FeatureWavefrontSize32,
      FeatureScalarStores,
@@ -1028,6 +1038,7 @@ def FeatureISAVersion10_1_2 : FeatureSet<
      FeatureDot2Insts,
      FeatureDot5Insts,
      FeatureDot6Insts,
+     FeatureDot7Insts,
      FeatureNSAEncoding,
      FeatureWavefrontSize32,
      FeatureScalarStores,
@@ -1049,6 +1060,7 @@ def FeatureISAVersion10_3_0 : FeatureSet<
    FeatureDot2Insts,
    FeatureDot5Insts,
    FeatureDot6Insts,
+   FeatureDot7Insts,
    FeatureNSAEncoding,
    FeatureWavefrontSize32,
    FeatureShaderCyclesRegister]>;
@@ -1373,6 +1385,9 @@ def HasDot5Insts : Predicate<"Subtarget->hasDot5Insts()">,
 def HasDot6Insts : Predicate<"Subtarget->hasDot6Insts()">,
   AssemblerPredicate<(all_of FeatureDot6Insts)>;
 
+def HasDot7Insts : Predicate<"Subtarget->hasDot7Insts()">,
+  AssemblerPredicate<(all_of FeatureDot7Insts)>;
+
 def HasGetWaveIdInst : Predicate<"Subtarget->hasGetWaveIdInst()">,
   AssemblerPredicate<(all_of FeatureGetWaveIdInst)>;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
index 0bd6297c529698fda822e69ca976da74df02d824..3d0c5fa25db13390a2230a82c2924f609ee52564 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
@@ -112,7 +112,7 @@ bool AMDGPUAnnotateUniformValues::isClobberedInFunction(LoadInst * Load) {
 
 void AMDGPUAnnotateUniformValues::visitBranchInst(BranchInst &I) {
   if (DA->isUniform(&I))
-    setUniformMetadata(I.getParent()->getTerminator());
+    setUniformMetadata(&I);
 }
 
 void AMDGPUAnnotateUniformValues::visitLoadInst(LoadInst &I) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index ed5ed98033a50c3ae4b41ee40f5a5491aa03dabe..c30c1105aca225fb25226c185190c0bf00fc19d0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -97,12 +97,14 @@ extern "C" void LLVM_EXTERNAL_VISIBILITY LLVMInitializeAMDGPUAsmPrinter() {
 
 AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
                                    std::unique_ptr<MCStreamer> Streamer)
-  : AsmPrinter(TM, std::move(Streamer)) {
+    : AsmPrinter(TM, std::move(Streamer)) {
   if (TM.getTargetTriple().getOS() == Triple::AMDHSA) {
     if (isHsaAbiVersion2(getGlobalSTI())) {
       HSAMetadataStream.reset(new HSAMD::MetadataStreamerV2());
-    } else {
+    } else if (isHsaAbiVersion3(getGlobalSTI())) {
       HSAMetadataStream.reset(new HSAMD::MetadataStreamerV3());
+    } else {
+      HSAMetadataStream.reset(new HSAMD::MetadataStreamerV4());
     }
   }
 }
@@ -122,34 +124,34 @@ AMDGPUTargetStreamer* AMDGPUAsmPrinter::getTargetStreamer() const {
 }
 
 void AMDGPUAsmPrinter::emitStartOfAsmFile(Module &M) {
-  if (isHsaAbiVersion3(getGlobalSTI())) {
-    std::string ExpectedTarget;
-    raw_string_ostream ExpectedTargetOS(ExpectedTarget);
-    IsaInfo::streamIsaVersion(getGlobalSTI(), ExpectedTargetOS);
-
-    getTargetStreamer()->EmitDirectiveAMDGCNTarget(ExpectedTarget);
-  }
+  // TODO: Which one is called first, emitStartOfAsmFile or
+  // emitFunctionBodyStart?
+  if (getTargetStreamer() && !getTargetStreamer()->getTargetID())
+    initializeTargetID(M);
 
   if (TM.getTargetTriple().getOS() != Triple::AMDHSA &&
       TM.getTargetTriple().getOS() != Triple::AMDPAL)
     return;
 
+  if (isHsaAbiVersion3Or4(getGlobalSTI()))
+    getTargetStreamer()->EmitDirectiveAMDGCNTarget();
+
   if (TM.getTargetTriple().getOS() == Triple::AMDHSA)
-    HSAMetadataStream->begin(M);
+    HSAMetadataStream->begin(M, *getTargetStreamer()->getTargetID());
 
   if (TM.getTargetTriple().getOS() == Triple::AMDPAL)
     getTargetStreamer()->getPALMetadata()->readFromIR(M);
 
-  if (isHsaAbiVersion3(getGlobalSTI()))
+  if (isHsaAbiVersion3Or4(getGlobalSTI()))
     return;
 
-  // HSA emits NT_AMDGPU_HSA_CODE_OBJECT_VERSION for code objects v2.
+  // HSA emits NT_AMD_HSA_CODE_OBJECT_VERSION for code objects v2.
   if (TM.getTargetTriple().getOS() == Triple::AMDHSA)
     getTargetStreamer()->EmitDirectiveHSACodeObjectVersion(2, 1);
 
-  // HSA and PAL emit NT_AMDGPU_HSA_ISA for code objects v2.
+  // HSA and PAL emit NT_AMD_HSA_ISA_VERSION for code objects v2.
   IsaVersion Version = getIsaVersion(getGlobalSTI()->getCPU());
-  getTargetStreamer()->EmitDirectiveHSACodeObjectISA(
+  getTargetStreamer()->EmitDirectiveHSACodeObjectISAV2(
       Version.Major, Version.Minor, Version.Stepping, "AMD", "AMDGPU");
 }
 
@@ -159,15 +161,11 @@ void AMDGPUAsmPrinter::emitEndOfAsmFile(Module &M) {
     return;
 
   if (TM.getTargetTriple().getOS() != Triple::AMDHSA ||
-      isHsaAbiVersion2(getGlobalSTI())) {
-    // Emit ISA Version (NT_AMD_AMDGPU_ISA).
-    std::string ISAVersionString;
-    raw_string_ostream ISAVersionStream(ISAVersionString);
-    IsaInfo::streamIsaVersion(getGlobalSTI(), ISAVersionStream);
-    getTargetStreamer()->EmitISAVersion(ISAVersionStream.str());
-  }
+      isHsaAbiVersion2(getGlobalSTI()))
+    getTargetStreamer()->EmitISAVersion();
 
   // Emit HSA Metadata (NT_AMD_AMDGPU_HSA_METADATA).
+  // Emit HSA Metadata (NT_AMD_HSA_METADATA).
   if (TM.getTargetTriple().getOS() == Triple::AMDHSA) {
     HSAMetadataStream->end();
     bool Success = HSAMetadataStream->emitTo(*getTargetStreamer());
@@ -192,11 +190,37 @@ bool AMDGPUAsmPrinter::isBlockOnlyReachableByFallthrough(
 
 void AMDGPUAsmPrinter::emitFunctionBodyStart() {
   const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
+  const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
+  const Function &F = MF->getFunction();
+
+  // TODO: Which one is called first, emitStartOfAsmFile or
+  // emitFunctionBodyStart?
+  if (getTargetStreamer() && !getTargetStreamer()->getTargetID())
+    initializeTargetID(*F.getParent());
+
+  const auto &FunctionTargetID = STM.getTargetID();
+  // Make sure function's xnack settings are compatible with module's
+  // xnack settings.
+  if (FunctionTargetID.isXnackSupported() &&
+      FunctionTargetID.getXnackSetting() != IsaInfo::TargetIDSetting::Any &&
+      FunctionTargetID.getXnackSetting() != getTargetStreamer()->getTargetID()->getXnackSetting()) {
+    OutContext.reportError({}, "xnack setting of '" + Twine(MF->getName()) +
+                           "' function does not match module xnack setting");
+    return;
+  }
+  // Make sure function's sramecc settings are compatible with module's
+  // sramecc settings.
+  if (FunctionTargetID.isSramEccSupported() &&
+      FunctionTargetID.getSramEccSetting() != IsaInfo::TargetIDSetting::Any &&
+      FunctionTargetID.getSramEccSetting() != getTargetStreamer()->getTargetID()->getSramEccSetting()) {
+    OutContext.reportError({}, "sramecc setting of '" + Twine(MF->getName()) +
+                           "' function does not match module sramecc setting");
+    return;
+  }
+
   if (!MFI.isEntryFunction())
     return;
 
-  const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
-  const Function &F = MF->getFunction();
   if ((STM.isMesaKernel(F) || isHsaAbiVersion2(getGlobalSTI())) &&
       (F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
        F.getCallingConv() == CallingConv::SPIR_KERNEL)) {
@@ -232,26 +256,25 @@ void AMDGPUAsmPrinter::emitFunctionBodyEnd() {
   if (ReadOnlySection.getAlignment() < 64)
     ReadOnlySection.setAlignment(Align(64));
 
-  const MCSubtargetInfo &STI = MF->getSubtarget();
+  const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
 
   SmallString<128> KernelName;
   getNameWithPrefix(KernelName, &MF->getFunction());
   getTargetStreamer()->EmitAmdhsaKernelDescriptor(
-      STI, KernelName, getAmdhsaKernelDescriptor(*MF, CurrentProgramInfo),
+      STM, KernelName, getAmdhsaKernelDescriptor(*MF, CurrentProgramInfo),
       CurrentProgramInfo.NumVGPRsForWavesPerEU,
       CurrentProgramInfo.NumSGPRsForWavesPerEU -
-          IsaInfo::getNumExtraSGPRs(&STI,
+          IsaInfo::getNumExtraSGPRs(&STM,
                                     CurrentProgramInfo.VCCUsed,
                                     CurrentProgramInfo.FlatUsed),
-      CurrentProgramInfo.VCCUsed, CurrentProgramInfo.FlatUsed,
-      hasXNACK(STI));
+      CurrentProgramInfo.VCCUsed, CurrentProgramInfo.FlatUsed);
 
   Streamer.PopSection();
 }
 
 void AMDGPUAsmPrinter::emitFunctionEntryLabel() {
   if (TM.getTargetTriple().getOS() == Triple::AMDHSA &&
-      isHsaAbiVersion3(getGlobalSTI())) {
+      isHsaAbiVersion3Or4(getGlobalSTI())) {
     AsmPrinter::emitFunctionEntryLabel();
     return;
   }
@@ -401,6 +424,8 @@ amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(
     const MachineFunction &MF,
     const SIProgramInfo &PI) const {
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
+  const Function &F = MF.getFunction();
+
   amdhsa::kernel_descriptor_t KernelDescriptor;
   memset(&KernelDescriptor, 0x0, sizeof(KernelDescriptor));
 
@@ -410,6 +435,10 @@ amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(
 
   KernelDescriptor.group_segment_fixed_size = PI.LDSSize;
   KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
+
+  Align MaxKernArgAlign;
+  KernelDescriptor.kernarg_size = STM.getKernArgSegmentSize(F, MaxKernArgAlign);
+
   KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1();
   KernelDescriptor.compute_pgm_rsrc2 = PI.ComputePGMRSrc2;
   KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
@@ -598,6 +627,36 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   return false;
 }
 
+// TODO: Fold this into emitFunctionBodyStart.
+void AMDGPUAsmPrinter::initializeTargetID(const Module &M) {
+  // In the beginning all features are either 'Any' or 'NotSupported',
+  // depending on global target features. This will cover empty modules.
+  getTargetStreamer()->initializeTargetID(
+      *getGlobalSTI(), getGlobalSTI()->getFeatureString());
+
+  // If module is empty, we are done.
+  if (M.empty())
+    return;
+
+  // If module is not empty, need to find first 'Off' or 'On' feature
+  // setting per feature from functions in module.
+  for (auto &F : M) {
+    auto &TSTargetID = getTargetStreamer()->getTargetID();
+    if ((!TSTargetID->isXnackSupported() || TSTargetID->isXnackOnOrOff()) &&
+        (!TSTargetID->isSramEccSupported() || TSTargetID->isSramEccOnOrOff()))
+      break;
+
+    const GCNSubtarget &STM = TM.getSubtarget<GCNSubtarget>(F);
+    const IsaInfo::AMDGPUTargetID &STMTargetID = STM.getTargetID();
+    if (TSTargetID->isXnackSupported())
+      if (TSTargetID->getXnackSetting() == IsaInfo::TargetIDSetting::Any)
+        TSTargetID->setXnackSetting(STMTargetID.getXnackSetting());
+    if (TSTargetID->isSramEccSupported())
+      if (TSTargetID->getSramEccSetting() == IsaInfo::TargetIDSetting::Any)
+        TSTargetID->setSramEccSetting(STMTargetID.getSramEccSetting());
+  }
+}
+
 uint64_t AMDGPUAsmPrinter::getFunctionCodeSize(const MachineFunction &MF) const {
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
   const SIInstrInfo *TII = STM.getInstrInfo();
@@ -632,8 +691,8 @@ static bool hasAnyNonFlatUseOfReg(const MachineRegisterInfo &MRI,
 
 int32_t AMDGPUAsmPrinter::SIFunctionResourceInfo::getTotalNumSGPRs(
   const GCNSubtarget &ST) const {
-  return NumExplicitSGPR + IsaInfo::getNumExtraSGPRs(&ST,
-                                                     UsesVCC, UsesFlatScratch);
+  return NumExplicitSGPR + IsaInfo::getNumExtraSGPRs(
+      &ST, UsesVCC, UsesFlatScratch, ST.getTargetID().isXnackOnOrAny());
 }
 
 int32_t AMDGPUAsmPrinter::SIFunctionResourceInfo::getTotalNumVGPRs(
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
index 9e1e26d65d8c15737716d86947e7e852ad5e7f10..fc29f9f2051cfe8dcbc168150e5372e8c085c8a6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
@@ -56,6 +56,8 @@ private:
     int32_t getTotalNumVGPRs(const GCNSubtarget &ST) const;
   };
 
+  void initializeTargetID(const Module &M);
+
   SIProgramInfo CurrentProgramInfo;
   DenseMap<const Function *, SIFunctionResourceInfo> CallGraphResourceInfo;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index 3f913cd9cba8055eff75f0d72ae94a65b0b6fb78..3e9fdcb1618e63cc0c7008aee37f89793f24bfe7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -48,6 +48,8 @@ private:
   const GCNSubtarget *ST;
   bool IsPixelShader;
 
+  Value *buildReduction(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
+                        Value *const Identity) const;
   Value *buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
                    Value *const Identity) const;
   Value *buildShiftRight(IRBuilder<> &B, Value *V, Value *const Identity) const;
@@ -279,6 +281,45 @@ static Value *buildNonAtomicBinOp(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
   return B.CreateSelect(Cond, LHS, RHS);
 }
 
+// Use the builder to create a reduction of V across the wavefront, with all
+// lanes active, returning the same result in all lanes.
+Value *AMDGPUAtomicOptimizer::buildReduction(IRBuilder<> &B,
+                                             AtomicRMWInst::BinOp Op, Value *V,
+                                             Value *const Identity) const {
+  Type *const Ty = V->getType();
+  Module *M = B.GetInsertBlock()->getModule();
+  Function *UpdateDPP =
+      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, Ty);
+
+  // Reduce within each row of 16 lanes.
+  for (unsigned Idx = 0; Idx < 4; Idx++) {
+    V = buildNonAtomicBinOp(
+        B, Op, V,
+        B.CreateCall(UpdateDPP,
+                     {Identity, V, B.getInt32(DPP::ROW_XMASK0 | 1 << Idx),
+                      B.getInt32(0xf), B.getInt32(0xf), B.getFalse()}));
+  }
+
+  // Reduce within each pair of rows (i.e. 32 lanes).
+  assert(ST->hasPermLaneX16());
+  V = buildNonAtomicBinOp(
+      B, Op, V,
+      B.CreateIntrinsic(
+          Intrinsic::amdgcn_permlanex16, {},
+          {V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()}));
+
+  if (ST->isWave32())
+    return V;
+
+  // Pick an arbitrary lane from 0..31 and an arbitrary lane from 32..63 and
+  // combine them with a scalar operation.
+  Function *ReadLane =
+      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, {});
+  Value *const Lane0 = B.CreateCall(ReadLane, {V, B.getInt32(0)});
+  Value *const Lane32 = B.CreateCall(ReadLane, {V, B.getInt32(32)});
+  return buildNonAtomicBinOp(B, Op, Lane0, Lane32);
+}
+
 // Use the builder to create an inclusive scan of V across the wavefront, with
 // all lanes active.
 Value *AMDGPUAtomicOptimizer::buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
@@ -287,10 +328,6 @@ Value *AMDGPUAtomicOptimizer::buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
   Module *M = B.GetInsertBlock()->getModule();
   Function *UpdateDPP =
       Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, Ty);
-  Function *PermLaneX16 =
-      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_permlanex16, {});
-  Function *ReadLane =
-      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, {});
 
   for (unsigned Idx = 0; Idx < 4; Idx++) {
     V = buildNonAtomicBinOp(
@@ -317,9 +354,10 @@ Value *AMDGPUAtomicOptimizer::buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
 
     // Combine lane 15 into lanes 16..31 (and, for wave 64, lane 47 into lanes
     // 48..63).
-    Value *const PermX =
-        B.CreateCall(PermLaneX16, {V, V, B.getInt32(-1), B.getInt32(-1),
-                                   B.getFalse(), B.getFalse()});
+    assert(ST->hasPermLaneX16());
+    Value *const PermX = B.CreateIntrinsic(
+        Intrinsic::amdgcn_permlanex16, {},
+        {V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()});
     V = buildNonAtomicBinOp(
         B, Op, V,
         B.CreateCall(UpdateDPP,
@@ -327,7 +365,8 @@ Value *AMDGPUAtomicOptimizer::buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
                       B.getInt32(0xa), B.getInt32(0xf), B.getFalse()}));
     if (!ST->isWave32()) {
       // Combine lane 31 into lanes 32..63.
-      Value *const Lane31 = B.CreateCall(ReadLane, {V, B.getInt32(31)});
+      Value *const Lane31 = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {},
+                                              {V, B.getInt32(31)});
       V = buildNonAtomicBinOp(
           B, Op, V,
           B.CreateCall(UpdateDPP,
@@ -346,10 +385,6 @@ Value *AMDGPUAtomicOptimizer::buildShiftRight(IRBuilder<> &B, Value *V,
   Module *M = B.GetInsertBlock()->getModule();
   Function *UpdateDPP =
       Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, Ty);
-  Function *ReadLane =
-      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, {});
-  Function *WriteLane =
-      Intrinsic::getDeclaration(M, Intrinsic::amdgcn_writelane, {});
 
   if (ST->hasDPPWavefrontShifts()) {
     // GFX9 has DPP wavefront shift operations.
@@ -357,6 +392,11 @@ Value *AMDGPUAtomicOptimizer::buildShiftRight(IRBuilder<> &B, Value *V,
                      {Identity, V, B.getInt32(DPP::WAVE_SHR1), B.getInt32(0xf),
                       B.getInt32(0xf), B.getFalse()});
   } else {
+    Function *ReadLane =
+        Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, {});
+    Function *WriteLane =
+        Intrinsic::getDeclaration(M, Intrinsic::amdgcn_writelane, {});
+
     // On GFX10 all DPP operations are confined to a single row. To get cross-
     // row operations we have to use permlane or readlane.
     Value *Old = V;
@@ -480,6 +520,8 @@ void AMDGPUAtomicOptimizer::optimizeAtomic(Instruction &I,
   Value *ExclScan = nullptr;
   Value *NewV = nullptr;
 
+  const bool NeedResult = !I.use_empty();
+
   // If we have a divergent value in each lane, we need to combine the value
   // using DPP.
   if (ValDivergent) {
@@ -489,31 +531,23 @@ void AMDGPUAtomicOptimizer::optimizeAtomic(Instruction &I,
 
     const AtomicRMWInst::BinOp ScanOp =
         Op == AtomicRMWInst::Sub ? AtomicRMWInst::Add : Op;
-    NewV = buildScan(B, ScanOp, NewV, Identity);
-    ExclScan = buildShiftRight(B, NewV, Identity);
-
-    // Read the value from the last lane, which has accumlated the values of
-    // each active lane in the wavefront. This will be our new value which we
-    // will provide to the atomic operation.
-    Value *const LastLaneIdx = B.getInt32(ST->getWavefrontSize() - 1);
-    if (TyBitWidth == 64) {
-      Value *const ExtractLo = B.CreateTrunc(NewV, B.getInt32Ty());
-      Value *const ExtractHi =
-          B.CreateTrunc(B.CreateLShr(NewV, 32), B.getInt32Ty());
-      CallInst *const ReadLaneLo = B.CreateIntrinsic(
-          Intrinsic::amdgcn_readlane, {}, {ExtractLo, LastLaneIdx});
-      CallInst *const ReadLaneHi = B.CreateIntrinsic(
-          Intrinsic::amdgcn_readlane, {}, {ExtractHi, LastLaneIdx});
-      Value *const PartialInsert = B.CreateInsertElement(
-          UndefValue::get(VecTy), ReadLaneLo, B.getInt32(0));
-      Value *const Insert =
-          B.CreateInsertElement(PartialInsert, ReadLaneHi, B.getInt32(1));
-      NewV = B.CreateBitCast(Insert, Ty);
-    } else if (TyBitWidth == 32) {
+    if (!NeedResult && ST->hasPermLaneX16()) {
+      // On GFX10 the permlanex16 instruction helps us build a reduction without
+      // too many readlanes and writelanes, which are generally bad for
+      // performance.
+      NewV = buildReduction(B, ScanOp, NewV, Identity);
+    } else {
+      NewV = buildScan(B, ScanOp, NewV, Identity);
+      if (NeedResult)
+        ExclScan = buildShiftRight(B, NewV, Identity);
+
+      // Read the value from the last lane, which has accumlated the values of
+      // each active lane in the wavefront. This will be our new value which we
+      // will provide to the atomic operation.
+      Value *const LastLaneIdx = B.getInt32(ST->getWavefrontSize() - 1);
+      assert(TyBitWidth == 32);
       NewV = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {},
                                {NewV, LastLaneIdx});
-    } else {
-      llvm_unreachable("Unhandled atomic bit width");
     }
 
     // Finally mark the readlanes in the WWM section.
@@ -583,7 +617,6 @@ void AMDGPUAtomicOptimizer::optimizeAtomic(Instruction &I,
   // original instruction.
   B.SetInsertPoint(&I);
 
-  const bool NeedResult = !I.use_empty();
   if (NeedResult) {
     // Create a PHI node to get our new atomic result into the exit block.
     PHINode *const PHI = B.CreatePHI(Ty, 2);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index c7c4ed45589fd173a2128ae3954160db2bebb4ad..a942a740535a29370e1a1086676111f96e86d66e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -656,7 +656,7 @@ bool AMDGPUCallLowering::lowerFormalArguments(
       }
     }
 
-    ArgInfo OrigArg(VRegs[Idx], Arg.getType());
+    ArgInfo OrigArg(VRegs[Idx], Arg);
     const unsigned OrigArgIdx = Idx + AttributeList::FirstArgIndex;
     setArgFlags(OrigArg, OrigArgIdx, DL, F);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 1a5f7aafbb439d325470ab1118dcf953ab3f0cdf..29f9c20dc8fdb77b16eeda728ba02125a8fea2d4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -113,14 +113,6 @@ def gi_mubuf_offset :
     GIComplexOperandMatcher<s64, "selectMUBUFOffset">,
     GIComplexPatternEquiv<MUBUFOffset>;
 
-def gi_mubuf_addr64_atomic :
-    GIComplexOperandMatcher<s64, "selectMUBUFAddr64Atomic">,
-    GIComplexPatternEquiv<MUBUFAddr64Atomic>;
-
-def gi_mubuf_offset_atomic :
-    GIComplexOperandMatcher<s64, "selectMUBUFOffsetAtomic">,
-    GIComplexPatternEquiv<MUBUFOffsetAtomic>;
-
 def gi_smrd_buffer_imm :
     GIComplexOperandMatcher<s64, "selectSMRDBufferImm">,
     GIComplexPatternEquiv<SMRDBufferImm>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
index 39f9092ce77c4b6ce20ee89c88930a2d43640dc9..8eeda7b67b732b7e7cd03f4ce7769149592ca990 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
@@ -226,8 +226,8 @@ MetadataStreamerV2::getHSADebugProps(const MachineFunction &MF,
 void MetadataStreamerV2::emitVersion() {
   auto &Version = HSAMetadata.mVersion;
 
-  Version.push_back(VersionMajor);
-  Version.push_back(VersionMinor);
+  Version.push_back(VersionMajorV2);
+  Version.push_back(VersionMinorV2);
 }
 
 void MetadataStreamerV2::emitPrintf(const Module &Mod) {
@@ -435,7 +435,8 @@ bool MetadataStreamerV2::emitTo(AMDGPUTargetStreamer &TargetStreamer) {
   return TargetStreamer.EmitHSAMetadata(getHSAMetadata());
 }
 
-void MetadataStreamerV2::begin(const Module &Mod) {
+void MetadataStreamerV2::begin(const Module &Mod,
+                               const IsaInfo::AMDGPUTargetID &TargetID) {
   emitVersion();
   emitPrintf(Mod);
 }
@@ -608,8 +609,8 @@ MetadataStreamerV3::getWorkGroupDimensions(MDNode *Node) const {
 
 void MetadataStreamerV3::emitVersion() {
   auto Version = HSAMetadataDoc->getArrayNode();
-  Version.push_back(Version.getDocument()->getNode(VersionMajor));
-  Version.push_back(Version.getDocument()->getNode(VersionMinor));
+  Version.push_back(Version.getDocument()->getNode(VersionMajorV3));
+  Version.push_back(Version.getDocument()->getNode(VersionMinorV3));
   getRootMetadata("amdhsa.version") = Version;
 }
 
@@ -881,7 +882,8 @@ bool MetadataStreamerV3::emitTo(AMDGPUTargetStreamer &TargetStreamer) {
   return TargetStreamer.EmitHSAMetadata(*HSAMetadataDoc, true);
 }
 
-void MetadataStreamerV3::begin(const Module &Mod) {
+void MetadataStreamerV3::begin(const Module &Mod,
+                               const IsaInfo::AMDGPUTargetID &TargetID) {
   emitVersion();
   emitPrintf(Mod);
   getRootMetadata("amdhsa.kernels") = HSAMetadataDoc->getArrayNode();
@@ -921,6 +923,30 @@ void MetadataStreamerV3::emitKernel(const MachineFunction &MF,
   Kernels.push_back(Kern);
 }
 
+//===----------------------------------------------------------------------===//
+// HSAMetadataStreamerV4
+//===----------------------------------------------------------------------===//
+
+void MetadataStreamerV4::emitVersion() {
+  auto Version = HSAMetadataDoc->getArrayNode();
+  Version.push_back(Version.getDocument()->getNode(VersionMajorV4));
+  Version.push_back(Version.getDocument()->getNode(VersionMinorV4));
+  getRootMetadata("amdhsa.version") = Version;
+}
+
+void MetadataStreamerV4::emitTargetID(const IsaInfo::AMDGPUTargetID &TargetID) {
+  getRootMetadata("amdhsa.target") =
+      HSAMetadataDoc->getNode(TargetID.toString(), /*Copy=*/true);
+}
+
+void MetadataStreamerV4::begin(const Module &Mod,
+                               const IsaInfo::AMDGPUTargetID &TargetID) {
+  emitVersion();
+  emitTargetID(TargetID);
+  emitPrintf(Mod);
+  getRootMetadata("amdhsa.kernels") = HSAMetadataDoc->getArrayNode();
+}
+
 } // end namespace HSAMD
 } // end namespace AMDGPU
 } // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
index 1c6db14b85cdbcc21ee2159b464d45df73a24119..4824b4cf37c70a2c6e3f785c8a02027ed64d9076 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUHSAMETADATASTREAMER_H
 #define LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUHSAMETADATASTREAMER_H
 
+#include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/BinaryFormat/MsgPackDocument.h"
 #include "llvm/Support/AMDGPUMetadata.h"
 #include "llvm/Support/Alignment.h"
@@ -40,7 +41,8 @@ public:
 
   virtual bool emitTo(AMDGPUTargetStreamer &TargetStreamer) = 0;
 
-  virtual void begin(const Module &Mod) = 0;
+  virtual void begin(const Module &Mod,
+                     const IsaInfo::AMDGPUTargetID &TargetID) = 0;
 
   virtual void end() = 0;
 
@@ -48,8 +50,9 @@ public:
                           const SIProgramInfo &ProgramInfo) = 0;
 };
 
-class MetadataStreamerV3 final : public MetadataStreamer {
-private:
+// TODO: Rename MetadataStreamerV3 -> MetadataStreamerMsgPackV3.
+class MetadataStreamerV3 : public MetadataStreamer {
+protected:
   std::unique_ptr<msgpack::Document> HSAMetadataDoc =
       std::make_unique<msgpack::Document>();
 
@@ -108,7 +111,8 @@ public:
 
   bool emitTo(AMDGPUTargetStreamer &TargetStreamer) override;
 
-  void begin(const Module &Mod) override;
+  void begin(const Module &Mod,
+             const IsaInfo::AMDGPUTargetID &TargetID) override;
 
   void end() override;
 
@@ -116,6 +120,21 @@ public:
                   const SIProgramInfo &ProgramInfo) override;
 };
 
+// TODO: Rename MetadataStreamerV4 -> MetadataStreamerMsgPackV4.
+class MetadataStreamerV4 final : public MetadataStreamerV3 {
+  void emitVersion();
+
+  void emitTargetID(const IsaInfo::AMDGPUTargetID &TargetID);
+
+public:
+  MetadataStreamerV4() = default;
+  ~MetadataStreamerV4() = default;
+
+  void begin(const Module &Mod,
+             const IsaInfo::AMDGPUTargetID &TargetID) override;
+};
+
+// TODO: Rename MetadataStreamerV2 -> MetadataStreamerYamlV2.
 class MetadataStreamerV2 final : public MetadataStreamer {
 private:
   Metadata HSAMetadata;
@@ -172,7 +191,8 @@ public:
 
   bool emitTo(AMDGPUTargetStreamer &TargetStreamer) override;
 
-  void begin(const Module &Mod) override;
+  void begin(const Module &Mod,
+             const IsaInfo::AMDGPUTargetID &TargetID) override;
 
   void end() override;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 28a21a1270ff8373a11eda2678bdf59d053d244f..fdb1cf898be31f1c829df66b59bf106dfd2b8b3e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -188,11 +188,7 @@ private:
                           SDValue &Offset1, unsigned Size) const;
   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
-                   SDValue &Idxen, SDValue &Addr64, SDValue &CPol, SDValue &TFE,
-                   SDValue &SWZ) const;
-  bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
-                         SDValue &SOffset, SDValue &Offset, SDValue &CPol,
-                         SDValue &TFE, SDValue &SWZ) const;
+                   SDValue &Idxen, SDValue &Addr64) const;
   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
                          SDValue &SOffset, SDValue &Offset) const;
   bool SelectMUBUFScratchOffen(SDNode *Parent,
@@ -202,9 +198,6 @@ private:
                                 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
                                 SDValue &Offset) const;
 
-  bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
-                         SDValue &Offset, SDValue &CPol, SDValue &TFE,
-                         SDValue &SWZ) const;
   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
                          SDValue &Offset) const;
 
@@ -1390,8 +1383,7 @@ bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr,
                                      SDValue &SOffset, SDValue &Offset,
                                      SDValue &Offen, SDValue &Idxen,
-                                     SDValue &Addr64, SDValue &CPol,
-                                     SDValue &TFE, SDValue &SWZ) const {
+                                     SDValue &Addr64) const {
   // Subtarget prefers to use flat instruction
   // FIXME: This should be a pattern predicate and not reach here
   if (Subtarget->useFlatForGlobal())
@@ -1399,11 +1391,6 @@ bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr,
 
   SDLoc DL(Addr);
 
-  if (!CPol)
-    CPol = CurDAG->getTargetConstant(0, DL, MVT::i32);
-  TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
-  SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1);
-
   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
@@ -1480,8 +1467,7 @@ bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr,
 
 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
                                            SDValue &VAddr, SDValue &SOffset,
-                                           SDValue &Offset, SDValue &CPol,
-                                           SDValue &TFE, SDValue &SWZ) const {
+                                           SDValue &Offset) const {
   SDValue Ptr, Offen, Idxen, Addr64;
 
   // addr64 bit was removed for volcanic islands.
@@ -1489,8 +1475,7 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
   if (!Subtarget->hasAddr64())
     return false;
 
-  if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
-                   CPol, TFE, SWZ))
+  if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
     return false;
 
   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
@@ -1507,14 +1492,6 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
   return false;
 }
 
-bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
-                                           SDValue &VAddr, SDValue &SOffset,
-                                           SDValue &Offset) const {
-  SDValue CPol, TFE, SWZ;
-
-  return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, CPol, TFE, SWZ);
-}
-
 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
   auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
   return PSV && PSV->isStack();
@@ -1633,15 +1610,13 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
 }
 
 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
-                                           SDValue &SOffset, SDValue &Offset,
-                                           SDValue &CPol, SDValue &TFE,
-                                           SDValue &SWZ) const {
+                                           SDValue &SOffset, SDValue &Offset
+                                           ) const {
   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
   const SIInstrInfo *TII =
     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
 
-  if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
-                   CPol, TFE, SWZ))
+  if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
     return false;
 
   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
@@ -1660,14 +1635,6 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
   return false;
 }
 
-bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
-                                           SDValue &Soffset, SDValue &Offset
-                                           ) const {
-  SDValue CPol, TFE, SWZ;
-
-  return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, CPol, TFE, SWZ);
-}
-
 // Find a load or store from corresponding pattern root.
 // Roots may be build_vector, bitconvert or their combinations.
 static MemSDNode* findMemSDNode(SDNode *N) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 8ef9c99e8b352c8ad7431a206218821b314e0d23..9b177088a2aa4dda00635df1e7414a680385fbc3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -596,12 +596,6 @@ class DwordAddrPat<ValueType vt, RegisterClass rc> : AMDGPUPat <
   (vt rc:$addr)
 >;
 
-// fshr pattern
-class FSHRPattern <Instruction BIT_ALIGN> : AMDGPUPat <
-  (fshr i32:$src0, i32:$src1, i32:$src2),
-  (BIT_ALIGN $src0, $src1, $src2)
->;
-
 // rotr pattern
 class ROTRPattern <Instruction BIT_ALIGN> : AMDGPUPat <
   (rotr i32:$src0, i32:$src1),
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 2925f70ece6249b8fc0f3988743831a465b4c1be..5c8da8bee265162b6f66f1fb061d970aebdc170c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -18,7 +18,9 @@
 #include "AMDGPUInstrInfo.h"
 #include "AMDGPUTargetMachine.h"
 #include "SIMachineFunctionInfo.h"
+#include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/ADT/ScopeExit.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
@@ -1595,17 +1597,37 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .clampScalar(0, S32, S64)
     .lower();
 
+  // TODO: Only Try to form v2s16 with legal packed instructions.
   getActionDefinitionsBuilder(G_FSHR)
     .legalFor({{S32, S32}})
+    .lowerFor({{V2S16, V2S16}})
+    .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16))
     .scalarize(0)
     .lower();
 
+  if (ST.hasVOP3PInsts()) {
+    getActionDefinitionsBuilder(G_FSHL)
+      .lowerFor({{V2S16, V2S16}})
+      .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16))
+      .scalarize(0)
+      .lower();
+  } else {
+    getActionDefinitionsBuilder(G_FSHL)
+      .scalarize(0)
+      .lower();
+  }
+
   getActionDefinitionsBuilder(G_READCYCLECOUNTER)
     .legalFor({S64});
 
   getActionDefinitionsBuilder(G_FENCE)
     .alwaysLegal();
 
+  getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+      .scalarize(0)
+      .minScalar(0, S32)
+      .lower();
+
   getActionDefinitionsBuilder({
       // TODO: Verify V_BFI_B32 is generated from expanded bit ops
       G_FCOPYSIGN,
@@ -1619,9 +1641,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       G_SADDO, G_SSUBO,
 
        // TODO: Implement
-      G_FMINIMUM, G_FMAXIMUM,
-      G_FSHL
-    }).lower();
+      G_FMINIMUM, G_FMAXIMUM}).lower();
 
   getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
         G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
@@ -4519,27 +4539,55 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(
 bool AMDGPULegalizerInfo::legalizeTrapIntrinsic(MachineInstr &MI,
                                                 MachineRegisterInfo &MRI,
                                                 MachineIRBuilder &B) const {
-  // Is non-HSA path or trap-handler disabled? then, insert s_endpgm instruction
-  if (ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
-      !ST.isTrapHandlerEnabled()) {
-    B.buildInstr(AMDGPU::S_ENDPGM).addImm(0);
-  } else {
-    // Pass queue pointer to trap handler as input, and insert trap instruction
-    // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
-    MachineRegisterInfo &MRI = *B.getMRI();
+  if (!ST.isTrapHandlerEnabled() ||
+      ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
+    return legalizeTrapEndpgm(MI, MRI, B);
+
+  if (Optional<uint8_t> HsaAbiVer = AMDGPU::getHsaAbiVersion(&ST)) {
+    switch (*HsaAbiVer) {
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V2:
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
+      return legalizeTrapHsaQueuePtr(MI, MRI, B);
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
+      return ST.supportsGetDoorbellID() ?
+          legalizeTrapHsa(MI, MRI, B) :
+          legalizeTrapHsaQueuePtr(MI, MRI, B);
+    }
+  }
 
-    Register LiveIn =
-      MRI.createGenericVirtualRegister(LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
-    if (!loadInputValue(LiveIn, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
-      return false;
+  llvm_unreachable("Unknown trap handler");
+}
 
-    Register SGPR01(AMDGPU::SGPR0_SGPR1);
-    B.buildCopy(SGPR01, LiveIn);
-    B.buildInstr(AMDGPU::S_TRAP)
-        .addImm(GCNSubtarget::TrapIDLLVMTrap)
-        .addReg(SGPR01, RegState::Implicit);
-  }
+bool AMDGPULegalizerInfo::legalizeTrapEndpgm(
+    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
+  B.buildInstr(AMDGPU::S_ENDPGM).addImm(0);
+  MI.eraseFromParent();
+  return true;
+}
+
+bool AMDGPULegalizerInfo::legalizeTrapHsaQueuePtr(
+    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
+  // Pass queue pointer to trap handler as input, and insert trap instruction
+  // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
+  Register LiveIn =
+    MRI.createGenericVirtualRegister(LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
+  if (!loadInputValue(LiveIn, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
+    return false;
+
+  Register SGPR01(AMDGPU::SGPR0_SGPR1);
+  B.buildCopy(SGPR01, LiveIn);
+  B.buildInstr(AMDGPU::S_TRAP)
+      .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
+      .addReg(SGPR01, RegState::Implicit);
+
+  MI.eraseFromParent();
+  return true;
+}
 
+bool AMDGPULegalizerInfo::legalizeTrapHsa(
+    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
+  B.buildInstr(AMDGPU::S_TRAP)
+      .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
   MI.eraseFromParent();
   return true;
 }
@@ -4548,8 +4596,8 @@ bool AMDGPULegalizerInfo::legalizeDebugTrapIntrinsic(
     MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
   // Is non-HSA path or trap-handler disabled? then, report a warning
   // accordingly
-  if (ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
-      !ST.isTrapHandlerEnabled()) {
+  if (!ST.isTrapHandlerEnabled() ||
+      ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
     DiagnosticInfoUnsupported NoTrap(B.getMF().getFunction(),
                                      "debugtrap handler not supported",
                                      MI.getDebugLoc(), DS_Warning);
@@ -4557,7 +4605,8 @@ bool AMDGPULegalizerInfo::legalizeDebugTrapIntrinsic(
     Ctx.diagnose(NoTrap);
   } else {
     // Insert debug-trap instruction
-    B.buildInstr(AMDGPU::S_TRAP).addImm(GCNSubtarget::TrapIDLLVMDebugTrap);
+    B.buildInstr(AMDGPU::S_TRAP)
+        .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap));
   }
 
   MI.eraseFromParent();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 87e8b2128a2590bbb380629b90fda78a4001dea5..de603e2d9cec1b87faebd2f13a0f436439083f13 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -183,6 +183,12 @@ public:
 
   bool legalizeTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
                              MachineIRBuilder &B) const;
+  bool legalizeTrapEndpgm(MachineInstr &MI, MachineRegisterInfo &MRI,
+                          MachineIRBuilder &B) const;
+  bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI,
+                               MachineIRBuilder &B) const;
+  bool legalizeTrapHsa(MachineInstr &MI, MachineRegisterInfo &MRI,
+                       MachineIRBuilder &B) const;
   bool legalizeDebugTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
                                   MachineIRBuilder &B) const;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPTNote.h b/llvm/lib/Target/AMDGPU/AMDGPUPTNote.h
index 756bc948b1dd5d0c1cff28375b6c505e369ca9b0..8af7979dba8b9bdb4030903c685071cc20fe30cb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPTNote.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPTNote.h
@@ -26,22 +26,6 @@ const char SectionName[] = ".note";
 const char NoteNameV2[] = "AMD";
 const char NoteNameV3[] = "AMDGPU";
 
-// TODO: Remove this file once we drop code object v2.
-enum NoteType{
-    NT_AMDGPU_HSA_RESERVED_0 = 0,
-    NT_AMDGPU_HSA_CODE_OBJECT_VERSION = 1,
-    NT_AMDGPU_HSA_HSAIL = 2,
-    NT_AMDGPU_HSA_ISA = 3,
-    NT_AMDGPU_HSA_PRODUCER = 4,
-    NT_AMDGPU_HSA_PRODUCER_OPTIONS = 5,
-    NT_AMDGPU_HSA_EXTENSION = 6,
-    NT_AMDGPU_HSA_RESERVED_7 = 7,
-    NT_AMDGPU_HSA_RESERVED_8 = 8,
-    NT_AMDGPU_HSA_RESERVED_9 = 9,
-    NT_AMDGPU_HSA_HLDEBUG_DEBUG = 101,
-    NT_AMDGPU_HSA_HLDEBUG_TARGET = 102
-};
-
 } // End namespace ElfNote
 } // End namespace AMDGPU
 } // End namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
index 0ca0ea18551aace0842a3d1c6774006cb03eced5..c58b15f0eb94ee2e896383bb7362e6c90f2a7482 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
@@ -249,6 +249,9 @@ void AMDGPUPreLegalizerCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
     AU.addRequired<MachineDominatorTree>();
     AU.addPreserved<MachineDominatorTree>();
   }
+
+  AU.addRequired<GISelCSEAnalysisWrapperPass>();
+  AU.addPreserved<GISelCSEAnalysisWrapperPass>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
@@ -270,8 +273,13 @@ bool AMDGPUPreLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
       IsOptNone ? nullptr : &getAnalysis<MachineDominatorTree>();
   AMDGPUPreLegalizerCombinerInfo PCInfo(EnableOpt, F.hasOptSize(),
                                         F.hasMinSize(), KB, MDT);
+  // Enable CSE.
+  GISelCSEAnalysisWrapper &Wrapper =
+      getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
+  auto *CSEInfo = &Wrapper.get(TPC->getCSEConfig());
+
   Combiner C(PCInfo, TPC);
-  return C.combineMachineInstrs(MF, /*CSEInfo*/ nullptr);
+  return C.combineMachineInstrs(MF, CSEInfo);
 }
 
 char AMDGPUPreLegalizerCombiner::ID = 0;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index dfab7ccc6aae7dee079e3016f9fefcd9e018a048..4b2ec0118ec47380995e11526e9adbd492bd8e4c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -267,6 +267,7 @@ GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
     HasDot4Insts(false),
     HasDot5Insts(false),
     HasDot6Insts(false),
+    HasDot7Insts(false),
     HasMAIInsts(false),
     HasPkFmacF16Inst(false),
     HasAtomicFaddInsts(false),
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 9db4e8c8472fdd0bdcb7a907d8bd6a706a8e8c63..ceabee546eba132ac6270b1e3470832e965a1a82 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -249,9 +249,8 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeSIModeRegisterPass(*PR);
   initializeSIWholeQuadModePass(*PR);
   initializeSILowerControlFlowPass(*PR);
-  initializeSIRemoveShortExecBranchesPass(*PR);
   initializeSIPreEmitPeepholePass(*PR);
-  initializeSIInsertSkipsPass(*PR);
+  initializeSILateBranchLoweringPass(*PR);
   initializeSIMemoryLegalizerPass(*PR);
   initializeSIOptimizeExecMaskingPass(*PR);
   initializeSIPreAllocateWWMRegsPass(*PR);
@@ -1215,9 +1214,9 @@ void GCNPassConfig::addPreEmitPass() {
   if (getOptLevel() > CodeGenOpt::None)
     addPass(&SIInsertHardClausesID);
 
-  addPass(&SIRemoveShortExecBranchesID);
-  addPass(&SIInsertSkipsPassID);
-  addPass(&SIPreEmitPeepholeID);
+  addPass(&SILateBranchLoweringPassID);
+  if (getOptLevel() > CodeGenOpt::None)
+    addPass(&SIPreEmitPeepholeID);
   // The hazard recognizer that runs as part of the post-ra scheduler does not
   // guarantee to be able handle all hazards correctly. This is because if there
   // are multiple scheduling regions in a basic block, the regions are scheduled
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index d7fa1279e993c31fb6cfda6ecbb61b1ca254e3c3..ebc685207fe13b350e9972dfa2c8106288337f3a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -310,8 +310,17 @@ unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const {
   return getHardwareNumberOfRegisters(false) / NumVGPRs;
 }
 
-unsigned GCNTTIImpl::getRegisterBitWidth(bool Vector) const {
-  return (Vector && ST->hasPackedFP32Ops()) ? 64 : 32;
+TypeSize
+GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+  switch (K) {
+  case TargetTransformInfo::RGK_Scalar:
+    return TypeSize::getFixed(32);
+  case TargetTransformInfo::RGK_FixedWidthVector:
+    return TypeSize::getFixed(ST->hasPackedFP32Ops() ? 64 : 32);
+  case TargetTransformInfo::RGK_ScalableVector:
+    return TypeSize::getScalable(0);
+  }
+  llvm_unreachable("Unsupported register kind");
 }
 
 unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {
@@ -1103,7 +1112,8 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
 }
 
 unsigned GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *VT,
-                                    int Index, VectorType *SubTp) {
+                                    ArrayRef<int> Mask, int Index,
+                                    VectorType *SubTp) {
   if (ST->hasVOP3PInsts()) {
     if (cast<FixedVectorType>(VT)->getNumElements() == 2 &&
         DL.getTypeSizeInBits(VT->getElementType()) == 16) {
@@ -1121,7 +1131,7 @@ unsigned GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *VT,
     }
   }
 
-  return BaseT::getShuffleCost(Kind, VT, Index, SubTp);
+  return BaseT::getShuffleCost(Kind, VT, Mask, Index, SubTp);
 }
 
 bool GCNTTIImpl::areInlineCompatible(const Function *Caller,
@@ -1223,8 +1233,9 @@ unsigned R600TTIImpl::getNumberOfRegisters(bool Vec) const {
   return getHardwareNumberOfRegisters(Vec);
 }
 
-unsigned R600TTIImpl::getRegisterBitWidth(bool Vector) const {
-  return 32;
+TypeSize
+R600TTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+  return TypeSize::getFixed(32);
 }
 
 unsigned R600TTIImpl::getMinVectorRegisterBitWidth() const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 9eaad39f985b72e9f67f85a2963024379b915527..02b1df1d44d710899efeb78db69ec2010be2474a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -121,7 +121,7 @@ public:
   unsigned getHardwareNumberOfRegisters(bool Vector) const;
   unsigned getNumberOfRegisters(bool Vector) const;
   unsigned getNumberOfRegisters(unsigned RCID) const;
-  unsigned getRegisterBitWidth(bool Vector) const;
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const;
   unsigned getMinVectorRegisterBitWidth() const;
   unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const;
   unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
@@ -197,8 +197,8 @@ public:
 
   unsigned getVectorSplitCost() { return 0; }
 
-  unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
-                          VectorType *SubTp);
+  unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
+                          ArrayRef<int> Mask, int Index, VectorType *SubTp);
 
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
@@ -243,7 +243,7 @@ public:
                              TTI::PeelingPreferences &PP);
   unsigned getHardwareNumberOfRegisters(bool Vec) const;
   unsigned getNumberOfRegisters(bool Vec) const;
-  unsigned getRegisterBitWidth(bool Vector) const;
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const;
   unsigned getMinVectorRegisterBitWidth() const;
   unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const;
   bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, Align Alignment,
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 5547cd6c1c320bdbea7fa80c0b4dc765917dffe8..b2a79ca2ce7b69064756f7c924b499a140231bba 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -338,9 +338,6 @@ public:
   bool isGDS() const { return isImmTy(ImmTyGDS); }
   bool isLDS() const { return isImmTy(ImmTyLDS); }
   bool isCPol() const { return isImmTy(ImmTyCPol); }
-  // "CPol_GLC1" is a MatchClass of the CPOL_GLC1 operand with the default and
-  // forced value of the GLC operand.
-  bool isCPol_GLC1() const { return isImmTy(ImmTyCPol); }
   bool isSWZ() const { return isImmTy(ImmTySWZ); }
   bool isTFE() const { return isImmTy(ImmTyTFE); }
   bool isD16() const { return isImmTy(ImmTyD16); }
@@ -1215,7 +1212,8 @@ private:
   bool ParseDirectiveHSACodeObjectISA();
   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
   bool ParseDirectiveAMDKernelCodeT();
-  bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
+  // TODO: Possibly make subtargetHasRegister const.
+  bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo);
   bool ParseDirectiveAMDGPUHsaKernel();
 
   bool ParseDirectiveISAVersion();
@@ -1294,7 +1292,7 @@ public:
       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
       AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
       MCContext &Ctx = getContext();
-      if (ISA.Major >= 6 && isHsaAbiVersion3(&getSTI())) {
+      if (ISA.Major >= 6 && isHsaAbiVersion3Or4(&getSTI())) {
         MCSymbol *Sym =
             Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_number"));
         Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
@@ -1311,7 +1309,7 @@ public:
         Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
         Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
       }
-      if (ISA.Major >= 6 && isHsaAbiVersion3(&getSTI())) {
+      if (ISA.Major >= 6 && isHsaAbiVersion3Or4(&getSTI())) {
         initializeGprCountSymbol(IS_VGPR);
         initializeGprCountSymbol(IS_SGPR);
       } else
@@ -1319,10 +1317,6 @@ public:
     }
   }
 
-  bool hasXNACK() const {
-    return AMDGPU::hasXNACK(getSTI());
-  }
-
   bool hasMIMG_R128() const {
     return AMDGPU::hasMIMG_R128(getSTI());
   }
@@ -1539,7 +1533,6 @@ private:
   bool validateMIMGD16(const MCInst &Inst);
   bool validateMIMGDim(const MCInst &Inst);
   bool validateMIMGMSAA(const MCInst &Inst);
-  bool validateLdsDirect(const MCInst &Inst);
   bool validateOpSel(const MCInst &Inst);
   bool validateVccOperand(unsigned Reg) const;
   bool validateVOP3Literal(const MCInst &Inst, const OperandVector &Operands);
@@ -1549,6 +1542,7 @@ private:
   bool validateDivScale(const MCInst &Inst);
   bool validateCoherencyBits(const MCInst &Inst, const OperandVector &Operands,
                              const SMLoc &IDLoc);
+  Optional<StringRef> validateLdsDirect(const MCInst &Inst);
   unsigned getConstantBusLimit(unsigned Opcode) const;
   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
@@ -1583,6 +1577,8 @@ private:
   void lex();
 
 public:
+  void onBeginOfFile() override;
+
   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
   OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
 
@@ -1620,7 +1616,6 @@ public:
   void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
 
   AMDGPUOperand::Ptr defaultCPol() const;
-  AMDGPUOperand::Ptr defaultCPol_GLC1() const;
 
   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
   AMDGPUOperand::Ptr defaultSMEMOffset() const;
@@ -2715,7 +2710,7 @@ AMDGPUAsmParser::parseRegister(bool RestoreOnFailure) {
   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
     return nullptr;
   }
-  if (isHsaAbiVersion3(&getSTI())) {
+  if (isHsaAbiVersion3Or4(&getSTI())) {
     if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
       return nullptr;
   } else
@@ -3768,7 +3763,7 @@ static bool IsRevOpcode(const unsigned Opcode)
   }
 }
 
-bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
+Optional<StringRef> AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
 
   using namespace SIInstrFlags;
   const unsigned Opcode = Inst.getOpcode();
@@ -3776,33 +3771,29 @@ bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
 
   // lds_direct register is defined so that it can be used
   // with 9-bit operands only. Ignore encodings which do not accept these.
-  if ((Desc.TSFlags & (VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA)) == 0)
-    return true;
+  const auto Enc = VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA;
+  if ((Desc.TSFlags & Enc) == 0)
+    return None;
 
-  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
-  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
-  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
+  for (auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
+    auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
+    if (SrcIdx == -1)
+      break;
+    const auto &Src = Inst.getOperand(SrcIdx);
+    if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
 
-  const int SrcIndices[] = { Src1Idx, Src2Idx };
+      if (isGFX90A())
+        return StringRef("lds_direct is not supported on this GPU");
 
-  // lds_direct cannot be specified as either src1 or src2.
-  for (int SrcIdx : SrcIndices) {
-    if (SrcIdx == -1) break;
-    const MCOperand &Src = Inst.getOperand(SrcIdx);
-    if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
-      return false;
+      if (IsRevOpcode(Opcode) || (Desc.TSFlags & SIInstrFlags::SDWA))
+        return StringRef("lds_direct cannot be used with this instruction");
+
+      if (SrcName != OpName::src0)
+        return StringRef("lds_direct may be used as src0 only");
     }
   }
 
-  if (Src0Idx == -1)
-    return true;
-
-  const MCOperand &Src = Inst.getOperand(Src0Idx);
-  if (!Src.isReg() || Src.getReg() != LDS_DIRECT)
-    return true;
-
-  // lds_direct is specified as src0. Check additional limitations.
-  return (Desc.TSFlags & SIInstrFlags::SDWA) == 0 && !IsRevOpcode(Opcode);
+  return None;
 }
 
 SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const {
@@ -4133,9 +4124,8 @@ bool AMDGPUAsmParser::validateCoherencyBits(const MCInst &Inst,
 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
                                           const SMLoc &IDLoc,
                                           const OperandVector &Operands) {
-  if (!validateLdsDirect(Inst)) {
-    Error(getRegLoc(AMDGPU::LDS_DIRECT, Operands),
-      "invalid use of lds_direct");
+  if (auto ErrMsg = validateLdsDirect(Inst)) {
+    Error(getRegLoc(LDS_DIRECT, Operands), *ErrMsg);
     return false;
   }
   if (!validateSOPLiteral(Inst)) {
@@ -4395,21 +4385,19 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
   if (getSTI().getTargetTriple().getArch() != Triple::amdgcn)
     return TokError("directive only supported for amdgcn architecture");
 
-  std::string Target;
-
-  SMLoc TargetStart = getLoc();
-  if (getParser().parseEscapedString(Target))
+  std::string TargetIDDirective;
+  SMLoc TargetStart = getTok().getLoc();
+  if (getParser().parseEscapedString(TargetIDDirective))
     return true;
-  SMRange TargetRange = SMRange(TargetStart, getLoc());
 
-  std::string ExpectedTarget;
-  raw_string_ostream ExpectedTargetOS(ExpectedTarget);
-  IsaInfo::streamIsaVersion(&getSTI(), ExpectedTargetOS);
+  SMRange TargetRange = SMRange(TargetStart, getTok().getLoc());
+  if (getTargetStreamer().getTargetID()->toString() != TargetIDDirective)
+    return getParser().Error(TargetRange.Start,
+        (Twine(".amdgcn_target directive's target id ") +
+         Twine(TargetIDDirective) +
+         Twine(" does not match the specified target id ") +
+         Twine(getTargetStreamer().getTargetID()->toString())).str());
 
-  if (Target != ExpectedTargetOS.str())
-    return Error(TargetRange.Start, "target must match options", TargetRange);
-
-  getTargetStreamer().EmitDirectiveAMDGCNTarget(Target);
   return false;
 }
 
@@ -4482,7 +4470,6 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
   unsigned UserSGPRCount = 0;
   bool ReserveVCC = true;
   bool ReserveFlatScr = true;
-  bool ReserveXNACK = hasXNACK();
   Optional<bool> EnableWavefrontSize32;
 
   while (true) {
@@ -4525,6 +4512,10 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
       if (!isUInt<sizeof(KD.private_segment_fixed_size) * CHAR_BIT>(Val))
         return OutOfRangeError(ValRange);
       KD.private_segment_fixed_size = Val;
+    } else if (ID == ".amdhsa_kernarg_size") {
+      if (!isUInt<sizeof(KD.kernarg_size) * CHAR_BIT>(Val))
+        return OutOfRangeError(ValRange);
+      KD.kernarg_size = Val;
     } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") {
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
@@ -4624,7 +4615,9 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
         return Error(IDRange.Start, "directive requires gfx8+", IDRange);
       if (!isUInt<1>(Val))
         return OutOfRangeError(ValRange);
-      ReserveXNACK = Val;
+      if (Val != getTargetStreamer().getTargetID()->isXnackOnOrAny())
+        return getParser().Error(IDRange.Start, ".amdhsa_reserve_xnack_mask does not match target id",
+                                 IDRange);
     } else if (ID == ".amdhsa_float_round_mode_32") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
                        COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange);
@@ -4715,7 +4708,8 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
   unsigned VGPRBlocks;
   unsigned SGPRBlocks;
   if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
-                         ReserveXNACK, EnableWavefrontSize32, NextFreeVGPR,
+                         getTargetStreamer().getTargetID()->isXnackOnOrAny(),
+                         EnableWavefrontSize32, NextFreeVGPR,
                          VGPRRange, NextFreeSGPR, SGPRRange, VGPRBlocks,
                          SGPRBlocks))
     return true;
@@ -4752,7 +4746,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
 
   getTargetStreamer().EmitAmdhsaKernelDescriptor(
       getSTI(), KernelName, KD, NextFreeVGPR, NextFreeSGPR, ReserveVCC,
-      ReserveFlatScr, ReserveXNACK);
+      ReserveFlatScr);
   return false;
 }
 
@@ -4778,9 +4772,9 @@ bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
   // targeted GPU.
   if (isToken(AsmToken::EndOfStatement)) {
     AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
-    getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
-                                                      ISA.Stepping,
-                                                      "AMD", "AMDGPU");
+    getTargetStreamer().EmitDirectiveHSACodeObjectISAV2(ISA.Major, ISA.Minor,
+                                                        ISA.Stepping,
+                                                        "AMD", "AMDGPU");
     return false;
   }
 
@@ -4805,8 +4799,8 @@ bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
   if (!parseString(ArchName, "invalid arch name"))
     return true;
 
-  getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
-                                                    VendorName, ArchName);
+  getTargetStreamer().EmitDirectiveHSACodeObjectISAV2(Major, Minor, Stepping,
+                                                      VendorName, ArchName);
   return false;
 }
 
@@ -4915,19 +4909,11 @@ bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
                  "architectures");
   }
 
-  auto ISAVersionStringFromASM = getToken().getStringContents();
-
-  std::string ISAVersionStringFromSTI;
-  raw_string_ostream ISAVersionStreamFromSTI(ISAVersionStringFromSTI);
-  IsaInfo::streamIsaVersion(&getSTI(), ISAVersionStreamFromSTI);
-
-  if (ISAVersionStringFromASM != ISAVersionStreamFromSTI.str()) {
-    return Error(getLoc(),
-                 ".amd_amdgpu_isa directive does not match triple and/or mcpu "
-                 "arguments specified through the command line");
-  }
+  auto TargetIDDirective = getLexer().getTok().getStringContents();
+  if (getTargetStreamer().getTargetID()->toString() != TargetIDDirective)
+    return Error(getParser().getTok().getLoc(), "target id must match options");
 
-  getTargetStreamer().EmitISAVersion(ISAVersionStreamFromSTI.str());
+  getTargetStreamer().EmitISAVersion();
   Lex();
 
   return false;
@@ -4937,7 +4923,7 @@ bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
   const char *AssemblerDirectiveBegin;
   const char *AssemblerDirectiveEnd;
   std::tie(AssemblerDirectiveBegin, AssemblerDirectiveEnd) =
-      isHsaAbiVersion3(&getSTI())
+      isHsaAbiVersion3Or4(&getSTI())
           ? std::make_tuple(HSAMD::V3::AssemblerDirectiveBegin,
                             HSAMD::V3::AssemblerDirectiveEnd)
           : std::make_tuple(HSAMD::AssemblerDirectiveBegin,
@@ -4954,7 +4940,7 @@ bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
                           HSAMetadataString))
     return true;
 
-  if (isHsaAbiVersion3(&getSTI())) {
+  if (isHsaAbiVersion3Or4(&getSTI())) {
     if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
       return Error(getLoc(), "invalid HSA metadata");
   } else {
@@ -5104,12 +5090,9 @@ bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
   StringRef IDVal = DirectiveID.getString();
 
-  if (isHsaAbiVersion3(&getSTI())) {
-    if (IDVal == ".amdgcn_target")
-      return ParseDirectiveAMDGCNTarget();
-
+  if (isHsaAbiVersion3Or4(&getSTI())) {
     if (IDVal == ".amdhsa_kernel")
-      return ParseDirectiveAMDHSAKernel();
+     return ParseDirectiveAMDHSAKernel();
 
     // TODO: Restructure/combine with PAL metadata directive.
     if (IDVal == AMDGPU::HSAMD::V3::AssemblerDirectiveBegin)
@@ -5134,6 +5117,9 @@ bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
       return ParseDirectiveHSAMetadata();
   }
 
+  if (IDVal == ".amdgcn_target")
+    return ParseDirectiveAMDGCNTarget();
+
   if (IDVal == ".amdgpu_lds")
     return ParseDirectiveAMDGPULDS();
 
@@ -5147,7 +5133,7 @@ bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
 }
 
 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
-                                           unsigned RegNo) const {
+                                           unsigned RegNo) {
 
   for (MCRegAliasIterator R(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, &MRI, true);
        R.isValid(); ++R) {
@@ -5179,7 +5165,7 @@ bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
   case AMDGPU::XNACK_MASK:
   case AMDGPU::XNACK_MASK_LO:
   case AMDGPU::XNACK_MASK_HI:
-    return (isVI() || isGFX9()) && hasXNACK();
+    return (isVI() || isGFX9()) && getTargetStreamer().getTargetID()->isXnackSupported();
   case AMDGPU::SGPR_NULL:
     return isGFX10Plus();
   default:
@@ -6910,11 +6896,6 @@ AMDGPUOperand::Ptr AMDGPUAsmParser::defaultCPol() const {
   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyCPol);
 }
 
-AMDGPUOperand::Ptr AMDGPUAsmParser::defaultCPol_GLC1() const {
-  return AMDGPUOperand::CreateImm(this, CPol::GLC, SMLoc(),
-                                  AMDGPUOperand::ImmTyCPol);
-}
-
 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
                                    const OperandVector &Operands,
                                    bool IsAtomic,
@@ -7275,6 +7256,18 @@ static const OptionalOperand AMDGPUOptionalOperandTable[] = {
   {"abid", AMDGPUOperand::ImmTyABID, false, nullptr}
 };
 
+void AMDGPUAsmParser::onBeginOfFile() {
+  if (!getParser().getStreamer().getTargetStreamer() ||
+      getSTI().getTargetTriple().getArch() == Triple::r600)
+    return;
+
+  if (!getTargetStreamer().getTargetID())
+    getTargetStreamer().initializeTargetID(getSTI(), getSTI().getFeatureString());
+
+  if (isHsaAbiVersion3Or4(&getSTI()))
+    getTargetStreamer().EmitDirectiveAMDGCNTarget();
+}
+
 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
 
   OperandMatchResultTy res = parseOptionalOpr(Operands);
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index e7930ba151c135528e8b4a9237af4febed0f63f3..74705e47c3536e1ad0c5c485c60e87c1771faed7 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -6,15 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">;
-def MUBUFAddr64Atomic : ComplexPattern<i64, 4, "SelectMUBUFAddr64">;
+def MUBUFAddr64 : ComplexPattern<i64, 4, "SelectMUBUFAddr64">;
+def MUBUFOffset : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
 
 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
 
-def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">;
-def MUBUFOffsetAtomic : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
-
 def BUFAddrKind {
   int Offset = 0;
   int OffEn  = 1;
@@ -120,6 +117,7 @@ class MTBUF_Real <MTBUF_Pseudo ps> :
   let Constraints        = ps.Constraints;
   let DisableEncoding    = ps.DisableEncoding;
   let TSFlags            = ps.TSFlags;
+  let SchedRW            = ps.SchedRW;
 
   bits<12> offset;
   bits<5>  cpol;
@@ -350,6 +348,7 @@ class MUBUF_Real <MUBUF_Pseudo ps> :
   let DisableEncoding      = ps.DisableEncoding;
   let TSFlags              = ps.TSFlags;
   let UseNamedOperandTable = ps.UseNamedOperandTable;
+  let SchedRW              = ps.SchedRW;
 
   bits<12> offset;
   bits<5>  cpol;
@@ -402,19 +401,19 @@ class getMUBUFInsDA<list<RegisterClass> vdataList,
   RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret;
   dag InsNoData = !if(!empty(vaddrList),
     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
-         offset:$offset, CPol:$cpol),
+         offset:$offset, CPol_0:$cpol),
     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
-         offset:$offset, CPol:$cpol)
+         offset:$offset, CPol_0:$cpol)
   );
   dag InsData = !if(!empty(vaddrList),
     (ins vdata_op:$vdata,                    SReg_128:$srsrc,
-         SCSrc_b32:$soffset, offset:$offset, CPol:$cpol),
+         SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol),
     (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
-         SCSrc_b32:$soffset, offset:$offset, CPol:$cpol)
+         SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol)
   );
   dag ret = !con(
               !if(!empty(vdataList), InsNoData, InsData),
-              !if(isLds, (ins SWZ:$swz), (ins TFE:$tfe, SWZ:$swz))
+              !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz))
              );
 }
 
@@ -506,15 +505,15 @@ class MUBUF_Load_Pseudo <string opName,
 }
 
 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
-  (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, CPol:$cpol, i1:$tfe, i1:$swz))),
-  (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, CPol:$cpol, i1:$tfe, i1:$swz))
+  (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))),
+  (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset))
 >;
 
 class MUBUF_Addr64_Load_Pat <Instruction inst,
                             ValueType load_vt = i32,
                             SDPatternOperator ld = null_frag> : Pat <
-  (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, CPol:$cpol, i1:$tfe, i1:$swz))),
-  (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, CPol:$cpol, i1:$tfe, i1:$swz))
+  (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))),
+  (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset))
 >;
 
 multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
@@ -585,12 +584,12 @@ multiclass MUBUF_Pseudo_Stores<string opName,
 
   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt,
     [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
-                                             i16:$offset, CPol:$cpol, i1:$tfe, i1:$swz))]>,
+                                             i16:$offset))]>,
     MUBUFAddr64Table<0, NAME>;
 
   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt,
     [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
-                                             i16:$offset, CPol:$cpol, i1:$tfe, i1:$swz))]>,
+                                             i16:$offset))]>,
     MUBUFAddr64Table<1, NAME>;
 
   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>;
@@ -679,7 +678,7 @@ class MUBUF_Atomic_Pseudo<string opName,
   let has_glc = 0;
   let has_dlc = 0;
   let has_tfe = 0;
-  let has_sccb = 0;
+  let has_sccb = 1;
   let maybeAtomic = 1;
   let AsmMatchConverter = "cvtMubufAtomic";
 }
@@ -757,14 +756,14 @@ multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
   let FPAtomic = isFP in
   def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
     [(set vdataType:$vdata,
-     (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset),
+     (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset),
              vdataType:$vdata_in))]>,
     MUBUFAddr64Table <0, NAME # "_RTN">;
 
   let FPAtomic = isFP in
   def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
     [(set vdataType:$vdata,
-     (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset),
+     (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset),
               vdataType:$vdata_in))]>,
     MUBUFAddr64Table <1, NAME # "_RTN">;
 
@@ -1539,20 +1538,20 @@ def : GCNPat<
 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
                               PatFrag constant_ld> : GCNPat <
      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
-                                   i16:$offset, CPol:$cpol, i1:$tfe, i1:$swz))),
-     (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, CPol:$cpol, $tfe, $swz)
+                                   i16:$offset))),
+     (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
   >;
 
 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
                                      ValueType vt, PatFrag atomic_ld> {
   def : GCNPat <
      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))),
-     (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
+     (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
   >;
 
   def : GCNPat <
-    (vt (atomic_ld (MUBUFOffsetAtomic v4i32:$rsrc, i32:$soffset, i16:$offset))),
-    (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0)
+    (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))),
+    (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset))
   >;
 }
 
@@ -1572,9 +1571,8 @@ multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
                                PatFrag ld> {
 
   def : GCNPat <
-    (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
-                         i16:$offset, CPol:$cpol, i1:$tfe, i1:$swz))),
-    (Instr_OFFSET $srsrc, $soffset, $offset, CPol:$cpol, $tfe, $swz)
+    (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))),
+    (Instr_OFFSET $srsrc, $soffset, $offset)
   >;
 }
 
@@ -1612,12 +1610,12 @@ multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen,
                                 ValueType vt, PatFrag ld_frag> {
   def : GCNPat <
     (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in),
-    (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $in)
+    (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in)
   >;
 
   def : GCNPat <
     (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in),
-    (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $in)
+    (InstrOffset $srsrc, $soffset, $offset, $in)
   >;
 }
 
@@ -1663,12 +1661,12 @@ multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo In
   // Store follows atomic op convention so address is first
   def : GCNPat <
      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val),
-     (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
+     (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset)
   >;
 
   def : GCNPat <
-    (atomic_st (MUBUFOffsetAtomic v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
-    (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0)
+    (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
+    (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset))
   >;
 }
 let SubtargetPredicate = isGFX6GFX7 in {
@@ -1681,9 +1679,8 @@ multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
                                PatFrag st> {
 
   def : GCNPat <
-    (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
-                                i16:$offset, CPol:$cpol, i1:$tfe, i1:$swz)),
-    (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, CPol:$cpol, $tfe, $swz)
+    (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)),
+    (Instr_OFFSET $vdata, $srsrc, $soffset, $offset)
   >;
 }
 
@@ -2259,7 +2256,8 @@ defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
 // GFX8, GFX9 (VI).
 //===----------------------------------------------------------------------===//
 
-class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc> :
+class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc,
+                          bit has_sccb = ps.has_sccb> :
   MUBUF_Real<ps>,
   Enc64,
   SIMCInstr<ps.PseudoInstr, Enc>,
@@ -2270,7 +2268,7 @@ class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc> :
   let Inst{12}    = ps.offen;
   let Inst{13}    = ps.idxen;
   let Inst{14}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
-  let Inst{15}    = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value);
+  let Inst{15}    = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value);
   let Inst{16}    = ps.lds;
   let Inst{17}    = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
   let Inst{24-18} = op;
@@ -2281,29 +2279,31 @@ class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc> :
   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
 }
 
-class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
-  MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI> {
+class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> :
+  MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> {
   let AssemblerPredicate = isGFX8GFX9NotGFX90A;
   let DecoderNamespace = "GFX8";
 
   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
 }
 
-class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps> :
-  MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A> {
+class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps,
+                         bit has_sccb = ps.has_sccb> :
+  MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> {
   let AssemblerPredicate = isGFX90APlus;
   let DecoderNamespace = "GFX90A";
-  let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands);
+  let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""),
+                                !subst("$tfe", "", ps.AsmOperands));
 
   let Inst{55}    = acc;
 }
 
 multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> {
   def _vi :     MUBUF_Real_vi<op, ps>;
-  def _gfx90a : MUBUF_Real_gfx90a<op, ps>;
+  def _gfx90a : MUBUF_Real_gfx90a<op, ps, !and(ps.has_sccb,!not(ps.FPAtomic))>;
 }
 
-multiclass MUBUF_Real_AllAddr_vi<bits<7> op, bit isAtomic = 0, bit isAtomicRet = 0> {
+multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
   defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
   defm _OFFEN  : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
   defm _IDXEN  : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
@@ -2379,7 +2379,7 @@ multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {
 }
 
 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
-  MUBUF_Real_AllAddr_vi<op, 1, 0> {
+  MUBUF_Real_AllAddr_vi<op> {
   defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
   defm _OFFEN_RTN  : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
   defm _IDXEN_RTN  : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
@@ -2483,7 +2483,7 @@ defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>;
 
 } // End SubtargetPredicate = HasAtomicFaddInsts
 
-let SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus in {
+let SubtargetPredicate = isGFX90APlus in {
   defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>;
   defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>;
   defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>;
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 7aa256821167e93dd7bb2ec75fb28448e13b7381..0688336cec2ddcc457f447aed940d46c69ce9580 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -119,7 +119,7 @@ add_llvm_target(AMDGPUCodeGen
   SIFormMemoryClauses.cpp
   SIFrameLowering.cpp
   SIInsertHardClauses.cpp
-  SIInsertSkips.cpp
+  SILateBranchLowering.cpp
   SIInsertWaitcnts.cpp
   SIInstrInfo.cpp
   SIISelLowering.cpp
@@ -137,7 +137,6 @@ add_llvm_target(AMDGPUCodeGen
   SIPreEmitPeephole.cpp
   SIProgramInfo.cpp
   SIRegisterInfo.cpp
-  SIRemoveShortExecBranches.cpp
   SIShrinkInstructions.cpp
   SIWholeQuadMode.cpp
   GCNILPSched.cpp
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index c856b2a0bbd21c34aa65ed55e344d4bae9686f87..d07ab15f664de16177a2ad2f75955ade1b118cc6 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -63,8 +63,9 @@ class DS_Real <DS_Pseudo ds> :
 
   // copy relevant pseudo op flags
   let SubtargetPredicate = ds.SubtargetPredicate;
-  let OtherPredicates = ds.OtherPredicates;
+  let OtherPredicates    = ds.OtherPredicates;
   let AsmMatchConverter  = ds.AsmMatchConverter;
+  let SchedRW            = ds.SchedRW;
 
   // encoding fields
   bits<10> vdst;
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index 7cd5070f596c2cade7b77521ba14ec609bc0bfc7..ccdd76514d786be972da0c2db8d7ffe79d45b130 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -1620,7 +1620,6 @@ AMDGPUDisassembler::decodeKernelDescriptorDirective(
 
   uint16_t TwoByteBuffer = 0;
   uint32_t FourByteBuffer = 0;
-  uint64_t EightByteBuffer = 0;
 
   StringRef ReservedBytes;
   StringRef Indent = "\t";
@@ -1641,11 +1640,19 @@ AMDGPUDisassembler::decodeKernelDescriptorDirective(
              << FourByteBuffer << '\n';
     return MCDisassembler::Success;
 
+  case amdhsa::KERNARG_SIZE_OFFSET:
+    FourByteBuffer = DE.getU32(Cursor);
+    KdStream << Indent << ".amdhsa_kernarg_size "
+             << FourByteBuffer << '\n';
+    return MCDisassembler::Success;
+
   case amdhsa::RESERVED0_OFFSET:
-    // 8 reserved bytes, must be 0.
-    EightByteBuffer = DE.getU64(Cursor);
-    if (EightByteBuffer) {
-      return MCDisassembler::Fail;
+    // 4 reserved bytes, must be 0.
+    ReservedBytes = DE.getBytes(Cursor, 4);
+    for (int I = 0; I < 4; ++I) {
+      if (ReservedBytes[I] != 0) {
+        return MCDisassembler::Fail;
+      }
     }
     return MCDisassembler::Success;
 
diff --git a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td
index 8d3e138ba56aed32dae1e5e3a5f4e502e09d10cd..596c3d7baea0555a843868a0e5ecc7c89f376b5c 100644
--- a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td
+++ b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td
@@ -532,7 +532,10 @@ def MULADD_UINT24_eg : R600_3OP <0x10, "MULADD_UINT24",
 def : UMad24Pat<MULADD_UINT24_eg>;
 
 def BIT_ALIGN_INT_eg : R600_3OP <0xC, "BIT_ALIGN_INT", [], VecALU>;
-def : FSHRPattern <BIT_ALIGN_INT_eg>;
+def : AMDGPUPat <
+  (fshr i32:$src0, i32:$src1, i32:$src2),
+  (BIT_ALIGN_INT_eg $src0, $src1, $src2)
+>;
 def : ROTRPattern <BIT_ALIGN_INT_eg>;
 def MULADD_eg : MULADD_Common<0x14>;
 def MULADD_IEEE_eg : MULADD_IEEE_Common<0x18>;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index ede74605a81c7228a23ce0b3d4e76b9402a870f8..2a19106428c396988f1d2430e1443f5a9a538c87 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -82,11 +82,12 @@ class FLAT_Real <bits<7> op, FLAT_Pseudo ps> :
   let isCodeGenOnly = 0;
 
   // copy relevant pseudo op flags
-  let SubtargetPredicate = ps.SubtargetPredicate;
-  let AsmMatchConverter  = ps.AsmMatchConverter;
-  let OtherPredicates = ps.OtherPredicates;
-  let TSFlags = ps.TSFlags;
+  let SubtargetPredicate   = ps.SubtargetPredicate;
+  let AsmMatchConverter    = ps.AsmMatchConverter;
+  let OtherPredicates      = ps.OtherPredicates;
+  let TSFlags              = ps.TSFlags;
   let UseNamedOperandTable = ps.UseNamedOperandTable;
+  let SchedRW              = ps.SchedRW;
 
   // encoding fields
   bits<8> vaddr;
diff --git a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
index eb261049248052ad3a1a6efc2c9f271ba975939b..1a28419e5f0dcdd7c88ac0a30cd90e6306e3e2fe 100644
--- a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
@@ -54,21 +54,20 @@ namespace {
 class GCNDPPCombine : public MachineFunctionPass {
   MachineRegisterInfo *MRI;
   const SIInstrInfo *TII;
+  const GCNSubtarget *ST;
 
   using RegSubRegPair = TargetInstrInfo::RegSubRegPair;
 
   MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;
 
-  MachineInstr *createDPPInst(MachineInstr &OrigMI,
-                              MachineInstr &MovMI,
+  MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
                               RegSubRegPair CombOldVGPR,
-                              MachineOperand *OldOpnd,
-                              bool CombBCZ) const;
+                              MachineOperand *OldOpnd, bool CombBCZ,
+                              bool IsShrinkable) const;
 
-  MachineInstr *createDPPInst(MachineInstr &OrigMI,
-                              MachineInstr &MovMI,
-                              RegSubRegPair CombOldVGPR,
-                              bool CombBCZ) const;
+  MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
+                              RegSubRegPair CombOldVGPR, bool CombBCZ,
+                              bool IsShrinkable) const;
 
   bool hasNoImmOrEqual(MachineInstr &MI,
                        unsigned OpndName,
@@ -99,7 +98,8 @@ public:
   }
 
 private:
-  int getDPPOp(unsigned Op) const;
+  int getDPPOp(unsigned Op, bool IsShrinkable) const;
+  bool isShrinkable(MachineInstr &OrigMI, unsigned OrigOp) const;
 };
 
 } // end anonymous namespace
@@ -114,11 +114,32 @@ FunctionPass *llvm::createGCNDPPCombinePass() {
   return new GCNDPPCombine();
 }
 
-int GCNDPPCombine::getDPPOp(unsigned Op) const {
+bool GCNDPPCombine::isShrinkable(MachineInstr &OrigMI, unsigned OrigOp) const {
+  if (!TII->isVOP3(OrigOp)) {
+    return false;
+  }
+  if (!TII->hasVALU32BitEncoding(OrigOp)) {
+    LLVM_DEBUG(dbgs() << "  Inst hasn't e32 equivalent\n");
+    return false;
+  }
+  // check if other than abs|neg modifiers are set (opsel for example)
+  const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);
+  if (!hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src0_modifiers, 0, Mask) ||
+      !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src1_modifiers, 0, Mask) ||
+      !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::clamp, 0) ||
+      !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::omod, 0)) {
+    LLVM_DEBUG(dbgs() << "  Inst has non-default modifiers\n");
+    return false;
+  }
+  return true;
+}
+
+int GCNDPPCombine::getDPPOp(unsigned Op, bool IsShrinkable) const {
   auto DPP32 = AMDGPU::getDPPOp32(Op);
-  if (DPP32 == -1) {
+  if (IsShrinkable) {
+    assert(DPP32 == -1);
     auto E32 = AMDGPU::getVOPe32(Op);
-    DPP32 = (E32 == -1)? -1 : AMDGPU::getDPPOp32(E32);
+    DPP32 = (E32 == -1) ? -1 : AMDGPU::getDPPOp32(E32);
   }
   return (DPP32 == -1 || TII->pseudoToMCOpcode(DPP32) == -1) ? -1 : DPP32;
 }
@@ -151,12 +172,13 @@ MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
                                            MachineInstr &MovMI,
                                            RegSubRegPair CombOldVGPR,
-                                           bool CombBCZ) const {
+                                           bool CombBCZ,
+                                           bool IsShrinkable) const {
   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
          MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
 
   auto OrigOp = OrigMI.getOpcode();
-  auto DPPOp = getDPPOp(OrigOp);
+  auto DPPOp = getDPPOp(OrigOp, IsShrinkable);
   if (DPPOp == -1) {
     LLVM_DEBUG(dbgs() << "  failed: no DPP opcode\n");
     return nullptr;
@@ -314,11 +336,9 @@ static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) {
   return false;
 }
 
-MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
-                                           MachineInstr &MovMI,
-                                           RegSubRegPair CombOldVGPR,
-                                           MachineOperand *OldOpndValue,
-                                           bool CombBCZ) const {
+MachineInstr *GCNDPPCombine::createDPPInst(
+    MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair CombOldVGPR,
+    MachineOperand *OldOpndValue, bool CombBCZ, bool IsShrinkable) const {
   assert(CombOldVGPR.Reg);
   if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) {
     auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
@@ -338,7 +358,7 @@ MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
       return nullptr;
     }
   }
-  return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ);
+  return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ, IsShrinkable);
 }
 
 // returns true if MI doesn't have OpndName immediate operand or the
@@ -503,21 +523,8 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
       continue;
     }
 
-    if (TII->isVOP3(OrigOp)) {
-      if (!TII->hasVALU32BitEncoding(OrigOp)) {
-        LLVM_DEBUG(dbgs() << "  failed: VOP3 hasn't e32 equivalent\n");
-        break;
-      }
-      // check if other than abs|neg modifiers are set (opsel for example)
-      const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);
-      if (!hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src0_modifiers, 0, Mask) ||
-          !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src1_modifiers, 0, Mask) ||
-          !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::clamp, 0) ||
-          !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::omod, 0)) {
-        LLVM_DEBUG(dbgs() << "  failed: VOP3 has non-default modifiers\n");
-        break;
-      }
-    } else if (!TII->isVOP1(OrigOp) && !TII->isVOP2(OrigOp)) {
+    bool IsShrinkable = isShrinkable(OrigMI, OrigOp);
+    if (!(IsShrinkable || TII->isVOP1(OrigOp) || TII->isVOP2(OrigOp))) {
       LLVM_DEBUG(dbgs() << "  failed: not VOP1/2/3\n");
       break;
     }
@@ -542,7 +549,7 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
     LLVM_DEBUG(dbgs() << "  combining: " << OrigMI);
     if (Use == Src0) {
       if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
-                                        OldOpndValue, CombBCZ)) {
+                                        OldOpndValue, CombBCZ, IsShrinkable)) {
         DPPMIs.push_back(DPPInst);
         Rollback = false;
       }
@@ -553,8 +560,9 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
       BB->insert(OrigMI, NewMI);
       if (TII->commuteInstruction(*NewMI)) {
         LLVM_DEBUG(dbgs() << "  commuted:  " << *NewMI);
-        if (auto *DPPInst = createDPPInst(*NewMI, MovMI, CombOldVGPR,
-                                          OldOpndValue, CombBCZ)) {
+        if (auto *DPPInst =
+                createDPPInst(*NewMI, MovMI, CombOldVGPR, OldOpndValue, CombBCZ,
+                              IsShrinkable)) {
           DPPMIs.push_back(DPPInst);
           Rollback = false;
         }
@@ -587,12 +595,12 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
 }
 
 bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) {
-  auto &ST = MF.getSubtarget<GCNSubtarget>();
-  if (!ST.hasDPP() || skipFunction(MF.getFunction()))
+  ST = &MF.getSubtarget<GCNSubtarget>();
+  if (!ST->hasDPP() || skipFunction(MF.getFunction()))
     return false;
 
   MRI = &MF.getRegInfo();
-  TII = ST.getInstrInfo();
+  TII = ST->getInstrInfo();
 
   bool Changed = false;
   for (auto &MBB : MF) {
@@ -602,7 +610,7 @@ bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) {
         Changed = true;
         ++NumDPPMovsCombined;
       } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO) {
-        if (ST.has64BitDPP() && combineDPPMov(MI)) {
+        if (ST->has64BitDPP() && combineDPPMov(MI)) {
           Changed = true;
           ++NumDPPMovsCombined;
         } else {
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index fc45f7ee11dc0e8816fd7bc6c66e04e83f42de28..415b1cb4854b324eb9a4e0130f948bc91054a5cf 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -41,24 +41,16 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   using AMDGPUSubtarget::getMaxWavesPerEU;
 
 public:
-  enum TrapHandlerAbi {
-    TrapHandlerAbiNone = 0,
-    TrapHandlerAbiHsa = 1
+  // Following 2 enums are documented at:
+  //   - https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
+  enum class TrapHandlerAbi {
+    NONE   = 0x00,
+    AMDHSA = 0x01,
   };
 
-  enum TrapID {
-    TrapIDHardwareReserved = 0,
-    TrapIDHSADebugTrap = 1,
-    TrapIDLLVMTrap = 2,
-    TrapIDLLVMDebugTrap = 3,
-    TrapIDDebugBreakpoint = 7,
-    TrapIDDebugReserved8 = 8,
-    TrapIDDebugReservedFE = 0xfe,
-    TrapIDDebugReservedFF = 0xff
-  };
-
-  enum TrapRegValues {
-    LLVMTrapHandlerRegValue = 1
+  enum class TrapID {
+    LLVMAMDHSATrap      = 0x02,
+    LLVMAMDHSADebugTrap = 0x03,
   };
 
 private:
@@ -150,6 +142,7 @@ protected:
   bool HasDot4Insts;
   bool HasDot5Insts;
   bool HasDot6Insts;
+  bool HasDot7Insts;
   bool HasMAIInsts;
   bool HasPkFmacF16Inst;
   bool HasAtomicFaddInsts;
@@ -254,6 +247,10 @@ public:
     return RegBankInfo.get();
   }
 
+  const AMDGPU::IsaInfo::AMDGPUTargetID &getTargetID() const {
+    return TargetID;
+  }
+
   // Nothing implemented, just prevent crashes on use.
   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
     return &TSInfo;
@@ -387,7 +384,12 @@ public:
   }
 
   TrapHandlerAbi getTrapHandlerAbi() const {
-    return isAmdHsaOS() ? TrapHandlerAbiHsa : TrapHandlerAbiNone;
+    return isAmdHsaOS() ? TrapHandlerAbi::AMDHSA : TrapHandlerAbi::NONE;
+  }
+
+  bool supportsGetDoorbellID() const {
+    // The S_GETREG DOORBELL_ID is supported by all GFX9 onward targets.
+    return getGeneration() >= GFX9;
   }
 
   /// True if the offset field of DS instructions works as expected. On SI, the
@@ -687,6 +689,10 @@ public:
     return HasDot6Insts;
   }
 
+  bool hasDot7Insts() const {
+    return HasDot7Insts;
+  }
+
   bool hasMAIInsts() const {
     return HasMAIInsts;
   }
@@ -805,6 +811,9 @@ public:
     return GFX8Insts;
   }
 
+  /// \returns true if the subtarget has the v_permlanex16_b32 instruction.
+  bool hasPermLaneX16() const { return getGeneration() >= GFX10; }
+
   bool hasDPP() const {
     return HasDPP;
   }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index 54bcd6cafa612b2d975364034c145b9ab2edacfb..3533ff1cb312042084a9749d07789c35bc93d9e0 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -31,6 +31,20 @@ using namespace llvm::AMDGPU;
 // AMDGPUTargetStreamer
 //===----------------------------------------------------------------------===//
 
+static void convertIsaVersionV2(uint32_t &Major, uint32_t &Minor,
+                                uint32_t &Stepping, bool Sramecc, bool Xnack) {
+  if (Major == 9 && Minor == 0) {
+    switch (Stepping) {
+      case 0:
+      case 2:
+      case 4:
+      case 6:
+        if (Xnack)
+          Stepping++;
+    }
+  }
+}
+
 bool AMDGPUTargetStreamer::EmitHSAMetadataV2(StringRef HSAMetadataString) {
   HSAMD::Metadata HSAMetadata;
   if (HSAMD::fromString(HSAMetadataString, HSAMetadata))
@@ -182,8 +196,8 @@ void AMDGPUTargetAsmStreamer::finish() {
   getPALMetadata()->reset();
 }
 
-void AMDGPUTargetAsmStreamer::EmitDirectiveAMDGCNTarget(StringRef Target) {
-  OS << "\t.amdgcn_target \"" << Target << "\"\n";
+void AMDGPUTargetAsmStreamer::EmitDirectiveAMDGCNTarget() {
+  OS << "\t.amdgcn_target \"" << getTargetID()->toString() << "\"\n";
 }
 
 void AMDGPUTargetAsmStreamer::EmitDirectiveHSACodeObjectVersion(
@@ -193,15 +207,14 @@ void AMDGPUTargetAsmStreamer::EmitDirectiveHSACodeObjectVersion(
 }
 
 void
-AMDGPUTargetAsmStreamer::EmitDirectiveHSACodeObjectISA(uint32_t Major,
-                                                       uint32_t Minor,
-                                                       uint32_t Stepping,
-                                                       StringRef VendorName,
-                                                       StringRef ArchName) {
-  OS << "\t.hsa_code_object_isa " <<
-        Twine(Major) << "," << Twine(Minor) << "," << Twine(Stepping) <<
-        ",\"" << VendorName << "\",\"" << ArchName << "\"\n";
-
+AMDGPUTargetAsmStreamer::EmitDirectiveHSACodeObjectISAV2(uint32_t Major,
+                                                         uint32_t Minor,
+                                                         uint32_t Stepping,
+                                                         StringRef VendorName,
+                                                         StringRef ArchName) {
+  convertIsaVersionV2(Major, Minor, Stepping, TargetID->isSramEccOnOrAny(), TargetID->isXnackOnOrAny());
+  OS << "\t.hsa_code_object_isa " << Twine(Major) << "," << Twine(Minor) << ","
+     << Twine(Stepping) << ",\"" << VendorName << "\",\"" << ArchName << "\"\n";
 }
 
 void
@@ -227,8 +240,8 @@ void AMDGPUTargetAsmStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
      << Alignment.value() << '\n';
 }
 
-bool AMDGPUTargetAsmStreamer::EmitISAVersion(StringRef IsaVersionString) {
-  OS << "\t.amd_amdgpu_isa \"" << IsaVersionString << "\"\n";
+bool AMDGPUTargetAsmStreamer::EmitISAVersion() {
+  OS << "\t.amd_amdgpu_isa \"" << getTargetID()->toString() << "\"\n";
   return true;
 }
 
@@ -285,7 +298,7 @@ bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) {
 void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
     const MCSubtargetInfo &STI, StringRef KernelName,
     const amdhsa::kernel_descriptor_t &KD, uint64_t NextVGPR, uint64_t NextSGPR,
-    bool ReserveVCC, bool ReserveFlatScr, bool ReserveXNACK) {
+    bool ReserveVCC, bool ReserveFlatScr) {
   IsaVersion IVersion = getIsaVersion(STI.getCPU());
 
   OS << "\t.amdhsa_kernel " << KernelName << '\n';
@@ -298,6 +311,7 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
      << '\n';
   OS << "\t\t.amdhsa_private_segment_fixed_size "
      << KD.private_segment_fixed_size << '\n';
+  OS << "\t\t.amdhsa_kernarg_size " << KD.kernarg_size << '\n';
 
   PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_buffer", KD,
               kernel_code_properties,
@@ -358,8 +372,20 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
     OS << "\t\t.amdhsa_reserve_vcc " << ReserveVCC << '\n';
   if (IVersion.Major >= 7 && !ReserveFlatScr)
     OS << "\t\t.amdhsa_reserve_flat_scratch " << ReserveFlatScr << '\n';
-  if (IVersion.Major >= 8 && ReserveXNACK != hasXNACK(STI))
-    OS << "\t\t.amdhsa_reserve_xnack_mask " << ReserveXNACK << '\n';
+
+  if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(&STI)) {
+    switch (*HsaAbiVer) {
+    default:
+      break;
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V2:
+      break;
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
+      if (getTargetID()->isXnackSupported())
+        OS << "\t\t.amdhsa_reserve_xnack_mask " << getTargetID()->isXnackOnOrAny() << '\n';
+      break;
+    }
+  }
 
   PRINT_FIELD(OS, ".amdhsa_float_round_mode_32", KD,
               compute_pgm_rsrc1,
@@ -432,23 +458,7 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
 
 AMDGPUTargetELFStreamer::AMDGPUTargetELFStreamer(MCStreamer &S,
                                                  const MCSubtargetInfo &STI)
-    : AMDGPUTargetStreamer(S), Streamer(S), Os(STI.getTargetTriple().getOS()) {
-  MCAssembler &MCA = getStreamer().getAssembler();
-  unsigned EFlags = MCA.getELFHeaderEFlags();
-
-  EFlags &= ~ELF::EF_AMDGPU_MACH;
-  EFlags |= getElfMach(STI.getCPU());
-
-  EFlags &= ~ELF::EF_AMDGPU_XNACK;
-  if (AMDGPU::hasXNACK(STI))
-    EFlags |= ELF::EF_AMDGPU_XNACK;
-
-  EFlags &= ~ELF::EF_AMDGPU_SRAM_ECC;
-  if (AMDGPU::hasSRAMECC(STI))
-    EFlags |= ELF::EF_AMDGPU_SRAM_ECC;
-
-  MCA.setELFHeaderEFlags(EFlags);
-}
+    : AMDGPUTargetStreamer(S), STI(STI), Streamer(S) {}
 
 MCELFStreamer &AMDGPUTargetELFStreamer::getStreamer() {
   return static_cast<MCELFStreamer &>(Streamer);
@@ -458,6 +468,9 @@ MCELFStreamer &AMDGPUTargetELFStreamer::getStreamer() {
 // We use it for emitting the accumulated PAL metadata as a .note record.
 // The PAL metadata is reset after it is emitted.
 void AMDGPUTargetELFStreamer::finish() {
+  MCAssembler &MCA = getStreamer().getAssembler();
+  MCA.setELFHeaderEFlags(getEFlags());
+
   std::string Blob;
   const char *Vendor = getPALMetadata()->getVendor();
   unsigned Type = getPALMetadata()->getType();
@@ -483,7 +496,7 @@ void AMDGPUTargetELFStreamer::EmitNote(
   unsigned NoteFlags = 0;
   // TODO Apparently, this is currently needed for OpenCL as mentioned in
   // https://reviews.llvm.org/D74995
-  if (Os == Triple::AMDHSA)
+  if (STI.getTargetTriple().getOS() == Triple::AMDHSA)
     NoteFlags = ELF::SHF_ALLOC;
 
   S.PushSection();
@@ -499,24 +512,150 @@ void AMDGPUTargetELFStreamer::EmitNote(
   S.PopSection();
 }
 
-void AMDGPUTargetELFStreamer::EmitDirectiveAMDGCNTarget(StringRef Target) {}
+unsigned AMDGPUTargetELFStreamer::getEFlags() {
+  switch (STI.getTargetTriple().getArch()) {
+  default:
+    llvm_unreachable("Unsupported Arch");
+  case Triple::r600:
+    return getEFlagsR600();
+  case Triple::amdgcn:
+    return getEFlagsAMDGCN();
+  }
+}
+
+unsigned AMDGPUTargetELFStreamer::getEFlagsR600() {
+  assert(STI.getTargetTriple().getArch() == Triple::r600);
+
+  return getElfMach(STI.getCPU());
+}
+
+unsigned AMDGPUTargetELFStreamer::getEFlagsAMDGCN() {
+  assert(STI.getTargetTriple().getArch() == Triple::amdgcn);
+
+  switch (STI.getTargetTriple().getOS()) {
+  default:
+    // TODO: Why are some tests have "mingw" listed as OS?
+    // llvm_unreachable("Unsupported OS");
+  case Triple::UnknownOS:
+    return getEFlagsUnknownOS();
+  case Triple::AMDHSA:
+    return getEFlagsAMDHSA();
+  case Triple::AMDPAL:
+    return getEFlagsAMDPAL();
+  case Triple::Mesa3D:
+    return getEFlagsMesa3D();
+  }
+}
+
+unsigned AMDGPUTargetELFStreamer::getEFlagsUnknownOS() {
+  // TODO: Why are some tests have "mingw" listed as OS?
+  // assert(STI.getTargetTriple().getOS() == Triple::UnknownOS);
+
+  return getEFlagsV3();
+}
+
+unsigned AMDGPUTargetELFStreamer::getEFlagsAMDHSA() {
+  assert(STI.getTargetTriple().getOS() == Triple::AMDHSA);
+
+  if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(&STI)) {
+    switch (*HsaAbiVer) {
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V2:
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
+      return getEFlagsV3();
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
+      return getEFlagsV4();
+    }
+  }
+
+  llvm_unreachable("HSA OS ABI Version identification must be defined");
+}
+
+unsigned AMDGPUTargetELFStreamer::getEFlagsAMDPAL() {
+  assert(STI.getTargetTriple().getOS() == Triple::AMDPAL);
+
+  return getEFlagsV3();
+}
+
+unsigned AMDGPUTargetELFStreamer::getEFlagsMesa3D() {
+  assert(STI.getTargetTriple().getOS() == Triple::Mesa3D);
+
+  return getEFlagsV3();
+}
+
+unsigned AMDGPUTargetELFStreamer::getEFlagsV3() {
+  unsigned EFlagsV3 = 0;
+
+  // mach.
+  EFlagsV3 |= getElfMach(STI.getCPU());
+
+  // xnack.
+  if (getTargetID()->isXnackOnOrAny())
+    EFlagsV3 |= ELF::EF_AMDGPU_FEATURE_XNACK_V3;
+  // sramecc.
+  if (getTargetID()->isSramEccOnOrAny())
+    EFlagsV3 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_V3;
+
+  return EFlagsV3;
+}
+
+unsigned AMDGPUTargetELFStreamer::getEFlagsV4() {
+  unsigned EFlagsV4 = 0;
+
+  // mach.
+  EFlagsV4 |= getElfMach(STI.getCPU());
+
+  // xnack.
+  switch (getTargetID()->getXnackSetting()) {
+  case AMDGPU::IsaInfo::TargetIDSetting::Unsupported:
+    EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_UNSUPPORTED_V4;
+    break;
+  case AMDGPU::IsaInfo::TargetIDSetting::Any:
+    EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_ANY_V4;
+    break;
+  case AMDGPU::IsaInfo::TargetIDSetting::Off:
+    EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_OFF_V4;
+    break;
+  case AMDGPU::IsaInfo::TargetIDSetting::On:
+    EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_ON_V4;
+    break;
+  }
+  // sramecc.
+  switch (getTargetID()->getSramEccSetting()) {
+  case AMDGPU::IsaInfo::TargetIDSetting::Unsupported:
+    EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_UNSUPPORTED_V4;
+    break;
+  case AMDGPU::IsaInfo::TargetIDSetting::Any:
+    EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_ANY_V4;
+    break;
+  case AMDGPU::IsaInfo::TargetIDSetting::Off:
+    EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_OFF_V4;
+    break;
+  case AMDGPU::IsaInfo::TargetIDSetting::On:
+    EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_ON_V4;
+    break;
+  }
+
+  return EFlagsV4;
+}
+
+void AMDGPUTargetELFStreamer::EmitDirectiveAMDGCNTarget() {}
 
 void AMDGPUTargetELFStreamer::EmitDirectiveHSACodeObjectVersion(
     uint32_t Major, uint32_t Minor) {
 
   EmitNote(ElfNote::NoteNameV2, MCConstantExpr::create(8, getContext()),
-           ElfNote::NT_AMDGPU_HSA_CODE_OBJECT_VERSION, [&](MCELFStreamer &OS) {
+           ELF::NT_AMD_HSA_CODE_OBJECT_VERSION, [&](MCELFStreamer &OS) {
              OS.emitInt32(Major);
              OS.emitInt32(Minor);
            });
 }
 
 void
-AMDGPUTargetELFStreamer::EmitDirectiveHSACodeObjectISA(uint32_t Major,
-                                                       uint32_t Minor,
-                                                       uint32_t Stepping,
-                                                       StringRef VendorName,
-                                                       StringRef ArchName) {
+AMDGPUTargetELFStreamer::EmitDirectiveHSACodeObjectISAV2(uint32_t Major,
+                                                         uint32_t Minor,
+                                                         uint32_t Stepping,
+                                                         StringRef VendorName,
+                                                         StringRef ArchName) {
   uint16_t VendorNameSize = VendorName.size() + 1;
   uint16_t ArchNameSize = ArchName.size() + 1;
 
@@ -524,8 +663,9 @@ AMDGPUTargetELFStreamer::EmitDirectiveHSACodeObjectISA(uint32_t Major,
     sizeof(Major) + sizeof(Minor) + sizeof(Stepping) +
     VendorNameSize + ArchNameSize;
 
+  convertIsaVersionV2(Major, Minor, Stepping, TargetID->isSramEccOnOrAny(), TargetID->isXnackOnOrAny());
   EmitNote(ElfNote::NoteNameV2, MCConstantExpr::create(DescSZ, getContext()),
-           ElfNote::NT_AMDGPU_HSA_ISA, [&](MCELFStreamer &OS) {
+           ELF::NT_AMD_HSA_ISA_VERSION, [&](MCELFStreamer &OS) {
              OS.emitInt16(VendorNameSize);
              OS.emitInt16(ArchNameSize);
              OS.emitInt32(Major);
@@ -573,7 +713,7 @@ void AMDGPUTargetELFStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
   SymbolELF->setSize(MCConstantExpr::create(Size, getContext()));
 }
 
-bool AMDGPUTargetELFStreamer::EmitISAVersion(StringRef IsaVersionString) {
+bool AMDGPUTargetELFStreamer::EmitISAVersion() {
   // Create two labels to mark the beginning and end of the desc field
   // and a MCExpr to calculate the size of the desc field.
   auto &Context = getContext();
@@ -583,10 +723,10 @@ bool AMDGPUTargetELFStreamer::EmitISAVersion(StringRef IsaVersionString) {
     MCSymbolRefExpr::create(DescEnd, Context),
     MCSymbolRefExpr::create(DescBegin, Context), Context);
 
-  EmitNote(ElfNote::NoteNameV2, DescSZ, ELF::NT_AMD_AMDGPU_ISA,
+  EmitNote(ElfNote::NoteNameV2, DescSZ, ELF::NT_AMD_HSA_ISA_NAME,
            [&](MCELFStreamer &OS) {
              OS.emitLabel(DescBegin);
-             OS.emitBytes(IsaVersionString);
+             OS.emitBytes(getTargetID()->toString());
              OS.emitLabel(DescEnd);
            });
   return true;
@@ -634,7 +774,7 @@ bool AMDGPUTargetELFStreamer::EmitHSAMetadata(
     MCSymbolRefExpr::create(DescEnd, Context),
     MCSymbolRefExpr::create(DescBegin, Context), Context);
 
-  EmitNote(ElfNote::NoteNameV2, DescSZ, ELF::NT_AMD_AMDGPU_HSA_METADATA,
+  EmitNote(ElfNote::NoteNameV2, DescSZ, ELF::NT_AMD_HSA_METADATA,
            [&](MCELFStreamer &OS) {
              OS.emitLabel(DescBegin);
              OS.emitBytes(HSAMetadataString);
@@ -672,8 +812,7 @@ bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) {
 void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor(
     const MCSubtargetInfo &STI, StringRef KernelName,
     const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR,
-    uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr,
-    bool ReserveXNACK) {
+    uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) {
   auto &Streamer = getStreamer();
   auto &Context = Streamer.getContext();
 
@@ -700,8 +839,11 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor(
   Streamer.emitLabel(KernelDescriptorSymbol);
   Streamer.emitInt32(KernelDescriptor.group_segment_fixed_size);
   Streamer.emitInt32(KernelDescriptor.private_segment_fixed_size);
+  Streamer.emitInt32(KernelDescriptor.kernarg_size);
+
   for (uint8_t Res : KernelDescriptor.reserved0)
     Streamer.emitInt8(Res);
+
   // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The
   // expression being created is:
   //   (start of kernel code) - (start of kernel descriptor)
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
index a29fc94ca523e0d02002fd91019fb8b3c074b110..cef34a5e5a5981fd939e1c6e40d4ca4eacd6cfce 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUTARGETSTREAMER_H
 #define LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUTARGETSTREAMER_H
 
+#include "Utils/AMDGPUBaseInfo.h"
 #include "Utils/AMDGPUPALMetadata.h"
 #include "llvm/MC/MCStreamer.h"
 
@@ -39,6 +40,9 @@ class AMDGPUTargetStreamer : public MCTargetStreamer {
   AMDGPUPALMetadata PALMetadata;
 
 protected:
+  // TODO: Move HSAMetadataStream to AMDGPUTargetStreamer.
+  Optional<AMDGPU::IsaInfo::AMDGPUTargetID> TargetID;
+
   MCContext &getContext() const { return Streamer.getContext(); }
 
 public:
@@ -46,15 +50,15 @@ public:
 
   AMDGPUPALMetadata *getPALMetadata() { return &PALMetadata; }
 
-  virtual void EmitDirectiveAMDGCNTarget(StringRef Target) = 0;
+  virtual void EmitDirectiveAMDGCNTarget() = 0;
 
   virtual void EmitDirectiveHSACodeObjectVersion(uint32_t Major,
                                                  uint32_t Minor) = 0;
 
-  virtual void EmitDirectiveHSACodeObjectISA(uint32_t Major, uint32_t Minor,
-                                             uint32_t Stepping,
-                                             StringRef VendorName,
-                                             StringRef ArchName) = 0;
+  virtual void EmitDirectiveHSACodeObjectISAV2(uint32_t Major, uint32_t Minor,
+                                               uint32_t Stepping,
+                                               StringRef VendorName,
+                                               StringRef ArchName) = 0;
 
   virtual void EmitAMDKernelCodeT(const amd_kernel_code_t &Header) = 0;
 
@@ -64,7 +68,7 @@ public:
                              Align Alignment) = 0;
 
   /// \returns True on success, false on failure.
-  virtual bool EmitISAVersion(StringRef IsaVersionString) = 0;
+  virtual bool EmitISAVersion() = 0;
 
   /// \returns True on success, false on failure.
   virtual bool EmitHSAMetadataV2(StringRef HSAMetadataString);
@@ -90,11 +94,27 @@ public:
   virtual void EmitAmdhsaKernelDescriptor(
       const MCSubtargetInfo &STI, StringRef KernelName,
       const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR,
-      uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr,
-      bool ReserveXNACK) = 0;
+      uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) = 0;
 
   static StringRef getArchNameFromElfMach(unsigned ElfMach);
   static unsigned getElfMach(StringRef GPU);
+
+  const Optional<AMDGPU::IsaInfo::AMDGPUTargetID> &getTargetID() const {
+    return TargetID;
+  }
+  Optional<AMDGPU::IsaInfo::AMDGPUTargetID> &getTargetID() {
+    return TargetID;
+  }
+  void initializeTargetID(const MCSubtargetInfo &STI) {
+    assert(TargetID == None && "TargetID can only be initialized once");
+    TargetID.emplace(STI);
+  }
+  void initializeTargetID(const MCSubtargetInfo &STI, StringRef FeatureString) {
+    initializeTargetID(STI);
+
+    assert(getTargetID() != None && "TargetID is None");
+    getTargetID()->setTargetIDFromFeaturesString(FeatureString);
+  }
 };
 
 class AMDGPUTargetAsmStreamer final : public AMDGPUTargetStreamer {
@@ -104,14 +124,14 @@ public:
 
   void finish() override;
 
-  void EmitDirectiveAMDGCNTarget(StringRef Target) override;
+  void EmitDirectiveAMDGCNTarget() override;
 
   void EmitDirectiveHSACodeObjectVersion(uint32_t Major,
                                          uint32_t Minor) override;
 
-  void EmitDirectiveHSACodeObjectISA(uint32_t Major, uint32_t Minor,
-                                     uint32_t Stepping, StringRef VendorName,
-                                     StringRef ArchName) override;
+  void EmitDirectiveHSACodeObjectISAV2(uint32_t Major, uint32_t Minor,
+                                       uint32_t Stepping, StringRef VendorName,
+                                       StringRef ArchName) override;
 
   void EmitAMDKernelCodeT(const amd_kernel_code_t &Header) override;
 
@@ -120,7 +140,7 @@ public:
   void emitAMDGPULDS(MCSymbol *Sym, unsigned Size, Align Alignment) override;
 
   /// \returns True on success, false on failure.
-  bool EmitISAVersion(StringRef IsaVersionString) override;
+  bool EmitISAVersion() override;
 
   /// \returns True on success, false on failure.
   bool EmitHSAMetadata(msgpack::Document &HSAMetadata, bool Strict) override;
@@ -134,17 +154,29 @@ public:
   void EmitAmdhsaKernelDescriptor(
       const MCSubtargetInfo &STI, StringRef KernelName,
       const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR,
-      uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr,
-      bool ReserveXNACK) override;
+      uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override;
 };
 
 class AMDGPUTargetELFStreamer final : public AMDGPUTargetStreamer {
+  const MCSubtargetInfo &STI;
   MCStreamer &Streamer;
-  Triple::OSType Os;
 
   void EmitNote(StringRef Name, const MCExpr *DescSize, unsigned NoteType,
                 function_ref<void(MCELFStreamer &)> EmitDesc);
 
+  unsigned getEFlags();
+
+  unsigned getEFlagsR600();
+  unsigned getEFlagsAMDGCN();
+
+  unsigned getEFlagsUnknownOS();
+  unsigned getEFlagsAMDHSA();
+  unsigned getEFlagsAMDPAL();
+  unsigned getEFlagsMesa3D();
+
+  unsigned getEFlagsV3();
+  unsigned getEFlagsV4();
+
 public:
   AMDGPUTargetELFStreamer(MCStreamer &S, const MCSubtargetInfo &STI);
 
@@ -152,14 +184,14 @@ public:
 
   void finish() override;
 
-  void EmitDirectiveAMDGCNTarget(StringRef Target) override;
+  void EmitDirectiveAMDGCNTarget() override;
 
   void EmitDirectiveHSACodeObjectVersion(uint32_t Major,
                                          uint32_t Minor) override;
 
-  void EmitDirectiveHSACodeObjectISA(uint32_t Major, uint32_t Minor,
-                                     uint32_t Stepping, StringRef VendorName,
-                                     StringRef ArchName) override;
+  void EmitDirectiveHSACodeObjectISAV2(uint32_t Major, uint32_t Minor,
+                                       uint32_t Stepping, StringRef VendorName,
+                                       StringRef ArchName) override;
 
   void EmitAMDKernelCodeT(const amd_kernel_code_t &Header) override;
 
@@ -168,7 +200,7 @@ public:
   void emitAMDGPULDS(MCSymbol *Sym, unsigned Size, Align Alignment) override;
 
   /// \returns True on success, false on failure.
-  bool EmitISAVersion(StringRef IsaVersionString) override;
+  bool EmitISAVersion() override;
 
   /// \returns True on success, false on failure.
   bool EmitHSAMetadata(msgpack::Document &HSAMetadata, bool Strict) override;
@@ -182,8 +214,7 @@ public:
   void EmitAmdhsaKernelDescriptor(
       const MCSubtargetInfo &STI, StringRef KernelName,
       const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR,
-      uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr,
-      bool ReserveXNACK) override;
+      uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override;
 };
 
 }
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index 9ef87fec0ca11f697848ed227a79a293292edf66..b6abfdf53efc349e4ba158a5decb20ae1d7491cf 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -671,6 +671,7 @@ enum SDWA9EncValues : unsigned {
 
 namespace DPP {
 
+// clang-format off
 enum DppCtrl : unsigned {
   QUAD_PERM_FIRST   = 0,
   QUAD_PERM_ID      = 0xE4, // identity permutation
@@ -707,12 +708,15 @@ enum DppCtrl : unsigned {
   DPP_UNUSED8_LAST  = 0x14F,
   ROW_NEWBCAST_FIRST= 0x150,
   ROW_NEWBCAST_LAST = 0x15F,
+  ROW_SHARE0        = 0x150,
   ROW_SHARE_FIRST   = 0x150,
   ROW_SHARE_LAST    = 0x15F,
+  ROW_XMASK0        = 0x160,
   ROW_XMASK_FIRST   = 0x160,
   ROW_XMASK_LAST    = 0x16F,
   DPP_LAST          = ROW_XMASK_LAST
 };
+// clang-format on
 
 enum DppFiMode {
   DPP_FI_0  = 0,
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index c9490da5efbdd6c7da149f0a89f86e825d8cf7a2..e7588b7161504508942058b2b2f12a9525e3b88f 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -838,6 +838,13 @@ static Register buildScratchExecCopy(LivePhysRegs &LiveRegs,
   return ScratchExecCopy;
 }
 
+// A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR.
+// Otherwise we are spilling to memory.
+static bool spilledToMemory(const MachineFunction &MF, int SaveIndex) {
+  const MachineFrameInfo &MFI = MF.getFrameInfo();
+  return MFI.getStackID(SaveIndex) != TargetStackID::SGPRSpill;
+}
+
 void SIFrameLowering::emitPrologue(MachineFunction &MF,
                                    MachineBasicBlock &MBB) const {
   SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
@@ -869,23 +876,8 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF,
   // turn on all lanes before doing the spill to memory.
   Register ScratchExecCopy;
 
-  bool HasFPSaveIndex = FuncInfo->FramePointerSaveIndex.hasValue();
-  bool SpillFPToMemory = false;
-  // A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR.
-  // Otherwise we are spilling the FP to memory.
-  if (HasFPSaveIndex) {
-    SpillFPToMemory = MFI.getStackID(*FuncInfo->FramePointerSaveIndex) !=
-                      TargetStackID::SGPRSpill;
-  }
-
-  bool HasBPSaveIndex = FuncInfo->BasePointerSaveIndex.hasValue();
-  bool SpillBPToMemory = false;
-  // A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR.
-  // Otherwise we are spilling the BP to memory.
-  if (HasBPSaveIndex) {
-    SpillBPToMemory = MFI.getStackID(*FuncInfo->BasePointerSaveIndex) !=
-                      TargetStackID::SGPRSpill;
-  }
+  Optional<int> FPSaveIndex = FuncInfo->FramePointerSaveIndex;
+  Optional<int> BPSaveIndex = FuncInfo->BasePointerSaveIndex;
 
   for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg
          : FuncInfo->getSGPRSpillVGPRs()) {
@@ -901,8 +893,9 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF,
                      Reg.FI.getValue());
   }
 
-  if (HasFPSaveIndex && SpillFPToMemory) {
-    assert(!MFI.isDeadObjectIndex(FuncInfo->FramePointerSaveIndex.getValue()));
+  if (FPSaveIndex && spilledToMemory(MF, *FPSaveIndex)) {
+    const int FramePtrFI = *FPSaveIndex;
+    assert(!MFI.isDeadObjectIndex(FramePtrFI));
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
@@ -916,12 +909,12 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF,
         .addReg(FramePtrReg);
 
     buildPrologSpill(ST, LiveRegs, MBB, MBBI, TII, TmpVGPR,
-                     FuncInfo->getScratchRSrcReg(), StackPtrReg,
-                     FuncInfo->FramePointerSaveIndex.getValue());
+                     FuncInfo->getScratchRSrcReg(), StackPtrReg, FramePtrFI);
   }
 
-  if (HasBPSaveIndex && SpillBPToMemory) {
-    assert(!MFI.isDeadObjectIndex(*FuncInfo->BasePointerSaveIndex));
+  if (BPSaveIndex && spilledToMemory(MF, *BPSaveIndex)) {
+    const int BasePtrFI = *BPSaveIndex;
+    assert(!MFI.isDeadObjectIndex(BasePtrFI));
 
     if (!ScratchExecCopy)
       ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true);
@@ -935,8 +928,7 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF,
         .addReg(BasePtrReg);
 
     buildPrologSpill(ST, LiveRegs, MBB, MBBI, TII, TmpVGPR,
-                     FuncInfo->getScratchRSrcReg(), StackPtrReg,
-                     *FuncInfo->BasePointerSaveIndex);
+                     FuncInfo->getScratchRSrcReg(), StackPtrReg, BasePtrFI);
   }
 
   if (ScratchExecCopy) {
@@ -949,13 +941,13 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF,
   }
 
   // In this case, spill the FP to a reserved VGPR.
-  if (HasFPSaveIndex && !SpillFPToMemory) {
-    const int FI = FuncInfo->FramePointerSaveIndex.getValue();
-    assert(!MFI.isDeadObjectIndex(FI));
+  if (FPSaveIndex && !spilledToMemory(MF, *FPSaveIndex)) {
+    const int FramePtrFI = *FPSaveIndex;
+    assert(!MFI.isDeadObjectIndex(FramePtrFI));
 
-    assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
+    assert(MFI.getStackID(FramePtrFI) == TargetStackID::SGPRSpill);
     ArrayRef<SIMachineFunctionInfo::SpilledReg> Spill =
-        FuncInfo->getSGPRToVGPRSpills(FI);
+        FuncInfo->getSGPRToVGPRSpills(FramePtrFI);
     assert(Spill.size() == 1);
 
     // Save FP before setting it up.
@@ -967,8 +959,8 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF,
   }
 
   // In this case, spill the BP to a reserved VGPR.
-  if (HasBPSaveIndex && !SpillBPToMemory) {
-    const int BasePtrFI = *FuncInfo->BasePointerSaveIndex;
+  if (BPSaveIndex && !spilledToMemory(MF, *BPSaveIndex)) {
+    const int BasePtrFI = *BPSaveIndex;
     assert(!MFI.isDeadObjectIndex(BasePtrFI));
 
     assert(MFI.getStackID(BasePtrFI) == TargetStackID::SGPRSpill);
@@ -1107,19 +1099,8 @@ void SIFrameLowering::emitEpilogue(MachineFunction &MF,
   const Register BasePtrReg =
       TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register();
 
-  bool HasFPSaveIndex = FuncInfo->FramePointerSaveIndex.hasValue();
-  bool SpillFPToMemory = false;
-  if (HasFPSaveIndex) {
-    SpillFPToMemory = MFI.getStackID(*FuncInfo->FramePointerSaveIndex) !=
-                      TargetStackID::SGPRSpill;
-  }
-
-  bool HasBPSaveIndex = FuncInfo->BasePointerSaveIndex.hasValue();
-  bool SpillBPToMemory = false;
-  if (HasBPSaveIndex) {
-    SpillBPToMemory = MFI.getStackID(*FuncInfo->BasePointerSaveIndex) !=
-                      TargetStackID::SGPRSpill;
-  }
+  Optional<int> FPSaveIndex = FuncInfo->FramePointerSaveIndex;
+  Optional<int> BPSaveIndex = FuncInfo->BasePointerSaveIndex;
 
   if (RoundedSize != 0 && hasFP(MF)) {
     BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_SUB_U32), StackPtrReg)
@@ -1141,10 +1122,10 @@ void SIFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 
   Register ScratchExecCopy;
-  if (HasFPSaveIndex) {
-    const int FI = FuncInfo->FramePointerSaveIndex.getValue();
-    assert(!MFI.isDeadObjectIndex(FI));
-    if (SpillFPToMemory) {
+  if (FPSaveIndex) {
+    const int FramePtrFI = *FPSaveIndex;
+    assert(!MFI.isDeadObjectIndex(FramePtrFI));
+    if (spilledToMemory(MF, FramePtrFI)) {
       if (!ScratchExecCopy)
         ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false);
 
@@ -1153,14 +1134,14 @@ void SIFrameLowering::emitEpilogue(MachineFunction &MF,
       if (!TempVGPR)
         report_fatal_error("failed to find free scratch register");
       buildEpilogReload(ST, LiveRegs, MBB, MBBI, TII, TempVGPR,
-                        FuncInfo->getScratchRSrcReg(), StackPtrReg, FI);
+                        FuncInfo->getScratchRSrcReg(), StackPtrReg, FramePtrFI);
       BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), FramePtrReg)
           .addReg(TempVGPR, RegState::Kill);
     } else {
       // Reload from VGPR spill.
-      assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
+      assert(MFI.getStackID(FramePtrFI) == TargetStackID::SGPRSpill);
       ArrayRef<SIMachineFunctionInfo::SpilledReg> Spill =
-          FuncInfo->getSGPRToVGPRSpills(FI);
+          FuncInfo->getSGPRToVGPRSpills(FramePtrFI);
       assert(Spill.size() == 1);
       BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READLANE_B32), FramePtrReg)
           .addReg(Spill[0].VGPR)
@@ -1168,10 +1149,10 @@ void SIFrameLowering::emitEpilogue(MachineFunction &MF,
     }
   }
 
-  if (HasBPSaveIndex) {
-    const int BasePtrFI = *FuncInfo->BasePointerSaveIndex;
+  if (BPSaveIndex) {
+    const int BasePtrFI = *BPSaveIndex;
     assert(!MFI.isDeadObjectIndex(BasePtrFI));
-    if (SpillBPToMemory) {
+    if (spilledToMemory(MF, BasePtrFI)) {
       if (!ScratchExecCopy)
         ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false);
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5a3eaa6e2c367547369e2aed9732a35be25a0c71..02fda2d2ddd3a4382c242a9d9c972ce6101a125c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19,6 +19,7 @@
 #include "SIRegisterInfo.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/LegacyDivergenceAnalysis.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
@@ -2829,6 +2830,7 @@ static bool canGuaranteeTCO(CallingConv::ID CC) {
 static bool mayTailCallThisCC(CallingConv::ID CC) {
   switch (CC) {
   case CallingConv::C:
+  case CallingConv::AMDGPU_Gfx:
     return true;
   default:
     return canGuaranteeTCO(CC);
@@ -5156,12 +5158,35 @@ SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
 }
 
 SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
+  if (!Subtarget->isTrapHandlerEnabled() ||
+      Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
+    return lowerTrapEndpgm(Op, DAG);
+
+  if (Optional<uint8_t> HsaAbiVer = AMDGPU::getHsaAbiVersion(Subtarget)) {
+    switch (*HsaAbiVer) {
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V2:
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
+      return lowerTrapHsaQueuePtr(Op, DAG);
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
+      return Subtarget->supportsGetDoorbellID() ?
+          lowerTrapHsa(Op, DAG) : lowerTrapHsaQueuePtr(Op, DAG);
+    }
+  }
+
+  llvm_unreachable("Unknown trap handler");
+}
+
+SDValue SITargetLowering::lowerTrapEndpgm(
+    SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue Chain = Op.getOperand(0);
+  return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain);
+}
 
-  if (Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
-      !Subtarget->isTrapHandlerEnabled())
-    return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain);
+SDValue SITargetLowering::lowerTrapHsaQueuePtr(
+    SDValue Op, SelectionDAG &DAG) const {
+  SDLoc SL(Op);
+  SDValue Chain = Op.getOperand(0);
 
   MachineFunction &MF = DAG.getMachineFunction();
   SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
@@ -5172,22 +5197,37 @@ SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
   SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64);
   SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01,
                                    QueuePtr, SDValue());
+
+  uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
   SDValue Ops[] = {
     ToReg,
-    DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMTrap, SL, MVT::i16),
+    DAG.getTargetConstant(TrapID, SL, MVT::i16),
     SGPR01,
     ToReg.getValue(1)
   };
   return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
 }
 
+SDValue SITargetLowering::lowerTrapHsa(
+    SDValue Op, SelectionDAG &DAG) const {
+  SDLoc SL(Op);
+  SDValue Chain = Op.getOperand(0);
+
+  uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
+  SDValue Ops[] = {
+    Chain,
+    DAG.getTargetConstant(TrapID, SL, MVT::i16)
+  };
+  return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
+}
+
 SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue Chain = Op.getOperand(0);
   MachineFunction &MF = DAG.getMachineFunction();
 
-  if (Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
-      !Subtarget->isTrapHandlerEnabled()) {
+  if (!Subtarget->isTrapHandlerEnabled() ||
+      Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
     DiagnosticInfoUnsupported NoTrap(MF.getFunction(),
                                      "debugtrap handler not supported",
                                      Op.getDebugLoc(),
@@ -5197,9 +5237,10 @@ SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
     return Chain;
   }
 
+  uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap);
   SDValue Ops[] = {
     Chain,
-    DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMDebugTrap, SL, MVT::i16)
+    DAG.getTargetConstant(TrapID, SL, MVT::i16)
   };
   return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
 }
@@ -9772,15 +9813,19 @@ SDValue SITargetLowering::performIntMed3ImmCombine(
   }
 
   // If there isn't a 16-bit med3 operation, convert to 32-bit.
-  MVT NVT = MVT::i32;
-  unsigned ExtOp = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
+  if (VT == MVT::i16) {
+    MVT NVT = MVT::i32;
+    unsigned ExtOp = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
 
-  SDValue Tmp1 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(0));
-  SDValue Tmp2 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(1));
-  SDValue Tmp3 = DAG.getNode(ExtOp, SL, NVT, Op1);
+    SDValue Tmp1 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(0));
+    SDValue Tmp2 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(1));
+    SDValue Tmp3 = DAG.getNode(ExtOp, SL, NVT, Op1);
 
-  SDValue Med3 = DAG.getNode(Med3Opc, SL, NVT, Tmp1, Tmp2, Tmp3);
-  return DAG.getNode(ISD::TRUNCATE, SL, VT, Med3);
+    SDValue Med3 = DAG.getNode(Med3Opc, SL, NVT, Tmp1, Tmp2, Tmp3);
+    return DAG.getNode(ISD::TRUNCATE, SL, VT, Med3);
+  }
+
+  return SDValue();
 }
 
 static ConstantFPSDNode *getSplatConstantFP(SDValue Op) {
@@ -10486,7 +10531,7 @@ SDValue SITargetLowering::performFMACombine(SDNode *N,
   EVT VT = N->getValueType(0);
   SDLoc SL(N);
 
-  if (!Subtarget->hasDot2Insts() || VT != MVT::f32)
+  if (!Subtarget->hasDot7Insts() || VT != MVT::f32)
     return SDValue();
 
   // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 334cd6dd096b04bb532dcf4266efb92813181ca0..d06683572360380e1fabe3b8a28f261afec10319 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -144,7 +144,11 @@ private:
   SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
+
   SDValue lowerTRAP(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerTrapEndpgm(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerTrapHsaQueuePtr(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerTrapHsa(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const;
 
   SDNode *adjustWritemask(MachineSDNode *&N, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index af51434514dfbb7591f3e7ee115df9103077947d..ba8ef5ec241c9443c2a8e4e867118be5492e144f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1116,7 +1116,9 @@ def CPol_0 : NamedOperandU32Default0<"CPol", NamedMatchClass<"CPol">>;
 def CPol_GLC1 : NamedOperandU32Default1<"CPol", NamedMatchClass<"CPol">>;
 
 def TFE : NamedOperandBit<"TFE", NamedMatchClass<"TFE">>;
+def TFE_0 : NamedOperandBit_0<"TFE", NamedMatchClass<"TFE">>;
 def SWZ : NamedOperandBit<"SWZ", NamedMatchClass<"SWZ">>;
+def SWZ_0 : NamedOperandBit_0<"SWZ", NamedMatchClass<"SWZ">>;
 def UNorm : NamedOperandBit<"UNorm", NamedMatchClass<"UNorm">>;
 def DA : NamedOperandBit<"DA", NamedMatchClass<"DA">>;
 def R128A16 : NamedOperandBit<"R128A16", NamedMatchClass<"R128A16">>;
@@ -1371,11 +1373,6 @@ def DSTOMOD {
   int NONE = 0;
 }
 
-def TRAPID{
-  int LLVM_TRAP = 2;
-  int LLVM_DEBUG_TRAP = 3;
-}
-
 def HWREG {
   int MODE = 1;
   int STATUS = 2;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 796d85fb8af655d6e2fb25653cf988303cfae71b..381f262abfeaa55477e7f124fe0a93066cfa12c4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -479,7 +479,7 @@ def SI_CALL : SPseudoInstSI <
 
 // Tail call handling pseudo
 def SI_TCRETURN : SPseudoInstSI <(outs),
-  (ins SSrc_b64:$src0, unknown:$callee, i32imm:$fpdiff),
+  (ins SReg_64:$src0, unknown:$callee, i32imm:$fpdiff),
   [(AMDGPUtc_return i64:$src0, tglobaladdr:$callee, i32:$fpdiff)]> {
   let Size = 4;
   let isCall = 1;
diff --git a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp b/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
similarity index 59%
rename from llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
rename to llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
index 439453e535486d5b39b71118222887c7cc0dcd6e..3fb96f15313d4895662af3850b58cb0355798e20 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
+++ b/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
@@ -1,4 +1,4 @@
-//===-- SIInsertSkips.cpp - Use predicates for control flow ---------------===//
+//===-- SILateBranchLowering.cpp - Final preparation of branches ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -14,28 +14,23 @@
 #include "AMDGPU.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
-#include "llvm/ADT/DepthFirstIterator.h"
+#include "SIMachineFunctionInfo.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/InitializePasses.h"
 
 using namespace llvm;
 
-#define DEBUG_TYPE "si-insert-skips"
+#define DEBUG_TYPE "si-late-branch-lowering"
 
 namespace {
 
-class SIInsertSkips : public MachineFunctionPass {
+class SILateBranchLowering : public MachineFunctionPass {
 private:
   const SIRegisterInfo *TRI = nullptr;
   const SIInstrInfo *TII = nullptr;
   MachineDominatorTree *MDT = nullptr;
 
-  MachineBasicBlock *EarlyExitBlock = nullptr;
-  bool EarlyExitClearsExec = false;
-
-  void ensureEarlyExitBlock(MachineBasicBlock &MBB, bool ClearExec);
-
-  void earlyTerm(MachineInstr &MI);
+  void earlyTerm(MachineInstr &MI, MachineBasicBlock *EarlyExitBlock);
 
 public:
   static char ID;
@@ -43,12 +38,12 @@ public:
   unsigned MovOpc;
   Register ExecReg;
 
-  SIInsertSkips() : MachineFunctionPass(ID) {}
+  SILateBranchLowering() : MachineFunctionPass(ID) {}
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   StringRef getPassName() const override {
-    return "SI insert s_cbranch_execz instructions";
+    return "SI Final Branch Preparation";
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -60,15 +55,15 @@ public:
 
 } // end anonymous namespace
 
-char SIInsertSkips::ID = 0;
+char SILateBranchLowering::ID = 0;
 
-INITIALIZE_PASS_BEGIN(SIInsertSkips, DEBUG_TYPE,
+INITIALIZE_PASS_BEGIN(SILateBranchLowering, DEBUG_TYPE,
                       "SI insert s_cbranch_execz instructions", false, false)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
-INITIALIZE_PASS_END(SIInsertSkips, DEBUG_TYPE,
+INITIALIZE_PASS_END(SILateBranchLowering, DEBUG_TYPE,
                     "SI insert s_cbranch_execz instructions", false, false)
 
-char &llvm::SIInsertSkipsPassID = SIInsertSkips::ID;
+char &llvm::SILateBranchLoweringPassID = SILateBranchLowering::ID;
 
 static void generateEndPgm(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator I, DebugLoc DL,
@@ -89,27 +84,6 @@ static void generateEndPgm(MachineBasicBlock &MBB,
   BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ENDPGM)).addImm(0);
 }
 
-void SIInsertSkips::ensureEarlyExitBlock(MachineBasicBlock &MBB,
-                                         bool ClearExec) {
-  MachineFunction *MF = MBB.getParent();
-  DebugLoc DL;
-
-  if (!EarlyExitBlock) {
-    EarlyExitBlock = MF->CreateMachineBasicBlock();
-    MF->insert(MF->end(), EarlyExitBlock);
-    generateEndPgm(*EarlyExitBlock, EarlyExitBlock->end(), DL, TII,
-                   MF->getFunction().getCallingConv() ==
-                       CallingConv::AMDGPU_PS);
-    EarlyExitClearsExec = false;
-  }
-
-  if (ClearExec && !EarlyExitClearsExec) {
-    auto ExitI = EarlyExitBlock->getFirstNonPHI();
-    BuildMI(*EarlyExitBlock, ExitI, DL, TII->get(MovOpc), ExecReg).addImm(0);
-    EarlyExitClearsExec = true;
-  }
-}
-
 static void splitBlock(MachineBasicBlock &MBB, MachineInstr &MI,
                        MachineDominatorTree *MDT) {
   MachineBasicBlock *SplitBB = MBB.splitAt(MI, /*UpdateLiveIns*/ true);
@@ -125,12 +99,11 @@ static void splitBlock(MachineBasicBlock &MBB, MachineInstr &MI,
   MDT->getBase().applyUpdates(DTUpdates);
 }
 
-void SIInsertSkips::earlyTerm(MachineInstr &MI) {
+void SILateBranchLowering::earlyTerm(MachineInstr &MI,
+                                     MachineBasicBlock *EarlyExitBlock) {
   MachineBasicBlock &MBB = *MI.getParent();
   const DebugLoc DL = MI.getDebugLoc();
 
-  ensureEarlyExitBlock(MBB, true);
-
   auto BranchMI = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC0))
                       .addMBB(EarlyExitBlock);
   auto Next = std::next(MI.getIterator());
@@ -142,7 +115,7 @@ void SIInsertSkips::earlyTerm(MachineInstr &MI) {
   MDT->getBase().insertEdge(&MBB, EarlyExitBlock);
 }
 
-bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
+bool SILateBranchLowering::runOnMachineFunction(MachineFunction &MF) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   TII = ST.getInstrInfo();
   TRI = &TII->getRegisterInfo();
@@ -152,6 +125,7 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
   ExecReg = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
 
   SmallVector<MachineInstr *, 4> EarlyTermInstrs;
+  SmallVector<MachineInstr *, 1> EpilogInstrs;
   bool MadeChange = false;
 
   for (MachineBasicBlock &MBB : MF) {
@@ -163,7 +137,7 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
       switch (MI.getOpcode()) {
       case AMDGPU::S_BRANCH:
         // Optimize out branches to the next block.
-        // FIXME: Shouldn't this be handled by BranchFolding?
+        // This only occurs in -O0 when BranchFolding is not executed.
         if (MBB.isLayoutSuccessor(MI.getOperand(0).getMBB())) {
           assert(&MI == &MBB.back());
           MI.eraseFromParent();
@@ -175,20 +149,73 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
         EarlyTermInstrs.push_back(&MI);
         break;
 
+      case AMDGPU::SI_RETURN_TO_EPILOG:
+        EpilogInstrs.push_back(&MI);
+        break;
+
       default:
         break;
       }
     }
   }
 
-  for (MachineInstr *Instr : EarlyTermInstrs) {
-    // Early termination in GS does nothing
-    if (MF.getFunction().getCallingConv() != CallingConv::AMDGPU_GS)
-      earlyTerm(*Instr);
-    Instr->eraseFromParent();
+  // Lower any early exit branches first
+  if (!EarlyTermInstrs.empty()) {
+    MachineBasicBlock *EarlyExitBlock = MF.CreateMachineBasicBlock();
+    DebugLoc DL;
+
+    MF.insert(MF.end(), EarlyExitBlock);
+    BuildMI(*EarlyExitBlock, EarlyExitBlock->end(), DL, TII->get(MovOpc),
+            ExecReg)
+        .addImm(0);
+    generateEndPgm(*EarlyExitBlock, EarlyExitBlock->end(), DL, TII,
+                   MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS);
+
+    for (MachineInstr *Instr : EarlyTermInstrs) {
+      // Early termination in GS does nothing
+      if (MF.getFunction().getCallingConv() != CallingConv::AMDGPU_GS)
+        earlyTerm(*Instr, EarlyExitBlock);
+      Instr->eraseFromParent();
+    }
+
+    EarlyTermInstrs.clear();
+    MadeChange = true;
+  }
+
+  // Now check return to epilog instructions occur at function end
+  if (!EpilogInstrs.empty()) {
+    MachineBasicBlock *EmptyMBBAtEnd = nullptr;
+    assert(!MF.getInfo<SIMachineFunctionInfo>()->returnsVoid());
+
+    // If there are multiple returns to epilog then all will
+    // become jumps to new empty end block.
+    if (EpilogInstrs.size() > 1) {
+      EmptyMBBAtEnd = MF.CreateMachineBasicBlock();
+      MF.insert(MF.end(), EmptyMBBAtEnd);
+    }
+
+    for (auto MI : EpilogInstrs) {
+      auto MBB = MI->getParent();
+      if (MBB == &MF.back() && MI == &MBB->back())
+        continue;
+
+      // SI_RETURN_TO_EPILOG is not the last instruction.
+      // Jump to empty block at function end.
+      if (!EmptyMBBAtEnd) {
+        EmptyMBBAtEnd = MF.CreateMachineBasicBlock();
+        MF.insert(MF.end(), EmptyMBBAtEnd);
+      }
+
+      MBB->addSuccessor(EmptyMBBAtEnd);
+      MDT->getBase().insertEdge(MBB, EmptyMBBAtEnd);
+      BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::S_BRANCH))
+          .addMBB(EmptyMBBAtEnd);
+      MI->eraseFromParent();
+      MadeChange = true;
+    }
+
+    EpilogInstrs.clear();
   }
-  EarlyTermInstrs.clear();
-  EarlyExitBlock = nullptr;
 
   return MadeChange;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index da144063c2b70bf3bebe4892efe8b06ed0c11720..50d18947aa737866e897a5104d86c178757f1053 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -1841,7 +1841,7 @@ bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
     for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) {
 
       // Unbundle instructions after the post-RA scheduler.
-      if (MI->isBundle()) {
+      if (MI->isBundle() && MI->mayLoadOrStore()) {
         MachineBasicBlock::instr_iterator II(MI->getIterator());
         for (MachineBasicBlock::instr_iterator I = ++II, E = MBB.instr_end();
              I != E && I->isBundledWithPred(); ++I) {
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
index 162e96655df248e7643fd3711504ac119f782992..5f89f382668331cdf75db8b83adf4ee4f83e493e 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
@@ -416,15 +416,20 @@ bool SIOptimizeExecMaskingPreRA::runOnMachineFunction(MachineFunction &MF) {
         continue;
 
       Register SavedExec = I->getOperand(0).getReg();
-      if (SavedExec.isVirtual() && MRI->hasOneNonDBGUse(SavedExec) &&
-          MRI->use_instr_nodbg_begin(SavedExec)->getParent() ==
-              I->getParent()) {
-        LLVM_DEBUG(dbgs() << "Redundant EXEC COPY: " << *I << '\n');
-        LIS->RemoveMachineInstrFromMaps(*I);
-        I->eraseFromParent();
-        MRI->replaceRegWith(SavedExec, ExecReg);
-        LIS->removeInterval(SavedExec);
-        Changed = true;
+      if (SavedExec.isVirtual() && MRI->hasOneNonDBGUse(SavedExec)) {
+        MachineInstr *SingleExecUser = &*MRI->use_instr_nodbg_begin(SavedExec);
+        int Idx = SingleExecUser->findRegisterUseOperandIdx(SavedExec);
+        assert(Idx != -1);
+        if (SingleExecUser->getParent() == I->getParent() &&
+            !SingleExecUser->getOperand(Idx).isImplicit() &&
+            TII->isOperandLegal(*SingleExecUser, Idx, &I->getOperand(1))) {
+          LLVM_DEBUG(dbgs() << "Redundant EXEC COPY: " << *I << '\n');
+          LIS->RemoveMachineInstrFromMaps(*I);
+          I->eraseFromParent();
+          MRI->replaceRegWith(SavedExec, ExecReg);
+          LIS->removeInterval(SavedExec);
+          Changed = true;
+        }
       }
       break;
     }
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 9ca43512cd9128721f40345c1d6ae09ac029d41b..cc06cd8ae7173c09cdda3ca924a84e1188424778 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -14,13 +14,20 @@
 #include "AMDGPU.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
-#include "SIMachineFunctionInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "si-pre-emit-peephole"
 
+static unsigned SkipThreshold;
+
+static cl::opt<unsigned, true> SkipThresholdFlag(
+    "amdgpu-skip-threshold", cl::Hidden,
+    cl::desc(
+        "Number of instructions before jumping over divergent control flow"),
+    cl::location(SkipThreshold), cl::init(12));
+
 namespace {
 
 class SIPreEmitPeephole : public MachineFunctionPass {
@@ -30,6 +37,13 @@ private:
 
   bool optimizeVccBranch(MachineInstr &MI) const;
   bool optimizeSetGPR(MachineInstr &First, MachineInstr &MI) const;
+  bool getBlockDestinations(MachineBasicBlock &SrcMBB,
+                            MachineBasicBlock *&TrueMBB,
+                            MachineBasicBlock *&FalseMBB,
+                            SmallVectorImpl<MachineOperand> &Cond);
+  bool mustRetainExeczBranch(const MachineBasicBlock &From,
+                             const MachineBasicBlock &To) const;
+  bool removeExeczBranch(MachineInstr &MI, MachineBasicBlock &SrcMBB);
 
 public:
   static char ID;
@@ -219,8 +233,11 @@ bool SIPreEmitPeephole::optimizeSetGPR(MachineInstr &First,
     return false;
 
   // Scan back to find an identical S_SET_GPR_IDX_ON
-  for (MachineBasicBlock::iterator I = std::next(First.getIterator()),
-       E = MI.getIterator(); I != E; ++I) {
+  for (MachineBasicBlock::instr_iterator I = std::next(First.getIterator()),
+                                         E = MI.getIterator();
+       I != E; ++I) {
+    if (I->isBundle())
+      continue;
     switch (I->getOpcode()) {
     case AMDGPU::S_SET_GPR_IDX_MODE:
       return false;
@@ -249,9 +266,77 @@ bool SIPreEmitPeephole::optimizeSetGPR(MachineInstr &First,
     }
   }
 
-  MI.eraseFromParent();
+  MI.eraseFromBundle();
   for (MachineInstr *RI : ToRemove)
-    RI->eraseFromParent();
+    RI->eraseFromBundle();
+  return true;
+}
+
+bool SIPreEmitPeephole::getBlockDestinations(
+    MachineBasicBlock &SrcMBB, MachineBasicBlock *&TrueMBB,
+    MachineBasicBlock *&FalseMBB, SmallVectorImpl<MachineOperand> &Cond) {
+  if (TII->analyzeBranch(SrcMBB, TrueMBB, FalseMBB, Cond))
+    return false;
+
+  if (!FalseMBB)
+    FalseMBB = SrcMBB.getNextNode();
+
+  return true;
+}
+
+bool SIPreEmitPeephole::mustRetainExeczBranch(
+    const MachineBasicBlock &From, const MachineBasicBlock &To) const {
+  unsigned NumInstr = 0;
+  const MachineFunction *MF = From.getParent();
+
+  for (MachineFunction::const_iterator MBBI(&From), ToI(&To), End = MF->end();
+       MBBI != End && MBBI != ToI; ++MBBI) {
+    const MachineBasicBlock &MBB = *MBBI;
+
+    for (MachineBasicBlock::const_iterator I = MBB.begin(), E = MBB.end();
+         I != E; ++I) {
+      // When a uniform loop is inside non-uniform control flow, the branch
+      // leaving the loop might never be taken when EXEC = 0.
+      // Hence we should retain cbranch out of the loop lest it become infinite.
+      if (I->isConditionalBranch())
+        return true;
+
+      if (TII->hasUnwantedEffectsWhenEXECEmpty(*I))
+        return true;
+
+      // These instructions are potentially expensive even if EXEC = 0.
+      if (TII->isSMRD(*I) || TII->isVMEM(*I) || TII->isFLAT(*I) ||
+          TII->isDS(*I) || I->getOpcode() == AMDGPU::S_WAITCNT)
+        return true;
+
+      ++NumInstr;
+      if (NumInstr >= SkipThreshold)
+        return true;
+    }
+  }
+
+  return false;
+}
+
+// Returns true if the skip branch instruction is removed.
+bool SIPreEmitPeephole::removeExeczBranch(MachineInstr &MI,
+                                          MachineBasicBlock &SrcMBB) {
+  MachineBasicBlock *TrueMBB = nullptr;
+  MachineBasicBlock *FalseMBB = nullptr;
+  SmallVector<MachineOperand, 1> Cond;
+
+  if (!getBlockDestinations(SrcMBB, TrueMBB, FalseMBB, Cond))
+    return false;
+
+  // Consider only the forward branches.
+  if ((SrcMBB.getNumber() >= TrueMBB->getNumber()) ||
+      mustRetainExeczBranch(*FalseMBB, *TrueMBB))
+    return false;
+
+  LLVM_DEBUG(dbgs() << "Removing the execz branch: " << MI);
+  MI.eraseFromParent();
+  SrcMBB.removeSuccessor(TrueMBB);
+
   return true;
 }
 
@@ -259,13 +344,14 @@ bool SIPreEmitPeephole::runOnMachineFunction(MachineFunction &MF) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   TII = ST.getInstrInfo();
   TRI = &TII->getRegisterInfo();
-  MachineBasicBlock *EmptyMBBAtEnd = nullptr;
   bool Changed = false;
 
+  MF.RenumberBlocks();
+
   for (MachineBasicBlock &MBB : MF) {
     MachineBasicBlock::iterator MBBE = MBB.getFirstTerminator();
     MachineBasicBlock::iterator TermI = MBBE;
-    // Check first terminator for VCC branches to optimize
+    // Check first terminator for branches to optimize
     if (TermI != MBB.end()) {
       MachineInstr &MI = *TermI;
       switch (MI.getOpcode()) {
@@ -273,38 +359,13 @@ bool SIPreEmitPeephole::runOnMachineFunction(MachineFunction &MF) {
       case AMDGPU::S_CBRANCH_VCCNZ:
         Changed |= optimizeVccBranch(MI);
         continue;
+      case AMDGPU::S_CBRANCH_EXECZ:
+        Changed |= removeExeczBranch(MI, MBB);
+        continue;
       default:
         break;
       }
     }
-    // Check all terminators for SI_RETURN_TO_EPILOG
-    // FIXME: This is not an optimization and should be moved somewhere else.
-    while (TermI != MBB.end()) {
-      MachineInstr &MI = *TermI;
-      if (MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) {
-        assert(!MF.getInfo<SIMachineFunctionInfo>()->returnsVoid());
-
-        // Graphics shaders returning non-void shouldn't contain S_ENDPGM,
-        // because external bytecode will be appended at the end.
-        if (&MBB != &MF.back() || &MI != &MBB.back()) {
-          // SI_RETURN_TO_EPILOG is not the last instruction. Add an empty block
-          // at the end and jump there.
-          if (!EmptyMBBAtEnd) {
-            EmptyMBBAtEnd = MF.CreateMachineBasicBlock();
-            MF.insert(MF.end(), EmptyMBBAtEnd);
-          }
-
-          MBB.addSuccessor(EmptyMBBAtEnd);
-          BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::S_BRANCH))
-              .addMBB(EmptyMBBAtEnd);
-          MI.eraseFromParent();
-          MBBE = MBB.getFirstTerminator();
-          TermI = MBBE;
-          continue;
-        }
-      }
-      TermI++;
-    }
 
     if (!ST.hasVGPRIndexMode())
       continue;
@@ -315,7 +376,10 @@ bool SIPreEmitPeephole::runOnMachineFunction(MachineFunction &MF) {
     // Scan the block for two S_SET_GPR_IDX_ON instructions to see if a
     // second is not needed. Do expensive checks in the optimizeSetGPR()
     // and limit the distance to 20 instructions for compile time purposes.
-    for (MachineBasicBlock::iterator MBBI = MBB.begin(); MBBI != MBBE; ) {
+    // Note: this needs to work on bundles as S_SET_GPR_IDX* instructions
+    // may be bundled with the instructions they modify.
+    for (MachineBasicBlock::instr_iterator MBBI = MBB.instr_begin();
+         MBBI != MBBE;) {
       MachineInstr &MI = *MBBI;
       ++MBBI;
 
diff --git a/llvm/lib/Target/AMDGPU/SIRemoveShortExecBranches.cpp b/llvm/lib/Target/AMDGPU/SIRemoveShortExecBranches.cpp
deleted file mode 100644
index 104dea8fdff519c5cb3d343d032d7ab7606055ae..0000000000000000000000000000000000000000
--- a/llvm/lib/Target/AMDGPU/SIRemoveShortExecBranches.cpp
+++ /dev/null
@@ -1,159 +0,0 @@
-//===-- SIRemoveShortExecBranches.cpp ------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-/// This pass optmizes the s_cbranch_execz instructions.
-/// The pass removes this skip instruction for short branches,
-/// if there is no unwanted sideeffect in the fallthrough code sequence.
-///
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPU.h"
-#include "GCNSubtarget.h"
-#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/Support/CommandLine.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "si-remove-short-exec-branches"
-
-static unsigned SkipThreshold;
-
-static cl::opt<unsigned, true> SkipThresholdFlag(
-    "amdgpu-skip-threshold", cl::Hidden,
-    cl::desc(
-        "Number of instructions before jumping over divergent control flow"),
-    cl::location(SkipThreshold), cl::init(12));
-
-namespace {
-
-class SIRemoveShortExecBranches : public MachineFunctionPass {
-private:
-  const SIInstrInfo *TII = nullptr;
-  bool getBlockDestinations(MachineBasicBlock &SrcMBB,
-                            MachineBasicBlock *&TrueMBB,
-                            MachineBasicBlock *&FalseMBB,
-                            SmallVectorImpl<MachineOperand> &Cond);
-  bool mustRetainExeczBranch(const MachineBasicBlock &From,
-                             const MachineBasicBlock &To) const;
-  bool removeExeczBranch(MachineInstr &MI, MachineBasicBlock &SrcMBB);
-
-public:
-  static char ID;
-
-  SIRemoveShortExecBranches() : MachineFunctionPass(ID) {
-    initializeSIRemoveShortExecBranchesPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-};
-
-} // End anonymous namespace.
-
-INITIALIZE_PASS(SIRemoveShortExecBranches, DEBUG_TYPE,
-                "SI remove short exec branches", false, false)
-
-char SIRemoveShortExecBranches::ID = 0;
-
-char &llvm::SIRemoveShortExecBranchesID = SIRemoveShortExecBranches::ID;
-
-bool SIRemoveShortExecBranches::getBlockDestinations(
-    MachineBasicBlock &SrcMBB, MachineBasicBlock *&TrueMBB,
-    MachineBasicBlock *&FalseMBB, SmallVectorImpl<MachineOperand> &Cond) {
-  if (TII->analyzeBranch(SrcMBB, TrueMBB, FalseMBB, Cond))
-    return false;
-
-  if (!FalseMBB)
-    FalseMBB = SrcMBB.getNextNode();
-
-  return true;
-}
-
-bool SIRemoveShortExecBranches::mustRetainExeczBranch(
-    const MachineBasicBlock &From, const MachineBasicBlock &To) const {
-  unsigned NumInstr = 0;
-  const MachineFunction *MF = From.getParent();
-
-  for (MachineFunction::const_iterator MBBI(&From), ToI(&To), End = MF->end();
-       MBBI != End && MBBI != ToI; ++MBBI) {
-    const MachineBasicBlock &MBB = *MBBI;
-
-    for (MachineBasicBlock::const_iterator I = MBB.begin(), E = MBB.end();
-         I != E; ++I) {
-      // When a uniform loop is inside non-uniform control flow, the branch
-      // leaving the loop might never be taken when EXEC = 0.
-      // Hence we should retain cbranch out of the loop lest it become infinite.
-      if (I->isConditionalBranch())
-        return true;
-
-      if (TII->hasUnwantedEffectsWhenEXECEmpty(*I))
-        return true;
-
-      if (TII->isKillTerminator(I->getOpcode()))
-        return true;
-
-      // These instructions are potentially expensive even if EXEC = 0.
-      if (TII->isSMRD(*I) || TII->isVMEM(*I) || TII->isFLAT(*I) ||
-          TII->isDS(*I) || I->getOpcode() == AMDGPU::S_WAITCNT)
-        return true;
-
-      ++NumInstr;
-      if (NumInstr >= SkipThreshold)
-        return true;
-    }
-  }
-
-  return false;
-}
-
-// Returns true if the skip branch instruction is removed.
-bool SIRemoveShortExecBranches::removeExeczBranch(MachineInstr &MI,
-                                                  MachineBasicBlock &SrcMBB) {
-  MachineBasicBlock *TrueMBB = nullptr;
-  MachineBasicBlock *FalseMBB = nullptr;
-  SmallVector<MachineOperand, 1> Cond;
-
-  if (!getBlockDestinations(SrcMBB, TrueMBB, FalseMBB, Cond))
-    return false;
-
-  // Consider only the forward branches.
-  if ((SrcMBB.getNumber() >= TrueMBB->getNumber()) ||
-      mustRetainExeczBranch(*FalseMBB, *TrueMBB))
-    return false;
-
-  LLVM_DEBUG(dbgs() << "Removing the execz branch: " << MI);
-  MI.eraseFromParent();
-  SrcMBB.removeSuccessor(TrueMBB);
-
-  return true;
-}
-
-bool SIRemoveShortExecBranches::runOnMachineFunction(MachineFunction &MF) {
-  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
-  TII = ST.getInstrInfo();
-  MF.RenumberBlocks();
-  bool Changed = false;
-
-  for (MachineBasicBlock &MBB : MF) {
-    MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
-    if (MBBI == MBB.end())
-      continue;
-
-    MachineInstr &MI = *MBBI;
-    switch (MI.getOpcode()) {
-    case AMDGPU::S_CBRANCH_EXECZ:
-      Changed = removeExeczBranch(MI, MBB);
-      break;
-    default:
-      break;
-    }
-  }
-
-  return Changed;
-}
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index 82b0ed06327438941706dbf72a1806b5ed69182f..ac9a504520e7cf32baca998b9d57d0c968449b06 100644
--- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -335,23 +335,22 @@ void SIWholeQuadMode::markDefs(const MachineInstr &UseMI, LiveRange &LR,
   struct PhiEntry {
     const VNInfo *Phi;
     unsigned PredIdx;
-    unsigned VisitIdx;
     LaneBitmask DefinedLanes;
 
-    PhiEntry(const VNInfo *Phi, unsigned PredIdx, unsigned VisitIdx,
-             LaneBitmask DefinedLanes)
-        : Phi(Phi), PredIdx(PredIdx), VisitIdx(VisitIdx),
-          DefinedLanes(DefinedLanes) {}
+    PhiEntry(const VNInfo *Phi, unsigned PredIdx, LaneBitmask DefinedLanes)
+        : Phi(Phi), PredIdx(PredIdx), DefinedLanes(DefinedLanes) {}
   };
-  SmallSetVector<const VNInfo *, 4> Visited;
+  using VisitKey = std::pair<const VNInfo *, LaneBitmask>;
   SmallVector<PhiEntry, 2> PhiStack;
+  SmallSet<VisitKey, 4> Visited;
   LaneBitmask DefinedLanes;
-  unsigned NextPredIdx; // Only used for processing phi nodes
+  unsigned NextPredIdx = 0; // Only used for processing phi nodes
   do {
     const VNInfo *NextValue = nullptr;
+    const VisitKey Key(Value, DefinedLanes);
 
-    if (!Visited.count(Value)) {
-      Visited.insert(Value);
+    if (!Visited.count(Key)) {
+      Visited.insert(Key);
       // On first visit to a phi then start processing first predecessor
       NextPredIdx = 0;
     }
@@ -367,14 +366,14 @@ void SIWholeQuadMode::markDefs(const MachineInstr &UseMI, LiveRange &LR,
       auto PE = MBB->pred_end();
       for (; PI != PE && !NextValue; ++PI, ++Idx) {
         if (const VNInfo *VN = LR.getVNInfoBefore(LIS->getMBBEndIdx(*PI))) {
-          if (!Visited.count(VN))
+          if (!Visited.count(VisitKey(VN, DefinedLanes)))
             NextValue = VN;
         }
       }
 
       // If there are more predecessors to process; add phi to stack
       if (PI != PE)
-        PhiStack.emplace_back(Value, Idx, Visited.size(), DefinedLanes);
+        PhiStack.emplace_back(Value, Idx, DefinedLanes);
     } else {
       MachineInstr *MI = LIS->getInstructionFromIndex(Value->def);
       assert(MI && "Def has no defining instruction");
@@ -404,7 +403,7 @@ void SIWholeQuadMode::markDefs(const MachineInstr &UseMI, LiveRange &LR,
           // Definition not complete; need to process input value
           LiveQueryResult LRQ = LR.Query(LIS->getInstructionIndex(*MI));
           if (const VNInfo *VN = LRQ.valueIn()) {
-            if (!Visited.count(VN))
+            if (!Visited.count(VisitKey(VN, DefinedLanes)))
               NextValue = VN;
           }
         }
@@ -424,9 +423,6 @@ void SIWholeQuadMode::markDefs(const MachineInstr &UseMI, LiveRange &LR,
       NextValue = Entry.Phi;
       NextPredIdx = Entry.PredIdx;
       DefinedLanes = Entry.DefinedLanes;
-      // Rewind visited set to correct state
-      while (Visited.size() > Entry.VisitIdx)
-        Visited.pop_back();
       PhiStack.pop_back();
     }
 
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 2df33f196816ea4798cd555a4e0ecd92b51f6a23..71890a008abda9dd581a155ac70451fd6ff63310 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -57,10 +57,11 @@ class SM_Real <SM_Pseudo ps>
   Instruction Opcode = !cast<Instruction>(NAME);
 
   // copy relevant pseudo op flags
-  let SubtargetPredicate = ps.SubtargetPredicate;
-  let AsmMatchConverter  = ps.AsmMatchConverter;
+  let SubtargetPredicate   = ps.SubtargetPredicate;
+  let AsmMatchConverter    = ps.AsmMatchConverter;
   let UseNamedOperandTable = ps.UseNamedOperandTable;
-  let SMRD = ps.SMRD;
+  let SMRD                 = ps.SMRD;
+  let SchedRW              = ps.SchedRW;
 
   let TSFlags = ps.TSFlags;
 
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 115aff6cc7f8058384c2784d2a093e56fbce9ddf..a55340407047f8244ab1f679b81efe326f688d9a 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -66,6 +66,7 @@ class SOP1_Real<bits<8> op, SOP1_Pseudo ps, string real_name = ps.Mnemonic> :
   // copy relevant pseudo op flags
   let SubtargetPredicate = ps.SubtargetPredicate;
   let AsmMatchConverter  = ps.AsmMatchConverter;
+  let SchedRW            = ps.SchedRW;
 
   // encoding
   bits<7> sdst;
@@ -369,10 +370,11 @@ class SOP2_Real<bits<7> op, SOP_Pseudo ps, string real_name = ps.Mnemonic> :
   let isCodeGenOnly = 0;
 
   // copy relevant pseudo op flags
-  let SubtargetPredicate = ps.SubtargetPredicate;
-  let AsmMatchConverter  = ps.AsmMatchConverter;
+  let SubtargetPredicate   = ps.SubtargetPredicate;
+  let AsmMatchConverter    = ps.AsmMatchConverter;
   let UseNamedOperandTable = ps.UseNamedOperandTable;
-  let TSFlags = ps.TSFlags;
+  let TSFlags              = ps.TSFlags;
+  let SchedRW              = ps.SchedRW;
 
   // encoding
   bits<7> sdst;
@@ -703,6 +705,7 @@ class SOPK_Real<bits<5> op, SOPK_Pseudo ps> :
   let AsmMatchConverter  = ps.AsmMatchConverter;
   let DisableEncoding    = ps.DisableEncoding;
   let Constraints        = ps.Constraints;
+  let SchedRW            = ps.SchedRW;
 
   // encoding
   bits<7>  sdst;
@@ -953,11 +956,12 @@ class SOPC_Real<bits<7> op, SOPC_Pseudo ps, string real_name = ps.Mnemonic> :
   let isCodeGenOnly = 0;
 
   // copy relevant pseudo op flags
-  let SubtargetPredicate = ps.SubtargetPredicate;
-  let OtherPredicates    = ps.OtherPredicates;
-  let AsmMatchConverter  = ps.AsmMatchConverter;
+  let SubtargetPredicate   = ps.SubtargetPredicate;
+  let OtherPredicates      = ps.OtherPredicates;
+  let AsmMatchConverter    = ps.AsmMatchConverter;
   let UseNamedOperandTable = ps.UseNamedOperandTable;
-  let TSFlags = ps.TSFlags;
+  let TSFlags              = ps.TSFlags;
+  let SchedRW              = ps.SchedRW;
 
   // encoding
   bits<8> src0;
@@ -1081,11 +1085,12 @@ class SOPP_Real<bits<7> op, SOPP_Pseudo ps, string real_name = ps.Mnemonic> :
   let isCodeGenOnly = 0;
 
   // copy relevant pseudo op flags
-  let SubtargetPredicate = ps.SubtargetPredicate;
-  let OtherPredicates    = ps.OtherPredicates;
-  let AsmMatchConverter  = ps.AsmMatchConverter;
+  let SubtargetPredicate   = ps.SubtargetPredicate;
+  let OtherPredicates      = ps.OtherPredicates;
+  let AsmMatchConverter    = ps.AsmMatchConverter;
   let UseNamedOperandTable = ps.UseNamedOperandTable;
-  let TSFlags = ps.TSFlags;
+  let TSFlags              = ps.TSFlags;
+  let SchedRW              = ps.SchedRW;
   bits <16> simm16;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index e8d76cc3f89c56fb855237b17325fe40b1bf7dd8..4127bb9b8ada36f11f488ce5b054ec2778b8ff69 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -30,7 +30,8 @@
 
 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion(
   "amdhsa-code-object-version", llvm::cl::Hidden,
-  llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(3));
+  llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4),
+  llvm::cl::ZeroOrMore);
 
 namespace {
 
@@ -96,23 +97,36 @@ Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) {
     return ELF::ELFABIVERSION_AMDGPU_HSA_V2;
   case 3:
     return ELF::ELFABIVERSION_AMDGPU_HSA_V3;
+  case 4:
+    return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
   default:
-    return ELF::ELFABIVERSION_AMDGPU_HSA_V3;
+    report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") +
+                       Twine(AmdhsaCodeObjectVersion));
   }
 }
 
 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) {
-  if (const auto &&HsaAbiVer = getHsaAbiVersion(STI))
-    return HsaAbiVer.getValue() == ELF::ELFABIVERSION_AMDGPU_HSA_V2;
+  if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
+    return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2;
   return false;
 }
 
 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) {
-  if (const auto &&HsaAbiVer = getHsaAbiVersion(STI))
-    return HsaAbiVer.getValue() == ELF::ELFABIVERSION_AMDGPU_HSA_V3;
+  if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
+    return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3;
+  return false;
+}
+
+bool isHsaAbiVersion4(const MCSubtargetInfo *STI) {
+  if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
+    return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4;
   return false;
 }
 
+bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI) {
+  return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI);
+}
+
 #define GET_MIMGBaseOpcodesTable_IMPL
 #define GET_MIMGDimInfoTable_IMPL
 #define GET_MIMGInfoTable_IMPL
@@ -247,7 +261,8 @@ int getMCOpcode(uint16_t Opcode, unsigned Gen) {
 namespace IsaInfo {
 
 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI)
-    : XnackSetting(TargetIDSetting::Any), SramEccSetting(TargetIDSetting::Any) {
+    : STI(STI), XnackSetting(TargetIDSetting::Any),
+      SramEccSetting(TargetIDSetting::Any) {
   if (!STI.getFeatureBits().test(FeatureSupportsXNACK))
     XnackSetting = TargetIDSetting::Unsupported;
   if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC))
@@ -334,25 +349,104 @@ void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) {
   }
 }
 
-void streamIsaVersion(const MCSubtargetInfo *STI, raw_ostream &Stream) {
-  auto TargetTriple = STI->getTargetTriple();
-  auto Version = getIsaVersion(STI->getCPU());
+std::string AMDGPUTargetID::toString() const {
+  std::string StringRep = "";
+  raw_string_ostream StreamRep(StringRep);
+
+  auto TargetTriple = STI.getTargetTriple();
+  auto Version = getIsaVersion(STI.getCPU());
 
-  Stream << TargetTriple.getArchName() << '-'
-         << TargetTriple.getVendorName() << '-'
-         << TargetTriple.getOSName() << '-'
-         << TargetTriple.getEnvironmentName() << '-'
-         << "gfx"
-         << Version.Major
-         << Version.Minor
-         << hexdigit(Version.Stepping, true);
+  StreamRep << TargetTriple.getArchName() << '-'
+            << TargetTriple.getVendorName() << '-'
+            << TargetTriple.getOSName() << '-'
+            << TargetTriple.getEnvironmentName() << '-';
+
+  std::string Processor = "";
+  // TODO: Following else statement is present here because we used various
+  // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803').
+  // Remove once all aliases are removed from GCNProcessors.td.
+  if (Version.Major >= 9)
+    Processor = STI.getCPU().str();
+  else
+    Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) +
+                 Twine(Version.Stepping))
+                    .str();
+
+  std::string Features = "";
+  if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) {
+    switch (*HsaAbiVersion) {
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V2:
+      // Code object V2 only supported specific processors and had fixed
+      // settings for the XNACK.
+      if (Processor == "gfx600") {
+      } else if (Processor == "gfx601") {
+      } else if (Processor == "gfx602") {
+      } else if (Processor == "gfx700") {
+      } else if (Processor == "gfx701") {
+      } else if (Processor == "gfx702") {
+      } else if (Processor == "gfx703") {
+      } else if (Processor == "gfx704") {
+      } else if (Processor == "gfx705") {
+      } else if (Processor == "gfx801") {
+        if (!isXnackOnOrAny())
+          report_fatal_error(
+              "AMD GPU code object V2 does not support processor " + Processor +
+              " without XNACK");
+      } else if (Processor == "gfx802") {
+      } else if (Processor == "gfx803") {
+      } else if (Processor == "gfx805") {
+      } else if (Processor == "gfx810") {
+        if (!isXnackOnOrAny())
+          report_fatal_error(
+              "AMD GPU code object V2 does not support processor " + Processor +
+              " without XNACK");
+      } else if (Processor == "gfx900") {
+        if (isXnackOnOrAny())
+          Processor = "gfx901";
+      } else if (Processor == "gfx902") {
+        if (isXnackOnOrAny())
+          Processor = "gfx903";
+      } else if (Processor == "gfx904") {
+        if (isXnackOnOrAny())
+          Processor = "gfx905";
+      } else if (Processor == "gfx906") {
+        if (isXnackOnOrAny())
+          Processor = "gfx907";
+      } else {
+        report_fatal_error(
+            "AMD GPU code object V2 does not support processor " + Processor);
+      }
+      break;
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
+      // xnack.
+      if (isXnackOnOrAny())
+        Features += "+xnack";
+      // In code object v2 and v3, "sramecc" feature was spelled with a
+      // hyphen ("sram-ecc").
+      if (isSramEccOnOrAny())
+        Features += "+sram-ecc";
+      break;
+    case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
+      // sramecc.
+      if (getSramEccSetting() == TargetIDSetting::Off)
+        Features += ":sramecc-";
+      else if (getSramEccSetting() == TargetIDSetting::On)
+        Features += ":sramecc+";
+      // xnack.
+      if (getXnackSetting() == TargetIDSetting::Off)
+        Features += ":xnack-";
+      else if (getXnackSetting() == TargetIDSetting::On)
+        Features += ":xnack+";
+      break;
+    default:
+      break;
+    }
+  }
 
-  if (hasXNACK(*STI))
-    Stream << "+xnack";
-  if (hasSRAMECC(*STI))
-    Stream << "+sramecc";
+  StreamRep << Processor << Features;
 
-  Stream.flush();
+  StreamRep.flush();
+  return StringRep;
 }
 
 unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index fa5575dffb1e5a43baf6f7f528b14e3731b1c746..6eeefdb52cc3b0818d20e9b4a2dd34d35435b47e 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -44,6 +44,12 @@ bool isHsaAbiVersion2(const MCSubtargetInfo *STI);
 /// \returns True if HSA OS ABI Version identification is 3,
 /// false otherwise.
 bool isHsaAbiVersion3(const MCSubtargetInfo *STI);
+/// \returns True if HSA OS ABI Version identification is 4,
+/// false otherwise.
+bool isHsaAbiVersion4(const MCSubtargetInfo *STI);
+/// \returns True if HSA OS ABI Version identification is 3 or 4,
+/// false otherwise.
+bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI);
 
 struct GcnBufferFormatInfo {
   unsigned Format;
@@ -78,6 +84,7 @@ enum class TargetIDSetting {
 
 class AMDGPUTargetID {
 private:
+  const MCSubtargetInfo &STI;
   TargetIDSetting XnackSetting;
   TargetIDSetting SramEccSetting;
 
@@ -145,10 +152,10 @@ public:
 
   void setTargetIDFromFeaturesString(StringRef FS);
   void setTargetIDFromTargetIDStream(StringRef TargetID);
-};
 
-/// Streams isa version string for given subtarget \p STI into \p Stream.
-void streamIsaVersion(const MCSubtargetInfo *STI, raw_ostream &Stream);
+  /// \returns String representation of an object.
+  std::string toString() const;
+};
 
 /// \returns Wavefront size for given subtarget \p STI.
 unsigned getWavefrontSize(const MCSubtargetInfo *STI);
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
index ef4abd560491c9f7db6d30031adc1517cb6f1f2e..c876be31c3529db9a72ce51f698525964cb93f92 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
@@ -41,7 +41,7 @@ void AMDGPUPALMetadata::readFromIR(Module &M) {
     }
     return;
   }
-  BlobType = ELF::NT_AMD_AMDGPU_PAL_METADATA;
+  BlobType = ELF::NT_AMD_PAL_METADATA;
   NamedMD = M.getNamedMetadata("amdgpu.pal.metadata");
   if (!NamedMD || !NamedMD->getNumOperands()) {
     // Emit msgpack metadata by default
@@ -69,7 +69,7 @@ void AMDGPUPALMetadata::readFromIR(Module &M) {
 // Metadata.
 bool AMDGPUPALMetadata::setFromBlob(unsigned Type, StringRef Blob) {
   BlobType = Type;
-  if (Type == ELF::NT_AMD_AMDGPU_PAL_METADATA)
+  if (Type == ELF::NT_AMD_PAL_METADATA)
     return setFromLegacyBlob(Blob);
   return setFromMsgPackBlob(Blob);
 }
@@ -688,7 +688,7 @@ void AMDGPUPALMetadata::toString(std::string &String) {
 // a .note record of the specified AMD type. Returns an empty blob if
 // there is no PAL metadata,
 void AMDGPUPALMetadata::toBlob(unsigned Type, std::string &Blob) {
-  if (Type == ELF::NT_AMD_AMDGPU_PAL_METADATA)
+  if (Type == ELF::NT_AMD_PAL_METADATA)
     toLegacyBlob(Blob);
   else if (Type)
     toMsgPackBlob(Blob);
@@ -825,7 +825,7 @@ const char *AMDGPUPALMetadata::getVendor() const {
 }
 
 // Get .note record type of metadata blob to be emitted:
-// ELF::NT_AMD_AMDGPU_PAL_METADATA (legacy key=val format), or
+// ELF::NT_AMD_PAL_METADATA (legacy key=val format), or
 // ELF::NT_AMDGPU_METADATA (MsgPack format), or
 // 0 (no PAL metadata).
 unsigned AMDGPUPALMetadata::getType() const {
@@ -834,12 +834,12 @@ unsigned AMDGPUPALMetadata::getType() const {
 
 // Return whether the blob type is legacy PAL metadata.
 bool AMDGPUPALMetadata::isLegacy() const {
-  return BlobType == ELF::NT_AMD_AMDGPU_PAL_METADATA;
+  return BlobType == ELF::NT_AMD_PAL_METADATA;
 }
 
 // Set legacy PAL metadata format.
 void AMDGPUPALMetadata::setLegacy() {
-  BlobType = ELF::NT_AMD_AMDGPU_PAL_METADATA;
+  BlobType = ELF::NT_AMD_PAL_METADATA;
 }
 
 // Erase all PAL metadata.
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h
index 8fa1f738487cc209df849512e56a183fd1c7941d..0c272da1a55bd6b41d281621ca7bec20a3d395bd 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h
@@ -95,7 +95,7 @@ public:
   const char *getVendor() const;
 
   // Get .note record type of metadata blob to be emitted:
-  // ELF::NT_AMD_AMDGPU_PAL_METADATA (legacy key=val format), or
+  // ELF::NT_AMD_PAL_METADATA (legacy key=val format), or
   // ELF::NT_AMDGPU_METADATA (MsgPack format), or
   // 0 (no PAL metadata).
   unsigned getType() const;
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index b8f57c9f0344bc76dcae74395440a6d4d71450dc..ac0731853e1397b0eeb80cc1c4da9d02bc790072 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -79,6 +79,7 @@ class VOP1_Real <VOP1_Pseudo ps, int EncodingFamily> :
   let UseNamedOperandTable = ps.UseNamedOperandTable;
   let Uses                 = ps.Uses;
   let Defs                 = ps.Defs;
+  let SchedRW              = ps.SchedRW;
 }
 
 class VOP1_SDWA_Pseudo <string OpName, VOPProfile P, list<dag> pattern=[]> :
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index 2b8d5d9a5209477508ed36bc481414bccadbee4e..8134ed72ec00c88e2d1ca673babf622b5ae3a722 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -101,6 +101,7 @@ class VOP2_Real <VOP2_Pseudo ps, int EncodingFamily> :
   let UseNamedOperandTable = ps.UseNamedOperandTable;
   let Uses                 = ps.Uses;
   let Defs                 = ps.Defs;
+  let SchedRW              = ps.SchedRW;
 }
 
 class VOP2_SDWA_Pseudo <string OpName, VOPProfile P, list<dag> pattern=[]> :
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 073e4dff45781bc08640798b4d3279ae65460f9b..880a6c618478cf957dd3c4050b35ba4be3078826 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -287,19 +287,24 @@ class SDot2Pat<Instruction Inst> : GCNPat <
 let IsDOT = 1 in {
 let SubtargetPredicate = HasDot2Insts in {
 
-def V_DOT2_F32_F16 : VOP3PInst<"v_dot2_f32_f16",
-  VOP3_Profile<VOP_F32_V2F16_V2F16_F32>,
-  AMDGPUfdot2, 1/*ExplicitClamp*/>;
 def V_DOT2_I32_I16 : VOP3PInst<"v_dot2_i32_i16",
   VOP3_Profile<VOP_I32_V2I16_V2I16_I32>, int_amdgcn_sdot2, 1>;
 def V_DOT2_U32_U16 : VOP3PInst<"v_dot2_u32_u16",
   VOP3_Profile<VOP_I32_V2I16_V2I16_I32>, int_amdgcn_udot2, 1>;
+
+} // End SubtargetPredicate = HasDot2Insts
+
+let SubtargetPredicate = HasDot7Insts in {
+
+def V_DOT2_F32_F16 : VOP3PInst<"v_dot2_f32_f16",
+  VOP3_Profile<VOP_F32_V2F16_V2F16_F32>,
+  AMDGPUfdot2, 1/*ExplicitClamp*/>;
 def V_DOT4_U32_U8  : VOP3PInst<"v_dot4_u32_u8",
   VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>, int_amdgcn_udot4, 1>;
 def V_DOT8_U32_U4  : VOP3PInst<"v_dot8_u32_u4",
   VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>, int_amdgcn_udot8, 1>;
 
-} // End SubtargetPredicate = HasDot2Insts
+} // End SubtargetPredicate = HasDot7Insts
 
 let SubtargetPredicate = HasDot1Insts in {
 
@@ -564,13 +569,18 @@ defm V_FMA_MIXHI_F16 : VOP3P_Real_vi <0x22>;
 
 let SubtargetPredicate = HasDot2Insts in {
 
-defm V_DOT2_F32_F16 : VOP3P_Real_vi <0x23>;
 defm V_DOT2_I32_I16 : VOP3P_Real_vi <0x26>;
 defm V_DOT2_U32_U16 : VOP3P_Real_vi <0x27>;
+
+} // End SubtargetPredicate = HasDot2Insts
+
+let SubtargetPredicate = HasDot7Insts in {
+
+defm V_DOT2_F32_F16 : VOP3P_Real_vi <0x23>;
 defm V_DOT4_U32_U8  : VOP3P_Real_vi <0x29>;
 defm V_DOT8_U32_U4  : VOP3P_Real_vi <0x2b>;
 
-} // End SubtargetPredicate = HasDot2Insts
+} // End SubtargetPredicate = HasDot7Insts
 
 let SubtargetPredicate = HasDot1Insts in {
 
@@ -657,13 +667,18 @@ defm V_FMA_MIXHI_F16  : VOP3P_Real_gfx10<0x22>;
 
 let SubtargetPredicate = HasDot2Insts in {
 
-defm V_DOT2_F32_F16 : VOP3P_Real_gfx10 <0x13>;
 defm V_DOT2_I32_I16 : VOP3P_Real_gfx10 <0x14>;
 defm V_DOT2_U32_U16 : VOP3P_Real_gfx10 <0x15>;
+
+} // End SubtargetPredicate = HasDot2Insts
+
+let SubtargetPredicate = HasDot7Insts in {
+
+defm V_DOT2_F32_F16 : VOP3P_Real_gfx10 <0x13>;
 defm V_DOT4_U32_U8  : VOP3P_Real_gfx10 <0x17>;
 defm V_DOT8_U32_U4  : VOP3P_Real_gfx10 <0x19>;
 
-} // End SubtargetPredicate = HasDot2Insts
+} // End SubtargetPredicate = HasDot7Insts
 
 let SubtargetPredicate = HasDot1Insts in {
 
diff --git a/llvm/lib/Target/AMDGPU/VOPCInstructions.td b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
index 7ca244fcc70a5560fb4ee8d5e436be269cd59f64..b8f27c7710d8f69786ac09ab539dc329040bc811 100644
--- a/llvm/lib/Target/AMDGPU/VOPCInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
@@ -121,6 +121,7 @@ class VOPC_Real <VOPC_Pseudo ps, int EncodingFamily> :
   let UseNamedOperandTable = ps.UseNamedOperandTable;
   let Uses                 = ps.Uses;
   let Defs                 = ps.Defs;
+  let SchedRW              = ps.SchedRW;
 }
 
 class VOPC_SDWA_Pseudo <string OpName, VOPProfile P, list<dag> pattern=[]> :
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index 45b64dde69cf44604df440020205057e1e1cb112..f0afc6d1b17b69ba498d8a0840bb138f912024c7 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -162,6 +162,7 @@ class VOP3_Real <VOP_Pseudo ps, int EncodingFamily> :
   let UseNamedOperandTable = ps.UseNamedOperandTable;
   let Uses                 = ps.Uses;
   let Defs                 = ps.Defs;
+  let SchedRW              = ps.SchedRW;
 
   VOPProfile Pfl = ps.Pfl;
 }
@@ -519,7 +520,6 @@ class VOP_SDWA_Real <VOP_SDWA_Pseudo ps> :
 
   let Defs = ps.Defs;
   let Uses = ps.Uses;
-  let SchedRW = ps.SchedRW;
   let hasSideEffects = ps.hasSideEffects;
 
   let Constraints     = ps.Constraints;
@@ -535,6 +535,7 @@ class VOP_SDWA_Real <VOP_SDWA_Pseudo ps> :
   let Constraints          = ps.Constraints;
   let DisableEncoding      = ps.DisableEncoding;
   let TSFlags              = ps.TSFlags;
+  let SchedRW              = ps.SchedRW;
 }
 
 class Base_VOP_SDWA9_Real <VOP_SDWA_Pseudo ps> :
@@ -563,6 +564,7 @@ class Base_VOP_SDWA9_Real <VOP_SDWA_Pseudo ps> :
   let Constraints          = ps.Constraints;
   let DisableEncoding      = ps.DisableEncoding;
   let TSFlags              = ps.TSFlags;
+  let SchedRW              = ps.SchedRW;
 }
 
 class VOP_SDWA9_Real <VOP_SDWA_Pseudo ps> :
@@ -664,6 +666,7 @@ class VOP_DPP_Real <VOP_DPP_Pseudo ps, int EncodingFamily> :
   let Constraints          = ps.Constraints;
   let DisableEncoding      = ps.DisableEncoding;
   let TSFlags              = ps.TSFlags;
+  let SchedRW              = ps.SchedRW;
 }
 
 class VOP_DPP <string OpName, VOPProfile P, bit IsDPP16,
diff --git a/llvm/lib/Target/ARM/ARM.h b/llvm/lib/Target/ARM/ARM.h
index 21ce3b19e170c6e92709f2603fd39e250e906501..5500783f74dbef9587f3589ef9758d62cb659b4b 100644
--- a/llvm/lib/Target/ARM/ARM.h
+++ b/llvm/lib/Target/ARM/ARM.h
@@ -58,6 +58,7 @@ createARMInstructionSelector(const ARMBaseTargetMachine &TM, const ARMSubtarget
 Pass *createMVEGatherScatterLoweringPass();
 FunctionPass *createARMSLSHardeningPass();
 FunctionPass *createARMIndirectThunks();
+Pass *createMVELaneInterleavingPass();
 
 void LowerARMMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
                                   ARMAsmPrinter &AP);
@@ -76,6 +77,7 @@ void initializeARMBlockPlacementPass(PassRegistry &);
 void initializeMVETailPredicationPass(PassRegistry &);
 void initializeMVEGatherScatterLoweringPass(PassRegistry &);
 void initializeARMSLSHardeningPass(PassRegistry &);
+void initializeMVELaneInterleavingPass(PassRegistry &);
 
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 802096e040dfa865adeef02f643c712b96a0cc00..d9be8ab471aa080d6fbe8ab3a5eb5a2e3f12d105 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -21,6 +21,7 @@
 #include "ARMRegisterInfo.h"
 #include "ARMSelectionDAGInfo.h"
 #include "ARMSubtarget.h"
+#include "ARMTargetTransformInfo.h"
 #include "MCTargetDesc/ARMAddressingModes.h"
 #include "MCTargetDesc/ARMBaseInfo.h"
 #include "Utils/ARMBaseInfo.h"
@@ -6966,35 +6967,6 @@ static bool isVEXTMask(ArrayRef<int> M, EVT VT,
   return true;
 }
 
-/// isVREVMask - Check if a vector shuffle corresponds to a VREV
-/// instruction with the specified blocksize.  (The order of the elements
-/// within each block of the vector is reversed.)
-static bool isVREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
-  assert((BlockSize==16 || BlockSize==32 || BlockSize==64) &&
-         "Only possible block sizes for VREV are: 16, 32, 64");
-
-  unsigned EltSz = VT.getScalarSizeInBits();
-  if (EltSz == 64)
-    return false;
-
-  unsigned NumElts = VT.getVectorNumElements();
-  unsigned BlockElts = M[0] + 1;
-  // If the first shuffle index is UNDEF, be optimistic.
-  if (M[0] < 0)
-    BlockElts = BlockSize / EltSz;
-
-  if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz)
-    return false;
-
-  for (unsigned i = 0; i < NumElts; ++i) {
-    if (M[i] < 0) continue; // ignore UNDEF indices
-    if ((unsigned) M[i] != (i - i%BlockElts) + (BlockElts - 1 - i%BlockElts))
-      return false;
-  }
-
-  return true;
-}
-
 static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
   // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
   // range, then 0 is placed into the resulting vector. So pretty much any mask
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index 7d1c9017e3dc516cdd99831ff4205ac2891f5541..c4830e7351f5ffa928238e90300dfdbd27defd76 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -1900,8 +1900,13 @@ let Predicates = [HasMVEInt] in {
   def : Pat<(insertelt (v4f32 MQPR:$src1), (f32 SPR:$src2), imm:$lane),
             (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), SPR:$src2, (SSubReg_f32_reg imm:$lane))>;
 
-  def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm:$lane),
+  def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm_even:$lane),
             (MVE_VMOV_to_lane_16 MQPR:$src1, (COPY_TO_REGCLASS (f16 HPR:$src2), rGPR), imm:$lane)>;
+  def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm_odd:$lane),
+            (COPY_TO_REGCLASS (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)),
+                                (VINSH (EXTRACT_SUBREG MQPR:$src1, (SSubReg_f16_reg imm_odd:$lane)),
+                                       (COPY_TO_REGCLASS HPR:$src2, SPR)),
+                                (SSubReg_f16_reg imm_odd:$lane)), MQPR)>;
   def : Pat<(extractelt (v8f16 MQPR:$src), imm_even:$lane),
             (EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_even:$lane))>;
   def : Pat<(extractelt (v8f16 MQPR:$src), imm_odd:$lane),
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb.td b/llvm/lib/Target/ARM/ARMInstrThumb.td
index 3a33dfeecdc9ae89b03877856a5d0a713062124e..64d4dc0b112ab576b46e56ba3b3c87e6417bd74f 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb.td
@@ -1659,19 +1659,16 @@ def : T1Pat<(post_store tGPR:$Rt, tGPR:$Rn, 4),
             (tSTMIA_UPD tGPR:$Rn, tGPR:$Rt)>;
 
 // If it's impossible to use [r,r] address mode for sextload, select to
-// ldr{b|h} + sxt{b|h} instead.
-def : T1Pat<(sextloadi8 t_addrmode_is1:$addr),
-            (tSXTB (tLDRBi t_addrmode_is1:$addr))>,
-      Requires<[IsThumb, IsThumb1Only, HasV6]>;
-def : T1Pat<(sextloadi8 t_addrmode_rr:$addr),
-            (tSXTB (tLDRBr t_addrmode_rr:$addr))>,
+// ldsr{b|h} r, 0 instead, in a hope that the mov 0 will be more likely to be
+// commoned out than a sxth.
+let AddedComplexity = 10 in {
+def : T1Pat<(sextloadi8 tGPR:$Rn),
+            (tLDRSB tGPR:$Rn, (tMOVi8 0))>,
       Requires<[IsThumb, IsThumb1Only, HasV6]>;
-def : T1Pat<(sextloadi16 t_addrmode_is2:$addr),
-            (tSXTH (tLDRHi t_addrmode_is2:$addr))>,
-      Requires<[IsThumb, IsThumb1Only, HasV6]>;
-def : T1Pat<(sextloadi16 t_addrmode_rr:$addr),
-            (tSXTH (tLDRHr t_addrmode_rr:$addr))>,
+def : T1Pat<(sextloadi16 tGPR:$Rn),
+            (tLDRSH tGPR:$Rn, (tMOVi8 0))>,
       Requires<[IsThumb, IsThumb1Only, HasV6]>;
+}
 
 def : T1Pat<(sextloadi8 t_addrmode_is1:$addr),
             (tASRri (tLSLri (tLDRBi t_addrmode_is1:$addr), 24), 24)>;
diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
index 69cdfe269c183022a7d71d43c2c6410688a0ae30..9637c9df900f2dd25e8c0a6368bc00505a0b095b 100644
--- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
+++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
@@ -1460,14 +1460,15 @@ MachineInstr* ARMLowOverheadLoops::ExpandLoopStart(LowOverheadLoop &LoLoop) {
 
 void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) {
   auto RemovePredicate = [](MachineInstr *MI) {
+    if (MI->isDebugInstr())
+      return;
     LLVM_DEBUG(dbgs() << "ARM Loops: Removing predicate from: " << *MI);
-    if (int PIdx = llvm::findFirstVPTPredOperandIdx(*MI)) {
-      assert(MI->getOperand(PIdx).getImm() == ARMVCC::Then &&
-             "Expected Then predicate!");
-      MI->getOperand(PIdx).setImm(ARMVCC::None);
-      MI->getOperand(PIdx+1).setReg(0);
-    } else
-      llvm_unreachable("trying to unpredicate a non-predicated instruction");
+    int PIdx = llvm::findFirstVPTPredOperandIdx(*MI);
+    assert(PIdx >= 1 && "Trying to unpredicate a non-predicated instruction");
+    assert(MI->getOperand(PIdx).getImm() == ARMVCC::Then &&
+           "Expected Then predicate!");
+    MI->getOperand(PIdx).setImm(ARMVCC::None);
+    MI->getOperand(PIdx + 1).setReg(0);
   };
 
   for (auto &Block : LoLoop.getVPTBlocks()) {
@@ -1511,8 +1512,13 @@ void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) {
         // - Insert a new vpst to predicate the instruction(s) that following
         //   the divergent vpr def.
         MachineInstr *Divergent = VPTState::getDivergent(Block);
+        MachineBasicBlock *MBB = Divergent->getParent();
         auto DivergentNext = ++MachineBasicBlock::iterator(Divergent);
+        while (DivergentNext != MBB->end() && DivergentNext->isDebugInstr())
+          ++DivergentNext;
+
         bool DivergentNextIsPredicated =
+            DivergentNext != MBB->end() &&
             getVPTInstrPredicate(*DivergentNext) != ARMVCC::None;
 
         for (auto I = ++MachineBasicBlock::iterator(VPST), E = DivergentNext;
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index 23a330f6041497dae04ecd6b796cab3209dcde42..c09df077e257d4f39f91717f7731f0cda4f63c20 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -102,6 +102,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeARMTarget() {
   initializeARMBlockPlacementPass(Registry);
   initializeMVEGatherScatterLoweringPass(Registry);
   initializeARMSLSHardeningPass(Registry);
+  initializeMVELaneInterleavingPass(Registry);
 }
 
 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -416,6 +417,7 @@ void ARMPassConfig::addIRPasses() {
         }));
 
   addPass(createMVEGatherScatterLoweringPass());
+  addPass(createMVELaneInterleavingPass());
 
   TargetPassConfig::addIRPasses();
 
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 92e523eed6bee5475d2d4f4fb91cfe9bbc76a351..07c63d5045acf43b29368551a3c0247522472d2e 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -817,13 +817,12 @@ int ARMTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
 
   if (ST->hasMVEIntegerOps() && (Opcode == Instruction::InsertElement ||
                                  Opcode == Instruction::ExtractElement)) {
-    // We say MVE moves costs at least the MVEVectorCostFactor, even though
-    // they are scalar instructions. This helps prevent mixing scalar and
-    // vector, to prevent vectorising where we end up just scalarising the
-    // result anyway.
-    return std::max(BaseT::getVectorInstrCost(Opcode, ValTy, Index),
-                    ST->getMVEVectorCostFactor(TTI::TCK_RecipThroughput)) *
-           cast<FixedVectorType>(ValTy)->getNumElements() / 2;
+    // Integer cross-lane moves are more expensive than float, which can
+    // sometimes just be vmovs. Integer involve being passes to GPR registers,
+    // causing more of a delay.
+    std::pair<unsigned, MVT> LT =
+        getTLI()->getTypeLegalizationCost(DL, ValTy->getScalarType());
+    return LT.first * (ValTy->getScalarType()->isIntegerTy() ? 4 : 1);
   }
 
   return BaseT::getVectorInstrCost(Opcode, ValTy, Index);
@@ -1131,7 +1130,8 @@ int ARMTTIImpl::getMemcpyCost(const Instruction *I) {
 }
 
 int ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
-                               int Index, VectorType *SubTp) {
+                               ArrayRef<int> Mask, int Index,
+                               VectorType *SubTp) {
   if (ST->hasNEON()) {
     if (Kind == TTI::SK_Broadcast) {
       static const CostTblEntry NEONDupTbl[] = {
@@ -1218,11 +1218,20 @@ int ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
         return LT.first * Entry->Cost *
                ST->getMVEVectorCostFactor(TTI::TCK_RecipThroughput);
     }
+
+    if (!Mask.empty()) {
+      std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
+      if (Mask.size() <= LT.second.getVectorNumElements() &&
+          (isVREVMask(Mask, LT.second, 16) || isVREVMask(Mask, LT.second, 32) ||
+           isVREVMask(Mask, LT.second, 64)))
+        return ST->getMVEVectorCostFactor(TTI::TCK_RecipThroughput) * LT.first;
+    }
   }
+
   int BaseCost = ST->hasMVEIntegerOps() && Tp->isVectorTy()
                      ? ST->getMVEVectorCostFactor(TTI::TCK_RecipThroughput)
                      : 1;
-  return BaseCost * BaseT::getShuffleCost(Kind, Tp, Index, SubTp);
+  return BaseCost * BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
 }
 
 int ARMTTIImpl::getArithmeticInstrCost(unsigned Opcode, Type *Ty,
@@ -2116,6 +2125,10 @@ bool ARMTTIImpl::emitGetActiveLaneMask() const {
 }
 void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                          TTI::UnrollingPreferences &UP) {
+  // Enable Upper bound unrolling universally, not dependant upon the conditions
+  // below.
+  UP.UpperBound = true;
+
   // Only currently enable these preferences for M-Class cores.
   if (!ST->isMClass())
     return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP);
@@ -2178,7 +2191,6 @@ void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
 
   UP.Partial = true;
   UP.Runtime = true;
-  UP.UpperBound = true;
   UP.UnrollRemainder = true;
   UP.DefaultUnrollRuntimeCount = 4;
   UP.UnrollAndJam = true;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index cb9c48e662ac275be02ecd555df0c5ba517dccc6..79e63b292468c875b1d95f083254eb2eee7c594f 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -149,16 +149,20 @@ public:
     return 13;
   }
 
-  unsigned getRegisterBitWidth(bool Vector) const {
-    if (Vector) {
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+    switch (K) {
+    case TargetTransformInfo::RGK_Scalar:
+      return TypeSize::getFixed(32);
+    case TargetTransformInfo::RGK_FixedWidthVector:
       if (ST->hasNEON())
-        return 128;
+        return TypeSize::getFixed(128);
       if (ST->hasMVEIntegerOps())
-        return 128;
-      return 0;
+        return TypeSize::getFixed(128);
+      return TypeSize::getFixed(0);
+    case TargetTransformInfo::RGK_ScalableVector:
+      return TypeSize::getScalable(0);
     }
-
-    return 32;
+    llvm_unreachable("Unsupported register kind");
   }
 
   unsigned getMaxInterleaveFactor(unsigned VF) {
@@ -183,8 +187,8 @@ public:
 
   int getNumMemOps(const IntrinsicInst *I) const;
 
-  int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
-                     VectorType *SubTp);
+  int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask,
+                     int Index, VectorType *SubTp);
 
   bool preferInLoopReduction(unsigned Opcode, Type *Ty,
                              TTI::ReductionFlags Flags) const;
@@ -282,6 +286,35 @@ public:
   /// @}
 };
 
+/// isVREVMask - Check if a vector shuffle corresponds to a VREV
+/// instruction with the specified blocksize.  (The order of the elements
+/// within each block of the vector is reversed.)
+inline bool isVREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
+  assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
+         "Only possible block sizes for VREV are: 16, 32, 64");
+
+  unsigned EltSz = VT.getScalarSizeInBits();
+  if (EltSz != 8 && EltSz != 16 && EltSz != 32)
+    return false;
+
+  unsigned BlockElts = M[0] + 1;
+  // If the first shuffle index is UNDEF, be optimistic.
+  if (M[0] < 0)
+    BlockElts = BlockSize / EltSz;
+
+  if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz)
+    return false;
+
+  for (unsigned i = 0, e = M.size(); i < e; ++i) {
+    if (M[i] < 0)
+      continue; // ignore UNDEF indices
+    if ((unsigned)M[i] != (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
+      return false;
+  }
+
+  return true;
+}
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_ARM_ARMTARGETTRANSFORMINFO_H
diff --git a/llvm/lib/Target/ARM/CMakeLists.txt b/llvm/lib/Target/ARM/CMakeLists.txt
index 2b03e9fb3f59e3a738df3c336e0893e64cbdcf55..89abc579460b62dc4cb04edc745a0720042c76d6 100644
--- a/llvm/lib/Target/ARM/CMakeLists.txt
+++ b/llvm/lib/Target/ARM/CMakeLists.txt
@@ -56,6 +56,7 @@ add_llvm_target(ARMCodeGen
   ARMTargetTransformInfo.cpp
   MLxExpansionPass.cpp
   MVEGatherScatterLowering.cpp
+  MVELaneInterleavingPass.cpp
   MVETailPredication.cpp
   MVEVPTBlockPass.cpp
   MVETPAndVPTOptimisationsPass.cpp
diff --git a/llvm/lib/Target/ARM/MVELaneInterleavingPass.cpp b/llvm/lib/Target/ARM/MVELaneInterleavingPass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ce01245fbd14200a00a10244ab69afdfbce96f54
--- /dev/null
+++ b/llvm/lib/Target/ARM/MVELaneInterleavingPass.cpp
@@ -0,0 +1,342 @@
+//===- MVELaneInterleaving.cpp - Inverleave for MVE instructions ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass interleaves around sext/zext/trunc instructions. MVE does not have
+// a single sext/zext or trunc instruction that takes the bottom half of a
+// vector and extends to a full width, like NEON has with MOVL. Instead it is
+// expected that this happens through top/bottom instructions. So the MVE
+// equivalent VMOVLT/B instructions take either the even or odd elements of the
+// input and extend them to the larger type, producing a vector with half the
+// number of elements each of double the bitwidth. As there is no simple
+// instruction, we often have to turn sext/zext/trunc into a series of lane
+// moves (or stack loads/stores, which we do not do yet).
+//
+// This pass takes vector code that starts at truncs, looks for interconnected
+// blobs of operations that end with sext/zext (or constants/splats) of the
+// form:
+//   %sa = sext v8i16 %a to v8i32
+//   %sb = sext v8i16 %b to v8i32
+//   %add = add v8i32 %sa, %sb
+//   %r = trunc %add to v8i16
+// And adds shuffles to allow the use of VMOVL/VMOVN instrctions:
+//   %sha = shuffle v8i16 %a, undef, <0, 2, 4, 6, 1, 3, 5, 7>
+//   %sa = sext v8i16 %sha to v8i32
+//   %shb = shuffle v8i16 %b, undef, <0, 2, 4, 6, 1, 3, 5, 7>
+//   %sb = sext v8i16 %shb to v8i32
+//   %add = add v8i32 %sa, %sb
+//   %r = trunc %add to v8i16
+//   %shr = shuffle v8i16 %r, undef, <0, 4, 1, 5, 2, 6, 3, 7>
+// Which can then be split and lowered to MVE instructions efficiently:
+//   %sa_b = VMOVLB.s16 %a
+//   %sa_t = VMOVLT.s16 %a
+//   %sb_b = VMOVLB.s16 %b
+//   %sb_t = VMOVLT.s16 %b
+//   %add_b = VADD.i32 %sa_b, %sb_b
+//   %add_t = VADD.i32 %sa_t, %sb_t
+//   %r = VMOVNT.i16 %add_b, %add_t
+//
+//===----------------------------------------------------------------------===//
+
+#include "ARM.h"
+#include "ARMBaseInstrInfo.h"
+#include "ARMSubtarget.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsARM.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Casting.h"
+#include <algorithm>
+#include <cassert>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "mve-laneinterleave"
+
+cl::opt<bool> EnableInterleave(
+    "enable-mve-interleave", cl::Hidden, cl::init(true),
+    cl::desc("Enable interleave MVE vector operation lowering"));
+
+namespace {
+
+class MVELaneInterleaving : public FunctionPass {
+public:
+  static char ID; // Pass identification, replacement for typeid
+
+  explicit MVELaneInterleaving() : FunctionPass(ID) {
+    initializeMVELaneInterleavingPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnFunction(Function &F) override;
+
+  StringRef getPassName() const override { return "MVE lane interleaving"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<TargetPassConfig>();
+    FunctionPass::getAnalysisUsage(AU);
+  }
+};
+
+} // end anonymous namespace
+
+char MVELaneInterleaving::ID = 0;
+
+INITIALIZE_PASS(MVELaneInterleaving, DEBUG_TYPE, "MVE lane interleaving", false,
+                false)
+
+Pass *llvm::createMVELaneInterleavingPass() {
+  return new MVELaneInterleaving();
+}
+
+static bool isProfitableToInterleave(SmallSetVector<Instruction *, 4> &Exts,
+                                     SmallSetVector<Instruction *, 4> &Truncs) {
+  // This is not always beneficial to transform. Exts can be incorporated into
+  // loads, Truncs can be folded into stores.
+  // Truncs are usually the same number of instructions,
+  //  VSTRH.32(A);VSTRH.32(B) vs VSTRH.16(VMOVNT A, B) with interleaving
+  // Exts are unfortunately more instructions in the general case:
+  //  A=VLDRH.32; B=VLDRH.32;
+  // vs with interleaving:
+  //  T=VLDRH.16; A=VMOVNB T; B=VMOVNT T
+  // But those VMOVL may be folded into a VMULL.
+
+  // But expensive extends/truncs are always good to remove.
+  for (auto *E : Exts)
+    if (!isa<LoadInst>(E->getOperand(0))) {
+      LLVM_DEBUG(dbgs() << "Beneficial due to " << *E << "\n");
+      return true;
+    }
+  for (auto *T : Truncs)
+    if (T->hasOneUse() && !isa<StoreInst>(*T->user_begin())) {
+      LLVM_DEBUG(dbgs() << "Beneficial due to " << *T << "\n");
+      return true;
+    }
+
+  // Otherwise, we know we have a load(ext), see if any of the Extends are a
+  // vmull. This is a simple heuristic and certainly not perfect.
+  for (auto *E : Exts) {
+    if (!E->hasOneUse() ||
+        cast<Instruction>(*E->user_begin())->getOpcode() != Instruction::Mul) {
+      LLVM_DEBUG(dbgs() << "Not beneficial due to " << *E << "\n");
+      return false;
+    }
+  }
+  return true;
+}
+
+static bool tryInterleave(Instruction *Start,
+                          SmallPtrSetImpl<Instruction *> &Visited) {
+  LLVM_DEBUG(dbgs() << "tryInterleave from " << *Start << "\n");
+  auto *VT = cast<FixedVectorType>(Start->getType());
+
+  if (!isa<Instruction>(Start->getOperand(0)))
+    return false;
+
+  // Look for connected operations starting from Ext's, terminating at Truncs.
+  std::vector<Instruction *> Worklist;
+  Worklist.push_back(Start);
+  Worklist.push_back(cast<Instruction>(Start->getOperand(0)));
+
+  SmallSetVector<Instruction *, 4> Truncs;
+  SmallSetVector<Instruction *, 4> Exts;
+  SmallSetVector<Use *, 4> OtherLeafs;
+  SmallSetVector<Instruction *, 4> Ops;
+
+  while (!Worklist.empty()) {
+    Instruction *I = Worklist.back();
+    Worklist.pop_back();
+
+    switch (I->getOpcode()) {
+    // Truncs
+    case Instruction::Trunc:
+      if (Truncs.count(I))
+        continue;
+      Truncs.insert(I);
+      Visited.insert(I);
+      break;
+
+    // Extend leafs
+    case Instruction::SExt:
+    case Instruction::ZExt:
+      if (Exts.count(I))
+        continue;
+      for (auto *Use : I->users())
+        Worklist.push_back(cast<Instruction>(Use));
+      Exts.insert(I);
+      break;
+
+    // Binary/tertiary ops
+    case Instruction::Add:
+    case Instruction::Sub:
+    case Instruction::Mul:
+    case Instruction::AShr:
+    case Instruction::LShr:
+    case Instruction::Shl:
+    case Instruction::ICmp:
+    case Instruction::Select:
+      if (Ops.count(I))
+        continue;
+      Ops.insert(I);
+
+      for (Use &Op : I->operands()) {
+        if (isa<Instruction>(Op))
+          Worklist.push_back(cast<Instruction>(&Op));
+        else
+          OtherLeafs.insert(&Op);
+      }
+
+      for (auto *Use : I->users())
+        Worklist.push_back(cast<Instruction>(Use));
+      break;
+
+    default:
+      LLVM_DEBUG(dbgs() << "  Unhandled instruction: " << *I << "\n");
+      return false;
+    }
+  }
+
+  if (Exts.empty() && OtherLeafs.empty())
+    return false;
+
+  LLVM_DEBUG({
+    dbgs() << "Found group:\n  Exts:";
+    for (auto *I : Exts)
+      dbgs() << "  " << *I << "\n";
+    dbgs() << "  Ops:";
+    for (auto *I : Ops)
+      dbgs() << "  " << *I << "\n";
+    dbgs() << "  OtherLeafs:";
+    for (auto *I : OtherLeafs)
+      dbgs() << "  " << *I << "\n";
+    dbgs() << "Truncs:";
+    for (auto *I : Truncs)
+      dbgs() << "  " << *I << "\n";
+  });
+
+  assert(!Truncs.empty() && "Expected some truncs");
+
+  // Check types
+  unsigned NumElts = VT->getNumElements();
+  unsigned BaseElts = VT->getScalarSizeInBits() == 16
+                          ? 8
+                          : (VT->getScalarSizeInBits() == 8 ? 16 : 0);
+  if (BaseElts == 0 || NumElts % BaseElts != 0) {
+    LLVM_DEBUG(dbgs() << "  Type is unsupported\n");
+    return false;
+  }
+  if (Start->getOperand(0)->getType()->getScalarSizeInBits() !=
+      VT->getScalarSizeInBits() * 2) {
+    LLVM_DEBUG(dbgs() << "  Type not double sized\n");
+    return false;
+  }
+  for (Instruction *I : Exts)
+    if (I->getOperand(0)->getType() != VT) {
+      LLVM_DEBUG(dbgs() << "  Wrong type on " << *I << "\n");
+      return false;
+    }
+  for (Instruction *I : Truncs)
+    if (I->getType() != VT) {
+      LLVM_DEBUG(dbgs() << "  Wrong type on " << *I << "\n");
+      return false;
+    }
+
+  // Check that it looks beneficial
+  if (!isProfitableToInterleave(Exts, Truncs))
+    return false;
+
+  // Create new shuffles around the extends / truncs / other leaves.
+  IRBuilder<> Builder(Start);
+
+  SmallVector<int, 16> LeafMask;
+  SmallVector<int, 16> TruncMask;
+  // LeafMask : 0, 2, 4, 6, 1, 3, 5, 7   8, 10, 12, 14,  9, 11, 13, 15
+  // TruncMask: 0, 4, 1, 5, 2, 6, 3, 7   8, 12,  9, 13, 10, 14, 11, 15
+  for (unsigned Base = 0; Base < NumElts; Base += BaseElts) {
+    for (unsigned i = 0; i < BaseElts / 2; i++)
+      LeafMask.push_back(Base + i * 2);
+    for (unsigned i = 0; i < BaseElts / 2; i++)
+      LeafMask.push_back(Base + i * 2 + 1);
+  }
+  for (unsigned Base = 0; Base < NumElts; Base += BaseElts) {
+    for (unsigned i = 0; i < BaseElts / 2; i++) {
+      TruncMask.push_back(Base + i);
+      TruncMask.push_back(Base + i + BaseElts / 2);
+    }
+  }
+
+  for (Instruction *I : Exts) {
+    LLVM_DEBUG(dbgs() << "Replacing ext " << *I << "\n");
+    Builder.SetInsertPoint(I);
+    Value *Shuffle = Builder.CreateShuffleVector(I->getOperand(0), LeafMask);
+    bool Sext = isa<SExtInst>(I);
+    Value *Ext = Sext ? Builder.CreateSExt(Shuffle, I->getType())
+                      : Builder.CreateZExt(Shuffle, I->getType());
+    I->replaceAllUsesWith(Ext);
+    LLVM_DEBUG(dbgs() << "  with " << *Shuffle << "\n");
+  }
+
+  for (Use *I : OtherLeafs) {
+    LLVM_DEBUG(dbgs() << "Replacing leaf " << *I << "\n");
+    Builder.SetInsertPoint(cast<Instruction>(I->getUser()));
+    Value *Shuffle = Builder.CreateShuffleVector(I->get(), LeafMask);
+    I->getUser()->setOperand(I->getOperandNo(), Shuffle);
+    LLVM_DEBUG(dbgs() << "  with " << *Shuffle << "\n");
+  }
+
+  for (Instruction *I : Truncs) {
+    LLVM_DEBUG(dbgs() << "Replacing trunc " << *I << "\n");
+
+    Builder.SetInsertPoint(I->getParent(), ++I->getIterator());
+    Value *Shuf = Builder.CreateShuffleVector(I, TruncMask);
+    I->replaceAllUsesWith(Shuf);
+    cast<Instruction>(Shuf)->setOperand(0, I);
+
+    LLVM_DEBUG(dbgs() << "  with " << *Shuf << "\n");
+  }
+
+  return true;
+}
+
+bool MVELaneInterleaving::runOnFunction(Function &F) {
+  if (!EnableInterleave)
+    return false;
+  auto &TPC = getAnalysis<TargetPassConfig>();
+  auto &TM = TPC.getTM<TargetMachine>();
+  auto *ST = &TM.getSubtarget<ARMSubtarget>(F);
+  if (!ST->hasMVEIntegerOps())
+    return false;
+
+  bool Changed = false;
+
+  SmallPtrSet<Instruction *, 16> Visited;
+  for (Instruction &I : reverse(instructions(F))) {
+    if (I.getType()->isVectorTy() &&
+        (isa<TruncInst>(I) || isa<FPTruncInst>(I)) && !Visited.count(&I))
+      Changed |= tryInterleave(&I, Visited);
+  }
+
+  return Changed;
+}
diff --git a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp
index f21ea278ccd4e66a3e9ab9285b8d922d0df843a1..3fd404e6d55b3fe120a8eb5dd36776cd0e5c6906 100644
--- a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp
+++ b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp
@@ -273,11 +273,28 @@ bool MVETPAndVPTOptimisations::MergeLoopEnd(MachineLoop *ML) {
 
   // Check if there is an illegal instruction (a call) in the low overhead loop
   // and if so revert it now before we get any further. While loops also need to
-  // check the preheaders.
-  SmallPtrSet<MachineBasicBlock *, 4> MBBs(ML->block_begin(), ML->block_end());
-  if (LoopStart->getOpcode() == ARM::t2WhileLoopStartLR)
-    MBBs.insert(ML->getHeader()->pred_begin(), ML->getHeader()->pred_end());
-  for (MachineBasicBlock *MBB : MBBs) {
+  // check the preheaders, but can be reverted to a DLS loop if needed.
+  auto *PreHeader = ML->getLoopPreheader();
+  if (LoopStart->getOpcode() == ARM::t2WhileLoopStartLR && PreHeader &&
+      LoopStart->getParent() != PreHeader) {
+    for (MachineInstr &MI : *PreHeader) {
+      if (MI.isCall()) {
+        // Create a t2DoLoopStart at the end of the preheader.
+        MachineInstrBuilder MIB =
+            BuildMI(*PreHeader, PreHeader->getFirstTerminator(),
+                    LoopStart->getDebugLoc(), TII->get(ARM::t2DoLoopStart));
+        MIB.add(LoopStart->getOperand(0));
+        MIB.add(LoopStart->getOperand(1));
+
+        // Revert the t2WhileLoopStartLR to a CMP and Br.
+        RevertWhileLoopStartLR(LoopStart, TII, ARM::t2Bcc, true);
+        LoopStart = MIB;
+        break;
+      }
+    }
+  }
+
+  for (MachineBasicBlock *MBB : ML->blocks()) {
     for (MachineInstr &MI : *MBB) {
       if (MI.isCall()) {
         LLVM_DEBUG(dbgs() << "Found call in loop, reverting: " << MI);
diff --git a/llvm/lib/Target/ARM/MVETailPredUtils.h b/llvm/lib/Target/ARM/MVETailPredUtils.h
index 4798c6612b79eba163a9a77a41f12aa969dde924..b0c003120fa5b62c7222b57c0e467f8ad5cba1f8 100644
--- a/llvm/lib/Target/ARM/MVETailPredUtils.h
+++ b/llvm/lib/Target/ARM/MVETailPredUtils.h
@@ -77,24 +77,38 @@ static inline bool isLoopStart(MachineInstr &MI) {
 
 // WhileLoopStart holds the exit block, so produce a subs Op0, Op1, 0 and then a
 // beq that branches to the exit branch.
+// If UseCmp is true, this will create a t2CMP instead of a t2SUBri, meaning the
+// value of LR into the loop will not be setup. This is used if the LR setup is
+// done via another means (via a t2DoLoopStart, for example).
 inline void RevertWhileLoopStartLR(MachineInstr *MI, const TargetInstrInfo *TII,
-                                   unsigned BrOpc = ARM::t2Bcc) {
+                                   unsigned BrOpc = ARM::t2Bcc,
+                                   bool UseCmp = false) {
   MachineBasicBlock *MBB = MI->getParent();
   assert(MI->getOpcode() == ARM::t2WhileLoopStartLR &&
          "Only expected a t2WhileLoopStartLR in RevertWhileLoopStartLR!");
 
-  // Subs
-  MachineInstrBuilder MIB =
-      BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2SUBri));
-  MIB.add(MI->getOperand(0));
-  MIB.add(MI->getOperand(1));
-  MIB.addImm(0);
-  MIB.addImm(ARMCC::AL);
-  MIB.addReg(ARM::NoRegister);
-  MIB.addReg(ARM::CPSR, RegState::Define);
+  // Subs/Cmp
+  if (UseCmp) {
+    MachineInstrBuilder MIB =
+        BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2CMPri));
+    MIB.add(MI->getOperand(1));
+    MIB.addImm(0);
+    MIB.addImm(ARMCC::AL);
+    MIB.addReg(ARM::NoRegister);
+  } else {
+    MachineInstrBuilder MIB =
+        BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2SUBri));
+    MIB.add(MI->getOperand(0));
+    MIB.add(MI->getOperand(1));
+    MIB.addImm(0);
+    MIB.addImm(ARMCC::AL);
+    MIB.addReg(ARM::NoRegister);
+    MIB.addReg(ARM::CPSR, RegState::Define);
+  }
 
   // Branch
-  MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
+  MachineInstrBuilder MIB =
+      BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
   MIB.add(MI->getOperand(2)); // branch target
   MIB.addImm(ARMCC::EQ);      // condition code
   MIB.addReg(ARM::CPSR);
diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp
index da7ec32703a5b42daff2de8c39327f4ee8cf7298..bedf159430dc4b06f5c72bc2c4da7e0abc05bfd5 100644
--- a/llvm/lib/Target/BPF/BTFDebug.cpp
+++ b/llvm/lib/Target/BPF/BTFDebug.cpp
@@ -1224,8 +1224,8 @@ void BTFDebug::processGlobals(bool ProcessingMapDef) {
     const DataLayout &DL = Global.getParent()->getDataLayout();
     uint32_t Size = DL.getTypeAllocSize(Global.getType()->getElementType());
 
-    DataSecEntries[std::string(SecName)]->addVar(VarId, Asm->getSymbol(&Global),
-                                                 Size);
+    DataSecEntries[std::string(SecName)]->addDataSecEntry(VarId,
+        Asm->getSymbol(&Global), Size);
   }
 }
 
@@ -1303,7 +1303,19 @@ void BTFDebug::processFuncPrototypes(const Function *F) {
   uint8_t Scope = BTF::FUNC_EXTERN;
   auto FuncTypeEntry =
       std::make_unique<BTFTypeFunc>(SP->getName(), ProtoTypeId, Scope);
-  addType(std::move(FuncTypeEntry));
+  uint32_t FuncId = addType(std::move(FuncTypeEntry));
+  if (F->hasSection()) {
+    StringRef SecName = F->getSection();
+
+    if (DataSecEntries.find(std::string(SecName)) == DataSecEntries.end()) {
+      DataSecEntries[std::string(SecName)] =
+          std::make_unique<BTFKindDataSec>(Asm, std::string(SecName));
+    }
+
+    // We really don't know func size, set it to 0.
+    DataSecEntries[std::string(SecName)]->addDataSecEntry(FuncId,
+        Asm->getSymbol(F), 0);
+  }
 }
 
 void BTFDebug::endModule() {
diff --git a/llvm/lib/Target/BPF/BTFDebug.h b/llvm/lib/Target/BPF/BTFDebug.h
index fb20ec59574baf98716ff25b5804314c1ea0a568..76f1901779bb726aef2b46847c9fcb0b7d9ac055 100644
--- a/llvm/lib/Target/BPF/BTFDebug.h
+++ b/llvm/lib/Target/BPF/BTFDebug.h
@@ -187,7 +187,7 @@ public:
   uint32_t getSize() override {
     return BTFTypeBase::getSize() + BTF::BTFDataSecVarSize * Vars.size();
   }
-  void addVar(uint32_t Id, const MCSymbol *Sym, uint32_t Size) {
+  void addDataSecEntry(uint32_t Id, const MCSymbol *Sym, uint32_t Size) {
     Vars.push_back(std::make_tuple(Id, Sym, Size));
   }
   std::string getName() { return Name; }
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 30c3d3d4f5701b808fdc183d6a661b311a7745f7..153c7e9d9489048de026c70ea9899373e079b5b9 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -308,6 +308,8 @@ Register HexagonTargetLowering::getRegisterByName(
                      .Case("m1", Hexagon::M1)
                      .Case("usr", Hexagon::USR)
                      .Case("ugp", Hexagon::UGP)
+                     .Case("cs0", Hexagon::CS0)
+                     .Case("cs1", Hexagon::CS1)
                      .Default(Register());
   if (Reg)
     return Reg;
@@ -701,7 +703,7 @@ SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
                                                      SelectionDAG &DAG) const {
   SDValue Chain = Op.getOperand(0);
   SDLoc dl(Op);
-  SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
+  SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
   return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain);
 }
 
diff --git a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
index cd894c555adcd2cc39aa2423227fe824586adc62..c2875bc8b1c0fb986ce13a165527f9ebcd062bf8 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
@@ -95,21 +95,41 @@ def unalignedstore: PatFrag<(ops node:$v, node:$a), (store $v, $a), [{
 
 // HVX loads
 
-multiclass HvxLd_pat<InstHexagon MI, PatFrag Load, ValueType ResType,
+multiclass HvxLdfi_pat<InstHexagon MI, PatFrag Load, ValueType ResType,
+                       PatFrag ImmPred> {
+  def: Pat<(ResType (Load (add (i32 AddrFI:$fi), ImmPred:$Off))),
+           (MI AddrFI:$fi, imm:$Off)>;
+  def: Pat<(ResType (Load (IsOrAdd (i32 AddrFI:$fi), ImmPred:$Off))),
+           (MI AddrFI:$fi, imm:$Off)>;
+  def: Pat<(ResType (Load AddrFI:$fi)), (ResType (MI AddrFI:$fi, 0))>;
+}
+
+multiclass HvxLdgi_pat<InstHexagon MI, PatFrag Load, ValueType ResType,
                      PatFrag ImmPred> {
+  def: Pat<(ResType (Load (add I32:$Rt, ImmPred:$Off))),
+           (MI I32:$Rt, imm:$Off)>;
   def: Pat<(ResType (Load I32:$Rt)),
            (MI I32:$Rt, 0)>;
-  def: Pat<(ResType (Load (add I32:$Rt, ImmPred:$s))),
-           (MI I32:$Rt, imm:$s)>;
+}
+
+multiclass HvxLdc_pat<InstHexagon MI, PatFrag Load, ValueType ResType> {
   // The HVX selection code for shuffles can generate vector constants.
   // Calling "Select" on the resulting loads from CP fails without these
   // patterns.
-  def: Pat<(ResType (Load (HexagonCP tconstpool:$A))),
-           (MI (A2_tfrsi imm:$A), 0)>;
-  def: Pat<(ResType (Load (HexagonAtPcrel tconstpool:$A))),
-           (MI (C4_addipc imm:$A), 0)>;
+  def: Pat<(ResType (Load (HexagonCP tconstpool:$Addr))),
+           (MI (A2_tfrsi imm:$Addr), 0)>;
+  def: Pat<(ResType (Load (HexagonAtPcrel tconstpool:$Addr))),
+           (MI (C4_addipc imm:$Addr), 0)>;
+}
+
+multiclass HvxLd_pat<InstHexagon MI, PatFrag Load, ValueType ResType,
+                     PatFrag ImmPred> {
+  defm: HvxLdfi_pat<MI, Load, ResType, ImmPred>;
+  defm: HvxLdgi_pat<MI, Load, ResType, ImmPred>;
+  defm: HvxLdc_pat <MI, Load, ResType>;
 }
 
+// Aligned loads: everything, plus loads with valignaddr node.
 multiclass HvxLda_pat<InstHexagon MI, PatFrag Load, ValueType ResType,
                       PatFrag ImmPred> {
   let AddedComplexity = 50 in {
@@ -122,41 +142,61 @@ multiclass HvxLda_pat<InstHexagon MI, PatFrag Load, ValueType ResType,
 }
 
 let Predicates = [UseHVX] in {
+  // alignedload will match a non-temporal load as well, so try non-temporal
+  // first.
   defm: HvxLda_pat<V6_vL32b_nt_ai, alignednontemporalload, VecI8,  IsVecOff>;
   defm: HvxLda_pat<V6_vL32b_nt_ai, alignednontemporalload, VecI16, IsVecOff>;
   defm: HvxLda_pat<V6_vL32b_nt_ai, alignednontemporalload, VecI32, IsVecOff>;
+  defm: HvxLda_pat<V6_vL32b_ai,               alignedload, VecI8,  IsVecOff>;
+  defm: HvxLda_pat<V6_vL32b_ai,               alignedload, VecI16, IsVecOff>;
+  defm: HvxLda_pat<V6_vL32b_ai,               alignedload, VecI32, IsVecOff>;
 
-  defm: HvxLda_pat<V6_vL32b_ai, alignedload, VecI8,  IsVecOff>;
-  defm: HvxLda_pat<V6_vL32b_ai, alignedload, VecI16, IsVecOff>;
-  defm: HvxLda_pat<V6_vL32b_ai, alignedload, VecI32, IsVecOff>;
-
-  defm: HvxLd_pat<V6_vL32Ub_ai,  unalignedload, VecI8,  IsVecOff>;
-  defm: HvxLd_pat<V6_vL32Ub_ai,  unalignedload, VecI16, IsVecOff>;
-  defm: HvxLd_pat<V6_vL32Ub_ai,  unalignedload, VecI32, IsVecOff>;
+  defm: HvxLd_pat<V6_vL32Ub_ai,             unalignedload, VecI8,  IsVecOff>;
+  defm: HvxLd_pat<V6_vL32Ub_ai,             unalignedload, VecI16, IsVecOff>;
+  defm: HvxLd_pat<V6_vL32Ub_ai,             unalignedload, VecI32, IsVecOff>;
 }
 
+
 // HVX stores
 
-multiclass HvxSt_pat<InstHexagon MI, PatFrag Store, PatFrag ImmPred,
-                     PatFrag Value> {
+multiclass HvxStfi_pat<InstHexagon MI, PatFrag Store, PatFrag Value,
+                       PatFrag ImmPred> {
+  def: Pat<(Store Value:$Vs, (add (i32 AddrFI:$fi), ImmPred:$Off)),
+           (MI AddrFI:$fi, imm:$Off, Value:$Vs)>;
+  def: Pat<(Store Value:$Vs, (IsOrAdd (i32 AddrFI:$fi), ImmPred:$Off)),
+           (MI AddrFI:$fi, imm:$Off, Value:$Vs)>;
+  def: Pat<(Store Value:$Vs, AddrFI:$fi),
+           (MI AddrFI:$fi, 0, Value:$Vs)>;
+}
+
+multiclass HvxStgi_pat<InstHexagon MI, PatFrag Store, PatFrag Value,
+                       PatFrag ImmPred> {
+  def: Pat<(Store Value:$Vs, (add I32:$Rt, ImmPred:$Off)),
+           (MI I32:$Rt, imm:$Off, Value:$Vs)>;
+  def: Pat<(Store Value:$Vs, (IsOrAdd I32:$Rt, ImmPred:$Off)),
+           (MI I32:$Rt, imm:$Off, Value:$Vs)>;
   def: Pat<(Store Value:$Vs, I32:$Rt),
            (MI I32:$Rt, 0, Value:$Vs)>;
-  def: Pat<(Store Value:$Vs, (add I32:$Rt, ImmPred:$s)),
-           (MI I32:$Rt, imm:$s, Value:$Vs)>;
 }
 
-let Predicates = [UseHVX] in {
-  defm: HvxSt_pat<V6_vS32b_nt_ai, alignednontemporalstore, IsVecOff, HVI8>;
-  defm: HvxSt_pat<V6_vS32b_nt_ai, alignednontemporalstore, IsVecOff, HVI16>;
-  defm: HvxSt_pat<V6_vS32b_nt_ai, alignednontemporalstore, IsVecOff, HVI32>;
-
-  defm: HvxSt_pat<V6_vS32b_ai, alignedstore, IsVecOff, HVI8>;
-  defm: HvxSt_pat<V6_vS32b_ai, alignedstore, IsVecOff, HVI16>;
-  defm: HvxSt_pat<V6_vS32b_ai, alignedstore, IsVecOff, HVI32>;
+multiclass HvxSt_pat<InstHexagon MI, PatFrag Store, PatFrag Value,
+                     PatFrag ImmPred> {
+  defm: HvxStfi_pat<MI, Store, Value, ImmPred>;
+  defm: HvxStgi_pat<MI, Store, Value, ImmPred>;
+}
 
-  defm: HvxSt_pat<V6_vS32Ub_ai, unalignedstore, IsVecOff, HVI8>;
-  defm: HvxSt_pat<V6_vS32Ub_ai, unalignedstore, IsVecOff, HVI16>;
-  defm: HvxSt_pat<V6_vS32Ub_ai, unalignedstore, IsVecOff, HVI32>;
+let Predicates = [UseHVX] in {
+  // alignedstore will match a non-temporal store as well, so try non-temporal
+  // first.
+  defm: HvxSt_pat<V6_vS32b_nt_ai, alignednontemporalstore,  HVI8, IsVecOff>;
+  defm: HvxSt_pat<V6_vS32b_nt_ai, alignednontemporalstore, HVI16, IsVecOff>;
+  defm: HvxSt_pat<V6_vS32b_nt_ai, alignednontemporalstore, HVI32, IsVecOff>;
+  defm: HvxSt_pat<V6_vS32b_ai,               alignedstore,  HVI8, IsVecOff>;
+  defm: HvxSt_pat<V6_vS32b_ai,               alignedstore, HVI16, IsVecOff>;
+  defm: HvxSt_pat<V6_vS32b_ai,               alignedstore, HVI32, IsVecOff>;
+  defm: HvxSt_pat<V6_vS32Ub_ai,            unalignedstore,  HVI8, IsVecOff>;
+  defm: HvxSt_pat<V6_vS32Ub_ai,            unalignedstore, HVI16, IsVecOff>;
+  defm: HvxSt_pat<V6_vS32Ub_ai,            unalignedstore, HVI32, IsVecOff>;
 }
 
 // Bitcasts between same-size vector types are no-ops, except for the
diff --git a/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp b/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp
index 5ece577e82851fb960348285c1122135615c939a..6e55bc6b5c2cdf53b7ee21359520f8dcaf2fa024 100644
--- a/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp
@@ -17,8 +17,10 @@
 #include "HexagonSubtarget.h"
 #include "HexagonTargetMachine.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -30,6 +32,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Type.h"
 #include "llvm/MC/MachineLocation.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
@@ -41,6 +44,15 @@
 
 using namespace llvm;
 
+static cl::opt<unsigned> FrameIndexSearchRange(
+    "hexagon-frame-index-search-range", cl::init(32), cl::Hidden,
+    cl::desc("Limit on instruction search range in frame index elimination"));
+
+static cl::opt<unsigned> FrameIndexReuseLimit(
+    "hexagon-frame-index-reuse-limit", cl::init(~0), cl::Hidden,
+    cl::desc("Limit on the number of reused registers in frame index "
+    "elimination"));
+
 HexagonRegisterInfo::HexagonRegisterInfo(unsigned HwMode)
     : HexagonGenRegisterInfo(Hexagon::R31, 0/*DwarfFlavor*/, 0/*EHFlavor*/,
                              0/*PC*/, HwMode) {}
@@ -133,7 +145,7 @@ const uint32_t *HexagonRegisterInfo::getCallPreservedMask(
 
 
 BitVector HexagonRegisterInfo::getReservedRegs(const MachineFunction &MF)
-  const {
+      const {
   BitVector Reserved(getNumRegs());
   Reserved.set(Hexagon::R29);
   Reserved.set(Hexagon::R30);
@@ -188,10 +200,10 @@ BitVector HexagonRegisterInfo::getReservedRegs(const MachineFunction &MF)
   return Reserved;
 }
 
-
 void HexagonRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
                                               int SPAdj, unsigned FIOp,
                                               RegScavenger *RS) const {
+  static unsigned ReuseCount = 0;
   //
   // Hexagon_TODO: Do we need to enforce this for Hexagon?
   assert(SPAdj == 0 && "Unexpected");
@@ -210,7 +222,6 @@ void HexagonRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   int Offset = HFI.getFrameIndexReference(MF, FI, BP).getFixed();
   // Add the offset from the instruction.
   int RealOffset = Offset + MI.getOperand(FIOp+1).getImm();
-  bool IsKill = false;
 
   unsigned Opc = MI.getOpcode();
   switch (Opc) {
@@ -228,18 +239,107 @@ void HexagonRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   if (!HII.isValidOffset(Opc, RealOffset, this)) {
     // If the offset is not valid, calculate the address in a temporary
     // register and use it with offset 0.
+    int InstOffset = 0;
+    // The actual base register (BP) is typically shared between many
+    // instructions where frame indices are being replaced. In scalar
+    // instructions the offset range is large, and the need for an extra
+    // add instruction is infrequent. Vector loads/stores, however, have
+    // a much smaller offset range: [-8, 7), or #s4. In those cases it
+    // makes sense to "standardize" the immediate in the "addi" instruction
+    // so that multiple loads/stores could be based on it.
+    bool IsPair = false;
+    switch (MI.getOpcode()) {
+      // All of these instructions have the same format: base+#s4.
+      case Hexagon::PS_vloadrw_ai:
+      case Hexagon::PS_vloadrw_nt_ai:
+      case Hexagon::PS_vstorerw_ai:
+      case Hexagon::PS_vstorerw_nt_ai:
+        IsPair = true;
+        LLVM_FALLTHROUGH;
+      case Hexagon::PS_vloadrv_ai:
+      case Hexagon::PS_vloadrv_nt_ai:
+      case Hexagon::PS_vstorerv_ai:
+      case Hexagon::PS_vstorerv_nt_ai:
+      case Hexagon::V6_vL32b_ai:
+      case Hexagon::V6_vS32b_ai: {
+        unsigned HwLen = HST.getVectorLength();
+        if (RealOffset % HwLen == 0) {
+          int VecOffset = RealOffset / HwLen;
+          // Rewrite the offset as "base + [-8, 7)".
+          VecOffset += 8;
+          // Pairs are expanded into two instructions: make sure that both
+          // can use the same base (i.e. VecOffset+1 is not a different
+          // multiple of 16 than VecOffset).
+          if (!IsPair || (VecOffset + 1) % 16 != 0) {
+            RealOffset = (VecOffset & -16) * HwLen;
+            InstOffset = (VecOffset % 16 - 8) * HwLen;
+          }
+        }
+      }
+    }
+
+    // Search backwards in the block for "Reg = A2_addi BP, RealOffset".
+    // This will give us a chance to avoid creating a new register.
+    Register ReuseBP;
+
+    if (ReuseCount < FrameIndexReuseLimit) {
+      unsigned SearchCount = 0, SearchRange = FrameIndexSearchRange;
+      SmallSet<Register,2> SeenVRegs;
+      bool PassedCall = false;
+      LiveRegUnits Defs(*this), Uses(*this);
+
+      for (auto I = std::next(II.getReverse()), E = MB.rend(); I != E; ++I) {
+        if (SearchCount == SearchRange)
+          break;
+        ++SearchCount;
+        const MachineInstr &BI = *I;
+        LiveRegUnits::accumulateUsedDefed(BI, Defs, Uses, this);
+        PassedCall |= BI.isCall();
+        for (const MachineOperand &Op : BI.operands()) {
+          if (SeenVRegs.size() > 1)
+            break;
+          if (Op.isReg() && Op.getReg().isVirtual())
+            SeenVRegs.insert(Op.getReg());
+        }
+        if (BI.getOpcode() != Hexagon::A2_addi)
+          continue;
+        if (BI.getOperand(1).getReg() != BP)
+          continue;
+        const auto &Op2 = BI.getOperand(2);
+        if (!Op2.isImm() || Op2.getImm() != RealOffset)
+          continue;
+
+        Register R = BI.getOperand(0).getReg();
+        if (R.isPhysical()) {
+          if (Defs.available(R))
+            ReuseBP = R;
+        } else if (R.isVirtual()) {
+          // Extending a range of a virtual register can be dangerous,
+          // since the scavenger will need to find a physical register
+          // for it. Avoid extending the range past a function call,
+          // and avoid overlapping it with another virtual register.
+          if (!PassedCall && SeenVRegs.size() <= 1)
+            ReuseBP = R;
+        }
+        break;
+      }
+      if (ReuseBP)
+        ++ReuseCount;
+    }
+
     auto &MRI = MF.getRegInfo();
-    Register TmpR = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
-    const DebugLoc &DL = MI.getDebugLoc();
-    BuildMI(MB, II, DL, HII.get(Hexagon::A2_addi), TmpR)
-      .addReg(BP)
-      .addImm(RealOffset);
-    BP = TmpR;
-    RealOffset = 0;
-    IsKill = true;
+    if (!ReuseBP) {
+      ReuseBP = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
+      const DebugLoc &DL = MI.getDebugLoc();
+      BuildMI(MB, II, DL, HII.get(Hexagon::A2_addi), ReuseBP)
+        .addReg(BP)
+        .addImm(RealOffset);
+    }
+    BP = ReuseBP;
+    RealOffset = InstOffset;
   }
 
-  MI.getOperand(FIOp).ChangeToRegister(BP, false, false, IsKill);
+  MI.getOperand(FIOp).ChangeToRegister(BP, false, false, false);
   MI.getOperand(FIOp+1).ChangeToImmediate(RealOffset);
 }
 
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 063a5571c13b66c9276455959ec1a05388c8bfb5..bfbba49949389d11bbb2cfb0220ef397c60a78a7 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -98,8 +98,18 @@ unsigned HexagonTTIImpl::getMaxInterleaveFactor(unsigned VF) {
   return useHVX() ? 2 : 1;
 }
 
-unsigned HexagonTTIImpl::getRegisterBitWidth(bool Vector) const {
-  return Vector ? getMinVectorRegisterBitWidth() : 32;
+TypeSize
+HexagonTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+  switch (K) {
+  case TargetTransformInfo::RGK_Scalar:
+    return TypeSize::getFixed(32);
+  case TargetTransformInfo::RGK_FixedWidthVector:
+    return TypeSize::getFixed(getMinVectorRegisterBitWidth());
+  case TargetTransformInfo::RGK_ScalableVector:
+    return TypeSize::getScalable(0);
+  }
+
+  llvm_unreachable("Unsupported register kind");
 }
 
 unsigned HexagonTTIImpl::getMinVectorRegisterBitWidth() const {
@@ -162,7 +172,9 @@ unsigned HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
     VectorType *VecTy = cast<VectorType>(Src);
     unsigned VecWidth = VecTy->getPrimitiveSizeInBits().getFixedSize();
     if (useHVX() && ST.isTypeForHVX(VecTy)) {
-      unsigned RegWidth = getRegisterBitWidth(true);
+      unsigned RegWidth =
+          getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+              .getFixedSize();
       assert(RegWidth && "Non-zero vector register width expected");
       // Cost of HVX loads.
       if (VecWidth % RegWidth == 0)
@@ -207,7 +219,8 @@ unsigned HexagonTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
 }
 
 unsigned HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
-      int Index, Type *SubTp) {
+                                        ArrayRef<int> Mask, int Index,
+                                        Type *SubTp) {
   return 1;
 }
 
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 61d50b9457a6f1a7cf2706510a4c92a5e45419e4..fd04d1df0dea6159db3717b4aba79551c2ea58fa 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -81,7 +81,7 @@ public:
 
   unsigned getNumberOfRegisters(bool vector) const;
   unsigned getMaxInterleaveFactor(unsigned VF);
-  unsigned getRegisterBitWidth(bool Vector) const;
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
   unsigned getMinVectorRegisterBitWidth() const;
   ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const;
 
@@ -122,8 +122,8 @@ public:
   getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                         unsigned AddressSpace,
                         TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency);
-  unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
-            Type *SubTp);
+  unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, ArrayRef<int> Mask,
+                          int Index, Type *SubTp);
   unsigned getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
                                   const Value *Ptr, bool VariableMask,
                                   Align Alignment, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/M68k/M68kInstrArithmetic.td b/llvm/lib/Target/M68k/M68kInstrArithmetic.td
index f4714d2534bda6f075419b18e48e3f454bb552bb..81286c8f162c2921ac2edce1d7c62e269cbf8a57 100644
--- a/llvm/lib/Target/M68k/M68kInstrArithmetic.td
+++ b/llvm/lib/Target/M68k/M68kInstrArithmetic.td
@@ -38,7 +38,7 @@
 ///             |         |         | EFFECTIVE ADDRESS
 ///  x  x  x  x |   REG   | OP MODE |   MODE  |   REG
 /// ----------------------------------------------------
-class MxArithEncoding<MxBead4Bits CMD, MxEncOpMode OPMODE, MxBeadReg REG,
+class MxArithEncoding<MxBead4Bits CMD, MxEncOpMode OPMODE, MxBead REG,
                       MxEncEA EA, MxEncExt EXT>
     : MxEncoding<EA.Reg, EA.DA, EA.Mode, OPMODE.B0, OPMODE.B1, OPMODE.B2, REG,
                  CMD,EXT.Imm, EXT.B8, EXT.Scale, EXT.WL, EXT.DAReg>;
@@ -53,7 +53,7 @@ class MxArithEncoding<MxBead4Bits CMD, MxEncOpMode OPMODE, MxBeadReg REG,
 /// Ry - source
 /// M  - address mode switch
 class MxArithXEncoding<MxBead4Bits CMD, MxEncSize SIZE, MxBead1Bit MODE,
-                       MxBeadReg SRC, MxBeadReg DST>
+                       MxBeadDReg SRC, MxBeadDReg DST>
     : MxEncoding<SRC, MODE, MxBead2Bits<0b00>, SIZE, MxBead1Bit<0b1>, DST, CMD>;
 
 /// Encoding for Immediate forms
@@ -88,13 +88,13 @@ let Defs = [CCR] in {
 let Constraints = "$src = $dst" in {
 
 // $reg, $ccr <- $reg op $reg
-class MxBiArOp_RFRR_xEA<string MN, SDNode NODE, MxType TYPE, bits<4> CMD>
+class MxBiArOp_RFRR_xEA<string MN, SDNode NODE, MxType TYPE, bits<4> CMD, MxBead REG>
     : MxInst<(outs TYPE.ROp:$dst), (ins TYPE.ROp:$src, TYPE.ROp:$opd),
            MN#"."#TYPE.Prefix#"\t$opd, $dst",
            [(set TYPE.VT:$dst, CCR, (NODE TYPE.VT:$src, TYPE.VT:$opd))],
            MxArithEncoding<MxBead4Bits<CMD>,
                            !cast<MxEncOpMode>("MxOpMode"#TYPE.Size#TYPE.RLet#"EA"),
-                           MxBeadReg<0>,
+                           REG,
                            !cast<MxEncEA>("MxEncEA"#TYPE.RLet#"_2"),
                            MxExtEmpty>>;
 
@@ -110,7 +110,7 @@ class MxBiArOp_RFRR_EAd<string MN, SDNode NODE, MxType TYPE, bits<4> CMD>
            [(set TYPE.VT:$dst, CCR, (NODE TYPE.VT:$src, TYPE.VT:$opd))],
            MxArithEncoding<MxBead4Bits<CMD>,
                            !cast<MxEncOpMode>("MxOpMode"#TYPE.Size#"EAd"),
-                           MxBeadReg<2>, MxEncEAd_0, MxExtEmpty>>;
+                           MxBeadDReg<2>, MxEncEAd_0, MxExtEmpty>>;
 
 // $reg <- $reg op $imm
 class MxBiArOp_RFRI_xEA<string MN, SDNode NODE, MxType TYPE, bits<4> CMD>
@@ -119,7 +119,7 @@ class MxBiArOp_RFRI_xEA<string MN, SDNode NODE, MxType TYPE, bits<4> CMD>
               [(set TYPE.VT:$dst, CCR, (NODE TYPE.VT:$src, TYPE.IPat:$opd))],
               MxArithEncoding<MxBead4Bits<CMD>,
                               !cast<MxEncOpMode>("MxOpMode"#TYPE.Size#TYPE.RLet#"EA"),
-                              MxBeadReg<0>, MxEncEAi,
+                              MxBeadDReg<0>, MxEncEAi,
                               !cast<MxEncExt>("MxExtI"#TYPE.Size#"_2")>>;
 
 // Again, there are two ways to write an immediate to Dn register either dEA
@@ -141,7 +141,7 @@ class MxBiArOp_RFRM<string MN, SDNode NODE, MxType TYPE, MxOperand OPD, ComplexP
            [(set TYPE.VT:$dst, CCR, (NODE TYPE.VT:$src, (TYPE.Load PAT:$opd)))],
            MxArithEncoding<MxBead4Bits<CMD>,
                            !cast<MxEncOpMode>("MxOpMode"#TYPE.Size#TYPE.RLet#"EA"),
-                           MxBeadReg<0>, EA, EXT>>;
+                           MxBeadDReg<0>, EA, EXT>>;
 
 } // Constraints
 
@@ -157,7 +157,7 @@ class MxBiArOp_FMR<string MN, SDNode NODE, MxType TYPE,
         [],
         MxArithEncoding<MxBead4Bits<CMD>,
                         !cast<MxEncOpMode>("MxOpMode"#TYPE.Size#"EA"#TYPE.RLet),
-                        MxBeadReg<1>, EA, EXT>>;
+                        MxBeadDReg<1>, EA, EXT>>;
 
 class MxBiArOp_FMI<string MN, SDNode NODE, MxType TYPE,
                    MxOperand MEMOpd, ComplexPattern MEMPat,
@@ -262,9 +262,9 @@ multiclass MxBiArOp_DF<string MN, SDNode NODE, bit isComm,
 
   let isCommutable = isComm in {
 
-    def NAME#"8dd"  : MxBiArOp_RFRR_xEA<MN, NODE, MxType8d,  CMD>;
-    def NAME#"16dd" : MxBiArOp_RFRR_xEA<MN, NODE, MxType16d, CMD>;
-    def NAME#"32dd" : MxBiArOp_RFRR_xEA<MN, NODE, MxType32d, CMD>;
+    def NAME#"8dd"  : MxBiArOp_RFRR_xEA<MN, NODE, MxType8d,  CMD, MxBeadDReg<0>>;
+    def NAME#"16dd" : MxBiArOp_RFRR_xEA<MN, NODE, MxType16d, CMD, MxBeadDReg<0>>;
+    def NAME#"32dd" : MxBiArOp_RFRR_xEA<MN, NODE, MxType32d, CMD, MxBeadDReg<0>>;
 
   } // isComm
 
@@ -291,7 +291,7 @@ multiclass MxBiArOp_AF<string MN, SDNode NODE, bit isComm,
   def NAME#"32ri" : MxBiArOp_RFRI_xEA<MN, NODE, MxType32r, CMD>;
 
   let isCommutable = isComm in
-  def NAME#"32rr" : MxBiArOp_RFRR_xEA<MN, NODE, MxType32r, CMD>;
+  def NAME#"32rr" : MxBiArOp_RFRR_xEA<MN, NODE, MxType32r, CMD, MxBeadReg<0>>;
 
 } // MxBiArOp_AF
 
@@ -313,7 +313,7 @@ class MxBiArOp_RFRRF<string MN, SDNode NODE, MxType TYPE, bits<4> CMD>
              [(set TYPE.VT:$dst, CCR, (NODE TYPE.VT:$src, TYPE.VT:$opd, CCR))],
              MxArithXEncoding<MxBead4Bits<CMD>,
                               !cast<MxEncSize>("MxEncSize"#TYPE.Size),
-                              MxBead1Bit<0>, MxBeadReg<2>, MxBeadReg<0>>>;
+                              MxBead1Bit<0>, MxBeadDReg<2>, MxBeadDReg<0>>>;
 
 } // Constraints
 } // Uses, Defs
@@ -372,7 +372,7 @@ class MxCmp_RR<MxType TYPE>
              [(set CCR, (MxCmp TYPE.VT:$lhs, TYPE.VT:$rhs))],
              MxArithEncoding<MxBead4Bits<0xB>,
                              !cast<MxEncOpMode>("MxOpMode"#TYPE.Size#"dEA"),
-                             MxBeadReg<1>, MxEncEAd_0, MxExtEmpty>>;
+                             MxBeadDReg<1>, MxEncEAd_0, MxExtEmpty>>;
 
 class MxCmp_RI<MxType TYPE>
     : MxInst<(outs), (ins TYPE.IOp:$imm, TYPE.ROp:$reg),
@@ -412,7 +412,7 @@ class MxCmp_RM<MxType TYPE, MxOperand MEMOpd, ComplexPattern MEMPat,
              [(set CCR, (MxCmp (load MEMPat:$mem), TYPE.ROp:$reg))],
              MxArithEncoding<MxBead4Bits<0xB>,
                              !cast<MxEncOpMode>("MxOpMode"#TYPE.Size#"dEA"),
-                             MxBeadReg<0>, EA, EXT>>;
+                             MxBeadDReg<0>, EA, EXT>>;
 } // let mayLoad = 1
 
 } // let Defs = [CCR]
@@ -474,7 +474,7 @@ def MxExtOpmode_lb : MxBead3Bits<0b111>;
 ///  0  1  0  0  1  0  0 |  OPMODE | 0  0  0 |   REG
 /// ---------------------------------------------------
 class MxExtEncoding<MxBead3Bits OPMODE>
-    : MxEncoding<MxBeadReg<0>, MxBead3Bits<0b000>, OPMODE,
+    : MxEncoding<MxBeadDReg<0>, MxBead3Bits<0b000>, OPMODE,
                  MxBead3Bits<0b100>, MxBead4Bits<0b0100>>;
 
 let Defs = [CCR] in
@@ -508,7 +508,7 @@ def MxUDiMuOpmode : MxBead3Bits<0b011>;
 ///  x  x  x  x |   REG   | OP MODE |   MODE  |   REG
 /// ----------------------------------------------------
 class MxDiMuEncoding<MxBead4Bits CMD, MxBead3Bits OPMODE, MxEncEA EA, MxEncExt EXT>
-    : MxEncoding<EA.Reg, EA.DA, EA.Mode, OPMODE, MxBeadReg<0>, CMD,
+    : MxEncoding<EA.Reg, EA.DA, EA.Mode, OPMODE, MxBeadDReg<0>, CMD,
                  EXT.Imm, EXT.B8, EXT.Scale, EXT.WL, EXT.DAReg>;
 
 let Defs = [CCR] in {
@@ -522,7 +522,7 @@ class MxDiMuOp_DD<string MN, bits<4> CMD, MxBead3Bits OPMODE,
 // $reg <- $reg op $imm
 class MxDiMuOp_DI<string MN, bits<4> CMD, MxBead3Bits OPMODE,
                   MxOperand DST, MxOperand OPD>
-    : MxInst<(outs DST:$dst), (ins DST:$src, unknown:$opd), MN#"\t$opd, $dst", [],
+    : MxInst<(outs DST:$dst), (ins DST:$src, OPD:$opd), MN#"\t$opd, $dst", [],
              MxDiMuEncoding<MxBead4Bits<CMD>, OPMODE, MxEncEAi, MxExtI16_2>>;
 } // let Constraints
 } // Defs = [CCR]
@@ -545,6 +545,12 @@ multiclass MxDiMuOp<string MN, bits<4> CMD, bit isComm = 0> {
 
 defm DIV : MxDiMuOp<"div", 0x8>;
 
+// This is used to cast immediates to 16-bits for operations which don't
+// support smaller immediate sizes.
+def as_i16imm : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i16);
+}]>;
+
 // RR i8
 def : Pat<(sdiv i8:$dst, i8:$opd),
           (EXTRACT_SUBREG
@@ -591,22 +597,22 @@ def : Pat<(urem i16:$dst, i16:$opd),
 // RI i8
 def : Pat<(sdiv i8:$dst, MximmSExt8:$opd),
           (EXTRACT_SUBREG
-            (SDIVd32i16 (MOVSXd32d8 $dst), imm:$opd),
+            (SDIVd32i16 (MOVSXd32d8 $dst), (as_i16imm $opd)),
              MxSubRegIndex8Lo)>;
 
 def : Pat<(udiv i8:$dst, MximmSExt8:$opd),
           (EXTRACT_SUBREG
-            (UDIVd32i16 (MOVZXd32d8 $dst), imm:$opd),
+            (UDIVd32i16 (MOVZXd32d8 $dst), (as_i16imm $opd)),
              MxSubRegIndex8Lo)>;
 
 def : Pat<(srem i8:$dst, MximmSExt8:$opd),
           (EXTRACT_SUBREG
-            (ASR32di (ASR32di (SDIVd32i16 (MOVSXd32d8 $dst), imm:$opd), 8), 8),
+            (ASR32di (ASR32di (SDIVd32i16 (MOVSXd32d8 $dst), (as_i16imm $opd)), 8), 8),
              MxSubRegIndex8Lo)>;
 
 def : Pat<(urem i8:$dst, MximmSExt8:$opd),
           (EXTRACT_SUBREG
-            (LSR32di (LSR32di (UDIVd32i16 (MOVZXd32d8 $dst), imm:$opd), 8), 8),
+            (LSR32di (LSR32di (UDIVd32i16 (MOVZXd32d8 $dst), (as_i16imm $opd)), 8), 8),
              MxSubRegIndex8Lo)>;
 
 // RI i16
diff --git a/llvm/lib/Target/M68k/M68kInstrBits.td b/llvm/lib/Target/M68k/M68kInstrBits.td
index 96d536520939b0d5250671eeffff494c715b41b4..d97ca50f74a9a565b566d2bcab59a87863e873e1 100644
--- a/llvm/lib/Target/M68k/M68kInstrBits.td
+++ b/llvm/lib/Target/M68k/M68kInstrBits.td
@@ -32,7 +32,7 @@
 /// ------------+---------+---------+---------+---------
 ///  0  0  0  0 |   REG   | 1  0  0 |   MODE  |   REG
 /// ------------+---------+---------+---------+---------
-class MxBTSTEnc_R<MxBeadReg REG, MxEncEA EA, MxEncExt EXT>
+class MxBTSTEnc_R<MxBeadDReg REG, MxEncEA EA, MxEncExt EXT>
     : MxEncoding<EA.Reg, EA.DA, EA.Mode, MxBead3Bits<0b100>, REG, MxBead4Bits<0b0000>,
                  EXT.Imm, EXT.B8, EXT.Scale, EXT.WL, EXT.DAReg>;
 
@@ -52,7 +52,7 @@ let Defs = [CCR] in {
 class MxBTST_RR<MxType TYPE>
     : MxInst<(outs), (ins TYPE.ROp:$dst, TYPE.ROp:$bitno), "btst\t$bitno, $dst",
              [(set CCR, (MxBt TYPE.VT:$dst, TYPE.VT:$bitno))],
-             MxBTSTEnc_R<MxBeadReg<1>, MxEncEAd_0, MxExtEmpty>>;
+             MxBTSTEnc_R<MxBeadDReg<1>, MxEncEAd_0, MxExtEmpty>>;
 
 class MxBTST_RI<MxType TYPE>
     : MxInst<(outs), (ins TYPE.ROp:$dst, TYPE.IOp:$bitno), "btst\t$bitno, $dst",
@@ -63,7 +63,7 @@ class MxBTST_MR<MxType TYPE, MxOperand MEMOpd, ComplexPattern MEMPat,
                 MxEncEA EA, MxEncExt EXT>
     : MxInst<(outs), (ins MEMOpd:$dst, TYPE.ROp:$bitno), "btst\t$bitno, $dst",
              [(set CCR, (MxBt (TYPE.Load MEMPat:$dst), TYPE.VT:$bitno))],
-             MxBTSTEnc_R<MxBeadReg<1>, EA, EXT>>;
+             MxBTSTEnc_R<MxBeadDReg<1>, EA, EXT>>;
 
 class MxBTST_MI<MxType TYPE, MxOperand MEMOpd, ComplexPattern MEMPat,
                 MxEncEA EA, MxEncExt EXT>
diff --git a/llvm/lib/Target/M68k/M68kInstrCompiler.td b/llvm/lib/Target/M68k/M68kInstrCompiler.td
index 4d1d207451581fc61a314db0c97a780155362559..bcb815dbc4ebf6efa9b1d6c44900885aae918bf9 100644
--- a/llvm/lib/Target/M68k/M68kInstrCompiler.td
+++ b/llvm/lib/Target/M68k/M68kInstrCompiler.td
@@ -57,7 +57,6 @@ def : Pat<(i32 (MxWrapperPC tblockaddress :$src)), (LEA32q tblockaddress :$src)>
 let usesCustomInserter = 1, Uses = [CCR] in
 class MxCMove<MxType TYPE>
     : MxPseudo<(outs TYPE.ROp:$dst), (ins TYPE.ROp:$t, TYPE.ROp:$f, i8imm:$cond),
-               "",
                [(set TYPE.VT:$dst,
                      (TYPE.VT (MxCmov TYPE.VT:$t, TYPE.VT:$f, imm:$cond, CCR)))]>;
 
@@ -78,11 +77,11 @@ def CMOV32r : MxCMove<MxType32r>;
 let Defs = [SP, CCR], Uses = [SP] in {
 
   def ADJCALLSTACKDOWN
-    : MxPseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2), "#ADJCALLSTACKDOWN",
+    : MxPseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2),
                [(MxCallSeqStart timm:$amt1, timm:$amt2)]>;
 
   def ADJCALLSTACKUP
-    : MxPseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2), "#ADJCALLSTACKUP",
+    : MxPseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2),
                [(MxCallSeqEnd timm:$amt1, timm:$amt2)]>;
 
 } // Defs
@@ -124,5 +123,4 @@ def : Pat<(MxTCRet (i32 texternalsym:$dst), imm:$adj),
 let Defs = [SP, CCR], Uses = [SP] in
 let usesCustomInserter = 1 in
 def SALLOCA : MxPseudo<(outs MxARD32:$dst), (ins MxARD32:$size),
-                       "# variable sized alloca for segmented stacks",
                        [(set iPTR:$dst, (MxSegAlloca iPTR:$size))]>;
diff --git a/llvm/lib/Target/M68k/M68kInstrControl.td b/llvm/lib/Target/M68k/M68kInstrControl.td
index 567e6b0427fbfb01647660016317e0213949900b..70847472686125845d713a04896c26ebaff422c5 100644
--- a/llvm/lib/Target/M68k/M68kInstrControl.td
+++ b/llvm/lib/Target/M68k/M68kInstrControl.td
@@ -264,7 +264,7 @@ let isTerminator = 1, isReturn = 1, isBarrier = 1, hasCtrlDep = 1 in {
 def RTS : MxInst<(outs), (ins), "rts", [], MxEncFixed<0x4E75>>;
 
 let isCodeGenOnly = 1 in
-def RET : MxPseudo<(outs), (ins i32imm:$adj, variable_ops), "",
+def RET : MxPseudo<(outs), (ins i32imm:$adj, variable_ops),
                    [(MxRet timm:$adj)]>;
 } // isTerminator = 1, isReturn = 1, isBarrier = 1, hasCtrlDep = 1
 
@@ -277,11 +277,11 @@ let Uses = [CCR], Defs = [CCR], isPseudo = 1 in {
 // FIXME These are pseudo ops that should be replaced with Pat<> patterns.
 // However, Pat<> can't replicate the destination reg into the inputs of the
 // result.
-def SETCS_C8d : MxPseudo<(outs MxDRD8:$dst), (ins), "",
+def SETCS_C8d : MxPseudo<(outs MxDRD8:$dst), (ins),
                          [(set MxDRD8:$dst, (MxSetCC_C MxCONDcs, CCR))]>;
-def SETCS_C16d : MxPseudo<(outs MxDRD16:$dst), (ins), "",
+def SETCS_C16d : MxPseudo<(outs MxDRD16:$dst), (ins),
                           [(set MxDRD16:$dst, (MxSetCC_C MxCONDcs, CCR))]>;
-def SETCS_C32d : MxPseudo<(outs MxXRD32:$dst), (ins), "",
+def SETCS_C32d : MxPseudo<(outs MxXRD32:$dst), (ins),
                           [(set MxXRD32:$dst, (MxSetCC_C MxCONDcs, CCR))]>;
 } // Uses = [CCR], Defs = [CCR], isPseudo = 1
 
diff --git a/llvm/lib/Target/M68k/M68kInstrData.td b/llvm/lib/Target/M68k/M68kInstrData.td
index 74db3a3daca5034906c8323c9e0cdbb340fb9182..40b9e4a2a7fa576e9112d816a94c7f0174585d55 100644
--- a/llvm/lib/Target/M68k/M68kInstrData.td
+++ b/llvm/lib/Target/M68k/M68kInstrData.td
@@ -588,10 +588,10 @@ def POP32r : MxPseudo<(outs XR32:$reg), (ins)>;
 
 let Defs = [CCR] in {
 class MxPseudoMove_RR<MxType DST, MxType SRC, list<dag> PAT = []>
-    : MxPseudo<(outs DST.ROp:$dst), (ins SRC.ROp:$src), "", PAT>;
+    : MxPseudo<(outs DST.ROp:$dst), (ins SRC.ROp:$src), PAT>;
 
 class MxPseudoMove_RM<MxType DST, MxOperand SRCOpd, list<dag> PAT = []>
-    : MxPseudo<(outs DST.ROp:$dst), (ins SRCOpd:$src), "", PAT>;
+    : MxPseudo<(outs DST.ROp:$dst), (ins SRCOpd:$src), PAT>;
 }
 
 /// This group of Pseudos is analogues to the real x86 extending moves, but
diff --git a/llvm/lib/Target/M68k/M68kInstrFormats.td b/llvm/lib/Target/M68k/M68kInstrFormats.td
index b00db4915df2d5a171b35d67c7ed35ceb5a982c8..1d950bd0377aed354fced5aa8d776b37feed3546 100644
--- a/llvm/lib/Target/M68k/M68kInstrFormats.td
+++ b/llvm/lib/Target/M68k/M68kInstrFormats.td
@@ -95,16 +95,17 @@ class MxBead4Bits <bits<4> b> : MxBead<0x4, b{0}, b{1}, b{2}, b{3}>;
 class MxBeadDAReg  <bits<3> o, bit a = 0> : MxBead<0x5, o{0}, o{1}, o{2}, a>;
 class MxBeadDA     <bits<3> o, bit a = 0> : MxBead<0x6, o{0}, o{1}, o{2}, a>;
 class MxBeadReg    <bits<3> o, bit a = 0> : MxBead<0x7, o{0}, o{1}, o{2}, a>;
-class MxBead8Disp  <bits<3> o, bit a = 0> : MxBead<0x8, o{0}, o{1}, o{2}, a>;
+class MxBeadDReg   <bits<3> o, bit a = 0> : MxBead<0x8, o{0}, o{1}, o{2}, a>;
+class MxBead8Disp  <bits<3> o, bit a = 0> : MxBead<0x9, o{0}, o{1}, o{2}, a>;
 
 /// Add Immediate to the instruction. 8-bit version is padded with zeros to fit
 /// the word.
-class MxBead8Imm   <bits<3> o, bit a = 0> : MxBead<0x9, o{0}, o{1}, o{2}, a>;
-class MxBead16Imm  <bits<3> o, bit a = 0> : MxBead<0xA, o{0}, o{1}, o{2}, a>;
-class MxBead32Imm  <bits<3> o, bit a = 0> : MxBead<0xB, o{0}, o{1}, o{2}, a>;
+class MxBead8Imm   <bits<3> o, bit a = 0> : MxBead<0xA, o{0}, o{1}, o{2}, a>;
+class MxBead16Imm  <bits<3> o, bit a = 0> : MxBead<0xB, o{0}, o{1}, o{2}, a>;
+class MxBead32Imm  <bits<3> o, bit a = 0> : MxBead<0xC, o{0}, o{1}, o{2}, a>;
 
 /// Encodes an immediate 0-7(alt. 1-8) into 3 bit field
-class MxBead3Imm   <bits<3> o, bit a = 0> : MxBead<0xC, o{0}, o{1}, o{2}, a>;
+class MxBead3Imm   <bits<3> o, bit a = 0> : MxBead<0xD, o{0}, o{1}, o{2}, a>;
 
 
 class MxEncoding<MxBead n0  = MxBeadTerm, MxBead n1  = MxBeadTerm,
@@ -202,7 +203,7 @@ class MxEncEA<MxBead reg, MxBead mode, MxBead da = MxBeadIgnore> {
 // FIXME: Is there a way to factorize the addressing mode suffix (i.e.
 // 'r', 'd', 'a' etc.) and use something like multiclass to replace?
 def MxEncEAr_0: MxEncEA<MxBeadDAReg<0>, MxBead2Bits<0b00>>;
-def MxEncEAd_0: MxEncEA<MxBeadReg<0>, MxBead2Bits<0b00>, MxBead1Bit<0>>;
+def MxEncEAd_0: MxEncEA<MxBeadDReg<0>, MxBead2Bits<0b00>, MxBead1Bit<0>>;
 def MxEncEAa_0: MxEncEA<MxBeadReg<0>, MxBead2Bits<0b00>, MxBead1Bit<1>>;
 def MxEncEAj_0: MxEncEA<MxBeadReg<0>, MxBead2Bits<0b01>, MxBead1Bit<0>>;
 def MxEncEAo_0: MxEncEA<MxBeadReg<0>, MxBead2Bits<0b01>, MxBead1Bit<1>>;
@@ -214,7 +215,7 @@ def MxEncEAa_0_reflected : MxEncEA<MxBeadReg<0>, MxBead3Bits<0b001>>;
 def MxEncEAr_0_reflected : MxEncEA<MxBeadReg<0>, MxBead2Bits<0b00>, MxBeadDA<0>>;
 
 def MxEncEAr_1: MxEncEA<MxBeadDAReg<1>, MxBead2Bits<0b00>>;
-def MxEncEAd_1: MxEncEA<MxBeadReg<1>, MxBead2Bits<0b00>, MxBead1Bit<0>>;
+def MxEncEAd_1: MxEncEA<MxBeadDReg<1>, MxBead2Bits<0b00>, MxBead1Bit<0>>;
 def MxEncEAa_1: MxEncEA<MxBeadReg<1>, MxBead2Bits<0b00>, MxBead1Bit<1>>;
 def MxEncEAj_1: MxEncEA<MxBeadReg<1>, MxBead2Bits<0b01>, MxBead1Bit<0>>;
 def MxEncEAo_1: MxEncEA<MxBeadReg<1>, MxBead2Bits<0b01>, MxBead1Bit<1>>;
@@ -223,7 +224,7 @@ def MxEncEAp_1: MxEncEA<MxBeadReg<1>, MxBead2Bits<0b10>, MxBead1Bit<1>>;
 def MxEncEAf_1: MxEncEA<MxBeadReg<1>, MxBead2Bits<0b11>, MxBead1Bit<0>>;
 
 def MxEncEAr_2: MxEncEA<MxBeadDAReg<2>, MxBead2Bits<0b00>>;
-def MxEncEAd_2: MxEncEA<MxBeadReg<2>, MxBead2Bits<0b00>, MxBead1Bit<0>>;
+def MxEncEAd_2: MxEncEA<MxBeadDReg<2>, MxBead2Bits<0b00>, MxBead1Bit<0>>;
 def MxEncEAa_2: MxEncEA<MxBeadReg<2>, MxBead2Bits<0b00>, MxBead1Bit<1>>;
 def MxEncEAj_2: MxEncEA<MxBeadReg<2>, MxBead2Bits<0b01>, MxBead1Bit<0>>;
 def MxEncEAo_2: MxEncEA<MxBeadReg<2>, MxBead2Bits<0b01>, MxBead1Bit<1>>;
@@ -364,7 +365,7 @@ class MxInst<dag outs, dag ins,
 }
 
 // M68k PSEUDO INSTRUCTION
-class MxPseudo<dag outs, dag ins, string asmStr = "", list<dag> pattern = []>
-    : MxInst<outs, ins, asmStr, pattern> {
+class MxPseudo<dag outs, dag ins, list<dag> pattern = []>
+    : MxInst<outs, ins, "; error: this should not be emitted", pattern> {
   let isPseudo = 1;
 }
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.cpp b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
index 94b003f873f7816d859df5fb6a218dddf91a458c..67b3a9f12c90e9c33b72b1ed62d0066164a67607 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.cpp
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
@@ -16,6 +16,7 @@
 #include "M68kInstrBuilder.h"
 #include "M68kMachineFunction.h"
 #include "M68kTargetMachine.h"
+#include "MCTargetDesc/M68kMCCodeEmitter.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.h b/llvm/lib/Target/M68k/M68kInstrInfo.h
index e15b0a1181bac9445f83d2a6eb376e85c13df34a..a503b02c5a825f9ba78275c416d4288dfd91ce47 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.h
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.h
@@ -30,9 +30,6 @@ namespace llvm {
 class M68kSubtarget;
 
 namespace M68k {
-// Forward declaration
-const uint8_t *getMCInstrBeads(unsigned Opcode);
-
 // These MUST be kept in sync with codes definitions in M68kInstrInfo.td
 enum CondCode {
   COND_T = 0,   // True
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.td b/llvm/lib/Target/M68k/M68kInstrInfo.td
index e5d6783022cee62e1ef1a667eaf5ac95a8586e03..973a6179efbece6e8e52a5c6e08f6e63dfc91fe6 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.td
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.td
@@ -206,14 +206,14 @@ def MxARD32 : MxRegOp<i32, AR32, MxSize32, "a">;
 def MxARD16_TC : MxRegOp<i16, AR16_TC, MxSize16, "a">;
 def MxARD32_TC : MxRegOp<i32, AR32_TC, MxSize32, "a">;
 
-// TODO finish parser wiring
-def MxMemAsmOperand : AsmOperandClass {
- let Name = "MxMemOp";
+class MxOpClass<string name> : AsmOperandClass {
+  let Name = name;
+  let ParserMethod = "parse"#name;
 }
 
 class MxMemOp<dag ops, MxSize size, string letter,
               string printMethod = "printOperand",
-              AsmOperandClass parserMatchClass = MxMemAsmOperand>
+              AsmOperandClass parserMatchClass = ImmAsmOperand>
     : Operand<iPTR>, MxOperand<iPTR, size, letter, ?> {
   let PrintMethod = printMethod;
   let MIOperandInfo = ops;
@@ -225,13 +225,14 @@ class MxMemOp<dag ops, MxSize size, string letter,
 // register specified by the register field. The reference is classified as
 // a data reference with the exception of the jump and jump-to-subroutine
 // instructions.
-def MxARI8        : MxMemOp<(ops AR32), MxSize8,  "j", "printARI8Mem">;
-def MxARI16       : MxMemOp<(ops AR32), MxSize16, "j", "printARI16Mem">;
-def MxARI32       : MxMemOp<(ops AR32), MxSize32, "j", "printARI32Mem">;
+def MxARI         : MxOpClass<"ARI">;
+def MxARI8        : MxMemOp<(ops AR32), MxSize8,  "j", "printARI8Mem", MxARI>;
+def MxARI16       : MxMemOp<(ops AR32), MxSize16, "j", "printARI16Mem", MxARI>;
+def MxARI32       : MxMemOp<(ops AR32), MxSize32, "j", "printARI32Mem", MxARI>;
 
-def MxARI8_TC     : MxMemOp<(ops AR32_TC), MxSize8,  "j", "printARI8Mem">;
-def MxARI16_TC    : MxMemOp<(ops AR32_TC), MxSize16, "j", "printARI16Mem">;
-def MxARI32_TC    : MxMemOp<(ops AR32_TC), MxSize32, "j", "printARI32Mem">;
+def MxARI8_TC     : MxMemOp<(ops AR32_TC), MxSize8,  "j", "printARI8Mem", MxARI>;
+def MxARI16_TC    : MxMemOp<(ops AR32_TC), MxSize16, "j", "printARI16Mem", MxARI>;
+def MxARI32_TC    : MxMemOp<(ops AR32_TC), MxSize32, "j", "printARI32Mem", MxARI>;
 
 // ADDRESS REGISTER INDIRECT WITH POSTINCREMENT. The address of the operand is
 // in the address register specified by the register field. After the operand
@@ -240,13 +241,14 @@ def MxARI32_TC    : MxMemOp<(ops AR32_TC), MxSize32, "j", "printARI32Mem">;
 // is the stack pointer and the operand size is byte, the address is incremented
 // by two rather than one to keep the stack pointer on a word boundary.
 // The reference is classified as a data reference.
-def MxARIPI8      : MxMemOp<(ops AR32), MxSize8,  "o", "printARIPI8Mem">;
-def MxARIPI16     : MxMemOp<(ops AR32), MxSize16, "o", "printARIPI16Mem">;
-def MxARIPI32     : MxMemOp<(ops AR32), MxSize32, "o", "printARIPI32Mem">;
+def MxARIPI       : MxOpClass<"ARIPI">;
+def MxARIPI8      : MxMemOp<(ops AR32), MxSize8,  "o", "printARIPI8Mem", MxARIPI>;
+def MxARIPI16     : MxMemOp<(ops AR32), MxSize16, "o", "printARIPI16Mem", MxARIPI>;
+def MxARIPI32     : MxMemOp<(ops AR32), MxSize32, "o", "printARIPI32Mem", MxARIPI>;
 
-def MxARIPI8_TC   : MxMemOp<(ops AR32_TC), MxSize8,  "o", "printARIPI8Mem">;
-def MxARIPI16_TC  : MxMemOp<(ops AR32_TC), MxSize16, "o", "printARIPI16Mem">;
-def MxARIPI32_TC  : MxMemOp<(ops AR32_TC), MxSize32, "o", "printARIPI32Mem">;
+def MxARIPI8_TC   : MxMemOp<(ops AR32_TC), MxSize8,  "o", "printARIPI8Mem", MxARIPI>;
+def MxARIPI16_TC  : MxMemOp<(ops AR32_TC), MxSize16, "o", "printARIPI16Mem", MxARIPI>;
+def MxARIPI32_TC  : MxMemOp<(ops AR32_TC), MxSize32, "o", "printARIPI32Mem", MxARIPI>;
 
 // ADDRESS REGISTER INDIRECT WITH PREDECREMENT. The address of the operand is in
 // the address register specified by the register field. Before the operand
@@ -255,26 +257,28 @@ def MxARIPI32_TC  : MxMemOp<(ops AR32_TC), MxSize32, "o", "printARIPI32Mem">;
 // the stack pointer and the operand size is byte, the address is decremented by
 // two rather than one to keep the stack pointer on a word boundary.
 // The reference is classified as a data reference.
-def MxARIPD8      : MxMemOp<(ops AR32), MxSize8,  "e", "printARIPD8Mem">;
-def MxARIPD16     : MxMemOp<(ops AR32), MxSize16, "e", "printARIPD16Mem">;
-def MxARIPD32     : MxMemOp<(ops AR32), MxSize32, "e", "printARIPD32Mem">;
+def MxARIPD       : MxOpClass<"ARIPD">;
+def MxARIPD8      : MxMemOp<(ops AR32), MxSize8,  "e", "printARIPD8Mem", MxARIPD>;
+def MxARIPD16     : MxMemOp<(ops AR32), MxSize16, "e", "printARIPD16Mem", MxARIPD>;
+def MxARIPD32     : MxMemOp<(ops AR32), MxSize32, "e", "printARIPD32Mem", MxARIPD>;
 
-def MxARIPD8_TC   : MxMemOp<(ops AR32_TC), MxSize8,  "e", "printARIPD8Mem">;
-def MxARIPD16_TC  : MxMemOp<(ops AR32_TC), MxSize16, "e", "printARIPD16Mem">;
-def MxARIPD32_TC  : MxMemOp<(ops AR32_TC), MxSize32, "e", "printARIPD32Mem">;
+def MxARIPD8_TC   : MxMemOp<(ops AR32_TC), MxSize8,  "e", "printARIPD8Mem", MxARIPD>;
+def MxARIPD16_TC  : MxMemOp<(ops AR32_TC), MxSize16, "e", "printARIPD16Mem", MxARIPD>;
+def MxARIPD32_TC  : MxMemOp<(ops AR32_TC), MxSize32, "e", "printARIPD32Mem", MxARIPD>;
 
 // ADDRESS REGISTER INDIRECT WITH DISPLACEMENT. This addressing mode requires one
 // word of extension. The address of the operand is the sum of the address in
 // the address register and the sign-extended 16-bit displacement integer in the
 // extension word. The reference is classified as a data reference with the
 // exception of the jump and jump-to-subroutine instructions.
-def MxARID8       : MxMemOp<(ops i16imm, AR32), MxSize8,  "p", "printARID8Mem">;
-def MxARID16      : MxMemOp<(ops i16imm, AR32), MxSize16, "p", "printARID16Mem">;
-def MxARID32      : MxMemOp<(ops i16imm, AR32), MxSize32, "p", "printARID32Mem">;
+def MxARID        : MxOpClass<"ARID">;
+def MxARID8       : MxMemOp<(ops i16imm, AR32), MxSize8,  "p", "printARID8Mem", MxARID>;
+def MxARID16      : MxMemOp<(ops i16imm, AR32), MxSize16, "p", "printARID16Mem", MxARID>;
+def MxARID32      : MxMemOp<(ops i16imm, AR32), MxSize32, "p", "printARID32Mem", MxARID>;
 
-def MxARID8_TC    : MxMemOp<(ops i16imm, AR32_TC), MxSize8,  "p", "printARID8Mem">;
-def MxARID16_TC   : MxMemOp<(ops i16imm, AR32_TC), MxSize16, "p", "printARID16Mem">;
-def MxARID32_TC   : MxMemOp<(ops i16imm, AR32_TC), MxSize32, "p", "printARID32Mem">;
+def MxARID8_TC    : MxMemOp<(ops i16imm, AR32_TC), MxSize8,  "p", "printARID8Mem", MxARID>;
+def MxARID16_TC   : MxMemOp<(ops i16imm, AR32_TC), MxSize16, "p", "printARID16Mem", MxARID>;
+def MxARID32_TC   : MxMemOp<(ops i16imm, AR32_TC), MxSize32, "p", "printARID32Mem", MxARID>;
 
 // ADDRESS REGISTER INDIRECT WITH INDEX. This addressing mode requires one word
 // of extension. The address of the operand is the sum of the address in the
@@ -282,21 +286,23 @@ def MxARID32_TC   : MxMemOp<(ops i16imm, AR32_TC), MxSize32, "p", "printARID32Me
 // bits of the extension word, and the contents of the index register.
 // The reference is classified as a data reference with the exception of the
 // jump and jump-to-subroutine instructions
-def MxARII8      : MxMemOp<(ops i8imm, AR32, XR32), MxSize8,  "f", "printARII8Mem">;
-def MxARII16     : MxMemOp<(ops i8imm, AR32, XR32), MxSize16, "f", "printARII16Mem">;
-def MxARII32     : MxMemOp<(ops i8imm, AR32, XR32), MxSize32, "f", "printARII32Mem">;
+def MxARII       : MxOpClass<"ARII">;
+def MxARII8      : MxMemOp<(ops i8imm, AR32, XR32), MxSize8,  "f", "printARII8Mem", MxARII>;
+def MxARII16     : MxMemOp<(ops i8imm, AR32, XR32), MxSize16, "f", "printARII16Mem", MxARII>;
+def MxARII32     : MxMemOp<(ops i8imm, AR32, XR32), MxSize32, "f", "printARII32Mem", MxARII>;
 
-def MxARII8_TC   : MxMemOp<(ops i8imm, AR32_TC, XR32_TC), MxSize8,  "f", "printARII8Mem">;
-def MxARII16_TC  : MxMemOp<(ops i8imm, AR32_TC, XR32_TC), MxSize16, "f", "printARII16Mem">;
-def MxARII32_TC  : MxMemOp<(ops i8imm, AR32_TC, XR32_TC), MxSize32, "f", "printARII32Mem">;
+def MxARII8_TC   : MxMemOp<(ops i8imm, AR32_TC, XR32_TC), MxSize8,  "f", "printARII8Mem", MxARII>;
+def MxARII16_TC  : MxMemOp<(ops i8imm, AR32_TC, XR32_TC), MxSize16, "f", "printARII16Mem", MxARII>;
+def MxARII32_TC  : MxMemOp<(ops i8imm, AR32_TC, XR32_TC), MxSize32, "f", "printARII32Mem", MxARII>;
 
 // ABSOLUTE SHORT ADDRESS. This addressing mode requires one word of extension.
 // The address of the operand is the extension word. The 16-bit address is sign
 // extended before it is used.  The reference is classified as a data reference
 // with the exception of the jump and jump-tosubroutine instructions.
-def MxAS8      : MxMemOp<(ops OtherVT), MxSize8,  "B", "printAS8Mem">;
-def MxAS16     : MxMemOp<(ops OtherVT), MxSize16, "B", "printAS16Mem">;
-def MxAS32     : MxMemOp<(ops OtherVT), MxSize32, "B", "printAS32Mem">;
+def MxAddr     : MxOpClass<"Addr">;
+def MxAS8      : MxMemOp<(ops OtherVT), MxSize8,  "B", "printAS8Mem", MxAddr>;
+def MxAS16     : MxMemOp<(ops OtherVT), MxSize16, "B", "printAS16Mem", MxAddr>;
+def MxAS32     : MxMemOp<(ops OtherVT), MxSize32, "B", "printAS32Mem", MxAddr>;
 
 // ABSOLUTE LONG ADDRESS. This addressing mode requires two words of extension.
 // The address of the operand is developed by the concatenation of the extension
@@ -304,9 +310,12 @@ def MxAS32     : MxMemOp<(ops OtherVT), MxSize32, "B", "printAS32Mem">;
 // order part of the address is the second extension word. The reference is
 // classified as a data reference with the exception of the jump and jump
 // to-subroutine instructions.
-def MxAL8      : MxMemOp<(ops OtherVT), MxSize8,  "b", "printAL8Mem">;
-def MxAL16     : MxMemOp<(ops OtherVT), MxSize16, "b", "printAL16Mem">;
-def MxAL32     : MxMemOp<(ops OtherVT), MxSize32, "b", "printAL32Mem">;
+def MxAL8      : MxMemOp<(ops OtherVT), MxSize8,  "b", "printAL8Mem", MxAddr>;
+def MxAL16     : MxMemOp<(ops OtherVT), MxSize16, "b", "printAL16Mem", MxAddr>;
+def MxAL32     : MxMemOp<(ops OtherVT), MxSize32, "b", "printAL32Mem", MxAddr>;
+
+def MxPCD : MxOpClass<"PCD">;
+def MxPCI : MxOpClass<"PCI">;
 
 let OperandType = "OPERAND_PCREL" in {
 // PROGRAM COUNTER WITH DISPLACEMENT. This addressing mode requires one word of
@@ -314,9 +323,9 @@ let OperandType = "OPERAND_PCREL" in {
 // counter and the Sign-extended 16-bit displacement integer in the extension
 // word. The value in the program counter is the address of the extension word.
 // The reference is classified as a program reference.
-def MxPCD8     : MxMemOp<(ops i16imm), MxSize8,  "q", "printPCD8Mem">;
-def MxPCD16    : MxMemOp<(ops i16imm), MxSize16, "q", "printPCD16Mem">;
-def MxPCD32    : MxMemOp<(ops i16imm), MxSize32, "q", "printPCD32Mem">;
+def MxPCD8     : MxMemOp<(ops i16imm), MxSize8,  "q", "printPCD8Mem", MxPCD>;
+def MxPCD16    : MxMemOp<(ops i16imm), MxSize16, "q", "printPCD16Mem", MxPCD>;
+def MxPCD32    : MxMemOp<(ops i16imm), MxSize32, "q", "printPCD32Mem", MxPCD>;
 
 // PROGRAM COUNTER WITH INDEX. This addressing mode requires one word of
 // extension. The address is the sum of the address in the program counter, the
@@ -324,14 +333,18 @@ def MxPCD32    : MxMemOp<(ops i16imm), MxSize32, "q", "printPCD32Mem">;
 // word, and the contents of the index register.  The value in the program
 // counter is the address of the extension word. This reference is classified as
 // a program reference.
-def MxPCI8   : MxMemOp<(ops i8imm, XR32), MxSize8,  "k", "printPCI8Mem">;
-def MxPCI16  : MxMemOp<(ops i8imm, XR32), MxSize16, "k", "printPCI16Mem">;
-def MxPCI32  : MxMemOp<(ops i8imm, XR32), MxSize32, "k", "printPCI32Mem">;
+def MxPCI8   : MxMemOp<(ops i8imm, XR32), MxSize8,  "k", "printPCI8Mem", MxPCI>;
+def MxPCI16  : MxMemOp<(ops i8imm, XR32), MxSize16, "k", "printPCI16Mem", MxPCI>;
+def MxPCI32  : MxMemOp<(ops i8imm, XR32), MxSize32, "k", "printPCI32Mem", MxPCI>;
 } // OPERAND_PCREL
 
+def MxImm : MxOpClass<"MxImm">;
+
 class MxOp<ValueType vt, MxSize size, string letter>
     : Operand<vt>,
-      MxOperand<vt, size, letter, ?>;
+      MxOperand<vt, size, letter, ?> {
+  let ParserMatchClass = MxImm;
+}
 
 let OperandType = "OPERAND_IMMEDIATE",
     PrintMethod = "printImmediate" in {
@@ -349,6 +362,7 @@ def Mxi32imm : MxOp<i32, MxSize32, "i">;
 } // OPERAND_IMMEDIATE
 
 let OperandType = "OPERAND_PCREL",
+    ParserMatchClass = MxImm,
     PrintMethod = "printPCRelImm" in {
 
 // Branch targets have OtherVT type and print as pc-relative values.
diff --git a/llvm/lib/Target/M68k/M68kInstrShiftRotate.td b/llvm/lib/Target/M68k/M68kInstrShiftRotate.td
index f777a5d33e217d2ee8bea4361f09e8c73845859a..cab6876380769a7abe0652c7a1a04a045931185d 100644
--- a/llvm/lib/Target/M68k/M68kInstrShiftRotate.td
+++ b/llvm/lib/Target/M68k/M68kInstrShiftRotate.td
@@ -38,11 +38,11 @@ def MxROOP_RO  : MxBead2Bits<0b11>;
 ///  1  1  1  0 | REG/IMM | D | SIZE |R/I|  OP  |   REG
 /// ------------+---------+---+------+---+------+---------
 class MxSREncoding_R<MxBead1Bit DIRECTION, MxBead2Bits ROOP, MxEncSize SIZE>
-    : MxEncoding<MxBeadReg<0>, ROOP, MxBead1Bit<1>, SIZE, DIRECTION,
-                 MxBeadReg<2>, MxBead4Bits<0b1110>>;
+    : MxEncoding<MxBeadDReg<0>, ROOP, MxBead1Bit<1>, SIZE, DIRECTION,
+                 MxBeadDReg<2>, MxBead4Bits<0b1110>>;
 
 class MxSREncoding_I<MxBead1Bit DIRECTION, MxBead2Bits ROOP, MxEncSize SIZE>
-    : MxEncoding<MxBeadReg<0>, ROOP, MxBead1Bit<0>, SIZE, DIRECTION,
+    : MxEncoding<MxBeadDReg<0>, ROOP, MxBead1Bit<0>, SIZE, DIRECTION,
                  MxBead3Imm<2, 1>, MxBead4Bits<0b1110>>;
 
 // $reg <- $reg op $reg
diff --git a/llvm/lib/Target/M68k/M68kRegisterInfo.td b/llvm/lib/Target/M68k/M68kRegisterInfo.td
index 76e762c718b08e317107e2732dfffde08489433e..e2ea2967f75b4a8879953fa75cceacc51f7e308a 100644
--- a/llvm/lib/Target/M68k/M68kRegisterInfo.td
+++ b/llvm/lib/Target/M68k/M68kRegisterInfo.td
@@ -15,8 +15,8 @@
 
 class MxReg<string N, bits<16> ENC,
             list<Register> SUBREGS = [], list<SubRegIndex> SUBIDX,
-            list<int> DWREGS = []>
-    : Register<N>, DwarfRegNum<DWREGS> {
+            list<int> DWREGS = [], list<string> ALTNAMES = []>
+    : Register<N, ALTNAMES>, DwarfRegNum<DWREGS> {
   let Namespace     = "M68k";
   let HWEncoding    = ENC;
   let SubRegs       = SUBREGS;
@@ -29,46 +29,45 @@ let Namespace = "M68k" in {
   def MxSubRegIndex16Lo : SubRegIndex<16, 0>;
 }
 
-// Generate Data registers and theirs smaller variants
-foreach Index = 0-7 in {
-  def "BD"#Index : MxReg<"d"#Index, Index, [], [], [Index]>;
-
-  def "WD"#Index
-    : MxReg<"d"#Index, Index,
-            [!cast<Register>("BD"#Index)], [MxSubRegIndex8Lo],
-            [Index]>;
-
-  def "D"#Index
-    : MxReg<"d"#Index, Index,
-            [!cast<Register>("WD"#Index)], [MxSubRegIndex16Lo],
-            [Index]>;
-
-} // foreach
-
-// Generate Address registers and theirs smaller variants
-foreach Index = 0-7 in {
-  def "WA"#Index
-     : MxReg<"a"#Index, Index, [], [], [!add(8,Index)]>;
-
-  def "A"#Index
-     : MxReg<"a"#Index, Index,
-             [!cast<Register>("WA"#Index)], [MxSubRegIndex16Lo],
-             [!add(8,Index)]>;
+multiclass MxDataRegister<int INDEX, string REG_NAME, list<string> ALTNAMES = []> {
+  def "B"#NAME : MxReg<REG_NAME, INDEX, [], [], [INDEX], ALTNAMES>;
+  def "W"#NAME
+    : MxReg<REG_NAME, INDEX,
+            [!cast<Register>("B"#NAME)], [MxSubRegIndex8Lo],
+            [INDEX], ALTNAMES>;
+  def NAME
+    : MxReg<REG_NAME, INDEX,
+            [!cast<Register>("W"#NAME)], [MxSubRegIndex16Lo],
+            [INDEX], ALTNAMES>;
 }
 
-// Alias Registers
-class MxAliasReg<string N, MxReg REG>
-    : MxReg<N, REG.HWEncoding, [], [], REG.DwarfNumbers> {
-  let Aliases = [REG];
+multiclass MxAddressRegister<int INDEX, string REG_NAME, list<string> ALTNAMES = []> {
+  def "W"#NAME
+    : MxReg<REG_NAME, INDEX, [], [], [!add(8,INDEX)], ALTNAMES>;
+  def NAME
+    : MxReg<REG_NAME, INDEX,
+            [!cast<Register>("W"#NAME)], [MxSubRegIndex16Lo],
+            [!add(8,INDEX)], ALTNAMES>;
 }
 
-def BP  : MxAliasReg<"bp",  A5>;
-def FP  : MxAliasReg<"fp",  A6>;
-def SP : MxAliasReg<"sp", A7>;
+defm D0 : MxDataRegister<0, "d0">;
+defm D1 : MxDataRegister<1, "d1">;
+defm D2 : MxDataRegister<2, "d2">;
+defm D3 : MxDataRegister<3, "d3">;
+defm D4 : MxDataRegister<4, "d4">;
+defm D5 : MxDataRegister<5, "d5">;
+defm D6 : MxDataRegister<6, "d6">;
+defm D7 : MxDataRegister<7, "d7">;
+
+defm A0 : MxAddressRegister<0, "a0">;
+defm A1 : MxAddressRegister<1, "a1">;
+defm A2 : MxAddressRegister<2, "a2">;
+defm A3 : MxAddressRegister<3, "a3">;
+defm A4 : MxAddressRegister<4, "a4">;
+defm A5 : MxAddressRegister<5, "a5", ["bp"]>;
+defm A6 : MxAddressRegister<6, "a6", ["fp"]>;
+defm SP : MxAddressRegister<7, "sp", ["usp", "ssp", "isp", "a7"]>;
 
-def USP : MxAliasReg<"usp", A7>;
-def SSP : MxAliasReg<"ssp", A7>;
-def ISP : MxAliasReg<"isp", A7>;
 
 // Pseudo Registers
 class MxPseudoReg<string N, list<Register> SUBREGS = [], list<SubRegIndex> SUBIDX = []>
@@ -92,10 +91,10 @@ def DR16 : MxRegClass<[i16], 16, (sequence "WD%u", 0, 7)>;
 def DR32 : MxRegClass<[i32], 32, (sequence "D%u",  0, 7)>;
 
 // Address Registers
-def AR16 : MxRegClass<[i16], 16, (sequence "WA%u", 0, 6)>;
+def AR16 : MxRegClass<[i16], 16, (add (sequence "WA%u", 0, 6), WSP)>;
 def AR32 : MxRegClass<[i32], 32, (add (sequence "A%u", 0, 6), SP)>;
 
-def AR32_NOSP : MxRegClass<[i32], 32, (add (sequence "A%u", 0, 6))>;
+def AR32_NOSP : MxRegClass<[i32], 32, (sequence "A%u", 0, 6)>;
 
 // Index Register Classes
 // FIXME try alternative ordering like `D0, D1, A0, A1, ...`
@@ -124,7 +123,5 @@ def XR16_TC : MxRegClass<[i16], 16, (add DR16_TC, AR16_TC)>;
 def XR32_TC : MxRegClass<[i32], 32, (add DR32_TC, AR32_TC)>;
 
 // These classes provide spill/restore order if used with MOVEM instruction
-def SPILL   : MxRegClass<[i32], 32, (add (add (sequence "D%u", 0, 7),
-                                              (sequence "A%u", 0, 6)), SP)>;
-def SPILL_R : MxRegClass<[i32], 32, (add SP, (add (sequence "A%u", 6, 0),
-                                                  (sequence "D%u", 7, 0)))>;
+def SPILL   : MxRegClass<[i32], 32, (add XR32)>;
+def SPILL_R : MxRegClass<[i32], 32, (add SP, (sequence "A%u", 6, 0), (sequence "D%u", 7, 0))>;
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kBaseInfo.h b/llvm/lib/Target/M68k/MCTargetDesc/M68kBaseInfo.h
index 36592fda1a962e4f80e343dd8bdb1ddcc169cbe3..7c56cfdf3123e33570cc13863bc16a95814144e1 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kBaseInfo.h
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kBaseInfo.h
@@ -58,11 +58,12 @@ enum {
   DAReg = 0x5,
   DA = 0x6,
   Reg = 0x7,
-  Disp8 = 0x8,
-  Imm8 = 0x9,
-  Imm16 = 0xA,
-  Imm32 = 0xB,
-  Imm3 = 0xC,
+  DReg = 0x8,
+  Disp8 = 0x9,
+  Imm8 = 0xA,
+  Imm16 = 0xB,
+  Imm32 = 0xC,
+  Imm3 = 0xD,
 };
 
 // Ctrl payload
@@ -181,7 +182,7 @@ static inline bool isAddressRegister(unsigned RegNo) {
   case M68k::WA4:
   case M68k::WA5:
   case M68k::WA6:
-  case M68k::WA7:
+  case M68k::WSP:
   case M68k::A0:
   case M68k::A1:
   case M68k::A2:
@@ -189,7 +190,6 @@ static inline bool isAddressRegister(unsigned RegNo) {
   case M68k::A4:
   case M68k::A5:
   case M68k::A6:
-  case M68k::A7:
   case M68k::SP:
     return true;
   default:
@@ -236,7 +236,7 @@ static inline unsigned getMaskedSpillRegister(unsigned order) {
   case 14:
     return M68k::A6;
   case 15:
-    return M68k::A7;
+    return M68k::SP;
   }
 }
 
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
index 59c49e85331957b328dc56a18bd6393d9b78666a..9708abaadf981ef6418b317ed34e8c7eeaaf144b 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
@@ -11,6 +11,7 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/M68kMCCodeEmitter.h"
 #include "MCTargetDesc/M68kBaseInfo.h"
 #include "MCTargetDesc/M68kFixupKinds.h"
 #include "MCTargetDesc/M68kMCTargetDesc.h"
@@ -27,13 +28,6 @@
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/raw_ostream.h"
 
-namespace llvm {
-namespace M68k {
-// Forward declarations
-const uint8_t *getMCInstrBeads(unsigned);
-} // end namespace M68k
-} // end namespace llvm
-
 using namespace llvm;
 
 #define DEBUG_TYPE "m68k-mccodeemitter"
@@ -127,6 +121,7 @@ unsigned M68kMCCodeEmitter::encodeReg(unsigned ThisByte, uint8_t Bead,
     Reg = false;
     DA = true;
     break;
+  case M68kBeads::DReg:
   case M68kBeads::Reg:
     Reg = true;
     DA = false;
@@ -357,6 +352,7 @@ void M68kMCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
       break;
     case M68kBeads::DAReg:
     case M68kBeads::DA:
+    case M68kBeads::DReg:
     case M68kBeads::Reg:
       Offset +=
           encodeReg(ThisByte, Bead, MI, Desc, Buffer, Offset, Fixups, STI);
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.h b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.h
new file mode 100644
index 0000000000000000000000000000000000000000..242a1297206a284ce560f88159c0ce41c4067a1c
--- /dev/null
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.h
@@ -0,0 +1,28 @@
+//===-- M68kMCCodeEmitter.h - M68k Code Emitter ----------------*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declarations for the code emitter which are useful
+/// outside of the emitter itself.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_M68K_MCTARGETDESC_M68KMCCODEEMITTER_H
+#define LLVM_LIB_TARGET_M68K_MCTARGETDESC_M68KMCCODEEMITTER_H
+
+#include <cstdint>
+
+namespace llvm {
+namespace M68k {
+
+const uint8_t *getMCInstrBeads(unsigned);
+
+} // namespace M68k
+} // namespace llvm
+
+#endif
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index 56643f64e6c2c15a737da597dd2945778a3b3531..0143f4f4b62a6b7cefc576d4f25999b23bba41f9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -200,7 +200,7 @@ static bool isALoadChain(Value *Start) {
       return false;
   }
   return true;
-};
+}
 
 void NVPTXLowerArgs::handleByValParam(Argument *Arg) {
   Function *Func = Arg->getParent();
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index 6f071040dd9dfcc5bf1ae4c9ea6ab2242dda3531..7d73d8f9cfd84f6e61d918a244f784e2fcc6f5ba 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -71,7 +71,9 @@ public:
 
   // Only <2 x half> should be vectorized, so always return 32 for the vector
   // register size.
-  unsigned getRegisterBitWidth(bool Vector) const { return 32; }
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+    return TypeSize::getFixed(32);
+  }
   unsigned getMinVectorRegisterBitWidth() const { return 32; }
 
   // We don't want to prevent inlining because of target-cpu and -features
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index c2840ebc5342603db685ab797fb94076e3a03eee..0503cbcdf6293595633ee3178849a914c349629c 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -252,9 +252,13 @@ def FeatureMMA : SubtargetFeature<"mma", "HasMMA", "true",
                                   "Enable MMA instructions",
                                   [FeatureP8Vector, FeatureP9Altivec,
                                    FeaturePairedVectorMemops]>;
-def FeatureROPProtection :
-  SubtargetFeature<"rop-protection", "HasROPProtection", "false",
-                   "Add ROP protection">;
+def FeatureROPProtect :
+  SubtargetFeature<"rop-protect", "HasROPProtect", "true",
+                   "Add ROP protect">;
+
+def FeaturePrivileged :
+  SubtargetFeature<"privileged", "HasPrivileged", "true",
+                   "Add privileged instructions">;
 
 def FeaturePredictableSelectIsExpensive :
   SubtargetFeature<"predictable-select-expensive",
@@ -323,8 +327,7 @@ def ProcessorFeatures {
      FeatureDirectMove,
      FeatureICBT,
      FeaturePartwordAtomic,
-     FeaturePredictableSelectIsExpensive,
-     FeatureROPProtection
+     FeaturePredictableSelectIsExpensive
     ];
 
   list<SubtargetFeature> P8SpecificFeatures = [FeatureAddiLoadFusion,
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 0f6212621ac2f3e3d7197eeb45804cbb9673107c..dd7b7a345d66db3fc3746f8b2b472ed006ac4ab8 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -1167,7 +1167,10 @@ static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, uint64_t Imm,
   const PPCSubtarget &Subtarget =
       CurDAG->getMachineFunction().getSubtarget<PPCSubtarget>();
 
-  if (Subtarget.hasPrefixInstrs()) {
+  // If we have prefixed instructions and there is a chance we can
+  // materialize the constant with fewer prefixed instructions than
+  // non-prefixed, try that.
+  if (Subtarget.hasPrefixInstrs() && InstCntDirect != 1) {
     unsigned InstCntDirectP = 0;
     SDNode *ResultP = selectI64ImmDirectPrefix(CurDAG, dl, Imm, InstCntDirectP);
     // Use the prefix case in either of two cases:
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 4fa1689a77c4793d83a71d7aa072fa11a02051b5..65ca0d81fc2ae3cffcadf2f1a60db4c4496a7a37 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -8820,6 +8820,18 @@ bool llvm::convertToNonDenormSingle(APInt &ArgAPInt) {
   return Success;
 }
 
+// Nondestructive check for convertTonNonDenormSingle.
+bool llvm::checkConvertToNonDenormSingle(APFloat &ArgAPFloat) {
+  // Only convert if it loses info, since XXSPLTIDP should
+  // handle the other case.
+  APFloat APFloatToConvert = ArgAPFloat;
+  bool LosesInfo = true;
+  APFloatToConvert.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
+                           &LosesInfo);
+
+  return (!LosesInfo && !APFloatToConvert.isDenormal());
+}
+
 // If this is a case we can't handle, return null and let the default
 // expansion code take care of it.  If we CAN select this case, and if it
 // selects to a single instruction, return Op.  Otherwise, if we can codegen
@@ -10111,7 +10123,7 @@ SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
           PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, WideVec,
           DAG.getConstant(Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo
                                                      : VecNo,
-                          dl, MVT::i64));
+                          dl, getPointerTy(DAG.getDataLayout())));
       RetOps.push_back(Extract);
     }
     return DAG.getMergeValues(RetOps, dl);
@@ -10395,7 +10407,7 @@ SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
   for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
     unsigned VecNum = Subtarget.isLittleEndian() ? NumVecs - 1 - Idx : Idx;
     SDValue Elt = DAG.getNode(PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, Value,
-                              DAG.getConstant(VecNum, dl, MVT::i64));
+                              DAG.getConstant(VecNum, dl, getPointerTy(DAG.getDataLayout())));
     SDValue Store =
         DAG.getStore(StoreChain, dl, Elt, BasePtr,
                      SN->getPointerInfo().getWithOffset(Idx * 16),
@@ -16115,10 +16127,8 @@ bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
   case MVT::f32:
   case MVT::f64:
     if (Subtarget.hasPrefixInstrs()) {
-      // With prefixed instructions, we can materialize anything that can be
-      // represented with a 32-bit immediate, not just positive zero.
-      APFloat APFloatOfImm = Imm;
-      return convertToNonDenormSingle(APFloatOfImm);
+      // we can materialize all immediatess via XXSPLTI32DX and XXSPLTIDP.
+      return true;
     }
     LLVM_FALLTHROUGH;
   case MVT::ppcf128:
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index f69b5aceccdc92a95a652dfcee1543a405313c25..9e85d6f510141b477159c4882e62323395b354fe 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -1332,6 +1332,7 @@ namespace llvm {
 
   bool convertToNonDenormSingle(APInt &ArgAPInt);
   bool convertToNonDenormSingle(APFloat &ArgAPFloat);
+  bool checkConvertToNonDenormSingle(APFloat &ArgAPFloat);
 
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 4d0595689d9e8c46b08731a3649d668124a847d8..6887f098ba1bf65ad19dc2debaa2eb7066a6be89 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -1108,6 +1108,7 @@ bool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
   case PPC::XXLXORspz:
   case PPC::XXLXORdpz:
   case PPC::XXLEQVOnes:
+  case PPC::XXSPLTI32DX:
   case PPC::V_SET0B:
   case PPC::V_SET0H:
   case PPC::V_SET0:
@@ -4416,21 +4417,17 @@ bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
     // Sign-extend to 64-bits.
     // DefMI may be folded with another imm form instruction, the result Imm is
     // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
+    APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
+    if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth))
+      return false;
+    if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth))
+      return false;
     Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
 
     if (Imm % III.ImmMustBeMultipleOf)
       return false;
     if (III.TruncateImmTo)
       Imm &= ((1 << III.TruncateImmTo) - 1);
-    if (III.SignedImm) {
-      APInt ActualValue(64, Imm, true);
-      if (!ActualValue.isSignedIntN(III.ImmWidth))
-        return false;
-    } else {
-      uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
-      if ((uint64_t)Imm > UnsignedMax)
-        return false;
-    }
   }
   else
     return false;
@@ -5160,7 +5157,8 @@ bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
   unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
   unsigned StackOffset = MI.getOperand(1).getImm();
   Register StackReg = MI.getOperand(2).getReg();
-  if (StackReg == PPC::X1 && StackOffset == TOCSaveOffset)
+  Register SPReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
+  if (StackReg == SPReg && StackOffset == TOCSaveOffset)
     return true;
 
   return false;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index bdd3728df7b9ea0b3358cc751dd48c59f723822c..aa820eee4dccaaea16741344e48d52d2c8cbad16 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -399,6 +399,30 @@ def getFPAs32BitInt : SDNodeXForm<fpimm, [{
                                    SDLoc(N), MVT::i32);
 }]>;
 
+// Check if the value can be converted to be single precision immediate, which
+// can be exploited by XXSPLTIDP. Ensure that it cannot be converted to single
+// precision before exploiting with XXSPLTI32DX.
+def nzFPImmAsi64 : PatLeaf<(fpimm), [{
+  APFloat APFloatOfN = N->getValueAPF();
+  return !N->isExactlyValue(+0.0) && !checkConvertToNonDenormSingle(APFloatOfN);
+}]>;
+
+// Get the Hi bits of a 64 bit immediate.
+def getFPAs64BitIntHi : SDNodeXForm<fpimm, [{
+  APFloat APFloatOfN = N->getValueAPF();
+  uint32_t Hi = (uint32_t)((APFloatOfN.bitcastToAPInt().getZExtValue() &
+                            0xFFFFFFFF00000000LL) >> 32);
+  return CurDAG->getTargetConstant(Hi, SDLoc(N), MVT::i32);
+}]>;
+
+// Get the Lo bits of a 64 bit immediate.
+def getFPAs64BitIntLo : SDNodeXForm<fpimm, [{
+  APFloat APFloatOfN = N->getValueAPF();
+  uint32_t Lo = (uint32_t)(APFloatOfN.bitcastToAPInt().getZExtValue() &
+                           0xFFFFFFFF);
+  return CurDAG->getTargetConstant(Lo, SDLoc(N), MVT::i32);
+}]>;
+
 def imm34 : PatLeaf<(imm), [{
   return isInt<34>(N->getSExtValue());
 }]>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index 5981bca372088c22e553f8f07f925793ec6a0156..9a6631c085550e4fb43f9b7bbbfa1d54a9ea36fa 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -18,10 +18,10 @@ def SDT_PPCPairBuild : SDTypeProfile<1, 2, [
   SDTCisVT<0, v256i1>, SDTCisVT<1, v4i32>, SDTCisVT<2, v4i32>
 ]>;
 def SDT_PPCAccExtractVsx : SDTypeProfile<1, 2, [
-  SDTCisVT<0, v4i32>, SDTCisVT<1, v512i1>, SDTCisInt<2>
+  SDTCisVT<0, v4i32>, SDTCisVT<1, v512i1>, SDTCisPtrTy<2>
 ]>;
 def SDT_PPCPairExtractVsx : SDTypeProfile<1, 2, [
-  SDTCisVT<0, v4i32>, SDTCisVT<1, v256i1>, SDTCisInt<2>
+  SDTCisVT<0, v4i32>, SDTCisVT<1, v256i1>, SDTCisPtrTy<2>
 ]>;
 def SDT_PPCxxmfacc : SDTypeProfile<1, 1, [
   SDTCisVT<0, v512i1>, SDTCisVT<1, v512i1>
@@ -1608,13 +1608,13 @@ let Predicates = [MMA] in {
                                               v16i8:$vs3, v16i8:$vs2)),
             (XXMTACC Concats.VecsToVecQuad)>;
   def : Pat<(v512i1 (PPCxxmfacc v512i1:$AS)), (XXMFACC acc:$AS)>;
-  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 0))),
+  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 0)),
             Extracts.Vec0>;
-  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 1))),
+  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 1)),
             Extracts.Vec1>;
-  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 2))),
+  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 2)),
             Extracts.Vec2>;
-  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 3))),
+  def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 3)),
             Extracts.Vec3>;
 }
 
@@ -1623,9 +1623,9 @@ let Predicates = [PairedVectorMemops] in {
             Concats.VecsToVecPair0>;
   def : Pat<(v256i1 (int_ppc_vsx_assemble_pair v16i8:$vs1, v16i8:$vs0)),
             Concats.VecsToVecPair0>;
-  def : Pat<(v4i32 (PPCPairExtractVsx vsrpevenrc:$v, (i64 0))),
+  def : Pat<(v4i32 (PPCPairExtractVsx vsrpevenrc:$v, 0)),
             (v4i32 (EXTRACT_SUBREG $v, sub_vsx0))>;
-  def : Pat<(v4i32 (PPCPairExtractVsx vsrpevenrc:$v, (i64 1))),
+  def : Pat<(v4i32 (PPCPairExtractVsx vsrpevenrc:$v, 1)),
             (v4i32 (EXTRACT_SUBREG $v, sub_vsx1))>;
 }
 
@@ -1867,14 +1867,6 @@ let Predicates = [PrefixInstrs] in {
                                       "xxspltidp $XT, $IMM32", IIC_VecGeneral,
                                       [(set v2f64:$XT,
                                             (PPCxxspltidp i32:$IMM32))]>;
-  def XXSPLTI32DX :
-      8RR_DForm_IMM32_XT6_IX<32, 0, (outs vsrc:$XT),
-                             (ins vsrc:$XTi, u1imm:$IX, i32imm:$IMM32),
-                             "xxsplti32dx $XT, $IX, $IMM32", IIC_VecGeneral,
-                             [(set v2i64:$XT,
-                                   (PPCxxsplti32dx v2i64:$XTi, i32:$IX,
-                                                   i32:$IMM32))]>,
-                             RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">;
   def XXPERMX :
     8RR_XX4Form_IMM3_XTABC6<34, 0, (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB,
                             vsrc:$XC, u3imm:$UIM),
@@ -1898,6 +1890,19 @@ let Predicates = [PrefixInstrs] in {
                        IIC_VecGeneral, []>;
 }
 
+// XXSPLI32DX needs extra flags to make sure the compiler does not attempt
+// to spill part of the instruction when the values are similar.
+let isReMaterializable = 1, isMoveImm = 1, Predicates = [PrefixInstrs] in {
+  def XXSPLTI32DX :
+      8RR_DForm_IMM32_XT6_IX<32, 0, (outs vsrc:$XT),
+                             (ins vsrc:$XTi, u1imm:$IX, i32imm:$IMM32),
+                             "xxsplti32dx $XT, $IX, $IMM32", IIC_VecGeneral,
+                             [(set v2i64:$XT,
+                                   (PPCxxsplti32dx v2i64:$XTi, i32:$IX,
+                                                   i32:$IMM32))]>,
+                             RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">;
+}
+
 let Predicates = [IsISA3_1] in {
   def SETBC : XForm_XT5_BI5<31, 384, (outs gprc:$RT), (ins crbitrc:$BI),
                             "setbc $RT, $BI", IIC_IntCompare, []>;
@@ -2623,6 +2628,19 @@ let AddedComplexity = 400, Predicates = [PrefixInstrs] in {
            (COPY_TO_REGCLASS (XXSPLTIDP (getFPAs32BitInt fpimm:$A)),
                              VSFRC)>;
 
+// To replace constant pool with XXSPLTI32DX for scalars.
+def : Pat<(f32 nzFPImmAsi64:$A),
+          (COPY_TO_REGCLASS (XXSPLTI32DX (XXSPLTI32DX(IMPLICIT_DEF), 0,
+                                        (getFPAs64BitIntHi $A)),
+                                        1, (getFPAs64BitIntLo $A)),
+                            VSRC)>;
+
+def : Pat<(f64 nzFPImmAsi64:$A),
+          (COPY_TO_REGCLASS (XXSPLTI32DX (XXSPLTI32DX (IMPLICIT_DEF), 0,
+                                        (getFPAs64BitIntHi $A)),
+                                        1, (getFPAs64BitIntLo $A)),
+                            VSRC)>;
+
   // Anonymous patterns for XXEVAL
   // AND
   // and(A, B, C)
diff --git a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
index c8b01aaef8280084fbbca3d2bdc95706b6d698e7..806908f23f2b9e9aad83ac5ea3554edc4b9b5fb5 100644
--- a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
@@ -226,28 +226,30 @@ getKnownLeadingZeroCount(MachineInstr *MI, const PPCInstrInfo *TII) {
 void PPCMIPeephole::UpdateTOCSaves(
   std::map<MachineInstr *, bool> &TOCSaves, MachineInstr *MI) {
   assert(TII->isTOCSaveMI(*MI) && "Expecting a TOC save instruction here");
-  assert(MF->getSubtarget<PPCSubtarget>().isELFv2ABI() &&
-         "TOC-save removal only supported on ELFv2");
-  PPCFunctionInfo *FI = MF->getInfo<PPCFunctionInfo>();
-
-  MachineBasicBlock *Entry = &MF->front();
-  uint64_t CurrBlockFreq = MBFI->getBlockFreq(MI->getParent()).getFrequency();
-
-  // If the block in which the TOC save resides is in a block that
-  // post-dominates Entry, or a block that is hotter than entry (keep in mind
-  // that early MachineLICM has already run so the TOC save won't be hoisted)
-  // we can just do the save in the prologue.
-  if (CurrBlockFreq > EntryFreq || MPDT->dominates(MI->getParent(), Entry))
-    FI->setMustSaveTOC(true);
-
-  // If we are saving the TOC in the prologue, all the TOC saves can be removed
-  // from the code.
-  if (FI->mustSaveTOC()) {
-    for (auto &TOCSave : TOCSaves)
-      TOCSave.second = false;
-    // Add new instruction to map.
-    TOCSaves[MI] = false;
-    return;
+  // FIXME: Saving TOC in prologue hasn't been implemented well in AIX ABI part,
+  // here only support it under ELFv2.
+  if (MF->getSubtarget<PPCSubtarget>().isELFv2ABI()) {
+    PPCFunctionInfo *FI = MF->getInfo<PPCFunctionInfo>();
+
+    MachineBasicBlock *Entry = &MF->front();
+    uint64_t CurrBlockFreq = MBFI->getBlockFreq(MI->getParent()).getFrequency();
+
+    // If the block in which the TOC save resides is in a block that
+    // post-dominates Entry, or a block that is hotter than entry (keep in mind
+    // that early MachineLICM has already run so the TOC save won't be hoisted)
+    // we can just do the save in the prologue.
+    if (CurrBlockFreq > EntryFreq || MPDT->dominates(MI->getParent(), Entry))
+      FI->setMustSaveTOC(true);
+
+    // If we are saving the TOC in the prologue, all the TOC saves can be
+    // removed from the code.
+    if (FI->mustSaveTOC()) {
+      for (auto &TOCSave : TOCSaves)
+        TOCSave.second = false;
+      // Add new instruction to map.
+      TOCSaves[MI] = false;
+      return;
+    }
   }
 
   bool Keep = true;
@@ -476,10 +478,12 @@ bool PPCMIPeephole::simplifyCode(void) {
         }
         break;
       }
+      case PPC::STW:
       case PPC::STD: {
         MachineFrameInfo &MFI = MF->getFrameInfo();
         if (MFI.hasVarSizedObjects() ||
-            !MF->getSubtarget<PPCSubtarget>().isELFv2ABI())
+            (!MF->getSubtarget<PPCSubtarget>().isELFv2ABI() &&
+             !MF->getSubtarget<PPCSubtarget>().isAIXABI()))
           break;
         // When encountering a TOC save instruction, call UpdateTOCSaves
         // to add it to the TOCSaves map and mark any existing TOC saves
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index bf98ea8a01d03b05c45b6cd77a1d731a5a63c80c..9bff07a1de0d1699f3bfb461e6eb877c07ef435f 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -87,7 +87,8 @@ void PPCSubtarget::initializeEnvironment() {
   HasP9Vector = false;
   HasP9Altivec = false;
   HasMMA = false;
-  HasROPProtection = false;
+  HasROPProtect = false;
+  HasPrivileged = false;
   HasP10Vector = false;
   HasPrefixInstrs = false;
   HasPCRelativeMemops = false;
@@ -182,9 +183,7 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   StackAlignment = getPlatformStackAlignment();
 
   // Determine endianness.
-  // FIXME: Part of the TargetMachine.
-  IsLittleEndian = (TargetTriple.getArch() == Triple::ppc64le ||
-                    TargetTriple.getArch() == Triple::ppcle);
+  IsLittleEndian = TM.isLittleEndian();
 }
 
 bool PPCSubtarget::enableMachineScheduler() const { return true; }
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h
index 04eb4a2f3da7934aa6b12bcb8d261179ba318aef..abc7ea1d14128175938547869e0718bedb3a1c54 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.h
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h
@@ -112,7 +112,8 @@ protected:
   bool HasPrefixInstrs;
   bool HasPCRelativeMemops;
   bool HasMMA;
-  bool HasROPProtection;
+  bool HasROPProtect;
+  bool HasPrivileged;
   bool HasFCPSGN;
   bool HasFSQRT;
   bool HasFRE, HasFRES, HasFRSQRTE, HasFRSQRTES;
@@ -274,7 +275,8 @@ public:
   bool hasPrefixInstrs() const { return HasPrefixInstrs; }
   bool hasPCRelativeMemops() const { return HasPCRelativeMemops; }
   bool hasMMA() const { return HasMMA; }
-  bool hasROPProtection() const { return HasROPProtection; }
+  bool hasROPProtect() const { return HasROPProtect; }
+  bool hasPrivileged() const { return HasPrivileged; }
   bool pairedVectorMemops() const { return PairedVectorMemops; }
   bool hasMFOCRF() const { return HasMFOCRF; }
   bool hasISEL() const { return HasISEL; }
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index 47fe6564041709c6378e64054c4793e38962b341..32b19d5ddd10dcba8f21f1959d51fa3a5a98b61f 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -126,13 +126,17 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCTarget() {
   initializeGlobalISel(PR);
 }
 
+static bool isLittleEndianTriple(const Triple &T) {
+  return T.getArch() == Triple::ppc64le || T.getArch() == Triple::ppcle;
+}
+
 /// Return the datalayout string of a subtarget.
 static std::string getDataLayoutString(const Triple &T) {
   bool is64Bit = T.getArch() == Triple::ppc64 || T.getArch() == Triple::ppc64le;
   std::string Ret;
 
   // Most PPC* platforms are big endian, PPC(64)LE is little endian.
-  if (T.getArch() == Triple::ppc64le || T.getArch() == Triple::ppcle)
+  if (isLittleEndianTriple(T))
     Ret = "e";
   else
     Ret = "E";
@@ -317,7 +321,8 @@ PPCTargetMachine::PPCTargetMachine(const Target &T, const Triple &TT,
                         getEffectiveRelocModel(TT, RM),
                         getEffectivePPCCodeModel(TT, CM, JIT), OL),
       TLOF(createTLOF(getTargetTriple())),
-      TargetABI(computeTargetABI(TT, Options)) {
+      TargetABI(computeTargetABI(TT, Options)),
+      Endianness(isLittleEndianTriple(TT) ? Endian::LITTLE : Endian::BIG) {
   initAsmInfo();
 }
 
@@ -540,6 +545,12 @@ PPCTargetMachine::getTargetTransformInfo(const Function &F) {
   return TargetTransformInfo(PPCTTIImpl(this, F));
 }
 
+bool PPCTargetMachine::isLittleEndian() const {
+  assert(Endianness != Endian::NOT_DETECTED &&
+         "Unable to determine endianness");
+  return Endianness == Endian::LITTLE;
+}
+
 static MachineSchedRegistry
 PPCPreRASchedRegistry("ppc-prera",
                       "Run PowerPC PreRA specific scheduler",
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.h b/llvm/lib/Target/PowerPC/PPCTargetMachine.h
index 21faa4e710e3eef04ac00a4eee66779c95ad26a0..ed9e74b72d1e433ac4a0e592910a1dd9fe00f303 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.h
@@ -25,9 +25,12 @@ namespace llvm {
 class PPCTargetMachine final : public LLVMTargetMachine {
 public:
   enum PPCABI { PPC_ABI_UNKNOWN, PPC_ABI_ELFv1, PPC_ABI_ELFv2 };
+  enum Endian { NOT_DETECTED, LITTLE, BIG };
+
 private:
   std::unique_ptr<TargetLoweringObjectFile> TLOF;
   PPCABI TargetABI;
+  Endian Endianness = Endian::NOT_DETECTED;
 
   mutable StringMap<std::unique_ptr<PPCSubtarget>> SubtargetMap;
 
@@ -63,6 +66,8 @@ public:
     // Addrspacecasts are always noops.
     return true;
   }
+
+  bool isLittleEndian() const;
 };
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index b3d8100fe016c27dd26ea449281123150f19beae..f8145fe4d3efb4d1e167f56a68866de32f40bf86 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -871,16 +871,18 @@ const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const {
   }
 }
 
-unsigned PPCTTIImpl::getRegisterBitWidth(bool Vector) const {
-  if (Vector) {
-    if (ST->hasAltivec()) return 128;
-    return 0;
+TypeSize
+PPCTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+  switch (K) {
+  case TargetTransformInfo::RGK_Scalar:
+    return TypeSize::getFixed(ST->isPPC64() ? 64 : 32);
+  case TargetTransformInfo::RGK_FixedWidthVector:
+    return TypeSize::getFixed(ST->hasAltivec() ? 128 : 0);
+  case TargetTransformInfo::RGK_ScalableVector:
+    return TypeSize::getScalable(0);
   }
 
-  if (ST->isPPC64())
-    return 64;
-  return 32;
-
+  llvm_unreachable("Unsupported register kind");
 }
 
 unsigned PPCTTIImpl::getCacheLineSize() const {
@@ -984,8 +986,8 @@ int PPCTTIImpl::getArithmeticInstrCost(unsigned Opcode, Type *Ty,
   return vectorCostAdjustment(Cost, Opcode, Ty, nullptr);
 }
 
-int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
-                               Type *SubTp) {
+int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
+                               ArrayRef<int> Mask, int Index, Type *SubTp) {
   // Legalize the type.
   std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
 
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index bc946715156fe7dc4f357cae3a4a491a086b5a64..87ff0e32bdeaa522d5f647292347c8e8d89781fc 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -93,7 +93,7 @@ public:
   unsigned getNumberOfRegisters(unsigned ClassID) const;
   unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const;
   const char* getRegisterClassName(unsigned ClassID) const;
-  unsigned getRegisterBitWidth(bool Vector) const;
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
   unsigned getCacheLineSize() const override;
   unsigned getPrefetchDistance() const override;
   unsigned getMaxInterleaveFactor(unsigned VF);
@@ -107,7 +107,8 @@ public:
       TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None,
       ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
       const Instruction *CxtI = nullptr);
-  int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp);
+  int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, ArrayRef<int> Mask,
+                     int Index, Type *SubTp);
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                        TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
                        const Instruction *I = nullptr);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
index 8ce2184c7a411e108380bf208fa8cfc5f43c9802..120c6d95bfc06fe0754eeb531ee586835e84a0c6 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
@@ -138,7 +138,8 @@ RISCVMCExpr::VariantKind RISCVMCExpr::getVariantKindForName(StringRef name) {
 
 StringRef RISCVMCExpr::getVariantKindName(VariantKind Kind) {
   switch (Kind) {
-  default:
+  case VK_RISCV_Invalid:
+  case VK_RISCV_None:
     llvm_unreachable("Invalid ELF symbol kind");
   case VK_RISCV_LO:
     return "lo";
@@ -160,7 +161,14 @@ StringRef RISCVMCExpr::getVariantKindName(VariantKind Kind) {
     return "tls_ie_pcrel_hi";
   case VK_RISCV_TLS_GD_HI:
     return "tls_gd_pcrel_hi";
+  case VK_RISCV_CALL:
+    return "call";
+  case VK_RISCV_CALL_PLT:
+    return "call_plt";
+  case VK_RISCV_32_PCREL:
+    return "32_pcrel";
   }
+  llvm_unreachable("Invalid ELF symbol kind");
 }
 
 static void fixELFSymbolsInTLSFixupsImpl(const MCExpr *Expr, MCAssembler &Asm) {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
index 49aedcb183cceb28751aec90a24048dc3ef301a2..4c2eeab639bd4959372e91e14faa0dca28f2fcb5 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
@@ -37,7 +37,7 @@ public:
     VK_RISCV_CALL,
     VK_RISCV_CALL_PLT,
     VK_RISCV_32_PCREL,
-    VK_RISCV_Invalid
+    VK_RISCV_Invalid // Must be the last item
   };
 
 private:
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index a0b7c9bb9c2ff4ebe2f3d9700e81211b4fa1fed3..d23dd62e9834e61ed8de8bf0cad70e96dc0f41f3 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -30,11 +30,10 @@ class MachineInstr;
 class MachineOperand;
 class PassRegistry;
 
-void LowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
-                                    const AsmPrinter &AP);
+bool lowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
+                                    AsmPrinter &AP);
 bool LowerRISCVMachineOperandToMCOperand(const MachineOperand &MO,
                                          MCOperand &MCOp, const AsmPrinter &AP);
-bool LowerEPIMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI);
 
 FunctionPass *createRISCVISelDag(RISCVTargetMachine &TM);
 
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index e7a17a44a8e2acdedec1c40a25f965b7880d8aa0..ce84a91cc3fddd3a61ff7d6803e4bcdac0a5c7bc 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -95,8 +95,8 @@ void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
     return;
 
   MCInst TmpInst;
-  LowerRISCVMachineInstrToMCInst(MI, TmpInst, *this);
-  EmitToStreamer(*OutStreamer, TmpInst);
+  if (!lowerRISCVMachineInstrToMCInst(MI, TmpInst, *this))
+    EmitToStreamer(*OutStreamer, TmpInst);
 }
 
 bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index ec9a395699526c4bd5bb679165cecf5b2d0d8fde..581f26c64abc9d1cba9c88201d6075f6d579c74f 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -62,6 +62,8 @@ private:
   bool expandVSetVL(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
   bool expandVMSET_VMCLR(MachineBasicBlock &MBB,
                          MachineBasicBlock::iterator MBBI, unsigned Opcode);
+  bool expandVSPILL(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
+  bool expandVRELOAD(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
 };
 
 char RISCVExpandPseudo::ID = 0;
@@ -123,6 +125,30 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
   case RISCV::PseudoVMSET_M_B64:
     // vmset.m vd => vmxnor.mm vd, vd, vd
     return expandVMSET_VMCLR(MBB, MBBI, RISCV::VMXNOR_MM);
+  case RISCV::PseudoVSPILL2_M1:
+  case RISCV::PseudoVSPILL2_M2:
+  case RISCV::PseudoVSPILL2_M4:
+  case RISCV::PseudoVSPILL3_M1:
+  case RISCV::PseudoVSPILL3_M2:
+  case RISCV::PseudoVSPILL4_M1:
+  case RISCV::PseudoVSPILL4_M2:
+  case RISCV::PseudoVSPILL5_M1:
+  case RISCV::PseudoVSPILL6_M1:
+  case RISCV::PseudoVSPILL7_M1:
+  case RISCV::PseudoVSPILL8_M1:
+    return expandVSPILL(MBB, MBBI);
+  case RISCV::PseudoVRELOAD2_M1:
+  case RISCV::PseudoVRELOAD2_M2:
+  case RISCV::PseudoVRELOAD2_M4:
+  case RISCV::PseudoVRELOAD3_M1:
+  case RISCV::PseudoVRELOAD3_M2:
+  case RISCV::PseudoVRELOAD4_M1:
+  case RISCV::PseudoVRELOAD4_M2:
+  case RISCV::PseudoVRELOAD5_M1:
+  case RISCV::PseudoVRELOAD6_M1:
+  case RISCV::PseudoVRELOAD7_M1:
+  case RISCV::PseudoVRELOAD8_M1:
+    return expandVRELOAD(MBB, MBBI);
   }
 
   return false;
@@ -253,6 +279,96 @@ bool RISCVExpandPseudo::expandVMSET_VMCLR(MachineBasicBlock &MBB,
   return true;
 }
 
+bool RISCVExpandPseudo::expandVSPILL(MachineBasicBlock &MBB,
+                                     MachineBasicBlock::iterator MBBI) {
+  const TargetRegisterInfo *TRI =
+      MBB.getParent()->getSubtarget().getRegisterInfo();
+  DebugLoc DL = MBBI->getDebugLoc();
+  Register SrcReg = MBBI->getOperand(0).getReg();
+  Register Base = MBBI->getOperand(1).getReg();
+  Register VL = MBBI->getOperand(2).getReg();
+  auto ZvlssegInfo = TII->isRVVSpillForZvlsseg(MBBI->getOpcode());
+  if (!ZvlssegInfo)
+    return false;
+  unsigned NF = ZvlssegInfo->first;
+  unsigned LMUL = ZvlssegInfo->second;
+  assert(NF * LMUL <= 8 && "Invalid NF/LMUL combinations.");
+  unsigned Opcode = RISCV::VS1R_V;
+  unsigned SubRegIdx = RISCV::sub_vrm1_0;
+  static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
+                "Unexpected subreg numbering");
+  if (LMUL == 2) {
+    Opcode = RISCV::VS2R_V;
+    SubRegIdx = RISCV::sub_vrm2_0;
+    static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
+                  "Unexpected subreg numbering");
+  } else if (LMUL == 4) {
+    Opcode = RISCV::VS4R_V;
+    SubRegIdx = RISCV::sub_vrm4_0;
+    static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
+                  "Unexpected subreg numbering");
+  } else
+    assert(LMUL == 1 && "LMUL must be 1, 2, or 4.");
+
+  for (unsigned I = 0; I < NF; ++I) {
+    BuildMI(MBB, MBBI, DL, TII->get(Opcode))
+        .addReg(TRI->getSubReg(SrcReg, SubRegIdx + I))
+        .addReg(Base)
+        .addMemOperand(*(MBBI->memoperands_begin()));
+    if (I != NF - 1)
+      BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADD), Base)
+          .addReg(Base)
+          .addReg(VL);
+  }
+  MBBI->eraseFromParent();
+  return true;
+}
+
+bool RISCVExpandPseudo::expandVRELOAD(MachineBasicBlock &MBB,
+                                      MachineBasicBlock::iterator MBBI) {
+  const TargetRegisterInfo *TRI =
+      MBB.getParent()->getSubtarget().getRegisterInfo();
+  DebugLoc DL = MBBI->getDebugLoc();
+  Register DestReg = MBBI->getOperand(0).getReg();
+  Register Base = MBBI->getOperand(1).getReg();
+  Register VL = MBBI->getOperand(2).getReg();
+  auto ZvlssegInfo = TII->isRVVSpillForZvlsseg(MBBI->getOpcode());
+  if (!ZvlssegInfo)
+    return false;
+  unsigned NF = ZvlssegInfo->first;
+  unsigned LMUL = ZvlssegInfo->second;
+  assert(NF * LMUL <= 8 && "Invalid NF/LMUL combinations.");
+  unsigned Opcode = RISCV::VL1RE8_V;
+  unsigned SubRegIdx = RISCV::sub_vrm1_0;
+  static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
+                "Unexpected subreg numbering");
+  if (LMUL == 2) {
+    Opcode = RISCV::VL2RE8_V;
+    SubRegIdx = RISCV::sub_vrm2_0;
+    static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
+                  "Unexpected subreg numbering");
+  } else if (LMUL == 4) {
+    Opcode = RISCV::VL4RE8_V;
+    SubRegIdx = RISCV::sub_vrm4_0;
+    static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
+                  "Unexpected subreg numbering");
+  } else
+    assert(LMUL == 1 && "LMUL must be 1, 2, or 4.");
+
+  for (unsigned I = 0; I < NF; ++I) {
+    BuildMI(MBB, MBBI, DL, TII->get(Opcode),
+            TRI->getSubReg(DestReg, SubRegIdx + I))
+        .addReg(Base)
+        .addMemOperand(*(MBBI->memoperands_begin()));
+    if (I != NF - 1)
+      BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADD), Base)
+          .addReg(Base)
+          .addReg(VL);
+  }
+  MBBI->eraseFromParent();
+  return true;
+}
+
 } // end of anonymous namespace
 
 INITIALIZE_PASS(RISCVExpandPseudo, "riscv-expand-pseudo",
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 53ec621f34bfe8e8d596e6dfda1aaafd3e0dd33c..7a21e62b54471914e697fb15d3575947d2a0430e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -667,11 +667,8 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       ReplaceNode(Node, New.getNode());
       return;
     }
-    int64_t Imm = ConstNode->getSExtValue();
-    if (XLenVT == MVT::i64) {
-      ReplaceNode(Node, selectImm(CurDAG, DL, Imm, XLenVT));
-      return;
-    }
+    ReplaceNode(Node, selectImm(CurDAG, DL, ConstNode->getSExtValue(), XLenVT));
+    return;
     break;
   }
   case ISD::FrameIndex: {
@@ -1376,10 +1373,8 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       return ReplaceNode(Node, NewNode);
     }
 
-    SDNode *NewNode = CurDAG->getMachineNode(
-        TargetOpcode::INSERT_SUBREG, DL, VT, V, SubV,
-        CurDAG->getTargetConstant(SubRegIdx, DL, XLenVT));
-    return ReplaceNode(Node, NewNode);
+    SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
+    return ReplaceNode(Node, Insert.getNode());
   }
   case ISD::EXTRACT_SUBVECTOR: {
     SDValue V = Node->getOperand(0);
@@ -1421,10 +1416,8 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       return ReplaceNode(Node, NewNode);
     }
 
-    SDNode *NewNode = CurDAG->getMachineNode(
-        TargetOpcode::EXTRACT_SUBREG, DL, VT, V,
-        CurDAG->getTargetConstant(SubRegIdx, DL, XLenVT));
-    return ReplaceNode(Node, NewNode);
+    SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
+    return ReplaceNode(Node, Extract.getNode());
   }
   }
 
@@ -1523,7 +1516,7 @@ bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) {
 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) {
   if (N.getOpcode() == ISD::AND) {
     auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
-    if (C && CheckAndMask(N.getOperand(0), C, UINT64_C(0xFFFFFFFF))) {
+    if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) {
       Val = N.getOperand(0);
       return true;
     }
@@ -1538,27 +1531,6 @@ bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) {
   return false;
 }
 
-// Match (srl (and val, mask), imm) where the result would be a
-// zero-extended 32-bit integer. i.e. the mask is 0xffffffff or the result
-// is equivalent to this (SimplifyDemandedBits may have removed lower bits
-// from the mask that aren't necessary due to the right-shifting).
-bool RISCVDAGToDAGISel::MatchSRLIW(SDNode *N) const {
-  assert(N->getOpcode() == ISD::SRL);
-  assert(N->getOperand(0).getOpcode() == ISD::AND);
-  assert(isa<ConstantSDNode>(N->getOperand(1)));
-  assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
-
-  // The IsRV64 predicate is checked after PatFrag predicates so we can get
-  // here even on RV32.
-  if (!Subtarget->is64Bit())
-    return false;
-
-  SDValue And = N->getOperand(0);
-  uint64_t ShAmt = N->getConstantOperandVal(1);
-  uint64_t Mask = And.getConstantOperandVal(1);
-  return (Mask | maskTrailingOnes<uint64_t>(ShAmt)) == 0xffffffff;
-}
-
 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
 // on RV64).
 // SLLIUW is the same as SLLI except for the fact that it clears the bits
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index 4fa6f5402f781d36237c5ff3990f4dc83cf2a0ae..e83e62d2090e9556315337588a37644a8f86e126 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -57,7 +57,6 @@ public:
   bool selectSExti32(SDValue N, SDValue &Val);
   bool selectZExti32(SDValue N, SDValue &Val);
 
-  bool MatchSRLIW(SDNode *N) const;
   bool MatchSLLIUW(SDNode *N) const;
 
   bool selectVLOp(SDValue N, SDValue &VL);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 8a873e3f16c73bcf84ba493fca13a7101ef467ed..7c7b2dbf465f193fd2163e66a5c4a6dd75a98eb0 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -208,6 +208,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
     setOperationAction(ISD::UADDO, MVT::i32, Custom);
     setOperationAction(ISD::USUBO, MVT::i32, Custom);
+    setOperationAction(ISD::UADDSAT, MVT::i32, Custom);
+    setOperationAction(ISD::USUBSAT, MVT::i32, Custom);
   }
 
   if (!Subtarget.hasStdExtM()) {
@@ -474,9 +476,16 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
 
+      setOperationAction(ISD::MLOAD, VT, Custom);
+      setOperationAction(ISD::MSTORE, VT, Custom);
+      setOperationAction(ISD::MGATHER, VT, Custom);
+      setOperationAction(ISD::MSCATTER, VT, Custom);
+
+      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
 
+      setOperationAction(ISD::STEP_VECTOR, VT, Custom);
       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
     }
 
@@ -514,6 +523,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
       setOperationAction(ISD::FCOPYSIGN, VT, Legal);
 
+      setOperationAction(ISD::MLOAD, VT, Custom);
+      setOperationAction(ISD::MSTORE, VT, Custom);
+      setOperationAction(ISD::MGATHER, VT, Custom);
+      setOperationAction(ISD::MSCATTER, VT, Custom);
+
+      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
 
@@ -559,6 +574,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
         setOperationAction(ISD::TRUNCATE, VT, Custom);
 
+        setOperationAction(ISD::BITCAST, VT, Custom);
+
         // Operations below are different for between masks and other vectors.
         if (VT.getVectorElementType() == MVT::i1) {
           setOperationAction(ISD::AND, VT, Custom);
@@ -570,6 +587,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
 
+        setOperationAction(ISD::MLOAD, VT, Custom);
+        setOperationAction(ISD::MSTORE, VT, Custom);
+        setOperationAction(ISD::MGATHER, VT, Custom);
+        setOperationAction(ISD::MSCATTER, VT, Custom);
         setOperationAction(ISD::ADD, VT, Custom);
         setOperationAction(ISD::MUL, VT, Custom);
         setOperationAction(ISD::SUB, VT, Custom);
@@ -604,8 +625,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
         setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
 
-        setOperationAction(ISD::BITCAST, VT, Custom);
-
         // Custom-lower reduction operations to set up the corresponding custom
         // nodes' operands.
         setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
@@ -641,6 +660,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
         setOperationAction(ISD::LOAD, VT, Custom);
         setOperationAction(ISD::STORE, VT, Custom);
+        setOperationAction(ISD::MLOAD, VT, Custom);
+        setOperationAction(ISD::MSTORE, VT, Custom);
+        setOperationAction(ISD::MGATHER, VT, Custom);
+        setOperationAction(ISD::MSCATTER, VT, Custom);
         setOperationAction(ISD::FADD, VT, Custom);
         setOperationAction(ISD::FSUB, VT, Custom);
         setOperationAction(ISD::FMUL, VT, Custom);
@@ -684,9 +707,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setTargetDAGCombine(ISD::OR);
   }
 
-  if (Subtarget.hasStdExtV())
-    setTargetDAGCombine(ISD::FCOPYSIGN);
-
   if (Subtarget.hasStdExtV()) {
     // EPI & VPred intrinsics may have illegal operands/results
     for (auto VT : {MVT::i1, MVT::i8, MVT::i16, MVT::i32, MVT::nxv1i32}) {
@@ -772,6 +792,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::FMAXNUM, VT, Legal);
     }
   }
+
+  if (Subtarget.hasStdExtV()) {
+    setTargetDAGCombine(ISD::FCOPYSIGN);
+    setTargetDAGCombine(ISD::MGATHER);
+    setTargetDAGCombine(ISD::MSCATTER);
+  }
 }
 
 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
@@ -1046,83 +1072,8 @@ RISCVTargetLowering::lowerZERO_EXTEND_VECTOR_INREG(SDValue Op,
   return lowerExtendVectorInReg(Op, DAG, RISCVISD::ZERO_EXTEND_BITS_INREG);
 }
 
-SDValue RISCVTargetLowering::lowerMGATHER(SDValue Op, SelectionDAG &DAG) const {
-  SDLoc DL(Op);
-  EVT VT = Op.getValueType();
-  assert(VT.isScalableVector() && "Unexpected type");
-
-  SDValue Indices = Op.getOperand(4);
-  EVT IndexVT = Indices.getValueType();
-
-  SDValue Offsets;
-  if (Op.getConstantOperandVal(5) == 1) {
-    Offsets = Indices;
-  } else {
-    SDValue VSLLOperands[] = {
-        DAG.getTargetConstant(Intrinsic::epi_vsll, DL, MVT::i64), Indices,
-        DAG.getConstant(Log2_64(Op.getConstantOperandVal(5)), DL,
-                        MVT::i64),           // log2(Scale).
-        DAG.getRegister(RISCV::X0, MVT::i64) // VLMAX.
-    };
-    // FIXME: This may overflow. RVV-0.9 provides mechanisms to decouple the
-    // index width from the data element width, solving this problem.
-    Offsets = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, VSLLOperands);
-  }
-
-  SDValue VLXEOperands[] = {
-      Op.getOperand(0), // Chain.
-      DAG.getTargetConstant(Intrinsic::epi_vload_indexed_mask, DL, MVT::i64),
-      Op.getOperand(1), // Merge.
-      Op.getOperand(3), // Base address.
-      Offsets,
-      Op.getOperand(2),                    // Mask.
-      DAG.getRegister(RISCV::X0, MVT::i64) // VLMAX.
-  };
-
-  return DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, Op->getVTList(), VLXEOperands);
-}
-
-SDValue RISCVTargetLowering::lowerMSCATTER(SDValue Op,
-                                           SelectionDAG &DAG) const {
-  SDLoc DL(Op);
-  EVT VT = Op.getOperand(1).getValueType();
-  assert(VT.isScalableVector() && "Unexpected type");
-
-  SDValue Indices = Op.getOperand(4);
-  EVT IndexVT = Indices.getValueType();
-
-  SDValue Offsets;
-  if (Op.getConstantOperandVal(5) == 1) {
-    Offsets = Indices;
-  } else {
-    SDValue VSLLOperands[] = {
-        DAG.getTargetConstant(Intrinsic::epi_vsll, DL, MVT::i64), Indices,
-        DAG.getConstant(Log2_64(Op.getConstantOperandVal(5)), DL,
-                        MVT::i64),           // log2(Scale).
-        DAG.getRegister(RISCV::X0, MVT::i64) // VLMAX.
-    };
-    // FIXME: This may overflow. RVV-0.9 provides mechanisms to decouple the
-    // index width from the data element width, solving this problem.
-    Offsets = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, VSLLOperands);
-  }
-
-  // Val, OutChain = GATHER (InChain, PassThru, Mask, BasePtr, Index, Scale)
-  //      OutChain = SCATTER(InChain, Value,    Mask, BasePtr, Index, Scale)
-  SDValue VSXEOperands[] = {
-      Op.getOperand(0), // Chain.
-      DAG.getTargetConstant(Intrinsic::epi_vstore_indexed_mask, DL, MVT::i64),
-      Op.getOperand(1), // Data
-      Op.getOperand(3), // Base address.
-      Offsets,
-      Op.getOperand(2),                    // Mask.
-      DAG.getRegister(RISCV::X0, MVT::i64) // VLMAX.
-  };
-
-  return DAG.getNode(ISD::INTRINSIC_VOID, DL, Op->getVTList(), VSXEOperands);
-}
-
 SDValue RISCVTargetLowering::lowerVECLIBCALL(SDValue Op, SelectionDAG &DAG,
-    ArrayRef<VTToLibCall> VTToLC) const 
+    ArrayRef<VTToLibCall> VTToLC) const
 {
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
@@ -1414,6 +1365,9 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
   SDValue Mask, VL;
   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
+  MVT XLenVT = Subtarget.getXLenVT();
+  unsigned NumElts = Op.getNumOperands();
+
   if (VT.getVectorElementType() == MVT::i1) {
     if (ISD::isBuildVectorAllZeros(Op.getNode())) {
       SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
@@ -1425,6 +1379,74 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
       return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
     }
 
+    // Lower constant mask BUILD_VECTORs via an integer vector type, in
+    // scalar integer chunks whose bit-width depends on the number of mask
+    // bits and XLEN.
+    // First, determine the most appropriate scalar integer type to use. This
+    // is at most XLenVT, but may be shrunk to a smaller vector element type
+    // according to the size of the final vector - use i8 chunks rather than
+    // XLenVT if we're producing a v8i1. This results in more consistent
+    // codegen across RV32 and RV64.
+    // If we have to use more than one INSERT_VECTOR_ELT then this optimization
+    // is likely to increase code size; avoid peforming it in such a case.
+    unsigned NumViaIntegerBits =
+        std::min(std::max(NumElts, 8u), Subtarget.getXLen());
+    if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
+        (!DAG.shouldOptForSize() || NumElts <= NumViaIntegerBits)) {
+      // Now we can create our integer vector type. Note that it may be larger
+      // than the resulting mask type: v4i1 would use v1i8 as its integer type.
+      MVT IntegerViaVecVT =
+          MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
+                           divideCeil(NumElts, NumViaIntegerBits));
+
+      uint64_t Bits = 0;
+      unsigned BitPos = 0, IntegerEltIdx = 0;
+      SDValue Vec = DAG.getUNDEF(IntegerViaVecVT);
+
+      for (unsigned I = 0; I < NumElts; I++, BitPos++) {
+        // Once we accumulate enough bits to fill our scalar type, insert into
+        // our vector and clear our accumulated data.
+        if (I != 0 && I % NumViaIntegerBits == 0) {
+          if (NumViaIntegerBits <= 32)
+            Bits = SignExtend64(Bits, 32);
+          SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
+          Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec,
+                            Elt, DAG.getConstant(IntegerEltIdx, DL, XLenVT));
+          Bits = 0;
+          BitPos = 0;
+          IntegerEltIdx++;
+        }
+        SDValue V = Op.getOperand(I);
+        bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
+        Bits |= ((uint64_t)BitValue << BitPos);
+      }
+
+      // Insert the (remaining) scalar value into position in our integer
+      // vector type.
+      if (NumViaIntegerBits <= 32)
+        Bits = SignExtend64(Bits, 32);
+      SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
+      Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt,
+                        DAG.getConstant(IntegerEltIdx, DL, XLenVT));
+
+      if (NumElts < NumViaIntegerBits) {
+        // If we're producing a smaller vector than our minimum legal integer
+        // type, bitcast to the equivalent (known-legal) mask type, and extract
+        // our final mask.
+        assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
+        Vec = DAG.getBitcast(MVT::v8i1, Vec);
+        Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
+                          DAG.getConstant(0, DL, XLenVT));
+      } else {
+        // Else we must have produced an integer type with the same size as the
+        // mask type; bitcast for the final result.
+        assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
+        Vec = DAG.getBitcast(VT, Vec);
+      }
+
+      return Vec;
+    }
+
     return SDValue();
   }
 
@@ -1439,10 +1461,10 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
   // instruction. An all-undef vector is matched by getSplatValue, above.
   if (VT.isInteger()) {
     bool IsVID = true;
-    for (unsigned i = 0, e = Op.getNumOperands(); i < e && IsVID; i++)
-      IsVID &= Op.getOperand(i).isUndef() ||
-               (isa<ConstantSDNode>(Op.getOperand(i)) &&
-                Op.getConstantOperandVal(i) == i);
+    for (unsigned I = 0; I < NumElts && IsVID; I++)
+      IsVID &= Op.getOperand(I).isUndef() ||
+               (isa<ConstantSDNode>(Op.getOperand(I)) &&
+                Op.getConstantOperandVal(I) == I);
 
     if (IsVID) {
       SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
@@ -1450,14 +1472,143 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
     }
   }
 
+  // Attempt to detect "hidden" splats, which only reveal themselves as splats
+  // when re-interpreted as a vector with a larger element type. For example,
+  //   v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
+  // could be instead splat as
+  //   v2i32 = build_vector i32 0x00010000, i32 0x00010000
+  // TODO: This optimization could also work on non-constant splats, but it
+  // would require bit-manipulation instructions to construct the splat value.
+  SmallVector<SDValue> Sequence;
+  unsigned EltBitSize = VT.getScalarSizeInBits();
+  const auto *BV = cast<BuildVectorSDNode>(Op);
+  if (VT.isInteger() && EltBitSize < 64 &&
+      ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
+      BV->getRepeatedSequence(Sequence) &&
+      (Sequence.size() * EltBitSize) <= 64) {
+    unsigned SeqLen = Sequence.size();
+    MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
+    MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, NumElts / SeqLen);
+    assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
+            ViaIntVT == MVT::i64) &&
+           "Unexpected sequence type");
+
+    unsigned EltIdx = 0;
+    uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
+    uint64_t SplatValue = 0;
+    // Construct the amalgamated value which can be splatted as this larger
+    // vector type.
+    for (const auto &SeqV : Sequence) {
+      if (!SeqV.isUndef())
+        SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
+                       << (EltIdx * EltBitSize));
+      EltIdx++;
+    }
+
+    // On RV64, sign-extend from 32 to 64 bits where possible in order to
+    // achieve better constant materializion.
+    if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
+      SplatValue = SignExtend64(SplatValue, 32);
+
+    // Since we can't introduce illegal i64 types at this stage, we can only
+    // perform an i64 splat on RV32 if it is its own sign-extended value. That
+    // way we can use RVV instructions to splat.
+    assert((ViaIntVT.bitsLE(XLenVT) ||
+            (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
+           "Unexpected bitcast sequence");
+    if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
+      SDValue ViaVL =
+          DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
+      MVT ViaContainerVT =
+          RISCVTargetLowering::getContainerForFixedLengthVector(DAG, ViaVecVT,
+                                                                Subtarget);
+      SDValue Splat =
+          DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
+                      DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
+      Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
+      return DAG.getBitcast(VT, Splat);
+    }
+  }
+
+  // Try and optimize BUILD_VECTORs with "dominant values" - these are values
+  // which constitute a large proportion of the elements. In such cases we can
+  // splat a vector with the dominant element and make up the shortfall with
+  // INSERT_VECTOR_ELTs.
+  // Note that this includes vectors of 2 elements by association. The
+  // upper-most element is the "dominant" one, allowing us to use a splat to
+  // "insert" the upper element, and an insert of the lower element at position
+  // 0, which improves codegen.
+  SDValue DominantValue;
+  unsigned MostCommonCount = 0;
+  DenseMap<SDValue, unsigned> ValueCounts;
+  unsigned NumUndefElts =
+      count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
+
+  for (SDValue V : Op->op_values()) {
+    if (V.isUndef())
+      continue;
+
+    ValueCounts.insert(std::make_pair(V, 0));
+    unsigned &Count = ValueCounts[V];
+
+    // Is this value dominant? In case of a tie, prefer the highest element as
+    // it's cheaper to insert near the beginning of a vector than it is at the
+    // end.
+    if (++Count >= MostCommonCount) {
+      DominantValue = V;
+      MostCommonCount = Count;
+    }
+  }
+
+  assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
+  unsigned NumDefElts = NumElts - NumUndefElts;
+  unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
+
+  // Don't perform this optimization when optimizing for size, since
+  // materializing elements and inserting them tends to cause code bloat.
+  if (!DAG.shouldOptForSize() &&
+      ((MostCommonCount > DominantValueCountThreshold) ||
+       (ValueCounts.size() <= Log2_32(NumDefElts)))) {
+    // Start by splatting the most common element.
+    SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
+
+    DenseSet<SDValue> Processed{DominantValue};
+    MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
+    for (const auto &OpIdx : enumerate(Op->ops())) {
+      const SDValue &V = OpIdx.value();
+      if (V.isUndef() || !Processed.insert(V).second)
+        continue;
+      if (ValueCounts[V] == 1) {
+        Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
+                          DAG.getConstant(OpIdx.index(), DL, XLenVT));
+      } else {
+        // Blend in all instances of this value using a VSELECT, using a
+        // mask where each bit signals whether that element is the one
+        // we're after.
+        SmallVector<SDValue> Ops;
+        transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
+          return DAG.getConstant(V == V1, DL, XLenVT);
+        });
+        Vec = DAG.getNode(ISD::VSELECT, DL, VT,
+                          DAG.getBuildVector(SelMaskTy, DL, Ops),
+                          DAG.getSplatBuildVector(VT, DL, V), Vec);
+      }
+    }
+
+    return Vec;
+  }
+
   return SDValue();
 }
 
 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
                                    const RISCVSubtarget &Subtarget) {
   SDValue V1 = Op.getOperand(0);
+  SDValue V2 = Op.getOperand(1);
   SDLoc DL(Op);
+  MVT XLenVT = Subtarget.getXLenVT();
   MVT VT = Op.getSimpleValueType();
+  unsigned NumElts = VT.getVectorNumElements();
   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
 
   if (SVN->isSplat()) {
@@ -1467,11 +1618,10 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
           DAG, VT, Subtarget);
 
       V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
-      assert(Lane < (int)VT.getVectorNumElements() && "Unexpected lane!");
+      assert(Lane < (int)NumElts && "Unexpected lane!");
 
       SDValue Mask, VL;
       std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
-      MVT XLenVT = Subtarget.getXLenVT();
       SDValue Gather =
           DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
                       DAG.getConstant(Lane, DL, XLenVT), Mask, VL);
@@ -1479,6 +1629,29 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
     }
   }
 
+  // Detect shuffles which can be re-expressed as vector selects.
+  SmallVector<SDValue> MaskVals;
+  // By default we preserve the original operand order, and select LHS as true
+  // and RHS as false. However, since RVV vector selects may feature splats but
+  // only on the LHS, we may choose to invert our mask and instead select
+  // between RHS and LHS.
+  bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
+
+  bool IsSelect = all_of(enumerate(SVN->getMask()), [&](const auto &MaskIdx) {
+    int MaskIndex = MaskIdx.value();
+    bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ SwapOps;
+    MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
+    return MaskIndex < 0 || MaskIdx.index() == (unsigned)MaskIndex % NumElts;
+  });
+
+  if (IsSelect) {
+    assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
+    MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
+    SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
+    return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SwapOps ? V2 : V1,
+                       SwapOps ? V1 : V2);
+  }
+
   return SDValue();
 }
 
@@ -1569,10 +1742,6 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
   case ISD::ZERO_EXTEND_VECTOR_INREG:
     return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
-  case ISD::MGATHER:
-    return lowerMGATHER(Op, DAG);
-  case ISD::MSCATTER:
-    return lowerMSCATTER(Op, DAG);
   case ISD::FEXP:
     return lowerFEXP(Op, DAG);
   case ISD::FSIN:
@@ -1884,6 +2053,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return lowerINSERT_SUBVECTOR(Op, DAG);
   case ISD::EXTRACT_SUBVECTOR:
     return lowerEXTRACT_SUBVECTOR(Op, DAG);
+  case ISD::STEP_VECTOR:
+    return lowerSTEP_VECTOR(Op, DAG);
   case ISD::VECTOR_REVERSE:
     return lowerVECTOR_REVERSE(Op, DAG);
   case ISD::BUILD_VECTOR:
@@ -1895,9 +2066,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     // better than going through the stack, as the default expansion does.
     SDLoc DL(Op);
     MVT VT = Op.getSimpleValueType();
-    assert(VT.isFixedLengthVector() && "Unexpected CONCAT_VECTORS lowering");
     unsigned NumOpElts =
-        Op.getOperand(0).getSimpleValueType().getVectorNumElements();
+        Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
     SDValue Vec = DAG.getUNDEF(VT);
     for (const auto &OpIdx : enumerate(Op->ops()))
       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, OpIdx.value(),
@@ -1908,6 +2078,10 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return lowerFixedLengthVectorLoadToRVV(Op, DAG);
   case ISD::STORE:
     return lowerFixedLengthVectorStoreToRVV(Op, DAG);
+  case ISD::MLOAD:
+    return lowerMLOAD(Op, DAG);
+  case ISD::MSTORE:
+    return lowerMSTORE(Op, DAG);
   case ISD::SETCC:
     return lowerFixedLengthVectorSetccToRVV(Op, DAG);
   case ISD::ADD:
@@ -1973,6 +2147,10 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return lowerFixedLengthVectorSelectToRVV(Op, DAG);
   case ISD::FCOPYSIGN:
     return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
+  case ISD::MGATHER:
+    return lowerMGATHER(Op, DAG);
+  case ISD::MSCATTER:
+    return lowerMSCATTER(Op, DAG);
   }
 }
 
@@ -3789,8 +3967,12 @@ SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
   SDValue ValInVec;
 
   if (IsLegalInsert) {
-    if (isNullConstant(Idx))
-      return DAG.getNode(RISCVISD::VMV_S_XF_VL, DL, ContainerVT, Vec, Val, VL);
+    if (isNullConstant(Idx)) {
+      Vec = DAG.getNode(RISCVISD::VMV_S_XF_VL, DL, ContainerVT, Vec, Val, VL);
+      if (!VecVT.isFixedLengthVector())
+        return Vec;
+      return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
+    }
     ValInVec = DAG.getNode(RISCVISD::VMV_S_XF_VL, DL, ContainerVT,
                            DAG.getUNDEF(ContainerVT), Val, VL);
   } else {
@@ -4553,9 +4735,9 @@ SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
     SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
     SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
                                   SubVec, SlideupAmt, Mask, VL);
-    if (!VecVT.isFixedLengthVector())
-      return Slideup;
-    return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
+    if (VecVT.isFixedLengthVector())
+      Slideup = convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
+    return DAG.getBitcast(Op.getValueType(), Slideup);
   }
 
   unsigned SubRegIdx, RemIdx;
@@ -4702,8 +4884,9 @@ SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
                     DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
     // Now we can use a cast-like subvector extract to get the result.
-    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
-                       DAG.getConstant(0, DL, XLenVT));
+    Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
+                            DAG.getConstant(0, DL, XLenVT));
+    return DAG.getBitcast(Op.getValueType(), Slidedown);
   }
 
   unsigned SubRegIdx, RemIdx;
@@ -4752,6 +4935,17 @@ SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
   return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
 }
 
+// Implement step_vector to the vid instruction.
+SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
+                                              SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  assert(Op.getConstantOperandAPInt(0) == 1 && "Unexpected step value");
+  MVT VT = Op.getSimpleValueType();
+  SDValue Mask, VL;
+  std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
+  return DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
+}
+
 // Implement vector_reverse using vrgather.vv with indices determined by
 // subtracting the id of each element from (VLMAX-1). This will convert
 // the indices like so:
@@ -4875,6 +5069,71 @@ RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
       Store->getMemoryVT(), Store->getMemOperand());
 }
 
+SDValue RISCVTargetLowering::lowerMLOAD(SDValue Op, SelectionDAG &DAG) const {
+  auto *Load = cast<MaskedLoadSDNode>(Op);
+
+  SDLoc DL(Op);
+  MVT VT = Op.getSimpleValueType();
+  MVT XLenVT = Subtarget.getXLenVT();
+
+  SDValue Mask = Load->getMask();
+  SDValue PassThru = Load->getPassThru();
+  SDValue VL;
+
+  MVT ContainerVT = VT;
+  if (VT.isFixedLengthVector()) {
+    ContainerVT = getContainerForFixedLengthVector(VT);
+    MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
+
+    Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
+    PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
+    VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
+  } else
+    VL = DAG.getRegister(RISCV::X0, XLenVT);
+
+  SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
+  SDValue IntID = DAG.getTargetConstant(Intrinsic::riscv_vle_mask, DL, XLenVT);
+  SDValue Ops[] = {Load->getChain(),   IntID, PassThru,
+                   Load->getBasePtr(), Mask,  VL};
+  SDValue Result =
+      DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
+                              Load->getMemoryVT(), Load->getMemOperand());
+  SDValue Chain = Result.getValue(1);
+
+  if (VT.isFixedLengthVector())
+    Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
+
+  return DAG.getMergeValues({Result, Chain}, DL);
+}
+
+SDValue RISCVTargetLowering::lowerMSTORE(SDValue Op, SelectionDAG &DAG) const {
+  auto *Store = cast<MaskedStoreSDNode>(Op);
+
+  SDLoc DL(Op);
+  SDValue Val = Store->getValue();
+  SDValue Mask = Store->getMask();
+  MVT VT = Val.getSimpleValueType();
+  MVT XLenVT = Subtarget.getXLenVT();
+  SDValue VL;
+
+  MVT ContainerVT = VT;
+  if (VT.isFixedLengthVector()) {
+    ContainerVT = getContainerForFixedLengthVector(VT);
+    MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
+
+    Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
+    Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
+    VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
+  } else
+    VL = DAG.getRegister(RISCV::X0, XLenVT);
+
+  SDValue IntID = DAG.getTargetConstant(Intrinsic::riscv_vse_mask, DL, XLenVT);
+  return DAG.getMemIntrinsicNode(
+      ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other),
+      {Store->getChain(), IntID, Val, Store->getBasePtr(), Mask, VL},
+      Store->getMemoryVT(), Store->getMemOperand());
+}
+
 SDValue
 RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op,
                                                       SelectionDAG &DAG) const {
@@ -5097,6 +5356,157 @@ SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
   return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
 }
 
+// Custom lower MGATHER to a legalized form for RVV. It will then be matched to
+// a RVV indexed load. The RVV indexed load instructions only support the
+// "unsigned unscaled" addressing mode; indices are implicitly zero-extended or
+// truncated to XLEN and are treated as byte offsets. Any signed or scaled
+// indexing is extended to the XLEN value type and scaled accordingly.
+SDValue RISCVTargetLowering::lowerMGATHER(SDValue Op, SelectionDAG &DAG) const {
+  auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
+  SDLoc DL(Op);
+
+  SDValue Index = MGN->getIndex();
+  SDValue Mask = MGN->getMask();
+  SDValue PassThru = MGN->getPassThru();
+
+  MVT VT = Op.getSimpleValueType();
+  MVT IndexVT = Index.getSimpleValueType();
+  MVT XLenVT = Subtarget.getXLenVT();
+
+  assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
+         "Unexpected VTs!");
+  assert(MGN->getBasePtr().getSimpleValueType() == XLenVT &&
+         "Unexpected pointer type");
+  // Targets have to explicitly opt-in for extending vector loads.
+  assert(MGN->getExtensionType() == ISD::NON_EXTLOAD &&
+         "Unexpected extending MGATHER");
+
+  // If the mask is known to be all ones, optimize to an unmasked intrinsic;
+  // the selection of the masked intrinsics doesn't do this for us.
+  bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
+
+  SDValue VL;
+  MVT ContainerVT = VT;
+  if (VT.isFixedLengthVector()) {
+    // We need to use the larger of the result and index type to determine the
+    // scalable type to use so we don't increase LMUL for any operand/result.
+    if (VT.bitsGE(IndexVT)) {
+      ContainerVT = getContainerForFixedLengthVector(VT);
+      IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
+                                 ContainerVT.getVectorElementCount());
+    } else {
+      IndexVT = getContainerForFixedLengthVector(IndexVT);
+      ContainerVT = MVT::getVectorVT(ContainerVT.getVectorElementType(),
+                                     IndexVT.getVectorElementCount());
+    }
+
+    Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
+
+    if (!IsUnmasked) {
+      MVT MaskVT =
+          MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
+      Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
+      PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
+    }
+
+    VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
+  } else
+    VL = DAG.getRegister(RISCV::X0, XLenVT);
+
+  unsigned IntID =
+      IsUnmasked ? Intrinsic::riscv_vloxei : Intrinsic::riscv_vloxei_mask;
+  SmallVector<SDValue, 8> Ops{MGN->getChain(),
+                              DAG.getTargetConstant(IntID, DL, XLenVT)};
+  if (!IsUnmasked)
+    Ops.push_back(PassThru);
+  Ops.push_back(MGN->getBasePtr());
+  Ops.push_back(Index);
+  if (!IsUnmasked)
+    Ops.push_back(Mask);
+  Ops.push_back(VL);
+
+  SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
+  SDValue Result =
+      DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
+                              MGN->getMemoryVT(), MGN->getMemOperand());
+  SDValue Chain = Result.getValue(1);
+
+  if (VT.isFixedLengthVector())
+    Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
+
+  return DAG.getMergeValues({Result, Chain}, DL);
+}
+
+// Custom lower MSCATTER to a legalized form for RVV. It will then be matched to
+// a RVV indexed store. The RVV indexed store instructions only support the
+// "unsigned unscaled" addressing mode; indices are implicitly zero-extended or
+// truncated to XLEN and are treated as byte offsets. Any signed or scaled
+// indexing is extended to the XLEN value type and scaled accordingly.
+SDValue RISCVTargetLowering::lowerMSCATTER(SDValue Op,
+                                           SelectionDAG &DAG) const {
+  auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
+  SDLoc DL(Op);
+  SDValue Index = MSN->getIndex();
+  SDValue Mask = MSN->getMask();
+  SDValue Val = MSN->getValue();
+
+  MVT VT = Val.getSimpleValueType();
+  MVT IndexVT = Index.getSimpleValueType();
+  MVT XLenVT = Subtarget.getXLenVT();
+
+  assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
+         "Unexpected VTs!");
+  assert(MSN->getBasePtr().getSimpleValueType() == XLenVT &&
+         "Unexpected pointer type");
+  // Targets have to explicitly opt-in for extending vector loads and
+  // truncating vector stores.
+  assert(!MSN->isTruncatingStore() && "Unexpected extending MSCATTER");
+
+  // If the mask is known to be all ones, optimize to an unmasked intrinsic;
+  // the selection of the masked intrinsics doesn't do this for us.
+  bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
+
+  SDValue VL;
+  if (VT.isFixedLengthVector()) {
+    // We need to use the larger of the value and index type to determine the
+    // scalable type to use so we don't increase LMUL for any operand/result.
+    if (VT.bitsGE(IndexVT)) {
+      VT = getContainerForFixedLengthVector(VT);
+      IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
+                                 VT.getVectorElementCount());
+    } else {
+      IndexVT = getContainerForFixedLengthVector(IndexVT);
+      VT = MVT::getVectorVT(VT.getVectorElementType(),
+                            IndexVT.getVectorElementCount());
+    }
+
+    Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
+    Val = convertToScalableVector(VT, Val, DAG, Subtarget);
+
+    if (!IsUnmasked) {
+      MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
+      Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
+    }
+
+    VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
+  } else
+    VL = DAG.getRegister(RISCV::X0, XLenVT);
+
+  unsigned IntID =
+      IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
+  SmallVector<SDValue, 8> Ops{MSN->getChain(),
+                              DAG.getTargetConstant(IntID, DL, XLenVT)};
+  Ops.push_back(Val);
+  Ops.push_back(MSN->getBasePtr());
+  Ops.push_back(Index);
+  if (!IsUnmasked)
+    Ops.push_back(Mask);
+  Ops.push_back(VL);
+
+  return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, MSN->getVTList(), Ops,
+                                 MSN->getMemoryVT(), MSN->getMemOperand());
+}
+
 // Returns the opcode of the target-specific SDNode that implements the 32-bit
 // form of the given Opcode.
 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
@@ -5258,7 +5668,6 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
            "Unexpected custom legalisation");
     bool IsAdd = N->getOpcode() == ISD::UADDO;
-    SDLoc DL(N);
     // Create an ADDW or SUBW.
     SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
     SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
@@ -5278,6 +5687,28 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
     Results.push_back(Overflow);
     return;
   }
+  case ISD::UADDSAT:
+  case ISD::USUBSAT: {
+    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
+           "Unexpected custom legalisation");
+    if (Subtarget.hasStdExtZbb()) {
+      // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using
+      // sign extend allows overflow of the lower 32 bits to be detected on
+      // the promoted size.
+      SDValue LHS =
+          DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
+      SDValue RHS =
+          DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
+      SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
+      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
+      return;
+    }
+
+    // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom
+    // promotion for UADDO/USUBO.
+    Results.push_back(expandAddSubSat(N, DAG));
+    return;
+  }
   case ISD::BITCAST: {
     assert(((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
              Subtarget.hasStdExtF()) ||
@@ -5307,7 +5738,6 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
     // This is similar to customLegalizeToWOp, except that we pass the second
     // operand (a TargetConstant) straight through: it is already of type
     // XLenVT.
-    SDLoc DL(N);
     RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
     SDValue NewOp0 =
         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
@@ -5322,7 +5752,6 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
     // There is no SHFLIW instruction, but we can just promote the operation.
     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
            "Unexpected custom legalisation");
-    SDLoc DL(N);
     SDValue NewOp0 =
         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
     SDValue NewRes =
@@ -5412,7 +5841,6 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
     // transferred to the destination register. We issue two of these from the
     // upper- and lower- halves of the SEW-bit vector element, slid down to the
     // first element.
-    SDLoc DL(N);
     SDValue Vec = N->getOperand(0);
     SDValue Idx = N->getOperand(1);
 
@@ -5535,21 +5963,6 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
 
       break;
     }
-    case Intrinsic::experimental_vector_stepvector: {
-      EVT Ty = N->getValueType(0);
-      switch (Ty.getSimpleVT().SimpleTy) {
-      default:
-        llvm_unreachable("Unexpected result type to legalize");
-      case MVT::nxv1i32:
-        SDValue Promoted =
-            DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::nxv1i64,
-                        DAG.getConstant(IntNo, DL, MVT::i64));
-        SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, Ty, Promoted);
-        Results.push_back(Trunc);
-        break;
-      }
-      break;
-    }
 #define VP_INTRINSIC(X) case Intrinsic::X:
       VP_INTRINSIC_WO_CHAIN_SET
 #undef VP_INTRINSIC
@@ -6190,6 +6603,58 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
                        DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
   }
+  case ISD::MGATHER:
+  case ISD::MSCATTER: {
+    if (!DCI.isBeforeLegalize())
+      break;
+    MaskedGatherScatterSDNode *MGSN = cast<MaskedGatherScatterSDNode>(N);
+    SDValue Index = MGSN->getIndex();
+    EVT IndexVT = Index.getValueType();
+    MVT XLenVT = Subtarget.getXLenVT();
+    // RISCV indexed loads only support the "unsigned unscaled" addressing
+    // mode, so anything else must be manually legalized.
+    bool NeedsIdxLegalization = MGSN->isIndexScaled() ||
+                                (MGSN->isIndexSigned() &&
+                                 IndexVT.getVectorElementType().bitsLT(XLenVT));
+    if (!NeedsIdxLegalization)
+      break;
+
+    SDLoc DL(N);
+
+    // Any index legalization should first promote to XLenVT, so we don't lose
+    // bits when scaling. This may create an illegal index type so we let
+    // LLVM's legalization take care of the splitting.
+    if (IndexVT.getVectorElementType().bitsLT(XLenVT)) {
+      IndexVT = IndexVT.changeVectorElementType(XLenVT);
+      Index = DAG.getNode(MGSN->isIndexSigned() ? ISD::SIGN_EXTEND
+                                                : ISD::ZERO_EXTEND,
+                          DL, IndexVT, Index);
+    }
+
+    unsigned Scale = N->getConstantOperandVal(5);
+    if (MGSN->isIndexScaled() && Scale != 1) {
+      // Manually scale the indices by the element size.
+      // TODO: Sanitize the scale operand here?
+      assert(isPowerOf2_32(Scale) && "Expecting power-of-two types");
+      SDValue SplatScale = DAG.getConstant(Log2_32(Scale), DL, IndexVT);
+      Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, SplatScale);
+    }
+
+    ISD::MemIndexType NewIndexTy = ISD::UNSIGNED_UNSCALED;
+    if (const auto *MGN = dyn_cast<MaskedGatherSDNode>(N)) {
+      return DAG.getMaskedGather(
+          N->getVTList(), MGSN->getMemoryVT(), DL,
+          {MGSN->getChain(), MGN->getPassThru(), MGSN->getMask(),
+           MGSN->getBasePtr(), Index, MGN->getScale()},
+          MGN->getMemOperand(), NewIndexTy, MGN->getExtensionType());
+    }
+    const auto *MSN = cast<MaskedScatterSDNode>(N);
+    return DAG.getMaskedScatter(
+        N->getVTList(), MGSN->getMemoryVT(), DL,
+        {MGSN->getChain(), MSN->getValue(), MGSN->getMask(), MGSN->getBasePtr(),
+         Index, MGSN->getScale()},
+        MGSN->getMemOperand(), NewIndexTy, MSN->isTruncatingStore());
+  }
   }
 
   return SDValue();
@@ -6265,16 +6730,36 @@ bool RISCVTargetLowering::targetShrinkDemandedConstant(
   // Clear all non-demanded bits initially.
   APInt ShrunkMask = Mask & DemandedBits;
 
+  // Try to make a smaller immediate by setting undemanded bits.
+
+  APInt ExpandedMask = Mask | ~DemandedBits;
+
+  auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
+    return ShrunkMask.isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
+  };
+  auto UseMask = [Mask, Op, VT, &TLO](const APInt &NewMask) -> bool {
+    if (NewMask == Mask)
+      return true;
+    SDLoc DL(Op);
+    SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
+    SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
+    return TLO.CombineTo(Op, NewOp);
+  };
+
   // If the shrunk mask fits in sign extended 12 bits, let the target
   // independent code apply it.
   if (ShrunkMask.isSignedIntN(12))
     return false;
 
-  // Try to make a smaller immediate by setting undemanded bits.
+  // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
+  if (VT == MVT::i64) {
+    APInt NewMask = APInt(64, 0xffffffff);
+    if (IsLegalMask(NewMask))
+      return UseMask(NewMask);
+  }
 
-  // We need to be able to make a negative number through a combination of mask
-  // and undemanded bits.
-  APInt ExpandedMask = Mask | ~DemandedBits;
+  // For the remaining optimizations, we need to be able to make a negative
+  // number through a combination of mask and undemanded bits.
   if (!ExpandedMask.isNegative())
     return false;
 
@@ -6292,18 +6777,8 @@ bool RISCVTargetLowering::targetShrinkDemandedConstant(
     return false;
 
   // Sanity check that our new mask is a subset of the demanded mask.
-  assert(NewMask.isSubsetOf(ExpandedMask));
-
-  // If we aren't changing the mask, just return true to keep it and prevent
-  // the caller from optimizing.
-  if (NewMask == Mask)
-    return true;
-
-  // Replace the constant with the new mask.
-  SDLoc DL(Op);
-  SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
-  SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
-  return TLO.CombineTo(Op, NewOp);
+  assert(IsLegalMask(NewMask));
+  return UseMask(NewMask);
 }
 
 void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
@@ -6673,6 +7148,19 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
   return TailMBB;
 }
 
+static MachineInstr *elideCopies(MachineInstr *MI,
+                                 const MachineRegisterInfo &MRI) {
+  while (true) {
+    if (!MI->isFullCopy())
+      return MI;
+    if (!Register::isVirtualRegister(MI->getOperand(1).getReg()))
+      return nullptr;
+    MI = MRI.getVRegDef(MI->getOperand(1).getReg());
+    if (!MI)
+      return nullptr;
+  }
+}
+
 static MachineBasicBlock *addVSetVL(MachineInstr &MI, MachineBasicBlock *BB,
                                     int VLIndex, unsigned SEWIndex,
                                     RISCVVLMUL VLMul, bool ForceTailAgnostic) {
@@ -6734,8 +7222,11 @@ static MachineBasicBlock *addVSetVL(MachineInstr &MI, MachineBasicBlock *BB,
     // If the tied operand is an IMPLICIT_DEF we can keep TailAgnostic.
     const MachineOperand &UseMO = MI.getOperand(UseOpIdx);
     MachineInstr *UseMI = MRI.getVRegDef(UseMO.getReg());
-    if (UseMI && UseMI->isImplicitDef())
-      TailAgnostic = true;
+    if (UseMI) {
+      UseMI = elideCopies(UseMI, MRI);
+      if (UseMI && UseMI->isImplicitDef())
+        TailAgnostic = true;
+    }
   }
 
   // For simplicity we reuse the vtype representation here.
@@ -8922,6 +9413,10 @@ Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
   return Result;
 }
 
+bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
+  return false;
+}
+
 bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
                                                      EVT VT) const {
   VT = VT.getScalarType();
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 7fdfffca13cb6277ae167d3fcd02fced1b30196e..6d187166861f0fddd68cced8b18b88beb6685865 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -490,6 +490,8 @@ public:
   static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
                                               const RISCVSubtarget &Subtarget);
 
+  bool shouldRemoveExtendFromGSIndex(EVT VT) const override;
+
 private:
   void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo,
                         const SmallVectorImpl<ISD::InputArg> &Ins,
@@ -532,10 +534,6 @@ private:
                                  int Opcode) const;
   SDValue lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
-  SDValue lowerMLOAD(SDValue Op, SelectionDAG &DAG) const;
-  SDValue lowerMSTORE(SDValue Op, SelectionDAG &DAG) const;
-  SDValue lowerMGATHER(SDValue Op, SelectionDAG &DAG) const;
-  SDValue lowerMSCATTER(SDValue Op, SelectionDAG &DAG) const;
   using VTToLibCall = std::pair<EVT, RTLIB::Libcall>;
   SDValue lowerVECLIBCALL(SDValue Op, SelectionDAG &DAG,
                           ArrayRef<VTToLibCall> TypeToCall) const;
@@ -546,10 +544,15 @@ private:
   SDValue lowerFPVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerSTEP_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerVECTOR_REVERSE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerABS(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerMLOAD(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerMSTORE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFixedLengthVectorFCOPYSIGNToRVV(SDValue Op,
                                                SelectionDAG &DAG) const;
+  SDValue lowerMGATHER(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerMSCATTER(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFixedLengthVectorLoadToRVV(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFixedLengthVectorStoreToRVV(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFixedLengthVectorSetccToRVV(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index f74eb5ddd33943688018d7aad001a7b7ef59cf32..cabe3a33d88858f8a48f252762223bce90588fe2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -23,6 +23,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/MC/MCInstBuilder.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/IR/IntrinsicsEPI.h"
@@ -68,6 +69,15 @@ RISCVInstrInfo::RISCVInstrInfo(RISCVSubtarget &STI)
     : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP),
       STI(STI) {}
 
+MCInst RISCVInstrInfo::getNop() const {
+  if (STI.getFeatureBits()[RISCV::FeatureStdExtC])
+    return MCInstBuilder(RISCV::C_NOP);
+  return MCInstBuilder(RISCV::ADDI)
+      .addReg(RISCV::X0)
+      .addReg(RISCV::X0)
+      .addImm(0);
+}
+
 unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
                                              int &FrameIndex) const {
   switch (MI.getOpcode()) {
@@ -77,14 +87,17 @@ unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
   case RISCV::PseudoVRELOAD_M2:
   case RISCV::PseudoVRELOAD_M4:
   case RISCV::PseudoVRELOAD_M8:
-
-  case RISCV::PseudoEPIVRELOAD_VRN2M1:
-  case RISCV::PseudoEPIVRELOAD_VRN3M1:
-  case RISCV::PseudoEPIVRELOAD_VRN4M1:
-  case RISCV::PseudoEPIVRELOAD_VRN5M1:
-  case RISCV::PseudoEPIVRELOAD_VRN6M1:
-  case RISCV::PseudoEPIVRELOAD_VRN7M1:
-  case RISCV::PseudoEPIVRELOAD_VRN8M1:
+  case RISCV::PseudoVRELOAD2_M1:
+  case RISCV::PseudoVRELOAD2_M2:
+  case RISCV::PseudoVRELOAD2_M4:
+  case RISCV::PseudoVRELOAD3_M1:
+  case RISCV::PseudoVRELOAD3_M2:
+  case RISCV::PseudoVRELOAD4_M1:
+  case RISCV::PseudoVRELOAD4_M2:
+  case RISCV::PseudoVRELOAD5_M1:
+  case RISCV::PseudoVRELOAD6_M1:
+  case RISCV::PseudoVRELOAD7_M1:
+  case RISCV::PseudoVRELOAD8_M1:
     assert(MI.getOperand(1).isFI());
     FrameIndex = MI.getOperand(1).getIndex();
     assert(MI.getOperand(0).isReg());
@@ -120,14 +133,17 @@ unsigned RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
   case RISCV::PseudoVSPILL_M2:
   case RISCV::PseudoVSPILL_M4:
   case RISCV::PseudoVSPILL_M8:
-
-  case RISCV::PseudoEPIVSPILL_VRN2M1:
-  case RISCV::PseudoEPIVSPILL_VRN3M1:
-  case RISCV::PseudoEPIVSPILL_VRN4M1:
-  case RISCV::PseudoEPIVSPILL_VRN5M1:
-  case RISCV::PseudoEPIVSPILL_VRN6M1:
-  case RISCV::PseudoEPIVSPILL_VRN7M1:
-  case RISCV::PseudoEPIVSPILL_VRN8M1:
+  case RISCV::PseudoVSPILL2_M1:
+  case RISCV::PseudoVSPILL2_M2:
+  case RISCV::PseudoVSPILL2_M4:
+  case RISCV::PseudoVSPILL3_M1:
+  case RISCV::PseudoVSPILL3_M2:
+  case RISCV::PseudoVSPILL4_M1:
+  case RISCV::PseudoVSPILL4_M2:
+  case RISCV::PseudoVSPILL5_M1:
+  case RISCV::PseudoVSPILL6_M1:
+  case RISCV::PseudoVSPILL7_M1:
+  case RISCV::PseudoVSPILL8_M1:
     assert(MI.getOperand(1).isFI());
     FrameIndex = MI.getOperand(1).getIndex();
     assert(MI.getOperand(0).isReg());
@@ -277,71 +293,56 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   MachineFrameInfo &MFI = MF->getFrameInfo();
 
   unsigned Opcode;
-  bool IsScalableVector = false;
-  if (RISCV::GPRRegClass.hasSubClassEq(RC))
+  bool IsScalableVector = true;
+  bool IsZvlsseg = true;
+  if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
              RISCV::SW : RISCV::SD;
-  else if (RISCV::FPR16RegClass.hasSubClassEq(RC))
+    IsScalableVector = false;
+  } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::FSH;
-  else if (RISCV::FPR32RegClass.hasSubClassEq(RC))
+    IsScalableVector = false;
+  } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::FSW;
-  else if (RISCV::FPR64RegClass.hasSubClassEq(RC))
+    IsScalableVector = false;
+  } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::FSD;
-  else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
+    IsScalableVector = false;
+  } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoVSPILL_M1;
-    IsScalableVector = true;
+    IsZvlsseg = false;
   } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoVSPILL_M2;
-    IsScalableVector = true;
+    IsZvlsseg = false;
   } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoVSPILL_M4;
-    IsScalableVector = true;
+    IsZvlsseg = false;
   } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoVSPILL_M8;
-    IsScalableVector = true;
-  } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_VRN2M1))
-        .addReg(SrcReg, getKillRegState(IsKill))
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_VRN3M1))
-        .addReg(SrcReg, getKillRegState(IsKill))
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_VRN4M1))
-        .addReg(SrcReg, getKillRegState(IsKill))
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_VRN5M1))
-        .addReg(SrcReg, getKillRegState(IsKill))
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_VRN6M1))
-        .addReg(SrcReg, getKillRegState(IsKill))
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_VRN7M1))
-        .addReg(SrcReg, getKillRegState(IsKill))
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_VRN8M1))
-        .addReg(SrcReg, getKillRegState(IsKill))
-        .addFrameIndex(FI);
-    return;
-  } else
+    IsZvlsseg = false;
+  } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL2_M1;
+  else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL2_M2;
+  else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL2_M4;
+  else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL3_M1;
+  else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL3_M2;
+  else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL4_M1;
+  else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL4_M2;
+  else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL5_M1;
+  else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL6_M1;
+  else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL7_M1;
+  else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVSPILL8_M1;
+  else
     llvm_unreachable("Can't store this register to stack slot");
 
   if (IsScalableVector) {
@@ -350,10 +351,16 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
         MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
 
     MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(Opcode))
-        .addReg(SrcReg, getKillRegState(IsKill))
-        .addFrameIndex(FI)
-        .addMemOperand(MMO);
+    auto MIB = BuildMI(MBB, I, DL, get(Opcode))
+                   .addReg(SrcReg, getKillRegState(IsKill))
+                   .addFrameIndex(FI)
+                   .addMemOperand(MMO);
+    if (IsZvlsseg) {
+      // For spilling/reloading Zvlsseg registers, append the dummy field for
+      // the scaled vector length. The argument will be used when expanding
+      // these pseudo instructions.
+      MIB.addReg(RISCV::X0);
+    }
   } else {
     MachineMemOperand *MMO = MF->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
@@ -380,64 +387,56 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   MachineFrameInfo &MFI = MF->getFrameInfo();
 
   unsigned Opcode;
-  bool IsScalableVector = false;
-  if (RISCV::GPRRegClass.hasSubClassEq(RC))
+  bool IsScalableVector = true;
+  bool IsZvlsseg = true;
+  if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
              RISCV::LW : RISCV::LD;
-  else if (RISCV::FPR16RegClass.hasSubClassEq(RC))
+    IsScalableVector = false;
+  } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::FLH;
-  else if (RISCV::FPR32RegClass.hasSubClassEq(RC))
+    IsScalableVector = false;
+  } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::FLW;
-  else if (RISCV::FPR64RegClass.hasSubClassEq(RC))
+    IsScalableVector = false;
+  } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::FLD;
-  else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
+    IsScalableVector = false;
+  } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoVRELOAD_M1;
-    IsScalableVector = true;
+    IsZvlsseg = false;
   } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoVRELOAD_M2;
-    IsScalableVector = true;
+    IsZvlsseg = false;
   } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoVRELOAD_M4;
-    IsScalableVector = true;
+    IsZvlsseg = false;
   } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoVRELOAD_M8;
-    IsScalableVector = true;
-  } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_VRN2M1), DstReg)
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_VRN3M1), DstReg)
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_VRN4M1), DstReg)
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_VRN5M1), DstReg)
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_VRN6M1), DstReg)
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_VRN7M1), DstReg)
-        .addFrameIndex(FI);
-    return;
-  } else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC)) {
-    MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_VRN8M1), DstReg)
-        .addFrameIndex(FI);
-    return;
-  } else
+    IsZvlsseg = false;
+  } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD2_M1;
+  else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD2_M2;
+  else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD2_M4;
+  else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD3_M1;
+  else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD3_M2;
+  else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD4_M1;
+  else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD4_M2;
+  else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD5_M1;
+  else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD6_M1;
+  else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD7_M1;
+  else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::PseudoVRELOAD8_M1;
+  else
     llvm_unreachable("Can't load this register from stack slot");
 
   if (IsScalableVector) {
@@ -446,9 +445,15 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
         MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
 
     MFI.setStackID(FI, TargetStackID::ScalableVector);
-    BuildMI(MBB, I, DL, get(Opcode), DstReg)
-        .addFrameIndex(FI)
-        .addMemOperand(MMO);
+    auto MIB = BuildMI(MBB, I, DL, get(Opcode), DstReg)
+                   .addFrameIndex(FI)
+                   .addMemOperand(MMO);
+    if (IsZvlsseg) {
+      // For spilling/reloading Zvlsseg registers, append the dummy field for
+      // the scaled vector length. The argument will be used when expanding
+      // these pseudo instructions.
+      MIB.addReg(RISCV::X0);
+    }
   } else {
     MachineMemOperand *MMO = MF->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
@@ -1404,3 +1409,44 @@ Register RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF,
 
   return FactorRegister;
 }
+
+Optional<std::pair<unsigned, unsigned>>
+RISCVInstrInfo::isRVVSpillForZvlsseg(unsigned Opcode) const {
+  switch (Opcode) {
+  default:
+    return None;
+  case RISCV::PseudoVSPILL2_M1:
+  case RISCV::PseudoVRELOAD2_M1:
+    return std::make_pair(2u, 1u);
+  case RISCV::PseudoVSPILL2_M2:
+  case RISCV::PseudoVRELOAD2_M2:
+    return std::make_pair(2u, 2u);
+  case RISCV::PseudoVSPILL2_M4:
+  case RISCV::PseudoVRELOAD2_M4:
+    return std::make_pair(2u, 4u);
+  case RISCV::PseudoVSPILL3_M1:
+  case RISCV::PseudoVRELOAD3_M1:
+    return std::make_pair(3u, 1u);
+  case RISCV::PseudoVSPILL3_M2:
+  case RISCV::PseudoVRELOAD3_M2:
+    return std::make_pair(3u, 2u);
+  case RISCV::PseudoVSPILL4_M1:
+  case RISCV::PseudoVRELOAD4_M1:
+    return std::make_pair(4u, 1u);
+  case RISCV::PseudoVSPILL4_M2:
+  case RISCV::PseudoVRELOAD4_M2:
+    return std::make_pair(4u, 2u);
+  case RISCV::PseudoVSPILL5_M1:
+  case RISCV::PseudoVRELOAD5_M1:
+    return std::make_pair(5u, 1u);
+  case RISCV::PseudoVSPILL6_M1:
+  case RISCV::PseudoVRELOAD6_M1:
+    return std::make_pair(6u, 1u);
+  case RISCV::PseudoVSPILL7_M1:
+  case RISCV::PseudoVRELOAD7_M1:
+    return std::make_pair(7u, 1u);
+  case RISCV::PseudoVSPILL8_M1:
+  case RISCV::PseudoVRELOAD8_M1:
+    return std::make_pair(8u, 1u);
+  }
+}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 9657834e23e53ee0823bfe79b4b1b789536caa72..c775b639fb85c92edf7bce52226c3311a1e8c65a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -28,6 +28,8 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
 public:
   explicit RISCVInstrInfo(RISCVSubtarget &STI);
 
+  MCInst getNop() const override;
+
   unsigned isLoadFromStackSlot(const MachineInstr &MI,
                                int &FrameIndex) const override;
   unsigned isStoreToStackSlot(const MachineInstr &MI,
@@ -144,6 +146,9 @@ public:
                                  MachineBasicBlock::iterator II,
                                  int64_t Amount) const;
 
+  Optional<std::pair<unsigned, unsigned>>
+  isRVVSpillForZvlsseg(unsigned Opcode) const;
+
 protected:
   const RISCVSubtarget &STI;
 };
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 90c8102b5bbe911e245d168c497c180aaeea01d0..7e2f7a57fb6b0f6f2359723d9c4fd2ec6812929d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -309,39 +309,6 @@ def uimm6gt32 : ImmLeaf<XLenVT, [{
 def AddrFI : ComplexPattern<iPTR, 1, "SelectAddrFI", [frameindex], []>;
 def BaseAddr : ComplexPattern<iPTR, 1, "SelectBaseAddr">;
 
-// Extract least significant 12 bits from an immediate value and sign extend
-// them.
-def LO12Sext : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(SignExtend64<12>(N->getZExtValue()),
-                                   SDLoc(N), N->getValueType(0));
-}]>;
-
-// Extract the most significant 20 bits from an immediate value. Add 1 if bit
-// 11 is 1, to compensate for the low 12 bits in the matching immediate addi
-// or ld/st being negative.
-def HI20 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(((N->getZExtValue()+0x800) >> 12) & 0xfffff,
-                                   SDLoc(N), N->getValueType(0));
-}]>;
-
-// Extract least significant 12 bits from the upper half of an immediate value
-// and sign extend them.
-def HLO12Sext : SDNodeXForm<imm, [{
-  uint64_t Val = N->getZExtValue() >> 32;
-  return CurDAG->getTargetConstant(SignExtend64<12>(Val),
-                                   SDLoc(N), N->getValueType(0));
-}]>;
-
-// Extract the most significant 20 bits from the upper half of an immediate
-// value. Add 1 if bit 32+11 is 1, to compensate for the low 12 bits in the
-// matching immediate addi or ld/st being negative.
-def HHI20 : SDNodeXForm<imm, [{
-  uint64_t Val = N->getZExtValue() >> 32;
-  return CurDAG->getTargetConstant(((Val+0x800) >> 12) & 0xfffff,
-                                   SDLoc(N), N->getValueType(0));
-}]>;
-
-
 // Return the negation of an immediate value.
 def NegImm : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(-N->getSExtValue(), SDLoc(N),
@@ -354,6 +321,12 @@ def ImmSub32 : SDNodeXForm<imm, [{
                                    N->getValueType(0));
 }]>;
 
+// Return an immediate value plus 32.
+def ImmPlus32 : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue() + 32, SDLoc(N),
+                                   N->getValueType(0));
+}]>;
+
 // Return an immediate subtracted from XLen.
 def ImmSubFromXLen : SDNodeXForm<imm, [{
   uint64_t XLen = Subtarget->getXLen();
@@ -412,7 +385,7 @@ class Shift_ri<bit arithshift, bits<3> funct3, string opcodestr>
     : RVInstIShift<arithshift, funct3, OPC_OP_IMM, (outs GPR:$rd),
                    (ins GPR:$rs1, uimmlog2xlen:$shamt), opcodestr,
                    "$rd, $rs1, $shamt">,
-      Sched<[WriteShift, ReadShift]>;
+      Sched<[WriteShiftImm, ReadShiftImm]>;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class ALU_rr<bits<7> funct7, bits<3> funct3, string opcodestr>
@@ -437,7 +410,7 @@ class ShiftW_ri<bit arithshift, bits<3> funct3, string opcodestr>
     : RVInstIShiftW<arithshift, funct3, OPC_OP_IMM_32, (outs GPR:$rd),
                     (ins GPR:$rs1, uimm5:$shamt), opcodestr,
                     "$rd, $rs1, $shamt">,
-      Sched<[WriteShift32, ReadShift32]>;
+      Sched<[WriteShiftImm32, ReadShiftImm32]>;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class ALUW_rr<bits<7> funct7, bits<3> funct3, string opcodestr>
@@ -510,12 +483,12 @@ def SRAI : Shift_ri<1, 0b101, "srai">;
 
 def ADD  : ALU_rr<0b0000000, 0b000, "add">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 def SUB  : ALU_rr<0b0100000, 0b000, "sub">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
-def SLL  : ALU_rr<0b0000000, 0b001, "sll">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
+def SLL  : ALU_rr<0b0000000, 0b001, "sll">, Sched<[WriteShiftReg, ReadShiftReg, ReadShiftReg]>;
 def SLT  : ALU_rr<0b0000000, 0b010, "slt">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 def SLTU : ALU_rr<0b0000000, 0b011, "sltu">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 def XOR  : ALU_rr<0b0000000, 0b100, "xor">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
-def SRL  : ALU_rr<0b0000000, 0b101, "srl">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
-def SRA  : ALU_rr<0b0100000, 0b101, "sra">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
+def SRL  : ALU_rr<0b0000000, 0b101, "srl">, Sched<[WriteShiftReg, ReadShiftReg, ReadShiftReg]>;
+def SRA  : ALU_rr<0b0100000, 0b101, "sra">, Sched<[WriteShiftReg, ReadShiftReg, ReadShiftReg]>;
 def OR   : ALU_rr<0b0000000, 0b110, "or">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 def AND  : ALU_rr<0b0000000, 0b111, "and">, Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 
@@ -597,11 +570,11 @@ def ADDW  : ALUW_rr<0b0000000, 0b000, "addw">,
 def SUBW  : ALUW_rr<0b0100000, 0b000, "subw">,
             Sched<[WriteIALU32, ReadIALU32, ReadIALU32]>;
 def SLLW  : ALUW_rr<0b0000000, 0b001, "sllw">,
-            Sched<[WriteIALU32, ReadIALU32, ReadIALU32]>;
+            Sched<[WriteShiftReg32, ReadShiftReg32, ReadShiftReg32]>;
 def SRLW  : ALUW_rr<0b0000000, 0b101, "srlw">,
-            Sched<[WriteIALU32, ReadIALU32, ReadIALU32]>;
+            Sched<[WriteShiftReg32, ReadShiftReg32, ReadShiftReg32]>;
 def SRAW  : ALUW_rr<0b0100000, 0b101, "sraw">,
-            Sched<[WriteIALU32, ReadIALU32, ReadIALU32]>;
+            Sched<[WriteShiftReg32, ReadShiftReg32, ReadShiftReg32]>;
 } // Predicates = [IsRV64]
 
 //===----------------------------------------------------------------------===//
@@ -881,11 +854,6 @@ def assertzexti32 : PatFrag<(ops node:$src), (assertzext node:$src), [{
 }]>;
 def zexti32 : ComplexPattern<i64, 1, "selectZExti32">;
 
-def SRLIWPat : PatFrag<(ops node:$A, node:$B),
-                       (srl (and node:$A, imm), node:$B), [{
-  return MatchSRLIW(N);
-}]>;
-
 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
 // on RV64). Also used to optimize the same sequence without SLLIUW.
 def SLLIUWPat : PatFrag<(ops node:$A, node:$B),
@@ -893,13 +861,6 @@ def SLLIUWPat : PatFrag<(ops node:$A, node:$B),
   return MatchSLLIUW(N);
 }]>;
 
-/// Immediates
-
-def : Pat<(simm12:$imm), (ADDI X0, simm12:$imm)>;
-def : Pat<(simm32hi20:$imm), (LUI (HI20 imm:$imm))>;
-def : Pat<(i32 (simm32:$imm)), (ADDI (LUI (HI20 imm:$imm)), (LO12Sext imm:$imm))>,
-      Requires<[IsRV32]>;
-
 /// Simple arithmetic operations
 
 def : PatGprGpr<add, ADD>;
@@ -1199,6 +1160,11 @@ def : Pat<(i64 (shl (and GPR:$rs1, 0xffffffff), uimm5:$shamt)),
 // shl/and can appear in the other order too.
 def : Pat<(i64 (SLLIUWPat GPR:$rs1, uimm5:$shamt)),
           (SRLI (SLLI GPR:$rs1, 32), (ImmSubFrom32 uimm5:$shamt))>;
+
+// If we're shifting a value left by 0-31 bits, and then masking to 32-bits,
+// use 2 shifts instead of 3.
+def : Pat<(i64 (and (shl GPR:$rs1, uimm5:$shamt), 0xffffffff)),
+          (SRLI (SLLI GPR:$rs1, (ImmPlus32 uimm5:$shamt)), 32)>;
 }
 
 let Predicates = [IsRV64] in {
@@ -1217,7 +1183,7 @@ def : Pat<(sext_inreg (sub GPR:$rs1, GPR:$rs2), i32),
           (SUBW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(sext_inreg (shl GPR:$rs1, uimm5:$shamt), i32),
           (SLLIW GPR:$rs1, uimm5:$shamt)>;
-def : Pat<(i64 (SRLIWPat GPR:$rs1, uimm5:$shamt)),
+def : Pat<(i64 (srl (and GPR:$rs1, 0xffffffff), uimm5:$shamt)),
           (SRLIW GPR:$rs1, uimm5:$shamt)>;
 def : Pat<(i64 (srl (shl GPR:$rs1, (i64 32)), uimm6gt32:$shamt)),
           (SRLIW GPR:$rs1, (ImmSub32 uimm6gt32:$shamt))>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoB.td b/llvm/lib/Target/RISCV/RISCVInstrInfoB.td
index 1b287409eecdafca39a6dad0fe89e1b801b4d821..abdffecaf8b250aeed4b6f3fb402f8193a2f12e4 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoB.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoB.td
@@ -170,20 +170,28 @@ class RVBTernaryImm5<bits<2> funct2, bits<3> funct3_b, RISCVOpcode opcode,
 //===----------------------------------------------------------------------===//
 
 let Predicates = [HasStdExtZbbOrZbp] in {
-def ANDN  : ALU_rr<0b0100000, 0b111, "andn">, Sched<[]>;
-def ORN   : ALU_rr<0b0100000, 0b110, "orn">, Sched<[]>;
-def XNOR  : ALU_rr<0b0100000, 0b100, "xnor">, Sched<[]>;
+def ANDN  : ALU_rr<0b0100000, 0b111, "andn">,
+            Sched<[WriteIALU, ReadIALU, ReadIALU]>;
+def ORN   : ALU_rr<0b0100000, 0b110, "orn">,
+            Sched<[WriteIALU, ReadIALU, ReadIALU]>;
+def XNOR  : ALU_rr<0b0100000, 0b100, "xnor">,
+            Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 } // Predicates = [HasStdExtZbbOrZbp]
 
 let Predicates = [HasStdExtZba] in {
-def SH1ADD : ALU_rr<0b0010000, 0b010, "sh1add">, Sched<[]>;
-def SH2ADD : ALU_rr<0b0010000, 0b100, "sh2add">, Sched<[]>;
-def SH3ADD : ALU_rr<0b0010000, 0b110, "sh3add">, Sched<[]>;
+def SH1ADD : ALU_rr<0b0010000, 0b010, "sh1add">,
+             Sched<[WriteSHXADD, ReadSHXADD, ReadSHXADD]>;
+def SH2ADD : ALU_rr<0b0010000, 0b100, "sh2add">,
+             Sched<[WriteSHXADD, ReadSHXADD, ReadSHXADD]>;
+def SH3ADD : ALU_rr<0b0010000, 0b110, "sh3add">,
+             Sched<[WriteSHXADD, ReadSHXADD, ReadSHXADD]>;
 } // Predicates = [HasStdExtZba]
 
 let Predicates = [HasStdExtZbbOrZbp] in {
-def ROL   : ALU_rr<0b0110000, 0b001, "rol">, Sched<[]>;
-def ROR   : ALU_rr<0b0110000, 0b101, "ror">, Sched<[]>;
+def ROL   : ALU_rr<0b0110000, 0b001, "rol">,
+            Sched<[WriteRotateReg, ReadRotateReg, ReadRotateReg]>;
+def ROR   : ALU_rr<0b0110000, 0b101, "ror">,
+            Sched<[WriteRotateReg, ReadRotateReg, ReadRotateReg]>;
 } // Predicates = [HasStdExtZbbOrZbp]
 
 let Predicates = [HasStdExtZbs] in {
@@ -205,7 +213,8 @@ def XPERMH : ALU_rr<0b0010100, 0b110, "xperm.h">, Sched<[]>;
 } // Predicates = [HasStdExtZbp]
 
 let Predicates = [HasStdExtZbbOrZbp] in
-def RORI  : RVBShift_ri<0b01100, 0b101, OPC_OP_IMM, "rori">, Sched<[]>;
+def RORI  : RVBShift_ri<0b01100, 0b101, OPC_OP_IMM, "rori">,
+            Sched<[WriteRotateImm, ReadRotateImm]>;
 
 let Predicates = [HasStdExtZbs] in {
 def BCLRI : RVBShift_ri<0b01001, 0b001, OPC_OP_IMM, "bclri">, Sched<[]>;
@@ -234,11 +243,11 @@ def FSRI : RVBTernaryImm6<0b101, OPC_OP_IMM, "fsri",
 
 let Predicates = [HasStdExtZbb] in {
 def CLZ  : RVBUnary<0b0110000, 0b00000, 0b001, RISCVOpcode<0b0010011>, "clz">,
-           Sched<[]>;
+           Sched<[WriteCLZ, ReadCLZ]>;
 def CTZ  : RVBUnary<0b0110000, 0b00001, 0b001, RISCVOpcode<0b0010011>, "ctz">,
-           Sched<[]>;
+           Sched<[WriteCTZ, ReadCTZ]>;
 def CPOP : RVBUnary<0b0110000, 0b00010, 0b001, RISCVOpcode<0b0010011>, "cpop">,
-           Sched<[]>;
+           Sched<[WriteCPOP, ReadCPOP]>;
 } // Predicates = [HasStdExtZbb]
 
 let Predicates = [HasStdExtZbm, IsRV64] in
@@ -247,9 +256,9 @@ def BMATFLIP : RVBUnary<0b0110000, 0b00011, 0b001, RISCVOpcode<0b0010011>,
 
 let Predicates = [HasStdExtZbb] in {
 def SEXTB : RVBUnary<0b0110000, 0b00100, 0b001, RISCVOpcode<0b0010011>,
-                     "sext.b">, Sched<[]>;
+                     "sext.b">, Sched<[WriteIALU, ReadIALU]>;
 def SEXTH : RVBUnary<0b0110000, 0b00101, 0b001, RISCVOpcode<0b0010011>,
-                     "sext.h">, Sched<[]>;
+                     "sext.h">, Sched<[WriteIALU, ReadIALU]>;
 } // Predicates = [HasStdExtZbb]
 
 let Predicates = [HasStdExtZbr] in {
@@ -285,10 +294,14 @@ def CLMULH : ALU_rr<0b0000101, 0b011, "clmulh">, Sched<[]>;
 } // Predicates = [HasStdExtZbc]
 
 let Predicates = [HasStdExtZbb] in {
-def MIN  : ALU_rr<0b0000101, 0b100, "min">, Sched<[]>;
-def MINU : ALU_rr<0b0000101, 0b101, "minu">, Sched<[]>;
-def MAX  : ALU_rr<0b0000101, 0b110, "max">, Sched<[]>;
-def MAXU : ALU_rr<0b0000101, 0b111, "maxu">, Sched<[]>;
+def MIN  : ALU_rr<0b0000101, 0b100, "min">,
+           Sched<[WriteIALU, ReadIALU, ReadIALU]>;
+def MINU : ALU_rr<0b0000101, 0b101, "minu">,
+           Sched<[WriteIALU, ReadIALU, ReadIALU]>;
+def MAX  : ALU_rr<0b0000101, 0b110, "max">,
+           Sched<[WriteIALU, ReadIALU, ReadIALU]>;
+def MAXU : ALU_rr<0b0000101, 0b111, "maxu">,
+           Sched<[WriteIALU, ReadIALU, ReadIALU]>;
 } // Predicates = [HasStdExtZbb]
 
 let Predicates = [HasStdExtZbp] in {
@@ -323,16 +336,23 @@ def UNSHFLI : RVBShfl_ri<0b000010, 0b101, OPC_OP_IMM, "unshfli">, Sched<[]>;
 } // Predicates = [HasStdExtZbp]
 
 let Predicates = [HasStdExtZba, IsRV64] in {
-def SLLIUW : RVBShift_ri<0b00001, 0b001, OPC_OP_IMM_32, "slli.uw">, Sched<[]>;
-def ADDUW : ALUW_rr<0b0000100, 0b000, "add.uw">, Sched<[]>;
-def SH1ADDUW : ALUW_rr<0b0010000, 0b010, "sh1add.uw">, Sched<[]>;
-def SH2ADDUW : ALUW_rr<0b0010000, 0b100, "sh2add.uw">, Sched<[]>;
-def SH3ADDUW : ALUW_rr<0b0010000, 0b110, "sh3add.uw">, Sched<[]>;
+def SLLIUW : RVBShift_ri<0b00001, 0b001, OPC_OP_IMM_32, "slli.uw">,
+             Sched<[WriteShiftImm32, ReadShiftImm32]>;
+def ADDUW : ALUW_rr<0b0000100, 0b000, "add.uw">,
+            Sched<[WriteIALU32, ReadIALU32, ReadIALU32]>;
+def SH1ADDUW : ALUW_rr<0b0010000, 0b010, "sh1add.uw">,
+               Sched<[WriteSHXADD32, ReadSHXADD32, ReadSHXADD32]>;
+def SH2ADDUW : ALUW_rr<0b0010000, 0b100, "sh2add.uw">,
+               Sched<[WriteSHXADD32, ReadSHXADD32, ReadSHXADD32]>;
+def SH3ADDUW : ALUW_rr<0b0010000, 0b110, "sh3add.uw">,
+               Sched<[WriteSHXADD32, ReadSHXADD32, ReadSHXADD32]>;
 } // Predicates = [HasStdExtZbb, IsRV64]
 
 let Predicates = [HasStdExtZbbOrZbp, IsRV64] in {
-def ROLW  : ALUW_rr<0b0110000, 0b001, "rolw">, Sched<[]>;
-def RORW  : ALUW_rr<0b0110000, 0b101, "rorw">, Sched<[]>;
+def ROLW  : ALUW_rr<0b0110000, 0b001, "rolw">,
+            Sched<[WriteRotateReg32, ReadRotateReg32, ReadRotateReg32]>;
+def RORW  : ALUW_rr<0b0110000, 0b101, "rorw">,
+            Sched<[WriteRotateReg32, ReadRotateReg32, ReadRotateReg32]>;
 } // Predicates = [HasStdExtZbbOrZbp, IsRV64]
 
 let Predicates = [HasStdExtZbs, IsRV64] in {
@@ -354,7 +374,8 @@ def XPERMW : ALU_rr<0b0010100, 0b000, "xperm.w">, Sched<[]>;
 } // Predicates = [HasStdExtZbp, IsRV64]
 
 let Predicates = [HasStdExtZbbOrZbp, IsRV64] in
-def RORIW : RVBShiftW_ri<0b0110000, 0b101, OPC_OP_IMM_32, "roriw">, Sched<[]>;
+def RORIW : RVBShiftW_ri<0b0110000, 0b101, OPC_OP_IMM_32, "roriw">,
+            Sched<[WriteRotateImm32, ReadRotateImm32]>;
 
 let Predicates = [HasStdExtZbs, IsRV64] in {
 // NOTE: These instructions have been removed from the 0.94 spec. As a result
@@ -383,11 +404,11 @@ def FSRIW : RVBTernaryImm5<0b10, 0b101, OPC_OP_IMM_32,
 
 let Predicates = [HasStdExtZbb, IsRV64] in {
 def CLZW   : RVBUnary<0b0110000, 0b00000, 0b001, RISCVOpcode<0b0011011>,
-                      "clzw">, Sched<[]>;
+                      "clzw">, Sched<[WriteCLZ32, ReadCLZ32]>;
 def CTZW   : RVBUnary<0b0110000, 0b00001, 0b001, RISCVOpcode<0b0011011>,
-                      "ctzw">, Sched<[]>;
+                      "ctzw">, Sched<[WriteCTZ32, ReadCTZ32]>;
 def CPOPW  : RVBUnary<0b0110000, 0b00010, 0b001, RISCVOpcode<0b0011011>,
-                      "cpopw">, Sched<[]>;
+                      "cpopw">, Sched<[WriteCPOP32, ReadCPOP32]>;
 } // Predicates = [HasStdExtZbb, IsRV64]
 
 let Predicates = [HasStdExtZbp, IsRV64] in {
@@ -413,7 +434,8 @@ def BFPW : ALUW_rr<0b0100100, 0b111, "bfpw">, Sched<[]>;
 let Predicates = [HasStdExtZbbOrZbp, IsRV32] in {
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 def ZEXTH_RV32 : RVInstR<0b0000100, 0b100, OPC_OP, (outs GPR:$rd),
-                         (ins GPR:$rs1), "zext.h", "$rd, $rs1">, Sched<[]> {
+                         (ins GPR:$rs1), "zext.h", "$rd, $rs1">,
+                 Sched<[WriteIALU, ReadIALU]> {
   let rs2 = 0b00000;
 }
 } // Predicates = [HasStdExtZbbOrZbp, IsRV32]
@@ -421,7 +443,8 @@ def ZEXTH_RV32 : RVInstR<0b0000100, 0b100, OPC_OP, (outs GPR:$rd),
 let Predicates = [HasStdExtZbbOrZbp, IsRV64] in {
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 def ZEXTH_RV64 : RVInstR<0b0000100, 0b100, OPC_OP_32, (outs GPR:$rd),
-                         (ins GPR:$rs1), "zext.h", "$rd, $rs1">, Sched<[]> {
+                         (ins GPR:$rs1), "zext.h", "$rd, $rs1">,
+                 Sched<[WriteIALU, ReadIALU]> {
   let rs2 = 0b00000;
 }
 } // Predicates = [HasStdExtZbbOrZbp, IsRV64]
@@ -436,7 +459,7 @@ def ZEXTH_RV64 : RVInstR<0b0000100, 0b100, OPC_OP_32, (outs GPR:$rd),
 let Predicates = [HasStdExtZbbOrZbp, IsRV32] in {
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 def REV8_RV32 : RVInstI<0b101, OPC_OP_IMM, (outs GPR:$rd), (ins GPR:$rs1),
-                        "rev8", "$rd, $rs1">, Sched<[]> {
+                        "rev8", "$rd, $rs1">, Sched<[WriteREV8, ReadREV8]> {
   let imm12 = { 0b01101, 0b0011000 };
 }
 } // Predicates = [HasStdExtZbbOrZbp, IsRV32]
@@ -444,7 +467,7 @@ def REV8_RV32 : RVInstI<0b101, OPC_OP_IMM, (outs GPR:$rd), (ins GPR:$rs1),
 let Predicates = [HasStdExtZbbOrZbp, IsRV64] in {
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 def REV8_RV64 : RVInstI<0b101, OPC_OP_IMM, (outs GPR:$rd), (ins GPR:$rs1),
-                        "rev8", "$rd, $rs1">, Sched<[]> {
+                        "rev8", "$rd, $rs1">, Sched<[WriteREV8, ReadREV8]> {
   let imm12 = { 0b01101, 0b0111000 };
 }
 } // Predicates = [HasStdExtZbbOrZbp, IsRV64]
@@ -452,7 +475,7 @@ def REV8_RV64 : RVInstI<0b101, OPC_OP_IMM, (outs GPR:$rd), (ins GPR:$rs1),
 let Predicates = [HasStdExtZbbOrZbp] in {
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 def ORCB : RVInstI<0b101, OPC_OP_IMM, (outs GPR:$rd), (ins GPR:$rs1),
-                   "orc.b", "$rd, $rs1">, Sched<[]> {
+                   "orc.b", "$rd, $rs1">, Sched<[WriteORCB, ReadORCB]> {
   let imm12 = { 0b00101, 0b0000111 };
 }
 } // Predicates = [HasStdExtZbbOrZbp]
@@ -871,6 +894,6 @@ def : Pat<(i64 (sext_inreg (or (shl GPR:$rs2, (i64 16)),
                            i32)),
           (PACKW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i64 (or (and (assertsexti32 GPR:$rs2), 0xFFFFFFFFFFFF0000),
-                   (SRLIWPat GPR:$rs1, (i64 16)))),
+                   (srl (and GPR:$rs1, 0xFFFFFFFF), (i64 16)))),
           (PACKUW GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasStdExtZbp, IsRV64]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
index 232b2e05f40e5e8e8d47ffa15639be7c37f04a8a..86f96c1529b128eefb9d8b56b1be5ef3b8f3053d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
@@ -435,9 +435,9 @@ def C_LUI : RVInst16CI<0b011, 0b01, (outs GPRNoX0X2:$rd),
 }
 
 def C_SRLI : Shift_right<0b00, "c.srli", GPRC, uimmlog2xlennonzero>,
-             Sched<[WriteShift, ReadShift]>;
+             Sched<[WriteShiftImm, ReadShiftImm]>;
 def C_SRAI : Shift_right<0b01, "c.srai", GPRC, uimmlog2xlennonzero>,
-             Sched<[WriteShift, ReadShift]>;
+             Sched<[WriteShiftImm, ReadShiftImm]>;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 def C_ANDI : RVInst16CB<0b100, 0b01, (outs GPRC:$rs1_wb), (ins GPRC:$rs1, simm6:$imm),
@@ -480,7 +480,7 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 def C_SLLI : RVInst16CI<0b000, 0b10, (outs GPRNoX0:$rd_wb),
                         (ins GPRNoX0:$rd, uimmlog2xlennonzero:$imm),
                         "c.slli", "$rd, $imm">,
-             Sched<[WriteShift, ReadShift]> {
+             Sched<[WriteShiftImm, ReadShiftImm]> {
   let Constraints = "$rd = $rd_wb";
   let Inst{6-2} = imm{4-0};
 }
@@ -653,7 +653,7 @@ def C_ADD_HINT : RVInst16CR<0b1001, 0b10, (outs GPRX0:$rs1_wb),
 def C_SLLI_HINT : RVInst16CI<0b000, 0b10, (outs GPRX0:$rd_wb),
                              (ins GPRX0:$rd, uimmlog2xlennonzero:$imm),
                              "c.slli", "$rd, $imm">,
-                  Sched<[WriteShift, ReadShift]> {
+                  Sched<[WriteShiftImm, ReadShiftImm]> {
   let Constraints = "$rd = $rd_wb";
   let Inst{6-2} = imm{4-0};
   let Inst{11-7} = 0;
@@ -662,7 +662,7 @@ def C_SLLI_HINT : RVInst16CI<0b000, 0b10, (outs GPRX0:$rd_wb),
 
 def C_SLLI64_HINT : RVInst16CI<0b000, 0b10, (outs GPR:$rd_wb), (ins GPR:$rd),
                                "c.slli64", "$rd">,
-                    Sched<[WriteShift, ReadShift]> {
+                    Sched<[WriteShiftImm, ReadShiftImm]> {
   let Constraints = "$rd = $rd_wb";
   let Inst{6-2} = 0;
   let Inst{12} = 0;
@@ -671,7 +671,7 @@ def C_SLLI64_HINT : RVInst16CI<0b000, 0b10, (outs GPR:$rd_wb), (ins GPR:$rd),
 def C_SRLI64_HINT : RVInst16CI<0b100, 0b01, (outs GPRC:$rd_wb),
                                (ins GPRC:$rd),
                                "c.srli64", "$rd">,
-                    Sched<[WriteShift, ReadShift]> {
+                    Sched<[WriteShiftImm, ReadShiftImm]> {
   let Constraints = "$rd = $rd_wb";
   let Inst{6-2} = 0;
   let Inst{11-10} = 0;
@@ -681,7 +681,7 @@ def C_SRLI64_HINT : RVInst16CI<0b100, 0b01, (outs GPRC:$rd_wb),
 def C_SRAI64_HINT : RVInst16CI<0b100, 0b01, (outs GPRC:$rd_wb),
                                (ins GPRC:$rd),
                                "c.srai64", "$rd">,
-                    Sched<[WriteShift, ReadShift]> {
+                    Sched<[WriteShiftImm, ReadShiftImm]> {
   let Constraints = "$rd = $rd_wb";
   let Inst{6-2} = 0;
   let Inst{11-10} = 1;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
index 7b6668e64c0e18b8af03bfcc0ef7db892bb3e6f1..7ba6db78b0434a0a53f1b3839f8321f4642f662a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
@@ -82,16 +82,16 @@ def FSD : RVInstS<0b011, OPC_STORE_FP, (outs),
           Sched<[WriteFST64, ReadStoreData, ReadFMemBase]>;
 
 def FMADD_D  : FPFMAD_rrr_frm<OPC_MADD, "fmadd.d">,
-               Sched<[WriteFMulAdd64, ReadFMulAdd64, ReadFMulAdd64, ReadFMulAdd64]>;
+               Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>;
 def          : FPFMADDynFrmAlias<FMADD_D, "fmadd.d">;
 def FMSUB_D  : FPFMAD_rrr_frm<OPC_MSUB, "fmsub.d">,
-               Sched<[WriteFMulSub64, ReadFMulSub64, ReadFMulSub64, ReadFMulSub64]>;
+               Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>;
 def          : FPFMADDynFrmAlias<FMSUB_D, "fmsub.d">;
 def FNMSUB_D : FPFMAD_rrr_frm<OPC_NMSUB, "fnmsub.d">,
-               Sched<[WriteFMulSub64, ReadFMulSub64, ReadFMulSub64, ReadFMulSub64]>;
+               Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>;
 def          : FPFMADDynFrmAlias<FNMSUB_D, "fnmsub.d">;
 def FNMADD_D : FPFMAD_rrr_frm<OPC_NMADD, "fnmadd.d">,
-               Sched<[WriteFMulAdd64, ReadFMulAdd64, ReadFMulAdd64, ReadFMulAdd64]>;
+               Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>;
 def          : FPFMADDynFrmAlias<FNMADD_D, "fnmadd.d">;
 
 def FADD_D : FPALUD_rr_frm<0b0000001, "fadd.d">,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoEPI.td b/llvm/lib/Target/RISCV/RISCVInstrInfoEPI.td
index fdfd2dadaddc503b6b34c7d113b933311b394dc0..064a302a01c78ce8f2cabe7153b36ddfe13c57dc 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoEPI.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoEPI.td
@@ -171,43 +171,6 @@ def "PseudoEPIVMCLR_" # MInfo.MX : Pseudo<(outs VR:$rd), (ins GPR:$vl, ixlenimm:
 def "PseudoEPIVMSET_" # MInfo.MX : Pseudo<(outs VR:$rd), (ins GPR:$vl, ixlenimm:$sew), []>;
 }
 
-
-//===----------------------------------------------------------------------===//
-// Pseudo instructions we need for SPILL and RELOAD
-//===----------------------------------------------------------------------===//
-
-multiclass pseudo_spill_reload<LMULInfo MInfo> {
-  defvar reg_class = MInfo.vrclass;
-
-  let hasSideEffects = 1, mayLoad = 0, mayStore = 1, isCodeGenOnly = 1,
-      Uses = [VL, VTYPE] in
-  def "PseudoEPIVSPILL_VR" # MInfo.MX : Pseudo<(outs), (ins reg_class:$rs1, GPR:$rs2), []>;
-
-  let hasSideEffects = 1, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1,
-      Uses = [VL, VTYPE] in
-  def "PseudoEPIVRELOAD_VR" # MInfo.MX : Pseudo<(outs reg_class:$rs1), (ins GPR:$rs2), []>;
-
-}
-
-foreach m = MxListNonFractional.m in {
-  defm "" : pseudo_spill_reload<m>;
-}
-
-// Tuples.
-foreach tsize = 2 ... 8 in {
-
-defvar TupleRegClass = !cast<RegisterClass>("VRN"#tsize#"M1");
-
-let hasSideEffects = 1, mayLoad = 0, mayStore = 1, isCodeGenOnly = 1,
-    Uses = [VL, VTYPE] in
-def PseudoEPIVSPILL_VRN#tsize#M1 : Pseudo<(outs), (ins TupleRegClass:$rs1, GPR:$rs2), []>;
-
-let hasSideEffects = 1, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1,
-    Uses = [VL, VTYPE] in
-def PseudoEPIVRELOAD_VRN#tsize#M1 : Pseudo<(outs TupleRegClass:$rs1), (ins GPR:$rs2), []>;
-
-} // tsize
-
 }
 
 //===----------------------------------------------------------------------===//
@@ -494,8 +457,8 @@ multiclass pseudo_unary<VReg result_reg_class,
 }
 
 // Special case for masking that does not have a merge operand.
-multiclass pseudo_unary_nomerge<VReg result_reg_class,
-                                VReg op_reg_class,
+multiclass pseudo_unary_nomerge<RegisterClass result_reg_class,
+                                RegisterClass op_reg_class,
                                 LMULInfo MInfo,
                                 string constraints = ""> {
   let Constraints = constraints,
@@ -978,8 +941,8 @@ multiclass pseudo_binary_m_vv_vf {
 }
 
 multiclass pseudo_ternary<VReg result_reg_class,
-                          VReg op1_reg_class,
-                          RegisterClass op2_reg_class,
+                          RegisterClass op1_reg_class,
+                          VReg op2_reg_class,
                           LMULInfo MInfo,
                           string constraints,
                           list<SchedReadWrite> sched = []> {
@@ -1623,8 +1586,8 @@ class FPZero<DAGOperand operand> {
                 (FMV_D_X X0), (FMV_W_X X0));
 }
 
-multiclass epi_pat_load_store<LLVMType type,
-                              LLVMType mask_type,
+multiclass epi_pat_load_store<ValueType type,
+                              ValueType mask_type,
                               int eew,
                               LMULInfo MInfo,
                               VReg reg_class>
@@ -3240,7 +3203,7 @@ foreach fvti = EPIAllFloatVectors in {
 //===----------------------------------------------------------------------===//
 
 // FIXME: Add patterns that can select masked versions
-multiclass pat_vop_binary_v<SDNode vop,
+multiclass pat_vop_binary_v<SDPatternOperator vop,
                             string instruction_name,
                             ValueType result_type,
                             ValueType op_type,
@@ -3264,7 +3227,7 @@ multiclass pat_vop_binary_v<SDNode vop,
               swap>.Value>;
 }
 
-multiclass pat_vop_binary_v_nomask<SDNode vop,
+multiclass pat_vop_binary_v_nomask<SDPatternOperator vop,
                                    string instruction_name,
                                    string kind,
                                    ValueType result_type,
@@ -3288,7 +3251,7 @@ multiclass pat_vop_binary_v_nomask<SDNode vop,
               swap>.Value>;
 }
 
-multiclass pat_vop_binary_xif<SDNode vop,
+multiclass pat_vop_binary_xif<SDPatternOperator vop,
                               string instruction_name,
                               string kind,
                               ValueType result_type,
@@ -3315,7 +3278,7 @@ multiclass pat_vop_binary_xif<SDNode vop,
               swap>.Value>;
 }
 
-multiclass pat_vop_binary_int_v_vv_vx<SDNode vop,
+multiclass pat_vop_binary_int_v_vv_vx<SDPatternOperator vop,
                                       string instruction_name>
 {
   foreach vti = EPIAllIntegerVectors in
@@ -3331,7 +3294,7 @@ multiclass pat_vop_binary_int_v_vv_vx<SDNode vop,
   }
 }
 
-multiclass pat_vop_binary_int_v_vv_vx_vi<SDNode vop,
+multiclass pat_vop_binary_int_v_vv_vx_vi<SDPatternOperator vop,
                                          string instruction_name,
                                          DAGOperand imm_kind = simm5>
 {
@@ -3352,7 +3315,7 @@ multiclass pat_vop_binary_int_v_vv_vx_vi<SDNode vop,
   }
 }
 
-multiclass pat_vop_binary_fp_v_vv_vf<SDNode vop,
+multiclass pat_vop_binary_fp_v_vv_vf<SDPatternOperator vop,
                                      string instruction_name>
 {
   foreach fvti = EPIAllFloatVectors in
@@ -3369,7 +3332,7 @@ multiclass pat_vop_binary_fp_v_vv_vf<SDNode vop,
 }
 
 // FIXME: Add patterns that can select masked versions
-multiclass pat_vop_binary_reduction_int<SDNode vop,
+multiclass pat_vop_binary_reduction_int<SDPatternOperator vop,
                                         string instruction_name>
 {
   foreach vti = EPIAllIntegerVectors in {
@@ -3391,7 +3354,7 @@ multiclass pat_vop_binary_reduction_int<SDNode vop,
 }
 
 // FIXME: Add patterns that can select masked versions
-multiclass pat_vop_binary_reduction_fp<SDNode vop,
+multiclass pat_vop_binary_reduction_fp<SDPatternOperator vop,
                                        string instruction_name>
 {
   foreach fvti = EPIAllFloatVectors in {
@@ -3414,7 +3377,7 @@ multiclass pat_vop_binary_reduction_fp<SDNode vop,
 }
 
 // FIXME: Add patterns that can select masked versions
-multiclass pat_vop_ternary_v<SDNode vop,
+multiclass pat_vop_ternary_v<SDPatternOperator vop,
                              string instruction_name,
                              ValueType vector_type,
                              ValueType mask_type,
@@ -3435,7 +3398,7 @@ multiclass pat_vop_ternary_v<SDNode vop,
              sew)>;
 }
 
-multiclass pat_vop_ternary_xf<SDNode vop,
+multiclass pat_vop_ternary_xf<SDPatternOperator vop,
                               string instruction_name,
                               string kind,
                               ValueType vector_type,
@@ -3459,7 +3422,7 @@ multiclass pat_vop_ternary_xf<SDNode vop,
              sew)>;
 }
 
-multiclass pat_vop_ternary_fp_v_vv_vf<SDNode vop,
+multiclass pat_vop_ternary_fp_v_vv_vf<SDPatternOperator vop,
                                       string instruction_name>
 {
   foreach fvti = EPIAllFloatVectors in
@@ -3476,7 +3439,7 @@ multiclass pat_vop_ternary_fp_v_vv_vf<SDNode vop,
 }
 
 // FIXME: Add patterns that can select masked versions
-multiclass pat_vop_unary<SDNode vop,
+multiclass pat_vop_unary<SDPatternOperator vop,
                          string instruction_name,
                          ValueType vector_type,
                          ValueType mask_type,
@@ -3494,7 +3457,7 @@ multiclass pat_vop_unary<SDNode vop,
              sew)>;
 }
 
-multiclass pat_vop_unary_fp_v_v<SDNode vop,
+multiclass pat_vop_unary_fp_v_v<SDPatternOperator vop,
                                 string instruction_name>
 {
   foreach vti = EPIAllFloatVectors in
@@ -3589,12 +3552,6 @@ foreach vti = EPIAllIntegerVectors in
             (!cast<Instruction>("PseudoEPIVMV_V_I_" # vti.MInfo.MX)
              simm5:$value,
              VLMax, vti.SEW)>;
-
-  def : Pat<(vti.Vector (int_experimental_vector_stepvector)),
-            (!cast<Instruction>("PseudoEPIVID_V_" # vti.MInfo.MX)
-             (vti.Vector (IMPLICIT_DEF)),
-             (vti.Mask zero_reg),
-             VLMax, vti.SEW)>;
 }
 
 foreach vti = EPIAllIntegerVectors in {
@@ -3634,7 +3591,7 @@ foreach fvti = EPIAllFloatVectors in {
 // Patterns. Full vector relational operations
 //===----------------------------------------------------------------------===//
 
-multiclass pat_vop_binary_int_m_vv<SDNode vop,
+multiclass pat_vop_binary_int_m_vv<SDPatternOperator vop,
                                    string instruction_name,
                                    bit swap = 0>
 {
@@ -3647,7 +3604,7 @@ multiclass pat_vop_binary_int_m_vv<SDNode vop,
   }
 }
 
-multiclass pat_vop_binary_int_m_vv_vx<SDNode vop,
+multiclass pat_vop_binary_int_m_vv_vx<SDPatternOperator vop,
                                       string instruction_name,
                                       bit swap = 0>
 {
@@ -3664,7 +3621,7 @@ multiclass pat_vop_binary_int_m_vv_vx<SDNode vop,
   }
 }
 
-multiclass pat_vop_binary_int_m_vv_vx_vi<SDNode vop,
+multiclass pat_vop_binary_int_m_vv_vx_vi<SDPatternOperator vop,
                                          string instruction_name,
                                          bit swap = 0>
 {
@@ -3704,7 +3661,7 @@ defm "" : pat_vop_binary_int_m_vv<setge, "PseudoEPIVMSLE", /* swap */ 1>;
 // Patterns. Full vector mask logical operations
 //===----------------------------------------------------------------------===//
 
-multiclass pat_vop_binary_m_mm<SDNode vop,
+multiclass pat_vop_binary_m_mm<SDPatternOperator vop,
                                string instruction_name>
 {
   foreach mti = EPIAllMasks in
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
index 14077c2b09e26a183e3a9dcebb7163742d85aa09..54b2a7d0c33fa934fe812ca0954a8b9aef47e0a9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -117,16 +117,16 @@ def FSW : RVInstS<0b010, OPC_STORE_FP, (outs),
           Sched<[WriteFST32, ReadStoreData, ReadFMemBase]>;
 
 def FMADD_S  : FPFMAS_rrr_frm<OPC_MADD, "fmadd.s">,
-               Sched<[WriteFMulAdd32, ReadFMulAdd32, ReadFMulAdd32, ReadFMulAdd32]>;
+               Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>;
 def          : FPFMASDynFrmAlias<FMADD_S, "fmadd.s">;
 def FMSUB_S  : FPFMAS_rrr_frm<OPC_MSUB, "fmsub.s">,
-               Sched<[WriteFMulSub32, ReadFMulSub32, ReadFMulSub32, ReadFMulSub32]>;
+               Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>;
 def          : FPFMASDynFrmAlias<FMSUB_S, "fmsub.s">;
 def FNMSUB_S : FPFMAS_rrr_frm<OPC_NMSUB, "fnmsub.s">,
-               Sched<[WriteFMulSub32, ReadFMulSub32, ReadFMulSub32, ReadFMulSub32]>;
+               Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>;
 def          : FPFMASDynFrmAlias<FNMSUB_S, "fnmsub.s">;
 def FNMADD_S : FPFMAS_rrr_frm<OPC_NMADD, "fnmadd.s">,
-               Sched<[WriteFMulAdd32, ReadFMulAdd32, ReadFMulAdd32, ReadFMulAdd32]>;
+               Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>;
 def          : FPFMASDynFrmAlias<FNMADD_S, "fnmadd.s">;
 
 def FADD_S : FPALUS_rr_frm<0b0000000, "fadd.s">,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
index 2bfdc9312ebbf051c7dc0cb354c050d3d93433a0..e841d7fdea0b52a37488c5103e38629d87f44dfc 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
@@ -94,3 +94,21 @@ def : Pat<(and (riscv_remuw (assertzexti32 GPR:$rs1),
 def : Pat<(srem (sexti32 (i64 GPR:$rs1)), (sexti32 (i64 GPR:$rs2))),
           (REMW GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasStdExtM, IsRV64]
+
+let Predicates = [HasStdExtM, IsRV64, NotHasStdExtZba] in {
+// Special case for calculating the full 64-bit product of a 32x32 unsigned
+// multiply where the inputs aren't known to be zero extended. We can shift the
+// inputs left by 32 and use a MULHU. This saves two SRLIs needed to finish
+// zeroing the upper 32 bits.
+// TODO: If one of the operands is zero extended and the other isn't, we might
+// still be better off shifting both left by 32.
+def : Pat<(i64 (mul (and GPR:$rs1, 0xffffffff), (and GPR:$rs2, 0xffffffff))),
+          (MULHU (SLLI GPR:$rs1, 32), (SLLI GPR:$rs2, 32))>;
+// Prevent matching the first part of this pattern to mulw. The mul here has
+// additionals users or the ANDs would have been removed. The above pattern
+// will be used for the other users. If we form a mulw we'll keep the ANDs alive
+// and they'll still become SLLI+SRLI.
+def : Pat<(sext_inreg (mul (and GPR:$rs1, 0xffffffff),
+                           (and GPR:$rs2, 0xffffffff)), i32),
+          (ADDIW (MULHU (SLLI GPR:$rs1, 32), (SLLI GPR:$rs2, 32)), 0)>;
+} // Predicates = [HasStdExtM, IsRV64, NotHasStdExtZba]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 281b08ee8b4a2ec805f87edf9be39c746f80ce35..71b8f4ee870f0003f9c5e63cf2c6ba256e809e99 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -821,6 +821,8 @@ defm VFSGNJX_V : VALU_FV_V_F<"vfsgnjx", 0b001010>;
 
 def : InstAlias<"vfneg.v $vd, $vs$vm",
                 (VFSGNJN_VV VR:$vd, VR:$vs, VR:$vs, VMaskOp:$vm)>;
+def : InstAlias<"vfabs.v $vd, $vs$vm",
+                (VFSGNJX_VV VR:$vd, VR:$vs, VR:$vs, VMaskOp:$vm)>;
 
 // Vector Floating-Point Compare Instructions
 let RVVConstraint = NoConstraint in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 013282e2e53741408b552bb52de413c7abeda2dc..4a408ae784176cc947214705a5c0596544fec3a2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -670,7 +670,7 @@ class VPseudoSLoadMask<VReg RetClass, bits<7> EEW>:
 }
 
 class VPseudoILoadNoMask<VReg RetClass, VReg IdxClass, bits<7> EEW, bits<3> LMUL,
-                         bit Ordered>:
+                         bit Ordered, bit EarlyClobber>:
       Pseudo<(outs RetClass:$rd),
              (ins GPR:$rs1, IdxClass:$rs2, GPR:$vl, ixlenimm:$sew),[]>,
       RISCVVPseudo,
@@ -683,11 +683,12 @@ class VPseudoILoadNoMask<VReg RetClass, VReg IdxClass, bits<7> EEW, bits<3> LMUL
   let HasVLOp = 1;
   let HasSEWOp = 1;
   let HasDummyMask = 1;
+  let Constraints = !if(!eq(EarlyClobber, 1), "@earlyclobber $rd", "");
   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 }
 
 class VPseudoILoadMask<VReg RetClass, VReg IdxClass, bits<7> EEW, bits<3> LMUL,
-                       bit Ordered>:
+                       bit Ordered, bit EarlyClobber>:
       Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
               (ins GetVRegNoV0<RetClass>.R:$merge,
                    GPR:$rs1, IdxClass:$rs2,
@@ -698,7 +699,7 @@ class VPseudoILoadMask<VReg RetClass, VReg IdxClass, bits<7> EEW, bits<3> LMUL,
   let mayStore = 0;
   let hasSideEffects = 0;
   let usesCustomInserter = 1;
-  let Constraints = "$rd = $merge";
+  let Constraints = !if(!eq(EarlyClobber, 1), "@earlyclobber $rd, $rd = $merge", "$rd = $merge");
   let Uses = [VL, VTYPE];
   let HasVLOp = 1;
   let HasSEWOp = 1;
@@ -1364,11 +1365,12 @@ multiclass VPseudoILoad<bit Ordered> {
           defvar idx_lmul = !cast<LMULInfo>("V_" # IdxLInfo);
           defvar Vreg = lmul.vrclass;
           defvar IdxVreg = idx_lmul.vrclass;
+          defvar HasConstraint = !ne(sew, eew);
           let VLMul = lmul.value in {
             def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo :
-              VPseudoILoadNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>;
+              VPseudoILoadNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>;
             def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" :
-              VPseudoILoadMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>;
+              VPseudoILoadMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>;
           }
         }
       }
@@ -3171,6 +3173,20 @@ let hasSideEffects = 0, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1 in {
   def PseudoVRELOAD_M8 : VPseudo<VL8RE8_V, V_M8, (outs VRM8:$rs1), (ins GPR:$rs2)>;
 }
 
+foreach lmul = MxList.m in {
+  foreach nf = NFSet<lmul>.L in {
+    defvar vreg = SegRegClass<lmul, nf>.RC;
+    let hasSideEffects = 0, mayLoad = 0, mayStore = 1, isCodeGenOnly = 1 in {
+      def "PseudoVSPILL" # nf # "_" # lmul.MX :
+        Pseudo<(outs), (ins vreg:$rs1, GPR:$rs2, GPR:$vlenb), []>;
+    }
+    let hasSideEffects = 0, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1 in {
+      def "PseudoVRELOAD" # nf # "_" # lmul.MX :
+        Pseudo<(outs vreg:$rs1), (ins GPR:$rs2, GPR:$vlenb), []>;
+    }
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // 6. Configuration-Setting Instructions
 //===----------------------------------------------------------------------===//
@@ -3869,10 +3885,10 @@ foreach vti = AllIntegerVectors in {
                                                                (DecImm simm5_plus1:$rs2),
                                                                GPR:$vl,
                                                                vti.SEW)>;
-  def : Pat<(vti.Mask (int_riscv_vmslt_mask (vti.Mask V0),
+  def : Pat<(vti.Mask (int_riscv_vmslt_mask (vti.Mask VR:$merge),
                                             (vti.Vector vti.RegClass:$rs1),
                                             (vti.Scalar simm5_plus1:$rs2),
-                                            (vti.Mask VR:$merge),
+                                            (vti.Mask V0),
                                             (XLenVT (VLOp GPR:$vl)))),
             (!cast<Instruction>("PseudoVMSLE_VI_"#vti.LMul.MX#"_MASK")
                                                       VR:$merge,
@@ -3882,17 +3898,17 @@ foreach vti = AllIntegerVectors in {
                                                       GPR:$vl,
                                                       vti.SEW)>;
 
- def : Pat<(vti.Mask (int_riscv_vmsltu (vti.Vector vti.RegClass:$rs1),
+  def : Pat<(vti.Mask (int_riscv_vmsltu (vti.Vector vti.RegClass:$rs1),
                                         (vti.Scalar simm5_plus1:$rs2),
                                         (XLenVT (VLOp GPR:$vl)))),
             (!cast<Instruction>("PseudoVMSLEU_VI_"#vti.LMul.MX) vti.RegClass:$rs1,
                                                                 (DecImm simm5_plus1:$rs2),
                                                                 GPR:$vl,
                                                                 vti.SEW)>;
-  def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask V0),
+  def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask VR:$merge),
                                              (vti.Vector vti.RegClass:$rs1),
                                              (vti.Scalar simm5_plus1:$rs2),
-                                             (vti.Mask VR:$merge),
+                                             (vti.Mask V0),
                                              (XLenVT (VLOp GPR:$vl)))),
             (!cast<Instruction>("PseudoVMSLEU_VI_"#vti.LMul.MX#"_MASK")
                                                       VR:$merge,
@@ -3910,11 +3926,11 @@ foreach vti = AllIntegerVectors in {
                                                                vti.RegClass:$rs1,
                                                                GPR:$vl,
                                                                vti.SEW)>;
-  def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask V0),
-                                            (vti.Vector vti.RegClass:$rs1),
-                                            (vti.Scalar 0),
-                                            (vti.Mask VR:$merge),
-                                            (XLenVT (VLOp GPR:$vl)))),
+  def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask VR:$merge),
+                                             (vti.Vector vti.RegClass:$rs1),
+                                             (vti.Scalar 0),
+                                             (vti.Mask V0),
+                                             (XLenVT (VLOp GPR:$vl)))),
             (!cast<Instruction>("PseudoVMSNE_VV_"#vti.LMul.MX#"_MASK")
                                                      VR:$merge,
                                                      vti.RegClass:$rs1,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index b967e4947d143c367247078dc4cf88bc06296ea2..9ff5777dad493f23e31a357582ee873e7cf04462 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -33,21 +33,6 @@ def SplatPat       : ComplexPattern<vAny, 1, "selectVSplat",      [splat_vector,
 def SplatPat_simm5 : ComplexPattern<vAny, 1, "selectVSplatSimm5", [splat_vector, rv32_splat_i64], [], 2>;
 def SplatPat_uimm5 : ComplexPattern<vAny, 1, "selectVSplatUimm5", [splat_vector, rv32_splat_i64], [], 2>;
 
-def masked_load :
-  PatFrag<(ops node:$ptr, node:$mask, node:$maskedoff),
-          (masked_ld node:$ptr, undef, node:$mask, node:$maskedoff), [{
-  return !cast<MaskedLoadSDNode>(N)->isExpandingLoad() &&
-    cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD &&
-    cast<MaskedLoadSDNode>(N)->isUnindexed();
-}]>;
-def masked_store :
-  PatFrag<(ops node:$val, node:$ptr, node:$mask),
-          (masked_st node:$val, node:$ptr, undef, node:$mask), [{
-  return !cast<MaskedStoreSDNode>(N)->isTruncatingStore() &&
-         !cast<MaskedStoreSDNode>(N)->isCompressingStore() &&
-         cast<MaskedStoreSDNode>(N)->isUnindexed();
-}]>;
-
 class SwapHelper<dag Prefix, dag A, dag B, dag Suffix, bit swap> {
    dag Value = !con(Prefix, !if(swap, B, A), !if(swap, A, B), Suffix);
 }
@@ -68,25 +53,6 @@ multiclass VPatUSLoadStoreSDNode<ValueType type,
             (store_instr reg_class:$rs2, BaseAddr:$rs1, avl, sew)>;
 }
 
-multiclass VPatUSLoadStoreSDNodeMask<ValueType type,
-                                     ValueType mask_type,
-                                     int sew,
-                                     LMULInfo vlmul,
-                                     OutPatFrag avl,
-                                     VReg reg_class>
-{
-  defvar load_instr = !cast<Instruction>("PseudoVLE"#sew#"_V_"#vlmul.MX#"_MASK");
-  defvar store_instr = !cast<Instruction>("PseudoVSE"#sew#"_V_"#vlmul.MX#"_MASK");
-  // Load
-  def : Pat<(type (masked_load BaseAddr:$rs1, (mask_type V0), type:$merge)),
-            (load_instr reg_class:$merge, BaseAddr:$rs1, (mask_type V0),
-                        avl, sew)>;
-  // Store
-  def : Pat<(masked_store type:$rs2, BaseAddr:$rs1, (mask_type V0)),
-            (store_instr reg_class:$rs2, BaseAddr:$rs1, (mask_type V0),
-                         avl, sew)>;
-}
-
 multiclass VPatUSLoadStoreWholeVRSDNode<ValueType type,
                                         int sew,
                                         LMULInfo vlmul,
@@ -394,9 +360,6 @@ foreach vti = !listconcat(FractionalGroupIntegerVectors,
                           FractionalGroupFloatVectors) in
   defm "" : VPatUSLoadStoreSDNode<vti.Vector, vti.SEW, vti.LMul,
                                   vti.AVL, vti.RegClass>;
-foreach vti = AllVectors in
-  defm "" : VPatUSLoadStoreSDNodeMask<vti.Vector, vti.Mask, vti.SEW, vti.LMul,
-                                      vti.AVL, vti.RegClass>;
 foreach vti = [VI8M1, VI16M1, VI32M1, VI64M1, VF16M1, VF32M1, VF64M1] in
   defm "" : VPatUSLoadStoreWholeVRSDNode<vti.Vector, vti.SEW, vti.LMul,
                                          vti.RegClass>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
index 85a03122c86107deab49b05c9714eda4d1957f56..1954da8d949c63e6d8962cfeab5b01569ef6928f 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
@@ -60,7 +60,7 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class FPCmpH_rr<bits<3> funct3, string opcodestr>
     : RVInstR<0b1010010, funct3, OPC_OP_FP, (outs GPR:$rd),
               (ins FPR16:$rs1, FPR16:$rs2), opcodestr, "$rd, $rs1, $rs2">,
-      Sched<[]>;
+      Sched<[WriteFCmp16, ReadFCmp16, ReadFCmp16]>;
 
 //===----------------------------------------------------------------------===//
 // Instructions
@@ -71,7 +71,7 @@ let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in
 def FLH : RVInstI<0b001, OPC_LOAD_FP, (outs FPR16:$rd),
                   (ins GPR:$rs1, simm12:$imm12),
                    "flh", "$rd, ${imm12}(${rs1})">,
-          Sched<[]>;
+          Sched<[WriteFLD16, ReadFMemBase]>;
 
 // Operands for stores are in the order srcreg, base, offset rather than
 // reflecting the order these fields are specified in the instruction
@@ -80,94 +80,93 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
 def FSH : RVInstS<0b001, OPC_STORE_FP, (outs),
                   (ins FPR16:$rs2, GPR:$rs1, simm12:$imm12),
                    "fsh", "$rs2, ${imm12}(${rs1})">,
-          Sched<[]>;
+          Sched<[WriteFST16, ReadStoreData, ReadFMemBase]>;
 
 def FMADD_H  : FPFMAH_rrr_frm<OPC_MADD, "fmadd.h">,
-               Sched<[]>;
+               Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>;
 def          : FPFMAHDynFrmAlias<FMADD_H, "fmadd.h">;
 def FMSUB_H  : FPFMAH_rrr_frm<OPC_MSUB, "fmsub.h">,
-               Sched<[]>;
+               Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>;
 def          : FPFMAHDynFrmAlias<FMSUB_H, "fmsub.h">;
 def FNMSUB_H : FPFMAH_rrr_frm<OPC_NMSUB, "fnmsub.h">,
-               Sched<[]>;
+               Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>;
 def          : FPFMAHDynFrmAlias<FNMSUB_H, "fnmsub.h">;
 def FNMADD_H : FPFMAH_rrr_frm<OPC_NMADD, "fnmadd.h">,
-               Sched<[]>;
+               Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>;
 def          : FPFMAHDynFrmAlias<FNMADD_H, "fnmadd.h">;
 
 def FADD_H : FPALUH_rr_frm<0b0000010, "fadd.h">,
-             Sched<[]>;
+             Sched<[WriteFALU16, ReadFALU16, ReadFALU16]>;
 def        : FPALUHDynFrmAlias<FADD_H, "fadd.h">;
 def FSUB_H : FPALUH_rr_frm<0b0000110, "fsub.h">,
-             Sched<[]>;
+             Sched<[WriteFALU16, ReadFALU16, ReadFALU16]>;
 def        : FPALUHDynFrmAlias<FSUB_H, "fsub.h">;
 def FMUL_H : FPALUH_rr_frm<0b0001010, "fmul.h">,
-             Sched<[]>;
+             Sched<[WriteFMul16, ReadFMul16, ReadFMul16]>;
 def        : FPALUHDynFrmAlias<FMUL_H, "fmul.h">;
 def FDIV_H : FPALUH_rr_frm<0b0001110, "fdiv.h">,
-             Sched<[]>;
+             Sched<[WriteFDiv16, ReadFDiv16, ReadFDiv16]>;
 def        : FPALUHDynFrmAlias<FDIV_H, "fdiv.h">;
 
 def FSQRT_H : FPUnaryOp_r_frm<0b0101110, FPR16, FPR16, "fsqrt.h">,
-              Sched<[]> {
+              Sched<[WriteFSqrt16, ReadFSqrt16]> {
   let rs2 = 0b00000;
 }
 def         : FPUnaryOpDynFrmAlias<FSQRT_H, "fsqrt.h", FPR16, FPR16>;
 
 def FSGNJ_H  : FPALUH_rr<0b0010010, 0b000, "fsgnj.h">,
-               Sched<[]>;
+               Sched<[WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16]>;
 def FSGNJN_H : FPALUH_rr<0b0010010, 0b001, "fsgnjn.h">,
-               Sched<[]>;
+               Sched<[WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16]>;
 def FSGNJX_H : FPALUH_rr<0b0010010, 0b010, "fsgnjx.h">,
-               Sched<[]>;
-
+               Sched<[WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16]>;
 def FMIN_H   : FPALUH_rr<0b0010110, 0b000, "fmin.h">,
-               Sched<[]>;
+               Sched<[WriteFMinMax16, ReadFMinMax16, ReadFMinMax16]>;
 def FMAX_H   : FPALUH_rr<0b0010110, 0b001, "fmax.h">,
-               Sched<[]>;
+               Sched<[WriteFMinMax16, ReadFMinMax16, ReadFMinMax16]>;
 
 def FCVT_W_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.w.h">,
-               Sched<[]> {
+               Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]> {
   let rs2 = 0b00000;
 }
 def          : FPUnaryOpDynFrmAlias<FCVT_W_H, "fcvt.w.h", GPR, FPR16>;
 
 def FCVT_WU_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.wu.h">,
-                Sched<[]> {
+                Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]> {
   let rs2 = 0b00001;
 }
 def           : FPUnaryOpDynFrmAlias<FCVT_WU_H, "fcvt.wu.h", GPR, FPR16>;
 
 def FCVT_H_W : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.w">,
-               Sched<[]> {
+               Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]> {
   let rs2 = 0b00000;
 }
 def          : FPUnaryOpDynFrmAlias<FCVT_H_W, "fcvt.h.w", FPR16, GPR>;
 
 def FCVT_H_WU : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.wu">,
-                Sched<[]> {
+                Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]> {
   let rs2 = 0b00001;
 }
 def           : FPUnaryOpDynFrmAlias<FCVT_H_WU, "fcvt.h.wu", FPR16, GPR>;
 
 def FCVT_H_S : FPUnaryOp_r_frm<0b0100010, FPR16, FPR32, "fcvt.h.s">,
-               Sched<[]> {
+               Sched<[WriteFCvtF32ToF16, ReadFCvtF32ToF16]> {
   let rs2 = 0b00000;
 }
 def          : FPUnaryOpDynFrmAlias<FCVT_H_S, "fcvt.h.s", FPR16, FPR32>;
 
 def FCVT_S_H : FPUnaryOp_r<0b0100000, 0b000, FPR32, FPR16, "fcvt.s.h">,
-               Sched<[]> {
+               Sched<[WriteFCvtF16ToF32, ReadFCvtF16ToF32]> {
   let rs2 = 0b00010;
 }
 
 def FMV_X_H : FPUnaryOp_r<0b1110010, 0b000, GPR, FPR16, "fmv.x.h">,
-              Sched<[]> {
+              Sched<[WriteFMovF16ToI16, ReadFMovF16ToI16]> {
   let rs2 = 0b00000;
 }
 
 def FMV_H_X : FPUnaryOp_r<0b1111010, 0b000, FPR16, GPR, "fmv.h.x">,
-              Sched<[]> {
+              Sched<[WriteFMovI16ToF16, ReadFMovI16ToF16]> {
   let rs2 = 0b00000;
 }
 
@@ -176,32 +175,32 @@ def FLT_H : FPCmpH_rr<0b001, "flt.h">;
 def FLE_H : FPCmpH_rr<0b000, "fle.h">;
 
 def FCLASS_H : FPUnaryOp_r<0b1110010, 0b001, GPR, FPR16, "fclass.h">,
-               Sched<[]> {
+               Sched<[WriteFClass16, ReadFClass16]> {
   let rs2 = 0b00000;
 }
 } // Predicates = [HasStdExtZfh]
 
 let Predicates = [HasStdExtZfh, IsRV64] in {
 def FCVT_L_H  : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.l.h">,
-                Sched<[]> {
+                Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]> {
   let rs2 = 0b00010;
 }
 def           : FPUnaryOpDynFrmAlias<FCVT_L_H, "fcvt.l.h", GPR, FPR16>;
 
 def FCVT_LU_H  : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.lu.h">,
-                 Sched<[]> {
+                 Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]> {
   let rs2 = 0b00011;
 }
 def            : FPUnaryOpDynFrmAlias<FCVT_LU_H, "fcvt.lu.h", GPR, FPR16>;
 
 def FCVT_H_L : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.l">,
-               Sched<[]> {
+               Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]> {
   let rs2 = 0b00010;
 }
 def          : FPUnaryOpDynFrmAlias<FCVT_H_L, "fcvt.h.l", FPR16, GPR>;
 
 def FCVT_H_LU : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.lu">,
-                Sched<[]> {
+                Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]> {
   let rs2 = 0b00011;
 }
 def           : FPUnaryOpDynFrmAlias<FCVT_H_LU, "fcvt.h.lu", FPR16, GPR>;
@@ -209,13 +208,13 @@ def           : FPUnaryOpDynFrmAlias<FCVT_H_LU, "fcvt.h.lu", FPR16, GPR>;
 
 let Predicates = [HasStdExtZfh, HasStdExtD] in {
 def FCVT_H_D : FPUnaryOp_r_frm<0b0100010, FPR16, FPR64, "fcvt.h.d">,
-               Sched<[]> {
+               Sched<[WriteFCvtF64ToF16, ReadFCvtF64ToF16]> {
   let rs2 = 0b00001;
 }
 def          : FPUnaryOpDynFrmAlias<FCVT_H_D, "fcvt.h.d", FPR16, FPR64>;
 
 def FCVT_D_H : FPUnaryOp_r<0b0100001, 0b000, FPR64, FPR16, "fcvt.d.h">,
-               Sched<[]> {
+               Sched<[WriteFCvtF16ToF64, ReadFCvtF16ToF64]> {
   let rs2 = 0b00010;
 }
 } // Predicates = [HasStdExtZfh, HasStdExtD]
@@ -280,10 +279,7 @@ def : PatFpr16Fpr16<fcopysign, FSGNJ_H>;
 def : Pat<(fcopysign FPR16:$rs1, (fneg FPR16:$rs2)), (FSGNJN_H $rs1, $rs2)>;
 def : Pat<(fcopysign FPR16:$rs1, FPR32:$rs2),
           (FSGNJ_H $rs1, (FCVT_H_S $rs2, 0b111))>;
-def : Pat<(fcopysign FPR16:$rs1, FPR64:$rs2),
-          (FSGNJ_H $rs1, (FCVT_H_D $rs2, 0b111))>;
 def : Pat<(fcopysign FPR32:$rs1, FPR16:$rs2), (FSGNJ_S $rs1, (FCVT_S_H $rs2))>;
-def : Pat<(fcopysign FPR64:$rs1, FPR16:$rs2), (FSGNJ_D $rs1, (FCVT_D_H $rs2))>;
 
 // fmadd: rs1 * rs2 + rs3
 def : Pat<(fma FPR16:$rs1, FPR16:$rs2, FPR16:$rs3),
@@ -368,4 +364,9 @@ let Predicates = [HasStdExtZfh, HasStdExtD] in {
 // f64 -> f16, f16 -> f64
 def : Pat<(fpround FPR64:$rs1), (FCVT_H_D FPR64:$rs1, 0b111)>;
 def : Pat<(fpextend FPR16:$rs1), (FCVT_D_H FPR16:$rs1)>;
+
+/// Float arithmetic operations
+def : Pat<(fcopysign FPR16:$rs1, FPR64:$rs2),
+          (FSGNJ_H $rs1, (FCVT_H_D $rs2, 0b111))>;
+def : Pat<(fcopysign FPR64:$rs1, FPR16:$rs2), (FSGNJ_D $rs1, (FCVT_D_H $rs2))>;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp
index 8f3baece4692a4aa2e22070b5cf1771f93de3485..69d937867703265394a2c3d66db42ef570381219 100644
--- a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp
@@ -204,40 +204,7 @@ static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
   return true;
 }
 
-void llvm::LowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
-                                          const AsmPrinter &AP) {
-  if (LowerEPIMachineInstrToMCInst(MI, OutMI))
-    return;
-
-  if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
-    return;
-
-  OutMI.setOpcode(MI->getOpcode());
-
-  for (const MachineOperand &MO : MI->operands()) {
-    MCOperand MCOp;
-    if (LowerRISCVMachineOperandToMCOperand(MO, MCOp, AP))
-      OutMI.addOperand(MCOp);
-  }
-
-  if (OutMI.getOpcode() == RISCV::PseudoReadVLENB) {
-    OutMI.setOpcode(RISCV::CSRRS);
-    OutMI.addOperand(MCOperand::createImm(
-        RISCVSysReg::lookupSysRegByName("VLENB")->Encoding));
-    OutMI.addOperand(MCOperand::createReg(RISCV::X0));
-    return;
-  }
-
-  if (OutMI.getOpcode() == RISCV::PseudoReadVL) {
-    OutMI.setOpcode(RISCV::CSRRS);
-    OutMI.addOperand(MCOperand::createImm(
-        RISCVSysReg::lookupSysRegByName("VL")->Encoding));
-    OutMI.addOperand(MCOperand::createReg(RISCV::X0));
-    return;
-  }
-}
-
-bool llvm::LowerEPIMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI) {
+static bool lowerEPIMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI) {
   const RISCVEPIPseudosTable::EPIPseudoInfo *EPI =
       RISCVEPIPseudosTable::getEPIPseudoInfo(MI->getOpcode());
   if (!EPI)
@@ -266,7 +233,7 @@ bool llvm::LowerEPIMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI) {
     MCOperand MCOp;
     switch (MO.getType()) {
     default:
-      report_fatal_error("LowerEPIMachineInstrToMCInst: unknown operand type");
+      report_fatal_error("lowerEPIMachineInstrToMCInst: unknown operand type");
     case MachineOperand::MO_Register: {
       // Ignore all implicit register operands.
       if (MO.isImplicit())
@@ -300,3 +267,49 @@ bool llvm::LowerEPIMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI) {
   }
   return true;
 }
+
+bool llvm::lowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
+                                          AsmPrinter &AP) {
+  if (lowerEPIMachineInstrToMCInst(MI, OutMI))
+    return false;
+
+  if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
+    return false;
+
+  OutMI.setOpcode(MI->getOpcode());
+
+  for (const MachineOperand &MO : MI->operands()) {
+    MCOperand MCOp;
+    if (LowerRISCVMachineOperandToMCOperand(MO, MCOp, AP))
+      OutMI.addOperand(MCOp);
+  }
+
+  switch (OutMI.getOpcode()) {
+  case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
+    const Function &F = MI->getParent()->getParent()->getFunction();
+    if (F.hasFnAttribute("patchable-function-entry")) {
+      unsigned Num;
+      if (F.getFnAttribute("patchable-function-entry")
+              .getValueAsString()
+              .getAsInteger(10, Num))
+        return false;
+      AP.emitNops(Num);
+      return true;
+    }
+    break;
+  }
+  case RISCV::PseudoReadVLENB:
+    OutMI.setOpcode(RISCV::CSRRS);
+    OutMI.addOperand(MCOperand::createImm(
+        RISCVSysReg::lookupSysRegByName("VLENB")->Encoding));
+    OutMI.addOperand(MCOperand::createReg(RISCV::X0));
+    break;
+  case RISCV::PseudoReadVL:
+    OutMI.setOpcode(RISCV::CSRRS);
+    OutMI.addOperand(
+        MCOperand::createImm(RISCVSysReg::lookupSysRegByName("VL")->Encoding));
+    OutMI.addOperand(MCOperand::createReg(RISCV::X0));
+    break;
+  }
+  return false;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 3a65db0c31f720d902685c0e6515d97af3ec14aa..614a40e08e25dc18f892895532ec8fdd4b3f7aa5 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -151,110 +151,6 @@ bool RISCVRegisterInfo::hasReservedSpillSlot(const MachineFunction &MF,
   return true;
 }
 
-static Register computeVRSpillReloadInstructions(
-    MachineBasicBlock::iterator II, const Register &ValueReg,
-    const Register &AddrReg, unsigned LMUL, bool IsReload, unsigned TupleSize,
-    bool KillVReg, Register VLenBReg = 0, bool KillAddr = false) {
-  MachineBasicBlock &MBB = *II->getParent();
-  MachineFunction &MF = *MBB.getParent();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
-  const TargetRegisterInfo &RI = *MF.getSubtarget().getRegisterInfo();
-  const RISCVInstrInfo &TII = *MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
-  DebugLoc DL = II->getDebugLoc();
-
-  assert(TupleSize != 1 && "TupleSize can't be 1");
-
-  if (LMUL == 1) {
-    if (!TupleSize) {
-      // This is the same as a single register and we only get there now
-      // via tuples. Individual vector registers are handled elsewhere.
-      if (IsReload)
-        BuildMI(MBB, II, DL, TII.get(RISCV::VL1RE8_V), ValueReg)
-            .addReg(AddrReg, getKillRegState(KillAddr || !VLenBReg));
-      else
-        BuildMI(MBB, II, DL, TII.get(RISCV::VS1R_V))
-            .addReg(ValueReg, getKillRegState(KillVReg))
-            .addReg(AddrReg, getKillRegState(KillAddr || !VLenBReg));
-
-      return AddrReg;
-    } else {
-      assert(TupleSize == 2 && "Unexpected tuple size");
-      Register VRegFirst = RI.getSubReg(ValueReg, RISCV::sub_vrm1_0);
-      Register VRegSecond = RI.getSubReg(ValueReg, RISCV::sub_vrm1_1);
-
-      // Compute the second handle already so we can kill the first handle
-      // when spilling the first register.
-      assert(VLenBReg == 0 && "Unexpected register");
-      VLenBReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-      BuildMI(MBB, II, DL, TII.get(RISCV::PseudoEPIReadVLENB), VLenBReg);
-
-      Register AddrRegSecond = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-      BuildMI(MBB, II, DL, TII.get(RISCV::ADD), AddrRegSecond)
-          .addReg(AddrReg)
-          .addReg(VLenBReg);
-
-      // First part.
-      computeVRSpillReloadInstructions(II, VRegFirst, AddrReg, LMUL, IsReload,
-                                       /* TupleSize */ 0, KillVReg);
-      // Second part.
-      computeVRSpillReloadInstructions(II, VRegSecond, AddrRegSecond, LMUL,
-                                       IsReload, /* TupleSize */ 0, KillVReg);
-      // We're done. Return zero_reg to make sure nobody attempts to use this
-      // for now.
-      return RISCV::NoRegister;
-    }
-  }
-
-  assert(TupleSize == 0 && "No tuple vectors of LMUL>1 yet");
-
-  Register VRegEven, VRegOdd;
-  switch (LMUL) {
-  default:
-    llvm_unreachable("Unexpected LMUL value");
-  case 2:
-    VRegEven = RI.getSubReg(ValueReg, RISCV::sub_vrm1_0);
-    VRegOdd = RI.getSubReg(ValueReg, RISCV::sub_vrm1_1);
-    break;
-  case 4:
-    VRegEven = RI.getSubReg(ValueReg, RISCV::sub_vrm2_0);
-    VRegOdd = RI.getSubReg(ValueReg, RISCV::sub_vrm2_1);
-    break;
-  case 8:
-    VRegEven = RI.getSubReg(ValueReg, RISCV::sub_vrm4_0);
-    VRegOdd = RI.getSubReg(ValueReg, RISCV::sub_vrm4_1);
-    break;
-  }
-
-  // Compute VLENB if it hasn't been computed.
-  if (!VLenBReg) {
-    VLenBReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-    BuildMI(MBB, II, DL, TII.get(RISCV::PseudoEPIReadVLENB), VLenBReg);
-
-    // The handle should only be killed in the VS1R/VL1R for the last register
-    // in the register group. This boolean is forwarded through the odd
-    // recursive calls only.
-    // Since VLENB hasn't been computed this must be the toplevel recursive
-    // call.
-    KillAddr = true;
-  }
-
-  // Recursive call on the even subregister. The handle corresponding to the
-  // latest spill/reload in the recursion is returned as a result.
-  Register LastAddrReg =
-      computeVRSpillReloadInstructions(II, VRegEven, AddrReg, LMUL / 2,
-                                       IsReload, TupleSize, KillVReg, VLenBReg);
-
-  Register AddrRegOdd = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-  BuildMI(MBB, II, DL, TII.get(RISCV::ADD), AddrRegOdd)
-      .addReg(LastAddrReg, RegState::Kill)
-      .addReg(VLenBReg);
-
-  // Recursive call on the odd subregister. Return latest spill/reload handle.
-  return computeVRSpillReloadInstructions(II, VRegOdd, AddrRegOdd, LMUL / 2,
-                                          IsReload, TupleSize, KillVReg, VLenBReg,
-                                          KillAddr);
-}
-
 static bool isRVVWholeLoadStore(unsigned Opcode) {
   switch (Opcode) {
   default:
@@ -283,29 +179,6 @@ static bool isRVVWholeLoadStore(unsigned Opcode) {
   }
 }
 
-static bool isEPISpillReload(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return false;
-  case RISCV::PseudoEPIVRELOAD_VRN2M1:
-  case RISCV::PseudoEPIVRELOAD_VRN3M1:
-  case RISCV::PseudoEPIVRELOAD_VRN4M1:
-  case RISCV::PseudoEPIVRELOAD_VRN5M1:
-  case RISCV::PseudoEPIVRELOAD_VRN6M1:
-  case RISCV::PseudoEPIVRELOAD_VRN7M1:
-  case RISCV::PseudoEPIVRELOAD_VRN8M1:
-
-  case RISCV::PseudoEPIVSPILL_VRN2M1:
-  case RISCV::PseudoEPIVSPILL_VRN3M1:
-  case RISCV::PseudoEPIVSPILL_VRN4M1:
-  case RISCV::PseudoEPIVSPILL_VRN5M1:
-  case RISCV::PseudoEPIVSPILL_VRN6M1:
-  case RISCV::PseudoEPIVSPILL_VRN7M1:
-  case RISCV::PseudoEPIVSPILL_VRN8M1:
-    return true;
-  }
-}
-
 void RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
                                             int SPAdj, unsigned FIOperandNum,
                                             RegScavenger *RS) const {
@@ -323,9 +196,10 @@ void RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
       getFrameLowering(MF)->getFrameIndexReference(MF, FrameIndex, FrameReg);
   bool isRVV = RISCVVPseudosTable::getPseudoInfo(MI.getOpcode()) ||
                isRVVWholeLoadStore(MI.getOpcode()) ||
+               TII->isRVVSpillForZvlsseg(MI.getOpcode()) ||
                // EPI remaining cases.
-               RISCVEPIPseudosTable::getEPIPseudoInfo(MI.getOpcode()) ||
-               isEPISpillReload(MI.getOpcode());
+               RISCVEPIPseudosTable::getEPIPseudoInfo(MI.getOpcode());
+
   if (!isRVV)
     Offset += StackOffset::getFixed(MI.getOperand(FIOperandNum + 1).getImm());
 
@@ -342,6 +216,13 @@ void RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
     // Modify Offset and FrameReg appropriately
     Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
     TII->movImm(MBB, II, DL, ScratchReg, Offset.getFixed());
+    if (MI.getOpcode() == RISCV::ADDI) {
+      BuildMI(MBB, II, DL, TII->get(RISCV::ADD), MI.getOperand(0).getReg())
+        .addReg(FrameReg)
+        .addReg(ScratchReg, RegState::Kill);
+      MI.eraseFromParent();
+      return;
+    }
     BuildMI(MBB, II, DL, TII->get(RISCV::ADD), ScratchReg)
         .addReg(FrameReg)
         .addReg(ScratchReg, RegState::Kill);
@@ -400,57 +281,14 @@ void RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 
   }
 
-  // Lower EPI spill/reload for tuples with the actual instructions.
-  if (MI.getOpcode() == RISCV::PseudoEPIVSPILL_VRN2M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_VRN2M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVSPILL_VRN3M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_VRN3M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVSPILL_VRN4M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_VRN4M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVSPILL_VRN5M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_VRN5M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVSPILL_VRN6M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_VRN6M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVSPILL_VRN7M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_VRN7M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVSPILL_VRN8M1 ||
-      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_VRN8M1) {
-
-    // Make sure we spill/reload all the bits using whole register
-    // instructions.
-    MachineOperand &ValueReg = MI.getOperand(0);
-    MachineOperand &AddrReg = MI.getOperand(FIOperandNum);
-    bool IsReload;
-    unsigned LMUL;
-    unsigned TupleSize = 0;
-    switch (MI.getOpcode()) {
-    default:
-      llvm_unreachable("Unexpected instruction");
-#define TUPLE_SPILL_RELOAD(N)                                                  \
-  case RISCV::PseudoEPIVSPILL_VRN##N##M1:                                      \
-    IsReload = false;                                                          \
-    LMUL = 1;                                                                  \
-    TupleSize = N;                                                             \
-    break;                                                                     \
-  case RISCV::PseudoEPIVRELOAD_VRN##N##M1:                                     \
-    IsReload = true;                                                           \
-    LMUL = 1;                                                                  \
-    TupleSize = N;                                                             \
-    break;
-      TUPLE_SPILL_RELOAD(2)
-      TUPLE_SPILL_RELOAD(3)
-      TUPLE_SPILL_RELOAD(4)
-      TUPLE_SPILL_RELOAD(5)
-      TUPLE_SPILL_RELOAD(6)
-      TUPLE_SPILL_RELOAD(7)
-      TUPLE_SPILL_RELOAD(8)
-    }
-    computeVRSpillReloadInstructions(II, ValueReg.getReg(), AddrReg.getReg(),
-                                     LMUL, IsReload, TupleSize,
-                                     /* KillVReg */ ValueReg.isKill());
-
-    // Remove the pseudo.
-    MI.eraseFromParent();
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  auto ZvlssegInfo = TII->isRVVSpillForZvlsseg(MI.getOpcode());
+  if (ZvlssegInfo) {
+    int64_t ScalableValue = MFI.getObjectSize(FrameIndex) / ZvlssegInfo->first;
+    Register FactorRegister =
+        TII->getVLENFactoredAmount(MF, MBB, II, ScalableValue);
+    MI.getOperand(FIOperandNum + 1)
+        .ChangeToRegister(FactorRegister, /*isDef=*/false);
   }
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVSchedAvispado.td b/llvm/lib/Target/RISCV/RISCVSchedAvispado.td
index a90be23dc0e7441036c8a58c2c56394ed88f385e..2b4b138d2527b662ee0eb253f620176925b2618e 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedAvispado.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedAvispado.td
@@ -19,7 +19,7 @@ def AvispadoModel : SchedMachineModel {
   let IssueWidth = 1;        // One instruction per cycle can be dispatched.
   let LoadLatency = 3;
   let MispredictPenalty = 3;
-  let UnsupportedFeatures = [HasStdExtZvamo, HasStdExtZvlsseg];
+  let UnsupportedFeatures = [HasStdExtZbb, HasStdExtZvamo, HasStdExtZvlsseg];
 
   // We should be able to remove this when all the instructions
   // have scheduling information.
@@ -60,8 +60,10 @@ def : WriteRes<WriteJmpReg, [AvispadoUnitB]>;
 // Integer arithmetic and logic
 def : WriteRes<WriteIALU32, [AvispadoUnitALU]>;
 def : WriteRes<WriteIALU, [AvispadoUnitALU]>;
-def : WriteRes<WriteShift32, [AvispadoUnitALU]>;
-def : WriteRes<WriteShift, [AvispadoUnitALU]>;
+def : WriteRes<WriteShiftReg32, [AvispadoUnitALU]>;
+def : WriteRes<WriteShiftImm32, [AvispadoUnitALU]>;
+def : WriteRes<WriteShiftReg, [AvispadoUnitALU]>;
+def : WriteRes<WriteShiftImm, [AvispadoUnitALU]>;
 
 // Integer multiplication
 let Latency = 4 in {
@@ -151,14 +153,12 @@ def : WriteRes<WriteFMovI64ToF64, [AvispadoUnitFPALU]>;
 // FP multiplication
 let Latency = 5 in {
 def : WriteRes<WriteFMul32, [AvispadoUnitFPALU]>;
-def : WriteRes<WriteFMulAdd32, [AvispadoUnitFPALU]>;
-def : WriteRes<WriteFMulSub32, [AvispadoUnitFPALU]>;
+def : WriteRes<WriteFMA32, [AvispadoUnitFPALU]>;
 }
 
 let Latency = 7 in {
 def : WriteRes<WriteFMul64, [AvispadoUnitFPALU]>;
-def : WriteRes<WriteFMulAdd64, [AvispadoUnitFPALU]>;
-def : WriteRes<WriteFMulSub64, [AvispadoUnitFPALU]>;
+def : WriteRes<WriteFMA64, [AvispadoUnitFPALU]>;
 }
 
 // FP division
@@ -189,8 +189,10 @@ def : ReadAdvance<ReadStoreData, 0>;
 def : ReadAdvance<ReadMemBase, 0>;
 def : ReadAdvance<ReadIALU, 0>;
 def : ReadAdvance<ReadIALU32, 0>;
-def : ReadAdvance<ReadShift, 0>;
-def : ReadAdvance<ReadShift32, 0>;
+def : ReadAdvance<ReadShiftReg, 0>;
+def : ReadAdvance<ReadShiftImm, 0>;
+def : ReadAdvance<ReadShiftReg32, 0>;
+def : ReadAdvance<ReadShiftImm32, 0>;
 def : ReadAdvance<ReadIDiv, 0>;
 def : ReadAdvance<ReadIDiv32, 0>;
 def : ReadAdvance<ReadIMul, 0>;
@@ -207,11 +209,9 @@ def : ReadAdvance<ReadFMemBase, 0>;
 def : ReadAdvance<ReadFALU32, 0>;
 def : ReadAdvance<ReadFALU64, 0>;
 def : ReadAdvance<ReadFMul32, 0>;
-def : ReadAdvance<ReadFMulAdd32, 0>;
-def : ReadAdvance<ReadFMulSub32, 0>;
+def : ReadAdvance<ReadFMA32, 0>;
 def : ReadAdvance<ReadFMul64, 0>;
-def : ReadAdvance<ReadFMulAdd64, 0>;
-def : ReadAdvance<ReadFMulSub64, 0>;
+def : ReadAdvance<ReadFMA64, 0>;
 def : ReadAdvance<ReadFDiv32, 0>;
 def : ReadAdvance<ReadFDiv64, 0>;
 def : ReadAdvance<ReadFSqrt32, 0>;
@@ -295,4 +295,53 @@ def : ReadAdvance<ReadVPUFALU, 0>;
 def : ReadAdvance<ReadVPUMem, 0>;
 def : ReadAdvance<ReadVPUStoreData,       0>;
 
+// Unsupported extensions
+defm : UnsupportedSchedZba;
+defm : UnsupportedSchedZbb;
+
+// Zfh is unsupported
+let Unsupported = true in {
+def : WriteRes<WriteFALU16, []>;
+def : WriteRes<WriteFClass16, []>;
+def : WriteRes<WriteFCvtF16ToF64, []>;
+def : WriteRes<WriteFCvtF64ToF16, []>;
+def : WriteRes<WriteFCvtI64ToF16, []>;
+def : WriteRes<WriteFCvtF32ToF16, []>;
+def : WriteRes<WriteFCvtI32ToF16, []>;
+def : WriteRes<WriteFCvtF16ToI64, []>;
+def : WriteRes<WriteFCvtF16ToF32, []>;
+def : WriteRes<WriteFCvtF16ToI32, []>;
+def : WriteRes<WriteFDiv16, []>;
+def : WriteRes<WriteFCmp16, []>;
+def : WriteRes<WriteFLD16, []>;
+def : WriteRes<WriteFMA16, []>;
+def : WriteRes<WriteFMinMax16, []>;
+def : WriteRes<WriteFMul16, []>;
+def : WriteRes<WriteFMovI16ToF16, []>;
+def : WriteRes<WriteFMovF16ToI16, []>;
+def : WriteRes<WriteFSGNJ16, []>;
+def : WriteRes<WriteFST16, []>;
+def : WriteRes<WriteFSqrt16, []>;
+
+def : ReadAdvance<ReadFALU16, 0>;
+def : ReadAdvance<ReadFClass16, 0>;
+def : ReadAdvance<ReadFCvtF16ToF64, 0>;
+def : ReadAdvance<ReadFCvtF64ToF16, 0>;
+def : ReadAdvance<ReadFCvtI64ToF16, 0>;
+def : ReadAdvance<ReadFCvtF32ToF16, 0>;
+def : ReadAdvance<ReadFCvtI32ToF16, 0>;
+def : ReadAdvance<ReadFCvtF16ToI64, 0>;
+def : ReadAdvance<ReadFCvtF16ToF32, 0>;
+def : ReadAdvance<ReadFCvtF16ToI32, 0>;
+def : ReadAdvance<ReadFDiv16, 0>;
+def : ReadAdvance<ReadFCmp16, 0>;
+def : ReadAdvance<ReadFMA16, 0>;
+def : ReadAdvance<ReadFMinMax16, 0>;
+def : ReadAdvance<ReadFMul16, 0>;
+def : ReadAdvance<ReadFMovI16ToF16, 0>;
+def : ReadAdvance<ReadFMovF16ToI16, 0>;
+def : ReadAdvance<ReadFSGNJ16, 0>;
+def : ReadAdvance<ReadFSqrt16, 0>;
+} // Unsupported = true
+
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedRocket.td b/llvm/lib/Target/RISCV/RISCVSchedRocket.td
index 2c790edfae9a83c8fc62cba1a1bd8a883b0173ad..241134a731a3789874700bd089de540c887444d7 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedRocket.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedRocket.td
@@ -52,8 +52,10 @@ def : WriteRes<WriteJmpReg, [RocketUnitB]>;
 // Integer arithmetic and logic
 def : WriteRes<WriteIALU32, [RocketUnitALU]>;
 def : WriteRes<WriteIALU, [RocketUnitALU]>;
-def : WriteRes<WriteShift32, [RocketUnitALU]>;
-def : WriteRes<WriteShift, [RocketUnitALU]>;
+def : WriteRes<WriteShiftImm32, [RocketUnitALU]>;
+def : WriteRes<WriteShiftImm, [RocketUnitALU]>;
+def : WriteRes<WriteShiftReg32, [RocketUnitALU]>;
+def : WriteRes<WriteShiftReg, [RocketUnitALU]>;
 
 // Integer multiplication
 let Latency = 4 in {
@@ -143,14 +145,12 @@ def : WriteRes<WriteFMovI64ToF64, [RocketUnitFPALU]>;
 // FP multiplication
 let Latency = 5 in {
 def : WriteRes<WriteFMul32, [RocketUnitFPALU]>;
-def : WriteRes<WriteFMulAdd32, [RocketUnitFPALU]>;
-def : WriteRes<WriteFMulSub32, [RocketUnitFPALU]>;
+def : WriteRes<WriteFMA32, [RocketUnitFPALU]>;
 }
 
 let Latency = 7 in {
 def : WriteRes<WriteFMul64, [RocketUnitFPALU]>;
-def : WriteRes<WriteFMulAdd64, [RocketUnitFPALU]>;
-def : WriteRes<WriteFMulSub64, [RocketUnitFPALU]>;
+def : WriteRes<WriteFMA64, [RocketUnitFPALU]>;
 }
 
 // FP division
@@ -181,8 +181,10 @@ def : ReadAdvance<ReadStoreData, 0>;
 def : ReadAdvance<ReadMemBase, 0>;
 def : ReadAdvance<ReadIALU, 0>;
 def : ReadAdvance<ReadIALU32, 0>;
-def : ReadAdvance<ReadShift, 0>;
-def : ReadAdvance<ReadShift32, 0>;
+def : ReadAdvance<ReadShiftImm, 0>;
+def : ReadAdvance<ReadShiftImm32, 0>;
+def : ReadAdvance<ReadShiftReg, 0>;
+def : ReadAdvance<ReadShiftReg32, 0>;
 def : ReadAdvance<ReadIDiv, 0>;
 def : ReadAdvance<ReadIDiv32, 0>;
 def : ReadAdvance<ReadIMul, 0>;
@@ -199,11 +201,9 @@ def : ReadAdvance<ReadFMemBase, 0>;
 def : ReadAdvance<ReadFALU32, 0>;
 def : ReadAdvance<ReadFALU64, 0>;
 def : ReadAdvance<ReadFMul32, 0>;
-def : ReadAdvance<ReadFMulAdd32, 0>;
-def : ReadAdvance<ReadFMulSub32, 0>;
+def : ReadAdvance<ReadFMA32, 0>;
 def : ReadAdvance<ReadFMul64, 0>;
-def : ReadAdvance<ReadFMulAdd64, 0>;
-def : ReadAdvance<ReadFMulSub64, 0>;
+def : ReadAdvance<ReadFMA64, 0>;
 def : ReadAdvance<ReadFDiv32, 0>;
 def : ReadAdvance<ReadFDiv64, 0>;
 def : ReadAdvance<ReadFSqrt32, 0>;
@@ -231,5 +231,54 @@ def : ReadAdvance<ReadFMovI64ToF64, 0>;
 def : ReadAdvance<ReadFClass32, 0>;
 def : ReadAdvance<ReadFClass64, 0>;
 
+// Zfh is unsupported
+let Unsupported = true in {
+def : WriteRes<WriteFALU16, []>;
+def : WriteRes<WriteFClass16, []>;
+def : WriteRes<WriteFCvtF16ToF64, []>;
+def : WriteRes<WriteFCvtF64ToF16, []>;
+def : WriteRes<WriteFCvtI64ToF16, []>;
+def : WriteRes<WriteFCvtF32ToF16, []>;
+def : WriteRes<WriteFCvtI32ToF16, []>;
+def : WriteRes<WriteFCvtF16ToI64, []>;
+def : WriteRes<WriteFCvtF16ToF32, []>;
+def : WriteRes<WriteFCvtF16ToI32, []>;
+def : WriteRes<WriteFDiv16, []>;
+def : WriteRes<WriteFCmp16, []>;
+def : WriteRes<WriteFLD16, []>;
+def : WriteRes<WriteFMA16, []>;
+def : WriteRes<WriteFMinMax16, []>;
+def : WriteRes<WriteFMul16, []>;
+def : WriteRes<WriteFMovI16ToF16, []>;
+def : WriteRes<WriteFMovF16ToI16, []>;
+def : WriteRes<WriteFSGNJ16, []>;
+def : WriteRes<WriteFST16, []>;
+def : WriteRes<WriteFSqrt16, []>;
+
+def : ReadAdvance<ReadFALU16, 0>;
+def : ReadAdvance<ReadFClass16, 0>;
+def : ReadAdvance<ReadFCvtF16ToF64, 0>;
+def : ReadAdvance<ReadFCvtF64ToF16, 0>;
+def : ReadAdvance<ReadFCvtI64ToF16, 0>;
+def : ReadAdvance<ReadFCvtF32ToF16, 0>;
+def : ReadAdvance<ReadFCvtI32ToF16, 0>;
+def : ReadAdvance<ReadFCvtF16ToI64, 0>;
+def : ReadAdvance<ReadFCvtF16ToF32, 0>;
+def : ReadAdvance<ReadFCvtF16ToI32, 0>;
+def : ReadAdvance<ReadFDiv16, 0>;
+def : ReadAdvance<ReadFCmp16, 0>;
+def : ReadAdvance<ReadFMA16, 0>;
+def : ReadAdvance<ReadFMinMax16, 0>;
+def : ReadAdvance<ReadFMul16, 0>;
+def : ReadAdvance<ReadFMovI16ToF16, 0>;
+def : ReadAdvance<ReadFMovF16ToI16, 0>;
+def : ReadAdvance<ReadFSGNJ16, 0>;
+def : ReadAdvance<ReadFSqrt16, 0>;
+} // Unsupported = true
+
+defm : UnsupportedSchedZba;
+defm : UnsupportedSchedZbb;
+
 include "RISCVSchedNotV.td"
+
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index f662912db99f249a713f73e53c8b4c75f028bdac..388e922797794508a5d20ac284f957cf4a624db9 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -45,8 +45,10 @@ def : WriteRes<WriteJmpReg, [SiFive7PipeB]>;
 let Latency = 3 in {
 def : WriteRes<WriteIALU, [SiFive7PipeAB]>;
 def : WriteRes<WriteIALU32, [SiFive7PipeAB]>;
-def : WriteRes<WriteShift, [SiFive7PipeAB]>;
-def : WriteRes<WriteShift32, [SiFive7PipeAB]>;
+def : WriteRes<WriteShiftImm, [SiFive7PipeAB]>;
+def : WriteRes<WriteShiftImm32, [SiFive7PipeAB]>;
+def : WriteRes<WriteShiftReg, [SiFive7PipeAB]>;
+def : WriteRes<WriteShiftReg32, [SiFive7PipeAB]>;
 }
 
 // Integer multiplication
@@ -101,8 +103,7 @@ def : WriteRes<WriteAtomicLDD, [SiFive7PipeA]>;
 let Latency = 5 in {
 def : WriteRes<WriteFALU32, [SiFive7PipeB]>;
 def : WriteRes<WriteFMul32, [SiFive7PipeB]>;
-def : WriteRes<WriteFMulAdd32, [SiFive7PipeB]>;
-def : WriteRes<WriteFMulSub32, [SiFive7PipeB]>;
+def : WriteRes<WriteFMA32, [SiFive7PipeB]>;
 }
 let Latency = 3 in {
 def : WriteRes<WriteFSGNJ32, [SiFive7PipeB]>;
@@ -118,8 +119,7 @@ def : WriteRes<WriteFSqrt32, [SiFive7PipeB, SiFive7FDiv]> { let Latency = 27;
 let Latency = 7 in {
 def : WriteRes<WriteFALU64, [SiFive7PipeB]>;
 def : WriteRes<WriteFMul64, [SiFive7PipeB]>;
-def : WriteRes<WriteFMulAdd64, [SiFive7PipeB]>;
-def : WriteRes<WriteFMulSub64, [SiFive7PipeB]>;
+def : WriteRes<WriteFMA64, [SiFive7PipeB]>;
 }
 let Latency = 3 in {
 def : WriteRes<WriteFSGNJ64, [SiFive7PipeB]>;
@@ -160,7 +160,6 @@ def : WriteRes<WriteNop, []>;
 
 def : InstRW<[WriteIALU], (instrs COPY)>;
 
-
 //===----------------------------------------------------------------------===//
 // Bypass and advance
 def : ReadAdvance<ReadJmp, 0>;
@@ -170,8 +169,10 @@ def : ReadAdvance<ReadStoreData, 0>;
 def : ReadAdvance<ReadMemBase, 0>;
 def : ReadAdvance<ReadIALU, 0>;
 def : ReadAdvance<ReadIALU32, 0>;
-def : ReadAdvance<ReadShift, 0>;
-def : ReadAdvance<ReadShift32, 0>;
+def : ReadAdvance<ReadShiftImm, 0>;
+def : ReadAdvance<ReadShiftImm32, 0>;
+def : ReadAdvance<ReadShiftReg, 0>;
+def : ReadAdvance<ReadShiftReg32, 0>;
 def : ReadAdvance<ReadIDiv, 0>;
 def : ReadAdvance<ReadIDiv32, 0>;
 def : ReadAdvance<ReadIMul, 0>;
@@ -188,11 +189,9 @@ def : ReadAdvance<ReadFMemBase, 0>;
 def : ReadAdvance<ReadFALU32, 0>;
 def : ReadAdvance<ReadFALU64, 0>;
 def : ReadAdvance<ReadFMul32, 0>;
-def : ReadAdvance<ReadFMulAdd32, 0>;
-def : ReadAdvance<ReadFMulSub32, 0>;
+def : ReadAdvance<ReadFMA32, 0>;
 def : ReadAdvance<ReadFMul64, 0>;
-def : ReadAdvance<ReadFMulAdd64, 0>;
-def : ReadAdvance<ReadFMulSub64, 0>;
+def : ReadAdvance<ReadFMA64, 0>;
 def : ReadAdvance<ReadFDiv32, 0>;
 def : ReadAdvance<ReadFDiv64, 0>;
 def : ReadAdvance<ReadFSqrt32, 0>;
@@ -220,5 +219,53 @@ def : ReadAdvance<ReadFMovI64ToF64, 0>;
 def : ReadAdvance<ReadFClass32, 0>;
 def : ReadAdvance<ReadFClass64, 0>;
 
+// Zfh is unsupported
+let Unsupported = true in {
+def : WriteRes<WriteFALU16, []>;
+def : WriteRes<WriteFClass16, []>;
+def : WriteRes<WriteFCvtF16ToF64, []>;
+def : WriteRes<WriteFCvtF64ToF16, []>;
+def : WriteRes<WriteFCvtI64ToF16, []>;
+def : WriteRes<WriteFCvtF32ToF16, []>;
+def : WriteRes<WriteFCvtI32ToF16, []>;
+def : WriteRes<WriteFCvtF16ToI64, []>;
+def : WriteRes<WriteFCvtF16ToF32, []>;
+def : WriteRes<WriteFCvtF16ToI32, []>;
+def : WriteRes<WriteFDiv16, []>;
+def : WriteRes<WriteFCmp16, []>;
+def : WriteRes<WriteFLD16, []>;
+def : WriteRes<WriteFMA16, []>;
+def : WriteRes<WriteFMinMax16, []>;
+def : WriteRes<WriteFMul16, []>;
+def : WriteRes<WriteFMovI16ToF16, []>;
+def : WriteRes<WriteFMovF16ToI16, []>;
+def : WriteRes<WriteFSGNJ16, []>;
+def : WriteRes<WriteFST16, []>;
+def : WriteRes<WriteFSqrt16, []>;
+
+def : ReadAdvance<ReadFALU16, 0>;
+def : ReadAdvance<ReadFClass16, 0>;
+def : ReadAdvance<ReadFCvtF16ToF64, 0>;
+def : ReadAdvance<ReadFCvtF64ToF16, 0>;
+def : ReadAdvance<ReadFCvtI64ToF16, 0>;
+def : ReadAdvance<ReadFCvtF32ToF16, 0>;
+def : ReadAdvance<ReadFCvtI32ToF16, 0>;
+def : ReadAdvance<ReadFCvtF16ToI64, 0>;
+def : ReadAdvance<ReadFCvtF16ToF32, 0>;
+def : ReadAdvance<ReadFCvtF16ToI32, 0>;
+def : ReadAdvance<ReadFDiv16, 0>;
+def : ReadAdvance<ReadFCmp16, 0>;
+def : ReadAdvance<ReadFMA16, 0>;
+def : ReadAdvance<ReadFMinMax16, 0>;
+def : ReadAdvance<ReadFMul16, 0>;
+def : ReadAdvance<ReadFMovI16ToF16, 0>;
+def : ReadAdvance<ReadFMovF16ToI16, 0>;
+def : ReadAdvance<ReadFSGNJ16, 0>;
+def : ReadAdvance<ReadFSqrt16, 0>;
+} // Unsupported = true
+
+defm : UnsupportedSchedZba;
+defm : UnsupportedSchedZbb;
+
 include "RISCVSchedNotV.td"
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedule.td b/llvm/lib/Target/RISCV/RISCVSchedule.td
index 0333cead094c71bc1f6af5dfa1bbf16ab490b3be..05d8d7ea1c0919617de496c846292c7710914d7c 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedule.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedule.td
@@ -9,8 +9,10 @@
 /// Define scheduler resources associated with def operands.
 def WriteIALU       : SchedWrite;    // 32 or 64-bit integer ALU operations
 def WriteIALU32     : SchedWrite;    // 32-bit integer ALU operations on RV64I
-def WriteShift32    : SchedWrite;    // 32-bit shift operations on RV64Ix
-def WriteShift      : SchedWrite;    // 32 or 64-bit shift operations
+def WriteShiftImm   : SchedWrite;    // 32 or 64-bit shift by immediate operations
+def WriteShiftImm32 : SchedWrite;    // 32-bit shift by immediate operations on RV64Ix
+def WriteShiftReg   : SchedWrite;    // 32 or 64-bit shift by immediate operations
+def WriteShiftReg32 : SchedWrite;    // 32-bit shift by immediate operations on RV64Ix
 def WriteIDiv       : SchedWrite;    // 32-bit or 64-bit divide and remainder
 def WriteIDiv32     : SchedWrite;    // 32-bit divide and remainder on RV64I
 def WriteIMul       : SchedWrite;    // 32-bit or 64-bit multiply
@@ -36,26 +38,33 @@ def WriteAtomicLDW  : SchedWrite;    // Atomic load word
 def WriteAtomicLDD  : SchedWrite;    // Atomic load double word
 def WriteAtomicSTW  : SchedWrite;    // Atomic store word
 def WriteAtomicSTD  : SchedWrite;    // Atomic store double word
+def WriteFALU16     : SchedWrite;    // FP 16-bit computation
 def WriteFALU32     : SchedWrite;    // FP 32-bit computation
 def WriteFALU64     : SchedWrite;    // FP 64-bit computation
+def WriteFMul16     : SchedWrite;    // 16-bit floating point multiply
+def WriteFMA16      : SchedWrite;    // 16-bit floating point fused multiply-add
 def WriteFMul32     : SchedWrite;    // 32-bit floating point multiply
-def WriteFMulAdd32  : SchedWrite;    // 32-bit floating point multiply add
-def WriteFMulSub32  : SchedWrite;    // 32-bit floating point multiply sub
+def WriteFMA32      : SchedWrite;    // 32-bit floating point fused multiply-add
 def WriteFMul64     : SchedWrite;    // 64-bit floating point multiply
-def WriteFMulAdd64  : SchedWrite;      // 64-bit floating point multiply add
-def WriteFMulSub64  : SchedWrite;    // 64-bit floating point multiply sub
+def WriteFMA64      : SchedWrite;    // 64-bit floating point fused multiply-add
+def WriteFDiv16     : SchedWrite;    // 16-bit floating point divide
 def WriteFDiv32     : SchedWrite;    // 32-bit floating point divide
 def WriteFDiv64     : SchedWrite;    // 64-bit floating point divide
+def WriteFSqrt16    : SchedWrite;    // 16-bit floating point sqrt
 def WriteFSqrt32    : SchedWrite;    // 32-bit floating point sqrt
 def WriteFSqrt64    : SchedWrite;    // 64-bit floating point sqrt
 
 // Integer to float conversions
+def WriteFCvtI32ToF16  : SchedWrite;
 def WriteFCvtI32ToF32  : SchedWrite;
 def WriteFCvtI32ToF64  : SchedWrite;
+def WriteFCvtI64ToF16  : SchedWrite;    // RV64I only
 def WriteFCvtI64ToF32  : SchedWrite;    // RV64I only
 def WriteFCvtI64ToF64  : SchedWrite;    // RV64I only
 
 //Float to integer conversions
+def WriteFCvtF16ToI32  : SchedWrite;
+def WriteFCvtF16ToI64  : SchedWrite;    // RV64I only
 def WriteFCvtF32ToI32  : SchedWrite;
 def WriteFCvtF32ToI64  : SchedWrite;    // RV64I only
 def WriteFCvtF64ToI32  : SchedWrite;
@@ -64,30 +73,56 @@ def WriteFCvtF64ToI64  : SchedWrite;    // RV64I only
 // Float to float conversions
 def WriteFCvtF32ToF64  : SchedWrite;
 def WriteFCvtF64ToF32  : SchedWrite;
+def WriteFCvtF16ToF32  : SchedWrite;
+def WriteFCvtF32ToF16  : SchedWrite;
+def WriteFCvtF16ToF64  : SchedWrite;
+def WriteFCvtF64ToF16  : SchedWrite;
 
-def WriteFConv32    : SchedWrite;    // 32-bit floating point convert
-def WriteFConv64    : SchedWrite;    // 64-bit floating point convert
+def WriteFClass16   : SchedWrite;    // 16-bit floating point classify
 def WriteFClass32   : SchedWrite;    // 32-bit floating point classify
 def WriteFClass64   : SchedWrite;    // 64-bit floating point classify
+def WriteFCmp16     : SchedWrite;    // 16-bit floating point compare
 def WriteFCmp32     : SchedWrite;    // 32-bit floating point compare
 def WriteFCmp64     : SchedWrite;    // 64-bit floating point compare
+def WriteFSGNJ16    : SchedWrite;    // 16-bit floating point sign-injection
 def WriteFSGNJ32    : SchedWrite;    // 32-bit floating point sign-injection
 def WriteFSGNJ64    : SchedWrite;    // 64-bit floating point sign-injection
+def WriteFMinMax16  : SchedWrite;    // 16-bit floating point min or max
 def WriteFMinMax32  : SchedWrite;    // 32-bit floating point min or max
 def WriteFMinMax64  : SchedWrite;    // 64-bit floating point min or max
 
+def WriteFMovF16ToI16     : SchedWrite;
+def WriteFMovI16ToF16     : SchedWrite;
 def WriteFMovF32ToI32     : SchedWrite;
 def WriteFMovI32ToF32     : SchedWrite;
 def WriteFMovF64ToI64     : SchedWrite;    // RV64I only
 def WriteFMovI64ToF64     : SchedWrite;    // RV64I only
 
-def WriteFMov32       : SchedWrite;    // 32-bit floating point move
-def WriteFMov64       : SchedWrite;    // 64-bit floating point move
+def WriteFLD16        : SchedWrite;    // Floating point sp load
 def WriteFLD32        : SchedWrite;    // Floating point sp load
 def WriteFLD64        : SchedWrite;    // Floating point dp load
+def WriteFST16        : SchedWrite;    // Floating point sp store
 def WriteFST32        : SchedWrite;    // Floating point sp store
 def WriteFST64        : SchedWrite;    // Floating point dp store
 
+// Zba extension
+def WriteSHXADD      : SchedWrite; // sh1add/sh2add/sh3add
+def WriteSHXADD32    : SchedWrite; // sh1add.uw/sh2add.uw/sh3add.uw
+
+// Zbb extension
+def WriteRotateImm   : SchedWrite;
+def WriteRotateImm32 : SchedWrite;
+def WriteRotateReg   : SchedWrite;
+def WriteRotateReg32 : SchedWrite;
+def WriteCLZ         : SchedWrite;
+def WriteCLZ32       : SchedWrite;
+def WriteCTZ         : SchedWrite;
+def WriteCTZ32       : SchedWrite;
+def WriteCPOP        : SchedWrite;
+def WriteCPOP32      : SchedWrite;
+def WriteREV8        : SchedWrite;
+def WriteORCB        : SchedWrite;
+
 /// Define scheduler resources associated with use operands.
 def ReadJmp         : SchedRead;
 def ReadJalr        : SchedRead;
@@ -97,8 +132,10 @@ def ReadFMemBase    : SchedRead;
 def ReadStoreData   : SchedRead;
 def ReadIALU        : SchedRead;
 def ReadIALU32      : SchedRead;    // 32-bit integer ALU operations on RV64I
-def ReadShift       : SchedRead;
-def ReadShift32     : SchedRead;    // 32-bit shift operations on RV64Ix
+def ReadShiftImm    : SchedRead;
+def ReadShiftImm32  : SchedRead;    // 32-bit shift by immediate operations on RV64Ix
+def ReadShiftReg    : SchedRead;
+def ReadShiftReg32  : SchedRead;    // 32-bit shift by register operations on RV64Ix
 def ReadIDiv        : SchedRead;
 def ReadIDiv32      : SchedRead;
 def ReadIMul        : SchedRead;
@@ -111,41 +148,116 @@ def ReadAtomicLDW   : SchedRead;    // Atomic load word
 def ReadAtomicLDD   : SchedRead;    // Atomic load double word
 def ReadAtomicSTW   : SchedRead;    // Atomic store word
 def ReadAtomicSTD   : SchedRead;    // Atomic store double word
+def ReadFALU16      : SchedRead;    // FP 16-bit computation
 def ReadFALU32      : SchedRead;    // FP 32-bit computation
 def ReadFALU64      : SchedRead;    // FP 64-bit computation
+def ReadFMul16      : SchedRead;    // 16-bit floating point multiply
+def ReadFMA16       : SchedRead;    // 16-bit floating point fused multiply-add
 def ReadFMul32      : SchedRead;    // 32-bit floating point multiply
-def ReadFMulAdd32   : SchedRead;    // 32-bit floating point multiply add
-def ReadFMulSub32   : SchedRead;    // 32-bit floating point multiply sub
+def ReadFMA32       : SchedRead;    // 32-bit floating point fused multiply-add
 def ReadFMul64      : SchedRead;    // 64-bit floating point multiply
-def ReadFMulAdd64   : SchedRead;    // 64-bit floating point multiply add
-def ReadFMulSub64   : SchedRead;    // 64-bit floating point multiply sub
+def ReadFMA64       : SchedRead;    // 64-bit floating point fused multiply-add
+def ReadFDiv16      : SchedRead;    // 16-bit floating point divide
 def ReadFDiv32      : SchedRead;    // 32-bit floating point divide
 def ReadFDiv64      : SchedRead;    // 64-bit floating point divide
+def ReadFSqrt16     : SchedRead;    // 16-bit floating point sqrt
 def ReadFSqrt32     : SchedRead;    // 32-bit floating point sqrt
 def ReadFSqrt64     : SchedRead;    // 64-bit floating point sqrt
+def ReadFCmp16      : SchedRead;
 def ReadFCmp32      : SchedRead;
 def ReadFCmp64      : SchedRead;
+def ReadFSGNJ16     : SchedRead;
 def ReadFSGNJ32     : SchedRead;
 def ReadFSGNJ64     : SchedRead;
+def ReadFMinMax16   : SchedRead;
 def ReadFMinMax32   : SchedRead;
 def ReadFMinMax64   : SchedRead;
+def ReadFCvtF16ToI32     : SchedRead;
+def ReadFCvtF16ToI64     : SchedRead;
 def ReadFCvtF32ToI32     : SchedRead;
 def ReadFCvtF32ToI64     : SchedRead;
 def ReadFCvtF64ToI32     : SchedRead;
 def ReadFCvtF64ToI64     : SchedRead;
+def ReadFCvtI32ToF16     : SchedRead;
 def ReadFCvtI32ToF32     : SchedRead;
 def ReadFCvtI32ToF64     : SchedRead;
+def ReadFCvtI64ToF16     : SchedRead;
 def ReadFCvtI64ToF32     : SchedRead;
 def ReadFCvtI64ToF64     : SchedRead;
+def ReadFMovF16ToI16     : SchedRead;
+def ReadFMovI16ToF16     : SchedRead;
 def ReadFMovF32ToI32     : SchedRead;
 def ReadFMovI32ToF32     : SchedRead;
 def ReadFMovF64ToI64     : SchedRead;
 def ReadFMovI64ToF64     : SchedRead;
 def ReadFCvtF32ToF64     : SchedRead;
 def ReadFCvtF64ToF32     : SchedRead;
+def ReadFCvtF16ToF32     : SchedRead;
+def ReadFCvtF32ToF16     : SchedRead;
+def ReadFCvtF16ToF64     : SchedRead;
+def ReadFCvtF64ToF16     : SchedRead;
+def ReadFClass16         : SchedRead;
 def ReadFClass32         : SchedRead;
 def ReadFClass64         : SchedRead;
 
+// Zba extension
+def ReadSHXADD      : SchedRead; // sh1add/sh2add/sh3add
+def ReadSHXADD32    : SchedRead; // sh1add.uw/sh2add.uw/sh3add.uw
+
+// Zbb extension
+def ReadRotateImm   : SchedRead;
+def ReadRotateImm32 : SchedRead;
+def ReadRotateReg   : SchedRead;
+def ReadRotateReg32 : SchedRead;
+def ReadCLZ         : SchedRead;
+def ReadCLZ32       : SchedRead;
+def ReadCTZ         : SchedRead;
+def ReadCTZ32       : SchedRead;
+def ReadCPOP        : SchedRead;
+def ReadCPOP32      : SchedRead;
+def ReadREV8        : SchedRead;
+def ReadORCB        : SchedRead;
+
+multiclass UnsupportedSchedZba {
+let Unsupported = true in {
+def : WriteRes<WriteSHXADD, []>;
+def : WriteRes<WriteSHXADD32, []>;
+
+def : ReadAdvance<ReadSHXADD, 0>;
+def : ReadAdvance<ReadSHXADD32, 0>;
+}
+}
+
+multiclass UnsupportedSchedZbb {
+let Unsupported = true in {
+def : WriteRes<WriteRotateImm, []>;
+def : WriteRes<WriteRotateImm32, []>;
+def : WriteRes<WriteRotateReg, []>;
+def : WriteRes<WriteRotateReg32, []>;
+def : WriteRes<WriteCLZ, []>;
+def : WriteRes<WriteCLZ32, []>;
+def : WriteRes<WriteCTZ, []>;
+def : WriteRes<WriteCTZ32, []>;
+def : WriteRes<WriteCPOP, []>;
+def : WriteRes<WriteCPOP32, []>;
+def : WriteRes<WriteREV8, []>;
+def : WriteRes<WriteORCB, []>;
+
+def : ReadAdvance<ReadRotateImm, 0>;
+def : ReadAdvance<ReadRotateImm32, 0>;
+def : ReadAdvance<ReadRotateReg, 0>;
+def : ReadAdvance<ReadRotateReg32, 0>;
+def : ReadAdvance<ReadCLZ, 0>;
+def : ReadAdvance<ReadCLZ32, 0>;
+def : ReadAdvance<ReadCTZ, 0>;
+def : ReadAdvance<ReadCTZ32, 0>;
+def : ReadAdvance<ReadCPOP, 0>;
+def : ReadAdvance<ReadCPOP32, 0>;
+def : ReadAdvance<ReadREV8, 0>;
+def : ReadAdvance<ReadORCB, 0>;
+}
+}
+
 //===----------------------------------------------------------------------===//
 // V-extension Scheduling.
 //===----------------------------------------------------------------------===//
@@ -172,5 +284,3 @@ def ReadVPUFALU       : SchedRead; // .vf
 def ReadVPUScalar     : SchedRead; // .vs
 def ReadVPUMem        : SchedRead;
 def ReadVPUStoreData  : SchedRead;
-
-
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
index f6463c02511434e4c9d892b05fa666d36e053db8..1606352208bd66daa767f219db2019e486049568 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
@@ -105,26 +105,31 @@ const RegisterBankInfo *RISCVSubtarget::getRegBankInfo() const {
   return RegBankInfo.get();
 }
 
+#define STRINGIZE_INTERNAL(arg) #arg
+#define STRINGIZE(arg) STRINGIZE_INTERNAL(arg)
+
+#define MINIMUM_VLEN 64
+
 unsigned RISCVSubtarget::getMaxRVVVectorSizeInBits() const {
   assert(hasStdExtV() && "Tried to get vector length without V support!");
   if (RVVVectorBitsMax == 0)
     return 0;
-  assert(RVVVectorBitsMax >= 128 && isPowerOf2_32(RVVVectorBitsMax) &&
-         "V extension requires vector length to be at least 128 and a power of "
+  assert(RVVVectorBitsMax >= MINIMUM_VLEN && isPowerOf2_32(RVVVectorBitsMax) &&
+         "V extension requires vector length to be at least " STRINGIZE(MINIMUM_VLEN) " and a power of "
          "2!");
   assert(RVVVectorBitsMax >= RVVVectorBitsMin &&
          "Minimum V extension vector length should not be larger than its "
          "maximum!");
   unsigned Max = std::max(RVVVectorBitsMin, RVVVectorBitsMax);
-  return PowerOf2Floor(Max < 128 ? 0 : Max);
+  return PowerOf2Floor(Max < MINIMUM_VLEN ? 0 : Max);
 }
 
 unsigned RISCVSubtarget::getMinRVVVectorSizeInBits() const {
   assert(hasStdExtV() &&
          "Tried to get vector length without V extension support!");
   assert((RVVVectorBitsMin == 0 ||
-          (RVVVectorBitsMin >= 128 && isPowerOf2_32(RVVVectorBitsMin))) &&
-         "V extension requires vector length to be at least 128 and a power of "
+          (RVVVectorBitsMin >= MINIMUM_VLEN && isPowerOf2_32(RVVVectorBitsMin))) &&
+         "V extension requires vector length to be at least " STRINGIZE(MINIMUM_VLEN) " and a power of "
          "2!");
   assert((RVVVectorBitsMax >= RVVVectorBitsMin || RVVVectorBitsMax == 0) &&
          "Minimum V extension vector length should not be larger than its "
@@ -132,7 +137,7 @@ unsigned RISCVSubtarget::getMinRVVVectorSizeInBits() const {
   unsigned Min = RVVVectorBitsMin;
   if (RVVVectorBitsMax != 0)
     Min = std::min(RVVVectorBitsMin, RVVVectorBitsMax);
-  return PowerOf2Floor(Min < 128 ? 0 : Min);
+  return PowerOf2Floor(Min < MINIMUM_VLEN ? 0 : Min);
 }
 
 unsigned RISCVSubtarget::getMaxLMULForFixedLengthVectors() const {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 24725a9c6ea027fb1045db7193c03a7299e4165d..8f1bc1536e932ea887e7bd61ae8eed8136d63f78 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -167,7 +167,7 @@ unsigned RISCVTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
 }
 
 unsigned RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
-                                      int Index, VectorType *SubTp) {
+                                      ArrayRef<int> Mask, int Index, VectorType *SubTp) {
   if (isa<ScalableVectorType>(Tp) &&
       (!SubTp || isa<ScalableVectorType>(SubTp))) {
     switch (Kind) {
@@ -187,7 +187,7 @@ unsigned RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                         cast<ScalableVectorType>(SubTp));
     }
   }
-  return BaseT::getShuffleCost(Kind, Tp, Index, SubTp);
+  return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
 }
 
 /// Estimate the overhead of scalarizing an instructions unique
@@ -260,17 +260,6 @@ unsigned RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
   return LegalizationFactor * Log2_32(BitRatio);
 }
 
-unsigned RISCVTTIImpl::getRegisterBitWidth(bool Vector) const {
-  if (!Vector)
-    return ST->is64Bit() ? 64 : 32;
-
-  // Largest vector register type will be `vscale * 8 * 64` bits for LMUL = 8
-  // (largest LMUL value). Since vscale is unknown at compile time, the largest
-  // possible register (register-group to be precise) bit width will be at least
-  // `64 * 8`.
-  return ST->hasStdExtV() ? getMinVectorRegisterBitWidth() * 8 : 0;
-}
-
 bool RISCVTTIImpl::shouldMaximizeVectorBandwidth(bool OptSize) const {
   return (ST->hasStdExtV() && true);
 }
@@ -281,12 +270,6 @@ unsigned RISCVTTIImpl::getMinVectorRegisterBitWidth() const {
   return ST->hasStdExtV() ? getMaxElementWidth() : 0;
 }
 
-unsigned RISCVTTIImpl::getVectorRegisterBitWidth(unsigned WidthFactor) const {
-  assert(WidthFactor <= 8 && isPowerOf2_32(WidthFactor) &&
-         "Possible RISC-V LMUL values are 1, 2, 4 and 8.");
-  return ST->hasStdExtV() ? getMinVectorRegisterBitWidth() * WidthFactor : 0;
-}
-
 ElementCount RISCVTTIImpl::getMinimumVF(unsigned ElemWidth,
                                         bool IsScalable) const {
   return ST->hasStdExtV() && IsScalable
@@ -297,10 +280,9 @@ ElementCount RISCVTTIImpl::getMinimumVF(unsigned ElemWidth,
              : ElementCount::getNull();
 }
 
-unsigned RISCVTTIImpl::getVectorRegisterUsage(unsigned VFKnownMin,
-                                              unsigned ElementTypeSize,
-                                              unsigned SafeDepDist) const {
-
+unsigned RISCVTTIImpl::getVectorRegisterUsage(
+    TargetTransformInfo::RegisterKind K, unsigned VFKnownMin,
+    unsigned ElementTypeSize, unsigned SafeDepDist) const {
   // FIXME: For the time being we assume dependency distance is always safe.
   // Once we have dependency distance computations for scalable vectors, we need
   // to figure out its relationship with register group usage;
@@ -309,7 +291,8 @@ unsigned RISCVTTIImpl::getVectorRegisterUsage(unsigned VFKnownMin,
 }
 
 std::pair<ElementCount, ElementCount>
-RISCVTTIImpl::getFeasibleMaxVFRange(unsigned SmallestType, unsigned WidestType,
+RISCVTTIImpl::getFeasibleMaxVFRange(TargetTransformInfo::RegisterKind K,
+                                    unsigned SmallestType, unsigned WidestType,
                                     unsigned MaxSafeRegisterWidth,
                                     unsigned RegWidthFactor) const {
   // check for SEW <= ELEN in the base ISA
@@ -319,8 +302,9 @@ RISCVTTIImpl::getFeasibleMaxVFRange(unsigned SmallestType, unsigned WidestType,
   // valid range of VFs.
   SmallestType = std::max<unsigned>(8, SmallestType);
   WidestType = std::max<unsigned>(8, WidestType);
-  unsigned WidestRegister =
-      std::min(getVectorRegisterBitWidth(RegWidthFactor), MaxSafeRegisterWidth);
+  unsigned WidestRegister = std::min<unsigned>(
+      getMinVectorRegisterBitWidth() * RegWidthFactor,
+      MaxSafeRegisterWidth);
   unsigned SmallestRegister =
       std::min(getMinVectorRegisterBitWidth(), MaxSafeRegisterWidth);
   bool IsScalable = useScalableVectorType();
@@ -354,16 +338,10 @@ int RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
   return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
 }
 
-unsigned RISCVTTIImpl::getGatherScatterOpCost(
-    unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
-    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
-  // We can do gather/scatter using a single instruction.
-  // FIXME: The actual cost is likely to be higher than that.
-  if (isa<ScalableVectorType>(DataTy))
-    return 1;
-
-  return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
-                                       Alignment, CostKind, I);
+TargetTransformInfo::PopcntSupportKind
+RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) {
+  assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
+  return ST->hasStdExtZbb() ? TTI::PSK_FastHardware : TTI::PSK_Software;
 }
 
 bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
@@ -471,3 +449,33 @@ int RISCVTTIImpl::getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy,
 
   return LegalizationCost + /*Cost of horizontal reduction*/ 2;
 }
+
+unsigned RISCVTTIImpl::getGatherScatterOpCost(
+    unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
+    Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
+  // We can do gather/scatter using a single instruction.
+  // FIXME: The actual cost is likely to be higher than that.
+  if (isa<ScalableVectorType>(DataTy))
+    return 1;
+
+  if (CostKind != TTI::TCK_RecipThroughput)
+    return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
+                                         Alignment, CostKind, I);
+
+  if ((Opcode == Instruction::Load &&
+       !isLegalMaskedGather(DataTy, Align(Alignment))) ||
+      (Opcode == Instruction::Store &&
+       !isLegalMaskedScatter(DataTy, Align(Alignment))))
+    return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
+                                         Alignment, CostKind, I);
+
+  if (!isa<FixedVectorType>(DataTy))
+    return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
+                                         Alignment, CostKind, I);
+
+  auto *VTy = cast<FixedVectorType>(DataTy);
+  unsigned NumLoads = VTy->getNumElements();
+  unsigned MemOpCost =
+      getMemoryOpCost(Opcode, VTy->getElementType(), Alignment, 0, CostKind, I);
+  return NumLoads * MemOpCost;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 38a77812b8942dc8021981eb1c8efcc4aac518d4..391ad88fa4a5d5501cb7aef97c008d4df693a2a3 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -150,8 +150,8 @@ public:
   bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) const;
   bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) const;
   unsigned getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index);
-  unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
-                          VectorType *SubTp);
+  unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
+                          ArrayRef<int> Mask, int Index, VectorType *SubTp);
   unsigned getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
                                             ArrayRef<Type *> Tys);
   unsigned getScalarizationOverhead(VectorType *InTy, const APInt &DemandedElts,
@@ -160,15 +160,15 @@ public:
                             TTI::CastContextHint CCH,
                             TTI::TargetCostKind CostKind,
                             const Instruction *I = nullptr);
-  unsigned getRegisterBitWidth(bool Vector) const;
   bool shouldMaximizeVectorBandwidth(bool OptSize) const;
   unsigned getMinVectorRegisterBitWidth() const;
-  unsigned getVectorRegisterBitWidth(unsigned WidthFactor) const;
   ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const;
-  unsigned getVectorRegisterUsage(unsigned VFKnownMin, unsigned ElementTypeSize,
+  unsigned getVectorRegisterUsage(TargetTransformInfo::RegisterKind K,
+                                  unsigned VFKnownMin, unsigned ElementTypeSize,
                                   unsigned SafeDepDist) const;
   std::pair<ElementCount, ElementCount>
-  getFeasibleMaxVFRange(unsigned SmallestType, unsigned WidestType,
+  getFeasibleMaxVFRange(TargetTransformInfo::RegisterKind K,
+                        unsigned SmallestType, unsigned WidestType,
                         unsigned MaxSafeRegisterWidth = -1U,
                         unsigned RegWidthFactor = 1) const;
   int getCmpSelInstrCost(
@@ -177,10 +177,7 @@ public:
       TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
       const Instruction *I = nullptr);
 
-  unsigned getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
-                                  const Value *Ptr, bool VariableMask,
-                                  Align Alignment, TTI::TargetCostKind CostKind,
-                                  const Instruction *I = nullptr);
+  TargetTransformInfo::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
 
   bool shouldExpandReduction(const IntrinsicInst *II) const;
   bool supportsScalableVectors() const { return ST->hasStdExtV(); }
@@ -191,6 +188,79 @@ public:
   int getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy,
                              bool IsPairwise, bool IsUnsigned,
                              TTI::TargetCostKind CostKind);
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+    switch (K) {
+    case TargetTransformInfo::RGK_Scalar:
+      return TypeSize::getFixed(ST->getXLen());
+    case TargetTransformInfo::RGK_FixedWidthVector:
+      return TypeSize::getFixed(
+          ST->hasStdExtV() ? ST->getMinRVVVectorSizeInBits() : 0);
+    case TargetTransformInfo::RGK_ScalableVector:
+      return TypeSize::getScalable(
+          ST->hasStdExtV() ? ST->getMinRVVVectorSizeInBits() : 0);
+    }
+
+    llvm_unreachable("Unsupported register kind");
+  }
+
+  unsigned getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
+                                  const Value *Ptr, bool VariableMask,
+                                  Align Alignment, TTI::TargetCostKind CostKind,
+                                  const Instruction *I);
+
+  bool isLegalElementTypeForRVV(Type *ScalarTy) {
+    if (ScalarTy->isPointerTy())
+      return true;
+
+    if (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
+        ScalarTy->isIntegerTy(32) || ScalarTy->isIntegerTy(64))
+      return true;
+
+    if (ScalarTy->isHalfTy())
+      return ST->hasStdExtZfh();
+    if (ScalarTy->isFloatTy())
+      return ST->hasStdExtF();
+    if (ScalarTy->isDoubleTy())
+      return ST->hasStdExtD();
+
+    return false;
+  }
+
+  bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) {
+    if (!ST->hasStdExtV())
+      return false;
+
+    // Only support fixed vectors if we know the minimum vector size.
+    if (isa<FixedVectorType>(DataType) && ST->getMinRVVVectorSizeInBits() == 0)
+      return false;
+
+    return isLegalElementTypeForRVV(DataType->getScalarType());
+  }
+
+  bool isLegalMaskedLoad(Type *DataType, Align Alignment) {
+    return isLegalMaskedLoadStore(DataType, Alignment);
+  }
+  bool isLegalMaskedStore(Type *DataType, Align Alignment) {
+    return isLegalMaskedLoadStore(DataType, Alignment);
+  }
+
+  bool isLegalMaskedGatherScatter(Type *DataType, Align Alignment) {
+    if (!ST->hasStdExtV())
+      return false;
+
+    // Only support fixed vectors if we know the minimum vector size.
+    if (isa<FixedVectorType>(DataType) && ST->getMinRVVVectorSizeInBits() == 0)
+      return false;
+
+    return isLegalElementTypeForRVV(DataType->getScalarType());
+  }
+
+  bool isLegalMaskedGather(Type *DataType, Align Alignment) {
+    return isLegalMaskedGatherScatter(DataType, Alignment);
+  }
+  bool isLegalMaskedScatter(Type *DataType, Align Alignment) {
+    return isLegalMaskedGatherScatter(DataType, Alignment);
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
index c537020cdee81fbc43aa3585cbfa91012c006c9d..8c4567cd1c4ed17bf389203e035a9169eb10280e 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
@@ -21,7 +21,8 @@ SystemZMCAsmInfo::SystemZMCAsmInfo(const Triple &TT) {
 
   MaxInstLength = 6;
 
-  CommentString = "#";
+  CommentString = AssemblerDialect == AD_HLASM ? "*" : "#";
+  RestrictCommentStringToStartOfStatement = (AssemblerDialect == AD_HLASM);
   ZeroDirective = "\t.space\t";
   Data64bitsDirective = "\t.quad\t";
   UsesELFSectionDirectiveForBSS = true;
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 535f164baf285f08c0641b3c21fbef979e2ba0f5..1274dab7c0253db4be2fd3cbde12b7e9adde970c 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -323,12 +323,18 @@ unsigned SystemZTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
   return 0;
 }
 
-unsigned SystemZTTIImpl::getRegisterBitWidth(bool Vector) const {
-  if (!Vector)
-    return 64;
-  if (ST->hasVector())
-    return 128;
-  return 0;
+TypeSize
+SystemZTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+  switch (K) {
+  case TargetTransformInfo::RGK_Scalar:
+    return TypeSize::getFixed(64);
+  case TargetTransformInfo::RGK_FixedWidthVector:
+    return TypeSize::getFixed(ST->hasVector() ? 128 : 0);
+  case TargetTransformInfo::RGK_ScalableVector:
+    return TypeSize::getScalable(0);
+  }
+
+  llvm_unreachable("Unsupported register kind");
 }
 
 unsigned SystemZTTIImpl::getMinPrefetchStride(unsigned NumMemAccesses,
@@ -548,7 +554,8 @@ int SystemZTTIImpl::getArithmeticInstrCost(
 }
 
 int SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
-                                   int Index, VectorType *SubTp) {
+                                   ArrayRef<int> Mask, int Index,
+                                   VectorType *SubTp) {
   if (ST->hasVector()) {
     unsigned NumVectors = getNumVectorRegs(Tp);
 
@@ -581,7 +588,7 @@ int SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
     }
   }
 
-  return BaseT::getShuffleCost(Kind, Tp, Index, SubTp);
+  return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
 }
 
 // Return the log2 difference of the element sizes of the two vector types.
@@ -749,8 +756,13 @@ int SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     }
   }
   else if (ST->hasVector()) {
+    // Vector to scalar cast.
     auto *SrcVecTy = cast<FixedVectorType>(Src);
-    auto *DstVecTy = cast<FixedVectorType>(Dst);
+    auto *DstVecTy = dyn_cast<FixedVectorType>(Dst);
+    if (!DstVecTy) {
+      // TODO: tune vector-to-scalar cast.
+      return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
+    }
     unsigned VF = SrcVecTy->getNumElements();
     unsigned NumDstVectors = getNumVectorRegs(Dst);
     unsigned NumSrcVectors = getNumVectorRegs(Src);
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index c97e099f99435f2a3037167c69851153d4642e81..0fe50c2efd161657b0c1a19018c77e2c3c7a669f 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -62,7 +62,7 @@ public:
   /// @{
 
   unsigned getNumberOfRegisters(unsigned ClassID) const;
-  unsigned getRegisterBitWidth(bool Vector) const;
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
 
   unsigned getCacheLineSize() const override { return 256; }
   unsigned getPrefetchDistance() const override { return 4500; }
@@ -87,8 +87,8 @@ public:
       TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None,
       ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
       const Instruction *CxtI = nullptr);
-  int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
-                     VectorType *SubTp);
+  int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask,
+                     int Index, VectorType *SubTp);
   unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy);
   unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy);
   unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
diff --git a/llvm/lib/Target/VE/VEInstrInfo.td b/llvm/lib/Target/VE/VEInstrInfo.td
index b6862cf7b30d5ae86e4e836fc8ebfbba96d00aeb..2f77daae713049d01cba3c338ae792f6311e59d0 100644
--- a/llvm/lib/Target/VE/VEInstrInfo.td
+++ b/llvm/lib/Target/VE/VEInstrInfo.td
@@ -793,7 +793,7 @@ multiclass PFCHm<string opcStr, bits<8>opc> {
 let Constraints = "$dest = $sd", DisableEncoding = "$sd",
     mayStore=1, mayLoad = 1, hasSideEffects = 0 in
 multiclass RRCAStgm<string opcStr, bits<8>opc, RegisterClass RC, ValueType Ty,
-                    Operand immOp, Operand MEM, Operand ADDR,
+                    Operand immOp, Operand MEM, ComplexPattern ADDR,
                     SDPatternOperator OpNode = null_frag> {
   def r : RRM<opc, (outs RC:$dest), (ins MEM:$addr, RC:$sy, RC:$sd),
               !strconcat(opcStr, " $dest, $addr, $sy"),
@@ -1719,10 +1719,10 @@ def : Pat<(i64 (anyext i32:$sy)),
 
 // extload, sextload and zextload stuff
 multiclass EXT64m<SDPatternOperator from,
-                  SDPatternOperator torri,
-                  SDPatternOperator torii,
-                  SDPatternOperator tozri,
-                  SDPatternOperator tozii> {
+                  RM torri,
+                  RM torii,
+                  RM tozri,
+                  RM tozii> {
   def : Pat<(i64 (from ADDRrri:$addr)),
             (INSERT_SUBREG (i64 (IMPLICIT_DEF)), (torri MEMrri:$addr),
                            sub_i32)>;
@@ -1748,10 +1748,10 @@ defm : EXT64m<extloadi32, LDLSXrri, LDLSXrii, LDLSXzri, LDLSXzii>;
 
 // anyextload
 multiclass EXT32m<SDPatternOperator from,
-                  SDPatternOperator torri,
-                  SDPatternOperator torii,
-                  SDPatternOperator tozri,
-                  SDPatternOperator tozii> {
+                  RM torri,
+                  RM torii,
+                  RM tozri,
+                  RM tozii> {
   def : Pat<(from ADDRrri:$addr), (torri MEMrri:$addr)>;
   def : Pat<(from ADDRrii:$addr), (torii MEMrii:$addr)>;
   def : Pat<(from ADDRzri:$addr), (tozri MEMzri:$addr)>;
@@ -1762,10 +1762,10 @@ defm : EXT32m<extloadi16, LD2BZXrri, LD2BZXrii, LD2BZXzri, LD2BZXzii>;
 
 // truncstore
 multiclass TRUNC64m<SDPatternOperator from,
-                    SDPatternOperator torri,
-                    SDPatternOperator torii,
-                    SDPatternOperator tozri,
-                    SDPatternOperator tozii> {
+                    RM torri,
+                    RM torii,
+                    RM tozri,
+                    RM tozii> {
   def : Pat<(from i64:$src, ADDRrri:$addr),
             (torri MEMrri:$addr, (EXTRACT_SUBREG $src, sub_i32))>;
   def : Pat<(from i64:$src, ADDRrii:$addr),
@@ -1781,8 +1781,8 @@ defm : TRUNC64m<truncstorei32, STLrri, STLrii, STLzri, ST1Bzii>;
 
 // Atomic loads
 multiclass ATMLDm<SDPatternOperator from,
-                  SDPatternOperator torri, SDPatternOperator torii,
-                  SDPatternOperator tozri, SDPatternOperator tozii> {
+                  RM torri, RM torii,
+                  RM tozri, RM tozii> {
   def : Pat<(from ADDRrri:$addr), (torri MEMrri:$addr)>;
   def : Pat<(from ADDRrii:$addr), (torii MEMrii:$addr)>;
   def : Pat<(from ADDRzri:$addr), (tozri MEMzri:$addr)>;
@@ -1794,9 +1794,9 @@ defm : ATMLDm<atomic_load_32, LDLZXrri, LDLZXrii, LDLZXzri, LDLZXzii>;
 defm : ATMLDm<atomic_load_64, LDrri, LDrii, LDzri, LDzii>;
 
 // Optimized atomic loads with sext
-multiclass SXATMLDm<SDPatternOperator from, Operand TY,
-                    SDPatternOperator torri, SDPatternOperator torii,
-                    SDPatternOperator tozri, SDPatternOperator tozii> {
+multiclass SXATMLDm<SDPatternOperator from, ValueType TY,
+                    RM torri, RM torii,
+                    RM tozri, RM tozii> {
   def : Pat<(i64 (sext_inreg (i64 (anyext (from ADDRrri:$addr))), TY)),
             (i2l (torri MEMrri:$addr))>;
   def : Pat<(i64 (sext_inreg (i64 (anyext (from ADDRrii:$addr))), TY)),
@@ -1807,8 +1807,8 @@ multiclass SXATMLDm<SDPatternOperator from, Operand TY,
             (i2l (tozii MEMzii:$addr))>;
 }
 multiclass SXATMLD32m<SDPatternOperator from,
-                      SDPatternOperator torri, SDPatternOperator torii,
-                      SDPatternOperator tozri, SDPatternOperator tozii> {
+                      RM torri, RM torii,
+                      RM tozri, RM tozii> {
   def : Pat<(i64 (sext (from ADDRrri:$addr))),
             (i2l (torri MEMrri:$addr))>;
   def : Pat<(i64 (sext (from ADDRrii:$addr))),
@@ -1824,9 +1824,9 @@ defm : SXATMLDm<atomic_load_16, i16, LD2BSXrri, LD2BSXrii, LD2BSXzri,
 defm : SXATMLD32m<atomic_load_32, LDLSXrri, LDLSXrii, LDLSXzri, LDLSXzii>;
 
 // Optimized atomic loads with zext
-multiclass ZXATMLDm<SDPatternOperator from, Operand VAL,
-                    SDPatternOperator torri, SDPatternOperator torii,
-                    SDPatternOperator tozri, SDPatternOperator tozii> {
+multiclass ZXATMLDm<SDPatternOperator from, int VAL,
+                    RM torri, RM torii,
+                    RM tozri, RM tozii> {
   def : Pat<(i64 (and (anyext (from ADDRrri:$addr)), VAL)),
             (i2l (torri MEMrri:$addr))>;
   def : Pat<(i64 (and (anyext (from ADDRrii:$addr)), VAL)),
@@ -1836,9 +1836,9 @@ multiclass ZXATMLDm<SDPatternOperator from, Operand VAL,
   def : Pat<(i64 (and (anyext (from ADDRzii:$addr)), VAL)),
             (i2l (tozii MEMzii:$addr))>;
 }
-multiclass ZXATMLD32m<SDPatternOperator from, Operand VAL,
-                      SDPatternOperator torri, SDPatternOperator torii,
-                      SDPatternOperator tozri, SDPatternOperator tozii> {
+multiclass ZXATMLD32m<SDPatternOperator from, int VAL,
+                      RM torri, RM torii,
+                      RM tozri, RM tozii> {
   def : Pat<(i64 (zext (from ADDRrri:$addr))),
             (i2l (torri MEMrri:$addr))>;
   def : Pat<(i64 (zext (from ADDRrii:$addr))),
@@ -1857,8 +1857,8 @@ defm : ZXATMLD32m<atomic_load_32, 0xFFFFFFFF, LDLZXrri, LDLZXrii, LDLZXzri,
 
 // Atomic stores
 multiclass ATMSTm<SDPatternOperator from, ValueType ty,
-                  SDPatternOperator torri, SDPatternOperator torii,
-                  SDPatternOperator tozri, SDPatternOperator tozii> {
+                  RM torri, RM torii,
+                  RM tozri, RM tozii> {
   def : Pat<(from ADDRrri:$addr, ty:$src), (torri MEMrri:$addr, $src)>;
   def : Pat<(from ADDRrii:$addr, ty:$src), (torii MEMrii:$addr, $src)>;
   def : Pat<(from ADDRzri:$addr, ty:$src), (tozri MEMzri:$addr, $src)>;
@@ -1872,10 +1872,10 @@ defm : ATMSTm<atomic_store_64, i64, STrri, STrii, STzri, STzii>;
 // Optimized atomic stores with truncate
 multiclass TRATMSTm<SDPatternOperator from,
                   ValueType ty,
-                  SDPatternOperator torri,
-                  SDPatternOperator torii,
-                  SDPatternOperator tozri,
-                  SDPatternOperator tozii> {
+                  RM torri,
+                  RM torii,
+                  RM tozri,
+                  RM tozii> {
   def : Pat<(from ADDRrri:$addr, (i32 (trunc i64:$src))),
             (torri MEMrri:$addr, (EXTRACT_SUBREG $src, sub_i32))>;
   def : Pat<(from ADDRrii:$addr, (i32 (trunc i64:$src))),
@@ -1929,10 +1929,10 @@ def : Pat<(br bb:$addr), (BRCFLa bb:$addr)>;
 
 // brcc
 // integer brcc
-multiclass BRCCIm<ValueType ty, SDPatternOperator BrOpNode1,
-                 SDPatternOperator BrOpNode2,
-                 SDPatternOperator CmpOpNode1,
-                 SDPatternOperator CmpOpNode2> {
+multiclass BRCCIm<ValueType ty, CF BrOpNode1,
+                 CF BrOpNode2,
+                 RR CmpOpNode1,
+                 RR CmpOpNode2> {
   def : Pat<(brcc CCSIOp:$cond, ty:$l, simm7:$r, bb:$addr),
             (BrOpNode2 (icond2ccSwap $cond), (LO7 $r), $l, bb:$addr)>;
   def : Pat<(brcc CCSIOp:$cond, ty:$l, ty:$r, bb:$addr),
@@ -1947,8 +1947,7 @@ defm : BRCCIm<i32, BRCFWrr, BRCFWir, CMPUWrr, CMPUWir>;
 defm : BRCCIm<i64, BRCFLrr, BRCFLir, CMPULrr, CMPULir>;
 
 // floating point brcc
-multiclass BRCCFm<ValueType ty, SDPatternOperator BrOpNode1,
-                 SDPatternOperator BrOpNode2> {
+multiclass BRCCFm<ValueType ty, CF BrOpNode1, CF BrOpNode2> {
   def : Pat<(brcc cond:$cond, ty:$l, simm7fp:$r, bb:$addr),
             (BrOpNode2 (fcond2ccSwap $cond), (LO7FP $r), $l, bb:$addr)>;
   def : Pat<(brcc cond:$cond, ty:$l, ty:$r, bb:$addr),
diff --git a/llvm/lib/Target/VE/VEInstrPatternsVec.td b/llvm/lib/Target/VE/VEInstrPatternsVec.td
index 0084876f9f1b9c31e8ccef0c158dbb949faf990d..dc3c913c918afb3f34c298ecc4a45083f8cbb815 100644
--- a/llvm/lib/Target/VE/VEInstrPatternsVec.td
+++ b/llvm/lib/Target/VE/VEInstrPatternsVec.td
@@ -16,7 +16,7 @@
 //===----------------------------------------------------------------------===//
 
 multiclass vbrd_elem32<ValueType v32, ValueType s32, SDPatternOperator ImmOp,
-                       SDNodeXForm ImmCast, SDNodeXForm SuperRegCast> {
+                       SDNodeXForm ImmCast, OutPatFrag SuperRegCast> {
   // VBRDil
   def : Pat<(v32 (vec_broadcast (s32 ImmOp:$sy), i32:$vl)),
             (VBRDil (ImmCast $sy), i32:$vl)>;
@@ -38,8 +38,8 @@ multiclass vbrd_elem64<ValueType v64, ValueType s64,
 }
 
 multiclass extract_insert_elem32<ValueType v32, ValueType s32,
-                                 SDNodeXForm SubRegCast,
-                                 SDNodeXForm SuperRegCast> {
+                                 OutPatFrag SubRegCast,
+                                 OutPatFrag SuperRegCast> {
   // LVSvi
   def: Pat<(s32 (extractelt v32:$vec, uimm7:$idx)),
            (SubRegCast (LVSvi v32:$vec, (ULO7 $idx)))>;
@@ -73,7 +73,7 @@ multiclass extract_insert_elem64<ValueType v64, ValueType s64> {
 
 multiclass patterns_elem32<ValueType v32, ValueType s32,
                            SDPatternOperator ImmOp, SDNodeXForm ImmCast,
-                           SDNodeXForm SubRegCast, SDNodeXForm SuperRegCast> {
+                           OutPatFrag SubRegCast, OutPatFrag SuperRegCast> {
   defm : vbrd_elem32<v32, s32, ImmOp, ImmCast, SuperRegCast>;
   defm : extract_insert_elem32<v32, s32, SubRegCast, SuperRegCast>;
 }
diff --git a/llvm/lib/Target/VE/VETargetTransformInfo.h b/llvm/lib/Target/VE/VETargetTransformInfo.h
index 68af665974851cf3d7fa39cb2ef8d88b90d867f0..6730c43258d22140b791863d42f24eeb94260db5 100644
--- a/llvm/lib/Target/VE/VETargetTransformInfo.h
+++ b/llvm/lib/Target/VE/VETargetTransformInfo.h
@@ -50,12 +50,18 @@ public:
     return 64;
   }
 
-  unsigned getRegisterBitWidth(bool Vector) const {
-    if (Vector) {
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
+    switch (K) {
+    case TargetTransformInfo::RGK_Scalar:
+      return TypeSize::getFixed(64);
+    case TargetTransformInfo::RGK_FixedWidthVector:
       // TODO report vregs once vector isel is stable.
-      return 0;
+      return TypeSize::getFixed(0);
+    case TargetTransformInfo::RGK_ScalableVector:
+      return TypeSize::getScalable(0);
     }
-    return 64;
+
+    llvm_unreachable("Unsupported register kind");
   }
 
   unsigned getMinVectorRegisterBitWidth() const {
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index c1706735115491627d783a89df7662692adf6cfd..d57c7ec17187bc430703c08f17bfd55c5c9d0970 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -169,6 +169,11 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
   }
 };
 
+// Perhaps this should go somewhere common.
+static wasm::WasmLimits DefaultLimits() {
+  return {wasm::WASM_LIMITS_FLAG_NONE, 0, 0};
+}
+
 static MCSymbolWasm *GetOrCreateFunctionTableSymbol(MCContext &Ctx,
                                                     const StringRef &Name) {
   MCSymbolWasm *Sym = cast_or_null<MCSymbolWasm>(Ctx.lookupSymbol(Name));
@@ -487,6 +492,28 @@ public:
         WebAssemblyOperand::IntOp{static_cast<int64_t>(BT)}));
   }
 
+  bool parseLimits(wasm::WasmLimits *Limits) {
+    auto Tok = Lexer.getTok();
+    if (!Tok.is(AsmToken::Integer))
+      return error("Expected integer constant, instead got: ", Tok);
+    int64_t Val = Tok.getIntVal();
+    assert(Val >= 0);
+    Limits->Minimum = Val;
+    Parser.Lex();
+
+    if (isNext(AsmToken::Comma)) {
+      Limits->Flags |= wasm::WASM_LIMITS_FLAG_HAS_MAX;
+      auto Tok = Lexer.getTok();
+      if (!Tok.is(AsmToken::Integer))
+        return error("Expected integer constant, instead got: ", Tok);
+      int64_t Val = Tok.getIntVal();
+      assert(Val >= 0);
+      Limits->Maximum = Val;
+      Parser.Lex();
+    }
+    return false;
+  }
+
   bool parseFunctionTableOperand(std::unique_ptr<WebAssemblyOperand> *Op) {
     if (STI->checkFeatures("+reference-types")) {
       // If the reference-types feature is enabled, there is an explicit table
@@ -819,24 +846,31 @@ public:
     }
 
     if (DirectiveID.getString() == ".tabletype") {
+      // .tabletype SYM, ELEMTYPE[, MINSIZE[, MAXSIZE]]
       auto SymName = expectIdent();
       if (SymName.empty())
         return true;
       if (expect(AsmToken::Comma, ","))
         return true;
-      auto TypeTok = Lexer.getTok();
-      auto TypeName = expectIdent();
-      if (TypeName.empty())
+
+      auto ElemTypeTok = Lexer.getTok();
+      auto ElemTypeName = expectIdent();
+      if (ElemTypeName.empty())
+        return true;
+      Optional<wasm::ValType> ElemType = parseType(ElemTypeName);
+      if (!ElemType)
+        return error("Unknown type in .tabletype directive: ", ElemTypeTok);
+
+      wasm::WasmLimits Limits = DefaultLimits();
+      if (isNext(AsmToken::Comma) && parseLimits(&Limits))
         return true;
-      auto Type = parseType(TypeName);
-      if (!Type)
-        return error("Unknown type in .tabletype directive: ", TypeTok);
 
       // Now that we have the name and table type, we can actually create the
       // symbol
       auto WasmSym = cast<MCSymbolWasm>(Ctx.getOrCreateSymbol(SymName));
       WasmSym->setType(wasm::WASM_SYMBOL_TYPE_TABLE);
-      WasmSym->setTableType(Type.getValue());
+      wasm::WasmTableType Type = {uint8_t(ElemType.getValue()), Limits};
+      WasmSym->setTableType(Type);
       TOut.emitTableType(WasmSym);
       return expect(AsmToken::EndOfStatement, "EOL");
     }
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index 5b77b8495adf84833965c2840bcfc124a8cbb283..3508ec0ba98f4440a9b8f2d159a8e7731362e513 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -196,8 +196,6 @@ inline unsigned GetDefaultP2AlignAny(unsigned Opc) {
   WASM_LOAD_STORE(LOAD8_SPLAT)
   WASM_LOAD_STORE(LOAD_LANE_I8x16)
   WASM_LOAD_STORE(STORE_LANE_I8x16)
-  WASM_LOAD_STORE(PREFETCH_T)
-  WASM_LOAD_STORE(PREFETCH_NT)
   return 0;
   WASM_LOAD_STORE(LOAD16_S_I32)
   WASM_LOAD_STORE(LOAD16_U_I32)
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
index 652d7a00a63cd19ab2ae87e8ab34c10687ab5e42..3dd961efca42483ffa402cc23a4bcf13ec473dc2 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
@@ -79,8 +79,15 @@ void WebAssemblyTargetAsmStreamer::emitGlobalType(const MCSymbolWasm *Sym) {
 
 void WebAssemblyTargetAsmStreamer::emitTableType(const MCSymbolWasm *Sym) {
   assert(Sym->isTable());
+  const wasm::WasmTableType &Type = Sym->getTableType();
   OS << "\t.tabletype\t" << Sym->getName() << ", "
-     << WebAssembly::typeToString(Sym->getTableType());
+     << WebAssembly::typeToString(static_cast<wasm::ValType>(Type.ElemType));
+  bool HasMaximum = Type.Limits.Flags & wasm::WASM_LIMITS_FLAG_HAS_MAX;
+  if (Type.Limits.Minimum != 0 || HasMaximum) {
+    OS << ", " << Type.Limits.Minimum;
+    if (HasMaximum)
+      OS << ", " << Type.Limits.Maximum;
+  }
   OS << '\n';
 }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.td b/llvm/lib/Target/WebAssembly/WebAssembly.td
index 2c18bf2c3abea933757884ab634286f81c2bc24b..c1872dd91c58c119cf69e0f8b3fd23ac19369f7a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssembly.td
+++ b/llvm/lib/Target/WebAssembly/WebAssembly.td
@@ -25,12 +25,6 @@ include "llvm/Target/Target.td"
 def FeatureSIMD128 : SubtargetFeature<"simd128", "SIMDLevel", "SIMD128",
                                       "Enable 128-bit SIMD">;
 
-def FeatureUnimplementedSIMD128 :
-      SubtargetFeature<"unimplemented-simd128",
-                       "SIMDLevel", "UnimplementedSIMD128",
-                       "Enable 128-bit SIMD not yet implemented in engines",
-                       [FeatureSIMD128]>;
-
 def FeatureAtomics : SubtargetFeature<"atomics", "HasAtomics", "true",
                                       "Enable Atomics">;
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
index 994baf797c7cc56c7ca7af0869da9c0f64c7751f..5b54ffdc2511d5f42a5fb27a44103be13f58b432 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
@@ -139,12 +139,9 @@ private:
     case MVT::v8i16:
     case MVT::v4i32:
     case MVT::v4f32:
-      if (Subtarget->hasSIMD128())
-        return VT;
-      break;
     case MVT::v2i64:
     case MVT::v2f64:
-      if (Subtarget->hasUnimplementedSIMD128())
+      if (Subtarget->hasSIMD128())
         return VT;
       break;
     default:
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
index d75afdcefb7d75344a5fd6de97ef7ea9e9f43544..3a82dd45a5f651669eda8ac034fbf94e9e58be34 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
@@ -29,10 +29,10 @@ HANDLE_NODETYPE(SWIZZLE)
 HANDLE_NODETYPE(VEC_SHL)
 HANDLE_NODETYPE(VEC_SHR_S)
 HANDLE_NODETYPE(VEC_SHR_U)
-HANDLE_NODETYPE(WIDEN_LOW_S)
-HANDLE_NODETYPE(WIDEN_LOW_U)
-HANDLE_NODETYPE(WIDEN_HIGH_S)
-HANDLE_NODETYPE(WIDEN_HIGH_U)
+HANDLE_NODETYPE(EXTEND_LOW_S)
+HANDLE_NODETYPE(EXTEND_LOW_U)
+HANDLE_NODETYPE(EXTEND_HIGH_S)
+HANDLE_NODETYPE(EXTEND_HIGH_U)
 HANDLE_NODETYPE(THROW)
 HANDLE_NODETYPE(CATCH)
 HANDLE_NODETYPE(MEMORY_COPY)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index a515a5f4aef2f81313320ff45853432e1cfb33e5..85d2d2f60a53df3364bf6accfa30083e275b6bbf 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -761,16 +761,6 @@ bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.align = MemAlign;
     return true;
   }
-  case Intrinsic::wasm_prefetch_t:
-  case Intrinsic::wasm_prefetch_nt: {
-    Info.opc = ISD::INTRINSIC_VOID;
-    Info.memVT = MVT::i8;
-    Info.ptrVal = I.getArgOperand(0);
-    Info.offset = 0;
-    Info.align = Align(1);
-    Info.flags = MachineMemOperand::MOLoad;
-    return true;
-  }
   default:
     return false;
   }
@@ -1705,18 +1695,14 @@ SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
   SDValue Result;
   // Prefer swizzles over vector consts over splats
-  if (NumSwizzleLanes >= NumSplatLanes &&
-      (!Subtarget->hasUnimplementedSIMD128() ||
-       NumSwizzleLanes >= NumConstantLanes)) {
+  if (NumSwizzleLanes >= NumSplatLanes && NumSwizzleLanes >= NumConstantLanes) {
     Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
                          SwizzleIndices);
     auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
     IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
       return Swizzled == GetSwizzleSrcs(I, Lane);
     };
-  } else if (NumConstantLanes >= NumSplatLanes &&
-             Subtarget->hasUnimplementedSIMD128()) {
-    // If we support v128.const, emit it directly
+  } else if (NumConstantLanes >= NumSplatLanes) {
     SmallVector<SDValue, 16> ConstLanes;
     for (const SDValue &Lane : Op->op_values()) {
       if (IsConstant(Lane)) {
@@ -1731,55 +1717,6 @@ SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
       return IsConstant(Lane);
     };
-  } else if (NumConstantLanes >= NumSplatLanes && VecT.isInteger()) {
-    // Otherwise, if this is an integer vector, pack the lane values together so
-    // we can construct the 128-bit constant from a pair of i64s using a splat
-    // followed by at most one i64x2.replace_lane. Also keep track of the lanes
-    // that actually matter so we can avoid the replace_lane in more cases.
-    std::array<uint64_t, 2> I64s{{0, 0}};
-    std::array<uint64_t, 2> ConstLaneMasks{{0, 0}};
-    size_t LaneBits = 128 / Lanes;
-    size_t HalfLanes = Lanes / 2;
-    for (size_t I = 0; I < Lanes; ++I) {
-      const SDValue &Lane = Op.getOperand(I);
-      if (IsConstant(Lane)) {
-        // How much we need to shift Val to position it in an i64
-        auto Shift = LaneBits * (I % HalfLanes);
-        auto Mask = maskTrailingOnes<uint64_t>(LaneBits);
-        auto Val = cast<ConstantSDNode>(Lane.getNode())->getZExtValue() & Mask;
-        I64s[I / HalfLanes] |= Val << Shift;
-        ConstLaneMasks[I / HalfLanes] |= Mask << Shift;
-      }
-    }
-    // Check whether all constant lanes in the second half of the vector are
-    // equivalent in the first half or vice versa to determine whether splatting
-    // either side will be sufficient to materialize the constant. As a special
-    // case, if the first and second halves have no constant lanes in common, we
-    // can just combine them.
-    bool FirstHalfSufficient = (I64s[0] & ConstLaneMasks[1]) == I64s[1];
-    bool SecondHalfSufficient = (I64s[1] & ConstLaneMasks[0]) == I64s[0];
-    bool CombinedSufficient = (ConstLaneMasks[0] & ConstLaneMasks[1]) == 0;
-
-    uint64_t Splatted;
-    if (SecondHalfSufficient) {
-      Splatted = I64s[1];
-    } else if (CombinedSufficient) {
-      Splatted = I64s[0] | I64s[1];
-    } else {
-      Splatted = I64s[0];
-    }
-
-    Result = DAG.getSplatBuildVector(MVT::v2i64, DL,
-                                     DAG.getConstant(Splatted, DL, MVT::i64));
-    if (!FirstHalfSufficient && !SecondHalfSufficient && !CombinedSufficient) {
-      Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v2i64, Result,
-                           DAG.getConstant(I64s[1], DL, MVT::i64),
-                           DAG.getConstant(1, DL, MVT::i32));
-    }
-    Result = DAG.getBitcast(VecT, Result);
-    IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
-      return IsConstant(Lane);
-    };
   } else {
     // Use a splat, but possibly a load_splat
     LoadSDNode *SplattedLoad;
@@ -1961,8 +1898,8 @@ performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
   return DAG.getBitcast(DstType, NewShuffle);
 }
 
-static SDValue performVectorWidenCombine(SDNode *N,
-                                         TargetLowering::DAGCombinerInfo &DCI) {
+static SDValue
+performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
   auto &DAG = DCI.DAG;
   assert(N->getOpcode() == ISD::SIGN_EXTEND ||
          N->getOpcode() == ISD::ZERO_EXTEND);
@@ -1996,10 +1933,10 @@ static SDValue performVectorWidenCombine(SDNode *N,
   bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
   bool IsLow = Index == 0;
 
-  unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::WIDEN_LOW_S
-                                : WebAssemblyISD::WIDEN_HIGH_S)
-                       : (IsLow ? WebAssemblyISD::WIDEN_LOW_U
-                                : WebAssemblyISD::WIDEN_HIGH_U);
+  unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
+                                : WebAssemblyISD::EXTEND_HIGH_S)
+                       : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
+                                : WebAssemblyISD::EXTEND_HIGH_U);
 
   return DAG.getNode(Op, SDLoc(N), ResVT, Source);
 }
@@ -2014,6 +1951,6 @@ WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
     return performVECTOR_SHUFFLECombine(N, DCI);
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
-    return performVectorWidenCombine(N, DCI);
+    return performVectorExtendCombine(N, DCI);
   }
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
index a1173ce116474c736352a05aeba44f4a6481ff8b..f6b9efa85cb9c32661797a509cd218feab1ebf39 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
@@ -26,10 +26,6 @@ def HasSIMD128 :
     Predicate<"Subtarget->hasSIMD128()">,
     AssemblerPredicate<(all_of FeatureSIMD128), "simd128">;
 
-def HasUnimplementedSIMD128 :
-    Predicate<"Subtarget->hasUnimplementedSIMD128()">,
-    AssemblerPredicate<(all_of FeatureUnimplementedSIMD128), "unimplemented-simd128">;
-
 def HasAtomics :
     Predicate<"Subtarget->hasAtomics()">,
     AssemblerPredicate<(all_of FeatureAtomics), "atomics">;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index 98422a8264e74419e0dd706178b8e7ed2e6fb43d..9afb7a07779620f96454cf5e077bdfca20f6c32f 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -266,8 +266,8 @@ multiclass SIMDLoadZero<Vec vec, bits<32> simdop> {
 
 // TODO: Also support v4f32 and v2f64 once the instructions are merged
 // to the proposal
-defm "" : SIMDLoadZero<I32x4, 252>;
-defm "" : SIMDLoadZero<I64x2, 253>;
+defm "" : SIMDLoadZero<I32x4, 0x5c>;
+defm "" : SIMDLoadZero<I64x2, 0x5d>;
 
 foreach vec = [I32x4, I64x2] in {
 defvar loadpat = !cast<Intrinsic>("int_wasm_load"#vec.lane_bits#"_zero");
@@ -302,10 +302,10 @@ multiclass SIMDLoadLane<Vec vec, bits<32> simdop> {
 
 // TODO: Also support v4f32 and v2f64 once the instructions are merged
 // to the proposal
-defm "" : SIMDLoadLane<I8x16, 88>;
-defm "" : SIMDLoadLane<I16x8, 89>;
-defm "" : SIMDLoadLane<I32x4, 90>;
-defm "" : SIMDLoadLane<I64x2, 91>;
+defm "" : SIMDLoadLane<I8x16, 0x54>;
+defm "" : SIMDLoadLane<I16x8, 0x55>;
+defm "" : SIMDLoadLane<I32x4, 0x56>;
+defm "" : SIMDLoadLane<I64x2, 0x57>;
 
 // Select loads with no constant offset.
 multiclass LoadLanePatNoOffset<Vec vec, SDPatternOperator kind> {
@@ -375,10 +375,10 @@ multiclass SIMDStoreLane<Vec vec, bits<32> simdop> {
 
 // TODO: Also support v4f32 and v2f64 once the instructions are merged
 // to the proposal
-defm "" : SIMDStoreLane<I8x16, 92>;
-defm "" : SIMDStoreLane<I16x8, 93>;
-defm "" : SIMDStoreLane<I32x4, 94>;
-defm "" : SIMDStoreLane<I64x2, 95>;
+defm "" : SIMDStoreLane<I8x16, 0x58>;
+defm "" : SIMDStoreLane<I16x8, 0x59>;
+defm "" : SIMDStoreLane<I32x4, 0x5a>;
+defm "" : SIMDStoreLane<I64x2, 0x5b>;
 
 // Select stores with no constant offset.
 multiclass StoreLanePatNoOffset<Vec vec, Intrinsic kind> {
@@ -404,8 +404,7 @@ defm : StoreLanePatNoOffset<I64x2, int_wasm_store64_lane>;
 
 // Constant: v128.const
 multiclass ConstVec<Vec vec, dag ops, dag pat, string args> {
-  let isMoveImm = 1, isReMaterializable = 1,
-      Predicates = [HasUnimplementedSIMD128] in
+  let isMoveImm = 1, isReMaterializable = 1 in
   defm CONST_V128_#vec : SIMD_I<(outs V128:$dst), ops, (outs), ops,
                                  [(set V128:$dst, (vec.vt pat))],
                                  "v128.const\t$dst, "#args,
@@ -792,22 +791,6 @@ def : Pat<(select
           (SELECT_V128 $rhs, $lhs, $cond)>;
 } // foreach vec
 
-// Sign select
-multiclass SIMDSignSelect<Vec vec, bits<32> simdop> {
-  defm SIGNSELECT_#vec :
-    SIMD_I<(outs V128:$dst), (ins V128:$v1, V128:$v2, V128:$c), (outs), (ins),
-           [(set (vec.vt V128:$dst),
-             (vec.vt (int_wasm_signselect
-               (vec.vt V128:$v1), (vec.vt V128:$v2), (vec.vt V128:$c))))],
-           vec.prefix#".signselect\t$dst, $v1, $v2, $c",
-           vec.prefix#".signselect", simdop>;
-}
-
-defm : SIMDSignSelect<I8x16, 125>;
-defm : SIMDSignSelect<I16x8, 126>;
-defm : SIMDSignSelect<I32x4, 127>;
-defm : SIMDSignSelect<I64x2, 148>;
-
 //===----------------------------------------------------------------------===//
 // Integer unary arithmetic
 //===----------------------------------------------------------------------===//
@@ -934,19 +917,19 @@ multiclass SIMDBinaryInt<SDPatternOperator node, string name, bits<32> baseInst>
   defm "" : SIMDBinary<I64x2, node, name, !add(baseInst, 96)>;
 }
 
-// Integer addition: add / add_saturate_s / add_saturate_u
+// Integer addition: add / add_sat_s / add_sat_u
 let isCommutable = 1 in {
 defm ADD : SIMDBinaryInt<add, "add", 110>;
-defm ADD_SAT_S : SIMDBinaryIntSmall<saddsat, "add_saturate_s", 111>;
-defm ADD_SAT_U : SIMDBinaryIntSmall<uaddsat, "add_saturate_u", 112>;
+defm ADD_SAT_S : SIMDBinaryIntSmall<saddsat, "add_sat_s", 111>;
+defm ADD_SAT_U : SIMDBinaryIntSmall<uaddsat, "add_sat_u", 112>;
 } // isCommutable = 1
 
-// Integer subtraction: sub / sub_saturate_s / sub_saturate_u
+// Integer subtraction: sub / sub_sat_s / sub_sat_u
 defm SUB : SIMDBinaryInt<sub, "sub", 113>;
 defm SUB_SAT_S :
-  SIMDBinaryIntSmall<int_wasm_sub_saturate_signed, "sub_saturate_s", 114>;
+  SIMDBinaryIntSmall<int_wasm_sub_sat_signed, "sub_sat_s", 114>;
 defm SUB_SAT_U :
-  SIMDBinaryIntSmall<int_wasm_sub_saturate_unsigned, "sub_saturate_u", 115>;
+  SIMDBinaryIntSmall<int_wasm_sub_sat_unsigned, "sub_sat_u", 115>;
 
 // Integer multiplication: mul
 let isCommutable = 1 in
@@ -997,31 +980,31 @@ multiclass SIMDExtBinary<Vec vec, Intrinsic node, string name, bits<32> simdop>
 }
 
 defm EXTMUL_LOW_S :
-  SIMDExtBinary<I16x8, int_wasm_extmul_low_signed, "extmul_low_i8x16_s", 154>;
+  SIMDExtBinary<I16x8, int_wasm_extmul_low_signed, "extmul_low_i8x16_s", 0x9c>;
 defm EXTMUL_HIGH_S :
-  SIMDExtBinary<I16x8, int_wasm_extmul_high_signed, "extmul_high_i8x16_s", 157>;
+  SIMDExtBinary<I16x8, int_wasm_extmul_high_signed, "extmul_high_i8x16_s", 0x9d>;
 defm EXTMUL_LOW_U :
-  SIMDExtBinary<I16x8, int_wasm_extmul_low_unsigned, "extmul_low_i8x16_u", 158>;
+  SIMDExtBinary<I16x8, int_wasm_extmul_low_unsigned, "extmul_low_i8x16_u", 0x9e>;
 defm EXTMUL_HIGH_U :
-  SIMDExtBinary<I16x8, int_wasm_extmul_high_unsigned, "extmul_high_i8x16_u", 159>;
+  SIMDExtBinary<I16x8, int_wasm_extmul_high_unsigned, "extmul_high_i8x16_u", 0x9f>;
 
 defm EXTMUL_LOW_S :
-  SIMDExtBinary<I32x4, int_wasm_extmul_low_signed, "extmul_low_i16x8_s", 187>;
+  SIMDExtBinary<I32x4, int_wasm_extmul_low_signed, "extmul_low_i16x8_s", 0xbc>;
 defm EXTMUL_HIGH_S :
-  SIMDExtBinary<I32x4, int_wasm_extmul_high_signed, "extmul_high_i16x8_s", 189>;
+  SIMDExtBinary<I32x4, int_wasm_extmul_high_signed, "extmul_high_i16x8_s", 0xbd>;
 defm EXTMUL_LOW_U :
-  SIMDExtBinary<I32x4, int_wasm_extmul_low_unsigned, "extmul_low_i16x8_u", 190>;
+  SIMDExtBinary<I32x4, int_wasm_extmul_low_unsigned, "extmul_low_i16x8_u", 0xbe>;
 defm EXTMUL_HIGH_U :
-  SIMDExtBinary<I32x4, int_wasm_extmul_high_unsigned, "extmul_high_i16x8_u", 191>;
+  SIMDExtBinary<I32x4, int_wasm_extmul_high_unsigned, "extmul_high_i16x8_u", 0xbf>;
 
 defm EXTMUL_LOW_S :
-  SIMDExtBinary<I64x2, int_wasm_extmul_low_signed, "extmul_low_i32x4_s", 210>;
+  SIMDExtBinary<I64x2, int_wasm_extmul_low_signed, "extmul_low_i32x4_s", 0xdc>;
 defm EXTMUL_HIGH_S :
-  SIMDExtBinary<I64x2, int_wasm_extmul_high_signed, "extmul_high_i32x4_s", 211>;
+  SIMDExtBinary<I64x2, int_wasm_extmul_high_signed, "extmul_high_i32x4_s", 0xdd>;
 defm EXTMUL_LOW_U :
-  SIMDExtBinary<I64x2, int_wasm_extmul_low_unsigned, "extmul_low_i32x4_u", 214>;
+  SIMDExtBinary<I64x2, int_wasm_extmul_low_unsigned, "extmul_low_i32x4_u", 0xde>;
 defm EXTMUL_HIGH_U :
-  SIMDExtBinary<I64x2, int_wasm_extmul_high_unsigned, "extmul_high_i32x4_u", 215>;
+  SIMDExtBinary<I64x2, int_wasm_extmul_high_unsigned, "extmul_high_i32x4_u", 0xdf>;
 
 //===----------------------------------------------------------------------===//
 // Floating-point unary arithmetic
@@ -1042,14 +1025,14 @@ defm NEG : SIMDUnaryFP<fneg, "neg", 225>;
 defm SQRT : SIMDUnaryFP<fsqrt, "sqrt", 227>;
 
 // Rounding: ceil, floor, trunc, nearest
-defm CEIL : SIMDUnary<F32x4, int_wasm_ceil, "ceil", 216>;
-defm FLOOR : SIMDUnary<F32x4, int_wasm_floor, "floor", 217>;
-defm TRUNC: SIMDUnary<F32x4, int_wasm_trunc, "trunc", 218>;
-defm NEAREST: SIMDUnary<F32x4, int_wasm_nearest, "nearest", 219>;
-defm CEIL : SIMDUnary<F64x2, int_wasm_ceil, "ceil", 220>;
-defm FLOOR : SIMDUnary<F64x2, int_wasm_floor, "floor", 221>;
-defm TRUNC: SIMDUnary<F64x2, int_wasm_trunc, "trunc", 222>;
-defm NEAREST: SIMDUnary<F64x2, int_wasm_nearest, "nearest", 223>;
+defm CEIL : SIMDUnary<F32x4, int_wasm_ceil, "ceil", 0x67>;
+defm FLOOR : SIMDUnary<F32x4, int_wasm_floor, "floor", 0x68>;
+defm TRUNC: SIMDUnary<F32x4, int_wasm_trunc, "trunc", 0x69>;
+defm NEAREST: SIMDUnary<F32x4, int_wasm_nearest, "nearest", 0x6a>;
+defm CEIL : SIMDUnary<F64x2, int_wasm_ceil, "ceil", 0x74>;
+defm FLOOR : SIMDUnary<F64x2, int_wasm_floor, "floor", 0x75>;
+defm TRUNC: SIMDUnary<F64x2, int_wasm_trunc, "trunc", 0x7a>;
+defm NEAREST: SIMDUnary<F64x2, int_wasm_nearest, "nearest", 0x94>;
 
 //===----------------------------------------------------------------------===//
 // Floating-point binary arithmetic
@@ -1106,42 +1089,42 @@ defm "" : SIMDConvert<I32x4, F32x4, fp_to_uint, "trunc_sat_f32x4_u", 249>;
 defm "" : SIMDConvert<F32x4, I32x4, sint_to_fp, "convert_i32x4_s", 250>;
 defm "" : SIMDConvert<F32x4, I32x4, uint_to_fp, "convert_i32x4_u", 251>;
 
-// Lower llvm.wasm.trunc.saturate.* to saturating instructions
+// Lower llvm.wasm.trunc.sat.* to saturating instructions
 def : Pat<(v4i32 (int_wasm_trunc_saturate_signed (v4f32 V128:$src))),
           (fp_to_sint_I32x4 $src)>;
 def : Pat<(v4i32 (int_wasm_trunc_saturate_unsigned (v4f32 V128:$src))),
           (fp_to_uint_I32x4 $src)>;
 
-// Widening operations
-def widen_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
-def widen_low_s : SDNode<"WebAssemblyISD::WIDEN_LOW_S", widen_t>;
-def widen_high_s : SDNode<"WebAssemblyISD::WIDEN_HIGH_S", widen_t>;
-def widen_low_u : SDNode<"WebAssemblyISD::WIDEN_LOW_U", widen_t>;
-def widen_high_u : SDNode<"WebAssemblyISD::WIDEN_HIGH_U", widen_t>;
+// Extending operations
+def extend_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
+def extend_low_s : SDNode<"WebAssemblyISD::EXTEND_LOW_S", extend_t>;
+def extend_high_s : SDNode<"WebAssemblyISD::EXTEND_HIGH_S", extend_t>;
+def extend_low_u : SDNode<"WebAssemblyISD::EXTEND_LOW_U", extend_t>;
+def extend_high_u : SDNode<"WebAssemblyISD::EXTEND_HIGH_U", extend_t>;
 
 // TODO: refactor this to be uniform for i64x2 if the numbering is not changed.
-multiclass SIMDWiden<Vec vec, bits<32> baseInst> {
-  defm "" : SIMDConvert<vec, vec.split, widen_low_s,
-                        "widen_low_"#vec.split.prefix#"_s", baseInst>;
-  defm "" : SIMDConvert<vec, vec.split, widen_high_s,
-                        "widen_high_"#vec.split.prefix#"_s", !add(baseInst, 1)>;
-  defm "" : SIMDConvert<vec, vec.split, widen_low_u,
-                        "widen_low_"#vec.split.prefix#"_u", !add(baseInst, 2)>;
-  defm "" : SIMDConvert<vec, vec.split, widen_high_u,
-                        "widen_high_"#vec.split.prefix#"_u", !add(baseInst, 3)>;
+multiclass SIMDExtend<Vec vec, bits<32> baseInst> {
+  defm "" : SIMDConvert<vec, vec.split, extend_low_s,
+                        "extend_low_"#vec.split.prefix#"_s", baseInst>;
+  defm "" : SIMDConvert<vec, vec.split, extend_high_s,
+                        "extend_high_"#vec.split.prefix#"_s", !add(baseInst, 1)>;
+  defm "" : SIMDConvert<vec, vec.split, extend_low_u,
+                        "extend_low_"#vec.split.prefix#"_u", !add(baseInst, 2)>;
+  defm "" : SIMDConvert<vec, vec.split, extend_high_u,
+                        "extend_high_"#vec.split.prefix#"_u", !add(baseInst, 3)>;
 }
 
-defm "" : SIMDWiden<I16x8, 135>;
-defm "" : SIMDWiden<I32x4, 167>;
+defm "" : SIMDExtend<I16x8, 135>;
+defm "" : SIMDExtend<I32x4, 167>;
 
-defm "" : SIMDConvert<I64x2, I32x4, int_wasm_widen_low_signed,
-                      "widen_low_i32x4_s", 199>;
-defm "" : SIMDConvert<I64x2, I32x4, int_wasm_widen_high_signed,
-                      "widen_high_i32x4_s", 200>;
-defm "" : SIMDConvert<I64x2, I32x4, int_wasm_widen_low_unsigned,
-                      "widen_low_i32x4_u", 201>;
-defm "" : SIMDConvert<I64x2, I32x4, int_wasm_widen_high_unsigned,
-                      "widen_high_i32x4_u", 202>;
+defm "" : SIMDConvert<I64x2, I32x4, int_wasm_extend_low_signed,
+                      "extend_low_i32x4_s", 199>;
+defm "" : SIMDConvert<I64x2, I32x4, int_wasm_extend_high_signed,
+                      "extend_high_i32x4_s", 200>;
+defm "" : SIMDConvert<I64x2, I32x4, int_wasm_extend_low_unsigned,
+                      "extend_low_i32x4_u", 201>;
+defm "" : SIMDConvert<I64x2, I32x4, int_wasm_extend_high_unsigned,
+                      "extend_high_i32x4_u", 202>;
 
 // Narrowing operations
 multiclass SIMDNarrow<Vec vec, bits<32> baseInst> {
@@ -1249,112 +1232,31 @@ def : Pat<(t1 (bitconvert (t2 V128:$v))), (t1 V128:$v)>;
 
 // Extended pairwise addition
 defm "" : SIMDConvert<I16x8, I8x16, int_wasm_extadd_pairwise_signed,
-                      "extadd_pairwise_i8x16_s", 0xc2>;
+                      "extadd_pairwise_i8x16_s", 0x7c>;
 defm "" : SIMDConvert<I16x8, I8x16, int_wasm_extadd_pairwise_unsigned,
-                      "extadd_pairwise_i8x16_u", 0xc3>;
+                      "extadd_pairwise_i8x16_u", 0x7d>;
 defm "" : SIMDConvert<I32x4, I16x8, int_wasm_extadd_pairwise_signed,
-                      "extadd_pairwise_i16x8_s", 0xa5>;
+                      "extadd_pairwise_i16x8_s", 0x7e>;
 defm "" : SIMDConvert<I32x4, I16x8, int_wasm_extadd_pairwise_unsigned,
-                      "extadd_pairwise_i16x8_u", 0xa6>;
+                      "extadd_pairwise_i16x8_u", 0x7f>;
 
 // Prototype f64x2 conversions
-defm "" : SIMDConvert<F64x2, I32x4, int_wasm_convert_low_signed,
-                      "convert_low_i32x4_s", 0x53>;
-defm "" : SIMDConvert<F64x2, I32x4, int_wasm_convert_low_unsigned,
-                      "convert_low_i32x4_u", 0x54>;
-defm "" : SIMDConvert<I32x4, F64x2, int_wasm_trunc_saturate_zero_signed,
-                      "trunc_sat_zero_f64x2_s", 0x55>;
-defm "" : SIMDConvert<I32x4, F64x2, int_wasm_trunc_saturate_zero_unsigned,
-                      "trunc_sat_zero_f64x2_u", 0x56>;
 defm "" : SIMDConvert<F32x4, F64x2, int_wasm_demote_zero,
-                      "demote_zero_f64x2", 0x57>;
+                      "demote_zero_f64x2", 0x5e>;
 defm "" : SIMDConvert<F64x2, F32x4, int_wasm_promote_low,
-                      "promote_low_f32x4", 0x69>;
-
-// Prototype i8x16 to i32x4 widening
-defm WIDEN_I8x16_TO_I32x4_S :
-  SIMD_I<(outs V128:$dst), (ins V128:$vec, vec_i8imm_op:$idx),
-         (outs), (ins vec_i8imm_op:$idx),
-         [(set (I32x4.vt V128:$dst),
-            (I32x4.vt (int_wasm_widen_signed
-              (I8x16.vt V128:$vec), (i32 timm:$idx))))],
-         "i32x4.widen_i8x16_s\t$dst, $vec, $idx",
-         "i32x4.widen_i8x16_s\t$idx", 0x67>;
-defm WIDEN_I8x16_TO_I32x4_U :
-  SIMD_I<(outs V128:$dst), (ins V128:$vec, vec_i8imm_op:$idx),
-         (outs), (ins vec_i8imm_op:$idx),
-         [(set (I32x4.vt V128:$dst),
-            (I32x4.vt (int_wasm_widen_unsigned
-              (I8x16.vt V128:$vec), (i32 timm:$idx))))],
-         "i32x4.widen_i8x16_u\t$dst, $vec, $idx",
-         "i32x4.widen_i8x16_u\t$idx", 0x68>;
-
-
-//===----------------------------------------------------------------------===//
-// Quasi-Fused Multiply- Add and Subtract (QFMA/QFMS)
-//===----------------------------------------------------------------------===//
-
-multiclass SIMDQFM<Vec vec, bits<32> simdopA, bits<32> simdopS> {
-  defm QFMA_#vec :
-    SIMD_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c),
-           (outs), (ins),
-           [(set (vec.vt V128:$dst), (int_wasm_qfma
-              (vec.vt V128:$a), (vec.vt V128:$b), (vec.vt V128:$c)))],
-           vec.prefix#".qfma\t$dst, $a, $b, $c", vec.prefix#".qfma", simdopA>;
-  defm QFMS_#vec :
-    SIMD_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c),
-           (outs), (ins),
-           [(set (vec.vt V128:$dst), (int_wasm_qfms
-              (vec.vt V128:$a), (vec.vt V128:$b), (vec.vt V128:$c)))],
-           vec.prefix#".qfms\t$dst, $a, $b, $c", vec.prefix#".qfms", simdopS>;
-}
-
-defm "" : SIMDQFM<F32x4, 180, 212>;
-defm "" : SIMDQFM<F64x2, 254, 255>;
+                      "promote_low_f32x4", 0x5f>;
+defm "" : SIMDConvert<I32x4, F64x2, int_wasm_trunc_sat_zero_signed,
+                      "trunc_sat_zero_f64x2_s", 0xfc>;
+defm "" : SIMDConvert<I32x4, F64x2, int_wasm_trunc_sat_zero_unsigned,
+                      "trunc_sat_zero_f64x2_u", 0xfd>;
+defm "" : SIMDConvert<F64x2, I32x4, int_wasm_convert_low_signed,
+                      "convert_low_i32x4_s", 0xfe>;
+defm "" : SIMDConvert<F64x2, I32x4, int_wasm_convert_low_unsigned,
+                      "convert_low_i32x4_u", 0xff>;
 
 //===----------------------------------------------------------------------===//
 // Saturating Rounding Q-Format Multiplication
 //===----------------------------------------------------------------------===//
 
 defm Q15MULR_SAT_S :
-  SIMDBinary<I16x8, int_wasm_q15mulr_saturate_signed, "q15mulr_sat_s", 156>;
-
-//===----------------------------------------------------------------------===//
-// Experimental prefetch instructions: prefetch.t, prefetch.nt
-//===----------------------------------------------------------------------===//
-
-let mayLoad = true, UseNamedOperandTable = true in {
-defm PREFETCH_T_A32 :
-  SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
-         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
-         "prefetch.t\t${off}(${addr})$p2align",
-         "prefetch.t\t$off$p2align", 0xc5>;
-defm PREFETCH_T_A64 :
-  SIMD_I<(outs), (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
-         (outs), (ins P2Align:$p2align, offset64_op:$off), [],
-         "prefetch.t\t${off}(${addr})$p2align",
-         "prefetch.t\t$off$p2align", 0xc5>;
-defm PREFETCH_NT_A32 :
-  SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
-         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
-         "prefetch.nt\t${off}(${addr})$p2align",
-         "prefetch.nt\t$off$p2align", 0xc6>;
-defm PREFETCH_NT_A64 :
-  SIMD_I<(outs), (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
-         (outs), (ins P2Align:$p2align, offset64_op:$off), [],
-         "prefetch.nt\t${off}(${addr})$p2align",
-         "prefetch.nt\t$off$p2align", 0xc6>;
-} // mayLoad, UseNamedOperandTable
-
-multiclass PrefetchPatNoOffset<Intrinsic kind, string inst> {
-  def : Pat<(kind I32:$addr), (!cast<NI>(inst # "_A32") 0, 0, $addr)>,
-        Requires<[HasAddr32]>;
-  def : Pat<(kind I64:$addr), (!cast<NI>(inst # "_A64") 0, 0, $addr)>,
-        Requires<[HasAddr64]>;
-}
-
-foreach inst = [["PREFETCH_T", "int_wasm_prefetch_t"],
-                ["PREFETCH_NT", "int_wasm_prefetch_nt"]] in {
-defvar node = !cast<Intrinsic>(inst[1]);
-defm : PrefetchPatNoOffset<node, inst[0]>;
-}
+  SIMDBinary<I16x8, int_wasm_q15mulr_sat_signed, "q15mulr_sat_s", 0x82>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h b/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
index a1c872ef2135368cf40f5743c589e484d2039212..43d5871f0aa0b72451d0ffaf924ae666513612a4 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h
@@ -36,7 +36,6 @@ class WebAssemblySubtarget final : public WebAssemblyGenSubtargetInfo {
   enum SIMDEnum {
     NoSIMD,
     SIMD128,
-    UnimplementedSIMD128,
   } SIMDLevel = NoSIMD;
 
   bool HasAtomics = false;
@@ -90,9 +89,6 @@ public:
   // Predicates used by WebAssemblyInstrInfo.td.
   bool hasAddr64() const { return TargetTriple.isArch64Bit(); }
   bool hasSIMD128() const { return SIMDLevel >= SIMD128; }
-  bool hasUnimplementedSIMD128() const {
-    return SIMDLevel >= UnimplementedSIMD128;
-  }
   bool hasAtomics() const { return HasAtomics; }
   bool hasNontrappingFPToInt() const { return HasNontrappingFPToInt; }
   bool hasSignExt() const { return HasSignExt; }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index a2111afbfcbaf6fd534da626be4e4c336fc77a0b..1357860c27c97531baf8a1cd5155e9cbabd38390 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -36,11 +36,18 @@ unsigned WebAssemblyTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
   return Result;
 }
 
-unsigned WebAssemblyTTIImpl::getRegisterBitWidth(bool Vector) const {
-  if (Vector && getST()->hasSIMD128())
-    return 128;
+TypeSize WebAssemblyTTIImpl::getRegisterBitWidth(
+    TargetTransformInfo::RegisterKind K) const {
+  switch (K) {
+  case TargetTransformInfo::RGK_Scalar:
+    return TypeSize::getFixed(64);
+  case TargetTransformInfo::RGK_FixedWidthVector:
+    return TypeSize::getFixed(getST()->hasSIMD128() ? 128 : 64);
+  case TargetTransformInfo::RGK_ScalableVector:
+    return TypeSize::getScalable(0);
+  }
 
-  return 64;
+  llvm_unreachable("Unsupported register kind");
 }
 
 unsigned WebAssemblyTTIImpl::getArithmeticInstrCost(
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index 3515a3e149d1e38a9df8cb81f748300c8e5d3211..8b9719fe7ff2ef2fc76f6b86165ea4ae8e1c1f70 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -57,7 +57,7 @@ public:
   /// @{
 
   unsigned getNumberOfRegisters(unsigned ClassID) const;
-  unsigned getRegisterBitWidth(bool Vector) const;
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
   unsigned getArithmeticInstrCost(
       unsigned Opcode, Type *Ty,
       TTI::TargetCostKind CostKind = TTI::TCK_SizeAndLatency,
diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index 3d8f77ebe503b45c99331a6ef2756cf6d1d470d1..b37cd25c7de1a63e1e6225615626fea0b62996f9 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -1446,6 +1446,10 @@ bool X86FastISel::X86SelectCmp(const Instruction *I) {
   if (!isTypeLegal(I->getOperand(0)->getType(), VT))
     return false;
 
+  // Below code only works for scalars.
+  if (VT.isVector())
+    return false;
+
   // Try to optimize or fold the cmp.
   CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
   unsigned ResultReg = 0;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 0cd08b4c52aa75f945b1b876a525e40206bc416f..bf89782ea017ad26ffa3acf0c80dce9434d11098 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -6338,8 +6338,8 @@ static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT,
 // Match (xor X, -1) -> X.
 // Match extract_subvector(xor X, -1) -> extract_subvector(X).
 // Match concat_vectors(xor X, -1, xor Y, -1) -> concat_vectors(X, Y).
-static SDValue IsNOT(SDValue V, SelectionDAG &DAG, bool OneUse = false) {
-  V = OneUse ? peekThroughOneUseBitcasts(V) : peekThroughBitcasts(V);
+static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
+  V = peekThroughBitcasts(V);
   if (V.getOpcode() == ISD::XOR &&
       ISD::isBuildVectorAllOnes(V.getOperand(1).getNode()))
     return V.getOperand(0);
@@ -13705,9 +13705,15 @@ static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
     V = extract128BitVector(V, ExtractIdx, DAG, DL);
   }
 
-  if (Opcode == X86ISD::MOVDDUP && !V.getValueType().isVector())
-    V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
-                    DAG.getBitcast(MVT::f64, V));
+  // On AVX we can use VBROADCAST directly for scalar sources.
+  if (Opcode == X86ISD::MOVDDUP && !V.getValueType().isVector()) {
+    V = DAG.getBitcast(MVT::f64, V);
+    if (Subtarget.hasAVX()) {
+      V = DAG.getNode(X86ISD::VBROADCAST, DL, MVT::v2f64, V);
+      return DAG.getBitcast(VT, V);
+    }
+    V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V);
+  }
 
   // If this is a scalar, do the broadcast on this type and bitcast.
   if (!V.getValueType().isVector()) {
@@ -22707,23 +22713,21 @@ static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
 
 /// Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then
 /// concatenate the result back.
-static SDValue splitIntVSETCC(SDValue Op, SelectionDAG &DAG) {
-  EVT VT = Op.getValueType();
+static SDValue splitIntVSETCC(EVT VT, SDValue LHS, SDValue RHS,
+                              ISD::CondCode Cond, SelectionDAG &DAG,
+                              const SDLoc &dl) {
+  assert(VT.isInteger() && VT == LHS.getValueType() &&
+         VT == RHS.getValueType() && "Unsupported VTs!");
 
-  assert(Op.getOpcode() == ISD::SETCC && "Unsupported operation");
-  assert(Op.getOperand(0).getValueType().isInteger() &&
-         VT == Op.getOperand(0).getValueType() && "Unsupported VTs!");
-
-  SDLoc dl(Op);
-  SDValue CC = Op.getOperand(2);
+  SDValue CC = DAG.getCondCode(Cond);
 
   // Extract the LHS Lo/Hi vectors
   SDValue LHS1, LHS2;
-  std::tie(LHS1, LHS2) = splitVector(Op.getOperand(0), DAG, dl);
+  std::tie(LHS1, LHS2) = splitVector(LHS, DAG, dl);
 
   // Extract the RHS Lo/Hi vectors
   SDValue RHS1, RHS2;
-  std::tie(RHS1, RHS2) = splitVector(Op.getOperand(1), DAG, dl);
+  std::tie(RHS1, RHS2) = splitVector(RHS, DAG, dl);
 
   // Issue the operation on the smaller types and concatenate the result back
   EVT LoVT, HiVT;
@@ -23073,11 +23077,11 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
 
   // Break 256-bit integer vector compare into smaller ones.
   if (VT.is256BitVector() && !Subtarget.hasInt256())
-    return splitIntVSETCC(Op, DAG);
+    return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl);
 
   if (VT == MVT::v32i16 || VT == MVT::v64i8) {
     assert(!Subtarget.hasBWI() && "Unexpected VT with AVX512BW!");
-    return splitIntVSETCC(Op, DAG);
+    return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl);
   }
 
   // If this is a SETNE against the signed minimum value, change it to SETGT.
@@ -27488,35 +27492,28 @@ static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
   // to a vXi16 type. Do the multiplies, shift the results and pack the half
   // lane results back together.
 
-  MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
+  // We'll take different approaches for signed and unsigned.
+  // For unsigned we'll use punpcklbw/punpckhbw to zero extend the bytes and
+  // use pmullw to calculate the full 16-bit product.
+  // For signed we'll use punpcklbw/punpckhbw to extend the bytes to words by
+  // placing the bytes in the upper byte of each word with zeros in the lower
+  // byte. This allows us to use pmulhw to calculate the full 16-bit product.
+  // This trick means we don't need to sign extend the bytes to use pmullw.
 
-  static const int PSHUFDMask[] = { 8,  9, 10, 11, 12, 13, 14, 15,
-                                   -1, -1, -1, -1, -1, -1, -1, -1};
-
-  // Extract the lo parts and zero/sign extend to i16.
-  // Only use SSE4.1 instructions for signed v16i8 where using unpack requires
-  // shifts to sign extend. Using unpack for unsigned only requires an xor to
-  // create zeros and a copy due to tied registers contraints pre-avx. But using
-  // zero_extend_vector_inreg would require an additional pshufd for the high
-  // part.
+  MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
+  SDValue Zero = DAG.getConstant(0, dl, VT);
 
   SDValue ALo, AHi;
-  if (IsSigned && VT == MVT::v16i8 && Subtarget.hasSSE41()) {
-    ALo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, ExVT, A);
-
-    AHi = DAG.getVectorShuffle(VT, dl, A, A, PSHUFDMask);
-    AHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, ExVT, AHi);
-  } else if (IsSigned) {
-    ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), A));
-    AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), A));
-
-    ALo = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, ALo, 8, DAG);
-    AHi = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, AHi, 8, DAG);
+  if (IsSigned) {
+    ALo = DAG.getBitcast(
+        ExVT, getUnpackl(DAG, dl, VT, Zero, A));
+    AHi = DAG.getBitcast(
+        ExVT, getUnpackh(DAG, dl, VT, Zero, A));
   } else {
     ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A,
-                                          DAG.getConstant(0, dl, VT)));
+                                          Zero));
     AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A,
-                                          DAG.getConstant(0, dl, VT)));
+                                          Zero));
   }
 
   SDValue BLo, BHi;
@@ -27529,8 +27526,12 @@ static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
         SDValue HiOp = B.getOperand(i + j + 8);
 
         if (IsSigned) {
-          LoOp = DAG.getSExtOrTrunc(LoOp, dl, MVT::i16);
-          HiOp = DAG.getSExtOrTrunc(HiOp, dl, MVT::i16);
+          LoOp = DAG.getAnyExtOrTrunc(LoOp, dl, MVT::i16);
+          HiOp = DAG.getAnyExtOrTrunc(HiOp, dl, MVT::i16);
+          LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp,
+                             DAG.getConstant(8, dl, MVT::i16));
+          HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp,
+                             DAG.getConstant(8, dl, MVT::i16));
         } else {
           LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16);
           HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16);
@@ -27543,32 +27544,23 @@ static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
 
     BLo = DAG.getBuildVector(ExVT, dl, LoOps);
     BHi = DAG.getBuildVector(ExVT, dl, HiOps);
-  } else if (IsSigned && VT == MVT::v16i8 && Subtarget.hasSSE41()) {
-    BLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, ExVT, B);
-
-    BHi = DAG.getVectorShuffle(VT, dl, B, B, PSHUFDMask);
-    BHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, ExVT, BHi);
   } else if (IsSigned) {
-    BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), B));
-    BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), B));
-
-    BLo = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, BLo, 8, DAG);
-    BHi = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, BHi, 8, DAG);
+    BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B));
+    BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B));
   } else {
-    BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B,
-                                          DAG.getConstant(0, dl, VT)));
-    BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B,
-                                          DAG.getConstant(0, dl, VT)));
+    BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero));
+    BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero));
   }
 
   // Multiply, lshr the upper 8bits to the lower 8bits of the lo/hi results and
   // pack back to vXi8.
-  SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
-  SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
+  unsigned MulOpc = IsSigned ? ISD::MULHS : ISD::MUL;
+  SDValue RLo = DAG.getNode(MulOpc, dl, ExVT, ALo, BLo);
+  SDValue RHi = DAG.getNode(MulOpc, dl, ExVT, AHi, BHi);
   RLo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, RLo, 8, DAG);
   RHi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, RHi, 8, DAG);
 
-  // Bitcast back to VT and then pack all the even elements from Lo and Hi.
+  // Pack all the even elements from Lo and Hi.
   return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
 }
 
@@ -34303,6 +34295,14 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     Known = Known.trunc(BitWidth);
     break;
   }
+  case X86ISD::VBROADCAST: {
+    SDValue Src = Op.getOperand(0);
+    if (!Src.getSimpleValueType().isVector()) {
+      Known = DAG.computeKnownBits(Src, Depth + 1);
+      return;
+    }
+    break;
+  }
   case X86ISD::ANDNP: {
     KnownBits Known2;
     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
@@ -34330,6 +34330,16 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     Known.Zero.setBitsFrom(16);
     break;
   }
+  case X86ISD::PMULUDQ: {
+    KnownBits Known2;
+    Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+
+    Known = Known.trunc(BitWidth / 2).zext(BitWidth);
+    Known2 = Known2.trunc(BitWidth / 2).zext(BitWidth);
+    Known = KnownBits::computeForMul(Known, Known2);
+    break;
+  }
   case X86ISD::CMOV: {
     Known = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
     // If we don't know any bits, early out.
@@ -34527,6 +34537,13 @@ unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
     return 1;
   }
 
+  case X86ISD::VBROADCAST: {
+    SDValue Src = Op.getOperand(0);
+    if (!Src.getSimpleValueType().isVector())
+      return DAG.ComputeNumSignBits(Src, Depth + 1);
+    break;
+  }
+
   case X86ISD::VSHLI: {
     SDValue Src = Op.getOperand(0);
     const APInt &ShiftVal = Op.getConstantOperandAPInt(1);
@@ -35272,6 +35289,20 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
     return CanonicalizeShuffleInput(RootVT, V1);
   }
 
+  // See if the shuffle is a hidden identity shuffle - repeated args in HOPs
+  // etc. can be simplified.
+  if (VT1 == VT2 && VT1.getSizeInBits() == RootSizeInBits) {
+    SmallVector<int> ScaledMask, IdentityMask;
+    unsigned NumElts = VT1.getVectorNumElements();
+    if (BaseMask.size() <= NumElts &&
+        scaleShuffleElements(BaseMask, NumElts, ScaledMask)) {
+      for (unsigned i = 0; i != NumElts; ++i)
+        IdentityMask.push_back(i);
+      if (isTargetShuffleEquivalent(RootVT, ScaledMask, IdentityMask, V1, V2))
+        return CanonicalizeShuffleInput(RootVT, V1);
+    }
+  }
+
   // Handle 128/256-bit lane shuffles of 512-bit vectors.
   if (RootVT.is512BitVector() &&
       (NumBaseMaskElts == 2 || NumBaseMaskElts == 4)) {
@@ -36033,6 +36064,12 @@ static SDValue canonicalizeShuffleMaskWithHorizOp(
   if (!isHoriz && !isPack)
     return SDValue();
 
+  // Do all ops have a single use?
+  bool OneUseOps = llvm::all_of(Ops, [](SDValue Op) {
+    return Op.hasOneUse() &&
+           peekThroughBitcasts(Op) == peekThroughOneUseBitcasts(Op);
+  });
+
   int NumElts = VT0.getVectorNumElements();
   int NumLanes = VT0.getSizeInBits() / 128;
   int NumEltsPerLane = NumElts / NumLanes;
@@ -36127,7 +36164,8 @@ static SDValue canonicalizeShuffleMaskWithHorizOp(
       scaleShuffleElements(TargetMask128, 2, WideMask128)) {
     assert(isUndefOrZeroOrInRange(WideMask128, 0, 4) && "Illegal shuffle");
     bool SingleOp = (Ops.size() == 1);
-    if (!isHoriz || shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) {
+    if (!isHoriz || OneUseOps ||
+        shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) {
       SDValue Lo = isInRange(WideMask128[0], 0, 2) ? BC0 : BC1;
       SDValue Hi = isInRange(WideMask128[1], 0, 2) ? BC0 : BC1;
       Lo = Lo.getOperand(WideMask128[0] & 1);
@@ -36852,7 +36890,9 @@ static SDValue canonicalizeShuffleWithBinOps(SDValue N, SelectionDAG &DAG,
   }
   case X86ISD::VBROADCAST:
   case X86ISD::MOVDDUP:
-  case X86ISD::PSHUFD: {
+  case X86ISD::PSHUFD:
+  case X86ISD::VPERMI:
+  case X86ISD::VPERMILPI: {
     if (N.getOperand(0).getValueType() == ShuffleVT &&
         N->isOnlyUserOf(N.getOperand(0).getNode())) {
       SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
@@ -36882,6 +36922,14 @@ static SDValue canonicalizeShuffleWithBinOps(SDValue N, SelectionDAG &DAG,
     break;
   }
   // Binary and Binary+Permute Shuffles.
+  case X86ISD::INSERTPS: {
+    // Don't merge INSERTPS if it contains zero'd elements.
+    unsigned InsertPSMask = N.getConstantOperandVal(2);
+    unsigned ZeroMask = InsertPSMask & 0xF;
+    if (ZeroMask != 0)
+      break;
+    LLVM_FALLTHROUGH;
+  }
   case X86ISD::BLENDI:
   case X86ISD::SHUFP:
   case X86ISD::UNPCKH:
@@ -37822,28 +37870,15 @@ static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG,
   return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask);
 }
 
-/// Eliminate a redundant shuffle of a horizontal math op.
+// Eliminate a redundant shuffle of a horizontal math op.
+// TODO: Merge this into canonicalizeShuffleMaskWithHorizOp.
 static SDValue foldShuffleOfHorizOp(SDNode *N, SelectionDAG &DAG) {
-  // TODO: Can we use getTargetShuffleInputs instead?
   unsigned Opcode = N->getOpcode();
-  if (Opcode != X86ISD::MOVDDUP && Opcode != X86ISD::VBROADCAST)
-    if (Opcode != X86ISD::UNPCKL && Opcode != X86ISD::UNPCKH)
-      if (Opcode != X86ISD::SHUFP)
-        if (Opcode != ISD::VECTOR_SHUFFLE || !N->getOperand(1).isUndef())
-          return SDValue();
+  if (Opcode != X86ISD::UNPCKL && Opcode != X86ISD::UNPCKH)
+    if (Opcode != X86ISD::SHUFP)
+      return SDValue();
 
-  // For a broadcast, peek through an extract element of index 0 to find the
-  // horizontal op: broadcast (ext_vec_elt HOp, 0)
   EVT VT = N->getValueType(0);
-  if (Opcode == X86ISD::VBROADCAST) {
-    SDValue SrcOp = N->getOperand(0);
-    if (SrcOp.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
-        SrcOp.getValueType() == MVT::f64 &&
-        SrcOp.getOperand(0).getValueType() == VT &&
-        isNullConstant(SrcOp.getOperand(1)))
-      N = SrcOp.getNode();
-  }
-
   SDValue HOp = N->getOperand(0);
   if (HOp.getOpcode() != X86ISD::HADD && HOp.getOpcode() != X86ISD::FHADD &&
       HOp.getOpcode() != X86ISD::HSUB && HOp.getOpcode() != X86ISD::FHSUB)
@@ -37897,67 +37932,6 @@ static SDValue foldShuffleOfHorizOp(SDNode *N, SelectionDAG &DAG) {
     return SDValue();
   }
 
-  // 128-bit horizontal math instructions are defined to operate on adjacent
-  // lanes of each operand as:
-  // v4X32: A[0] + A[1] , A[2] + A[3] , B[0] + B[1] , B[2] + B[3]
-  // ...similarly for v2f64 and v8i16.
-  if (!HOp.getOperand(0).isUndef() && !HOp.getOperand(1).isUndef() &&
-      HOp.getOperand(0) != HOp.getOperand(1))
-    return SDValue();
-
-  // The shuffle that we are eliminating may have allowed the horizontal op to
-  // have an undemanded (undefined) operand. Duplicate the other (defined)
-  // operand to ensure that the results are defined across all lanes without the
-  // shuffle.
-  auto updateHOp = [](SDValue HorizOp, SelectionDAG &DAG) {
-    SDValue X;
-    if (HorizOp.getOperand(0).isUndef()) {
-      assert(!HorizOp.getOperand(1).isUndef() && "Not expecting foldable h-op");
-      X = HorizOp.getOperand(1);
-    } else if (HorizOp.getOperand(1).isUndef()) {
-      assert(!HorizOp.getOperand(0).isUndef() && "Not expecting foldable h-op");
-      X = HorizOp.getOperand(0);
-    } else {
-      return HorizOp;
-    }
-    return DAG.getNode(HorizOp.getOpcode(), SDLoc(HorizOp),
-                       HorizOp.getValueType(), X, X);
-  };
-
-  // When the operands of a horizontal math op are identical, the low half of
-  // the result is the same as the high half. If a target shuffle is also
-  // replicating low and high halves (and without changing the type/length of
-  // the vector), we don't need the shuffle.
-  if (Opcode == X86ISD::MOVDDUP || Opcode == X86ISD::VBROADCAST) {
-    if (HOp.getScalarValueSizeInBits() == 64 && HOp.getValueType() == VT) {
-      // movddup (hadd X, X) --> hadd X, X
-      // broadcast (extract_vec_elt (hadd X, X), 0) --> hadd X, X
-      assert((HOp.getValueType() == MVT::v2f64 ||
-              HOp.getValueType() == MVT::v4f64) && "Unexpected type for h-op");
-      return updateHOp(HOp, DAG);
-    }
-    return SDValue();
-  }
-
-  // shuffle (hadd X, X), undef, [low half...high half] --> hadd X, X
-  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
-
-  // TODO: Other mask possibilities like {1,1} and {1,0} could be added here,
-  // but this should be tied to whatever horizontal op matching and shuffle
-  // canonicalization are producing.
-  if (HOp.getValueSizeInBits() == 128 &&
-      (isShuffleEquivalent(Mask, {0, 0}) ||
-       isShuffleEquivalent(Mask, {0, 1, 0, 1}) ||
-       isShuffleEquivalent(Mask, {0, 1, 2, 3, 0, 1, 2, 3})))
-    return updateHOp(HOp, DAG);
-
-  if (HOp.getValueSizeInBits() == 256 &&
-      (isShuffleEquivalent(Mask, {0, 0, 2, 2}) ||
-       isShuffleEquivalent(Mask, {0, 1, 0, 1, 4, 5, 4, 5}) ||
-       isShuffleEquivalent(
-           Mask, {0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11})))
-    return updateHOp(HOp, DAG);
-
   return SDValue();
 }
 
@@ -38924,6 +38898,19 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
     if (SimplifyDemandedBits(Src, OriginalDemandedBits, DemandedElts, Known,
                              TLO, Depth + 1))
       return true;
+    // If we don't need the upper bits, attempt to narrow the broadcast source.
+    // Don't attempt this on AVX512 as it might affect broadcast folding.
+    // TODO: Should we attempt this for i32/i16 splats? They tend to be slower.
+    if ((BitWidth == 64) && SrcVT.isScalarInteger() && !Subtarget.hasAVX512() &&
+        OriginalDemandedBits.countLeadingZeros() >= (BitWidth / 2)) {
+      MVT NewSrcVT = MVT::getIntegerVT(BitWidth / 2);
+      SDValue NewSrc =
+          TLO.DAG.getNode(ISD::TRUNCATE, SDLoc(Src), NewSrcVT, Src);
+      MVT NewVT = MVT::getVectorVT(NewSrcVT, VT.getVectorNumElements() * 2);
+      SDValue NewBcst =
+          TLO.DAG.getNode(X86ISD::VBROADCAST, SDLoc(Op), NewVT, NewSrc);
+      return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, NewBcst));
+    }
     break;
   }
   case X86ISD::PCMPGT:
@@ -39147,17 +39134,22 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
       Op, DemandedBits, DemandedElts, DAG, Depth);
 }
 
-// Helper to peek through bitops/setcc to determine size of source vector.
+// Helper to peek through bitops/trunc/setcc to determine size of source vector.
 // Allows combineBitcastvxi1 to determine what size vector generated a <X x i1>.
-static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size) {
+static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
+                                      bool AllowTruncate) {
   switch (Src.getOpcode()) {
+  case ISD::TRUNCATE:
+    if (!AllowTruncate)
+      return false;
+    LLVM_FALLTHROUGH;
   case ISD::SETCC:
     return Src.getOperand(0).getValueSizeInBits() == Size;
   case ISD::AND:
   case ISD::XOR:
   case ISD::OR:
-    return checkBitcastSrcVectorSize(Src.getOperand(0), Size) &&
-           checkBitcastSrcVectorSize(Src.getOperand(1), Size);
+    return checkBitcastSrcVectorSize(Src.getOperand(0), Size, AllowTruncate) &&
+           checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate);
   }
   return false;
 }
@@ -39212,6 +39204,7 @@ static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
                                           SDValue Src, const SDLoc &DL) {
   switch (Src.getOpcode()) {
   case ISD::SETCC:
+  case ISD::TRUNCATE:
     return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
   case ISD::AND:
   case ISD::XOR:
@@ -39295,7 +39288,8 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
     SExtVT = MVT::v4i32;
     // For cases such as (i4 bitcast (v4i1 setcc v4i64 v1, v2))
     // sign-extend to a 256-bit operation to avoid truncation.
-    if (Subtarget.hasAVX() && checkBitcastSrcVectorSize(Src, 256)) {
+    if (Subtarget.hasAVX() &&
+        checkBitcastSrcVectorSize(Src, 256, Subtarget.hasAVX2())) {
       SExtVT = MVT::v4i64;
       PropagateSExt = true;
     }
@@ -39307,8 +39301,8 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
     // If the setcc operand is 128-bit, prefer sign-extending to 128-bit over
     // 256-bit because the shuffle is cheaper than sign extending the result of
     // the compare.
-    if (Subtarget.hasAVX() && (checkBitcastSrcVectorSize(Src, 256) ||
-                               checkBitcastSrcVectorSize(Src, 512))) {
+    if (Subtarget.hasAVX() && (checkBitcastSrcVectorSize(Src, 256, true) ||
+                               checkBitcastSrcVectorSize(Src, 512, true))) {
       SExtVT = MVT::v8i32;
       PropagateSExt = true;
     }
@@ -39333,7 +39327,7 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
       break;
     }
     // Split if this is a <64 x i8> comparison result.
-    if (checkBitcastSrcVectorSize(Src, 512)) {
+    if (checkBitcastSrcVectorSize(Src, 512, false)) {
       SExtVT = MVT::v64i8;
       break;
     }
@@ -43154,35 +43148,41 @@ static SDValue combineHorizOpWithShuffle(SDNode *N, SelectionDAG &DAG,
   SDValue N1 = N->getOperand(1);
   EVT SrcVT = N0.getValueType();
 
+  SDValue BC0 = peekThroughBitcasts(N0);
+  SDValue BC1 = peekThroughBitcasts(N1);
+
   // Attempt to fold HOP(LOSUBVECTOR(SHUFFLE(X)),HISUBVECTOR(SHUFFLE(X)))
   // to SHUFFLE(HOP(LOSUBVECTOR(X),HISUBVECTOR(X))), this is mainly for
   // truncation trees that help us avoid lane crossing shuffles.
   // TODO: There's a lot more we can do for PACK/HADD style shuffle combines.
   // TODO: We don't handle vXf64 shuffles yet.
-  if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
-      N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
-      N0.getConstantOperandAPInt(1) == 0 &&
-      N1.getConstantOperandAPInt(1) == SrcVT.getVectorNumElements() &&
-      N0.getOperand(0) == N1.getOperand(0) && VT.is128BitVector() &&
-      N0.getOperand(0).getValueType().is256BitVector() &&
-      SrcVT.getScalarSizeInBits() <= 32) {
-    // TODO - support target/faux shuffles.
-    SDValue Vec = peekThroughBitcasts(N0.getOperand(0));
-    if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Vec)) {
+  if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32 &&
+      BC0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+      BC1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+      BC0.getOperand(0) == BC1.getOperand(0) &&
+      BC0.getOperand(0).getValueType().is256BitVector() &&
+      BC0.getConstantOperandAPInt(1) == 0 &&
+      BC1.getConstantOperandAPInt(1) ==
+          BC0.getValueType().getVectorNumElements()) {
+    SmallVector<SDValue> ShuffleOps;
+    SmallVector<int> ShuffleMask, ScaledMask;
+    SDValue Vec = peekThroughBitcasts(BC0.getOperand(0));
+    if (getTargetShuffleInputs(Vec, ShuffleOps, ShuffleMask, DAG)) {
+      resolveTargetShuffleInputsAndMask(ShuffleOps, ShuffleMask);
       // To keep the HOP LHS/RHS coherency, we must be able to scale the unary
-      // shuffle to a vXi64 width - we can probably relax this in the future.
-      SmallVector<int, 4> ShuffleMask;
-      if (SVN->getOperand(1).isUndef() &&
-          scaleShuffleElements(SVN->getMask(), 4, ShuffleMask)) {
+      // shuffle to a v4X64 width - we can probably relax this in the future.
+      if (!isAnyZero(ShuffleMask) && ShuffleOps.size() == 1 &&
+          ShuffleOps[0].getValueType().is256BitVector() &&
+          scaleShuffleElements(ShuffleMask, 4, ScaledMask)) {
         SDLoc DL(N);
         SDValue Lo, Hi;
         MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32;
-        std::tie(Lo, Hi) = DAG.SplitVector(SVN->getOperand(0), DL);
-        Lo = DAG.getBitcast(N0.getValueType(), Lo);
-        Hi = DAG.getBitcast(N1.getValueType(), Hi);
+        std::tie(Lo, Hi) = DAG.SplitVector(ShuffleOps[0], DL);
+        Lo = DAG.getBitcast(SrcVT, Lo);
+        Hi = DAG.getBitcast(SrcVT, Hi);
         SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi);
         Res = DAG.getBitcast(ShufVT, Res);
-        Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ShuffleMask);
+        Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ScaledMask);
         return DAG.getBitcast(VT, Res);
       }
     }
@@ -46086,10 +46086,8 @@ static SDValue combineVectorTruncation(SDNode *N, SelectionDAG &DAG,
   EVT InVT = In.getValueType();
   unsigned NumElems = OutVT.getVectorNumElements();
 
-  // TODO: On AVX2, the behavior of X86ISD::PACKUS is different from that on
-  // SSE2, and we need to take care of it specially.
-  // AVX512 provides vpmovdb.
-  if (!Subtarget.hasSSE2() || Subtarget.hasAVX2())
+  // AVX512 provides fast truncate ops.
+  if (!Subtarget.hasSSE2() || Subtarget.hasAVX512())
     return SDValue();
 
   EVT OutSVT = OutVT.getVectorElementType();
@@ -49035,10 +49033,17 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
     SDValue N01In = N01Elt.getOperand(0);
     SDValue N10In = N10Elt.getOperand(0);
     SDValue N11In = N11Elt.getOperand(0);
+
     // First time we find an input capture it.
     if (!In0) {
       In0 = N00In;
       In1 = N01In;
+
+      // The input vector sizes must match the output.
+      // TODO: Insert cast ops to allow different types.
+      if (In0.getValueSizeInBits() != VT.getSizeInBits() ||
+          In1.getValueSizeInBits() != VT.getSizeInBits())
+        return SDValue();
     }
     // Mul is commutative so the input vectors can be in any order.
     // Canonicalize to make the compares easier.
diff --git a/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp b/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp
index 9b6e54682f8cd8dcd80ee45ebf39cdc932a6d79d..cebcbf01ed75d078630a9d528bb6f5cb932bc745 100644
--- a/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp
@@ -43,17 +43,55 @@ using namespace PatternMatch;
 
 #define DEBUG_TYPE "lower-amx-intrinsics"
 
+#ifndef NDEBUG
 static bool isV256I32Ty(Type *Ty) {
   if (auto *FVT = dyn_cast<FixedVectorType>(Ty))
     return FVT->getNumElements() == 256 &&
            FVT->getElementType()->isIntegerTy(32);
   return false;
 }
+#endif
 
-static BasicBlock *createLoop(BasicBlock *Preheader, BasicBlock *Exit,
-                              Value *Bound, Value *Step, StringRef Name,
-                              IRBuilderBase &B, DomTreeUpdater &DTU, Loop *L,
-                              LoopInfo &LI) {
+namespace {
+class X86LowerAMXIntrinsics {
+  Function &Func;
+
+public:
+  X86LowerAMXIntrinsics(Function &F, DomTreeUpdater &DomTU, LoopInfo *LoopI)
+      : Func(F), DTU(DomTU), LI(LoopI) {}
+  bool visit();
+
+private:
+  DomTreeUpdater &DTU;
+  LoopInfo *LI;
+  BasicBlock *createLoop(BasicBlock *Preheader, BasicBlock *Exit, Value *Bound,
+                         Value *Step, StringRef Name, IRBuilderBase &B,
+                         Loop *L);
+  template <bool IsTileLoad>
+  Value *createTileLoadStoreLoops(BasicBlock *Start, BasicBlock *End,
+                                  IRBuilderBase &B, Value *Row, Value *Col,
+                                  Value *Ptr, Value *Stride, Value *Tile);
+  template <Intrinsic::ID IntrID>
+  typename std::enable_if<IntrID == Intrinsic::x86_tdpbssd_internal ||
+                              IntrID == Intrinsic::x86_tdpbf16ps_internal,
+                          Value *>::type
+  createTileDPLoops(BasicBlock *Start, BasicBlock *End, IRBuilderBase &B,
+                    Value *Row, Value *Col, Value *K, Value *Acc, Value *LHS,
+                    Value *RHS);
+  template <bool IsTileLoad>
+  bool lowerTileLoadStore(Instruction *TileLoadStore);
+  template <Intrinsic::ID IntrID>
+  typename std::enable_if<IntrID == Intrinsic::x86_tdpbssd_internal ||
+                              IntrID == Intrinsic::x86_tdpbf16ps_internal,
+                          bool>::type
+  lowerTileDP(Instruction *TileDP);
+  bool lowerTileZero(Instruction *TileZero);
+};
+
+BasicBlock *X86LowerAMXIntrinsics::createLoop(BasicBlock *Preheader,
+                                              BasicBlock *Exit, Value *Bound,
+                                              Value *Step, StringRef Name,
+                                              IRBuilderBase &B, Loop *L) {
   LLVMContext &Ctx = Preheader->getContext();
   BasicBlock *Header =
       BasicBlock::Create(Ctx, Name + ".header", Preheader->getParent(), Exit);
@@ -86,35 +124,37 @@ static BasicBlock *createLoop(BasicBlock *Preheader, BasicBlock *Exit,
       {DominatorTree::Insert, Latch, Exit},
       {DominatorTree::Insert, Preheader, Header},
   });
-
-  L->addBasicBlockToLoop(Header, LI);
-  L->addBasicBlockToLoop(Body, LI);
-  L->addBasicBlockToLoop(Latch, LI);
+  if (LI) {
+    L->addBasicBlockToLoop(Header, *LI);
+    L->addBasicBlockToLoop(Body, *LI);
+    L->addBasicBlockToLoop(Latch, *LI);
+  }
   return Body;
 }
 
 template <bool IsTileLoad>
-static Value *createTileLoadStoreLoops(BasicBlock *Start, BasicBlock *End,
-                                       IRBuilderBase &B, DomTreeUpdater &DTU,
-                                       LoopInfo &LI, Value *Row, Value *Col,
-                                       Value *Ptr, Value *Stride, Value *Tile) {
+Value *X86LowerAMXIntrinsics::createTileLoadStoreLoops(
+    BasicBlock *Start, BasicBlock *End, IRBuilderBase &B, Value *Row,
+    Value *Col, Value *Ptr, Value *Stride, Value *Tile) {
   std::string IntrinName = IsTileLoad ? "tileload" : "tilestore";
-  Loop *RowLoop = LI.AllocateLoop();
-  Loop *ColLoop = LI.AllocateLoop();
-  RowLoop->addChildLoop(ColLoop);
-  if (Loop *ParentL = LI.getLoopFor(Start))
-    ParentL->addChildLoop(RowLoop);
-  else
-    LI.addTopLevelLoop(RowLoop);
+  Loop *RowLoop = nullptr;
+  Loop *ColLoop = nullptr;
+  if (LI) {
+    RowLoop = LI->AllocateLoop();
+    ColLoop = LI->AllocateLoop();
+    RowLoop->addChildLoop(ColLoop);
+    if (Loop *ParentL = LI->getLoopFor(Start))
+      ParentL->addChildLoop(RowLoop);
+    else
+      LI->addTopLevelLoop(RowLoop);
+  }
 
-  BasicBlock *RowBody =
-      createLoop(Start, End, Row, B.getInt16(1), IntrinName + ".scalarize.rows",
-                 B, DTU, RowLoop, LI);
+  BasicBlock *RowBody = createLoop(Start, End, Row, B.getInt16(1),
+                                   IntrinName + ".scalarize.rows", B, RowLoop);
   BasicBlock *RowLatch = RowBody->getSingleSuccessor();
 
-  BasicBlock *ColBody =
-      createLoop(RowBody, RowLatch, Col, B.getInt16(1),
-                 IntrinName + ".scalarize.cols", B, DTU, ColLoop, LI);
+  BasicBlock *ColBody = createLoop(RowBody, RowLatch, Col, B.getInt16(1),
+                                   IntrinName + ".scalarize.cols", B, ColLoop);
 
   BasicBlock *ColLoopLatch = ColBody->getSingleSuccessor();
   BasicBlock *ColLoopHeader = ColBody->getSinglePredecessor();
@@ -181,35 +221,44 @@ static Value *createTileLoadStoreLoops(BasicBlock *Start, BasicBlock *End,
   }
 }
 
-static Value *createTileDPBSSDLoops(BasicBlock *Start, BasicBlock *End,
-                                    IRBuilderBase &B, DomTreeUpdater &DTU,
-                                    LoopInfo &LI, Value *Row, Value *Col,
-                                    Value *K, Value *Acc, Value *LHS,
-                                    Value *RHS) {
-  Loop *RowLoop = LI.AllocateLoop();
-  Loop *ColLoop = LI.AllocateLoop();
-  Loop *InnerLoop = LI.AllocateLoop();
-  ColLoop->addChildLoop(InnerLoop);
-  RowLoop->addChildLoop(ColLoop);
-  if (Loop *ParentL = LI.getLoopFor(Start))
-    ParentL->addChildLoop(RowLoop);
-  else
-    LI.addTopLevelLoop(RowLoop);
+template <Intrinsic::ID IntrID>
+typename std::enable_if<IntrID == Intrinsic::x86_tdpbssd_internal ||
+                            IntrID == Intrinsic::x86_tdpbf16ps_internal,
+                        Value *>::type
+X86LowerAMXIntrinsics::createTileDPLoops(BasicBlock *Start, BasicBlock *End,
+                                         IRBuilderBase &B, Value *Row,
+                                         Value *Col, Value *K, Value *Acc,
+                                         Value *LHS, Value *RHS) {
+  std::string IntrinName =
+      IntrID == Intrinsic::x86_tdpbssd_internal ? "tiledpbssd" : "tdpbf16ps";
+  Loop *RowLoop = nullptr;
+  Loop *ColLoop = nullptr;
+  Loop *InnerLoop = nullptr;
+  if (LI) {
+    RowLoop = LI->AllocateLoop();
+    ColLoop = LI->AllocateLoop();
+    InnerLoop = LI->AllocateLoop();
+    ColLoop->addChildLoop(InnerLoop);
+    RowLoop->addChildLoop(ColLoop);
+    if (Loop *ParentL = LI->getLoopFor(Start))
+      ParentL->addChildLoop(RowLoop);
+    else
+      LI->addTopLevelLoop(RowLoop);
+  }
 
-  BasicBlock *RowBody =
-      createLoop(Start, End, Row, B.getInt16(1), "tiledpbssd.scalarize.rows", B,
-                 DTU, RowLoop, LI);
+  BasicBlock *RowBody = createLoop(Start, End, Row, B.getInt16(1),
+                                   IntrinName + ".scalarize.rows", B, RowLoop);
   BasicBlock *RowLatch = RowBody->getSingleSuccessor();
 
-  BasicBlock *ColBody =
-      createLoop(RowBody, RowLatch, Col, B.getInt16(1),
-                 "tiledpbssd.scalarize.cols", B, DTU, ColLoop, LI);
+  BasicBlock *ColBody = createLoop(RowBody, RowLatch, Col, B.getInt16(1),
+                                   IntrinName + ".scalarize.cols", B, ColLoop);
+
   BasicBlock *ColLoopLatch = ColBody->getSingleSuccessor();
 
   B.SetInsertPoint(ColBody->getTerminator());
   BasicBlock *InnerBody =
       createLoop(ColBody, ColLoopLatch, K, B.getInt16(1),
-                 "tiledpbssd.scalarize.inner", B, DTU, InnerLoop, LI);
+                 IntrinName + ".scalarize.inner", B, InnerLoop);
 
   BasicBlock *ColLoopHeader = ColBody->getSinglePredecessor();
   BasicBlock *RowLoopHeader = RowBody->getSinglePredecessor();
@@ -273,39 +322,82 @@ static Value *createTileDPBSSDLoops(BasicBlock *Start, BasicBlock *End,
   PHINode *VecCPhi = B.CreatePHI(V256I32Ty, 2, "vec.c.inner.phi");
   VecCPhi->addIncoming(VecCPhiColLoop, ColBody);
 
-  // tiledpbssd.scalarize.inner.body:
-  // calculate idxa, idxb
-  // %eltc = extractelement <256 x i32> %vec.c.inner.phi, i16 %idxc
-  // %elta = extractelement <256 x i32> %veca, i16 %idxa
-  // %eltav4i8 = bitcast i32 %elta to <4 x i8>
-  // %eltb = extractelement <256 x i32> %vecb, i16 %idxb
-  // %eltbv4i8 = bitcast i32 %eltb to <4 x i8>
-  // %eltav4i32 = sext <4 x i8> %eltav4i8 to <4 x i32>
-  // %eltbv4i32 = sext <4 x i8> %eltbv4i8 to <4 x i32>
-  // %mulab = mul <4 x i32> %eltbv4i32, %eltav4i32
-  // %acc = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %131)
-  // %neweltc = add i32 %elt, %acc
-  // %NewVecC = insertelement <256 x i32> %vec.c.inner.phi, i32 %neweltc,
-  // i16 %idxc
-
   B.SetInsertPoint(InnerBody->getTerminator());
   Value *IdxA =
       B.CreateAdd(B.CreateMul(CurrentRow, B.getInt16(16)), CurrentInner);
   Value *IdxB =
       B.CreateAdd(B.CreateMul(CurrentInner, B.getInt16(16)), CurrentCol);
-
-  FixedVectorType *V4I8Ty = FixedVectorType::get(B.getInt8Ty(), 4);
-  FixedVectorType *V4I32Ty = FixedVectorType::get(B.getInt32Ty(), 4);
-  Value *EltC = B.CreateExtractElement(VecCPhi, IdxC);
-  Value *EltA = B.CreateExtractElement(VecA, IdxA);
-  Value *SubVecA = B.CreateBitCast(EltA, V4I8Ty);
-  Value *EltB = B.CreateExtractElement(VecB, IdxB);
-  Value *SubVecB = B.CreateBitCast(EltB, V4I8Ty);
-  Value *SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty);
-  Value *SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty);
-  Value *SubVecR = B.CreateAddReduce(B.CreateMul(SEXTSubVecA, SEXTSubVecB));
-  Value *ResElt = B.CreateAdd(EltC, SubVecR);
-  Value *NewVecC = B.CreateInsertElement(VecCPhi, ResElt, IdxC);
+  Value *NewVecC = nullptr;
+
+  if (IntrID == Intrinsic::x86_tdpbssd_internal) {
+    // tiledpbssd.scalarize.inner.body:
+    // calculate idxa, idxb
+    // %eltc = extractelement <256 x i32> %vec.c.inner.phi, i16 %idxc
+    // %elta = extractelement <256 x i32> %veca, i16 %idxa
+    // %eltav4i8 = bitcast i32 %elta to <4 x i8>
+    // %eltb = extractelement <256 x i32> %vecb, i16 %idxb
+    // %eltbv4i8 = bitcast i32 %eltb to <4 x i8>
+    // %eltav4i32 = sext <4 x i8> %eltav4i8 to <4 x i32>
+    // %eltbv4i32 = sext <4 x i8> %eltbv4i8 to <4 x i32>
+    // %mulab = mul <4 x i32> %eltbv4i32, %eltav4i32
+    // %acc = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %131)
+    // %neweltc = add i32 %elt, %acc
+    // %NewVecC = insertelement <256 x i32> %vec.c.inner.phi, i32 %neweltc,
+    // i16 %idxc
+    FixedVectorType *V4I8Ty = FixedVectorType::get(B.getInt8Ty(), 4);
+    FixedVectorType *V4I32Ty = FixedVectorType::get(B.getInt32Ty(), 4);
+    Value *EltC = B.CreateExtractElement(VecCPhi, IdxC);
+    Value *EltA = B.CreateExtractElement(VecA, IdxA);
+    Value *SubVecA = B.CreateBitCast(EltA, V4I8Ty);
+    Value *EltB = B.CreateExtractElement(VecB, IdxB);
+    Value *SubVecB = B.CreateBitCast(EltB, V4I8Ty);
+    Value *SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty);
+    Value *SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty);
+    Value *SubVecR = B.CreateAddReduce(B.CreateMul(SEXTSubVecA, SEXTSubVecB));
+    Value *ResElt = B.CreateAdd(EltC, SubVecR);
+    NewVecC = B.CreateInsertElement(VecCPhi, ResElt, IdxC);
+  } else if (IntrID == Intrinsic::x86_tdpbf16ps_internal) {
+    // tiledpbf16ps.scalarize.inner.body:
+    // calculate idxa, idxb, idxc
+    // %eltc = extractelement <256 x i32> %vec.c.inner.phi, i16 %idxc
+    // %eltcf32 = bitcast i32 %eltc to float
+    // %elta = extractelement <256 x i32> %veca, i16 %idxa
+    // %eltav2i16 = bitcast i32 %elta to <2 x i16>
+    // %eltb = extractelement <256 x i32> %vecb, i16 %idxb
+    // %eltbv2i16 = bitcast i32 %eltb to <2 x i16>
+    // %shufflea = shufflevector <2 x i16> %elta, <2 x i16> zeroinitializer, <4
+    // x i32> <i32 2, i32 0, i32 3, i32 1>
+    // %eltav2f32 = bitcast <4 x i16> %shufflea to <2 x float>
+    // %shuffleb = shufflevector <2 x i16> %eltb, <2 xi16> zeroinitializer, <4 x
+    // i32> <i32 2, i32 0, i32 3, i32 1>
+    // %eltbv2f32 = bitcast <4 x i16> %shuffleb to <2 x float>
+    // %mulab = fmul <2 x float> %eltav2f32, %eltbv2f32
+    // %acc = call float
+    // @llvm.vector.reduce.fadd.v2f32(float %eltcf32, <2 x float> %mulab)
+    // %neweltc = bitcast float %acc to i32
+    // %NewVecC = insertelement <256 x i32> %vec.c.inner.phi, i32 %neweltc,
+    // i16 %idxc
+    // %NewVecD = insertelement <256 x i32> %vec.d.inner.phi, i32 %neweltc,
+    // i16 %idxc
+    FixedVectorType *V2I16Ty = FixedVectorType::get(B.getInt16Ty(), 2);
+    FixedVectorType *V2F32Ty = FixedVectorType::get(B.getFloatTy(), 2);
+    Value *EltC = B.CreateExtractElement(VecCPhi, IdxC);
+    Value *EltCF32 = B.CreateBitCast(EltC, B.getFloatTy());
+    Value *EltA = B.CreateExtractElement(VecA, IdxA);
+    Value *SubVecA = B.CreateBitCast(EltA, V2I16Ty);
+    Value *EltB = B.CreateExtractElement(VecB, IdxB);
+    Value *SubVecB = B.CreateBitCast(EltB, V2I16Ty);
+    Value *ZeroV2I16 = Constant::getNullValue(V2I16Ty);
+    int ShuffleMask[4] = {2, 0, 3, 1};
+    auto ShuffleArray = makeArrayRef(ShuffleMask);
+    Value *AV2F32 = B.CreateBitCast(
+        B.CreateShuffleVector(SubVecA, ZeroV2I16, ShuffleArray), V2F32Ty);
+    Value *BV2F32 = B.CreateBitCast(
+        B.CreateShuffleVector(SubVecB, ZeroV2I16, ShuffleArray), V2F32Ty);
+    Value *SubVecR = B.CreateFAddReduce(EltCF32, B.CreateFMul(AV2F32, BV2F32));
+    Value *ResElt = B.CreateBitCast(SubVecR, B.getInt32Ty());
+    NewVecC = B.CreateInsertElement(VecCPhi, ResElt, IdxC);
+  }
 
   // tiledpbssd.scalarize.cols.latch:
   // %NewEltC = extractelement <256 x i32> %vec.c.phi.col, i16 %idxc
@@ -324,33 +416,17 @@ static Value *createTileDPBSSDLoops(BasicBlock *Start, BasicBlock *End,
   return NewVecD;
 }
 
-namespace {
-class X86LowerAMXIntrinsics {
-  Function &Func;
-
-public:
-  X86LowerAMXIntrinsics(Function &F, DominatorTree *DT, LoopInfo *LI)
-      : Func(F), DT(DT), LI(LI) {}
-  bool visit();
-
-private:
-  DominatorTree *DT;
-  LoopInfo *LI;
-  template <bool IsTileLoad>
-  bool lowerTileLoadStore(Instruction *TileLoadStore);
-  bool lowerTileDPBSSD(Instruction *TileDPBSSD);
-  bool lowerTileZero(Instruction *TileZero);
-};
-
-bool X86LowerAMXIntrinsics::lowerTileDPBSSD(Instruction *TileDPBSSD) {
+template <Intrinsic::ID IntrID>
+typename std::enable_if<IntrID == Intrinsic::x86_tdpbssd_internal ||
+                            IntrID == Intrinsic::x86_tdpbf16ps_internal,
+                        bool>::type
+X86LowerAMXIntrinsics::lowerTileDP(Instruction *TileDP) {
   Value *M, *N, *K, *C, *A, *B;
-  match(TileDPBSSD, m_Intrinsic<Intrinsic::x86_tdpbssd_internal>(
-                        m_Value(M), m_Value(N), m_Value(K), m_Value(C),
-                        m_Value(A), m_Value(B)));
-  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
-  Instruction *InsertI = TileDPBSSD;
-  IRBuilder<> PreBuilder(TileDPBSSD);
-  PreBuilder.SetInsertPoint(TileDPBSSD);
+  match(TileDP, m_Intrinsic<IntrID>(m_Value(M), m_Value(N), m_Value(K),
+                                    m_Value(C), m_Value(A), m_Value(B)));
+  Instruction *InsertI = TileDP;
+  IRBuilder<> PreBuilder(TileDP);
+  PreBuilder.SetInsertPoint(TileDP);
   // We visit the loop with (m, n/4, k/4):
   // %n_dword = lshr i16 %n, 2
   // %k_dword = lshr i16 %k, 2
@@ -358,18 +434,17 @@ bool X86LowerAMXIntrinsics::lowerTileDPBSSD(Instruction *TileDPBSSD) {
   Value *KDWord = PreBuilder.CreateLShr(K, PreBuilder.getInt16(2));
   BasicBlock *Start = InsertI->getParent();
   BasicBlock *End =
-      SplitBlock(InsertI->getParent(), InsertI, DT, LI, nullptr, "continue");
-  IRBuilder<> Builder(TileDPBSSD);
-  Value *ResVec = createTileDPBSSDLoops(Start, End, Builder, DTU, *LI, M,
-                                        NDWord, KDWord, C, A, B);
+      SplitBlock(InsertI->getParent(), InsertI, &DTU, LI, nullptr, "continue");
+  IRBuilder<> Builder(TileDP);
+  Value *ResVec = createTileDPLoops<IntrID>(Start, End, Builder, M, NDWord,
+                                            KDWord, C, A, B);
   // we cannot assume there always be bitcast after tiledpbssd. So we need to
   // insert one bitcast as required
   Builder.SetInsertPoint(End->getFirstNonPHI());
   Value *ResAMX =
       Builder.CreateBitCast(ResVec, Type::getX86_AMXTy(Builder.getContext()));
-  // Delete tiledpbssd intrinsic and do some clean-up.
-  for (auto UI = TileDPBSSD->use_begin(), UE = TileDPBSSD->use_end();
-       UI != UE;) {
+  // Delete TileDP intrinsic and do some clean-up.
+  for (auto UI = TileDP->use_begin(), UE = TileDP->use_end(); UI != UE;) {
     Instruction *I = cast<Instruction>((UI++)->getUser());
     Value *Vec;
     if (match(I, m_BitCast(m_Value(Vec)))) {
@@ -377,8 +452,8 @@ bool X86LowerAMXIntrinsics::lowerTileDPBSSD(Instruction *TileDPBSSD) {
       I->eraseFromParent();
     }
   }
-  TileDPBSSD->replaceAllUsesWith(ResAMX);
-  TileDPBSSD->eraseFromParent();
+  TileDP->replaceAllUsesWith(ResAMX);
+  TileDP->eraseFromParent();
   return true;
 }
 
@@ -394,7 +469,6 @@ bool X86LowerAMXIntrinsics::lowerTileLoadStore(Instruction *TileLoadStore) {
                              m_Value(M), m_Value(N), m_Value(Ptr),
                              m_Value(Stride), m_Value(Tile)));
 
-  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
   Instruction *InsertI = TileLoadStore;
   IRBuilder<> PreBuilder(TileLoadStore);
   PreBuilder.SetInsertPoint(TileLoadStore);
@@ -402,10 +476,10 @@ bool X86LowerAMXIntrinsics::lowerTileLoadStore(Instruction *TileLoadStore) {
   Value *StrideDWord = PreBuilder.CreateLShr(Stride, PreBuilder.getInt64(2));
   BasicBlock *Start = InsertI->getParent();
   BasicBlock *End =
-      SplitBlock(InsertI->getParent(), InsertI, DT, LI, nullptr, "continue");
+      SplitBlock(InsertI->getParent(), InsertI, &DTU, LI, nullptr, "continue");
   IRBuilder<> Builder(TileLoadStore);
   Value *ResVec = createTileLoadStoreLoops<IsTileLoad>(
-      Start, End, Builder, DTU, *LI, M, NDWord, Ptr, StrideDWord,
+      Start, End, Builder, M, NDWord, Ptr, StrideDWord,
       IsTileLoad ? nullptr : Tile);
   if (IsTileLoad) {
     // we cannot assume there always be bitcast after tileload. So we need to
@@ -456,6 +530,7 @@ bool X86LowerAMXIntrinsics::visit() {
         case Intrinsic::x86_tileloadd64_internal:
         case Intrinsic::x86_tilestored64_internal:
         case Intrinsic::x86_tilezero_internal:
+        case Intrinsic::x86_tdpbf16ps_internal:
           WorkList.push_back(Inst);
           break;
         default:
@@ -468,7 +543,10 @@ bool X86LowerAMXIntrinsics::visit() {
   for (auto *Inst : WorkList) {
     switch (Inst->getIntrinsicID()) {
     case Intrinsic::x86_tdpbssd_internal:
-      C = lowerTileDPBSSD(Inst) || C;
+      C = lowerTileDP<Intrinsic::x86_tdpbssd_internal>(Inst) || C;
+      break;
+    case Intrinsic::x86_tdpbf16ps_internal:
+      C = lowerTileDP<Intrinsic::x86_tdpbf16ps_internal>(Inst) || C;
       break;
     case Intrinsic::x86_tileloadd64_internal:
       C = lowerTileLoadStore<true>(Inst) || C;
@@ -505,18 +583,19 @@ public:
         TM->getOptLevel() != CodeGenOpt::None)
       return false;
 
-    auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-    auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+    auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
+    auto *DT = DTWP ? &DTWP->getDomTree() : nullptr;
+    auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>();
+    auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr;
+    DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
 
-    X86LowerAMXIntrinsics LAT(F, &DT, &LI);
+    X86LowerAMXIntrinsics LAT(F, DTU, LI);
     return LAT.visit();
   }
   StringRef getPassName() const override { return "Lower AMX intrinsics"; }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<DominatorTreeWrapperPass>();
     AU.addPreserved<DominatorTreeWrapperPass>();
-    AU.addRequired<LoopInfoWrapperPass>();
     AU.addPreserved<LoopInfoWrapperPass>();
     AU.addRequired<TargetPassConfig>();
   }
@@ -528,8 +607,6 @@ static const char PassName[] = "Lower AMX intrinsics";
 char X86LowerAMXIntrinsicsLegacyPass::ID = 0;
 INITIALIZE_PASS_BEGIN(X86LowerAMXIntrinsicsLegacyPass, DEBUG_TYPE, PassName,
                       false, false)
-INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
 INITIALIZE_PASS_END(X86LowerAMXIntrinsicsLegacyPass, DEBUG_TYPE, PassName,
                     false, false)
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index 1c91e87e69d53b0878a227902f8d52b6847a0824..cd5d3d6d90d7048f2defea51b7f50da513dffb6d 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -1,4 +1,4 @@
-//===-- X86PreTileConfig.cpp - Tile Register Configure---------------------===//
+//===-- X86PreTileConfig.cpp - Tile Register Pre-configure-----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -87,10 +87,10 @@ public:
 char X86PreTileConfig::ID = 0;
 
 INITIALIZE_PASS_BEGIN(X86PreTileConfig, "tilepreconfig",
-                      "Tile Register Configure", false, false)
+                      "Tile Register Pre-configure", false, false)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
 INITIALIZE_PASS_END(X86PreTileConfig, "tilepreconfig",
-                    "Tile Register Configure", false, false)
+                    "Tile Register Pre-configure", false, false)
 
 void X86PreTileConfig::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
@@ -296,6 +296,12 @@ static void reloadTileConfig(MachineInstr *MI, int FI,
   MachineBasicBlock *MBB = MI->getParent();
   BBVisitedInfo[MBB] = BBInfo(CfgNeedInsert, MBB, MI);
 
+  // The entry BB is special, since it always has a ldtilecfg before AMX
+  // instruction. We don't need to check if its predecessor BBs have call.
+  // FIXME: This case happens only when the entry BB is in a loop. We need to
+  // hoist the first tile config point out of the loop in future.
+  BBVisitedInfo[MBB].HasCallBeforeAMX = true;
+
   WorkList.push_back(MBB);
   while (!WorkList.empty()) {
     MBB = WorkList.pop_back_val();
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 0f19accb834891a8e24eac195819c8d1cb6fa411..32b90e31bec37bde6fce7110d5380a0a4a1a3def 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -261,7 +261,7 @@ X86TargetMachine::getSubtargetImpl(const Function &F) const {
     StringRef Val = PreferVecWidthAttr.getValueAsString();
     unsigned Width;
     if (!Val.getAsInteger(0, Width)) {
-      Key += "prefer-vector-width=";
+      Key += 'p';
       Key += Val;
       PreferVectorWidthOverride = Width;
     }
@@ -274,7 +274,7 @@ X86TargetMachine::getSubtargetImpl(const Function &F) const {
     StringRef Val = MinLegalVecWidthAttr.getValueAsString();
     unsigned Width;
     if (!Val.getAsInteger(0, Width)) {
-      Key += "min-legal-vector-width=";
+      Key += 'm';
       Key += Val;
       RequiredVectorWidth = Width;
     }
@@ -284,7 +284,6 @@ X86TargetMachine::getSubtargetImpl(const Function &F) const {
   Key += CPU;
 
   // Add tune CPU to the Key.
-  Key += "tune=";
   Key += TuneCPU;
 
   // Keep track of the start of the feature portion of the string.
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 71455237fb616546311b30bc730f5c40c85afdcc..0a9582edd2115054b42d05f708e8f42daf2dca73 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -129,26 +129,30 @@ unsigned X86TTIImpl::getNumberOfRegisters(unsigned ClassID) const {
   return 8;
 }
 
-unsigned X86TTIImpl::getRegisterBitWidth(bool Vector) const {
+TypeSize
+X86TTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
   unsigned PreferVectorWidth = ST->getPreferVectorWidth();
-  if (Vector) {
+  switch (K) {
+  case TargetTransformInfo::RGK_Scalar:
+    return TypeSize::getFixed(ST->is64Bit() ? 64 : 32);
+  case TargetTransformInfo::RGK_FixedWidthVector:
     if (ST->hasAVX512() && PreferVectorWidth >= 512)
-      return 512;
+      return TypeSize::getFixed(512);
     if (ST->hasAVX() && PreferVectorWidth >= 256)
-      return 256;
+      return TypeSize::getFixed(256);
     if (ST->hasSSE1() && PreferVectorWidth >= 128)
-      return 128;
-    return 0;
+      return TypeSize::getFixed(128);
+    return TypeSize::getFixed(0);
+  case TargetTransformInfo::RGK_ScalableVector:
+    return TypeSize::getScalable(0);
   }
 
-  if (ST->is64Bit())
-    return 64;
-
-  return 32;
+  llvm_unreachable("Unsupported register kind");
 }
 
 unsigned X86TTIImpl::getLoadStoreVecRegBitWidth(unsigned) const {
-  return getRegisterBitWidth(true);
+  return getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+      .getFixedSize();
 }
 
 unsigned X86TTIImpl::getMaxInterleaveFactor(unsigned VF) {
@@ -960,7 +964,8 @@ int X86TTIImpl::getArithmeticInstrCost(unsigned Opcode, Type *Ty,
 }
 
 int X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *BaseTp,
-                               int Index, VectorType *SubTp) {
+                               ArrayRef<int> Mask, int Index,
+                               VectorType *SubTp) {
   // 64-bit packed float vectors (v2f32) are widened to type v4f32.
   // 64-bit packed integer vectors (v2i32) are widened to type v4i32.
   std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, BaseTp);
@@ -1006,7 +1011,7 @@ int X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *BaseTp,
         auto *SubTy = FixedVectorType::get(BaseTp->getElementType(),
                                            SubLT.second.getVectorNumElements());
         int ExtractIndex = alignDown((Index % NumElts), NumSubElts);
-        int ExtractCost = getShuffleCost(TTI::SK_ExtractSubvector, VecTy,
+        int ExtractCost = getShuffleCost(TTI::SK_ExtractSubvector, VecTy, None,
                                          ExtractIndex, SubTy);
 
         // If the original size is 32-bits or more, we can use pshufd. Otherwise
@@ -1080,11 +1085,11 @@ int X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *BaseTp,
                                               LegalVT.getVectorNumElements());
 
       unsigned NumOfShuffles = (NumOfSrcs - 1) * NumOfDests;
-      return NumOfShuffles *
-             getShuffleCost(TTI::SK_PermuteTwoSrc, SingleOpTy, 0, nullptr);
+      return NumOfShuffles * getShuffleCost(TTI::SK_PermuteTwoSrc, SingleOpTy,
+                                            None, 0, nullptr);
     }
 
-    return BaseT::getShuffleCost(Kind, BaseTp, Index, SubTp);
+    return BaseT::getShuffleCost(Kind, BaseTp, Mask, Index, SubTp);
   }
 
   // For 2-input shuffles, we must account for splitting the 2 inputs into many.
@@ -1392,7 +1397,7 @@ int X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *BaseTp,
     if (const auto *Entry = CostTableLookup(SSE1ShuffleTbl, Kind, LT.second))
       return LT.first * Entry->Cost;
 
-  return BaseT::getShuffleCost(Kind, BaseTp, Index, SubTp);
+  return BaseT::getShuffleCost(Kind, BaseTp, Mask, Index, SubTp);
 }
 
 int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
@@ -3085,7 +3090,8 @@ int X86TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
       EVT VT = TLI->getValueType(DL, Val);
       if (VT.getScalarType() != MScalarTy || VT.getSizeInBits() >= 128)
         SubTy = FixedVectorType::get(ScalarType, SubNumElts);
-      ShuffleCost = getShuffleCost(TTI::SK_PermuteTwoSrc, SubTy, 0, SubTy);
+      ShuffleCost =
+          getShuffleCost(TTI::SK_PermuteTwoSrc, SubTy, None, 0, SubTy);
     }
     int IntOrFpCost = ScalarType->isFloatingPointTy() ? 0 : 1;
     return ShuffleCost + IntOrFpCost + RegisterFileMoveCost;
@@ -3288,14 +3294,14 @@ int X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy,
   if (VT.isSimple() && LT.second != VT.getSimpleVT() &&
       LT.second.getVectorNumElements() == NumElem)
     // Promotion requires expand/truncate for data and a shuffle for mask.
-    Cost += getShuffleCost(TTI::SK_PermuteTwoSrc, SrcVTy, 0, nullptr) +
-            getShuffleCost(TTI::SK_PermuteTwoSrc, MaskTy, 0, nullptr);
+    Cost += getShuffleCost(TTI::SK_PermuteTwoSrc, SrcVTy, None, 0, nullptr) +
+            getShuffleCost(TTI::SK_PermuteTwoSrc, MaskTy, None, 0, nullptr);
 
   else if (LT.second.getVectorNumElements() > NumElem) {
     auto *NewMaskTy = FixedVectorType::get(MaskTy->getElementType(),
                                            LT.second.getVectorNumElements());
     // Expanding requires fill mask with zeroes
-    Cost += getShuffleCost(TTI::SK_InsertSubvector, NewMaskTy, 0, MaskTy);
+    Cost += getShuffleCost(TTI::SK_InsertSubvector, NewMaskTy, None, 0, MaskTy);
   }
 
   // Pre-AVX512 - each maskmov load costs 2 + store costs ~8.
@@ -3529,7 +3535,7 @@ int X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
     if (Size > 128) {
       auto *SubTy = FixedVectorType::get(ValVTy->getElementType(), NumVecElts);
       ReductionCost +=
-          getShuffleCost(TTI::SK_ExtractSubvector, Ty, NumVecElts, SubTy);
+          getShuffleCost(TTI::SK_ExtractSubvector, Ty, None, NumVecElts, SubTy);
       Ty = SubTy;
     } else if (Size == 128) {
       // Reducing from 128 bits is a permute of v2f64/v2i64.
@@ -3541,7 +3547,7 @@ int X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
         ShufTy =
             FixedVectorType::get(Type::getInt64Ty(ValVTy->getContext()), 2);
       ReductionCost +=
-          getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, 0, nullptr);
+          getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, None, 0, nullptr);
     } else if (Size == 64) {
       // Reducing from 64 bits is a shuffle of v4f32/v4i32.
       FixedVectorType *ShufTy;
@@ -3552,7 +3558,7 @@ int X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
         ShufTy =
             FixedVectorType::get(Type::getInt32Ty(ValVTy->getContext()), 4);
       ReductionCost +=
-          getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, 0, nullptr);
+          getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, None, 0, nullptr);
     } else {
       // Reducing from smaller size is a shift by immediate.
       auto *ShiftTy = FixedVectorType::get(
@@ -3833,7 +3839,7 @@ int X86TTIImpl::getMinMaxReductionCost(VectorType *ValTy, VectorType *CondTy,
     if (Size > 128) {
       auto *SubTy = FixedVectorType::get(ValVTy->getElementType(), NumVecElts);
       MinMaxCost +=
-          getShuffleCost(TTI::SK_ExtractSubvector, Ty, NumVecElts, SubTy);
+          getShuffleCost(TTI::SK_ExtractSubvector, Ty, None, NumVecElts, SubTy);
       Ty = SubTy;
     } else if (Size == 128) {
       // Reducing from 128 bits is a permute of v2f64/v2i64.
@@ -3844,7 +3850,7 @@ int X86TTIImpl::getMinMaxReductionCost(VectorType *ValTy, VectorType *CondTy,
       else
         ShufTy = FixedVectorType::get(Type::getInt64Ty(ValTy->getContext()), 2);
       MinMaxCost +=
-          getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, 0, nullptr);
+          getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, None, 0, nullptr);
     } else if (Size == 64) {
       // Reducing from 64 bits is a shuffle of v4f32/v4i32.
       FixedVectorType *ShufTy;
@@ -3853,7 +3859,7 @@ int X86TTIImpl::getMinMaxReductionCost(VectorType *ValTy, VectorType *CondTy,
       else
         ShufTy = FixedVectorType::get(Type::getInt32Ty(ValTy->getContext()), 4);
       MinMaxCost +=
-          getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, 0, nullptr);
+          getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, None, 0, nullptr);
     } else {
       // Reducing from smaller size is a shift by immediate.
       auto *ShiftTy = FixedVectorType::get(
@@ -4666,7 +4672,7 @@ int X86TTIImpl::getInterleavedMemoryOpCostAVX512(
         (NumOfMemOps > 1) ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc;
 
     unsigned ShuffleCost =
-        getShuffleCost(ShuffleKind, SingleMemOpTy, 0, nullptr);
+        getShuffleCost(ShuffleKind, SingleMemOpTy, None, 0, nullptr);
 
     unsigned NumOfLoadsInInterleaveGrp =
         Indices.size() ? Indices.size() : Factor;
@@ -4722,7 +4728,7 @@ int X86TTIImpl::getInterleavedMemoryOpCostAVX512(
   // shuffle.
   unsigned NumOfSources = Factor; // The number of values to be merged.
   unsigned ShuffleCost =
-      getShuffleCost(TTI::SK_PermuteTwoSrc, SingleMemOpTy, 0, nullptr);
+      getShuffleCost(TTI::SK_PermuteTwoSrc, SingleMemOpTy, None, 0, nullptr);
   unsigned NumOfShufflesPerStore = NumOfSources - 1;
 
   // The SK_MergeTwoSrc shuffle clobbers one of src operands.
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index bbf78708c822c9283706792053e6b13e922d2da6..8b2f919347c7e767d31c6511d7a1d7fa066ae0b7 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -115,7 +115,7 @@ public:
   /// @{
 
   unsigned getNumberOfRegisters(unsigned ClassID) const;
-  unsigned getRegisterBitWidth(bool Vector) const;
+  TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
   unsigned getLoadStoreVecRegBitWidth(unsigned AS) const;
   unsigned getMaxInterleaveFactor(unsigned VF);
   int getArithmeticInstrCost(
@@ -127,8 +127,8 @@ public:
       TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None,
       ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
       const Instruction *CxtI = nullptr);
-  int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
-                     VectorType *SubTp);
+  int getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask,
+                     int Index, VectorType *SubTp);
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                        TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
                        const Instruction *I = nullptr);
diff --git a/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp b/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp
index f3904b921e605b18654d2ab140cb81db95de128f..80d3033244de406e04a9f5e84c148b07a9633e84 100644
--- a/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp
+++ b/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp
@@ -112,8 +112,8 @@ static void doList(opt::InputArgList& Args) {
   std::unique_ptr<MemoryBuffer> B;
   for (auto *Arg : Args.filtered(OPT_INPUT)) {
     // Create or open the archive object.
-    ErrorOr<std::unique_ptr<MemoryBuffer>> MaybeBuf =
-        MemoryBuffer::getFile(Arg->getValue(), -1, false);
+    ErrorOr<std::unique_ptr<MemoryBuffer>> MaybeBuf = MemoryBuffer::getFile(
+        Arg->getValue(), /*IsText=*/false, /*RequiresNullTerminator=*/false);
     fatalOpenError(errorCodeToError(MaybeBuf.getError()), Arg->getValue());
 
     if (identify_magic(MaybeBuf.get()->getBuffer()) == file_magic::archive) {
@@ -339,8 +339,8 @@ int llvm::libDriverMain(ArrayRef<const char *> ArgsArr) {
       continue;
 
     // Open a file.
-    ErrorOr<std::unique_ptr<MemoryBuffer>> MOrErr =
-        MemoryBuffer::getFile(Path, -1, false);
+    ErrorOr<std::unique_ptr<MemoryBuffer>> MOrErr = MemoryBuffer::getFile(
+        Path, /*IsText=*/false, /*RequiresNullTerminator=*/false);
     fatalOpenError(errorCodeToError(MOrErr.getError()), Path);
     MemoryBufferRef MBRef = (*MOrErr)->getMemBufferRef();
 
diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index fe5cd7671213cf2fcbadf1c3d82de5af6f5cd1a8..dd72ac413613359d76b04906d99b0b596e65647a 100644
--- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -312,12 +312,6 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote,
           AAMDNodes AAInfo;
           OrigLoad->getAAMetadata(AAInfo);
           newLoad->setAAMetadata(AAInfo);
-          // And other metadata.
-          newLoad->copyMetadata(
-              *OrigLoad,
-              {LLVMContext::MD_nontemporal, LLVMContext::MD_nonnull,
-               LLVMContext::MD_dereferenceable, LLVMContext::MD_align,
-               LLVMContext::MD_noundef, LLVMContext::MD_range});
 
           Args.push_back(newLoad);
           ArgAttrVec.push_back(AttributeSet());
@@ -1052,6 +1046,7 @@ PreservedAnalyses ArgumentPromotionPass::run(LazyCallGraph::SCC &C,
       // swaps out the particular function mapped to a particular node in the
       // graph.
       C.getOuterRefSCC().replaceNodeFunction(N, *NewF);
+      FAM.clear(OldF, OldF.getName());
       OldF.eraseFromParent();
     }
 
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 21fa11aadea8feccf9108570741e3fd341c6f8e4..fa32a22059ac34c941c9fb1cf1d654ee9f4c5d81 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1750,8 +1750,9 @@ struct AANonNullImpl : AANonNull {
 
     AANonNull::initialize(A);
 
-    bool CanBeNull = true;
-    if (V.getPointerDereferenceableBytes(A.getDataLayout(), CanBeNull)) {
+    bool CanBeNull, CanBeFreed;
+    if (V.getPointerDereferenceableBytes(A.getDataLayout(), CanBeNull,
+                                         CanBeFreed)) {
       if (!CanBeNull) {
         indicateOptimisticFixpoint();
         return;
@@ -3548,10 +3549,10 @@ struct AADereferenceableImpl : AADereferenceable {
     const IRPosition &IRP = this->getIRPosition();
     NonNullAA = &A.getAAFor<AANonNull>(*this, IRP, DepClassTy::NONE);
 
-    bool CanBeNull;
+    bool CanBeNull, CanBeFreed;
     takeKnownDerefBytesMaximum(
         IRP.getAssociatedValue().getPointerDereferenceableBytes(
-            A.getDataLayout(), CanBeNull));
+            A.getDataLayout(), CanBeNull, CanBeFreed));
 
     bool IsFnInterface = IRP.isFnInterfaceKind();
     Function *FnScope = IRP.getAnchorScope();
@@ -3661,8 +3662,9 @@ struct AADereferenceableFloating : AADereferenceableImpl {
       if (!Stripped && this == &AA) {
         // Use IR information if we did not strip anything.
         // TODO: track globally.
-        bool CanBeNull;
-        DerefBytes = Base->getPointerDereferenceableBytes(DL, CanBeNull);
+        bool CanBeNull, CanBeFreed;
+        DerefBytes =
+          Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
         T.GlobalState.indicatePessimisticFixpoint();
       } else {
         const DerefState &DS = AA.getState();
diff --git a/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp b/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
index 0b763e423fe0e7c2bfaa5890db17d890f37cf9bf..3154d2468c6483b1063ccc84e86b793a315f7991 100644
--- a/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
+++ b/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp
@@ -295,6 +295,7 @@ bool DeadArgumentEliminationPass::RemoveDeadArgumentsFromCallers(Function &Fn) {
         Changed = true;
       }
       UnusedArgs.push_back(Arg.getArgNo());
+      Fn.removeParamUndefImplyingAttrs(Arg.getArgNo());
     }
   }
 
@@ -312,6 +313,8 @@ bool DeadArgumentEliminationPass::RemoveDeadArgumentsFromCallers(Function &Fn) {
 
       Value *Arg = CB->getArgOperand(ArgNo);
       CB->setArgOperand(ArgNo, UndefValue::get(Arg->getType()));
+      CB->removeParamUndefImplyingAttrs(ArgNo);
+
       ++NumArgumentsReplacedWithUndef;
       Changed = true;
     }
diff --git a/llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp b/llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp
index 1a8bb225a626c78ae45bcaf8da6f89f6e0984d3d..1a4be75dcf2fc5d1fddc5e58ffe68bd348b79a50 100644
--- a/llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp
@@ -73,6 +73,7 @@ static Attribute::AttrKind parseAttrKind(StringRef Kind) {
       .Case("sspstrong", Attribute::StackProtectStrong)
       .Case("strictfp", Attribute::StrictFP)
       .Case("uwtable", Attribute::UWTable)
+      .Case("vscale_range", Attribute::VScaleRange)
       .Default(Attribute::None);
 }
 
diff --git a/llvm/lib/Transforms/IPO/SCCP.cpp b/llvm/lib/Transforms/IPO/SCCP.cpp
index fdffffba0c2d2c534a0bb92028e35fa8e97b099a..6cf1d307921d90aa787fc353b89e794a6da9dde5 100644
--- a/llvm/lib/Transforms/IPO/SCCP.cpp
+++ b/llvm/lib/Transforms/IPO/SCCP.cpp
@@ -1,3 +1,15 @@
+//===-- SCCP.cpp ----------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements Interprocedural Sparse Conditional Constant Propagation.
+//
+//===----------------------------------------------------------------------===//
+
 #include "llvm/Transforms/IPO/SCCP.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/PostDominators.h"
diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
index 9b7af455e69d3ba3b9b2abf822f3114ea2859a0f..5ad0ba20b3e0224b8bb2dd00517c74566c1bee24 100644
--- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
+++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
@@ -542,8 +542,11 @@ ContextTrieNode &SampleContextTracker::promoteMergeContextSamplesTree(
   } else {
     // Destination node exists, merge samples for the context tree
     mergeContextNode(FromNode, *ToNode, ContextStrToRemove);
-    LLVM_DEBUG(dbgs() << "  Context promoted and merged to: "
-                      << ToNode->getFunctionSamples()->getContext() << "\n");
+    LLVM_DEBUG({
+      if (ToNode->getFunctionSamples())
+        dbgs() << "  Context promoted and merged to: "
+               << ToNode->getFunctionSamples()->getContext() << "\n";
+    });
 
     // Recursively promote and merge children
     for (auto &It : FromNode.getAllChildContext()) {
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 2ecff87f492f18f458e70bf322ae661874c537ec..548a8ad216b1ee8f7110d60715fd6a8a24f7ce05 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -321,11 +321,16 @@ struct CandidateComparer {
     if (LHS.CallsiteCount != RHS.CallsiteCount)
       return LHS.CallsiteCount < RHS.CallsiteCount;
 
+    const FunctionSamples *LCS = LHS.CalleeSamples;
+    const FunctionSamples *RCS = RHS.CalleeSamples;
+    assert(LCS && RCS && "Expect non-null FunctionSamples");
+
+    // Tie breaker using number of samples try to favor smaller functions first
+    if (LCS->getBodySamples().size() != RCS->getBodySamples().size())
+      return LCS->getBodySamples().size() > RCS->getBodySamples().size();
+
     // Tie breaker using GUID so we have stable/deterministic inlining order
-    assert(LHS.CalleeSamples && RHS.CalleeSamples &&
-           "Expect non-null FunctionSamples");
-    return LHS.CalleeSamples->getGUID(LHS.CalleeSamples->getName()) <
-           RHS.CalleeSamples->getGUID(RHS.CalleeSamples->getName());
+    return LCS->getGUID(LCS->getName()) < RCS->getGUID(RCS->getName());
   }
 };
 
@@ -755,14 +760,8 @@ static void
 updateIDTMetaData(Instruction &Inst,
                   const SmallVectorImpl<InstrProfValueData> &CallTargets,
                   uint64_t Sum) {
-  assert((Sum != 0 || (CallTargets.size() == 1 &&
-                       CallTargets[0].Count == NOMORE_ICP_MAGICNUM)) &&
-         "If sum is 0, assume only one element in CallTargets with count "
-         "being NOMORE_ICP_MAGICNUM");
-
   uint32_t NumVals = 0;
   // OldSum is the existing total count in the value profile data.
-  // It will be replaced by Sum if Sum is not 0.
   uint64_t OldSum = 0;
   std::unique_ptr<InstrProfValueData[]> ValueData =
       std::make_unique<InstrProfValueData[]>(MaxNumPromotions);
@@ -771,34 +770,44 @@ updateIDTMetaData(Instruction &Inst,
                                ValueData.get(), NumVals, OldSum, true);
 
   DenseMap<uint64_t, uint64_t> ValueCountMap;
-  // Initialize ValueCountMap with existing value profile data.
-  if (Valid) {
-    for (uint32_t I = 0; I < NumVals; I++)
-      ValueCountMap[ValueData[I].Value] = ValueData[I].Count;
-  }
-
-  for (const auto &Data : CallTargets) {
-    auto Pair = ValueCountMap.try_emplace(Data.Value, Data.Count);
-    if (Pair.second)
-      continue;
-    // Whenever the count is NOMORE_ICP_MAGICNUM for a value, keep it
-    // in the ValueCountMap. If both the count in CallTargets and the
-    // count in ValueCountMap is not NOMORE_ICP_MAGICNUM, keep the
-    // count in CallTargets.
-    if (Pair.first->second != NOMORE_ICP_MAGICNUM &&
-        Data.Count == NOMORE_ICP_MAGICNUM) {
+  if (Sum == 0) {
+    assert((CallTargets.size() == 1 &&
+            CallTargets[0].Count == NOMORE_ICP_MAGICNUM) &&
+           "If sum is 0, assume only one element in CallTargets "
+           "with count being NOMORE_ICP_MAGICNUM");
+    // Initialize ValueCountMap with existing value profile data.
+    if (Valid) {
+      for (uint32_t I = 0; I < NumVals; I++)
+        ValueCountMap[ValueData[I].Value] = ValueData[I].Count;
+    }
+    auto Pair =
+        ValueCountMap.try_emplace(CallTargets[0].Value, CallTargets[0].Count);
+    // If the target already exists in value profile, decrease the total
+    // count OldSum and reset the target's count to NOMORE_ICP_MAGICNUM.
+    if (!Pair.second) {
       OldSum -= Pair.first->second;
       Pair.first->second = NOMORE_ICP_MAGICNUM;
-    } else if (Pair.first->second == NOMORE_ICP_MAGICNUM &&
-               Data.Count != NOMORE_ICP_MAGICNUM) {
+    }
+    Sum = OldSum;
+  } else {
+    // Initialize ValueCountMap with existing NOMORE_ICP_MAGICNUM
+    // counts in the value profile.
+    if (Valid) {
+      for (uint32_t I = 0; I < NumVals; I++) {
+        if (ValueData[I].Count == NOMORE_ICP_MAGICNUM)
+          ValueCountMap[ValueData[I].Value] = ValueData[I].Count;
+      }
+    }
+
+    for (const auto &Data : CallTargets) {
+      auto Pair = ValueCountMap.try_emplace(Data.Value, Data.Count);
+      if (Pair.second)
+        continue;
+      // The target represented by Data.Value has already been promoted.
+      // Keep the count as NOMORE_ICP_MAGICNUM in the profile and decrease
+      // Sum by Data.Count.
       assert(Sum >= Data.Count && "Sum should never be less than Data.Count");
       Sum -= Data.Count;
-    } else if (Pair.first->second != NOMORE_ICP_MAGICNUM &&
-               Data.Count != NOMORE_ICP_MAGICNUM) {
-      // Sum will be used in this case. Although the existing count
-      // for the current value in value profile will be overriden,
-      // no need to update OldSum.
-      Pair.first->second = Data.Count;
     }
   }
 
@@ -818,8 +827,7 @@ updateIDTMetaData(Instruction &Inst,
   uint32_t MaxMDCount =
       std::min(NewCallTargets.size(), static_cast<size_t>(MaxNumPromotions));
   annotateValueSite(*Inst.getParent()->getParent()->getParent(), Inst,
-                    NewCallTargets, Sum ? Sum : OldSum, IPVK_IndirectCallTarget,
-                    MaxMDCount);
+                    NewCallTargets, Sum, IPVK_IndirectCallTarget, MaxMDCount);
 }
 
 /// Attempt to promote indirect call and also inline the promoted call.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 4b996c83960777d5aa03b9af3192b2cda5f6613a..e40cfc42511fef77fcfcb9f903ec7773ab5752ff 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -1575,13 +1575,16 @@ static Type *shrinkFPConstant(ConstantFP *CFP) {
 // TODO: Make these support undef elements.
 static Type *shrinkFPConstantVector(Value *V) {
   auto *CV = dyn_cast<Constant>(V);
-  auto *CVVTy = dyn_cast<VectorType>(V->getType());
+  auto *CVVTy = dyn_cast<FixedVectorType>(V->getType());
   if (!CV || !CVVTy)
     return nullptr;
 
   Type *MinType = nullptr;
 
-  unsigned NumElts = cast<FixedVectorType>(CVVTy)->getNumElements();
+  unsigned NumElts = CVVTy->getNumElements();
+
+  // For fixed-width vectors we find the minimal type by looking
+  // through the constant values of the vector.
   for (unsigned i = 0; i != NumElts; ++i) {
     auto *CFP = dyn_cast_or_null<ConstantFP>(CV->getAggregateElement(i));
     if (!CFP)
@@ -1613,7 +1616,15 @@ static Type *getMinimumFPType(Value *V) {
     if (Type *T = shrinkFPConstant(CFP))
       return T;
 
-  // Try to shrink a vector of FP constants.
+  // We can only correctly find a minimum type for a scalable vector when it is
+  // a splat. For splats of constant values the fpext is wrapped up as a
+  // ConstantExpr.
+  if (auto *FPCExt = dyn_cast<ConstantExpr>(V))
+    if (FPCExt->getOpcode() == Instruction::FPExt)
+      return FPCExt->getOperand(0)->getType();
+
+  // Try to shrink a vector of FP constants. This returns nullptr on scalable
+  // vectors
   if (Type *T = shrinkFPConstantVector(V))
     return T;
 
@@ -1933,11 +1944,8 @@ Instruction *InstCombinerImpl::visitIntToPtr(IntToPtrInst &CI) {
   unsigned AS = CI.getAddressSpace();
   if (CI.getOperand(0)->getType()->getScalarSizeInBits() !=
       DL.getPointerSizeInBits(AS)) {
-    Type *Ty = DL.getIntPtrType(CI.getContext(), AS);
-    // Handle vectors of pointers.
-    if (auto *CIVTy = dyn_cast<VectorType>(CI.getType()))
-      Ty = VectorType::get(Ty, CIVTy->getElementCount());
-
+    Type *Ty = CI.getOperand(0)->getType()->getWithNewType(
+        DL.getIntPtrType(CI.getContext(), AS));
     Value *P = Builder.CreateZExtOrTrunc(CI.getOperand(0), Ty);
     return new IntToPtrInst(P, CI.getType());
   }
@@ -1976,16 +1984,14 @@ Instruction *InstCombinerImpl::visitPtrToInt(PtrToIntInst &CI) {
   // do a ptrtoint to intptr_t then do a trunc or zext.  This allows the cast
   // to be exposed to other transforms.
   Value *SrcOp = CI.getPointerOperand();
+  Type *SrcTy = SrcOp->getType();
   Type *Ty = CI.getType();
   unsigned AS = CI.getPointerAddressSpace();
   unsigned TySize = Ty->getScalarSizeInBits();
   unsigned PtrSize = DL.getPointerSizeInBits(AS);
   if (TySize != PtrSize) {
-    Type *IntPtrTy = DL.getIntPtrType(CI.getContext(), AS);
-    // Handle vectors of pointers.
-    if (auto *VecTy = dyn_cast<VectorType>(Ty))
-      IntPtrTy = VectorType::get(IntPtrTy, VecTy->getElementCount());
-
+    Type *IntPtrTy =
+        SrcTy->getWithNewType(DL.getIntPtrType(CI.getContext(), AS));
     Value *P = Builder.CreatePtrToInt(SrcOp, IntPtrTy);
     return CastInst::CreateIntegerCast(P, Ty, /*isSigned=*/false);
   }
@@ -2602,8 +2608,8 @@ Instruction *InstCombinerImpl::visitBitCast(BitCastInst &CI) {
 
       // If the source pointer is dereferenceable, then assume it points to an
       // allocated object and apply "inbounds" to the GEP.
-      bool CanBeNull;
-      if (Src->getPointerDereferenceableBytes(DL, CanBeNull)) {
+      bool CanBeNull, CanBeFreed;
+      if (Src->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed)) {
         // In a non-default address space (not 0), a null pointer can not be
         // assumed inbounds, so ignore that case (dereferenceable_or_null).
         // The reason is that 'null' is not treated differently in these address
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 90a6fd50ef4405fa905f29b7236d43d0b9839c11..a3eba049737472597b50ea45ca470d5adec9645d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -327,6 +327,35 @@ Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI,
     return UnaryOperator::CreateFNegFMF(NewSel, TI);
   }
 
+  // Min/max intrinsic with a common operand can have the common operand pulled
+  // after the select. This is the same transform as below for binops, but
+  // specialized for intrinsic matching and without the restrictive uses clause.
+  auto *TII = dyn_cast<IntrinsicInst>(TI);
+  auto *FII = dyn_cast<IntrinsicInst>(FI);
+  if (TII && FII && TII->getIntrinsicID() == FII->getIntrinsicID() &&
+      (TII->hasOneUse() || FII->hasOneUse())) {
+    Value *T0, *T1, *F0, *F1;
+    if (match(TII, m_MaxOrMin(m_Value(T0), m_Value(T1))) &&
+        match(FII, m_MaxOrMin(m_Value(F0), m_Value(F1)))) {
+      if (T0 == F0) {
+        Value *NewSel = Builder.CreateSelect(Cond, T1, F1, "minmaxop", &SI);
+        return CallInst::Create(TII->getCalledFunction(), {NewSel, T0});
+      }
+      if (T0 == F1) {
+        Value *NewSel = Builder.CreateSelect(Cond, T1, F0, "minmaxop", &SI);
+        return CallInst::Create(TII->getCalledFunction(), {NewSel, T0});
+      }
+      if (T1 == F0) {
+        Value *NewSel = Builder.CreateSelect(Cond, T0, F1, "minmaxop", &SI);
+        return CallInst::Create(TII->getCalledFunction(), {NewSel, T1});
+      }
+      if (T1 == F1) {
+        Value *NewSel = Builder.CreateSelect(Cond, T0, F0, "minmaxop", &SI);
+        return CallInst::Create(TII->getCalledFunction(), {NewSel, T1});
+      }
+    }
+  }
+
   // Only handle binary operators (including two-operand getelementptr) with
   // one-use here. As with the cast case above, it may be possible to relax the
   // one-use constraint, but that needs be examined carefully since it may not
@@ -2604,34 +2633,50 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
       return BinaryOperator::CreateAnd(CondVal, TrueVal);
     }
 
+    auto *One = ConstantInt::getTrue(SelType);
+    auto *Zero = ConstantInt::getFalse(SelType);
+
     // select a, false, b -> select !a, b, false
     if (match(TrueVal, m_Zero())) {
       Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName());
-      return SelectInst::Create(NotCond, FalseVal,
-                                ConstantInt::getFalse(SelType));
+      return SelectInst::Create(NotCond, FalseVal, Zero);
     }
     // select a, b, true -> select !a, true, b
     if (match(FalseVal, m_One())) {
       Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName());
-      return SelectInst::Create(NotCond, ConstantInt::getTrue(SelType),
-                                TrueVal);
+      return SelectInst::Create(NotCond, One, TrueVal);
     }
 
     // select a, a, b -> select a, true, b
     if (CondVal == TrueVal)
-      return replaceOperand(SI, 1, ConstantInt::getTrue(SelType));
+      return replaceOperand(SI, 1, One);
     // select a, b, a -> select a, b, false
     if (CondVal == FalseVal)
-      return replaceOperand(SI, 2, ConstantInt::getFalse(SelType));
+      return replaceOperand(SI, 2, Zero);
 
     // select a, !a, b -> select !a, b, false
     if (match(TrueVal, m_Not(m_Specific(CondVal))))
-      return SelectInst::Create(TrueVal, FalseVal,
-                                ConstantInt::getFalse(SelType));
+      return SelectInst::Create(TrueVal, FalseVal, Zero);
     // select a, b, !a -> select !a, true, b
     if (match(FalseVal, m_Not(m_Specific(CondVal))))
-      return SelectInst::Create(FalseVal, ConstantInt::getTrue(SelType),
-                                TrueVal);
+      return SelectInst::Create(FalseVal, One, TrueVal);
+
+    Value *A, *B;
+    // select (select a, true, b), true, b -> select a, true, b
+    if (match(CondVal, m_Select(m_Value(A), m_One(), m_Value(B))) &&
+        match(TrueVal, m_One()) && match(FalseVal, m_Specific(B)))
+      return replaceOperand(SI, 0, A);
+    // select (select a, b, false), b, false -> select a, b, false
+    if (match(CondVal, m_Select(m_Value(A), m_Value(B), m_Zero())) &&
+        match(TrueVal, m_Specific(B)) && match(FalseVal, m_Zero()))
+      return replaceOperand(SI, 0, A);
+
+    if (Value *S = SimplifyWithOpReplaced(TrueVal, CondVal, One, SQ,
+                                          /* AllowRefinement */ true))
+      return replaceOperand(SI, 1, S);
+    if (Value *S = SimplifyWithOpReplaced(FalseVal, CondVal, Zero, SQ,
+                                          /* AllowRefinement */ true))
+      return replaceOperand(SI, 2, S);
   }
 
   // Selecting between two integer or vector splat integer constants?
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 65becd7904af1f97a2d5121d45b3f9f669623879..c8fb8aebe53ae7e4d8c88f5c6b3a0a2cff9e6a14 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -2425,13 +2425,21 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
     // analysis of unions. If "A" is also a bitcast, wait for A/X to be merged.
     unsigned OffsetBits = DL.getIndexTypeSizeInBits(GEPType);
     APInt Offset(OffsetBits, 0);
-    if (!isa<BitCastInst>(SrcOp) && GEP.accumulateConstantOffset(DL, Offset)) {
+
+    // If the bitcast argument is an allocation, The bitcast is for convertion
+    // to actual type of allocation. Removing such bitcasts, results in having
+    // GEPs with i8* base and pure byte offsets. That means GEP is not aware of
+    // struct or array hierarchy.
+    // By avoiding such GEPs, phi translation and MemoryDependencyAnalysis have
+    // a better chance to succeed.
+    if (!isa<BitCastInst>(SrcOp) && GEP.accumulateConstantOffset(DL, Offset) &&
+        !isAllocationFn(SrcOp, &TLI)) {
       // If this GEP instruction doesn't move the pointer, just replace the GEP
       // with a bitcast of the real input to the dest type.
       if (!Offset) {
         // If the bitcast is of an allocation, and the allocation will be
         // converted to match the type of the cast, don't touch this.
-        if (isa<AllocaInst>(SrcOp) || isAllocationFn(SrcOp, &TLI)) {
+        if (isa<AllocaInst>(SrcOp)) {
           // See if the bitcast simplifies, if so, don't nuke this GEP yet.
           if (Instruction *I = visitBitCast(*BCI)) {
             if (I != BCI) {
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index a16ae68925a5c29fdc29d3e1f71e3c586aeadd4f..63b8db7916a03148d6bd96a03525b8949785dba5 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -16,9 +16,38 @@
 /// issues within their own code.
 ///
 /// The analysis is based on automatic propagation of data flow labels (also
-/// known as taint labels) through a program as it performs computation.  Each
-/// byte of application memory is backed by two bytes of shadow memory which
-/// hold the label.  On Linux/x86_64, memory is laid out as follows:
+/// known as taint labels) through a program as it performs computation.
+///
+/// There are two possible memory layouts. In the first one, each byte of
+/// application memory is backed by a shadow memory byte. The shadow byte can
+/// represent up to 8 labels. To enable this you must specify the
+/// -dfsan-fast-8-labels flag. On Linux/x86_64, memory is then laid out as
+/// follows:
+///
+/// +--------------------+ 0x800000000000 (top of memory)
+/// | application memory |
+/// +--------------------+ 0x700000008000 (kAppAddr)
+/// |                    |
+/// |       unused       |
+/// |                    |
+/// +--------------------+ 0x300200000000 (kUnusedAddr)
+/// |    union table     |
+/// +--------------------+ 0x300000000000 (kUnionTableAddr)
+/// |       origin       |
+/// +--------------------+ 0x200000008000 (kOriginAddr)
+/// |   shadow memory    |
+/// +--------------------+ 0x100000008000 (kShadowAddr)
+/// |       unused       |
+/// +--------------------+ 0x000000010000
+/// | reserved by kernel |
+/// +--------------------+ 0x000000000000
+///
+///
+/// In the second memory layout, each byte of application memory is backed by
+/// two bytes of shadow memory which hold the label. That means we can represent
+/// either 16 labels (with -dfsan-fast-16-labels flag) or 2^16 labels (on the
+/// default legacy mode) per byte. On Linux/x86_64, memory is then laid out as
+/// follows:
 ///
 /// +--------------------+ 0x800000000000 (top of memory)
 /// | application memory |
@@ -36,6 +65,7 @@
 /// | reserved by kernel |
 /// +--------------------+ 0x000000000000
 ///
+///
 /// To derive a shadow memory address from an application memory address,
 /// bits 44-46 are cleared to bring the address into the range
 /// [0x000000008000,0x100000000000).  Then the address is shifted left by 1 to
@@ -200,6 +230,14 @@ static cl::opt<bool> ClFast16Labels(
              "labels to 16."),
     cl::Hidden, cl::init(false));
 
+// Use a distinct bit for each base label, enabling faster unions with less
+// instrumentation.  Limits the max number of base labels to 8.
+static cl::opt<bool> ClFast8Labels(
+    "dfsan-fast-8-labels",
+    cl::desc("Use more efficient instrumentation, limiting the number of "
+             "labels to 8."),
+    cl::Hidden, cl::init(false));
+
 // Controls whether the pass tracks the control flow of select instructions.
 static cl::opt<bool> ClTrackSelectControlFlow(
     "dfsan-track-select-control-flow",
@@ -341,8 +379,6 @@ class DataFlowSanitizer {
   friend class DFSanVisitor;
 
   enum {
-    ShadowWidthBits = 16,
-    ShadowWidthBytes = ShadowWidthBits / 8,
     OriginWidthBits = 32,
     OriginWidthBytes = OriginWidthBits / 8
   };
@@ -383,6 +419,9 @@ class DataFlowSanitizer {
     WK_Custom
   };
 
+  unsigned ShadowWidthBits;
+  unsigned ShadowWidthBytes;
+
   Module *Mod;
   LLVMContext *Ctx;
   Type *Int8Ptr;
@@ -419,7 +458,7 @@ class DataFlowSanitizer {
   FunctionCallee DFSanUnionFn;
   FunctionCallee DFSanCheckedUnionFn;
   FunctionCallee DFSanUnionLoadFn;
-  FunctionCallee DFSanUnionLoadFast16LabelsFn;
+  FunctionCallee DFSanUnionLoadFastLabelsFn;
   FunctionCallee DFSanLoadLabelAndOriginFn;
   FunctionCallee DFSanUnimplementedFn;
   FunctionCallee DFSanSetLabelFn;
@@ -442,6 +481,7 @@ class DataFlowSanitizer {
 
   Value *getShadowOffset(Value *Addr, IRBuilder<> &IRB);
   Value *getShadowAddress(Value *Addr, Instruction *Pos);
+  Value *getShadowAddress(Value *Addr, Instruction *Pos, Value *ShadowOffset);
   std::pair<Value *, Value *>
   getShadowOriginAddress(Value *Addr, Align InstAlignment, Instruction *Pos);
   bool isInstrumented(const Function *F);
@@ -462,6 +502,9 @@ class DataFlowSanitizer {
 
   bool init(Module &M);
 
+  /// Returns whether fast8 or fast16 mode has been specified.
+  bool hasFastLabelsEnabled();
+
   /// Returns whether the pass tracks origins. Support only fast16 mode in TLS
   /// ABI mode.
   bool shouldTrackOrigins();
@@ -733,6 +776,14 @@ private:
 
 DataFlowSanitizer::DataFlowSanitizer(
     const std::vector<std::string> &ABIListFiles) {
+  if (ClFast8Labels && ClFast16Labels) {
+    report_fatal_error(
+        "cannot set both -dfsan-fast-8-labels and -dfsan-fast-16-labels");
+  }
+
+  ShadowWidthBits = ClFast8Labels ? 8 : 16;
+  ShadowWidthBytes = ShadowWidthBits / 8;
+
   std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
   llvm::append_range(AllABIListFiles, ClABIListFiles);
   // FIXME: should we propagate vfs::FileSystem to this constructor?
@@ -827,6 +878,11 @@ bool DataFlowSanitizer::isZeroShadow(Value *V) {
   return isa<ConstantAggregateZero>(V);
 }
 
+bool DataFlowSanitizer::hasFastLabelsEnabled() {
+  static const bool HasFastLabelsEnabled = ClFast8Labels || ClFast16Labels;
+  return HasFastLabelsEnabled;
+}
+
 bool DataFlowSanitizer::shouldTrackOrigins() {
   static const bool ShouldTrackOrigins =
       ClTrackOrigins && getInstrumentedABI() == DataFlowSanitizer::IA_TLS &&
@@ -835,7 +891,8 @@ bool DataFlowSanitizer::shouldTrackOrigins() {
 }
 
 bool DataFlowSanitizer::shouldTrackFieldsAndIndices() {
-  return getInstrumentedABI() == DataFlowSanitizer::IA_TLS && ClFast16Labels;
+  return getInstrumentedABI() == DataFlowSanitizer::IA_TLS &&
+         hasFastLabelsEnabled();
 }
 
 Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) {
@@ -1000,11 +1057,15 @@ bool DataFlowSanitizer::init(Module &M) {
 
   switch (TargetTriple.getArch()) {
   case Triple::x86_64:
-    ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x700000000000LL);
+    ShadowPtrMask = ClFast8Labels
+                        ? ConstantInt::getSigned(IntptrTy, ~0x600000000000LL)
+                        : ConstantInt::getSigned(IntptrTy, ~0x700000000000LL);
     break;
   case Triple::mips64:
   case Triple::mips64el:
-    ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0xF000000000LL);
+    ShadowPtrMask = ClFast8Labels
+                        ? ConstantInt::getSigned(IntptrTy, ~0xE000000000LL)
+                        : ConstantInt::getSigned(IntptrTy, ~0xF000000000LL);
     break;
   case Triple::aarch64:
   case Triple::aarch64_be:
@@ -1238,7 +1299,7 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
                          Attribute::ReadOnly);
     AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
                          Attribute::ZExt);
-    DFSanUnionLoadFast16LabelsFn = Mod->getOrInsertFunction(
+    DFSanUnionLoadFastLabelsFn = Mod->getOrInsertFunction(
         "__dfsan_union_load_fast16labels", DFSanUnionLoadFnTy, AL);
   }
   {
@@ -1290,7 +1351,7 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
   DFSanRuntimeFunctions.insert(
       DFSanUnionLoadFn.getCallee()->stripPointerCasts());
   DFSanRuntimeFunctions.insert(
-      DFSanUnionLoadFast16LabelsFn.getCallee()->stripPointerCasts());
+      DFSanUnionLoadFastLabelsFn.getCallee()->stripPointerCasts());
   DFSanRuntimeFunctions.insert(
       DFSanLoadLabelAndOriginFn.getCallee()->stripPointerCasts());
   DFSanRuntimeFunctions.insert(
@@ -1757,8 +1818,7 @@ DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment,
   // Returns ((Addr & shadow_mask) + origin_base) & ~4UL
   IRBuilder<> IRB(Pos);
   Value *ShadowOffset = getShadowOffset(Addr, IRB);
-  Value *ShadowPtr = IRB.CreateIntToPtr(
-      IRB.CreateMul(ShadowOffset, ShadowPtrMul), PrimitiveShadowPtrTy);
+  Value *ShadowPtr = getShadowAddress(Addr, Pos, ShadowOffset);
   Value *OriginPtr = nullptr;
   if (shouldTrackOrigins()) {
     Value *OriginLong = IRB.CreateAdd(ShadowOffset, OriginBase);
@@ -1774,12 +1834,21 @@ DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment,
   return {ShadowPtr, OriginPtr};
 }
 
+Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos,
+                                           Value *ShadowOffset) {
+  IRBuilder<> IRB(Pos);
+
+  if (!ShadowPtrMul->isOne())
+    ShadowOffset = IRB.CreateMul(ShadowOffset, ShadowPtrMul);
+
+  return IRB.CreateIntToPtr(ShadowOffset, PrimitiveShadowPtrTy);
+}
+
 Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos) {
   // Returns (Addr & shadow_mask) x 2
   IRBuilder<> IRB(Pos);
   Value *ShadowOffset = getShadowOffset(Addr, IRB);
-  return IRB.CreateIntToPtr(IRB.CreateMul(ShadowOffset, ShadowPtrMul),
-                            PrimitiveShadowPtrTy);
+  return getShadowAddress(Addr, Pos, ShadowOffset);
 }
 
 Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
@@ -1829,7 +1898,7 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
   Value *PV2 = collapseToPrimitiveShadow(V2, Pos);
 
   IRBuilder<> IRB(Pos);
-  if (ClFast16Labels) {
+  if (DFS.hasFastLabelsEnabled()) {
     CCS.Block = Pos->getParent();
     CCS.Shadow = IRB.CreateOr(PV1, PV2);
   } else if (AvoidNewBlocks) {
@@ -1978,27 +2047,53 @@ bool DFSanFunction::useCallbackLoadLabelAndOrigin(uint64_t Size,
 std::pair<Value *, Value *> DFSanFunction::loadFast16ShadowFast(
     Value *ShadowAddr, Value *OriginAddr, uint64_t Size, Align ShadowAlign,
     Align OriginAlign, Value *FirstOrigin, Instruction *Pos) {
-  // First OR all the WideShadows, then OR individual shadows within the
-  // combined WideShadow. This is fewer instructions than ORing shadows
-  // individually.
   const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
+  const uint64_t ShadowSize = Size * DFS.ShadowWidthBytes;
+
+  assert(Size >= 4 && "Not large enough load size for fast path!");
+
+  // Used for origin tracking.
   std::vector<Value *> Shadows;
   std::vector<Value *> Origins;
+
+  // Load instructions in LLVM can have arbitrary byte sizes (e.g., 3, 12, 20)
+  // but this function is only used in a subset of cases that make it possible
+  // to optimize the instrumentation.
+  //
+  // Specifically, when the shadow size in bytes (i.e., loaded bytes x shadow
+  // per byte) is either:
+  // - a multiple of 8  (common)
+  // - equal to 4       (only for load32 in fast-8 mode)
+  //
+  // For the second case, we can fit the wide shadow in a 32-bit integer. In all
+  // other cases, we use a 64-bit integer to hold the wide shadow.
+  Type *WideShadowTy =
+      ShadowSize == 4 ? Type::getInt32Ty(*DFS.Ctx) : Type::getInt64Ty(*DFS.Ctx);
+
   IRBuilder<> IRB(Pos);
-  Value *WideAddr =
-      IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
+  Value *WideAddr = IRB.CreateBitCast(ShadowAddr, WideShadowTy->getPointerTo());
   Value *CombinedWideShadow =
-      IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
+      IRB.CreateAlignedLoad(WideShadowTy, WideAddr, ShadowAlign);
+
   if (ShouldTrackOrigins) {
     Shadows.push_back(CombinedWideShadow);
     Origins.push_back(FirstOrigin);
   }
-  for (uint64_t Ofs = 64 / DFS.ShadowWidthBits; Ofs != Size;
-       Ofs += 64 / DFS.ShadowWidthBits) {
-    WideAddr = IRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr,
+
+  // First OR all the WideShadows (i.e., 64bit or 32bit shadow chunks) linearly;
+  // then OR individual shadows within the combined WideShadow by binary ORing.
+  // This is fewer instructions than ORing shadows individually, since it
+  // needs logN shift/or instructions (N being the bytes of the combined wide
+  // shadow).
+  unsigned WideShadowBitWidth = WideShadowTy->getIntegerBitWidth();
+  const uint64_t BytesPerWideShadow = WideShadowBitWidth / DFS.ShadowWidthBits;
+
+  for (uint64_t ByteOfs = BytesPerWideShadow; ByteOfs < Size;
+       ByteOfs += BytesPerWideShadow) {
+    WideAddr = IRB.CreateGEP(WideShadowTy, WideAddr,
                              ConstantInt::get(DFS.IntptrTy, 1));
     Value *NextWideShadow =
-        IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
+        IRB.CreateAlignedLoad(WideShadowTy, WideAddr, ShadowAlign);
     CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow);
     if (ShouldTrackOrigins) {
       Shadows.push_back(NextWideShadow);
@@ -2008,7 +2103,8 @@ std::pair<Value *, Value *> DFSanFunction::loadFast16ShadowFast(
           IRB.CreateAlignedLoad(DFS.OriginTy, OriginAddr, OriginAlign));
     }
   }
-  for (unsigned Width = 32; Width >= DFS.ShadowWidthBits; Width >>= 1) {
+  for (unsigned Width = WideShadowBitWidth / 2; Width >= DFS.ShadowWidthBits;
+       Width >>= 1) {
     Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width);
     CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow);
   }
@@ -2023,24 +2119,33 @@ Value *DFSanFunction::loadLegacyShadowFast(Value *ShadowAddr, uint64_t Size,
                                            Align ShadowAlign,
                                            Instruction *Pos) {
   // Fast path for the common case where each byte has identical shadow: load
-  // shadow 64 bits at a time, fall out to a __dfsan_union_load call if any
-  // shadow is non-equal.
+  // shadow 64 (or 32) bits at a time, fall out to a __dfsan_union_load call if
+  // any shadow is non-equal.
   BasicBlock *FallbackBB = BasicBlock::Create(*DFS.Ctx, "", F);
   IRBuilder<> FallbackIRB(FallbackBB);
   CallInst *FallbackCall = FallbackIRB.CreateCall(
       DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
   FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
 
+  const uint64_t ShadowSize = Size * DFS.ShadowWidthBytes;
+  assert(Size >= 4 && "Not large enough load size for fast path!");
+
+  // Same as in loadFast16AShadowsFast. In the case of load32, we can fit the
+  // wide shadow in a 32-bit integer instead.
+  Type *WideShadowTy =
+      ShadowSize == 4 ? Type::getInt32Ty(*DFS.Ctx) : Type::getInt64Ty(*DFS.Ctx);
+
   // Compare each of the shadows stored in the loaded 64 bits to each other,
   // by computing (WideShadow rotl ShadowWidthBits) == WideShadow.
   IRBuilder<> IRB(Pos);
-  Value *WideAddr =
-      IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
+  unsigned WideShadowBitWidth = WideShadowTy->getIntegerBitWidth();
+  Value *WideAddr = IRB.CreateBitCast(ShadowAddr, WideShadowTy->getPointerTo());
   Value *WideShadow =
-      IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
+      IRB.CreateAlignedLoad(WideShadowTy, WideAddr, ShadowAlign);
   Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.PrimitiveShadowTy);
   Value *ShlShadow = IRB.CreateShl(WideShadow, DFS.ShadowWidthBits);
-  Value *ShrShadow = IRB.CreateLShr(WideShadow, 64 - DFS.ShadowWidthBits);
+  Value *ShrShadow =
+      IRB.CreateLShr(WideShadow, WideShadowBitWidth - DFS.ShadowWidthBits);
   Value *RotShadow = IRB.CreateOr(ShlShadow, ShrShadow);
   Value *ShadowsEq = IRB.CreateICmpEQ(WideShadow, RotShadow);
 
@@ -2063,15 +2168,17 @@ Value *DFSanFunction::loadLegacyShadowFast(Value *ShadowAddr, uint64_t Size,
   ReplaceInstWithInst(Head->getTerminator(), LastBr);
   DT.addNewBlock(FallbackBB, Head);
 
-  for (uint64_t Ofs = 64 / DFS.ShadowWidthBits; Ofs != Size;
-       Ofs += 64 / DFS.ShadowWidthBits) {
+  const uint64_t BytesPerWideShadow = WideShadowBitWidth / DFS.ShadowWidthBits;
+
+  for (uint64_t ByteOfs = BytesPerWideShadow; ByteOfs < Size;
+       ByteOfs += BytesPerWideShadow) {
     BasicBlock *NextBB = BasicBlock::Create(*DFS.Ctx, "", F);
     DT.addNewBlock(NextBB, LastBr->getParent());
     IRBuilder<> NextIRB(NextBB);
-    WideAddr = NextIRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr,
+    WideAddr = NextIRB.CreateGEP(WideShadowTy, WideAddr,
                                  ConstantInt::get(DFS.IntptrTy, 1));
     Value *NextWideShadow =
-        NextIRB.CreateAlignedLoad(NextIRB.getInt64Ty(), WideAddr, ShadowAlign);
+        NextIRB.CreateAlignedLoad(WideShadowTy, WideAddr, ShadowAlign);
     ShadowsEq = NextIRB.CreateICmpEQ(WideShadow, NextWideShadow);
     LastBr->setSuccessor(0, NextBB);
     LastBr = NextIRB.CreateCondBr(ShadowsEq, FallbackBB, FallbackBB);
@@ -2158,6 +2265,8 @@ std::pair<Value *, Value *> DFSanFunction::loadShadowOrigin(Value *Addr,
     Origin = IRB.CreateAlignedLoad(DFS.OriginTy, OriginAddr, OriginAlign);
   }
 
+  // When the byte size is small enough, we can load the shadow directly with
+  // just a few instructions.
   switch (Size) {
   case 1: {
     LoadInst *LI = new LoadInst(DFS.PrimitiveShadowTy, ShadowAddr, "", Pos);
@@ -2175,17 +2284,21 @@ std::pair<Value *, Value *> DFSanFunction::loadShadowOrigin(Value *Addr,
     return {combineShadows(Load, Load1, Pos), Origin};
   }
   }
+  uint64_t ShadowSize = Size * DFS.ShadowWidthBytes;
+  bool HasSizeForFastPath = ShadowSize % 8 == 0 || ShadowSize == 4;
+  bool HasFastLabelsEnabled = DFS.hasFastLabelsEnabled();
 
-  if (ClFast16Labels && Size % (64 / DFS.ShadowWidthBits) == 0)
+  if (HasFastLabelsEnabled && HasSizeForFastPath)
     return loadFast16ShadowFast(ShadowAddr, OriginAddr, Size, ShadowAlign,
                                 OriginAlign, Origin, Pos);
 
-  if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidthBits) == 0)
+  if (!AvoidNewBlocks && HasSizeForFastPath)
     return {loadLegacyShadowFast(ShadowAddr, Size, ShadowAlign, Pos), Origin};
 
   IRBuilder<> IRB(Pos);
-  FunctionCallee &UnionLoadFn =
-      ClFast16Labels ? DFS.DFSanUnionLoadFast16LabelsFn : DFS.DFSanUnionLoadFn;
+  FunctionCallee &UnionLoadFn = HasFastLabelsEnabled
+                                    ? DFS.DFSanUnionLoadFastLabelsFn
+                                    : DFS.DFSanUnionLoadFn;
   CallInst *FallbackCall = IRB.CreateCall(
       UnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
   FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
@@ -2406,7 +2519,10 @@ void DFSanFunction::storePrimitiveShadowOrigin(Value *Addr, uint64_t Size,
   std::tie(ShadowAddr, OriginAddr) =
       DFS.getShadowOriginAddress(Addr, InstAlignment, Pos);
 
-  const unsigned ShadowVecSize = 128 / DFS.ShadowWidthBits;
+  const unsigned ShadowVecSize = 8;
+  assert(ShadowVecSize * DFS.ShadowWidthBits <= 128 &&
+         "Shadow vector is too large!");
+
   uint64_t Offset = 0;
   uint64_t LeftSize = Size;
   if (LeftSize >= ShadowVecSize) {
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 27715ff86ff204e47555150a00bfaae0d93edd2b..04761f0d89f0b63462347a7b6c64ea8b49b31228 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -267,7 +267,7 @@ private:
     bool InGlobal;
     bool InTls;
 
-    void init(Triple &TargetTriple);
+    void init(Triple &TargetTriple, bool InstrumentWithCalls);
     unsigned getObjectAlignment() const { return 1U << Scale; }
   };
   ShadowMapping Mapping;
@@ -284,6 +284,9 @@ private:
   bool OutlinedChecks;
   bool UseShortGranules;
   bool InstrumentLandingPads;
+  bool InstrumentWithCalls;
+  bool InstrumentStack;
+  bool UsePageAliases;
 
   bool HasMatchAllTag = false;
   uint8_t MatchAllTag = 0;
@@ -479,7 +482,13 @@ void HWAddressSanitizer::initializeModule() {
 
   TargetTriple = Triple(M.getTargetTriple());
 
-  Mapping.init(TargetTriple);
+  // x86_64 uses userspace pointer aliases, currently heap-only with callback
+  // instrumentation only.
+  UsePageAliases = TargetTriple.getArch() == Triple::x86_64;
+  InstrumentWithCalls = UsePageAliases ? true : ClInstrumentWithCalls;
+  InstrumentStack = UsePageAliases ? false : ClInstrumentStack;
+
+  Mapping.init(TargetTriple, InstrumentWithCalls);
 
   C = &(M.getContext());
   IRBuilder<> IRB(*C);
@@ -521,7 +530,7 @@ void HWAddressSanitizer::initializeModule() {
     createHwasanCtorComdat();
     bool InstrumentGlobals =
         ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
-    if (InstrumentGlobals)
+    if (InstrumentGlobals && !UsePageAliases)
       instrumentGlobals();
 
     bool InstrumentPersonalityFunctions =
@@ -699,7 +708,7 @@ static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
 }
 
 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
-  if (TargetTriple.isAArch64())
+  if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64)
     return;
 
   IRBuilder<> IRB(I);
@@ -721,6 +730,7 @@ Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
                                                    unsigned AccessSizeIndex,
                                                    Instruction *InsertBefore) {
+  assert(!UsePageAliases);
   const int64_t AccessInfo =
       (CompileKernel << HWASanAccessInfo::CompileKernelShift) +
       (HasMatchAllTag << HWASanAccessInfo::HasMatchAllShift) +
@@ -849,7 +859,7 @@ bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
       (!O.Alignment || *O.Alignment >= (1ULL << Mapping.Scale) ||
        *O.Alignment >= O.TypeSize / 8)) {
     size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeSize);
-    if (ClInstrumentWithCalls) {
+    if (InstrumentWithCalls) {
       IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
                      IRB.CreatePointerCast(Addr, IntptrTy));
     } else {
@@ -884,7 +894,7 @@ bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
     Size = AlignedSize;
 
   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
-  if (ClInstrumentWithCalls) {
+  if (InstrumentWithCalls) {
     IRB.CreateCall(HwasanTagMemoryFunc,
                    {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
                     ConstantInt::get(IntptrTy, AlignedSize)});
@@ -976,6 +986,7 @@ Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
 // Add a tag to an address.
 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
                                       Value *PtrLong, Value *Tag) {
+  assert(!UsePageAliases);
   Value *TaggedPtrLong;
   if (CompileKernel) {
     // Kernel addresses have 0xFF in the most significant byte.
@@ -993,6 +1004,7 @@ Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
 
 // Remove tag from an address.
 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
+  assert(!UsePageAliases);
   Value *UntaggedPtrLong;
   if (CompileKernel) {
     // Kernel addresses have 0xFF in the most significant byte.
@@ -1207,7 +1219,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) {
   DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> AllocaDbgMap;
   for (auto &BB : F) {
     for (auto &Inst : BB) {
-      if (ClInstrumentStack)
+      if (InstrumentStack)
         if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
           if (isInterestingAlloca(*AI))
             AllocasToInstrument.push_back(AI);
@@ -1343,6 +1355,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) {
 }
 
 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
+  assert(!UsePageAliases);
   Constant *Initializer = GV->getInitializer();
   uint64_t SizeInBytes =
       M.getDataLayout().getTypeAllocSize(Initializer->getType());
@@ -1515,13 +1528,20 @@ void HWAddressSanitizer::instrumentPersonalityFunctions() {
   }
 }
 
-void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
+void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
+                                             bool InstrumentWithCalls) {
   Scale = kDefaultShadowScale;
-  if (ClMappingOffset.getNumOccurrences() > 0) {
+  if (TargetTriple.isOSFuchsia()) {
+    // Fuchsia is always PIE, which means that the beginning of the address
+    // space is always available.
+    InGlobal = false;
+    InTls = false;
+    Offset = 0;
+  } else if (ClMappingOffset.getNumOccurrences() > 0) {
     InGlobal = false;
     InTls = false;
     Offset = ClMappingOffset;
-  } else if (ClEnableKhwasan || ClInstrumentWithCalls) {
+  } else if (ClEnableKhwasan || InstrumentWithCalls) {
     InGlobal = false;
     InTls = false;
     Offset = 0;
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 843a27faf1687602fc96383bf5d70a7d2cf03e46..166dd108f81bf4858d1e70c7dca2b93f09df41dd 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -328,13 +328,20 @@ PreservedAnalyses ModuleSanitizerCoveragePass::run(Module &M,
 std::pair<Value *, Value *>
 ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
                                            Type *Ty) {
-  GlobalVariable *SecStart = new GlobalVariable(
-      M, Ty->getPointerElementType(), false, GlobalVariable::ExternalLinkage,
-      nullptr, getSectionStart(Section));
+  // Use ExternalWeak so that if all sections are discarded due to section
+  // garbage collection, the linker will not report undefined symbol errors.
+  // Windows defines the start/stop symbols in compiler-rt so no need for
+  // ExternalWeak.
+  GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()
+                                          ? GlobalVariable::ExternalLinkage
+                                          : GlobalVariable::ExternalWeakLinkage;
+  GlobalVariable *SecStart =
+      new GlobalVariable(M, Ty->getPointerElementType(), false, Linkage,
+                         nullptr, getSectionStart(Section));
   SecStart->setVisibility(GlobalValue::HiddenVisibility);
-  GlobalVariable *SecEnd = new GlobalVariable(
-      M, Ty->getPointerElementType(), false, GlobalVariable::ExternalLinkage,
-      nullptr, getSectionEnd(Section));
+  GlobalVariable *SecEnd =
+      new GlobalVariable(M, Ty->getPointerElementType(), false, Linkage,
+                         nullptr, getSectionEnd(Section));
   SecEnd->setVisibility(GlobalValue::HiddenVisibility);
   IRBuilder<> IRB(M.getContext());
   if (!TargetTriple.isOSBinFormatCOFF())
diff --git a/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp b/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
index bb5270af7f2317305a8660880f11894ff53bd4da..c5bdfb4d8f6ad04ebfeb4b93a22862c1aa9c00fd 100644
--- a/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
+++ b/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
@@ -100,9 +100,9 @@ static void runImpl(Function &F, const TargetLibraryInfo &TLI) {
     }
   }
 
-  Instruction *IP = &*F.begin()->begin();
   for (const auto &KV : Mapping)
-    ORE.emit(OptimizationRemarkAnalysis(REMARK_PASS, "AnnotationSummary", IP)
+    ORE.emit(OptimizationRemarkAnalysis(REMARK_PASS, "AnnotationSummary",
+                                        F.getSubprogram(), &F.front())
              << "Annotated " << NV("count", KV.second) << " instructions with "
              << NV("type", KV.first));
 
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index f133f865945f87408181d261e77aedee23f67dd9..16cde39ecff93d0e65c5d67f9b28819f1c0a05df 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -166,8 +166,8 @@ getConstraint(CmpInst::Predicate Pred, Value *Op0, Value *Op1,
   if (Pred != CmpInst::ICMP_ULE && Pred != CmpInst::ICMP_ULT)
     return {};
 
-  auto ADec = decompose(Op0->stripPointerCasts());
-  auto BDec = decompose(Op1->stripPointerCasts());
+  auto ADec = decompose(Op0->stripPointerCastsSameRepresentation());
+  auto BDec = decompose(Op1->stripPointerCastsSameRepresentation());
   // Skip if decomposing either of the values failed.
   if (ADec.empty() || BDec.empty())
     return {};
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 180a82917fa9edca779094817f93575788839bdd..9c7d430788214ae01aa7fa501cf4f68e1ccc413e 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -109,6 +109,9 @@ struct SimpleValue {
 
   static bool canHandle(Instruction *Inst) {
     // This can only handle non-void readnone functions.
+    if (isa<GCRelocateInst>(Inst))
+      // Migration assistant for PR49607, to be removed once complete
+      return true;
     if (CallInst *CI = dyn_cast<CallInst>(Inst))
       return CI->doesNotAccessMemory() && !CI->getType()->isVoidTy();
     return isa<CastInst>(Inst) || isa<UnaryOperator>(Inst) ||
@@ -280,6 +283,13 @@ static unsigned getHashValueImpl(SimpleValue Val) {
     return hash_combine(II->getOpcode(), LHS, RHS);
   }
 
+  // gc.relocate is 'special' call: its second and third operands are
+  // not real values, but indices into statepoint's argument list.
+  // Get values they point to.
+  if (const GCRelocateInst *GCR = dyn_cast<GCRelocateInst>(Inst))
+    return hash_combine(GCR->getOpcode(), GCR->getOperand(0),
+                        GCR->getBasePtr(), GCR->getDerivedPtr());
+
   // Mix in the opcode.
   return hash_combine(
       Inst->getOpcode(),
@@ -341,6 +351,13 @@ static bool isEqualImpl(SimpleValue LHS, SimpleValue RHS) {
            LII->getArgOperand(1) == RII->getArgOperand(0);
   }
 
+  // See comment above in `getHashValue()`.
+  if (const GCRelocateInst *GCR1 = dyn_cast<GCRelocateInst>(LHSI))
+    if (const GCRelocateInst *GCR2 = dyn_cast<GCRelocateInst>(RHSI))
+      return GCR1->getOperand(0) == GCR2->getOperand(0) &&
+             GCR1->getBasePtr() == GCR2->getBasePtr() &&
+             GCR1->getDerivedPtr() == GCR2->getDerivedPtr();
+
   // Min/max can occur with commuted operands, non-canonical predicates,
   // and/or non-canonical operands.
   // Selects can be non-trivially equivalent via inverted conditions and swaps.
@@ -454,13 +471,6 @@ template <> struct DenseMapInfo<CallValue> {
 unsigned DenseMapInfo<CallValue>::getHashValue(CallValue Val) {
   Instruction *Inst = Val.Inst;
 
-  // gc.relocate is 'special' call: its second and third operands are
-  // not real values, but indices into statepoint's argument list.
-  // Get values they point to.
-  if (const GCRelocateInst *GCR = dyn_cast<GCRelocateInst>(Inst))
-    return hash_combine(GCR->getOpcode(), GCR->getOperand(0),
-                        GCR->getBasePtr(), GCR->getDerivedPtr());
-
   // Hash all of the operands as pointers and mix in the opcode.
   return hash_combine(
       Inst->getOpcode(),
@@ -472,13 +482,6 @@ bool DenseMapInfo<CallValue>::isEqual(CallValue LHS, CallValue RHS) {
   if (LHS.isSentinel() || RHS.isSentinel())
     return LHSI == RHSI;
 
-  // See comment above in `getHashValue()`.
-  if (const GCRelocateInst *GCR1 = dyn_cast<GCRelocateInst>(LHSI))
-    if (const GCRelocateInst *GCR2 = dyn_cast<GCRelocateInst>(RHSI))
-      return GCR1->getOperand(0) == GCR2->getOperand(0) &&
-             GCR1->getBasePtr() == GCR2->getBasePtr() &&
-             GCR1->getDerivedPtr() == GCR2->getDerivedPtr();
-
   return LHSI->isIdenticalTo(RHSI);
 }
 
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 63a2166021f2dd8a58cf195ac3131cd973bdc7c0..e0942a5106d2ffdd1e78389380b77a784614bd8f 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -397,7 +397,9 @@ void GVN::ValueTable::add(Value *V, uint32_t num) {
 }
 
 uint32_t GVN::ValueTable::lookupOrAddCall(CallInst *C) {
-  if (AA->doesNotAccessMemory(C)) {
+  // The gc.relocate specific check is to simplify migration under PR49607, and
+  // is to be removed once complete.
+  if (AA->doesNotAccessMemory(C) || isa<GCRelocateInst>(C)) {
     Expression exp = createExpr(C);
     uint32_t e = assignExpNewValueNum(exp).first;
     valueNumbering[C] = e;
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index 6e09dec198c2c6b7288160e8e80d629de0a3a5a4..2e44ee8f20b71b273a33246c46b3d227f62c1522 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -360,7 +360,7 @@ void InductiveRangeCheck::extractRangeChecksFromCond(
     return;
 
   // TODO: Do the same for OR, XOR, NOT etc?
-  if (match(Condition, m_And(m_Value(), m_Value()))) {
+  if (match(Condition, m_LogicalAnd(m_Value(), m_Value()))) {
     extractRangeChecksFromCond(L, SE, cast<User>(Condition)->getOperandUse(0),
                                Checks, Visited);
     extractRangeChecksFromCond(L, SE, cast<User>(Condition)->getOperandUse(1),
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 2fb4a67ca1c982556bd9c25c75367d7b7949e477..e4852958703a4cab9f8039953f0abf049de8d461 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -31,6 +31,7 @@
 #include "llvm/Analysis/LazyValueInfo.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -1388,10 +1389,14 @@ bool JumpThreadingPass::simplifyPartiallyRedundantLoad(LoadInst *LoadI) {
            "Attempting to CSE volatile or atomic loads");
     // If this is a load on a phi pointer, phi-translate it and search
     // for available load/store to the pointer in predecessors.
-    Value *Ptr = LoadedPtr->DoPHITranslation(LoadBB, PredBB);
-    PredAvailable = FindAvailablePtrLoadStore(
-        Ptr, LoadI->getType(), LoadI->isAtomic(), PredBB, BBIt,
-        DefMaxInstsToScan, AA, &IsLoadCSE, &NumScanedInst);
+    Type *AccessTy = LoadI->getType();
+    const auto &DL = LoadI->getModule()->getDataLayout();
+    MemoryLocation Loc(LoadedPtr->DoPHITranslation(LoadBB, PredBB),
+                       LocationSize::precise(DL.getTypeStoreSize(AccessTy)),
+                       AATags);
+    PredAvailable = findAvailablePtrLoadStore(Loc, AccessTy, LoadI->isAtomic(),
+                                              PredBB, BBIt, DefMaxInstsToScan,
+                                              AA, &IsLoadCSE, &NumScanedInst);
 
     // If PredBB has a single predecessor, continue scanning through the
     // single predecessor.
@@ -1401,8 +1406,8 @@ bool JumpThreadingPass::simplifyPartiallyRedundantLoad(LoadInst *LoadI) {
       SinglePredBB = SinglePredBB->getSinglePredecessor();
       if (SinglePredBB) {
         BBIt = SinglePredBB->end();
-        PredAvailable = FindAvailablePtrLoadStore(
-            Ptr, LoadI->getType(), LoadI->isAtomic(), SinglePredBB, BBIt,
+        PredAvailable = findAvailablePtrLoadStore(
+            Loc, AccessTy, LoadI->isAtomic(), SinglePredBB, BBIt,
             (DefMaxInstsToScan - NumScanedInst), AA, &IsLoadCSE,
             &NumScanedInst);
       }
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 2875b301bbf107cef1b4246d5a840825b764d329..587b58ff562207a6b5e563b265bfd55fdc20da34 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -162,6 +162,7 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
                  OptimizationRemarkEmitter *ORE);
 static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                            const DominatorTree *DT,
+                                           const TargetLibraryInfo *TLI,
                                            const Loop *CurLoop,
                                            const LoopSafetyInfo *SafetyInfo,
                                            OptimizationRemarkEmitter *ORE,
@@ -921,7 +922,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                              ORE) &&
           worthSinkOrHoistInst(I, CurLoop->getLoopPreheader(), ORE, BFI) &&
           isSafeToExecuteUnconditionally(
-              I, DT, CurLoop, SafetyInfo, ORE,
+              I, DT, TLI, CurLoop, SafetyInfo, ORE,
               CurLoop->getLoopPreheader()->getTerminator())) {
         hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
               MSSAU, SE, ORE);
@@ -1502,24 +1503,23 @@ static Instruction *cloneInstructionInExitBlock(
     }
   }
 
-  // Build LCSSA PHI nodes for any in-loop operands. Note that this is
-  // particularly cheap because we can rip off the PHI node that we're
+  // Build LCSSA PHI nodes for any in-loop operands (if legal).  Note that
+  // this is particularly cheap because we can rip off the PHI node that we're
   // replacing for the number and blocks of the predecessors.
   // OPT: If this shows up in a profile, we can instead finish sinking all
   // invariant instructions, and then walk their operands to re-establish
   // LCSSA. That will eliminate creating PHI nodes just to nuke them when
   // sinking bottom-up.
   for (Use &Op : New->operands())
-    if (Instruction *OInst = dyn_cast<Instruction>(Op))
-      if (Loop *OLoop = LI->getLoopFor(OInst->getParent()))
-        if (!OLoop->contains(&PN)) {
-          PHINode *OpPN =
-              PHINode::Create(OInst->getType(), PN.getNumIncomingValues(),
-                              OInst->getName() + ".lcssa", &ExitBlock.front());
-          for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
-            OpPN->addIncoming(OInst, PN.getIncomingBlock(i));
-          Op = OpPN;
-        }
+    if (LI->wouldBeOutOfLoopUseRequiringLCSSA(Op.get(), PN.getParent())) {
+      auto *OInst = cast<Instruction>(Op.get());
+      PHINode *OpPN =
+        PHINode::Create(OInst->getType(), PN.getNumIncomingValues(),
+                        OInst->getName() + ".lcssa", &ExitBlock.front());
+      for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
+        OpPN->addIncoming(OInst, PN.getIncomingBlock(i));
+      Op = OpPN;
+    }
   return New;
 }
 
@@ -1662,17 +1662,8 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
                  BlockFrequencyInfo *BFI, const Loop *CurLoop,
                  ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
                  OptimizationRemarkEmitter *ORE) {
-  LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n");
-  ORE->emit([&]() {
-    return OptimizationRemark(DEBUG_TYPE, "InstSunk", &I)
-           << "sinking " << ore::NV("Inst", &I);
-  });
   bool Changed = false;
-  if (isa<LoadInst>(I))
-    ++NumMovedLoads;
-  else if (isa<CallInst>(I))
-    ++NumMovedCalls;
-  ++NumSunk;
+  LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n");
 
   // Iterate over users to be ready for actual sinking. Replace users via
   // unreachable blocks with undef and make all user PHIs trivially replaceable.
@@ -1724,6 +1715,16 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
   if (VisitedUsers.empty())
     return Changed;
 
+  ORE->emit([&]() {
+    return OptimizationRemark(DEBUG_TYPE, "InstSunk", &I)
+           << "sinking " << ore::NV("Inst", &I);
+  });
+  if (isa<LoadInst>(I))
+    ++NumMovedLoads;
+  else if (isa<CallInst>(I))
+    ++NumMovedCalls;
+  ++NumSunk;
+
 #ifndef NDEBUG
   SmallVector<BasicBlock *, 32> ExitBlocks;
   CurLoop->getUniqueExitBlocks(ExitBlocks);
@@ -1815,11 +1816,12 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
 /// or if it is a trapping instruction and is guaranteed to execute.
 static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                            const DominatorTree *DT,
+                                           const TargetLibraryInfo *TLI,
                                            const Loop *CurLoop,
                                            const LoopSafetyInfo *SafetyInfo,
                                            OptimizationRemarkEmitter *ORE,
                                            const Instruction *CtxI) {
-  if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT))
+  if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT, TLI))
     return true;
 
   bool GuaranteedToExecute =
@@ -1856,19 +1858,21 @@ class LoopPromoter : public LoadAndStorePromoter {
   AAMDNodes AATags;
   ICFLoopSafetyInfo &SafetyInfo;
 
+  // We're about to add a use of V in a loop exit block.  Insert an LCSSA phi
+  // (if legal) if doing so would add an out-of-loop use to an instruction
+  // defined in-loop.
   Value *maybeInsertLCSSAPHI(Value *V, BasicBlock *BB) const {
-    if (Instruction *I = dyn_cast<Instruction>(V))
-      if (Loop *L = LI.getLoopFor(I->getParent()))
-        if (!L->contains(BB)) {
-          // We need to create an LCSSA PHI node for the incoming value and
-          // store that.
-          PHINode *PN = PHINode::Create(I->getType(), PredCache.size(BB),
-                                        I->getName() + ".lcssa", &BB->front());
-          for (BasicBlock *Pred : PredCache.get(BB))
-            PN->addIncoming(I, Pred);
-          return PN;
-        }
-    return V;
+    if (!LI.wouldBeOutOfLoopUseRequiringLCSSA(V, BB))
+      return V;
+
+    Instruction *I = cast<Instruction>(V);
+    // We need to create an LCSSA PHI node for the incoming value and
+    // store that.
+    PHINode *PN = PHINode::Create(I->getType(), PredCache.size(BB),
+                                  I->getName() + ".lcssa", &BB->front());
+    for (BasicBlock *Pred : PredCache.get(BB))
+      PN->addIncoming(I, Pred);
+    return PN;
   }
 
 public:
@@ -2091,8 +2095,9 @@ bool llvm::promoteLoopAccessesToScalars(
         // to execute does as well.  Thus we can increase our guaranteed
         // alignment as well. 
         if (!DereferenceableInPH || (InstAlignment > Alignment))
-          if (isSafeToExecuteUnconditionally(*Load, DT, CurLoop, SafetyInfo,
-                                             ORE, Preheader->getTerminator())) {
+          if (isSafeToExecuteUnconditionally(*Load, DT, TLI, CurLoop,
+                                             SafetyInfo, ORE,
+                                             Preheader->getTerminator())) {
             DereferenceableInPH = true;
             Alignment = std::max(Alignment, InstAlignment);
           }
@@ -2139,7 +2144,7 @@ bool llvm::promoteLoopAccessesToScalars(
         if (!DereferenceableInPH) {
           DereferenceableInPH = isDereferenceableAndAlignedPointer(
               Store->getPointerOperand(), Store->getValueOperand()->getType(),
-              Store->getAlign(), MDL, Preheader->getTerminator(), DT);
+              Store->getAlign(), MDL, Preheader->getTerminator(), DT, TLI);
         }
       } else
         return false; // Not a load or store.
diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
index 604eb64a94e0abec000b68b49b8872c52f742879..5b5196529734fcfc05ccacb5afc21e6b927b7098 100644
--- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
@@ -176,8 +176,12 @@ static bool findLoopComponents(
   LLVM_DEBUG(dbgs() << "Found limit: "; Limit->dump());
 
   assert(InductionPHI->getNumIncomingValues() == 2);
-  assert(InductionPHI->getIncomingValueForBlock(Latch) == Increment &&
-         "PHI value is not increment inst");
+
+  if (InductionPHI->getIncomingValueForBlock(Latch) != Increment) {
+    LLVM_DEBUG(
+        dbgs() << "Incoming value from latch is not the increment inst\n");
+    return false;
+  }
 
   auto *CI = dyn_cast<ConstantInt>(
       InductionPHI->getIncomingValueForBlock(L->getLoopPreheader()));
@@ -190,8 +194,7 @@ static bool findLoopComponents(
   return true;
 }
 
-static bool checkPHIs(struct FlattenInfo &FI,
-                      const TargetTransformInfo *TTI) {
+static bool checkPHIs(FlattenInfo &FI, const TargetTransformInfo *TTI) {
   // All PHIs in the inner and outer headers must either be:
   // - The induction PHI, which we are going to rewrite as one induction in
   //   the new loop. This is already checked by findLoopComponents.
@@ -272,7 +275,7 @@ static bool checkPHIs(struct FlattenInfo &FI,
 }
 
 static bool
-checkOuterLoopInsts(struct FlattenInfo &FI,
+checkOuterLoopInsts(FlattenInfo &FI,
                     SmallPtrSetImpl<Instruction *> &IterationInstructions,
                     const TargetTransformInfo *TTI) {
   // Check for instructions in the outer but not inner loop. If any of these
@@ -330,7 +333,7 @@ checkOuterLoopInsts(struct FlattenInfo &FI,
   return true;
 }
 
-static bool checkIVUsers(struct FlattenInfo &FI) {
+static bool checkIVUsers(FlattenInfo &FI) {
   // We require all uses of both induction variables to match this pattern:
   //
   //   (OuterPHI * InnerLimit) + InnerPHI
@@ -426,8 +429,8 @@ static bool checkIVUsers(struct FlattenInfo &FI) {
 
 // Return an OverflowResult dependant on if overflow of the multiplication of
 // InnerLimit and OuterLimit can be assumed not to happen.
-static OverflowResult checkOverflow(struct FlattenInfo &FI,
-                                    DominatorTree *DT, AssumptionCache *AC) {
+static OverflowResult checkOverflow(FlattenInfo &FI, DominatorTree *DT,
+                                    AssumptionCache *AC) {
   Function *F = FI.OuterLoop->getHeader()->getParent();
   const DataLayout &DL = F->getParent()->getDataLayout();
 
@@ -465,9 +468,9 @@ static OverflowResult checkOverflow(struct FlattenInfo &FI,
   return OverflowResult::MayOverflow;
 }
 
-static bool CanFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
-                               LoopInfo *LI, ScalarEvolution *SE,
-                               AssumptionCache *AC, const TargetTransformInfo *TTI) {
+static bool CanFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
+                               ScalarEvolution *SE, AssumptionCache *AC,
+                               const TargetTransformInfo *TTI) {
   SmallPtrSet<Instruction *, 8> IterationInstructions;
   if (!findLoopComponents(FI.InnerLoop, IterationInstructions, FI.InnerInductionPHI,
                           FI.InnerLimit, FI.InnerIncrement, FI.InnerBranch, SE))
@@ -509,9 +512,8 @@ static bool CanFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
   return true;
 }
 
-static bool DoFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
-                              LoopInfo *LI, ScalarEvolution *SE,
-                              AssumptionCache *AC,
+static bool DoFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
+                              ScalarEvolution *SE, AssumptionCache *AC,
                               const TargetTransformInfo *TTI) {
   Function *F = FI.OuterLoop->getHeader()->getParent();
   LLVM_DEBUG(dbgs() << "Checks all passed, doing the transformation\n");
@@ -572,9 +574,9 @@ static bool DoFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
   return true;
 }
 
-static bool CanWidenIV(struct FlattenInfo &FI, DominatorTree *DT,
-                       LoopInfo *LI, ScalarEvolution *SE,
-                       AssumptionCache *AC, const TargetTransformInfo *TTI) {
+static bool CanWidenIV(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
+                       ScalarEvolution *SE, AssumptionCache *AC,
+                       const TargetTransformInfo *TTI) {
   if (!WidenIV) {
     LLVM_DEBUG(dbgs() << "Widening the IVs is disabled\n");
     return false;
@@ -622,9 +624,8 @@ static bool CanWidenIV(struct FlattenInfo &FI, DominatorTree *DT,
   return CanFlattenLoopPair(FI, DT, LI, SE, AC, TTI);
 }
 
-static bool FlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
-                            LoopInfo *LI, ScalarEvolution *SE,
-                            AssumptionCache *AC,
+static bool FlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
+                            ScalarEvolution *SE, AssumptionCache *AC,
                             const TargetTransformInfo *TTI) {
   LLVM_DEBUG(
       dbgs() << "Loop flattening running on outer loop "
@@ -664,7 +665,7 @@ bool Flatten(DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE,
     auto *OuterLoop = InnerLoop->getParentLoop();
     if (!OuterLoop)
       continue;
-    struct FlattenInfo FI(OuterLoop, InnerLoop);
+    FlattenInfo FI(OuterLoop, InnerLoop);
     Changed |= FlattenLoopPair(FI, DT, LI, SE, AC, TTI);
   }
   return Changed;
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 0162bf1307af0d27ae317ca408891c24cc7bae31..5da619fc08f269eae98c826bab0d7bb059793b33 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopNestAnalysis.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ScalarEvolution.h"
@@ -617,6 +618,22 @@ bool LoopInterchangeLegality::tightlyNested(Loop *OuterLoop, Loop *InnerLoop) {
       containsUnsafeInstructions(InnerLoopPreHeader))
     return false;
 
+  BasicBlock *InnerLoopExit = InnerLoop->getExitBlock();
+  // Ensure the inner loop exit block flows to the outer loop latch possibly
+  // through empty blocks.
+  const BasicBlock &SuccInner =
+      LoopNest::skipEmptyBlockUntil(InnerLoopExit, OuterLoopLatch);
+  if (&SuccInner != OuterLoopLatch) {
+    LLVM_DEBUG(dbgs() << "Inner loop exit block " << *InnerLoopExit
+                      << " does not lead to the outer loop latch.\n";);
+    return false;
+  }
+  // The inner loop exit block does flow to the outer loop latch and not some
+  // other BBs, now make sure it contains safe instructions, since it will be
+  // moved into the (new) inner loop after interchange.
+  if (containsUnsafeInstructions(InnerLoopExit))
+    return false;
+
   LLVM_DEBUG(dbgs() << "Loops are perfectly nested\n");
   // We have a perfect loop nest.
   return true;
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index 60a9602096bbe962a0566f59d76780b3c75fe77b..0bb3ec46703c5a7c1405b9534882781e76bd4d94 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -14,6 +14,7 @@
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/TimeProfiler.h"
 
 using namespace llvm;
@@ -291,8 +292,17 @@ PreservedAnalyses FunctionToLoopPassAdaptor::run(Function &F,
     else
       PI.runAfterPass<Loop>(*Pass, *L, PassPA);
 
-    // FIXME: We should verify the set of analyses relevant to Loop passes
-    // are preserved.
+#ifndef NDEBUG
+    // LoopAnalysisResults should always be valid.
+    // Note that we don't LAR.SE.verify() because that can change observed SE
+    // queries. See PR44815.
+    if (VerifyDomInfo)
+      LAR.DT.verify();
+    if (VerifyLoopInfo)
+      LAR.LI.verify(LAR.DT);
+    if (LAR.MSSA && VerifyMemorySSA)
+      LAR.MSSA->verifyMemorySSA();
+#endif
 
     // If the loop hasn't been deleted, we need to handle invalidation here.
     if (!Updater.skipCurrentLoop())
diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
index da13075dfee2634785b6ee510932218e33d62952..ead8082f30360da9954c4b71731d15b950da1aa8 100644
--- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -24,6 +24,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Scalar.h"
 
@@ -41,15 +42,15 @@ STATISTIC(ExpectIntrinsicsHandled,
 // only be used in extreme cases, we could make this ratio higher. As it stands,
 // programmers may be using __builtin_expect() / llvm.expect to annotate that a
 // branch is likely or unlikely to be taken.
-//
-// There is a known dependency on this ratio in CodeGenPrepare when transforming
-// 'select' instructions. It may be worthwhile to hoist these values to some
-// shared space, so they can be used directly by other passes.
 
-cl::opt<uint32_t> llvm::LikelyBranchWeight(
+// WARNING: these values are internal implementation detail of the pass.
+// They should not be exposed to the outside of the pass, front-end codegen
+// should emit @llvm.expect intrinsics instead of using these weights directly.
+// Transforms should use TargetTransformInfo's getPredictableBranchThreshold().
+static cl::opt<uint32_t> LikelyBranchWeight(
     "likely-branch-weight", cl::Hidden, cl::init(2000),
     cl::desc("Weight of the branch likely to be taken (default = 2000)"));
-cl::opt<uint32_t> llvm::UnlikelyBranchWeight(
+static cl::opt<uint32_t> UnlikelyBranchWeight(
     "unlikely-branch-weight", cl::Hidden, cl::init(1),
     cl::desc("Weight of the branch unlikely to be taken (default = 1)"));
 
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index e07eb65932007fa446853d129db674fee72ade3a..544f67cd79e2ff56ba43569c08f81345b9c7bbe4 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -413,7 +413,9 @@ public:
   /// \p VT * N.
   unsigned getNumOps(Type *ST, unsigned N) {
     return std::ceil((ST->getPrimitiveSizeInBits() * N).getFixedSize() /
-                     double(TTI.getRegisterBitWidth(true)));
+                     double(TTI.getRegisterBitWidth(
+                                   TargetTransformInfo::RGK_FixedWidthVector)
+                                .getFixedSize()));
   }
 
   /// Return the set of vectors that a matrix value is lowered to.
@@ -1013,7 +1015,8 @@ public:
                           const MatrixTy &B, bool AllowContraction,
                           IRBuilder<> &Builder, bool isTiled) {
     const unsigned VF = std::max<unsigned>(
-        TTI.getRegisterBitWidth(true) /
+        TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+                .getFixedSize() /
             Result.getElementType()->getPrimitiveSizeInBits().getFixedSize(),
         1U);
     unsigned R = Result.getNumRows();
@@ -1179,10 +1182,11 @@ public:
     const unsigned M = LShape.NumColumns;
     auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
 
-    const unsigned VF =
-        std::max<unsigned>(TTI.getRegisterBitWidth(true) /
-                               EltType->getPrimitiveSizeInBits().getFixedSize(),
-                           1U);
+    const unsigned VF = std::max<unsigned>(
+        TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+                .getFixedSize() /
+            EltType->getPrimitiveSizeInBits().getFixedSize(),
+        1U);
 
     // Cost model for tiling
     //
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 5c1c6b2a2d8f24f6df70303be2e14747790df3de..f98a06490d98f5fdfde9d5d539b434868fea44ca 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -308,6 +308,7 @@ INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
 INITIALIZE_PASS_END(MemCpyOptLegacyPass, "memcpyopt", "MemCpy Optimization",
                     false, false)
 
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index 53e256f8b852a47c1f6a18238c8e92639f9fbf66..2603ea0f92ed3ab580a0e7fce2eddb99962be2c8 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -782,8 +782,9 @@ static BDVState meetBDVState(const BDVState &LHS, const BDVState &RHS) {
   if (LHS.getStatus() == BDVState::Base && RHS.getStatus() == BDVState::Base &&
       LHS.getBaseValue() != RHS.getBaseValue()) {
     NewStatus = BDVState::Conflict;
-    BaseValue = nullptr;
   }
+  if (NewStatus == BDVState::Conflict)
+    BaseValue = nullptr;
   return BDVState(LHS.getOriginalValue(), NewStatus, BaseValue);
 }
 
@@ -855,8 +856,11 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
       F(IE->getOperand(0));
       F(IE->getOperand(1));
     } else if (auto *SV = dyn_cast<ShuffleVectorInst>(BDV)) {
+      // For a canonical broadcast, ignore the undef argument
+      // (without this, we insert a parallel base shuffle for every broadcast)
       F(SV->getOperand(0));
-      F(SV->getOperand(1));
+      if (!SV->isZeroEltSplat())
+        F(SV->getOperand(1));
     } else {
       llvm_unreachable("unexpected BDV type");
     }
@@ -900,14 +904,51 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
   }
 #endif
 
+  // Iterate forward through the value graph pruning any node from the state
+  // list where all of the inputs are base pointers.  The purpose of this is to
+  // reuse existing values when the derived pointer we were asked to materialize
+  // a base pointer for happens to be a base pointer itself.  (Or a sub-graph
+  // feeding it does.)
+  SmallVector<Value *> ToRemove;
+  do {
+    ToRemove.clear();
+    for (auto Pair : States) {
+      Value *BDV = Pair.first;
+      auto canPruneInput = [&](Value *V) {
+        Value *BDV = findBaseOrBDV(V, Cache);
+        if (V->stripPointerCasts() != BDV)
+          return false;
+        // The assumption is that anything not in the state list is
+        // propagates a base pointer.
+        return States.count(BDV) == 0;
+      };
+
+      bool CanPrune = true;
+      visitBDVOperands(BDV, [&](Value *Op) {
+        CanPrune = CanPrune && canPruneInput(Op);
+      });
+      if (CanPrune)
+        ToRemove.push_back(BDV);
+    }
+    for (Value *V : ToRemove) {
+      States.erase(V);
+      // Cache the fact V is it's own base for later usage.
+      Cache[V] = V;
+    }
+  } while (!ToRemove.empty());
+
+  // Did we manage to prove that Def itself must be a base pointer?
+  if (!States.count(Def))
+    return Def;
+
   // Return a phi state for a base defining value.  We'll generate a new
   // base state for known bases and expect to find a cached state otherwise.
   auto GetStateForBDV = [&](Value *BaseValue, Value *Input) {
-    if (isKnownBaseResult(BaseValue) && areBothVectorOrScalar(BaseValue, Input))
-      return BDVState(BaseValue, BDVState::Base, BaseValue);
     auto I = States.find(BaseValue);
-    assert(I != States.end() && "lookup failed!");
-    return I->second;
+    if (I != States.end())
+      return I->second;
+    assert(areBothVectorOrScalar(BaseValue, Input));
+    return BDVState(BaseValue, BDVState::Base, BaseValue);
   };
 
   bool Progress = true;
@@ -1017,40 +1058,23 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
     if (!State.isConflict())
       continue;
 
-    /// Create and insert a new instruction which will represent the base of
-    /// the given instruction 'I'.
-    auto MakeBaseInstPlaceholder = [](Instruction *I) -> Instruction* {
+    auto getMangledName = [](Instruction *I) -> std::string {
       if (isa<PHINode>(I)) {
-        BasicBlock *BB = I->getParent();
-        int NumPreds = pred_size(BB);
-        assert(NumPreds > 0 && "how did we reach here");
-        std::string Name = suffixed_name_or(I, ".base", "base_phi");
-        return PHINode::Create(I->getType(), NumPreds, Name, I);
-      } else if (SelectInst *SI = dyn_cast<SelectInst>(I)) {
-        // The undef will be replaced later
-        UndefValue *Undef = UndefValue::get(SI->getType());
-        std::string Name = suffixed_name_or(I, ".base", "base_select");
-        return SelectInst::Create(SI->getCondition(), Undef, Undef, Name, SI);
-      } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {
-        UndefValue *Undef = UndefValue::get(EE->getVectorOperand()->getType());
-        std::string Name = suffixed_name_or(I, ".base", "base_ee");
-        return ExtractElementInst::Create(Undef, EE->getIndexOperand(), Name,
-                                          EE);
-      } else if (auto *IE = dyn_cast<InsertElementInst>(I)) {
-        UndefValue *VecUndef = UndefValue::get(IE->getOperand(0)->getType());
-        UndefValue *ScalarUndef = UndefValue::get(IE->getOperand(1)->getType());
-        std::string Name = suffixed_name_or(I, ".base", "base_ie");
-        return InsertElementInst::Create(VecUndef, ScalarUndef,
-                                         IE->getOperand(2), Name, IE);
+        return suffixed_name_or(I, ".base", "base_phi");
+      } else if (isa<SelectInst>(I)) {
+        return suffixed_name_or(I, ".base", "base_select");
+      } else if (isa<ExtractElementInst>(I)) {
+        return suffixed_name_or(I, ".base", "base_ee");
+      } else if (isa<InsertElementInst>(I)) {
+        return suffixed_name_or(I, ".base", "base_ie");
       } else {
-        auto *SV = cast<ShuffleVectorInst>(I);
-        UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType());
-        std::string Name = suffixed_name_or(I, ".base", "base_sv");
-        return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(),
-                                     Name, SV);
+        return suffixed_name_or(I, ".base", "base_sv");
       }
     };
-    Instruction *BaseInst = MakeBaseInstPlaceholder(I);
+
+    Instruction *BaseInst = I->clone();
+    BaseInst->insertBefore(I);
+    BaseInst->setName(getMangledName(I));
     // Add metadata marking this as a base value
     BaseInst->setMetadata("is_base_value", MDNode::get(I->getContext(), {}));
     States[I] = BDVState(I, BDVState::Conflict, BaseInst);
@@ -1071,7 +1095,8 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
   auto getBaseForInput = [&](Value *Input, Instruction *InsertPt) {
     Value *BDV = findBaseOrBDV(Input, Cache);
     Value *Base = nullptr;
-    if (isKnownBaseResult(BDV) && areBothVectorOrScalar(BDV, Input)) {
+    if (!States.count(BDV)) {
+      assert(areBothVectorOrScalar(BDV, Input));
       Base = BDV;
     } else {
       // Either conflict or base.
@@ -1104,26 +1129,21 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
 
     if (PHINode *BasePHI = dyn_cast<PHINode>(State.getBaseValue())) {
       PHINode *PN = cast<PHINode>(BDV);
-      unsigned NumPHIValues = PN->getNumIncomingValues();
+      const unsigned NumPHIValues = PN->getNumIncomingValues();
+
+      // The IR verifier requires phi nodes with multiple entries from the
+      // same basic block to have the same incoming value for each of those
+      // entries.  Since we're inserting bitcasts in the loop, make sure we
+      // do so at least once per incoming block.
+      DenseMap<BasicBlock *, Value*> BlockToValue;
       for (unsigned i = 0; i < NumPHIValues; i++) {
         Value *InVal = PN->getIncomingValue(i);
         BasicBlock *InBB = PN->getIncomingBlock(i);
-
-        // If we've already seen InBB, add the same incoming value
-        // we added for it earlier.  The IR verifier requires phi
-        // nodes with multiple entries from the same basic block
-        // to have the same incoming value for each of those
-        // entries.  If we don't do this check here and basephi
-        // has a different type than base, we'll end up adding two
-        // bitcasts (and hence two distinct values) as incoming
-        // values for the same basic block.
-
-        int BlockIndex = BasePHI->getBasicBlockIndex(InBB);
-        if (BlockIndex != -1) {
-          Value *OldBase = BasePHI->getIncomingValue(BlockIndex);
-          BasePHI->addIncoming(OldBase, InBB);
-
+        if (!BlockToValue.count(InBB))
+          BlockToValue[InBB] = getBaseForInput(InVal, InBB->getTerminator());
+        else {
 #ifndef NDEBUG
+          Value *OldBase = BlockToValue[InBB];
           Value *Base = getBaseForInput(InVal, nullptr);
           // In essence this assert states: the only way two values
           // incoming from the same basic block may be different is by
@@ -1134,16 +1154,10 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
           assert(Base->stripPointerCasts() == OldBase->stripPointerCasts() &&
                  "Sanity -- findBaseOrBDV should be pure!");
 #endif
-          continue;
         }
-
-        // Find the instruction which produces the base for each input.  We may
-        // need to insert a bitcast in the incoming block.
-        // TODO: Need to split critical edges if insertion is needed
-        Value *Base = getBaseForInput(InVal, InBB->getTerminator());
-        BasePHI->addIncoming(Base, InBB);
+        Value *Base = BlockToValue[InBB];
+        BasePHI->setIncomingValue(i, Base);
       }
-      assert(BasePHI->getNumIncomingValues() == NumPHIValues);
     } else if (SelectInst *BaseSI =
                    dyn_cast<SelectInst>(State.getBaseValue())) {
       SelectInst *SI = cast<SelectInst>(BDV);
@@ -1176,7 +1190,13 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
         BaseSV->setOperand(OperandIdx, Base);
       };
       UpdateOperand(0); // vector operand
-      UpdateOperand(1); // vector operand
+      if (!BdvSV->isZeroEltSplat())
+        UpdateOperand(1); // vector operand
+      else {
+        // Never read, so just use undef
+        Value *InVal = BdvSV->getOperand(1);
+        BaseSV->setOperand(1, UndefValue::get(InVal->getType()));
+      }
     }
   }
 
@@ -1203,14 +1223,6 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
                << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none")
                << " to: " << Base->getName() << "\n");
 
-    if (Cache.count(BDV)) {
-      assert(isKnownBaseResult(Base) &&
-             "must be something we 'know' is a base pointer");
-      // Once we transition from the BDV relation being store in the Cache to
-      // the base relation being stored, it must be stable
-      assert((!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) &&
-             "base relation should be stable");
-    }
     Cache[BDV] = Base;
   }
   assert(Cache.count(Def));
@@ -3016,11 +3028,7 @@ static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
   // We may have base pointers which are now live that weren't before.  We need
   // to update the PointerToBase structure to reflect this.
   for (auto V : Updated)
-    if (Info.PointerToBase.insert({V, V}).second) {
-      assert(isKnownBaseResult(V) &&
-             "Can't find base for unexpected live value!");
-      continue;
-    }
+    Info.PointerToBase.insert({V, V});
 
 #ifndef NDEBUG
   for (auto V : Updated)
diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index 0e5a787cea70cd5539f2b3386ad5f901830e9fce..cf77cf70e32326c3d3058a257d2191a6269f1f2e 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -2684,11 +2684,13 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
   // don't know how to split those exit blocks.
   // FIXME: We should teach SplitBlock to handle this and remove this
   // restriction.
-  for (auto *ExitBB : ExitBlocks)
+  for (auto *ExitBB : ExitBlocks) {
     if (isa<CleanupPadInst>(ExitBB->getFirstNonPHI())) {
-      dbgs() << "Cannot unswitch because of cleanuppad in exit block\n";
+      LLVM_DEBUG(
+          dbgs() << "Cannot unswitch because of cleanuppad in exit block\n");
       return false;
     }
+  }
 
   LLVM_DEBUG(
       dbgs() << "Considering " << UnswitchCandidates.size()
@@ -2899,10 +2901,20 @@ static bool unswitchLoop(Loop &L, DominatorTree &DT, LoopInfo &LI,
     return true;
   }
 
-  // If we're not doing non-trivial unswitching, we're done. We both accept
-  // a parameter but also check a local flag that can be used for testing
-  // a debugging.
-  if (!NonTrivial && !EnableNonTrivialUnswitch)
+  // Check whether we should continue with non-trivial conditions.
+  // EnableNonTrivialUnswitch: Global variable that forces non-trivial
+  //                           unswitching for testing and debugging.
+  // NonTrivial: Parameter that enables non-trivial unswitching for this
+  //             invocation of the transform. But this should be allowed only
+  //             for targets without branch divergence.
+  //
+  // FIXME: If divergence analysis becomes available to a loop
+  // transform, we should allow unswitching for non-trivial uniform
+  // branches even on targets that have divergence.
+  // https://bugs.llvm.org/show_bug.cgi?id=48819
+  bool ContinueWithNonTrivial =
+      EnableNonTrivialUnswitch || (NonTrivial && !TTI.hasBranchDivergence());
+  if (!ContinueWithNonTrivial)
     return false;
 
   // Skip non-trivial unswitching for optsize functions.
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index d1a3ae5c0ed079ea12fc40ddd6b0dd8fa0528f08..2d5f1dd5e9637c6b453c1dc51ddfc9eed6855feb 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -228,8 +228,7 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
   // These dominator edges will be redirected from Pred.
   std::vector<DominatorTree::UpdateType> Updates;
   if (DTU) {
-    SmallSetVector<BasicBlock *, 2> UniqueSuccessors(succ_begin(BB),
-                                                     succ_end(BB));
+    SmallPtrSet<BasicBlock *, 2> UniqueSuccessors(succ_begin(BB), succ_end(BB));
     Updates.reserve(1 + (2 * UniqueSuccessors.size()));
     // Add insert edges first. Experimentally, for the particular case of two
     // blocks that can be merged, with a single successor and single predecessor
@@ -569,8 +568,8 @@ static BasicBlock *SplitBlockImpl(BasicBlock *Old, Instruction *SplitPt,
   if (DTU) {
     SmallVector<DominatorTree::UpdateType, 8> Updates;
     // Old dominates New. New node dominates all other nodes dominated by Old.
-    SmallSetVector<BasicBlock *, 8> UniqueSuccessorsOfOld(succ_begin(New),
-                                                          succ_end(New));
+    SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfOld(succ_begin(New),
+                                                       succ_end(New));
     Updates.push_back({DominatorTree::Insert, Old, New});
     Updates.reserve(Updates.size() + 2 * UniqueSuccessorsOfOld.size());
     for (BasicBlock *UniqueSuccessorOfOld : UniqueSuccessorsOfOld) {
@@ -635,8 +634,8 @@ BasicBlock *llvm::splitBlockBefore(BasicBlock *Old, Instruction *SplitPt,
     SmallVector<DominatorTree::UpdateType, 8> DTUpdates;
     // New dominates Old. The predecessor nodes of the Old node dominate
     // New node.
-    SmallSetVector<BasicBlock *, 8> UniquePredecessorsOfOld(pred_begin(New),
-                                                            pred_end(New));
+    SmallPtrSet<BasicBlock *, 8> UniquePredecessorsOfOld(pred_begin(New),
+                                                         pred_end(New));
     DTUpdates.push_back({DominatorTree::Insert, New, Old});
     DTUpdates.reserve(DTUpdates.size() + 2 * UniquePredecessorsOfOld.size());
     for (BasicBlock *UniquePredecessorOfOld : UniquePredecessorsOfOld) {
@@ -675,7 +674,7 @@ static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
     } else {
       // Split block expects NewBB to have a non-empty set of predecessors.
       SmallVector<DominatorTree::UpdateType, 8> Updates;
-      SmallSetVector<BasicBlock *, 8> UniquePreds(Preds.begin(), Preds.end());
+      SmallPtrSet<BasicBlock *, 8> UniquePreds(Preds.begin(), Preds.end());
       Updates.push_back({DominatorTree::Insert, NewBB, OldBB});
       Updates.reserve(Updates.size() + 2 * UniquePreds.size());
       for (auto *UniquePred : UniquePreds) {
@@ -1141,8 +1140,8 @@ SplitBlockAndInsertIfThenImpl(Value *Cond, Instruction *SplitBefore,
   BasicBlock *Head = SplitBefore->getParent();
   BasicBlock *Tail = Head->splitBasicBlock(SplitBefore->getIterator());
   if (DTU) {
-    SmallSetVector<BasicBlock *, 8> UniqueSuccessorsOfHead(succ_begin(Tail),
-                                                           succ_end(Tail));
+    SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfHead(succ_begin(Tail),
+                                                        succ_end(Tail));
     Updates.push_back({DominatorTree::Insert, Head, Tail});
     Updates.reserve(Updates.size() + 2 * UniqueSuccessorsOfHead.size());
     for (BasicBlock *UniqueSuccessorOfHead : UniqueSuccessorsOfHead) {
diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index 0ff28e9b3c29e03a28a914adb1fa770166b17938..6e139e2d8d7ad6c7e419759af8a590fb72378d26 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -72,7 +72,7 @@ BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
   }
 
   if (CodeInfo) {
-    CodeInfo->ContainsCalls          |= hasCalls;
+    CodeInfo->ContainsCalls |= hasCalls;
     CodeInfo->ContainsDynamicAllocas |= hasDynamicAllocas;
   }
   return NewBB;
@@ -125,6 +125,11 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
       AttributeList::get(NewFunc->getContext(), OldAttrs.getFnAttributes(),
                          OldAttrs.getRetAttributes(), NewArgAttrs));
 
+  // Everything else beyond this point deals with function instructions,
+  // so if we are dealing with a function declaration, we're done.
+  if (OldFunc->isDeclaration())
+    return;
+
   // When we remap instructions within the same module, we want to avoid
   // duplicating inlined DISubprograms, so record all subprograms we find as we
   // duplicate instructions and then freeze them in the MD map. We also record
@@ -149,7 +154,7 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
   } else {
     assert((NewFunc->getParent() == nullptr ||
             NewFunc->getParent() != OldFunc->getParent()) &&
-           "Set SameModule to true if the new function is in the same module");
+           "Expected NewFunc to have different parents, or no parent");
 
     if (Changes == CloneFunctionChangeType::DifferentModule) {
       assert(NewFunc->getParent() &&
@@ -160,17 +165,10 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
     }
   }
 
-  // Everything else beyond this point deals with function instructions,
-  // so if we are dealing with a function declaration, we're done.
-  if (OldFunc->isDeclaration())
-    return;
-
   // Loop over all of the basic blocks in the function, cloning them as
   // appropriate.  Note that we save BE this way in order to handle cloning of
   // recursive functions into themselves.
-  for (Function::const_iterator BI = OldFunc->begin(), BE = OldFunc->end();
-       BI != BE; ++BI) {
-    const BasicBlock &BB = *BI;
+  for (const BasicBlock &BB : *OldFunc) {
 
     // Create a new basic block and copy instructions into it!
     BasicBlock *CBB = CloneBasicBlock(&BB, VMap, NameSuffix, NewFunc, CodeInfo,
@@ -186,8 +184,8 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
     // implementation, which generates an invalid blockaddress when
     // cloning a function.)
     if (BB.hasAddressTaken()) {
-      Constant *OldBBAddr = BlockAddress::get(const_cast<Function*>(OldFunc),
-                                              const_cast<BasicBlock*>(&BB));
+      Constant *OldBBAddr = BlockAddress::get(const_cast<Function *>(OldFunc),
+                                              const_cast<BasicBlock *>(&BB));
       VMap[OldBBAddr] = BlockAddress::get(NewFunc, CBB);
     }
 
@@ -225,30 +223,26 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
            "Subprogram should be in DIFinder->subprogram_count()...");
   }
 
+  const auto RemapFlag = ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges;
   // Duplicate the metadata that is attached to the cloned function.
   // Subprograms/CUs/types that were already mapped to themselves won't be
   // duplicated.
   SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
   OldFunc->getAllMetadata(MDs);
   for (auto MD : MDs) {
-    NewFunc->addMetadata(
-        MD.first,
-        *MapMetadata(MD.second, VMap,
-                     ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges,
-                     TypeMapper, Materializer));
+    NewFunc->addMetadata(MD.first, *MapMetadata(MD.second, VMap, RemapFlag,
+                                                TypeMapper, Materializer));
   }
 
-  // Loop over all of the instructions in the function, fixing up operand
-  // references as we go.  This uses VMap to do all the hard work.
-  for (Function::iterator BB =
-           cast<BasicBlock>(VMap[&OldFunc->front()])->getIterator(),
-                          BE = NewFunc->end();
+  // Loop over all of the instructions in the new function, fixing up operand
+  // references as we go. This uses VMap to do all the hard work.
+  for (Function::iterator
+           BB = cast<BasicBlock>(VMap[&OldFunc->front()])->getIterator(),
+           BE = NewFunc->end();
        BB != BE; ++BB)
     // Loop over all instructions, fixing each one as we find it...
     for (Instruction &II : *BB)
-      RemapInstruction(&II, VMap,
-                       ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges,
-                       TypeMapper, Materializer);
+      RemapInstruction(&II, VMap, RemapFlag, TypeMapper, Materializer);
 
   // Only update !llvm.dbg.cu for DifferentModule (not CloneModule). In the
   // same module, the compile unit will already be listed (or not). When
@@ -266,7 +260,7 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
   // visiting the metadata attached to global values, which would allow this
   // code to be deleted. Alternatively, perhaps give responsibility for this
   // update to CloneFunctionInto's callers.
-  auto* NewModule = NewFunc->getParent();
+  auto *NewModule = NewFunc->getParent();
   auto *NMD = NewModule->getOrInsertNamedMetadata("llvm.dbg.cu");
   // Avoid multiple insertions of the same DICompileUnit to NMD.
   SmallPtrSet<const void *, 8> Visited;
@@ -289,7 +283,7 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
 ///
 Function *llvm::CloneFunction(Function *F, ValueToValueMapTy &VMap,
                               ClonedCodeInfo *CodeInfo) {
-  std::vector<Type*> ArgTypes;
+  std::vector<Type *> ArgTypes;
 
   // The user might be deleting arguments to the function by specifying them in
   // the VMap.  If so, we need to not add the arguments to the arg ty vector
@@ -299,8 +293,9 @@ Function *llvm::CloneFunction(Function *F, ValueToValueMapTy &VMap,
       ArgTypes.push_back(I.getType());
 
   // Create a new function type...
-  FunctionType *FTy = FunctionType::get(F->getFunctionType()->getReturnType(),
-                                    ArgTypes, F->getFunctionType()->isVarArg());
+  FunctionType *FTy =
+      FunctionType::get(F->getFunctionType()->getReturnType(), ArgTypes,
+                        F->getFunctionType()->isVarArg());
 
   // Create the new function...
   Function *NewF = Function::Create(FTy, F->getLinkage(), F->getAddressSpace(),
@@ -308,61 +303,60 @@ Function *llvm::CloneFunction(Function *F, ValueToValueMapTy &VMap,
 
   // Loop over the arguments, copying the names of the mapped arguments over...
   Function::arg_iterator DestI = NewF->arg_begin();
-  for (const Argument & I : F->args())
+  for (const Argument &I : F->args())
     if (VMap.count(&I) == 0) {     // Is this argument preserved?
       DestI->setName(I.getName()); // Copy the name over...
       VMap[&I] = &*DestI++;        // Add mapping to VMap
     }
 
-  SmallVector<ReturnInst*, 8> Returns;  // Ignore returns cloned.
+  SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
   CloneFunctionInto(NewF, F, VMap, CloneFunctionChangeType::LocalChangesOnly,
                     Returns, "", CodeInfo);
 
   return NewF;
 }
 
-
-
 namespace {
-  /// This is a private class used to implement CloneAndPruneFunctionInto.
-  struct PruningFunctionCloner {
-    Function *NewFunc;
-    const Function *OldFunc;
-    ValueToValueMapTy &VMap;
-    bool ModuleLevelChanges;
-    const char *NameSuffix;
-    ClonedCodeInfo *CodeInfo;
-
-  public:
-    PruningFunctionCloner(Function *newFunc, const Function *oldFunc,
-                          ValueToValueMapTy &valueMap, bool moduleLevelChanges,
-                          const char *nameSuffix, ClonedCodeInfo *codeInfo)
-        : NewFunc(newFunc), OldFunc(oldFunc), VMap(valueMap),
-          ModuleLevelChanges(moduleLevelChanges), NameSuffix(nameSuffix),
-          CodeInfo(codeInfo) {}
-
-    /// The specified block is found to be reachable, clone it and
-    /// anything that it can reach.
-    void CloneBlock(const BasicBlock *BB,
-                    BasicBlock::const_iterator StartingInst,
-                    std::vector<const BasicBlock*> &ToClone);
-  };
-}
+/// This is a private class used to implement CloneAndPruneFunctionInto.
+struct PruningFunctionCloner {
+  Function *NewFunc;
+  const Function *OldFunc;
+  ValueToValueMapTy &VMap;
+  bool ModuleLevelChanges;
+  const char *NameSuffix;
+  ClonedCodeInfo *CodeInfo;
+
+public:
+  PruningFunctionCloner(Function *newFunc, const Function *oldFunc,
+                        ValueToValueMapTy &valueMap, bool moduleLevelChanges,
+                        const char *nameSuffix, ClonedCodeInfo *codeInfo)
+      : NewFunc(newFunc), OldFunc(oldFunc), VMap(valueMap),
+        ModuleLevelChanges(moduleLevelChanges), NameSuffix(nameSuffix),
+        CodeInfo(codeInfo) {}
+
+  /// The specified block is found to be reachable, clone it and
+  /// anything that it can reach.
+  void CloneBlock(const BasicBlock *BB, BasicBlock::const_iterator StartingInst,
+                  std::vector<const BasicBlock *> &ToClone);
+};
+} // namespace
 
 /// The specified block is found to be reachable, clone it and
 /// anything that it can reach.
-void PruningFunctionCloner::CloneBlock(const BasicBlock *BB,
-                                       BasicBlock::const_iterator StartingInst,
-                                       std::vector<const BasicBlock*> &ToClone){
+void PruningFunctionCloner::CloneBlock(
+    const BasicBlock *BB, BasicBlock::const_iterator StartingInst,
+    std::vector<const BasicBlock *> &ToClone) {
   WeakTrackingVH &BBEntry = VMap[BB];
 
   // Have we already cloned this block?
-  if (BBEntry) return;
+  if (BBEntry)
+    return;
 
   // Nope, clone it now.
   BasicBlock *NewBB;
   BBEntry = NewBB = BasicBlock::Create(BB->getContext());
-  if (BB->hasName()) NewBB->setName(BB->getName()+NameSuffix);
+  if (BB->hasName())
+    NewBB->setName(BB->getName() + NameSuffix);
 
   // It is only legal to clone a function if a block address within that
   // function is never referenced outside of the function.  Given that, we
@@ -374,8 +368,8 @@ void PruningFunctionCloner::CloneBlock(const BasicBlock *BB,
   // Note that we don't need to fix the mapping for unreachable blocks;
   // the default mapping there is safe.
   if (BB->hasAddressTaken()) {
-    Constant *OldBBAddr = BlockAddress::get(const_cast<Function*>(OldFunc),
-                                            const_cast<BasicBlock*>(BB));
+    Constant *OldBBAddr = BlockAddress::get(const_cast<Function *>(OldFunc),
+                                            const_cast<BasicBlock *>(BB));
     VMap[OldBBAddr] = BlockAddress::get(NewFunc, NewBB);
   }
 
@@ -383,8 +377,8 @@ void PruningFunctionCloner::CloneBlock(const BasicBlock *BB,
 
   // Loop over all instructions, and copy them over, DCE'ing as we go.  This
   // loop doesn't include the terminator.
-  for (BasicBlock::const_iterator II = StartingInst, IE = --BB->end();
-       II != IE; ++II) {
+  for (BasicBlock::const_iterator II = StartingInst, IE = --BB->end(); II != IE;
+       ++II) {
 
     Instruction *NewInst = II->clone();
 
@@ -414,7 +408,7 @@ void PruningFunctionCloner::CloneBlock(const BasicBlock *BB,
     }
 
     if (II->hasName())
-      NewInst->setName(II->getName()+NameSuffix);
+      NewInst->setName(II->getName() + NameSuffix);
     VMap[&*II] = NewInst; // Add instruction map to value.
     NewBB->getInstList().push_back(NewInst);
     hasCalls |= (isa<CallInst>(II) && !isa<DbgInfoIntrinsic>(II));
@@ -460,9 +454,9 @@ void PruningFunctionCloner::CloneBlock(const BasicBlock *BB,
       Value *V = VMap.lookup(SI->getCondition());
       Cond = dyn_cast_or_null<ConstantInt>(V);
     }
-    if (Cond) {     // Constant fold to uncond branch!
+    if (Cond) { // Constant fold to uncond branch!
       SwitchInst::ConstCaseHandle Case = *SI->findCaseValue(Cond);
-      BasicBlock *Dest = const_cast<BasicBlock*>(Case.getCaseSuccessor());
+      BasicBlock *Dest = const_cast<BasicBlock *>(Case.getCaseSuccessor());
       VMap[OldTI] = BranchInst::Create(Dest, NewBB);
       ToClone.push_back(Dest);
       TerminatorDone = true;
@@ -472,9 +466,9 @@ void PruningFunctionCloner::CloneBlock(const BasicBlock *BB,
   if (!TerminatorDone) {
     Instruction *NewInst = OldTI->clone();
     if (OldTI->hasName())
-      NewInst->setName(OldTI->getName()+NameSuffix);
+      NewInst->setName(OldTI->getName() + NameSuffix);
     NewBB->getInstList().push_back(NewInst);
-    VMap[OldTI] = NewInst;             // Add instruction map to value.
+    VMap[OldTI] = NewInst; // Add instruction map to value.
 
     if (CodeInfo)
       if (auto *CB = dyn_cast<CallBase>(OldTI))
@@ -486,10 +480,10 @@ void PruningFunctionCloner::CloneBlock(const BasicBlock *BB,
   }
 
   if (CodeInfo) {
-    CodeInfo->ContainsCalls          |= hasCalls;
+    CodeInfo->ContainsCalls |= hasCalls;
     CodeInfo->ContainsDynamicAllocas |= hasDynamicAllocas;
-    CodeInfo->ContainsDynamicAllocas |= hasStaticAllocas &&
-      BB != &BB->getParent()->front();
+    CodeInfo->ContainsDynamicAllocas |=
+        hasStaticAllocas && BB != &BB->getParent()->front();
   }
 }
 
@@ -527,7 +521,7 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
   }
 
   // Clone the entry block, and anything recursively reachable from it.
-  std::vector<const BasicBlock*> CloneWorklist;
+  std::vector<const BasicBlock *> CloneWorklist;
   PFC.CloneBlock(StartingBB, StartingInst->getIterator(), CloneWorklist);
   while (!CloneWorklist.empty()) {
     const BasicBlock *BB = CloneWorklist.back();
@@ -540,11 +534,12 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
   // insert it into the new function in the right order.  If not, ignore it.
   //
   // Defer PHI resolution until rest of function is resolved.
-  SmallVector<const PHINode*, 16> PHIToResolve;
+  SmallVector<const PHINode *, 16> PHIToResolve;
   for (const BasicBlock &BI : *OldFunc) {
     Value *V = VMap.lookup(&BI);
     BasicBlock *NewBB = cast_or_null<BasicBlock>(V);
-    if (!NewBB) continue;  // Dead block.
+    if (!NewBB)
+      continue; // Dead block.
 
     // Add the new block to the new function.
     NewFunc->getBasicBlockList().push_back(NewBB);
@@ -569,7 +564,7 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
 
   // Defer PHI resolution until rest of function is resolved, PHI resolution
   // requires the CFG to be up-to-date.
-  for (unsigned phino = 0, e = PHIToResolve.size(); phino != e; ) {
+  for (unsigned phino = 0, e = PHIToResolve.size(); phino != e;) {
     const PHINode *OPN = PHIToResolve[phino];
     unsigned NumPreds = OPN->getNumIncomingValues();
     const BasicBlock *OldBB = OPN->getParent();
@@ -578,21 +573,22 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
     // Map operands for blocks that are live and remove operands for blocks
     // that are dead.
     for (; phino != PHIToResolve.size() &&
-         PHIToResolve[phino]->getParent() == OldBB; ++phino) {
+           PHIToResolve[phino]->getParent() == OldBB;
+         ++phino) {
       OPN = PHIToResolve[phino];
       PHINode *PN = cast<PHINode>(VMap[OPN]);
       for (unsigned pred = 0, e = NumPreds; pred != e; ++pred) {
         Value *V = VMap.lookup(PN->getIncomingBlock(pred));
         if (BasicBlock *MappedBlock = cast_or_null<BasicBlock>(V)) {
-          Value *InVal = MapValue(PN->getIncomingValue(pred),
-                                  VMap,
-                        ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges);
+          Value *InVal =
+              MapValue(PN->getIncomingValue(pred), VMap,
+                       ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges);
           assert(InVal && "Unknown input value?");
           PN->setIncomingValue(pred, InVal);
           PN->setIncomingBlock(pred, MappedBlock);
         } else {
           PN->removeIncomingValue(pred, false);
-          --pred;  // Revisit the next entry.
+          --pred; // Revisit the next entry.
           --e;
         }
       }
@@ -608,7 +604,7 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
     if (NumPreds != PN->getNumIncomingValues()) {
       assert(NumPreds < PN->getNumIncomingValues());
       // Count how many times each predecessor comes to this block.
-      std::map<BasicBlock*, unsigned> PredCount;
+      std::map<BasicBlock *, unsigned> PredCount;
       for (BasicBlock *Pred : predecessors(NewBB))
         --PredCount[Pred];
 
@@ -728,11 +724,15 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
     }
 
     BranchInst *BI = dyn_cast<BranchInst>(I->getTerminator());
-    if (!BI || BI->isConditional()) { ++I; continue; }
+    if (!BI || BI->isConditional()) {
+      ++I;
+      continue;
+    }
 
     BasicBlock *Dest = BI->getSuccessor(0);
     if (!Dest->getSinglePredecessor()) {
-      ++I; continue;
+      ++I;
+      continue;
     }
 
     // We shouldn't be able to get single-entry PHI nodes here, as instsimplify
@@ -765,7 +765,6 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
       Returns.push_back(RI);
 }
 
-
 /// This works exactly like CloneFunctionInto,
 /// except that it does some simple constant prop and DCE on the fly.  The
 /// effect of this is to copy significantly less code in cases where (for
@@ -773,13 +772,10 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
 /// constant arguments cause a significant amount of code in the callee to be
 /// dead.  Since this doesn't produce an exact copy of the input, it can't be
 /// used for things like CloneFunction or CloneModule.
-void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
-                                     ValueToValueMapTy &VMap,
-                                     bool ModuleLevelChanges,
-                                     SmallVectorImpl<ReturnInst*> &Returns,
-                                     const char *NameSuffix,
-                                     ClonedCodeInfo *CodeInfo,
-                                     Instruction *TheCall) {
+void llvm::CloneAndPruneFunctionInto(
+    Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap,
+    bool ModuleLevelChanges, SmallVectorImpl<ReturnInst *> &Returns,
+    const char *NameSuffix, ClonedCodeInfo *CodeInfo, Instruction *TheCall) {
   CloneAndPruneIntoFromInst(NewFunc, OldFunc, &OldFunc->front().front(), VMap,
                             ModuleLevelChanges, Returns, NameSuffix, CodeInfo);
 }
@@ -930,10 +926,9 @@ BasicBlock *llvm::DuplicateInstructionsInSplitBetween(
   return NewBB;
 }
 
-void llvm::cloneNoAliasScopes(
-    ArrayRef<MDNode *> NoAliasDeclScopes,
-    DenseMap<MDNode *, MDNode *> &ClonedScopes,
-    StringRef Ext, LLVMContext &Context) {
+void llvm::cloneNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
+                              DenseMap<MDNode *, MDNode *> &ClonedScopes,
+                              StringRef Ext, LLVMContext &Context) {
   MDBuilder MDB(Context);
 
   for (auto *ScopeList : NoAliasDeclScopes) {
@@ -956,9 +951,9 @@ void llvm::cloneNoAliasScopes(
   }
 }
 
-void llvm::adaptNoAliasScopes(
-    Instruction *I, const DenseMap<MDNode *, MDNode *> &ClonedScopes,
-    LLVMContext &Context) {
+void llvm::adaptNoAliasScopes(Instruction *I,
+                              const DenseMap<MDNode *, MDNode *> &ClonedScopes,
+                              LLVMContext &Context) {
   auto CloneScopeList = [&](const MDNode *ScopeList) -> MDNode * {
     bool NeedsReplacement = false;
     SmallVector<Metadata *, 8> NewScopeList;
@@ -990,9 +985,9 @@ void llvm::adaptNoAliasScopes(
   replaceWhenNeeded(LLVMContext::MD_alias_scope);
 }
 
-void llvm::cloneAndAdaptNoAliasScopes(
-    ArrayRef<MDNode *> NoAliasDeclScopes,
-    ArrayRef<BasicBlock *> NewBlocks, LLVMContext &Context, StringRef Ext) {
+void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
+                                      ArrayRef<BasicBlock *> NewBlocks,
+                                      LLVMContext &Context, StringRef Ext) {
   if (NoAliasDeclScopes.empty())
     return;
 
@@ -1007,9 +1002,9 @@ void llvm::cloneAndAdaptNoAliasScopes(
       adaptNoAliasScopes(&I, ClonedScopes, Context);
 }
 
-void llvm::cloneAndAdaptNoAliasScopes(
-    ArrayRef<MDNode *> NoAliasDeclScopes, Instruction *IStart,
-    Instruction *IEnd, LLVMContext &Context, StringRef Ext) {
+void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
+                                      Instruction *IStart, Instruction *IEnd,
+                                      LLVMContext &Context, StringRef Ext) {
   if (NoAliasDeclScopes.empty())
     return;
 
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 4ad33c14585d4f8ee093e496c8b8e231eba29f7b..cfd350ff612ae428c13e8f2ec85536e42ca8ae9a 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -970,6 +970,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::StackProtectStrong:
       case Attribute::StrictFP:
       case Attribute::UWTable:
+      case Attribute::VScaleRange:
       case Attribute::NoCfCheck:
       case Attribute::MustProgress:
       case Attribute::NoProfile:
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index e9eb6cb2d783a5c6112042ac833770d6d3c6496d..da876ce03711d172deb68e544c74f4661f986e60 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -258,7 +258,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
       Builder.CreateBr(TheOnlyDest);
       BasicBlock *BB = SI->getParent();
 
-      SmallSetVector<BasicBlock *, 8> RemovedSuccessors;
+      SmallSet<BasicBlock *, 8> RemovedSuccessors;
 
       // Remove entries from PHI nodes which we no longer branch to...
       BasicBlock *SuccToKeep = TheOnlyDest;
@@ -330,7 +330,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
     if (auto *BA =
           dyn_cast<BlockAddress>(IBI->getAddress()->stripPointerCasts())) {
       BasicBlock *TheOnlyDest = BA->getBasicBlock();
-      SmallSetVector<BasicBlock *, 8> RemovedSuccessors;
+      SmallSet<BasicBlock *, 8> RemovedSuccessors;
 
       // Insert the new branch.
       Builder.CreateBr(TheOnlyDest);
@@ -1618,8 +1618,8 @@ void llvm::insertDebugValuesForPHIs(BasicBlock *BB,
   // so that if a dbg.value is being rewritten to use more than one of the
   // inserted PHIs in the same destination BB, we can update the same dbg.value
   // with all the new PHIs instead of creating one copy for each.
-  SmallDenseMap<std::pair<BasicBlock *, DbgVariableIntrinsic *>,
-                DbgVariableIntrinsic *>
+  MapVector<std::pair<BasicBlock *, DbgVariableIntrinsic *>,
+            DbgVariableIntrinsic *>
       NewDbgValueMap;
   // Then iterate through the new PHIs and look to see if they use one of the
   // previously mapped PHIs. If so, create a new dbg.value intrinsic that will
@@ -2136,7 +2136,7 @@ unsigned llvm::changeToUnreachable(Instruction *I, bool UseLLVMTrap,
   if (MSSAU)
     MSSAU->changeToUnreachable(I);
 
-  SmallSetVector<BasicBlock *, 8> UniqueSuccessors;
+  SmallSet<BasicBlock *, 8> UniqueSuccessors;
 
   // Loop over all of the successors, removing BB's entry from any PHI
   // nodes.
@@ -2397,7 +2397,7 @@ static bool markAliveBlocks(Function &F,
         }
       };
 
-      SmallMapVector<BasicBlock *, int, 8> NumPerSuccessorCases;
+      SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
       // Set of unique CatchPads.
       SmallDenseMap<CatchPadInst *, detail::DenseSetEmpty, 4,
                     CatchPadDenseMapInfo, detail::DenseSetPair<CatchPadInst *>>
@@ -2407,22 +2407,25 @@ static bool markAliveBlocks(Function &F,
                                              E = CatchSwitch->handler_end();
            I != E; ++I) {
         BasicBlock *HandlerBB = *I;
-        ++NumPerSuccessorCases[HandlerBB];
+        if (DTU)
+          ++NumPerSuccessorCases[HandlerBB];
         auto *CatchPad = cast<CatchPadInst>(HandlerBB->getFirstNonPHI());
         if (!HandlerSet.insert({CatchPad, Empty}).second) {
-          --NumPerSuccessorCases[HandlerBB];
+          if (DTU)
+            --NumPerSuccessorCases[HandlerBB];
           CatchSwitch->removeHandler(I);
           --I;
           --E;
           Changed = true;
         }
       }
-      std::vector<DominatorTree::UpdateType> Updates;
-      for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
-        if (I.second == 0)
-          Updates.push_back({DominatorTree::Delete, BB, I.first});
-      if (DTU)
+      if (DTU) {
+        std::vector<DominatorTree::UpdateType> Updates;
+        for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
+          if (I.second == 0)
+            Updates.push_back({DominatorTree::Delete, BB, I.first});
         DTU->applyUpdates(Updates);
+      }
     }
 
     Changed |= ConstantFoldTerminator(BB, true, nullptr, DTU);
@@ -2508,7 +2511,7 @@ bool llvm::removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
   // their internal references. Update DTU if available.
   std::vector<DominatorTree::UpdateType> Updates;
   for (auto *BB : BlocksToRemove) {
-    SmallSetVector<BasicBlock *, 8> UniqueSuccessors;
+    SmallSet<BasicBlock *, 8> UniqueSuccessors;
     for (BasicBlock *Successor : successors(BB)) {
       // Only remove references to BB in reachable successors of BB.
       if (Reachable.count(Successor))
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index befacb5917629f0c29ff0658c948fbd4c9d8f4ae..cd1f6f0c78a518a59de67f610cac427673f66067 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -211,9 +211,7 @@ static unsigned countToEliminateCompares(Loop &L, unsigned MaxPeelCount,
 
     // Do not consider predicates that are known to be true or false
     // independently of the loop iteration.
-    if (SE.isKnownPredicate(Pred, LeftSCEV, RightSCEV) ||
-        SE.isKnownPredicate(ICmpInst::getInversePredicate(Pred), LeftSCEV,
-                            RightSCEV))
+    if (SE.evaluatePredicate(Pred, LeftSCEV, RightSCEV))
       continue;
 
     // Check if we have a condition with one AddRec and one non AddRec
diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index 4c262f60014ce3a3b9b8dbb0a9969c90a8f7e00a..91280762aaa7d5dae43a9935a7c9ba00ae128095 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -16,7 +16,6 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/IR/AssemblyAnnotationWriter.h"
@@ -24,7 +23,6 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
@@ -539,21 +537,6 @@ void PredicateInfoBuilder::buildPredicateInfo() {
   renameUses(OpsToRename);
 }
 
-// Create a ssa_copy declaration with custom mangling, because
-// Intrinsic::getDeclaration does not handle overloaded unnamed types properly:
-// all unnamed types get mangled to the same string. We use the pointer
-// to the type as name here, as it guarantees unique names for different
-// types and we remove the declarations when destroying PredicateInfo.
-// It is a workaround for PR38117, because solving it in a fully general way is
-// tricky (FIXME).
-static Function *getCopyDeclaration(Module *M, Type *Ty) {
-  std::string Name = "llvm.ssa.copy." + utostr((uintptr_t) Ty);
-  return cast<Function>(
-      M->getOrInsertFunction(Name,
-                             getType(M->getContext(), Intrinsic::ssa_copy, Ty))
-          .getCallee());
-}
-
 // Given the renaming stack, make all the operands currently on the stack real
 // by inserting them into the IR.  Return the last operation's value.
 Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
@@ -585,9 +568,8 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
     // order in the case of multiple predicateinfo in the same block.
     if (isa<PredicateWithEdge>(ValInfo)) {
       IRBuilder<> B(getBranchTerminator(ValInfo));
-      Function *IF = getCopyDeclaration(F.getParent(), Op->getType());
-      if (IF->users().empty())
-        PI.CreatedDeclarations.insert(IF);
+      Function *IF = Intrinsic::getDeclaration(
+          F.getParent(), Intrinsic::ssa_copy, Op->getType());
       CallInst *PIC =
           B.CreateCall(IF, Op, Op->getName() + "." + Twine(Counter++));
       PI.PredicateMap.insert({PIC, ValInfo});
@@ -599,9 +581,8 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
       // Insert the predicate directly after the assume. While it also holds
       // directly before it, assume(i1 true) is not a useful fact.
       IRBuilder<> B(PAssume->AssumeInst->getNextNode());
-      Function *IF = getCopyDeclaration(F.getParent(), Op->getType());
-      if (IF->users().empty())
-        PI.CreatedDeclarations.insert(IF);
+      Function *IF = Intrinsic::getDeclaration(
+          F.getParent(), Intrinsic::ssa_copy, Op->getType());
       CallInst *PIC = B.CreateCall(IF, Op);
       PI.PredicateMap.insert({PIC, ValInfo});
       Result.Def = PIC;
@@ -780,23 +761,6 @@ PredicateInfo::PredicateInfo(Function &F, DominatorTree &DT,
   Builder.buildPredicateInfo();
 }
 
-// Remove all declarations we created . The PredicateInfo consumers are
-// responsible for remove the ssa_copy calls created.
-PredicateInfo::~PredicateInfo() {
-  // Collect function pointers in set first, as SmallSet uses a SmallVector
-  // internally and we have to remove the asserting value handles first.
-  SmallPtrSet<Function *, 20> FunctionPtrs;
-  for (auto &F : CreatedDeclarations)
-    FunctionPtrs.insert(&*F);
-  CreatedDeclarations.clear();
-
-  for (Function *F : FunctionPtrs) {
-    assert(F->user_begin() == F->user_end() &&
-           "PredicateInfo consumer did not remove all SSA copies.");
-    F->eraseFromParent();
-  }
-}
-
 Optional<PredicateConstraint> PredicateBase::getConstraint() const {
   switch (Type) {
   case PT_Assume:
@@ -863,19 +827,6 @@ void PredicateInfoPrinterLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<AssumptionCacheTracker>();
 }
 
-// Replace ssa_copy calls created by PredicateInfo with their operand.
-static void replaceCreatedSSACopys(PredicateInfo &PredInfo, Function &F) {
-  for (Instruction &Inst : llvm::make_early_inc_range(instructions(F))) {
-    const auto *PI = PredInfo.getPredicateInfoFor(&Inst);
-    auto *II = dyn_cast<IntrinsicInst>(&Inst);
-    if (!PI || !II || II->getIntrinsicID() != Intrinsic::ssa_copy)
-      continue;
-
-    Inst.replaceAllUsesWith(II->getOperand(0));
-    Inst.eraseFromParent();
-  }
-}
-
 bool PredicateInfoPrinterLegacyPass::runOnFunction(Function &F) {
   auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
@@ -883,8 +834,6 @@ bool PredicateInfoPrinterLegacyPass::runOnFunction(Function &F) {
   PredInfo->print(dbgs());
   if (VerifyPredicateInfo)
     PredInfo->verifyPredicateInfo();
-
-  replaceCreatedSSACopys(*PredInfo, F);
   return false;
 }
 
@@ -896,7 +845,6 @@ PreservedAnalyses PredicateInfoPrinterPass::run(Function &F,
   auto PredInfo = std::make_unique<PredicateInfo>(F, DT, AC);
   PredInfo->print(OS);
 
-  replaceCreatedSSACopys(*PredInfo, F);
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 1f9fa611a9b2ea3551e8297a817719fb94f1a971..9a5e864170bc210451292e4e4e2f2fd4c06aaaeb 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -63,6 +63,7 @@
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
 #include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -909,24 +910,27 @@ bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
     LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
                       << "Through successor TI: " << *TI);
 
-    SmallMapVector<BasicBlock *, int, 8> NumPerSuccessorCases;
+    SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
     for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
       --i;
       auto *Successor = i->getCaseSuccessor();
-      ++NumPerSuccessorCases[Successor];
+      if (DTU)
+        ++NumPerSuccessorCases[Successor];
       if (DeadCases.count(i->getCaseValue())) {
         Successor->removePredecessor(PredDef);
         SI.removeCase(i);
-        --NumPerSuccessorCases[Successor];
+        if (DTU)
+          --NumPerSuccessorCases[Successor];
       }
     }
 
-    std::vector<DominatorTree::UpdateType> Updates;
-    for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
-      if (I.second == 0)
-        Updates.push_back({DominatorTree::Delete, PredDef, I.first});
-    if (DTU)
+    if (DTU) {
+      std::vector<DominatorTree::UpdateType> Updates;
+      for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
+        if (I.second == 0)
+          Updates.push_back({DominatorTree::Delete, PredDef, I.first});
       DTU->applyUpdates(Updates);
+    }
 
     LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n");
     return true;
@@ -957,7 +961,7 @@ bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
   if (!TheRealDest)
     TheRealDest = ThisDef;
 
-  SmallSetVector<BasicBlock *, 2> RemovedSuccs;
+  SmallPtrSet<BasicBlock *, 2> RemovedSuccs;
 
   // Remove PHI node entries for dead edges.
   BasicBlock *CheckEdge = TheRealDest;
@@ -1096,8 +1100,13 @@ static void CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(
                          (NewBonusInst->getName() + ".merge").str());
     SSAUpdate.AddAvailableValue(BB, &BonusInst);
     SSAUpdate.AddAvailableValue(PredBlock, NewBonusInst);
-    for (Use &U : make_early_inc_range(BonusInst.uses()))
-      SSAUpdate.RewriteUseAfterInsertions(U);
+    for (Use &U : make_early_inc_range(BonusInst.uses())) {
+      auto *UI = cast<Instruction>(U.getUser());
+      if (UI->getParent() != PredBlock)
+        SSAUpdate.RewriteUseAfterInsertions(U);
+      else // Use is in the same block as, and comes before, NewBonusInst.
+        SSAUpdate.RewriteUse(U);
+    }
   }
 }
 
@@ -1171,7 +1180,7 @@ bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding(
     // Reconstruct the new switch statement we will be building.
     if (PredDefault != BBDefault) {
       PredDefault->removePredecessor(Pred);
-      if (PredDefault != BB)
+      if (DTU && PredDefault != BB)
         Updates.push_back({DominatorTree::Delete, Pred, PredDefault});
       PredDefault = BBDefault;
       ++NewSuccessors[BBDefault];
@@ -1253,7 +1262,7 @@ bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding(
       (void)I;
       AddPredecessorToBlock(NewSuccessor.first, Pred, BB);
     }
-    if (!is_contained(successors(Pred), NewSuccessor.first))
+    if (DTU && !is_contained(successors(Pred), NewSuccessor.first))
       Updates.push_back({DominatorTree::Insert, Pred, NewSuccessor.first});
   }
 
@@ -1293,18 +1302,21 @@ bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding(
         InfLoopBlock =
             BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
         BranchInst::Create(InfLoopBlock, InfLoopBlock);
-        Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
+        if (DTU)
+          Updates.push_back(
+              {DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
       }
       NewSI->setSuccessor(i, InfLoopBlock);
     }
 
-  if (InfLoopBlock)
-    Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock});
+  if (DTU) {
+    if (InfLoopBlock)
+      Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock});
 
-  Updates.push_back({DominatorTree::Delete, Pred, BB});
+    Updates.push_back({DominatorTree::Delete, Pred, BB});
 
-  if (DTU)
     DTU->applyUpdates(Updates);
+  }
 
   ++NumFoldValueComparisonIntoPredecessors;
   return true;
@@ -1581,10 +1593,13 @@ HoistTerminator:
   // Update any PHI nodes in our new successors.
   for (BasicBlock *Succ : successors(BB1)) {
     AddPredecessorToBlock(Succ, BIParent, BB1);
-    Updates.push_back({DominatorTree::Insert, BIParent, Succ});
+    if (DTU)
+      Updates.push_back({DominatorTree::Insert, BIParent, Succ});
   }
-  for (BasicBlock *Succ : successors(BI))
-    Updates.push_back({DominatorTree::Delete, BIParent, Succ});
+
+  if (DTU)
+    for (BasicBlock *Succ : successors(BI))
+      Updates.push_back({DominatorTree::Delete, BIParent, Succ});
 
   EraseTerminatorAndDCECond(BI);
   if (DTU)
@@ -2468,7 +2483,8 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, DomTreeUpdater *DTU,
         BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge",
                            RealDest->getParent(), RealDest);
     BranchInst *CritEdgeBranch = BranchInst::Create(RealDest, EdgeBB);
-    Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest});
+    if (DTU)
+      Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest});
     CritEdgeBranch->setDebugLoc(BI->getDebugLoc());
 
     // Update PHI nodes.
@@ -2527,11 +2543,12 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, DomTreeUpdater *DTU,
         PredBBTI->setSuccessor(i, EdgeBB);
       }
 
-    Updates.push_back({DominatorTree::Insert, PredBB, EdgeBB});
-    Updates.push_back({DominatorTree::Delete, PredBB, BB});
+    if (DTU) {
+      Updates.push_back({DominatorTree::Insert, PredBB, EdgeBB});
+      Updates.push_back({DominatorTree::Delete, PredBB, BB});
 
-    if (DTU)
       DTU->applyUpdates(Updates);
+    }
 
     // Recurse, simplifying any other constants.
     return FoldCondBranchOnPHI(BI, DTU, DL, AC) || true;
@@ -2840,30 +2857,52 @@ static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI,
   }
 }
 
-// Determine if the two branches share a common destination,
-// and deduce a glue that we need to use to join branch's conditions
-// to arrive at the common destination.
+/// Determine if the two branches share a common destination and deduce a glue
+/// that joins the branches' conditions to arrive at the common destination if
+/// that would be profitable.
 static Optional<std::pair<Instruction::BinaryOps, bool>>
-CheckIfCondBranchesShareCommonDestination(BranchInst *BI, BranchInst *PBI) {
+shouldFoldCondBranchesToCommonDestination(BranchInst *BI, BranchInst *PBI,
+                                          const TargetTransformInfo *TTI) {
   assert(BI && PBI && BI->isConditional() && PBI->isConditional() &&
          "Both blocks must end with a conditional branches.");
   assert(is_contained(predecessors(BI->getParent()), PBI->getParent()) &&
          "PredBB must be a predecessor of BB.");
 
-  if (PBI->getSuccessor(0) == BI->getSuccessor(0))
-    return {{Instruction::Or, false}};
-  else if (PBI->getSuccessor(1) == BI->getSuccessor(1))
-    return {{Instruction::And, false}};
-  else if (PBI->getSuccessor(0) == BI->getSuccessor(1))
-    return {{Instruction::And, true}};
-  else if (PBI->getSuccessor(1) == BI->getSuccessor(0))
-    return {{Instruction::Or, true}};
+  // We have the potential to fold the conditions together, but if the
+  // predecessor branch is predictable, we may not want to merge them.
+  uint64_t PTWeight, PFWeight;
+  BranchProbability PBITrueProb, Likely;
+  if (TTI && PBI->extractProfMetadata(PTWeight, PFWeight) &&
+      (PTWeight + PFWeight) != 0) {
+    PBITrueProb =
+        BranchProbability::getBranchProbability(PTWeight, PTWeight + PFWeight);
+    Likely = TTI->getPredictableBranchThreshold();
+  }
+
+  if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
+    // Speculate the 2nd condition unless the 1st is probably true.
+    if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
+      return {{Instruction::Or, false}};
+  } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
+    // Speculate the 2nd condition unless the 1st is probably false.
+    if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
+      return {{Instruction::And, false}};
+  } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
+    // Speculate the 2nd condition unless the 1st is probably true.
+    if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
+      return {{Instruction::And, true}};
+  } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
+    // Speculate the 2nd condition unless the 1st is probably false.
+    if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
+      return {{Instruction::Or, true}};
+  }
   return None;
 }
 
-static bool PerformBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
+static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
                                              DomTreeUpdater *DTU,
-                                             MemorySSAUpdater *MSSAU) {
+                                             MemorySSAUpdater *MSSAU,
+                                             const TargetTransformInfo *TTI) {
   BasicBlock *BB = BI->getParent();
   BasicBlock *PredBlock = PBI->getParent();
 
@@ -2871,7 +2910,7 @@ static bool PerformBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
   Instruction::BinaryOps Opc;
   bool InvertPredCond;
   std::tie(Opc, InvertPredCond) =
-      *CheckIfCondBranchesShareCommonDestination(BI, PBI);
+      *shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI);
 
   LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB);
 
@@ -2962,8 +3001,16 @@ static bool PerformBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
 
   // Now that the Cond was cloned into the predecessor basic block,
   // or/and the two conditions together.
-  Instruction *NewCond = cast<Instruction>(Builder.CreateBinOp(
-      Opc, PBI->getCondition(), VMap[BI->getCondition()], "or.cond"));
+  Value *NewCond = nullptr;
+  Value *BICond = VMap[BI->getCondition()];
+
+  if (impliesPoison(BICond, PBI->getCondition()))
+    NewCond = Builder.CreateBinOp(Opc, PBI->getCondition(), BICond, "or.cond");
+  else
+    NewCond =
+        Opc == Instruction::And
+            ? Builder.CreateLogicalAnd(PBI->getCondition(), BICond, "or.cond")
+            : Builder.CreateLogicalOr(PBI->getCondition(), BICond, "or.cond");
   PBI->setCondition(NewCond);
 
   // Copy any debug value intrinsics into the end of PredBlock.
@@ -2994,8 +3041,6 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
 
   BasicBlock *BB = BI->getParent();
 
-  const unsigned PredCount = pred_size(BB);
-
   bool Changed = false;
 
   TargetTransformInfo::TargetCostKind CostKind =
@@ -3008,32 +3053,6 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
       Cond->getParent() != BB || !Cond->hasOneUse())
     return Changed;
 
-  // Only allow this transformation if computing the condition doesn't involve
-  // too many instructions and these involved instructions can be executed
-  // unconditionally. We denote all involved instructions except the condition
-  // as "bonus instructions", and only allow this transformation when the
-  // number of the bonus instructions we'll need to create when cloning into
-  // each predecessor does not exceed a certain threshold.
-  unsigned NumBonusInsts = 0;
-  for (Instruction &I : *BB) {
-    // Don't check the branch condition comparison itself.
-    if (&I == Cond)
-      continue;
-    // Ignore dbg intrinsics, and the terminator.
-    if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I))
-      continue;
-    // I must be safe to execute unconditionally.
-    if (!isSafeToSpeculativelyExecute(&I))
-      return Changed;
-
-    // Account for the cost of duplicating this instruction into each
-    // predecessor.
-    NumBonusInsts += PredCount;
-    // Early exits once we reach the limit.
-    if (NumBonusInsts > BonusInstThreshold)
-      return Changed;
-  }
-
   // Cond is known to be a compare or binary operator.  Check to make sure that
   // neither operand is a potentially-trapping constant expression.
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(0)))
@@ -3047,6 +3066,8 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
   if (is_contained(successors(BB), BB))
     return Changed;
 
+  // With which predecessors will we want to deal with?
+  SmallVector<BasicBlock *, 8> Preds;
   for (BasicBlock *PredBlock : predecessors(BB)) {
     BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
 
@@ -3059,8 +3080,8 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
     // Determine if the two branches share a common destination.
     Instruction::BinaryOps Opc;
     bool InvertPredCond;
-    if (auto Recepie = CheckIfCondBranchesShareCommonDestination(BI, PBI))
-      std::tie(Opc, InvertPredCond) = *Recepie;
+    if (auto Recipe = shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI))
+      std::tie(Opc, InvertPredCond) = *Recipe;
     else
       continue;
 
@@ -3077,7 +3098,46 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
         continue;
     }
 
-    return PerformBranchToCommonDestFolding(BI, PBI, DTU, MSSAU);
+    // Ok, we do want to deal with this predecessor. Record it.
+    Preds.emplace_back(PredBlock);
+  }
+
+  // If there aren't any predecessors into which we can fold,
+  // don't bother checking the cost.
+  if (Preds.empty())
+    return Changed;
+
+  // Only allow this transformation if computing the condition doesn't involve
+  // too many instructions and these involved instructions can be executed
+  // unconditionally. We denote all involved instructions except the condition
+  // as "bonus instructions", and only allow this transformation when the
+  // number of the bonus instructions we'll need to create when cloning into
+  // each predecessor does not exceed a certain threshold.
+  unsigned NumBonusInsts = 0;
+  const unsigned PredCount = Preds.size();
+  for (Instruction &I : *BB) {
+    // Don't check the branch condition comparison itself.
+    if (&I == Cond)
+      continue;
+    // Ignore dbg intrinsics, and the terminator.
+    if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I))
+      continue;
+    // I must be safe to execute unconditionally.
+    if (!isSafeToSpeculativelyExecute(&I))
+      return Changed;
+
+    // Account for the cost of duplicating this instruction into each
+    // predecessor.
+    NumBonusInsts += PredCount;
+    // Early exits once we reach the limit.
+    if (NumBonusInsts > BonusInstThreshold)
+      return Changed;
+  }
+
+  // Ok, we have the budget. Perform the transformation.
+  for (BasicBlock *PredBlock : Preds) {
+    auto *PBI = cast<BranchInst>(PredBlock->getTerminator());
+    return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI);
   }
   return Changed;
 }
@@ -3607,7 +3667,8 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
     BasicBlock *InfLoopBlock =
         BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
     BranchInst::Create(InfLoopBlock, InfLoopBlock);
-    Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
+    if (DTU)
+      Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
     OtherDest = InfLoopBlock;
   }
 
@@ -3634,11 +3695,12 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
   PBI->setSuccessor(0, CommonDest);
   PBI->setSuccessor(1, OtherDest);
 
-  Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest});
-  Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest});
+  if (DTU) {
+    Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest});
+    Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest});
 
-  if (DTU)
     DTU->applyUpdates(Updates);
+  }
 
   // Update branch weight for PBI.
   uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
@@ -3727,7 +3789,7 @@ bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm,
   BasicBlock *KeepEdge1 = TrueBB;
   BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr;
 
-  SmallSetVector<BasicBlock *, 2> RemovedSuccessors;
+  SmallPtrSet<BasicBlock *, 2> RemovedSuccessors;
 
   // Then remove the rest.
   for (BasicBlock *Succ : successors(OldTerm)) {
@@ -3957,17 +4019,19 @@ bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt(
       SIW.setSuccessorWeight(0, *NewW);
     }
     SIW.addCase(Cst, NewBB, NewW);
-    Updates.push_back({DominatorTree::Insert, Pred, NewBB});
+    if (DTU)
+      Updates.push_back({DominatorTree::Insert, Pred, NewBB});
   }
 
   // NewBB branches to the phi block, add the uncond branch and the phi entry.
   Builder.SetInsertPoint(NewBB);
   Builder.SetCurrentDebugLocation(SI->getDebugLoc());
   Builder.CreateBr(SuccBlock);
-  Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock});
   PHIUse->addIncoming(NewCst, NewBB);
-  if (DTU)
+  if (DTU) {
+    Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock});
     DTU->applyUpdates(Updates);
+  }
   return true;
 }
 
@@ -4053,7 +4117,8 @@ bool SimplifyCFGOpt::SimplifyBranchOnICmpChain(BranchInst *BI,
 
     OldTI->eraseFromParent();
 
-    Updates.push_back({DominatorTree::Insert, BB, EdgeBB});
+    if (DTU)
+      Updates.push_back({DominatorTree::Insert, BB, EdgeBB});
 
     // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
     // for the edge we just added.
@@ -4341,16 +4406,19 @@ static bool removeEmptyCleanup(CleanupReturnInst *RI, DomTreeUpdater *DTU) {
   // We use make_early_inc_range here because we may remove some predecessors.
   for (BasicBlock *PredBB : llvm::make_early_inc_range(predecessors(BB))) {
     if (UnwindDest == nullptr) {
-      if (DTU)
+      if (DTU) {
         DTU->applyUpdates(Updates);
-      Updates.clear();
+        Updates.clear();
+      }
       removeUnwindEdge(PredBB, DTU);
       ++NumInvokes;
     } else {
       Instruction *TI = PredBB->getTerminator();
       TI->replaceUsesOfWith(BB, UnwindDest);
-      Updates.push_back({DominatorTree::Insert, PredBB, UnwindDest});
-      Updates.push_back({DominatorTree::Delete, PredBB, BB});
+      if (DTU) {
+        Updates.push_back({DominatorTree::Insert, PredBB, UnwindDest});
+        Updates.push_back({DominatorTree::Delete, PredBB, BB});
+      }
     }
   }
 
@@ -4557,7 +4625,8 @@ bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
         EraseTerminatorAndDCECond(BI);
         Changed = true;
       }
-      Updates.push_back({DominatorTree::Delete, Predecessor, BB});
+      if (DTU)
+        Updates.push_back({DominatorTree::Delete, Predecessor, BB});
     } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
       SwitchInstProfUpdateWrapper SU(*SI);
       for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) {
@@ -4571,21 +4640,23 @@ bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
         Changed = true;
       }
       // Note that the default destination can't be removed!
-      if (SI->getDefaultDest() != BB)
+      if (DTU && SI->getDefaultDest() != BB)
         Updates.push_back({DominatorTree::Delete, Predecessor, BB});
     } else if (auto *II = dyn_cast<InvokeInst>(TI)) {
       if (II->getUnwindDest() == BB) {
-        if (DTU)
+        if (DTU) {
           DTU->applyUpdates(Updates);
-        Updates.clear();
+          Updates.clear();
+        }
         removeUnwindEdge(TI->getParent(), DTU);
         Changed = true;
       }
     } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) {
       if (CSI->getUnwindDest() == BB) {
-        if (DTU)
+        if (DTU) {
           DTU->applyUpdates(Updates);
-        Updates.clear();
+          Updates.clear();
+        }
         removeUnwindEdge(TI->getParent(), DTU);
         Changed = true;
         continue;
@@ -4601,23 +4672,28 @@ bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
           Changed = true;
         }
       }
-      Updates.push_back({DominatorTree::Delete, Predecessor, BB});
+      if (DTU)
+        Updates.push_back({DominatorTree::Delete, Predecessor, BB});
       if (CSI->getNumHandlers() == 0) {
         if (CSI->hasUnwindDest()) {
           // Redirect all predecessors of the block containing CatchSwitchInst
           // to instead branch to the CatchSwitchInst's unwind destination.
-          for (auto *PredecessorOfPredecessor : predecessors(Predecessor)) {
-            Updates.push_back({DominatorTree::Insert, PredecessorOfPredecessor,
-                               CSI->getUnwindDest()});
-            Updates.push_back(
-                {DominatorTree::Delete, PredecessorOfPredecessor, Predecessor});
+          if (DTU) {
+            for (auto *PredecessorOfPredecessor : predecessors(Predecessor)) {
+              Updates.push_back({DominatorTree::Insert,
+                                 PredecessorOfPredecessor,
+                                 CSI->getUnwindDest()});
+              Updates.push_back({DominatorTree::Delete,
+                                 PredecessorOfPredecessor, Predecessor});
+            }
           }
           Predecessor->replaceAllUsesWith(CSI->getUnwindDest());
         } else {
           // Rewrite all preds to unwind to caller (or from invoke to call).
-          if (DTU)
+          if (DTU) {
             DTU->applyUpdates(Updates);
-          Updates.clear();
+            Updates.clear();
+          }
           SmallVector<BasicBlock *, 8> EHPreds(predecessors(Predecessor));
           for (BasicBlock *EHPred : EHPreds)
             removeUnwindEdge(EHPred, DTU);
@@ -4631,7 +4707,8 @@ bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
       (void)CRI;
       assert(CRI->hasUnwindDest() && CRI->getUnwindDest() == BB &&
              "Expected to always have an unwind to BB.");
-      Updates.push_back({DominatorTree::Delete, Predecessor, BB});
+      if (DTU)
+        Updates.push_back({DominatorTree::Delete, Predecessor, BB});
       new UnreachableInst(TI->getContext(), TI);
       TI->eraseFromParent();
       Changed = true;
@@ -4678,8 +4755,9 @@ static void createUnreachableSwitchDefault(SwitchInst *Switch,
                        {DominatorTree::Delete, BB, OrigDefaultBlock}});
   SplitBlock(&*NewDefaultBlock, &NewDefaultBlock->front(), DTU);
   SmallVector<DominatorTree::UpdateType, 2> Updates;
-  for (auto *Successor : successors(NewDefaultBlock))
-    Updates.push_back({DominatorTree::Delete, NewDefaultBlock, Successor});
+  if (DTU)
+    for (auto *Successor : successors(NewDefaultBlock))
+      Updates.push_back({DominatorTree::Delete, NewDefaultBlock, Successor});
   auto *NewTerminator = NewDefaultBlock->getTerminator();
   new UnreachableInst(Switch->getContext(), NewTerminator);
   EraseTerminatorAndDCECond(NewTerminator);
@@ -4831,15 +4909,17 @@ static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU,
 
   // Gather dead cases.
   SmallVector<ConstantInt *, 8> DeadCases;
-  SmallMapVector<BasicBlock *, int, 8> NumPerSuccessorCases;
+  SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
   for (auto &Case : SI->cases()) {
     auto *Successor = Case.getCaseSuccessor();
-    ++NumPerSuccessorCases[Successor];
+    if (DTU)
+      ++NumPerSuccessorCases[Successor];
     const APInt &CaseVal = Case.getCaseValue()->getValue();
     if (Known.Zero.intersects(CaseVal) || !Known.One.isSubsetOf(CaseVal) ||
         (CaseVal.getMinSignedBits() > MaxSignificantBitsInCond)) {
       DeadCases.push_back(Case.getCaseValue());
-      --NumPerSuccessorCases[Successor];
+      if (DTU)
+        --NumPerSuccessorCases[Successor];
       LLVM_DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseVal
                         << " is dead.\n");
     }
@@ -4874,12 +4954,13 @@ static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU,
     SIW.removeCase(CaseI);
   }
 
-  std::vector<DominatorTree::UpdateType> Updates;
-  for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
-    if (I.second == 0)
-      Updates.push_back({DominatorTree::Delete, SI->getParent(), I.first});
-  if (DTU)
+  if (DTU) {
+    std::vector<DominatorTree::UpdateType> Updates;
+    for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
+      if (I.second == 0)
+        Updates.push_back({DominatorTree::Delete, SI->getParent(), I.first});
     DTU->applyUpdates(Updates);
+  }
 
   return true;
 }
@@ -5243,7 +5324,7 @@ static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI,
   BasicBlock *SelectBB = SI->getParent();
   BasicBlock *DestBB = PHI->getParent();
 
-  if (!is_contained(predecessors(DestBB), SelectBB))
+  if (DTU && !is_contained(predecessors(DestBB), SelectBB))
     Updates.push_back({DominatorTree::Insert, SelectBB, DestBB});
   Builder.CreateBr(DestBB);
 
@@ -5259,7 +5340,8 @@ static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI,
     if (Succ == DestBB)
       continue;
     Succ->removePredecessor(SelectBB);
-    Updates.push_back({DominatorTree::Delete, SelectBB, Succ});
+    if (DTU)
+      Updates.push_back({DominatorTree::Delete, SelectBB, Succ});
   }
   SI->eraseFromParent();
   if (DTU)
@@ -5807,7 +5889,8 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
 
   if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
     Builder.CreateBr(LookupBB);
-    Updates.push_back({DominatorTree::Insert, BB, LookupBB});
+    if (DTU)
+      Updates.push_back({DominatorTree::Insert, BB, LookupBB});
     // Note: We call removeProdecessor later since we need to be able to get the
     // PHI value for the default case in case we're using a bit mask.
   } else {
@@ -5815,7 +5898,8 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
         TableIndex, ConstantInt::get(MinCaseVal->getType(), TableSize));
     RangeCheckBranch =
         Builder.CreateCondBr(Cmp, LookupBB, SI->getDefaultDest());
-    Updates.push_back({DominatorTree::Insert, BB, LookupBB});
+    if (DTU)
+      Updates.push_back({DominatorTree::Insert, BB, LookupBB});
   }
 
   // Populate the BB that does the lookups.
@@ -5853,8 +5937,10 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
     Value *LoBit = Builder.CreateTrunc(
         Shifted, Type::getInt1Ty(Mod.getContext()), "switch.lobit");
     Builder.CreateCondBr(LoBit, LookupBB, SI->getDefaultDest());
-    Updates.push_back({DominatorTree::Insert, MaskBB, LookupBB});
-    Updates.push_back({DominatorTree::Insert, MaskBB, SI->getDefaultDest()});
+    if (DTU) {
+      Updates.push_back({DominatorTree::Insert, MaskBB, LookupBB});
+      Updates.push_back({DominatorTree::Insert, MaskBB, SI->getDefaultDest()});
+    }
     Builder.SetInsertPoint(LookupBB);
     AddPredecessorToBlock(SI->getDefaultDest(), MaskBB, BB);
   }
@@ -5864,7 +5950,8 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
     // do not delete PHINodes here.
     SI->getDefaultDest()->removePredecessor(BB,
                                             /*KeepOneInputPHIs=*/true);
-    Updates.push_back({DominatorTree::Delete, BB, SI->getDefaultDest()});
+    if (DTU)
+      Updates.push_back({DominatorTree::Delete, BB, SI->getDefaultDest()});
   }
 
   bool ReturnedEarly = false;
@@ -5903,11 +5990,12 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
 
   if (!ReturnedEarly) {
     Builder.CreateBr(CommonDest);
-    Updates.push_back({DominatorTree::Insert, LookupBB, CommonDest});
+    if (DTU)
+      Updates.push_back({DominatorTree::Insert, LookupBB, CommonDest});
   }
 
   // Remove the switch.
-  SmallSetVector<BasicBlock *, 8> RemovedSuccessors;
+  SmallPtrSet<BasicBlock *, 8> RemovedSuccessors;
   for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
     BasicBlock *Succ = SI->getSuccessor(i);
 
@@ -6089,7 +6177,7 @@ bool SimplifyCFGOpt::simplifyIndirectBr(IndirectBrInst *IBI) {
 
   // Eliminate redundant destinations.
   SmallPtrSet<Value *, 8> Succs;
-  SmallSetVector<BasicBlock *, 8> RemovedSuccs;
+  SmallPtrSet<BasicBlock *, 8> RemovedSuccs;
   for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; ++i) {
     BasicBlock *Dest = IBI->getDestination(i);
     if (!Dest->hasAddressTaken() || !Succs.insert(Dest).second) {
@@ -6179,15 +6267,16 @@ static bool TryToMergeLandingPad(LandingPadInst *LPad, BranchInst *BI,
 
     // We've found an identical block.  Update our predecessors to take that
     // path instead and make ourselves dead.
-    SmallPtrSet<BasicBlock *, 16> Preds;
-    Preds.insert(pred_begin(BB), pred_end(BB));
+    SmallPtrSet<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB));
     for (BasicBlock *Pred : Preds) {
       InvokeInst *II = cast<InvokeInst>(Pred->getTerminator());
       assert(II->getNormalDest() != BB && II->getUnwindDest() == BB &&
              "unexpected successor");
       II->setUnwindDest(OtherPred);
-      Updates.push_back({DominatorTree::Insert, Pred, OtherPred});
-      Updates.push_back({DominatorTree::Delete, Pred, BB});
+      if (DTU) {
+        Updates.push_back({DominatorTree::Insert, Pred, OtherPred});
+        Updates.push_back({DominatorTree::Delete, Pred, BB});
+      }
     }
 
     // The debug info in OtherPred doesn't cover the merged control flow that
@@ -6199,11 +6288,11 @@ static bool TryToMergeLandingPad(LandingPadInst *LPad, BranchInst *BI,
         Inst.eraseFromParent();
     }
 
-    SmallPtrSet<BasicBlock *, 16> Succs;
-    Succs.insert(succ_begin(BB), succ_end(BB));
+    SmallPtrSet<BasicBlock *, 16> Succs(succ_begin(BB), succ_end(BB));
     for (BasicBlock *Succ : Succs) {
       Succ->removePredecessor(BB);
-      Updates.push_back({DominatorTree::Delete, BB, Succ});
+      if (DTU)
+        Updates.push_back({DominatorTree::Delete, BB, Succ});
     }
 
     IRBuilder<> Builder(BI);
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index 290c04a7ad10ea7f8b199e5fa1d952b7da7641ea..d4f325cd468286c79ae4c29657e91144ef5aa87e 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -99,6 +99,24 @@ namespace {
   };
 }
 
+/// Find a point in code which dominates all given instructions. We can safely
+/// assume that, whatever fact we can prove at the found point, this fact is
+/// also true for each of the given instructions.
+static Instruction *findCommonDominator(ArrayRef<Instruction *> Instructions,
+                                        DominatorTree &DT) {
+  Instruction *CommonDom = nullptr;
+  for (auto *Insn : Instructions)
+    if (!CommonDom || DT.dominates(Insn, CommonDom))
+      CommonDom = Insn;
+    else if (!DT.dominates(CommonDom, Insn))
+      // If there is no dominance relation, use common dominator.
+      CommonDom =
+          DT.findNearestCommonDominator(CommonDom->getParent(),
+                                        Insn->getParent())->getTerminator();
+  assert(CommonDom && "Common dominator not found?");
+  return CommonDom;
+}
+
 /// Fold an IV operand into its use.  This removes increments of an
 /// aligned IV when used by a instruction that ignores the low bits.
 ///
@@ -261,14 +279,14 @@ void SimplifyIndvar::eliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) {
   const SCEV *S = SE->getSCEVAtScope(ICmp->getOperand(IVOperIdx), ICmpLoop);
   const SCEV *X = SE->getSCEVAtScope(ICmp->getOperand(1 - IVOperIdx), ICmpLoop);
 
-  // If the condition is always true or always false, replace it with
-  // a constant value.
-  if (SE->isKnownPredicate(Pred, S, X)) {
-    ICmp->replaceAllUsesWith(ConstantInt::getTrue(ICmp->getContext()));
-    DeadInsts.emplace_back(ICmp);
-    LLVM_DEBUG(dbgs() << "INDVARS: Eliminated comparison: " << *ICmp << '\n');
-  } else if (SE->isKnownPredicate(ICmpInst::getInversePredicate(Pred), S, X)) {
-    ICmp->replaceAllUsesWith(ConstantInt::getFalse(ICmp->getContext()));
+  // If the condition is always true or always false in the given context,
+  // replace it with a constant value.
+  SmallVector<Instruction *, 4> Users;
+  for (auto *U : ICmp->users())
+    Users.push_back(cast<Instruction>(U));
+  const Instruction *CtxI = findCommonDominator(Users, *DT);
+  if (auto Ev = SE->evaluatePredicateAt(Pred, S, X, CtxI)) {
+    ICmp->replaceAllUsesWith(ConstantInt::getBool(ICmp->getContext(), *Ev));
     DeadInsts.emplace_back(ICmp);
     LLVM_DEBUG(dbgs() << "INDVARS: Eliminated comparison: " << *ICmp << '\n');
   } else if (makeIVComparisonInvariant(ICmp, IVOperand)) {
@@ -1575,17 +1593,7 @@ bool WidenIV::widenWithVariantUse(WidenIV::NarrowIVDefUse DU) {
   // We'll prove some facts that should be true in the context of ext users. If
   // there is no users, we are done now. If there are some, pick their common
   // dominator as context.
-  Instruction *Context = nullptr;
-  for (auto *Ext : ExtUsers) {
-    if (!Context || DT->dominates(Ext, Context))
-      Context = Ext;
-    else if (!DT->dominates(Context, Ext))
-      // For users that don't have dominance relation, use common dominator.
-      Context =
-          DT->findNearestCommonDominator(Context->getParent(), Ext->getParent())
-              ->getTerminator();
-  }
-  assert(Context && "Context not found?");
+  const Instruction *CtxI = findCommonDominator(ExtUsers, *DT);
 
   if (!CanSignExtend && !CanZeroExtend) {
     // Because InstCombine turns 'sub nuw' to 'add' losing the no-wrap flag, we
@@ -1601,8 +1609,8 @@ bool WidenIV::widenWithVariantUse(WidenIV::NarrowIVDefUse DU) {
       return false;
     if (!SE->isKnownNegative(RHS))
       return false;
-    bool ProvedSubNUW = SE->isKnownPredicateAt(
-        ICmpInst::ICMP_UGE, LHS, SE->getNegativeSCEV(RHS), Context);
+    bool ProvedSubNUW = SE->isKnownPredicateAt(ICmpInst::ICMP_UGE, LHS,
+                                               SE->getNegativeSCEV(RHS), CtxI);
     if (!ProvedSubNUW)
       return false;
     // In fact, our 'add' is 'sub nuw'. We will need to widen the 2nd operand as
diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp
index ae839bd3a3d365416b5d9b4242647fa33412babe..d8751888ad21ebf193ab8d25608ce590ebd06097 100644
--- a/llvm/lib/Transforms/Utils/ValueMapper.cpp
+++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp
@@ -945,7 +945,8 @@ void Mapper::remapInstruction(Instruction *I) {
     AttributeList Attrs = CB->getAttributes();
     for (unsigned i = 0; i < Attrs.getNumAttrSets(); ++i) {
       for (Attribute::AttrKind TypedAttr :
-           {Attribute::ByVal, Attribute::StructRet, Attribute::ByRef}) {
+             {Attribute::ByVal, Attribute::StructRet, Attribute::ByRef,
+              Attribute::InAlloca}) {
         if (Type *Ty = Attrs.getAttribute(i, TypedAttr).getValueAsType()) {
           Attrs = Attrs.replaceAttributeType(C, i, TypedAttr,
                                              TypeMapper->remapType(Ty));
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index ce10e1d8a9f06f2e09f5b63ab65dc93e7992a17d..9f19223e8ccf952ee00af58e3ca13218e76022ec 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -251,16 +251,6 @@ bool LoopVectorizationRequirements::doesNotMeet(
     Function *F, Loop *L, const LoopVectorizeHints &Hints) {
   const char *PassName = Hints.vectorizeAnalysisPassName();
   bool Failed = false;
-  if (ExactFPMathInst && !Hints.allowReordering()) {
-    ORE.emit([&]() {
-      return OptimizationRemarkAnalysisFPCommute(
-                 PassName, "CantReorderFPOps", ExactFPMathInst->getDebugLoc(),
-                 ExactFPMathInst->getParent())
-             << "loop not vectorized: cannot prove it is safe to reorder "
-                "floating-point operations";
-    });
-    Failed = true;
-  }
 
   // Test if runtime memcheck thresholds are exceeded.
   bool PragmaThresholdReached =
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 5bb32f69970ae2c83b369f987d0b587ff11c49c9..27d5b613810ac92e906e8b16e0dff80aa3dcd02c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -275,10 +275,9 @@ public:
   /// best selected VPlan.
   void executePlan(InnerLoopVectorizer &LB, DominatorTree *DT);
 
-  void printPlans(raw_ostream &O) {
-    for (const auto &Plan : VPlans)
-      O << *Plan;
-  }
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  void printPlans(raw_ostream &O);
+#endif
 
   /// Look through the existing plans and return true if we have one with all
   /// the vectorization factors in question.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0378a7029639be2fd1d54b47d40cf61eb2ab5e03..6f2d966167bfa227c4b00aa43fe8e9b6fe0f0452 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -372,6 +372,10 @@ cl::opt<bool> llvm::EnableLoopVectorization(
     "vectorize-loops", cl::init(true), cl::Hidden,
     cl::desc("Run the Loop vectorization passes"));
 
+cl::opt<bool> PrintVPlansInDotFormat(
+    "vplan-print-in-dot-format", cl::init(false), cl::Hidden,
+    cl::desc("Use dot format instead of plain text when dumping VPlans"));
+
 /// A helper function that returns the type of loaded or stored value.
 static Type *getMemInstValueType(Value *I) {
   assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
@@ -383,19 +387,11 @@ static Type *getMemInstValueType(Value *I) {
 
 /// A helper function that returns true if the given type is irregular. The
 /// type is irregular if its allocated size doesn't equal the store size of an
-/// element of the corresponding vector type at the given vectorization factor.
-static bool hasIrregularType(Type *Ty, const DataLayout &DL, ElementCount VF) {
-  // Determine if an array of VF elements of type Ty is "bitcast compatible"
-  // with a <VF x Ty> vector.
-  if (VF.isVector()) {
-    auto *VectorTy = VectorType::get(Ty, VF);
-    return TypeSize::get(VF.getKnownMinValue() *
-                             DL.getTypeAllocSize(Ty).getFixedValue(),
-                         VF.isScalable()) != DL.getTypeStoreSize(VectorTy);
-  }
-
-  // If the vectorization factor is one, we just check if an array of type Ty
-  // requires padding between elements.
+/// element of the corresponding vector type.
+static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
+  // Determine if an array of N elements of type Ty is "bitcast compatible"
+  // with a <N x Ty> vector.
+  // This is only true if there is no padding between the array elements.
   return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
 }
 
@@ -661,7 +657,8 @@ protected:
   /// represented as.
   void truncateToMinimalBitwidths(VPTransformState &State);
 
-  /// This function adds (StartIdx, StartIdx + Step, StartIdx + 2*Step, ...)
+  /// This function adds
+  /// (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...)
   /// to each vector element of Val. The sequence starts at StartIndex.
   /// \p Opcode is relevant for FP induction variable.
   virtual Value *
@@ -1455,7 +1452,7 @@ public:
   /// Return the cost model decision for the given instruction \p I and vector
   /// width \p VF. Return CM_Unknown if this instruction did not pass
   /// through the cost modeling.
-  InstWidening getWideningDecision(Instruction *I, ElementCount VF) {
+  InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
     assert(VF.isVector() && "Expected VF to be a vector VF");
     // Cost model is not run in the VPlan-native path - return conservative
     // result until this changes.
@@ -1523,27 +1520,27 @@ public:
 
   /// Returns true if the target machine supports masked store operation
   /// for the given \p DataType and kind of access to \p Ptr.
-  bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) {
+  bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) const {
     return Legal->isConsecutivePtr(Ptr) &&
            TTI.isLegalMaskedStore(DataType, Alignment);
   }
 
   /// Returns true if the target machine supports masked load operation
   /// for the given \p DataType and kind of access to \p Ptr.
-  bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) {
+  bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) const {
     return Legal->isConsecutivePtr(Ptr) &&
            TTI.isLegalMaskedLoad(DataType, Alignment);
   }
 
   /// Returns true if the target machine supports masked scatter operation
   /// for the given \p DataType.
-  bool isLegalMaskedScatter(Type *DataType, Align Alignment) {
+  bool isLegalMaskedScatter(Type *DataType, Align Alignment) const {
     return TTI.isLegalMaskedScatter(DataType, Alignment);
   }
 
   /// Returns true if the target machine supports masked gather operation
   /// for the given \p DataType.
-  bool isLegalMaskedGather(Type *DataType, Align Alignment) {
+  bool isLegalMaskedGather(Type *DataType, Align Alignment) const {
     return TTI.isLegalMaskedGather(DataType, Alignment);
   }
 
@@ -1574,8 +1571,9 @@ public:
   /// instructions that may divide by zero.
   /// If a non-zero VF has been calculated, we check if I will be scalarized
   /// predication for that VF.
-  bool isScalarWithPredication(Instruction *I,
-                               ElementCount VF = ElementCount::getFixed(1));
+  bool
+  isScalarWithPredication(Instruction *I,
+                          ElementCount VF = ElementCount::getFixed(1)) const;
 
   // Returns true if \p I is an instruction that will be predicated either
   // through scalar predication or masked load/store or masked gather/scatter.
@@ -1635,7 +1633,7 @@ public:
   /// Returns true if all loop blocks should be masked to fold tail loop.
   bool foldTailByMasking() const { return FoldTailByMasking; }
 
-  bool blockNeedsPredication(BasicBlock *BB) {
+  bool blockNeedsPredication(BasicBlock *BB) const {
     return foldTailByMasking() || Legal->blockNeedsPredication(BB);
   }
 
@@ -1658,7 +1656,7 @@ public:
   /// Estimate cost of an intrinsic call instruction CI if it were vectorized
   /// with factor VF.  Return the cost of the instruction, including
   /// scalarization overhead if it's needed.
-  InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF);
+  InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
 
   /// Estimate cost of a call instruction CI if it were vectorized with factor
   /// VF. Return the cost of the instruction, including scalarization overhead
@@ -1666,7 +1664,7 @@ public:
   /// scalarized -
   /// i.e. either vector version isn't available, or is too expensive.
   InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF,
-                                    bool &NeedToScalarize);
+                                    bool &NeedToScalarize) const;
 
   /// Invalidates decisions already taken by the cost model.
   void invalidateCostModelingDecisions() {
@@ -1738,7 +1736,8 @@ private:
 
   /// Estimate the overhead of scalarizing an instruction. This is a
   /// convenience wrapper for the type-based getScalarizationOverhead API.
-  InstructionCost getScalarizationOverhead(Instruction *I, ElementCount VF);
+  InstructionCost getScalarizationOverhead(Instruction *I,
+                                           ElementCount VF) const;
 
   /// Returns whether the instruction is a load or store and will be a emitted
   /// as a vector operation.
@@ -1857,7 +1856,7 @@ private:
 
   /// Returns a range containing only operands needing to be extracted.
   SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
-                                                   ElementCount VF) {
+                                                   ElementCount VF) const {
     return SmallVector<Value *, 4>(make_filter_range(
         Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));
   }
@@ -2558,98 +2557,36 @@ Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx, Value *Step,
          "Induction Step must be an integer or FP");
   assert(Step->getType() == STy && "Step has wrong type");
 
-  if (STy->isIntegerTy()) {
-    // Create a vector of consecutive numbers from zero to VF.
-    if (VF.isScalable()) {
-      // Get the stepvector from intrinsic - <0, 1, 2 ... vscale*VF-1>
-      Value *Indices =
-          Builder.CreateIntrinsic(Intrinsic::experimental_vector_stepvector,
-                                  Val->getType(), {}, nullptr, "stepvec.base");
-      assert(Indices->getType() == Val->getType() && "Invalid consecutive vec");
-
-      if (StartIdx) {
-        // here VF = k in <vscale x k x type>. We need to scale StartIdx by
-        // vscale, since StartIdx = VF * Part.
-        Value *ScaledStartIdx = Builder.CreateVScale(
-            ConstantInt::get(STy, StartIdx), "startidx.vscale");
-        Value *StartIdxSplat = Builder.CreateVectorSplat(
-            VLen, ScaledStartIdx, "stepvec.start");
-        Indices = Builder.CreateAdd(Indices, StartIdxSplat);
-      }
+  // Create a vector of consecutive numbers from zero to VF.
+  VectorType *InitVecValVTy = ValVTy;
+  Type *InitVecValSTy = STy;
+  if (STy->isFloatingPointTy()) {
+    InitVecValSTy =
+        IntegerType::get(STy->getContext(), STy->getScalarSizeInBits());
+    InitVecValVTy = VectorType::get(InitVecValSTy, VLen);
+  }
+  Value *InitVec = Builder.CreateStepVector(InitVecValVTy);
 
-      // FIXME: The newly created binary instructions should contain nsw/nuw
-      // flags, which can be found from the original scalar operations.
-      if (!isa<Constant>(Step) || !cast<Constant>(Step)->isOneValue()) {
-        Step =
-            Builder.CreateVectorSplat(VLen, Step, "splat.step");
-        assert(Step->getType() == Val->getType() && "Invalid step vec");
-        Step = Builder.CreateMul(Indices, Step, "stepvec.scaled");
-      } else {
-        Step = Indices;
-      }
-    } else {
-      SmallVector<Constant *, 8> Indices;
-      for (unsigned i = 0; i < VLen.getKnownMinValue(); ++i)
-        Indices.push_back(ConstantInt::get(STy, StartIdx + i));
-
-      // Add the consecutive indices to the vector value.
-      Constant *Cv = ConstantVector::get(Indices);
-      assert(Cv->getType() == Val->getType() && "Invalid consecutive vec");
-      Step = Builder.CreateVectorSplat(VLen, Step, "");
-      assert(Step->getType() == Val->getType() && "Invalid step vec");
-      // FIXME: The newly created binary instructions should contain nsw/nuw
-      // flags, which can be found from the original scalar operations.
-      Step = Builder.CreateMul(Cv, Step);
-    }
-    // Vectorized induction variable is created by adding broadcasted index and
-    // the step vector for each part.
+  // Add on StartIdx
+  Value *StartIdxSplat = Builder.CreateVectorSplat(
+      VLen, ConstantInt::get(InitVecValSTy, StartIdx));
+  InitVec = Builder.CreateAdd(InitVec, StartIdxSplat);
+
+  if (STy->isIntegerTy()) {
+    Step = Builder.CreateVectorSplat(VLen, Step);
+    assert(Step->getType() == Val->getType() && "Invalid step vec");
+    // FIXME: The newly created binary instructions should contain nsw/nuw flags,
+    // which can be found from the original scalar operations.
+    Step = Builder.CreateMul(InitVec, Step);
     return Builder.CreateAdd(Val, Step, "induction");
   }
 
   // Floating point induction.
   assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
          "Binary Opcode should be specified for FP induction");
-
-  if (VF.isScalable()) {
-    auto *IntVTy = VectorType::get(Builder.getInt32Ty(), VLen);
-    Value *Indices =
-        Builder.CreateIntrinsic(Intrinsic::experimental_vector_stepvector,
-                                IntVTy, {}, nullptr, "stepvec.base");
-    if (StartIdx) {
-      // here VF = k in <vscale x k x type>. We need to scale StartIdx by
-      // vscale, since StartIdx = VF * Part.
-      Value *ScaledStartIdx =
-          Builder.CreateVScale(Builder.getInt32(StartIdx), "startidx.vscale");
-      Value *StartIdxSplat =
-          Builder.CreateVectorSplat(VLen, ScaledStartIdx, "stepvec.start");
-      Indices = Builder.CreateAdd(Indices, StartIdxSplat);
-    }
-
-    Indices = Builder.CreateUIToFP(Indices, Val->getType());
-
-    Step = Builder.CreateVectorSplat(VLen, Step, "splat.step");
-    Value *StepxInd = Builder.CreateFMul(Indices, Step, "stepvec.scaled");
-    FastMathFlags Flags;
-    Flags.setFast();
-    if (isa<Instruction>(StepxInd))
-      cast<Instruction>(StepxInd)->setFastMathFlags(Flags);
-
-    Value *BOp = Builder.CreateBinOp(BinOp, Val, StepxInd, "induction");
-    if (isa<Instruction>(BOp))
-      cast<Instruction>(BOp)->setFastMathFlags(Flags);
-    return BOp;
-  }
-
-  // Create a vector of consecutive numbers from zero to VF.
-  SmallVector<Constant *, 8> Indices;
-  for (unsigned i = 0; i < VLen.getKnownMinValue(); ++i)
-    Indices.push_back(ConstantFP::get(STy, (double)(StartIdx + i)));
-
-  // Add the consecutive indices to the vector value.
-  // Floating-point operations inherit FMF via the builder's flags.
-  Constant *Cv = ConstantVector::get(Indices);
+  InitVec = Builder.CreateUIToFP(InitVec, ValVTy);
   Step = Builder.CreateVectorSplat(VLen, Step);
-  Value *MulOp = Builder.CreateFMul(Cv, Step);
+  Value *MulOp = Builder.CreateFMul(InitVec, Step);
   return Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
 }
 
@@ -2723,19 +2660,7 @@ void InnerLoopVectorizer::packScalarIntoVectorValue(VPValue *Def,
 
 Value *InnerLoopVectorizer::reverseVector(Value *Vec) {
   assert(Vec->getType()->isVectorTy() && "Invalid type");
-
-  if (isa<ScalableVectorType>(Vec->getType())) {
-    Function *ReverseFunc = Intrinsic::getDeclaration(
-        LoopVectorPreHeader->getModule(),
-        Intrinsic::experimental_vector_reverse, {Vec->getType()});
-    return Builder.CreateCall(ReverseFunc, {Vec}, "reverse");
-  }
-
-  SmallVector<int, 8> ShuffleMask;
-  for (unsigned i = 0; i < VF.getKnownMinValue(); ++i)
-    ShuffleMask.push_back(VF.getKnownMinValue() - i - 1);
-
-  return Builder.CreateShuffleVector(Vec, ShuffleMask, "reverse");
+  return Builder.CreateVectorReverse(Vec, "reverse");
 }
 
 bool InnerLoopVectorizer::preferPredicatedVectorOps() const {
@@ -3021,30 +2946,21 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(
     bool InBounds = false;
     if (auto *gep = dyn_cast<GetElementPtrInst>(Ptr->stripPointerCasts()))
       InBounds = gep->isInBounds();
-
     if (Reverse) {
       // If the address is consecutive but reversed, then the
       // wide store needs to start at the last vector element.
-
-      // If the address is consecutive but reversed, then the
-      // wide store needs to start at the last vector element.
-      Value *ScaledVF = nullptr;
-      if (VF.isScalable()) {
-        ScaledVF =
-            Builder.CreateVScale(Builder.getInt32(VF.getKnownMinValue()));
-      }
-
-      Value *Index = VF.isScalable()
-                         ? Builder.CreateMul(ScaledVF, Builder.getInt32(-Part))
-                         : Builder.getInt32(-Part * VF.getKnownMinValue());
+      // RunTimeVF =  VScale * VF.getKnownMinValue()
+      // For fixed-width VScale is 1, then RunTimeVF = VF.getKnownMinValue()
+      Value *RunTimeVF = getRuntimeVF(Builder, Builder.getInt32Ty(), VF);
+      // NumElt = -Part * RunTimeVF
+      Value *NumElt = Builder.CreateMul(Builder.getInt32(-Part), RunTimeVF);
+      // LastLane = 1 - RunTimeVF
+      Value *LastLane = Builder.CreateSub(Builder.getInt32(1), RunTimeVF);
       PartPtr =
-          cast<GetElementPtrInst>(Builder.CreateGEP(ScalarDataTy, Ptr, Index));
+          cast<GetElementPtrInst>(Builder.CreateGEP(ScalarDataTy, Ptr, NumElt));
       PartPtr->setIsInBounds(InBounds);
-      Value *RevIndex = VF.isScalable()
-                            ? Builder.CreateSub(Builder.getInt32(1), ScaledVF)
-                            : Builder.getInt32(1 - VF.getKnownMinValue());
       PartPtr = cast<GetElementPtrInst>(
-          Builder.CreateGEP(ScalarDataTy, PartPtr, RevIndex));
+          Builder.CreateGEP(ScalarDataTy, PartPtr, LastLane));
       PartPtr->setIsInBounds(InBounds);
       if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
         BlockInMaskParts[Part] = reverseVector(BlockInMaskParts[Part]);
@@ -4020,7 +3936,7 @@ static void cse(BasicBlock *BB) {
 
 InstructionCost
 LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, ElementCount VF,
-                                              bool &NeedToScalarize) {
+                                              bool &NeedToScalarize) const {
   Function *F = CI->getCalledFunction();
   Type *ScalarRetTy = CI->getType();
   SmallVector<Type *, 4> Tys, ScalarTys;
@@ -4075,7 +3991,7 @@ static Type *MaybeVectorizeType(Type *Elt, ElementCount VF) {
 
 InstructionCost
 LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
-                                                   ElementCount VF) {
+                                                   ElementCount VF) const {
   Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
   assert(ID && "Expected intrinsic call!");
   Type *RetTy = MaybeVectorizeType(CI->getType(), VF);
@@ -4615,6 +4531,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi, VPTransformState &State) {
 
   setDebugLocFromInst(Builder, LoopExitInst);
 
+  Type *PhiTy = Phi->getType();
   // If tail is folded by masking, the vector value to leave the loop should be
   // a Select choosing between the vectorized LoopExitInst and vectorized Phi,
   // instead of the former. For an inloop reduction the reduction will already
@@ -4638,10 +4555,9 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi, VPTransformState &State) {
       // cheaper for the select to remain in the loop than be sunk out of it,
       // and so use the select value for the phi instead of the old
       // LoopExitValue.
-      RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi];
       if (PreferPredicatedReductionSelect ||
           TTI->preferPredicatedReductionSelect(
-              RdxDesc.getOpcode(), Phi->getType(),
+              RdxDesc.getOpcode(), PhiTy,
               TargetTransformInfo::ReductionFlags())) {
         auto *VecRdxPhi =
             cast<PHINode>(State.get(State.Plan->getVPValue(Phi), Part));
@@ -4671,7 +4587,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi, VPTransformState &State) {
   // If the vector reduction can be performed in a smaller type, we truncate
   // then extend the loop exit value to enable InstCombine to evaluate the
   // entire expression in the smaller type.
-  if (VF.isVector() && Phi->getType() != RdxDesc.getRecurrenceType()) {
+  if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
     assert(!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!");
     assert(!VF.isScalable() && "scalable vectors not yet supported.");
     Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
@@ -4733,16 +4649,15 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi, VPTransformState &State) {
         createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx);
     // If the reduction can be performed in a smaller type, we need to extend
     // the reduction to the wider type before we branch to the original loop.
-    if (Phi->getType() != RdxDesc.getRecurrenceType())
-      ReducedPartRdx =
-        RdxDesc.isSigned()
-        ? Builder.CreateSExt(ReducedPartRdx, Phi->getType())
-        : Builder.CreateZExt(ReducedPartRdx, Phi->getType());
+    if (PhiTy != RdxDesc.getRecurrenceType())
+      ReducedPartRdx = RdxDesc.isSigned()
+                           ? Builder.CreateSExt(ReducedPartRdx, PhiTy)
+                           : Builder.CreateZExt(ReducedPartRdx, PhiTy);
   }
 
   // Create a phi node that merges control-flow from the backedge-taken check
   // block and the middle block.
-  PHINode *BCBlockPhi = PHINode::Create(Phi->getType(), 2, "bc.merge.rdx",
+  PHINode *BCBlockPhi = PHINode::Create(PhiTy, 2, "bc.merge.rdx",
                                         LoopScalarPreHeader->getTerminator());
   for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I)
     BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[I]);
@@ -5160,12 +5075,12 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
     // phi as base and a vectorized version of the step value
     // (<step*0, ..., step*N>) as offset.
     for (unsigned Part = 0; Part < State.UF; ++Part) {
-      SmallVector<Constant *, 8> Indices;
+      Type *VecPhiType = VectorType::get(PhiType, State.VF);
+      Value *StartOffset =
+          ConstantInt::get(VecPhiType, Part * State.VF.getKnownMinValue());
       // Create a vector of consecutive numbers from zero to VF.
-      for (unsigned i = 0; i < State.VF.getKnownMinValue(); ++i)
-        Indices.push_back(
-            ConstantInt::get(PhiType, i + Part * State.VF.getKnownMinValue()));
-      Constant *StartOffset = ConstantVector::get(Indices);
+      StartOffset =
+          Builder.CreateAdd(StartOffset, Builder.CreateStepVector(VecPhiType));
 
       Value *GEP = Builder.CreateGEP(
           ScStValueType->getPointerElementType(), NewPointerPhi,
@@ -5888,8 +5803,8 @@ void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
   Scalars[VF].insert(Worklist.begin(), Worklist.end());
 }
 
-bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
-                                                         ElementCount VF) {
+bool LoopVectorizationCostModel::isScalarWithPredication(
+    Instruction *I, ElementCount VF) const {
   if (!blockNeedsPredication(I->getParent()))
     return false;
   switch (I->getOpcode()) {
@@ -5936,7 +5851,7 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
   // requires padding and will be scalarized.
   auto &DL = I->getModule()->getDataLayout();
   auto *ScalarTy = getMemInstValueType(I);
-  if (hasIrregularType(ScalarTy, DL, VF))
+  if (hasIrregularType(ScalarTy, DL))
     return false;
 
   // Check if masking is required.
@@ -5983,7 +5898,7 @@ bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(
   // requires padding and will be scalarized.
   auto &DL = I->getModule()->getDataLayout();
   auto *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType();
-  if (hasIrregularType(ScalarTy, DL, VF))
+  if (hasIrregularType(ScalarTy, DL))
     return false;
 
   return true;
@@ -6416,7 +6331,11 @@ LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
   unsigned SmallestType, WidestType;
   std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
   unsigned WidestRegister =
-      TTI.getVectorRegisterBitWidth(VectorRegisterWidthFactor);
+      TTI.getRegisterBitWidth(TTI.useScalableVectorType() // FIXME
+                                  ? TargetTransformInfo::RGK_ScalableVector
+                                  : TargetTransformInfo::RGK_FixedWidthVector)
+          .getKnownMinValue();
+  WidestRegister *= VectorRegisterWidthFactor;
 
   // Get the maximum safe dependence distance in bits computed by LAA.
   // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
@@ -6443,7 +6362,10 @@ LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
   ElementCount FeasibleMaxVFLowerBound = ElementCount::getNull();
   ElementCount FeasibleMaxVFUpperBound = ElementCount::getNull();
   std::tie(FeasibleMaxVFLowerBound, FeasibleMaxVFUpperBound) =
-      TTI.getFeasibleMaxVFRange(SmallestType, WidestType,
+      TTI.getFeasibleMaxVFRange(TTI.useScalableVectorType() // FIXME
+                                    ? TargetTransformInfo::RGK_ScalableVector
+                                    : TargetTransformInfo::RGK_FixedWidthVector,
+                                SmallestType, WidestType,
                                 MaxSafeVectorWidthInBits,
                                 VectorRegisterWidthFactor);
 
@@ -7170,8 +7092,11 @@ LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<ElementCount> VFs) {
     if (Ty->isTokenTy())
       return 0U;
     unsigned TypeSize = DL.getTypeSizeInBits(Ty->getScalarType());
-    return TTI.getVectorRegisterUsage(VF.getKnownMinValue(), TypeSize,
-                                      MaxSafeDepDist);
+    return TTI.getVectorRegisterUsage(
+        TTI.useScalableVectorType() // FIXME
+            ? TargetTransformInfo::RGK_ScalableVector
+            : TargetTransformInfo::RGK_FixedWidthVector,
+        VF.getKnownMinValue(), TypeSize, MaxSafeDepDist);
   };
 
   for (unsigned int i = 0, s = IdxToInstr.size(); i < s; ++i) {
@@ -7605,12 +7530,20 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
   // we might create due to scalarization.
   Cost += getScalarizationOverhead(I, VF);
 
-  // If we have a predicated store, it may not be executed for each vector
-  // lane. Scale the cost by the probability of executing the predicated
-  // block.
+  // If we have a predicated load/store, it will need extra i1 extracts and
+  // conditional branches, but may not be executed for each vector lane. Scale
+  // the cost by the probability of executing the predicated block.
   if (isPredicatedInst(I)) {
     Cost /= getReciprocalPredBlockProb();
 
+    // Add the cost of an i1 extract and a branch
+    auto *Vec_i1Ty =
+        VectorType::get(IntegerType::getInt1Ty(ValTy->getContext()), VF);
+    Cost += TTI.getScalarizationOverhead(
+        Vec_i1Ty, APInt::getAllOnesValue(VF.getKnownMinValue()),
+        /*Insert=*/false, /*Extract=*/true);
+    Cost += TTI.getCFInstrCost(Instruction::Br, TTI::TCK_RecipThroughput);
+
     if (useEmulatedMaskMemRefHack(I))
       // Artificially setting to a high enough value to practically disable
       // vectorization with such operations.
@@ -7643,7 +7576,8 @@ LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
 
   bool Reverse = ConsecutiveStride < 0;
   if (Reverse)
-    Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0);
+    Cost +=
+        TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, None, 0);
   return Cost;
 }
 
@@ -7727,8 +7661,9 @@ LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
     // TODO: Add support for reversed masked interleaved access.
     assert(!Legal->isMaskRequired(I) &&
            "Reverse masked interleaved access not supported.");
-    Cost += Group->getNumMembers() *
-            TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0);
+    Cost +=
+        Group->getNumMembers() *
+        TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, None, 0);
   }
   return Cost;
 }
@@ -7917,7 +7852,7 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
 
 InstructionCost
 LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
-                                                     ElementCount VF) {
+                                                     ElementCount VF) const {
 
   if (VF.isScalable())
     return InstructionCost::getInvalid();
@@ -8189,7 +8124,7 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I, ElementCount VF,
     if (VF.isVector() && Legal->isFirstOrderRecurrence(Phi))
       return TTI.getShuffleCost(
           TargetTransformInfo::SK_ExtractSubvector, cast<VectorType>(VectorTy),
-          VF.getKnownMinValue() - 1,
+          None, VF.getKnownMinValue() - 1,
           VectorType::get(RetTy, ElementCount::get(1, VF.isScalable())));
 
     // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
@@ -8560,8 +8495,10 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
     // If the user doesn't provide a vectorization factor, determine a
     // reasonable one.
     if (UserVF.isZero()) {
-      VF = ElementCount::getFixed(
-          determineVPlanVF(TTI->getRegisterBitWidth(true /* Vector*/), CM));
+      VF = ElementCount::getFixed(determineVPlanVF(
+          TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+              .getFixedSize(),
+          CM));
       LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n");
 
       // Make sure we have a VF > 1 for stress testing.
@@ -8705,6 +8642,16 @@ void LoopVectorizationPlanner::executePlan(InnerLoopVectorizer &ILV,
   ILV.printDebugTracesAtEnd();
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void LoopVectorizationPlanner::printPlans(raw_ostream &O) {
+  for (const auto &Plan : VPlans)
+    if (PrintVPlansInDotFormat)
+      Plan->printDOT(O);
+    else
+      Plan->print(O);
+}
+#endif
+
 void LoopVectorizationPlanner::collectTriviallyDeadInstructions(
     SmallPtrSetImpl<Instruction *> &DeadInstructions) {
 
@@ -9502,7 +9449,6 @@ VPPredicatedWidenRecipe *VPRecipeBuilder::tryToPredicatedWiden(Instruction *I,
 
 VPBasicBlock *VPRecipeBuilder::handleReplication(
     Instruction *I, VFRange &Range, VPBasicBlock *VPBB,
-    DenseMap<Instruction *, VPReplicateRecipe *> &PredInst2Recipe,
     VPlanPtr &Plan) {
   bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
       [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
@@ -9520,10 +9466,16 @@ VPBasicBlock *VPRecipeBuilder::handleReplication(
   // Find if I uses a predicated instruction. If so, it will use its scalar
   // value. Avoid hoisting the insert-element which packs the scalar value into
   // a vector value, as that happens iff all users use the vector value.
-  for (auto &Op : I->operands())
-    if (auto *PredInst = dyn_cast<Instruction>(Op))
-      if (PredInst2Recipe.find(PredInst) != PredInst2Recipe.end())
-        PredInst2Recipe[PredInst]->setAlsoPack(false);
+  for (VPValue *Op : Recipe->operands()) {
+    auto *PredR = dyn_cast_or_null<VPPredInstPHIRecipe>(Op->getDef());
+    if (!PredR)
+      continue;
+    auto *RepR =
+        cast_or_null<VPReplicateRecipe>(PredR->getOperand(0)->getDef());
+    assert(RepR->isPredicated() &&
+           "expected Replicate recipe to be predicated");
+    RepR->setAlsoPack(false);
+  }
 
   // Finalize the recipe for Instr, first if it is not predicated.
   if (!IsPredicated) {
@@ -9535,7 +9487,6 @@ VPBasicBlock *VPRecipeBuilder::handleReplication(
   assert(VPBB->getSuccessors().empty() &&
          "VPBB has successors when handling predicated replication.");
   // Record predicated instructions for above packing optimizations.
-  PredInst2Recipe[I] = Recipe;
   VPBlockBase *Region = createReplicateRegion(I, Recipe, Plan);
   VPBlockUtils::insertBlockAfter(Region, VPBB);
   auto *RegSucc = new VPBasicBlock();
@@ -9668,11 +9619,6 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
     VFRange &Range, SmallPtrSetImpl<Instruction *> &DeadInstructions,
     const DenseMap<Instruction *, Instruction *> &SinkAfter) {
 
-  // Hold a mapping from predicated instructions to their recipes, in order to
-  // fix their AlsoPack behavior if a user is determined to replicate and use a
-  // scalar instead of vector value.
-  DenseMap<Instruction *, VPReplicateRecipe *> PredInst2Recipe;
-
   SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
 
   VPRecipeBuilder RecipeBuilder(OrigLoop, TLI, Legal, CM, PSE, Builder);
@@ -9776,8 +9722,8 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
 
       // Otherwise, if all widening options failed, Instruction is to be
       // replicated. This may create a successor for VPBB.
-      VPBasicBlock *NextVPBB = RecipeBuilder.handleReplication(
-          Instr, Range, VPBB, PredInst2Recipe, Plan);
+      VPBasicBlock *NextVPBB =
+          RecipeBuilder.handleReplication(Instr, Range, VPBB, Plan);
       if (NextVPBB != VPBB) {
         VPBB = NextVPBB;
         VPBB->setName(BB->hasName() ? BB->getName() + "." + Twine(VPBBsForBB++)
@@ -9980,9 +9926,10 @@ void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
                                VPSlotTracker &SlotTracker) const {
-  O << Indent << "\"INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";
+  O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";
   IG->getInsertPos()->printAsOperand(O, false);
   O << ", ";
   getAddr()->printAsOperand(O, SlotTracker);
@@ -9993,8 +9940,9 @@ void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
   }
   for (unsigned i = 0; i < IG->getFactor(); ++i)
     if (Instruction *I = IG->getMember(i))
-      O << "\\l\" +\n" << Indent << "\"  " << VPlanIngredient(I) << " " << i;
+      O << "\n" << Indent << "  " << VPlanIngredient(I) << " " << i;
 }
+#endif
 
 void VPWidenCallRecipe::execute(VPTransformState &State) {
   State.ILV->widenCallInstruction(*cast<CallInst>(getUnderlyingInstr()), this,
@@ -10032,7 +9980,7 @@ void VPWidenEVLMaskRecipe::execute(VPTransformState &State) {
 Value *InnerLoopVectorizer::createEVLMask(Value *EVL) {
   Value *EVLSplat = Builder.CreateVectorSplat(VF, EVL, "evl");
   Value *StepVec =
-      Builder.CreateIntrinsic(Intrinsic::experimental_vector_stepvector,
+      Builder.CreateIntrinsic(Intrinsic::experimental_stepvector,
                               EVLSplat->getType(), {}, nullptr, "step.vec");
   return Builder.CreateICmpULT(StepVec, EVLSplat, "evl.mask");
 }
@@ -10632,6 +10580,21 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     return false;
   }
 
+  if (!Requirements.canVectorizeFPMath(Hints)) {
+    ORE->emit([&]() {
+      auto *ExactFPMathInst = Requirements.getExactFPInst();
+      return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
+                                                 ExactFPMathInst->getDebugLoc(),
+                                                 ExactFPMathInst->getParent())
+             << "loop not vectorized: cannot prove it is safe to reorder "
+                "floating-point operations";
+    });
+    LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
+                         "reorder floating-point operations\n");
+    Hints.emitRemarkWithHints();
+    return false;
+  }
+
   bool UseInterleaved = TTI->enableInterleavedAccessVectorization();
   InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI());
 
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f7c1cfebb182a86956172fd305600aa4e03a9564..d56f2fe0d989b46075738102ea59a543e9e310c2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -287,10 +287,11 @@ static bool isCommutative(Instruction *I) {
 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
 /// ret <4 x i8> %ins4
 /// InstCombiner transforms this into a shuffle and vector mul
+/// Mask will return the Shuffle Mask equivalent to the extracted elements.
 /// TODO: Can we split off and reuse the shuffle mask detection from
 /// TargetTransformInfo::getInstructionThroughput?
 static Optional<TargetTransformInfo::ShuffleKind>
-isShuffle(ArrayRef<Value *> VL) {
+isShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) {
   auto *EI0 = cast<ExtractElementInst>(VL[0]);
   unsigned Size =
       cast<FixedVectorType>(EI0->getVectorOperandType())->getNumElements();
@@ -308,9 +309,12 @@ isShuffle(ArrayRef<Value *> VL) {
     if (!Idx)
       return None;
     // Undefined behavior if Idx is negative or >= Size.
-    if (Idx->getValue().uge(Size))
+    if (Idx->getValue().uge(Size)) {
+      Mask.push_back(UndefMaskElem);
       continue;
+    }
     unsigned IntIdx = Idx->getValue().getZExtValue();
+    Mask.push_back(IntIdx);
     // We can extractelement from undef or poison vector.
     if (isa<UndefValue>(Vec))
       continue;
@@ -570,7 +574,9 @@ public:
     if (MaxVectorRegSizeOption.getNumOccurrences())
       MaxVecRegSize = MaxVectorRegSizeOption;
     else
-      MaxVecRegSize = TTI->getRegisterBitWidth(true);
+      MaxVecRegSize =
+          TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+              .getFixedSize();
 
     if (MinVectorRegSizeOption.getNumOccurrences())
       MinVecRegSize = MinVectorRegSizeOption;
@@ -937,10 +943,16 @@ public:
                                ScalarEvolution &SE) {
       auto *LI1 = dyn_cast<LoadInst>(V1);
       auto *LI2 = dyn_cast<LoadInst>(V2);
-      if (LI1 && LI2)
-        return isConsecutiveAccess(LI1, LI2, DL, SE)
-                   ? VLOperands::ScoreConsecutiveLoads
-                   : VLOperands::ScoreFail;
+      if (LI1 && LI2) {
+        if (LI1->getParent() != LI2->getParent())
+          return VLOperands::ScoreFail;
+
+        Optional<int> Dist =
+            getPointersDiff(LI1->getPointerOperand(), LI2->getPointerOperand(),
+                            DL, SE, /*StrictCheck=*/true);
+        return (Dist && *Dist == 1) ? VLOperands::ScoreConsecutiveLoads
+                                    : VLOperands::ScoreFail;
+      }
 
       auto *C1 = dyn_cast<Constant>(V1);
       auto *C2 = dyn_cast<Constant>(V2);
@@ -2867,13 +2879,9 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
           Ptr0 = PointerOps[CurrentOrder.front()];
           PtrN = PointerOps[CurrentOrder.back()];
         }
-        const SCEV *Scev0 = SE->getSCEV(Ptr0);
-        const SCEV *ScevN = SE->getSCEV(PtrN);
-        const auto *Diff =
-            dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
-        uint64_t Size = DL->getTypeAllocSize(ScalarTy);
+        Optional<int> Diff = getPointersDiff(Ptr0, PtrN, *DL, *SE);
         // Check that the sorted loads are consecutive.
-        if (Diff && Diff->getAPInt() == (VL.size() - 1) * Size) {
+        if (static_cast<unsigned>(*Diff) == VL.size() - 1) {
           if (CurrentOrder.empty()) {
             // Original loads are consecutive and does not require reordering.
             ++NumOpsWantToKeepOriginalOrder;
@@ -3146,13 +3154,9 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
           Ptr0 = PointerOps[CurrentOrder.front()];
           PtrN = PointerOps[CurrentOrder.back()];
         }
-        const SCEV *Scev0 = SE->getSCEV(Ptr0);
-        const SCEV *ScevN = SE->getSCEV(PtrN);
-        const auto *Diff =
-            dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
-        uint64_t Size = DL->getTypeAllocSize(ScalarTy);
+        Optional<int> Dist = getPointersDiff(Ptr0, PtrN, *DL, *SE);
         // Check that the sorted pointer operands are consecutive.
-        if (Diff && Diff->getAPInt() == (VL.size() - 1) * Size) {
+        if (static_cast<unsigned>(*Dist) == VL.size() - 1) {
           if (CurrentOrder.empty()) {
             // Original stores are consecutive and does not require reordering.
             ++NumOpsWantToKeepOriginalOrder;
@@ -3469,21 +3473,25 @@ InstructionCost BoUpSLP::getEntryCost(TreeEntry *E) {
   InstructionCost ReuseShuffleCost = 0;
   if (NeedToShuffleReuses) {
     ReuseShuffleCost =
-        TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
+        TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy,
+                            E->ReuseShuffleIndices);
   }
   if (E->State == TreeEntry::NeedToGather) {
     if (allConstant(VL))
       return 0;
     if (isSplat(VL)) {
       return ReuseShuffleCost +
-             TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
+             TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, None,
+                                 0);
     }
     if (E->getOpcode() == Instruction::ExtractElement &&
         allSameType(VL) && allSameBlock(VL)) {
-      Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL);
+      SmallVector<int> Mask;
+      Optional<TargetTransformInfo::ShuffleKind> ShuffleKind =
+          isShuffle(VL, Mask);
       if (ShuffleKind.hasValue()) {
         InstructionCost Cost =
-            TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
+            TTI->getShuffleCost(ShuffleKind.getValue(), VecTy, Mask);
         for (auto *V : VL) {
           // If all users of instruction are going to be vectorized and this
           // instruction itself is not going to be vectorized, consider this
@@ -3547,8 +3555,10 @@ InstructionCost BoUpSLP::getEntryCost(TreeEntry *E) {
         }
         CommonCost = ReuseShuffleCost;
       } else if (!E->ReorderIndices.empty()) {
+        SmallVector<int> NewMask;
+        inversePermutation(E->ReorderIndices, NewMask);
         CommonCost = TTI->getShuffleCost(
-            TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
+            TargetTransformInfo::SK_PermuteSingleSrc, VecTy, NewMask);
       }
       for (unsigned I = 0, E = VL.size(); I < E; ++I) {
         Instruction *EI = cast<Instruction>(VL[I]);
@@ -3772,9 +3782,12 @@ InstructionCost BoUpSLP::getEntryCost(TreeEntry *E) {
             Instruction::Load, VecTy, cast<LoadInst>(VL0)->getPointerOperand(),
             /*VariableMask=*/false, alignment, CostKind, VL0);
       }
-      if (!NeedToShuffleReuses && !E->ReorderIndices.empty())
+      if (!NeedToShuffleReuses && !E->ReorderIndices.empty()) {
+        SmallVector<int> NewMask;
+        inversePermutation(E->ReorderIndices, NewMask);
         VecLdCost += TTI->getShuffleCost(
-            TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
+            TargetTransformInfo::SK_PermuteSingleSrc, VecTy, NewMask);
+      }
       LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecLdCost, ScalarLdCost));
       return ReuseShuffleCost + VecLdCost - ScalarLdCost;
     }
@@ -3789,9 +3802,12 @@ InstructionCost BoUpSLP::getEntryCost(TreeEntry *E) {
       InstructionCost ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
       InstructionCost VecStCost = TTI->getMemoryOpCost(
           Instruction::Store, VecTy, Alignment, 0, CostKind, VL0);
-      if (IsReorder)
+      if (IsReorder) {
+        SmallVector<int> NewMask;
+        inversePermutation(E->ReorderIndices, NewMask);
         VecStCost += TTI->getShuffleCost(
-            TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
+            TargetTransformInfo::SK_PermuteSingleSrc, VecTy, NewMask);
+      }
       LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecStCost, ScalarStCost));
       return VecStCost - ScalarStCost;
     }
@@ -3858,7 +3874,15 @@ InstructionCost BoUpSLP::getEntryCost(TreeEntry *E) {
         VecCost += TTI->getCastInstrCost(E->getAltOpcode(), VecTy, Src1Ty,
                                          TTI::CastContextHint::None, CostKind);
       }
-      VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, 0);
+
+      SmallVector<int> Mask(E->Scalars.size());
+      for (unsigned I = 0, End = E->Scalars.size(); I < End; ++I) {
+        auto *OpInst = cast<Instruction>(E->Scalars[I]);
+        assert(E->isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode");
+        Mask[I] = I + (OpInst->getOpcode() == E->getAltOpcode() ? End : 0);
+      }
+      VecCost +=
+          TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, Mask, 0);
       LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost));
       return ReuseShuffleCost + VecCost - ScalarCost;
     }
@@ -4911,8 +4935,14 @@ BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
   // sign extend the extracted values below.
   auto *ScalarRoot = VectorizableTree[0]->Scalars[0];
   if (MinBWs.count(ScalarRoot)) {
-    if (auto *I = dyn_cast<Instruction>(VectorRoot))
-      Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
+    if (auto *I = dyn_cast<Instruction>(VectorRoot)) {
+      // If current instr is a phi and not the last phi, insert it after the
+      // last phi node.
+      if (isa<PHINode>(I))
+        Builder.SetInsertPoint(&*I->getParent()->getFirstInsertionPt());
+      else
+        Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
+    }
     auto BundleWidth = VectorizableTree[0]->Scalars.size();
     auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
     auto *VecTy = FixedVectorType::get(MinTy, BundleWidth);
@@ -5151,11 +5181,53 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
   bool ReSchedule = false;
   LLVM_DEBUG(dbgs() << "SLP:  bundle: " << *S.OpValue << "\n");
 
+  auto &&TryScheduleBundle = [this, OldScheduleEnd, SLP](bool ReSchedule,
+                                                         ScheduleData *Bundle) {
+    // The scheduling region got new instructions at the lower end (or it is a
+    // new region for the first bundle). This makes it necessary to
+    // recalculate all dependencies.
+    // It is seldom that this needs to be done a second time after adding the
+    // initial bundle to the region.
+    if (ScheduleEnd != OldScheduleEnd) {
+      for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode())
+        doForAllOpcodes(I, [](ScheduleData *SD) { SD->clearDependencies(); });
+      ReSchedule = true;
+    }
+    if (ReSchedule) {
+      resetSchedule();
+      initialFillReadyList(ReadyInsts);
+    }
+    if (Bundle) {
+      LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle
+                        << " in block " << BB->getName() << "\n");
+      calculateDependencies(Bundle, /*InsertInReadyList=*/true, SLP);
+    }
+
+    // Now try to schedule the new bundle or (if no bundle) just calculate
+    // dependencies. As soon as the bundle is "ready" it means that there are no
+    // cyclic dependencies and we can schedule it. Note that's important that we
+    // don't "schedule" the bundle yet (see cancelScheduling).
+    while (((!Bundle && ReSchedule) || (Bundle && !Bundle->isReady())) &&
+           !ReadyInsts.empty()) {
+      ScheduleData *Picked = ReadyInsts.pop_back_val();
+      if (Picked->isSchedulingEntity() && Picked->isReady())
+        schedule(Picked, ReadyInsts);
+    }
+  };
+
   // Make sure that the scheduling region contains all
   // instructions of the bundle.
   for (Value *V : VL) {
-    if (!extendSchedulingRegion(V, S))
+    if (!extendSchedulingRegion(V, S)) {
+      // If the scheduling region got new instructions at the lower end (or it
+      // is a new region for the first bundle). This makes it necessary to
+      // recalculate all dependencies.
+      // Otherwise the compiler may crash trying to incorrectly calculate
+      // dependencies and emit instruction in the wrong order at the actual
+      // scheduling.
+      TryScheduleBundle(/*ReSchedule=*/false, nullptr);
       return None;
+    }
   }
 
   for (Value *V : VL) {
@@ -5184,42 +5256,8 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
     BundleMember->FirstInBundle = Bundle;
     PrevInBundle = BundleMember;
   }
-  if (ScheduleEnd != OldScheduleEnd) {
-    // The scheduling region got new instructions at the lower end (or it is a
-    // new region for the first bundle). This makes it necessary to
-    // recalculate all dependencies.
-    // It is seldom that this needs to be done a second time after adding the
-    // initial bundle to the region.
-    for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
-      doForAllOpcodes(I, [](ScheduleData *SD) {
-        SD->clearDependencies();
-      });
-    }
-    ReSchedule = true;
-  }
-  if (ReSchedule) {
-    resetSchedule();
-    initialFillReadyList(ReadyInsts);
-  }
   assert(Bundle && "Failed to find schedule bundle");
-
-  LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "
-                    << BB->getName() << "\n");
-
-  calculateDependencies(Bundle, true, SLP);
-
-  // Now try to schedule the new bundle. As soon as the bundle is "ready" it
-  // means that there are no cyclic dependencies and we can schedule it.
-  // Note that's important that we don't "schedule" the bundle yet (see
-  // cancelScheduling).
-  while (!Bundle->isReady() && !ReadyInsts.empty()) {
-
-    ScheduleData *pickedSD = ReadyInsts.pop_back_val();
-
-    if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
-      schedule(pickedSD, ReadyInsts);
-    }
-  }
+  TryScheduleBundle(ReSchedule, Bundle);
   if (!Bundle->isReady()) {
     cancelScheduling(VL, S.OpValue);
     return None;
@@ -5302,41 +5340,39 @@ bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
   BasicBlock::reverse_iterator UpperEnd = BB->rend();
   BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
   BasicBlock::iterator LowerEnd = BB->end();
-  while (true) {
+  while (UpIter != UpperEnd && DownIter != LowerEnd && &*UpIter != I &&
+         &*DownIter != I) {
     if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
       LLVM_DEBUG(dbgs() << "SLP:  exceeded schedule region size limit\n");
       return false;
     }
 
-    if (UpIter != UpperEnd) {
-      if (&*UpIter == I) {
-        initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
-        ScheduleStart = I;
-        if (isOneOf(S, I) != I)
-          CheckSheduleForI(I);
-        LLVM_DEBUG(dbgs() << "SLP:  extend schedule region start to " << *I
-                          << "\n");
-        return true;
-      }
-      ++UpIter;
-    }
-    if (DownIter != LowerEnd) {
-      if (&*DownIter == I) {
-        initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
-                         nullptr);
-        ScheduleEnd = I->getNextNode();
-        if (isOneOf(S, I) != I)
-          CheckSheduleForI(I);
-        assert(ScheduleEnd && "tried to vectorize a terminator?");
-        LLVM_DEBUG(dbgs() << "SLP:  extend schedule region end to " << *I
-                          << "\n");
-        return true;
-      }
-      ++DownIter;
-    }
-    assert((UpIter != UpperEnd || DownIter != LowerEnd) &&
-           "instruction not found in block");
+    ++UpIter;
+    ++DownIter;
+  }
+  if (DownIter == LowerEnd || (UpIter != UpperEnd && &*UpIter == I)) {
+    assert(I->getParent() == ScheduleStart->getParent() &&
+           "Instruction is in wrong basic block.");
+    initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
+    ScheduleStart = I;
+    if (isOneOf(S, I) != I)
+      CheckSheduleForI(I);
+    LLVM_DEBUG(dbgs() << "SLP:  extend schedule region start to " << *I
+                      << "\n");
+    return true;
   }
+  assert((UpIter == UpperEnd || (DownIter != LowerEnd && &*DownIter == I)) &&
+         "Expected to reach top of the basic block or instruction down the "
+         "lower end.");
+  assert(I->getParent() == ScheduleEnd->getParent() &&
+         "Instruction is in wrong basic block.");
+  initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
+                   nullptr);
+  ScheduleEnd = I->getNextNode();
+  if (isOneOf(S, I) != I)
+    CheckSheduleForI(I);
+  assert(ScheduleEnd && "tried to vectorize a terminator?");
+  LLVM_DEBUG(dbgs() << "SLP:  extend schedule region end to " << *I << "\n");
   return true;
 }
 
@@ -6071,20 +6107,41 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
 
   int E = Stores.size();
   SmallBitVector Tails(E, false);
-  SmallVector<int, 16> ConsecutiveChain(E, E + 1);
   int MaxIter = MaxStoreLookup.getValue();
+  SmallVector<std::pair<int, int>, 16> ConsecutiveChain(
+      E, std::make_pair(E, INT_MAX));
+  SmallVector<SmallBitVector, 4> CheckedPairs(E, SmallBitVector(E, false));
   int IterCnt;
   auto &&FindConsecutiveAccess = [this, &Stores, &Tails, &IterCnt, MaxIter,
+                                  &CheckedPairs,
                                   &ConsecutiveChain](int K, int Idx) {
     if (IterCnt >= MaxIter)
       return true;
+    if (CheckedPairs[Idx].test(K))
+      return ConsecutiveChain[K].second == 1 &&
+             ConsecutiveChain[K].first == Idx;
     ++IterCnt;
-    if (!isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE))
+    CheckedPairs[Idx].set(K);
+    CheckedPairs[K].set(Idx);
+    Optional<int> Diff = getPointersDiff(Stores[K]->getPointerOperand(),
+                                         Stores[Idx]->getPointerOperand(), *DL,
+                                         *SE, /*StrictCheck=*/true);
+    if (!Diff || *Diff == 0)
+      return false;
+    int Val = *Diff;
+    if (Val < 0) {
+      if (ConsecutiveChain[Idx].second > -Val) {
+        Tails.set(K);
+        ConsecutiveChain[Idx] = std::make_pair(K, -Val);
+      }
+      return false;
+    }
+    if (ConsecutiveChain[K].second <= Val)
       return false;
 
     Tails.set(Idx);
-    ConsecutiveChain[K] = Idx;
-    return true;
+    ConsecutiveChain[K] = std::make_pair(Idx, Val);
+    return Val == 1;
   };
   // Do a quadratic search on all of the given stores in reverse order and find
   // all of the pairs of stores that follow each other.
@@ -6104,29 +6161,44 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
   // For stores that start but don't end a link in the chain:
   for (int Cnt = E; Cnt > 0; --Cnt) {
     int I = Cnt - 1;
-    if (ConsecutiveChain[I] == E + 1 || Tails.test(I))
+    if (ConsecutiveChain[I].first == E || Tails.test(I))
       continue;
     // We found a store instr that starts a chain. Now follow the chain and try
     // to vectorize it.
     BoUpSLP::ValueList Operands;
     // Collect the chain into a list.
-    while (I != E + 1 && !VectorizedStores.count(Stores[I])) {
+    while (I != E && !VectorizedStores.count(Stores[I])) {
       Operands.push_back(Stores[I]);
+      Tails.set(I);
+      if (ConsecutiveChain[I].second != 1) {
+        // Mark the new end in the chain and go back, if required. It might be
+        // required if the original stores come in reversed order, for example.
+        if (ConsecutiveChain[I].first != E &&
+            Tails.test(ConsecutiveChain[I].first) &&
+            !VectorizedStores.count(Stores[ConsecutiveChain[I].first])) {
+          Tails.reset(ConsecutiveChain[I].first);
+          if (Cnt < ConsecutiveChain[I].first + 2)
+            Cnt = ConsecutiveChain[I].first + 2;
+        }
+        break;
+      }
       // Move to the next value in the chain.
-      I = ConsecutiveChain[I];
+      I = ConsecutiveChain[I].first;
     }
+    assert(!Operands.empty() && "Expected non-empty list of stores.");
 
-    // If a vector register can't hold 1 element, we are done.
     unsigned MaxVecRegSize = R.getMaxVecRegSize();
     unsigned EltSize = R.getVectorElementSize(Operands[0]);
-    if (MaxVecRegSize % EltSize != 0)
-      continue;
+    unsigned MaxElts = llvm::PowerOf2Floor(MaxVecRegSize / EltSize);
+
+    unsigned MinVF = std::max(2U, R.getMinVecRegSize() / EltSize);
+    unsigned MaxVF = std::min(R.getMaximumVF(EltSize, Instruction::Store),
+                              MaxElts);
 
-    unsigned MaxElts = MaxVecRegSize / EltSize;
     // FIXME: Is division-by-2 the correct step? Should we assert that the
     // register size is a power-of-2?
     unsigned StartIdx = 0;
-    for (unsigned Size = llvm::PowerOf2Ceil(MaxElts); Size >= 2; Size /= 2) {
+    for (unsigned Size = MaxVF; Size >= MinVF; Size /= 2) {
       for (unsigned Cnt = StartIdx, E = Operands.size(); Cnt + Size <= E;) {
         ArrayRef<Value *> Slice = makeArrayRef(Operands).slice(Cnt, Size);
         if (!VectorizedStores.count(Slice.front()) &&
@@ -6494,8 +6566,7 @@ class HorizontalReduction {
       // in this case.
       // Do not perform analysis of remaining operands of ParentStackElem.first
       // instruction, this whole instruction is an extra argument.
-      RecurKind ParentRdxKind = getRdxKind(ParentStackElem.first);
-      ParentStackElem.second = getNumberOfOperands(ParentRdxKind);
+      ParentStackElem.second = getNumberOfOperands(ParentStackElem.first);
     } else {
       // We ran into something like:
       // ParentStackElem.first += ... + ExtraArg + ...
@@ -6564,10 +6635,9 @@ class HorizontalReduction {
                          Value *RHS, const Twine &Name, Instruction *I) {
     Value *Op = createOp(Builder, RdxKind, LHS, RHS, Name);
     if (RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RdxKind)) {
-      if (auto *Sel = dyn_cast<SelectInst>(Op)) {
-        propagateIRFlags(Sel->getCondition(),
-                         cast<SelectInst>(I)->getCondition());
-      }
+      if (auto *Sel = dyn_cast<SelectInst>(Op))
+        if (auto *SelI = dyn_cast<SelectInst>(I))
+          propagateIRFlags(Sel->getCondition(), SelI->getCondition());
     }
     propagateIRFlags(Op, I);
     return Op;
@@ -6596,7 +6666,6 @@ class HorizontalReduction {
     if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
       return RecurKind::FMin;
 
-
     if (auto *Select = dyn_cast<SelectInst>(I)) {
       // These would also match llvm.{u,s}{min,max} intrinsic call
       // if were not guarded by the SelectInst check above.
@@ -6666,63 +6735,54 @@ class HorizontalReduction {
     return RecurKind::None;
   }
 
-  /// Return true if this operation is a cmp+select idiom.
-  static bool isCmpSel(RecurKind Kind) {
-    return RecurrenceDescriptor::isIntMinMaxRecurrenceKind(Kind);
-  }
-
   /// Get the index of the first operand.
-  static unsigned getFirstOperandIndex(RecurKind Kind) {
-    // We allow calling this before 'Kind' is set, so handle that specially.
-    if (Kind == RecurKind::None)
-      return 0;
-    return isCmpSel(Kind) ? 1 : 0;
+  static unsigned getFirstOperandIndex(Instruction *I) {
+    return isa<SelectInst>(I) ? 1 : 0;
   }
 
   /// Total number of operands in the reduction operation.
-  static unsigned getNumberOfOperands(RecurKind Kind) {
-    return isCmpSel(Kind) ? 3 : 2;
+  static unsigned getNumberOfOperands(Instruction *I) {
+    return isa<SelectInst>(I) ? 3 : 2;
   }
 
   /// Checks if the instruction is in basic block \p BB.
   /// For a min/max reduction check that both compare and select are in \p BB.
-  static bool hasSameParent(RecurKind Kind, Instruction *I, BasicBlock *BB,
-                            bool IsRedOp) {
-    if (IsRedOp && isCmpSel(Kind)) {
-      auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
-      return I->getParent() == BB && Cmp->getParent() == BB;
+  static bool hasSameParent(Instruction *I, BasicBlock *BB, bool IsRedOp) {
+    auto *Sel = dyn_cast<SelectInst>(I);
+    if (IsRedOp && Sel) {
+      auto *Cmp = cast<Instruction>(Sel->getCondition());
+      return Sel->getParent() == BB && Cmp->getParent() == BB;
     }
     return I->getParent() == BB;
   }
 
   /// Expected number of uses for reduction operations/reduced values.
-  static bool hasRequiredNumberOfUses(RecurKind Kind, Instruction *I,
-                                      bool IsReductionOp) {
+  static bool hasRequiredNumberOfUses(bool MatchCmpSel, Instruction *I) {
     // SelectInst must be used twice while the condition op must have single
     // use only.
-    if (isCmpSel(Kind))
-      return I->hasNUses(2) &&
-             (!IsReductionOp ||
-              cast<SelectInst>(I)->getCondition()->hasOneUse());
+    if (MatchCmpSel) {
+      if (auto *Sel = dyn_cast<SelectInst>(I))
+        return Sel->hasNUses(2) && Sel->getCondition()->hasOneUse();
+      return I->hasNUses(2);
+    }
 
     // Arithmetic reduction operation must be used once only.
     return I->hasOneUse();
   }
 
   /// Initializes the list of reduction operations.
-  void initReductionOps(RecurKind Kind) {
-    if (isCmpSel(Kind))
+  void initReductionOps(Instruction *I) {
+    if (isa<SelectInst>(I))
       ReductionOps.assign(2, ReductionOpsType());
     else
       ReductionOps.assign(1, ReductionOpsType());
   }
 
   /// Add all reduction operations for the reduction instruction \p I.
-  void addReductionOps(RecurKind Kind, Instruction *I) {
-    assert(Kind != RecurKind::None && "Expected reduction operation.");
-    if (isCmpSel(Kind)) {
-      ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
-      ReductionOps[1].emplace_back(I);
+  void addReductionOps(Instruction *I) {
+    if (auto *Sel = dyn_cast<SelectInst>(I)) {
+      ReductionOps[0].emplace_back(Sel->getCondition());
+      ReductionOps[1].emplace_back(Sel);
     } else {
       ReductionOps[0].emplace_back(I);
     }
@@ -6731,12 +6791,12 @@ class HorizontalReduction {
   static Value *getLHS(RecurKind Kind, Instruction *I) {
     if (Kind == RecurKind::None)
       return nullptr;
-    return I->getOperand(getFirstOperandIndex(Kind));
+    return I->getOperand(getFirstOperandIndex(I));
   }
   static Value *getRHS(RecurKind Kind, Instruction *I) {
     if (Kind == RecurKind::None)
       return nullptr;
-    return I->getOperand(getFirstOperandIndex(Kind) + 1);
+    return I->getOperand(getFirstOperandIndex(I) + 1);
   }
 
 public:
@@ -6788,8 +6848,8 @@ public:
     // Post order traverse the reduction tree starting at B. We only handle true
     // trees containing only binary operators.
     SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
-    Stack.push_back(std::make_pair(B, getFirstOperandIndex(RdxKind)));
-    initReductionOps(RdxKind);
+    Stack.push_back(std::make_pair(B, getFirstOperandIndex(B)));
+    initReductionOps(B);
     while (!Stack.empty()) {
       Instruction *TreeN = Stack.back().first;
       unsigned EdgeToVisit = Stack.back().second++;
@@ -6797,12 +6857,12 @@ public:
       bool IsReducedValue = TreeRdxKind != RdxKind;
 
       // Postorder visit.
-      if (IsReducedValue || EdgeToVisit == getNumberOfOperands(TreeRdxKind)) {
+      if (IsReducedValue || EdgeToVisit == getNumberOfOperands(TreeN)) {
         if (IsReducedValue)
           ReducedVals.push_back(TreeN);
         else {
-          auto I = ExtraArgs.find(TreeN);
-          if (I != ExtraArgs.end() && !I->second) {
+          auto ExtraArgsIter = ExtraArgs.find(TreeN);
+          if (ExtraArgsIter != ExtraArgs.end() && !ExtraArgsIter->second) {
             // Check if TreeN is an extra argument of its parent operation.
             if (Stack.size() <= 1) {
               // TreeN can't be an extra argument as it is a root reduction
@@ -6815,7 +6875,7 @@ public:
             markExtraArg(Stack[Stack.size() - 2], TreeN);
             ExtraArgs.erase(TreeN);
           } else
-            addReductionOps(RdxKind, TreeN);
+            addReductionOps(TreeN);
         }
         // Retract.
         Stack.pop_back();
@@ -6824,14 +6884,14 @@ public:
 
       // Visit left or right.
       Value *EdgeVal = TreeN->getOperand(EdgeToVisit);
-      auto *I = dyn_cast<Instruction>(EdgeVal);
-      if (!I) {
+      auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
+      if (!EdgeInst) {
         // Edge value is not a reduction instruction or a leaf instruction.
         // (It may be a constant, function argument, or something else.)
         markExtraArg(Stack.back(), EdgeVal);
         continue;
       }
-      RecurKind EdgeRdxKind = getRdxKind(I);
+      RecurKind EdgeRdxKind = getRdxKind(EdgeInst);
       // Continue analysis if the next operand is a reduction operation or
       // (possibly) a leaf value. If the leaf value opcode is not set,
       // the first met operation != reduction operation is considered as the
@@ -6840,25 +6900,26 @@ public:
       // Each tree node needs to have minimal number of users except for the
       // ultimate reduction.
       const bool IsRdxInst = EdgeRdxKind == RdxKind;
-      if (I != Phi && I != B &&
-          hasSameParent(RdxKind, I, B->getParent(), IsRdxInst) &&
-          hasRequiredNumberOfUses(RdxKind, I, IsRdxInst) &&
-          (!LeafOpcode || LeafOpcode == I->getOpcode() || IsRdxInst)) {
+      if (EdgeInst != Phi && EdgeInst != B &&
+          hasSameParent(EdgeInst, B->getParent(), IsRdxInst) &&
+          hasRequiredNumberOfUses(isa<SelectInst>(B), EdgeInst) &&
+          (!LeafOpcode || LeafOpcode == EdgeInst->getOpcode() || IsRdxInst)) {
         if (IsRdxInst) {
           // We need to be able to reassociate the reduction operations.
-          if (!isVectorizable(EdgeRdxKind, I)) {
+          if (!isVectorizable(EdgeRdxKind, EdgeInst)) {
             // I is an extra argument for TreeN (its parent operation).
-            markExtraArg(Stack.back(), I);
+            markExtraArg(Stack.back(), EdgeInst);
             continue;
           }
         } else if (!LeafOpcode) {
-          LeafOpcode = I->getOpcode();
+          LeafOpcode = EdgeInst->getOpcode();
         }
-        Stack.push_back(std::make_pair(I, getFirstOperandIndex(EdgeRdxKind)));
+        Stack.push_back(
+            std::make_pair(EdgeInst, getFirstOperandIndex(EdgeInst)));
         continue;
       }
       // I is an extra argument for TreeN (its parent operation).
-      markExtraArg(Stack.back(), I);
+      markExtraArg(Stack.back(), EdgeInst);
     }
     return true;
   }
@@ -7001,7 +7062,7 @@ public:
       // Emit a reduction. If the root is a select (min/max idiom), the insert
       // point is the compare condition of that select.
       Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
-      if (isCmpSel(RdxKind))
+      if (isa<SelectInst>(RdxRootInst))
         Builder.SetInsertPoint(getCmpForMinMaxReduction(RdxRootInst));
       else
         Builder.SetInsertPoint(RdxRootInst);
@@ -7043,10 +7104,9 @@ public:
       // select, we also have to RAUW for the compare instruction feeding the
       // reduction root. That's because the original compare may have extra uses
       // besides the final select of the reduction.
-      if (isCmpSel(RdxKind)) {
+      if (auto *ScalarSelect = dyn_cast<SelectInst>(ReductionRoot)) {
         if (auto *VecSelect = dyn_cast<SelectInst>(VectorizedTree)) {
-          Instruction *ScalarCmp =
-              getCmpForMinMaxReduction(cast<Instruction>(ReductionRoot));
+          Instruction *ScalarCmp = getCmpForMinMaxReduction(ScalarSelect);
           ScalarCmp->replaceAllUsesWith(VecSelect->getCondition());
         }
       }
@@ -7461,10 +7521,11 @@ bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
                                                    BasicBlock *BB, BoUpSLP &R) {
   SmallVector<Value *, 16> BuildVectorInsts;
   SmallVector<Value *, 16> BuildVectorOpds;
+  SmallVector<int> Mask;
   if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, BuildVectorInsts) ||
       (llvm::all_of(BuildVectorOpds,
                     [](Value *V) { return isa<ExtractElementInst>(V); }) &&
-       isShuffle(BuildVectorOpds)))
+       isShuffle(BuildVectorOpds, Mask)))
     return false;
 
   // Vectorize starting with the build vector operands ignoring the BuildVector
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index b2fba440d03e77bc8c7fe88ffe14468773a33e42..d698e970e533df28e4f4b796e42c081e8f06800f 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -194,7 +194,6 @@ public:
   /// \p Range.Start to \p Range.End.
   VPBasicBlock *handleReplication(
       Instruction *I, VFRange &Range, VPBasicBlock *VPBB,
-      DenseMap<Instruction *, VPReplicateRecipe *> &PredInst2Recipe,
       VPlanPtr &Plan);
 };
 } // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 1536a6b88426d200d37193216f83c1e9fb39cc4f..e3e3ed912e82380d05ff6ebdd65e806033655fe4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -55,6 +55,7 @@ extern cl::opt<bool> EnableVPlanNativePath;
 
 #define DEBUG_TYPE "vplan"
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) {
   const VPInstruction *Instr = dyn_cast<VPInstruction>(&V);
   VPSlotTracker SlotTracker(
@@ -62,6 +63,7 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) {
   V.print(OS, SlotTracker);
   return OS;
 }
+#endif
 
 Value *VPLane::getAsRuntimeExpr(IRBuilder<> &Builder,
                                 const ElementCount &VF) const {
@@ -88,6 +90,7 @@ VPValue::~VPValue() {
     Def->removeDefinedValue(this);
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPValue::print(raw_ostream &OS, VPSlotTracker &SlotTracker) const {
   if (const VPRecipeBase *R = dyn_cast_or_null<VPRecipeBase>(Def))
     R->print(OS, "", SlotTracker);
@@ -110,6 +113,7 @@ void VPDef::dump() const {
   print(dbgs(), "", SlotTracker);
   dbgs() << "\n";
 }
+#endif
 
 // Get the top-most entry block of \p Start. This is the entry block of the
 // containing VPlan. This function is templated to support both const and non-const blocks
@@ -404,6 +408,44 @@ void VPBasicBlock::dropAllReferences(VPValue *NewValue) {
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPBasicBlock::print(raw_ostream &O, const Twine &Indent,
+                         VPSlotTracker &SlotTracker) const {
+  O << Indent << getName() << ":\n";
+  if (const VPValue *Pred = getPredicate()) {
+    O << Indent << "BlockPredicate:";
+    Pred->printAsOperand(O, SlotTracker);
+    if (const auto *PredInst = dyn_cast<VPInstruction>(Pred))
+      O << " (" << PredInst->getParent()->getName() << ")";
+    O << '\n';
+  }
+
+  auto RecipeIndent = Indent + "  ";
+  for (const VPRecipeBase &Recipe : *this) {
+    Recipe.print(O, RecipeIndent, SlotTracker);
+    O << '\n';
+  }
+
+  if (getSuccessors().empty()) {
+    O << Indent << "No successors\n";
+  } else {
+    O << Indent << "Successor(s): ";
+    ListSeparator LS;
+    for (auto *Succ : getSuccessors())
+      O << LS << Succ->getName();
+    O << '\n';
+  }
+
+  if (const VPValue *CBV = getCondBit()) {
+    O << Indent << "CondBit: ";
+    CBV->printAsOperand(O, SlotTracker);
+    if (const auto *CBI = dyn_cast<VPInstruction>(CBV))
+      O << " (" << CBI->getParent()->getName() << ")";
+    O << '\n';
+  }
+}
+#endif
+
 void VPRegionBlock::dropAllReferences(VPValue *NewValue) {
   for (VPBlockBase *Block : depth_first(Entry))
     // Drop all references in VPBasicBlocks and replace all uses with
@@ -459,6 +501,19 @@ void VPRegionBlock::execute(VPTransformState *State) {
   State->Instance.reset();
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
+                          VPSlotTracker &SlotTracker) const {
+  O << Indent << (isReplicator() ? "<xVFxUF> " : "<x1> ") << getName() << ": {";
+  auto NewIndent = Indent + "  ";
+  for (auto *BlockBase : depth_first(Entry)) {
+    O << '\n';
+    BlockBase->print(O, NewIndent, SlotTracker);
+  }
+  O << Indent << "}\n";
+}
+#endif
+
 void VPRecipeBase::insertBefore(VPRecipeBase *InsertPos) {
   assert(!Parent && "Recipe already in some VPBasicBlock");
   assert(InsertPos->getParent() &&
@@ -564,6 +619,7 @@ void VPInstruction::execute(VPTransformState &State) {
     generateInstruction(State, Part);
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPInstruction::dump() const {
   VPSlotTracker SlotTracker(getParent()->getPlan());
   print(dbgs(), "", SlotTracker);
@@ -604,6 +660,7 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
     Operand->printAsOperand(O, SlotTracker);
   }
 }
+#endif
 
 /// Generate the code inside the body of the vectorized loop. Assumes a single
 /// LoopVectorBody basic-block was created for this. Introduce additional
@@ -701,7 +758,25 @@ void VPlan::execute(VPTransformState *State) {
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD
-void VPlan::dump() const { dbgs() << *this << '\n'; }
+void VPlan::print(raw_ostream &O) const {
+  VPSlotTracker SlotTracker(this);
+
+  O << "VPlan '" << Name << "' {";
+  for (const VPBlockBase *Block : depth_first(getEntry())) {
+    O << '\n';
+    Block->print(O, "", SlotTracker);
+  }
+  O << "}\n";
+}
+
+LLVM_DUMP_METHOD
+void VPlan::printDOT(raw_ostream &O) const {
+  VPlanPrinter Printer(O, *this);
+  Printer.dump();
+}
+
+LLVM_DUMP_METHOD
+void VPlan::dump() const { print(dbgs()); }
 #endif
 
 void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopPreHeaderBB,
@@ -744,6 +819,7 @@ void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopPreHeaderBB,
   assert(DT->verify(DominatorTree::VerificationLevel::Fast));
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 const Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
   return (isa<VPRegionBlock>(Block) ? "cluster_N" : "N") +
          Twine(getOrCreateBID(Block));
@@ -827,46 +903,32 @@ void VPlanPrinter::dumpEdges(const VPBlockBase *Block) {
 }
 
 void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
+  // Implement dot-formatted dump by performing plain-text dump into the
+  // temporary storage followed by some post-processing.
   OS << Indent << getUID(BasicBlock) << " [label =\n";
   bumpIndent(1);
-  OS << Indent << "\"" << DOT::EscapeString(BasicBlock->getName()) << ":\\n\"";
-  bumpIndent(1);
+  std::string Str;
+  raw_string_ostream SS(Str);
+  // Use no indentation as we need to wrap the lines into quotes ourselves.
+  BasicBlock->print(SS, "", SlotTracker);
 
-  // Dump the block predicate.
-  const VPValue *Pred = BasicBlock->getPredicate();
-  if (Pred) {
-    OS << " +\n" << Indent << " \"BlockPredicate: \"";
-    if (const VPInstruction *PredI = dyn_cast<VPInstruction>(Pred)) {
-      PredI->printAsOperand(OS, SlotTracker);
-      OS << " (" << DOT::EscapeString(PredI->getParent()->getName())
-         << ")\\l\"";
-    } else
-      Pred->printAsOperand(OS, SlotTracker);
-  }
+  // We need to process each line of the output separately, so split
+  // single-string plain-text dump.
+  SmallVector<StringRef, 0> Lines;
+  StringRef(Str).rtrim('\n').split(Lines, "\n");
 
-  for (const VPRecipeBase &Recipe : *BasicBlock) {
-    OS << " +\n" << Indent << "\"";
-    // Don't indent inside the recipe printer as we printed it before the
-    // opening quote already.
-    Recipe.print(OS, "", SlotTracker);
-    OS << "\\l\"";
-  }
+  auto EmitLine = [&](StringRef Line, StringRef Suffix) {
+    OS << Indent << '"' << DOT::EscapeString(Line.str()) << "\\l\"" << Suffix;
+  };
 
-  // Dump the condition bit.
-  const VPValue *CBV = BasicBlock->getCondBit();
-  if (CBV) {
-    OS << " +\n" << Indent << " \"CondBit: ";
-    if (const VPInstruction *CBI = dyn_cast<VPInstruction>(CBV)) {
-      CBI->printAsOperand(OS, SlotTracker);
-      OS << " (" << DOT::EscapeString(CBI->getParent()->getName()) << ")\\l\"";
-    } else {
-      CBV->printAsOperand(OS, SlotTracker);
-      OS << "\"";
-    }
-  }
+  // Don't need the "+" after the last line.
+  for (auto Line : make_range(Lines.begin(), Lines.end() - 1))
+    EmitLine(Line, " +\n");
+  EmitLine(Lines.back(), "\n");
+
+  bumpIndent(-1);
+  OS << Indent << "]\n";
 
-  bumpIndent(-2);
-  OS << "\n" << Indent << "]\n";
   dumpEdges(BasicBlock);
 }
 
@@ -886,25 +948,21 @@ void VPlanPrinter::dumpRegion(const VPRegionBlock *Region) {
   dumpEdges(Region);
 }
 
-void VPlanPrinter::printAsIngredient(raw_ostream &O, const Value *V) {
-  std::string IngredientString;
-  raw_string_ostream RSO(IngredientString);
+void VPlanIngredient::print(raw_ostream &O) const {
   if (auto *Inst = dyn_cast<Instruction>(V)) {
     if (!Inst->getType()->isVoidTy()) {
-      Inst->printAsOperand(RSO, false);
-      RSO << " = ";
+      Inst->printAsOperand(O, false);
+      O << " = ";
     }
-    RSO << Inst->getOpcodeName() << " ";
+    O << Inst->getOpcodeName() << " ";
     unsigned E = Inst->getNumOperands();
     if (E > 0) {
-      Inst->getOperand(0)->printAsOperand(RSO, false);
+      Inst->getOperand(0)->printAsOperand(O, false);
       for (unsigned I = 1; I < E; ++I)
-        Inst->getOperand(I)->printAsOperand(RSO << ", ", false);
+        Inst->getOperand(I)->printAsOperand(O << ", ", false);
     }
   } else // !Inst
-    V->printAsOperand(RSO, false);
-  RSO.flush();
-  O << DOT::EscapeString(IngredientString);
+    V->printAsOperand(O, false);
 }
 
 void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
@@ -1065,6 +1123,7 @@ void VPWidenMemoryInstructionRecipe::print(raw_ostream &O, const Twine &Indent,
 
   printOperands(O, SlotTracker);
 }
+#endif
 
 void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) {
   Value *CanonicalIV = State.CanonicalIV;
@@ -1094,7 +1153,7 @@ void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) {
       // Note that, if interleaving is forced with predicated vectorizations the
       // loop vectorizer would have already bailed out.
       VStep = Builder.CreateIntrinsic(
-          Intrinsic::experimental_vector_stepvector,
+          Intrinsic::experimental_stepvector,
           VectorType::get(STy, State.VF), {}, nullptr,
           "stepvector");
 
@@ -1118,12 +1177,14 @@ void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) {
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenCanonicalIVRecipe::print(raw_ostream &O, const Twine &Indent,
                                      VPSlotTracker &SlotTracker) const {
   O << Indent << "EMIT ";
   getVPValue()->printAsOperand(O, SlotTracker);
   O << " = WIDEN-CANONICAL-INDUCTION";
 }
+#endif
 
 void VPWidenEVLRecipe::print(raw_ostream &O, const Twine &Indent,
                              VPSlotTracker &SlotTracker) const {
@@ -1174,6 +1235,7 @@ void VPValue::replaceAllUsesWith(VPValue *New) {
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const {
   if (const Value *UV = getUnderlyingValue()) {
     OS << "ir<";
@@ -1194,6 +1256,7 @@ void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
     Op->printAsOperand(O, SlotTracker);
   });
 }
+#endif
 
 void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
                                           Old2NewTy &Old2New,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f6ff40d57c06b2a631b4a08484b031643cc80b57..acb108c4991851bb3e9991ce8ca871cdd4f0ad6c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -579,16 +579,6 @@ public:
   /// Delete all blocks reachable from a given VPBlockBase, inclusive.
   static void deleteCFG(VPBlockBase *Entry);
 
-  void printAsOperand(raw_ostream &OS, bool PrintType) const {
-    OS << getName();
-  }
-
-  void print(raw_ostream &OS) const {
-    // TODO: Only printing VPBB name for now since we only have dot printing
-    // support for VPInstructions/Recipes.
-    printAsOperand(OS, false);
-  }
-
   /// Return true if it is legal to hoist instructions into this block.
   bool isLegalToHoistInto() {
     // There are currently no constraints that prevent an instruction to be
@@ -599,6 +589,30 @@ public:
   /// Replace all operands of VPUsers in the block with \p NewValue and also
   /// replaces all uses of VPValues defined in the block with NewValue.
   virtual void dropAllReferences(VPValue *NewValue) = 0;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  void printAsOperand(raw_ostream &OS, bool PrintType) const {
+    OS << getName();
+  }
+
+  /// Print plain-text dump of this VPBlockBase to \p O, prefixing all lines
+  /// with \p Indent. \p SlotTracker is used to print unnamed VPValue's using
+  /// consequtive numbers.
+  ///
+  /// Note that the numbering is applied to the whole VPlan, so printing
+  /// individual blocks is consistent with the whole VPlan printing.
+  virtual void print(raw_ostream &O, const Twine &Indent,
+                     VPSlotTracker &SlotTracker) const = 0;
+
+  /// Print plain-text dump of this VPlan to \p O.
+  void print(raw_ostream &O) const {
+    VPSlotTracker SlotTracker(getPlan());
+    print(O, "", SlotTracker);
+  }
+
+  /// Dump this VPBlockBase to dbgs().
+  LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
+#endif
 };
 
 /// VPRecipeBase is a base class modeling a sequence of one or more output IR
@@ -755,12 +769,14 @@ public:
   /// provided.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the VPInstruction to \p O.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
 
   /// Print the VPInstruction to dbgs() (for debugging).
-  void dump() const;
+  LLVM_DUMP_METHOD void dump() const;
+#endif
 
   /// Return true if this instruction may modify memory.
   bool mayWriteToMemory() const {
@@ -814,9 +830,11 @@ public:
   /// Produce widened copies of all Ingredients.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 };
 
 class VPPredicatedWidenRecipe : public VPRecipeBase, public VPValue {
@@ -888,9 +906,11 @@ public:
   /// Produce a widened version of the call instruction.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 };
 
 /// A recipe for widening select instructions.
@@ -917,9 +937,11 @@ public:
   /// Produce a widened version of the select instruction.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 };
 
 /// A recipe for handling GEP instructions.
@@ -955,9 +977,11 @@ public:
   /// Generate the gep nodes.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 };
 
 /// A recipe for handling phi nodes of integer and floating-point inductions,
@@ -988,9 +1012,11 @@ public:
   /// needed by their users.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 
   /// Returns the start value of the induction.
   VPValue *getStartValue() { return getOperand(0); }
@@ -1050,9 +1076,11 @@ public:
   /// Generate the phi/select nodes.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 
   /// Returns the start value of the phi, if it is a reduction.
   VPValue *getStartValue() {
@@ -1108,9 +1136,11 @@ public:
   /// Generate the phi/select nodes.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 };
 
 /// VPInterleaveRecipe is a recipe for transforming an interleave group of load
@@ -1171,9 +1201,11 @@ public:
   /// Generate the wide load or store, and shuffles.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 
   const InterleaveGroup<Instruction> *getInterleaveGroup() { return IG; }
 };
@@ -1211,9 +1243,11 @@ public:
   /// Generate the reduction in the loop
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 
   /// The VPValue of the scalar Chain being accumulated.
   VPValue *getChainOp() const { return getOperand(0); }
@@ -1271,13 +1305,17 @@ public:
 
   void setAlsoPack(bool Pack) { AlsoPack = Pack; }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 
   bool isUniform() const { return IsUniform; }
 
   bool isPacked() const { return AlsoPack; }
+
+  bool isPredicated() const { return IsPredicated; }
 };
 
 /// A recipe for generating conditional branches on the bits of a mask.
@@ -1298,16 +1336,17 @@ public:
   /// conditional branch.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override {
-    O << " +\n" << Indent << "\"BRANCH-ON-MASK ";
+    O << Indent << "BRANCH-ON-MASK ";
     if (VPValue *Mask = getMask())
       Mask->printAsOperand(O, SlotTracker);
     else
       O << " All-One";
-    O << "\\l\"";
   }
+#endif
 
   /// Return the mask used by this recipe. Note that a full mask is represented
   /// by a nullptr.
@@ -1340,9 +1379,11 @@ public:
   /// Generates phi nodes for live-outs as needed to retain SSA form.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 };
 
 /// A Recipe for widening load/store operations.
@@ -1407,9 +1448,11 @@ public:
   /// Generate the wide load/store.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 };
 
 class VPPredicatedWidenMemoryInstructionRecipe : public VPRecipeBase {
@@ -1500,9 +1543,11 @@ public:
   /// step = <VF*UF, VF*UF, ..., VF*UF>.
   void execute(VPTransformState &State) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+#endif
 };
 
 /// A recipe to generate Explicit Vector Length (EVL) value to be used with
@@ -1649,6 +1694,17 @@ public:
 
   void dropAllReferences(VPValue *NewValue) override;
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p
+  /// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
+  ///
+  /// Note that the numbering is applied to the whole VPlan, so printing
+  /// individual blocks is consistent with the whole VPlan printing.
+  void print(raw_ostream &O, const Twine &Indent,
+             VPSlotTracker &SlotTracker) const override;
+  using VPBlockBase::print; // Get the print(raw_stream &O) version.
+#endif
+
 private:
   /// Create an IR BasicBlock to hold the output instructions generated by this
   /// VPBasicBlock, and return it. Update the CFGState accordingly.
@@ -1740,6 +1796,18 @@ public:
   void execute(struct VPTransformState *State) override;
 
   void dropAllReferences(VPValue *NewValue) override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// Print this VPRegionBlock to \p O (recursively), prefixing all lines with
+  /// \p Indent. \p SlotTracker is used to print unnamed VPValue's using
+  /// consequtive numbers.
+  ///
+  /// Note that the numbering is applied to the whole VPlan, so printing
+  /// individual regions is consistent with the whole VPlan printing.
+  void print(raw_ostream &O, const Twine &Indent,
+             VPSlotTracker &SlotTracker) const override;
+  using VPBlockBase::print; // Get the print(raw_stream &O) version.
+#endif
 };
 
 //===----------------------------------------------------------------------===//
@@ -2005,8 +2073,16 @@ public:
   VPLoopInfo &getVPLoopInfo() { return VPLInfo; }
   const VPLoopInfo &getVPLoopInfo() const { return VPLInfo; }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// Print this VPlan to \p O.
+  void print(raw_ostream &O) const;
+
+  /// Print this VPlan in DOT format to \p O.
+  void printDOT(raw_ostream &O) const;
+
   /// Dump the plan to stderr (for debugging).
-  void dump() const;
+  LLVM_DUMP_METHOD void dump() const;
+#endif
 
   /// Returns a range mapping the values the range \p Operands to their
   /// corresponding VPValues.
@@ -2026,14 +2102,10 @@ private:
                                   BasicBlock *LoopExitBB);
 };
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 /// VPlanPrinter prints a given VPlan to a given output stream. The printing is
 /// indented and follows the dot format.
 class VPlanPrinter {
-  friend inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan);
-  friend inline raw_ostream &operator<<(raw_ostream &OS,
-                                        const struct VPlanIngredient &I);
-
-private:
   raw_ostream &OS;
   const VPlan &Plan;
   unsigned Depth = 0;
@@ -2044,9 +2116,6 @@ private:
 
   VPSlotTracker SlotTracker;
 
-  VPlanPrinter(raw_ostream &O, const VPlan &P)
-      : OS(O), Plan(P), SlotTracker(&P) {}
-
   /// Handle indentation.
   void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
 
@@ -2076,27 +2145,31 @@ private:
   void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
                 const Twine &Label);
 
-  void dump();
+public:
+  VPlanPrinter(raw_ostream &O, const VPlan &P)
+      : OS(O), Plan(P), SlotTracker(&P) {}
 
-  static void printAsIngredient(raw_ostream &O, const Value *V);
+  LLVM_DUMP_METHOD void dump();
 };
 
 struct VPlanIngredient {
   const Value *V;
 
   VPlanIngredient(const Value *V) : V(V) {}
+
+  void print(raw_ostream &O) const;
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const VPlanIngredient &I) {
-  VPlanPrinter::printAsIngredient(OS, I.V);
+  I.print(OS);
   return OS;
 }
 
 inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan) {
-  VPlanPrinter Printer(OS, Plan);
-  Printer.dump();
+  Plan.print(OS);
   return OS;
 }
+#endif
 
 //===----------------------------------------------------------------------===//
 // VPlan Utilities
@@ -2312,8 +2385,10 @@ class VPlanSlp {
                                        SmallPtrSetImpl<VPValue *> &Candidates,
                                        VPInterleavedAccessInfo &IAI);
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print bundle \p Values to dbgs().
   void dumpBundle(ArrayRef<VPValue *> Values);
+#endif
 
 public:
   VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB) : IAI(IAI), BB(BB) {}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
index 39c879d45647258de65e01effeecc07d365da4ab..fd02805d971f729e2d5f52a8613893334aa670ee 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
@@ -349,6 +349,7 @@ SmallVector<VPlanSlp::MultiNodeOpTy, 4> VPlanSlp::reorderMultiNodeOps() {
   return FinalOrder;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPlanSlp::dumpBundle(ArrayRef<VPValue *> Values) {
   dbgs() << " Ops: ";
   for (auto Op : Values) {
@@ -361,6 +362,7 @@ void VPlanSlp::dumpBundle(ArrayRef<VPValue *> Values) {
   }
   dbgs() << "\n";
 }
+#endif
 
 VPInstruction *VPlanSlp::buildGraph(ArrayRef<VPValue *> Values) {
   assert(!Values.empty() && "Need some operands!");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 83d741e90828f419349ca46d6a5353ec055df949..afe6ece43d5fb214b7f461246b28d21e4e99be45 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -123,11 +123,13 @@ public:
   /// for any other purpose, as the values may change as LLVM evolves.
   unsigned getVPValueID() const { return SubclassID; }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const;
   void print(raw_ostream &OS, VPSlotTracker &Tracker) const;
 
   /// Dump the value to stderr (for debugging).
   void dump() const;
+#endif
 
   unsigned getNumUsers() const { return Users.size(); }
   void addUser(VPUser &User) { Users.push_back(&User); }
@@ -199,8 +201,10 @@ class VPUser {
   SmallVector<VPValue *, 2> Operands;
 
 protected:
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the operands to \p O.
   void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const;
+#endif
 
 public:
   VPUser() {}
@@ -359,12 +363,14 @@ public:
   /// for any other purpose, as the values may change as LLVM evolves.
   unsigned getVPDefID() const { return SubclassID; }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Dump the VPDef to stderr (for debugging).
   void dump() const;
 
   /// Each concrete VPDef prints itself.
   virtual void print(raw_ostream &O, const Twine &Indent,
                      VPSlotTracker &SlotTracker) const = 0;
+#endif
 };
 
 class VPlan;
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 787f146bdddca30ddc437b098c4989d259065527..f43ac6f5f119b8591ce044ad70ad526d7d6581d4 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -192,8 +192,18 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
   InstructionCost NewCost =
       TTI.getMemoryOpCost(Instruction::Load, MinVecTy, Alignment, AS);
   // Optionally, we are shuffling the loaded vector element(s) into place.
+  // For the mask set everything but element 0 to undef to prevent poison from
+  // propagating from the extra loaded memory. This will also optionally
+  // shrink/grow the vector from the loaded size to the output size.
+  // We assume this operation has no cost in codegen if there was no offset.
+  // Note that we could use freeze to avoid poison problems, but then we might
+  // still need a shuffle to change the vector size.
+  unsigned OutputNumElts = Ty->getNumElements();
+  SmallVector<int, 16> Mask(OutputNumElts, UndefMaskElem);
+  assert(OffsetEltIndex < MinVecNumElts && "Address offset too big");
+  Mask[0] = OffsetEltIndex;
   if (OffsetEltIndex)
-    NewCost += TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, MinVecTy);
+    NewCost += TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, MinVecTy, Mask);
 
   // We can aggressively convert to the vector form because the backend can
   // invert this transform if it does not result in a performance win.
@@ -205,17 +215,6 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
   IRBuilder<> Builder(Load);
   Value *CastedPtr = Builder.CreateBitCast(SrcPtr, MinVecTy->getPointerTo(AS));
   Value *VecLd = Builder.CreateAlignedLoad(MinVecTy, CastedPtr, Alignment);
-
-  // Set everything but element 0 to undef to prevent poison from propagating
-  // from the extra loaded memory. This will also optionally shrink/grow the
-  // vector from the loaded size to the output size.
-  // We assume this operation has no cost in codegen if there was no offset.
-  // Note that we could use freeze to avoid poison problems, but then we might
-  // still need a shuffle to change the vector size.
-  unsigned OutputNumElts = Ty->getNumElements();
-  SmallVector<int, 16> Mask(OutputNumElts, UndefMaskElem);
-  assert(OffsetEltIndex < MinVecNumElts && "Address offset too big");
-  Mask[0] = OffsetEltIndex;
   VecLd = Builder.CreateShuffleVector(VecLd, Mask);
 
   replaceValue(I, *VecLd);
@@ -516,15 +515,6 @@ bool VectorCombine::foldBitcastShuf(Instruction &I) {
   if (!SrcTy || !DestTy || I.getOperand(0)->getType() != SrcTy)
     return false;
 
-  // The new shuffle must not cost more than the old shuffle. The bitcast is
-  // moved ahead of the shuffle, so assume that it has the same cost as before.
-  InstructionCost DestCost =
-      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, DestTy);
-  InstructionCost SrcCost =
-      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, SrcTy);
-  if (DestCost > SrcCost || !DestCost.isValid())
-    return false;
-
   unsigned DestNumElts = DestTy->getNumElements();
   unsigned SrcNumElts = SrcTy->getNumElements();
   SmallVector<int, 16> NewMask;
@@ -542,6 +532,16 @@ bool VectorCombine::foldBitcastShuf(Instruction &I) {
     if (!widenShuffleMaskElts(ScaleFactor, Mask, NewMask))
       return false;
   }
+
+  // The new shuffle must not cost more than the old shuffle. The bitcast is
+  // moved ahead of the shuffle, so assume that it has the same cost as before.
+  InstructionCost DestCost = TTI.getShuffleCost(
+      TargetTransformInfo::SK_PermuteSingleSrc, DestTy, NewMask);
+  InstructionCost SrcCost =
+      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, SrcTy, Mask);
+  if (DestCost > SrcCost || !DestCost.isValid())
+    return false;
+
   // bitcast (shuf V, MaskC) --> shuf (bitcast V), MaskC'
   ++NumShufOfBitcast;
   Value *CastV = Builder.CreateBitCast(V, DestTy);
@@ -725,8 +725,10 @@ bool VectorCombine::foldExtractedCmps(Instruction &I) {
   int ExpensiveIndex = ConvertToShuf == Ext0 ? Index0 : Index1;
   auto *CmpTy = cast<FixedVectorType>(CmpInst::makeCmpResultType(X->getType()));
   InstructionCost NewCost = TTI.getCmpSelInstrCost(CmpOpcode, X->getType());
-  NewCost +=
-      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, CmpTy);
+  SmallVector<int, 32> ShufMask(VecTy->getNumElements(), UndefMaskElem);
+  ShufMask[CheapIndex] = ExpensiveIndex;
+  NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, CmpTy,
+                                ShufMask);
   NewCost += TTI.getArithmeticInstrCost(I.getOpcode(), CmpTy);
   NewCost += TTI.getVectorInstrCost(Ext0->getOpcode(), CmpTy, CheapIndex);
 
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index cb9bb700b1ef86a09a456340c33b0653d82341a2..07a438bf581c72147572787263c06c43ea4b919a 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -7,7 +7,7 @@
 # build, see runtimes/CMakeLists.txt, except that we currently check whether
 # compiler-rt is being built to determine whether to first build builtins
 # or not so we need that information in this file as well.
-set(LLVM_ALL_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind;openmp")
+set(LLVM_ALL_RUNTIMES "compiler-rt;libc;libcxx;libcxxabi;libunwind;openmp")
 set(LLVM_ENABLE_RUNTIMES "" CACHE STRING
   "Semicolon-separated list of runtimes to build (${LLVM_ALL_RUNTIMES}), or \"all\".")
 if(LLVM_ENABLE_RUNTIMES STREQUAL "all" )
@@ -166,6 +166,12 @@ endif()
 # for all variables that will apply to runtimes.
 foreach(entry ${runtimes})
   get_filename_component(projName ${entry} NAME)
+  if(projName STREQUAL "libc")
+    # For now, we will use the name "llvmlibc" for the libc project as it is
+    # not a full libc yet. Also, if we leave it as is, the "lib" prefix gets
+    # stripped below and the targets endup having the name "c", "check-c" etc.
+    set(projName "llvmlibc")
+  endif()
   string(REPLACE "-" "_" canon_name ${projName})
   string(TOUPPER ${canon_name} canon_name)
   list(APPEND prefixes ${canon_name})
diff --git a/llvm/test/Analysis/BasicAA/assume.ll b/llvm/test/Analysis/BasicAA/assume.ll
index fe83a8c3df0e3dbc5b6b016f195cb09d157fa51f..bc6be3ef0157eed58cee0d54f1e1853b6e06b2af 100644
--- a/llvm/test/Analysis/BasicAA/assume.ll
+++ b/llvm/test/Analysis/BasicAA/assume.ll
@@ -20,4 +20,21 @@ define void @test1(i8* %P, i8* %Q) nounwind ssp {
 ; CHECK: NoModRef:   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false) <->   tail call void @llvm.assume(i1 true)
 }
 
+; Same but with operand bundles
+define void @test2(i8* %P, i8* %Q) nounwind ssp {
+  tail call void @llvm.assume(i1 true) [ "nonnull"(i8* %P) ]
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  ret void
+
+; CHECK-LABEL: Function: test2:
+
+; CHECK: MayAlias:	i8* %P, i8* %Q
+; CHECK: NoModRef:  Ptr: i8* %P	<->  tail call void @llvm.assume(i1 true) [ "nonnull"(i8* %P) ]
+; CHECK: NoModRef:  Ptr: i8* %Q	<->  tail call void @llvm.assume(i1 true) [ "nonnull"(i8* %P) ]
+; CHECK: Both ModRef:  Ptr: i8* %P	<->  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+; CHECK: Both ModRef:  Ptr: i8* %Q	<->  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+; CHECK: NoModRef:   tail call void @llvm.assume(i1 true) [ "nonnull"(i8* %P) ] <->   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+; CHECK: NoModRef:   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false) <->   tail call void @llvm.assume(i1 true) [ "nonnull"(i8* %P) ]
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/Analysis/BasicAA/gep-modulo.ll b/llvm/test/Analysis/BasicAA/gep-modulo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e5aa4901adee9b26b3a5fad721bd65f465bf7702
--- /dev/null
+++ b/llvm/test/Analysis/BasicAA/gep-modulo.ll
@@ -0,0 +1,148 @@
+; RUN: opt -basic-aa -aa-eval -print-all-alias-modref-info %s 2>&1 | FileCheck %s
+
+target datalayout = "p:64:64:64"
+
+; %gep.idx and %gep.6 must-alias if %mul overflows (e.g. %idx == 52).
+define void @may_overflow_mul_add_i8([16 x i8]* %ptr, i8 %idx) {
+; CHECK-LABEL: Function: may_overflow_mul_add_i8: 3 pointers, 0 call sites
+; CHECK-NEXT:    MayAlias:  [16 x i8]* %ptr, i8* %gep.idx
+; CHECK-NEXT:    PartialAlias: [16 x i8]* %ptr, i8* %gep.6
+; CHECK-NEXT:    MayAlias:  i8* %gep.6, i8* %gep.idx
+;
+  %mul = mul i8 %idx, 5
+  %add = add i8 %mul, 2
+  %gep.idx = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i8 %add
+  store i8 0, i8* %gep.idx, align 1
+  %gep.6 = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i32 6
+  store i8 1, i8* %gep.6, align 1
+  ret void
+}
+
+define void @nuw_nsw_mul_add_i8([16 x i8]* %ptr, i8 %idx) {
+; CHECK-LABEL: Function: nuw_nsw_mul_add_i8: 3 pointers, 0 call sites
+; CHECK-NEXT:    MayAlias: [16 x i8]* %ptr, i8* %gep.idx
+; CHECK-NEXT:    PartialAlias: [16 x i8]* %ptr, i8* %gep.6
+; CHECK-NEXT:    NoAlias:  i8* %gep.6, i8* %gep.idx
+;
+  %mul = mul nuw nsw i8 %idx, 5
+  %add = add nuw nsw i8 %mul, 2
+  %gep.idx = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i8 %add
+  store i8 0, i8* %gep.idx, align 1
+  %gep.6 = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i32 6
+  store i8 1, i8* %gep.6, align 1
+  ret void
+}
+
+; %gep.idx and %gep.3 must-alias if %mul overflows (e.g. %idx == 52).
+define void @may_overflow_mul_sub_i8([16 x i8]* %ptr, i8 %idx) {
+; CHECK-LABEL: Function: may_overflow_mul_sub_i8: 3 pointers, 0 call sites
+; CHECK-NEXT:    MayAlias:  [16 x i8]* %ptr, i8* %gep.idx
+; CHECK-NEXT:    PartialAlias: [16 x i8]* %ptr, i8* %gep.3
+; CHECK-NEXT:    MayAlias:  i8* %gep.3, i8* %gep.idx
+;
+  %mul = mul i8 %idx, 5
+  %sub = sub i8 %mul, 1
+  %gep.idx = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i8 %sub
+  store i8 0, i8* %gep.idx, align 1
+  %gep.3 = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i32 3
+  store i8 1, i8* %gep.3, align 1
+  ret void
+}
+
+define void @nuw_nsw_mul_sub_i8([16 x i8]* %ptr, i8 %idx) {
+; CHECK-LABEL: Function: nuw_nsw_mul_sub_i8: 3 pointers, 0 call sites
+; CHECK-NEXT:    MayAlias:  [16 x i8]* %ptr, i8* %gep.idx
+; CHECK-NEXT:    PartialAlias: [16 x i8]* %ptr, i8* %gep.3
+; CHECK-NEXT:    NoAlias:  i8* %gep.3, i8* %gep.idx
+;
+  %mul = mul nuw nsw i8 %idx, 5
+  %sub = sub nuw nsw i8 %mul, 1
+  %gep.idx = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i8 %sub
+  store i8 0, i8* %gep.idx, align 1
+  %gep.3 = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i32 3
+  store i8 1, i8* %gep.3, align 1
+  ret void
+}
+
+; %gep.idx and %gep.3 must-alias if %mul overflows
+; (e.g. %idx == 3689348814741910323).
+define void @may_overflow_mul_sub_i64([16 x i8]* %ptr, i64 %idx) {
+; CHECK-LABEL: Function: may_overflow_mul_sub_i64: 3 pointers, 0 call sites
+; CHECK-NEXT:    MayAlias:  [16 x i8]* %ptr, i8* %gep.idx
+; CHECK-NEXT:    PartialAlias: [16 x i8]* %ptr, i8* %gep.3
+; CHECK-NEXT:    NoAlias:  i8* %gep.3, i8* %gep.idx
+;
+  %mul = mul i64 %idx, 5
+  %sub = sub i64 %mul, 1
+  %gep.idx = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i64 %sub
+  store i8 0, i8* %gep.idx, align 1
+  %gep.3 = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i64 3
+  store i8 1, i8* %gep.3, align 1
+  ret void
+}
+
+; %gep.idx and %gep.3 must-alias if %mul overflows (e.g. %idx == 110).
+define void @may_overflow_i32_sext([16 x i8]* %ptr, i32 %idx) {
+; CHECK-LABEL: Function: may_overflow_i32_sext: 3 pointers, 0 call sites
+; CHECK-NEXT:    MayAlias:  [16 x i8]* %ptr, i8* %gep.idx
+; CHECK-NEXT:    PartialAlias:  [16 x i8]* %ptr, i8* %gep.3
+; CHECK-NEXT:    MayAlias:	i8* %gep.3, i8* %gep.idx
+;
+  %mul = mul i32 %idx, 678152731
+  %sub = sub i32 %mul, 1582356375
+  %sub.ext = sext i32 %sub to i64
+  %gep.idx = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i64 %sub.ext
+  store i8 0, i8* %gep.idx, align 1
+  %gep.3 = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i32 3
+  store i8 1, i8* %gep.3, align 1
+  ret void
+}
+
+define void @nuw_nsw_i32_sext([16 x i8]* %ptr, i32 %idx) {
+; CHECK-LABEL: Function: nuw_nsw_i32_sext: 3 pointers, 0 call sites
+; CHECK-NEXT:    NoAlias:  [16 x i8]* %ptr, i8* %gep.idx
+; CHECK-NEXT:    PartialAlias:  [16 x i8]* %ptr, i8* %gep.3
+; CHECK-NEXT:    NoAlias:	i8* %gep.3, i8* %gep.idx
+;
+  %mul = mul nuw nsw i32 %idx, 678152731
+  %sub = sub nuw nsw i32 %mul, 1582356375
+  %sub.ext = sext i32 %sub to i64
+  %gep.idx = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i64 %sub.ext
+  store i8 0, i8* %gep.idx, align 1
+  %gep.3 = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i32 3
+  store i8 1, i8* %gep.3, align 1
+  ret void
+}
+
+; %gep.idx and %gep.3 must-alias if %mul overflows (e.g. %idx == 110).
+define void @may_overflow_i32_zext([16 x i8]* %ptr, i32 %idx) {
+; CHECK-LABEL: Function: may_overflow_i32_zext: 3 pointers, 0 call sites
+; CHECK-NEXT:    MayAlias:  [16 x i8]* %ptr, i8* %gep.idx
+; CHECK-NEXT:    PartialAlias:  [16 x i8]* %ptr, i8* %gep.3
+; CHECK-NEXT:    MayAlias:	i8* %gep.3, i8* %gep.idx
+;
+  %mul = mul i32 %idx, 678152731
+  %sub = sub i32 %mul, 1582356375
+  %sub.ext = zext i32 %sub to i64
+  %gep.idx = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i64 %sub.ext
+  store i8 0, i8* %gep.idx, align 1
+  %gep.3 = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i32 3
+  store i8 1, i8* %gep.3, align 1
+  ret void
+}
+
+define void @nuw_nsw_i32_zext([16 x i8]* %ptr, i32 %idx) {
+; CHECK-LABEL: Function: nuw_nsw_i32_zext: 3 pointers, 0 call sites
+; CHECK-NEXT:    NoAlias:  [16 x i8]* %ptr, i8* %gep.idx
+; CHECK-NEXT:    PartialAlias:  [16 x i8]* %ptr, i8* %gep.3
+; CHECK-NEXT:    NoAlias:	i8* %gep.3, i8* %gep.idx
+;
+  %mul = mul nuw nsw i32 %idx, 678152731
+  %sub = sub nuw nsw i32 %mul, 1582356375
+  %sub.ext = zext i32 %sub to i64
+  %gep.idx = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i64 %sub.ext
+  store i8 0, i8* %gep.idx, align 1
+  %gep.3 = getelementptr [16 x i8], [16 x i8]* %ptr, i32 0, i32 3
+  store i8 1, i8* %gep.3, align 1
+  ret void
+}
diff --git a/llvm/test/Analysis/BasicAA/invalidation.ll b/llvm/test/Analysis/BasicAA/invalidation.ll
index 27e94cb6a2e2c8e2e883f97fbed19fc87f5f09a5..facc0305b4bd4789a263c15b0729a2d41b49fe61 100644
--- a/llvm/test/Analysis/BasicAA/invalidation.ll
+++ b/llvm/test/Analysis/BasicAA/invalidation.ll
@@ -13,18 +13,6 @@
 ; CHECK-DT-INVALIDATE: Running pass: AAEvaluator
 ; CHECK-DT-INVALIDATE: Running analysis: BasicAA
 ;
-; Check LoopInfo specifically.
-; RUN: opt -disable-output -disable-verify -debug-pass-manager %s 2>&1 \
-; RUN:     -passes='require<loops>,require<aa>,invalidate<loops>,aa-eval' -aa-pipeline='basic-aa' \
-; RUN:     | FileCheck %s --check-prefix=CHECK-LI-INVALIDATE
-; CHECK-LI-INVALIDATE: Running pass: RequireAnalysisPass
-; CHECK-LI-INVALIDATE: Running analysis: BasicAA
-; CHECK-LI-INVALIDATE: Running pass: InvalidateAnalysisPass
-; CHECK-LI-INVALIDATE: Invalidating analysis: LoopAnalysis
-; CHECK-LI-INVALIDATE: Invalidating analysis: BasicAA
-; CHECK-LI-INVALIDATE: Running pass: AAEvaluator
-; CHECK-LI-INVALIDATE: Running analysis: BasicAA
-;
 ; Check PhiValues specifically.
 ; RUN: opt -disable-output -disable-verify -debug-pass-manager %s 2>&1 \
 ; RUN:     -passes='require<phi-values>,require<aa>,invalidate<phi-values>,aa-eval' -aa-pipeline='basic-aa' \
diff --git a/llvm/test/Analysis/BasicAA/q.bad.ll b/llvm/test/Analysis/BasicAA/q.bad.ll
index 0d22f37cc251dedd2d50a922a61f46bc635c0b2d..ac27143c57d1734be9b07de4504821ad681bea73 100644
--- a/llvm/test/Analysis/BasicAA/q.bad.ll
+++ b/llvm/test/Analysis/BasicAA/q.bad.ll
@@ -178,3 +178,11 @@ define void @constantOffsetHeuristic_i8_i8(i8* %mem, i8 %val) {
   %c = bitcast i8* %c.8 to i32*
   ret void
 }
+
+; CHECK-LABEL: different_large_bitwidths
+; MayAlias: i64* %p1, i64* %p2
+define void @different_large_bitwidths(i8* %a, i64 %i, i128 %j) {
+  %p1 = getelementptr i8, i8* %a, i64 %i
+  %p2 = getelementptr i8, i8* %a, i128 %j
+  ret void
+}
diff --git a/llvm/test/Analysis/BasicAA/zext.ll b/llvm/test/Analysis/BasicAA/zext.ll
index 39e67dcca74ce467090437ab09885848436b2663..8e5d5fe0cf2cd8eb86c25fe5ec567300338d33f1 100644
--- a/llvm/test/Analysis/BasicAA/zext.ll
+++ b/llvm/test/Analysis/BasicAA/zext.ll
@@ -240,5 +240,51 @@ entry:
   ret float %x4
 }
 
+; CHECK-LABEL: Function: test_shl_nuw_zext
+; CHECK: MustAlias: i8* %p.1, i8* %p.2
+define void @test_shl_nuw_zext(i8* %p, i32 %x) {
+  %shl = shl nuw i32 %x, 1
+  %shl.ext = zext i32 %shl to i64
+  %ext = zext i32 %x to i64
+  %ext.shl = shl nuw i64 %ext, 1
+  %p.1 = getelementptr i8, i8* %p, i64 %shl.ext
+  %p.2 = getelementptr i8, i8* %p, i64 %ext.shl
+  ret void
+}
+
+; CHECK-LABEL: Function: test_shl_nsw_sext
+; CHECK: MustAlias: i8* %p.1, i8* %p.2
+define void @test_shl_nsw_sext(i8* %p, i32 %x) {
+  %shl = shl nsw i32 %x, 1
+  %shl.ext = sext i32 %shl to i64
+  %ext = sext i32 %x to i64
+  %ext.shl = shl nsw i64 %ext, 1
+  %p.1 = getelementptr i8, i8* %p, i64 %shl.ext
+  %p.2 = getelementptr i8, i8* %p, i64 %ext.shl
+  ret void
+}
+
+; CHECK-LABEL: Function: test_implicit_sext
+; CHECK: MayAlias: i8* %p.1, i8* %p.2
+define void @test_implicit_sext(i8* %p, i32 %x) {
+  %add = add i32 %x, 1
+  %ext = sext i32 %x to i64
+  %ext.add = add i64 %ext, 1
+  %p.1 = getelementptr i8, i8* %p, i32 %add
+  %p.2 = getelementptr i8, i8* %p, i64 %ext.add
+  ret void
+}
+
+; CHECK-LABEL: Function: test_partial_decomposition
+; CHECK: MustAlias: i8* %p.1, i8* %p.2
+define void @test_partial_decomposition(i8* %p, i32 %x) {
+  %add = add i32 %x, 1
+  %add.1 = add nsw i32 %add, 1
+  %add.2 = add nsw i32 %add, 1
+  %p.1 = getelementptr i8, i8* %p, i32 %add.1
+  %p.2 = getelementptr i8, i8* %p, i32 %add.2
+  ret void
+}
+
 ; Function Attrs: nounwind
 declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Analysis/CostModel/AArch64/neon-stepvector.ll b/llvm/test/Analysis/CostModel/AArch64/neon-stepvector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c38e7582f9bc4e3bdcca4cf21f5763825240dbc5
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/neon-stepvector.ll
@@ -0,0 +1,34 @@
+; RUN: opt -cost-model -analyze -mtriple=aarch64--linux-gnu -mattr=+neon  < %s | FileCheck %s
+
+; Check icmp for legal integer vectors.
+define void @stepvector_legal_int() {
+; CHECK-LABEL: 'stepvector_legal_int'
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %1 = call <2 x i64> @llvm.experimental.stepvector.v2i64()
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %2 = call <4 x i32> @llvm.experimental.stepvector.v4i32()
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = call <8 x i16> @llvm.experimental.stepvector.v8i16()
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = call <16 x i8> @llvm.experimental.stepvector.v16i8()
+  %1 = call <2 x i64> @llvm.experimental.stepvector.v2i64()
+  %2 = call <4 x i32> @llvm.experimental.stepvector.v4i32()
+  %3 = call <8 x i16> @llvm.experimental.stepvector.v8i16()
+  %4 = call <16 x i8> @llvm.experimental.stepvector.v16i8()
+  ret void
+}
+
+; Check icmp for an illegal integer vector.
+define void @stepvector_illegal_int() {
+; CHECK-LABEL: 'stepvector_illegal_int'
+; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %1 = call <4 x i64> @llvm.experimental.stepvector.v4i64()
+; CHECK: Cost Model: Found an estimated cost of 4 for instruction:   %2 = call <16 x i32> @llvm.experimental.stepvector.v16i32()
+  %1 = call <4 x i64> @llvm.experimental.stepvector.v4i64()
+  %2 = call <16 x i32> @llvm.experimental.stepvector.v16i32()
+  ret void
+}
+
+
+declare <2 x i64> @llvm.experimental.stepvector.v2i64()
+declare <4 x i32> @llvm.experimental.stepvector.v4i32()
+declare <8 x i16> @llvm.experimental.stepvector.v8i16()
+declare <16 x i8> @llvm.experimental.stepvector.v16i8()
+
+declare <4 x i64> @llvm.experimental.stepvector.v4i64()
+declare <16 x i32> @llvm.experimental.stepvector.v16i32()
diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-and.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-and.ll
index 2df09c5f7bef34288177bc358130d1dc9fbc8f1b..cbf04bfa8238553b2b80de7520d29d877ad2f9db 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-and.ll
@@ -5,12 +5,12 @@ define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 129 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 364 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 455 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 637 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1001 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 91 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 181 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 362 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-or.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-or.ll
index 3888495a3fc8b21353bb9421e55a96a19a117b0b..dba196f0f042e35750cea45ddccb7a8de1b2b693 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-or.ll
@@ -5,12 +5,12 @@ define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 129 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 364 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 455 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 637 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1001 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 91 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 181 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 362 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-stepvector.ll b/llvm/test/Analysis/CostModel/AArch64/sve-stepvector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c3109248838bf8399c230cc1495fea21178a3db9
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-stepvector.ll
@@ -0,0 +1,39 @@
+; RUN: opt -cost-model -analyze -mtriple=aarch64--linux-gnu -mattr=+sve  < %s 2>%t | FileCheck %s
+
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+; Check icmp for legal integer vectors.
+define void @stepvector_legal_int() {
+; CHECK-LABEL: 'stepvector_legal_int'
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %1 = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %2 = call <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = call <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = call <vscale x 16 x i8> @llvm.experimental.stepvector.nxv16i8()
+  %1 = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+  %2 = call <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+  %3 = call <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+  %4 = call <vscale x 16 x i8> @llvm.experimental.stepvector.nxv16i8()
+  ret void
+}
+
+; Check icmp for an illegal integer vector.
+define void @stepvector_illegal_int() {
+; CHECK-LABEL: 'stepvector_illegal_int'
+; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %1 = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK: Cost Model: Found an estimated cost of 4 for instruction:   %2 = call <vscale x 16 x i32> @llvm.experimental.stepvector.nxv16i32()
+  %1 = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+  %2 = call <vscale x 16 x i32> @llvm.experimental.stepvector.nxv16i32()
+  ret void
+}
+
+
+declare <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+declare <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+declare <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+declare <vscale x 16 x i8> @llvm.experimental.stepvector.nxv16i8()
+
+declare <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+declare <vscale x 16 x i32> @llvm.experimental.stepvector.nxv16i32()
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll b/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
index 7609deb86b8403da58ff5c61f65f688d6cb2353c..07592b1f8d4c494dc617fab28026dd1cd819c8be 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
@@ -4,13 +4,13 @@
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 190 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 382 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll b/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
index 362efbb5615d813b83e0dabdd6e9f7d4410d6272..c78c115fe6b8cb1070ce2da183c0b09ad69e4287 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
@@ -4,13 +4,13 @@
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 190 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 382 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll b/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll
index ce052b3398e51b1f348a481f919fb329c94e2ab7..5a74d68f8a7364f4678e64a69b51825747cffd06 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll
@@ -67,21 +67,21 @@ define i32 @sadd(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'sadd'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 242 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 866 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 370 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 738 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 498 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1890 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 274 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 546 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1874 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7332 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 170 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 530 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1060 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1098 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7316 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 28968 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1044 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2088 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; V8M-SIZE-LABEL: 'sadd'
@@ -124,21 +124,21 @@ define i32 @sadd(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'sadd'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 150 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 203 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 787 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 211 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 779 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 3092 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 203 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 404 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 3084 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12310 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 790 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
@@ -225,21 +225,21 @@ define i32 @uadd(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'uadd'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1040 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1032 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4112 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; V8M-SIZE-LABEL: 'uadd'
@@ -282,21 +282,21 @@ define i32 @uadd(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'uadd'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 145 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1032 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1028 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4104 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
@@ -383,21 +383,21 @@ define i32 @ssub(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'ssub'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 242 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 866 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 370 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 738 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 498 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1890 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 274 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 546 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1874 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7332 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 170 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 530 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1060 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1098 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7316 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 28968 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1044 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2088 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; V8M-SIZE-LABEL: 'ssub'
@@ -440,21 +440,21 @@ define i32 @ssub(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'ssub'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 150 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 203 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 787 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 211 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 779 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 3092 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 203 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 404 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 3084 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12310 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 790 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
@@ -541,21 +541,21 @@ define i32 @usub(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'usub'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1040 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1032 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4112 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; V8M-SIZE-LABEL: 'usub'
@@ -598,21 +598,21 @@ define i32 @usub(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'usub'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 145 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1032 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1028 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4104 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
@@ -699,21 +699,21 @@ define i32 @smul(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'smul'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 120 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 768 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1056 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2112 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 380 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1464 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 118 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 348 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1016 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 420 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1512 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.smul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 228 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 616 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.smul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I8 = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.smul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1252 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.smul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4712 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.smul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 356 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.smul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 872 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.smul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; V8M-SIZE-LABEL: 'smul'
@@ -756,21 +756,21 @@ define i32 @smul(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'smul'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 101 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 165 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 325 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 218 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 816 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 368 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 332 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1174 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.smul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.smul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I8 = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.smul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1164 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.smul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4374 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.smul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 268 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.smul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 534 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.smul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
@@ -857,21 +857,21 @@ define i32 @umul(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'umul'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 492 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 984 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1968 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 376 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1456 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 116 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 344 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1008 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1504 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.umul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 608 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.umul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I8 = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 92 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.umul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1248 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.umul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4704 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.umul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 352 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.umul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 864 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.umul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; V8M-SIZE-LABEL: 'umul'
@@ -914,21 +914,21 @@ define i32 @umul(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'umul'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 181 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 216 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 812 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 184 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 364 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1170 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.umul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 274 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.umul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I8 = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 undef, i8 undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.umul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1162 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.umul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4370 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.umul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 266 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.umul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 530 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.umul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll b/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll
index 251392af24c70233cfe54be603476c47a3814854..2aefb3a8fd597b4d8f3e51594f719b5579df1280 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll
@@ -85,22 +85,22 @@ define i32 @add(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'add'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 118 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 378 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1330 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 302 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 602 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1202 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -160,22 +160,22 @@ define i32 @add(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'add'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 153 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -291,22 +291,22 @@ define i32 @sub(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'sub'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 118 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 378 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1330 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 302 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 602 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1202 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -366,22 +366,22 @@ define i32 @sub(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'sub'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 153 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith-usat.ll b/llvm/test/Analysis/CostModel/ARM/arith-usat.ll
index 342956cb83e889d39625d9b80facf3d5198d284b..9baec1e7d25f0f537fe4a6924065ed629cdf822b 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith-usat.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith-usat.ll
@@ -85,22 +85,22 @@ define i32 @add(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'add'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 448 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -160,22 +160,22 @@ define i32 @add(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'add'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -291,22 +291,22 @@ define i32 @sub(i32 %arg) {
 ;
 ; MVE-RECIP-LABEL: 'sub'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 448 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -366,22 +366,22 @@ define i32 @sub(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'sub'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith.ll b/llvm/test/Analysis/CostModel/ARM/arith.ll
index cbeaa8e97f0658e3b5e13bc09e4ab1ca03f43a91..f1f11e2484aca4794b9085101dece8472c443248 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith.ll
@@ -375,12 +375,12 @@ define void @i64() {
 
 define void @vi8() {
 ; CHECK-MVE1-LABEL: 'vi8'
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = add <2 x i8> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = sub <2 x i8> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = mul <2 x i8> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = ashr <2 x i8> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g2 = lshr <2 x i8> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h2 = shl <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i8> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i8> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i8> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i8> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i8> undef, undef
@@ -414,12 +414,12 @@ define void @vi8() {
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE2-LABEL: 'vi8'
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = add <2 x i8> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = sub <2 x i8> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = mul <2 x i8> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %f2 = ashr <2 x i8> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %g2 = lshr <2 x i8> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %h2 = shl <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i8> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i8> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = and <2 x i8> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j2 = or <2 x i8> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k2 = xor <2 x i8> undef, undef
@@ -609,12 +609,12 @@ define void @vi8() {
 ; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-SIZE-LABEL: 'vi8'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = add <2 x i8> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = sub <2 x i8> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = mul <2 x i8> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %f2 = ashr <2 x i8> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %g2 = lshr <2 x i8> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %h2 = shl <2 x i8> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i8> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i8> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i8> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i8> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i8> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i8> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i8> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i8> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i8> undef, undef
@@ -688,12 +688,12 @@ define void @vi8() {
 
 define void @vi16() {
 ; CHECK-MVE1-LABEL: 'vi16'
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = add <2 x i16> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = sub <2 x i16> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = mul <2 x i16> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = ashr <2 x i16> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g2 = lshr <2 x i16> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h2 = shl <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i16> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i16> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i16> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i16> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i16> undef, undef
@@ -727,12 +727,12 @@ define void @vi16() {
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE2-LABEL: 'vi16'
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = add <2 x i16> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = sub <2 x i16> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = mul <2 x i16> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %f2 = ashr <2 x i16> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %g2 = lshr <2 x i16> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %h2 = shl <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i16> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i16> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = and <2 x i16> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j2 = or <2 x i16> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k2 = xor <2 x i16> undef, undef
@@ -922,12 +922,12 @@ define void @vi16() {
 ; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-SIZE-LABEL: 'vi16'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = add <2 x i16> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = sub <2 x i16> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = mul <2 x i16> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %f2 = ashr <2 x i16> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %g2 = lshr <2 x i16> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %h2 = shl <2 x i16> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i16> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i16> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i16> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i16> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i16> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i16> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i16> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i16> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i16> undef, undef
@@ -1001,12 +1001,12 @@ define void @vi16() {
 
 define void @vi32() {
 ; CHECK-MVE1-LABEL: 'vi32'
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = add <2 x i32> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = sub <2 x i32> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = mul <2 x i32> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = ashr <2 x i32> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g2 = lshr <2 x i32> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h2 = shl <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i32> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i32> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i32> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i32> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i32> undef, undef
@@ -1040,12 +1040,12 @@ define void @vi32() {
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE2-LABEL: 'vi32'
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = add <2 x i32> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = sub <2 x i32> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = mul <2 x i32> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %f2 = ashr <2 x i32> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %g2 = lshr <2 x i32> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %h2 = shl <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i32> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i32> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = and <2 x i32> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j2 = or <2 x i32> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k2 = xor <2 x i32> undef, undef
@@ -1235,12 +1235,12 @@ define void @vi32() {
 ; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-SIZE-LABEL: 'vi32'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = add <2 x i32> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = sub <2 x i32> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = mul <2 x i32> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %f2 = ashr <2 x i32> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %g2 = lshr <2 x i32> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %h2 = shl <2 x i32> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %c2 = add <2 x i32> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %d2 = sub <2 x i32> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = mul <2 x i32> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = ashr <2 x i32> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %g2 = lshr <2 x i32> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %h2 = shl <2 x i32> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i32> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i32> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i32> undef, undef
@@ -1314,21 +1314,21 @@ define void @vi32() {
 
 define void @vi64() {
 ; CHECK-MVE1-LABEL: 'vi64'
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c2 = add <2 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d2 = sub <2 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = mul <2 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f2 = ashr <2 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g2 = lshr <2 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %h2 = shl <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c2 = add <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d2 = sub <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e2 = mul <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = ashr <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %g2 = lshr <2 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %h2 = shl <2 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %c4 = add <4 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %d4 = sub <4 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e4 = mul <4 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f4 = ashr <4 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %g4 = lshr <4 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %h4 = shl <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %c4 = add <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %d4 = sub <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = mul <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f4 = ashr <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %g4 = lshr <4 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %h4 = shl <4 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i4 = and <4 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j4 = or <4 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k4 = xor <4 x i64> undef, undef
@@ -1341,33 +1341,33 @@ define void @vi64() {
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i8 = and <8 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j8 = or <8 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k8 = xor <8 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %c16 = add <16 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %d16 = sub <16 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %e16 = mul <16 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %f16 = ashr <16 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %g16 = lshr <16 x i64> undef, undef
-; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %h16 = shl <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %c16 = add <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %d16 = sub <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e16 = mul <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %f16 = ashr <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %g16 = lshr <16 x i64> undef, undef
+; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %h16 = shl <16 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i16 = and <16 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j16 = or <16 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k16 = xor <16 x i64> undef, undef
 ; CHECK-MVE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE2-LABEL: 'vi64'
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c2 = add <2 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d2 = sub <2 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = mul <2 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f2 = ashr <2 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g2 = lshr <2 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %h2 = shl <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c2 = add <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d2 = sub <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e2 = mul <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = ashr <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %g2 = lshr <2 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %h2 = shl <2 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i2 = and <2 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j2 = or <2 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k2 = xor <2 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %c4 = add <4 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %d4 = sub <4 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e4 = mul <4 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f4 = ashr <4 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %g4 = lshr <4 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %h4 = shl <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %c4 = add <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %d4 = sub <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = mul <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f4 = ashr <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %g4 = lshr <4 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %h4 = shl <4 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i4 = and <4 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j4 = or <4 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k4 = xor <4 x i64> undef, undef
@@ -1380,24 +1380,24 @@ define void @vi64() {
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i8 = and <8 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j8 = or <8 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k8 = xor <8 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %c16 = add <16 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %d16 = sub <16 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %e16 = mul <16 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %f16 = ashr <16 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %g16 = lshr <16 x i64> undef, undef
-; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %h16 = shl <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %c16 = add <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %d16 = sub <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e16 = mul <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %f16 = ashr <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %g16 = lshr <16 x i64> undef, undef
+; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %h16 = shl <16 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %i16 = and <16 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %j16 = or <16 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %k16 = xor <16 x i64> undef, undef
 ; CHECK-MVE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE4-LABEL: 'vi64'
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %c2 = add <2 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %d2 = sub <2 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %e2 = mul <2 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f2 = ashr <2 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %g2 = lshr <2 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %h2 = shl <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c2 = add <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d2 = sub <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e2 = mul <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = ashr <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %g2 = lshr <2 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %h2 = shl <2 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i2 = and <2 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j2 = or <2 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k2 = xor <2 x i64> undef, undef
@@ -1410,21 +1410,21 @@ define void @vi64() {
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i4 = and <4 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j4 = or <4 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k4 = xor <4 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %c8 = add <8 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %d8 = sub <8 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %e8 = mul <8 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %f8 = ashr <8 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %g8 = lshr <8 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %h8 = shl <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %c8 = add <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %d8 = sub <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e8 = mul <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f8 = ashr <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %g8 = lshr <8 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %h8 = shl <8 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %i8 = and <8 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %j8 = or <8 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %k8 = xor <8 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %c16 = add <16 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %d16 = sub <16 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %e16 = mul <16 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %f16 = ashr <16 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %g16 = lshr <16 x i64> undef, undef
-; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %h16 = shl <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %c16 = add <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %d16 = sub <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e16 = mul <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %f16 = ashr <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %g16 = lshr <16 x i64> undef, undef
+; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %h16 = shl <16 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %i16 = and <16 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %j16 = or <16 x i64> undef, undef
 ; CHECK-MVE4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %k16 = xor <16 x i64> undef, undef
@@ -1548,21 +1548,21 @@ define void @vi64() {
 ; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-SIZE-LABEL: 'vi64'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c2 = add <2 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d2 = sub <2 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = mul <2 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f2 = ashr <2 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g2 = lshr <2 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %h2 = shl <2 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c2 = add <2 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d2 = sub <2 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e2 = mul <2 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = ashr <2 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %g2 = lshr <2 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %h2 = shl <2 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %c4 = add <4 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %d4 = sub <4 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e4 = mul <4 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f4 = ashr <4 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %g4 = lshr <4 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %h4 = shl <4 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %c4 = add <4 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %d4 = sub <4 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = mul <4 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f4 = ashr <4 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %g4 = lshr <4 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %h4 = shl <4 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i4 = and <4 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j4 = or <4 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k4 = xor <4 x i64> undef, undef
@@ -1575,12 +1575,12 @@ define void @vi64() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i8 = and <8 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j8 = or <8 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k8 = xor <8 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %c16 = add <16 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %d16 = sub <16 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %e16 = mul <16 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %f16 = ashr <16 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %g16 = lshr <16 x i64> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %h16 = shl <16 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %c16 = add <16 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %d16 = sub <16 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e16 = mul <16 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %f16 = ashr <16 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %g16 = lshr <16 x i64> undef, undef
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %h16 = shl <16 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i16 = and <16 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j16 = or <16 x i64> undef, undef
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k16 = xor <16 x i64> undef, undef
diff --git a/llvm/test/Analysis/CostModel/ARM/cast.ll b/llvm/test/Analysis/CostModel/ARM/cast.ll
index f8377ad3f0c19e010609ec0da86483b7c655ce01..bf2488eec97b269d8d7c9ac85a6cfc0ce0bd5cbf 100644
--- a/llvm/test/Analysis/CostModel/ARM/cast.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cast.ll
@@ -557,36 +557,36 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r87hf = fpext <4 x half> undef to <4 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r88hf = fpext <8 x half> undef to <8 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %r89hf = fpext <16 x half> undef to <16 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r90h = fptoui <2 x half> undef to <2 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r91h = fptosi <2 x half> undef to <2 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r92h = fptoui <2 x half> undef to <2 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r93h = fptosi <2 x half> undef to <2 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r94h = fptoui <2 x half> undef to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r95h = fptosi <2 x half> undef to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r96h = fptoui <2 x half> undef to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r97h = fptosi <2 x half> undef to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r98h = fptoui <2 x half> undef to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r99h = fptosi <2 x half> undef to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r90 = fptoui <2 x float> undef to <2 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r91 = fptosi <2 x float> undef to <2 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r92 = fptoui <2 x float> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r93 = fptosi <2 x float> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r94 = fptoui <2 x float> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r95 = fptosi <2 x float> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r96 = fptoui <2 x float> undef to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r97 = fptosi <2 x float> undef to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %r98 = fptoui <2 x float> undef to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %r99 = fptosi <2 x float> undef to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r90h = fptoui <2 x half> undef to <2 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r91h = fptosi <2 x half> undef to <2 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r92h = fptoui <2 x half> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r93h = fptosi <2 x half> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r94h = fptoui <2 x half> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r95h = fptosi <2 x half> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r96h = fptoui <2 x half> undef to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r97h = fptosi <2 x half> undef to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %r98h = fptoui <2 x half> undef to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %r99h = fptosi <2 x half> undef to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r100 = fptoui <2 x double> undef to <2 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r101 = fptosi <2 x double> undef to <2 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r102 = fptoui <2 x double> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r103 = fptosi <2 x double> undef to <2 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r104 = fptoui <2 x double> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r105 = fptosi <2 x double> undef to <2 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r106 = fptoui <2 x double> undef to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %r107 = fptosi <2 x double> undef to <2 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %r108 = fptoui <2 x double> undef to <2 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %r109 = fptosi <2 x double> undef to <2 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r110 = fptoui <4 x float> undef to <4 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r111 = fptosi <4 x float> undef to <4 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112 = fptoui <4 x float> undef to <4 x i8>
@@ -595,8 +595,8 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115 = fptosi <4 x float> undef to <4 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116 = fptoui <4 x float> undef to <4 x i32>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117 = fptosi <4 x float> undef to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 274 for instruction: %r118 = fptoui <4 x float> undef to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 274 for instruction: %r119 = fptosi <4 x float> undef to <4 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r110h = fptoui <4 x half> undef to <4 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r111h = fptosi <4 x half> undef to <4 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r112h = fptoui <4 x half> undef to <4 x i8>
@@ -605,18 +605,18 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r115h = fptosi <4 x half> undef to <4 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r116h = fptoui <4 x half> undef to <4 x i32>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r117h = fptosi <4 x half> undef to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r118h = fptoui <4 x half> undef to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r119h = fptosi <4 x half> undef to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 274 for instruction: %r118h = fptoui <4 x half> undef to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 274 for instruction: %r119h = fptosi <4 x half> undef to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %r120 = fptoui <4 x double> undef to <4 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %r121 = fptosi <4 x double> undef to <4 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %r122 = fptoui <4 x double> undef to <4 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %r123 = fptosi <4 x double> undef to <4 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %r124 = fptoui <4 x double> undef to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %r125 = fptosi <4 x double> undef to <4 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %r126 = fptoui <4 x double> undef to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %r127 = fptosi <4 x double> undef to <4 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %r128 = fptoui <4 x double> undef to <4 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %r129 = fptosi <4 x double> undef to <4 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r130 = fptoui <8 x float> undef to <8 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r131 = fptosi <8 x float> undef to <8 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r132 = fptoui <8 x float> undef to <8 x i8>
@@ -625,8 +625,8 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r135 = fptosi <8 x float> undef to <8 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r136 = fptoui <8 x float> undef to <8 x i32>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r137 = fptosi <8 x float> undef to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 328 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 328 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1096 for instruction: %r138 = fptoui <8 x float> undef to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1096 for instruction: %r139 = fptosi <8 x float> undef to <8 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r130h = fptoui <8 x half> undef to <8 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r131h = fptosi <8 x half> undef to <8 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r132h = fptoui <8 x half> undef to <8 x i8>
@@ -635,18 +635,18 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r135h = fptosi <8 x half> undef to <8 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r136h = fptoui <8 x half> undef to <8 x i32>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r137h = fptosi <8 x half> undef to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r138h = fptoui <8 x half> undef to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r139h = fptosi <8 x half> undef to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 328 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 328 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1098 for instruction: %r138h = fptoui <8 x half> undef to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1098 for instruction: %r139h = fptosi <8 x half> undef to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 586 for instruction: %r140 = fptoui <8 x double> undef to <8 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 586 for instruction: %r141 = fptosi <8 x double> undef to <8 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 586 for instruction: %r142 = fptoui <8 x double> undef to <8 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 586 for instruction: %r143 = fptosi <8 x double> undef to <8 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 586 for instruction: %r144 = fptoui <8 x double> undef to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 586 for instruction: %r145 = fptosi <8 x double> undef to <8 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 584 for instruction: %r146 = fptoui <8 x double> undef to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 584 for instruction: %r147 = fptosi <8 x double> undef to <8 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1088 for instruction: %r148 = fptoui <8 x double> undef to <8 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1088 for instruction: %r149 = fptosi <8 x double> undef to <8 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r150 = fptoui <16 x float> undef to <16 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r151 = fptosi <16 x float> undef to <16 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r152 = fptoui <16 x float> undef to <16 x i8>
@@ -655,8 +655,8 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r155 = fptosi <16 x float> undef to <16 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r156 = fptoui <16 x float> undef to <16 x i32>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r157 = fptosi <16 x float> undef to <16 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1312 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1312 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4384 for instruction: %r158 = fptoui <16 x float> undef to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4384 for instruction: %r159 = fptosi <16 x float> undef to <16 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r150h = fptoui <16 x half> undef to <16 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r151h = fptosi <16 x half> undef to <16 x i1>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r152h = fptoui <16 x half> undef to <16 x i8>
@@ -665,18 +665,18 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %r155h = fptosi <16 x half> undef to <16 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r156h = fptoui <16 x half> undef to <16 x i32>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r157h = fptosi <16 x half> undef to <16 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1320 for instruction: %r158h = fptoui <16 x half> undef to <16 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1320 for instruction: %r159h = fptosi <16 x half> undef to <16 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1320 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1320 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1312 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1312 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4392 for instruction: %r158h = fptoui <16 x half> undef to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4392 for instruction: %r159h = fptosi <16 x half> undef to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2346 for instruction: %r160 = fptoui <16 x double> undef to <16 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2346 for instruction: %r161 = fptosi <16 x double> undef to <16 x i1>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2346 for instruction: %r162 = fptoui <16 x double> undef to <16 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2346 for instruction: %r163 = fptosi <16 x double> undef to <16 x i8>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2344 for instruction: %r164 = fptoui <16 x double> undef to <16 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2344 for instruction: %r165 = fptosi <16 x double> undef to <16 x i16>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2336 for instruction: %r166 = fptoui <16 x double> undef to <16 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2336 for instruction: %r167 = fptosi <16 x double> undef to <16 x i32>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4352 for instruction: %r168 = fptoui <16 x double> undef to <16 x i64>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4352 for instruction: %r169 = fptosi <16 x double> undef to <16 x i64>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r170 = uitofp <2 x i1> undef to <2 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r171 = sitofp <2 x i1> undef to <2 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r172 = uitofp <2 x i8> undef to <2 x float>
@@ -697,16 +697,16 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r177h = sitofp <2 x i32> undef to <2 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r178h = uitofp <2 x i64> undef to <2 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r179h = sitofp <2 x i64> undef to <2 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r180 = uitofp <2 x i1> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r181 = sitofp <2 x i1> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r182 = uitofp <2 x i8> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r183 = sitofp <2 x i8> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r184 = uitofp <2 x i16> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r185 = sitofp <2 x i16> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r186 = uitofp <2 x i32> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r187 = sitofp <2 x i32> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r188 = uitofp <2 x i64> undef to <2 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %r189 = sitofp <2 x i64> undef to <2 x double>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r190 = uitofp <4 x i1> undef to <4 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r191 = sitofp <4 x i1> undef to <4 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r192 = uitofp <4 x i8> undef to <4 x float>
@@ -727,16 +727,16 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r197h = sitofp <4 x i32> undef to <4 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r198h = uitofp <4 x i64> undef to <4 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r199h = sitofp <4 x i64> undef to <4 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %r200 = uitofp <4 x i1> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %r201 = sitofp <4 x i1> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %r202 = uitofp <4 x i8> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %r203 = sitofp <4 x i8> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %r204 = uitofp <4 x i16> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %r205 = sitofp <4 x i16> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %r206 = uitofp <4 x i32> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %r207 = sitofp <4 x i32> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %r208 = uitofp <4 x i64> undef to <4 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %r209 = sitofp <4 x i64> undef to <4 x double>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r210 = uitofp <8 x i1> undef to <8 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r211 = sitofp <8 x i1> undef to <8 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r212 = uitofp <8 x i8> undef to <8 x float>
@@ -757,16 +757,16 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %r217h = sitofp <8 x i32> undef to <8 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r218h = uitofp <8 x i64> undef to <8 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r219h = sitofp <8 x i64> undef to <8 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 522 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 394 for instruction: %r220 = uitofp <8 x i1> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 394 for instruction: %r221 = sitofp <8 x i1> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 394 for instruction: %r222 = uitofp <8 x i8> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 394 for instruction: %r223 = sitofp <8 x i8> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 394 for instruction: %r224 = uitofp <8 x i16> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 394 for instruction: %r225 = sitofp <8 x i16> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 394 for instruction: %r226 = uitofp <8 x i16> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 394 for instruction: %r227 = sitofp <8 x i16> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %r228 = uitofp <8 x i64> undef to <8 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %r229 = sitofp <8 x i64> undef to <8 x double>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r230 = uitofp <16 x i1> undef to <16 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r231 = sitofp <16 x i1> undef to <16 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %r232 = uitofp <16 x i8> undef to <16 x float>
@@ -787,16 +787,16 @@ define i32 @casts() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %r237h = sitofp <16 x i32> undef to <16 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %r238h = uitofp <16 x i64> undef to <16 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %r239h = sitofp <16 x i64> undef to <16 x half>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2090 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2090 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2090 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2090 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2088 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2088 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2088 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2088 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2048 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2048 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1578 for instruction: %r240 = uitofp <16 x i1> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1578 for instruction: %r241 = sitofp <16 x i1> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1578 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1578 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1576 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1576 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1576 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1576 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1536 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1536 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'casts'
diff --git a/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
index 4406e27acabad643946ff4574c52064a2fba8293..afa616260b3e92461b6fb58ef581edec25c04155 100644
--- a/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
@@ -81,14 +81,14 @@ define i32 @load_extends() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef, align 1
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef, align 2
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef, align 2
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef, align 4
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv16i8 = load <16 x i8>, <16 x i8>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef, align 4
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i16 = load <8 x i16>, <8 x i16>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %loadv2i32 = load <2 x i32>, <2 x i32>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i32 = load <4 x i32>, <4 x i32>* undef, align 16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r0 = sext i8 %loadi8 to i16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r1 = zext i8 %loadi8 to i16
@@ -774,9 +774,9 @@ define i32 @store_trunc() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1632, i16* undef, align 2
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %i1664, i16* undef, align 2
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %i3264, i32* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: store <2 x i8> %v2816, <2 x i8>* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: store <2 x i8> %v2832, <2 x i8>* undef, align 2
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: store <2 x i8> %v2864, <2 x i8>* undef, align 2
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4816, <4 x i8>* undef, align 4
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4832, <4 x i8>* undef, align 4
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4864, <4 x i8>* undef, align 4
@@ -786,13 +786,13 @@ define i32 @store_trunc() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16816, <16 x i8>* undef, align 16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16832, <16 x i8>* undef, align 16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16864, <16 x i8>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: store <2 x i16> %v21632, <2 x i16>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: store <2 x i16> %v21664, <2 x i16>* undef, align 4
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> %v41632, <4 x i16>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> %v41664, <4 x i16>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v81632, <8 x i16>* undef, align 16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v81664, <8 x i16>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: store <2 x i32> %v23264, <2 x i32>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> %v43264, <4 x i32>* undef, align 16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
@@ -1273,11 +1273,11 @@ define i32 @load_fpextends() {
 ; CHECK-MVE-RECIP-LABEL: 'load_fpextends'
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf16 = load half, half* undef, align 2
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadf32 = load float, float* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %loadv2f16 = load <2 x half>, <2 x half>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv4f16 = load <4 x half>, <4 x half>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8f16 = load <8 x half>, <8 x half>* undef, align 16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv16f16 = load <16 x half>, <16 x half>* undef, align 32
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %loadv2f32 = load <2 x float>, <2 x float>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4f32 = load <4 x float>, <4 x float>* undef, align 16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %loadv8f32 = load <8 x float>, <8 x float>* undef, align 32
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r1 = fpext half %loadf16 to float
@@ -1567,13 +1567,13 @@ define i32 @load_fptrunc() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1632, half* undef, align 2
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store half %i1664, half* undef, align 2
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float %i3264, float* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: store <2 x half> %v21632, <2 x half>* undef, align 4
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: store <2 x half> %v21664, <2 x half>* undef, align 4
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x half> %v41632, <4 x half>* undef, align 8
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <4 x half> %v41664, <4 x half>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x half> %v81632, <8 x half>* undef, align 16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x half> %v81664, <8 x half>* undef, align 16
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: store <2 x float> %v23264, <2 x float>* undef, align 8
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x float> %v43264, <4 x float>* undef, align 16
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
diff --git a/llvm/test/Analysis/CostModel/ARM/cmps.ll b/llvm/test/Analysis/CostModel/ARM/cmps.ll
index 75f2bd58b212858ac190cb3107e76b1ae90a2826..bda8763ef18dad4ab9dc701f34e7cff8c7245931 100644
--- a/llvm/test/Analysis/CostModel/ARM/cmps.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cmps.ll
@@ -22,9 +22,9 @@ define i32 @cmps() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a10 = fcmp olt <8 x half> undef, undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a12 = fcmp oge <2 x double> undef, undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %p = icmp eq i32* undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %q = icmp eq <4 x i32*> undef, undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %q = icmp eq <4 x i32*> undef, undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'cmps'
@@ -183,8 +183,8 @@ define void @minmax() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c5 = icmp slt i32* undef, undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s5 = select i1 %c5, i32* undef, i32* undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %c6 = icmp slt <4 x i32*> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %s6 = select <4 x i1> %c6, <4 x i32*> undef, <4 x i32*> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %c6 = icmp slt <4 x i32*> undef, undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s6 = select <4 x i1> %c6, <4 x i32*> undef, <4 x i32*> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'minmax'
diff --git a/llvm/test/Analysis/CostModel/ARM/divrem.ll b/llvm/test/Analysis/CostModel/ARM/divrem.ll
index de5ec2034a35d4f52ef18d83705c22997edc1cc9..0d9fae25cbb3eec6044e2abefe57f774b940b15d 100644
--- a/llvm/test/Analysis/CostModel/ARM/divrem.ll
+++ b/llvm/test/Analysis/CostModel/ARM/divrem.ll
@@ -423,22 +423,22 @@ define void @vi8() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi8'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i8> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i8> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i8> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i8> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i8> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi8'
@@ -538,22 +538,22 @@ define void @vi16() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi16'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i16> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i16> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i16> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i16> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i16> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi16'
@@ -653,22 +653,22 @@ define void @vi32() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi32'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i32> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i32> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i32> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i32> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i32> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi32'
@@ -768,22 +768,22 @@ define void @vi64() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi64'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t1 = sdiv <2 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t2 = udiv <2 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f1 = sdiv <4 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f2 = udiv <4 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %t1 = sdiv <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %t2 = udiv <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %t3 = srem <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %t4 = urem <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f1 = sdiv <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f2 = udiv <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %f3 = srem <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %f4 = urem <4 x i64> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e1 = sdiv <8 x i64> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e2 = udiv <8 x i64> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i64> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %s1 = sdiv <16 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %s2 = udiv <16 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %s1 = sdiv <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %s2 = udiv <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %s3 = srem <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %s4 = urem <16 x i64> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi64'
@@ -873,12 +873,12 @@ define void @vf16() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf16'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = frem <2 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = frem <4 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = frem <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x half> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf16'
@@ -928,12 +928,12 @@ define void @vf32() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf32'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = frem <2 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = frem <4 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = frem <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x float> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf32'
@@ -983,12 +983,12 @@ define void @vf64() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf64'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = frem <2 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = frem <4 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = frem <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x double> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf64'
@@ -1048,22 +1048,22 @@ define void @vi8_2() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi8_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i8> undef, <i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i8> undef, <i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i8> undef, <i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i8> undef, <i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i8> undef, <i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i8> undef, <i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i8> undef, <i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i8> undef, <i8 2, i8 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i8> undef, <i8 2, i8 2, i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i8> undef, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi8_2'
@@ -1163,22 +1163,22 @@ define void @vi16_2() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi16_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i16> undef, <i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i16> undef, <i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i16> undef, <i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i16> undef, <i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i16> undef, <i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i16> undef, <i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i16> undef, <i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i16> undef, <i16 2, i16 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i16> undef, <i16 2, i16 2, i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i16> undef, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi16_2'
@@ -1278,22 +1278,22 @@ define void @vi32_2() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi32_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t1 = sdiv <2 x i32> undef, <i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t2 = udiv <2 x i32> undef, <i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i32> undef, <i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i32> undef, <i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i32> undef, <i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i32> undef, <i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i32> undef, <i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i32> undef, <i32 2, i32 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i32> undef, <i32 2, i32 2, i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e1 = sdiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e2 = udiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s1 = sdiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s2 = udiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i32> undef, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi32_2'
@@ -1393,22 +1393,22 @@ define void @vi64_2() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi64_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t1 = sdiv <2 x i64> undef, <i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t2 = udiv <2 x i64> undef, <i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i64> undef, <i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i64> undef, <i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f1 = sdiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f2 = udiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %t1 = sdiv <2 x i64> undef, <i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %t2 = udiv <2 x i64> undef, <i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %t3 = srem <2 x i64> undef, <i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %t4 = urem <2 x i64> undef, <i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f1 = sdiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f2 = udiv <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %f3 = srem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %f4 = urem <4 x i64> undef, <i64 2, i64 2, i64 2, i64 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e1 = sdiv <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e2 = udiv <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %s1 = sdiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %s2 = udiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s3 = srem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %s4 = urem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %s1 = sdiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %s2 = udiv <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %s3 = srem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %s4 = urem <16 x i64> undef, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi64_2'
@@ -1498,12 +1498,12 @@ define void @vf16_2() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf16_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x half> undef, <half 0xH4000, half 0xH4000>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = frem <2 x half> undef, <half 0xH4000, half 0xH4000>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = frem <4 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = frem <8 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, <half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x half> undef, <half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x half> undef, <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf16_2'
@@ -1553,12 +1553,12 @@ define void @vf32_2() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf32_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x float> undef, <float 2.000000e+00, float 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = frem <2 x float> undef, <float 2.000000e+00, float 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = frem <4 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = frem <8 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x float> undef, <float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x float> undef, <float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x float> undef, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf32_2'
@@ -1608,12 +1608,12 @@ define void @vf64_2() {
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf64_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %1 = fdiv <2 x double> undef, <double 2.000000e+00, double 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %2 = frem <2 x double> undef, <double 2.000000e+00, double 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %3 = fdiv <4 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %4 = frem <4 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %5 = fdiv <8 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %6 = frem <8 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x double> undef, <double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x double> undef, <double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x double> undef, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf64_2'
diff --git a/llvm/test/Analysis/CostModel/ARM/fparith.ll b/llvm/test/Analysis/CostModel/ARM/fparith.ll
index cb3d66edfa20c180dde685481959b23a2adf514e..3403ab25e490c61edd421f10f717874aa22922b5 100644
--- a/llvm/test/Analysis/CostModel/ARM/fparith.ll
+++ b/llvm/test/Analysis/CostModel/ARM/fparith.ll
@@ -61,15 +61,15 @@ define void @f64() {
 
 define void @vf32() {
 ; CHECK-MVE-LABEL: 'vf32'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = fadd <2 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = fsub <2 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = fmul <2 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c4 = fadd <4 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d4 = fsub <4 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e4 = fmul <4 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %c8 = fadd <8 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %d8 = fsub <8 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e8 = fmul <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = fadd <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = fsub <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = fmul <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c4 = fadd <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d4 = fsub <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e4 = fmul <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %c8 = fadd <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %d8 = fsub <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e8 = fmul <8 x float> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVEFP-LABEL: 'vf32'
@@ -98,15 +98,15 @@ define void @vf32() {
 
 define void @vf16() {
 ; CHECK-MVE-LABEL: 'vf16'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = fadd <2 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = fsub <2 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = fmul <2 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c4 = fadd <4 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d4 = fsub <4 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e4 = fmul <4 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %c8 = fadd <8 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %d8 = fsub <8 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e8 = fmul <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = fadd <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = fsub <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = fmul <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c4 = fadd <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d4 = fsub <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e4 = fmul <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %c8 = fadd <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %d8 = fsub <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e8 = fmul <8 x half> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVEFP-LABEL: 'vf16'
@@ -135,27 +135,27 @@ define void @vf16() {
 
 define void @vf64() {
 ; CHECK-MVE-LABEL: 'vf64'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = fadd <2 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = fsub <2 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = fmul <2 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c4 = fadd <4 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d4 = fsub <4 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e4 = fmul <4 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %c8 = fadd <8 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %d8 = fsub <8 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e8 = fmul <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = fadd <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = fsub <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = fmul <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c4 = fadd <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d4 = fsub <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e4 = fmul <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %c8 = fadd <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %d8 = fsub <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e8 = fmul <8 x double> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVEFP-LABEL: 'vf64'
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %c2 = fadd <2 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %d2 = fsub <2 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %e2 = fmul <2 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %c4 = fadd <4 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %d4 = fsub <4 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %e4 = fmul <4 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %c8 = fadd <8 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %d8 = fsub <8 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e8 = fmul <8 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c2 = fadd <2 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d2 = fsub <2 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e2 = fmul <2 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c4 = fadd <4 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d4 = fsub <4 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e4 = fmul <4 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %c8 = fadd <8 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %d8 = fsub <8 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e8 = fmul <8 x double> undef, undef
 ; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %c2 = fadd <2 x double> undef, undef
diff --git a/llvm/test/Analysis/CostModel/ARM/intrinsic-cost-kinds.ll b/llvm/test/Analysis/CostModel/ARM/intrinsic-cost-kinds.ll
index f64393cbe88c1f41d3a1d62608a3009045b561da..b8a6a9569c9a6bf9522637afb7bd32122bedcb5f 100644
--- a/llvm/test/Analysis/CostModel/ARM/intrinsic-cost-kinds.ll
+++ b/llvm/test/Analysis/CostModel/ARM/intrinsic-cost-kinds.ll
@@ -95,7 +95,7 @@ define void @fmuladd(float %a, float %b, float %c, <16 x float> %va, <16 x float
 define void @log2(float %a, <16 x float> %va) {
 ; THRU-LABEL: 'log2'
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %s = call float @llvm.log2.f32(float %a)
-; THRU-NEXT:  Cost Model: Found an estimated cost of 672 for instruction: %v = call <16 x float> @llvm.log2.v16f32(<16 x float> %va)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v = call <16 x float> @llvm.log2.v16f32(<16 x float> %va)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'log2'
@@ -105,12 +105,12 @@ define void @log2(float %a, <16 x float> %va) {
 ;
 ; SIZE-LABEL: 'log2'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s = call float @llvm.log2.f32(float %a)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %v = call <16 x float> @llvm.log2.v16f32(<16 x float> %va)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v = call <16 x float> @llvm.log2.v16f32(<16 x float> %va)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'log2'
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %s = call float @llvm.log2.f32(float %a)
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 672 for instruction: %v = call <16 x float> @llvm.log2.v16f32(<16 x float> %va)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v = call <16 x float> @llvm.log2.v16f32(<16 x float> %va)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %s = call float @llvm.log2.f32(float %a)
@@ -121,7 +121,7 @@ define void @log2(float %a, <16 x float> %va) {
 define void @constrained_fadd(float %a, <16 x float> %va) {
 ; THRU-LABEL: 'constrained_fadd'
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s = call float @llvm.experimental.constrained.fadd.f32(float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.ignore")
-; THRU-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %t = call <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float> %va, <16 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.ignore")
+; THRU-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %t = call <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float> %va, <16 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.ignore")
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'constrained_fadd'
@@ -131,12 +131,12 @@ define void @constrained_fadd(float %a, <16 x float> %va) {
 ;
 ; SIZE-LABEL: 'constrained_fadd'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s = call float @llvm.experimental.constrained.fadd.f32(float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.ignore")
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %t = call <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float> %va, <16 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.ignore")
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %t = call <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float> %va, <16 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.ignore")
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'constrained_fadd'
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s = call float @llvm.experimental.constrained.fadd.f32(float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.ignore")
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %t = call <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float> %va, <16 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.ignore")
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %t = call <16 x float> @llvm.experimental.constrained.fadd.v16f32(<16 x float> %va, <16 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.ignore")
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %s = call float @llvm.experimental.constrained.fadd.f32(float %a, float %a, metadata !"round.dynamic", metadata !"fpexcept.ignore")
@@ -147,7 +147,7 @@ define void @constrained_fadd(float %a, <16 x float> %va) {
 define void @fmaximum(float %a, float %b, <16 x float> %va, <16 x float> %vb) {
 ; THRU-LABEL: 'fmaximum'
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
-; THRU-NEXT:  Cost Model: Found an estimated cost of 928 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'fmaximum'
@@ -157,12 +157,12 @@ define void @fmaximum(float %a, float %b, <16 x float> %va, <16 x float> %vb) {
 ;
 ; SIZE-LABEL: 'fmaximum'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 784 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'fmaximum'
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 928 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %s = call float @llvm.maximum.f32(float %a, float %b)
@@ -225,7 +225,7 @@ define void @ctlz(i32 %a, <16 x i32> %va) {
 define void @fshl(i32 %a, i32 %b, i32 %c, <16 x i32> %va, <16 x i32> %vb, <16 x i32> %vc) {
 ; THRU-LABEL: 'fshl'
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %s = call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
-; THRU-NEXT:  Cost Model: Found an estimated cost of 832 for instruction: %v = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %va, <16 x i32> %vb, <16 x i32> %vc)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %va, <16 x i32> %vb, <16 x i32> %vc)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'fshl'
@@ -235,12 +235,12 @@ define void @fshl(i32 %a, i32 %b, i32 %c, <16 x i32> %va, <16 x i32> %vb, <16 x
 ;
 ; SIZE-LABEL: 'fshl'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s = call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 805 for instruction: %v = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %va, <16 x i32> %vb, <16 x i32> %vc)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 229 for instruction: %v = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %va, <16 x i32> %vb, <16 x i32> %vc)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'fshl'
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %s = call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 826 for instruction: %v = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %va, <16 x i32> %vb, <16 x i32> %vc)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 250 for instruction: %v = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %va, <16 x i32> %vb, <16 x i32> %vc)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %s = call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
@@ -250,7 +250,7 @@ define void @fshl(i32 %a, i32 %b, i32 %c, <16 x i32> %va, <16 x i32> %vb, <16 x
 
 define void @maskedgather(<16 x float*> %va, <16 x i1> %vb, <16 x float> %vc) {
 ; THRU-LABEL: 'maskedgather'
-; THRU-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'maskedgather'
@@ -258,11 +258,11 @@ define void @maskedgather(<16 x float*> %va, <16 x i1> %vb, <16 x float> %vc) {
 ; LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE-LABEL: 'maskedgather'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'maskedgather'
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
@@ -271,7 +271,7 @@ define void @maskedgather(<16 x float*> %va, <16 x i1> %vb, <16 x float> %vc) {
 
 define void @maskedscatter(<16 x float> %va, <16 x float*> %vb, <16 x i1> %vc) {
 ; THRU-LABEL: 'maskedscatter'
-; THRU-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'maskedscatter'
@@ -279,11 +279,11 @@ define void @maskedscatter(<16 x float> %va, <16 x float*> %vb, <16 x i1> %vc) {
 ; LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE-LABEL: 'maskedscatter'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'maskedscatter'
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
@@ -292,7 +292,7 @@ define void @maskedscatter(<16 x float> %va, <16 x float*> %vb, <16 x i1> %vc) {
 
 define void @reduce_fmax(<16 x float> %va) {
 ; THRU-LABEL: 'reduce_fmax'
-; THRU-NEXT:  Cost Model: Found an estimated cost of 696 for instruction: %v = call float @llvm.vector.reduce.fmax.v16f32(<16 x float> %va)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 133 for instruction: %v = call float @llvm.vector.reduce.fmax.v16f32(<16 x float> %va)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'reduce_fmax'
@@ -300,11 +300,11 @@ define void @reduce_fmax(<16 x float> %va) {
 ; LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fmax'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 685 for instruction: %v = call float @llvm.vector.reduce.fmax.v16f32(<16 x float> %va)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 122 for instruction: %v = call float @llvm.vector.reduce.fmax.v16f32(<16 x float> %va)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'reduce_fmax'
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 694 for instruction: %v = call float @llvm.vector.reduce.fmax.v16f32(<16 x float> %va)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 131 for instruction: %v = call float @llvm.vector.reduce.fmax.v16f32(<16 x float> %va)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %v = call float @llvm.vector.reduce.fmax.v16f32(<16 x float> %va)
diff --git a/llvm/test/Analysis/CostModel/ARM/load_store.ll b/llvm/test/Analysis/CostModel/ARM/load_store.ll
index 2ca4acda0fc2d0419d6e0df85c731142139eb2fb..52f6e3cccac6415b2a7e721e2b14f2d6f31ed22f 100644
--- a/llvm/test/Analysis/CostModel/ARM/load_store.ll
+++ b/llvm/test/Analysis/CostModel/ARM/load_store.ll
@@ -69,16 +69,16 @@ define void @stores() {
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store i128 undef, i128* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store float undef, float* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store double undef, double* undef, align 4
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x i32> undef, <2 x i32>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: store <2 x i16> undef, <2 x i16>* undef, align 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: store <2 x i32> undef, <2 x i32>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> undef, <8 x i16>* undef, align 2
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> undef, <16 x i8>* undef, align 1
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x float> undef, <4 x float>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x double> undef, <4 x double>* undef, align 4
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: store <2 x float> undef, <2 x float>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: store <2 x float> undef, <2 x float>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x double> undef, <2 x double>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i64> undef, <2 x i64>* undef, align 1
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> undef, <4 x i32>* undef, align 1
@@ -256,16 +256,16 @@ define void @loads() {
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = load i128, i128* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load float, float* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = load double, double* undef, align 4
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %8 = load <2 x i8>, <2 x i8>* undef, align 1
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %9 = load <2 x i16>, <2 x i16>* undef, align 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %10 = load <2 x i32>, <2 x i32>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %8 = load <2 x i8>, <2 x i8>* undef, align 1
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %9 = load <2 x i16>, <2 x i16>* undef, align 2
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %10 = load <2 x i32>, <2 x i32>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %11 = load <2 x i64>, <2 x i64>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %12 = load <4 x i32>, <4 x i32>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %13 = load <8 x i16>, <8 x i16>* undef, align 2
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = load <16 x i8>, <16 x i8>* undef, align 1
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = load <4 x float>, <4 x float>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = load <4 x double>, <4 x double>* undef, align 4
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %17 = load <2 x float>, <2 x float>* undef, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %17 = load <2 x float>, <2 x float>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %18 = load <2 x double>, <2 x double>* undef, align 4
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %19 = load <2 x i64>, <2 x i64>* undef, align 1
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = load <4 x i32>, <4 x i32>* undef, align 1
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-abs.ll b/llvm/test/Analysis/CostModel/ARM/mve-abs.ll
index a69531c0379a405d33b3708880990a86f6f8aa3a..ace2d1bee070cf6116fcc6bb44daddbc31fb2507 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-abs.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-abs.ll
@@ -31,22 +31,22 @@ declare <64 x i8>  @llvm.abs.v64i8(<64 x i8>, i1)
 define i32 @abs(i32 %arg) {
 ; MVE-RECIP-LABEL: 'abs'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I64 = call i64 @llvm.abs.i64(i64 undef, i1 false)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 448 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.abs.i32(i32 undef, i1 false)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.abs.i16(i16 undef, i1 false)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = call <4 x i16> @llvm.abs.v4i16(<4 x i16> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.abs.i8(i8 undef, i1 false)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I8 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V2I8 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I8 = call <4 x i8> @llvm.abs.v4i8(<4 x i8> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I8 = call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 false)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 false)
@@ -56,22 +56,22 @@ define i32 @abs(i32 %arg) {
 ;
 ; MVE-SIZE-LABEL: 'abs'
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I64 = call i64 @llvm.abs.i64(i64 undef, i1 false)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.abs.i32(i32 undef, i1 false)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.abs.i16(i16 undef, i1 false)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.abs.v4i16(<4 x i16> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.abs.i8(i8 undef, i1 false)
-; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I8 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
+; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I8 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I8 = call <4 x i8> @llvm.abs.v4i8(<4 x i8> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 false)
 ; MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 false)
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll b/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll
index ca53d85e556a346c0b9c5ae292e5b0f63811d039..c2af5e1cacb53b70da8bbfdacbd76909972dea85 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll
@@ -4,21 +4,21 @@
 
 define void @icmp() {
 ; CHECK-LABEL: 'icmp'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2i8 = icmp slt <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v2i8 = icmp slt <2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = icmp slt <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = icmp slt <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8 = icmp slt <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1028 for instruction: %v32i8 = icmp slt <32 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2i16 = icmp slt <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %v32i8 = icmp slt <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v2i16 = icmp slt <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = icmp slt <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16 = icmp slt <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %v16i16 = icmp slt <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2i32 = icmp slt <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v16i16 = icmp slt <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v2i32 = icmp slt <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32 = icmp slt <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v8i32 = icmp slt <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %v16i32 = icmp slt <16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2i64 = icmp slt <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v4i64 = icmp slt <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v8i32 = icmp slt <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %v16i32 = icmp slt <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v2i64 = icmp slt <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %v4i64 = icmp slt <4 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %v8i64 = icmp slt <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -47,31 +47,31 @@ define void @icmp() {
 
 define void @fcmp() {
 ; CHECK-MVE-LABEL: 'fcmp'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2f16 = fcmp olt <2 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v4f16 = fcmp olt <4 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v8f16 = fcmp olt <8 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2f32 = fcmp olt <2 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v4f32 = fcmp olt <4 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v8f32 = fcmp olt <8 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %v16f32 = fcmp olt <16 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2f64 = fcmp olt <2 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v4f64 = fcmp olt <4 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v8f64 = fcmp olt <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2f16 = fcmp olt <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v4f16 = fcmp olt <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v8f16 = fcmp olt <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2f32 = fcmp olt <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v4f32 = fcmp olt <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v8f32 = fcmp olt <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v16f32 = fcmp olt <16 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2f64 = fcmp olt <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v4f64 = fcmp olt <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v8f64 = fcmp olt <8 x double> undef, undef
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-MVEFP-LABEL: 'fcmp'
 ; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp olt <2 x half> undef, undef
 ; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = fcmp olt <4 x half> undef, undef
 ; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = fcmp olt <8 x half> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef
 ; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fcmp olt <2 x float> undef, undef
 ; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = fcmp olt <4 x float> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v8f32 = fcmp olt <8 x float> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %v16f32 = fcmp olt <16 x float> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2f64 = fcmp olt <2 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v4f64 = fcmp olt <4 x double> undef, undef
-; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %v8f64 = fcmp olt <8 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v8f32 = fcmp olt <8 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %v16f32 = fcmp olt <16 x float> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2f64 = fcmp olt <2 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4f64 = fcmp olt <4 x double> undef, undef
+; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8f64 = fcmp olt <8 x double> undef, undef
 ; CHECK-MVEFP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2f16 = fcmp olt <2 x half> undef, undef
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-gather-scatter-cost.ll b/llvm/test/Analysis/CostModel/ARM/mve-gather-scatter-cost.ll
index fe75e5087ec4a1230a9efef6644ba665939c00a0..c368991faea65384908dff274e76cceadd63cd53 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-gather-scatter-cost.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-gather-scatter-cost.ll
@@ -3,32 +3,32 @@
 
 define i32 @masked_gather() {
 ; CHECK-LABEL: 'masked_gather'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> undef, i32 4, <4 x i1> undef, <4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> undef, i32 4, <2 x i1> undef, <2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> undef, i32 4, <16 x i1> undef, <16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> undef, i32 4, <4 x i1> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> undef, i32 4, <2 x i1> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> undef, i32 4, <16 x i1> undef, <16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %V16F16 = call <16 x half> @llvm.masked.gather.v16f16.v16p0f16(<16 x half*> undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V8F16 = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4F16 = call <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*> undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F16 = call <2 x half> @llvm.masked.gather.v2f16.v2p0f16(<2 x half*> undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*> undef, i32 4, <4 x i1> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> undef, i32 4, <2 x i1> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> undef, i32 4, <16 x i1> undef, <16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> undef, i32 4, <8 x i1> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16F16 = call <16 x half> @llvm.masked.gather.v16f16.v16p0f16(<16 x half*> undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8F16 = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4F16 = call <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*> undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F16 = call <2 x half> @llvm.masked.gather.v2f16.v2p0f16(<2 x half*> undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*> undef, i32 4, <4 x i1> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> undef, i32 4, <2 x i1> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> undef, i32 4, <16 x i1> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> undef, i32 4, <8 x i1> undef, <8 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> undef, i32 4, <4 x i1> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0i16(<16 x i16*> undef, i32 2, <16 x i1> undef, <16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0i16(<16 x i16*> undef, i32 2, <16 x i1> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> undef, i32 2, <8 x i1> undef, <8 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I16 = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2112 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> undef, i32 1, <32 x i1> undef, <32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I16 = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> undef, i32 1, <32 x i1> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> undef, i32 1, <8 x i1> undef, <8 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V4I8 = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I8 = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I32p = call <4 x i32*> @llvm.masked.gather.v4p0i32.v4p0p0i32(<4 x i32**> undef, i32 4, <4 x i1> undef, <4 x i32*> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I8 = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32p = call <4 x i32*> @llvm.masked.gather.v4p0i32.v4p0p0i32(<4 x i32**> undef, i32 4, <4 x i1> undef, <4 x i32*> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> undef, i32 4, <4 x i1> undef, <4 x double> undef)
@@ -70,31 +70,31 @@ define i32 @masked_gather() {
 
 define i32 @masked_scatter() {
 ; CHECK-LABEL: 'masked_scatter'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> undef, <4 x double*> undef, i32 4, <4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> undef, <2 x double*> undef, i32 4, <2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> undef, <16 x float*> undef, i32 4, <16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> undef, <8 x float*> undef, i32 4, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> undef, <4 x double*> undef, i32 4, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> undef, <2 x double*> undef, i32 4, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> undef, <16 x float*> undef, i32 4, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> undef, <8 x float*> undef, i32 4, <8 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> undef, <4 x float*> undef, i32 4, <4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> undef, <2 x float*> undef, i32 4, <2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: call void @llvm.masked.scatter.v16f16.v16p0f16(<16 x half> undef, <16 x half*> undef, i32 2, <16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> undef, <8 x half*> undef, i32 2, <8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half> undef, <4 x half*> undef, i32 2, <4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2f16.v2p0f16(<2 x half> undef, <2 x half*> undef, i32 2, <2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i64.v4p0i64(<4 x i64> undef, <4 x i64*> undef, i32 4, <4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> undef, <2 x i64*> undef, i32 4, <2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> undef, <16 x i32*> undef, i32 4, <16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> undef, <8 x i32*> undef, i32 4, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> undef, <2 x float*> undef, i32 4, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.scatter.v16f16.v16p0f16(<16 x half> undef, <16 x half*> undef, i32 2, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> undef, <8 x half*> undef, i32 2, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half> undef, <4 x half*> undef, i32 2, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f16.v2p0f16(<2 x half> undef, <2 x half*> undef, i32 2, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.masked.scatter.v4i64.v4p0i64(<4 x i64> undef, <4 x i64*> undef, i32 4, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> undef, <2 x i64*> undef, i32 4, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> undef, <16 x i32*> undef, i32 4, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> undef, <8 x i32*> undef, i32 4, <8 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> undef, <4 x i32*> undef, i32 4, <4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> undef, <2 x i32*> undef, i32 4, <2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 544 for instruction: call void @llvm.masked.scatter.v16i16.v16p0i16(<16 x i16> undef, <16 x i16*> undef, i32 2, <16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> undef, <8 x i16*> undef, i32 2, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> undef, <2 x i32*> undef, i32 4, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.v16i16.v16p0i16(<16 x i16> undef, <16 x i16*> undef, i32 2, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> undef, <8 x i16*> undef, i32 2, <8 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16> undef, <4 x i16*> undef, i32 2, <4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i16.v2p0i16(<2 x i16> undef, <2 x i16*> undef, i32 2, <2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2112 for instruction: call void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8> undef, <32 x i8*> undef, i32 1, <32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> undef, <16 x i8*> undef, i32 1, <16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8i8.v8p0i8(<8 x i8> undef, <8 x i8*> undef, i32 1, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v2i16.v2p0i16(<2 x i16> undef, <2 x i16*> undef, i32 2, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: call void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8> undef, <32 x i8*> undef, i32 1, <32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> undef, <16 x i8*> undef, i32 1, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.masked.scatter.v8i8.v8p0i8(<8 x i8> undef, <8 x i8*> undef, i32 1, <8 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> undef, <4 x i8*> undef, i32 1, <4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8> undef, <2 x i8*> undef, i32 1, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8> undef, <2 x i8*> undef, i32 1, <2 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> undef, <4 x double*> undef, i32 4, <4 x i1> undef)
@@ -205,8 +205,8 @@ define void @gep_v4f32(float* %base, i16* %base16, i8* %base8, <4 x i32> %ind32,
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res3 = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep3, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> %res3, <4 x float*> %gep3, i32 4, <4 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %gepu = getelementptr float, float* %base, <4 x i32> %ind32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %resu = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gepu, i32 1, <4 x i1> %mask, <4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> %resu, <4 x float*> %gepu, i32 1, <4 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %resu = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gepu, i32 1, <4 x i1> %mask, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> %resu, <4 x float*> %gepu, i32 1, <4 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepos = getelementptr i8, i8* %base8, <4 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %geposb = bitcast <4 x i8*> %gepos to <4 x float*>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %resos = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %geposb, i32 4, <4 x i1> %mask, <4 x float> undef)
@@ -336,26 +336,26 @@ define void @gep_v4i8(i8* %base, <4 x i8> %ind8, <4 x i1> %mask)  {
 define void @gep_v8i16(i16* %base, i8* %base8, i32* %base32, <8 x i32> %ind32, <8 x i16> %ind16, <8 x i8> %ind8, <8 x i1> %mask)  {
 ; CHECK-LABEL: 'gep_v8i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep1 = getelementptr i16, i16* %base, <8 x i32> %ind32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %res1 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gep1, i32 2, <8 x i1> %mask, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %res1, <8 x i16*> %gep1, i32 2, <8 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %res1 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gep1, i32 2, <8 x i1> %mask, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %res1, <8 x i16*> %gep1, i32 2, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %indzext = zext <8 x i16> %ind16 to <8 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep2 = getelementptr i16, i16* %base, <8 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res2 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gep2, i32 2, <8 x i1> %mask, <8 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %res2, <8 x i16*> %gep2, i32 2, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %indsext = sext <8 x i16> %ind16 to <8 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep3 = getelementptr i16, i16* %base, <8 x i32> %indsext
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %res3 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gep3, i32 2, <8 x i1> %mask, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %res3, <8 x i16*> %gep3, i32 2, <8 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %resu = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gep2, i32 1, <8 x i1> %mask, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %resu, <8 x i16*> %gep2, i32 1, <8 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %res3 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gep3, i32 2, <8 x i1> %mask, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %res3, <8 x i16*> %gep3, i32 2, <8 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %resu = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gep2, i32 1, <8 x i1> %mask, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %resu, <8 x i16*> %gep2, i32 1, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepos = getelementptr i8, i8* %base8, <8 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %geposb = bitcast <8 x i8*> %gepos to <8 x i16*>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %resos = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %geposb, i32 2, <8 x i1> %mask, <8 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %resos, <8 x i16*> %geposb, i32 2, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepbs = getelementptr i32, i32* %base32, <8 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepbsb = bitcast <8 x i32*> %gepbs to <8 x i16*>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %resbs = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gepbsb, i32 2, <8 x i1> %mask, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %resbs, <8 x i16*> %gepbsb, i32 2, <8 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %resbs = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %gepbsb, i32 2, <8 x i1> %mask, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %resbs, <8 x i16*> %gepbsb, i32 2, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %indzext4 = zext <8 x i16> %ind16 to <8 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep4 = getelementptr i16, i16* %base, <8 x i32> %indzext4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %indtrunc = trunc <8 x i32> %ind32 to <8 x i16>
@@ -417,26 +417,26 @@ define void @gep_v8i16(i16* %base, i8* %base8, i32* %base32, <8 x i32> %ind32, <
 define void @gep_v8f16(half* %base, i8* %base8, i32* %base32, <8 x i32> %ind32, <8 x i16> %ind16, <8 x i1> %mask)  {
 ; CHECK-LABEL: 'gep_v8f16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %gep1 = getelementptr half, half* %base, <8 x i32> %ind32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %res1 = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gep1, i32 2, <8 x i1> %mask, <8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %res1, <8 x half*> %gep1, i32 2, <8 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res1 = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gep1, i32 2, <8 x i1> %mask, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %res1, <8 x half*> %gep1, i32 2, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %indzext = zext <8 x i16> %ind16 to <8 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %gep2 = getelementptr half, half* %base, <8 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res2 = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gep2, i32 2, <8 x i1> %mask, <8 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %res2, <8 x half*> %gep2, i32 2, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %indsext = sext <8 x i16> %ind16 to <8 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %gep3 = getelementptr half, half* %base, <8 x i32> %indsext
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %res3 = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gep3, i32 2, <8 x i1> %mask, <8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %res3, <8 x half*> %gep3, i32 2, <8 x i1> %mask)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %resu = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gep2, i32 1, <8 x i1> %mask, <8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %resu, <8 x half*> %gep2, i32 1, <8 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res3 = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gep3, i32 2, <8 x i1> %mask, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %res3, <8 x half*> %gep3, i32 2, <8 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %resu = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gep2, i32 1, <8 x i1> %mask, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %resu, <8 x half*> %gep2, i32 1, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepos = getelementptr i8, i8* %base8, <8 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %geposb = bitcast <8 x i8*> %gepos to <8 x half*>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %resos = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %geposb, i32 2, <8 x i1> %mask, <8 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %resos, <8 x half*> %geposb, i32 2, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepbs = getelementptr i32, i32* %base32, <8 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepbsb = bitcast <8 x i32*> %gepbs to <8 x half*>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %resbs = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gepbsb, i32 2, <8 x i1> %mask, <8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %resbs, <8 x half*> %gepbsb, i32 2, <8 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %resbs = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %gepbsb, i32 2, <8 x i1> %mask, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %resbs, <8 x half*> %gepbsb, i32 2, <8 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   ; no offset ext
@@ -509,20 +509,20 @@ define void @gep_v8i8(i8* %base, <8 x i8> %ind8, <8 x i1> %mask)  {
 define void @gep_v16i8(i8* %base, i16* %base16, <16 x i8> %ind8, <16 x i32> %ind32, <16 x i1> %mask)  {
 ; CHECK-LABEL: 'gep_v16i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep1 = getelementptr i8, i8* %base, <16 x i32> %ind32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %res1 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gep1, i32 1, <16 x i1> %mask, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %res1, <16 x i8*> %gep1, i32 2, <16 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %res1 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gep1, i32 1, <16 x i1> %mask, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %res1, <16 x i8*> %gep1, i32 2, <16 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %indzext = zext <16 x i8> %ind8 to <16 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep2 = getelementptr i8, i8* %base, <16 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %res2 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gep2, i32 2, <16 x i1> %mask, <16 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %res2, <16 x i8*> %gep2, i32 2, <16 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %indsext = sext <16 x i8> %ind8 to <16 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep3 = getelementptr i8, i8* %base, <16 x i32> %indsext
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %res3 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gep3, i32 2, <16 x i1> %mask, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %res3, <16 x i8*> %gep3, i32 2, <16 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %res3 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gep3, i32 2, <16 x i1> %mask, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %res3, <16 x i8*> %gep3, i32 2, <16 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepbs = getelementptr i16, i16* %base16, <16 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepbsb = bitcast <16 x i16*> %gepbs to <16 x i8*>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %resbs = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gepbsb, i32 2, <16 x i1> %mask, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %resbs, <16 x i8*> %gepbsb, i32 2, <16 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %resbs = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gepbsb, i32 2, <16 x i1> %mask, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %resbs, <16 x i8*> %gepbsb, i32 2, <16 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %indzext4 = zext <16 x i8> %ind8 to <16 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep4 = getelementptr i8, i8* %base, <16 x i32> %indzext
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %indtrunc = trunc <16 x i32> %ind32 to <16 x i8>
@@ -564,8 +564,8 @@ define void @gep_v16i8(i8* %base, i16* %base16, <16 x i8> %ind8, <16 x i32> %ind
 define void @gep_v16i8p(<16 x i8*> %base, i32 %off, <16 x i1> %mask)  {
 ; CHECK-LABEL: 'gep_v16i8p'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gepbs = getelementptr i8, <16 x i8*> %base, i32 %off
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: %resbs = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gepbs, i32 2, <16 x i1> %mask, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 528 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %resbs, <16 x i8*> %gepbs, i32 2, <16 x i1> %mask)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %resbs = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %gepbs, i32 2, <16 x i1> %mask, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %resbs, <16 x i8*> %gepbs, i32 2, <16 x i1> %mask)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %gepbs = getelementptr i8, <16 x i8*> %base, i32 %off
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll b/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll
index ede3bdda30ca58e25dcd4c3437ca05a185405cf4..b075ba754904627aaea1bf598a01e20694624b08 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll
@@ -33,22 +33,22 @@ declare <64 x i8>  @llvm.smin.v64i8(<64 x i8>, <64 x i8>)
 define i32 @smin(i32 %arg) {
 ; MVE-RECIP-LABEL: 'smin'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 152 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 304 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I8 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I8 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I8 = call <4 x i8> @llvm.smin.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I8 = call <8 x i8> @llvm.smin.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -140,22 +140,22 @@ declare <64 x i8>  @llvm.smax.v64i8(<64 x i8>, <64 x i8>)
 define i32 @smax(i32 %arg) {
 ; MVE-RECIP-LABEL: 'smax'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 152 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 304 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I8 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I8 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I8 = call <4 x i8> @llvm.smax.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I8 = call <8 x i8> @llvm.smax.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -248,22 +248,22 @@ declare <64 x i8>  @llvm.umin.v64i8(<64 x i8>, <64 x i8>)
 define i32 @umin(i32 %arg) {
 ; MVE-RECIP-LABEL: 'umin'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 152 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 304 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I8 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I8 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I8 = call <4 x i8> @llvm.umin.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I8 = call <8 x i8> @llvm.umin.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -355,22 +355,22 @@ declare <64 x i8>  @llvm.umax.v64i8(<64 x i8>, <64 x i8>)
 define i32 @sub(i32 %arg) {
 ; MVE-RECIP-LABEL: 'sub'
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 152 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 304 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2I8 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V2I8 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I8 = call <4 x i8> @llvm.umax.v4i8(<4 x i8> undef, <4 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I8 = call <8 x i8> @llvm.umax.v8i8(<8 x i8> undef, <8 x i8> undef)
 ; MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -455,45 +455,45 @@ declare <32 x half> @llvm.minnum.v32f16(<32 x half>, <32 x half>)
 define float @minnum(float %arg) {
 ; MVEI-RECIP-LABEL: 'minnum'
 ; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = call double @llvm.minnum.f64(double undef, double undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8F64 = call <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2F64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4F64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8F64 = call <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
 ; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F32 = call float @llvm.minnum.f32(float undef, float undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F32 = call <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F32 = call <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8F32 = call <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %V16F32 = call <16 x float> @llvm.minnum.v16f32(<16 x float> undef, <16 x float> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2F32 = call <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4F32 = call <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8F32 = call <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16F32 = call <16 x float> @llvm.minnum.v16f32(<16 x float> undef, <16 x float> undef)
 ; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F16 = call half @llvm.minnum.f16(half undef, half undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8F16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %V16F16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 1344 for instruction: %V32F16 = call <32 x half> @llvm.minnum.v32f16(<32 x half> undef, <32 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2F16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4F16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8F16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16F16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 352 for instruction: %V32F16 = call <32 x half> @llvm.minnum.v32f16(<32 x half> undef, <32 x half> undef)
 ; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float undef
 ;
 ; MVEI-SIZE-LABEL: 'minnum'
 ; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.minnum.f64(double undef, double undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8F64 = call <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = call <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
 ; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.minnum.f32(float undef, float undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8F32 = call <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %V16F32 = call <16 x float> @llvm.minnum.v16f32(<16 x float> undef, <16 x float> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16F32 = call <16 x float> @llvm.minnum.v16f32(<16 x float> undef, <16 x float> undef)
 ; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call half @llvm.minnum.f16(half undef, half undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8F16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %V16F16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1056 for instruction: %V32F16 = call <32 x half> @llvm.minnum.v32f16(<32 x half> undef, <32 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16F16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32F16 = call <32 x half> @llvm.minnum.v32f16(<32 x half> undef, <32 x half> undef)
 ; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float undef
 ;
 ; MVEF-RECIP-LABEL: 'minnum'
 ; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = call double @llvm.minnum.f64(double undef, double undef)
-; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
-; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
-; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8F64 = call <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
+; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2F64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
+; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4F64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
+; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8F64 = call <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
 ; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.minnum.f32(float undef, float undef)
 ; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = call <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
 ; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
@@ -509,9 +509,9 @@ define float @minnum(float %arg) {
 ;
 ; MVEF-SIZE-LABEL: 'minnum'
 ; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.minnum.f64(double undef, double undef)
-; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
-; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
-; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8F64 = call <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
+; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
+; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
+; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = call <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
 ; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.minnum.f32(float undef, float undef)
 ; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
 ; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
@@ -567,45 +567,45 @@ declare <32 x half> @llvm.maxnum.v32f16(<32 x half>, <32 x half>)
 define float @maxnum(float %arg) {
 ; MVEI-RECIP-LABEL: 'maxnum'
 ; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = call double @llvm.maxnum.f64(double undef, double undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8F64 = call <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2F64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4F64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8F64 = call <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
 ; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F32 = call float @llvm.maxnum.f32(float undef, float undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F32 = call <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F32 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8F32 = call <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %V16F32 = call <16 x float> @llvm.maxnum.v16f32(<16 x float> undef, <16 x float> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2F32 = call <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4F32 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8F32 = call <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16F32 = call <16 x float> @llvm.maxnum.v16f32(<16 x float> undef, <16 x float> undef)
 ; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F16 = call half @llvm.maxnum.f16(half undef, half undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8F16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %V16F16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
-; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 1344 for instruction: %V32F16 = call <32 x half> @llvm.maxnum.v32f16(<32 x half> undef, <32 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2F16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4F16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8F16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16F16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
+; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 352 for instruction: %V32F16 = call <32 x half> @llvm.maxnum.v32f16(<32 x half> undef, <32 x half> undef)
 ; MVEI-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float undef
 ;
 ; MVEI-SIZE-LABEL: 'maxnum'
 ; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.maxnum.f64(double undef, double undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8F64 = call <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = call <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
 ; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.maxnum.f32(float undef, float undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8F32 = call <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %V16F32 = call <16 x float> @llvm.maxnum.v16f32(<16 x float> undef, <16 x float> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F32 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F32 = call <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16F32 = call <16 x float> @llvm.maxnum.v16f32(<16 x float> undef, <16 x float> undef)
 ; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call half @llvm.maxnum.f16(half undef, half undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8F16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 272 for instruction: %V16F16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
-; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1056 for instruction: %V32F16 = call <32 x half> @llvm.maxnum.v32f16(<32 x half> undef, <32 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16F16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
+; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32F16 = call <32 x half> @llvm.maxnum.v32f16(<32 x half> undef, <32 x half> undef)
 ; MVEI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float undef
 ;
 ; MVEF-RECIP-LABEL: 'maxnum'
 ; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = call double @llvm.maxnum.f64(double undef, double undef)
-; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
-; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
-; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V8F64 = call <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
+; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2F64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
+; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4F64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
+; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8F64 = call <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
 ; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.maxnum.f32(float undef, float undef)
 ; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = call <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
 ; MVEF-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
@@ -621,9 +621,9 @@ define float @maxnum(float %arg) {
 ;
 ; MVEF-SIZE-LABEL: 'maxnum'
 ; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.maxnum.f64(double undef, double undef)
-; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
-; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
-; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V8F64 = call <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
+; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
+; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
+; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = call <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
 ; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.maxnum.f32(float undef, float undef)
 ; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
 ; MVEF-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-vecreduce-add.ll b/llvm/test/Analysis/CostModel/ARM/mve-vecreduce-add.ll
index 8b59de716fcc73d47b7369f9bc6290c56c199934..c670d6f6baeeab97aeab4e62e40761820fa78555 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-vecreduce-add.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-vecreduce-add.ll
@@ -3,8 +3,8 @@
 
 define void @add_i8() {
 ; CHECK-LABEL: 'add_i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0 = call i8 @llvm.vector.reduce.add.v1i8(<1 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0 = call i8 @llvm.vector.reduce.add.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a3 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a4 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
@@ -26,13 +26,13 @@ define void @add_i8() {
 define void @add_i16() {
 ; CHECK-LABEL: 'add_i16'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0za = zext <1 x i8> undef to <1 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0z = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a0za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0z = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a0za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0sa = sext <1 x i8> undef to <1 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0s = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a0sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0s = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a0sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a1za = zext <2 x i8> undef to <2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1z = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a1za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1z = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a1za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a1sa = sext <2 x i8> undef to <2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1s = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a1sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1s = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a1sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2za = zext <4 x i8> undef to <4 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2z = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a2za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2sa = sext <4 x i8> undef to <4 x i16>
@@ -45,8 +45,8 @@ define void @add_i16() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a4z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %a4za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %a4sa = sext <16 x i8> undef to <16 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a4s = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %a4sa)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5 = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a6 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a5 = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a6 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a7 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a8 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a9 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> undef)
@@ -98,13 +98,13 @@ define void @add_i16() {
 define void @add_i32() {
 ; CHECK-LABEL: 'add_i32'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0za = zext <1 x i8> undef to <1 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0z = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a0za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0z = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a0za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0sa = sext <1 x i8> undef to <1 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0s = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a0sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0s = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a0sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a1za = zext <2 x i8> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1z = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a1za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1z = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a1za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a1sa = sext <2 x i8> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1s = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a1sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1s = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a1sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a2za = zext <4 x i8> undef to <4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a2za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a2sa = sext <4 x i8> undef to <4 x i32>
@@ -118,13 +118,13 @@ define void @add_i32() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %a4sa = sext <16 x i8> undef to <16 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a4s = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a4sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a5za = zext <1 x i16> undef to <1 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5z = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a5za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a5z = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a5za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a5sa = sext <1 x i16> undef to <1 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5s = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a5sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a5s = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a5sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a6za = zext <2 x i16> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a6z = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a6za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a6z = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a6za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a6sa = sext <2 x i16> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a6s = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a6sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a6s = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a6sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a7za = zext <4 x i16> undef to <4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a7z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a7za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a7sa = sext <4 x i16> undef to <4 x i32>
@@ -137,8 +137,8 @@ define void @add_i32() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a9z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a9za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %a9sa = sext <16 x i16> undef to <16 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a9s = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a9sa)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a10 = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a11 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a10 = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a11 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a12 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a13 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a14 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
@@ -219,71 +219,71 @@ define void @add_i32() {
 
 define void @add_i64() {
 ; CHECK-LABEL: 'add_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a0za = zext <1 x i8> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a0za)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a0sa = sext <1 x i8> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a0sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a0za = zext <1 x i8> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a0z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a0za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a0sa = sext <1 x i8> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a0s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a0sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a1za = zext <2 x i8> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a1z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a1z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %a1sa = sext <2 x i8> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a1s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a1s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %a2za = zext <4 x i8> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a2z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a2za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a2z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a2za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %a2sa = sext <4 x i8> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a2s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a2sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a2s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a2sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %a3za = zext <8 x i8> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a3z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a3za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a3z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a3za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %a3sa = sext <8 x i8> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a3s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a3sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a3s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a3sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %a4za = zext <16 x i8> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a4z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a4za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a4z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a4za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %a4sa = sext <16 x i8> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a4s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a4sa)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a5za = zext <1 x i16> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a5za)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a5sa = sext <1 x i16> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a5sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a4s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a4sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a5za = zext <1 x i16> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a5z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a5za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %a5sa = sext <1 x i16> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a5s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a5sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a6za = zext <2 x i16> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a6z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a6za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a6z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a6za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %a6sa = sext <2 x i16> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a6s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a6sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a6s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a6sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %a7za = zext <4 x i16> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a7z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a7za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a7z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a7za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %a7sa = sext <4 x i16> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a7s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a7sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a7s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a7sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %a8za = zext <8 x i16> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a8z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a8za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a8z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a8za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %a8sa = sext <8 x i16> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a8s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a8sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a8s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a8sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 296 for instruction: %a9za = zext <16 x i16> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a9z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a9za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a9z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a9za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1320 for instruction: %a9sa = sext <16 x i16> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a9s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a9sa)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a10za = zext <1 x i32> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a10z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a10za)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a10sa = sext <1 x i32> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a10s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a10sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a9s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a9sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a10za = zext <1 x i32> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a10z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a10za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a10sa = sext <1 x i32> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a10s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a10sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a11za = zext <2 x i32> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a11z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a11za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a11z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a11za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a11sa = sext <2 x i32> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a11s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a11sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a11s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a11sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %a12za = zext <4 x i32> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a12z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a12za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a12z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a12za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a12sa = sext <4 x i32> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a12s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a12sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a12s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a12sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %a13za = zext <8 x i32> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a13z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a13za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a13z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a13za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %a13sa = sext <8 x i32> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a13s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a13sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a13s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a13sa)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %a14za = zext <16 x i32> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a14z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a14za)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a14z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a14za)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1056 for instruction: %a14sa = sext <16 x i32> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a14s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a14sa)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a15 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a16 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a17 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a18 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a19 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a14s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a14sa)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a15 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a16 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a17 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a18 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a19 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %a0za = zext <1 x i8> undef to <1 x i64>
@@ -392,9 +392,9 @@ define void @add_i64() {
 define void @mla_i8() {
 ; CHECK-LABEL: 'mla_i8'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0m = mul <1 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0 = call i8 @llvm.vector.reduce.add.v1i8(<1 x i8> %a0m)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a1m = mul <2 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> %a1m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0 = call i8 @llvm.vector.reduce.add.v1i8(<1 x i8> %a0m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %a1m = mul <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> %a1m)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2m = mul <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %a2m)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a3m = mul <8 x i8> undef, undef
@@ -426,19 +426,19 @@ define void @mla_i16() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0za = zext <1 x i8> undef to <1 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0zb = zext <1 x i8> undef to <1 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0zm = mul <1 x i16> %a0za, %a0zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0z = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a0zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0z = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a0zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0sa = sext <1 x i8> undef to <1 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0sb = sext <1 x i8> undef to <1 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0sm = mul <1 x i16> %a0sa, %a0sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0s = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a0sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0s = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a0sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a1za = zext <2 x i8> undef to <2 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a1zb = zext <2 x i8> undef to <2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %a1zm = mul <2 x i16> %a1za, %a1zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1z = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a1zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %a1zm = mul <2 x i16> %a1za, %a1zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1z = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a1zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a1sa = sext <2 x i8> undef to <2 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a1sb = sext <2 x i8> undef to <2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %a1sm = mul <2 x i16> %a1sa, %a1sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1s = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a1sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %a1sm = mul <2 x i16> %a1sa, %a1sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1s = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a1sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2za = zext <4 x i8> undef to <4 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2zb = zext <4 x i8> undef to <4 x i16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2zm = mul <4 x i16> %a2za, %a2zb
@@ -464,9 +464,9 @@ define void @mla_i16() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a4sm = mul <16 x i16> %a4sa, %a4sb
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a4s = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %a4sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a5m = mul <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5 = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a5m)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a6m = mul <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a6 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a6m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a5 = call i16 @llvm.vector.reduce.add.v1i16(<1 x i16> %a5m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %a6m = mul <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a6 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %a6m)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a7m = mul <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a7 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a7m)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a8m = mul <8 x i16> undef, undef
@@ -548,19 +548,19 @@ define void @mla_i32() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0za = zext <1 x i8> undef to <1 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0zb = zext <1 x i8> undef to <1 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0zm = mul <1 x i32> %a0za, %a0zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0z = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a0zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0z = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a0zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0sa = sext <1 x i8> undef to <1 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0sb = sext <1 x i8> undef to <1 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a0sm = mul <1 x i32> %a0sa, %a0sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0s = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a0sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0s = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a0sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a1za = zext <2 x i8> undef to <2 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a1zb = zext <2 x i8> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %a1zm = mul <2 x i32> %a1za, %a1zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1z = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a1zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %a1zm = mul <2 x i32> %a1za, %a1zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1z = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a1zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a1sa = sext <2 x i8> undef to <2 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a1sb = sext <2 x i8> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %a1sm = mul <2 x i32> %a1sa, %a1sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a1s = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a1sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %a1sm = mul <2 x i32> %a1sa, %a1sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a1s = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a1sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a2za = zext <4 x i8> undef to <4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a2zb = zext <4 x i8> undef to <4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a2zm = mul <4 x i32> %a2za, %a2zb
@@ -588,19 +588,19 @@ define void @mla_i32() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a5za = zext <1 x i16> undef to <1 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a5zb = zext <1 x i16> undef to <1 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a5zm = mul <1 x i32> %a5za, %a5zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5z = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a5zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a5z = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a5zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a5sa = sext <1 x i16> undef to <1 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a5sb = sext <1 x i16> undef to <1 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a5sm = mul <1 x i32> %a5sa, %a5sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5s = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a5sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a5s = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a5sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a6za = zext <2 x i16> undef to <2 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a6zb = zext <2 x i16> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %a6zm = mul <2 x i32> %a6za, %a6zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a6z = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a6zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %a6zm = mul <2 x i32> %a6za, %a6zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a6z = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a6zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a6sa = sext <2 x i16> undef to <2 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a6sb = sext <2 x i16> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %a6sm = mul <2 x i32> %a6sa, %a6sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a6s = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a6sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %a6sm = mul <2 x i32> %a6sa, %a6sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a6s = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a6sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a7za = zext <4 x i16> undef to <4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a7zb = zext <4 x i16> undef to <4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a7zm = mul <4 x i32> %a7za, %a7zb
@@ -626,9 +626,9 @@ define void @mla_i32() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a9sm = mul <16 x i32> %a9sa, %a9sb
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a9s = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a9sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a10m = mul <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a10 = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a10m)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a11m = mul <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %a11 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a11m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a10 = call i32 @llvm.vector.reduce.add.v1i32(<1 x i32> %a10m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %a11m = mul <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %a11 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a11m)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a12m = mul <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a12 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a12m)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a13m = mul <8 x i32> undef, undef
@@ -757,136 +757,136 @@ define void @mla_i32() {
 
 define void @mla_i64() {
 ; CHECK-LABEL: 'mla_i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a0za = zext <1 x i8> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a0zb = zext <1 x i8> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a0za = zext <1 x i8> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a0zb = zext <1 x i8> undef to <1 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0zm = mul <1 x i64> %a0za, %a0zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a0zm)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a0sa = sext <1 x i8> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a0sb = sext <1 x i8> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a0z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a0zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a0sa = sext <1 x i8> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a0sb = sext <1 x i8> undef to <1 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a0sm = mul <1 x i64> %a0sa, %a0sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a0sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a0s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a0sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a1za = zext <2 x i8> undef to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a1zb = zext <2 x i8> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a1zm = mul <2 x i64> %a1za, %a1zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a1z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %a1zm = mul <2 x i64> %a1za, %a1zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a1z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %a1sa = sext <2 x i8> undef to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %a1sb = sext <2 x i8> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a1sm = mul <2 x i64> %a1sa, %a1sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a1s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %a1sm = mul <2 x i64> %a1sa, %a1sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a1s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a1sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %a2za = zext <4 x i8> undef to <4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %a2zb = zext <4 x i8> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %a2zm = mul <4 x i64> %a2za, %a2zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a2z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a2zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %a2zm = mul <4 x i64> %a2za, %a2zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a2z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a2zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %a2sa = sext <4 x i8> undef to <4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %a2sb = sext <4 x i8> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %a2sm = mul <4 x i64> %a2sa, %a2sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a2s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a2sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %a2sm = mul <4 x i64> %a2sa, %a2sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a2s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a2sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %a3za = zext <8 x i8> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %a3zb = zext <8 x i8> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a3zm = mul <8 x i64> %a3za, %a3zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a3z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a3zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a3z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a3zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %a3sa = sext <8 x i8> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %a3sb = sext <8 x i8> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a3sm = mul <8 x i64> %a3sa, %a3sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a3s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a3sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a3s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a3sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %a4za = zext <16 x i8> undef to <16 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %a4zb = zext <16 x i8> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 800 for instruction: %a4zm = mul <16 x i64> %a4za, %a4zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a4z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a4zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %a4zm = mul <16 x i64> %a4za, %a4zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a4z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a4zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %a4sa = sext <16 x i8> undef to <16 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1322 for instruction: %a4sb = sext <16 x i8> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 800 for instruction: %a4sm = mul <16 x i64> %a4sa, %a4sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a4s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a4sm)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a5za = zext <1 x i16> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a5zb = zext <1 x i16> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %a4sm = mul <16 x i64> %a4sa, %a4sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a4s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a4sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a5za = zext <1 x i16> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a5zb = zext <1 x i16> undef to <1 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a5zm = mul <1 x i64> %a5za, %a5zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a5zm)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a5sa = sext <1 x i16> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a5sb = sext <1 x i16> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a5z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a5zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %a5sa = sext <1 x i16> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %a5sb = sext <1 x i16> undef to <1 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a5sm = mul <1 x i64> %a5sa, %a5sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a5sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a5s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a5sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a6za = zext <2 x i16> undef to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a6zb = zext <2 x i16> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a6zm = mul <2 x i64> %a6za, %a6zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a6z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a6zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %a6zm = mul <2 x i64> %a6za, %a6zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a6z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a6zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %a6sa = sext <2 x i16> undef to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %a6sb = sext <2 x i16> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a6sm = mul <2 x i64> %a6sa, %a6sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a6s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a6sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %a6sm = mul <2 x i64> %a6sa, %a6sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a6s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a6sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %a7za = zext <4 x i16> undef to <4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %a7zb = zext <4 x i16> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %a7zm = mul <4 x i64> %a7za, %a7zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a7z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a7zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %a7zm = mul <4 x i64> %a7za, %a7zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a7z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a7zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %a7sa = sext <4 x i16> undef to <4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %a7sb = sext <4 x i16> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %a7sm = mul <4 x i64> %a7sa, %a7sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a7s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a7sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %a7sm = mul <4 x i64> %a7sa, %a7sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a7s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a7sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %a8za = zext <8 x i16> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %a8zb = zext <8 x i16> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a8zm = mul <8 x i64> %a8za, %a8zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a8z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a8zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a8z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a8zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %a8sa = sext <8 x i16> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 330 for instruction: %a8sb = sext <8 x i16> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a8sm = mul <8 x i64> %a8sa, %a8sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a8s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a8sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a8s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a8sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 296 for instruction: %a9za = zext <16 x i16> undef to <16 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 296 for instruction: %a9zb = zext <16 x i16> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 800 for instruction: %a9zm = mul <16 x i64> %a9za, %a9zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a9z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a9zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %a9zm = mul <16 x i64> %a9za, %a9zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a9z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a9zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1320 for instruction: %a9sa = sext <16 x i16> undef to <16 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1320 for instruction: %a9sb = sext <16 x i16> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 800 for instruction: %a9sm = mul <16 x i64> %a9sa, %a9sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a9s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a9sm)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a10za = zext <1 x i32> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a10zb = zext <1 x i32> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %a9sm = mul <16 x i64> %a9sa, %a9sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a9s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a9sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a10za = zext <1 x i32> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a10zb = zext <1 x i32> undef to <1 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a10zm = mul <1 x i64> %a10za, %a10zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a10z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a10zm)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a10sa = sext <1 x i32> undef to <1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a10sb = sext <1 x i32> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a10z = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a10zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a10sa = sext <1 x i32> undef to <1 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %a10sb = sext <1 x i32> undef to <1 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a10sm = mul <1 x i64> %a10sa, %a10sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a10s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a10sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a10s = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a10sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a11za = zext <2 x i32> undef to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a11zb = zext <2 x i32> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a11zm = mul <2 x i64> %a11za, %a11zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a11z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a11zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %a11zm = mul <2 x i64> %a11za, %a11zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a11z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a11zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a11sa = sext <2 x i32> undef to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a11sb = sext <2 x i32> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a11sm = mul <2 x i64> %a11sa, %a11sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a11s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a11sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %a11sm = mul <2 x i64> %a11sa, %a11sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a11s = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a11sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %a12za = zext <4 x i32> undef to <4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %a12zb = zext <4 x i32> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %a12zm = mul <4 x i64> %a12za, %a12zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a12z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a12zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %a12zm = mul <4 x i64> %a12za, %a12zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a12z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a12zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a12sa = sext <4 x i32> undef to <4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a12sb = sext <4 x i32> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %a12sm = mul <4 x i64> %a12sa, %a12sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a12s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a12sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %a12sm = mul <4 x i64> %a12sa, %a12sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a12s = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a12sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %a13za = zext <8 x i32> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %a13zb = zext <8 x i32> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a13zm = mul <8 x i64> %a13za, %a13zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a13z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a13zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a13z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a13zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %a13sa = sext <8 x i32> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %a13sb = sext <8 x i32> undef to <8 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a13sm = mul <8 x i64> %a13sa, %a13sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a13s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a13sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a13s = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a13sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %a14za = zext <16 x i32> undef to <16 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %a14zb = zext <16 x i32> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 800 for instruction: %a14zm = mul <16 x i64> %a14za, %a14zb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a14z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a14zm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %a14zm = mul <16 x i64> %a14za, %a14zb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a14z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a14zm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1056 for instruction: %a14sa = sext <16 x i32> undef to <16 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1056 for instruction: %a14sb = sext <16 x i32> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 800 for instruction: %a14sm = mul <16 x i64> %a14sa, %a14sb
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a14s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a14sm)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 416 for instruction: %a14sm = mul <16 x i64> %a14sa, %a14sb
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a14s = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a14sm)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a15m = mul <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a15 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a15m)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a16m = mul <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %a16 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a16m)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %a17m = mul <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %a17 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a17m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a15 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a15m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %a16m = mul <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %a16 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a16m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %a17m = mul <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %a17 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a17m)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %a18m = mul <8 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %a18 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a18m)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %a19m = mul <16 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 730 for instruction: %a19 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a19m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 408 for instruction: %a18 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a18m)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %a19m = mul <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 808 for instruction: %a19 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %a19m)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %a0za = zext <1 x i8> undef to <1 x i64>
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-and.ll b/llvm/test/Analysis/CostModel/ARM/reduce-and.ll
index 26120b8657d1ef75ff9fc7bc80eb8a3a986e6d16..2bd23dd27719dfcdbee4bca12640006b72e55910 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-and.ll
@@ -4,13 +4,13 @@
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 150 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 391 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 488 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 682 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1070 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 193 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 385 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-or.ll b/llvm/test/Analysis/CostModel/ARM/reduce-or.ll
index 2027e6f16d58a7dc04fde9f76b7d053bb5d3faec..c29f0cfd609dc912dddd081aacd1b3ecbee84d92 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-or.ll
@@ -4,13 +4,13 @@
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 150 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 391 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 488 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 682 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1070 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 193 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 385 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll b/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll
index 868ebdc24756f14f9ef5fb6fdf3d6802d278f643..5a72f3be05cc15be188e367ca5794baa03d9f107 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll
@@ -21,11 +21,11 @@ define i32 @reduce_i64(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i64'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 282 for instruction: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 970 for instruction: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 148 for instruction: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 568 for instruction: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1128 for instruction: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1  = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
@@ -54,11 +54,11 @@ define i32 @reduce_i32(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 240 for instruction: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 696 for instruction: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 2504 for instruction: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 376 for instruction: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 712 for instruction: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
@@ -89,12 +89,12 @@ define i32 @reduce_i16(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 788 for instruction: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 1176 for instruction: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 2976 for instruction: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 10160 for instruction: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 400 for instruction: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 532 for instruction: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 860 for instruction: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1516 for instruction: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
@@ -128,13 +128,13 @@ define i32 @reduce_i8(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 788 for instruction: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 4128 for instruction: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 5668 for instruction: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 12844 for instruction: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 41532 for instruction: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 400 for instruction: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1044 for instruction: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1304 for instruction: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1952 for instruction: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 3248 for instruction: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2   = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll b/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll
index 61ec7a4637d72dff1e1f029ed8d398aaea60518e..cd4da54354362e320c178072a65cd7a46040332d 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll
@@ -21,11 +21,11 @@ define i32 @reduce_i64(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i64'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 282 for instruction: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 970 for instruction: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 148 for instruction: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 568 for instruction: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1128 for instruction: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1  = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
@@ -54,11 +54,11 @@ define i32 @reduce_i32(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 240 for instruction: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 696 for instruction: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 2504 for instruction: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 376 for instruction: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 712 for instruction: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
@@ -89,12 +89,12 @@ define i32 @reduce_i16(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 788 for instruction: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 1176 for instruction: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 2976 for instruction: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 10160 for instruction: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 400 for instruction: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 532 for instruction: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 860 for instruction: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1516 for instruction: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
@@ -128,13 +128,13 @@ define i32 @reduce_i8(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 788 for instruction: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 4128 for instruction: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 5668 for instruction: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 12844 for instruction: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 41532 for instruction: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 400 for instruction: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1044 for instruction: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1304 for instruction: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1952 for instruction: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 3248 for instruction: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2   = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll b/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll
index 96f24f3c92e93a4ac39e110fef4a8e52af214015..0447e3826ca552cb1995bd6b6ea5122624f2530d 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll
@@ -21,11 +21,11 @@ define i32 @reduce_i64(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i64'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 282 for instruction: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 970 for instruction: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 148 for instruction: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 568 for instruction: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1128 for instruction: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1  = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
@@ -54,11 +54,11 @@ define i32 @reduce_i32(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 240 for instruction: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 696 for instruction: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 2504 for instruction: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 376 for instruction: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 712 for instruction: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
@@ -89,12 +89,12 @@ define i32 @reduce_i16(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 788 for instruction: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 1176 for instruction: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 2976 for instruction: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 10160 for instruction: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 400 for instruction: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 532 for instruction: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 860 for instruction: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1516 for instruction: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
@@ -128,13 +128,13 @@ define i32 @reduce_i8(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 788 for instruction: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 4128 for instruction: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 5668 for instruction: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 12844 for instruction: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 41532 for instruction: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 400 for instruction: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1044 for instruction: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1304 for instruction: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1952 for instruction: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 3248 for instruction: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2   = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll b/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll
index d77ed48b4a5b51b9ec736a1e1f3fbcf9b38305d6..62510ae70f2bc1a8eeb85600210e8f1350cca4f4 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll
@@ -21,11 +21,11 @@ define i32 @reduce_i64(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i64'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 282 for instruction: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 970 for instruction: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 148 for instruction: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 288 for instruction: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 568 for instruction: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1128 for instruction: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1  = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
@@ -54,11 +54,11 @@ define i32 @reduce_i32(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 240 for instruction: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 696 for instruction: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 2504 for instruction: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 208 for instruction: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 376 for instruction: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 712 for instruction: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
@@ -89,12 +89,12 @@ define i32 @reduce_i16(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 788 for instruction: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 1176 for instruction: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 2976 for instruction: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 10160 for instruction: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 400 for instruction: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 532 for instruction: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 860 for instruction: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1516 for instruction: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2  = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
@@ -128,13 +128,13 @@ define i32 @reduce_i8(i32 %arg) {
 ; NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 788 for instruction: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 4128 for instruction: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 5668 for instruction: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 12844 for instruction: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
-; MVE-NEXT:  Cost Model: Found an estimated cost of 41532 for instruction: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 400 for instruction: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1044 for instruction: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1304 for instruction: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 1952 for instruction: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
+; MVE-NEXT:  Cost Model: Found an estimated cost of 3248 for instruction: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
 ; MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2   = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/select.ll b/llvm/test/Analysis/CostModel/ARM/select.ll
index 67a558003923cc2f84021d84c859cad14f785923..173904d9e60b74140bc1f3e5f6151643bdd5d781 100644
--- a/llvm/test/Analysis/CostModel/ARM/select.ll
+++ b/llvm/test/Analysis/CostModel/ARM/select.ll
@@ -18,28 +18,28 @@ define void @selects() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4 = select i1 undef, i64 undef, i64 undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = select i1 undef, float undef, float undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = select i1 undef, double undef, double undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v7 = select <2 x i1> undef, <2 x i8> undef, <2 x i8> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v7 = select <2 x i1> undef, <2 x i8> undef, <2 x i8> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8 = select <4 x i1> undef, <4 x i8> undef, <4 x i8> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v9 = select <8 x i1> undef, <8 x i8> undef, <8 x i8> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v10 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v11 = select <2 x i1> undef, <2 x i16> undef, <2 x i16> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v11 = select <2 x i1> undef, <2 x i16> undef, <2 x i16> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v12 = select <4 x i1> undef, <4 x i16> undef, <4 x i16> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v13 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v13b = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v14 = select <2 x i1> undef, <2 x i32> undef, <2 x i32> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v14 = select <2 x i1> undef, <2 x i32> undef, <2 x i32> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v15 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v15b = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v15c = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v16 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 576 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v17 = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v18 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v19 = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v20 = select <1 x i1> undef, <1 x i32> undef, <1 x i32> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v19 = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v20 = select <1 x i1> undef, <1 x i32> undef, <1 x i32> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v21 = select <3 x i1> undef, <3 x float> undef, <3 x float> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %v22 = select <5 x i1> undef, <5 x double> undef, <5 x double> undef
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v22 = select <5 x i1> undef, <5 x double> undef, <5 x double> undef
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-NEON-RECIP-LABEL: 'selects'
diff --git a/llvm/test/Analysis/CostModel/ARM/shuffle.ll b/llvm/test/Analysis/CostModel/ARM/shuffle.ll
index e1dec2cb6e854354bc9a8c963c4f7ecad6d891cc..65f7ea0393a1fbc45695234dd31fcc46bf98f76d 100644
--- a/llvm/test/Analysis/CostModel/ARM/shuffle.ll
+++ b/llvm/test/Analysis/CostModel/ARM/shuffle.ll
@@ -4,19 +4,19 @@
 
 define void @broadcast() {
 ; CHECK-MVE-LABEL: 'broadcast'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> zeroinitializer
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> zeroinitializer
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> zeroinitializer
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> zeroinitializer
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> zeroinitializer
@@ -24,8 +24,8 @@ define void @broadcast() {
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> zeroinitializer
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> zeroinitializer
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> zeroinitializer
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-NEON-LABEL: 'broadcast'
@@ -88,28 +88,28 @@ define void @broadcast() {
 ;; Reverse shuffles should be lowered to vrev and possibly a vext (for quadwords, on neon)
 define void @reverse() {
 ; CHECK-MVE-LABEL: 'reverse'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2f32 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-NEON-LABEL: 'reverse'
@@ -233,28 +233,28 @@ define void @concat() {
 
 define void @select() {
 ; CHECK-MVE-LABEL: 'select'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 0, i32 3>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2f32 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-NEON-LABEL: 'select'
@@ -316,21 +316,21 @@ define void @select() {
 
 define void @vrev2() {
 ; CHECK-MVE-LABEL: 'vrev2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-NEON-LABEL: 'vrev2'
@@ -378,14 +378,14 @@ define void @vrev2() {
 
 define void @vrev4() {
 ; CHECK-MVE-LABEL: 'vrev4'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-NEON-LABEL: 'vrev4'
diff --git a/llvm/test/Analysis/CostModel/PowerPC/reduce-and.ll b/llvm/test/Analysis/CostModel/PowerPC/reduce-and.ll
index 15f697fd100792e53e662e8ed909b1281b64dda2..68768258bc2f2415e115e0a0155a495c8c5796c8 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/reduce-and.ll
@@ -4,13 +4,13 @@
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 193 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 386 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/PowerPC/reduce-or.ll b/llvm/test/Analysis/CostModel/PowerPC/reduce-or.ll
index b37396fd1e6bbc2c928cbe8765f3c64d2b3d1e2e..1ccae8185b3db1d08fa8530304c080cf5d7ff1b5 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/reduce-or.ll
@@ -4,13 +4,13 @@
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 193 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 386 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-gather.ll b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-gather.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5de1f3eb97d884f165649987c7ffb1223e3836dc
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-gather.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -cost-model -analyze -mtriple=riscv64 -mattr=+experimental-v,+f,+d,+experimental-zfh -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 < %s 2>%t | FileCheck %s
+; Sanity check that we don't crash querying costs when vectors are not enabled.
+; RUN: opt -cost-model -analyze -mtriple=riscv64
+
+define i32 @masked_gather() {
+; CHECK-LABEL: 'masked_gather'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> undef, i32 1, <8 x i1> undef, <8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> undef, i32 1, <4 x i1> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> undef, i32 8, <2 x i1> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0f64(<1 x double*> undef, i32 8, <1 x i1> undef, <1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> undef, i32 1, <16 x i1> undef, <16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> undef, i32 1, <8 x i1> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = call <1 x float> @llvm.masked.gather.v1f32.v1p0f32(<1 x float*> undef, i32 4, <1 x i1> undef, <1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32F16 = call <32 x half> @llvm.masked.gather.v32f16.v32p0f16(<32 x half*> undef, i32 1, <32 x i1> undef, <32 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F16 = call <16 x half> @llvm.masked.gather.v16f16.v16p0f16(<16 x half*> undef, i32 1, <16 x i1> undef, <16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F16 = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F16 = call <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*> undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16 = call <2 x half> @llvm.masked.gather.v2f16.v2p0f16(<2 x half*> undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = call <1 x half> @llvm.masked.gather.v1f16.v1p0f16(<1 x half*> undef, i32 2, <1 x i1> undef, <1 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> undef, i32 1, <8 x i1> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*> undef, i32 1, <4 x i1> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> undef, i32 8, <2 x i1> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0i64(<1 x i64*> undef, i32 8, <1 x i1> undef, <1 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> undef, i32 1, <16 x i1> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> undef, i32 1, <8 x i1> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = call <1 x i32> @llvm.masked.gather.v1i32.v1p0i32(<1 x i32*> undef, i32 4, <1 x i1> undef, <1 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0i16(<32 x i16*> undef, i32 1, <32 x i1> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0i16(<16 x i16*> undef, i32 1, <16 x i1> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = call <1 x i16> @llvm.masked.gather.v1i16.v1p0i16(<1 x i16*> undef, i32 2, <1 x i1> undef, <1 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0i8(<64 x i8*> undef, i32 1, <64 x i1> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> undef, i32 1, <32 x i1> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I8 = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I8 = call <1 x i8> @llvm.masked.gather.v1i8.v1p0i8(<1 x i8*> undef, i32 1, <1 x i1> undef, <1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+;
+  %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> undef, i32 1, <8 x i1> undef, <8 x double> undef)
+  %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> undef, i32 1, <4 x i1> undef, <4 x double> undef)
+  %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> undef, i32 8, <2 x i1> undef, <2 x double> undef)
+  %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0f64(<1 x double*> undef, i32 8, <1 x i1> undef, <1 x double> undef)
+
+  %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> undef, i32 1, <16 x i1> undef, <16 x float> undef)
+  %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> undef, i32 1, <8 x i1> undef, <8 x float> undef)
+  %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> undef, i32 4, <4 x i1> undef, <4 x float> undef)
+  %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> undef, i32 4, <2 x i1> undef, <2 x float> undef)
+  %V1F32 = call <1 x float> @llvm.masked.gather.v1f32.v1p0f32(<1 x float*> undef, i32 4, <1 x i1> undef, <1 x float> undef)
+
+  %V32F16 = call <32 x half> @llvm.masked.gather.v32f16.v32p0f16(<32 x half*> undef, i32 1, <32 x i1> undef, <32 x half> undef)
+  %V16F16 = call <16 x half> @llvm.masked.gather.v16f16.v16p0f16(<16 x half*> undef, i32 1, <16 x i1> undef, <16 x half> undef)
+  %V8F16 = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> undef, i32 2, <8 x i1> undef, <8 x half> undef)
+  %V4F16 = call <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*> undef, i32 2, <4 x i1> undef, <4 x half> undef)
+  %V2F16 = call <2 x half> @llvm.masked.gather.v2f16.v2p0f16(<2 x half*> undef, i32 2, <2 x i1> undef, <2 x half> undef)
+  %V1F16 = call <1 x half> @llvm.masked.gather.v1f16.v1p0f16(<1 x half*> undef, i32 2, <1 x i1> undef, <1 x half> undef)
+
+  %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> undef, i32 1, <8 x i1> undef, <8 x i64> undef)
+  %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*> undef, i32 1, <4 x i1> undef, <4 x i64> undef)
+  %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> undef, i32 8, <2 x i1> undef, <2 x i64> undef)
+  %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0i64(<1 x i64*> undef, i32 8, <1 x i1> undef, <1 x i64> undef)
+
+  %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> undef, i32 1, <16 x i1> undef, <16 x i32> undef)
+  %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> undef, i32 1, <8 x i1> undef, <8 x i32> undef)
+  %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+  %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+  %V1I32 = call <1 x i32> @llvm.masked.gather.v1i32.v1p0i32(<1 x i32*> undef, i32 4, <1 x i1> undef, <1 x i32> undef)
+
+  %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0i16(<32 x i16*> undef, i32 1, <32 x i1> undef, <32 x i16> undef)
+  %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0i16(<16 x i16*> undef, i32 1, <16 x i1> undef, <16 x i16> undef)
+  %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+  %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+  %V2I16 = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+  %V1I16 = call <1 x i16> @llvm.masked.gather.v1i16.v1p0i16(<1 x i16*> undef, i32 2, <1 x i1> undef, <1 x i16> undef)
+
+  %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0i8(<64 x i8*> undef, i32 1, <64 x i1> undef, <64 x i8> undef)
+  %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> undef, i32 1, <32 x i1> undef, <32 x i8> undef)
+  %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+  %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+  %V4I8 = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+  %V2I8 = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+  %V1I8 = call <1 x i8> @llvm.masked.gather.v1i8.v1p0i8(<1 x i8*> undef, i32 1, <1 x i1> undef, <1 x i8> undef)
+
+  ret i32 0
+}
+
+declare <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*>, i32, <8 x i1>, <8 x double>)
+declare <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*>, i32, <4 x i1>, <4 x double>)
+declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*>, i32, <2 x i1>, <2 x double>)
+declare <1 x double> @llvm.masked.gather.v1f64.v1p0f64(<1 x double*>, i32, <1 x i1>, <1 x double>)
+
+declare <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*>, i32, <16 x i1>, <16 x float>)
+declare <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*>, i32, <8 x i1>, <8 x float>)
+declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1>, <4 x float>)
+declare <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*>, i32, <2 x i1>, <2 x float>)
+declare <1 x float> @llvm.masked.gather.v1f32.v1p0f32(<1 x float*>, i32, <1 x i1>, <1 x float>)
+
+declare <32 x half> @llvm.masked.gather.v32f16.v32p0f16(<32 x half*>, i32, <32 x i1>, <32 x half>)
+declare <16 x half> @llvm.masked.gather.v16f16.v16p0f16(<16 x half*>, i32, <16 x i1>, <16 x half>)
+declare <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*>, i32, <8 x i1>, <8 x half>)
+declare <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*>, i32, <4 x i1>, <4 x half>)
+declare <2 x half> @llvm.masked.gather.v2f16.v2p0f16(<2 x half*>, i32, <2 x i1>, <2 x half>)
+declare <1 x half> @llvm.masked.gather.v1f16.v1p0f16(<1 x half*>, i32, <1 x i1>, <1 x half>)
+
+declare <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*>, i32, <8 x i1>, <8 x i64>)
+declare <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*>, i32, <4 x i1>, <4 x i64>)
+declare <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*>, i32, <2 x i1>, <2 x i64>)
+declare <1 x i64> @llvm.masked.gather.v1i64.v1p0i64(<1 x i64*>, i32, <1 x i1>, <1 x i64>)
+
+declare <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*>, i32, <16 x i1>, <16 x i32>)
+declare <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*>, i32, <8 x i1>, <8 x i32>)
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
+declare <1 x i32> @llvm.masked.gather.v1i32.v1p0i32(<1 x i32*>, i32, <1 x i1>, <1 x i32>)
+
+declare <32 x i16> @llvm.masked.gather.v32i16.v32p0i16(<32 x i16*>, i32, <32 x i1>, <32 x i16>)
+declare <16 x i16> @llvm.masked.gather.v16i16.v16p0i16(<16 x i16*>, i32, <16 x i1>, <16 x i16>)
+declare <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*>, i32, <8 x i1>, <8 x i16>)
+declare <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>)
+declare <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*>, i32, <2 x i1>, <2 x i16>)
+declare <1 x i16> @llvm.masked.gather.v1i16.v1p0i16(<1 x i16*>, i32, <1 x i1>, <1 x i16>)
+
+declare <64 x i8> @llvm.masked.gather.v64i8.v64p0i8(<64 x i8*>, i32, <64 x i1>, <64 x i8>)
+declare <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*>, i32, <32 x i1>, <32 x i8>)
+declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32, <16 x i1>, <16 x i8>)
+declare <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>)
+declare <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>)
+declare <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*>, i32, <2 x i1>, <2 x i8>)
+declare <1 x i8> @llvm.masked.gather.v1i8.v1p0i8(<1 x i8*>, i32, <1 x i1>, <1 x i8>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-scatter.ll b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-scatter.ll
new file mode 100644
index 0000000000000000000000000000000000000000..90323e26e8faf35cc6bd82a09749b85fe5ed926d
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-scatter.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -cost-model -analyze -mtriple=riscv64 -mattr=+experimental-v,+f,+d,+experimental-zfh -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 < %s 2>%t | FileCheck %s
+; Sanity check that we don't crash querying costs when vectors are not enabled.
+; RUN: opt -cost-model -analyze -mtriple=riscv64
+
+define i32 @masked_scatter() {
+; CHECK-LABEL: 'masked_scatter'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> undef, <8 x double*> undef, i32 1, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> undef, <4 x double*> undef, i32 1, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> undef, <2 x double*> undef, i32 8, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1f64.v1p0f64(<1 x double> undef, <1 x double*> undef, i32 8, <1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> undef, <16 x float*> undef, i32 1, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> undef, <8 x float*> undef, i32 1, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> undef, <4 x float*> undef, i32 2, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> undef, <2 x float*> undef, i32 2, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1f32.v1p0f32(<1 x float> undef, <1 x float*> undef, i32 2, <1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v32f16.v32p0f16(<32 x half> undef, <32 x half*> undef, i32 1, <32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v16f16.v16p0f16(<16 x half> undef, <16 x half*> undef, i32 1, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> undef, <8 x half*> undef, i32 2, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half> undef, <4 x half*> undef, i32 2, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v2f16.v2p0f16(<2 x half> undef, <2 x half*> undef, i32 2, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1f16.v1p0f16(<1 x half> undef, <1 x half*> undef, i32 2, <1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> undef, <8 x i64*> undef, i32 1, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.scatter.v4i64.v4p0i64(<4 x i64> undef, <4 x i64*> undef, i32 1, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> undef, <2 x i64*> undef, i32 8, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1i64.v1p0i64(<1 x i64> undef, <1 x i64*> undef, i32 8, <1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> undef, <16 x i32*> undef, i32 1, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> undef, <8 x i32*> undef, i32 1, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> undef, <4 x i32*> undef, i32 4, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> undef, <2 x i32*> undef, i32 4, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1i32.v1p0i32(<1 x i32> undef, <1 x i32*> undef, i32 4, <1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v32i16.v32p0i16(<32 x i16> undef, <32 x i16*> undef, i32 1, <32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v16i16.v16p0i16(<16 x i16> undef, <16 x i16*> undef, i32 1, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> undef, <8 x i16*> undef, i32 2, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16> undef, <4 x i16*> undef, i32 2, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v2i16.v2p0i16(<2 x i16> undef, <2 x i16*> undef, i32 2, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1i16.v1p0i16(<1 x i16> undef, <1 x i16*> undef, i32 2, <1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.scatter.v64i8.v64p0i8(<64 x i8> undef, <64 x i8*> undef, i32 1, <64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8> undef, <32 x i8*> undef, i32 1, <32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> undef, <16 x i8*> undef, i32 1, <16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v8i8.v8p0i8(<8 x i8> undef, <8 x i8*> undef, i32 1, <8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> undef, <4 x i8*> undef, i32 1, <4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8> undef, <2 x i8*> undef, i32 1, <2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1i8.v1p0i8(<1 x i8> undef, <1 x i8*> undef, i32 1, <1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+;
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> undef, <8 x double*> undef, i32 1, <8 x i1> undef)
+  call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> undef, <4 x double*> undef, i32 1, <4 x i1> undef)
+  call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> undef, <2 x double*> undef, i32 8, <2 x i1> undef)
+  call void @llvm.masked.scatter.v1f64.v1p0f64(<1 x double> undef, <1 x double*> undef, i32 8, <1 x i1> undef)
+
+  call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> undef, <16 x float*> undef, i32 1, <16 x i1> undef)
+  call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> undef, <8 x float*> undef, i32 1, <8 x i1> undef)
+  call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> undef, <4 x float*> undef, i32 2, <4 x i1> undef)
+  call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> undef, <2 x float*> undef, i32 2, <2 x i1> undef)
+  call void @llvm.masked.scatter.v1f32.v1p0f32(<1 x float> undef, <1 x float*> undef, i32 2, <1 x i1> undef)
+
+  call void @llvm.masked.scatter.v32f16.v32p0f16(<32 x half> undef, <32 x half*> undef, i32 1, <32 x i1> undef)
+  call void @llvm.masked.scatter.v16f16.v16p0f16(<16 x half> undef, <16 x half*> undef, i32 1, <16 x i1> undef)
+  call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> undef, <8 x half*> undef, i32 2, <8 x i1> undef)
+  call void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half> undef, <4 x half*> undef, i32 2, <4 x i1> undef)
+  call void @llvm.masked.scatter.v2f16.v2p0f16(<2 x half> undef, <2 x half*> undef, i32 2, <2 x i1> undef)
+  call void @llvm.masked.scatter.v1f16.v1p0f16(<1 x half> undef, <1 x half*> undef, i32 2, <1 x i1> undef)
+
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> undef, <8 x i64*> undef, i32 1, <8 x i1> undef)
+  call void @llvm.masked.scatter.v4i64.v4p0i64(<4 x i64> undef, <4 x i64*> undef, i32 1, <4 x i1> undef)
+  call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> undef, <2 x i64*> undef, i32 8, <2 x i1> undef)
+  call void @llvm.masked.scatter.v1i64.v1p0i64(<1 x i64> undef, <1 x i64*> undef, i32 8, <1 x i1> undef)
+
+  call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> undef, <16 x i32*> undef, i32 1, <16 x i1> undef)
+  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> undef, <8 x i32*> undef, i32 1, <8 x i1> undef)
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> undef, <4 x i32*> undef, i32 4, <4 x i1> undef)
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> undef, <2 x i32*> undef, i32 4, <2 x i1> undef)
+  call void @llvm.masked.scatter.v1i32.v1p0i32(<1 x i32> undef, <1 x i32*> undef, i32 4, <1 x i1> undef)
+
+  call void @llvm.masked.scatter.v32i16.v32p0i16(<32 x i16> undef, <32 x i16*> undef, i32 1, <32 x i1> undef)
+  call void @llvm.masked.scatter.v16i16.v16p0i16(<16 x i16> undef, <16 x i16*> undef, i32 1, <16 x i1> undef)
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> undef, <8 x i16*> undef, i32 2, <8 x i1> undef)
+  call void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16> undef, <4 x i16*> undef, i32 2, <4 x i1> undef)
+  call void @llvm.masked.scatter.v2i16.v2p0i16(<2 x i16> undef, <2 x i16*> undef, i32 2, <2 x i1> undef)
+  call void @llvm.masked.scatter.v1i16.v1p0i16(<1 x i16> undef, <1 x i16*> undef, i32 2, <1 x i1> undef)
+
+  call void @llvm.masked.scatter.v64i8.v64p0i8(<64 x i8> undef, <64 x i8*> undef, i32 1, <64 x i1> undef)
+  call void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8> undef, <32 x i8*> undef, i32 1, <32 x i1> undef)
+  call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> undef, <16 x i8*> undef, i32 1, <16 x i1> undef)
+  call void @llvm.masked.scatter.v8i8.v8p0i8(<8 x i8> undef, <8 x i8*> undef, i32 1, <8 x i1> undef)
+  call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> undef, <4 x i8*> undef, i32 1, <4 x i1> undef)
+  call void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8> undef, <2 x i8*> undef, i32 1, <2 x i1> undef)
+  call void @llvm.masked.scatter.v1i8.v1p0i8(<1 x i8> undef, <1 x i8*> undef, i32 1, <1 x i1> undef)
+
+  ret i32 0
+}
+
+declare void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double>, <8 x double*>, i32, <8 x i1>)
+declare void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double>, <4 x double*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double>, <2 x double*>, i32, <2 x i1>)
+declare void @llvm.masked.scatter.v1f64.v1p0f64(<1 x double>, <1 x double*>, i32, <1 x i1>)
+
+declare void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float>, <16 x float*>, i32, <16 x i1>)
+declare void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float>, <8 x float*>, i32, <8 x i1>)
+declare void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float>, <4 x float*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float>, <2 x float*>, i32, <2 x i1>)
+declare void @llvm.masked.scatter.v1f32.v1p0f32(<1 x float>, <1 x float*>, i32, <1 x i1>)
+
+declare void @llvm.masked.scatter.v32f16.v32p0f16(<32 x half>, <32 x half*>, i32, <32 x i1>)
+declare void @llvm.masked.scatter.v16f16.v16p0f16(<16 x half>, <16 x half*>, i32, <16 x i1>)
+declare void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half>, <8 x half*>, i32, <8 x i1>)
+declare void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half>, <4 x half*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v2f16.v2p0f16(<2 x half>, <2 x half*>, i32, <2 x i1>)
+declare void @llvm.masked.scatter.v1f16.v1p0f16(<1 x half>, <1 x half*>, i32, <1 x i1>)
+
+declare void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64>, <8 x i64*>, i32, <8 x i1>)
+declare void @llvm.masked.scatter.v4i64.v4p0i64(<4 x i64>, <4 x i64*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64>, <2 x i64*>, i32, <2 x i1>)
+declare void @llvm.masked.scatter.v1i64.v1p0i64(<1 x i64>, <1 x i64*>, i32, <1 x i1>)
+
+declare void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32>, <16 x i32*>, i32, <16 x i1>)
+declare void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32>, <8 x i32*>, i32, <8 x i1>)
+declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>, <4 x i32*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32>, <2 x i32*>, i32, <2 x i1>)
+declare void @llvm.masked.scatter.v1i32.v1p0i32(<1 x i32>, <1 x i32*>, i32, <1 x i1>)
+
+declare void @llvm.masked.scatter.v32i16.v32p0i16(<32 x i16>, <32 x i16*>, i32, <32 x i1>)
+declare void @llvm.masked.scatter.v16i16.v16p0i16(<16 x i16>, <16 x i16*>, i32, <16 x i1>)
+declare void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16>, <8 x i16*>, i32, <8 x i1>)
+declare void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16>, <4 x i16*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v2i16.v2p0i16(<2 x i16>, <2 x i16*>, i32, <2 x i1>)
+declare void @llvm.masked.scatter.v1i16.v1p0i16(<1 x i16>, <1 x i16*>, i32, <1 x i1>)
+
+declare void @llvm.masked.scatter.v64i8.v64p0i8(<64 x i8>, <64 x i8*>, i32, <64 x i1>)
+declare void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8>, <32 x i8*>, i32, <32 x i1>)
+declare void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8>, <16 x i8*>, i32, <16 x i1>)
+declare void @llvm.masked.scatter.v8i8.v8p0i8(<8 x i8>, <8 x i8*>, i32, <8 x i1>)
+declare void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8>, <4 x i8*>, i32, <4 x i1>)
+declare void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8>, <2 x i8*>, i32, <2 x i1>)
+declare void @llvm.masked.scatter.v1i8.v1p0i8(<1 x i8>, <1 x i8*>, i32, <1 x i1>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll
index d5aae4153b252b90fa02b0959074bf103783c50f..1405464af6a50ecaa8ed47542b436c4c951c1354 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-and.ll
@@ -1,18 +1,29 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=riscv32 -cost-model -cost-kind=throughput -analyze | FileCheck %s
-; RUN: opt < %s -mtriple=riscv64 -cost-model -cost-kind=throughput -analyze | FileCheck %s
+; RUN: opt < %s -mtriple=riscv32 -cost-model -cost-kind=throughput -analyze | FileCheck %s --check-prefix=RISCV32
+; RUN: opt < %s -mtriple=riscv64 -cost-model -cost-kind=throughput -analyze | FileCheck %s --check-prefix=RISCV64
 
 define i32 @reduce_i1(i32 %arg) {
-; CHECK-LABEL: 'reduce_i1'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 190 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 382 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; RISCV32-LABEL: 'reduce_i1'
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; RISCV64-LABEL: 'reduce_i1'
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
   %V2   = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll
index 3f14f265c19087647f99345cf7e333e204d7353a..9d675de362c7649c6985576d8121ea32a73a6ab1 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-or.ll
@@ -1,18 +1,29 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=riscv32 -cost-model -cost-kind=throughput -analyze | FileCheck %s
-; RUN: opt < %s -mtriple=riscv64 -cost-model -cost-kind=throughput -analyze | FileCheck %s
+; RUN: opt < %s -mtriple=riscv32 -cost-model -cost-kind=throughput -analyze | FileCheck %s --check-prefix=RISCV32
+; RUN: opt < %s -mtriple=riscv64 -cost-model -cost-kind=throughput -analyze | FileCheck %s --check-prefix=RISCV64
 
 define i32 @reduce_i1(i32 %arg) {
-; CHECK-LABEL: 'reduce_i1'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 190 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 382 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; RISCV32-LABEL: 'reduce_i1'
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; RISCV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; RISCV64-LABEL: 'reduce_i1'
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; RISCV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
   %V2   = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/SystemZ/reduce-and.ll b/llvm/test/Analysis/CostModel/SystemZ/reduce-and.ll
index fccd496058ecc28db62423f086a2e8ecff2b8893..4eab88794f83c917af36b323266ef6fdb0a256c0 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/reduce-and.ll
@@ -4,13 +4,13 @@
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/SystemZ/reduce-or.ll b/llvm/test/Analysis/CostModel/SystemZ/reduce-or.ll
index 18d0f2b838d8ac692aa91746f5a2843a973cb457..700e4d6a8f8aba4921cce43cfedbf57be3d3378c 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/reduce-or.ll
@@ -4,13 +4,13 @@
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
index 98b4dc333c0a17fc790f4ced01184845eace7994..28723ed97e409808e6fd9bde50a8c328a5949d92 100644
--- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
+++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
@@ -531,16 +531,16 @@ define void @crash(i8* %ptr) {
 ; CHECK-LABEL: 'crash'
 ; CHECK-NEXT:  Classifying expressions for: @crash
 ; CHECK-NEXT:    %text.addr.5 = phi i8* [ %incdec.ptr112, %while.cond111 ], [ null, %while.body ]
-; CHECK-NEXT:    --> {null,+,-1}<nw><%while.cond111> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %while.cond111: Computable, %while.body: Variant }
+; CHECK-NEXT:    -->  {null,+,-1}<nw><%while.cond111> U: full-set S: full-set		Exits: <<Unknown>>		LoopDispositions: { %while.cond111: Computable, %while.body: Variant }
 ; CHECK-NEXT:    %incdec.ptr112 = getelementptr inbounds i8, i8* %text.addr.5, i64 -1
-; CHECK-NEXT:    --> {(-1 + null)<nuw><nsw>,+,-1}<nw><%while.cond111> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %while.cond111: Computable, %while.body: Variant }
+; CHECK-NEXT:    -->  {(-1 + null)<nuw><nsw>,+,-1}<nw><%while.cond111> U: full-set S: full-set		Exits: <<Unknown>>		LoopDispositions: { %while.cond111: Computable, %while.body: Variant }
 ; CHECK-NEXT:    %lastout.2271 = phi i8* [ %incdec.ptr126, %while.body125 ], [ %ptr, %while.end117 ]
-; CHECK-NEXT:    --> {%ptr,+,1}<nuw><%while.body125> U: full-set S: full-set Exits: {(-2 + null)<nuw><nsw>,+,-1}<nw><%while.cond111> LoopDispositions: { %while.body125: Computable }
+; CHECK-NEXT:    -->  {%ptr,+,1}<nuw><%while.body125> U: full-set S: full-set		Exits: {(-2 + null)<nuw><nsw>,+,-1}<nw><%while.cond111>		LoopDispositions: { %while.body125: Computable }
 ; CHECK-NEXT:    %incdec.ptr126 = getelementptr inbounds i8, i8* %lastout.2271, i64 1
-; CHECK-NEXT:    --> {(1 + %ptr)<nuw>,+,1}<nuw><%while.body125> U: [1,0) S: [1,0) Exits: {(-1 + null)<nuw><nsw>,+,-1}<nw><%while.cond111> LoopDispositions: { %while.body125: Computable }
+; CHECK-NEXT:    -->  {(1 + %ptr)<nuw>,+,1}<nuw><%while.body125> U: [1,0) S: [1,0)		Exits: {(-1 + null)<nuw><nsw>,+,-1}<nw><%while.cond111>		LoopDispositions: { %while.body125: Computable }
 ; CHECK-NEXT:  Determining loop execution counts for: @crash
 ; CHECK-NEXT:  Loop %while.body125: backedge-taken count is {(-2 + (-1 * %ptr) + null),+,-1}<nw><%while.cond111>
-; CHECK-NEXT:  Loop %while.body125: max backedge-taken count is -1
+; CHECK-NEXT:  Loop %while.body125: max backedge-taken count is -2
 ; CHECK-NEXT:  Loop %while.body125: Predicated backedge-taken count is {(-2 + (-1 * %ptr) + null),+,-1}<nw><%while.cond111>
 ; CHECK-NEXT:   Predicates:
 ; CHECK:       Loop %while.body125: Trip multiple is 1
diff --git a/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll b/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll
index 4eb9c9ea50a6b9791e22ff451c37583860cfc84d..5f34ad42ab5f650005a8d468ec182f489c5cfca5 100644
--- a/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll
+++ b/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll
@@ -193,11 +193,11 @@ define void @test_shl2() {
 ; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,5) S: [0,5) Exits: 4 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %iv.shl = phi i64 [ 4, %entry ], [ %iv.shl.next, %loop ]
-; CHECK-NEXT:    --> %iv.shl U: [0,-3) S: [-9223372036854775808,9223372036854775801) Exits: 64 LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    --> %iv.shl U: [4,65) S: [4,65) Exits: 64 LoopDispositions: { %loop: Variant }
 ; CHECK-NEXT:    %iv.next = add i64 %iv, 1
 ; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,6) S: [1,6) Exits: 5 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %iv.shl.next = shl i64 %iv.shl, 1
-; CHECK-NEXT:    --> (2 * %iv.shl) U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: 128 LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    --> (2 * %iv.shl)<nuw><nsw> U: [8,129) S: [8,129) Exits: 128 LoopDispositions: { %loop: Variant }
 ; CHECK-NEXT:  Determining loop execution counts for: @test_shl2
 ; CHECK-NEXT:  Loop %loop: backedge-taken count is 4
 ; CHECK-NEXT:  Loop %loop: max backedge-taken count is 4
@@ -227,7 +227,7 @@ define void @test_shl3(i1 %c) {
 ; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,5) S: [0,5) Exits: 4 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %iv.shl = phi i64 [ 4, %entry ], [ %iv.shl.next, %loop ]
-; CHECK-NEXT:    --> %iv.shl U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    --> %iv.shl U: [4,65) S: [4,65) Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
 ; CHECK-NEXT:    %iv.next = add i64 %iv, 1
 ; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,6) S: [1,6) Exits: 5 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %iv.shl.next = shl i64 %iv.shl, %shiftamt
@@ -260,11 +260,11 @@ define void @test_shl4() {
 ; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,61) S: [0,61) Exits: 60 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %iv.shl = phi i64 [ 4, %entry ], [ %iv.shl.next, %loop ]
-; CHECK-NEXT:    --> %iv.shl U: [0,-3) S: [-9223372036854775808,9223372036854775801) Exits: 4611686018427387904 LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    --> %iv.shl U: [4,4611686018427387905) S: [4,4611686018427387905) Exits: 4611686018427387904 LoopDispositions: { %loop: Variant }
 ; CHECK-NEXT:    %iv.next = add i64 %iv, 1
 ; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,62) S: [1,62) Exits: 61 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %iv.shl.next = shl i64 %iv.shl, 1
-; CHECK-NEXT:    --> (2 * %iv.shl) U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: -9223372036854775808 LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    --> (2 * %iv.shl)<nuw> U: [8,-9223372036854775807) S: [-9223372036854775808,9223372036854775801) Exits: -9223372036854775808 LoopDispositions: { %loop: Variant }
 ; CHECK-NEXT:  Determining loop execution counts for: @test_shl4
 ; CHECK-NEXT:  Loop %loop: backedge-taken count is 60
 ; CHECK-NEXT:  Loop %loop: max backedge-taken count is 60
@@ -324,7 +324,7 @@ define void @test_shl6(i1 %c) {
 ; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK-NEXT:    --> {0,+,1}<%loop> U: [0,5) S: [0,5) Exits: 4 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %iv.shl = phi i64 [ 4, %entry ], [ %iv.shl.next, %loop ]
-; CHECK-NEXT:    --> %iv.shl U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: 16 LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    --> %iv.shl U: [4,65) S: [4,65) Exits: 16 LoopDispositions: { %loop: Variant }
 ; CHECK-NEXT:    %iv.next = add i64 %iv, 1
 ; CHECK-NEXT:    --> {1,+,1}<%loop> U: [1,6) S: [1,6) Exits: 5 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %shiftamt = and i64 %iv, 1
diff --git a/llvm/test/Analysis/ScalarEvolution/zext-wrap.ll b/llvm/test/Analysis/ScalarEvolution/zext-wrap.ll
index 66bedcea7edf2790889ed0c86549355ccb49c988..d52589a15dde03001a217014a273e7c659cb0dff 100644
--- a/llvm/test/Analysis/ScalarEvolution/zext-wrap.ll
+++ b/llvm/test/Analysis/ScalarEvolution/zext-wrap.ll
@@ -15,7 +15,7 @@ bb.i:           ; preds = %bb1.i, %bb.nph
 
 ; This cast shouldn't be folded into the addrec.
 ; CHECK: %tmp = zext i8 %l_95.0.i1 to i16
-; CHECK: -->  (zext i8 {0,+,-1}<nw><%bb.i> to i16){{ U: [^ ]+ S: [^ ]+}}{{ *}}Exits: 2
+; CHECK: -->  (zext i8 {0,+,-1}<%bb.i> to i16){{ U: [^ ]+ S: [^ ]+}}{{ *}}Exits: 2
 
         %tmp = zext i8 %l_95.0.i1 to i16
 
diff --git a/llvm/test/Analysis/ValueTracking/deref-abstract-gc.ll b/llvm/test/Analysis/ValueTracking/deref-abstract-gc.ll
new file mode 100644
index 0000000000000000000000000000000000000000..70fd9eda5f8f2e420cbf6c9b9ecab911a5bfa6de
--- /dev/null
+++ b/llvm/test/Analysis/ValueTracking/deref-abstract-gc.ll
@@ -0,0 +1,24 @@
+; RUN: opt -print-memderefs -analyze -S < %s -enable-new-pm=0 -use-dereferenceable-at-point-semantics=1 | FileCheck %s --check-prefixes=CHECK
+; RUN: opt -passes=print-memderefs -S < %s -disable-output  -use-dereferenceable-at-point-semantics=1 2>&1 | FileCheck %s --check-prefixes=CHECK
+
+target datalayout = "e-i32:32:64"
+
+; For the abstract machine model (before RS4GC), gc managed objects
+; conceptually live forever.  But there may be non-managed objects which are
+; freed.
+; CHECK-LABEL: 'abstract_model'
+; CHECK: %gc_ptr
+; CHECK-NOT: %other_ptr
+; FIXME: Can infer the gc pointer case
+define void @abstract_model(i32 addrspace(1)* dereferenceable(8) %gc_ptr,
+                            i32* dereferenceable(8) %other_ptr)
+    gc "statepoint-example" {
+entry:
+  call void @mayfree()
+  load i32, i32 addrspace(1)* %gc_ptr
+  load i32, i32* %other_ptr
+  ret void
+}
+
+; Can free any object accessible in memory
+declare void @mayfree()
diff --git a/llvm/test/Analysis/ValueTracking/deref-bitcast-of-gep.ll b/llvm/test/Analysis/ValueTracking/deref-bitcast-of-gep.ll
index 18ee280dbe3d869e858a14ab50e5f0c7aa78c011..23b4614fca8137b1cc61e8814ad575445e5b6d22 100644
--- a/llvm/test/Analysis/ValueTracking/deref-bitcast-of-gep.ll
+++ b/llvm/test/Analysis/ValueTracking/deref-bitcast-of-gep.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -licm < %s | FileCheck %s
 
 ; Note: the !invariant.load is there just solely to let us call @use()
@@ -10,12 +11,16 @@ declare void @use(i32)
 
 define void @f_0(i8* align 4 dereferenceable(1024) %ptr) {
 ; CHECK-LABEL: @f_0(
-; CHECK: entry:
-; CHECK:  %val = load i32, i32* %ptr.i32
-; CHECK:  br label %loop
-; CHECK: loop:
-; CHECK:  call void @use(i32 0)
-; CHECK-NEXT:  call void @use(i32 %val)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR_GEP:%.*]] = getelementptr i8, i8* [[PTR:%.*]], i32 32
+; CHECK-NEXT:    [[PTR_I32:%.*]] = bitcast i8* [[PTR_GEP]] to i32*
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[PTR_I32]], align 4
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @use(i32 0)
+; CHECK-NEXT:    call void @use(i32 [[VAL]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
 
 
 entry:
@@ -32,18 +37,27 @@ loop:
 
 define void @f_1(i8* align 4 dereferenceable_or_null(1024) %ptr) {
 ; CHECK-LABEL: @f_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR_GEP:%.*]] = getelementptr i8, i8* [[PTR:%.*]], i32 32
+; CHECK-NEXT:    [[PTR_I32:%.*]] = bitcast i8* [[PTR_GEP]] to i32*
+; CHECK-NEXT:    [[PTR_IS_NULL:%.*]] = icmp eq i8* [[PTR]], null
+; CHECK-NEXT:    br i1 [[PTR_IS_NULL]], label [[LEAVE:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[PTR_I32]], align 4
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @use(i32 0)
+; CHECK-NEXT:    call void @use(i32 [[VAL]])
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       leave:
+; CHECK-NEXT:    ret void
+;
 entry:
   %ptr.gep = getelementptr i8, i8* %ptr, i32 32
   %ptr.i32 = bitcast i8* %ptr.gep to i32*
   %ptr_is_null = icmp eq i8* %ptr, null
   br i1 %ptr_is_null, label %leave, label %loop
 
-; CHECK: loop.preheader:
-; CHECK:   %val = load i32, i32* %ptr.i32
-; CHECK:   br label %loop
-; CHECK: loop:
-; CHECK:  call void @use(i32 0)
-; CHECK-NEXT:  call void @use(i32 %val)
 
 loop:
   call void @use(i32 0)
@@ -57,10 +71,21 @@ leave:
 
 define void @f_2(i8* align 4 dereferenceable_or_null(1024) %ptr) {
 ; CHECK-LABEL: @f_2(
-; CHECK-NOT: load
-; CHECK:  call void @use(i32 0)
-; CHECK-NEXT:  %val = load i32, i32* %ptr.i32, align 4, !invariant.load !0
-; CHECK-NEXT:  call void @use(i32 %val)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR_GEP:%.*]] = getelementptr i8, i8* [[PTR:%.*]], i32 30
+; CHECK-NEXT:    [[PTR_I32:%.*]] = bitcast i8* [[PTR_GEP]] to i32*
+; CHECK-NEXT:    [[PTR_IS_NULL:%.*]] = icmp eq i8* [[PTR]], null
+; CHECK-NEXT:    br i1 [[PTR_IS_NULL]], label [[LEAVE:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @use(i32 0)
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[PTR_I32]], align 4, !invariant.load !0
+; CHECK-NEXT:    call void @use(i32 [[VAL]])
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       leave:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   ;; Can't hoist, since the alignment does not work out -- (<4 byte
@@ -83,13 +108,15 @@ leave:
 
 define void @checkLaunder(i8* align 4 dereferenceable(1024) %p) {
 ; CHECK-LABEL: @checkLaunder(
-; CHECK: entry:
-; CHECK:  %l = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
-; CHECK:  %val = load i8, i8* %l
-; CHECK:  br label %loop
-; CHECK: loop:
-; CHECK:  call void @use(i32 0)
-; CHECK-NEXT:  call void @use8(i8 %val)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L:%.*]] = call i8* @llvm.launder.invariant.group.p0i8(i8* [[P:%.*]])
+; CHECK-NEXT:    [[VAL:%.*]] = load i8, i8* [[L]], align 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @use(i32 0)
+; CHECK-NEXT:    call void @use8(i8 [[VAL]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
 
 entry:
   %l = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
diff --git a/llvm/test/Analysis/ValueTracking/known-non-equal.ll b/llvm/test/Analysis/ValueTracking/known-non-equal.ll
index 197088a85e8145411bf18920c1fbbd4eaf0fa468..09420455164601528ae078c0547e17708882ae6e 100644
--- a/llvm/test/Analysis/ValueTracking/known-non-equal.ll
+++ b/llvm/test/Analysis/ValueTracking/known-non-equal.ll
@@ -214,4 +214,275 @@ define i1 @mul_constantexpr(i16 %a) {
   ret i1 %cmp
 }
 
+define i1 @mul_nuw(i16 %x) {
+; CHECK-LABEL: @mul_nuw(
+; CHECK-NEXT:    ret i1 false
+;
+  %nz = or i16 %x, 2
+  %mul = mul nuw i16 %nz, 2
+  %cmp = icmp eq i16 %nz, %mul
+  ret i1 %cmp
+}
+
+define i1 @mul_nuw_comm(i16 %x) {
+; CHECK-LABEL: @mul_nuw_comm(
+; CHECK-NEXT:    ret i1 false
+;
+  %nz = or i16 %x, 2
+  %mul = mul nuw i16 %nz, 2
+  %cmp = icmp eq i16 %mul, %nz
+  ret i1 %cmp
+}
+
+define i1 @mul_nsw(i16 %x) {
+; CHECK-LABEL: @mul_nsw(
+; CHECK-NEXT:    ret i1 false
+;
+  %nz = or i16 %x, 2
+  %mul = mul nsw i16 %nz, 2
+  %cmp = icmp eq i16 %nz, %mul
+  ret i1 %cmp
+}
+
+define i1 @mul_nsw_comm(i16 %x) {
+; CHECK-LABEL: @mul_nsw_comm(
+; CHECK-NEXT:    ret i1 false
+;
+  %nz = or i16 %x, 2
+  %mul = mul nsw i16 %nz, 2
+  %cmp = icmp eq i16 %mul, %nz
+  ret i1 %cmp
+}
+
+define i1 @mul_may_wrap(i16 %x) {
+; CHECK-LABEL: @mul_may_wrap(
+; CHECK-NEXT:    [[NZ:%.*]] = or i16 [[X:%.*]], 2
+; CHECK-NEXT:    [[MUL:%.*]] = mul i16 [[NZ]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[NZ]], [[MUL]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %nz = or i16 %x, 2
+  %mul = mul i16 %nz, 2
+  %cmp = icmp eq i16 %nz, %mul
+  ret i1 %cmp
+}
+
+define i1 @mul_may_be_zero(i16 %x) {
+; CHECK-LABEL: @mul_may_be_zero(
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i16 [[X:%.*]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[X]], [[MUL]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul nuw i16 %x, 2
+  %cmp = icmp eq i16 %x, %mul
+  ret i1 %cmp
+}
+
+define i1 @mul_other_may_be_zero_or_one(i16 %x, i16 %y) {
+; CHECK-LABEL: @mul_other_may_be_zero_or_one(
+; CHECK-NEXT:    [[NZ:%.*]] = or i16 [[X:%.*]], 2
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i16 [[NZ]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[NZ]], [[MUL]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %nz = or i16 %x, 2
+  %mul = mul nuw i16 %nz, %y
+  %cmp = icmp eq i16 %nz, %mul
+  ret i1 %cmp
+}
+
+define i1 @known_non_equal_phis(i8 %p, i8* %pq, i8 %n, i8 %r) {
+; CHECK-LABEL: @known_non_equal_phis(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = mul nsw i8 [[A]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 2, %entry ], [ %next, %loop ]
+  %B = phi i8 [ 3, %entry ], [ %A, %loop ]
+  %next = mul nsw i8 %A, 2
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp ne i8 %A, %B
+  ret i1 %cmp
+}
+
+define i1 @known_non_equal_phis_fail(i8 %p, i8* %pq, i8 %n, i8 %r) {
+; CHECK-LABEL: @known_non_equal_phis_fail(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[B:%.*]] = phi i8 [ 2, [[ENTRY]] ], [ [[A]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = mul nsw i8 [[A]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 2, %entry ], [ %next, %loop ]
+  %B = phi i8 [ 2, %entry ], [ %A, %loop ]
+  %next = mul nsw i8 %A, 2
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp ne i8 %A, %B
+  ret i1 %cmp
+}
+
+define i1 @shl_nuw(i16 %x) {
+; CHECK-LABEL: @shl_nuw(
+; CHECK-NEXT:    ret i1 false
+;
+  %nz = or i16 %x, 2
+  %mul = shl nuw i16 %nz, 1
+  %cmp = icmp eq i16 %nz, %mul
+  ret i1 %cmp
+}
+
+define i1 @shl_nsw(i16 %x) {
+; CHECK-LABEL: @shl_nsw(
+; CHECK-NEXT:    ret i1 false
+;
+  %nz = or i16 %x, 2
+  %mul = shl nsw i16 %nz, 1
+  %cmp = icmp eq i16 %nz, %mul
+  ret i1 %cmp
+}
+
+define i1 @shl_may_wrap(i16 %x) {
+; CHECK-LABEL: @shl_may_wrap(
+; CHECK-NEXT:    [[NZ:%.*]] = or i16 [[X:%.*]], 2
+; CHECK-NEXT:    [[MUL:%.*]] = shl i16 [[NZ]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[NZ]], [[MUL]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %nz = or i16 %x, 2
+  %mul = shl i16 %nz, 1
+  %cmp = icmp eq i16 %nz, %mul
+  ret i1 %cmp
+}
+
+define i1 @shl_shift_may_be_zero(i16 %x, i16 %shift) {
+; CHECK-LABEL: @shl_shift_may_be_zero(
+; CHECK-NEXT:    [[NZ:%.*]] = or i16 [[X:%.*]], 2
+; CHECK-NEXT:    [[MUL:%.*]] = shl nuw i16 [[NZ]], [[SHIFT:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[NZ]], [[MUL]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %nz = or i16 %x, 2
+  %mul = shl nuw i16 %nz, %shift
+  %cmp = icmp eq i16 %nz, %mul
+  ret i1 %cmp
+}
+
+define i1 @shl_op_may_be_zero(i16 %x) {
+; CHECK-LABEL: @shl_op_may_be_zero(
+; CHECK-NEXT:    [[MUL:%.*]] = shl nuw i16 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[X]], [[MUL]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = shl nuw i16 %x, 1
+  %cmp = icmp eq i16 %x, %mul
+  ret i1 %cmp
+}
+
+; The additional muls in these tests are necessary to actually
+; test the isKnownNonEqual() code, rather than InstSimplify's own
+; comparison folding.
+
+define i1 @shl_shl_nuw(i8 %B, i8 %shift) {
+; CHECK-LABEL: @shl_shl_nuw(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add i8 %B, 1
+  %A.op = shl nuw i8 %A, %shift
+  %B.op = shl nuw i8 %B, %shift
+  %A.op2 = mul nuw i8 %A.op, 3
+  %B.op2 = mul nuw i8 %B.op, 3
+  %cmp = icmp eq i8 %A.op2, %B.op2
+  ret i1 %cmp
+}
+
+define i1 @shl_shl_nsw(i8 %B, i8 %shift) {
+; CHECK-LABEL: @shl_shl_nsw(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add i8 %B, 1
+  %A.op = shl nsw i8 %A, %shift
+  %B.op = shl nsw i8 %B, %shift
+  %A.op2 = mul nuw i8 %A.op, 3
+  %B.op2 = mul nuw i8 %B.op, 3
+  %cmp = icmp eq i8 %A.op2, %B.op2
+  ret i1 %cmp
+}
+
+define i1 @shl_shl_may_wrap(i8 %B, i8 %shift) {
+; CHECK-LABEL: @shl_shl_may_wrap(
+; CHECK-NEXT:    [[A:%.*]] = add i8 [[B:%.*]], 1
+; CHECK-NEXT:    [[A_OP:%.*]] = shl i8 [[A]], [[SHIFT:%.*]]
+; CHECK-NEXT:    [[B_OP:%.*]] = shl nsw i8 [[B]], [[SHIFT]]
+; CHECK-NEXT:    [[A_OP2:%.*]] = mul nuw i8 [[A_OP]], 3
+; CHECK-NEXT:    [[B_OP2:%.*]] = mul nuw i8 [[B_OP]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A_OP2]], [[B_OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %A = add i8 %B, 1
+  %A.op = shl i8 %A, %shift
+  %B.op = shl nsw i8 %B, %shift
+  %A.op2 = mul nuw i8 %A.op, 3
+  %B.op2 = mul nuw i8 %B.op, 3
+  %cmp = icmp eq i8 %A.op2, %B.op2
+  ret i1 %cmp
+}
+
+define i1 @shl_shl_mixed_wrap(i8 %B, i8 %shift) {
+; CHECK-LABEL: @shl_shl_mixed_wrap(
+; CHECK-NEXT:    [[A:%.*]] = add i8 [[B:%.*]], 1
+; CHECK-NEXT:    [[A_OP:%.*]] = shl nuw i8 [[A]], [[SHIFT:%.*]]
+; CHECK-NEXT:    [[B_OP:%.*]] = shl nsw i8 [[B]], [[SHIFT]]
+; CHECK-NEXT:    [[A_OP2:%.*]] = mul nuw i8 [[A_OP]], 3
+; CHECK-NEXT:    [[B_OP2:%.*]] = mul nuw i8 [[B_OP]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A_OP2]], [[B_OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %A = add i8 %B, 1
+  %A.op = shl nuw i8 %A, %shift
+  %B.op = shl nsw i8 %B, %shift
+  %A.op2 = mul nuw i8 %A.op, 3
+  %B.op2 = mul nuw i8 %B.op, 3
+  %cmp = icmp eq i8 %A.op2, %B.op2
+  ret i1 %cmp
+}
+
+define i1 @shl_shl_may_be_equal(i8 %A, i8 %B, i8 %shift) {
+; CHECK-LABEL: @shl_shl_may_be_equal(
+; CHECK-NEXT:    [[A_OP:%.*]] = shl nuw i8 [[A:%.*]], [[SHIFT:%.*]]
+; CHECK-NEXT:    [[B_OP:%.*]] = shl nuw i8 [[B:%.*]], [[SHIFT]]
+; CHECK-NEXT:    [[A_OP2:%.*]] = mul nuw i8 [[A_OP]], 3
+; CHECK-NEXT:    [[B_OP2:%.*]] = mul nuw i8 [[B_OP]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A_OP2]], [[B_OP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %A.op = shl nuw i8 %A, %shift
+  %B.op = shl nuw i8 %B, %shift
+  %A.op2 = mul nuw i8 %A.op, 3
+  %B.op2 = mul nuw i8 %B.op, 3
+  %cmp = icmp eq i8 %A.op2, %B.op2
+  ret i1 %cmp
+}
+
 !0 = !{ i8 1, i8 5 }
diff --git a/llvm/test/Analysis/ValueTracking/memory-dereferenceable.ll b/llvm/test/Analysis/ValueTracking/memory-dereferenceable.ll
index 95ba9f2a1ddf9d6f8e43c8b713ad7096f0b71b1a..231fa83417c20e545d1da5f388ded0c030296e24 100644
--- a/llvm/test/Analysis/ValueTracking/memory-dereferenceable.ll
+++ b/llvm/test/Analysis/ValueTracking/memory-dereferenceable.ll
@@ -1,5 +1,8 @@
-; RUN: opt -print-memderefs -analyze -S < %s -enable-new-pm=0 | FileCheck %s
-; RUN: opt -passes=print-memderefs -S < %s -disable-output 2>&1 | FileCheck %s
+; RUN: opt -print-memderefs -analyze -S < %s -enable-new-pm=0 -use-dereferenceable-at-point-semantics=0 | FileCheck %s --check-prefixes=CHECK,GLOBAL
+; RUN: opt -passes=print-memderefs -S < %s -disable-output  -use-dereferenceable-at-point-semantics=0 2>&1 | FileCheck %s --check-prefixes=CHECK,GLOBAL
+; RUN: opt -print-memderefs -analyze -S < %s -enable-new-pm=0 -use-dereferenceable-at-point-semantics=1 | FileCheck %s --check-prefixes=CHECK,POINT
+; RUN: opt -passes=print-memderefs -S < %s -disable-output  -use-dereferenceable-at-point-semantics=1 2>&1 | FileCheck %s --check-prefixes=CHECK,POINT
+
 
 ; Uses the print-deref (+ analyze to print) pass to run
 ; isDereferenceablePointer() on many load instruction operands
@@ -20,42 +23,34 @@ declare i32* @foo()
 @globalptr.align1 = external global i8, align 1
 @globalptr.align16 = external global i8, align 16
 
+; Loads from sret arguments
+; CHECK-LABEL: 'test_sret'
+; CHECK: %sret_gep{{.*}}(aligned)
+; CHECK-NOT: %sret_gep_outside
+define void @test_sret(%struct.A* sret(%struct.A) %result) {
+  %sret_gep = getelementptr inbounds %struct.A, %struct.A* %result, i64 0, i32 1, i64 2
+  load i8, i8* %sret_gep
+
+  %sret_gep_outside = getelementptr %struct.A, %struct.A* %result, i64 0, i32 1, i64 7
+  load i8, i8* %sret_gep_outside
+  ret void
+}
+
 ; CHECK-LABEL: 'test'
-define void @test(%struct.A* sret(%struct.A) %result,
-                  i32 addrspace(1)* dereferenceable(8) %dparam,
+define void @test(i32 addrspace(1)* dereferenceable(8) %dparam,
                   i8 addrspace(1)* dereferenceable(32) align 1 %dparam.align1,
-                  i8 addrspace(1)* dereferenceable(32) align 16 %dparam.align16,
-                  i8* byval(i8) %i8_byval,
-                  %struct.A* byval(%struct.A) %A_byval)
+                  i8 addrspace(1)* dereferenceable(32) align 16 %dparam.align16)
     gc "statepoint-example" {
 ; CHECK: The following are dereferenceable:
 entry:
-; CHECK: %globalptr{{.*}}(aligned)
-    %globalptr = getelementptr inbounds [6 x i8], [6 x i8]* @globalstr, i32 0, i32 0
-    %load1 = load i8, i8* %globalptr
-
-; CHECK: %alloca{{.*}}(aligned)
-    %alloca = alloca i1
-    %load2 = load i1, i1* %alloca
-
-    ; Load from empty array alloca
-; CHECK-NOT: %empty_alloca
-    %empty_alloca = alloca i8, i64 0
-    %empty_load = load i8, i8* %empty_alloca
-
-    ; Loads from sret arguments
-; CHECK: %sret_gep{{.*}}(aligned)
-    %sret_gep = getelementptr inbounds %struct.A, %struct.A* %result, i64 0, i32 1, i64 2
-    load i8, i8* %sret_gep
+  call void @mayfree()
 
-; CHECK-NOT: %sret_gep_outside
-    %sret_gep_outside = getelementptr %struct.A, %struct.A* %result, i64 0, i32 1, i64 7
-    load i8, i8* %sret_gep_outside
-
-; CHECK: %dparam{{.*}}(unaligned)
+; GLOBAL: %dparam{{.*}}(unaligned)
+; POINT-NOT: %dparam{{.*}}(unaligned)
     %load3 = load i32, i32 addrspace(1)* %dparam
 
-; CHECK: %relocate{{.*}}(unaligned)
+; GLOBAL: %relocate{{.*}}(unaligned)
+; POINT-NOT: %relocate{{.*}}(unaligned)
     %tok = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32 addrspace(1)* %dparam)]
     %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 0, i32 0)
     %load4 = load i32, i32 addrspace(1)* %relocate
@@ -71,7 +66,8 @@ entry:
     %load6 = load i32, i32* %nd_load
 
     ; Load from a dereferenceable load
-; CHECK: %d4_load{{.*}}(unaligned)
+; GLOBAL: %d4_load{{.*}}(unaligned)
+; POINT-NOT: %d4_load{{.*}}(unaligned)
     %d4_load = load i32*, i32** @globali32ptr, !dereferenceable !0
     %load7 = load i32, i32* %d4_load
 
@@ -86,57 +82,28 @@ entry:
     %load9 = load i32, i32* %d_or_null_load
 
     ; Load from a non-null pointer with dereferenceable_or_null
-; CHECK: %d_or_null_non_null_load{{.*}}(unaligned)
+; GLOBAL: %d_or_null_non_null_load{{.*}}(unaligned)
+; POINT-NOT: %d_or_null_non_null_load{{.*}}(unaligned)
     %d_or_null_non_null_load = load i32*, i32** @globali32ptr, !nonnull !2, !dereferenceable_or_null !0
     %load10 = load i32, i32* %d_or_null_non_null_load
 
-    ; It's OK to overrun static array size as long as we stay within underlying object size
-; CHECK: %within_allocation{{.*}}(aligned)
-    %within_allocation = getelementptr inbounds %struct.A, %struct.A* @globalstruct, i64 0, i32 0, i64 10
-    %load11 = load i8, i8* %within_allocation
-
-    ; GEP is outside the underlying object size
-; CHECK-NOT: %outside_allocation
-    %outside_allocation = getelementptr inbounds %struct.A, %struct.A* @globalstruct, i64 0, i32 1, i64 10
-    %load12 = load i8, i8* %outside_allocation
-
-    ; Loads from aligned globals
-; CHECK: @globalptr.align1{{.*}}(unaligned)
-; CHECK: @globalptr.align16{{.*}}(aligned)
-    %load13 = load i8, i8* @globalptr.align1, align 16
-    %load14 = load i8, i8* @globalptr.align16, align 16
-
     ; Loads from aligned arguments
-; CHECK: %dparam.align1{{.*}}(unaligned)
-; CHECK: %dparam.align16{{.*}}(aligned)
+; GLOBAL: %dparam.align1{{.*}}(unaligned)
+; POINT-NOT: %dparam.align1{{.*}}(unaligned)
+; POINT-NOT: %dparam.align16{{.*}}(aligned)
+; GLOBAL: %dparam.align16{{.*}}(aligned)
     %load15 = load i8, i8 addrspace(1)* %dparam.align1, align 16
     %load16 = load i8, i8 addrspace(1)* %dparam.align16, align 16
 
-    ; Loads from byval arguments
-; CHECK: %i8_byval{{.*}}(aligned)
-    %i8_byval_load = load i8, i8* %i8_byval
-
-; CHECK-NOT: %byval_cast
-    %byval_cast = bitcast i8* %i8_byval to i32*
-    %bad_byval_load = load i32, i32* %byval_cast
-
-; CHECK: %byval_gep{{.*}}(aligned)
-    %byval_gep = getelementptr inbounds %struct.A, %struct.A* %A_byval, i64 0, i32 1, i64 2
-    load i8, i8* %byval_gep
-
-    ; Loads from aligned allocas
-; CHECK: %alloca.align1{{.*}}(unaligned)
-; CHECK: %alloca.align16{{.*}}(aligned)
-    %alloca.align1 = alloca i1, align 1
-    %alloca.align16 = alloca i1, align 16
-    %load17 = load i1, i1* %alloca.align1, align 16
-    %load18 = load i1, i1* %alloca.align16, align 16
-
     ; Loads from GEPs
-; CHECK: %gep.align1.offset1{{.*}}(unaligned)
-; CHECK: %gep.align16.offset1{{.*}}(unaligned)
-; CHECK: %gep.align1.offset16{{.*}}(unaligned)
-; CHECK: %gep.align16.offset16{{.*}}(aligned)
+; GLOBAL: %gep.align1.offset1{{.*}}(unaligned)
+; GLOBAL: %gep.align16.offset1{{.*}}(unaligned)
+; GLOBAL: %gep.align1.offset16{{.*}}(unaligned)
+; GLOBAL: %gep.align16.offset16{{.*}}(aligned)
+; POINT-NOT: %gep.align1.offset1{{.*}}(unaligned)
+; POINT-NOT: %gep.align16.offset1{{.*}}(unaligned)
+; POINT-NOT: %gep.align1.offset16{{.*}}(unaligned)
+; POINT-NOT: %gep.align16.offset16{{.*}}(aligned)
     %gep.align1.offset1 = getelementptr inbounds i8, i8 addrspace(1)* %dparam.align1, i32 1
     %gep.align16.offset1 = getelementptr inbounds i8, i8 addrspace(1)* %dparam.align16, i32 1
     %gep.align1.offset16 = getelementptr inbounds i8, i8 addrspace(1)* %dparam.align1, i32 16
@@ -147,8 +114,10 @@ entry:
     %load22 = load i8, i8 addrspace(1)* %gep.align16.offset16, align 16
 
 ; CHECK-NOT: %no_deref_return
-; CHECK: %deref_return{{.*}}(unaligned)
-; CHECK: %deref_and_aligned_return{{.*}}(aligned)
+; GLOBAL: %deref_return{{.*}}(unaligned)
+; GLOBAL: %deref_and_aligned_return{{.*}}(aligned)
+; POINT-NOT: %deref_return{{.*}}(unaligned)
+; POINT-NOT: %deref_and_aligned_return{{.*}}(aligned)
     %no_deref_return = call i32* @foo()
     %deref_return = call dereferenceable(32) i32* @foo()
     %deref_and_aligned_return = call dereferenceable(32) align 16 i32* @foo()
@@ -157,30 +126,170 @@ entry:
     %load25 = load i32, i32* %deref_and_aligned_return, align 16
 
     ; Load from a dereferenceable and aligned load
-; CHECK: %d4_unaligned_load{{.*}}(unaligned)
-; CHECK: %d4_aligned_load{{.*}}(aligned)
+; GLOBAL: %d4_unaligned_load{{.*}}(unaligned)
+; GLOBAL: %d4_aligned_load{{.*}}(aligned)
+; POINT-NOT: %d4_unaligned_load{{.*}}(unaligned)
+; POINT-NOT: %d4_aligned_load{{.*}}(aligned)
     %d4_unaligned_load = load i32*, i32** @globali32ptr, !dereferenceable !0
     %d4_aligned_load = load i32*, i32** @globali32ptr, !dereferenceable !0, !align !{i64 16}
     %load26 = load i32, i32* %d4_unaligned_load, align 16
     %load27 = load i32, i32* %d4_aligned_load, align 16
+    ret void
+}
 
-   ; Alloca with no explicit alignment is aligned to preferred alignment of
-   ; the type (specified by datalayout string).
+; Loads from aligned allocas
+; CHECK-LABEL: 'alloca_aligned'
+; CHECK: %alloca.align1{{.*}}(unaligned)
+; CHECK: %alloca.align16{{.*}}(aligned)
+define void @alloca_aligned() {
+   %alloca.align1 = alloca i1, align 1
+   %alloca.align16 = alloca i1, align 16
+   call void @mayfree()
+   %load17 = load i1, i1* %alloca.align1, align 16
+   %load18 = load i1, i1* %alloca.align16, align 16
+   ret void
+}
+
+; CHECK-LABEL: 'alloca_basic'
+; CHECK: %alloca{{.*}}(aligned)
+define void @alloca_basic() {
+  %alloca = alloca i1
+  call void @mayfree()
+  %load2 = load i1, i1* %alloca
+  ret void
+}
+
+; Load from empty array alloca
+; CHECK-LABEL: 'alloca_empty'
+; CHECK-NOT: %empty_alloca
+define void @alloca_empty() {
+  %empty_alloca = alloca i8, i64 0
+  call void @mayfree()
+  %empty_load = load i8, i8* %empty_alloca
+  ret void
+}
+
+; Alloca with no explicit alignment is aligned to preferred alignment of
+; the type (specified by datalayout string).
+; CHECK-LABEL: 'alloca_perfalign'
 ; CHECK: %alloca.noalign{{.*}}(aligned)
-    %alloca.noalign = alloca i32
-    %load28 = load i32, i32* %alloca.noalign, align 8
+define void @alloca_perfalign() {
+   %alloca.noalign = alloca i32
+   call void @mayfree()
+   %load28 = load i32, i32* %alloca.noalign, align 8
+   ret void
+}
 
-    ret void
+; CHECK-LABEL: 'global'
+; CHECK: @globalptr.align1{{.*}}(unaligned)
+; CHECK: @globalptr.align16{{.*}}(aligned)
+; CHECK: %globalptr{{.*}}(aligned)
+define void @global() {
+  %load13 = load i8, i8* @globalptr.align1, align 16
+  %load14 = load i8, i8* @globalptr.align16, align 16
+
+  %globalptr = getelementptr inbounds [6 x i8], [6 x i8]* @globalstr, i32 0, i32 0
+  %load1 = load i8, i8* %globalptr
+  ret void
 }
 
-; CHECK: The following are dereferenceable:
-; CHECK: %ptr = inttoptr i32 %val to i32*, !dereferenceable !0
+; It's OK to overrun static array size as long as we stay within underlying
+; object size
+; CHECK-LABEL: 'global_allocationsize'
+; CHECK: %within_allocation{{.*}}(aligned)
+; CHECK-NOT: %outside_allocation
+define void @global_allocationsize() {
+  %within_allocation = getelementptr inbounds %struct.A, %struct.A* @globalstruct, i64 0, i32 0, i64 10
+  %load11 = load i8, i8* %within_allocation
+
+  %outside_allocation = getelementptr inbounds %struct.A, %struct.A* @globalstruct, i64 0, i32 1, i64 10
+  %load12 = load i8, i8* %outside_allocation
+  ret void
+}
+
+; Loads from byval arguments
+; CHECK-LABEL: 'byval'
+; CHECK: %i8_byval{{.*}}(aligned)
+; CHECK-NOT: %bad_byval_cast
+; CHECK: %byval_gep{{.*}}(aligned)
+; CHECK: %good_byval_cast{{.*}}(unaligned)
+define void @byval(i8* byval(i8) %i8_byval,
+                   %struct.A* byval(%struct.A) %A_byval) {
+  call void @mayfree()
+  load i8, i8* %i8_byval
+
+  %bad_byval_cast = bitcast i8* %i8_byval to i32*
+  load i32, i32* %bad_byval_cast
+
+  %byval_gep = getelementptr inbounds %struct.A, %struct.A* %A_byval, i64 0, i32 1, i64 2
+  load i8, i8* %byval_gep
+  %good_byval_cast = bitcast %struct.A* %A_byval to i32*
+  load i32, i32* %good_byval_cast
+
+  ret void
+}
+
+; CHECK-LABEL: 'f_0'
+; GLOBAL: %ptr = inttoptr i32 %val to i32*, !dereferenceable !0
+; POINT-NOT: %ptr = inttoptr i32 %val to i32*, !dereferenceable !0
 define i32 @f_0(i32 %val) {
   %ptr = inttoptr i32 %val to i32*, !dereferenceable !0
+  call void @mayfree()
   %load29 = load i32, i32* %ptr, align 8
   ret i32 %load29
 }
 
+
+; The most basic case showing the difference between legacy global deref
+; attribute semantics and the new point-in-time semantics.
+; CHECK-LABEL: 'negative'
+; GLOBAL: %p
+; POINT-NOT: %p
+define void @negative(i32* dereferenceable(8) %p) {
+  call void @mayfree()
+  %v = load i32, i32* %p
+  ret void
+}
+
+; CHECK-LABEL: 'infer_func_attrs1'
+; CHECK: %p
+define void @infer_func_attrs1(i32* dereferenceable(8) %p) nofree nosync {
+  call void @mayfree()
+  %v = load i32, i32* %p
+  ret void
+}
+
+; CHECK-LABEL: 'infer_func_attrs2'
+; GLOBAL: %p
+; POINT-NOT: %p
+; FIXME: Can be inferred from attributes
+define void @infer_func_attrs2(i32* dereferenceable(8) %p) readonly {
+  call void @mayfree()
+  %v = load i32, i32* %p
+  ret void
+}
+
+; CHECK-LABEL: 'infer_noalias1'
+; GLOBAL: %p
+; POINT-NOT: %p
+; FIXME: Can be inferred from attributes
+define void @infer_noalias1(i32* dereferenceable(8) noalias nofree %p) {
+  call void @mayfree()
+  %v = load i32, i32* %p
+  ret void
+}
+
+; CHECK-LABEL: 'infer_noalias2'
+; GLOBAL: %p
+; POINT-NOT: %p
+; FIXME: Can be inferred from attributes
+define void @infer_noalias2(i32* dereferenceable(8) noalias readonly %p) nosync {
+  call void @mayfree()
+  %v = load i32, i32* %p
+  ret void
+}
+
+
 ; Just check that we don't crash.
 ; CHECK-LABEL: 'opaque_type_crasher'
 define void @opaque_type_crasher(%TypeOpaque* dereferenceable(16) %a) {
@@ -203,6 +312,9 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
 
 declare i32 addrspace(1)* @func1(i32 addrspace(1)* returned) nounwind argmemonly
 
+; Can free any object accessible in memory
+declare void @mayfree()
+
 !0 = !{i64 4}
 !1 = !{i64 2}
 !2 = !{}
diff --git a/llvm/test/Analysis/ValueTracking/monotonic-phi.ll b/llvm/test/Analysis/ValueTracking/monotonic-phi.ll
index 3204bda49f0b1125466158540a551d01c04ba429..e79e949401dd193ff15a620cd87d910343e764cd 100644
--- a/llvm/test/Analysis/ValueTracking/monotonic-phi.ll
+++ b/llvm/test/Analysis/ValueTracking/monotonic-phi.ll
@@ -1,7 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instsimplify -S < %s | FileCheck %s
 
-; CHECK-LABEL: @test1
-define i1 @test1(i8 %p, i8* %pq, i8 %n, i8 %r) {
+define i1 @test_add_nsw(i8 %p, i8* %pq, i8 %n, i8 %r) {
+; CHECK-LABEL: @test_add_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 1, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = add nsw i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
 entry:
   br label %loop
 loop:
@@ -12,12 +23,23 @@ loop:
 exit:
   %add = or i8 %A, %r
   %cmp = icmp eq i8 %add, 0
-  ; CHECK: ret i1 false
   ret i1 %cmp
 }
 
-; CHECK-LABEL: @test2
-define i1 @test2(i8 %p, i8* %pq, i8 %n, i8 %r) {
+define i1 @test_add_may_wrap(i8 %p, i8* %pq, i8 %n, i8 %r) {
+; CHECK-LABEL: @test_add_may_wrap(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 1, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = add i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[ADD:%.*]] = or i8 [[A]], [[R:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[ADD]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
 entry:
   br label %loop
 loop:
@@ -28,12 +50,21 @@ loop:
 exit:
   %add = or i8 %A, %r
   %cmp = icmp eq i8 %add, 0
-  ; CHECK-NOT: ret i1 false
   ret i1 %cmp
 }
 
-; CHECK-LABEL: @test3
-define i1 @test3(i8 %p, i8* %pq, i8 %n, i8 %r) {
+define i1 @test_add_nuw(i8 %p, i8* %pq, i8 %n, i8 %r) {
+; CHECK-LABEL: @test_add_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 1, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = add nuw i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
 entry:
   br label %loop
 loop:
@@ -44,6 +75,402 @@ loop:
 exit:
   %add = or i8 %A, %r
   %cmp = icmp eq i8 %add, 0
-  ; CHECK: ret i1 false
+  ret i1 %cmp
+}
+
+define i1 @test_add_zero_start(i8 %p, i8* %pq, i8 %n, i8 %r) {
+; CHECK-LABEL: @test_add_zero_start(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = add nuw i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[ADD:%.*]] = or i8 [[A]], [[R:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[ADD]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 0, %entry ], [ %next, %loop ]
+  %next = add nuw i8 %A, 1
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %add = or i8 %A, %r
+  %cmp = icmp eq i8 %add, 0
+  ret i1 %cmp
+}
+
+define i1 @test_add_nuw_negative_start(i8 %p, i8* %pq, i8 %n, i8 %r) {
+; CHECK-LABEL: @test_add_nuw_negative_start(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ -2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = add nuw i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ -2, %entry ], [ %next, %loop ]
+  %next = add nuw i8 %A, 1
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %add = or i8 %A, %r
+  %cmp = icmp eq i8 %add, 0
+  ret i1 %cmp
+}
+
+define i1 @test_add_nsw_negative_start(i8 %p, i8* %pq, i8 %n, i8 %r) {
+; CHECK-LABEL: @test_add_nsw_negative_start(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ -2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = add nsw i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[ADD:%.*]] = or i8 [[A]], [[R:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[ADD]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ -2, %entry ], [ %next, %loop ]
+  %next = add nsw i8 %A, 1
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %add = or i8 %A, %r
+  %cmp = icmp eq i8 %add, 0
+  ret i1 %cmp
+}
+
+define i1 @test_add_nsw_negative_start_and_step(i8 %p, i8* %pq, i8 %n, i8 %r) {
+; CHECK-LABEL: @test_add_nsw_negative_start_and_step(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ -1, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = add nsw i8 [[A]], -1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ -1, %entry ], [ %next, %loop ]
+  %next = add nsw i8 %A, -1
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %add = or i8 %A, %r
+  %cmp = icmp eq i8 %add, 0
+  ret i1 %cmp
+}
+
+define i1 @test_mul_nsw(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_mul_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = mul nsw i8 [[A]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 2, %entry ], [ %next, %loop ]
+  %next = mul nsw i8 %A, 2
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_mul_may_wrap(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_mul_may_wrap(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = mul i8 [[A]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 2, %entry ], [ %next, %loop ]
+  %next = mul i8 %A, 2
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_mul_nuw(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_mul_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = mul nuw i8 [[A]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 2, %entry ], [ %next, %loop ]
+  %next = mul nuw i8 %A, 2
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_mul_zero_start(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_mul_zero_start(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = mul nuw i8 [[A]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 0, %entry ], [ %next, %loop ]
+  %next = mul nuw i8 %A, 2
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_mul_nuw_negative_step(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_mul_nuw_negative_step(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = mul nuw i8 [[A]], -2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 2, %entry ], [ %next, %loop ]
+  %next = mul nuw i8 %A, -2
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_mul_nsw_negative_step(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_mul_nsw_negative_step(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = mul nsw i8 [[A]], -2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 2, %entry ], [ %next, %loop ]
+  %next = mul nsw i8 %A, -2
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_mul_nuw_negative_start(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_mul_nuw_negative_start(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ -2, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = mul nuw i8 [[A]], 2
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ -2, %entry ], [ %next, %loop ]
+  %next = mul nuw i8 %A, 2
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_shl_nuw(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_shl_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 1, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = shl nuw i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 1, %entry ], [ %next, %loop ]
+  %next = shl nuw i8 %A, 1
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_shl_nsw(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_shl_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 1, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = shl nsw i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 1, %entry ], [ %next, %loop ]
+  %next = shl nsw i8 %A, 1
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_shl_dynamic_shift(i8 %p, i8* %pq, i8 %n, i8 %shift) {
+; CHECK-LABEL: @test_shl_dynamic_shift(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 1, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = shl nuw i8 [[A]], [[SHIFT:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 1, %entry ], [ %next, %loop ]
+  %next = shl nuw i8 %A, %shift
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_shl_may_wrap(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_shl_may_wrap(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 1, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = shl i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 1, %entry ], [ %next, %loop ]
+  %next = shl i8 %A, 1
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
+  ret i1 %cmp
+}
+
+define i1 @test_shl_zero_start(i8 %p, i8* %pq, i8 %n) {
+; CHECK-LABEL: @test_shl_zero_start(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[NEXT]] = shl nuw i8 [[A]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  br label %loop
+loop:
+  %A = phi i8 [ 0, %entry ], [ %next, %loop ]
+  %next = shl nuw i8 %A, 1
+  %cmp1 = icmp eq i8 %A, %n
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  %cmp = icmp eq i8 %A, 0
   ret i1 %cmp
 }
diff --git a/llvm/test/Assembler/inalloca-parse-error0.ll b/llvm/test/Assembler/inalloca-parse-error0.ll
new file mode 100644
index 0000000000000000000000000000000000000000..24fe82baaeff2c4dac9e63361a2c8151f6e47adc
--- /dev/null
+++ b/llvm/test/Assembler/inalloca-parse-error0.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:40: error: expected '('{{$}}
+define void @test_inalloca(i8* inalloca) {
+  ret void
+}
diff --git a/llvm/test/Assembler/invalid-immarg.ll b/llvm/test/Assembler/invalid-immarg.ll
index f2203d2609fd3fa10f7f7271c7c701191d5b5c7c..023a528ea7bb0729de7eb80a6c7e1d55d1e712c3 100644
--- a/llvm/test/Assembler/invalid-immarg.ll
+++ b/llvm/test/Assembler/invalid-immarg.ll
@@ -4,7 +4,7 @@
 declare void @llvm.immarg.byval(i32* byval(i32) immarg)
 
 ; CHECK: Attribute 'immarg' is incompatible with other attributes
-declare void @llvm.immarg.inalloca(i32* inalloca immarg)
+declare void @llvm.immarg.inalloca(i32* inalloca(i32) immarg)
 
 ; CHECK: Attribute 'immarg' is incompatible with other attributes
 declare void @llvm.immarg.inreg(i32 inreg immarg)
diff --git a/clang/test/Driver/Inputs/multilib_armhf_linux_tree/usr/include/arm-linux-gnueabihf/.keep b/llvm/test/Bindings/OCaml/.ocamlformat
similarity index 100%
rename from clang/test/Driver/Inputs/multilib_armhf_linux_tree/usr/include/arm-linux-gnueabihf/.keep
rename to llvm/test/Bindings/OCaml/.ocamlformat
diff --git a/llvm/test/Bindings/OCaml/Utils/Testsuite.ml b/llvm/test/Bindings/OCaml/Utils/Testsuite.ml
new file mode 100644
index 0000000000000000000000000000000000000000..7a8955a7a2fd842d448a2657d79b3d6c6b05fd31
--- /dev/null
+++ b/llvm/test/Bindings/OCaml/Utils/Testsuite.ml
@@ -0,0 +1,33 @@
+(* Tiny unit test framework - really just to help find which line is busted *)
+let exit_status = ref 0
+
+let suite_name = ref ""
+
+let group_name = ref ""
+
+let case_num = ref 0
+
+let print_checkpoints = false
+
+let group name =
+  group_name := !suite_name ^ "/" ^ name;
+  case_num := 0;
+  if print_checkpoints then prerr_endline ("  " ^ name ^ "...")
+
+let insist ?(exit_on_fail = false) cond =
+  incr case_num;
+  if not cond then exit_status := 10;
+  ( match (print_checkpoints, cond) with
+  | false, true -> ()
+  | false, false ->
+      prerr_endline
+        ( "FAILED: " ^ !suite_name ^ "/" ^ !group_name ^ " #"
+        ^ string_of_int !case_num )
+  | true, true -> prerr_endline ("    " ^ string_of_int !case_num)
+  | true, false -> prerr_endline ("    " ^ string_of_int !case_num ^ " FAIL") );
+  if exit_on_fail && not cond then exit !exit_status else ()
+
+let suite name f =
+  suite_name := name;
+  if print_checkpoints then prerr_endline (name ^ ":");
+  f ()
diff --git a/llvm/test/Bindings/OCaml/Utils/lit.local.cfg b/llvm/test/Bindings/OCaml/Utils/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..53edf1edae2a721557aa92444710b073639fd3f5
--- /dev/null
+++ b/llvm/test/Bindings/OCaml/Utils/lit.local.cfg
@@ -0,0 +1,2 @@
+# This is a directory for utility functions. No test here.
+config.suffixes = ['.dummy']
diff --git a/llvm/test/Bindings/OCaml/core.ml b/llvm/test/Bindings/OCaml/core.ml
index c1bf8f908751ae5ee2ca7360f411990938b1dc47..cedf83af18d73849a5ac2d9e5640e68a4849d465 100644
--- a/llvm/test/Bindings/OCaml/core.ml
+++ b/llvm/test/Bindings/OCaml/core.ml
@@ -1,7 +1,7 @@
-(* RUN: rm -rf %t && mkdir -p %t && cp %s %t/core.ml
- * RUN: %ocamlc -g -w +A -package llvm.analysis -package llvm.bitwriter -linkpkg %t/core.ml -o %t/executable
+(* RUN: rm -rf %t && mkdir -p %t && cp %s %t/core.ml && cp %S/Utils/Testsuite.ml %t/Testsuite.ml
+ * RUN: %ocamlc -g -w +A -package llvm.analysis -package llvm.bitwriter -I %t/ -linkpkg %t/Testsuite.ml %t/core.ml -o %t/executable
  * RUN: %t/executable %t/bitcode.bc
- * RUN: %ocamlopt -g -w +A -package llvm.analysis -package llvm.bitwriter -linkpkg %t/core.ml -o %t/executable
+ * RUN: %ocamlopt -g -w +A -package llvm.analysis -package llvm.bitwriter -I %t/ -linkpkg %t/Testsuite.ml %t/core.ml -o %t/executable
  * RUN: %t/executable %t/bitcode.bc
  * RUN: llvm-dis < %t/bitcode.bc > %t/dis.ll
  * RUN: FileCheck %s < %t/dis.ll
@@ -17,13 +17,7 @@
 open Llvm
 open Llvm_bitwriter
 
-
-(* Tiny unit test framework - really just to help find which line is busted *)
-let exit_status = ref 0
-let suite_name = ref ""
-let group_name = ref ""
-let case_num = ref 0
-let print_checkpoints = false
+open Testsuite
 let context = global_context ()
 let i1_type = Llvm.i1_type context
 let i8_type = Llvm.i8_type context
@@ -35,32 +29,6 @@ let float_type = Llvm.float_type context
 let double_type = Llvm.double_type context
 let fp128_type = Llvm.fp128_type context
 
-let group name =
-  group_name := !suite_name ^ "/" ^ name;
-  case_num := 0;
-  if print_checkpoints then
-    prerr_endline ("  " ^ name ^ "...")
-
-let insist cond =
-  incr case_num;
-  if not cond then
-    exit_status := 10;
-  match print_checkpoints, cond with
-  | false, true -> ()
-  | false, false ->
-      prerr_endline ("FAILED: " ^ !suite_name ^ "/" ^ !group_name ^ " #" ^ (string_of_int !case_num))
-  | true, true ->
-      prerr_endline ("    " ^ (string_of_int !case_num))
-  | true, false ->
-      prerr_endline ("    " ^ (string_of_int !case_num) ^ " FAIL")
-
-let suite name f =
-  suite_name := name;
-  if print_checkpoints then
-    prerr_endline (name ^ ":");
-  f ()
-
-
 (*===-- Fixture -----------------------------------------------------------===*)
 
 let filename = Sys.argv.(1)
@@ -548,14 +516,14 @@ let test_global_variables () =
            set_initializer forty_two32 in
     insist (not (is_declaration g));
     insist (not (is_declaration g2));
-    insist ((global_initializer g) == (global_initializer g2));
+    insist ((global_initializer g) = (global_initializer g2));
 
     let g = define_qualified_global "QGVar02" forty_two32 3 m in
     let g2 = declare_qualified_global i32_type "QGVar03" 3 m ++
            set_initializer forty_two32 in
     insist (not (is_declaration g));
     insist (not (is_declaration g2));
-    insist ((global_initializer g) == (global_initializer g2));
+    insist ((global_initializer g) = (global_initializer g2));
   end;
 
   (* CHECK: GVar04{{.*}}thread_local
@@ -596,6 +564,10 @@ let test_global_variables () =
   begin group "iteration";
     let m = create_module context "temp" in
 
+    insist (get_module_identifier m = "temp");
+    set_module_identifer m "temp2";
+    insist (get_module_identifier m = "temp2");
+
     insist (At_end m = global_begin m);
     insist (At_start m = global_end m);
 
diff --git a/llvm/test/Bindings/OCaml/debuginfo.ml b/llvm/test/Bindings/OCaml/debuginfo.ml
new file mode 100644
index 0000000000000000000000000000000000000000..d07d7592fb2637a06a795ce3f0c43dcb3ff157d6
--- /dev/null
+++ b/llvm/test/Bindings/OCaml/debuginfo.ml
@@ -0,0 +1,413 @@
+(* RUN: rm -rf %t && mkdir -p %t && cp %s %t/debuginfo.ml && cp %S/Utils/Testsuite.ml %t/Testsuite.ml
+ * RUN: %ocamlc -g -w +A -package llvm.all_backends -package llvm.target -package llvm.analysis -package llvm.debuginfo -I %t/ -linkpkg %t/Testsuite.ml %t/debuginfo.ml -o %t/executable
+ * RUN: %t/executable | FileCheck %s
+ * RUN: %ocamlopt -g -w +A -package llvm.all_backends -package llvm.target -package llvm.analysis -package llvm.debuginfo -I %t/ -linkpkg %t/Testsuite.ml %t/debuginfo.ml -o %t/executable
+ * RUN: %t/executable | FileCheck %s
+ * XFAIL: vg_leak
+ *)
+
+open Testsuite
+
+let context = Llvm.global_context ()
+
+let filename = "di_test_file"
+
+let directory = "di_test_dir"
+
+let module_name = "di_test_module"
+
+let null_metadata = Llvm_debuginfo.llmetadata_null ()
+
+let string_of_metadata md =
+  Llvm.string_of_llvalue (Llvm.metadata_as_value context md)
+
+let stdout_metadata md = Printf.printf "%s\n" (string_of_metadata md)
+
+let prepare_target llmod =
+  Llvm_all_backends.initialize ();
+  let triple = Llvm_target.Target.default_triple () in
+  let lltarget = Llvm_target.Target.by_triple triple in
+  let llmachine = Llvm_target.TargetMachine.create ~triple lltarget in
+  let lldly =
+    Llvm_target.DataLayout.as_string
+      (Llvm_target.TargetMachine.data_layout llmachine)
+  in
+  let _ = Llvm.set_target_triple triple llmod in
+  let _ = Llvm.set_data_layout lldly llmod in
+  ()
+
+let new_module () =
+  let m = Llvm.create_module context module_name in
+  let () = prepare_target m in
+  m
+
+let test_get_module () =
+  group "module_level_tests";
+  let m = new_module () in
+  let cur_ver = Llvm_debuginfo.debug_metadata_version () in
+  insist (cur_ver > 0);
+  let m_ver = Llvm_debuginfo.get_module_debug_metadata_version m in
+  (* We haven't added any debug info to the module *)
+  insist (m_ver = 0);
+  let dibuilder = Llvm_debuginfo.dibuilder m in
+  let di_version_key = "Debug Info Version" in
+  let ver =
+    Llvm.value_as_metadata @@ Llvm.const_int (Llvm.i32_type context) cur_ver
+  in
+  let () =
+    Llvm.add_module_flag m Llvm.ModuleFlagBehavior.Warning di_version_key ver
+  in
+  let file_di =
+    Llvm_debuginfo.dibuild_create_file dibuilder ~filename ~directory
+  in
+  stdout_metadata file_di;
+  (* CHECK: [[FILE_PTR:<0x[0-9a-f]*>]] = !DIFile(filename: "di_test_file", directory: "di_test_dir")
+  *)
+  insist
+    ( Llvm_debuginfo.di_file_get_filename ~file:file_di = filename
+    && Llvm_debuginfo.di_file_get_directory ~file:file_di = directory );
+  insist
+    ( Llvm_debuginfo.get_metadata_kind file_di
+    = Llvm_debuginfo.MetadataKind.DIFileMetadataKind );
+  let cu_di =
+    Llvm_debuginfo.dibuild_create_compile_unit dibuilder
+      Llvm_debuginfo.DWARFSourceLanguageKind.C89 ~file_ref:file_di
+      ~producer:"TestGen" ~is_optimized:false ~flags:"" ~runtime_ver:0
+      ~split_name:"" Llvm_debuginfo.DWARFEmissionKind.LineTablesOnly ~dwoid:0
+      ~di_inlining:false ~di_profiling:false ~sys_root:"" ~sdk:""
+  in
+  stdout_metadata cu_di;
+  (* CHECK: [[CMPUNIT_PTR:<0x[0-9a-f]*>]] = distinct !DICompileUnit(language: DW_LANG_C89, file: [[FILE_PTR]], producer: "TestGen", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, splitDebugInlining: false)
+  *)
+  insist
+    ( Llvm_debuginfo.get_metadata_kind cu_di
+    = Llvm_debuginfo.MetadataKind.DICompileUnitMetadataKind );
+  let m_di =
+    Llvm_debuginfo.dibuild_create_module dibuilder ~parent_ref:cu_di
+      ~name:module_name ~config_macros:"" ~include_path:"" ~sys_root:""
+  in
+  insist
+    ( Llvm_debuginfo.get_metadata_kind m_di
+    = Llvm_debuginfo.MetadataKind.DIModuleMetadataKind );
+  insist (Llvm_debuginfo.get_module_debug_metadata_version m = cur_ver);
+  stdout_metadata m_di;
+  (* CHECK: [[MODULE_PTR:<0x[0-9a-f]*>]] = !DIModule(scope: null, name: "di_test_module")
+  *)
+  (m, dibuilder, file_di, m_di)
+
+let flags_zero = Llvm_debuginfo.diflags_get Llvm_debuginfo.DIFlag.Zero
+
+let int_ty_di bits dibuilder =
+  Llvm_debuginfo.dibuild_create_basic_type dibuilder ~name:"int"
+    ~size_in_bits:bits ~encoding:0x05
+    (* llvm::dwarf::DW_ATE_signed *) flags_zero
+
+let test_get_function m dibuilder file_di m_di =
+  group "function_level_tests";
+
+  (* Create a function of type "void foo (int)". *)
+  let int_ty_di = int_ty_di 32 dibuilder in
+  stdout_metadata int_ty_di;
+  (* CHECK: [[INT32_PTR:<0x[0-9a-f]*>]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  *)
+  let param_types = [| null_metadata; int_ty_di |] in
+  let fty_di =
+    Llvm_debuginfo.dibuild_create_subroutine_type dibuilder ~file:file_di
+      ~param_types flags_zero
+  in
+  insist
+    ( Llvm_debuginfo.get_metadata_kind fty_di
+    = Llvm_debuginfo.MetadataKind.DISubroutineTypeMetadataKind );
+  (* To be able to print and verify the type array of the subroutine type,
+   * since we have no way to access it from fty_di, we build it again. *)
+  let fty_di_args =
+    Llvm_debuginfo.dibuild_get_or_create_type_array dibuilder ~data:param_types
+  in
+  stdout_metadata fty_di_args;
+  (* CHECK: [[FARGS_PTR:<0x[0-9a-f]*>]] = !{null, [[INT32_PTR]]}
+  *)
+  stdout_metadata fty_di;
+  (* CHECK: [[SBRTNTY_PTR:<0x[0-9a-f]*>]] = !DISubroutineType(types: [[FARGS_PTR]])
+  *)
+  (* Let's create the LLVM-IR function now. *)
+  let name = "tfun" in
+  let fty =
+    Llvm.function_type (Llvm.void_type context) [| Llvm.i32_type context |]
+  in
+  let f = Llvm.define_function name fty m in
+  let f_di =
+    Llvm_debuginfo.dibuild_create_function dibuilder ~scope:m_di ~name
+      ~linkage_name:name ~file:file_di ~line_no:10 ~ty:fty_di
+      ~is_local_to_unit:false ~is_definition:true ~scope_line:10
+      ~flags:flags_zero ~is_optimized:false
+  in
+  stdout_metadata f_di;
+  (* CHECK: [[SBPRG_PTR:<0x[0-9a-f]*>]] = distinct !DISubprogram(name: "tfun", linkageName: "tfun", scope: [[MODULE_PTR]], file: [[FILE_PTR]], line: 10, type: [[SBRTNTY_PTR]], scopeLine: 10, spFlags: DISPFlagDefinition, unit: [[CMPUNIT_PTR]], retainedNodes: {{<0x[0-9a-f]*>}})
+  *)
+  Llvm_debuginfo.set_subprogram f f_di;
+  ( match Llvm_debuginfo.get_subprogram f with
+  | Some f_di' -> insist (f_di = f_di')
+  | None -> insist false );
+  insist
+    ( Llvm_debuginfo.get_metadata_kind f_di
+    = Llvm_debuginfo.MetadataKind.DISubprogramMetadataKind );
+  insist (Llvm_debuginfo.di_subprogram_get_line f_di = 10);
+  (f, f_di)
+
+let test_bbinstr f f_di file_di dibuilder =
+  group "basic_block and instructions tests";
+  (* Create this pattern:
+   *   if (arg0 != 0) {
+   *      foo(arg0);
+   *   }
+   *   return;
+   *)
+  let arg0 = (Llvm.params f).(0) in
+  let builder = Llvm.builder_at_end context (Llvm.entry_block f) in
+  let zero = Llvm.const_int (Llvm.i32_type context) 0 in
+  let cmpi = Llvm.build_icmp Llvm.Icmp.Ne zero arg0 "cmpi" builder in
+  let truebb = Llvm.append_block context "truebb" f in
+  let falsebb = Llvm.append_block context "falsebb" f in
+  let _ = Llvm.build_cond_br cmpi truebb falsebb builder in
+  let foodecl =
+    Llvm.declare_function "foo"
+      (Llvm.element_type (Llvm.type_of f))
+      (Llvm.global_parent f)
+  in
+  let _ =
+    Llvm.position_at_end truebb builder;
+    let scope =
+      Llvm_debuginfo.dibuild_create_lexical_block dibuilder ~scope:f_di
+        ~file:file_di ~line:9 ~column:4
+    in
+    let file_of_f_di = Llvm_debuginfo.di_scope_get_file ~scope:f_di in
+    let file_of_scope = Llvm_debuginfo.di_scope_get_file ~scope in
+    insist
+      ( match (file_of_f_di, file_of_scope) with
+      | Some file_of_f_di', Some file_of_scope' ->
+          file_of_f_di' = file_di && file_of_scope' = file_di
+      | _ -> false );
+    let foocall = Llvm.build_call foodecl [| arg0 |] "" builder in
+    let foocall_loc =
+      Llvm_debuginfo.dibuild_create_debug_location context ~line:10 ~column:12
+        ~scope
+    in
+    Llvm_debuginfo.instr_set_debug_loc foocall (Some foocall_loc);
+    insist
+      ( match Llvm_debuginfo.instr_get_debug_loc foocall with
+      | Some foocall_loc' -> foocall_loc' = foocall_loc
+      | None -> false );
+    stdout_metadata scope;
+    (* CHECK: [[BLOCK_PTR:<0x[0-9a-f]*>]] = distinct !DILexicalBlock(scope: [[SBPRG_PTR]], file: [[FILE_PTR]], line: 9, column: 4)
+     *)
+    stdout_metadata foocall_loc;
+    (* CHECK: !DILocation(line: 10, column: 12, scope: [[BLOCK_PTR]])
+     *)
+    insist
+      ( Llvm_debuginfo.di_location_get_scope ~location:foocall_loc = scope
+      && Llvm_debuginfo.di_location_get_line ~location:foocall_loc = 10
+      && Llvm_debuginfo.di_location_get_column ~location:foocall_loc = 12 );
+    insist
+      ( Llvm_debuginfo.get_metadata_kind foocall_loc
+        = Llvm_debuginfo.MetadataKind.DILocationMetadataKind
+      && Llvm_debuginfo.get_metadata_kind scope
+         = Llvm_debuginfo.MetadataKind.DILexicalBlockMetadataKind );
+    Llvm.build_br falsebb builder
+  in
+  let _ =
+    Llvm.position_at_end falsebb builder;
+    Llvm.build_ret_void builder
+  in
+  (* Printf.printf "%s\n" (Llvm.string_of_llmodule (Llvm.global_parent f)); *)
+  ()
+
+let test_global_variable_expression dibuilder f_di m_di =
+  group "global variable expression tests";
+  let cexpr_di =
+    Llvm_debuginfo.dibuild_create_constant_value_expression dibuilder 0
+  in
+  stdout_metadata cexpr_di;
+  (* CHECK: [[DICEXPR:!DIExpression\(DW_OP_constu, 0, DW_OP_stack_value\)]]
+   *)
+  insist
+    ( Llvm_debuginfo.get_metadata_kind cexpr_di
+    = Llvm_debuginfo.MetadataKind.DIExpressionMetadataKind );
+  let ty = int_ty_di 64 dibuilder in
+  stdout_metadata ty;
+  (* CHECK: [[INT64TY_PTR:<0x[0-9a-f]*>]] = !DIBasicType(name: "int", size: 64, encoding: DW_ATE_signed)
+   *)
+  let gvexpr_di =
+    Llvm_debuginfo.dibuild_create_global_variable_expression dibuilder
+      ~scope:m_di ~name:"my_global" ~linkage:"" ~file:f_di ~line:5 ~ty
+      ~is_local_to_unit:true ~expr:cexpr_di ~decl:null_metadata ~align_in_bits:0
+  in
+  insist
+    ( Llvm_debuginfo.get_metadata_kind gvexpr_di
+    = Llvm_debuginfo.MetadataKind.DIGlobalVariableExpressionMetadataKind );
+  ( match
+      Llvm_debuginfo.di_global_variable_expression_get_variable gvexpr_di
+    with
+  | Some gvexpr_var_di ->
+      insist
+        ( Llvm_debuginfo.get_metadata_kind gvexpr_var_di
+        = Llvm_debuginfo.MetadataKind.DIGlobalVariableMetadataKind );
+      stdout_metadata gvexpr_var_di
+      (* CHECK: [[GV_PTR:<0x[0-9a-f]*>]] = distinct !DIGlobalVariable(name: "my_global", scope: [[MODULE_PTR]], file: [[FILE_PTR]], line: 5, type: [[INT64TY_PTR]], isLocal: true, isDefinition: true)
+       *)
+  | None -> insist false );
+  stdout_metadata gvexpr_di;
+  (* CHECK: [[GVEXP_PTR:<0x[0-9a-f]*>]] = !DIGlobalVariableExpression(var: [[GV_PTR]], expr: [[DICEXPR]])
+   *)
+  ()
+
+let test_types dibuilder file_di m_di =
+  group "type tests";
+  let namespace_di =
+    Llvm_debuginfo.dibuild_create_namespace dibuilder ~parent_ref:m_di
+      ~name:"NameSpace1" ~export_symbols:false
+  in
+  stdout_metadata namespace_di;
+  (* CHECK: [[NAMESPACE_PTR:<0x[0-9a-f]*>]] = !DINamespace(name: "NameSpace1", scope: [[MODULE_PTR]])
+   *)
+  let int64_ty_di = int_ty_di 64 dibuilder in
+  let structty_args = [| int64_ty_di; int64_ty_di; int64_ty_di |] in
+  let struct_ty_di =
+    Llvm_debuginfo.dibuild_create_struct_type dibuilder ~scope:namespace_di
+      ~name:"StructType1" ~file:file_di ~line_number:20 ~size_in_bits:192
+      ~align_in_bits:0 flags_zero ~derived_from:null_metadata
+      ~elements:structty_args Llvm_debuginfo.DWARFSourceLanguageKind.C89
+      ~vtable_holder:null_metadata ~unique_id:"StructType1"
+  in
+  (* Since there's no way to fetch the element types which is now
+   * a type array, we build that again for checking. *)
+  let structty_di_eltypes =
+    Llvm_debuginfo.dibuild_get_or_create_type_array dibuilder
+      ~data:structty_args
+  in
+  stdout_metadata structty_di_eltypes;
+  (* CHECK: [[STRUCTELT_PTR:<0x[0-9a-f]*>]] = !{[[INT64TY_PTR]], [[INT64TY_PTR]], [[INT64TY_PTR]]}
+   *)
+  stdout_metadata struct_ty_di;
+  (* CHECK: [[STRUCT_PTR:<0x[0-9a-f]*>]] = !DICompositeType(tag: DW_TAG_structure_type, name: "StructType1", scope: [[NAMESPACE_PTR]], file: [[FILE_PTR]], line: 20, size: 192, elements: [[STRUCTELT_PTR]], identifier: "StructType1")
+   *)
+  insist
+    ( Llvm_debuginfo.get_metadata_kind struct_ty_di
+    = Llvm_debuginfo.MetadataKind.DICompositeTypeMetadataKind );
+  let structptr_di =
+    Llvm_debuginfo.dibuild_create_pointer_type dibuilder
+      ~pointee_ty:struct_ty_di ~size_in_bits:192 ~align_in_bits:0
+      ~address_space:0 ~name:""
+  in
+  stdout_metadata structptr_di;
+  (* CHECK: [[STRUCTPTR_PTR:<0x[0-9a-f]*>]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: [[STRUCT_PTR]], size: 192, dwarfAddressSpace: 0)
+   *)
+  insist
+    ( Llvm_debuginfo.get_metadata_kind structptr_di
+    = Llvm_debuginfo.MetadataKind.DIDerivedTypeMetadataKind );
+  let enumerator1 =
+    Llvm_debuginfo.dibuild_create_enumerator dibuilder ~name:"Test_A" ~value:0
+      ~is_unsigned:true
+  in
+  stdout_metadata enumerator1;
+  (* CHECK: [[ENUMERATOR1_PTR:<0x[0-9a-f]*>]] = !DIEnumerator(name: "Test_A", value: 0, isUnsigned: true)
+   *)
+  let enumerator2 =
+    Llvm_debuginfo.dibuild_create_enumerator dibuilder ~name:"Test_B" ~value:1
+      ~is_unsigned:true
+  in
+  stdout_metadata enumerator2;
+  (* CHECK: [[ENUMERATOR2_PTR:<0x[0-9a-f]*>]] = !DIEnumerator(name: "Test_B", value: 1, isUnsigned: true)
+   *)
+  let enumerator3 =
+    Llvm_debuginfo.dibuild_create_enumerator dibuilder ~name:"Test_C" ~value:2
+      ~is_unsigned:true
+  in
+  insist
+    ( Llvm_debuginfo.get_metadata_kind enumerator1
+      = Llvm_debuginfo.MetadataKind.DIEnumeratorMetadataKind
+    && Llvm_debuginfo.get_metadata_kind enumerator2
+       = Llvm_debuginfo.MetadataKind.DIEnumeratorMetadataKind
+    && Llvm_debuginfo.get_metadata_kind enumerator3
+       = Llvm_debuginfo.MetadataKind.DIEnumeratorMetadataKind );
+  stdout_metadata enumerator3;
+  (* CHECK: [[ENUMERATOR3_PTR:<0x[0-9a-f]*>]] = !DIEnumerator(name: "Test_C", value: 2, isUnsigned: true)
+   *)
+  let elements = [| enumerator1; enumerator2; enumerator3 |] in
+  let enumeration_ty_di =
+    Llvm_debuginfo.dibuild_create_enumeration_type dibuilder ~scope:namespace_di
+      ~name:"EnumTest" ~file:file_di ~line_number:1 ~size_in_bits:64
+      ~align_in_bits:0 ~elements ~class_ty:int64_ty_di
+  in
+  let elements_arr =
+    Llvm_debuginfo.dibuild_get_or_create_array dibuilder ~data:elements
+  in
+  stdout_metadata elements_arr;
+  (* CHECK: [[ELEMENTS_PTR:<0x[0-9a-f]*>]] = !{[[ENUMERATOR1_PTR]], [[ENUMERATOR2_PTR]], [[ENUMERATOR3_PTR]]}
+   *)
+  stdout_metadata enumeration_ty_di;
+  (* CHECK: [[ENUMERATION_PTR:<0x[0-9a-f]*>]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "EnumTest", scope: [[NAMESPACE_PTR]], file: [[FILE_PTR]], line: 1, baseType: [[INT64TY_PTR]], size: 64, elements: [[ELEMENTS_PTR]])
+   *)
+  insist
+    ( Llvm_debuginfo.get_metadata_kind enumeration_ty_di
+    = Llvm_debuginfo.MetadataKind.DICompositeTypeMetadataKind );
+  let int32_ty_di = int_ty_di 32 dibuilder in
+  let class_mem1 =
+    Llvm_debuginfo.dibuild_create_member_type dibuilder ~scope:namespace_di
+      ~name:"Field1" ~file:file_di ~line_number:3 ~size_in_bits:32
+      ~align_in_bits:0 ~offset_in_bits:0 flags_zero ~ty:int32_ty_di
+  in
+  stdout_metadata class_mem1;
+  (* CHECK: [[MEMB1_PTR:<0x[0-9a-f]*>]] = !DIDerivedType(tag: DW_TAG_member, name: "Field1", scope: [[NAMESPACE_PTR]], file: [[FILE_PTR]], line: 3, baseType: [[INT32_PTR]], size: 32)
+   *)
+  insist (Llvm_debuginfo.di_type_get_name class_mem1 = "Field1");
+  insist (Llvm_debuginfo.di_type_get_line class_mem1 = 3);
+  let class_mem2 =
+    Llvm_debuginfo.dibuild_create_member_type dibuilder ~scope:namespace_di
+      ~name:"Field2" ~file:file_di ~line_number:4 ~size_in_bits:64
+      ~align_in_bits:8 ~offset_in_bits:32 flags_zero ~ty:int64_ty_di
+  in
+  stdout_metadata class_mem2;
+  (* CHECK: [[MEMB2_PTR:<0x[0-9a-f]*>]] = !DIDerivedType(tag: DW_TAG_member, name: "Field2", scope: [[NAMESPACE_PTR]], file: [[FILE_PTR]], line: 4, baseType: [[INT64TY_PTR]], size: 64, align: 8, offset: 32)
+   *)
+  insist (Llvm_debuginfo.di_type_get_offset_in_bits class_mem2 = 32);
+  insist (Llvm_debuginfo.di_type_get_size_in_bits class_mem2 = 64);
+  insist (Llvm_debuginfo.di_type_get_align_in_bits class_mem2 = 8);
+  let class_elements = [| class_mem1; class_mem2 |] in
+  insist
+    ( Llvm_debuginfo.get_metadata_kind class_mem1
+      = Llvm_debuginfo.MetadataKind.DIDerivedTypeMetadataKind
+    && Llvm_debuginfo.get_metadata_kind class_mem2
+       = Llvm_debuginfo.MetadataKind.DIDerivedTypeMetadataKind );
+  stdout_metadata
+    (Llvm_debuginfo.dibuild_get_or_create_type_array dibuilder
+       ~data:class_elements);
+  (* CHECK: [[CLASSMEM_PTRS:<0x[0-9a-f]*>]] = !{[[MEMB1_PTR]], [[MEMB2_PTR]]}
+   *)
+  let classty_di =
+    Llvm_debuginfo.dibuild_create_class_type dibuilder ~scope:namespace_di
+      ~name:"MyClass" ~file:file_di ~line_number:1 ~size_in_bits:96
+      ~align_in_bits:0 ~offset_in_bits:0 flags_zero ~derived_from:null_metadata
+      ~elements:class_elements ~vtable_holder:null_metadata
+      ~template_params_node:null_metadata ~unique_identifier:"MyClass"
+  in
+  stdout_metadata classty_di;
+  (* [[CLASS_PTR:<0x[0-9a-f]*>]] = !DICompositeType(tag: DW_TAG_structure_type, name: "MyClass", scope: [[NAMESPACE_PTR]], file: [[FILE_PTR]], line: 1, size: 96, elements: [[CLASSMEM_PTRS]], identifier: "MyClass")
+   *)
+  insist
+    ( Llvm_debuginfo.get_metadata_kind classty_di
+    = Llvm_debuginfo.MetadataKind.DICompositeTypeMetadataKind );
+  ()
+
+let () =
+  let m, dibuilder, file_di, m_di = test_get_module () in
+  let f, fun_di = test_get_function m dibuilder file_di m_di in
+  let () = test_bbinstr f fun_di file_di dibuilder in
+  let () = test_global_variable_expression dibuilder file_di m_di in
+  let () = test_types dibuilder file_di m_di in
+  Llvm_debuginfo.dibuild_finalize dibuilder;
+  ( match Llvm_analysis.verify_module m with
+  | Some err ->
+      prerr_endline ("Verification of module failed: " ^ err);
+      exit_status := 1
+  | None -> () );
+  exit !exit_status
diff --git a/llvm/test/Bindings/OCaml/irreader.ml b/llvm/test/Bindings/OCaml/irreader.ml
index 49b0d9190ee7edb778d169168194c56cabf7f8a6..7d8e4a97e38dc638dd219de60d0783a208fa5e18 100644
--- a/llvm/test/Bindings/OCaml/irreader.ml
+++ b/llvm/test/Bindings/OCaml/irreader.ml
@@ -38,7 +38,7 @@ let test_irreader () =
     let m   = parse_ir context buf in
     match lookup_global "foo" m with
     | Some foo ->
-        insist ((global_initializer foo) = (const_int (i32_type context) 42))
+        insist ((global_initializer foo) = (Some (const_int (i32_type context) 42)))
     | None ->
         failwith "global"
   end;
diff --git a/llvm/test/Bitcode/Inputs/inalloca-upgrade.bc b/llvm/test/Bitcode/Inputs/inalloca-upgrade.bc
new file mode 100644
index 0000000000000000000000000000000000000000..51012caacddcead2fb9e9a9aa4137ac88aa41be0
Binary files /dev/null and b/llvm/test/Bitcode/Inputs/inalloca-upgrade.bc differ
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 089180ed4725bdcad74d1c3e0299e16ec2d66b7a..936a2953901a982aa312059beb3beb79449be6c5 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -214,8 +214,8 @@ define void @f35() optnone noinline
         ret void;
 }
 
-define void @f36(i8* inalloca %0) {
-; CHECK: define void @f36(i8* inalloca %0) {
+define void @f36(i8* inalloca(i8) %0) {
+; CHECK: define void @f36(i8* inalloca(i8) %0) {
         ret void
 }
 
@@ -422,6 +422,31 @@ define void @f71() hot
   ret void
 }
 
+; CHECK: define void @f72() #45
+define void @f72() vscale_range(8)
+{
+  ret void
+}
+
+; CHECK: define void @f73() #46
+define void @f73() vscale_range(1,8)
+{
+  ret void
+}
+
+; CHECK: define void @f74() #47
+define void @f74() vscale_range(1,0)
+{
+  ret void
+}
+
+; CHECK: define void @f75()
+; CHECK-NOT: define void @f75() #
+define void @f75() vscale_range(0,0)
+{
+  ret void
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { readnone }
@@ -467,4 +492,7 @@ define void @f71() hot
 ; CHECK: attributes #42 = { nocallback }
 ; CHECK: attributes #43 = { cold }
 ; CHECK: attributes #44 = { hot }
+; CHECK: attributes #45 = { vscale_range(8,8) }
+; CHECK: attributes #46 = { vscale_range(1,8) }
+; CHECK: attributes #47 = { vscale_range(1,0) }
 ; CHECK: attributes #[[NOBUILTIN]] = { nobuiltin }
diff --git a/llvm/test/Bitcode/compatibility-3.6.ll b/llvm/test/Bitcode/compatibility-3.6.ll
index 05e6b71f147786272afe592006e0334a87ce3e27..882f0416e8936967a3afeeff6eff0889666f5d85 100644
--- a/llvm/test/Bitcode/compatibility-3.6.ll
+++ b/llvm/test/Bitcode/compatibility-3.6.ll
@@ -406,7 +406,7 @@ declare void @f.param.inreg(i8 inreg)
 declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 ; CHECK: declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 declare void @f.param.inalloca(i8* inalloca)
-; CHECK: declare void @f.param.inalloca(i8* inalloca)
+; CHECK: declare void @f.param.inalloca(i8* inalloca(i8))
 declare void @f.param.sret(i8* sret(i8))
 ; CHECK: declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
@@ -993,7 +993,7 @@ exit:
 
 define void @instructions.call_musttail(i8* inalloca %val) {
   musttail call void @f.param.inalloca(i8* inalloca %val)
-  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca %val)
+  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca(i8) %val)
 
   ret void
 }
diff --git a/llvm/test/Bitcode/compatibility-3.7.ll b/llvm/test/Bitcode/compatibility-3.7.ll
index b31509ec0a867373eafcf01d29f96a6e30135956..65dae1b6a7553667d488aa73ec4d7bf4883f2810 100644
--- a/llvm/test/Bitcode/compatibility-3.7.ll
+++ b/llvm/test/Bitcode/compatibility-3.7.ll
@@ -412,7 +412,7 @@ declare void @f.param.inreg(i8 inreg)
 declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 ; CHECK: declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 declare void @f.param.inalloca(i8* inalloca)
-; CHECK: declare void @f.param.inalloca(i8* inalloca)
+; CHECK: declare void @f.param.inalloca(i8* inalloca(i8))
 declare void @f.param.sret(i8* sret(i8))
 ; CHECK: declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
@@ -1034,7 +1034,7 @@ exit:
 
 define void @instructions.call_musttail(i8* inalloca %val) {
   musttail call void @f.param.inalloca(i8* inalloca %val)
-  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca %val)
+  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca(i8) %val)
 
   ret void
 }
diff --git a/llvm/test/Bitcode/compatibility-3.8.ll b/llvm/test/Bitcode/compatibility-3.8.ll
index 72b01f6a1d98b3b3ce441dbd28529d854de7a0d9..51b4740ea42e3ca0f81d1db608dca022167d1909 100644
--- a/llvm/test/Bitcode/compatibility-3.8.ll
+++ b/llvm/test/Bitcode/compatibility-3.8.ll
@@ -437,7 +437,7 @@ declare void @f.param.inreg(i8 inreg)
 declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 ; CHECK: declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 declare void @f.param.inalloca(i8* inalloca)
-; CHECK: declare void @f.param.inalloca(i8* inalloca)
+; CHECK: declare void @f.param.inalloca(i8* inalloca(i8))
 declare void @f.param.sret(i8* sret(i8))
 ; CHECK: declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
@@ -1182,7 +1182,7 @@ exit:
 
 define void @instructions.call_musttail(i8* inalloca %val) {
   musttail call void @f.param.inalloca(i8* inalloca %val)
-  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca %val)
+  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca(i8) %val)
 
   ret void
 }
diff --git a/llvm/test/Bitcode/compatibility-3.9.ll b/llvm/test/Bitcode/compatibility-3.9.ll
index f5b409ab2578904d26814e1f86ce5728f2cdf510..e203e5144c28ce5e0df37bc052b67bdb1f4b3cad 100644
--- a/llvm/test/Bitcode/compatibility-3.9.ll
+++ b/llvm/test/Bitcode/compatibility-3.9.ll
@@ -506,7 +506,7 @@ declare void @f.param.inreg(i8 inreg)
 declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 ; CHECK: declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 declare void @f.param.inalloca(i8* inalloca)
-; CHECK: declare void @f.param.inalloca(i8* inalloca)
+; CHECK: declare void @f.param.inalloca(i8* inalloca(i8))
 declare void @f.param.sret(i8* sret(i8))
 ; CHECK: declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
@@ -1253,7 +1253,7 @@ exit:
 
 define void @instructions.call_musttail(i8* inalloca %val) {
   musttail call void @f.param.inalloca(i8* inalloca %val)
-  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca %val)
+  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca(i8) %val)
 
   ret void
 }
diff --git a/llvm/test/Bitcode/compatibility-4.0.ll b/llvm/test/Bitcode/compatibility-4.0.ll
index c7874106d2b9914b34079e9fb5fc1e5db554bfe8..b0b65fe75da81d4b2421f5e560ada9258dc0e9b8 100644
--- a/llvm/test/Bitcode/compatibility-4.0.ll
+++ b/llvm/test/Bitcode/compatibility-4.0.ll
@@ -506,7 +506,7 @@ declare void @f.param.inreg(i8 inreg)
 declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 ; CHECK: declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 declare void @f.param.inalloca(i8* inalloca)
-; CHECK: declare void @f.param.inalloca(i8* inalloca)
+; CHECK: declare void @f.param.inalloca(i8* inalloca(i8))
 declare void @f.param.sret(i8* sret(i8))
 ; CHECK: declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
@@ -1253,7 +1253,7 @@ exit:
 
 define void @instructions.call_musttail(i8* inalloca %val) {
   musttail call void @f.param.inalloca(i8* inalloca %val)
-  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca %val)
+  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca(i8) %val)
 
   ret void
 }
diff --git a/llvm/test/Bitcode/compatibility-5.0.ll b/llvm/test/Bitcode/compatibility-5.0.ll
index e63ff3a7cc066c820d183685638a84754a625ce1..93916d9ff890232b5e89d755a8b1effc39b6734c 100644
--- a/llvm/test/Bitcode/compatibility-5.0.ll
+++ b/llvm/test/Bitcode/compatibility-5.0.ll
@@ -510,7 +510,7 @@ declare void @f.param.inreg(i8 inreg)
 declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 ; CHECK: declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 declare void @f.param.inalloca(i8* inalloca)
-; CHECK: declare void @f.param.inalloca(i8* inalloca)
+; CHECK: declare void @f.param.inalloca(i8* inalloca(i8))
 declare void @f.param.sret(i8* sret(i8))
 ; CHECK: declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
@@ -1265,7 +1265,7 @@ exit:
 
 define void @instructions.call_musttail(i8* inalloca %val) {
   musttail call void @f.param.inalloca(i8* inalloca %val)
-  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca %val)
+  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca(i8) %val)
 
   ret void
 }
diff --git a/llvm/test/Bitcode/compatibility-6.0.ll b/llvm/test/Bitcode/compatibility-6.0.ll
index 980dd92563c7edd9aba9af90ca8749f46035ef76..bd5dfd3457928ff04cc6575961296a9308e9619b 100644
--- a/llvm/test/Bitcode/compatibility-6.0.ll
+++ b/llvm/test/Bitcode/compatibility-6.0.ll
@@ -517,7 +517,7 @@ declare void @f.param.inreg(i8 inreg)
 declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 ; CHECK: declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 declare void @f.param.inalloca(i8* inalloca)
-; CHECK: declare void @f.param.inalloca(i8* inalloca)
+; CHECK: declare void @f.param.inalloca(i8* inalloca(i8))
 declare void @f.param.sret(i8* sret(i8))
 ; CHECK: declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
@@ -1276,7 +1276,7 @@ exit:
 
 define void @instructions.call_musttail(i8* inalloca %val) {
   musttail call void @f.param.inalloca(i8* inalloca %val)
-  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca %val)
+  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca(i8) %val)
 
   ret void
 }
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 34c5cfd41204615253f31a026bed3bba5fd5fe81..95bb2639e09bcdcf41066c001e14cf311d010c23 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -532,8 +532,8 @@ declare void @f.param.inreg(i8 inreg)
 ; CHECK: declare void @f.param.inreg(i8 inreg)
 declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
 ; CHECK: declare void @f.param.byval({ i8, i8 }* byval({ i8, i8 }))
-declare void @f.param.inalloca(i8* inalloca)
-; CHECK: declare void @f.param.inalloca(i8* inalloca)
+declare void @f.param.inalloca(i8* inalloca(i8))
+; CHECK: declare void @f.param.inalloca(i8* inalloca(i8))
 declare void @f.param.sret(i8* sret(i8))
 ; CHECK: declare void @f.param.sret(i8* sret(i8))
 declare void @f.param.noalias(i8* noalias)
@@ -1520,9 +1520,9 @@ exit:
   ret void
 }
 
-define void @instructions.call_musttail(i8* inalloca %val) {
-  musttail call void @f.param.inalloca(i8* inalloca %val)
-  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca %val)
+define void @instructions.call_musttail(i8* inalloca(i8) %val) {
+  musttail call void @f.param.inalloca(i8* inalloca(i8) %val)
+  ; CHECK: musttail call void @f.param.inalloca(i8* inalloca(i8) %val)
 
   ret void
 }
diff --git a/llvm/test/Bitcode/inalloca-upgrade.test b/llvm/test/Bitcode/inalloca-upgrade.test
new file mode 100644
index 0000000000000000000000000000000000000000..20d41365b3601cb4855bbc0a385ca3101ba1e2b7
--- /dev/null
+++ b/llvm/test/Bitcode/inalloca-upgrade.test
@@ -0,0 +1,7 @@
+RUN: llvm-dis %p/Inputs/inalloca-upgrade.bc -o - | FileCheck %s
+
+Make sure we upgrade old-style IntAttribute inalloca records to a
+fully typed version correctly.
+
+CHECK: call void @bar({ i32*, i8 }* inalloca({ i32*, i8 }) %ptr)
+CHECK: invoke void @bar({ i32*, i8 }* inalloca({ i32*, i8 }) %ptr)
diff --git a/llvm/test/Bitcode/inalloca.ll b/llvm/test/Bitcode/inalloca.ll
index 84abe176d65e992a6af8e0f6d27a801c4e5ffbce..3b56f571b15b25afab27d1aec249778ec51c9892 100644
--- a/llvm/test/Bitcode/inalloca.ll
+++ b/llvm/test/Bitcode/inalloca.ll
@@ -3,17 +3,17 @@
 
 ; inalloca should roundtrip.
 
-define void @foo(i32* inalloca %args) {
+define void @foo(i32* inalloca(i32) %args) {
   ret void
 }
-; CHECK-LABEL: define void @foo(i32* inalloca %args)
+; CHECK-LABEL: define void @foo(i32* inalloca(i32) %args)
 
 define void @bar() {
   ; Use the maximum alignment, since we stuff our bit with alignment.
   %args = alloca inalloca i32, align 536870912
-  call void @foo(i32* inalloca %args)
+  call void @foo(i32* inalloca(i32) %args)
   ret void
 }
 ; CHECK-LABEL: define void @bar() {
 ; CHECK: %args = alloca inalloca i32, align 536870912
-; CHECK: call void @foo(i32* inalloca %args)
+; CHECK: call void @foo(i32* inalloca(i32) %args)
diff --git a/llvm/test/Bitcode/intrinsics-with-unnamed-types.ll b/llvm/test/Bitcode/intrinsics-with-unnamed-types.ll
new file mode 100644
index 0000000000000000000000000000000000000000..02d86ec2c5da9bb10d19ad9f1ee93a3026f6bf99
--- /dev/null
+++ b/llvm/test/Bitcode/intrinsics-with-unnamed-types.ll
@@ -0,0 +1,31 @@
+; RUN: llvm-as -o - %s | llvm-dis -o - 2>&1 | FileCheck %s
+
+; Make sure we can assemble and disassemble IR containing intrinsics with
+; unnamed types.
+
+%1 = type opaque
+%0 = type opaque
+
+; CHECK-LABEL: @f0(
+; CHECK: %c1 = call %0* @llvm.ssa.copy.p0s_s.0(%0* %arg)
+; CHECK: %c2 = call %1* @llvm.ssa.copy.p0s_s.1(%1* %tmp)
+; CHECK: %c3 = call %0** @llvm.ssa.copy.p0p0s_s.1(%0** %arg2)
+; CHECK: %c4 = call %1** @llvm.ssa.copy.p0p0s_s.0(%1** %tmp2)
+
+define void @f0(%0* %arg, %1* %tmp, %1** %tmp2, %0** %arg2) {
+bb:
+  %cmp1 = icmp ne %0* %arg, null
+  %c1 = call %0* @llvm.ssa.copy.p0s_s.0(%0* %arg)
+  %c2 = call %1* @llvm.ssa.copy.p0s_s.1(%1* %tmp)
+  %c3 = call %0** @llvm.ssa.copy.p0p0s_s.1(%0** %arg2)
+  %c4 = call %1** @llvm.ssa.copy.p0p0s_s.0(%1** %tmp2)
+  ret void
+}
+
+declare %0* @llvm.ssa.copy.p0s_s.0(%0* returned)
+
+declare %1* @llvm.ssa.copy.p0s_s.1(%1* returned)
+
+declare %0** @llvm.ssa.copy.p0p0s_s.1(%0** returned)
+
+declare %1** @llvm.ssa.copy.p0p0s_s.0(%1** returned)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/byval-call.ll b/llvm/test/CodeGen/AArch64/GlobalISel/byval-call.ll
new file mode 100644
index 0000000000000000000000000000000000000000..778c823552a1f4dacde5a9852dac2b98907bd15c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/byval-call.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+declare void @byval_i32(i32* byval(i32) %ptr)
+
+define void @call_byval_i32(i32* %incoming) {
+; CHECK-LABEL: call_byval_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #32 // =32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    str w8, [sp]
+; CHECK-NEXT:    bl byval_i32
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32 // =32
+; CHECK-NEXT:    ret
+  call void @byval_i32(i32* byval(i32) %incoming)
+  ret void
+}
+
+declare void @byval_a64i32([64 x i32]* byval([64 x i32]) %ptr)
+
+define void @call_byval_a64i32([64 x i32]* %incoming) {
+; CHECK-LABEL: call_byval_a64i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #288 // =288
+; CHECK-NEXT:    stp x29, x30, [sp, #256] // 16-byte Folded Spill
+; CHECK-NEXT:    str x28, [sp, #272] // 8-byte Folded Spill
+; CHECK-NEXT:    add x29, sp, #256 // =256
+; CHECK-NEXT:    .cfi_def_cfa w29, 32
+; CHECK-NEXT:    .cfi_offset w28, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    str q0, [sp]
+; CHECK-NEXT:    ldr q0, [x0, #16]
+; CHECK-NEXT:    str q0, [sp, #16]
+; CHECK-NEXT:    ldr q0, [x0, #32]
+; CHECK-NEXT:    str q0, [sp, #32]
+; CHECK-NEXT:    ldr q0, [x0, #48]
+; CHECK-NEXT:    str q0, [sp, #48]
+; CHECK-NEXT:    ldr q0, [x0, #64]
+; CHECK-NEXT:    str q0, [sp, #64]
+; CHECK-NEXT:    ldr q0, [x0, #80]
+; CHECK-NEXT:    str q0, [sp, #80]
+; CHECK-NEXT:    ldr q0, [x0, #96]
+; CHECK-NEXT:    str q0, [sp, #96]
+; CHECK-NEXT:    ldr q0, [x0, #112]
+; CHECK-NEXT:    str q0, [sp, #112]
+; CHECK-NEXT:    ldr q0, [x0, #128]
+; CHECK-NEXT:    str q0, [sp, #128]
+; CHECK-NEXT:    ldr q0, [x0, #144]
+; CHECK-NEXT:    str q0, [sp, #144]
+; CHECK-NEXT:    ldr q0, [x0, #160]
+; CHECK-NEXT:    str q0, [sp, #160]
+; CHECK-NEXT:    ldr q0, [x0, #176]
+; CHECK-NEXT:    str q0, [sp, #176]
+; CHECK-NEXT:    ldr q0, [x0, #192]
+; CHECK-NEXT:    str q0, [sp, #192]
+; CHECK-NEXT:    ldr q0, [x0, #208]
+; CHECK-NEXT:    str q0, [sp, #208]
+; CHECK-NEXT:    ldr q0, [x0, #224]
+; CHECK-NEXT:    str q0, [sp, #224]
+; CHECK-NEXT:    ldr q0, [x0, #240]
+; CHECK-NEXT:    str q0, [sp, #240]
+; CHECK-NEXT:    bl byval_a64i32
+; CHECK-NEXT:    ldr x28, [sp, #272] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp, #256] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #288 // =288
+; CHECK-NEXT:    ret
+  call void @byval_a64i32([64 x i32]* byval([64 x i32]) %incoming)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-vectors.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-vectors.ll
index ee73e58798c76b2804fbe999e47df72fc496c79e..f34f0981c21161e3edbd733d9f5967703abfdbbb 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-vectors.ll
@@ -31,3 +31,16 @@ define i24 @test_v3i8(<3 x i8> %a) {
   ret i24 %res
 }
 
+
+define <1 x half> @test_v1s16(<1 x float> %x) {
+  ; CHECK-LABEL: name: test_v1s16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $d0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+  ; CHECK:   [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[UV]](s32)
+  ; CHECK:   $h0 = COPY [[FPTRUNC]](s16)
+  ; CHECK:   RET_ReallyLR implicit $h0
+  %tmp = fptrunc <1 x float> %x to <1 x half>
+  ret <1 x half> %tmp
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets-target-features.mir b/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets-target-features.mir
new file mode 100644
index 0000000000000000000000000000000000000000..639c51d92d9c1829f0ebf1e7c85c4cb01b43c3c3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets-target-features.mir
@@ -0,0 +1,241 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-unknown-unknown -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=DEFAULT,CHECK
+# RUN: llc -mtriple aarch64-apple-darwin -code-model=large -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=LARGE-MACHO,CHECK
+# RUN: llc -mtriple aarch64-apple-darwin -code-model=small -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=SMALL-MACHO,CHECK
+# RUN: llc -mtriple aarch64-linux-elf -code-model=large -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=LARGE-ELF,CHECK
+# RUN: llc -mtriple aarch64-linux-elf -code-model=tiny -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=TINY,CHECK
+# RUN: llc -mtriple aarch64-windows-coff -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=WINDOWS,CHECK
+
+# Each of these tests has a trivial pattern for folding a G_PTR_ADD into a
+# G_GLOBAL_VALUE.
+#
+# Check that given different code models/target features, we do/don't fold.
+
+--- |
+    @external_linkage = external hidden global i32
+    @common_linkage = common local_unnamed_addr global i32 0, align 4
+    @internal_linkage = internal unnamed_addr global i32 0, align 4
+    @extern_weak_linkage = extern_weak hidden global i32
+    @dll_import = external dllimport global i32
+
+    define void @test_external_linkage() { ret void }
+    define void @test_internal_linkage() { ret void }
+    define void @test_common_linkage() { ret void }
+    define void @test_extern_weak_linkage() { ret void }
+    define void @never_fold_tagged_globals() #0 { ret void }
+    define void @test_dll_import() { ret void }
+
+    attributes #0 = { "target-features"="+tagged-globals" }
+...
+---
+name:            test_external_linkage
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    ; Large + Mach-O goes via GOT, so we can't fold.
+
+    ; DEFAULT-LABEL: name: test_external_linkage
+    ; DEFAULT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_linkage + 1
+    ; DEFAULT: $x0 = COPY [[GV]](p0)
+    ; DEFAULT: RET_ReallyLR implicit $x0
+    ; LARGE-MACHO-LABEL: name: test_external_linkage
+    ; LARGE-MACHO: %global:_(p0) = G_GLOBAL_VALUE @external_linkage
+    ; LARGE-MACHO: %imm:_(s64) = G_CONSTANT i64 1
+    ; LARGE-MACHO: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; LARGE-MACHO: $x0 = COPY %ptr_add(p0)
+    ; LARGE-MACHO: RET_ReallyLR implicit $x0
+    ; SMALL-MACHO-LABEL: name: test_external_linkage
+    ; SMALL-MACHO: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_linkage + 1
+    ; SMALL-MACHO: $x0 = COPY [[GV]](p0)
+    ; SMALL-MACHO: RET_ReallyLR implicit $x0
+    ; LARGE-ELF-LABEL: name: test_external_linkage
+    ; LARGE-ELF: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_linkage + 1
+    ; LARGE-ELF: $x0 = COPY [[GV]](p0)
+    ; LARGE-ELF: RET_ReallyLR implicit $x0
+    ; TINY-LABEL: name: test_external_linkage
+    ; TINY: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_linkage + 1
+    ; TINY: $x0 = COPY [[GV]](p0)
+    ; TINY: RET_ReallyLR implicit $x0
+    ; WINDOWS-LABEL: name: test_external_linkage
+    ; WINDOWS: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_linkage + 1
+    ; WINDOWS: $x0 = COPY [[GV]](p0)
+    ; WINDOWS: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @external_linkage
+    %imm:_(s64) = G_CONSTANT i64 1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    $x0 = COPY %ptr_add(p0)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            test_internal_linkage
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    ; Large + Mach-O goes via GOT, so we can't fold.
+
+    ; DEFAULT-LABEL: name: test_internal_linkage
+    ; DEFAULT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @internal_linkage + 1
+    ; DEFAULT: $x0 = COPY [[GV]](p0)
+    ; DEFAULT: RET_ReallyLR implicit $x0
+    ; LARGE-MACHO-LABEL: name: test_internal_linkage
+    ; LARGE-MACHO: %global:_(p0) = G_GLOBAL_VALUE @internal_linkage
+    ; LARGE-MACHO: %imm:_(s64) = G_CONSTANT i64 1
+    ; LARGE-MACHO: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; LARGE-MACHO: $x0 = COPY %ptr_add(p0)
+    ; LARGE-MACHO: RET_ReallyLR implicit $x0
+    ; SMALL-MACHO-LABEL: name: test_internal_linkage
+    ; SMALL-MACHO: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @internal_linkage + 1
+    ; SMALL-MACHO: $x0 = COPY [[GV]](p0)
+    ; SMALL-MACHO: RET_ReallyLR implicit $x0
+    ; LARGE-ELF-LABEL: name: test_internal_linkage
+    ; LARGE-ELF: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @internal_linkage + 1
+    ; LARGE-ELF: $x0 = COPY [[GV]](p0)
+    ; LARGE-ELF: RET_ReallyLR implicit $x0
+    ; TINY-LABEL: name: test_internal_linkage
+    ; TINY: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @internal_linkage + 1
+    ; TINY: $x0 = COPY [[GV]](p0)
+    ; TINY: RET_ReallyLR implicit $x0
+    ; WINDOWS-LABEL: name: test_internal_linkage
+    ; WINDOWS: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @internal_linkage + 1
+    ; WINDOWS: $x0 = COPY [[GV]](p0)
+    ; WINDOWS: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @internal_linkage
+    %imm:_(s64) = G_CONSTANT i64 1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    $x0 = COPY %ptr_add(p0)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            test_common_linkage
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    ; DEFAULT-LABEL: name: test_common_linkage
+    ; DEFAULT: %global:_(p0) = G_GLOBAL_VALUE @common_linkage
+    ; DEFAULT: %imm:_(s64) = G_CONSTANT i64 1
+    ; DEFAULT: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; DEFAULT: $x0 = COPY %ptr_add(p0)
+    ; DEFAULT: RET_ReallyLR implicit $x0
+    ; LARGE-MACHO-LABEL: name: test_common_linkage
+    ; LARGE-MACHO: %global:_(p0) = G_GLOBAL_VALUE @common_linkage
+    ; LARGE-MACHO: %imm:_(s64) = G_CONSTANT i64 1
+    ; LARGE-MACHO: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; LARGE-MACHO: $x0 = COPY %ptr_add(p0)
+    ; LARGE-MACHO: RET_ReallyLR implicit $x0
+    ; SMALL-MACHO-LABEL: name: test_common_linkage
+    ; SMALL-MACHO: %global:_(p0) = G_GLOBAL_VALUE @common_linkage
+    ; SMALL-MACHO: %imm:_(s64) = G_CONSTANT i64 1
+    ; SMALL-MACHO: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; SMALL-MACHO: $x0 = COPY %ptr_add(p0)
+    ; SMALL-MACHO: RET_ReallyLR implicit $x0
+    ; LARGE-ELF-LABEL: name: test_common_linkage
+    ; LARGE-ELF: %global:_(p0) = G_GLOBAL_VALUE @common_linkage
+    ; LARGE-ELF: %imm:_(s64) = G_CONSTANT i64 1
+    ; LARGE-ELF: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; LARGE-ELF: $x0 = COPY %ptr_add(p0)
+    ; LARGE-ELF: RET_ReallyLR implicit $x0
+    ; TINY-LABEL: name: test_common_linkage
+    ; TINY: %global:_(p0) = G_GLOBAL_VALUE @common_linkage
+    ; TINY: %imm:_(s64) = G_CONSTANT i64 1
+    ; TINY: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; TINY: $x0 = COPY %ptr_add(p0)
+    ; TINY: RET_ReallyLR implicit $x0
+    ; WINDOWS-LABEL: name: test_common_linkage
+    ; WINDOWS: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @common_linkage + 1
+    ; WINDOWS: $x0 = COPY [[GV]](p0)
+    ; WINDOWS: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @common_linkage
+    %imm:_(s64) = G_CONSTANT i64 1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    $x0 = COPY %ptr_add(p0)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            test_extern_weak_linkage
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    ; DEFAULT-LABEL: name: test_extern_weak_linkage
+    ; DEFAULT: %global:_(p0) = G_GLOBAL_VALUE @extern_weak_linkage
+    ; DEFAULT: %imm:_(s64) = G_CONSTANT i64 1
+    ; DEFAULT: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; DEFAULT: $x0 = COPY %ptr_add(p0)
+    ; DEFAULT: RET_ReallyLR implicit $x0
+    ; LARGE-MACHO-LABEL: name: test_extern_weak_linkage
+    ; LARGE-MACHO: %global:_(p0) = G_GLOBAL_VALUE @extern_weak_linkage
+    ; LARGE-MACHO: %imm:_(s64) = G_CONSTANT i64 1
+    ; LARGE-MACHO: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; LARGE-MACHO: $x0 = COPY %ptr_add(p0)
+    ; LARGE-MACHO: RET_ReallyLR implicit $x0
+    ; SMALL-MACHO-LABEL: name: test_extern_weak_linkage
+    ; SMALL-MACHO: %global:_(p0) = G_GLOBAL_VALUE @extern_weak_linkage
+    ; SMALL-MACHO: %imm:_(s64) = G_CONSTANT i64 1
+    ; SMALL-MACHO: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; SMALL-MACHO: $x0 = COPY %ptr_add(p0)
+    ; SMALL-MACHO: RET_ReallyLR implicit $x0
+    ; LARGE-ELF-LABEL: name: test_extern_weak_linkage
+    ; LARGE-ELF: %global:_(p0) = G_GLOBAL_VALUE @extern_weak_linkage
+    ; LARGE-ELF: %imm:_(s64) = G_CONSTANT i64 1
+    ; LARGE-ELF: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; LARGE-ELF: $x0 = COPY %ptr_add(p0)
+    ; LARGE-ELF: RET_ReallyLR implicit $x0
+    ; TINY-LABEL: name: test_extern_weak_linkage
+    ; TINY: %global:_(p0) = G_GLOBAL_VALUE @extern_weak_linkage
+    ; TINY: %imm:_(s64) = G_CONSTANT i64 1
+    ; TINY: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; TINY: $x0 = COPY %ptr_add(p0)
+    ; TINY: RET_ReallyLR implicit $x0
+    ; WINDOWS-LABEL: name: test_extern_weak_linkage
+    ; WINDOWS: %global:_(p0) = G_GLOBAL_VALUE @extern_weak_linkage
+    ; WINDOWS: %imm:_(s64) = G_CONSTANT i64 1
+    ; WINDOWS: %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    ; WINDOWS: $x0 = COPY %ptr_add(p0)
+    ; WINDOWS: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @extern_weak_linkage
+    %imm:_(s64) = G_CONSTANT i64 1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    $x0 = COPY %ptr_add(p0)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            never_fold_tagged_globals
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: never_fold_tagged_globals
+    ; CHECK-NOT: %global:_(p0) = G_GLOBAL_VALUE @external_linkage + 1
+    %global:_(p0) = G_GLOBAL_VALUE @external_linkage
+    %imm:_(s64) = G_CONSTANT i64 1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    $x0 = COPY %ptr_add(p0)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            test_dll_import
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_dll_import
+    ; CHECK-NOT: %global:_(p0) = G_GLOBAL_VALUE @dll_import + 1
+    %global:_(p0) = G_GLOBAL_VALUE @dll_import
+    %imm:_(s64) = G_CONSTANT i64 1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %imm(s64)
+    $x0 = COPY %ptr_add(p0)
+    RET_ReallyLR implicit $x0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir b/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir
new file mode 100644
index 0000000000000000000000000000000000000000..514cef0e703d1189c5f30e4cb6f96550718029e9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir
@@ -0,0 +1,284 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-apple-darwin -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+    @g = external hidden global i32
+
+    %opaque = type opaque
+    @unsized = external hidden global %opaque
+
+    define void @one_ptr_add() { ret void }
+    define void @add_to_offset() { ret void }
+    define void @two_ptr_adds_same_offset() { ret void }
+    define void @two_ptr_adds_different_offset() { ret void }
+    define void @ptr_add_chain() { ret void }
+
+    define void @dont_fold_negative_offset() { ret void }
+    define void @dont_min_offset_less_than_curr_offset() { ret void }
+    define void @dont_fold_max_offset() { ret void }
+    define void @dont_fold_offset_larger_than_type_alloc() { ret void }
+    define void @dont_fold_unsized_type() { ret void }
+...
+---
+name:            one_ptr_add
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0
+
+    ; We should fold the offset 1 into the G_GLOBAL_VALUE.
+
+    ; CHECK-LABEL: name: one_ptr_add
+    ; CHECK: liveins: $x0
+    ; CHECK: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @g + 1
+    ; CHECK: $x0 = COPY [[GV]](p0)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @g
+    %offset:_(s64) = G_CONSTANT i64 1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    $x0 = COPY %ptr_add
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            add_to_offset
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0
+
+    ; We should fold the offset 1 into the G_GLOBAL_VALUE, resulting in a
+    ; final offset of 4.
+
+    ; CHECK-LABEL: name: add_to_offset
+    ; CHECK: liveins: $x0
+    ; CHECK: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @g + 4
+    ; CHECK: $x0 = COPY [[GV]](p0)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @g + 3
+    %offset:_(s64) = G_CONSTANT i64 1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    $x0 = COPY %ptr_add
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            two_ptr_adds_same_offset
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; We're allowed to have more than one G_PTR_ADD use. We should fold 1 into
+    ; the G_GLOBAL_VALUE's offset.
+
+    ; CHECK-LABEL: name: two_ptr_adds_same_offset
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: %val1:_(s64) = COPY $x0
+    ; CHECK: %val2:_(s64) = COPY $x1
+    ; CHECK: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @g + 1
+    ; CHECK: G_STORE %val1(s64), [[GV]](p0) :: (store 8)
+    ; CHECK: G_STORE %val2(s64), [[GV]](p0) :: (store 8)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %val1:_(s64) = COPY $x0
+    %val2:_(s64) = COPY $x1
+    %global:_(p0) = G_GLOBAL_VALUE @g
+    %offset:_(s64) = G_CONSTANT i64 1
+    %ptr_add1:_(p0) = G_PTR_ADD %global, %offset(s64)
+    %ptr_add2:_(p0) = G_PTR_ADD %global, %offset(s64)
+    G_STORE %val1:_(s64), %ptr_add1 :: (store 8)
+    G_STORE %val2:_(s64), %ptr_add2 :: (store 8)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            two_ptr_adds_different_offset
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; The lowest offset G_PTR_ADD (2) should be folded into the G_GLOBAL_VALUE.
+    ;
+    ; The other G_PTR_ADD should have its offset decremented by 2.
+
+    ; CHECK-LABEL: name: two_ptr_adds_different_offset
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: %val1:_(s64) = COPY $x0
+    ; CHECK: %val2:_(s64) = COPY $x1
+    ; CHECK: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @g + 2
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; CHECK: %ptr_add2:_(p0) = G_PTR_ADD [[GV]], [[C]](s64)
+    ; CHECK: G_STORE %val1(s64), [[GV]](p0) :: (store 8)
+    ; CHECK: G_STORE %val2(s64), %ptr_add2(p0) :: (store 8)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %val1:_(s64) = COPY $x0
+    %val2:_(s64) = COPY $x1
+    %global:_(p0) = G_GLOBAL_VALUE @g
+    %offset1:_(s64) = G_CONSTANT i64 2
+    %offset2:_(s64) = G_CONSTANT i64 10
+    %ptr_add1:_(p0) = G_PTR_ADD %global, %offset1(s64)
+    %ptr_add2:_(p0) = G_PTR_ADD %global, %offset2(s64)
+    G_STORE %val1:_(s64), %ptr_add1 :: (store 8)
+    G_STORE %val2:_(s64), %ptr_add2 :: (store 8)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            ptr_add_chain
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0
+    ; We should be able to fold all of the G_PTR_ADDs, except for the last one
+    ; into the G_GLOBAL_VALUE.
+    ;
+    ; (TypeAllocSize = 4, so the offset on the G_GLOBAL_VALUE can't go above
+    ; that.)
+
+    ; CHECK-LABEL: name: ptr_add_chain
+    ; CHECK: liveins: $x0
+    ; CHECK: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @g + 1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+    ; CHECK: %dont_fold_me:_(p0) = G_PTR_ADD [[GV]], [[C]](s64)
+    ; CHECK: $x0 = COPY %dont_fold_me(p0)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @g
+    %offset:_(s64) = G_CONSTANT i64 1
+    %ptr_add1:_(p0) = G_PTR_ADD %global, %offset(s64)
+    %ptr_add2:_(p0) = G_PTR_ADD %ptr_add1, %offset(s64)
+    %ptr_add3:_(p0) = G_PTR_ADD %ptr_add2, %offset(s64)
+    %ptr_add4:_(p0) = G_PTR_ADD %ptr_add3, %offset(s64)
+    %dont_fold_me:_(p0) = G_PTR_ADD %ptr_add4, %offset(s64)
+    $x0 = COPY %dont_fold_me
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            dont_fold_negative_offset
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0
+
+    ; Do not add negative offsets to G_GLOBAL_VALUE.
+
+    ; CHECK-LABEL: name: dont_fold_negative_offset
+    ; CHECK: liveins: $x0
+    ; CHECK: %global:_(p0) = G_GLOBAL_VALUE @g
+    ; CHECK: %offset:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    ; CHECK: $x0 = COPY %ptr_add(p0)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @g
+    %offset:_(s64) = G_CONSTANT i64 -1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    $x0 = COPY %ptr_add
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            dont_min_offset_less_than_curr_offset
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0
+
+    ; Do not create smaller offsets. Ensures combine termination.
+
+    ; CHECK-LABEL: name: dont_min_offset_less_than_curr_offset
+    ; CHECK: liveins: $x0
+    ; CHECK: %global:_(p0) = G_GLOBAL_VALUE @g + 3
+    ; CHECK: %offset:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    ; CHECK: $x0 = COPY %ptr_add(p0)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @g + 3
+    %offset:_(s64) = G_CONSTANT i64 -1
+    %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    $x0 = COPY %ptr_add
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            dont_fold_max_offset
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $x0
+
+    ; 1 << 21 is the largest offset expressible in all object formats.
+    ; Don't fold it.
+
+    ; CHECK-LABEL: name: dont_fold_max_offset
+    ; CHECK: liveins: $x0
+    ; CHECK: %global:_(p0) = G_GLOBAL_VALUE @g
+    ; CHECK: %offset:_(s64) = G_CONSTANT i64 4292870144
+    ; CHECK: %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    ; CHECK: $x0 = COPY %ptr_add(p0)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @g
+    %offset:_(s64) = G_CONSTANT i64 4292870144 ; 1 << 21
+    %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    $x0 = COPY %ptr_add
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            dont_fold_offset_larger_than_type_alloc
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+
+    ; Type alloc size = 4, offset = 16. Don't fold.
+
+    ; CHECK-LABEL: name: dont_fold_offset_larger_than_type_alloc
+    ; CHECK: %global:_(p0) = G_GLOBAL_VALUE @g
+    ; CHECK: %offset:_(s64) = G_CONSTANT i64 16
+    ; CHECK: %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    ; CHECK: $x0 = COPY %ptr_add(p0)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @g
+    %offset:_(s64) = G_CONSTANT i64 16
+    %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    $x0 = COPY %ptr_add(p0)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            dont_fold_unsized_type
+alignment:       4
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    ; Check that we don't touch unsized globals.
+
+    ; CHECK-LABEL: name: dont_fold_unsized_type
+    ; CHECK: %global:_(p0) = G_GLOBAL_VALUE @unsized
+    ; CHECK: %offset:_(s64) = G_CONSTANT i64 16
+    ; CHECK: %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    ; CHECK: $x0 = COPY %ptr_add(p0)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %global:_(p0) = G_GLOBAL_VALUE @unsized
+    %offset:_(s64) = G_CONSTANT i64 16
+    %ptr_add:_(p0) = G_PTR_ADD %global, %offset(s64)
+    $x0 = COPY %ptr_add(p0)
+    RET_ReallyLR implicit $x0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/implicit_def_rbs_crash.mir b/llvm/test/CodeGen/AArch64/GlobalISel/implicit_def_rbs_crash.mir
new file mode 100644
index 0000000000000000000000000000000000000000..04aefb9207e79d31045e811865cbddc26764b211
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/implicit_def_rbs_crash.mir
@@ -0,0 +1,22 @@
+# RUN: llc -O0 -mtriple arm64-- -run-pass=regbankselect -verify-machineinstrs %s -o - | FileCheck %s
+
+# Check we don't crash given an non-generic implicit_def. These may
+# come from swifterror handling in the translator.
+# CHECK: IMPLICIT_DEF
+---
+name:            implicit_def_crash
+alignment:       4
+legalized:       true
+regBankSelected: false
+selected:        false
+failedISel:      false
+registers:
+  - { id: 0, class: gpr64all, preferred-register: '' }
+  - { id: 1, class: _, preferred-register: '' }
+  - { id: 2, class: gpr64all, preferred-register: '' }
+  - { id: 3, class: gpr64all, preferred-register: '' }
+body:             |
+  bb.1:
+    %0:gpr64all = IMPLICIT_DEF
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bzero-unsupported.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bzero-unsupported.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f12a125b2f73b71ae16ca6636d1cb1cf55178b5d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bzero-unsupported.mir
@@ -0,0 +1,14 @@
+# RUN: not llc -mtriple=aarch64 -global-isel-abort=1 -run-pass=legalizer -verify-machineinstrs %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=aarch64-linux-gnu -global-isel-abort=1 -run-pass=legalizer -verify-machineinstrs %s -o /dev/null 2>&1 | FileCheck %s
+...
+---
+name:            bzero
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    # CHECK: LLVM ERROR: unable to legalize instruction: G_BZERO
+    liveins: $x0, $x1
+    %ptr:_(p0) = COPY $x0
+    %width:_(s64) = COPY $x1
+    G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bzero.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bzero.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9e4908396d9d2fde38c2a570f9578661d142a813
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bzero.mir
@@ -0,0 +1,45 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-apple-ios -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+# Check that we can legalize G_BZERO on Darwin.
+
+...
+---
+name:            bzero
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: bzero
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: %ptr:_(p0) = COPY $x0
+    ; CHECK: %width:_(s64) = COPY $x1
+    ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: $x0 = COPY %ptr(p0)
+    ; CHECK: $x1 = COPY %width(s64)
+    ; CHECK: BL &bzero, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1
+    ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %width:_(s64) = COPY $x1
+    G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
+
+...
+---
+name:            bzero_tail_call
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: bzero_tail_call
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: %ptr:_(p0) = COPY $x0
+    ; CHECK: %width:_(s64) = COPY $x1
+    ; CHECK: $x0 = COPY %ptr(p0)
+    ; CHECK: $x1 = COPY %width(s64)
+    ; CHECK: TCRETURNdi &bzero, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit $x0, implicit $x1
+    %ptr:_(p0) = COPY $x0
+    %width:_(s64) = COPY $x1
+    G_BZERO %ptr(p0), %width(s64), 1 :: (store 4)
+    RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ctlz.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ctlz.mir
new file mode 100644
index 0000000000000000000000000000000000000000..428a39c5248631cb25752b2f5b43bd21a29df618
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ctlz.mir
@@ -0,0 +1,210 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=arm64-unknown-unknown -global-isel -run-pass=legalizer -global-isel-abort=1 %s -o - | FileCheck %s
+---
+name:            test_v8s8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_v8s8
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<8 x s8>) = COPY $d0
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(<8 x s8>) = G_CTLZ [[COPY]](<8 x s8>)
+    ; CHECK: $d0 = COPY [[CTLZ]](<8 x s8>)
+    ; CHECK: RET_ReallyLR implicit $d0
+    %0:_(<8 x s8>) = COPY $d0
+    %1:_(<8 x s8>) = G_CTLZ %0(<8 x s8>)
+    $d0 = COPY %1(<8 x s8>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4s16
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_v4s16
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(<4 x s16>) = G_CTLZ [[COPY]](<4 x s16>)
+    ; CHECK: $d0 = COPY [[CTLZ]](<4 x s16>)
+    ; CHECK: RET_ReallyLR implicit $d0
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_CTLZ %0(<4 x s16>)
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v2s32
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_v2s32
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(<2 x s32>) = G_CTLZ [[COPY]](<2 x s32>)
+    ; CHECK: $d0 = COPY [[CTLZ]](<2 x s32>)
+    ; CHECK: RET_ReallyLR implicit $d0
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_CTLZ %0(<2 x s32>)
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_s64
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_s64
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(s64) = G_CTLZ [[COPY1]](s64)
+    ; CHECK: $d0 = COPY [[CTLZ]](s64)
+    ; CHECK: RET_ReallyLR implicit $d0
+    %0:_(s64) = COPY $d0
+    %2:_(s64) = COPY %0(s64)
+    %1:_(s64) = G_CTLZ %2(s64)
+    $d0 = COPY %1(s64)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_s32
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $s0
+    ; CHECK-LABEL: name: test_s32
+    ; CHECK: liveins: $s0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[COPY1]](s32)
+    ; CHECK: $s0 = COPY [[CTLZ]](s32)
+    ; CHECK: RET_ReallyLR implicit $s0
+    %0:_(s32) = COPY $s0
+    %2:_(s32) = COPY %0(s32)
+    %1:_(s32) = G_CTLZ %2(s32)
+    $s0 = COPY %1(s32)
+    RET_ReallyLR implicit $s0
+
+...
+---
+name:            test_v16s8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: test_v16s8
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<16 x s8>) = COPY $q0
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(<16 x s8>) = G_CTLZ [[COPY]](<16 x s8>)
+    ; CHECK: $q0 = COPY [[CTLZ]](<16 x s8>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<16 x s8>) = COPY $q0
+    %1:_(<16 x s8>) = G_CTLZ %0(<16 x s8>)
+    $q0 = COPY %1(<16 x s8>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v8s16
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: test_v8s16
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(<8 x s16>) = G_CTLZ [[COPY]](<8 x s16>)
+    ; CHECK: $q0 = COPY [[CTLZ]](<8 x s16>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_CTLZ %0(<8 x s16>)
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v4s32
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: test_v4s32
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(<4 x s32>) = G_CTLZ [[COPY]](<4 x s32>)
+    ; CHECK: $q0 = COPY [[CTLZ]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_CTLZ %0(<4 x s32>)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+
+# The ZERO_UNDEF variants just lower into the vanilla ones.
+---
+name:            test_s32_zeroundef
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $s0
+    ; CHECK-LABEL: name: test_s32_zeroundef
+    ; CHECK: liveins: $s0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[COPY1]](s32)
+    ; CHECK: $s0 = COPY [[CTLZ]](s32)
+    ; CHECK: RET_ReallyLR implicit $s0
+    %0:_(s32) = COPY $s0
+    %2:_(s32) = COPY %0(s32)
+    %1:_(s32) = G_CTLZ_ZERO_UNDEF %2(s32)
+    $s0 = COPY %1(s32)
+    RET_ReallyLR implicit $s0
+
+...
+---
+name:            test_s64_zeroundef
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_s64_zeroundef
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(s64) = G_CTLZ [[COPY1]](s64)
+    ; CHECK: $d0 = COPY [[CTLZ]](s64)
+    ; CHECK: RET_ReallyLR implicit $d0
+    %0:_(s64) = COPY $d0
+    %2:_(s64) = COPY %0(s64)
+    %1:_(s64) = G_CTLZ_ZERO_UNDEF %2(s64)
+    $d0 = COPY %1(s64)
+    RET_ReallyLR implicit $d0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fptoi.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fptoi.mir
index 9bc639679bea17eeb78c410589f4e9079a79685b..94390563f6bab5f50f9930cd0be1e7c5ec9f7834 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fptoi.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fptoi.mir
@@ -1,32 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -run-pass=legalizer %s -o - | FileCheck %s
-
---- |
-  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-  target triple = "aarch64--"
-
-  define void @test_fptosi_s32_s32() { ret void }
-  define void @test_fptoui_s32_s32() { ret void }
-  define void @test_fptosi_s32_s64() { ret void }
-  define void @test_fptoui_s32_s64() { ret void }
-
-  define void @test_fptosi_s64_s32() { ret void }
-  define void @test_fptoui_s64_s32() { ret void }
-  define void @test_fptosi_s64_s64() { ret void }
-  define void @test_fptoui_s64_s64() { ret void }
-
-  define void @test_fptosi_s1_s32() { ret void }
-  define void @test_fptoui_s1_s32() { ret void }
-
-  define void @test_fptosi_s8_s64() { ret void }
-  define void @test_fptoui_s8_s64() { ret void }
-
-  define void @test_fptosi_s16_s32() { ret void }
-  define void @test_fptoui_s16_s32() { ret void }
-
-  define void @test_fptoui_v4s32() { ret void }
-  define void @test_fptosi_v4s32() { ret void }
-...
+# RUN: llc -mtriple=aarch64-- -O0 -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
 name:            test_fptosi_s32_s32
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshl.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshl.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f1945cdaf263cc9e03c2e4c771eaafd63ad52de1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshl.mir
@@ -0,0 +1,66 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=arm64-unknown-unknown -global-isel -run-pass=legalizer -global-isel-abort=1 %s -o - | FileCheck %s
+---
+name:            test_s32
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: test_s32
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY1]], [[C1]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s64)
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[AND1]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK: $w0 = COPY [[OR]](s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:_(s32) = COPY $w0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = G_FSHL %0(s32), %0, %1
+    $w0 = COPY %2(s32)
+    RET_ReallyLR implicit $w0
+
+...
+
+---
+name:            test_s64
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_s64
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY1]], [[C1]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s64) = G_AND [[XOR]], [[C]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[AND]](s64)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[C2]](s64)
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[AND1]](s64)
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; CHECK: $x0 = COPY [[OR]](s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = G_FSHL %0(s64), %0, %1(s64)
+    $x0 = COPY %2(s64)
+    RET_ReallyLR implicit $x0
+
+...
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshr.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshr.mir
new file mode 100644
index 0000000000000000000000000000000000000000..725461130edc06bc4c6533b7cfacf0595630e008
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fshr.mir
@@ -0,0 +1,66 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=arm64-unknown-unknown -global-isel -run-pass=legalizer -global-isel-abort=1 %s -o - | FileCheck %s
+---
+name:            test_s32
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: test_s32
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY1]], [[C1]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C2]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND1]](s32)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[AND]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK: $w0 = COPY [[OR]](s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:_(s32) = COPY $w0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = G_FSHR %0(s32), %0, %1
+    $w0 = COPY %2(s32)
+    RET_ReallyLR implicit $w0
+
+...
+
+---
+name:            test_s64
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_s64
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY1]], [[C1]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s64) = G_AND [[XOR]], [[C]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C2]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[AND1]](s64)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[AND]](s64)
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK: $x0 = COPY [[OR]](s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = G_FSHR %0(s64), %0, %1(s64)
+    $x0 = COPY %2(s64)
+    RET_ReallyLR implicit $x0
+
+...
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global-pic.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global-pic.mir
index 3fbd0125b31f4f3fb0659bca3701c6cd90d395ac..706bab2d0092fd342e5bd815e6591f52f54e2b50 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global-pic.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global-pic.mir
@@ -6,6 +6,7 @@
   target triple = "aarch64--"
   @var = external global i8
   define i8* @test_global() { ret i8* undef }
+  define i8* @test_global_with_offset() { ret i8* undef }
 ...
 ---
 name:            test_global
@@ -17,15 +18,6 @@ body: |
     ; We don't want to lower to G_ADD_LOW when we need a GOT access, or when the code
     ; model isn't 'Small'.
 
-    ; CHECK-LABEL: name: test_global
-    ; CHECK: [[ADRP:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var
-    ; CHECK: [[ADD_LOW:%[0-9]+]]:_(p0) = G_ADD_LOW [[ADRP]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var
-    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[ADD_LOW]](p0)
-    ; CHECK: $x0 = COPY [[PTRTOINT]](s64)
-    ; CMLARGE-LABEL: name: test_global
-    ; CMLARGE: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var
-    ; CMLARGE: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
-    ; CMLARGE: $x0 = COPY [[PTRTOINT]](s64)
     ; PIC-LABEL: name: test_global
     ; PIC: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var
     ; PIC: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
@@ -34,3 +26,17 @@ body: |
     %1:_(s64) = G_PTRTOINT %0
     $x0 = COPY %1
 ...
+---
+name:            test_global_with_offset
+registers:
+  - { id: 0, class: _ }
+body: |
+  bb.0:
+    ; PIC-LABEL: name: test_global_with_offset
+    ; PIC: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var + 1
+    ; PIC: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
+    ; PIC: $x0 = COPY [[PTRTOINT]](s64)
+    %0(p0) = G_GLOBAL_VALUE @var + 1
+    %1:_(s64) = G_PTRTOINT %0
+    $x0 = COPY %1
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global.mir
index da84fb43ca93201fd99679031a5d77796a0cf7b2..4338db9df94a4510800372e4856e53d7510d6399 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global.mir
@@ -7,6 +7,7 @@
   target triple = "aarch64--"
   @var = external dso_local global i8
   define i8* @test_global() { ret i8* undef }
+  define i8* @test_global_with_offset() { ret i8* undef }
 ...
 ---
 name:            test_global
@@ -17,16 +18,11 @@ body: |
 
     ; We don't want to lower to G_ADD_LOW when we need a GOT access, or when the code
     ; model isn't 'Small'.
-
     ; CHECK-LABEL: name: test_global
     ; CHECK: [[ADRP:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var
     ; CHECK: [[ADD_LOW:%[0-9]+]]:_(p0) = G_ADD_LOW [[ADRP]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var
     ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[ADD_LOW]](p0)
     ; CHECK: $x0 = COPY [[PTRTOINT]](s64)
-    ; PIC-LABEL: name: test_global
-    ; PIC: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var
-    ; PIC: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
-    ; PIC: $x0 = COPY [[PTRTOINT]](s64)
     ; CMLARGE-LABEL: name: test_global
     ; CMLARGE: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var
     ; CMLARGE: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
@@ -35,3 +31,23 @@ body: |
     %1:_(s64) = G_PTRTOINT %0
     $x0 = COPY %1
 ...
+---
+name:            test_global_with_offset
+body: |
+  bb.0:
+    ; When we legalize into ADRP + G_ADD_LOW, both should inherit the offset
+    ; from the original G_GLOBAL_VALUE.
+    ;
+    ; CHECK-LABEL: name: test_global_with_offset
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var + 1
+    ; CHECK: [[ADD_LOW:%[0-9]+]]:_(p0) = G_ADD_LOW [[ADRP]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var + 1
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[ADD_LOW]](p0)
+    ; CHECK: $x0 = COPY [[PTRTOINT]](s64)
+    ; CMLARGE-LABEL: name: test_global_with_offset
+    ; CMLARGE: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var + 1
+    ; CMLARGE: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
+    ; CMLARGE: $x0 = COPY [[PTRTOINT]](s64)
+    %0:_(p0) = G_GLOBAL_VALUE @var + 1
+    %1:_(s64) = G_PTRTOINT %0
+    $x0 = COPY %1
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sbfx.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sbfx.mir
new file mode 100644
index 0000000000000000000000000000000000000000..ccf7b1b926123ca37b1deed9fa1171a2e8d6ca6c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-sbfx.mir
@@ -0,0 +1,37 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+...
+---
+name:            s32
+body: |
+  bb.0.entry:
+    liveins: $w0
+    ; CHECK-LABEL: name: s32
+    ; CHECK: %copy:_(s32) = COPY $w0
+    ; CHECK: %lsb:_(s32) = G_CONSTANT i32 1
+    ; CHECK: %width:_(s32) = G_CONSTANT i32 2
+    ; CHECK: %sbfx:_(s32) = G_SBFX %copy, %lsb, %width
+    ; CHECK: $w0 = COPY %sbfx(s32)
+    %copy:_(s32) = COPY $w0
+    %lsb:_(s32) = G_CONSTANT i32 1
+    %width:_(s32) = G_CONSTANT i32 2
+    %sbfx:_(s32) = G_SBFX %copy(s32), %lsb, %width
+    $w0 = COPY %sbfx(s32)
+...
+---
+name:            s64
+body: |
+  bb.0.entry:
+    liveins: $x0
+    ; CHECK-LABEL: name: s64
+    ; CHECK: %copy:_(s64) = COPY $x0
+    ; CHECK: %lsb:_(s64) = G_CONSTANT i64 1
+    ; CHECK: %width:_(s64) = G_CONSTANT i64 2
+    ; CHECK: %sbfx:_(s64) = G_SBFX %copy, %lsb, %width
+    ; CHECK: $x0 = COPY %sbfx(s64)
+    %copy:_(s64) = COPY $x0
+    %lsb:_(s64) = G_CONSTANT i64 1
+    %width:_(s64) = G_CONSTANT i64 2
+    %sbfx:_(s64) = G_SBFX %copy(s64), %lsb, %width
+    $x0 = COPY %sbfx(s64)
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ubfx.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ubfx.mir
new file mode 100644
index 0000000000000000000000000000000000000000..0b32bf5923bd6c50bab1a9c01fe3741367bb992f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ubfx.mir
@@ -0,0 +1,37 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+...
+---
+name:            s32
+body: |
+  bb.0.entry:
+    liveins: $w0
+    ; CHECK-LABEL: name: s32
+    ; CHECK: %copy:_(s32) = COPY $w0
+    ; CHECK: %lsb:_(s32) = G_CONSTANT i32 1
+    ; CHECK: %width:_(s32) = G_CONSTANT i32 2
+    ; CHECK: %ubfx:_(s32) = G_UBFX %copy, %lsb, %width
+    ; CHECK: $w0 = COPY %ubfx(s32)
+    %copy:_(s32) = COPY $w0
+    %lsb:_(s32) = G_CONSTANT i32 1
+    %width:_(s32) = G_CONSTANT i32 2
+    %ubfx:_(s32) = G_UBFX %copy(s32), %lsb, %width
+    $w0 = COPY %ubfx(s32)
+...
+---
+name:            s64
+body: |
+  bb.0.entry:
+    liveins: $x0
+    ; CHECK-LABEL: name: s64
+    ; CHECK: %copy:_(s64) = COPY $x0
+    ; CHECK: %lsb:_(s64) = G_CONSTANT i64 1
+    ; CHECK: %width:_(s64) = G_CONSTANT i64 2
+    ; CHECK: %ubfx:_(s64) = G_UBFX %copy, %lsb, %width
+    ; CHECK: $x0 = COPY %ubfx(s64)
+    %copy:_(s64) = COPY $x0
+    %lsb:_(s64) = G_CONSTANT i64 1
+    %width:_(s64) = G_CONSTANT i64 2
+    %ubfx:_(s64) = G_UBFX %copy(s64), %lsb, %width
+    $x0 = COPY %ubfx(s64)
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 69d8cdec866bf59a99aa662fd18882cae126a156..36fd84757463d4197035b6a4d2c91626f42b530f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -285,9 +285,16 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_FSHL (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_FSHR (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_ROTR (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: G_FSHR (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: G_ROTL (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_ICMP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
@@ -541,8 +548,8 @@
 # DEBUG-NEXT: .. the first uncovered type index: 2, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_CTLZ_ZERO_UNDEF (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_CTPOP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -618,6 +625,7 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_MEMCPY (opcode {{[0-9]+}}): 3 type indices, 1 imm index
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_MEMMOVE (opcode {{[0-9]+}}): 3 type indices, 1 imm index
@@ -628,6 +636,9 @@
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_BZERO (opcode {{[0-9]+}}): 2 type indices, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_VECREDUCE_SEQ_FADD (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -673,7 +684,13 @@
 # DEBUG-NEXT: G_VECREDUCE_UMIN (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
-
+# DEBUG-NEXT: G_SBFX (opcode {{[0-9]+}}): 1 type index, 0 imm indices
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: G_UBFX (opcode {{[0-9]+}}): 1 type index, 0 imm indices
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # CHECK-NOT: ill-defined
 
 ---
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/lifetime-marker-no-dce.mir b/llvm/test/CodeGen/AArch64/GlobalISel/lifetime-marker-no-dce.mir
new file mode 100644
index 0000000000000000000000000000000000000000..16f2d70cd604bd36f31ed15e540eec8f6eb1b199
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/lifetime-marker-no-dce.mir
@@ -0,0 +1,24 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+# Check that we don't DCE the lifetime markers even though they don't have any users.
+---
+name:            test_lifetime_no_dce
+alignment:       4
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+stack:
+  - { id: 0, size: 4, alignment: 4 }
+machineFunctionInfo: {}
+body:             |
+  bb.1:
+    ;%0:_(p0) = G_FRAME_INDEX %stack.0.slot
+    ; CHECK-LABEL: name: test_lifetime_no_dce
+    ; CHECK: LIFETIME_START %stack.0
+    ; CHECK: LIFETIME_END %stack.0
+    ; CHECK: RET_ReallyLR
+    LIFETIME_START %stack.0
+    LIFETIME_END %stack.0
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/no-neon-no-fp.ll b/llvm/test/CodeGen/AArch64/GlobalISel/no-neon-no-fp.ll
index 6408f2e0b0d5a6d34fcae8c343d200507cd8156b..822c6252fbe3b47739e89d43a528efc59a7755cf 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/no-neon-no-fp.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/no-neon-no-fp.ll
@@ -1,13 +1,24 @@
-; RUN: not --crash llc -o - -verify-machineinstrs -global-isel -global-isel-abort=1 -stop-after=legalizer %s 2>&1 | FileCheck %s
+; RUN: llc -o - -verify-machineinstrs -global-isel -global-isel-abort=2 %s 2>&1 | FileCheck %s
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-unknown"
 
-; CHECK: unable to legalize instruction: G_STORE %1:_(s128), %0:_(p0) :: (store 16 into %ir.ptr) (in function: foo)
+; We should fall back in the translator if we don't have no-neon/no-fp support.
+; CHECK: Instruction selection used fallback path for foo
 define void @foo(i128 *%ptr) #0 align 2 {
 entry:
   store i128 0, i128* %ptr, align 16
   ret void
 }
 
+; This test below will crash the legalizer due to trying to use legacy rules,
+; if we don't fall back in the translator.
+declare i1 @zoo()
+; CHECK: Instruction selection used fallback path for bar
+define i32 @bar() #0 {
+  %1 = call zeroext i1 @zoo()
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
 attributes #0 = { "use-soft-float"="false" "target-features"="-fp-armv8,-neon" }
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-bzero.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-bzero.mir
new file mode 100644
index 0000000000000000000000000000000000000000..73fd9d17230ca29c7948941d52ac99fe2072c51f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-bzero.mir
@@ -0,0 +1,147 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-apple-ios -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=DARWIN
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=UNKNOWN
+#
+# Check that on Darwin we can combine to G_BZERO. Without Darwin, this should
+# stay as memset.
+
+--- |
+  define void @bzero_unknown_width() { unreachable }
+  define void @bzero_tail_unknown_width() { unreachable }
+  define void @bzero_constant_width() { unreachable }
+  define void @bzero_constant_width_minsize() minsize { unreachable }
+  define void @not_zero() minsize { unreachable }
+...
+---
+name:            bzero_unknown_width
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; Always use G_BZERO when the memset width is unknown on Darwin.
+
+    ; DARWIN-LABEL: name: bzero_unknown_width
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %width:_(s64) = COPY $x1
+    ; DARWIN: G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: bzero_unknown_width
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %zero:_(s8) = G_CONSTANT i8 0
+    ; UNKNOWN: %width:_(s64) = COPY $x1
+    ; UNKNOWN: G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %zero:_(s8) = G_CONSTANT i8 0
+    %width:_(s64) = COPY $x1
+    G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
+...
+---
+name:            bzero_tail_unknown_width
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; DARWIN-LABEL: name: bzero_tail_unknown_width
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %width:_(s64) = COPY $x1
+    ; DARWIN: G_BZERO %ptr(p0), %width(s64), 1 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: bzero_tail_unknown_width
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %zero:_(s8) = G_CONSTANT i8 0
+    ; UNKNOWN: %width:_(s64) = COPY $x1
+    ; UNKNOWN: G_MEMSET %ptr(p0), %zero(s8), %width(s64), 1 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %zero:_(s8) = G_CONSTANT i8 0
+    %width:_(s64) = COPY $x1
+    G_MEMSET %ptr(p0), %zero(s8), %width(s64), 1 :: (store 4)
+    RET_ReallyLR
+...
+---
+name:            bzero_constant_width
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; For values >256, we should use G_BZERO on Darwin.
+
+    ; DARWIN-LABEL: name: bzero_constant_width
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %width:_(s64) = G_CONSTANT i64 1024
+    ; DARWIN: G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: bzero_constant_width
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %zero:_(s8) = G_CONSTANT i8 0
+    ; UNKNOWN: %width:_(s64) = G_CONSTANT i64 1024
+    ; UNKNOWN: G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %zero:_(s8) = G_CONSTANT i8 0
+    %width:_(s64) = G_CONSTANT i64 1024
+    G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
+...
+---
+name:            bzero_constant_width_minsize
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; With minsize, we should always use G_BZERO to avoid a copy from wzr.
+
+    ; DARWIN-LABEL: name: bzero_constant_width_minsize
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %width:_(s64) = G_CONSTANT i64 256
+    ; DARWIN: G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: bzero_constant_width_minsize
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %zero:_(s8) = G_CONSTANT i8 0
+    ; UNKNOWN: %width:_(s64) = G_CONSTANT i64 256
+    ; UNKNOWN: G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %zero:_(s8) = G_CONSTANT i8 0
+    %width:_(s64) = G_CONSTANT i64 256
+    G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
+...
+---
+name:            not_zero
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; When the value isn't 0, don't create G_BZERO.
+
+    ; DARWIN-LABEL: name: not_zero
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %not_zero:_(s8) = G_CONSTANT i8 1
+    ; DARWIN: %width:_(s64) = G_CONSTANT i64 256
+    ; DARWIN: G_MEMSET %ptr(p0), %not_zero(s8), %width(s64), 0 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: not_zero
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %not_zero:_(s8) = G_CONSTANT i8 1
+    ; UNKNOWN: %width:_(s64) = G_CONSTANT i64 256
+    ; UNKNOWN: G_MEMSET %ptr(p0), %not_zero(s8), %width(s64), 0 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %not_zero:_(s8) = G_CONSTANT i8 1
+    %width:_(s64) = G_CONSTANT i64 256
+    G_MEMSET %ptr(p0), %not_zero(s8), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-add-low.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-add-low.mir
new file mode 100644
index 0000000000000000000000000000000000000000..2272aaf286733bb248254d1c32798bcee219ff81
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-add-low.mir
@@ -0,0 +1,70 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+
+    @x = external hidden local_unnamed_addr global i32*, align 8
+
+    define void @select_add_low_without_offset() { ret void }
+    define void @select_add_low_with_offset() { ret void }
+    define void @select_add_low_without_adrp() { ret void }
+
+...
+---
+name:            select_add_low_without_offset
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: select_add_low_without_offset
+    ; CHECK: liveins: $x0
+    ; CHECK: %add_low:gpr64 = MOVaddr target-flags(aarch64-page) @x, target-flags(aarch64-pageoff, aarch64-nc) @x
+    ; CHECK: $x0 = COPY %add_low
+    ; CHECK: RET_ReallyLR implicit $x0
+    %copy:gpr(p0) = COPY $x0
+    %adrp:gpr64(p0) = ADRP target-flags(aarch64-page) @x
+    %add_low:gpr(p0) = G_ADD_LOW %adrp(p0), target-flags(aarch64-pageoff, aarch64-nc) @x
+    $x0 = COPY %add_low
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            select_add_low_with_offset
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: select_add_low_with_offset
+    ; CHECK: liveins: $x0
+    ; CHECK: %add_low:gpr64 = MOVaddr target-flags(aarch64-page) @x + 1, target-flags(aarch64-pageoff, aarch64-nc) @x + 1
+    ; CHECK: $x0 = COPY %add_low
+    ; CHECK: RET_ReallyLR implicit $x0
+    %copy:gpr(p0) = COPY $x0
+    %adrp:gpr64(p0) = ADRP target-flags(aarch64-page) @x + 1
+    %add_low:gpr(p0) = G_ADD_LOW %adrp(p0), target-flags(aarch64-pageoff, aarch64-nc) @x + 1
+    $x0 = COPY %add_low
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            select_add_low_without_adrp
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: select_add_low_without_adrp
+    ; CHECK: liveins: $x0
+    ; CHECK: %ptr:gpr64sp = COPY $x0
+    ; CHECK: %add_low:gpr64sp = ADDXri %ptr, target-flags(aarch64-pageoff, aarch64-nc) @x, 0
+    ; CHECK: $x0 = COPY %add_low
+    ; CHECK: RET_ReallyLR implicit $x0
+    %ptr:gpr(p0) = COPY $x0
+    %add_low:gpr(p0) = G_ADD_LOW %ptr(p0), target-flags(aarch64-pageoff, aarch64-nc) @x
+    $x0 = COPY %add_low
+    RET_ReallyLR implicit $x0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-ctlz.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-ctlz.mir
index 9873d49b2a3884a0ce1bf42fd9e409aa0ac476f7..7b0112f9e7f2b60befe1588591ded5555a82ed72 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-ctlz.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-ctlz.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -mtriple=arm64-unknown-unknown -global-isel -run-pass=instruction-select %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=arm64-unknown-unknown -global-isel -run-pass=instruction-select -global-isel-abort=1 %s -o - | FileCheck %s
 
 name:            test_v8s8
 alignment:       4
@@ -179,22 +179,3 @@ body:             |
     RET_ReallyLR implicit $q0
 
 ...
----
-name:            test_v2s64
-alignment:       4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0:
-    liveins: $q0
-    ; CHECK-LABEL: name: test_v2s64
-    ; CHECK: liveins: $q0
-    ; CHECK: [[COPY:%[0-9]+]]:fpr(<2 x s64>) = COPY $q0
-    ; CHECK: [[CTLZ:%[0-9]+]]:fpr(<2 x s64>) = G_CTLZ [[COPY]](<2 x s64>)
-    ; CHECK: $q0 = COPY [[CTLZ]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
-    %0:fpr(<2 x s64>) = COPY $q0
-    %1:fpr(<2 x s64>) = G_CTLZ %0(<2 x s64>)
-    $q0 = COPY %1(<2 x s64>)
-    RET_ReallyLR implicit $q0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-gv-with-offset.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-gv-with-offset.mir
new file mode 100644
index 0000000000000000000000000000000000000000..7533731b2bd8e8df3576d0571689a09182ff5cb5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-gv-with-offset.mir
@@ -0,0 +1,38 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -code-model=large -run-pass=instruction-select -verify-machineinstrs -O0 %s -o - | FileCheck %s --check-prefix=LARGE
+# RUN: llc -mtriple=aarch64 -code-model=small -run-pass=instruction-select -verify-machineinstrs -O0 %s -o - | FileCheck %s --check-prefix=SMALL
+# RUN: llc -mtriple=aarch64 -code-model=tiny -run-pass=instruction-select -verify-machineinstrs -O0 %s -o - | FileCheck %s --check-prefix=TINY
+
+--- |
+   @g = external hidden global i32
+   define void @select_gv_with_offset() { ret void }
+...
+---
+name:            select_gv_with_offset
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; LARGE-LABEL: name: select_gv_with_offset
+    ; LARGE: liveins: $x0
+    ; LARGE: [[MOVZXi:%[0-9]+]]:gpr64 = MOVZXi target-flags(aarch64-g0, aarch64-nc) @g + 1, 0
+    ; LARGE: [[MOVKXi:%[0-9]+]]:gpr64 = MOVKXi [[MOVZXi]], target-flags(aarch64-g1, aarch64-nc) @g + 1, 16
+    ; LARGE: [[MOVKXi1:%[0-9]+]]:gpr64 = MOVKXi [[MOVKXi]], target-flags(aarch64-g2, aarch64-nc) @g + 1, 32
+    ; LARGE: %g:gpr64 = MOVKXi [[MOVKXi1]], target-flags(aarch64-g3) @g + 1, 48
+    ; LARGE: $x0 = COPY %g
+    ; LARGE: RET_ReallyLR implicit $x0
+    ; SMALL-LABEL: name: select_gv_with_offset
+    ; SMALL: liveins: $x0
+    ; SMALL: %g:gpr64 = MOVaddr target-flags(aarch64-page) @g + 1, target-flags(aarch64-pageoff, aarch64-nc) @g + 1
+    ; SMALL: $x0 = COPY %g
+    ; SMALL: RET_ReallyLR implicit $x0
+    ; TINY-LABEL: name: select_gv_with_offset
+    ; TINY: liveins: $x0
+    ; TINY: %g:gpr64 = ADR @g + 1
+    ; TINY: $x0 = COPY %g
+    ; TINY: RET_ReallyLR implicit $x0
+    %g:gpr(p0) = G_GLOBAL_VALUE @g + 1
+    $x0 = COPY %g(p0)
+    RET_ReallyLR implicit $x0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-sbfx.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-sbfx.mir
new file mode 100644
index 0000000000000000000000000000000000000000..daae44e5fec5940dae0910982d46958a7ef330b0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-sbfx.mir
@@ -0,0 +1,70 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+...
+---
+name:            sbfx_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0
+    ; CHECK-LABEL: name: sbfx_s32
+    ; CHECK: liveins: $w0
+    ; CHECK: %copy:gpr32 = COPY $w0
+    ; CHECK: %sbfx:gpr32 = SBFMWri %copy, 0, 10
+    ; CHECK: $w0 = COPY %sbfx
+    ; CHECK: RET_ReallyLR implicit $w0
+    %copy:gpr(s32) = COPY $w0
+    %cst1:gpr(s32) = G_CONSTANT i32 0
+    %cst2:gpr(s32) = G_CONSTANT i32 10
+    %sbfx:gpr(s32) = G_SBFX %copy, %cst1, %cst2
+    $w0 = COPY %sbfx
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            sbfx_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: sbfx_s64
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64 = COPY $x0
+    ; CHECK: %sbfx:gpr64 = SBFMXri %copy, 0, 10
+    ; CHECK: $x0 = COPY %sbfx
+    ; CHECK: RET_ReallyLR implicit $x0
+    %copy:gpr(s64) = COPY $x0
+    %cst1:gpr(s64) = G_CONSTANT i64 0
+    %cst2:gpr(s64) = G_CONSTANT i64 10
+    %sbfx:gpr(s64) = G_SBFX %copy, %cst1, %cst2
+    $x0 = COPY %sbfx
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            sbfx_s32_31_31
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0
+    ; This is just an asr, so it's okay.
+
+    ; CHECK-LABEL: name: sbfx_s32_31_31
+    ; CHECK: liveins: $w0
+    ; CHECK: %copy:gpr32 = COPY $w0
+    ; CHECK: %sbfx:gpr32 = SBFMWri %copy, 31, 31
+    ; CHECK: $w0 = COPY %sbfx
+    ; CHECK: RET_ReallyLR implicit $w0
+    %copy:gpr(s32) = COPY $w0
+    %cst1:gpr(s32) = G_CONSTANT i32 31
+    %cst2:gpr(s32) = G_CONSTANT i32 31
+    %sbfx:gpr(s32) = G_SBFX %copy, %cst1, %cst2
+    $w0 = COPY %sbfx
+    RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir
index 62c28b906dea56fcd58f574c1433f12f4c954dfd..5bbd2a73c14ea843181cbd923dc3c87fd2d0b1c2 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-store.mir
@@ -41,7 +41,8 @@
 
   @x = external hidden local_unnamed_addr global i32*, align 8
   define void @store_adrp_add_low() { ret void }
-
+  define void @store_adrp_add_low_foldable_offset() { ret void }
+  define void @store_adrp_add_low_unfoldable_offset() { ret void }
 ...
 
 ---
@@ -622,3 +623,43 @@ body:             |
     %adrp:gpr64(p0) = ADRP target-flags(aarch64-page) @x
     %add_low:gpr(p0) = G_ADD_LOW %adrp(p0), target-flags(aarch64-pageoff, aarch64-nc) @x
     G_STORE %copy(p0), %add_low(p0) :: (store 8 into @x)
+
+...
+---
+name:            store_adrp_add_low_foldable_offset
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: store_adrp_add_low_foldable_offset
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64all = COPY $x0
+    ; CHECK: %adrp:gpr64common = ADRP target-flags(aarch64-page) @x + 8
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY %copy
+    ; CHECK: STRXui [[COPY]], %adrp, target-flags(aarch64-pageoff, aarch64-nc) @x + 8 :: (store 8 into @x)
+    %copy:gpr(p0) = COPY $x0
+    %adrp:gpr64(p0) = ADRP target-flags(aarch64-page) @x + 8
+    %add_low:gpr(p0) = G_ADD_LOW %adrp(p0), target-flags(aarch64-pageoff, aarch64-nc) @x + 8
+    G_STORE %copy(p0), %add_low(p0) :: (store 8 into @x)
+
+...
+---
+name:            store_adrp_add_low_unfoldable_offset
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: store_adrp_add_low_unfoldable_offset
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64all = COPY $x0
+    ; CHECK: %add_low:gpr64common = MOVaddr target-flags(aarch64-page) @x + 3, target-flags(aarch64-pageoff, aarch64-nc) @x + 3
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY %copy
+    ; CHECK: STRXui [[COPY]], %add_low, 0 :: (store 8 into @x)
+    %copy:gpr(p0) = COPY $x0
+    %adrp:gpr64(p0) = ADRP target-flags(aarch64-page) @x + 3
+    %add_low:gpr(p0) = G_ADD_LOW %adrp(p0), target-flags(aarch64-pageoff, aarch64-nc) @x + 3
+    G_STORE %copy(p0), %add_low(p0) :: (store 8 into @x)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-ubfx.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-ubfx.mir
new file mode 100644
index 0000000000000000000000000000000000000000..3f629b69b6b0ce29dbefcbcbc32b81b059354895
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-ubfx.mir
@@ -0,0 +1,71 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+...
+---
+name:            ubfx_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0
+    ; CHECK-LABEL: name: ubfx_s32
+    ; CHECK: liveins: $w0
+    ; CHECK: %copy:gpr32 = COPY $w0
+    ; CHECK: %ubfx:gpr32 = UBFMWri %copy, 0, 10
+    ; CHECK: $w0 = COPY %ubfx
+    ; CHECK: RET_ReallyLR implicit $w0
+    %copy:gpr(s32) = COPY $w0
+    %cst1:gpr(s32) = G_CONSTANT i32 0
+    %cst2:gpr(s32) = G_CONSTANT i32 10
+    %ubfx:gpr(s32) = G_UBFX %copy, %cst1, %cst2
+    $w0 = COPY %ubfx
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            ubfx_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: ubfx_s64
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:gpr64 = COPY $x0
+    ; CHECK: %ubfx:gpr64 = UBFMXri %copy, 0, 10
+    ; CHECK: $x0 = COPY %ubfx
+    ; CHECK: RET_ReallyLR implicit $x0
+    %copy:gpr(s64) = COPY $x0
+    %cst1:gpr(s64) = G_CONSTANT i64 0
+    %cst2:gpr(s64) = G_CONSTANT i64 10
+    %ubfx:gpr(s64) = G_UBFX %copy, %cst1, %cst2
+    $x0 = COPY %ubfx
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            ubfx_s32_31_31
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0
+
+    ; This is just a lsr, so it's okay.
+
+    ; CHECK-LABEL: name: ubfx_s32_31_31
+    ; CHECK: liveins: $w0
+    ; CHECK: %copy:gpr32 = COPY $w0
+    ; CHECK: %ubfx:gpr32 = UBFMWri %copy, 31, 31
+    ; CHECK: $w0 = COPY %ubfx
+    ; CHECK: RET_ReallyLR implicit $w0
+    %copy:gpr(s32) = COPY $w0
+    %cst1:gpr(s32) = G_CONSTANT i32 31
+    %cst2:gpr(s32) = G_CONSTANT i32 31
+    %ubfx:gpr(s32) = G_UBFX %copy, %cst1, %cst2
+    $w0 = COPY %ubfx
+    RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/arm64-clrsb.ll b/llvm/test/CodeGen/AArch64/arm64-clrsb.ll
index 149a466a11470037541f81abad025678e5ba0255..412c2b00a5ac09d5b7e8b9ca3a6d24f8b9fa4bce 100644
--- a/llvm/test/CodeGen/AArch64/arm64-clrsb.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-clrsb.ll
@@ -41,3 +41,38 @@ entry:
 ; GISEL-LABEL: clrsb64
 ; GISEL:   cls [[TEMP:x[0-9]+]], [[TEMP]]
 }
+
+; Function Attrs: nounwind ssp
+; FALLBACK-NOT: remark{{.*}}clrsb32_zeroundef
+define i32 @clrsb32_zeroundef(i32 %x) #2 {
+entry:
+  %shr = ashr i32 %x, 31
+  %xor = xor i32 %shr, %x
+  %mul = shl i32 %xor, 1
+  %add = or i32 %mul, 1
+  %0 = tail call i32 @llvm.ctlz.i32(i32 %add, i1 true)
+
+  ret i32 %0
+; CHECK-LABEL: clrsb32_zeroundef
+; CHECK:   cls [[TEMP:w[0-9]+]], [[TEMP]]
+
+; GISEL-LABEL: clrsb32_zeroundef
+; GISEL: cls [[TEMP:w[0-9]+]], [[TEMP]]
+}
+
+; Function Attrs: nounwind ssp
+; FALLBACK-NOT: remark{{.*}}clrsb64
+define i64 @clrsb64_zeroundef(i64 %x) #3 {
+entry:
+  %shr = ashr i64 %x, 63
+  %xor = xor i64 %shr, %x
+  %mul = shl nsw i64 %xor, 1
+  %add = or i64 %mul, 1
+  %0 = tail call i64 @llvm.ctlz.i64(i64 %add, i1 true)
+
+  ret i64 %0
+; CHECK-LABEL: clrsb64_zeroundef
+; CHECK:   cls [[TEMP:x[0-9]+]], [[TEMP]]
+; GISEL-LABEL: clrsb64_zeroundef
+; GISEL:   cls [[TEMP:x[0-9]+]], [[TEMP]]
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-csel.ll b/llvm/test/CodeGen/AArch64/arm64-csel.ll
index f031710a4dcb6f65ec83405b1d481897d3285160..44e951ed69e1dcdde60029fab24fdbabee2107b9 100644
--- a/llvm/test/CodeGen/AArch64/arm64-csel.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-csel.ll
@@ -269,3 +269,44 @@ define i64 @foo23(i64 %x) {
   %res = select i1 %cmp, i64 1, i64 6
   ret i64 %res
 }
+
+define i16 @foo24(i8* nocapture readonly %A, i8* nocapture readonly %B) {
+; CHECK-LABEL: foo24:
+; CHECK:       ldrb    w[[W8:[0-9]+]], [x1]
+; CHECK-NEXT:  ldrb    w[[W9:[0-9]+]], [x0]
+; CHECK-NEXT:  cmp     w[[W8]], #33
+; CHECK-NEXT:  cset    w[[W8]], hi
+; CHECK-NEXT:  cmp     w[[W9]], #3
+; CHECK-NEXT:  cinc    w0, w[[W8]], hi
+; CHECK-NEXT:  ret
+entry:
+  %0 = load i8, i8* %A, align 1
+  %cmp = icmp ugt i8 %0, 3
+  %conv1 = zext i1 %cmp to i16
+  %1 = load i8, i8* %B, align 1
+  %cmp4 = icmp ugt i8 %1, 33
+  %conv5 = zext i1 %cmp4 to i16
+  %add = add nuw nsw i16 %conv5, %conv1
+  ret i16 %add
+}
+
+define i64 @foo25(i64* nocapture readonly %A, i64* nocapture readonly %B) {
+; CHECK-LABEL: foo25:
+; CHECK:       ldr    x[[X8:[0-9]+]], [x1]
+; CHECK-NEXT:  ldr    x[[X9:[0-9]+]], [x0]
+; CHECK-NEXT:  cmp    x[[X8]], #33
+; CHECK-NEXT:  cset   w[[W8]], hi
+; CHECK-NEXT:  cmp    x[[X9]], #3
+; CHECK-NEXT:  cinc   x0, x[[X8]], hi
+; CHECK-NEXT:  ret
+entry:
+  %0 = load i64, i64* %A, align 1
+  %cmp = icmp ugt i64 %0, 3
+  %conv1 = zext i1 %cmp to i64
+  %1 = load i64, i64* %B, align 1
+  %cmp4 = icmp ugt i64 %1, 33
+  %conv5 = zext i1 %cmp4 to i64
+  %add = add nuw nsw i64 %conv5, %conv1
+  ret i64 %add
+}
+
diff --git a/llvm/test/CodeGen/AArch64/arm64-vcvt.ll b/llvm/test/CodeGen/AArch64/arm64-vcvt.ll
index 67eba3f4e3075452327d571cace72100c4bce3c7..43ed1aba735c179a997e1dab60084d59de7aef0a 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vcvt.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vcvt.ll
@@ -590,7 +590,7 @@ define <2 x float> @frintn_2s(<2 x float> %A) nounwind {
 ;CHECK-NOT: ld1
 ;CHECK: frintn.2s v0, v0
 ;CHECK-NEXT: ret
-	%tmp3 = call <2 x float> @llvm.aarch64.neon.frintn.v2f32(<2 x float> %A)
+	%tmp3 = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %A)
 	ret <2 x float> %tmp3
 }
 
@@ -599,7 +599,7 @@ define <4 x float> @frintn_4s(<4 x float> %A) nounwind {
 ;CHECK-NOT: ld1
 ;CHECK: frintn.4s v0, v0
 ;CHECK-NEXT: ret
-	%tmp3 = call <4 x float> @llvm.aarch64.neon.frintn.v4f32(<4 x float> %A)
+	%tmp3 = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %A)
 	ret <4 x float> %tmp3
 }
 
@@ -608,13 +608,13 @@ define <2 x double> @frintn_2d(<2 x double> %A) nounwind {
 ;CHECK-NOT: ld1
 ;CHECK: frintn.2d v0, v0
 ;CHECK-NEXT: ret
-	%tmp3 = call <2 x double> @llvm.aarch64.neon.frintn.v2f64(<2 x double> %A)
+	%tmp3 = call <2 x double> @llvm.roundeven.v2f64(<2 x double> %A)
 	ret <2 x double> %tmp3
 }
 
-declare <2 x float> @llvm.aarch64.neon.frintn.v2f32(<2 x float>) nounwind readnone
-declare <4 x float> @llvm.aarch64.neon.frintn.v4f32(<4 x float>) nounwind readnone
-declare <2 x double> @llvm.aarch64.neon.frintn.v2f64(<2 x double>) nounwind readnone
+declare <2 x float> @llvm.roundeven.v2f32(<2 x float>) nounwind readnone
+declare <4 x float> @llvm.roundeven.v4f32(<4 x float>) nounwind readnone
+declare <2 x double> @llvm.roundeven.v2f64(<2 x double>) nounwind readnone
 
 ; FALLBACK-NOT: remark{{.*}}frintp_2s
 define <2 x float> @frintp_2s(<2 x float> %A) nounwind {
diff --git a/llvm/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll b/llvm/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll
index 524b6e0528e0880376f7214c21fbb3a3dce442ff..65bb08ec2e4ecdea245cb5e68c1080133364f83f 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll
@@ -245,6 +245,20 @@ define %v4f16 @test_v4f16.round(%v4f16 %a) {
   %1 =  call %v4f16 @llvm.round.v4f16(%v4f16 %a)
   ret %v4f16 %1
 }
+define %v4f16 @test_v4f16.roundeven(%v4f16 %a) {
+  ; CHECK-LABEL:          test_v4f16.roundeven:
+  ; CHECK-NOFP16-COUNT-4: frintn s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintn.4h
+  ; CHECK-FP16-NEXT:      ret
+  ; GISEL-LABEL:          test_v4f16.roundeven:
+  ; GISEL-NOFP16-COUNT-4: frintn s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           frintn.4h
+  ; GISEL-FP16-NEXT:      ret
+  %1 =  call %v4f16 @llvm.roundeven.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
 
 declare %v4f16 @llvm.sqrt.v4f16(%v4f16) #0
 declare %v4f16 @llvm.powi.v4f16(%v4f16, i32) #0
@@ -264,6 +278,7 @@ declare %v4f16 @llvm.trunc.v4f16(%v4f16) #0
 declare %v4f16 @llvm.rint.v4f16(%v4f16) #0
 declare %v4f16 @llvm.nearbyint.v4f16(%v4f16) #0
 declare %v4f16 @llvm.round.v4f16(%v4f16) #0
+declare %v4f16 @llvm.roundeven.v4f16(%v4f16) #0
 
 ;;;
 
@@ -502,6 +517,20 @@ define %v8f16 @test_v8f16.round(%v8f16 %a) {
   %1 =  call %v8f16 @llvm.round.v8f16(%v8f16 %a)
   ret %v8f16 %1
 }
+define %v8f16 @test_v8f16.roundeven(%v8f16 %a) {
+  ; CHECK-LABEL:          test_v8f16.roundeven:
+  ; CHECK-NOFP16-COUNT-8: frintn s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintn.8h
+  ; CHECK-FP16-NEXT:      ret
+  ; GISEL-LABEL:          test_v8f16.roundeven:
+  ; GISEL-NOFP16-COUNT-8: frintn s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           frintn.8h
+  ; GISEL-FP16-NEXT:      ret
+  %1 =  call %v8f16 @llvm.roundeven.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
 
 declare %v8f16 @llvm.sqrt.v8f16(%v8f16) #0
 declare %v8f16 @llvm.powi.v8f16(%v8f16, i32) #0
@@ -521,6 +550,7 @@ declare %v8f16 @llvm.trunc.v8f16(%v8f16) #0
 declare %v8f16 @llvm.rint.v8f16(%v8f16) #0
 declare %v8f16 @llvm.nearbyint.v8f16(%v8f16) #0
 declare %v8f16 @llvm.round.v8f16(%v8f16) #0
+declare %v8f16 @llvm.roundeven.v8f16(%v8f16) #0
 
 ;;; Float vectors
 
diff --git a/llvm/test/CodeGen/AArch64/csinc-cmp-removal.mir b/llvm/test/CodeGen/AArch64/csinc-cmp-removal.mir
new file mode 100644
index 0000000000000000000000000000000000000000..932c9aa213146ef4999ee7fe146705f0183819ce
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/csinc-cmp-removal.mir
@@ -0,0 +1,313 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=peephole-opt -verify-machineinstrs %s -o - | FileCheck %s
+---
+name:            remove_subswr_after_csincwr
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: remove_subswr_after_csincwr
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $x1
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+  ; CHECK:   [[DEF:%[0-9]+]]:gpr64 = IMPLICIT_DEF
+  ; CHECK:   [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr killed [[DEF]], [[COPY]], implicit-def $nzcv
+  ; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32common = CSINCWr $wzr, $wzr, 1, implicit $nzcv
+  ; CHECK:   [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri killed [[CSINCWr]], 1, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 1, %bb.2, implicit $nzcv
+  ; CHECK:   B %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   B %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $x1
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    %1:gpr64common = COPY $x1
+    %2:gpr64 = IMPLICIT_DEF
+    %3:gpr64 = SUBSXrr killed %2:gpr64, %1:gpr64common, implicit-def $nzcv
+    %4:gpr32common = CSINCWr $wzr, $wzr, 1, implicit $nzcv
+    %5:gpr32 = SUBSWri killed %4:gpr32common, 1, 0, implicit-def $nzcv
+    Bcc 1, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    B %bb.2
+
+  bb.2:
+    RET_ReallyLR
+
+...
+---
+name:            remove_subsxr_after_csincxr_invertcc
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: remove_subsxr_after_csincxr_invertcc
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $x1
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+  ; CHECK:   [[DEF:%[0-9]+]]:gpr64 = IMPLICIT_DEF
+  ; CHECK:   [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr killed [[DEF]], [[COPY]], implicit-def $nzcv
+  ; CHECK:   [[CSINCXr:%[0-9]+]]:gpr64common = CSINCXr $xzr, $xzr, 1, implicit $nzcv
+  ; CHECK:   [[SUBSXri:%[0-9]+]]:gpr64 = SUBSXri killed [[CSINCXr]], 0, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 0, %bb.2, implicit $nzcv
+  ; CHECK:   B %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   B %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $x1
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    %1:gpr64common = COPY $x1
+    %2:gpr64 = IMPLICIT_DEF
+    %3:gpr64 = SUBSXrr killed %2:gpr64, %1:gpr64common, implicit-def $nzcv
+    %4:gpr64common = CSINCXr $xzr, $xzr, 1, implicit $nzcv
+    %5:gpr64 = SUBSXri killed %4:gpr64common, 0, 0, implicit-def $nzcv
+    Bcc 0, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    B %bb.2
+
+  bb.2:
+    RET_ReallyLR
+
+...
+---
+name:            cflags_alive_in_succs
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: cflags_alive_in_succs
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $nzcv
+  ; CHECK:   [[CSINCXr:%[0-9]+]]:gpr64common = CSINCXr $xzr, $xzr, 1, implicit $nzcv
+  ; CHECK:   [[ADDSXri:%[0-9]+]]:gpr64 = ADDSXri killed [[CSINCXr]], 0, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 0, %bb.2, implicit $nzcv
+  ; CHECK:   B %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $nzcv
+  ; CHECK:   B %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $nzcv
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    %1:gpr64common = CSINCXr $xzr, $xzr, 1, implicit $nzcv
+    %2:gpr64 = ADDSXri killed %1:gpr64common, 0, 0, implicit-def $nzcv
+    Bcc 0, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1:
+    liveins: $nzcv
+    successors: %bb.2(0x80000000)
+    B %bb.2
+
+  bb.2:
+    RET_ReallyLR
+
+...
+---
+name:            nz_used_after_cmp
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: nz_used_after_cmp
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $nzcv
+  ; CHECK:   [[CSINCXr:%[0-9]+]]:gpr64common = CSINCXr $xzr, $xzr, 1, implicit $nzcv
+  ; CHECK:   [[ADDSXri:%[0-9]+]]:gpr64 = ADDSXri killed [[CSINCXr]], 0, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 4, %bb.2, implicit $nzcv
+  ; CHECK:   B %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   B %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $nzcv
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    %1:gpr64common = CSINCXr $xzr, $xzr, 1, implicit $nzcv
+    %2:gpr64 = ADDSXri killed %1:gpr64common, 0, 0, implicit-def $nzcv
+    Bcc 4, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    B %bb.2
+
+  bb.2:
+    RET_ReallyLR
+
+...
+---
+name:            remove_addswr_after_csincwr_invertcc
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: remove_addswr_after_csincwr_invertcc
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $nzcv
+  ; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32common = CSINCWr $wzr, $wzr, 1, implicit $nzcv
+  ; CHECK:   [[ADDSWri:%[0-9]+]]:gpr32 = ADDSWri killed [[CSINCWr]], 0, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 1, %bb.2, implicit $nzcv
+  ; CHECK:   B %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   B %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $nzcv
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    %1:gpr32common = CSINCWr $wzr, $wzr, 1, implicit $nzcv
+    %2:gpr32 = ADDSWri killed %1:gpr32common, 0, 0, implicit-def $nzcv
+    Bcc 1, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    B %bb.2
+
+  bb.2:
+    RET_ReallyLR
+
+...
+---
+name:            cv_used_after_cmp
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: cv_used_after_cmp
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $nzcv
+  ; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32common = CSINCWr $wzr, $wzr, 1, implicit $nzcv
+  ; CHECK:   [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri killed [[CSINCWr]], 0, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 2, %bb.2, implicit $nzcv
+  ; CHECK:   B %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   B %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $nzcv
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    %1:gpr32common = CSINCWr $wzr, $wzr, 1, implicit $nzcv
+    %2:gpr32 = SUBSWri killed %1:gpr32common, 0, 0, implicit-def $nzcv
+    Bcc 2, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    B %bb.2
+
+  bb.2:
+    RET_ReallyLR
+
+...
+---
+name:            csinc_add0
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: csinc_add0
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $nzcv
+  ; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32common = CSINCWr $wzr, $wzr, 4, implicit $nzcv
+  ; CHECK:   [[ADDSWri:%[0-9]+]]:gpr32 = ADDSWri killed [[CSINCWr]], 0, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 1, %bb.2, implicit $nzcv
+  ; CHECK:   B %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   B %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $nzcv
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    %1:gpr32common = CSINCWr $wzr, $wzr, 4, implicit $nzcv
+    %2:gpr32 = ADDSWri killed %1:gpr32common, 0, 0, implicit-def $nzcv
+    Bcc 1, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    B %bb.2
+
+  bb.2:
+    RET_ReallyLR
+
+...
+---
+name:            remove_subs1_after_csincN_invertcc
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: remove_subs1_after_csincN_invertcc
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $nzcv
+  ; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32common = CSINCWr $wzr, $wzr, 5, implicit $nzcv
+  ; CHECK:   [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri killed [[CSINCWr]], 1, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 4, %bb.2, implicit $nzcv
+  ; CHECK:   B %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   B %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $nzcv
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    %1:gpr32common = CSINCWr $wzr, $wzr, 5, implicit $nzcv
+    %2:gpr32 = SUBSWri killed %1:gpr32common, 1, 0, implicit-def $nzcv
+    Bcc 4, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    B %bb.2
+
+  bb.2:
+    RET_ReallyLR
+
+...
+---
+name:            csinc_wrong_cc
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: csinc_wrong_cc
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $nzcv
+  ; CHECK:   [[CSINCXr:%[0-9]+]]:gpr64common = CSINCXr $xzr, $xzr, 2, implicit $nzcv
+  ; CHECK:   [[SUBSXri:%[0-9]+]]:gpr64 = SUBSXri killed [[CSINCXr]], 1, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 0, %bb.2, implicit $nzcv
+  ; CHECK:   B %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   B %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $nzcv
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    %1:gpr64common = CSINCXr $xzr, $xzr, 2, implicit $nzcv
+    %2:gpr64 = SUBSXri killed %1:gpr64common, 1, 0, implicit-def $nzcv
+    Bcc 0, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    B %bb.2
+
+  bb.2:
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll b/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..981818f6834833984c4ca8008c911aa002902a60
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -o - %s  -mattr=+neon,+fullfp16 | FileCheck %s
+
+define double @t1(double %x) {
+; CHECK-LABEL: t1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs d0, d0
+; CHECK-NEXT:    scvtf d0, d0
+; CHECK-NEXT:    ret
+entry:
+  %conv = fptosi double %x to i64
+  %conv1 = sitofp i64 %conv to double
+  ret double %conv1
+}
+
+define float @t2(float %x) {
+; CHECK-LABEL: t2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs s0, s0
+; CHECK-NEXT:    scvtf s0, s0
+; CHECK-NEXT:    ret
+entry:
+  %conv = fptosi float %x to i32
+  %conv1 = sitofp i32 %conv to float
+  ret float %conv1
+}
+
+define half @t3(half %x)  {
+; CHECK-LABEL: t3:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs h0, h0
+; CHECK-NEXT:    scvtf h0, h0
+; CHECK-NEXT:    ret
+entry:
+  %conv = fptosi half %x to i32
+  %conv1 = sitofp i32 %conv to half
+  ret half %conv1
+}
+
+define double @t4(double %x) {
+; CHECK-LABEL: t4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu d0, d0
+; CHECK-NEXT:    ucvtf d0, d0
+; CHECK-NEXT:    ret
+entry:
+  %conv = fptoui double %x to i64
+  %conv1 = uitofp i64 %conv to double
+  ret double %conv1
+}
+
+define float @t5(float %x) {
+; CHECK-LABEL: t5:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu s0, s0
+; CHECK-NEXT:    ucvtf s0, s0
+; CHECK-NEXT:    ret
+entry:
+  %conv = fptoui float %x to i32
+  %conv1 = uitofp i32 %conv to float
+  ret float %conv1
+}
+
+define half @t6(half %x)  {
+; CHECK-LABEL: t6:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu h0, h0
+; CHECK-NEXT:    ucvtf h0, h0
+; CHECK-NEXT:    ret
+entry:
+  %conv = fptoui half %x to i32
+  %conv1 = uitofp i32 %conv to half
+  ret half %conv1
+}
diff --git a/llvm/test/CodeGen/AArch64/f16-instructions.ll b/llvm/test/CodeGen/AArch64/f16-instructions.ll
index a324034f5b200895fbbab6973f965e9714c2590e..bb445f08d1edab93f9f0f13df4cd25c35c00ed64 100644
--- a/llvm/test/CodeGen/AArch64/f16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/f16-instructions.ll
@@ -796,6 +796,7 @@ declare half @llvm.trunc.f16(half %a) #0
 declare half @llvm.rint.f16(half %a) #0
 declare half @llvm.nearbyint.f16(half %a) #0
 declare half @llvm.round.f16(half %a) #0
+declare half @llvm.roundeven.f16(half %a) #0
 declare half @llvm.fmuladd.f16(half %a, half %b, half %c) #0
 declare half @llvm.aarch64.neon.frecpe.f16(half %a) #0
 declare half @llvm.aarch64.neon.frecpx.f16(half %a) #0
@@ -1313,6 +1314,32 @@ define half @test_round(half %a) #0 {
   ret half %r
 }
 
+; CHECK-CVT-LABEL: test_roundeven:
+; CHECK-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
+; CHECK-CVT-NEXT: frintn [[INT32:s[0-9]+]], [[FLOAT32]]
+; CHECK-CVT-NEXT: fcvt h0, [[INT32]]
+; CHECK-CVT-NEXT: ret
+
+; GISEL-CVT-LABEL: test_roundeven:
+; GISEL-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
+; GISEL-CVT-NEXT: frintn [[INT32:s[0-9]+]], [[FLOAT32]]
+; GISEL-CVT-NEXT: fcvt h0, [[INT32]]
+; GISEL-CVT-NEXT: ret
+
+
+; CHECK-FP16-LABEL: test_roundeven:
+; CHECK-FP16-NEXT: frintn h0, h0
+; CHECK-FP16-NEXT: ret
+
+; GISEL-FP16-LABEL: test_roundeven:
+; GISEL-FP16-NEXT: frintn h0, h0
+; GISEL-FP16-NEXT: ret
+
+define half @test_roundeven(half %a) #0 {
+  %r = call half @llvm.roundeven.f16(half %a)
+  ret half %r
+}
+
 ; CHECK-CVT-LABEL: test_fmuladd:
 ; CHECK-CVT-NEXT: fcvt s1, h1
 ; CHECK-CVT-NEXT: fcvt s0, h0
diff --git a/llvm/test/CodeGen/AArch64/fold-global-offsets.ll b/llvm/test/CodeGen/AArch64/fold-global-offsets.ll
index 40235791c524d7e277da33089684bc3126743e02..24168f9121758fe7a40ba0ff2f9a0a4dca5c1b88 100644
--- a/llvm/test/CodeGen/AArch64/fold-global-offsets.ll
+++ b/llvm/test/CodeGen/AArch64/fold-global-offsets.ll
@@ -1,69 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-linux-gnu | FileCheck %s
+; RUN: llc < %s -global-isel -mtriple=arm64-linux-gnu | FileCheck %s --check-prefix=GISEL
 
 @x1 = external hidden global [2 x i64]
 @x2 = external hidden global [16777216 x i64]
 @x3 = external hidden global { [9 x i8*], [8 x i8*] }
 
 define i64 @f1() {
-  ; CHECK: f1:
-  ; CHECK: adrp x8, x1+16
-  ; CHECK: ldr x0, [x8, :lo12:x1+16]
+; CHECK-LABEL: f1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x1+16
+; CHECK-NEXT:    ldr x0, [x8, :lo12:x1+16]
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: f1:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, x1+16
+; GISEL-NEXT:    ldr x0, [x8, :lo12:x1+16]
+; GISEL-NEXT:    ret
   %l = load i64, i64* getelementptr ([2 x i64], [2 x i64]* @x1, i64 0, i64 2)
   ret i64 %l
 }
 
 define i64 @f2() {
-  ; CHECK: f2:
-  ; CHECK: adrp x8, x1
-  ; CHECK: add x8, x8, :lo12:x1
-  ; CHECK: ldr x0, [x8, #24]
+; CHECK-LABEL: f2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x1
+; CHECK-NEXT:    add x8, x8, :lo12:x1
+; CHECK-NEXT:    ldr x0, [x8, #24]
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: f2:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, x1
+; GISEL-NEXT:    add x8, x8, :lo12:x1
+; GISEL-NEXT:    ldr x0, [x8, #24]
+; GISEL-NEXT:    ret
+
   %l = load i64, i64* getelementptr ([2 x i64], [2 x i64]* @x1, i64 0, i64 3)
   ret i64 %l
 }
 
 define i64 @f3() {
-  ; CHECK: f3:
-  ; CHECK: adrp x8, x1+1
-  ; CHECK: add x8, x8, :lo12:x1+1
-  ; CHECK: ldr x0, [x8]
+; CHECK-LABEL: f3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x1+1
+; CHECK-NEXT:    add x8, x8, :lo12:x1+1
+; CHECK-NEXT:    ldr x0, [x8]
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: f3:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, x1+1
+; GISEL-NEXT:    add x8, x8, :lo12:x1+1
+; GISEL-NEXT:    ldr x0, [x8]
+; GISEL-NEXT:    ret
   %l = load i64, i64* bitcast (i8* getelementptr (i8, i8* bitcast ([2 x i64]* @x1 to i8*), i64 1) to i64*)
   ret i64 %l
 }
 
 define [2 x i64] @f4() {
-  ; CHECK: f4:
-  ; CHECK: adrp x8, x2+8
-  ; CHECK: add x8, x8, :lo12:x2+8
-  ; CHECK: ldp x0, x1, [x8]
+; FIXME: GlobalISel misses the opportunity to form a LDP here.
+;
+; CHECK-LABEL: f4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x2+8
+; CHECK-NEXT:    add x8, x8, :lo12:x2+8
+; CHECK-NEXT:    ldp x0, x1, [x8]
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: f4:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x9, x2+8
+; GISEL-NEXT:    adrp x8, x2+8
+; GISEL-NEXT:    add x9, x9, :lo12:x2+8
+; GISEL-NEXT:    ldr x0, [x8, :lo12:x2+8]
+; GISEL-NEXT:    ldr x1, [x9, #8]
+; GISEL-NEXT:    ret
   %l = load [2 x i64], [2 x i64]* bitcast (i8* getelementptr (i8, i8* bitcast ([16777216 x i64]* @x2 to i8*), i64 8) to [2 x i64]*)
   ret [2 x i64] %l
 }
 
 define i64 @f5() {
-  ; CHECK: f5:
-  ; CHECK: adrp x8, x2+2097144
-  ; CHECK: ldr x0, [x8, :lo12:x2+2097144]
-  ; CHECK: ret
+; CHECK-LABEL: f5:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x2+2097144
+; CHECK-NEXT:    ldr x0, [x8, :lo12:x2+2097144]
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: f5:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, x2+2097144
+; GISEL-NEXT:    ldr x0, [x8, :lo12:x2+2097144]
+; GISEL-NEXT:    ret
   %l = load i64, i64* getelementptr ([16777216 x i64], [16777216 x i64]* @x2, i64 0, i64 262143)
   ret i64 %l
 }
 
 define i64 @f6() {
-  ; CHECK: f6:
-  ; CHECK: adrp x8, x2
-  ; CHECK: add x8, x8, :lo12:x2
-  ; CHECK: mov w9, #2097152
-  ; CHECK: ldr x0, [x8, x9]
-  ; CHECK: ret
+; CHECK-LABEL: f6:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, x2
+; CHECK-NEXT:    add x8, x8, :lo12:x2
+; CHECK-NEXT:    mov w9, #2097152
+; CHECK-NEXT:    ldr x0, [x8, x9]
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: f6:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x9, x2
+; GISEL-NEXT:    mov w8, #2097152
+; GISEL-NEXT:    add x9, x9, :lo12:x2
+; GISEL-NEXT:    ldr x0, [x9, x8]
+; GISEL-NEXT:    ret
   %l = load i64, i64* getelementptr ([16777216 x i64], [16777216 x i64]* @x2, i64 0, i64 262144)
   ret i64 %l
 }
 
 define i32 @f7() {
+; FIXME: GlobalISel doesn't handle vectors well.
+;
+; CHECK-LABEL: f7:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, x3+108
+; CHECK-NEXT:    ldr w0, [x8, :lo12:x3+108]
+; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: f7:
+; GISEL:       // %bb.0: // %entry
+; GISEL-NEXT:    adrp x8, x3+88
+; GISEL-NEXT:    add x8, x8, :lo12:x3+88
+; GISEL-NEXT:    mov v0.d[1], x8
+; GISEL-NEXT:    mov w9, #64
+; GISEL-NEXT:    mov d1, v0.d[1]
+; GISEL-NEXT:    sub x8, x9, #64 // =64
+; GISEL-NEXT:    fmov x11, d1
+; GISEL-NEXT:    fmov x10, d0
+; GISEL-NEXT:    lsl x12, x11, x8
+; GISEL-NEXT:    cmp x9, #64 // =64
+; GISEL-NEXT:    lsr x8, x11, x8
+; GISEL-NEXT:    orr x11, x12, x10, lsr #0
+; GISEL-NEXT:    csel x8, x11, x8, lo
+; GISEL-NEXT:    cmp x9, #0 // =0
+; GISEL-NEXT:    csel x8, x10, x8, eq
+; GISEL-NEXT:    ldr w0, [x8, #20]
+; GISEL-NEXT:    ret
+
 entry:
-  ; CHECK: f7
-  ; CHECK: adrp x8, x3+108
-  ; CHECK: ldr w0, [x8, :lo12:x3+108]
   %l = load i32, i32* getelementptr (i32, i32* inttoptr (i64 trunc (i128 lshr (i128 bitcast (<2 x i64> <i64 undef, i64 ptrtoint (i8** getelementptr inbounds ({ [9 x i8*], [8 x i8*] }, { [9 x i8*], [8 x i8*] }* @x3, i64 0, inrange i32 1, i64 2) to i64)> to i128), i128 64) to i64) to i32*), i64 5)
   ret i32 %l
 }
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
index 3c412a5f7e0e30d23f336f20708589ad8a067931..f2694ab08a0db66916ab4293437be7e5a9942f90 100644
--- a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
@@ -266,6 +266,13 @@ define float @round_f32(float %x) #0 {
   ret float %val
 }
 
+; CHECK-LABEL: roundeven_f32:
+; CHECK: frintn s0, s0
+define float @roundeven_f32(float %x) #0 {
+  %val = call float @llvm.experimental.constrained.roundeven.f32(float %x, metadata !"fpexcept.strict") #0
+  ret float %val
+}
+
 ; CHECK-LABEL: trunc_f32:
 ; CHECK: frintz s0, s0
 define float @trunc_f32(float %x) #0 {
@@ -729,6 +736,13 @@ define double @round_f64(double %x) #0 {
   ret double %val
 }
 
+; CHECK-LABEL: roundeven_f64:
+; CHECK: frintn d0, d0
+define double @roundeven_f64(double %x) #0 {
+  %val = call double @llvm.experimental.constrained.roundeven.f64(double %x, metadata !"fpexcept.strict") #0
+  ret double %val
+}
+
 ; CHECK-LABEL: trunc_f64:
 ; CHECK: frintz d0, d0
 define double @trunc_f64(double %x) #0 {
@@ -1474,6 +1488,7 @@ declare float @llvm.experimental.constrained.floor.f32(float, metadata)
 declare i32 @llvm.experimental.constrained.lround.f32(float, metadata)
 declare i64 @llvm.experimental.constrained.llround.f32(float, metadata)
 declare float @llvm.experimental.constrained.round.f32(float, metadata)
+declare float @llvm.experimental.constrained.roundeven.f32(float, metadata)
 declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
 declare i1 @llvm.experimental.constrained.fcmps.f32(float, float, metadata, metadata)
 declare i1 @llvm.experimental.constrained.fcmp.f32(float, float, metadata, metadata)
@@ -1515,6 +1530,7 @@ declare double @llvm.experimental.constrained.floor.f64(double, metadata)
 declare i32 @llvm.experimental.constrained.lround.f64(double, metadata)
 declare i64 @llvm.experimental.constrained.llround.f64(double, metadata)
 declare double @llvm.experimental.constrained.round.f64(double, metadata)
+declare double @llvm.experimental.constrained.roundeven.f64(double, metadata)
 declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
 declare i1 @llvm.experimental.constrained.fcmps.f64(double, double, metadata, metadata)
 declare i1 @llvm.experimental.constrained.fcmp.f64(double, double, metadata, metadata)
diff --git a/llvm/test/CodeGen/AArch64/frintn.ll b/llvm/test/CodeGen/AArch64/frintn.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2dc03db39a1a891e7875a5b777fa0b8bfa68b14c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/frintn.ll
@@ -0,0 +1,41 @@
+; RUN: llc -mtriple=aarch64-eabi -mattr=+fullfp16 %s -o - | FileCheck %s
+
+; The llvm.aarch64.neon.frintn intrinsic should be auto-upgraded to the
+; target-independent roundeven intrinsic.
+
+define <4 x half> @frintn_4h(<4 x half> %A) nounwind {
+;CHECK-LABEL: frintn_4h:
+;CHECK: frintn v0.4h, v0.4h
+;CHECK-NEXT: ret
+	%tmp3 = call <4 x half> @llvm.aarch64.neon.frintn.v4f16(<4 x half> %A)
+	ret <4 x half> %tmp3
+}
+
+define <2 x float> @frintn_2s(<2 x float> %A) nounwind {
+;CHECK-LABEL: frintn_2s:
+;CHECK: frintn v0.2s, v0.2s
+;CHECK-NEXT: ret
+	%tmp3 = call <2 x float> @llvm.aarch64.neon.frintn.v2f32(<2 x float> %A)
+	ret <2 x float> %tmp3
+}
+
+define <4 x float> @frintn_4s(<4 x float> %A) nounwind {
+;CHECK-LABEL: frintn_4s:
+;CHECK: frintn v0.4s, v0.4s
+;CHECK-NEXT: ret
+	%tmp3 = call <4 x float> @llvm.aarch64.neon.frintn.v4f32(<4 x float> %A)
+	ret <4 x float> %tmp3
+}
+
+define <2 x double> @frintn_2d(<2 x double> %A) nounwind {
+;CHECK-LABEL: frintn_2d:
+;CHECK: frintn v0.2d, v0.2d
+;CHECK-NEXT: ret
+	%tmp3 = call <2 x double> @llvm.aarch64.neon.frintn.v2f64(<2 x double> %A)
+	ret <2 x double> %tmp3
+}
+
+declare <4 x half> @llvm.aarch64.neon.frintn.v4f16(<4 x half>) nounwind readnone
+declare <2 x float> @llvm.aarch64.neon.frintn.v2f32(<2 x float>) nounwind readnone
+declare <4 x float> @llvm.aarch64.neon.frintn.v4f32(<4 x float>) nounwind readnone
+declare <2 x double> @llvm.aarch64.neon.frintn.v2f64(<2 x double>) nounwind readnone
diff --git a/llvm/test/CodeGen/AArch64/half.ll b/llvm/test/CodeGen/AArch64/half.ll
index b815c53d02bcf91e9b7243b1365b049c147004ab..ab64cc04374ff8d2725970687f64e2c2c073e0ed 100644
--- a/llvm/test/CodeGen/AArch64/half.ll
+++ b/llvm/test/CodeGen/AArch64/half.ll
@@ -107,12 +107,12 @@ define i16 @test_fccmp(i1 %a, i16 %in) {
 ; CHECK-NEXT:    movk w9, #15428, lsl #16
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    cset w8, pl
-; CHECK-NEXT:    fccmp s0, s1, #8, pl
-; CHECK-NEXT:    mov w9, #4
-; CHECK-NEXT:    csinc w9, w9, wzr, mi
-; CHECK-NEXT:    add w0, w8, w9
+; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    mov w10, #4
+; CHECK-NEXT:    fccmp s0, s2, #8, pl
+; CHECK-NEXT:    csinc w8, w10, wzr, mi
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cinc w0, w8, pl
 ; CHECK-NEXT:    ret
   %f16 = bitcast i16 %in to half
   %cmp0 = fcmp ogt half 0xH3333, %f16
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
index aea36d969108c3818a9cd4cf1d2ea83b2a05f3ab..3c4eff39c60b96bc27677d6e67f56ec71db16138 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
@@ -67,7 +67,7 @@ entry:
 
 attributes #0 = { "sign-return-address"="non-leaf" }
 
-; CHECK-NOT:        [[OUTLINED_FUNCTION_{{.*}}]]
+; CHECK-NOT:        OUTLINED_FUNCTION_{{.*}}
 ; CHECK-NOT:         .cfi_b_key_frame
 ; CHECK-NOT:         paci{{[a,b]}}sp
 ; CHECK-NOT:         hint #2{{[5,7]}}
diff --git a/llvm/test/CodeGen/AArch64/neon-stepvector.ll b/llvm/test/CodeGen/AArch64/neon-stepvector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..05308bf5f6d76597c445efde3c5510ad1ffd37c3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-stepvector.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+neon < %s | FileCheck %s --check-prefixes=CHECK
+
+; LEGAL INTEGER TYPES
+
+define <2 x i64> @stepvector_v2i64() {
+; CHECK-LABEL: .LCPI0_0:
+; CHECK-NEXT:    .xword 0
+; CHECK-NEXT:    .xword 1
+; CHECK-LABEL: stepvector_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <2 x i64> @llvm.experimental.stepvector.v2i64()
+  ret <2 x i64> %0
+}
+
+define <4 x i32> @stepvector_v4i32() {
+; CHECK-LABEL: .LCPI1_0:
+; CHECK-NEXT:    .word 0
+; CHECK-NEXT:    .word 1
+; CHECK-NEXT:    .word 2
+; CHECK-NEXT:    .word 3
+; CHECK-LABEL: stepvector_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI1_0
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <4 x i32> @llvm.experimental.stepvector.v4i32()
+  ret <4 x i32> %0
+}
+
+define <8 x i16> @stepvector_v8i16() {
+; CHECK-LABEL: .LCPI2_0:
+; CHECK-NEXT:    .hword 0
+; CHECK-NEXT:    .hword 1
+; CHECK-NEXT:    .hword 2
+; CHECK-NEXT:    .hword 3
+; CHECK-NEXT:    .hword 4
+; CHECK-NEXT:    .hword 5
+; CHECK-NEXT:    .hword 6
+; CHECK-NEXT:    .hword 7
+; CHECK-LABEL: stepvector_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI2_0
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <8 x i16> @llvm.experimental.stepvector.v8i16()
+  ret <8 x i16> %0
+}
+
+define <16 x i8> @stepvector_v16i8() {
+; CHECK-LABEL: .LCPI3_0:
+; CHECK-NEXT:    .byte 0
+; CHECK-NEXT:    .byte 1
+; CHECK-NEXT:    .byte 2
+; CHECK-NEXT:    .byte 3
+; CHECK-NEXT:    .byte 4
+; CHECK-NEXT:    .byte 5
+; CHECK-NEXT:    .byte 6
+; CHECK-NEXT:    .byte 7
+; CHECK-NEXT:    .byte 8
+; CHECK-NEXT:    .byte 9
+; CHECK-NEXT:    .byte 10
+; CHECK-NEXT:    .byte 11
+; CHECK-NEXT:    .byte 12
+; CHECK-NEXT:    .byte 13
+; CHECK-NEXT:    .byte 14
+; CHECK-NEXT:    .byte 15
+; CHECK-LABEL: stepvector_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI3_0
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <16 x i8> @llvm.experimental.stepvector.v16i8()
+  ret <16 x i8> %0
+}
+
+; ILLEGAL INTEGER TYPES
+
+define <4 x i64> @stepvector_v4i64() {
+; CHECK-LABEL: .LCPI4_0:
+; CHECK-NEXT:    .xword 0
+; CHECK-NEXT:    .xword 1
+; CHECK-LABEL: .LCPI4_1:
+; CHECK-NEXT:    .xword 2
+; CHECK-NEXT:    .xword 3
+; CHECK-LABEL: stepvector_v4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI4_0
+; CHECK-NEXT:    adrp x9, .LCPI4_1
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI4_0]
+; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI4_1]
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <4 x i64> @llvm.experimental.stepvector.v4i64()
+  ret <4 x i64> %0
+}
+
+define <16 x i32> @stepvector_v16i32() {
+; CHECK-LABEL: .LCPI5_0:
+; CHECK-NEXT:    .word 0
+; CHECK-NEXT:    .word 1
+; CHECK-NEXT:    .word 2
+; CHECK-NEXT:    .word 3
+; CHECK-LABEL: .LCPI5_1:
+; CHECK-NEXT:    .word 4
+; CHECK-NEXT:    .word 5
+; CHECK-NEXT:    .word 6
+; CHECK-NEXT:    .word 7
+; CHECK-LABEL: .LCPI5_2:
+; CHECK-NEXT:    .word 8
+; CHECK-NEXT:    .word 9
+; CHECK-NEXT:    .word 10
+; CHECK-NEXT:    .word 11
+; CHECK-LABEL: .LCPI5_3:
+; CHECK-NEXT:    .word 12
+; CHECK-NEXT:    .word 13
+; CHECK-NEXT:    .word 14
+; CHECK-NEXT:    .word 15
+; CHECK-LABEL: stepvector_v16i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI5_0
+; CHECK-NEXT:    adrp x9, .LCPI5_1
+; CHECK-NEXT:    adrp x10, .LCPI5_2
+; CHECK-NEXT:    adrp x11, .LCPI5_3
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI5_0]
+; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI5_1]
+; CHECK-NEXT:    ldr q2, [x10, :lo12:.LCPI5_2]
+; CHECK-NEXT:    ldr q3, [x11, :lo12:.LCPI5_3]
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <16 x i32> @llvm.experimental.stepvector.v16i32()
+  ret <16 x i32> %0
+}
+
+define <2 x i32> @stepvector_v2i32() {
+; CHECK-LABEL: .LCPI6_0:
+; CHECK-NEXT:    .word 0
+; CHECK-NEXT:    .word 1
+; CHECK-LABEL: stepvector_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI6_0
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI6_0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <2 x i32> @llvm.experimental.stepvector.v2i32()
+  ret <2 x i32> %0
+}
+
+define <4 x i16> @stepvector_v4i16() {
+; CHECK-LABEL: .LCPI7_0:
+; CHECK-NEXT:    .hword 0
+; CHECK-NEXT:    .hword 1
+; CHECK-NEXT:    .hword 2
+; CHECK-NEXT:    .hword 3
+; CHECK-LABEL: stepvector_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI7_0
+; CHECK-NEXT:    ldr d0, [x8, :lo12:.LCPI7_0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <4 x i16> @llvm.experimental.stepvector.v4i16()
+  ret <4 x i16> %0
+}
+
+
+declare <2 x i64> @llvm.experimental.stepvector.v2i64()
+declare <4 x i32> @llvm.experimental.stepvector.v4i32()
+declare <8 x i16> @llvm.experimental.stepvector.v8i16()
+declare <16 x i8> @llvm.experimental.stepvector.v16i8()
+
+declare <4 x i64> @llvm.experimental.stepvector.v4i64()
+declare <16 x i32> @llvm.experimental.stepvector.v16i32()
+declare <2 x i32> @llvm.experimental.stepvector.v2i32()
+declare <4 x i16> @llvm.experimental.stepvector.v4i16()
diff --git a/llvm/test/CodeGen/AArch64/sve-cmp-select.ll b/llvm/test/CodeGen/AArch64/sve-cmp-select.ll
new file mode 100644
index 0000000000000000000000000000000000000000..baaed7ad56559d9545caa20a0e80b702f9e414a6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-cmp-select.ll
@@ -0,0 +1,41 @@
+; RUN: llc -mtriple=aarch64-linux-unknown -mattr=+sve -o - < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix="WARN" --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+define <vscale x 16 x i8> @vselect_cmp_ne(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+  ; CHECK-LABEL: vselect_cmp_ne
+  ; CHECK:       // %bb.0:
+	; CHECK-NEXT:    ptrue	p0.b
+	; CHECK-NEXT:    cmpne	p0.b, p0/z, z0.b, z1.b
+	; CHECK-NEXT:    sel	z0.b, p0, z1.b, z2.b
+	; CHECK-NEXT:    ret
+  %cmp = icmp ne <vscale x 16 x i8> %a, %b
+  %d = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c
+  ret <vscale x 16 x i8> %d
+}
+
+define <vscale x 16 x i8> @vselect_cmp_sgt(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+  ; CHECK-LABEL: vselect_cmp_sgt
+  ; CHECK:       // %bb.0:
+  ; CHECK-NEXT: 	ptrue	p0.b
+  ; CHECK-NEXT: 	cmpgt	p0.b, p0/z, z0.b, z1.b
+  ; CHECK-NEXT: 	sel	z0.b, p0, z1.b, z2.b
+  ; CHECK-NEXT: 	ret
+  %cmp = icmp sgt <vscale x 16 x i8> %a, %b
+  %d = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c
+  ret <vscale x 16 x i8> %d
+}
+
+define <vscale x 16 x i8> @vselect_cmp_ugt(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
+  ; CHECK-LABEL: vselect_cmp_ugt
+  ; CHECK:       // %bb.0:
+  ; CHECK-NEXT: 	ptrue	p0.b
+  ; CHECK-NEXT: 	cmphi	p0.b, p0/z, z0.b, z1.b
+  ; CHECK-NEXT: 	sel	z0.b, p0, z1.b, z2.b
+  ; CHECK-NEXT: 	ret
+  %cmp = icmp ugt <vscale x 16 x i8> %a, %b
+  %d = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c
+  ret <vscale x 16 x i8> %d
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
index fdd0acd9702438140dd3cd8e3a5bd6240fcd3053..667513b77e43f64ad98243b46f0c8acc18dcc3a1 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
@@ -1710,6 +1710,220 @@ define void @fsub_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
   ret void
 }
 
+;
+; FABS
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x half> @fabs_v4f16(<4 x half> %op) #0 {
+; CHECK-LABEL: fabs_v4f16:
+; CHECK: fabs v0.4h, v0.4h
+; CHECK: ret
+  %res = call <4 x half> @llvm.fabs.v4f16(<4 x half> %op)
+  ret <4 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x half> @fabs_v8f16(<8 x half> %op) #0 {
+; CHECK-LABEL: fabs_v8f16:
+; CHECK: fabs v0.8h, v0.8h
+; CHECK: ret
+  %res = call <8 x half> @llvm.fabs.v8f16(<8 x half> %op)
+  ret <8 x half> %res
+}
+
+define void @fabs_v16f16(<16 x half>* %a) #0 {
+; CHECK-LABEL: fabs_v16f16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
+; CHECK: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <16 x half>, <16 x half>* %a
+  %res = call <16 x half> @llvm.fabs.v16f16(<16 x half> %op)
+  store <16 x half> %res, <16 x half>* %a
+  ret void
+}
+
+define void @fabs_v32f16(<32 x half>* %a) #0 {
+; CHECK-LABEL: fabs_v32f16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
+; CHECK: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <32 x half>, <32 x half>* %a
+  %res = call <32 x half> @llvm.fabs.v32f16(<32 x half> %op)
+  store <32 x half> %res, <32 x half>* %a
+  ret void
+}
+
+define void @fabs_v64f16(<64 x half>* %a) #0 {
+; CHECK-LABEL: fabs_v64f16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
+; CHECK: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <64 x half>, <64 x half>* %a
+  %res = call <64 x half> @llvm.fabs.v64f16(<64 x half> %op)
+  store <64 x half> %res, <64 x half>* %a
+  ret void
+}
+
+define void @fabs_v128f16(<128 x half>* %a) #0 {
+; CHECK-LABEL: fabs_v128f16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
+; CHECK: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <128 x half>, <128 x half>* %a
+  %res = call <128 x half> @llvm.fabs.v128f16(<128 x half> %op)
+  store <128 x half> %res, <128 x half>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @fabs_v2f32(<2 x float> %op) #0 {
+; CHECK-LABEL: fabs_v2f32:
+; CHECK: fabs v0.2s, v0.2s
+; CHECK: ret
+  %res = call <2 x float> @llvm.fabs.v2f32(<2 x float> %op)
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @fabs_v4f32(<4 x float> %op) #0 {
+; CHECK-LABEL: fabs_v4f32:
+; CHECK: fabs v0.4s, v0.4s
+; CHECK: ret
+  %res = call <4 x float> @llvm.fabs.v4f32(<4 x float> %op)
+  ret <4 x float> %res
+}
+
+define void @fabs_v8f32(<8 x float>* %a) #0 {
+; CHECK-LABEL: fabs_v8f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
+; CHECK: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <8 x float>, <8 x float>* %a
+  %res = call <8 x float> @llvm.fabs.v8f32(<8 x float> %op)
+  store <8 x float> %res, <8 x float>* %a
+  ret void
+}
+
+define void @fabs_v16f32(<16 x float>* %a) #0 {
+; CHECK-LABEL: fabs_v16f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
+; CHECK: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <16 x float>, <16 x float>* %a
+  %res = call <16 x float> @llvm.fabs.v16f32(<16 x float> %op)
+  store <16 x float> %res, <16 x float>* %a
+  ret void
+}
+
+define void @fabs_v32f32(<32 x float>* %a) #0 {
+; CHECK-LABEL: fabs_v32f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
+; CHECK: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <32 x float>, <32 x float>* %a
+  %res = call <32 x float> @llvm.fabs.v32f32(<32 x float> %op)
+  store <32 x float> %res, <32 x float>* %a
+  ret void
+}
+
+define void @fabs_v64f32(<64 x float>* %a) #0 {
+; CHECK-LABEL: fabs_v64f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
+; CHECK: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <64 x float>, <64 x float>* %a
+  %res = call <64 x float> @llvm.fabs.v64f32(<64 x float> %op)
+  store <64 x float> %res, <64 x float>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @fabs_v1f64(<1 x double> %op) #0 {
+; CHECK-LABEL: fabs_v1f64:
+; CHECK: fabs d0, d0
+; CHECK: ret
+  %res = call <1 x double> @llvm.fabs.v1f64(<1 x double> %op)
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @fabs_v2f64(<2 x double> %op) #0 {
+; CHECK-LABEL: fabs_v2f64:
+; CHECK: fabs v0.2d, v0.2d
+; CHECK: ret
+  %res = call <2 x double> @llvm.fabs.v2f64(<2 x double> %op)
+  ret <2 x double> %res
+}
+
+define void @fabs_v4f64(<4 x double>* %a) #0 {
+; CHECK-LABEL: fabs_v4f64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
+; CHECK: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <4 x double>, <4 x double>* %a
+  %res = call <4 x double> @llvm.fabs.v4f64(<4 x double> %op)
+  store <4 x double> %res, <4 x double>* %a
+  ret void
+}
+
+define void @fabs_v8f64(<8 x double>* %a) #0 {
+; CHECK-LABEL: fabs_v8f64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
+; CHECK: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <8 x double>, <8 x double>* %a
+  %res = call <8 x double> @llvm.fabs.v8f64(<8 x double> %op)
+  store <8 x double> %res, <8 x double>* %a
+  ret void
+}
+
+define void @fabs_v16f64(<16 x double>* %a) #0 {
+; CHECK-LABEL: fabs_v16f64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
+; CHECK: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <16 x double>, <16 x double>* %a
+  %res = call <16 x double> @llvm.fabs.v16f64(<16 x double> %op)
+  store <16 x double> %res, <16 x double>* %a
+  ret void
+}
+
+define void @fabs_v32f64(<32 x double>* %a) #0 {
+; CHECK-LABEL: fabs_v32f64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
+; CHECK: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK: fabs [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op = load <32 x double>, <32 x double>* %a
+  %res = call <32 x double> @llvm.fabs.v32f64(<32 x double> %op)
+  store <32 x double> %res, <32 x double>* %a
+  ret void
+}
+
 attributes #0 = { "target-features"="+sve" }
 
 declare <4 x half> @llvm.fma.v4f16(<4 x half>, <4 x half>, <4 x half>)
@@ -1749,3 +1963,22 @@ declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
 declare <8 x double> @llvm.sqrt.v8f64(<8 x double>)
 declare <16 x double> @llvm.sqrt.v16f64(<16 x double>)
 declare <32 x double> @llvm.sqrt.v32f64(<32 x double>)
+
+declare <4 x half> @llvm.fabs.v4f16(<4 x half>)
+declare <8 x half> @llvm.fabs.v8f16(<8 x half>)
+declare <16 x half> @llvm.fabs.v16f16(<16 x half>)
+declare <32 x half> @llvm.fabs.v32f16(<32 x half>)
+declare <64 x half> @llvm.fabs.v64f16(<64 x half>)
+declare <128 x half> @llvm.fabs.v128f16(<128 x half>)
+declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
+declare <4 x float> @llvm.fabs.v4f32(<4 x float>)
+declare <8 x float> @llvm.fabs.v8f32(<8 x float>)
+declare <16 x float> @llvm.fabs.v16f32(<16 x float>)
+declare <32 x float> @llvm.fabs.v32f32(<32 x float>)
+declare <64 x float> @llvm.fabs.v64f32(<64 x float>)
+declare <1 x double> @llvm.fabs.v1f64(<1 x double>)
+declare <2 x double> @llvm.fabs.v2f64(<2 x double>)
+declare <4 x double> @llvm.fabs.v4f64(<4 x double>)
+declare <8 x double> @llvm.fabs.v8f64(<8 x double>)
+declare <16 x double> @llvm.fabs.v16f64(<16 x double>)
+declare <32 x double> @llvm.fabs.v32f64(<32 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-rounding.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-rounding.ll
index a3a677a5d9f82ca419f740c6c6e8613b8a86d69c..1d7472707b0a4bd7516d08dd2f8fa013491ad900 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-rounding.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-rounding.ll
@@ -1255,6 +1255,253 @@ define void @frinta_v32f64(<32 x double>* %a) #0 {
   ret void
 }
 
+;
+; ROUNDEVEN -> FRINTN
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x half> @frintn_v4f16(<4 x half> %op) #0 {
+; CHECK-LABEL: frintn_v4f16:
+; CHECK: frintn v0.4h, v0.4h
+; CHECK-NEXT: ret
+  %res = call <4 x half> @llvm.roundeven.v4f16(<4 x half> %op)
+  ret <4 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x half> @frintn_v8f16(<8 x half> %op) #0 {
+; CHECK-LABEL: frintn_v8f16:
+; CHECK: frintn v0.8h, v0.8h
+; CHECK-NEXT: ret
+  %res = call <8 x half> @llvm.roundeven.v8f16(<8 x half> %op)
+  ret <8 x half> %res
+}
+
+define void @frintn_v16f16(<16 x half>* %a) #0 {
+; CHECK-LABEL: frintn_v16f16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK-DAG: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-NEXT: frintn [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; CHECK-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <16 x half>, <16 x half>* %a
+  %res = call <16 x half> @llvm.roundeven.v16f16(<16 x half> %op)
+  store <16 x half> %res, <16 x half>* %a
+  ret void
+}
+
+define void @frintn_v32f16(<32 x half>* %a) #0 {
+; CHECK-LABEL: frintn_v32f16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512-DAG: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: frintn [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_512-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].h, vl16
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1h { [[OP_LO:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1h { [[OP_HI:z[0-9]+]].h }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: frintn [[RES_LO:z[0-9]+]].h, [[PG]]/m, [[OP_LO]].h
+; VBITS_EQ_256-DAG: frintn [[RES_HI:z[0-9]+]].h, [[PG]]/m, [[OP_HI]].h
+; VBITS_EQ_256-DAG: st1h { [[RES_LO]].h }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1h { [[RES_HI]].h }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <32 x half>, <32 x half>* %a
+  %res = call <32 x half> @llvm.roundeven.v32f16(<32 x half> %op)
+  store <32 x half> %res, <32 x half>* %a
+  ret void
+}
+
+define void @frintn_v64f16(<64 x half>* %a) #0 {
+; CHECK-LABEL: frintn_v64f16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024-DAG: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: frintn [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_1024-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <64 x half>, <64 x half>* %a
+  %res = call <64 x half> @llvm.roundeven.v64f16(<64 x half> %op)
+  store <64 x half> %res, <64 x half>* %a
+  ret void
+}
+
+define void @frintn_v128f16(<128 x half>* %a) #0 {
+; CHECK-LABEL: frintn_v128f16:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].h, vl128
+; VBITS_GE_2048-DAG: ld1h { [[OP:z[0-9]+]].h }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: frintn [[RES:z[0-9]+]].h, [[PG]]/m, [[OP]].h
+; VBITS_GE_2048-NEXT: st1h { [[RES]].h }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <128 x half>, <128 x half>* %a
+  %res = call <128 x half> @llvm.roundeven.v128f16(<128 x half> %op)
+  store <128 x half> %res, <128 x half>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @frintn_v2f32(<2 x float> %op) #0 {
+; CHECK-LABEL: frintn_v2f32:
+; CHECK: frintn v0.2s, v0.2s
+; CHECK-NEXT: ret
+  %res = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %op)
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @frintn_v4f32(<4 x float> %op) #0 {
+; CHECK-LABEL: frintn_v4f32:
+; CHECK: frintn v0.4s, v0.4s
+; CHECK-NEXT: ret
+  %res = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %op)
+  ret <4 x float> %res
+}
+
+define void @frintn_v8f32(<8 x float>* %a) #0 {
+; CHECK-LABEL: frintn_v8f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK-DAG: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-NEXT: frintn [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; CHECK-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <8 x float>, <8 x float>* %a
+  %res = call <8 x float> @llvm.roundeven.v8f32(<8 x float> %op)
+  store <8 x float> %res, <8 x float>* %a
+  ret void
+}
+
+define void @frintn_v16f32(<16 x float>* %a) #0 {
+; CHECK-LABEL: frintn_v16f32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512-DAG: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: frintn [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_512-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].s, vl8
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1w { [[OP_LO:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1w { [[OP_HI:z[0-9]+]].s }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: frintn [[RES_LO:z[0-9]+]].s, [[PG]]/m, [[OP_LO]].s
+; VBITS_EQ_256-DAG: frintn [[RES_HI:z[0-9]+]].s, [[PG]]/m, [[OP_HI]].s
+; VBITS_EQ_256-DAG: st1w { [[RES_LO]].s }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1w { [[RES_HI]].s }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <16 x float>, <16 x float>* %a
+  %res = call <16 x float> @llvm.roundeven.v16f32(<16 x float> %op)
+  store <16 x float> %res, <16 x float>* %a
+  ret void
+}
+
+define void @frintn_v32f32(<32 x float>* %a) #0 {
+; CHECK-LABEL: frintn_v32f32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024-DAG: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: frintn [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_1024-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <32 x float>, <32 x float>* %a
+  %res = call <32 x float> @llvm.roundeven.v32f32(<32 x float> %op)
+  store <32 x float> %res, <32 x float>* %a
+  ret void
+}
+
+define void @frintn_v64f32(<64 x float>* %a) #0 {
+; CHECK-LABEL: frintn_v64f32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048-DAG: ld1w { [[OP:z[0-9]+]].s }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: frintn [[RES:z[0-9]+]].s, [[PG]]/m, [[OP]].s
+; VBITS_GE_2048-NEXT: st1w { [[RES]].s }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <64 x float>, <64 x float>* %a
+  %res = call <64 x float> @llvm.roundeven.v64f32(<64 x float> %op)
+  store <64 x float> %res, <64 x float>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @frintn_v1f64(<1 x double> %op) #0 {
+; CHECK-LABEL: frintn_v1f64:
+; CHECK: frintn d0, d0
+; CHECK-NEXT: ret
+  %res = call <1 x double> @llvm.roundeven.v1f64(<1 x double> %op)
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @frintn_v2f64(<2 x double> %op) #0 {
+; CHECK-LABEL: frintn_v2f64:
+; CHECK: frintn v0.2d, v0.2d
+; CHECK-NEXT: ret
+  %res = call <2 x double> @llvm.roundeven.v2f64(<2 x double> %op)
+  ret <2 x double> %res
+}
+
+define void @frintn_v4f64(<4 x double>* %a) #0 {
+; CHECK-LABEL: frintn_v4f64:
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl4
+; CHECK-DAG: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-NEXT: frintn [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; CHECK-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK-NEXT: ret
+  %op = load <4 x double>, <4 x double>* %a
+  %res = call <4 x double> @llvm.roundeven.v4f64(<4 x double> %op)
+  store <4 x double> %res, <4 x double>* %a
+  ret void
+}
+
+define void @frintn_v8f64(<8 x double>* %a) #0 {
+; CHECK-LABEL: frintn_v8f64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512-DAG: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_512-NEXT: frintn [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_512-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_512-NEXT: ret
+
+; Ensure sensible type legalisation.
+; VBITS_EQ_256-DAG: ptrue [[PG:p[0-9]+]].d, vl4
+; VBITS_EQ_256-DAG: add x[[A_HI:[0-9]+]], x0, #32
+; VBITS_EQ_256-DAG: ld1d { [[OP_LO:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_EQ_256-DAG: ld1d { [[OP_HI:z[0-9]+]].d }, [[PG]]/z, [x[[A_HI]]]
+; VBITS_EQ_256-DAG: frintn [[RES_LO:z[0-9]+]].d, [[PG]]/m, [[OP_LO]].d
+; VBITS_EQ_256-DAG: frintn [[RES_HI:z[0-9]+]].d, [[PG]]/m, [[OP_HI]].d
+; VBITS_EQ_256-DAG: st1d { [[RES_LO]].d }, [[PG]], [x0]
+; VBITS_EQ_256-DAG: st1d { [[RES_HI]].d }, [[PG]], [x[[A_HI]]
+; VBITS_EQ_256-NEXT: ret
+  %op = load <8 x double>, <8 x double>* %a
+  %res = call <8 x double> @llvm.roundeven.v8f64(<8 x double> %op)
+  store <8 x double> %res, <8 x double>* %a
+  ret void
+}
+
+define void @frintn_v16f64(<16 x double>* %a) #0 {
+; CHECK-LABEL: frintn_v16f64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024-DAG: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_1024-NEXT: frintn [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_1024-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_1024-NEXT: ret
+  %op = load <16 x double>, <16 x double>* %a
+  %res = call <16 x double> @llvm.roundeven.v16f64(<16 x double> %op)
+  store <16 x double> %res, <16 x double>* %a
+  ret void
+}
+
+define void @frintn_v32f64(<32 x double>* %a) #0 {
+; CHECK-LABEL: frintn_v32f64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048-DAG: ld1d { [[OP:z[0-9]+]].d }, [[PG]]/z, [x0]
+; VBITS_GE_2048-NEXT: frintn [[RES:z[0-9]+]].d, [[PG]]/m, [[OP]].d
+; VBITS_GE_2048-NEXT: st1d { [[RES]].d }, [[PG]], [x0]
+; VBITS_GE_2048-NEXT: ret
+  %op = load <32 x double>, <32 x double>* %a
+  %res = call <32 x double> @llvm.roundeven.v32f64(<32 x double> %op)
+  store <32 x double> %res, <32 x double>* %a
+  ret void
+}
+
 ;
 ; TRUNC -> FRINTZ
 ;
@@ -1599,6 +1846,25 @@ declare <8 x double> @llvm.round.v8f64(<8 x double>)
 declare <16 x double> @llvm.round.v16f64(<16 x double>)
 declare <32 x double> @llvm.round.v32f64(<32 x double>)
 
+declare <4 x half> @llvm.roundeven.v4f16(<4 x half>)
+declare <8 x half> @llvm.roundeven.v8f16(<8 x half>)
+declare <16 x half> @llvm.roundeven.v16f16(<16 x half>)
+declare <32 x half> @llvm.roundeven.v32f16(<32 x half>)
+declare <64 x half> @llvm.roundeven.v64f16(<64 x half>)
+declare <128 x half> @llvm.roundeven.v128f16(<128 x half>)
+declare <2 x float> @llvm.roundeven.v2f32(<2 x float>)
+declare <4 x float> @llvm.roundeven.v4f32(<4 x float>)
+declare <8 x float> @llvm.roundeven.v8f32(<8 x float>)
+declare <16 x float> @llvm.roundeven.v16f32(<16 x float>)
+declare <32 x float> @llvm.roundeven.v32f32(<32 x float>)
+declare <64 x float> @llvm.roundeven.v64f32(<64 x float>)
+declare <1 x double> @llvm.roundeven.v1f64(<1 x double>)
+declare <2 x double> @llvm.roundeven.v2f64(<2 x double>)
+declare <4 x double> @llvm.roundeven.v4f64(<4 x double>)
+declare <8 x double> @llvm.roundeven.v8f64(<8 x double>)
+declare <16 x double> @llvm.roundeven.v16f64(<16 x double>)
+declare <32 x double> @llvm.roundeven.v32f64(<32 x double>)
+
 declare <4 x half> @llvm.trunc.v4f16(<4 x half>)
 declare <8 x half> @llvm.trunc.v8f16(<8 x half>)
 declare <16 x half> @llvm.trunc.v16f16(<16 x half>)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll
index 348c785ebae2a7e60ebd494466943dded1259813..7ad31257ebfc4b931c90214b93913181b5087b56 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-subvector.ll
@@ -32,6 +32,61 @@ target triple = "aarch64-unknown-linux-gnu"
 ; Don't use SVE when its registers are no bigger than NEON.
 ; NO_SVE-NOT: ptrue
 
+define void @subvector_v8i16(<8 x i16> *%in, <8 x i16>* %out) #0 {
+; CHECK-LABEL: subvector_v8i16:
+; CHECK: ldr [[DATA:q[0-9]+]], [x0]
+; CHECK: str [[DATA]], [x1]
+; CHECK: ret
+  %a = load <8 x i16>, <8 x i16>* %in
+  br label %bb1
+
+bb1:
+  store <8 x i16> %a, <8 x i16>* %out
+  ret void
+}
+
+define void @subvector_v16i16(<16 x i16> *%in, <16 x i16>* %out) #0 {
+; CHECK-LABEL: subvector_v16i16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK: ld1h { [[DATA:z[0-9]+.h]] }, [[PG]]/z, [x0]
+; CHECK: st1h { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <16 x i16>, <16 x i16>* %in
+  br label %bb1
+
+bb1:
+  store <16 x i16> %a, <16 x i16>* %out
+  ret void
+}
+
+define void @subvector_v32i16(<32 x i16> *%in, <32 x i16>* %out) #0 {
+; CHECK-LABEL: subvector_v32i16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512: ld1h { [[DATA:z[0-9]+.h]] }, [[PG]]/z, [x0]
+; VBITS_GE_512: st1h { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <32 x i16>, <32 x i16>* %in
+  br label %bb1
+
+bb1:
+  store <32 x i16> %a, <32 x i16>* %out
+  ret void
+}
+
+define void @subvector_v64i16(<64 x i16> *%in, <64 x i16>* %out) #0 {
+; CHECK-LABEL: subvector_v64i16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024: ld1h { [[DATA:z[0-9]+.h]] }, [[PG]]/z, [x0]
+; VBITS_GE_1024: st1h { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <64 x i16>, <64 x i16>* %in
+  br label %bb1
+
+bb1:
+  store <64 x i16> %a, <64 x i16>* %out
+  ret void
+}
+
 define void @subvector_v8i32(<8 x i32> *%in, <8 x i32>* %out) #0 {
 ; CHECK-LABEL: subvector_v8i32:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
@@ -51,7 +106,7 @@ define void @subvector_v16i32(<16 x i32> *%in, <16 x i32>* %out) #0 {
 ; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
 ; VBITS_GE_512: ld1w { [[DATA:z[0-9]+.s]] }, [[PG]]/z, [x0]
 ; VBITS_GE_512: st1w { [[DATA]] }, [[PG]], [x1]
-; CHECKT: ret
+; CHECK: ret
   %a = load <16 x i32>, <16 x i32>* %in
   br label %bb1
 
@@ -88,6 +143,201 @@ bb1:
   ret void
 }
 
+
+define void @subvector_v8i64(<8 x i64> *%in, <8 x i64>* %out) #0 {
+; CHECK-LABEL: subvector_v8i64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512: ld1d { [[DATA:z[0-9]+.d]] }, [[PG]]/z, [x0]
+; VBITS_GE_512: st1d { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <8 x i64>, <8 x i64>* %in
+  br label %bb1
+
+bb1:
+  store <8 x i64> %a, <8 x i64>* %out
+  ret void
+}
+
+define void @subvector_v16i64(<16 x i64> *%in, <16 x i64>* %out) #0 {
+; CHECK-LABEL: subvector_v16i64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ld1d { [[DATA:z[0-9]+.d]] }, [[PG]]/z, [x0]
+; VBITS_GE_1024: st1d { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <16 x i64>, <16 x i64>* %in
+  br label %bb1
+
+bb1:
+  store <16 x i64> %a, <16 x i64>* %out
+  ret void
+}
+
+define void @subvector_v32i64(<32 x i64> *%in, <32 x i64>* %out) #0 {
+; CHECK-LABEL: subvector_v32i64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ld1d { [[DATA:z[0-9]+.d]] }, [[PG]]/z, [x0]
+; VBITS_GE_2048: st1d { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <32 x i64>, <32 x i64>* %in
+  br label %bb1
+
+bb1:
+  store <32 x i64> %a, <32 x i64>* %out
+  ret void
+}
+
+define void @subvector_v8f16(<8 x half> *%in, <8 x half>* %out) #0 {
+; CHECK-LABEL: subvector_v8f16:
+; CHECK: ldr [[DATA:q[0-9]+]], [x0]
+; CHECK: str [[DATA]], [x1]
+; CHECK: ret
+  %a = load <8 x half>, <8 x half>* %in
+  br label %bb1
+
+bb1:
+  store <8 x half> %a, <8 x half>* %out
+  ret void
+}
+
+define void @subvector_v16f16(<16 x half> *%in, <16 x half>* %out) #0 {
+; CHECK-LABEL: subvector_v16f16:
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl16
+; CHECK: ld1h { [[DATA:z[0-9]+.h]] }, [[PG]]/z, [x0]
+; CHECK: st1h { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <16 x half>, <16 x half>* %in
+  br label %bb1
+
+bb1:
+  store <16 x half> %a, <16 x half>* %out
+  ret void
+}
+
+define void @subvector_v32f16(<32 x half> *%in, <32 x half>* %out) #0 {
+; CHECK-LABEL: subvector_v32f16:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].h, vl32
+; VBITS_GE_512: ld1h { [[DATA:z[0-9]+.h]] }, [[PG]]/z, [x0]
+; VBITS_GE_512: st1h { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <32 x half>, <32 x half>* %in
+  br label %bb1
+
+bb1:
+  store <32 x half> %a, <32 x half>* %out
+  ret void
+}
+
+define void @subvector_v64f16(<64 x half> *%in, <64 x half>* %out) #0 {
+; CHECK-LABEL: subvector_v64f16:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].h, vl64
+; VBITS_GE_1024: ld1h { [[DATA:z[0-9]+.h]] }, [[PG]]/z, [x0]
+; VBITS_GE_1024: st1h { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <64 x half>, <64 x half>* %in
+  br label %bb1
+
+bb1:
+  store <64 x half> %a, <64 x half>* %out
+  ret void
+}
+
+define void @subvector_v8f32(<8 x float> *%in, <8 x float>* %out) #0 {
+; CHECK-LABEL: subvector_v8f32:
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
+; CHECK: ld1w { [[DATA:z[0-9]+.s]] }, [[PG]]/z, [x0]
+; CHECK: st1w { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <8 x float>, <8 x float>* %in
+  br label %bb1
+
+bb1:
+  store <8 x float> %a, <8 x float>* %out
+  ret void
+}
+
+define void @subvector_v16f32(<16 x float> *%in, <16 x float>* %out) #0 {
+; CHECK-LABEL: subvector_v16f32:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].s, vl16
+; VBITS_GE_512: ld1w { [[DATA:z[0-9]+.s]] }, [[PG]]/z, [x0]
+; VBITS_GE_512: st1w { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <16 x float>, <16 x float>* %in
+  br label %bb1
+
+bb1:
+  store <16 x float> %a, <16 x float>* %out
+  ret void
+}
+
+define void @subvector_v32f32(<32 x float> *%in, <32 x float>* %out) #0 {
+; CHECK-LABEL: subvector_v32f32:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].s, vl32
+; VBITS_GE_1024: ld1w { [[DATA:z[0-9]+.s]] }, [[PG]]/z, [x0]
+; VBITS_GE_1024: st1w { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <32 x float>, <32 x float>* %in
+  br label %bb1
+
+bb1:
+  store <32 x float> %a, <32 x float>* %out
+  ret void
+}
+
+define void @subvector_v64f32(<64 x float> *%in, <64 x float>* %out) #0 {
+; CHECK-LABEL: subvector_v64f32:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].s, vl64
+; VBITS_GE_2048: ld1w { [[DATA:z[0-9]+.s]] }, [[PG]]/z, [x0]
+; VBITS_GE_2048: st1w { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <64 x float>, <64 x float>* %in
+  br label %bb1
+
+bb1:
+  store <64 x float> %a, <64 x float>* %out
+  ret void
+}
+define void @subvector_v8f64(<8 x double> *%in, <8 x double>* %out) #0 {
+; CHECK-LABEL: subvector_v8f64:
+; VBITS_GE_512: ptrue [[PG:p[0-9]+]].d, vl8
+; VBITS_GE_512: ld1d { [[DATA:z[0-9]+.d]] }, [[PG]]/z, [x0]
+; VBITS_GE_512: st1d { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <8 x double>, <8 x double>* %in
+  br label %bb1
+
+bb1:
+  store <8 x double> %a, <8 x double>* %out
+  ret void
+}
+
+define void @subvector_v16f64(<16 x double> *%in, <16 x double>* %out) #0 {
+; CHECK-LABEL: subvector_v16f64:
+; VBITS_GE_1024: ptrue [[PG:p[0-9]+]].d, vl16
+; VBITS_GE_1024: ld1d { [[DATA:z[0-9]+.d]] }, [[PG]]/z, [x0]
+; VBITS_GE_1024: st1d { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <16 x double>, <16 x double>* %in
+  br label %bb1
+
+bb1:
+  store <16 x double> %a, <16 x double>* %out
+  ret void
+}
+
+define void @subvector_v32f64(<32 x double> *%in, <32 x double>* %out) #0 {
+; CHECK-LABEL: subvector_v32f64:
+; VBITS_GE_2048: ptrue [[PG:p[0-9]+]].d, vl32
+; VBITS_GE_2048: ld1d { [[DATA:z[0-9]+.d]] }, [[PG]]/z, [x0]
+; VBITS_GE_2048: st1d { [[DATA]] }, [[PG]], [x1]
+; CHECK: ret
+  %a = load <32 x double>, <32 x double>* %in
+  br label %bb1
+
+bb1:
+  store <32 x double> %a, <32 x double>* %out
+  ret void
+}
+
 define <8 x i1> @no_warn_dropped_scalable(<8 x i32>* %in) #0 {
 ; CHECK-LABEL: no_warn_dropped_scalable:
 ; CHECK: ptrue [[PG:p[0-9]+]].s, vl8
@@ -112,11 +362,11 @@ define void @no_subvector_binop_hang(<8 x i32>* %in, <8 x i32>* %out, i1 %cond)
 ; CHECK-NEXT:    ptrue [[PG:p[0-9]+]].s, vl8
 ; CHECK-NEXT:    ld1w { [[A:z[0-9]+]].s }, [[PG]]/z, [x0]
 ; CHECK-NEXT:    ld1w { [[B:z[0-9]+]].s }, [[PG]]/z, [x1]
-; CHECK-NEXT:    tbz w2, #0, .LBB5_2
+; CHECK-NEXT:    tbz w2, #0, [[LABEL:\.[A-z0-9_]+]]
 ; CHECK-NEXT:  // %bb.1: // %bb.1
 ; CHECK-NEXT:    orr [[OR:z[0-9]+]].d, [[A]].d, [[B]].d
 ; CHECK-NEXT:    st1w { [[OR]].s }, [[PG]], [x1]
-; CHECK-NEXT:  .LBB5_2: // %bb.2
+; CHECK-NEXT:  [[LABEL]]: // %bb.2
 ; CHECK-NEXT:    ret
   %a = load <8 x i32>, <8 x i32>* %in
   %b = load <8 x i32>, <8 x i32>* %out
diff --git a/llvm/test/CodeGen/AArch64/sve-fmul-idempotency.ll b/llvm/test/CodeGen/AArch64/sve-fmul-idempotency.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e716aa091c61bd73da7506d009ed7aebc062421b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fmul-idempotency.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -aarch64-sve-intrinsic-opts < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+; Idempotent fmuls -- should compile to just a ret.
+define <vscale x 8 x half> @idempotent_fmul_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: @idempotent_fmul_f16(
+; CHECK-NEXT:    ret <vscale x 8 x half> [[A:%.*]]
+;
+  %1 = call <vscale x 8 x half> @llvm.aarch64.sve.dup.x.nxv8f16(half 1.0)
+  %2 = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %1)
+  ret <vscale x 8 x half> %2
+}
+
+define <vscale x 4 x float> @idempotent_fmul_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: @idempotent_fmul_f32(
+; CHECK-NEXT:    ret <vscale x 4 x float> [[A:%.*]]
+;
+  %1 = call <vscale x 4 x float> @llvm.aarch64.sve.dup.x.nxv4f32(float 1.0)
+  %2 = call <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %1)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 2 x double> @idempotent_fmul_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: @idempotent_fmul_f64(
+; CHECK-NEXT:    ret <vscale x 2 x double> [[A:%.*]]
+;
+  %1 = call <vscale x 2 x double> @llvm.aarch64.sve.dup.x.nxv2f64(double 1.0)
+  %2 = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %1)
+  ret <vscale x 2 x double> %2
+}
+
+define <vscale x 2 x double> @idempotent_fmul_different_argument_order(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: @idempotent_fmul_different_argument_order(
+; CHECK-NEXT:    ret <vscale x 2 x double> [[A:%.*]]
+;
+  %1 = call <vscale x 2 x double> @llvm.aarch64.sve.dup.x.nxv2f64(double 1.0)
+  ; Different argument order to the above tests.
+  %2 = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %1, <vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %2
+}
+
+define <vscale x 8 x half> @idempotent_fmul_with_predicated_dup(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: @idempotent_fmul_with_predicated_dup(
+; CHECK-NEXT:    ret <vscale x 8 x half> [[A:%.*]]
+;
+  %1 = call <vscale x 8 x half> @llvm.aarch64.sve.dup.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> %pg, half 1.0)
+  %2 = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %1)
+  ret <vscale x 8 x half> %2
+}
+
+define <vscale x 8 x half> @idempotent_fmul_two_dups(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+  ; Edge case -- make sure that the case where we're fmultiplying two dups
+  ; together is sane.
+; CHECK-LABEL: @idempotent_fmul_two_dups(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x half> @llvm.aarch64.sve.dup.x.nxv8f16(half 0xH3C00)
+; CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+;
+  %1 = call <vscale x 8 x half> @llvm.aarch64.sve.dup.x.nxv8f16(half 1.0)
+  %2 = call <vscale x 8 x half> @llvm.aarch64.sve.dup.x.nxv8f16(half 1.0)
+  %3 = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %1, <vscale x 8 x half> %2)
+  ret <vscale x 8 x half> %3
+}
+
+; Non-idempotent fmuls -- we don't expect these to be optimised out.
+define <vscale x 8 x half> @non_idempotent_fmul_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: @non_idempotent_fmul_f16(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x half> @llvm.aarch64.sve.dup.x.nxv8f16(half 0xH4000)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> [[PG:%.*]], <vscale x 8 x half> [[A:%.*]], <vscale x 8 x half> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 8 x half> [[TMP2]]
+;
+  %1 = call <vscale x 8 x half> @llvm.aarch64.sve.dup.x.nxv8f16(half 2.0)
+  %2 = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %1)
+  ret <vscale x 8 x half> %2
+}
+
+define <vscale x 4 x float> @non_idempotent_fmul_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: @non_idempotent_fmul_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.dup.x.nxv4f32(float 2.000000e+00)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1> [[PG:%.*]], <vscale x 4 x float> [[A:%.*]], <vscale x 4 x float> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP2]]
+;
+  %1 = call <vscale x 4 x float> @llvm.aarch64.sve.dup.x.nxv4f32(float 2.0)
+  %2 = call <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %1)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 2 x double> @non_idempotent_fmul_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: @non_idempotent_fmul_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.aarch64.sve.dup.x.nxv2f64(double 2.000000e+00)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> [[PG:%.*]], <vscale x 2 x double> [[A:%.*]], <vscale x 2 x double> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP2]]
+;
+  %1 = call <vscale x 2 x double> @llvm.aarch64.sve.dup.x.nxv2f64(double 2.0)
+  %2 = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %1)
+  ret <vscale x 2 x double> %2
+}
+
+define <vscale x 2 x double> @non_idempotent_fmul_with_predicated_dup(<vscale x 2 x i1> %pg1, <vscale x 2 x i1> %pg2, <vscale x 2 x double> %a) {
+  ; Different predicates
+; CHECK-LABEL: @non_idempotent_fmul_with_predicated_dup(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> [[PG1:%.*]], double 1.000000e+00)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> [[PG2:%.*]], <vscale x 2 x double> [[A:%.*]], <vscale x 2 x double> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 2 x double> [[TMP2]]
+;
+  %1 = call <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> %pg1, double 1.0)
+  %2 = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %pg2, <vscale x 2 x double> %a, <vscale x 2 x double> %1)
+  ret <vscale x 2 x double> %2
+}
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.dup.x.nxv8f16(half)
+declare <vscale x 4 x float> @llvm.aarch64.sve.dup.x.nxv4f32(float)
+declare <vscale x 2 x double> @llvm.aarch64.sve.dup.x.nxv2f64(double)
+
+declare <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double)
+declare <vscale x 8 x half> @llvm.aarch64.sve.dup.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half)
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll
index 64cb89edd679e6bb6ebb4b5dfd36e84cba33fb0e..57778847b545f4a5f612d11d15902e6148a13d63 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll
@@ -78,7 +78,194 @@ define <vscale x 2 x i64> @gld1sw_index(<vscale x 2 x i1> %pg, i32* %base, <vsca
   ret <vscale x 2 x i64> %res
 }
 
+;
+; LD1H, LD1W, LD1D: base + 64-bit sxtw'd scaled offset
+;   e.g. ld1h z0.d, p0/z, [x0, z0.d, sxtw #1]
+;
+
+define <vscale x 2 x i64> @gld1h_index_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1h_index_sxtw
+; CHECK:	    ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+; CHECK-NEXT:	ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                             i16* %base,
+                                                                             <vscale x 2 x i64> %sxtw)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1w_index_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1w_index_sxtw
+; CHECK:	    ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+; CHECK-NEXT:	ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                             i32* %base,
+                                                                             <vscale x 2 x i64> %sxtw)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1d_index_sxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_index_sxtw
+; CHECK:	    ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+; CHECK-NEXT:	ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                             i64* %base,
+                                                                             <vscale x 2 x i64> %sxtw)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @gld1d_index_double_sxtw(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_index_double_sxtw
+; CHECK:	    ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+; CHECK-NEXT:	ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.index.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                                double* %base,
+                                                                                <vscale x 2 x i64> %sxtw)
+  ret <vscale x 2 x double> %load
+}
+
+;
+; LD1SH, LD1SW: base + 64-bit sxtw'd scaled offset
+;   e.g. ld1sh z0.d, p0/z, [x0, z0.d, sxtw #1]
+;
+
+define <vscale x 2 x i64> @gld1sh_index_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sh_index_sxtw
+; CHECK:	    ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+; CHECK-NEXT:	ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                             i16* %base,
+                                                                             <vscale x 2 x i64> %sxtw)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sw_index_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sw_index_sxtw
+; CHECK:	    ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+; CHECK-NEXT:	ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                             i32* %base,
+                                                                             <vscale x 2 x i64> %sxtw)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; LD1H, LD1W, LD1D: base + 64-bit sxtw'd scaled offset
+;   e.g. ld1h z0.d, p0/z, [x0, z0.d, uxtw #1]
+;
+
+define <vscale x 2 x i64> @gld1h_index_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1h_index_uxtw
+; CHECK:	    ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+; CHECK-NEXT:	ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                             i16* %base,
+                                                                             <vscale x 2 x i64> %uxtw)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1w_index_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1w_index_uxtw
+; CHECK:	    ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+; CHECK-NEXT:	ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                             i32* %base,
+                                                                             <vscale x 2 x i64> %uxtw)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1d_index_uxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_index_uxtw
+; CHECK:	    ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+; CHECK-NEXT:	ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                             i64* %base,
+                                                                             <vscale x 2 x i64> %uxtw)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @gld1d_index_double_uxtw(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_index_double_uxtw
+; CHECK:	    ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+; CHECK-NEXT:	ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.index.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                                double* %base,
+                                                                                <vscale x 2 x i64> %uxtw)
+  ret <vscale x 2 x double> %load
+}
+
+;
+; LD1SH, LD1SW: base + 64-bit uxtw'd scaled offset
+;   e.g. ld1sh z0.d, p0/z, [x0, z0.d, uxtw #1]
+;
+
+define <vscale x 2 x i64> @gld1sh_index_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sh_index_uxtw
+; CHECK:	    ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+; CHECK-NEXT:	ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                             i16* %base,
+                                                                             <vscale x 2 x i64> %uxtw)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sw_index_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sw_index_uxtw
+; CHECK:	    ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+; CHECK-NEXT:	ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                             i32* %base,
+                                                                             <vscale x 2 x i64> %uxtw)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
 declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i64>)
 declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i64>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i64>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.index.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i64>)
+
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll
index 7cf641a264274470fa8b3b44516750dc033f01c5..21c08d152ef3a046cc282951f77ef56f729fde43 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll
@@ -100,8 +100,251 @@ define <vscale x 2 x i64> @gld1sw_d(<vscale x 2 x i1> %pg, i32* %base, <vscale x
   ret <vscale x 2 x i64> %res
 }
 
+;
+; LD1B, LD1W, LD1H, LD1D: base + 64-bit sxtw'd unscaled offset
+;   e.g. ld1h { z0.d }, p0/z, [x0, z0.d, sxtw]
+;
+
+define <vscale x 2 x i64> @gld1b_d_sxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1b_d_sxtw:
+; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                     i8* %base,
+                                                                     <vscale x 2 x i64> %sxtw)
+  %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1h_d_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1h_d_sxtw:
+; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                       i16* %base,
+                                                                       <vscale x 2 x i64> %sxtw)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1w_d_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: gld1w_d_sxtw:
+; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %offsets)
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                       i32* %base,
+                                                                       <vscale x 2 x i64> %sxtw)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1d_d_sxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_d_sxtw:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                       i64* %base,
+                                                                       <vscale x 2 x i64> %sxtw)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @gld1d_d_double_sxtw(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_d_double_sxtw:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                       double* %base,
+                                                                       <vscale x 2 x i64> %sxtw)
+  ret <vscale x 2 x double> %load
+}
+
+;
+; LD1SB, LD1SW, LD1SH: base + 64-bit sxtw'd unscaled offset
+;   e.g. ld1sh { z0.d }, p0/z, [x0, z0.d]
+;
+
+define <vscale x 2 x i64> @gld1sb_d_sxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sb_d_sxtw:
+; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                     i8* %base,
+                                                                     <vscale x 2 x i64> %sxtw)
+  %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sh_d_sxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sh_d_sxtw:
+; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                       i16* %base,
+                                                                       <vscale x 2 x i64> %sxtw)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sw_d_sxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: gld1sw_d_sxtw:
+; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT: ret
+  %sxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %offsets)
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                       i32* %base,
+                                                                       <vscale x 2 x i64> %sxtw)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+;
+; LD1B, LD1W, LD1H, LD1D: base + 64-bit uxtw'd unscaled offset
+;   e.g. ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]
+;
+
+define <vscale x 2 x i64> @gld1b_d_uxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1b_d_uxtw:
+; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                     i8* %base,
+                                                                     <vscale x 2 x i64> %uxtw)
+  %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1h_d_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1h_d_uxtw:
+; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                       i16* %base,
+                                                                       <vscale x 2 x i64> %uxtw)
+  %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1w_d_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: gld1w_d_uxtw:
+; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %offsets)
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                       i32* %base,
+                                                                       <vscale x 2 x i64> %uxtw)
+  %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1d_d_uxtw(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_d_uxtw:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                       i64* %base,
+                                                                       <vscale x 2 x i64> %uxtw)
+  ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x double> @gld1d_d_double_uxtw(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1d_d_double_uxtw:
+; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                       double* %base,
+                                                                       <vscale x 2 x i64> %uxtw)
+  ret <vscale x 2 x double> %load
+}
+
+;
+; LD1SB, LD1SW, LD1SH: base + 64-bit uxtw'd unscaled offset
+;   e.g. ld1sh { z0.d }, p0/z, [x0, z0.d]
+;
+
+define <vscale x 2 x i64> @gld1sb_d_uxtw(<vscale x 2 x i1> %pg, i8* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sb_d_uxtw:
+; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> %pg,
+                                                                     i8* %base,
+                                                                     <vscale x 2 x i64> %uxtw)
+  %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sh_d_uxtw(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: gld1sh_d_uxtw:
+; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %b)
+  %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> %pg,
+                                                                       i16* %base,
+                                                                       <vscale x 2 x i64> %uxtw)
+  %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @gld1sw_d_uxtw(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %offsets) {
+; CHECK-LABEL: gld1sw_d_uxtw:
+; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT: ret
+  %uxtw = call <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64> undef,
+                                                                 <vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x i64> %offsets)
+  %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> %pg,
+                                                                       i32* %base,
+                                                                       <vscale x 2 x i64> %uxtw)
+  %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %res
+}
+
 declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1>, i8*, <vscale x 2 x i64>)
 declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i64>)
 declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i64>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i64>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i64>)
+
+declare <vscale x 2 x i64> @llvm.aarch64.sve.sxtw.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.uxtw.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-mul-idempotency.ll b/llvm/test/CodeGen/AArch64/sve-mul-idempotency.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d07e100f9d57bf884bb1eef801ae7525d09706ad
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-mul-idempotency.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -aarch64-sve-intrinsic-opts < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+; Idempotent muls -- should compile to just a ret.
+define <vscale x 8 x i16> @idempotent_mul_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: @idempotent_mul_i16(
+; CHECK-NEXT:    ret <vscale x 8 x i16> [[A:%.*]]
+;
+  %1 = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.x.nxv8i16(i16 1)
+  %2 = call <vscale x 8 x i16> @llvm.aarch64.sve.mul.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %1)
+  ret <vscale x 8 x i16> %2
+}
+
+define <vscale x 4 x i32> @idempotent_mul_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: @idempotent_mul_i32(
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[A:%.*]]
+;
+  %1 = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.x.nxv4i32(i32 1)
+  %2 = call <vscale x 4 x i32> @llvm.aarch64.sve.mul.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %1)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 2 x i64> @idempotent_mul_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: @idempotent_mul_i64(
+; CHECK-NEXT:    ret <vscale x 2 x i64> [[A:%.*]]
+;
+  %1 = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.x.nxv2i64(i64 1)
+  %2 = call <vscale x 2 x i64> @llvm.aarch64.sve.mul.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %1)
+  ret <vscale x 2 x i64> %2
+}
+
+define <vscale x 2 x i64> @idempotent_mul_different_argument_order(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: @idempotent_mul_different_argument_order(
+; CHECK-NEXT:    ret <vscale x 2 x i64> [[A:%.*]]
+;
+  %1 = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.x.nxv2i64(i64 1)
+  ; Different argument order to the above tests.
+  %2 = call <vscale x 2 x i64> @llvm.aarch64.sve.mul.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %1, <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i64> %2
+}
+
+define <vscale x 8 x i16> @idempotent_mul_with_predicated_dup(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: @idempotent_mul_with_predicated_dup(
+; CHECK-NEXT:    ret <vscale x 8 x i16> [[A:%.*]]
+;
+  %1 = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> %pg, i16 1)
+  %2 = call <vscale x 8 x i16> @llvm.aarch64.sve.mul.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %1)
+  ret <vscale x 8 x i16> %2
+}
+
+define <vscale x 8 x i16> @idempotent_mul_two_dups(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+  ; Edge case -- make sure that the case where we're multiplying two dups
+  ; together is sane.
+; CHECK-LABEL: @idempotent_mul_two_dups(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.x.nxv8i16(i16 1)
+; CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+;
+  %1 = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.x.nxv8i16(i16 1)
+  %2 = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.x.nxv8i16(i16 1)
+  %3 = call <vscale x 8 x i16> @llvm.aarch64.sve.mul.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2)
+  ret <vscale x 8 x i16> %3
+}
+
+; Non-idempotent muls -- we don't expect these to be optimised out.
+define <vscale x 8 x i16> @non_idempotent_mul_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: @non_idempotent_mul_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.x.nxv8i16(i16 2)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.mul.nxv8i16(<vscale x 8 x i1> [[PG:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 8 x i16> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
+;
+  %1 = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.x.nxv8i16(i16 2)
+  %2 = call <vscale x 8 x i16> @llvm.aarch64.sve.mul.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %1)
+  ret <vscale x 8 x i16> %2
+}
+
+define <vscale x 4 x i32> @non_idempotent_mul_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: @non_idempotent_mul_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.x.nxv4i32(i32 2)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.mul.nxv4i32(<vscale x 4 x i1> [[PG:%.*]], <vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
+;
+  %1 = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.x.nxv4i32(i32 2)
+  %2 = call <vscale x 4 x i32> @llvm.aarch64.sve.mul.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %1)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 2 x i64> @non_idempotent_mul_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: @non_idempotent_mul_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.x.nxv2i64(i64 2)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.mul.nxv2i64(<vscale x 2 x i1> [[PG:%.*]], <vscale x 2 x i64> [[A:%.*]], <vscale x 2 x i64> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
+;
+  %1 = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.x.nxv2i64(i64 2)
+  %2 = call <vscale x 2 x i64> @llvm.aarch64.sve.mul.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %1)
+  ret <vscale x 2 x i64> %2
+}
+
+define <vscale x 2 x i64> @non_idempotent_mul_with_predicated_dup(<vscale x 2 x i1> %pg1, <vscale x 2 x i1> %pg2, <vscale x 2 x i64> %a) {
+  ; Different predicates
+; CHECK-LABEL: @non_idempotent_mul_with_predicated_dup(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> [[PG1:%.*]], i64 1)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.mul.nxv2i64(<vscale x 2 x i1> [[PG2:%.*]], <vscale x 2 x i64> [[A:%.*]], <vscale x 2 x i64> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
+;
+  %1 = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> %pg1, i64 1)
+  %2 = call <vscale x 2 x i64> @llvm.aarch64.sve.mul.nxv2i64(<vscale x 2 x i1> %pg2, <vscale x 2 x i64> %a, <vscale x 2 x i64> %1)
+  ret <vscale x 2 x i64> %2
+}
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.dup.x.nxv8i16(i16)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.dup.x.nxv4i32(i32)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.dup.x.nxv2i64(i64)
+
+declare <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16)
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.mul.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.mul.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.mul.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-stepvector.ll b/llvm/test/CodeGen/AArch64/sve-stepvector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1b162ec577c21726553bbf76a449c1dac9bfdb8f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-stepvector.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s --check-prefixes=CHECK
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s < %t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+; LEGAL INTEGER TYPES
+
+define <vscale x 2 x i64> @stepvector_nxv2i64() {
+; CHECK-LABEL: stepvector_nxv2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    index z0.d, #0, #1
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+  ret <vscale x 2 x i64> %0
+}
+
+define <vscale x 4 x i32> @stepvector_nxv4i32() {
+; CHECK-LABEL: stepvector_nxv4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    index z0.s, #0, #1
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+  ret <vscale x 4 x i32> %0
+}
+
+define <vscale x 8 x i16> @stepvector_nxv8i16() {
+; CHECK-LABEL: stepvector_nxv8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    index z0.h, #0, #1
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+  ret <vscale x 8 x i16> %0
+}
+
+define <vscale x 16 x i8> @stepvector_nxv16i8() {
+; CHECK-LABEL: stepvector_nxv16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    index z0.b, #0, #1
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <vscale x 16 x i8> @llvm.experimental.stepvector.nxv16i8()
+  ret <vscale x 16 x i8> %0
+}
+
+; ILLEGAL INTEGER TYPES
+
+define <vscale x 4 x i64> @stepvector_nxv4i64() {
+; CHECK-LABEL: stepvector_nxv4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    index z0.d, #0, #1
+; CHECK-NEXT:    add z1.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+  ret <vscale x 4 x i64> %0
+}
+
+define <vscale x 16 x i32> @stepvector_nxv16i32() {
+; CHECK-LABEL: stepvector_nxv16i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cntw x9
+; CHECK-NEXT:    cnth x8
+; CHECK-NEXT:    index z0.s, #0, #1
+; CHECK-NEXT:    mov z1.s, w9
+; CHECK-NEXT:    mov z3.s, w8
+; CHECK-NEXT:    add z1.s, z0.s, z1.s
+; CHECK-NEXT:    add z2.s, z0.s, z3.s
+; CHECK-NEXT:    add z3.s, z1.s, z3.s
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <vscale x 16 x i32> @llvm.experimental.stepvector.nxv16i32()
+  ret <vscale x 16 x i32> %0
+}
+
+define <vscale x 2 x i32> @stepvector_nxv2i32() {
+; CHECK-LABEL: stepvector_nxv2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    index z0.d, #0, #1
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <vscale x 2 x i32> @llvm.experimental.stepvector.nxv2i32()
+  ret <vscale x 2 x i32> %0
+}
+
+define <vscale x 4 x i16> @stepvector_nxv4i16() {
+; CHECK-LABEL: stepvector_nxv4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    index z0.s, #0, #1
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <vscale x 4 x i16> @llvm.experimental.stepvector.nxv4i16()
+  ret <vscale x 4 x i16> %0
+}
+
+define <vscale x 8 x i8> @stepvector_nxv8i8() {
+; CHECK-LABEL: stepvector_nxv8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    index z0.h, #0, #1
+; CHECK-NEXT:    ret
+entry:
+  %0 = call <vscale x 8 x i8> @llvm.experimental.stepvector.nxv8i8()
+  ret <vscale x 8 x i8> %0
+}
+
+declare <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+declare <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+declare <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+declare <vscale x 16 x i8> @llvm.experimental.stepvector.nxv16i8()
+
+declare <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+declare <vscale x 16 x i32> @llvm.experimental.stepvector.nxv16i32()
+declare <vscale x 2 x i32> @llvm.experimental.stepvector.nxv2i32()
+declare <vscale x 8 x i8> @llvm.experimental.stepvector.nxv8i8()
+declare <vscale x 4 x i16> @llvm.experimental.stepvector.nxv4i16()
diff --git a/llvm/test/CodeGen/AArch64/vec-libcalls.ll b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
index 80ec45ca49cb2c383c112b4a3e41f28527ddabe8..e96a4b815d6bb91be57b4316ba2ab9810f612f8f 100644
--- a/llvm/test/CodeGen/AArch64/vec-libcalls.ll
+++ b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
@@ -29,6 +29,7 @@ declare <3 x float> @llvm.log2.v3f32(<3 x float>)
 declare <3 x float> @llvm.nearbyint.v3f32(<3 x float>)
 declare <3 x float> @llvm.rint.v3f32(<3 x float>)
 declare <3 x float> @llvm.round.v3f32(<3 x float>)
+declare <3 x float> @llvm.roundeven.v3f32(<3 x float>)
 declare <3 x float> @llvm.sqrt.v3f32(<3 x float>)
 declare <3 x float> @llvm.trunc.v3f32(<3 x float>)
 
@@ -478,6 +479,15 @@ define <3 x float> @round_v3f32(<3 x float> %x) nounwind {
   ret <3 x float> %r
 }
 
+define <3 x float> @roundeven_v3f32(<3 x float> %x) nounwind {
+; CHECK-LABEL: roundeven_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintn v0.4s, v0.4s
+; CHECK-NEXT:    ret
+  %r = call <3 x float> @llvm.roundeven.v3f32(<3 x float> %x)
+  ret <3 x float> %r
+}
+
 define <3 x float> @sqrt_v3f32(<3 x float> %x) nounwind {
 ; CHECK-LABEL: sqrt_v3f32:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-urem-pow-2.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-urem-pow-2.mir
index 93b723c5e730b16922f7abe0f135180e24657537..0c5c66ba4a89d98067b9bbebaad5ba36371e1076 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-urem-pow-2.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-urem-pow-2.mir
@@ -29,10 +29,8 @@ body:             |
 
     ; GCN-LABEL: name: urem_s32_var_const1
     ; GCN: liveins: $vgpr0
-    ; GCN: %const:_(s32) = G_CONSTANT i32 1
-    ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GCN: [[ADD:%[0-9]+]]:_(s32) = G_ADD %const, [[C]]
-    ; GCN: $vgpr0 = COPY [[ADD]](s32)
+    ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GCN: $vgpr0 = COPY [[C]](s32)
     %var:_(s32) = COPY $vgpr0
     %const:_(s32) = G_CONSTANT i32 1
     %rem:_(s32) = G_UREM %var, %const
@@ -49,10 +47,8 @@ body:             |
     ; GCN-LABEL: name: urem_s32_var_const2
     ; GCN: liveins: $vgpr0
     ; GCN: %var:_(s32) = COPY $vgpr0
-    ; GCN: %const:_(s32) = G_CONSTANT i32 2
-    ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; GCN: [[ADD:%[0-9]+]]:_(s32) = G_ADD %const, [[C]]
-    ; GCN: %rem:_(s32) = G_AND %var, [[ADD]]
+    ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GCN: %rem:_(s32) = G_AND %var, [[C]]
     ; GCN: $vgpr0 = COPY %rem(s32)
     %var:_(s32) = COPY $vgpr0
     %const:_(s32) = G_CONSTANT i32 2
@@ -147,12 +143,44 @@ body:             |
     ; GCN: %var:_(<2 x s16>) = COPY $vgpr0
     ; GCN: %four:_(s32) = G_CONSTANT i32 4
     ; GCN: %four_vec:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC %four(s32), %four(s32)
+    ; GCN: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GCN: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
+    ; GCN: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD %four_vec, [[BUILD_VECTOR]]
+    ; GCN: %rem:_(<2 x s16>) = G_AND %var, [[ADD]]
+    ; GCN: $vgpr0 = COPY %rem(<2 x s16>)
+    %var:_(<2 x s16>) = COPY $vgpr0
+    %shift_amt:_(s32) = COPY $vgpr1
+    %four:_(s32) = G_CONSTANT i32 4
+    %four_vec:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC %four, %four
+    %rem:_(<2 x s16>) = G_UREM %var, %four_vec
+    $vgpr0 = COPY %rem
+...
+
+# The shl is a known power of two, but we do not know if the final
+# value is a power of 2 due to the truncation.
+---
+name: urem_v2s16_var_nonconst_build_vector_trunc
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GCN-LABEL: name: urem_v2s16_var_nonconst_build_vector_trunc
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN: %var:_(<2 x s16>) = COPY $vgpr0
+    ; GCN: %shift_amt:_(<2 x s16>) = COPY $vgpr1
+    ; GCN: %two:_(s32) = G_CONSTANT i32 2
+    ; GCN: %four:_(s32) = G_CONSTANT i32 4
+    ; GCN: %shift:_(s32) = G_SHL %two, %shift_amt(<2 x s16>)
+    ; GCN: %four_vec:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC %four(s32), %shift(s32)
     ; GCN: %rem:_(<2 x s16>) = G_UREM %var, %four_vec
     ; GCN: $vgpr0 = COPY %rem(<2 x s16>)
     %var:_(<2 x s16>) = COPY $vgpr0
     %shift_amt:_(<2 x s16>) = COPY $vgpr1
+    %two:_(s32) = G_CONSTANT i32 2
     %four:_(s32) = G_CONSTANT i32 4
-    %four_vec:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC %four, %four
+    %shift:_(s32) = G_SHL %two, %shift_amt
+    %four_vec:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC %four, %shift
     %rem:_(<2 x s16>) = G_UREM %var, %four_vec
     $vgpr0 = COPY %rem
 ...
@@ -194,7 +222,10 @@ body:             |
     ; GCN: %pow2_1:_(s32) = G_CONSTANT i32 4096
     ; GCN: %pow2_2:_(s32) = G_CONSTANT i32 2048
     ; GCN: %pow2_vec:_(<2 x s32>) = G_BUILD_VECTOR %pow2_1(s32), %pow2_2(s32)
-    ; GCN: %rem:_(<2 x s32>) = G_UREM %var, %pow2_vec
+    ; GCN: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GCN: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+    ; GCN: [[ADD:%[0-9]+]]:_(<2 x s32>) = G_ADD %pow2_vec, [[BUILD_VECTOR]]
+    ; GCN: %rem:_(<2 x s32>) = G_AND %var, [[ADD]]
     ; GCN: $vgpr0_vgpr1 = COPY %rem(<2 x s32>)
     %var:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %pow2_1:_(s32) = G_CONSTANT i32 4096
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
index a57e681d0d845a0426039916315fa60cb2380368..a1b9c2c7df32f115cdd9239f94ef75e6c1648111 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
@@ -26,8 +26,8 @@ define amdgpu_ps i128 @extractelement_vgpr_v4i128_sgpr_idx(<4 x i128> addrspace(
 ; GFX9-NEXT:    global_load_dwordx4 v[14:17], v[0:1], off offset:48
 ; GFX9-NEXT:    s_lshl_b32 s0, s2, 1
 ; GFX9-NEXT:    s_lshl_b32 s2, s0, 1
-; GFX9-NEXT:    s_set_gpr_idx_on s2, gpr_idx(SRC0)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_set_gpr_idx_on s2, gpr_idx(SRC0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v18, v2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5e98c4e6abd1863a9c5a31831a50cc5c15435741
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -0,0 +1,7474 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -o - %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - %s | FileCheck -check-prefixes=GCN,GFX10 %s
+
+define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
+; GFX6-LABEL: s_fshl_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v0, 7
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_sub_i32 s3, 0, 7
+; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX6-NEXT:    s_movk_i32 s3, 0x7f
+; GFX6-NEXT:    s_and_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s1, s1, s3
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, 7
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 7, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 7, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, 6, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX6-NEXT:    v_lshl_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshr_b32_e32 v1, s1, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, 7
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    s_sub_i32 s3, 0, 7
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX8-NEXT:    s_movk_i32 s3, 0x7f
+; GFX8-NEXT:    s_and_b32 s2, s2, s3
+; GFX8-NEXT:    s_and_b32 s1, s1, s3
+; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, 7
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v1, vcc, 7, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v1, vcc, 7, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_sub_u16_e32 v1, 6, v0
+; GFX8-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX8-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v0, v0, s0
+; GFX8-NEXT:    v_lshrrev_b16_e64 v1, v1, s1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v0, 7
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_sub_i32 s3, 0, 7
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX9-NEXT:    s_movk_i32 s3, 0x7f
+; GFX9-NEXT:    s_and_b32 s2, s2, s3
+; GFX9-NEXT:    s_and_b32 s1, s1, s3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 7
+; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v1, 7, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v1, 7, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_sub_u16_e32 v1, 6, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX9-NEXT:    v_lshlrev_b16_e64 v0, v0, s0
+; GFX9-NEXT:    v_lshrrev_b16_e64 v1, v1, s1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, 7
+; GFX10-NEXT:    s_sub_i32 s3, 0, 7
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX10-NEXT:    s_movk_i32 s3, 0x7f
+; GFX10-NEXT:    s_and_b32 s2, s2, s3
+; GFX10-NEXT:    s_and_b32 s1, s1, s3
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, 7
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 7, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 7, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 7, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 7, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_sub_nc_u16_e64 v1, 6, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v0, s0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v1, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i7 @llvm.fshl.i7(i7 %lhs, i7 %rhs, i7 %amt)
+  ret i7 %result
+}
+
+define i7 @v_fshl_i7(i7 %lhs, i7 %rhs, i7 %amt) {
+; GFX6-LABEL: v_fshl_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v3, 7
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX6-NEXT:    s_sub_i32 s4, 0, 7
+; GFX6-NEXT:    v_and_b32_e32 v2, 0x7f, v2
+; GFX6-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX6-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX6-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX6-NEXT:    v_mov_b32_e32 v4, 0x7f
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_mul_lo_u32 v3, v3, 7
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 7, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 7, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 6, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v2, v0
+; GFX6-NEXT:    v_and_b32_e32 v2, v3, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v3, 7
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX8-NEXT:    s_sub_i32 s4, 0, 7
+; GFX8-NEXT:    v_and_b32_e32 v2, 0x7f, v2
+; GFX8-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX8-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX8-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0x7f
+; GFX8-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 1, v1
+; GFX8-NEXT:    v_mul_lo_u32 v3, v3, 7
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, 7, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, 7, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX8-NEXT:    v_sub_u16_e32 v3, 6, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v2, v0
+; GFX8-NEXT:    v_and_b32_e32 v2, v3, v4
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v3, 7
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX9-NEXT:    s_sub_i32 s4, 0, 7
+; GFX9-NEXT:    v_and_b32_e32 v2, 0x7f, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX9-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x7f
+; GFX9-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, 1, v1
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, 7
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 7, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 7, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT:    v_sub_u16_e32 v3, 6, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, v2, v0
+; GFX9-NEXT:    v_and_b32_e32 v2, v3, v4
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v3, 7
+; GFX10-NEXT:    s_sub_i32 s4, 0, 7
+; GFX10-NEXT:    v_and_b32_e32 v2, 0x7f, v2
+; GFX10-NEXT:    v_and_b32_e32 v1, 0x7f, v1
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 1, v1
+; GFX10-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX10-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v4
+; GFX10-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX10-NEXT:    v_mul_lo_u32 v3, v3, 7
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 7, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 7, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0x7f
+; GFX10-NEXT:    v_sub_nc_u16_e64 v4, 6, v2
+; GFX10-NEXT:    v_and_b32_e32 v2, v2, v3
+; GFX10-NEXT:    v_and_b32_e32 v3, v4, v3
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v2, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v3, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i7 @llvm.fshl.i7(i7 %lhs, i7 %rhs, i7 %amt)
+  ret i7 %result
+}
+
+define amdgpu_ps i8 @s_fshl_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
+; GFX6-LABEL: s_fshl_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX6-NEXT:    s_and_b32 s3, s2, 7
+; GFX6-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_and_b32 s3, s2, 7
+; GFX8-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_and_b32 s3, s2, 7
+; GFX9-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX10-NEXT:    s_and_b32 s3, s2, 7
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 %amt)
+  ret i8 %result
+}
+
+define i8 @v_fshl_i8(i8 %lhs, i8 %rhs, i8 %amt) {
+; GFX6-LABEL: v_fshl_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v3, 7, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v3, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v3, 7, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v3, v0
+; GFX8-NEXT:    v_mov_b32_e32 v3, 1
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v3, 7, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, v3, v0
+; GFX9-NEXT:    v_mov_b32_e32 v3, 1
+; GFX9-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v3, -1, v2
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX10-NEXT:    v_and_b32_e32 v3, 7, v3
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 1, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v2, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v3, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 %amt)
+  ret i8 %result
+}
+
+define amdgpu_ps i8 @s_fshl_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
+; GFX6-LABEL: s_fshl_i8_4:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 4
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i8_4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 4
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i8_4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 4
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i8_4:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 4
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 4)
+  ret i8 %result
+}
+
+define i8 @v_fshl_i8_4(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_fshl_i8_4:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 4, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i8_4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 4
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 4, v0
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i8_4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 4, v0
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i8_4:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 4, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 4, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 4)
+  ret i8 %result
+}
+
+define amdgpu_ps i8 @s_fshl_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
+; GFX6-LABEL: s_fshl_i8_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 5
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 3
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i8_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 5
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 3
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i8_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 5
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 3
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i8_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 5
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 3
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 5)
+  ret i8 %result
+}
+
+define i8 @v_fshl_i8_5(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_fshl_i8_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 3, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i8_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 5, v0
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i8_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, 3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 5, v0
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i8_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 5, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 3, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 5)
+  ret i8 %result
+}
+
+define amdgpu_ps i16 @s_fshl_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 inreg %amt.arg) {
+; GFX6-LABEL: s_fshl_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s5, s2, 7
+; GFX6-NEXT:    s_movk_i32 s6, 0xff
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX6-NEXT:    s_and_b32 s5, s1, s6
+; GFX6-NEXT:    s_lshr_b32 s4, s2, 8
+; GFX6-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX6-NEXT:    s_lshr_b32 s5, s5, 1
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 9
+; GFX6-NEXT:    s_lshr_b32 s2, s5, s2
+; GFX6-NEXT:    s_or_b32 s0, s0, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, 7
+; GFX6-NEXT:    s_andn2_b32 s4, 7, s4
+; GFX6-NEXT:    s_and_b32 s1, s1, 0x7f
+; GFX6-NEXT:    s_lshl_b32 s2, s3, s2
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s4
+; GFX6-NEXT:    s_or_b32 s1, s2, s1
+; GFX6-NEXT:    s_and_b32 s1, s1, s6
+; GFX6-NEXT:    s_and_b32 s0, s0, s6
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s6, s2, 7
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s6
+; GFX8-NEXT:    s_movk_i32 s6, 0xff
+; GFX8-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX8-NEXT:    s_and_b32 s1, s1, s6
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX8-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s5, 7
+; GFX8-NEXT:    s_lshl_b32 s1, s3, s1
+; GFX8-NEXT:    s_and_b32 s3, s4, s6
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_andn2_b32 s2, 7, s5
+; GFX8-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX8-NEXT:    s_lshr_b32 s2, s3, s2
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    s_and_b32 s1, s1, s6
+; GFX8-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX8-NEXT:    s_and_b32 s0, s0, s6
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s6, s2, 7
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s6
+; GFX9-NEXT:    s_movk_i32 s6, 0xff
+; GFX9-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX9-NEXT:    s_and_b32 s1, s1, s6
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX9-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s5, 7
+; GFX9-NEXT:    s_lshl_b32 s1, s3, s1
+; GFX9-NEXT:    s_and_b32 s3, s4, s6
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_andn2_b32 s2, 7, s5
+; GFX9-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX9-NEXT:    s_lshr_b32 s2, s3, s2
+; GFX9-NEXT:    s_or_b32 s1, s1, s2
+; GFX9-NEXT:    s_and_b32 s1, s1, s6
+; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX9-NEXT:    s_and_b32 s0, s0, s6
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX10-NEXT:    s_movk_i32 s6, 0xff
+; GFX10-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX10-NEXT:    s_and_b32 s4, s4, s6
+; GFX10-NEXT:    s_and_b32 s1, s1, s6
+; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX10-NEXT:    s_and_b32 s7, s2, 7
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s7
+; GFX10-NEXT:    s_and_b32 s7, s5, 7
+; GFX10-NEXT:    s_andn2_b32 s5, 7, s5
+; GFX10-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX10-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s7
+; GFX10-NEXT:    s_lshr_b32 s4, s4, s5
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX10-NEXT:    s_or_b32 s2, s3, s4
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_and_b32 s1, s2, s6
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    s_and_b32 s0, s0, s6
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %amt = bitcast i16 %amt.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.fshl.v2i8(<2 x i8> %lhs, <2 x i8> %rhs, <2 x i8> %amt)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define i16 @v_fshl_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
+; GFX6-LABEL: v_fshl_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v6, 0xff, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 8, v2
+; GFX6-NEXT:    v_and_b32_e32 v5, 7, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 1, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v5, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, v2, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 9, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, 7, v4
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, 7, v4
+; GFX6-NEXT:    v_and_b32_e32 v1, 0x7f, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v4, v1
+; GFX6-NEXT:    v_mov_b32_e32 v5, 0xff
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v5
+; GFX6-NEXT:    v_and_b32_e32 v0, v0, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v6, 7, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v6, v0
+; GFX8-NEXT:    v_mov_b32_e32 v6, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v5
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_e32 v1, 7, v5
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, v1, v3
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v3, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, v2, v3
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v6, 7, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, v6, v0
+; GFX9-NEXT:    v_mov_b32_e32 v6, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v5
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v1, 7, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, v1, v3
+; GFX9-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v3, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v2, v2, v3
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_xor_b32_e32 v7, -1, v2
+; GFX10-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX10-NEXT:    v_xor_b32_e32 v6, -1, v3
+; GFX10-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 8, v0
+; GFX10-NEXT:    v_and_b32_e32 v3, 7, v3
+; GFX10-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX10-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX10-NEXT:    v_lshrrev_b16_e64 v4, 1, v4
+; GFX10-NEXT:    v_and_b32_e32 v7, 7, v7
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 1, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v3, v3, v5
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v2, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v4, v6, v4
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v7, v1
+; GFX10-NEXT:    v_or_b32_e32 v2, v3, v4
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, v2, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %amt = bitcast i16 %amt.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.fshl.v2i8(<2 x i8> %lhs, <2 x i8> %rhs, <2 x i8> %amt)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define amdgpu_ps i32 @s_fshl_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 inreg %amt.arg) {
+; GFX6-LABEL: s_fshl_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s9, s2, 7
+; GFX6-NEXT:    s_movk_i32 s10, 0xff
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX6-NEXT:    s_lshr_b32 s5, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s9
+; GFX6-NEXT:    s_and_b32 s9, s1, s10
+; GFX6-NEXT:    s_lshr_b32 s6, s2, 8
+; GFX6-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX6-NEXT:    s_lshr_b32 s8, s2, 24
+; GFX6-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX6-NEXT:    s_lshr_b32 s9, s9, 1
+; GFX6-NEXT:    s_lshr_b32 s2, s9, s2
+; GFX6-NEXT:    s_or_b32 s0, s0, s2
+; GFX6-NEXT:    s_and_b32 s2, s6, 7
+; GFX6-NEXT:    s_lshl_b32 s2, s3, s2
+; GFX6-NEXT:    s_lshr_b32 s3, s1, 9
+; GFX6-NEXT:    s_movk_i32 s9, 0x7f
+; GFX6-NEXT:    s_andn2_b32 s6, 7, s6
+; GFX6-NEXT:    s_and_b32 s3, s3, s9
+; GFX6-NEXT:    s_lshr_b32 s3, s3, s6
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s3, s7, 7
+; GFX6-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX6-NEXT:    s_lshr_b32 s4, s1, 17
+; GFX6-NEXT:    s_andn2_b32 s6, 7, s7
+; GFX6-NEXT:    s_and_b32 s4, s4, s9
+; GFX6-NEXT:    s_lshr_b32 s4, s4, s6
+; GFX6-NEXT:    s_and_b32 s2, s2, s10
+; GFX6-NEXT:    s_or_b32 s3, s3, s4
+; GFX6-NEXT:    s_and_b32 s4, s8, 7
+; GFX6-NEXT:    s_andn2_b32 s6, 7, s8
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 25
+; GFX6-NEXT:    s_lshl_b32 s4, s5, s4
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s6
+; GFX6-NEXT:    s_and_b32 s0, s0, s10
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 8
+; GFX6-NEXT:    s_or_b32 s1, s4, s1
+; GFX6-NEXT:    s_or_b32 s0, s0, s2
+; GFX6-NEXT:    s_and_b32 s2, s3, s10
+; GFX6-NEXT:    s_and_b32 s1, s1, s10
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s13, 0xff
+; GFX8-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s1, 24
+; GFX8-NEXT:    s_and_b32 s1, s1, s13
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s9, s2, 8
+; GFX8-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s2, 24
+; GFX8-NEXT:    s_and_b32 s12, s2, 7
+; GFX8-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s5, s0, 24
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s12
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s9, 7
+; GFX8-NEXT:    s_lshl_b32 s1, s3, s1
+; GFX8-NEXT:    s_and_b32 s3, s6, s13
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_andn2_b32 s2, 7, s9
+; GFX8-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX8-NEXT:    s_lshr_b32 s2, s3, s2
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    s_and_b32 s2, s10, 7
+; GFX8-NEXT:    s_lshl_b32 s2, s4, s2
+; GFX8-NEXT:    s_and_b32 s4, s7, s13
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_andn2_b32 s3, 7, s10
+; GFX8-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX8-NEXT:    s_lshr_b32 s3, s4, s3
+; GFX8-NEXT:    s_and_b32 s1, s1, s13
+; GFX8-NEXT:    s_or_b32 s2, s2, s3
+; GFX8-NEXT:    s_and_b32 s3, s11, 7
+; GFX8-NEXT:    s_lshl_b32 s3, s5, s3
+; GFX8-NEXT:    s_andn2_b32 s4, 7, s11
+; GFX8-NEXT:    s_lshr_b32 s5, s8, 1
+; GFX8-NEXT:    s_and_b32 s0, s0, s13
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX8-NEXT:    s_lshr_b32 s4, s5, s4
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s2, s13
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_or_b32 s3, s3, s4
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s3, s13
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s13, 0xff
+; GFX9-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX9-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s1, 24
+; GFX9-NEXT:    s_and_b32 s1, s1, s13
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s9, s2, 8
+; GFX9-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s2, 24
+; GFX9-NEXT:    s_and_b32 s12, s2, 7
+; GFX9-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s12
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s9, 7
+; GFX9-NEXT:    s_lshl_b32 s1, s3, s1
+; GFX9-NEXT:    s_and_b32 s3, s6, s13
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_andn2_b32 s2, 7, s9
+; GFX9-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX9-NEXT:    s_lshr_b32 s2, s3, s2
+; GFX9-NEXT:    s_or_b32 s1, s1, s2
+; GFX9-NEXT:    s_and_b32 s2, s10, 7
+; GFX9-NEXT:    s_lshl_b32 s2, s4, s2
+; GFX9-NEXT:    s_and_b32 s4, s7, s13
+; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX9-NEXT:    s_andn2_b32 s3, 7, s10
+; GFX9-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX9-NEXT:    s_lshr_b32 s3, s4, s3
+; GFX9-NEXT:    s_and_b32 s1, s1, s13
+; GFX9-NEXT:    s_or_b32 s2, s2, s3
+; GFX9-NEXT:    s_and_b32 s3, s11, 7
+; GFX9-NEXT:    s_lshl_b32 s3, s5, s3
+; GFX9-NEXT:    s_andn2_b32 s4, 7, s11
+; GFX9-NEXT:    s_lshr_b32 s5, s8, 1
+; GFX9-NEXT:    s_and_b32 s0, s0, s13
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_lshr_b32 s4, s5, s4
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s2, s13
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX9-NEXT:    s_or_b32 s3, s3, s4
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s3, s13
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s11, 0xff
+; GFX10-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s8, s1, 24
+; GFX10-NEXT:    s_and_b32 s1, s1, s11
+; GFX10-NEXT:    s_lshr_b32 s9, s2, 8
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s12, s2, 24
+; GFX10-NEXT:    s_and_b32 s13, s2, 7
+; GFX10-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX10-NEXT:    s_and_b32 s2, s6, s11
+; GFX10-NEXT:    s_and_b32 s6, s9, 7
+; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    s_andn2_b32 s9, 7, s9
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 1
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s6
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s5, s0, 24
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s13
+; GFX10-NEXT:    s_lshr_b32 s2, s2, s9
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_or_b32 s1, s3, s2
+; GFX10-NEXT:    s_and_b32 s2, s7, s11
+; GFX10-NEXT:    s_and_b32 s3, s10, 7
+; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    s_andn2_b32 s6, 7, s10
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 1
+; GFX10-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX10-NEXT:    s_lshr_b32 s2, s2, s6
+; GFX10-NEXT:    s_and_b32 s4, s12, 7
+; GFX10-NEXT:    s_andn2_b32 s6, 7, s12
+; GFX10-NEXT:    s_lshr_b32 s7, s8, 1
+; GFX10-NEXT:    s_or_b32 s2, s3, s2
+; GFX10-NEXT:    s_and_b32 s1, s1, s11
+; GFX10-NEXT:    s_lshl_b32 s4, s5, s4
+; GFX10-NEXT:    s_lshr_b32 s5, s7, s6
+; GFX10-NEXT:    s_and_b32 s0, s0, s11
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX10-NEXT:    s_or_b32 s3, s4, s5
+; GFX10-NEXT:    s_and_b32 s2, s2, s11
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, 16
+; GFX10-NEXT:    s_and_b32 s2, s3, s11
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %amt = bitcast i32 %amt.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.fshl.v4i8(<4 x i8> %lhs, <4 x i8> %rhs, <4 x i8> %amt)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define i32 @v_fshl_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
+; GFX6-LABEL: v_fshl_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v10, 0xff, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 8, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v8, 24, v2
+; GFX6-NEXT:    v_and_b32_e32 v9, 7, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v10, 1, v10
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v9, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, v2, v10
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, 7, v6
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 9, v1
+; GFX6-NEXT:    s_movk_i32 s4, 0x7f
+; GFX6-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, v6, v3
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_and_b32_e32 v3, 7, v7
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v7
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, v3, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 17, v1
+; GFX6-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX6-NEXT:    v_mov_b32_e32 v9, 0xff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v6, v4
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v8
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v9
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, 7, v8
+; GFX6-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 25, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, v4, v5
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v6, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, v0, v9
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, v3, v9
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v9
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v8, 7, v2
+; GFX8-NEXT:    v_mov_b32_e32 v10, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v11, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v8, v8, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, v2, v11
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, v8, v2
+; GFX8-NEXT:    v_and_b32_e32 v8, 7, v5
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v5
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_and_b32_e32 v5, 7, v5
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v4, v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v4, v5, v4
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, v8, v3
+; GFX8-NEXT:    v_mov_b32_e32 v9, 0xff
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v6
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    v_and_b32_e32 v4, 7, v6
+; GFX8-NEXT:    v_and_b32_sdwa v6, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_e32 v5, 7, v5
+; GFX8-NEXT:    v_lshrrev_b16_e32 v6, 1, v6
+; GFX8-NEXT:    v_lshrrev_b16_e32 v5, v5, v6
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v4, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v6, -1, v7
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v5
+; GFX8-NEXT:    v_and_b32_e32 v5, 7, v7
+; GFX8-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, v6, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX8-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v8, 7, v2
+; GFX9-NEXT:    v_mov_b32_e32 v10, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v11, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v8, v8, v0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v2, v2, v11
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, v8, v2
+; GFX9-NEXT:    v_and_b32_e32 v8, 7, v5
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX9-NEXT:    v_and_b32_e32 v5, 7, v5
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v4, v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v4, v5, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, v8, v3
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0xff
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v6
+; GFX9-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX9-NEXT:    v_and_b32_e32 v4, 7, v6
+; GFX9-NEXT:    v_and_b32_sdwa v6, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_e32 v5, 7, v5
+; GFX9-NEXT:    v_lshrrev_b16_e32 v6, 1, v6
+; GFX9-NEXT:    v_lshrrev_b16_e32 v5, v5, v6
+; GFX9-NEXT:    v_lshlrev_b16_sdwa v4, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_xor_b32_e32 v6, -1, v7
+; GFX9-NEXT:    v_or_b32_e32 v4, v4, v5
+; GFX9-NEXT:    v_and_b32_e32 v5, 7, v7
+; GFX9-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_lshlrev_b16_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, v6, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_mov_b32_e32 v1, 8
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX9-NEXT:    v_and_or_b32 v1, v2, s4, v1
+; GFX9-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 8, v2
+; GFX10-NEXT:    v_and_b32_e32 v11, 7, v2
+; GFX10-NEXT:    v_xor_b32_e32 v10, -1, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v11, v0
+; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v8
+; GFX10-NEXT:    v_and_b32_e32 v8, 7, v8
+; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
+; GFX10-NEXT:    v_mov_b32_e32 v15, 0xff
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_xor_b32_e32 v13, -1, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX10-NEXT:    v_and_b32_e32 v12, s4, v1
+; GFX10-NEXT:    v_and_b32_e32 v6, s4, v6
+; GFX10-NEXT:    v_and_b32_sdwa v1, v1, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v3, v8, v3
+; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v9
+; GFX10-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX10-NEXT:    v_and_b32_e32 v9, 7, v9
+; GFX10-NEXT:    v_and_b32_e32 v11, 7, v11
+; GFX10-NEXT:    v_lshrrev_b16_e64 v6, 1, v6
+; GFX10-NEXT:    v_and_b32_e32 v8, 7, v8
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 1, v1
+; GFX10-NEXT:    v_and_b32_e32 v13, 7, v13
+; GFX10-NEXT:    v_lshrrev_b16_e64 v7, 1, v7
+; GFX10-NEXT:    v_lshrrev_b16_e64 v6, v11, v6
+; GFX10-NEXT:    v_lshlrev_b16_e64 v2, v2, v5
+; GFX10-NEXT:    v_and_b32_e32 v10, 7, v10
+; GFX10-NEXT:    v_lshrrev_b16_e64 v12, 1, v12
+; GFX10-NEXT:    v_lshrrev_b16_e64 v5, v13, v7
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, v9, v4
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v8, v1
+; GFX10-NEXT:    v_or_b32_e32 v3, v3, v6
+; GFX10-NEXT:    v_lshrrev_b16_e64 v7, v10, v12
+; GFX10-NEXT:    v_or_b32_e32 v2, v2, v5
+; GFX10-NEXT:    v_mov_b32_e32 v6, 8
+; GFX10-NEXT:    v_or_b32_e32 v1, v4, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v7
+; GFX10-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX10-NEXT:    v_lshlrev_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
+; GFX10-NEXT:    v_and_or_b32 v0, v0, s4, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %amt = bitcast i32 %amt.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.fshl.v4i8(<4 x i8> %lhs, <4 x i8> %rhs, <4 x i8> %amt)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define amdgpu_ps i24 @s_fshl_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt) {
+; GFX6-LABEL: s_fshl_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_sub_i32 s3, 0, 24
+; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX6-NEXT:    s_mov_b32 s3, 0xffffff
+; GFX6-NEXT:    s_and_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s1, s1, s3
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 24, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 24, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, 23, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX6-NEXT:    v_lshl_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshr_b32_e32 v1, s1, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    s_sub_i32 s3, 0, 24
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xffffff
+; GFX8-NEXT:    s_and_b32 s2, s2, s3
+; GFX8-NEXT:    s_and_b32 s1, s1, s3
+; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v1, vcc, 24, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v1, vcc, 24, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, 23, v0
+; GFX8-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX8-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX8-NEXT:    v_lshlrev_b32_e64 v0, v0, s0
+; GFX8-NEXT:    v_lshrrev_b32_e64 v1, v1, s1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_sub_i32 s3, 0, 24
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xffffff
+; GFX9-NEXT:    s_and_b32 s2, s2, s3
+; GFX9-NEXT:    s_and_b32 s1, s1, s3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v1, 24, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v1, 24, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v1, 23, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX9-NEXT:    v_lshrrev_b32_e64 v1, v1, s1
+; GFX9-NEXT:    v_lshl_or_b32 v0, s0, v0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX10-NEXT:    s_sub_i32 s3, 0, 24
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX10-NEXT:    s_mov_b32 s3, 0xffffff
+; GFX10-NEXT:    s_and_b32 s2, s2, s3
+; GFX10-NEXT:    s_and_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 24, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 24, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 23, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX10-NEXT:    v_lshrrev_b32_e64 v1, v1, s1
+; GFX10-NEXT:    v_lshl_or_b32 v0, s0, v0, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i24 @llvm.fshl.i24(i24 %lhs, i24 %rhs, i24 %amt)
+  ret i24 %result
+}
+
+define i24 @v_fshl_i24(i24 %lhs, i24 %rhs, i24 %amt) {
+; GFX6-LABEL: v_fshl_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v3, 24
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX6-NEXT:    s_sub_i32 s4, 0, 24
+; GFX6-NEXT:    v_and_b32_e32 v2, 0xffffff, v2
+; GFX6-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX6-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX6-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX6-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_mul_lo_u32 v3, v3, 24
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 24, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 24, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 23, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v2, v0
+; GFX6-NEXT:    v_and_b32_e32 v2, v3, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v3, 24
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX8-NEXT:    s_sub_i32 s4, 0, 24
+; GFX8-NEXT:    v_and_b32_e32 v2, 0xffffff, v2
+; GFX8-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX8-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX8-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX8-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX8-NEXT:    v_mul_lo_u32 v3, v3, 24
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, 24, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, 24, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, 23, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, v2, v0
+; GFX8-NEXT:    v_and_b32_e32 v2, v3, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v3, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX9-NEXT:    s_sub_i32 s4, 0, 24
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffffff, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX9-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX9-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, 24
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 24, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 24, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v3, 23, v2
+; GFX9-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX9-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, v3, v1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, v2, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v3, 24
+; GFX10-NEXT:    s_sub_i32 s4, 0, 24
+; GFX10-NEXT:    v_and_b32_e32 v2, 0xffffff, v2
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX10-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v4
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX10-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX10-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_mul_lo_u32 v3, v3, 24
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 24, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 24, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 23, v2
+; GFX10-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX10-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, v3, v1
+; GFX10-NEXT:    v_lshl_or_b32 v0, v0, v2, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i24 @llvm.fshl.i24(i24 %lhs, i24 %rhs, i24 %amt)
+  ret i24 %result
+}
+
+define amdgpu_ps i48 @s_fshl_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 inreg %amt.arg) {
+; GFX6-LABEL: s_fshl_v2i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshr_b32 s6, s0, 8
+; GFX6-NEXT:    s_movk_i32 s10, 0xff
+; GFX6-NEXT:    s_and_b32 s6, s6, s10
+; GFX6-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX6-NEXT:    s_lshr_b32 s8, s0, 24
+; GFX6-NEXT:    s_and_b32 s0, s0, s10
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s6
+; GFX6-NEXT:    s_and_b32 s6, s7, s10
+; GFX6-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s9, s1, 8
+; GFX6-NEXT:    s_and_b32 s1, s1, s10
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s6
+; GFX6-NEXT:    s_and_b32 s6, s9, s10
+; GFX6-NEXT:    s_or_b32 s1, s8, s1
+; GFX6-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s6
+; GFX6-NEXT:    s_lshr_b32 s6, s2, 8
+; GFX6-NEXT:    s_and_b32 s6, s6, s10
+; GFX6-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX6-NEXT:    s_lshr_b32 s8, s2, 24
+; GFX6-NEXT:    s_and_b32 s2, s2, s10
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 8
+; GFX6-NEXT:    s_or_b32 s2, s2, s6
+; GFX6-NEXT:    s_and_b32 s6, s7, s10
+; GFX6-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s9, s3, 8
+; GFX6-NEXT:    s_and_b32 s3, s3, s10
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 8
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX6-NEXT:    s_or_b32 s2, s2, s6
+; GFX6-NEXT:    s_and_b32 s6, s9, s10
+; GFX6-NEXT:    s_or_b32 s3, s8, s3
+; GFX6-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_or_b32 s3, s3, s6
+; GFX6-NEXT:    s_lshr_b32 s6, s4, 8
+; GFX6-NEXT:    s_and_b32 s6, s6, s10
+; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX6-NEXT:    s_lshr_b32 s8, s4, 24
+; GFX6-NEXT:    s_and_b32 s4, s4, s10
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 8
+; GFX6-NEXT:    s_or_b32 s4, s4, s6
+; GFX6-NEXT:    s_and_b32 s6, s7, s10
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_or_b32 s4, s4, s6
+; GFX6-NEXT:    s_sub_i32 s6, 0, 24
+; GFX6-NEXT:    v_mul_lo_u32 v1, s6, v0
+; GFX6-NEXT:    s_lshr_b32 s9, s5, 8
+; GFX6-NEXT:    s_and_b32 s5, s5, s10
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 8
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT:    s_and_b32 s7, s9, s10
+; GFX6-NEXT:    s_or_b32 s5, s8, s5
+; GFX6-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v1, 24
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX6-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX6-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX6-NEXT:    s_or_b32 s5, s5, s7
+; GFX6-NEXT:    s_mov_b32 s7, 0xffffff
+; GFX6-NEXT:    v_mul_lo_u32 v3, s6, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX6-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 23, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, s7, v0
+; GFX6-NEXT:    v_lshl_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, 24
+; GFX6-NEXT:    s_lshr_b32 s0, s2, 1
+; GFX6-NEXT:    v_and_b32_e32 v2, s7, v2
+; GFX6-NEXT:    v_lshr_b32_e32 v2, s0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s5, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v1
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v1
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 23, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX6-NEXT:    s_lshr_b32 s0, s3, 1
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX6-NEXT:    v_lshl_b32_e32 v1, s1, v1
+; GFX6-NEXT:    v_lshr_b32_e32 v2, s0, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_and_b32_e32 v2, s10, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, s10, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s10, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s10, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s10, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s10, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_v2i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 8
+; GFX8-NEXT:    s_movk_i32 s10, 0xff
+; GFX8-NEXT:    s_and_b32 s6, s6, s10
+; GFX8-NEXT:    s_bfe_u32 s11, 8, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s0, 24
+; GFX8-NEXT:    s_and_b32 s0, s0, s10
+; GFX8-NEXT:    s_lshl_b32 s6, s6, s11
+; GFX8-NEXT:    s_or_b32 s0, s0, s6
+; GFX8-NEXT:    s_and_b32 s6, s7, s10
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s9, s1, 8
+; GFX8-NEXT:    s_and_b32 s1, s1, s10
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s11
+; GFX8-NEXT:    s_or_b32 s0, s0, s6
+; GFX8-NEXT:    s_and_b32 s6, s9, s10
+; GFX8-NEXT:    s_or_b32 s1, s8, s1
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s6
+; GFX8-NEXT:    s_lshr_b32 s6, s2, 8
+; GFX8-NEXT:    s_and_b32 s6, s6, s10
+; GFX8-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s2, 24
+; GFX8-NEXT:    s_and_b32 s2, s2, s10
+; GFX8-NEXT:    s_lshl_b32 s6, s6, s11
+; GFX8-NEXT:    s_or_b32 s2, s2, s6
+; GFX8-NEXT:    s_and_b32 s6, s7, s10
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s9, s3, 8
+; GFX8-NEXT:    s_and_b32 s3, s3, s10
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX8-NEXT:    s_lshl_b32 s3, s3, s11
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX8-NEXT:    s_or_b32 s2, s2, s6
+; GFX8-NEXT:    s_and_b32 s6, s9, s10
+; GFX8-NEXT:    s_or_b32 s3, s8, s3
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX8-NEXT:    s_or_b32 s3, s3, s6
+; GFX8-NEXT:    s_lshr_b32 s6, s4, 8
+; GFX8-NEXT:    s_and_b32 s6, s6, s10
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s4, 24
+; GFX8-NEXT:    s_and_b32 s4, s4, s10
+; GFX8-NEXT:    s_lshl_b32 s6, s6, s11
+; GFX8-NEXT:    s_or_b32 s4, s4, s6
+; GFX8-NEXT:    s_and_b32 s6, s7, s10
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX8-NEXT:    s_or_b32 s4, s4, s6
+; GFX8-NEXT:    s_sub_i32 s6, 0, 24
+; GFX8-NEXT:    v_mul_lo_u32 v1, s6, v0
+; GFX8-NEXT:    s_lshr_b32 s9, s5, 8
+; GFX8-NEXT:    s_and_b32 s5, s5, s10
+; GFX8-NEXT:    s_lshl_b32 s5, s5, s11
+; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX8-NEXT:    s_and_b32 s7, s9, s10
+; GFX8-NEXT:    s_or_b32 s5, s8, s5
+; GFX8-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v1, 24
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX8-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX8-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX8-NEXT:    s_or_b32 s5, s5, s7
+; GFX8-NEXT:    s_mov_b32 s7, 0xffffff
+; GFX8-NEXT:    v_mul_lo_u32 v3, s6, v1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s4, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 24, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX8-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 24, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
+; GFX8-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 23, v0
+; GFX8-NEXT:    v_and_b32_e32 v0, s7, v0
+; GFX8-NEXT:    v_lshlrev_b32_e64 v0, v0, s0
+; GFX8-NEXT:    v_mul_lo_u32 v1, v1, 24
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 1
+; GFX8-NEXT:    v_and_b32_e32 v2, s7, v2
+; GFX8-NEXT:    v_lshrrev_b32_e64 v2, v2, s0
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, s5, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 24, v1
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 24, v1
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 23, v1
+; GFX8-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX8-NEXT:    s_lshr_b32 s0, s3, 1
+; GFX8-NEXT:    v_lshlrev_b32_e64 v1, v1, s1
+; GFX8-NEXT:    v_lshrrev_b32_e64 v2, v2, s0
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX8-NEXT:    v_mov_b32_e32 v4, 8
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s10
+; GFX8-NEXT:    v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT:    v_and_b32_e32 v2, s10, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_v2i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_lshr_b32 s7, s0, 8
+; GFX9-NEXT:    s_movk_i32 s11, 0xff
+; GFX9-NEXT:    s_and_b32 s7, s7, s11
+; GFX9-NEXT:    s_bfe_u32 s12, 8, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s0, 24
+; GFX9-NEXT:    s_and_b32 s0, s0, s11
+; GFX9-NEXT:    s_lshl_b32 s7, s7, s12
+; GFX9-NEXT:    s_or_b32 s0, s0, s7
+; GFX9-NEXT:    s_and_b32 s7, s8, s11
+; GFX9-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s10, s1, 8
+; GFX9-NEXT:    s_and_b32 s1, s1, s11
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s12
+; GFX9-NEXT:    s_or_b32 s0, s0, s7
+; GFX9-NEXT:    s_and_b32 s7, s10, s11
+; GFX9-NEXT:    s_or_b32 s1, s9, s1
+; GFX9-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX9-NEXT:    s_or_b32 s1, s1, s7
+; GFX9-NEXT:    s_lshr_b32 s7, s2, 8
+; GFX9-NEXT:    s_and_b32 s7, s7, s11
+; GFX9-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s2, 24
+; GFX9-NEXT:    s_and_b32 s2, s2, s11
+; GFX9-NEXT:    s_lshl_b32 s7, s7, s12
+; GFX9-NEXT:    s_or_b32 s2, s2, s7
+; GFX9-NEXT:    s_and_b32 s7, s8, s11
+; GFX9-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s10, s3, 8
+; GFX9-NEXT:    s_and_b32 s3, s3, s11
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX9-NEXT:    s_or_b32 s2, s2, s7
+; GFX9-NEXT:    s_and_b32 s7, s10, s11
+; GFX9-NEXT:    s_lshl_b32 s3, s3, s12
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX9-NEXT:    s_or_b32 s3, s9, s3
+; GFX9-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX9-NEXT:    s_or_b32 s3, s3, s7
+; GFX9-NEXT:    s_lshr_b32 s7, s4, 8
+; GFX9-NEXT:    s_and_b32 s7, s7, s11
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    s_lshr_b32 s8, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s4, 24
+; GFX9-NEXT:    s_and_b32 s4, s4, s11
+; GFX9-NEXT:    s_lshl_b32 s7, s7, s12
+; GFX9-NEXT:    s_or_b32 s4, s4, s7
+; GFX9-NEXT:    s_and_b32 s7, s8, s11
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX9-NEXT:    s_or_b32 s4, s4, s7
+; GFX9-NEXT:    s_sub_i32 s7, 0, 24
+; GFX9-NEXT:    v_mul_lo_u32 v1, s7, v0
+; GFX9-NEXT:    s_lshr_b32 s10, s5, 8
+; GFX9-NEXT:    s_and_b32 s5, s5, s11
+; GFX9-NEXT:    s_lshl_b32 s5, s5, s12
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    s_and_b32 s8, s10, s11
+; GFX9-NEXT:    s_or_b32 s5, s9, s5
+; GFX9-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v1, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX9-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX9-NEXT:    s_or_b32 s5, s5, s8
+; GFX9-NEXT:    s_mov_b32 s8, 0xffffff
+; GFX9-NEXT:    v_mul_lo_u32 v3, s7, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v2, 24, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v2, 24, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, 23, v0
+; GFX9-NEXT:    s_lshr_b32 s2, s2, 1
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 24
+; GFX9-NEXT:    v_and_b32_e32 v2, s8, v2
+; GFX9-NEXT:    v_and_b32_e32 v0, s8, v0
+; GFX9-NEXT:    v_lshrrev_b32_e64 v2, v2, s2
+; GFX9-NEXT:    v_sub_u32_e32 v1, s5, v1
+; GFX9-NEXT:    v_lshl_or_b32 v0, s0, v0, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v2, 24, v1
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v2, 24, v1
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffffff
+; GFX9-NEXT:    v_sub_u32_e32 v2, 23, v1
+; GFX9-NEXT:    s_lshr_b32 s0, s3, 1
+; GFX9-NEXT:    v_and_b32_e32 v2, v2, v3
+; GFX9-NEXT:    v_and_b32_e32 v1, v1, v3
+; GFX9-NEXT:    v_lshrrev_b32_e64 v2, v2, s0
+; GFX9-NEXT:    v_lshl_or_b32 v1, s1, v1, v2
+; GFX9-NEXT:    s_mov_b32 s6, 8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v2, s6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_e32 v4, s11, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_and_or_b32 v2, v0, s11, v2
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
+; GFX9-NEXT:    v_or3_b32 v0, v2, v0, v4
+; GFX9-NEXT:    v_and_or_b32 v1, v3, s11, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_v2i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, 24
+; GFX10-NEXT:    s_movk_i32 s8, 0xff
+; GFX10-NEXT:    s_lshr_b32 s11, s1, 8
+; GFX10-NEXT:    s_bfe_u32 s10, 8, 0x100000
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX10-NEXT:    s_and_b32 s1, s1, s8
+; GFX10-NEXT:    s_lshr_b32 s9, s0, 24
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s10
+; GFX10-NEXT:    s_lshr_b32 s6, s0, 8
+; GFX10-NEXT:    s_or_b32 s1, s9, s1
+; GFX10-NEXT:    s_sub_i32 s9, 0, 24
+; GFX10-NEXT:    s_and_b32 s6, s6, s8
+; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; GFX10-NEXT:    s_and_b32 s0, s0, s8
+; GFX10-NEXT:    s_lshl_b32 s6, s6, s10
+; GFX10-NEXT:    s_lshr_b32 s12, s4, 24
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX10-NEXT:    s_or_b32 s0, s0, s6
+; GFX10-NEXT:    s_and_b32 s6, s7, s8
+; GFX10-NEXT:    s_lshr_b32 s7, s4, 8
+; GFX10-NEXT:    v_mul_lo_u32 v2, s9, v0
+; GFX10-NEXT:    v_mul_lo_u32 v3, s9, v1
+; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX10-NEXT:    s_and_b32 s7, s7, s8
+; GFX10-NEXT:    s_or_b32 s0, s0, s6
+; GFX10-NEXT:    s_and_b32 s6, s11, s8
+; GFX10-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_and_b32 s4, s4, s8
+; GFX10-NEXT:    s_lshl_b32 s7, s7, s10
+; GFX10-NEXT:    s_and_b32 s9, s11, s8
+; GFX10-NEXT:    s_or_b32 s4, s4, s7
+; GFX10-NEXT:    s_bfe_u32 s7, s9, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s13, s5, 8
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v2
+; GFX10-NEXT:    v_mul_hi_u32 v2, v1, v3
+; GFX10-NEXT:    s_and_b32 s5, s5, s8
+; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX10-NEXT:    s_lshl_b32 s5, s5, s10
+; GFX10-NEXT:    s_or_b32 s4, s4, s7
+; GFX10-NEXT:    s_and_b32 s7, s13, s8
+; GFX10-NEXT:    s_or_b32 s5, s12, s5
+; GFX10-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX10-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v2
+; GFX10-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX10-NEXT:    s_lshr_b32 s9, s2, 16
+; GFX10-NEXT:    s_or_b32 s5, s5, s7
+; GFX10-NEXT:    s_lshr_b32 s7, s2, 8
+; GFX10-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX10-NEXT:    s_lshr_b32 s11, s2, 24
+; GFX10-NEXT:    s_and_b32 s2, s2, s8
+; GFX10-NEXT:    s_lshr_b32 s12, s3, 8
+; GFX10-NEXT:    s_and_b32 s3, s3, s8
+; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s10
+; GFX10-NEXT:    v_mul_lo_u32 v1, v1, 24
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s4, v0
+; GFX10-NEXT:    s_and_b32 s4, s7, s8
+; GFX10-NEXT:    s_and_b32 s7, s9, s8
+; GFX10-NEXT:    s_lshl_b32 s4, s4, s10
+; GFX10-NEXT:    s_or_b32 s3, s11, s3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 24, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, s5, v1
+; GFX10-NEXT:    s_or_b32 s2, s2, s4
+; GFX10-NEXT:    s_bfe_u32 s4, s7, 0x100000
+; GFX10-NEXT:    s_mov_b32 s5, 0xffffff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 24, v1
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v1
+; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 24, v0
+; GFX10-NEXT:    s_or_b32 s2, s2, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX10-NEXT:    s_and_b32 s4, s12, s8
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 1
+; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 24, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v1
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 23, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v0, s5, v0
+; GFX10-NEXT:    v_and_b32_e32 v2, s5, v3
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0xffffff
+; GFX10-NEXT:    v_sub_nc_u32_e32 v4, 23, v1
+; GFX10-NEXT:    v_lshrrev_b32_e64 v2, v2, s2
+; GFX10-NEXT:    s_bfe_u32 s2, s3, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s3, s4, 16
+; GFX10-NEXT:    v_and_b32_e32 v4, v4, v3
+; GFX10-NEXT:    s_or_b32 s2, s2, s3
+; GFX10-NEXT:    v_lshl_or_b32 v0, s0, v0, v2
+; GFX10-NEXT:    s_lshr_b32 s0, s2, 1
+; GFX10-NEXT:    v_and_b32_e32 v1, v1, v3
+; GFX10-NEXT:    v_lshrrev_b32_e64 v2, v4, s0
+; GFX10-NEXT:    s_or_b32 s0, s1, s6
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX10-NEXT:    v_lshl_or_b32 v1, s0, v1, v2
+; GFX10-NEXT:    s_mov_b32 s0, 8
+; GFX10-NEXT:    v_lshlrev_b32_sdwa v2, s0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT:    v_and_b32_e32 v3, s8, v1
+; GFX10-NEXT:    v_and_b32_sdwa v4, v1, s8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    v_and_or_b32 v2, v0, s8, v2
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT:    v_or3_b32 v0, v2, v0, v3
+; GFX10-NEXT:    v_and_or_b32 v1, v1, s8, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i48 %lhs.arg to <2 x i24>
+  %rhs = bitcast i48 %rhs.arg to <2 x i24>
+  %amt = bitcast i48 %amt.arg to <2 x i24>
+  %result = call <2 x i24> @llvm.fshl.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
+  %cast.result = bitcast <2 x i24> %result to i48
+  ret i48 %cast.result
+}
+
+define <2 x i24> @v_fshl_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
+; GFX6-LABEL: v_fshl_v2i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v6, 24
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GFX6-NEXT:    s_sub_i32 s4, 0, 24
+; GFX6-NEXT:    v_and_b32_e32 v4, 0xffffff, v4
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v8, 24
+; GFX6-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GFX6-NEXT:    v_mul_lo_u32 v7, s4, v6
+; GFX6-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; GFX6-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v7, v8
+; GFX6-NEXT:    v_mov_b32_e32 v8, 0xffffff
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v8
+; GFX6-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX6-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, 24, v4
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, 24, v4
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v6, s4, v7
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, 23, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, v4, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v4, v0
+; GFX6-NEXT:    v_mul_hi_u32 v6, v7, v6
+; GFX6-NEXT:    v_and_b32_e32 v4, v5, v8
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v7, v6
+; GFX6-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GFX6-NEXT:    v_and_b32_e32 v6, v9, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, v6, v2
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_mul_lo_u32 v5, v5, 24
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v4, v5
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, 24, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, 24, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, 23, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, v3, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX6-NEXT:    v_and_b32_e32 v3, v4, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, v3, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v2i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v6, 24
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GFX8-NEXT:    s_sub_i32 s4, 0, 24
+; GFX8-NEXT:    v_and_b32_e32 v4, 0xffffff, v4
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v8, 24
+; GFX8-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GFX8-NEXT:    v_mul_lo_u32 v7, s4, v6
+; GFX8-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v7
+; GFX8-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v7, v8
+; GFX8-NEXT:    v_mov_b32_e32 v8, 0xffffff
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v8
+; GFX8-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX8-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v4, v6
+; GFX8-NEXT:    v_subrev_u32_e32 v6, vcc, 24, v4
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v6, vcc, 24, v4
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_mul_lo_u32 v6, s4, v7
+; GFX8-NEXT:    v_sub_u32_e32 v9, vcc, 23, v4
+; GFX8-NEXT:    v_and_b32_e32 v4, v4, v8
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, v4, v0
+; GFX8-NEXT:    v_mul_hi_u32 v6, v7, v6
+; GFX8-NEXT:    v_and_b32_e32 v4, v5, v8
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v7, v6
+; GFX8-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GFX8-NEXT:    v_and_b32_e32 v6, v9, v8
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, v6, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_mul_lo_u32 v5, v5, 24
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v4, v5
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, 24, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, 24, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, 23, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v8
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, v2, v1
+; GFX8-NEXT:    v_and_b32_e32 v2, v3, v8
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX8-NEXT:    v_and_b32_e32 v3, v4, v8
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, v3, v2
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v2i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v6, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GFX9-NEXT:    s_sub_i32 s4, 0, 24
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v8, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GFX9-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GFX9-NEXT:    v_and_b32_e32 v4, 0xffffff, v4
+; GFX9-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0xffffff
+; GFX9-NEXT:    v_mul_lo_u32 v7, s4, v6
+; GFX9-NEXT:    v_and_b32_e32 v5, v5, v9
+; GFX9-NEXT:    v_and_b32_e32 v2, v2, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GFX9-NEXT:    v_and_b32_e32 v3, v3, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
+; GFX9-NEXT:    v_add_u32_e32 v6, v6, v7
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v7, v8
+; GFX9-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GFX9-NEXT:    v_mul_lo_u32 v8, s4, v7
+; GFX9-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX9-NEXT:    v_mul_hi_u32 v8, v7, v8
+; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v6
+; GFX9-NEXT:    v_subrev_u32_e32 v6, 24, v4
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX9-NEXT:    v_add_u32_e32 v7, v7, v8
+; GFX9-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v6, 24, v4
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v7, v7, 24
+; GFX9-NEXT:    v_sub_u32_e32 v6, 23, v4
+; GFX9-NEXT:    v_and_b32_e32 v6, v6, v9
+; GFX9-NEXT:    v_and_b32_e32 v4, v4, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, v6, v2
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, v4, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, v5, v7
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 24, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 24, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v4, 23, v2
+; GFX9-NEXT:    v_and_b32_e32 v4, v4, v9
+; GFX9-NEXT:    v_and_b32_e32 v2, v2, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, v4, v3
+; GFX9-NEXT:    v_lshl_or_b32 v1, v1, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v2i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v6, 24
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v7, 24
+; GFX10-NEXT:    s_sub_i32 s4, 0, 24
+; GFX10-NEXT:    v_mov_b32_e32 v12, 0xffffff
+; GFX10-NEXT:    v_and_b32_e32 v4, 0xffffff, v4
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; GFX10-NEXT:    v_and_b32_e32 v5, v5, v12
+; GFX10-NEXT:    v_and_b32_e32 v2, v2, v12
+; GFX10-NEXT:    v_and_b32_e32 v3, v3, v12
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX10-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX10-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GFX10-NEXT:    v_mul_lo_u32 v8, s4, v6
+; GFX10-NEXT:    v_mul_lo_u32 v9, s4, v7
+; GFX10-NEXT:    v_mul_hi_u32 v8, v6, v8
+; GFX10-NEXT:    v_mul_hi_u32 v9, v7, v9
+; GFX10-NEXT:    v_add_nc_u32_e32 v6, v6, v8
+; GFX10-NEXT:    v_add_nc_u32_e32 v7, v7, v9
+; GFX10-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GFX10-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GFX10-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX10-NEXT:    v_mul_lo_u32 v7, v7, 24
+; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v6
+; GFX10-NEXT:    v_sub_nc_u32_e32 v5, v5, v7
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, 24, v4
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, 24, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v5
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, 24, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, 24, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v4, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v5
+; GFX10-NEXT:    v_sub_nc_u32_e32 v6, 23, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v11, v6, v12
+; GFX10-NEXT:    v_sub_nc_u32_e32 v7, 23, v5
+; GFX10-NEXT:    v_and_b32_e32 v10, v5, v12
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, v11, v2
+; GFX10-NEXT:    v_and_b32_e32 v6, v7, v12
+; GFX10-NEXT:    v_and_b32_e32 v7, v15, v12
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, v6, v3
+; GFX10-NEXT:    v_lshl_or_b32 v0, v0, v7, v2
+; GFX10-NEXT:    v_lshl_or_b32 v1, v1, v10, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i24> @llvm.fshl.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
+  ret <2 x i24> %result
+}
+
+define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
+; GFX6-LABEL: s_fshl_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    s_not_b32 s1, s2
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    s_not_b32 s1, s2
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    s_not_b32 s1, s2
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s1, 1
+; GFX10-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX10-NEXT:    s_not_b32 s1, s2
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, v0, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  ret i32 %result
+}
+
+define amdgpu_ps i32 @s_fshl_i32_5(i32 inreg %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: s_fshl_i32_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, -5
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i32_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, -5
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i32_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, -5
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i32_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s1, -5
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 5)
+  ret i32 %result
+}
+
+define amdgpu_ps i32 @s_fshl_i32_8(i32 inreg %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: s_fshl_i32_8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, -8
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i32_8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, -8
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i32_8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, -8
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i32_8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s1, -8
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 8)
+  ret i32 %result
+}
+
+define i32 @v_fshl_i32(i32 %lhs, i32 %rhs, i32 %amt) {
+; GFX6-LABEL: v_fshl_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v1, v0, v1, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v1, v0, v1, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v1, v0, v1, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v1, v0, v1, 1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v1, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  ret i32 %result
+}
+
+define i32 @v_fshl_i32_5(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_fshl_i32_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v1, -5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i32_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v1, -5
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i32_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v1, -5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i32_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v1, -5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 5)
+  ret i32 %result
+}
+
+define i32 @v_fshl_i32_8(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_fshl_i32_8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v1, -8
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i32_8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v1, -8
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i32_8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v1, -8
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i32_8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v1, -8
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 8)
+  ret i32 %result
+}
+
+define amdgpu_ps float @v_fshl_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt) {
+; GFX6-LABEL: v_fshl_i32_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_alignbit_b32 v1, s0, v1, 1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v1, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i32_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_alignbit_b32 v1, s0, v1, 1
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v1, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i32_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_alignbit_b32 v1, s0, v1, 1
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX9-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v1, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i32_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v1, s0, s1, 1
+; GFX10-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX10-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, v1, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  %cast.result = bitcast i32 %result to float
+  ret float %cast.result
+}
+
+define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt) {
+; GFX6-LABEL: v_fshl_i32_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_not_b32 s1, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i32_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_not_b32 s1, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i32_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_not_b32 s1, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i32_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX10-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX10-NEXT:    s_not_b32 s1, s1
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, v0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  %cast.result = bitcast i32 %result to float
+  ret float %cast.result
+}
+
+define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
+; GFX6-LABEL: v_fshl_i32_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    s_not_b32 s1, s2
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i32_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    s_not_b32 s1, s2
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i32_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    s_not_b32 s1, s2
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, 1
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i32_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s1, 1
+; GFX10-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX10-NEXT:    s_not_b32 s1, s2
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, v0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  %cast.result = bitcast i32 %result to float
+  ret float %cast.result
+}
+
+define <2 x i32> @v_fshl_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
+; GFX6-LABEL: v_fshl_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v2, v0, v2, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v2, v4
+; GFX6-NEXT:    v_alignbit_b32 v2, v1, v3, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_xor_b32_e32 v3, -1, v5
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v2, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v2, v0, v2, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v2, v4
+; GFX8-NEXT:    v_alignbit_b32 v2, v1, v3, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v5
+; GFX8-NEXT:    v_alignbit_b32 v1, v1, v2, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v2, v0, v2, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v2, v4
+; GFX9-NEXT:    v_alignbit_b32 v2, v1, v3, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_xor_b32_e32 v3, -1, v5
+; GFX9-NEXT:    v_alignbit_b32 v1, v1, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v2, v0, v2, 1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX10-NEXT:    v_alignbit_b32 v3, v1, v3, 1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 1, v1
+; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v5
+; GFX10-NEXT:    v_alignbit_b32 v0, v7, v2, v4
+; GFX10-NEXT:    v_alignbit_b32 v1, v6, v3, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt)
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
+; GFX6-LABEL: v_fshl_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v3, v0, v3, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v6
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v3, v6
+; GFX6-NEXT:    v_alignbit_b32 v3, v1, v4, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v7
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v3, v4
+; GFX6-NEXT:    v_alignbit_b32 v3, v2, v5, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v8
+; GFX6-NEXT:    v_alignbit_b32 v2, v2, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v3, v0, v3, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_xor_b32_e32 v6, -1, v6
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v3, v6
+; GFX8-NEXT:    v_alignbit_b32 v3, v1, v4, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v7
+; GFX8-NEXT:    v_alignbit_b32 v1, v1, v3, v4
+; GFX8-NEXT:    v_alignbit_b32 v3, v2, v5, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v8
+; GFX8-NEXT:    v_alignbit_b32 v2, v2, v3, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v3, v0, v3, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_xor_b32_e32 v6, -1, v6
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v3, v6
+; GFX9-NEXT:    v_alignbit_b32 v3, v1, v4, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v7
+; GFX9-NEXT:    v_alignbit_b32 v1, v1, v3, v4
+; GFX9-NEXT:    v_alignbit_b32 v3, v2, v5, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v8
+; GFX9-NEXT:    v_alignbit_b32 v2, v2, v3, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v3, v0, v3, 1
+; GFX10-NEXT:    v_alignbit_b32 v4, v1, v4, 1
+; GFX10-NEXT:    v_alignbit_b32 v5, v2, v5, 1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v6, -1, v6
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_xor_b32_e32 v7, -1, v7
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v3, v6
+; GFX10-NEXT:    v_alignbit_b32 v1, v1, v4, v7
+; GFX10-NEXT:    v_alignbit_b32 v2, v2, v5, v8
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i32> @llvm.fshl.v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt)
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @v_fshl_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
+; GFX6-LABEL: v_fshl_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v4, v0, v4, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v4, v8
+; GFX6-NEXT:    v_alignbit_b32 v4, v1, v5, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v9
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v4, v5
+; GFX6-NEXT:    v_alignbit_b32 v4, v2, v6, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v10
+; GFX6-NEXT:    v_alignbit_b32 v2, v2, v4, v5
+; GFX6-NEXT:    v_alignbit_b32 v4, v3, v7, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v11
+; GFX6-NEXT:    v_alignbit_b32 v3, v3, v4, v5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v4, v0, v4, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v4, v8
+; GFX8-NEXT:    v_alignbit_b32 v4, v1, v5, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v9
+; GFX8-NEXT:    v_alignbit_b32 v1, v1, v4, v5
+; GFX8-NEXT:    v_alignbit_b32 v4, v2, v6, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v10
+; GFX8-NEXT:    v_alignbit_b32 v2, v2, v4, v5
+; GFX8-NEXT:    v_alignbit_b32 v4, v3, v7, 1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v11
+; GFX8-NEXT:    v_alignbit_b32 v3, v3, v4, v5
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v4, v0, v4, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v4, v8
+; GFX9-NEXT:    v_alignbit_b32 v4, v1, v5, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v9
+; GFX9-NEXT:    v_alignbit_b32 v1, v1, v4, v5
+; GFX9-NEXT:    v_alignbit_b32 v4, v2, v6, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v10
+; GFX9-NEXT:    v_alignbit_b32 v2, v2, v4, v5
+; GFX9-NEXT:    v_alignbit_b32 v4, v3, v7, 1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v11
+; GFX9-NEXT:    v_alignbit_b32 v3, v3, v4, v5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v22, v1, v5, 1
+; GFX10-NEXT:    v_alignbit_b32 v18, v0, v4, 1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 1, v1
+; GFX10-NEXT:    v_xor_b32_e32 v9, -1, v9
+; GFX10-NEXT:    v_alignbit_b32 v5, v2, v6, 1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v23, 1, v2
+; GFX10-NEXT:    v_xor_b32_e32 v10, -1, v10
+; GFX10-NEXT:    v_alignbit_b32 v13, v3, v7, 1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 1, v3
+; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v11
+; GFX10-NEXT:    v_alignbit_b32 v0, v15, v18, v8
+; GFX10-NEXT:    v_alignbit_b32 v1, v19, v22, v9
+; GFX10-NEXT:    v_alignbit_b32 v2, v23, v5, v10
+; GFX10-NEXT:    v_alignbit_b32 v3, v14, v13, v11
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt)
+  ret <4 x i32> %result
+}
+
+define amdgpu_ps i16 @s_fshl_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt) {
+; GFX6-LABEL: s_fshl_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s3, s2, 15
+; GFX6-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s3, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, 1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s3, s2, 15
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s3, 1, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s3, s2, 15
+; GFX10-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s4
+; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  ret i16 %result
+}
+
+define amdgpu_ps i16 @s_fshl_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: s_fshl_i16_4:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 12
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i16_4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 4, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, 12, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i16_4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 4, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, 12, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i16_4:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 4, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, 12, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 4)
+  ret i16 %result
+}
+
+define amdgpu_ps i16 @s_fshl_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: s_fshl_i16_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 5
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 11
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i16_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 5, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, 11, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i16_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 5, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, 11, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i16_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 5, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, 11, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 5)
+  ret i16 %result
+}
+
+define i16 @v_fshl_i16(i16 %lhs, i16 %rhs, i16 %amt) {
+; GFX6-LABEL: v_fshl_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v3, 15, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT:    v_bfe_u32 v3, v3, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v3, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v3, 15, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 1, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v3, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v3, 15, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, 1, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, v3, v0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v3, -1, v2
+; GFX10-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 1, v1
+; GFX10-NEXT:    v_and_b32_e32 v3, 15, v3
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v2, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v3, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  ret i16 %result
+}
+
+define i16 @v_fshl_i16_4(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_fshl_i16_4:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 12, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i16_4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 4, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 12, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i16_4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 4, v0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, 12, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i16_4:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 4, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 12, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 4)
+  ret i16 %result
+}
+
+define i16 @v_fshl_i16_5(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_fshl_i16_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 11, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i16_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 5, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 11, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i16_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 5, v0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, 11, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i16_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 5, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 11, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 5)
+  ret i16 %result
+}
+
+define amdgpu_ps half @v_fshl_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt) {
+; GFX6-LABEL: v_fshl_i16_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_and_b32_e32 v1, 15, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX6-NEXT:    v_lshl_b32_e32 v1, s0, v1
+; GFX6-NEXT:    s_and_b32 s0, s1, 0xffff
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshr_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i16_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_and_b32_e32 v1, 15, v0
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v1, v1, s0
+; GFX8-NEXT:    s_bfe_u32 s0, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, 1, 0x100000
+; GFX8-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX8-NEXT:    v_lshrrev_b16_e64 v0, v0, s0
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i16_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_and_b32_e32 v1, 15, v0
+; GFX9-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT:    v_lshlrev_b16_e64 v1, v1, s0
+; GFX9-NEXT:    s_bfe_u32 s0, s1, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, 1, 0x100000
+; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX9-NEXT:    v_lshrrev_b16_e64 v0, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i16_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s2, 1, 0x100000
+; GFX10-NEXT:    v_and_b32_e32 v1, 15, v1
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v0, s0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v1, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  %cast.result = bitcast i16 %result to half
+  ret half %cast.result
+}
+
+define amdgpu_ps half @v_fshl_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt) {
+; GFX6-LABEL: v_fshl_i16_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s2, s1, 15
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, s1, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i16_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s2, s1, 15
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 1, v0
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, s1, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i16_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s2, s1, 15
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 1, v0
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, s1, v0
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i16_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 1, v0
+; GFX10-NEXT:    s_andn2_b32 s2, 15, s1
+; GFX10-NEXT:    s_and_b32 s1, s1, 15
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, s2, v0
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  %cast.result = bitcast i16 %result to half
+  ret half %cast.result
+}
+
+define amdgpu_ps half @v_fshl_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt) {
+; GFX6-LABEL: v_fshl_i16_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s2, s1, 15
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX6-NEXT:    s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, s2, v0
+; GFX6-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i16_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s2, s1, 15
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, s2, v0
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, 1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i16_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s2, s1, 15
+; GFX9-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, s2, v0
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, 1, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i16_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s2, s1, 15
+; GFX10-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, 1, 0x100000
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, s2, v0
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s3
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  %cast.result = bitcast i16 %result to half
+  ret half %cast.result
+}
+
+define amdgpu_ps i32 @s_fshl_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <2 x i16> inreg %amt) {
+; GFX6-LABEL: s_fshl_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s5, s2, 15
+; GFX6-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX6-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX6-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX6-NEXT:    s_and_b32 s5, s1, 0xffff
+; GFX6-NEXT:    s_lshr_b32 s5, s5, 1
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s2, s5, s2
+; GFX6-NEXT:    s_or_b32 s0, s0, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, 15
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_andn2_b32 s4, 15, s4
+; GFX6-NEXT:    s_lshl_b32 s2, s3, s2
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 17
+; GFX6-NEXT:    s_bfe_u32 s3, s4, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX6-NEXT:    s_or_b32 s1, s2, s1
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s6, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX8-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s6
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s6, 1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s6
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s5, 15
+; GFX8-NEXT:    s_andn2_b32 s2, 15, s5
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s3, s1
+; GFX8-NEXT:    s_lshr_b32 s3, s4, s6
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s3, s2
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000f
+; GFX9-NEXT:    s_and_b32 s4, s2, s3
+; GFX9-NEXT:    s_andn2_b32 s2, s3, s2
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s3, s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_and_b32 s1, s1, s4
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX9-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX9-NEXT:    s_and_b32 s1, s1, s4
+; GFX9-NEXT:    s_and_b32 s2, s2, s4
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_lshr_b32 s2, s3, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s5, 0xffff
+; GFX10-NEXT:    s_mov_b32 s3, 0xf000f
+; GFX10-NEXT:    s_and_b32 s7, s1, s5
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s7, s7, 1
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX10-NEXT:    s_and_b32 s4, s2, s3
+; GFX10-NEXT:    s_andn2_b32 s2, s3, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s7, s1
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX10-NEXT:    s_and_b32 s1, s1, s5
+; GFX10-NEXT:    s_and_b32 s5, s2, s5
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s6
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s5
+; GFX10-NEXT:    s_lshr_b32 s2, s4, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  %cast = bitcast <2 x i16> %result to i32
+  ret i32 %cast
+}
+
+define <2 x i16> @v_fshl_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
+; GFX6-LABEL: v_fshl_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v5, 15, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_bfe_u32 v5, v5, 0, 16
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v5, v0
+; GFX6-NEXT:    v_and_b32_e32 v5, 0xffff, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 1, v5
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, v2, v5
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v4
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 17, v1
+; GFX6-NEXT:    v_bfe_u32 v3, v4, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v3, v1
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v4, 15, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX8-NEXT:    v_lshrrev_b16_e32 v5, 1, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v4, v4, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, v2, v5
+; GFX8-NEXT:    v_or_b32_e32 v2, v4, v2
+; GFX8-NEXT:    v_and_b32_e32 v4, 15, v3
+; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v3
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v4, 1
+; GFX8-NEXT:    v_and_b32_e32 v3, 15, v3
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, v3, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 0xf000f
+; GFX9-NEXT:    v_and_b32_e32 v3, s4, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX9-NEXT:    v_pk_lshrrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, v3, v0
+; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v3, -1, v2
+; GFX10-NEXT:    s_mov_b32 s4, 0xf000f
+; GFX10-NEXT:    v_pk_lshrrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX10-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX10-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, v2, v0
+; GFX10-NEXT:    v_pk_lshrrev_b16 v1, v3, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  ret <2 x i16> %result
+}
+
+define <2 x i16> @v_fshl_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_fshl_v2i16_4_8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v3, 0xffff, v1
+; GFX6-NEXT:    s_bfe_u32 s4, 4, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
+; GFX6-NEXT:    s_bfe_u32 s4, 11, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, s4, v3
+; GFX6-NEXT:    s_bfe_u32 s4, 8, 0x100000
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 17, v1
+; GFX6-NEXT:    s_bfe_u32 s4, 7, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v2i16_4_8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b16_e32 v3, 1, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 4, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v3, 11, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT:    v_mov_b32_e32 v3, 1
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 7, v1
+; GFX8-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v2i16_4_8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v2, 16
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v3, 16
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX9-NEXT:    s_sub_i32 s4, 0, 16
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, s4, v2
+; GFX9-NEXT:    v_mul_lo_u32 v5, s4, v3
+; GFX9-NEXT:    v_mul_hi_u32 v4, v2, v4
+; GFX9-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_mul_hi_u32 v2, 4, v2
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v5
+; GFX9-NEXT:    v_mul_hi_u32 v3, 8, v3
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, 4, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 16, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 16, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 4, v3
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 16, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 16, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 8, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 16, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 16, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 16, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 16, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v4, v3
+; GFX9-NEXT:    v_pk_sub_i16 v3, 16, v2 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, v2, v0
+; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v3, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v2i16_4_8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v2, 16
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v3, 16
+; GFX10-NEXT:    s_sub_i32 s4, 0, 16
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GFX10-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_lo_u32 v4, s4, v2
+; GFX10-NEXT:    v_mul_lo_u32 v5, s4, v3
+; GFX10-NEXT:    v_mul_hi_u32 v4, v2, v4
+; GFX10-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v5
+; GFX10-NEXT:    v_mul_hi_u32 v2, 8, v2
+; GFX10-NEXT:    v_mul_hi_u32 v3, 4, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 4, v3
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 8, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 4, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v4, 16, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 16, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v5, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 16, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v4, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 16, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v5, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX10-NEXT:    v_and_or_b32 v2, v3, 0xffff, v2
+; GFX10-NEXT:    v_pk_sub_i16 v3, 16, v2 op_sel_hi:[0,1]
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, v2, v0
+; GFX10-NEXT:    v_pk_lshrrev_b16 v1, v3, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> <i16 4, i16 8>)
+  ret <2 x i16> %result
+}
+
+define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <2 x i16> %amt) {
+; GFX6-LABEL: v_fshl_v2i16_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v0
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX6-NEXT:    v_lshl_b32_e32 v2, s0, v2
+; GFX6-NEXT:    s_and_b32 s0, s1, 0xffff
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshr_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v1
+; GFX6-NEXT:    v_xor_b32_e32 v1, -1, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, 15, v1
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    s_lshr_b32 s0, s1, 17
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_lshl_b32_e32 v2, s2, v2
+; GFX6-NEXT:    v_lshr_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_v2i16_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_and_b32_e32 v2, 15, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    v_lshlrev_b16_e64 v2, v2, s0
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_bfe_u32 s0, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, 1, 0x100000
+; GFX8-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX8-NEXT:    v_lshrrev_b16_e64 v0, v0, s0
+; GFX8-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT:    v_and_b32_e32 v2, 15, v1
+; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v1
+; GFX8-NEXT:    v_and_b32_e32 v1, 15, v1
+; GFX8-NEXT:    s_lshr_b32 s0, s3, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v2, v2, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v1, v1, s0
+; GFX8-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_v2i16_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX9-NEXT:    v_and_b32_e32 v1, s2, v0
+; GFX9-NEXT:    v_pk_lshlrev_b16 v1, v1, s0
+; GFX9-NEXT:    s_lshr_b32 s0, s1, 16
+; GFX9-NEXT:    s_and_b32 s1, s1, 0xffff
+; GFX9-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX9-NEXT:    v_and_b32_e32 v0, s2, v0
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s1, s0
+; GFX9-NEXT:    v_pk_lshrrev_b16 v0, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_v2i16_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX10-NEXT:    s_and_b32 s1, s1, 0xffff
+; GFX10-NEXT:    v_and_b32_e32 v0, s2, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, s2, v1
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX10-NEXT:    s_lshr_b32 s2, s3, 1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, v0, s0
+; GFX10-NEXT:    v_pk_lshrrev_b16 v1, v1, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <2 x i16> inreg %amt) {
+; GFX6-LABEL: v_fshl_v2i16_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v0
+; GFX6-NEXT:    s_and_b32 s4, s1, 15
+; GFX6-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX6-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, s1, v1
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX6-NEXT:    v_or_b32_e32 v1, s0, v1
+; GFX6-NEXT:    s_and_b32 s0, s3, 15
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s3
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 17, v0
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s0, s2, s0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, s1, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_v2i16_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s4, s1, 15
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 1, v0
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, s1, v1
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX8-NEXT:    v_or_b32_e32 v1, s0, v1
+; GFX8-NEXT:    s_and_b32 s0, s3, 15
+; GFX8-NEXT:    v_mov_b32_e32 v2, 1
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s3
+; GFX8-NEXT:    s_lshl_b32 s0, s2, s0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, s1, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_v2i16_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX9-NEXT:    s_and_b32 s3, s1, s2
+; GFX9-NEXT:    s_andn2_b32 s1, s2, s1
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX9-NEXT:    v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX9-NEXT:    s_lshl_b32 s2, s2, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    v_pk_lshrrev_b16 v0, s1, v0
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_v2i16_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX10-NEXT:    v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX10-NEXT:    s_and_b32 s3, s1, s2
+; GFX10-NEXT:    s_andn2_b32 s1, s2, s1
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX10-NEXT:    v_pk_lshrrev_b16 v0, s1, v0
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <2 x i16> inreg %amt) {
+; GFX6-LABEL: v_fshl_v2i16_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s3, s1, 15
+; GFX6-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT:    s_lshr_b32 s2, s1, 16
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, s3, v0
+; GFX6-NEXT:    s_and_b32 s3, s0, 0xffff
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s1, s3, s1
+; GFX6-NEXT:    v_or_b32_e32 v0, s1, v0
+; GFX6-NEXT:    s_and_b32 s1, s2, 15
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, s1, v1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 17
+; GFX6-NEXT:    s_bfe_u32 s1, s2, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX6-NEXT:    v_or_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_v2i16_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s4, s1, 15
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, s4, v0
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s3
+; GFX8-NEXT:    v_or_b32_e32 v1, s0, v1
+; GFX8-NEXT:    s_and_b32 s0, s3, 15
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_lshr_b32 s0, s2, s4
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_v2i16_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX9-NEXT:    s_and_b32 s3, s1, s2
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, s3, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xffff
+; GFX9-NEXT:    s_andn2_b32 s1, s2, s1
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_and_b32 s0, s0, s3
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX9-NEXT:    s_lshr_b32 s2, s2, 1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX9-NEXT:    s_and_b32 s0, s0, s3
+; GFX9-NEXT:    s_and_b32 s1, s1, s3
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX9-NEXT:    s_lshr_b32 s1, s2, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_v2i16_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s3, 0xffff
+; GFX10-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX10-NEXT:    s_and_b32 s5, s0, s3
+; GFX10-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s5, s5, 1
+; GFX10-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX10-NEXT:    s_and_b32 s4, s1, s2
+; GFX10-NEXT:    s_andn2_b32 s1, s2, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s5, s0
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, s4, v0
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX10-NEXT:    s_and_b32 s0, s0, s3
+; GFX10-NEXT:    s_and_b32 s3, s1, s3
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s3
+; GFX10-NEXT:    s_lshr_b32 s1, s2, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+; ; FIXME
+; define amdgpu_ps i48 @s_fshl_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <3 x i16> inreg %amt) {
+;   %result = call <3 x i16> @llvm.fshl.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
+;   %cast = bitcast <3 x i16> %result to i48
+;   ret i48 %cast
+; }
+
+; ; FIXME
+; define <3 x half> @v_fshl_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt) {
+;   %result = call <3 x i16> @llvm.fshl.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
+;   %cast.result = bitcast <3 x i16> %result to <3 x half>
+;   ret <3 x half> %cast.result
+; }
+
+define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs, <4 x i16> inreg %amt) {
+; GFX6-LABEL: s_fshl_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s12, s8, 15
+; GFX6-NEXT:    s_bfe_u32 s12, s12, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s12
+; GFX6-NEXT:    s_mov_b32 s12, 0xffff
+; GFX6-NEXT:    s_andn2_b32 s8, 15, s8
+; GFX6-NEXT:    s_and_b32 s4, s4, s12
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX6-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s4, s4, s8
+; GFX6-NEXT:    s_or_b32 s0, s0, s4
+; GFX6-NEXT:    s_and_b32 s4, s9, 15
+; GFX6-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX6-NEXT:    s_and_b32 s4, s5, s12
+; GFX6-NEXT:    s_andn2_b32 s8, 15, s9
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX6-NEXT:    s_bfe_u32 s5, s8, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s4, s4, s5
+; GFX6-NEXT:    s_or_b32 s1, s1, s4
+; GFX6-NEXT:    s_and_b32 s4, s10, 15
+; GFX6-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX6-NEXT:    s_andn2_b32 s5, 15, s10
+; GFX6-NEXT:    s_lshl_b32 s2, s2, s4
+; GFX6-NEXT:    s_and_b32 s4, s6, s12
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX6-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s4, s4, s5
+; GFX6-NEXT:    s_or_b32 s2, s2, s4
+; GFX6-NEXT:    s_and_b32 s4, s11, 15
+; GFX6-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX6-NEXT:    s_andn2_b32 s5, 15, s11
+; GFX6-NEXT:    s_lshl_b32 s3, s3, s4
+; GFX6-NEXT:    s_and_b32 s4, s7, s12
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX6-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s4, s4, s5
+; GFX6-NEXT:    s_or_b32 s3, s3, s4
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_bfe_u32 s1, s2, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s2, s3, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s12, s4, 15
+; GFX8-NEXT:    s_bfe_u32 s12, s12, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX8-NEXT:    s_andn2_b32 s4, 15, s4
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s12
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s12, 1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s2, s12
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s2, s4
+; GFX8-NEXT:    s_or_b32 s0, s0, s2
+; GFX8-NEXT:    s_and_b32 s2, s10, 15
+; GFX8-NEXT:    s_andn2_b32 s4, 15, s10
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s6, s2
+; GFX8-NEXT:    s_lshr_b32 s6, s8, s12
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s4, s6, s4
+; GFX8-NEXT:    s_or_b32 s2, s2, s4
+; GFX8-NEXT:    s_and_b32 s4, s5, 15
+; GFX8-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
+; GFX8-NEXT:    s_andn2_b32 s5, 15, s5
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s5, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s12
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s4
+; GFX8-NEXT:    s_or_b32 s1, s1, s3
+; GFX8-NEXT:    s_and_b32 s3, s11, 15
+; GFX8-NEXT:    s_andn2_b32 s4, 15, s11
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s5, s9, s12
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s7, s3
+; GFX8-NEXT:    s_lshr_b32 s4, s5, s4
+; GFX8-NEXT:    s_or_b32 s3, s3, s4
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s2, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s6, 0xf000f
+; GFX9-NEXT:    s_and_b32 s7, s4, s6
+; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s7, 16
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s7
+; GFX9-NEXT:    s_lshl_b32 s7, s8, s9
+; GFX9-NEXT:    s_mov_b32 s8, 0xffff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
+; GFX9-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX9-NEXT:    s_and_b32 s2, s2, s8
+; GFX9-NEXT:    s_lshr_b32 s2, s2, 1
+; GFX9-NEXT:    s_lshr_b32 s7, s7, 1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s7
+; GFX9-NEXT:    s_andn2_b32 s4, s6, s4
+; GFX9-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s4, 16
+; GFX9-NEXT:    s_and_b32 s2, s2, s8
+; GFX9-NEXT:    s_and_b32 s4, s4, s8
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s7, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    s_or_b32 s0, s0, s2
+; GFX9-NEXT:    s_and_b32 s2, s5, s6
+; GFX9-NEXT:    s_andn2_b32 s4, s6, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_lshl_b32 s2, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    s_lshr_b32 s2, s3, 16
+; GFX9-NEXT:    s_and_b32 s3, s3, s8
+; GFX9-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX9-NEXT:    s_lshr_b32 s2, s2, 1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
+; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_and_b32 s2, s2, s8
+; GFX9-NEXT:    s_and_b32 s4, s4, s8
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s4
+; GFX9-NEXT:    s_lshr_b32 s3, s3, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    s_or_b32 s1, s1, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s9, 0xffff
+; GFX10-NEXT:    s_mov_b32 s6, 0xf000f
+; GFX10-NEXT:    s_and_b32 s11, s2, s9
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX10-NEXT:    s_and_b32 s7, s4, s6
+; GFX10-NEXT:    s_lshr_b32 s11, s11, 1
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 1
+; GFX10-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX10-NEXT:    s_andn2_b32 s4, s6, s4
+; GFX10-NEXT:    s_lshr_b32 s10, s7, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s11, s2
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s7
+; GFX10-NEXT:    s_lshl_b32 s7, s8, s10
+; GFX10-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX10-NEXT:    s_and_b32 s10, s4, s9
+; GFX10-NEXT:    s_and_b32 s2, s2, s9
+; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX10-NEXT:    s_lshr_b32 s2, s2, s10
+; GFX10-NEXT:    s_lshr_b32 s4, s8, s4
+; GFX10-NEXT:    s_and_b32 s8, s3, s9
+; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX10-NEXT:    s_and_b32 s4, s5, s6
+; GFX10-NEXT:    s_lshr_b32 s8, s8, 1
+; GFX10-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
+; GFX10-NEXT:    s_andn2_b32 s5, s6, s5
+; GFX10-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX10-NEXT:    s_lshl_b32 s4, s6, s7
+; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX10-NEXT:    s_and_b32 s7, s5, s9
+; GFX10-NEXT:    s_and_b32 s3, s3, s9
+; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX10-NEXT:    s_lshr_b32 s3, s3, s7
+; GFX10-NEXT:    s_lshr_b32 s5, s6, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX10-NEXT:    s_or_b32 s0, s0, s2
+; GFX10-NEXT:    s_or_b32 s1, s1, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i16> @llvm.fshl.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
+  %cast.result = bitcast <4 x i16> %result to <2 x i32>
+  ret <2 x i32> %cast.result
+}
+
+define <4 x half> @v_fshl_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt) {
+; GFX6-LABEL: v_fshl_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v12, 15, v8
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v8, 15, v8
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX6-NEXT:    v_bfe_u32 v12, v12, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_bfe_u32 v8, v8, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v12, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v8, v4
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, 15, v9
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v9
+; GFX6-NEXT:    v_bfe_u32 v4, v4, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, v4, v1
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v5
+; GFX6-NEXT:    v_and_b32_e32 v8, 15, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_bfe_u32 v5, v8, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v5, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, 15, v10
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v10
+; GFX6-NEXT:    v_bfe_u32 v4, v4, 0, 16
+; GFX6-NEXT:    v_mov_b32_e32 v12, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v5, 15, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, v4, v2
+; GFX6-NEXT:    v_and_b32_e32 v4, v6, v12
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_bfe_u32 v5, v5, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v5, v4
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, 15, v11
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v11
+; GFX6-NEXT:    v_bfe_u32 v4, v4, 0, 16
+; GFX6-NEXT:    v_and_b32_e32 v5, 15, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, v4, v3
+; GFX6-NEXT:    v_and_b32_e32 v4, v7, v12
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_bfe_u32 v5, v5, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v5, v4
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v8, 15, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX8-NEXT:    v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT:    v_lshrrev_b16_e32 v9, 1, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v8, v8, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v4, v4, v9
+; GFX8-NEXT:    v_or_b32_e32 v4, v8, v4
+; GFX8-NEXT:    v_and_b32_e32 v8, 15, v6
+; GFX8-NEXT:    v_xor_b32_e32 v6, -1, v6
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v8, 1
+; GFX8-NEXT:    v_and_b32_e32 v6, 15, v6
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, v6, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, 15, v5
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v5
+; GFX8-NEXT:    v_and_b32_e32 v5, 15, v5
+; GFX8-NEXT:    v_lshrrev_b16_e32 v6, 1, v3
+; GFX8-NEXT:    v_lshrrev_b16_e32 v5, v5, v6
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, v2, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v5
+; GFX8-NEXT:    v_and_b32_e32 v5, 15, v7
+; GFX8-NEXT:    v_xor_b32_e32 v6, -1, v7
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v5, 1
+; GFX8-NEXT:    v_and_b32_e32 v6, 15, v6
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v3, v6, v3
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    v_mov_b32_e32 v3, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 0xf000f
+; GFX9-NEXT:    v_and_b32_e32 v6, s4, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX9-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v4, v2
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, v6, v0
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v5
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, s4, v5
+; GFX9-NEXT:    v_pk_lshlrev_b16 v1, v2, v1
+; GFX9-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, 1, v3 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v4, v2
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v6, -1, v4
+; GFX10-NEXT:    v_xor_b32_e32 v7, -1, v5
+; GFX10-NEXT:    s_mov_b32 s4, 0xf000f
+; GFX10-NEXT:    v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
+; GFX10-NEXT:    v_and_b32_e32 v11, s4, v4
+; GFX10-NEXT:    v_and_b32_e32 v15, s4, v6
+; GFX10-NEXT:    v_and_b32_e32 v19, s4, v5
+; GFX10-NEXT:    v_and_b32_e32 v6, s4, v7
+; GFX10-NEXT:    v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, v11, v0
+; GFX10-NEXT:    v_pk_lshrrev_b16 v2, v15, v2
+; GFX10-NEXT:    v_pk_lshlrev_b16 v1, v19, v1
+; GFX10-NEXT:    v_pk_lshrrev_b16 v3, v6, v3
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.fshl.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
+  %cast.result = bitcast <4 x i16> %result to <4 x half>
+  ret <4 x half> %cast.result
+}
+
+define amdgpu_ps i64 @s_fshl_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt) {
+; GCN-LABEL: s_fshl_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_and_b64 s[6:7], s[4:5], 63
+; GCN-NEXT:    s_andn2_b64 s[4:5], 63, s[4:5]
+; GCN-NEXT:    s_lshr_b64 s[2:3], s[2:3], 1
+; GCN-NEXT:    s_lshl_b64 s[0:1], s[0:1], s6
+; GCN-NEXT:    s_lshr_b64 s[2:3], s[2:3], s4
+; GCN-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_fshl_i64_5(i64 inreg %lhs, i64 inreg %rhs) {
+; GCN-LABEL: s_fshl_i64_5:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_lshr_b32 s2, s3, 27
+; GCN-NEXT:    s_lshl_b64 s[0:1], s[0:1], 5
+; GCN-NEXT:    s_mov_b32 s3, 0
+; GCN-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 5)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_fshl_i64_32(i64 inreg %lhs, i64 inreg %rhs) {
+; GCN-LABEL: s_fshl_i64_32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_mov_b32 s1, s0
+; GCN-NEXT:    s_mov_b32 s0, 0
+; GCN-NEXT:    s_mov_b32 s2, s3
+; GCN-NEXT:    s_mov_b32 s3, s0
+; GCN-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 32)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_fshl_i64_48(i64 inreg %lhs, i64 inreg %rhs) {
+; GCN-LABEL: s_fshl_i64_48:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_lshl_b32 s1, s0, 16
+; GCN-NEXT:    s_mov_b32 s0, 0
+; GCN-NEXT:    s_lshr_b64 s[2:3], s[2:3], 16
+; GCN-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 48)
+  ret i64 %result
+}
+
+define i64 @v_fshl_i64(i64 %lhs, i64 %rhs, i64 %amt) {
+; GFX6-LABEL: v_fshl_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v5, 63, v4
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[2:3], 1
+; GFX6-NEXT:    v_and_b32_e32 v4, 63, v4
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], v5
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[2:3], v4
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v5, 63, v4
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX8-NEXT:    v_and_b32_e32 v4, 63, v4
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v5, v[0:1]
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], v4, v[2:3]
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v5, 63, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX9-NEXT:    v_and_b32_e32 v4, 63, v4
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v5, v[0:1]
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], v4, v[2:3]
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v4
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX10-NEXT:    v_and_b32_e32 v7, 63, v4
+; GFX10-NEXT:    v_and_b32_e32 v5, 63, v5
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], v7, v[0:1]
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], v5, v[2:3]
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  ret i64 %result
+}
+
+define i64 @v_fshl_i64_5(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_fshl_i64_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 5
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 27, v3
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i64_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], 5, v[0:1]
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 27, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i64_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 5, v[0:1]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 27, v3
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i64_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 5, v[0:1]
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 27, v3
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 5)
+  ret i64 %result
+}
+
+define i64 @v_fshl_i64_32(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_fshl_i64_32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, v0
+; GFX6-NEXT:    v_mov_b32_e32 v0, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i64_32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i64_32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i64_32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v1, v0
+; GFX10-NEXT:    v_mov_b32_e32 v0, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 32)
+  ret i64 %result
+}
+
+define i64 @v_fshl_i64_48(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_fshl_i64_48:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v4, v0
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[2:3], 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i64_48:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v4, v0
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], 16, v[2:3]
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i64_48:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v0
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], 16, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i64_48:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v4, v0
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], 16, v[2:3]
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
+; GFX10-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 48)
+  ret i64 %result
+}
+
+define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64 %amt) {
+; GFX6-LABEL: v_fshl_i64_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_and_b32_e32 v1, 63, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    v_and_b32_e32 v2, 63, v0
+; GFX6-NEXT:    v_lshl_b64 v[0:1], s[0:1], v1
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[2:3], 1
+; GFX6-NEXT:    v_lshr_b64 v[2:3], s[0:1], v2
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i64_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_and_b32_e32 v1, 63, v0
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    v_and_b32_e32 v2, 63, v0
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v1, s[0:1]
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[2:3], 1
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], v2, s[0:1]
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i64_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_and_b32_e32 v1, 63, v0
+; GFX9-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT:    v_and_b32_e32 v2, 63, v0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v1, s[0:1]
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[2:3], 1
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], v2, s[0:1]
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i64_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 63, v0
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT:    v_and_b32_e32 v2, 63, v1
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], v0, s[0:1]
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg %amt) {
+; GFX6-LABEL: v_fshl_i64_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[0:1], 1
+; GFX6-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX6-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[0:1], s2
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s4
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i64_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], 1, v[0:1]
+; GFX8-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX8-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s4
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i64_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], 1, v[0:1]
+; GFX9-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX9-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s4
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i64_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], 1, v[0:1]
+; GFX10-NEXT:    s_andn2_b64 s[4:5], 63, s[2:3]
+; GFX10-NEXT:    s_and_b64 s[2:3], s[2:3], 63
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], s4, v[0:1]
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x float> @v_fshl_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg %amt) {
+; GFX6-LABEL: v_fshl_i64_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX6-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], s4
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i64_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX8-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[0:1], 1
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], s4, v[0:1]
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i64_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX9-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], 1
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], s4, v[0:1]
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i64_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX10-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], s4, v[0:1]
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x i64> @s_fshl_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs, <2 x i64> inreg %amt) {
+; GFX6-LABEL: s_fshl_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b64 s[12:13], s[8:9], 63
+; GFX6-NEXT:    s_andn2_b64 s[8:9], 63, s[8:9]
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[4:5], 1
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[4:5], s8
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s12
+; GFX6-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT:    s_and_b64 s[4:5], s[10:11], 63
+; GFX6-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
+; GFX6-NEXT:    s_andn2_b64 s[8:9], 63, s[10:11]
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[6:7], 1
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[4:5], s8
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b64 s[12:13], s[8:9], 63
+; GFX8-NEXT:    s_andn2_b64 s[8:9], 63, s[8:9]
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[4:5], 1
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[4:5], s8
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s12
+; GFX8-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT:    s_and_b64 s[4:5], s[10:11], 63
+; GFX8-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
+; GFX8-NEXT:    s_andn2_b64 s[8:9], 63, s[10:11]
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[6:7], 1
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[4:5], s8
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b64 s[12:13], s[8:9], 63
+; GFX9-NEXT:    s_andn2_b64 s[8:9], 63, s[8:9]
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], 1
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], s8
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s12
+; GFX9-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX9-NEXT:    s_and_b64 s[4:5], s[10:11], 63
+; GFX9-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
+; GFX9-NEXT:    s_andn2_b64 s[8:9], 63, s[10:11]
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[6:7], 1
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], s8
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b64 s[12:13], s[8:9], 63
+; GFX10-NEXT:    s_andn2_b64 s[8:9], 63, s[8:9]
+; GFX10-NEXT:    s_lshr_b64 s[4:5], s[4:5], 1
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[6:7], 1
+; GFX10-NEXT:    s_lshr_b64 s[4:5], s[4:5], s8
+; GFX10-NEXT:    s_and_b64 s[8:9], s[10:11], 63
+; GFX10-NEXT:    s_andn2_b64 s[10:11], 63, s[10:11]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s12
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[2:3], s8
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[6:7], s10
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
+  ret <2 x i64> %result
+}
+
+define <2 x i64> @v_fshl_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
+; GFX6-LABEL: v_fshl_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v9, 63, v8
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[4:5], 1
+; GFX6-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[4:5], v8
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], v9
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v10
+; GFX6-NEXT:    v_lshr_b64 v[6:7], v[6:7], 1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, 63, v10
+; GFX6-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[2:3], v4
+; GFX6-NEXT:    v_lshr_b64 v[6:7], v[6:7], v8
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v6
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v7
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v9, 63, v8
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], 1, v[4:5]
+; GFX8-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], v8, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v9, v[0:1]
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v10
+; GFX8-NEXT:    v_lshrrev_b64 v[6:7], 1, v[6:7]
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_e32 v4, 63, v10
+; GFX8-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX8-NEXT:    v_lshrrev_b64 v[6:7], v8, v[6:7]
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v6
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v7
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v9, 63, v8
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], 1, v[4:5]
+; GFX9-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], v8, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v9, v[0:1]
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v10
+; GFX9-NEXT:    v_lshrrev_b64 v[6:7], 1, v[6:7]
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX9-NEXT:    v_and_b32_e32 v4, 63, v10
+; GFX9-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX9-NEXT:    v_lshrrev_b64 v[6:7], v8, v[6:7]
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX9-NEXT:    v_or_b32_e32 v2, v2, v6
+; GFX9-NEXT:    v_or_b32_e32 v3, v3, v7
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v9, -1, v8
+; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v10
+; GFX10-NEXT:    v_lshrrev_b64 v[4:5], 1, v[4:5]
+; GFX10-NEXT:    v_lshrrev_b64 v[6:7], 1, v[6:7]
+; GFX10-NEXT:    v_and_b32_e32 v15, 63, v8
+; GFX10-NEXT:    v_and_b32_e32 v19, 63, v9
+; GFX10-NEXT:    v_and_b32_e32 v13, 63, v11
+; GFX10-NEXT:    v_and_b32_e32 v9, 63, v10
+; GFX10-NEXT:    v_lshlrev_b64 v[11:12], v15, v[0:1]
+; GFX10-NEXT:    v_lshrrev_b64 v[4:5], v19, v[4:5]
+; GFX10-NEXT:    v_lshrrev_b64 v[6:7], v13, v[6:7]
+; GFX10-NEXT:    v_lshlrev_b64 v[15:16], v9, v[2:3]
+; GFX10-NEXT:    v_or_b32_e32 v0, v11, v4
+; GFX10-NEXT:    v_or_b32_e32 v1, v12, v5
+; GFX10-NEXT:    v_or_b32_e32 v2, v15, v6
+; GFX10-NEXT:    v_or_b32_e32 v3, v16, v7
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps i128 @s_fshl_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg %amt) {
+; GFX6-LABEL: s_fshl_i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s10, 0x7f
+; GFX6-NEXT:    s_mov_b32 s11, 0
+; GFX6-NEXT:    s_and_b64 s[12:13], s[8:9], s[10:11]
+; GFX6-NEXT:    s_andn2_b64 s[8:9], s[10:11], s[8:9]
+; GFX6-NEXT:    s_sub_i32 s9, s12, 64
+; GFX6-NEXT:    s_sub_i32 s13, 64, s12
+; GFX6-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX6-NEXT:    s_lshl_b64 s[10:11], s[0:1], s12
+; GFX6-NEXT:    s_lshr_b64 s[14:15], s[0:1], s13
+; GFX6-NEXT:    s_lshl_b64 s[12:13], s[2:3], s12
+; GFX6-NEXT:    s_or_b64 s[12:13], s[14:15], s[12:13]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s9
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[12:13], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX6-NEXT:    s_sub_i32 s9, 1, 64
+; GFX6-NEXT:    s_sub_i32 s14, 64, 1
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[12:13], s[4:5], 1
+; GFX6-NEXT:    s_lshl_b64 s[14:15], s[6:7], s14
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[6:7], 1
+; GFX6-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[6:7], s9
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[12:13], s[6:7]
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], 0
+; GFX6-NEXT:    s_sub_i32 s14, s8, 64
+; GFX6-NEXT:    s_sub_i32 s12, 64, s8
+; GFX6-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX6-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[0:1], s8
+; GFX6-NEXT:    s_lshl_b64 s[12:13], s[0:1], s12
+; GFX6-NEXT:    s_lshr_b64 s[8:9], s[4:5], s8
+; GFX6-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[0:1], s14
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[6:7], 0
+; GFX6-NEXT:    s_or_b64 s[0:1], s[10:11], s[0:1]
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s10, 0x7f
+; GFX8-NEXT:    s_mov_b32 s11, 0
+; GFX8-NEXT:    s_and_b64 s[12:13], s[8:9], s[10:11]
+; GFX8-NEXT:    s_andn2_b64 s[8:9], s[10:11], s[8:9]
+; GFX8-NEXT:    s_sub_i32 s9, s12, 64
+; GFX8-NEXT:    s_sub_i32 s13, 64, s12
+; GFX8-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX8-NEXT:    s_lshl_b64 s[10:11], s[0:1], s12
+; GFX8-NEXT:    s_lshr_b64 s[14:15], s[0:1], s13
+; GFX8-NEXT:    s_lshl_b64 s[12:13], s[2:3], s12
+; GFX8-NEXT:    s_or_b64 s[12:13], s[14:15], s[12:13]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s9
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[12:13], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX8-NEXT:    s_sub_i32 s9, 1, 64
+; GFX8-NEXT:    s_sub_i32 s14, 64, 1
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[12:13], s[4:5], 1
+; GFX8-NEXT:    s_lshl_b64 s[14:15], s[6:7], s14
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[6:7], 1
+; GFX8-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[6:7], s9
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[12:13], s[6:7]
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], 0
+; GFX8-NEXT:    s_sub_i32 s14, s8, 64
+; GFX8-NEXT:    s_sub_i32 s12, 64, s8
+; GFX8-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX8-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[0:1], s8
+; GFX8-NEXT:    s_lshl_b64 s[12:13], s[0:1], s12
+; GFX8-NEXT:    s_lshr_b64 s[8:9], s[4:5], s8
+; GFX8-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[0:1], s14
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[6:7], 0
+; GFX8-NEXT:    s_or_b64 s[0:1], s[10:11], s[0:1]
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s10, 0x7f
+; GFX9-NEXT:    s_mov_b32 s11, 0
+; GFX9-NEXT:    s_and_b64 s[12:13], s[8:9], s[10:11]
+; GFX9-NEXT:    s_andn2_b64 s[8:9], s[10:11], s[8:9]
+; GFX9-NEXT:    s_sub_i32 s9, s12, 64
+; GFX9-NEXT:    s_sub_i32 s13, 64, s12
+; GFX9-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX9-NEXT:    s_lshl_b64 s[10:11], s[0:1], s12
+; GFX9-NEXT:    s_lshr_b64 s[14:15], s[0:1], s13
+; GFX9-NEXT:    s_lshl_b64 s[12:13], s[2:3], s12
+; GFX9-NEXT:    s_or_b64 s[12:13], s[14:15], s[12:13]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s9
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[12:13], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s9, 1, 64
+; GFX9-NEXT:    s_sub_i32 s14, 64, 1
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[12:13], s[4:5], 1
+; GFX9-NEXT:    s_lshl_b64 s[14:15], s[6:7], s14
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[6:7], 1
+; GFX9-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[6:7], s9
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[12:13], s[6:7]
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], 0
+; GFX9-NEXT:    s_sub_i32 s14, s8, 64
+; GFX9-NEXT:    s_sub_i32 s12, 64, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX9-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[0:1], s8
+; GFX9-NEXT:    s_lshl_b64 s[12:13], s[0:1], s12
+; GFX9-NEXT:    s_lshr_b64 s[8:9], s[4:5], s8
+; GFX9-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], s14
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[6:7], 0
+; GFX9-NEXT:    s_or_b64 s[0:1], s[10:11], s[0:1]
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s10, 0x7f
+; GFX10-NEXT:    s_mov_b32 s11, 0
+; GFX10-NEXT:    s_and_b64 s[12:13], s[8:9], s[10:11]
+; GFX10-NEXT:    s_andn2_b64 s[8:9], s[10:11], s[8:9]
+; GFX10-NEXT:    s_sub_i32 s9, s12, 64
+; GFX10-NEXT:    s_sub_i32 s10, 64, s12
+; GFX10-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[14:15], s[2:3], s12
+; GFX10-NEXT:    s_lshr_b64 s[10:11], s[0:1], s10
+; GFX10-NEXT:    s_lshl_b64 s[12:13], s[0:1], s12
+; GFX10-NEXT:    s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s9
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[12:13], s[12:13], 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s9, 1, 64
+; GFX10-NEXT:    s_sub_i32 s10, 64, 1
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[4:5], 1
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[6:7], s10
+; GFX10-NEXT:    s_lshr_b64 s[14:15], s[6:7], 1
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[6:7], s9
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[14:15], 0
+; GFX10-NEXT:    s_sub_i32 s14, s8, 64
+; GFX10-NEXT:    s_sub_i32 s9, 64, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[0:1], s8
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[4:5], s9
+; GFX10-NEXT:    s_lshr_b64 s[8:9], s[4:5], s8
+; GFX10-NEXT:    s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX10-NEXT:    s_lshr_b64 s[4:5], s[4:5], s14
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[8:9], 0
+; GFX10-NEXT:    s_or_b64 s[0:1], s[12:13], s[0:1]
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  ret i128 %result
+}
+
+define i128 @v_fshl_i128(i128 %lhs, i128 %rhs, i128 %amt) {
+; GFX6-LABEL: v_fshl_i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_movk_i32 s4, 0x7f
+; GFX6-NEXT:    v_and_b32_e32 v14, s4, v8
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX6-NEXT:    v_and_b32_e32 v15, s4, v8
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, 64, v14
+; GFX6-NEXT:    v_subrev_i32_e32 v16, vcc, 64, v14
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[0:1], v8
+; GFX6-NEXT:    v_lshl_b64 v[10:11], v[2:3], v14
+; GFX6-NEXT:    v_lshl_b64 v[12:13], v[0:1], v14
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], v16
+; GFX6-NEXT:    v_or_b32_e32 v8, v8, v10
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX6-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX6-NEXT:    v_cndmask_b32_e32 v10, 0, v12, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v14
+; GFX6-NEXT:    s_sub_i32 s4, 1, 64
+; GFX6-NEXT:    s_sub_i32 s5, 64, 1
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    v_cndmask_b32_e32 v12, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v13, v1, v3, vcc
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[4:5], 1
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[6:7], s5
+; GFX6-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[6:7], s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s6
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s7
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[6:7], 1
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s6
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, 64, v15
+; GFX6-NEXT:    v_subrev_i32_e32 v14, vcc, 64, v15
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[0:1], v15
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[2:3], v6
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[2:3], v15
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[2:3], v14
+; GFX6-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v15
+; GFX6-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v15
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT:    v_or_b32_e32 v0, v10, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, v11, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, v12, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v13, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_movk_i32 s4, 0x7f
+; GFX8-NEXT:    v_and_b32_e32 v14, s4, v8
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX8-NEXT:    v_and_b32_e32 v15, s4, v8
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, 64, v14
+; GFX8-NEXT:    v_subrev_u32_e32 v16, vcc, 64, v14
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], v8, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[10:11], v14, v[2:3]
+; GFX8-NEXT:    v_lshlrev_b64 v[12:13], v14, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v16, v[0:1]
+; GFX8-NEXT:    v_or_b32_e32 v8, v8, v10
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX8-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, 0, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v14
+; GFX8-NEXT:    s_sub_i32 s4, 1, 64
+; GFX8-NEXT:    s_sub_i32 s5, 64, 1
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v13, v1, v3, vcc
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], 1, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], s5, v[6:7]
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s4, v[6:7]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s7
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], 1, v[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, 64, v15
+; GFX8-NEXT:    v_subrev_u32_e32 v14, vcc, 64, v15
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], v15, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], v6, v[2:3]
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], v15, v[2:3]
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], v14, v[2:3]
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v15
+; GFX8-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v15
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT:    v_or_b32_e32 v0, v10, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, v11, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, v12, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v13, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7f
+; GFX9-NEXT:    v_and_b32_e32 v14, s4, v8
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX9-NEXT:    v_and_b32_e32 v15, s4, v8
+; GFX9-NEXT:    v_sub_u32_e32 v8, 64, v14
+; GFX9-NEXT:    v_subrev_u32_e32 v16, 64, v14
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v8, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[10:11], v14, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[12:13], v14, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v16, v[0:1]
+; GFX9-NEXT:    v_or_b32_e32 v8, v8, v10
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX9-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v12, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v14
+; GFX9-NEXT:    s_sub_i32 s4, 1, 64
+; GFX9-NEXT:    s_sub_i32 s5, 64, 1
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    v_cndmask_b32_e32 v12, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v13, v1, v3, vcc
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], 1, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], s5, v[6:7]
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s4, v[6:7]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s7
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], 1, v[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v6, 64, v15
+; GFX9-NEXT:    v_subrev_u32_e32 v14, 64, v15
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], v15, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], v6, v[2:3]
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v15, v[2:3]
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], v14, v[2:3]
+; GFX9-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v15
+; GFX9-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v15
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT:    v_or_b32_e32 v0, v10, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, v11, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, v12, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v13, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v9, -1, v8
+; GFX10-NEXT:    s_movk_i32 s4, 0x7f
+; GFX10-NEXT:    v_mov_b32_e32 v27, v2
+; GFX10-NEXT:    v_and_b32_e32 v18, s4, v8
+; GFX10-NEXT:    v_mov_b32_e32 v28, v3
+; GFX10-NEXT:    v_and_b32_e32 v19, s4, v9
+; GFX10-NEXT:    s_sub_i32 s4, 64, 1
+; GFX10-NEXT:    v_lshrrev_b64 v[8:9], 1, v[4:5]
+; GFX10-NEXT:    v_lshlrev_b64 v[10:11], s4, v[6:7]
+; GFX10-NEXT:    s_sub_i32 s4, 1, 64
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[15:16], s4, v[6:7]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_or_b32_e32 v8, v8, v10
+; GFX10-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_and_b32 s5, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s4, 1, s4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v14, 64, v18
+; GFX10-NEXT:    v_lshrrev_b64 v[6:7], 1, v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v15, v8, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v16, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s5
+; GFX10-NEXT:    v_lshrrev_b64 v[8:9], v14, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[10:11], v18, v[27:28]
+; GFX10-NEXT:    v_sub_nc_u32_e32 v16, 64, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v21, 0, v6, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v22, 0, v7, s4
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v23, 64, v18
+; GFX10-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v8, 64, v19
+; GFX10-NEXT:    v_lshrrev_b64 v[14:15], v19, v[4:5]
+; GFX10-NEXT:    v_lshlrev_b64 v[16:17], v16, v[21:22]
+; GFX10-NEXT:    v_lshlrev_b64 v[12:13], v18, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], v23, v[0:1]
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 64, v18
+; GFX10-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX10-NEXT:    v_lshrrev_b64 v[8:9], v8, v[21:22]
+; GFX10-NEXT:    v_or_b32_e32 v14, v14, v16
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s4, 64, v19
+; GFX10-NEXT:    v_or_b32_e32 v15, v15, v17
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v0, v10, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v1, v11, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], v19, v[21:22]
+; GFX10-NEXT:    v_cndmask_b32_e64 v23, v8, v14, s4
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 0, v19
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 0, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v9, v15, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v31, 0, v12, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0, v13, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v23, v4, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v10, v27, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v6, v5, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, v0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, v1, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v11, v28, s6
+; GFX10-NEXT:    v_or_b32_e32 v0, v31, v4
+; GFX10-NEXT:    v_or_b32_e32 v1, v7, v5
+; GFX10-NEXT:    v_or_b32_e32 v2, v15, v6
+; GFX10-NEXT:    v_or_b32_e32 v3, v3, v8
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  ret i128 %result
+}
+
+define amdgpu_ps <4 x float> @v_fshl_i128_ssv(i128 inreg %lhs, i128 inreg %rhs, i128 %amt) {
+; GFX6-LABEL: v_fshl_i128_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s8, 0x7f
+; GFX6-NEXT:    v_and_b32_e32 v6, s8, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    v_and_b32_e32 v7, s8, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, 64, v6
+; GFX6-NEXT:    v_lshr_b64 v[0:1], s[0:1], v0
+; GFX6-NEXT:    v_lshl_b64 v[2:3], s[2:3], v6
+; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, 64, v6
+; GFX6-NEXT:    s_sub_i32 s10, 1, 64
+; GFX6-NEXT:    s_sub_i32 s8, 64, 1
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_lshl_b64 v[0:1], s[0:1], v8
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v6
+; GFX6-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s2
+; GFX6-NEXT:    v_mov_b32_e32 v3, s3
+; GFX6-NEXT:    s_lshr_b64 s[2:3], s[4:5], 1
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[6:7], s8
+; GFX6-NEXT:    v_lshl_b64 v[4:5], s[0:1], v6
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[6:7], 1
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[6:7], s10
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v10, v1, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 64, v7
+; GFX6-NEXT:    v_lshr_b64 v[0:1], s[2:3], v7
+; GFX6-NEXT:    v_lshl_b64 v[2:3], s[0:1], v2
+; GFX6-NEXT:    v_subrev_i32_e32 v11, vcc, 64, v7
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_lshr_b64 v[0:1], s[0:1], v11
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
+; GFX6-NEXT:    v_lshr_b64 v[4:5], s[0:1], v7
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s2
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v7
+; GFX6-NEXT:    v_mov_b32_e32 v3, s3
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX6-NEXT:    v_or_b32_e32 v0, v8, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, v9, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, v6, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v10, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i128_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s8, 0x7f
+; GFX8-NEXT:    v_and_b32_e32 v6, s8, v0
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    v_and_b32_e32 v7, s8, v0
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, 64, v6
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v0, s[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v6, s[2:3]
+; GFX8-NEXT:    v_subrev_u32_e32 v8, vcc, 64, v6
+; GFX8-NEXT:    s_sub_i32 s10, 1, 64
+; GFX8-NEXT:    s_sub_i32 s8, 64, 1
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v8, s[0:1]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v6
+; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    s_lshr_b64 s[2:3], s[4:5], 1
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[6:7], s8
+; GFX8-NEXT:    v_lshlrev_b64 v[4:5], v6, s[0:1]
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[6:7], 1
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[6:7], s10
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v1, v3, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 64, v7
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v7, s[2:3]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v2, s[0:1]
+; GFX8-NEXT:    v_subrev_u32_e32 v11, vcc, 64, v7
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v11, s[0:1]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], v7, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v7
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX8-NEXT:    v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, v9, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v10, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i128_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s8, 0x7f
+; GFX9-NEXT:    v_and_b32_e32 v6, s8, v0
+; GFX9-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT:    v_and_b32_e32 v7, s8, v0
+; GFX9-NEXT:    v_sub_u32_e32 v0, 64, v6
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v0, s[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v6, s[2:3]
+; GFX9-NEXT:    v_subrev_u32_e32 v8, 64, v6
+; GFX9-NEXT:    s_sub_i32 s10, 1, 64
+; GFX9-NEXT:    s_sub_i32 s8, 64, 1
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v8, s[0:1]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v6
+; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[4:5], 1
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[6:7], s8
+; GFX9-NEXT:    v_lshlrev_b64 v[4:5], v6, s[0:1]
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[6:7], 1
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[6:7], s10
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], 0
+; GFX9-NEXT:    v_sub_u32_e32 v2, 64, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, v1, v3, vcc
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v7, s[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v2, s[0:1]
+; GFX9-NEXT:    v_subrev_u32_e32 v11, 64, v7
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v11, s[0:1]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], v7, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v7
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX9-NEXT:    v_or_b32_e32 v0, v8, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, v9, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, v6, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v10, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i128_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s8, 0x7f
+; GFX10-NEXT:    s_sub_i32 s14, 1, 64
+; GFX10-NEXT:    v_and_b32_e32 v12, s8, v0
+; GFX10-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX10-NEXT:    s_sub_i32 s10, 64, 1
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 64, v12
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_and_b32_e32 v13, s8, v0
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], v1, s[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], v12, s[2:3]
+; GFX10-NEXT:    s_lshr_b64 s[8:9], s[4:5], 1
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[6:7], s10
+; GFX10-NEXT:    s_lshr_b64 s[12:13], s[6:7], 1
+; GFX10-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX10-NEXT:    s_cselect_b64 s[6:7], s[8:9], s[6:7]
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, 64, v13
+; GFX10-NEXT:    s_cselect_b64 s[6:7], s[4:5], s[6:7]
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, 64, v12
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[12:13], 0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v14, 64, v13
+; GFX10-NEXT:    v_lshrrev_b64 v[6:7], v13, s[6:7]
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], v0, s[8:9]
+; GFX10-NEXT:    v_lshlrev_b64 v[15:16], v10, s[0:1]
+; GFX10-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 64, v12
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], v14, s[8:9]
+; GFX10-NEXT:    v_lshlrev_b64 v[4:5], v12, s[0:1]
+; GFX10-NEXT:    v_or_b32_e32 v6, v6, v8
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s0, 64, v13
+; GFX10-NEXT:    v_or_b32_e32 v7, v7, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v15, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v16, v3, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], v13, s[8:9]
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 0, v12
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 0, v13
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s0
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, 0, v5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v19, v8, s2, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s6, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s7, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v10, s3, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s0
+; GFX10-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, v15, v1
+; GFX10-NEXT:    v_or_b32_e32 v2, v19, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, v6, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  %cast.result = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast.result
+}
+
+define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 inreg %amt) {
+; GFX6-LABEL: v_fshl_i128_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s6, 0x7f
+; GFX6-NEXT:    s_mov_b32 s7, 0
+; GFX6-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX6-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX6-NEXT:    s_sub_i32 s5, s8, 64
+; GFX6-NEXT:    s_sub_i32 s9, 64, s8
+; GFX6-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX6-NEXT:    s_lshl_b64 s[6:7], s[0:1], s8
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[0:1], s9
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GFX6-NEXT:    s_or_b64 s[8:9], s[10:11], s[8:9]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s5
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[6:7], 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX6-NEXT:    s_sub_i32 s1, 64, 1
+; GFX6-NEXT:    s_sub_i32 s0, 1, 64
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[0:1], 1
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[2:3], s1
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[2:3], 1
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[2:3], s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s5
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s8
+; GFX6-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX6-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[0:1]
+; GFX6-NEXT:    s_sub_i32 s0, s4, 64
+; GFX6-NEXT:    s_sub_i32 s1, 64, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[0:1], s4
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[2:3], s1
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[2:3], s4
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[2:3], s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s5
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX6-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX6-NEXT:    s_and_b32 s0, 1, s8
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s5
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT:    v_or_b32_e32 v0, s6, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, s7, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i128_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s6, 0x7f
+; GFX8-NEXT:    s_mov_b32 s7, 0
+; GFX8-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX8-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX8-NEXT:    s_sub_i32 s5, s8, 64
+; GFX8-NEXT:    s_sub_i32 s9, 64, s8
+; GFX8-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[0:1], s8
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[0:1], s9
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GFX8-NEXT:    s_or_b64 s[8:9], s[10:11], s[8:9]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s5
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[6:7], 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX8-NEXT:    s_sub_i32 s1, 64, 1
+; GFX8-NEXT:    s_sub_i32 s0, 1, 64
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], 1, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], s1, v[2:3]
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], 1, v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], s0, v[2:3]
+; GFX8-NEXT:    s_and_b32 s0, 1, s5
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s8
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, s0
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[0:1]
+; GFX8-NEXT:    s_sub_i32 s0, s4, 64
+; GFX8-NEXT:    s_sub_i32 s1, 64, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], s4, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], s1, v[2:3]
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], s4, v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], s0, v[2:3]
+; GFX8-NEXT:    s_and_b32 s0, 1, s5
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT:    s_and_b32 s0, 1, s8
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT:    v_or_b32_e32 v0, s6, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, s7, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i128_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s6, 0x7f
+; GFX9-NEXT:    s_mov_b32 s7, 0
+; GFX9-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX9-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX9-NEXT:    s_sub_i32 s5, s8, 64
+; GFX9-NEXT:    s_sub_i32 s9, 64, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX9-NEXT:    s_lshl_b64 s[6:7], s[0:1], s8
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[0:1], s9
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GFX9-NEXT:    s_or_b64 s[8:9], s[10:11], s[8:9]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s5
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[6:7], 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s1, 64, 1
+; GFX9-NEXT:    s_sub_i32 s0, 1, 64
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], 1, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], s1, v[2:3]
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], 1, v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], s0, v[2:3]
+; GFX9-NEXT:    s_and_b32 s0, 1, s5
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s8
+; GFX9-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX9-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[0:1]
+; GFX9-NEXT:    s_sub_i32 s0, s4, 64
+; GFX9-NEXT:    s_sub_i32 s1, 64, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], s4, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], s1, v[2:3]
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], s4, v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], s0, v[2:3]
+; GFX9-NEXT:    s_and_b32 s0, 1, s5
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX9-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX9-NEXT:    s_and_b32 s0, 1, s8
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT:    v_or_b32_e32 v0, s6, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, s7, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i128_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s6, 0x7f
+; GFX10-NEXT:    s_mov_b32 s7, 0
+; GFX10-NEXT:    v_lshrrev_b64 v[4:5], 1, v[0:1]
+; GFX10-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX10-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX10-NEXT:    s_sub_i32 s5, s8, 64
+; GFX10-NEXT:    s_sub_i32 s6, 64, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[2:3], s8
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[0:1], s6
+; GFX10-NEXT:    s_lshl_b64 s[8:9], s[0:1], s8
+; GFX10-NEXT:    s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s5
+; GFX10-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[6:7], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s0, 64, 1
+; GFX10-NEXT:    v_lshlrev_b64 v[6:7], s0, v[2:3]
+; GFX10-NEXT:    s_sub_i32 s0, 1, 64
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[11:12], s0, v[2:3]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX10-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX10-NEXT:    s_and_b32 s1, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v11, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v12, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s0
+; GFX10-NEXT:    s_sub_i32 s0, 64, s4
+; GFX10-NEXT:    v_lshrrev_b64 v[4:5], s4, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[6:7], s0, v[2:3]
+; GFX10-NEXT:    s_sub_i32 s0, s4, 64
+; GFX10-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[11:12], s0, v[2:3]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX10-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX10-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX10-NEXT:    s_and_b32 s1, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], s4, v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v11, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v12, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s0
+; GFX10-NEXT:    v_or_b32_e32 v0, s8, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, s9, v1
+; GFX10-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  %cast.result = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast.result
+}
+
+define amdgpu_ps <4 x float> @v_fshl_i128_vss(i128 %lhs, i128 inreg %rhs, i128 inreg %amt) {
+; GFX6-LABEL: v_fshl_i128_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s6, 0x7f
+; GFX6-NEXT:    s_mov_b32 s7, 0
+; GFX6-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX6-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX6-NEXT:    s_sub_i32 s6, 64, s8
+; GFX6-NEXT:    s_sub_i32 s5, s8, 64
+; GFX6-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[0:1], s6
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[2:3], s8
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    v_lshl_b64 v[8:9], v[0:1], s8
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], s5
+; GFX6-NEXT:    s_and_b32 s5, 1, s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX6-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX6-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX6-NEXT:    s_and_b32 s5, 1, s9
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX6-NEXT:    s_sub_i32 s5, 1, 64
+; GFX6-NEXT:    s_sub_i32 s10, 64, 1
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[8:9], s[0:1], 1
+; GFX6-NEXT:    s_lshl_b64 s[10:11], s[2:3], s10
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[2:3], 1
+; GFX6-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX6-NEXT:    s_lshr_b64 s[2:3], s[2:3], s5
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX6-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[6:7], 0
+; GFX6-NEXT:    s_sub_i32 s10, s4, 64
+; GFX6-NEXT:    s_sub_i32 s8, 64, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX6-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[2:3], s4
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[0:1], s4
+; GFX6-NEXT:    s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX6-NEXT:    s_lshr_b64 s[2:3], s[2:3], s10
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[6:7], 0
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v6
+; GFX6-NEXT:    v_or_b32_e32 v1, s1, v7
+; GFX6-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i128_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s6, 0x7f
+; GFX8-NEXT:    s_mov_b32 s7, 0
+; GFX8-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX8-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX8-NEXT:    s_sub_i32 s6, 64, s8
+; GFX8-NEXT:    s_sub_i32 s5, s8, 64
+; GFX8-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], s6, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], s8, v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    v_lshlrev_b64 v[8:9], s8, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], s5, v[0:1]
+; GFX8-NEXT:    s_and_b32 s5, 1, s7
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT:    s_and_b32 s5, 1, s9
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX8-NEXT:    s_sub_i32 s5, 1, 64
+; GFX8-NEXT:    s_sub_i32 s10, 64, 1
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[8:9], s[0:1], 1
+; GFX8-NEXT:    s_lshl_b64 s[10:11], s[2:3], s10
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[2:3], 1
+; GFX8-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX8-NEXT:    s_lshr_b64 s[2:3], s[2:3], s5
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX8-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[6:7], 0
+; GFX8-NEXT:    s_sub_i32 s10, s4, 64
+; GFX8-NEXT:    s_sub_i32 s8, 64, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[2:3], s4
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[0:1], s4
+; GFX8-NEXT:    s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX8-NEXT:    s_lshr_b64 s[2:3], s[2:3], s10
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[6:7], 0
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v6
+; GFX8-NEXT:    v_or_b32_e32 v1, s1, v7
+; GFX8-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshl_i128_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s6, 0x7f
+; GFX9-NEXT:    s_mov_b32 s7, 0
+; GFX9-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX9-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX9-NEXT:    s_sub_i32 s6, 64, s8
+; GFX9-NEXT:    s_sub_i32 s5, s8, 64
+; GFX9-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], s6, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], s8, v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    v_lshlrev_b64 v[8:9], s8, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], s5, v[0:1]
+; GFX9-NEXT:    s_and_b32 s5, 1, s7
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX9-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX9-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX9-NEXT:    s_and_b32 s5, 1, s9
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX9-NEXT:    s_sub_i32 s5, 1, 64
+; GFX9-NEXT:    s_sub_i32 s10, 64, 1
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[8:9], s[0:1], 1
+; GFX9-NEXT:    s_lshl_b64 s[10:11], s[2:3], s10
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[2:3], 1
+; GFX9-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[2:3], s5
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX9-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[6:7], 0
+; GFX9-NEXT:    s_sub_i32 s10, s4, 64
+; GFX9-NEXT:    s_sub_i32 s8, 64, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[2:3], s4
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[0:1], s4
+; GFX9-NEXT:    s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[2:3], s10
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[6:7], 0
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v6
+; GFX9-NEXT:    v_or_b32_e32 v1, s1, v7
+; GFX9-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshl_i128_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s6, 0x7f
+; GFX10-NEXT:    s_mov_b32 s7, 0
+; GFX10-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX10-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX10-NEXT:    s_sub_i32 s9, 64, s8
+; GFX10-NEXT:    s_sub_i32 s5, s8, 64
+; GFX10-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[4:5], s9, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[6:7], s8, v[2:3]
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], s8, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[11:12], s5, v[0:1]
+; GFX10-NEXT:    s_and_b32 s5, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s12, 1, s6
+; GFX10-NEXT:    s_sub_i32 s13, 1, 64
+; GFX10-NEXT:    s_sub_i32 s8, 64, 1
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s5
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    s_cselect_b32 s14, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[0:1], 1
+; GFX10-NEXT:    s_lshl_b64 s[8:9], s[2:3], s8
+; GFX10-NEXT:    s_lshr_b64 s[10:11], s[2:3], 1
+; GFX10-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[2:3], s13
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v4, vcc_lo
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX10-NEXT:    s_cmp_lg_u32 s14, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v12, v5, vcc_lo
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s12
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[10:11], 0
+; GFX10-NEXT:    s_sub_i32 s10, s4, 64
+; GFX10-NEXT:    s_sub_i32 s5, 64, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v0, v2, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[0:1], s4
+; GFX10-NEXT:    s_lshl_b64 s[8:9], s[2:3], s5
+; GFX10-NEXT:    s_lshr_b64 s[4:5], s[2:3], s4
+; GFX10-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[2:3], s10
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX10-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v6
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[4:5], 0
+; GFX10-NEXT:    v_or_b32_e32 v1, s1, v7
+; GFX10-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  %cast.result = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast.result
+}
+
+define amdgpu_ps i128 @s_fshl_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: s_fshl_i128_65:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s8, 0x41
+; GFX6-NEXT:    s_sub_i32 s16, s8, 64
+; GFX6-NEXT:    s_sub_i32 s12, 64, s8
+; GFX6-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX6-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[12:13], s[0:1], s12
+; GFX6-NEXT:    s_lshl_b64 s[14:15], s[2:3], s8
+; GFX6-NEXT:    s_lshl_b64 s[10:11], s[0:1], s8
+; GFX6-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s16
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[12:13], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX6-NEXT:    s_sub_i32 s14, 63, 64
+; GFX6-NEXT:    s_sub_i32 s12, 64, 63
+; GFX6-NEXT:    s_cmp_lt_u32 63, 64
+; GFX6-NEXT:    s_mov_b32 s9, 0
+; GFX6-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 63, 0
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_lshr_b32 s0, s5, 31
+; GFX6-NEXT:    s_mov_b32 s1, s9
+; GFX6-NEXT:    s_lshl_b64 s[12:13], s[6:7], s12
+; GFX6-NEXT:    s_lshr_b32 s8, s7, 31
+; GFX6-NEXT:    s_or_b64 s[0:1], s[0:1], s[12:13]
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[8:9], 0
+; GFX6-NEXT:    s_or_b64 s[0:1], s[10:11], s[0:1]
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i128_65:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s8, 0x41
+; GFX8-NEXT:    s_sub_i32 s16, s8, 64
+; GFX8-NEXT:    s_sub_i32 s12, 64, s8
+; GFX8-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX8-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[12:13], s[0:1], s12
+; GFX8-NEXT:    s_lshl_b64 s[14:15], s[2:3], s8
+; GFX8-NEXT:    s_lshl_b64 s[10:11], s[0:1], s8
+; GFX8-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s16
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[12:13], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX8-NEXT:    s_sub_i32 s14, 63, 64
+; GFX8-NEXT:    s_sub_i32 s12, 64, 63
+; GFX8-NEXT:    s_cmp_lt_u32 63, 64
+; GFX8-NEXT:    s_mov_b32 s9, 0
+; GFX8-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 63, 0
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_lshr_b32 s0, s5, 31
+; GFX8-NEXT:    s_mov_b32 s1, s9
+; GFX8-NEXT:    s_lshl_b64 s[12:13], s[6:7], s12
+; GFX8-NEXT:    s_lshr_b32 s8, s7, 31
+; GFX8-NEXT:    s_or_b64 s[0:1], s[0:1], s[12:13]
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[8:9], 0
+; GFX8-NEXT:    s_or_b64 s[0:1], s[10:11], s[0:1]
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_i128_65:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s8, 0x41
+; GFX9-NEXT:    s_sub_i32 s16, s8, 64
+; GFX9-NEXT:    s_sub_i32 s12, 64, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX9-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[12:13], s[0:1], s12
+; GFX9-NEXT:    s_lshl_b64 s[14:15], s[2:3], s8
+; GFX9-NEXT:    s_lshl_b64 s[10:11], s[0:1], s8
+; GFX9-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s16
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[12:13], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s14, 63, 64
+; GFX9-NEXT:    s_sub_i32 s12, 64, 63
+; GFX9-NEXT:    s_cmp_lt_u32 63, 64
+; GFX9-NEXT:    s_mov_b32 s9, 0
+; GFX9-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 63, 0
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_lshr_b32 s0, s5, 31
+; GFX9-NEXT:    s_mov_b32 s1, s9
+; GFX9-NEXT:    s_lshl_b64 s[12:13], s[6:7], s12
+; GFX9-NEXT:    s_lshr_b32 s8, s7, 31
+; GFX9-NEXT:    s_or_b64 s[0:1], s[0:1], s[12:13]
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[8:9], 0
+; GFX9-NEXT:    s_or_b64 s[0:1], s[10:11], s[0:1]
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_i128_65:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s12, 0x41
+; GFX10-NEXT:    s_sub_i32 s14, s12, 64
+; GFX10-NEXT:    s_sub_i32 s8, 64, s12
+; GFX10-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[2:3], s12
+; GFX10-NEXT:    s_lshr_b64 s[8:9], s[0:1], s8
+; GFX10-NEXT:    s_lshl_b64 s[12:13], s[0:1], s12
+; GFX10-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s14
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_cselect_b64 s[10:11], s[12:13], 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s14, 63, 64
+; GFX10-NEXT:    s_sub_i32 s0, 64, 63
+; GFX10-NEXT:    s_cmp_lt_u32 63, 64
+; GFX10-NEXT:    s_mov_b32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 63, 0
+; GFX10-NEXT:    s_mov_b32 s9, s1
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[12:13], s[6:7], s0
+; GFX10-NEXT:    s_lshr_b32 s8, s5, 31
+; GFX10-NEXT:    s_lshr_b32 s0, s7, 31
+; GFX10-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_cselect_b64 s[6:7], s[8:9], s[6:7]
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_cselect_b64 s[6:7], s[0:1], 0
+; GFX10-NEXT:    s_or_b64 s[0:1], s[10:11], s[4:5]
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 65)
+  ret i128 %result
+}
+
+define i128 @v_fshl_i128_65(i128 %lhs, i128 %rhs) {
+; GFX6-LABEL: v_fshl_i128_65:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_movk_i32 s4, 0x41
+; GFX6-NEXT:    s_sub_i32 s6, 64, s4
+; GFX6-NEXT:    s_sub_i32 s5, s4, 64
+; GFX6-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[0:1], s6
+; GFX6-NEXT:    v_lshl_b64 v[10:11], v[2:3], s4
+; GFX6-NEXT:    v_lshl_b64 v[12:13], v[0:1], s4
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], s5
+; GFX6-NEXT:    s_and_b32 s4, 1, s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_or_b32_e32 v8, v8, v10
+; GFX6-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX6-NEXT:    s_and_b32 s4, 1, s8
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v10, 0, v12, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_sub_i32 s4, 63, 64
+; GFX6-NEXT:    s_sub_i32 s5, 64, 63
+; GFX6-NEXT:    s_cmp_lt_u32 63, 64
+; GFX6-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 63, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[6:7], s5
+; GFX6-NEXT:    v_lshrrev_b32_e32 v8, 31, v7
+; GFX6-NEXT:    v_lshrrev_b32_e32 v9, 31, v5
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    v_lshr_b64 v[6:7], v[6:7], s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s6
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_or_b32_e32 v0, v9, v0
+; GFX6-NEXT:    s_and_b32 s4, 1, s7
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s6
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX6-NEXT:    v_or_b32_e32 v0, v10, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, v11, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_i128_65:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_movk_i32 s4, 0x41
+; GFX8-NEXT:    s_sub_i32 s6, 64, s4
+; GFX8-NEXT:    s_sub_i32 s5, s4, 64
+; GFX8-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], s6, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[10:11], s4, v[2:3]
+; GFX8-NEXT:    v_lshlrev_b64 v[12:13], s4, v[0:1]
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], s5, v[0:1]
+; GFX8-NEXT:    s_and_b32 s4, 1, s7
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_or_b32_e32 v8, v8, v10
+; GFX8-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX8-NEXT:    s_and_b32 s4, 1, s8
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, 0, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_sub_i32 s4, 63, 64
+; GFX8-NEXT:    s_sub_i32 s5, 64, 63
+; GFX8-NEXT:    s_cmp_lt_u32 63, 64
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 63, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], s5, v[6:7]
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 31, v7
+; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 31, v5
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[6:7], s4, v[6:7]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_or_b32_e32 v0, v9, v0
+; GFX8-NEXT:    s_and_b32 s4, 1, s7
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX8-NEXT:    v_or_b32_e32 v0, v10, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, v11, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_i128_65:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x41
+; GFX9-NEXT:    s_sub_i32 s6, 64, s4
+; GFX9-NEXT:    s_sub_i32 s5, s4, 64
+; GFX9-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], s6, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[10:11], s4, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[12:13], s4, v[0:1]
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], s5, v[0:1]
+; GFX9-NEXT:    s_and_b32 s4, 1, s7
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_or_b32_e32 v8, v8, v10
+; GFX9-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX9-NEXT:    s_and_b32 s4, 1, s8
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v12, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, 0, v13, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_sub_i32 s4, 63, 64
+; GFX9-NEXT:    s_sub_i32 s5, 64, 63
+; GFX9-NEXT:    s_cmp_lt_u32 63, 64
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 63, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], s5, v[6:7]
+; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 31, v7
+; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 31, v5
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[6:7], s4, v[6:7]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_or_b32_e32 v0, v9, v0
+; GFX9-NEXT:    s_and_b32 s4, 1, s7
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX9-NEXT:    v_or_b32_e32 v0, v10, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, v11, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_i128_65:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_movk_i32 s4, 0x41
+; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 31, v5
+; GFX10-NEXT:    s_sub_i32 s5, 64, s4
+; GFX10-NEXT:    v_lshlrev_b64 v[10:11], s4, v[2:3]
+; GFX10-NEXT:    v_lshrrev_b64 v[8:9], s5, v[0:1]
+; GFX10-NEXT:    s_sub_i32 s5, s4, 64
+; GFX10-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX10-NEXT:    v_lshlrev_b64 v[12:13], s4, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], s5, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_sub_i32 s5, 64, 63
+; GFX10-NEXT:    v_or_b32_e32 v15, v9, v11
+; GFX10-NEXT:    v_or_b32_e32 v14, v8, v10
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], s5, v[6:7]
+; GFX10-NEXT:    s_and_b32 s6, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s7, 1, s4
+; GFX10-NEXT:    s_sub_i32 s4, 63, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0, v12, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v27, 0, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s6
+; GFX10-NEXT:    v_lshrrev_b64 v[23:24], s4, v[6:7]
+; GFX10-NEXT:    s_cmp_lt_u32 63, 64
+; GFX10-NEXT:    v_or_b32_e32 v6, v19, v8
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 63, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s6, 0, s7
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v23, v6, s4
+; GFX10-NEXT:    s_and_b32 s5, 1, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v24, v9, s4
+; GFX10-NEXT:    s_and_b32 s4, 1, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s5, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v19, v0, v2, s6
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 31, v7
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v1, v15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v8, v5, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, v0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v10, v3, s6
+; GFX10-NEXT:    v_or_b32_e32 v0, v11, v4
+; GFX10-NEXT:    v_or_b32_e32 v1, v27, v5
+; GFX10-NEXT:    v_or_b32_e32 v2, v19, v6
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 65)
+  ret i128 %result
+}
+
+define amdgpu_ps <2 x i128> @s_fshl_v2i128(<2 x i128> inreg %lhs, <2 x i128> inreg %rhs, <2 x i128> inreg %amt) {
+; GFX6-LABEL: s_fshl_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s18, 0x7f
+; GFX6-NEXT:    s_mov_b32 s19, 0
+; GFX6-NEXT:    s_and_b64 s[22:23], s[16:17], s[18:19]
+; GFX6-NEXT:    s_andn2_b64 s[16:17], s[18:19], s[16:17]
+; GFX6-NEXT:    s_sub_i32 s17, s22, 64
+; GFX6-NEXT:    s_sub_i32 s23, 64, s22
+; GFX6-NEXT:    s_cmp_lt_u32 s22, 64
+; GFX6-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s22, 0
+; GFX6-NEXT:    s_cselect_b32 s29, 1, 0
+; GFX6-NEXT:    s_lshl_b64 s[24:25], s[0:1], s22
+; GFX6-NEXT:    s_lshr_b64 s[26:27], s[0:1], s23
+; GFX6-NEXT:    s_lshl_b64 s[22:23], s[2:3], s22
+; GFX6-NEXT:    s_or_b64 s[22:23], s[26:27], s[22:23]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s17
+; GFX6-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX6-NEXT:    s_cselect_b64 s[24:25], s[24:25], 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[22:23], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s29, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX6-NEXT:    s_sub_i32 s28, 1, 64
+; GFX6-NEXT:    s_sub_i32 s29, 64, 1
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s30, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[22:23], s[8:9], 1
+; GFX6-NEXT:    s_lshl_b64 s[26:27], s[10:11], s29
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[10:11], 1
+; GFX6-NEXT:    s_or_b64 s[22:23], s[22:23], s[26:27]
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[10:11], s28
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_cselect_b64 s[10:11], s[22:23], s[10:11]
+; GFX6-NEXT:    s_cmp_lg_u32 s30, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], 0
+; GFX6-NEXT:    s_sub_i32 s26, s16, 64
+; GFX6-NEXT:    s_sub_i32 s22, 64, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s16, 64
+; GFX6-NEXT:    s_cselect_b32 s27, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b32 s30, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[0:1], s16
+; GFX6-NEXT:    s_lshl_b64 s[22:23], s[0:1], s22
+; GFX6-NEXT:    s_lshr_b64 s[16:17], s[8:9], s16
+; GFX6-NEXT:    s_or_b64 s[16:17], s[16:17], s[22:23]
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[0:1], s26
+; GFX6-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[16:17], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s30, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[10:11], 0
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX6-NEXT:    s_and_b64 s[8:9], s[20:21], s[18:19]
+; GFX6-NEXT:    s_andn2_b64 s[10:11], s[18:19], s[20:21]
+; GFX6-NEXT:    s_or_b64 s[0:1], s[24:25], s[0:1]
+; GFX6-NEXT:    s_sub_i32 s11, s8, 64
+; GFX6-NEXT:    s_sub_i32 s9, 64, s8
+; GFX6-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX6-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s21, 1, 0
+; GFX6-NEXT:    s_lshl_b64 s[16:17], s[4:5], s8
+; GFX6-NEXT:    s_lshr_b64 s[18:19], s[4:5], s9
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[6:7], s8
+; GFX6-NEXT:    s_or_b64 s[8:9], s[18:19], s[8:9]
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[4:5], s11
+; GFX6-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX6-NEXT:    s_cselect_b64 s[16:17], s[16:17], 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX6-NEXT:    s_cmp_lg_u32 s21, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[8:9], s[12:13], 1
+; GFX6-NEXT:    s_lshl_b64 s[18:19], s[14:15], s29
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[14:15], 1
+; GFX6-NEXT:    s_or_b64 s[8:9], s[8:9], s[18:19]
+; GFX6-NEXT:    s_lshr_b64 s[14:15], s[14:15], s28
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[14:15]
+; GFX6-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[4:5], 0
+; GFX6-NEXT:    s_sub_i32 s18, s10, 64
+; GFX6-NEXT:    s_sub_i32 s14, 64, s10
+; GFX6-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX6-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX6-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[12:13], s[4:5], s10
+; GFX6-NEXT:    s_lshl_b64 s[14:15], s[4:5], s14
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[8:9], s10
+; GFX6-NEXT:    s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[4:5], s18
+; GFX6-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[10:11], s[4:5]
+; GFX6-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX6-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[12:13], 0
+; GFX6-NEXT:    s_or_b64 s[4:5], s[16:17], s[4:5]
+; GFX6-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s18, 0x7f
+; GFX8-NEXT:    s_mov_b32 s19, 0
+; GFX8-NEXT:    s_and_b64 s[22:23], s[16:17], s[18:19]
+; GFX8-NEXT:    s_andn2_b64 s[16:17], s[18:19], s[16:17]
+; GFX8-NEXT:    s_sub_i32 s17, s22, 64
+; GFX8-NEXT:    s_sub_i32 s23, 64, s22
+; GFX8-NEXT:    s_cmp_lt_u32 s22, 64
+; GFX8-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s22, 0
+; GFX8-NEXT:    s_cselect_b32 s29, 1, 0
+; GFX8-NEXT:    s_lshl_b64 s[24:25], s[0:1], s22
+; GFX8-NEXT:    s_lshr_b64 s[26:27], s[0:1], s23
+; GFX8-NEXT:    s_lshl_b64 s[22:23], s[2:3], s22
+; GFX8-NEXT:    s_or_b64 s[22:23], s[26:27], s[22:23]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s17
+; GFX8-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX8-NEXT:    s_cselect_b64 s[24:25], s[24:25], 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[22:23], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s29, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX8-NEXT:    s_sub_i32 s28, 1, 64
+; GFX8-NEXT:    s_sub_i32 s29, 64, 1
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s30, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[22:23], s[8:9], 1
+; GFX8-NEXT:    s_lshl_b64 s[26:27], s[10:11], s29
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[10:11], 1
+; GFX8-NEXT:    s_or_b64 s[22:23], s[22:23], s[26:27]
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[10:11], s28
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_cselect_b64 s[10:11], s[22:23], s[10:11]
+; GFX8-NEXT:    s_cmp_lg_u32 s30, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], 0
+; GFX8-NEXT:    s_sub_i32 s26, s16, 64
+; GFX8-NEXT:    s_sub_i32 s22, 64, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s16, 64
+; GFX8-NEXT:    s_cselect_b32 s27, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b32 s30, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[0:1], s16
+; GFX8-NEXT:    s_lshl_b64 s[22:23], s[0:1], s22
+; GFX8-NEXT:    s_lshr_b64 s[16:17], s[8:9], s16
+; GFX8-NEXT:    s_or_b64 s[16:17], s[16:17], s[22:23]
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[0:1], s26
+; GFX8-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[16:17], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s30, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[10:11], 0
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX8-NEXT:    s_and_b64 s[8:9], s[20:21], s[18:19]
+; GFX8-NEXT:    s_andn2_b64 s[10:11], s[18:19], s[20:21]
+; GFX8-NEXT:    s_or_b64 s[0:1], s[24:25], s[0:1]
+; GFX8-NEXT:    s_sub_i32 s11, s8, 64
+; GFX8-NEXT:    s_sub_i32 s9, 64, s8
+; GFX8-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX8-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s21, 1, 0
+; GFX8-NEXT:    s_lshl_b64 s[16:17], s[4:5], s8
+; GFX8-NEXT:    s_lshr_b64 s[18:19], s[4:5], s9
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[6:7], s8
+; GFX8-NEXT:    s_or_b64 s[8:9], s[18:19], s[8:9]
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[4:5], s11
+; GFX8-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX8-NEXT:    s_cselect_b64 s[16:17], s[16:17], 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX8-NEXT:    s_cmp_lg_u32 s21, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[8:9], s[12:13], 1
+; GFX8-NEXT:    s_lshl_b64 s[18:19], s[14:15], s29
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[14:15], 1
+; GFX8-NEXT:    s_or_b64 s[8:9], s[8:9], s[18:19]
+; GFX8-NEXT:    s_lshr_b64 s[14:15], s[14:15], s28
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[14:15]
+; GFX8-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[4:5], 0
+; GFX8-NEXT:    s_sub_i32 s18, s10, 64
+; GFX8-NEXT:    s_sub_i32 s14, 64, s10
+; GFX8-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX8-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[12:13], s[4:5], s10
+; GFX8-NEXT:    s_lshl_b64 s[14:15], s[4:5], s14
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[8:9], s10
+; GFX8-NEXT:    s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[4:5], s18
+; GFX8-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[10:11], s[4:5]
+; GFX8-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX8-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[12:13], 0
+; GFX8-NEXT:    s_or_b64 s[4:5], s[16:17], s[4:5]
+; GFX8-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshl_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s18, 0x7f
+; GFX9-NEXT:    s_mov_b32 s19, 0
+; GFX9-NEXT:    s_and_b64 s[22:23], s[16:17], s[18:19]
+; GFX9-NEXT:    s_andn2_b64 s[16:17], s[18:19], s[16:17]
+; GFX9-NEXT:    s_sub_i32 s17, s22, 64
+; GFX9-NEXT:    s_sub_i32 s23, 64, s22
+; GFX9-NEXT:    s_cmp_lt_u32 s22, 64
+; GFX9-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s22, 0
+; GFX9-NEXT:    s_cselect_b32 s29, 1, 0
+; GFX9-NEXT:    s_lshl_b64 s[24:25], s[0:1], s22
+; GFX9-NEXT:    s_lshr_b64 s[26:27], s[0:1], s23
+; GFX9-NEXT:    s_lshl_b64 s[22:23], s[2:3], s22
+; GFX9-NEXT:    s_or_b64 s[22:23], s[26:27], s[22:23]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s17
+; GFX9-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX9-NEXT:    s_cselect_b64 s[24:25], s[24:25], 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[22:23], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s29, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s28, 1, 64
+; GFX9-NEXT:    s_sub_i32 s29, 64, 1
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s30, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[22:23], s[8:9], 1
+; GFX9-NEXT:    s_lshl_b64 s[26:27], s[10:11], s29
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[10:11], 1
+; GFX9-NEXT:    s_or_b64 s[22:23], s[22:23], s[26:27]
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[10:11], s28
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_cselect_b64 s[10:11], s[22:23], s[10:11]
+; GFX9-NEXT:    s_cmp_lg_u32 s30, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], 0
+; GFX9-NEXT:    s_sub_i32 s26, s16, 64
+; GFX9-NEXT:    s_sub_i32 s22, 64, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s16, 64
+; GFX9-NEXT:    s_cselect_b32 s27, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b32 s30, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[0:1], s16
+; GFX9-NEXT:    s_lshl_b64 s[22:23], s[0:1], s22
+; GFX9-NEXT:    s_lshr_b64 s[16:17], s[8:9], s16
+; GFX9-NEXT:    s_or_b64 s[16:17], s[16:17], s[22:23]
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], s26
+; GFX9-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[16:17], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s30, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[10:11], 0
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX9-NEXT:    s_and_b64 s[8:9], s[20:21], s[18:19]
+; GFX9-NEXT:    s_andn2_b64 s[10:11], s[18:19], s[20:21]
+; GFX9-NEXT:    s_or_b64 s[0:1], s[24:25], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s11, s8, 64
+; GFX9-NEXT:    s_sub_i32 s9, 64, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
+; GFX9-NEXT:    s_lshl_b64 s[16:17], s[4:5], s8
+; GFX9-NEXT:    s_lshr_b64 s[18:19], s[4:5], s9
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[6:7], s8
+; GFX9-NEXT:    s_or_b64 s[8:9], s[18:19], s[8:9]
+; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], s11
+; GFX9-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX9-NEXT:    s_cselect_b64 s[16:17], s[16:17], 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX9-NEXT:    s_cmp_lg_u32 s21, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[8:9], s[12:13], 1
+; GFX9-NEXT:    s_lshl_b64 s[18:19], s[14:15], s29
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[14:15], 1
+; GFX9-NEXT:    s_or_b64 s[8:9], s[8:9], s[18:19]
+; GFX9-NEXT:    s_lshr_b64 s[14:15], s[14:15], s28
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[14:15]
+; GFX9-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[4:5], 0
+; GFX9-NEXT:    s_sub_i32 s18, s10, 64
+; GFX9-NEXT:    s_sub_i32 s14, 64, s10
+; GFX9-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[12:13], s[4:5], s10
+; GFX9-NEXT:    s_lshl_b64 s[14:15], s[4:5], s14
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[8:9], s10
+; GFX9-NEXT:    s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], s18
+; GFX9-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[10:11], s[4:5]
+; GFX9-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX9-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[12:13], 0
+; GFX9-NEXT:    s_or_b64 s[4:5], s[16:17], s[4:5]
+; GFX9-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshl_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s18, 0x7f
+; GFX10-NEXT:    s_mov_b32 s19, 0
+; GFX10-NEXT:    s_mov_b32 s30, s0
+; GFX10-NEXT:    s_and_b64 s[22:23], s[16:17], s[18:19]
+; GFX10-NEXT:    s_andn2_b64 s[16:17], s[18:19], s[16:17]
+; GFX10-NEXT:    s_sub_i32 s17, s22, 64
+; GFX10-NEXT:    s_sub_i32 s23, 64, s22
+; GFX10-NEXT:    s_cmp_lt_u32 s22, 64
+; GFX10-NEXT:    s_mov_b32 s31, s1
+; GFX10-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s22, 0
+; GFX10-NEXT:    s_cselect_b32 s29, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[24:25], s[30:31], s23
+; GFX10-NEXT:    s_lshl_b64 s[26:27], s[2:3], s22
+; GFX10-NEXT:    s_lshl_b64 s[22:23], s[30:31], s22
+; GFX10-NEXT:    s_or_b64 s[24:25], s[24:25], s[26:27]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[30:31], s17
+; GFX10-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX10-NEXT:    s_cselect_b64 s[22:23], s[22:23], 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[24:25], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s29, 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s28, 1, 64
+; GFX10-NEXT:    s_sub_i32 s29, 64, 1
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    s_cselect_b32 s30, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[8:9], 1
+; GFX10-NEXT:    s_lshl_b64 s[24:25], s[10:11], s29
+; GFX10-NEXT:    s_lshr_b64 s[26:27], s[10:11], 1
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[24:25]
+; GFX10-NEXT:    s_lshr_b64 s[10:11], s[10:11], s28
+; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT:    s_cmp_lg_u32 s30, 0
+; GFX10-NEXT:    s_cselect_b64 s[46:47], s[8:9], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[26:27], 0
+; GFX10-NEXT:    s_sub_i32 s26, s16, 64
+; GFX10-NEXT:    s_sub_i32 s17, 64, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s16, 64
+; GFX10-NEXT:    s_cselect_b32 s27, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b32 s30, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[10:11], s[46:47], s16
+; GFX10-NEXT:    s_lshl_b64 s[24:25], s[8:9], s17
+; GFX10-NEXT:    s_lshr_b64 s[16:17], s[8:9], s16
+; GFX10-NEXT:    s_or_b64 s[10:11], s[10:11], s[24:25]
+; GFX10-NEXT:    s_lshr_b64 s[8:9], s[8:9], s26
+; GFX10-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[10:11], s[8:9]
+; GFX10-NEXT:    s_cmp_lg_u32 s30, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[46:47], s[8:9]
+; GFX10-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[16:17], 0
+; GFX10-NEXT:    s_andn2_b64 s[10:11], s[18:19], s[20:21]
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX10-NEXT:    s_and_b64 s[8:9], s[20:21], s[18:19]
+; GFX10-NEXT:    s_or_b64 s[0:1], s[22:23], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s11, s8, 64
+; GFX10-NEXT:    s_sub_i32 s9, 64, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[16:17], s[4:5], s9
+; GFX10-NEXT:    s_lshl_b64 s[18:19], s[6:7], s8
+; GFX10-NEXT:    s_lshl_b64 s[8:9], s[4:5], s8
+; GFX10-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GFX10-NEXT:    s_lshl_b64 s[4:5], s[4:5], s11
+; GFX10-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[16:17], s[4:5]
+; GFX10-NEXT:    s_cmp_lg_u32 s21, 0
+; GFX10-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[4:5], s[12:13], 1
+; GFX10-NEXT:    s_lshl_b64 s[16:17], s[14:15], s29
+; GFX10-NEXT:    s_lshr_b64 s[18:19], s[14:15], 1
+; GFX10-NEXT:    s_or_b64 s[4:5], s[4:5], s[16:17]
+; GFX10-NEXT:    s_lshr_b64 s[14:15], s[14:15], s28
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[14:15]
+; GFX10-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[12:13], s[4:5]
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    s_cselect_b64 s[12:13], s[18:19], 0
+; GFX10-NEXT:    s_sub_i32 s18, s10, 64
+; GFX10-NEXT:    s_sub_i32 s11, 64, s10
+; GFX10-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[14:15], s[4:5], s10
+; GFX10-NEXT:    s_lshl_b64 s[16:17], s[12:13], s11
+; GFX10-NEXT:    s_lshr_b64 s[10:11], s[12:13], s10
+; GFX10-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX10-NEXT:    s_lshr_b64 s[12:13], s[12:13], s18
+; GFX10-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX10-NEXT:    s_cselect_b64 s[12:13], s[14:15], s[12:13]
+; GFX10-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[12:13]
+; GFX10-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX10-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX10-NEXT:    s_or_b64 s[4:5], s[8:9], s[4:5]
+; GFX10-NEXT:    s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
+  ret <2 x i128> %result
+}
+
+define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt) {
+; GFX6-LABEL: v_fshl_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_movk_i32 s6, 0x7f
+; GFX6-NEXT:    v_and_b32_e32 v23, s6, v16
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, 64, v23
+; GFX6-NEXT:    v_lshr_b64 v[17:18], v[0:1], v17
+; GFX6-NEXT:    v_lshl_b64 v[21:22], v[2:3], v23
+; GFX6-NEXT:    s_sub_i32 s7, 64, 1
+; GFX6-NEXT:    s_sub_i32 s8, 1, 64
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    v_or_b32_e32 v24, v17, v21
+; GFX6-NEXT:    v_or_b32_e32 v25, v18, v22
+; GFX6-NEXT:    v_lshr_b64 v[17:18], v[8:9], 1
+; GFX6-NEXT:    v_lshl_b64 v[21:22], v[10:11], s7
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    v_or_b32_e32 v19, v17, v21
+; GFX6-NEXT:    v_or_b32_e32 v21, v18, v22
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_and_b32 s4, 1, s4
+; GFX6-NEXT:    v_lshr_b64 v[17:18], v[10:11], s8
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s5
+; GFX6-NEXT:    v_lshr_b64 v[10:11], v[10:11], 1
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v16
+; GFX6-NEXT:    v_cndmask_b32_e32 v18, v18, v21, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v17, v17, v19, vcc
+; GFX6-NEXT:    v_and_b32_e32 v21, s6, v16
+; GFX6-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v9, v18, v9, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v10, 0, v10, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v11, 0, v11, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v16, vcc, 64, v21
+; GFX6-NEXT:    v_lshl_b64 v[16:17], v[10:11], v16
+; GFX6-NEXT:    v_lshr_b64 v[18:19], v[8:9], v21
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v23
+; GFX6-NEXT:    v_or_b32_e32 v18, v18, v16
+; GFX6-NEXT:    v_subrev_i32_e32 v16, vcc, 64, v23
+; GFX6-NEXT:    v_or_b32_e32 v19, v19, v17
+; GFX6-NEXT:    v_lshl_b64 v[16:17], v[0:1], v16
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], v23
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v23
+; GFX6-NEXT:    v_cndmask_b32_e32 v22, 0, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v16, v24, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v16, v17, v25, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v17, v0, v2, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v16, v16, v3, s[4:5]
+; GFX6-NEXT:    v_subrev_i32_e64 v0, s[4:5], 64, v21
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[10:11], v0
+; GFX6-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v21
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v18, 0, v1, vcc
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[10:11], v21
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v21
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, v19, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v8, 0, v0, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v9, 0, v1, s[4:5]
+; GFX6-NEXT:    v_or_b32_e32 v0, v22, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v18, v3
+; GFX6-NEXT:    v_or_b32_e32 v2, v17, v8
+; GFX6-NEXT:    v_or_b32_e32 v3, v16, v9
+; GFX6-NEXT:    v_and_b32_e32 v16, s6, v20
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v20
+; GFX6-NEXT:    v_and_b32_e32 v17, s6, v8
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, 64, v16
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[4:5], v8
+; GFX6-NEXT:    v_lshl_b64 v[10:11], v[6:7], v16
+; GFX6-NEXT:    v_subrev_i32_e32 v18, vcc, 64, v16
+; GFX6-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX6-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX6-NEXT:    v_lshl_b64 v[8:9], v[4:5], v16
+; GFX6-NEXT:    v_lshl_b64 v[4:5], v[4:5], v18
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX6-NEXT:    v_cndmask_b32_e32 v18, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v19, 0, v9, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    v_cndmask_b32_e32 v16, v4, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v20, v5, v7, vcc
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[12:13], 1
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[14:15], s7
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_and_b32 s6, 1, s4
+; GFX6-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX6-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[14:15], s8
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX6-NEXT:    s_and_b32 s5, 1, s5
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[14:15], 1
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX6-NEXT:    s_and_b32 s4, 1, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, 64, v17
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[4:5], v17
+; GFX6-NEXT:    v_lshl_b64 v[10:11], v[6:7], v10
+; GFX6-NEXT:    v_subrev_i32_e32 v12, vcc, 64, v17
+; GFX6-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX6-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[6:7], v17
+; GFX6-NEXT:    v_lshr_b64 v[6:7], v[6:7], v12
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v17
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v17
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX6-NEXT:    v_or_b32_e32 v4, v18, v4
+; GFX6-NEXT:    v_or_b32_e32 v5, v19, v5
+; GFX6-NEXT:    v_or_b32_e32 v6, v16, v6
+; GFX6-NEXT:    v_or_b32_e32 v7, v20, v7
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_movk_i32 s6, 0x7f
+; GFX8-NEXT:    v_and_b32_e32 v23, s6, v16
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, 64, v23
+; GFX8-NEXT:    v_lshrrev_b64 v[17:18], v17, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[21:22], v23, v[2:3]
+; GFX8-NEXT:    s_sub_i32 s7, 64, 1
+; GFX8-NEXT:    s_sub_i32 s8, 1, 64
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    v_or_b32_e32 v24, v17, v21
+; GFX8-NEXT:    v_or_b32_e32 v25, v18, v22
+; GFX8-NEXT:    v_lshrrev_b64 v[17:18], 1, v[8:9]
+; GFX8-NEXT:    v_lshlrev_b64 v[21:22], s7, v[10:11]
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    v_or_b32_e32 v19, v17, v21
+; GFX8-NEXT:    v_or_b32_e32 v21, v18, v22
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_and_b32 s4, 1, s4
+; GFX8-NEXT:    v_lshrrev_b64 v[17:18], s8, v[10:11]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s5
+; GFX8-NEXT:    v_lshrrev_b64 v[10:11], 1, v[10:11]
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, v18, v21, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v17, v17, v19, vcc
+; GFX8-NEXT:    v_and_b32_e32 v21, s6, v16
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, v18, v9, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, 0, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, 0, v11, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v16, vcc, 64, v21
+; GFX8-NEXT:    v_lshlrev_b64 v[16:17], v16, v[10:11]
+; GFX8-NEXT:    v_lshrrev_b64 v[18:19], v21, v[8:9]
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v23
+; GFX8-NEXT:    v_or_b32_e32 v18, v18, v16
+; GFX8-NEXT:    v_subrev_u32_e32 v16, vcc, 64, v23
+; GFX8-NEXT:    v_or_b32_e32 v19, v19, v17
+; GFX8-NEXT:    v_lshlrev_b64 v[16:17], v16, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v23, v[0:1]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v23
+; GFX8-NEXT:    v_cndmask_b32_e32 v22, 0, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v24, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v16, v17, v25, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v17, v0, v2, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v16, v16, v3, s[4:5]
+; GFX8-NEXT:    v_subrev_u32_e64 v0, s[4:5], 64, v21
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], v0, v[10:11]
+; GFX8-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v21
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, 0, v1, vcc
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v21, v[10:11]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v21
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v19, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, v0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, 0, v1, s[4:5]
+; GFX8-NEXT:    v_or_b32_e32 v0, v22, v2
+; GFX8-NEXT:    v_or_b32_e32 v1, v18, v3
+; GFX8-NEXT:    v_or_b32_e32 v2, v17, v8
+; GFX8-NEXT:    v_or_b32_e32 v3, v16, v9
+; GFX8-NEXT:    v_and_b32_e32 v16, s6, v20
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v20
+; GFX8-NEXT:    v_and_b32_e32 v17, s6, v8
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, 64, v16
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], v8, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[10:11], v16, v[6:7]
+; GFX8-NEXT:    v_subrev_u32_e32 v18, vcc, 64, v16
+; GFX8-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX8-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX8-NEXT:    v_lshlrev_b64 v[8:9], v16, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[4:5], v18, v[4:5]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v19, 0, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    v_cndmask_b32_e32 v16, v4, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v20, v5, v7, vcc
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], 1, v[12:13]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], s7, v[14:15]
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_and_b32 s6, 1, s4
+; GFX8-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX8-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], s8, v[14:15]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX8-NEXT:    s_and_b32 s5, 1, s5
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], 1, v[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX8-NEXT:    s_and_b32 s4, 1, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v10, vcc, 64, v17
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], v17, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[10:11], v10, v[6:7]
+; GFX8-NEXT:    v_subrev_u32_e32 v12, vcc, 64, v17
+; GFX8-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX8-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], v17, v[6:7]
+; GFX8-NEXT:    v_lshrrev_b64 v[6:7], v12, v[6:7]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX8-NEXT:    v_or_b32_e32 v4, v18, v4
+; GFX8-NEXT:    v_or_b32_e32 v5, v19, v5
+; GFX8-NEXT:    v_or_b32_e32 v6, v16, v6
+; GFX8-NEXT:    v_or_b32_e32 v7, v20, v7
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshl_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s6, 0x7f
+; GFX9-NEXT:    v_and_b32_e32 v23, s6, v16
+; GFX9-NEXT:    v_sub_u32_e32 v17, 64, v23
+; GFX9-NEXT:    v_lshrrev_b64 v[17:18], v17, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[21:22], v23, v[2:3]
+; GFX9-NEXT:    s_sub_i32 s7, 64, 1
+; GFX9-NEXT:    s_sub_i32 s8, 1, 64
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    v_or_b32_e32 v24, v17, v21
+; GFX9-NEXT:    v_or_b32_e32 v25, v18, v22
+; GFX9-NEXT:    v_lshrrev_b64 v[17:18], 1, v[8:9]
+; GFX9-NEXT:    v_lshlrev_b64 v[21:22], s7, v[10:11]
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    v_or_b32_e32 v19, v17, v21
+; GFX9-NEXT:    v_or_b32_e32 v21, v18, v22
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_and_b32 s4, 1, s4
+; GFX9-NEXT:    v_lshrrev_b64 v[17:18], s8, v[10:11]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s5
+; GFX9-NEXT:    v_lshrrev_b64 v[10:11], 1, v[10:11]
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v16
+; GFX9-NEXT:    v_cndmask_b32_e32 v18, v18, v21, vcc
+; GFX9-NEXT:    v_and_b32_e32 v21, s6, v16
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v17, v17, v19, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v18, v9, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, 0, v11, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v16, 64, v21
+; GFX9-NEXT:    v_lshlrev_b64 v[16:17], v16, v[10:11]
+; GFX9-NEXT:    v_lshrrev_b64 v[18:19], v21, v[8:9]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v23
+; GFX9-NEXT:    v_or_b32_e32 v18, v18, v16
+; GFX9-NEXT:    v_subrev_u32_e32 v16, 64, v23
+; GFX9-NEXT:    v_or_b32_e32 v19, v19, v17
+; GFX9-NEXT:    v_lshlrev_b64 v[16:17], v16, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v23, v[0:1]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v23
+; GFX9-NEXT:    v_cndmask_b32_e32 v22, 0, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v16, v24, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v25, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v17, v0, v2, s[4:5]
+; GFX9-NEXT:    v_subrev_u32_e32 v0, 64, v21
+; GFX9-NEXT:    v_cndmask_b32_e64 v16, v16, v3, s[4:5]
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], v0, v[10:11]
+; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v21
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v18, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v18, 0, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v21, v[10:11]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v21
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v19, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, v0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, v1, s[4:5]
+; GFX9-NEXT:    v_or_b32_e32 v0, v22, v2
+; GFX9-NEXT:    v_or_b32_e32 v1, v18, v3
+; GFX9-NEXT:    v_or_b32_e32 v2, v17, v8
+; GFX9-NEXT:    v_or_b32_e32 v3, v16, v9
+; GFX9-NEXT:    v_and_b32_e32 v16, s6, v20
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v20
+; GFX9-NEXT:    v_and_b32_e32 v17, s6, v8
+; GFX9-NEXT:    v_sub_u32_e32 v8, 64, v16
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v8, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[10:11], v16, v[6:7]
+; GFX9-NEXT:    v_subrev_u32_e32 v18, 64, v16
+; GFX9-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX9-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX9-NEXT:    v_lshlrev_b64 v[8:9], v16, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[4:5], v18, v[4:5]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v18, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v19, 0, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    v_cndmask_b32_e32 v16, v4, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v20, v5, v7, vcc
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], 1, v[12:13]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], s7, v[14:15]
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_and_b32 s6, 1, s4
+; GFX9-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX9-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], s8, v[14:15]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX9-NEXT:    s_and_b32 s5, 1, s5
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], 1, v[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX9-NEXT:    s_and_b32 s4, 1, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v10, 64, v17
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v17, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[10:11], v10, v[6:7]
+; GFX9-NEXT:    v_subrev_u32_e32 v12, 64, v17
+; GFX9-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX9-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v17, v[6:7]
+; GFX9-NEXT:    v_lshrrev_b64 v[6:7], v12, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v17
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v17
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v7, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX9-NEXT:    v_or_b32_e32 v4, v18, v4
+; GFX9-NEXT:    v_or_b32_e32 v5, v19, v5
+; GFX9-NEXT:    v_or_b32_e32 v6, v16, v6
+; GFX9-NEXT:    v_or_b32_e32 v7, v20, v7
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshl_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v17, -1, v16
+; GFX10-NEXT:    s_movk_i32 s7, 0x7f
+; GFX10-NEXT:    s_sub_i32 s8, 64, 1
+; GFX10-NEXT:    v_and_b32_e32 v27, s7, v16
+; GFX10-NEXT:    v_lshlrev_b64 v[18:19], s8, v[10:11]
+; GFX10-NEXT:    v_and_b32_e32 v28, s7, v17
+; GFX10-NEXT:    v_lshrrev_b64 v[16:17], 1, v[8:9]
+; GFX10-NEXT:    s_sub_i32 s9, 1, 64
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[21:22], s9, v[10:11]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_or_b32_e32 v16, v16, v18
+; GFX10-NEXT:    v_or_b32_e32 v17, v17, v19
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_and_b32 s5, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s4, 1, s4
+; GFX10-NEXT:    v_mov_b32_e32 v29, v2
+; GFX10-NEXT:    v_mov_b32_e32 v30, v3
+; GFX10-NEXT:    v_sub_nc_u32_e32 v23, 64, v27
+; GFX10-NEXT:    v_lshrrev_b64 v[10:11], 1, v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v21, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v22, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s5
+; GFX10-NEXT:    v_lshrrev_b64 v[16:17], v23, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[18:19], v27, v[29:30]
+; GFX10-NEXT:    v_sub_nc_u32_e32 v25, 64, v28
+; GFX10-NEXT:    v_cndmask_b32_e32 v34, v21, v8, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v35, v22, v9, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, 0, v11, s4
+; GFX10-NEXT:    v_or_b32_e32 v18, v16, v18
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v31, 64, v27
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v16, 64, v28
+; GFX10-NEXT:    v_lshrrev_b64 v[23:24], v28, v[34:35]
+; GFX10-NEXT:    v_lshlrev_b64 v[25:26], v25, v[10:11]
+; GFX10-NEXT:    v_lshlrev_b64 v[21:22], v27, v[0:1]
+; GFX10-NEXT:    v_or_b32_e32 v19, v17, v19
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], v31, v[0:1]
+; GFX10-NEXT:    v_lshrrev_b64 v[16:17], v16, v[10:11]
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 64, v27
+; GFX10-NEXT:    v_or_b32_e32 v23, v23, v25
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s4, 64, v28
+; GFX10-NEXT:    v_or_b32_e32 v24, v24, v26
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 0, v28
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v0, v18, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, v1, v19, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], v28, v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v16, v16, v23, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v17, v24, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, 0, v21, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0, v22, vcc_lo
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v16, v34, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v10, v35, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, v0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v23, 0, v1, s4
+; GFX10-NEXT:    v_xor_b32_e32 v16, -1, v20
+; GFX10-NEXT:    v_or_b32_e32 v0, v21, v8
+; GFX10-NEXT:    v_or_b32_e32 v1, v11, v9
+; GFX10-NEXT:    v_lshrrev_b64 v[8:9], 1, v[12:13]
+; GFX10-NEXT:    v_lshlrev_b64 v[10:11], s8, v[14:15]
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 0, v27
+; GFX10-NEXT:    v_and_b32_e32 v27, s7, v16
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_lshrrev_b64 v[16:17], s9, v[14:15]
+; GFX10-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX10-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_and_b32 s5, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s4, 1, s4
+; GFX10-NEXT:    v_and_b32_e32 v24, s7, v20
+; GFX10-NEXT:    v_lshrrev_b64 v[8:9], 1, v[14:15]
+; GFX10-NEXT:    v_cndmask_b32_e64 v22, v19, v30, s6
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, v16, v10, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v31, v17, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v18, v29, s6
+; GFX10-NEXT:    v_sub_nc_u32_e32 v18, 64, v24
+; GFX10-NEXT:    v_lshlrev_b64 v[14:15], v24, v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v31, v13, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v19, v12, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, v9, s4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v31, 64, v27
+; GFX10-NEXT:    v_lshrrev_b64 v[35:36], v18, v[4:5]
+; GFX10-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 64, v24
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, 64, v27
+; GFX10-NEXT:    v_lshlrev_b64 v[16:17], v24, v[4:5]
+; GFX10-NEXT:    v_lshrrev_b64 v[18:19], v27, v[12:13]
+; GFX10-NEXT:    v_lshlrev_b64 v[20:21], v31, v[8:9]
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 64, v24
+; GFX10-NEXT:    v_lshlrev_b64 v[3:4], v3, v[4:5]
+; GFX10-NEXT:    v_or_b32_e32 v5, v36, v15
+; GFX10-NEXT:    v_or_b32_e32 v14, v35, v14
+; GFX10-NEXT:    v_lshrrev_b64 v[10:11], v10, v[8:9]
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, 0, v16, vcc_lo
+; GFX10-NEXT:    v_or_b32_e32 v16, v18, v20
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s4, 64, v27
+; GFX10-NEXT:    v_or_b32_e32 v18, v19, v21
+; GFX10-NEXT:    v_cndmask_b32_e32 v31, v3, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v4, v5, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[3:4], v27, v[8:9]
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 0, v24
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v11, v18, s4
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 0, v27
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0, v17, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v5, v7, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, 0, v4, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v19, v31, v6, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v10, v12, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, v3, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v13, s5
+; GFX10-NEXT:    v_or_b32_e32 v3, v22, v23
+; GFX10-NEXT:    v_or_b32_e32 v7, v14, v11
+; GFX10-NEXT:    v_or_b32_e32 v4, v15, v5
+; GFX10-NEXT:    v_or_b32_e32 v6, v19, v10
+; GFX10-NEXT:    v_or_b32_e32 v5, v9, v8
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
+  ret <2 x i128> %result
+}
+
+declare i7 @llvm.fshl.i7(i7, i7, i7) #0
+declare i8 @llvm.fshl.i8(i8, i8, i8) #0
+declare <2 x i8> @llvm.fshl.v2i8(<2 x i8>, <2 x i8>, <2 x i8>) #0
+declare <4 x i8> @llvm.fshl.v4i8(<4 x i8>, <4 x i8>, <4 x i8>) #0
+
+declare i16 @llvm.fshl.i16(i16, i16, i16) #0
+declare <2 x i16> @llvm.fshl.v2i16(<2 x i16>, <2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.fshl.v3i16(<3 x i16>, <3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.fshl.v4i16(<4 x i16>, <4 x i16>, <4 x i16>) #0
+declare <5 x i16> @llvm.fshl.v5i16(<5 x i16>, <5 x i16>, <5 x i16>) #0
+declare <6 x i16> @llvm.fshl.v6i16(<6 x i16>, <6 x i16>, <6 x i16>) #0
+declare <8 x i16> @llvm.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>) #0
+
+declare i24 @llvm.fshl.i24(i24, i24, i24) #0
+declare <2 x i24> @llvm.fshl.v2i24(<2 x i24>, <2 x i24>, <2 x i24>) #0
+
+declare i32 @llvm.fshl.i32(i32, i32, i32) #0
+declare <2 x i32> @llvm.fshl.v2i32(<2 x i32>, <2 x i32>, <2 x i32>) #0
+declare <3 x i32> @llvm.fshl.v3i32(<3 x i32>, <3 x i32>, <3 x i32>) #0
+declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) #0
+declare <5 x i32> @llvm.fshl.v5i32(<5 x i32>, <5 x i32>, <5 x i32>) #0
+declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>) #0
+
+declare i48 @llvm.fshl.i48(i48, i48, i48) #0
+
+declare i64 @llvm.fshl.i64(i64, i64, i64) #0
+declare <2 x i64> @llvm.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>) #0
+
+declare i128 @llvm.fshl.i128(i128, i128, i128) #0
+declare <2 x i128> @llvm.fshl.v2i128(<2 x i128>, <2 x i128>, <2 x i128>) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c1c8cc1363ef72f9fe518828eda68f7e3b26930f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -0,0 +1,7572 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -o - %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - %s | FileCheck -check-prefixes=GCN,GFX10 %s
+
+define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
+; GFX6-LABEL: s_fshr_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v0, 7
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_sub_i32 s3, 0, 7
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX6-NEXT:    s_movk_i32 s3, 0x7f
+; GFX6-NEXT:    s_and_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s1, s1, s3
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, 7
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 7, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 7, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, 6, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX6-NEXT:    v_lshl_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_lshr_b32_e32 v0, s1, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, 7
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    s_sub_i32 s3, 0, 7
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX8-NEXT:    s_movk_i32 s3, 0x7f
+; GFX8-NEXT:    s_and_b32 s2, s2, s3
+; GFX8-NEXT:    s_and_b32 s1, s1, s3
+; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, 7
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v1, vcc, 7, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v1, vcc, 7, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_sub_u16_e32 v1, 6, v0
+; GFX8-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX8-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v1, v1, s0
+; GFX8-NEXT:    v_lshrrev_b16_e64 v0, v0, s1
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v0, 7
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_sub_i32 s3, 0, 7
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX9-NEXT:    s_movk_i32 s3, 0x7f
+; GFX9-NEXT:    s_and_b32 s2, s2, s3
+; GFX9-NEXT:    s_and_b32 s1, s1, s3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 7
+; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v1, 7, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v1, 7, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 7, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_sub_u16_e32 v1, 6, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX9-NEXT:    v_lshlrev_b16_e64 v1, v1, s0
+; GFX9-NEXT:    v_lshrrev_b16_e64 v0, v0, s1
+; GFX9-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, 7
+; GFX10-NEXT:    s_sub_i32 s3, 0, 7
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX10-NEXT:    s_movk_i32 s3, 0x7f
+; GFX10-NEXT:    s_and_b32 s2, s2, s3
+; GFX10-NEXT:    s_and_b32 s1, s1, s3
+; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, 7
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 7, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 7, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 7, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 7, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_sub_nc_u16_e64 v1, 6, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, v0, s1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, v1, s0
+; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i7 @llvm.fshr.i7(i7 %lhs, i7 %rhs, i7 %amt)
+  ret i7 %result
+}
+
+define i7 @v_fshr_i7(i7 %lhs, i7 %rhs, i7 %amt) {
+; GFX6-LABEL: v_fshr_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v3, 7
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX6-NEXT:    s_sub_i32 s4, 0, 7
+; GFX6-NEXT:    v_and_b32_e32 v2, 0x7f, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX6-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX6-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX6-NEXT:    v_mov_b32_e32 v4, 0x7f
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX6-NEXT:    v_mul_lo_u32 v3, v3, 7
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 7, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 7, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 6, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX6-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v3, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v3, 7
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX8-NEXT:    s_sub_i32 s4, 0, 7
+; GFX8-NEXT:    v_and_b32_e32 v2, 0x7f, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX8-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX8-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX8-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0x7f
+; GFX8-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_mul_lo_u32 v3, v3, 7
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, 7, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, 7, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX8-NEXT:    v_sub_u16_e32 v3, 6, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX8-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v3, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v3, 7
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX9-NEXT:    s_sub_i32 s4, 0, 7
+; GFX9-NEXT:    v_and_b32_e32 v2, 0x7f, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX9-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x7f
+; GFX9-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, 7
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 7, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 7, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 7, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT:    v_sub_u16_e32 v3, 6, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX9-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, v3, v0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v3, 7
+; GFX10-NEXT:    s_sub_i32 s4, 0, 7
+; GFX10-NEXT:    v_and_b32_e32 v2, 0x7f, v2
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 1, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, 0x7f, v1
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX10-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v4
+; GFX10-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX10-NEXT:    v_mul_lo_u32 v3, v3, 7
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 7, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 7, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0x7f
+; GFX10-NEXT:    v_sub_nc_u16_e64 v4, 6, v2
+; GFX10-NEXT:    v_and_b32_e32 v2, v2, v3
+; GFX10-NEXT:    v_and_b32_e32 v7, v4, v3
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v2, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v7, v0
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i7 @llvm.fshr.i7(i7 %lhs, i7 %rhs, i7 %amt)
+  ret i7 %result
+}
+
+define amdgpu_ps i8 @s_fshr_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
+; GFX6-LABEL: s_fshr_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s3, s2, 7
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX6-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX8-NEXT:    s_and_b32 s3, s2, 7
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX9-NEXT:    s_and_b32 s3, s2, 7
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX10-NEXT:    s_and_b32 s3, s2, 7
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 %amt)
+  ret i8 %result
+}
+
+define i8 @v_fshr_i8(i8 %lhs, i8 %rhs, i8 %amt) {
+; GFX6-LABEL: v_fshr_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v3, 7, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v2, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v3, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v3, 7, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v2, v0
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v3, 7, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, v2, v0
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v3, -1, v2
+; GFX10-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 1, v0
+; GFX10-NEXT:    v_and_b32_e32 v3, 7, v3
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v2, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v3, v0
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 %amt)
+  ret i8 %result
+}
+
+define amdgpu_ps i8 @s_fshr_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
+; GFX6-LABEL: s_fshr_i8_4:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 4
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i8_4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 4
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i8_4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 4
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i8_4:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 4
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 4
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 4)
+  ret i8 %result
+}
+
+define i8 @v_fshr_i8_4(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_fshr_i8_4:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 4, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i8_4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 4
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 4, v0
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i8_4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 4, v0
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i8_4:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 4, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 4, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 4)
+  ret i8 %result
+}
+
+define amdgpu_ps i8 @s_fshr_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
+; GFX6-LABEL: s_fshr_i8_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 3
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 5
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i8_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 3
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 5
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i8_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 3
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 5
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i8_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s1, s1, 0xff
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 3
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 5
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 5)
+  ret i8 %result
+}
+
+define i8 @v_fshr_i8_5(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_fshr_i8_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 5, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i8_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 5
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 3, v0
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i8_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, 5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 3, v0
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i8_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 3, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 5, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 5)
+  ret i8 %result
+}
+
+define amdgpu_ps i16 @s_fshr_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 inreg %amt.arg) {
+; GFX6-LABEL: s_fshr_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s7, 0xff
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX6-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX6-NEXT:    s_and_b32 s6, s2, 7
+; GFX6-NEXT:    s_and_b32 s1, s1, s7
+; GFX6-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s6
+; GFX6-NEXT:    s_andn2_b32 s2, 7, s5
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s2, s3, s2
+; GFX6-NEXT:    s_and_b32 s1, s5, 7
+; GFX6-NEXT:    s_and_b32 s3, s4, s7
+; GFX6-NEXT:    s_lshr_b32 s1, s3, s1
+; GFX6-NEXT:    s_or_b32 s1, s2, s1
+; GFX6-NEXT:    s_and_b32 s1, s1, s7
+; GFX6-NEXT:    s_and_b32 s0, s0, s7
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX8-NEXT:    s_and_b32 s6, s2, 7
+; GFX8-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_movk_i32 s2, 0xff
+; GFX8-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX8-NEXT:    s_and_b32 s1, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_and_b32 s4, s4, s2
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s6
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s5, 7
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_andn2_b32 s5, 7, s5
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX8-NEXT:    s_lshl_b32 s3, s3, s5
+; GFX8-NEXT:    s_lshr_b32 s1, s4, s1
+; GFX8-NEXT:    s_or_b32 s1, s3, s1
+; GFX8-NEXT:    s_and_b32 s0, s0, s2
+; GFX8-NEXT:    s_and_b32 s1, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX9-NEXT:    s_and_b32 s6, s2, 7
+; GFX9-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_movk_i32 s2, 0xff
+; GFX9-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX9-NEXT:    s_and_b32 s1, s1, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_and_b32 s4, s4, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s6
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s5, 7
+; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX9-NEXT:    s_andn2_b32 s5, 7, s5
+; GFX9-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX9-NEXT:    s_lshl_b32 s3, s3, s5
+; GFX9-NEXT:    s_lshr_b32 s1, s4, s1
+; GFX9-NEXT:    s_or_b32 s1, s3, s1
+; GFX9-NEXT:    s_and_b32 s0, s0, s2
+; GFX9-NEXT:    s_and_b32 s1, s1, s2
+; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX10-NEXT:    s_movk_i32 s7, 0xff
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX10-NEXT:    s_and_b32 s4, s4, s7
+; GFX10-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX10-NEXT:    s_and_b32 s6, s2, 7
+; GFX10-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX10-NEXT:    s_and_b32 s1, s1, s7
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_and_b32 s2, s5, 7
+; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX10-NEXT:    s_andn2_b32 s5, 7, s5
+; GFX10-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s5
+; GFX10-NEXT:    s_lshr_b32 s2, s4, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s6
+; GFX10-NEXT:    s_or_b32 s2, s3, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_and_b32 s1, s2, s7
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    s_and_b32 s0, s0, s7
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %amt = bitcast i16 %amt.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.fshr.v2i8(<2 x i8> %lhs, <2 x i8> %rhs, <2 x i8> %amt)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define i16 @v_fshr_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
+; GFX6-LABEL: v_fshr_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX6-NEXT:    v_and_b32_e32 v6, 7, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX6-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v2, v0
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v5
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v6, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 1, v3
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_and_b32_e32 v1, 7, v5
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v1, v3
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX8-NEXT:    v_and_b32_e32 v6, 7, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v2, v0
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v5
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_e32 v1, 7, v5
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 1, v3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, v2, v3
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX8-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NEXT:    v_and_b32_e32 v6, 7, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX9-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, v2, v0
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v5
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v1, 7, v5
+; GFX9-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 1, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, v2, v3
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 8, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_and_b32_e32 v7, 7, v2
+; GFX10-NEXT:    v_xor_b32_e32 v6, -1, v3
+; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX10-NEXT:    v_and_b32_e32 v3, 7, v3
+; GFX10-NEXT:    v_and_b32_e32 v5, s4, v5
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 1, v4
+; GFX10-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX10-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 1, v0
+; GFX10-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX10-NEXT:    v_lshrrev_b16_e64 v3, v3, v5
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, v6, v4
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v7, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v2, v0
+; GFX10-NEXT:    v_or_b32_e32 v2, v4, v3
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, v2, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %amt = bitcast i16 %amt.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.fshr.v2i8(<2 x i8> %lhs, <2 x i8> %rhs, <2 x i8> %amt)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define amdgpu_ps i32 @s_fshr_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 inreg %amt.arg) {
+; GFX6-LABEL: s_fshr_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s13, 0xff
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX6-NEXT:    s_lshr_b32 s5, s0, 24
+; GFX6-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX6-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX6-NEXT:    s_lshr_b32 s8, s1, 24
+; GFX6-NEXT:    s_lshr_b32 s9, s2, 8
+; GFX6-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX6-NEXT:    s_lshr_b32 s11, s2, 24
+; GFX6-NEXT:    s_and_b32 s12, s2, 7
+; GFX6-NEXT:    s_and_b32 s1, s1, s13
+; GFX6-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s12
+; GFX6-NEXT:    s_andn2_b32 s2, 7, s9
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s2, s3, s2
+; GFX6-NEXT:    s_and_b32 s1, s9, 7
+; GFX6-NEXT:    s_and_b32 s3, s6, s13
+; GFX6-NEXT:    s_lshr_b32 s1, s3, s1
+; GFX6-NEXT:    s_andn2_b32 s3, 7, s10
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 1
+; GFX6-NEXT:    s_or_b32 s1, s2, s1
+; GFX6-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX6-NEXT:    s_and_b32 s2, s10, 7
+; GFX6-NEXT:    s_and_b32 s4, s7, s13
+; GFX6-NEXT:    s_lshr_b32 s2, s4, s2
+; GFX6-NEXT:    s_and_b32 s1, s1, s13
+; GFX6-NEXT:    s_or_b32 s2, s3, s2
+; GFX6-NEXT:    s_and_b32 s3, s11, 7
+; GFX6-NEXT:    s_andn2_b32 s4, 7, s11
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 1
+; GFX6-NEXT:    s_and_b32 s0, s0, s13
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s13
+; GFX6-NEXT:    s_lshl_b32 s4, s5, s4
+; GFX6-NEXT:    s_lshr_b32 s3, s8, s3
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s3, s4, s3
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s3, s13
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s13, 0xff
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s5, s0, 24
+; GFX8-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s1, 24
+; GFX8-NEXT:    s_and_b32 s1, s1, s13
+; GFX8-NEXT:    s_lshr_b32 s9, s2, 8
+; GFX8-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s2, 24
+; GFX8-NEXT:    s_and_b32 s12, s2, 7
+; GFX8-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_andn2_b32 s2, 7, s9
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s12
+; GFX8-NEXT:    s_lshl_b32 s2, s3, s2
+; GFX8-NEXT:    s_and_b32 s3, s6, s13
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s9, 7
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s3, s1
+; GFX8-NEXT:    s_andn2_b32 s3, 7, s10
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 1
+; GFX8-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX8-NEXT:    s_and_b32 s4, s7, s13
+; GFX8-NEXT:    s_or_b32 s1, s2, s1
+; GFX8-NEXT:    s_and_b32 s2, s10, 7
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s4, s2
+; GFX8-NEXT:    s_and_b32 s1, s1, s13
+; GFX8-NEXT:    s_or_b32 s2, s3, s2
+; GFX8-NEXT:    s_and_b32 s3, s11, 7
+; GFX8-NEXT:    s_andn2_b32 s4, 7, s11
+; GFX8-NEXT:    s_lshl_b32 s5, s5, 1
+; GFX8-NEXT:    s_and_b32 s0, s0, s13
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s2, s13
+; GFX8-NEXT:    s_lshl_b32 s4, s5, s4
+; GFX8-NEXT:    s_lshr_b32 s3, s8, s3
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_or_b32 s3, s4, s3
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s3, s13
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s13, 0xff
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s0, 24
+; GFX9-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX9-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s1, 24
+; GFX9-NEXT:    s_and_b32 s1, s1, s13
+; GFX9-NEXT:    s_lshr_b32 s9, s2, 8
+; GFX9-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s2, 24
+; GFX9-NEXT:    s_and_b32 s12, s2, 7
+; GFX9-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_andn2_b32 s2, 7, s9
+; GFX9-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s12
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s2
+; GFX9-NEXT:    s_and_b32 s3, s6, s13
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s9, 7
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s1, s3, s1
+; GFX9-NEXT:    s_andn2_b32 s3, 7, s10
+; GFX9-NEXT:    s_lshl_b32 s4, s4, 1
+; GFX9-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX9-NEXT:    s_and_b32 s4, s7, s13
+; GFX9-NEXT:    s_or_b32 s1, s2, s1
+; GFX9-NEXT:    s_and_b32 s2, s10, 7
+; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s2, s4, s2
+; GFX9-NEXT:    s_and_b32 s1, s1, s13
+; GFX9-NEXT:    s_or_b32 s2, s3, s2
+; GFX9-NEXT:    s_and_b32 s3, s11, 7
+; GFX9-NEXT:    s_andn2_b32 s4, 7, s11
+; GFX9-NEXT:    s_lshl_b32 s5, s5, 1
+; GFX9-NEXT:    s_and_b32 s0, s0, s13
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s2, s13
+; GFX9-NEXT:    s_lshl_b32 s4, s5, s4
+; GFX9-NEXT:    s_lshr_b32 s3, s8, s3
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX9-NEXT:    s_or_b32 s3, s4, s3
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s3, s13
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX10-NEXT:    s_movk_i32 s13, 0xff
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s5, s0, 24
+; GFX10-NEXT:    s_and_b32 s6, s6, s13
+; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s8, s1, 24
+; GFX10-NEXT:    s_and_b32 s1, s1, s13
+; GFX10-NEXT:    s_lshr_b32 s9, s2, 8
+; GFX10-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s11, s2, 24
+; GFX10-NEXT:    s_and_b32 s12, s2, 7
+; GFX10-NEXT:    s_andn2_b32 s2, 7, s2
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_and_b32 s2, s9, 7
+; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    s_andn2_b32 s9, 7, s9
+; GFX10-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s12
+; GFX10-NEXT:    s_lshr_b32 s2, s6, s2
+; GFX10-NEXT:    s_and_b32 s6, s7, s13
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s9
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_or_b32 s1, s3, s2
+; GFX10-NEXT:    s_and_b32 s2, s10, 7
+; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    s_andn2_b32 s3, 7, s10
+; GFX10-NEXT:    s_lshl_b32 s4, s4, 1
+; GFX10-NEXT:    s_lshr_b32 s2, s6, s2
+; GFX10-NEXT:    s_lshl_b32 s3, s4, s3
+; GFX10-NEXT:    s_andn2_b32 s4, 7, s11
+; GFX10-NEXT:    s_lshl_b32 s5, s5, 1
+; GFX10-NEXT:    s_and_b32 s6, s11, 7
+; GFX10-NEXT:    s_or_b32 s2, s3, s2
+; GFX10-NEXT:    s_and_b32 s1, s1, s13
+; GFX10-NEXT:    s_lshl_b32 s4, s5, s4
+; GFX10-NEXT:    s_lshr_b32 s5, s8, s6
+; GFX10-NEXT:    s_and_b32 s0, s0, s13
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX10-NEXT:    s_or_b32 s3, s4, s5
+; GFX10-NEXT:    s_and_b32 s2, s2, s13
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, 16
+; GFX10-NEXT:    s_and_b32 s2, s3, s13
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %amt = bitcast i32 %amt.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.fshr.v4i8(<4 x i8> %lhs, <4 x i8> %rhs, <4 x i8> %amt)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define i32 @v_fshr_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
+; GFX6-LABEL: v_fshr_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v9, 8, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v11, 24, v2
+; GFX6-NEXT:    v_and_b32_e32 v12, 7, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v8, 24, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v2, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v12, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, 7, v9
+; GFX6-NEXT:    v_xor_b32_e32 v9, -1, v9
+; GFX6-NEXT:    v_and_b32_e32 v6, s4, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v1, v6
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v10
+; GFX6-NEXT:    v_and_b32_e32 v9, 7, v9
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 1, v3
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, v9, v3
+; GFX6-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v3, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, v6, v4
+; GFX6-NEXT:    v_and_b32_e32 v3, 7, v10
+; GFX6-NEXT:    v_and_b32_e32 v6, v7, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, v3, v6
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v11
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT:    v_and_b32_e32 v4, 7, v11
+; GFX6-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 1, v5
+; GFX6-NEXT:    v_and_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, v3, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, v6, v5
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v4, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v4, v5, v4
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, v4, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v8, 7, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v9, 1, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, v2, v9
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v8, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v8
+; GFX8-NEXT:    v_and_b32_e32 v8, 7, v5
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v5
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX8-NEXT:    v_and_b32_e32 v5, 7, v5
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 1, v3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, v5, v3
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v6
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    v_and_b32_e32 v4, 7, v6
+; GFX8-NEXT:    v_mov_b32_e32 v6, 1
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v8, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_and_b32_e32 v5, 7, v5
+; GFX8-NEXT:    v_mov_b32_e32 v9, 0xff
+; GFX8-NEXT:    v_lshlrev_b16_e32 v5, v5, v8
+; GFX8-NEXT:    v_and_b32_sdwa v8, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b16_e32 v4, v4, v8
+; GFX8-NEXT:    v_or_b32_e32 v4, v5, v4
+; GFX8-NEXT:    v_and_b32_e32 v5, 7, v7
+; GFX8-NEXT:    v_xor_b32_e32 v7, -1, v7
+; GFX8-NEXT:    v_and_b32_e32 v7, 7, v7
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v7, v0
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 8
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX8-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v8, 7, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v9, 1, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, v2, v9
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v8, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX9-NEXT:    v_or_b32_e32 v2, v2, v8
+; GFX9-NEXT:    v_and_b32_e32 v8, 7, v5
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v5
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT:    v_and_b32_e32 v5, 7, v5
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 1, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, v5, v3
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v6
+; GFX9-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX9-NEXT:    v_and_b32_e32 v4, 7, v6
+; GFX9-NEXT:    v_mov_b32_e32 v6, 1
+; GFX9-NEXT:    v_lshlrev_b16_sdwa v8, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_and_b32_e32 v5, 7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0xff
+; GFX9-NEXT:    v_lshlrev_b16_e32 v5, v5, v8
+; GFX9-NEXT:    v_and_b32_sdwa v8, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b16_e32 v4, v4, v8
+; GFX9-NEXT:    v_or_b32_e32 v4, v5, v4
+; GFX9-NEXT:    v_and_b32_e32 v5, 7, v7
+; GFX9-NEXT:    v_xor_b32_e32 v7, -1, v7
+; GFX9-NEXT:    v_and_b32_e32 v7, 7, v7
+; GFX9-NEXT:    v_lshlrev_b16_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, v7, v0
+; GFX9-NEXT:    v_lshrrev_b16_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    v_mov_b32_e32 v1, 8
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX9-NEXT:    v_and_or_b32 v1, v2, s4, v1
+; GFX9-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX9-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 8, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
+; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v6
+; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 24, v2
+; GFX10-NEXT:    v_and_b32_e32 v15, 7, v8
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 1, v0
+; GFX10-NEXT:    v_and_b32_e32 v14, 7, v11
+; GFX10-NEXT:    v_lshlrev_b16_e64 v3, 1, v3
+; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v10
+; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 8, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v15, v0
+; GFX10-NEXT:    v_mov_b32_e32 v15, 0xff
+; GFX10-NEXT:    v_lshlrev_b16_e64 v3, v14, v3
+; GFX10-NEXT:    v_xor_b32_e32 v14, -1, v12
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 24, v1
+; GFX10-NEXT:    v_and_b32_e32 v8, s4, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, v1, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_e32 v10, 7, v10
+; GFX10-NEXT:    v_and_b32_e32 v6, 7, v6
+; GFX10-NEXT:    v_and_b32_e32 v7, s4, v7
+; GFX10-NEXT:    v_and_b32_e32 v11, 7, v11
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 1, v4
+; GFX10-NEXT:    v_and_b32_e32 v15, 7, v14
+; GFX10-NEXT:    v_lshlrev_b16_e64 v5, 1, v5
+; GFX10-NEXT:    v_and_b32_e32 v12, 7, v12
+; GFX10-NEXT:    v_and_b32_e32 v2, 7, v2
+; GFX10-NEXT:    v_lshrrev_b16_e64 v6, v6, v7
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, v11, v4
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v10, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v5, v15, v5
+; GFX10-NEXT:    v_lshrrev_b16_e64 v7, v12, v9
+; GFX10-NEXT:    v_lshrrev_b16_e64 v2, v2, v8
+; GFX10-NEXT:    v_or_b32_e32 v3, v3, v6
+; GFX10-NEXT:    v_or_b32_e32 v1, v4, v1
+; GFX10-NEXT:    v_mov_b32_e32 v6, 8
+; GFX10-NEXT:    v_or_b32_e32 v4, v5, v7
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX10-NEXT:    v_lshlrev_b32_sdwa v2, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT:    v_and_b32_e32 v3, s4, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_and_or_b32 v0, v0, s4, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %amt = bitcast i32 %amt.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.fshr.v4i8(<4 x i8> %lhs, <4 x i8> %rhs, <4 x i8> %amt)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define amdgpu_ps i24 @s_fshr_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt) {
+; GFX6-LABEL: s_fshr_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_sub_i32 s3, 0, 24
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX6-NEXT:    s_mov_b32 s3, 0xffffff
+; GFX6-NEXT:    s_and_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s1, s1, s3
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 24, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 24, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, 23, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX6-NEXT:    v_lshl_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_lshr_b32_e32 v0, s1, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    s_sub_i32 s3, 0, 24
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX8-NEXT:    s_mov_b32 s3, 0xffffff
+; GFX8-NEXT:    s_and_b32 s2, s2, s3
+; GFX8-NEXT:    s_and_b32 s1, s1, s3
+; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v1, vcc, 24, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v1, vcc, 24, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, 23, v0
+; GFX8-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX8-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX8-NEXT:    v_lshlrev_b32_e64 v1, v1, s0
+; GFX8-NEXT:    v_lshrrev_b32_e64 v0, v0, s1
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_sub_i32 s3, 0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX9-NEXT:    s_mov_b32 s3, 0xffffff
+; GFX9-NEXT:    s_and_b32 s2, s2, s3
+; GFX9-NEXT:    s_and_b32 s1, s1, s3
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v1, 24, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v1, 24, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v1, 23, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX9-NEXT:    v_lshrrev_b32_e64 v0, v0, s1
+; GFX9-NEXT:    v_lshl_or_b32 v0, s0, v1, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX10-NEXT:    s_sub_i32 s3, 0, 24
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, s3, v0
+; GFX10-NEXT:    s_mov_b32 s3, 0xffffff
+; GFX10-NEXT:    s_and_b32 s2, s2, s3
+; GFX10-NEXT:    s_and_b32 s1, s1, s3
+; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 24, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 24, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 23, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, s3, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, s3, v1
+; GFX10-NEXT:    v_lshrrev_b32_e64 v0, v0, s1
+; GFX10-NEXT:    v_lshl_or_b32 v0, s0, v1, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i24 @llvm.fshr.i24(i24 %lhs, i24 %rhs, i24 %amt)
+  ret i24 %result
+}
+
+define i24 @v_fshr_i24(i24 %lhs, i24 %rhs, i24 %amt) {
+; GFX6-LABEL: v_fshr_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v3, 24
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX6-NEXT:    s_sub_i32 s4, 0, 24
+; GFX6-NEXT:    v_and_b32_e32 v2, 0xffffff, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX6-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX6-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX6-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX6-NEXT:    v_mul_lo_u32 v3, v3, 24
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 24, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 24, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 23, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX6-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v3, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v3, 24
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX8-NEXT:    s_sub_i32 s4, 0, 24
+; GFX8-NEXT:    v_and_b32_e32 v2, 0xffffff, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX8-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX8-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX8-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_mul_lo_u32 v3, v3, 24
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, 24, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, 24, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, 23, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX8-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, v3, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v3, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX9-NEXT:    s_sub_i32 s4, 0, 24
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xffffff, v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX9-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX9-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, 24
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 24, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 24, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v3, 23, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX9-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, v3, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v3, 24
+; GFX10-NEXT:    s_sub_i32 s4, 0, 24
+; GFX10-NEXT:    v_and_b32_e32 v2, 0xffffff, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_lo_u32 v4, s4, v3
+; GFX10-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v4
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX10-NEXT:    v_mul_hi_u32 v3, v2, v3
+; GFX10-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX10-NEXT:    v_mul_lo_u32 v3, v3, 24
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 24, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 24, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 23, v2
+; GFX10-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX10-NEXT:    v_and_b32_e32 v3, v3, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX10-NEXT:    v_lshl_or_b32 v0, v0, v3, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i24 @llvm.fshr.i24(i24 %lhs, i24 %rhs, i24 %amt)
+  ret i24 %result
+}
+
+define amdgpu_ps i48 @s_fshr_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 inreg %amt.arg) {
+; GFX6-LABEL: s_fshr_v2i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s10, 0xff
+; GFX6-NEXT:    s_lshr_b32 s9, s1, 8
+; GFX6-NEXT:    s_and_b32 s1, s1, s10
+; GFX6-NEXT:    s_lshr_b32 s6, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s8, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s1, s8, s1
+; GFX6-NEXT:    s_and_b32 s6, s6, s10
+; GFX6-NEXT:    s_lshr_b32 s8, s2, 8
+; GFX6-NEXT:    s_and_b32 s8, s8, s10
+; GFX6-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX6-NEXT:    s_and_b32 s0, s0, s10
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s6
+; GFX6-NEXT:    s_and_b32 s6, s7, s10
+; GFX6-NEXT:    s_and_b32 s7, s9, s10
+; GFX6-NEXT:    s_lshr_b32 s9, s2, 16
+; GFX6-NEXT:    s_lshr_b32 s11, s2, 24
+; GFX6-NEXT:    s_and_b32 s2, s2, s10
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 8
+; GFX6-NEXT:    s_or_b32 s2, s2, s8
+; GFX6-NEXT:    s_and_b32 s8, s9, s10
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX6-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_lshr_b32 s12, s3, 8
+; GFX6-NEXT:    s_and_b32 s3, s3, s10
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 8
+; GFX6-NEXT:    s_or_b32 s2, s2, s8
+; GFX6-NEXT:    s_and_b32 s8, s12, s10
+; GFX6-NEXT:    s_or_b32 s3, s11, s3
+; GFX6-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX6-NEXT:    s_or_b32 s3, s3, s8
+; GFX6-NEXT:    s_lshr_b32 s8, s4, 8
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    s_and_b32 s8, s8, s10
+; GFX6-NEXT:    s_lshr_b32 s9, s4, 16
+; GFX6-NEXT:    s_lshr_b32 s11, s4, 24
+; GFX6-NEXT:    s_and_b32 s4, s4, s10
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 8
+; GFX6-NEXT:    s_or_b32 s4, s4, s8
+; GFX6-NEXT:    s_and_b32 s8, s9, s10
+; GFX6-NEXT:    s_sub_i32 s9, 0, 24
+; GFX6-NEXT:    v_mul_lo_u32 v1, s9, v0
+; GFX6-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT:    s_or_b32 s4, s4, s8
+; GFX6-NEXT:    s_lshr_b32 s12, s5, 8
+; GFX6-NEXT:    s_and_b32 s5, s5, s10
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v1, 24
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 8
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX6-NEXT:    s_and_b32 s8, s12, s10
+; GFX6-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v0
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v2, s9, v1
+; GFX6-NEXT:    s_or_b32 s5, s11, s5
+; GFX6-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX6-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX6-NEXT:    s_or_b32 s5, s5, s8
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX6-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX6-NEXT:    s_mov_b32 s8, 0xffffff
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 23, v0
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, 24
+; GFX6-NEXT:    s_lshl_b32 s4, s6, 17
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    v_and_b32_e32 v0, s8, v0
+; GFX6-NEXT:    s_or_b32 s0, s4, s0
+; GFX6-NEXT:    v_and_b32_e32 v2, s8, v3
+; GFX6-NEXT:    v_lshl_b32_e32 v2, s0, v2
+; GFX6-NEXT:    v_lshr_b32_e32 v0, s2, v0
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s5, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v1
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 24, v1
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX6-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 23, v1
+; GFX6-NEXT:    s_lshl_b32 s0, s7, 17
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 1
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX6-NEXT:    v_lshl_b32_e32 v2, s0, v2
+; GFX6-NEXT:    v_lshr_b32_e32 v1, s3, v1
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_and_b32_e32 v2, s10, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, s10, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s10, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s10, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s10, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s10, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_v2i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s10, 0xff
+; GFX8-NEXT:    s_lshr_b32 s9, s1, 8
+; GFX8-NEXT:    s_bfe_u32 s11, 8, 0x100000
+; GFX8-NEXT:    s_and_b32 s1, s1, s10
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s8, s0, 24
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s11
+; GFX8-NEXT:    s_or_b32 s1, s8, s1
+; GFX8-NEXT:    s_and_b32 s6, s6, s10
+; GFX8-NEXT:    s_lshr_b32 s8, s2, 8
+; GFX8-NEXT:    s_and_b32 s8, s8, s10
+; GFX8-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX8-NEXT:    s_and_b32 s0, s0, s10
+; GFX8-NEXT:    s_lshl_b32 s6, s6, s11
+; GFX8-NEXT:    s_or_b32 s0, s0, s6
+; GFX8-NEXT:    s_and_b32 s6, s7, s10
+; GFX8-NEXT:    s_and_b32 s7, s9, s10
+; GFX8-NEXT:    s_lshr_b32 s9, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s12, s2, 24
+; GFX8-NEXT:    s_and_b32 s2, s2, s10
+; GFX8-NEXT:    s_lshl_b32 s8, s8, s11
+; GFX8-NEXT:    s_or_b32 s2, s2, s8
+; GFX8-NEXT:    s_and_b32 s8, s9, s10
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX8-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT:    s_lshr_b32 s13, s3, 8
+; GFX8-NEXT:    s_and_b32 s3, s3, s10
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX8-NEXT:    s_lshl_b32 s3, s3, s11
+; GFX8-NEXT:    s_or_b32 s2, s2, s8
+; GFX8-NEXT:    s_and_b32 s8, s13, s10
+; GFX8-NEXT:    s_or_b32 s3, s12, s3
+; GFX8-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX8-NEXT:    s_or_b32 s3, s3, s8
+; GFX8-NEXT:    s_lshr_b32 s8, s4, 8
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT:    s_and_b32 s8, s8, s10
+; GFX8-NEXT:    s_lshr_b32 s9, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s12, s4, 24
+; GFX8-NEXT:    s_and_b32 s4, s4, s10
+; GFX8-NEXT:    s_lshl_b32 s8, s8, s11
+; GFX8-NEXT:    s_or_b32 s4, s4, s8
+; GFX8-NEXT:    s_and_b32 s8, s9, s10
+; GFX8-NEXT:    s_sub_i32 s9, 0, 24
+; GFX8-NEXT:    v_mul_lo_u32 v1, s9, v0
+; GFX8-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX8-NEXT:    s_or_b32 s4, s4, s8
+; GFX8-NEXT:    s_lshr_b32 s13, s5, 8
+; GFX8-NEXT:    s_and_b32 s5, s5, s10
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v1, 24
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX8-NEXT:    s_lshl_b32 s5, s5, s11
+; GFX8-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX8-NEXT:    s_and_b32 s8, s13, s10
+; GFX8-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s4, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 24, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 24, v0
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_mul_lo_u32 v2, s9, v1
+; GFX8-NEXT:    s_or_b32 s5, s12, s5
+; GFX8-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GFX8-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX8-NEXT:    s_or_b32 s5, s5, s8
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_mov_b32 s8, 0xffffff
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, 23, v0
+; GFX8-NEXT:    v_mul_lo_u32 v1, v1, 24
+; GFX8-NEXT:    s_lshl_b32 s4, s6, 17
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX8-NEXT:    v_and_b32_e32 v0, s8, v0
+; GFX8-NEXT:    s_or_b32 s0, s4, s0
+; GFX8-NEXT:    v_and_b32_e32 v2, s8, v3
+; GFX8-NEXT:    v_lshlrev_b32_e64 v2, v2, s0
+; GFX8-NEXT:    v_lshrrev_b32_e64 v0, v0, s2
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, s5, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 24, v1
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 24, v1
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 23, v1
+; GFX8-NEXT:    s_lshl_b32 s0, s7, 17
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 1
+; GFX8-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v4
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    v_lshlrev_b32_e64 v2, v2, s0
+; GFX8-NEXT:    v_lshrrev_b32_e64 v1, v1, s3
+; GFX8-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX8-NEXT:    v_mov_b32_e32 v4, 8
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s10
+; GFX8-NEXT:    v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT:    v_and_b32_e32 v2, s10, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_v2i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s11, 0xff
+; GFX9-NEXT:    s_lshr_b32 s10, s1, 8
+; GFX9-NEXT:    s_bfe_u32 s12, 8, 0x100000
+; GFX9-NEXT:    s_and_b32 s1, s1, s11
+; GFX9-NEXT:    s_lshr_b32 s7, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s9, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s12
+; GFX9-NEXT:    s_or_b32 s1, s9, s1
+; GFX9-NEXT:    s_and_b32 s7, s7, s11
+; GFX9-NEXT:    s_lshr_b32 s9, s2, 8
+; GFX9-NEXT:    s_and_b32 s9, s9, s11
+; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX9-NEXT:    s_and_b32 s0, s0, s11
+; GFX9-NEXT:    s_lshl_b32 s7, s7, s12
+; GFX9-NEXT:    s_or_b32 s0, s0, s7
+; GFX9-NEXT:    s_and_b32 s7, s8, s11
+; GFX9-NEXT:    s_and_b32 s8, s10, s11
+; GFX9-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s13, s2, 24
+; GFX9-NEXT:    s_and_b32 s2, s2, s11
+; GFX9-NEXT:    s_lshl_b32 s9, s9, s12
+; GFX9-NEXT:    s_or_b32 s2, s2, s9
+; GFX9-NEXT:    s_and_b32 s9, s10, s11
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX9-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_lshr_b32 s14, s3, 8
+; GFX9-NEXT:    s_and_b32 s3, s3, s11
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s9, s9, 16
+; GFX9-NEXT:    s_lshl_b32 s3, s3, s12
+; GFX9-NEXT:    s_or_b32 s2, s2, s9
+; GFX9-NEXT:    s_and_b32 s9, s14, s11
+; GFX9-NEXT:    s_or_b32 s3, s13, s3
+; GFX9-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s9, s9, 16
+; GFX9-NEXT:    s_or_b32 s3, s3, s9
+; GFX9-NEXT:    s_lshr_b32 s9, s4, 8
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    s_and_b32 s9, s9, s11
+; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s13, s4, 24
+; GFX9-NEXT:    s_and_b32 s4, s4, s11
+; GFX9-NEXT:    s_lshl_b32 s9, s9, s12
+; GFX9-NEXT:    s_or_b32 s4, s4, s9
+; GFX9-NEXT:    s_and_b32 s9, s10, s11
+; GFX9-NEXT:    s_sub_i32 s10, 0, 24
+; GFX9-NEXT:    v_mul_lo_u32 v1, s10, v0
+; GFX9-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s9, s9, 16
+; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT:    s_or_b32 s4, s4, s9
+; GFX9-NEXT:    s_lshr_b32 s14, s5, 8
+; GFX9-NEXT:    s_and_b32 s5, s5, s11
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v1, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX9-NEXT:    s_lshl_b32 s5, s5, s12
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX9-NEXT:    s_and_b32 s9, s14, s11
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v2, 24, v0
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GFX9-NEXT:    s_or_b32 s5, s13, s5
+; GFX9-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX9-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GFX9-NEXT:    s_lshl_b32 s9, s9, 16
+; GFX9-NEXT:    s_or_b32 s5, s5, s9
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 24, v0
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
+; GFX9-NEXT:    v_mul_hi_u32 v1, s5, v1
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, 24
+; GFX9-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX9-NEXT:    s_mov_b32 s9, 0xffffff
+; GFX9-NEXT:    v_sub_u32_e32 v3, 23, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, s9, v0
+; GFX9-NEXT:    s_lshl_b32 s4, s7, 17
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX9-NEXT:    v_sub_u32_e32 v1, s5, v1
+; GFX9-NEXT:    s_or_b32 s0, s4, s0
+; GFX9-NEXT:    v_and_b32_e32 v3, s9, v3
+; GFX9-NEXT:    v_lshrrev_b32_e64 v0, v0, s2
+; GFX9-NEXT:    v_lshl_or_b32 v0, s0, v3, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 24, v1
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 24, v1
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v1
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffffff
+; GFX9-NEXT:    v_sub_u32_e32 v3, 23, v1
+; GFX9-NEXT:    v_and_b32_e32 v1, v1, v2
+; GFX9-NEXT:    s_lshl_b32 s0, s8, 17
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 1
+; GFX9-NEXT:    v_and_b32_e32 v3, v3, v2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    v_lshrrev_b32_e64 v1, v1, s3
+; GFX9-NEXT:    v_lshl_or_b32 v1, s0, v3, v1
+; GFX9-NEXT:    s_mov_b32 s6, 8
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_sdwa v2, s6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_and_b32_e32 v4, s11, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_and_or_b32 v2, v0, s11, v2
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 24, v4
+; GFX9-NEXT:    v_or3_b32 v0, v2, v0, v4
+; GFX9-NEXT:    v_and_or_b32 v1, v3, s11, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_v2i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, 24
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, 24
+; GFX10-NEXT:    s_sub_i32 s12, 0, 24
+; GFX10-NEXT:    s_movk_i32 s9, 0xff
+; GFX10-NEXT:    s_lshr_b32 s14, s4, 8
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GFX10-NEXT:    s_lshr_b32 s15, s4, 16
+; GFX10-NEXT:    s_bfe_u32 s10, 8, 0x100000
+; GFX10-NEXT:    s_and_b32 s14, s14, s9
+; GFX10-NEXT:    s_and_b32 s16, s4, s9
+; GFX10-NEXT:    s_lshl_b32 s14, s14, s10
+; GFX10-NEXT:    s_and_b32 s15, s15, s9
+; GFX10-NEXT:    s_or_b32 s14, s16, s14
+; GFX10-NEXT:    s_lshr_b32 s4, s4, 24
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; GFX10-NEXT:    s_bfe_u32 s14, s14, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s6, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s11, s1, 8
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX10-NEXT:    s_and_b32 s1, s1, s9
+; GFX10-NEXT:    s_and_b32 s6, s6, s9
+; GFX10-NEXT:    s_lshr_b32 s8, s0, 24
+; GFX10-NEXT:    v_mul_lo_u32 v2, s12, v0
+; GFX10-NEXT:    v_mul_lo_u32 v3, s12, v1
+; GFX10-NEXT:    s_bfe_u32 s12, s15, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s15, s5, 8
+; GFX10-NEXT:    s_lshl_b32 s12, s12, 16
+; GFX10-NEXT:    s_and_b32 s5, s5, s9
+; GFX10-NEXT:    s_or_b32 s12, s14, s12
+; GFX10-NEXT:    s_lshl_b32 s5, s5, s10
+; GFX10-NEXT:    v_mul_hi_u32 v2, v0, v2
+; GFX10-NEXT:    s_and_b32 s14, s15, s9
+; GFX10-NEXT:    s_or_b32 s4, s4, s5
+; GFX10-NEXT:    s_bfe_u32 s5, s14, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s10
+; GFX10-NEXT:    s_or_b32 s4, s4, s5
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v2
+; GFX10-NEXT:    v_mul_hi_u32 v2, v1, v3
+; GFX10-NEXT:    s_or_b32 s1, s8, s1
+; GFX10-NEXT:    s_lshr_b32 s8, s2, 8
+; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX10-NEXT:    v_mul_hi_u32 v0, s12, v0
+; GFX10-NEXT:    s_and_b32 s0, s0, s9
+; GFX10-NEXT:    s_lshl_b32 s6, s6, s10
+; GFX10-NEXT:    s_and_b32 s8, s8, s9
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v2
+; GFX10-NEXT:    s_or_b32 s0, s0, s6
+; GFX10-NEXT:    s_and_b32 s6, s7, s9
+; GFX10-NEXT:    s_and_b32 s7, s11, s9
+; GFX10-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GFX10-NEXT:    v_mul_hi_u32 v1, s4, v1
+; GFX10-NEXT:    s_lshr_b32 s11, s2, 16
+; GFX10-NEXT:    s_and_b32 s13, s2, s9
+; GFX10-NEXT:    s_lshl_b32 s5, s8, s10
+; GFX10-NEXT:    s_and_b32 s8, s11, s9
+; GFX10-NEXT:    s_lshr_b32 s11, s3, 8
+; GFX10-NEXT:    s_and_b32 s3, s3, s9
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s12, v0
+; GFX10-NEXT:    v_mul_lo_u32 v1, v1, 24
+; GFX10-NEXT:    s_or_b32 s5, s13, s5
+; GFX10-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 24, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s10
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffffff
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, s4, v1
+; GFX10-NEXT:    s_mov_b32 s4, 0xffffff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 24, v1
+; GFX10-NEXT:    s_or_b32 s2, s2, s3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 24, v0
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX10-NEXT:    s_and_b32 s3, s11, s9
+; GFX10-NEXT:    s_or_b32 s5, s5, s8
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v1
+; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 23, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX10-NEXT:    s_lshl_b32 s6, s6, 17
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX10-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 24, v1
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v1
+; GFX10-NEXT:    v_lshrrev_b32_e64 v0, v0, s5
+; GFX10-NEXT:    s_or_b32 s0, s6, s0
+; GFX10-NEXT:    s_or_b32 s2, s2, s3
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX10-NEXT:    v_lshl_or_b32 v0, s0, v2, v0
+; GFX10-NEXT:    s_lshl_b32 s0, s7, 17
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 1
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 23, v1
+; GFX10-NEXT:    v_and_b32_e32 v1, v1, v4
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    v_and_b32_e32 v2, v3, v4
+; GFX10-NEXT:    v_lshrrev_b32_e64 v1, v1, s2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
+; GFX10-NEXT:    v_lshl_or_b32 v1, s0, v2, v1
+; GFX10-NEXT:    s_mov_b32 s0, 8
+; GFX10-NEXT:    v_lshlrev_b32_sdwa v2, s0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT:    v_and_b32_e32 v3, s9, v1
+; GFX10-NEXT:    v_and_b32_sdwa v4, v1, s9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    v_and_or_b32 v2, v0, s9, v2
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT:    v_or3_b32 v0, v2, v0, v3
+; GFX10-NEXT:    v_and_or_b32 v1, v1, s9, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i48 %lhs.arg to <2 x i24>
+  %rhs = bitcast i48 %rhs.arg to <2 x i24>
+  %amt = bitcast i48 %amt.arg to <2 x i24>
+  %result = call <2 x i24> @llvm.fshr.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
+  %cast.result = bitcast <2 x i24> %result to i48
+  ret i48 %cast.result
+}
+
+define <2 x i24> @v_fshr_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
+; GFX6-LABEL: v_fshr_v2i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v6, 24
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GFX6-NEXT:    s_sub_i32 s4, 0, 24
+; GFX6-NEXT:    v_and_b32_e32 v4, 0xffffff, v4
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v8, 24
+; GFX6-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_mul_lo_u32 v7, s4, v6
+; GFX6-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; GFX6-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v7, v8
+; GFX6-NEXT:    v_mov_b32_e32 v8, 0xffffff
+; GFX6-NEXT:    v_and_b32_e32 v5, v5, v8
+; GFX6-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX6-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v8
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, 24, v4
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, 24, v4
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v6, s4, v7
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, 23, v4
+; GFX6-NEXT:    v_and_b32_e32 v9, v9, v8
+; GFX6-NEXT:    v_and_b32_e32 v4, v4, v8
+; GFX6-NEXT:    v_mul_hi_u32 v6, v7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v9, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, v4, v2
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; GFX6-NEXT:    v_mul_hi_u32 v6, v5, v6
+; GFX6-NEXT:    v_and_b32_e32 v3, v3, v8
+; GFX6-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v5, v6
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, 24, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, 24, v2
+; GFX6-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, 23, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, v2, v8
+; GFX6-NEXT:    v_and_b32_e32 v4, v4, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, v4, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v2i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v6, 24
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GFX8-NEXT:    s_sub_i32 s4, 0, 24
+; GFX8-NEXT:    v_and_b32_e32 v4, 0xffffff, v4
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v8, 24
+; GFX8-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 1, v1
+; GFX8-NEXT:    v_mul_lo_u32 v7, s4, v6
+; GFX8-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v7
+; GFX8-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GFX8-NEXT:    v_rcp_iflag_f32_e32 v7, v8
+; GFX8-NEXT:    v_mov_b32_e32 v8, 0xffffff
+; GFX8-NEXT:    v_and_b32_e32 v5, v5, v8
+; GFX8-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX8-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
+; GFX8-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v8
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v4, v6
+; GFX8-NEXT:    v_subrev_u32_e32 v6, vcc, 24, v4
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v6, vcc, 24, v4
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_mul_lo_u32 v6, s4, v7
+; GFX8-NEXT:    v_sub_u32_e32 v9, vcc, 23, v4
+; GFX8-NEXT:    v_and_b32_e32 v9, v9, v8
+; GFX8-NEXT:    v_and_b32_e32 v4, v4, v8
+; GFX8-NEXT:    v_mul_hi_u32 v6, v7, v6
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, v9, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, v4, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v7, v6
+; GFX8-NEXT:    v_mul_hi_u32 v6, v5, v6
+; GFX8-NEXT:    v_and_b32_e32 v3, v3, v8
+; GFX8-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v5, v6
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, 24, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, 24, v2
+; GFX8-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, 23, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, v2, v8
+; GFX8-NEXT:    v_and_b32_e32 v4, v4, v8
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, v4, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, v2, v3
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v2i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v6, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GFX9-NEXT:    s_sub_i32 s4, 0, 24
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v8, 24
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GFX9-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GFX9-NEXT:    v_and_b32_e32 v4, 0xffffff, v4
+; GFX9-NEXT:    v_mul_f32_e32 v8, 0x4f7ffffe, v8
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0xffffff
+; GFX9-NEXT:    v_mul_lo_u32 v7, s4, v6
+; GFX9-NEXT:    v_and_b32_e32 v5, v5, v9
+; GFX9-NEXT:    v_and_b32_e32 v2, v2, v9
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_mul_hi_u32 v7, v6, v7
+; GFX9-NEXT:    v_and_b32_e32 v3, v3, v9
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_add_u32_e32 v6, v6, v7
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v7, v8
+; GFX9-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GFX9-NEXT:    v_mul_lo_u32 v8, s4, v7
+; GFX9-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX9-NEXT:    v_mul_hi_u32 v8, v7, v8
+; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v6
+; GFX9-NEXT:    v_subrev_u32_e32 v6, 24, v4
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX9-NEXT:    v_add_u32_e32 v7, v7, v8
+; GFX9-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v6, 24, v4
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v4
+; GFX9-NEXT:    v_mul_lo_u32 v7, v7, 24
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v6, 23, v4
+; GFX9-NEXT:    v_and_b32_e32 v4, v4, v9
+; GFX9-NEXT:    v_and_b32_e32 v6, v6, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, v4, v2
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, v6, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, v5, v7
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 24, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 24, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 24, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v4, 23, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, v2, v9
+; GFX9-NEXT:    v_and_b32_e32 v4, v4, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, v2, v3
+; GFX9-NEXT:    v_lshl_or_b32 v1, v1, v4, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v2i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v6, 24
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v7, 24
+; GFX10-NEXT:    s_sub_i32 s4, 0, 24
+; GFX10-NEXT:    v_mov_b32_e32 v12, 0xffffff
+; GFX10-NEXT:    v_and_b32_e32 v4, 0xffffff, v4
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v6, v6
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v7, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_and_b32_e32 v5, v5, v12
+; GFX10-NEXT:    v_and_b32_e32 v2, v2, v12
+; GFX10-NEXT:    v_and_b32_e32 v3, v3, v12
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX10-NEXT:    v_mul_f32_e32 v7, 0x4f7ffffe, v7
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GFX10-NEXT:    v_mul_lo_u32 v8, s4, v6
+; GFX10-NEXT:    v_mul_lo_u32 v9, s4, v7
+; GFX10-NEXT:    v_mul_hi_u32 v8, v6, v8
+; GFX10-NEXT:    v_mul_hi_u32 v9, v7, v9
+; GFX10-NEXT:    v_add_nc_u32_e32 v6, v6, v8
+; GFX10-NEXT:    v_add_nc_u32_e32 v7, v7, v9
+; GFX10-NEXT:    v_mul_hi_u32 v6, v4, v6
+; GFX10-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GFX10-NEXT:    v_mul_lo_u32 v6, v6, 24
+; GFX10-NEXT:    v_mul_lo_u32 v7, v7, 24
+; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v6
+; GFX10-NEXT:    v_sub_nc_u32_e32 v5, v5, v7
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, 24, v4
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, 24, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v5
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, 24, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, 24, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v4, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 24, v5
+; GFX10-NEXT:    v_and_b32_e32 v4, v11, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX10-NEXT:    v_sub_nc_u32_e32 v6, 23, v11
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, v4, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v7, 23, v5
+; GFX10-NEXT:    v_and_b32_e32 v5, v5, v12
+; GFX10-NEXT:    v_and_b32_e32 v11, v6, v12
+; GFX10-NEXT:    v_and_b32_e32 v4, v7, v12
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, v5, v3
+; GFX10-NEXT:    v_lshl_or_b32 v0, v0, v11, v2
+; GFX10-NEXT:    v_lshl_or_b32 v1, v1, v4, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i24> @llvm.fshr.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
+  ret <2 x i24> %result
+}
+
+define amdgpu_ps i32 @s_fshr_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
+; GFX6-LABEL: s_fshr_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s1, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  ret i32 %result
+}
+
+define amdgpu_ps i32 @s_fshr_i32_5(i32 inreg %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: s_fshr_i32_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, 5
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i32_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, 5
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i32_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, 5
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i32_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s1, 5
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 5)
+  ret i32 %result
+}
+
+define amdgpu_ps i32 @s_fshr_i32_8(i32 inreg %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: s_fshr_i32_8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, 8
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i32_8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, 8
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i32_8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, 8
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i32_8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s1, 8
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 8)
+  ret i32 %result
+}
+
+define i32 @v_fshr_i32(i32 %lhs, i32 %rhs, i32 %amt) {
+; GFX6-LABEL: v_fshr_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v1, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  ret i32 %result
+}
+
+define i32 @v_fshr_i32_5(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_fshr_i32_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v1, 5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i32_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v1, 5
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i32_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v1, 5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i32_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v1, 5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 5)
+  ret i32 %result
+}
+
+define i32 @v_fshr_i32_8(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_fshr_i32_8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v1, 8
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i32_8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v1, 8
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i32_8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v1, 8
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i32_8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v1, 8
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 8)
+  ret i32 %result
+}
+
+define amdgpu_ps float @v_fshr_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt) {
+; GFX6-LABEL: v_fshr_i32_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v1, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i32_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v1, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i32_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v1, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i32_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s1, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  %cast.result = bitcast i32 %result to float
+  ret float %cast.result
+}
+
+define amdgpu_ps float @v_fshr_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt) {
+; GFX6-LABEL: v_fshr_i32_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i32_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i32_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i32_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, v0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  %cast.result = bitcast i32 %result to float
+  ret float %cast.result
+}
+
+define amdgpu_ps float @v_fshr_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
+; GFX6-LABEL: v_fshr_i32_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i32_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i32_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_alignbit_b32 v0, s0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i32_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s1, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 %amt)
+  %cast.result = bitcast i32 %result to float
+  ret float %cast.result
+}
+
+define <2 x i32> @v_fshr_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
+; GFX6-LABEL: v_fshr_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v2, v4
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v3, v5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v2, v4
+; GFX8-NEXT:    v_alignbit_b32 v1, v1, v3, v5
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v2, v4
+; GFX9-NEXT:    v_alignbit_b32 v1, v1, v3, v5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v2, v4
+; GFX10-NEXT:    v_alignbit_b32 v1, v1, v3, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt)
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @v_fshr_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
+; GFX6-LABEL: v_fshr_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v3, v6
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v4, v7
+; GFX6-NEXT:    v_alignbit_b32 v2, v2, v5, v8
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v3, v6
+; GFX8-NEXT:    v_alignbit_b32 v1, v1, v4, v7
+; GFX8-NEXT:    v_alignbit_b32 v2, v2, v5, v8
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v3, v6
+; GFX9-NEXT:    v_alignbit_b32 v1, v1, v4, v7
+; GFX9-NEXT:    v_alignbit_b32 v2, v2, v5, v8
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v3, v6
+; GFX10-NEXT:    v_alignbit_b32 v1, v1, v4, v7
+; GFX10-NEXT:    v_alignbit_b32 v2, v2, v5, v8
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i32> @llvm.fshr.v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt)
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @v_fshr_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
+; GFX6-LABEL: v_fshr_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_alignbit_b32 v0, v0, v4, v8
+; GFX6-NEXT:    v_alignbit_b32 v1, v1, v5, v9
+; GFX6-NEXT:    v_alignbit_b32 v2, v2, v6, v10
+; GFX6-NEXT:    v_alignbit_b32 v3, v3, v7, v11
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_alignbit_b32 v0, v0, v4, v8
+; GFX8-NEXT:    v_alignbit_b32 v1, v1, v5, v9
+; GFX8-NEXT:    v_alignbit_b32 v2, v2, v6, v10
+; GFX8-NEXT:    v_alignbit_b32 v3, v3, v7, v11
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_alignbit_b32 v0, v0, v4, v8
+; GFX9-NEXT:    v_alignbit_b32 v1, v1, v5, v9
+; GFX9-NEXT:    v_alignbit_b32 v2, v2, v6, v10
+; GFX9-NEXT:    v_alignbit_b32 v3, v3, v7, v11
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v4, v8
+; GFX10-NEXT:    v_alignbit_b32 v1, v1, v5, v9
+; GFX10-NEXT:    v_alignbit_b32 v2, v2, v6, v10
+; GFX10-NEXT:    v_alignbit_b32 v3, v3, v7, v11
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt)
+  ret <4 x i32> %result
+}
+
+define amdgpu_ps i16 @s_fshr_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt) {
+; GFX6-LABEL: s_fshr_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s3, s2, 15
+; GFX6-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX6-NEXT:    s_bfe_u32 s2, s3, 0x100000
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s3, s2, 15
+; GFX8-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX8-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s3, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s3, s2, 15
+; GFX9-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX9-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, s3, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s3, s2, 15
+; GFX10-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX10-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  ret i16 %result
+}
+
+define amdgpu_ps i16 @s_fshr_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: s_fshr_i16_4:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 12
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 4
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i16_4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 12, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, 4, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i16_4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 12, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, 4, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i16_4:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 12, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, 4, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 4)
+  ret i16 %result
+}
+
+define amdgpu_ps i16 @s_fshr_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: s_fshr_i16_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 11
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 5
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i16_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 11, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, 5, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i16_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 11, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, 5, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i16_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 11, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, 5, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 5)
+  ret i16 %result
+}
+
+define i16 @v_fshr_i16(i16 %lhs, i16 %rhs, i16 %amt) {
+; GFX6-LABEL: v_fshr_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v3, 15, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v2, v0
+; GFX6-NEXT:    v_bfe_u32 v2, v3, 0, 16
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v3, 15, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v2, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, v3, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v3, 15, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, v2, v0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, v3, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v3, -1, v2
+; GFX10-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 1, v0
+; GFX10-NEXT:    v_and_b32_e32 v3, 15, v3
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, v2, v1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, v3, v0
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  ret i16 %result
+}
+
+define i16 @v_fshr_i16_4(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_fshr_i16_4:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 12, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 4, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i16_4:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 12, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 4, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i16_4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 12, v0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, 4, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i16_4:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 12, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 4, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 4)
+  ret i16 %result
+}
+
+define i16 @v_fshr_i16_5(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_fshr_i16_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 11, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 5, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i16_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 11, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 5, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i16_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 11, v0
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, 5, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i16_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 11, v0
+; GFX10-NEXT:    v_lshrrev_b16_e64 v1, 5, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 5)
+  ret i16 %result
+}
+
+define amdgpu_ps half @v_fshr_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt) {
+; GFX6-LABEL: v_fshr_i16_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_and_b32_e32 v1, 15, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshl_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    s_and_b32 s0, s1, 0xffff
+; GFX6-NEXT:    v_lshr_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i16_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_and_b32_e32 v1, 15, v0
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    s_bfe_u32 s2, 1, 0x100000
+; GFX8-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    v_lshlrev_b16_e64 v0, v0, s0
+; GFX8-NEXT:    v_lshrrev_b16_e64 v1, v1, s1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i16_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_and_b32_e32 v1, 15, v0
+; GFX9-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT:    s_bfe_u32 s2, 1, 0x100000
+; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    v_lshlrev_b16_e64 v0, v0, s0
+; GFX9-NEXT:    v_lshrrev_b16_e64 v1, v1, s1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i16_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX10-NEXT:    s_bfe_u32 s2, 1, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    v_and_b32_e32 v1, 15, v1
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, v0, s1
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, v1, s0
+; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  %cast.result = bitcast i16 %result to half
+  ret half %cast.result
+}
+
+define amdgpu_ps half @v_fshr_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt) {
+; GFX6-LABEL: v_fshr_i16_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s2, s1, 15
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s1
+; GFX6-NEXT:    s_bfe_u32 s1, s2, 0x100000
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, s1, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i16_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s2, s1, 15
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX8-NEXT:    s_bfe_u32 s3, 1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, s2, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i16_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s2, s1, 15
+; GFX9-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX9-NEXT:    s_bfe_u32 s3, 1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s1
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, s2, v0
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i16_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b32 s2, s1, 15
+; GFX10-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX10-NEXT:    s_bfe_u32 s3, 1, 0x100000
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, s2, v0
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  %cast.result = bitcast i16 %result to half
+  ret half %cast.result
+}
+
+define amdgpu_ps half @v_fshr_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt) {
+; GFX6-LABEL: v_fshr_i16_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b32 s2, s1, 15
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, s1, v0
+; GFX6-NEXT:    s_bfe_u32 s1, s2, 0x100000
+; GFX6-NEXT:    s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i16_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b32 s2, s1, 15
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, s1, v0
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i16_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s2, s1, 15
+; GFX9-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, s1, v0
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i16_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 1, v0
+; GFX10-NEXT:    s_andn2_b32 s2, 15, s1
+; GFX10-NEXT:    s_and_b32 s1, s1, 15
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, s2, v0
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 %amt)
+  %cast.result = bitcast i16 %result to half
+  ret half %cast.result
+}
+
+define amdgpu_ps i32 @s_fshr_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <2 x i16> inreg %amt) {
+; GFX6-LABEL: s_fshr_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s5, 0xffff
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX6-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX6-NEXT:    s_and_b32 s6, s1, s5
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s3, s3, s4
+; GFX6-NEXT:    s_bfe_u32 s7, 14, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s4, s1, 17
+; GFX6-NEXT:    s_lshr_b32 s6, s6, 1
+; GFX6-NEXT:    s_lshr_b32 s4, s4, s7
+; GFX6-NEXT:    s_lshr_b32 s6, s6, s7
+; GFX6-NEXT:    s_or_b32 s3, s3, s4
+; GFX6-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 1
+; GFX6-NEXT:    s_xor_b32 s2, s2, -1
+; GFX6-NEXT:    s_and_b32 s7, s2, 15
+; GFX6-NEXT:    s_and_b32 s1, s1, s5
+; GFX6-NEXT:    s_or_b32 s0, s0, s6
+; GFX6-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX6-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX6-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 1
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s7
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s6, 15
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 1
+; GFX6-NEXT:    s_andn2_b32 s2, 15, s6
+; GFX6-NEXT:    s_lshl_b32 s1, s3, s1
+; GFX6-NEXT:    s_and_b32 s3, s4, s5
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s2, s3, s2
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s5, 1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s6, s6, s5
+; GFX8-NEXT:    s_bfe_u32 s7, 14, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX8-NEXT:    s_lshr_b32 s6, s6, s7
+; GFX8-NEXT:    s_or_b32 s0, s0, s6
+; GFX8-NEXT:    s_lshr_b32 s6, s4, s5
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b32 s3, s3, s5
+; GFX8-NEXT:    s_lshr_b32 s6, s6, s7
+; GFX8-NEXT:    s_xor_b32 s2, s2, -1
+; GFX8-NEXT:    s_and_b32 s7, s2, 15
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_or_b32 s3, s3, s6
+; GFX8-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX8-NEXT:    s_andn2_b32 s2, 15, s2
+; GFX8-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s5
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s7
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s6, 15
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, s5
+; GFX8-NEXT:    s_andn2_b32 s2, 15, s6
+; GFX8-NEXT:    s_lshl_b32 s1, s3, s1
+; GFX8-NEXT:    s_bfe_u32 s3, s4, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s5
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s3, s2
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000f
+; GFX9-NEXT:    s_and_b32 s4, s2, s3
+; GFX9-NEXT:    s_andn2_b32 s2, s3, s2
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 0x10001
+; GFX9-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s5
+; GFX9-NEXT:    s_mov_b32 s3, 0xffff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s2, s1, 16
+; GFX9-NEXT:    s_and_b32 s1, s1, s3
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_and_b32 s3, s4, s3
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX10-NEXT:    s_mov_b32 s3, 0xf000f
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 0x10001
+; GFX10-NEXT:    s_lshl_b32 s4, s4, 1
+; GFX10-NEXT:    s_and_b32 s5, s2, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
+; GFX10-NEXT:    s_andn2_b32 s2, s3, s2
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s2, s3, s4
+; GFX10-NEXT:    s_mov_b32 s3, 0xffff
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX10-NEXT:    s_and_b32 s1, s1, s3
+; GFX10-NEXT:    s_and_b32 s3, s5, s3
+; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshr_b32 s3, s4, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  %cast = bitcast <2 x i16> %result to i32
+  ret i32 %cast
+}
+
+define <2 x i16> @v_fshr_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
+; GFX6-LABEL: v_fshr_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s5, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v4, s5, v1
+; GFX6-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    s_bfe_u32 s6, 14, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, s6, v4
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 17, v1
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, s4, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, s6, v4
+; GFX6-NEXT:    v_and_b32_e32 v6, 15, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX6-NEXT:    v_and_b32_e32 v1, s5, v1
+; GFX6-NEXT:    v_bfe_u32 v6, v6, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v6, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, 15, v5
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v5
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, v1, v3
+; GFX6-NEXT:    v_and_b32_e32 v3, s5, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b16_e32 v4, 1, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 1, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v4, 14, v4
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    v_mov_b32_e32 v4, 1
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v5, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v5, 14, v5
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v5
+; GFX8-NEXT:    v_lshlrev_b16_e32 v5, 1, v1
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_and_b32_e32 v6, 15, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX8-NEXT:    v_and_b32_e32 v2, 15, v2
+; GFX8-NEXT:    v_lshrrev_b16_e32 v5, 1, v5
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, v6, v3
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, v2, v5
+; GFX8-NEXT:    v_or_b32_e32 v2, v3, v2
+; GFX8-NEXT:    v_and_b32_e32 v3, 15, v4
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX8-NEXT:    v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 1, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v3, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, v4, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 0xf000f
+; GFX9-NEXT:    v_and_b32_e32 v3, s4, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, v2, v0
+; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v3, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v3, -1, v2
+; GFX10-NEXT:    s_mov_b32 s4, 0xf000f
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX10-NEXT:    v_and_b32_e32 v2, s4, v2
+; GFX10-NEXT:    v_and_b32_e32 v3, s4, v3
+; GFX10-NEXT:    v_pk_lshrrev_b16 v1, v2, v1
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, v3, v0
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  ret <2 x i16> %result
+}
+
+define <2 x i16> @v_fshr_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_fshr_v2i16_4_8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_sub_i32 s4, 0, 4
+; GFX6-NEXT:    s_and_b32 s6, s4, 15
+; GFX6-NEXT:    v_and_b32_e32 v3, 0xffff, v1
+; GFX6-NEXT:    s_xor_b32 s4, s4, -1
+; GFX6-NEXT:    s_sub_i32 s5, 0, 8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 1, v3
+; GFX6-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, s4, v3
+; GFX6-NEXT:    s_and_b32 s4, s5, 15
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX6-NEXT:    s_xor_b32 s5, s5, -1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 17, v1
+; GFX6-NEXT:    s_bfe_u32 s4, s5, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, s4, v1
+; GFX6-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, s6, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v2i16_4_8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_sub_i32 s4, 0, 4
+; GFX8-NEXT:    s_and_b32 s6, s4, 15
+; GFX8-NEXT:    s_sub_i32 s5, 0, 8
+; GFX8-NEXT:    s_xor_b32 s4, s4, -1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v3, 1, v1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v3, s4, v3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, s6, v0
+; GFX8-NEXT:    s_and_b32 s4, s5, 15
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v3, s4
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v3, 1
+; GFX8-NEXT:    s_xor_b32 s5, s5, -1
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, s5, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v2i16_4_8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v2, 16
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v3, 16
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX9-NEXT:    s_sub_i32 s4, 0, 16
+; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_lo_u32 v4, s4, v2
+; GFX9-NEXT:    v_mul_lo_u32 v5, s4, v3
+; GFX9-NEXT:    v_mul_hi_u32 v4, v2, v4
+; GFX9-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_mul_hi_u32 v2, 4, v2
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v5
+; GFX9-NEXT:    v_mul_hi_u32 v3, 8, v3
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, 4, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 16, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 16, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 4, v3
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 16, v2
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 16, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, 8, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 16, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 16, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 16, v3
+; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, 16, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v4, v3
+; GFX9-NEXT:    v_pk_sub_i16 v3, 16, v2 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, v3, v0
+; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v2, v1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v2i16_4_8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v2, 16
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v3, 16
+; GFX10-NEXT:    s_sub_i32 s4, 0, 16
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
+; GFX10-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX10-NEXT:    v_mul_lo_u32 v4, s4, v2
+; GFX10-NEXT:    v_mul_lo_u32 v5, s4, v3
+; GFX10-NEXT:    v_mul_hi_u32 v4, v2, v4
+; GFX10-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v5
+; GFX10-NEXT:    v_mul_hi_u32 v2, 8, v2
+; GFX10-NEXT:    v_mul_hi_u32 v3, 4, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 4, v3
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 8, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 4, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v4, 16, v2
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 16, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v5, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 16, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v4, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 16, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v5, 16, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX10-NEXT:    v_and_or_b32 v2, v3, 0xffff, v2
+; GFX10-NEXT:    v_pk_sub_i16 v3, 16, v2 op_sel_hi:[0,1]
+; GFX10-NEXT:    v_pk_lshrrev_b16 v1, v2, v1
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, v3, v0
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> <i16 4, i16 8>)
+  ret <2 x i16> %result
+}
+
+define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <2 x i16> %amt) {
+; GFX6-LABEL: v_fshr_v2i16_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    s_and_b32 s5, s1, s4
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX6-NEXT:    s_bfe_u32 s3, 1, 0x100000
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX6-NEXT:    s_lshl_b32 s2, s2, s3
+; GFX6-NEXT:    s_lshr_b32 s5, s5, 1
+; GFX6-NEXT:    s_bfe_u32 s6, 14, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s3, s1, 17
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v0
+; GFX6-NEXT:    s_lshr_b32 s5, s5, s6
+; GFX6-NEXT:    s_lshr_b32 s3, s3, s6
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    s_or_b32 s0, s0, s5
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 1
+; GFX6-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX6-NEXT:    v_lshl_b32_e32 v2, s0, v2
+; GFX6-NEXT:    s_and_b32 s0, s1, s4
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshr_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT:    v_and_b32_e32 v2, 15, v1
+; GFX6-NEXT:    v_xor_b32_e32 v1, -1, v1
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX6-NEXT:    v_and_b32_e32 v1, 15, v1
+; GFX6-NEXT:    s_and_b32 s0, s3, s4
+; GFX6-NEXT:    v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_lshl_b32_e32 v2, s2, v2
+; GFX6-NEXT:    v_lshr_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_v2i16_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s5, s5, s4
+; GFX8-NEXT:    s_bfe_u32 s6, 14, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshr_b32 s5, s5, s6
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s5
+; GFX8-NEXT:    v_and_b32_e32 v2, 15, v0
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v2, v2, s0
+; GFX8-NEXT:    s_bfe_u32 s0, s1, 0x100000
+; GFX8-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshr_b32 s5, s3, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v0, v0, s0
+; GFX8-NEXT:    s_lshl_b32 s3, s3, s4
+; GFX8-NEXT:    s_bfe_u32 s0, s3, 0x100000
+; GFX8-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT:    v_and_b32_e32 v2, 15, v1
+; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v1
+; GFX8-NEXT:    s_lshl_b32 s2, s2, s4
+; GFX8-NEXT:    s_lshr_b32 s5, s5, s6
+; GFX8-NEXT:    s_or_b32 s2, s2, s5
+; GFX8-NEXT:    v_and_b32_e32 v1, 15, v1
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
+; GFX8-NEXT:    v_lshlrev_b16_e64 v2, v2, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v1, v1, s0
+; GFX8-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_v2i16_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX9-NEXT:    v_and_b32_e32 v1, s2, v0
+; GFX9-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, s2, v0
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 0x10001
+; GFX9-NEXT:    s_lshl_b32 s2, s2, 1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, v0, s0
+; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v1, s1
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_v2i16_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    v_and_b32_e32 v0, s2, v0
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 0x10001
+; GFX10-NEXT:    v_and_b32_e32 v1, s2, v1
+; GFX10-NEXT:    s_lshl_b32 s2, s3, 1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT:    v_pk_lshrrev_b16 v0, v0, s1
+; GFX10-NEXT:    v_pk_lshlrev_b16 v1, v1, s0
+; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <2 x i16> inreg %amt) {
+; GFX6-LABEL: v_fshr_v2i16_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v0
+; GFX6-NEXT:    s_bfe_u32 s3, 1, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT:    s_bfe_u32 s5, 14, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 17, v0
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, s5, v1
+; GFX6-NEXT:    v_or_b32_e32 v1, s0, v1
+; GFX6-NEXT:    s_lshl_b32 s0, s2, s3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, s5, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_or_b32_e32 v2, s0, v2
+; GFX6-NEXT:    s_xor_b32 s0, s1, -1
+; GFX6-NEXT:    s_and_b32 s2, s0, 15
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX6-NEXT:    s_andn2_b32 s0, 15, s0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, s0, v0
+; GFX6-NEXT:    s_and_b32 s0, s1, 15
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, s2, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 1, v3
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, s0, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; GFX6-NEXT:    s_bfe_u32 s0, s1, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, s0, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_v2i16_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 1
+; GFX8-NEXT:    s_bfe_u32 s3, 1, 0x100000
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v3, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s3
+; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 14, v1
+; GFX8-NEXT:    v_or_b32_e32 v1, s0, v1
+; GFX8-NEXT:    s_lshl_b32 s0, s2, s3
+; GFX8-NEXT:    v_lshrrev_b16_e32 v3, 14, v3
+; GFX8-NEXT:    v_or_b32_e32 v3, s0, v3
+; GFX8-NEXT:    s_xor_b32 s0, s1, -1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 1, v0
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    s_and_b32 s2, s0, 15
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    s_andn2_b32 s0, 15, s0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, 1, v4
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, s0, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, s2, v1
+; GFX8-NEXT:    s_and_b32 s0, s1, 15
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 1, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, s0, v3
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, s1, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_v2i16_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX9-NEXT:    s_and_b32 s3, s1, s2
+; GFX9-NEXT:    s_andn2_b32 s1, s2, s1
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 0x10001
+; GFX9-NEXT:    s_lshl_b32 s2, s2, 1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s1
+; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT:    v_pk_lshrrev_b16 v0, s3, v0
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_v2i16_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 0x10001
+; GFX10-NEXT:    s_lshl_b32 s3, s3, 1
+; GFX10-NEXT:    s_and_b32 s4, s1, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT:    s_andn2_b32 s1, s2, s1
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX10-NEXT:    v_pk_lshrrev_b16 v0, s4, v0
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <2 x i16> inreg %amt) {
+; GFX6-LABEL: v_fshr_v2i16_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s3, 0xffff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT:    s_bfe_u32 s2, 1, 0x100000
+; GFX6-NEXT:    s_and_b32 s4, s0, s3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, s2, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, s2, v1
+; GFX6-NEXT:    s_bfe_u32 s5, 14, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 17
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX6-NEXT:    s_lshr_b32 s2, s2, s5
+; GFX6-NEXT:    s_lshr_b32 s4, s4, s5
+; GFX6-NEXT:    v_or_b32_e32 v1, s2, v1
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 1
+; GFX6-NEXT:    s_xor_b32 s1, s1, -1
+; GFX6-NEXT:    s_and_b32 s5, s1, 15
+; GFX6-NEXT:    s_and_b32 s0, s0, s3
+; GFX6-NEXT:    v_or_b32_e32 v0, s4, v0
+; GFX6-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX6-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, s5, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    s_and_b32 s0, s4, 15
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 1
+; GFX6-NEXT:    s_andn2_b32 s1, 15, s4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, s0, v1
+; GFX6-NEXT:    s_and_b32 s0, s2, s3
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 1
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX6-NEXT:    v_or_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT:    v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_v2i16_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s3, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s4
+; GFX8-NEXT:    s_bfe_u32 s5, 14, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 1, v0
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s5
+; GFX8-NEXT:    v_or_b32_e32 v1, s3, v1
+; GFX8-NEXT:    s_lshr_b32 s3, s2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, 1
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s5
+; GFX8-NEXT:    s_xor_b32 s1, s1, -1
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    v_or_b32_e32 v0, s3, v0
+; GFX8-NEXT:    s_and_b32 s5, s1, 15
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s1
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, s5, v1
+; GFX8-NEXT:    v_or_b32_e32 v1, s0, v1
+; GFX8-NEXT:    s_and_b32 s0, s3, 15
+; GFX8-NEXT:    s_lshl_b32 s2, s2, s4
+; GFX8-NEXT:    s_andn2_b32 s1, 15, s3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, s0, v0
+; GFX8-NEXT:    s_bfe_u32 s0, s2, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_v2i16_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX9-NEXT:    s_and_b32 s3, s1, s2
+; GFX9-NEXT:    s_andn2_b32 s1, s2, s1
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_mov_b32 s2, 0xffff
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, s1, v0
+; GFX9-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX9-NEXT:    s_and_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX9-NEXT:    s_and_b32 s2, s3, s2
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_v2i16_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX10-NEXT:    s_mov_b32 s2, 0xf000f
+; GFX10-NEXT:    s_mov_b32 s3, 0xffff
+; GFX10-NEXT:    s_and_b32 s4, s1, s2
+; GFX10-NEXT:    s_andn2_b32 s1, s2, s1
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, s1, v0
+; GFX10-NEXT:    s_and_b32 s0, s0, s3
+; GFX10-NEXT:    s_and_b32 s1, s4, s3
+; GFX10-NEXT:    s_lshr_b32 s3, s4, 16
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshr_b32 s1, s2, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+; ; FIXME
+; define amdgpu_ps i48 @s_fshr_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <3 x i16> inreg %amt) {
+;   %result = call <3 x i16> @llvm.fshr.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
+;   %cast = bitcast <3 x i16> %result to i48
+;   ret i48 %cast
+; }
+
+; ; FIXME
+; define <3 x half> @v_fshr_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt) {
+;   %result = call <3 x i16> @llvm.fshr.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
+;   %cast.result = bitcast <3 x i16> %result to <3 x half>
+;   ret <3 x half> %cast.result
+; }
+
+define amdgpu_ps <2 x i32> @s_fshr_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs, <4 x i16> inreg %amt) {
+; GFX6-LABEL: s_fshr_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s12, 0xffff
+; GFX6-NEXT:    s_lshl_b32 s9, s9, 16
+; GFX6-NEXT:    s_and_b32 s8, s8, s12
+; GFX6-NEXT:    s_or_b32 s8, s9, s8
+; GFX6-NEXT:    s_lshl_b32 s9, s11, 16
+; GFX6-NEXT:    s_and_b32 s11, s4, s12
+; GFX6-NEXT:    s_and_b32 s10, s10, s12
+; GFX6-NEXT:    s_or_b32 s9, s9, s10
+; GFX6-NEXT:    s_bfe_u32 s10, 1, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s11, s11, 1
+; GFX6-NEXT:    s_bfe_u32 s13, 14, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s10
+; GFX6-NEXT:    s_lshr_b32 s11, s11, s13
+; GFX6-NEXT:    s_or_b32 s0, s0, s11
+; GFX6-NEXT:    s_and_b32 s11, s5, s12
+; GFX6-NEXT:    s_lshr_b32 s11, s11, 1
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 1
+; GFX6-NEXT:    s_xor_b32 s8, s8, -1
+; GFX6-NEXT:    s_lshl_b32 s1, s1, s10
+; GFX6-NEXT:    s_lshr_b32 s11, s11, s13
+; GFX6-NEXT:    s_and_b32 s14, s8, 15
+; GFX6-NEXT:    s_and_b32 s4, s4, s12
+; GFX6-NEXT:    s_or_b32 s1, s1, s11
+; GFX6-NEXT:    s_lshr_b32 s11, s8, 16
+; GFX6-NEXT:    s_andn2_b32 s8, 15, s8
+; GFX6-NEXT:    s_bfe_u32 s14, s14, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX6-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s4, s4, s8
+; GFX6-NEXT:    s_lshl_b32 s0, s0, s14
+; GFX6-NEXT:    s_or_b32 s0, s0, s4
+; GFX6-NEXT:    s_and_b32 s4, s11, 15
+; GFX6-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 1
+; GFX6-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX6-NEXT:    s_and_b32 s4, s5, s12
+; GFX6-NEXT:    s_andn2_b32 s8, 15, s11
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 1
+; GFX6-NEXT:    s_bfe_u32 s5, s8, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s4, s4, s5
+; GFX6-NEXT:    s_or_b32 s1, s1, s4
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s1, s2, s10
+; GFX6-NEXT:    s_and_b32 s2, s6, s12
+; GFX6-NEXT:    s_lshr_b32 s2, s2, 1
+; GFX6-NEXT:    s_lshr_b32 s2, s2, s13
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_lshl_b32 s2, s3, s10
+; GFX6-NEXT:    s_and_b32 s3, s7, s12
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX6-NEXT:    s_lshr_b32 s3, s3, s13
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    s_lshl_b32 s3, s6, 1
+; GFX6-NEXT:    s_xor_b32 s5, s9, -1
+; GFX6-NEXT:    s_and_b32 s3, s3, s12
+; GFX6-NEXT:    s_lshl_b32 s4, s7, 1
+; GFX6-NEXT:    s_and_b32 s7, s5, 15
+; GFX6-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX6-NEXT:    s_andn2_b32 s5, 15, s5
+; GFX6-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX6-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s3, s3, s5
+; GFX6-NEXT:    s_lshl_b32 s1, s1, s7
+; GFX6-NEXT:    s_or_b32 s1, s1, s3
+; GFX6-NEXT:    s_and_b32 s3, s6, 15
+; GFX6-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s3, s4, s12
+; GFX6-NEXT:    s_andn2_b32 s5, 15, s6
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 1
+; GFX6-NEXT:    s_bfe_u32 s4, s5, 0x100000
+; GFX6-NEXT:    s_lshr_b32 s3, s3, s4
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX6-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s8, 1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s9, s2, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s9, s9, s8
+; GFX8-NEXT:    s_bfe_u32 s10, 14, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX8-NEXT:    s_lshr_b32 s9, s9, s10
+; GFX8-NEXT:    s_or_b32 s0, s0, s9
+; GFX8-NEXT:    s_lshr_b32 s9, s7, s8
+; GFX8-NEXT:    s_lshl_b32 s2, s2, s8
+; GFX8-NEXT:    s_xor_b32 s4, s4, -1
+; GFX8-NEXT:    s_lshl_b32 s6, s6, s8
+; GFX8-NEXT:    s_lshr_b32 s9, s9, s10
+; GFX8-NEXT:    s_and_b32 s11, s4, 15
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_or_b32 s6, s6, s9
+; GFX8-NEXT:    s_lshr_b32 s9, s4, 16
+; GFX8-NEXT:    s_andn2_b32 s4, 15, s4
+; GFX8-NEXT:    s_bfe_u32 s11, s11, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s2, s8
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s11
+; GFX8-NEXT:    s_or_b32 s0, s0, s2
+; GFX8-NEXT:    s_and_b32 s2, s9, 15
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s7, s7, s8
+; GFX8-NEXT:    s_andn2_b32 s4, 15, s9
+; GFX8-NEXT:    s_lshl_b32 s2, s6, s2
+; GFX8-NEXT:    s_bfe_u32 s6, s7, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s6, s6, s8
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s4, s6, s4
+; GFX8-NEXT:    s_or_b32 s2, s2, s4
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s6, s3, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s6, s6, s8
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshr_b32 s2, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX8-NEXT:    s_lshr_b32 s6, s6, s10
+; GFX8-NEXT:    s_or_b32 s1, s1, s6
+; GFX8-NEXT:    s_lshr_b32 s6, s4, s8
+; GFX8-NEXT:    s_lshl_b32 s3, s3, s8
+; GFX8-NEXT:    s_xor_b32 s5, s5, -1
+; GFX8-NEXT:    s_lshl_b32 s2, s2, s8
+; GFX8-NEXT:    s_lshr_b32 s6, s6, s10
+; GFX8-NEXT:    s_and_b32 s7, s5, 15
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_or_b32 s2, s2, s6
+; GFX8-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX8-NEXT:    s_andn2_b32 s5, 15, s5
+; GFX8-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s8
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s5
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s7
+; GFX8-NEXT:    s_or_b32 s1, s1, s3
+; GFX8-NEXT:    s_and_b32 s3, s6, 15
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, s8
+; GFX8-NEXT:    s_lshl_b32 s2, s2, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s4, 0x100000
+; GFX8-NEXT:    s_andn2_b32 s5, 15, s6
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s8
+; GFX8-NEXT:    s_bfe_u32 s4, s5, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s4
+; GFX8-NEXT:    s_or_b32 s2, s2, s3
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX9-NEXT:    s_mov_b32 s8, 0x10001
+; GFX9-NEXT:    s_mov_b32 s6, 0xf000f
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX9-NEXT:    s_lshl_b32 s9, s9, 1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s9
+; GFX9-NEXT:    s_and_b32 s7, s4, s6
+; GFX9-NEXT:    s_andn2_b32 s4, s6, s4
+; GFX9-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s4, s9, s10
+; GFX9-NEXT:    s_mov_b32 s9, 0xffff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s7, 16
+; GFX9-NEXT:    s_and_b32 s2, s2, s9
+; GFX9-NEXT:    s_and_b32 s7, s7, s9
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s7
+; GFX9-NEXT:    s_lshr_b32 s4, s4, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    s_or_b32 s0, s0, s2
+; GFX9-NEXT:    s_and_b32 s2, s5, s6
+; GFX9-NEXT:    s_andn2_b32 s4, s6, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX9-NEXT:    s_lshl_b32 s5, s5, 1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_lshl_b32 s4, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX9-NEXT:    s_and_b32 s3, s3, s9
+; GFX9-NEXT:    s_and_b32 s2, s2, s9
+; GFX9-NEXT:    s_lshr_b32 s2, s3, s2
+; GFX9-NEXT:    s_lshr_b32 s3, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    s_or_b32 s1, s1, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s7, 0x10001
+; GFX10-NEXT:    s_mov_b32 s6, 0xf000f
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s7
+; GFX10-NEXT:    s_lshl_b32 s8, s8, 1
+; GFX10-NEXT:    s_and_b32 s9, s4, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s8
+; GFX10-NEXT:    s_andn2_b32 s4, s6, s4
+; GFX10-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX10-NEXT:    s_lshl_b32 s4, s8, s10
+; GFX10-NEXT:    s_mov_b32 s8, 0xffff
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s7
+; GFX10-NEXT:    s_lshl_b32 s4, s4, 1
+; GFX10-NEXT:    s_and_b32 s7, s5, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX10-NEXT:    s_andn2_b32 s4, s6, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX10-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX10-NEXT:    s_and_b32 s11, s9, s8
+; GFX10-NEXT:    s_and_b32 s2, s2, s8
+; GFX10-NEXT:    s_lshr_b32 s9, s9, 16
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX10-NEXT:    s_lshl_b32 s4, s5, s6
+; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX10-NEXT:    s_and_b32 s6, s7, s8
+; GFX10-NEXT:    s_and_b32 s3, s3, s8
+; GFX10-NEXT:    s_lshr_b32 s7, s7, 16
+; GFX10-NEXT:    s_lshr_b32 s2, s2, s11
+; GFX10-NEXT:    s_lshr_b32 s9, s10, s9
+; GFX10-NEXT:    s_lshr_b32 s3, s3, s6
+; GFX10-NEXT:    s_lshr_b32 s5, s5, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX10-NEXT:    s_or_b32 s0, s0, s2
+; GFX10-NEXT:    s_or_b32 s1, s1, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i16> @llvm.fshr.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
+  %cast.result = bitcast <4 x i16> %result to <2 x i32>
+  ret <2 x i32> %cast.result
+}
+
+define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt) {
+; GFX6-LABEL: v_fshr_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v12, 0xffff
+; GFX6-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
+; GFX6-NEXT:    v_and_b32_e32 v8, v8, v12
+; GFX6-NEXT:    v_or_b32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v9, 16, v11
+; GFX6-NEXT:    v_and_b32_e32 v10, v10, v12
+; GFX6-NEXT:    s_mov_b32 s5, 0xffff
+; GFX6-NEXT:    v_or_b32_e32 v9, v9, v10
+; GFX6-NEXT:    v_and_b32_e32 v10, s5, v4
+; GFX6-NEXT:    s_bfe_u32 s4, 1, 0x100000
+; GFX6-NEXT:    v_lshrrev_b32_e32 v10, 1, v10
+; GFX6-NEXT:    s_bfe_u32 s6, 14, 0x100000
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v10, s6, v10
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v10
+; GFX6-NEXT:    v_and_b32_e32 v10, s5, v5
+; GFX6-NEXT:    v_lshrrev_b32_e32 v10, 1, v10
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v10, s6, v10
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_and_b32_e32 v11, 15, v8
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v10
+; GFX6-NEXT:    v_lshrrev_b32_e32 v10, 16, v8
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX6-NEXT:    v_and_b32_e32 v8, 15, v8
+; GFX6-NEXT:    v_and_b32_e32 v4, s5, v4
+; GFX6-NEXT:    v_bfe_u32 v11, v11, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_bfe_u32 v8, v8, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v8, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, v11, v0
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, 15, v10
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v10
+; GFX6-NEXT:    v_bfe_u32 v4, v4, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 1, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, v4, v1
+; GFX6-NEXT:    v_and_b32_e32 v4, s5, v5
+; GFX6-NEXT:    v_and_b32_e32 v8, 15, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_bfe_u32 v5, v8, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v5, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, s5, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, s4, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, s6, v4
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, s5, v7
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, s4, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, s6, v4
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 1, v6
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v9
+; GFX6-NEXT:    v_and_b32_e32 v8, 15, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 1, v7
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v6
+; GFX6-NEXT:    v_and_b32_e32 v6, 15, v6
+; GFX6-NEXT:    v_and_b32_e32 v4, s5, v4
+; GFX6-NEXT:    v_bfe_u32 v8, v8, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_bfe_u32 v6, v6, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v6, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, v8, v2
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, 15, v7
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v7
+; GFX6-NEXT:    v_bfe_u32 v4, v4, 0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, v4, v3
+; GFX6-NEXT:    v_and_b32_e32 v4, s5, v5
+; GFX6-NEXT:    v_and_b32_e32 v6, 15, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_bfe_u32 v5, v6, 0, 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, v5, v4
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b16_e32 v7, 1, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v6, 1, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v7, 14, v7
+; GFX8-NEXT:    v_or_b32_e32 v6, v6, v7
+; GFX8-NEXT:    v_mov_b32_e32 v7, 1
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v8, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v8, 14, v8
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v8
+; GFX8-NEXT:    v_lshlrev_b16_e32 v8, 1, v2
+; GFX8-NEXT:    v_and_b32_e32 v10, 15, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX8-NEXT:    v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT:    v_lshrrev_b16_e32 v8, 1, v8
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v4, v4, v8
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v9
+; GFX8-NEXT:    v_lshlrev_b16_e32 v6, v10, v6
+; GFX8-NEXT:    v_or_b32_e32 v4, v6, v4
+; GFX8-NEXT:    v_and_b32_e32 v6, 15, v9
+; GFX8-NEXT:    v_and_b32_e32 v8, 15, v8
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, 1, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, v6, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, v8, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v6, 1, v3
+; GFX8-NEXT:    v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 1, v1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v6, 14, v6
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT:    v_lshrrev_b16_sdwa v6, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v5
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v6, 14, v6
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v6
+; GFX8-NEXT:    v_lshlrev_b16_e32 v6, 1, v3
+; GFX8-NEXT:    v_lshlrev_b16_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_and_b32_e32 v8, 15, v5
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v5
+; GFX8-NEXT:    v_and_b32_e32 v5, 15, v5
+; GFX8-NEXT:    v_lshrrev_b16_e32 v6, 1, v6
+; GFX8-NEXT:    v_lshrrev_b16_e32 v5, v5, v6
+; GFX8-NEXT:    v_lshlrev_b16_e32 v4, v8, v4
+; GFX8-NEXT:    v_xor_b32_e32 v6, -1, v7
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v5
+; GFX8-NEXT:    v_and_b32_e32 v5, 15, v7
+; GFX8-NEXT:    v_and_b32_e32 v6, 15, v6
+; GFX8-NEXT:    v_lshrrev_b16_e32 v3, 1, v3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, v5, v1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v3, v6, v3
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 0xf000f
+; GFX9-NEXT:    v_and_b32_e32 v6, s4, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX9-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v0, v4, v0
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v5
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v6, v2
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, s4, v5
+; GFX9-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX9-NEXT:    v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v1, v4, v1
+; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v2, v3
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v6, -1, v4
+; GFX10-NEXT:    v_xor_b32_e32 v7, -1, v5
+; GFX10-NEXT:    s_mov_b32 s4, 0xf000f
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX10-NEXT:    v_and_b32_e32 v4, s4, v4
+; GFX10-NEXT:    v_and_b32_e32 v6, s4, v6
+; GFX10-NEXT:    v_and_b32_e32 v5, s4, v5
+; GFX10-NEXT:    v_and_b32_e32 v7, s4, v7
+; GFX10-NEXT:    v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX10-NEXT:    v_pk_lshrrev_b16 v2, v4, v2
+; GFX10-NEXT:    v_pk_lshlrev_b16 v0, v6, v0
+; GFX10-NEXT:    v_pk_lshrrev_b16 v3, v5, v3
+; GFX10-NEXT:    v_pk_lshlrev_b16 v1, v7, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.fshr.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
+  %cast.result = bitcast <4 x i16> %result to <4 x half>
+  ret <4 x half> %cast.result
+}
+
+define amdgpu_ps i64 @s_fshr_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt) {
+; GFX6-LABEL: s_fshr_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b64 s[6:7], s[4:5], 63
+; GFX6-NEXT:    s_andn2_b64 s[4:5], 63, s[4:5]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s4
+; GFX6-NEXT:    s_lshr_b64 s[2:3], s[2:3], s6
+; GFX6-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b64 s[6:7], s[4:5], 63
+; GFX8-NEXT:    s_andn2_b64 s[4:5], 63, s[4:5]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s4
+; GFX8-NEXT:    s_lshr_b64 s[2:3], s[2:3], s6
+; GFX8-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b64 s[6:7], s[4:5], 63
+; GFX9-NEXT:    s_andn2_b64 s[4:5], 63, s[4:5]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s4
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[2:3], s6
+; GFX9-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_andn2_b64 s[6:7], 63, s[4:5]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT:    s_and_b64 s[4:5], s[4:5], 63
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s6
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[2:3], s4
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_fshr_i64_5(i64 inreg %lhs, i64 inreg %rhs) {
+; GCN-LABEL: s_fshr_i64_5:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_lshl_b32 s1, s0, 27
+; GCN-NEXT:    s_mov_b32 s0, 0
+; GCN-NEXT:    s_lshr_b64 s[2:3], s[2:3], 5
+; GCN-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 5)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_fshr_i64_32(i64 inreg %lhs, i64 inreg %rhs) {
+; GCN-LABEL: s_fshr_i64_32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_mov_b32 s1, s0
+; GCN-NEXT:    s_mov_b32 s0, 0
+; GCN-NEXT:    s_mov_b32 s2, s3
+; GCN-NEXT:    s_mov_b32 s3, s0
+; GCN-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 32)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_fshr_i64_48(i64 inreg %lhs, i64 inreg %rhs) {
+; GCN-LABEL: s_fshr_i64_48:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_lshr_b32 s2, s3, 16
+; GCN-NEXT:    s_lshl_b64 s[0:1], s[0:1], 16
+; GCN-NEXT:    s_mov_b32 s3, 0
+; GCN-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 48)
+  ret i64 %result
+}
+
+define i64 @v_fshr_i64(i64 %lhs, i64 %rhs, i64 %amt) {
+; GFX6-LABEL: v_fshr_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v5, 63, v4
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
+; GFX6-NEXT:    v_and_b32_e32 v4, 63, v4
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], v4
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[2:3], v5
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v5, 63, v4
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX8-NEXT:    v_and_b32_e32 v4, 63, v4
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v4, v[0:1]
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], v5, v[2:3]
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v5, 63, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v4
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX9-NEXT:    v_and_b32_e32 v4, 63, v4
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v4, v[0:1]
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], v5, v[2:3]
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v4
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX10-NEXT:    v_and_b32_e32 v4, 63, v4
+; GFX10-NEXT:    v_and_b32_e32 v7, 63, v5
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], v4, v[2:3]
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], v7, v[0:1]
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  ret i64 %result
+}
+
+define i64 @v_fshr_i64_5(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_fshr_i64_5:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v4, v0
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[2:3], 5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 27, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i64_5:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v4, v0
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], 5, v[2:3]
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 27, v4
+; GFX8-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i64_5:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v0
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], 5, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 27, v4
+; GFX9-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i64_5:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v4, v0
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], 5, v[2:3]
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 27, v4
+; GFX10-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 5)
+  ret i64 %result
+}
+
+define i64 @v_fshr_i64_32(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_fshr_i64_32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, v0
+; GFX6-NEXT:    v_mov_b32_e32 v0, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i64_32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i64_32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i64_32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v1, v0
+; GFX10-NEXT:    v_mov_b32_e32 v0, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 32)
+  ret i64 %result
+}
+
+define i64 @v_fshr_i64_48(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_fshr_i64_48:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 16
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i64_48:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], 16, v[0:1]
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i64_48:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 16, v[0:1]
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i64_48:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 16, v[0:1]
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 48)
+  ret i64 %result
+}
+
+define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64 %amt) {
+; GFX6-LABEL: v_fshr_i64_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_and_b32_e32 v2, 63, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    v_and_b32_e32 v0, 63, v0
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT:    v_lshl_b64 v[0:1], s[0:1], v0
+; GFX6-NEXT:    v_lshr_b64 v[2:3], s[2:3], v2
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i64_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_and_b32_e32 v2, 63, v0
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    v_and_b32_e32 v0, 63, v0
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v0, s[0:1]
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i64_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_and_b32_e32 v2, 63, v0
+; GFX9-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, 63, v0
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v0, s[0:1]
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i64_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 63, v0
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT:    v_and_b32_e32 v2, 63, v1
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], v0, s[2:3]
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], v2, s[0:1]
+; GFX10-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, v3, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x float> @v_fshr_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg %amt) {
+; GFX6-LABEL: v_fshr_i64_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX6-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[0:1], s4
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s2
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i64_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX8-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s4, v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s2
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i64_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX9-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s4, v[0:1]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s2
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i64_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX10-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], s4, v[0:1]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg %amt) {
+; GFX6-LABEL: v_fshr_i64_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
+; GFX6-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX6-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], s2
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[0:1], s4
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i64_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX8-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX8-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], s2, v[0:1]
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[0:1], s4
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i64_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX9-NEXT:    s_and_b64 s[4:5], s[2:3], 63
+; GFX9-NEXT:    s_andn2_b64 s[2:3], 63, s[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], s2, v[0:1]
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], s4
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i64_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX10-NEXT:    s_andn2_b64 s[4:5], 63, s[2:3]
+; GFX10-NEXT:    s_and_b64 s[2:3], s[2:3], 63
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], s4, v[0:1]
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x i64> @s_fshr_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs, <2 x i64> inreg %amt) {
+; GFX6-LABEL: s_fshr_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_and_b64 s[12:13], s[8:9], 63
+; GFX6-NEXT:    s_andn2_b64 s[8:9], 63, s[8:9]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s8
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[4:5], s12
+; GFX6-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT:    s_and_b64 s[4:5], s[10:11], 63
+; GFX6-NEXT:    s_andn2_b64 s[8:9], 63, s[10:11]
+; GFX6-NEXT:    s_lshl_b64 s[2:3], s[2:3], 1
+; GFX6-NEXT:    s_lshl_b64 s[2:3], s[2:3], s8
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[6:7], s4
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_and_b64 s[12:13], s[8:9], 63
+; GFX8-NEXT:    s_andn2_b64 s[8:9], 63, s[8:9]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s8
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[4:5], s12
+; GFX8-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT:    s_and_b64 s[4:5], s[10:11], 63
+; GFX8-NEXT:    s_andn2_b64 s[8:9], 63, s[10:11]
+; GFX8-NEXT:    s_lshl_b64 s[2:3], s[2:3], 1
+; GFX8-NEXT:    s_lshl_b64 s[2:3], s[2:3], s8
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[6:7], s4
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b64 s[12:13], s[8:9], 63
+; GFX9-NEXT:    s_andn2_b64 s[8:9], 63, s[8:9]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s8
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], s12
+; GFX9-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX9-NEXT:    s_and_b64 s[4:5], s[10:11], 63
+; GFX9-NEXT:    s_andn2_b64 s[8:9], 63, s[10:11]
+; GFX9-NEXT:    s_lshl_b64 s[2:3], s[2:3], 1
+; GFX9-NEXT:    s_lshl_b64 s[2:3], s[2:3], s8
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[6:7], s4
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_andn2_b64 s[12:13], 63, s[8:9]
+; GFX10-NEXT:    s_and_b64 s[8:9], s[8:9], 63
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT:    s_lshr_b64 s[4:5], s[4:5], s8
+; GFX10-NEXT:    s_andn2_b64 s[8:9], 63, s[10:11]
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT:    s_and_b64 s[10:11], s[10:11], 63
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s12
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[2:3], s8
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[6:7], s10
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
+  ret <2 x i64> %result
+}
+
+define <2 x i64> @v_fshr_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
+; GFX6-LABEL: v_fshr_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v9, 63, v8
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
+; GFX6-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], v8
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[4:5], v9
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v10
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, 63, v10
+; GFX6-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[2:3], v8
+; GFX6-NEXT:    v_lshr_b64 v[6:7], v[6:7], v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v6
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v7
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v9, 63, v8
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX8-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v8, v[0:1]
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], v9, v[4:5]
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v10
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_and_b32_e32 v4, 63, v10
+; GFX8-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v8, v[2:3]
+; GFX8-NEXT:    v_lshrrev_b64 v[6:7], v4, v[6:7]
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v6
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v7
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v9, 63, v8
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX9-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v8, v[0:1]
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], v9, v[4:5]
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v10
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX9-NEXT:    v_and_b32_e32 v4, 63, v10
+; GFX9-NEXT:    v_and_b32_e32 v8, 63, v8
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v8, v[2:3]
+; GFX9-NEXT:    v_lshrrev_b64 v[6:7], v4, v[6:7]
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v5
+; GFX9-NEXT:    v_or_b32_e32 v2, v2, v6
+; GFX9-NEXT:    v_or_b32_e32 v3, v3, v7
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v9, -1, v8
+; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v10
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX10-NEXT:    v_and_b32_e32 v19, 63, v8
+; GFX10-NEXT:    v_and_b32_e32 v15, 63, v9
+; GFX10-NEXT:    v_and_b32_e32 v9, 63, v11
+; GFX10-NEXT:    v_and_b32_e32 v13, 63, v10
+; GFX10-NEXT:    v_lshrrev_b64 v[4:5], v19, v[4:5]
+; GFX10-NEXT:    v_lshlrev_b64 v[11:12], v15, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[15:16], v9, v[2:3]
+; GFX10-NEXT:    v_lshrrev_b64 v[6:7], v13, v[6:7]
+; GFX10-NEXT:    v_or_b32_e32 v0, v11, v4
+; GFX10-NEXT:    v_or_b32_e32 v1, v12, v5
+; GFX10-NEXT:    v_or_b32_e32 v2, v15, v6
+; GFX10-NEXT:    v_or_b32_e32 v3, v16, v7
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps i128 @s_fshr_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg %amt) {
+; GFX6-LABEL: s_fshr_i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s10, 0x7f
+; GFX6-NEXT:    s_mov_b32 s11, 0
+; GFX6-NEXT:    s_and_b64 s[12:13], s[8:9], s[10:11]
+; GFX6-NEXT:    s_andn2_b64 s[8:9], s[10:11], s[8:9]
+; GFX6-NEXT:    s_sub_i32 s9, 1, 64
+; GFX6-NEXT:    s_sub_i32 s13, 64, 1
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[14:15], s[0:1], s13
+; GFX6-NEXT:    s_lshl_b64 s[16:17], s[2:3], 1
+; GFX6-NEXT:    s_lshl_b64 s[10:11], s[0:1], 1
+; GFX6-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s9
+; GFX6-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX6-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[14:15], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX6-NEXT:    s_sub_i32 s13, s8, 64
+; GFX6-NEXT:    s_sub_i32 s9, 64, s8
+; GFX6-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX6-NEXT:    s_lshl_b64 s[2:3], s[10:11], s8
+; GFX6-NEXT:    s_lshr_b64 s[14:15], s[10:11], s9
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[0:1], s8
+; GFX6-NEXT:    s_or_b64 s[8:9], s[14:15], s[8:9]
+; GFX6-NEXT:    s_lshl_b64 s[10:11], s[10:11], s13
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[0:1], s[8:9]
+; GFX6-NEXT:    s_sub_i32 s14, s12, 64
+; GFX6-NEXT:    s_sub_i32 s13, 64, s12
+; GFX6-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX6-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[6:7], s12
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[4:5], s12
+; GFX6-NEXT:    s_lshl_b64 s[12:13], s[6:7], s13
+; GFX6-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[10:11], s[6:7]
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[0:1], 0
+; GFX6-NEXT:    s_or_b64 s[0:1], s[2:3], s[4:5]
+; GFX6-NEXT:    s_or_b64 s[2:3], s[8:9], s[6:7]
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s10, 0x7f
+; GFX8-NEXT:    s_mov_b32 s11, 0
+; GFX8-NEXT:    s_and_b64 s[12:13], s[8:9], s[10:11]
+; GFX8-NEXT:    s_andn2_b64 s[8:9], s[10:11], s[8:9]
+; GFX8-NEXT:    s_sub_i32 s9, 1, 64
+; GFX8-NEXT:    s_sub_i32 s13, 64, 1
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[14:15], s[0:1], s13
+; GFX8-NEXT:    s_lshl_b64 s[16:17], s[2:3], 1
+; GFX8-NEXT:    s_lshl_b64 s[10:11], s[0:1], 1
+; GFX8-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s9
+; GFX8-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX8-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[14:15], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX8-NEXT:    s_sub_i32 s13, s8, 64
+; GFX8-NEXT:    s_sub_i32 s9, 64, s8
+; GFX8-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX8-NEXT:    s_lshl_b64 s[2:3], s[10:11], s8
+; GFX8-NEXT:    s_lshr_b64 s[14:15], s[10:11], s9
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[0:1], s8
+; GFX8-NEXT:    s_or_b64 s[8:9], s[14:15], s[8:9]
+; GFX8-NEXT:    s_lshl_b64 s[10:11], s[10:11], s13
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[0:1], s[8:9]
+; GFX8-NEXT:    s_sub_i32 s14, s12, 64
+; GFX8-NEXT:    s_sub_i32 s13, 64, s12
+; GFX8-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX8-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[6:7], s12
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[4:5], s12
+; GFX8-NEXT:    s_lshl_b64 s[12:13], s[6:7], s13
+; GFX8-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[10:11], s[6:7]
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[0:1], 0
+; GFX8-NEXT:    s_or_b64 s[0:1], s[2:3], s[4:5]
+; GFX8-NEXT:    s_or_b64 s[2:3], s[8:9], s[6:7]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s10, 0x7f
+; GFX9-NEXT:    s_mov_b32 s11, 0
+; GFX9-NEXT:    s_and_b64 s[12:13], s[8:9], s[10:11]
+; GFX9-NEXT:    s_andn2_b64 s[8:9], s[10:11], s[8:9]
+; GFX9-NEXT:    s_sub_i32 s9, 1, 64
+; GFX9-NEXT:    s_sub_i32 s13, 64, 1
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[14:15], s[0:1], s13
+; GFX9-NEXT:    s_lshl_b64 s[16:17], s[2:3], 1
+; GFX9-NEXT:    s_lshl_b64 s[10:11], s[0:1], 1
+; GFX9-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s9
+; GFX9-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX9-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[14:15], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s13, s8, 64
+; GFX9-NEXT:    s_sub_i32 s9, 64, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX9-NEXT:    s_lshl_b64 s[2:3], s[10:11], s8
+; GFX9-NEXT:    s_lshr_b64 s[14:15], s[10:11], s9
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[0:1], s8
+; GFX9-NEXT:    s_or_b64 s[8:9], s[14:15], s[8:9]
+; GFX9-NEXT:    s_lshl_b64 s[10:11], s[10:11], s13
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[0:1], s[8:9]
+; GFX9-NEXT:    s_sub_i32 s14, s12, 64
+; GFX9-NEXT:    s_sub_i32 s13, 64, s12
+; GFX9-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX9-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[6:7], s12
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[4:5], s12
+; GFX9-NEXT:    s_lshl_b64 s[12:13], s[6:7], s13
+; GFX9-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[10:11], s[6:7]
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[0:1], 0
+; GFX9-NEXT:    s_or_b64 s[0:1], s[2:3], s[4:5]
+; GFX9-NEXT:    s_or_b64 s[2:3], s[8:9], s[6:7]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s10, 0x7f
+; GFX10-NEXT:    s_mov_b32 s11, 0
+; GFX10-NEXT:    s_and_b64 s[12:13], s[8:9], s[10:11]
+; GFX10-NEXT:    s_andn2_b64 s[8:9], s[10:11], s[8:9]
+; GFX10-NEXT:    s_sub_i32 s9, 1, 64
+; GFX10-NEXT:    s_sub_i32 s10, 64, 1
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[10:11], s[0:1], s10
+; GFX10-NEXT:    s_lshl_b64 s[14:15], s[2:3], 1
+; GFX10-NEXT:    s_lshl_b64 s[16:17], s[0:1], 1
+; GFX10-NEXT:    s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s9
+; GFX10-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX10-NEXT:    s_cselect_b64 s[14:15], s[16:17], 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s13, s8, 64
+; GFX10-NEXT:    s_sub_i32 s2, 64, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[0:1], s8
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[14:15], s2
+; GFX10-NEXT:    s_lshl_b64 s[8:9], s[14:15], s8
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[10:11]
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[14:15], s13
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[0:1], s[2:3]
+; GFX10-NEXT:    s_sub_i32 s14, s12, 64
+; GFX10-NEXT:    s_sub_i32 s10, 64, s12
+; GFX10-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[4:5], s12
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[6:7], s10
+; GFX10-NEXT:    s_lshr_b64 s[12:13], s[6:7], s12
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[12:13], 0
+; GFX10-NEXT:    s_or_b64 s[0:1], s[8:9], s[0:1]
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  ret i128 %result
+}
+
+define i128 @v_fshr_i128(i128 %lhs, i128 %rhs, i128 %amt) {
+; GFX6-LABEL: v_fshr_i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_movk_i32 s4, 0x7f
+; GFX6-NEXT:    v_and_b32_e32 v14, s4, v8
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX6-NEXT:    v_and_b32_e32 v15, s4, v8
+; GFX6-NEXT:    s_sub_i32 s5, 64, 1
+; GFX6-NEXT:    s_sub_i32 s4, 1, 64
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[0:1], s5
+; GFX6-NEXT:    v_lshl_b64 v[10:11], v[2:3], 1
+; GFX6-NEXT:    v_lshl_b64 v[12:13], v[0:1], 1
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s6
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX6-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX6-NEXT:    s_and_b32 s4, 1, s7
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, 0, v12, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v9, 0, v13, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 64, v15
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[8:9], v2
+; GFX6-NEXT:    v_lshl_b64 v[10:11], v[0:1], v15
+; GFX6-NEXT:    v_subrev_i32_e32 v16, vcc, 64, v15
+; GFX6-NEXT:    v_lshl_b64 v[12:13], v[8:9], v15
+; GFX6-NEXT:    v_or_b32_e32 v10, v2, v10
+; GFX6-NEXT:    v_or_b32_e32 v11, v3, v11
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[8:9], v16
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v15
+; GFX6-NEXT:    v_cndmask_b32_e32 v12, 0, v12, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v13, 0, v13, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v15
+; GFX6-NEXT:    v_cndmask_b32_e32 v10, v2, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v11, v3, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 64, v14
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[4:5], v14
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[6:7], v2
+; GFX6-NEXT:    v_subrev_i32_e32 v15, vcc, 64, v14
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[6:7], v15
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[6:7], v14
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT:    v_or_b32_e32 v0, v12, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, v13, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, v10, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v11, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_movk_i32 s4, 0x7f
+; GFX8-NEXT:    v_and_b32_e32 v14, s4, v8
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX8-NEXT:    v_and_b32_e32 v15, s4, v8
+; GFX8-NEXT:    s_sub_i32 s5, 64, 1
+; GFX8-NEXT:    s_sub_i32 s4, 1, 64
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], s5, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[10:11], 1, v[2:3]
+; GFX8-NEXT:    v_lshlrev_b64 v[12:13], 1, v[0:1]
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], s4, v[0:1]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX8-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX8-NEXT:    s_and_b32 s4, 1, s7
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, 0, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, 0, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 64, v15
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], v2, v[8:9]
+; GFX8-NEXT:    v_lshlrev_b64 v[10:11], v15, v[0:1]
+; GFX8-NEXT:    v_subrev_u32_e32 v16, vcc, 64, v15
+; GFX8-NEXT:    v_lshlrev_b64 v[12:13], v15, v[8:9]
+; GFX8-NEXT:    v_or_b32_e32 v10, v2, v10
+; GFX8-NEXT:    v_or_b32_e32 v11, v3, v11
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v16, v[8:9]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v15
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, 0, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v13, 0, v13, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v15
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v3, v1, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 64, v14
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v14, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v2, v[6:7]
+; GFX8-NEXT:    v_subrev_u32_e32 v15, vcc, 64, v14
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v15, v[6:7]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], v14, v[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT:    v_or_b32_e32 v0, v12, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, v13, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, v10, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v11, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7f
+; GFX9-NEXT:    v_and_b32_e32 v14, s4, v8
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v8
+; GFX9-NEXT:    v_and_b32_e32 v15, s4, v8
+; GFX9-NEXT:    s_sub_i32 s5, 64, 1
+; GFX9-NEXT:    s_sub_i32 s4, 1, 64
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], s5, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[10:11], 1, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[12:13], 1, v[0:1]
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], s4, v[0:1]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX9-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX9-NEXT:    s_and_b32 s4, 1, s7
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, 0, v12, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v13, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, 64, v15
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], v2, v[8:9]
+; GFX9-NEXT:    v_lshlrev_b64 v[10:11], v15, v[0:1]
+; GFX9-NEXT:    v_subrev_u32_e32 v16, 64, v15
+; GFX9-NEXT:    v_lshlrev_b64 v[12:13], v15, v[8:9]
+; GFX9-NEXT:    v_or_b32_e32 v10, v2, v10
+; GFX9-NEXT:    v_or_b32_e32 v11, v3, v11
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v16, v[8:9]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v15
+; GFX9-NEXT:    v_cndmask_b32_e32 v12, 0, v12, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v13, 0, v13, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v15
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, v2, v0, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, 64, v14
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, v3, v1, vcc
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v14, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v2, v[6:7]
+; GFX9-NEXT:    v_subrev_u32_e32 v15, 64, v14
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v15, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], v14, v[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT:    v_or_b32_e32 v0, v12, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, v13, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, v10, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v11, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_sub_i32 s4, 64, 1
+; GFX10-NEXT:    s_sub_i32 s6, 1, 64
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[9:10], s4, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[11:12], 1, v[2:3]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[13:14], 1, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX10-NEXT:    s_and_b32 s4, 1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], s6, v[0:1]
+; GFX10-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s4
+; GFX10-NEXT:    v_or_b32_e32 v10, v10, v12
+; GFX10-NEXT:    v_xor_b32_e32 v15, -1, v8
+; GFX10-NEXT:    s_movk_i32 s5, 0x7f
+; GFX10-NEXT:    s_and_b32 s6, 1, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v11, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v10, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s6
+; GFX10-NEXT:    v_and_b32_e32 v19, s5, v15
+; GFX10-NEXT:    v_and_b32_e32 v20, s5, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0, v13, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v11, 64, v19
+; GFX10-NEXT:    v_sub_nc_u32_e32 v17, 64, v20
+; GFX10-NEXT:    v_mov_b32_e32 v25, v4
+; GFX10-NEXT:    v_mov_b32_e32 v26, v5
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v8, 64, v19
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], v11, v[9:10]
+; GFX10-NEXT:    v_lshlrev_b64 v[11:12], v19, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[13:14], v19, v[9:10]
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v21, 64, v20
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], v8, v[9:10]
+; GFX10-NEXT:    v_lshrrev_b64 v[15:16], v20, v[25:26]
+; GFX10-NEXT:    v_lshlrev_b64 v[17:18], v17, v[6:7]
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 64, v19
+; GFX10-NEXT:    v_or_b32_e32 v10, v3, v12
+; GFX10-NEXT:    v_or_b32_e32 v11, v2, v11
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], v21, v[6:7]
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s4, 64, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, 0, v13, vcc_lo
+; GFX10-NEXT:    v_or_b32_e32 v13, v15, v17
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v9, v10, vcc_lo
+; GFX10-NEXT:    v_or_b32_e32 v10, v16, v18
+; GFX10-NEXT:    v_lshrrev_b64 v[6:7], v20, v[6:7]
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 0, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v8, v11, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v13, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v10, s4
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 0, v20
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v15, v1, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v11, v0, s6
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, v25, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v3, v26, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v6, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, v7, s4
+; GFX10-NEXT:    v_or_b32_e32 v0, v23, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, v10, v1
+; GFX10-NEXT:    v_or_b32_e32 v2, v8, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, v9, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  ret i128 %result
+}
+
+define amdgpu_ps <4 x float> @v_fshr_i128_ssv(i128 inreg %lhs, i128 inreg %rhs, i128 %amt) {
+; GFX6-LABEL: v_fshr_i128_ssv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_i32 s14, 1, 64
+; GFX6-NEXT:    s_sub_i32 s10, 64, 1
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_movk_i32 s8, 0x7f
+; GFX6-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    v_and_b32_e32 v6, s8, v0
+; GFX6-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[0:1], s10
+; GFX6-NEXT:    s_lshl_b64 s[12:13], s[2:3], 1
+; GFX6-NEXT:    v_and_b32_e32 v7, s8, v0
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[0:1], 1
+; GFX6-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s14
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, 64, v7
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX6-NEXT:    v_lshr_b64 v[0:1], s[8:9], v0
+; GFX6-NEXT:    v_lshl_b64 v[2:3], s[0:1], v7
+; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, 64, v7
+; GFX6-NEXT:    v_lshl_b64 v[4:5], s[8:9], v7
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_lshl_b64 v[0:1], s[8:9], v8
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v10, v1, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 64, v6
+; GFX6-NEXT:    v_lshr_b64 v[0:1], s[4:5], v6
+; GFX6-NEXT:    v_lshl_b64 v[2:3], s[6:7], v2
+; GFX6-NEXT:    v_subrev_i32_e32 v11, vcc, 64, v6
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_lshr_b64 v[0:1], s[6:7], v11
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v6
+; GFX6-NEXT:    v_lshr_b64 v[4:5], s[6:7], v6
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v6
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX6-NEXT:    v_or_b32_e32 v0, v8, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, v9, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, v7, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v10, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i128_ssv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_i32 s14, 1, 64
+; GFX8-NEXT:    s_sub_i32 s10, 64, 1
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_movk_i32 s8, 0x7f
+; GFX8-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    v_and_b32_e32 v6, s8, v0
+; GFX8-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[0:1], s10
+; GFX8-NEXT:    s_lshl_b64 s[12:13], s[2:3], 1
+; GFX8-NEXT:    v_and_b32_e32 v7, s8, v0
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[0:1], 1
+; GFX8-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s14
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, 64, v7
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v0, s[8:9]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v7, s[0:1]
+; GFX8-NEXT:    v_subrev_u32_e32 v8, vcc, 64, v7
+; GFX8-NEXT:    v_lshlrev_b64 v[4:5], v7, s[8:9]
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], v8, s[8:9]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v1, v3, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 64, v6
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v6, s[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v2, s[6:7]
+; GFX8-NEXT:    v_subrev_u32_e32 v11, vcc, 64, v6
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v11, s[6:7]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v6
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], v6, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v6
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX8-NEXT:    v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, v9, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v10, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i128_ssv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_i32 s14, 1, 64
+; GFX9-NEXT:    s_sub_i32 s10, 64, 1
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_movk_i32 s8, 0x7f
+; GFX9-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    v_and_b32_e32 v6, s8, v0
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    v_xor_b32_e32 v0, -1, v0
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[0:1], s10
+; GFX9-NEXT:    s_lshl_b64 s[12:13], s[2:3], 1
+; GFX9-NEXT:    v_and_b32_e32 v7, s8, v0
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[0:1], 1
+; GFX9-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s14
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    v_sub_u32_e32 v0, 64, v7
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v0, s[8:9]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v7, s[0:1]
+; GFX9-NEXT:    v_subrev_u32_e32 v8, 64, v7
+; GFX9-NEXT:    v_lshlrev_b64 v[4:5], v7, s[8:9]
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v8, s[8:9]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v0, v2, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, 64, v6
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, v1, v3, vcc
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v6, s[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v2, s[6:7]
+; GFX9-NEXT:    v_subrev_u32_e32 v11, 64, v6
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v11, s[6:7]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v6
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], v6, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX9-NEXT:    v_or_b32_e32 v0, v8, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, v9, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, v7, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v10, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i128_ssv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    s_sub_i32 s14, 1, 64
+; GFX10-NEXT:    s_sub_i32 s9, 64, 1
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    s_movk_i32 s8, 0x7f
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_and_b32_e32 v13, s8, v1
+; GFX10-NEXT:    v_and_b32_e32 v12, s8, v0
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[8:9], s[0:1], s9
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[2:3], 1
+; GFX10-NEXT:    s_lshl_b64 s[12:13], s[0:1], 1
+; GFX10-NEXT:    s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s14
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, 64, v13
+; GFX10-NEXT:    s_cselect_b64 s[10:11], s[12:13], 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v8, 64, v12
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[2:3], s[0:1]
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], v0, s[10:11]
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], v13, s[8:9]
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, 64, v13
+; GFX10-NEXT:    v_lshrrev_b64 v[6:7], v12, s[4:5]
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], v8, s[6:7]
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 64, v13
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s0, 64, v12
+; GFX10-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v0, 64, v12
+; GFX10-NEXT:    v_lshlrev_b64 v[15:16], v10, s[10:11]
+; GFX10-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX10-NEXT:    v_or_b32_e32 v6, v6, v8
+; GFX10-NEXT:    v_or_b32_e32 v7, v7, v9
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], v0, s[6:7]
+; GFX10-NEXT:    v_lshlrev_b64 v[4:5], v13, s[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v15, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v16, v3, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], v12, s[6:7]
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s2, 0, v13
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 0, v12
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s0
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, 0, v5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v19, v8, s8, s2
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s4, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s5, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v10, s9, s2
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s0
+; GFX10-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, v15, v1
+; GFX10-NEXT:    v_or_b32_e32 v2, v19, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, v6, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  %cast.result = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast.result
+}
+
+define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 inreg %amt) {
+; GFX6-LABEL: v_fshr_i128_svs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s6, 0x7f
+; GFX6-NEXT:    s_mov_b32 s7, 0
+; GFX6-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX6-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX6-NEXT:    s_sub_i32 s5, 1, 64
+; GFX6-NEXT:    s_sub_i32 s9, 64, 1
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s14, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[0:1], s9
+; GFX6-NEXT:    s_lshl_b64 s[12:13], s[2:3], 1
+; GFX6-NEXT:    s_lshl_b64 s[6:7], s[0:1], 1
+; GFX6-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s5
+; GFX6-NEXT:    s_cmp_lg_u32 s14, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[6:7], 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX6-NEXT:    s_sub_i32 s9, s4, 64
+; GFX6-NEXT:    s_sub_i32 s5, 64, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX6-NEXT:    s_lshl_b64 s[2:3], s[6:7], s4
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[6:7], s5
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[0:1], s4
+; GFX6-NEXT:    s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX6-NEXT:    s_lshl_b64 s[6:7], s[6:7], s9
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX6-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT:    s_sub_i32 s5, 64, s8
+; GFX6-NEXT:    s_sub_i32 s4, s8, 64
+; GFX6-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX6-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[0:1], s8
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[2:3], s5
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[2:3], s8
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[2:3], s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s6
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX6-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX6-NEXT:    s_and_b32 s4, 1, s7
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s6
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT:    v_or_b32_e32 v0, s2, v0
+; GFX6-NEXT:    v_or_b32_e32 v1, s3, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, s0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, s1, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i128_svs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s6, 0x7f
+; GFX8-NEXT:    s_mov_b32 s7, 0
+; GFX8-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX8-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX8-NEXT:    s_sub_i32 s5, 1, 64
+; GFX8-NEXT:    s_sub_i32 s9, 64, 1
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s14, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[0:1], s9
+; GFX8-NEXT:    s_lshl_b64 s[12:13], s[2:3], 1
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[0:1], 1
+; GFX8-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s5
+; GFX8-NEXT:    s_cmp_lg_u32 s14, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[6:7], 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX8-NEXT:    s_sub_i32 s9, s4, 64
+; GFX8-NEXT:    s_sub_i32 s5, 64, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX8-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX8-NEXT:    s_lshl_b64 s[2:3], s[6:7], s4
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[6:7], s5
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[0:1], s4
+; GFX8-NEXT:    s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], s9
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX8-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT:    s_sub_i32 s5, 64, s8
+; GFX8-NEXT:    s_sub_i32 s4, s8, 64
+; GFX8-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], s8, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], s8, v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], s4, v[2:3]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT:    s_and_b32 s4, 1, s7
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT:    v_or_b32_e32 v0, s2, v0
+; GFX8-NEXT:    v_or_b32_e32 v1, s3, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, s0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, s1, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i128_svs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s6, 0x7f
+; GFX9-NEXT:    s_mov_b32 s7, 0
+; GFX9-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX9-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX9-NEXT:    s_sub_i32 s5, 1, 64
+; GFX9-NEXT:    s_sub_i32 s9, 64, 1
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s14, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[0:1], s9
+; GFX9-NEXT:    s_lshl_b64 s[12:13], s[2:3], 1
+; GFX9-NEXT:    s_lshl_b64 s[6:7], s[0:1], 1
+; GFX9-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s5
+; GFX9-NEXT:    s_cmp_lg_u32 s14, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[6:7], 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s9, s4, 64
+; GFX9-NEXT:    s_sub_i32 s5, 64, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX9-NEXT:    s_lshl_b64 s[2:3], s[6:7], s4
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[6:7], s5
+; GFX9-NEXT:    s_lshl_b64 s[4:5], s[0:1], s4
+; GFX9-NEXT:    s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX9-NEXT:    s_lshl_b64 s[6:7], s[6:7], s9
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX9-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX9-NEXT:    s_sub_i32 s5, 64, s8
+; GFX9-NEXT:    s_sub_i32 s4, s8, 64
+; GFX9-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], s8, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], s8, v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], s4, v[2:3]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX9-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX9-NEXT:    s_and_b32 s4, 1, s7
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT:    v_or_b32_e32 v0, s2, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, s3, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, s0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, s1, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i128_svs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s6, 0x7f
+; GFX10-NEXT:    s_mov_b32 s7, 0
+; GFX10-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX10-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX10-NEXT:    s_sub_i32 s5, 1, 64
+; GFX10-NEXT:    s_sub_i32 s6, 64, 1
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[4:5], s8, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    s_cselect_b32 s14, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[0:1], s6
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[2:3], 1
+; GFX10-NEXT:    s_lshl_b64 s[12:13], s[0:1], 1
+; GFX10-NEXT:    s_or_b64 s[6:7], s[6:7], s[10:11]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s5
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_cselect_b64 s[10:11], s[12:13], 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[6:7], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s14, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s9, s4, 64
+; GFX10-NEXT:    s_sub_i32 s2, 64, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX10-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[6:7], s[0:1], s4
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[10:11], s2
+; GFX10-NEXT:    s_lshl_b64 s[4:5], s[10:11], s4
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX10-NEXT:    s_lshl_b64 s[6:7], s[10:11], s9
+; GFX10-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[4:5], 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX10-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[0:1], s[2:3]
+; GFX10-NEXT:    s_sub_i32 s0, 64, s8
+; GFX10-NEXT:    v_lshlrev_b64 v[6:7], s0, v[2:3]
+; GFX10-NEXT:    s_sub_i32 s0, s8, 64
+; GFX10-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[11:12], s0, v[2:3]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX10-NEXT:    v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX10-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX10-NEXT:    s_and_b32 s1, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], s8, v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v11, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v12, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s0
+; GFX10-NEXT:    v_or_b32_e32 v0, s4, v0
+; GFX10-NEXT:    v_or_b32_e32 v1, s5, v1
+; GFX10-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  %cast.result = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast.result
+}
+
+define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 inreg %amt) {
+; GFX6-LABEL: v_fshr_i128_vss:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s6, 0x7f
+; GFX6-NEXT:    s_mov_b32 s7, 0
+; GFX6-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX6-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX6-NEXT:    s_sub_i32 s6, 64, 1
+; GFX6-NEXT:    s_sub_i32 s5, 1, 64
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[0:1], s6
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[2:3], 1
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    v_lshl_b64 v[8:9], v[0:1], 1
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], s5
+; GFX6-NEXT:    s_and_b32 s5, 1, s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX6-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX6-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX6-NEXT:    s_and_b32 s5, 1, s9
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX6-NEXT:    s_sub_i32 s5, s4, 64
+; GFX6-NEXT:    s_sub_i32 s6, 64, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[4:5], s6
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[0:1], s4
+; GFX6-NEXT:    v_lshl_b64 v[8:9], v[4:5], s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s9
+; GFX6-NEXT:    s_sub_i32 s10, s8, 64
+; GFX6-NEXT:    s_sub_i32 s9, 64, s8
+; GFX6-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX6-NEXT:    v_or_b32_e32 v6, v2, v6
+; GFX6-NEXT:    v_or_b32_e32 v7, v3, v7
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[4:5], s5
+; GFX6-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[2:3], s8
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[0:1], s8
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[2:3], s9
+; GFX6-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX6-NEXT:    s_lshr_b64 s[2:3], s[2:3], s10
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[4:5], 0
+; GFX6-NEXT:    v_or_b32_e32 v0, s0, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, s1, v5
+; GFX6-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i128_vss:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s6, 0x7f
+; GFX8-NEXT:    s_mov_b32 s7, 0
+; GFX8-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX8-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX8-NEXT:    s_sub_i32 s6, 64, 1
+; GFX8-NEXT:    s_sub_i32 s5, 1, 64
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], s6, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], 1, v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    v_lshlrev_b64 v[8:9], 1, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], s5, v[0:1]
+; GFX8-NEXT:    s_and_b32 s5, 1, s7
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX8-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX8-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX8-NEXT:    s_and_b32 s5, 1, s9
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX8-NEXT:    s_sub_i32 s5, s4, 64
+; GFX8-NEXT:    s_sub_i32 s6, 64, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], s6, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], s4, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[8:9], s4, v[4:5]
+; GFX8-NEXT:    s_and_b32 s4, 1, s7
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s9
+; GFX8-NEXT:    s_sub_i32 s10, s8, 64
+; GFX8-NEXT:    s_sub_i32 s9, 64, s8
+; GFX8-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX8-NEXT:    v_or_b32_e32 v6, v2, v6
+; GFX8-NEXT:    v_or_b32_e32 v7, v3, v7
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], s5, v[4:5]
+; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[2:3], s8
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[0:1], s8
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[2:3], s9
+; GFX8-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX8-NEXT:    s_lshr_b64 s[2:3], s[2:3], s10
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[4:5], 0
+; GFX8-NEXT:    v_or_b32_e32 v0, s0, v4
+; GFX8-NEXT:    v_or_b32_e32 v1, s1, v5
+; GFX8-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: v_fshr_i128_vss:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s6, 0x7f
+; GFX9-NEXT:    s_mov_b32 s7, 0
+; GFX9-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX9-NEXT:    s_andn2_b64 s[4:5], s[6:7], s[4:5]
+; GFX9-NEXT:    s_sub_i32 s6, 64, 1
+; GFX9-NEXT:    s_sub_i32 s5, 1, 64
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], s6, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], 1, v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    v_lshlrev_b64 v[8:9], 1, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], s5, v[0:1]
+; GFX9-NEXT:    s_and_b32 s5, 1, s7
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX9-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX9-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX9-NEXT:    s_and_b32 s5, 1, s9
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX9-NEXT:    s_sub_i32 s5, s4, 64
+; GFX9-NEXT:    s_sub_i32 s6, 64, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], s6, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], s4, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[8:9], s4, v[4:5]
+; GFX9-NEXT:    s_and_b32 s4, 1, s7
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s9
+; GFX9-NEXT:    s_sub_i32 s10, s8, 64
+; GFX9-NEXT:    s_sub_i32 s9, 64, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX9-NEXT:    v_or_b32_e32 v6, v2, v6
+; GFX9-NEXT:    v_or_b32_e32 v7, v3, v7
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], s5, v[4:5]
+; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[2:3], s8
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[0:1], s8
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[2:3], s9
+; GFX9-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[2:3], s10
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[4:5], 0
+; GFX9-NEXT:    v_or_b32_e32 v0, s0, v4
+; GFX9-NEXT:    v_or_b32_e32 v1, s1, v5
+; GFX9-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: v_fshr_i128_vss:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s6, 0x7f
+; GFX10-NEXT:    s_mov_b32 s7, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[6:7], 1, v[2:3]
+; GFX10-NEXT:    s_and_b64 s[8:9], s[4:5], s[6:7]
+; GFX10-NEXT:    s_andn2_b64 s[6:7], s[6:7], s[4:5]
+; GFX10-NEXT:    s_sub_i32 s4, 64, 1
+; GFX10-NEXT:    s_sub_i32 s5, 1, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[4:5], s4, v[0:1]
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_lshlrev_b64 v[13:14], s5, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], 1, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX10-NEXT:    s_and_b32 s4, 1, vcc_lo
+; GFX10-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s4
+; GFX10-NEXT:    v_or_b32_e32 v5, v5, v7
+; GFX10-NEXT:    s_and_b32 s5, 1, s7
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0, v8, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v13, v6, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v14, v5, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0, v9, vcc_lo
+; GFX10-NEXT:    s_sub_i32 s5, s6, 64
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
+; GFX10-NEXT:    s_sub_i32 s4, 64, s6
+; GFX10-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[11:12], s4, v[4:5]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[6:7], s6, v[0:1]
+; GFX10-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], s6, v[4:5]
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    s_and_b32 s4, 1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b64 v[4:5], s5, v[4:5]
+; GFX10-NEXT:    v_or_b32_e32 v2, v11, v6
+; GFX10-NEXT:    v_or_b32_e32 v3, v12, v7
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s4
+; GFX10-NEXT:    s_sub_i32 s10, s8, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0, v8, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0, v9, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s4
+; GFX10-NEXT:    s_and_b32 s4, 1, s6
+; GFX10-NEXT:    s_sub_i32 s6, 64, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s4
+; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[4:5], s[0:1], s8
+; GFX10-NEXT:    s_lshl_b64 s[6:7], s[2:3], s6
+; GFX10-NEXT:    s_lshr_b64 s[8:9], s[2:3], s8
+; GFX10-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[2:3], s10
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc_lo
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX10-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v1, vcc_lo
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    v_or_b32_e32 v0, s0, v6
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[8:9], 0
+; GFX10-NEXT:    v_or_b32_e32 v1, s1, v7
+; GFX10-NEXT:    v_or_b32_e32 v2, s2, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
+  %cast.result = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast.result
+}
+
+define amdgpu_ps i128 @s_fshr_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: s_fshr_i128_65:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_i32 s14, 63, 64
+; GFX6-NEXT:    s_sub_i32 s10, 64, 63
+; GFX6-NEXT:    s_cmp_lt_u32 63, 64
+; GFX6-NEXT:    s_mov_b32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 63, 0
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[0:1], s10
+; GFX6-NEXT:    s_lshl_b32 s13, s2, 31
+; GFX6-NEXT:    s_mov_b32 s12, s8
+; GFX6-NEXT:    s_lshl_b32 s9, s0, 31
+; GFX6-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s14
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_movk_i32 s10, 0x41
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX6-NEXT:    s_sub_i32 s14, s10, 64
+; GFX6-NEXT:    s_sub_i32 s12, 64, s10
+; GFX6-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX6-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[6:7], s10
+; GFX6-NEXT:    s_lshl_b64 s[12:13], s[6:7], s12
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[4:5], s10
+; GFX6-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[10:11], s[6:7]
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX6-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[0:1], 0
+; GFX6-NEXT:    s_or_b64 s[0:1], s[8:9], s[4:5]
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i128_65:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_i32 s14, 63, 64
+; GFX8-NEXT:    s_sub_i32 s10, 64, 63
+; GFX8-NEXT:    s_cmp_lt_u32 63, 64
+; GFX8-NEXT:    s_mov_b32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 63, 0
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[0:1], s10
+; GFX8-NEXT:    s_lshl_b32 s13, s2, 31
+; GFX8-NEXT:    s_mov_b32 s12, s8
+; GFX8-NEXT:    s_lshl_b32 s9, s0, 31
+; GFX8-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s14
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_movk_i32 s10, 0x41
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX8-NEXT:    s_sub_i32 s14, s10, 64
+; GFX8-NEXT:    s_sub_i32 s12, 64, s10
+; GFX8-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX8-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[6:7], s10
+; GFX8-NEXT:    s_lshl_b64 s[12:13], s[6:7], s12
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[4:5], s10
+; GFX8-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[10:11], s[6:7]
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX8-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[0:1], 0
+; GFX8-NEXT:    s_or_b64 s[0:1], s[8:9], s[4:5]
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i128_65:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_i32 s14, 63, 64
+; GFX9-NEXT:    s_sub_i32 s10, 64, 63
+; GFX9-NEXT:    s_cmp_lt_u32 63, 64
+; GFX9-NEXT:    s_mov_b32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 63, 0
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[0:1], s10
+; GFX9-NEXT:    s_lshl_b32 s13, s2, 31
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_lshl_b32 s9, s0, 31
+; GFX9-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s14
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_movk_i32 s10, 0x41
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s14, s10, 64
+; GFX9-NEXT:    s_sub_i32 s12, 64, s10
+; GFX9-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX9-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[6:7], s10
+; GFX9-NEXT:    s_lshl_b64 s[12:13], s[6:7], s12
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[4:5], s10
+; GFX9-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[10:11], s[6:7]
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX9-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[0:1], 0
+; GFX9-NEXT:    s_or_b64 s[0:1], s[8:9], s[4:5]
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_i128_65:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sub_i32 s14, 63, 64
+; GFX10-NEXT:    s_sub_i32 s9, 64, 63
+; GFX10-NEXT:    s_cmp_lt_u32 63, 64
+; GFX10-NEXT:    s_mov_b32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 63, 0
+; GFX10-NEXT:    s_mov_b32 s12, s8
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[10:11], s[0:1], s9
+; GFX10-NEXT:    s_lshl_b32 s13, s2, 31
+; GFX10-NEXT:    s_lshl_b32 s9, s0, 31
+; GFX10-NEXT:    s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s14
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_movk_i32 s12, 0x41
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s14, s12, 64
+; GFX10-NEXT:    s_sub_i32 s10, 64, s12
+; GFX10-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[4:5], s12
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[6:7], s10
+; GFX10-NEXT:    s_lshr_b64 s[12:13], s[6:7], s12
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[6:7], s14
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s15, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[12:13], 0
+; GFX10-NEXT:    s_or_b64 s[0:1], s[8:9], s[0:1]
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 65)
+  ret i128 %result
+}
+
+define i128 @v_fshr_i128_65(i128 %lhs, i128 %rhs) {
+; GFX6-LABEL: v_fshr_i128_65:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_sub_i32 s5, 64, 63
+; GFX6-NEXT:    s_sub_i32 s4, 63, 64
+; GFX6-NEXT:    s_cmp_lt_u32 63, 64
+; GFX6-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 63, 0
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[0:1], s5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v10, 31, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v11, 31, v2
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s6
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX6-NEXT:    s_and_b32 s4, 1, s7
+; GFX6-NEXT:    v_cndmask_b32_e32 v10, 0, v10, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_movk_i32 s4, 0x41
+; GFX6-NEXT:    s_sub_i32 s5, s4, 64
+; GFX6-NEXT:    s_sub_i32 s6, 64, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX6-NEXT:    v_cndmask_b32_e32 v11, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v12, v1, v3, vcc
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[4:5], s4
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[6:7], s6
+; GFX6-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[6:7], s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s7
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[6:7], s5
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s8
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s7
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT:    v_or_b32_e32 v1, v10, v1
+; GFX6-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v12, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_i128_65:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_sub_i32 s5, 64, 63
+; GFX8-NEXT:    s_sub_i32 s4, 63, 64
+; GFX8-NEXT:    s_cmp_lt_u32 63, 64
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 63, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], s5, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b32_e32 v10, 31, v0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v11, 31, v2
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], s4, v[0:1]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX8-NEXT:    s_and_b32 s4, 1, s7
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, 0, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_movk_i32 s4, 0x41
+; GFX8-NEXT:    s_sub_i32 s5, s4, 64
+; GFX8-NEXT:    s_sub_i32 s6, 64, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, v1, v3, vcc
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s4, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], s6, v[6:7]
+; GFX8-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], s4, v[6:7]
+; GFX8-NEXT:    s_and_b32 s4, 1, s7
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s5, v[6:7]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s8
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s7
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT:    v_or_b32_e32 v1, v10, v1
+; GFX8-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v12, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_i128_65:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_sub_i32 s5, 64, 63
+; GFX9-NEXT:    s_sub_i32 s4, 63, 64
+; GFX9-NEXT:    s_cmp_lt_u32 63, 64
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 63, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], s5, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b32_e32 v10, 31, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v11, 31, v2
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], s4, v[0:1]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX9-NEXT:    s_and_b32 s4, 1, s7
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_movk_i32 s4, 0x41
+; GFX9-NEXT:    s_sub_i32 s5, s4, 64
+; GFX9-NEXT:    s_sub_i32 s6, 64, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s4, 64
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v12, v1, v3, vcc
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s4, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], s6, v[6:7]
+; GFX9-NEXT:    s_cselect_b32 s7, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], s4, v[6:7]
+; GFX9-NEXT:    s_and_b32 s4, 1, s7
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s5, v[6:7]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s8
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s7
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX9-NEXT:    v_or_b32_e32 v1, v10, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v12, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_i128_65:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_sub_i32 s4, 64, 63
+; GFX10-NEXT:    s_sub_i32 s5, 63, 64
+; GFX10-NEXT:    s_cmp_lt_u32 63, 64
+; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 31, v0
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 63, 0
+; GFX10-NEXT:    v_lshrrev_b64 v[8:9], s4, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[14:15], s5, v[0:1]
+; GFX10-NEXT:    s_and_b32 s5, 1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 31, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, 0, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s5
+; GFX10-NEXT:    s_movk_i32 s6, 0x41
+; GFX10-NEXT:    s_and_b32 s4, 1, s4
+; GFX10-NEXT:    s_sub_i32 s5, 64, s6
+; GFX10-NEXT:    v_or_b32_e32 v12, v9, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, v14, v8, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b64 v[10:11], s5, v[6:7]
+; GFX10-NEXT:    v_lshrrev_b64 v[8:9], s6, v[4:5]
+; GFX10-NEXT:    s_sub_i32 s5, s6, 64
+; GFX10-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v15, v12, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[15:16], s5, v[6:7]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX10-NEXT:    v_or_b32_e32 v8, v8, v10
+; GFX10-NEXT:    v_or_b32_e32 v9, v9, v11
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    v_lshrrev_b64 v[6:7], s6, v[6:7]
+; GFX10-NEXT:    s_and_b32 s5, 1, s5
+; GFX10-NEXT:    s_and_b32 s6, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v15, v8, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v16, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s5, 0, s5
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v19, v2, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v12, v3, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v15, v5, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v11, v4, s5
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0, v7, vcc_lo
+; GFX10-NEXT:    v_or_b32_e32 v1, v23, v1
+; GFX10-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX10-NEXT:    v_or_b32_e32 v3, v3, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 65)
+  ret i128 %result
+}
+
+define amdgpu_ps <2 x i128> @s_fshr_v2i128(<2 x i128> inreg %lhs, <2 x i128> inreg %rhs, <2 x i128> inreg %amt) {
+; GFX6-LABEL: s_fshr_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_movk_i32 s18, 0x7f
+; GFX6-NEXT:    s_mov_b32 s19, 0
+; GFX6-NEXT:    s_and_b64 s[22:23], s[16:17], s[18:19]
+; GFX6-NEXT:    s_andn2_b64 s[16:17], s[18:19], s[16:17]
+; GFX6-NEXT:    s_sub_i32 s30, 1, 64
+; GFX6-NEXT:    s_sub_i32 s31, 64, 1
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[26:27], s[0:1], s31
+; GFX6-NEXT:    s_lshl_b64 s[28:29], s[2:3], 1
+; GFX6-NEXT:    s_lshl_b64 s[24:25], s[0:1], 1
+; GFX6-NEXT:    s_or_b64 s[26:27], s[26:27], s[28:29]
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], s30
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_cselect_b64 s[24:25], s[24:25], 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[26:27], s[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX6-NEXT:    s_sub_i32 s23, s16, 64
+; GFX6-NEXT:    s_sub_i32 s17, 64, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s16, 64
+; GFX6-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s16, 0
+; GFX6-NEXT:    s_cselect_b32 s29, 1, 0
+; GFX6-NEXT:    s_lshl_b64 s[2:3], s[24:25], s16
+; GFX6-NEXT:    s_lshr_b64 s[26:27], s[24:25], s17
+; GFX6-NEXT:    s_lshl_b64 s[16:17], s[0:1], s16
+; GFX6-NEXT:    s_or_b64 s[16:17], s[26:27], s[16:17]
+; GFX6-NEXT:    s_lshl_b64 s[24:25], s[24:25], s23
+; GFX6-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], 0
+; GFX6-NEXT:    s_cselect_b64 s[16:17], s[16:17], s[24:25]
+; GFX6-NEXT:    s_cmp_lg_u32 s29, 0
+; GFX6-NEXT:    s_cselect_b64 s[16:17], s[0:1], s[16:17]
+; GFX6-NEXT:    s_sub_i32 s26, s22, 64
+; GFX6-NEXT:    s_sub_i32 s24, 64, s22
+; GFX6-NEXT:    s_cmp_lt_u32 s22, 64
+; GFX6-NEXT:    s_cselect_b32 s27, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s22, 0
+; GFX6-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[10:11], s22
+; GFX6-NEXT:    s_lshl_b64 s[24:25], s[10:11], s24
+; GFX6-NEXT:    s_lshr_b64 s[22:23], s[8:9], s22
+; GFX6-NEXT:    s_or_b64 s[22:23], s[22:23], s[24:25]
+; GFX6-NEXT:    s_lshr_b64 s[10:11], s[10:11], s26
+; GFX6-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX6-NEXT:    s_cselect_b64 s[10:11], s[22:23], s[10:11]
+; GFX6-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX6-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX6-NEXT:    s_cselect_b64 s[10:11], s[0:1], 0
+; GFX6-NEXT:    s_or_b64 s[0:1], s[2:3], s[8:9]
+; GFX6-NEXT:    s_or_b64 s[2:3], s[16:17], s[10:11]
+; GFX6-NEXT:    s_and_b64 s[8:9], s[20:21], s[18:19]
+; GFX6-NEXT:    s_andn2_b64 s[10:11], s[18:19], s[20:21]
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[18:19], s[4:5], s31
+; GFX6-NEXT:    s_lshl_b64 s[20:21], s[6:7], 1
+; GFX6-NEXT:    s_lshl_b64 s[16:17], s[4:5], 1
+; GFX6-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[4:5], s30
+; GFX6-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX6-NEXT:    s_cselect_b64 s[16:17], s[16:17], 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[18:19], s[4:5]
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX6-NEXT:    s_sub_i32 s9, s10, 64
+; GFX6-NEXT:    s_sub_i32 s11, 64, s10
+; GFX6-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX6-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX6-NEXT:    s_cselect_b32 s21, 1, 0
+; GFX6-NEXT:    s_lshl_b64 s[6:7], s[16:17], s10
+; GFX6-NEXT:    s_lshr_b64 s[18:19], s[16:17], s11
+; GFX6-NEXT:    s_lshl_b64 s[10:11], s[4:5], s10
+; GFX6-NEXT:    s_or_b64 s[10:11], s[18:19], s[10:11]
+; GFX6-NEXT:    s_lshl_b64 s[16:17], s[16:17], s9
+; GFX6-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[6:7], 0
+; GFX6-NEXT:    s_cselect_b64 s[10:11], s[10:11], s[16:17]
+; GFX6-NEXT:    s_cmp_lg_u32 s21, 0
+; GFX6-NEXT:    s_cselect_b64 s[10:11], s[4:5], s[10:11]
+; GFX6-NEXT:    s_sub_i32 s18, s8, 64
+; GFX6-NEXT:    s_sub_i32 s16, 64, s8
+; GFX6-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX6-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[14:15], s8
+; GFX6-NEXT:    s_lshl_b64 s[16:17], s[14:15], s16
+; GFX6-NEXT:    s_lshr_b64 s[8:9], s[12:13], s8
+; GFX6-NEXT:    s_or_b64 s[8:9], s[8:9], s[16:17]
+; GFX6-NEXT:    s_lshr_b64 s[14:15], s[14:15], s18
+; GFX6-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[14:15]
+; GFX6-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX6-NEXT:    s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; GFX6-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX6-NEXT:    s_cselect_b64 s[12:13], s[4:5], 0
+; GFX6-NEXT:    s_or_b64 s[4:5], s[6:7], s[8:9]
+; GFX6-NEXT:    s_or_b64 s[6:7], s[10:11], s[12:13]
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_movk_i32 s18, 0x7f
+; GFX8-NEXT:    s_mov_b32 s19, 0
+; GFX8-NEXT:    s_and_b64 s[22:23], s[16:17], s[18:19]
+; GFX8-NEXT:    s_andn2_b64 s[16:17], s[18:19], s[16:17]
+; GFX8-NEXT:    s_sub_i32 s30, 1, 64
+; GFX8-NEXT:    s_sub_i32 s31, 64, 1
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[26:27], s[0:1], s31
+; GFX8-NEXT:    s_lshl_b64 s[28:29], s[2:3], 1
+; GFX8-NEXT:    s_lshl_b64 s[24:25], s[0:1], 1
+; GFX8-NEXT:    s_or_b64 s[26:27], s[26:27], s[28:29]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], s30
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_cselect_b64 s[24:25], s[24:25], 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[26:27], s[0:1]
+; GFX8-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX8-NEXT:    s_sub_i32 s23, s16, 64
+; GFX8-NEXT:    s_sub_i32 s17, 64, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s16, 64
+; GFX8-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s16, 0
+; GFX8-NEXT:    s_cselect_b32 s29, 1, 0
+; GFX8-NEXT:    s_lshl_b64 s[2:3], s[24:25], s16
+; GFX8-NEXT:    s_lshr_b64 s[26:27], s[24:25], s17
+; GFX8-NEXT:    s_lshl_b64 s[16:17], s[0:1], s16
+; GFX8-NEXT:    s_or_b64 s[16:17], s[26:27], s[16:17]
+; GFX8-NEXT:    s_lshl_b64 s[24:25], s[24:25], s23
+; GFX8-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], 0
+; GFX8-NEXT:    s_cselect_b64 s[16:17], s[16:17], s[24:25]
+; GFX8-NEXT:    s_cmp_lg_u32 s29, 0
+; GFX8-NEXT:    s_cselect_b64 s[16:17], s[0:1], s[16:17]
+; GFX8-NEXT:    s_sub_i32 s26, s22, 64
+; GFX8-NEXT:    s_sub_i32 s24, 64, s22
+; GFX8-NEXT:    s_cmp_lt_u32 s22, 64
+; GFX8-NEXT:    s_cselect_b32 s27, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s22, 0
+; GFX8-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[10:11], s22
+; GFX8-NEXT:    s_lshl_b64 s[24:25], s[10:11], s24
+; GFX8-NEXT:    s_lshr_b64 s[22:23], s[8:9], s22
+; GFX8-NEXT:    s_or_b64 s[22:23], s[22:23], s[24:25]
+; GFX8-NEXT:    s_lshr_b64 s[10:11], s[10:11], s26
+; GFX8-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX8-NEXT:    s_cselect_b64 s[10:11], s[22:23], s[10:11]
+; GFX8-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX8-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX8-NEXT:    s_cselect_b64 s[10:11], s[0:1], 0
+; GFX8-NEXT:    s_or_b64 s[0:1], s[2:3], s[8:9]
+; GFX8-NEXT:    s_or_b64 s[2:3], s[16:17], s[10:11]
+; GFX8-NEXT:    s_and_b64 s[8:9], s[20:21], s[18:19]
+; GFX8-NEXT:    s_andn2_b64 s[10:11], s[18:19], s[20:21]
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[18:19], s[4:5], s31
+; GFX8-NEXT:    s_lshl_b64 s[20:21], s[6:7], 1
+; GFX8-NEXT:    s_lshl_b64 s[16:17], s[4:5], 1
+; GFX8-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[4:5], s30
+; GFX8-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX8-NEXT:    s_cselect_b64 s[16:17], s[16:17], 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[18:19], s[4:5]
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX8-NEXT:    s_sub_i32 s9, s10, 64
+; GFX8-NEXT:    s_sub_i32 s11, 64, s10
+; GFX8-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX8-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX8-NEXT:    s_cselect_b32 s21, 1, 0
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[16:17], s10
+; GFX8-NEXT:    s_lshr_b64 s[18:19], s[16:17], s11
+; GFX8-NEXT:    s_lshl_b64 s[10:11], s[4:5], s10
+; GFX8-NEXT:    s_or_b64 s[10:11], s[18:19], s[10:11]
+; GFX8-NEXT:    s_lshl_b64 s[16:17], s[16:17], s9
+; GFX8-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[6:7], 0
+; GFX8-NEXT:    s_cselect_b64 s[10:11], s[10:11], s[16:17]
+; GFX8-NEXT:    s_cmp_lg_u32 s21, 0
+; GFX8-NEXT:    s_cselect_b64 s[10:11], s[4:5], s[10:11]
+; GFX8-NEXT:    s_sub_i32 s18, s8, 64
+; GFX8-NEXT:    s_sub_i32 s16, 64, s8
+; GFX8-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[4:5], s[14:15], s8
+; GFX8-NEXT:    s_lshl_b64 s[16:17], s[14:15], s16
+; GFX8-NEXT:    s_lshr_b64 s[8:9], s[12:13], s8
+; GFX8-NEXT:    s_or_b64 s[8:9], s[8:9], s[16:17]
+; GFX8-NEXT:    s_lshr_b64 s[14:15], s[14:15], s18
+; GFX8-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[14:15]
+; GFX8-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX8-NEXT:    s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; GFX8-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX8-NEXT:    s_cselect_b64 s[12:13], s[4:5], 0
+; GFX8-NEXT:    s_or_b64 s[4:5], s[6:7], s[8:9]
+; GFX8-NEXT:    s_or_b64 s[6:7], s[10:11], s[12:13]
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s18, 0x7f
+; GFX9-NEXT:    s_mov_b32 s19, 0
+; GFX9-NEXT:    s_and_b64 s[22:23], s[16:17], s[18:19]
+; GFX9-NEXT:    s_andn2_b64 s[16:17], s[18:19], s[16:17]
+; GFX9-NEXT:    s_sub_i32 s30, 1, 64
+; GFX9-NEXT:    s_sub_i32 s31, 64, 1
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[26:27], s[0:1], s31
+; GFX9-NEXT:    s_lshl_b64 s[28:29], s[2:3], 1
+; GFX9-NEXT:    s_lshl_b64 s[24:25], s[0:1], 1
+; GFX9-NEXT:    s_or_b64 s[26:27], s[26:27], s[28:29]
+; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], s30
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_cselect_b64 s[24:25], s[24:25], 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[26:27], s[0:1]
+; GFX9-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s23, s16, 64
+; GFX9-NEXT:    s_sub_i32 s17, 64, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s16, 64
+; GFX9-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s16, 0
+; GFX9-NEXT:    s_cselect_b32 s29, 1, 0
+; GFX9-NEXT:    s_lshl_b64 s[2:3], s[24:25], s16
+; GFX9-NEXT:    s_lshr_b64 s[26:27], s[24:25], s17
+; GFX9-NEXT:    s_lshl_b64 s[16:17], s[0:1], s16
+; GFX9-NEXT:    s_or_b64 s[16:17], s[26:27], s[16:17]
+; GFX9-NEXT:    s_lshl_b64 s[24:25], s[24:25], s23
+; GFX9-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], 0
+; GFX9-NEXT:    s_cselect_b64 s[16:17], s[16:17], s[24:25]
+; GFX9-NEXT:    s_cmp_lg_u32 s29, 0
+; GFX9-NEXT:    s_cselect_b64 s[16:17], s[0:1], s[16:17]
+; GFX9-NEXT:    s_sub_i32 s26, s22, 64
+; GFX9-NEXT:    s_sub_i32 s24, 64, s22
+; GFX9-NEXT:    s_cmp_lt_u32 s22, 64
+; GFX9-NEXT:    s_cselect_b32 s27, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s22, 0
+; GFX9-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[10:11], s22
+; GFX9-NEXT:    s_lshl_b64 s[24:25], s[10:11], s24
+; GFX9-NEXT:    s_lshr_b64 s[22:23], s[8:9], s22
+; GFX9-NEXT:    s_or_b64 s[22:23], s[22:23], s[24:25]
+; GFX9-NEXT:    s_lshr_b64 s[10:11], s[10:11], s26
+; GFX9-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX9-NEXT:    s_cselect_b64 s[10:11], s[22:23], s[10:11]
+; GFX9-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX9-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX9-NEXT:    s_cselect_b64 s[10:11], s[0:1], 0
+; GFX9-NEXT:    s_or_b64 s[0:1], s[2:3], s[8:9]
+; GFX9-NEXT:    s_or_b64 s[2:3], s[16:17], s[10:11]
+; GFX9-NEXT:    s_and_b64 s[8:9], s[20:21], s[18:19]
+; GFX9-NEXT:    s_andn2_b64 s[10:11], s[18:19], s[20:21]
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[18:19], s[4:5], s31
+; GFX9-NEXT:    s_lshl_b64 s[20:21], s[6:7], 1
+; GFX9-NEXT:    s_lshl_b64 s[16:17], s[4:5], 1
+; GFX9-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], s30
+; GFX9-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX9-NEXT:    s_cselect_b64 s[16:17], s[16:17], 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[18:19], s[4:5]
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX9-NEXT:    s_sub_i32 s9, s10, 64
+; GFX9-NEXT:    s_sub_i32 s11, 64, s10
+; GFX9-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
+; GFX9-NEXT:    s_lshl_b64 s[6:7], s[16:17], s10
+; GFX9-NEXT:    s_lshr_b64 s[18:19], s[16:17], s11
+; GFX9-NEXT:    s_lshl_b64 s[10:11], s[4:5], s10
+; GFX9-NEXT:    s_or_b64 s[10:11], s[18:19], s[10:11]
+; GFX9-NEXT:    s_lshl_b64 s[16:17], s[16:17], s9
+; GFX9-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[6:7], 0
+; GFX9-NEXT:    s_cselect_b64 s[10:11], s[10:11], s[16:17]
+; GFX9-NEXT:    s_cmp_lg_u32 s21, 0
+; GFX9-NEXT:    s_cselect_b64 s[10:11], s[4:5], s[10:11]
+; GFX9-NEXT:    s_sub_i32 s18, s8, 64
+; GFX9-NEXT:    s_sub_i32 s16, 64, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[14:15], s8
+; GFX9-NEXT:    s_lshl_b64 s[16:17], s[14:15], s16
+; GFX9-NEXT:    s_lshr_b64 s[8:9], s[12:13], s8
+; GFX9-NEXT:    s_or_b64 s[8:9], s[8:9], s[16:17]
+; GFX9-NEXT:    s_lshr_b64 s[14:15], s[14:15], s18
+; GFX9-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[8:9], s[14:15]
+; GFX9-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX9-NEXT:    s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; GFX9-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX9-NEXT:    s_cselect_b64 s[12:13], s[4:5], 0
+; GFX9-NEXT:    s_or_b64 s[4:5], s[6:7], s[8:9]
+; GFX9-NEXT:    s_or_b64 s[6:7], s[10:11], s[12:13]
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_fshr_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_movk_i32 s18, 0x7f
+; GFX10-NEXT:    s_mov_b32 s19, 0
+; GFX10-NEXT:    s_sub_i32 s30, 1, 64
+; GFX10-NEXT:    s_and_b64 s[22:23], s[16:17], s[18:19]
+; GFX10-NEXT:    s_andn2_b64 s[16:17], s[18:19], s[16:17]
+; GFX10-NEXT:    s_sub_i32 s31, 64, 1
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    s_mov_b32 s62, s10
+; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    s_mov_b32 s63, s11
+; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[24:25], s[0:1], s31
+; GFX10-NEXT:    s_lshl_b64 s[26:27], s[2:3], 1
+; GFX10-NEXT:    s_lshl_b64 s[28:29], s[0:1], 1
+; GFX10-NEXT:    s_or_b64 s[24:25], s[24:25], s[26:27]
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], s30
+; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX10-NEXT:    s_cselect_b64 s[26:27], s[28:29], 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[24:25], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX10-NEXT:    s_cselect_b64 s[46:47], s[2:3], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s23, s16, 64
+; GFX10-NEXT:    s_sub_i32 s2, 64, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s16, 64
+; GFX10-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s16, 0
+; GFX10-NEXT:    s_cselect_b32 s29, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[24:25], s[46:47], s16
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[26:27], s2
+; GFX10-NEXT:    s_lshl_b64 s[16:17], s[26:27], s16
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[24:25]
+; GFX10-NEXT:    s_lshl_b64 s[24:25], s[26:27], s23
+; GFX10-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX10-NEXT:    s_cselect_b64 s[78:79], s[16:17], 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[24:25]
+; GFX10-NEXT:    s_cmp_lg_u32 s29, 0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[46:47], s[2:3]
+; GFX10-NEXT:    s_sub_i32 s26, s22, 64
+; GFX10-NEXT:    s_sub_i32 s23, 64, s22
+; GFX10-NEXT:    s_cmp_lt_u32 s22, 64
+; GFX10-NEXT:    s_cselect_b32 s27, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s22, 0
+; GFX10-NEXT:    s_cselect_b32 s28, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[8:9], s22
+; GFX10-NEXT:    s_lshl_b64 s[24:25], s[62:63], s23
+; GFX10-NEXT:    s_lshr_b64 s[22:23], s[62:63], s22
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[24:25]
+; GFX10-NEXT:    s_lshr_b64 s[10:11], s[62:63], s26
+; GFX10-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT:    s_cmp_lg_u32 s28, 0
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s27, 0
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[22:23], 0
+; GFX10-NEXT:    s_or_b64 s[0:1], s[78:79], s[0:1]
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX10-NEXT:    s_and_b64 s[8:9], s[20:21], s[18:19]
+; GFX10-NEXT:    s_andn2_b64 s[10:11], s[18:19], s[20:21]
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[16:17], s[4:5], s31
+; GFX10-NEXT:    s_lshl_b64 s[18:19], s[6:7], 1
+; GFX10-NEXT:    s_lshl_b64 s[20:21], s[4:5], 1
+; GFX10-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GFX10-NEXT:    s_lshl_b64 s[4:5], s[4:5], s30
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_cselect_b64 s[18:19], s[20:21], 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[16:17], s[4:5]
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX10-NEXT:    s_sub_i32 s9, s10, 64
+; GFX10-NEXT:    s_sub_i32 s6, 64, s10
+; GFX10-NEXT:    s_cmp_lt_u32 s10, 64
+; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s10, 0
+; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[16:17], s[4:5], s10
+; GFX10-NEXT:    s_lshr_b64 s[6:7], s[18:19], s6
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[18:19], s10
+; GFX10-NEXT:    s_or_b64 s[6:7], s[6:7], s[16:17]
+; GFX10-NEXT:    s_lshl_b64 s[16:17], s[18:19], s9
+; GFX10-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX10-NEXT:    s_cselect_b64 s[10:11], s[10:11], 0
+; GFX10-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[16:17]
+; GFX10-NEXT:    s_cmp_lg_u32 s21, 0
+; GFX10-NEXT:    s_cselect_b64 s[6:7], s[4:5], s[6:7]
+; GFX10-NEXT:    s_sub_i32 s18, s8, 64
+; GFX10-NEXT:    s_sub_i32 s9, 64, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s8, 64
+; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX10-NEXT:    s_lshr_b64 s[4:5], s[12:13], s8
+; GFX10-NEXT:    s_lshl_b64 s[16:17], s[14:15], s9
+; GFX10-NEXT:    s_lshr_b64 s[8:9], s[14:15], s8
+; GFX10-NEXT:    s_or_b64 s[4:5], s[4:5], s[16:17]
+; GFX10-NEXT:    s_lshr_b64 s[14:15], s[14:15], s18
+; GFX10-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[14:15]
+; GFX10-NEXT:    s_cmp_lg_u32 s20, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[12:13], s[4:5]
+; GFX10-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX10-NEXT:    s_cselect_b64 s[8:9], s[8:9], 0
+; GFX10-NEXT:    s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX10-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
+  ret <2 x i128> %result
+}
+
+define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt) {
+; GFX6-LABEL: v_fshr_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_sub_i32 s6, 64, 1
+; GFX6-NEXT:    s_sub_i32 s7, 1, 64
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    v_lshr_b64 v[17:18], v[0:1], s6
+; GFX6-NEXT:    v_lshl_b64 v[21:22], v[2:3], 1
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    v_or_b32_e32 v19, v17, v21
+; GFX6-NEXT:    v_or_b32_e32 v21, v18, v22
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    v_lshl_b64 v[17:18], v[0:1], 1
+; GFX6-NEXT:    s_and_b32 s4, 1, s4
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s5
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v19, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
+; GFX6-NEXT:    s_movk_i32 s8, 0x7f
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v16
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v21, vcc
+; GFX6-NEXT:    v_and_b32_e32 v19, s8, v2
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v17, 0, v17, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v18, 0, v18, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 64, v19
+; GFX6-NEXT:    v_lshr_b64 v[2:3], v[17:18], v2
+; GFX6-NEXT:    v_lshl_b64 v[21:22], v[0:1], v19
+; GFX6-NEXT:    v_and_b32_e32 v25, s8, v16
+; GFX6-NEXT:    v_or_b32_e32 v23, v2, v21
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 64, v25
+; GFX6-NEXT:    v_or_b32_e32 v24, v3, v22
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[10:11], v2
+; GFX6-NEXT:    v_lshr_b64 v[21:22], v[8:9], v25
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GFX6-NEXT:    v_or_b32_e32 v21, v21, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 64, v19
+; GFX6-NEXT:    v_or_b32_e32 v22, v22, v3
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[17:18], v2
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v23, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v24, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, v1, s[4:5]
+; GFX6-NEXT:    v_subrev_i32_e64 v0, s[4:5], 64, v25
+; GFX6-NEXT:    v_lshl_b64 v[16:17], v[17:18], v19
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[10:11], v0
+; GFX6-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v25
+; GFX6-NEXT:    v_cndmask_b32_e32 v16, 0, v16, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v17, 0, v17, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v18, v0, v21, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v19, v1, v22, s[4:5]
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[10:11], v25
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v25
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, v18, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[4:5]
+; GFX6-NEXT:    v_or_b32_e32 v0, v16, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v9, v19, v9, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v20
+; GFX6-NEXT:    v_or_b32_e32 v1, v17, v9
+; GFX6-NEXT:    v_and_b32_e32 v17, s8, v8
+; GFX6-NEXT:    s_cmp_lt_u32 1, 64
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v10
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v11
+; GFX6-NEXT:    v_lshr_b64 v[8:9], v[4:5], s6
+; GFX6-NEXT:    v_lshl_b64 v[10:11], v[6:7], 1
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 1, 0
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_and_b32 s4, 1, s4
+; GFX6-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX6-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX6-NEXT:    v_lshl_b64 v[8:9], v[4:5], 1
+; GFX6-NEXT:    v_lshl_b64 v[4:5], v[4:5], s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s5
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, 0, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v9, 0, v9, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, 64, v17
+; GFX6-NEXT:    v_lshr_b64 v[6:7], v[8:9], v6
+; GFX6-NEXT:    v_lshl_b64 v[10:11], v[4:5], v17
+; GFX6-NEXT:    v_subrev_i32_e32 v18, vcc, 64, v17
+; GFX6-NEXT:    v_or_b32_e32 v10, v6, v10
+; GFX6-NEXT:    v_or_b32_e32 v11, v7, v11
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[8:9], v17
+; GFX6-NEXT:    v_lshl_b64 v[8:9], v[8:9], v18
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v17
+; GFX6-NEXT:    v_and_b32_e32 v16, s8, v20
+; GFX6-NEXT:    v_cndmask_b32_e32 v18, 0, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v19, 0, v7, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v9, v11, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v17
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, v6, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v9, v7, v5, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, 64, v16
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[12:13], v16
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[14:15], v6
+; GFX6-NEXT:    v_subrev_i32_e32 v10, vcc, 64, v16
+; GFX6-NEXT:    v_or_b32_e32 v11, v4, v6
+; GFX6-NEXT:    v_or_b32_e32 v17, v5, v7
+; GFX6-NEXT:    v_lshr_b64 v[6:7], v[14:15], v10
+; GFX6-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[14:15], v16
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v6, v11, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v16
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v7, v17, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, v6, v12, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v10, 0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v7, v7, v13, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v11, 0, v5, vcc
+; GFX6-NEXT:    v_or_b32_e32 v4, v18, v6
+; GFX6-NEXT:    v_or_b32_e32 v5, v19, v7
+; GFX6-NEXT:    v_or_b32_e32 v6, v8, v10
+; GFX6-NEXT:    v_or_b32_e32 v7, v9, v11
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshr_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_sub_i32 s6, 64, 1
+; GFX8-NEXT:    s_sub_i32 s7, 1, 64
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    v_lshrrev_b64 v[17:18], s6, v[0:1]
+; GFX8-NEXT:    v_lshlrev_b64 v[21:22], 1, v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    v_or_b32_e32 v19, v17, v21
+; GFX8-NEXT:    v_or_b32_e32 v21, v18, v22
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    v_lshlrev_b64 v[17:18], 1, v[0:1]
+; GFX8-NEXT:    s_and_b32 s4, 1, s4
+; GFX8-NEXT:    v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v19, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
+; GFX8-NEXT:    s_movk_i32 s8, 0x7f
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v21, vcc
+; GFX8-NEXT:    v_and_b32_e32 v19, s8, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v17, 0, v17, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, 0, v18, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 64, v19
+; GFX8-NEXT:    v_lshrrev_b64 v[2:3], v2, v[17:18]
+; GFX8-NEXT:    v_lshlrev_b64 v[21:22], v19, v[0:1]
+; GFX8-NEXT:    v_and_b32_e32 v25, s8, v16
+; GFX8-NEXT:    v_or_b32_e32 v23, v2, v21
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 64, v25
+; GFX8-NEXT:    v_or_b32_e32 v24, v3, v22
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v2, v[10:11]
+; GFX8-NEXT:    v_lshrrev_b64 v[21:22], v25, v[8:9]
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GFX8-NEXT:    v_or_b32_e32 v21, v21, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 64, v19
+; GFX8-NEXT:    v_or_b32_e32 v22, v22, v3
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], v2, v[17:18]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v23, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v24, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v1, s[4:5]
+; GFX8-NEXT:    v_subrev_u32_e64 v0, s[4:5], 64, v25
+; GFX8-NEXT:    v_lshlrev_b64 v[16:17], v19, v[17:18]
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v0, v[10:11]
+; GFX8-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v25
+; GFX8-NEXT:    v_cndmask_b32_e32 v16, 0, v16, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v17, 0, v17, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v18, v0, v21, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v19, v1, v22, s[4:5]
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], v25, v[10:11]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v25
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v18, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[4:5]
+; GFX8-NEXT:    v_or_b32_e32 v0, v16, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v19, v9, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v20
+; GFX8-NEXT:    v_or_b32_e32 v1, v17, v9
+; GFX8-NEXT:    v_and_b32_e32 v17, s8, v8
+; GFX8-NEXT:    s_cmp_lt_u32 1, 64
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v10
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v11
+; GFX8-NEXT:    v_lshrrev_b64 v[8:9], s6, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[10:11], 1, v[6:7]
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 1, 0
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_and_b32 s4, 1, s4
+; GFX8-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX8-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX8-NEXT:    v_lshlrev_b64 v[8:9], 1, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[4:5], s7, v[4:5]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, 0, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, 0, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, 64, v17
+; GFX8-NEXT:    v_lshrrev_b64 v[6:7], v6, v[8:9]
+; GFX8-NEXT:    v_lshlrev_b64 v[10:11], v17, v[4:5]
+; GFX8-NEXT:    v_subrev_u32_e32 v18, vcc, 64, v17
+; GFX8-NEXT:    v_or_b32_e32 v10, v6, v10
+; GFX8-NEXT:    v_or_b32_e32 v11, v7, v11
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], v17, v[8:9]
+; GFX8-NEXT:    v_lshlrev_b64 v[8:9], v18, v[8:9]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v17
+; GFX8-NEXT:    v_and_b32_e32 v16, s8, v20
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, 0, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v19, 0, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v9, v11, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v6, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v7, v5, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, 64, v16
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], v16, v[12:13]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], v6, v[14:15]
+; GFX8-NEXT:    v_subrev_u32_e32 v10, vcc, 64, v16
+; GFX8-NEXT:    v_or_b32_e32 v11, v4, v6
+; GFX8-NEXT:    v_or_b32_e32 v17, v5, v7
+; GFX8-NEXT:    v_lshrrev_b64 v[6:7], v10, v[14:15]
+; GFX8-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], v16, v[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v11, vcc
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v17, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, v12, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, 0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, v13, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v11, 0, v5, vcc
+; GFX8-NEXT:    v_or_b32_e32 v4, v18, v6
+; GFX8-NEXT:    v_or_b32_e32 v5, v19, v7
+; GFX8-NEXT:    v_or_b32_e32 v6, v8, v10
+; GFX8-NEXT:    v_or_b32_e32 v7, v9, v11
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fshr_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_sub_i32 s6, 64, 1
+; GFX9-NEXT:    s_sub_i32 s7, 1, 64
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    v_lshrrev_b64 v[17:18], s6, v[0:1]
+; GFX9-NEXT:    v_lshlrev_b64 v[21:22], 1, v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    v_or_b32_e32 v19, v17, v21
+; GFX9-NEXT:    v_or_b32_e32 v21, v18, v22
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    v_lshlrev_b64 v[17:18], 1, v[0:1]
+; GFX9-NEXT:    s_and_b32 s4, 1, s4
+; GFX9-NEXT:    v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v19, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
+; GFX9-NEXT:    s_movk_i32 s8, 0x7f
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v16
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v21, vcc
+; GFX9-NEXT:    v_and_b32_e32 v19, s8, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v17, 0, v17, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v18, 0, v18, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v2, 64, v19
+; GFX9-NEXT:    v_lshrrev_b64 v[2:3], v2, v[17:18]
+; GFX9-NEXT:    v_lshlrev_b64 v[21:22], v19, v[0:1]
+; GFX9-NEXT:    v_and_b32_e32 v25, s8, v16
+; GFX9-NEXT:    v_or_b32_e32 v23, v2, v21
+; GFX9-NEXT:    v_sub_u32_e32 v2, 64, v25
+; GFX9-NEXT:    v_or_b32_e32 v24, v3, v22
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v2, v[10:11]
+; GFX9-NEXT:    v_lshrrev_b64 v[21:22], v25, v[8:9]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX9-NEXT:    v_or_b32_e32 v21, v21, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v2, 64, v19
+; GFX9-NEXT:    v_or_b32_e32 v22, v22, v3
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], v2, v[17:18]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v23, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v24, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
+; GFX9-NEXT:    v_subrev_u32_e32 v0, 64, v25
+; GFX9-NEXT:    v_lshlrev_b64 v[16:17], v19, v[17:18]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v1, s[4:5]
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v0, v[10:11]
+; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v25
+; GFX9-NEXT:    v_cndmask_b32_e32 v16, 0, v16, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v17, 0, v17, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v18, v0, v21, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v19, v1, v22, s[4:5]
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], v25, v[10:11]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v25
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, v18, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[4:5]
+; GFX9-NEXT:    v_or_b32_e32 v0, v16, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, v1, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, v19, v9, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v20
+; GFX9-NEXT:    v_or_b32_e32 v1, v17, v9
+; GFX9-NEXT:    v_and_b32_e32 v17, s8, v8
+; GFX9-NEXT:    s_cmp_lt_u32 1, 64
+; GFX9-NEXT:    v_or_b32_e32 v2, v2, v10
+; GFX9-NEXT:    v_or_b32_e32 v3, v3, v11
+; GFX9-NEXT:    v_lshrrev_b64 v[8:9], s6, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[10:11], 1, v[6:7]
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 1, 0
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_and_b32 s4, 1, s4
+; GFX9-NEXT:    v_or_b32_e32 v10, v8, v10
+; GFX9-NEXT:    v_or_b32_e32 v11, v9, v11
+; GFX9-NEXT:    v_lshlrev_b64 v[8:9], 1, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[4:5], s7, v[4:5]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, 0, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, 0, v9, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v6, 64, v17
+; GFX9-NEXT:    v_lshrrev_b64 v[6:7], v6, v[8:9]
+; GFX9-NEXT:    v_lshlrev_b64 v[10:11], v17, v[4:5]
+; GFX9-NEXT:    v_subrev_u32_e32 v18, 64, v17
+; GFX9-NEXT:    v_or_b32_e32 v10, v6, v10
+; GFX9-NEXT:    v_or_b32_e32 v11, v7, v11
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], v17, v[8:9]
+; GFX9-NEXT:    v_lshlrev_b64 v[8:9], v18, v[8:9]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v17
+; GFX9-NEXT:    v_and_b32_e32 v16, s8, v20
+; GFX9-NEXT:    v_cndmask_b32_e32 v18, 0, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v19, 0, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v9, v11, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v17
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, v6, v4, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v6, 64, v16
+; GFX9-NEXT:    v_cndmask_b32_e32 v9, v7, v5, vcc
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], v16, v[12:13]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], v6, v[14:15]
+; GFX9-NEXT:    v_subrev_u32_e32 v10, 64, v16
+; GFX9-NEXT:    v_or_b32_e32 v11, v4, v6
+; GFX9-NEXT:    v_or_b32_e32 v17, v5, v7
+; GFX9-NEXT:    v_lshrrev_b64 v[6:7], v10, v[14:15]
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], v16, v[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v11, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v16
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v17, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v12, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v10, 0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, v13, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, 0, v5, vcc
+; GFX9-NEXT:    v_or_b32_e32 v4, v18, v6
+; GFX9-NEXT:    v_or_b32_e32 v5, v19, v7
+; GFX9-NEXT:    v_or_b32_e32 v6, v8, v10
+; GFX9-NEXT:    v_or_b32_e32 v7, v9, v11
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_fshr_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_sub_i32 s5, 64, 1
+; GFX10-NEXT:    s_sub_i32 s6, 1, 64
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_lshrrev_b64 v[27:28], s5, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[21:22], 1, v[2:3]
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[23:24], 1, v[0:1]
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    s_and_b32 s4, 1, vcc_lo
+; GFX10-NEXT:    v_lshlrev_b64 v[0:1], s6, v[0:1]
+; GFX10-NEXT:    v_xor_b32_e32 v19, -1, v16
+; GFX10-NEXT:    v_or_b32_e32 v21, v27, v21
+; GFX10-NEXT:    v_or_b32_e32 v18, v28, v22
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s4
+; GFX10-NEXT:    s_movk_i32 s7, 0x7f
+; GFX10-NEXT:    s_and_b32 s8, 1, s8
+; GFX10-NEXT:    v_and_b32_e32 v31, s7, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v17, 0, v23, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v18, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v21, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s8
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, 0, v24, vcc_lo
+; GFX10-NEXT:    v_sub_nc_u32_e32 v19, 64, v31
+; GFX10-NEXT:    v_and_b32_e32 v26, s7, v16
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v16, 64, v31
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], v19, v[17:18]
+; GFX10-NEXT:    v_mov_b32_e32 v35, v10
+; GFX10-NEXT:    v_mov_b32_e32 v36, v11
+; GFX10-NEXT:    v_sub_nc_u32_e32 v25, 64, v26
+; GFX10-NEXT:    v_lshlrev_b64 v[21:22], v31, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[23:24], v31, v[17:18]
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 64, v31
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v29, 64, v26
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s4, 64, v26
+; GFX10-NEXT:    v_lshrrev_b64 v[27:28], s5, v[4:5]
+; GFX10-NEXT:    v_or_b32_e32 v21, v2, v21
+; GFX10-NEXT:    v_or_b32_e32 v22, v3, v22
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], v16, v[17:18]
+; GFX10-NEXT:    v_lshlrev_b64 v[18:19], v25, v[35:36]
+; GFX10-NEXT:    v_lshrrev_b64 v[16:17], v26, v[8:9]
+; GFX10-NEXT:    v_cndmask_b32_e32 v23, 0, v23, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v39, 0, v24, vcc_lo
+; GFX10-NEXT:    s_cmp_lt_u32 1, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v2, v21, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v22, v3, v22, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[2:3], v29, v[35:36]
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v31
+; GFX10-NEXT:    v_or_b32_e32 v16, v16, v18
+; GFX10-NEXT:    v_or_b32_e32 v17, v17, v19
+; GFX10-NEXT:    v_lshlrev_b64 v[10:11], 1, v[6:7]
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v18, v21, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v31, v22, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s4
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v26
+; GFX10-NEXT:    s_cmp_eq_u32 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v19, v3, v17, s4
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_lshlrev_b64 v[16:17], 1, v[4:5]
+; GFX10-NEXT:    v_lshlrev_b64 v[4:5], s6, v[4:5]
+; GFX10-NEXT:    s_and_b32 s6, 1, s5
+; GFX10-NEXT:    v_cndmask_b32_e32 v21, v2, v8, vcc_lo
+; GFX10-NEXT:    v_xor_b32_e32 v25, -1, v20
+; GFX10-NEXT:    v_or_b32_e32 v2, v27, v10
+; GFX10-NEXT:    v_or_b32_e32 v3, v28, v11
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s6, 0, s6
+; GFX10-NEXT:    s_and_b32 s8, 1, s8
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], v26, v[35:36]
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, v16, s5
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v19, v9, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v4, v2, s6
+; GFX10-NEXT:    v_and_b32_e32 v30, s7, v25
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v5, v3, s6
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, v17, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v19, 0, v0, s4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 64, v30
+; GFX10-NEXT:    v_or_b32_e32 v0, v23, v21
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v11, v6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v23, s7, v20
+; GFX10-NEXT:    v_lshrrev_b64 v[5:6], v2, v[8:9]
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, 64, v30
+; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, v1, s4
+; GFX10-NEXT:    v_lshlrev_b64 v[10:11], v30, v[3:4]
+; GFX10-NEXT:    v_sub_nc_u32_e32 v20, 64, v23
+; GFX10-NEXT:    v_or_b32_e32 v1, v39, v16
+; GFX10-NEXT:    v_or_b32_e32 v2, v18, v19
+; GFX10-NEXT:    v_lshlrev_b64 v[16:17], v30, v[8:9]
+; GFX10-NEXT:    v_lshrrev_b64 v[18:19], v23, v[12:13]
+; GFX10-NEXT:    v_or_b32_e32 v10, v5, v10
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v5, 64, v23
+; GFX10-NEXT:    v_lshlrev_b64 v[20:21], v20, v[14:15]
+; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 64, v30
+; GFX10-NEXT:    v_lshlrev_b64 v[7:8], v7, v[8:9]
+; GFX10-NEXT:    v_or_b32_e32 v9, v6, v11
+; GFX10-NEXT:    v_lshrrev_b64 v[34:35], v5, v[14:15]
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s4, 64, v23
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0, v16, vcc_lo
+; GFX10-NEXT:    v_or_b32_e32 v16, v18, v20
+; GFX10-NEXT:    v_or_b32_e32 v18, v19, v21
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v7, v10, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v8, v9, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[7:8], v23, v[14:15]
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 0, v30
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v34, v16, s4
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 0, v23
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v35, v18, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v14, 0, v17, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v10, v3, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v4, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v5, v12, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v6, v13, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, v7, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, 0, v8, s4
+; GFX10-NEXT:    v_or_b32_e32 v3, v31, v26
+; GFX10-NEXT:    v_or_b32_e32 v4, v11, v4
+; GFX10-NEXT:    v_or_b32_e32 v5, v14, v5
+; GFX10-NEXT:    v_or_b32_e32 v6, v15, v6
+; GFX10-NEXT:    v_or_b32_e32 v7, v9, v7
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
+  ret <2 x i128> %result
+}
+
+declare i7 @llvm.fshr.i7(i7, i7, i7) #0
+declare i8 @llvm.fshr.i8(i8, i8, i8) #0
+declare <2 x i8> @llvm.fshr.v2i8(<2 x i8>, <2 x i8>, <2 x i8>) #0
+declare <4 x i8> @llvm.fshr.v4i8(<4 x i8>, <4 x i8>, <4 x i8>) #0
+
+declare i16 @llvm.fshr.i16(i16, i16, i16) #0
+declare <2 x i16> @llvm.fshr.v2i16(<2 x i16>, <2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.fshr.v3i16(<3 x i16>, <3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.fshr.v4i16(<4 x i16>, <4 x i16>, <4 x i16>) #0
+declare <5 x i16> @llvm.fshr.v5i16(<5 x i16>, <5 x i16>, <5 x i16>) #0
+declare <6 x i16> @llvm.fshr.v6i16(<6 x i16>, <6 x i16>, <6 x i16>) #0
+declare <8 x i16> @llvm.fshr.v8i16(<8 x i16>, <8 x i16>, <8 x i16>) #0
+
+declare i24 @llvm.fshr.i24(i24, i24, i24) #0
+declare <2 x i24> @llvm.fshr.v2i24(<2 x i24>, <2 x i24>, <2 x i24>) #0
+
+declare i32 @llvm.fshr.i32(i32, i32, i32) #0
+declare <2 x i32> @llvm.fshr.v2i32(<2 x i32>, <2 x i32>, <2 x i32>) #0
+declare <3 x i32> @llvm.fshr.v3i32(<3 x i32>, <3 x i32>, <3 x i32>) #0
+declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) #0
+declare <5 x i32> @llvm.fshr.v5i32(<5 x i32>, <5 x i32>, <5 x i32>) #0
+declare <16 x i32> @llvm.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>) #0
+
+declare i48 @llvm.fshr.i48(i48, i48, i48) #0
+
+declare i64 @llvm.fshr.i64(i64, i64, i64) #0
+declare <2 x i64> @llvm.fshr.v2i64(<2 x i64>, <2 x i64>, <2 x i64>) #0
+
+declare i128 @llvm.fshr.i128(i128, i128, i128) #0
+declare <2 x i128> @llvm.fshr.v2i128(<2 x i128>, <2 x i128>, <2 x i128>) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
index 79a346240c17c6d65ea01902e0e194b8b0e676ac..c0807b83f8415c9b9fc19a8c940e35ab1d497f57 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
@@ -3912,8 +3912,13 @@ define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0
   ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
   ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C5]](s32)
   ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
-  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
-  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C6]](s32)
+  ; CHECK:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   G_MEMCPY [[PTR_ADD2]](p5), [[FRAME_INDEX]](p5), [[C7]](s32), 0 :: (dereferenceable store 8 into stack, align 4, addrspace 5), (dereferenceable load 8 from %ir.val, align 4, addrspace 5)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
   ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
   ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
   ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD1]](p4)
@@ -3934,6 +3939,113 @@ define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0
   ret void
 }
 
+declare void @void_func_byval_a3i32_byval_i8_align32([3 x i32] addrspace(5)* byval([3 x i32]) %arg0, i8 addrspace(5)* byval(i8) align 32 %arg1, i32 %arg2) #0
+
+define void @call_byval_3ai32_byval_i8_align32([3 x i32] addrspace(5)* %incoming0, i8 addrspace(5)* align 32 %incoming1) #0 {
+  ; CHECK-LABEL: name: call_byval_3ai32_byval_i8_align32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr0, $vgpr1, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(p5) = COPY $vgpr0
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p5) = COPY $vgpr1
+  ; CHECK:   [[COPY10:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 999
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @void_func_byval_a3i32_byval_i8_align32
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(p5) = COPY $sgpr32
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY19]], [[C1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   G_MEMCPY [[PTR_ADD]](p5), [[COPY8]](p5), [[C2]](s32), 0 :: (dereferenceable store 12 into stack, align 4, addrspace 5), (dereferenceable load 12 from %ir.incoming0, align 4, addrspace 5)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   G_MEMCPY [[PTR_ADD1]](p5), [[COPY9]](p5), [[C4]](s32), 0 :: (dereferenceable store 1 into stack + 32, align 32, addrspace 5), (dereferenceable load 1 from %ir.incoming1, align 32, addrspace 5)
+  ; CHECK:   $vgpr0 = COPY [[C]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY12]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[COPY13]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY14]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY16]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY17]](s32)
+  ; CHECK:   $vgpr31 = COPY [[COPY18]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @void_func_byval_a3i32_byval_i8_align32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 36, implicit-def $scc
+  ; CHECK:   [[COPY21:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY10]]
+  ; CHECK:   S_SETPC_B64_return [[COPY21]]
+  call void @void_func_byval_a3i32_byval_i8_align32([3 x i32] addrspace(5)* byval([3 x i32]) %incoming0, i8 addrspace(5)* align 32 %incoming1, i32 999)
+  ret void
+}
+
+declare void @void_func_byval_a4i64_align4([4 x i64] addrspace(5)* byval([4 x i64]) align 4 %arg0) #0
+
+; Make sure we are aware of the higher alignment of the incoming value
+; than implied by the outgoing byval alignment in the memory operand.
+define void @call_byval_a4i64_align4_higher_source_align([4 x i64] addrspace(5)* align 256 %incoming_high_align) #0 {
+  ; CHECK-LABEL: name: call_byval_a4i64_align4_higher_source_align
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr0, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(p5) = COPY $vgpr0
+  ; CHECK:   [[COPY9:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @void_func_byval_a4i64_align4
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(p5) = COPY $sgpr32
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY18]], [[C]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+  ; CHECK:   G_MEMCPY [[PTR_ADD]](p5), [[COPY8]](p5), [[C1]](s32), 0 :: (dereferenceable store 32 into stack, align 4, addrspace 5), (dereferenceable load 32 from %ir.incoming_high_align, align 256, addrspace 5)
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY19]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[COPY12]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[COPY17]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @void_func_byval_a4i64_align4, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 32, implicit-def $scc
+  ; CHECK:   [[COPY20:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY9]]
+  ; CHECK:   S_SETPC_B64_return [[COPY20]]
+  call void @void_func_byval_a4i64_align4([4 x i64] addrspace(5)* byval([4 x i64]) align 4 %incoming_high_align)
+  ret void
+}
+
 define amdgpu_kernel void @test_call_external_void_func_v2i8() #0 {
   ; CHECK-LABEL: name: test_call_external_void_func_v2i8
   ; CHECK: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-non-entry-func.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-non-entry-func.ll
index 4473c86a6e60c70d2a20c4d46307a1a2e35700e1..e3638569bbf12dad14275191c2a3a5f561fe287c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-non-entry-func.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-global-non-entry-func.ll
@@ -24,7 +24,6 @@ define void @func_use_lds_global() {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[6:7]
 ; GFX9-NEXT:    s_trap 2
 ; GFX9-NEXT:    ds_write_b32 v0, v0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
@@ -47,7 +46,6 @@ define void @func_use_lds_global_constexpr_cast() {
 ; GFX9-LABEL: func_use_lds_global_constexpr_cast:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[6:7]
 ; GFX9-NEXT:    s_trap 2
 ; GFX9-NEXT:    global_store_dword v[0:1], v0, off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
new file mode 100644
index 0000000000000000000000000000000000000000..93af75e3d6ed22126f9e0451da50840d1a893d4d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
@@ -0,0 +1,1254 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9  %s
+
+---
+name: test_fshl_s32_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshl_s32_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[COPY]], [[COPY1]], [[C]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[C1]]
+    ; SI: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[LSHR]], [[FSHR]], [[XOR]](s32)
+    ; SI: $vgpr0 = COPY [[FSHR1]](s32)
+    ; VI-LABEL: name: test_fshl_s32_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; VI: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[COPY]], [[COPY1]], [[C]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[C1]]
+    ; VI: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[LSHR]], [[FSHR]], [[XOR]](s32)
+    ; VI: $vgpr0 = COPY [[FSHR1]](s32)
+    ; GFX9-LABEL: name: test_fshl_s32_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[COPY]], [[COPY1]], [[C]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[C1]]
+    ; GFX9: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[LSHR]], [[FSHR]], [[XOR]](s32)
+    ; GFX9: $vgpr0 = COPY [[FSHR1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = G_FSHL %0, %1, %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fshl_v2s32_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fshl_v2s32_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[UV]], [[UV2]], [[C]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[UV4]], [[C1]]
+    ; SI: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[LSHR]], [[FSHR]], [[XOR]](s32)
+    ; SI: [[FSHR2:%[0-9]+]]:_(s32) = G_FSHR [[UV1]], [[UV3]], [[C]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
+    ; SI: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[UV5]], [[C1]]
+    ; SI: [[FSHR3:%[0-9]+]]:_(s32) = G_FSHR [[LSHR1]], [[FSHR2]], [[XOR1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSHR1]](s32), [[FSHR3]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fshl_v2s32_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; VI: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[UV]], [[UV2]], [[C]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[UV4]], [[C1]]
+    ; VI: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[LSHR]], [[FSHR]], [[XOR]](s32)
+    ; VI: [[FSHR2:%[0-9]+]]:_(s32) = G_FSHR [[UV1]], [[UV3]], [[C]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
+    ; VI: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[UV5]], [[C1]]
+    ; VI: [[FSHR3:%[0-9]+]]:_(s32) = G_FSHR [[LSHR1]], [[FSHR2]], [[XOR1]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSHR1]](s32), [[FSHR3]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fshl_v2s32_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[UV]], [[UV2]], [[C]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[UV4]], [[C1]]
+    ; GFX9: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[LSHR]], [[FSHR]], [[XOR]](s32)
+    ; GFX9: [[FSHR2:%[0-9]+]]:_(s32) = G_FSHR [[UV1]], [[UV3]], [[C]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[UV5]], [[C1]]
+    ; GFX9: [[FSHR3:%[0-9]+]]:_(s32) = G_FSHR [[LSHR1]], [[FSHR2]], [[XOR1]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSHR1]](s32), [[FSHR3]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %3:_(<2 x s32>) = G_FSHL %0, %1, %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_fshl_s16_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshl_s16_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C1]]
+    ; SI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C]]
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[AND]](s16)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[ZEXT]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C3]]
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC1]], [[TRUNC2]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fshl_s16_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C1]]
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C]]
+    ; VI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C2]](s16)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[LSHR]], [[AND1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR1]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fshl_s16_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C1]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C2]](s16)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[LSHR]], [[AND1]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s16) = G_TRUNC %0
+    %4:_(s16) = G_TRUNC %1
+    %5:_(s16) = G_TRUNC %2
+    %6:_(s16) = G_FSHL %3, %4, %5
+    %7:_(s32) = G_ANYEXT %6
+    $vgpr0 = COPY %7
+...
+
+---
+name: test_fshl_v2s16_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshl_v2s16_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
+    ; SI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C2]]
+    ; SI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C1]]
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[AND]](s16)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[ZEXT]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C4]]
+    ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C4]]
+    ; SI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC2]], [[TRUNC3]]
+    ; SI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
+    ; SI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C2]]
+    ; SI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; SI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C4]]
+    ; SI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[COPY7]](s32)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND5]](s16)
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; SI: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C4]]
+    ; SI: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[ZEXT3]](s32)
+    ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC4]], [[TRUNC5]]
+    ; SI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; SI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL2]]
+    ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
+    ; SI: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
+    ; VI-LABEL: name: test_fshl_v2s16_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
+    ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C1]]
+    ; VI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC4]], [[C2]]
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C1]]
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C3]](s16)
+    ; VI: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[LSHR3]], [[AND1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR4]]
+    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C1]]
+    ; VI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC5]], [[C2]]
+    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
+    ; VI: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C3]](s16)
+    ; VI: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[LSHR5]], [[AND3]](s16)
+    ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR6]]
+    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; VI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; VI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
+    ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
+    ; VI: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fshl_v2s16_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[COPY2]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[C1]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[COPY2]], [[BUILD_VECTOR_TRUNC1]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[C2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY]], [[AND]](<2 x s16>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY1]], [[BUILD_VECTOR_TRUNC2]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR]], [[AND1]](<2 x s16>)
+    ; GFX9: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9: $vgpr0 = COPY [[OR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = COPY $vgpr2
+    %3:_(<2 x s16>) = G_FSHL %0, %1, %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fshl_s64_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fshl_s64_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; SI: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[C1]]
+    ; SI: [[AND1:%[0-9]+]]:_(s64) = G_AND [[XOR]], [[C]]
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[TRUNC]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[TRUNC1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; SI: $vgpr0_vgpr1 = COPY [[OR]](s64)
+    ; VI-LABEL: name: test_fshl_s64_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; VI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; VI: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[C1]]
+    ; VI: [[AND1:%[0-9]+]]:_(s64) = G_AND [[XOR]], [[C]]
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[TRUNC]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[TRUNC1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; VI: $vgpr0_vgpr1 = COPY [[OR]](s64)
+    ; GFX9-LABEL: name: test_fshl_s64_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; GFX9: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[C1]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(s64) = G_AND [[XOR]], [[C]]
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[TRUNC]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[LSHR]], [[TRUNC1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[OR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = COPY $vgpr4_vgpr5
+    %3:_(s64) = G_FSHL %0, %1, %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_fshl_s8_s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshl_s8_s8
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[COPY4]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY5]], [[COPY6]]
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[COPY8]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C3]]
+    ; SI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[AND2]](s32)
+    ; SI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; SI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C3]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[AND3]](s32)
+    ; SI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
+    ; SI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C3]]
+    ; SI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY14]], [[C3]]
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[AND5]](s32)
+    ; SI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY15]], [[COPY16]]
+    ; SI: [[COPY17:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; SI: $vgpr0 = COPY [[COPY17]](s32)
+    ; VI-LABEL: name: test_fshl_s8_s8
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[COPY4]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; VI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; VI: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY5]], [[COPY6]]
+    ; VI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; VI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[COPY8]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; VI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C3]]
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND2]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
+    ; VI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; VI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C4]]
+    ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[AND3]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[AND4]], [[TRUNC3]](s16)
+    ; VI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
+    ; VI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; VI: [[COPY12:%[0-9]+]]:_(s16) = COPY [[LSHR]](s16)
+    ; VI: [[AND6:%[0-9]+]]:_(s16) = G_AND [[COPY12]], [[C4]]
+    ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[AND5]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[AND6]], [[TRUNC4]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; VI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; VI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; VI: $vgpr0 = COPY [[COPY13]](s32)
+    ; GFX9-LABEL: name: test_fshl_s8_s8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[COPY4]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY5]], [[COPY6]]
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[COPY8]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C3]]
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C4]]
+    ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[AND3]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[AND4]], [[TRUNC3]](s16)
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
+    ; GFX9: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; GFX9: [[COPY12:%[0-9]+]]:_(s16) = COPY [[LSHR]](s16)
+    ; GFX9: [[AND6:%[0-9]+]]:_(s16) = G_AND [[COPY12]], [[C4]]
+    ; GFX9: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[AND5]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[AND6]], [[TRUNC4]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR1]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; GFX9: [[COPY13:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; GFX9: $vgpr0 = COPY [[COPY13]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s8) = G_TRUNC %0
+    %4:_(s8) = G_TRUNC %1
+    %5:_(s8) = G_TRUNC %2
+    %6:_(s8) = G_FSHL %3, %4, %5
+    %7:_(s32) = G_ANYEXT %6
+    $vgpr0 = COPY %7
+...
+
+---
+name: test_fshl_s24_s24
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshl_s24_s24
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C2]]
+    ; SI: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
+    ; SI: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
+    ; SI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C3]]
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[AND1]]
+    ; SI: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; SI: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; SI: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; SI: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; SI: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; SI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; SI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; SI: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; SI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; SI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[COPY5]], [[COPY6]]
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C2]]
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[AND2]](s32)
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C2]]
+    ; SI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C2]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[AND3]](s32)
+    ; SI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; SI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C2]]
+    ; SI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C2]]
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[AND5]](s32)
+    ; SI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY13]], [[COPY14]]
+    ; SI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; SI: $vgpr0 = COPY [[COPY15]](s32)
+    ; VI-LABEL: name: test_fshl_s24_s24
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C2]]
+    ; VI: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
+    ; VI: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
+    ; VI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C3]]
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[AND1]]
+    ; VI: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; VI: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; VI: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; VI: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; VI: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; VI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; VI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; VI: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; VI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; VI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[COPY5]], [[COPY6]]
+    ; VI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; VI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C2]]
+    ; VI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[AND2]](s32)
+    ; VI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; VI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C2]]
+    ; VI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C2]]
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[AND3]](s32)
+    ; VI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; VI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C2]]
+    ; VI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; VI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C2]]
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[AND5]](s32)
+    ; VI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; VI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY13]], [[COPY14]]
+    ; VI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; VI: $vgpr0 = COPY [[COPY15]](s32)
+    ; GFX9-LABEL: name: test_fshl_s24_s24
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C2]]
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C3]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[COPY5]], [[COPY6]]
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C2]]
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[AND2]](s32)
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C2]]
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C2]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[AND3]](s32)
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX9: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C2]]
+    ; GFX9: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C2]]
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[AND5]](s32)
+    ; GFX9: [[COPY13:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; GFX9: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY13]], [[COPY14]]
+    ; GFX9: [[COPY15:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; GFX9: $vgpr0 = COPY [[COPY15]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s24) = G_TRUNC %0
+    %4:_(s24) = G_TRUNC %1
+    %5:_(s24) = G_TRUNC %2
+    %6:_(s24) = G_FSHL %3, %4, %5
+    %7:_(s32) = G_ANYEXT %6
+    $vgpr0 = COPY %7
+...
+
+---
+name: test_fshl_v3s16_v3s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; SI-LABEL: name: test_fshl_v3s16_v3s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; SI: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
+    ; SI: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
+    ; SI: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
+    ; SI: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
+    ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; SI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
+    ; SI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
+    ; SI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+    ; SI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
+    ; SI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C2]]
+    ; SI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C1]]
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[AND]](s16)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[ZEXT]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C4]]
+    ; SI: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C4]]
+    ; SI: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC3]], [[TRUNC4]]
+    ; SI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
+    ; SI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C2]]
+    ; SI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY9]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; SI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; SI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C4]]
+    ; SI: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[COPY10]](s32)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND5]](s16)
+    ; SI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
+    ; SI: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C4]]
+    ; SI: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[ZEXT3]](s32)
+    ; SI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC5]], [[TRUNC6]]
+    ; SI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C1]]
+    ; SI: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C2]]
+    ; SI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
+    ; SI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[AND8]](s16)
+    ; SI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY13]], [[ZEXT4]](s32)
+    ; SI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[SHL2]](s32)
+    ; SI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; SI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; SI: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY15]], [[C4]]
+    ; SI: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[COPY14]](s32)
+    ; SI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND9]](s16)
+    ; SI: [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR10]](s32)
+    ; SI: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY16]], [[C4]]
+    ; SI: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[AND11]], [[ZEXT5]](s32)
+    ; SI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR11]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s16) = G_OR [[TRUNC7]], [[TRUNC8]]
+    ; SI: [[DEF1:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF2]](<4 x s16>)
+    ; SI: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
+    ; SI: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
+    ; SI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; SI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; SI: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C]](s32)
+    ; SI: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL3]]
+    ; SI: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](s32)
+    ; SI: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; SI: [[COPY17:%[0-9]+]]:_(s32) = COPY [[BITCAST6]](s32)
+    ; SI: [[AND12:%[0-9]+]]:_(s32) = G_AND [[COPY17]], [[C4]]
+    ; SI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND12]], [[C]](s32)
+    ; SI: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT8]], [[SHL4]]
+    ; SI: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](s32)
+    ; SI: [[COPY18:%[0-9]+]]:_(s32) = COPY [[LSHR12]](s32)
+    ; SI: [[AND13:%[0-9]+]]:_(s32) = G_AND [[COPY18]], [[C4]]
+    ; SI: [[COPY19:%[0-9]+]]:_(s32) = COPY [[BITCAST7]](s32)
+    ; SI: [[AND14:%[0-9]+]]:_(s32) = G_AND [[COPY19]], [[C4]]
+    ; SI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND14]], [[C]](s32)
+    ; SI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[AND13]], [[SHL5]]
+    ; SI: [[BITCAST10:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
+    ; SI: $vgpr0 = COPY [[BITCAST8]](<2 x s16>)
+    ; SI: $vgpr1 = COPY [[BITCAST9]](<2 x s16>)
+    ; SI: $vgpr2 = COPY [[BITCAST10]](<2 x s16>)
+    ; VI-LABEL: name: test_fshl_v3s16_v3s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; VI: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
+    ; VI: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
+    ; VI: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
+    ; VI: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
+    ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
+    ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; VI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
+    ; VI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
+    ; VI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+    ; VI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
+    ; VI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
+    ; VI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+    ; VI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C1]]
+    ; VI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC6]], [[C2]]
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C1]]
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
+    ; VI: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C3]](s16)
+    ; VI: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[LSHR6]], [[AND1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR7]]
+    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C1]]
+    ; VI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC7]], [[C2]]
+    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
+    ; VI: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC4]], [[C3]](s16)
+    ; VI: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[LSHR8]], [[AND3]](s16)
+    ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR9]]
+    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C1]]
+    ; VI: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC8]], [[C2]]
+    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
+    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[AND4]](s16)
+    ; VI: [[LSHR10:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC5]], [[C3]](s16)
+    ; VI: [[LSHR11:%[0-9]+]]:_(s16) = G_LSHR [[LSHR10]], [[AND5]](s16)
+    ; VI: [[OR2:%[0-9]+]]:_(s16) = G_OR [[SHL2]], [[LSHR11]]
+    ; VI: [[DEF1:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF2]](<4 x s16>)
+    ; VI: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
+    ; VI: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
+    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; VI: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; VI: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
+    ; VI: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](s32)
+    ; VI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; VI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST6]](s32)
+    ; VI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C4]]
+    ; VI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C]](s32)
+    ; VI: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL4]]
+    ; VI: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](s32)
+    ; VI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR12]](s32)
+    ; VI: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C4]]
+    ; VI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[BITCAST7]](s32)
+    ; VI: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C4]]
+    ; VI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND8]], [[C]](s32)
+    ; VI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[AND7]], [[SHL5]]
+    ; VI: [[BITCAST10:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
+    ; VI: $vgpr0 = COPY [[BITCAST8]](<2 x s16>)
+    ; VI: $vgpr1 = COPY [[BITCAST9]](<2 x s16>)
+    ; VI: $vgpr2 = COPY [[BITCAST10]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fshl_v3s16_v3s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; GFX9: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
+    ; GFX9: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
+    ; GFX9: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
+    ; GFX9: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[DEF1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY8]](s32), [[COPY9]](s32)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
+    ; GFX9: [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX9: [[COPY13:%[0-9]+]]:_(s32) = COPY [[DEF1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
+    ; GFX9: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+    ; GFX9: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
+    ; GFX9: [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST4]](s32)
+    ; GFX9: [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
+    ; GFX9: [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST5]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF1]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; GFX9: [[COPY17:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[C1]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[BUILD_VECTOR_TRUNC4]], [[BUILD_VECTOR_TRUNC6]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[COPY18:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY18]](s32), [[C2]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BUILD_VECTOR_TRUNC4]], [[BUILD_VECTOR_TRUNC7]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR]], [[BUILD_VECTOR_TRUNC6]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[COPY19:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC8:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY19]](s32), [[C3]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR_TRUNC]], [[AND]](<2 x s16>)
+    ; GFX9: [[LSHR6:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[BUILD_VECTOR_TRUNC2]], [[BUILD_VECTOR_TRUNC8]](<2 x s16>)
+    ; GFX9: [[LSHR7:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR6]], [[AND1]](<2 x s16>)
+    ; GFX9: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL]], [[LSHR7]]
+    ; GFX9: [[COPY20:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[COPY21:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC9:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY20]](s32), [[COPY21]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(<2 x s16>) = G_AND [[BUILD_VECTOR_TRUNC5]], [[BUILD_VECTOR_TRUNC9]]
+    ; GFX9: [[COPY22:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[COPY23:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC10:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY22]](s32), [[COPY23]](s32)
+    ; GFX9: [[XOR1:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BUILD_VECTOR_TRUNC5]], [[BUILD_VECTOR_TRUNC10]]
+    ; GFX9: [[AND3:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR1]], [[BUILD_VECTOR_TRUNC9]]
+    ; GFX9: [[COPY24:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; GFX9: [[COPY25:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC11:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY24]](s32), [[COPY25]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR_TRUNC1]], [[AND2]](<2 x s16>)
+    ; GFX9: [[LSHR8:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[BUILD_VECTOR_TRUNC3]], [[BUILD_VECTOR_TRUNC11]](<2 x s16>)
+    ; GFX9: [[LSHR9:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR8]], [[AND3]](<2 x s16>)
+    ; GFX9: [[OR1:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR9]]
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[OR]](<2 x s16>)
+    ; GFX9: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
+    ; GFX9: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[OR1]](<2 x s16>)
+    ; GFX9: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF2]](<4 x s16>)
+    ; GFX9: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST8]], [[C]](s32)
+    ; GFX9: [[BITCAST9:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST9]], [[C]](s32)
+    ; GFX9: [[COPY26:%[0-9]+]]:_(s32) = COPY [[BITCAST6]](s32)
+    ; GFX9: [[COPY27:%[0-9]+]]:_(s32) = COPY [[LSHR10]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC12:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY26]](s32), [[COPY27]](s32)
+    ; GFX9: [[COPY28:%[0-9]+]]:_(s32) = COPY [[BITCAST7]](s32)
+    ; GFX9: [[COPY29:%[0-9]+]]:_(s32) = COPY [[BITCAST8]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC13:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY28]](s32), [[COPY29]](s32)
+    ; GFX9: [[COPY30:%[0-9]+]]:_(s32) = COPY [[LSHR12]](s32)
+    ; GFX9: [[COPY31:%[0-9]+]]:_(s32) = COPY [[BITCAST9]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC14:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY30]](s32), [[COPY31]](s32)
+    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC12]](<2 x s16>)
+    ; GFX9: $vgpr1 = COPY [[BUILD_VECTOR_TRUNC13]](<2 x s16>)
+    ; GFX9: $vgpr2 = COPY [[BUILD_VECTOR_TRUNC14]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = COPY $vgpr2
+    %3:_(<2 x s16>) = COPY $vgpr3
+    %4:_(<2 x s16>) = COPY $vgpr4
+    %5:_(<2 x s16>) = COPY $vgpr5
+    %6:_(<2 x s16>) = G_IMPLICIT_DEF
+    %7:_(<6 x s16>) = G_CONCAT_VECTORS %0(<2 x s16>), %1(<2 x s16>), %6(<2 x s16>)
+    %8:_(<3 x s16>), %9:_(<3 x s16>) = G_UNMERGE_VALUES %7(<6 x s16>)
+    %10:_(<6 x s16>) = G_CONCAT_VECTORS %2(<2 x s16>), %3(<2 x s16>), %6(<2 x s16>)
+    %11:_(<3 x s16>), %12:_(<3 x s16>) = G_UNMERGE_VALUES %10(<6 x s16>)
+    %13:_(<6 x s16>) = G_CONCAT_VECTORS %4(<2 x s16>), %5(<2 x s16>), %6(<2 x s16>)
+    %14:_(<3 x s16>), %15:_(<3 x s16>) = G_UNMERGE_VALUES %13(<6 x s16>)
+    %16:_(<3 x s16>) = G_FSHL %8, %11, %14(<3 x s16>)
+    %17:_(<3 x s16>) = G_IMPLICIT_DEF
+    %18:_(<6 x s16>) = G_CONCAT_VECTORS %16(<3 x s16>), %17(<3 x s16>)
+    %19:_(<2 x s16>), %20:_(<2 x s16>), %21:_(<2 x s16>) = G_UNMERGE_VALUES %18(<6 x s16>)
+    $vgpr0 = COPY %19(<2 x s16>)
+    $vgpr1 = COPY %20(<2 x s16>)
+    $vgpr2 = COPY %21(<2 x s16>)
+...
+
+---
+name: test_fshl_v4s16_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fshl_v4s16_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
+    ; SI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
+    ; SI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
+    ; SI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+    ; SI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
+    ; SI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C2]]
+    ; SI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C1]]
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[AND]](s16)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[ZEXT]](s32)
+    ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C4]]
+    ; SI: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C4]]
+    ; SI: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC4]], [[TRUNC5]]
+    ; SI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
+    ; SI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C2]]
+    ; SI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; SI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C4]]
+    ; SI: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[COPY7]](s32)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND5]](s16)
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
+    ; SI: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C4]]
+    ; SI: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[ZEXT3]](s32)
+    ; SI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC6]], [[TRUNC7]]
+    ; SI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C1]]
+    ; SI: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C2]]
+    ; SI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
+    ; SI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[AND8]](s16)
+    ; SI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[ZEXT4]](s32)
+    ; SI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[SHL2]](s32)
+    ; SI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; SI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; SI: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C4]]
+    ; SI: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[COPY11]](s32)
+    ; SI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND9]](s16)
+    ; SI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR10]](s32)
+    ; SI: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C4]]
+    ; SI: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[AND11]], [[ZEXT5]](s32)
+    ; SI: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR11]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s16) = G_OR [[TRUNC8]], [[TRUNC9]]
+    ; SI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C1]]
+    ; SI: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC3]], [[C2]]
+    ; SI: [[AND13:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C1]]
+    ; SI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[AND12]](s16)
+    ; SI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; SI: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY14]], [[ZEXT6]](s32)
+    ; SI: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
+    ; SI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; SI: [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; SI: [[AND14:%[0-9]+]]:_(s32) = G_AND [[COPY16]], [[C4]]
+    ; SI: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[AND14]], [[COPY15]](s32)
+    ; SI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[AND13]](s16)
+    ; SI: [[COPY17:%[0-9]+]]:_(s32) = COPY [[LSHR12]](s32)
+    ; SI: [[AND15:%[0-9]+]]:_(s32) = G_AND [[COPY17]], [[C4]]
+    ; SI: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[AND15]], [[ZEXT7]](s32)
+    ; SI: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR13]](s32)
+    ; SI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC10]], [[TRUNC11]]
+    ; SI: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; SI: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; SI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C]](s32)
+    ; SI: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT8]], [[SHL4]]
+    ; SI: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](s32)
+    ; SI: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; SI: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; SI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C]](s32)
+    ; SI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT10]], [[SHL5]]
+    ; SI: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
+    ; SI: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; VI-LABEL: name: test_fshl_v4s16_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; VI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
+    ; VI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; VI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
+    ; VI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+    ; VI: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
+    ; VI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; VI: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
+    ; VI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
+    ; VI: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C1]]
+    ; VI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC8]], [[C2]]
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C1]]
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
+    ; VI: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC4]], [[C3]](s16)
+    ; VI: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[LSHR6]], [[AND1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR7]]
+    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C1]]
+    ; VI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC9]], [[C2]]
+    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C1]]
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
+    ; VI: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC5]], [[C3]](s16)
+    ; VI: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[LSHR8]], [[AND3]](s16)
+    ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR9]]
+    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C1]]
+    ; VI: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC10]], [[C2]]
+    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR2]], [[C1]]
+    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[AND4]](s16)
+    ; VI: [[LSHR10:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC6]], [[C3]](s16)
+    ; VI: [[LSHR11:%[0-9]+]]:_(s16) = G_LSHR [[LSHR10]], [[AND5]](s16)
+    ; VI: [[OR2:%[0-9]+]]:_(s16) = G_OR [[SHL2]], [[LSHR11]]
+    ; VI: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C1]]
+    ; VI: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC11]], [[C2]]
+    ; VI: [[AND7:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C1]]
+    ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[AND6]](s16)
+    ; VI: [[LSHR12:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC7]], [[C3]](s16)
+    ; VI: [[LSHR13:%[0-9]+]]:_(s16) = G_LSHR [[LSHR12]], [[AND7]](s16)
+    ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[SHL3]], [[LSHR13]]
+    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; VI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; VI: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
+    ; VI: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](s32)
+    ; VI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; VI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; VI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+    ; VI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
+    ; VI: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
+    ; VI: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fshl_v4s16_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[UV4]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[C1]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV4]], [[BUILD_VECTOR_TRUNC1]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[C2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV]], [[AND]](<2 x s16>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV2]], [[BUILD_VECTOR_TRUNC2]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR]], [[AND1]](<2 x s16>)
+    ; GFX9: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL]], [[LSHR1]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(<2 x s16>) = G_AND [[UV5]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY8]](s32), [[COPY9]](s32)
+    ; GFX9: [[XOR1:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV5]], [[BUILD_VECTOR_TRUNC4]]
+    ; GFX9: [[AND3:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR1]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV1]], [[AND2]](<2 x s16>)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV3]], [[BUILD_VECTOR_TRUNC5]](<2 x s16>)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[LSHR2]], [[AND3]](<2 x s16>)
+    ; GFX9: [[OR1:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[OR]](<2 x s16>), [[OR1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    %3:_(<4 x s16>) = G_FSHL %0, %1, %2
+    $vgpr0_vgpr1 = COPY %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
index b26b28f6bf3416920e4929926c499ac4ec9983a6..432866061c994d29683ebcd34e09182e9312ce79 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
@@ -90,11 +90,27 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
-    ; SI: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s16)
-    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s16)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C1]]
+    ; SI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[ZEXT]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND]](s16)
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C3]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC1]], [[TRUNC2]]
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; VI-LABEL: name: test_fshr_s16_s16
     ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -103,8 +119,17 @@ body: |
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
-    ; VI: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s16)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s16)
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C1]]
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C]]
+    ; VI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C2]](s16)
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[SHL]], [[AND1]](s16)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[AND]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: test_fshr_s16_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
@@ -113,8 +138,17 @@ body: |
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
-    ; GFX9: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s16)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s16)
+    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C1]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C2]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[SHL]], [[AND1]](s16)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[AND]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -137,35 +171,194 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; SI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
-    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; SI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<2 x s16>)
-    ; SI: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[UV]], [[UV2]], [[UV4]](s16)
-    ; SI: [[FSHR1:%[0-9]+]]:_(s16) = G_FSHR [[UV1]], [[UV3]], [[UV5]](s16)
-    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSHR]](s16), [[FSHR1]](s16)
-    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; SI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; SI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C2]]
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[AND]](s16)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C5]]
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C4]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C5]]
+    ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC]], [[TRUNC1]]
+    ; SI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; SI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; SI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; SI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C5]]
+    ; SI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[COPY7]](s32)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND5]](s16)
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; SI: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C5]]
+    ; SI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[ZEXT3]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC2]], [[TRUNC3]]
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; SI: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
+    ; SI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY11]], [[COPY10]](s32)
+    ; SI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; SI: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY13]], [[COPY12]](s32)
+    ; SI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; SI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; SI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY15]], [[C1]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[COPY14]], [[SHL4]]
+    ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
+    ; SI: [[XOR2:%[0-9]+]]:_(<2 x s16>) = G_XOR [[COPY2]], [[BITCAST3]]
+    ; SI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[XOR2]](<2 x s16>)
+    ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
+    ; SI: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C1]](s32)
+    ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+    ; SI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C2]]
+    ; SI: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC4]], [[C3]]
+    ; SI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C2]]
+    ; SI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[AND8]](s16)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; SI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT]], [[ZEXT4]](s32)
+    ; SI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[SHL5]](s32)
+    ; SI: [[COPY16:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY17:%[0-9]+]]:_(s32) = COPY [[SHL2]](s32)
+    ; SI: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY17]], [[C5]]
+    ; SI: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[COPY16]](s32)
+    ; SI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND9]](s16)
+    ; SI: [[COPY18:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
+    ; SI: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY18]], [[C5]]
+    ; SI: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[AND11]], [[ZEXT5]](s32)
+    ; SI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
+    ; SI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC6]], [[TRUNC7]]
+    ; SI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C2]]
+    ; SI: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC5]], [[C3]]
+    ; SI: [[AND13:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
+    ; SI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[AND12]](s16)
+    ; SI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[OR1]](s16)
+    ; SI: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT1]], [[ZEXT6]](s32)
+    ; SI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[SHL6]](s32)
+    ; SI: [[COPY19:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY20:%[0-9]+]]:_(s32) = COPY [[SHL3]](s32)
+    ; SI: [[AND14:%[0-9]+]]:_(s32) = G_AND [[COPY20]], [[C5]]
+    ; SI: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[AND14]], [[COPY19]](s32)
+    ; SI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[AND13]](s16)
+    ; SI: [[COPY21:%[0-9]+]]:_(s32) = COPY [[LSHR10]](s32)
+    ; SI: [[AND15:%[0-9]+]]:_(s32) = G_AND [[COPY21]], [[C5]]
+    ; SI: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[AND15]], [[ZEXT7]](s32)
+    ; SI: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR11]](s32)
+    ; SI: [[OR4:%[0-9]+]]:_(s16) = G_OR [[TRUNC8]], [[TRUNC9]]
+    ; SI: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; SI: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
+    ; SI: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C1]](s32)
+    ; SI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT8]], [[SHL7]]
+    ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
+    ; SI: $vgpr0 = COPY [[BITCAST5]](<2 x s16>)
     ; VI-LABEL: name: test_fshr_v2s16_v2s16
     ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; VI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
-    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; VI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<2 x s16>)
-    ; VI: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[UV]], [[UV2]], [[UV4]](s16)
-    ; VI: [[FSHR1:%[0-9]+]]:_(s16) = G_FSHR [[UV1]], [[UV3]], [[UV5]](s16)
-    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSHR]](s16), [[FSHR1]](s16)
-    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C2]]
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C]](s16)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[LSHR2]], [[AND1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR3]]
+    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; VI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
+    ; VI: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C]](s16)
+    ; VI: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[LSHR4]], [[AND3]](s16)
+    ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR5]]
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; VI: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
+    ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
+    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC4]], [[C]](s16)
+    ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC5]], [[C]](s16)
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; VI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C1]](s32)
+    ; VI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[COPY3]], [[SHL4]]
+    ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
+    ; VI: [[XOR2:%[0-9]+]]:_(<2 x s16>) = G_XOR [[COPY2]], [[BITCAST3]]
+    ; VI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[XOR2]](<2 x s16>)
+    ; VI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
+    ; VI: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C1]](s32)
+    ; VI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C2]]
+    ; VI: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC6]], [[C3]]
+    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C2]]
+    ; VI: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[OR]], [[AND4]](s16)
+    ; VI: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[SHL2]], [[C]](s16)
+    ; VI: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[LSHR8]], [[AND5]](s16)
+    ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[SHL5]], [[LSHR9]]
+    ; VI: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C2]]
+    ; VI: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC7]], [[C3]]
+    ; VI: [[AND7:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
+    ; VI: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[OR1]], [[AND6]](s16)
+    ; VI: [[LSHR10:%[0-9]+]]:_(s16) = G_LSHR [[SHL3]], [[C]](s16)
+    ; VI: [[LSHR11:%[0-9]+]]:_(s16) = G_LSHR [[LSHR10]], [[AND7]](s16)
+    ; VI: [[OR4:%[0-9]+]]:_(s16) = G_OR [[SHL6]], [[LSHR11]]
+    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
+    ; VI: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+    ; VI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL7]]
+    ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
+    ; VI: $vgpr0 = COPY [[BITCAST5]](<2 x s16>)
     ; GFX9-LABEL: name: test_fshr_v2s16_v2s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
     ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
-    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX9: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<2 x s16>)
-    ; GFX9: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[UV]], [[UV2]], [[UV4]](s16)
-    ; GFX9: [[FSHR1:%[0-9]+]]:_(s16) = G_FSHR [[UV1]], [[UV3]], [[UV5]](s16)
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSHR]](s16), [[FSHR1]](s16)
-    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[COPY2]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[C1]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[COPY2]], [[BUILD_VECTOR_TRUNC1]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[C2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY]], [[BUILD_VECTOR_TRUNC2]](<2 x s16>)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL]], [[AND1]](<2 x s16>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY1]], [[AND]](<2 x s16>)
+    ; GFX9: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: $vgpr0 = COPY [[OR]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
     %2:_(<2 x s16>) = COPY $vgpr2
@@ -183,20 +376,53 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
     ; SI: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
-    ; SI: [[FSHR:%[0-9]+]]:_(s64) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s64)
-    ; SI: $vgpr0_vgpr1 = COPY [[FSHR]](s64)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; SI: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[C1]]
+    ; SI: [[AND1:%[0-9]+]]:_(s64) = G_AND [[XOR]], [[C]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C2]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[TRUNC]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[TRUNC1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI: $vgpr0_vgpr1 = COPY [[OR]](s64)
     ; VI-LABEL: name: test_fshr_s64_s64
     ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
     ; VI: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
-    ; VI: [[FSHR:%[0-9]+]]:_(s64) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s64)
-    ; VI: $vgpr0_vgpr1 = COPY [[FSHR]](s64)
+    ; VI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; VI: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; VI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[C1]]
+    ; VI: [[AND1:%[0-9]+]]:_(s64) = G_AND [[XOR]], [[C]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C2]](s32)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[TRUNC]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[TRUNC1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: $vgpr0_vgpr1 = COPY [[OR]](s64)
     ; GFX9-LABEL: name: test_fshr_s64_s64
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
     ; GFX9: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
-    ; GFX9: [[FSHR:%[0-9]+]]:_(s64) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s64)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[FSHR]](s64)
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; GFX9: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[C1]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(s64) = G_AND [[XOR]], [[C]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C2]](s32)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SHL]], [[TRUNC]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[AND]](s64)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[TRUNC1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[OR]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = COPY $vgpr4_vgpr5
@@ -214,32 +440,115 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; SI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; SI: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; SI: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
-    ; SI: [[FSHR:%[0-9]+]]:_(s8) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s8)
-    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s8)
-    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[COPY4]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY5]], [[COPY6]]
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[COPY8]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C3]]
+    ; SI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[AND2]](s32)
+    ; SI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; SI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY12]], [[AND3]](s32)
+    ; SI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; SI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C3]]
+    ; SI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY14]], [[C3]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[AND4]](s32)
+    ; SI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[SHL1]](s32)
+    ; SI: [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY15]], [[COPY16]]
+    ; SI: [[COPY17:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; SI: $vgpr0 = COPY [[COPY17]](s32)
     ; VI-LABEL: name: test_fshr_s8_s8
     ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; VI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; VI: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; VI: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
-    ; VI: [[FSHR:%[0-9]+]]:_(s8) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s8)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s8)
-    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[COPY4]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; VI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; VI: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY5]], [[COPY6]]
+    ; VI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; VI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[COPY8]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; VI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C3]]
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND2]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
+    ; VI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
+    ; VI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C3]]
+    ; VI: [[COPY11:%[0-9]+]]:_(s16) = COPY [[SHL]](s16)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[AND3]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[COPY11]], [[TRUNC2]](s16)
+    ; VI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; VI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C3]]
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
+    ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[AND4]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[AND5]], [[TRUNC4]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL1]](s16)
+    ; VI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; VI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; VI: $vgpr0 = COPY [[COPY13]](s32)
     ; GFX9-LABEL: name: test_fshr_s8_s8
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
-    ; GFX9: [[FSHR:%[0-9]+]]:_(s8) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s8)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s8)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[COPY4]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY5]], [[COPY6]]
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[COPY8]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C3]]
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C3]]
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s16) = COPY [[SHL]](s16)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[AND3]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[COPY11]], [[TRUNC2]](s16)
+    ; GFX9: [[COPY12:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
+    ; GFX9: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[AND4]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[AND5]], [[TRUNC4]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL1]](s16)
+    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; GFX9: [[COPY13:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; GFX9: $vgpr0 = COPY [[COPY13]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
@@ -261,32 +570,158 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; SI: [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
-    ; SI: [[TRUNC1:%[0-9]+]]:_(s24) = G_TRUNC [[COPY1]](s32)
-    ; SI: [[TRUNC2:%[0-9]+]]:_(s24) = G_TRUNC [[COPY2]](s32)
-    ; SI: [[FSHR:%[0-9]+]]:_(s24) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s24)
-    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s24)
-    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C2]]
+    ; SI: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
+    ; SI: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
+    ; SI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C3]]
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[AND1]]
+    ; SI: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; SI: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; SI: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; SI: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; SI: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; SI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; SI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; SI: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; SI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; SI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[COPY5]], [[COPY6]]
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C2]]
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[AND2]](s32)
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C2]]
+    ; SI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[AND3]](s32)
+    ; SI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; SI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C2]]
+    ; SI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C2]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[AND4]](s32)
+    ; SI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[SHL1]](s32)
+    ; SI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY13]], [[COPY14]]
+    ; SI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; SI: $vgpr0 = COPY [[COPY15]](s32)
     ; VI-LABEL: name: test_fshr_s24_s24
     ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; VI: [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
-    ; VI: [[TRUNC1:%[0-9]+]]:_(s24) = G_TRUNC [[COPY1]](s32)
-    ; VI: [[TRUNC2:%[0-9]+]]:_(s24) = G_TRUNC [[COPY2]](s32)
-    ; VI: [[FSHR:%[0-9]+]]:_(s24) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s24)
-    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s24)
-    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C2]]
+    ; VI: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
+    ; VI: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
+    ; VI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C3]]
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[AND1]]
+    ; VI: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; VI: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; VI: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; VI: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; VI: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; VI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; VI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; VI: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; VI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; VI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[COPY5]], [[COPY6]]
+    ; VI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; VI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C2]]
+    ; VI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[AND2]](s32)
+    ; VI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; VI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C2]]
+    ; VI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[AND3]](s32)
+    ; VI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; VI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C2]]
+    ; VI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C2]]
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[AND4]](s32)
+    ; VI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[SHL1]](s32)
+    ; VI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY13]], [[COPY14]]
+    ; VI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; VI: $vgpr0 = COPY [[COPY15]](s32)
     ; GFX9-LABEL: name: test_fshr_s24_s24
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s24) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s24) = G_TRUNC [[COPY2]](s32)
-    ; GFX9: [[FSHR:%[0-9]+]]:_(s24) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s24)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s24)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C2]]
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND1]](s32)
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[UITOFP]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41EFFFFFC0000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C3]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMUL]](s32)
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[AND1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SUB]], [[FPTOUI]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI]], [[UMULH]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[ADD]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UMULH1]], [[AND1]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[MUL1]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SUB1]](s32), [[AND1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[SUB2]], [[SUB1]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT]](s32), [[AND1]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SELECT]], [[AND1]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[SUB3]], [[SELECT]]
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[COPY5]], [[COPY6]]
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C2]]
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[AND2]](s32)
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[SUB4]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C2]]
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[AND3]](s32)
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C2]]
+    ; GFX9: [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C2]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[AND4]](s32)
+    ; GFX9: [[COPY13:%[0-9]+]]:_(s32) = COPY [[SHL1]](s32)
+    ; GFX9: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY13]], [[COPY14]]
+    ; GFX9: [[COPY15:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; GFX9: $vgpr0 = COPY [[COPY15]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
@@ -297,3 +732,929 @@ body: |
     %7:_(s32) = G_ANYEXT %6
     $vgpr0 = COPY %7
 ...
+
+---
+name: test_fshr_v3s16_v3s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; SI-LABEL: name: test_fshr_v3s16_v3s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; SI: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
+    ; SI: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
+    ; SI: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
+    ; SI: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; SI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
+    ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; SI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
+    ; SI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+    ; SI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
+    ; SI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST4]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; SI: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[BITCAST5]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C1]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY9]], [[C]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
+    ; SI: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; SI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; SI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; SI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[C3]], [[C4]]
+    ; SI: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[C3]], [[C5]]
+    ; SI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C4]]
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[AND3]](s16)
+    ; SI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SHL2]](s32)
+    ; SI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; SI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C1]]
+    ; SI: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C6]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
+    ; SI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; SI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C1]]
+    ; SI: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s16) = G_OR [[TRUNC]], [[TRUNC1]]
+    ; SI: [[AND7:%[0-9]+]]:_(s16) = G_AND [[C3]], [[C4]]
+    ; SI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C3]], [[C5]]
+    ; SI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C4]]
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[AND7]](s16)
+    ; SI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY13]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
+    ; SI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; SI: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY15]], [[C1]]
+    ; SI: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[AND9]], [[COPY14]](s32)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND8]](s16)
+    ; SI: [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
+    ; SI: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY16]], [[C1]]
+    ; SI: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[ZEXT3]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
+    ; SI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC2]], [[TRUNC3]]
+    ; SI: [[COPY17:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY18:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; SI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[COPY17]](s32)
+    ; SI: [[COPY19:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY20:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; SI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY20]], [[COPY19]](s32)
+    ; SI: [[COPY21:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[COPY22:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[COPY22]], [[C]](s32)
+    ; SI: [[OR4:%[0-9]+]]:_(s32) = G_OR [[COPY21]], [[SHL6]]
+    ; SI: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](s32)
+    ; SI: [[XOR2:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BITCAST6]], [[BITCAST8]]
+    ; SI: [[BITCAST9:%[0-9]+]]:_(s32) = G_BITCAST [[XOR2]](<2 x s16>)
+    ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST9]](s32)
+    ; SI: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST9]], [[C]](s32)
+    ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR10]](s32)
+    ; SI: [[AND11:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C4]]
+    ; SI: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC4]], [[C5]]
+    ; SI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C4]]
+    ; SI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[AND11]](s16)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR2]](s16)
+    ; SI: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT]], [[ZEXT4]](s32)
+    ; SI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[SHL7]](s32)
+    ; SI: [[COPY23:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY24:%[0-9]+]]:_(s32) = COPY [[SHL4]](s32)
+    ; SI: [[AND13:%[0-9]+]]:_(s32) = G_AND [[COPY24]], [[C1]]
+    ; SI: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[AND13]], [[COPY23]](s32)
+    ; SI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND12]](s16)
+    ; SI: [[COPY25:%[0-9]+]]:_(s32) = COPY [[LSHR11]](s32)
+    ; SI: [[AND14:%[0-9]+]]:_(s32) = G_AND [[COPY25]], [[C1]]
+    ; SI: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[AND14]], [[ZEXT5]](s32)
+    ; SI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR12]](s32)
+    ; SI: [[OR5:%[0-9]+]]:_(s16) = G_OR [[TRUNC6]], [[TRUNC7]]
+    ; SI: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C4]]
+    ; SI: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC5]], [[C5]]
+    ; SI: [[AND16:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C4]]
+    ; SI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[AND15]](s16)
+    ; SI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[OR3]](s16)
+    ; SI: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT1]], [[ZEXT6]](s32)
+    ; SI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[SHL8]](s32)
+    ; SI: [[COPY26:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY27:%[0-9]+]]:_(s32) = COPY [[SHL5]](s32)
+    ; SI: [[AND17:%[0-9]+]]:_(s32) = G_AND [[COPY27]], [[C1]]
+    ; SI: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[AND17]], [[COPY26]](s32)
+    ; SI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[AND16]](s16)
+    ; SI: [[COPY28:%[0-9]+]]:_(s32) = COPY [[LSHR13]](s32)
+    ; SI: [[AND18:%[0-9]+]]:_(s32) = G_AND [[COPY28]], [[C1]]
+    ; SI: [[LSHR14:%[0-9]+]]:_(s32) = G_LSHR [[AND18]], [[ZEXT7]](s32)
+    ; SI: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR14]](s32)
+    ; SI: [[OR6:%[0-9]+]]:_(s16) = G_OR [[TRUNC8]], [[TRUNC9]]
+    ; SI: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
+    ; SI: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
+    ; SI: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C]](s32)
+    ; SI: [[OR7:%[0-9]+]]:_(s32) = G_OR [[ZEXT8]], [[SHL9]]
+    ; SI: [[BITCAST10:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](s32)
+    ; SI: [[AND19:%[0-9]+]]:_(s16) = G_AND [[C3]], [[C4]]
+    ; SI: [[XOR5:%[0-9]+]]:_(s16) = G_XOR [[C3]], [[C5]]
+    ; SI: [[AND20:%[0-9]+]]:_(s16) = G_AND [[XOR5]], [[C4]]
+    ; SI: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[AND19]](s16)
+    ; SI: [[COPY29:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; SI: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[COPY29]], [[ZEXT10]](s32)
+    ; SI: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[SHL10]](s32)
+    ; SI: [[COPY30:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY31:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; SI: [[AND21:%[0-9]+]]:_(s32) = G_AND [[COPY31]], [[C1]]
+    ; SI: [[LSHR15:%[0-9]+]]:_(s32) = G_LSHR [[AND21]], [[COPY30]](s32)
+    ; SI: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[AND20]](s16)
+    ; SI: [[COPY32:%[0-9]+]]:_(s32) = COPY [[LSHR15]](s32)
+    ; SI: [[AND22:%[0-9]+]]:_(s32) = G_AND [[COPY32]], [[C1]]
+    ; SI: [[LSHR16:%[0-9]+]]:_(s32) = G_LSHR [[AND22]], [[ZEXT11]](s32)
+    ; SI: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR16]](s32)
+    ; SI: [[OR8:%[0-9]+]]:_(s16) = G_OR [[TRUNC10]], [[TRUNC11]]
+    ; SI: [[AND23:%[0-9]+]]:_(s16) = G_AND [[C3]], [[C4]]
+    ; SI: [[XOR6:%[0-9]+]]:_(s16) = G_XOR [[C3]], [[C5]]
+    ; SI: [[AND24:%[0-9]+]]:_(s16) = G_AND [[XOR6]], [[C4]]
+    ; SI: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[AND23]](s16)
+    ; SI: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; SI: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[DEF1]], [[ZEXT12]](s32)
+    ; SI: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[SHL11]](s32)
+    ; SI: [[COPY33:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[LSHR17:%[0-9]+]]:_(s32) = G_LSHR [[C2]], [[COPY33]](s32)
+    ; SI: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[AND24]](s16)
+    ; SI: [[COPY34:%[0-9]+]]:_(s32) = COPY [[LSHR17]](s32)
+    ; SI: [[AND25:%[0-9]+]]:_(s32) = G_AND [[COPY34]], [[C1]]
+    ; SI: [[LSHR18:%[0-9]+]]:_(s32) = G_LSHR [[AND25]], [[ZEXT13]](s32)
+    ; SI: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR18]](s32)
+    ; SI: [[OR9:%[0-9]+]]:_(s16) = G_OR [[TRUNC12]], [[TRUNC13]]
+    ; SI: [[COPY35:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY36:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; SI: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[COPY36]], [[COPY35]](s32)
+    ; SI: [[COPY37:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY38:%[0-9]+]]:_(s32) = COPY [[DEF1]](s32)
+    ; SI: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[COPY38]], [[COPY37]](s32)
+    ; SI: [[COPY39:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[COPY40:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; SI: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[COPY40]], [[C]](s32)
+    ; SI: [[OR10:%[0-9]+]]:_(s32) = G_OR [[COPY39]], [[SHL14]]
+    ; SI: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](s32)
+    ; SI: [[XOR7:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BITCAST7]], [[BITCAST11]]
+    ; SI: [[BITCAST12:%[0-9]+]]:_(s32) = G_BITCAST [[XOR7]](<2 x s16>)
+    ; SI: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST12]](s32)
+    ; SI: [[LSHR19:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST12]], [[C]](s32)
+    ; SI: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR19]](s32)
+    ; SI: [[AND26:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C4]]
+    ; SI: [[XOR8:%[0-9]+]]:_(s16) = G_XOR [[TRUNC14]], [[C5]]
+    ; SI: [[AND27:%[0-9]+]]:_(s16) = G_AND [[XOR8]], [[C4]]
+    ; SI: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[AND26]](s16)
+    ; SI: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[OR8]](s16)
+    ; SI: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT2]], [[ZEXT14]](s32)
+    ; SI: [[TRUNC16:%[0-9]+]]:_(s16) = G_TRUNC [[SHL15]](s32)
+    ; SI: [[COPY41:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY42:%[0-9]+]]:_(s32) = COPY [[SHL12]](s32)
+    ; SI: [[AND28:%[0-9]+]]:_(s32) = G_AND [[COPY42]], [[C1]]
+    ; SI: [[LSHR20:%[0-9]+]]:_(s32) = G_LSHR [[AND28]], [[COPY41]](s32)
+    ; SI: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[AND27]](s16)
+    ; SI: [[COPY43:%[0-9]+]]:_(s32) = COPY [[LSHR20]](s32)
+    ; SI: [[AND29:%[0-9]+]]:_(s32) = G_AND [[COPY43]], [[C1]]
+    ; SI: [[LSHR21:%[0-9]+]]:_(s32) = G_LSHR [[AND29]], [[ZEXT15]](s32)
+    ; SI: [[TRUNC17:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR21]](s32)
+    ; SI: [[OR11:%[0-9]+]]:_(s16) = G_OR [[TRUNC16]], [[TRUNC17]]
+    ; SI: [[AND30:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C4]]
+    ; SI: [[XOR9:%[0-9]+]]:_(s16) = G_XOR [[TRUNC15]], [[C5]]
+    ; SI: [[AND31:%[0-9]+]]:_(s16) = G_AND [[XOR9]], [[C4]]
+    ; SI: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[AND30]](s16)
+    ; SI: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[OR9]](s16)
+    ; SI: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT3]], [[ZEXT16]](s32)
+    ; SI: [[TRUNC18:%[0-9]+]]:_(s16) = G_TRUNC [[SHL16]](s32)
+    ; SI: [[COPY44:%[0-9]+]]:_(s32) = COPY [[C6]](s32)
+    ; SI: [[COPY45:%[0-9]+]]:_(s32) = COPY [[SHL13]](s32)
+    ; SI: [[AND32:%[0-9]+]]:_(s32) = G_AND [[COPY45]], [[C1]]
+    ; SI: [[LSHR22:%[0-9]+]]:_(s32) = G_LSHR [[AND32]], [[COPY44]](s32)
+    ; SI: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[AND31]](s16)
+    ; SI: [[COPY46:%[0-9]+]]:_(s32) = COPY [[LSHR22]](s32)
+    ; SI: [[AND33:%[0-9]+]]:_(s32) = G_AND [[COPY46]], [[C1]]
+    ; SI: [[LSHR23:%[0-9]+]]:_(s32) = G_LSHR [[AND33]], [[ZEXT17]](s32)
+    ; SI: [[TRUNC19:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR23]](s32)
+    ; SI: [[OR12:%[0-9]+]]:_(s16) = G_OR [[TRUNC18]], [[TRUNC19]]
+    ; SI: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[OR11]](s16)
+    ; SI: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[OR12]](s16)
+    ; SI: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXT19]], [[C]](s32)
+    ; SI: [[OR13:%[0-9]+]]:_(s32) = G_OR [[ZEXT18]], [[SHL17]]
+    ; SI: [[BITCAST13:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR13]](s32)
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[BITCAST14:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST10]](<2 x s16>)
+    ; SI: [[LSHR24:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST14]], [[C]](s32)
+    ; SI: [[BITCAST15:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST13]](<2 x s16>)
+    ; SI: [[LSHR25:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST15]], [[C]](s32)
+    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF2]](<4 x s16>)
+    ; SI: [[BITCAST16:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[LSHR26:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST16]], [[C]](s32)
+    ; SI: [[BITCAST17:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[LSHR27:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST17]], [[C]](s32)
+    ; SI: [[COPY47:%[0-9]+]]:_(s32) = COPY [[BITCAST14]](s32)
+    ; SI: [[AND34:%[0-9]+]]:_(s32) = G_AND [[COPY47]], [[C1]]
+    ; SI: [[COPY48:%[0-9]+]]:_(s32) = COPY [[LSHR24]](s32)
+    ; SI: [[AND35:%[0-9]+]]:_(s32) = G_AND [[COPY48]], [[C1]]
+    ; SI: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[AND35]], [[C]](s32)
+    ; SI: [[OR14:%[0-9]+]]:_(s32) = G_OR [[AND34]], [[SHL18]]
+    ; SI: [[BITCAST18:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR14]](s32)
+    ; SI: [[COPY49:%[0-9]+]]:_(s32) = COPY [[BITCAST15]](s32)
+    ; SI: [[AND36:%[0-9]+]]:_(s32) = G_AND [[COPY49]], [[C1]]
+    ; SI: [[COPY50:%[0-9]+]]:_(s32) = COPY [[BITCAST16]](s32)
+    ; SI: [[AND37:%[0-9]+]]:_(s32) = G_AND [[COPY50]], [[C1]]
+    ; SI: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[AND37]], [[C]](s32)
+    ; SI: [[OR15:%[0-9]+]]:_(s32) = G_OR [[AND36]], [[SHL19]]
+    ; SI: [[BITCAST19:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR15]](s32)
+    ; SI: [[COPY51:%[0-9]+]]:_(s32) = COPY [[LSHR26]](s32)
+    ; SI: [[AND38:%[0-9]+]]:_(s32) = G_AND [[COPY51]], [[C1]]
+    ; SI: [[COPY52:%[0-9]+]]:_(s32) = COPY [[BITCAST17]](s32)
+    ; SI: [[AND39:%[0-9]+]]:_(s32) = G_AND [[COPY52]], [[C1]]
+    ; SI: [[SHL20:%[0-9]+]]:_(s32) = G_SHL [[AND39]], [[C]](s32)
+    ; SI: [[OR16:%[0-9]+]]:_(s32) = G_OR [[AND38]], [[SHL20]]
+    ; SI: [[BITCAST20:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR16]](s32)
+    ; SI: $vgpr0 = COPY [[BITCAST18]](<2 x s16>)
+    ; SI: $vgpr1 = COPY [[BITCAST19]](<2 x s16>)
+    ; SI: $vgpr2 = COPY [[BITCAST20]](<2 x s16>)
+    ; VI-LABEL: name: test_fshr_v3s16_v3s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; VI: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
+    ; VI: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
+    ; VI: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
+    ; VI: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; VI: [[DEF1:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
+    ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; VI: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
+    ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; VI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
+    ; VI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+    ; VI: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
+    ; VI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; VI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST4]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
+    ; VI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; VI: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; VI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[BITCAST5]](s32)
+    ; VI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C1]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+    ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
+    ; VI: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; VI: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[C3]], [[C4]]
+    ; VI: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[C3]], [[C5]]
+    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C4]]
+    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND3]](s16)
+    ; VI: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C3]](s16)
+    ; VI: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[LSHR6]], [[AND4]](s16)
+    ; VI: [[OR2:%[0-9]+]]:_(s16) = G_OR [[SHL2]], [[LSHR7]]
+    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[C3]], [[C4]]
+    ; VI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C3]], [[C5]]
+    ; VI: [[AND6:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C4]]
+    ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND5]](s16)
+    ; VI: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC4]], [[C3]](s16)
+    ; VI: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[LSHR8]], [[AND6]](s16)
+    ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[SHL3]], [[LSHR9]]
+    ; VI: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
+    ; VI: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[TRUNC4]], [[C3]](s16)
+    ; VI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; VI: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+    ; VI: [[OR4:%[0-9]+]]:_(s32) = G_OR [[COPY9]], [[SHL6]]
+    ; VI: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](s32)
+    ; VI: [[XOR2:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BITCAST6]], [[BITCAST8]]
+    ; VI: [[BITCAST9:%[0-9]+]]:_(s32) = G_BITCAST [[XOR2]](<2 x s16>)
+    ; VI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST9]](s32)
+    ; VI: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST9]], [[C]](s32)
+    ; VI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR10]](s32)
+    ; VI: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C4]]
+    ; VI: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC6]], [[C5]]
+    ; VI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C4]]
+    ; VI: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[OR2]], [[AND7]](s16)
+    ; VI: [[LSHR11:%[0-9]+]]:_(s16) = G_LSHR [[SHL4]], [[C3]](s16)
+    ; VI: [[LSHR12:%[0-9]+]]:_(s16) = G_LSHR [[LSHR11]], [[AND8]](s16)
+    ; VI: [[OR5:%[0-9]+]]:_(s16) = G_OR [[SHL7]], [[LSHR12]]
+    ; VI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C4]]
+    ; VI: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC7]], [[C5]]
+    ; VI: [[AND10:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C4]]
+    ; VI: [[SHL8:%[0-9]+]]:_(s16) = G_SHL [[OR3]], [[AND9]](s16)
+    ; VI: [[LSHR13:%[0-9]+]]:_(s16) = G_LSHR [[SHL5]], [[C3]](s16)
+    ; VI: [[LSHR14:%[0-9]+]]:_(s16) = G_LSHR [[LSHR13]], [[AND10]](s16)
+    ; VI: [[OR6:%[0-9]+]]:_(s16) = G_OR [[SHL8]], [[LSHR14]]
+    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
+    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
+    ; VI: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; VI: [[OR7:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL9]]
+    ; VI: [[BITCAST10:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](s32)
+    ; VI: [[AND11:%[0-9]+]]:_(s16) = G_AND [[C3]], [[C4]]
+    ; VI: [[XOR5:%[0-9]+]]:_(s16) = G_XOR [[C3]], [[C5]]
+    ; VI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[XOR5]], [[C4]]
+    ; VI: [[SHL10:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[AND11]](s16)
+    ; VI: [[LSHR15:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC5]], [[C3]](s16)
+    ; VI: [[LSHR16:%[0-9]+]]:_(s16) = G_LSHR [[LSHR15]], [[AND12]](s16)
+    ; VI: [[OR8:%[0-9]+]]:_(s16) = G_OR [[SHL10]], [[LSHR16]]
+    ; VI: [[AND13:%[0-9]+]]:_(s16) = G_AND [[C3]], [[C4]]
+    ; VI: [[XOR6:%[0-9]+]]:_(s16) = G_XOR [[C3]], [[C5]]
+    ; VI: [[AND14:%[0-9]+]]:_(s16) = G_AND [[XOR6]], [[C4]]
+    ; VI: [[SHL11:%[0-9]+]]:_(s16) = G_SHL [[DEF1]], [[AND13]](s16)
+    ; VI: [[LSHR17:%[0-9]+]]:_(s16) = G_LSHR [[DEF1]], [[C3]](s16)
+    ; VI: [[LSHR18:%[0-9]+]]:_(s16) = G_LSHR [[LSHR17]], [[AND14]](s16)
+    ; VI: [[OR9:%[0-9]+]]:_(s16) = G_OR [[SHL11]], [[LSHR18]]
+    ; VI: [[SHL12:%[0-9]+]]:_(s16) = G_SHL [[TRUNC5]], [[C3]](s16)
+    ; VI: [[SHL13:%[0-9]+]]:_(s16) = G_SHL [[DEF1]], [[C3]](s16)
+    ; VI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; VI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; VI: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[COPY11]], [[C]](s32)
+    ; VI: [[OR10:%[0-9]+]]:_(s32) = G_OR [[COPY10]], [[SHL14]]
+    ; VI: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](s32)
+    ; VI: [[XOR7:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BITCAST7]], [[BITCAST11]]
+    ; VI: [[BITCAST12:%[0-9]+]]:_(s32) = G_BITCAST [[XOR7]](<2 x s16>)
+    ; VI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST12]](s32)
+    ; VI: [[LSHR19:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST12]], [[C]](s32)
+    ; VI: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR19]](s32)
+    ; VI: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C4]]
+    ; VI: [[XOR8:%[0-9]+]]:_(s16) = G_XOR [[TRUNC8]], [[C5]]
+    ; VI: [[AND16:%[0-9]+]]:_(s16) = G_AND [[XOR8]], [[C4]]
+    ; VI: [[SHL15:%[0-9]+]]:_(s16) = G_SHL [[OR8]], [[AND15]](s16)
+    ; VI: [[LSHR20:%[0-9]+]]:_(s16) = G_LSHR [[SHL12]], [[C3]](s16)
+    ; VI: [[LSHR21:%[0-9]+]]:_(s16) = G_LSHR [[LSHR20]], [[AND16]](s16)
+    ; VI: [[OR11:%[0-9]+]]:_(s16) = G_OR [[SHL15]], [[LSHR21]]
+    ; VI: [[AND17:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C4]]
+    ; VI: [[XOR9:%[0-9]+]]:_(s16) = G_XOR [[TRUNC9]], [[C5]]
+    ; VI: [[AND18:%[0-9]+]]:_(s16) = G_AND [[XOR9]], [[C4]]
+    ; VI: [[SHL16:%[0-9]+]]:_(s16) = G_SHL [[OR9]], [[AND17]](s16)
+    ; VI: [[LSHR22:%[0-9]+]]:_(s16) = G_LSHR [[SHL13]], [[C3]](s16)
+    ; VI: [[LSHR23:%[0-9]+]]:_(s16) = G_LSHR [[LSHR22]], [[AND18]](s16)
+    ; VI: [[OR12:%[0-9]+]]:_(s16) = G_OR [[SHL16]], [[LSHR23]]
+    ; VI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR11]](s16)
+    ; VI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR12]](s16)
+    ; VI: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+    ; VI: [[OR13:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL17]]
+    ; VI: [[BITCAST13:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR13]](s32)
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[BITCAST14:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST10]](<2 x s16>)
+    ; VI: [[LSHR24:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST14]], [[C]](s32)
+    ; VI: [[BITCAST15:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST13]](<2 x s16>)
+    ; VI: [[LSHR25:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST15]], [[C]](s32)
+    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF2]](<4 x s16>)
+    ; VI: [[BITCAST16:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[LSHR26:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST16]], [[C]](s32)
+    ; VI: [[BITCAST17:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[LSHR27:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST17]], [[C]](s32)
+    ; VI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST14]](s32)
+    ; VI: [[AND19:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C1]]
+    ; VI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR24]](s32)
+    ; VI: [[AND20:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C1]]
+    ; VI: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[AND20]], [[C]](s32)
+    ; VI: [[OR14:%[0-9]+]]:_(s32) = G_OR [[AND19]], [[SHL18]]
+    ; VI: [[BITCAST18:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR14]](s32)
+    ; VI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST15]](s32)
+    ; VI: [[AND21:%[0-9]+]]:_(s32) = G_AND [[COPY14]], [[C1]]
+    ; VI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[BITCAST16]](s32)
+    ; VI: [[AND22:%[0-9]+]]:_(s32) = G_AND [[COPY15]], [[C1]]
+    ; VI: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[AND22]], [[C]](s32)
+    ; VI: [[OR15:%[0-9]+]]:_(s32) = G_OR [[AND21]], [[SHL19]]
+    ; VI: [[BITCAST19:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR15]](s32)
+    ; VI: [[COPY16:%[0-9]+]]:_(s32) = COPY [[LSHR26]](s32)
+    ; VI: [[AND23:%[0-9]+]]:_(s32) = G_AND [[COPY16]], [[C1]]
+    ; VI: [[COPY17:%[0-9]+]]:_(s32) = COPY [[BITCAST17]](s32)
+    ; VI: [[AND24:%[0-9]+]]:_(s32) = G_AND [[COPY17]], [[C1]]
+    ; VI: [[SHL20:%[0-9]+]]:_(s32) = G_SHL [[AND24]], [[C]](s32)
+    ; VI: [[OR16:%[0-9]+]]:_(s32) = G_OR [[AND23]], [[SHL20]]
+    ; VI: [[BITCAST20:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR16]](s32)
+    ; VI: $vgpr0 = COPY [[BITCAST18]](<2 x s16>)
+    ; VI: $vgpr1 = COPY [[BITCAST19]](<2 x s16>)
+    ; VI: $vgpr2 = COPY [[BITCAST20]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fshr_v3s16_v3s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; GFX9: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
+    ; GFX9: [[COPY4:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr4
+    ; GFX9: [[COPY5:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr5
+    ; GFX9: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[DEF1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY8]](s32), [[COPY9]](s32)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY2]](<2 x s16>)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY3]](<2 x s16>)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
+    ; GFX9: [[COPY12:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX9: [[COPY13:%[0-9]+]]:_(s32) = COPY [[DEF1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
+    ; GFX9: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[COPY4]](<2 x s16>)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C]](s32)
+    ; GFX9: [[BITCAST5:%[0-9]+]]:_(s32) = G_BITCAST [[COPY5]](<2 x s16>)
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
+    ; GFX9: [[COPY14:%[0-9]+]]:_(s32) = COPY [[BITCAST4]](s32)
+    ; GFX9: [[COPY15:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
+    ; GFX9: [[COPY16:%[0-9]+]]:_(s32) = COPY [[BITCAST5]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[DEF1]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; GFX9: [[COPY17:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[C1]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[BUILD_VECTOR_TRUNC4]], [[BUILD_VECTOR_TRUNC6]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[COPY18:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY18]](s32), [[C2]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BUILD_VECTOR_TRUNC4]], [[BUILD_VECTOR_TRUNC7]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR]], [[BUILD_VECTOR_TRUNC6]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[COPY19:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC8:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY19]](s32), [[C3]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC8]](<2 x s16>)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL]], [[AND1]](<2 x s16>)
+    ; GFX9: [[LSHR6:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[BUILD_VECTOR_TRUNC2]], [[AND]](<2 x s16>)
+    ; GFX9: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR6]]
+    ; GFX9: [[COPY20:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[COPY21:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC9:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY20]](s32), [[COPY21]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(<2 x s16>) = G_AND [[BUILD_VECTOR_TRUNC5]], [[BUILD_VECTOR_TRUNC9]]
+    ; GFX9: [[COPY22:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[COPY23:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC10:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY22]](s32), [[COPY23]](s32)
+    ; GFX9: [[XOR1:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BUILD_VECTOR_TRUNC5]], [[BUILD_VECTOR_TRUNC10]]
+    ; GFX9: [[AND3:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR1]], [[BUILD_VECTOR_TRUNC9]]
+    ; GFX9: [[COPY24:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; GFX9: [[COPY25:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC11:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY24]](s32), [[COPY25]](s32)
+    ; GFX9: [[SHL2:%[0-9]+]]:_(<2 x s16>) = G_SHL [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC11]](<2 x s16>)
+    ; GFX9: [[SHL3:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL2]], [[AND3]](<2 x s16>)
+    ; GFX9: [[LSHR7:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[BUILD_VECTOR_TRUNC3]], [[AND2]](<2 x s16>)
+    ; GFX9: [[OR1:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL3]], [[LSHR7]]
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[OR]](<2 x s16>)
+    ; GFX9: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C]](s32)
+    ; GFX9: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[OR1]](<2 x s16>)
+    ; GFX9: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C]](s32)
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[DEF2]](<4 x s16>)
+    ; GFX9: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX9: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST8]], [[C]](s32)
+    ; GFX9: [[BITCAST9:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX9: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST9]], [[C]](s32)
+    ; GFX9: [[COPY26:%[0-9]+]]:_(s32) = COPY [[BITCAST6]](s32)
+    ; GFX9: [[COPY27:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC12:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY26]](s32), [[COPY27]](s32)
+    ; GFX9: [[COPY28:%[0-9]+]]:_(s32) = COPY [[BITCAST7]](s32)
+    ; GFX9: [[COPY29:%[0-9]+]]:_(s32) = COPY [[BITCAST8]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC13:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY28]](s32), [[COPY29]](s32)
+    ; GFX9: [[COPY30:%[0-9]+]]:_(s32) = COPY [[LSHR10]](s32)
+    ; GFX9: [[COPY31:%[0-9]+]]:_(s32) = COPY [[BITCAST9]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC14:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY30]](s32), [[COPY31]](s32)
+    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR_TRUNC12]](<2 x s16>)
+    ; GFX9: $vgpr1 = COPY [[BUILD_VECTOR_TRUNC13]](<2 x s16>)
+    ; GFX9: $vgpr2 = COPY [[BUILD_VECTOR_TRUNC14]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = COPY $vgpr2
+    %3:_(<2 x s16>) = COPY $vgpr3
+    %4:_(<2 x s16>) = COPY $vgpr4
+    %5:_(<2 x s16>) = COPY $vgpr5
+    %6:_(<2 x s16>) = G_IMPLICIT_DEF
+    %7:_(<6 x s16>) = G_CONCAT_VECTORS %0(<2 x s16>), %1(<2 x s16>), %6(<2 x s16>)
+    %8:_(<3 x s16>), %9:_(<3 x s16>) = G_UNMERGE_VALUES %7(<6 x s16>)
+    %10:_(<6 x s16>) = G_CONCAT_VECTORS %2(<2 x s16>), %3(<2 x s16>), %6(<2 x s16>)
+    %11:_(<3 x s16>), %12:_(<3 x s16>) = G_UNMERGE_VALUES %10(<6 x s16>)
+    %13:_(<6 x s16>) = G_CONCAT_VECTORS %4(<2 x s16>), %5(<2 x s16>), %6(<2 x s16>)
+    %14:_(<3 x s16>), %15:_(<3 x s16>) = G_UNMERGE_VALUES %13(<6 x s16>)
+    %16:_(<3 x s16>) = G_FSHR %8, %11, %14(<3 x s16>)
+    %17:_(<3 x s16>) = G_IMPLICIT_DEF
+    %18:_(<6 x s16>) = G_CONCAT_VECTORS %16(<3 x s16>), %17(<3 x s16>)
+    %19:_(<2 x s16>), %20:_(<2 x s16>), %21:_(<2 x s16>) = G_UNMERGE_VALUES %18(<6 x s16>)
+    $vgpr0 = COPY %19(<2 x s16>)
+    $vgpr1 = COPY %20(<2 x s16>)
+    $vgpr2 = COPY %21(<2 x s16>)
+...
+
+---
+name: test_fshr_v4s16_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fshr_v4s16_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
+    ; SI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+    ; SI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; SI: [[AND:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; SI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; SI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; SI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C2]]
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[AND]](s16)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; SI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C5]]
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C4]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[AND1]](s16)
+    ; SI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C5]]
+    ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s16) = G_OR [[TRUNC]], [[TRUNC1]]
+    ; SI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; SI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; SI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[AND4]](s16)
+    ; SI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; SI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C5]]
+    ; SI: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[COPY7]](s32)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[AND5]](s16)
+    ; SI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; SI: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C5]]
+    ; SI: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[ZEXT3]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC2]], [[TRUNC3]]
+    ; SI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; SI: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
+    ; SI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY11]], [[COPY10]](s32)
+    ; SI: [[COPY12:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; SI: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY13]], [[COPY12]](s32)
+    ; SI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; SI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; SI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY15]], [[C1]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[COPY14]], [[SHL4]]
+    ; SI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
+    ; SI: [[XOR2:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV4]], [[BITCAST3]]
+    ; SI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[XOR2]](<2 x s16>)
+    ; SI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
+    ; SI: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C1]](s32)
+    ; SI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+    ; SI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C2]]
+    ; SI: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC4]], [[C3]]
+    ; SI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C2]]
+    ; SI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[AND8]](s16)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; SI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT]], [[ZEXT4]](s32)
+    ; SI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[SHL5]](s32)
+    ; SI: [[COPY16:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY17:%[0-9]+]]:_(s32) = COPY [[SHL2]](s32)
+    ; SI: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY17]], [[C5]]
+    ; SI: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[COPY16]](s32)
+    ; SI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[AND9]](s16)
+    ; SI: [[COPY18:%[0-9]+]]:_(s32) = COPY [[LSHR8]](s32)
+    ; SI: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY18]], [[C5]]
+    ; SI: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[AND11]], [[ZEXT5]](s32)
+    ; SI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
+    ; SI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC6]], [[TRUNC7]]
+    ; SI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C2]]
+    ; SI: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC5]], [[C3]]
+    ; SI: [[AND13:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
+    ; SI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[AND12]](s16)
+    ; SI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[OR1]](s16)
+    ; SI: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT1]], [[ZEXT6]](s32)
+    ; SI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[SHL6]](s32)
+    ; SI: [[COPY19:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY20:%[0-9]+]]:_(s32) = COPY [[SHL3]](s32)
+    ; SI: [[AND14:%[0-9]+]]:_(s32) = G_AND [[COPY20]], [[C5]]
+    ; SI: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[AND14]], [[COPY19]](s32)
+    ; SI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[AND13]](s16)
+    ; SI: [[COPY21:%[0-9]+]]:_(s32) = COPY [[LSHR10]](s32)
+    ; SI: [[AND15:%[0-9]+]]:_(s32) = G_AND [[COPY21]], [[C5]]
+    ; SI: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[AND15]], [[ZEXT7]](s32)
+    ; SI: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR11]](s32)
+    ; SI: [[OR4:%[0-9]+]]:_(s16) = G_OR [[TRUNC8]], [[TRUNC9]]
+    ; SI: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; SI: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
+    ; SI: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C1]](s32)
+    ; SI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT8]], [[SHL7]]
+    ; SI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
+    ; SI: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; SI: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C1]](s32)
+    ; SI: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C1]](s32)
+    ; SI: [[AND16:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; SI: [[XOR5:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; SI: [[AND17:%[0-9]+]]:_(s16) = G_AND [[XOR5]], [[C2]]
+    ; SI: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[AND16]](s16)
+    ; SI: [[COPY22:%[0-9]+]]:_(s32) = COPY [[BITCAST6]](s32)
+    ; SI: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[COPY22]], [[ZEXT10]](s32)
+    ; SI: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[SHL8]](s32)
+    ; SI: [[COPY23:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY24:%[0-9]+]]:_(s32) = COPY [[BITCAST7]](s32)
+    ; SI: [[AND18:%[0-9]+]]:_(s32) = G_AND [[COPY24]], [[C5]]
+    ; SI: [[LSHR14:%[0-9]+]]:_(s32) = G_LSHR [[AND18]], [[COPY23]](s32)
+    ; SI: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[AND17]](s16)
+    ; SI: [[COPY25:%[0-9]+]]:_(s32) = COPY [[LSHR14]](s32)
+    ; SI: [[AND19:%[0-9]+]]:_(s32) = G_AND [[COPY25]], [[C5]]
+    ; SI: [[LSHR15:%[0-9]+]]:_(s32) = G_LSHR [[AND19]], [[ZEXT11]](s32)
+    ; SI: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR15]](s32)
+    ; SI: [[OR6:%[0-9]+]]:_(s16) = G_OR [[TRUNC10]], [[TRUNC11]]
+    ; SI: [[AND20:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; SI: [[XOR6:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; SI: [[AND21:%[0-9]+]]:_(s16) = G_AND [[XOR6]], [[C2]]
+    ; SI: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[AND20]](s16)
+    ; SI: [[COPY26:%[0-9]+]]:_(s32) = COPY [[LSHR12]](s32)
+    ; SI: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[COPY26]], [[ZEXT12]](s32)
+    ; SI: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[SHL9]](s32)
+    ; SI: [[COPY27:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY28:%[0-9]+]]:_(s32) = COPY [[LSHR13]](s32)
+    ; SI: [[AND22:%[0-9]+]]:_(s32) = G_AND [[COPY28]], [[C5]]
+    ; SI: [[LSHR16:%[0-9]+]]:_(s32) = G_LSHR [[AND22]], [[COPY27]](s32)
+    ; SI: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[AND21]](s16)
+    ; SI: [[COPY29:%[0-9]+]]:_(s32) = COPY [[LSHR16]](s32)
+    ; SI: [[AND23:%[0-9]+]]:_(s32) = G_AND [[COPY29]], [[C5]]
+    ; SI: [[LSHR17:%[0-9]+]]:_(s32) = G_LSHR [[AND23]], [[ZEXT13]](s32)
+    ; SI: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR17]](s32)
+    ; SI: [[OR7:%[0-9]+]]:_(s16) = G_OR [[TRUNC12]], [[TRUNC13]]
+    ; SI: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; SI: [[LSHR18:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST8]], [[C1]](s32)
+    ; SI: [[COPY30:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY31:%[0-9]+]]:_(s32) = COPY [[BITCAST8]](s32)
+    ; SI: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[COPY31]], [[COPY30]](s32)
+    ; SI: [[COPY32:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY33:%[0-9]+]]:_(s32) = COPY [[LSHR18]](s32)
+    ; SI: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[COPY33]], [[COPY32]](s32)
+    ; SI: [[COPY34:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; SI: [[COPY35:%[0-9]+]]:_(s32) = COPY [[C5]](s32)
+    ; SI: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[COPY35]], [[C1]](s32)
+    ; SI: [[OR8:%[0-9]+]]:_(s32) = G_OR [[COPY34]], [[SHL12]]
+    ; SI: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](s32)
+    ; SI: [[XOR7:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV5]], [[BITCAST9]]
+    ; SI: [[BITCAST10:%[0-9]+]]:_(s32) = G_BITCAST [[XOR7]](<2 x s16>)
+    ; SI: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST10]](s32)
+    ; SI: [[LSHR19:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST10]], [[C1]](s32)
+    ; SI: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR19]](s32)
+    ; SI: [[AND24:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C2]]
+    ; SI: [[XOR8:%[0-9]+]]:_(s16) = G_XOR [[TRUNC14]], [[C3]]
+    ; SI: [[AND25:%[0-9]+]]:_(s16) = G_AND [[XOR8]], [[C2]]
+    ; SI: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[AND24]](s16)
+    ; SI: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[OR6]](s16)
+    ; SI: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT2]], [[ZEXT14]](s32)
+    ; SI: [[TRUNC16:%[0-9]+]]:_(s16) = G_TRUNC [[SHL13]](s32)
+    ; SI: [[COPY36:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY37:%[0-9]+]]:_(s32) = COPY [[SHL10]](s32)
+    ; SI: [[AND26:%[0-9]+]]:_(s32) = G_AND [[COPY37]], [[C5]]
+    ; SI: [[LSHR20:%[0-9]+]]:_(s32) = G_LSHR [[AND26]], [[COPY36]](s32)
+    ; SI: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[AND25]](s16)
+    ; SI: [[COPY38:%[0-9]+]]:_(s32) = COPY [[LSHR20]](s32)
+    ; SI: [[AND27:%[0-9]+]]:_(s32) = G_AND [[COPY38]], [[C5]]
+    ; SI: [[LSHR21:%[0-9]+]]:_(s32) = G_LSHR [[AND27]], [[ZEXT15]](s32)
+    ; SI: [[TRUNC17:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR21]](s32)
+    ; SI: [[OR9:%[0-9]+]]:_(s16) = G_OR [[TRUNC16]], [[TRUNC17]]
+    ; SI: [[AND28:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C2]]
+    ; SI: [[XOR9:%[0-9]+]]:_(s16) = G_XOR [[TRUNC15]], [[C3]]
+    ; SI: [[AND29:%[0-9]+]]:_(s16) = G_AND [[XOR9]], [[C2]]
+    ; SI: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[AND28]](s16)
+    ; SI: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[OR7]](s16)
+    ; SI: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT3]], [[ZEXT16]](s32)
+    ; SI: [[TRUNC18:%[0-9]+]]:_(s16) = G_TRUNC [[SHL14]](s32)
+    ; SI: [[COPY39:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI: [[COPY40:%[0-9]+]]:_(s32) = COPY [[SHL11]](s32)
+    ; SI: [[AND30:%[0-9]+]]:_(s32) = G_AND [[COPY40]], [[C5]]
+    ; SI: [[LSHR22:%[0-9]+]]:_(s32) = G_LSHR [[AND30]], [[COPY39]](s32)
+    ; SI: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[AND29]](s16)
+    ; SI: [[COPY41:%[0-9]+]]:_(s32) = COPY [[LSHR22]](s32)
+    ; SI: [[AND31:%[0-9]+]]:_(s32) = G_AND [[COPY41]], [[C5]]
+    ; SI: [[LSHR23:%[0-9]+]]:_(s32) = G_LSHR [[AND31]], [[ZEXT17]](s32)
+    ; SI: [[TRUNC19:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR23]](s32)
+    ; SI: [[OR10:%[0-9]+]]:_(s16) = G_OR [[TRUNC18]], [[TRUNC19]]
+    ; SI: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
+    ; SI: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[OR10]](s16)
+    ; SI: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[ZEXT19]], [[C1]](s32)
+    ; SI: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT18]], [[SHL15]]
+    ; SI: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR11]](s32)
+    ; SI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x s16>), [[BITCAST11]](<2 x s16>)
+    ; SI: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; VI-LABEL: name: test_fshr_v4s16_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 15
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; VI: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; VI: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[XOR]], [[C2]]
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[AND]](s16)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[C]](s16)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[LSHR2]], [[AND1]](s16)
+    ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[SHL]], [[LSHR3]]
+    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; VI: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[XOR1]], [[C2]]
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[AND2]](s16)
+    ; VI: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C]](s16)
+    ; VI: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[LSHR4]], [[AND3]](s16)
+    ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[SHL1]], [[LSHR5]]
+    ; VI: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; VI: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C1]](s32)
+    ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
+    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC4]], [[C]](s16)
+    ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC5]], [[C]](s16)
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; VI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C1]](s32)
+    ; VI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[COPY3]], [[SHL4]]
+    ; VI: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
+    ; VI: [[XOR2:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV4]], [[BITCAST3]]
+    ; VI: [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[XOR2]](<2 x s16>)
+    ; VI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST4]](s32)
+    ; VI: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST4]], [[C1]](s32)
+    ; VI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR7]](s32)
+    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C2]]
+    ; VI: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC6]], [[C3]]
+    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[XOR3]], [[C2]]
+    ; VI: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[OR]], [[AND4]](s16)
+    ; VI: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[SHL2]], [[C]](s16)
+    ; VI: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[LSHR8]], [[AND5]](s16)
+    ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[SHL5]], [[LSHR9]]
+    ; VI: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C2]]
+    ; VI: [[XOR4:%[0-9]+]]:_(s16) = G_XOR [[TRUNC7]], [[C3]]
+    ; VI: [[AND7:%[0-9]+]]:_(s16) = G_AND [[XOR4]], [[C2]]
+    ; VI: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[OR1]], [[AND6]](s16)
+    ; VI: [[LSHR10:%[0-9]+]]:_(s16) = G_LSHR [[SHL3]], [[C]](s16)
+    ; VI: [[LSHR11:%[0-9]+]]:_(s16) = G_LSHR [[LSHR10]], [[AND7]](s16)
+    ; VI: [[OR4:%[0-9]+]]:_(s16) = G_OR [[SHL6]], [[LSHR11]]
+    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
+    ; VI: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+    ; VI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL7]]
+    ; VI: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](s32)
+    ; VI: [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; VI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST6]](s32)
+    ; VI: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST6]], [[C1]](s32)
+    ; VI: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR12]](s32)
+    ; VI: [[BITCAST7:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST7]](s32)
+    ; VI: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST7]], [[C1]](s32)
+    ; VI: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR13]](s32)
+    ; VI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; VI: [[XOR5:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; VI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[XOR5]], [[C2]]
+    ; VI: [[SHL8:%[0-9]+]]:_(s16) = G_SHL [[TRUNC8]], [[AND8]](s16)
+    ; VI: [[LSHR14:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC10]], [[C]](s16)
+    ; VI: [[LSHR15:%[0-9]+]]:_(s16) = G_LSHR [[LSHR14]], [[AND9]](s16)
+    ; VI: [[OR6:%[0-9]+]]:_(s16) = G_OR [[SHL8]], [[LSHR15]]
+    ; VI: [[AND10:%[0-9]+]]:_(s16) = G_AND [[C]], [[C2]]
+    ; VI: [[XOR6:%[0-9]+]]:_(s16) = G_XOR [[C]], [[C3]]
+    ; VI: [[AND11:%[0-9]+]]:_(s16) = G_AND [[XOR6]], [[C2]]
+    ; VI: [[SHL9:%[0-9]+]]:_(s16) = G_SHL [[TRUNC9]], [[AND10]](s16)
+    ; VI: [[LSHR16:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC11]], [[C]](s16)
+    ; VI: [[LSHR17:%[0-9]+]]:_(s16) = G_LSHR [[LSHR16]], [[AND11]](s16)
+    ; VI: [[OR7:%[0-9]+]]:_(s16) = G_OR [[SHL9]], [[LSHR17]]
+    ; VI: [[BITCAST8:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; VI: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST8]](s32)
+    ; VI: [[LSHR18:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST8]], [[C1]](s32)
+    ; VI: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR18]](s32)
+    ; VI: [[SHL10:%[0-9]+]]:_(s16) = G_SHL [[TRUNC12]], [[C]](s16)
+    ; VI: [[SHL11:%[0-9]+]]:_(s16) = G_SHL [[TRUNC13]], [[C]](s16)
+    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; VI: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C1]](s32)
+    ; VI: [[OR8:%[0-9]+]]:_(s32) = G_OR [[COPY4]], [[SHL12]]
+    ; VI: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](s32)
+    ; VI: [[XOR7:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV5]], [[BITCAST9]]
+    ; VI: [[BITCAST10:%[0-9]+]]:_(s32) = G_BITCAST [[XOR7]](<2 x s16>)
+    ; VI: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST10]](s32)
+    ; VI: [[LSHR19:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST10]], [[C1]](s32)
+    ; VI: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR19]](s32)
+    ; VI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C2]]
+    ; VI: [[XOR8:%[0-9]+]]:_(s16) = G_XOR [[TRUNC14]], [[C3]]
+    ; VI: [[AND13:%[0-9]+]]:_(s16) = G_AND [[XOR8]], [[C2]]
+    ; VI: [[SHL13:%[0-9]+]]:_(s16) = G_SHL [[OR6]], [[AND12]](s16)
+    ; VI: [[LSHR20:%[0-9]+]]:_(s16) = G_LSHR [[SHL10]], [[C]](s16)
+    ; VI: [[LSHR21:%[0-9]+]]:_(s16) = G_LSHR [[LSHR20]], [[AND13]](s16)
+    ; VI: [[OR9:%[0-9]+]]:_(s16) = G_OR [[SHL13]], [[LSHR21]]
+    ; VI: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C2]]
+    ; VI: [[XOR9:%[0-9]+]]:_(s16) = G_XOR [[TRUNC15]], [[C3]]
+    ; VI: [[AND15:%[0-9]+]]:_(s16) = G_AND [[XOR9]], [[C2]]
+    ; VI: [[SHL14:%[0-9]+]]:_(s16) = G_SHL [[OR7]], [[AND14]](s16)
+    ; VI: [[LSHR22:%[0-9]+]]:_(s16) = G_LSHR [[SHL11]], [[C]](s16)
+    ; VI: [[LSHR23:%[0-9]+]]:_(s16) = G_LSHR [[LSHR22]], [[AND15]](s16)
+    ; VI: [[OR10:%[0-9]+]]:_(s16) = G_OR [[SHL14]], [[LSHR23]]
+    ; VI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
+    ; VI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR10]](s16)
+    ; VI: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+    ; VI: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL15]]
+    ; VI: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR11]](s32)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x s16>), [[BITCAST11]](<2 x s16>)
+    ; VI: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fshr_v4s16_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[UV4]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[C1]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV4]], [[BUILD_VECTOR_TRUNC1]]
+    ; GFX9: [[AND1:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY5]](s32), [[C2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV]], [[BUILD_VECTOR_TRUNC2]](<2 x s16>)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL]], [[AND1]](<2 x s16>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV2]], [[AND]](<2 x s16>)
+    ; GFX9: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(<2 x s16>) = G_AND [[UV5]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY8]](s32), [[COPY9]](s32)
+    ; GFX9: [[XOR1:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV5]], [[BUILD_VECTOR_TRUNC4]]
+    ; GFX9: [[AND3:%[0-9]+]]:_(<2 x s16>) = G_AND [[XOR1]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
+    ; GFX9: [[SHL2:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV1]], [[BUILD_VECTOR_TRUNC5]](<2 x s16>)
+    ; GFX9: [[SHL3:%[0-9]+]]:_(<2 x s16>) = G_SHL [[SHL2]], [[AND3]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV3]], [[AND2]](<2 x s16>)
+    ; GFX9: [[OR1:%[0-9]+]]:_(<2 x s16>) = G_OR [[SHL3]], [[LSHR1]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[OR]](<2 x s16>), [[OR1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    %3:_(<4 x s16>) = G_FSHR %0, %1, %2
+    $vgpr0_vgpr1 = COPY %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..87b0c5bd57a7b6f549d7d3898fdcfe2854616c6b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir
@@ -0,0 +1,578 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=GFX8
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx906 -O0 -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=GFX9
+
+---
+name: test_smulo_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: test_smulo_s32
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[COPY]], [[COPY1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SMULH]](s32), [[ASHR]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: $vgpr0 = COPY [[MUL]](s32)
+    ; GFX8: $vgpr1 = COPY [[SEXT]](s32)
+    ; GFX9-LABEL: name: test_smulo_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[COPY]], [[COPY1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SMULH]](s32), [[ASHR]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: $vgpr0 = COPY [[MUL]](s32)
+    ; GFX9: $vgpr1 = COPY [[SEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32), %3:_(s1) = G_SMULO %0, %1
+    %4:_(s32) = G_SEXT %3
+    $vgpr0 = COPY %2
+    $vgpr1 = COPY %4
+...
+
+---
+name: test_smulo_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: test_smulo_v2s32
+    ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX8: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[UV]], [[UV2]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[UV]], [[UV2]]
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SMULH]](s32), [[ASHR]]
+    ; GFX8: [[SMULH1:%[0-9]+]]:_(s32) = G_SMULH [[UV1]], [[UV3]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UV1]], [[UV3]]
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[MUL1]], [[C]](s32)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SMULH1]](s32), [[ASHR1]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[MUL]](s32), [[MUL1]](s32)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 1
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[ANYEXT1]](s32)
+    ; GFX8: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 1
+    ; GFX8: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX8: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR1]](<2 x s32>)
+    ; GFX9-LABEL: name: test_smulo_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[UV]], [[UV2]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[UV]], [[UV2]]
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SMULH]](s32), [[ASHR]]
+    ; GFX9: [[SMULH1:%[0-9]+]]:_(s32) = G_SMULH [[UV1]], [[UV3]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UV1]], [[UV3]]
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[MUL1]], [[C]](s32)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SMULH1]](s32), [[ASHR1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[MUL]](s32), [[MUL1]](s32)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX9: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ANYEXT]], 1
+    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX9: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ANYEXT1]], 1
+    ; GFX9: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR1]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>), %3:_(<2 x s1>) = G_SMULO %0, %1
+    %4:_(<2 x s32>) = G_SEXT %3
+    $vgpr0_vgpr1 = COPY %2
+    $vgpr2_vgpr3 = COPY %4
+...
+
+---
+name: test_smulo_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: test_smulo_s16
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 16
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 16
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX8: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 16
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: $vgpr0 = COPY [[SEXT_INREG3]](s32)
+    ; GFX8: $vgpr1 = COPY [[SEXT]](s32)
+    ; GFX9-LABEL: name: test_smulo_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 16
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 16
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX9: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 16
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: $vgpr0 = COPY [[SEXT_INREG3]](s32)
+    ; GFX9: $vgpr1 = COPY [[SEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16), %6:_(s1) = G_SMULO %2, %3
+    %5:_(s32) = G_SEXT %4
+    %7:_(s32) = G_SEXT %6
+    $vgpr0 = COPY %5
+    $vgpr1 = COPY %7
+...
+
+---
+name: test_smulo_s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: test_smulo_s8
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX8: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 8
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: $vgpr0 = COPY [[SEXT_INREG3]](s32)
+    ; GFX8: $vgpr1 = COPY [[SEXT]](s32)
+    ; GFX9-LABEL: name: test_smulo_s8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX9: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 8
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: $vgpr0 = COPY [[SEXT_INREG3]](s32)
+    ; GFX9: $vgpr1 = COPY [[SEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s8), %6:_(s1) = G_SMULO %2, %3
+    %5:_(s32) = G_SEXT %4
+    %7:_(s32) = G_SEXT %6
+    $vgpr0 = COPY %5
+    $vgpr1 = COPY %7
+...
+
+---
+name: test_smulo_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; GFX8-LABEL: name: test_smulo_v2s16
+    ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 16
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; GFX8: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 16
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX8: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 16
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; GFX8: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
+    ; GFX8: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
+    ; GFX8: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 16
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG3]], [[SEXT_INREG4]]
+    ; GFX8: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL1]], 16
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[SEXT_INREG5]]
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C]]
+    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[MUL1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C]]
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX8: [[COPY8:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+    ; GFX8: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY8]], 1
+    ; GFX8: [[COPY9:%[0-9]+]]:_(s32) = COPY [[ANYEXT1]](s32)
+    ; GFX8: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY9]], 1
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG6]](s32), [[SEXT_INREG7]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX8: [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX8: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY10]], 16
+    ; GFX8: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY11]], 16
+    ; GFX8: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG8]](s32), [[SEXT_INREG9]](s32)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR1]](<2 x s32>)
+    ; GFX8: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_smulo_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; GFX9: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 16
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; GFX9: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 16
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX9: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 16
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; GFX9: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 16
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
+    ; GFX9: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 16
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG3]], [[SEXT_INREG4]]
+    ; GFX9: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL1]], 16
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[SEXT_INREG5]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[MUL1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX9: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ANYEXT]], 1
+    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX9: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ANYEXT1]], 1
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG6]](s32), [[SEXT_INREG7]](s32)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[BUILD_VECTOR_TRUNC]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX9: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY8]], 16
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY9]], 16
+    ; GFX9: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG8]](s32), [[SEXT_INREG9]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR1]](<2 x s32>)
+    ; GFX9: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s16>) = G_TRUNC %0
+    %3:_(<2 x s16>) = G_TRUNC %1
+    %4:_(<2 x s16>), %6:_(<2 x s1>) = G_SMULO %2, %3
+    %7:_(<2 x s32>) = G_SEXT %6
+    %5:_(<2 x s32>) = G_SEXT %4
+    $vgpr0_vgpr1 = COPY %5
+    $vgpr2_vgpr3 = COPY %7
+...
+
+
+---
+name: test_smulo_v2s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX8-LABEL: name: test_smulo_v2s8
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; GFX8: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX8: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX8: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 8
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
+    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; GFX8: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG3]], [[SEXT_INREG4]]
+    ; GFX8: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL1]], 8
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[SEXT_INREG5]]
+    ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[MUL]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[MUL1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; GFX8: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX8: $vgpr1 = COPY [[ANYEXT1]](s32)
+    ; GFX8: $vgpr2 = COPY [[ANYEXT2]](s32)
+    ; GFX9-LABEL: name: test_smulo_v2s8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX9: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX9: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 8
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; GFX9: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG3]], [[SEXT_INREG4]]
+    ; GFX9: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL1]], 8
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[SEXT_INREG5]]
+    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[MUL]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[MUL1]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C1]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9: $vgpr1 = COPY [[ANYEXT1]](s32)
+    ; GFX9: $vgpr2 = COPY [[ANYEXT2]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %5:_(s8) = G_TRUNC %0
+    %6:_(s8) = G_TRUNC %1
+    %7:_(s8) = G_TRUNC %2
+    %8:_(s8) = G_TRUNC %3
+    %11:_(<2 x s8>) = G_BUILD_VECTOR %5, %6
+    %12:_(<2 x s8>) = G_BUILD_VECTOR %7, %8
+    %13:_(<2 x s8>), %19:_(<2 x s1>) = G_SMULO %11, %12
+    %20:_(<2 x s32>) = G_SEXT %19
+    %14:_(s8), %15:_(s8) = G_UNMERGE_VALUES %13
+    %21:_(s1), %22:_(s1) = G_UNMERGE_VALUES %19
+    %17:_(s16) = G_MERGE_VALUES %14, %15
+    %18:_(s32) = G_ANYEXT %17
+    %23:_(s32) = G_ANYEXT %21
+    %24:_(s32) = G_ANYEXT %22
+    $vgpr0 = COPY %18
+    $vgpr1 = COPY %23
+    $vgpr2 = COPY %24
+...
+
+---
+name: test_smulo_v4s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX8-LABEL: name: test_smulo_v4s8
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX8: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX8: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX8: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 8
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; GFX8: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX8: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG3]], [[SEXT_INREG4]]
+    ; GFX8: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL1]], 8
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[SEXT_INREG5]]
+    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
+    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX8: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG6]], [[SEXT_INREG7]]
+    ; GFX8: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL2]], 8
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL2]](s32), [[SEXT_INREG8]]
+    ; GFX8: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX8: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY8]], 8
+    ; GFX8: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; GFX8: [[SEXT_INREG10:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY9]], 8
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG9]], [[SEXT_INREG10]]
+    ; GFX8: [[SEXT_INREG11:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL3]], 8
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL3]](s32), [[SEXT_INREG11]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX8: [[COPY10:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C3]]
+    ; GFX8: [[COPY11:%[0-9]+]]:_(s32) = COPY [[MUL1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; GFX8: [[COPY12:%[0-9]+]]:_(s32) = COPY [[MUL2]](s32)
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C3]]
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; GFX8: [[COPY13:%[0-9]+]]:_(s32) = COPY [[MUL3]](s32)
+    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C3]]
+    ; GFX8: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; GFX8: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX8: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX8: $vgpr1 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_smulo_v4s8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX9: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 8
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX9: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG3]], [[SEXT_INREG4]]
+    ; GFX9: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL1]], 8
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[SEXT_INREG5]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX9: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX9: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG6]], [[SEXT_INREG7]]
+    ; GFX9: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL2]], 8
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL2]](s32), [[SEXT_INREG8]]
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX9: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY8]], 8
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; GFX9: [[SEXT_INREG10:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY9]], 8
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG9]], [[SEXT_INREG10]]
+    ; GFX9: [[SEXT_INREG11:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL3]], 8
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL3]](s32), [[SEXT_INREG11]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C3]]
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s32) = COPY [[MUL1]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; GFX9: [[COPY12:%[0-9]+]]:_(s32) = COPY [[MUL2]](s32)
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C3]]
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+    ; GFX9: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; GFX9: [[COPY13:%[0-9]+]]:_(s32) = COPY [[MUL3]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C3]]
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+    ; GFX9: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX9: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX9: $vgpr1 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8), %3:_(s8), %4:_(s8), %5:_(s8) = G_UNMERGE_VALUES %0
+    %6:_(s8), %7:_(s8), %8:_(s8), %9:_(s8) = G_UNMERGE_VALUES %1
+    %10:_(<4 x s8>) = G_BUILD_VECTOR %2:_(s8), %3:_(s8), %4:_(s8), %5:_(s8)
+    %11:_(<4 x s8>) = G_BUILD_VECTOR %6:_(s8), %7:_(s8), %8:_(s8), %9:_(s8)
+    %12:_(<4 x s8>), %18:_(<4 x s1>) = G_SMULO %10:_, %11:_
+    %13:_(s8), %14:_(s8), %15:_(s8), %16:_(s8) = G_UNMERGE_VALUES %12:_(<4 x s8>)
+    %19:_(s1), %20:_(s1), %21:_(s1), %22:_(s1) = G_UNMERGE_VALUES %18:_(<4 x s1>)
+    %17:_(s32) = G_MERGE_VALUES %13, %14, %15, %16
+    %23:_(s32) = G_ANYEXT %19
+    $vgpr0 = COPY %17
+    $vgpr1 = COPY %23
+...
+---
+name: test_smulo_s24
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: test_smulo_s24
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 24
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 24
+    ; GFX8: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SMULH]](s32), [[ASHR]]
+    ; GFX8: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 24
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX8: [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP]], [[ICMP1]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 24
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[OR]](s1)
+    ; GFX8: $vgpr0 = COPY [[SEXT_INREG3]](s32)
+    ; GFX8: $vgpr1 = COPY [[SEXT]](s32)
+    ; GFX9-LABEL: name: test_smulo_s24
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 24
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 24
+    ; GFX9: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SMULH]](s32), [[ASHR]]
+    ; GFX9: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL]], 24
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[SEXT_INREG2]]
+    ; GFX9: [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP]], [[ICMP1]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 24
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[OR]](s1)
+    ; GFX9: $vgpr0 = COPY [[SEXT_INREG3]](s32)
+    ; GFX9: $vgpr1 = COPY [[SEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s24) = G_TRUNC %0
+    %3:_(s24) = G_TRUNC %1
+    %4:_(s24), %6:_(s1) = G_SMULO %2, %3
+    %5:_(s32) = G_SEXT %4
+    %7:_(s32) = G_SEXT %6
+    $vgpr0 = COPY %5
+    $vgpr1 = COPY %7
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c3e130924acbbbf02bb8474948f74763316ebba6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir
@@ -0,0 +1,680 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=GFX8
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx906 -O0 -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=GFX9
+
+---
+name: test_umulo_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: test_umulo_s32
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[COPY1]]
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UMULH]](s32), [[C]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; GFX8: $vgpr0 = COPY [[MUL]](s32)
+    ; GFX8: $vgpr1 = COPY [[ZEXT]](s32)
+    ; GFX9-LABEL: name: test_umulo_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[COPY1]]
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UMULH]](s32), [[C]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; GFX9: $vgpr0 = COPY [[MUL]](s32)
+    ; GFX9: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32), %3:_(s1) = G_UMULO %0, %1
+    %4:_(s32) = G_ZEXT %3
+    $vgpr0 = COPY %2
+    $vgpr1 = COPY %4
+...
+
+---
+name: test_umulo_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: test_umulo_v2s32
+    ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[UV2]]
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[UV]], [[UV2]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UMULH]](s32), [[C]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[UV3]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UV1]], [[UV3]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UMULH1]](s32), [[C]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[MUL]](s32), [[MUL1]](s32)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[ANYEXT1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; GFX8: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX8: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR1]](<2 x s32>)
+    ; GFX9-LABEL: name: test_umulo_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[UV2]]
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[UV]], [[UV2]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UMULH]](s32), [[C]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[UV3]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UV1]], [[UV3]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UMULH1]](s32), [[C]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[MUL]](s32), [[MUL1]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C1]]
+    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C1]]
+    ; GFX9: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR1]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>), %3:_(<2 x s1>) = G_UMULO %0, %1
+    %4:_(<2 x s32>) = G_ZEXT %3
+    $vgpr0_vgpr1 = COPY %2
+    $vgpr2_vgpr3 = COPY %4
+...
+
+---
+name: test_umulo_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX8-LABEL: name: test_umulo_s64
+    ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[UV1]], [[UV2]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UV]], [[UV3]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[UV2]]
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL]], [[MUL1]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[UV1]], [[UV3]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[UV2]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[UV3]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL2]], [[UMULH1]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH2]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ZEXT4]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[UV3]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[ADD2]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO8]](s32), [[ADD3]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UV4]], [[UV6]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[UV5]], [[UV6]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[UV4]], [[UV7]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[UV4]], [[UV6]]
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[MUL4]], [[MUL5]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ADD4]], [[UMULH4]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[MUL3]](s32), [[ADD5]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MV]](s64), [[C]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s64) = G_ZEXT [[ICMP]](s1)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[MV1]](s64)
+    ; GFX8: $vgpr2_vgpr3 = COPY [[ZEXT5]](s64)
+    ; GFX9-LABEL: name: test_umulo_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[UV1]], [[UV2]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UV]], [[UV3]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[UV2]]
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL]], [[MUL1]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[UV1]], [[UV3]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[UV2]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[UV3]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL2]], [[UMULH1]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH2]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ZEXT4]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[UV3]]
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UMULH3]], [[ADD2]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO8]](s32), [[ADD3]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[UV4]], [[UV6]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[UV5]], [[UV6]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[UV4]], [[UV7]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[UV4]], [[UV6]]
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[MUL4]], [[MUL5]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ADD4]], [[UMULH4]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[MUL3]](s32), [[ADD5]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MV]](s64), [[C]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s64) = G_ZEXT [[ICMP]](s1)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[MV1]](s64)
+    ; GFX9: $vgpr2_vgpr3 = COPY [[ZEXT5]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64), %3:_(s1) = G_UMULO %0, %1
+    %4:_(s64) = G_ZEXT %3
+    $vgpr0_vgpr1 = COPY %2
+    $vgpr2_vgpr3 = COPY %4
+...
+
+---
+name: test_umulo_s24
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: test_umulo_s24
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[AND1]]
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UMULH]](s32), [[C1]]
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX8: [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP]], [[ICMP1]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s1)
+    ; GFX8: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX8: $vgpr1 = COPY [[ZEXT]](s32)
+    ; GFX9-LABEL: name: test_umulo_s24
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[AND1]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UMULH]](s32), [[C1]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX9: [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP]], [[ICMP1]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s1)
+    ; GFX9: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX9: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s24) = G_TRUNC %0
+    %3:_(s24) = G_TRUNC %1
+    %4:_(s24), %6:_(s1) = G_UMULO %2, %3
+    %5:_(s32) = G_ZEXT %4
+    %7:_(s32) = G_ZEXT %6
+    $vgpr0 = COPY %5
+    $vgpr1 = COPY %7
+...
+
+
+---
+name: test_umulo_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: test_umulo_s16
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; GFX8: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX8: $vgpr1 = COPY [[ZEXT]](s32)
+    ; GFX9-LABEL: name: test_umulo_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; GFX9: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX9: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16), %6:_(s1) = G_UMULO %2, %3
+    %5:_(s32) = G_ZEXT %4
+    %7:_(s32) = G_ZEXT %6
+    $vgpr0 = COPY %5
+    $vgpr1 = COPY %7
+...
+
+---
+name: test_umulo_s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX8-LABEL: name: test_umulo_s8
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; GFX8: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX8: $vgpr1 = COPY [[ZEXT]](s32)
+    ; GFX9-LABEL: name: test_umulo_s8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; GFX9: $vgpr0 = COPY [[AND3]](s32)
+    ; GFX9: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s8), %6:_(s1) = G_UMULO %2, %3
+    %5:_(s32) = G_ZEXT %4
+    %7:_(s32) = G_ZEXT %6
+    $vgpr0 = COPY %5
+    $vgpr1 = COPY %7
+...
+
+---
+name: test_umulo_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; GFX8-LABEL: name: test_umulo_v2s16
+    ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX8: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
+    ; GFX8: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[AND4]]
+    ; GFX8: [[AND5:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[AND5]]
+    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C]]
+    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[MUL1]](s32)
+    ; GFX8: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C]]
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL]]
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX8: [[COPY8:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+    ; GFX8: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C2]]
+    ; GFX8: [[COPY9:%[0-9]+]]:_(s32) = COPY [[ANYEXT1]](s32)
+    ; GFX8: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C2]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND8]](s32), [[AND9]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+    ; GFX8: [[COPY10:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX8: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C]]
+    ; GFX8: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C]]
+    ; GFX8: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND10]](s32), [[AND11]](s32)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR1]](<2 x s32>)
+    ; GFX8: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_umulo_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
+    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[AND4]]
+    ; GFX9: [[AND5:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[AND5]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[MUL1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[COPY7]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX9: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C1]]
+    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX9: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND6]](s32), [[AND7]](s32)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[BUILD_VECTOR_TRUNC]](<2 x s16>)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C2]](s32)
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX9: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C]]
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C]]
+    ; GFX9: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND8]](s32), [[AND9]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR1]](<2 x s32>)
+    ; GFX9: $vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s16>) = G_TRUNC %0
+    %3:_(<2 x s16>) = G_TRUNC %1
+    %4:_(<2 x s16>), %6:_(<2 x s1>) = G_UMULO %2, %3
+    %7:_(<2 x s32>) = G_ZEXT %6
+    %5:_(<2 x s32>) = G_ZEXT %4
+    $vgpr0_vgpr1 = COPY %5
+    $vgpr2_vgpr3 = COPY %7
+...
+
+
+---
+name: test_umulo_v2s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX8-LABEL: name: test_umulo_v2s8
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX8: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C]]
+    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; GFX8: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[AND4]]
+    ; GFX8: [[AND5:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[AND5]]
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[MUL]](s32)
+    ; GFX8: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[MUL1]](s32)
+    ; GFX8: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C2]](s16)
+    ; GFX8: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX8: $vgpr1 = COPY [[ANYEXT1]](s32)
+    ; GFX8: $vgpr2 = COPY [[ANYEXT2]](s32)
+    ; GFX9-LABEL: name: test_umulo_v2s8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C]]
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[COPY3]](s32)
+    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[AND4]]
+    ; GFX9: [[AND5:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[AND5]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[MUL]](s32)
+    ; GFX9: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C1]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[MUL1]](s32)
+    ; GFX9: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C2]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9: $vgpr1 = COPY [[ANYEXT1]](s32)
+    ; GFX9: $vgpr2 = COPY [[ANYEXT2]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %5:_(s8) = G_TRUNC %0
+    %6:_(s8) = G_TRUNC %1
+    %7:_(s8) = G_TRUNC %2
+    %8:_(s8) = G_TRUNC %3
+    %11:_(<2 x s8>) = G_BUILD_VECTOR %5, %6
+    %12:_(<2 x s8>) = G_BUILD_VECTOR %7, %8
+    %13:_(<2 x s8>), %19:_(<2 x s1>) = G_UMULO %11, %12
+    %20:_(<2 x s32>) = G_ZEXT %19
+    %14:_(s8), %15:_(s8) = G_UNMERGE_VALUES %13
+    %21:_(s1), %22:_(s1) = G_UNMERGE_VALUES %19
+    %17:_(s16) = G_MERGE_VALUES %14, %15
+    %18:_(s32) = G_ANYEXT %17
+    %23:_(s32) = G_ANYEXT %21
+    %24:_(s32) = G_ANYEXT %22
+    $vgpr0 = COPY %18
+    $vgpr1 = COPY %23
+    $vgpr2 = COPY %24
+...
+
+---
+name: test_umulo_v4s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX8-LABEL: name: test_umulo_v4s8
+    ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX8: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX8: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX8: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX8: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C3]]
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX8: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; GFX8: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX8: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C3]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[AND4]]
+    ; GFX8: [[AND5:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C3]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[AND5]]
+    ; GFX8: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX8: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C3]]
+    ; GFX8: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX8: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C3]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[AND7]]
+    ; GFX8: [[AND8:%[0-9]+]]:_(s32) = G_AND [[MUL2]], [[C3]]
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL2]](s32), [[AND8]]
+    ; GFX8: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX8: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C3]]
+    ; GFX8: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; GFX8: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C3]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[AND10]]
+    ; GFX8: [[AND11:%[0-9]+]]:_(s32) = G_AND [[MUL3]], [[C3]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL3]](s32), [[AND11]]
+    ; GFX8: [[COPY10:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX8: [[AND12:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C3]]
+    ; GFX8: [[COPY11:%[0-9]+]]:_(s32) = COPY [[MUL1]](s32)
+    ; GFX8: [[AND13:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND12]], [[SHL]]
+    ; GFX8: [[COPY12:%[0-9]+]]:_(s32) = COPY [[MUL2]](s32)
+    ; GFX8: [[AND14:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C3]]
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND14]], [[C1]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; GFX8: [[COPY13:%[0-9]+]]:_(s32) = COPY [[MUL3]](s32)
+    ; GFX8: [[AND15:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C3]]
+    ; GFX8: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C2]](s32)
+    ; GFX8: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX8: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX8: $vgpr1 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_umulo_v4s8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C3]]
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C3]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[AND4]]
+    ; GFX9: [[AND5:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C3]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[AND5]]
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX9: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C3]]
+    ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX9: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C3]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[AND7]]
+    ; GFX9: [[AND8:%[0-9]+]]:_(s32) = G_AND [[MUL2]], [[C3]]
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL2]](s32), [[AND8]]
+    ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX9: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C3]]
+    ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; GFX9: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C3]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[AND10]]
+    ; GFX9: [[AND11:%[0-9]+]]:_(s32) = G_AND [[MUL3]], [[C3]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL3]](s32), [[AND11]]
+    ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; GFX9: [[AND12:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C3]]
+    ; GFX9: [[COPY11:%[0-9]+]]:_(s32) = COPY [[MUL1]](s32)
+    ; GFX9: [[AND13:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND12]], [[SHL]]
+    ; GFX9: [[COPY12:%[0-9]+]]:_(s32) = COPY [[MUL2]](s32)
+    ; GFX9: [[AND14:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C3]]
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND14]], [[C1]](s32)
+    ; GFX9: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; GFX9: [[COPY13:%[0-9]+]]:_(s32) = COPY [[MUL3]](s32)
+    ; GFX9: [[AND15:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C3]]
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C2]](s32)
+    ; GFX9: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
+    ; GFX9: $vgpr0 = COPY [[OR2]](s32)
+    ; GFX9: $vgpr1 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8), %3:_(s8), %4:_(s8), %5:_(s8) = G_UNMERGE_VALUES %0
+    %6:_(s8), %7:_(s8), %8:_(s8), %9:_(s8) = G_UNMERGE_VALUES %1
+    %10:_(<4 x s8>) = G_BUILD_VECTOR %2:_(s8), %3:_(s8), %4:_(s8), %5:_(s8)
+    %11:_(<4 x s8>) = G_BUILD_VECTOR %6:_(s8), %7:_(s8), %8:_(s8), %9:_(s8)
+    %12:_(<4 x s8>), %18:_(<4 x s1>) = G_UMULO %10:_, %11:_
+    %13:_(s8), %14:_(s8), %15:_(s8), %16:_(s8) = G_UNMERGE_VALUES %12:_(<4 x s8>)
+    %19:_(s1), %20:_(s1), %21:_(s1), %22:_(s1) = G_UNMERGE_VALUES %18:_(<4 x s1>)
+    %17:_(s32) = G_MERGE_VALUES %13, %14, %15, %16
+    %23:_(s32) = G_ANYEXT %19
+    $vgpr0 = COPY %17
+    $vgpr1 = COPY %23
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll
index f0d76065ddd5a03b174993232cec8df19a70e600..1b8689d10a1e2e75684ceeaf90bb792304ff29af 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll
@@ -166,12 +166,13 @@ define amdgpu_ps void @branch(float %arg0, float %arg1) {
 ; SI-NEXT:    s_xor_b64 s[2:3], vcc, -1
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
 ; SI-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execz BB2_3
 ; SI-NEXT:  ; %bb.1: ; %.demote
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 BB2_4
 ; SI-NEXT:  ; %bb.2: ; %.demote
 ; SI-NEXT:    s_mov_b64 exec, 0
-; SI-NEXT:  ; %bb.3: ; %.continue
+; SI-NEXT:  BB2_3: ; %.continue
 ; SI-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
 ; SI-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
@@ -192,12 +193,13 @@ define amdgpu_ps void @branch(float %arg0, float %arg1) {
 ; GFX9-NEXT:    s_xor_b64 s[2:3], vcc, -1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
 ; GFX9-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz BB2_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB2_4
 ; GFX9-NEXT:  ; %bb.2: ; %.demote
 ; GFX9-NEXT:    s_mov_b64 exec, 0
-; GFX9-NEXT:  ; %bb.3: ; %.continue
+; GFX9-NEXT:  BB2_3: ; %.continue
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
 ; GFX9-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
@@ -218,12 +220,13 @@ define amdgpu_ps void @branch(float %arg0, float %arg1) {
 ; GFX10-32-NEXT:    s_xor_b32 s1, vcc_lo, -1
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s2, s1
 ; GFX10-32-NEXT:    s_xor_b32 s1, exec_lo, s2
+; GFX10-32-NEXT:    s_cbranch_execz BB2_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-32-NEXT:    s_andn2_b32 s0, s0, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB2_4
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-32-NEXT:    s_mov_b32 exec_lo, 0
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue
+; GFX10-32-NEXT:  BB2_3: ; %.continue
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX10-32-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
 ; GFX10-32-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
@@ -244,12 +247,13 @@ define amdgpu_ps void @branch(float %arg0, float %arg1) {
 ; GFX10-64-NEXT:    s_xor_b64 s[2:3], vcc, -1
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
 ; GFX10-64-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; GFX10-64-NEXT:    s_cbranch_execz BB2_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-64-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB2_4
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-64-NEXT:    s_mov_b64 exec, 0
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue
+; GFX10-64-NEXT:  BB2_3: ; %.continue
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX10-64-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
 ; GFX10-64-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
@@ -284,13 +288,14 @@ define amdgpu_ps <4 x float> @wqm_demote_1(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; SI-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v1
 ; SI-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; SI-NEXT:    s_xor_b64 s[14:15], exec, s[14:15]
+; SI-NEXT:    s_cbranch_execz BB3_3
 ; SI-NEXT:  ; %bb.1: ; %.demote
 ; SI-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; SI-NEXT:    s_cbranch_scc0 BB3_4
 ; SI-NEXT:  ; %bb.2: ; %.demote
 ; SI-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; SI-NEXT:    s_and_b64 exec, exec, s[16:17]
-; SI-NEXT:  ; %bb.3: ; %.continue
+; SI-NEXT:  BB3_3: ; %.continue
 ; SI-NEXT:    s_or_b64 exec, exec, s[14:15]
 ; SI-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -312,13 +317,14 @@ define amdgpu_ps <4 x float> @wqm_demote_1(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX9-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; GFX9-NEXT:    s_xor_b64 s[14:15], exec, s[14:15]
+; GFX9-NEXT:    s_cbranch_execz BB3_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote
 ; GFX9-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB3_4
 ; GFX9-NEXT:  ; %bb.2: ; %.demote
 ; GFX9-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[16:17]
-; GFX9-NEXT:  ; %bb.3: ; %.continue
+; GFX9-NEXT:  BB3_3: ; %.continue
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[14:15]
 ; GFX9-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -340,13 +346,14 @@ define amdgpu_ps <4 x float> @wqm_demote_1(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-32-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, 0, v1
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s13, vcc_lo
 ; GFX10-32-NEXT:    s_xor_b32 s13, exec_lo, s13
+; GFX10-32-NEXT:    s_cbranch_execz BB3_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-32-NEXT:    s_andn2_b32 s12, s12, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB3_4
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-32-NEXT:    s_wqm_b32 s28, s12
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s28
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue
+; GFX10-32-NEXT:  BB3_3: ; %.continue
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s13
 ; GFX10-32-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
 ; GFX10-32-NEXT:    s_waitcnt vmcnt(0)
@@ -368,13 +375,14 @@ define amdgpu_ps <4 x float> @wqm_demote_1(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-64-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v1
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; GFX10-64-NEXT:    s_xor_b64 s[28:29], exec, s[14:15]
+; GFX10-64-NEXT:    s_cbranch_execz BB3_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-64-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB3_4
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-64-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[16:17]
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue
+; GFX10-64-NEXT:  BB3_3: ; %.continue
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[28:29]
 ; GFX10-64-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
 ; GFX10-64-NEXT:    s_waitcnt vmcnt(0)
@@ -416,13 +424,14 @@ define amdgpu_ps <4 x float> @wqm_demote_2(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; SI-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
 ; SI-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; SI-NEXT:    s_xor_b64 s[14:15], exec, s[14:15]
+; SI-NEXT:    s_cbranch_execz BB4_3
 ; SI-NEXT:  ; %bb.1: ; %.demote
 ; SI-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; SI-NEXT:    s_cbranch_scc0 BB4_4
 ; SI-NEXT:  ; %bb.2: ; %.demote
 ; SI-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; SI-NEXT:    s_and_b64 exec, exec, s[16:17]
-; SI-NEXT:  ; %bb.3: ; %.continue
+; SI-NEXT:  BB4_3: ; %.continue
 ; SI-NEXT:    s_or_b64 exec, exec, s[14:15]
 ; SI-NEXT:    v_add_f32_e32 v0, v0, v0
 ; SI-NEXT:    s_and_b64 exec, exec, s[12:13]
@@ -444,13 +453,14 @@ define amdgpu_ps <4 x float> @wqm_demote_2(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX9-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; GFX9-NEXT:    s_xor_b64 s[14:15], exec, s[14:15]
+; GFX9-NEXT:    s_cbranch_execz BB4_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote
 ; GFX9-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB4_4
 ; GFX9-NEXT:  ; %bb.2: ; %.demote
 ; GFX9-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[16:17]
-; GFX9-NEXT:  ; %bb.3: ; %.continue
+; GFX9-NEXT:  BB4_3: ; %.continue
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[14:15]
 ; GFX9-NEXT:    v_add_f32_e32 v0, v0, v0
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[12:13]
@@ -472,13 +482,14 @@ define amdgpu_ps <4 x float> @wqm_demote_2(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-32-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, 0, v0
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s13, vcc_lo
 ; GFX10-32-NEXT:    s_xor_b32 s13, exec_lo, s13
+; GFX10-32-NEXT:    s_cbranch_execz BB4_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-32-NEXT:    s_andn2_b32 s12, s12, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB4_4
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-32-NEXT:    s_wqm_b32 s28, s12
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s28
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue
+; GFX10-32-NEXT:  BB4_3: ; %.continue
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s13
 ; GFX10-32-NEXT:    v_add_f32_e32 v0, v0, v0
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s12
@@ -500,13 +511,14 @@ define amdgpu_ps <4 x float> @wqm_demote_2(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-64-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; GFX10-64-NEXT:    s_xor_b64 s[28:29], exec, s[14:15]
+; GFX10-64-NEXT:    s_cbranch_execz BB4_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-64-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB4_4
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-64-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[16:17]
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue
+; GFX10-64-NEXT:  BB4_3: ; %.continue
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[28:29]
 ; GFX10-64-NEXT:    v_add_f32_e32 v0, v0, v0
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[12:13]
@@ -660,13 +672,14 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execz BB6_3
 ; SI-NEXT:  ; %bb.1: ; %.demote0
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 BB6_7
 ; SI-NEXT:  ; %bb.2: ; %.demote0
 ; SI-NEXT:    s_wqm_b64 s[6:7], s[0:1]
 ; SI-NEXT:    s_and_b64 exec, exec, s[6:7]
-; SI-NEXT:  ; %bb.3: ; %.continue0
+; SI-NEXT:  BB6_3: ; %.continue0
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0, s[4:5]
@@ -682,12 +695,13 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; SI-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
 ; SI-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[6:7]
+; SI-NEXT:    s_cbranch_execz BB6_6
 ; SI-NEXT:  ; %bb.4: ; %.demote1
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 BB6_7
 ; SI-NEXT:  ; %bb.5: ; %.demote1
 ; SI-NEXT:    s_mov_b64 exec, 0
-; SI-NEXT:  ; %bb.6: ; %.continue1
+; SI-NEXT:  BB6_6: ; %.continue1
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    v_mov_b32_e32 v0, s2
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
@@ -706,13 +720,14 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX9-NEXT:    s_cbranch_execz BB6_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote0
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX9-NEXT:  ; %bb.2: ; %.demote0
 ; GFX9-NEXT:    s_wqm_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[4:5]
-; GFX9-NEXT:  ; %bb.3: ; %.continue0
+; GFX9-NEXT:  BB6_3: ; %.continue0
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0, s[2:3]
@@ -728,12 +743,13 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX9-NEXT:    s_xor_b64 s[2:3], s[2:3], -1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
 ; GFX9-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz BB6_6
 ; GFX9-NEXT:  ; %bb.4: ; %.demote1
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX9-NEXT:  ; %bb.5: ; %.demote1
 ; GFX9-NEXT:    s_mov_b64 exec, 0
-; GFX9-NEXT:  ; %bb.6: ; %.continue1
+; GFX9-NEXT:  BB6_6: ; %.continue1
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; GFX9-NEXT:    v_bfrev_b32_e32 v1, 60
@@ -752,13 +768,14 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX10-32-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-32-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX10-32-NEXT:    s_cbranch_execz BB6_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote0
 ; GFX10-32-NEXT:    s_andn2_b32 s0, s0, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote0
 ; GFX10-32-NEXT:    s_wqm_b32 s2, s0
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s2
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue0
+; GFX10-32-NEXT:  BB6_3: ; %.continue0
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX10-32-NEXT:    s_mov_b32 s1, s0
 ; GFX10-32-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0, s1
@@ -772,12 +789,13 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX10-32-NEXT:    s_xor_b32 s1, s1, -1
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s2, s1
 ; GFX10-32-NEXT:    s_xor_b32 s1, exec_lo, s2
+; GFX10-32-NEXT:    s_cbranch_execz BB6_6
 ; GFX10-32-NEXT:  ; %bb.4: ; %.demote1
 ; GFX10-32-NEXT:    s_andn2_b32 s0, s0, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX10-32-NEXT:  ; %bb.5: ; %.demote1
 ; GFX10-32-NEXT:    s_mov_b32 exec_lo, 0
-; GFX10-32-NEXT:  ; %bb.6: ; %.continue1
+; GFX10-32-NEXT:  BB6_6: ; %.continue1
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX10-32-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; GFX10-32-NEXT:    v_bfrev_b32_e32 v1, 60
@@ -796,13 +814,14 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX10-64-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX10-64-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX10-64-NEXT:    s_cbranch_execz BB6_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote0
 ; GFX10-64-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote0
 ; GFX10-64-NEXT:    s_wqm_b64 s[4:5], s[0:1]
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[4:5]
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue0
+; GFX10-64-NEXT:  BB6_3: ; %.continue0
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX10-64-NEXT:    s_mov_b64 s[2:3], s[0:1]
 ; GFX10-64-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0, s[2:3]
@@ -816,12 +835,13 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX10-64-NEXT:    s_xor_b64 s[2:3], s[2:3], -1
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
 ; GFX10-64-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; GFX10-64-NEXT:    s_cbranch_execz BB6_6
 ; GFX10-64-NEXT:  ; %bb.4: ; %.demote1
 ; GFX10-64-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX10-64-NEXT:  ; %bb.5: ; %.demote1
 ; GFX10-64-NEXT:    s_mov_b64 exec, 0
-; GFX10-64-NEXT:  ; %bb.6: ; %.continue1
+; GFX10-64-NEXT:  BB6_6: ; %.continue1
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX10-64-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; GFX10-64-NEXT:    v_bfrev_b32_e32 v1, 60
@@ -883,13 +903,14 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execz BB7_3
 ; SI-NEXT:  ; %bb.1: ; %.demote0
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 BB7_9
 ; SI-NEXT:  ; %bb.2: ; %.demote0
 ; SI-NEXT:    s_wqm_b64 s[8:9], s[0:1]
 ; SI-NEXT:    s_and_b64 exec, exec, s[8:9]
-; SI-NEXT:  ; %bb.3: ; %.continue0.preheader
+; SI-NEXT:  BB7_3: ; %.continue0.preheader
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_mov_b64 s[4:5], 0
 ; SI-NEXT:    v_mov_b32_e32 v0, s6
@@ -948,13 +969,14 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX9-NEXT:    s_cbranch_execz BB7_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote0
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB7_9
 ; GFX9-NEXT:  ; %bb.2: ; %.demote0
 ; GFX9-NEXT:    s_wqm_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[6:7]
-; GFX9-NEXT:  ; %bb.3: ; %.continue0.preheader
+; GFX9-NEXT:  BB7_3: ; %.continue0.preheader
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
@@ -1013,13 +1035,14 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX10-32-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s2, vcc_lo
 ; GFX10-32-NEXT:    s_xor_b32 s2, exec_lo, s2
+; GFX10-32-NEXT:    s_cbranch_execz BB7_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote0
 ; GFX10-32-NEXT:    s_andn2_b32 s0, s0, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB7_9
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote0
 ; GFX10-32-NEXT:    s_wqm_b32 s3, s0
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s3
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue0.preheader
+; GFX10-32-NEXT:  BB7_3: ; %.continue0.preheader
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX10-32-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX10-32-NEXT:    s_branch BB7_5
@@ -1075,13 +1098,14 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX10-64-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX10-64-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX10-64-NEXT:    s_cbranch_execz BB7_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote0
 ; GFX10-64-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB7_9
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote0
 ; GFX10-64-NEXT:    s_wqm_b64 s[6:7], s[0:1]
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[6:7]
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue0.preheader
+; GFX10-64-NEXT:  BB7_3: ; %.continue0.preheader
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX10-64-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10-64-NEXT:    s_mov_b64 s[2:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
index 6d4ffa6db73bc0f59bf3f36002130e5b7e65f1c5..f3dc84b12c5ec68552a92e7395b9f71f20efb72a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
@@ -207,21 +207,28 @@ define i32 @v_urem_i32_pow2k_denom(i32 %num) {
 ; CHECK-LABEL: v_urem_i32_pow2k_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_add_i32 s4, 0x1000, -1
-; CHECK-NEXT:    v_and_b32_e32 v0, s4, v0
+; CHECK-NEXT:    v_and_b32_e32 v0, 0xfff, v0
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem i32 %num, 4096
   ret i32 %result
 }
 
 define <2 x i32> @v_urem_v2i32_pow2k_denom(<2 x i32> %num) {
-; CHECK-LABEL: v_urem_v2i32_pow2k_denom:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_add_i32 s4, 0x1000, -1
-; CHECK-NEXT:    v_and_b32_e32 v0, s4, v0
-; CHECK-NEXT:    v_and_b32_e32 v1, s4, v1
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_urem_v2i32_pow2k_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_add_i32 s4, 0x1000, -1
+; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
+; GISEL-NEXT:    v_and_b32_e32 v1, s4, v1
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i32_pow2k_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s4, 0xfff
+; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_and_b32_e32 v1, s4, v1
+; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = urem <2 x i32> %num, <i32 4096, i32 4096>
   ret <2 x i32> %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index 6219bc0f19b8b0f8bdea5a87c964299295abe182..7411807ad755fe0af1e920ed135aff1854f72817 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -949,38 +949,43 @@ define i64 @v_urem_i64_pow2k_denom(i64 %num) {
 ; CHECK-LABEL: v_urem_i64_pow2k_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_add_u32 s4, 0x1000, -1
-; CHECK-NEXT:    s_cselect_b32 s5, 1, 0
-; CHECK-NEXT:    s_and_b32 s5, s5, 1
-; CHECK-NEXT:    s_cmp_lg_u32 s5, 0
-; CHECK-NEXT:    s_addc_u32 s5, 0, -1
-; CHECK-NEXT:    v_and_b32_e32 v0, s4, v0
-; CHECK-NEXT:    v_and_b32_e32 v1, s5, v1
+; CHECK-NEXT:    v_and_b32_e32 v0, 0xfff, v0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem i64 %num, 4096
   ret i64 %result
 }
 
 define <2 x i64> @v_urem_v2i64_pow2k_denom(<2 x i64> %num) {
-; CHECK-LABEL: v_urem_v2i64_pow2k_denom:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_movk_i32 s4, 0x1000
-; CHECK-NEXT:    s_add_u32 s5, s4, -1
-; CHECK-NEXT:    s_cselect_b32 s6, 1, 0
-; CHECK-NEXT:    s_and_b32 s6, s6, 1
-; CHECK-NEXT:    s_cmp_lg_u32 s6, 0
-; CHECK-NEXT:    s_addc_u32 s6, 0, -1
-; CHECK-NEXT:    s_add_u32 s4, s4, -1
-; CHECK-NEXT:    s_cselect_b32 s7, 1, 0
-; CHECK-NEXT:    v_and_b32_e32 v0, s5, v0
-; CHECK-NEXT:    s_and_b32 s5, s7, 1
-; CHECK-NEXT:    v_and_b32_e32 v1, s6, v1
-; CHECK-NEXT:    s_cmp_lg_u32 s5, 0
-; CHECK-NEXT:    s_addc_u32 s5, 0, -1
-; CHECK-NEXT:    v_and_b32_e32 v2, s4, v2
-; CHECK-NEXT:    v_and_b32_e32 v3, s5, v3
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_urem_v2i64_pow2k_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s4, 0x1000
+; GISEL-NEXT:    s_add_u32 s5, s4, -1
+; GISEL-NEXT:    s_cselect_b32 s6, 1, 0
+; GISEL-NEXT:    s_and_b32 s6, s6, 1
+; GISEL-NEXT:    s_cmp_lg_u32 s6, 0
+; GISEL-NEXT:    s_addc_u32 s6, 0, -1
+; GISEL-NEXT:    s_add_u32 s4, s4, -1
+; GISEL-NEXT:    s_cselect_b32 s7, 1, 0
+; GISEL-NEXT:    v_and_b32_e32 v0, s5, v0
+; GISEL-NEXT:    s_and_b32 s5, s7, 1
+; GISEL-NEXT:    v_and_b32_e32 v1, s6, v1
+; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
+; GISEL-NEXT:    s_addc_u32 s5, 0, -1
+; GISEL-NEXT:    v_and_b32_e32 v2, s4, v2
+; GISEL-NEXT:    v_and_b32_e32 v3, s5, v3
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i64_pow2k_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s4, 0xfff
+; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_and_b32_e32 v2, s4, v2
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:    v_mov_b32_e32 v3, 0
+; CGP-NEXT:    s_setpc_b64 s[30:31]
   %result = urem <2 x i64> %num, <i64 4096, i64 4096>
   ret <2 x i64> %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/and.ll b/llvm/test/CodeGen/AMDGPU/and.ll
index 5c53e347c638c5dd46f38276aae4fa8942deca96..78bca7d199ef0c1b9306d39662e0676f1fdec98f 100644
--- a/llvm/test/CodeGen/AMDGPU/and.ll
+++ b/llvm/test/CodeGen/AMDGPU/and.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global,-xnack -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index bccb3f68dcee4b0eddaf6c2e8acc028f2ffb6e72..cdd4db7f8dbc63ab1c136ee28f28a4b9bada74a5 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s
+; RUN: llc -march=amdgcn -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.raw.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index a1280e1f9791eba68f008755f48a9c58ede11f37..9e06bac336303f0b556d3c9c2649cc73d6be350a 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -1,65 +1,594 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX7LESS %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX89,GFX8 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX89,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX1064 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX1032 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 
 ; Show what the atomic optimization pass will do for global pointers.
 
-; GCN-LABEL: add_i32_constant:
-; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
-; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
-; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
-; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: s_mul_i32 s[[value:[0-9]+]], s[[popcount]], 5
-; GCN: v_mov_b32_e32 v[[data:[0-9]+]], s[[value]]
-; GCN: {{flat|buffer|global}}_atomic_add v[[data]]
 define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) {
+; GFX7LESS-LABEL: add_i32_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX7LESS-NEXT:    s_cbranch_execz BB0_2
+; GFX7LESS-NEXT:  ; %bb.1:
+; GFX7LESS-NEXT:    s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT:    s_mul_i32 s6, s6, 5
+; GFX7LESS-NEXT:    s_mov_b32 s10, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s8, s2
+; GFX7LESS-NEXT:    s_mov_b32 s9, s3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB0_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX7LESS-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX89-LABEL: add_i32_constant:
+; GFX89:       ; %bb.0: ; %entry
+; GFX89-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX89-NEXT:    s_mov_b64 s[6:7], exec
+; GFX89-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX89-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX89-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX89-NEXT:    ; implicit-def: $vgpr1
+; GFX89-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX89-NEXT:    s_cbranch_execz BB0_2
+; GFX89-NEXT:  ; %bb.1:
+; GFX89-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX89-NEXT:    s_mov_b32 s8, s2
+; GFX89-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
+; GFX89-NEXT:    s_mul_i32 s2, s2, 5
+; GFX89-NEXT:    s_mov_b32 s11, 0xf000
+; GFX89-NEXT:    s_mov_b32 s10, -1
+; GFX89-NEXT:    s_mov_b32 s9, s3
+; GFX89-NEXT:    v_mov_b32_e32 v1, s2
+; GFX89-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
+; GFX89-NEXT:    s_waitcnt vmcnt(0)
+; GFX89-NEXT:    buffer_wbinvl1_vol
+; GFX89-NEXT:  BB0_2:
+; GFX89-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX89-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX89-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX89-NEXT:    s_mov_b32 s3, 0xf000
+; GFX89-NEXT:    s_mov_b32 s2, -1
+; GFX89-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
+; GFX89-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX89-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i32_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
+; GFX1064-NEXT:    ; implicit-def: $vgpr1
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB0_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT:    s_mul_i32 s6, s6, 5
+; GFX1064-NEXT:    s_mov_b32 s10, -1
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s6
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mov_b32 s8, s2
+; GFX1064-NEXT:    s_mov_b32 s9, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB0_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i32_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
+; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB0_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s5, s5
+; GFX1032-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT:    s_mul_i32 s5, s5, 5
+; GFX1032-NEXT:    s_mov_b32 s10, -1
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s5
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mov_b32 s8, s2
+; GFX1032-NEXT:    s_mov_b32 s9, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB0_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i32 addrspace(1)* %inout, i32 5 acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: add_i32_uniform:
-; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
-; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
-; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
-; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GCN: {{flat|buffer|global}}_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 addrspace(1)* %inout, i32 %additive) {
+; GFX7LESS-LABEL: add_i32_uniform:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GFX7LESS-NEXT:    s_cbranch_execz BB1_2
+; GFX7LESS-NEXT:  ; %bb.1:
+; GFX7LESS-NEXT:    s_mov_b32 s15, 0xf000
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mul_i32 s1, s0, s1
+; GFX7LESS-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-NEXT:    s_mov_b32 s12, s6
+; GFX7LESS-NEXT:    s_mov_b32 s13, s7
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB1_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX7LESS-NEXT:    v_add_i32_e32 v0, vcc, s1, v0
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i32_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dword s0, s[0:1], 0x34
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GFX8-NEXT:    s_cbranch_execz BB1_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mul_i32 s1, s0, s1
+; GFX8-NEXT:    s_mov_b32 s15, 0xf000
+; GFX8-NEXT:    s_mov_b32 s14, -1
+; GFX8-NEXT:    s_mov_b32 s12, s6
+; GFX8-NEXT:    s_mov_b32 s13, s7
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB1_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
+; GFX8-NEXT:    s_mov_b32 s6, -1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i32_uniform:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x34
+; GFX9-NEXT:    s_mov_b64 s[8:9], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s9, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT:    s_cbranch_execz BB1_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s3, s[8:9]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mul_i32 s3, s2, s3
+; GFX9-NEXT:    s_mov_b32 s15, 0xf000
+; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    s_mov_b32 s12, s6
+; GFX9-NEXT:    s_mov_b32 s13, s7
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB1_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i32_uniform:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_clause 0x1
+; GFX1064-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX1064-NEXT:    s_load_dword s2, s[0:1], 0x34
+; GFX1064-NEXT:    s_mov_b64 s[8:9], exec
+; GFX1064-NEXT:    ; implicit-def: $vgpr1
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s8, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s9, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB1_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s3, s[8:9]
+; GFX1064-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mul_i32 s3, s2, s3
+; GFX1064-NEXT:    s_mov_b32 s10, -1
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s3
+; GFX1064-NEXT:    s_mov_b32 s8, s6
+; GFX1064-NEXT:    s_mov_b32 s9, s7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB1_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s6, -1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i32_uniform:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_clause 0x1
+; GFX1032-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX1032-NEXT:    s_load_dword s2, s[0:1], 0x34
+; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB1_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s1, s3
+; GFX1032-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mul_i32 s1, s2, s1
+; GFX1032-NEXT:    s_mov_b32 s10, -1
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1032-NEXT:    s_mov_b32 s8, s6
+; GFX1032-NEXT:    s_mov_b32 s9, s7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB1_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s6, -1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i32 addrspace(1)* %inout, i32 %additive acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: add_i32_varying:
-; GFX7LESS-NOT: v_mbcnt_lo_u32_b32
-; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
-; GFX7LESS-NOT: s_bcnt1_i32_b64
-; GFX7LESS: buffer_atomic_add v{{[0-9]+}}
-; DPPCOMB: v_add_u32_dpp
-; DPPCOMB: v_add_u32_dpp
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
-; GFX89: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX10: s_mov_b32 s[[copy_value:[0-9]+]], s[[scalar_value]]
-; GFX10: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[copy_value]]
-; GFX8MORE: buffer_atomic_add v[[value]]
 define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) {
+; GFX7LESS-LABEL: add_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    s_mov_b32 s10, s6
+; GFX7LESS-NEXT:    s_mov_b32 s11, s7
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s8, s2
+; GFX7LESS-NEXT:    s_mov_b32 s9, s3
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_add v0, off, s[8:11], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s4, s0
+; GFX7LESS-NEXT:    s_mov_b32 s5, s1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s6, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    s_cbranch_execz BB2_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_mov_b32 s11, 0xf000
+; GFX8-NEXT:    s_mov_b32 s10, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s8, s2
+; GFX8-NEXT:    s_mov_b32 s9, s3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_atomic_add v0, off, s[8:11], 0 glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB2_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s4, v0
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s6, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_cbranch_execz BB2_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    s_mov_b32 s11, 0xf000
+; GFX9-NEXT:    s_mov_b32 s10, -1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s8, s2
+; GFX9-NEXT:    s_mov_b32 s9, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_add v0, off, s[8:11], 0 glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB2_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    v_add_u32_e32 v0, s4, v0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s6, v1, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_readlane_b32 s7, v1, 31
+; GFX1064-NEXT:    v_writelane_b32 v3, s6, 16
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_readlane_b32 s8, v1, 47
+; GFX1064-NEXT:    v_readlane_b32 s9, v1, 63
+; GFX1064-NEXT:    v_writelane_b32 v3, s7, 32
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GFX1064-NEXT:    s_mov_b32 s4, s9
+; GFX1064-NEXT:    v_writelane_b32 v3, s8, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[6:7]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_mov_b32 s6, -1
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB2_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, s4
+; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mov_b32 s4, s2
+; GFX1064-NEXT:    s_mov_b32 s5, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_add v0, off, s[4:7], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB2_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s2, v0
+; GFX1064-NEXT:    s_mov_b32 s2, s6
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-NEXT:    v_readlane_b32 s5, v1, 15
+; GFX1032-NEXT:    v_readlane_b32 s6, v1, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_writelane_b32 v3, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_mov_b32 s4, s6
+; GFX1032-NEXT:    s_mov_b32 s6, -1
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s8, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB2_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, s4
+; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mov_b32 s4, s2
+; GFX1032-NEXT:    s_mov_b32 s5, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_add v0, off, s[4:7], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB2_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s2, v0
+; GFX1032-NEXT:    s_mov_b32 s2, s6
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw add i32 addrspace(1)* %inout, i32 %lane acq_rel
@@ -67,49 +596,464 @@ entry:
   ret void
 }
 
-; GCN-LABEL: add_i64_constant:
-; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
-; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
-; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
-; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN-DAG: s_mul_i32 s[[value:[0-9]+]], s[[popcount]], 5
-; GCN-DAG: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5
-; GCN: v_mov_b32_e32 v[[value_lo:[0-9]+]], s[[value]]
-; GCN: {{flat|buffer|global}}_atomic_add_x2 v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
 define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out, i64 addrspace(1)* %inout) {
+; GFX7LESS-LABEL: add_i64_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX7LESS-NEXT:    s_cbranch_execz BB3_2
+; GFX7LESS-NEXT:  ; %bb.1:
+; GFX7LESS-NEXT:    s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT:    s_mul_i32 s6, s6, 5
+; GFX7LESS-NEXT:    s_mov_b32 s10, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s8, s2
+; GFX7LESS-NEXT:    s_mov_b32 s9, s3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB3_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
+; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
+; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s5
+; GFX7LESS-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
+; GFX7LESS-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX89-LABEL: add_i64_constant:
+; GFX89:       ; %bb.0: ; %entry
+; GFX89-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX89-NEXT:    s_mov_b64 s[6:7], exec
+; GFX89-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX89-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX89-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX89-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX89-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX89-NEXT:    s_cbranch_execz BB3_2
+; GFX89-NEXT:  ; %bb.1:
+; GFX89-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX89-NEXT:    s_mov_b32 s8, s2
+; GFX89-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
+; GFX89-NEXT:    s_mul_i32 s2, s2, 5
+; GFX89-NEXT:    s_mov_b32 s11, 0xf000
+; GFX89-NEXT:    s_mov_b32 s10, -1
+; GFX89-NEXT:    s_mov_b32 s9, s3
+; GFX89-NEXT:    v_mov_b32_e32 v1, s2
+; GFX89-NEXT:    v_mov_b32_e32 v2, 0
+; GFX89-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
+; GFX89-NEXT:    s_waitcnt vmcnt(0)
+; GFX89-NEXT:    buffer_wbinvl1_vol
+; GFX89-NEXT:  BB3_2:
+; GFX89-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX89-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX89-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX89-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX89-NEXT:    v_mov_b32_e32 v1, s2
+; GFX89-NEXT:    v_mov_b32_e32 v2, s3
+; GFX89-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v0, 5, v[1:2]
+; GFX89-NEXT:    s_mov_b32 s3, 0xf000
+; GFX89-NEXT:    s_mov_b32 s2, -1
+; GFX89-NEXT:    s_nop 2
+; GFX89-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX89-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i64_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
+; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB3_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_mul_i32 s6, s6, 5
+; GFX1064-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s6
+; GFX1064-NEXT:    s_mov_b32 s10, -1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mov_b32 s8, s2
+; GFX1064-NEXT:    s_mov_b32 s9, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB3_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX1064-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], v0, 5, s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i64_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
+; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB3_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s5, s5
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_mul_i32 s5, s5, 5
+; GFX1032-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s5
+; GFX1032-NEXT:    s_mov_b32 s10, -1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mov_b32 s8, s2
+; GFX1032-NEXT:    s_mov_b32 s9, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB3_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX1032-NEXT:    v_mad_u64_u32 v[0:1], s2, v0, 5, s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(1)* %inout, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: add_i64_uniform:
-; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
-; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
-; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
-; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: {{flat|buffer|global}}_atomic_add_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 addrspace(1)* %inout, i64 %additive) {
+; GFX7LESS-LABEL: add_i64_uniform:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[8:9], exec
+; GFX7LESS-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s8, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s9, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    s_cbranch_execz BB4_2
+; GFX7LESS-NEXT:  ; %bb.1:
+; GFX7LESS-NEXT:    s_mov_b32 s15, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s12, s6
+; GFX7LESS-NEXT:    s_mov_b32 s13, s7
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[8:9]
+; GFX7LESS-NEXT:    s_mul_i32 s7, s1, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    v_mul_hi_u32 v1, s0, v1
+; GFX7LESS-NEXT:    s_mul_i32 s6, s0, s6
+; GFX7LESS-NEXT:    v_add_i32_e32 v2, vcc, s7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[12:15], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB4_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
+; GFX7LESS-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX7LESS-NEXT:    v_mul_hi_u32 v2, s0, v0
+; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX7LESS-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX7LESS-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: add_i64_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_mov_b64 s[8:9], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s9, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    s_cbranch_execz BB4_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s12, s6
+; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[8:9]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_mul_hi_u32 v1, s0, v1
+; GFX8-NEXT:    s_mov_b32 s13, s7
+; GFX8-NEXT:    s_mul_i32 s7, s1, s6
+; GFX8-NEXT:    s_mul_i32 s6, s0, s6
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s7, v1
+; GFX8-NEXT:    s_mov_b32 s15, 0xf000
+; GFX8-NEXT:    s_mov_b32 s14, -1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[12:15], 0 glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB4_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX8-NEXT:    v_mul_hi_u32 v3, s0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
+; GFX8-NEXT:    s_mov_b32 s6, -1
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: add_i64_uniform:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT:    s_mov_b64 s[8:9], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s9, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT:    s_cbranch_execz BB4_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s12, s6
+; GFX9-NEXT:    s_bcnt1_i32_b64 s6, s[8:9]
+; GFX9-NEXT:    s_mov_b32 s13, s7
+; GFX9-NEXT:    s_mul_i32 s7, s3, s6
+; GFX9-NEXT:    s_mul_hi_u32 s8, s2, s6
+; GFX9-NEXT:    s_add_i32 s8, s8, s7
+; GFX9-NEXT:    s_mul_i32 s6, s2, s6
+; GFX9-NEXT:    s_mov_b32 s15, 0xf000
+; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_mov_b32_e32 v2, s8
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[12:15], 0 glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB4_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: add_i64_uniform:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_clause 0x1
+; GFX1064-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX1064-NEXT:    s_mov_b64 s[8:9], exec
+; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s8, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s9, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB4_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s8, s[8:9]
+; GFX1064-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mul_i32 s9, s3, s8
+; GFX1064-NEXT:    s_mul_hi_u32 s10, s2, s8
+; GFX1064-NEXT:    s_mul_i32 s8, s2, s8
+; GFX1064-NEXT:    s_add_i32 s10, s10, s9
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s8
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s10
+; GFX1064-NEXT:    s_mov_b32 s10, -1
+; GFX1064-NEXT:    s_mov_b32 s8, s6
+; GFX1064-NEXT:    s_mov_b32 s9, s7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB4_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX1064-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1064-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s6, -1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1064-NEXT:    v_add_co_u32_e64 v0, vcc, s0, v0
+; GFX1064-NEXT:    v_add_co_ci_u32_e32 v1, vcc, s1, v1, vcc
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i64_uniform:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_clause 0x1
+; GFX1032-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX1032-NEXT:    s_mov_b32 s8, exec_lo
+; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s8, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB4_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s1, s8
+; GFX1032-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mul_i32 s8, s3, s1
+; GFX1032-NEXT:    s_mul_hi_u32 s9, s2, s1
+; GFX1032-NEXT:    s_mul_i32 s1, s2, s1
+; GFX1032-NEXT:    s_add_i32 s9, s9, s8
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s9
+; GFX1032-NEXT:    s_mov_b32 s10, -1
+; GFX1032-NEXT:    s_mov_b32 s8, s6
+; GFX1032-NEXT:    s_mov_b32 s9, s7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_add_x2 v[1:2], off, s[8:11], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB4_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX1032-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1032-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s6, -1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1032-NEXT:    v_add_co_u32_e64 v0, vcc_lo, s0, v0
+; GFX1032-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw add i64 addrspace(1)* %inout, i64 %additive acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: add_i64_varying:
-; GCN-NOT: v_mbcnt_lo_u32_b32
-; GCN-NOT: v_mbcnt_hi_u32_b32
-; GCN-NOT: s_bcnt1_i32_b64
-; GCN: {{flat|buffer|global}}_atomic_add_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @add_i64_varying(i64 addrspace(1)* %out, i64 addrspace(1)* %inout) {
+; GFX7LESS-LABEL: add_i64_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT:    s_mov_b32 s10, s6
+; GFX7LESS-NEXT:    s_mov_b32 s11, s7
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s8, s2
+; GFX7LESS-NEXT:    s_mov_b32 s9, s3
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s4, s0
+; GFX7LESS-NEXT:    s_mov_b32 s5, s1
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX89-LABEL: add_i64_varying:
+; GFX89:       ; %bb.0: ; %entry
+; GFX89-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT:    s_mov_b32 s3, 0xf000
+; GFX89-NEXT:    s_mov_b32 s2, -1
+; GFX89-NEXT:    v_mov_b32_e32 v1, 0
+; GFX89-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX89-NEXT:    s_mov_b32 s0, s4
+; GFX89-NEXT:    s_mov_b32 s1, s5
+; GFX89-NEXT:    s_mov_b32 s4, s6
+; GFX89-NEXT:    s_mov_b32 s5, s7
+; GFX89-NEXT:    s_mov_b32 s6, s2
+; GFX89-NEXT:    s_mov_b32 s7, s3
+; GFX89-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    buffer_atomic_add_x2 v[0:1], off, s[4:7], 0 glc
+; GFX89-NEXT:    s_waitcnt vmcnt(0)
+; GFX89-NEXT:    buffer_wbinvl1_vol
+; GFX89-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX89-NEXT:    s_endpgm
+;
+; GFX10-LABEL: add_i64_varying:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX10-NEXT:    s_mov_b32 s6, -1
+; GFX10-NEXT:    s_mov_b32 s11, s7
+; GFX10-NEXT:    s_mov_b32 s10, s6
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s8, s2
+; GFX10-NEXT:    s_mov_b32 s9, s3
+; GFX10-NEXT:    s_mov_b32 s4, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    buffer_gl0_inv
+; GFX10-NEXT:    buffer_gl1_inv
+; GFX10-NEXT:    s_mov_b32 s5, s1
+; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX10-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %zext = zext i32 %lane to i64
@@ -118,58 +1062,624 @@ entry:
   ret void
 }
 
-; GCN-LABEL: sub_i32_constant:
-; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
-; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
-; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
-; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: s_mul_i32 s[[value:[0-9]+]], s[[popcount]], 5
-; GCN: v_mov_b32_e32 v[[data:[0-9]+]], s[[value]]
-; GCN: {{flat|buffer|global}}_atomic_sub v[[data]]
 define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) {
+; GFX7LESS-LABEL: sub_i32_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX7LESS-NEXT:    s_cbranch_execz BB6_2
+; GFX7LESS-NEXT:  ; %bb.1:
+; GFX7LESS-NEXT:    s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT:    s_mul_i32 s6, s6, 5
+; GFX7LESS-NEXT:    s_mov_b32 s10, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s8, s2
+; GFX7LESS-NEXT:    s_mov_b32 s9, s3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB6_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i32_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b64 s[6:7], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    s_cbranch_execz BB6_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s8, s2
+; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
+; GFX8-NEXT:    s_mul_i32 s2, s2, 5
+; GFX8-NEXT:    s_mov_b32 s11, 0xf000
+; GFX8-NEXT:    s_mov_b32 s10, -1
+; GFX8-NEXT:    s_mov_b32 s9, s3
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB6_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX8-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s4, v0
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i32_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[6:7], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_cbranch_execz BB6_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s8, s2
+; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
+; GFX9-NEXT:    s_mul_i32 s2, s2, 5
+; GFX9-NEXT:    s_mov_b32 s11, 0xf000
+; GFX9-NEXT:    s_mov_b32 s10, -1
+; GFX9-NEXT:    s_mov_b32 s9, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB6_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX9-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i32_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
+; GFX1064-NEXT:    ; implicit-def: $vgpr1
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB6_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT:    s_mul_i32 s6, s6, 5
+; GFX1064-NEXT:    s_mov_b32 s10, -1
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s6
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mov_b32 s8, s2
+; GFX1064-NEXT:    s_mov_b32 s9, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB6_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i32_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
+; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB6_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s5, s5
+; GFX1032-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT:    s_mul_i32 s5, s5, 5
+; GFX1032-NEXT:    s_mov_b32 s10, -1
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s5
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mov_b32 s8, s2
+; GFX1032-NEXT:    s_mov_b32 s9, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB6_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i32 addrspace(1)* %inout, i32 5 acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: sub_i32_uniform:
-; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
-; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
-; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
-; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
-; GCN: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GCN: {{flat|buffer|global}}_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 addrspace(1)* %inout, i32 %subitive) {
+; GFX7LESS-LABEL: sub_i32_uniform:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_load_dword s0, s[0:1], 0xd
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GFX7LESS-NEXT:    s_cbranch_execz BB7_2
+; GFX7LESS-NEXT:  ; %bb.1:
+; GFX7LESS-NEXT:    s_mov_b32 s15, 0xf000
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mul_i32 s1, s0, s1
+; GFX7LESS-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-NEXT:    s_mov_b32 s12, s6
+; GFX7LESS-NEXT:    s_mov_b32 s13, s7
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_sub v1, off, s[12:15], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB7_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s1, v0
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i32_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dword s0, s[0:1], 0x34
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1
+; GFX8-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GFX8-NEXT:    s_cbranch_execz BB7_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mul_i32 s1, s0, s1
+; GFX8-NEXT:    s_mov_b32 s15, 0xf000
+; GFX8-NEXT:    s_mov_b32 s14, -1
+; GFX8-NEXT:    s_mov_b32 s12, s6
+; GFX8-NEXT:    s_mov_b32 s13, s7
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_atomic_sub v1, off, s[12:15], 0 glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB7_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
+; GFX8-NEXT:    s_mov_b32 s6, -1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i32_uniform:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x34
+; GFX9-NEXT:    s_mov_b64 s[8:9], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s9, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1
+; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT:    s_cbranch_execz BB7_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    s_bcnt1_i32_b64 s3, s[8:9]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mul_i32 s3, s2, s3
+; GFX9-NEXT:    s_mov_b32 s15, 0xf000
+; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    s_mov_b32 s12, s6
+; GFX9-NEXT:    s_mov_b32 s13, s7
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_sub v1, off, s[12:15], 0 glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB7_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    v_sub_u32_e32 v0, s0, v0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i32_uniform:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_clause 0x1
+; GFX1064-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX1064-NEXT:    s_load_dword s2, s[0:1], 0x34
+; GFX1064-NEXT:    s_mov_b64 s[8:9], exec
+; GFX1064-NEXT:    ; implicit-def: $vgpr1
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s8, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s9, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB7_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s3, s[8:9]
+; GFX1064-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mul_i32 s3, s2, s3
+; GFX1064-NEXT:    s_mov_b32 s10, -1
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s3
+; GFX1064-NEXT:    s_mov_b32 s8, s6
+; GFX1064-NEXT:    s_mov_b32 s9, s7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB7_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s6, -1
+; GFX1064-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i32_uniform:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_clause 0x1
+; GFX1032-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX1032-NEXT:    s_load_dword s2, s[0:1], 0x34
+; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB7_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s1, s3
+; GFX1032-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mul_i32 s1, s2, s1
+; GFX1032-NEXT:    s_mov_b32 s10, -1
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1032-NEXT:    s_mov_b32 s8, s6
+; GFX1032-NEXT:    s_mov_b32 s9, s7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB7_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s6, -1
+; GFX1032-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i32 addrspace(1)* %inout, i32 %subitive acq_rel
   store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: sub_i32_varying:
-; GFX7LESS-NOT: v_mbcnt_lo_u32_b32
-; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
-; GFX7LESS-NOT: s_bcnt1_i32_b64
-; GFX7LESS: buffer_atomic_sub v{{[0-9]+}}
-; DPPCOMB: v_add_u32_dpp
-; DPPCOMB: v_add_u32_dpp
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE64: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
-; GFX89: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX10: s_mov_b32 s[[copy_value:[0-9]+]], s[[scalar_value]]
-; GFX10: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[copy_value]]
-; GFX8MORE: buffer_atomic_sub v[[value]]
 define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out, i32 addrspace(1)* %inout) {
+; GFX7LESS-LABEL: sub_i32_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    s_mov_b32 s10, s6
+; GFX7LESS-NEXT:    s_mov_b32 s11, s7
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s8, s2
+; GFX7LESS-NEXT:    s_mov_b32 s9, s3
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s4, s0
+; GFX7LESS-NEXT:    s_mov_b32 s5, s1
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i32_varying:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s6, v2, 63
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr0
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    s_cbranch_execz BB8_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_mov_b32 s11, 0xf000
+; GFX8-NEXT:    s_mov_b32 s10, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s8, s2
+; GFX8-NEXT:    s_mov_b32 s9, s3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB8_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s4, v0
+; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i32_varying:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s6, v2, 63
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr0
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_cbranch_execz BB8_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    s_mov_b32 s11, 0xf000
+; GFX9-NEXT:    s_mov_b32 s10, -1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s8, s2
+; GFX9-NEXT:    s_mov_b32 s9, s3
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB8_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i32_varying:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1064-NEXT:    v_readlane_b32 s6, v1, 15
+; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_readlane_b32 s7, v1, 31
+; GFX1064-NEXT:    v_writelane_b32 v3, s6, 16
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GFX1064-NEXT:    v_readlane_b32 s8, v1, 47
+; GFX1064-NEXT:    v_readlane_b32 s9, v1, 63
+; GFX1064-NEXT:    v_writelane_b32 v3, s7, 32
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GFX1064-NEXT:    s_mov_b32 s4, s9
+; GFX1064-NEXT:    v_writelane_b32 v3, s8, 48
+; GFX1064-NEXT:    s_mov_b64 exec, s[6:7]
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_mov_b32 s6, -1
+; GFX1064-NEXT:    ; implicit-def: $vgpr0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[8:9], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB8_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, s4
+; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mov_b32 s4, s2
+; GFX1064-NEXT:    s_mov_b32 s5, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_sub v0, off, s[4:7], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB8_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX1064-NEXT:    s_mov_b32 s2, s6
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i32_varying:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
+; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-NEXT:    v_readlane_b32 s5, v1, 15
+; GFX1032-NEXT:    v_readlane_b32 s6, v1, 31
+; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX1032-NEXT:    s_or_saveexec_b32 s4, -1
+; GFX1032-NEXT:    v_writelane_b32 v3, s5, 16
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s4
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_mov_b32 s4, s6
+; GFX1032-NEXT:    s_mov_b32 s6, -1
+; GFX1032-NEXT:    ; implicit-def: $vgpr0
+; GFX1032-NEXT:    s_and_saveexec_b32 s8, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB8_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    v_mov_b32_e32 v0, s4
+; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mov_b32 s4, s2
+; GFX1032-NEXT:    s_mov_b32 s5, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_sub v0, off, s[4:7], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB8_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
+; GFX1032-NEXT:    s_mov_b32 s2, s6
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw sub i32 addrspace(1)* %inout, i32 %lane acq_rel
@@ -177,49 +1687,510 @@ entry:
   ret void
 }
 
-; GCN-LABEL: sub_i64_constant:
-; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
-; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
-; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
-; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN-DAG: s_mul_i32 s[[value:[0-9]+]], s[[popcount]], 5
-; GCN-DAG: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5
-; GCN: v_mov_b32_e32 v[[value_lo:[0-9]+]], s[[value]]
-; GCN: {{flat|buffer|global}}_atomic_sub_x2 v{{\[}}[[value_lo]]:[[value_hi]]{{\]}}
 define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out, i64 addrspace(1)* %inout) {
+; GFX7LESS-LABEL: sub_i64_constant:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX7LESS-NEXT:    s_cbranch_execz BB9_2
+; GFX7LESS-NEXT:  ; %bb.1:
+; GFX7LESS-NEXT:    s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT:    s_mul_i32 s6, s6, 5
+; GFX7LESS-NEXT:    s_mov_b32 s10, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s8, s2
+; GFX7LESS-NEXT:    s_mov_b32 s9, s3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB9_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
+; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
+; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s5
+; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GFX7LESS-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i64_constant:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b64 s[6:7], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT:    s_cbranch_execz BB9_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s8, s2
+; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
+; GFX8-NEXT:    s_mul_i32 s2, s2, 5
+; GFX8-NEXT:    s_mov_b32 s11, 0xf000
+; GFX8-NEXT:    s_mov_b32 s10, -1
+; GFX8-NEXT:    s_mov_b32 s9, s3
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB9_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX8-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
+; GFX8-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s5
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s4, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s3, 0xf000
+; GFX8-NEXT:    s_mov_b32 s2, -1
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i64_constant:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[6:7], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_cbranch_execz BB9_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s8, s2
+; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[6:7]
+; GFX9-NEXT:    s_mul_i32 s2, s2, 5
+; GFX9-NEXT:    s_mov_b32 s11, 0xf000
+; GFX9-NEXT:    s_mov_b32 s10, -1
+; GFX9-NEXT:    s_mov_b32 s9, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB9_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX9-NEXT:    v_mul_hi_u32_u24_e32 v1, 5, v0
+; GFX9-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s4, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s3, 0xf000
+; GFX9-NEXT:    s_mov_b32 s2, -1
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i64_constant:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
+; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB9_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_mul_i32 s6, s6, 5
+; GFX1064-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s6
+; GFX1064-NEXT:    s_mov_b32 s10, -1
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mov_b32 s8, s2
+; GFX1064-NEXT:    s_mov_b32 s9, s3
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB9_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT:    v_mul_u32_u24_e32 v1, 5, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX1064-NEXT:    v_mul_hi_u32_u24_e32 v2, 5, v0
+; GFX1064-NEXT:    v_sub_co_u32_e64 v0, vcc, s2, v1
+; GFX1064-NEXT:    s_mov_b32 s2, -1
+; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s3, v2, vcc
+; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i64_constant:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
+; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB9_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s5, s5
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_mul_i32 s5, s5, 5
+; GFX1032-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s5
+; GFX1032-NEXT:    s_mov_b32 s10, -1
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mov_b32 s8, s2
+; GFX1032-NEXT:    s_mov_b32 s9, s3
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB9_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT:    v_mul_u32_u24_e32 v1, 5, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX1032-NEXT:    v_mul_hi_u32_u24_e32 v2, 5, v0
+; GFX1032-NEXT:    v_sub_co_u32_e64 v0, vcc_lo, s2, v1
+; GFX1032-NEXT:    s_mov_b32 s2, -1
+; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v2, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i64 addrspace(1)* %inout, i64 5 acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: sub_i64_uniform:
-; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
-; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
-; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
-; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
-; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
-; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
-; GCN: {{flat|buffer|global}}_atomic_sub_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 addrspace(1)* %inout, i64 %subitive) {
+; GFX7LESS-LABEL: sub_i64_uniform:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[8:9], exec
+; GFX7LESS-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s8, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s9, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    s_cbranch_execz BB10_2
+; GFX7LESS-NEXT:  ; %bb.1:
+; GFX7LESS-NEXT:    s_mov_b32 s15, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s12, s6
+; GFX7LESS-NEXT:    s_mov_b32 s13, s7
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[8:9]
+; GFX7LESS-NEXT:    s_mul_i32 s7, s1, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    v_mul_hi_u32 v1, s0, v1
+; GFX7LESS-NEXT:    s_mul_i32 s6, s0, s6
+; GFX7LESS-NEXT:    v_add_i32_e32 v2, vcc, s7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[12:15], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:  BB10_2:
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX7LESS-NEXT:    v_readfirstlane_b32 s3, v2
+; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
+; GFX7LESS-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX7LESS-NEXT:    v_mul_hi_u32 v2, s0, v0
+; GFX7LESS-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX7LESS-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; GFX7LESS-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i64_uniform:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_mov_b64 s[8:9], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s9, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    s_cbranch_execz BB10_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s12, s6
+; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[8:9]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_mul_hi_u32 v1, s0, v1
+; GFX8-NEXT:    s_mov_b32 s13, s7
+; GFX8-NEXT:    s_mul_i32 s7, s1, s6
+; GFX8-NEXT:    s_mul_i32 s6, s0, s6
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s7, v1
+; GFX8-NEXT:    s_mov_b32 s15, 0xf000
+; GFX8-NEXT:    s_mov_b32 s14, -1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[12:15], 0 glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:  BB10_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mul_lo_u32 v1, s1, v0
+; GFX8-NEXT:    v_mul_hi_u32 v3, s0, v0
+; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v3, v1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
+; GFX8-NEXT:    s_mov_b32 s6, -1
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i64_uniform:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT:    s_mov_b64 s[8:9], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s9, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT:    s_cbranch_execz BB10_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s12, s6
+; GFX9-NEXT:    s_bcnt1_i32_b64 s6, s[8:9]
+; GFX9-NEXT:    s_mov_b32 s13, s7
+; GFX9-NEXT:    s_mul_i32 s7, s3, s6
+; GFX9-NEXT:    s_mul_hi_u32 s8, s2, s6
+; GFX9-NEXT:    s_add_i32 s8, s8, s7
+; GFX9-NEXT:    s_mul_i32 s6, s2, s6
+; GFX9-NEXT:    s_mov_b32 s15, 0xf000
+; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_mov_b32_e32 v2, s8
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[12:15], 0 glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:  BB10_2:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX9-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v2, v1, vcc
+; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i64_uniform:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    s_clause 0x1
+; GFX1064-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX1064-NEXT:    s_mov_b64 s[8:9], exec
+; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s8, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s9, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB10_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s8, s[8:9]
+; GFX1064-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    s_mul_i32 s9, s3, s8
+; GFX1064-NEXT:    s_mul_hi_u32 s10, s2, s8
+; GFX1064-NEXT:    s_mul_i32 s8, s2, s8
+; GFX1064-NEXT:    s_add_i32 s10, s10, s9
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s8
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s10
+; GFX1064-NEXT:    s_mov_b32 s10, -1
+; GFX1064-NEXT:    s_mov_b32 s8, s6
+; GFX1064-NEXT:    s_mov_b32 s9, s7
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:    buffer_gl1_inv
+; GFX1064-NEXT:  BB10_2:
+; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX1064-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX1064-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1064-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX1064-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT:    s_mov_b32 s6, -1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1064-NEXT:    v_sub_co_u32_e64 v0, vcc, s0, v0
+; GFX1064-NEXT:    v_sub_co_ci_u32_e32 v1, vcc, s1, v1, vcc
+; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i64_uniform:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    s_clause 0x1
+; GFX1032-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX1032-NEXT:    s_mov_b32 s8, exec_lo
+; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s8, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB10_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s1, s8
+; GFX1032-NEXT:    s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    s_mul_i32 s8, s3, s1
+; GFX1032-NEXT:    s_mul_hi_u32 s9, s2, s1
+; GFX1032-NEXT:    s_mul_i32 s1, s2, s1
+; GFX1032-NEXT:    s_add_i32 s9, s9, s8
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s9
+; GFX1032-NEXT:    s_mov_b32 s10, -1
+; GFX1032-NEXT:    s_mov_b32 s8, s6
+; GFX1032-NEXT:    s_mov_b32 s9, s7
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    buffer_atomic_sub_x2 v[1:2], off, s[8:11], 0 glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:    buffer_gl1_inv
+; GFX1032-NEXT:  BB10_2:
+; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    v_mul_lo_u32 v3, s3, v0
+; GFX1032-NEXT:    v_mul_hi_u32 v4, s2, v0
+; GFX1032-NEXT:    v_mul_lo_u32 v0, s2, v0
+; GFX1032-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX1032-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX1032-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT:    s_mov_b32 s6, -1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v4, v3
+; GFX1032-NEXT:    v_sub_co_u32_e64 v0, vcc_lo, s0, v0
+; GFX1032-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
+; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX1032-NEXT:    s_endpgm
 entry:
   %old = atomicrmw sub i64 addrspace(1)* %inout, i64 %subitive acq_rel
   store i64 %old, i64 addrspace(1)* %out
   ret void
 }
 
-; GCN-LABEL: sub_i64_varying:
-; GCN-NOT: v_mbcnt_lo_u32_b32
-; GCN-NOT: v_mbcnt_hi_u32_b32
-; GCN-NOT: s_bcnt1_i32_b64
-; GCN: {{flat|buffer|global}}_atomic_sub_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @sub_i64_varying(i64 addrspace(1)* %out, i64 addrspace(1)* %inout) {
+; GFX7LESS-LABEL: sub_i64_varying:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s6, -1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT:    s_mov_b32 s10, s6
+; GFX7LESS-NEXT:    s_mov_b32 s11, s7
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_mov_b32 s8, s2
+; GFX7LESS-NEXT:    s_mov_b32 s9, s3
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    buffer_wbinvl1
+; GFX7LESS-NEXT:    s_mov_b32 s4, s0
+; GFX7LESS-NEXT:    s_mov_b32 s5, s1
+; GFX7LESS-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX89-LABEL: sub_i64_varying:
+; GFX89:       ; %bb.0: ; %entry
+; GFX89-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX89-NEXT:    s_mov_b32 s3, 0xf000
+; GFX89-NEXT:    s_mov_b32 s2, -1
+; GFX89-NEXT:    v_mov_b32_e32 v1, 0
+; GFX89-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX89-NEXT:    s_mov_b32 s0, s4
+; GFX89-NEXT:    s_mov_b32 s1, s5
+; GFX89-NEXT:    s_mov_b32 s4, s6
+; GFX89-NEXT:    s_mov_b32 s5, s7
+; GFX89-NEXT:    s_mov_b32 s6, s2
+; GFX89-NEXT:    s_mov_b32 s7, s3
+; GFX89-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[4:7], 0 glc
+; GFX89-NEXT:    s_waitcnt vmcnt(0)
+; GFX89-NEXT:    buffer_wbinvl1_vol
+; GFX89-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX89-NEXT:    s_endpgm
+;
+; GFX10-LABEL: sub_i64_varying:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX10-NEXT:    s_mov_b32 s6, -1
+; GFX10-NEXT:    s_mov_b32 s11, s7
+; GFX10-NEXT:    s_mov_b32 s10, s6
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s8, s2
+; GFX10-NEXT:    s_mov_b32 s9, s3
+; GFX10-NEXT:    s_mov_b32 s4, s0
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    buffer_gl0_inv
+; GFX10-NEXT:    buffer_gl1_inv
+; GFX10-NEXT:    s_mov_b32 s5, s1
+; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX10-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %zext = zext i32 %lane to i64
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 1753cdcf407be3bfc8419720f4edcba5d2a42ee0..a3166e46501cffef55cd1c22620ee46e00b1f9a5 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1032 %s
+; RUN: llc -march=amdgcn -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX7LESS %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX1064 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX1032 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 
@@ -575,471 +575,155 @@ entry:
   ret void
 }
 
-define amdgpu_kernel void @add_i32_varying_gfx1032(i32 addrspace(1)* %out) {
-;
-;
-; GFX7LESS-LABEL: add_i32_varying_gfx1032:
+define amdgpu_kernel void @add_i32_varying_nouse() {
+; GFX7LESS-LABEL: add_i32_varying_nouse:
 ; GFX7LESS:       ; %bb.0: ; %entry
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
 ; GFX7LESS-NEXT:    s_mov_b32 m0, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    ds_add_rtn_u32 v0, v1, v0
+; GFX7LESS-NEXT:    ds_add_u32 v1, v0
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s2, -1
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7LESS-NEXT:    s_endpgm
 ;
-; GFX8-LABEL: add_i32_varying_gfx1032:
+; GFX8-LABEL: add_i32_varying_nouse:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_mov_b32_e32 v2, v0
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX8-NEXT:    s_not_b64 exec, exec
-; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX8-NEXT:    s_not_b64 exec, exec
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
 ; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
-; GFX8-NEXT:    s_nop 0
-; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v1, 63
+; GFX8-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX8-NEXT:    s_mov_b32 s0, s2
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8-NEXT:    ; implicit-def: $vgpr0
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX8-NEXT:    s_cbranch_execz BB3_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    ds_add_rtn_u32 v0, v0, v3
+; GFX8-NEXT:    ds_add_u32 v0, v2
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:  BB3_2:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX8-NEXT:    v_mov_b32_e32 v0, v1
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
-; GFX8-NEXT:    s_mov_b32 s3, 0xf000
-; GFX8-NEXT:    s_mov_b32 s2, -1
-; GFX8-NEXT:    s_nop 0
-; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX8-NEXT:    s_endpgm
 ;
-; GFX9-LABEL: add_i32_varying_gfx1032:
+; GFX9-LABEL: add_i32_varying_nouse:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX9-NEXT:    s_not_b64 exec, exec
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_not_b64 exec, exec
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
 ; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
-; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v1, 63
+; GFX9-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-NEXT:    s_mov_b32 s0, s2
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT:    ; implicit-def: $vgpr0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_cbranch_execz BB3_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    ds_add_rtn_u32 v0, v0, v3
+; GFX9-NEXT:    ds_add_u32 v0, v2
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:  BB3_2:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX9-NEXT:    v_mov_b32_e32 v0, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0
-; GFX9-NEXT:    s_mov_b32 s3, 0xf000
-; GFX9-NEXT:    s_mov_b32 s2, -1
-; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX9-NEXT:    s_endpgm
-;
-; GFX1064-LABEL: add_i32_varying_gfx1032:
-; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
-; GFX1064-NEXT:    s_not_b64 exec, exec
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    s_not_b64 exec, exec
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
-; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
-; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
-; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
-; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
-; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
-; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
-; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT:    s_mov_b32 s2, -1
-; GFX1064-NEXT:    ; implicit-def: $vgpr0
-; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT:    s_cbranch_execz BB3_2
-; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
-; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
-; GFX1064-NEXT:    s_mov_b32 s3, s7
-; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1064-NEXT:    ds_add_rtn_u32 v0, v7, v4
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    buffer_gl0_inv
-; GFX1064-NEXT:  BB3_2:
-; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
-; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s3, v0
-; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX1064-NEXT:    s_endpgm
-;
-; GFX1032-LABEL: add_i32_varying_gfx1032:
-; GFX1032:       ; %bb.0: ; %entry
-; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
-; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
-; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
-; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
-; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
-; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
-; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT:    s_mov_b32 s2, -1
-; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz BB3_2
-; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
-; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
-; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1032-NEXT:    ds_add_rtn_u32 v0, v7, v4
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    buffer_gl0_inv
-; GFX1032-NEXT:  BB3_2:
-; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
-; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
-; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s3, v0
-; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX1032-NEXT:    s_endpgm
-entry:
-  %lane = call i32 @llvm.amdgcn.workitem.id.x()
-  %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel
-  store i32 %old, i32 addrspace(1)* %out
-  ret void
-}
-
-define amdgpu_kernel void @add_i32_varying_gfx1064(i32 addrspace(1)* %out) {
-;
-;
-; GFX7LESS-LABEL: add_i32_varying_gfx1064:
-; GFX7LESS:       ; %bb.0: ; %entry
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
-; GFX7LESS-NEXT:    s_mov_b32 m0, -1
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    ds_add_rtn_u32 v0, v1, v0
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s2, -1
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX7LESS-NEXT:    s_endpgm
-;
-; GFX8-LABEL: add_i32_varying_gfx1064:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_mov_b32_e32 v2, v0
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX8-NEXT:    s_not_b64 exec, exec
-; GFX8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX8-NEXT:    s_not_b64 exec, exec
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX8-NEXT:    s_nop 1
-; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX8-NEXT:    v_readlane_b32 s4, v2, 63
-; GFX8-NEXT:    s_nop 0
-; GFX8-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8-NEXT:    ; implicit-def: $vgpr0
-; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX8-NEXT:    s_cbranch_execz BB4_2
-; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    s_mov_b32 m0, -1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    ds_add_rtn_u32 v0, v0, v3
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:  BB4_2:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX8-NEXT:    v_mov_b32_e32 v0, v1
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
-; GFX8-NEXT:    s_mov_b32 s3, 0xf000
-; GFX8-NEXT:    s_mov_b32 s2, -1
-; GFX8-NEXT:    s_nop 0
-; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-LABEL: add_i32_varying_gfx1064:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT:    s_not_b64 exec, exec
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    s_not_b64 exec, exec
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-NEXT:    s_nop 1
-; GFX9-NEXT:    v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-NEXT:    v_readlane_b32 s4, v2, 63
-; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT:    ; implicit-def: $vgpr0
-; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX9-NEXT:    s_cbranch_execz BB4_2
-; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    ds_add_rtn_u32 v0, v0, v3
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:  BB4_2:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX9-NEXT:    v_mov_b32_e32 v0, v1
-; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0
-; GFX9-NEXT:    s_mov_b32 s3, 0xf000
-; GFX9-NEXT:    s_mov_b32 s2, -1
-; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
-; GFX1064-LABEL: add_i32_varying_gfx1064:
+; GFX1064-LABEL: add_i32_varying_nouse:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064-NEXT:    v_mov_b32_e32 v3, 0
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1064-NEXT:    v_readlane_b32 s4, v1, 31
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s4
-; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1064-NEXT:    v_readlane_b32 s4, v1, 15
-; GFX1064-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX1064-NEXT:    v_readlane_b32 s5, v1, 31
-; GFX1064-NEXT:    v_writelane_b32 v3, s4, 16
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v1, v2
+; GFX1064-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
-; GFX1064-NEXT:    v_readlane_b32 s7, v1, 63
-; GFX1064-NEXT:    v_readlane_b32 s6, v1, 47
-; GFX1064-NEXT:    v_writelane_b32 v3, s5, 32
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GFX1064-NEXT:    v_readlane_b32 s2, v1, 0
+; GFX1064-NEXT:    v_readlane_b32 s3, v1, 32
+; GFX1064-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
-; GFX1064-NEXT:    v_writelane_b32 v3, s6, 48
-; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
+; GFX1064-NEXT:    s_add_i32 s0, s2, s3
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT:    s_mov_b32 s2, -1
-; GFX1064-NEXT:    ; implicit-def: $vgpr0
-; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT:    s_cbranch_execz BB4_2
-; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
-; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
-; GFX1064-NEXT:    s_mov_b32 s3, s7
-; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1064-NEXT:    ds_add_rtn_u32 v0, v7, v4
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    buffer_gl0_inv
-; GFX1064-NEXT:  BB4_2:
-; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
-; GFX1064-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1064-NEXT:    v_add_nc_u32_e32 v0, s3, v0
-; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX1064-NEXT:    s_endpgm
-;
-; GFX1032-LABEL: add_i32_varying_gfx1064:
-; GFX1032:       ; %bb.0: ; %entry
-; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
-; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
-; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
-; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1032-NEXT:    v_mov_b32_e32 v3, 0
-; GFX1032-NEXT:    v_readlane_b32 s3, v1, 15
-; GFX1032-NEXT:    v_readlane_b32 s4, v1, 31
-; GFX1032-NEXT:    v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
-; GFX1032-NEXT:    v_writelane_b32 v3, s3, 16
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT:    s_mov_b32 s2, -1
-; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz BB4_2
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB3_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s0
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_add_u32 v0, v3
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:  BB3_2:
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: add_i32_varying_nouse:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s0, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v1, v2
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
+; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB3_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
-; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var32@abs32@lo
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1032-NEXT:    ds_add_rtn_u32 v0, v7, v4
+; GFX1032-NEXT:    ds_add_u32 v3, v0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
-; GFX1032-NEXT:  BB4_2:
-; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
-; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
-; GFX1032-NEXT:    v_mov_b32_e32 v0, v3
-; GFX1032-NEXT:    v_add_nc_u32_e32 v0, s3, v0
-; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX1032-NEXT:  BB3_2:
 ; GFX1032-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %old = atomicrmw add i32 addrspace(3)* @local_var32, i32 %lane acq_rel
-  store i32 %old, i32 addrspace(1)* %out
   ret void
 }
 
@@ -1055,18 +739,18 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX7LESS-NEXT:    s_cbranch_execz BB5_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB4_2
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX7LESS-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX7LESS-NEXT:    s_mul_i32 s5, s4, 5
-; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7LESS-NEXT:    s_mov_b32 m0, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:  BB5_2:
+; GFX7LESS-NEXT:  BB4_2:
 ; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1090,18 +774,18 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX8-NEXT:    s_cbranch_execz BB5_2
+; GFX8-NEXT:    s_cbranch_execz BB4_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX8-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
 ; GFX8-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:  BB5_2:
+; GFX8-NEXT:  BB4_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1124,17 +808,17 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX9-NEXT:    s_cbranch_execz BB5_2
+; GFX9-NEXT:    s_cbranch_execz BB4_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX9-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
 ; GFX9-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:  BB5_2:
+; GFX9-NEXT:  BB4_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1157,19 +841,19 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s5, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX1064-NEXT:    s_cbranch_execz BB5_2
+; GFX1064-NEXT:    s_cbranch_execz BB4_2
 ; GFX1064-NEXT:  ; %bb.1:
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX1064-NEXT:    s_mul_i32 s5, s4, 5
-; GFX1064-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s5
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1064-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
-; GFX1064-NEXT:  BB5_2:
+; GFX1064-NEXT:  BB4_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1189,19 +873,19 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz BB5_2
+; GFX1032-NEXT:    s_cbranch_execz BB4_2
 ; GFX1032-NEXT:  ; %bb.1:
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_mul_i32 s3, s3, 5
 ; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX1032-NEXT:    s_mul_i32 s4, s3, 5
-; GFX1032-NEXT:    v_mul_hi_u32_u24_e64 v2, s3, 5
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
-; GFX1032-NEXT:  BB5_2:
+; GFX1032-NEXT:  BB4_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1230,7 +914,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX7LESS-NEXT:    s_cbranch_execz BB6_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB5_2
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
@@ -1245,7 +929,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:  BB6_2:
+; GFX7LESS-NEXT:  BB5_2:
 ; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7LESS-NEXT:    s_mov_b32 s6, -1
@@ -1273,7 +957,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT:    s_cbranch_execz BB6_2
+; GFX8-NEXT:    s_cbranch_execz BB5_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s6
@@ -1288,7 +972,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:  BB6_2:
+; GFX8-NEXT:  BB5_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_mov_b32 s4, s0
@@ -1316,7 +1000,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT:    s_cbranch_execz BB6_2
+; GFX9-NEXT:    s_cbranch_execz BB5_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1330,7 +1014,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:  BB6_2:
+; GFX9-NEXT:  BB5_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mul_lo_u32 v3, s3, v0
@@ -1358,7 +1042,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT:    s_cbranch_execz BB6_2
+; GFX1064-NEXT:    s_cbranch_execz BB5_2
 ; GFX1064-NEXT:  ; %bb.1:
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
 ; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
@@ -1374,7 +1058,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX1064-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
-; GFX1064-NEXT:  BB6_2:
+; GFX1064-NEXT:  BB5_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1399,7 +1083,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz BB6_2
+; GFX1032-NEXT:    s_cbranch_execz BB5_2
 ; GFX1032-NEXT:  ; %bb.1:
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s5, s5
 ; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
@@ -1415,7 +1099,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX1032-NEXT:    ds_add_rtn_u64 v[1:2], v3, v[1:2]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
-; GFX1032-NEXT:  BB6_2:
+; GFX1032-NEXT:  BB5_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1437,9 +1121,6 @@ entry:
   ret void
 }
 
-; GCN-NOT: v_mbcnt_lo_u32_b32
-; GCN-NOT: v_mbcnt_hi_u32_b32
-; GCN-NOT: s_bcnt1_i32_b64
 define amdgpu_kernel void @add_i64_varying(i64 addrspace(1)* %out) {
 ;
 ;
@@ -1484,35 +1165,20 @@ define amdgpu_kernel void @add_i64_varying(i64 addrspace(1)* %out) {
 ; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
-; GFX1064-LABEL: add_i64_varying:
-; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT:    s_mov_b32 s2, -1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1064-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    buffer_gl0_inv
-; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
-; GFX1064-NEXT:    s_endpgm
-;
-; GFX1032-LABEL: add_i64_varying:
-; GFX1032:       ; %bb.0: ; %entry
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1032-NEXT:    s_mov_b32 s2, -1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1032-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    buffer_gl0_inv
-; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
-; GFX1032-NEXT:    s_endpgm
+; GFX10-LABEL: add_i64_varying:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX10-NEXT:    s_mov_b32 s2, -1
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    buffer_gl0_inv
+; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX10-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %zext = zext i32 %lane to i64
@@ -1533,7 +1199,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX7LESS-NEXT:    s_cbranch_execz BB8_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB7_2
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-NEXT:    s_mul_i32 s2, s2, 5
@@ -1543,7 +1209,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:  BB8_2:
+; GFX7LESS-NEXT:  BB7_2:
 ; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1563,7 +1229,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT:    s_cbranch_execz BB8_2
+; GFX8-NEXT:    s_cbranch_execz BB7_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX8-NEXT:    s_mul_i32 s2, s2, 5
@@ -1573,7 +1239,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:  BB8_2:
+; GFX8-NEXT:  BB7_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1594,7 +1260,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT:    s_cbranch_execz BB8_2
+; GFX9-NEXT:    s_cbranch_execz BB7_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX9-NEXT:    s_mul_i32 s2, s2, 5
@@ -1603,7 +1269,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:  BB8_2:
+; GFX9-NEXT:  BB7_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1624,7 +1290,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT:    s_cbranch_execz BB8_2
+; GFX1064-NEXT:    s_cbranch_execz BB7_2
 ; GFX1064-NEXT:  ; %bb.1:
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
@@ -1635,7 +1301,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1064-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
-; GFX1064-NEXT:  BB8_2:
+; GFX1064-NEXT:  BB7_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1655,7 +1321,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz BB8_2
+; GFX1032-NEXT:    s_cbranch_execz BB7_2
 ; GFX1032-NEXT:  ; %bb.1:
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
@@ -1666,7 +1332,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
-; GFX1032-NEXT:  BB8_2:
+; GFX1032-NEXT:  BB7_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
@@ -1696,7 +1362,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[6:7], vcc
-; GFX7LESS-NEXT:    s_cbranch_execz BB9_2
+; GFX7LESS-NEXT:    s_cbranch_execz BB8_2
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1707,7 +1373,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:  BB9_2:
+; GFX7LESS-NEXT:  BB8_2:
 ; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    v_readfirstlane_b32 s1, v1
@@ -1728,7 +1394,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[6:7], vcc
-; GFX8-NEXT:    s_cbranch_execz BB9_2
+; GFX8-NEXT:    s_cbranch_execz BB8_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1739,7 +1405,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:  BB9_2:
+; GFX8-NEXT:  BB8_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
@@ -1760,7 +1426,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT:    s_cbranch_execz BB9_2
+; GFX9-NEXT:    s_cbranch_execz BB8_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1770,7 +1436,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:  BB9_2:
+; GFX9-NEXT:  BB8_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mul_lo_u32 v0, s2, v0
@@ -1792,7 +1458,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT:    s_cbranch_execz BB9_2
+; GFX1064-NEXT:    s_cbranch_execz BB8_2
 ; GFX1064-NEXT:  ; %bb.1:
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
@@ -1804,7 +1470,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX1064-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
-; GFX1064-NEXT:  BB9_2:
+; GFX1064-NEXT:  BB8_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1826,7 +1492,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz BB9_2
+; GFX1032-NEXT:    s_cbranch_execz BB8_2
 ; GFX1032-NEXT:  ; %bb.1:
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s1, s3
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
@@ -1838,7 +1504,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX1032-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
-; GFX1032-NEXT:  BB9_2:
+; GFX1032-NEXT:  BB8_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1902,7 +1568,7 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr0
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX8-NEXT:    s_cbranch_execz BB10_2
+; GFX8-NEXT:    s_cbranch_execz BB9_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s4
@@ -1910,7 +1576,7 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    ds_sub_rtn_u32 v0, v0, v3
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:  BB10_2:
+; GFX8-NEXT:  BB9_2:
 ; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
@@ -1953,14 +1619,14 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX9-NEXT:    s_cbranch_execz BB10_2
+; GFX9-NEXT:    s_cbranch_execz BB9_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_sub_rtn_u32 v0, v0, v3
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:  BB10_2:
+; GFX9-NEXT:  BB9_2:
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
@@ -2012,7 +1678,7 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-NEXT:    s_mov_b32 s2, -1
 ; GFX1064-NEXT:    ; implicit-def: $vgpr0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT:    s_cbranch_execz BB10_2
+; GFX1064-NEXT:    s_cbranch_execz BB9_2
 ; GFX1064-NEXT:  ; %bb.1:
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, s7
@@ -2022,7 +1688,7 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-NEXT:    ds_sub_rtn_u32 v0, v7, v4
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    buffer_gl0_inv
-; GFX1064-NEXT:  BB10_2:
+; GFX1064-NEXT:  BB9_2:
 ; GFX1064-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s3, v0
@@ -2063,7 +1729,7 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz BB10_2
+; GFX1032-NEXT:    s_cbranch_execz BB9_2
 ; GFX1032-NEXT:  ; %bb.1:
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, local_var32@abs32@lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, s4
@@ -2072,7 +1738,7 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    ds_sub_rtn_u32 v0, v7, v4
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    buffer_gl0_inv
-; GFX1032-NEXT:  BB10_2:
+; GFX1032-NEXT:  BB9_2:
 ; GFX1032-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GFX1032-NEXT:    v_readfirstlane_b32 s3, v0
@@ -2089,6 +1755,158 @@ entry:
   ret void
 }
 
+define amdgpu_kernel void @sub_i32_varying_nouse() {
+; GFX7LESS-LABEL: sub_i32_varying_nouse:
+; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
+; GFX7LESS-NEXT:    s_mov_b32 m0, -1
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    ds_sub_u32 v1, v0
+; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT:    s_endpgm
+;
+; GFX8-LABEL: sub_i32_varying_nouse:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    v_mov_b32_e32 v1, v0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_not_b64 exec, exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8-NEXT:    s_nop 1
+; GFX8-NEXT:    v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8-NEXT:    v_readlane_b32 s2, v1, 63
+; GFX8-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    s_cbranch_execz BB10_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    ds_sub_u32 v0, v2
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:  BB10_2:
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sub_i32_varying_nouse:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_not_b64 exec, exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-NEXT:    v_readlane_b32 s2, v1, 63
+; GFX9-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    s_cbranch_execz BB10_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    ds_sub_u32 v0, v2
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:  BB10_2:
+; GFX9-NEXT:    s_endpgm
+;
+; GFX1064-LABEL: sub_i32_varying_nouse:
+; GFX1064:       ; %bb.0: ; %entry
+; GFX1064-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1064-NEXT:    s_not_b64 exec, exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1064-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
+; GFX1064-NEXT:    v_add_nc_u32_e32 v1, v1, v2
+; GFX1064-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX1064-NEXT:    s_or_saveexec_b64 s[0:1], -1
+; GFX1064-NEXT:    v_readlane_b32 s2, v1, 0
+; GFX1064-NEXT:    v_readlane_b32 s3, v1, 32
+; GFX1064-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
+; GFX1064-NEXT:    s_add_i32 s0, s2, s3
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    s_cbranch_execz BB10_2
+; GFX1064-NEXT:  ; %bb.1:
+; GFX1064-NEXT:    v_mov_b32_e32 v0, local_var32@abs32@lo
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s0
+; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1064-NEXT:    ds_sub_u32 v0, v3
+; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT:    buffer_gl0_inv
+; GFX1064-NEXT:  BB10_2:
+; GFX1064-NEXT:    s_endpgm
+;
+; GFX1032-LABEL: sub_i32_varying_nouse:
+; GFX1032:       ; %bb.0: ; %entry
+; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s0, -1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v1
+; GFX1032-NEXT:    v_permlanex16_b32 v2, v2, -1, -1
+; GFX1032-NEXT:    v_add_nc_u32_e32 v1, v1, v2
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
+; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_cbranch_execz BB10_2
+; GFX1032-NEXT:  ; %bb.1:
+; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var32@abs32@lo
+; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX1032-NEXT:    ds_sub_u32 v3, v0
+; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT:    buffer_gl0_inv
+; GFX1032-NEXT:  BB10_2:
+; GFX1032-NEXT:    s_endpgm
+entry:
+  %lane = call i32 @llvm.amdgcn.workitem.id.x()
+  %old = atomicrmw sub i32 addrspace(3)* @local_var32, i32 %lane acq_rel
+  ret void
+}
+
 define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ;
 ;
@@ -2104,10 +1922,10 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ; GFX7LESS-NEXT:    s_cbranch_execz BB11_2
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX7LESS-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX7LESS-NEXT:    s_mul_i32 s5, s4, 5
-; GFX7LESS-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX7LESS-NEXT:    s_mov_b32 m0, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
@@ -2139,9 +1957,9 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ; GFX8-NEXT:    s_cbranch_execz BB11_2
 ; GFX8-NEXT:  ; %bb.1:
 ; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX8-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
 ; GFX8-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2174,9 +1992,9 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ; GFX9-NEXT:    s_cbranch_execz BB11_2
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX9-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
 ; GFX9-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
@@ -2208,10 +2026,10 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1064-NEXT:    s_cbranch_execz BB11_2
 ; GFX1064-NEXT:  ; %bb.1:
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-NEXT:    s_mul_i32 s4, s4, 5
 ; GFX1064-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX1064-NEXT:    s_mul_i32 s5, s4, 5
-; GFX1064-NEXT:    v_mul_hi_u32_u24_e64 v2, s4, 5
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s5
+; GFX1064-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1064-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
@@ -2243,10 +2061,10 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032-NEXT:    s_cbranch_execz BB11_2
 ; GFX1032-NEXT:  ; %bb.1:
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-NEXT:    s_mul_i32 s3, s3, 5
 ; GFX1032-NEXT:    v_mov_b32_e32 v3, local_var64@abs32@lo
-; GFX1032-NEXT:    s_mul_i32 s4, s3, 5
-; GFX1032-NEXT:    v_mul_hi_u32_u24_e64 v2, s3, 5
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_sub_rtn_u64 v[1:2], v3, v[1:2]
@@ -2491,9 +2309,6 @@ entry:
   ret void
 }
 
-; GCN-NOT: v_mbcnt_lo_u32_b32
-; GCN-NOT: v_mbcnt_hi_u32_b32
-; GCN-NOT: s_bcnt1_i32_b64
 define amdgpu_kernel void @sub_i64_varying(i64 addrspace(1)* %out) {
 ;
 ;
@@ -2538,35 +2353,20 @@ define amdgpu_kernel void @sub_i64_varying(i64 addrspace(1)* %out) {
 ; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
 ;
-; GFX1064-LABEL: sub_i64_varying:
-; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT:    s_mov_b32 s2, -1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1064-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    buffer_gl0_inv
-; GFX1064-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
-; GFX1064-NEXT:    s_endpgm
-;
-; GFX1032-LABEL: sub_i64_varying:
-; GFX1032:       ; %bb.0: ; %entry
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1032-NEXT:    s_mov_b32 s2, -1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1032-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    buffer_gl0_inv
-; GFX1032-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
-; GFX1032-NEXT:    s_endpgm
+; GFX10-LABEL: sub_i64_varying:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, local_var64@abs32@lo
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX10-NEXT:    s_mov_b32 s2, -1
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    buffer_gl0_inv
+; GFX10-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX10-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %zext = zext i32 %lane to i64
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index f303aad8389a877f223309d13c593521fa43c15f..ddbf168e773465c70f94389128b0d96f764383d5 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-- -amdgpu-atomic-optimizations=true -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX7 %s
-; RUN: llc  -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s
-; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
+; RUN: llc  -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX89,GFX8 %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX89,GFX9 %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064 %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1032 %s
 
@@ -46,73 +46,39 @@ define amdgpu_ps void @add_i32_constant(<4 x i32> inreg %out, <4 x i32> inreg %i
 ; GFX7-NEXT:  BB0_6: ; %UnifiedReturnBlock
 ; GFX7-NEXT:    s_endpgm
 ;
-; GFX8-LABEL: add_i32_constant:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_mov_b64 s[10:11], exec
-; GFX8-NEXT:    ; implicit-def: $vgpr0
-; GFX8-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
-; GFX8-NEXT:    s_cbranch_execz BB0_4
-; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    s_mov_b64 s[12:13], exec
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s12, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s13, v0
-; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8-NEXT:    ; implicit-def: $vgpr1
-; GFX8-NEXT:    s_and_saveexec_b64 s[10:11], vcc
-; GFX8-NEXT:    s_cbranch_execz BB0_3
-; GFX8-NEXT:  ; %bb.2:
-; GFX8-NEXT:    s_bcnt1_i32_b64 s12, s[12:13]
-; GFX8-NEXT:    s_mul_i32 s12, s12, 5
-; GFX8-NEXT:    v_mov_b32_e32 v1, s12
-; GFX8-NEXT:    buffer_atomic_add v1, off, s[4:7], 0 glc
-; GFX8-NEXT:  BB0_3:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[10:11]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX8-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
-; GFX8-NEXT:  BB0_4: ; %Flow
-; GFX8-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GFX8-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX8-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_vccnz BB0_6
-; GFX8-NEXT:  ; %bb.5: ; %if
-; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX8-NEXT:  BB0_6: ; %UnifiedReturnBlock
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-LABEL: add_i32_constant:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_mov_b64 s[10:11], exec
-; GFX9-NEXT:    ; implicit-def: $vgpr0
-; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
-; GFX9-NEXT:    s_cbranch_execz BB0_4
-; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_mov_b64 s[12:13], exec
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s12, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s13, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT:    ; implicit-def: $vgpr1
-; GFX9-NEXT:    s_and_saveexec_b64 s[10:11], vcc
-; GFX9-NEXT:    s_cbranch_execz BB0_3
-; GFX9-NEXT:  ; %bb.2:
-; GFX9-NEXT:    s_bcnt1_i32_b64 s12, s[12:13]
-; GFX9-NEXT:    s_mul_i32 s12, s12, 5
-; GFX9-NEXT:    v_mov_b32_e32 v1, s12
-; GFX9-NEXT:    buffer_atomic_add v1, off, s[4:7], 0 glc
-; GFX9-NEXT:  BB0_3:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[10:11]
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX9-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
-; GFX9-NEXT:  BB0_4: ; %Flow
-; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GFX9-NEXT:    s_wqm_b64 s[4:5], -1
-; GFX9-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_vccnz BB0_6
-; GFX9-NEXT:  ; %bb.5: ; %if
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX9-NEXT:  BB0_6: ; %UnifiedReturnBlock
-; GFX9-NEXT:    s_endpgm
+; GFX89-LABEL: add_i32_constant:
+; GFX89:       ; %bb.0: ; %entry
+; GFX89-NEXT:    s_mov_b64 s[10:11], exec
+; GFX89-NEXT:    ; implicit-def: $vgpr0
+; GFX89-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
+; GFX89-NEXT:    s_cbranch_execz BB0_4
+; GFX89-NEXT:  ; %bb.1:
+; GFX89-NEXT:    s_mov_b64 s[12:13], exec
+; GFX89-NEXT:    v_mbcnt_lo_u32_b32 v0, s12, 0
+; GFX89-NEXT:    v_mbcnt_hi_u32_b32 v0, s13, v0
+; GFX89-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX89-NEXT:    ; implicit-def: $vgpr1
+; GFX89-NEXT:    s_and_saveexec_b64 s[10:11], vcc
+; GFX89-NEXT:    s_cbranch_execz BB0_3
+; GFX89-NEXT:  ; %bb.2:
+; GFX89-NEXT:    s_bcnt1_i32_b64 s12, s[12:13]
+; GFX89-NEXT:    s_mul_i32 s12, s12, 5
+; GFX89-NEXT:    v_mov_b32_e32 v1, s12
+; GFX89-NEXT:    buffer_atomic_add v1, off, s[4:7], 0 glc
+; GFX89-NEXT:  BB0_3:
+; GFX89-NEXT:    s_or_b64 exec, exec, s[10:11]
+; GFX89-NEXT:    s_waitcnt vmcnt(0)
+; GFX89-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX89-NEXT:    v_mad_u32_u24 v0, v0, 5, s4
+; GFX89-NEXT:  BB0_4: ; %Flow
+; GFX89-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX89-NEXT:    s_wqm_b64 s[4:5], -1
+; GFX89-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; GFX89-NEXT:    s_cbranch_vccnz BB0_6
+; GFX89-NEXT:  ; %bb.5: ; %if
+; GFX89-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX89-NEXT:  BB0_6: ; %UnifiedReturnBlock
+; GFX89-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: add_i32_constant:
 ; GFX1064:       ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index bd0ec2efec2d3cb2a375f9c6799a85afec8fc88f..a73bf61340e0d5eb53bd0519c02c6b25dbc42367 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s
+; RUN: llc -march=amdgcn -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.raw.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index 3a29c101babd0ad5b0d79c65fb815a955869e6f8..43f52bdf192bfd4b038194da48b5dc1e8b4bde3f 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s
+; RUN: llc -march=amdgcn -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX7LESS %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX89,DPPCOMB %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN64,GFX8MORE,GFX8MORE64,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN32,GFX8MORE,GFX8MORE32,GFX10 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.struct.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-v3.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-v3.ll
index 2ad0b626fd9261ea0406b7e130d115fa77a4ab77..59acd206159d8e4a6ffa43d3a4c68f85f0e2d8e0 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-v3.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 < %s | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -amdgpu-verify-hsa-metadata -filetype=obj -o /dev/null < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 --amdhsa-code-object-version=3 < %s | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 --amdhsa-code-object-version=3 -amdgpu-verify-hsa-metadata -filetype=obj -o /dev/null < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK-LABEL: {{^}}min_64_max_64:
 ; CHECK: SGPRBlocks: 0
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
index 38c758935ed2024ffe031a0b41b1c19812cbd0f2..710c1ede71282eed9e969161bbbc95503e902c89 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 < %s | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 --amdhsa-code-object-version=2 < %s | FileCheck --check-prefix=CHECK %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 --amdhsa-code-object-version=2 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=HSAMD %s
 
 ; CHECK-LABEL: {{^}}min_64_max_64:
@@ -129,7 +129,7 @@ define amdgpu_kernel void @min_1024_max_1024() #3 {
 }
 attributes #3 = {"amdgpu-flat-work-group-size"="1024,1024"}
 
-; HSAMD: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
+; HSAMD: NT_AMD_HSA_METADATA (AMD HSA Metadata)
 ; HSAMD: Version: [ 1, 0 ]
 ; HSAMD: Kernels:
 ; HSAMD: - Name:                 min_64_max_64
diff --git a/llvm/test/CodeGen/AMDGPU/break-smem-soft-clauses.mir b/llvm/test/CodeGen/AMDGPU/break-smem-soft-clauses.mir
index 62a1d08e173640b2ccd35e804945dc0730de595f..a10cc7f6a693b1b812e838b871373046868d630e 100644
--- a/llvm/test/CodeGen/AMDGPU/break-smem-soft-clauses.mir
+++ b/llvm/test/CodeGen/AMDGPU/break-smem-soft-clauses.mir
@@ -1,5 +1,5 @@
 # RUN: llc -march=amdgcn -mcpu=carrizo -verify-machineinstrs -run-pass  post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,XNACK %s
-# RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass  post-RA-hazard-rec %s -o - | FileCheck --check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=fiji -mattr=-xnack -verify-machineinstrs -run-pass  post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN %s
 
 ---
 # Trivial clause at beginning of program
diff --git a/llvm/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir b/llvm/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir
index a52faa79d52c08dfd53ecc3750773b1a3e55305a..7e7db6b62460a5a4ad24801ee2f8b2ee5fbfc1ac 100644
--- a/llvm/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir
+++ b/llvm/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=amdgcn -mcpu=tonga -run-pass post-RA-sched -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=tonga -mattr=-xnack -run-pass post-RA-sched -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
 
 # GCN:      FLAT_LOAD_DWORD
 # GCN-NEXT: FLAT_LOAD_DWORD
diff --git a/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll b/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll
index c638beb9a2246d55c793983f53c16161e862066e..456bdedcd6b28258a604f75767dc9992e8475767 100644
--- a/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll
+++ b/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll
@@ -1,3 +1,90 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx600 < %s | FileCheck --check-prefixes=V3-GFX600 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=tahiti < %s | FileCheck --check-prefixes=V3-GFX600 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx601 < %s | FileCheck --check-prefixes=V3-GFX601 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=pitcairn < %s | FileCheck --check-prefixes=V3-GFX601 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=verde < %s | FileCheck --check-prefixes=V3-GFX601 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx602 < %s | FileCheck --check-prefixes=V3-GFX602 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=hainan < %s | FileCheck --check-prefixes=V3-GFX602 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=oland < %s | FileCheck --check-prefixes=V3-GFX602 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx700 < %s | FileCheck --check-prefixes=V3-GFX700 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=kaveri < %s | FileCheck --check-prefixes=V3-GFX700 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx701 < %s | FileCheck --check-prefixes=V3-GFX701 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=hawaii < %s | FileCheck --check-prefixes=V3-GFX701 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx702 < %s | FileCheck --check-prefixes=V3-GFX702 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx703 < %s | FileCheck --check-prefixes=V3-GFX703 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=kabini < %s | FileCheck --check-prefixes=V3-GFX703 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=mullins < %s | FileCheck --check-prefixes=V3-GFX703 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx704 < %s | FileCheck --check-prefixes=V3-GFX704 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=bonaire < %s | FileCheck --check-prefixes=V3-GFX704 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx705 < %s | FileCheck --check-prefixes=V3-GFX705 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx801 < %s | FileCheck --check-prefixes=V3-GFX801-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx801 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX801-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx801 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX801-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=carrizo < %s | FileCheck --check-prefixes=V3-GFX801-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=carrizo -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX801-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=carrizo -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX801-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx802 < %s | FileCheck --check-prefixes=V3-GFX802 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=iceland < %s | FileCheck --check-prefixes=V3-GFX802 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=tonga < %s | FileCheck --check-prefixes=V3-GFX802 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx803 < %s | FileCheck --check-prefixes=V3-GFX803 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=fiji < %s | FileCheck --check-prefixes=V3-GFX803 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=polaris10 < %s | FileCheck --check-prefixes=V3-GFX803 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=polaris11 < %s | FileCheck --check-prefixes=V3-GFX803 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx805 < %s | FileCheck --check-prefixes=V3-GFX805 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=tongapro < %s | FileCheck --check-prefixes=V3-GFX805 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx810 < %s | FileCheck --check-prefixes=V3-GFX810-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx810 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX810-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx810 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX810-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=stoney < %s | FileCheck --check-prefixes=V3-GFX810-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=stoney -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX810-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=stoney -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX810-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx900 < %s | FileCheck --check-prefixes=V3-GFX900-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx900 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX900-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx900 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX900-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx902 < %s | FileCheck --check-prefixes=V3-GFX902-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx902 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX902-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx902 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX902-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx904 < %s | FileCheck --check-prefixes=V3-GFX904-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx904 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX904-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx904 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX904-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx906 < %s | FileCheck --check-prefixes=V3-GFX906-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx906 -mattr=-sramecc < %s | FileCheck --check-prefixes=V3-GFX906-NOSRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx906 -mattr=+sramecc < %s | FileCheck --check-prefixes=V3-GFX906-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx906 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX906-SRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx906 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX906-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx906 -mattr=-sramecc,-xnack < %s | FileCheck --check-prefixes=V3-GFX906-NOSRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx906 -mattr=+sramecc,-xnack < %s | FileCheck --check-prefixes=V3-GFX906-SRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx906 -mattr=-sramecc,+xnack < %s | FileCheck --check-prefixes=V3-GFX906-NOSRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx906 -mattr=+sramecc,+xnack < %s | FileCheck --check-prefixes=V3-GFX906-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx908 < %s | FileCheck --check-prefixes=V3-GFX908-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx908 -mattr=-sramecc < %s | FileCheck --check-prefixes=V3-GFX908-NOSRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx908 -mattr=+sramecc < %s | FileCheck --check-prefixes=V3-GFX908-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx908 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX908-SRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx908 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX908-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx908 -mattr=-sramecc,-xnack < %s | FileCheck --check-prefixes=V3-GFX908-NOSRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx908 -mattr=+sramecc,-xnack < %s | FileCheck --check-prefixes=V3-GFX908-SRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx908 -mattr=-sramecc,+xnack < %s | FileCheck --check-prefixes=V3-GFX908-NOSRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx908 -mattr=+sramecc,+xnack < %s | FileCheck --check-prefixes=V3-GFX908-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx909 < %s | FileCheck --check-prefixes=V3-GFX909-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx909 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX909-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx909 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX909-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx90c < %s | FileCheck --check-prefixes=V3-GFX90C-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx90c -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX90C-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx90c -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX90C-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1010 < %s | FileCheck --check-prefixes=V3-GFX1010-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1010 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX1010-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1010 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX1010-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1011 < %s | FileCheck --check-prefixes=V3-GFX1011-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1011 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX1011-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1011 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX1011-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1012 < %s | FileCheck --check-prefixes=V3-GFX1012-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1012 -mattr=-xnack < %s | FileCheck --check-prefixes=V3-GFX1012-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1012 -mattr=+xnack < %s | FileCheck --check-prefixes=V3-GFX1012-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1030 < %s | FileCheck --check-prefixes=V3-GFX1030 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1031 < %s | FileCheck --check-prefixes=V3-GFX1031 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1032 < %s | FileCheck --check-prefixes=V3-GFX1032 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx1033 < %s | FileCheck --check-prefixes=V3-GFX1033 %s
+
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 < %s | FileCheck --check-prefixes=GFX600 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti < %s | FileCheck --check-prefixes=GFX600 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx601 < %s | FileCheck --check-prefixes=GFX601 %s
@@ -17,8 +104,12 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx704 < %s | FileCheck --check-prefixes=GFX704 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=bonaire < %s | FileCheck --check-prefixes=GFX704 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx705 < %s | FileCheck --check-prefixes=GFX705 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=gfx801 < %s | FileCheck --check-prefixes=GFX801 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -mcpu=carrizo < %s | FileCheck --check-prefixes=GFX801 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx801 < %s | FileCheck --check-prefixes=GFX801 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx801 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX801-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx801 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX801-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=carrizo < %s | FileCheck --check-prefixes=GFX801 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=carrizo -mattr=-xnack < %s | FileCheck --check-prefixes=GFX801-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=carrizo -mattr=+xnack < %s | FileCheck --check-prefixes=GFX801-XNACK %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 < %s | FileCheck --check-prefixes=GFX802 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=iceland < %s | FileCheck --check-prefixes=GFX802 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga < %s | FileCheck --check-prefixes=GFX802 %s
@@ -29,23 +120,102 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx805 < %s | FileCheck --check-prefixes=GFX805 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tongapro < %s | FileCheck --check-prefixes=GFX805 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 < %s | FileCheck --check-prefixes=GFX810 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX810-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX810-XNACK %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=stoney < %s | FileCheck --check-prefixes=GFX810 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=stoney -mattr=-xnack < %s | FileCheck --check-prefixes=GFX810-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=stoney -mattr=+xnack < %s | FileCheck --check-prefixes=GFX810-XNACK %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=GFX900 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX900-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX900-XNACK %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx902 < %s | FileCheck --check-prefixes=GFX902 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx902 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX902-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx902 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX902-XNACK %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx904 < %s | FileCheck --check-prefixes=GFX904 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx904 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX904-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx904 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX904-XNACK %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 < %s | FileCheck --check-prefixes=GFX906 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-sramecc < %s | FileCheck --check-prefixes=GFX906-NOSRAMECC %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+sramecc < %s | FileCheck --check-prefixes=GFX906-SRAMECC %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX906-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX906-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-sramecc,-xnack < %s | FileCheck --check-prefixes=GFX906-NOSRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+sramecc,-xnack < %s | FileCheck --check-prefixes=GFX906-SRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-sramecc,+xnack < %s | FileCheck --check-prefixes=GFX906-NOSRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+sramecc,+xnack < %s | FileCheck --check-prefixes=GFX906-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck --check-prefixes=GFX908 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-sramecc < %s | FileCheck --check-prefixes=GFX908-NOSRAMECC %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=+sramecc < %s | FileCheck --check-prefixes=GFX908-SRAMECC %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX908-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX908-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-sramecc,-xnack < %s | FileCheck --check-prefixes=GFX908-NOSRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=+sramecc,-xnack < %s | FileCheck --check-prefixes=GFX908-SRAMECC-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-sramecc,+xnack < %s | FileCheck --check-prefixes=GFX908-NOSRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=+sramecc,+xnack < %s | FileCheck --check-prefixes=GFX908-SRAMECC-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx909 < %s | FileCheck --check-prefixes=GFX909 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx909 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX909-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx909 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX909-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90c < %s | FileCheck --check-prefixes=GFX90C %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90c -mattr=-xnack < %s | FileCheck --check-prefixes=GFX90C-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90c -mattr=+xnack < %s | FileCheck --check-prefixes=GFX90C-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX1010 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX1010-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX1010-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1011 < %s | FileCheck --check-prefixes=GFX1011 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1011 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX1011-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1011 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX1011-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1012 < %s | FileCheck --check-prefixes=GFX1012 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1012 -mattr=-xnack < %s | FileCheck --check-prefixes=GFX1012-NOXNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1012 -mattr=+xnack < %s | FileCheck --check-prefixes=GFX1012-XNACK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck --check-prefixes=GFX1030 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1031 < %s | FileCheck --check-prefixes=GFX1031 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1032 < %s | FileCheck --check-prefixes=GFX1032 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1033 < %s | FileCheck --check-prefixes=GFX1033 %s
 
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack < %s | FileCheck --check-prefixes=XNACK-GFX900 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx902 -mattr=-xnack < %s | FileCheck --check-prefixes=NO-XNACK-GFX902 %s
-
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx904 -mattr=+sramecc < %s | FileCheck --check-prefixes=SRAM-ECC-GFX904 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+sramecc < %s | FileCheck --check-prefixes=SRAM-ECC-GFX906 %s
-
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx904 -mattr=+sramecc,+xnack < %s | FileCheck --check-prefixes=SRAM-ECC-XNACK-GFX904 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+sramecc,+xnack < %s | FileCheck --check-prefixes=SRAM-ECC-XNACK-GFX906 %s
-
-; FIXME: With the default attributes these directives are not accurate for
-; xnack and sramecc. Subsequent Target-ID patches will address this.
+; V3-GFX600: .amdgcn_target "amdgcn-amd-amdhsa--gfx600"
+; V3-GFX601: .amdgcn_target "amdgcn-amd-amdhsa--gfx601"
+; V3-GFX602: .amdgcn_target "amdgcn-amd-amdhsa--gfx602"
+; V3-GFX700: .amdgcn_target "amdgcn-amd-amdhsa--gfx700"
+; V3-GFX701: .amdgcn_target "amdgcn-amd-amdhsa--gfx701"
+; V3-GFX702: .amdgcn_target "amdgcn-amd-amdhsa--gfx702"
+; V3-GFX703: .amdgcn_target "amdgcn-amd-amdhsa--gfx703"
+; V3-GFX704: .amdgcn_target "amdgcn-amd-amdhsa--gfx704"
+; V3-GFX705: .amdgcn_target "amdgcn-amd-amdhsa--gfx705"
+; V3-GFX801-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx801"
+; V3-GFX801-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx801+xnack"
+; V3-GFX802: .amdgcn_target "amdgcn-amd-amdhsa--gfx802"
+; V3-GFX803: .amdgcn_target "amdgcn-amd-amdhsa--gfx803"
+; V3-GFX805: .amdgcn_target "amdgcn-amd-amdhsa--gfx805"
+; V3-GFX810-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx810"
+; V3-GFX810-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx810+xnack"
+; V3-GFX900-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+; V3-GFX900-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx900+xnack"
+; V3-GFX902-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx902"
+; V3-GFX902-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx902+xnack"
+; V3-GFX904-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx904"
+; V3-GFX904-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx904+xnack"
+; V3-GFX906-NOSRAMECC-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906"
+; V3-GFX906-SRAMECC-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906+sram-ecc"
+; V3-GFX906-NOSRAMECC-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906+xnack"
+; V3-GFX906-SRAMECC-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906+xnack+sram-ecc"
+; V3-GFX908-NOSRAMECC-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908"
+; V3-GFX908-SRAMECC-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908+sram-ecc"
+; V3-GFX908-NOSRAMECC-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908+xnack"
+; V3-GFX908-SRAMECC-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908+xnack+sram-ecc"
+; V3-GFX909-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx909"
+; V3-GFX909-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx909+xnack"
+; V3-GFX90C-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx90c"
+; V3-GFX90C-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx90c+xnack"
+; V3-GFX1010-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1010"
+; V3-GFX1010-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1010+xnack"
+; V3-GFX1011-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1011"
+; V3-GFX1011-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1011+xnack"
+; V3-GFX1012-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1012"
+; V3-GFX1012-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1012+xnack"
+; V3-GFX1030: .amdgcn_target "amdgcn-amd-amdhsa--gfx1030"
+; V3-GFX1031: .amdgcn_target "amdgcn-amd-amdhsa--gfx1031"
+; V3-GFX1032: .amdgcn_target "amdgcn-amd-amdhsa--gfx1032"
+; V3-GFX1033: .amdgcn_target "amdgcn-amd-amdhsa--gfx1033"
 
 ; GFX600: .amdgcn_target "amdgcn-amd-amdhsa--gfx600"
 ; GFX601: .amdgcn_target "amdgcn-amd-amdhsa--gfx601"
@@ -57,23 +227,60 @@
 ; GFX704: .amdgcn_target "amdgcn-amd-amdhsa--gfx704"
 ; GFX705: .amdgcn_target "amdgcn-amd-amdhsa--gfx705"
 ; GFX801: .amdgcn_target "amdgcn-amd-amdhsa--gfx801"
+; GFX801-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx801:xnack-"
+; GFX801-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx801:xnack+"
 ; GFX802: .amdgcn_target "amdgcn-amd-amdhsa--gfx802"
 ; GFX803: .amdgcn_target "amdgcn-amd-amdhsa--gfx803"
 ; GFX805: .amdgcn_target "amdgcn-amd-amdhsa--gfx805"
 ; GFX810: .amdgcn_target "amdgcn-amd-amdhsa--gfx810"
+; GFX810-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx810:xnack-"
+; GFX810-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx810:xnack+"
 ; GFX900: .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+; GFX900-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-"
+; GFX900-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+"
 ; GFX902: .amdgcn_target "amdgcn-amd-amdhsa--gfx902"
+; GFX902-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx902:xnack-"
+; GFX902-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx902:xnack+"
 ; GFX904: .amdgcn_target "amdgcn-amd-amdhsa--gfx904"
+; GFX904-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx904:xnack-"
+; GFX904-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx904:xnack+"
 ; GFX906: .amdgcn_target "amdgcn-amd-amdhsa--gfx906"
-
-; XNACK-GFX900: .amdgcn_target "amdgcn-amd-amdhsa--gfx900+xnack"
-; NO-XNACK-GFX902: .amdgcn_target "amdgcn-amd-amdhsa--gfx902"
-
-; SRAM-ECC-GFX904: .amdgcn_target "amdgcn-amd-amdhsa--gfx904+sramecc"
-; SRAM-ECC-GFX906: "amdgcn-amd-amdhsa--gfx906+sramecc"
-
-; SRAM-ECC-XNACK-GFX904: .amdgcn_target "amdgcn-amd-amdhsa--gfx904+xnack+sramecc"
-; SRAM-ECC-XNACK-GFX906: .amdgcn_target "amdgcn-amd-amdhsa--gfx906+xnack+sramecc"
+; GFX906-NOSRAMECC: .amdgcn_target "amdgcn-amd-amdhsa--gfx906:sramecc-"
+; GFX906-SRAMECC: .amdgcn_target "amdgcn-amd-amdhsa--gfx906:sramecc+"
+; GFX906-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906:xnack-"
+; GFX906-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906:xnack+"
+; GFX906-NOSRAMECC-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906:sramecc-:xnack-"
+; GFX906-SRAMECC-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906:sramecc+:xnack-"
+; GFX906-NOSRAMECC-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906:sramecc-:xnack+"
+; GFX906-SRAMECC-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx906:sramecc+:xnack+"
+; GFX908: .amdgcn_target "amdgcn-amd-amdhsa--gfx908"
+; GFX908-NOSRAMECC: .amdgcn_target "amdgcn-amd-amdhsa--gfx908:sramecc-"
+; GFX908-SRAMECC: .amdgcn_target "amdgcn-amd-amdhsa--gfx908:sramecc+"
+; GFX908-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908:xnack-"
+; GFX908-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908:xnack+"
+; GFX908-NOSRAMECC-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908:sramecc-:xnack-"
+; GFX908-SRAMECC-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908:sramecc+:xnack-"
+; GFX908-NOSRAMECC-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908:sramecc-:xnack+"
+; GFX908-SRAMECC-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx908:sramecc+:xnack+"
+; GFX909: .amdgcn_target "amdgcn-amd-amdhsa--gfx909"
+; GFX909-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx909:xnack-"
+; GFX909-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx909:xnack+"
+; GFX90C: .amdgcn_target "amdgcn-amd-amdhsa--gfx90c"
+; GFX90C-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx90c:xnack-"
+; GFX90C-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx90c:xnack+"
+; GFX1010: .amdgcn_target "amdgcn-amd-amdhsa--gfx1010"
+; GFX1010-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1010:xnack-"
+; GFX1010-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1010:xnack+"
+; GFX1011: .amdgcn_target "amdgcn-amd-amdhsa--gfx1011"
+; GFX1011-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1011:xnack-"
+; GFX1011-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1011:xnack+"
+; GFX1012: .amdgcn_target "amdgcn-amd-amdhsa--gfx1012"
+; GFX1012-NOXNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1012:xnack-"
+; GFX1012-XNACK: .amdgcn_target "amdgcn-amd-amdhsa--gfx1012:xnack+"
+; GFX1030: .amdgcn_target "amdgcn-amd-amdhsa--gfx1030"
+; GFX1031: .amdgcn_target "amdgcn-amd-amdhsa--gfx1031"
+; GFX1032: .amdgcn_target "amdgcn-amd-amdhsa--gfx1032"
+; GFX1033: .amdgcn_target "amdgcn-amd-amdhsa--gfx1033"
 
 define amdgpu_kernel void @directive_amdgcn_target() {
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/early-term.mir b/llvm/test/CodeGen/AMDGPU/early-term.mir
index fc896c54512e3a0f653682687a30e18880627b67..39ff92bd581981366b095f803aedb3753d1c74b7 100644
--- a/llvm/test/CodeGen/AMDGPU/early-term.mir
+++ b/llvm/test/CodeGen/AMDGPU/early-term.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=si-insert-skips -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=si-late-branch-lowering -verify-machineinstrs  %s -o - | FileCheck %s
 
 --- |
   define amdgpu_ps void @early_term_scc0_end_block() {
diff --git a/llvm/test/CodeGen/AMDGPU/elf-header-flags-sram-ecc.ll b/llvm/test/CodeGen/AMDGPU/elf-header-flags-sramecc.ll
similarity index 53%
rename from llvm/test/CodeGen/AMDGPU/elf-header-flags-sram-ecc.ll
rename to llvm/test/CodeGen/AMDGPU/elf-header-flags-sramecc.ll
index b977bb0f6c1a373d28a77c5d2b7e50990776b4f6..69662de040326bb81cb589bb753552b5e9e5aa00 100644
--- a/llvm/test/CodeGen/AMDGPU/elf-header-flags-sram-ecc.ll
+++ b/llvm/test/CodeGen/AMDGPU/elf-header-flags-sramecc.ll
@@ -1,30 +1,41 @@
-; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx906 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NO-SRAM-ECC-GFX906 %s
+; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx906 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=SRAM-ECC-GFX906 %s
 ; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx906 -mattr=-sramecc < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NO-SRAM-ECC-GFX906 %s
 ; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx906 -mattr=+sramecc < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=SRAM-ECC-GFX906 %s
 ; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx906 -mattr=+sramecc,+xnack < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=SRAM-ECC-XNACK-GFX906 %s
 
+; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx908 < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=SRAM-ECC-GFX908 %s
 ; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx908 -mattr=+sramecc < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=SRAM-ECC-GFX908 %s
 
+; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx90a < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=SRAM-ECC-GFX90A %s
+; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx90a < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=SRAM-ECC-GFX90A %s
+
 ; NO-SRAM-ECC-GFX906:      Flags [
+; NO-SRAM-ECC-GFX906-NEXT:   EF_AMDGPU_FEATURE_XNACK_V3   (0x100)
 ; NO-SRAM-ECC-GFX906-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX906 (0x2F)
 ; NO-SRAM-ECC-GFX906-NEXT: ]
 
 ; SRAM-ECC-GFX906:      Flags [
+; SRAM-ECC-GFX906-NEXT:   EF_AMDGPU_FEATURE_SRAMECC_V3 (0x200)
+; SRAM-ECC-GFX906-NEXT:   EF_AMDGPU_FEATURE_XNACK_V3   (0x100)
 ; SRAM-ECC-GFX906-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX906 (0x2F)
-; SRAM-ECC-GFX906-NEXT:   EF_AMDGPU_SRAM_ECC           (0x200)
 ; SRAM-ECC-GFX906-NEXT: ]
 
 ; SRAM-ECC-XNACK-GFX906:      Flags [
+; SRAM-ECC-XNACK-GFX906-NEXT:   EF_AMDGPU_FEATURE_SRAMECC_V3 (0x200)
+; SRAM-ECC-XNACK-GFX906-NEXT:   EF_AMDGPU_FEATURE_XNACK_V3   (0x100)
 ; SRAM-ECC-XNACK-GFX906-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX906 (0x2F)
-; SRAM-ECC-XNACK-GFX906-NEXT:   EF_AMDGPU_SRAM_ECC           (0x200)
-; SRAM-ECC-XNACK-GFX906-NEXT:   EF_AMDGPU_XNACK              (0x100)
 ; SRAM-ECC-XNACK-GFX906-NEXT: ]
 
-; SRAM-ECC-GFX908: Flags [ (0x230)
+; SRAM-ECC-GFX908: Flags [
+; SRAM-ECC-GFX908:    EF_AMDGPU_FEATURE_SRAMECC_V3 (0x200)
 ; SRAM-ECC-GFX908:    EF_AMDGPU_MACH_AMDGCN_GFX908 (0x30)
-; SRAM-ECC-GFX908:    EF_AMDGPU_SRAM_ECC (0x200)
 ; SRAM-ECC-GFX908:  ]
 
+; SRAM-ECC-GFX90A: Flags [
+; SRAM-ECC-GFX90A:    EF_AMDGPU_FEATURE_SRAMECC_V3 (0x200)
+; SRAM-ECC-GFX90A:    EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)
+; SRAM-ECC-GFX90A:  ]
+
 define amdgpu_kernel void @elf_header() {
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/elf-header-flags-xnack.ll b/llvm/test/CodeGen/AMDGPU/elf-header-flags-xnack.ll
index 30607351a818af604e178d19e17a7f860ce433e4..22dcaf971c4837537f2db22866a6ef79f10bb99e 100644
--- a/llvm/test/CodeGen/AMDGPU/elf-header-flags-xnack.ll
+++ b/llvm/test/CodeGen/AMDGPU/elf-header-flags-xnack.ll
@@ -1,14 +1,16 @@
-; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx801 -mattr=-xnack < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NO-XNACK-GFX801 %s
-; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx802 -mattr=+xnack < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=XNACK-GFX802 %s
+; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=XNACK-GFX801 %s
+; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx801 -mattr=+xnack < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=XNACK-GFX801 %s
+; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx802 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NO-XNACK-GFX802 %s
+; RUN: llc -filetype=obj -march=amdgcn -mcpu=gfx802 -mattr=-xnack < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NO-XNACK-GFX802 %s
 
-; NO-XNACK-GFX801:      Flags [
-; NO-XNACK-GFX801-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX801 (0x28)
-; NO-XNACK-GFX801-NEXT: ]
+; XNACK-GFX801:      Flags [
+; XNACK-GFX801-NEXT:   EF_AMDGPU_FEATURE_XNACK_V3   (0x100)
+; XNACK-GFX801-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX801 (0x28)
+; XNACK-GFX801-NEXT: ]
 
-; XNACK-GFX802:      Flags [
-; XNACK-GFX802-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX802 (0x29)
-; XNACK-GFX802-NEXT:   EF_AMDGPU_XNACK              (0x100)
-; XNACK-GFX802-NEXT: ]
+; NO-XNACK-GFX802:      Flags [
+; NO-XNACK-GFX802-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX802 (0x29)
+; NO-XNACK-GFX802-NEXT: ]
 
 define amdgpu_kernel void @elf_header() {
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll b/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll
index 3fe61092b53faa6ba1978bbfdd88820c778c5240..1cbde24c96a8b66e911381800f02c73ee23bdf93 100644
--- a/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll
+++ b/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll
@@ -13,11 +13,11 @@
 
 ; NONE:   OS/ABI: SystemV       (0x0)
 ; HSA:    OS/ABI: AMDGPU_HSA    (0x40)
-; HSA:    ABIVersion: 1
+; HSA:    ABIVersion: 2
 ; PAL:    OS/ABI: AMDGPU_PAL    (0x41)
 ; PAL:    ABIVersion: 0
 ; MESA3D: OS/ABI: AMDGPU_MESA3D (0x42)
-; MESA3D:    ABIVersion: 0
+; MESA3D: ABIVersion: 0
 
 define amdgpu_kernel void @elf_header() {
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/elf-notes.ll b/llvm/test/CodeGen/AMDGPU/elf-notes.ll
index 05ac3c92f4bac78bf8800f4a22d5559bf50055d7..0507e868fea5391cb4185e6028fe062a920a1024 100644
--- a/llvm/test/CodeGen/AMDGPU/elf-notes.ll
+++ b/llvm/test/CodeGen/AMDGPU/elf-notes.ll
@@ -16,13 +16,13 @@
 ; OSABI-UNK-NOT: .amd_amdgpu_pal_metadata
 
 ; OSABI-UNK-ELF-NOT: Unknown note type
-; OSABI-UNK-ELF: NT_AMD_AMDGPU_ISA (ISA Version)
-; OSABI-UNK-ELF: ISA Version:
+; OSABI-UNK-ELF: NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)
+; OSABI-UNK-ELF: AMD HSA ISA Name:
 ; OSABI-UNK-ELF: amdgcn-amd-unknown--gfx802
 ; OSABI-UNK-ELF-NOT: Unknown note type
-; OSABI-UNK-ELF-NOT: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
+; OSABI-UNK-ELF-NOT: NT_AMD_HSA_METADATA (AMD HSA Metadata)
 ; OSABI-UNK-ELF-NOT: Unknown note type
-; OSABI-UNK-ELF-NOT: NT_AMD_AMDGPU_PAL_METADATA (PAL Metadata)
+; OSABI-UNK-ELF-NOT: NT_AMD_PAL_METADATA (AMD PAL Metadata)
 ; OSABI-UNK-ELF-NOT: Unknown note type
 
 ; OSABI-HSA: .hsa_code_object_version
@@ -31,12 +31,12 @@
 ; OSABI-HSA: .amd_amdgpu_hsa_metadata
 ; OSABI-HSA-NOT: .amd_amdgpu_pal_metadata
 
-; OSABI-HSA-ELF: Unknown note type: (0x00000001)
-; OSABI-HSA-ELF: Unknown note type: (0x00000003)
-; OSABI-HSA-ELF: NT_AMD_AMDGPU_ISA (ISA Version)
-; OSABI-HSA-ELF: ISA Version:
+; OSABI-HSA-ELF: NT_AMD_HSA_CODE_OBJECT_VERSION (AMD HSA Code Object Version)
+; OSABI-HSA-ELF: NT_AMD_HSA_ISA_VERSION (AMD HSA ISA Version)
+; OSABI-HSA-ELF: NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)
+; OSABI-HSA-ELF: AMD HSA ISA Name:
 ; OSABI-HSA-ELF: amdgcn-amd-amdhsa--gfx802
-; OSABI-HSA-ELF: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
+; OSABI-HSA-ELF: NT_AMD_HSA_METADATA (AMD HSA Metadata)
 ; OSABI-HSA-ELF: HSA Metadata:
 ; OSABI-HSA-ELF: ---
 ; OSABI-HSA-ELF: Version: [ 1, 0 ]
@@ -51,18 +51,18 @@
 ; OSABI-HSA-ELF:       WavefrontSize:   64
 ; OSABI-HSA-ELF:       NumSGPRs:        96
 ; OSABI-HSA-ELF: ...
-; OSABI-HSA-ELF-NOT: NT_AMD_AMDGPU_PAL_METADATA (PAL Metadata)
+; OSABI-HSA-ELF-NOT: NT_AMD_PAL_METADATA (AMD PAL Metadata)
 
 ; OSABI-PAL-NOT: .hsa_code_object_version
 ; OSABI-PAL: .hsa_code_object_isa
 ; OSABI-PAL: .amd_amdgpu_isa "amdgcn-amd-amdpal--gfx802"
 ; OSABI-PAL-NOT: .amd_amdgpu_hsa_metadata
 
-; OSABI-PAL-ELF: Unknown note type: (0x00000003)
-; OSABI-PAL-ELF: NT_AMD_AMDGPU_ISA (ISA Version)
-; OSABI-PAL-ELF: ISA Version:
+; OSABI-PAL-ELF: NT_AMD_HSA_ISA_VERSION (AMD HSA ISA Version)
+; OSABI-PAL-ELF: NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)
+; OSABI-PAL-ELF: AMD HSA ISA Name:
 ; OSABI-PAL-ELF: amdgcn-amd-amdpal--gfx802
-; OSABI-PAL-ELF-NOT: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
+; OSABI-PAL-ELF-NOT: NT_AMD_HSA_METADATA (AMD HSA Metadata)
 ; OSABI-PAL-ELF: NT_AMDGPU_METADATA (AMDGPU Metadata)
 ; OSABI-PAL-ELF: AMDGPU Metadata:
 ; OSABI-PAL-ELF: amdpal.pipelines:
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll
index b938ff52f21a2885b3837189ed885f656e79d654..729a05c12c744c9d2ea29666ba90c4c34626feb8 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll
@@ -1,14 +1,17 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,GFX89 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SIVI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,GFX89,SIVI %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX89 %s
 
 ; GCN-LABEL: {{^}}extract_vector_elt_v2i16:
 ; GCN: s_load_dword [[VEC:s[0-9]+]]
-; GCN: s_lshr_b32 [[ELT1:s[0-9]+]], [[VEC]], 16
-; GCN-DAG: v_mov_b32_e32 [[VELT0:v[0-9]+]], [[VEC]]
-; GCN-DAG: v_mov_b32_e32 [[VELT1:v[0-9]+]], [[ELT1]]
-; GCN-DAG: buffer_store_short [[VELT0]]
-; GCN-DAG: buffer_store_short [[VELT1]]
+; SIVI: s_lshr_b32 [[ELT1:s[0-9]+]], [[VEC]], 16
+; SIVI-DAG: v_mov_b32_e32 [[VELT0:v[0-9]+]], [[VEC]]
+; SIVI-DAG: v_mov_b32_e32 [[VELT1:v[0-9]+]], [[ELT1]]
+; SIVI-DAG: buffer_store_short [[VELT0]]
+; SIVI-DAG: buffer_store_short [[VELT1]]
+; GFX9: v_mov_b32_e32 [[VVEC:v[0-9]+]], [[VEC]]
+; GFX9: global_store_short_d16_hi v{{[0-9]+}}, [[VVEC]],
+; GFX9: buffer_store_short [[VVEC]],
 define amdgpu_kernel void @extract_vector_elt_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace(4)* %vec.ptr) #0 {
   %vec = load <2 x i16>, <2 x i16> addrspace(4)* %vec.ptr
   %p0 = extractelement <2 x i16> %vec, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.ll b/llvm/test/CodeGen/AMDGPU/fabs.ll
index badaa16bbfcc5d447f93dc1603f13feea187260c..83cdcac32e95580a016e8d486c827fe905e20879 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global,-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s
 
 
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
index 268e327e678b8732ad823f1b2514456bd11fa4e5..02835b6a5465b6cebd95653141fe36f09cd02fa9 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
@@ -1,20 +1,27 @@
 ; RUN: llc -march=amdgcn -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=VI-NOXNACK -check-prefix=GCN %s
 
-; RUN: llc -march=amdgcn -mcpu=carrizo -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=VI-NOXNACK  -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=stoney -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=VI-NOXNACK  -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=carrizo -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefixes=VI-NOXNACK,GCN %s
+; RUN: llc -march=amdgcn -mcpu=stoney -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefixes=VI-NOXNACK,GCN %s
 
 ; RUN: llc -march=amdgcn -mcpu=carrizo -mattr=+xnack -verify-machineinstrs < %s | FileCheck -check-prefix=VI-XNACK  -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=stoney -mattr=+xnack -verify-machineinstrs < %s | FileCheck -check-prefix=VI-XNACK  -check-prefix=GCN %s
 
-; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=kaveri --amdhsa-code-object-version=2 -verify-machineinstrs < %s | FileCheck -check-prefix=HSA-CI -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=carrizo --amdhsa-code-object-version=2 -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=HSA-VI-NOXNACK -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=carrizo --amdhsa-code-object-version=2 -mattr=+xnack -verify-machineinstrs < %s | FileCheck -check-prefix=HSA-VI-XNACK -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=kaveri --amdhsa-code-object-version=2 -verify-machineinstrs < %s | FileCheck -check-prefixes=CI,HSA-CI-V2,GCN %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=carrizo --amdhsa-code-object-version=2 -mattr=+xnack -verify-machineinstrs < %s | FileCheck -check-prefixes=VI-XNACK,HSA-VI-XNACK-V2,GCN %s
+
+; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefixes=VI-NOXNACK,HSA-VI-NOXNACK,GCN %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=+xnack -verify-machineinstrs < %s | FileCheck -check-prefixes=VI-XNACK,HSA-VI-XNACK,GCN %s
 
 ; GCN-LABEL: {{^}}no_vcc_no_flat:
-; HSA-CI: is_xnack_enabled = 0
-; HSA-VI-NOXNACK: is_xnack_enabled = 0
-; HSA-VI-XNACK: is_xnack_enabled = 1
+
+; HSA-CI-V2: is_xnack_enabled = 0
+; HSA-VI-XNACK-V2: is_xnack_enabled = 1
+
+; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
+; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
+; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: ; NumSgprs: 8
 ; VI-NOXNACK: ; NumSgprs: 8
@@ -26,9 +33,13 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}vcc_no_flat:
-; HSA-CI: is_xnack_enabled = 0
-; HSA-VI-NOXNACK: is_xnack_enabled = 0
-; HSA-VI-XNACK: is_xnack_enabled = 1
+
+; HSA-CI-V2: is_xnack_enabled = 0
+; HSA-VI-XNACK-V2: is_xnack_enabled = 1
+
+; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
+; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
+; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: ; NumSgprs: 10
 ; VI-NOXNACK: ; NumSgprs: 10
@@ -40,16 +51,17 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}no_vcc_flat:
-; HSA-CI: is_xnack_enabled = 0
-; HSA-VI-NOXNACK: is_xnack_enabled = 0
-; HSA-VI-XNACK: is_xnack_enabled = 1
+
+; HSA-CI-V2: is_xnack_enabled = 0
+; HSA-VI-XNACK-V2: is_xnack_enabled = 1
+
+; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
+; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
+; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: ; NumSgprs: 12
 ; VI-NOXNACK: ; NumSgprs: 14
 ; VI-XNACK: ; NumSgprs: 14
-; HSA-CI: ; NumSgprs: 12
-; HSA-VI-NOXNACK: ; NumSgprs: 14
-; HSA-VI-XNACK: ; NumSgprs: 14
 define amdgpu_kernel void @no_vcc_flat() {
 entry:
   call void asm sideeffect "", "~{s7},~{flat_scratch}"()
@@ -57,15 +69,17 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}vcc_flat:
-; HSA-NOXNACK: is_xnack_enabled = 0
-; HSA-XNACK: is_xnack_enabled = 1
+
+; HSA-CI-V2: is_xnack_enabled = 0
+; HSA-VI-XNACK-V2: is_xnack_enabled = 1
+
+; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
+; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
+; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
 
 ; CI: ; NumSgprs: 12
 ; VI-NOXNACK: ; NumSgprs: 14
 ; VI-XNACK: ; NumSgprs: 14
-; HSA-CI: ; NumSgprs: 12
-; HSA-VI-NOXNACK: ; NumSgprs: 14
-; HSA-VI-XNACK: ; NumSgprs: 14
 define amdgpu_kernel void @vcc_flat() {
 entry:
   call void asm sideeffect "", "~{s7},~{vcc},~{flat_scratch}"()
@@ -76,6 +90,14 @@ entry:
 ; scratch usage and implicit flat uses.
 
 ; GCN-LABEL: {{^}}use_flat_scr:
+
+; HSA-CI-V2: is_xnack_enabled = 0
+; HSA-VI-XNACK-V2: is_xnack_enabled = 1
+
+; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
+; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
+; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
+
 ; CI: NumSgprs: 4
 ; VI-NOXNACK: NumSgprs: 6
 ; VI-XNACK: NumSgprs: 6
@@ -86,6 +108,14 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}use_flat_scr_lo:
+
+; HSA-CI-V2: is_xnack_enabled = 0
+; HSA-VI-XNACK-V2: is_xnack_enabled = 1
+
+; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
+; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
+; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
+
 ; CI: NumSgprs: 4
 ; VI-NOXNACK: NumSgprs: 6
 ; VI-XNACK: NumSgprs: 6
@@ -96,6 +126,14 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}use_flat_scr_hi:
+
+; HSA-CI-V2: is_xnack_enabled = 0
+; HSA-VI-XNACK-V2: is_xnack_enabled = 1
+
+; NOT-HSA-CI: .amdhsa_reserve_xnack_mask
+; HSA-VI-NOXNACK: .amdhsa_reserve_xnack_mask 0
+; HSA-VI-XNACK: .amdhsa_reserve_xnack_mask 1
+
 ; CI: NumSgprs: 4
 ; VI-NOXNACK: NumSgprs: 6
 ; VI-XNACK: NumSgprs: 6
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index c1f270143feb7c3e991a1fcb74740109b3c4f40c..15eefaebd4bf969049a6edb654d02d9546e1b934 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -3294,66 +3294,30 @@ define void @tail_call_byval_align16(<32 x i32> %val, double %tmp) #0 {
 ; GFX9-LABEL: tail_call_byval_align16:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
-; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX9-NEXT:    v_writelane_b32 v40, s33, 2
-; GFX9-NEXT:    s_mov_b32 s33, s32
-; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s33 offset:8
-; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s33 offset:12
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_add_u32 s32, s32, 0x800
+; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:12
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, byval_align16_f64_arg@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, byval_align16_f64_arg@rel32@hi+12
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:4
 ; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
-; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
-; GFX9-NEXT:    s_sub_u32 s32, s32, 0x800
-; GFX9-NEXT:    v_readlane_b32 s33, v40, 2
-; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], -1
-; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_mov_b64 exec, s[6:7]
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[4:5]
 ;
 ; GFX10-LABEL: tail_call_byval_align16:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    s_or_saveexec_b32 s4, -1
-; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
-; GFX10-NEXT:    s_mov_b32 s33, s32
-; GFX10-NEXT:    s_add_u32 s32, s32, 0x400
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s33 offset:12
-; GFX10-NEXT:    buffer_load_dword v33, off, s[0:3], s33 offset:8
+; GFX10-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:12
+; GFX10-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:8
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, byval_align16_f64_arg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, byval_align16_f64_arg@rel32@hi+12
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt vmcnt(1)
 ; GFX10-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:4
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    buffer_store_dword v33, off, s[0:3], s32
-; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
-; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
-; GFX10-NEXT:    s_sub_u32 s32, s32, 0x400
-; GFX10-NEXT:    v_readlane_b32 s33, v40, 2
-; GFX10-NEXT:    s_or_saveexec_b32 s6, -1
-; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_mov_b32 exec_lo, s6
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[4:5]
 entry:
   %alloca = alloca double, align 8, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll
index f475f4a6c631604ca150c2536df9971a81ac96f6..88a54eec2e00c387fd1ae7b62d8eb9bec9734372 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK:              ---
 ; CHECK:      amdhsa.kernels:
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
index 32a5c22ce1193dfc0240ef793a54f27a70adcb0a..fa3ba63fe6e52e1825da802e6ebb8cfd3ca22421 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
@@ -1,9 +1,9 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 --amdhsa-code-object-version=3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 %struct.A = type { i8, float }
 %opencl.image1d_t = type opaque
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll
index 10a151a560649c00aa41687c19c371041155d755..1105caf980ef4c5526506e5dd40423ab8b434a7c 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
 
 ; CHECK:              ---
 ; CHECK:      amdhsa.kernels:
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
index 9d2541508eb7be3d91f46562b3ddcd1f09317fc2..a1cee23387a89f1e7d7a5ae0614ae853e0ea230b 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK:              ---
 ; CHECK:      amdhsa.kernels:
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
index 1753037fa0f96d19b9e9917d0671455492133dcb..11123e0624f3a2a8d34a06f326f5db9f17ded97e 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK:              ---
 ; CHECK:      amdhsa.kernels:
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll
index cff46dba261fe2802ecb94474f223c0c306fa857..72f692cf1b6871a037e9fbd79406958d1b5fefc4 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK %s
 
 %opencl.image1d_t = type opaque
 %opencl.image1d_array_t = type opaque
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1-v3.ll
index 7a4c31140bee4715bbb8d27d166cc72fa263511f..3b6fff3a681fa82213f0a4ae61852ec0946b9ece 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2-v3.ll
index c8fd01e7d8198d16ae378caff5282f2abc77f5fd..241979860dc1eb61605ce3d81e240bcb3c054b6d 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3-v3.ll
index d9088cc99ed49a4bc7524833c612e18462ec2c2d..89191e6dfa1ae2c66d14c5283ec62917b6add55b 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-wavefrontsize.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-wavefrontsize.ll
index 844668184981fb70845009cefe53440107207e6f..7955dcc434c3b096975d5c0159c485c974073e15 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-wavefrontsize.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-wavefrontsize.ll
@@ -1,13 +1,10 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 --amdhsa-code-object-version=2 -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GCN,GFX10-32 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 --amdhsa-code-object-version=2 -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefixes=GCN,GFX10-64 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GCN,GFX10-32 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefixes=GCN,GFX10-64 %s
 
-; GCN:      ---
-; GCN:      Kernels:
-; GCN:        - Name: wavefrontsize
-; GCN:          CodeProps:
-; GFX10-32:       WavefrontSize: 32
-; GFX10-64:       WavefrontSize: 64
-; GCN:      ...
+; GCN:      amdhsa.kernels:
+; GCN:      .name: wavefrontsize
+; GFX10-32: .wavefront_size: 32
+; GFX10-64: .wavefront_size: 64
 define amdgpu_kernel void @wavefrontsize() {
 entry:
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-note-no-func.ll b/llvm/test/CodeGen/AMDGPU/hsa-note-no-func.ll
index fac19dc52b67c54f398a47e16a4aed809b04d38a..83b0b2c9352d2f26c6f394adfe3191e5220642c5 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-note-no-func.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-note-no-func.ll
@@ -1,60 +1,59 @@
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx600 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-SI600 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx601 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-SI601 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx700 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI700 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx701 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI701 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx702 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI702 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx703 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI703 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx704 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI704 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=bonaire --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI704 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=mullins --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI703 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=hawaii --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI701 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kabini --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI703 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI700 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo --amdhsa-code-object-version=2 -mattr=-flat-for-global | FileCheck --check-prefix=HSA --check-prefix=HSA-VI801 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=tonga --amdhsa-code-object-version=2 -mattr=-flat-for-global | FileCheck --check-prefix=HSA --check-prefix=HSA-VI802 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=fiji --amdhsa-code-object-version=2 -mattr=-flat-for-global | FileCheck --check-prefix=HSA --check-prefix=HSA-VI803 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=polaris10 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI803 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=polaris11 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI803 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx801 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI801 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx802 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI802 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx803 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI803 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx810 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI810 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX900 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx902 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX902 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx904 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX904 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx906 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX906 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx908 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX908 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx909 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX909 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX1010 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1011 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX1011 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1012 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX1012 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1030 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX1030 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1031 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX1031 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1032 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX1032 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1033 --amdhsa-code-object-version=2 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX1033 %s
+; RUN: llc < %s -mtriple=amdgcn-- -mcpu=gfx600 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=NONHSA-SI600 %s
+; RUN: llc < %s -mtriple=amdgcn-- -mcpu=gfx601 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=NONHSA-SI601 %s
+; RUN: llc < %s -mtriple=amdgcn-- -mcpu=gfx602 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=NONHSA-SI602 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx700 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI700 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI700 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx701 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI701 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=hawaii --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI701 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx702 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI702 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx703 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI703 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kabini --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI703 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=mullins --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI703 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx704 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI704 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=bonaire --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI704 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx705 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-CI705 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx801 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-VI801 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo --amdhsa-code-object-version=2 -mattr=-flat-for-global | FileCheck --check-prefixes=HSA,HSA-VI801 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx802 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-VI802 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=iceland --amdhsa-code-object-version=2 -mattr=-flat-for-global | FileCheck --check-prefixes=HSA,HSA-VI802 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=tonga --amdhsa-code-object-version=2 -mattr=-flat-for-global | FileCheck --check-prefixes=HSA,HSA-VI802 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx803 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-VI803 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=fiji --amdhsa-code-object-version=2 -mattr=-flat-for-global | FileCheck --check-prefixes=HSA,HSA-VI803 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=polaris10 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-VI803 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=polaris11 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-VI803 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx805 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-VI805 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=tongapro --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-VI805 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx810 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-VI810 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=stoney --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-VI810 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 --amdhsa-code-object-version=2 -mattr=-xnack | FileCheck --check-prefixes=HSA,HSA-GFX900 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-GFX901 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx902 --amdhsa-code-object-version=2 -mattr=-xnack | FileCheck --check-prefixes=HSA,HSA-GFX902 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx902 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-GFX903 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx904 --amdhsa-code-object-version=2 -mattr=-xnack | FileCheck --check-prefixes=HSA,HSA-GFX904 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx904 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-GFX905 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx906 --amdhsa-code-object-version=2 -mattr=-xnack | FileCheck --check-prefixes=HSA,HSA-GFX906 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx906 --amdhsa-code-object-version=2 | FileCheck --check-prefixes=HSA,HSA-GFX907 %s
 
 ; HSA: .hsa_code_object_version 2,1
-; HSA-SI600: .hsa_code_object_isa 6,0,0,"AMD","AMDGPU"
-; HSA-SI601: .hsa_code_object_isa 6,0,1,"AMD","AMDGPU"
+; NONHSA-SI600: .amd_amdgpu_isa "amdgcn-unknown-unknown--gfx600"
+; NONHSA-SI601: .amd_amdgpu_isa "amdgcn-unknown-unknown--gfx601"
+; NONHSA-SI602: .amd_amdgpu_isa "amdgcn-unknown-unknown--gfx602"
 ; HSA-CI700: .hsa_code_object_isa 7,0,0,"AMD","AMDGPU"
 ; HSA-CI701: .hsa_code_object_isa 7,0,1,"AMD","AMDGPU"
 ; HSA-CI702: .hsa_code_object_isa 7,0,2,"AMD","AMDGPU"
 ; HSA-CI703: .hsa_code_object_isa 7,0,3,"AMD","AMDGPU"
 ; HSA-CI704: .hsa_code_object_isa 7,0,4,"AMD","AMDGPU"
+; HSA-CI705: .hsa_code_object_isa 7,0,5,"AMD","AMDGPU"
 ; HSA-VI801: .hsa_code_object_isa 8,0,1,"AMD","AMDGPU"
 ; HSA-VI802: .hsa_code_object_isa 8,0,2,"AMD","AMDGPU"
 ; HSA-VI803: .hsa_code_object_isa 8,0,3,"AMD","AMDGPU"
+; HSA-VI805: .hsa_code_object_isa 8,0,5,"AMD","AMDGPU"
 ; HSA-VI810: .hsa_code_object_isa 8,1,0,"AMD","AMDGPU"
 ; HSA-GFX900: .hsa_code_object_isa 9,0,0,"AMD","AMDGPU"
+; HSA-GFX901: .hsa_code_object_isa 9,0,1,"AMD","AMDGPU"
 ; HSA-GFX902: .hsa_code_object_isa 9,0,2,"AMD","AMDGPU"
+; HSA-GFX903: .hsa_code_object_isa 9,0,3,"AMD","AMDGPU"
 ; HSA-GFX904: .hsa_code_object_isa 9,0,4,"AMD","AMDGPU"
+; HSA-GFX905: .hsa_code_object_isa 9,0,5,"AMD","AMDGPU"
 ; HSA-GFX906: .hsa_code_object_isa 9,0,6,"AMD","AMDGPU"
-; HSA-GFX908: .hsa_code_object_isa 9,0,8,"AMD","AMDGPU"
-; HSA-GFX909: .hsa_code_object_isa 9,0,9,"AMD","AMDGPU"
-; HSA-GFX1010: .hsa_code_object_isa 10,1,0,"AMD","AMDGPU"
-; HSA-GFX1011: .hsa_code_object_isa 10,1,1,"AMD","AMDGPU"
-; HSA-GFX1012: .hsa_code_object_isa 10,1,2,"AMD","AMDGPU"
-; HSA-GFX1030: .hsa_code_object_isa 10,3,0,"AMD","AMDGPU"
-; HSA-GFX1031: .hsa_code_object_isa 10,3,1,"AMD","AMDGPU"
-; HSA-GFX1032: .hsa_code_object_isa 10,3,2,"AMD","AMDGPU"
-; HSA-GFX1033: .hsa_code_object_isa 10,3,3,"AMD","AMDGPU"
+; HSA-GFX907: .hsa_code_object_isa 9,0,7,"AMD","AMDGPU"
diff --git a/llvm/test/CodeGen/AMDGPU/hsa.ll b/llvm/test/CodeGen/AMDGPU/hsa.ll
index 6493b29fe09e384dba14625b2f27e8dbf8d529a8..862586a356ccd7c222e007fecf070c12c2d35e56 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa.ll
@@ -4,8 +4,8 @@
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo --amdhsa-code-object-version=2 -mattr=-flat-for-global | FileCheck --check-prefix=HSA-VI %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -filetype=obj --amdhsa-code-object-version=2 | llvm-readobj -symbols -s -sd - | FileCheck --check-prefix=ELF %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri --amdhsa-code-object-version=2 | llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri --amdhsa-code-object-version=2 | llvm-readobj -symbols -s -sd - | FileCheck %s --check-prefix=ELF
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 --amdhsa-code-object-version=2 -mattr=+wavefrontsize32,-wavefrontsize64 | FileCheck --check-prefix=HSA --check-prefix=GFX10 --check-prefix=GFX10-W32 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 --amdhsa-code-object-version=2 -mattr=-wavefrontsize32,+wavefrontsize64 | FileCheck --check-prefix=HSA --check-prefix=GFX10 --check-prefix=GFX10-W64 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 | FileCheck --check-prefix=GFX10 --check-prefix=GFX10-W32 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 | FileCheck --check-prefix=GFX10 --check-prefix=GFX10-W64 %s
 
 ; The SHT_NOTE section contains the output from the .hsa_code_object_*
 ; directives.
@@ -49,12 +49,10 @@
 ; HSA: enable_sgpr_kernarg_segment_ptr = 1
 
 ; PRE-GFX10: enable_wavefront_size32 = 0
-; GFX10-W32: enable_wavefront_size32 = 1
-; GFX10-W64: enable_wavefront_size32 = 0
+; GFX10-W32: .amdhsa_wavefront_size32 1
+; GFX10-W64: .amdhsa_wavefront_size32 0
 
 ; PRE-GFX10: wavefront_size = 6
-; GFX10-W32: wavefront_size = 5
-; GFX10-W64: wavefront_size = 6
 
 ; HSA: call_convention = -1
 ; HSA: .end_amd_kernel_code_t
@@ -66,7 +64,7 @@
 ; HSA-VI: s_mov_b32 s[[HI:[0-9]]], 0x1100f000
 ; Make sure we generate flat store for HSA
 ; PRE-GFX10: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
-; GFX10: global_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}
+; GFX10: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off
 
 ; HSA: .Lfunc_end0:
 ; HSA: .size   simple, .Lfunc_end0-simple
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem-ds.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem-ds.mir
index 6ce629a0dc055911116847c67ad26308b44fdeb9..7b37990dfa45e597510e2ab53c7571fff5095f9e 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem-ds.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem-ds.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=polaris10 -run-pass si-remove-short-exec-branches -amdgpu-skip-threshold=1 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=polaris10 -run-pass si-pre-emit-peephole -amdgpu-skip-threshold=1 -verify-machineinstrs  %s -o - | FileCheck %s
 
 ---
 
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-gws.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-gws.mir
index 5424ad39b4d9c63140176aaa27522eaf50fd22ac..95b537367219777ac00d2a8d37225cad32219454 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-gws.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-gws.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass si-remove-short-exec-branches -amdgpu-skip-threshold=1 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass si-pre-emit-peephole -amdgpu-skip-threshold=1 -verify-machineinstrs  %s -o - | FileCheck %s
 # Make sure mandatory skips are inserted to ensure GWS ops aren't run with exec = 0
 
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-ignored-insts.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-ignored-insts.mir
index 928324492d51eb8eda1b79156b7c46053127d009..97c8b50c50cb0647cca47153cc155b78f6349771 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-ignored-insts.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-ignored-insts.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass si-remove-short-exec-branches -amdgpu-skip-threshold=3 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass si-pre-emit-peephole -amdgpu-skip-threshold=3 %s -o - | FileCheck %s
 
 ---
 
diff --git a/llvm/test/CodeGen/AMDGPU/kernarg-size.ll b/llvm/test/CodeGen/AMDGPU/kernarg-size.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ff409b938319beddfbd24f44c2df6ff04b76bc0b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/kernarg-size.ll
@@ -0,0 +1,21 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefix=HSA %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefix=HSA %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 < %s | FileCheck --check-prefix=HSA %s
+
+declare void @llvm.trap() #0
+declare void @llvm.debugtrap() #1
+
+; HSA:      .amdhsa_kernel trap
+; HSA-NEXT:     .amdhsa_group_segment_fixed_size 0
+; HSA-NEXT:     .amdhsa_private_segment_fixed_size 0
+; HSA-NEXT:     .amdhsa_kernarg_size 8
+; HSA-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
+; HSA:      .end_amdhsa_kernel
+
+define amdgpu_kernel void @trap(i32 addrspace(1)* nocapture readonly %arg0) {
+  store volatile i32 1, i32 addrspace(1)* %arg0
+  call void @llvm.trap()
+  unreachable
+  store volatile i32 2, i32 addrspace(1)* %arg0
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll b/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll
index 702ba881be89a22c95da3a454f84d1de54f05bea..8c9bdff36873921026607d630ad4c10a73e66f1e 100644
--- a/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/kill-infinite-loop.ll
@@ -50,7 +50,7 @@ end:
 ; CHECK-LABEL: only_kill
 ; CHECK: exp null off, off, off, off done vm
 ; CHECK-NEXT: s_endpgm
-; SIInsertSkips inserts an extra null export here, but it should be harmless.
+; SILateBranchLowering inserts an extra null export here, but it should be harmless.
 ; CHECK: exp null off, off, off, off done vm
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @only_kill() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/large-alloca-compute.ll b/llvm/test/CodeGen/AMDGPU/large-alloca-compute.ll
index 7a784611df8f49948f3dca2c961fc571a2c5994d..440214deb8ee07c6d71adb8ef3a0e0b84834913f 100644
--- a/llvm/test/CodeGen/AMDGPU/large-alloca-compute.ll
+++ b/llvm/test/CodeGen/AMDGPU/large-alloca-compute.ll
@@ -1,9 +1,9 @@
 ; RUN: llc -march=amdgcn -mcpu=bonaire -show-mc-encoding < %s | FileCheck --check-prefixes=GCN,CI,ALL %s
 ; RUN: llc -march=amdgcn -mcpu=carrizo --show-mc-encoding < %s | FileCheck --check-prefixes=GCN,VI,ALL %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 --show-mc-encoding < %s | FileCheck --check-prefixes=GCN,GFX9,ALL %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -mtriple=amdgcn-unknown-amdhsa --amdhsa-code-object-version=2 < %s -mattr=-flat-for-global | FileCheck --check-prefixes=GCNHSA,ALL %s
-; RUN: llc -march=amdgcn -mcpu=carrizo -mtriple=amdgcn-unknown-amdhsa --amdhsa-code-object-version=2 -mattr=-flat-for-global < %s | FileCheck --check-prefixes=GCNHSA,ALL %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mtriple=amdgcn-unknown-amdhsa --amdhsa-code-object-version=2 -mattr=-flat-for-global < %s | FileCheck --check-prefixes=GCNHSA,GFX10HSA,ALL %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -mtriple=amdgcn-unknown-amdhsa --amdhsa-code-object-version=4 < %s -mattr=-flat-for-global | FileCheck --check-prefixes=GCNHSA,ALL %s
+; RUN: llc -march=amdgcn -mcpu=carrizo -mtriple=amdgcn-unknown-amdhsa --amdhsa-code-object-version=4 -mattr=-flat-for-global < %s | FileCheck --check-prefixes=GCNHSA,ALL %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mtriple=amdgcn-unknown-amdhsa --amdhsa-code-object-version=4 -mattr=-flat-for-global < %s | FileCheck --check-prefixes=GCNHSA,GFX10HSA,ALL %s
 
 ; FIXME: align on alloca seems to be ignored for private_segment_alignment
 
@@ -19,30 +19,6 @@
 ; GFX9-DAG: s_mov_b32 s{{[0-9]+}}, 0xe00000
 
 
-; GCNHSA: .amd_kernel_code_t
-
-; GCNHSA: enable_sgpr_private_segment_wave_byte_offset = 1
-; GCNHSA: user_sgpr_count = 8
-; GCNHSA: enable_sgpr_workgroup_id_x = 1
-; GCNHSA: enable_sgpr_workgroup_id_y = 0
-; GCNHSA: enable_sgpr_workgroup_id_z = 0
-; GCNHSA: enable_sgpr_workgroup_info = 0
-; GCNHSA: enable_vgpr_workitem_id = 0
-
-; GCNHSA: enable_sgpr_private_segment_buffer = 1
-; GCNHSA: enable_sgpr_dispatch_ptr = 0
-; GCNHSA: enable_sgpr_queue_ptr = 0
-; GCNHSA: enable_sgpr_kernarg_segment_ptr = 1
-; GCNHSA: enable_sgpr_dispatch_id = 0
-; GCNHSA: enable_sgpr_flat_scratch_init = 1
-; GCNHSA: enable_sgpr_private_segment_size = 0
-; GCNHSA: enable_sgpr_grid_workgroup_count_x = 0
-; GCNHSA: enable_sgpr_grid_workgroup_count_y = 0
-; GCNHSA: enable_sgpr_grid_workgroup_count_z = 0
-; GCNHSA: workitem_private_segment_byte_size = 32772
-; GCNHSA: private_segment_alignment = 4
-; GCNHSA: .end_amd_kernel_code_t
-
 ; GFX10HSA: s_add_u32 [[FLAT_SCR_LO:s[0-9]+]], s{{[0-9]+}}, s{{[0-9]+}}
 ; GFX10HSA-DAG: s_addc_u32 [[FLAT_SCR_HI:s[0-9]+]], s{{[0-9]+}}, 0
 ; GFX10HSA-DAG: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), [[FLAT_SCR_LO]]
@@ -51,6 +27,39 @@
 ; GCNHSA: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, s[0:3], 0 offen
 ; GCNHSA: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, s[0:3], 0 offen
 
+; GCNHSA: .amdhsa_kernel large_alloca_compute_shader
+; GCNHSA:         .amdhsa_group_segment_fixed_size 0
+; GCNHSA:         .amdhsa_private_segment_fixed_size 32772
+; GCNHSA:         .amdhsa_user_sgpr_private_segment_buffer 1
+; GCNHSA:         .amdhsa_user_sgpr_dispatch_ptr 0
+; GCNHSA:         .amdhsa_user_sgpr_queue_ptr 0
+; GCNHSA:         .amdhsa_user_sgpr_kernarg_segment_ptr 1
+; GCNHSA:         .amdhsa_user_sgpr_dispatch_id 0
+; GCNHSA:         .amdhsa_user_sgpr_flat_scratch_init 1
+; GCNHSA:         .amdhsa_user_sgpr_private_segment_size 0
+; GCNHSA:         .amdhsa_system_sgpr_private_segment_wavefront_offset 1
+; GCNHSA:         .amdhsa_system_sgpr_workgroup_id_x 1
+; GCNHSA:         .amdhsa_system_sgpr_workgroup_id_y 0
+; GCNHSA:         .amdhsa_system_sgpr_workgroup_id_z 0
+; GCNHSA:         .amdhsa_system_sgpr_workgroup_info 0
+; GCNHSA:         .amdhsa_system_vgpr_workitem_id 0
+; GCNHSA:         .amdhsa_next_free_vgpr 3
+; GCNHSA:         .amdhsa_next_free_sgpr 10
+; GCNHSA:         .amdhsa_float_round_mode_32 0
+; GCNHSA:         .amdhsa_float_round_mode_16_64 0
+; GCNHSA:         .amdhsa_float_denorm_mode_32 3
+; GCNHSA:         .amdhsa_float_denorm_mode_16_64 3
+; GCNHSA:         .amdhsa_dx10_clamp 1
+; GCNHSA:         .amdhsa_ieee_mode 1
+; GCNHSA:         .amdhsa_exception_fp_ieee_invalid_op 0
+; GCNHSA:         .amdhsa_exception_fp_denorm_src 0
+; GCNHSA:         .amdhsa_exception_fp_ieee_div_zero 0
+; GCNHSA:         .amdhsa_exception_fp_ieee_overflow 0
+; GCNHSA:         .amdhsa_exception_fp_ieee_underflow 0
+; GCNHSA:         .amdhsa_exception_fp_ieee_inexact 0
+; GCNHSA:         .amdhsa_exception_int_div_zero 0
+; GCNHSA: .end_amdhsa_kernel
+
 ; Scratch size = alloca size + emergency stack slot, align {{.*}}, addrspace(5)
 ; ALL: ; ScratchSize: 32772
 define amdgpu_kernel void @large_alloca_compute_shader(i32 %x, i32 %y) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll b/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll
index 823c6a001ff99a34410e2da31f37b7b6084bcca3..cd59bb42d6e1a57d41308b64d3c753ac4b12b59b 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -o - -amdgpu-disable-lower-module-lds=true %s 2> %t | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -o - -amdgpu-disable-lower-module-lds=true %s 2> %t | FileCheck -check-prefixes=GFX8 %s
 ; RUN: FileCheck -check-prefix=ERR %s < %t
 
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - -amdgpu-disable-lower-module-lds=true %s 2> %t | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - -amdgpu-disable-lower-module-lds=true %s 2> %t | FileCheck -check-prefixes=GFX9 %s
 ; RUN: FileCheck -check-prefix=ERR %s < %t
 
 @lds = internal addrspace(3) global float undef, align 4
@@ -25,7 +25,6 @@ define void @func_use_lds_global() {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    ds_write_b32 v0, v0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[6:7]
 ; GFX9-NEXT:    s_trap 2
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -35,12 +34,18 @@ define void @func_use_lds_global() {
 
 ; ERR: warning: <unknown>:0:0: in function func_use_lds_global_constexpr_cast void (): local memory global used by non-kernel function
 define void @func_use_lds_global_constexpr_cast() {
-; GCN-LABEL: func_use_lds_global_constexpr_cast:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_mov_b64 s[0:1], s[6:7]
-; GCN-NEXT:    s_trap 2
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GFX8-LABEL: func_use_lds_global_constexpr_cast:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b64 s[0:1], s[6:7]
+; GFX8-NEXT:    s_trap 2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: func_use_lds_global_constexpr_cast:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_trap 2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
   store i32 ptrtoint (float addrspace(3)* @lds to i32), i32 addrspace(1)* undef, align 4
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.demote.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.demote.ll
index 0b0fb98cacb8eb72cbecbedadfc705180d286b54..9edd1a397b78d3e94f20a66716f1659573063ac4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.demote.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.demote.ll
@@ -167,12 +167,13 @@ define amdgpu_ps void @branch(float %arg0, float %arg1) {
 ; SI-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v0
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], s[0:1]
 ; SI-NEXT:    s_xor_b64 s[0:1], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execz BB2_3
 ; SI-NEXT:  ; %bb.1: ; %.demote
 ; SI-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
 ; SI-NEXT:    s_cbranch_scc0 BB2_4
 ; SI-NEXT:  ; %bb.2: ; %.demote
 ; SI-NEXT:    s_mov_b64 exec, 0
-; SI-NEXT:  ; %bb.3: ; %.continue
+; SI-NEXT:  BB2_3: ; %.continue
 ; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
 ; SI-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
@@ -194,12 +195,13 @@ define amdgpu_ps void @branch(float %arg0, float %arg1) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz BB2_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote
 ; GFX9-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB2_4
 ; GFX9-NEXT:  ; %bb.2: ; %.demote
 ; GFX9-NEXT:    s_mov_b64 exec, 0
-; GFX9-NEXT:  ; %bb.3: ; %.continue
+; GFX9-NEXT:  BB2_3: ; %.continue
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
 ; GFX9-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
@@ -221,12 +223,13 @@ define amdgpu_ps void @branch(float %arg0, float %arg1) {
 ; GFX10-32-NEXT:    v_cmp_eq_u32_e64 s0, 1, v0
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s2, s0
 ; GFX10-32-NEXT:    s_xor_b32 s0, exec_lo, s2
+; GFX10-32-NEXT:    s_cbranch_execz BB2_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-32-NEXT:    s_andn2_b32 s1, s1, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB2_4
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-32-NEXT:    s_mov_b32 exec_lo, 0
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue
+; GFX10-32-NEXT:  BB2_3: ; %.continue
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX10-32-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
 ; GFX10-32-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
@@ -248,12 +251,13 @@ define amdgpu_ps void @branch(float %arg0, float %arg1) {
 ; GFX10-64-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v0
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[4:5], s[0:1]
 ; GFX10-64-NEXT:    s_xor_b64 s[0:1], exec, s[4:5]
+; GFX10-64-NEXT:    s_cbranch_execz BB2_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-64-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB2_4
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-64-NEXT:    s_mov_b64 exec, 0
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue
+; GFX10-64-NEXT:  BB2_3: ; %.continue
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX10-64-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
 ; GFX10-64-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
@@ -289,13 +293,14 @@ define amdgpu_ps <4 x float> @wqm_demote_1(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; SI-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v1
 ; SI-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; SI-NEXT:    s_xor_b64 s[14:15], exec, s[14:15]
+; SI-NEXT:    s_cbranch_execz BB3_3
 ; SI-NEXT:  ; %bb.1: ; %.demote
 ; SI-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; SI-NEXT:    s_cbranch_scc0 BB3_4
 ; SI-NEXT:  ; %bb.2: ; %.demote
 ; SI-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; SI-NEXT:    s_and_b64 exec, exec, s[16:17]
-; SI-NEXT:  ; %bb.3: ; %.continue
+; SI-NEXT:  BB3_3: ; %.continue
 ; SI-NEXT:    s_or_b64 exec, exec, s[14:15]
 ; SI-NEXT:    image_sample v0, v0, s[0:7], s[8:11] dmask:0x1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -317,13 +322,14 @@ define amdgpu_ps <4 x float> @wqm_demote_1(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX9-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; GFX9-NEXT:    s_xor_b64 s[14:15], exec, s[14:15]
+; GFX9-NEXT:    s_cbranch_execz BB3_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote
 ; GFX9-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB3_4
 ; GFX9-NEXT:  ; %bb.2: ; %.demote
 ; GFX9-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[16:17]
-; GFX9-NEXT:  ; %bb.3: ; %.continue
+; GFX9-NEXT:  BB3_3: ; %.continue
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[14:15]
 ; GFX9-NEXT:    image_sample v0, v0, s[0:7], s[8:11] dmask:0x1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -345,13 +351,14 @@ define amdgpu_ps <4 x float> @wqm_demote_1(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-32-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, 0, v1
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s13, vcc_lo
 ; GFX10-32-NEXT:    s_xor_b32 s13, exec_lo, s13
+; GFX10-32-NEXT:    s_cbranch_execz BB3_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-32-NEXT:    s_andn2_b32 s12, s12, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB3_4
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-32-NEXT:    s_wqm_b32 s28, s12
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s28
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue
+; GFX10-32-NEXT:  BB3_3: ; %.continue
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s13
 ; GFX10-32-NEXT:    image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
 ; GFX10-32-NEXT:    s_waitcnt vmcnt(0)
@@ -373,13 +380,14 @@ define amdgpu_ps <4 x float> @wqm_demote_1(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-64-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v1
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; GFX10-64-NEXT:    s_xor_b64 s[28:29], exec, s[14:15]
+; GFX10-64-NEXT:    s_cbranch_execz BB3_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-64-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB3_4
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-64-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[16:17]
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue
+; GFX10-64-NEXT:  BB3_3: ; %.continue
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[28:29]
 ; GFX10-64-NEXT:    image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D
 ; GFX10-64-NEXT:    s_waitcnt vmcnt(0)
@@ -421,13 +429,14 @@ define amdgpu_ps <4 x float> @wqm_demote_2(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; SI-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
 ; SI-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; SI-NEXT:    s_xor_b64 s[14:15], exec, s[14:15]
+; SI-NEXT:    s_cbranch_execz BB4_3
 ; SI-NEXT:  ; %bb.1: ; %.demote
 ; SI-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; SI-NEXT:    s_cbranch_scc0 BB4_4
 ; SI-NEXT:  ; %bb.2: ; %.demote
 ; SI-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; SI-NEXT:    s_and_b64 exec, exec, s[16:17]
-; SI-NEXT:  ; %bb.3: ; %.continue
+; SI-NEXT:  BB4_3: ; %.continue
 ; SI-NEXT:    s_or_b64 exec, exec, s[14:15]
 ; SI-NEXT:    v_add_f32_e32 v0, v0, v0
 ; SI-NEXT:    s_and_b64 exec, exec, s[12:13]
@@ -449,13 +458,14 @@ define amdgpu_ps <4 x float> @wqm_demote_2(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX9-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; GFX9-NEXT:    s_xor_b64 s[14:15], exec, s[14:15]
+; GFX9-NEXT:    s_cbranch_execz BB4_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote
 ; GFX9-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB4_4
 ; GFX9-NEXT:  ; %bb.2: ; %.demote
 ; GFX9-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[16:17]
-; GFX9-NEXT:  ; %bb.3: ; %.continue
+; GFX9-NEXT:  BB4_3: ; %.continue
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[14:15]
 ; GFX9-NEXT:    v_add_f32_e32 v0, v0, v0
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[12:13]
@@ -477,13 +487,14 @@ define amdgpu_ps <4 x float> @wqm_demote_2(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-32-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, 0, v0
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s13, vcc_lo
 ; GFX10-32-NEXT:    s_xor_b32 s13, exec_lo, s13
+; GFX10-32-NEXT:    s_cbranch_execz BB4_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-32-NEXT:    s_andn2_b32 s12, s12, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB4_4
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-32-NEXT:    s_wqm_b32 s28, s12
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s28
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue
+; GFX10-32-NEXT:  BB4_3: ; %.continue
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s13
 ; GFX10-32-NEXT:    v_add_f32_e32 v0, v0, v0
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s12
@@ -505,13 +516,14 @@ define amdgpu_ps <4 x float> @wqm_demote_2(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-64-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[14:15], vcc
 ; GFX10-64-NEXT:    s_xor_b64 s[28:29], exec, s[14:15]
+; GFX10-64-NEXT:    s_cbranch_execz BB4_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote
 ; GFX10-64-NEXT:    s_andn2_b64 s[12:13], s[12:13], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB4_4
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote
 ; GFX10-64-NEXT:    s_wqm_b64 s[16:17], s[12:13]
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[16:17]
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue
+; GFX10-64-NEXT:  BB4_3: ; %.continue
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[28:29]
 ; GFX10-64-NEXT:    v_add_f32_e32 v0, v0, v0
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[12:13]
@@ -659,13 +671,14 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; SI-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; SI-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; SI-NEXT:    s_cbranch_execz BB6_3
 ; SI-NEXT:  ; %bb.1: ; %.demote0
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 BB6_7
 ; SI-NEXT:  ; %bb.2: ; %.demote0
 ; SI-NEXT:    s_wqm_b64 s[4:5], s[0:1]
 ; SI-NEXT:    s_and_b64 exec, exec, s[4:5]
-; SI-NEXT:  ; %bb.3: ; %.continue0
+; SI-NEXT:  BB6_3: ; %.continue0
 ; SI-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; SI-NEXT:    s_mov_b64 s[2:3], s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0, s[2:3]
@@ -681,12 +694,13 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; SI-NEXT:    s_or_b64 s[2:3], s[2:3], vcc
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
 ; SI-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execz BB6_6
 ; SI-NEXT:  ; %bb.4: ; %.demote1
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 BB6_7
 ; SI-NEXT:  ; %bb.5: ; %.demote1
 ; SI-NEXT:    s_mov_b64 exec, 0
-; SI-NEXT:  ; %bb.6: ; %.continue1
+; SI-NEXT:  BB6_6: ; %.continue1
 ; SI-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; SI-NEXT:    v_bfrev_b32_e32 v0, 60
 ; SI-NEXT:    v_mov_b32_e32 v1, 0x3c00
@@ -705,13 +719,14 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX9-NEXT:    s_cbranch_execz BB6_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote0
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX9-NEXT:  ; %bb.2: ; %.demote0
 ; GFX9-NEXT:    s_wqm_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[4:5]
-; GFX9-NEXT:  ; %bb.3: ; %.continue0
+; GFX9-NEXT:  BB6_3: ; %.continue0
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_mov_b64 s[2:3], s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0, s[2:3]
@@ -727,12 +742,13 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], vcc
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
 ; GFX9-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz BB6_6
 ; GFX9-NEXT:  ; %bb.4: ; %.demote1
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX9-NEXT:  ; %bb.5: ; %.demote1
 ; GFX9-NEXT:    s_mov_b64 exec, 0
-; GFX9-NEXT:  ; %bb.6: ; %.continue1
+; GFX9-NEXT:  BB6_6: ; %.continue1
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; GFX9-NEXT:    v_bfrev_b32_e32 v1, 60
@@ -751,13 +767,14 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX10-32-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-32-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX10-32-NEXT:    s_cbranch_execz BB6_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote0
 ; GFX10-32-NEXT:    s_andn2_b32 s0, s0, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote0
 ; GFX10-32-NEXT:    s_wqm_b32 s2, s0
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s2
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue0
+; GFX10-32-NEXT:  BB6_3: ; %.continue0
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX10-32-NEXT:    s_mov_b32 s1, s0
 ; GFX10-32-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0, s1
@@ -771,12 +788,13 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX10-32-NEXT:    s_or_b32 s1, s1, vcc_lo
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s2, s1
 ; GFX10-32-NEXT:    s_xor_b32 s1, exec_lo, s2
+; GFX10-32-NEXT:    s_cbranch_execz BB6_6
 ; GFX10-32-NEXT:  ; %bb.4: ; %.demote1
 ; GFX10-32-NEXT:    s_andn2_b32 s0, s0, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX10-32-NEXT:  ; %bb.5: ; %.demote1
 ; GFX10-32-NEXT:    s_mov_b32 exec_lo, 0
-; GFX10-32-NEXT:  ; %bb.6: ; %.continue1
+; GFX10-32-NEXT:  BB6_6: ; %.continue1
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s1
 ; GFX10-32-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; GFX10-32-NEXT:    v_bfrev_b32_e32 v1, 60
@@ -795,13 +813,14 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX10-64-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX10-64-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX10-64-NEXT:    s_cbranch_execz BB6_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote0
 ; GFX10-64-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote0
 ; GFX10-64-NEXT:    s_wqm_b64 s[4:5], s[0:1]
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[4:5]
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue0
+; GFX10-64-NEXT:  BB6_3: ; %.continue0
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX10-64-NEXT:    s_mov_b64 s[2:3], s[0:1]
 ; GFX10-64-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0, s[2:3]
@@ -815,12 +834,13 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
 ; GFX10-64-NEXT:    s_or_b64 s[2:3], s[2:3], vcc
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
 ; GFX10-64-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; GFX10-64-NEXT:    s_cbranch_execz BB6_6
 ; GFX10-64-NEXT:  ; %bb.4: ; %.demote1
 ; GFX10-64-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB6_7
 ; GFX10-64-NEXT:  ; %bb.5: ; %.demote1
 ; GFX10-64-NEXT:    s_mov_b64 exec, 0
-; GFX10-64-NEXT:  ; %bb.6: ; %.continue1
+; GFX10-64-NEXT:  BB6_6: ; %.continue1
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX10-64-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; GFX10-64-NEXT:    v_bfrev_b32_e32 v1, 60
@@ -875,13 +895,14 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execz BB7_3
 ; SI-NEXT:  ; %bb.1: ; %.demote0
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 BB7_9
 ; SI-NEXT:  ; %bb.2: ; %.demote0
 ; SI-NEXT:    s_wqm_b64 s[6:7], s[0:1]
 ; SI-NEXT:    s_and_b64 exec, exec, s[6:7]
-; SI-NEXT:  ; %bb.3: ; %.continue0.preheader
+; SI-NEXT:  BB7_3: ; %.continue0.preheader
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_mov_b64 s[4:5], 0
 ; SI-NEXT:    s_branch BB7_5
@@ -940,13 +961,14 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz BB7_3
 ; GFX9-NEXT:  ; %bb.1: ; %.demote0
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX9-NEXT:    s_cbranch_scc0 BB7_9
 ; GFX9-NEXT:  ; %bb.2: ; %.demote0
 ; GFX9-NEXT:    s_wqm_b64 s[6:7], s[0:1]
 ; GFX9-NEXT:    s_and_b64 exec, exec, s[6:7]
-; GFX9-NEXT:  ; %bb.3: ; %.continue0.preheader
+; GFX9-NEXT:  BB7_3: ; %.continue0.preheader
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_branch BB7_5
@@ -1005,13 +1027,14 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX10-32-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GFX10-32-NEXT:    s_and_saveexec_b32 s2, vcc_lo
 ; GFX10-32-NEXT:    s_xor_b32 s2, exec_lo, s2
+; GFX10-32-NEXT:    s_cbranch_execz BB7_3
 ; GFX10-32-NEXT:  ; %bb.1: ; %.demote0
 ; GFX10-32-NEXT:    s_andn2_b32 s0, s0, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 BB7_9
 ; GFX10-32-NEXT:  ; %bb.2: ; %.demote0
 ; GFX10-32-NEXT:    s_wqm_b32 s3, s0
 ; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s3
-; GFX10-32-NEXT:  ; %bb.3: ; %.continue0.preheader
+; GFX10-32-NEXT:  BB7_3: ; %.continue0.preheader
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX10-32-NEXT:    s_mov_b32 s2, 0
 ; GFX10-32-NEXT:    s_branch BB7_5
@@ -1067,13 +1090,14 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX10-64-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX10-64-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX10-64-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX10-64-NEXT:    s_cbranch_execz BB7_3
 ; GFX10-64-NEXT:  ; %bb.1: ; %.demote0
 ; GFX10-64-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 BB7_9
 ; GFX10-64-NEXT:  ; %bb.2: ; %.demote0
 ; GFX10-64-NEXT:    s_wqm_b64 s[6:7], s[0:1]
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[6:7]
-; GFX10-64-NEXT:  ; %bb.3: ; %.continue0.preheader
+; GFX10-64-NEXT:  BB7_3: ; %.continue0.preheader
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX10-64-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX10-64-NEXT:    s_branch BB7_5
diff --git a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
index aedb081e75f0496bcb2cb36624ba439eb3903985..d40556e5de1261bf4c01cff00322cfd670919f27 100644
--- a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
@@ -2,8 +2,6 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefix=GFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefix=VI %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefix=CI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefix=CI %s
 
 define amdgpu_kernel void @s_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
 ; GFX9-LABEL: s_lshr_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/opt_exec_copy_fold.mir b/llvm/test/CodeGen/AMDGPU/opt_exec_copy_fold.mir
new file mode 100644
index 0000000000000000000000000000000000000000..4ebfa2cc9643a852552a8be2fdce3e1051e243b9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/opt_exec_copy_fold.mir
@@ -0,0 +1,23 @@
+# RUN: llc -run-pass si-optimize-exec-masking-pre-ra -march=amdgcn -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN %s
+---
+# GCN-LABEL: name: opt_exec_copy_fold
+# GCN:             %2:vreg_64 = COPY $exec
+name:            opt_exec_copy_fold
+tracksRegLiveness: true
+liveins:
+  - { reg: '$sgpr0_sgpr1' }
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+
+    %0:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr0_sgpr1, implicit $exec
+    %1:sreg_64 = V_CMP_NE_U32_e64 0, %0, implicit $exec
+    %2:vreg_64 = COPY $exec
+    %3:sreg_64 = V_CMP_EQ_U64_e64 %1, %2, implicit $exec
+    $scc = COPY %3
+    S_CBRANCH_SCC0 %bb.1, implicit $scc
+  
+  bb.1:
+...
+
+
diff --git a/llvm/test/CodeGen/AMDGPU/readlane_exec0.mir b/llvm/test/CodeGen/AMDGPU/readlane_exec0.mir
index e7660a14de916c6e168a21043e302362ceac5dec..597ac24cc53321015d39e8402eedc4b8e30f29f9 100644
--- a/llvm/test/CodeGen/AMDGPU/readlane_exec0.mir
+++ b/llvm/test/CodeGen/AMDGPU/readlane_exec0.mir
@@ -1,4 +1,4 @@
-# RUN: llc -o - %s -march=amdgcn -mcpu=fiji  -run-pass=si-insert-skips -verify-machineinstrs | FileCheck -check-prefix=GCN %s
+# RUN: llc -o - %s -march=amdgcn -mcpu=fiji -run-pass=si-late-branch-lowering -verify-machineinstrs | FileCheck -check-prefix=GCN %s
 
 # GCN-LABEL: readlane_exec0
 # GCN: bb.0
diff --git a/llvm/test/CodeGen/AMDGPU/rel32.ll b/llvm/test/CodeGen/AMDGPU/rel32.ll
index 4e9878eea23a8f0469282807fb8f918ad3f3d8d7..44e3b54d0e67b49c393b1f16cc0ac0c66491a536 100644
--- a/llvm/test/CodeGen/AMDGPU/rel32.ll
+++ b/llvm/test/CodeGen/AMDGPU/rel32.ll
@@ -1,11 +1,12 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck %s
 
 @g = protected local_unnamed_addr addrspace(4) externally_initialized global i32 0, align 4
 
 ; CHECK-LABEL: rel32_neg_offset:
 ; CHECK: s_getpc_b64 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{]}}
-; CHECK: s_add_u32 s[[LO]], s[[LO]], g@rel32@lo-4
-; CHECK: s_addc_u32 s[[HI]], s[[HI]], g@rel32@hi+4
+; CHECK-NEXT: s_add_u32 s[[LO]], s[[LO]], g@rel32@lo-4
+; CHECK-NEXT: s_addc_u32 s[[HI]], s[[HI]], g@rel32@hi+4
 define i32 addrspace(4)* @rel32_neg_offset() {
   %r = getelementptr i32, i32 addrspace(4)* @g, i64 -2
   ret i32 addrspace(4)* %r
diff --git a/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-gpr-idx-mode.mir b/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-gpr-idx-mode.mir
index 0f0d210799a92eb25767342f341aaafcdf93e7b2..3dddb0fef23033fdfd29a90dac4193e9f082269a 100644
--- a/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-gpr-idx-mode.mir
+++ b/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-gpr-idx-mode.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=si-remove-short-exec-branches -amdgpu-skip-threshold=10 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=si-pre-emit-peephole -amdgpu-skip-threshold=10 -verify-machineinstrs  %s -o - | FileCheck %s
 # Make sure mandatory skips are not removed around mode defs.
 # FIXME: -amdgpu-skip-threshold seems to be backwards.
 
diff --git a/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-special-instructions.mir b/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-special-instructions.mir
index ee72fa99a1297568ad298a6abecbce57290b4800..58b1ab9ace01a5598cfa18200d9b65662fbffc8f 100644
--- a/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-special-instructions.mir
+++ b/llvm/test/CodeGen/AMDGPU/remove-short-exec-branches-special-instructions.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=si-remove-short-exec-branches -amdgpu-skip-threshold=10 -verify-machineinstrs  %s -o - | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=si-pre-emit-peephole -amdgpu-skip-threshold=10 -verify-machineinstrs  %s -o - | FileCheck %s
 # Make sure mandatory skips are not removed around mode defs.
 # FIXME: -amdgpu-skip-threshold seems to be backwards.
 
diff --git a/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll b/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll
index 410ced9a517afd7b1d24ef11ad0f9a15753f7210..20943ffa05f10b6e7c519cde198d9e16561be4e4 100644
--- a/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -mattr=-flat-for-global,-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 
 ; TODO: Some of those tests fail with OS == amdhsa due to unreasonable register
 ;       allocation differences.
diff --git a/llvm/test/CodeGen/AMDGPU/s_mulk_i32.ll b/llvm/test/CodeGen/AMDGPU/s_mulk_i32.ll
index 86c0dcf3e9f6030da3193713d23cfd3bb6c30a72..25ab7201924e025190d572f9b4093fd4275a4b2b 100644
--- a/llvm/test/CodeGen/AMDGPU/s_mulk_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/s_mulk_i32.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -mattr=-flat-for-global,-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 
 ; SI-LABEL: {{^}}s_mulk_i32_k0:
 ; SI: s_load_dword [[VAL:s[0-9]+]]
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index 091959adcd71d6b6ed8381d1d2898b75ce7bc4d9..8ae54e8d91ae4d6f936d5eff242365b1d1a5966e 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -1202,15 +1202,14 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v5, 24, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
-; GCN-NEXT:    v_mul_hi_u32 v6, 24, v3
-; GCN-NEXT:    v_mul_hi_u32 v0, 0, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, 24, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, 24, v3
 ; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, s8, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s8, v0
@@ -1420,15 +1419,14 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) {
 ; GCN-NEXT:    v_addc_u32_e64 v4, vcc, v4, v6, s[4:5]
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; GCN-NEXT:    v_mul_hi_u32 v6, 24, v3
 ; GCN-NEXT:    v_mul_lo_u32 v5, v4, 24
-; GCN-NEXT:    v_mul_hi_u32 v7, 24, v4
-; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
+; GCN-NEXT:    v_mul_hi_u32 v3, 24, v3
+; GCN-NEXT:    v_mul_hi_u32 v6, 24, v4
 ; GCN-NEXT:    v_mul_hi_u32 v4, 0, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v13, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0, v5
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v13, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v4, v12, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
@@ -1633,15 +1631,14 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) {
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    s_mov_b32 s4, 0x8000
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; GCN-NEXT:    v_mul_hi_u32 v5, s4, v3
-; GCN-NEXT:    v_mul_hi_u32 v6, s4, v4
-; GCN-NEXT:    v_lshlrev_b32_e32 v7, 15, v4
-; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GCN-NEXT:    v_mul_hi_u32 v3, s4, v3
+; GCN-NEXT:    v_mul_hi_u32 v5, s4, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v6, 15, v4
 ; GCN-NEXT:    v_mul_hi_u32 v4, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v13, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, 0, v5
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v13, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 0, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v4, v12, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v5, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
diff --git a/llvm/test/CodeGen/AMDGPU/set-gpr-idx-peephole.mir b/llvm/test/CodeGen/AMDGPU/set-gpr-idx-peephole.mir
index c423f757c38d4030844cd6164519402dea5b1153..95f62e5e7cd3bff8d95ecefd61f8d197744cadec 100644
--- a/llvm/test/CodeGen/AMDGPU/set-gpr-idx-peephole.mir
+++ b/llvm/test/CodeGen/AMDGPU/set-gpr-idx-peephole.mir
@@ -356,3 +356,113 @@ body:             |
   V_MOV_B32_indirect undef $vgpr0, undef $vgpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3(tied-def 3)
   S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
 ...
+
+---
+name:            simple_bundle
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: simple_bundle
+    ; GCN: BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    ; GCN:   S_SET_GPR_IDX_ON $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    ; GCN:   $vgpr16 = V_MOV_B32_e32 undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    ; GCN: }
+    ; GCN: BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    ; GCN:   $vgpr15 = V_MOV_B32_e32 undef $vgpr0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    ; GCN:   S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+    ; GCN: }
+  BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    S_SET_GPR_IDX_ON $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    $vgpr16 = V_MOV_B32_e32 undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+  }
+  BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    S_SET_GPR_IDX_ON killed $sgpr2, 1, implicit-def $mode, implicit-def  $m0, implicit $mode, implicit undef $m0
+    $vgpr15 = V_MOV_B32_e32 undef $vgpr0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+  }
+...
+
+---
+name:            salu_in_between_bundle
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: salu_in_between_bundle
+    ; GCN: BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    ; GCN:   S_SET_GPR_IDX_ON $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    ; GCN:   $vgpr16 = V_MOV_B32_e32 undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    ; GCN: }
+    ; GCN: $sgpr0 = S_MOV_B32 $sgpr2
+    ; GCN: BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    ; GCN:   $vgpr15 = V_MOV_B32_e32 undef $vgpr0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    ; GCN:   S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+    ; GCN: }
+  BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    S_SET_GPR_IDX_ON $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    $vgpr16 = V_MOV_B32_e32 undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+  }
+  $sgpr0 = S_MOV_B32 $sgpr2
+  BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    S_SET_GPR_IDX_ON killed $sgpr2, 1, implicit-def $mode, implicit-def  $m0, implicit $mode, implicit undef $m0
+    $vgpr15 = V_MOV_B32_e32 undef $vgpr0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+  }
+...
+
+---
+name:            valu_in_between_bundle
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: valu_in_between_bundle
+    ; GCN: BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    ; GCN:   S_SET_GPR_IDX_ON $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    ; GCN:   $vgpr16 = V_MOV_B32_e32 undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    ; GCN:   S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+    ; GCN: }
+    ; GCN: $vgpr20 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN: BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    ; GCN:   S_SET_GPR_IDX_ON killed $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    ; GCN:   $vgpr15 = V_MOV_B32_e32 undef $vgpr0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    ; GCN:   S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+    ; GCN: }
+  BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    S_SET_GPR_IDX_ON $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    $vgpr16 = V_MOV_B32_e32 undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+  }
+  $vgpr20 = V_MOV_B32_e32 1, implicit $exec
+  BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    S_SET_GPR_IDX_ON killed $sgpr2, 1, implicit-def $mode, implicit-def  $m0, implicit $mode, implicit undef $m0
+    $vgpr15 = V_MOV_B32_e32 undef $vgpr0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+  }
+...
+
+---
+name:            changed_index_bundle
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: changed_index_bundle
+    ; GCN: BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    ; GCN:   S_SET_GPR_IDX_ON $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    ; GCN:   $vgpr16 = V_MOV_B32_e32 undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    ; GCN:   S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+    ; GCN: }
+    ; GCN: $sgpr2 = S_MOV_B32 1
+    ; GCN: BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    ; GCN:   S_SET_GPR_IDX_ON killed $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    ; GCN:   $vgpr15 = V_MOV_B32_e32 undef $vgpr0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    ; GCN:   S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+    ; GCN: }
+  BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    S_SET_GPR_IDX_ON $sgpr2, 1, implicit-def $mode, implicit-def $m0, implicit $mode, implicit undef $m0
+    $vgpr16 = V_MOV_B32_e32 undef $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+  }
+  $sgpr2 = S_MOV_B32 1
+  BUNDLE implicit-def $m0, implicit-def $m0_lo16, implicit-def $m0_hi16, implicit-def $mode, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit $sgpr2, implicit $m0, implicit $mode, implicit undef $vgpr0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 {
+    S_SET_GPR_IDX_ON killed $sgpr2, 1, implicit-def $mode, implicit-def  $m0, implicit $mode, implicit undef $m0
+    $vgpr15 = V_MOV_B32_e32 undef $vgpr0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+    S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
+  }
+...
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-carry.mir b/llvm/test/CodeGen/AMDGPU/shrink-carry.mir
index d828f0be4319af4426d6b945e93aa9b52eb18b44..74b81d8d29a35bf26d9c67674851efa3d8c73997 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-carry.mir
+++ b/llvm/test/CodeGen/AMDGPU/shrink-carry.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -start-before si-shrink-instructions -stop-before si-insert-skips -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -start-before si-shrink-instructions -stop-before si-late-branch-lowering -o - %s | FileCheck -check-prefix=GCN %s
 
 # GCN-LABEL: name: subbrev{{$}}
 # GCN:       V_SUBBREV_U32_e32 0, undef $vgpr0, implicit-def $vcc, implicit killed $vcc, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/skip-branch-taildup-ret.mir b/llvm/test/CodeGen/AMDGPU/skip-branch-taildup-ret.mir
index 5979720d0cc7a924a571345ae8ed233f9e5e4af7..4c53c51d1ce4e1bd9b0638d815925e9dd62aa116 100644
--- a/llvm/test/CodeGen/AMDGPU/skip-branch-taildup-ret.mir
+++ b/llvm/test/CodeGen/AMDGPU/skip-branch-taildup-ret.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -run-pass=si-remove-short-exec-branches -amdgpu-skip-threshold=1000000 -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -run-pass=si-pre-emit-peephole -amdgpu-skip-threshold=1000000 -o -  %s | FileCheck %s
 
 ---
 name: skip_branch_taildup_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
index f535e28c671850c98c8e62ad63f0d8037e728343..bc84fe8adab6040063d98fdbc4fa0b583aba8047 100644
--- a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
+++ b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
@@ -1002,13 +1002,14 @@ define amdgpu_ps void @if_after_kill_block(float %arg, float %arg1, float %arg2,
 ; SI-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v1
 ; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execz BB13_3
 ; SI-NEXT:  ; %bb.1: ; %bb3
 ; SI-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
 ; SI-NEXT:    s_andn2_b64 s[2:3], s[2:3], vcc
 ; SI-NEXT:    s_cbranch_scc0 BB13_6
 ; SI-NEXT:  ; %bb.2: ; %bb3
 ; SI-NEXT:    s_andn2_b64 exec, exec, vcc
-; SI-NEXT:  ; %bb.3: ; %bb4
+; SI-NEXT:  BB13_3: ; %bb4
 ; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SI-NEXT:    s_mov_b32 s1, s0
 ; SI-NEXT:    s_mov_b32 s2, s0
@@ -1043,13 +1044,14 @@ define amdgpu_ps void @if_after_kill_block(float %arg, float %arg1, float %arg2,
 ; GFX10-WAVE64-NEXT:    s_mov_b32 s0, 0
 ; GFX10-WAVE64-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX10-WAVE64-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX10-WAVE64-NEXT:    s_cbranch_execz BB13_3
 ; GFX10-WAVE64-NEXT:  ; %bb.1: ; %bb3
 ; GFX10-WAVE64-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
 ; GFX10-WAVE64-NEXT:    s_andn2_b64 s[2:3], s[2:3], vcc
 ; GFX10-WAVE64-NEXT:    s_cbranch_scc0 BB13_6
 ; GFX10-WAVE64-NEXT:  ; %bb.2: ; %bb3
 ; GFX10-WAVE64-NEXT:    s_andn2_b64 exec, exec, vcc
-; GFX10-WAVE64-NEXT:  ; %bb.3: ; %bb4
+; GFX10-WAVE64-NEXT:  BB13_3: ; %bb4
 ; GFX10-WAVE64-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX10-WAVE64-NEXT:    s_mov_b32 s1, s0
 ; GFX10-WAVE64-NEXT:    s_mov_b32 s2, s0
@@ -1082,13 +1084,14 @@ define amdgpu_ps void @if_after_kill_block(float %arg, float %arg1, float %arg2,
 ; GFX10-WAVE32-NEXT:    s_mov_b32 s0, 0
 ; GFX10-WAVE32-NEXT:    s_and_saveexec_b32 s2, vcc_lo
 ; GFX10-WAVE32-NEXT:    s_xor_b32 s2, exec_lo, s2
+; GFX10-WAVE32-NEXT:    s_cbranch_execz BB13_3
 ; GFX10-WAVE32-NEXT:  ; %bb.1: ; %bb3
 ; GFX10-WAVE32-NEXT:    v_cmp_ngt_f32_e32 vcc_lo, 0, v0
 ; GFX10-WAVE32-NEXT:    s_andn2_b32 s1, s1, vcc_lo
 ; GFX10-WAVE32-NEXT:    s_cbranch_scc0 BB13_6
 ; GFX10-WAVE32-NEXT:  ; %bb.2: ; %bb3
 ; GFX10-WAVE32-NEXT:    s_andn2_b32 exec_lo, exec_lo, vcc_lo
-; GFX10-WAVE32-NEXT:  ; %bb.3: ; %bb4
+; GFX10-WAVE32-NEXT:  BB13_3: ; %bb4
 ; GFX10-WAVE32-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX10-WAVE32-NEXT:    s_mov_b32 s1, s0
 ; GFX10-WAVE32-NEXT:    s_mov_b32 s2, s0
@@ -1154,12 +1157,13 @@ define amdgpu_ps void @cbranch_kill(i32 inreg %0, float %val0, float %val1) {
 ; SI-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v1
 ; SI-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; SI-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; SI-NEXT:    s_cbranch_execz BB14_3
 ; SI-NEXT:  ; %bb.1: ; %kill
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 BB14_6
 ; SI-NEXT:  ; %bb.2: ; %kill
 ; SI-NEXT:    s_mov_b64 exec, 0
-; SI-NEXT:  ; %bb.3: ; %Flow
+; SI-NEXT:  BB14_3: ; %Flow
 ; SI-NEXT:    s_or_saveexec_b64 s[0:1], s[2:3]
 ; SI-NEXT:    ; implicit-def: $vgpr2
 ; SI-NEXT:    s_xor_b64 exec, exec, s[0:1]
@@ -1167,7 +1171,7 @@ define amdgpu_ps void @cbranch_kill(i32 inreg %0, float %val0, float %val1) {
 ; SI-NEXT:    v_mul_f32_e32 v2, v0, v1
 ; SI-NEXT:  ; %bb.5: ; %export
 ; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
-; SI-NEXT:    exp mrt0 v2, v2, v2, v2 done compr vm
+; SI-NEXT:    exp mrt0 v2, v2, v2, v2 done vm
 ; SI-NEXT:    s_endpgm
 ; SI-NEXT:  BB14_6:
 ; SI-NEXT:    s_mov_b64 exec, 0
@@ -1190,12 +1194,13 @@ define amdgpu_ps void @cbranch_kill(i32 inreg %0, float %val0, float %val1) {
 ; GFX10-WAVE64-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v1
 ; GFX10-WAVE64-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX10-WAVE64-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX10-WAVE64-NEXT:    s_cbranch_execz BB14_3
 ; GFX10-WAVE64-NEXT:  ; %bb.1: ; %kill
 ; GFX10-WAVE64-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX10-WAVE64-NEXT:    s_cbranch_scc0 BB14_6
 ; GFX10-WAVE64-NEXT:  ; %bb.2: ; %kill
 ; GFX10-WAVE64-NEXT:    s_mov_b64 exec, 0
-; GFX10-WAVE64-NEXT:  ; %bb.3: ; %Flow
+; GFX10-WAVE64-NEXT:  BB14_3: ; %Flow
 ; GFX10-WAVE64-NEXT:    s_or_saveexec_b64 s[0:1], s[2:3]
 ; GFX10-WAVE64-NEXT:    ; implicit-def: $vgpr2
 ; GFX10-WAVE64-NEXT:    s_xor_b64 exec, exec, s[0:1]
@@ -1203,7 +1208,7 @@ define amdgpu_ps void @cbranch_kill(i32 inreg %0, float %val0, float %val1) {
 ; GFX10-WAVE64-NEXT:    v_mul_f32_e32 v2, v0, v1
 ; GFX10-WAVE64-NEXT:  ; %bb.5: ; %export
 ; GFX10-WAVE64-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX10-WAVE64-NEXT:    exp mrt0 v2, v2, v2, v2 done compr vm
+; GFX10-WAVE64-NEXT:    exp mrt0 v2, v2, v2, v2 done vm
 ; GFX10-WAVE64-NEXT:    s_endpgm
 ; GFX10-WAVE64-NEXT:  BB14_6:
 ; GFX10-WAVE64-NEXT:    s_mov_b64 exec, 0
@@ -1226,12 +1231,13 @@ define amdgpu_ps void @cbranch_kill(i32 inreg %0, float %val0, float %val1) {
 ; GFX10-WAVE32-NEXT:    v_cmp_ge_f32_e32 vcc_lo, 0, v1
 ; GFX10-WAVE32-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-WAVE32-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX10-WAVE32-NEXT:    s_cbranch_execz BB14_3
 ; GFX10-WAVE32-NEXT:  ; %bb.1: ; %kill
 ; GFX10-WAVE32-NEXT:    s_andn2_b32 s0, s0, exec_lo
 ; GFX10-WAVE32-NEXT:    s_cbranch_scc0 BB14_6
 ; GFX10-WAVE32-NEXT:  ; %bb.2: ; %kill
 ; GFX10-WAVE32-NEXT:    s_mov_b32 exec_lo, 0
-; GFX10-WAVE32-NEXT:  ; %bb.3: ; %Flow
+; GFX10-WAVE32-NEXT:  BB14_3: ; %Flow
 ; GFX10-WAVE32-NEXT:    s_or_saveexec_b32 s0, s1
 ; GFX10-WAVE32-NEXT:    ; implicit-def: $vgpr2
 ; GFX10-WAVE32-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
@@ -1239,7 +1245,7 @@ define amdgpu_ps void @cbranch_kill(i32 inreg %0, float %val0, float %val1) {
 ; GFX10-WAVE32-NEXT:    v_mul_f32_e32 v2, v0, v1
 ; GFX10-WAVE32-NEXT:  ; %bb.5: ; %export
 ; GFX10-WAVE32-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX10-WAVE32-NEXT:    exp mrt0 v2, v2, v2, v2 done compr vm
+; GFX10-WAVE32-NEXT:    exp mrt0 v2, v2, v2, v2 done vm
 ; GFX10-WAVE32-NEXT:    s_endpgm
 ; GFX10-WAVE32-NEXT:  BB14_6:
 ; GFX10-WAVE32-NEXT:    s_mov_b32 exec_lo, 0
@@ -1260,8 +1266,7 @@ live:
 
 export:
   %proxy = phi float [ undef, %kill ], [ %scale, %live ]
-  %proxyh = bitcast float %proxy to <2 x half>
-  call void @llvm.amdgcn.exp.compr.v2f16(i32 immarg 0, i32 immarg 15, <2 x half> %proxyh, <2 x half> %proxyh, i1 immarg true, i1 immarg true) #3
+  call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 15, float %proxy, float %proxy, float %proxy, float %proxy, i1 immarg true, i1 immarg true) #3
   ret void
 }
 
@@ -1303,7 +1308,7 @@ define amdgpu_ps void @complex_loop(i32 inreg %cmpa, i32 %cmpb, i32 %cmpc) {
 ; SI-NEXT:  BB15_6: ; %Flow
 ; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; SI-NEXT:  BB15_7: ; %._crit_edge
-; SI-NEXT:    exp mrt0 v2, v2, v0, v0 done compr vm
+; SI-NEXT:    exp mrt0 v2, v2, v0, v0 done vm
 ; SI-NEXT:    s_endpgm
 ; SI-NEXT:  BB15_8:
 ; SI-NEXT:    s_mov_b64 exec, 0
@@ -1346,7 +1351,7 @@ define amdgpu_ps void @complex_loop(i32 inreg %cmpa, i32 %cmpb, i32 %cmpc) {
 ; GFX10-WAVE64-NEXT:  BB15_6: ; %Flow
 ; GFX10-WAVE64-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX10-WAVE64-NEXT:  BB15_7: ; %._crit_edge
-; GFX10-WAVE64-NEXT:    exp mrt0 v2, v2, v0, v0 done compr vm
+; GFX10-WAVE64-NEXT:    exp mrt0 v2, v2, v0, v0 done vm
 ; GFX10-WAVE64-NEXT:    s_endpgm
 ; GFX10-WAVE64-NEXT:  BB15_8:
 ; GFX10-WAVE64-NEXT:    s_mov_b64 exec, 0
@@ -1389,7 +1394,7 @@ define amdgpu_ps void @complex_loop(i32 inreg %cmpa, i32 %cmpb, i32 %cmpc) {
 ; GFX10-WAVE32-NEXT:  BB15_6: ; %Flow
 ; GFX10-WAVE32-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX10-WAVE32-NEXT:  BB15_7: ; %._crit_edge
-; GFX10-WAVE32-NEXT:    exp mrt0 v2, v2, v0, v0 done compr vm
+; GFX10-WAVE32-NEXT:    exp mrt0 v2, v2, v0, v0 done vm
 ; GFX10-WAVE32-NEXT:    s_endpgm
 ; GFX10-WAVE32-NEXT:  BB15_8:
 ; GFX10-WAVE32-NEXT:    s_mov_b32 exec_lo, 0
@@ -1418,8 +1423,8 @@ latch:
 
 ._crit_edge:
   %tmp = phi i32 [ -1, %.entry ], [ %ctr.next, %latch ]
-  %out = bitcast i32 %tmp to <2 x half>
-  call void @llvm.amdgcn.exp.compr.v2f16(i32 immarg 0, i32 immarg 15, <2 x half> %out, <2 x half> undef, i1 immarg true, i1 immarg true)
+  %out = bitcast i32 %tmp to float
+  call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 15, float %out, float %out, float undef, float undef, i1 immarg true, i1 immarg true)
   ret void
 }
 
@@ -1473,7 +1478,7 @@ bb.1:
   ret void
 }
 
-declare void @llvm.amdgcn.exp.compr.v2f16(i32 immarg, i32 immarg, <2 x half>, <2 x half>, i1 immarg, i1 immarg) #3
+declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg) #3
 declare float @llvm.amdgcn.image.sample.l.2darray.f32.f32(i32 immarg, float, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #1
 declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare void @llvm.amdgcn.kill(i1) #0
diff --git a/llvm/test/CodeGen/AMDGPU/smed3.ll b/llvm/test/CodeGen/AMDGPU/smed3.ll
index 16aff2edba956469e6e564fa4e12be08d5b5d328..494510430ad555116861382267985d89aff60cc9 100644
--- a/llvm/test/CodeGen/AMDGPU/smed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/smed3.ll
@@ -80,6 +80,19 @@ define amdgpu_kernel void @v_test_smed3_r_i_i_i64(i64 addrspace(1)* %out, i64 ad
   ret void
 }
 
+; Regression test for performIntMed3ImmCombine extending arguments to 32 bit
+; which failed for 64 bit arguments. Previously asserted / crashed.
+; GCN-LABEL: {{^}}test_intMed3ImmCombine_no_32bit_extend:
+; GCN: v_cmp_lt_i64
+; GCN: v_cmp_gt_i64
+define i64 @test_intMed3ImmCombine_no_32bit_extend(i64 %x) {
+  %smax = call i64 @llvm.smax.i64(i64 %x, i64 -2)
+  %smin = call i64 @llvm.smin.i64(i64 %smax, i64 2)
+  ret i64 %smin
+}
+declare i64 @llvm.smax.i64(i64, i64)
+declare i64 @llvm.smin.i64(i64, i64)
+
 ; GCN-LABEL: {{^}}v_test_smed3_r_i_i_i16:
 ; SICIVI: v_med3_i32 v{{[0-9]+}}, v{{[0-9]+}}, 12, 17
 ; GFX9: v_med3_i16 v{{[0-9]+}}, v{{[0-9]+}}, 12, 17
diff --git a/llvm/test/CodeGen/AMDGPU/spill-before-exec.mir b/llvm/test/CodeGen/AMDGPU/spill-before-exec.mir
index 76e7d73cdf6caf05032a3f4f981f4397d4263bdf..fe5b4eb450464a628b3d82a7d8b400e2f74b0255 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-before-exec.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill-before-exec.mir
@@ -1,5 +1,5 @@
 # REQUIRES: asserts
-# RUN: llc -mtriple=amdgcn--- -verify-machineinstrs -debug-only=regalloc -run-pass=greedy -o /dev/null %s 2>&1 | FileCheck %s
+# RUN: llc -mtriple=amdgcn-- -verify-machineinstrs -debug-only=regalloc -run-pass=greedy -o /dev/null %s 2>&1 | FileCheck %s
 
 ---
 # Check that physreg candidate is not used since cannot be spilled in a block,
diff --git a/llvm/test/CodeGen/AMDGPU/sram-ecc-default.ll b/llvm/test/CodeGen/AMDGPU/sram-ecc-default.ll
index 2ab384f5c77ecf7d2f6a3e0419204f2af3097dbc..7b9d1e228d088496aa1c8ae5ca5360434c7b5391 100644
--- a/llvm/test/CodeGen/AMDGPU/sram-ecc-default.ll
+++ b/llvm/test/CodeGen/AMDGPU/sram-ecc-default.ll
@@ -1,11 +1,13 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,NO-ECC %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,NO-ECC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+sramecc < %s | FileCheck -check-prefixes=GCN,NO-ECC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-sramecc < %s | FileCheck -check-prefixes=GCN,NO-ECC %s
+; RUN: llc -march=amdgcn -mcpu=gfx902 -mattr=+sramecc < %s | FileCheck -check-prefixes=GCN,NO-ECC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx904 -mattr=+sramecc < %s | FileCheck -check-prefixes=GCN,NO-ECC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx906 -mattr=+sramecc < %s | FileCheck -check-prefixes=GCN,ECC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx906 -mattr=-sramecc < %s | FileCheck -check-prefixes=GCN,NO-ECC %s
 
 ; Make sure the correct set of targets are marked with
-; FeatureDoesNotSupportSRAMECC, and +sram-ecc is ignored if it's never
+; FeatureDoesNotSupportSRAMECC, and +sramecc is ignored if it's never
 ; supported.
 
 ; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg:
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index cd0b7f77af4302237d8d8952f3c3bdbc6aeb6fc3..261d466f0142efb2ff7d13c50f9179d69e634421 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -1390,15 +1390,14 @@ define amdgpu_kernel void @s_test_srem_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v5, 24, v0
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
-; GCN-NEXT:    v_mul_hi_u32 v6, 24, v3
-; GCN-NEXT:    v_mul_hi_u32 v0, 0, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, 24, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, 24, v3
 ; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v0, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v1, s8, v1
 ; GCN-NEXT:    v_mul_hi_u32 v2, s8, v0
@@ -1605,15 +1604,14 @@ define i64 @v_test_srem_k_num_i64(i64 %x) {
 ; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v5, s[4:5]
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v5, 24, v2
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
-; GCN-NEXT:    v_mul_hi_u32 v6, 24, v3
-; GCN-NEXT:    v_mul_hi_u32 v2, 0, v2
+; GCN-NEXT:    v_mul_hi_u32 v2, 24, v2
+; GCN-NEXT:    v_mul_hi_u32 v5, 24, v3
 ; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v12, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v12, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 0, v2
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v4, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v11, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v3, v0, v3
 ; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
@@ -1816,15 +1814,14 @@ define i64 @v_test_srem_pow2_k_num_i64(i64 %x) {
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    s_mov_b32 s4, 0x8000
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v4, s4, v2
-; GCN-NEXT:    v_mul_hi_u32 v5, s4, v3
-; GCN-NEXT:    v_lshlrev_b32_e32 v6, 15, v3
-; GCN-NEXT:    v_mul_hi_u32 v2, 0, v2
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GCN-NEXT:    v_mul_hi_u32 v2, s4, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, s4, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v5, 15, v3
 ; GCN-NEXT:    v_mul_hi_u32 v3, 0, v3
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v12, v5, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v12, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 0, v2
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v4, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v11, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v3, v0, v3
 ; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
diff --git a/llvm/test/CodeGen/AMDGPU/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
index 5ec6023734097a9101f38117a960f21b12f46c1b..4aede123e48b297ff82ddd8602d580db76e443e2 100644
--- a/llvm/test/CodeGen/AMDGPU/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
@@ -226,8 +226,8 @@ define <3 x i16> @v_ssubsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
 ; GFX9-LABEL: v_ssubsat_v3i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2 clamp
 ; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v3 clamp
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i16> @llvm.ssub.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
   ret <3 x i16> %result
diff --git a/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll b/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll
index 3ca0f86f4651ecdd479549564c86770ed48a62f2..7fa9e0f02cf467c05ad3fc168ac7d8951055e01b 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900  < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji --amdhsa-code-object-version=3 < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 < %s | FileCheck -check-prefix=GFX9 %s
 
 ; Make sure the stack is never realigned for entry functions.
 
@@ -20,6 +20,7 @@ define amdgpu_kernel void @max_alignment_128() #0 {
 ; VI-NEXT:    .amdhsa_kernel max_alignment_128
 ; VI-NEXT:     .amdhsa_group_segment_fixed_size 0
 ; VI-NEXT:     .amdhsa_private_segment_fixed_size 256
+; VI-NEXT:     .amdhsa_kernarg_size 0
 ; VI-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; VI-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
@@ -67,6 +68,7 @@ define amdgpu_kernel void @max_alignment_128() #0 {
 ; GFX9-NEXT:    .amdhsa_kernel max_alignment_128
 ; GFX9-NEXT:     .amdhsa_group_segment_fixed_size 0
 ; GFX9-NEXT:     .amdhsa_private_segment_fixed_size 256
+; GFX9-NEXT:     .amdhsa_kernarg_size 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; GFX9-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
@@ -83,6 +85,7 @@ define amdgpu_kernel void @max_alignment_128() #0 {
 ; GFX9-NEXT:     .amdhsa_next_free_vgpr 1
 ; GFX9-NEXT:     .amdhsa_next_free_sgpr 8
 ; GFX9-NEXT:     .amdhsa_reserve_vcc 0
+; GFX9-NEXT:     .amdhsa_reserve_xnack_mask 1
 ; GFX9-NEXT:     .amdhsa_float_round_mode_32 0
 ; GFX9-NEXT:     .amdhsa_float_round_mode_16_64 0
 ; GFX9-NEXT:     .amdhsa_float_denorm_mode_32 3
@@ -121,6 +124,7 @@ define amdgpu_kernel void @stackrealign_attr() #1 {
 ; VI-NEXT:    .amdhsa_kernel stackrealign_attr
 ; VI-NEXT:     .amdhsa_group_segment_fixed_size 0
 ; VI-NEXT:     .amdhsa_private_segment_fixed_size 8
+; VI-NEXT:     .amdhsa_kernarg_size 0
 ; VI-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; VI-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
@@ -168,6 +172,7 @@ define amdgpu_kernel void @stackrealign_attr() #1 {
 ; GFX9-NEXT:    .amdhsa_kernel stackrealign_attr
 ; GFX9-NEXT:     .amdhsa_group_segment_fixed_size 0
 ; GFX9-NEXT:     .amdhsa_private_segment_fixed_size 8
+; GFX9-NEXT:     .amdhsa_kernarg_size 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; GFX9-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
@@ -184,6 +189,7 @@ define amdgpu_kernel void @stackrealign_attr() #1 {
 ; GFX9-NEXT:     .amdhsa_next_free_vgpr 1
 ; GFX9-NEXT:     .amdhsa_next_free_sgpr 8
 ; GFX9-NEXT:     .amdhsa_reserve_vcc 0
+; GFX9-NEXT:     .amdhsa_reserve_xnack_mask 1
 ; GFX9-NEXT:     .amdhsa_float_round_mode_32 0
 ; GFX9-NEXT:     .amdhsa_float_round_mode_16_64 0
 ; GFX9-NEXT:     .amdhsa_float_denorm_mode_32 3
@@ -222,6 +228,7 @@ define amdgpu_kernel void @alignstack_attr() #2 {
 ; VI-NEXT:    .amdhsa_kernel alignstack_attr
 ; VI-NEXT:     .amdhsa_group_segment_fixed_size 0
 ; VI-NEXT:     .amdhsa_private_segment_fixed_size 128
+; VI-NEXT:     .amdhsa_kernarg_size 0
 ; VI-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; VI-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
@@ -269,6 +276,7 @@ define amdgpu_kernel void @alignstack_attr() #2 {
 ; GFX9-NEXT:    .amdhsa_kernel alignstack_attr
 ; GFX9-NEXT:     .amdhsa_group_segment_fixed_size 0
 ; GFX9-NEXT:     .amdhsa_private_segment_fixed_size 128
+; GFX9-NEXT:     .amdhsa_kernarg_size 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; GFX9-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
@@ -285,6 +293,7 @@ define amdgpu_kernel void @alignstack_attr() #2 {
 ; GFX9-NEXT:     .amdhsa_next_free_vgpr 1
 ; GFX9-NEXT:     .amdhsa_next_free_sgpr 8
 ; GFX9-NEXT:     .amdhsa_reserve_vcc 0
+; GFX9-NEXT:     .amdhsa_reserve_xnack_mask 1
 ; GFX9-NEXT:     .amdhsa_float_round_mode_32 0
 ; GFX9-NEXT:     .amdhsa_float_round_mode_16_64 0
 ; GFX9-NEXT:     .amdhsa_float_denorm_mode_32 3
diff --git a/llvm/test/CodeGen/AMDGPU/syncscopes.ll b/llvm/test/CodeGen/AMDGPU/syncscopes.ll
index e78967bbf8ca4717ae7202b1b681c9e0790c29ab..2a7c87ea33850ef138cb9cfa937a4e5baa2077e2 100644
--- a/llvm/test/CodeGen/AMDGPU/syncscopes.ll
+++ b/llvm/test/CodeGen/AMDGPU/syncscopes.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -stop-after=si-insert-skips < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -stop-after=si-late-branch-lowering < %s | FileCheck --check-prefix=GCN %s
 
 ; GCN-LABEL: name: syncscopes
 ; GCN: FLAT_STORE_DWORD killed renamable $vgpr1_vgpr2, killed renamable $vgpr0, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("agent") seq_cst 4 into %ir.agent_out)
diff --git a/llvm/test/CodeGen/AMDGPU/tail-call-amdgpu-gfx.ll b/llvm/test/CodeGen/AMDGPU/tail-call-amdgpu-gfx.ll
new file mode 100644
index 0000000000000000000000000000000000000000..75850a933b3ec0be0b32489df1cd9c137e4828c3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tail-call-amdgpu-gfx.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN -enable-var-scope %s
+
+; Callee with SGPR and VGPR arguments
+define hidden amdgpu_gfx float @callee(float %v.arg0, float inreg %s.arg1) {
+; GCN-LABEL: callee:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f32_e32 v0, s4, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %add = fadd float %v.arg0, %s.arg1
+  ret float %add
+}
+
+define amdgpu_gfx float @caller(float %arg0) {
+; GCN-LABEL: caller:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT:    s_mov_b32 s4, 2.0
+; GCN-NEXT:    s_getpc_b64 s[6:7]
+; GCN-NEXT:    s_add_u32 s6, s6, callee@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s7, s7, callee@rel32@hi+12
+; GCN-NEXT:    s_setpc_b64 s[6:7]
+  %add = fadd float %arg0, 1.0
+  %call = tail call amdgpu_gfx float @callee(float %add, float 2.0)
+  ret float %call
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9c166b7cc61babf3a1368fe46e9cb0c593336d0b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll
@@ -0,0 +1,30 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+; ASM:  amdhsa.target: amdgcn-amd-amdhsa--gfx900
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x12C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_ANY_V4 (0x100)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900   (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() {
+entry:
+  ret void
+}
+
+define void @func1() {
+entry:
+  ret void
+}
+
+define void @func2() {
+entry:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f1d98b6e462fbd398b331e67a0f416de8bc23793
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll
@@ -0,0 +1,29 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx700"
+; ASM:  amdhsa.target: amdgcn-amd-amdhsa--gfx700
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x22)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX700 (0x22)
+; ELF-NEXT: ]
+
+define void @func0() {
+entry:
+  ret void
+}
+
+define void @func1() {
+entry:
+  ret void
+}
+
+define void @func2() {
+entry:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c32eb66b7deb98376d4af998c9fdb09c51d89bef
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target  "amdgcn-amd-amdhsa--gfx900:xnack-"
+; ASM:  amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-'
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x22C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900   (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() #0 {
+entry:
+  ret void
+}
+
+define void @func1() #0 {
+entry:
+  ret void
+}
+
+define void @func2() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "target-features"="-xnack" }
diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a15b005ca79fac4dca57a5c8db59b9a9b0cca5d9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target  "amdgcn-amd-amdhsa--gfx900:xnack+"
+; ASM:  amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+'
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x32C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900  (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() #0 {
+entry:
+  ret void
+}
+
+define void @func1() #0 {
+entry:
+  ret void
+}
+
+define void @func2() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "target-features"="+xnack" }
diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f77949204942c136662c18ec7fa987f89c390924
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target  "amdgcn-amd-amdhsa--gfx900:xnack-"
+; ASM:  amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-'
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x22C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900   (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() {
+entry:
+  ret void
+}
+
+define void @func1() #0 {
+entry:
+  ret void
+}
+
+define void @func2() {
+entry:
+  ret void
+}
+
+attributes #0 = { "target-features"="-xnack" }
diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..042254a03ec548586cbe40f48c592d6778165f40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target  "amdgcn-amd-amdhsa--gfx900:xnack-"
+; ASM:  amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-'
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x22C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900   (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() #0 {
+entry:
+  ret void
+}
+
+define void @func1() {
+entry:
+  ret void
+}
+
+define void @func2() {
+entry:
+  ret void
+}
+
+attributes #0 = { "target-features"="-xnack" }
diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d5f6135994d36c559b55eeae04762ba3f9362d02
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target  "amdgcn-amd-amdhsa--gfx900:xnack+"
+; ASM:  amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+'
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x32C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900  (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() {
+entry:
+  ret void
+}
+
+define void @func1() #0 {
+entry:
+  ret void
+}
+
+define void @func2() {
+entry:
+  ret void
+}
+
+attributes #0 = { "target-features"="+xnack" }
diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d0df9e463e0634cc6797fa5d23d67482305ef79f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target  "amdgcn-amd-amdhsa--gfx900:xnack+"
+; ASM:  amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+'
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x32C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900  (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() #0 {
+entry:
+  ret void
+}
+
+define void @func1() {
+entry:
+  ret void
+}
+
+define void @func2() {
+entry:
+  ret void
+}
+
+attributes #0 = { "target-features"="+xnack" }
diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-invalid-any-off-on.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-invalid-any-off-on.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0d24aa8f848d398880fea91e7672f5b9f0bb1960
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-invalid-any-off-on.ll
@@ -0,0 +1,21 @@
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s 2>&1 | FileCheck --check-prefixes=ERR %s
+
+; ERR: error: xnack setting of 'func2' function does not match module xnack setting
+
+define void @func0() {
+entry:
+  ret void
+}
+
+define void @func1() #0 {
+entry:
+  ret void
+}
+
+define void @func2() #1 {
+entry:
+  ret void
+}
+
+attributes #0 = { "target-features"="-xnack" }
+attributes #1 = { "target-features"="+xnack" }
diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll
new file mode 100644
index 0000000000000000000000000000000000000000..90d56dd1d6c145812982674952db3b6655d0673e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll
@@ -0,0 +1,20 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900"
+; ASM:  amdhsa.target: amdgcn-amd-amdhsa--gfx900
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x12C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_ANY_V4 (0x100)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900   (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() {
+entry:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7d2969b0a8054386e1c68350bf83cf6e049bd72c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll
@@ -0,0 +1,19 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx700"
+; ASM:  amdhsa.target: amdgcn-amd-amdhsa--gfx700
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x22)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX700 (0x22)
+; ELF-NEXT: ]
+
+define void @func0() {
+entry:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9758d2e8e8a177b94e7e5027ff7e296bd714346e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll
@@ -0,0 +1,22 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target  "amdgcn-amd-amdhsa--gfx900:xnack-"
+; ASM:  amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-'
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x22C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900   (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "target-features"="-xnack" }
diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6474971e67534bec0709e434e678872f8d2ea46e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll
@@ -0,0 +1,22 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s
+
+; ASM: .amdgcn_target  "amdgcn-amd-amdhsa--gfx900:xnack+"
+; ASM:  amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+'
+; ASM:  amdhsa.version:
+; ASM:    - 1
+; ASM:    - 1
+
+; ELF:      OS/ABI: AMDGPU_HSA (0x40)
+; ELF:      ABIVersion: 2
+; ELF:      Flags [ (0x32C)
+; ELF-NEXT:   EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300)
+; ELF-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900  (0x2C)
+; ELF-NEXT: ]
+
+define void @func0() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "target-features"="+xnack" }
diff --git a/llvm/test/CodeGen/AMDGPU/transform-block-with-return-to-epilog.ll b/llvm/test/CodeGen/AMDGPU/transform-block-with-return-to-epilog.ll
index 9f42347f5ec856d6885628925daed909def13e59..e5a019e5d04afafc36c985dcc2916f8d1c921200 100644
--- a/llvm/test/CodeGen/AMDGPU/transform-block-with-return-to-epilog.ll
+++ b/llvm/test/CodeGen/AMDGPU/transform-block-with-return-to-epilog.ll
@@ -108,25 +108,26 @@ define amdgpu_ps { <4 x float> } @test_return_to_epilog_with_optimized_kill(floa
   ; GCN:   liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5
   ; GCN:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
   ; GCN: bb.4.Flow1:
-  ; GCN:   successors: %bb.5(0x40000000)
+  ; GCN:   successors: %bb.5(0x40000000), %bb.7(0x40000000)
   ; GCN:   liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
   ; GCN:   renamable $sgpr2_sgpr3 = S_OR_SAVEEXEC_B64 killed renamable $sgpr2_sgpr3, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GCN:   $exec = S_XOR_B64 $exec, renamable $sgpr2_sgpr3, implicit-def $scc
+  ; GCN:   S_CBRANCH_EXECZ %bb.7, implicit $exec
   ; GCN: bb.5.kill0:
-  ; GCN:   successors: %bb.8(0x40000000), %bb.7(0x40000000)
+  ; GCN:   successors: %bb.6(0x40000000), %bb.8(0x40000000)
   ; GCN:   liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
   ; GCN:   dead renamable $sgpr0_sgpr1 = S_ANDN2_B64 killed renamable $sgpr0_sgpr1, $exec, implicit-def $scc
-  ; GCN:   S_CBRANCH_SCC0 %bb.7, implicit $scc
-  ; GCN: bb.8.kill0:
-  ; GCN:   successors: %bb.6(0x80000000)
+  ; GCN:   S_CBRANCH_SCC0 %bb.8, implicit $scc
+  ; GCN: bb.6.kill0:
+  ; GCN:   successors: %bb.7(0x80000000)
   ; GCN:   liveins: $sgpr2_sgpr3, $scc
   ; GCN:   $exec = S_MOV_B64 0
-  ; GCN: bb.6.end:
+  ; GCN: bb.7.end:
   ; GCN:   successors: %bb.9(0x80000000)
   ; GCN:   liveins: $sgpr2_sgpr3
   ; GCN:   $exec = S_OR_B64 $exec, killed renamable $sgpr2_sgpr3, implicit-def $scc
   ; GCN:   S_BRANCH %bb.9
-  ; GCN: bb.7:
+  ; GCN: bb.8:
   ; GCN:   $exec = S_MOV_B64 0
   ; GCN:   EXP_DONE 9, undef $vgpr0, undef $vgpr0, undef $vgpr0, undef $vgpr0, 1, 0, 0, implicit $exec
   ; GCN:   S_ENDPGM 0
diff --git a/llvm/test/CodeGen/AMDGPU/trap-abis.ll b/llvm/test/CodeGen/AMDGPU/trap-abis.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4fe18cb5608a7ce8c4c3ae934e5189d7a5a4f568
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/trap-abis.ll
@@ -0,0 +1,1181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx900 --amdhsa-code-object-version=2 -verify-machineinstrs < %s | FileCheck --check-prefix=NOHSA-TRAP-GFX900-V2 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 --amdhsa-code-object-version=3 -verify-machineinstrs < %s | FileCheck --check-prefix=NOHSA-TRAP-GFX900-V3 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 --amdhsa-code-object-version=4 -verify-machineinstrs < %s | FileCheck --check-prefix=NOHSA-TRAP-GFX900-V4 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 --amdhsa-code-object-version=2 -verify-machineinstrs < %s | FileCheck --check-prefix=HSA-TRAP-GFX803-V2 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 --amdhsa-code-object-version=3 -verify-machineinstrs < %s | FileCheck --check-prefix=HSA-TRAP-GFX803-V3 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 --amdhsa-code-object-version=4 -verify-machineinstrs < %s | FileCheck --check-prefix=HSA-TRAP-GFX803-V4 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=2 -verify-machineinstrs < %s | FileCheck --check-prefix=HSA-TRAP-GFX900-V2 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=3 -verify-machineinstrs < %s | FileCheck --check-prefix=HSA-TRAP-GFX900-V3 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 -verify-machineinstrs < %s | FileCheck --check-prefix=HSA-TRAP-GFX900-V4 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-trap-handler --amdhsa-code-object-version=2 -verify-machineinstrs < %s | FileCheck --check-prefix=HSA-NOTRAP-GFX900-V2 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-trap-handler --amdhsa-code-object-version=3 -verify-machineinstrs < %s | FileCheck --check-prefix=HSA-NOTRAP-GFX900-V3 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-trap-handler --amdhsa-code-object-version=4 -verify-machineinstrs < %s | FileCheck --check-prefix=HSA-NOTRAP-GFX900-V4 %s
+
+declare void @llvm.trap() #0
+declare void @llvm.debugtrap() #1
+
+define amdgpu_kernel void @trap(i32 addrspace(1)* nocapture readonly %arg0) {
+; NOHSA-TRAP-GFX900-V2-LABEL: trap:
+; NOHSA-TRAP-GFX900-V2:       ; %bb.0:
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; NOHSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v0, 0
+; NOHSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v1, 1
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; NOHSA-TRAP-GFX900-V2-NEXT:    global_store_dword v0, v1, s[0:1]
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_endpgm
+;
+; NOHSA-TRAP-GFX900-V3-LABEL: trap:
+; NOHSA-TRAP-GFX900-V3:       ; %bb.0:
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; NOHSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v0, 0
+; NOHSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v1, 1
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; NOHSA-TRAP-GFX900-V3-NEXT:    global_store_dword v0, v1, s[0:1]
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_endpgm
+;
+; NOHSA-TRAP-GFX900-V4-LABEL: trap:
+; NOHSA-TRAP-GFX900-V4:       ; %bb.0:
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; NOHSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v0, 0
+; NOHSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v1, 1
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; NOHSA-TRAP-GFX900-V4-NEXT:    global_store_dword v0, v1, s[0:1]
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_endpgm
+;
+; HSA-TRAP-GFX803-V2-LABEL: trap:
+; HSA-TRAP-GFX803-V2:         .amd_kernel_code_t
+; HSA-TRAP-GFX803-V2-NEXT:     amd_code_version_major = 1
+; HSA-TRAP-GFX803-V2-NEXT:     amd_code_version_minor = 2
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_kind = 1
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_version_major = 8
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_version_minor = 0
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_version_stepping = 3
+; HSA-TRAP-GFX803-V2-NEXT:     kernel_code_entry_byte_offset = 256
+; HSA-TRAP-GFX803-V2-NEXT:     kernel_code_prefetch_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     granulated_workitem_vgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     granulated_wavefront_sgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     priority = 0
+; HSA-TRAP-GFX803-V2-NEXT:     float_mode = 240
+; HSA-TRAP-GFX803-V2-NEXT:     priv = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_dx10_clamp = 1
+; HSA-TRAP-GFX803-V2-NEXT:     debug_mode = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_ieee_mode = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_wgp_mode = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_mem_ordered = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_fwd_progress = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
+; HSA-TRAP-GFX803-V2-NEXT:     user_sgpr_count = 8
+; HSA-TRAP-GFX803-V2-NEXT:     enable_trap_handler = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_id_x = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_id_y = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_id_z = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_info = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_vgpr_workitem_id = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_exception_msb = 0
+; HSA-TRAP-GFX803-V2-NEXT:     granulated_lds_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_exception = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_private_segment_buffer = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_dispatch_ptr = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_queue_ptr = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_dispatch_id = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_flat_scratch_init = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_private_segment_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_grid_workgroup_count_z = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_wavefront_size32 = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_ordered_append_gds = 0
+; HSA-TRAP-GFX803-V2-NEXT:     private_element_size = 1
+; HSA-TRAP-GFX803-V2-NEXT:     is_ptr64 = 1
+; HSA-TRAP-GFX803-V2-NEXT:     is_dynamic_callstack = 0
+; HSA-TRAP-GFX803-V2-NEXT:     is_debug_enabled = 0
+; HSA-TRAP-GFX803-V2-NEXT:     is_xnack_enabled = 0
+; HSA-TRAP-GFX803-V2-NEXT:     workitem_private_segment_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     workgroup_group_segment_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     gds_segment_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     kernarg_segment_byte_size = 8
+; HSA-TRAP-GFX803-V2-NEXT:     workgroup_fbarrier_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     wavefront_sgpr_count = 8
+; HSA-TRAP-GFX803-V2-NEXT:     workitem_vgpr_count = 3
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_vgpr_first = 0
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_vgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_sgpr_first = 0
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_sgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     debug_wavefront_private_segment_offset_sgpr = 0
+; HSA-TRAP-GFX803-V2-NEXT:     debug_private_segment_buffer_sgpr = 0
+; HSA-TRAP-GFX803-V2-NEXT:     kernarg_segment_alignment = 4
+; HSA-TRAP-GFX803-V2-NEXT:     group_segment_alignment = 4
+; HSA-TRAP-GFX803-V2-NEXT:     private_segment_alignment = 4
+; HSA-TRAP-GFX803-V2-NEXT:     wavefront_size = 6
+; HSA-TRAP-GFX803-V2-NEXT:     call_convention = -1
+; HSA-TRAP-GFX803-V2-NEXT:     runtime_loader_kernel_symbol = 0
+; HSA-TRAP-GFX803-V2-NEXT:    .end_amd_kernel_code_t
+; HSA-TRAP-GFX803-V2-NEXT:  ; %bb.0:
+; HSA-TRAP-GFX803-V2-NEXT:    s_load_dwordx2 s[2:3], s[6:7], 0x0
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v2, 1
+; HSA-TRAP-GFX803-V2-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX803-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v0, s2
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v1, s3
+; HSA-TRAP-GFX803-V2-NEXT:    flat_store_dword v[0:1], v2
+; HSA-TRAP-GFX803-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V2-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX803-V3-LABEL: trap:
+; HSA-TRAP-GFX803-V3:       ; %bb.0:
+; HSA-TRAP-GFX803-V3-NEXT:    s_load_dwordx2 s[2:3], s[6:7], 0x0
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v2, 1
+; HSA-TRAP-GFX803-V3-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX803-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v0, s2
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v1, s3
+; HSA-TRAP-GFX803-V3-NEXT:    flat_store_dword v[0:1], v2
+; HSA-TRAP-GFX803-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V3-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX803-V4-LABEL: trap:
+; HSA-TRAP-GFX803-V4:       ; %bb.0:
+; HSA-TRAP-GFX803-V4-NEXT:    s_load_dwordx2 s[2:3], s[6:7], 0x0
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v2, 1
+; HSA-TRAP-GFX803-V4-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX803-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v0, s2
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v1, s3
+; HSA-TRAP-GFX803-V4-NEXT:    flat_store_dword v[0:1], v2
+; HSA-TRAP-GFX803-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V4-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX900-V2-LABEL: trap:
+; HSA-TRAP-GFX900-V2:         .amd_kernel_code_t
+; HSA-TRAP-GFX900-V2-NEXT:     amd_code_version_major = 1
+; HSA-TRAP-GFX900-V2-NEXT:     amd_code_version_minor = 2
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_kind = 1
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_version_major = 9
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_version_minor = 0
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_version_stepping = 0
+; HSA-TRAP-GFX900-V2-NEXT:     kernel_code_entry_byte_offset = 256
+; HSA-TRAP-GFX900-V2-NEXT:     kernel_code_prefetch_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     granulated_workitem_vgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     granulated_wavefront_sgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     priority = 0
+; HSA-TRAP-GFX900-V2-NEXT:     float_mode = 240
+; HSA-TRAP-GFX900-V2-NEXT:     priv = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_dx10_clamp = 1
+; HSA-TRAP-GFX900-V2-NEXT:     debug_mode = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_ieee_mode = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_wgp_mode = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_mem_ordered = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_fwd_progress = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
+; HSA-TRAP-GFX900-V2-NEXT:     user_sgpr_count = 8
+; HSA-TRAP-GFX900-V2-NEXT:     enable_trap_handler = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_x = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_y = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_z = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_info = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_vgpr_workitem_id = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_exception_msb = 0
+; HSA-TRAP-GFX900-V2-NEXT:     granulated_lds_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_exception = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_buffer = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_ptr = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_queue_ptr = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_id = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_flat_scratch_init = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_z = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_wavefront_size32 = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_ordered_append_gds = 0
+; HSA-TRAP-GFX900-V2-NEXT:     private_element_size = 1
+; HSA-TRAP-GFX900-V2-NEXT:     is_ptr64 = 1
+; HSA-TRAP-GFX900-V2-NEXT:     is_dynamic_callstack = 0
+; HSA-TRAP-GFX900-V2-NEXT:     is_debug_enabled = 0
+; HSA-TRAP-GFX900-V2-NEXT:     is_xnack_enabled = 1
+; HSA-TRAP-GFX900-V2-NEXT:     workitem_private_segment_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     workgroup_group_segment_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     gds_segment_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     kernarg_segment_byte_size = 8
+; HSA-TRAP-GFX900-V2-NEXT:     workgroup_fbarrier_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     wavefront_sgpr_count = 8
+; HSA-TRAP-GFX900-V2-NEXT:     workitem_vgpr_count = 2
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_vgpr_first = 0
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_vgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_sgpr_first = 0
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_sgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     debug_wavefront_private_segment_offset_sgpr = 0
+; HSA-TRAP-GFX900-V2-NEXT:     debug_private_segment_buffer_sgpr = 0
+; HSA-TRAP-GFX900-V2-NEXT:     kernarg_segment_alignment = 4
+; HSA-TRAP-GFX900-V2-NEXT:     group_segment_alignment = 4
+; HSA-TRAP-GFX900-V2-NEXT:     private_segment_alignment = 4
+; HSA-TRAP-GFX900-V2-NEXT:     wavefront_size = 6
+; HSA-TRAP-GFX900-V2-NEXT:     call_convention = -1
+; HSA-TRAP-GFX900-V2-NEXT:     runtime_loader_kernel_symbol = 0
+; HSA-TRAP-GFX900-V2-NEXT:    .end_amd_kernel_code_t
+; HSA-TRAP-GFX900-V2-NEXT:  ; %bb.0:
+; HSA-TRAP-GFX900-V2-NEXT:    s_load_dwordx2 s[2:3], s[6:7], 0x0
+; HSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-TRAP-GFX900-V2-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX900-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX900-V2-NEXT:    global_store_dword v0, v1, s[2:3]
+; HSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V2-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX900-V3-LABEL: trap:
+; HSA-TRAP-GFX900-V3:       ; %bb.0:
+; HSA-TRAP-GFX900-V3-NEXT:    s_load_dwordx2 s[2:3], s[6:7], 0x0
+; HSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-TRAP-GFX900-V3-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX900-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX900-V3-NEXT:    global_store_dword v0, v1, s[2:3]
+; HSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V3-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX900-V4-LABEL: trap:
+; HSA-TRAP-GFX900-V4:       ; %bb.0:
+; HSA-TRAP-GFX900-V4-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-TRAP-GFX900-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX900-V4-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V4-NEXT:    s_trap 2
+;
+; HSA-NOTRAP-GFX900-V2-LABEL: trap:
+; HSA-NOTRAP-GFX900-V2:         .amd_kernel_code_t
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_code_version_major = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_code_version_minor = 2
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_kind = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_version_major = 9
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_version_minor = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_version_stepping = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernel_code_entry_byte_offset = 256
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernel_code_prefetch_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     granulated_workitem_vgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     granulated_wavefront_sgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     priority = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     float_mode = 240
+; HSA-NOTRAP-GFX900-V2-NEXT:     priv = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_dx10_clamp = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     debug_mode = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_ieee_mode = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_wgp_mode = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_mem_ordered = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_fwd_progress = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     user_sgpr_count = 8
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_trap_handler = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_x = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_y = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_z = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_info = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_vgpr_workitem_id = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_exception_msb = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     granulated_lds_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_exception = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_buffer = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_ptr = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_queue_ptr = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_id = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_flat_scratch_init = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_z = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_wavefront_size32 = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_ordered_append_gds = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     private_element_size = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_ptr64 = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_dynamic_callstack = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_debug_enabled = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_xnack_enabled = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     workitem_private_segment_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     workgroup_group_segment_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     gds_segment_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernarg_segment_byte_size = 8
+; HSA-NOTRAP-GFX900-V2-NEXT:     workgroup_fbarrier_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     wavefront_sgpr_count = 8
+; HSA-NOTRAP-GFX900-V2-NEXT:     workitem_vgpr_count = 2
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_vgpr_first = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_vgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_sgpr_first = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_sgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     debug_wavefront_private_segment_offset_sgpr = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     debug_private_segment_buffer_sgpr = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernarg_segment_alignment = 4
+; HSA-NOTRAP-GFX900-V2-NEXT:     group_segment_alignment = 4
+; HSA-NOTRAP-GFX900-V2-NEXT:     private_segment_alignment = 4
+; HSA-NOTRAP-GFX900-V2-NEXT:     wavefront_size = 6
+; HSA-NOTRAP-GFX900-V2-NEXT:     call_convention = -1
+; HSA-NOTRAP-GFX900-V2-NEXT:     runtime_loader_kernel_symbol = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:    .end_amd_kernel_code_t
+; HSA-NOTRAP-GFX900-V2-NEXT:  ; %bb.0:
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-NOTRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-NOTRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-NOTRAP-GFX900-V2-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_endpgm
+;
+; HSA-NOTRAP-GFX900-V3-LABEL: trap:
+; HSA-NOTRAP-GFX900-V3:       ; %bb.0:
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-NOTRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-NOTRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-NOTRAP-GFX900-V3-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_endpgm
+;
+; HSA-NOTRAP-GFX900-V4-LABEL: trap:
+; HSA-NOTRAP-GFX900-V4:       ; %bb.0:
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-NOTRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-NOTRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-NOTRAP-GFX900-V4-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_endpgm
+  store volatile i32 1, i32 addrspace(1)* %arg0
+  call void @llvm.trap()
+  unreachable
+  store volatile i32 2, i32 addrspace(1)* %arg0
+  ret void
+}
+
+define amdgpu_kernel void @non_entry_trap(i32 addrspace(1)* nocapture readonly %arg0) local_unnamed_addr {
+; NOHSA-TRAP-GFX900-V2-LABEL: non_entry_trap:
+; NOHSA-TRAP-GFX900-V2:       ; %bb.0: ; %entry
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; NOHSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v0, 0
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; NOHSA-TRAP-GFX900-V2-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V2-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v1
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_and_b64 vcc, exec, vcc
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_cbranch_vccz BB1_2
+; NOHSA-TRAP-GFX900-V2-NEXT:  ; %bb.1: ; %ret
+; NOHSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v1, 3
+; NOHSA-TRAP-GFX900-V2-NEXT:    global_store_dword v0, v1, s[0:1]
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_endpgm
+; NOHSA-TRAP-GFX900-V2-NEXT:  BB1_2: ; %trap
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_endpgm
+;
+; NOHSA-TRAP-GFX900-V3-LABEL: non_entry_trap:
+; NOHSA-TRAP-GFX900-V3:       ; %bb.0: ; %entry
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; NOHSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v0, 0
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; NOHSA-TRAP-GFX900-V3-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V3-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v1
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_and_b64 vcc, exec, vcc
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_cbranch_vccz BB1_2
+; NOHSA-TRAP-GFX900-V3-NEXT:  ; %bb.1: ; %ret
+; NOHSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v1, 3
+; NOHSA-TRAP-GFX900-V3-NEXT:    global_store_dword v0, v1, s[0:1]
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_endpgm
+; NOHSA-TRAP-GFX900-V3-NEXT:  BB1_2: ; %trap
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_endpgm
+;
+; NOHSA-TRAP-GFX900-V4-LABEL: non_entry_trap:
+; NOHSA-TRAP-GFX900-V4:       ; %bb.0: ; %entry
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; NOHSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v0, 0
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; NOHSA-TRAP-GFX900-V4-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V4-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v1
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_and_b64 vcc, exec, vcc
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_cbranch_vccz BB1_2
+; NOHSA-TRAP-GFX900-V4-NEXT:  ; %bb.1: ; %ret
+; NOHSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v1, 3
+; NOHSA-TRAP-GFX900-V4-NEXT:    global_store_dword v0, v1, s[0:1]
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_endpgm
+; NOHSA-TRAP-GFX900-V4-NEXT:  BB1_2: ; %trap
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_endpgm
+;
+; HSA-TRAP-GFX803-V2-LABEL: non_entry_trap:
+; HSA-TRAP-GFX803-V2:         .amd_kernel_code_t
+; HSA-TRAP-GFX803-V2-NEXT:     amd_code_version_major = 1
+; HSA-TRAP-GFX803-V2-NEXT:     amd_code_version_minor = 2
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_kind = 1
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_version_major = 8
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_version_minor = 0
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_version_stepping = 3
+; HSA-TRAP-GFX803-V2-NEXT:     kernel_code_entry_byte_offset = 256
+; HSA-TRAP-GFX803-V2-NEXT:     kernel_code_prefetch_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     granulated_workitem_vgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     granulated_wavefront_sgpr_count = 1
+; HSA-TRAP-GFX803-V2-NEXT:     priority = 0
+; HSA-TRAP-GFX803-V2-NEXT:     float_mode = 240
+; HSA-TRAP-GFX803-V2-NEXT:     priv = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_dx10_clamp = 1
+; HSA-TRAP-GFX803-V2-NEXT:     debug_mode = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_ieee_mode = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_wgp_mode = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_mem_ordered = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_fwd_progress = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
+; HSA-TRAP-GFX803-V2-NEXT:     user_sgpr_count = 8
+; HSA-TRAP-GFX803-V2-NEXT:     enable_trap_handler = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_id_x = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_id_y = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_id_z = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_info = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_vgpr_workitem_id = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_exception_msb = 0
+; HSA-TRAP-GFX803-V2-NEXT:     granulated_lds_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_exception = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_private_segment_buffer = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_dispatch_ptr = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_queue_ptr = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_dispatch_id = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_flat_scratch_init = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_private_segment_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_grid_workgroup_count_z = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_wavefront_size32 = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_ordered_append_gds = 0
+; HSA-TRAP-GFX803-V2-NEXT:     private_element_size = 1
+; HSA-TRAP-GFX803-V2-NEXT:     is_ptr64 = 1
+; HSA-TRAP-GFX803-V2-NEXT:     is_dynamic_callstack = 0
+; HSA-TRAP-GFX803-V2-NEXT:     is_debug_enabled = 0
+; HSA-TRAP-GFX803-V2-NEXT:     is_xnack_enabled = 0
+; HSA-TRAP-GFX803-V2-NEXT:     workitem_private_segment_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     workgroup_group_segment_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     gds_segment_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     kernarg_segment_byte_size = 8
+; HSA-TRAP-GFX803-V2-NEXT:     workgroup_fbarrier_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     wavefront_sgpr_count = 10
+; HSA-TRAP-GFX803-V2-NEXT:     workitem_vgpr_count = 3
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_vgpr_first = 0
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_vgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_sgpr_first = 0
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_sgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     debug_wavefront_private_segment_offset_sgpr = 0
+; HSA-TRAP-GFX803-V2-NEXT:     debug_private_segment_buffer_sgpr = 0
+; HSA-TRAP-GFX803-V2-NEXT:     kernarg_segment_alignment = 4
+; HSA-TRAP-GFX803-V2-NEXT:     group_segment_alignment = 4
+; HSA-TRAP-GFX803-V2-NEXT:     private_segment_alignment = 4
+; HSA-TRAP-GFX803-V2-NEXT:     wavefront_size = 6
+; HSA-TRAP-GFX803-V2-NEXT:     call_convention = -1
+; HSA-TRAP-GFX803-V2-NEXT:     runtime_loader_kernel_symbol = 0
+; HSA-TRAP-GFX803-V2-NEXT:    .end_amd_kernel_code_t
+; HSA-TRAP-GFX803-V2-NEXT:  ; %bb.0: ; %entry
+; HSA-TRAP-GFX803-V2-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX803-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v1, s1
+; HSA-TRAP-GFX803-V2-NEXT:    flat_load_dword v0, v[0:1] glc
+; HSA-TRAP-GFX803-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V2-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v0
+; HSA-TRAP-GFX803-V2-NEXT:    s_and_b64 vcc, exec, vcc
+; HSA-TRAP-GFX803-V2-NEXT:    s_cbranch_vccz BB1_2
+; HSA-TRAP-GFX803-V2-NEXT:  ; %bb.1: ; %ret
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v2, 3
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v1, s1
+; HSA-TRAP-GFX803-V2-NEXT:    flat_store_dword v[0:1], v2
+; HSA-TRAP-GFX803-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V2-NEXT:    s_endpgm
+; HSA-TRAP-GFX803-V2-NEXT:  BB1_2: ; %trap
+; HSA-TRAP-GFX803-V2-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX803-V2-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX803-V3-LABEL: non_entry_trap:
+; HSA-TRAP-GFX803-V3:       ; %bb.0: ; %entry
+; HSA-TRAP-GFX803-V3-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX803-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v1, s1
+; HSA-TRAP-GFX803-V3-NEXT:    flat_load_dword v0, v[0:1] glc
+; HSA-TRAP-GFX803-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V3-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v0
+; HSA-TRAP-GFX803-V3-NEXT:    s_and_b64 vcc, exec, vcc
+; HSA-TRAP-GFX803-V3-NEXT:    s_cbranch_vccz BB1_2
+; HSA-TRAP-GFX803-V3-NEXT:  ; %bb.1: ; %ret
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v2, 3
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v1, s1
+; HSA-TRAP-GFX803-V3-NEXT:    flat_store_dword v[0:1], v2
+; HSA-TRAP-GFX803-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V3-NEXT:    s_endpgm
+; HSA-TRAP-GFX803-V3-NEXT:  BB1_2: ; %trap
+; HSA-TRAP-GFX803-V3-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX803-V3-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX803-V4-LABEL: non_entry_trap:
+; HSA-TRAP-GFX803-V4:       ; %bb.0: ; %entry
+; HSA-TRAP-GFX803-V4-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX803-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v1, s1
+; HSA-TRAP-GFX803-V4-NEXT:    flat_load_dword v0, v[0:1] glc
+; HSA-TRAP-GFX803-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V4-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v0
+; HSA-TRAP-GFX803-V4-NEXT:    s_and_b64 vcc, exec, vcc
+; HSA-TRAP-GFX803-V4-NEXT:    s_cbranch_vccz BB1_2
+; HSA-TRAP-GFX803-V4-NEXT:  ; %bb.1: ; %ret
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v2, 3
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v1, s1
+; HSA-TRAP-GFX803-V4-NEXT:    flat_store_dword v[0:1], v2
+; HSA-TRAP-GFX803-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V4-NEXT:    s_endpgm
+; HSA-TRAP-GFX803-V4-NEXT:  BB1_2: ; %trap
+; HSA-TRAP-GFX803-V4-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX803-V4-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX900-V2-LABEL: non_entry_trap:
+; HSA-TRAP-GFX900-V2:         .amd_kernel_code_t
+; HSA-TRAP-GFX900-V2-NEXT:     amd_code_version_major = 1
+; HSA-TRAP-GFX900-V2-NEXT:     amd_code_version_minor = 2
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_kind = 1
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_version_major = 9
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_version_minor = 0
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_version_stepping = 0
+; HSA-TRAP-GFX900-V2-NEXT:     kernel_code_entry_byte_offset = 256
+; HSA-TRAP-GFX900-V2-NEXT:     kernel_code_prefetch_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     granulated_workitem_vgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     granulated_wavefront_sgpr_count = 1
+; HSA-TRAP-GFX900-V2-NEXT:     priority = 0
+; HSA-TRAP-GFX900-V2-NEXT:     float_mode = 240
+; HSA-TRAP-GFX900-V2-NEXT:     priv = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_dx10_clamp = 1
+; HSA-TRAP-GFX900-V2-NEXT:     debug_mode = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_ieee_mode = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_wgp_mode = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_mem_ordered = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_fwd_progress = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
+; HSA-TRAP-GFX900-V2-NEXT:     user_sgpr_count = 8
+; HSA-TRAP-GFX900-V2-NEXT:     enable_trap_handler = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_x = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_y = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_z = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_info = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_vgpr_workitem_id = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_exception_msb = 0
+; HSA-TRAP-GFX900-V2-NEXT:     granulated_lds_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_exception = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_buffer = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_ptr = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_queue_ptr = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_id = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_flat_scratch_init = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_z = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_wavefront_size32 = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_ordered_append_gds = 0
+; HSA-TRAP-GFX900-V2-NEXT:     private_element_size = 1
+; HSA-TRAP-GFX900-V2-NEXT:     is_ptr64 = 1
+; HSA-TRAP-GFX900-V2-NEXT:     is_dynamic_callstack = 0
+; HSA-TRAP-GFX900-V2-NEXT:     is_debug_enabled = 0
+; HSA-TRAP-GFX900-V2-NEXT:     is_xnack_enabled = 1
+; HSA-TRAP-GFX900-V2-NEXT:     workitem_private_segment_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     workgroup_group_segment_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     gds_segment_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     kernarg_segment_byte_size = 8
+; HSA-TRAP-GFX900-V2-NEXT:     workgroup_fbarrier_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     wavefront_sgpr_count = 10
+; HSA-TRAP-GFX900-V2-NEXT:     workitem_vgpr_count = 2
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_vgpr_first = 0
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_vgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_sgpr_first = 0
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_sgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     debug_wavefront_private_segment_offset_sgpr = 0
+; HSA-TRAP-GFX900-V2-NEXT:     debug_private_segment_buffer_sgpr = 0
+; HSA-TRAP-GFX900-V2-NEXT:     kernarg_segment_alignment = 4
+; HSA-TRAP-GFX900-V2-NEXT:     group_segment_alignment = 4
+; HSA-TRAP-GFX900-V2-NEXT:     private_segment_alignment = 4
+; HSA-TRAP-GFX900-V2-NEXT:     wavefront_size = 6
+; HSA-TRAP-GFX900-V2-NEXT:     call_convention = -1
+; HSA-TRAP-GFX900-V2-NEXT:     runtime_loader_kernel_symbol = 0
+; HSA-TRAP-GFX900-V2-NEXT:    .end_amd_kernel_code_t
+; HSA-TRAP-GFX900-V2-NEXT:  ; %bb.0: ; %entry
+; HSA-TRAP-GFX900-V2-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-TRAP-GFX900-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX900-V2-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; HSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V2-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v1
+; HSA-TRAP-GFX900-V2-NEXT:    s_and_b64 vcc, exec, vcc
+; HSA-TRAP-GFX900-V2-NEXT:    s_cbranch_vccz BB1_2
+; HSA-TRAP-GFX900-V2-NEXT:  ; %bb.1: ; %ret
+; HSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v1, 3
+; HSA-TRAP-GFX900-V2-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V2-NEXT:    s_endpgm
+; HSA-TRAP-GFX900-V2-NEXT:  BB1_2: ; %trap
+; HSA-TRAP-GFX900-V2-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX900-V2-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX900-V3-LABEL: non_entry_trap:
+; HSA-TRAP-GFX900-V3:       ; %bb.0: ; %entry
+; HSA-TRAP-GFX900-V3-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-TRAP-GFX900-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX900-V3-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; HSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V3-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v1
+; HSA-TRAP-GFX900-V3-NEXT:    s_and_b64 vcc, exec, vcc
+; HSA-TRAP-GFX900-V3-NEXT:    s_cbranch_vccz BB1_2
+; HSA-TRAP-GFX900-V3-NEXT:  ; %bb.1: ; %ret
+; HSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v1, 3
+; HSA-TRAP-GFX900-V3-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V3-NEXT:    s_endpgm
+; HSA-TRAP-GFX900-V3-NEXT:  BB1_2: ; %trap
+; HSA-TRAP-GFX900-V3-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; HSA-TRAP-GFX900-V3-NEXT:    s_trap 2
+;
+; HSA-TRAP-GFX900-V4-LABEL: non_entry_trap:
+; HSA-TRAP-GFX900-V4:       ; %bb.0: ; %entry
+; HSA-TRAP-GFX900-V4-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-TRAP-GFX900-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX900-V4-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; HSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V4-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v1
+; HSA-TRAP-GFX900-V4-NEXT:    s_and_b64 vcc, exec, vcc
+; HSA-TRAP-GFX900-V4-NEXT:    s_cbranch_vccz BB1_2
+; HSA-TRAP-GFX900-V4-NEXT:  ; %bb.1: ; %ret
+; HSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v1, 3
+; HSA-TRAP-GFX900-V4-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V4-NEXT:    s_endpgm
+; HSA-TRAP-GFX900-V4-NEXT:  BB1_2: ; %trap
+; HSA-TRAP-GFX900-V4-NEXT:    s_trap 2
+;
+; HSA-NOTRAP-GFX900-V2-LABEL: non_entry_trap:
+; HSA-NOTRAP-GFX900-V2:         .amd_kernel_code_t
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_code_version_major = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_code_version_minor = 2
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_kind = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_version_major = 9
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_version_minor = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_version_stepping = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernel_code_entry_byte_offset = 256
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernel_code_prefetch_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     granulated_workitem_vgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     granulated_wavefront_sgpr_count = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     priority = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     float_mode = 240
+; HSA-NOTRAP-GFX900-V2-NEXT:     priv = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_dx10_clamp = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     debug_mode = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_ieee_mode = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_wgp_mode = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_mem_ordered = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_fwd_progress = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     user_sgpr_count = 8
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_trap_handler = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_x = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_y = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_z = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_info = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_vgpr_workitem_id = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_exception_msb = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     granulated_lds_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_exception = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_buffer = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_ptr = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_queue_ptr = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_id = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_flat_scratch_init = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_z = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_wavefront_size32 = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_ordered_append_gds = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     private_element_size = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_ptr64 = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_dynamic_callstack = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_debug_enabled = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_xnack_enabled = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     workitem_private_segment_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     workgroup_group_segment_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     gds_segment_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernarg_segment_byte_size = 8
+; HSA-NOTRAP-GFX900-V2-NEXT:     workgroup_fbarrier_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     wavefront_sgpr_count = 10
+; HSA-NOTRAP-GFX900-V2-NEXT:     workitem_vgpr_count = 2
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_vgpr_first = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_vgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_sgpr_first = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_sgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     debug_wavefront_private_segment_offset_sgpr = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     debug_private_segment_buffer_sgpr = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernarg_segment_alignment = 4
+; HSA-NOTRAP-GFX900-V2-NEXT:     group_segment_alignment = 4
+; HSA-NOTRAP-GFX900-V2-NEXT:     private_segment_alignment = 4
+; HSA-NOTRAP-GFX900-V2-NEXT:     wavefront_size = 6
+; HSA-NOTRAP-GFX900-V2-NEXT:     call_convention = -1
+; HSA-NOTRAP-GFX900-V2-NEXT:     runtime_loader_kernel_symbol = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:    .end_amd_kernel_code_t
+; HSA-NOTRAP-GFX900-V2-NEXT:  ; %bb.0: ; %entry
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-NOTRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-NOTRAP-GFX900-V2-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V2-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v1
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_and_b64 vcc, exec, vcc
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_cbranch_vccz BB1_2
+; HSA-NOTRAP-GFX900-V2-NEXT:  ; %bb.1: ; %ret
+; HSA-NOTRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v1, 3
+; HSA-NOTRAP-GFX900-V2-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_endpgm
+; HSA-NOTRAP-GFX900-V2-NEXT:  BB1_2: ; %trap
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_endpgm
+;
+; HSA-NOTRAP-GFX900-V3-LABEL: non_entry_trap:
+; HSA-NOTRAP-GFX900-V3:       ; %bb.0: ; %entry
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-NOTRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-NOTRAP-GFX900-V3-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V3-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v1
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_and_b64 vcc, exec, vcc
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_cbranch_vccz BB1_2
+; HSA-NOTRAP-GFX900-V3-NEXT:  ; %bb.1: ; %ret
+; HSA-NOTRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v1, 3
+; HSA-NOTRAP-GFX900-V3-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_endpgm
+; HSA-NOTRAP-GFX900-V3-NEXT:  BB1_2: ; %trap
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_endpgm
+;
+; HSA-NOTRAP-GFX900-V4-LABEL: non_entry_trap:
+; HSA-NOTRAP-GFX900-V4:       ; %bb.0: ; %entry
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-NOTRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-NOTRAP-GFX900-V4-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V4-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v1
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_and_b64 vcc, exec, vcc
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_cbranch_vccz BB1_2
+; HSA-NOTRAP-GFX900-V4-NEXT:  ; %bb.1: ; %ret
+; HSA-NOTRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v1, 3
+; HSA-NOTRAP-GFX900-V4-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_endpgm
+; HSA-NOTRAP-GFX900-V4-NEXT:  BB1_2: ; %trap
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_endpgm
+entry:
+  %tmp29 = load volatile i32, i32 addrspace(1)* %arg0
+  %cmp = icmp eq i32 %tmp29, -1
+  br i1 %cmp, label %ret, label %trap
+
+trap:
+  call void @llvm.trap()
+  unreachable
+
+ret:
+  store volatile i32 3, i32 addrspace(1)* %arg0
+  ret void
+}
+
+define amdgpu_kernel void @debugtrap(i32 addrspace(1)* nocapture readonly %arg0) {
+; NOHSA-TRAP-GFX900-V2-LABEL: debugtrap:
+; NOHSA-TRAP-GFX900-V2:       ; %bb.0:
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; NOHSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v0, 0
+; NOHSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v1, 1
+; NOHSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v2, 2
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; NOHSA-TRAP-GFX900-V2-NEXT:    global_store_dword v0, v1, s[0:1]
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V2-NEXT:    global_store_dword v0, v2, s[0:1]
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V2-NEXT:    s_endpgm
+;
+; NOHSA-TRAP-GFX900-V3-LABEL: debugtrap:
+; NOHSA-TRAP-GFX900-V3:       ; %bb.0:
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; NOHSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v0, 0
+; NOHSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v1, 1
+; NOHSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v2, 2
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; NOHSA-TRAP-GFX900-V3-NEXT:    global_store_dword v0, v1, s[0:1]
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V3-NEXT:    global_store_dword v0, v2, s[0:1]
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V3-NEXT:    s_endpgm
+;
+; NOHSA-TRAP-GFX900-V4-LABEL: debugtrap:
+; NOHSA-TRAP-GFX900-V4:       ; %bb.0:
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; NOHSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v0, 0
+; NOHSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v1, 1
+; NOHSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v2, 2
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; NOHSA-TRAP-GFX900-V4-NEXT:    global_store_dword v0, v1, s[0:1]
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V4-NEXT:    global_store_dword v0, v2, s[0:1]
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; NOHSA-TRAP-GFX900-V4-NEXT:    s_endpgm
+;
+; HSA-TRAP-GFX803-V2-LABEL: debugtrap:
+; HSA-TRAP-GFX803-V2:         .amd_kernel_code_t
+; HSA-TRAP-GFX803-V2-NEXT:     amd_code_version_major = 1
+; HSA-TRAP-GFX803-V2-NEXT:     amd_code_version_minor = 2
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_kind = 1
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_version_major = 8
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_version_minor = 0
+; HSA-TRAP-GFX803-V2-NEXT:     amd_machine_version_stepping = 3
+; HSA-TRAP-GFX803-V2-NEXT:     kernel_code_entry_byte_offset = 256
+; HSA-TRAP-GFX803-V2-NEXT:     kernel_code_prefetch_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     granulated_workitem_vgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     granulated_wavefront_sgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     priority = 0
+; HSA-TRAP-GFX803-V2-NEXT:     float_mode = 240
+; HSA-TRAP-GFX803-V2-NEXT:     priv = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_dx10_clamp = 1
+; HSA-TRAP-GFX803-V2-NEXT:     debug_mode = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_ieee_mode = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_wgp_mode = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_mem_ordered = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_fwd_progress = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
+; HSA-TRAP-GFX803-V2-NEXT:     user_sgpr_count = 8
+; HSA-TRAP-GFX803-V2-NEXT:     enable_trap_handler = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_id_x = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_id_y = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_id_z = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_workgroup_info = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_vgpr_workitem_id = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_exception_msb = 0
+; HSA-TRAP-GFX803-V2-NEXT:     granulated_lds_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_exception = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_private_segment_buffer = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_dispatch_ptr = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_queue_ptr = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_dispatch_id = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_flat_scratch_init = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_private_segment_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_sgpr_grid_workgroup_count_z = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_wavefront_size32 = 0
+; HSA-TRAP-GFX803-V2-NEXT:     enable_ordered_append_gds = 0
+; HSA-TRAP-GFX803-V2-NEXT:     private_element_size = 1
+; HSA-TRAP-GFX803-V2-NEXT:     is_ptr64 = 1
+; HSA-TRAP-GFX803-V2-NEXT:     is_dynamic_callstack = 0
+; HSA-TRAP-GFX803-V2-NEXT:     is_debug_enabled = 0
+; HSA-TRAP-GFX803-V2-NEXT:     is_xnack_enabled = 0
+; HSA-TRAP-GFX803-V2-NEXT:     workitem_private_segment_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     workgroup_group_segment_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     gds_segment_byte_size = 0
+; HSA-TRAP-GFX803-V2-NEXT:     kernarg_segment_byte_size = 8
+; HSA-TRAP-GFX803-V2-NEXT:     workgroup_fbarrier_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     wavefront_sgpr_count = 8
+; HSA-TRAP-GFX803-V2-NEXT:     workitem_vgpr_count = 4
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_vgpr_first = 0
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_vgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_sgpr_first = 0
+; HSA-TRAP-GFX803-V2-NEXT:     reserved_sgpr_count = 0
+; HSA-TRAP-GFX803-V2-NEXT:     debug_wavefront_private_segment_offset_sgpr = 0
+; HSA-TRAP-GFX803-V2-NEXT:     debug_private_segment_buffer_sgpr = 0
+; HSA-TRAP-GFX803-V2-NEXT:     kernarg_segment_alignment = 4
+; HSA-TRAP-GFX803-V2-NEXT:     group_segment_alignment = 4
+; HSA-TRAP-GFX803-V2-NEXT:     private_segment_alignment = 4
+; HSA-TRAP-GFX803-V2-NEXT:     wavefront_size = 6
+; HSA-TRAP-GFX803-V2-NEXT:     call_convention = -1
+; HSA-TRAP-GFX803-V2-NEXT:     runtime_loader_kernel_symbol = 0
+; HSA-TRAP-GFX803-V2-NEXT:    .end_amd_kernel_code_t
+; HSA-TRAP-GFX803-V2-NEXT:  ; %bb.0:
+; HSA-TRAP-GFX803-V2-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v2, 1
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v3, 2
+; HSA-TRAP-GFX803-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-V2-NEXT:    v_mov_b32_e32 v1, s1
+; HSA-TRAP-GFX803-V2-NEXT:    flat_store_dword v[0:1], v2
+; HSA-TRAP-GFX803-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V2-NEXT:    s_trap 3
+; HSA-TRAP-GFX803-V2-NEXT:    flat_store_dword v[0:1], v3
+; HSA-TRAP-GFX803-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V2-NEXT:    s_endpgm
+;
+; HSA-TRAP-GFX803-V3-LABEL: debugtrap:
+; HSA-TRAP-GFX803-V3:       ; %bb.0:
+; HSA-TRAP-GFX803-V3-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v2, 1
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v3, 2
+; HSA-TRAP-GFX803-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-V3-NEXT:    v_mov_b32_e32 v1, s1
+; HSA-TRAP-GFX803-V3-NEXT:    flat_store_dword v[0:1], v2
+; HSA-TRAP-GFX803-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V3-NEXT:    s_trap 3
+; HSA-TRAP-GFX803-V3-NEXT:    flat_store_dword v[0:1], v3
+; HSA-TRAP-GFX803-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V3-NEXT:    s_endpgm
+;
+; HSA-TRAP-GFX803-V4-LABEL: debugtrap:
+; HSA-TRAP-GFX803-V4:       ; %bb.0:
+; HSA-TRAP-GFX803-V4-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v2, 1
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v3, 2
+; HSA-TRAP-GFX803-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v0, s0
+; HSA-TRAP-GFX803-V4-NEXT:    v_mov_b32_e32 v1, s1
+; HSA-TRAP-GFX803-V4-NEXT:    flat_store_dword v[0:1], v2
+; HSA-TRAP-GFX803-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V4-NEXT:    s_trap 3
+; HSA-TRAP-GFX803-V4-NEXT:    flat_store_dword v[0:1], v3
+; HSA-TRAP-GFX803-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX803-V4-NEXT:    s_endpgm
+;
+; HSA-TRAP-GFX900-V2-LABEL: debugtrap:
+; HSA-TRAP-GFX900-V2:         .amd_kernel_code_t
+; HSA-TRAP-GFX900-V2-NEXT:     amd_code_version_major = 1
+; HSA-TRAP-GFX900-V2-NEXT:     amd_code_version_minor = 2
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_kind = 1
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_version_major = 9
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_version_minor = 0
+; HSA-TRAP-GFX900-V2-NEXT:     amd_machine_version_stepping = 0
+; HSA-TRAP-GFX900-V2-NEXT:     kernel_code_entry_byte_offset = 256
+; HSA-TRAP-GFX900-V2-NEXT:     kernel_code_prefetch_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     granulated_workitem_vgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     granulated_wavefront_sgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     priority = 0
+; HSA-TRAP-GFX900-V2-NEXT:     float_mode = 240
+; HSA-TRAP-GFX900-V2-NEXT:     priv = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_dx10_clamp = 1
+; HSA-TRAP-GFX900-V2-NEXT:     debug_mode = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_ieee_mode = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_wgp_mode = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_mem_ordered = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_fwd_progress = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
+; HSA-TRAP-GFX900-V2-NEXT:     user_sgpr_count = 8
+; HSA-TRAP-GFX900-V2-NEXT:     enable_trap_handler = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_x = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_y = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_z = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_info = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_vgpr_workitem_id = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_exception_msb = 0
+; HSA-TRAP-GFX900-V2-NEXT:     granulated_lds_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_exception = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_buffer = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_ptr = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_queue_ptr = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_id = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_flat_scratch_init = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_z = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_wavefront_size32 = 0
+; HSA-TRAP-GFX900-V2-NEXT:     enable_ordered_append_gds = 0
+; HSA-TRAP-GFX900-V2-NEXT:     private_element_size = 1
+; HSA-TRAP-GFX900-V2-NEXT:     is_ptr64 = 1
+; HSA-TRAP-GFX900-V2-NEXT:     is_dynamic_callstack = 0
+; HSA-TRAP-GFX900-V2-NEXT:     is_debug_enabled = 0
+; HSA-TRAP-GFX900-V2-NEXT:     is_xnack_enabled = 1
+; HSA-TRAP-GFX900-V2-NEXT:     workitem_private_segment_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     workgroup_group_segment_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     gds_segment_byte_size = 0
+; HSA-TRAP-GFX900-V2-NEXT:     kernarg_segment_byte_size = 8
+; HSA-TRAP-GFX900-V2-NEXT:     workgroup_fbarrier_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     wavefront_sgpr_count = 8
+; HSA-TRAP-GFX900-V2-NEXT:     workitem_vgpr_count = 3
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_vgpr_first = 0
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_vgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_sgpr_first = 0
+; HSA-TRAP-GFX900-V2-NEXT:     reserved_sgpr_count = 0
+; HSA-TRAP-GFX900-V2-NEXT:     debug_wavefront_private_segment_offset_sgpr = 0
+; HSA-TRAP-GFX900-V2-NEXT:     debug_private_segment_buffer_sgpr = 0
+; HSA-TRAP-GFX900-V2-NEXT:     kernarg_segment_alignment = 4
+; HSA-TRAP-GFX900-V2-NEXT:     group_segment_alignment = 4
+; HSA-TRAP-GFX900-V2-NEXT:     private_segment_alignment = 4
+; HSA-TRAP-GFX900-V2-NEXT:     wavefront_size = 6
+; HSA-TRAP-GFX900-V2-NEXT:     call_convention = -1
+; HSA-TRAP-GFX900-V2-NEXT:     runtime_loader_kernel_symbol = 0
+; HSA-TRAP-GFX900-V2-NEXT:    .end_amd_kernel_code_t
+; HSA-TRAP-GFX900-V2-NEXT:  ; %bb.0:
+; HSA-TRAP-GFX900-V2-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-TRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v2, 2
+; HSA-TRAP-GFX900-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX900-V2-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V2-NEXT:    s_trap 3
+; HSA-TRAP-GFX900-V2-NEXT:    global_store_dword v0, v2, s[0:1]
+; HSA-TRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V2-NEXT:    s_endpgm
+;
+; HSA-TRAP-GFX900-V3-LABEL: debugtrap:
+; HSA-TRAP-GFX900-V3:       ; %bb.0:
+; HSA-TRAP-GFX900-V3-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-TRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v2, 2
+; HSA-TRAP-GFX900-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX900-V3-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V3-NEXT:    s_trap 3
+; HSA-TRAP-GFX900-V3-NEXT:    global_store_dword v0, v2, s[0:1]
+; HSA-TRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V3-NEXT:    s_endpgm
+;
+; HSA-TRAP-GFX900-V4-LABEL: debugtrap:
+; HSA-TRAP-GFX900-V4:       ; %bb.0:
+; HSA-TRAP-GFX900-V4-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-TRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v2, 2
+; HSA-TRAP-GFX900-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-TRAP-GFX900-V4-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V4-NEXT:    s_trap 3
+; HSA-TRAP-GFX900-V4-NEXT:    global_store_dword v0, v2, s[0:1]
+; HSA-TRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-TRAP-GFX900-V4-NEXT:    s_endpgm
+;
+; HSA-NOTRAP-GFX900-V2-LABEL: debugtrap:
+; HSA-NOTRAP-GFX900-V2:         .amd_kernel_code_t
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_code_version_major = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_code_version_minor = 2
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_kind = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_version_major = 9
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_version_minor = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     amd_machine_version_stepping = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernel_code_entry_byte_offset = 256
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernel_code_prefetch_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     granulated_workitem_vgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     granulated_wavefront_sgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     priority = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     float_mode = 240
+; HSA-NOTRAP-GFX900-V2-NEXT:     priv = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_dx10_clamp = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     debug_mode = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_ieee_mode = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_wgp_mode = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_mem_ordered = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_fwd_progress = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_wave_byte_offset = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     user_sgpr_count = 8
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_trap_handler = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_x = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_y = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_id_z = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_workgroup_info = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_vgpr_workitem_id = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_exception_msb = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     granulated_lds_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_exception = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_buffer = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_ptr = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_queue_ptr = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_kernarg_segment_ptr = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_dispatch_id = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_flat_scratch_init = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_private_segment_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_x = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_y = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_sgpr_grid_workgroup_count_z = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_wavefront_size32 = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     enable_ordered_append_gds = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     private_element_size = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_ptr64 = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_dynamic_callstack = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_debug_enabled = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     is_xnack_enabled = 1
+; HSA-NOTRAP-GFX900-V2-NEXT:     workitem_private_segment_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     workgroup_group_segment_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     gds_segment_byte_size = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernarg_segment_byte_size = 8
+; HSA-NOTRAP-GFX900-V2-NEXT:     workgroup_fbarrier_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     wavefront_sgpr_count = 8
+; HSA-NOTRAP-GFX900-V2-NEXT:     workitem_vgpr_count = 3
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_vgpr_first = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_vgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_sgpr_first = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     reserved_sgpr_count = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     debug_wavefront_private_segment_offset_sgpr = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     debug_private_segment_buffer_sgpr = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:     kernarg_segment_alignment = 4
+; HSA-NOTRAP-GFX900-V2-NEXT:     group_segment_alignment = 4
+; HSA-NOTRAP-GFX900-V2-NEXT:     private_segment_alignment = 4
+; HSA-NOTRAP-GFX900-V2-NEXT:     wavefront_size = 6
+; HSA-NOTRAP-GFX900-V2-NEXT:     call_convention = -1
+; HSA-NOTRAP-GFX900-V2-NEXT:     runtime_loader_kernel_symbol = 0
+; HSA-NOTRAP-GFX900-V2-NEXT:    .end_amd_kernel_code_t
+; HSA-NOTRAP-GFX900-V2-NEXT:  ; %bb.0:
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-NOTRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-NOTRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-NOTRAP-GFX900-V2-NEXT:    v_mov_b32_e32 v2, 2
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-NOTRAP-GFX900-V2-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V2-NEXT:    global_store_dword v0, v2, s[0:1]
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V2-NEXT:    s_endpgm
+;
+; HSA-NOTRAP-GFX900-V3-LABEL: debugtrap:
+; HSA-NOTRAP-GFX900-V3:       ; %bb.0:
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-NOTRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-NOTRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-NOTRAP-GFX900-V3-NEXT:    v_mov_b32_e32 v2, 2
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-NOTRAP-GFX900-V3-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V3-NEXT:    global_store_dword v0, v2, s[0:1]
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V3-NEXT:    s_endpgm
+;
+; HSA-NOTRAP-GFX900-V4-LABEL: debugtrap:
+; HSA-NOTRAP-GFX900-V4:       ; %bb.0:
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_load_dwordx2 s[0:1], s[6:7], 0x0
+; HSA-NOTRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v0, 0
+; HSA-NOTRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v1, 1
+; HSA-NOTRAP-GFX900-V4-NEXT:    v_mov_b32_e32 v2, 2
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_waitcnt lgkmcnt(0)
+; HSA-NOTRAP-GFX900-V4-NEXT:    global_store_dword v0, v1, s[0:1]
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V4-NEXT:    global_store_dword v0, v2, s[0:1]
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_waitcnt vmcnt(0)
+; HSA-NOTRAP-GFX900-V4-NEXT:    s_endpgm
+  store volatile i32 1, i32 addrspace(1)* %arg0
+  call void @llvm.debugtrap()
+  store volatile i32 2, i32 addrspace(1)* %arg0
+  ret void
+}
+
+attributes #0 = { nounwind noreturn }
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index 779c37b65a28651d118425af6b2c257b9892cc14..95303d81def0879564862c55613239cd48e0be4b 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -969,14 +969,14 @@ define amdgpu_kernel void @s_test_udiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
 ; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
+; GCN-NEXT:    v_mul_hi_u32 v8, v3, v6
+; GCN-NEXT:    v_mul_lo_u32 v6, v3, v6
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; GCN-NEXT:    v_mul_hi_u32 v10, v3, v4
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v8, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v10, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
@@ -999,27 +999,24 @@ define amdgpu_kernel void @s_test_udiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v3, v1, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, 24
 ; GCN-NEXT:    v_mul_hi_u32 v0, v0, 24
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, 24
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, 24
 ; GCN-NEXT:    v_mov_b32_e32 v5, s7
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v3, vcc
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v1, s7, v0
 ; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
-; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
 ; GCN-NEXT:    v_mul_lo_u32 v3, s6, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
 ; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 24, v3
 ; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
 ; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s6, v3
@@ -1031,21 +1028,21 @@ define amdgpu_kernel void @s_test_udiv_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s7, v4
 ; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
 ; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
-; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v2, s[0:1]
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
-; GCN-NEXT:    v_subb_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v2, s[0:1], 0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v1
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
-; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s7, v2
-; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v3
 ; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v2
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s7, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v2, vcc
 ; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; GCN-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/urem-seteq-illegal-types.ll b/llvm/test/CodeGen/AMDGPU/urem-seteq-illegal-types.ll
index ea8068a8f5ad890aa38585a21df9df722f6b0d64..eee9a4e6973879be5709ba2c35a7c19b4ee62ccb 100644
--- a/llvm/test/CodeGen/AMDGPU/urem-seteq-illegal-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/urem-seteq-illegal-types.ll
@@ -9,7 +9,7 @@ define i1 @test_urem_odd(i13 %X) nounwind {
 ; CHECK-NEXT:    s_mov_b32 s4, 0xcccccccd
 ; CHECK-NEXT:    v_mul_hi_u32 v1, v0, s4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 2, v1
-; CHECK-NEXT:    v_mul_lo_u32 v1, v1, 5
+; CHECK-NEXT:    v_mul_u32_u24_e32 v1, 5, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -28,7 +28,7 @@ define i1 @test_urem_even(i27 %X) nounwind {
 ; CHECK-NEXT:    s_mov_b32 s4, 0x92492493
 ; CHECK-NEXT:    v_mul_hi_u32 v0, v0, s4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v0, 2, v0
-; CHECK-NEXT:    v_mul_lo_u32 v0, v0, 14
+; CHECK-NEXT:    v_mul_u32_u24_e32 v0, 14, v0
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v1, v0
 ; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -46,7 +46,7 @@ define i1 @test_urem_odd_setne(i4 %X) nounwind {
 ; CHECK-NEXT:    s_mov_b32 s4, 0xcccccccd
 ; CHECK-NEXT:    v_mul_hi_u32 v1, v0, s4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 2, v1
-; CHECK-NEXT:    v_mul_lo_u32 v1, v1, 5
+; CHECK-NEXT:    v_mul_u32_u24_e32 v1, 5, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -62,10 +62,9 @@ define i1 @test_urem_negative_odd(i9 %X) nounwind {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x1ff, v0
 ; CHECK-NEXT:    s_mov_b32 s4, 0x2050c9f9
-; CHECK-NEXT:    s_movk_i32 s5, 0x1fb
 ; CHECK-NEXT:    v_mul_hi_u32 v1, v0, s4
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 6, v1
-; CHECK-NEXT:    v_mul_lo_u32 v1, v1, s5
+; CHECK-NEXT:    v_mul_u32_u24_e32 v1, 0x1fb, v1
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll
index a0eba73e7d0ff07d84d21a712f1bfc43948b5473..8458512b3f7547e937598241df7e3e098791d33e 100644
--- a/llvm/test/CodeGen/AMDGPU/urem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/urem64.ll
@@ -779,14 +779,14 @@ define amdgpu_kernel void @s_test_urem_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_hi_u32 v5, v0, v6
 ; GCN-NEXT:    v_mul_lo_u32 v7, v0, v4
 ; GCN-NEXT:    v_mul_hi_u32 v9, v0, v4
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
+; GCN-NEXT:    v_mul_hi_u32 v8, v3, v6
+; GCN-NEXT:    v_mul_lo_u32 v6, v3, v6
 ; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; GCN-NEXT:    v_mul_hi_u32 v10, v3, v4
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v2, v9, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v8, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v10, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
 ; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v4
@@ -809,27 +809,24 @@ define amdgpu_kernel void @s_test_urem_k_num_i64(i64 addrspace(1)* %out, i64 %x)
 ; GCN-NEXT:    v_mul_lo_u32 v4, v4, v6
 ; GCN-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
 ; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v8, v1, vcc
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v3, v6, s[0:1]
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v3, v1, s[0:1]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, 24
-; GCN-NEXT:    v_mul_hi_u32 v0, v0, 24
-; GCN-NEXT:    v_mul_hi_u32 v3, v3, 24
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v3, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT:    v_mul_lo_u32 v1, s6, v1
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, 24
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, 24
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, 24
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v2, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v1, s7, v0
 ; GCN-NEXT:    v_mul_hi_u32 v2, s6, v0
-; GCN-NEXT:    v_mul_lo_u32 v3, s7, v0
 ; GCN-NEXT:    v_mul_lo_u32 v0, s6, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, s7
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
-; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_sub_i32_e32 v2, vcc, 0, v1
-; GCN-NEXT:    v_mov_b32_e32 v3, s7
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
 ; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
 ; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s6, v0
diff --git a/llvm/test/CodeGen/AMDGPU/usubsat.ll b/llvm/test/CodeGen/AMDGPU/usubsat.ll
index eeabd31f90624ee76e6f1558f337aaae5bac2a3c..13f61b6ea131f3d226e6ef9d95384616f8ce4c15 100644
--- a/llvm/test/CodeGen/AMDGPU/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/usubsat.ll
@@ -147,8 +147,8 @@ define <3 x i16> @v_usubsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
 ; GFX9-LABEL: v_usubsat_v3i16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_sub_u16 v0, v0, v2 clamp
 ; GFX9-NEXT:    v_pk_sub_u16 v1, v1, v3 clamp
+; GFX9-NEXT:    v_pk_sub_u16 v0, v0, v2 clamp
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i16> @llvm.usub.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
   ret <3 x i16> %result
diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll
index 384d86dbc828c53176c7e4a73507b13caf0fce60..118beab8d295f8a0568f9a84b104a7f90a074f4d 100644
--- a/llvm/test/CodeGen/ARM/O3-pipeline.ll
+++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll
@@ -10,6 +10,7 @@
 ; CHECK-NEXT:      Dominator Tree Construction
 ; CHECK-NEXT:      Natural Loop Information
 ; CHECK-NEXT:      MVE gather/scatter lowering
+; CHECK-NEXT:      MVE lane interleaving
 ; CHECK-NEXT:      Module Verifier
 ; CHECK-NEXT:      Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:      Canonicalize natural loops
diff --git a/llvm/test/CodeGen/ARM/load.ll b/llvm/test/CodeGen/ARM/load.ll
index f33294267f518b3658de716d1d6fb516140116e6..d23d2693ec01128c5eea96a766506c37cf11eb29 100644
--- a/llvm/test/CodeGen/ARM/load.ll
+++ b/llvm/test/CodeGen/ARM/load.ll
@@ -96,8 +96,8 @@ entry:
 ; Immediate offset of zero
 
 ; CHECK-LABEL: ldrsb_ri_zero
-; CHECK-T1: ldrb    r0, [r0]
-; CHECK-T1: sxtb    r0, r0
+; CHECK-T1: movs    r1, #0
+; CHECK-T1: ldrsb   r0, [r0, r1]
 ; CHECK-T2: ldrsb.w r0, [r0]
 define i32 @ldrsb_ri_zero(i8* %p) {
 entry:
@@ -107,8 +107,8 @@ entry:
 }
 
 ; CHECK-LABEL: ldrsh_ri_zero
-; CHECK-T1: ldrh    r0, [r0]
-; CHECK-T1: sxth    r0, r0
+; CHECK-T1: movs    r1, #0
+; CHECK-T1: ldrsh   r0, [r0, r1]
 ; CHECK-T2: ldrsh.w r0, [r0]
 define i32 @ldrsh_ri_zero(i16* %p) {
 entry:
diff --git a/llvm/test/CodeGen/ARM/select-imm.ll b/llvm/test/CodeGen/ARM/select-imm.ll
index 5cdfe3326e922164e03ceaee7df30aeb40033cfb..b06394fa2bb9e5ad2cff7e9c8e8a01e452e34cf0 100644
--- a/llvm/test/CodeGen/ARM/select-imm.ll
+++ b/llvm/test/CodeGen/ARM/select-imm.ll
@@ -1,104 +1,212 @@
-; RUN: llc -mtriple=arm-eabi %s -o - | FileCheck %s --check-prefix=ARM
-
-; RUN: llc -mtriple=arm-eabi -mcpu=arm1156t2-s -mattr=+thumb2 %s -o - \
-; RUN:  | FileCheck %s --check-prefix=ARMT2
-
-; RUN: llc -mtriple=thumb-eabi -mcpu=cortex-m0 %s -o - \
-; RUN:  | FileCheck %s --check-prefix=THUMB1
-
-; RUN: llc -mtriple=thumb-eabi -mcpu=arm1156t2-s -mattr=+thumb2 %s -o - \
-; RUN:  | FileCheck %s --check-prefix=THUMB2
-
-; RUN: llc -mtriple=thumbv8m.base-eabi %s -o - \
-; RUN:  | FileCheck %s --check-prefix=V8MBASE
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=arm-eabi | FileCheck %s --check-prefix=ARM
+; RUN: llc < %s -mtriple=arm-eabi -mcpu=arm1156t2-s -mattr=+thumb2 | FileCheck %s --check-prefix=ARMT2
+; RUN: llc < %s -mtriple=thumb-eabi -mcpu=cortex-m0 | FileCheck %s --check-prefix=THUMB1
+; RUN: llc < %s -mtriple=thumb-eabi -mcpu=arm1156t2-s -mattr=+thumb2 | FileCheck %s --check-prefix=THUMB2
+; RUN: llc < %s -mtriple=thumbv8m.base-eabi | FileCheck %s --check-prefix=V8MBASE
 
 define i32 @t1(i32 %c) nounwind readnone {
-entry:
 ; ARM-LABEL: t1:
-; ARM: mov [[R1:r[0-9]+]], #101
-; ARM: orr [[R1b:r[0-9]+]], [[R1]], #256
-; ARM: movgt {{r[0-1]}}, #123
-
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    mov r1, #101
+; ARM-NEXT:    cmp r0, #1
+; ARM-NEXT:    orr r1, r1, #256
+; ARM-NEXT:    movgt r1, #123
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
 ; ARMT2-LABEL: t1:
-; ARMT2: movw [[R:r[0-1]]], #357
-; ARMT2: movwgt [[R]], #123
-
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    movw r1, #357
+; ARMT2-NEXT:    cmp r0, #1
+; ARMT2-NEXT:    movwgt r1, #123
+; ARMT2-NEXT:    mov r0, r1
+; ARMT2-NEXT:    bx lr
+;
 ; THUMB1-LABEL: t1:
-; THUMB1: cmp     r0, #1
-; THUMB1: bgt     .LBB0_2
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    cmp r0, #1
+; THUMB1-NEXT:    bgt .LBB0_2
+; THUMB1-NEXT:  @ %bb.1: @ %entry
+; THUMB1-NEXT:    movs r0, #255
+; THUMB1-NEXT:    adds r0, #102
+; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:  .LBB0_2:
+; THUMB1-NEXT:    movs r0, #123
+; THUMB1-NEXT:    bx lr
+;
 ; THUMB2-LABEL: t1:
-; THUMB2: movw [[R:r[0-1]]], #357
-; THUMB2: movgt [[R]], #123
-
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    movw r1, #357
+; THUMB2-NEXT:    cmp r0, #1
+; THUMB2-NEXT:    it gt
+; THUMB2-NEXT:    movgt r1, #123
+; THUMB2-NEXT:    mov r0, r1
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t1:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    cmp r0, #1
+; V8MBASE-NEXT:    bgt .LBB0_2
+; V8MBASE-NEXT:  @ %bb.1: @ %entry
+; V8MBASE-NEXT:    movw r0, #357
+; V8MBASE-NEXT:    bx lr
+; V8MBASE-NEXT:  .LBB0_2:
+; V8MBASE-NEXT:    movs r0, #123
+; V8MBASE-NEXT:    bx lr
+entry:
   %0 = icmp sgt i32 %c, 1
   %1 = select i1 %0, i32 123, i32 357
   ret i32 %1
 }
 
 define i32 @t2(i32 %c) nounwind readnone {
-entry:
 ; ARM-LABEL: t2:
-; ARM: mov [[R:r[0-9]+]], #101
-; ARM: orr [[R]], [[R]], #256
-; ARM: movle [[R]], #123
-
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    mov r1, #101
+; ARM-NEXT:    cmp r0, #1
+; ARM-NEXT:    orr r1, r1, #256
+; ARM-NEXT:    movle r1, #123
+; ARM-NEXT:    mov r0, r1
+; ARM-NEXT:    mov pc, lr
+;
 ; ARMT2-LABEL: t2:
-; ARMT2: mov [[R:r[0-1]]], #123
-; ARMT2: movwgt [[R]], #357
-
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    mov r1, #123
+; ARMT2-NEXT:    cmp r0, #1
+; ARMT2-NEXT:    movwgt r1, #357
+; ARMT2-NEXT:    mov r0, r1
+; ARMT2-NEXT:    bx lr
+;
 ; THUMB1-LABEL: t2:
-; THUMB1: cmp r{{[0-9]+}}, #1
-; THUMB1: bgt
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    cmp r0, #1
+; THUMB1-NEXT:    bgt .LBB1_2
+; THUMB1-NEXT:  @ %bb.1: @ %entry
+; THUMB1-NEXT:    movs r0, #123
+; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:  .LBB1_2:
+; THUMB1-NEXT:    movs r0, #255
+; THUMB1-NEXT:    adds r0, #102
+; THUMB1-NEXT:    bx lr
+;
 ; THUMB2-LABEL: t2:
-; THUMB2: mov{{(s|\.w)}} [[R:r[0-1]]], #123
-; THUMB2: movwgt [[R]], #357
-
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    movs r1, #123
+; THUMB2-NEXT:    cmp r0, #1
+; THUMB2-NEXT:    it gt
+; THUMB2-NEXT:    movwgt r1, #357
+; THUMB2-NEXT:    mov r0, r1
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t2:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    mov r1, r0
+; V8MBASE-NEXT:    movw r0, #357
+; V8MBASE-NEXT:    cmp r1, #1
+; V8MBASE-NEXT:    bgt .LBB1_2
+; V8MBASE-NEXT:  @ %bb.1: @ %entry
+; V8MBASE-NEXT:    movs r0, #123
+; V8MBASE-NEXT:  .LBB1_2: @ %entry
+; V8MBASE-NEXT:    bx lr
+entry:
   %0 = icmp sgt i32 %c, 1
   %1 = select i1 %0, i32 357, i32 123
   ret i32 %1
 }
 
 define i32 @t3(i32 %a) nounwind readnone {
-entry:
 ; ARM-LABEL: t3:
-; ARM: rsbs r1, r0, #0
-; ARM: adc  r0, r0, r1
-
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    sub r0, r0, #160
+; ARM-NEXT:    rsbs r1, r0, #0
+; ARM-NEXT:    adc r0, r0, r1
+; ARM-NEXT:    mov pc, lr
+;
 ; ARMT2-LABEL: t3:
-; ARMT2: clz r0, r0
-; ARMT2: lsr r0, r0, #5
-
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    sub r0, r0, #160
+; ARMT2-NEXT:    clz r0, r0
+; ARMT2-NEXT:    lsr r0, r0, #5
+; ARMT2-NEXT:    bx lr
+;
 ; THUMB1-LABEL: t3:
-; THUMB1: rsbs r1, r0, #0
-; THUMB1: adcs r0, r1
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    subs r0, #160
+; THUMB1-NEXT:    rsbs r1, r0, #0
+; THUMB1-NEXT:    adcs r0, r1
+; THUMB1-NEXT:    bx lr
+;
 ; THUMB2-LABEL: t3:
-; THUMB2: clz r0, r0
-; THUMB2: lsrs r0, r0, #5
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    subs r0, #160
+; THUMB2-NEXT:    clz r0, r0
+; THUMB2-NEXT:    lsrs r0, r0, #5
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t3:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    subs r0, #160
+; V8MBASE-NEXT:    rsbs r1, r0, #0
+; V8MBASE-NEXT:    adcs r0, r1
+; V8MBASE-NEXT:    bx lr
+entry:
   %0 = icmp eq i32 %a, 160
   %1 = zext i1 %0 to i32
   ret i32 %1
 }
 
 define i32 @t4(i32 %a, i32 %b, i32 %x) nounwind {
-entry:
 ; ARM-LABEL: t4:
-; ARM: mvn [[R0:r[0-9]+]], #170
-; ARM: sub [[R0:r[0-9]+]], [[R0:r[0-9]+]], #11141120
-; ARM: mov{{lt|ge}}
-
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    mvn r3, #170
+; ARM-NEXT:    cmp r0, r1
+; ARM-NEXT:    sub r3, r3, #11141120
+; ARM-NEXT:    movge r3, r2
+; ARM-NEXT:    mov r0, r3
+; ARM-NEXT:    mov pc, lr
+;
 ; ARMT2-LABEL: t4:
-; ARMT2: movwlt [[R0:r[0-9]+]], #65365
-; ARMT2: movtlt [[R0]], #65365
-
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    cmp r0, r1
+; ARMT2-NEXT:    movwlt r2, #65365
+; ARMT2-NEXT:    movtlt r2, #65365
+; ARMT2-NEXT:    mov r0, r2
+; ARMT2-NEXT:    bx lr
+;
 ; THUMB1-LABEL: t4:
-; THUMB1: cmp r{{[0-9]+}}, r{{[0-9]+}}
-; THUMB1: b{{lt|ge}}
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    cmp r0, r1
+; THUMB1-NEXT:    bge .LBB3_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    ldr r2, .LCPI3_0
+; THUMB1-NEXT:  .LBB3_2: @ %entry
+; THUMB1-NEXT:    mov r0, r2
+; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:    .p2align 2
+; THUMB1-NEXT:  @ %bb.3:
+; THUMB1-NEXT:  .LCPI3_0:
+; THUMB1-NEXT:    .long 4283826005 @ 0xff55ff55
+;
 ; THUMB2-LABEL: t4:
-; THUMB2: mvnlt [[R0:r[0-9]+]], #11141290
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    cmp r0, r1
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    mvnlt r2, #11141290
+; THUMB2-NEXT:    mov r0, r2
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t4:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    cmp r0, r1
+; V8MBASE-NEXT:    bge .LBB3_2
+; V8MBASE-NEXT:  @ %bb.1:
+; V8MBASE-NEXT:    movw r2, #65365
+; V8MBASE-NEXT:    movt r2, #65365
+; V8MBASE-NEXT:  .LBB3_2: @ %entry
+; V8MBASE-NEXT:    mov r0, r2
+; V8MBASE-NEXT:    bx lr
+entry:
   %0 = icmp slt i32 %a, %b
   %1 = select i1 %0, i32 4283826005, i32 %x
   ret i32 %1
@@ -106,105 +214,206 @@ entry:
 
 ; rdar://9758317
 define i32 @t5(i32 %a) nounwind {
-entry:
 ; ARM-LABEL: t5:
-; ARM-NOT: mov
-; ARM: sub  r0, r0, #1
-; ARM-NOT: mov
-; ARM: rsbs r1, r0, #0
-; ARM: adc  r0, r0, r1
-
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    sub r0, r0, #1
+; ARM-NEXT:    rsbs r1, r0, #0
+; ARM-NEXT:    adc r0, r0, r1
+; ARM-NEXT:    mov pc, lr
+;
+; ARMT2-LABEL: t5:
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    sub r0, r0, #1
+; ARMT2-NEXT:    clz r0, r0
+; ARMT2-NEXT:    lsr r0, r0, #5
+; ARMT2-NEXT:    bx lr
+;
 ; THUMB1-LABEL: t5:
-; THUMB1-NOT: bne
-; THUMB1: rsbs r0, r1, #0
-; THUMB1: adcs r0, r1
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    subs r1, r0, #1
+; THUMB1-NEXT:    rsbs r0, r1, #0
+; THUMB1-NEXT:    adcs r0, r1
+; THUMB1-NEXT:    bx lr
+;
 ; THUMB2-LABEL: t5:
-; THUMB2-NOT: mov
-; THUMB2: subs r0, #1
-; THUMB2: clz  r0, r0
-; THUMB2: lsrs r0, r0, #5
-
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    subs r0, #1
+; THUMB2-NEXT:    clz r0, r0
+; THUMB2-NEXT:    lsrs r0, r0, #5
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t5:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    subs r1, r0, #1
+; V8MBASE-NEXT:    rsbs r0, r1, #0
+; V8MBASE-NEXT:    adcs r0, r1
+; V8MBASE-NEXT:    bx lr
+entry:
   %cmp = icmp eq i32 %a, 1
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @t6(i32 %a) nounwind {
-entry:
 ; ARM-LABEL: t6:
-; ARM-NOT: mov
-; ARM: cmp r0, #0
-; ARM: movne r0, #1
-
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    cmp r0, #0
+; ARM-NEXT:    movne r0, #1
+; ARM-NEXT:    mov pc, lr
+;
+; ARMT2-LABEL: t6:
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    cmp r0, #0
+; ARMT2-NEXT:    movwne r0, #1
+; ARMT2-NEXT:    bx lr
+;
 ; THUMB1-LABEL: t6:
-; THUMB1: subs r1, r0, #1
-; THUMB1: sbcs r0, r1
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    subs r1, r0, #1
+; THUMB1-NEXT:    sbcs r0, r1
+; THUMB1-NEXT:    bx lr
+;
 ; THUMB2-LABEL: t6:
-; THUMB2-NOT: mov
-; THUMB2: cmp r0, #0
-; THUMB2: it ne
-; THUMB2: movne r0, #1
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    cmp r0, #0
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r0, #1
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t6:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    subs r1, r0, #1
+; V8MBASE-NEXT:    sbcs r0, r1
+; V8MBASE-NEXT:    bx lr
+entry:
   %tobool = icmp ne i32 %a, 0
   %lnot.ext = zext i1 %tobool to i32
   ret i32 %lnot.ext
 }
 
 define i32 @t7(i32 %a, i32 %b) nounwind readnone {
-entry:
 ; ARM-LABEL: t7:
-; ARM: subs r0, r0, r1
-; ARM: movne   r0, #1
-; ARM: lsl     r0, r0, #2
-
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    subs r0, r0, r1
+; ARM-NEXT:    movne r0, #1
+; ARM-NEXT:    lsl r0, r0, #2
+; ARM-NEXT:    mov pc, lr
+;
 ; ARMT2-LABEL: t7:
-; ARMT2: subs r0, r0, r1
-; ARMT2: movwne r0, #1
-; ARMT2: lsl     r0, r0, #2
-
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    subs r0, r0, r1
+; ARMT2-NEXT:    movwne r0, #1
+; ARMT2-NEXT:    lsl r0, r0, #2
+; ARMT2-NEXT:    bx lr
+;
 ; THUMB1-LABEL: t7:
-; THUMB1: subs r0, r0, r1
-; THUMB1: subs r1, r0, #1
-; THUMB1: sbcs r0, r1
-; THUMB1: lsls r0, r0, #2
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    subs r0, r0, r1
+; THUMB1-NEXT:    subs r1, r0, #1
+; THUMB1-NEXT:    sbcs r0, r1
+; THUMB1-NEXT:    lsls r0, r0, #2
+; THUMB1-NEXT:    bx lr
+;
 ; THUMB2-LABEL: t7:
-; THUMB2: subs r0, r0, r1
-; THUMB2: it ne
-; THUMB2: movne r0, #1
-; THUMB2: lsls    r0, r0, #2
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    subs r0, r0, r1
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r0, #1
+; THUMB2-NEXT:    lsls r0, r0, #2
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t7:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    subs r0, r0, r1
+; V8MBASE-NEXT:    subs r1, r0, #1
+; V8MBASE-NEXT:    sbcs r0, r1
+; V8MBASE-NEXT:    lsls r0, r0, #2
+; V8MBASE-NEXT:    bx lr
+entry:
   %0 = icmp ne i32 %a, %b
   %1 = select i1 %0, i32 4, i32 0
   ret i32 %1
 }
 
-define void @t8(i32 %a) {
-entry:
-
 ; ARM scheduler emits icmp/zext before both calls, so isn't relevant
-
+define void @t8(i32 %a) {
+; ARM-LABEL: t8:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    .save {r4, lr}
+; ARM-NEXT:    push {r4, lr}
+; ARM-NEXT:    mov r1, r0
+; ARM-NEXT:    sub r0, r0, #5
+; ARM-NEXT:    rsbs r2, r0, #0
+; ARM-NEXT:    adc r4, r0, r2
+; ARM-NEXT:    mov r0, #9
+; ARM-NEXT:    bl t7
+; ARM-NEXT:    mov r1, r0
+; ARM-NEXT:    mov r0, r4
+; ARM-NEXT:    pop {r4, lr}
+; ARM-NEXT:    b t7
+;
 ; ARMT2-LABEL: t8:
-; ARMT2: bl t7
-; ARMT2: mov r1, r0
-; ARMT2: sub r0, r4, #5
-; ARMT2: clz r0, r0
-; ARMT2: lsr r0, r0, #5
-
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    .save {r4, lr}
+; ARMT2-NEXT:    push {r4, lr}
+; ARMT2-NEXT:    mov r4, r0
+; ARMT2-NEXT:    mov r0, #9
+; ARMT2-NEXT:    mov r1, r4
+; ARMT2-NEXT:    bl t7
+; ARMT2-NEXT:    mov r1, r0
+; ARMT2-NEXT:    sub r0, r4, #5
+; ARMT2-NEXT:    clz r0, r0
+; ARMT2-NEXT:    lsr r0, r0, #5
+; ARMT2-NEXT:    pop {r4, lr}
+; ARMT2-NEXT:    b t7
+;
 ; THUMB1-LABEL: t8:
-; THUMB1: bl t7
-; THUMB1: mov r1, r0
-; THUMB1: subs r2, r4, #5
-; THUMB1: rsbs r0, r2, #0
-; THUMB1: adcs r0, r2
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    .save {r4, lr}
+; THUMB1-NEXT:    push {r4, lr}
+; THUMB1-NEXT:    mov r4, r0
+; THUMB1-NEXT:    movs r0, #9
+; THUMB1-NEXT:    mov r1, r4
+; THUMB1-NEXT:    bl t7
+; THUMB1-NEXT:    mov r1, r0
+; THUMB1-NEXT:    subs r2, r4, #5
+; THUMB1-NEXT:    rsbs r0, r2, #0
+; THUMB1-NEXT:    adcs r0, r2
+; THUMB1-NEXT:    bl t7
+; THUMB1-NEXT:    pop {r4, pc}
+;
 ; THUMB2-LABEL: t8:
-; THUMB2: bl t7
-; THUMB2: mov r1, r0
-; THUMB2: subs r0, r4, #5
-; THUMB2: clz r0, r0
-; THUMB2: lsrs r0, r0, #5
-
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    .save {r4, lr}
+; THUMB2-NEXT:    push {r4, lr}
+; THUMB2-NEXT:    mov r4, r0
+; THUMB2-NEXT:    movs r0, #9
+; THUMB2-NEXT:    mov r1, r4
+; THUMB2-NEXT:    bl t7
+; THUMB2-NEXT:    mov r1, r0
+; THUMB2-NEXT:    subs r0, r4, #5
+; THUMB2-NEXT:    clz r0, r0
+; THUMB2-NEXT:    lsrs r0, r0, #5
+; THUMB2-NEXT:    pop.w {r4, lr}
+; THUMB2-NEXT:    b t7
+;
+; V8MBASE-LABEL: t8:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    .save {r4, lr}
+; V8MBASE-NEXT:    push {r4, lr}
+; V8MBASE-NEXT:    mov r1, r0
+; V8MBASE-NEXT:    subs r0, r0, #5
+; V8MBASE-NEXT:    rsbs r4, r0, #0
+; V8MBASE-NEXT:    adcs r4, r0
+; V8MBASE-NEXT:    movs r0, #9
+; V8MBASE-NEXT:    bl t7
+; V8MBASE-NEXT:    mov r1, r0
+; V8MBASE-NEXT:    mov r0, r4
+; V8MBASE-NEXT:    pop {r4}
+; V8MBASE-NEXT:    pop {r2}
+; V8MBASE-NEXT:    mov lr, r2
+; V8MBASE-NEXT:    b t7
+entry:
   %cmp = icmp eq i32 %a, 5
   %conv = zext i1 %cmp to i32
   %call = tail call i32 @t7(i32 9, i32 %a)
@@ -212,45 +421,127 @@ entry:
   ret void
 }
 
-define void @t9(i8* %a, i8 %b) {
-entry:
-
 ; ARM scheduler emits icmp/zext before both calls, so isn't relevant
-
+define void @t9(i8* %a, i8 %b) {
+; ARM-LABEL: t9:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    .save {r4, lr}
+; ARM-NEXT:    push {r4, lr}
+; ARM-NEXT:    ldrsb r4, [r0]
+; ARM-NEXT:    mov r0, #1
+; ARM-NEXT:    bl f
+; ARM-NEXT:    and r0, r4, #255
+; ARM-NEXT:    cmp r0, r0
+; ARM-NEXT:    bne .LBB8_3
+; ARM-NEXT:  @ %bb.1: @ %while.body.preheader
+; ARM-NEXT:    add r1, r4, #1
+; ARM-NEXT:    mov r2, r0
+; ARM-NEXT:  .LBB8_2: @ %while.body
+; ARM-NEXT:    @ =>This Inner Loop Header: Depth=1
+; ARM-NEXT:    add r2, r2, #1
+; ARM-NEXT:    add r1, r1, #1
+; ARM-NEXT:    and r3, r2, #255
+; ARM-NEXT:    cmp r3, r0
+; ARM-NEXT:    blt .LBB8_2
+; ARM-NEXT:  .LBB8_3: @ %while.end
+; ARM-NEXT:    pop {r4, lr}
+; ARM-NEXT:    mov pc, lr
+;
 ; ARMT2-LABEL: t9:
-; ARMT2: bl f
-; ARMT2: uxtb r0, r4
-; ARMT2: cmp  r0, r0
-; ARMT2: add  r1, r4, #1
-; ARMT2: mov  r2, r0
-; ARMT2: add  r2, r2, #1
-; ARMT2: add  r1, r1, #1
-; ARMT2: uxtb r3, r2
-; ARMT2: cmp  r3, r0
-
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    .save {r4, lr}
+; ARMT2-NEXT:    push {r4, lr}
+; ARMT2-NEXT:    ldrsb r4, [r0]
+; ARMT2-NEXT:    mov r0, #1
+; ARMT2-NEXT:    bl f
+; ARMT2-NEXT:    uxtb r0, r4
+; ARMT2-NEXT:    cmp r0, r0
+; ARMT2-NEXT:    popne {r4, pc}
+; ARMT2-NEXT:  .LBB8_1: @ %while.body.preheader
+; ARMT2-NEXT:    add r1, r4, #1
+; ARMT2-NEXT:    mov r2, r0
+; ARMT2-NEXT:  .LBB8_2: @ %while.body
+; ARMT2-NEXT:    @ =>This Inner Loop Header: Depth=1
+; ARMT2-NEXT:    add r2, r2, #1
+; ARMT2-NEXT:    add r1, r1, #1
+; ARMT2-NEXT:    uxtb r3, r2
+; ARMT2-NEXT:    cmp r3, r0
+; ARMT2-NEXT:    blt .LBB8_2
+; ARMT2-NEXT:  @ %bb.3: @ %while.end
+; ARMT2-NEXT:    pop {r4, pc}
+;
 ; THUMB1-LABEL: t9:
-; THUMB1: bl f
-; THUMB1: sxtb r1, r4
-; THUMB1: uxtb r0, r1
-; THUMB1: cmp  r0, r0
-; THUMB1: adds r1, r1, #1
-; THUMB1: mov  r2, r0
-; THUMB1: adds r1, r1, #1
-; THUMB1: adds r2, r2, #1
-; THUMB1: uxtb r3, r2
-; THUMB1: cmp  r3, r0
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    .save {r4, lr}
+; THUMB1-NEXT:    push {r4, lr}
+; THUMB1-NEXT:    movs r1, #0
+; THUMB1-NEXT:    ldrsb r4, [r0, r1]
+; THUMB1-NEXT:    movs r0, #1
+; THUMB1-NEXT:    bl f
+; THUMB1-NEXT:    uxtb r0, r4
+; THUMB1-NEXT:    cmp r0, r0
+; THUMB1-NEXT:    bne .LBB8_3
+; THUMB1-NEXT:  @ %bb.1: @ %while.body.preheader
+; THUMB1-NEXT:    adds r1, r4, #1
+; THUMB1-NEXT:    mov r2, r0
+; THUMB1-NEXT:  .LBB8_2: @ %while.body
+; THUMB1-NEXT:    @ =>This Inner Loop Header: Depth=1
+; THUMB1-NEXT:    adds r1, r1, #1
+; THUMB1-NEXT:    adds r2, r2, #1
+; THUMB1-NEXT:    uxtb r3, r2
+; THUMB1-NEXT:    cmp r3, r0
+; THUMB1-NEXT:    blt .LBB8_2
+; THUMB1-NEXT:  .LBB8_3: @ %while.end
+; THUMB1-NEXT:    pop {r4, pc}
+;
 ; THUMB2-LABEL: t9:
-; THUMB2: bl f
-; THUMB2: uxtb r0, r4
-; THUMB2: cmp  r0, r0
-; THUMB2: adds r1, r4, #1
-; THUMB2: mov  r2, r0
-; THUMB2: adds r2, #1
-; THUMB2: adds r1, #1
-; THUMB2: uxtb r3, r2
-; THUMB2: cmp  r3, r0
-
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    .save {r4, lr}
+; THUMB2-NEXT:    push {r4, lr}
+; THUMB2-NEXT:    ldrsb.w r4, [r0]
+; THUMB2-NEXT:    movs r0, #1
+; THUMB2-NEXT:    bl f
+; THUMB2-NEXT:    uxtb r0, r4
+; THUMB2-NEXT:    cmp r0, r0
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    popne {r4, pc}
+; THUMB2-NEXT:  .LBB8_1: @ %while.body.preheader
+; THUMB2-NEXT:    adds r1, r4, #1
+; THUMB2-NEXT:    mov r2, r0
+; THUMB2-NEXT:  .LBB8_2: @ %while.body
+; THUMB2-NEXT:    @ =>This Inner Loop Header: Depth=1
+; THUMB2-NEXT:    adds r2, #1
+; THUMB2-NEXT:    adds r1, #1
+; THUMB2-NEXT:    uxtb r3, r2
+; THUMB2-NEXT:    cmp r3, r0
+; THUMB2-NEXT:    blt .LBB8_2
+; THUMB2-NEXT:  @ %bb.3: @ %while.end
+; THUMB2-NEXT:    pop {r4, pc}
+;
+; V8MBASE-LABEL: t9:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    .save {r4, lr}
+; V8MBASE-NEXT:    push {r4, lr}
+; V8MBASE-NEXT:    movs r1, #0
+; V8MBASE-NEXT:    ldrsb r4, [r0, r1]
+; V8MBASE-NEXT:    movs r0, #1
+; V8MBASE-NEXT:    bl f
+; V8MBASE-NEXT:    uxtb r0, r4
+; V8MBASE-NEXT:    cmp r0, r0
+; V8MBASE-NEXT:    bne .LBB8_3
+; V8MBASE-NEXT:  @ %bb.1: @ %while.body.preheader
+; V8MBASE-NEXT:    adds r1, r4, #1
+; V8MBASE-NEXT:    mov r2, r0
+; V8MBASE-NEXT:  .LBB8_2: @ %while.body
+; V8MBASE-NEXT:    @ =>This Inner Loop Header: Depth=1
+; V8MBASE-NEXT:    adds r1, r1, #1
+; V8MBASE-NEXT:    adds r2, r2, #1
+; V8MBASE-NEXT:    uxtb r3, r2
+; V8MBASE-NEXT:    cmp r3, r0
+; V8MBASE-NEXT:    blt .LBB8_2
+; V8MBASE-NEXT:  .LBB8_3: @ %while.end
+; V8MBASE-NEXT:    pop {r4, pc}
+entry:
   %0 = load i8, i8* %a
   %conv = sext i8 %0 to i32
   %conv119 = zext i8 %0 to i32
@@ -275,8 +566,104 @@ while.end:
 
 declare void @f(i1 zeroext)
 
-
 define i1 @t10() {
+; ARM-LABEL: t10:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    .save {r11, lr}
+; ARM-NEXT:    push {r11, lr}
+; ARM-NEXT:    .pad #8
+; ARM-NEXT:    sub sp, sp, #8
+; ARM-NEXT:    mvn r0, #2
+; ARM-NEXT:    mvn r1, #7
+; ARM-NEXT:    str r0, [sp, #4]
+; ARM-NEXT:    mvn r0, #7
+; ARM-NEXT:    str r0, [sp]
+; ARM-NEXT:    mvn r0, #2
+; ARM-NEXT:    bl __aeabi_idivmod
+; ARM-NEXT:    sub r0, r1, r0, lsl #3
+; ARM-NEXT:    add r0, r0, #3
+; ARM-NEXT:    rsbs r1, r0, #0
+; ARM-NEXT:    adc r0, r0, r1
+; ARM-NEXT:    add sp, sp, #8
+; ARM-NEXT:    pop {r11, lr}
+; ARM-NEXT:    mov pc, lr
+;
+; ARMT2-LABEL: t10:
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    .save {r11, lr}
+; ARMT2-NEXT:    push {r11, lr}
+; ARMT2-NEXT:    .pad #8
+; ARMT2-NEXT:    sub sp, sp, #8
+; ARMT2-NEXT:    mvn r0, #2
+; ARMT2-NEXT:    str r0, [sp, #4]
+; ARMT2-NEXT:    mvn r0, #7
+; ARMT2-NEXT:    str r0, [sp]
+; ARMT2-NEXT:    mvn r0, #2
+; ARMT2-NEXT:    mvn r1, #7
+; ARMT2-NEXT:    bl __aeabi_idivmod
+; ARMT2-NEXT:    sub r0, r1, r0, lsl #3
+; ARMT2-NEXT:    add r0, r0, #3
+; ARMT2-NEXT:    clz r0, r0
+; ARMT2-NEXT:    lsr r0, r0, #5
+; ARMT2-NEXT:    add sp, sp, #8
+; ARMT2-NEXT:    pop {r11, pc}
+;
+; THUMB1-LABEL: t10:
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    .save {r4, r5, r7, lr}
+; THUMB1-NEXT:    push {r4, r5, r7, lr}
+; THUMB1-NEXT:    .pad #8
+; THUMB1-NEXT:    sub sp, #8
+; THUMB1-NEXT:    movs r0, #7
+; THUMB1-NEXT:    mvns r4, r0
+; THUMB1-NEXT:    str r4, [sp]
+; THUMB1-NEXT:    adds r5, r4, #5
+; THUMB1-NEXT:    str r5, [sp, #4]
+; THUMB1-NEXT:    mov r0, r5
+; THUMB1-NEXT:    mov r1, r4
+; THUMB1-NEXT:    bl __aeabi_idivmod
+; THUMB1-NEXT:    muls r0, r4, r0
+; THUMB1-NEXT:    adds r0, r0, r1
+; THUMB1-NEXT:    subs r1, r0, r5
+; THUMB1-NEXT:    rsbs r0, r1, #0
+; THUMB1-NEXT:    adcs r0, r1
+; THUMB1-NEXT:    add sp, #8
+; THUMB1-NEXT:    pop {r4, r5, r7, pc}
+;
+; THUMB2-LABEL: t10:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    .save {r7, lr}
+; THUMB2-NEXT:    push {r7, lr}
+; THUMB2-NEXT:    .pad #8
+; THUMB2-NEXT:    sub sp, #8
+; THUMB2-NEXT:    mvn r0, #2
+; THUMB2-NEXT:    str r0, [sp, #4]
+; THUMB2-NEXT:    mvn r0, #7
+; THUMB2-NEXT:    str r0, [sp]
+; THUMB2-NEXT:    mvn r0, #2
+; THUMB2-NEXT:    mvn r1, #7
+; THUMB2-NEXT:    bl __aeabi_idivmod
+; THUMB2-NEXT:    sub.w r0, r1, r0, lsl #3
+; THUMB2-NEXT:    adds r0, #3
+; THUMB2-NEXT:    clz r0, r0
+; THUMB2-NEXT:    lsrs r0, r0, #5
+; THUMB2-NEXT:    add sp, #8
+; THUMB2-NEXT:    pop {r7, pc}
+;
+; V8MBASE-LABEL: t10:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    .pad #8
+; V8MBASE-NEXT:    sub sp, #8
+; V8MBASE-NEXT:    movs r0, #7
+; V8MBASE-NEXT:    mvns r0, r0
+; V8MBASE-NEXT:    str r0, [sp]
+; V8MBASE-NEXT:    adds r0, r0, #5
+; V8MBASE-NEXT:    str r0, [sp, #4]
+; V8MBASE-NEXT:    movs r1, #0
+; V8MBASE-NEXT:    rsbs r0, r1, #0
+; V8MBASE-NEXT:    adcs r0, r1
+; V8MBASE-NEXT:    add sp, #8
+; V8MBASE-NEXT:    bx lr
 entry:
   %q = alloca i32
   %p = alloca i32
@@ -290,29 +677,121 @@ entry:
   %add = add nsw i32 %mul, %rem
   %cmp = icmp eq i32 %add, %0
   ret i1 %cmp
-
-; ARM-LABEL: t10:
-; ARM: rsbs r1, r0, #0
-; ARM: adc  r0, r0, r1
-
-; ARMT2-LABEL: t10:
-; ARMT2: clz r0, r0
-; ARMT2: lsr r0, r0, #5
-
-; THUMB1-LABEL: t10:
-; THUMB1: rsbs r0, r1, #0
-; THUMB1: adcs r0, r1
-
-; THUMB2-LABEL: t10:
-; THUMB2: clz r0, r0
-; THUMB2: lsrs r0, r0, #5
-
-; V8MBASE-LABEL: t10:
-; V8MBASE-NOT: movs r0, #0
-; V8MBASE: movs r0, #7
 }
 
 define i1 @t11() {
+; ARM-LABEL: t11:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    .pad #4
+; ARM-NEXT:    sub sp, sp, #4
+; ARM-NEXT:    ldr r0, [sp]
+; ARM-NEXT:    mov r1, #40960
+; ARM-NEXT:    orr r1, r1, #-33554432
+; ARM-NEXT:    orr r0, r0, #40960
+; ARM-NEXT:    and r0, r0, r1
+; ARM-NEXT:    orr r0, r0, #3
+; ARM-NEXT:    str r0, [sp]
+; ARM-NEXT:    mov r0, #1
+; ARM-NEXT:    add sp, sp, #4
+; ARM-NEXT:    mov pc, lr
+;
+; ARMT2-LABEL: t11:
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    .pad #4
+; ARMT2-NEXT:    sub sp, sp, #4
+; ARMT2-NEXT:    ldr r1, [sp]
+; ARMT2-NEXT:    mov r0, #33
+; ARMT2-NEXT:    movw r2, #52429
+; ARMT2-NEXT:    movt r2, #52428
+; ARMT2-NEXT:    bfi r1, r0, #0, #12
+; ARMT2-NEXT:    mov r0, #10
+; ARMT2-NEXT:    bfi r1, r0, #12, #13
+; ARMT2-NEXT:    mov r0, r1
+; ARMT2-NEXT:    bfc r0, #12, #20
+; ARMT2-NEXT:    umull r2, r3, r0, r2
+; ARMT2-NEXT:    lsr r2, r3, #3
+; ARMT2-NEXT:    add r2, r2, r2, lsl #2
+; ARMT2-NEXT:    sub r0, r0, r2, lsl #1
+; ARMT2-NEXT:    movw r2, #40960
+; ARMT2-NEXT:    movt r2, #65024
+; ARMT2-NEXT:    and r1, r1, r2
+; ARMT2-NEXT:    orr r0, r1, r0
+; ARMT2-NEXT:    str r0, [sp]
+; ARMT2-NEXT:    bfc r0, #12, #20
+; ARMT2-NEXT:    sub r0, r0, #3
+; ARMT2-NEXT:    clz r0, r0
+; ARMT2-NEXT:    lsr r0, r0, #5
+; ARMT2-NEXT:    add sp, sp, #4
+; ARMT2-NEXT:    bx lr
+;
+; THUMB1-LABEL: t11:
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    .pad #4
+; THUMB1-NEXT:    sub sp, #4
+; THUMB1-NEXT:    movs r0, #5
+; THUMB1-NEXT:    lsls r0, r0, #13
+; THUMB1-NEXT:    ldr r1, [sp]
+; THUMB1-NEXT:    orrs r1, r0
+; THUMB1-NEXT:    ldr r0, .LCPI10_0
+; THUMB1-NEXT:    ands r0, r1
+; THUMB1-NEXT:    adds r0, r0, #3
+; THUMB1-NEXT:    str r0, [sp]
+; THUMB1-NEXT:    movs r1, #0
+; THUMB1-NEXT:    rsbs r0, r1, #0
+; THUMB1-NEXT:    adcs r0, r1
+; THUMB1-NEXT:    add sp, #4
+; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:    .p2align 2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:  .LCPI10_0:
+; THUMB1-NEXT:    .long 4261453824 @ 0xfe00a000
+;
+; THUMB2-LABEL: t11:
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    .pad #4
+; THUMB2-NEXT:    sub sp, #4
+; THUMB2-NEXT:    ldr r1, [sp]
+; THUMB2-NEXT:    movs r0, #33
+; THUMB2-NEXT:    movw r2, #52429
+; THUMB2-NEXT:    bfi r1, r0, #0, #12
+; THUMB2-NEXT:    movs r0, #10
+; THUMB2-NEXT:    bfi r1, r0, #12, #13
+; THUMB2-NEXT:    mov r0, r1
+; THUMB2-NEXT:    movt r2, #52428
+; THUMB2-NEXT:    bfc r0, #12, #20
+; THUMB2-NEXT:    umull r2, r3, r0, r2
+; THUMB2-NEXT:    lsrs r2, r3, #3
+; THUMB2-NEXT:    add.w r2, r2, r2, lsl #2
+; THUMB2-NEXT:    sub.w r0, r0, r2, lsl #1
+; THUMB2-NEXT:    movw r2, #40960
+; THUMB2-NEXT:    movt r2, #65024
+; THUMB2-NEXT:    ands r1, r2
+; THUMB2-NEXT:    orrs r0, r1
+; THUMB2-NEXT:    str r0, [sp]
+; THUMB2-NEXT:    bfc r0, #12, #20
+; THUMB2-NEXT:    subs r0, #3
+; THUMB2-NEXT:    clz r0, r0
+; THUMB2-NEXT:    lsrs r0, r0, #5
+; THUMB2-NEXT:    add sp, #4
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t11:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    .pad #4
+; V8MBASE-NEXT:    sub sp, #4
+; V8MBASE-NEXT:    movw r0, #40960
+; V8MBASE-NEXT:    ldr r1, [sp]
+; V8MBASE-NEXT:    orrs r1, r0
+; V8MBASE-NEXT:    movw r0, #40960
+; V8MBASE-NEXT:    movt r0, #65024
+; V8MBASE-NEXT:    ands r0, r1
+; V8MBASE-NEXT:    adds r0, r0, #3
+; V8MBASE-NEXT:    str r0, [sp]
+; V8MBASE-NEXT:    movs r1, #0
+; V8MBASE-NEXT:    rsbs r0, r1, #0
+; V8MBASE-NEXT:    adcs r0, r1
+; V8MBASE-NEXT:    add sp, #4
+; V8MBASE-NEXT:    bx lr
 entry:
   %bit = alloca i32
   %load = load i32, i32* %bit
@@ -330,67 +809,87 @@ entry:
   %clear12 = and i32 %set10, 4095
   %cmp = icmp eq i32 %clear12, 3
   ret i1 %cmp
-
-; ARM-LABEL: t11:
-; ARM: rsbs r1, r0, #0
-; ARM: adc  r0, r0, r1
-
-; ARMT2-LABEL: t11:
-; ARMT2: clz r0, r0
-; ARMT2: lsr r0, r0, #5
-
-; THUMB1-LABEL: t11:
-; THUMB1-NOT: movs r0, #0
-; THUMB1: movs r0, #5
-
-; THUMB2-LABEL: t11:
-; THUMB2: clz r0, r0
-; THUMB2: lsrs r0, r0, #5
-
-; V8MBASE-LABEL: t11:
-; V8MBASE-NOT: movs r0, #0
-; V8MBASE: movw	r0, #40960
 }
 
 define i32 @t12(i32 %a) nounwind {
-entry:
 ; ARM-LABEL: t12:
-; ARM-NOT: mov
-; ARM: cmp r0, #0
-; ARM: movne r0, #1
-
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    cmp r0, #0
+; ARM-NEXT:    movne r0, #1
+; ARM-NEXT:    lsl r0, r0, #1
+; ARM-NEXT:    mov pc, lr
+;
+; ARMT2-LABEL: t12:
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    cmp r0, #0
+; ARMT2-NEXT:    movwne r0, #1
+; ARMT2-NEXT:    lsl r0, r0, #1
+; ARMT2-NEXT:    bx lr
+;
 ; THUMB1-LABEL: t12:
-; THUMB1: subs r1, r0, #1
-; THUMB1: sbcs r0, r1
-; THUMB1: lsls r0, r0, #1
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    subs r1, r0, #1
+; THUMB1-NEXT:    sbcs r0, r1
+; THUMB1-NEXT:    lsls r0, r0, #1
+; THUMB1-NEXT:    bx lr
+;
 ; THUMB2-LABEL: t12:
-; THUMB2-NOT: mov
-; THUMB2: cmp r0, #0
-; THUMB2: it ne
-; THUMB2: movne r0, #1
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    cmp r0, #0
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r0, #1
+; THUMB2-NEXT:    lsls r0, r0, #1
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t12:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    subs r1, r0, #1
+; V8MBASE-NEXT:    sbcs r0, r1
+; V8MBASE-NEXT:    lsls r0, r0, #1
+; V8MBASE-NEXT:    bx lr
+entry:
   %tobool = icmp ne i32 %a, 0
   %lnot.ext = select i1 %tobool, i32 2, i32 0
   ret i32 %lnot.ext
 }
 
 define i32 @t13(i32 %a) nounwind {
-entry:
 ; ARM-LABEL: t13:
-; ARM-NOT: mov
-; ARM: cmp r0, #0
-; ARM: movne r0, #3
-
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    cmp r0, #0
+; ARM-NEXT:    movne r0, #3
+; ARM-NEXT:    mov pc, lr
+;
+; ARMT2-LABEL: t13:
+; ARMT2:       @ %bb.0: @ %entry
+; ARMT2-NEXT:    cmp r0, #0
+; ARMT2-NEXT:    movwne r0, #3
+; ARMT2-NEXT:    bx lr
+;
 ; THUMB1-LABEL: t13:
-; THUMB1: cmp r0, #0
-; THUMB1: beq
-; THUMB1: movs r0, #3
-
+; THUMB1:       @ %bb.0: @ %entry
+; THUMB1-NEXT:    cmp r0, #0
+; THUMB1-NEXT:    beq .LBB12_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    movs r0, #3
+; THUMB1-NEXT:  .LBB12_2: @ %entry
+; THUMB1-NEXT:    bx lr
+;
 ; THUMB2-LABEL: t13:
-; THUMB2-NOT: mov
-; THUMB2: cmp r0, #0
-; THUMB2: it ne
-; THUMB2: movne r0, #3
+; THUMB2:       @ %bb.0: @ %entry
+; THUMB2-NEXT:    cmp r0, #0
+; THUMB2-NEXT:    it ne
+; THUMB2-NEXT:    movne r0, #3
+; THUMB2-NEXT:    bx lr
+;
+; V8MBASE-LABEL: t13:
+; V8MBASE:       @ %bb.0: @ %entry
+; V8MBASE-NEXT:    cbz r0, .LBB12_2
+; V8MBASE-NEXT:  @ %bb.1:
+; V8MBASE-NEXT:    movs r0, #3
+; V8MBASE-NEXT:  .LBB12_2: @ %entry
+; V8MBASE-NEXT:    bx lr
+entry:
   %tobool = icmp ne i32 %a, 0
   %lnot.ext = select i1 %tobool, i32 3, i32 0
   ret i32 %lnot.ext
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak-section.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak-section.ll
index 23332c9d9aa1d1a4aea24f115740fa0bb55bb893..d47a9d6c504a4906049a6d3c99cb6eab322e3032 100644
--- a/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak-section.ll
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-func-weak-section.ll
@@ -23,9 +23,9 @@ declare !dbg !4 extern_weak dso_local i32 @global_func(i8 signext) local_unnamed
 ; CHECK-NEXT:        .byte   0
 ; CHECK-NEXT:        .long   24
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   88
-; CHECK-NEXT:        .long   88
-; CHECK-NEXT:        .long   72
+; CHECK-NEXT:        .long   112
+; CHECK-NEXT:        .long   112
+; CHECK-NEXT:        .long   76
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
 ; CHECK-NEXT:        .long   218103808               # 0xd000000
 ; CHECK-NEXT:        .long   2
@@ -48,6 +48,12 @@ declare !dbg !4 extern_weak dso_local i32 @global_func(i8 signext) local_unnamed
 ; CHECK-NEXT:        .long   60                      # BTF_KIND_FUNC(id = 6)
 ; CHECK-NEXT:        .long   201326594               # 0xc000002
 ; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   72                      # BTF_KIND_DATASEC(id = 7)
+; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   global_func
+; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .byte   0                       # string offset=0
 ; CHECK-NEXT:        .ascii  "int"                   # string offset=1
 ; CHECK-NEXT:        .byte   0
@@ -61,6 +67,7 @@ declare !dbg !4 extern_weak dso_local i32 @global_func(i8 signext) local_unnamed
 ; CHECK-NEXT:        .byte   0
 ; CHECK-NEXT:        .ascii  "global_func"           # string offset=60
 ; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "abc"                   # string offset=72
 
 attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-section.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-section.ll
index e01da7e209fd483b4be97b50288a95f16fab757a..520847449950eabfbe10323f6de539450354a620 100644
--- a/llvm/test/CodeGen/BPF/BTF/extern-var-section.ll
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-section.ll
@@ -28,8 +28,8 @@ entry:
 ; CHECK-NEXT:        .byte   0
 ; CHECK-NEXT:        .long   24
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   128
-; CHECK-NEXT:        .long   128
+; CHECK-NEXT:        .long   140
+; CHECK-NEXT:        .long   140
 ; CHECK-NEXT:        .long   79
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
 ; CHECK-NEXT:        .long   218103808               # 0xd000000
@@ -58,7 +58,10 @@ entry:
 ; CHECK-NEXT:        .long   5
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   75                      # BTF_KIND_DATASEC(id = 8)
-; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   251658242               # 0xf000002
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   global_func
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   7
 ; CHECK-NEXT:        .long   ch
diff --git a/llvm/test/CodeGen/BPF/BTF/extern-var-weak-section.ll b/llvm/test/CodeGen/BPF/BTF/extern-var-weak-section.ll
index 6e64d9b4e4828eeba87a86d4c82adf404005dafc..bdf7fb49c560931de3d6e3505e9a4d4090183622 100644
--- a/llvm/test/CodeGen/BPF/BTF/extern-var-weak-section.ll
+++ b/llvm/test/CodeGen/BPF/BTF/extern-var-weak-section.ll
@@ -28,8 +28,8 @@ declare !dbg !6 extern_weak dso_local i32 @global_func(i8 signext) local_unnamed
 ; CHECK-NEXT:        .byte   0
 ; CHECK-NEXT:        .long   24
 ; CHECK-NEXT:        .long   0
-; CHECK-NEXT:        .long   128
-; CHECK-NEXT:        .long   128
+; CHECK-NEXT:        .long   140
+; CHECK-NEXT:        .long   140
 ; CHECK-NEXT:        .long   79
 ; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
 ; CHECK-NEXT:        .long   218103808               # 0xd000000
@@ -58,7 +58,10 @@ declare !dbg !6 extern_weak dso_local i32 @global_func(i8 signext) local_unnamed
 ; CHECK-NEXT:        .long   5
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   75                      # BTF_KIND_DATASEC(id = 8)
-; CHECK-NEXT:        .long   251658241               # 0xf000001
+; CHECK-NEXT:        .long   251658242               # 0xf000002
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   global_func
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   7
 ; CHECK-NEXT:        .long   ch
diff --git a/llvm/test/CodeGen/Generic/2008-01-25-dag-combine-mul.ll b/llvm/test/CodeGen/Generic/2008-01-25-dag-combine-mul.ll
index 4558f09c4b96fe0a2f4888b6dbd40570897cba9d..8c8d8a103c576e2df435909fd4144ad0de48f0e0 100644
--- a/llvm/test/CodeGen/Generic/2008-01-25-dag-combine-mul.ll
+++ b/llvm/test/CodeGen/Generic/2008-01-25-dag-combine-mul.ll
@@ -1,6 +1,9 @@
 ; RUN: llc < %s
 ; rdar://5707064
 
+; XCore default subtarget does not support 8-byte alignment on stack.
+; XFAIL: xcore
+
 define i32 @f(i16* %pc) {
 entry:
 	%acc = alloca i64, align 8		; <i64*> [#uses=4]
diff --git a/llvm/test/CodeGen/Generic/2014-02-05-OpaqueConstants.ll b/llvm/test/CodeGen/Generic/2014-02-05-OpaqueConstants.ll
index 1497bbb0c9703bd4f1bb0affd0351a01a632f911..3ca31efd88819d5cd7caa439651b60aac09b4715 100644
--- a/llvm/test/CodeGen/Generic/2014-02-05-OpaqueConstants.ll
+++ b/llvm/test/CodeGen/Generic/2014-02-05-OpaqueConstants.ll
@@ -1,6 +1,6 @@
 ; Test that opaque constants are not creating an infinite DAGCombine loop
 ; RUN: llc < %s
-; XFAIL: r600, xcore
+; XFAIL: r600
 
 @a = common global i32* null, align 8
 @c = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/Generic/inline-asm-mem-clobber.ll b/llvm/test/CodeGen/Generic/inline-asm-mem-clobber.ll
index be1e0a39b3b0dbcf265e4b60139418447c20befe..a4f21a0955fec60bfe3ce4ee95e2a7c958c850ba 100644
--- a/llvm/test/CodeGen/Generic/inline-asm-mem-clobber.ll
+++ b/llvm/test/CodeGen/Generic/inline-asm-mem-clobber.ll
@@ -1,5 +1,8 @@
 ; RUN: llc -O2 -no-integrated-as < %s | FileCheck %s
 
+; XCore default subtarget does not support 8-byte alignment on stack.
+; XFAIL: xcore
+
 @G = common global i32 0, align 4
 
 define i32 @foo(i8* %p) nounwind uwtable {
diff --git a/llvm/test/CodeGen/Hexagon/hvx-reuse-fi-base.ll b/llvm/test/CodeGen/Hexagon/hvx-reuse-fi-base.ll
new file mode 100644
index 0000000000000000000000000000000000000000..73da83b921e204bd5b2d1242f47b2ff99ee7e817
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/hvx-reuse-fi-base.ll
@@ -0,0 +1,212 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=hexagon < %s | FileCheck %s
+target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048"
+target triple = "hexagon"
+
+@g0 = external dso_local global <64 x i32>, align 128
+@g1 = external hidden unnamed_addr constant [110 x i8], align 1
+@g2 = external hidden unnamed_addr constant [102 x i8], align 1
+@g3 = external hidden unnamed_addr constant [110 x i8], align 1
+
+declare dso_local void @f0() #0
+
+declare dso_local void @f1(i8*, ...) #0
+
+; Function Attrs: nounwind readnone
+declare <32 x i32> @llvm.hexagon.V6.vandqrt.128B(<128 x i1>, i32) #1
+
+; Function Attrs: nounwind readnone
+declare <128 x i1> @llvm.hexagon.V6.vandvrt.128B(<32 x i32>, i32) #1
+
+; Function Attrs: nounwind readnone
+declare <32 x i32> @llvm.hexagon.V6.lvsplatw.128B(i32) #1
+
+; Function Attrs: nounwind readnone
+declare <64 x i32> @llvm.hexagon.V6.vrmpyubi.128B(<64 x i32>, i32, i32 immarg) #1
+
+; Function Attrs: nounwind readnone
+declare <64 x i32> @llvm.hexagon.V6.vaddubh.128B(<32 x i32>, <32 x i32>) #1
+
+define dso_local void @f2() #0 {
+; CHECK-LABEL: f2:
+; CHECK:       // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1:0 = combine(#2,##16843009)
+; CHECK-NEXT:     allocframe(r29,#1536):raw
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = vsplat(r1)
+; CHECK-NEXT:     r17:16 = combine(#-1,#1)
+; CHECK-NEXT:     r29 = and(r29,#-256)
+; CHECK-NEXT:     memd(r30+#-8) = r17:16
+; CHECK-NEXT:    } // 8-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vsplat(r16)
+; CHECK-NEXT:     r2 = add(r29,#2048)
+; CHECK-NEXT:     memd(r30+#-16) = r19:18
+; CHECK-NEXT:    } // 8-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     q0 = vand(v0,r0)
+; CHECK-NEXT:     r18 = ##-2147483648
+; CHECK-NEXT:     vmem(r2+#-7) = v0
+; CHECK-NEXT:    } // 128-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vand(q0,r17)
+; CHECK-NEXT:     r0 = ##g1
+; CHECK-NEXT:     memd(r30+#-24) = r21:20
+; CHECK-NEXT:    } // 8-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r19 = ##g0+128
+; CHECK-NEXT:     vmem(r2+#-6) = v0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v3:2.h = vadd(v0.ub,v1.ub)
+; CHECK-NEXT:     r20 = ##g0
+; CHECK-NEXT:     vmem(r29+#5) = v1
+; CHECK-NEXT:    } // 128-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     vmem(r29+#6) = v2
+; CHECK-NEXT:    } // 256-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v31:30.uw = vrmpy(v3:2.ub,r18.ub,#0)
+; CHECK-NEXT:     vmem(r29+#7) = v3
+; CHECK-NEXT:    } // 256-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     vmem(r19+#0) = v31
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     call f1
+; CHECK-NEXT:     vmem(r20+#0) = v30
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = add(r29,#2048)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vmem(r0+#-7)
+; CHECK-NEXT:    } // 128-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1:0.h = vadd(v0.ub,v0.ub)
+; CHECK-NEXT:     r0 = ##g2
+; CHECK-NEXT:     vmem(r29+#2) = v0.new
+; CHECK-NEXT:    } // 256-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     vmem(r29+#3) = v1
+; CHECK-NEXT:    } // 256-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1:0.uw = vrmpy(v1:0.ub,r17.ub,#0)
+; CHECK-NEXT:     vmem(r19+#0) = v1.new
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     call f1
+; CHECK-NEXT:     vmem(r20+#0) = v0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = ##2147483647
+; CHECK-NEXT:     v0 = vmem(r29+#2)
+; CHECK-NEXT:    } // 256-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = vmem(r29+#3)
+; CHECK-NEXT:    } // 256-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1:0.uw = vrmpy(v1:0.ub,r0.ub,#1)
+; CHECK-NEXT:     r0 = ##g3
+; CHECK-NEXT:     vmem(r19+#0) = v1.new
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     call f1
+; CHECK-NEXT:     vmem(r20+#0) = v0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vmem(r29+#6)
+; CHECK-NEXT:    } // 256-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = vmem(r29+#7)
+; CHECK-NEXT:    } // 256-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1:0.uw = vrmpy(v1:0.ub,r18.ub,#1)
+; CHECK-NEXT:     vmem(r19+#0) = v1.new
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     call f0
+; CHECK-NEXT:     vmem(r20+#0) = v0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = #0
+; CHECK-NEXT:     v0 = vmem(r29+#6)
+; CHECK-NEXT:    } // 256-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = vmem(r29+#7)
+; CHECK-NEXT:    } // 256-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1:0.uw = vrmpy(v1:0.ub,r0.ub,#1)
+; CHECK-NEXT:     vmem(r19+#0) = v1.new
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     call f0
+; CHECK-NEXT:     vmem(r20+#0) = v0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = add(r29,#2048)
+; CHECK-NEXT:     v1 = vmem(r29+#5)
+; CHECK-NEXT:    } // 128-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vmem(r0+#-7)
+; CHECK-NEXT:    } // 128-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1:0.h = vadd(v0.ub,v1.ub)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1:0.uw = vrmpy(v1:0.ub,r16.ub,#1)
+; CHECK-NEXT:     r17:16 = memd(r30+#-8)
+; CHECK-NEXT:     vmem(r19+#0) = v1.new
+; CHECK-NEXT:    } // 8-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r19:18 = memd(r30+#-16)
+; CHECK-NEXT:     vmem(r20+#0) = v0
+; CHECK-NEXT:    } // 8-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = memd(r30+#-24)
+; CHECK-NEXT:     r31:30 = dealloc_return(r30):raw
+; CHECK-NEXT:    } // 8-byte Folded Reload
+b0:
+  %v0 = alloca <32 x i32>, align 128
+  %v1 = call <32 x i32> @llvm.hexagon.V6.lvsplatw.128B(i32 1)
+  %v2 = call <128 x i1> @llvm.hexagon.V6.vandvrt.128B(<32 x i32> %v1, i32 16843009)
+  %v3 = call <32 x i32> @llvm.hexagon.V6.vandqrt.128B(<128 x i1> %v2, i32 -1)
+  store <32 x i32> %v3, <32 x i32>* %v0, align 128
+  %v4 = call <32 x i32> @llvm.hexagon.V6.lvsplatw.128B(i32 2)
+  %v5 = call <64 x i32> @llvm.hexagon.V6.vaddubh.128B(<32 x i32> undef, <32 x i32> %v4)
+  %v6 = call <64 x i32> @llvm.hexagon.V6.vrmpyubi.128B(<64 x i32> %v5, i32 -2147483648, i32 0)
+  store <64 x i32> %v6, <64 x i32>* @g0, align 128
+  call void (i8*, ...) @f1(i8* getelementptr inbounds ([110 x i8], [110 x i8]* @g1, i32 0, i32 0)) #2
+  %v7 = call <32 x i32> @llvm.hexagon.V6.lvsplatw.128B(i32 1)
+  %v8 = call <64 x i32> @llvm.hexagon.V6.vaddubh.128B(<32 x i32> %v7, <32 x i32> undef)
+  %v9 = call <64 x i32> @llvm.hexagon.V6.vrmpyubi.128B(<64 x i32> %v8, i32 -1, i32 0)
+  store <64 x i32> %v9, <64 x i32>* @g0, align 128
+  call void (i8*, ...) @f1(i8* getelementptr inbounds ([102 x i8], [102 x i8]* @g2, i32 0, i32 0)) #2
+  %v10 = call <32 x i32> @llvm.hexagon.V6.lvsplatw.128B(i32 1)
+  %v11 = call <64 x i32> @llvm.hexagon.V6.vaddubh.128B(<32 x i32> %v10, <32 x i32> undef)
+  %v12 = call <64 x i32> @llvm.hexagon.V6.vrmpyubi.128B(<64 x i32> %v11, i32 2147483647, i32 1)
+  store <64 x i32> %v12, <64 x i32>* @g0, align 128
+  call void (i8*, ...) @f1(i8* getelementptr inbounds ([110 x i8], [110 x i8]* @g3, i32 0, i32 0)) #2
+  %v13 = call <32 x i32> @llvm.hexagon.V6.lvsplatw.128B(i32 2)
+  %v14 = call <64 x i32> @llvm.hexagon.V6.vaddubh.128B(<32 x i32> undef, <32 x i32> %v13)
+  %v15 = call <64 x i32> @llvm.hexagon.V6.vrmpyubi.128B(<64 x i32> %v14, i32 -2147483648, i32 1)
+  store <64 x i32> %v15, <64 x i32>* @g0, align 128
+  call void @f0() #2
+  %v16 = call <32 x i32> @llvm.hexagon.V6.lvsplatw.128B(i32 2)
+  %v17 = call <64 x i32> @llvm.hexagon.V6.vaddubh.128B(<32 x i32> undef, <32 x i32> %v16)
+  %v18 = call <64 x i32> @llvm.hexagon.V6.vrmpyubi.128B(<64 x i32> %v17, i32 0, i32 1)
+  store <64 x i32> %v18, <64 x i32>* @g0, align 128
+  call void @f0() #2
+  %v19 = call <32 x i32> @llvm.hexagon.V6.lvsplatw.128B(i32 1)
+  %v20 = call <32 x i32> @llvm.hexagon.V6.lvsplatw.128B(i32 2)
+  %v21 = call <64 x i32> @llvm.hexagon.V6.vaddubh.128B(<32 x i32> %v19, <32 x i32> %v20)
+  %v22 = call <64 x i32> @llvm.hexagon.V6.vrmpyubi.128B(<64 x i32> %v21, i32 1, i32 1)
+  store <64 x i32> %v22, <64 x i32>* @g0, align 128
+  ret void
+}
+
+attributes #0 = { nounwind "use-soft-float"="false" "target-cpu"="hexagonv66" "target-features"="+hvxv66,+hvx-length128b" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind optsize }
diff --git a/llvm/test/CodeGen/Hexagon/namedreg.ll b/llvm/test/CodeGen/Hexagon/namedreg.ll
index 72ca50868828f712b8072508710db971836e03cf..a905332b2dee5d3efe3ee9899dd492fb5a2a2fad 100644
--- a/llvm/test/CodeGen/Hexagon/namedreg.ll
+++ b/llvm/test/CodeGen/Hexagon/namedreg.ll
@@ -4,10 +4,29 @@ entry:
   %0 = call i32 @llvm.read_register.i32(metadata !0)
   ret i32 %0
 }
-
 declare i32 @llvm.read_register.i32(metadata) #1
 
+define dso_local i32 @rcs0() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !1)
+  ret i32 %0
+}
+
+define dso_local i32 @rcs1() #0 {
+entry:
+  %0 = call i32 @llvm.read_register.i32(metadata !2)
+  ret i32 %0
+}
+
+
+
 !llvm.named.register.r19 = !{!0}
+!llvm.named.register.cs0 = !{!1}
+!llvm.named.register.cs1 = !{!2}
 
 !0 = !{!"r19"}
+!1 = !{!"cs0"}
+!2 = !{!"cs1"}
 ; CHECK: r0 = r19
+; CHECK: r0 = cs0
+; CHECK: r0 = cs1
diff --git a/llvm/test/CodeGen/Hexagon/vec-align.ll b/llvm/test/CodeGen/Hexagon/vec-align.ll
index 6d03e42e5424c37dabb732410b4dcfff5cab6bb1..8178f53f05672f0379442fe88b56cd77f034d572 100644
--- a/llvm/test/CodeGen/Hexagon/vec-align.ll
+++ b/llvm/test/CodeGen/Hexagon/vec-align.ll
@@ -2,11 +2,10 @@
 
 ; Make sure we generate stack alignment.
 ; CHECK: [[REG1:r[0-9]*]] = and(r29,#-64)
-; CHECK: = add([[REG1]],#128)
-; CHECK: = add([[REG1]],#64)
-; Make sure we do not generate another -64 off SP.
-; CHECK: vmem(
-; CHECK-NOT: r{{[0-9]*}} = add(r29,#-64)
+; CHECK: vmem([[REG1]]+#2) =
+; CHECK: vmem([[REG1]]+#1) =
+; CHECK: = vmem([[REG1]]+#2)
+; CHECK: = vmem([[REG1]]+#1)
 
 target triple = "hexagon"
 
@@ -42,5 +41,5 @@ declare <16 x i32> @llvm.hexagon.V6.vaddw(<16 x i32>, <16 x i32>) #1
 
 declare void @f2(...) #0
 
-attributes #0 = { nounwind "target-cpu"="hexagonv60" "target-features"="+hvxv60,+hvx-length64b" }
+attributes #0 = { nounwind "target-cpu"="hexagonv65" "target-features"="+hvxv65,+hvx-length64b" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/M68k/Arith/divide-by-constant.ll b/llvm/test/CodeGen/M68k/Arith/divide-by-constant.ll
index 04114d97e457aac39c7be13c4ed655b0d788bb09..b695202f8ec2ec37e0ee3cd19ee69d4a3e517cf1 100644
--- a/llvm/test/CodeGen/M68k/Arith/divide-by-constant.ll
+++ b/llvm/test/CodeGen/M68k/Arith/divide-by-constant.ll
@@ -44,7 +44,6 @@ define zeroext i8 @test3(i8 zeroext %x, i8 zeroext %c) {
 ; CHECK-NEXT:    lsr.l #8, %d0
 ; CHECK-NEXT:    lsr.w #1, %d0
 ; CHECK-NEXT:    and.l #65535, %d0
-; CHECK-NEXT:    and.l #255, %d0
 ; CHECK-NEXT:    rts
 entry:
   %div = udiv i8 %c, 3
diff --git a/llvm/test/CodeGen/M68k/lit.local.cfg b/llvm/test/CodeGen/M68k/lit.local.cfg
index 8e4684705573a649bf787e370b3d42d11ceae07f..f42093af07708857748f27fe927e763aebb98f76 100644
--- a/llvm/test/CodeGen/M68k/lit.local.cfg
+++ b/llvm/test/CodeGen/M68k/lit.local.cfg
@@ -6,7 +6,8 @@ extract_section_path = os.path.join(config.llvm_src_root,
         'utils', 'extract-section.py')
 
 config.substitutions.append(('extract-section',
-    extract_section_path + ' --byte-indicator --hex-width=2'))
+    "'%s' %s %s" % (config.python_executable,
+                    extract_section_path, '--byte-indicator --hex-width=2')))
 
 if not 'M68k' in config.root.targets:
     config.unsupported = True
diff --git a/llvm/test/CodeGen/MIR/Generic/frame-info.mir b/llvm/test/CodeGen/MIR/Generic/frame-info.mir
index e25bdcfb0d0ac20fb44d0a62bbfd3e403646c608..8fd9362e6db4ab320464d03687fa8900aaf4830f 100644
--- a/llvm/test/CodeGen/MIR/Generic/frame-info.mir
+++ b/llvm/test/CodeGen/MIR/Generic/frame-info.mir
@@ -42,6 +42,7 @@ tracksRegLiveness: true
 # CHECK-NEXT: hasOpaqueSPAdjustment: false
 # CHECK-NEXT: hasVAStart: false
 # CHECK-NEXT: hasMustTailInVarArgFunc: false
+# CHECK-NEXT: hasTailCall: false
 # CHECK-NEXT: localFrameSize: 0
 # CHECK-NEXT: savePoint:       ''
 # CHECK-NEXT: restorePoint:    ''
@@ -73,6 +74,7 @@ tracksRegLiveness: true
 # CHECK-NEXT: hasOpaqueSPAdjustment: true
 # CHECK-NEXT: hasVAStart: true
 # CHECK-NEXT: hasMustTailInVarArgFunc: true
+# CHECK-NEXT: hasTailCall: true
 # CHECK: body
 frameInfo:
   isFrameAddressTaken: true
@@ -88,6 +90,7 @@ frameInfo:
   cvBytesOfCalleeSavedRegisters: 8
   hasOpaqueSPAdjustment: true
   hasVAStart:      true
+  hasTailCall:     true
   hasMustTailInVarArgFunc: true
   localFrameSize: 256
 body: |
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll
index 9ae0072c9499a8282c43e0d9a5a6e48be8a4bf21..091a92e72b1910dceda6cc5e881a95c40dafa66c 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-data.ll
@@ -162,7 +162,7 @@
 
 ; OBJ:      Sections [
 ; OBJ:        Section {
-; OBJ-NEXT:     Index: 1
+; OBJ-NEXT:     Index: [[#OBJ_INDX:]]
 ; OBJ-NEXT:     Name: .text
 ; OBJ-NEXT:     PhysicalAddress: 0x0
 ; OBJ-NEXT:     VirtualAddress: 0x0
@@ -176,7 +176,7 @@
 ; OBJ-NEXT:   }
 
 ; OBJ:        Section {
-; OBJ-NEXT:     Index: 2
+; OBJ-NEXT:     Index: [[#OBJ_INDX+1]]
 ; OBJ-NEXT:     Name: .data
 ; OBJ-NEXT:     PhysicalAddress: 0x0
 ; OBJ-NEXT:     VirtualAddress: 0x0
@@ -190,7 +190,7 @@
 ; OBJ-NEXT:   }
 
 ; OBJ:        Section {
-; OBJ-NEXT:     Index: 3
+; OBJ-NEXT:     Index: [[#OBJ_INDX+2]]
 ; OBJ-NEXT:     Name: .bss
 ; OBJ-NEXT:     PhysicalAddress: 0x80
 ; OBJ-NEXT:     VirtualAddress: 0x80
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
index 8313dcba13f95bf056eb593aa964001928c03453..72752576c90ff61c4bb15edf5836f49b571eeccd 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
@@ -158,7 +158,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     NumberOfAuxEntries: 0
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 1
+; SYM-NEXT:     Index: [[#INDX:]]
 ; SYM-NEXT:     Name: .bar
 ; SYM-NEXT:     Value (RelocatableAddress): 0x0
 ; SYM-NEXT:     Section: N_UNDEF
@@ -166,7 +166,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_EXT (0x2)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 2
+; SYM-NEXT:       Index: [[#INDX+1]]
 ; SYM-NEXT:       SectionLen: 0
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -178,7 +178,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 3
+; SYM-NEXT:     Index: [[#INDX+2]]
 ; SYM-NEXT:     Name: bar
 ; SYM-NEXT:     Value (RelocatableAddress): 0x0
 ; SYM-NEXT:     Section: N_UNDEF
@@ -186,7 +186,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_EXT (0x2)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 4
+; SYM-NEXT:       Index: [[#INDX+3]]
 ; SYM-NEXT:       SectionLen: 0
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -198,7 +198,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 5
+; SYM-NEXT:     Index: [[#INDX+4]]
 ; SYM-NEXT:     Name: .text
 ; SYM-NEXT:     Value (RelocatableAddress): 0x0
 ; SYM-NEXT:     Section: .text
@@ -206,7 +206,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 6
+; SYM-NEXT:       Index: [[#INDX+5]]
 ; SYM-NEXT:       SectionLen: 64
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -218,7 +218,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 7
+; SYM-NEXT:     Index: [[#INDX+6]]
 ; SYM-NEXT:     Name: .foo
 ; SYM-NEXT:     Value (RelocatableAddress): 0x0
 ; SYM-NEXT:     Section: .text
@@ -226,8 +226,8 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_EXT (0x2)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 8
-; SYM-NEXT:       ContainingCsectSymbolIndex: 5
+; SYM-NEXT:       Index: [[#INDX+7]]
+; SYM-NEXT:       ContainingCsectSymbolIndex: [[#INDX+4]]
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
 ; SYM-NEXT:       SymbolAlignmentLog2: 0
@@ -238,7 +238,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 9
+; SYM-NEXT:     Index: [[#INDX+8]]
 ; SYM-NEXT:     Name: .data
 ; SYM-NEXT:     Value (RelocatableAddress): 0x40
 ; SYM-NEXT:     Section: .data
@@ -246,7 +246,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 10
+; SYM-NEXT:       Index: [[#INDX+9]]
 ; SYM-NEXT:       SectionLen: 52
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -258,7 +258,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 11
+; SYM-NEXT:     Index: [[#INDX+10]]
 ; SYM-NEXT:     Name: globalA
 ; SYM-NEXT:     Value (RelocatableAddress): 0x40
 ; SYM-NEXT:     Section: .data
@@ -266,8 +266,8 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_EXT (0x2)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 12
-; SYM-NEXT:       ContainingCsectSymbolIndex: 9
+; SYM-NEXT:       Index: [[#INDX+11]]
+; SYM-NEXT:       ContainingCsectSymbolIndex: [[#INDX+8]]
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
 ; SYM-NEXT:       SymbolAlignmentLog2: 0
@@ -278,7 +278,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 13
+; SYM-NEXT:     Index: [[#INDX+12]]
 ; SYM-NEXT:     Name: globalB
 ; SYM-NEXT:     Value (RelocatableAddress): 0x44
 ; SYM-NEXT:     Section: .data
@@ -286,8 +286,8 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_EXT (0x2)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 14
-; SYM-NEXT:       ContainingCsectSymbolIndex: 9
+; SYM-NEXT:       Index: [[#INDX+13]]
+; SYM-NEXT:       ContainingCsectSymbolIndex: [[#INDX+8]]
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
 ; SYM-NEXT:       SymbolAlignmentLog2: 0
@@ -298,7 +298,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 15
+; SYM-NEXT:     Index: [[#INDX+14]]
 ; SYM-NEXT:     Name: arr
 ; SYM-NEXT:     Value (RelocatableAddress): 0x48
 ; SYM-NEXT:     Section: .data
@@ -306,8 +306,8 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_EXT (0x2)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 16
-; SYM-NEXT:       ContainingCsectSymbolIndex: 9
+; SYM-NEXT:       Index: [[#INDX+15]]
+; SYM-NEXT:       ContainingCsectSymbolIndex: [[#INDX+8]]
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
 ; SYM-NEXT:       SymbolAlignmentLog2: 0
@@ -318,7 +318,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 17
+; SYM-NEXT:     Index: [[#INDX+16]]
 ; SYM-NEXT:     Name: p
 ; SYM-NEXT:     Value (RelocatableAddress): 0x70
 ; SYM-NEXT:     Section: .data
@@ -326,8 +326,8 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_EXT (0x2)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 18
-; SYM-NEXT:       ContainingCsectSymbolIndex: 9
+; SYM-NEXT:       Index: [[#INDX+17]]
+; SYM-NEXT:       ContainingCsectSymbolIndex: [[#INDX+8]]
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
 ; SYM-NEXT:       SymbolAlignmentLog2: 0
@@ -338,7 +338,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 19
+; SYM-NEXT:     Index: [[#INDX+18]]
 ; SYM-NEXT:     Name: foo
 ; SYM-NEXT:     Value (RelocatableAddress): 0x74
 ; SYM-NEXT:     Section: .data
@@ -346,7 +346,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_EXT (0x2)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 20
+; SYM-NEXT:       Index: [[#INDX+19]]
 ; SYM-NEXT:       SectionLen: 12
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -358,7 +358,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 21
+; SYM-NEXT:     Index: [[#INDX+20]]
 ; SYM-NEXT:     Name: TOC
 ; SYM-NEXT:     Value (RelocatableAddress): 0x80
 ; SYM-NEXT:     Section: .data
@@ -366,7 +366,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 22
+; SYM-NEXT:       Index: [[#INDX+21]]
 ; SYM-NEXT:       SectionLen: 0
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -378,7 +378,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 23
+; SYM-NEXT:     Index: [[#INDX+22]]
 ; SYM-NEXT:     Name: globalA
 ; SYM-NEXT:     Value (RelocatableAddress): 0x80
 ; SYM-NEXT:     Section: .data
@@ -386,7 +386,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 24
+; SYM-NEXT:       Index: [[#INDX+23]]
 ; SYM-NEXT:       SectionLen: 4
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -398,7 +398,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 25
+; SYM-NEXT:     Index: [[#INDX+24]]
 ; SYM-NEXT:     Name: globalB
 ; SYM-NEXT:     Value (RelocatableAddress): 0x84
 ; SYM-NEXT:     Section: .data
@@ -406,7 +406,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 26
+; SYM-NEXT:       Index: [[#INDX+25]]
 ; SYM-NEXT:       SectionLen: 4
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll
index 82244723b868986aa49cad1b9b3832c7c372b767..e4cce73c600e7b86f692e780ec076062d8d7da0f 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-rodata.ll
@@ -138,7 +138,7 @@
 ; SYMS-NEXT:     NumberOfAuxEntries: 1
 ; SYMS-NEXT:     CSECT Auxiliary Entry {
 ; SYMS-NEXT:       Index: [[#INDX+3]]
-; SYMS-NEXT:       ContainingCsectSymbolIndex: 3
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX]]
 ; SYMS-NEXT:       ParameterHashIndex: 0x0
 ; SYMS-NEXT:       TypeChkSectNum: 0x0
 ; SYMS-NEXT:       SymbolAlignmentLog2: 0
@@ -159,7 +159,7 @@
 ; SYMS-NEXT:     NumberOfAuxEntries: 1
 ; SYMS-NEXT:     CSECT Auxiliary Entry {
 ; SYMS-NEXT:       Index: [[#INDX+5]]
-; SYMS-NEXT:       ContainingCsectSymbolIndex: 3
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX]]
 ; SYMS-NEXT:       ParameterHashIndex: 0x0
 ; SYMS-NEXT:       TypeChkSectNum: 0x0
 ; SYMS-NEXT:       SymbolAlignmentLog2: 0
@@ -180,7 +180,7 @@
 ; SYMS-NEXT:     NumberOfAuxEntries: 1
 ; SYMS-NEXT:     CSECT Auxiliary Entry {
 ; SYMS-NEXT:       Index: [[#INDX+7]]
-; SYMS-NEXT:       ContainingCsectSymbolIndex: 3
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX]]
 ; SYMS-NEXT:       ParameterHashIndex: 0x0
 ; SYMS-NEXT:       TypeChkSectNum: 0x0
 ; SYMS-NEXT:       SymbolAlignmentLog2: 0
@@ -201,7 +201,7 @@
 ; SYMS-NEXT:     NumberOfAuxEntries: 1
 ; SYMS-NEXT:     CSECT Auxiliary Entry {
 ; SYMS-NEXT:       Index: [[#INDX+9]]
-; SYMS-NEXT:       ContainingCsectSymbolIndex: 3
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX]]
 ; SYMS-NEXT:       ParameterHashIndex: 0x0
 ; SYMS-NEXT:       TypeChkSectNum: 0x0
 ; SYMS-NEXT:       SymbolAlignmentLog2: 0
@@ -222,7 +222,7 @@
 ; SYMS-NEXT:     NumberOfAuxEntries: 1
 ; SYMS-NEXT:     CSECT Auxiliary Entry {
 ; SYMS-NEXT:       Index: [[#INDX+11]]
-; SYMS-NEXT:       ContainingCsectSymbolIndex: 3
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX]]
 ; SYMS-NEXT:       ParameterHashIndex: 0x0
 ; SYMS-NEXT:       TypeChkSectNum: 0x0
 ; SYMS-NEXT:       SymbolAlignmentLog2: 0
@@ -243,7 +243,7 @@
 ; SYMS-NEXT:     NumberOfAuxEntries: 1
 ; SYMS-NEXT:     CSECT Auxiliary Entry {
 ; SYMS-NEXT:       Index: [[#INDX+13]]
-; SYMS-NEXT:       ContainingCsectSymbolIndex: 3
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX]]
 ; SYMS-NEXT:       ParameterHashIndex: 0x0
 ; SYMS-NEXT:       TypeChkSectNum: 0x0
 ; SYMS-NEXT:       SymbolAlignmentLog2: 0
@@ -264,7 +264,7 @@
 ; SYMS-NEXT:     NumberOfAuxEntries: 1
 ; SYMS-NEXT:     CSECT Auxiliary Entry {
 ; SYMS-NEXT:       Index: [[#INDX+15]]
-; SYMS-NEXT:       ContainingCsectSymbolIndex: 3
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX]]
 ; SYMS-NEXT:       ParameterHashIndex: 0x0
 ; SYMS-NEXT:       TypeChkSectNum: 0x0
 ; SYMS-NEXT:       SymbolAlignmentLog2: 0
@@ -285,7 +285,7 @@
 ; SYMS-NEXT:     NumberOfAuxEntries: 1
 ; SYMS-NEXT:     CSECT Auxiliary Entry {
 ; SYMS-NEXT:       Index: [[#INDX+17]]
-; SYMS-NEXT:       ContainingCsectSymbolIndex: 3
+; SYMS-NEXT:       ContainingCsectSymbolIndex: [[#INDX]]
 ; SYMS-NEXT:       ParameterHashIndex: 0x0
 ; SYMS-NEXT:       TypeChkSectNum: 0x0
 ; SYMS-NEXT:       SymbolAlignmentLog2: 0
diff --git a/llvm/test/CodeGen/PowerPC/bswap-load-store.ll b/llvm/test/CodeGen/PowerPC/bswap-load-store.ll
index 093f7f55635704d29f54195b17fd88e7743264ca..2a8a765f1528b67fba27787bdc60546e403705b8 100644
--- a/llvm/test/CodeGen/PowerPC/bswap-load-store.ll
+++ b/llvm/test/CodeGen/PowerPC/bswap-load-store.ll
@@ -1,82 +1,180 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -mcpu=ppc32 | FileCheck %s -check-prefix=X32
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=ppc64 | FileCheck %s -check-prefix=X64
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -mcpu=pwr7 | FileCheck %s -check-prefix=PWR7
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -mcpu=pwr7 | FileCheck %s -check-prefix=X32
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs < %s -mtriple=ppc32--     -mcpu=ppc32 | FileCheck %s --check-prefixes=X32
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs < %s -mtriple=ppc32--     -mcpu=pwr7  | FileCheck %s --check-prefixes=X32,PWR7_32
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs < %s -mtriple=powerpc64-- -mcpu=ppc64 | FileCheck %s --check-prefixes=X64
+; RUN: llc -ppc-asm-full-reg-names -verify-machineinstrs < %s -mtriple=powerpc64-- -mcpu=pwr7  | FileCheck %s --check-prefixes=PWR7_64
 
 
 define void @STWBRX(i32 %i, i8* %ptr, i32 %off) {
-        %tmp1 = getelementptr i8, i8* %ptr, i32 %off                ; <i8*> [#uses=1]
-        %tmp1.upgrd.1 = bitcast i8* %tmp1 to i32*               ; <i32*> [#uses=1]
-        %tmp13 = tail call i32 @llvm.bswap.i32( i32 %i )                ; <i32> [#uses=1]
-        store i32 %tmp13, i32* %tmp1.upgrd.1
-        ret void
+; X32-LABEL: STWBRX:
+; X32:       # %bb.0:
+; X32-NEXT:    stwbrx r3, r4, r5
+; X32-NEXT:    blr
+;
+; X64-LABEL: STWBRX:
+; X64:       # %bb.0:
+; X64-NEXT:    extsw r5, r5
+; X64-NEXT:    stwbrx r3, r4, r5
+; X64-NEXT:    blr
+;
+; PWR7_64-LABEL: STWBRX:
+; PWR7_64:       # %bb.0:
+; PWR7_64-NEXT:    extsw r5, r5
+; PWR7_64-NEXT:    stwbrx r3, r4, r5
+; PWR7_64-NEXT:    blr
+  %tmp1 = getelementptr i8, i8* %ptr, i32 %off
+  %tmp1.upgrd.1 = bitcast i8* %tmp1 to i32*
+  %tmp13 = tail call i32 @llvm.bswap.i32( i32 %i )
+  store i32 %tmp13, i32* %tmp1.upgrd.1
+  ret void
 }
 
 define i32 @LWBRX(i8* %ptr, i32 %off) {
-        %tmp1 = getelementptr i8, i8* %ptr, i32 %off                ; <i8*> [#uses=1]
-        %tmp1.upgrd.2 = bitcast i8* %tmp1 to i32*               ; <i32*> [#uses=1]
-        %tmp = load i32, i32* %tmp1.upgrd.2          ; <i32> [#uses=1]
-        %tmp14 = tail call i32 @llvm.bswap.i32( i32 %tmp )              ; <i32> [#uses=1]
-        ret i32 %tmp14
+; X32-LABEL: LWBRX:
+; X32:       # %bb.0:
+; X32-NEXT:    lwbrx r3, r3, r4
+; X32-NEXT:    blr
+;
+; X64-LABEL: LWBRX:
+; X64:       # %bb.0:
+; X64-NEXT:    extsw r4, r4
+; X64-NEXT:    lwbrx r3, r3, r4
+; X64-NEXT:    blr
+;
+; PWR7_64-LABEL: LWBRX:
+; PWR7_64:       # %bb.0:
+; PWR7_64-NEXT:    extsw r4, r4
+; PWR7_64-NEXT:    lwbrx r3, r3, r4
+; PWR7_64-NEXT:    blr
+  %tmp1 = getelementptr i8, i8* %ptr, i32 %off
+  %tmp1.upgrd.2 = bitcast i8* %tmp1 to i32*
+  %tmp = load i32, i32* %tmp1.upgrd.2
+  %tmp14 = tail call i32 @llvm.bswap.i32( i32 %tmp )
+  ret i32 %tmp14
 }
 
 define void @STHBRX(i16 %s, i8* %ptr, i32 %off) {
-        %tmp1 = getelementptr i8, i8* %ptr, i32 %off                ; <i8*> [#uses=1]
-        %tmp1.upgrd.3 = bitcast i8* %tmp1 to i16*               ; <i16*> [#uses=1]
-        %tmp5 = call i16 @llvm.bswap.i16( i16 %s )              ; <i16> [#uses=1]
-        store i16 %tmp5, i16* %tmp1.upgrd.3
-        ret void
+; X32-LABEL: STHBRX:
+; X32:       # %bb.0:
+; X32-NEXT:    sthbrx r3, r4, r5
+; X32-NEXT:    blr
+;
+; X64-LABEL: STHBRX:
+; X64:       # %bb.0:
+; X64-NEXT:    extsw r5, r5
+; X64-NEXT:    sthbrx r3, r4, r5
+; X64-NEXT:    blr
+;
+; PWR7_64-LABEL: STHBRX:
+; PWR7_64:       # %bb.0:
+; PWR7_64-NEXT:    extsw r5, r5
+; PWR7_64-NEXT:    sthbrx r3, r4, r5
+; PWR7_64-NEXT:    blr
+  %tmp1 = getelementptr i8, i8* %ptr, i32 %off
+  %tmp1.upgrd.3 = bitcast i8* %tmp1 to i16*
+  %tmp5 = call i16 @llvm.bswap.i16( i16 %s )
+  store i16 %tmp5, i16* %tmp1.upgrd.3
+  ret void
 }
 
 define i16 @LHBRX(i8* %ptr, i32 %off) {
-        %tmp1 = getelementptr i8, i8* %ptr, i32 %off                ; <i8*> [#uses=1]
-        %tmp1.upgrd.4 = bitcast i8* %tmp1 to i16*               ; <i16*> [#uses=1]
-        %tmp = load i16, i16* %tmp1.upgrd.4          ; <i16> [#uses=1]
-        %tmp6 = call i16 @llvm.bswap.i16( i16 %tmp )            ; <i16> [#uses=1]
-        ret i16 %tmp6
+; X32-LABEL: LHBRX:
+; X32:       # %bb.0:
+; X32-NEXT:    lhbrx r3, r3, r4
+; X32-NEXT:    blr
+;
+; X64-LABEL: LHBRX:
+; X64:       # %bb.0:
+; X64-NEXT:    extsw r4, r4
+; X64-NEXT:    lhbrx r3, r3, r4
+; X64-NEXT:    blr
+;
+; PWR7_64-LABEL: LHBRX:
+; PWR7_64:       # %bb.0:
+; PWR7_64-NEXT:    extsw r4, r4
+; PWR7_64-NEXT:    lhbrx r3, r3, r4
+; PWR7_64-NEXT:    blr
+  %tmp1 = getelementptr i8, i8* %ptr, i32 %off
+  %tmp1.upgrd.4 = bitcast i8* %tmp1 to i16*
+  %tmp = load i16, i16* %tmp1.upgrd.4
+  %tmp6 = call i16 @llvm.bswap.i16( i16 %tmp )
+  ret i16 %tmp6
 }
 
 define void @STDBRX(i64 %i, i8* %ptr, i64 %off) {
-        %tmp1 = getelementptr i8, i8* %ptr, i64 %off                ; <i8*> [#uses=1]
-        %tmp1.upgrd.1 = bitcast i8* %tmp1 to i64*               ; <i64*> [#uses=1]
-        %tmp13 = tail call i64 @llvm.bswap.i64( i64 %i )                ; <i64> [#uses=1]
-        store i64 %tmp13, i64* %tmp1.upgrd.1
-        ret void
+; PWR7_32-LABEL: STDBRX:
+; PWR7_32:       # %bb.0:
+; PWR7_32-NEXT:    li r6, 4
+; PWR7_32-NEXT:    add r7, r5, r8
+; PWR7_32-NEXT:    stwbrx r4, r5, r8
+; PWR7_32-NEXT:    stwbrx r3, r7, r6
+; PWR7_32-NEXT:    blr
+;
+; X64-LABEL: STDBRX:
+; X64:       # %bb.0:
+; X64-NEXT:    rotldi r6, r3, 16
+; X64-NEXT:    rotldi r7, r3, 8
+; X64-NEXT:    rldimi r7, r6, 8, 48
+; X64-NEXT:    rotldi r6, r3, 24
+; X64-NEXT:    rldimi r7, r6, 16, 40
+; X64-NEXT:    rotldi r6, r3, 32
+; X64-NEXT:    rldimi r7, r6, 24, 32
+; X64-NEXT:    rotldi r6, r3, 48
+; X64-NEXT:    rldimi r7, r6, 40, 16
+; X64-NEXT:    rotldi r6, r3, 56
+; X64-NEXT:    rldimi r7, r6, 48, 8
+; X64-NEXT:    rldimi r7, r3, 56, 0
+; X64-NEXT:    stdx r7, r4, r5
+; X64-NEXT:    blr
+;
+; PWR7_64-LABEL: STDBRX:
+; PWR7_64:       # %bb.0:
+; PWR7_64-NEXT:    stdbrx r3, r4, r5
+; PWR7_64-NEXT:    blr
+  %tmp1 = getelementptr i8, i8* %ptr, i64 %off
+  %tmp1.upgrd.1 = bitcast i8* %tmp1 to i64*
+  %tmp13 = tail call i64 @llvm.bswap.i64( i64 %i )
+  store i64 %tmp13, i64* %tmp1.upgrd.1
+  ret void
 }
 
 define i64 @LDBRX(i8* %ptr, i64 %off) {
-        %tmp1 = getelementptr i8, i8* %ptr, i64 %off                ; <i8*> [#uses=1]
-        %tmp1.upgrd.2 = bitcast i8* %tmp1 to i64*               ; <i64*> [#uses=1]
-        %tmp = load i64, i64* %tmp1.upgrd.2          ; <i64> [#uses=1]
-        %tmp14 = tail call i64 @llvm.bswap.i64( i64 %tmp )              ; <i64> [#uses=1]
-        ret i64 %tmp14
+; PWR7_32-LABEL: LDBRX:
+; PWR7_32:       # %bb.0:
+; PWR7_32-NEXT:    li r5, 4
+; PWR7_32-NEXT:    add r7, r3, r6
+; PWR7_32-NEXT:    lwbrx r4, r3, r6
+; PWR7_32-NEXT:    lwbrx r3, r7, r5
+; PWR7_32-NEXT:    blr
+;
+; X64-LABEL: LDBRX:
+; X64:       # %bb.0:
+; X64-NEXT:    ldx r4, r3, r4
+; X64-NEXT:    rotldi r5, r4, 16
+; X64-NEXT:    rotldi r3, r4, 8
+; X64-NEXT:    rldimi r3, r5, 8, 48
+; X64-NEXT:    rotldi r5, r4, 24
+; X64-NEXT:    rldimi r3, r5, 16, 40
+; X64-NEXT:    rotldi r5, r4, 32
+; X64-NEXT:    rldimi r3, r5, 24, 32
+; X64-NEXT:    rotldi r5, r4, 48
+; X64-NEXT:    rldimi r3, r5, 40, 16
+; X64-NEXT:    rotldi r5, r4, 56
+; X64-NEXT:    rldimi r3, r5, 48, 8
+; X64-NEXT:    rldimi r3, r4, 56, 0
+; X64-NEXT:    blr
+;
+; PWR7_64-LABEL: LDBRX:
+; PWR7_64:       # %bb.0:
+; PWR7_64-NEXT:    ldbrx r3, r3, r4
+; PWR7_64-NEXT:    blr
+  %tmp1 = getelementptr i8, i8* %ptr, i64 %off
+  %tmp1.upgrd.2 = bitcast i8* %tmp1 to i64*
+  %tmp = load i64, i64* %tmp1.upgrd.2
+  %tmp14 = tail call i64 @llvm.bswap.i64( i64 %tmp )
+  ret i64 %tmp14
 }
 
-declare i32 @llvm.bswap.i32(i32)
-
 declare i16 @llvm.bswap.i16(i16)
-
+declare i32 @llvm.bswap.i32(i32)
 declare i64 @llvm.bswap.i64(i64)
-
-
-; X32: stwbrx
-; X32: lwbrx
-; X32: sthbrx
-; X32: lhbrx
-; X32-NOT: ldbrx
-; X32-NOT: stdbrx
-
-; X64: stwbrx
-; X64: lwbrx
-; X64: sthbrx
-; X64: lhbrx
-; X64-NOT: ldbrx
-; X64-NOT: stdbrx
-
-; PWR7: stwbrx
-; PWR7: lwbrx
-; PWR7: sthbrx
-; PWR7: lhbrx
-; PWR7: stdbrx
-; PWR7: ldbrx
-
diff --git a/llvm/test/CodeGen/PowerPC/constant-pool.ll b/llvm/test/CodeGen/PowerPC/constant-pool.ll
index 602c67df0dec3dddfc6310387da0542a97210d70..69c2582f40053c2bc2475442260063a01c1c6dcd 100644
--- a/llvm/test/CodeGen/PowerPC/constant-pool.ll
+++ b/llvm/test/CodeGen/PowerPC/constant-pool.ll
@@ -9,7 +9,9 @@
  define float @FloatConstantPool() {
 ; CHECK-LABEL: FloatConstantPool:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    plfs f1, .LCPI0_0@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs1, 0, 0
+; CHECK-NEXT:    xxsplti32dx vs1, 1, 8388577
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: FloatConstantPool:
@@ -24,7 +26,9 @@ entry:
  define double @DoubleConstantPool() {
 ; CHECK-LABEL: DoubleConstantPool:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    plfd f1, .LCPI1_0@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs1, 0, 1048574
+; CHECK-NEXT:    xxsplti32dx vs1, 1, 780229072
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: DoubleConstantPool:
@@ -39,8 +43,12 @@ entry:
  define ppc_fp128 @LongDoubleConstantPool() {
 ; CHECK-LABEL: LongDoubleConstantPool:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    plfd f1, .LCPI2_0@PCREL(0), 1
-; CHECK-NEXT:    plfd f2, .LCPI2_1@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs1, 0, 56623104
+; CHECK-NEXT:    xxsplti32dx vs2, 0, -2146625897
+; CHECK-NEXT:    xxsplti32dx vs1, 1, -609716532
+; CHECK-NEXT:    xxsplti32dx vs2, 1, 1339675259
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: LongDoubleConstantPool:
@@ -185,9 +193,11 @@ entry:
 define double @two_constants(double %a) {
 ; CHECK-LABEL: two_constants:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    plfd f0, .LCPI11_0@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs0, 0, 1074446467
+; CHECK-NEXT:    xxsplti32dx vs0, 1, 309237645
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    plfd f1, .LCPI11_1@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs1, 0, 1073922179
+; CHECK-NEXT:    xxsplti32dx vs1, 1, 309237645
 ; CHECK-NEXT:    xsadddp f1, f0, f1
 ; CHECK-NEXT:    blr
 ;
@@ -212,11 +222,15 @@ define double @two_constants_two_bb(i32 %m, double %a) {
 ; CHECK-NEXT:    cmplwi r3, 0
 ; CHECK-NEXT:    beq cr0, .LBB12_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    plfd f1, .LCPI12_0@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs1, 0, 1074935889
+; CHECK-NEXT:    xxsplti32dx vs1, 1, -343597384
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NEXT:    blr
 ; CHECK-NEXT:  .LBB12_2: # %if.end
-; CHECK-NEXT:    plfd f0, .LCPI12_1@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs0, 0, 1076085391
+; CHECK-NEXT:    xxsplti32dx vs0, 1, 1546188227
 ; CHECK-NEXT:    xsadddp f1, f1, f0
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: two_constants_two_bb:
@@ -248,11 +262,14 @@ return:
 define double @three_constants_f64(double %a, double %c) {
 ; CHECK-LABEL: three_constants_f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    plfd f0, .LCPI13_0@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs0, 0, 1074446467
+; CHECK-NEXT:    xxsplti32dx vs0, 1, 309237645
 ; CHECK-NEXT:    xsadddp f0, f1, f0
-; CHECK-NEXT:    plfd f1, .LCPI13_1@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs1, 0, 1073922179
+; CHECK-NEXT:    xxsplti32dx vs1, 1, 309237645
 ; CHECK-NEXT:    xsadddp f0, f0, f1
-; CHECK-NEXT:    plfd f1, .LCPI13_2@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs1, 0, 1073948393
+; CHECK-NEXT:    xxsplti32dx vs1, 1, 2027224564
 ; CHECK-NEXT:    xsadddp f1, f0, f1
 ; CHECK-NEXT:    blr
 ;
@@ -340,21 +357,26 @@ entry:
 
 define ppc_fp128 @three_constants_ppcf128(ppc_fp128 %a, ppc_fp128 %c) {
 ; CHECK-LABEL: three_constants_ppcf128:
-; CHECK:         .localentry three_constants_ppcf128, 1
-; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    std r0, 16(r1)
 ; CHECK-NEXT:    stdu r1, -32(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    plfd f3, .LCPI16_0@PCREL(0), 1
-; CHECK-NEXT:    xxlxor f4, f4, f4
+; CHECK-DAG:     xxlxor f4, f4, f4
+; CHECK-DAG:     xxsplti32dx vs3, 0, 1074935889
+; CHECK-NEXT:    xxsplti32dx vs3, 1, -343597384
+; CHECK-NEXT:    # kill: def $f3 killed $f3 killed $vsl3
 ; CHECK-NEXT:    bl __gcc_qadd@notoc
-; CHECK-NEXT:    plfd f3, .LCPI16_1@PCREL(0), 1
-; CHECK-NEXT:    xxlxor f4, f4, f4
+; CHECK-DAG:     xxlxor f4, f4, f4
+; CHECK-DAG:     xxsplti32dx vs3, 0, 1074935889
+; CHECK-NEXT:    xxsplti32dx vs3, 1, -1719329096
+; CHECK-NEXT:    # kill: def $f3 killed $f3 killed $vsl3
 ; CHECK-NEXT:    bl __gcc_qadd@notoc
-; CHECK-NEXT:    plfd f3, .LCPI16_2@PCREL(0), 1
-; CHECK-NEXT:    xxlxor f4, f4, f4
+; CHECK-DAG:     xxlxor f4, f4, f4
+; CHECK-DAG:     xxsplti32dx vs3, 0, 1074935889
+; CHECK-NEXT:    xxsplti32dx vs3, 1, 8724152
+; CHECK-NEXT:    # kill: def $f3 killed $f3 killed $vsl3
 ; CHECK-NEXT:    bl __gcc_qadd@notoc
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
diff --git a/llvm/test/CodeGen/PowerPC/future-check-features.ll b/llvm/test/CodeGen/PowerPC/future-check-features.ll
index aaae1089ea01e47a53b23a354dc18dc06ccd0257..1dca4dafd5b641ed0d20541c8f55361ec9f125ae 100644
--- a/llvm/test/CodeGen/PowerPC/future-check-features.ll
+++ b/llvm/test/CodeGen/PowerPC/future-check-features.ll
@@ -1,7 +1,7 @@
-; RUN: llc -mattr=pcrelative-memops,prefix-instrs,paired-vector-memops,mma,rop-protection \
+; RUN: llc -mattr=pcrelative-memops,prefix-instrs,paired-vector-memops,mma,rop-protect,privileged \
 ; RUN:   -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names %s -o - 2>&1 | FileCheck %s
-; RUN: llc -mattr=pcrelative-memops,prefix-instrs,paired-vector-memops,mma,rop-protection \
+; RUN: llc -mattr=pcrelative-memops,prefix-instrs,paired-vector-memops,mma,rop-protect,privileged \
 ; RUN:   -verify-machineinstrs -mtriple=powerpc64-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names %s -o - 2>&1 | FileCheck %s
 
diff --git a/llvm/test/CodeGen/PowerPC/out-of-range-dform.ll b/llvm/test/CodeGen/PowerPC/out-of-range-dform.ll
new file mode 100644
index 0000000000000000000000000000000000000000..13b68a18ac79410c5e16509e2edd25f0f7387faf
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/out-of-range-dform.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s \
+; RUN:   -check-prefix=CHECK-P9
+
+@_ZL3num = external dso_local unnamed_addr global float, align 4
+
+define dso_local void @main() local_unnamed_addr personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-P9-LABEL: main:
+; CHECK-P9:       # %bb.0: # %bb
+; CHECK-P9-NEXT:    mflr r0
+; CHECK-P9-NEXT:    std r0, 16(r1)
+; CHECK-P9-NEXT:    stdu r1, -32(r1)
+; CHECK-P9-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P9-NEXT:    .cfi_offset lr, 16
+; CHECK-P9-NEXT:    bl malloc
+; CHECK-P9-NEXT:    nop
+; CHECK-P9-NEXT:    addis r4, r2, _ZL3num@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, -25400
+; CHECK-P9-NEXT:    lfs f0, _ZL3num@toc@l(r4)
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI0_0@toc@ha
+; CHECK-P9-NEXT:    lfs f1, .LCPI0_0@toc@l(r4)
+; CHECK-P9-NEXT:    li r4, 0
+; CHECK-P9-NEXT:    xsmulsp f0, f0, f1
+; CHECK-P9-NEXT:    cmpldi r4, 0
+; CHECK-P9-NEXT:    beq- cr0, .LBB0_2
+; CHECK-P9-NEXT:    .p2align 5
+; CHECK-P9-NEXT:  .LBB0_1: # %bb5
+; CHECK-P9-NEXT:    #
+; CHECK-P9-NEXT:    addi r3, r3, 25400
+; CHECK-P9-NEXT:    addi r4, r4, 25400
+; CHECK-P9-NEXT:    stfs f0, 15240(r3)
+; CHECK-P9-NEXT:    cmpldi r4, 0
+; CHECK-P9-NEXT:    bne+ cr0, .LBB0_1
+; CHECK-P9-NEXT:  .LBB0_2: # %bb16
+bb:
+  %i = tail call noalias dereferenceable_or_null(6451600) i8* @malloc()
+  %i1 = bitcast i8* %i to float*
+  br label %bb2
+
+bb2:                                              ; preds = %bb5, %bb
+  %i3 = phi i64 [ 0, %bb ], [ %i15, %bb5 ]
+  %i4 = icmp eq i64 %i3, 0
+  br i1 %i4, label %bb16, label %bb5
+
+bb5:                                              ; preds = %bb2
+  %i6 = mul nuw nsw i64 %i3, 1270
+  %i7 = add nuw nsw i64 %i6, 0
+  %i8 = getelementptr inbounds float, float* %i1, i64 %i7
+  store float undef, float* %i8, align 4
+  %i9 = add nuw nsw i64 %i3, 3
+  %i10 = load float, float* @_ZL3num, align 4
+  %i11 = fmul float %i10, 0x3E00000000000000
+  %i12 = mul nuw nsw i64 %i9, 1270
+  %i13 = add nuw nsw i64 %i12, 0
+  %i14 = getelementptr inbounds float, float* %i1, i64 %i13
+  store float %i11, float* %i14, align 4
+  %i15 = add nuw nsw i64 %i3, 5
+  br label %bb2
+
+bb16:                                             ; preds = %bb2
+  unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i8* @malloc() local_unnamed_addr
diff --git a/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll b/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll
index 0836c4cb7bbe292f710aaf3b8a42b3ede78a2d83..b75539c59918d16604ff2f576f19d499b7379060 100644
--- a/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-splatImm-CPload-pcrel.ll
@@ -122,19 +122,23 @@ entry:
 define dso_local double @testDoubleNonRepresentableScalar() local_unnamed_addr {
 ; CHECK-LE-LABEL: testDoubleNonRepresentableScalar:
 ; CHECK-LE:       # %bb.0: # %entry
-; CHECK-LE-NEXT:    plfd f1, .LCPI3_0@PCREL(0), 1
+; CHECK-LE-NEXT:    xxsplti32dx vs1, 0, 1081435463
+; CHECK-LE-NEXT:    xxsplti32dx vs1, 1, -1374389535
+; CHECK-LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-NOPCREL-BE-LABEL: testDoubleNonRepresentableScalar:
 ; CHECK-NOPCREL-BE:       # %bb.0: # %entry
-; CHECK-NOPCREL-BE-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
-; CHECK-NOPCREL-BE-NEXT:    lfd f1, .LCPI3_0@toc@l(r3)
+; CHECK-NOPCREL-BE-NEXT:    xxsplti32dx vs1, 0, 1081435463
+; CHECK-NOPCREL-BE-NEXT:    xxsplti32dx vs1, 1, -1374389535
+; CHECK-NOPCREL-BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NOPCREL-BE-NEXT:    blr
 ;
 ; CHECK-NOPCREL-LE-LABEL: testDoubleNonRepresentableScalar:
 ; CHECK-NOPCREL-LE:       # %bb.0: # %entry
-; CHECK-NOPCREL-LE-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
-; CHECK-NOPCREL-LE-NEXT:    lfd f1, .LCPI3_0@toc@l(r3)
+; CHECK-NOPCREL-LE-NEXT:    xxsplti32dx vs1, 0, 1081435463
+; CHECK-NOPCREL-LE-NEXT:    xxsplti32dx vs1, 1, -1374389535
+; CHECK-NOPCREL-LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NOPCREL-LE-NEXT:    blr
 ;
 ; CHECK-NOPREFIX-LABEL: testDoubleNonRepresentableScalar:
@@ -145,7 +149,9 @@ define dso_local double @testDoubleNonRepresentableScalar() local_unnamed_addr {
 ;
 ; CHECK-BE-LABEL: testDoubleNonRepresentableScalar:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    plfd f1, .LCPI3_0@PCREL(0), 1
+; CHECK-BE-NEXT:    xxsplti32dx vs1, 0, 1081435463
+; CHECK-BE-NEXT:    xxsplti32dx vs1, 1, -1374389535
+; CHECK-BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-BE-NEXT:    blr
 entry:
   ret double 3.423300e+02
@@ -154,19 +160,23 @@ entry:
 define dso_local float @testFloatDenormScalar() local_unnamed_addr {
 ; CHECK-LE-LABEL: testFloatDenormScalar:
 ; CHECK-LE:       # %bb.0: # %entry
-; CHECK-LE-NEXT:    plfs f1, .LCPI4_0@PCREL(0), 1
+; CHECK-LE-NEXT:    xxsplti32dx vs1, 0, 0
+; CHECK-LE-NEXT:    xxsplti32dx vs1, 1, 7136238
+; CHECK-LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-NOPCREL-BE-LABEL: testFloatDenormScalar:
 ; CHECK-NOPCREL-BE:       # %bb.0: # %entry
-; CHECK-NOPCREL-BE-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
-; CHECK-NOPCREL-BE-NEXT:    lfs f1, .LCPI4_0@toc@l(r3)
+; CHECK-NOPCREL-BE-NEXT:    xxsplti32dx vs1, 0, 0
+; CHECK-NOPCREL-BE-NEXT:    xxsplti32dx vs1, 1, 7136238
+; CHECK-NOPCREL-BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NOPCREL-BE-NEXT:    blr
 ;
 ; CHECK-NOPCREL-LE-LABEL: testFloatDenormScalar:
 ; CHECK-NOPCREL-LE:       # %bb.0: # %entry
-; CHECK-NOPCREL-LE-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
-; CHECK-NOPCREL-LE-NEXT:    lfs f1, .LCPI4_0@toc@l(r3)
+; CHECK-NOPCREL-LE-NEXT:    xxsplti32dx vs1, 0, 0
+; CHECK-NOPCREL-LE-NEXT:    xxsplti32dx vs1, 1, 7136238
+; CHECK-NOPCREL-LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NOPCREL-LE-NEXT:    blr
 ;
 ; CHECK-NOPREFIX-LABEL: testFloatDenormScalar:
@@ -177,7 +187,9 @@ define dso_local float @testFloatDenormScalar() local_unnamed_addr {
 ;
 ; CHECK-BE-LABEL: testFloatDenormScalar:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    plfs f1, .LCPI4_0@PCREL(0), 1
+; CHECK-BE-NEXT:    xxsplti32dx vs1, 0, 0
+; CHECK-BE-NEXT:    xxsplti32dx vs1, 1, 7136238
+; CHECK-BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-BE-NEXT:    blr
 entry:
   ret float 0x380B38FB80000000
@@ -186,19 +198,23 @@ entry:
 define dso_local double @testFloatDenormToDoubleScalar() local_unnamed_addr {
 ; CHECK-LE-LABEL: testFloatDenormToDoubleScalar:
 ; CHECK-LE:       # %bb.0: # %entry
-; CHECK-LE-NEXT:    plfs f1, .LCPI5_0@PCREL(0), 1
+; CHECK-LE-NEXT:    xxsplti32dx vs1, 0, 940259579
+; CHECK-LE-NEXT:    xxsplti32dx vs1, 1, -2147483648
+; CHECK-LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-NOPCREL-BE-LABEL: testFloatDenormToDoubleScalar:
 ; CHECK-NOPCREL-BE:       # %bb.0: # %entry
-; CHECK-NOPCREL-BE-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
-; CHECK-NOPCREL-BE-NEXT:    lfs f1, .LCPI5_0@toc@l(r3)
+; CHECK-NOPCREL-BE-NEXT:    xxsplti32dx vs1, 0, 940259579
+; CHECK-NOPCREL-BE-NEXT:    xxsplti32dx vs1, 1, -2147483648
+; CHECK-NOPCREL-BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NOPCREL-BE-NEXT:    blr
 ;
 ; CHECK-NOPCREL-LE-LABEL: testFloatDenormToDoubleScalar:
 ; CHECK-NOPCREL-LE:       # %bb.0: # %entry
-; CHECK-NOPCREL-LE-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
-; CHECK-NOPCREL-LE-NEXT:    lfs f1, .LCPI5_0@toc@l(r3)
+; CHECK-NOPCREL-LE-NEXT:    xxsplti32dx vs1, 0, 940259579
+; CHECK-NOPCREL-LE-NEXT:    xxsplti32dx vs1, 1, -2147483648
+; CHECK-NOPCREL-LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-NOPCREL-LE-NEXT:    blr
 ;
 ; CHECK-NOPREFIX-LABEL: testFloatDenormToDoubleScalar:
@@ -209,7 +225,9 @@ define dso_local double @testFloatDenormToDoubleScalar() local_unnamed_addr {
 ;
 ; CHECK-BE-LABEL: testFloatDenormToDoubleScalar:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    plfs f1, .LCPI5_0@PCREL(0), 1
+; CHECK-BE-NEXT:    xxsplti32dx vs1, 0, 940259579
+; CHECK-BE-NEXT:    xxsplti32dx vs1, 1, -2147483648
+; CHECK-BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-BE-NEXT:    blr
 entry:
   ret double 0x380B38FB80000000
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-leaf.ll b/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-leaf.ll
index f2da036a37c50b7cb0d8c7868bb6f1e9a9f3b90b..ae7db80de19940811bd0e81f5abf549d63b430d5 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-leaf.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-leaf.ll
@@ -173,7 +173,9 @@ define dso_local double @UsesX2AsConstPoolTOC() local_unnamed_addr {
 ; CHECK-LARGE:     add r2, r2, r12
 ; CHECK-S-NOT:       .localentry
 ; CHECK-ALL:       # %bb.0: # %entry
-; CHECK-S-NEXT:    plfd f1, .LCPI7_0@PCREL(0), 1
+; CHECK-S-NEXT:    xxsplti32dx vs1, 0, 1078011044
+; CHECK-S-NEXT:    xxsplti32dx vs1, 1, -337824948
+; CHECK-S-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
 ; CHECK-S-NEXT:    blr
 entry:
   ret double 0x404124A4EBDD334C
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-linkeropt.ll b/llvm/test/CodeGen/PowerPC/pcrel-linkeropt.ll
index f6b4760659a10605ea6b8b054eb68d7cb85aad68..1e42c695c5853ee41f267cccf69bdb254d44d376 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-linkeropt.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-linkeropt.ll
@@ -35,6 +35,9 @@
 @FuncPtrOut = external local_unnamed_addr global void (...)*, align 8
 
 define dso_local void @ReadWrite8() local_unnamed_addr #0 {
+; In this test the stb r3, 0(r4) cannot be optimized because it
+; uses the register r3 and that register is defined by lbz r3, 0(r3)
+; which is defined between the pld and the stb.
 ; CHECK-LABEL: ReadWrite8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pld r3, input8@got@pcrel(0), 1
@@ -44,9 +47,6 @@ define dso_local void @ReadWrite8() local_unnamed_addr #0 {
 ; CHECK-NEXT:    lbz r3, 0(r3)
 ; CHECK-NEXT:    stb r3, 0(r4)
 ; CHECK-NEXT:    blr
-; In this test the stb r3, 0(r4) cannot be optimized because it
-; uses the register r3 and that register is defined by lbz r3, 0(r3)
-; which is defined between the pld and the stb.
 entry:
   %0 = load i8, i8* @input8, align 1
   store i8 %0, i8* @output8, align 1
@@ -54,6 +54,9 @@ entry:
 }
 
 define dso_local void @ReadWrite16() local_unnamed_addr #0 {
+; In this test the sth r3, 0(r4) cannot be optimized because it
+; uses the register r3 and that register is defined by lhz r3, 0(r3)
+; which is defined between the pld and the sth.
 ; CHECK-LABEL: ReadWrite16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pld r3, input16@got@pcrel(0), 1
@@ -63,9 +66,6 @@ define dso_local void @ReadWrite16() local_unnamed_addr #0 {
 ; CHECK-NEXT:    lhz r3, 0(r3)
 ; CHECK-NEXT:    sth r3, 0(r4)
 ; CHECK-NEXT:    blr
-; In this test the sth r3, 0(r4) cannot be optimized because it
-; uses the register r3 and that register is defined by lhz r3, 0(r3)
-; which is defined between the pld and the sth.
 entry:
   %0 = load i16, i16* @input16, align 2
   store i16 %0, i16* @output16, align 2
@@ -144,7 +144,8 @@ define dso_local void @ReadWritef64() local_unnamed_addr #0 {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pld r3, inputf64@got@pcrel(0), 1
 ; CHECK-NEXT:  .Lpcrel5:
-; CHECK-NEXT:    plfd f1, .LCPI6_0@PCREL(0), 1
+; CHECK-NEXT:    xxsplti32dx vs1, 0, 1075524403
+; CHECK-NEXT:    xxsplti32dx vs1, 1, 858993459
 ; CHECK-NEXT:    .reloc .Lpcrel5-8,R_PPC64_PCREL_OPT,.-(.Lpcrel5-8)
 ; CHECK-NEXT:    lfd f0, 0(r3)
 ; CHECK-NEXT:    pld r3, outputf64@got@pcrel(0), 1
diff --git a/llvm/test/CodeGen/PowerPC/pcrel.ll b/llvm/test/CodeGen/PowerPC/pcrel.ll
index 1d3d96a92904f24c8bb2feb2a101f9383420a020..37c5b7c0be64ad46f8712729e96b186424bfbc2f 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel.ll
@@ -8,13 +8,14 @@
 
 ; Constant Pool Index.
 ; CHECK-S-LABEL: ConstPool
-; CHECK-S:       plfd f1, .LCPI0_0@PCREL(0), 1
+; CHECK-S:       xxsplti32dx vs1, 0, 1081002676
+; CHECK-S-NEXT:       xxsplti32dx vs1, 1, 962072674
 ; CHECK-S:       blr
 
 ; CHECK-O-LABEL: ConstPool
-; CHECK-O:       plfd 1, 0(0), 1
-; CHECK-O-NEXT:  R_PPC64_PCREL34  .rodata.cst8
-; CHECK-O:       blr
+; CHECK-O:       xxsplti32dx 1, 0, 1081002676
+; CHECK-O-NEXT:  xxsplti32dx 1, 1, 962072674
+; CHECK-O-NEXT:  blr
 define dso_local double @ConstPool() local_unnamed_addr {
   entry:
     ret double 0x406ECAB439581062
diff --git a/llvm/test/CodeGen/PowerPC/remove-redundant-toc-saves.ll b/llvm/test/CodeGen/PowerPC/remove-redundant-toc-saves.ll
index e20b9816f3d4ff3ece4fbec74ec5c84acd84bd71..58c959bc0efadb02e469871ed6594c741f9bbc6d 100644
--- a/llvm/test/CodeGen/PowerPC/remove-redundant-toc-saves.ll
+++ b/llvm/test/CodeGen/PowerPC/remove-redundant-toc-saves.ll
@@ -1,9 +1,20 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-ibm-aix-xcoff < %s | FileCheck %s --check-prefix=AIX64
+; RUN: llc -verify-machineinstrs -mtriple=powerpc-ibm-aix-xcoff < %s | FileCheck %s --check-prefix=AIX32
+
 define signext i32 @test1(i32 signext %i, i32 (i32)* nocapture %Func, i32 (i32)* nocapture %Func2) {
 entry:
 ; CHECK-LABEL: test1:
 ; CHECK:    std 2, 24(1)
 ; CHECK-NOT:    std 2, 24(1)
+
+; AIX64-LABEL: test1:
+; AIX64: std 2, 40(1)
+; AIX64-NOT: std 2, 40(1)
+
+; AIX32-LABEL: test1:
+; AIX32: stw 2, 20(1)
+; AIX32-NOT: std 2, 20(1)
   %call = tail call signext i32 %Func(i32 signext %i)
   %call1 = tail call signext i32 %Func2(i32 signext %i)
   %add2 = add nsw i32 %call1, %call
@@ -15,6 +26,14 @@ entry:
 ; CHECK-LABEL: test2:
 ; CHECK:    std 2, 24(1)
 ; CHECK-NOT:    std 2, 24(1)
+
+; AIX64-LABEL: test2:
+; AIX64: std 2, 40(1)
+; AIX64-NOT: std 2, 40(1)
+
+; AIX32-LABEL: test2:
+; AIX32: stw 2, 20(1)
+; AIX32-NOT: std 2, 20(1)
   %call = tail call signext i32 %Func(i32 signext %i)
   %tobool = icmp eq i32 %j, 0
   br i1 %tobool, label %if.end, label %if.then
@@ -38,6 +57,12 @@ define signext i32 @test3(i32 signext %i, i32 (i32)* nocapture %Func, i32 (i32)*
 ; CHECK-LABEL: test3:
 ; CHECK:    std 2, 24(1)
 ; CHECK-NOT:    std 2, 24(1)
+
+; AIX64-LABEL: test3:
+; AIX64-COUNT-3: std 2, 40(1)
+
+; AIX32-LABEL: test3:
+; AIX32-COUNT-3: stw 2, 20(1)
 entry:
   %tobool = icmp eq i32 %i, 0
   br i1 %tobool, label %if.else, label %if.then
@@ -62,6 +87,13 @@ define signext i32 @test4(i32 signext %i, i32 (i32)* nocapture %Func, i32 (i32)*
 ; CHECK:    std 2, 24(1)
 ; CHECK-NOT:    std 2, 24(1)
 
+; AIX64-LABEL: test4:
+; AIX64: std 2, 40(1)
+; AIX64-NOT: std 2, 40(1)
+
+; AIX32-LABEL: test4:
+; AIX32: stw 2, 20(1)
+; AIX32-NOT: std 2, 20(1)
 entry:
   %call = tail call signext i32 %Func(i32 signext %i)
   %tobool = icmp eq i32 %i, 0
@@ -87,6 +119,11 @@ entry:
 ; CHECK-LABEL: test5:
 ; CHECK:    std 2, 24(1)
 
+; AIX64-LABEL: test5:
+; AIX64: std 2, 40(1)
+
+; AIX32-LABEL: test5:
+; AIX32: stw 2, 20(1)
   %tobool = icmp eq i32 %i, 0
   br i1 %tobool, label %if.end, label %if.then
 
@@ -108,6 +145,13 @@ entry:
 ; CHECK:    std 2, 24(1)
 ; CHECK:    std 2, 24(1)
 
+; AIX64-LABEL: test6:
+; AIX64: std 2, 40(1)
+; AIX64: std 2, 40(1)
+
+; AIX32-LABEL: test6:
+; AIX32: stw 2, 20(1)
+; AIX32: stw 2, 20(1)
   %conv = sext i32 %i to i64
   %0 = alloca i8, i64 %conv, align 16
   %1 = bitcast i8* %0 to i32*
diff --git a/llvm/test/CodeGen/PowerPC/urem-seteq-illegal-types.ll b/llvm/test/CodeGen/PowerPC/urem-seteq-illegal-types.ll
index c0f8749f78dc435f40b65eccc3c9577aeaf8caab..40d402d424e60575da443457ae4051c0cd9f6c73 100644
--- a/llvm/test/CodeGen/PowerPC/urem-seteq-illegal-types.ll
+++ b/llvm/test/CodeGen/PowerPC/urem-seteq-illegal-types.ll
@@ -88,9 +88,8 @@ define i1 @test_urem_odd_setne(i4 %X) nounwind {
 ; PPC64LE-NEXT:    clrlwi 3, 3, 28
 ; PPC64LE-NEXT:    ori 4, 4, 52429
 ; PPC64LE-NEXT:    mulhwu 4, 3, 4
-; PPC64LE-NEXT:    rlwinm 5, 4, 0, 0, 29
 ; PPC64LE-NEXT:    srwi 4, 4, 2
-; PPC64LE-NEXT:    add 4, 4, 5
+; PPC64LE-NEXT:    rlwimi 4, 4, 2, 28, 29
 ; PPC64LE-NEXT:    sub 3, 3, 4
 ; PPC64LE-NEXT:    cntlzw 3, 3
 ; PPC64LE-NEXT:    not 3, 3
diff --git a/llvm/test/CodeGen/RISCV/alu32.ll b/llvm/test/CodeGen/RISCV/alu32.ll
index 975716e53c049039ada3e40274f79bab408c45cd..d9fb08b5076221eca45f9e4cf853d66d2ddb5692 100644
--- a/llvm/test/CodeGen/RISCV/alu32.ll
+++ b/llvm/test/CodeGen/RISCV/alu32.ll
@@ -129,6 +129,25 @@ define i32 @srli(i32 %a) nounwind {
   ret i32 %1
 }
 
+; This makes sure SimplifyDemandedBits doesn't prevent us from matching SRLIW
+; on RV64.
+define i32 @srli_demandedbits(i32 %0) {
+; RV32I-LABEL: srli_demandedbits:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srli a0, a0, 3
+; RV32I-NEXT:    ori a0, a0, 1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: srli_demandedbits:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srliw a0, a0, 3
+; RV64I-NEXT:    ori a0, a0, 1
+; RV64I-NEXT:    ret
+  %2 = lshr i32 %0, 3
+  %3 = or i32 %2, 1
+  ret i32 %3
+}
+
 define i32 @srai(i32 %a) nounwind {
 ; RV32I-LABEL: srai:
 ; RV32I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/epi-full-vector-fabs.ll b/llvm/test/CodeGen/RISCV/epi-full-vector-fabs.ll
index 9f397a24c9f55b498b2a04e8fba55eed91e7861e..37ab7852a17149ca4207b82e7319e843fb9b25b2 100644
--- a/llvm/test/CodeGen/RISCV/epi-full-vector-fabs.ll
+++ b/llvm/test/CodeGen/RISCV/epi-full-vector-fabs.ll
@@ -7,7 +7,7 @@ define <vscale x 1 x double> @nxv1f64(<vscale x 1 x double> %a) nounwind {
 ; CHECK-LABEL: nxv1f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %res = call <vscale x 1 x double> @llvm.fabs.nxv1f64(<vscale x 1 x double> %a)
   ret <vscale x 1 x double> %res
@@ -17,7 +17,7 @@ define <vscale x 2 x float> @nxv2f32(<vscale x 2 x float> %a) nounwind {
 ; CHECK-LABEL: nxv2f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float> %a)
   ret <vscale x 2 x float> %res
@@ -27,7 +27,7 @@ define <vscale x 2 x double> @nxv2f64(<vscale x 2 x double> %a) nounwind {
 ; CHECK-LABEL: nxv2f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> %a)
   ret <vscale x 2 x double> %res
@@ -37,7 +37,7 @@ define <vscale x 4 x float> @nxv4f32(<vscale x 4 x float> %a) nounwind {
 ; CHECK-LABEL: nxv4f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> %a)
   ret <vscale x 4 x float> %res
@@ -47,7 +47,7 @@ define <vscale x 4 x double> @nxv4f64(<vscale x 4 x double> %a) nounwind {
 ; CHECK-LABEL: nxv4f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x double> @llvm.fabs.nxv4f64(<vscale x 4 x double> %a)
   ret <vscale x 4 x double> %res
@@ -57,7 +57,7 @@ define <vscale x 8 x float> @nxv8f32(<vscale x 8 x float> %a) nounwind {
 ; CHECK-LABEL: nxv8f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x float> @llvm.fabs.nxv8f32(<vscale x 8 x float> %a)
   ret <vscale x 8 x float> %res
@@ -67,7 +67,7 @@ define <vscale x 8 x double> @nxv8f64(<vscale x 8 x double> %a) nounwind {
 ; CHECK-LABEL: nxv8f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x double> @llvm.fabs.nxv8f64(<vscale x 8 x double> %a)
   ret <vscale x 8 x double> %res
@@ -77,7 +77,7 @@ define <vscale x 16 x float> @nxv16f32(<vscale x 16 x float> %a) nounwind {
 ; CHECK-LABEL: nxv16f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %res = call <vscale x 16 x float> @llvm.fabs.nxv16f32(<vscale x 16 x float> %a)
   ret <vscale x 16 x float> %res
diff --git a/llvm/test/CodeGen/RISCV/epi-masked-gather.ll b/llvm/test/CodeGen/RISCV/epi-masked-gather.ll
index 1a0955ad17e4dcfc09e73dfd24155193721a32e4..9fb2f32bce71f0a1787bede5a24f4c06ea0c1d67 100644
--- a/llvm/test/CodeGen/RISCV/epi-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/epi-masked-gather.ll
@@ -7,7 +7,7 @@ define <vscale x 1 x i64> @nxv1i64(i64* %ptr, <vscale x 1 x i64> %indices, <vsca
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vsll.vi v25, v8, 3
-; CHECK-NEXT:    vluxei64.v v9, (a0), v25, v0.t
+; CHECK-NEXT:    vloxei64.v v9, (a0), v25, v0.t
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
   %1 = getelementptr i64, i64* %ptr, <vscale x 1 x i64> %indices
@@ -40,7 +40,7 @@ define <vscale x 8 x double> @nxv8f64(double* %ptr, <vscale x 8 x i64> %indices,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vsll.vi v8, v8, 3
-; CHECK-NEXT:    vluxei64.v v16, (a0), v8, v0.t
+; CHECK-NEXT:    vloxei64.v v16, (a0), v8, v0.t
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
   %1 = getelementptr double, double* %ptr, <vscale x 8 x i64> %indices
@@ -53,8 +53,8 @@ define <vscale x 2 x i32> @nxv2xi32(i32* %ptr, <vscale x 2 x i64> %indices, <vsc
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vsll.vi v26, v8, 2
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei64.v v10, (a0), v26, v0.t
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vloxei64.v v10, (a0), v26, v0.t
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
   %1 = getelementptr i32, i32* %ptr, <vscale x 2 x i64> %indices
@@ -65,8 +65,8 @@ define <vscale x 2 x i32> @nxv2xi32(i32* %ptr, <vscale x 2 x i64> %indices, <vsc
 define <vscale x 2 x i32> @nxv2xi32_full(<vscale x 2 x i32*> %ptr, <vscale x 2 x i1> %mask, <vscale x 2 x i32> %passthru) {
 ; CHECK-LABEL: nxv2xi32_full:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei64.v v10, (zero), v8, v0.t
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vloxei64.v v10, (zero), v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/epi-masked-scatter.ll b/llvm/test/CodeGen/RISCV/epi-masked-scatter.ll
index 70256066898566fffe3fb68736a07f5ca8b8fd3a..d10a70b856503ba4d76c048312e2393df638b758 100644
--- a/llvm/test/CodeGen/RISCV/epi-masked-scatter.ll
+++ b/llvm/test/CodeGen/RISCV/epi-masked-scatter.ll
@@ -7,7 +7,7 @@ define void @nxv1i64(<vscale x 1 x i64> %data, i64* %ptr, <vscale x 1 x i64> %in
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vsll.vi v25, v9, 3
-; CHECK-NEXT:    vsuxei64.v v8, (a0), v25, v0.t
+; CHECK-NEXT:    vsoxei64.v v8, (a0), v25, v0.t
 ; CHECK-NEXT:    ret
   %1 = getelementptr i64, i64* %ptr, <vscale x 1 x i64> %indices
   call void @llvm.masked.scatter.nxv1i64.nxv1p0i64(<vscale x 1 x i64> %data, <vscale x 1 x i64*> %1, i32 8, <vscale x 1 x i1> %mask)
@@ -39,7 +39,7 @@ define void @nxv8f64(<vscale x 8 x double> %data, double* %ptr, <vscale x 8 x i6
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vsll.vi v16, v16, 3
-; CHECK-NEXT:    vsuxei64.v v8, (a0), v16, v0.t
+; CHECK-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; CHECK-NEXT:    ret
   %1 = getelementptr double, double* %ptr, <vscale x 8 x i64> %indices
   call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %data, <vscale x 8 x double*> %1, i32 8, <vscale x 8 x i1> %mask)
@@ -52,7 +52,7 @@ define void @nxv2i32(<vscale x 2 x i32> %data, i32* %ptr, <vscale x 2 x i64> %in
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vsll.vi v26, v10, 2
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vsuxei64.v v8, (a0), v26, v0.t
+; CHECK-NEXT:    vsoxei64.v v8, (a0), v26, v0.t
 ; CHECK-NEXT:    ret
   %1 = getelementptr i32, i32* %ptr, <vscale x 2 x i64> %indices
   call void @llvm.masked.scatter.nxv2i32.nxv2p0i32(<vscale x 2 x i32> %data, <vscale x 2 x i32*> %1, i32 4, <vscale x 2 x i1> %mask)
@@ -63,7 +63,7 @@ define void @nxv2i32_full(<vscale x 2 x i32> %data, <vscale x 2 x i32*> %ptr, <v
 ; CHECK-LABEL: nxv2i32_full:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vsuxei64.v v8, (zero), v10, v0.t
+; CHECK-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
 ; CHECK-NEXT:    ret
   call void @llvm.masked.scatter.nxv2i32.nxv2p0i32(<vscale x 2 x i32> %data, <vscale x 2 x i32*> %ptr, i32 4, <vscale x 2 x i1> %mask)
   ret void
diff --git a/llvm/test/CodeGen/RISCV/epi-spill-vector-tuple.ll b/llvm/test/CodeGen/RISCV/epi-spill-vector-tuple.ll
index d9b4095a056d151817d03ed2410485273314e2f1..c1599f39ba60ad72cb455c60a8362ca3cff1ae3a 100644
--- a/llvm/test/CodeGen/RISCV/epi-spill-vector-tuple.ll
+++ b/llvm/test/CodeGen/RISCV/epi-spill-vector-tuple.ll
@@ -63,18 +63,18 @@ define dso_local void @bar(double* nocapture %pa, double* nocapture readonly %pb
 ; SPILL-O2-NEXT:    vle64.v v25, (s1)
 ; SPILL-O2-NEXT:    vle64.v v26, (a1)
 ; SPILL-O2-NEXT:    vzip2.vv v0, v25, v26
-; SPILL-O2-NEXT:    addi a1, sp, 8
-; SPILL-O2-NEXT:    rdvlenb a0
-; SPILL-O2-NEXT:    add a0, a1, a0
-; SPILL-O2-NEXT:    vs1r.v v0, (a1)
-; SPILL-O2-NEXT:    vs1r.v v1, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 8
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vs1r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs1r.v v1, (a0) # Unknown-size Folded Spill
 ; SPILL-O2-NEXT:    call foo
 ; SPILL-O2-NEXT:    vsetvli a0, s0, e64,m1,ta,mu
-; SPILL-O2-NEXT:    addi a1, sp, 8
-; SPILL-O2-NEXT:    rdvlenb a0
-; SPILL-O2-NEXT:    add a0, a1, a0
-; SPILL-O2-NEXT:    vl1r.v v0, (a1)
-; SPILL-O2-NEXT:    vl1r.v v1, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 8
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vl1r.v v0, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl1r.v v1, (a0) # Unknown-size Folded Reload
 ; SPILL-O2-NEXT:    vse64.v v0, (s1)
 ; SPILL-O2-NEXT:    vse64.v v1, (s1)
 ; SPILL-O2-NEXT:    csrr a0, vlenb
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-tuple-spill-emergency.ll b/llvm/test/CodeGen/RISCV/epi-vector-tuple-spill-emergency.ll
index 083eea9c65f37146803a8568e073bc6640d25617..352e7834a0de8cca2a1b60b7ace43a1ebe40ff17 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-tuple-spill-emergency.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-tuple-spill-emergency.ll
@@ -117,11 +117,11 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vmv1r.v v25, v31
 ; CHECK-NEXT:    vfnmsac.vv v25, v4, v12
 ; CHECK-NEXT:    vmv1r.v v14, v8
-; CHECK-NEXT:    addi a3, sp, 72
-; CHECK-NEXT:    rdvlenb a2
+; CHECK-NEXT:    addi a2, sp, 72
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    vs1r.v v1, (a2) # Unknown-size Folded Spill
 ; CHECK-NEXT:    add a2, a2, a3
-; CHECK-NEXT:    vs1r.v v1, (a3)
-; CHECK-NEXT:    vs1r.v v2, (a2)
+; CHECK-NEXT:    vs1r.v v2, (a2) # Unknown-size Folded Spill
 ; CHECK-NEXT:    vfnmsac.vv v14, v2, v12
 ; CHECK-NEXT:    vfmacc.vv v31, v4, v12
 ; CHECK-NEXT:    vfmacc.vv v8, v2, v12
@@ -270,13 +270,13 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vtrn.vv v3, v20, v20
 ; CHECK-NEXT:    vtrn.vv v3, v4, v20
 ; CHECK-NEXT:    vfnmsac.vv v16, v2, v3
+; CHECK-NEXT:    addi a5, sp, 72
 ; CHECK-NEXT:    sd a1, 8(sp)
-; CHECK-NEXT:    addi a1, sp, 72
-; CHECK-NEXT:    rdvlenb a5
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    vl1r.v v3, (a5) # Unknown-size Folded Reload
 ; CHECK-NEXT:    add a5, a5, a1
-; CHECK-NEXT:    vl1r.v v3, (a1)
+; CHECK-NEXT:    vl1r.v v4, (a5) # Unknown-size Folded Reload
 ; CHECK-NEXT:    ld a1, 8(sp)
-; CHECK-NEXT:    vl1r.v v4, (a5)
 ; CHECK-NEXT:    vfmul.vv v20, v23, v3
 ; CHECK-NEXT:    vfneg.v v23, v23, v0.t
 ; CHECK-NEXT:    vtrn.vv v1, v23, v23
diff --git a/llvm/test/CodeGen/RISCV/large-stack.ll b/llvm/test/CodeGen/RISCV/large-stack.ll
index e4cf5eb283999e3717c38dd802339cdec12184b7..962d88907ee07cb3843726e0f9595bd2b2c98388 100644
--- a/llvm/test/CodeGen/RISCV/large-stack.ll
+++ b/llvm/test/CodeGen/RISCV/large-stack.ll
@@ -101,7 +101,6 @@ define void @test_emergency_spill_slot(i32 %a) {
 ; RV32I-WITHFP-NEXT:    lui a2, 1048478
 ; RV32I-WITHFP-NEXT:    addi a2, a2, 1388
 ; RV32I-WITHFP-NEXT:    add a2, s0, a2
-; RV32I-WITHFP-NEXT:    mv a2, a2
 ; RV32I-WITHFP-NEXT:    add a1, a2, a1
 ; RV32I-WITHFP-NEXT:    #APP
 ; RV32I-WITHFP-NEXT:    nop
diff --git a/llvm/test/CodeGen/RISCV/mul.ll b/llvm/test/CodeGen/RISCV/mul.ll
index 647004df3f0cf5d898ca34ebb73951f6496bbdf3..a79049663cbad5f9fd39594bb101b6b76150533f 100644
--- a/llvm/test/CodeGen/RISCV/mul.ll
+++ b/llvm/test/CodeGen/RISCV/mul.ll
@@ -293,6 +293,57 @@ define zeroext i32 @mulhu(i32 zeroext %a, i32 zeroext %b) nounwind {
   ret i32 %5
 }
 
+define i32 @mulhsu(i32 %a, i32 %b) nounwind {
+; RV32I-LABEL: mulhsu:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    srai a3, a1, 31
+; RV32I-NEXT:    mv a1, zero
+; RV32I-NEXT:    call __muldi3@plt
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: mulhsu:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    srai a2, a1, 31
+; RV32IM-NEXT:    mulhu a1, a0, a1
+; RV32IM-NEXT:    mul a0, a0, a2
+; RV32IM-NEXT:    add a0, a1, a0
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: mulhsu:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: mulhsu:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    sext.w a1, a1
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = zext i32 %a to i64
+  %2 = sext i32 %b to i64
+  %3 = mul i64 %1, %2
+  %4 = lshr i64 %3, 32
+  %5 = trunc i64 %4 to i32
+  ret i32 %5
+}
+
 define i32 @muli32_p65(i32 %a) nounwind {
 ; RV32I-LABEL: muli32_p65:
 ; RV32I:       # %bb.0:
@@ -993,10 +1044,10 @@ define i128 @muli128_m3840(i128 %a) nounwind {
 ; RV32I-NEXT:    sltu t4, t3, t1
 ; RV32I-NEXT:    sub t0, t2, t0
 ; RV32I-NEXT:    mv a2, t4
-; RV32I-NEXT:    beq a5, a3, .LBB26_2
+; RV32I-NEXT:    beq a5, a3, .LBB27_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    sltu a2, a5, a3
-; RV32I-NEXT:  .LBB26_2:
+; RV32I-NEXT:  .LBB27_2:
 ; RV32I-NEXT:    sub a1, a7, a6
 ; RV32I-NEXT:    sltu a4, a1, a2
 ; RV32I-NEXT:    sub a4, t0, a4
@@ -1091,10 +1142,10 @@ define i128 @muli128_m63(i128 %a) nounwind {
 ; RV32I-NEXT:    slli a5, a3, 6
 ; RV32I-NEXT:    or t2, a5, a1
 ; RV32I-NEXT:    mv t3, a7
-; RV32I-NEXT:    beq a3, t2, .LBB27_2
+; RV32I-NEXT:    beq a3, t2, .LBB28_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    sltu t3, a3, t2
-; RV32I-NEXT:  .LBB27_2:
+; RV32I-NEXT:  .LBB28_2:
 ; RV32I-NEXT:    srli t1, a3, 26
 ; RV32I-NEXT:    slli a1, a4, 6
 ; RV32I-NEXT:    or a1, a1, t1
@@ -1180,3 +1231,79 @@ define i128 @muli128_m63(i128 %a) nounwind {
   %1 = mul i128 %a, -63
   ret i128 %1
 }
+
+define i64 @mulhsu_i64(i64 %a, i64 %b) nounwind {
+; RV32I-LABEL: mulhsu_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -64
+; RV32I-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    srai a4, a3, 31
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw zero, 36(sp)
+; RV32I-NEXT:    sw zero, 32(sp)
+; RV32I-NEXT:    sw a1, 28(sp)
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    addi a0, sp, 40
+; RV32I-NEXT:    addi a1, sp, 24
+; RV32I-NEXT:    addi a2, sp, 8
+; RV32I-NEXT:    sw a4, 16(sp)
+; RV32I-NEXT:    call __multi3@plt
+; RV32I-NEXT:    lw a0, 48(sp)
+; RV32I-NEXT:    lw a1, 52(sp)
+; RV32I-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 64
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: mulhsu_i64:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi sp, sp, -64
+; RV32IM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    srai a4, a3, 31
+; RV32IM-NEXT:    sw a3, 12(sp)
+; RV32IM-NEXT:    sw a2, 8(sp)
+; RV32IM-NEXT:    sw zero, 36(sp)
+; RV32IM-NEXT:    sw zero, 32(sp)
+; RV32IM-NEXT:    sw a1, 28(sp)
+; RV32IM-NEXT:    sw a0, 24(sp)
+; RV32IM-NEXT:    sw a4, 20(sp)
+; RV32IM-NEXT:    addi a0, sp, 40
+; RV32IM-NEXT:    addi a1, sp, 24
+; RV32IM-NEXT:    addi a2, sp, 8
+; RV32IM-NEXT:    sw a4, 16(sp)
+; RV32IM-NEXT:    call __multi3@plt
+; RV32IM-NEXT:    lw a0, 48(sp)
+; RV32IM-NEXT:    lw a1, 52(sp)
+; RV32IM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    addi sp, sp, 64
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: mulhsu_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    srai a3, a1, 63
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __multi3@plt
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: mulhsu_i64:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    srai a2, a1, 63
+; RV64IM-NEXT:    mulhu a1, a0, a1
+; RV64IM-NEXT:    mul a0, a0, a2
+; RV64IM-NEXT:    add a0, a1, a0
+; RV64IM-NEXT:    ret
+  %1 = zext i64 %a to i128
+  %2 = sext i64 %b to i128
+  %3 = mul i128 %1, %2
+  %4 = lshr i128 %3, 64
+  %5 = trunc i128 %4 to i64
+  ret i64 %5
+}
+
diff --git a/llvm/test/CodeGen/RISCV/patchable-function-entry.ll b/llvm/test/CodeGen/RISCV/patchable-function-entry.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9c4eb2c414dfdf9b4025265a6f6baf98f1c90103
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/patchable-function-entry.ll
@@ -0,0 +1,71 @@
+;; Test the function attribute "patchable-function-entry".
+; RUN: llc -mtriple=riscv32 --riscv-no-aliases < %s | FileCheck %s --check-prefixes=CHECK,RV32,NORVC
+; RUN: llc -mtriple=riscv64 --riscv-no-aliases < %s | FileCheck %s --check-prefixes=CHECK,RV64,NORVC
+; RUN: llc -mtriple=riscv32 -mattr=+c --riscv-no-aliases < %s | FileCheck %s --check-prefixes=CHECK,RV32,RVC
+; RUN: llc -mtriple=riscv64 -mattr=+c --riscv-no-aliases < %s | FileCheck %s --check-prefixes=CHECK,RV64,RVC
+
+define void @f0() "patchable-function-entry"="0" {
+; CHECK-LABEL: f0:
+; CHECK-NEXT:  .Lfunc_begin0:
+; CHECK-NOT:     {{addi|c.nop}}
+; NORVC:         jalr zero, 0(ra)
+; RVC:           c.jr ra
+; CHECK-NOT:   .section __patchable_function_entries
+  ret void
+}
+
+define void @f1() "patchable-function-entry"="1" {
+; CHECK-LABEL: f1:
+; CHECK-NEXT: .Lfunc_begin1:
+; NORVC:         addi zero, zero, 0
+; NORVC-NEXT:    jalr zero, 0(ra)
+; RVC:           c.nop
+; RVC-NEXT:      c.jr ra
+; CHECK:       .section __patchable_function_entries,"awo",@progbits,f1{{$}}
+; 32:          .p2align 2
+; 32-NEXT:     .word .Lfunc_begin1
+; 64:          .p2align 3
+; 64-NEXT:     .quad .Lfunc_begin1
+  ret void
+}
+
+$f5 = comdat any
+define void @f5() "patchable-function-entry"="5" comdat {
+; CHECK-LABEL: f5:
+; CHECK-NEXT:  .Lfunc_begin2:
+; NORVC-COUNT-5: addi zero, zero, 0
+; NORVC-NEXT:    jalr zero, 0(ra)
+; RVC-COUNT-5:   c.nop
+; RVC-NEXT:      c.jr ra
+; CHECK:       .section __patchable_function_entries,"aGwo",@progbits,f5,comdat,f5{{$}}
+; RV32:        .p2align 2
+; RV32-NEXT:   .word .Lfunc_begin2
+; RV64:        .p2align 3
+; RV64-NEXT:   .quad .Lfunc_begin2
+  ret void
+}
+
+;; -fpatchable-function-entry=3,2
+;; "patchable-function-prefix" emits data before the function entry label.
+define void @f3_2() "patchable-function-entry"="1" "patchable-function-prefix"="2" {
+; CHECK-LABEL: .type f3_2,@function
+; CHECK-NEXT:  .Ltmp0: # @f3_2
+; NORVC-COUNT-2: addi zero, zero, 0
+; RVC-COUNT-2:   c.nop
+; CHECK-NEXT:  f3_2:
+; CHECK:       # %bb.0:
+; NORVC-NEXT:    addi zero, zero, 0
+; NORVC-NEXT:    addi sp, sp, -16
+; RVC-NEXT:      c.nop
+; RVC-NEXT:      c.addi sp, -16
+;; .size does not include the prefix.
+; CHECK:      .Lfunc_end3:
+; CHECK-NEXT: .size f3_2, .Lfunc_end3-f3_2
+; CHECK:      .section __patchable_function_entries,"awo",@progbits,f3_2{{$}}
+; RV32:       .p2align 2
+; RV32-NEXT:  .word .Ltmp0
+; RV64:       .p2align 3
+; RV64-NEXT:  .quad .Ltmp0
+  %frame = alloca i8, i32 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rv32Zba.ll b/llvm/test/CodeGen/RISCV/rv32zba.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv32Zba.ll
rename to llvm/test/CodeGen/RISCV/rv32zba.ll
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbbp.ll b/llvm/test/CodeGen/RISCV/rv32zbb-zbp.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv32Zbbp.ll
rename to llvm/test/CodeGen/RISCV/rv32zbb-zbp.ll
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbb.ll b/llvm/test/CodeGen/RISCV/rv32zbb.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv32Zbb.ll
rename to llvm/test/CodeGen/RISCV/rv32zbb.ll
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbp.ll b/llvm/test/CodeGen/RISCV/rv32zbp.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv32Zbp.ll
rename to llvm/test/CodeGen/RISCV/rv32zbp.ll
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbs.ll b/llvm/test/CodeGen/RISCV/rv32zbs.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv32Zbs.ll
rename to llvm/test/CodeGen/RISCV/rv32zbs.ll
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbt.ll b/llvm/test/CodeGen/RISCV/rv32zbt.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv32Zbt.ll
rename to llvm/test/CodeGen/RISCV/rv32zbt.ll
diff --git a/llvm/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll b/llvm/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll
index b14ed50315d77487c466c941ef8bcd30caed1091..8c6c5d79de81ce7cb9f5b3f8e5bd48725d4bfcc7 100644
--- a/llvm/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll
+++ b/llvm/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll
@@ -1478,8 +1478,7 @@ define signext i32 @sext_slliw_zext(i32 zeroext %a) nounwind {
 define zeroext i32 @zext_slliw_aext(i32 %a) nounwind {
 ; RV64I-LABEL: zext_slliw_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 7
-; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a0, a0, 39
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, 7
@@ -1489,8 +1488,7 @@ define zeroext i32 @zext_slliw_aext(i32 %a) nounwind {
 define zeroext i32 @zext_slliw_sext(i32 signext %a) nounwind {
 ; RV64I-LABEL: zext_slliw_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 8
-; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a0, a0, 40
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, 8
@@ -1500,8 +1498,7 @@ define zeroext i32 @zext_slliw_sext(i32 signext %a) nounwind {
 define zeroext i32 @zext_slliw_zext(i32 zeroext %a) nounwind {
 ; RV64I-LABEL: zext_slliw_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 9
-; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    slli a0, a0, 41
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, 9
diff --git a/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll b/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll
index c4a4de7681b08c160195df640f125bd1f806092c..682f351478ed8ed21d2875bc64d1cbc1bfe41076 100644
--- a/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll
+++ b/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll
@@ -10,13 +10,11 @@ define signext i32 @addw(i32 signext %s, i32 signext %n, i32 signext %k) nounwin
 ; CHECK-NEXT:    add a2, a2, a1
 ; CHECK-NEXT:    addi a3, a0, 1
 ; CHECK-NEXT:    mul a3, a2, a3
-; CHECK-NEXT:    slli a2, a2, 32
-; CHECK-NEXT:    srli a2, a2, 32
 ; CHECK-NEXT:    sub a1, a1, a0
 ; CHECK-NEXT:    addi a1, a1, -2
 ; CHECK-NEXT:    slli a1, a1, 32
-; CHECK-NEXT:    srli a1, a1, 32
-; CHECK-NEXT:    mul a1, a2, a1
+; CHECK-NEXT:    slli a2, a2, 32
+; CHECK-NEXT:    mulhu a1, a2, a1
 ; CHECK-NEXT:    srli a1, a1, 1
 ; CHECK-NEXT:    add a0, a3, a0
 ; CHECK-NEXT:    addw a0, a0, a1
@@ -57,13 +55,11 @@ define signext i32 @subw(i32 signext %s, i32 signext %n, i32 signext %k) nounwin
 ; CHECK-NEXT:    not a2, a0
 ; CHECK-NEXT:    add a3, a2, a1
 ; CHECK-NEXT:    mul a2, a3, a2
-; CHECK-NEXT:    slli a3, a3, 32
-; CHECK-NEXT:    srli a3, a3, 32
 ; CHECK-NEXT:    sub a1, a1, a0
 ; CHECK-NEXT:    addi a1, a1, -2
 ; CHECK-NEXT:    slli a1, a1, 32
-; CHECK-NEXT:    srli a1, a1, 32
-; CHECK-NEXT:    mul a1, a3, a1
+; CHECK-NEXT:    slli a3, a3, 32
+; CHECK-NEXT:    mulhu a1, a3, a1
 ; CHECK-NEXT:    srli a1, a1, 1
 ; CHECK-NEXT:    sub a0, a2, a0
 ; CHECK-NEXT:    subw a0, a0, a1
diff --git a/llvm/test/CodeGen/RISCV/rv64Zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll
similarity index 92%
rename from llvm/test/CodeGen/RISCV/rv64Zba.ll
rename to llvm/test/CodeGen/RISCV/rv64zba.ll
index 9dcbaae60822fa8d13649fceb31446e77b8ddbed..1f25ef3cef9bb382ceb47df37bea9c81362fef35 100644
--- a/llvm/test/CodeGen/RISCV/rv64Zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -126,6 +126,32 @@ define i64 @zextw_i64(i64 %a) nounwind {
   ret i64 %and
 }
 
+; This makes sure targetShrinkDemandedConstant changes the and immmediate to
+; allow zext.w or slli+srli.
+define i64 @zextw_demandedbits_i64(i64 %0) {
+; RV64I-LABEL: zextw_demandedbits_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ori a0, a0, 1
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: zextw_demandedbits_i64:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    ori a0, a0, 1
+; RV64IB-NEXT:    zext.w a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBA-LABEL: zextw_demandedbits_i64:
+; RV64IBA:       # %bb.0:
+; RV64IBA-NEXT:    ori a0, a0, 1
+; RV64IBA-NEXT:    zext.w a0, a0
+; RV64IBA-NEXT:    ret
+  %2 = and i64 %0, 4294967294
+  %3 = or i64 %2, 1
+  ret i64 %3
+}
+
 define signext i16 @sh1add(i64 %0, i16* %1) {
 ; RV64I-LABEL: sh1add:
 ; RV64I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv64Zbbp.ll b/llvm/test/CodeGen/RISCV/rv64zbb-zbp.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv64Zbbp.ll
rename to llvm/test/CodeGen/RISCV/rv64zbb-zbp.ll
diff --git a/llvm/test/CodeGen/RISCV/rv64Zbb.ll b/llvm/test/CodeGen/RISCV/rv64zbb.ll
similarity index 55%
rename from llvm/test/CodeGen/RISCV/rv64Zbb.ll
rename to llvm/test/CodeGen/RISCV/rv64zbb.ll
index b0f0fb8a5f35c210dbf8fe60b50b59d6b23d222f..a173235d280c8abaff250e712636819a60890b6b 100644
--- a/llvm/test/CodeGen/RISCV/rv64Zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zbb.ll
@@ -94,6 +94,396 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
   ret i32 %1
 }
 
+define signext i32 @log2_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: log2_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    beqz a0, .LBB1_2
+; RV64I-NEXT:  # %bb.1: # %cond.false
+; RV64I-NEXT:    srliw a1, a0, 1
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 8
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    not a0, a0
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    lui a2, 21845
+; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lui a1, 13107
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    and a2, a0, a1
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    add a0, a2, a0
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lui a1, 3855
+; RV64I-NEXT:    addiw a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    lui a1, 4112
+; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    srli a0, a0, 56
+; RV64I-NEXT:    addi a0, a0, -32
+; RV64I-NEXT:    j .LBB1_3
+; RV64I-NEXT:  .LBB1_2:
+; RV64I-NEXT:    addi a0, zero, 32
+; RV64I-NEXT:  .LBB1_3: # %cond.end
+; RV64I-NEXT:    addi a1, zero, 31
+; RV64I-NEXT:    sub a0, a1, a0
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: log2_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    zext.w a0, a0
+; RV64IB-NEXT:    clz a0, a0
+; RV64IB-NEXT:    addi a1, zero, 63
+; RV64IB-NEXT:    sub a0, a1, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBB-LABEL: log2_i32:
+; RV64IBB:       # %bb.0:
+; RV64IBB-NEXT:    slli a0, a0, 32
+; RV64IBB-NEXT:    srli a0, a0, 32
+; RV64IBB-NEXT:    clz a0, a0
+; RV64IBB-NEXT:    addi a1, zero, 63
+; RV64IBB-NEXT:    sub a0, a1, a0
+; RV64IBB-NEXT:    ret
+  %1 = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
+  %2 = sub i32 31, %1
+  ret i32 %2
+}
+
+define signext i32 @log2_ceil_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: log2_ceil_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addiw a2, a0, -1
+; RV64I-NEXT:    addi s0, zero, 32
+; RV64I-NEXT:    addi a1, zero, 32
+; RV64I-NEXT:    beqz a2, .LBB2_2
+; RV64I-NEXT:  # %bb.1: # %cond.false
+; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srliw a1, a0, 1
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 8
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    not a0, a0
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    lui a2, 21845
+; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lui a1, 13107
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    and a2, a0, a1
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    add a0, a2, a0
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lui a1, 3855
+; RV64I-NEXT:    addiw a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    lui a1, 4112
+; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    srli a0, a0, 56
+; RV64I-NEXT:    addi a1, a0, -32
+; RV64I-NEXT:  .LBB2_2: # %cond.end
+; RV64I-NEXT:    sub a0, s0, a1
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: log2_ceil_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    addi a0, a0, -1
+; RV64IB-NEXT:    zext.w a0, a0
+; RV64IB-NEXT:    clz a0, a0
+; RV64IB-NEXT:    addi a1, zero, 64
+; RV64IB-NEXT:    sub a0, a1, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBB-LABEL: log2_ceil_i32:
+; RV64IBB:       # %bb.0:
+; RV64IBB-NEXT:    addi a0, a0, -1
+; RV64IBB-NEXT:    slli a0, a0, 32
+; RV64IBB-NEXT:    srli a0, a0, 32
+; RV64IBB-NEXT:    clz a0, a0
+; RV64IBB-NEXT:    addi a1, zero, 64
+; RV64IBB-NEXT:    sub a0, a1, a0
+; RV64IBB-NEXT:    ret
+  %1 = sub i32 %a, 1
+  %2 = call i32 @llvm.ctlz.i32(i32 %1, i1 false)
+  %3 = sub i32 32, %2
+  ret i32 %3
+}
+
+define signext i32 @findLastSet_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: findLastSet_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    srliw a1, s0, 1
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 8
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    not a0, a0
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    lui a2, 21845
+; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lui a1, 13107
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    and a2, a0, a1
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    add a0, a2, a0
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lui a1, 3855
+; RV64I-NEXT:    addiw a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    lui a1, 4112
+; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    addi a0, zero, -1
+; RV64I-NEXT:    beqz s0, .LBB3_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    srli a0, a1, 56
+; RV64I-NEXT:    addi a0, a0, -32
+; RV64I-NEXT:    xori a0, a0, 31
+; RV64I-NEXT:  .LBB3_2:
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: findLastSet_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    clzw a1, a0
+; RV64IB-NEXT:    xori a1, a1, 31
+; RV64IB-NEXT:    addi a2, zero, -1
+; RV64IB-NEXT:    cmov a0, a0, a1, a2
+; RV64IB-NEXT:    ret
+;
+; RV64IBB-LABEL: findLastSet_i32:
+; RV64IBB:       # %bb.0:
+; RV64IBB-NEXT:    mv a1, a0
+; RV64IBB-NEXT:    addi a0, zero, -1
+; RV64IBB-NEXT:    beqz a1, .LBB3_2
+; RV64IBB-NEXT:  # %bb.1:
+; RV64IBB-NEXT:    clzw a0, a1
+; RV64IBB-NEXT:    xori a0, a0, 31
+; RV64IBB-NEXT:  .LBB3_2:
+; RV64IBB-NEXT:    ret
+  %1 = call i32 @llvm.ctlz.i32(i32 %a, i1 true)
+  %2 = xor i32 31, %1
+  %3 = icmp eq i32 %a, 0
+  %4 = select i1 %3, i32 -1, i32 %2
+  ret i32 %4
+}
+
+define i32 @ctlz_lshr_i32(i32 signext %a) {
+; RV64I-LABEL: ctlz_lshr_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    .cfi_def_cfa_offset 16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    srliw a0, a0, 1
+; RV64I-NEXT:    beqz a0, .LBB4_2
+; RV64I-NEXT:  # %bb.1: # %cond.false
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 8
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 16
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    not a0, a0
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    lui a2, 21845
+; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lui a1, 13107
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    and a2, a0, a1
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    add a0, a2, a0
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lui a1, 3855
+; RV64I-NEXT:    addiw a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    lui a1, 4112
+; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    srli a0, a0, 56
+; RV64I-NEXT:    addi a0, a0, -32
+; RV64I-NEXT:    j .LBB4_3
+; RV64I-NEXT:  .LBB4_2:
+; RV64I-NEXT:    addi a0, zero, 32
+; RV64I-NEXT:  .LBB4_3: # %cond.end
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: ctlz_lshr_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    srliw a0, a0, 1
+; RV64IB-NEXT:    clz a0, a0
+; RV64IB-NEXT:    addi a0, a0, -32
+; RV64IB-NEXT:    ret
+;
+; RV64IBB-LABEL: ctlz_lshr_i32:
+; RV64IBB:       # %bb.0:
+; RV64IBB-NEXT:    srliw a0, a0, 1
+; RV64IBB-NEXT:    clz a0, a0
+; RV64IBB-NEXT:    addi a0, a0, -32
+; RV64IBB-NEXT:    ret
+  %1 = lshr i32 %a, 1
+  %2 = call i32 @llvm.ctlz.i32(i32 %1, i1 false)
+  ret i32 %2
+}
+
 declare i64 @llvm.ctlz.i64(i64, i1)
 
 define i64 @ctlz_i64(i64 %a) nounwind {
@@ -101,7 +491,7 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    beqz a0, .LBB1_2
+; RV64I-NEXT:    beqz a0, .LBB5_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    or a0, a0, a1
@@ -158,10 +548,10 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    j .LBB1_3
-; RV64I-NEXT:  .LBB1_2:
+; RV64I-NEXT:    j .LBB5_3
+; RV64I-NEXT:  .LBB5_2:
 ; RV64I-NEXT:    addi a0, zero, 64
-; RV64I-NEXT:  .LBB1_3: # %cond.end
+; RV64I-NEXT:  .LBB5_3: # %cond.end
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -186,7 +576,7 @@ define signext i32 @cttz_i32(i32 signext %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    beqz a0, .LBB2_2
+; RV64I-NEXT:    beqz a0, .LBB6_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
 ; RV64I-NEXT:    addi a1, a0, -1
 ; RV64I-NEXT:    not a0, a0
@@ -233,10 +623,10 @@ define signext i32 @cttz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    j .LBB2_3
-; RV64I-NEXT:  .LBB2_2:
+; RV64I-NEXT:    j .LBB6_3
+; RV64I-NEXT:  .LBB6_2:
 ; RV64I-NEXT:    addi a0, zero, 32
-; RV64I-NEXT:  .LBB2_3: # %cond.end
+; RV64I-NEXT:  .LBB6_3: # %cond.end
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -254,6 +644,244 @@ define signext i32 @cttz_i32(i32 signext %a) nounwind {
   ret i32 %1
 }
 
+define signext i32 @cttz_zero_undef_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: cttz_zero_undef_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi a1, a0, -1
+; RV64I-NEXT:    not a0, a0
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    lui a2, 21845
+; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lui a1, 13107
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    and a2, a0, a1
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    add a0, a2, a0
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lui a1, 3855
+; RV64I-NEXT:    addiw a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    lui a1, 4112
+; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    srli a0, a0, 56
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: cttz_zero_undef_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    ctz a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBB-LABEL: cttz_zero_undef_i32:
+; RV64IBB:       # %bb.0:
+; RV64IBB-NEXT:    ctz a0, a0
+; RV64IBB-NEXT:    ret
+  %1 = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+  ret i32 %1
+}
+
+define signext i32 @findFirstSet_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: findFirstSet_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    not a1, s0
+; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    lui a2, 21845
+; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lui a1, 13107
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    and a2, a0, a1
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    add a0, a2, a0
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lui a1, 3855
+; RV64I-NEXT:    addiw a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    lui a1, 4112
+; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    addi a0, zero, -1
+; RV64I-NEXT:    beqz s0, .LBB8_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    srli a0, a1, 56
+; RV64I-NEXT:  .LBB8_2:
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: findFirstSet_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    ctz a1, a0
+; RV64IB-NEXT:    addi a2, zero, -1
+; RV64IB-NEXT:    cmov a0, a0, a1, a2
+; RV64IB-NEXT:    ret
+;
+; RV64IBB-LABEL: findFirstSet_i32:
+; RV64IBB:       # %bb.0:
+; RV64IBB-NEXT:    mv a1, a0
+; RV64IBB-NEXT:    addi a0, zero, -1
+; RV64IBB-NEXT:    beqz a1, .LBB8_2
+; RV64IBB-NEXT:  # %bb.1:
+; RV64IBB-NEXT:    ctz a0, a1
+; RV64IBB-NEXT:  .LBB8_2:
+; RV64IBB-NEXT:    ret
+  %1 = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+  %2 = icmp eq i32 %a, 0
+  %3 = select i1 %2, i32 -1, i32 %1
+  ret i32 %3
+}
+
+define signext i32 @ffs_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: ffs_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    not a1, s0
+; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    srli a1, a0, 1
+; RV64I-NEXT:    lui a2, 21845
+; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lui a1, 13107
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    and a2, a0, a1
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    add a0, a2, a0
+; RV64I-NEXT:    srli a1, a0, 4
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lui a1, 3855
+; RV64I-NEXT:    addiw a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    lui a1, 4112
+; RV64I-NEXT:    addiw a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    addi a1, a1, 257
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    mv a0, zero
+; RV64I-NEXT:    beqz s0, .LBB9_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    srli a0, a1, 56
+; RV64I-NEXT:    addi a0, a0, 1
+; RV64I-NEXT:  .LBB9_2:
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: ffs_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    ctz a1, a0
+; RV64IB-NEXT:    addi a1, a1, 1
+; RV64IB-NEXT:    cmov a0, a0, a1, zero
+; RV64IB-NEXT:    ret
+;
+; RV64IBB-LABEL: ffs_i32:
+; RV64IBB:       # %bb.0:
+; RV64IBB-NEXT:    mv a1, a0
+; RV64IBB-NEXT:    mv a0, zero
+; RV64IBB-NEXT:    beqz a1, .LBB9_2
+; RV64IBB-NEXT:  # %bb.1:
+; RV64IBB-NEXT:    ctz a0, a1
+; RV64IBB-NEXT:    addi a0, a0, 1
+; RV64IBB-NEXT:  .LBB9_2:
+; RV64IBB-NEXT:    ret
+  %1 = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+  %2 = add i32 %1, 1
+  %3 = icmp eq i32 %a, 0
+  %4 = select i1 %3, i32 0, i32 %2
+  ret i32 %4
+}
+
 declare i64 @llvm.cttz.i64(i64, i1)
 
 define i64 @cttz_i64(i64 %a) nounwind {
@@ -261,7 +889,7 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    beqz a0, .LBB3_2
+; RV64I-NEXT:    beqz a0, .LBB10_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
 ; RV64I-NEXT:    addi a1, a0, -1
 ; RV64I-NEXT:    not a0, a0
@@ -308,10 +936,10 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    j .LBB3_3
-; RV64I-NEXT:  .LBB3_2:
+; RV64I-NEXT:    j .LBB10_3
+; RV64I-NEXT:  .LBB10_2:
 ; RV64I-NEXT:    addi a0, zero, 64
-; RV64I-NEXT:  .LBB3_3: # %cond.end
+; RV64I-NEXT:  .LBB10_3: # %cond.end
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -544,10 +1172,10 @@ define i64 @sexth_i64(i64 %a) nounwind {
 define signext i32 @min_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: min_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a0, a1, .LBB10_2
+; RV64I-NEXT:    blt a0, a1, .LBB17_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB10_2:
+; RV64I-NEXT:  .LBB17_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: min_i32:
@@ -567,10 +1195,10 @@ define signext i32 @min_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @min_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: min_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a0, a1, .LBB11_2
+; RV64I-NEXT:    blt a0, a1, .LBB18_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB11_2:
+; RV64I-NEXT:  .LBB18_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: min_i64:
@@ -590,10 +1218,10 @@ define i64 @min_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @max_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: max_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a1, a0, .LBB12_2
+; RV64I-NEXT:    blt a1, a0, .LBB19_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB12_2:
+; RV64I-NEXT:  .LBB19_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: max_i32:
@@ -613,10 +1241,10 @@ define signext i32 @max_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @max_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: max_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a1, a0, .LBB13_2
+; RV64I-NEXT:    blt a1, a0, .LBB20_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB13_2:
+; RV64I-NEXT:  .LBB20_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: max_i64:
@@ -636,10 +1264,10 @@ define i64 @max_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @minu_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: minu_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a0, a1, .LBB14_2
+; RV64I-NEXT:    bltu a0, a1, .LBB21_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB14_2:
+; RV64I-NEXT:  .LBB21_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: minu_i32:
@@ -659,10 +1287,10 @@ define signext i32 @minu_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @minu_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: minu_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a0, a1, .LBB15_2
+; RV64I-NEXT:    bltu a0, a1, .LBB22_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB15_2:
+; RV64I-NEXT:  .LBB22_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: minu_i64:
@@ -682,10 +1310,10 @@ define i64 @minu_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @maxu_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: maxu_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a1, a0, .LBB16_2
+; RV64I-NEXT:    bltu a1, a0, .LBB23_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB16_2:
+; RV64I-NEXT:  .LBB23_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: maxu_i32:
@@ -705,10 +1333,10 @@ define signext i32 @maxu_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @maxu_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: maxu_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a1, a0, .LBB17_2
+; RV64I-NEXT:    bltu a1, a0, .LBB24_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB17_2:
+; RV64I-NEXT:  .LBB24_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: maxu_i64:
diff --git a/llvm/test/CodeGen/RISCV/rv64Zbp.ll b/llvm/test/CodeGen/RISCV/rv64zbp.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv64Zbp.ll
rename to llvm/test/CodeGen/RISCV/rv64zbp.ll
diff --git a/llvm/test/CodeGen/RISCV/rv64Zbs.ll b/llvm/test/CodeGen/RISCV/rv64zbs.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv64Zbs.ll
rename to llvm/test/CodeGen/RISCV/rv64zbs.ll
diff --git a/llvm/test/CodeGen/RISCV/rv64Zbt.ll b/llvm/test/CodeGen/RISCV/rv64zbt.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rv64Zbt.ll
rename to llvm/test/CodeGen/RISCV/rv64zbt.ll
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll
new file mode 100644
index 0000000000000000000000000000000000000000..53fe40a707b50d118dfa184e44298b4f90b1c2bd
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 < %s | FileCheck %s
+
+define <32 x i1> @bitcast_v4i8_v32i1(<4 x i8> %a, <32 x i1> %b) {
+; CHECK-LABEL: bitcast_v4i8_v32i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsetvli a0, a0, e8,m2,ta,mu
+; CHECK-NEXT:    vmxor.mm v0, v0, v8
+; CHECK-NEXT:    ret
+  %c = bitcast <4 x i8> %a to <32 x i1>
+  %d = xor <32 x i1> %b, %c
+  ret <32 x i1> %d
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll
index c3ea32110c288faf3bb0ef68914c1cedf8480631..b56fe797f5c24fdf678e556f1bf0907bc5da69e4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse.ll
@@ -369,96 +369,67 @@ define void @bitreverse_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV32:       # %bb.0:
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_0)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_0)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v27, v25, v26
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_1)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_1)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v28, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v29, v25, v28
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_2)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_2)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v30, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v29, v29, v30
-; LMULMAX2-RV32-NEXT:    vor.vv v27, v29, v27
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_3)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_3)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v29, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v30, v25, v29
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_4)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_4)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v31, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v30, v30, v31
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_5)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_5)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v31, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v8, v25, v31
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_6)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_6)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v9, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v8, v8, v9
-; LMULMAX2-RV32-NEXT:    vor.vv v30, v8, v30
-; LMULMAX2-RV32-NEXT:    vor.vv v27, v30, v27
-; LMULMAX2-RV32-NEXT:    vsll.vv v30, v25, v31
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_7)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_7)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v31, (a1)
+; LMULMAX2-RV32-NEXT:    addi a6, zero, 56
+; LMULMAX2-RV32-NEXT:    vsrl.vx v26, v25, a6
+; LMULMAX2-RV32-NEXT:    addi a2, zero, 40
+; LMULMAX2-RV32-NEXT:    vsrl.vx v27, v25, a2
+; LMULMAX2-RV32-NEXT:    lui a3, 16
+; LMULMAX2-RV32-NEXT:    addi a3, a3, -256
+; LMULMAX2-RV32-NEXT:    vand.vx v27, v27, a3
+; LMULMAX2-RV32-NEXT:    vor.vv v26, v27, v26
+; LMULMAX2-RV32-NEXT:    addi a4, zero, 5
+; LMULMAX2-RV32-NEXT:    vsetivli a5, 1, e8,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a4
+; LMULMAX2-RV32-NEXT:    vsetivli a4, 4, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.i v27, 0
+; LMULMAX2-RV32-NEXT:    lui a4, 1044480
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v27, v27, a4, v0
+; LMULMAX2-RV32-NEXT:    vsetivli a4, 2, e64,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vsrl.vi v28, v25, 8
+; LMULMAX2-RV32-NEXT:    vand.vv v27, v28, v27
+; LMULMAX2-RV32-NEXT:    vsrl.vi v28, v25, 24
+; LMULMAX2-RV32-NEXT:    lui a4, 4080
+; LMULMAX2-RV32-NEXT:    vand.vx v28, v28, a4
+; LMULMAX2-RV32-NEXT:    vor.vv v27, v27, v28
+; LMULMAX2-RV32-NEXT:    vor.vv v26, v27, v26
+; LMULMAX2-RV32-NEXT:    addi a5, zero, 255
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v27, a5
+; LMULMAX2-RV32-NEXT:    vmerge.vim v27, v27, 0, v0
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v30, v30, v31
-; LMULMAX2-RV32-NEXT:    vsll.vv v29, v25, v29
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_8)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_8)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v31, (a1)
+; LMULMAX2-RV32-NEXT:    vsll.vi v28, v25, 8
+; LMULMAX2-RV32-NEXT:    vand.vv v27, v28, v27
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a3
+; LMULMAX2-RV32-NEXT:    vmerge.vim v28, v28, 0, v0
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v29, v29, v31
-; LMULMAX2-RV32-NEXT:    vor.vv v29, v29, v30
-; LMULMAX2-RV32-NEXT:    vsll.vv v26, v25, v26
-; LMULMAX2-RV32-NEXT:    vsll.vv v25, v25, v28
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_9)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_9)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v28, (a1)
+; LMULMAX2-RV32-NEXT:    vsll.vi v29, v25, 24
+; LMULMAX2-RV32-NEXT:    vand.vv v28, v29, v28
+; LMULMAX2-RV32-NEXT:    vor.vv v27, v28, v27
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a4
+; LMULMAX2-RV32-NEXT:    vmerge.vim v28, v28, 0, v0
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v25, v25, v28
-; LMULMAX2-RV32-NEXT:    vor.vv v25, v26, v25
-; LMULMAX2-RV32-NEXT:    vor.vv v25, v25, v29
+; LMULMAX2-RV32-NEXT:    vsll.vx v29, v25, a2
+; LMULMAX2-RV32-NEXT:    vand.vv v28, v29, v28
+; LMULMAX2-RV32-NEXT:    vsll.vx v25, v25, a6
+; LMULMAX2-RV32-NEXT:    vor.vv v25, v25, v28
 ; LMULMAX2-RV32-NEXT:    vor.vv v25, v25, v27
+; LMULMAX2-RV32-NEXT:    vor.vv v25, v25, v26
 ; LMULMAX2-RV32-NEXT:    lui a1, 61681
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, -241
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV32-NEXT:    vand.vv v26, v25, v26
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_10)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_10)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v27, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsll.vv v26, v26, v27
+; LMULMAX2-RV32-NEXT:    vsll.vi v26, v26, 4
 ; LMULMAX2-RV32-NEXT:    lui a1, 986895
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, 240
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
+; LMULMAX2-RV32-NEXT:    vmv.v.x v27, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v25, v25, v28
-; LMULMAX2-RV32-NEXT:    vsrl.vv v25, v25, v27
+; LMULMAX2-RV32-NEXT:    vand.vv v25, v25, v27
+; LMULMAX2-RV32-NEXT:    vsrl.vi v25, v25, 4
 ; LMULMAX2-RV32-NEXT:    vor.vv v25, v25, v26
 ; LMULMAX2-RV32-NEXT:    lui a1, 209715
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, 819
@@ -466,19 +437,14 @@ define void @bitreverse_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV32-NEXT:    vand.vv v26, v25, v26
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_11)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_11)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v27, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsll.vv v26, v26, v27
+; LMULMAX2-RV32-NEXT:    vsll.vi v26, v26, 2
 ; LMULMAX2-RV32-NEXT:    lui a1, 838861
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, -820
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
+; LMULMAX2-RV32-NEXT:    vmv.v.x v27, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v25, v25, v28
-; LMULMAX2-RV32-NEXT:    vsrl.vv v25, v25, v27
+; LMULMAX2-RV32-NEXT:    vand.vv v25, v25, v27
+; LMULMAX2-RV32-NEXT:    vsrl.vi v25, v25, 2
 ; LMULMAX2-RV32-NEXT:    vor.vv v25, v25, v26
 ; LMULMAX2-RV32-NEXT:    lui a1, 349525
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, 1365
@@ -486,19 +452,14 @@ define void @bitreverse_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV32-NEXT:    vand.vv v26, v25, v26
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI2_12)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_12)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v27, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsll.vv v26, v26, v27
+; LMULMAX2-RV32-NEXT:    vsll.vi v26, v26, 1
 ; LMULMAX2-RV32-NEXT:    lui a1, 699051
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, -1366
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
+; LMULMAX2-RV32-NEXT:    vmv.v.x v27, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v25, v25, v28
-; LMULMAX2-RV32-NEXT:    vsrl.vv v25, v25, v27
+; LMULMAX2-RV32-NEXT:    vand.vv v25, v25, v27
+; LMULMAX2-RV32-NEXT:    vsrl.vi v25, v25, 1
 ; LMULMAX2-RV32-NEXT:    vor.vv v25, v25, v26
 ; LMULMAX2-RV32-NEXT:    vse64.v v25, (a0)
 ; LMULMAX2-RV32-NEXT:    ret
@@ -608,96 +569,67 @@ define void @bitreverse_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV32:       # %bb.0:
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v27, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_1)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_1)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v28, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v29, v25, v28
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_2)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v30, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v29, v29, v30
-; LMULMAX1-RV32-NEXT:    vor.vv v27, v29, v27
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_3)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_3)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v29, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v30, v25, v29
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_4)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_4)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v31, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v30, v30, v31
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_5)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_5)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v31, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v8, v25, v31
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_6)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_6)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v9, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v8, v8, v9
-; LMULMAX1-RV32-NEXT:    vor.vv v30, v8, v30
-; LMULMAX1-RV32-NEXT:    vor.vv v27, v30, v27
-; LMULMAX1-RV32-NEXT:    vsll.vv v30, v25, v31
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_7)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_7)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v31, (a1)
+; LMULMAX1-RV32-NEXT:    addi a6, zero, 56
+; LMULMAX1-RV32-NEXT:    vsrl.vx v26, v25, a6
+; LMULMAX1-RV32-NEXT:    addi a2, zero, 40
+; LMULMAX1-RV32-NEXT:    vsrl.vx v27, v25, a2
+; LMULMAX1-RV32-NEXT:    lui a3, 16
+; LMULMAX1-RV32-NEXT:    addi a3, a3, -256
+; LMULMAX1-RV32-NEXT:    vand.vx v27, v27, a3
+; LMULMAX1-RV32-NEXT:    vor.vv v26, v27, v26
+; LMULMAX1-RV32-NEXT:    addi a4, zero, 5
+; LMULMAX1-RV32-NEXT:    vsetivli a5, 1, e8,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.s.x v0, a4
+; LMULMAX1-RV32-NEXT:    vsetivli a4, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.i v27, 0
+; LMULMAX1-RV32-NEXT:    lui a4, 1044480
+; LMULMAX1-RV32-NEXT:    vmerge.vxm v27, v27, a4, v0
+; LMULMAX1-RV32-NEXT:    vsetivli a4, 2, e64,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vsrl.vi v28, v25, 8
+; LMULMAX1-RV32-NEXT:    vand.vv v27, v28, v27
+; LMULMAX1-RV32-NEXT:    vsrl.vi v28, v25, 24
+; LMULMAX1-RV32-NEXT:    lui a4, 4080
+; LMULMAX1-RV32-NEXT:    vand.vx v28, v28, a4
+; LMULMAX1-RV32-NEXT:    vor.vv v27, v27, v28
+; LMULMAX1-RV32-NEXT:    vor.vv v26, v27, v26
+; LMULMAX1-RV32-NEXT:    addi a5, zero, 255
+; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.x v27, a5
+; LMULMAX1-RV32-NEXT:    vmerge.vim v27, v27, 0, v0
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v30, v30, v31
-; LMULMAX1-RV32-NEXT:    vsll.vv v29, v25, v29
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_8)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_8)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v31, (a1)
+; LMULMAX1-RV32-NEXT:    vsll.vi v28, v25, 8
+; LMULMAX1-RV32-NEXT:    vand.vv v27, v28, v27
+; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.x v28, a3
+; LMULMAX1-RV32-NEXT:    vmerge.vim v28, v28, 0, v0
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v29, v29, v31
-; LMULMAX1-RV32-NEXT:    vor.vv v29, v29, v30
-; LMULMAX1-RV32-NEXT:    vsll.vv v26, v25, v26
-; LMULMAX1-RV32-NEXT:    vsll.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_9)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_9)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v28, (a1)
+; LMULMAX1-RV32-NEXT:    vsll.vi v29, v25, 24
+; LMULMAX1-RV32-NEXT:    vand.vv v28, v29, v28
+; LMULMAX1-RV32-NEXT:    vor.vv v27, v28, v27
+; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.x v28, a4
+; LMULMAX1-RV32-NEXT:    vmerge.vim v28, v28, 0, v0
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    vor.vv v25, v26, v25
-; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v29
+; LMULMAX1-RV32-NEXT:    vsll.vx v29, v25, a2
+; LMULMAX1-RV32-NEXT:    vand.vv v28, v29, v28
+; LMULMAX1-RV32-NEXT:    vsll.vx v25, v25, a6
+; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v28
 ; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
 ; LMULMAX1-RV32-NEXT:    lui a1, 61681
 ; LMULMAX1-RV32-NEXT:    addi a1, a1, -241
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vand.vv v26, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_10)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_10)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v27, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsll.vv v26, v26, v27
+; LMULMAX1-RV32-NEXT:    vsll.vi v26, v26, 4
 ; LMULMAX1-RV32-NEXT:    lui a1, 986895
 ; LMULMAX1-RV32-NEXT:    addi a1, a1, 240
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v28, a1
+; LMULMAX1-RV32-NEXT:    vmv.v.x v27, a1
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 4
 ; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
 ; LMULMAX1-RV32-NEXT:    lui a1, 209715
 ; LMULMAX1-RV32-NEXT:    addi a1, a1, 819
@@ -705,19 +637,14 @@ define void @bitreverse_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vand.vv v26, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_11)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_11)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v27, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsll.vv v26, v26, v27
+; LMULMAX1-RV32-NEXT:    vsll.vi v26, v26, 2
 ; LMULMAX1-RV32-NEXT:    lui a1, 838861
 ; LMULMAX1-RV32-NEXT:    addi a1, a1, -820
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v28, a1
+; LMULMAX1-RV32-NEXT:    vmv.v.x v27, a1
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 2
 ; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
 ; LMULMAX1-RV32-NEXT:    lui a1, 349525
 ; LMULMAX1-RV32-NEXT:    addi a1, a1, 1365
@@ -725,19 +652,14 @@ define void @bitreverse_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vand.vv v26, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI2_12)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI2_12)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v27, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsll.vv v26, v26, v27
+; LMULMAX1-RV32-NEXT:    vsll.vi v26, v26, 1
 ; LMULMAX1-RV32-NEXT:    lui a1, 699051
 ; LMULMAX1-RV32-NEXT:    addi a1, a1, -1366
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v28, a1
+; LMULMAX1-RV32-NEXT:    vmv.v.x v27, a1
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 1
 ; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
 ; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
 ; LMULMAX1-RV32-NEXT:    ret
@@ -1311,96 +1233,67 @@ define void @bitreverse_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX2-RV32:       # %bb.0:
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vle64.v v26, (a0)
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_0)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_0)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v28, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v8, v26, v28
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_1)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_1)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v30, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v10, v26, v30
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_2)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_2)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v12, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v10, v10, v12
-; LMULMAX2-RV32-NEXT:    vor.vv v10, v10, v8
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_3)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_3)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v8, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v12, v26, v8
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_4)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_4)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v14, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v12, v12, v14
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_5)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_5)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v14, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v16, v26, v14
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_6)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_6)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v18, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v16, v16, v18
-; LMULMAX2-RV32-NEXT:    vor.vv v12, v16, v12
-; LMULMAX2-RV32-NEXT:    vor.vv v10, v12, v10
-; LMULMAX2-RV32-NEXT:    vsll.vv v12, v26, v14
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_7)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_7)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v14, (a1)
+; LMULMAX2-RV32-NEXT:    addi a6, zero, 56
+; LMULMAX2-RV32-NEXT:    vsrl.vx v28, v26, a6
+; LMULMAX2-RV32-NEXT:    addi a2, zero, 40
+; LMULMAX2-RV32-NEXT:    vsrl.vx v30, v26, a2
+; LMULMAX2-RV32-NEXT:    lui a3, 16
+; LMULMAX2-RV32-NEXT:    addi a3, a3, -256
+; LMULMAX2-RV32-NEXT:    vand.vx v30, v30, a3
+; LMULMAX2-RV32-NEXT:    vor.vv v28, v30, v28
+; LMULMAX2-RV32-NEXT:    addi a4, zero, 85
+; LMULMAX2-RV32-NEXT:    vsetivli a5, 1, e8,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a4
+; LMULMAX2-RV32-NEXT:    vsetivli a4, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.i v30, 0
+; LMULMAX2-RV32-NEXT:    lui a4, 1044480
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v30, v30, a4, v0
+; LMULMAX2-RV32-NEXT:    vsetivli a4, 4, e64,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vsrl.vi v8, v26, 8
+; LMULMAX2-RV32-NEXT:    vand.vv v30, v8, v30
+; LMULMAX2-RV32-NEXT:    vsrl.vi v8, v26, 24
+; LMULMAX2-RV32-NEXT:    lui a4, 4080
+; LMULMAX2-RV32-NEXT:    vand.vx v8, v8, a4
+; LMULMAX2-RV32-NEXT:    vor.vv v30, v30, v8
+; LMULMAX2-RV32-NEXT:    vor.vv v28, v30, v28
+; LMULMAX2-RV32-NEXT:    addi a5, zero, 255
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v30, a5
+; LMULMAX2-RV32-NEXT:    vmerge.vim v30, v30, 0, v0
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v12, v12, v14
-; LMULMAX2-RV32-NEXT:    vsll.vv v8, v26, v8
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_8)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_8)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v14, (a1)
+; LMULMAX2-RV32-NEXT:    vsll.vi v8, v26, 8
+; LMULMAX2-RV32-NEXT:    vand.vv v30, v8, v30
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v8, a3
+; LMULMAX2-RV32-NEXT:    vmerge.vim v8, v8, 0, v0
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v8, v8, v14
-; LMULMAX2-RV32-NEXT:    vor.vv v8, v8, v12
-; LMULMAX2-RV32-NEXT:    vsll.vv v28, v26, v28
-; LMULMAX2-RV32-NEXT:    vsll.vv v26, v26, v30
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_9)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_9)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v30, (a1)
+; LMULMAX2-RV32-NEXT:    vsll.vi v10, v26, 24
+; LMULMAX2-RV32-NEXT:    vand.vv v8, v10, v8
+; LMULMAX2-RV32-NEXT:    vor.vv v30, v8, v30
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v8, a4
+; LMULMAX2-RV32-NEXT:    vmerge.vim v8, v8, 0, v0
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v26, v26, v30
-; LMULMAX2-RV32-NEXT:    vor.vv v26, v28, v26
+; LMULMAX2-RV32-NEXT:    vsll.vx v10, v26, a2
+; LMULMAX2-RV32-NEXT:    vand.vv v8, v10, v8
+; LMULMAX2-RV32-NEXT:    vsll.vx v26, v26, a6
 ; LMULMAX2-RV32-NEXT:    vor.vv v26, v26, v8
-; LMULMAX2-RV32-NEXT:    vor.vv v26, v26, v10
+; LMULMAX2-RV32-NEXT:    vor.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vor.vv v26, v26, v28
 ; LMULMAX2-RV32-NEXT:    lui a1, 61681
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, -241
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vand.vv v28, v26, v28
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_10)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_10)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v30, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsll.vv v28, v28, v30
+; LMULMAX2-RV32-NEXT:    vsll.vi v28, v28, 4
 ; LMULMAX2-RV32-NEXT:    lui a1, 986895
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, 240
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vmv.v.x v8, a1
+; LMULMAX2-RV32-NEXT:    vmv.v.x v30, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v26, v26, v8
-; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vand.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vsrl.vi v26, v26, 4
 ; LMULMAX2-RV32-NEXT:    vor.vv v26, v26, v28
 ; LMULMAX2-RV32-NEXT:    lui a1, 209715
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, 819
@@ -1408,19 +1301,14 @@ define void @bitreverse_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vand.vv v28, v26, v28
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_11)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_11)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v30, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsll.vv v28, v28, v30
+; LMULMAX2-RV32-NEXT:    vsll.vi v28, v28, 2
 ; LMULMAX2-RV32-NEXT:    lui a1, 838861
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, -820
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vmv.v.x v8, a1
+; LMULMAX2-RV32-NEXT:    vmv.v.x v30, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v26, v26, v8
-; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vand.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vsrl.vi v26, v26, 2
 ; LMULMAX2-RV32-NEXT:    vor.vv v26, v26, v28
 ; LMULMAX2-RV32-NEXT:    lui a1, 349525
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, 1365
@@ -1428,19 +1316,14 @@ define void @bitreverse_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vand.vv v28, v26, v28
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI5_12)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI5_12)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v30, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsll.vv v28, v28, v30
+; LMULMAX2-RV32-NEXT:    vsll.vi v28, v28, 1
 ; LMULMAX2-RV32-NEXT:    lui a1, 699051
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, -1366
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vmv.v.x v8, a1
+; LMULMAX2-RV32-NEXT:    vmv.v.x v30, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vand.vv v26, v26, v8
-; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vand.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vsrl.vi v26, v26, 1
 ; LMULMAX2-RV32-NEXT:    vor.vv v26, v26, v28
 ; LMULMAX2-RV32-NEXT:    vse64.v v26, (a0)
 ; LMULMAX2-RV32-NEXT:    ret
@@ -1549,178 +1432,139 @@ define void @bitreverse_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV32-LABEL: bitreverse_v4i64:
 ; LMULMAX1-RV32:       # %bb.0:
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
 ; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV32-NEXT:    vle64.v v11, (a1)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v28, v11, v26
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_1)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_1)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v27, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v30, v11, v27
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_2)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_2)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v29, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v30, v30, v29
-; LMULMAX1-RV32-NEXT:    vor.vv v9, v30, v28
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_3)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_3)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v28, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v31, v11, v28
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_4)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_4)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v30, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v10, v31, v30
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_5)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_5)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v31, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v12, v11, v31
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_6)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_6)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v8, (a2)
+; LMULMAX1-RV32-NEXT:    vle64.v v30, (a1)
+; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
+; LMULMAX1-RV32-NEXT:    addi a6, zero, 56
+; LMULMAX1-RV32-NEXT:    vsrl.vx v26, v30, a6
+; LMULMAX1-RV32-NEXT:    addi a7, zero, 40
+; LMULMAX1-RV32-NEXT:    vsrl.vx v27, v30, a7
+; LMULMAX1-RV32-NEXT:    lui a4, 16
+; LMULMAX1-RV32-NEXT:    addi a4, a4, -256
+; LMULMAX1-RV32-NEXT:    vand.vx v27, v27, a4
+; LMULMAX1-RV32-NEXT:    vor.vv v27, v27, v26
+; LMULMAX1-RV32-NEXT:    addi a5, zero, 5
+; LMULMAX1-RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.s.x v0, a5
+; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.i v26, 0
+; LMULMAX1-RV32-NEXT:    lui a2, 1044480
+; LMULMAX1-RV32-NEXT:    vmerge.vxm v26, v26, a2, v0
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v12, v12, v8
-; LMULMAX1-RV32-NEXT:    vor.vv v10, v12, v10
-; LMULMAX1-RV32-NEXT:    vor.vv v12, v10, v9
-; LMULMAX1-RV32-NEXT:    vsll.vv v10, v11, v31
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_7)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_7)
+; LMULMAX1-RV32-NEXT:    vsrl.vi v28, v30, 8
+; LMULMAX1-RV32-NEXT:    vand.vv v28, v28, v26
+; LMULMAX1-RV32-NEXT:    vsrl.vi v29, v30, 24
+; LMULMAX1-RV32-NEXT:    lui a5, 4080
+; LMULMAX1-RV32-NEXT:    vand.vx v29, v29, a5
+; LMULMAX1-RV32-NEXT:    vor.vv v28, v28, v29
+; LMULMAX1-RV32-NEXT:    vor.vv v31, v28, v27
+; LMULMAX1-RV32-NEXT:    addi a2, zero, 255
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v9, (a2)
+; LMULMAX1-RV32-NEXT:    vmv.v.x v27, a2
+; LMULMAX1-RV32-NEXT:    vmerge.vim v27, v27, 0, v0
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v13, v10, v9
-; LMULMAX1-RV32-NEXT:    vsll.vv v14, v11, v28
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_8)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_8)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v10, (a2)
+; LMULMAX1-RV32-NEXT:    vsll.vi v28, v30, 8
+; LMULMAX1-RV32-NEXT:    vand.vv v29, v28, v27
+; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.x v28, a4
+; LMULMAX1-RV32-NEXT:    vmerge.vim v28, v28, 0, v0
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v14, v14, v10
-; LMULMAX1-RV32-NEXT:    vor.vv v13, v14, v13
-; LMULMAX1-RV32-NEXT:    vsll.vv v14, v11, v26
-; LMULMAX1-RV32-NEXT:    vsll.vv v15, v11, v27
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_9)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_9)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v11, (a2)
+; LMULMAX1-RV32-NEXT:    vsll.vi v8, v30, 24
+; LMULMAX1-RV32-NEXT:    vand.vv v8, v8, v28
+; LMULMAX1-RV32-NEXT:    vor.vv v8, v8, v29
+; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.x v29, a5
+; LMULMAX1-RV32-NEXT:    vmerge.vim v29, v29, 0, v0
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v15, v15, v11
-; LMULMAX1-RV32-NEXT:    vor.vv v14, v14, v15
-; LMULMAX1-RV32-NEXT:    vor.vv v13, v14, v13
-; LMULMAX1-RV32-NEXT:    vor.vv v15, v13, v12
+; LMULMAX1-RV32-NEXT:    vsll.vx v9, v30, a7
+; LMULMAX1-RV32-NEXT:    vand.vv v9, v9, v29
+; LMULMAX1-RV32-NEXT:    vsll.vx v30, v30, a6
+; LMULMAX1-RV32-NEXT:    vor.vv v30, v30, v9
+; LMULMAX1-RV32-NEXT:    vor.vv v30, v30, v8
+; LMULMAX1-RV32-NEXT:    vor.vv v31, v30, v31
 ; LMULMAX1-RV32-NEXT:    lui a2, 61681
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, -241
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v12, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.x v30, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v14, v15, v12
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_10)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_10)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v13, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsll.vv v16, v14, v13
+; LMULMAX1-RV32-NEXT:    vand.vv v8, v31, v30
+; LMULMAX1-RV32-NEXT:    vsll.vi v8, v8, 4
 ; LMULMAX1-RV32-NEXT:    lui a2, 986895
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, 240
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v14, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.x v9, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v15, v15, v14
-; LMULMAX1-RV32-NEXT:    vsrl.vv v15, v15, v13
-; LMULMAX1-RV32-NEXT:    vor.vv v17, v15, v16
+; LMULMAX1-RV32-NEXT:    vand.vv v31, v31, v9
+; LMULMAX1-RV32-NEXT:    vsrl.vi v31, v31, 4
+; LMULMAX1-RV32-NEXT:    vor.vv v31, v31, v8
 ; LMULMAX1-RV32-NEXT:    lui a2, 209715
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, 819
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v15, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.x v8, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v18, v17, v15
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_11)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_11)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v16, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsll.vv v18, v18, v16
+; LMULMAX1-RV32-NEXT:    vand.vv v10, v31, v8
+; LMULMAX1-RV32-NEXT:    vsll.vi v10, v10, 2
 ; LMULMAX1-RV32-NEXT:    lui a2, 838861
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, -820
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v19, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.x v11, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v17, v17, v19
-; LMULMAX1-RV32-NEXT:    vsrl.vv v17, v17, v16
-; LMULMAX1-RV32-NEXT:    vor.vv v17, v17, v18
+; LMULMAX1-RV32-NEXT:    vand.vv v31, v31, v11
+; LMULMAX1-RV32-NEXT:    vsrl.vi v31, v31, 2
+; LMULMAX1-RV32-NEXT:    vor.vv v31, v31, v10
 ; LMULMAX1-RV32-NEXT:    lui a2, 349525
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, 1365
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v18, a2
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v20, v17, v18
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI5_12)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI5_12)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v21, (a2)
+; LMULMAX1-RV32-NEXT:    vmv.v.x v10, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsll.vv v20, v20, v21
+; LMULMAX1-RV32-NEXT:    vand.vv v12, v31, v10
+; LMULMAX1-RV32-NEXT:    vsll.vi v12, v12, 1
 ; LMULMAX1-RV32-NEXT:    lui a2, 699051
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, -1366
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v22, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.x v13, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v17, v17, v22
-; LMULMAX1-RV32-NEXT:    vsrl.vv v17, v17, v21
-; LMULMAX1-RV32-NEXT:    vor.vv v17, v17, v20
-; LMULMAX1-RV32-NEXT:    vsrl.vv v20, v25, v26
-; LMULMAX1-RV32-NEXT:    vsrl.vv v23, v25, v27
-; LMULMAX1-RV32-NEXT:    vand.vv v29, v23, v29
-; LMULMAX1-RV32-NEXT:    vor.vv v29, v29, v20
-; LMULMAX1-RV32-NEXT:    vsrl.vv v20, v25, v28
-; LMULMAX1-RV32-NEXT:    vand.vv v30, v20, v30
-; LMULMAX1-RV32-NEXT:    vsrl.vv v20, v25, v31
-; LMULMAX1-RV32-NEXT:    vand.vv v8, v20, v8
-; LMULMAX1-RV32-NEXT:    vor.vv v30, v8, v30
-; LMULMAX1-RV32-NEXT:    vor.vv v29, v30, v29
-; LMULMAX1-RV32-NEXT:    vsll.vv v30, v25, v31
-; LMULMAX1-RV32-NEXT:    vand.vv v30, v30, v9
-; LMULMAX1-RV32-NEXT:    vsll.vv v28, v25, v28
-; LMULMAX1-RV32-NEXT:    vand.vv v28, v28, v10
-; LMULMAX1-RV32-NEXT:    vor.vv v28, v28, v30
-; LMULMAX1-RV32-NEXT:    vsll.vv v26, v25, v26
-; LMULMAX1-RV32-NEXT:    vsll.vv v25, v25, v27
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v11
-; LMULMAX1-RV32-NEXT:    vor.vv v25, v26, v25
+; LMULMAX1-RV32-NEXT:    vand.vv v31, v31, v13
+; LMULMAX1-RV32-NEXT:    vsrl.vi v31, v31, 1
+; LMULMAX1-RV32-NEXT:    vor.vv v31, v31, v12
+; LMULMAX1-RV32-NEXT:    vsrl.vx v12, v25, a6
+; LMULMAX1-RV32-NEXT:    vsrl.vx v14, v25, a7
+; LMULMAX1-RV32-NEXT:    vand.vx v14, v14, a4
+; LMULMAX1-RV32-NEXT:    vor.vv v12, v14, v12
+; LMULMAX1-RV32-NEXT:    vsrl.vi v14, v25, 8
+; LMULMAX1-RV32-NEXT:    vand.vv v26, v14, v26
+; LMULMAX1-RV32-NEXT:    vsrl.vi v14, v25, 24
+; LMULMAX1-RV32-NEXT:    vand.vx v14, v14, a5
+; LMULMAX1-RV32-NEXT:    vor.vv v26, v26, v14
+; LMULMAX1-RV32-NEXT:    vor.vv v26, v26, v12
+; LMULMAX1-RV32-NEXT:    vsll.vi v12, v25, 8
+; LMULMAX1-RV32-NEXT:    vand.vv v27, v12, v27
+; LMULMAX1-RV32-NEXT:    vsll.vi v12, v25, 24
+; LMULMAX1-RV32-NEXT:    vand.vv v28, v12, v28
+; LMULMAX1-RV32-NEXT:    vor.vv v27, v28, v27
+; LMULMAX1-RV32-NEXT:    vsll.vx v28, v25, a7
+; LMULMAX1-RV32-NEXT:    vand.vv v28, v28, v29
+; LMULMAX1-RV32-NEXT:    vsll.vx v25, v25, a6
 ; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v29
-; LMULMAX1-RV32-NEXT:    vand.vv v26, v25, v12
-; LMULMAX1-RV32-NEXT:    vsll.vv v26, v26, v13
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v14
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v13
+; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v27
 ; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vand.vv v26, v25, v15
-; LMULMAX1-RV32-NEXT:    vsll.vv v26, v26, v16
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v19
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v16
+; LMULMAX1-RV32-NEXT:    vand.vv v26, v25, v30
+; LMULMAX1-RV32-NEXT:    vsll.vi v26, v26, 4
+; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v9
+; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 4
 ; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vand.vv v26, v25, v18
-; LMULMAX1-RV32-NEXT:    vsll.vv v26, v26, v21
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v22
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v21
+; LMULMAX1-RV32-NEXT:    vand.vv v26, v25, v8
+; LMULMAX1-RV32-NEXT:    vsll.vi v26, v26, 2
+; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v11
+; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 2
+; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
+; LMULMAX1-RV32-NEXT:    vand.vv v26, v25, v10
+; LMULMAX1-RV32-NEXT:    vsll.vi v26, v26, 1
+; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v13
+; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 1
 ; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
 ; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vse64.v v17, (a1)
+; LMULMAX1-RV32-NEXT:    vse64.v v31, (a1)
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV64-LABEL: bitreverse_v4i64:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll
index 85da44f54341bc30f1bc58f9d272f1c0674763f6..e6432c1632553f00008ef41d9bd7ac158fd7b104 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bswap.ll
@@ -612,8 +612,6 @@ define void @bswap_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ;
 ; LMULMAX2-RV64-LABEL: bswap_v2i64:
 ; LMULMAX2-RV64:       # %bb.0:
-; LMULMAX2-RV64-NEXT:    addi sp, sp, -16
-; LMULMAX2-RV64-NEXT:    .cfi_def_cfa_offset 16
 ; LMULMAX2-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV64-NEXT:    vle64.v v25, (a0)
 ; LMULMAX2-RV64-NEXT:    vmv.x.s a1, v25
@@ -645,37 +643,35 @@ define void @bswap_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV64-NEXT:    slli a1, a1, 56
 ; LMULMAX2-RV64-NEXT:    or a1, a1, a4
 ; LMULMAX2-RV64-NEXT:    or a1, a1, a2
-; LMULMAX2-RV64-NEXT:    or a1, a1, t1
-; LMULMAX2-RV64-NEXT:    sd a1, 0(sp)
-; LMULMAX2-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    or t1, a1, t1
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 1, e64,m1,ta,mu
 ; LMULMAX2-RV64-NEXT:    vslidedown.vi v25, v25, 1
-; LMULMAX2-RV64-NEXT:    vmv.x.s a1, v25
-; LMULMAX2-RV64-NEXT:    srli a2, a1, 40
-; LMULMAX2-RV64-NEXT:    and a2, a2, a7
-; LMULMAX2-RV64-NEXT:    srli a4, a1, 56
-; LMULMAX2-RV64-NEXT:    or a2, a2, a4
-; LMULMAX2-RV64-NEXT:    srli a4, a1, 24
+; LMULMAX2-RV64-NEXT:    vmv.x.s a2, v25
+; LMULMAX2-RV64-NEXT:    srli a4, a2, 40
+; LMULMAX2-RV64-NEXT:    and a4, a4, a7
+; LMULMAX2-RV64-NEXT:    srli a1, a2, 56
+; LMULMAX2-RV64-NEXT:    or a1, a4, a1
+; LMULMAX2-RV64-NEXT:    srli a4, a2, 24
 ; LMULMAX2-RV64-NEXT:    and a4, a4, a6
-; LMULMAX2-RV64-NEXT:    srli a5, a1, 8
+; LMULMAX2-RV64-NEXT:    srli a5, a2, 8
 ; LMULMAX2-RV64-NEXT:    and a5, a5, t0
 ; LMULMAX2-RV64-NEXT:    or a4, a5, a4
-; LMULMAX2-RV64-NEXT:    or a2, a4, a2
-; LMULMAX2-RV64-NEXT:    slli a4, a1, 8
+; LMULMAX2-RV64-NEXT:    or a1, a4, a1
+; LMULMAX2-RV64-NEXT:    slli a4, a2, 8
 ; LMULMAX2-RV64-NEXT:    and a4, a4, t2
-; LMULMAX2-RV64-NEXT:    slli a5, a1, 24
+; LMULMAX2-RV64-NEXT:    slli a5, a2, 24
 ; LMULMAX2-RV64-NEXT:    and a5, a5, t3
 ; LMULMAX2-RV64-NEXT:    or a4, a5, a4
-; LMULMAX2-RV64-NEXT:    slli a5, a1, 40
+; LMULMAX2-RV64-NEXT:    slli a5, a2, 40
 ; LMULMAX2-RV64-NEXT:    and a3, a5, a3
-; LMULMAX2-RV64-NEXT:    slli a1, a1, 56
-; LMULMAX2-RV64-NEXT:    or a1, a1, a3
-; LMULMAX2-RV64-NEXT:    or a1, a1, a4
-; LMULMAX2-RV64-NEXT:    or a1, a1, a2
-; LMULMAX2-RV64-NEXT:    sd a1, 8(sp)
-; LMULMAX2-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV64-NEXT:    vle64.v v25, (sp)
+; LMULMAX2-RV64-NEXT:    slli a2, a2, 56
+; LMULMAX2-RV64-NEXT:    or a2, a2, a3
+; LMULMAX2-RV64-NEXT:    or a2, a2, a4
+; LMULMAX2-RV64-NEXT:    or a1, a2, a1
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.x v25, a1
+; LMULMAX2-RV64-NEXT:    vmv.s.x v25, t1
 ; LMULMAX2-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX2-RV64-NEXT:    addi sp, sp, 16
 ; LMULMAX2-RV64-NEXT:    ret
 ;
 ; LMULMAX1-RV32-LABEL: bswap_v2i64:
@@ -745,8 +741,6 @@ define void @bswap_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ;
 ; LMULMAX1-RV64-LABEL: bswap_v2i64:
 ; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    addi sp, sp, -16
-; LMULMAX1-RV64-NEXT:    .cfi_def_cfa_offset 16
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
 ; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v25
@@ -778,37 +772,35 @@ define void @bswap_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    slli a1, a1, 56
 ; LMULMAX1-RV64-NEXT:    or a1, a1, a4
 ; LMULMAX1-RV64-NEXT:    or a1, a1, a2
-; LMULMAX1-RV64-NEXT:    or a1, a1, t1
-; LMULMAX1-RV64-NEXT:    sd a1, 0(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    or t1, a1, t1
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 1, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    vslidedown.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v25
-; LMULMAX1-RV64-NEXT:    srli a2, a1, 40
-; LMULMAX1-RV64-NEXT:    and a2, a2, a7
-; LMULMAX1-RV64-NEXT:    srli a4, a1, 56
-; LMULMAX1-RV64-NEXT:    or a2, a2, a4
-; LMULMAX1-RV64-NEXT:    srli a4, a1, 24
+; LMULMAX1-RV64-NEXT:    vmv.x.s a2, v25
+; LMULMAX1-RV64-NEXT:    srli a4, a2, 40
+; LMULMAX1-RV64-NEXT:    and a4, a4, a7
+; LMULMAX1-RV64-NEXT:    srli a1, a2, 56
+; LMULMAX1-RV64-NEXT:    or a1, a4, a1
+; LMULMAX1-RV64-NEXT:    srli a4, a2, 24
 ; LMULMAX1-RV64-NEXT:    and a4, a4, a6
-; LMULMAX1-RV64-NEXT:    srli a5, a1, 8
+; LMULMAX1-RV64-NEXT:    srli a5, a2, 8
 ; LMULMAX1-RV64-NEXT:    and a5, a5, t0
 ; LMULMAX1-RV64-NEXT:    or a4, a5, a4
-; LMULMAX1-RV64-NEXT:    or a2, a4, a2
-; LMULMAX1-RV64-NEXT:    slli a4, a1, 8
+; LMULMAX1-RV64-NEXT:    or a1, a4, a1
+; LMULMAX1-RV64-NEXT:    slli a4, a2, 8
 ; LMULMAX1-RV64-NEXT:    and a4, a4, t2
-; LMULMAX1-RV64-NEXT:    slli a5, a1, 24
+; LMULMAX1-RV64-NEXT:    slli a5, a2, 24
 ; LMULMAX1-RV64-NEXT:    and a5, a5, t3
 ; LMULMAX1-RV64-NEXT:    or a4, a5, a4
-; LMULMAX1-RV64-NEXT:    slli a5, a1, 40
+; LMULMAX1-RV64-NEXT:    slli a5, a2, 40
 ; LMULMAX1-RV64-NEXT:    and a3, a5, a3
-; LMULMAX1-RV64-NEXT:    slli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    or a1, a1, a3
-; LMULMAX1-RV64-NEXT:    or a1, a1, a4
-; LMULMAX1-RV64-NEXT:    or a1, a1, a2
-; LMULMAX1-RV64-NEXT:    sd a1, 8(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (sp)
+; LMULMAX1-RV64-NEXT:    slli a2, a2, 56
+; LMULMAX1-RV64-NEXT:    or a2, a2, a3
+; LMULMAX1-RV64-NEXT:    or a2, a2, a4
+; LMULMAX1-RV64-NEXT:    or a1, a2, a1
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v25, a1
+; LMULMAX1-RV64-NEXT:    vmv.s.x v25, t1
 ; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    addi sp, sp, 16
 ; LMULMAX1-RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = load <2 x i64>, <2 x i64>* %y
@@ -2169,8 +2161,6 @@ define void @bswap_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ;
 ; LMULMAX1-RV64-LABEL: bswap_v4i64:
 ; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    addi sp, sp, -32
-; LMULMAX1-RV64-NEXT:    .cfi_def_cfa_offset 32
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    addi a6, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle64.v v26, (a6)
@@ -2204,33 +2194,34 @@ define void @bswap_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    slli a2, a2, 56
 ; LMULMAX1-RV64-NEXT:    or a2, a2, a3
 ; LMULMAX1-RV64-NEXT:    or a1, a2, a1
-; LMULMAX1-RV64-NEXT:    or a1, a1, a5
-; LMULMAX1-RV64-NEXT:    sd a1, 16(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    or t4, a1, a5
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 1, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    vslidedown.vi v26, v26, 1
-; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v26
-; LMULMAX1-RV64-NEXT:    srli a2, a1, 40
-; LMULMAX1-RV64-NEXT:    and a2, a2, t0
-; LMULMAX1-RV64-NEXT:    srli a3, a1, 56
-; LMULMAX1-RV64-NEXT:    or a2, a2, a3
-; LMULMAX1-RV64-NEXT:    srli a3, a1, 24
-; LMULMAX1-RV64-NEXT:    and a3, a3, a7
-; LMULMAX1-RV64-NEXT:    srli a5, a1, 8
-; LMULMAX1-RV64-NEXT:    and a5, a5, t1
-; LMULMAX1-RV64-NEXT:    or a3, a5, a3
-; LMULMAX1-RV64-NEXT:    or a2, a3, a2
-; LMULMAX1-RV64-NEXT:    slli a3, a1, 8
+; LMULMAX1-RV64-NEXT:    vmv.x.s a2, v26
+; LMULMAX1-RV64-NEXT:    srli a3, a2, 40
+; LMULMAX1-RV64-NEXT:    and a3, a3, t0
+; LMULMAX1-RV64-NEXT:    srli a5, a2, 56
+; LMULMAX1-RV64-NEXT:    or a3, a3, a5
+; LMULMAX1-RV64-NEXT:    srli a5, a2, 24
+; LMULMAX1-RV64-NEXT:    and a5, a5, a7
+; LMULMAX1-RV64-NEXT:    srli a1, a2, 8
+; LMULMAX1-RV64-NEXT:    and a1, a1, t1
+; LMULMAX1-RV64-NEXT:    or a1, a1, a5
+; LMULMAX1-RV64-NEXT:    or a1, a1, a3
+; LMULMAX1-RV64-NEXT:    slli a3, a2, 8
 ; LMULMAX1-RV64-NEXT:    and a3, a3, t2
-; LMULMAX1-RV64-NEXT:    slli a5, a1, 24
+; LMULMAX1-RV64-NEXT:    slli a5, a2, 24
 ; LMULMAX1-RV64-NEXT:    and a5, a5, t3
 ; LMULMAX1-RV64-NEXT:    or a3, a5, a3
-; LMULMAX1-RV64-NEXT:    slli a5, a1, 40
+; LMULMAX1-RV64-NEXT:    slli a5, a2, 40
 ; LMULMAX1-RV64-NEXT:    and a5, a5, a4
-; LMULMAX1-RV64-NEXT:    slli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    or a1, a1, a5
-; LMULMAX1-RV64-NEXT:    or a1, a1, a3
-; LMULMAX1-RV64-NEXT:    or a1, a1, a2
-; LMULMAX1-RV64-NEXT:    sd a1, 24(sp)
+; LMULMAX1-RV64-NEXT:    slli a2, a2, 56
+; LMULMAX1-RV64-NEXT:    or a2, a2, a5
+; LMULMAX1-RV64-NEXT:    or a2, a2, a3
+; LMULMAX1-RV64-NEXT:    or a1, a2, a1
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v26, a1
+; LMULMAX1-RV64-NEXT:    vmv.s.x v26, t4
 ; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v25
 ; LMULMAX1-RV64-NEXT:    srli a2, a1, 40
 ; LMULMAX1-RV64-NEXT:    and a2, a2, t0
@@ -2252,39 +2243,36 @@ define void @bswap_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    slli a1, a1, 56
 ; LMULMAX1-RV64-NEXT:    or a1, a1, a5
 ; LMULMAX1-RV64-NEXT:    or a1, a1, a3
-; LMULMAX1-RV64-NEXT:    or a1, a1, a2
-; LMULMAX1-RV64-NEXT:    sd a1, 0(sp)
+; LMULMAX1-RV64-NEXT:    or t4, a1, a2
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 1, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    vslidedown.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v25
-; LMULMAX1-RV64-NEXT:    srli a2, a1, 40
-; LMULMAX1-RV64-NEXT:    and a2, a2, t0
-; LMULMAX1-RV64-NEXT:    srli a3, a1, 56
-; LMULMAX1-RV64-NEXT:    or a2, a2, a3
-; LMULMAX1-RV64-NEXT:    srli a3, a1, 24
-; LMULMAX1-RV64-NEXT:    and a3, a3, a7
-; LMULMAX1-RV64-NEXT:    srli a5, a1, 8
-; LMULMAX1-RV64-NEXT:    and a5, a5, t1
-; LMULMAX1-RV64-NEXT:    or a3, a5, a3
-; LMULMAX1-RV64-NEXT:    or a2, a3, a2
-; LMULMAX1-RV64-NEXT:    slli a3, a1, 8
+; LMULMAX1-RV64-NEXT:    vmv.x.s a2, v25
+; LMULMAX1-RV64-NEXT:    srli a3, a2, 40
+; LMULMAX1-RV64-NEXT:    and a3, a3, t0
+; LMULMAX1-RV64-NEXT:    srli a5, a2, 56
+; LMULMAX1-RV64-NEXT:    or a3, a3, a5
+; LMULMAX1-RV64-NEXT:    srli a5, a2, 24
+; LMULMAX1-RV64-NEXT:    and a5, a5, a7
+; LMULMAX1-RV64-NEXT:    srli a1, a2, 8
+; LMULMAX1-RV64-NEXT:    and a1, a1, t1
+; LMULMAX1-RV64-NEXT:    or a1, a1, a5
+; LMULMAX1-RV64-NEXT:    or a1, a1, a3
+; LMULMAX1-RV64-NEXT:    slli a3, a2, 8
 ; LMULMAX1-RV64-NEXT:    and a3, a3, t2
-; LMULMAX1-RV64-NEXT:    slli a5, a1, 24
+; LMULMAX1-RV64-NEXT:    slli a5, a2, 24
 ; LMULMAX1-RV64-NEXT:    and a5, a5, t3
 ; LMULMAX1-RV64-NEXT:    or a3, a5, a3
-; LMULMAX1-RV64-NEXT:    slli a5, a1, 40
+; LMULMAX1-RV64-NEXT:    slli a5, a2, 40
 ; LMULMAX1-RV64-NEXT:    and a4, a5, a4
-; LMULMAX1-RV64-NEXT:    slli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    or a1, a1, a4
-; LMULMAX1-RV64-NEXT:    or a1, a1, a3
-; LMULMAX1-RV64-NEXT:    or a1, a1, a2
-; LMULMAX1-RV64-NEXT:    sd a1, 8(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (sp)
-; LMULMAX1-RV64-NEXT:    addi a1, sp, 16
-; LMULMAX1-RV64-NEXT:    vle64.v v26, (a1)
+; LMULMAX1-RV64-NEXT:    slli a2, a2, 56
+; LMULMAX1-RV64-NEXT:    or a2, a2, a4
+; LMULMAX1-RV64-NEXT:    or a2, a2, a3
+; LMULMAX1-RV64-NEXT:    or a1, a2, a1
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v25, a1
+; LMULMAX1-RV64-NEXT:    vmv.s.x v25, t4
 ; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
 ; LMULMAX1-RV64-NEXT:    vse64.v v26, (a6)
-; LMULMAX1-RV64-NEXT:    addi sp, sp, 32
 ; LMULMAX1-RV64-NEXT:    ret
   %a = load <4 x i64>, <4 x i64>* %x
   %b = load <4 x i64>, <4 x i64>* %y
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll
index aa70f31d96705766dfd465a06921eabb7180012a..0128f6ef84501f79e3f245674d6b3a814230344b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll
@@ -3790,11 +3790,11 @@ define void @ctlz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ;
 ; LMULMAX2-RV64-LABEL: ctlz_v2i64:
 ; LMULMAX2-RV64:       # %bb.0:
-; LMULMAX2-RV64-NEXT:    addi sp, sp, -16
-; LMULMAX2-RV64-NEXT:    .cfi_def_cfa_offset 16
 ; LMULMAX2-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX2-RV64-NEXT:    vmv.x.s a1, v25
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vslidedown.vi v26, v25, 1
+; LMULMAX2-RV64-NEXT:    vmv.x.s a1, v26
 ; LMULMAX2-RV64-NEXT:    srli a2, a1, 1
 ; LMULMAX2-RV64-NEXT:    or a1, a1, a2
 ; LMULMAX2-RV64-NEXT:    srli a2, a1, 2
@@ -3850,9 +3850,8 @@ define void @ctlz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV64-NEXT:    addi a5, a5, 257
 ; LMULMAX2-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX2-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX2-RV64-NEXT:    sd a1, 0(sp)
-; LMULMAX2-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
-; LMULMAX2-RV64-NEXT:    vslidedown.vi v25, v25, 1
+; LMULMAX2-RV64-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.x v26, a1
 ; LMULMAX2-RV64-NEXT:    vmv.x.s a1, v25
 ; LMULMAX2-RV64-NEXT:    srli a3, a1, 1
 ; LMULMAX2-RV64-NEXT:    or a1, a1, a3
@@ -3879,11 +3878,8 @@ define void @ctlz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV64-NEXT:    and a1, a1, a4
 ; LMULMAX2-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX2-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX2-RV64-NEXT:    sd a1, 8(sp)
-; LMULMAX2-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV64-NEXT:    vle64.v v25, (sp)
-; LMULMAX2-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX2-RV64-NEXT:    addi sp, sp, 16
+; LMULMAX2-RV64-NEXT:    vmv.s.x v26, a1
+; LMULMAX2-RV64-NEXT:    vse64.v v26, (a0)
 ; LMULMAX2-RV64-NEXT:    ret
 ;
 ; LMULMAX1-RV32-LABEL: ctlz_v2i64:
@@ -4027,11 +4023,11 @@ define void @ctlz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ;
 ; LMULMAX1-RV64-LABEL: ctlz_v2i64:
 ; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    addi sp, sp, -16
-; LMULMAX1-RV64-NEXT:    .cfi_def_cfa_offset 16
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v25
+; LMULMAX1-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v26, v25, 1
+; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v26
 ; LMULMAX1-RV64-NEXT:    srli a2, a1, 1
 ; LMULMAX1-RV64-NEXT:    or a1, a1, a2
 ; LMULMAX1-RV64-NEXT:    srli a2, a1, 2
@@ -4087,9 +4083,8 @@ define void @ctlz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    addi a5, a5, 257
 ; LMULMAX1-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX1-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    sd a1, 0(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vslidedown.vi v25, v25, 1
+; LMULMAX1-RV64-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v26, a1
 ; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v25
 ; LMULMAX1-RV64-NEXT:    srli a3, a1, 1
 ; LMULMAX1-RV64-NEXT:    or a1, a1, a3
@@ -4116,11 +4111,8 @@ define void @ctlz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    and a1, a1, a4
 ; LMULMAX1-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX1-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    sd a1, 8(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (sp)
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    addi sp, sp, 16
+; LMULMAX1-RV64-NEXT:    vmv.s.x v26, a1
+; LMULMAX1-RV64-NEXT:    vse64.v v26, (a0)
 ; LMULMAX1-RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = load <2 x i64>, <2 x i64>* %y
@@ -11756,12 +11748,12 @@ define void @ctlz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ;
 ; LMULMAX1-RV64-LABEL: ctlz_v4i64:
 ; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    addi sp, sp, -32
-; LMULMAX1-RV64-NEXT:    .cfi_def_cfa_offset 32
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    addi a6, a0, 16
-; LMULMAX1-RV64-NEXT:    vle64.v v26, (a6)
+; LMULMAX1-RV64-NEXT:    vle64.v v27, (a6)
 ; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 1, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v26, v27, 1
 ; LMULMAX1-RV64-NEXT:    vmv.x.s a2, v26
 ; LMULMAX1-RV64-NEXT:    srli a3, a2, 1
 ; LMULMAX1-RV64-NEXT:    or a2, a2, a3
@@ -11818,10 +11810,9 @@ define void @ctlz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    addi a5, a5, 257
 ; LMULMAX1-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX1-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    sd a1, 16(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vslidedown.vi v26, v26, 1
-; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v26
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v26, a1
+; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v27
 ; LMULMAX1-RV64-NEXT:    srli a2, a1, 1
 ; LMULMAX1-RV64-NEXT:    or a1, a1, a2
 ; LMULMAX1-RV64-NEXT:    srli a2, a1, 2
@@ -11847,8 +11838,10 @@ define void @ctlz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    and a1, a1, a4
 ; LMULMAX1-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX1-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    sd a1, 24(sp)
-; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v25
+; LMULMAX1-RV64-NEXT:    vmv.s.x v26, a1
+; LMULMAX1-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v27, v25, 1
+; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v27
 ; LMULMAX1-RV64-NEXT:    srli a2, a1, 1
 ; LMULMAX1-RV64-NEXT:    or a1, a1, a2
 ; LMULMAX1-RV64-NEXT:    srli a2, a1, 2
@@ -11874,8 +11867,8 @@ define void @ctlz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    and a1, a1, a4
 ; LMULMAX1-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX1-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    sd a1, 0(sp)
-; LMULMAX1-RV64-NEXT:    vslidedown.vi v25, v25, 1
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v27, a1
 ; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v25
 ; LMULMAX1-RV64-NEXT:    srli a2, a1, 1
 ; LMULMAX1-RV64-NEXT:    or a1, a1, a2
@@ -11902,14 +11895,9 @@ define void @ctlz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    and a1, a1, a4
 ; LMULMAX1-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX1-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    sd a1, 8(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (sp)
-; LMULMAX1-RV64-NEXT:    addi a1, sp, 16
-; LMULMAX1-RV64-NEXT:    vle64.v v26, (a1)
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
+; LMULMAX1-RV64-NEXT:    vmv.s.x v27, a1
+; LMULMAX1-RV64-NEXT:    vse64.v v27, (a0)
 ; LMULMAX1-RV64-NEXT:    vse64.v v26, (a6)
-; LMULMAX1-RV64-NEXT:    addi sp, sp, 32
 ; LMULMAX1-RV64-NEXT:    ret
   %a = load <4 x i64>, <4 x i64>* %x
   %b = load <4 x i64>, <4 x i64>* %y
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop.ll
index a007f503a5ac7a684dcbdff3d80584400c1e84d0..9678f8ffb404290adc1d9bf8279243721935df53 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop.ll
@@ -264,12 +264,7 @@ define void @ctpop_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV32:       # %bb.0:
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI3_0)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI3_0)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v25, v26
+; LMULMAX2-RV32-NEXT:    vsrl.vi v26, v25, 1
 ; LMULMAX2-RV32-NEXT:    lui a1, 349525
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, 1365
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
@@ -283,20 +278,10 @@ define void @ctpop_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV32-NEXT:    vand.vv v27, v25, v26
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI3_1)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI3_1)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v28, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v25, v25, v28
+; LMULMAX2-RV32-NEXT:    vsrl.vi v25, v25, 2
 ; LMULMAX2-RV32-NEXT:    vand.vv v25, v25, v26
 ; LMULMAX2-RV32-NEXT:    vadd.vv v25, v27, v25
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI3_2)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI3_2)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v25, v26
+; LMULMAX2-RV32-NEXT:    vsrl.vi v26, v25, 4
 ; LMULMAX2-RV32-NEXT:    vadd.vv v25, v25, v26
 ; LMULMAX2-RV32-NEXT:    lui a1, 61681
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, -241
@@ -310,12 +295,8 @@ define void @ctpop_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV32-NEXT:    vmul.vv v25, v25, v26
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI3_3)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI3_3)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v25, v25, v26
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 56
+; LMULMAX2-RV32-NEXT:    vsrl.vx v25, v25, a1
 ; LMULMAX2-RV32-NEXT:    vse64.v v25, (a0)
 ; LMULMAX2-RV32-NEXT:    ret
 ;
@@ -373,12 +354,7 @@ define void @ctpop_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV32:       # %bb.0:
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI3_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI3_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v26, v25, v26
+; LMULMAX1-RV32-NEXT:    vsrl.vi v26, v25, 1
 ; LMULMAX1-RV32-NEXT:    lui a1, 349525
 ; LMULMAX1-RV32-NEXT:    addi a1, a1, 1365
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
@@ -392,20 +368,10 @@ define void @ctpop_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vand.vv v27, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI3_1)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI3_1)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v28, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v28
+; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 2
 ; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v26
 ; LMULMAX1-RV32-NEXT:    vadd.vv v25, v27, v25
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI3_2)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI3_2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v26, v25, v26
+; LMULMAX1-RV32-NEXT:    vsrl.vi v26, v25, 4
 ; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
 ; LMULMAX1-RV32-NEXT:    lui a1, 61681
 ; LMULMAX1-RV32-NEXT:    addi a1, a1, -241
@@ -419,12 +385,8 @@ define void @ctpop_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV32-NEXT:    vmv.v.x v26, a1
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vmul.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI3_3)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI3_3)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v26
+; LMULMAX1-RV32-NEXT:    addi a1, zero, 56
+; LMULMAX1-RV32-NEXT:    vsrl.vx v25, v25, a1
 ; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
 ; LMULMAX1-RV32-NEXT:    ret
 ;
@@ -838,12 +800,7 @@ define void @ctpop_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX2-RV32:       # %bb.0:
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vle64.v v26, (a0)
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI7_0)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI7_0)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v28, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v28, v26, v28
+; LMULMAX2-RV32-NEXT:    vsrl.vi v28, v26, 1
 ; LMULMAX2-RV32-NEXT:    lui a1, 349525
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, 1365
 ; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
@@ -857,20 +814,10 @@ define void @ctpop_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vand.vv v30, v26, v28
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI7_1)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI7_1)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v8, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v26, v8
+; LMULMAX2-RV32-NEXT:    vsrl.vi v26, v26, 2
 ; LMULMAX2-RV32-NEXT:    vand.vv v26, v26, v28
 ; LMULMAX2-RV32-NEXT:    vadd.vv v26, v30, v26
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI7_2)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI7_2)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v28, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v28, v26, v28
+; LMULMAX2-RV32-NEXT:    vsrl.vi v28, v26, 4
 ; LMULMAX2-RV32-NEXT:    vadd.vv v26, v26, v28
 ; LMULMAX2-RV32-NEXT:    lui a1, 61681
 ; LMULMAX2-RV32-NEXT:    addi a1, a1, -241
@@ -884,12 +831,8 @@ define void @ctpop_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
 ; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vmul.vv v26, v26, v28
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI7_3)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI7_3)
-; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v28, (a1)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 56
+; LMULMAX2-RV32-NEXT:    vsrl.vx v26, v26, a1
 ; LMULMAX2-RV32-NEXT:    vse64.v v26, (a0)
 ; LMULMAX2-RV32-NEXT:    ret
 ;
@@ -946,75 +889,56 @@ define void @ctpop_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV32-LABEL: ctpop_v4i64:
 ; LMULMAX1-RV32:       # %bb.0:
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
 ; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV32-NEXT:    vle64.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI7_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI7_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v27, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v28, v26, v27
+; LMULMAX1-RV32-NEXT:    vle64.v v25, (a1)
+; LMULMAX1-RV32-NEXT:    vle64.v v26, (a0)
+; LMULMAX1-RV32-NEXT:    vsrl.vi v27, v25, 1
 ; LMULMAX1-RV32-NEXT:    lui a2, 349525
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, 1365
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v29, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.x v28, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v28, v28, v29
-; LMULMAX1-RV32-NEXT:    vsub.vv v26, v26, v28
+; LMULMAX1-RV32-NEXT:    vand.vv v27, v27, v28
+; LMULMAX1-RV32-NEXT:    vsub.vv v25, v25, v27
 ; LMULMAX1-RV32-NEXT:    lui a2, 209715
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, 819
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v28, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.x v27, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v30, v26, v28
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI7_1)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI7_1)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v31, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v26, v26, v31
-; LMULMAX1-RV32-NEXT:    vand.vv v26, v26, v28
-; LMULMAX1-RV32-NEXT:    vadd.vv v26, v30, v26
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI7_2)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI7_2)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v30, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v8, v26, v30
-; LMULMAX1-RV32-NEXT:    vadd.vv v26, v26, v8
+; LMULMAX1-RV32-NEXT:    vand.vv v29, v25, v27
+; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 2
+; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vadd.vv v25, v29, v25
+; LMULMAX1-RV32-NEXT:    vsrl.vi v29, v25, 4
+; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v29
 ; LMULMAX1-RV32-NEXT:    lui a2, 61681
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, -241
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v8, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.x v29, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v26, v26, v8
+; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v29
 ; LMULMAX1-RV32-NEXT:    lui a2, 4112
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, 257
 ; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.x v9, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.x v30, a2
 ; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmul.vv v26, v26, v9
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI7_3)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI7_3)
-; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v10, (a2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v26, v26, v10
-; LMULMAX1-RV32-NEXT:    vsrl.vv v27, v25, v27
-; LMULMAX1-RV32-NEXT:    vand.vv v27, v27, v29
-; LMULMAX1-RV32-NEXT:    vsub.vv v25, v25, v27
-; LMULMAX1-RV32-NEXT:    vand.vv v27, v25, v28
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v31
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v27, v25
-; LMULMAX1-RV32-NEXT:    vsrl.vv v27, v25, v30
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v27
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v8
-; LMULMAX1-RV32-NEXT:    vmul.vv v25, v25, v9
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v10
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vse64.v v26, (a1)
+; LMULMAX1-RV32-NEXT:    vmul.vv v25, v25, v30
+; LMULMAX1-RV32-NEXT:    addi a2, zero, 56
+; LMULMAX1-RV32-NEXT:    vsrl.vx v25, v25, a2
+; LMULMAX1-RV32-NEXT:    vsrl.vi v31, v26, 1
+; LMULMAX1-RV32-NEXT:    vand.vv v28, v31, v28
+; LMULMAX1-RV32-NEXT:    vsub.vv v26, v26, v28
+; LMULMAX1-RV32-NEXT:    vand.vv v28, v26, v27
+; LMULMAX1-RV32-NEXT:    vsrl.vi v26, v26, 2
+; LMULMAX1-RV32-NEXT:    vand.vv v26, v26, v27
+; LMULMAX1-RV32-NEXT:    vadd.vv v26, v28, v26
+; LMULMAX1-RV32-NEXT:    vsrl.vi v27, v26, 4
+; LMULMAX1-RV32-NEXT:    vadd.vv v26, v26, v27
+; LMULMAX1-RV32-NEXT:    vand.vv v26, v26, v29
+; LMULMAX1-RV32-NEXT:    vmul.vv v26, v26, v30
+; LMULMAX1-RV32-NEXT:    vsrl.vx v26, v26, a2
+; LMULMAX1-RV32-NEXT:    vse64.v v26, (a0)
+; LMULMAX1-RV32-NEXT:    vse64.v v25, (a1)
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV64-LABEL: ctpop_v4i64:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
index 2a7ebc4911028ea0e310f9fae8f76322939b87cf..7bec59f18821acbf913396b385240ae4fb63e6a1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
@@ -2638,8 +2638,6 @@ define void @cttz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ;
 ; LMULMAX2-RV64-LABEL: cttz_v2i64:
 ; LMULMAX2-RV64:       # %bb.0:
-; LMULMAX2-RV64-NEXT:    addi sp, sp, -16
-; LMULMAX2-RV64-NEXT:    .cfi_def_cfa_offset 16
 ; LMULMAX2-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX2-RV64-NEXT:    vle64.v v25, (a0)
 ; LMULMAX2-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
@@ -2690,7 +2688,8 @@ define void @cttz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV64-NEXT:    addi a5, a5, 257
 ; LMULMAX2-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX2-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX2-RV64-NEXT:    sd a1, 8(sp)
+; LMULMAX2-RV64-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.x v26, a1
 ; LMULMAX2-RV64-NEXT:    vmv.x.s a1, v25
 ; LMULMAX2-RV64-NEXT:    addi a3, a1, -1
 ; LMULMAX2-RV64-NEXT:    not a1, a1
@@ -2707,11 +2706,8 @@ define void @cttz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX2-RV64-NEXT:    and a1, a1, a4
 ; LMULMAX2-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX2-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX2-RV64-NEXT:    sd a1, 0(sp)
-; LMULMAX2-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX2-RV64-NEXT:    vle64.v v25, (sp)
-; LMULMAX2-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX2-RV64-NEXT:    addi sp, sp, 16
+; LMULMAX2-RV64-NEXT:    vmv.s.x v26, a1
+; LMULMAX2-RV64-NEXT:    vse64.v v26, (a0)
 ; LMULMAX2-RV64-NEXT:    ret
 ;
 ; LMULMAX1-RV32-LABEL: cttz_v2i64:
@@ -2823,8 +2819,6 @@ define void @cttz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ;
 ; LMULMAX1-RV64-LABEL: cttz_v2i64:
 ; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    addi sp, sp, -16
-; LMULMAX1-RV64-NEXT:    .cfi_def_cfa_offset 16
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
@@ -2875,7 +2869,8 @@ define void @cttz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    addi a5, a5, 257
 ; LMULMAX1-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX1-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    sd a1, 8(sp)
+; LMULMAX1-RV64-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v26, a1
 ; LMULMAX1-RV64-NEXT:    vmv.x.s a1, v25
 ; LMULMAX1-RV64-NEXT:    addi a3, a1, -1
 ; LMULMAX1-RV64-NEXT:    not a1, a1
@@ -2892,11 +2887,8 @@ define void @cttz_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    and a1, a1, a4
 ; LMULMAX1-RV64-NEXT:    mul a1, a1, a5
 ; LMULMAX1-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    sd a1, 0(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (sp)
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    addi sp, sp, 16
+; LMULMAX1-RV64-NEXT:    vmv.s.x v26, a1
+; LMULMAX1-RV64-NEXT:    vse64.v v26, (a0)
 ; LMULMAX1-RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = load <2 x i64>, <2 x i64>* %y
@@ -8124,8 +8116,6 @@ define void @cttz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ;
 ; LMULMAX1-RV64-LABEL: cttz_v4i64:
 ; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    addi sp, sp, -32
-; LMULMAX1-RV64-NEXT:    .cfi_def_cfa_offset 32
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    addi a6, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle64.v v26, (a6)
@@ -8178,11 +8168,12 @@ define void @cttz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    addi a1, a1, 257
 ; LMULMAX1-RV64-NEXT:    mul a4, a4, a1
 ; LMULMAX1-RV64-NEXT:    srli a4, a4, 56
-; LMULMAX1-RV64-NEXT:    sd a4, 24(sp)
-; LMULMAX1-RV64-NEXT:    vmv.x.s a4, v26
-; LMULMAX1-RV64-NEXT:    addi a2, a4, -1
-; LMULMAX1-RV64-NEXT:    not a4, a4
-; LMULMAX1-RV64-NEXT:    and a2, a4, a2
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v27, a4
+; LMULMAX1-RV64-NEXT:    vmv.x.s a2, v26
+; LMULMAX1-RV64-NEXT:    addi a4, a2, -1
+; LMULMAX1-RV64-NEXT:    not a2, a2
+; LMULMAX1-RV64-NEXT:    and a2, a2, a4
 ; LMULMAX1-RV64-NEXT:    srli a4, a2, 1
 ; LMULMAX1-RV64-NEXT:    and a4, a4, a7
 ; LMULMAX1-RV64-NEXT:    sub a2, a2, a4
@@ -8195,7 +8186,8 @@ define void @cttz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    and a2, a2, a5
 ; LMULMAX1-RV64-NEXT:    mul a2, a2, a1
 ; LMULMAX1-RV64-NEXT:    srli a2, a2, 56
-; LMULMAX1-RV64-NEXT:    sd a2, 16(sp)
+; LMULMAX1-RV64-NEXT:    vmv.s.x v27, a2
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 1, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    vslidedown.vi v26, v25, 1
 ; LMULMAX1-RV64-NEXT:    vmv.x.s a2, v26
 ; LMULMAX1-RV64-NEXT:    addi a4, a2, -1
@@ -8213,7 +8205,8 @@ define void @cttz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    and a2, a2, a5
 ; LMULMAX1-RV64-NEXT:    mul a2, a2, a1
 ; LMULMAX1-RV64-NEXT:    srli a2, a2, 56
-; LMULMAX1-RV64-NEXT:    sd a2, 8(sp)
+; LMULMAX1-RV64-NEXT:    vsetivli a4, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v26, a2
 ; LMULMAX1-RV64-NEXT:    vmv.x.s a2, v25
 ; LMULMAX1-RV64-NEXT:    addi a4, a2, -1
 ; LMULMAX1-RV64-NEXT:    not a2, a2
@@ -8230,14 +8223,9 @@ define void @cttz_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX1-RV64-NEXT:    and a2, a2, a5
 ; LMULMAX1-RV64-NEXT:    mul a1, a2, a1
 ; LMULMAX1-RV64-NEXT:    srli a1, a1, 56
-; LMULMAX1-RV64-NEXT:    sd a1, 0(sp)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (sp)
-; LMULMAX1-RV64-NEXT:    addi a1, sp, 16
-; LMULMAX1-RV64-NEXT:    vle64.v v26, (a1)
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vse64.v v26, (a6)
-; LMULMAX1-RV64-NEXT:    addi sp, sp, 32
+; LMULMAX1-RV64-NEXT:    vmv.s.x v26, a1
+; LMULMAX1-RV64-NEXT:    vse64.v v26, (a0)
+; LMULMAX1-RV64-NEXT:    vse64.v v27, (a6)
 ; LMULMAX1-RV64-NEXT:    ret
   %a = load <4 x i64>, <4 x i64>* %x
   %b = load <4 x i64>, <4 x i64>* %y
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
index 4844ee024b16976d1a171fcb26dc8ec78a73e46c..ea9eba3ea2df87c107112881c0c0db1c13c09dc9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
@@ -19,3 +19,83 @@ define void @buildvec_no_vid_v4f32(<4 x float>* %x) {
   store <4 x float> <float 0.0, float 4.0, float 0.0, float 2.0>, <4 x float>* %x
   ret void
 }
+
+define void @buildvec_dominant0_v4f32(<4 x float>* %x) {
+; CHECK-LABEL: buildvec_dominant0_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a1, %hi(.LCPI1_0)
+; CHECK-NEXT:    flw ft0, %lo(.LCPI1_0)(a1)
+; CHECK-NEXT:    fmv.w.x ft1, zero
+; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vfmv.s.f v25, ft1
+; CHECK-NEXT:    vfmv.v.f v26, ft0
+; CHECK-NEXT:    vsetivli a1, 3, e32,m1,tu,mu
+; CHECK-NEXT:    vslideup.vi v26, v25, 2
+; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vse32.v v26, (a0)
+; CHECK-NEXT:    ret
+  store <4 x float> <float 2.0, float 2.0, float 0.0, float 2.0>, <4 x float>* %x
+  ret void
+}
+
+define void @buildvec_dominant1_v4f32(<4 x float>* %x, float %f) {
+; CHECK-LABEL: buildvec_dominant1_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vfmv.s.f v25, ft0
+; CHECK-NEXT:    vfmv.v.f v26, fa0
+; CHECK-NEXT:    vsetivli a1, 2, e32,m1,tu,mu
+; CHECK-NEXT:    vslideup.vi v26, v25, 1
+; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vse32.v v26, (a0)
+; CHECK-NEXT:    ret
+  %v0 = insertelement <4 x float> undef, float %f, i32 0
+  %v1 = insertelement <4 x float> %v0, float 0.0, i32 1
+  %v2 = insertelement <4 x float> %v1, float %f, i32 2
+  %v3 = insertelement <4 x float> %v2, float %f, i32 3
+  store <4 x float> %v3, <4 x float>* %x
+  ret void
+}
+
+define void @buildvec_dominant2_v4f32(<4 x float>* %x, float %f) {
+; CHECK-LABEL: buildvec_dominant2_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a1, %hi(.LCPI3_0)
+; CHECK-NEXT:    flw ft0, %lo(.LCPI3_0)(a1)
+; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vfmv.s.f v25, ft0
+; CHECK-NEXT:    vfmv.v.f v26, fa0
+; CHECK-NEXT:    vsetivli a1, 2, e32,m1,tu,mu
+; CHECK-NEXT:    vslideup.vi v26, v25, 1
+; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vse32.v v26, (a0)
+; CHECK-NEXT:    ret
+  %v0 = insertelement <4 x float> undef, float %f, i32 0
+  %v1 = insertelement <4 x float> %v0, float 2.0, i32 1
+  %v2 = insertelement <4 x float> %v1, float %f, i32 2
+  %v3 = insertelement <4 x float> %v2, float %f, i32 3
+  store <4 x float> %v3, <4 x float>* %x
+  ret void
+}
+
+define void @buildvec_merge0_v4f32(<4 x float>* %x, float %f) {
+; CHECK-LABEL: buildvec_merge0_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, zero, 6
+; CHECK-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; CHECK-NEXT:    lui a2, %hi(.LCPI4_0)
+; CHECK-NEXT:    flw ft0, %lo(.LCPI4_0)(a2)
+; CHECK-NEXT:    vmv.s.x v0, a1
+; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vfmv.v.f v25, fa0
+; CHECK-NEXT:    vfmerge.vfm v25, v25, ft0, v0
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    ret
+  %v0 = insertelement <4 x float> undef, float %f, i32 0
+  %v1 = insertelement <4 x float> %v0, float 2.0, i32 1
+  %v2 = insertelement <4 x float> %v1, float 2.0, i32 2
+  %v3 = insertelement <4 x float> %v2, float %f, i32 3
+  store <4 x float> %v3, <4 x float>* %x
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d0a8183d0cc9adbbfcdb7aaa1ab8ed3863f80b2c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+experimental-zfh,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: llc -mtriple=riscv64 -mattr=+d,+experimental-zfh,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK
+
+define <4 x half> @shuffle_v4f16(<4 x half> %x, <4 x half> %y) {
+; CHECK-LABEL: shuffle_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 11
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %s = shufflevector <4 x half> %x, <4 x half> %y, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  ret <4 x half> %s
+}
+
+define <8 x float> @shuffle_v8f32(<8 x float> %x, <8 x float> %y) {
+; CHECK-LABEL: shuffle_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 236
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetivli a0, 8, e32,m2,ta,mu
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %s = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 12, i32 5, i32 6, i32 7>
+  ret <8 x float> %s
+}
+
+define <4 x double> @shuffle_fv_v4i16(<4 x double> %x) {
+; CHECK-LABEL: shuffle_fv_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 9
+; CHECK-NEXT:    lui a1, %hi(.LCPI2_0)
+; CHECK-NEXT:    fld ft0, %lo(.LCPI2_0)(a1)
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetivli a0, 4, e64,m2,ta,mu
+; CHECK-NEXT:    vfmerge.vfm v8, v8, ft0, v0
+; CHECK-NEXT:    ret
+  %s = shufflevector <4 x double> <double 2.0, double 2.0, double 2.0, double 2.0>, <4 x double> %x, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  ret <4 x double> %s
+}
+
+define <4 x double> @shuffle_vf_v4i16(<4 x double> %x) {
+; CHECK-LABEL: shuffle_vf_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 6
+; CHECK-NEXT:    lui a1, %hi(.LCPI3_0)
+; CHECK-NEXT:    fld ft0, %lo(.LCPI3_0)(a1)
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetivli a0, 4, e64,m2,ta,mu
+; CHECK-NEXT:    vfmerge.vfm v8, v8, ft0, v0
+; CHECK-NEXT:    ret
+  %s = shufflevector <4 x double> %x, <4 x double> <double 2.0, double 2.0, double 2.0, double 2.0>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  ret <4 x double> %s
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll
index 2aab87fcc3a442cb30d7a15168650fef9a0b901d..869e5e871e5bb7e27529cf500860296636b90989 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll
@@ -243,7 +243,7 @@ define void @fabs_v8f16(<8 x half>* %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
 ; CHECK-NEXT:    vle16.v v25, (a0)
-; CHECK-NEXT:    vfsgnjx.vv v25, v25, v25
+; CHECK-NEXT:    vfabs.v v25, v25
 ; CHECK-NEXT:    vse16.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <8 x half>, <8 x half>* %x
@@ -258,7 +258,7 @@ define void @fabs_v4f32(<4 x float>* %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v25, (a0)
-; CHECK-NEXT:    vfsgnjx.vv v25, v25, v25
+; CHECK-NEXT:    vfabs.v v25, v25
 ; CHECK-NEXT:    vse32.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <4 x float>, <4 x float>* %x
@@ -273,7 +273,7 @@ define void @fabs_v2f64(<2 x double>* %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v25, (a0)
-; CHECK-NEXT:    vfsgnjx.vv v25, v25, v25
+; CHECK-NEXT:    vfabs.v v25, v25
 ; CHECK-NEXT:    vse64.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <2 x double>, <2 x double>* %x
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
index 19db7da60dd2fc88355e91d74d53aae2be7b14f7..43626ca3f5ddc901a807a7471956c08c6aaecb92 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
@@ -43,20 +43,17 @@ define void @insertelt_v4i64(<4 x i64>* %x, i64 %y) {
 define void @insertelt_v3i64(<3 x i64>* %x, i64 %y) {
 ; RV32-LABEL: insertelt_v3i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    addi a3, a0, 16
-; RV32-NEXT:    vsetivli a4, 2, e32,m1,ta,mu
-; RV32-NEXT:    vle32.v v25, (a3)
-; RV32-NEXT:    vse32.v v25, (sp)
 ; RV32-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
 ; RV32-NEXT:    vle64.v v26, (a0)
 ; RV32-NEXT:    vsetivli a3, 8, e32,m2,ta,mu
 ; RV32-NEXT:    vmv.v.i v28, 0
 ; RV32-NEXT:    vsetivli a3, 2, e64,m2,tu,mu
 ; RV32-NEXT:    vslideup.vi v28, v26, 0
-; RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
-; RV32-NEXT:    vle32.v v26, (sp)
+; RV32-NEXT:    lw a3, 20(a0)
+; RV32-NEXT:    vsetivli a4, 4, e32,m1,ta,mu
+; RV32-NEXT:    lw a4, 16(a0)
+; RV32-NEXT:    vmv.v.x v26, a3
+; RV32-NEXT:    vmv.s.x v26, a4
 ; RV32-NEXT:    vsetivli a3, 4, e64,m2,tu,mu
 ; RV32-NEXT:    vslideup.vi v28, v26, 2
 ; RV32-NEXT:    vsetivli a3, 2, e32,m2,ta,mu
@@ -69,7 +66,6 @@ define void @insertelt_v3i64(<3 x i64>* %x, i64 %y) {
 ; RV32-NEXT:    vse64.v v28, (a0)
 ; RV32-NEXT:    sw a1, 16(a0)
 ; RV32-NEXT:    sw a2, 20(a0)
-; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v3i64:
@@ -181,7 +177,7 @@ define void @insertelt_v8i64_0(<8 x i64>* %x) {
 ; RV32-NEXT:    vle64.v v28, (a0)
 ; RV32-NEXT:    addi a1, zero, -1
 ; RV32-NEXT:    vmv.s.x v28, a1
-; RV32-NEXT:    vs4r.v v28, (a0)
+; RV32-NEXT:    vse64.v v28, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_v8i64_0:
@@ -238,7 +234,7 @@ define void @insertelt_c6_v8i64_0(<8 x i64>* %x) {
 ; RV32-NEXT:    vle64.v v28, (a0)
 ; RV32-NEXT:    addi a1, zero, 6
 ; RV32-NEXT:    vmv.s.x v28, a1
-; RV32-NEXT:    vs4r.v v28, (a0)
+; RV32-NEXT:    vse64.v v28, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: insertelt_c6_v8i64_0:
@@ -287,3 +283,35 @@ define void @insertelt_c6_v8i64(<8 x i64>* %x, i32 %idx) {
   store <8 x i64> %b, <8 x i64>* %x
   ret void
 }
+
+; Test that using a insertelement at element 0 by a later operation doesn't
+; crash the compiler.
+define void @insertelt_c6_v8i64_0_add(<8 x i64>* %x, <8 x i64>* %y) {
+; RV32-LABEL: insertelt_c6_v8i64_0_add:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a2, 8, e64,m4,ta,mu
+; RV32-NEXT:    vle64.v v28, (a0)
+; RV32-NEXT:    vle64.v v8, (a1)
+; RV32-NEXT:    addi a1, zero, 6
+; RV32-NEXT:    vmv.s.x v28, a1
+; RV32-NEXT:    vadd.vv v28, v28, v8
+; RV32-NEXT:    vse64.v v28, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: insertelt_c6_v8i64_0_add:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a2, 8, e64,m4,ta,mu
+; RV64-NEXT:    vle64.v v28, (a0)
+; RV64-NEXT:    vle64.v v8, (a1)
+; RV64-NEXT:    addi a1, zero, 6
+; RV64-NEXT:    vmv.s.x v28, a1
+; RV64-NEXT:    vadd.vv v28, v28, v8
+; RV64-NEXT:    vse64.v v28, (a0)
+; RV64-NEXT:    ret
+  %a = load <8 x i64>, <8 x i64>* %x
+  %b = insertelement <8 x i64> %a, i64 6, i32 0
+  %c = load <8 x i64>, <8 x i64>* %y
+  %d = add <8 x i64> %b, %c
+  store <8 x i64> %d, <8 x i64>* %x
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
index c862056d55105ba83dd2e73af72fab6890afdca7..c90e0d1aacd0b132c18b79069ec511ebe91cc551 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 define void @buildvec_vid_v16i8(<16 x i8>* %x) {
 ; CHECK-LABEL: buildvec_vid_v16i8:
@@ -65,3 +65,247 @@ define void @buildvec_vid_mpy_imm_v16i8(<16 x i8>* %x) {
   store <16 x i8> <i8 0, i8 3, i8 6, i8 9, i8 12, i8 15, i8 18, i8 21, i8 24, i8 27, i8 30, i8 33, i8 36, i8 39, i8 42, i8 45>, <16 x i8>* %x
   ret void
 }
+
+define void @buildvec_dominant0_v8i16(<8 x i16>* %x) {
+; CHECK-LABEL: buildvec_dominant0_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v25, zero
+; CHECK-NEXT:    vmv.v.i v26, 8
+; CHECK-NEXT:    vsetivli a1, 4, e16,m1,tu,mu
+; CHECK-NEXT:    vslideup.vi v26, v25, 3
+; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vse16.v v26, (a0)
+; CHECK-NEXT:    ret
+  store <8 x i16> <i16 8, i16 8, i16 undef, i16 0, i16 8, i16 undef, i16 8, i16 8>, <8 x i16>* %x
+  ret void
+}
+
+define void @buildvec_dominant1_v8i16(<8 x i16>* %x) {
+; CHECK-LABEL: buildvec_dominant1_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, 8
+; CHECK-NEXT:    vse16.v v25, (a0)
+; CHECK-NEXT:    ret
+  store <8 x i16> <i16 undef, i16 8, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef>, <8 x i16>* %x
+  ret void
+}
+
+define void @buildvec_dominant0_v2i8(<2 x i8>* %x) {
+; CHECK-LABEL: buildvec_dominant0_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  store <2 x i8> <i8 undef, i8 undef>, <2 x i8>* %x
+  ret void
+}
+
+define void @buildvec_dominant1_v2i8(<2 x i8>* %x) {
+; CHECK-LABEL: buildvec_dominant1_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -1
+; CHECK-NEXT:    vse8.v v25, (a0)
+; CHECK-NEXT:    ret
+  store <2 x i8> <i8 undef, i8 -1>, <2 x i8>* %x
+  ret void
+}
+
+define void @buildvec_dominant2_v2i8(<2 x i8>* %x) {
+; CHECK-LABEL: buildvec_dominant2_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -1
+; CHECK-NEXT:    vmv.s.x v25, zero
+; CHECK-NEXT:    vse8.v v25, (a0)
+; CHECK-NEXT:    ret
+  store <2 x i8> <i8 0, i8 -1>, <2 x i8>* %x
+  ret void
+}
+
+define void @buildvec_dominant0_v2i32(<2 x i64>* %x) {
+; RV32-LABEL: buildvec_dominant0_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a1, %hi(.LCPI10_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI10_0)
+; RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; RV32-NEXT:    vle32.v v25, (a1)
+; RV32-NEXT:    vse32.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: buildvec_dominant0_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV64-NEXT:    vmv.v.i v25, -1
+; RV64-NEXT:    lui a1, 3641
+; RV64-NEXT:    addiw a1, a1, -455
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -455
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -455
+; RV64-NEXT:    slli a1, a1, 13
+; RV64-NEXT:    addi a1, a1, -910
+; RV64-NEXT:    vmv.s.x v25, a1
+; RV64-NEXT:    vse64.v v25, (a0)
+; RV64-NEXT:    ret
+  store <2 x i64> <i64 2049638230412172402, i64 -1>, <2 x i64>* %x
+  ret void
+}
+
+define void @buildvec_dominant1_optsize_v2i32(<2 x i64>* %x) optsize {
+; RV32-LABEL: buildvec_dominant1_optsize_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a1, %hi(.LCPI11_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI11_0)
+; RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; RV32-NEXT:    vle32.v v25, (a1)
+; RV32-NEXT:    vse32.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: buildvec_dominant1_optsize_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    lui a1, %hi(.LCPI11_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI11_0)
+; RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a1)
+; RV64-NEXT:    vse64.v v25, (a0)
+; RV64-NEXT:    ret
+  store <2 x i64> <i64 2049638230412172402, i64 -1>, <2 x i64>* %x
+  ret void
+}
+
+define void @buildvec_seq_v8i8_v4i16(<8 x i8>* %x) {
+; CHECK-LABEL: buildvec_seq_v8i8_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, zero, 513
+; CHECK-NEXT:    vsetivli a2, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.v.x v25, a1
+; CHECK-NEXT:    vsetivli a1, 8, e8,m1,ta,mu
+; CHECK-NEXT:    vse8.v v25, (a0)
+; CHECK-NEXT:    ret
+  store <8 x i8> <i8 1, i8 2, i8 1, i8 2, i8 1, i8 2, i8 undef, i8 2>, <8 x i8>* %x
+  ret void
+}
+
+define void @buildvec_seq_v8i8_v2i32(<8 x i8>* %x) {
+; RV32-LABEL: buildvec_seq_v8i8_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a1, 48
+; RV32-NEXT:    addi a1, a1, 513
+; RV32-NEXT:    vsetivli a2, 2, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.x v25, a1
+; RV32-NEXT:    vsetivli a1, 8, e8,m1,ta,mu
+; RV32-NEXT:    vse8.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: buildvec_seq_v8i8_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    lui a1, 48
+; RV64-NEXT:    addiw a1, a1, 513
+; RV64-NEXT:    vsetivli a2, 2, e32,m1,ta,mu
+; RV64-NEXT:    vmv.v.x v25, a1
+; RV64-NEXT:    vsetivli a1, 8, e8,m1,ta,mu
+; RV64-NEXT:    vse8.v v25, (a0)
+; RV64-NEXT:    ret
+  store <8 x i8> <i8 1, i8 2, i8 3, i8 undef, i8 1, i8 2, i8 3, i8 undef>, <8 x i8>* %x
+  ret void
+}
+
+define void @buildvec_seq_v16i8_v2i64(<16 x i8>* %x) {
+; RV32-LABEL: buildvec_seq_v16i8_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a1, %hi(.LCPI14_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI14_0)
+; RV32-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; RV32-NEXT:    vle8.v v25, (a1)
+; RV32-NEXT:    vse8.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: buildvec_seq_v16i8_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    lui a1, 32880
+; RV64-NEXT:    addiw a1, a1, 1541
+; RV64-NEXT:    slli a1, a1, 16
+; RV64-NEXT:    addi a1, a1, 1027
+; RV64-NEXT:    slli a1, a1, 16
+; RV64-NEXT:    addi a1, a1, 513
+; RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; RV64-NEXT:    vmv.v.x v25, a1
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV64-NEXT:    vse8.v v25, (a0)
+; RV64-NEXT:    ret
+  store <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8>, <16 x i8>* %x
+  ret void
+}
+
+define void @buildvec_seq2_v16i8_v2i64(<16 x i8>* %x) {
+; RV32-LABEL: buildvec_seq2_v16i8_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a1, 528432
+; RV32-NEXT:    addi a1, a1, 513
+; RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; RV32-NEXT:    vmv.v.x v25, a1
+; RV32-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV32-NEXT:    vse8.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: buildvec_seq2_v16i8_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    lui a1, 528432
+; RV64-NEXT:    addiw a1, a1, 513
+; RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; RV64-NEXT:    vmv.v.x v25, a1
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV64-NEXT:    vse8.v v25, (a0)
+; RV64-NEXT:    ret
+  store <16 x i8> <i8 1, i8 2, i8 3, i8 129, i8 -1, i8 -1, i8 -1, i8 -1, i8 1, i8 2, i8 3, i8 129, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8>* %x
+  ret void
+}
+
+define void @buildvec_seq_v9i8(<9 x i8>* %x) {
+; RV32-LABEL: buildvec_seq_v9i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi a1, zero, 73
+; RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    vsetivli a1, 8, e8,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v25, 2
+; RV32-NEXT:    vmerge.vim v25, v25, 1, v0
+; RV32-NEXT:    addi a1, zero, 36
+; RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    vsetivli a1, 8, e8,m1,ta,mu
+; RV32-NEXT:    vmerge.vim v25, v25, 3, v0
+; RV32-NEXT:    vse8.v v25, (a0)
+; RV32-NEXT:    addi a1, zero, 3
+; RV32-NEXT:    sb a1, 8(a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: buildvec_seq_v9i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi a1, zero, 3
+; RV64-NEXT:    sb a1, 8(a0)
+; RV64-NEXT:    lui a1, 4104
+; RV64-NEXT:    addiw a1, a1, 385
+; RV64-NEXT:    slli a1, a1, 17
+; RV64-NEXT:    addi a1, a1, 259
+; RV64-NEXT:    slli a1, a1, 16
+; RV64-NEXT:    addi a1, a1, 513
+; RV64-NEXT:    sd a1, 0(a0)
+; RV64-NEXT:    ret
+  store <9 x i8> <i8 1, i8 2, i8 3, i8 1, i8 2, i8 3, i8 1, i8 2, i8 3>, <9 x i8>* %x
+  ret void
+}
+
+define void @buildvec_seq_v4i16_v2i32(<4 x i16>* %x) {
+; CHECK-LABEL: buildvec_seq_v4i16_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, zero, -127
+; CHECK-NEXT:    vsetivli a2, 2, e32,m1,ta,mu
+; CHECK-NEXT:    vmv.v.x v25, a1
+; CHECK-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vse16.v v25, (a0)
+; CHECK-NEXT:    ret
+  store <4 x i16> <i16 -127, i16 -1, i16 -127, i16 -1>, <4 x i16>* %x
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
new file mode 100644
index 0000000000000000000000000000000000000000..94bc6e9dc8fd1329d5143a9c4634c2730dbf0a6b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
+
+define <4 x i16> @shuffle_v4i16(<4 x i16> %x, <4 x i16> %y) {
+; CHECK-LABEL: shuffle_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 11
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vmerge.vvm v8, v9, v8, v0
+; CHECK-NEXT:    ret
+  %s = shufflevector <4 x i16> %x, <4 x i16> %y, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  ret <4 x i16> %s
+}
+
+define <8 x i32> @shuffle_v8i32(<8 x i32> %x, <8 x i32> %y) {
+; CHECK-LABEL: shuffle_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 203
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetivli a0, 8, e32,m2,ta,mu
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %s = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 0, i32 1, i32 10, i32 3, i32 12, i32 13, i32 6, i32 7>
+  ret <8 x i32> %s
+}
+
+define <4 x i16> @shuffle_xv_v4i16(<4 x i16> %x) {
+; CHECK-LABEL: shuffle_xv_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 9
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vmerge.vim v8, v8, 5, v0
+; CHECK-NEXT:    ret
+  %s = shufflevector <4 x i16> <i16 5, i16 5, i16 5, i16 5>, <4 x i16> %x, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  ret <4 x i16> %s
+}
+
+define <4 x i16> @shuffle_vx_v4i16(<4 x i16> %x) {
+; CHECK-LABEL: shuffle_vx_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 6
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vmerge.vim v8, v8, 5, v0
+; CHECK-NEXT:    ret
+  %s = shufflevector <4 x i16> %x, <4 x i16> <i16 5, i16 5, i16 5, i16 5>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  ret <4 x i16> %s
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-splat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-splat.ll
index 619ba8fdab13c70c54b90ef298835da23a1d4c9b..6f1f72a4f07a05dc1a611252c470df10911b8dea 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-splat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-splat.ll
@@ -48,44 +48,35 @@ define void @splat_v4i32(<4 x i32>* %x, i32 %y) {
 define void @splat_v2i64(<2 x i64>* %x, i64 %y) {
 ; LMULMAX8-RV32-LABEL: splat_v2i64:
 ; LMULMAX8-RV32:       # %bb.0:
-; LMULMAX8-RV32-NEXT:    addi sp, sp, -16
-; LMULMAX8-RV32-NEXT:    .cfi_def_cfa_offset 16
-; LMULMAX8-RV32-NEXT:    sw a2, 12(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 8(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 4(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 0(sp)
-; LMULMAX8-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX8-RV32-NEXT:    vle32.v v25, (sp)
+; LMULMAX8-RV32-NEXT:    addi a3, zero, 5
+; LMULMAX8-RV32-NEXT:    vsetivli a4, 1, e8,m1,ta,mu
+; LMULMAX8-RV32-NEXT:    vmv.s.x v0, a3
+; LMULMAX8-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; LMULMAX8-RV32-NEXT:    vmv.v.x v25, a2
+; LMULMAX8-RV32-NEXT:    vmerge.vxm v25, v25, a1, v0
 ; LMULMAX8-RV32-NEXT:    vse32.v v25, (a0)
-; LMULMAX8-RV32-NEXT:    addi sp, sp, 16
 ; LMULMAX8-RV32-NEXT:    ret
 ;
 ; LMULMAX2-RV32-LABEL: splat_v2i64:
 ; LMULMAX2-RV32:       # %bb.0:
-; LMULMAX2-RV32-NEXT:    addi sp, sp, -16
-; LMULMAX2-RV32-NEXT:    .cfi_def_cfa_offset 16
-; LMULMAX2-RV32-NEXT:    sw a2, 12(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 8(sp)
-; LMULMAX2-RV32-NEXT:    sw a2, 4(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 0(sp)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v25, (sp)
+; LMULMAX2-RV32-NEXT:    addi a3, zero, 5
+; LMULMAX2-RV32-NEXT:    vsetivli a4, 1, e8,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a3
+; LMULMAX2-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v25, a2
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v25, v25, a1, v0
 ; LMULMAX2-RV32-NEXT:    vse32.v v25, (a0)
-; LMULMAX2-RV32-NEXT:    addi sp, sp, 16
 ; LMULMAX2-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV32-LABEL: splat_v2i64:
 ; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    addi sp, sp, -16
-; LMULMAX1-RV32-NEXT:    .cfi_def_cfa_offset 16
-; LMULMAX1-RV32-NEXT:    sw a2, 12(sp)
-; LMULMAX1-RV32-NEXT:    sw a1, 8(sp)
-; LMULMAX1-RV32-NEXT:    sw a2, 4(sp)
-; LMULMAX1-RV32-NEXT:    sw a1, 0(sp)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v25, (sp)
+; LMULMAX1-RV32-NEXT:    addi a3, zero, 5
+; LMULMAX1-RV32-NEXT:    vsetivli a4, 1, e8,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.s.x v0, a3
+; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.x v25, a2
+; LMULMAX1-RV32-NEXT:    vmerge.vxm v25, v25, a1, v0
 ; LMULMAX1-RV32-NEXT:    vse32.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    addi sp, sp, 16
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX8-RV64-LABEL: splat_v2i64:
@@ -206,74 +197,37 @@ define void @splat_v8i32(<8 x i32>* %x, i32 %y) {
 define void @splat_v4i64(<4 x i64>* %x, i64 %y) {
 ; LMULMAX8-RV32-LABEL: splat_v4i64:
 ; LMULMAX8-RV32:       # %bb.0:
-; LMULMAX8-RV32-NEXT:    addi sp, sp, -64
-; LMULMAX8-RV32-NEXT:    .cfi_def_cfa_offset 64
-; LMULMAX8-RV32-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; LMULMAX8-RV32-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; LMULMAX8-RV32-NEXT:    .cfi_offset ra, -4
-; LMULMAX8-RV32-NEXT:    .cfi_offset s0, -8
-; LMULMAX8-RV32-NEXT:    addi s0, sp, 64
-; LMULMAX8-RV32-NEXT:    .cfi_def_cfa s0, 0
-; LMULMAX8-RV32-NEXT:    andi sp, sp, -32
-; LMULMAX8-RV32-NEXT:    sw a2, 28(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 24(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 20(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 16(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 12(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 8(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 4(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 0(sp)
-; LMULMAX8-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
-; LMULMAX8-RV32-NEXT:    vle32.v v26, (sp)
+; LMULMAX8-RV32-NEXT:    addi a3, zero, 85
+; LMULMAX8-RV32-NEXT:    vsetivli a4, 1, e8,m1,ta,mu
+; LMULMAX8-RV32-NEXT:    vmv.s.x v0, a3
+; LMULMAX8-RV32-NEXT:    vsetivli a3, 8, e32,m2,ta,mu
+; LMULMAX8-RV32-NEXT:    vmv.v.x v26, a2
+; LMULMAX8-RV32-NEXT:    vmerge.vxm v26, v26, a1, v0
 ; LMULMAX8-RV32-NEXT:    vse32.v v26, (a0)
-; LMULMAX8-RV32-NEXT:    addi sp, s0, -64
-; LMULMAX8-RV32-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; LMULMAX8-RV32-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; LMULMAX8-RV32-NEXT:    addi sp, sp, 64
 ; LMULMAX8-RV32-NEXT:    ret
 ;
 ; LMULMAX2-RV32-LABEL: splat_v4i64:
 ; LMULMAX2-RV32:       # %bb.0:
-; LMULMAX2-RV32-NEXT:    addi sp, sp, -64
-; LMULMAX2-RV32-NEXT:    .cfi_def_cfa_offset 64
-; LMULMAX2-RV32-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; LMULMAX2-RV32-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; LMULMAX2-RV32-NEXT:    .cfi_offset ra, -4
-; LMULMAX2-RV32-NEXT:    .cfi_offset s0, -8
-; LMULMAX2-RV32-NEXT:    addi s0, sp, 64
-; LMULMAX2-RV32-NEXT:    .cfi_def_cfa s0, 0
-; LMULMAX2-RV32-NEXT:    andi sp, sp, -32
-; LMULMAX2-RV32-NEXT:    sw a2, 28(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 24(sp)
-; LMULMAX2-RV32-NEXT:    sw a2, 20(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 16(sp)
-; LMULMAX2-RV32-NEXT:    sw a2, 12(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 8(sp)
-; LMULMAX2-RV32-NEXT:    sw a2, 4(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 0(sp)
-; LMULMAX2-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v26, (sp)
+; LMULMAX2-RV32-NEXT:    addi a3, zero, 85
+; LMULMAX2-RV32-NEXT:    vsetivli a4, 1, e8,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a3
+; LMULMAX2-RV32-NEXT:    vsetivli a3, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v26, a2
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v26, v26, a1, v0
 ; LMULMAX2-RV32-NEXT:    vse32.v v26, (a0)
-; LMULMAX2-RV32-NEXT:    addi sp, s0, -64
-; LMULMAX2-RV32-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; LMULMAX2-RV32-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; LMULMAX2-RV32-NEXT:    addi sp, sp, 64
 ; LMULMAX2-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV32-LABEL: splat_v4i64:
 ; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    addi sp, sp, -16
-; LMULMAX1-RV32-NEXT:    .cfi_def_cfa_offset 16
-; LMULMAX1-RV32-NEXT:    sw a2, 12(sp)
-; LMULMAX1-RV32-NEXT:    sw a1, 8(sp)
-; LMULMAX1-RV32-NEXT:    sw a2, 4(sp)
-; LMULMAX1-RV32-NEXT:    sw a1, 0(sp)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v25, (sp)
+; LMULMAX1-RV32-NEXT:    addi a3, zero, 5
+; LMULMAX1-RV32-NEXT:    vsetivli a4, 1, e8,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.s.x v0, a3
+; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.x v25, a2
+; LMULMAX1-RV32-NEXT:    vmerge.vxm v25, v25, a1, v0
 ; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
 ; LMULMAX1-RV32-NEXT:    vse32.v v25, (a1)
 ; LMULMAX1-RV32-NEXT:    vse32.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    addi sp, sp, 16
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX8-RV64-LABEL: splat_v4i64:
@@ -842,72 +796,23 @@ define void @splat_allones_with_use_v4i64(<4 x i64>* %x) {
 define void @vadd_vx_v16i64(<16 x i64>* %a, i64 %b, <16 x i64>* %c) {
 ; LMULMAX8-RV32-LABEL: vadd_vx_v16i64:
 ; LMULMAX8-RV32:       # %bb.0:
-; LMULMAX8-RV32-NEXT:    addi sp, sp, -256
-; LMULMAX8-RV32-NEXT:    .cfi_def_cfa_offset 256
-; LMULMAX8-RV32-NEXT:    sw ra, 252(sp) # 4-byte Folded Spill
-; LMULMAX8-RV32-NEXT:    sw s0, 248(sp) # 4-byte Folded Spill
-; LMULMAX8-RV32-NEXT:    .cfi_offset ra, -4
-; LMULMAX8-RV32-NEXT:    .cfi_offset s0, -8
-; LMULMAX8-RV32-NEXT:    addi s0, sp, 256
-; LMULMAX8-RV32-NEXT:    .cfi_def_cfa s0, 0
-; LMULMAX8-RV32-NEXT:    andi sp, sp, -128
 ; LMULMAX8-RV32-NEXT:    vsetivli a4, 16, e64,m8,ta,mu
 ; LMULMAX8-RV32-NEXT:    vle64.v v8, (a0)
-; LMULMAX8-RV32-NEXT:    sw a2, 124(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 120(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 116(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 112(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 108(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 104(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 100(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 96(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 92(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 88(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 84(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 80(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 76(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 72(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 68(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 64(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 60(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 56(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 52(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 48(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 44(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 40(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 36(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 32(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 28(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 24(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 20(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 16(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 12(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 8(sp)
-; LMULMAX8-RV32-NEXT:    sw a2, 4(sp)
-; LMULMAX8-RV32-NEXT:    sw a1, 0(sp)
+; LMULMAX8-RV32-NEXT:    lui a0, 349525
+; LMULMAX8-RV32-NEXT:    addi a0, a0, 1365
+; LMULMAX8-RV32-NEXT:    vsetivli a4, 1, e32,m1,ta,mu
+; LMULMAX8-RV32-NEXT:    vmv.s.x v0, a0
 ; LMULMAX8-RV32-NEXT:    addi a0, zero, 32
 ; LMULMAX8-RV32-NEXT:    vsetvli a0, a0, e32,m8,ta,mu
-; LMULMAX8-RV32-NEXT:    vle32.v v16, (sp)
+; LMULMAX8-RV32-NEXT:    vmv.v.x v16, a2
+; LMULMAX8-RV32-NEXT:    vmerge.vxm v16, v16, a1, v0
 ; LMULMAX8-RV32-NEXT:    vsetivli a0, 16, e64,m8,ta,mu
 ; LMULMAX8-RV32-NEXT:    vadd.vv v8, v8, v16
 ; LMULMAX8-RV32-NEXT:    vse64.v v8, (a3)
-; LMULMAX8-RV32-NEXT:    addi sp, s0, -256
-; LMULMAX8-RV32-NEXT:    lw s0, 248(sp) # 4-byte Folded Reload
-; LMULMAX8-RV32-NEXT:    lw ra, 252(sp) # 4-byte Folded Reload
-; LMULMAX8-RV32-NEXT:    addi sp, sp, 256
 ; LMULMAX8-RV32-NEXT:    ret
 ;
 ; LMULMAX2-RV32-LABEL: vadd_vx_v16i64:
 ; LMULMAX2-RV32:       # %bb.0:
-; LMULMAX2-RV32-NEXT:    addi sp, sp, -64
-; LMULMAX2-RV32-NEXT:    .cfi_def_cfa_offset 64
-; LMULMAX2-RV32-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; LMULMAX2-RV32-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; LMULMAX2-RV32-NEXT:    .cfi_offset ra, -4
-; LMULMAX2-RV32-NEXT:    .cfi_offset s0, -8
-; LMULMAX2-RV32-NEXT:    addi s0, sp, 64
-; LMULMAX2-RV32-NEXT:    .cfi_def_cfa s0, 0
-; LMULMAX2-RV32-NEXT:    andi sp, sp, -32
 ; LMULMAX2-RV32-NEXT:    addi a4, a0, 64
 ; LMULMAX2-RV32-NEXT:    vsetivli a5, 4, e64,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vle64.v v26, (a4)
@@ -916,16 +821,12 @@ define void @vadd_vx_v16i64(<16 x i64>* %a, i64 %b, <16 x i64>* %c) {
 ; LMULMAX2-RV32-NEXT:    vle64.v v30, (a0)
 ; LMULMAX2-RV32-NEXT:    addi a0, a0, 32
 ; LMULMAX2-RV32-NEXT:    vle64.v v8, (a0)
-; LMULMAX2-RV32-NEXT:    sw a2, 28(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 24(sp)
-; LMULMAX2-RV32-NEXT:    sw a2, 20(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 16(sp)
-; LMULMAX2-RV32-NEXT:    sw a2, 12(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 8(sp)
-; LMULMAX2-RV32-NEXT:    sw a2, 4(sp)
-; LMULMAX2-RV32-NEXT:    sw a1, 0(sp)
+; LMULMAX2-RV32-NEXT:    addi a0, zero, 85
+; LMULMAX2-RV32-NEXT:    vsetivli a4, 1, e8,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a0
 ; LMULMAX2-RV32-NEXT:    vsetivli a0, 8, e32,m2,ta,mu
-; LMULMAX2-RV32-NEXT:    vle32.v v10, (sp)
+; LMULMAX2-RV32-NEXT:    vmv.v.x v10, a2
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v10, v10, a1, v0
 ; LMULMAX2-RV32-NEXT:    vsetivli a0, 4, e64,m2,ta,mu
 ; LMULMAX2-RV32-NEXT:    vadd.vv v8, v8, v10
 ; LMULMAX2-RV32-NEXT:    vadd.vv v30, v30, v10
@@ -938,16 +839,10 @@ define void @vadd_vx_v16i64(<16 x i64>* %a, i64 %b, <16 x i64>* %c) {
 ; LMULMAX2-RV32-NEXT:    vse64.v v30, (a3)
 ; LMULMAX2-RV32-NEXT:    addi a0, a3, 32
 ; LMULMAX2-RV32-NEXT:    vse64.v v8, (a0)
-; LMULMAX2-RV32-NEXT:    addi sp, s0, -64
-; LMULMAX2-RV32-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; LMULMAX2-RV32-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; LMULMAX2-RV32-NEXT:    addi sp, sp, 64
 ; LMULMAX2-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV32-LABEL: vadd_vx_v16i64:
 ; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    addi sp, sp, -16
-; LMULMAX1-RV32-NEXT:    .cfi_def_cfa_offset 16
 ; LMULMAX1-RV32-NEXT:    addi a4, a0, 96
 ; LMULMAX1-RV32-NEXT:    vsetivli a5, 2, e64,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vle64.v v25, (a4)
@@ -964,12 +859,12 @@ define void @vadd_vx_v16i64(<16 x i64>* %a, i64 %b, <16 x i64>* %c) {
 ; LMULMAX1-RV32-NEXT:    vle64.v v31, (a0)
 ; LMULMAX1-RV32-NEXT:    addi a0, a0, 16
 ; LMULMAX1-RV32-NEXT:    vle64.v v8, (a0)
-; LMULMAX1-RV32-NEXT:    sw a2, 12(sp)
-; LMULMAX1-RV32-NEXT:    sw a1, 8(sp)
-; LMULMAX1-RV32-NEXT:    sw a2, 4(sp)
-; LMULMAX1-RV32-NEXT:    sw a1, 0(sp)
+; LMULMAX1-RV32-NEXT:    addi a0, zero, 5
+; LMULMAX1-RV32-NEXT:    vsetivli a4, 1, e8,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.s.x v0, a0
 ; LMULMAX1-RV32-NEXT:    vsetivli a0, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v9, (sp)
+; LMULMAX1-RV32-NEXT:    vmv.v.x v9, a2
+; LMULMAX1-RV32-NEXT:    vmerge.vxm v9, v9, a1, v0
 ; LMULMAX1-RV32-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vadd.vv v8, v8, v9
 ; LMULMAX1-RV32-NEXT:    vadd.vv v31, v31, v9
@@ -994,7 +889,6 @@ define void @vadd_vx_v16i64(<16 x i64>* %a, i64 %b, <16 x i64>* %c) {
 ; LMULMAX1-RV32-NEXT:    vse64.v v31, (a3)
 ; LMULMAX1-RV32-NEXT:    addi a0, a3, 16
 ; LMULMAX1-RV32-NEXT:    vse64.v v8, (a0)
-; LMULMAX1-RV32-NEXT:    addi sp, sp, 16
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX8-RV64-LABEL: vadd_vx_v16i64:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
index f3fe451ba44a156d145492ccc0075a41600009ac..9de8c531fb03cac68dff201ce47b2264a2b0415d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1-RV32
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1-RV64
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,LMULMAX2,LMULMAX2-RV32
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,LMULMAX2,LMULMAX2-RV64
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,LMULMAX1,LMULMAX1-RV32
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,LMULMAX1,LMULMAX1-RV64
 
 define void @add_v16i8(<16 x i8>* %x, <16 x i8>* %y) {
 ; CHECK-LABEL: add_v16i8:
@@ -837,30 +837,99 @@ define void @urem_v2i64(<2 x i64>* %x, <2 x i64>* %y) {
 }
 
 define void @mulhu_v16i8(<16 x i8>* %x) {
-; CHECK-LABEL: mulhu_v16i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
-; CHECK-NEXT:    vle8.v v25, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI52_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI52_0)
-; CHECK-NEXT:    vle8.v v26, (a1)
-; CHECK-NEXT:    lui a1, %hi(.LCPI52_1)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI52_1)
-; CHECK-NEXT:    vle8.v v27, (a1)
-; CHECK-NEXT:    vsrl.vv v26, v25, v26
-; CHECK-NEXT:    vmulhu.vv v26, v26, v27
-; CHECK-NEXT:    lui a1, %hi(.LCPI52_2)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI52_2)
-; CHECK-NEXT:    vle8.v v27, (a1)
-; CHECK-NEXT:    lui a1, %hi(.LCPI52_3)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI52_3)
-; CHECK-NEXT:    vle8.v v28, (a1)
-; CHECK-NEXT:    vsub.vv v25, v25, v26
-; CHECK-NEXT:    vmulhu.vv v25, v25, v27
-; CHECK-NEXT:    vadd.vv v25, v25, v26
-; CHECK-NEXT:    vsrl.vv v25, v25, v28
-; CHECK-NEXT:    vse8.v v25, (a0)
-; CHECK-NEXT:    ret
+; RV32-LABEL: mulhu_v16i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV32-NEXT:    vle8.v v25, (a0)
+; RV32-NEXT:    addi a1, zero, 513
+; RV32-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v26, 4
+; RV32-NEXT:    vmerge.vim v26, v26, 1, v0
+; RV32-NEXT:    lui a1, 1
+; RV32-NEXT:    addi a2, a1, 78
+; RV32-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a2
+; RV32-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; RV32-NEXT:    vmerge.vim v26, v26, 3, v0
+; RV32-NEXT:    lui a2, 8
+; RV32-NEXT:    addi a2, a2, 304
+; RV32-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a2
+; RV32-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; RV32-NEXT:    vmerge.vim v26, v26, 2, v0
+; RV32-NEXT:    lui a2, 3
+; RV32-NEXT:    addi a2, a2, -2044
+; RV32-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a2
+; RV32-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v27, 0
+; RV32-NEXT:    addi a2, zero, -128
+; RV32-NEXT:    vmerge.vxm v28, v27, a2, v0
+; RV32-NEXT:    addi a1, a1, 32
+; RV32-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV32-NEXT:    lui a1, %hi(.LCPI52_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI52_0)
+; RV32-NEXT:    vle8.v v29, (a1)
+; RV32-NEXT:    vmerge.vim v27, v27, 1, v0
+; RV32-NEXT:    vsrl.vv v27, v25, v27
+; RV32-NEXT:    vmulhu.vv v27, v27, v29
+; RV32-NEXT:    vsub.vv v25, v25, v27
+; RV32-NEXT:    vmulhu.vv v25, v25, v28
+; RV32-NEXT:    vadd.vv v25, v25, v27
+; RV32-NEXT:    vsrl.vv v25, v25, v26
+; RV32-NEXT:    vse8.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhu_v16i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV64-NEXT:    vle8.v v25, (a0)
+; RV64-NEXT:    addi a1, zero, 513
+; RV64-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; RV64-NEXT:    vmv.s.x v0, a1
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV64-NEXT:    vmv.v.i v26, 4
+; RV64-NEXT:    vmerge.vim v26, v26, 1, v0
+; RV64-NEXT:    lui a1, 1
+; RV64-NEXT:    addiw a2, a1, 78
+; RV64-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; RV64-NEXT:    vmv.s.x v0, a2
+; RV64-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; RV64-NEXT:    vmerge.vim v26, v26, 3, v0
+; RV64-NEXT:    lui a2, 8
+; RV64-NEXT:    addiw a2, a2, 304
+; RV64-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; RV64-NEXT:    vmv.s.x v0, a2
+; RV64-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; RV64-NEXT:    vmerge.vim v26, v26, 2, v0
+; RV64-NEXT:    lui a2, 3
+; RV64-NEXT:    addiw a2, a2, -2044
+; RV64-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; RV64-NEXT:    vmv.s.x v0, a2
+; RV64-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; RV64-NEXT:    vmv.v.i v27, 0
+; RV64-NEXT:    addi a2, zero, -128
+; RV64-NEXT:    vmerge.vxm v28, v27, a2, v0
+; RV64-NEXT:    addiw a1, a1, 32
+; RV64-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; RV64-NEXT:    vmv.s.x v0, a1
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV64-NEXT:    lui a1, %hi(.LCPI52_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI52_0)
+; RV64-NEXT:    vle8.v v29, (a1)
+; RV64-NEXT:    vmerge.vim v27, v27, 1, v0
+; RV64-NEXT:    vsrl.vv v27, v25, v27
+; RV64-NEXT:    vmulhu.vv v27, v27, v29
+; RV64-NEXT:    vsub.vv v25, v25, v27
+; RV64-NEXT:    vmulhu.vv v25, v25, v28
+; RV64-NEXT:    vadd.vv v25, v25, v27
+; RV64-NEXT:    vsrl.vv v25, v25, v26
+; RV64-NEXT:    vse8.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <16 x i8>, <16 x i8>* %x
   %b = udiv <16 x i8> %a, <i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25>
   store <16 x i8> %b, <16 x i8>* %x
@@ -872,24 +941,33 @@ define void @mulhu_v8i16(<8 x i16>* %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
 ; CHECK-NEXT:    vle16.v v25, (a0)
+; CHECK-NEXT:    addi a1, zero, 1
+; CHECK-NEXT:    vmv.s.x v26, a1
+; CHECK-NEXT:    addi a1, zero, 33
+; CHECK-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a1
+; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v27, 3
+; CHECK-NEXT:    vmerge.vim v27, v27, 2, v0
+; CHECK-NEXT:    vsetivli a1, 7, e16,m1,tu,mu
+; CHECK-NEXT:    vslideup.vi v27, v26, 6
+; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v28, 0
+; CHECK-NEXT:    lui a1, 1048568
+; CHECK-NEXT:    vmv1r.v v29, v28
+; CHECK-NEXT:    vmv.s.x v29, a1
+; CHECK-NEXT:    vsetivli a1, 7, e16,m1,tu,mu
+; CHECK-NEXT:    vslideup.vi v28, v26, 6
+; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
 ; CHECK-NEXT:    lui a1, %hi(.LCPI53_0)
 ; CHECK-NEXT:    addi a1, a1, %lo(.LCPI53_0)
 ; CHECK-NEXT:    vle16.v v26, (a1)
-; CHECK-NEXT:    lui a1, %hi(.LCPI53_1)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI53_1)
-; CHECK-NEXT:    vle16.v v27, (a1)
-; CHECK-NEXT:    vsrl.vv v26, v25, v26
-; CHECK-NEXT:    vmulhu.vv v26, v26, v27
-; CHECK-NEXT:    lui a1, %hi(.LCPI53_2)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI53_2)
-; CHECK-NEXT:    vle16.v v27, (a1)
-; CHECK-NEXT:    lui a1, %hi(.LCPI53_3)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI53_3)
-; CHECK-NEXT:    vle16.v v28, (a1)
+; CHECK-NEXT:    vsrl.vv v28, v25, v28
+; CHECK-NEXT:    vmulhu.vv v26, v28, v26
 ; CHECK-NEXT:    vsub.vv v25, v25, v26
-; CHECK-NEXT:    vmulhu.vv v25, v25, v27
+; CHECK-NEXT:    vmulhu.vv v25, v25, v29
 ; CHECK-NEXT:    vadd.vv v25, v25, v26
-; CHECK-NEXT:    vsrl.vv v25, v25, v28
+; CHECK-NEXT:    vsrl.vv v25, v25, v27
 ; CHECK-NEXT:    vse16.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <8 x i16>, <8 x i16>* %x
@@ -903,20 +981,26 @@ define void @mulhu_v4i32(<4 x i32>* %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    lui a1, 524288
+; CHECK-NEXT:    vmv.s.x v26, a1
+; CHECK-NEXT:    vmv.v.i v27, 0
+; CHECK-NEXT:    vsetivli a1, 3, e32,m1,tu,mu
+; CHECK-NEXT:    vslideup.vi v27, v26, 2
 ; CHECK-NEXT:    lui a1, %hi(.LCPI54_0)
 ; CHECK-NEXT:    addi a1, a1, %lo(.LCPI54_0)
+; CHECK-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v26, (a1)
 ; CHECK-NEXT:    vmulhu.vv v26, v25, v26
-; CHECK-NEXT:    lui a1, %hi(.LCPI54_1)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI54_1)
-; CHECK-NEXT:    vle32.v v27, (a1)
-; CHECK-NEXT:    lui a1, %hi(.LCPI54_2)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI54_2)
-; CHECK-NEXT:    vle32.v v28, (a1)
 ; CHECK-NEXT:    vsub.vv v25, v25, v26
 ; CHECK-NEXT:    vmulhu.vv v25, v25, v27
 ; CHECK-NEXT:    vadd.vv v25, v25, v26
-; CHECK-NEXT:    vsrl.vv v25, v25, v28
+; CHECK-NEXT:    addi a1, zero, 1
+; CHECK-NEXT:    vmv.s.x v26, a1
+; CHECK-NEXT:    vmv.v.i v27, 2
+; CHECK-NEXT:    vsetivli a1, 4, e32,m1,tu,mu
+; CHECK-NEXT:    vslideup.vi v27, v26, 3
+; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vsrl.vv v25, v25, v27
 ; CHECK-NEXT:    vse32.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <4 x i32>, <4 x i32>* %x
@@ -926,39 +1010,54 @@ define void @mulhu_v4i32(<4 x i32>* %x) {
 }
 
 define void @mulhu_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: mulhu_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI55_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI55_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmulhu.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI55_1)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI55_1)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhu_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI55_0)
-; LMULMAX1-RV64-NEXT:    addi a1, a1, %lo(.LCPI55_0)
-; LMULMAX1-RV64-NEXT:    vle64.v v26, (a1)
-; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI55_1)
-; LMULMAX1-RV64-NEXT:    addi a1, a1, %lo(.LCPI55_1)
-; LMULMAX1-RV64-NEXT:    vle64.v v27, (a1)
-; LMULMAX1-RV64-NEXT:    vmulhu.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vsrl.vv v25, v25, v27
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: mulhu_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a0)
+; RV32-NEXT:    lui a1, %hi(.LCPI55_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI55_0)
+; RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; RV32-NEXT:    vle32.v v26, (a1)
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vmulhu.vv v25, v25, v26
+; RV32-NEXT:    lui a1, %hi(.LCPI55_1)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI55_1)
+; RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; RV32-NEXT:    vle32.v v26, (a1)
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vsrl.vv v25, v25, v26
+; RV32-NEXT:    vse64.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhu_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a0)
+; RV64-NEXT:    vmv.v.i v26, 2
+; RV64-NEXT:    addi a1, zero, 1
+; RV64-NEXT:    vmv.s.x v26, a1
+; RV64-NEXT:    lui a1, 1035469
+; RV64-NEXT:    addiw a1, a1, -819
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -819
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -819
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -819
+; RV64-NEXT:    vmv.v.x v27, a1
+; RV64-NEXT:    lui a1, 1026731
+; RV64-NEXT:    addiw a1, a1, -1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -1365
+; RV64-NEXT:    vmv.s.x v27, a1
+; RV64-NEXT:    vmulhu.vv v25, v25, v27
+; RV64-NEXT:    vsrl.vv v25, v25, v26
+; RV64-NEXT:    vse64.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = udiv <2 x i64> %a, <i64 3, i64 5>
   store <2 x i64> %b, <2 x i64>* %x
@@ -966,20 +1065,45 @@ define void @mulhu_v2i64(<2 x i64>* %x) {
 }
 
 define void @mulhs_v16i8(<16 x i8>* %x) {
-; CHECK-LABEL: mulhs_v16i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
-; CHECK-NEXT:    vle8.v v25, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI56_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI56_0)
-; CHECK-NEXT:    vle8.v v26, (a1)
-; CHECK-NEXT:    lui a1, %hi(.LCPI56_1)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI56_1)
-; CHECK-NEXT:    vle8.v v27, (a1)
-; CHECK-NEXT:    vmulhu.vv v25, v25, v26
-; CHECK-NEXT:    vsrl.vv v25, v25, v27
-; CHECK-NEXT:    vse8.v v25, (a0)
-; CHECK-NEXT:    ret
+; RV32-LABEL: mulhs_v16i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV32-NEXT:    vle8.v v25, (a0)
+; RV32-NEXT:    lui a1, 5
+; RV32-NEXT:    addi a1, a1, -1452
+; RV32-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v26, 7
+; RV32-NEXT:    vmerge.vim v26, v26, 1, v0
+; RV32-NEXT:    addi a1, zero, -123
+; RV32-NEXT:    vmv.v.x v27, a1
+; RV32-NEXT:    addi a1, zero, 57
+; RV32-NEXT:    vmerge.vxm v27, v27, a1, v0
+; RV32-NEXT:    vmulhu.vv v25, v25, v27
+; RV32-NEXT:    vsrl.vv v25, v25, v26
+; RV32-NEXT:    vse8.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhs_v16i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV64-NEXT:    vle8.v v25, (a0)
+; RV64-NEXT:    lui a1, 5
+; RV64-NEXT:    addiw a1, a1, -1452
+; RV64-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; RV64-NEXT:    vmv.s.x v0, a1
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; RV64-NEXT:    vmv.v.i v26, 7
+; RV64-NEXT:    vmerge.vim v26, v26, 1, v0
+; RV64-NEXT:    addi a1, zero, -123
+; RV64-NEXT:    vmv.v.x v27, a1
+; RV64-NEXT:    addi a1, zero, 57
+; RV64-NEXT:    vmerge.vxm v27, v27, a1, v0
+; RV64-NEXT:    vmulhu.vv v25, v25, v27
+; RV64-NEXT:    vsrl.vv v25, v25, v26
+; RV64-NEXT:    vse8.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <16 x i8>, <16 x i8>* %x
   %b = udiv <16 x i8> %a, <i8 -9, i8 -9, i8 9, i8 -9, i8 9, i8 -9, i8 9, i8 -9, i8 -9, i8 9, i8 -9, i8 9, i8 -9, i8 -9, i8 9, i8 -9>
   store <16 x i8> %b, <16 x i8>* %x
@@ -987,19 +1111,47 @@ define void @mulhs_v16i8(<16 x i8>* %x) {
 }
 
 define void @mulhs_v8i16(<8 x i16>* %x) {
-; CHECK-LABEL: mulhs_v8i16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; CHECK-NEXT:    vle16.v v25, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI57_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI57_0)
-; CHECK-NEXT:    vle16.v v26, (a1)
-; CHECK-NEXT:    vmulh.vv v25, v25, v26
-; CHECK-NEXT:    vsra.vi v25, v25, 1
-; CHECK-NEXT:    vsrl.vi v26, v25, 15
-; CHECK-NEXT:    vadd.vv v25, v25, v26
-; CHECK-NEXT:    vse16.v v25, (a0)
-; CHECK-NEXT:    ret
+; RV32-LABEL: mulhs_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; RV32-NEXT:    vle16.v v25, (a0)
+; RV32-NEXT:    addi a1, zero, 105
+; RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    lui a1, 5
+; RV32-NEXT:    addi a1, a1, -1755
+; RV32-NEXT:    vsetivli a2, 8, e16,m1,ta,mu
+; RV32-NEXT:    vmv.v.x v26, a1
+; RV32-NEXT:    lui a1, 1048571
+; RV32-NEXT:    addi a1, a1, 1755
+; RV32-NEXT:    vmerge.vxm v26, v26, a1, v0
+; RV32-NEXT:    vmulh.vv v25, v25, v26
+; RV32-NEXT:    vsra.vi v25, v25, 1
+; RV32-NEXT:    vsrl.vi v26, v25, 15
+; RV32-NEXT:    vadd.vv v25, v25, v26
+; RV32-NEXT:    vse16.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhs_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; RV64-NEXT:    vle16.v v25, (a0)
+; RV64-NEXT:    addi a1, zero, 105
+; RV64-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; RV64-NEXT:    vmv.s.x v0, a1
+; RV64-NEXT:    lui a1, 5
+; RV64-NEXT:    addiw a1, a1, -1755
+; RV64-NEXT:    vsetivli a2, 8, e16,m1,ta,mu
+; RV64-NEXT:    vmv.v.x v26, a1
+; RV64-NEXT:    lui a1, 1048571
+; RV64-NEXT:    addiw a1, a1, 1755
+; RV64-NEXT:    vmerge.vxm v26, v26, a1, v0
+; RV64-NEXT:    vmulh.vv v25, v25, v26
+; RV64-NEXT:    vsra.vi v25, v25, 1
+; RV64-NEXT:    vsrl.vi v26, v25, 15
+; RV64-NEXT:    vadd.vv v25, v25, v26
+; RV64-NEXT:    vse16.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <8 x i16>, <8 x i16>* %x
   %b = sdiv <8 x i16> %a, <i16 -7, i16 7, i16 7, i16 -7, i16 7, i16 -7, i16 -7, i16 7>
   store <8 x i16> %b, <8 x i16>* %x
@@ -1007,33 +1159,48 @@ define void @mulhs_v8i16(<8 x i16>* %x) {
 }
 
 define void @mulhs_v4i32(<4 x i32>* %x) {
-; LMULMAX1-RV32-LABEL: mulhs_v4i32:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI58_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI58_0)
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vmulh.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vsrl.vi v26, v25, 31
-; LMULMAX1-RV32-NEXT:    vsra.vi v25, v25, 1
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse32.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhs_v4i32:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle32.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI58_0)
-; LMULMAX1-RV64-NEXT:    addi a1, a1, %lo(.LCPI58_0)
-; LMULMAX1-RV64-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV64-NEXT:    vmulh.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vsra.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vsrl.vi v26, v25, 31
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vse32.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: mulhs_v4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-NEXT:    vle32.v v25, (a0)
+; RV32-NEXT:    addi a1, zero, 5
+; RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    lui a1, 419430
+; RV32-NEXT:    addi a1, a1, 1639
+; RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.x v26, a1
+; RV32-NEXT:    lui a1, 629146
+; RV32-NEXT:    addi a1, a1, -1639
+; RV32-NEXT:    vmerge.vxm v26, v26, a1, v0
+; RV32-NEXT:    vmulh.vv v25, v25, v26
+; RV32-NEXT:    vsrl.vi v26, v25, 31
+; RV32-NEXT:    vsra.vi v25, v25, 1
+; RV32-NEXT:    vadd.vv v25, v25, v26
+; RV32-NEXT:    vse32.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhs_v4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV64-NEXT:    vle32.v v25, (a0)
+; RV64-NEXT:    lui a1, 13107
+; RV64-NEXT:    addiw a1, a1, 819
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, 973
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -819
+; RV64-NEXT:    slli a1, a1, 13
+; RV64-NEXT:    addi a1, a1, -1639
+; RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; RV64-NEXT:    vmv.v.x v26, a1
+; RV64-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV64-NEXT:    vmulh.vv v25, v25, v26
+; RV64-NEXT:    vsra.vi v25, v25, 1
+; RV64-NEXT:    vsrl.vi v26, v25, 31
+; RV64-NEXT:    vadd.vv v25, v25, v26
+; RV64-NEXT:    vse32.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <4 x i32>, <4 x i32>* %x
   %b = sdiv <4 x i32> %a, <i32 -5, i32 5, i32 -5, i32 5>
   store <4 x i32> %b, <4 x i32>* %x
@@ -1041,59 +1208,68 @@ define void @mulhs_v4i32(<4 x i32>* %x) {
 }
 
 define void @mulhs_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: mulhs_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI59_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI59_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmul.vv v26, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI59_1)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI59_1)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v27, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmulh.vv v25, v25, v27
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI59_2)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI59_2)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v26, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI59_3)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI59_3)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v27, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsra.vv v25, v25, v27
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhs_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI59_0)
-; LMULMAX1-RV64-NEXT:    addi a1, a1, %lo(.LCPI59_0)
-; LMULMAX1-RV64-NEXT:    vle64.v v26, (a1)
-; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI59_1)
-; LMULMAX1-RV64-NEXT:    addi a1, a1, %lo(.LCPI59_1)
-; LMULMAX1-RV64-NEXT:    vle64.v v27, (a1)
-; LMULMAX1-RV64-NEXT:    vmul.vv v26, v25, v26
-; LMULMAX1-RV64-NEXT:    vmulh.vv v25, v25, v27
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    addi a1, zero, 63
-; LMULMAX1-RV64-NEXT:    vsrl.vx v26, v25, a1
-; LMULMAX1-RV64-NEXT:    vid.v v27
-; LMULMAX1-RV64-NEXT:    vsra.vv v25, v25, v27
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: mulhs_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a0)
+; RV32-NEXT:    lui a1, 349525
+; RV32-NEXT:    addi a2, a1, 1365
+; RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.x v26, a2
+; RV32-NEXT:    addi a1, a1, 1366
+; RV32-NEXT:    vmv.s.x v26, a1
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vmulh.vv v26, v25, v26
+; RV32-NEXT:    addi a1, zero, 1
+; RV32-NEXT:    addi a2, zero, 3
+; RV32-NEXT:    vsetivli a3, 1, e8,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a2
+; RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v27, -1
+; RV32-NEXT:    vmerge.vim v27, v27, 0, v0
+; RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; RV32-NEXT:    vmul.vv v25, v25, v27
+; RV32-NEXT:    vadd.vv v25, v26, v25
+; RV32-NEXT:    addi a2, zero, 63
+; RV32-NEXT:    vsrl.vx v26, v25, a2
+; RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v27, a1
+; RV32-NEXT:    vmv.v.i v28, 0
+; RV32-NEXT:    vsetivli a1, 3, e32,m1,tu,mu
+; RV32-NEXT:    vslideup.vi v28, v27, 2
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vsra.vv v25, v25, v28
+; RV32-NEXT:    vadd.vv v25, v25, v26
+; RV32-NEXT:    vse64.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhs_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a0)
+; RV64-NEXT:    vmv.v.i v26, -1
+; RV64-NEXT:    vmv.s.x v26, zero
+; RV64-NEXT:    vmul.vv v26, v25, v26
+; RV64-NEXT:    lui a1, 21845
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a2, a1, 1365
+; RV64-NEXT:    vmv.v.x v27, a2
+; RV64-NEXT:    addi a1, a1, 1366
+; RV64-NEXT:    vmv.s.x v27, a1
+; RV64-NEXT:    vmulh.vv v25, v25, v27
+; RV64-NEXT:    vadd.vv v25, v25, v26
+; RV64-NEXT:    addi a1, zero, 63
+; RV64-NEXT:    vsrl.vx v26, v25, a1
+; RV64-NEXT:    vid.v v27
+; RV64-NEXT:    vsra.vv v25, v25, v27
+; RV64-NEXT:    vadd.vv v25, v25, v26
+; RV64-NEXT:    vse64.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = sdiv <2 x i64> %a, <i64 3, i64 -3>
   store <2 x i64> %b, <2 x i64>* %x
@@ -3788,37 +3964,21 @@ define void @extract_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 ; LMULMAX2-NEXT:    vse64.v v26, (a0)
 ; LMULMAX2-NEXT:    ret
 ;
-; LMULMAX1-RV32-LABEL: extract_v4i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    addi a2, a0, 16
-; LMULMAX1-RV32-NEXT:    vle64.v v26, (a2)
-; LMULMAX1-RV32-NEXT:    vle64.v v27, (a1)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, 16
-; LMULMAX1-RV32-NEXT:    vle64.v v28, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vadd.vv v26, v26, v28
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v27
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vse64.v v26, (a2)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: extract_v4i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    addi a2, a0, 16
-; LMULMAX1-RV64-NEXT:    vle64.v v26, (a2)
-; LMULMAX1-RV64-NEXT:    vle64.v v27, (a1)
-; LMULMAX1-RV64-NEXT:    addi a1, a1, 16
-; LMULMAX1-RV64-NEXT:    vle64.v v28, (a1)
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vadd.vv v26, v26, v28
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v27
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vse64.v v26, (a2)
-; LMULMAX1-RV64-NEXT:    ret
+; LMULMAX1-LABEL: extract_v4i64:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-NEXT:    vle64.v v25, (a0)
+; LMULMAX1-NEXT:    addi a2, a0, 16
+; LMULMAX1-NEXT:    vle64.v v26, (a2)
+; LMULMAX1-NEXT:    vle64.v v27, (a1)
+; LMULMAX1-NEXT:    addi a1, a1, 16
+; LMULMAX1-NEXT:    vle64.v v28, (a1)
+; LMULMAX1-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; LMULMAX1-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX1-NEXT:    vadd.vv v25, v25, v27
+; LMULMAX1-NEXT:    vse64.v v25, (a0)
+; LMULMAX1-NEXT:    vse64.v v26, (a2)
+; LMULMAX1-NEXT:    ret
   %a = load <4 x i64>, <4 x i64>* %x
   %b = load <4 x i64>, <4 x i64>* %y
   br label %"compute"
@@ -3829,61 +3989,120 @@ define void @extract_v4i64(<4 x i64>* %x, <4 x i64>* %y) {
 }
 
 define void @mulhu_v32i8(<32 x i8>* %x) {
-; LMULMAX2-LABEL: mulhu_v32i8:
-; LMULMAX2:       # %bb.0:
-; LMULMAX2-NEXT:    addi a1, zero, 32
-; LMULMAX2-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; LMULMAX2-NEXT:    vle8.v v26, (a0)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI129_0)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI129_0)
-; LMULMAX2-NEXT:    vle8.v v28, (a1)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI129_1)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI129_1)
-; LMULMAX2-NEXT:    vle8.v v30, (a1)
-; LMULMAX2-NEXT:    vsrl.vv v28, v26, v28
-; LMULMAX2-NEXT:    vmulhu.vv v28, v28, v30
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI129_2)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI129_2)
-; LMULMAX2-NEXT:    vle8.v v30, (a1)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI129_3)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI129_3)
-; LMULMAX2-NEXT:    vle8.v v8, (a1)
-; LMULMAX2-NEXT:    vsub.vv v26, v26, v28
-; LMULMAX2-NEXT:    vmulhu.vv v26, v26, v30
-; LMULMAX2-NEXT:    vadd.vv v26, v26, v28
-; LMULMAX2-NEXT:    vsrl.vv v26, v26, v8
-; LMULMAX2-NEXT:    vse8.v v26, (a0)
-; LMULMAX2-NEXT:    ret
-;
-; LMULMAX1-RV32-LABEL: mulhu_v32i8:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV32-NEXT:    vle8.v v25, (a1)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI129_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI129_0)
-; LMULMAX1-RV32-NEXT:    vle8.v v26, (a2)
-; LMULMAX1-RV32-NEXT:    vle8.v v27, (a0)
-; LMULMAX1-RV32-NEXT:    vdivu.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vdivu.vv v26, v27, v26
-; LMULMAX1-RV32-NEXT:    vse8.v v26, (a0)
-; LMULMAX1-RV32-NEXT:    vse8.v v25, (a1)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhu_v32i8:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV64-NEXT:    vle8.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI129_0)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI129_0)
-; LMULMAX1-RV64-NEXT:    vle8.v v26, (a2)
-; LMULMAX1-RV64-NEXT:    vle8.v v27, (a0)
-; LMULMAX1-RV64-NEXT:    vdivu.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vdivu.vv v26, v27, v26
-; LMULMAX1-RV64-NEXT:    vse8.v v26, (a0)
-; LMULMAX1-RV64-NEXT:    vse8.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    ret
+; LMULMAX2-RV32-LABEL: mulhu_v32i8:
+; LMULMAX2-RV32:       # %bb.0:
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 32
+; LMULMAX2-RV32-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vle8.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    lui a2, 8208
+; LMULMAX2-RV32-NEXT:    addi a2, a2, 513
+; LMULMAX2-RV32-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV32-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.i v28, 4
+; LMULMAX2-RV32-NEXT:    vmerge.vim v28, v28, 1, v0
+; LMULMAX2-RV32-NEXT:    lui a2, 66785
+; LMULMAX2-RV32-NEXT:    addi a2, a2, 78
+; LMULMAX2-RV32-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV32-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmerge.vim v28, v28, 3, v0
+; LMULMAX2-RV32-NEXT:    lui a2, 529160
+; LMULMAX2-RV32-NEXT:    addi a2, a2, 304
+; LMULMAX2-RV32-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV32-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmerge.vim v28, v28, 2, v0
+; LMULMAX2-RV32-NEXT:    lui a2, 163907
+; LMULMAX2-RV32-NEXT:    addi a2, a2, -2044
+; LMULMAX2-RV32-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV32-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.i v30, 0
+; LMULMAX2-RV32-NEXT:    addi a2, zero, -128
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v8, v30, a2, v0
+; LMULMAX2-RV32-NEXT:    lui a2, 66049
+; LMULMAX2-RV32-NEXT:    addi a2, a2, 32
+; LMULMAX2-RV32-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV32-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI129_0)
+; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI129_0)
+; LMULMAX2-RV32-NEXT:    vle8.v v10, (a1)
+; LMULMAX2-RV32-NEXT:    vmerge.vim v30, v30, 1, v0
+; LMULMAX2-RV32-NEXT:    vsrl.vv v30, v26, v30
+; LMULMAX2-RV32-NEXT:    vmulhu.vv v30, v30, v10
+; LMULMAX2-RV32-NEXT:    vsub.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vmulhu.vv v26, v26, v8
+; LMULMAX2-RV32-NEXT:    vadd.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    vse8.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    ret
+;
+; LMULMAX2-RV64-LABEL: mulhu_v32i8:
+; LMULMAX2-RV64:       # %bb.0:
+; LMULMAX2-RV64-NEXT:    addi a1, zero, 32
+; LMULMAX2-RV64-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vle8.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    lui a2, 8208
+; LMULMAX2-RV64-NEXT:    addiw a2, a2, 513
+; LMULMAX2-RV64-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV64-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.i v28, 4
+; LMULMAX2-RV64-NEXT:    vmerge.vim v28, v28, 1, v0
+; LMULMAX2-RV64-NEXT:    lui a2, 66785
+; LMULMAX2-RV64-NEXT:    addiw a2, a2, 78
+; LMULMAX2-RV64-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV64-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmerge.vim v28, v28, 3, v0
+; LMULMAX2-RV64-NEXT:    lui a2, 529160
+; LMULMAX2-RV64-NEXT:    addiw a2, a2, 304
+; LMULMAX2-RV64-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV64-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmerge.vim v28, v28, 2, v0
+; LMULMAX2-RV64-NEXT:    lui a2, 163907
+; LMULMAX2-RV64-NEXT:    addiw a2, a2, -2044
+; LMULMAX2-RV64-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV64-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.i v30, 0
+; LMULMAX2-RV64-NEXT:    addi a2, zero, -128
+; LMULMAX2-RV64-NEXT:    vmerge.vxm v8, v30, a2, v0
+; LMULMAX2-RV64-NEXT:    lui a2, 66049
+; LMULMAX2-RV64-NEXT:    addiw a2, a2, 32
+; LMULMAX2-RV64-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV64-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI129_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI129_0)
+; LMULMAX2-RV64-NEXT:    vle8.v v10, (a1)
+; LMULMAX2-RV64-NEXT:    vmerge.vim v30, v30, 1, v0
+; LMULMAX2-RV64-NEXT:    vsrl.vv v30, v26, v30
+; LMULMAX2-RV64-NEXT:    vmulhu.vv v30, v30, v10
+; LMULMAX2-RV64-NEXT:    vsub.vv v26, v26, v30
+; LMULMAX2-RV64-NEXT:    vmulhu.vv v26, v26, v8
+; LMULMAX2-RV64-NEXT:    vadd.vv v26, v26, v30
+; LMULMAX2-RV64-NEXT:    vsrl.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vse8.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    ret
+;
+; LMULMAX1-LABEL: mulhu_v32i8:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; LMULMAX1-NEXT:    addi a1, a0, 16
+; LMULMAX1-NEXT:    vle8.v v25, (a1)
+; LMULMAX1-NEXT:    lui a2, %hi(.LCPI129_0)
+; LMULMAX1-NEXT:    addi a2, a2, %lo(.LCPI129_0)
+; LMULMAX1-NEXT:    vle8.v v26, (a2)
+; LMULMAX1-NEXT:    vle8.v v27, (a0)
+; LMULMAX1-NEXT:    vdivu.vv v25, v25, v26
+; LMULMAX1-NEXT:    vdivu.vv v26, v27, v26
+; LMULMAX1-NEXT:    vse8.v v26, (a0)
+; LMULMAX1-NEXT:    vse8.v v25, (a1)
+; LMULMAX1-NEXT:    ret
   %a = load <32 x i8>, <32 x i8>* %x
   %b = udiv <32 x i8> %a, <i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25>
   store <32 x i8> %b, <32 x i8>* %x
@@ -3891,60 +4110,98 @@ define void @mulhu_v32i8(<32 x i8>* %x) {
 }
 
 define void @mulhu_v16i16(<16 x i16>* %x) {
-; LMULMAX2-LABEL: mulhu_v16i16:
-; LMULMAX2:       # %bb.0:
-; LMULMAX2-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
-; LMULMAX2-NEXT:    vle16.v v26, (a0)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI130_0)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI130_0)
-; LMULMAX2-NEXT:    vle16.v v28, (a1)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI130_1)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI130_1)
-; LMULMAX2-NEXT:    vle16.v v30, (a1)
-; LMULMAX2-NEXT:    vsrl.vv v28, v26, v28
-; LMULMAX2-NEXT:    vmulhu.vv v28, v28, v30
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI130_2)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI130_2)
-; LMULMAX2-NEXT:    vle16.v v30, (a1)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI130_3)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI130_3)
-; LMULMAX2-NEXT:    vle16.v v8, (a1)
-; LMULMAX2-NEXT:    vsub.vv v26, v26, v28
-; LMULMAX2-NEXT:    vmulhu.vv v26, v26, v30
-; LMULMAX2-NEXT:    vadd.vv v26, v26, v28
-; LMULMAX2-NEXT:    vsrl.vv v26, v26, v8
-; LMULMAX2-NEXT:    vse16.v v26, (a0)
-; LMULMAX2-NEXT:    ret
-;
-; LMULMAX1-RV32-LABEL: mulhu_v16i16:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV32-NEXT:    vle16.v v25, (a1)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI130_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI130_0)
-; LMULMAX1-RV32-NEXT:    vle16.v v26, (a2)
-; LMULMAX1-RV32-NEXT:    vle16.v v27, (a0)
-; LMULMAX1-RV32-NEXT:    vdivu.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vdivu.vv v26, v27, v26
-; LMULMAX1-RV32-NEXT:    vse16.v v26, (a0)
-; LMULMAX1-RV32-NEXT:    vse16.v v25, (a1)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhu_v16i16:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV64-NEXT:    vle16.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI130_0)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI130_0)
-; LMULMAX1-RV64-NEXT:    vle16.v v26, (a2)
-; LMULMAX1-RV64-NEXT:    vle16.v v27, (a0)
-; LMULMAX1-RV64-NEXT:    vdivu.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vdivu.vv v26, v27, v26
-; LMULMAX1-RV64-NEXT:    vse16.v v26, (a0)
-; LMULMAX1-RV64-NEXT:    vse16.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    ret
+; LMULMAX2-RV32-LABEL: mulhu_v16i16:
+; LMULMAX2-RV32:       # %bb.0:
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vle16.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    lui a1, 2
+; LMULMAX2-RV32-NEXT:    addi a1, a1, 289
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.i v28, 3
+; LMULMAX2-RV32-NEXT:    vmerge.vim v28, v28, 2, v0
+; LMULMAX2-RV32-NEXT:    lui a1, 4
+; LMULMAX2-RV32-NEXT:    addi a1, a1, 64
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmerge.vim v28, v28, 1, v0
+; LMULMAX2-RV32-NEXT:    vmv1r.v v12, v0
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 257
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.i v30, 0
+; LMULMAX2-RV32-NEXT:    lui a1, 1048568
+; LMULMAX2-RV32-NEXT:    lui a2, %hi(.LCPI130_0)
+; LMULMAX2-RV32-NEXT:    addi a2, a2, %lo(.LCPI130_0)
+; LMULMAX2-RV32-NEXT:    vle16.v v8, (a2)
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v10, v30, a1, v0
+; LMULMAX2-RV32-NEXT:    vmv1r.v v0, v12
+; LMULMAX2-RV32-NEXT:    vmerge.vim v30, v30, 1, v0
+; LMULMAX2-RV32-NEXT:    vsrl.vv v30, v26, v30
+; LMULMAX2-RV32-NEXT:    vmulhu.vv v30, v30, v8
+; LMULMAX2-RV32-NEXT:    vsub.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vmulhu.vv v26, v26, v10
+; LMULMAX2-RV32-NEXT:    vadd.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    vse16.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    ret
+;
+; LMULMAX2-RV64-LABEL: mulhu_v16i16:
+; LMULMAX2-RV64:       # %bb.0:
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vle16.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    lui a1, 2
+; LMULMAX2-RV64-NEXT:    addiw a1, a1, 289
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.i v28, 3
+; LMULMAX2-RV64-NEXT:    vmerge.vim v28, v28, 2, v0
+; LMULMAX2-RV64-NEXT:    lui a1, 4
+; LMULMAX2-RV64-NEXT:    addiw a1, a1, 64
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmerge.vim v28, v28, 1, v0
+; LMULMAX2-RV64-NEXT:    vmv1r.v v12, v0
+; LMULMAX2-RV64-NEXT:    addi a1, zero, 257
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.i v30, 0
+; LMULMAX2-RV64-NEXT:    lui a1, 1048568
+; LMULMAX2-RV64-NEXT:    lui a2, %hi(.LCPI130_0)
+; LMULMAX2-RV64-NEXT:    addi a2, a2, %lo(.LCPI130_0)
+; LMULMAX2-RV64-NEXT:    vle16.v v8, (a2)
+; LMULMAX2-RV64-NEXT:    vmerge.vxm v10, v30, a1, v0
+; LMULMAX2-RV64-NEXT:    vmv1r.v v0, v12
+; LMULMAX2-RV64-NEXT:    vmerge.vim v30, v30, 1, v0
+; LMULMAX2-RV64-NEXT:    vsrl.vv v30, v26, v30
+; LMULMAX2-RV64-NEXT:    vmulhu.vv v30, v30, v8
+; LMULMAX2-RV64-NEXT:    vsub.vv v26, v26, v30
+; LMULMAX2-RV64-NEXT:    vmulhu.vv v26, v26, v10
+; LMULMAX2-RV64-NEXT:    vadd.vv v26, v26, v30
+; LMULMAX2-RV64-NEXT:    vsrl.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vse16.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    ret
+;
+; LMULMAX1-LABEL: mulhu_v16i16:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; LMULMAX1-NEXT:    addi a1, a0, 16
+; LMULMAX1-NEXT:    vle16.v v25, (a1)
+; LMULMAX1-NEXT:    lui a2, %hi(.LCPI130_0)
+; LMULMAX1-NEXT:    addi a2, a2, %lo(.LCPI130_0)
+; LMULMAX1-NEXT:    vle16.v v26, (a2)
+; LMULMAX1-NEXT:    vle16.v v27, (a0)
+; LMULMAX1-NEXT:    vdivu.vv v25, v25, v26
+; LMULMAX1-NEXT:    vdivu.vv v26, v27, v26
+; LMULMAX1-NEXT:    vse16.v v26, (a0)
+; LMULMAX1-NEXT:    vse16.v v25, (a1)
+; LMULMAX1-NEXT:    ret
   %a = load <16 x i16>, <16 x i16>* %x
   %b = udiv <16 x i16> %a, <i16 7, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 7, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
   store <16 x i16> %b, <16 x i16>* %x
@@ -3956,50 +4213,63 @@ define void @mulhu_v8i32(<8 x i32>* %x) {
 ; LMULMAX2:       # %bb.0:
 ; LMULMAX2-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
 ; LMULMAX2-NEXT:    vle32.v v26, (a0)
+; LMULMAX2-NEXT:    addi a1, zero, 68
+; LMULMAX2-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; LMULMAX2-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
 ; LMULMAX2-NEXT:    lui a1, %hi(.LCPI131_0)
 ; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI131_0)
 ; LMULMAX2-NEXT:    vle32.v v28, (a1)
+; LMULMAX2-NEXT:    vmv.v.i v30, 0
+; LMULMAX2-NEXT:    lui a1, 524288
+; LMULMAX2-NEXT:    vmerge.vxm v30, v30, a1, v0
 ; LMULMAX2-NEXT:    vmulhu.vv v28, v26, v28
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI131_1)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI131_1)
-; LMULMAX2-NEXT:    vle32.v v30, (a1)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI131_2)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI131_2)
-; LMULMAX2-NEXT:    vle32.v v8, (a1)
 ; LMULMAX2-NEXT:    vsub.vv v26, v26, v28
 ; LMULMAX2-NEXT:    vmulhu.vv v26, v26, v30
 ; LMULMAX2-NEXT:    vadd.vv v26, v26, v28
-; LMULMAX2-NEXT:    vsrl.vv v26, v26, v8
+; LMULMAX2-NEXT:    addi a1, zero, 136
+; LMULMAX2-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; LMULMAX2-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-NEXT:    vmv.v.i v28, 2
+; LMULMAX2-NEXT:    vmerge.vim v28, v28, 1, v0
+; LMULMAX2-NEXT:    vsrl.vv v26, v26, v28
 ; LMULMAX2-NEXT:    vse32.v v26, (a0)
 ; LMULMAX2-NEXT:    ret
 ;
 ; LMULMAX1-RV32-LABEL: mulhu_v8i32:
 ; LMULMAX1-RV32:       # %bb.0:
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vle32.v v25, (a0)
 ; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV32-NEXT:    vle32.v v25, (a1)
+; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
+; LMULMAX1-RV32-NEXT:    lui a2, 524288
+; LMULMAX1-RV32-NEXT:    vmv.s.x v27, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.i v28, 0
+; LMULMAX1-RV32-NEXT:    vsetivli a2, 3, e32,m1,tu,mu
+; LMULMAX1-RV32-NEXT:    vslideup.vi v28, v27, 2
 ; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI131_0)
 ; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI131_0)
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a2)
-; LMULMAX1-RV32-NEXT:    vle32.v v27, (a0)
-; LMULMAX1-RV32-NEXT:    vmulhu.vv v28, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI131_1)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI131_1)
-; LMULMAX1-RV32-NEXT:    vle32.v v29, (a2)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI131_2)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI131_2)
-; LMULMAX1-RV32-NEXT:    vle32.v v30, (a2)
-; LMULMAX1-RV32-NEXT:    vsub.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    vmulhu.vv v25, v25, v29
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v30
-; LMULMAX1-RV32-NEXT:    vmulhu.vv v26, v27, v26
-; LMULMAX1-RV32-NEXT:    vsub.vv v27, v27, v26
-; LMULMAX1-RV32-NEXT:    vmulhu.vv v27, v27, v29
-; LMULMAX1-RV32-NEXT:    vadd.vv v26, v27, v26
+; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vle32.v v27, (a2)
+; LMULMAX1-RV32-NEXT:    vmulhu.vv v29, v26, v27
+; LMULMAX1-RV32-NEXT:    vsub.vv v26, v26, v29
+; LMULMAX1-RV32-NEXT:    vmulhu.vv v26, v26, v28
+; LMULMAX1-RV32-NEXT:    vadd.vv v26, v26, v29
+; LMULMAX1-RV32-NEXT:    addi a2, zero, 1
+; LMULMAX1-RV32-NEXT:    vmv.s.x v29, a2
+; LMULMAX1-RV32-NEXT:    vmv.v.i v30, 2
+; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,tu,mu
+; LMULMAX1-RV32-NEXT:    vslideup.vi v30, v29, 3
+; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
 ; LMULMAX1-RV32-NEXT:    vsrl.vv v26, v26, v30
-; LMULMAX1-RV32-NEXT:    vse32.v v26, (a0)
-; LMULMAX1-RV32-NEXT:    vse32.v v25, (a1)
+; LMULMAX1-RV32-NEXT:    vmulhu.vv v27, v25, v27
+; LMULMAX1-RV32-NEXT:    vsub.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vmulhu.vv v25, v25, v28
+; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v30
+; LMULMAX1-RV32-NEXT:    vse32.v v25, (a0)
+; LMULMAX1-RV32-NEXT:    vse32.v v26, (a1)
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV64-LABEL: mulhu_v8i32:
@@ -4023,6 +4293,60 @@ define void @mulhu_v8i32(<8 x i32>* %x) {
 }
 
 define void @mulhu_v4i64(<4 x i64>* %x) {
+; LMULMAX2-RV32-LABEL: mulhu_v4i64:
+; LMULMAX2-RV32:       # %bb.0:
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vle64.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI132_0)
+; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI132_0)
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vle32.v v28, (a1)
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmulhu.vv v28, v26, v28
+; LMULMAX2-RV32-NEXT:    vsub.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    lui a1, 524288
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v30, a1
+; LMULMAX2-RV32-NEXT:    vmv.v.i v8, 0
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 6, e32,m2,tu,mu
+; LMULMAX2-RV32-NEXT:    vslideup.vi v8, v30, 5
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmulhu.vv v26, v26, v8
+; LMULMAX2-RV32-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI132_1)
+; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI132_1)
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vle32.v v28, (a1)
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    vse64.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    ret
+;
+; LMULMAX2-RV64-LABEL: mulhu_v4i64:
+; LMULMAX2-RV64:       # %bb.0:
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vle64.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    addi a1, zero, -1
+; LMULMAX2-RV64-NEXT:    slli a1, a1, 63
+; LMULMAX2-RV64-NEXT:    vmv.s.x v28, a1
+; LMULMAX2-RV64-NEXT:    vmv.v.i v30, 0
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 3, e64,m2,tu,mu
+; LMULMAX2-RV64-NEXT:    vslideup.vi v30, v28, 2
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI132_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI132_0)
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 4, e64,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vle64.v v28, (a1)
+; LMULMAX2-RV64-NEXT:    vmulhu.vv v28, v26, v28
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI132_1)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI132_1)
+; LMULMAX2-RV64-NEXT:    vle64.v v8, (a1)
+; LMULMAX2-RV64-NEXT:    vsub.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vmulhu.vv v26, v26, v30
+; LMULMAX2-RV64-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vsrl.vv v26, v26, v8
+; LMULMAX2-RV64-NEXT:    vse64.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    ret
+;
 ; LMULMAX1-RV32-LABEL: mulhu_v4i64:
 ; LMULMAX1-RV32:       # %bb.0:
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
@@ -4047,34 +4371,65 @@ define void @mulhu_v4i64(<4 x i64>* %x) {
 ;
 ; LMULMAX1-RV64-LABEL: mulhu_v4i64:
 ; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_0)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI132_0)
+; LMULMAX1-RV64-NEXT:    addi a1, zero, 2
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
+; LMULMAX1-RV64-NEXT:    addi a2, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle64.v v26, (a2)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_1)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI132_1)
-; LMULMAX1-RV64-NEXT:    vle64.v v27, (a2)
-; LMULMAX1-RV64-NEXT:    vle64.v v28, (a0)
-; LMULMAX1-RV64-NEXT:    vmulhu.vv v26, v25, v26
-; LMULMAX1-RV64-NEXT:    vsub.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vmulhu.vv v25, v25, v27
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_2)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI132_2)
-; LMULMAX1-RV64-NEXT:    vle64.v v27, (a2)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_3)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI132_3)
-; LMULMAX1-RV64-NEXT:    vle64.v v29, (a2)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_4)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI132_4)
-; LMULMAX1-RV64-NEXT:    vle64.v v30, (a2)
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v26
+; LMULMAX1-RV64-NEXT:    vmv.v.i v27, 0
+; LMULMAX1-RV64-NEXT:    addi a3, zero, -1
+; LMULMAX1-RV64-NEXT:    slli a3, a3, 63
+; LMULMAX1-RV64-NEXT:    vmv.s.x v27, a3
+; LMULMAX1-RV64-NEXT:    lui a3, 1044935
+; LMULMAX1-RV64-NEXT:    addiw a3, a3, 455
+; LMULMAX1-RV64-NEXT:    slli a3, a3, 12
+; LMULMAX1-RV64-NEXT:    addi a3, a3, 455
+; LMULMAX1-RV64-NEXT:    slli a3, a3, 12
+; LMULMAX1-RV64-NEXT:    addi a3, a3, 455
+; LMULMAX1-RV64-NEXT:    slli a3, a3, 13
+; LMULMAX1-RV64-NEXT:    addi a3, a3, 911
+; LMULMAX1-RV64-NEXT:    vmv.v.x v28, a3
+; LMULMAX1-RV64-NEXT:    lui a3, 4681
+; LMULMAX1-RV64-NEXT:    addiw a3, a3, 585
+; LMULMAX1-RV64-NEXT:    slli a3, a3, 12
+; LMULMAX1-RV64-NEXT:    addi a3, a3, 585
+; LMULMAX1-RV64-NEXT:    slli a3, a3, 12
+; LMULMAX1-RV64-NEXT:    addi a3, a3, 585
+; LMULMAX1-RV64-NEXT:    slli a3, a3, 13
+; LMULMAX1-RV64-NEXT:    addi a3, a3, 1171
+; LMULMAX1-RV64-NEXT:    vmv.s.x v28, a3
+; LMULMAX1-RV64-NEXT:    vmulhu.vv v28, v26, v28
+; LMULMAX1-RV64-NEXT:    vsub.vv v26, v26, v28
+; LMULMAX1-RV64-NEXT:    vmulhu.vv v26, v26, v27
+; LMULMAX1-RV64-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX1-RV64-NEXT:    vmv.v.i v27, 3
+; LMULMAX1-RV64-NEXT:    vmv.s.x v27, a1
+; LMULMAX1-RV64-NEXT:    vsrl.vv v26, v26, v27
+; LMULMAX1-RV64-NEXT:    vmv.v.i v27, 2
+; LMULMAX1-RV64-NEXT:    addi a1, zero, 1
+; LMULMAX1-RV64-NEXT:    vmv.s.x v27, a1
+; LMULMAX1-RV64-NEXT:    lui a1, 1035469
+; LMULMAX1-RV64-NEXT:    addiw a1, a1, -819
+; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX1-RV64-NEXT:    addi a1, a1, -819
+; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX1-RV64-NEXT:    addi a1, a1, -819
+; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX1-RV64-NEXT:    addi a1, a1, -819
+; LMULMAX1-RV64-NEXT:    vmv.v.x v28, a1
+; LMULMAX1-RV64-NEXT:    lui a1, 1026731
+; LMULMAX1-RV64-NEXT:    addiw a1, a1, -1365
+; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX1-RV64-NEXT:    addi a1, a1, -1365
+; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX1-RV64-NEXT:    addi a1, a1, -1365
+; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX1-RV64-NEXT:    addi a1, a1, -1365
+; LMULMAX1-RV64-NEXT:    vmv.s.x v28, a1
+; LMULMAX1-RV64-NEXT:    vmulhu.vv v25, v25, v28
 ; LMULMAX1-RV64-NEXT:    vsrl.vv v25, v25, v27
-; LMULMAX1-RV64-NEXT:    vmulhu.vv v26, v28, v29
-; LMULMAX1-RV64-NEXT:    vsrl.vv v26, v26, v30
-; LMULMAX1-RV64-NEXT:    vse64.v v26, (a0)
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a1)
+; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
+; LMULMAX1-RV64-NEXT:    vse64.v v26, (a2)
 ; LMULMAX1-RV64-NEXT:    ret
   %a = load <4 x i64>, <4 x i64>* %x
   %b = udiv <4 x i64> %a, <i64 3, i64 5, i64 7, i64 9>
@@ -4083,50 +4438,84 @@ define void @mulhu_v4i64(<4 x i64>* %x) {
 }
 
 define void @mulhs_v32i8(<32 x i8>* %x) {
-; LMULMAX2-LABEL: mulhs_v32i8:
-; LMULMAX2:       # %bb.0:
-; LMULMAX2-NEXT:    addi a1, zero, 32
-; LMULMAX2-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; LMULMAX2-NEXT:    vle8.v v26, (a0)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI133_0)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI133_0)
-; LMULMAX2-NEXT:    vle8.v v28, (a1)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI133_1)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI133_1)
-; LMULMAX2-NEXT:    vle8.v v30, (a1)
-; LMULMAX2-NEXT:    vmulhu.vv v26, v26, v28
-; LMULMAX2-NEXT:    vsrl.vv v26, v26, v30
-; LMULMAX2-NEXT:    vse8.v v26, (a0)
-; LMULMAX2-NEXT:    ret
+; LMULMAX2-RV32-LABEL: mulhs_v32i8:
+; LMULMAX2-RV32:       # %bb.0:
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 32
+; LMULMAX2-RV32-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vle8.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    lui a2, 304453
+; LMULMAX2-RV32-NEXT:    addi a2, a2, -1452
+; LMULMAX2-RV32-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV32-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.i v28, 7
+; LMULMAX2-RV32-NEXT:    vmerge.vim v28, v28, 1, v0
+; LMULMAX2-RV32-NEXT:    addi a1, zero, -123
+; LMULMAX2-RV32-NEXT:    vmv.v.x v30, a1
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 57
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v30, v30, a1, v0
+; LMULMAX2-RV32-NEXT:    vmulhu.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vsrl.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    vse8.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    ret
+;
+; LMULMAX2-RV64-LABEL: mulhs_v32i8:
+; LMULMAX2-RV64:       # %bb.0:
+; LMULMAX2-RV64-NEXT:    addi a1, zero, 32
+; LMULMAX2-RV64-NEXT:    vsetvli a2, a1, e8,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vle8.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    lui a2, 304453
+; LMULMAX2-RV64-NEXT:    addiw a2, a2, -1452
+; LMULMAX2-RV64-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a2
+; LMULMAX2-RV64-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.i v28, 7
+; LMULMAX2-RV64-NEXT:    vmerge.vim v28, v28, 1, v0
+; LMULMAX2-RV64-NEXT:    addi a1, zero, -123
+; LMULMAX2-RV64-NEXT:    vmv.v.x v30, a1
+; LMULMAX2-RV64-NEXT:    addi a1, zero, 57
+; LMULMAX2-RV64-NEXT:    vmerge.vxm v30, v30, a1, v0
+; LMULMAX2-RV64-NEXT:    vmulhu.vv v26, v26, v30
+; LMULMAX2-RV64-NEXT:    vsrl.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vse8.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    ret
 ;
 ; LMULMAX1-RV32-LABEL: mulhs_v32i8:
 ; LMULMAX1-RV32:       # %bb.0:
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vle8.v v25, (a0)
 ; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV32-NEXT:    vle8.v v25, (a1)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI133_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI133_0)
-; LMULMAX1-RV32-NEXT:    vle8.v v26, (a2)
-; LMULMAX1-RV32-NEXT:    vle8.v v27, (a0)
-; LMULMAX1-RV32-NEXT:    vdivu.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vdivu.vv v26, v27, v26
-; LMULMAX1-RV32-NEXT:    vse8.v v26, (a0)
-; LMULMAX1-RV32-NEXT:    vse8.v v25, (a1)
+; LMULMAX1-RV32-NEXT:    vle8.v v26, (a1)
+; LMULMAX1-RV32-NEXT:    lui a2, 5
+; LMULMAX1-RV32-NEXT:    addi a2, a2, -1452
+; LMULMAX1-RV32-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.s.x v0, a2
+; LMULMAX1-RV32-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.i v27, -9
+; LMULMAX1-RV32-NEXT:    vmerge.vim v27, v27, 9, v0
+; LMULMAX1-RV32-NEXT:    vdivu.vv v26, v26, v27
+; LMULMAX1-RV32-NEXT:    vdivu.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vse8.v v25, (a0)
+; LMULMAX1-RV32-NEXT:    vse8.v v26, (a1)
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV64-LABEL: mulhs_v32i8:
 ; LMULMAX1-RV64:       # %bb.0:
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vle8.v v25, (a0)
 ; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV64-NEXT:    vle8.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI133_0)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI133_0)
-; LMULMAX1-RV64-NEXT:    vle8.v v26, (a2)
-; LMULMAX1-RV64-NEXT:    vle8.v v27, (a0)
-; LMULMAX1-RV64-NEXT:    vdivu.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vdivu.vv v26, v27, v26
-; LMULMAX1-RV64-NEXT:    vse8.v v26, (a0)
-; LMULMAX1-RV64-NEXT:    vse8.v v25, (a1)
+; LMULMAX1-RV64-NEXT:    vle8.v v26, (a1)
+; LMULMAX1-RV64-NEXT:    lui a2, 5
+; LMULMAX1-RV64-NEXT:    addiw a2, a2, -1452
+; LMULMAX1-RV64-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.s.x v0, a2
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.i v27, -9
+; LMULMAX1-RV64-NEXT:    vmerge.vim v27, v27, 9, v0
+; LMULMAX1-RV64-NEXT:    vdivu.vv v26, v26, v27
+; LMULMAX1-RV64-NEXT:    vdivu.vv v25, v25, v27
+; LMULMAX1-RV64-NEXT:    vse8.v v25, (a0)
+; LMULMAX1-RV64-NEXT:    vse8.v v26, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
   %a = load <32 x i8>, <32 x i8>* %x
   %b = udiv <32 x i8> %a, <i8 -9, i8 -9, i8 9, i8 -9, i8 9, i8 -9, i8 9, i8 -9, i8 -9, i8 9, i8 -9, i8 9, i8 -9, i8 -9, i8 9, i8 -9, i8 -9, i8 -9, i8 9, i8 -9, i8 9, i8 -9, i8 9, i8 -9, i8 -9, i8 9, i8 -9, i8 9, i8 -9, i8 -9, i8 9, i8 -9>
@@ -4135,49 +4524,67 @@ define void @mulhs_v32i8(<32 x i8>* %x) {
 }
 
 define void @mulhs_v16i16(<16 x i16>* %x) {
-; LMULMAX2-LABEL: mulhs_v16i16:
-; LMULMAX2:       # %bb.0:
-; LMULMAX2-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
-; LMULMAX2-NEXT:    vle16.v v26, (a0)
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI134_0)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI134_0)
-; LMULMAX2-NEXT:    vle16.v v28, (a1)
-; LMULMAX2-NEXT:    vmulh.vv v26, v26, v28
-; LMULMAX2-NEXT:    vsra.vi v26, v26, 1
-; LMULMAX2-NEXT:    vsrl.vi v28, v26, 15
-; LMULMAX2-NEXT:    vadd.vv v26, v26, v28
-; LMULMAX2-NEXT:    vse16.v v26, (a0)
-; LMULMAX2-NEXT:    ret
-;
-; LMULMAX1-RV32-LABEL: mulhs_v16i16:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV32-NEXT:    vle16.v v25, (a1)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI134_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI134_0)
-; LMULMAX1-RV32-NEXT:    vle16.v v26, (a2)
-; LMULMAX1-RV32-NEXT:    vle16.v v27, (a0)
-; LMULMAX1-RV32-NEXT:    vdiv.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vdiv.vv v26, v27, v26
-; LMULMAX1-RV32-NEXT:    vse16.v v26, (a0)
-; LMULMAX1-RV32-NEXT:    vse16.v v25, (a1)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhs_v16i16:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV64-NEXT:    vle16.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI134_0)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI134_0)
-; LMULMAX1-RV64-NEXT:    vle16.v v26, (a2)
-; LMULMAX1-RV64-NEXT:    vle16.v v27, (a0)
-; LMULMAX1-RV64-NEXT:    vdiv.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vdiv.vv v26, v27, v26
-; LMULMAX1-RV64-NEXT:    vse16.v v26, (a0)
-; LMULMAX1-RV64-NEXT:    vse16.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    ret
+; LMULMAX2-RV32-LABEL: mulhs_v16i16:
+; LMULMAX2-RV32:       # %bb.0:
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vle16.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    lui a1, 7
+; LMULMAX2-RV32-NEXT:    addi a1, a1, -1687
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV32-NEXT:    lui a1, 5
+; LMULMAX2-RV32-NEXT:    addi a1, a1, -1755
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 16, e16,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
+; LMULMAX2-RV32-NEXT:    lui a1, 1048571
+; LMULMAX2-RV32-NEXT:    addi a1, a1, 1755
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v28, v28, a1, v0
+; LMULMAX2-RV32-NEXT:    vmulh.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    vsra.vi v26, v26, 1
+; LMULMAX2-RV32-NEXT:    vsrl.vi v28, v26, 15
+; LMULMAX2-RV32-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    vse16.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    ret
+;
+; LMULMAX2-RV64-LABEL: mulhs_v16i16:
+; LMULMAX2-RV64:       # %bb.0:
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 16, e16,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vle16.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    lui a1, 7
+; LMULMAX2-RV64-NEXT:    addiw a1, a1, -1687
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 1, e16,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV64-NEXT:    lui a1, 5
+; LMULMAX2-RV64-NEXT:    addiw a1, a1, -1755
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 16, e16,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.x v28, a1
+; LMULMAX2-RV64-NEXT:    lui a1, 1048571
+; LMULMAX2-RV64-NEXT:    addiw a1, a1, 1755
+; LMULMAX2-RV64-NEXT:    vmerge.vxm v28, v28, a1, v0
+; LMULMAX2-RV64-NEXT:    vmulh.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vsra.vi v26, v26, 1
+; LMULMAX2-RV64-NEXT:    vsrl.vi v28, v26, 15
+; LMULMAX2-RV64-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vse16.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    ret
+;
+; LMULMAX1-LABEL: mulhs_v16i16:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; LMULMAX1-NEXT:    vle16.v v25, (a0)
+; LMULMAX1-NEXT:    addi a1, a0, 16
+; LMULMAX1-NEXT:    vle16.v v26, (a1)
+; LMULMAX1-NEXT:    addi a2, zero, 105
+; LMULMAX1-NEXT:    vsetivli a3, 1, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.s.x v0, a2
+; LMULMAX1-NEXT:    vsetivli a2, 8, e16,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.i v27, 7
+; LMULMAX1-NEXT:    vmerge.vim v27, v27, -7, v0
+; LMULMAX1-NEXT:    vdiv.vv v26, v26, v27
+; LMULMAX1-NEXT:    vdiv.vv v25, v25, v27
+; LMULMAX1-NEXT:    vse16.v v25, (a0)
+; LMULMAX1-NEXT:    vse16.v v26, (a1)
+; LMULMAX1-NEXT:    ret
   %a = load <16 x i16>, <16 x i16>* %x
   %b = sdiv <16 x i16> %a, <i16 -7, i16 7, i16 7, i16 -7, i16 7, i16 -7, i16 -7, i16 7, i16 -7, i16 7, i16 7, i16 -7, i16 7, i16 -7, i16 -7, i16 7>
   store <16 x i16> %b, <16 x i16>* %x
@@ -4185,40 +4592,93 @@ define void @mulhs_v16i16(<16 x i16>* %x) {
 }
 
 define void @mulhs_v8i32(<8 x i32>* %x) {
+; LMULMAX2-RV32-LABEL: mulhs_v8i32:
+; LMULMAX2-RV32:       # %bb.0:
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vle32.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 85
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV32-NEXT:    lui a1, 419430
+; LMULMAX2-RV32-NEXT:    addi a1, a1, 1639
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v28, a1
+; LMULMAX2-RV32-NEXT:    lui a1, 629146
+; LMULMAX2-RV32-NEXT:    addi a1, a1, -1639
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v28, v28, a1, v0
+; LMULMAX2-RV32-NEXT:    vmulh.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    vsrl.vi v28, v26, 31
+; LMULMAX2-RV32-NEXT:    vsra.vi v26, v26, 1
+; LMULMAX2-RV32-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    vse32.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    ret
+;
+; LMULMAX2-RV64-LABEL: mulhs_v8i32:
+; LMULMAX2-RV64:       # %bb.0:
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vle32.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    lui a1, 13107
+; LMULMAX2-RV64-NEXT:    addiw a1, a1, 819
+; LMULMAX2-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX2-RV64-NEXT:    addi a1, a1, 973
+; LMULMAX2-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX2-RV64-NEXT:    addi a1, a1, -819
+; LMULMAX2-RV64-NEXT:    slli a1, a1, 13
+; LMULMAX2-RV64-NEXT:    addi a1, a1, -1639
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 4, e64,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.x v28, a1
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmulh.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vsra.vi v26, v26, 1
+; LMULMAX2-RV64-NEXT:    vsrl.vi v28, v26, 31
+; LMULMAX2-RV64-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vse32.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    ret
+;
 ; LMULMAX1-RV32-LABEL: mulhs_v8i32:
 ; LMULMAX1-RV32:       # %bb.0:
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vle32.v v25, (a0)
 ; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV32-NEXT:    vle32.v v25, (a1)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI135_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI135_0)
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a2)
-; LMULMAX1-RV32-NEXT:    vle32.v v27, (a0)
-; LMULMAX1-RV32-NEXT:    vmulh.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vsrl.vi v28, v25, 31
-; LMULMAX1-RV32-NEXT:    vsra.vi v25, v25, 1
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v28
-; LMULMAX1-RV32-NEXT:    vmulh.vv v26, v27, v26
-; LMULMAX1-RV32-NEXT:    vsrl.vi v27, v26, 31
+; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
+; LMULMAX1-RV32-NEXT:    addi a2, zero, 5
+; LMULMAX1-RV32-NEXT:    vsetivli a3, 1, e8,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.s.x v0, a2
+; LMULMAX1-RV32-NEXT:    lui a2, 419430
+; LMULMAX1-RV32-NEXT:    addi a2, a2, 1639
+; LMULMAX1-RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vmv.v.x v27, a2
+; LMULMAX1-RV32-NEXT:    lui a2, 629146
+; LMULMAX1-RV32-NEXT:    addi a2, a2, -1639
+; LMULMAX1-RV32-NEXT:    vmerge.vxm v27, v27, a2, v0
+; LMULMAX1-RV32-NEXT:    vmulh.vv v26, v26, v27
+; LMULMAX1-RV32-NEXT:    vsrl.vi v28, v26, 31
 ; LMULMAX1-RV32-NEXT:    vsra.vi v26, v26, 1
-; LMULMAX1-RV32-NEXT:    vadd.vv v26, v26, v27
-; LMULMAX1-RV32-NEXT:    vse32.v v26, (a0)
-; LMULMAX1-RV32-NEXT:    vse32.v v25, (a1)
+; LMULMAX1-RV32-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX1-RV32-NEXT:    vmulh.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vsrl.vi v27, v25, 31
+; LMULMAX1-RV32-NEXT:    vsra.vi v25, v25, 1
+; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v27
+; LMULMAX1-RV32-NEXT:    vse32.v v25, (a0)
+; LMULMAX1-RV32-NEXT:    vse32.v v26, (a1)
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV64-LABEL: mulhs_v8i32:
 ; LMULMAX1-RV64:       # %bb.0:
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vle32.v v25, (a0)
 ; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
-; LMULMAX1-RV64-NEXT:    vle32.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI135_0)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI135_0)
-; LMULMAX1-RV64-NEXT:    vle32.v v26, (a2)
-; LMULMAX1-RV64-NEXT:    vle32.v v27, (a0)
-; LMULMAX1-RV64-NEXT:    vdiv.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vdiv.vv v26, v27, v26
-; LMULMAX1-RV64-NEXT:    vse32.v v26, (a0)
-; LMULMAX1-RV64-NEXT:    vse32.v v25, (a1)
+; LMULMAX1-RV64-NEXT:    vle32.v v26, (a1)
+; LMULMAX1-RV64-NEXT:    addi a2, zero, 3
+; LMULMAX1-RV64-NEXT:    slli a2, a2, 33
+; LMULMAX1-RV64-NEXT:    addi a2, a2, -5
+; LMULMAX1-RV64-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v27, a2
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vdiv.vv v26, v26, v27
+; LMULMAX1-RV64-NEXT:    vdiv.vv v25, v25, v27
+; LMULMAX1-RV64-NEXT:    vse32.v v25, (a0)
+; LMULMAX1-RV64-NEXT:    vse32.v v26, (a1)
 ; LMULMAX1-RV64-NEXT:    ret
   %a = load <8 x i32>, <8 x i32>* %x
   %b = sdiv <8 x i32> %a, <i32 -5, i32 5, i32 -5, i32 5, i32 -5, i32 5, i32 -5, i32 5>
@@ -4227,6 +4687,77 @@ define void @mulhs_v8i32(<8 x i32>* %x) {
 }
 
 define void @mulhs_v4i64(<4 x i64>* %x) {
+; LMULMAX2-RV32-LABEL: mulhs_v4i64:
+; LMULMAX2-RV32:       # %bb.0:
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vle64.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 51
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.i v28, -1
+; LMULMAX2-RV32-NEXT:    vmerge.vim v28, v28, 0, v0
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmul.vv v28, v26, v28
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 17
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV32-NEXT:    lui a1, 349525
+; LMULMAX2-RV32-NEXT:    addi a2, a1, 1365
+; LMULMAX2-RV32-NEXT:    vsetivli a3, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.x v30, a2
+; LMULMAX2-RV32-NEXT:    addi a1, a1, 1366
+; LMULMAX2-RV32-NEXT:    vmerge.vxm v30, v30, a1, v0
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmulh.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 63
+; LMULMAX2-RV32-NEXT:    vsrl.vx v28, v26, a1
+; LMULMAX2-RV32-NEXT:    addi a1, zero, 68
+; LMULMAX2-RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vmv.v.i v30, 0
+; LMULMAX2-RV32-NEXT:    vmerge.vim v30, v30, 1, v0
+; LMULMAX2-RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV32-NEXT:    vsra.vv v26, v26, v30
+; LMULMAX2-RV32-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV32-NEXT:    vse64.v v26, (a0)
+; LMULMAX2-RV32-NEXT:    ret
+;
+; LMULMAX2-RV64-LABEL: mulhs_v4i64:
+; LMULMAX2-RV64:       # %bb.0:
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vle64.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    addi a1, zero, 5
+; LMULMAX2-RV64-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a1
+; LMULMAX2-RV64-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-RV64-NEXT:    vmv.v.i v28, -1
+; LMULMAX2-RV64-NEXT:    vmerge.vim v28, v28, 0, v0
+; LMULMAX2-RV64-NEXT:    vmul.vv v28, v26, v28
+; LMULMAX2-RV64-NEXT:    lui a1, 21845
+; LMULMAX2-RV64-NEXT:    addiw a1, a1, 1365
+; LMULMAX2-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX2-RV64-NEXT:    addi a1, a1, 1365
+; LMULMAX2-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX2-RV64-NEXT:    addi a1, a1, 1365
+; LMULMAX2-RV64-NEXT:    slli a1, a1, 12
+; LMULMAX2-RV64-NEXT:    addi a2, a1, 1365
+; LMULMAX2-RV64-NEXT:    vmv.v.x v30, a2
+; LMULMAX2-RV64-NEXT:    addi a1, a1, 1366
+; LMULMAX2-RV64-NEXT:    vmerge.vxm v30, v30, a1, v0
+; LMULMAX2-RV64-NEXT:    vmulh.vv v26, v26, v30
+; LMULMAX2-RV64-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    addi a1, zero, 63
+; LMULMAX2-RV64-NEXT:    vsrl.vx v28, v26, a1
+; LMULMAX2-RV64-NEXT:    vmv.v.i v30, 1
+; LMULMAX2-RV64-NEXT:    vmerge.vim v30, v30, 0, v0
+; LMULMAX2-RV64-NEXT:    vsra.vv v26, v26, v30
+; LMULMAX2-RV64-NEXT:    vadd.vv v26, v26, v28
+; LMULMAX2-RV64-NEXT:    vse64.v v26, (a0)
+; LMULMAX2-RV64-NEXT:    ret
+;
 ; LMULMAX1-RV32-LABEL: mulhs_v4i64:
 ; LMULMAX1-RV32:       # %bb.0:
 ; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
@@ -4249,24 +4780,31 @@ define void @mulhs_v4i64(<4 x i64>* %x) {
 ; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
 ; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle64.v v25, (a1)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI136_0)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI136_0)
-; LMULMAX1-RV64-NEXT:    vle64.v v26, (a2)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI136_1)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI136_1)
-; LMULMAX1-RV64-NEXT:    vle64.v v27, (a2)
-; LMULMAX1-RV64-NEXT:    vle64.v v28, (a0)
-; LMULMAX1-RV64-NEXT:    vmul.vv v29, v25, v26
-; LMULMAX1-RV64-NEXT:    vmulh.vv v25, v25, v27
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v29
+; LMULMAX1-RV64-NEXT:    vle64.v v26, (a0)
+; LMULMAX1-RV64-NEXT:    vmv.v.i v27, -1
+; LMULMAX1-RV64-NEXT:    vmv.s.x v27, zero
+; LMULMAX1-RV64-NEXT:    vmul.vv v28, v25, v27
+; LMULMAX1-RV64-NEXT:    lui a2, 21845
+; LMULMAX1-RV64-NEXT:    addiw a2, a2, 1365
+; LMULMAX1-RV64-NEXT:    slli a2, a2, 12
+; LMULMAX1-RV64-NEXT:    addi a2, a2, 1365
+; LMULMAX1-RV64-NEXT:    slli a2, a2, 12
+; LMULMAX1-RV64-NEXT:    addi a2, a2, 1365
+; LMULMAX1-RV64-NEXT:    slli a2, a2, 12
+; LMULMAX1-RV64-NEXT:    addi a3, a2, 1365
+; LMULMAX1-RV64-NEXT:    vmv.v.x v29, a3
+; LMULMAX1-RV64-NEXT:    addi a2, a2, 1366
+; LMULMAX1-RV64-NEXT:    vmv.s.x v29, a2
+; LMULMAX1-RV64-NEXT:    vmulh.vv v25, v25, v29
+; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v28
 ; LMULMAX1-RV64-NEXT:    addi a2, zero, 63
-; LMULMAX1-RV64-NEXT:    vsrl.vx v29, v25, a2
+; LMULMAX1-RV64-NEXT:    vsrl.vx v28, v25, a2
 ; LMULMAX1-RV64-NEXT:    vid.v v30
 ; LMULMAX1-RV64-NEXT:    vsra.vv v25, v25, v30
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v29
-; LMULMAX1-RV64-NEXT:    vmul.vv v26, v28, v26
-; LMULMAX1-RV64-NEXT:    vmulh.vv v27, v28, v27
-; LMULMAX1-RV64-NEXT:    vadd.vv v26, v27, v26
+; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v28
+; LMULMAX1-RV64-NEXT:    vmul.vv v27, v26, v27
+; LMULMAX1-RV64-NEXT:    vmulh.vv v26, v26, v29
+; LMULMAX1-RV64-NEXT:    vadd.vv v26, v26, v27
 ; LMULMAX1-RV64-NEXT:    vsrl.vx v27, v26, a2
 ; LMULMAX1-RV64-NEXT:    vsra.vv v26, v26, v30
 ; LMULMAX1-RV64-NEXT:    vadd.vv v26, v26, v27
@@ -5084,24 +5622,24 @@ define void @add_vi_v4i32(<4 x i32>* %x) {
 }
 
 define void @add_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: add_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.i v26, -1
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: add_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vadd.vi v25, v25, -1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: add_vi_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a0)
+; RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v26, -1
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vadd.vv v25, v25, v26
+; RV32-NEXT:    vse64.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: add_vi_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a0)
+; RV64-NEXT:    vadd.vi v25, v25, -1
+; RV64-NEXT:    vse64.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 -1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -5159,26 +5697,13 @@ define void @add_iv_v4i32(<4 x i32>* %x) {
 }
 
 define void @add_iv_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: add_iv_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI160_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI160_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: add_iv_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vadd.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: add_iv_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vadd.vi v25, v25, 1
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -5335,25 +5860,25 @@ define void @sub_vi_v4i32(<4 x i32>* %x) {
 }
 
 define void @sub_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: sub_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.i v26, -1
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsub.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: sub_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    addi a1, zero, -1
-; LMULMAX1-RV64-NEXT:    vsub.vx v25, v25, a1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: sub_vi_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a0)
+; RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v26, -1
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vsub.vv v25, v25, v26
+; RV32-NEXT:    vse64.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sub_vi_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a0)
+; RV64-NEXT:    addi a1, zero, -1
+; RV64-NEXT:    vsub.vx v25, v25, a1
+; RV64-NEXT:    vse64.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 -1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -5411,26 +5936,13 @@ define void @sub_iv_v4i32(<4 x i32>* %x) {
 }
 
 define void @sub_iv_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: sub_iv_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI174_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI174_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsub.vv v25, v26, v25
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: sub_iv_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vrsub.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: sub_iv_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vrsub.vi v25, v25, 1
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -5680,26 +6192,13 @@ define void @and_vi_v4i32(<4 x i32>* %x) {
 }
 
 define void @and_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: and_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI190_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI190_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: and_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vand.vi v25, v25, -2
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: and_vi_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vand.vi v25, v25, -2
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 -2, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -5757,26 +6256,13 @@ define void @and_iv_v4i32(<4 x i32>* %x) {
 }
 
 define void @and_iv_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: and_iv_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI194_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI194_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vand.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: and_iv_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vand.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: and_iv_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vand.vi v25, v25, 1
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -5930,26 +6416,13 @@ define void @or_vi_v4i32(<4 x i32>* %x) {
 }
 
 define void @or_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: or_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI204_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI204_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: or_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vor.vi v25, v25, -2
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: or_vi_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vor.vi v25, v25, -2
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 -2, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -6007,26 +6480,13 @@ define void @or_iv_v4i32(<4 x i32>* %x) {
 }
 
 define void @or_iv_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: or_iv_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI208_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI208_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vor.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: or_iv_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vor.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: or_iv_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vor.vi v25, v25, 1
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -6180,24 +6640,24 @@ define void @xor_vi_v4i32(<4 x i32>* %x) {
 }
 
 define void @xor_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: xor_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmv.v.i v26, -1
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vxor.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: xor_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vnot.v v25, v25
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: xor_vi_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a0)
+; RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v26, -1
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vxor.vv v25, v25, v26
+; RV32-NEXT:    vse64.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: xor_vi_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a0)
+; RV64-NEXT:    vnot.v v25, v25
+; RV64-NEXT:    vse64.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 -1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -6255,26 +6715,13 @@ define void @xor_iv_v4i32(<4 x i32>* %x) {
 }
 
 define void @xor_iv_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: xor_iv_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI222_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI222_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vxor.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: xor_iv_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vxor.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: xor_iv_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vxor.vi v25, v25, 1
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 1, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -6428,26 +6875,13 @@ define void @lshr_vi_v4i32(<4 x i32>* %x) {
 }
 
 define void @lshr_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: lshr_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI232_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI232_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: lshr_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vsrl.vi v25, v25, 31
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: lshr_vi_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vsrl.vi v25, v25, 31
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 31, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -6553,26 +6987,13 @@ define void @ashr_vi_v4i32(<4 x i32>* %x) {
 }
 
 define void @ashr_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: ashr_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI239_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI239_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsra.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: ashr_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vsra.vi v25, v25, 31
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: ashr_vi_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vsra.vi v25, v25, 31
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 31, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -6678,26 +7099,13 @@ define void @shl_vi_v4i32(<4 x i32>* %x) {
 }
 
 define void @shl_vi_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: shl_vi_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI246_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI246_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsll.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: shl_vi_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    vsll.vi v25, v25, 31
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; CHECK-LABEL: shl_vi_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a0)
+; CHECK-NEXT:    vsll.vi v25, v25, 31
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = insertelement <2 x i64> undef, i64 31, i32 0
   %c = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
@@ -6963,33 +7371,33 @@ define void @mulhu_vx_v16i8(<16 x i8>* %x) {
 }
 
 define void @mulhu_vx_v8i16(<8 x i16>* %x) {
-; LMULMAX1-RV32-LABEL: mulhu_vx_v8i16:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle16.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, 2
-; LMULMAX1-RV32-NEXT:    addi a1, a1, 1171
-; LMULMAX1-RV32-NEXT:    vmulhu.vx v26, v25, a1
-; LMULMAX1-RV32-NEXT:    vsub.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 1
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 2
-; LMULMAX1-RV32-NEXT:    vse16.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhu_vx_v8i16:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle16.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, 2
-; LMULMAX1-RV64-NEXT:    addiw a1, a1, 1171
-; LMULMAX1-RV64-NEXT:    vmulhu.vx v26, v25, a1
-; LMULMAX1-RV64-NEXT:    vsub.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vsrl.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vsrl.vi v25, v25, 2
-; LMULMAX1-RV64-NEXT:    vse16.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: mulhu_vx_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; RV32-NEXT:    vle16.v v25, (a0)
+; RV32-NEXT:    lui a1, 2
+; RV32-NEXT:    addi a1, a1, 1171
+; RV32-NEXT:    vmulhu.vx v26, v25, a1
+; RV32-NEXT:    vsub.vv v25, v25, v26
+; RV32-NEXT:    vsrl.vi v25, v25, 1
+; RV32-NEXT:    vadd.vv v25, v25, v26
+; RV32-NEXT:    vsrl.vi v25, v25, 2
+; RV32-NEXT:    vse16.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhu_vx_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; RV64-NEXT:    vle16.v v25, (a0)
+; RV64-NEXT:    lui a1, 2
+; RV64-NEXT:    addiw a1, a1, 1171
+; RV64-NEXT:    vmulhu.vx v26, v25, a1
+; RV64-NEXT:    vsub.vv v25, v25, v26
+; RV64-NEXT:    vsrl.vi v25, v25, 1
+; RV64-NEXT:    vadd.vv v25, v25, v26
+; RV64-NEXT:    vsrl.vi v25, v25, 2
+; RV64-NEXT:    vse16.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <8 x i16>, <8 x i16>* %x
   %b = udiv <8 x i16> %a, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
   store <8 x i16> %b, <8 x i16>* %x
@@ -6997,27 +7405,27 @@ define void @mulhu_vx_v8i16(<8 x i16>* %x) {
 }
 
 define void @mulhu_vx_v4i32(<4 x i32>* %x) {
-; LMULMAX1-RV32-LABEL: mulhu_vx_v4i32:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, 838861
-; LMULMAX1-RV32-NEXT:    addi a1, a1, -819
-; LMULMAX1-RV32-NEXT:    vmulhu.vx v25, v25, a1
-; LMULMAX1-RV32-NEXT:    vsrl.vi v25, v25, 2
-; LMULMAX1-RV32-NEXT:    vse32.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhu_vx_v4i32:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle32.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, 838861
-; LMULMAX1-RV64-NEXT:    addiw a1, a1, -819
-; LMULMAX1-RV64-NEXT:    vmulhu.vx v25, v25, a1
-; LMULMAX1-RV64-NEXT:    vsrl.vi v25, v25, 2
-; LMULMAX1-RV64-NEXT:    vse32.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: mulhu_vx_v4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-NEXT:    vle32.v v25, (a0)
+; RV32-NEXT:    lui a1, 838861
+; RV32-NEXT:    addi a1, a1, -819
+; RV32-NEXT:    vmulhu.vx v25, v25, a1
+; RV32-NEXT:    vsrl.vi v25, v25, 2
+; RV32-NEXT:    vse32.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhu_vx_v4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV64-NEXT:    vle32.v v25, (a0)
+; RV64-NEXT:    lui a1, 838861
+; RV64-NEXT:    addiw a1, a1, -819
+; RV64-NEXT:    vmulhu.vx v25, v25, a1
+; RV64-NEXT:    vsrl.vi v25, v25, 2
+; RV64-NEXT:    vse32.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <4 x i32>, <4 x i32>* %x
   %b = udiv <4 x i32> %a, <i32 5, i32 5, i32 5, i32 5>
   store <4 x i32> %b, <4 x i32>* %x
@@ -7025,41 +7433,41 @@ define void @mulhu_vx_v4i32(<4 x i32>* %x) {
 }
 
 define void @mulhu_vx_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: mulhu_vx_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI265_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI265_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmulhu.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI265_1)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI265_1)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhu_vx_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, 1026731
-; LMULMAX1-RV64-NEXT:    addiw a1, a1, -1365
-; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
-; LMULMAX1-RV64-NEXT:    addi a1, a1, -1365
-; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
-; LMULMAX1-RV64-NEXT:    addi a1, a1, -1365
-; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
-; LMULMAX1-RV64-NEXT:    addi a1, a1, -1365
-; LMULMAX1-RV64-NEXT:    vmulhu.vx v25, v25, a1
-; LMULMAX1-RV64-NEXT:    vsrl.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: mulhu_vx_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a0)
+; RV32-NEXT:    addi a1, zero, 5
+; RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    lui a1, 699051
+; RV32-NEXT:    addi a2, a1, -1366
+; RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.x v26, a2
+; RV32-NEXT:    addi a1, a1, -1365
+; RV32-NEXT:    vmerge.vxm v26, v26, a1, v0
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vmulhu.vv v25, v25, v26
+; RV32-NEXT:    vsrl.vi v25, v25, 1
+; RV32-NEXT:    vse64.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhu_vx_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a0)
+; RV64-NEXT:    lui a1, 1026731
+; RV64-NEXT:    addiw a1, a1, -1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, -1365
+; RV64-NEXT:    vmulhu.vx v25, v25, a1
+; RV64-NEXT:    vsrl.vi v25, v25, 1
+; RV64-NEXT:    vse64.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = udiv <2 x i64> %a, <i64 3, i64 3>
   store <2 x i64> %b, <2 x i64>* %x
@@ -7083,31 +7491,31 @@ define void @mulhs_vx_v16i8(<16 x i8>* %x) {
 }
 
 define void @mulhs_vx_v8i16(<8 x i16>* %x) {
-; LMULMAX1-RV32-LABEL: mulhs_vx_v8i16:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle16.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, 5
-; LMULMAX1-RV32-NEXT:    addi a1, a1, -1755
-; LMULMAX1-RV32-NEXT:    vmulh.vx v25, v25, a1
-; LMULMAX1-RV32-NEXT:    vsra.vi v25, v25, 1
-; LMULMAX1-RV32-NEXT:    vsrl.vi v26, v25, 15
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse16.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhs_vx_v8i16:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle16.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, 5
-; LMULMAX1-RV64-NEXT:    addiw a1, a1, -1755
-; LMULMAX1-RV64-NEXT:    vmulh.vx v25, v25, a1
-; LMULMAX1-RV64-NEXT:    vsra.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vsrl.vi v26, v25, 15
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vse16.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: mulhs_vx_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; RV32-NEXT:    vle16.v v25, (a0)
+; RV32-NEXT:    lui a1, 5
+; RV32-NEXT:    addi a1, a1, -1755
+; RV32-NEXT:    vmulh.vx v25, v25, a1
+; RV32-NEXT:    vsra.vi v25, v25, 1
+; RV32-NEXT:    vsrl.vi v26, v25, 15
+; RV32-NEXT:    vadd.vv v25, v25, v26
+; RV32-NEXT:    vse16.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhs_vx_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; RV64-NEXT:    vle16.v v25, (a0)
+; RV64-NEXT:    lui a1, 5
+; RV64-NEXT:    addiw a1, a1, -1755
+; RV64-NEXT:    vmulh.vx v25, v25, a1
+; RV64-NEXT:    vsra.vi v25, v25, 1
+; RV64-NEXT:    vsrl.vi v26, v25, 15
+; RV64-NEXT:    vadd.vv v25, v25, v26
+; RV64-NEXT:    vse16.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <8 x i16>, <8 x i16>* %x
   %b = sdiv <8 x i16> %a, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
   store <8 x i16> %b, <8 x i16>* %x
@@ -7115,31 +7523,31 @@ define void @mulhs_vx_v8i16(<8 x i16>* %x) {
 }
 
 define void @mulhs_vx_v4i32(<4 x i32>* %x) {
-; LMULMAX1-RV32-LABEL: mulhs_vx_v4i32:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, 629146
-; LMULMAX1-RV32-NEXT:    addi a1, a1, -1639
-; LMULMAX1-RV32-NEXT:    vmulh.vx v25, v25, a1
-; LMULMAX1-RV32-NEXT:    vsrl.vi v26, v25, 31
-; LMULMAX1-RV32-NEXT:    vsra.vi v25, v25, 1
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse32.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhs_vx_v4i32:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle32.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, 629146
-; LMULMAX1-RV64-NEXT:    addiw a1, a1, -1639
-; LMULMAX1-RV64-NEXT:    vmulh.vx v25, v25, a1
-; LMULMAX1-RV64-NEXT:    vsra.vi v25, v25, 1
-; LMULMAX1-RV64-NEXT:    vsrl.vi v26, v25, 31
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vse32.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: mulhs_vx_v4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-NEXT:    vle32.v v25, (a0)
+; RV32-NEXT:    lui a1, 629146
+; RV32-NEXT:    addi a1, a1, -1639
+; RV32-NEXT:    vmulh.vx v25, v25, a1
+; RV32-NEXT:    vsrl.vi v26, v25, 31
+; RV32-NEXT:    vsra.vi v25, v25, 1
+; RV32-NEXT:    vadd.vv v25, v25, v26
+; RV32-NEXT:    vse32.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhs_vx_v4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV64-NEXT:    vle32.v v25, (a0)
+; RV64-NEXT:    lui a1, 629146
+; RV64-NEXT:    addiw a1, a1, -1639
+; RV64-NEXT:    vmulh.vx v25, v25, a1
+; RV64-NEXT:    vsra.vi v25, v25, 1
+; RV64-NEXT:    vsrl.vi v26, v25, 31
+; RV64-NEXT:    vadd.vv v25, v25, v26
+; RV64-NEXT:    vse32.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <4 x i32>, <4 x i32>* %x
   %b = sdiv <4 x i32> %a, <i32 -5, i32 -5, i32 -5, i32 -5>
   store <4 x i32> %b, <4 x i32>* %x
@@ -7147,44 +7555,45 @@ define void @mulhs_vx_v4i32(<4 x i32>* %x) {
 }
 
 define void @mulhs_vx_v2i64(<2 x i64>* %x) {
-; LMULMAX1-RV32-LABEL: mulhs_vx_v2i64:
-; LMULMAX1-RV32:       # %bb.0:
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI269_0)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI269_0)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vmulh.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI269_1)
-; LMULMAX1-RV32-NEXT:    addi a1, a1, %lo(.LCPI269_1)
-; LMULMAX1-RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vle32.v v26, (a1)
-; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV32-NEXT:    vsrl.vv v26, v25, v26
-; LMULMAX1-RV32-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV32-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV32-NEXT:    ret
-;
-; LMULMAX1-RV64-LABEL: mulhs_vx_v2i64:
-; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; LMULMAX1-RV64-NEXT:    vle64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, 21845
-; LMULMAX1-RV64-NEXT:    addiw a1, a1, 1365
-; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
-; LMULMAX1-RV64-NEXT:    addi a1, a1, 1365
-; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
-; LMULMAX1-RV64-NEXT:    addi a1, a1, 1365
-; LMULMAX1-RV64-NEXT:    slli a1, a1, 12
-; LMULMAX1-RV64-NEXT:    addi a1, a1, 1366
-; LMULMAX1-RV64-NEXT:    vmulh.vx v25, v25, a1
-; LMULMAX1-RV64-NEXT:    addi a1, zero, 63
-; LMULMAX1-RV64-NEXT:    vsrl.vx v26, v25, a1
-; LMULMAX1-RV64-NEXT:    vadd.vv v25, v25, v26
-; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
-; LMULMAX1-RV64-NEXT:    ret
+; RV32-LABEL: mulhs_vx_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a0)
+; RV32-NEXT:    addi a1, zero, 5
+; RV32-NEXT:    vsetivli a2, 1, e8,m1,ta,mu
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    lui a1, 349525
+; RV32-NEXT:    addi a2, a1, 1365
+; RV32-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.x v26, a2
+; RV32-NEXT:    addi a1, a1, 1366
+; RV32-NEXT:    vmerge.vxm v26, v26, a1, v0
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vmulh.vv v25, v25, v26
+; RV32-NEXT:    addi a1, zero, 63
+; RV32-NEXT:    vsrl.vx v26, v25, a1
+; RV32-NEXT:    vadd.vv v25, v25, v26
+; RV32-NEXT:    vse64.v v25, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhs_vx_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a0)
+; RV64-NEXT:    lui a1, 21845
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, 1365
+; RV64-NEXT:    slli a1, a1, 12
+; RV64-NEXT:    addi a1, a1, 1366
+; RV64-NEXT:    vmulh.vx v25, v25, a1
+; RV64-NEXT:    addi a1, zero, 63
+; RV64-NEXT:    vsrl.vx v26, v25, a1
+; RV64-NEXT:    vadd.vv v25, v25, v26
+; RV64-NEXT:    vse64.v v25, (a0)
+; RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = sdiv <2 x i64> %a, <i64 3, i64 3>
   store <2 x i64> %b, <2 x i64>* %x
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e0fbab5539203be64cb2eadfbf074a2b8317f3e4
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
@@ -0,0 +1,570 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,RV32-LMULMAX1
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,RV64-LMULMAX1
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,RV32-LMULMAX2
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,RV64-LMULMAX2
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=4 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,RV32-LMULMAX4
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=4 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,RV64-LMULMAX4
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,RV32-LMULMAX8
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,RV64-LMULMAX8
+
+define <3 x i1> @buildvec_mask_v1i1() {
+; CHECK-LABEL: buildvec_mask_v1i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 2
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    ret
+  ret <3 x i1> <i1 0, i1 1, i1 0>
+}
+
+define <3 x i1> @buildvec_mask_optsize_v1i1() optsize {
+; CHECK-LABEL: buildvec_mask_optsize_v1i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 2
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    ret
+  ret <3 x i1> <i1 0, i1 1, i1 0>
+}
+
+define <4 x i1> @buildvec_mask_v4i1() {
+; CHECK-LABEL: buildvec_mask_v4i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 6
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    ret
+  ret <4 x i1> <i1 0, i1 1, i1 1, i1 0>
+}
+
+define <8 x i1> @buildvec_mask_v8i1() {
+; CHECK-LABEL: buildvec_mask_v8i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 182
+; CHECK-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    ret
+  ret <8 x i1> <i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1>
+}
+
+define <10 x i1> @buildvec_mask_v10i1() {
+; CHECK-LABEL: buildvec_mask_v10i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 949
+; CHECK-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    ret
+  ret <10 x i1> <i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 1>
+}
+
+define <16 x i1> @buildvec_mask_v16i1() {
+; CHECK-RV32-LABEL: buildvec_mask_v16i1:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lui a0, 11
+; CHECK-RV32-NEXT:    addi a0, a0, 1718
+; CHECK-RV32-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; CHECK-RV32-NEXT:    vmv.s.x v0, a0
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64-LABEL: buildvec_mask_v16i1:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    lui a0, 11
+; CHECK-RV64-NEXT:    addiw a0, a0, 1718
+; CHECK-RV64-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; CHECK-RV64-NEXT:    vmv.s.x v0, a0
+; CHECK-RV64-NEXT:    ret
+  ret <16 x i1> <i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1>
+}
+
+define <16 x i1> @buildvec_mask_v16i1_undefs() {
+; CHECK-LABEL: buildvec_mask_v16i1_undefs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 1722
+; CHECK-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    ret
+  ret <16 x i1> <i1 undef, i1 1, i1 undef, i1 1, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 undef, i1 undef, i1 undef, i1 undef, i1 undef>
+}
+
+define <32 x i1> @buildvec_mask_v32i1() {
+; RV32-LMULMAX1-LABEL: buildvec_mask_v32i1:
+; RV32-LMULMAX1:       # %bb.0:
+; RV32-LMULMAX1-NEXT:    addi a0, zero, 1776
+; RV32-LMULMAX1-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; RV32-LMULMAX1-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 11
+; RV32-LMULMAX1-NEXT:    addi a0, a0, 1718
+; RV32-LMULMAX1-NEXT:    vmv.s.x v8, a0
+; RV32-LMULMAX1-NEXT:    ret
+;
+; RV64-LMULMAX1-LABEL: buildvec_mask_v32i1:
+; RV64-LMULMAX1:       # %bb.0:
+; RV64-LMULMAX1-NEXT:    addi a0, zero, 1776
+; RV64-LMULMAX1-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; RV64-LMULMAX1-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 11
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, 1718
+; RV64-LMULMAX1-NEXT:    vmv.s.x v8, a0
+; RV64-LMULMAX1-NEXT:    ret
+;
+; RV32-LMULMAX2-LABEL: buildvec_mask_v32i1:
+; RV32-LMULMAX2:       # %bb.0:
+; RV32-LMULMAX2-NEXT:    lui a0, 748384
+; RV32-LMULMAX2-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX2-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV32-LMULMAX2-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX2-NEXT:    ret
+;
+; RV64-LMULMAX2-LABEL: buildvec_mask_v32i1:
+; RV64-LMULMAX2:       # %bb.0:
+; RV64-LMULMAX2-NEXT:    lui a0, 748384
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, 1776
+; RV64-LMULMAX2-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV64-LMULMAX2-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX2-NEXT:    ret
+;
+; RV32-LMULMAX4-LABEL: buildvec_mask_v32i1:
+; RV32-LMULMAX4:       # %bb.0:
+; RV32-LMULMAX4-NEXT:    lui a0, 748384
+; RV32-LMULMAX4-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX4-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV32-LMULMAX4-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX4-NEXT:    ret
+;
+; RV64-LMULMAX4-LABEL: buildvec_mask_v32i1:
+; RV64-LMULMAX4:       # %bb.0:
+; RV64-LMULMAX4-NEXT:    lui a0, 748384
+; RV64-LMULMAX4-NEXT:    addiw a0, a0, 1776
+; RV64-LMULMAX4-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX4-NEXT:    ret
+;
+; RV32-LMULMAX8-LABEL: buildvec_mask_v32i1:
+; RV32-LMULMAX8:       # %bb.0:
+; RV32-LMULMAX8-NEXT:    lui a0, 748384
+; RV32-LMULMAX8-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX8-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV32-LMULMAX8-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX8-NEXT:    ret
+;
+; RV64-LMULMAX8-LABEL: buildvec_mask_v32i1:
+; RV64-LMULMAX8:       # %bb.0:
+; RV64-LMULMAX8-NEXT:    lui a0, 748384
+; RV64-LMULMAX8-NEXT:    addiw a0, a0, 1776
+; RV64-LMULMAX8-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV64-LMULMAX8-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX8-NEXT:    ret
+  ret <32 x i1> <i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1>
+}
+
+define <64 x i1> @buildvec_mask_v64i1() {
+; RV32-LMULMAX1-LABEL: buildvec_mask_v64i1:
+; RV32-LMULMAX1:       # %bb.0:
+; RV32-LMULMAX1-NEXT:    addi a0, zero, 1776
+; RV32-LMULMAX1-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; RV32-LMULMAX1-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 4
+; RV32-LMULMAX1-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX1-NEXT:    vmv.s.x v9, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 11
+; RV32-LMULMAX1-NEXT:    addi a0, a0, 1718
+; RV32-LMULMAX1-NEXT:    vmv.s.x v8, a0
+; RV32-LMULMAX1-NEXT:    vmv1r.v v10, v8
+; RV32-LMULMAX1-NEXT:    ret
+;
+; RV64-LMULMAX1-LABEL: buildvec_mask_v64i1:
+; RV64-LMULMAX1:       # %bb.0:
+; RV64-LMULMAX1-NEXT:    addi a0, zero, 1776
+; RV64-LMULMAX1-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; RV64-LMULMAX1-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 4
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, -1793
+; RV64-LMULMAX1-NEXT:    vmv.s.x v9, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 11
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, 1718
+; RV64-LMULMAX1-NEXT:    vmv.s.x v8, a0
+; RV64-LMULMAX1-NEXT:    vmv1r.v v10, v8
+; RV64-LMULMAX1-NEXT:    ret
+;
+; RV32-LMULMAX2-LABEL: buildvec_mask_v64i1:
+; RV32-LMULMAX2:       # %bb.0:
+; RV32-LMULMAX2-NEXT:    lui a0, 748384
+; RV32-LMULMAX2-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX2-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV32-LMULMAX2-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX2-NEXT:    lui a0, 748388
+; RV32-LMULMAX2-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX2-NEXT:    vmv.s.x v8, a0
+; RV32-LMULMAX2-NEXT:    ret
+;
+; RV64-LMULMAX2-LABEL: buildvec_mask_v64i1:
+; RV64-LMULMAX2:       # %bb.0:
+; RV64-LMULMAX2-NEXT:    lui a0, 748384
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, 1776
+; RV64-LMULMAX2-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV64-LMULMAX2-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX2-NEXT:    lui a0, 748388
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, -1793
+; RV64-LMULMAX2-NEXT:    vmv.s.x v8, a0
+; RV64-LMULMAX2-NEXT:    ret
+;
+; RV32-LMULMAX4-LABEL: buildvec_mask_v64i1:
+; RV32-LMULMAX4:       # %bb.0:
+; RV32-LMULMAX4-NEXT:    lui a0, 748388
+; RV32-LMULMAX4-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX4-NEXT:    vsetivli a1, 2, e32,m1,ta,mu
+; RV32-LMULMAX4-NEXT:    vmv.s.x v25, a0
+; RV32-LMULMAX4-NEXT:    lui a0, 748384
+; RV32-LMULMAX4-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX4-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX4-NEXT:    vsetivli a0, 2, e32,m1,tu,mu
+; RV32-LMULMAX4-NEXT:    vslideup.vi v0, v25, 1
+; RV32-LMULMAX4-NEXT:    ret
+;
+; RV64-LMULMAX4-LABEL: buildvec_mask_v64i1:
+; RV64-LMULMAX4:       # %bb.0:
+; RV64-LMULMAX4-NEXT:    lui a0, 1048429
+; RV64-LMULMAX4-NEXT:    addiw a0, a0, 1735
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 1023
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX4-NEXT:    addi a0, a0, -1189
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 17
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 1776
+; RV64-LMULMAX4-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX4-NEXT:    ret
+;
+; RV32-LMULMAX8-LABEL: buildvec_mask_v64i1:
+; RV32-LMULMAX8:       # %bb.0:
+; RV32-LMULMAX8-NEXT:    lui a0, 748388
+; RV32-LMULMAX8-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX8-NEXT:    vsetivli a1, 2, e32,m1,ta,mu
+; RV32-LMULMAX8-NEXT:    vmv.s.x v25, a0
+; RV32-LMULMAX8-NEXT:    lui a0, 748384
+; RV32-LMULMAX8-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX8-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX8-NEXT:    vsetivli a0, 2, e32,m1,tu,mu
+; RV32-LMULMAX8-NEXT:    vslideup.vi v0, v25, 1
+; RV32-LMULMAX8-NEXT:    ret
+;
+; RV64-LMULMAX8-LABEL: buildvec_mask_v64i1:
+; RV64-LMULMAX8:       # %bb.0:
+; RV64-LMULMAX8-NEXT:    lui a0, 1048429
+; RV64-LMULMAX8-NEXT:    addiw a0, a0, 1735
+; RV64-LMULMAX8-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX8-NEXT:    addi a0, a0, 1023
+; RV64-LMULMAX8-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX8-NEXT:    addi a0, a0, -1189
+; RV64-LMULMAX8-NEXT:    slli a0, a0, 17
+; RV64-LMULMAX8-NEXT:    addi a0, a0, 1776
+; RV64-LMULMAX8-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; RV64-LMULMAX8-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX8-NEXT:    ret
+  ret <64 x i1> <i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1>
+}
+
+define <128 x i1> @buildvec_mask_v128i1() {
+; RV32-LMULMAX1-LABEL: buildvec_mask_v128i1:
+; RV32-LMULMAX1:       # %bb.0:
+; RV32-LMULMAX1-NEXT:    addi a0, zero, 1776
+; RV32-LMULMAX1-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; RV32-LMULMAX1-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 11
+; RV32-LMULMAX1-NEXT:    addi a0, a0, 1718
+; RV32-LMULMAX1-NEXT:    vmv.s.x v8, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 8
+; RV32-LMULMAX1-NEXT:    addi a0, a0, 1718
+; RV32-LMULMAX1-NEXT:    vmv.s.x v12, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 4
+; RV32-LMULMAX1-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX1-NEXT:    vmv.s.x v9, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 14
+; RV32-LMULMAX1-NEXT:    addi a0, a0, 1722
+; RV32-LMULMAX1-NEXT:    vmv.s.x v14, a0
+; RV32-LMULMAX1-NEXT:    vmv1r.v v10, v8
+; RV32-LMULMAX1-NEXT:    vmv1r.v v11, v0
+; RV32-LMULMAX1-NEXT:    vmv1r.v v13, v9
+; RV32-LMULMAX1-NEXT:    ret
+;
+; RV64-LMULMAX1-LABEL: buildvec_mask_v128i1:
+; RV64-LMULMAX1:       # %bb.0:
+; RV64-LMULMAX1-NEXT:    addi a0, zero, 1776
+; RV64-LMULMAX1-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; RV64-LMULMAX1-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 11
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, 1718
+; RV64-LMULMAX1-NEXT:    vmv.s.x v8, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 8
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, 1718
+; RV64-LMULMAX1-NEXT:    vmv.s.x v12, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 4
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, -1793
+; RV64-LMULMAX1-NEXT:    vmv.s.x v9, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 14
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, 1722
+; RV64-LMULMAX1-NEXT:    vmv.s.x v14, a0
+; RV64-LMULMAX1-NEXT:    vmv1r.v v10, v8
+; RV64-LMULMAX1-NEXT:    vmv1r.v v11, v0
+; RV64-LMULMAX1-NEXT:    vmv1r.v v13, v9
+; RV64-LMULMAX1-NEXT:    ret
+;
+; RV32-LMULMAX2-LABEL: buildvec_mask_v128i1:
+; RV32-LMULMAX2:       # %bb.0:
+; RV32-LMULMAX2-NEXT:    lui a0, 748384
+; RV32-LMULMAX2-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX2-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV32-LMULMAX2-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX2-NEXT:    lui a0, 748388
+; RV32-LMULMAX2-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX2-NEXT:    vmv.s.x v8, a0
+; RV32-LMULMAX2-NEXT:    lui a0, 551776
+; RV32-LMULMAX2-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX2-NEXT:    vmv.s.x v9, a0
+; RV32-LMULMAX2-NEXT:    lui a0, 945060
+; RV32-LMULMAX2-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX2-NEXT:    vmv.s.x v10, a0
+; RV32-LMULMAX2-NEXT:    ret
+;
+; RV64-LMULMAX2-LABEL: buildvec_mask_v128i1:
+; RV64-LMULMAX2:       # %bb.0:
+; RV64-LMULMAX2-NEXT:    lui a0, 748384
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, 1776
+; RV64-LMULMAX2-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV64-LMULMAX2-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX2-NEXT:    lui a0, 748388
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, -1793
+; RV64-LMULMAX2-NEXT:    vmv.s.x v8, a0
+; RV64-LMULMAX2-NEXT:    lui a0, 551776
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, 1776
+; RV64-LMULMAX2-NEXT:    vmv.s.x v9, a0
+; RV64-LMULMAX2-NEXT:    lui a0, 945060
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, -1793
+; RV64-LMULMAX2-NEXT:    vmv.s.x v10, a0
+; RV64-LMULMAX2-NEXT:    ret
+;
+; RV32-LMULMAX4-LABEL: buildvec_mask_v128i1:
+; RV32-LMULMAX4:       # %bb.0:
+; RV32-LMULMAX4-NEXT:    lui a0, 748388
+; RV32-LMULMAX4-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX4-NEXT:    vsetivli a1, 2, e32,m1,ta,mu
+; RV32-LMULMAX4-NEXT:    vmv.s.x v25, a0
+; RV32-LMULMAX4-NEXT:    lui a0, 748384
+; RV32-LMULMAX4-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX4-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX4-NEXT:    vsetivli a0, 2, e32,m1,tu,mu
+; RV32-LMULMAX4-NEXT:    vslideup.vi v0, v25, 1
+; RV32-LMULMAX4-NEXT:    lui a0, 945060
+; RV32-LMULMAX4-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX4-NEXT:    vsetivli a1, 2, e32,m1,ta,mu
+; RV32-LMULMAX4-NEXT:    vmv.s.x v25, a0
+; RV32-LMULMAX4-NEXT:    lui a0, 551776
+; RV32-LMULMAX4-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX4-NEXT:    vmv.s.x v8, a0
+; RV32-LMULMAX4-NEXT:    vsetivli a0, 2, e32,m1,tu,mu
+; RV32-LMULMAX4-NEXT:    vslideup.vi v8, v25, 1
+; RV32-LMULMAX4-NEXT:    ret
+;
+; RV64-LMULMAX4-LABEL: buildvec_mask_v128i1:
+; RV64-LMULMAX4:       # %bb.0:
+; RV64-LMULMAX4-NEXT:    lui a0, 841543
+; RV64-LMULMAX4-NEXT:    addiw a0, a0, 511
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 14
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 859
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 17
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 1776
+; RV64-LMULMAX4-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; RV64-LMULMAX4-NEXT:    vmv.s.x v8, a0
+; RV64-LMULMAX4-NEXT:    lui a0, 1048429
+; RV64-LMULMAX4-NEXT:    addiw a0, a0, 1735
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 1023
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX4-NEXT:    addi a0, a0, -1189
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 17
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 1776
+; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX4-NEXT:    ret
+;
+; RV32-LMULMAX8-LABEL: buildvec_mask_v128i1:
+; RV32-LMULMAX8:       # %bb.0:
+; RV32-LMULMAX8-NEXT:    lui a0, 748388
+; RV32-LMULMAX8-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX8-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-LMULMAX8-NEXT:    vmv.s.x v25, a0
+; RV32-LMULMAX8-NEXT:    lui a0, 748384
+; RV32-LMULMAX8-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX8-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX8-NEXT:    vsetivli a0, 2, e32,m1,tu,mu
+; RV32-LMULMAX8-NEXT:    vslideup.vi v0, v25, 1
+; RV32-LMULMAX8-NEXT:    lui a0, 551776
+; RV32-LMULMAX8-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX8-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-LMULMAX8-NEXT:    vmv.s.x v25, a0
+; RV32-LMULMAX8-NEXT:    vsetivli a0, 3, e32,m1,tu,mu
+; RV32-LMULMAX8-NEXT:    vslideup.vi v0, v25, 2
+; RV32-LMULMAX8-NEXT:    lui a0, 945060
+; RV32-LMULMAX8-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX8-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-LMULMAX8-NEXT:    vmv.s.x v25, a0
+; RV32-LMULMAX8-NEXT:    vsetivli a0, 4, e32,m1,tu,mu
+; RV32-LMULMAX8-NEXT:    vslideup.vi v0, v25, 3
+; RV32-LMULMAX8-NEXT:    ret
+;
+; RV64-LMULMAX8-LABEL: buildvec_mask_v128i1:
+; RV64-LMULMAX8:       # %bb.0:
+; RV64-LMULMAX8-NEXT:    lui a0, 841543
+; RV64-LMULMAX8-NEXT:    addiw a0, a0, 511
+; RV64-LMULMAX8-NEXT:    slli a0, a0, 14
+; RV64-LMULMAX8-NEXT:    addi a0, a0, 859
+; RV64-LMULMAX8-NEXT:    slli a0, a0, 17
+; RV64-LMULMAX8-NEXT:    addi a0, a0, 1776
+; RV64-LMULMAX8-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV64-LMULMAX8-NEXT:    vmv.s.x v25, a0
+; RV64-LMULMAX8-NEXT:    lui a0, 1048429
+; RV64-LMULMAX8-NEXT:    addiw a0, a0, 1735
+; RV64-LMULMAX8-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX8-NEXT:    addi a0, a0, 1023
+; RV64-LMULMAX8-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX8-NEXT:    addi a0, a0, -1189
+; RV64-LMULMAX8-NEXT:    slli a0, a0, 17
+; RV64-LMULMAX8-NEXT:    addi a0, a0, 1776
+; RV64-LMULMAX8-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX8-NEXT:    vsetivli a0, 2, e64,m1,tu,mu
+; RV64-LMULMAX8-NEXT:    vslideup.vi v0, v25, 1
+; RV64-LMULMAX8-NEXT:    ret
+  ret <128 x i1> <i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 0, i1 1, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 1, i1 1, i1 1>
+}
+
+define <128 x i1> @buildvec_mask_optsize_v128i1() optsize {
+; RV32-LMULMAX1-LABEL: buildvec_mask_optsize_v128i1:
+; RV32-LMULMAX1:       # %bb.0:
+; RV32-LMULMAX1-NEXT:    addi a0, zero, 1776
+; RV32-LMULMAX1-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; RV32-LMULMAX1-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 11
+; RV32-LMULMAX1-NEXT:    addi a0, a0, 1718
+; RV32-LMULMAX1-NEXT:    vmv.s.x v8, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 8
+; RV32-LMULMAX1-NEXT:    addi a0, a0, 1718
+; RV32-LMULMAX1-NEXT:    vmv.s.x v12, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 4
+; RV32-LMULMAX1-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX1-NEXT:    vmv.s.x v9, a0
+; RV32-LMULMAX1-NEXT:    lui a0, 14
+; RV32-LMULMAX1-NEXT:    addi a0, a0, 1722
+; RV32-LMULMAX1-NEXT:    vmv.s.x v14, a0
+; RV32-LMULMAX1-NEXT:    vmv1r.v v10, v8
+; RV32-LMULMAX1-NEXT:    vmv1r.v v11, v0
+; RV32-LMULMAX1-NEXT:    vmv1r.v v13, v9
+; RV32-LMULMAX1-NEXT:    ret
+;
+; RV64-LMULMAX1-LABEL: buildvec_mask_optsize_v128i1:
+; RV64-LMULMAX1:       # %bb.0:
+; RV64-LMULMAX1-NEXT:    addi a0, zero, 1776
+; RV64-LMULMAX1-NEXT:    vsetivli a1, 1, e16,m1,ta,mu
+; RV64-LMULMAX1-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 11
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, 1718
+; RV64-LMULMAX1-NEXT:    vmv.s.x v8, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 8
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, 1718
+; RV64-LMULMAX1-NEXT:    vmv.s.x v12, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 4
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, -1793
+; RV64-LMULMAX1-NEXT:    vmv.s.x v9, a0
+; RV64-LMULMAX1-NEXT:    lui a0, 14
+; RV64-LMULMAX1-NEXT:    addiw a0, a0, 1722
+; RV64-LMULMAX1-NEXT:    vmv.s.x v14, a0
+; RV64-LMULMAX1-NEXT:    vmv1r.v v10, v8
+; RV64-LMULMAX1-NEXT:    vmv1r.v v11, v0
+; RV64-LMULMAX1-NEXT:    vmv1r.v v13, v9
+; RV64-LMULMAX1-NEXT:    ret
+;
+; RV32-LMULMAX2-LABEL: buildvec_mask_optsize_v128i1:
+; RV32-LMULMAX2:       # %bb.0:
+; RV32-LMULMAX2-NEXT:    lui a0, 748384
+; RV32-LMULMAX2-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX2-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV32-LMULMAX2-NEXT:    vmv.s.x v0, a0
+; RV32-LMULMAX2-NEXT:    lui a0, 748388
+; RV32-LMULMAX2-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX2-NEXT:    vmv.s.x v8, a0
+; RV32-LMULMAX2-NEXT:    lui a0, 551776
+; RV32-LMULMAX2-NEXT:    addi a0, a0, 1776
+; RV32-LMULMAX2-NEXT:    vmv.s.x v9, a0
+; RV32-LMULMAX2-NEXT:    lui a0, 945060
+; RV32-LMULMAX2-NEXT:    addi a0, a0, -1793
+; RV32-LMULMAX2-NEXT:    vmv.s.x v10, a0
+; RV32-LMULMAX2-NEXT:    ret
+;
+; RV64-LMULMAX2-LABEL: buildvec_mask_optsize_v128i1:
+; RV64-LMULMAX2:       # %bb.0:
+; RV64-LMULMAX2-NEXT:    lui a0, 748384
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, 1776
+; RV64-LMULMAX2-NEXT:    vsetivli a1, 1, e32,m1,ta,mu
+; RV64-LMULMAX2-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX2-NEXT:    lui a0, 748388
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, -1793
+; RV64-LMULMAX2-NEXT:    vmv.s.x v8, a0
+; RV64-LMULMAX2-NEXT:    lui a0, 551776
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, 1776
+; RV64-LMULMAX2-NEXT:    vmv.s.x v9, a0
+; RV64-LMULMAX2-NEXT:    lui a0, 945060
+; RV64-LMULMAX2-NEXT:    addiw a0, a0, -1793
+; RV64-LMULMAX2-NEXT:    vmv.s.x v10, a0
+; RV64-LMULMAX2-NEXT:    ret
+;
+; RV32-LMULMAX4-LABEL: buildvec_mask_optsize_v128i1:
+; RV32-LMULMAX4:       # %bb.0:
+; RV32-LMULMAX4-NEXT:    lui a0, %hi(.LCPI10_0)
+; RV32-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI10_0)
+; RV32-LMULMAX4-NEXT:    addi a1, zero, 64
+; RV32-LMULMAX4-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
+; RV32-LMULMAX4-NEXT:    vle1.v v0, (a0)
+; RV32-LMULMAX4-NEXT:    lui a0, %hi(.LCPI10_1)
+; RV32-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI10_1)
+; RV32-LMULMAX4-NEXT:    vle1.v v8, (a0)
+; RV32-LMULMAX4-NEXT:    ret
+;
+; RV64-LMULMAX4-LABEL: buildvec_mask_optsize_v128i1:
+; RV64-LMULMAX4:       # %bb.0:
+; RV64-LMULMAX4-NEXT:    lui a0, 841543
+; RV64-LMULMAX4-NEXT:    addiw a0, a0, 511
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 14
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 859
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 17
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 1776
+; RV64-LMULMAX4-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; RV64-LMULMAX4-NEXT:    vmv.s.x v8, a0
+; RV64-LMULMAX4-NEXT:    lui a0, 1048429
+; RV64-LMULMAX4-NEXT:    addiw a0, a0, 1735
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 1023
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 13
+; RV64-LMULMAX4-NEXT:    addi a0, a0, -1189
+; RV64-LMULMAX4-NEXT:    slli a0, a0, 17
+; RV64-LMULMAX4-NEXT:    addi a0, a0, 1776
+; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX4-NEXT:    ret
+;
+; RV32-LMULMAX8-LABEL: buildvec_mask_optsize_v128i1:
+; RV32-LMULMAX8:       # %bb.0:
+; RV32-LMULMAX8-NEXT:    lui a0, %hi(.LCPI10_0)
+; RV32-LMULMAX8-NEXT:    addi a0, a0, %lo(.LCPI10_0)
+; RV32-LMULMAX8-NEXT:    addi a1, zero, 128
+; RV32-LMULMAX8-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
+; RV32-LMULMAX8-NEXT:    vle1.v v0, (a0)
+; RV32-LMULMAX8-NEXT:    ret
+;
+; RV64-LMULMAX8-LABEL: buildvec_mask_optsize_v128i1:
+; RV64-LMULMAX8:       # %bb.0:
+; RV64-LMULMAX8-NEXT:    lui a0, %hi(.LCPI10_0)
+; RV64-LMULMAX8-NEXT:    addi a0, a0, %lo(.LCPI10_0)
+; RV64-LMULMAX8-NEXT:    addi a1, zero, 128
+; RV64-LMULMAX8-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
+; RV64-LMULMAX8-NEXT:    vle1.v v0, (a0)
+; RV64-LMULMAX8-NEXT:    ret
+  ret <128 x i1> <i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 0, i1 1, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 1, i1 1, i1 1>
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2505f97c4405caba508d934b7135a0a53ab85ca6
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -0,0 +1,2209 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=ilp32d \
+; RUN:     -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=lp64d \
+; RUN:     -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV64
+
+declare <1 x i8> @llvm.masked.gather.v1i8.v1p0i8(<1 x i8*>, i32, <1 x i1>, <1 x i8>)
+
+define <1 x i8> @mgather_v1i8(<1 x i8*> %ptrs, <1 x i1> %m, <1 x i8> %passthru) {
+; RV32-LABEL: mgather_v1i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v1i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e8,mf8,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <1 x i8> @llvm.masked.gather.v1i8.v1p0i8(<1 x i8*> %ptrs, i32 1, <1 x i1> %m, <1 x i8> %passthru)
+  ret <1 x i8> %v
+}
+
+declare <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*>, i32, <2 x i1>, <2 x i8>)
+
+define <2 x i8> @mgather_v2i8(<2 x i8*> %ptrs, <2 x i1> %m, <2 x i8> %passthru) {
+; RV32-LABEL: mgather_v2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf8,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> %ptrs, i32 1, <2 x i1> %m, <2 x i8> %passthru)
+  ret <2 x i8> %v
+}
+
+define <2 x i16> @mgather_v2i8_sextload_v2i16(<2 x i8*> %ptrs, <2 x i1> %m, <2 x i8> %passthru) {
+; RV32-LABEL: mgather_v2i8_sextload_v2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e16,m1,ta,mu
+; RV32-NEXT:    vsext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i8_sextload_v2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf8,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e16,m1,ta,mu
+; RV64-NEXT:    vsext.vf2 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> %ptrs, i32 1, <2 x i1> %m, <2 x i8> %passthru)
+  %ev = sext <2 x i8> %v to <2 x i16>
+  ret <2 x i16> %ev
+}
+
+define <2 x i16> @mgather_v2i8_zextload_v2i16(<2 x i8*> %ptrs, <2 x i1> %m, <2 x i8> %passthru) {
+; RV32-LABEL: mgather_v2i8_zextload_v2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e16,m1,ta,mu
+; RV32-NEXT:    vzext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i8_zextload_v2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf8,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e16,m1,ta,mu
+; RV64-NEXT:    vzext.vf2 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> %ptrs, i32 1, <2 x i1> %m, <2 x i8> %passthru)
+  %ev = zext <2 x i8> %v to <2 x i16>
+  ret <2 x i16> %ev
+}
+
+define <2 x i32> @mgather_v2i8_sextload_v2i32(<2 x i8*> %ptrs, <2 x i1> %m, <2 x i8> %passthru) {
+; RV32-LABEL: mgather_v2i8_sextload_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsext.vf4 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i8_sextload_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf8,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV64-NEXT:    vsext.vf4 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> %ptrs, i32 1, <2 x i1> %m, <2 x i8> %passthru)
+  %ev = sext <2 x i8> %v to <2 x i32>
+  ret <2 x i32> %ev
+}
+
+define <2 x i32> @mgather_v2i8_zextload_v2i32(<2 x i8*> %ptrs, <2 x i1> %m, <2 x i8> %passthru) {
+; RV32-LABEL: mgather_v2i8_zextload_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vzext.vf4 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i8_zextload_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf8,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV64-NEXT:    vzext.vf4 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> %ptrs, i32 1, <2 x i1> %m, <2 x i8> %passthru)
+  %ev = zext <2 x i8> %v to <2 x i32>
+  ret <2 x i32> %ev
+}
+
+define <2 x i64> @mgather_v2i8_sextload_v2i64(<2 x i8*> %ptrs, <2 x i1> %m, <2 x i8> %passthru) {
+; RV32-LABEL: mgather_v2i8_sextload_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV32-NEXT:    vsext.vf8 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i8_sextload_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf8,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV64-NEXT:    vsext.vf8 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> %ptrs, i32 1, <2 x i1> %m, <2 x i8> %passthru)
+  %ev = sext <2 x i8> %v to <2 x i64>
+  ret <2 x i64> %ev
+}
+
+define <2 x i64> @mgather_v2i8_zextload_v2i64(<2 x i8*> %ptrs, <2 x i1> %m, <2 x i8> %passthru) {
+; RV32-LABEL: mgather_v2i8_zextload_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV32-NEXT:    vzext.vf8 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i8_zextload_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf8,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV64-NEXT:    vzext.vf8 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i8> @llvm.masked.gather.v2i8.v2p0i8(<2 x i8*> %ptrs, i32 1, <2 x i1> %m, <2 x i8> %passthru)
+  %ev = zext <2 x i8> %v to <2 x i64>
+  ret <2 x i64> %ev
+}
+
+declare <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>)
+
+define <4 x i8> @mgather_v4i8(<4 x i8*> %ptrs, <4 x i1> %m, <4 x i8> %passthru) {
+; RV32-LABEL: mgather_v4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e8,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> %m, <4 x i8> %passthru)
+  ret <4 x i8> %v
+}
+
+define <4 x i8> @mgather_truemask_v4i8(<4 x i8*> %ptrs, <4 x i8> %passthru) {
+; RV32-LABEL: mgather_truemask_v4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e8,mf4,ta,mu
+; RV32-NEXT:    vloxei32.v v25, (zero), v8
+; RV32-NEXT:    vmv1r.v v8, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_v4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e8,mf4,ta,mu
+; RV64-NEXT:    vloxei64.v v25, (zero), v8
+; RV64-NEXT:    vmv1r.v v8, v25
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  %v = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> %mtrue, <4 x i8> %passthru)
+  ret <4 x i8> %v
+}
+
+define <4 x i8> @mgather_falsemask_v4i8(<4 x i8*> %ptrs, <4 x i8> %passthru) {
+; RV32-LABEL: mgather_falsemask_v4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_v4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> zeroinitializer, <4 x i8> %passthru)
+  ret <4 x i8> %v
+}
+
+declare <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>)
+
+define <8 x i8> @mgather_v8i8(<8 x i8*> %ptrs, <8 x i1> %m, <8 x i8> %passthru) {
+; RV32-LABEL: mgather_v8i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 8, e8,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v8i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 8, e8,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %ptrs, i32 1, <8 x i1> %m, <8 x i8> %passthru)
+  ret <8 x i8> %v
+}
+
+define <8 x i8> @mgather_baseidx_v8i8(i8* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i8> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsetivli a1, 8, e8,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v26, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsetivli a1, 8, e8,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v28, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <8 x i8> %idxs
+  %v = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %ptrs, i32 1, <8 x i1> %m, <8 x i8> %passthru)
+  ret <8 x i8> %v
+}
+
+declare <1 x i16> @llvm.masked.gather.v1i16.v1p0i16(<1 x i16*>, i32, <1 x i1>, <1 x i16>)
+
+define <1 x i16> @mgather_v1i16(<1 x i16*> %ptrs, <1 x i1> %m, <1 x i16> %passthru) {
+; RV32-LABEL: mgather_v1i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v1i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <1 x i16> @llvm.masked.gather.v1i16.v1p0i16(<1 x i16*> %ptrs, i32 2, <1 x i1> %m, <1 x i16> %passthru)
+  ret <1 x i16> %v
+}
+
+declare <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*>, i32, <2 x i1>, <2 x i16>)
+
+define <2 x i16> @mgather_v2i16(<2 x i16*> %ptrs, <2 x i1> %m, <2 x i16> %passthru) {
+; RV32-LABEL: mgather_v2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %ptrs, i32 2, <2 x i1> %m, <2 x i16> %passthru)
+  ret <2 x i16> %v
+}
+
+define <2 x i32> @mgather_v2i16_sextload_v2i32(<2 x i16*> %ptrs, <2 x i1> %m, <2 x i16> %passthru) {
+; RV32-LABEL: mgather_v2i16_sextload_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i16_sextload_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV64-NEXT:    vsext.vf2 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %ptrs, i32 2, <2 x i1> %m, <2 x i16> %passthru)
+  %ev = sext <2 x i16> %v to <2 x i32>
+  ret <2 x i32> %ev
+}
+
+define <2 x i32> @mgather_v2i16_zextload_v2i32(<2 x i16*> %ptrs, <2 x i1> %m, <2 x i16> %passthru) {
+; RV32-LABEL: mgather_v2i16_zextload_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vzext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i16_zextload_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV64-NEXT:    vzext.vf2 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %ptrs, i32 2, <2 x i1> %m, <2 x i16> %passthru)
+  %ev = zext <2 x i16> %v to <2 x i32>
+  ret <2 x i32> %ev
+}
+
+define <2 x i64> @mgather_v2i16_sextload_v2i64(<2 x i16*> %ptrs, <2 x i1> %m, <2 x i16> %passthru) {
+; RV32-LABEL: mgather_v2i16_sextload_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV32-NEXT:    vsext.vf4 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i16_sextload_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV64-NEXT:    vsext.vf4 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %ptrs, i32 2, <2 x i1> %m, <2 x i16> %passthru)
+  %ev = sext <2 x i16> %v to <2 x i64>
+  ret <2 x i64> %ev
+}
+
+define <2 x i64> @mgather_v2i16_zextload_v2i64(<2 x i16*> %ptrs, <2 x i1> %m, <2 x i16> %passthru) {
+; RV32-LABEL: mgather_v2i16_zextload_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV32-NEXT:    vzext.vf4 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i16_zextload_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV64-NEXT:    vzext.vf4 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %ptrs, i32 2, <2 x i1> %m, <2 x i16> %passthru)
+  %ev = zext <2 x i16> %v to <2 x i64>
+  ret <2 x i64> %ev
+}
+
+declare <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>)
+
+define <4 x i16> @mgather_v4i16(<4 x i16*> %ptrs, <4 x i1> %m, <4 x i16> %passthru) {
+; RV32-LABEL: mgather_v4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e16,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> %m, <4 x i16> %passthru)
+  ret <4 x i16> %v
+}
+
+define <4 x i16> @mgather_truemask_v4i16(<4 x i16*> %ptrs, <4 x i16> %passthru) {
+; RV32-LABEL: mgather_truemask_v4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e16,mf2,ta,mu
+; RV32-NEXT:    vloxei32.v v25, (zero), v8
+; RV32-NEXT:    vmv1r.v v8, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_v4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e16,mf2,ta,mu
+; RV64-NEXT:    vloxei64.v v25, (zero), v8
+; RV64-NEXT:    vmv1r.v v8, v25
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  %v = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> %mtrue, <4 x i16> %passthru)
+  ret <4 x i16> %v
+}
+
+define <4 x i16> @mgather_falsemask_v4i16(<4 x i16*> %ptrs, <4 x i16> %passthru) {
+; RV32-LABEL: mgather_falsemask_v4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_v4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> zeroinitializer, <4 x i16> %passthru)
+  ret <4 x i16> %v
+}
+
+declare <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*>, i32, <8 x i1>, <8 x i16>)
+
+define <8 x i16> @mgather_v8i16(<8 x i16*> %ptrs, <8 x i1> %m, <8 x i16> %passthru) {
+; RV32-LABEL: mgather_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> %m, <8 x i16> %passthru)
+  ret <8 x i16> %v
+}
+
+define <8 x i16> @mgather_baseidx_v8i8_v8i16(i16* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i16> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i8_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v26, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i8_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v28, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i8> %idxs
+  %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> %m, <8 x i16> %passthru)
+  ret <8 x i16> %v
+}
+
+define <8 x i16> @mgather_baseidx_sext_v8i8_v8i16(i16* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i16> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i8_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v26, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i8_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v28, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i16>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> %eidxs
+  %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> %m, <8 x i16> %passthru)
+  ret <8 x i16> %v
+}
+
+define <8 x i16> @mgather_baseidx_zext_v8i8_v8i16(i16* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i16> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i8_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v26, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i8_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v28, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i16>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> %eidxs
+  %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> %m, <8 x i16> %passthru)
+  ret <8 x i16> %v
+}
+
+define <8 x i16> @mgather_baseidx_v8i16(i16* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i16> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v26, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v28, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> %idxs
+  %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %ptrs, i32 2, <8 x i1> %m, <8 x i16> %passthru)
+  ret <8 x i16> %v
+}
+
+declare <1 x i32> @llvm.masked.gather.v1i32.v1p0i32(<1 x i32*>, i32, <1 x i1>, <1 x i32>)
+
+define <1 x i32> @mgather_v1i32(<1 x i32*> %ptrs, <1 x i1> %m, <1 x i32> %passthru) {
+; RV32-LABEL: mgather_v1i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v1i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e32,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <1 x i32> @llvm.masked.gather.v1i32.v1p0i32(<1 x i32*> %ptrs, i32 4, <1 x i1> %m, <1 x i32> %passthru)
+  ret <1 x i32> %v
+}
+
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
+
+define <2 x i32> @mgather_v2i32(<2 x i32*> %ptrs, <2 x i1> %m, <2 x i32> %passthru) {
+; RV32-LABEL: mgather_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %ptrs, i32 4, <2 x i1> %m, <2 x i32> %passthru)
+  ret <2 x i32> %v
+}
+
+define <2 x i64> @mgather_v2i32_sextload_v2i64(<2 x i32*> %ptrs, <2 x i1> %m, <2 x i32> %passthru) {
+; RV32-LABEL: mgather_v2i32_sextload_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV32-NEXT:    vsext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i32_sextload_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV64-NEXT:    vsext.vf2 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %ptrs, i32 4, <2 x i1> %m, <2 x i32> %passthru)
+  %ev = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %ev
+}
+
+define <2 x i64> @mgather_v2i32_zextload_v2i64(<2 x i32*> %ptrs, <2 x i1> %m, <2 x i32> %passthru) {
+; RV32-LABEL: mgather_v2i32_zextload_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV32-NEXT:    vzext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i32_zextload_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; RV64-NEXT:    vzext.vf2 v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %ptrs, i32 4, <2 x i1> %m, <2 x i32> %passthru)
+  %ev = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %ev
+}
+
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+
+define <4 x i32> @mgather_v4i32(<4 x i32*> %ptrs, <4 x i1> %m, <4 x i32> %passthru) {
+; RV32-LABEL: mgather_v4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e32,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> %m, <4 x i32> %passthru)
+  ret <4 x i32> %v
+}
+
+define <4 x i32> @mgather_truemask_v4i32(<4 x i32*> %ptrs, <4 x i32> %passthru) {
+; RV32-LABEL: mgather_truemask_v4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e32,m1,ta,mu
+; RV32-NEXT:    vloxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_v4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e32,m1,ta,mu
+; RV64-NEXT:    vloxei64.v v25, (zero), v8
+; RV64-NEXT:    vmv1r.v v8, v25
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  %v = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> %mtrue, <4 x i32> %passthru)
+  ret <4 x i32> %v
+}
+
+define <4 x i32> @mgather_falsemask_v4i32(<4 x i32*> %ptrs, <4 x i32> %passthru) {
+; RV32-LABEL: mgather_falsemask_v4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_v4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> zeroinitializer, <4 x i32> %passthru)
+  ret <4 x i32> %v
+}
+
+declare <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*>, i32, <8 x i1>, <8 x i32>)
+
+define <8 x i32> @mgather_v8i32(<8 x i32*> %ptrs, <8 x i1> %m, <8 x i32> %passthru) {
+; RV32-LABEL: mgather_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
+  ret <8 x i32> %v
+}
+
+define <8 x i32> @mgather_baseidx_v8i8_v8i32(i32* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i8_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i8_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i8> %idxs
+  %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
+  ret <8 x i32> %v
+}
+
+define <8 x i32> @mgather_baseidx_sext_v8i8_v8i32(i32* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i8_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i8_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %eidxs
+  %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
+  ret <8 x i32> %v
+}
+
+define <8 x i32> @mgather_baseidx_zext_v8i8_v8i32(i32* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i8_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i8_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %eidxs
+  %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
+  ret <8 x i32> %v
+}
+
+define <8 x i32> @mgather_baseidx_v8i16_v8i32(i32* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i16_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i16_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i16> %idxs
+  %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
+  ret <8 x i32> %v
+}
+
+define <8 x i32> @mgather_baseidx_sext_v8i16_v8i32(i32* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i16_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i16_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i16> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %eidxs
+  %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
+  ret <8 x i32> %v
+}
+
+define <8 x i32> @mgather_baseidx_zext_v8i16_v8i32(i32* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i16_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i16_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i16> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %eidxs
+  %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
+  ret <8 x i32> %v
+}
+
+define <8 x i32> @mgather_baseidx_v8i32(i32* %base, <8 x i32> %idxs, <8 x i1> %m, <8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsll.vi v26, v8, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %idxs
+  %v = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %ptrs, i32 4, <8 x i1> %m, <8 x i32> %passthru)
+  ret <8 x i32> %v
+}
+
+declare <1 x i64> @llvm.masked.gather.v1i64.v1p0i64(<1 x i64*>, i32, <1 x i1>, <1 x i64>)
+
+define <1 x i64> @mgather_v1i64(<1 x i64*> %ptrs, <1 x i1> %m, <1 x i64> %passthru) {
+; RV32-LABEL: mgather_v1i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e64,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v1i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e64,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <1 x i64> @llvm.masked.gather.v1i64.v1p0i64(<1 x i64*> %ptrs, i32 8, <1 x i1> %m, <1 x i64> %passthru)
+  ret <1 x i64> %v
+}
+
+declare <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*>, i32, <2 x i1>, <2 x i64>)
+
+define <2 x i64> @mgather_v2i64(<2 x i64*> %ptrs, <2 x i1> %m, <2 x i64> %passthru) {
+; RV32-LABEL: mgather_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e64,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e64,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> %ptrs, i32 8, <2 x i1> %m, <2 x i64> %passthru)
+  ret <2 x i64> %v
+}
+
+declare <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*>, i32, <4 x i1>, <4 x i64>)
+
+define <4 x i64> @mgather_v4i64(<4 x i64*> %ptrs, <4 x i1> %m, <4 x i64> %passthru) {
+; RV32-LABEL: mgather_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e64,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e64,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*> %ptrs, i32 8, <4 x i1> %m, <4 x i64> %passthru)
+  ret <4 x i64> %v
+}
+
+define <4 x i64> @mgather_truemask_v4i64(<4 x i64*> %ptrs, <4 x i64> %passthru) {
+; RV32-LABEL: mgather_truemask_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e64,m2,ta,mu
+; RV32-NEXT:    vloxei32.v v26, (zero), v8
+; RV32-NEXT:    vmv2r.v v8, v26
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e64,m2,ta,mu
+; RV64-NEXT:    vloxei64.v v8, (zero), v8
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  %v = call <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*> %ptrs, i32 8, <4 x i1> %mtrue, <4 x i64> %passthru)
+  ret <4 x i64> %v
+}
+
+define <4 x i64> @mgather_falsemask_v4i64(<4 x i64*> %ptrs, <4 x i64> %passthru) {
+; RV32-LABEL: mgather_falsemask_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x i64> @llvm.masked.gather.v4i64.v4p0i64(<4 x i64*> %ptrs, i32 8, <4 x i1> zeroinitializer, <4 x i64> %passthru)
+  ret <4 x i64> %v
+}
+
+declare <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*>, i32, <8 x i1>, <8 x i64>)
+
+define <8 x i64> @mgather_v8i64(<8 x i64*> %ptrs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (zero), v8, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_v8i8_v8i64(i64* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i8_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v26, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i8_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i8> %idxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_sext_v8i8_v8i64(i64* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i8_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf8 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i8_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_zext_v8i8_v8i64(i64* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i8_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf8 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i8_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_v8i16_v8i64(i64* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i16_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v26, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i16_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i16> %idxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(i64* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i16_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i16_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i16> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_zext_v8i16_v8i64(i64* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i16_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i16_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i16> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_v8i32_v8i64(i64* %base, <8 x i32> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i32_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsll.vi v26, v8, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v26, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i32_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i32> %idxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_sext_v8i32_v8i64(i64* %base, <8 x i32> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i32_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i32_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i32> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_zext_v8i32_v8i64(i64* %base, <8 x i32> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i32_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i32_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf2 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i32> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+define <8 x i64> @mgather_baseidx_v8i64(i64* %base, <8 x i64> %idxs, <8 x i1> %m, <8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v8, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsll.vi v28, v8, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %idxs
+  %v = call <8 x i64> @llvm.masked.gather.v8i64.v8p0i64(<8 x i64*> %ptrs, i32 8, <8 x i1> %m, <8 x i64> %passthru)
+  ret <8 x i64> %v
+}
+
+declare <1 x half> @llvm.masked.gather.v1f16.v1p0f16(<1 x half*>, i32, <1 x i1>, <1 x half>)
+
+define <1 x half> @mgather_v1f16(<1 x half*> %ptrs, <1 x i1> %m, <1 x half> %passthru) {
+; RV32-LABEL: mgather_v1f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v1f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <1 x half> @llvm.masked.gather.v1f16.v1p0f16(<1 x half*> %ptrs, i32 2, <1 x i1> %m, <1 x half> %passthru)
+  ret <1 x half> %v
+}
+
+declare <2 x half> @llvm.masked.gather.v2f16.v2p0f16(<2 x half*>, i32, <2 x i1>, <2 x half>)
+
+define <2 x half> @mgather_v2f16(<2 x half*> %ptrs, <2 x i1> %m, <2 x half> %passthru) {
+; RV32-LABEL: mgather_v2f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x half> @llvm.masked.gather.v2f16.v2p0f16(<2 x half*> %ptrs, i32 2, <2 x i1> %m, <2 x half> %passthru)
+  ret <2 x half> %v
+}
+
+declare <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*>, i32, <4 x i1>, <4 x half>)
+
+define <4 x half> @mgather_v4f16(<4 x half*> %ptrs, <4 x i1> %m, <4 x half> %passthru) {
+; RV32-LABEL: mgather_v4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e16,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*> %ptrs, i32 2, <4 x i1> %m, <4 x half> %passthru)
+  ret <4 x half> %v
+}
+
+define <4 x half> @mgather_truemask_v4f16(<4 x half*> %ptrs, <4 x half> %passthru) {
+; RV32-LABEL: mgather_truemask_v4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e16,mf2,ta,mu
+; RV32-NEXT:    vloxei32.v v25, (zero), v8
+; RV32-NEXT:    vmv1r.v v8, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_v4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e16,mf2,ta,mu
+; RV64-NEXT:    vloxei64.v v25, (zero), v8
+; RV64-NEXT:    vmv1r.v v8, v25
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  %v = call <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*> %ptrs, i32 2, <4 x i1> %mtrue, <4 x half> %passthru)
+  ret <4 x half> %v
+}
+
+define <4 x half> @mgather_falsemask_v4f16(<4 x half*> %ptrs, <4 x half> %passthru) {
+; RV32-LABEL: mgather_falsemask_v4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_v4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*> %ptrs, i32 2, <4 x i1> zeroinitializer, <4 x half> %passthru)
+  ret <4 x half> %v
+}
+
+declare <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*>, i32, <8 x i1>, <8 x half>)
+
+define <8 x half> @mgather_v8f16(<8 x half*> %ptrs, <8 x i1> %m, <8 x half> %passthru) {
+; RV32-LABEL: mgather_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> %m, <8 x half> %passthru)
+  ret <8 x half> %v
+}
+
+define <8 x half> @mgather_baseidx_v8i8_v8f16(half* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x half> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i8_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v26, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i8_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v28, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds half, half* %base, <8 x i8> %idxs
+  %v = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> %m, <8 x half> %passthru)
+  ret <8 x half> %v
+}
+
+define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(half* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x half> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i8_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v26, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i8_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v28, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i16>
+  %ptrs = getelementptr inbounds half, half* %base, <8 x i16> %eidxs
+  %v = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> %m, <8 x half> %passthru)
+  ret <8 x half> %v
+}
+
+define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(half* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x half> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i8_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v26, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i8_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v28, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i16>
+  %ptrs = getelementptr inbounds half, half* %base, <8 x i16> %eidxs
+  %v = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> %m, <8 x half> %passthru)
+  ret <8 x half> %v
+}
+
+define <8 x half> @mgather_baseidx_v8f16(half* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x half> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v26, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 8, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v28, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds half, half* %base, <8 x i16> %idxs
+  %v = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %ptrs, i32 2, <8 x i1> %m, <8 x half> %passthru)
+  ret <8 x half> %v
+}
+
+declare <1 x float> @llvm.masked.gather.v1f32.v1p0f32(<1 x float*>, i32, <1 x i1>, <1 x float>)
+
+define <1 x float> @mgather_v1f32(<1 x float*> %ptrs, <1 x i1> %m, <1 x float> %passthru) {
+; RV32-LABEL: mgather_v1f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v1f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e32,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <1 x float> @llvm.masked.gather.v1f32.v1p0f32(<1 x float*> %ptrs, i32 4, <1 x i1> %m, <1 x float> %passthru)
+  ret <1 x float> %v
+}
+
+declare <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*>, i32, <2 x i1>, <2 x float>)
+
+define <2 x float> @mgather_v2f32(<2 x float*> %ptrs, <2 x i1> %m, <2 x float> %passthru) {
+; RV32-LABEL: mgather_v2f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> %ptrs, i32 4, <2 x i1> %m, <2 x float> %passthru)
+  ret <2 x float> %v
+}
+
+declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1>, <4 x float>)
+
+define <4 x float> @mgather_v4f32(<4 x float*> %ptrs, <4 x i1> %m, <4 x float> %passthru) {
+; RV32-LABEL: mgather_v4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e32,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> %m, <4 x float> %passthru)
+  ret <4 x float> %v
+}
+
+define <4 x float> @mgather_truemask_v4f32(<4 x float*> %ptrs, <4 x float> %passthru) {
+; RV32-LABEL: mgather_truemask_v4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e32,m1,ta,mu
+; RV32-NEXT:    vloxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_v4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e32,m1,ta,mu
+; RV64-NEXT:    vloxei64.v v25, (zero), v8
+; RV64-NEXT:    vmv1r.v v8, v25
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  %v = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> %mtrue, <4 x float> %passthru)
+  ret <4 x float> %v
+}
+
+define <4 x float> @mgather_falsemask_v4f32(<4 x float*> %ptrs, <4 x float> %passthru) {
+; RV32-LABEL: mgather_falsemask_v4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_v4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> zeroinitializer, <4 x float> %passthru)
+  ret <4 x float> %v
+}
+
+declare <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*>, i32, <8 x i1>, <8 x float>)
+
+define <8 x float> @mgather_v8f32(<8 x float*> %ptrs, <8 x i1> %m, <8 x float> %passthru) {
+; RV32-LABEL: mgather_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
+  ret <8 x float> %v
+}
+
+define <8 x float> @mgather_baseidx_v8i8_v8f32(float* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i8_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i8_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i8> %idxs
+  %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
+  ret <8 x float> %v
+}
+
+define <8 x float> @mgather_baseidx_sext_v8i8_v8f32(float* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i8_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i8_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %eidxs
+  %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
+  ret <8 x float> %v
+}
+
+define <8 x float> @mgather_baseidx_zext_v8i8_v8f32(float* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i8_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i8_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %eidxs
+  %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
+  ret <8 x float> %v
+}
+
+define <8 x float> @mgather_baseidx_v8i16_v8f32(float* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i16_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i16_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i16> %idxs
+  %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
+  ret <8 x float> %v
+}
+
+define <8 x float> @mgather_baseidx_sext_v8i16_v8f32(float* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i16_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i16_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i16> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %eidxs
+  %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
+  ret <8 x float> %v
+}
+
+define <8 x float> @mgather_baseidx_zext_v8i16_v8f32(float* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i16_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i16_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i16> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %eidxs
+  %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
+  ret <8 x float> %v
+}
+
+define <8 x float> @mgather_baseidx_v8f32(float* %base, <8 x i32> %idxs, <8 x i1> %m, <8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsll.vi v26, v8, 2
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v26, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 8, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v28, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %idxs
+  %v = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %ptrs, i32 4, <8 x i1> %m, <8 x float> %passthru)
+  ret <8 x float> %v
+}
+
+declare <1 x double> @llvm.masked.gather.v1f64.v1p0f64(<1 x double*>, i32, <1 x i1>, <1 x double>)
+
+define <1 x double> @mgather_v1f64(<1 x double*> %ptrs, <1 x i1> %m, <1 x double> %passthru) {
+; RV32-LABEL: mgather_v1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e64,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e64,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <1 x double> @llvm.masked.gather.v1f64.v1p0f64(<1 x double*> %ptrs, i32 8, <1 x i1> %m, <1 x double> %passthru)
+  ret <1 x double> %v
+}
+
+declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*>, i32, <2 x i1>, <2 x double>)
+
+define <2 x double> @mgather_v2f64(<2 x double*> %ptrs, <2 x i1> %m, <2 x double> %passthru) {
+; RV32-LABEL: mgather_v2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e64,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e64,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 8, <2 x i1> %m, <2 x double> %passthru)
+  ret <2 x double> %v
+}
+
+declare <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*>, i32, <4 x i1>, <4 x double>)
+
+define <4 x double> @mgather_v4f64(<4 x double*> %ptrs, <4 x i1> %m, <4 x double> %passthru) {
+; RV32-LABEL: mgather_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e64,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e64,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> %ptrs, i32 8, <4 x i1> %m, <4 x double> %passthru)
+  ret <4 x double> %v
+}
+
+define <4 x double> @mgather_truemask_v4f64(<4 x double*> %ptrs, <4 x double> %passthru) {
+; RV32-LABEL: mgather_truemask_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e64,m2,ta,mu
+; RV32-NEXT:    vloxei32.v v26, (zero), v8
+; RV32-NEXT:    vmv2r.v v8, v26
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_v4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e64,m2,ta,mu
+; RV64-NEXT:    vloxei64.v v8, (zero), v8
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  %v = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> %ptrs, i32 8, <4 x i1> %mtrue, <4 x double> %passthru)
+  ret <4 x double> %v
+}
+
+define <4 x double> @mgather_falsemask_v4f64(<4 x double*> %ptrs, <4 x double> %passthru) {
+; RV32-LABEL: mgather_falsemask_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_v4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> %ptrs, i32 8, <4 x i1> zeroinitializer, <4 x double> %passthru)
+  ret <4 x double> %v
+}
+
+declare <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*>, i32, <8 x i1>, <8 x double>)
+
+define <8 x double> @mgather_v8f64(<8 x double*> %ptrs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (zero), v8, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_v8i8_v8f64(double* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i8_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v26, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i8_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i8> %idxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_sext_v8i8_v8f64(double* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i8_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf8 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i8_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_zext_v8i8_v8f64(double* %base, <8 x i8> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i8_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf8 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i8_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_v8i16_v8f64(double* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i16_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v8
+; RV32-NEXT:    vsll.vi v26, v26, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v26, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i16_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i16> %idxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_sext_v8i16_v8f64(double* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i16_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i16_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i16> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_zext_v8i16_v8f64(double* %base, <8 x i16> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i16_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i16_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf4 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i16> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_v8i32_v8f64(double* %base, <8 x i32> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8i32_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsll.vi v26, v8, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v26, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8i32_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i32> %idxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_sext_v8i32_v8f64(double* %base, <8 x i32> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_v8i32_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_v8i32_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i32> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_zext_v8i32_v8f64(double* %base, <8 x i32> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_v8i32_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_v8i32_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf2 v28, v8
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i32> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+define <8 x double> @mgather_baseidx_v8f64(double* %base, <8 x i64> %idxs, <8 x i1> %m, <8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v8, 3
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV32-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsll.vi v28, v8, 3
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v28, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %idxs
+  %v = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> %ptrs, i32 8, <8 x i1> %m, <8 x double> %passthru)
+  ret <8 x double> %v
+}
+
+declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32, <16 x i1>, <16 x i8>)
+
+define <16 x i8> @mgather_baseidx_v16i8(i8* %base, <16 x i8> %idxs, <16 x i1> %m, <16 x i8> %passthru) {
+; RV32-LABEL: mgather_baseidx_v16i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 16, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsetivli a1, 16, e8,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v28, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v16i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 16, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v16, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i8> %idxs
+  %v = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %ptrs, i32 2, <16 x i1> %m, <16 x i8> %passthru)
+  ret <16 x i8> %v
+}
+
+declare <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*>, i32, <32 x i1>, <32 x i8>)
+
+define <32 x i8> @mgather_baseidx_v32i8(i8* %base, <32 x i8> %idxs, <32 x i1> %m, <32 x i8> %passthru) {
+; RV32-LABEL: mgather_baseidx_v32i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi a1, zero, 32
+; RV32-NEXT:    vsetvli a2, a1, e32,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v16, v8
+; RV32-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v16, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_v32i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv1r.v v25, v0
+; RV64-NEXT:    vsetivli a1, 16, e8,m2,ta,mu
+; RV64-NEXT:    vslidedown.vi v26, v8, 16
+; RV64-NEXT:    vsetivli a1, 16, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v26
+; RV64-NEXT:    vsetivli a1, 16, e8,m2,ta,mu
+; RV64-NEXT:    vslidedown.vi v26, v10, 16
+; RV64-NEXT:    vsetivli a1, 2, e8,m1,ta,mu
+; RV64-NEXT:    vslidedown.vi v0, v0, 2
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v26, (a0), v16, v0.t
+; RV64-NEXT:    vsetivli a1, 16, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsetivli a1, 16, e8,m1,tu,mu
+; RV64-NEXT:    vmv1r.v v0, v25
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    addi a0, zero, 32
+; RV64-NEXT:    vsetvli a1, a0, e8,m2,ta,mu
+; RV64-NEXT:    vmv.v.i v8, 0
+; RV64-NEXT:    vsetivli a1, 16, e8,m2,tu,mu
+; RV64-NEXT:    vslideup.vi v8, v10, 0
+; RV64-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
+; RV64-NEXT:    vslideup.vi v8, v26, 16
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <32 x i8> %idxs
+  %v = call <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*> %ptrs, i32 2, <32 x i1> %m, <32 x i8> %passthru)
+  ret <32 x i8> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e8add2a9475104b47fd2f143841243299377772e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-fp.ll
@@ -0,0 +1,478 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v,+f,+d,+experimental-zfh -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v,+f,+d,+experimental-zfh -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+
+define void @masked_load_v1f16(<1 x half>* %a, <1 x half>* %m_ptr, <1 x half>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vle16.v v25, (a0), v0.t
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <1 x half>, <1 x half>* %m_ptr
+  %mask = fcmp oeq <1 x half> %m, zeroinitializer
+  %load = call <1 x half> @llvm.masked.load.v1f16(<1 x half>* %a, i32 8, <1 x i1> %mask, <1 x half> undef)
+  store <1 x half> %load, <1 x half>* %res_ptr
+  ret void
+}
+declare <1 x half> @llvm.masked.load.v1f16(<1 x half>*, i32, <1 x i1>, <1 x half>)
+
+define void @masked_load_v1f32(<1 x float>* %a, <1 x float>* %m_ptr, <1 x float>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a1)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vle32.v v25, (a0), v0.t
+; CHECK-NEXT:    vse32.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <1 x float>, <1 x float>* %m_ptr
+  %mask = fcmp oeq <1 x float> %m, zeroinitializer
+  %load = call <1 x float> @llvm.masked.load.v1f32(<1 x float>* %a, i32 8, <1 x i1> %mask, <1 x float> undef)
+  store <1 x float> %load, <1 x float>* %res_ptr
+  ret void
+}
+declare <1 x float> @llvm.masked.load.v1f32(<1 x float>*, i32, <1 x i1>, <1 x float>)
+
+define void @masked_load_v1f64(<1 x double>* %a, <1 x double>* %m_ptr, <1 x double>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 1, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a1)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v25, ft0
+; RV32-NEXT:    vle64.v v25, (a0), v0.t
+; RV32-NEXT:    vse64.v v25, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 1, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a1)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v25, ft0
+; RV64-NEXT:    vle64.v v25, (a0), v0.t
+; RV64-NEXT:    vse64.v v25, (a2)
+; RV64-NEXT:    ret
+  %m = load <1 x double>, <1 x double>* %m_ptr
+  %mask = fcmp oeq <1 x double> %m, zeroinitializer
+  %load = call <1 x double> @llvm.masked.load.v1f64(<1 x double>* %a, i32 8, <1 x i1> %mask, <1 x double> undef)
+  store <1 x double> %load, <1 x double>* %res_ptr
+  ret void
+}
+declare <1 x double> @llvm.masked.load.v1f64(<1 x double>*, i32, <1 x i1>, <1 x double>)
+
+define void @masked_load_v2f16(<2 x half>* %a, <2 x half>* %m_ptr, <2 x half>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vle16.v v25, (a0), v0.t
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <2 x half>, <2 x half>* %m_ptr
+  %mask = fcmp oeq <2 x half> %m, zeroinitializer
+  %load = call <2 x half> @llvm.masked.load.v2f16(<2 x half>* %a, i32 8, <2 x i1> %mask, <2 x half> undef)
+  store <2 x half> %load, <2 x half>* %res_ptr
+  ret void
+}
+declare <2 x half> @llvm.masked.load.v2f16(<2 x half>*, i32, <2 x i1>, <2 x half>)
+
+define void @masked_load_v2f32(<2 x float>* %a, <2 x float>* %m_ptr, <2 x float>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a1)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vle32.v v25, (a0), v0.t
+; CHECK-NEXT:    vse32.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <2 x float>, <2 x float>* %m_ptr
+  %mask = fcmp oeq <2 x float> %m, zeroinitializer
+  %load = call <2 x float> @llvm.masked.load.v2f32(<2 x float>* %a, i32 8, <2 x i1> %mask, <2 x float> undef)
+  store <2 x float> %load, <2 x float>* %res_ptr
+  ret void
+}
+declare <2 x float> @llvm.masked.load.v2f32(<2 x float>*, i32, <2 x i1>, <2 x float>)
+
+define void @masked_load_v2f64(<2 x double>* %a, <2 x double>* %m_ptr, <2 x double>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a1)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v25, ft0
+; RV32-NEXT:    vle64.v v25, (a0), v0.t
+; RV32-NEXT:    vse64.v v25, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a1)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v25, ft0
+; RV64-NEXT:    vle64.v v25, (a0), v0.t
+; RV64-NEXT:    vse64.v v25, (a2)
+; RV64-NEXT:    ret
+  %m = load <2 x double>, <2 x double>* %m_ptr
+  %mask = fcmp oeq <2 x double> %m, zeroinitializer
+  %load = call <2 x double> @llvm.masked.load.v2f64(<2 x double>* %a, i32 8, <2 x i1> %mask, <2 x double> undef)
+  store <2 x double> %load, <2 x double>* %res_ptr
+  ret void
+}
+declare <2 x double> @llvm.masked.load.v2f64(<2 x double>*, i32, <2 x i1>, <2 x double>)
+
+define void @masked_load_v4f16(<4 x half>* %a, <4 x half>* %m_ptr, <4 x half>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vle16.v v25, (a0), v0.t
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <4 x half>, <4 x half>* %m_ptr
+  %mask = fcmp oeq <4 x half> %m, zeroinitializer
+  %load = call <4 x half> @llvm.masked.load.v4f16(<4 x half>* %a, i32 8, <4 x i1> %mask, <4 x half> undef)
+  store <4 x half> %load, <4 x half>* %res_ptr
+  ret void
+}
+declare <4 x half> @llvm.masked.load.v4f16(<4 x half>*, i32, <4 x i1>, <4 x half>)
+
+define void @masked_load_v4f32(<4 x float>* %a, <4 x float>* %m_ptr, <4 x float>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a1)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vle32.v v25, (a0), v0.t
+; CHECK-NEXT:    vse32.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <4 x float>, <4 x float>* %m_ptr
+  %mask = fcmp oeq <4 x float> %m, zeroinitializer
+  %load = call <4 x float> @llvm.masked.load.v4f32(<4 x float>* %a, i32 8, <4 x i1> %mask, <4 x float> undef)
+  store <4 x float> %load, <4 x float>* %res_ptr
+  ret void
+}
+declare <4 x float> @llvm.masked.load.v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>)
+
+define void @masked_load_v4f64(<4 x double>* %a, <4 x double>* %m_ptr, <4 x double>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
+; RV32-NEXT:    vle64.v v26, (a1)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v26, ft0
+; RV32-NEXT:    vle64.v v26, (a0), v0.t
+; RV32-NEXT:    vse64.v v26, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
+; RV64-NEXT:    vle64.v v26, (a1)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v26, ft0
+; RV64-NEXT:    vle64.v v26, (a0), v0.t
+; RV64-NEXT:    vse64.v v26, (a2)
+; RV64-NEXT:    ret
+  %m = load <4 x double>, <4 x double>* %m_ptr
+  %mask = fcmp oeq <4 x double> %m, zeroinitializer
+  %load = call <4 x double> @llvm.masked.load.v4f64(<4 x double>* %a, i32 8, <4 x i1> %mask, <4 x double> undef)
+  store <4 x double> %load, <4 x double>* %res_ptr
+  ret void
+}
+declare <4 x double> @llvm.masked.load.v4f64(<4 x double>*, i32, <4 x i1>, <4 x double>)
+
+define void @masked_load_v8f16(<8 x half>* %a, <8 x half>* %m_ptr, <8 x half>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vle16.v v25, (a0), v0.t
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <8 x half>, <8 x half>* %m_ptr
+  %mask = fcmp oeq <8 x half> %m, zeroinitializer
+  %load = call <8 x half> @llvm.masked.load.v8f16(<8 x half>* %a, i32 8, <8 x i1> %mask, <8 x half> undef)
+  store <8 x half> %load, <8 x half>* %res_ptr
+  ret void
+}
+declare <8 x half> @llvm.masked.load.v8f16(<8 x half>*, i32, <8 x i1>, <8 x half>)
+
+define void @masked_load_v8f32(<8 x float>* %a, <8 x float>* %m_ptr, <8 x float>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e32,m2,ta,mu
+; CHECK-NEXT:    vle32.v v26, (a1)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v26, ft0
+; CHECK-NEXT:    vle32.v v26, (a0), v0.t
+; CHECK-NEXT:    vse32.v v26, (a2)
+; CHECK-NEXT:    ret
+  %m = load <8 x float>, <8 x float>* %m_ptr
+  %mask = fcmp oeq <8 x float> %m, zeroinitializer
+  %load = call <8 x float> @llvm.masked.load.v8f32(<8 x float>* %a, i32 8, <8 x i1> %mask, <8 x float> undef)
+  store <8 x float> %load, <8 x float>* %res_ptr
+  ret void
+}
+declare <8 x float> @llvm.masked.load.v8f32(<8 x float>*, i32, <8 x i1>, <8 x float>)
+
+define void @masked_load_v8f64(<8 x double>* %a, <8 x double>* %m_ptr, <8 x double>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 8, e64,m4,ta,mu
+; RV32-NEXT:    vle64.v v28, (a1)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v28, ft0
+; RV32-NEXT:    vle64.v v28, (a0), v0.t
+; RV32-NEXT:    vse64.v v28, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 8, e64,m4,ta,mu
+; RV64-NEXT:    vle64.v v28, (a1)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v28, ft0
+; RV64-NEXT:    vle64.v v28, (a0), v0.t
+; RV64-NEXT:    vse64.v v28, (a2)
+; RV64-NEXT:    ret
+  %m = load <8 x double>, <8 x double>* %m_ptr
+  %mask = fcmp oeq <8 x double> %m, zeroinitializer
+  %load = call <8 x double> @llvm.masked.load.v8f64(<8 x double>* %a, i32 8, <8 x i1> %mask, <8 x double> undef)
+  store <8 x double> %load, <8 x double>* %res_ptr
+  ret void
+}
+declare <8 x double> @llvm.masked.load.v8f64(<8 x double>*, i32, <8 x i1>, <8 x double>)
+
+define void @masked_load_v16f16(<16 x half>* %a, <16 x half>* %m_ptr, <16 x half>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e16,m2,ta,mu
+; CHECK-NEXT:    vle16.v v26, (a1)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v26, ft0
+; CHECK-NEXT:    vle16.v v26, (a0), v0.t
+; CHECK-NEXT:    vse16.v v26, (a2)
+; CHECK-NEXT:    ret
+  %m = load <16 x half>, <16 x half>* %m_ptr
+  %mask = fcmp oeq <16 x half> %m, zeroinitializer
+  %load = call <16 x half> @llvm.masked.load.v16f16(<16 x half>* %a, i32 8, <16 x i1> %mask, <16 x half> undef)
+  store <16 x half> %load, <16 x half>* %res_ptr
+  ret void
+}
+declare <16 x half> @llvm.masked.load.v16f16(<16 x half>*, i32, <16 x i1>, <16 x half>)
+
+define void @masked_load_v16f32(<16 x float>* %a, <16 x float>* %m_ptr, <16 x float>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e32,m4,ta,mu
+; CHECK-NEXT:    vle32.v v28, (a1)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v28, ft0
+; CHECK-NEXT:    vle32.v v28, (a0), v0.t
+; CHECK-NEXT:    vse32.v v28, (a2)
+; CHECK-NEXT:    ret
+  %m = load <16 x float>, <16 x float>* %m_ptr
+  %mask = fcmp oeq <16 x float> %m, zeroinitializer
+  %load = call <16 x float> @llvm.masked.load.v16f32(<16 x float>* %a, i32 8, <16 x i1> %mask, <16 x float> undef)
+  store <16 x float> %load, <16 x float>* %res_ptr
+  ret void
+}
+declare <16 x float> @llvm.masked.load.v16f32(<16 x float>*, i32, <16 x i1>, <16 x float>)
+
+define void @masked_load_v16f64(<16 x double>* %a, <16 x double>* %m_ptr, <16 x double>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v16f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV32-NEXT:    vle64.v v8, (a1)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v8, ft0
+; RV32-NEXT:    vle64.v v8, (a0), v0.t
+; RV32-NEXT:    vse64.v v8, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v16f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV64-NEXT:    vle64.v v8, (a1)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v8, ft0
+; RV64-NEXT:    vle64.v v8, (a0), v0.t
+; RV64-NEXT:    vse64.v v8, (a2)
+; RV64-NEXT:    ret
+  %m = load <16 x double>, <16 x double>* %m_ptr
+  %mask = fcmp oeq <16 x double> %m, zeroinitializer
+  %load = call <16 x double> @llvm.masked.load.v16f64(<16 x double>* %a, i32 8, <16 x i1> %mask, <16 x double> undef)
+  store <16 x double> %load, <16 x double>* %res_ptr
+  ret void
+}
+declare <16 x double> @llvm.masked.load.v16f64(<16 x double>*, i32, <16 x i1>, <16 x double>)
+
+define void @masked_load_v32f16(<32 x half>* %a, <32 x half>* %m_ptr, <32 x half>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e16,m4,ta,mu
+; CHECK-NEXT:    vle16.v v28, (a1)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v28, ft0
+; CHECK-NEXT:    vle16.v v28, (a0), v0.t
+; CHECK-NEXT:    vse16.v v28, (a2)
+; CHECK-NEXT:    ret
+  %m = load <32 x half>, <32 x half>* %m_ptr
+  %mask = fcmp oeq <32 x half> %m, zeroinitializer
+  %load = call <32 x half> @llvm.masked.load.v32f16(<32 x half>* %a, i32 8, <32 x i1> %mask, <32 x half> undef)
+  store <32 x half> %load, <32 x half>* %res_ptr
+  ret void
+}
+declare <32 x half> @llvm.masked.load.v32f16(<32 x half>*, i32, <32 x i1>, <32 x half>)
+
+define void @masked_load_v32f32(<32 x float>* %a, <32 x float>* %m_ptr, <32 x float>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v32f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e32,m8,ta,mu
+; CHECK-NEXT:    vle32.v v8, (a1)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v8, ft0
+; CHECK-NEXT:    vle32.v v8, (a0), v0.t
+; CHECK-NEXT:    vse32.v v8, (a2)
+; CHECK-NEXT:    ret
+  %m = load <32 x float>, <32 x float>* %m_ptr
+  %mask = fcmp oeq <32 x float> %m, zeroinitializer
+  %load = call <32 x float> @llvm.masked.load.v32f32(<32 x float>* %a, i32 8, <32 x i1> %mask, <32 x float> undef)
+  store <32 x float> %load, <32 x float>* %res_ptr
+  ret void
+}
+declare <32 x float> @llvm.masked.load.v32f32(<32 x float>*, i32, <32 x i1>, <32 x float>)
+
+define void @masked_load_v32f64(<32 x double>* %a, <32 x double>* %m_ptr, <32 x double>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v32f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi a3, a1, 128
+; RV32-NEXT:    vsetivli a4, 16, e64,m8,ta,mu
+; RV32-NEXT:    vle64.v v8, (a1)
+; RV32-NEXT:    vle64.v v16, (a3)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v8, ft0
+; RV32-NEXT:    vmfeq.vf v26, v16, ft0
+; RV32-NEXT:    vle64.v v8, (a0), v0.t
+; RV32-NEXT:    addi a0, a0, 128
+; RV32-NEXT:    vmv1r.v v0, v26
+; RV32-NEXT:    vle64.v v16, (a0), v0.t
+; RV32-NEXT:    vse64.v v8, (a2)
+; RV32-NEXT:    addi a0, a2, 128
+; RV32-NEXT:    vse64.v v16, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v32f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi a3, a1, 128
+; RV64-NEXT:    vsetivli a4, 16, e64,m8,ta,mu
+; RV64-NEXT:    vle64.v v8, (a1)
+; RV64-NEXT:    vle64.v v16, (a3)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v8, ft0
+; RV64-NEXT:    vmfeq.vf v26, v16, ft0
+; RV64-NEXT:    vle64.v v8, (a0), v0.t
+; RV64-NEXT:    addi a0, a0, 128
+; RV64-NEXT:    vmv1r.v v0, v26
+; RV64-NEXT:    vle64.v v16, (a0), v0.t
+; RV64-NEXT:    vse64.v v8, (a2)
+; RV64-NEXT:    addi a0, a2, 128
+; RV64-NEXT:    vse64.v v16, (a0)
+; RV64-NEXT:    ret
+  %m = load <32 x double>, <32 x double>* %m_ptr
+  %mask = fcmp oeq <32 x double> %m, zeroinitializer
+  %load = call <32 x double> @llvm.masked.load.v32f64(<32 x double>* %a, i32 8, <32 x i1> %mask, <32 x double> undef)
+  store <32 x double> %load, <32 x double>* %res_ptr
+  ret void
+}
+declare <32 x double> @llvm.masked.load.v32f64(<32 x double>*, i32, <32 x i1>, <32 x double>)
+
+define void @masked_load_v64f16(<64 x half>* %a, <64 x half>* %m_ptr, <64 x half>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v64f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 64
+; CHECK-NEXT:    vsetvli a3, a3, e16,m8,ta,mu
+; CHECK-NEXT:    vle16.v v8, (a1)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v8, ft0
+; CHECK-NEXT:    vle16.v v8, (a0), v0.t
+; CHECK-NEXT:    vse16.v v8, (a2)
+; CHECK-NEXT:    ret
+  %m = load <64 x half>, <64 x half>* %m_ptr
+  %mask = fcmp oeq <64 x half> %m, zeroinitializer
+  %load = call <64 x half> @llvm.masked.load.v64f16(<64 x half>* %a, i32 8, <64 x i1> %mask, <64 x half> undef)
+  store <64 x half> %load, <64 x half>* %res_ptr
+  ret void
+}
+declare <64 x half> @llvm.masked.load.v64f16(<64 x half>*, i32, <64 x i1>, <64 x half>)
+
+define void @masked_load_v64f32(<64 x float>* %a, <64 x float>* %m_ptr, <64 x float>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v64f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, a1, 128
+; CHECK-NEXT:    addi a4, zero, 32
+; CHECK-NEXT:    vsetvli a4, a4, e32,m8,ta,mu
+; CHECK-NEXT:    vle32.v v8, (a1)
+; CHECK-NEXT:    vle32.v v16, (a3)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v8, ft0
+; CHECK-NEXT:    vmfeq.vf v26, v16, ft0
+; CHECK-NEXT:    vle32.v v8, (a0), v0.t
+; CHECK-NEXT:    addi a0, a0, 128
+; CHECK-NEXT:    vmv1r.v v0, v26
+; CHECK-NEXT:    vle32.v v16, (a0), v0.t
+; CHECK-NEXT:    vse32.v v8, (a2)
+; CHECK-NEXT:    addi a0, a2, 128
+; CHECK-NEXT:    vse32.v v16, (a0)
+; CHECK-NEXT:    ret
+  %m = load <64 x float>, <64 x float>* %m_ptr
+  %mask = fcmp oeq <64 x float> %m, zeroinitializer
+  %load = call <64 x float> @llvm.masked.load.v64f32(<64 x float>* %a, i32 8, <64 x i1> %mask, <64 x float> undef)
+  store <64 x float> %load, <64 x float>* %res_ptr
+  ret void
+}
+declare <64 x float> @llvm.masked.load.v64f32(<64 x float>*, i32, <64 x i1>, <64 x float>)
+
+define void @masked_load_v128f16(<128 x half>* %a, <128 x half>* %m_ptr, <128 x half>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v128f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, a1, 128
+; CHECK-NEXT:    addi a4, zero, 64
+; CHECK-NEXT:    vsetvli a4, a4, e16,m8,ta,mu
+; CHECK-NEXT:    vle16.v v8, (a1)
+; CHECK-NEXT:    vle16.v v16, (a3)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v8, ft0
+; CHECK-NEXT:    vmfeq.vf v26, v16, ft0
+; CHECK-NEXT:    vle16.v v8, (a0), v0.t
+; CHECK-NEXT:    addi a0, a0, 128
+; CHECK-NEXT:    vmv1r.v v0, v26
+; CHECK-NEXT:    vle16.v v16, (a0), v0.t
+; CHECK-NEXT:    vse16.v v8, (a2)
+; CHECK-NEXT:    addi a0, a2, 128
+; CHECK-NEXT:    vse16.v v16, (a0)
+; CHECK-NEXT:    ret
+  %m = load <128 x half>, <128 x half>* %m_ptr
+  %mask = fcmp oeq <128 x half> %m, zeroinitializer
+  %load = call <128 x half> @llvm.masked.load.v128f16(<128 x half>* %a, i32 8, <128 x i1> %mask, <128 x half> undef)
+  store <128 x half> %load, <128 x half>* %res_ptr
+  ret void
+}
+declare <128 x half> @llvm.masked.load.v128f16(<128 x half>*, i32, <128 x i1>, <128 x half>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5d3ff7c6c43649ee44458507e16455eb6452c697
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-load-int.ll
@@ -0,0 +1,610 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+
+define void @masked_load_v1i8(<1 x i8>* %a, <1 x i8>* %m_ptr, <1 x i8>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v1i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle8.v v25, (a0), v0.t
+; CHECK-NEXT:    vse8.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <1 x i8>, <1 x i8>* %m_ptr
+  %mask = icmp eq <1 x i8> %m, zeroinitializer
+  %load = call <1 x i8> @llvm.masked.load.v1i8(<1 x i8>* %a, i32 8, <1 x i1> %mask, <1 x i8> undef)
+  store <1 x i8> %load, <1 x i8>* %res_ptr
+  ret void
+}
+declare <1 x i8> @llvm.masked.load.v1i8(<1 x i8>*, i32, <1 x i1>, <1 x i8>)
+
+define void @masked_load_v1i16(<1 x i16>* %a, <1 x i16>* %m_ptr, <1 x i16>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v1i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle16.v v25, (a0), v0.t
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <1 x i16>, <1 x i16>* %m_ptr
+  %mask = icmp eq <1 x i16> %m, zeroinitializer
+  %load = call <1 x i16> @llvm.masked.load.v1i16(<1 x i16>* %a, i32 8, <1 x i1> %mask, <1 x i16> undef)
+  store <1 x i16> %load, <1 x i16>* %res_ptr
+  ret void
+}
+declare <1 x i16> @llvm.masked.load.v1i16(<1 x i16>*, i32, <1 x i1>, <1 x i16>)
+
+define void @masked_load_v1i32(<1 x i32>* %a, <1 x i32>* %m_ptr, <1 x i32>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v1i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle32.v v25, (a0), v0.t
+; CHECK-NEXT:    vse32.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <1 x i32>, <1 x i32>* %m_ptr
+  %mask = icmp eq <1 x i32> %m, zeroinitializer
+  %load = call <1 x i32> @llvm.masked.load.v1i32(<1 x i32>* %a, i32 8, <1 x i1> %mask, <1 x i32> undef)
+  store <1 x i32> %load, <1 x i32>* %res_ptr
+  ret void
+}
+declare <1 x i32> @llvm.masked.load.v1i32(<1 x i32>*, i32, <1 x i1>, <1 x i32>)
+
+define void @masked_load_v1i64(<1 x i64>* %a, <1 x i64>* %m_ptr, <1 x i64>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v1i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 1, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a1)
+; RV32-NEXT:    vsetivli a1, 2, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v26, 0
+; RV32-NEXT:    vsetivli a1, 1, e64,m1,ta,mu
+; RV32-NEXT:    vmseq.vv v0, v25, v26
+; RV32-NEXT:    vle64.v v25, (a0), v0.t
+; RV32-NEXT:    vse64.v v25, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v1i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 1, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a1)
+; RV64-NEXT:    vmseq.vi v0, v25, 0
+; RV64-NEXT:    vle64.v v25, (a0), v0.t
+; RV64-NEXT:    vse64.v v25, (a2)
+; RV64-NEXT:    ret
+  %m = load <1 x i64>, <1 x i64>* %m_ptr
+  %mask = icmp eq <1 x i64> %m, zeroinitializer
+  %load = call <1 x i64> @llvm.masked.load.v1i64(<1 x i64>* %a, i32 8, <1 x i1> %mask, <1 x i64> undef)
+  store <1 x i64> %load, <1 x i64>* %res_ptr
+  ret void
+}
+declare <1 x i64> @llvm.masked.load.v1i64(<1 x i64>*, i32, <1 x i1>, <1 x i64>)
+
+define void @masked_load_v2i8(<2 x i8>* %a, <2 x i8>* %m_ptr, <2 x i8>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle8.v v25, (a0), v0.t
+; CHECK-NEXT:    vse8.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <2 x i8>, <2 x i8>* %m_ptr
+  %mask = icmp eq <2 x i8> %m, zeroinitializer
+  %load = call <2 x i8> @llvm.masked.load.v2i8(<2 x i8>* %a, i32 8, <2 x i1> %mask, <2 x i8> undef)
+  store <2 x i8> %load, <2 x i8>* %res_ptr
+  ret void
+}
+declare <2 x i8> @llvm.masked.load.v2i8(<2 x i8>*, i32, <2 x i1>, <2 x i8>)
+
+define void @masked_load_v2i16(<2 x i16>* %a, <2 x i16>* %m_ptr, <2 x i16>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle16.v v25, (a0), v0.t
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <2 x i16>, <2 x i16>* %m_ptr
+  %mask = icmp eq <2 x i16> %m, zeroinitializer
+  %load = call <2 x i16> @llvm.masked.load.v2i16(<2 x i16>* %a, i32 8, <2 x i1> %mask, <2 x i16> undef)
+  store <2 x i16> %load, <2 x i16>* %res_ptr
+  ret void
+}
+declare <2 x i16> @llvm.masked.load.v2i16(<2 x i16>*, i32, <2 x i1>, <2 x i16>)
+
+define void @masked_load_v2i32(<2 x i32>* %a, <2 x i32>* %m_ptr, <2 x i32>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle32.v v25, (a0), v0.t
+; CHECK-NEXT:    vse32.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <2 x i32>, <2 x i32>* %m_ptr
+  %mask = icmp eq <2 x i32> %m, zeroinitializer
+  %load = call <2 x i32> @llvm.masked.load.v2i32(<2 x i32>* %a, i32 8, <2 x i1> %mask, <2 x i32> undef)
+  store <2 x i32> %load, <2 x i32>* %res_ptr
+  ret void
+}
+declare <2 x i32> @llvm.masked.load.v2i32(<2 x i32>*, i32, <2 x i1>, <2 x i32>)
+
+define void @masked_load_v2i64(<2 x i64>* %a, <2 x i64>* %m_ptr, <2 x i64>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a1)
+; RV32-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v26, 0
+; RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; RV32-NEXT:    vmseq.vv v0, v25, v26
+; RV32-NEXT:    vle64.v v25, (a0), v0.t
+; RV32-NEXT:    vse64.v v25, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a1)
+; RV64-NEXT:    vmseq.vi v0, v25, 0
+; RV64-NEXT:    vle64.v v25, (a0), v0.t
+; RV64-NEXT:    vse64.v v25, (a2)
+; RV64-NEXT:    ret
+  %m = load <2 x i64>, <2 x i64>* %m_ptr
+  %mask = icmp eq <2 x i64> %m, zeroinitializer
+  %load = call <2 x i64> @llvm.masked.load.v2i64(<2 x i64>* %a, i32 8, <2 x i1> %mask, <2 x i64> undef)
+  store <2 x i64> %load, <2 x i64>* %res_ptr
+  ret void
+}
+declare <2 x i64> @llvm.masked.load.v2i64(<2 x i64>*, i32, <2 x i1>, <2 x i64>)
+
+define void @masked_load_v4i8(<4 x i8>* %a, <4 x i8>* %m_ptr, <4 x i8>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle8.v v25, (a0), v0.t
+; CHECK-NEXT:    vse8.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <4 x i8>, <4 x i8>* %m_ptr
+  %mask = icmp eq <4 x i8> %m, zeroinitializer
+  %load = call <4 x i8> @llvm.masked.load.v4i8(<4 x i8>* %a, i32 8, <4 x i1> %mask, <4 x i8> undef)
+  store <4 x i8> %load, <4 x i8>* %res_ptr
+  ret void
+}
+declare <4 x i8> @llvm.masked.load.v4i8(<4 x i8>*, i32, <4 x i1>, <4 x i8>)
+
+define void @masked_load_v4i16(<4 x i16>* %a, <4 x i16>* %m_ptr, <4 x i16>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle16.v v25, (a0), v0.t
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <4 x i16>, <4 x i16>* %m_ptr
+  %mask = icmp eq <4 x i16> %m, zeroinitializer
+  %load = call <4 x i16> @llvm.masked.load.v4i16(<4 x i16>* %a, i32 8, <4 x i1> %mask, <4 x i16> undef)
+  store <4 x i16> %load, <4 x i16>* %res_ptr
+  ret void
+}
+declare <4 x i16> @llvm.masked.load.v4i16(<4 x i16>*, i32, <4 x i1>, <4 x i16>)
+
+define void @masked_load_v4i32(<4 x i32>* %a, <4 x i32>* %m_ptr, <4 x i32>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle32.v v25, (a0), v0.t
+; CHECK-NEXT:    vse32.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <4 x i32>, <4 x i32>* %m_ptr
+  %mask = icmp eq <4 x i32> %m, zeroinitializer
+  %load = call <4 x i32> @llvm.masked.load.v4i32(<4 x i32>* %a, i32 8, <4 x i1> %mask, <4 x i32> undef)
+  store <4 x i32> %load, <4 x i32>* %res_ptr
+  ret void
+}
+declare <4 x i32> @llvm.masked.load.v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
+
+define void @masked_load_v4i64(<4 x i64>* %a, <4 x i64>* %m_ptr, <4 x i64>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
+; RV32-NEXT:    vle64.v v26, (a1)
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vmv.v.i v28, 0
+; RV32-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; RV32-NEXT:    vmseq.vv v0, v26, v28
+; RV32-NEXT:    vle64.v v26, (a0), v0.t
+; RV32-NEXT:    vse64.v v26, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
+; RV64-NEXT:    vle64.v v26, (a1)
+; RV64-NEXT:    vmseq.vi v0, v26, 0
+; RV64-NEXT:    vle64.v v26, (a0), v0.t
+; RV64-NEXT:    vse64.v v26, (a2)
+; RV64-NEXT:    ret
+  %m = load <4 x i64>, <4 x i64>* %m_ptr
+  %mask = icmp eq <4 x i64> %m, zeroinitializer
+  %load = call <4 x i64> @llvm.masked.load.v4i64(<4 x i64>* %a, i32 8, <4 x i1> %mask, <4 x i64> undef)
+  store <4 x i64> %load, <4 x i64>* %res_ptr
+  ret void
+}
+declare <4 x i64> @llvm.masked.load.v4i64(<4 x i64>*, i32, <4 x i1>, <4 x i64>)
+
+define void @masked_load_v8i8(<8 x i8>* %a, <8 x i8>* %m_ptr, <8 x i8>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle8.v v25, (a0), v0.t
+; CHECK-NEXT:    vse8.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <8 x i8>, <8 x i8>* %m_ptr
+  %mask = icmp eq <8 x i8> %m, zeroinitializer
+  %load = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %a, i32 8, <8 x i1> %mask, <8 x i8> undef)
+  store <8 x i8> %load, <8 x i8>* %res_ptr
+  ret void
+}
+declare <8 x i8> @llvm.masked.load.v8i8(<8 x i8>*, i32, <8 x i1>, <8 x i8>)
+
+define void @masked_load_v8i16(<8 x i16>* %a, <8 x i16>* %m_ptr, <8 x i16>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle16.v v25, (a0), v0.t
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <8 x i16>, <8 x i16>* %m_ptr
+  %mask = icmp eq <8 x i16> %m, zeroinitializer
+  %load = call <8 x i16> @llvm.masked.load.v8i16(<8 x i16>* %a, i32 8, <8 x i1> %mask, <8 x i16> undef)
+  store <8 x i16> %load, <8 x i16>* %res_ptr
+  ret void
+}
+declare <8 x i16> @llvm.masked.load.v8i16(<8 x i16>*, i32, <8 x i1>, <8 x i16>)
+
+define void @masked_load_v8i32(<8 x i32>* %a, <8 x i32>* %m_ptr, <8 x i32>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e32,m2,ta,mu
+; CHECK-NEXT:    vle32.v v26, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v26, 0
+; CHECK-NEXT:    vle32.v v26, (a0), v0.t
+; CHECK-NEXT:    vse32.v v26, (a2)
+; CHECK-NEXT:    ret
+  %m = load <8 x i32>, <8 x i32>* %m_ptr
+  %mask = icmp eq <8 x i32> %m, zeroinitializer
+  %load = call <8 x i32> @llvm.masked.load.v8i32(<8 x i32>* %a, i32 8, <8 x i1> %mask, <8 x i32> undef)
+  store <8 x i32> %load, <8 x i32>* %res_ptr
+  ret void
+}
+declare <8 x i32> @llvm.masked.load.v8i32(<8 x i32>*, i32, <8 x i1>, <8 x i32>)
+
+define void @masked_load_v8i64(<8 x i64>* %a, <8 x i64>* %m_ptr, <8 x i64>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 8, e64,m4,ta,mu
+; RV32-NEXT:    vle64.v v28, (a1)
+; RV32-NEXT:    vsetivli a1, 16, e32,m4,ta,mu
+; RV32-NEXT:    vmv.v.i v8, 0
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vmseq.vv v0, v28, v8
+; RV32-NEXT:    vle64.v v28, (a0), v0.t
+; RV32-NEXT:    vse64.v v28, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 8, e64,m4,ta,mu
+; RV64-NEXT:    vle64.v v28, (a1)
+; RV64-NEXT:    vmseq.vi v0, v28, 0
+; RV64-NEXT:    vle64.v v28, (a0), v0.t
+; RV64-NEXT:    vse64.v v28, (a2)
+; RV64-NEXT:    ret
+  %m = load <8 x i64>, <8 x i64>* %m_ptr
+  %mask = icmp eq <8 x i64> %m, zeroinitializer
+  %load = call <8 x i64> @llvm.masked.load.v8i64(<8 x i64>* %a, i32 8, <8 x i1> %mask, <8 x i64> undef)
+  store <8 x i64> %load, <8 x i64>* %res_ptr
+  ret void
+}
+declare <8 x i64> @llvm.masked.load.v8i64(<8 x i64>*, i32, <8 x i1>, <8 x i64>)
+
+define void @masked_load_v16i8(<16 x i8>* %a, <16 x i8>* %m_ptr, <16 x i8>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vle8.v v25, (a0), v0.t
+; CHECK-NEXT:    vse8.v v25, (a2)
+; CHECK-NEXT:    ret
+  %m = load <16 x i8>, <16 x i8>* %m_ptr
+  %mask = icmp eq <16 x i8> %m, zeroinitializer
+  %load = call <16 x i8> @llvm.masked.load.v16i8(<16 x i8>* %a, i32 8, <16 x i1> %mask, <16 x i8> undef)
+  store <16 x i8> %load, <16 x i8>* %res_ptr
+  ret void
+}
+declare <16 x i8> @llvm.masked.load.v16i8(<16 x i8>*, i32, <16 x i1>, <16 x i8>)
+
+define void @masked_load_v16i16(<16 x i16>* %a, <16 x i16>* %m_ptr, <16 x i16>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e16,m2,ta,mu
+; CHECK-NEXT:    vle16.v v26, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v26, 0
+; CHECK-NEXT:    vle16.v v26, (a0), v0.t
+; CHECK-NEXT:    vse16.v v26, (a2)
+; CHECK-NEXT:    ret
+  %m = load <16 x i16>, <16 x i16>* %m_ptr
+  %mask = icmp eq <16 x i16> %m, zeroinitializer
+  %load = call <16 x i16> @llvm.masked.load.v16i16(<16 x i16>* %a, i32 8, <16 x i1> %mask, <16 x i16> undef)
+  store <16 x i16> %load, <16 x i16>* %res_ptr
+  ret void
+}
+declare <16 x i16> @llvm.masked.load.v16i16(<16 x i16>*, i32, <16 x i1>, <16 x i16>)
+
+define void @masked_load_v16i32(<16 x i32>* %a, <16 x i32>* %m_ptr, <16 x i32>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v16i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e32,m4,ta,mu
+; CHECK-NEXT:    vle32.v v28, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v28, 0
+; CHECK-NEXT:    vle32.v v28, (a0), v0.t
+; CHECK-NEXT:    vse32.v v28, (a2)
+; CHECK-NEXT:    ret
+  %m = load <16 x i32>, <16 x i32>* %m_ptr
+  %mask = icmp eq <16 x i32> %m, zeroinitializer
+  %load = call <16 x i32> @llvm.masked.load.v16i32(<16 x i32>* %a, i32 8, <16 x i1> %mask, <16 x i32> undef)
+  store <16 x i32> %load, <16 x i32>* %res_ptr
+  ret void
+}
+declare <16 x i32> @llvm.masked.load.v16i32(<16 x i32>*, i32, <16 x i1>, <16 x i32>)
+
+define void @masked_load_v16i64(<16 x i64>* %a, <16 x i64>* %m_ptr, <16 x i64>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v16i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV32-NEXT:    vle64.v v8, (a1)
+; RV32-NEXT:    addi a1, zero, 32
+; RV32-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
+; RV32-NEXT:    vmv.v.i v16, 0
+; RV32-NEXT:    vsetivli a1, 16, e64,m8,ta,mu
+; RV32-NEXT:    vmseq.vv v0, v8, v16
+; RV32-NEXT:    vle64.v v8, (a0), v0.t
+; RV32-NEXT:    vse64.v v8, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v16i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV64-NEXT:    vle64.v v8, (a1)
+; RV64-NEXT:    vmseq.vi v0, v8, 0
+; RV64-NEXT:    vle64.v v8, (a0), v0.t
+; RV64-NEXT:    vse64.v v8, (a2)
+; RV64-NEXT:    ret
+  %m = load <16 x i64>, <16 x i64>* %m_ptr
+  %mask = icmp eq <16 x i64> %m, zeroinitializer
+  %load = call <16 x i64> @llvm.masked.load.v16i64(<16 x i64>* %a, i32 8, <16 x i1> %mask, <16 x i64> undef)
+  store <16 x i64> %load, <16 x i64>* %res_ptr
+  ret void
+}
+declare <16 x i64> @llvm.masked.load.v16i64(<16 x i64>*, i32, <16 x i1>, <16 x i64>)
+
+define void @masked_load_v32i8(<32 x i8>* %a, <32 x i8>* %m_ptr, <32 x i8>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e8,m2,ta,mu
+; CHECK-NEXT:    vle8.v v26, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v26, 0
+; CHECK-NEXT:    vle8.v v26, (a0), v0.t
+; CHECK-NEXT:    vse8.v v26, (a2)
+; CHECK-NEXT:    ret
+  %m = load <32 x i8>, <32 x i8>* %m_ptr
+  %mask = icmp eq <32 x i8> %m, zeroinitializer
+  %load = call <32 x i8> @llvm.masked.load.v32i8(<32 x i8>* %a, i32 8, <32 x i1> %mask, <32 x i8> undef)
+  store <32 x i8> %load, <32 x i8>* %res_ptr
+  ret void
+}
+declare <32 x i8> @llvm.masked.load.v32i8(<32 x i8>*, i32, <32 x i1>, <32 x i8>)
+
+define void @masked_load_v32i16(<32 x i16>* %a, <32 x i16>* %m_ptr, <32 x i16>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v32i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e16,m4,ta,mu
+; CHECK-NEXT:    vle16.v v28, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v28, 0
+; CHECK-NEXT:    vle16.v v28, (a0), v0.t
+; CHECK-NEXT:    vse16.v v28, (a2)
+; CHECK-NEXT:    ret
+  %m = load <32 x i16>, <32 x i16>* %m_ptr
+  %mask = icmp eq <32 x i16> %m, zeroinitializer
+  %load = call <32 x i16> @llvm.masked.load.v32i16(<32 x i16>* %a, i32 8, <32 x i1> %mask, <32 x i16> undef)
+  store <32 x i16> %load, <32 x i16>* %res_ptr
+  ret void
+}
+declare <32 x i16> @llvm.masked.load.v32i16(<32 x i16>*, i32, <32 x i1>, <32 x i16>)
+
+define void @masked_load_v32i32(<32 x i32>* %a, <32 x i32>* %m_ptr, <32 x i32>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v32i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e32,m8,ta,mu
+; CHECK-NEXT:    vle32.v v8, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vle32.v v8, (a0), v0.t
+; CHECK-NEXT:    vse32.v v8, (a2)
+; CHECK-NEXT:    ret
+  %m = load <32 x i32>, <32 x i32>* %m_ptr
+  %mask = icmp eq <32 x i32> %m, zeroinitializer
+  %load = call <32 x i32> @llvm.masked.load.v32i32(<32 x i32>* %a, i32 8, <32 x i1> %mask, <32 x i32> undef)
+  store <32 x i32> %load, <32 x i32>* %res_ptr
+  ret void
+}
+declare <32 x i32> @llvm.masked.load.v32i32(<32 x i32>*, i32, <32 x i1>, <32 x i32>)
+
+define void @masked_load_v32i64(<32 x i64>* %a, <32 x i64>* %m_ptr, <32 x i64>* %res_ptr) nounwind {
+; RV32-LABEL: masked_load_v32i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi a3, a1, 128
+; RV32-NEXT:    vsetivli a4, 16, e64,m8,ta,mu
+; RV32-NEXT:    vle64.v v8, (a3)
+; RV32-NEXT:    vle64.v v16, (a1)
+; RV32-NEXT:    addi a1, zero, 32
+; RV32-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
+; RV32-NEXT:    vmv.v.i v24, 0
+; RV32-NEXT:    vsetivli a1, 16, e64,m8,ta,mu
+; RV32-NEXT:    vmseq.vv v0, v16, v24
+; RV32-NEXT:    vmseq.vv v16, v8, v24
+; RV32-NEXT:    vle64.v v8, (a0), v0.t
+; RV32-NEXT:    addi a0, a0, 128
+; RV32-NEXT:    vmv1r.v v0, v16
+; RV32-NEXT:    vle64.v v16, (a0), v0.t
+; RV32-NEXT:    vse64.v v8, (a2)
+; RV32-NEXT:    addi a0, a2, 128
+; RV32-NEXT:    vse64.v v16, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_load_v32i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi a3, a1, 128
+; RV64-NEXT:    vsetivli a4, 16, e64,m8,ta,mu
+; RV64-NEXT:    vle64.v v8, (a1)
+; RV64-NEXT:    vle64.v v16, (a3)
+; RV64-NEXT:    vmseq.vi v0, v8, 0
+; RV64-NEXT:    vmseq.vi v26, v16, 0
+; RV64-NEXT:    vle64.v v8, (a0), v0.t
+; RV64-NEXT:    addi a0, a0, 128
+; RV64-NEXT:    vmv1r.v v0, v26
+; RV64-NEXT:    vle64.v v16, (a0), v0.t
+; RV64-NEXT:    vse64.v v8, (a2)
+; RV64-NEXT:    addi a0, a2, 128
+; RV64-NEXT:    vse64.v v16, (a0)
+; RV64-NEXT:    ret
+  %m = load <32 x i64>, <32 x i64>* %m_ptr
+  %mask = icmp eq <32 x i64> %m, zeroinitializer
+  %load = call <32 x i64> @llvm.masked.load.v32i64(<32 x i64>* %a, i32 8, <32 x i1> %mask, <32 x i64> undef)
+  store <32 x i64> %load, <32 x i64>* %res_ptr
+  ret void
+}
+declare <32 x i64> @llvm.masked.load.v32i64(<32 x i64>*, i32, <32 x i1>, <32 x i64>)
+
+define void @masked_load_v64i8(<64 x i8>* %a, <64 x i8>* %m_ptr, <64 x i8>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v64i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 64
+; CHECK-NEXT:    vsetvli a3, a3, e8,m4,ta,mu
+; CHECK-NEXT:    vle8.v v28, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v28, 0
+; CHECK-NEXT:    vle8.v v28, (a0), v0.t
+; CHECK-NEXT:    vse8.v v28, (a2)
+; CHECK-NEXT:    ret
+  %m = load <64 x i8>, <64 x i8>* %m_ptr
+  %mask = icmp eq <64 x i8> %m, zeroinitializer
+  %load = call <64 x i8> @llvm.masked.load.v64i8(<64 x i8>* %a, i32 8, <64 x i1> %mask, <64 x i8> undef)
+  store <64 x i8> %load, <64 x i8>* %res_ptr
+  ret void
+}
+declare <64 x i8> @llvm.masked.load.v64i8(<64 x i8>*, i32, <64 x i1>, <64 x i8>)
+
+define void @masked_load_v64i16(<64 x i16>* %a, <64 x i16>* %m_ptr, <64 x i16>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v64i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 64
+; CHECK-NEXT:    vsetvli a3, a3, e16,m8,ta,mu
+; CHECK-NEXT:    vle16.v v8, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vle16.v v8, (a0), v0.t
+; CHECK-NEXT:    vse16.v v8, (a2)
+; CHECK-NEXT:    ret
+  %m = load <64 x i16>, <64 x i16>* %m_ptr
+  %mask = icmp eq <64 x i16> %m, zeroinitializer
+  %load = call <64 x i16> @llvm.masked.load.v64i16(<64 x i16>* %a, i32 8, <64 x i1> %mask, <64 x i16> undef)
+  store <64 x i16> %load, <64 x i16>* %res_ptr
+  ret void
+}
+declare <64 x i16> @llvm.masked.load.v64i16(<64 x i16>*, i32, <64 x i1>, <64 x i16>)
+
+define void @masked_load_v64i32(<64 x i32>* %a, <64 x i32>* %m_ptr, <64 x i32>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v64i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, a1, 128
+; CHECK-NEXT:    addi a4, zero, 32
+; CHECK-NEXT:    vsetvli a4, a4, e32,m8,ta,mu
+; CHECK-NEXT:    vle32.v v8, (a1)
+; CHECK-NEXT:    vle32.v v16, (a3)
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vmseq.vi v26, v16, 0
+; CHECK-NEXT:    vle32.v v8, (a0), v0.t
+; CHECK-NEXT:    addi a0, a0, 128
+; CHECK-NEXT:    vmv1r.v v0, v26
+; CHECK-NEXT:    vle32.v v16, (a0), v0.t
+; CHECK-NEXT:    vse32.v v8, (a2)
+; CHECK-NEXT:    addi a0, a2, 128
+; CHECK-NEXT:    vse32.v v16, (a0)
+; CHECK-NEXT:    ret
+  %m = load <64 x i32>, <64 x i32>* %m_ptr
+  %mask = icmp eq <64 x i32> %m, zeroinitializer
+  %load = call <64 x i32> @llvm.masked.load.v64i32(<64 x i32>* %a, i32 8, <64 x i1> %mask, <64 x i32> undef)
+  store <64 x i32> %load, <64 x i32>* %res_ptr
+  ret void
+}
+declare <64 x i32> @llvm.masked.load.v64i32(<64 x i32>*, i32, <64 x i1>, <64 x i32>)
+
+define void @masked_load_v128i8(<128 x i8>* %a, <128 x i8>* %m_ptr, <128 x i8>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v128i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 128
+; CHECK-NEXT:    vsetvli a3, a3, e8,m8,ta,mu
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vle8.v v8, (a0), v0.t
+; CHECK-NEXT:    vse8.v v8, (a2)
+; CHECK-NEXT:    ret
+  %m = load <128 x i8>, <128 x i8>* %m_ptr
+  %mask = icmp eq <128 x i8> %m, zeroinitializer
+  %load = call <128 x i8> @llvm.masked.load.v128i8(<128 x i8>* %a, i32 8, <128 x i1> %mask, <128 x i8> undef)
+  store <128 x i8> %load, <128 x i8>* %res_ptr
+  ret void
+}
+declare <128 x i8> @llvm.masked.load.v128i8(<128 x i8>*, i32, <128 x i1>, <128 x i8>)
+
+define void @masked_load_v256i8(<256 x i8>* %a, <256 x i8>* %m_ptr, <256 x i8>* %res_ptr) nounwind {
+; CHECK-LABEL: masked_load_v256i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, a1, 128
+; CHECK-NEXT:    addi a4, zero, 128
+; CHECK-NEXT:    vsetvli a4, a4, e8,m8,ta,mu
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vle8.v v16, (a3)
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vmseq.vi v26, v16, 0
+; CHECK-NEXT:    vle8.v v8, (a0), v0.t
+; CHECK-NEXT:    addi a0, a0, 128
+; CHECK-NEXT:    vmv1r.v v0, v26
+; CHECK-NEXT:    vle8.v v16, (a0), v0.t
+; CHECK-NEXT:    vse8.v v8, (a2)
+; CHECK-NEXT:    addi a0, a2, 128
+; CHECK-NEXT:    vse8.v v16, (a0)
+; CHECK-NEXT:    ret
+  %m = load <256 x i8>, <256 x i8>* %m_ptr
+  %mask = icmp eq <256 x i8> %m, zeroinitializer
+  %load = call <256 x i8> @llvm.masked.load.v256i8(<256 x i8>* %a, i32 8, <256 x i1> %mask, <256 x i8> undef)
+  store <256 x i8> %load, <256 x i8>* %res_ptr
+  ret void
+}
+declare <256 x i8> @llvm.masked.load.v256i8(<256 x i8>*, i32, <256 x i1>, <256 x i8>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
new file mode 100644
index 0000000000000000000000000000000000000000..58716cf2b6bc5d87d407c59afa9fcb531c69fdc8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
@@ -0,0 +1,1919 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=ilp32d \
+; RUN:     -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=lp64d \
+; RUN:     -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV64
+
+declare void @llvm.masked.scatter.v1i8.v1p0i8(<1 x i8>, <1 x i8*>, i32, <1 x i1>)
+
+define void @mscatter_v1i8(<1 x i8> %val, <1 x i8*> %ptrs, <1 x i1> %m) {
+; RV32-LABEL: mscatter_v1i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v1i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e8,mf8,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v1i8.v1p0i8(<1 x i8> %val, <1 x i8*> %ptrs, i32 1, <1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8>, <2 x i8*>, i32, <2 x i1>)
+
+define void @mscatter_v2i8(<2 x i8> %val, <2 x i8*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e8,mf8,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8> %val, <2 x i8*> %ptrs, i32 1, <2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_v2i16_truncstore_v2i8(<2 x i16> %val, <2 x i8*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i16_truncstore_v2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV32-NEXT:    vsoxei32.v v25, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i16_truncstore_v2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsetivli a0, 1, e8,mf8,ta,mu
+; RV64-NEXT:    vsoxei64.v v25, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <2 x i16> %val to <2 x i8>
+  call void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8> %tval, <2 x i8*> %ptrs, i32 1, <2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_v2i32_truncstore_v2i8(<2 x i32> %val, <2 x i8*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i32_truncstore_v2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV32-NEXT:    vnsrl.wi v26, v25, 0
+; RV32-NEXT:    vsoxei32.v v26, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i32_truncstore_v2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV64-NEXT:    vnsrl.wi v26, v25, 0
+; RV64-NEXT:    vsetivli a0, 1, e8,mf8,ta,mu
+; RV64-NEXT:    vsoxei64.v v26, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <2 x i32> %val to <2 x i8>
+  call void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8> %tval, <2 x i8*> %ptrs, i32 1, <2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_v2i64_truncstore_v2i8(<2 x i64> %val, <2 x i8*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i64_truncstore_v2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsetivli a0, 2, e16,mf4,ta,mu
+; RV32-NEXT:    vnsrl.wi v26, v25, 0
+; RV32-NEXT:    vsetivli a0, 2, e8,mf8,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v26, 0
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV32-NEXT:    vsoxei32.v v25, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i64_truncstore_v2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsetivli a0, 2, e16,mf4,ta,mu
+; RV64-NEXT:    vnsrl.wi v26, v25, 0
+; RV64-NEXT:    vsetivli a0, 2, e8,mf8,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v26, 0
+; RV64-NEXT:    vsetivli a0, 1, e8,mf8,ta,mu
+; RV64-NEXT:    vsoxei64.v v25, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <2 x i64> %val to <2 x i8>
+  call void @llvm.masked.scatter.v2i8.v2p0i8(<2 x i8> %tval, <2 x i8*> %ptrs, i32 1, <2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8>, <4 x i8*>, i32, <4 x i1>)
+
+define void @mscatter_v4i8(<4 x i8> %val, <4 x i8*> %ptrs, <4 x i1> %m) {
+; RV32-LABEL: mscatter_v4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> %val, <4 x i8*> %ptrs, i32 1, <4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_v4i8(<4 x i8> %val, <4 x i8*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_v4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_v4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e8,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> %val, <4 x i8*> %ptrs, i32 1, <4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_v4i8(<4 x i8> %val, <4 x i8*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_v4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_v4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> %val, <4 x i8*> %ptrs, i32 1, <4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v8i8.v8p0i8(<8 x i8>, <8 x i8*>, i32, <8 x i1>)
+
+define void @mscatter_v8i8(<8 x i8> %val, <8 x i8*> %ptrs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_v8i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e8,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v8i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e8,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v8i8.v8p0i8(<8 x i8> %val, <8 x i8*> %ptrs, i32 1, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i8(<8 x i8> %val, i8* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v9
+; RV32-NEXT:    vsetivli a1, 4, e8,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v9
+; RV64-NEXT:    vsetivli a1, 4, e8,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <8 x i8> %idxs
+  call void @llvm.masked.scatter.v8i8.v8p0i8(<8 x i8> %val, <8 x i8*> %ptrs, i32 1, <8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v1i16.v1p0i16(<1 x i16>, <1 x i16*>, i32, <1 x i1>)
+
+define void @mscatter_v1i16(<1 x i16> %val, <1 x i16*> %ptrs, <1 x i1> %m) {
+; RV32-LABEL: mscatter_v1i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v1i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e16,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v1i16.v1p0i16(<1 x i16> %val, <1 x i16*> %ptrs, i32 2, <1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2i16.v2p0i16(<2 x i16>, <2 x i16*>, i32, <2 x i1>)
+
+define void @mscatter_v2i16(<2 x i16> %val, <2 x i16*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e16,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v2i16.v2p0i16(<2 x i16> %val, <2 x i16*> %ptrs, i32 2, <2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_v2i32_truncstore_v2i16(<2 x i32> %val, <2 x i16*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i32_truncstore_v2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsoxei32.v v25, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i32_truncstore_v2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsetivli a0, 1, e16,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v25, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <2 x i32> %val to <2 x i16>
+  call void @llvm.masked.scatter.v2i16.v2p0i16(<2 x i16> %tval, <2 x i16*> %ptrs, i32 2, <2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_v2i64_truncstore_v2i16(<2 x i64> %val, <2 x i16*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i64_truncstore_v2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsetivli a0, 2, e16,mf4,ta,mu
+; RV32-NEXT:    vnsrl.wi v26, v25, 0
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v26, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i64_truncstore_v2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsetivli a0, 2, e16,mf4,ta,mu
+; RV64-NEXT:    vnsrl.wi v26, v25, 0
+; RV64-NEXT:    vsetivli a0, 1, e16,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v26, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <2 x i64> %val to <2 x i16>
+  call void @llvm.masked.scatter.v2i16.v2p0i16(<2 x i16> %tval, <2 x i16*> %ptrs, i32 2, <2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16>, <4 x i16*>, i32, <4 x i1>)
+
+define void @mscatter_v4i16(<4 x i16> %val, <4 x i16*> %ptrs, <4 x i1> %m) {
+; RV32-LABEL: mscatter_v4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16> %val, <4 x i16*> %ptrs, i32 2, <4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_v4i16(<4 x i16> %val, <4 x i16*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_v4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_v4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16> %val, <4 x i16*> %ptrs, i32 2, <4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_v4i16(<4 x i16> %val, <4 x i16*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_v4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_v4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4i16.v4p0i16(<4 x i16> %val, <4 x i16*> %ptrs, i32 2, <4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16>, <8 x i16*>, i32, <8 x i1>)
+
+define void @mscatter_v8i16(<8 x i16> %val, <8 x i16*> %ptrs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %val, <8 x i16*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i8_v8i16(<8 x i16> %val, i16* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i8_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v9
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i8_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v9
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i8> %idxs
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %val, <8 x i16*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i8_v8i16(<8 x i16> %val, i16* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i8_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v9
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i8_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v9
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i16>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> %eidxs
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %val, <8 x i16*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i8_v8i16(<8 x i16> %val, i16* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i8_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf4 v26, v9
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i8_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v9
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i16>
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> %eidxs
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %val, <8 x i16*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i16(<8 x i16> %val, i16* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v9
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v9
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i16, i16* %base, <8 x i16> %idxs
+  call void @llvm.masked.scatter.v8i16.v8p0i16(<8 x i16> %val, <8 x i16*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v1i32.v1p0i32(<1 x i32>, <1 x i32*>, i32, <1 x i1>)
+
+define void @mscatter_v1i32(<1 x i32> %val, <1 x i32*> %ptrs, <1 x i1> %m) {
+; RV32-LABEL: mscatter_v1i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v1i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e32,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v1i32.v1p0i32(<1 x i32> %val, <1 x i32*> %ptrs, i32 4, <1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32>, <2 x i32*>, i32, <2 x i1>)
+
+define void @mscatter_v2i32(<2 x i32> %val, <2 x i32*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e32,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %val, <2 x i32*> %ptrs, i32 4, <2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_v2i64_truncstore_v2i32(<2 x i64> %val, <2 x i32*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i64_truncstore_v2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v25, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i64_truncstore_v2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsetivli a0, 1, e32,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v25, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <2 x i64> %val to <2 x i32>
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %tval, <2 x i32*> %ptrs, i32 4, <2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>, <4 x i32*>, i32, <4 x i1>)
+
+define void @mscatter_v4i32(<4 x i32> %val, <4 x i32*> %ptrs, <4 x i1> %m) {
+; RV32-LABEL: mscatter_v4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %val, <4 x i32*> %ptrs, i32 4, <4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_v4i32(<4 x i32> %val, <4 x i32*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_v4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_v4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %val, <4 x i32*> %ptrs, i32 4, <4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_v4i32(<4 x i32> %val, <4 x i32*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_v4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_v4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %val, <4 x i32*> %ptrs, i32 4, <4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32>, <8 x i32*>, i32, <8 x i1>)
+
+define void @mscatter_v8i32(<8 x i32> %val, <8 x i32*> %ptrs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %val, <8 x i32*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i8_v8i32(<8 x i32> %val, i32* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i8_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i8_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i8> %idxs
+  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %val, <8 x i32*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i8_v8i32(<8 x i32> %val, i32* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i8_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i8_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %eidxs
+  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %val, <8 x i32*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i8_v8i32(<8 x i32> %val, i32* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i8_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf4 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i8_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %eidxs
+  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %val, <8 x i32*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i16_v8i32(<8 x i32> %val, i32* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i16_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i16_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i16> %idxs
+  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %val, <8 x i32*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i16_v8i32(<8 x i32> %val, i32* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i16_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i16_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i16> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %eidxs
+  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %val, <8 x i32*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i16_v8i32(<8 x i32> %val, i32* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i16_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf2 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i16_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf4 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i16> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %eidxs
+  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %val, <8 x i32*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i32(<8 x i32> %val, i32* %base, <8 x i32> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsll.vi v26, v10, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <8 x i32> %idxs
+  call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %val, <8 x i32*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v1i64.v1p0i64(<1 x i64>, <1 x i64*>, i32, <1 x i1>)
+
+define void @mscatter_v1i64(<1 x i64> %val, <1 x i64*> %ptrs, <1 x i1> %m) {
+; RV32-LABEL: mscatter_v1i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e64,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v1i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e64,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v1i64.v1p0i64(<1 x i64> %val, <1 x i64*> %ptrs, i32 8, <1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64>, <2 x i64*>, i32, <2 x i1>)
+
+define void @mscatter_v2i64(<2 x i64> %val, <2 x i64*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e64,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e64,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> %val, <2 x i64*> %ptrs, i32 8, <2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v4i64.v4p0i64(<4 x i64>, <4 x i64*>, i32, <4 x i1>)
+
+define void @mscatter_v4i64(<4 x i64> %val, <4 x i64*> %ptrs, <4 x i1> %m) {
+; RV32-LABEL: mscatter_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e64,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e64,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4i64.v4p0i64(<4 x i64> %val, <4 x i64*> %ptrs, i32 8, <4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_v4i64(<4 x i64> %val, <4 x i64*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e64,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e64,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.v4i64.v4p0i64(<4 x i64> %val, <4 x i64*> %ptrs, i32 8, <4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_v4i64(<4 x i64> %val, <4 x i64*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4i64.v4p0i64(<4 x i64> %val, <4 x i64*> %ptrs, i32 8, <4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64>, <8 x i64*>, i32, <8 x i1>)
+
+define void @mscatter_v8i64(<8 x i64> %val, <8 x i64*> %ptrs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i8_v8i64(<8 x i64> %val, i64* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i8_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v12
+; RV32-NEXT:    vsll.vi v26, v26, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i8_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i8> %idxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i8_v8i64(<8 x i64> %val, i64* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i8_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf8 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i8_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, i64* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i8_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf8 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i8_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, i64* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i16_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v12
+; RV32-NEXT:    vsll.vi v26, v26, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i16_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i16> %idxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, i64* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i16_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i16_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i16> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, i64* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i16_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i16_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf4 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i16> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i32_v8i64(<8 x i64> %val, i64* %base, <8 x i32> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i32_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsll.vi v26, v12, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i32_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i32> %idxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i32_v8i64(<8 x i64> %val, i64* %base, <8 x i32> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i32_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i32_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i32> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i32_v8i64(<8 x i64> %val, i64* %base, <8 x i32> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i32_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i32_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf2 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i32> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i64(<8 x i64> %val, i64* %base, <8 x i64> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v12, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsll.vi v28, v12, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <8 x i64> %idxs
+  call void @llvm.masked.scatter.v8i64.v8p0i64(<8 x i64> %val, <8 x i64*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v1f16.v1p0f16(<1 x half>, <1 x half*>, i32, <1 x i1>)
+
+define void @mscatter_v1f16(<1 x half> %val, <1 x half*> %ptrs, <1 x i1> %m) {
+; RV32-LABEL: mscatter_v1f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v1f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e16,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v1f16.v1p0f16(<1 x half> %val, <1 x half*> %ptrs, i32 2, <1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2f16.v2p0f16(<2 x half>, <2 x half*>, i32, <2 x i1>)
+
+define void @mscatter_v2f16(<2 x half> %val, <2 x half*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e16,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v2f16.v2p0f16(<2 x half> %val, <2 x half*> %ptrs, i32 2, <2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half>, <4 x half*>, i32, <4 x i1>)
+
+define void @mscatter_v4f16(<4 x half> %val, <4 x half*> %ptrs, <4 x i1> %m) {
+; RV32-LABEL: mscatter_v4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half> %val, <4 x half*> %ptrs, i32 2, <4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_v4f16(<4 x half> %val, <4 x half*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_v4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_v4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e16,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half> %val, <4 x half*> %ptrs, i32 2, <4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_v4f16(<4 x half> %val, <4 x half*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_v4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_v4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4f16.v4p0f16(<4 x half> %val, <4 x half*> %ptrs, i32 2, <4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half>, <8 x half*>, i32, <8 x i1>)
+
+define void @mscatter_v8f16(<8 x half> %val, <8 x half*> %ptrs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %val, <8 x half*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i8_v8f16(<8 x half> %val, half* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i8_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v9
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i8_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v9
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds half, half* %base, <8 x i8> %idxs
+  call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %val, <8 x half*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i8_v8f16(<8 x half> %val, half* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i8_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v9
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i8_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v9
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i16>
+  %ptrs = getelementptr inbounds half, half* %base, <8 x i16> %eidxs
+  call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %val, <8 x half*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i8_v8f16(<8 x half> %val, half* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i8_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf4 v26, v9
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i8_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v9
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i16>
+  %ptrs = getelementptr inbounds half, half* %base, <8 x i16> %eidxs
+  call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %val, <8 x half*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8f16(<8 x half> %val, half* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v9
+; RV32-NEXT:    vsll.vi v26, v26, 1
+; RV32-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v9
+; RV64-NEXT:    vsll.vi v28, v28, 1
+; RV64-NEXT:    vsetivli a1, 4, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds half, half* %base, <8 x i16> %idxs
+  call void @llvm.masked.scatter.v8f16.v8p0f16(<8 x half> %val, <8 x half*> %ptrs, i32 2, <8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v1f32.v1p0f32(<1 x float>, <1 x float*>, i32, <1 x i1>)
+
+define void @mscatter_v1f32(<1 x float> %val, <1 x float*> %ptrs, <1 x i1> %m) {
+; RV32-LABEL: mscatter_v1f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v1f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e32,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v1f32.v1p0f32(<1 x float> %val, <1 x float*> %ptrs, i32 4, <1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float>, <2 x float*>, i32, <2 x i1>)
+
+define void @mscatter_v2f32(<2 x float> %val, <2 x float*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e32,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> %val, <2 x float*> %ptrs, i32 4, <2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float>, <4 x float*>, i32, <4 x i1>)
+
+define void @mscatter_v4f32(<4 x float> %val, <4 x float*> %ptrs, <4 x i1> %m) {
+; RV32-LABEL: mscatter_v4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> %val, <4 x float*> %ptrs, i32 4, <4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_v4f32(<4 x float> %val, <4 x float*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_v4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_v4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> %val, <4 x float*> %ptrs, i32 4, <4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_v4f32(<4 x float> %val, <4 x float*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_v4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_v4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4f32.v4p0f32(<4 x float> %val, <4 x float*> %ptrs, i32 4, <4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float>, <8 x float*>, i32, <8 x i1>)
+
+define void @mscatter_v8f32(<8 x float> %val, <8 x float*> %ptrs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> %val, <8 x float*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i8_v8f32(<8 x float> %val, float* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i8_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i8_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i8> %idxs
+  call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> %val, <8 x float*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i8_v8f32(<8 x float> %val, float* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i8_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i8_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %eidxs
+  call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> %val, <8 x float*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i8_v8f32(<8 x float> %val, float* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i8_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf4 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i8_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %eidxs
+  call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> %val, <8 x float*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i16_v8f32(<8 x float> %val, float* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i16_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i16_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i16> %idxs
+  call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> %val, <8 x float*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i16_v8f32(<8 x float> %val, float* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i16_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i16_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i16> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %eidxs
+  call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> %val, <8 x float*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i16_v8f32(<8 x float> %val, float* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i16_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vzext.vf2 v26, v10
+; RV32-NEXT:    vsll.vi v26, v26, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i16_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf4 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i16> %idxs to <8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %eidxs
+  call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> %val, <8 x float*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8f32(<8 x float> %val, float* %base, <8 x i32> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsll.vi v26, v10, 2
+; RV32-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v10
+; RV64-NEXT:    vsll.vi v28, v28, 2
+; RV64-NEXT:    vsetivli a1, 4, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <8 x i32> %idxs
+  call void @llvm.masked.scatter.v8f32.v8p0f32(<8 x float> %val, <8 x float*> %ptrs, i32 4, <8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v1f64.v1p0f64(<1 x double>, <1 x double*>, i32, <1 x i1>)
+
+define void @mscatter_v1f64(<1 x double> %val, <1 x double*> %ptrs, <1 x i1> %m) {
+; RV32-LABEL: mscatter_v1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e64,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e64,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v1f64.v1p0f64(<1 x double> %val, <1 x double*> %ptrs, i32 8, <1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double>, <2 x double*>, i32, <2 x i1>)
+
+define void @mscatter_v2f64(<2 x double> %val, <2 x double*> %ptrs, <2 x i1> %m) {
+; RV32-LABEL: mscatter_v2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 1, e64,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 1, e64,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %val, <2 x double*> %ptrs, i32 8, <2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double>, <4 x double*>, i32, <4 x i1>)
+
+define void @mscatter_v4f64(<4 x double> %val, <4 x double*> %ptrs, <4 x i1> %m) {
+; RV32-LABEL: mscatter_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e64,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e64,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> %val, <4 x double*> %ptrs, i32 8, <4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_v4f64(<4 x double> %val, <4 x double*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 2, e64,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_v4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 2, e64,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %mhead = insertelement <4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <4 x i1> %mhead, <4 x i1> undef, <4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> %val, <4 x double*> %ptrs, i32 8, <4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_v4f64(<4 x double> %val, <4 x double*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_v4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v4f64.v4p0f64(<4 x double> %val, <4 x double*> %ptrs, i32 8, <4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double>, <8 x double*>, i32, <8 x i1>)
+
+define void @mscatter_v8f64(<8 x double> %val, <8 x double*> %ptrs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a0, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a0, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i8_v8f64(<8 x double> %val, double* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i8_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v12
+; RV32-NEXT:    vsll.vi v26, v26, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i8_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i8> %idxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i8_v8f64(<8 x double> %val, double* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i8_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf8 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i8_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf8 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i8> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i8_v8f64(<8 x double> %val, double* %base, <8 x i8> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i8_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf8 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i8_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf8 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i8> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i16_v8f64(<8 x double> %val, double* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i16_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v12
+; RV32-NEXT:    vsll.vi v26, v26, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i16_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i16> %idxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, double* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i16_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i16_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf4 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i16> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i16_v8f64(<8 x double> %val, double* %base, <8 x i16> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i16_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i16_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf4 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i16> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8i32_v8f64(<8 x double> %val, double* %base, <8 x i32> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8i32_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; RV32-NEXT:    vsll.vi v26, v12, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v26, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8i32_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i32> %idxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_v8i32_v8f64(<8 x double> %val, double* %base, <8 x i32> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_v8i32_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_v8i32_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsext.vf2 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <8 x i32> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_v8i32_v8f64(<8 x double> %val, double* %base, <8 x i32> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_v8i32_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vzext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_v8i32_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vzext.vf2 v28, v12
+; RV64-NEXT:    vsll.vi v28, v28, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <8 x i32> %idxs to <8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %eidxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_v8f64(<8 x double> %val, double* %base, <8 x i64> %idxs, <8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v12, 3
+; RV32-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; RV64-NEXT:    vsll.vi v28, v12, 3
+; RV64-NEXT:    vsetivli a1, 4, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v28, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <8 x i64> %idxs
+  call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> %val, <8 x double*> %ptrs, i32 8, <8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8>, <16 x i8*>, i32, <16 x i1>)
+
+define void @mscatter_baseidx_v16i8(<16 x i8> %val, i8* %base, <16 x i8> %idxs, <16 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v16i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a1, 16, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v9
+; RV32-NEXT:    vsetivli a1, 8, e8,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v16i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 16, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v9
+; RV64-NEXT:    vsetivli a1, 8, e8,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <16 x i8> %idxs
+  call void @llvm.masked.scatter.v16i8.v16p0i8(<16 x i8> %val, <16 x i8*> %ptrs, i32 1, <16 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8>, <32 x i8*>, i32, <32 x i1>)
+
+define void @mscatter_baseidx_v32i8(<32 x i8> %val, i8* %base, <32 x i8> %idxs, <32 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_v32i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi a1, zero, 32
+; RV32-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v16, v10
+; RV32-NEXT:    vsetivli a1, 16, e8,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_v32i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a1, 16, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v10
+; RV64-NEXT:    vsetivli a1, 8, e8,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    vsetivli a1, 16, e8,m2,ta,mu
+; RV64-NEXT:    vslidedown.vi v26, v10, 16
+; RV64-NEXT:    vsetivli a1, 16, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v26
+; RV64-NEXT:    vsetivli a1, 16, e8,m2,ta,mu
+; RV64-NEXT:    vslidedown.vi v26, v8, 16
+; RV64-NEXT:    vsetivli a1, 2, e8,m1,ta,mu
+; RV64-NEXT:    vslidedown.vi v0, v0, 2
+; RV64-NEXT:    vsetivli a1, 8, e8,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v26, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <32 x i8> %idxs
+  call void @llvm.masked.scatter.v32i8.v32p0i8(<32 x i8> %val, <32 x i8*> %ptrs, i32 1, <32 x i1> %m)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..79ddde95226e5f6a4da60669f742179038569b51
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-fp.ll
@@ -0,0 +1,478 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v,+f,+d,+experimental-zfh -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v,+f,+d,+experimental-zfh -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+
+define void @masked_store_v1f16(<1 x half>* %val_ptr, <1 x half>* %a, <1 x half>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a2)
+; CHECK-NEXT:    vle16.v v26, (a0)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vse16.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <1 x half>, <1 x half>* %m_ptr
+  %mask = fcmp oeq <1 x half> %m, zeroinitializer
+  %val = load <1 x half>, <1 x half>* %val_ptr
+  call void @llvm.masked.store.v1f16.p0v1f16(<1 x half> %val, <1 x half>* %a, i32 8, <1 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v1f16.p0v1f16(<1 x half>, <1 x half>*, i32, <1 x i1>)
+
+define void @masked_store_v1f32(<1 x float>* %val_ptr, <1 x float>* %a, <1 x float>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a2)
+; CHECK-NEXT:    vle32.v v26, (a0)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vse32.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <1 x float>, <1 x float>* %m_ptr
+  %mask = fcmp oeq <1 x float> %m, zeroinitializer
+  %val = load <1 x float>, <1 x float>* %val_ptr
+  call void @llvm.masked.store.v1f32.p0v1f32(<1 x float> %val, <1 x float>* %a, i32 8, <1 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v1f32.p0v1f32(<1 x float>, <1 x float>*, i32, <1 x i1>)
+
+define void @masked_store_v1f64(<1 x double>* %val_ptr, <1 x double>* %a, <1 x double>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 1, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a2)
+; RV32-NEXT:    vle64.v v26, (a0)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v25, ft0
+; RV32-NEXT:    vse64.v v26, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 1, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a2)
+; RV64-NEXT:    vle64.v v26, (a0)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v25, ft0
+; RV64-NEXT:    vse64.v v26, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <1 x double>, <1 x double>* %m_ptr
+  %mask = fcmp oeq <1 x double> %m, zeroinitializer
+  %val = load <1 x double>, <1 x double>* %val_ptr
+  call void @llvm.masked.store.v1f64.p0v1f64(<1 x double> %val, <1 x double>* %a, i32 8, <1 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v1f64.p0v1f64(<1 x double>, <1 x double>*, i32, <1 x i1>)
+
+define void @masked_store_v2f16(<2 x half>* %val_ptr, <2 x half>* %a, <2 x half>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a2)
+; CHECK-NEXT:    vle16.v v26, (a0)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vse16.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <2 x half>, <2 x half>* %m_ptr
+  %mask = fcmp oeq <2 x half> %m, zeroinitializer
+  %val = load <2 x half>, <2 x half>* %val_ptr
+  call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %val, <2 x half>* %a, i32 8, <2 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v2f16.p0v2f16(<2 x half>, <2 x half>*, i32, <2 x i1>)
+
+define void @masked_store_v2f32(<2 x float>* %val_ptr, <2 x float>* %a, <2 x float>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a2)
+; CHECK-NEXT:    vle32.v v26, (a0)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vse32.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <2 x float>, <2 x float>* %m_ptr
+  %mask = fcmp oeq <2 x float> %m, zeroinitializer
+  %val = load <2 x float>, <2 x float>* %val_ptr
+  call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %a, i32 8, <2 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v2f32.p0v2f32(<2 x float>, <2 x float>*, i32, <2 x i1>)
+
+define void @masked_store_v2f64(<2 x double>* %val_ptr, <2 x double>* %a, <2 x double>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a2)
+; RV32-NEXT:    vle64.v v26, (a0)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v25, ft0
+; RV32-NEXT:    vse64.v v26, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a2)
+; RV64-NEXT:    vle64.v v26, (a0)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v25, ft0
+; RV64-NEXT:    vse64.v v26, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <2 x double>, <2 x double>* %m_ptr
+  %mask = fcmp oeq <2 x double> %m, zeroinitializer
+  %val = load <2 x double>, <2 x double>* %val_ptr
+  call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %val, <2 x double>* %a, i32 8, <2 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v2f64.p0v2f64(<2 x double>, <2 x double>*, i32, <2 x i1>)
+
+define void @masked_store_v4f16(<4 x half>* %val_ptr, <4 x half>* %a, <4 x half>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a2)
+; CHECK-NEXT:    vle16.v v26, (a0)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vse16.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <4 x half>, <4 x half>* %m_ptr
+  %mask = fcmp oeq <4 x half> %m, zeroinitializer
+  %val = load <4 x half>, <4 x half>* %val_ptr
+  call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %val, <4 x half>* %a, i32 8, <4 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v4f16.p0v4f16(<4 x half>, <4 x half>*, i32, <4 x i1>)
+
+define void @masked_store_v4f32(<4 x float>* %val_ptr, <4 x float>* %a, <4 x float>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a2)
+; CHECK-NEXT:    vle32.v v26, (a0)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vse32.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <4 x float>, <4 x float>* %m_ptr
+  %mask = fcmp oeq <4 x float> %m, zeroinitializer
+  %val = load <4 x float>, <4 x float>* %val_ptr
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %val, <4 x float>* %a, i32 8, <4 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32, <4 x i1>)
+
+define void @masked_store_v4f64(<4 x double>* %val_ptr, <4 x double>* %a, <4 x double>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
+; RV32-NEXT:    vle64.v v26, (a2)
+; RV32-NEXT:    vle64.v v28, (a0)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v26, ft0
+; RV32-NEXT:    vse64.v v28, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
+; RV64-NEXT:    vle64.v v26, (a2)
+; RV64-NEXT:    vle64.v v28, (a0)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v26, ft0
+; RV64-NEXT:    vse64.v v28, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <4 x double>, <4 x double>* %m_ptr
+  %mask = fcmp oeq <4 x double> %m, zeroinitializer
+  %val = load <4 x double>, <4 x double>* %val_ptr
+  call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> %val, <4 x double>* %a, i32 8, <4 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v4f64.p0v4f64(<4 x double>, <4 x double>*, i32, <4 x i1>)
+
+define void @masked_store_v8f16(<8 x half>* %val_ptr, <8 x half>* %a, <8 x half>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a2)
+; CHECK-NEXT:    vle16.v v26, (a0)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v25, ft0
+; CHECK-NEXT:    vse16.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <8 x half>, <8 x half>* %m_ptr
+  %mask = fcmp oeq <8 x half> %m, zeroinitializer
+  %val = load <8 x half>, <8 x half>* %val_ptr
+  call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %val, <8 x half>* %a, i32 8, <8 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v8f16.p0v8f16(<8 x half>, <8 x half>*, i32, <8 x i1>)
+
+define void @masked_store_v8f32(<8 x float>* %val_ptr, <8 x float>* %a, <8 x float>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e32,m2,ta,mu
+; CHECK-NEXT:    vle32.v v26, (a2)
+; CHECK-NEXT:    vle32.v v28, (a0)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v26, ft0
+; CHECK-NEXT:    vse32.v v28, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <8 x float>, <8 x float>* %m_ptr
+  %mask = fcmp oeq <8 x float> %m, zeroinitializer
+  %val = load <8 x float>, <8 x float>* %val_ptr
+  call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> %val, <8 x float>* %a, i32 8, <8 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v8f32.p0v8f32(<8 x float>, <8 x float>*, i32, <8 x i1>)
+
+define void @masked_store_v8f64(<8 x double>* %val_ptr, <8 x double>* %a, <8 x double>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 8, e64,m4,ta,mu
+; RV32-NEXT:    vle64.v v28, (a2)
+; RV32-NEXT:    vle64.v v8, (a0)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v28, ft0
+; RV32-NEXT:    vse64.v v8, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 8, e64,m4,ta,mu
+; RV64-NEXT:    vle64.v v28, (a2)
+; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v28, ft0
+; RV64-NEXT:    vse64.v v8, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <8 x double>, <8 x double>* %m_ptr
+  %mask = fcmp oeq <8 x double> %m, zeroinitializer
+  %val = load <8 x double>, <8 x double>* %val_ptr
+  call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> %val, <8 x double>* %a, i32 8, <8 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v8f64.p0v8f64(<8 x double>, <8 x double>*, i32, <8 x i1>)
+
+define void @masked_store_v16f16(<16 x half>* %val_ptr, <16 x half>* %a, <16 x half>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e16,m2,ta,mu
+; CHECK-NEXT:    vle16.v v26, (a2)
+; CHECK-NEXT:    vle16.v v28, (a0)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v26, ft0
+; CHECK-NEXT:    vse16.v v28, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <16 x half>, <16 x half>* %m_ptr
+  %mask = fcmp oeq <16 x half> %m, zeroinitializer
+  %val = load <16 x half>, <16 x half>* %val_ptr
+  call void @llvm.masked.store.v16f16.p0v16f16(<16 x half> %val, <16 x half>* %a, i32 8, <16 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v16f16.p0v16f16(<16 x half>, <16 x half>*, i32, <16 x i1>)
+
+define void @masked_store_v16f32(<16 x float>* %val_ptr, <16 x float>* %a, <16 x float>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e32,m4,ta,mu
+; CHECK-NEXT:    vle32.v v28, (a2)
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v28, ft0
+; CHECK-NEXT:    vse32.v v8, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <16 x float>, <16 x float>* %m_ptr
+  %mask = fcmp oeq <16 x float> %m, zeroinitializer
+  %val = load <16 x float>, <16 x float>* %val_ptr
+  call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> %val, <16 x float>* %a, i32 8, <16 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v16f32.p0v16f32(<16 x float>, <16 x float>*, i32, <16 x i1>)
+
+define void @masked_store_v16f64(<16 x double>* %val_ptr, <16 x double>* %a, <16 x double>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v16f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV32-NEXT:    vle64.v v8, (a2)
+; RV32-NEXT:    vle64.v v16, (a0)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v0, v8, ft0
+; RV32-NEXT:    vse64.v v16, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v16f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV64-NEXT:    vle64.v v8, (a2)
+; RV64-NEXT:    vle64.v v16, (a0)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v0, v8, ft0
+; RV64-NEXT:    vse64.v v16, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <16 x double>, <16 x double>* %m_ptr
+  %mask = fcmp oeq <16 x double> %m, zeroinitializer
+  %val = load <16 x double>, <16 x double>* %val_ptr
+  call void @llvm.masked.store.v16f64.p0v16f64(<16 x double> %val, <16 x double>* %a, i32 8, <16 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v16f64.p0v16f64(<16 x double>, <16 x double>*, i32, <16 x i1>)
+
+define void @masked_store_v32f16(<32 x half>* %val_ptr, <32 x half>* %a, <32 x half>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e16,m4,ta,mu
+; CHECK-NEXT:    vle16.v v28, (a2)
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v28, ft0
+; CHECK-NEXT:    vse16.v v8, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <32 x half>, <32 x half>* %m_ptr
+  %mask = fcmp oeq <32 x half> %m, zeroinitializer
+  %val = load <32 x half>, <32 x half>* %val_ptr
+  call void @llvm.masked.store.v32f16.p0v32f16(<32 x half> %val, <32 x half>* %a, i32 8, <32 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v32f16.p0v32f16(<32 x half>, <32 x half>*, i32, <32 x i1>)
+
+define void @masked_store_v32f32(<32 x float>* %val_ptr, <32 x float>* %a, <32 x float>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v32f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e32,m8,ta,mu
+; CHECK-NEXT:    vle32.v v8, (a2)
+; CHECK-NEXT:    vle32.v v16, (a0)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v8, ft0
+; CHECK-NEXT:    vse32.v v16, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <32 x float>, <32 x float>* %m_ptr
+  %mask = fcmp oeq <32 x float> %m, zeroinitializer
+  %val = load <32 x float>, <32 x float>* %val_ptr
+  call void @llvm.masked.store.v32f32.p0v32f32(<32 x float> %val, <32 x float>* %a, i32 8, <32 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v32f32.p0v32f32(<32 x float>, <32 x float>*, i32, <32 x i1>)
+
+define void @masked_store_v32f64(<32 x double>* %val_ptr, <32 x double>* %a, <32 x double>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v32f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV32-NEXT:    addi a3, a2, 128
+; RV32-NEXT:    vle64.v v8, (a3)
+; RV32-NEXT:    vle64.v v16, (a2)
+; RV32-NEXT:    fcvt.d.w ft0, zero
+; RV32-NEXT:    vmfeq.vf v1, v8, ft0
+; RV32-NEXT:    vle64.v v8, (a0)
+; RV32-NEXT:    addi a0, a0, 128
+; RV32-NEXT:    vle64.v v24, (a0)
+; RV32-NEXT:    vmfeq.vf v0, v16, ft0
+; RV32-NEXT:    vse64.v v8, (a1), v0.t
+; RV32-NEXT:    addi a0, a1, 128
+; RV32-NEXT:    vmv1r.v v0, v1
+; RV32-NEXT:    vse64.v v24, (a0), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v32f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV64-NEXT:    addi a3, a2, 128
+; RV64-NEXT:    vle64.v v8, (a3)
+; RV64-NEXT:    vle64.v v16, (a2)
+; RV64-NEXT:    fmv.d.x ft0, zero
+; RV64-NEXT:    vmfeq.vf v1, v8, ft0
+; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    addi a0, a0, 128
+; RV64-NEXT:    vle64.v v24, (a0)
+; RV64-NEXT:    vmfeq.vf v0, v16, ft0
+; RV64-NEXT:    vse64.v v8, (a1), v0.t
+; RV64-NEXT:    addi a0, a1, 128
+; RV64-NEXT:    vmv1r.v v0, v1
+; RV64-NEXT:    vse64.v v24, (a0), v0.t
+; RV64-NEXT:    ret
+  %m = load <32 x double>, <32 x double>* %m_ptr
+  %mask = fcmp oeq <32 x double> %m, zeroinitializer
+  %val = load <32 x double>, <32 x double>* %val_ptr
+  call void @llvm.masked.store.v32f32.p0v32f64(<32 x double> %val, <32 x double>* %a, i32 8, <32 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v32f32.p0v32f64(<32 x double>, <32 x double>*, i32, <32 x i1>)
+
+define void @masked_store_v64f16(<64 x half>* %val_ptr, <64 x half>* %a, <64 x half>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v64f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 64
+; CHECK-NEXT:    vsetvli a3, a3, e16,m8,ta,mu
+; CHECK-NEXT:    vle16.v v8, (a2)
+; CHECK-NEXT:    vle16.v v16, (a0)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v0, v8, ft0
+; CHECK-NEXT:    vse16.v v16, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <64 x half>, <64 x half>* %m_ptr
+  %mask = fcmp oeq <64 x half> %m, zeroinitializer
+  %val = load <64 x half>, <64 x half>* %val_ptr
+  call void @llvm.masked.store.v64f16.p0v64f16(<64 x half> %val, <64 x half>* %a, i32 8, <64 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v64f16.p0v64f16(<64 x half>, <64 x half>*, i32, <64 x i1>)
+
+define void @masked_store_v64f32(<64 x float>* %val_ptr, <64 x float>* %a, <64 x float>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v64f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e32,m8,ta,mu
+; CHECK-NEXT:    addi a3, a2, 128
+; CHECK-NEXT:    vle32.v v8, (a3)
+; CHECK-NEXT:    vle32.v v16, (a2)
+; CHECK-NEXT:    fmv.w.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v1, v8, ft0
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    addi a0, a0, 128
+; CHECK-NEXT:    vle32.v v24, (a0)
+; CHECK-NEXT:    vmfeq.vf v0, v16, ft0
+; CHECK-NEXT:    vse32.v v8, (a1), v0.t
+; CHECK-NEXT:    addi a0, a1, 128
+; CHECK-NEXT:    vmv1r.v v0, v1
+; CHECK-NEXT:    vse32.v v24, (a0), v0.t
+; CHECK-NEXT:    ret
+  %m = load <64 x float>, <64 x float>* %m_ptr
+  %mask = fcmp oeq <64 x float> %m, zeroinitializer
+  %val = load <64 x float>, <64 x float>* %val_ptr
+  call void @llvm.masked.store.v64f16.p0v64f32(<64 x float> %val, <64 x float>* %a, i32 8, <64 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v64f16.p0v64f32(<64 x float>, <64 x float>*, i32, <64 x i1>)
+
+define void @masked_store_v128f16(<128 x half>* %val_ptr, <128 x half>* %a, <128 x half>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v128f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 64
+; CHECK-NEXT:    vsetvli a3, a3, e16,m8,ta,mu
+; CHECK-NEXT:    addi a3, a2, 128
+; CHECK-NEXT:    vle16.v v8, (a3)
+; CHECK-NEXT:    vle16.v v16, (a2)
+; CHECK-NEXT:    fmv.h.x ft0, zero
+; CHECK-NEXT:    vmfeq.vf v1, v8, ft0
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    addi a0, a0, 128
+; CHECK-NEXT:    vle16.v v24, (a0)
+; CHECK-NEXT:    vmfeq.vf v0, v16, ft0
+; CHECK-NEXT:    vse16.v v8, (a1), v0.t
+; CHECK-NEXT:    addi a0, a1, 128
+; CHECK-NEXT:    vmv1r.v v0, v1
+; CHECK-NEXT:    vse16.v v24, (a0), v0.t
+; CHECK-NEXT:    ret
+  %m = load <128 x half>, <128 x half>* %m_ptr
+  %mask = fcmp oeq <128 x half> %m, zeroinitializer
+  %val = load <128 x half>, <128 x half>* %val_ptr
+  call void @llvm.masked.store.v128f16.p0v128f16(<128 x half> %val, <128 x half>* %a, i32 8, <128 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v128f16.p0v128f16(<128 x half>, <128 x half>*, i32, <128 x i1>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-int.ll
new file mode 100644
index 0000000000000000000000000000000000000000..04798df3bb3df4bc2a00050880f41bad6b0223f0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-store-int.ll
@@ -0,0 +1,658 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+
+define void @masked_store_v1i8(<1 x i8>* %val_ptr, <1 x i8>* %a, <1 x i8>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v1i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a2)
+; CHECK-NEXT:    vle8.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse8.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <1 x i8>, <1 x i8>* %m_ptr
+  %mask = icmp eq <1 x i8> %m, zeroinitializer
+  %val = load <1 x i8>, <1 x i8>* %val_ptr
+  call void @llvm.masked.store.v1i8.p0v1i8(<1 x i8> %val, <1 x i8>* %a, i32 8, <1 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v1i8.p0v1i8(<1 x i8>, <1 x i8>*, i32, <1 x i1>)
+
+define void @masked_store_v1i16(<1 x i16>* %val_ptr, <1 x i16>* %a, <1 x i16>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v1i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a2)
+; CHECK-NEXT:    vle16.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse16.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <1 x i16>, <1 x i16>* %m_ptr
+  %mask = icmp eq <1 x i16> %m, zeroinitializer
+  %val = load <1 x i16>, <1 x i16>* %val_ptr
+  call void @llvm.masked.store.v1i16.p0v1i16(<1 x i16> %val, <1 x i16>* %a, i32 8, <1 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v1i16.p0v1i16(<1 x i16>, <1 x i16>*, i32, <1 x i1>)
+
+define void @masked_store_v1i32(<1 x i32>* %val_ptr, <1 x i32>* %a, <1 x i32>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v1i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 1, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a2)
+; CHECK-NEXT:    vle32.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse32.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <1 x i32>, <1 x i32>* %m_ptr
+  %mask = icmp eq <1 x i32> %m, zeroinitializer
+  %val = load <1 x i32>, <1 x i32>* %val_ptr
+  call void @llvm.masked.store.v1i32.p0v1i32(<1 x i32> %val, <1 x i32>* %a, i32 8, <1 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v1i32.p0v1i32(<1 x i32>, <1 x i32>*, i32, <1 x i1>)
+
+define void @masked_store_v1i64(<1 x i64>* %val_ptr, <1 x i64>* %a, <1 x i64>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v1i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 1, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a2)
+; RV32-NEXT:    vsetivli a2, 2, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v26, 0
+; RV32-NEXT:    vsetivli a2, 1, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v27, (a0)
+; RV32-NEXT:    vmseq.vv v0, v25, v26
+; RV32-NEXT:    vse64.v v27, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v1i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 1, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a2)
+; RV64-NEXT:    vle64.v v26, (a0)
+; RV64-NEXT:    vmseq.vi v0, v25, 0
+; RV64-NEXT:    vse64.v v26, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <1 x i64>, <1 x i64>* %m_ptr
+  %mask = icmp eq <1 x i64> %m, zeroinitializer
+  %val = load <1 x i64>, <1 x i64>* %val_ptr
+  call void @llvm.masked.store.v1i64.p0v1i64(<1 x i64> %val, <1 x i64>* %a, i32 8, <1 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v1i64.p0v1i64(<1 x i64>, <1 x i64>*, i32, <1 x i1>)
+
+define void @masked_store_v2i8(<2 x i8>* %val_ptr, <2 x i8>* %a, <2 x i8>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a2)
+; CHECK-NEXT:    vle8.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse8.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <2 x i8>, <2 x i8>* %m_ptr
+  %mask = icmp eq <2 x i8> %m, zeroinitializer
+  %val = load <2 x i8>, <2 x i8>* %val_ptr
+  call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %val, <2 x i8>* %a, i32 8, <2 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v2i8.p0v2i8(<2 x i8>, <2 x i8>*, i32, <2 x i1>)
+
+define void @masked_store_v2i16(<2 x i16>* %val_ptr, <2 x i16>* %a, <2 x i16>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a2)
+; CHECK-NEXT:    vle16.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse16.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <2 x i16>, <2 x i16>* %m_ptr
+  %mask = icmp eq <2 x i16> %m, zeroinitializer
+  %val = load <2 x i16>, <2 x i16>* %val_ptr
+  call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %val, <2 x i16>* %a, i32 8, <2 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v2i16.p0v2i16(<2 x i16>, <2 x i16>*, i32, <2 x i1>)
+
+define void @masked_store_v2i32(<2 x i32>* %val_ptr, <2 x i32>* %a, <2 x i32>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 2, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a2)
+; CHECK-NEXT:    vle32.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse32.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <2 x i32>, <2 x i32>* %m_ptr
+  %mask = icmp eq <2 x i32> %m, zeroinitializer
+  %val = load <2 x i32>, <2 x i32>* %val_ptr
+  call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %a, i32 8, <2 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v2i32.p0v2i32(<2 x i32>, <2 x i32>*, i32, <2 x i1>)
+
+define void @masked_store_v2i64(<2 x i64>* %val_ptr, <2 x i64>* %a, <2 x i64>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v25, (a2)
+; RV32-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; RV32-NEXT:    vmv.v.i v26, 0
+; RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; RV32-NEXT:    vle64.v v27, (a0)
+; RV32-NEXT:    vmseq.vv v0, v25, v26
+; RV32-NEXT:    vse64.v v27, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; RV64-NEXT:    vle64.v v25, (a2)
+; RV64-NEXT:    vle64.v v26, (a0)
+; RV64-NEXT:    vmseq.vi v0, v25, 0
+; RV64-NEXT:    vse64.v v26, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <2 x i64>, <2 x i64>* %m_ptr
+  %mask = icmp eq <2 x i64> %m, zeroinitializer
+  %val = load <2 x i64>, <2 x i64>* %val_ptr
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %val, <2 x i64>* %a, i32 8, <2 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v2i64.p0v2i64(<2 x i64>, <2 x i64>*, i32, <2 x i1>)
+
+define void @masked_store_v4i8(<4 x i8>* %val_ptr, <4 x i8>* %a, <4 x i8>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a2)
+; CHECK-NEXT:    vle8.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse8.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <4 x i8>, <4 x i8>* %m_ptr
+  %mask = icmp eq <4 x i8> %m, zeroinitializer
+  %val = load <4 x i8>, <4 x i8>* %val_ptr
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %val, <4 x i8>* %a, i32 8, <4 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32, <4 x i1>)
+
+define void @masked_store_v4i16(<4 x i16>* %val_ptr, <4 x i16>* %a, <4 x i16>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a2)
+; CHECK-NEXT:    vle16.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse16.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <4 x i16>, <4 x i16>* %m_ptr
+  %mask = icmp eq <4 x i16> %m, zeroinitializer
+  %val = load <4 x i16>, <4 x i16>* %val_ptr
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %val, <4 x i16>* %a, i32 8, <4 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32, <4 x i1>)
+
+define void @masked_store_v4i32(<4 x i32>* %val_ptr, <4 x i32>* %a, <4 x i32>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a2)
+; CHECK-NEXT:    vle32.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse32.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <4 x i32>, <4 x i32>* %m_ptr
+  %mask = icmp eq <4 x i32> %m, zeroinitializer
+  %val = load <4 x i32>, <4 x i32>* %val_ptr
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %a, i32 8, <4 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+
+define void @masked_store_v4i64(<4 x i64>* %val_ptr, <4 x i64>* %a, <4 x i64>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
+; RV32-NEXT:    vle64.v v26, (a2)
+; RV32-NEXT:    vsetivli a2, 8, e32,m2,ta,mu
+; RV32-NEXT:    vmv.v.i v28, 0
+; RV32-NEXT:    vsetivli a2, 4, e64,m2,ta,mu
+; RV32-NEXT:    vle64.v v30, (a0)
+; RV32-NEXT:    vmseq.vv v0, v26, v28
+; RV32-NEXT:    vse64.v v30, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 4, e64,m2,ta,mu
+; RV64-NEXT:    vle64.v v26, (a2)
+; RV64-NEXT:    vle64.v v28, (a0)
+; RV64-NEXT:    vmseq.vi v0, v26, 0
+; RV64-NEXT:    vse64.v v28, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <4 x i64>, <4 x i64>* %m_ptr
+  %mask = icmp eq <4 x i64> %m, zeroinitializer
+  %val = load <4 x i64>, <4 x i64>* %val_ptr
+  call void @llvm.masked.store.v4i64.p0v4i64(<4 x i64> %val, <4 x i64>* %a, i32 8, <4 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v4i64.p0v4i64(<4 x i64>, <4 x i64>*, i32, <4 x i1>)
+
+define void @masked_store_v8i8(<8 x i8>* %val_ptr, <8 x i8>* %a, <8 x i8>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a2)
+; CHECK-NEXT:    vle8.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse8.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <8 x i8>, <8 x i8>* %m_ptr
+  %mask = icmp eq <8 x i8> %m, zeroinitializer
+  %val = load <8 x i8>, <8 x i8>* %val_ptr
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %val, <8 x i8>* %a, i32 8, <8 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32, <8 x i1>)
+
+define void @masked_store_v8i16(<8 x i16>* %val_ptr, <8 x i16>* %a, <8 x i16>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a2)
+; CHECK-NEXT:    vle16.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse16.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <8 x i16>, <8 x i16>* %m_ptr
+  %mask = icmp eq <8 x i16> %m, zeroinitializer
+  %val = load <8 x i16>, <8 x i16>* %val_ptr
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %val, <8 x i16>* %a, i32 8, <8 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>)
+
+define void @masked_store_v8i32(<8 x i32>* %val_ptr, <8 x i32>* %a, <8 x i32>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 8, e32,m2,ta,mu
+; CHECK-NEXT:    vle32.v v26, (a2)
+; CHECK-NEXT:    vle32.v v28, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v26, 0
+; CHECK-NEXT:    vse32.v v28, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <8 x i32>, <8 x i32>* %m_ptr
+  %mask = icmp eq <8 x i32> %m, zeroinitializer
+  %val = load <8 x i32>, <8 x i32>* %val_ptr
+  call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> %val, <8 x i32>* %a, i32 8, <8 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, i32, <8 x i1>)
+
+define void @masked_store_v8i64(<8 x i64>* %val_ptr, <8 x i64>* %a, <8 x i64>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 8, e64,m4,ta,mu
+; RV32-NEXT:    vle64.v v28, (a2)
+; RV32-NEXT:    vsetivli a2, 16, e32,m4,ta,mu
+; RV32-NEXT:    vmv.v.i v8, 0
+; RV32-NEXT:    vsetivli a2, 8, e64,m4,ta,mu
+; RV32-NEXT:    vle64.v v12, (a0)
+; RV32-NEXT:    vmseq.vv v0, v28, v8
+; RV32-NEXT:    vse64.v v12, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 8, e64,m4,ta,mu
+; RV64-NEXT:    vle64.v v28, (a2)
+; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    vmseq.vi v0, v28, 0
+; RV64-NEXT:    vse64.v v8, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <8 x i64>, <8 x i64>* %m_ptr
+  %mask = icmp eq <8 x i64> %m, zeroinitializer
+  %val = load <8 x i64>, <8 x i64>* %val_ptr
+  call void @llvm.masked.store.v8i64.p0v8i64(<8 x i64> %val, <8 x i64>* %a, i32 8, <8 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v8i64.p0v8i64(<8 x i64>, <8 x i64>*, i32, <8 x i1>)
+
+define void @masked_store_v16i8(<16 x i8>* %val_ptr, <16 x i8>* %a, <16 x i8>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a2)
+; CHECK-NEXT:    vle8.v v26, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    vse8.v v26, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <16 x i8>, <16 x i8>* %m_ptr
+  %mask = icmp eq <16 x i8> %m, zeroinitializer
+  %val = load <16 x i8>, <16 x i8>* %val_ptr
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %val, <16 x i8>* %a, i32 8, <16 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+
+define void @masked_store_v16i16(<16 x i16>* %val_ptr, <16 x i16>* %a, <16 x i16>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e16,m2,ta,mu
+; CHECK-NEXT:    vle16.v v26, (a2)
+; CHECK-NEXT:    vle16.v v28, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v26, 0
+; CHECK-NEXT:    vse16.v v28, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <16 x i16>, <16 x i16>* %m_ptr
+  %mask = icmp eq <16 x i16> %m, zeroinitializer
+  %val = load <16 x i16>, <16 x i16>* %val_ptr
+  call void @llvm.masked.store.v16i16.p0v16i16(<16 x i16> %val, <16 x i16>* %a, i32 8, <16 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v16i16.p0v16i16(<16 x i16>, <16 x i16>*, i32, <16 x i1>)
+
+define void @masked_store_v16i32(<16 x i32>* %val_ptr, <16 x i32>* %a, <16 x i32>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v16i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a3, 16, e32,m4,ta,mu
+; CHECK-NEXT:    vle32.v v28, (a2)
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v28, 0
+; CHECK-NEXT:    vse32.v v8, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <16 x i32>, <16 x i32>* %m_ptr
+  %mask = icmp eq <16 x i32> %m, zeroinitializer
+  %val = load <16 x i32>, <16 x i32>* %val_ptr
+  call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> %val, <16 x i32>* %a, i32 8, <16 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v16i32.p0v16i32(<16 x i32>, <16 x i32>*, i32, <16 x i1>)
+
+define void @masked_store_v16i64(<16 x i64>* %val_ptr, <16 x i64>* %a, <16 x i64>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v16i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV32-NEXT:    vle64.v v8, (a2)
+; RV32-NEXT:    addi a2, zero, 32
+; RV32-NEXT:    vsetvli a2, a2, e32,m8,ta,mu
+; RV32-NEXT:    vmv.v.i v16, 0
+; RV32-NEXT:    vsetivli a2, 16, e64,m8,ta,mu
+; RV32-NEXT:    vle64.v v24, (a0)
+; RV32-NEXT:    vmseq.vv v0, v8, v16
+; RV32-NEXT:    vse64.v v24, (a1), v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v16i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV64-NEXT:    vle64.v v8, (a2)
+; RV64-NEXT:    vle64.v v16, (a0)
+; RV64-NEXT:    vmseq.vi v0, v8, 0
+; RV64-NEXT:    vse64.v v16, (a1), v0.t
+; RV64-NEXT:    ret
+  %m = load <16 x i64>, <16 x i64>* %m_ptr
+  %mask = icmp eq <16 x i64> %m, zeroinitializer
+  %val = load <16 x i64>, <16 x i64>* %val_ptr
+  call void @llvm.masked.store.v16i64.p0v16i64(<16 x i64> %val, <16 x i64>* %a, i32 8, <16 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v16i64.p0v16i64(<16 x i64>, <16 x i64>*, i32, <16 x i1>)
+
+define void @masked_store_v32i8(<32 x i8>* %val_ptr, <32 x i8>* %a, <32 x i8>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e8,m2,ta,mu
+; CHECK-NEXT:    vle8.v v26, (a2)
+; CHECK-NEXT:    vle8.v v28, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v26, 0
+; CHECK-NEXT:    vse8.v v28, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <32 x i8>, <32 x i8>* %m_ptr
+  %mask = icmp eq <32 x i8> %m, zeroinitializer
+  %val = load <32 x i8>, <32 x i8>* %val_ptr
+  call void @llvm.masked.store.v32i8.p0v32i8(<32 x i8> %val, <32 x i8>* %a, i32 8, <32 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v32i8.p0v32i8(<32 x i8>, <32 x i8>*, i32, <32 x i1>)
+
+define void @masked_store_v32i16(<32 x i16>* %val_ptr, <32 x i16>* %a, <32 x i16>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v32i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e16,m4,ta,mu
+; CHECK-NEXT:    vle16.v v28, (a2)
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v28, 0
+; CHECK-NEXT:    vse16.v v8, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <32 x i16>, <32 x i16>* %m_ptr
+  %mask = icmp eq <32 x i16> %m, zeroinitializer
+  %val = load <32 x i16>, <32 x i16>* %val_ptr
+  call void @llvm.masked.store.v32i16.p0v32i16(<32 x i16> %val, <32 x i16>* %a, i32 8, <32 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v32i16.p0v32i16(<32 x i16>, <32 x i16>*, i32, <32 x i1>)
+
+define void @masked_store_v32i32(<32 x i32>* %val_ptr, <32 x i32>* %a, <32 x i32>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v32i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e32,m8,ta,mu
+; CHECK-NEXT:    vle32.v v8, (a2)
+; CHECK-NEXT:    vle32.v v16, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vse32.v v16, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <32 x i32>, <32 x i32>* %m_ptr
+  %mask = icmp eq <32 x i32> %m, zeroinitializer
+  %val = load <32 x i32>, <32 x i32>* %val_ptr
+  call void @llvm.masked.store.v32i32.p0v32i32(<32 x i32> %val, <32 x i32>* %a, i32 8, <32 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v32i32.p0v32i32(<32 x i32>, <32 x i32>*, i32, <32 x i1>)
+
+define void @masked_store_v32i64(<32 x i64>* %val_ptr, <32 x i64>* %a, <32 x i64>* %m_ptr) nounwind {
+; RV32-LABEL: masked_store_v32i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a3, a3, 4
+; RV32-NEXT:    sub sp, sp, a3
+; RV32-NEXT:    addi a3, a2, 128
+; RV32-NEXT:    vsetivli a4, 16, e64,m8,ta,mu
+; RV32-NEXT:    vle64.v v8, (a3)
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a3, a3, 3
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vle64.v v16, (a2)
+; RV32-NEXT:    addi a2, zero, 32
+; RV32-NEXT:    vsetvli a2, a2, e32,m8,ta,mu
+; RV32-NEXT:    vmv.v.i v24, 0
+; RV32-NEXT:    vsetivli a2, 16, e64,m8,ta,mu
+; RV32-NEXT:    vmseq.vv v1, v16, v24
+; RV32-NEXT:    addi a2, a0, 128
+; RV32-NEXT:    vle64.v v16, (a2)
+; RV32-NEXT:    vle64.v v8, (a0)
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 3
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl8re8.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vmseq.vv v0, v8, v24
+; RV32-NEXT:    addi a0, a1, 128
+; RV32-NEXT:    vse64.v v16, (a0), v0.t
+; RV32-NEXT:    vmv1r.v v0, v1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vl8re8.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vse64.v v8, (a1), v0.t
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: masked_store_v32i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli a3, 16, e64,m8,ta,mu
+; RV64-NEXT:    addi a3, a2, 128
+; RV64-NEXT:    vle64.v v8, (a3)
+; RV64-NEXT:    vle64.v v16, (a2)
+; RV64-NEXT:    vmseq.vi v1, v8, 0
+; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    addi a0, a0, 128
+; RV64-NEXT:    vle64.v v24, (a0)
+; RV64-NEXT:    vmseq.vi v0, v16, 0
+; RV64-NEXT:    vse64.v v8, (a1), v0.t
+; RV64-NEXT:    addi a0, a1, 128
+; RV64-NEXT:    vmv1r.v v0, v1
+; RV64-NEXT:    vse64.v v24, (a0), v0.t
+; RV64-NEXT:    ret
+  %m = load <32 x i64>, <32 x i64>* %m_ptr
+  %mask = icmp eq <32 x i64> %m, zeroinitializer
+  %val = load <32 x i64>, <32 x i64>* %val_ptr
+  call void @llvm.masked.store.v32i64.p0v32i64(<32 x i64> %val, <32 x i64>* %a, i32 8, <32 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v32i64.p0v32i64(<32 x i64>, <32 x i64>*, i32, <32 x i1>)
+
+define void @masked_store_v64i8(<64 x i8>* %val_ptr, <64 x i8>* %a, <64 x i8>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v64i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 64
+; CHECK-NEXT:    vsetvli a3, a3, e8,m4,ta,mu
+; CHECK-NEXT:    vle8.v v28, (a2)
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v28, 0
+; CHECK-NEXT:    vse8.v v8, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <64 x i8>, <64 x i8>* %m_ptr
+  %mask = icmp eq <64 x i8> %m, zeroinitializer
+  %val = load <64 x i8>, <64 x i8>* %val_ptr
+  call void @llvm.masked.store.v64i8.p0v64i8(<64 x i8> %val, <64 x i8>* %a, i32 8, <64 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v64i8.p0v64i8(<64 x i8>, <64 x i8>*, i32, <64 x i1>)
+
+define void @masked_store_v64i16(<64 x i16>* %val_ptr, <64 x i16>* %a, <64 x i16>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v64i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 64
+; CHECK-NEXT:    vsetvli a3, a3, e16,m8,ta,mu
+; CHECK-NEXT:    vle16.v v8, (a2)
+; CHECK-NEXT:    vle16.v v16, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vse16.v v16, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <64 x i16>, <64 x i16>* %m_ptr
+  %mask = icmp eq <64 x i16> %m, zeroinitializer
+  %val = load <64 x i16>, <64 x i16>* %val_ptr
+  call void @llvm.masked.store.v64i16.p0v64i16(<64 x i16> %val, <64 x i16>* %a, i32 8, <64 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v64i16.p0v64i16(<64 x i16>, <64 x i16>*, i32, <64 x i1>)
+
+define void @masked_store_v64i32(<64 x i32>* %val_ptr, <64 x i32>* %a, <64 x i32>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v64i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 32
+; CHECK-NEXT:    vsetvli a3, a3, e32,m8,ta,mu
+; CHECK-NEXT:    addi a3, a2, 128
+; CHECK-NEXT:    vle32.v v8, (a3)
+; CHECK-NEXT:    vle32.v v16, (a2)
+; CHECK-NEXT:    vmseq.vi v1, v8, 0
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    addi a0, a0, 128
+; CHECK-NEXT:    vle32.v v24, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v16, 0
+; CHECK-NEXT:    vse32.v v8, (a1), v0.t
+; CHECK-NEXT:    addi a0, a1, 128
+; CHECK-NEXT:    vmv1r.v v0, v1
+; CHECK-NEXT:    vse32.v v24, (a0), v0.t
+; CHECK-NEXT:    ret
+  %m = load <64 x i32>, <64 x i32>* %m_ptr
+  %mask = icmp eq <64 x i32> %m, zeroinitializer
+  %val = load <64 x i32>, <64 x i32>* %val_ptr
+  call void @llvm.masked.store.v64i32.p0v64i32(<64 x i32> %val, <64 x i32>* %a, i32 8, <64 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v64i32.p0v64i32(<64 x i32>, <64 x i32>*, i32, <64 x i1>)
+
+define void @masked_store_v128i8(<128 x i8>* %val_ptr, <128 x i8>* %a, <128 x i8>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v128i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 128
+; CHECK-NEXT:    vsetvli a3, a3, e8,m8,ta,mu
+; CHECK-NEXT:    vle8.v v8, (a2)
+; CHECK-NEXT:    vle8.v v16, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v8, 0
+; CHECK-NEXT:    vse8.v v16, (a1), v0.t
+; CHECK-NEXT:    ret
+  %m = load <128 x i8>, <128 x i8>* %m_ptr
+  %mask = icmp eq <128 x i8> %m, zeroinitializer
+  %val = load <128 x i8>, <128 x i8>* %val_ptr
+  call void @llvm.masked.store.v128i8.p0v128i8(<128 x i8> %val, <128 x i8>* %a, i32 8, <128 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v128i8.p0v128i8(<128 x i8>, <128 x i8>*, i32, <128 x i1>)
+
+define void @masked_store_v128i16(<128 x i16>* %val_ptr, <128 x i16>* %a, <128 x i16>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v128i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 64
+; CHECK-NEXT:    vsetvli a3, a3, e16,m8,ta,mu
+; CHECK-NEXT:    addi a3, a2, 128
+; CHECK-NEXT:    vle16.v v8, (a3)
+; CHECK-NEXT:    vle16.v v16, (a2)
+; CHECK-NEXT:    vmseq.vi v1, v8, 0
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    addi a0, a0, 128
+; CHECK-NEXT:    vle16.v v24, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v16, 0
+; CHECK-NEXT:    vse16.v v8, (a1), v0.t
+; CHECK-NEXT:    addi a0, a1, 128
+; CHECK-NEXT:    vmv1r.v v0, v1
+; CHECK-NEXT:    vse16.v v24, (a0), v0.t
+; CHECK-NEXT:    ret
+  %m = load <128 x i16>, <128 x i16>* %m_ptr
+  %mask = icmp eq <128 x i16> %m, zeroinitializer
+  %val = load <128 x i16>, <128 x i16>* %val_ptr
+  call void @llvm.masked.store.v128i16.p0v128i16(<128 x i16> %val, <128 x i16>* %a, i32 8, <128 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v128i16.p0v128i16(<128 x i16>, <128 x i16>*, i32, <128 x i1>)
+
+define void @masked_store_v256i8(<256 x i8>* %val_ptr, <256 x i8>* %a, <256 x i8>* %m_ptr) nounwind {
+; CHECK-LABEL: masked_store_v256i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a3, zero, 128
+; CHECK-NEXT:    vsetvli a3, a3, e8,m8,ta,mu
+; CHECK-NEXT:    addi a3, a2, 128
+; CHECK-NEXT:    vle8.v v8, (a3)
+; CHECK-NEXT:    vle8.v v16, (a2)
+; CHECK-NEXT:    vmseq.vi v1, v8, 0
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    addi a0, a0, 128
+; CHECK-NEXT:    vle8.v v24, (a0)
+; CHECK-NEXT:    vmseq.vi v0, v16, 0
+; CHECK-NEXT:    vse8.v v8, (a1), v0.t
+; CHECK-NEXT:    addi a0, a1, 128
+; CHECK-NEXT:    vmv1r.v v0, v1
+; CHECK-NEXT:    vse8.v v24, (a0), v0.t
+; CHECK-NEXT:    ret
+  %m = load <256 x i8>, <256 x i8>* %m_ptr
+  %mask = icmp eq <256 x i8> %m, zeroinitializer
+  %val = load <256 x i8>, <256 x i8>* %val_ptr
+  call void @llvm.masked.store.v256i8.p0v256i8(<256 x i8> %val, <256 x i8>* %a, i32 8, <256 x i1> %mask)
+  ret void
+}
+declare void @llvm.masked.store.v256i8.p0v256i8(<256 x i8>, <256 x i8>*, i32, <256 x i1>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-stepvector-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-stepvector-rv32.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7499425a73ddddb798a9c64edf9a15489f495219
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-stepvector-rv32.ll
@@ -0,0 +1,183 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2
+
+declare <2 x i8> @llvm.experimental.stepvector.v2i8()
+
+define <2 x i8> @stepvector_v2i8() {
+; CHECK-LABEL: stepvector_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 2, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <2 x i8> @llvm.experimental.stepvector.v2i8()
+  ret <2 x i8> %v
+}
+
+declare <4 x i8> @llvm.experimental.stepvector.v4i8()
+
+define <4 x i8> @stepvector_v4i8() {
+; CHECK-LABEL: stepvector_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 4, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <4 x i8> @llvm.experimental.stepvector.v4i8()
+  ret <4 x i8> %v
+}
+
+declare <8 x i8> @llvm.experimental.stepvector.v8i8()
+
+define <8 x i8> @stepvector_v8i8() {
+; CHECK-LABEL: stepvector_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <8 x i8> @llvm.experimental.stepvector.v8i8()
+  ret <8 x i8> %v
+}
+
+declare <16 x i8> @llvm.experimental.stepvector.v16i8()
+
+define <16 x i8> @stepvector_v16i8() {
+; CHECK-LABEL: stepvector_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 16, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <16 x i8> @llvm.experimental.stepvector.v16i8()
+  ret <16 x i8> %v
+}
+
+declare <2 x i16> @llvm.experimental.stepvector.v2i16()
+
+define <2 x i16> @stepvector_v2i16() {
+; CHECK-LABEL: stepvector_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 2, e16,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <2 x i16> @llvm.experimental.stepvector.v2i16()
+  ret <2 x i16> %v
+}
+
+declare <4 x i16> @llvm.experimental.stepvector.v4i16()
+
+define <4 x i16> @stepvector_v4i16() {
+; CHECK-LABEL: stepvector_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <4 x i16> @llvm.experimental.stepvector.v4i16()
+  ret <4 x i16> %v
+}
+
+declare <8 x i16> @llvm.experimental.stepvector.v8i16()
+
+define <8 x i16> @stepvector_v8i16() {
+; CHECK-LABEL: stepvector_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <8 x i16> @llvm.experimental.stepvector.v8i16()
+  ret <8 x i16> %v
+}
+
+declare <16 x i16> @llvm.experimental.stepvector.v16i16()
+
+define <16 x i16> @stepvector_v16i16() {
+; LMULMAX1-LABEL: stepvector_v16i16:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI7_0)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI7_0)
+; LMULMAX1-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; LMULMAX1-NEXT:    vle16.v v9, (a0)
+; LMULMAX1-NEXT:    vid.v v8
+; LMULMAX1-NEXT:    ret
+;
+; LMULMAX2-LABEL: stepvector_v16i16:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    vsetivli a0, 16, e16,m2,ta,mu
+; LMULMAX2-NEXT:    vid.v v8
+; LMULMAX2-NEXT:    ret
+  %v = call <16 x i16> @llvm.experimental.stepvector.v16i16()
+  ret <16 x i16> %v
+}
+
+declare <2 x i32> @llvm.experimental.stepvector.v2i32()
+
+define <2 x i32> @stepvector_v2i32() {
+; CHECK-LABEL: stepvector_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <2 x i32> @llvm.experimental.stepvector.v2i32()
+  ret <2 x i32> %v
+}
+
+declare <4 x i32> @llvm.experimental.stepvector.v4i32()
+
+define <4 x i32> @stepvector_v4i32() {
+; CHECK-LABEL: stepvector_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <4 x i32> @llvm.experimental.stepvector.v4i32()
+  ret <4 x i32> %v
+}
+
+declare <8 x i32> @llvm.experimental.stepvector.v8i32()
+
+define <8 x i32> @stepvector_v8i32() {
+; LMULMAX1-LABEL: stepvector_v8i32:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI10_0)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI10_0)
+; LMULMAX1-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-NEXT:    vle32.v v9, (a0)
+; LMULMAX1-NEXT:    vid.v v8
+; LMULMAX1-NEXT:    ret
+;
+; LMULMAX2-LABEL: stepvector_v8i32:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    vsetivli a0, 8, e32,m2,ta,mu
+; LMULMAX2-NEXT:    vid.v v8
+; LMULMAX2-NEXT:    ret
+  %v = call <8 x i32> @llvm.experimental.stepvector.v8i32()
+  ret <8 x i32> %v
+}
+
+declare <16 x i32> @llvm.experimental.stepvector.v16i32()
+
+define <16 x i32> @stepvector_v16i32() {
+; LMULMAX1-LABEL: stepvector_v16i32:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI11_0)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI11_0)
+; LMULMAX1-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-NEXT:    vle32.v v9, (a0)
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI11_1)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI11_1)
+; LMULMAX1-NEXT:    vle32.v v10, (a0)
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI11_2)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI11_2)
+; LMULMAX1-NEXT:    vle32.v v11, (a0)
+; LMULMAX1-NEXT:    vid.v v8
+; LMULMAX1-NEXT:    ret
+;
+; LMULMAX2-LABEL: stepvector_v16i32:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    lui a0, %hi(.LCPI11_0)
+; LMULMAX2-NEXT:    addi a0, a0, %lo(.LCPI11_0)
+; LMULMAX2-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-NEXT:    vle32.v v10, (a0)
+; LMULMAX2-NEXT:    vid.v v8
+; LMULMAX2-NEXT:    ret
+  %v = call <16 x i32> @llvm.experimental.stepvector.v16i32()
+  ret <16 x i32> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-stepvector-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-stepvector-rv64.ll
new file mode 100644
index 0000000000000000000000000000000000000000..23971a9c3be9840dac28d18c582e2426fafa2a11
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-stepvector-rv64.ll
@@ -0,0 +1,294 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=2 < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX2
+
+declare <2 x i8> @llvm.experimental.stepvector.v2i8()
+
+define <2 x i8> @stepvector_v2i8() {
+; CHECK-LABEL: stepvector_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 2, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <2 x i8> @llvm.experimental.stepvector.v2i8()
+  ret <2 x i8> %v
+}
+
+declare <4 x i8> @llvm.experimental.stepvector.v4i8()
+
+define <4 x i8> @stepvector_v4i8() {
+; CHECK-LABEL: stepvector_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 4, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <4 x i8> @llvm.experimental.stepvector.v4i8()
+  ret <4 x i8> %v
+}
+
+declare <8 x i8> @llvm.experimental.stepvector.v8i8()
+
+define <8 x i8> @stepvector_v8i8() {
+; CHECK-LABEL: stepvector_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 8, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <8 x i8> @llvm.experimental.stepvector.v8i8()
+  ret <8 x i8> %v
+}
+
+declare <16 x i8> @llvm.experimental.stepvector.v16i8()
+
+define <16 x i8> @stepvector_v16i8() {
+; CHECK-LABEL: stepvector_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 16, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <16 x i8> @llvm.experimental.stepvector.v16i8()
+  ret <16 x i8> %v
+}
+
+declare <2 x i16> @llvm.experimental.stepvector.v2i16()
+
+define <2 x i16> @stepvector_v2i16() {
+; CHECK-LABEL: stepvector_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 2, e16,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <2 x i16> @llvm.experimental.stepvector.v2i16()
+  ret <2 x i16> %v
+}
+
+declare <4 x i16> @llvm.experimental.stepvector.v4i16()
+
+define <4 x i16> @stepvector_v4i16() {
+; CHECK-LABEL: stepvector_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 4, e16,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <4 x i16> @llvm.experimental.stepvector.v4i16()
+  ret <4 x i16> %v
+}
+
+declare <8 x i16> @llvm.experimental.stepvector.v8i16()
+
+define <8 x i16> @stepvector_v8i16() {
+; CHECK-LABEL: stepvector_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <8 x i16> @llvm.experimental.stepvector.v8i16()
+  ret <8 x i16> %v
+}
+
+declare <16 x i16> @llvm.experimental.stepvector.v16i16()
+
+define <16 x i16> @stepvector_v16i16() {
+; LMULMAX1-LABEL: stepvector_v16i16:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI7_0)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI7_0)
+; LMULMAX1-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; LMULMAX1-NEXT:    vle16.v v9, (a0)
+; LMULMAX1-NEXT:    vid.v v8
+; LMULMAX1-NEXT:    ret
+;
+; LMULMAX2-LABEL: stepvector_v16i16:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    vsetivli a0, 16, e16,m2,ta,mu
+; LMULMAX2-NEXT:    vid.v v8
+; LMULMAX2-NEXT:    ret
+  %v = call <16 x i16> @llvm.experimental.stepvector.v16i16()
+  ret <16 x i16> %v
+}
+
+declare <2 x i32> @llvm.experimental.stepvector.v2i32()
+
+define <2 x i32> @stepvector_v2i32() {
+; CHECK-LABEL: stepvector_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 2, e32,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <2 x i32> @llvm.experimental.stepvector.v2i32()
+  ret <2 x i32> %v
+}
+
+declare <4 x i32> @llvm.experimental.stepvector.v4i32()
+
+define <4 x i32> @stepvector_v4i32() {
+; CHECK-LABEL: stepvector_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <4 x i32> @llvm.experimental.stepvector.v4i32()
+  ret <4 x i32> %v
+}
+
+declare <8 x i32> @llvm.experimental.stepvector.v8i32()
+
+define <8 x i32> @stepvector_v8i32() {
+; LMULMAX1-LABEL: stepvector_v8i32:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI10_0)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI10_0)
+; LMULMAX1-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-NEXT:    vle32.v v9, (a0)
+; LMULMAX1-NEXT:    vid.v v8
+; LMULMAX1-NEXT:    ret
+;
+; LMULMAX2-LABEL: stepvector_v8i32:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    vsetivli a0, 8, e32,m2,ta,mu
+; LMULMAX2-NEXT:    vid.v v8
+; LMULMAX2-NEXT:    ret
+  %v = call <8 x i32> @llvm.experimental.stepvector.v8i32()
+  ret <8 x i32> %v
+}
+
+declare <16 x i32> @llvm.experimental.stepvector.v16i32()
+
+define <16 x i32> @stepvector_v16i32() {
+; LMULMAX1-LABEL: stepvector_v16i32:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI11_0)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI11_0)
+; LMULMAX1-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
+; LMULMAX1-NEXT:    vle32.v v9, (a0)
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI11_1)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI11_1)
+; LMULMAX1-NEXT:    vle32.v v10, (a0)
+; LMULMAX1-NEXT:    lui a0, %hi(.LCPI11_2)
+; LMULMAX1-NEXT:    addi a0, a0, %lo(.LCPI11_2)
+; LMULMAX1-NEXT:    vle32.v v11, (a0)
+; LMULMAX1-NEXT:    vid.v v8
+; LMULMAX1-NEXT:    ret
+;
+; LMULMAX2-LABEL: stepvector_v16i32:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    lui a0, %hi(.LCPI11_0)
+; LMULMAX2-NEXT:    addi a0, a0, %lo(.LCPI11_0)
+; LMULMAX2-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
+; LMULMAX2-NEXT:    vle32.v v10, (a0)
+; LMULMAX2-NEXT:    vid.v v8
+; LMULMAX2-NEXT:    ret
+  %v = call <16 x i32> @llvm.experimental.stepvector.v16i32()
+  ret <16 x i32> %v
+}
+
+declare <2 x i64> @llvm.experimental.stepvector.v2i64()
+
+define <2 x i64> @stepvector_v2i64() {
+; CHECK-LABEL: stepvector_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli a0, 2, e64,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <2 x i64> @llvm.experimental.stepvector.v2i64()
+  ret <2 x i64> %v
+}
+
+declare <4 x i64> @llvm.experimental.stepvector.v4i64()
+
+define <4 x i64> @stepvector_v4i64() {
+; LMULMAX1-LABEL: stepvector_v4i64:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    addi a0, zero, 2
+; LMULMAX1-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.i v9, 3
+; LMULMAX1-NEXT:    vmv.s.x v9, a0
+; LMULMAX1-NEXT:    vid.v v8
+; LMULMAX1-NEXT:    ret
+;
+; LMULMAX2-LABEL: stepvector_v4i64:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    vsetivli a0, 4, e64,m2,ta,mu
+; LMULMAX2-NEXT:    vid.v v8
+; LMULMAX2-NEXT:    ret
+  %v = call <4 x i64> @llvm.experimental.stepvector.v4i64()
+  ret <4 x i64> %v
+}
+
+declare <8 x i64> @llvm.experimental.stepvector.v8i64()
+
+define <8 x i64> @stepvector_v8i64() {
+; LMULMAX1-LABEL: stepvector_v8i64:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    addi a0, zero, 2
+; LMULMAX1-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.i v10, 5
+; LMULMAX1-NEXT:    addi a1, zero, 4
+; LMULMAX1-NEXT:    vmv.s.x v10, a1
+; LMULMAX1-NEXT:    vmv.v.i v11, 7
+; LMULMAX1-NEXT:    addi a1, zero, 6
+; LMULMAX1-NEXT:    vmv.s.x v11, a1
+; LMULMAX1-NEXT:    vmv.v.i v9, 3
+; LMULMAX1-NEXT:    vmv.s.x v9, a0
+; LMULMAX1-NEXT:    vid.v v8
+; LMULMAX1-NEXT:    ret
+;
+; LMULMAX2-LABEL: stepvector_v8i64:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    lui a0, %hi(.LCPI14_0)
+; LMULMAX2-NEXT:    addi a0, a0, %lo(.LCPI14_0)
+; LMULMAX2-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-NEXT:    vle64.v v10, (a0)
+; LMULMAX2-NEXT:    vid.v v8
+; LMULMAX2-NEXT:    ret
+  %v = call <8 x i64> @llvm.experimental.stepvector.v8i64()
+  ret <8 x i64> %v
+}
+
+declare <16 x i64> @llvm.experimental.stepvector.v16i64()
+
+define <16 x i64> @stepvector_v16i64() {
+; LMULMAX1-LABEL: stepvector_v16i64:
+; LMULMAX1:       # %bb.0:
+; LMULMAX1-NEXT:    addi a0, zero, 2
+; LMULMAX1-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.i v10, 5
+; LMULMAX1-NEXT:    addi a1, zero, 4
+; LMULMAX1-NEXT:    vmv.s.x v10, a1
+; LMULMAX1-NEXT:    vmv.v.i v11, 7
+; LMULMAX1-NEXT:    addi a1, zero, 6
+; LMULMAX1-NEXT:    vmv.s.x v11, a1
+; LMULMAX1-NEXT:    vmv.v.i v12, 9
+; LMULMAX1-NEXT:    addi a1, zero, 8
+; LMULMAX1-NEXT:    vmv.s.x v12, a1
+; LMULMAX1-NEXT:    vmv.v.i v13, 11
+; LMULMAX1-NEXT:    addi a1, zero, 10
+; LMULMAX1-NEXT:    vmv.s.x v13, a1
+; LMULMAX1-NEXT:    vmv.v.i v14, 13
+; LMULMAX1-NEXT:    addi a1, zero, 12
+; LMULMAX1-NEXT:    vmv.s.x v14, a1
+; LMULMAX1-NEXT:    vmv.v.i v15, 15
+; LMULMAX1-NEXT:    addi a1, zero, 14
+; LMULMAX1-NEXT:    vmv.s.x v15, a1
+; LMULMAX1-NEXT:    vmv.v.i v9, 3
+; LMULMAX1-NEXT:    vmv.s.x v9, a0
+; LMULMAX1-NEXT:    vid.v v8
+; LMULMAX1-NEXT:    ret
+;
+; LMULMAX2-LABEL: stepvector_v16i64:
+; LMULMAX2:       # %bb.0:
+; LMULMAX2-NEXT:    lui a0, %hi(.LCPI15_0)
+; LMULMAX2-NEXT:    addi a0, a0, %lo(.LCPI15_0)
+; LMULMAX2-NEXT:    vsetivli a1, 4, e64,m2,ta,mu
+; LMULMAX2-NEXT:    vle64.v v10, (a0)
+; LMULMAX2-NEXT:    lui a0, %hi(.LCPI15_1)
+; LMULMAX2-NEXT:    addi a0, a0, %lo(.LCPI15_1)
+; LMULMAX2-NEXT:    vle64.v v12, (a0)
+; LMULMAX2-NEXT:    lui a0, %hi(.LCPI15_2)
+; LMULMAX2-NEXT:    addi a0, a0, %lo(.LCPI15_2)
+; LMULMAX2-NEXT:    vle64.v v14, (a0)
+; LMULMAX2-NEXT:    vid.v v8
+; LMULMAX2-NEXT:    ret
+  %v = call <16 x i64> @llvm.experimental.stepvector.v16i64()
+  ret <16 x i64> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll b/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll
index 85a7cd023f2d2a1b693cc66093636bca031742c6..93d2faef24a88da9b4d4d453311ab0d01940d405 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll
@@ -126,7 +126,7 @@ declare <vscale x 8 x float> @llvm.masked.load.nxv8f32(<vscale x 8 x float>*, i3
 define <vscale x 8 x double> @masked_load_nxv8f64(<vscale x 8 x double>* %a, <vscale x 8 x i1> %mask) nounwind {
 ; CHECK-LABEL: masked_load_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
   %load = call <vscale x 8 x double> @llvm.masked.load.nxv8f64(<vscale x 8 x double>* %a, i32 8, <vscale x 8 x i1> %mask, <vscale x 8 x double> undef)
@@ -148,7 +148,7 @@ declare <vscale x 16 x half> @llvm.masked.load.nxv16f16(<vscale x 16 x half>*, i
 define <vscale x 16 x float> @masked_load_nxv16f32(<vscale x 16 x float>* %a, <vscale x 16 x i1> %mask) nounwind {
 ; CHECK-LABEL: masked_load_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vle32.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
   %load = call <vscale x 16 x float> @llvm.masked.load.nxv16f32(<vscale x 16 x float>* %a, i32 4, <vscale x 16 x i1> %mask, <vscale x 16 x float> undef)
@@ -159,7 +159,7 @@ declare <vscale x 16 x float> @llvm.masked.load.nxv16f32(<vscale x 16 x float>*,
 define <vscale x 32 x half> @masked_load_nxv32f16(<vscale x 32 x half>* %a, <vscale x 32 x i1> %mask) nounwind {
 ; CHECK-LABEL: masked_load_nxv32f16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vle16.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
   %load = call <vscale x 32 x half> @llvm.masked.load.nxv32f16(<vscale x 32 x half>* %a, i32 2, <vscale x 32 x i1> %mask, <vscale x 32 x half> undef)
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll b/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll
index c7133f6d79001c2b80f8bb88c3fac7ff6fcf2c47..50136e98f64af8a90805cbb5ddc4056b1393ff1d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll
@@ -170,7 +170,7 @@ declare <vscale x 8 x i32> @llvm.masked.load.nxv8i32(<vscale x 8 x i32>*, i32, <
 define <vscale x 8 x i64> @masked_load_nxv8i64(<vscale x 8 x i64>* %a, <vscale x 8 x i1> %mask) nounwind {
 ; CHECK-LABEL: masked_load_nxv8i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
   %load = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64(<vscale x 8 x i64>* %a, i32 8, <vscale x 8 x i1> %mask, <vscale x 8 x i64> undef)
@@ -203,7 +203,7 @@ declare <vscale x 16 x i16> @llvm.masked.load.nxv16i16(<vscale x 16 x i16>*, i32
 define <vscale x 16 x i32> @masked_load_nxv16i32(<vscale x 16 x i32>* %a, <vscale x 16 x i1> %mask) nounwind {
 ; CHECK-LABEL: masked_load_nxv16i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vle32.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
   %load = call <vscale x 16 x i32> @llvm.masked.load.nxv16i32(<vscale x 16 x i32>* %a, i32 4, <vscale x 16 x i1> %mask, <vscale x 16 x i32> undef)
@@ -225,7 +225,7 @@ declare <vscale x 32 x i8> @llvm.masked.load.nxv32i8(<vscale x 32 x i8>*, i32, <
 define <vscale x 32 x i16> @masked_load_nxv32i16(<vscale x 32 x i16>* %a, <vscale x 32 x i1> %mask) nounwind {
 ; CHECK-LABEL: masked_load_nxv32i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vle16.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
   %load = call <vscale x 32 x i16> @llvm.masked.load.nxv32i16(<vscale x 32 x i16>* %a, i32 2, <vscale x 32 x i1> %mask, <vscale x 32 x i16> undef)
@@ -236,7 +236,7 @@ declare <vscale x 32 x i16> @llvm.masked.load.nxv32i16(<vscale x 32 x i16>*, i32
 define <vscale x 64 x i8> @masked_load_nxv64i8(<vscale x 64 x i8>* %a, <vscale x 64 x i1> %mask) nounwind {
 ; CHECK-LABEL: masked_load_nxv64i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
 ; CHECK-NEXT:    vle8.v v8, (a0), v0.t
 ; CHECK-NEXT:    ret
   %load = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8(<vscale x 64 x i8>* %a, i32 1, <vscale x 64 x i1> %mask, <vscale x 64 x i8> undef)
diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
new file mode 100644
index 0000000000000000000000000000000000000000..72321ead1eede9b91bfb2cb2a3341c11e6e2e755
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
@@ -0,0 +1,2302 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV64
+
+declare <vscale x 1 x i8> @llvm.masked.gather.nxv1i8.nxv1p0i8(<vscale x 1 x i8*>, i32, <vscale x 1 x i1>, <vscale x 1 x i8>)
+
+define <vscale x 1 x i8> @mgather_nxv1i8(<vscale x 1 x i8*> %ptrs, <vscale x 1 x i1> %m, <vscale x 1 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv1i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf8,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv1i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf8,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8.nxv1p0i8(<vscale x 1 x i8*> %ptrs, i32 1, <vscale x 1 x i1> %m, <vscale x 1 x i8> %passthru)
+  ret <vscale x 1 x i8> %v
+}
+
+declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8.nxv2p0i8(<vscale x 2 x i8*>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
+
+define <vscale x 2 x i8> @mgather_nxv2i8(<vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8.nxv2p0i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru)
+  ret <vscale x 2 x i8> %v
+}
+
+define <vscale x 2 x i16> @mgather_nxv2i8_sextload_nxv2i16(<vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv2i8_sextload_nxv2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vsext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i8_sextload_nxv2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vsext.vf2 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8.nxv2p0i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru)
+  %ev = sext <vscale x 2 x i8> %v to <vscale x 2 x i16>
+  ret <vscale x 2 x i16> %ev
+}
+
+define <vscale x 2 x i16> @mgather_nxv2i8_zextload_nxv2i16(<vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv2i8_zextload_nxv2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vzext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i8_zextload_nxv2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vzext.vf2 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8.nxv2p0i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru)
+  %ev = zext <vscale x 2 x i8> %v to <vscale x 2 x i16>
+  ret <vscale x 2 x i16> %ev
+}
+
+define <vscale x 2 x i32> @mgather_nxv2i8_sextload_nxv2i32(<vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv2i8_sextload_nxv2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vsext.vf4 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i8_sextload_nxv2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vsext.vf4 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8.nxv2p0i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru)
+  %ev = sext <vscale x 2 x i8> %v to <vscale x 2 x i32>
+  ret <vscale x 2 x i32> %ev
+}
+
+define <vscale x 2 x i32> @mgather_nxv2i8_zextload_nxv2i32(<vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv2i8_zextload_nxv2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vzext.vf4 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i8_zextload_nxv2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vzext.vf4 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8.nxv2p0i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru)
+  %ev = zext <vscale x 2 x i8> %v to <vscale x 2 x i32>
+  ret <vscale x 2 x i32> %ev
+}
+
+define <vscale x 2 x i64> @mgather_nxv2i8_sextload_nxv2i64(<vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv2i8_sextload_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vsext.vf8 v26, v9
+; RV32-NEXT:    vmv2r.v v8, v26
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i8_sextload_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vsext.vf8 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8.nxv2p0i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru)
+  %ev = sext <vscale x 2 x i8> %v to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ev
+}
+
+define <vscale x 2 x i64> @mgather_nxv2i8_zextload_nxv2i64(<vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv2i8_zextload_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vzext.vf8 v26, v9
+; RV32-NEXT:    vmv2r.v v8, v26
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i8_zextload_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vzext.vf8 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8.nxv2p0i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m, <vscale x 2 x i8> %passthru)
+  %ev = zext <vscale x 2 x i8> %v to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ev
+}
+
+declare <vscale x 4 x i8> @llvm.masked.gather.nxv4i8.nxv4p0i8(<vscale x 4 x i8*>, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)
+
+define <vscale x 4 x i8> @mgather_nxv4i8(<vscale x 4 x i8*> %ptrs, <vscale x 4 x i1> %m, <vscale x 4 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8.nxv4p0i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %m, <vscale x 4 x i8> %passthru)
+  ret <vscale x 4 x i8> %v
+}
+
+define <vscale x 4 x i8> @mgather_truemask_nxv4i8(<vscale x 4 x i8*> %ptrs, <vscale x 4 x i8> %passthru) {
+; RV32-LABEL: mgather_truemask_nxv4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf2,ta,mu
+; RV32-NEXT:    vloxei32.v v25, (zero), v8
+; RV32-NEXT:    vmv1r.v v8, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_nxv4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf2,ta,mu
+; RV64-NEXT:    vloxei64.v v25, (zero), v8
+; RV64-NEXT:    vmv1r.v v8, v25
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  %v = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8.nxv4p0i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %mtrue, <vscale x 4 x i8> %passthru)
+  ret <vscale x 4 x i8> %v
+}
+
+define <vscale x 4 x i8> @mgather_falsemask_nxv4i8(<vscale x 4 x i8*> %ptrs, <vscale x 4 x i8> %passthru) {
+; RV32-LABEL: mgather_falsemask_nxv4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv1r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_nxv4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv1r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8.nxv4p0i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i8> %passthru)
+  ret <vscale x 4 x i8> %v
+}
+
+declare <vscale x 8 x i8> @llvm.masked.gather.nxv8i8.nxv8p0i8(<vscale x 8 x i8*>, i32, <vscale x 8 x i1>, <vscale x 8 x i8>)
+
+define <vscale x 8 x i8> @mgather_nxv8i8(<vscale x 8 x i8*> %ptrs, <vscale x 8 x i1> %m, <vscale x 8 x i8> %passthru) {
+; RV32-LABEL: mgather_nxv8i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv8i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v16
+; RV64-NEXT:    ret
+  %v = call <vscale x 8 x i8> @llvm.masked.gather.nxv8i8.nxv8p0i8(<vscale x 8 x i8*> %ptrs, i32 1, <vscale x 8 x i1> %m, <vscale x 8 x i8> %passthru)
+  ret <vscale x 8 x i8> %v
+}
+
+define <vscale x 8 x i8> @mgather_baseidx_nxv8i8(i8* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i8> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (a0), v28, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (a0), v16, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <vscale x 8 x i8> %idxs
+  %v = call <vscale x 8 x i8> @llvm.masked.gather.nxv8i8.nxv8p0i8(<vscale x 8 x i8*> %ptrs, i32 1, <vscale x 8 x i1> %m, <vscale x 8 x i8> %passthru)
+  ret <vscale x 8 x i8> %v
+}
+
+declare <vscale x 1 x i16> @llvm.masked.gather.nxv1i16.nxv1p0i16(<vscale x 1 x i16*>, i32, <vscale x 1 x i1>, <vscale x 1 x i16>)
+
+define <vscale x 1 x i16> @mgather_nxv1i16(<vscale x 1 x i16*> %ptrs, <vscale x 1 x i1> %m, <vscale x 1 x i16> %passthru) {
+; RV32-LABEL: mgather_nxv1i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv1i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16.nxv1p0i16(<vscale x 1 x i16*> %ptrs, i32 2, <vscale x 1 x i1> %m, <vscale x 1 x i16> %passthru)
+  ret <vscale x 1 x i16> %v
+}
+
+declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16.nxv2p0i16(<vscale x 2 x i16*>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+
+define <vscale x 2 x i16> @mgather_nxv2i16(<vscale x 2 x i16*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru) {
+; RV32-LABEL: mgather_nxv2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16.nxv2p0i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru)
+  ret <vscale x 2 x i16> %v
+}
+
+define <vscale x 2 x i32> @mgather_nxv2i16_sextload_nxv2i32(<vscale x 2 x i16*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru) {
+; RV32-LABEL: mgather_nxv2i16_sextload_nxv2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vsext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i16_sextload_nxv2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vsext.vf2 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16.nxv2p0i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru)
+  %ev = sext <vscale x 2 x i16> %v to <vscale x 2 x i32>
+  ret <vscale x 2 x i32> %ev
+}
+
+define <vscale x 2 x i32> @mgather_nxv2i16_zextload_nxv2i32(<vscale x 2 x i16*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru) {
+; RV32-LABEL: mgather_nxv2i16_zextload_nxv2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vzext.vf2 v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i16_zextload_nxv2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vzext.vf2 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16.nxv2p0i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru)
+  %ev = zext <vscale x 2 x i16> %v to <vscale x 2 x i32>
+  ret <vscale x 2 x i32> %ev
+}
+
+define <vscale x 2 x i64> @mgather_nxv2i16_sextload_nxv2i64(<vscale x 2 x i16*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru) {
+; RV32-LABEL: mgather_nxv2i16_sextload_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vsext.vf4 v26, v9
+; RV32-NEXT:    vmv2r.v v8, v26
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i16_sextload_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vsext.vf4 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16.nxv2p0i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru)
+  %ev = sext <vscale x 2 x i16> %v to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ev
+}
+
+define <vscale x 2 x i64> @mgather_nxv2i16_zextload_nxv2i64(<vscale x 2 x i16*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru) {
+; RV32-LABEL: mgather_nxv2i16_zextload_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vzext.vf4 v26, v9
+; RV32-NEXT:    vmv2r.v v8, v26
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i16_zextload_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vzext.vf4 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16.nxv2p0i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %m, <vscale x 2 x i16> %passthru)
+  %ev = zext <vscale x 2 x i16> %v to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ev
+}
+
+declare <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0i16(<vscale x 4 x i16*>, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)
+
+define <vscale x 4 x i16> @mgather_nxv4i16(<vscale x 4 x i16*> %ptrs, <vscale x 4 x i1> %m, <vscale x 4 x i16> %passthru) {
+; RV32-LABEL: mgather_nxv4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %m, <vscale x 4 x i16> %passthru)
+  ret <vscale x 4 x i16> %v
+}
+
+define <vscale x 4 x i16> @mgather_truemask_nxv4i16(<vscale x 4 x i16*> %ptrs, <vscale x 4 x i16> %passthru) {
+; RV32-LABEL: mgather_truemask_nxv4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vloxei32.v v25, (zero), v8
+; RV32-NEXT:    vmv1r.v v8, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_nxv4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vloxei64.v v25, (zero), v8
+; RV64-NEXT:    vmv1r.v v8, v25
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  %v = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mtrue, <vscale x 4 x i16> %passthru)
+  ret <vscale x 4 x i16> %v
+}
+
+define <vscale x 4 x i16> @mgather_falsemask_nxv4i16(<vscale x 4 x i16*> %ptrs, <vscale x 4 x i16> %passthru) {
+; RV32-LABEL: mgather_falsemask_nxv4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv1r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_nxv4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv1r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i16> %passthru)
+  ret <vscale x 4 x i16> %v
+}
+
+declare <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0i16(<vscale x 8 x i16*>, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)
+
+define <vscale x 8 x i16> @mgather_nxv8i16(<vscale x 8 x i16*> %ptrs, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru) {
+; RV32-LABEL: mgather_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v16
+; RV64-NEXT:    ret
+  %v = call <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0i16(<vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru)
+  ret <vscale x 8 x i16> %v
+}
+
+define <vscale x 8 x i16> @mgather_baseidx_nxv8i8_nxv8i16(i16* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i8_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v28, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i8_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i16, i16* %base, <vscale x 8 x i8> %idxs
+  %v = call <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0i16(<vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru)
+  ret <vscale x 8 x i16> %v
+}
+
+define <vscale x 8 x i16> @mgather_baseidx_sext_nxv8i8_nxv8i16(i16* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i8_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v28, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i8_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i16>
+  %ptrs = getelementptr inbounds i16, i16* %base, <vscale x 8 x i16> %eidxs
+  %v = call <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0i16(<vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru)
+  ret <vscale x 8 x i16> %v
+}
+
+define <vscale x 8 x i16> @mgather_baseidx_zext_nxv8i8_nxv8i16(i16* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i8_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v28, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i8_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i16>
+  %ptrs = getelementptr inbounds i16, i16* %base, <vscale x 8 x i16> %eidxs
+  %v = call <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0i16(<vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru)
+  ret <vscale x 8 x i16> %v
+}
+
+define <vscale x 8 x i16> @mgather_baseidx_nxv8i16(i16* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v28, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i16, i16* %base, <vscale x 8 x i16> %idxs
+  %v = call <vscale x 8 x i16> @llvm.masked.gather.nxv8i16.nxv8p0i16(<vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x i16> %passthru)
+  ret <vscale x 8 x i16> %v
+}
+
+declare <vscale x 1 x i32> @llvm.masked.gather.nxv1i32.nxv1p0i32(<vscale x 1 x i32*>, i32, <vscale x 1 x i1>, <vscale x 1 x i32>)
+
+define <vscale x 1 x i32> @mgather_nxv1i32(<vscale x 1 x i32*> %ptrs, <vscale x 1 x i1> %m, <vscale x 1 x i32> %passthru) {
+; RV32-LABEL: mgather_nxv1i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv1i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32.nxv1p0i32(<vscale x 1 x i32*> %ptrs, i32 4, <vscale x 1 x i1> %m, <vscale x 1 x i32> %passthru)
+  ret <vscale x 1 x i32> %v
+}
+
+declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0i32(<vscale x 2 x i32*>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+
+define <vscale x 2 x i32> @mgather_nxv2i32(<vscale x 2 x i32*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i32> %passthru) {
+; RV32-LABEL: mgather_nxv2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %m, <vscale x 2 x i32> %passthru)
+  ret <vscale x 2 x i32> %v
+}
+
+define <vscale x 2 x i64> @mgather_nxv2i32_sextload_nxv2i64(<vscale x 2 x i32*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i32> %passthru) {
+; RV32-LABEL: mgather_nxv2i32_sextload_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vsext.vf2 v26, v9
+; RV32-NEXT:    vmv2r.v v8, v26
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i32_sextload_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vsext.vf2 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %m, <vscale x 2 x i32> %passthru)
+  %ev = sext <vscale x 2 x i32> %v to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ev
+}
+
+define <vscale x 2 x i64> @mgather_nxv2i32_zextload_nxv2i64(<vscale x 2 x i32*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i32> %passthru) {
+; RV32-LABEL: mgather_nxv2i32_zextload_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vzext.vf2 v26, v9
+; RV32-NEXT:    vmv2r.v v8, v26
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i32_zextload_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vzext.vf2 v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %m, <vscale x 2 x i32> %passthru)
+  %ev = zext <vscale x 2 x i32> %v to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %ev
+}
+
+declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
+
+define <vscale x 4 x i32> @mgather_nxv4i32(<vscale x 4 x i32*> %ptrs, <vscale x 4 x i1> %m, <vscale x 4 x i32> %passthru) {
+; RV32-LABEL: mgather_nxv4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> %m, <vscale x 4 x i32> %passthru)
+  ret <vscale x 4 x i32> %v
+}
+
+define <vscale x 4 x i32> @mgather_truemask_nxv4i32(<vscale x 4 x i32*> %ptrs, <vscale x 4 x i32> %passthru) {
+; RV32-LABEL: mgather_truemask_nxv4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV32-NEXT:    vloxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_nxv4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV64-NEXT:    vloxei64.v v26, (zero), v8
+; RV64-NEXT:    vmv2r.v v8, v26
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  %v = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> %mtrue, <vscale x 4 x i32> %passthru)
+  ret <vscale x 4 x i32> %v
+}
+
+define <vscale x 4 x i32> @mgather_falsemask_nxv4i32(<vscale x 4 x i32*> %ptrs, <vscale x 4 x i32> %passthru) {
+; RV32-LABEL: mgather_falsemask_nxv4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_nxv4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv2r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %passthru)
+  ret <vscale x 4 x i32> %v
+}
+
+declare <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0i32(<vscale x 8 x i32*>, i32, <vscale x 8 x i1>, <vscale x 8 x i32>)
+
+define <vscale x 8 x i32> @mgather_nxv8i32(<vscale x 8 x i32*> %ptrs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
+; RV32-LABEL: mgather_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (zero), v8, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (zero), v8, v0.t
+; RV64-NEXT:    vmv4r.v v8, v16
+; RV64-NEXT:    ret
+  %v = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0i32(<vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru)
+  ret <vscale x 8 x i32> %v
+}
+
+define <vscale x 8 x i32> @mgather_baseidx_nxv8i8_nxv8i32(i32* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i8_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i8_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i8> %idxs
+  %v = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0i32(<vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru)
+  ret <vscale x 8 x i32> %v
+}
+
+define <vscale x 8 x i32> @mgather_baseidx_sext_nxv8i8_nxv8i32(i32* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i8_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i8_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %eidxs
+  %v = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0i32(<vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru)
+  ret <vscale x 8 x i32> %v
+}
+
+define <vscale x 8 x i32> @mgather_baseidx_zext_nxv8i8_nxv8i32(i32* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i8_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i8_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %eidxs
+  %v = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0i32(<vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru)
+  ret <vscale x 8 x i32> %v
+}
+
+define <vscale x 8 x i32> @mgather_baseidx_nxv8i16_nxv8i32(i32* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i16_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i16_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i16> %idxs
+  %v = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0i32(<vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru)
+  ret <vscale x 8 x i32> %v
+}
+
+define <vscale x 8 x i32> @mgather_baseidx_sext_nxv8i16_nxv8i32(i32* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i16_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i16_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %eidxs
+  %v = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0i32(<vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru)
+  ret <vscale x 8 x i32> %v
+}
+
+define <vscale x 8 x i32> @mgather_baseidx_zext_nxv8i16_nxv8i32(i32* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i16_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i16_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf4 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %eidxs
+  %v = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0i32(<vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru)
+  ret <vscale x 8 x i32> %v
+}
+
+define <vscale x 8 x i32> @mgather_baseidx_nxv8i32(i32* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v8, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %idxs
+  %v = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0i32(<vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x i32> %passthru)
+  ret <vscale x 8 x i32> %v
+}
+
+declare <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0i64(<vscale x 1 x i64*>, i32, <vscale x 1 x i1>, <vscale x 1 x i64>)
+
+define <vscale x 1 x i64> @mgather_nxv1i64(<vscale x 1 x i64*> %ptrs, <vscale x 1 x i1> %m, <vscale x 1 x i64> %passthru) {
+; RV32-LABEL: mgather_nxv1i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv1i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0i64(<vscale x 1 x i64*> %ptrs, i32 8, <vscale x 1 x i1> %m, <vscale x 1 x i64> %passthru)
+  ret <vscale x 1 x i64> %v
+}
+
+declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0i64(<vscale x 2 x i64*>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
+
+define <vscale x 2 x i64> @mgather_nxv2i64(<vscale x 2 x i64*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x i64> %passthru) {
+; RV32-LABEL: mgather_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0i64(<vscale x 2 x i64*> %ptrs, i32 8, <vscale x 2 x i1> %m, <vscale x 2 x i64> %passthru)
+  ret <vscale x 2 x i64> %v
+}
+
+declare <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0i64(<vscale x 4 x i64*>, i32, <vscale x 4 x i1>, <vscale x 4 x i64>)
+
+define <vscale x 4 x i64> @mgather_nxv4i64(<vscale x 4 x i64*> %ptrs, <vscale x 4 x i1> %m, <vscale x 4 x i64> %passthru) {
+; RV32-LABEL: mgather_nxv4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (zero), v8, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0i64(<vscale x 4 x i64*> %ptrs, i32 8, <vscale x 4 x i1> %m, <vscale x 4 x i64> %passthru)
+  ret <vscale x 4 x i64> %v
+}
+
+define <vscale x 4 x i64> @mgather_truemask_nxv4i64(<vscale x 4 x i64*> %ptrs, <vscale x 4 x i64> %passthru) {
+; RV32-LABEL: mgather_truemask_nxv4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV32-NEXT:    vloxei32.v v28, (zero), v8
+; RV32-NEXT:    vmv4r.v v8, v28
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_nxv4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV64-NEXT:    vloxei64.v v8, (zero), v8
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  %v = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0i64(<vscale x 4 x i64*> %ptrs, i32 8, <vscale x 4 x i1> %mtrue, <vscale x 4 x i64> %passthru)
+  ret <vscale x 4 x i64> %v
+}
+
+define <vscale x 4 x i64> @mgather_falsemask_nxv4i64(<vscale x 4 x i64*> %ptrs, <vscale x 4 x i64> %passthru) {
+; RV32-LABEL: mgather_falsemask_nxv4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_nxv4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0i64(<vscale x 4 x i64*> %ptrs, i32 8, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i64> %passthru)
+  ret <vscale x 4 x i64> %v
+}
+
+declare <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*>, i32, <vscale x 8 x i1>, <vscale x 8 x i64>)
+
+define <vscale x 8 x i64> @mgather_nxv8i64(<vscale x 8 x i64*> %ptrs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v16, (zero), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (zero), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_nxv8i8_nxv8i64(i64* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i8_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i8_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i8> %idxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_sext_nxv8i8_nxv8i64(i64* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i8_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf8 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i8_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_zext_nxv8i8_nxv8i64(i64* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i8_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf8 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i8_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_nxv8i16_nxv8i64(i64* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i16_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i16_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i16> %idxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_sext_nxv8i16_nxv8i64(i64* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i16_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i16_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_zext_nxv8i16_nxv8i64(i64* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i16_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf4 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i16_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf4 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_nxv8i32_nxv8i64(i64* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i32_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v8, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i32_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i32> %idxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_sext_nxv8i32_nxv8i64(i64* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i32_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf2 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i32_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_zext_nxv8i32_nxv8i64(i64* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i32_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf2 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i32_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf2 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+define <vscale x 8 x i64> @mgather_baseidx_nxv8i64(i64* %base, <vscale x 8 x i64> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsll.vi v8, v8, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsll.vi v8, v8, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %idxs
+  %v = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0i64(<vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x i64> %passthru)
+  ret <vscale x 8 x i64> %v
+}
+
+declare <vscale x 16 x i64> @llvm.masked.gather.nxv16i64.nxv16p0f64(<vscale x 16 x i64*>, i32, <vscale x 16 x i1>, <vscale x 16 x i64>)
+
+declare <vscale x 16 x i64> @llvm.experimental.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64>, <vscale x 8 x i64>, i64 %idx)
+declare <vscale x 16 x i64*> @llvm.experimental.vector.insert.nxv8p0i64.nxv16p0i64(<vscale x 16 x i64*>, <vscale x 8 x i64*>, i64 %idx)
+
+define void @mgather_nxv16i64(<vscale x 8 x i64*> %ptrs0, <vscale x 8 x i64*> %ptrs1, <vscale x 16 x i1> %m, <vscale x 8 x i64> %passthru0, <vscale x 8 x i64> %passthru1, <vscale x 16 x i64>* %out) {
+; RV32-LABEL: mgather_nxv16i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vl8re64.v v24, (a0)
+; RV32-NEXT:    vsetvli a0, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v16, (zero), v8, v0.t
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    srli a0, a0, 3
+; RV32-NEXT:    vsetvli a2, zero, e8,mf4,ta,mu
+; RV32-NEXT:    vslidedown.vx v0, v0, a0
+; RV32-NEXT:    vsetvli a2, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v24, (zero), v12, v0.t
+; RV32-NEXT:    slli a0, a0, 6
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    vs8r.v v24, (a0)
+; RV32-NEXT:    vs8r.v v16, (a1)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv16i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    csrr a3, vlenb
+; RV64-NEXT:    slli a3, a3, 3
+; RV64-NEXT:    sub sp, sp, a3
+; RV64-NEXT:    vl8re64.v v24, (a0)
+; RV64-NEXT:    addi a0, sp, 16
+; RV64-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; RV64-NEXT:    vmv8r.v v16, v8
+; RV64-NEXT:    vl8re64.v v8, (a1)
+; RV64-NEXT:    vsetvli a0, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v24, (zero), v16, v0.t
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    srli a0, a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vslidedown.vx v0, v0, a0
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    addi a1, sp, 16
+; RV64-NEXT:    vl8re8.v v16, (a1) # Unknown-size Folded Reload
+; RV64-NEXT:    vloxei64.v v8, (zero), v16, v0.t
+; RV64-NEXT:    slli a0, a0, 6
+; RV64-NEXT:    add a0, a2, a0
+; RV64-NEXT:    vs8r.v v8, (a0)
+; RV64-NEXT:    vs8r.v v24, (a2)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 3
+; RV64-NEXT:    add sp, sp, a0
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    ret
+  %p0 = call <vscale x 16 x i64*> @llvm.experimental.vector.insert.nxv8p0i64.nxv16p0i64(<vscale x 16 x i64*> undef, <vscale x 8 x i64*> %ptrs0, i64 0)
+  %p1 = call <vscale x 16 x i64*> @llvm.experimental.vector.insert.nxv8p0i64.nxv16p0i64(<vscale x 16 x i64*> %p0, <vscale x 8 x i64*> %ptrs1, i64 8)
+
+  %pt0 = call <vscale x 16 x i64> @llvm.experimental.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> undef, <vscale x 8 x i64> %passthru0, i64 0)
+  %pt1 = call <vscale x 16 x i64> @llvm.experimental.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> %pt0, <vscale x 8 x i64> %passthru1, i64 8)
+
+  %v = call <vscale x 16 x i64> @llvm.masked.gather.nxv16i64.nxv16p0f64(<vscale x 16 x i64*> %p1, i32 8, <vscale x 16 x i1> %m, <vscale x 16 x i64> %pt1)
+  store <vscale x 16 x i64> %v, <vscale x 16 x i64>* %out
+  ret void
+}
+
+
+declare <vscale x 1 x half> @llvm.masked.gather.nxv1f16.nxv1p0f16(<vscale x 1 x half*>, i32, <vscale x 1 x i1>, <vscale x 1 x half>)
+
+define <vscale x 1 x half> @mgather_nxv1f16(<vscale x 1 x half*> %ptrs, <vscale x 1 x i1> %m, <vscale x 1 x half> %passthru) {
+; RV32-LABEL: mgather_nxv1f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf4,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv1f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf4,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <vscale x 1 x half> @llvm.masked.gather.nxv1f16.nxv1p0f16(<vscale x 1 x half*> %ptrs, i32 2, <vscale x 1 x i1> %m, <vscale x 1 x half> %passthru)
+  ret <vscale x 1 x half> %v
+}
+
+declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16.nxv2p0f16(<vscale x 2 x half*>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
+
+define <vscale x 2 x half> @mgather_nxv2f16(<vscale x 2 x half*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x half> %passthru) {
+; RV32-LABEL: mgather_nxv2f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16.nxv2p0f16(<vscale x 2 x half*> %ptrs, i32 2, <vscale x 2 x i1> %m, <vscale x 2 x half> %passthru)
+  ret <vscale x 2 x half> %v
+}
+
+declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16.nxv4p0f16(<vscale x 4 x half*>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
+
+define <vscale x 4 x half> @mgather_nxv4f16(<vscale x 4 x half*> %ptrs, <vscale x 4 x i1> %m, <vscale x 4 x half> %passthru) {
+; RV32-LABEL: mgather_nxv4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16.nxv4p0f16(<vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> %m, <vscale x 4 x half> %passthru)
+  ret <vscale x 4 x half> %v
+}
+
+define <vscale x 4 x half> @mgather_truemask_nxv4f16(<vscale x 4 x half*> %ptrs, <vscale x 4 x half> %passthru) {
+; RV32-LABEL: mgather_truemask_nxv4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vloxei32.v v25, (zero), v8
+; RV32-NEXT:    vmv1r.v v8, v25
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_nxv4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vloxei64.v v25, (zero), v8
+; RV64-NEXT:    vmv1r.v v8, v25
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  %v = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16.nxv4p0f16(<vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> %mtrue, <vscale x 4 x half> %passthru)
+  ret <vscale x 4 x half> %v
+}
+
+define <vscale x 4 x half> @mgather_falsemask_nxv4f16(<vscale x 4 x half*> %ptrs, <vscale x 4 x half> %passthru) {
+; RV32-LABEL: mgather_falsemask_nxv4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv1r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_nxv4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv1r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16.nxv4p0f16(<vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x half> %passthru)
+  ret <vscale x 4 x half> %v
+}
+
+declare <vscale x 8 x half> @llvm.masked.gather.nxv8f16.nxv8p0f16(<vscale x 8 x half*>, i32, <vscale x 8 x i1>, <vscale x 8 x half>)
+
+define <vscale x 8 x half> @mgather_nxv8f16(<vscale x 8 x half*> %ptrs, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru) {
+; RV32-LABEL: mgather_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v16
+; RV64-NEXT:    ret
+  %v = call <vscale x 8 x half> @llvm.masked.gather.nxv8f16.nxv8p0f16(<vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru)
+  ret <vscale x 8 x half> %v
+}
+
+define <vscale x 8 x half> @mgather_baseidx_nxv8i8_nxv8f16(half* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i8_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v28, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i8_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds half, half* %base, <vscale x 8 x i8> %idxs
+  %v = call <vscale x 8 x half> @llvm.masked.gather.nxv8f16.nxv8p0f16(<vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru)
+  ret <vscale x 8 x half> %v
+}
+
+define <vscale x 8 x half> @mgather_baseidx_sext_nxv8i8_nxv8f16(half* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i8_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v28, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i8_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i16>
+  %ptrs = getelementptr inbounds half, half* %base, <vscale x 8 x i16> %eidxs
+  %v = call <vscale x 8 x half> @llvm.masked.gather.nxv8f16.nxv8p0f16(<vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru)
+  ret <vscale x 8 x half> %v
+}
+
+define <vscale x 8 x half> @mgather_baseidx_zext_nxv8i8_nxv8f16(half* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i8_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v28, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i8_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i16>
+  %ptrs = getelementptr inbounds half, half* %base, <vscale x 8 x i16> %eidxs
+  %v = call <vscale x 8 x half> @llvm.masked.gather.nxv8f16.nxv8p0f16(<vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru)
+  ret <vscale x 8 x half> %v
+}
+
+define <vscale x 8 x half> @mgather_baseidx_nxv8f16(half* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v28, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds half, half* %base, <vscale x 8 x i16> %idxs
+  %v = call <vscale x 8 x half> @llvm.masked.gather.nxv8f16.nxv8p0f16(<vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m, <vscale x 8 x half> %passthru)
+  ret <vscale x 8 x half> %v
+}
+
+declare <vscale x 1 x float> @llvm.masked.gather.nxv1f32.nxv1p0f32(<vscale x 1 x float*>, i32, <vscale x 1 x i1>, <vscale x 1 x float>)
+
+define <vscale x 1 x float> @mgather_nxv1f32(<vscale x 1 x float*> %ptrs, <vscale x 1 x i1> %m, <vscale x 1 x float> %passthru) {
+; RV32-LABEL: mgather_nxv1f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,mf2,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv1f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,mf2,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <vscale x 1 x float> @llvm.masked.gather.nxv1f32.nxv1p0f32(<vscale x 1 x float*> %ptrs, i32 4, <vscale x 1 x i1> %m, <vscale x 1 x float> %passthru)
+  ret <vscale x 1 x float> %v
+}
+
+declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32.nxv2p0f32(<vscale x 2 x float*>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
+
+define <vscale x 2 x float> @mgather_nxv2f32(<vscale x 2 x float*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x float> %passthru) {
+; RV32-LABEL: mgather_nxv2f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32.nxv2p0f32(<vscale x 2 x float*> %ptrs, i32 4, <vscale x 2 x i1> %m, <vscale x 2 x float> %passthru)
+  ret <vscale x 2 x float> %v
+}
+
+declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0f32(<vscale x 4 x float*>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
+
+define <vscale x 4 x float> @mgather_nxv4f32(<vscale x 4 x float*> %ptrs, <vscale x 4 x i1> %m, <vscale x 4 x float> %passthru) {
+; RV32-LABEL: mgather_nxv4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0f32(<vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> %m, <vscale x 4 x float> %passthru)
+  ret <vscale x 4 x float> %v
+}
+
+define <vscale x 4 x float> @mgather_truemask_nxv4f32(<vscale x 4 x float*> %ptrs, <vscale x 4 x float> %passthru) {
+; RV32-LABEL: mgather_truemask_nxv4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV32-NEXT:    vloxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_nxv4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV64-NEXT:    vloxei64.v v26, (zero), v8
+; RV64-NEXT:    vmv2r.v v8, v26
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  %v = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0f32(<vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> %mtrue, <vscale x 4 x float> %passthru)
+  ret <vscale x 4 x float> %v
+}
+
+define <vscale x 4 x float> @mgather_falsemask_nxv4f32(<vscale x 4 x float*> %ptrs, <vscale x 4 x float> %passthru) {
+; RV32-LABEL: mgather_falsemask_nxv4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_nxv4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv2r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0f32(<vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> %passthru)
+  ret <vscale x 4 x float> %v
+}
+
+declare <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0f32(<vscale x 8 x float*>, i32, <vscale x 8 x i1>, <vscale x 8 x float>)
+
+define <vscale x 8 x float> @mgather_nxv8f32(<vscale x 8 x float*> %ptrs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
+; RV32-LABEL: mgather_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (zero), v8, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (zero), v8, v0.t
+; RV64-NEXT:    vmv4r.v v8, v16
+; RV64-NEXT:    ret
+  %v = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0f32(<vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru)
+  ret <vscale x 8 x float> %v
+}
+
+define <vscale x 8 x float> @mgather_baseidx_nxv8i8_nxv8f32(float* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i8_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i8_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i8> %idxs
+  %v = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0f32(<vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru)
+  ret <vscale x 8 x float> %v
+}
+
+define <vscale x 8 x float> @mgather_baseidx_sext_nxv8i8_nxv8f32(float* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i8_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i8_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %eidxs
+  %v = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0f32(<vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru)
+  ret <vscale x 8 x float> %v
+}
+
+define <vscale x 8 x float> @mgather_baseidx_zext_nxv8i8_nxv8f32(float* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i8_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i8_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %eidxs
+  %v = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0f32(<vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru)
+  ret <vscale x 8 x float> %v
+}
+
+define <vscale x 8 x float> @mgather_baseidx_nxv8i16_nxv8f32(float* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i16_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i16_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i16> %idxs
+  %v = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0f32(<vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru)
+  ret <vscale x 8 x float> %v
+}
+
+define <vscale x 8 x float> @mgather_baseidx_sext_nxv8i16_nxv8f32(float* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i16_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i16_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %eidxs
+  %v = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0f32(<vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru)
+  ret <vscale x 8 x float> %v
+}
+
+define <vscale x 8 x float> @mgather_baseidx_zext_nxv8i16_nxv8f32(float* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i16_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i16_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf4 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %eidxs
+  %v = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0f32(<vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru)
+  ret <vscale x 8 x float> %v
+}
+
+define <vscale x 8 x float> @mgather_baseidx_nxv8f32(float* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v8, 2
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v28, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v16, v8
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %idxs
+  %v = call <vscale x 8 x float> @llvm.masked.gather.nxv8f32.nxv8p0f32(<vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m, <vscale x 8 x float> %passthru)
+  ret <vscale x 8 x float> %v
+}
+
+declare <vscale x 1 x double> @llvm.masked.gather.nxv1f64.nxv1p0f64(<vscale x 1 x double*>, i32, <vscale x 1 x i1>, <vscale x 1 x double>)
+
+define <vscale x 1 x double> @mgather_nxv1f64(<vscale x 1 x double*> %ptrs, <vscale x 1 x i1> %m, <vscale x 1 x double> %passthru) {
+; RV32-LABEL: mgather_nxv1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; RV32-NEXT:    vloxei32.v v9, (zero), v8, v0.t
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v9, (zero), v8, v0.t
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %v = call <vscale x 1 x double> @llvm.masked.gather.nxv1f64.nxv1p0f64(<vscale x 1 x double*> %ptrs, i32 8, <vscale x 1 x i1> %m, <vscale x 1 x double> %passthru)
+  ret <vscale x 1 x double> %v
+}
+
+declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0f64(<vscale x 2 x double*>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+define <vscale x 2 x double> @mgather_nxv2f64(<vscale x 2 x double*> %ptrs, <vscale x 2 x i1> %m, <vscale x 2 x double> %passthru) {
+; RV32-LABEL: mgather_nxv2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (zero), v8, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (zero), v8, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %v = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0f64(<vscale x 2 x double*> %ptrs, i32 8, <vscale x 2 x i1> %m, <vscale x 2 x double> %passthru)
+  ret <vscale x 2 x double> %v
+}
+
+declare <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0f64(<vscale x 4 x double*>, i32, <vscale x 4 x i1>, <vscale x 4 x double>)
+
+define <vscale x 4 x double> @mgather_nxv4f64(<vscale x 4 x double*> %ptrs, <vscale x 4 x i1> %m, <vscale x 4 x double> %passthru) {
+; RV32-LABEL: mgather_nxv4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (zero), v8, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (zero), v8, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0f64(<vscale x 4 x double*> %ptrs, i32 8, <vscale x 4 x i1> %m, <vscale x 4 x double> %passthru)
+  ret <vscale x 4 x double> %v
+}
+
+define <vscale x 4 x double> @mgather_truemask_nxv4f64(<vscale x 4 x double*> %ptrs, <vscale x 4 x double> %passthru) {
+; RV32-LABEL: mgather_truemask_nxv4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV32-NEXT:    vloxei32.v v28, (zero), v8
+; RV32-NEXT:    vmv4r.v v8, v28
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_truemask_nxv4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV64-NEXT:    vloxei64.v v8, (zero), v8
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  %v = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0f64(<vscale x 4 x double*> %ptrs, i32 8, <vscale x 4 x i1> %mtrue, <vscale x 4 x double> %passthru)
+  ret <vscale x 4 x double> %v
+}
+
+define <vscale x 4 x double> @mgather_falsemask_nxv4f64(<vscale x 4 x double*> %ptrs, <vscale x 4 x double> %passthru) {
+; RV32-LABEL: mgather_falsemask_nxv4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_falsemask_nxv4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %v = call <vscale x 4 x double> @llvm.masked.gather.nxv4f64.nxv4p0f64(<vscale x 4 x double*> %ptrs, i32 8, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x double> %passthru)
+  ret <vscale x 4 x double> %v
+}
+
+declare <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*>, i32, <vscale x 8 x i1>, <vscale x 8 x double>)
+
+define <vscale x 8 x double> @mgather_nxv8f64(<vscale x 8 x double*> %ptrs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v16, (zero), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (zero), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_nxv8i8_nxv8f64(double* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i8_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i8_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i8> %idxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_sext_nxv8i8_nxv8f64(double* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i8_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf8 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i8_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_zext_nxv8i8_nxv8f64(double* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i8_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf8 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i8_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_nxv8i16_nxv8f64(double* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i16_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v8
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i16_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i16> %idxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_sext_nxv8i16_nxv8f64(double* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i16_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i16_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_zext_nxv8i16_nxv8f64(double* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i16_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf4 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i16_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf4 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_nxv8i32_nxv8f64(double* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8i32_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v8, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8i32_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i32> %idxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_sext_nxv8i32_nxv8f64(double* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_sext_nxv8i32_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf2 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_sext_nxv8i32_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_zext_nxv8i32_nxv8f64(double* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_zext_nxv8i32_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf2 v24, v8
+; RV32-NEXT:    vsll.vi v8, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_zext_nxv8i32_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf2 v24, v8
+; RV64-NEXT:    vsll.vi v8, v24, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+define <vscale x 8 x double> @mgather_baseidx_nxv8f64(double* %base, <vscale x 8 x i64> %idxs, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsll.vi v8, v8, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV32-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV32-NEXT:    vmv8r.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsll.vi v8, v8, 3
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; RV64-NEXT:    vloxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %idxs
+  %v = call <vscale x 8 x double> @llvm.masked.gather.nxv8f64.nxv8p0f64(<vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m, <vscale x 8 x double> %passthru)
+  ret <vscale x 8 x double> %v
+}
+
+declare <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0i8(<vscale x 16 x i8*>, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)
+
+define <vscale x 16 x i8> @mgather_baseidx_nxv16i8(i8* %base, <vscale x 16 x i8> %idxs, <vscale x 16 x i1> %m, <vscale x 16 x i8> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv16i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v16, v8
+; RV32-NEXT:    vsetvli a1, zero, e8,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v10, (a0), v16, v0.t
+; RV32-NEXT:    vmv2r.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv16i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v10, (a0), v16, v0.t
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    srli a1, a1, 3
+; RV64-NEXT:    vsetvli a2, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vslidedown.vx v0, v0, a1
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v9
+; RV64-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v11, (a0), v16, v0.t
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <vscale x 16 x i8> %idxs
+  %v = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0i8(<vscale x 16 x i8*> %ptrs, i32 2, <vscale x 16 x i1> %m, <vscale x 16 x i8> %passthru)
+  ret <vscale x 16 x i8> %v
+}
+
+declare <vscale x 32 x i8> @llvm.masked.gather.nxv32i8.nxv32p0i8(<vscale x 32 x i8*>, i32, <vscale x 32 x i1>, <vscale x 32 x i8>)
+
+define <vscale x 32 x i8> @mgather_baseidx_nxv32i8(i8* %base, <vscale x 32 x i8> %idxs, <vscale x 32 x i1> %m, <vscale x 32 x i8> %passthru) {
+; RV32-LABEL: mgather_baseidx_nxv32i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v16, v8
+; RV32-NEXT:    vsetvli a1, zero, e8,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v12, (a0), v16, v0.t
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    srli a1, a1, 2
+; RV32-NEXT:    vsetvli a2, zero, e8,mf2,ta,mu
+; RV32-NEXT:    vslidedown.vx v0, v0, a1
+; RV32-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v16, v10
+; RV32-NEXT:    vsetvli a1, zero, e8,m2,tu,mu
+; RV32-NEXT:    vloxei32.v v14, (a0), v16, v0.t
+; RV32-NEXT:    vmv4r.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mgather_baseidx_nxv32i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v8
+; RV64-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v12, (a0), v16, v0.t
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    srli a1, a1, 3
+; RV64-NEXT:    vsetvli a2, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vslidedown.vx v25, v0, a1
+; RV64-NEXT:    vmv1r.v v26, v0
+; RV64-NEXT:    vsetvli a2, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v9
+; RV64-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; RV64-NEXT:    vmv1r.v v0, v25
+; RV64-NEXT:    vloxei64.v v13, (a0), v16, v0.t
+; RV64-NEXT:    slli a2, a1, 1
+; RV64-NEXT:    vsetvli a3, zero, e8,mf2,ta,mu
+; RV64-NEXT:    vslidedown.vx v26, v26, a2
+; RV64-NEXT:    vsetvli a2, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vslidedown.vx v0, v26, a1
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v11
+; RV64-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; RV64-NEXT:    vloxei64.v v15, (a0), v16, v0.t
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v10
+; RV64-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; RV64-NEXT:    vmv1r.v v0, v26
+; RV64-NEXT:    vloxei64.v v14, (a0), v16, v0.t
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <vscale x 32 x i8> %idxs
+  %v = call <vscale x 32 x i8> @llvm.masked.gather.nxv32i8.nxv32p0i8(<vscale x 32 x i8*> %ptrs, i32 2, <vscale x 32 x i1> %m, <vscale x 32 x i8> %passthru)
+  ret <vscale x 32 x i8> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
new file mode 100644
index 0000000000000000000000000000000000000000..57a9e0019f7a352c1bb3b3ac20045bc8514a9984
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
@@ -0,0 +1,1938 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+experimental-zfh,+experimental-v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV64
+
+declare void @llvm.masked.scatter.nxv1i8.nxv1p0i8(<vscale x 1 x i8>, <vscale x 1 x i8*>, i32, <vscale x 1 x i1>)
+
+define void @mscatter_nxv1i8(<vscale x 1 x i8> %val, <vscale x 1 x i8*> %ptrs, <vscale x 1 x i1> %m) {
+; RV32-LABEL: mscatter_nxv1i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv1i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf8,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv1i8.nxv1p0i8(<vscale x 1 x i8> %val, <vscale x 1 x i8*> %ptrs, i32 1, <vscale x 1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv2i8.nxv2p0i8(<vscale x 2 x i8>, <vscale x 2 x i8*>, i32, <vscale x 2 x i1>)
+
+define void @mscatter_nxv2i8(<vscale x 2 x i8> %val, <vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv2i8.nxv2p0i8(<vscale x 2 x i8> %val, <vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_nxv2i16_truncstore_nxv2i8(<vscale x 2 x i16> %val, <vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i16_truncstore_nxv2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsoxei32.v v25, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i16_truncstore_nxv2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsoxei64.v v25, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <vscale x 2 x i16> %val to <vscale x 2 x i8>
+  call void @llvm.masked.scatter.nxv2i8.nxv2p0i8(<vscale x 2 x i8> %tval, <vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_nxv2i32_truncstore_nxv2i8(<vscale x 2 x i32> %val, <vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i32_truncstore_nxv2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; RV32-NEXT:    vnsrl.wi v26, v25, 0
+; RV32-NEXT:    vsoxei32.v v26, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i32_truncstore_nxv2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vnsrl.wi v26, v25, 0
+; RV64-NEXT:    vsoxei64.v v26, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <vscale x 2 x i32> %val to <vscale x 2 x i8>
+  call void @llvm.masked.scatter.nxv2i8.nxv2p0i8(<vscale x 2 x i8> %tval, <vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_nxv2i64_truncstore_nxv2i8(<vscale x 2 x i64> %val, <vscale x 2 x i8*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i64_truncstore_nxv2i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v26, v25, 0
+; RV32-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v26, 0
+; RV32-NEXT:    vsoxei32.v v25, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i64_truncstore_nxv2i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v26, v25, 0
+; RV64-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v26, 0
+; RV64-NEXT:    vsoxei64.v v25, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <vscale x 2 x i64> %val to <vscale x 2 x i8>
+  call void @llvm.masked.scatter.nxv2i8.nxv2p0i8(<vscale x 2 x i8> %tval, <vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv4i8.nxv4p0i8(<vscale x 4 x i8>, <vscale x 4 x i8*>, i32, <vscale x 4 x i1>)
+
+define void @mscatter_nxv4i8(<vscale x 4 x i8> %val, <vscale x 4 x i8*> %ptrs, <vscale x 4 x i1> %m) {
+; RV32-LABEL: mscatter_nxv4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4i8.nxv4p0i8(<vscale x 4 x i8> %val, <vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_nxv4i8(<vscale x 4 x i8> %val, <vscale x 4 x i8*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_nxv4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_nxv4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.nxv4i8.nxv4p0i8(<vscale x 4 x i8> %val, <vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_nxv4i8(<vscale x 4 x i8> %val, <vscale x 4 x i8*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_nxv4i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_nxv4i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4i8.nxv4p0i8(<vscale x 4 x i8> %val, <vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv8i8.nxv8p0i8(<vscale x 8 x i8>, <vscale x 8 x i8*>, i32, <vscale x 8 x i1>)
+
+define void @mscatter_nxv8i8(<vscale x 8 x i8> %val, <vscale x 8 x i8*> %ptrs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_nxv8i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv8i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v16, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv8i8.nxv8p0i8(<vscale x 8 x i8> %val, <vscale x 8 x i8*> %ptrs, i32 1, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i8(<vscale x 8 x i8> %val, i8* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v9
+; RV32-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v9
+; RV64-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i8, i8* %base, <vscale x 8 x i8> %idxs
+  call void @llvm.masked.scatter.nxv8i8.nxv8p0i8(<vscale x 8 x i8> %val, <vscale x 8 x i8*> %ptrs, i32 1, <vscale x 8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv1i16.nxv1p0i16(<vscale x 1 x i16>, <vscale x 1 x i16*>, i32, <vscale x 1 x i1>)
+
+define void @mscatter_nxv1i16(<vscale x 1 x i16> %val, <vscale x 1 x i16*> %ptrs, <vscale x 1 x i1> %m) {
+; RV32-LABEL: mscatter_nxv1i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv1i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv1i16.nxv1p0i16(<vscale x 1 x i16> %val, <vscale x 1 x i16*> %ptrs, i32 2, <vscale x 1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv2i16.nxv2p0i16(<vscale x 2 x i16>, <vscale x 2 x i16*>, i32, <vscale x 2 x i1>)
+
+define void @mscatter_nxv2i16(<vscale x 2 x i16> %val, <vscale x 2 x i16*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv2i16.nxv2p0i16(<vscale x 2 x i16> %val, <vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_nxv2i32_truncstore_nxv2i16(<vscale x 2 x i32> %val, <vscale x 2 x i16*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i32_truncstore_nxv2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsoxei32.v v25, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i32_truncstore_nxv2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsoxei64.v v25, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <vscale x 2 x i32> %val to <vscale x 2 x i16>
+  call void @llvm.masked.scatter.nxv2i16.nxv2p0i16(<vscale x 2 x i16> %tval, <vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_nxv2i64_truncstore_nxv2i16(<vscale x 2 x i64> %val, <vscale x 2 x i16*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i64_truncstore_nxv2i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vnsrl.wi v26, v25, 0
+; RV32-NEXT:    vsoxei32.v v26, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i64_truncstore_nxv2i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vnsrl.wi v26, v25, 0
+; RV64-NEXT:    vsoxei64.v v26, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <vscale x 2 x i64> %val to <vscale x 2 x i16>
+  call void @llvm.masked.scatter.nxv2i16.nxv2p0i16(<vscale x 2 x i16> %tval, <vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv4i16.nxv4p0i16(<vscale x 4 x i16>, <vscale x 4 x i16*>, i32, <vscale x 4 x i1>)
+
+define void @mscatter_nxv4i16(<vscale x 4 x i16> %val, <vscale x 4 x i16*> %ptrs, <vscale x 4 x i1> %m) {
+; RV32-LABEL: mscatter_nxv4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4i16.nxv4p0i16(<vscale x 4 x i16> %val, <vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_nxv4i16(<vscale x 4 x i16> %val, <vscale x 4 x i16*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_nxv4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_nxv4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.nxv4i16.nxv4p0i16(<vscale x 4 x i16> %val, <vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_nxv4i16(<vscale x 4 x i16> %val, <vscale x 4 x i16*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_nxv4i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_nxv4i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4i16.nxv4p0i16(<vscale x 4 x i16> %val, <vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv8i16.nxv8p0i16(<vscale x 8 x i16>, <vscale x 8 x i16*>, i32, <vscale x 8 x i1>)
+
+define void @mscatter_nxv8i16(<vscale x 8 x i16> %val, <vscale x 8 x i16*> %ptrs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v16, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv8i16.nxv8p0i16(<vscale x 8 x i16> %val, <vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i8_nxv8i16(<vscale x 8 x i16> %val, i16* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i8_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v10
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i8_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v10
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i16, i16* %base, <vscale x 8 x i8> %idxs
+  call void @llvm.masked.scatter.nxv8i16.nxv8p0i16(<vscale x 8 x i16> %val, <vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i8_nxv8i16(<vscale x 8 x i16> %val, i16* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v10
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v10
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i16>
+  %ptrs = getelementptr inbounds i16, i16* %base, <vscale x 8 x i16> %eidxs
+  call void @llvm.masked.scatter.nxv8i16.nxv8p0i16(<vscale x 8 x i16> %val, <vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i8_nxv8i16(<vscale x 8 x i16> %val, i16* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v10
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v16, v10
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i16>
+  %ptrs = getelementptr inbounds i16, i16* %base, <vscale x 8 x i16> %eidxs
+  call void @llvm.masked.scatter.nxv8i16.nxv8p0i16(<vscale x 8 x i16> %val, <vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i16(<vscale x 8 x i16> %val, i16* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v10
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v10
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i16, i16* %base, <vscale x 8 x i16> %idxs
+  call void @llvm.masked.scatter.nxv8i16.nxv8p0i16(<vscale x 8 x i16> %val, <vscale x 8 x i16*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv1i32.nxv1p0i32(<vscale x 1 x i32>, <vscale x 1 x i32*>, i32, <vscale x 1 x i1>)
+
+define void @mscatter_nxv1i32(<vscale x 1 x i32> %val, <vscale x 1 x i32*> %ptrs, <vscale x 1 x i1> %m) {
+; RV32-LABEL: mscatter_nxv1i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv1i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv1i32.nxv1p0i32(<vscale x 1 x i32> %val, <vscale x 1 x i32*> %ptrs, i32 4, <vscale x 1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv2i32.nxv2p0i32(<vscale x 2 x i32>, <vscale x 2 x i32*>, i32, <vscale x 2 x i1>)
+
+define void @mscatter_nxv2i32(<vscale x 2 x i32> %val, <vscale x 2 x i32*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv2i32.nxv2p0i32(<vscale x 2 x i32> %val, <vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %m)
+  ret void
+}
+
+define void @mscatter_nxv2i64_truncstore_nxv2i32(<vscale x 2 x i64> %val, <vscale x 2 x i32*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i64_truncstore_nxv2i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vnsrl.wi v25, v8, 0
+; RV32-NEXT:    vsoxei32.v v25, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i64_truncstore_nxv2i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vnsrl.wi v25, v8, 0
+; RV64-NEXT:    vsoxei64.v v25, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  %tval = trunc <vscale x 2 x i64> %val to <vscale x 2 x i32>
+  call void @llvm.masked.scatter.nxv2i32.nxv2p0i32(<vscale x 2 x i32> %tval, <vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32>, <vscale x 4 x i32*>, i32, <vscale x 4 x i1>)
+
+define void @mscatter_nxv4i32(<vscale x 4 x i32> %val, <vscale x 4 x i32*> %ptrs, <vscale x 4 x i1> %m) {
+; RV32-LABEL: mscatter_nxv4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> %val, <vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_nxv4i32(<vscale x 4 x i32> %val, <vscale x 4 x i32*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_nxv4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_nxv4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> %val, <vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_nxv4i32(<vscale x 4 x i32> %val, <vscale x 4 x i32*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_nxv4i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_nxv4i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> %val, <vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv8i32.nxv8p0i32(<vscale x 8 x i32>, <vscale x 8 x i32*>, i32, <vscale x 8 x i1>)
+
+define void @mscatter_nxv8i32(<vscale x 8 x i32> %val, <vscale x 8 x i32*> %ptrs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v16, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv8i32.nxv8p0i32(<vscale x 8 x i32> %val, <vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i8_nxv8i32(<vscale x 8 x i32> %val, i32* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i8_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i8_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i8> %idxs
+  call void @llvm.masked.scatter.nxv8i32.nxv8p0i32(<vscale x 8 x i32> %val, <vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i8_nxv8i32(<vscale x 8 x i32> %val, i32* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %eidxs
+  call void @llvm.masked.scatter.nxv8i32.nxv8p0i32(<vscale x 8 x i32> %val, <vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i8_nxv8i32(<vscale x 8 x i32> %val, i32* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %eidxs
+  call void @llvm.masked.scatter.nxv8i32.nxv8p0i32(<vscale x 8 x i32> %val, <vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, i32* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i16_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i16_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i16> %idxs
+  call void @llvm.masked.scatter.nxv8i32.nxv8p0i32(<vscale x 8 x i32> %val, <vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, i32* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %eidxs
+  call void @llvm.masked.scatter.nxv8i32.nxv8p0i32(<vscale x 8 x i32> %val, <vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i16_nxv8i32(<vscale x 8 x i32> %val, i32* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf4 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %eidxs
+  call void @llvm.masked.scatter.nxv8i32.nxv8p0i32(<vscale x 8 x i32> %val, <vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i32(<vscale x 8 x i32> %val, i32* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v12, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i32, i32* %base, <vscale x 8 x i32> %idxs
+  call void @llvm.masked.scatter.nxv8i32.nxv8p0i32(<vscale x 8 x i32> %val, <vscale x 8 x i32*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv1i64.nxv1p0i64(<vscale x 1 x i64>, <vscale x 1 x i64*>, i32, <vscale x 1 x i1>)
+
+define void @mscatter_nxv1i64(<vscale x 1 x i64> %val, <vscale x 1 x i64*> %ptrs, <vscale x 1 x i1> %m) {
+; RV32-LABEL: mscatter_nxv1i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv1i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv1i64.nxv1p0i64(<vscale x 1 x i64> %val, <vscale x 1 x i64*> %ptrs, i32 8, <vscale x 1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv2i64.nxv2p0i64(<vscale x 2 x i64>, <vscale x 2 x i64*>, i32, <vscale x 2 x i1>)
+
+define void @mscatter_nxv2i64(<vscale x 2 x i64> %val, <vscale x 2 x i64*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv2i64.nxv2p0i64(<vscale x 2 x i64> %val, <vscale x 2 x i64*> %ptrs, i32 8, <vscale x 2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv4i64.nxv4p0i64(<vscale x 4 x i64>, <vscale x 4 x i64*>, i32, <vscale x 4 x i1>)
+
+define void @mscatter_nxv4i64(<vscale x 4 x i64> %val, <vscale x 4 x i64*> %ptrs, <vscale x 4 x i1> %m) {
+; RV32-LABEL: mscatter_nxv4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4i64.nxv4p0i64(<vscale x 4 x i64> %val, <vscale x 4 x i64*> %ptrs, i32 8, <vscale x 4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_nxv4i64(<vscale x 4 x i64> %val, <vscale x 4 x i64*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_nxv4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_nxv4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.nxv4i64.nxv4p0i64(<vscale x 4 x i64> %val, <vscale x 4 x i64*> %ptrs, i32 8, <vscale x 4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_nxv4i64(<vscale x 4 x i64> %val, <vscale x 4 x i64*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_nxv4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_nxv4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4i64.nxv4p0i64(<vscale x 4 x i64> %val, <vscale x 4 x i64*> %ptrs, i32 8, <vscale x 4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64>, <vscale x 8 x i64*>, i32, <vscale x 8 x i1>)
+
+define void @mscatter_nxv8i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v16, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i8_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i8_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v16
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i8_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i8> %idxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i8_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf8 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i8_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf8 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i16_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v16
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i16_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i16> %idxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i16_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf4 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf4 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i32_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i32_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v16, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i32_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i32> %idxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i32_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i32_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf2 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i32_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i32_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i32_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf2 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i32_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf2 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i64(<vscale x 8 x i64> %val, i64* %base, <vscale x 8 x i64> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsll.vi v16, v16, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsll.vi v16, v16, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds i64, i64* %base, <vscale x 8 x i64> %idxs
+  call void @llvm.masked.scatter.nxv8i64.nxv8p0i64(<vscale x 8 x i64> %val, <vscale x 8 x i64*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv1f16.nxv1p0f16(<vscale x 1 x half>, <vscale x 1 x half*>, i32, <vscale x 1 x i1>)
+
+define void @mscatter_nxv1f16(<vscale x 1 x half> %val, <vscale x 1 x half*> %ptrs, <vscale x 1 x i1> %m) {
+; RV32-LABEL: mscatter_nxv1f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv1f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv1f16.nxv1p0f16(<vscale x 1 x half> %val, <vscale x 1 x half*> %ptrs, i32 2, <vscale x 1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv2f16.nxv2p0f16(<vscale x 2 x half>, <vscale x 2 x half*>, i32, <vscale x 2 x i1>)
+
+define void @mscatter_nxv2f16(<vscale x 2 x half> %val, <vscale x 2 x half*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv2f16.nxv2p0f16(<vscale x 2 x half> %val, <vscale x 2 x half*> %ptrs, i32 2, <vscale x 2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv4f16.nxv4p0f16(<vscale x 4 x half>, <vscale x 4 x half*>, i32, <vscale x 4 x i1>)
+
+define void @mscatter_nxv4f16(<vscale x 4 x half> %val, <vscale x 4 x half*> %ptrs, <vscale x 4 x i1> %m) {
+; RV32-LABEL: mscatter_nxv4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4f16.nxv4p0f16(<vscale x 4 x half> %val, <vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_nxv4f16(<vscale x 4 x half> %val, <vscale x 4 x half*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_nxv4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_nxv4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.nxv4f16.nxv4p0f16(<vscale x 4 x half> %val, <vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_nxv4f16(<vscale x 4 x half> %val, <vscale x 4 x half*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_nxv4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_nxv4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4f16.nxv4p0f16(<vscale x 4 x half> %val, <vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv8f16.nxv8p0f16(<vscale x 8 x half>, <vscale x 8 x half*>, i32, <vscale x 8 x i1>)
+
+define void @mscatter_nxv8f16(<vscale x 8 x half> %val, <vscale x 8 x half*> %ptrs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v16, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv8f16.nxv8p0f16(<vscale x 8 x half> %val, <vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i8_nxv8f16(<vscale x 8 x half> %val, half* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i8_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v10
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i8_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v10
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds half, half* %base, <vscale x 8 x i8> %idxs
+  call void @llvm.masked.scatter.nxv8f16.nxv8p0f16(<vscale x 8 x half> %val, <vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i8_nxv8f16(<vscale x 8 x half> %val, half* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v10
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v10
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i16>
+  %ptrs = getelementptr inbounds half, half* %base, <vscale x 8 x i16> %eidxs
+  call void @llvm.masked.scatter.nxv8f16.nxv8p0f16(<vscale x 8 x half> %val, <vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i8_nxv8f16(<vscale x 8 x half> %val, half* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v10
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v16, v10
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i16>
+  %ptrs = getelementptr inbounds half, half* %base, <vscale x 8 x i16> %eidxs
+  call void @llvm.masked.scatter.nxv8f16.nxv8p0f16(<vscale x 8 x half> %val, <vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8f16(<vscale x 8 x half> %val, half* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v10
+; RV32-NEXT:    vsll.vi v28, v28, 1
+; RV32-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v10
+; RV64-NEXT:    vsll.vi v16, v16, 1
+; RV64-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds half, half* %base, <vscale x 8 x i16> %idxs
+  call void @llvm.masked.scatter.nxv8f16.nxv8p0f16(<vscale x 8 x half> %val, <vscale x 8 x half*> %ptrs, i32 2, <vscale x 8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv1f32.nxv1p0f32(<vscale x 1 x float>, <vscale x 1 x float*>, i32, <vscale x 1 x i1>)
+
+define void @mscatter_nxv1f32(<vscale x 1 x float> %val, <vscale x 1 x float*> %ptrs, <vscale x 1 x i1> %m) {
+; RV32-LABEL: mscatter_nxv1f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv1f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv1f32.nxv1p0f32(<vscale x 1 x float> %val, <vscale x 1 x float*> %ptrs, i32 4, <vscale x 1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv2f32.nxv2p0f32(<vscale x 2 x float>, <vscale x 2 x float*>, i32, <vscale x 2 x i1>)
+
+define void @mscatter_nxv2f32(<vscale x 2 x float> %val, <vscale x 2 x float*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv2f32.nxv2p0f32(<vscale x 2 x float> %val, <vscale x 2 x float*> %ptrs, i32 4, <vscale x 2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv4f32.nxv4p0f32(<vscale x 4 x float>, <vscale x 4 x float*>, i32, <vscale x 4 x i1>)
+
+define void @mscatter_nxv4f32(<vscale x 4 x float> %val, <vscale x 4 x float*> %ptrs, <vscale x 4 x i1> %m) {
+; RV32-LABEL: mscatter_nxv4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4f32.nxv4p0f32(<vscale x 4 x float> %val, <vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_nxv4f32(<vscale x 4 x float> %val, <vscale x 4 x float*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_nxv4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_nxv4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.nxv4f32.nxv4p0f32(<vscale x 4 x float> %val, <vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_nxv4f32(<vscale x 4 x float> %val, <vscale x 4 x float*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_nxv4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_nxv4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4f32.nxv4p0f32(<vscale x 4 x float> %val, <vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv8f32.nxv8p0f32(<vscale x 8 x float>, <vscale x 8 x float*>, i32, <vscale x 8 x i1>)
+
+define void @mscatter_nxv8f32(<vscale x 8 x float> %val, <vscale x 8 x float*> %ptrs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v16, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv8f32.nxv8p0f32(<vscale x 8 x float> %val, <vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i8_nxv8f32(<vscale x 8 x float> %val, float* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i8_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i8_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i8> %idxs
+  call void @llvm.masked.scatter.nxv8f32.nxv8p0f32(<vscale x 8 x float> %val, <vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i8_nxv8f32(<vscale x 8 x float> %val, float* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %eidxs
+  call void @llvm.masked.scatter.nxv8f32.nxv8p0f32(<vscale x 8 x float> %val, <vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i8_nxv8f32(<vscale x 8 x float> %val, float* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf4 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %eidxs
+  call void @llvm.masked.scatter.nxv8f32.nxv8p0f32(<vscale x 8 x float> %val, <vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i16_nxv8f32(<vscale x 8 x float> %val, float* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i16_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i16_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i16> %idxs
+  call void @llvm.masked.scatter.nxv8f32.nxv8p0f32(<vscale x 8 x float> %val, <vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i16_nxv8f32(<vscale x 8 x float> %val, float* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %eidxs
+  call void @llvm.masked.scatter.nxv8f32.nxv8p0f32(<vscale x 8 x float> %val, <vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i16_nxv8f32(<vscale x 8 x float> %val, float* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vzext.vf2 v28, v12
+; RV32-NEXT:    vsll.vi v28, v28, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf4 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i32>
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %eidxs
+  call void @llvm.masked.scatter.nxv8f32.nxv8p0f32(<vscale x 8 x float> %val, <vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8f32(<vscale x 8 x float> %val, float* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v12, 2
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v16, v12
+; RV64-NEXT:    vsll.vi v16, v16, 2
+; RV64-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds float, float* %base, <vscale x 8 x i32> %idxs
+  call void @llvm.masked.scatter.nxv8f32.nxv8p0f32(<vscale x 8 x float> %val, <vscale x 8 x float*> %ptrs, i32 4, <vscale x 8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv1f64.nxv1p0f64(<vscale x 1 x double>, <vscale x 1 x double*>, i32, <vscale x 1 x i1>)
+
+define void @mscatter_nxv1f64(<vscale x 1 x double> %val, <vscale x 1 x double*> %ptrs, <vscale x 1 x i1> %m) {
+; RV32-LABEL: mscatter_nxv1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v9, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v9, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv1f64.nxv1p0f64(<vscale x 1 x double> %val, <vscale x 1 x double*> %ptrs, i32 8, <vscale x 1 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv2f64.nxv2p0f64(<vscale x 2 x double>, <vscale x 2 x double*>, i32, <vscale x 2 x i1>)
+
+define void @mscatter_nxv2f64(<vscale x 2 x double> %val, <vscale x 2 x double*> %ptrs, <vscale x 2 x i1> %m) {
+; RV32-LABEL: mscatter_nxv2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v10, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v10, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv2f64.nxv2p0f64(<vscale x 2 x double> %val, <vscale x 2 x double*> %ptrs, i32 8, <vscale x 2 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv4f64.nxv4p0f64(<vscale x 4 x double>, <vscale x 4 x double*>, i32, <vscale x 4 x i1>)
+
+define void @mscatter_nxv4f64(<vscale x 4 x double> %val, <vscale x 4 x double*> %ptrs, <vscale x 4 x i1> %m) {
+; RV32-LABEL: mscatter_nxv4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4f64.nxv4p0f64(<vscale x 4 x double> %val, <vscale x 4 x double*> %ptrs, i32 8, <vscale x 4 x i1> %m)
+  ret void
+}
+
+define void @mscatter_truemask_nxv4f64(<vscale x 4 x double> %val, <vscale x 4 x double*> %ptrs) {
+; RV32-LABEL: mscatter_truemask_nxv4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_truemask_nxv4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v12
+; RV64-NEXT:    ret
+  %mhead = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
+  %mtrue = shufflevector <vscale x 4 x i1> %mhead, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
+  call void @llvm.masked.scatter.nxv4f64.nxv4p0f64(<vscale x 4 x double> %val, <vscale x 4 x double*> %ptrs, i32 8, <vscale x 4 x i1> %mtrue)
+  ret void
+}
+
+define void @mscatter_falsemask_nxv4f64(<vscale x 4 x double> %val, <vscale x 4 x double*> %ptrs) {
+; RV32-LABEL: mscatter_falsemask_nxv4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_falsemask_nxv4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv4f64.nxv4p0f64(<vscale x 4 x double> %val, <vscale x 4 x double*> %ptrs, i32 8, <vscale x 4 x i1> zeroinitializer)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double>, <vscale x 8 x double*>, i32, <vscale x 8 x i1>)
+
+define void @mscatter_nxv8f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v16, v0.t
+; RV64-NEXT:    ret
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i8_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i8_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf4 v28, v16
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i8_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i8> %idxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i8_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf8 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i8_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i8_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i8> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf8 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i8_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf8 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i8> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i16_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i16_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsext.vf2 v28, v16
+; RV32-NEXT:    vsll.vi v28, v28, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i16_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i16> %idxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i16_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i16_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i16_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i16> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf4 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i16_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf4 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i16> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8i32_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8i32_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
+; RV32-NEXT:    vsll.vi v28, v16, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8i32_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i32> %idxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_sext_nxv8i32_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_sext_nxv8i32_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsext.vf2 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_sext_nxv8i32_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf2 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = sext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_zext_nxv8i32_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i32> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_zext_nxv8i32_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vzext.vf2 v24, v16
+; RV32-NEXT:    vsll.vi v16, v24, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_zext_nxv8i32_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vzext.vf2 v24, v16
+; RV64-NEXT:    vsll.vi v16, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %eidxs = zext <vscale x 8 x i32> %idxs to <vscale x 8 x i64>
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %eidxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv8f64(<vscale x 8 x double> %val, double* %base, <vscale x 8 x i64> %idxs, <vscale x 8 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsll.vi v16, v16, 3
+; RV32-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsll.vi v16, v16, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 8 x i64> %idxs
+  call void @llvm.masked.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> %val, <vscale x 8 x double*> %ptrs, i32 8, <vscale x 8 x i1> %m)
+  ret void
+}
+
+declare void @llvm.masked.scatter.nxv16f64.nxv16p0f64(<vscale x 16 x double>, <vscale x 16 x double*>, i32, <vscale x 16 x i1>)
+
+declare <vscale x 16 x double> @llvm.experimental.vector.insert.nxv8f64.nxv16f64(<vscale x 16 x double>, <vscale x 8 x double>, i64)
+declare <vscale x 16 x double*> @llvm.experimental.vector.insert.nxv8p0f64.nxv16p0f64(<vscale x 16 x double*>, <vscale x 8 x double*>, i64)
+
+define void @mscatter_nxv16f64(<vscale x 8 x double> %val0, <vscale x 8 x double> %val1, <vscale x 8 x double*> %ptrs0, <vscale x 8 x double*> %ptrs1, <vscale x 16 x i1> %m) {
+; RV32-LABEL: mscatter_nxv16f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vl4re32.v v28, (a0)
+; RV32-NEXT:    vl4re32.v v24, (a1)
+; RV32-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (zero), v28, v0.t
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    srli a0, a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
+; RV32-NEXT:    vslidedown.vx v0, v0, a0
+; RV32-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v16, (zero), v24, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_nxv16f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a2, a2, 3
+; RV64-NEXT:    sub sp, sp, a2
+; RV64-NEXT:    vl8re64.v v24, (a0)
+; RV64-NEXT:    addi a0, sp, 16
+; RV64-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; RV64-NEXT:    vl8re64.v v16, (a1)
+; RV64-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsoxei64.v v8, (zero), v24, v0.t
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    srli a0, a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vslidedown.vx v0, v0, a0
+; RV64-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; RV64-NEXT:    addi a0, sp, 16
+; RV64-NEXT:    vl8re8.v v8, (a0) # Unknown-size Folded Reload
+; RV64-NEXT:    vsoxei64.v v8, (zero), v16, v0.t
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 3
+; RV64-NEXT:    add sp, sp, a0
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    ret
+  %p0 = call <vscale x 16 x double*> @llvm.experimental.vector.insert.nxv8p0f64.nxv16p0f64(<vscale x 16 x double*> undef, <vscale x 8 x double*> %ptrs0, i64 0)
+  %p1 = call <vscale x 16 x double*> @llvm.experimental.vector.insert.nxv8p0f64.nxv16p0f64(<vscale x 16 x double*> %p0, <vscale x 8 x double*> %ptrs1, i64 8)
+  %v0 = call <vscale x 16 x double> @llvm.experimental.vector.insert.nxv8f64.nxv16f64(<vscale x 16 x double> undef, <vscale x 8 x double> %val0, i64 0)
+  %v1 = call <vscale x 16 x double> @llvm.experimental.vector.insert.nxv8f64.nxv16f64(<vscale x 16 x double> %v0, <vscale x 8 x double> %val1, i64 8)
+  call void @llvm.masked.scatter.nxv16f64.nxv16p0f64(<vscale x 16 x double> %v1, <vscale x 16 x double*> %p1, i32 8, <vscale x 16 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv16i8_nxv16f64(<vscale x 8 x double> %val0, <vscale x 8 x double> %val1, double* %base, <vscale x 16 x i8> %idxs, <vscale x 16 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv16i8_nxv16f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vl2r.v v2, (a1)
+; RV32-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
+; RV32-NEXT:    vsext.vf4 v24, v2
+; RV32-NEXT:    vsll.vi v24, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v24, v0.t
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    srli a1, a1, 3
+; RV32-NEXT:    vsetvli a2, zero, e8,mf4,ta,mu
+; RV32-NEXT:    vslidedown.vx v0, v0, a1
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv16i8_nxv16f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vl2r.v v2, (a1)
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v24, v2
+; RV64-NEXT:    vsll.vi v24, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    srli a1, a1, 3
+; RV64-NEXT:    vsetvli a2, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vslidedown.vx v0, v0, a1
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf8 v8, v3
+; RV64-NEXT:    vsll.vi v8, v8, 3
+; RV64-NEXT:    vsoxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 16 x i8> %idxs
+  %v0 = call <vscale x 16 x double> @llvm.experimental.vector.insert.nxv8f64.nxv16f64(<vscale x 16 x double> undef, <vscale x 8 x double> %val0, i64 0)
+  %v1 = call <vscale x 16 x double> @llvm.experimental.vector.insert.nxv8f64.nxv16f64(<vscale x 16 x double> %v0, <vscale x 8 x double> %val1, i64 8)
+  call void @llvm.masked.scatter.nxv16f64.nxv16p0f64(<vscale x 16 x double> %v1, <vscale x 16 x double*> %ptrs, i32 8, <vscale x 16 x i1> %m)
+  ret void
+}
+
+define void @mscatter_baseidx_nxv16i16_nxv16f64(<vscale x 8 x double> %val0, <vscale x 8 x double> %val1, double* %base, <vscale x 16 x i16> %idxs, <vscale x 16 x i1> %m) {
+; RV32-LABEL: mscatter_baseidx_nxv16i16_nxv16f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vl4re16.v v4, (a1)
+; RV32-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
+; RV32-NEXT:    vsext.vf2 v24, v4
+; RV32-NEXT:    vsll.vi v24, v24, 3
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v8, (a0), v24, v0.t
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    srli a1, a1, 3
+; RV32-NEXT:    vsetvli a2, zero, e8,mf4,ta,mu
+; RV32-NEXT:    vslidedown.vx v0, v0, a1
+; RV32-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV32-NEXT:    vsoxei32.v v16, (a0), v28, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mscatter_baseidx_nxv16i16_nxv16f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vl4re16.v v4, (a1)
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v24, v4
+; RV64-NEXT:    vsll.vi v24, v24, 3
+; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    srli a1, a1, 3
+; RV64-NEXT:    vsetvli a2, zero, e8,mf4,ta,mu
+; RV64-NEXT:    vslidedown.vx v0, v0, a1
+; RV64-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; RV64-NEXT:    vsext.vf4 v8, v6
+; RV64-NEXT:    vsll.vi v8, v8, 3
+; RV64-NEXT:    vsoxei64.v v16, (a0), v8, v0.t
+; RV64-NEXT:    ret
+  %ptrs = getelementptr inbounds double, double* %base, <vscale x 16 x i16> %idxs
+  %v0 = call <vscale x 16 x double> @llvm.experimental.vector.insert.nxv8f64.nxv16f64(<vscale x 16 x double> undef, <vscale x 8 x double> %val0, i64 0)
+  %v1 = call <vscale x 16 x double> @llvm.experimental.vector.insert.nxv8f64.nxv16f64(<vscale x 16 x double> %v0, <vscale x 8 x double> %val1, i64 8)
+  call void @llvm.masked.scatter.nxv16f64.nxv16p0f64(<vscale x 16 x double> %v1, <vscale x 16 x double*> %ptrs, i32 8, <vscale x 16 x i1> %m)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d549c03d9d027a7cea3f9f9f6f0b084423e7a6bb
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
@@ -0,0 +1,299 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -mattr=+m -O0 < %s \
+; RUN:    | FileCheck --check-prefix=SPILL-O0 %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -mattr=+m -O2 < %s \
+; RUN:    | FileCheck --check-prefix=SPILL-O2 %s
+
+define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(i32* %base, i32 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg_nxv1i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
+; SPILL-O0-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv1r.v v25, v1
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs1r.v v25, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg_nxv1i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    slli a2, a2, 1
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
+; SPILL-O2-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vs1r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs1r.v v1, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vl1r.v v7, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8 killed $v8 killed $v7_v8
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 1
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 1 x i32>,<vscale x 1 x i32>} @llvm.riscv.vlseg2.nxv1i32(i32* %base, i32 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 1 x i32>,<vscale x 1 x i32>} %0, 1
+  ret <vscale x 1 x i32> %1
+}
+
+define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(i32* %base, i32 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg_nxv2i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
+; SPILL-O0-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv1r.v v25, v1
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs1r.v v25, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg_nxv2i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    slli a2, a2, 1
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
+; SPILL-O2-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vs1r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs1r.v v1, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vl1r.v v7, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8 killed $v8 killed $v7_v8
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 1
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 2 x i32>,<vscale x 2 x i32>} @llvm.riscv.vlseg2.nxv2i32(i32* %base, i32 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 2 x i32>,<vscale x 2 x i32>} %0, 1
+  ret <vscale x 2 x i32> %1
+}
+
+define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(i32* %base, i32 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg_nxv4i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    slli a2, a2, 1
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
+; SPILL-O0-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv2r.v v26, v2
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs2r.v v26, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl2re8.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    slli a0, a0, 1
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg_nxv4i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    slli a2, a2, 2
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
+; SPILL-O2-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 1
+; SPILL-O2-NEXT:    vs2r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs2r.v v2, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 1
+; SPILL-O2-NEXT:    vl2r.v v6, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8m2 killed $v8m2 killed $v6m2_v8m2
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 2
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 4 x i32>,<vscale x 4 x i32>} @llvm.riscv.vlseg2.nxv4i32(i32* %base, i32 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 4 x i32>,<vscale x 4 x i32>} %0, 1
+  ret <vscale x 4 x i32> %1
+}
+
+define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(i32* %base, i32 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg_nxv8i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    slli a2, a2, 2
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
+; SPILL-O0-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv4r.v v28, v4
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs4r.v v28, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl4re8.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    slli a0, a0, 2
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg_nxv8i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    slli a2, a2, 3
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
+; SPILL-O2-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 2
+; SPILL-O2-NEXT:    vs4r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs4r.v v4, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 2
+; SPILL-O2-NEXT:    vl4r.v v4, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8m4 killed $v8m4 killed $v4m4_v8m4
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 3
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 8 x i32>,<vscale x 8 x i32>} @llvm.riscv.vlseg2.nxv8i32(i32* %base, i32 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 8 x i32>,<vscale x 8 x i32>} %0, 1
+  ret <vscale x 8 x i32> %1
+}
+
+define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(i32* %base, i32 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg3_nxv4i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    slli a2, a2, 1
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
+; SPILL-O0-NEXT:    vlseg3e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv2r.v v26, v2
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs2r.v v26, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl2re8.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    slli a0, a0, 1
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg3_nxv4i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    addi a3, zero, 6
+; SPILL-O2-NEXT:    mul a2, a2, a3
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
+; SPILL-O2-NEXT:    vlseg3e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 1
+; SPILL-O2-NEXT:    vs2r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs2r.v v2, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs2r.v v4, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 1
+; SPILL-O2-NEXT:    vl2r.v v6, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8m2 killed $v8m2 killed $v6m2_v8m2_v10m2
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    addi a1, zero, 6
+; SPILL-O2-NEXT:    mul a0, a0, a1
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>} @llvm.riscv.vlseg3.nxv4i32(i32* %base, i32 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>} %0, 1
+  ret <vscale x 4 x i32> %1
+}
+
+declare {<vscale x 1 x i32>,<vscale x 1 x i32>} @llvm.riscv.vlseg2.nxv1i32(i32* , i32)
+declare {<vscale x 2 x i32>,<vscale x 2 x i32>} @llvm.riscv.vlseg2.nxv2i32(i32* , i32)
+declare {<vscale x 4 x i32>,<vscale x 4 x i32>} @llvm.riscv.vlseg2.nxv4i32(i32* , i32)
+declare {<vscale x 8 x i32>,<vscale x 8 x i32>} @llvm.riscv.vlseg2.nxv8i32(i32* , i32)
+declare {<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>} @llvm.riscv.vlseg3.nxv4i32(i32* , i32)
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bbda9980380b25bd89cf597452691819dfe8d0c8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
@@ -0,0 +1,299 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -mattr=+m -O0 < %s \
+; RUN:    | FileCheck --check-prefix=SPILL-O0 %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -mattr=+m -O2 < %s \
+; RUN:    | FileCheck --check-prefix=SPILL-O2 %s
+
+define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(i32* %base, i64 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg_nxv1i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
+; SPILL-O0-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv1r.v v25, v1
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs1r.v v25, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg_nxv1i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    slli a2, a2, 1
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
+; SPILL-O2-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vs1r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs1r.v v1, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vl1r.v v7, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8 killed $v8 killed $v7_v8
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 1
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 1 x i32>,<vscale x 1 x i32>} @llvm.riscv.vlseg2.nxv1i32(i32* %base, i64 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 1 x i32>,<vscale x 1 x i32>} %0, 1
+  ret <vscale x 1 x i32> %1
+}
+
+define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(i32* %base, i64 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg_nxv2i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
+; SPILL-O0-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv1r.v v25, v1
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs1r.v v25, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg_nxv2i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    slli a2, a2, 1
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
+; SPILL-O2-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vs1r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs1r.v v1, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    vl1r.v v7, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8 killed $v8 killed $v7_v8
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 1
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 2 x i32>,<vscale x 2 x i32>} @llvm.riscv.vlseg2.nxv2i32(i32* %base, i64 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 2 x i32>,<vscale x 2 x i32>} %0, 1
+  ret <vscale x 2 x i32> %1
+}
+
+define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(i32* %base, i64 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg_nxv4i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    slli a2, a2, 1
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
+; SPILL-O0-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv2r.v v26, v2
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs2r.v v26, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl2re8.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    slli a0, a0, 1
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg_nxv4i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    slli a2, a2, 2
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
+; SPILL-O2-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 1
+; SPILL-O2-NEXT:    vs2r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs2r.v v2, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 1
+; SPILL-O2-NEXT:    vl2r.v v6, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8m2 killed $v8m2 killed $v6m2_v8m2
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 2
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 4 x i32>,<vscale x 4 x i32>} @llvm.riscv.vlseg2.nxv4i32(i32* %base, i64 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 4 x i32>,<vscale x 4 x i32>} %0, 1
+  ret <vscale x 4 x i32> %1
+}
+
+define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(i32* %base, i64 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg_nxv8i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    slli a2, a2, 2
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
+; SPILL-O0-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv4r.v v28, v4
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs4r.v v28, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl4re8.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    slli a0, a0, 2
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg_nxv8i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    slli a2, a2, 3
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
+; SPILL-O2-NEXT:    vlseg2e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 2
+; SPILL-O2-NEXT:    vs4r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs4r.v v4, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 2
+; SPILL-O2-NEXT:    vl4r.v v4, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl4r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8m4 killed $v8m4 killed $v4m4_v8m4
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    slli a0, a0, 3
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 8 x i32>,<vscale x 8 x i32>} @llvm.riscv.vlseg2.nxv8i32(i32* %base, i64 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 8 x i32>,<vscale x 8 x i32>} %0, 1
+  ret <vscale x 8 x i32> %1
+}
+
+define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(i32* %base, i64 %vl) nounwind {
+; SPILL-O0-LABEL: spill_zvlsseg3_nxv4i32:
+; SPILL-O0:       # %bb.0: # %entry
+; SPILL-O0-NEXT:    addi sp, sp, -16
+; SPILL-O0-NEXT:    csrr a2, vlenb
+; SPILL-O0-NEXT:    slli a2, a2, 1
+; SPILL-O0-NEXT:    sub sp, sp, a2
+; SPILL-O0-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
+; SPILL-O0-NEXT:    vlseg3e32.v v0, (a0)
+; SPILL-O0-NEXT:    vmv2r.v v26, v2
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vs2r.v v26, (a0) # Unknown-size Folded Spill
+; SPILL-O0-NEXT:    #APP
+; SPILL-O0-NEXT:    #NO_APP
+; SPILL-O0-NEXT:    addi a0, sp, 16
+; SPILL-O0-NEXT:    vl2re8.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O0-NEXT:    csrr a0, vlenb
+; SPILL-O0-NEXT:    slli a0, a0, 1
+; SPILL-O0-NEXT:    add sp, sp, a0
+; SPILL-O0-NEXT:    addi sp, sp, 16
+; SPILL-O0-NEXT:    ret
+;
+; SPILL-O2-LABEL: spill_zvlsseg3_nxv4i32:
+; SPILL-O2:       # %bb.0: # %entry
+; SPILL-O2-NEXT:    addi sp, sp, -16
+; SPILL-O2-NEXT:    csrr a2, vlenb
+; SPILL-O2-NEXT:    addi a3, zero, 6
+; SPILL-O2-NEXT:    mul a2, a2, a3
+; SPILL-O2-NEXT:    sub sp, sp, a2
+; SPILL-O2-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
+; SPILL-O2-NEXT:    vlseg3e32.v v0, (a0)
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 1
+; SPILL-O2-NEXT:    vs2r.v v0, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs2r.v v2, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vs2r.v v4, (a0) # Unknown-size Folded Spill
+; SPILL-O2-NEXT:    #APP
+; SPILL-O2-NEXT:    #NO_APP
+; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    csrr a1, vlenb
+; SPILL-O2-NEXT:    slli a1, a1, 1
+; SPILL-O2-NEXT:    vl2r.v v6, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    add a0, a0, a1
+; SPILL-O2-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
+; SPILL-O2-NEXT:    # kill: def $v8m2 killed $v8m2 killed $v6m2_v8m2_v10m2
+; SPILL-O2-NEXT:    csrr a0, vlenb
+; SPILL-O2-NEXT:    addi a1, zero, 6
+; SPILL-O2-NEXT:    mul a0, a0, a1
+; SPILL-O2-NEXT:    add sp, sp, a0
+; SPILL-O2-NEXT:    addi sp, sp, 16
+; SPILL-O2-NEXT:    ret
+entry:
+  %0 = tail call {<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>} @llvm.riscv.vlseg3.nxv4i32(i32* %base, i64 %vl)
+  call void asm sideeffect "",
+  "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  %1 = extractvalue {<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>} %0, 1
+  ret <vscale x 4 x i32> %1
+}
+
+declare {<vscale x 1 x i32>,<vscale x 1 x i32>} @llvm.riscv.vlseg2.nxv1i32(i32* , i64)
+declare {<vscale x 2 x i32>,<vscale x 2 x i32>} @llvm.riscv.vlseg2.nxv2i32(i32* , i64)
+declare {<vscale x 4 x i32>,<vscale x 4 x i32>} @llvm.riscv.vlseg2.nxv4i32(i32* , i64)
+declare {<vscale x 8 x i32>,<vscale x 8 x i32>} @llvm.riscv.vlseg2.nxv8i32(i32* , i64)
+declare {<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>} @llvm.riscv.vlseg3.nxv4i32(i32* , i64)
diff --git a/llvm/test/CodeGen/RISCV/rvv/stepvector-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/stepvector-rv32.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b11776d17b1597e5535770837c593b3ac911afcf
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/stepvector-rv32.ll
@@ -0,0 +1,233 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v -verify-machineinstrs < %s | FileCheck %s
+
+declare <vscale x 1 x i8> @llvm.experimental.stepvector.nxv1i8()
+
+define <vscale x 1 x i8> @stepvector_nxv1i8() {
+; CHECK-LABEL: stepvector_nxv1i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,mf8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 1 x i8> @llvm.experimental.stepvector.nxv1i8()
+  ret <vscale x 1 x i8> %v
+}
+
+declare <vscale x 2 x i8> @llvm.experimental.stepvector.nxv2i8()
+
+define <vscale x 2 x i8> @stepvector_nxv2i8() {
+; CHECK-LABEL: stepvector_nxv2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i8> @llvm.experimental.stepvector.nxv2i8()
+  ret <vscale x 2 x i8> %v
+}
+
+declare <vscale x 4 x i8> @llvm.experimental.stepvector.nxv4i8()
+
+define <vscale x 4 x i8> @stepvector_nxv4i8() {
+; CHECK-LABEL: stepvector_nxv4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,mf2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i8> @llvm.experimental.stepvector.nxv4i8()
+  ret <vscale x 4 x i8> %v
+}
+
+declare <vscale x 8 x i8> @llvm.experimental.stepvector.nxv8i8()
+
+define <vscale x 8 x i8> @stepvector_nxv8i8() {
+; CHECK-LABEL: stepvector_nxv8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i8> @llvm.experimental.stepvector.nxv8i8()
+  ret <vscale x 8 x i8> %v
+}
+
+declare <vscale x 16 x i8> @llvm.experimental.stepvector.nxv16i8()
+
+define <vscale x 16 x i8> @stepvector_nxv16i8() {
+; CHECK-LABEL: stepvector_nxv16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,m2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 16 x i8> @llvm.experimental.stepvector.nxv16i8()
+  ret <vscale x 16 x i8> %v
+}
+
+declare <vscale x 32 x i8> @llvm.experimental.stepvector.nxv32i8()
+
+define <vscale x 32 x i8> @stepvector_nxv32i8() {
+; CHECK-LABEL: stepvector_nxv32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,m4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x i8> @llvm.experimental.stepvector.nxv32i8()
+  ret <vscale x 32 x i8> %v
+}
+
+declare <vscale x 64 x i8> @llvm.experimental.stepvector.nxv64i8()
+
+define <vscale x 64 x i8> @stepvector_nxv64i8() {
+; CHECK-LABEL: stepvector_nxv64i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,m8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 64 x i8> @llvm.experimental.stepvector.nxv64i8()
+  ret <vscale x 64 x i8> %v
+}
+
+declare <vscale x 1 x i16> @llvm.experimental.stepvector.nxv1i16()
+
+define <vscale x 1 x i16> @stepvector_nxv1i16() {
+; CHECK-LABEL: stepvector_nxv1i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 1 x i16> @llvm.experimental.stepvector.nxv1i16()
+  ret <vscale x 1 x i16> %v
+}
+
+declare <vscale x 2 x i16> @llvm.experimental.stepvector.nxv2i16()
+
+define <vscale x 2 x i16> @stepvector_nxv2i16() {
+; CHECK-LABEL: stepvector_nxv2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i16> @llvm.experimental.stepvector.nxv2i16()
+  ret <vscale x 2 x i16> %v
+}
+
+declare <vscale x 4 x i16> @llvm.experimental.stepvector.nxv4i16()
+
+define <vscale x 4 x i16> @stepvector_nxv4i16() {
+; CHECK-LABEL: stepvector_nxv4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i16> @llvm.experimental.stepvector.nxv4i16()
+  ret <vscale x 4 x i16> %v
+}
+
+declare <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+
+define <vscale x 8 x i16> @stepvector_nxv8i16() {
+; CHECK-LABEL: stepvector_nxv8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+  ret <vscale x 8 x i16> %v
+}
+
+declare <vscale x 16 x i16> @llvm.experimental.stepvector.nxv16i16()
+
+define <vscale x 16 x i16> @stepvector_nxv16i16() {
+; CHECK-LABEL: stepvector_nxv16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,m4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 16 x i16> @llvm.experimental.stepvector.nxv16i16()
+  ret <vscale x 16 x i16> %v
+}
+
+declare <vscale x 32 x i16> @llvm.experimental.stepvector.nxv32i16()
+
+define <vscale x 32 x i16> @stepvector_nxv32i16() {
+; CHECK-LABEL: stepvector_nxv32i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,m8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x i16> @llvm.experimental.stepvector.nxv32i16()
+  ret <vscale x 32 x i16> %v
+}
+
+declare <vscale x 1 x i32> @llvm.experimental.stepvector.nxv1i32()
+
+define <vscale x 1 x i32> @stepvector_nxv1i32() {
+; CHECK-LABEL: stepvector_nxv1i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 1 x i32> @llvm.experimental.stepvector.nxv1i32()
+  ret <vscale x 1 x i32> %v
+}
+
+declare <vscale x 2 x i32> @llvm.experimental.stepvector.nxv2i32()
+
+define <vscale x 2 x i32> @stepvector_nxv2i32() {
+; CHECK-LABEL: stepvector_nxv2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i32> @llvm.experimental.stepvector.nxv2i32()
+  ret <vscale x 2 x i32> %v
+}
+
+declare <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+
+define <vscale x 4 x i32> @stepvector_nxv4i32() {
+; CHECK-LABEL: stepvector_nxv4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+  ret <vscale x 4 x i32> %v
+}
+
+declare <vscale x 8 x i32> @llvm.experimental.stepvector.nxv8i32()
+
+define <vscale x 8 x i32> @stepvector_nxv8i32() {
+; CHECK-LABEL: stepvector_nxv8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i32> @llvm.experimental.stepvector.nxv8i32()
+  ret <vscale x 8 x i32> %v
+}
+
+declare <vscale x 16 x i32> @llvm.experimental.stepvector.nxv16i32()
+
+define <vscale x 16 x i32> @stepvector_nxv16i32() {
+; CHECK-LABEL: stepvector_nxv16i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 16 x i32> @llvm.experimental.stepvector.nxv16i32()
+  ret <vscale x 16 x i32> %v
+}
+
+declare <vscale x 32 x i32> @llvm.experimental.stepvector.nxv32i32()
+
+define <vscale x 32 x i32> @stepvector_nxv32i32() {
+; CHECK-LABEL: stepvector_nxv32i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x i32> @llvm.experimental.stepvector.nxv32i32()
+  ret <vscale x 32 x i32> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/stepvector-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/stepvector-rv64.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dec25b517165aebd86c3580cb22a143f5ab4eaa0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/stepvector-rv64.ll
@@ -0,0 +1,280 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v -verify-machineinstrs < %s | FileCheck %s
+
+declare <vscale x 1 x i8> @llvm.experimental.stepvector.nxv1i8()
+
+define <vscale x 1 x i8> @stepvector_nxv1i8() {
+; CHECK-LABEL: stepvector_nxv1i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,mf8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 1 x i8> @llvm.experimental.stepvector.nxv1i8()
+  ret <vscale x 1 x i8> %v
+}
+
+declare <vscale x 2 x i8> @llvm.experimental.stepvector.nxv2i8()
+
+define <vscale x 2 x i8> @stepvector_nxv2i8() {
+; CHECK-LABEL: stepvector_nxv2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,mf4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i8> @llvm.experimental.stepvector.nxv2i8()
+  ret <vscale x 2 x i8> %v
+}
+
+declare <vscale x 4 x i8> @llvm.experimental.stepvector.nxv4i8()
+
+define <vscale x 4 x i8> @stepvector_nxv4i8() {
+; CHECK-LABEL: stepvector_nxv4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,mf2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i8> @llvm.experimental.stepvector.nxv4i8()
+  ret <vscale x 4 x i8> %v
+}
+
+declare <vscale x 8 x i8> @llvm.experimental.stepvector.nxv8i8()
+
+define <vscale x 8 x i8> @stepvector_nxv8i8() {
+; CHECK-LABEL: stepvector_nxv8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i8> @llvm.experimental.stepvector.nxv8i8()
+  ret <vscale x 8 x i8> %v
+}
+
+declare <vscale x 16 x i8> @llvm.experimental.stepvector.nxv16i8()
+
+define <vscale x 16 x i8> @stepvector_nxv16i8() {
+; CHECK-LABEL: stepvector_nxv16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,m2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 16 x i8> @llvm.experimental.stepvector.nxv16i8()
+  ret <vscale x 16 x i8> %v
+}
+
+declare <vscale x 32 x i8> @llvm.experimental.stepvector.nxv32i8()
+
+define <vscale x 32 x i8> @stepvector_nxv32i8() {
+; CHECK-LABEL: stepvector_nxv32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,m4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x i8> @llvm.experimental.stepvector.nxv32i8()
+  ret <vscale x 32 x i8> %v
+}
+
+declare <vscale x 64 x i8> @llvm.experimental.stepvector.nxv64i8()
+
+define <vscale x 64 x i8> @stepvector_nxv64i8() {
+; CHECK-LABEL: stepvector_nxv64i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8,m8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 64 x i8> @llvm.experimental.stepvector.nxv64i8()
+  ret <vscale x 64 x i8> %v
+}
+
+declare <vscale x 1 x i16> @llvm.experimental.stepvector.nxv1i16()
+
+define <vscale x 1 x i16> @stepvector_nxv1i16() {
+; CHECK-LABEL: stepvector_nxv1i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 1 x i16> @llvm.experimental.stepvector.nxv1i16()
+  ret <vscale x 1 x i16> %v
+}
+
+declare <vscale x 2 x i16> @llvm.experimental.stepvector.nxv2i16()
+
+define <vscale x 2 x i16> @stepvector_nxv2i16() {
+; CHECK-LABEL: stepvector_nxv2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i16> @llvm.experimental.stepvector.nxv2i16()
+  ret <vscale x 2 x i16> %v
+}
+
+declare <vscale x 4 x i16> @llvm.experimental.stepvector.nxv4i16()
+
+define <vscale x 4 x i16> @stepvector_nxv4i16() {
+; CHECK-LABEL: stepvector_nxv4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i16> @llvm.experimental.stepvector.nxv4i16()
+  ret <vscale x 4 x i16> %v
+}
+
+declare <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+
+define <vscale x 8 x i16> @stepvector_nxv8i16() {
+; CHECK-LABEL: stepvector_nxv8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i16> @llvm.experimental.stepvector.nxv8i16()
+  ret <vscale x 8 x i16> %v
+}
+
+declare <vscale x 16 x i16> @llvm.experimental.stepvector.nxv16i16()
+
+define <vscale x 16 x i16> @stepvector_nxv16i16() {
+; CHECK-LABEL: stepvector_nxv16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,m4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 16 x i16> @llvm.experimental.stepvector.nxv16i16()
+  ret <vscale x 16 x i16> %v
+}
+
+declare <vscale x 32 x i16> @llvm.experimental.stepvector.nxv32i16()
+
+define <vscale x 32 x i16> @stepvector_nxv32i16() {
+; CHECK-LABEL: stepvector_nxv32i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16,m8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x i16> @llvm.experimental.stepvector.nxv32i16()
+  ret <vscale x 32 x i16> %v
+}
+
+declare <vscale x 1 x i32> @llvm.experimental.stepvector.nxv1i32()
+
+define <vscale x 1 x i32> @stepvector_nxv1i32() {
+; CHECK-LABEL: stepvector_nxv1i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 1 x i32> @llvm.experimental.stepvector.nxv1i32()
+  ret <vscale x 1 x i32> %v
+}
+
+declare <vscale x 2 x i32> @llvm.experimental.stepvector.nxv2i32()
+
+define <vscale x 2 x i32> @stepvector_nxv2i32() {
+; CHECK-LABEL: stepvector_nxv2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i32> @llvm.experimental.stepvector.nxv2i32()
+  ret <vscale x 2 x i32> %v
+}
+
+declare <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+
+define <vscale x 4 x i32> @stepvector_nxv4i32() {
+; CHECK-LABEL: stepvector_nxv4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+  ret <vscale x 4 x i32> %v
+}
+
+declare <vscale x 8 x i32> @llvm.experimental.stepvector.nxv8i32()
+
+define <vscale x 8 x i32> @stepvector_nxv8i32() {
+; CHECK-LABEL: stepvector_nxv8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i32> @llvm.experimental.stepvector.nxv8i32()
+  ret <vscale x 8 x i32> %v
+}
+
+declare <vscale x 16 x i32> @llvm.experimental.stepvector.nxv16i32()
+
+define <vscale x 16 x i32> @stepvector_nxv16i32() {
+; CHECK-LABEL: stepvector_nxv16i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 16 x i32> @llvm.experimental.stepvector.nxv16i32()
+  ret <vscale x 16 x i32> %v
+}
+
+declare <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
+
+define <vscale x 1 x i64> @stepvector_nxv1i64() {
+; CHECK-LABEL: stepvector_nxv1i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
+  ret <vscale x 1 x i64> %v
+}
+
+declare <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+
+define <vscale x 2 x i64> @stepvector_nxv2i64() {
+; CHECK-LABEL: stepvector_nxv2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+  ret <vscale x 2 x i64> %v
+}
+
+declare <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+
+define <vscale x 4 x i64> @stepvector_nxv4i64() {
+; CHECK-LABEL: stepvector_nxv4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+  ret <vscale x 4 x i64> %v
+}
+
+declare <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+
+define <vscale x 8 x i64> @stepvector_nxv8i64() {
+; CHECK-LABEL: stepvector_nxv8i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    ret
+  %v = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+  ret <vscale x 8 x i64> %v
+}
+
+declare <vscale x 16 x i64> @llvm.experimental.stepvector.nxv16i64()
+
+define <vscale x 16 x i64> @stepvector_nxv16i64() {
+; CHECK-LABEL: stepvector_nxv16i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 16 x i64> @llvm.experimental.stepvector.nxv16i64()
+  ret <vscale x 16 x i64> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/tail-agnostic-impdef-copy.mir b/llvm/test/CodeGen/RISCV/rvv/tail-agnostic-impdef-copy.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5e34d25826b22da733674f566835b90a4931b89e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/tail-agnostic-impdef-copy.mir
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc %s -mtriple=riscv64 -mattr=experimental-v -riscv-v-vector-bits-min=128 -run-pass=finalize-isel -o - | FileCheck %s
+
+# This test makes sure we peak through the COPY instruction between the
+# IMPLICIT_DEF and PseudoVLE64_V_M8_MASK in order to select the tail agnostic
+# policy. The test is working if the second argument to PseudoVSETVLI has bit 6
+# set.
+
+--- |
+  ; ModuleID = 'test.ll'
+  source_filename = "test.ll"
+  target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
+  target triple = "riscv64"
+
+  ; Function Attrs: nounwind
+  define <vscale x 8 x i64> @masked_load_nxv8i64(<vscale x 8 x i64>* %a, <vscale x 8 x i1> %mask) #0 {
+    %load = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64.p0nxv8i64(<vscale x 8 x i64>* %a, i32 8, <vscale x 8 x i1> %mask, <vscale x 8 x i64> undef)
+    ret <vscale x 8 x i64> %load
+  }
+
+  ; Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+  declare <vscale x 8 x i64> @llvm.masked.load.nxv8i64.p0nxv8i64(<vscale x 8 x i64>*, i32 immarg, <vscale x 8 x i1>, <vscale x 8 x i64>) #1
+
+  attributes #0 = { nounwind "target-features"="+experimental-v" }
+  attributes #1 = { argmemonly nofree nosync nounwind readonly willreturn "target-features"="+experimental-v" }
+
+...
+---
+name:            masked_load_nxv8i64
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: vr }
+  - { id: 2, class: vrm8nov0 }
+  - { id: 3, class: vrm8 }
+  - { id: 4, class: vrm8nov0 }
+liveins:
+  - { reg: '$x10', virtual-reg: '%0' }
+  - { reg: '$v0', virtual-reg: '%1' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $x10, $v0
+
+    ; CHECK-LABEL: name: masked_load_nxv8i64
+    ; CHECK: liveins: $x10, $v0
+    ; CHECK: [[COPY:%[0-9]+]]:vr = COPY $v0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK: $v0 = COPY [[COPY]]
+    ; CHECK: [[DEF:%[0-9]+]]:vrm8 = IMPLICIT_DEF
+    ; CHECK: [[COPY2:%[0-9]+]]:vrm8nov0 = COPY [[DEF]]
+    ; CHECK: dead %5:gpr = PseudoVSETVLI $x0, 91, implicit-def $vl, implicit-def $vtype
+    ; CHECK: [[PseudoVLE64_V_M8_MASK:%[0-9]+]]:vrm8nov0 = PseudoVLE64_V_M8_MASK [[COPY2]], [[COPY1]], $v0, $noreg, 64, implicit $vl, implicit $vtype :: (load 64 from %ir.a, align 8)
+    ; CHECK: $v8m8 = COPY [[PseudoVLE64_V_M8_MASK]]
+    ; CHECK: PseudoRET implicit $v8m8
+    %1:vr = COPY $v0
+    %0:gpr = COPY $x10
+    $v0 = COPY %1
+    %3:vrm8 = IMPLICIT_DEF
+    %4:vrm8nov0 = COPY %3
+    %2:vrm8nov0 = PseudoVLE64_V_M8_MASK %4, %0, $v0, $x0, 64, implicit $vl, implicit $vtype :: (load 64 from %ir.a, align 8)
+    $v8m8 = COPY %2
+    PseudoRET implicit $v8m8
+
+...
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfabs-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfabs-sdnode.ll
index 314a7574f2df977471befdbb5612d7f56cbf7c6d..18bd8ceebdadde4eb5e822b17ceab6f7d0bbf0ff 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfabs-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfabs-sdnode.ll
@@ -10,7 +10,7 @@ define <vscale x 1 x half> @vfabs_nxv1f16(<vscale x 1 x half> %v) {
 ; CHECK-LABEL: vfabs_nxv1f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16,mf4,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 1 x half> @llvm.fabs.nxv1f16(<vscale x 1 x half> %v)
   ret <vscale x 1 x half> %r
@@ -22,7 +22,7 @@ define <vscale x 2 x half> @vfabs_nxv2f16(<vscale x 2 x half> %v) {
 ; CHECK-LABEL: vfabs_nxv2f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16,mf2,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 2 x half> @llvm.fabs.nxv2f16(<vscale x 2 x half> %v)
   ret <vscale x 2 x half> %r
@@ -34,7 +34,7 @@ define <vscale x 4 x half> @vfabs_nxv4f16(<vscale x 4 x half> %v) {
 ; CHECK-LABEL: vfabs_nxv4f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %v)
   ret <vscale x 4 x half> %r
@@ -46,7 +46,7 @@ define <vscale x 8 x half> @vfabs_nxv8f16(<vscale x 8 x half> %v) {
 ; CHECK-LABEL: vfabs_nxv8f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %v)
   ret <vscale x 8 x half> %r
@@ -58,7 +58,7 @@ define <vscale x 16 x half> @vfabs_nxv16f16(<vscale x 16 x half> %v) {
 ; CHECK-LABEL: vfabs_nxv16f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16,m4,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 16 x half> @llvm.fabs.nxv16f16(<vscale x 16 x half> %v)
   ret <vscale x 16 x half> %r
@@ -70,7 +70,7 @@ define <vscale x 32 x half> @vfabs_nxv32f16(<vscale x 32 x half> %v) {
 ; CHECK-LABEL: vfabs_nxv32f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16,m8,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 32 x half> @llvm.fabs.nxv32f16(<vscale x 32 x half> %v)
   ret <vscale x 32 x half> %r
@@ -82,7 +82,7 @@ define <vscale x 1 x float> @vfabs_nxv1f32(<vscale x 1 x float> %v) {
 ; CHECK-LABEL: vfabs_nxv1f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 1 x float> @llvm.fabs.nxv1f32(<vscale x 1 x float> %v)
   ret <vscale x 1 x float> %r
@@ -94,7 +94,7 @@ define <vscale x 2 x float> @vfabs_nxv2f32(<vscale x 2 x float> %v) {
 ; CHECK-LABEL: vfabs_nxv2f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float> %v)
   ret <vscale x 2 x float> %r
@@ -106,7 +106,7 @@ define <vscale x 4 x float> @vfabs_nxv4f32(<vscale x 4 x float> %v) {
 ; CHECK-LABEL: vfabs_nxv4f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> %v)
   ret <vscale x 4 x float> %r
@@ -118,7 +118,7 @@ define <vscale x 8 x float> @vfabs_nxv8f32(<vscale x 8 x float> %v) {
 ; CHECK-LABEL: vfabs_nxv8f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 8 x float> @llvm.fabs.nxv8f32(<vscale x 8 x float> %v)
   ret <vscale x 8 x float> %r
@@ -130,7 +130,7 @@ define <vscale x 16 x float> @vfabs_nxv16f32(<vscale x 16 x float> %v) {
 ; CHECK-LABEL: vfabs_nxv16f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 16 x float> @llvm.fabs.nxv16f32(<vscale x 16 x float> %v)
   ret <vscale x 16 x float> %r
@@ -142,7 +142,7 @@ define <vscale x 1 x double> @vfabs_nxv1f64(<vscale x 1 x double> %v) {
 ; CHECK-LABEL: vfabs_nxv1f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 1 x double> @llvm.fabs.nxv1f64(<vscale x 1 x double> %v)
   ret <vscale x 1 x double> %r
@@ -154,7 +154,7 @@ define <vscale x 2 x double> @vfabs_nxv2f64(<vscale x 2 x double> %v) {
 ; CHECK-LABEL: vfabs_nxv2f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> %v)
   ret <vscale x 2 x double> %r
@@ -166,7 +166,7 @@ define <vscale x 4 x double> @vfabs_nxv4f64(<vscale x 4 x double> %v) {
 ; CHECK-LABEL: vfabs_nxv4f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 4 x double> @llvm.fabs.nxv4f64(<vscale x 4 x double> %v)
   ret <vscale x 4 x double> %r
@@ -178,7 +178,7 @@ define <vscale x 8 x double> @vfabs_nxv8f64(<vscale x 8 x double> %v) {
 ; CHECK-LABEL: vfabs_nxv8f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vfsgnjx.vv v8, v8, v8
+; CHECK-NEXT:    vfabs.v v8, v8
 ; CHECK-NEXT:    ret
   %r = call <vscale x 8 x double> @llvm.fabs.nxv8f64(<vscale x 8 x double> %v)
   ret <vscale x 8 x double> %r
diff --git a/llvm/test/CodeGen/RISCV/rvv/vloxei-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vloxei-rv32.ll
index e347c434ff100db65b1f304cc989084f7bf0cf02..cfc54485a74186591a05a1236e569eab9f70ea59 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vloxei-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vloxei-rv32.ll
@@ -10,7 +10,8 @@ define <vscale x 1 x i8> @intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32(
@@ -54,7 +55,8 @@ define <vscale x 2 x i8> @intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32(
@@ -98,7 +100,8 @@ define <vscale x 4 x i8> @intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32(
@@ -142,7 +145,8 @@ define <vscale x 8 x i8> @intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32(
@@ -186,7 +190,8 @@ define <vscale x 16 x i8> @intrinsic_vloxei_v_nxv16i8_nxv16i8_nxv16i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i8_nxv16i8_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32(
@@ -230,7 +235,8 @@ define <vscale x 1 x i16> @intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32(
@@ -274,7 +280,8 @@ define <vscale x 2 x i16> @intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32(
@@ -318,7 +325,8 @@ define <vscale x 4 x i16> @intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32(
@@ -362,7 +370,8 @@ define <vscale x 8 x i16> @intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32(
@@ -406,7 +415,8 @@ define <vscale x 16 x i16> @intrinsic_vloxei_v_nxv16i16_nxv16i16_nxv16i32(<vscal
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i16_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32(
@@ -670,7 +680,8 @@ define <vscale x 1 x half> @intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vloxei.nxv1f16.nxv1i32(
@@ -714,7 +725,8 @@ define <vscale x 2 x half> @intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vloxei.nxv2f16.nxv2i32(
@@ -758,7 +770,8 @@ define <vscale x 4 x half> @intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vloxei.nxv4f16.nxv4i32(
@@ -802,7 +815,8 @@ define <vscale x 8 x half> @intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vloxei.nxv8f16.nxv8i32(
@@ -846,7 +860,8 @@ define <vscale x 16 x half> @intrinsic_vloxei_v_nxv16f16_nxv16f16_nxv16i32(<vsca
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16f16_nxv16f16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vloxei.nxv16f16.nxv16i32(
@@ -1110,7 +1125,8 @@ define <vscale x 1 x double> @intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i32(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i32(
@@ -1154,7 +1170,8 @@ define <vscale x 2 x double> @intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i32(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i32(
@@ -1198,7 +1215,8 @@ define <vscale x 4 x double> @intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i32(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i32(
@@ -1242,7 +1260,8 @@ define <vscale x 8 x double> @intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i32(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i32(
@@ -1286,7 +1305,8 @@ define <vscale x 1 x i8> @intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16(
@@ -1330,7 +1350,8 @@ define <vscale x 2 x i8> @intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16(
@@ -1374,7 +1395,8 @@ define <vscale x 4 x i8> @intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16(
@@ -1418,7 +1440,8 @@ define <vscale x 8 x i8> @intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16(
@@ -1462,7 +1485,8 @@ define <vscale x 16 x i8> @intrinsic_vloxei_v_nxv16i8_nxv16i8_nxv16i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i8_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16(
@@ -1506,7 +1530,8 @@ define <vscale x 32 x i8> @intrinsic_vloxei_v_nxv32i8_nxv32i8_nxv32i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv32i8_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16(
@@ -1814,7 +1839,8 @@ define <vscale x 1 x i32> @intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16(
@@ -1858,7 +1884,8 @@ define <vscale x 2 x i32> @intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16(
@@ -1902,7 +1929,8 @@ define <vscale x 4 x i32> @intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16(
@@ -1946,7 +1974,8 @@ define <vscale x 8 x i32> @intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16(
@@ -1990,7 +2019,8 @@ define <vscale x 16 x i32> @intrinsic_vloxei_v_nxv16i32_nxv16i32_nxv16i16(<vscal
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i32_nxv16i32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16(
@@ -2298,7 +2328,8 @@ define <vscale x 1 x float> @intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i16(
@@ -2342,7 +2373,8 @@ define <vscale x 2 x float> @intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i16(
@@ -2386,7 +2418,8 @@ define <vscale x 4 x float> @intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i16(
@@ -2430,7 +2463,8 @@ define <vscale x 8 x float> @intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i16(
@@ -2474,7 +2508,8 @@ define <vscale x 16 x float> @intrinsic_vloxei_v_nxv16f32_nxv16f32_nxv16i16(<vsc
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16f32_nxv16f32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i16(
@@ -2518,7 +2553,8 @@ define <vscale x 1 x double> @intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i16(
@@ -2562,7 +2598,8 @@ define <vscale x 2 x double> @intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i16(
@@ -2606,7 +2643,8 @@ define <vscale x 4 x double> @intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i16(
@@ -2650,7 +2688,8 @@ define <vscale x 8 x double> @intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i16(
@@ -3002,7 +3041,8 @@ define <vscale x 1 x i16> @intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8(
@@ -3046,7 +3086,8 @@ define <vscale x 2 x i16> @intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8(
@@ -3090,7 +3131,8 @@ define <vscale x 4 x i16> @intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8(
@@ -3134,7 +3176,8 @@ define <vscale x 8 x i16> @intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8(
@@ -3178,7 +3221,8 @@ define <vscale x 16 x i16> @intrinsic_vloxei_v_nxv16i16_nxv16i16_nxv16i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i16_nxv16i16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8(
@@ -3222,7 +3266,8 @@ define <vscale x 32 x i16> @intrinsic_vloxei_v_nxv32i16_nxv32i16_nxv32i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv32i16_nxv32i16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8(
@@ -3266,7 +3311,8 @@ define <vscale x 1 x i32> @intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8(
@@ -3310,7 +3356,8 @@ define <vscale x 2 x i32> @intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8(
@@ -3354,7 +3401,8 @@ define <vscale x 4 x i32> @intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8(
@@ -3398,7 +3446,8 @@ define <vscale x 8 x i32> @intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8(
@@ -3442,7 +3491,8 @@ define <vscale x 16 x i32> @intrinsic_vloxei_v_nxv16i32_nxv16i32_nxv16i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i32_nxv16i32_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8(
@@ -3486,7 +3536,8 @@ define <vscale x 1 x half> @intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vloxei.nxv1f16.nxv1i8(
@@ -3530,7 +3581,8 @@ define <vscale x 2 x half> @intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vloxei.nxv2f16.nxv2i8(
@@ -3574,7 +3626,8 @@ define <vscale x 4 x half> @intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vloxei.nxv4f16.nxv4i8(
@@ -3618,7 +3671,8 @@ define <vscale x 8 x half> @intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vloxei.nxv8f16.nxv8i8(
@@ -3662,7 +3716,8 @@ define <vscale x 16 x half> @intrinsic_vloxei_v_nxv16f16_nxv16f16_nxv16i8(<vscal
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16f16_nxv16f16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vloxei.nxv16f16.nxv16i8(
@@ -3706,7 +3761,8 @@ define <vscale x 32 x half> @intrinsic_vloxei_v_nxv32f16_nxv32f16_nxv32i8(<vscal
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv32f16_nxv32f16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vloxei.nxv32f16.nxv32i8(
@@ -3750,7 +3806,8 @@ define <vscale x 1 x float> @intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i8(
@@ -3794,7 +3851,8 @@ define <vscale x 2 x float> @intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i8(
@@ -3838,7 +3896,8 @@ define <vscale x 4 x float> @intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i8(
@@ -3882,7 +3941,8 @@ define <vscale x 8 x float> @intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i8(
@@ -3926,7 +3986,8 @@ define <vscale x 16 x float> @intrinsic_vloxei_v_nxv16f32_nxv16f32_nxv16i8(<vsca
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16f32_nxv16f32_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i8(
@@ -3970,7 +4031,8 @@ define <vscale x 1 x double> @intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i8(
@@ -4014,7 +4076,8 @@ define <vscale x 2 x double> @intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i8(
@@ -4058,7 +4121,8 @@ define <vscale x 4 x double> @intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i8(
@@ -4102,7 +4166,8 @@ define <vscale x 8 x double> @intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vloxei-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vloxei-rv64.ll
index 3ac5dc3f57469f877677ccf57118693f394c4672..05204e671f2c717b733edef08053929255fe7830 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vloxei-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vloxei-rv64.ll
@@ -10,7 +10,8 @@ define <vscale x 1 x i8> @intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i64(
@@ -54,7 +55,8 @@ define <vscale x 2 x i8> @intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i64(
@@ -98,7 +100,8 @@ define <vscale x 4 x i8> @intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i64(
@@ -142,7 +145,8 @@ define <vscale x 8 x i8> @intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i64(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i64(
@@ -186,7 +190,8 @@ define <vscale x 1 x i16> @intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i64(
@@ -230,7 +235,8 @@ define <vscale x 2 x i16> @intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i64(
@@ -274,7 +280,8 @@ define <vscale x 4 x i16> @intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i64(
@@ -318,7 +325,8 @@ define <vscale x 8 x i16> @intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i64(
@@ -362,7 +370,8 @@ define <vscale x 1 x i32> @intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i64(
@@ -406,7 +415,8 @@ define <vscale x 2 x i32> @intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i64(
@@ -450,7 +460,8 @@ define <vscale x 4 x i32> @intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i64(
@@ -494,7 +505,8 @@ define <vscale x 8 x i32> @intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i64(
@@ -714,7 +726,8 @@ define <vscale x 1 x half> @intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vloxei.nxv1f16.nxv1i64(
@@ -758,7 +771,8 @@ define <vscale x 2 x half> @intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vloxei.nxv2f16.nxv2i64(
@@ -802,7 +816,8 @@ define <vscale x 4 x half> @intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vloxei.nxv4f16.nxv4i64(
@@ -846,7 +861,8 @@ define <vscale x 8 x half> @intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i64(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vloxei.nxv8f16.nxv8i64(
@@ -890,7 +906,8 @@ define <vscale x 1 x float> @intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i64(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i64(
@@ -934,7 +951,8 @@ define <vscale x 2 x float> @intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i64(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i64(
@@ -978,7 +996,8 @@ define <vscale x 4 x float> @intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i64(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i64(
@@ -1022,7 +1041,8 @@ define <vscale x 8 x float> @intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i64(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei64.v v8, (a0), v8
+; CHECK-NEXT:    vloxei64.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i64(
@@ -1242,7 +1262,8 @@ define <vscale x 1 x i8> @intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i32(
@@ -1286,7 +1307,8 @@ define <vscale x 2 x i8> @intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i32(
@@ -1330,7 +1352,8 @@ define <vscale x 4 x i8> @intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i32(
@@ -1374,7 +1397,8 @@ define <vscale x 8 x i8> @intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i32(
@@ -1418,7 +1442,8 @@ define <vscale x 16 x i8> @intrinsic_vloxei_v_nxv16i8_nxv16i8_nxv16i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i8_nxv16i8_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i32(
@@ -1462,7 +1487,8 @@ define <vscale x 1 x i16> @intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i32(
@@ -1506,7 +1532,8 @@ define <vscale x 2 x i16> @intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i32(
@@ -1550,7 +1577,8 @@ define <vscale x 4 x i16> @intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i32(
@@ -1594,7 +1622,8 @@ define <vscale x 8 x i16> @intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i32(
@@ -1638,7 +1667,8 @@ define <vscale x 16 x i16> @intrinsic_vloxei_v_nxv16i16_nxv16i16_nxv16i32(<vscal
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i16_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i32(
@@ -1902,7 +1932,8 @@ define <vscale x 1 x i64> @intrinsic_vloxei_v_nxv1i64_nxv1i64_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i64_nxv1i64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i32(
@@ -1946,7 +1977,8 @@ define <vscale x 2 x i64> @intrinsic_vloxei_v_nxv2i64_nxv2i64_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i64_nxv2i64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i32(
@@ -1990,7 +2022,8 @@ define <vscale x 4 x i64> @intrinsic_vloxei_v_nxv4i64_nxv4i64_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i64_nxv4i64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i32(
@@ -2034,7 +2067,8 @@ define <vscale x 8 x i64> @intrinsic_vloxei_v_nxv8i64_nxv8i64_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i64_nxv8i64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i32(
@@ -2078,7 +2112,8 @@ define <vscale x 1 x half> @intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vloxei.nxv1f16.nxv1i32(
@@ -2122,7 +2157,8 @@ define <vscale x 2 x half> @intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vloxei.nxv2f16.nxv2i32(
@@ -2166,7 +2202,8 @@ define <vscale x 4 x half> @intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vloxei.nxv4f16.nxv4i32(
@@ -2210,7 +2247,8 @@ define <vscale x 8 x half> @intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vloxei.nxv8f16.nxv8i32(
@@ -2254,7 +2292,8 @@ define <vscale x 16 x half> @intrinsic_vloxei_v_nxv16f16_nxv16f16_nxv16i32(<vsca
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16f16_nxv16f16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vloxei.nxv16f16.nxv16i32(
@@ -2518,7 +2557,8 @@ define <vscale x 1 x double> @intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i32(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i32(
@@ -2562,7 +2602,8 @@ define <vscale x 2 x double> @intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i32(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i32(
@@ -2606,7 +2647,8 @@ define <vscale x 4 x double> @intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i32(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i32(
@@ -2650,7 +2692,8 @@ define <vscale x 8 x double> @intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i32(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vloxei32.v v8, (a0), v8
+; CHECK-NEXT:    vloxei32.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i32(
@@ -2694,7 +2737,8 @@ define <vscale x 1 x i8> @intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i8_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vloxei.nxv1i8.nxv1i16(
@@ -2738,7 +2782,8 @@ define <vscale x 2 x i8> @intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i8_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vloxei.nxv2i8.nxv2i16(
@@ -2782,7 +2827,8 @@ define <vscale x 4 x i8> @intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i8_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vloxei.nxv4i8.nxv4i16(
@@ -2826,7 +2872,8 @@ define <vscale x 8 x i8> @intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i8_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vloxei.nxv8i8.nxv8i16(
@@ -2870,7 +2917,8 @@ define <vscale x 16 x i8> @intrinsic_vloxei_v_nxv16i8_nxv16i8_nxv16i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i8_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vloxei.nxv16i8.nxv16i16(
@@ -2914,7 +2962,8 @@ define <vscale x 32 x i8> @intrinsic_vloxei_v_nxv32i8_nxv32i8_nxv32i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv32i8_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i8> @llvm.riscv.vloxei.nxv32i8.nxv32i16(
@@ -3222,7 +3271,8 @@ define <vscale x 1 x i32> @intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i16(
@@ -3266,7 +3316,8 @@ define <vscale x 2 x i32> @intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i16(
@@ -3310,7 +3361,8 @@ define <vscale x 4 x i32> @intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i16(
@@ -3354,7 +3406,8 @@ define <vscale x 8 x i32> @intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i16(
@@ -3398,7 +3451,8 @@ define <vscale x 16 x i32> @intrinsic_vloxei_v_nxv16i32_nxv16i32_nxv16i16(<vscal
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i32_nxv16i32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i16(
@@ -3442,7 +3496,8 @@ define <vscale x 1 x i64> @intrinsic_vloxei_v_nxv1i64_nxv1i64_nxv1i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i64_nxv1i64_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i16(
@@ -3486,7 +3541,8 @@ define <vscale x 2 x i64> @intrinsic_vloxei_v_nxv2i64_nxv2i64_nxv2i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i64_nxv2i64_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i16(
@@ -3530,7 +3586,8 @@ define <vscale x 4 x i64> @intrinsic_vloxei_v_nxv4i64_nxv4i64_nxv4i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i64_nxv4i64_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i16(
@@ -3574,7 +3631,8 @@ define <vscale x 8 x i64> @intrinsic_vloxei_v_nxv8i64_nxv8i64_nxv8i16(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i64_nxv8i64_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i16(
@@ -3882,7 +3940,8 @@ define <vscale x 1 x float> @intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i16(
@@ -3926,7 +3985,8 @@ define <vscale x 2 x float> @intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i16(
@@ -3970,7 +4030,8 @@ define <vscale x 4 x float> @intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i16(
@@ -4014,7 +4075,8 @@ define <vscale x 8 x float> @intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i16(
@@ -4058,7 +4120,8 @@ define <vscale x 16 x float> @intrinsic_vloxei_v_nxv16f32_nxv16f32_nxv16i16(<vsc
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16f32_nxv16f32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i16(
@@ -4102,7 +4165,8 @@ define <vscale x 1 x double> @intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i16(
@@ -4146,7 +4210,8 @@ define <vscale x 2 x double> @intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i16(
@@ -4190,7 +4255,8 @@ define <vscale x 4 x double> @intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i16(
@@ -4234,7 +4300,8 @@ define <vscale x 8 x double> @intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i16(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vloxei16.v v8, (a0), v8
+; CHECK-NEXT:    vloxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i16(
@@ -4586,7 +4653,8 @@ define <vscale x 1 x i16> @intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i16_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vloxei.nxv1i16.nxv1i8(
@@ -4630,7 +4698,8 @@ define <vscale x 2 x i16> @intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i16_nxv2i16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vloxei.nxv2i16.nxv2i8(
@@ -4674,7 +4743,8 @@ define <vscale x 4 x i16> @intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i16_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vloxei.nxv4i16.nxv4i8(
@@ -4718,7 +4788,8 @@ define <vscale x 8 x i16> @intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i16_nxv8i16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vloxei.nxv8i16.nxv8i8(
@@ -4762,7 +4833,8 @@ define <vscale x 16 x i16> @intrinsic_vloxei_v_nxv16i16_nxv16i16_nxv16i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i16_nxv16i16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vloxei.nxv16i16.nxv16i8(
@@ -4806,7 +4878,8 @@ define <vscale x 32 x i16> @intrinsic_vloxei_v_nxv32i16_nxv32i16_nxv32i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv32i16_nxv32i16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i16> @llvm.riscv.vloxei.nxv32i16.nxv32i8(
@@ -4850,7 +4923,8 @@ define <vscale x 1 x i32> @intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i32_nxv1i32_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vloxei.nxv1i32.nxv1i8(
@@ -4894,7 +4968,8 @@ define <vscale x 2 x i32> @intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i32_nxv2i32_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vloxei.nxv2i32.nxv2i8(
@@ -4938,7 +5013,8 @@ define <vscale x 4 x i32> @intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i32_nxv4i32_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vloxei.nxv4i32.nxv4i8(
@@ -4982,7 +5058,8 @@ define <vscale x 8 x i32> @intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i32_nxv8i32_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vloxei.nxv8i32.nxv8i8(
@@ -5026,7 +5103,8 @@ define <vscale x 16 x i32> @intrinsic_vloxei_v_nxv16i32_nxv16i32_nxv16i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16i32_nxv16i32_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vloxei.nxv16i32.nxv16i8(
@@ -5070,7 +5148,8 @@ define <vscale x 1 x i64> @intrinsic_vloxei_v_nxv1i64_nxv1i64_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1i64_nxv1i64_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vloxei.nxv1i64.nxv1i8(
@@ -5114,7 +5193,8 @@ define <vscale x 2 x i64> @intrinsic_vloxei_v_nxv2i64_nxv2i64_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2i64_nxv2i64_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vloxei.nxv2i64.nxv2i8(
@@ -5158,7 +5238,8 @@ define <vscale x 4 x i64> @intrinsic_vloxei_v_nxv4i64_nxv4i64_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4i64_nxv4i64_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vloxei.nxv4i64.nxv4i8(
@@ -5202,7 +5283,8 @@ define <vscale x 8 x i64> @intrinsic_vloxei_v_nxv8i64_nxv8i64_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8i64_nxv8i64_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vloxei.nxv8i64.nxv8i8(
@@ -5246,7 +5328,8 @@ define <vscale x 1 x half> @intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f16_nxv1f16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vloxei.nxv1f16.nxv1i8(
@@ -5290,7 +5373,8 @@ define <vscale x 2 x half> @intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f16_nxv2f16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vloxei.nxv2f16.nxv2i8(
@@ -5334,7 +5418,8 @@ define <vscale x 4 x half> @intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f16_nxv4f16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vloxei.nxv4f16.nxv4i8(
@@ -5378,7 +5463,8 @@ define <vscale x 8 x half> @intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f16_nxv8f16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vloxei.nxv8f16.nxv8i8(
@@ -5422,7 +5508,8 @@ define <vscale x 16 x half> @intrinsic_vloxei_v_nxv16f16_nxv16f16_nxv16i8(<vscal
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16f16_nxv16f16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vloxei.nxv16f16.nxv16i8(
@@ -5466,7 +5553,8 @@ define <vscale x 32 x half> @intrinsic_vloxei_v_nxv32f16_nxv32f16_nxv32i8(<vscal
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv32f16_nxv32f16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vloxei.nxv32f16.nxv32i8(
@@ -5510,7 +5598,8 @@ define <vscale x 1 x float> @intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f32_nxv1f32_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vloxei.nxv1f32.nxv1i8(
@@ -5554,7 +5643,8 @@ define <vscale x 2 x float> @intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f32_nxv2f32_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vloxei.nxv2f32.nxv2i8(
@@ -5598,7 +5688,8 @@ define <vscale x 4 x float> @intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f32_nxv4f32_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vloxei.nxv4f32.nxv4i8(
@@ -5642,7 +5733,8 @@ define <vscale x 8 x float> @intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i8(<vscale x
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f32_nxv8f32_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vloxei.nxv8f32.nxv8i8(
@@ -5686,7 +5778,8 @@ define <vscale x 16 x float> @intrinsic_vloxei_v_nxv16f32_nxv16f32_nxv16i8(<vsca
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv16f32_nxv16f32_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vloxei.nxv16f32.nxv16i8(
@@ -5730,7 +5823,8 @@ define <vscale x 1 x double> @intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv1f64_nxv1f64_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vloxei.nxv1f64.nxv1i8(
@@ -5774,7 +5868,8 @@ define <vscale x 2 x double> @intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv2f64_nxv2f64_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vloxei.nxv2f64.nxv2i8(
@@ -5818,7 +5913,8 @@ define <vscale x 4 x double> @intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv4f64_nxv4f64_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vloxei.nxv4f64.nxv4i8(
@@ -5862,7 +5958,8 @@ define <vscale x 8 x double> @intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i8(<vscale
 ; CHECK-LABEL: intrinsic_vloxei_v_nxv8f64_nxv8f64_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vloxei8.v v8, (a0), v8
+; CHECK-NEXT:    vloxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vloxei.nxv8f64.nxv8i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vluxei-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vluxei-rv32.ll
index 8b218263bb4ab764b71f6f26a07842dd53c1dc6d..edc4855600b728a5a8f69f170f68928000a9fd57 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vluxei-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vluxei-rv32.ll
@@ -10,7 +10,8 @@ define <vscale x 1 x i8> @intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32(
@@ -54,7 +55,8 @@ define <vscale x 2 x i8> @intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32(
@@ -98,7 +100,8 @@ define <vscale x 4 x i8> @intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32(
@@ -142,7 +145,8 @@ define <vscale x 8 x i8> @intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32(
@@ -186,7 +190,8 @@ define <vscale x 16 x i8> @intrinsic_vluxei_v_nxv16i8_nxv16i8_nxv16i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i8_nxv16i8_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32(
@@ -230,7 +235,8 @@ define <vscale x 1 x i16> @intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32(
@@ -274,7 +280,8 @@ define <vscale x 2 x i16> @intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32(
@@ -318,7 +325,8 @@ define <vscale x 4 x i16> @intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32(
@@ -362,7 +370,8 @@ define <vscale x 8 x i16> @intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32(
@@ -406,7 +415,8 @@ define <vscale x 16 x i16> @intrinsic_vluxei_v_nxv16i16_nxv16i16_nxv16i32(<vscal
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i16_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32(
@@ -670,7 +680,8 @@ define <vscale x 1 x half> @intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vluxei.nxv1f16.nxv1i32(
@@ -714,7 +725,8 @@ define <vscale x 2 x half> @intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vluxei.nxv2f16.nxv2i32(
@@ -758,7 +770,8 @@ define <vscale x 4 x half> @intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vluxei.nxv4f16.nxv4i32(
@@ -802,7 +815,8 @@ define <vscale x 8 x half> @intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vluxei.nxv8f16.nxv8i32(
@@ -846,7 +860,8 @@ define <vscale x 16 x half> @intrinsic_vluxei_v_nxv16f16_nxv16f16_nxv16i32(<vsca
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16f16_nxv16f16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vluxei.nxv16f16.nxv16i32(
@@ -1110,7 +1125,8 @@ define <vscale x 1 x double> @intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i32(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i32(
@@ -1154,7 +1170,8 @@ define <vscale x 2 x double> @intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i32(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i32(
@@ -1198,7 +1215,8 @@ define <vscale x 4 x double> @intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i32(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i32(
@@ -1242,7 +1260,8 @@ define <vscale x 8 x double> @intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i32(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i32(
@@ -1286,7 +1305,8 @@ define <vscale x 1 x i8> @intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16(
@@ -1330,7 +1350,8 @@ define <vscale x 2 x i8> @intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16(
@@ -1374,7 +1395,8 @@ define <vscale x 4 x i8> @intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16(
@@ -1418,7 +1440,8 @@ define <vscale x 8 x i8> @intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16(
@@ -1462,7 +1485,8 @@ define <vscale x 16 x i8> @intrinsic_vluxei_v_nxv16i8_nxv16i8_nxv16i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i8_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16(
@@ -1506,7 +1530,8 @@ define <vscale x 32 x i8> @intrinsic_vluxei_v_nxv32i8_nxv32i8_nxv32i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv32i8_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16(
@@ -1814,7 +1839,8 @@ define <vscale x 1 x i32> @intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16(
@@ -1858,7 +1884,8 @@ define <vscale x 2 x i32> @intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16(
@@ -1902,7 +1929,8 @@ define <vscale x 4 x i32> @intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16(
@@ -1946,7 +1974,8 @@ define <vscale x 8 x i32> @intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16(
@@ -1990,7 +2019,8 @@ define <vscale x 16 x i32> @intrinsic_vluxei_v_nxv16i32_nxv16i32_nxv16i16(<vscal
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i32_nxv16i32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16(
@@ -2298,7 +2328,8 @@ define <vscale x 1 x float> @intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i16(
@@ -2342,7 +2373,8 @@ define <vscale x 2 x float> @intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i16(
@@ -2386,7 +2418,8 @@ define <vscale x 4 x float> @intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i16(
@@ -2430,7 +2463,8 @@ define <vscale x 8 x float> @intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i16(
@@ -2474,7 +2508,8 @@ define <vscale x 16 x float> @intrinsic_vluxei_v_nxv16f32_nxv16f32_nxv16i16(<vsc
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16f32_nxv16f32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i16(
@@ -2518,7 +2553,8 @@ define <vscale x 1 x double> @intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i16(
@@ -2562,7 +2598,8 @@ define <vscale x 2 x double> @intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i16(
@@ -2606,7 +2643,8 @@ define <vscale x 4 x double> @intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i16(
@@ -2650,7 +2688,8 @@ define <vscale x 8 x double> @intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i16(
@@ -3002,7 +3041,8 @@ define <vscale x 1 x i16> @intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8(
@@ -3046,7 +3086,8 @@ define <vscale x 2 x i16> @intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8(
@@ -3090,7 +3131,8 @@ define <vscale x 4 x i16> @intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8(
@@ -3134,7 +3176,8 @@ define <vscale x 8 x i16> @intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8(
@@ -3178,7 +3221,8 @@ define <vscale x 16 x i16> @intrinsic_vluxei_v_nxv16i16_nxv16i16_nxv16i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i16_nxv16i16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8(
@@ -3222,7 +3266,8 @@ define <vscale x 32 x i16> @intrinsic_vluxei_v_nxv32i16_nxv32i16_nxv32i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv32i16_nxv32i16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8(
@@ -3266,7 +3311,8 @@ define <vscale x 1 x i32> @intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8(
@@ -3310,7 +3356,8 @@ define <vscale x 2 x i32> @intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8(
@@ -3354,7 +3401,8 @@ define <vscale x 4 x i32> @intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8(
@@ -3398,7 +3446,8 @@ define <vscale x 8 x i32> @intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8(
@@ -3442,7 +3491,8 @@ define <vscale x 16 x i32> @intrinsic_vluxei_v_nxv16i32_nxv16i32_nxv16i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i32_nxv16i32_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8(
@@ -3486,7 +3536,8 @@ define <vscale x 1 x half> @intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vluxei.nxv1f16.nxv1i8(
@@ -3530,7 +3581,8 @@ define <vscale x 2 x half> @intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vluxei.nxv2f16.nxv2i8(
@@ -3574,7 +3626,8 @@ define <vscale x 4 x half> @intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vluxei.nxv4f16.nxv4i8(
@@ -3618,7 +3671,8 @@ define <vscale x 8 x half> @intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vluxei.nxv8f16.nxv8i8(
@@ -3662,7 +3716,8 @@ define <vscale x 16 x half> @intrinsic_vluxei_v_nxv16f16_nxv16f16_nxv16i8(<vscal
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16f16_nxv16f16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vluxei.nxv16f16.nxv16i8(
@@ -3706,7 +3761,8 @@ define <vscale x 32 x half> @intrinsic_vluxei_v_nxv32f16_nxv32f16_nxv32i8(<vscal
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv32f16_nxv32f16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vluxei.nxv32f16.nxv32i8(
@@ -3750,7 +3806,8 @@ define <vscale x 1 x float> @intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i8(
@@ -3794,7 +3851,8 @@ define <vscale x 2 x float> @intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i8(
@@ -3838,7 +3896,8 @@ define <vscale x 4 x float> @intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i8(
@@ -3882,7 +3941,8 @@ define <vscale x 8 x float> @intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i8(
@@ -3926,7 +3986,8 @@ define <vscale x 16 x float> @intrinsic_vluxei_v_nxv16f32_nxv16f32_nxv16i8(<vsca
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16f32_nxv16f32_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i8(
@@ -3970,7 +4031,8 @@ define <vscale x 1 x double> @intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i8(
@@ -4014,7 +4076,8 @@ define <vscale x 2 x double> @intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i8(
@@ -4058,7 +4121,8 @@ define <vscale x 4 x double> @intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i8(
@@ -4102,7 +4166,8 @@ define <vscale x 8 x double> @intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vluxei-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vluxei-rv64.ll
index cef3916834acc8b0f3c1549e0f6c28ddaa1fe28c..77136f2ef1ac0daa6e350ee2af926bf86665760e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vluxei-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vluxei-rv64.ll
@@ -10,7 +10,8 @@ define <vscale x 1 x i8> @intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i64(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i64(
@@ -54,7 +55,8 @@ define <vscale x 2 x i8> @intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i64(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i64(
@@ -98,7 +100,8 @@ define <vscale x 4 x i8> @intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i64(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i64(
@@ -142,7 +145,8 @@ define <vscale x 8 x i8> @intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i64(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i64(
@@ -186,7 +190,8 @@ define <vscale x 1 x i16> @intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i64(
@@ -230,7 +235,8 @@ define <vscale x 2 x i16> @intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i64(
@@ -274,7 +280,8 @@ define <vscale x 4 x i16> @intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i64(
@@ -318,7 +325,8 @@ define <vscale x 8 x i16> @intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i64(
@@ -362,7 +370,8 @@ define <vscale x 1 x i32> @intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i64(
@@ -406,7 +415,8 @@ define <vscale x 2 x i32> @intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i64(
@@ -450,7 +460,8 @@ define <vscale x 4 x i32> @intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i64(
@@ -494,7 +505,8 @@ define <vscale x 8 x i32> @intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i64(
@@ -714,7 +726,8 @@ define <vscale x 1 x half> @intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vluxei.nxv1f16.nxv1i64(
@@ -758,7 +771,8 @@ define <vscale x 2 x half> @intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vluxei.nxv2f16.nxv2i64(
@@ -802,7 +816,8 @@ define <vscale x 4 x half> @intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vluxei.nxv4f16.nxv4i64(
@@ -846,7 +861,8 @@ define <vscale x 8 x half> @intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i64(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vluxei.nxv8f16.nxv8i64(
@@ -890,7 +906,8 @@ define <vscale x 1 x float> @intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i64(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i64(
@@ -934,7 +951,8 @@ define <vscale x 2 x float> @intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i64(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i64(
@@ -978,7 +996,8 @@ define <vscale x 4 x float> @intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i64(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i64(
@@ -1022,7 +1041,8 @@ define <vscale x 8 x float> @intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i64(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei64.v v8, (a0), v8
+; CHECK-NEXT:    vluxei64.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i64(
@@ -1242,7 +1262,8 @@ define <vscale x 1 x i8> @intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i32(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i32(
@@ -1286,7 +1307,8 @@ define <vscale x 2 x i8> @intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i32(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i32(
@@ -1330,7 +1352,8 @@ define <vscale x 4 x i8> @intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i32(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i32(
@@ -1374,7 +1397,8 @@ define <vscale x 8 x i8> @intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i32(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i32(
@@ -1418,7 +1442,8 @@ define <vscale x 16 x i8> @intrinsic_vluxei_v_nxv16i8_nxv16i8_nxv16i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i8_nxv16i8_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i32(
@@ -1462,7 +1487,8 @@ define <vscale x 1 x i16> @intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i32(
@@ -1506,7 +1532,8 @@ define <vscale x 2 x i16> @intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i32(
@@ -1550,7 +1577,8 @@ define <vscale x 4 x i16> @intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i32(
@@ -1594,7 +1622,8 @@ define <vscale x 8 x i16> @intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i32(
@@ -1638,7 +1667,8 @@ define <vscale x 16 x i16> @intrinsic_vluxei_v_nxv16i16_nxv16i16_nxv16i32(<vscal
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i16_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i32(
@@ -1902,7 +1932,8 @@ define <vscale x 1 x i64> @intrinsic_vluxei_v_nxv1i64_nxv1i64_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i64_nxv1i64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i32(
@@ -1946,7 +1977,8 @@ define <vscale x 2 x i64> @intrinsic_vluxei_v_nxv2i64_nxv2i64_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i64_nxv2i64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i32(
@@ -1990,7 +2022,8 @@ define <vscale x 4 x i64> @intrinsic_vluxei_v_nxv4i64_nxv4i64_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i64_nxv4i64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i32(
@@ -2034,7 +2067,8 @@ define <vscale x 8 x i64> @intrinsic_vluxei_v_nxv8i64_nxv8i64_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i64_nxv8i64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i32(
@@ -2078,7 +2112,8 @@ define <vscale x 1 x half> @intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vluxei.nxv1f16.nxv1i32(
@@ -2122,7 +2157,8 @@ define <vscale x 2 x half> @intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vluxei.nxv2f16.nxv2i32(
@@ -2166,7 +2202,8 @@ define <vscale x 4 x half> @intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vluxei.nxv4f16.nxv4i32(
@@ -2210,7 +2247,8 @@ define <vscale x 8 x half> @intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i32(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vluxei.nxv8f16.nxv8i32(
@@ -2254,7 +2292,8 @@ define <vscale x 16 x half> @intrinsic_vluxei_v_nxv16f16_nxv16f16_nxv16i32(<vsca
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16f16_nxv16f16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vluxei.nxv16f16.nxv16i32(
@@ -2518,7 +2557,8 @@ define <vscale x 1 x double> @intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i32(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i32(
@@ -2562,7 +2602,8 @@ define <vscale x 2 x double> @intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i32(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i32(
@@ -2606,7 +2647,8 @@ define <vscale x 4 x double> @intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i32(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i32(
@@ -2650,7 +2692,8 @@ define <vscale x 8 x double> @intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i32(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vluxei32.v v8, (a0), v8
+; CHECK-NEXT:    vluxei32.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i32(
@@ -2694,7 +2737,8 @@ define <vscale x 1 x i8> @intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i16(<vscale x 1 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i8_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vluxei.nxv1i8.nxv1i16(
@@ -2738,7 +2782,8 @@ define <vscale x 2 x i8> @intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i16(<vscale x 2 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i8_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vluxei.nxv2i8.nxv2i16(
@@ -2782,7 +2827,8 @@ define <vscale x 4 x i8> @intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i16(<vscale x 4 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i8_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vluxei.nxv4i8.nxv4i16(
@@ -2826,7 +2872,8 @@ define <vscale x 8 x i8> @intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i16(<vscale x 8 x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i8_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vluxei.nxv8i8.nxv8i16(
@@ -2870,7 +2917,8 @@ define <vscale x 16 x i8> @intrinsic_vluxei_v_nxv16i8_nxv16i8_nxv16i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i8_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vluxei.nxv16i8.nxv16i16(
@@ -2914,7 +2962,8 @@ define <vscale x 32 x i8> @intrinsic_vluxei_v_nxv32i8_nxv32i8_nxv32i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv32i8_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i8> @llvm.riscv.vluxei.nxv32i8.nxv32i16(
@@ -3222,7 +3271,8 @@ define <vscale x 1 x i32> @intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i16(
@@ -3266,7 +3316,8 @@ define <vscale x 2 x i32> @intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i16(
@@ -3310,7 +3361,8 @@ define <vscale x 4 x i32> @intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i16(
@@ -3354,7 +3406,8 @@ define <vscale x 8 x i32> @intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i16(
@@ -3398,7 +3451,8 @@ define <vscale x 16 x i32> @intrinsic_vluxei_v_nxv16i32_nxv16i32_nxv16i16(<vscal
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i32_nxv16i32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i16(
@@ -3442,7 +3496,8 @@ define <vscale x 1 x i64> @intrinsic_vluxei_v_nxv1i64_nxv1i64_nxv1i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i64_nxv1i64_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i16(
@@ -3486,7 +3541,8 @@ define <vscale x 2 x i64> @intrinsic_vluxei_v_nxv2i64_nxv2i64_nxv2i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i64_nxv2i64_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i16(
@@ -3530,7 +3586,8 @@ define <vscale x 4 x i64> @intrinsic_vluxei_v_nxv4i64_nxv4i64_nxv4i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i64_nxv4i64_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i16(
@@ -3574,7 +3631,8 @@ define <vscale x 8 x i64> @intrinsic_vluxei_v_nxv8i64_nxv8i64_nxv8i16(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i64_nxv8i64_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i16(
@@ -3882,7 +3940,8 @@ define <vscale x 1 x float> @intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i16(
@@ -3926,7 +3985,8 @@ define <vscale x 2 x float> @intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i16(
@@ -3970,7 +4030,8 @@ define <vscale x 4 x float> @intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i16(
@@ -4014,7 +4075,8 @@ define <vscale x 8 x float> @intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i16(
@@ -4058,7 +4120,8 @@ define <vscale x 16 x float> @intrinsic_vluxei_v_nxv16f32_nxv16f32_nxv16i16(<vsc
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16f32_nxv16f32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i16(
@@ -4102,7 +4165,8 @@ define <vscale x 1 x double> @intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i16(
@@ -4146,7 +4210,8 @@ define <vscale x 2 x double> @intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i16(
@@ -4190,7 +4255,8 @@ define <vscale x 4 x double> @intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i16(
@@ -4234,7 +4300,8 @@ define <vscale x 8 x double> @intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i16(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vluxei16.v v8, (a0), v8
+; CHECK-NEXT:    vluxei16.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i16(
@@ -4586,7 +4653,8 @@ define <vscale x 1 x i16> @intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i16_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vluxei.nxv1i16.nxv1i8(
@@ -4630,7 +4698,8 @@ define <vscale x 2 x i16> @intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i16_nxv2i16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vluxei.nxv2i16.nxv2i8(
@@ -4674,7 +4743,8 @@ define <vscale x 4 x i16> @intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i16_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vluxei.nxv4i16.nxv4i8(
@@ -4718,7 +4788,8 @@ define <vscale x 8 x i16> @intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i16_nxv8i16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vluxei.nxv8i16.nxv8i8(
@@ -4762,7 +4833,8 @@ define <vscale x 16 x i16> @intrinsic_vluxei_v_nxv16i16_nxv16i16_nxv16i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i16_nxv16i16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vluxei.nxv16i16.nxv16i8(
@@ -4806,7 +4878,8 @@ define <vscale x 32 x i16> @intrinsic_vluxei_v_nxv32i16_nxv32i16_nxv32i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv32i16_nxv32i16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i16> @llvm.riscv.vluxei.nxv32i16.nxv32i8(
@@ -4850,7 +4923,8 @@ define <vscale x 1 x i32> @intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i32_nxv1i32_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vluxei.nxv1i32.nxv1i8(
@@ -4894,7 +4968,8 @@ define <vscale x 2 x i32> @intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i32_nxv2i32_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vluxei.nxv2i32.nxv2i8(
@@ -4938,7 +5013,8 @@ define <vscale x 4 x i32> @intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i32_nxv4i32_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vluxei.nxv4i32.nxv4i8(
@@ -4982,7 +5058,8 @@ define <vscale x 8 x i32> @intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i32_nxv8i32_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vluxei.nxv8i32.nxv8i8(
@@ -5026,7 +5103,8 @@ define <vscale x 16 x i32> @intrinsic_vluxei_v_nxv16i32_nxv16i32_nxv16i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16i32_nxv16i32_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vluxei.nxv16i32.nxv16i8(
@@ -5070,7 +5148,8 @@ define <vscale x 1 x i64> @intrinsic_vluxei_v_nxv1i64_nxv1i64_nxv1i8(<vscale x 1
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1i64_nxv1i64_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vluxei.nxv1i64.nxv1i8(
@@ -5114,7 +5193,8 @@ define <vscale x 2 x i64> @intrinsic_vluxei_v_nxv2i64_nxv2i64_nxv2i8(<vscale x 2
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2i64_nxv2i64_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vluxei.nxv2i64.nxv2i8(
@@ -5158,7 +5238,8 @@ define <vscale x 4 x i64> @intrinsic_vluxei_v_nxv4i64_nxv4i64_nxv4i8(<vscale x 4
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4i64_nxv4i64_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vluxei.nxv4i64.nxv4i8(
@@ -5202,7 +5283,8 @@ define <vscale x 8 x i64> @intrinsic_vluxei_v_nxv8i64_nxv8i64_nxv8i8(<vscale x 8
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8i64_nxv8i64_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vluxei.nxv8i64.nxv8i8(
@@ -5246,7 +5328,8 @@ define <vscale x 1 x half> @intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f16_nxv1f16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vluxei.nxv1f16.nxv1i8(
@@ -5290,7 +5373,8 @@ define <vscale x 2 x half> @intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f16_nxv2f16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vluxei.nxv2f16.nxv2i8(
@@ -5334,7 +5418,8 @@ define <vscale x 4 x half> @intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f16_nxv4f16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vluxei.nxv4f16.nxv4i8(
@@ -5378,7 +5463,8 @@ define <vscale x 8 x half> @intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f16_nxv8f16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vluxei.nxv8f16.nxv8i8(
@@ -5422,7 +5508,8 @@ define <vscale x 16 x half> @intrinsic_vluxei_v_nxv16f16_nxv16f16_nxv16i8(<vscal
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16f16_nxv16f16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vluxei.nxv16f16.nxv16i8(
@@ -5466,7 +5553,8 @@ define <vscale x 32 x half> @intrinsic_vluxei_v_nxv32f16_nxv32f16_nxv32i8(<vscal
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv32f16_nxv32f16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vluxei.nxv32f16.nxv32i8(
@@ -5510,7 +5598,8 @@ define <vscale x 1 x float> @intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f32_nxv1f32_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vluxei.nxv1f32.nxv1i8(
@@ -5554,7 +5643,8 @@ define <vscale x 2 x float> @intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f32_nxv2f32_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vluxei.nxv2f32.nxv2i8(
@@ -5598,7 +5688,8 @@ define <vscale x 4 x float> @intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f32_nxv4f32_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vluxei.nxv4f32.nxv4i8(
@@ -5642,7 +5733,8 @@ define <vscale x 8 x float> @intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i8(<vscale x
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f32_nxv8f32_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vluxei.nxv8f32.nxv8i8(
@@ -5686,7 +5778,8 @@ define <vscale x 16 x float> @intrinsic_vluxei_v_nxv16f32_nxv16f32_nxv16i8(<vsca
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv16f32_nxv16f32_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vluxei.nxv16f32.nxv16i8(
@@ -5730,7 +5823,8 @@ define <vscale x 1 x double> @intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv1f64_nxv1f64_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v25, (a0), v8
+; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vluxei.nxv1f64.nxv1i8(
@@ -5774,7 +5868,8 @@ define <vscale x 2 x double> @intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv2f64_nxv2f64_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v26, (a0), v8
+; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vluxei.nxv2f64.nxv2i8(
@@ -5818,7 +5913,8 @@ define <vscale x 4 x double> @intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv4f64_nxv4f64_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v28, (a0), v8
+; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vluxei.nxv4f64.nxv4i8(
@@ -5862,7 +5958,8 @@ define <vscale x 8 x double> @intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i8(<vscale
 ; CHECK-LABEL: intrinsic_vluxei_v_nxv8f64_nxv8f64_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
-; CHECK-NEXT:    vluxei8.v v8, (a0), v8
+; CHECK-NEXT:    vluxei8.v v16, (a0), v8
+; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vluxei.nxv8f64.nxv8i8(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmslt-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vmslt-rv32.ll
index 894a232a167dbd619c64505cf4905ca963df45e2..a51949573c9790a334d6746ebc93f5595dbd329d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmslt-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmslt-rv32.ll
@@ -1504,9 +1504,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -15, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -15, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i8.i8(
@@ -1537,9 +1539,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -13, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -13, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i8.i8(
@@ -1570,9 +1574,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -11, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -11, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i8.i8(
@@ -1603,9 +1609,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -9, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmslt.mask.nxv8i8.i8(
@@ -1636,9 +1644,11 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmslt_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
-; CHECK-NEXT:    vmsle.vi v10, v8, -7, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsle.vi v25, v8, -7, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i1> @llvm.riscv.vmslt.mask.nxv16i8.i8(
@@ -1669,9 +1679,11 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmslt_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
-; CHECK-NEXT:    vmsle.vi v12, v8, -5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsle.vi v25, v8, -5, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i1> @llvm.riscv.vmslt.mask.nxv32i8.i8(
@@ -1702,9 +1714,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -3, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -3, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i16.i16(
@@ -1735,9 +1749,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -1, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i16.i16(
@@ -1768,9 +1784,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, 0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, 0, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i16.i16(
@@ -1801,9 +1819,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
-; CHECK-NEXT:    vmsle.vi v10, v8, 2, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsle.vi v25, v8, 2, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmslt.mask.nxv8i16.i16(
@@ -1834,9 +1854,11 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmslt_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
-; CHECK-NEXT:    vmsle.vi v12, v8, 4, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsle.vi v25, v8, 4, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i1> @llvm.riscv.vmslt.mask.nxv16i16.i16(
@@ -1867,9 +1889,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, 6, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, 6, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i32.i32(
@@ -1900,9 +1924,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, 8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, 8, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i32.i32(
@@ -1933,9 +1959,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
-; CHECK-NEXT:    vmsle.vi v10, v8, 10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsle.vi v25, v8, 10, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i32.i32(
@@ -1966,9 +1994,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
-; CHECK-NEXT:    vmsle.vi v12, v8, 12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsle.vi v25, v8, 12, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmslt.mask.nxv8i32.i32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmslt-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vmslt-rv64.ll
index 7802c2a84ff817a2c726be940133d13fa5b3024e..c75f37c440d503af8dca1d05861d43f55bebb664 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmslt-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmslt-rv64.ll
@@ -1801,9 +1801,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -15, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -15, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i8.i8(
@@ -1834,9 +1836,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -13, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -13, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i8.i8(
@@ -1867,9 +1871,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -11, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -11, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i8.i8(
@@ -1900,9 +1906,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -9, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmslt.mask.nxv8i8.i8(
@@ -1933,9 +1941,11 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmslt_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
-; CHECK-NEXT:    vmsle.vi v10, v8, -7, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsle.vi v25, v8, -7, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i1> @llvm.riscv.vmslt.mask.nxv16i8.i8(
@@ -1966,9 +1976,11 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmslt_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
-; CHECK-NEXT:    vmsle.vi v12, v8, -5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsle.vi v25, v8, -5, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i1> @llvm.riscv.vmslt.mask.nxv32i8.i8(
@@ -1999,9 +2011,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -3, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -3, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i16.i16(
@@ -2032,9 +2046,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, -1, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i16.i16(
@@ -2065,9 +2081,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, 0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, 0, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i16.i16(
@@ -2098,9 +2116,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
-; CHECK-NEXT:    vmsle.vi v10, v8, 2, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsle.vi v25, v8, 2, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmslt.mask.nxv8i16.i16(
@@ -2131,9 +2151,11 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmslt_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
-; CHECK-NEXT:    vmsle.vi v12, v8, 4, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsle.vi v25, v8, 4, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i1> @llvm.riscv.vmslt.mask.nxv16i16.i16(
@@ -2164,9 +2186,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, 6, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, 6, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i32.i32(
@@ -2197,9 +2221,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, 8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, 8, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i32.i32(
@@ -2230,9 +2256,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
-; CHECK-NEXT:    vmsle.vi v10, v8, 10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsle.vi v25, v8, 10, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i32.i32(
@@ -2263,9 +2291,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmslt_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
-; CHECK-NEXT:    vmsle.vi v12, v8, 12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsle.vi v25, v8, 12, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmslt.mask.nxv8i32.i32(
@@ -2296,9 +2326,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmslt_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-NEXT:    vmsle.vi v9, v8, 14, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsle.vi v25, v8, 14, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmslt.mask.nxv1i64.i64(
@@ -2329,9 +2361,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmslt_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
-; CHECK-NEXT:    vmsle.vi v10, v8, -16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsle.vi v25, v8, -16, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmslt.mask.nxv2i64.i64(
@@ -2362,9 +2396,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmslt_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmslt_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e64,m4,tu,mu
-; CHECK-NEXT:    vmsle.vi v12, v8, -14, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsle.vi v25, v8, -14, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmslt.mask.nxv4i64.i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsltu-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vmsltu-rv32.ll
index 8cd17ead0234a8113b4520f6fef061d80e95267c..ca78411b24c246c2028afb3f4369a6ed8792259b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsltu-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsltu-rv32.ll
@@ -1504,9 +1504,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -15, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -15, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i8.i8(
@@ -1537,9 +1539,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -13, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -13, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i8.i8(
@@ -1570,9 +1574,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -11, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -11, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i8.i8(
@@ -1603,9 +1609,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -9, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmsltu.mask.nxv8i8.i8(
@@ -1636,9 +1644,11 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v10, v8, -7, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsleu.vi v25, v8, -7, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i1> @llvm.riscv.vmsltu.mask.nxv16i8.i8(
@@ -1669,9 +1679,11 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsltu_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v12, v8, -5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsleu.vi v25, v8, -5, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i1> @llvm.riscv.vmsltu.mask.nxv32i8.i8(
@@ -1702,9 +1714,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -3, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -3, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i16.i16(
@@ -1735,9 +1749,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
-; CHECK-NEXT:    vmsne.vv v9, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsne.vv v25, v8, v8, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i16.i16(
@@ -1768,9 +1784,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, 0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, 0, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i16.i16(
@@ -1801,9 +1819,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v10, v8, 2, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsleu.vi v25, v8, 2, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmsltu.mask.nxv8i16.i16(
@@ -1834,9 +1854,11 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v12, v8, 4, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsleu.vi v25, v8, 4, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i1> @llvm.riscv.vmsltu.mask.nxv16i16.i16(
@@ -1867,9 +1889,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, 6, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, 6, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i32.i32(
@@ -1900,9 +1924,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, 8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, 8, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i32.i32(
@@ -1933,9 +1959,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v10, v8, 10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsleu.vi v25, v8, 10, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i32.i32(
@@ -1966,9 +1994,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v12, v8, 12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsleu.vi v25, v8, 12, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmsltu.mask.nxv8i32.i32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmsltu-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vmsltu-rv64.ll
index 83dc531750eede27c34e9bf525d04242ab375a6b..a145dd684bacd5a5bb5347b6222f6acd25d4b8db 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmsltu-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmsltu-rv64.ll
@@ -1801,9 +1801,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i8_i8(<vscale x 1 x i1> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -15, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -15, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i8.i8(
@@ -1834,9 +1836,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i8_i8(<vscale x 2 x i1> %0, <vscale x 2 x i8> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -13, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -13, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i8.i8(
@@ -1867,9 +1871,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i8_i8(<vscale x 4 x i1> %0, <vscale x 4 x i8> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -11, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -11, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i8.i8(
@@ -1900,9 +1906,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vi_nxv8i8_i8(<vscale x 8 x i1> %0, <vscale x 8 x i8> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -9, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -9, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmsltu.mask.nxv8i8.i8(
@@ -1933,9 +1941,11 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vi_nxv16i8_i8(<vscale x 16 x i1> %0, <vscale x 16 x i8> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v10, v8, -7, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsleu.vi v25, v8, -7, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i1> @llvm.riscv.vmsltu.mask.nxv16i8.i8(
@@ -1966,9 +1976,11 @@ entry:
 define <vscale x 32 x i1> @intrinsic_vmsltu_mask_vi_nxv32i8_i8(<vscale x 32 x i1> %0, <vscale x 32 x i8> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v12, v8, -5, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsleu.vi v25, v8, -5, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 32 x i1> @llvm.riscv.vmsltu.mask.nxv32i8.i8(
@@ -1999,9 +2011,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i16_i16(<vscale x 1 x i1> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, -3, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, -3, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i16.i16(
@@ -2032,9 +2046,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i16_i16(<vscale x 2 x i1> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
-; CHECK-NEXT:    vmsne.vv v9, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsne.vv v25, v8, v8, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i16.i16(
@@ -2065,9 +2081,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i16_i16(<vscale x 4 x i1> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, 0, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, 0, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i16.i16(
@@ -2098,9 +2116,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vi_nxv8i16_i16(<vscale x 8 x i1> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v10, v8, 2, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsleu.vi v25, v8, 2, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmsltu.mask.nxv8i16.i16(
@@ -2131,9 +2151,11 @@ entry:
 define <vscale x 16 x i1> @intrinsic_vmsltu_mask_vi_nxv16i16_i16(<vscale x 16 x i1> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v12, v8, 4, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsleu.vi v25, v8, 4, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 16 x i1> @llvm.riscv.vmsltu.mask.nxv16i16.i16(
@@ -2164,9 +2186,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i32_i32(<vscale x 1 x i1> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, 6, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, 6, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i32.i32(
@@ -2197,9 +2221,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i32_i32(<vscale x 2 x i1> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, 8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, 8, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i32.i32(
@@ -2230,9 +2256,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i32_i32(<vscale x 4 x i1> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v10, v8, 10, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsleu.vi v25, v8, 10, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i32.i32(
@@ -2263,9 +2291,11 @@ entry:
 define <vscale x 8 x i1> @intrinsic_vmsltu_mask_vi_nxv8i32_i32(<vscale x 8 x i1> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v12, v8, 12, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsleu.vi v25, v8, 12, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 8 x i1> @llvm.riscv.vmsltu.mask.nxv8i32.i32(
@@ -2296,9 +2326,11 @@ entry:
 define <vscale x 1 x i1> @intrinsic_vmsltu_mask_vi_nxv1i64_i64(<vscale x 1 x i1> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-NEXT:    vmsleu.vi v9, v8, 14, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmsleu.vi v25, v8, 14, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 1 x i1> @llvm.riscv.vmsltu.mask.nxv1i64.i64(
@@ -2329,9 +2361,11 @@ entry:
 define <vscale x 2 x i1> @intrinsic_vmsltu_mask_vi_nxv2i64_i64(<vscale x 2 x i1> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
-; CHECK-NEXT:    vmsleu.vi v10, v8, -16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v10
+; CHECK-NEXT:    vmsleu.vi v25, v8, -16, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 2 x i1> @llvm.riscv.vmsltu.mask.nxv2i64.i64(
@@ -2362,9 +2396,11 @@ entry:
 define <vscale x 4 x i1> @intrinsic_vmsltu_mask_vi_nxv4i64_i64(<vscale x 4 x i1> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vi_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v25, v0
 ; CHECK-NEXT:    vsetvli a0, a0, e64,m4,tu,mu
-; CHECK-NEXT:    vmsleu.vi v12, v8, -14, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vmsleu.vi v25, v8, -14, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
   %a = call <vscale x 4 x i1> @llvm.riscv.vmsltu.mask.nxv4i64.i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnclip-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vnclip-rv32.ll
index faafb67c43b9731500b09f00ff24f7fb0af32aa3..c59e3656c5cc7012a11704fac3a008e0572d631f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnclip-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnclip-rv32.ll
@@ -496,858 +496,858 @@ entry:
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16(
   <vscale x 1 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vnclip_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclip_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16(
   <vscale x 1 x i8>,
   <vscale x 1 x i16>,
-  i8,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vnclip_mask_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclip_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16(
   <vscale x 2 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vnclip_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclip_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16(
   <vscale x 2 x i8>,
   <vscale x 2 x i16>,
-  i8,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vnclip_mask_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclip_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16(
   <vscale x 4 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vnclip_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclip_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16(
   <vscale x 4 x i8>,
   <vscale x 4 x i16>,
-  i8,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vnclip_mask_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclip_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16(
   <vscale x 8 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vnclip_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclip_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16(
   <vscale x 8 x i8>,
   <vscale x 8 x i16>,
-  i8,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vnclip_mask_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclip_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16(
   <vscale x 16 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vnclip_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclip_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16(
   <vscale x 16 x i8>,
   <vscale x 16 x i16>,
-  i8,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vnclip_mask_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclip_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16(
   <vscale x 32 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vnclip_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclip_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16(
   <vscale x 32 x i8>,
   <vscale x 32 x i16>,
-  i8,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vnclip_mask_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclip_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32(
   <vscale x 1 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vnclip_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclip_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32(
   <vscale x 1 x i16>,
   <vscale x 1 x i32>,
-  i16,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vnclip_mask_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclip_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32(
   <vscale x 2 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vnclip_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclip_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32(
   <vscale x 2 x i16>,
   <vscale x 2 x i32>,
-  i16,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vnclip_mask_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclip_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32(
   <vscale x 4 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vnclip_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclip_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32(
   <vscale x 4 x i16>,
   <vscale x 4 x i32>,
-  i16,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vnclip_mask_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclip_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32(
   <vscale x 8 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vnclip_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclip_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32(
   <vscale x 8 x i16>,
   <vscale x 8 x i32>,
-  i16,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vnclip_mask_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclip_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32(
   <vscale x 16 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vnclip_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclip_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32(
   <vscale x 16 x i16>,
   <vscale x 16 x i32>,
-  i16,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vnclip_mask_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclip_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnclip_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclip_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnclip_mask_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclip_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnclip_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclip_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnclip_mask_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclip_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnclip_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclip_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnclip_mask_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclip_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnclip_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclip_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnclip_mask_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclip_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnclip_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclip_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnclip_mask_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclip_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnclip_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclip_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnclip_mask_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclip_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnclip_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclip_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnclip_mask_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclip_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnclip_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclip_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnclip_mask_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclip_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnclip_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclip_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnclip_mask_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclip_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnclip_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclip_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnclip_mask_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclip_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnclip_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclip_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnclip_mask_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclip_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnclip-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vnclip-rv64.ll
index 88d41f660048561b7df7a82f27bf4e0e5e05d411..9a9644ea46c96daa68fff55901a117cc0b5d3373 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnclip-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnclip-rv64.ll
@@ -676,1170 +676,1170 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16(
   <vscale x 1 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vnclip_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclip_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16(
   <vscale x 1 x i8>,
   <vscale x 1 x i16>,
-  i8,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vnclip_mask_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclip_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16(
   <vscale x 2 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vnclip_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclip_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16(
   <vscale x 2 x i8>,
   <vscale x 2 x i16>,
-  i8,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vnclip_mask_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclip_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16(
   <vscale x 4 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vnclip_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclip_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16(
   <vscale x 4 x i8>,
   <vscale x 4 x i16>,
-  i8,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vnclip_mask_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclip_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16(
   <vscale x 8 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vnclip_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclip_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16(
   <vscale x 8 x i8>,
   <vscale x 8 x i16>,
-  i8,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vnclip_mask_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclip_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16(
   <vscale x 16 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vnclip_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclip_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16(
   <vscale x 16 x i8>,
   <vscale x 16 x i16>,
-  i8,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vnclip_mask_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclip_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16(
   <vscale x 32 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vnclip_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclip_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16(
   <vscale x 32 x i8>,
   <vscale x 32 x i16>,
-  i8,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vnclip_mask_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclip_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32(
   <vscale x 1 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vnclip_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclip_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32(
   <vscale x 1 x i16>,
   <vscale x 1 x i32>,
-  i16,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vnclip_mask_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclip_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32(
   <vscale x 2 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vnclip_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclip_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32(
   <vscale x 2 x i16>,
   <vscale x 2 x i32>,
-  i16,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vnclip_mask_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclip_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32(
   <vscale x 4 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vnclip_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclip_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32(
   <vscale x 4 x i16>,
   <vscale x 4 x i32>,
-  i16,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vnclip_mask_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclip_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32(
   <vscale x 8 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vnclip_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclip_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32(
   <vscale x 8 x i16>,
   <vscale x 8 x i32>,
-  i16,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vnclip_mask_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclip_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32(
   <vscale x 16 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vnclip_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclip_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32(
   <vscale x 16 x i16>,
   <vscale x 16 x i32>,
-  i16,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vnclip_mask_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclip_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnclip.nxv1i32.nxv1i64.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vnclip.nxv1i32.nxv1i64(
   <vscale x 1 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vnclip_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnclip_vx_nxv1i32_nxv1i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnclip.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnclip.nxv1i32.nxv1i64(
     <vscale x 1 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnclip.mask.nxv1i32.nxv1i64.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vnclip.mask.nxv1i32.nxv1i64(
   <vscale x 1 x i32>,
   <vscale x 1 x i64>,
-  i32,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vnclip_mask_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnclip_mask_vx_nxv1i32_nxv1i64(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnclip.mask.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnclip.mask.nxv1i32.nxv1i64(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vnclip.nxv2i32.nxv2i64.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vnclip.nxv2i32.nxv2i64(
   <vscale x 2 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vnclip_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnclip_vx_nxv2i32_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnclip.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnclip.nxv2i32.nxv2i64(
     <vscale x 2 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vnclip.mask.nxv2i32.nxv2i64.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vnclip.mask.nxv2i32.nxv2i64(
   <vscale x 2 x i32>,
   <vscale x 2 x i64>,
-  i32,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vnclip_mask_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnclip_mask_vx_nxv2i32_nxv2i64(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnclip.mask.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnclip.mask.nxv2i32.nxv2i64(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vnclip.nxv4i32.nxv4i64.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vnclip.nxv4i32.nxv4i64(
   <vscale x 4 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vnclip_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnclip_vx_nxv4i32_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnclip.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnclip.nxv4i32.nxv4i64(
     <vscale x 4 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vnclip.mask.nxv4i32.nxv4i64.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vnclip.mask.nxv4i32.nxv4i64(
   <vscale x 4 x i32>,
   <vscale x 4 x i64>,
-  i32,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vnclip_mask_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnclip_mask_vx_nxv4i32_nxv4i64(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnclip.mask.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnclip.mask.nxv4i32.nxv4i64(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vnclip.nxv8i32.nxv8i64.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vnclip.nxv8i32.nxv8i64(
   <vscale x 8 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vnclip_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wx_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnclip_vx_nxv8i32_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnclip.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnclip.nxv8i32.nxv8i64(
     <vscale x 8 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vnclip.mask.nxv8i32.nxv8i64.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vnclip.mask.nxv8i32.nxv8i64(
   <vscale x 8 x i32>,
   <vscale x 8 x i64>,
-  i32,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vnclip_mask_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wx_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnclip_mask_vx_nxv8i32_nxv8i64(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnclip.mask.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnclip.mask.nxv8i32.nxv8i64(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i32> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnclip_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclip_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnclip_mask_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclip_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnclip_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclip_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnclip_mask_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclip_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnclip_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclip_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnclip_mask_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclip_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnclip_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclip_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnclip_mask_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclip_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnclip_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclip_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnclip_mask_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclip_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnclip_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclip_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnclip_mask_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclip_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnclip_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclip_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnclip_mask_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclip_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnclip_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclip_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnclip_mask_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclip_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnclip_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclip_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnclip_mask_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclip_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnclip_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclip_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnclip_mask_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclip_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnclip_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclip_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnclip_mask_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclip_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 1 x i32> @intrinsic_vnclip_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnclip_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnclip.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnclip.nxv1i32.nxv1i64(
     <vscale x 1 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i32> %a
 }
 
-define <vscale x 1 x i32> @intrinsic_vnclip_mask_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnclip_mask_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnclip.mask.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnclip.mask.nxv1i32.nxv1i64(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i32> %a
 }
 
-define <vscale x 2 x i32> @intrinsic_vnclip_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnclip_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vnclip.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnclip.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnclip.nxv2i32.nxv2i64(
     <vscale x 2 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i32> %a
 }
 
-define <vscale x 2 x i32> @intrinsic_vnclip_mask_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnclip_mask_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnclip.mask.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnclip.mask.nxv2i32.nxv2i64(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i32> %a
 }
 
-define <vscale x 4 x i32> @intrinsic_vnclip_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnclip_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
 ; CHECK-NEXT:    vnclip.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnclip.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnclip.nxv4i32.nxv4i64(
     <vscale x 4 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i32> %a
 }
 
-define <vscale x 4 x i32> @intrinsic_vnclip_mask_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnclip_mask_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnclip.mask.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnclip.mask.nxv4i32.nxv4i64(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i32> %a
 }
 
-define <vscale x 8 x i32> @intrinsic_vnclip_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_wi_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnclip_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,ta,mu
 ; CHECK-NEXT:    vnclip.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnclip.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnclip.nxv8i32.nxv8i64(
     <vscale x 8 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i32> %a
 }
 
-define <vscale x 8 x i32> @intrinsic_vnclip_mask_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclip_mask_wi_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnclip_mask_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
 ; CHECK-NEXT:    vnclip.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnclip.mask.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnclip.mask.nxv8i32.nxv8i64(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnclipu-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vnclipu-rv32.ll
index d0ad6e3d1cb89147fa6ab67d5dff5783cd2bd102..4ab2aa7c168146f01dacf95d14cece728113f118 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnclipu-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnclipu-rv32.ll
@@ -496,858 +496,858 @@ entry:
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16(
   <vscale x 1 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vnclipu_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclipu_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16(
   <vscale x 1 x i8>,
   <vscale x 1 x i16>,
-  i8,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vnclipu_mask_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclipu_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16(
   <vscale x 2 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vnclipu_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclipu_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16(
   <vscale x 2 x i8>,
   <vscale x 2 x i16>,
-  i8,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vnclipu_mask_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclipu_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16(
   <vscale x 4 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vnclipu_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclipu_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16(
   <vscale x 4 x i8>,
   <vscale x 4 x i16>,
-  i8,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vnclipu_mask_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclipu_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16(
   <vscale x 8 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vnclipu_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclipu_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16(
   <vscale x 8 x i8>,
   <vscale x 8 x i16>,
-  i8,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vnclipu_mask_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclipu_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16(
   <vscale x 16 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vnclipu_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclipu_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16(
   <vscale x 16 x i8>,
   <vscale x 16 x i16>,
-  i8,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vnclipu_mask_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclipu_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16(
   <vscale x 32 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vnclipu_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclipu_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16(
   <vscale x 32 x i8>,
   <vscale x 32 x i16>,
-  i8,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vnclipu_mask_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclipu_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32(
   <vscale x 1 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vnclipu_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclipu_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32(
   <vscale x 1 x i16>,
   <vscale x 1 x i32>,
-  i16,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vnclipu_mask_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclipu_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32(
   <vscale x 2 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vnclipu_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclipu_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32(
   <vscale x 2 x i16>,
   <vscale x 2 x i32>,
-  i16,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vnclipu_mask_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclipu_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32(
   <vscale x 4 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vnclipu_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclipu_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32(
   <vscale x 4 x i16>,
   <vscale x 4 x i32>,
-  i16,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vnclipu_mask_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclipu_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32(
   <vscale x 8 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vnclipu_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclipu_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32(
   <vscale x 8 x i16>,
   <vscale x 8 x i32>,
-  i16,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vnclipu_mask_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclipu_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32(
   <vscale x 16 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vnclipu_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclipu_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32(
   <vscale x 16 x i16>,
   <vscale x 16 x i32>,
-  i16,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vnclipu_mask_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclipu_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnclipu_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclipu_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnclipu_mask_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclipu_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnclipu_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclipu_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnclipu_mask_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclipu_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnclipu_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclipu_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnclipu_mask_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclipu_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnclipu_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclipu_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnclipu_mask_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclipu_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnclipu_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclipu_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnclipu_mask_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclipu_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnclipu_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclipu_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnclipu_mask_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclipu_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnclipu_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclipu_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnclipu_mask_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclipu_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnclipu_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclipu_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnclipu_mask_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclipu_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnclipu_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclipu_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnclipu_mask_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclipu_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnclipu_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclipu_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnclipu_mask_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclipu_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnclipu_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclipu_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnclipu_mask_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclipu_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnclipu-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vnclipu-rv64.ll
index 3d943da5af512c357fea4c0ec6e378a75e3203b6..fd21d8b90bc2bdd52bc7f4840d98d222e2679331 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnclipu-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnclipu-rv64.ll
@@ -676,1170 +676,1170 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16(
   <vscale x 1 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vnclipu_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclipu_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16(
   <vscale x 1 x i8>,
   <vscale x 1 x i16>,
-  i8,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vnclipu_mask_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclipu_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16(
   <vscale x 2 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vnclipu_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclipu_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16(
   <vscale x 2 x i8>,
   <vscale x 2 x i16>,
-  i8,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vnclipu_mask_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclipu_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16(
   <vscale x 4 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vnclipu_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclipu_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16(
   <vscale x 4 x i8>,
   <vscale x 4 x i16>,
-  i8,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vnclipu_mask_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclipu_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16(
   <vscale x 8 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vnclipu_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclipu_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16(
   <vscale x 8 x i8>,
   <vscale x 8 x i16>,
-  i8,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vnclipu_mask_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclipu_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16(
   <vscale x 16 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vnclipu_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclipu_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16(
   <vscale x 16 x i8>,
   <vscale x 16 x i16>,
-  i8,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vnclipu_mask_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclipu_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16(
   <vscale x 32 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vnclipu_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclipu_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16(
   <vscale x 32 x i8>,
   <vscale x 32 x i16>,
-  i8,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vnclipu_mask_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclipu_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32(
   <vscale x 1 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vnclipu_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclipu_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32(
   <vscale x 1 x i16>,
   <vscale x 1 x i32>,
-  i16,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vnclipu_mask_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclipu_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32(
   <vscale x 2 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vnclipu_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclipu_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32(
   <vscale x 2 x i16>,
   <vscale x 2 x i32>,
-  i16,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vnclipu_mask_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclipu_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32(
   <vscale x 4 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vnclipu_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclipu_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32(
   <vscale x 4 x i16>,
   <vscale x 4 x i32>,
-  i16,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vnclipu_mask_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclipu_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32(
   <vscale x 8 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vnclipu_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclipu_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32(
   <vscale x 8 x i16>,
   <vscale x 8 x i32>,
-  i16,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vnclipu_mask_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclipu_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32(
   <vscale x 16 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vnclipu_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclipu_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32(
   <vscale x 16 x i16>,
   <vscale x 16 x i32>,
-  i16,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vnclipu_mask_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclipu_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnclipu.nxv1i32.nxv1i64.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vnclipu.nxv1i32.nxv1i64(
   <vscale x 1 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vnclipu_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnclipu_vx_nxv1i32_nxv1i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnclipu.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnclipu.nxv1i32.nxv1i64(
     <vscale x 1 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnclipu.mask.nxv1i32.nxv1i64.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vnclipu.mask.nxv1i32.nxv1i64(
   <vscale x 1 x i32>,
   <vscale x 1 x i64>,
-  i32,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vnclipu_mask_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnclipu_mask_vx_nxv1i32_nxv1i64(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnclipu.mask.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnclipu.mask.nxv1i32.nxv1i64(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vnclipu.nxv2i32.nxv2i64.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vnclipu.nxv2i32.nxv2i64(
   <vscale x 2 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vnclipu_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnclipu_vx_nxv2i32_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnclipu.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnclipu.nxv2i32.nxv2i64(
     <vscale x 2 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vnclipu.mask.nxv2i32.nxv2i64.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vnclipu.mask.nxv2i32.nxv2i64(
   <vscale x 2 x i32>,
   <vscale x 2 x i64>,
-  i32,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vnclipu_mask_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnclipu_mask_vx_nxv2i32_nxv2i64(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnclipu.mask.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnclipu.mask.nxv2i32.nxv2i64(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vnclipu.nxv4i32.nxv4i64.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vnclipu.nxv4i32.nxv4i64(
   <vscale x 4 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vnclipu_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnclipu_vx_nxv4i32_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnclipu.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnclipu.nxv4i32.nxv4i64(
     <vscale x 4 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vnclipu.mask.nxv4i32.nxv4i64.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vnclipu.mask.nxv4i32.nxv4i64(
   <vscale x 4 x i32>,
   <vscale x 4 x i64>,
-  i32,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vnclipu_mask_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnclipu_mask_vx_nxv4i32_nxv4i64(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnclipu.mask.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnclipu.mask.nxv4i32.nxv4i64(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vnclipu.nxv8i32.nxv8i64.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vnclipu.nxv8i32.nxv8i64(
   <vscale x 8 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vnclipu_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wx_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnclipu_vx_nxv8i32_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnclipu.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnclipu.nxv8i32.nxv8i64(
     <vscale x 8 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vnclipu.mask.nxv8i32.nxv8i64.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vnclipu.mask.nxv8i32.nxv8i64(
   <vscale x 8 x i32>,
   <vscale x 8 x i64>,
-  i32,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vnclipu_mask_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wx_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnclipu_mask_vx_nxv8i32_nxv8i64(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnclipu.mask.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnclipu.mask.nxv8i32.nxv8i64(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i32> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnclipu_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclipu_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnclipu_mask_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnclipu_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnclipu_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclipu_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnclipu_mask_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnclipu_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnclipu_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclipu_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnclipu_mask_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnclipu_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnclipu_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclipu_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnclipu_mask_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnclipu_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnclipu_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclipu_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnclipu_mask_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnclipu_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnclipu_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclipu_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnclipu_mask_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnclipu_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnclipu_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclipu_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnclipu_mask_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnclipu_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnclipu_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclipu_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnclipu_mask_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnclipu_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnclipu_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclipu_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnclipu_mask_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnclipu_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnclipu_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclipu_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnclipu_mask_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnclipu_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnclipu_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclipu_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnclipu_mask_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnclipu_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 1 x i32> @intrinsic_vnclipu_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnclipu_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnclipu.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnclipu.nxv1i32.nxv1i64(
     <vscale x 1 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i32> %a
 }
 
-define <vscale x 1 x i32> @intrinsic_vnclipu_mask_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnclipu_mask_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnclipu.mask.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnclipu.mask.nxv1i32.nxv1i64(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i32> %a
 }
 
-define <vscale x 2 x i32> @intrinsic_vnclipu_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnclipu_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnclipu.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnclipu.nxv2i32.nxv2i64(
     <vscale x 2 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i32> %a
 }
 
-define <vscale x 2 x i32> @intrinsic_vnclipu_mask_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnclipu_mask_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnclipu.mask.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnclipu.mask.nxv2i32.nxv2i64(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i32> %a
 }
 
-define <vscale x 4 x i32> @intrinsic_vnclipu_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnclipu_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnclipu.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnclipu.nxv4i32.nxv4i64(
     <vscale x 4 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i32> %a
 }
 
-define <vscale x 4 x i32> @intrinsic_vnclipu_mask_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnclipu_mask_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnclipu.mask.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnclipu.mask.nxv4i32.nxv4i64(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i32> %a
 }
 
-define <vscale x 8 x i32> @intrinsic_vnclipu_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_wi_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnclipu_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,ta,mu
 ; CHECK-NEXT:    vnclipu.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnclipu.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnclipu.nxv8i32.nxv8i64(
     <vscale x 8 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i32> %a
 }
 
-define <vscale x 8 x i32> @intrinsic_vnclipu_mask_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnclipu_mask_wi_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnclipu_mask_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
 ; CHECK-NEXT:    vnclipu.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnclipu.mask.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnclipu.mask.nxv8i32.nxv8i64(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnsra-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vnsra-rv32.ll
index 5c3f1b95fb85e2c5a7d75eacb22ee1884e309e66..8d3df148a05c9133099566e95f336768529e3e28 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnsra-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnsra-rv32.ll
@@ -496,1352 +496,860 @@ entry:
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64.nxv1i32(
-  <vscale x 1 x i64>,
-  <vscale x 1 x i32>,
-  i32);
-
-define <vscale x 1 x i32> @intrinsic_vnsra_wv_nxv1i32_nxv1i64_nxv1i32(<vscale x 1 x i64> %0, <vscale x 1 x i32> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wv_nxv1i32_nxv1i64_nxv1i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,ta,mu
-; CHECK-NEXT:    vnsra.wv v25, v8, v9
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64.nxv1i32(
-    <vscale x 1 x i64> %0,
-    <vscale x 1 x i32> %1,
-    i32 %2)
-
-  ret <vscale x 1 x i32> %a
-}
-
-declare <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64.nxv1i32(
-  <vscale x 1 x i32>,
-  <vscale x 1 x i64>,
-  <vscale x 1 x i32>,
-  <vscale x 1 x i1>,
-  i32);
-
-define <vscale x 1 x i32> @intrinsic_vnsra_mask_wv_nxv1i32_nxv1i64_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv1i32_nxv1i64_nxv1i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
-; CHECK-NEXT:    vnsra.wv v8, v9, v10, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64.nxv1i32(
-    <vscale x 1 x i32> %0,
-    <vscale x 1 x i64> %1,
-    <vscale x 1 x i32> %2,
-    <vscale x 1 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 1 x i32> %a
-}
-
-declare <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64.nxv2i32(
-  <vscale x 2 x i64>,
-  <vscale x 2 x i32>,
-  i32);
-
-define <vscale x 2 x i32> @intrinsic_vnsra_wv_nxv2i32_nxv2i64_nxv2i32(<vscale x 2 x i64> %0, <vscale x 2 x i32> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wv_nxv2i32_nxv2i64_nxv2i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
-; CHECK-NEXT:    vnsra.wv v25, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64.nxv2i32(
-    <vscale x 2 x i64> %0,
-    <vscale x 2 x i32> %1,
-    i32 %2)
-
-  ret <vscale x 2 x i32> %a
-}
-
-declare <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64.nxv2i32(
-  <vscale x 2 x i32>,
-  <vscale x 2 x i64>,
-  <vscale x 2 x i32>,
-  <vscale x 2 x i1>,
-  i32);
-
-define <vscale x 2 x i32> @intrinsic_vnsra_mask_wv_nxv2i32_nxv2i64_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv2i32_nxv2i64_nxv2i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-NEXT:    vnsra.wv v8, v10, v9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64.nxv2i32(
-    <vscale x 2 x i32> %0,
-    <vscale x 2 x i64> %1,
-    <vscale x 2 x i32> %2,
-    <vscale x 2 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 2 x i32> %a
-}
-
-declare <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64.nxv4i32(
-  <vscale x 4 x i64>,
-  <vscale x 4 x i32>,
-  i32);
-
-define <vscale x 4 x i32> @intrinsic_vnsra_wv_nxv4i32_nxv4i64_nxv4i32(<vscale x 4 x i64> %0, <vscale x 4 x i32> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wv_nxv4i32_nxv4i64_nxv4i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
-; CHECK-NEXT:    vnsra.wv v26, v8, v12
-; CHECK-NEXT:    vmv2r.v v8, v26
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64.nxv4i32(
-    <vscale x 4 x i64> %0,
-    <vscale x 4 x i32> %1,
-    i32 %2)
-
-  ret <vscale x 4 x i32> %a
-}
-
-declare <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64.nxv4i32(
-  <vscale x 4 x i32>,
-  <vscale x 4 x i64>,
-  <vscale x 4 x i32>,
-  <vscale x 4 x i1>,
-  i32);
-
-define <vscale x 4 x i32> @intrinsic_vnsra_mask_wv_nxv4i32_nxv4i64_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv4i32_nxv4i64_nxv4i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
-; CHECK-NEXT:    vnsra.wv v8, v12, v10, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64.nxv4i32(
-    <vscale x 4 x i32> %0,
-    <vscale x 4 x i64> %1,
-    <vscale x 4 x i32> %2,
-    <vscale x 4 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 4 x i32> %a
-}
-
-declare <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64.nxv8i32(
-  <vscale x 8 x i64>,
-  <vscale x 8 x i32>,
-  i32);
-
-define <vscale x 8 x i32> @intrinsic_vnsra_wv_nxv8i32_nxv8i64_nxv8i32(<vscale x 8 x i64> %0, <vscale x 8 x i32> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wv_nxv8i32_nxv8i64_nxv8i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m4,ta,mu
-; CHECK-NEXT:    vnsra.wv v28, v8, v16
-; CHECK-NEXT:    vmv4r.v v8, v28
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64.nxv8i32(
-    <vscale x 8 x i64> %0,
-    <vscale x 8 x i32> %1,
-    i32 %2)
-
-  ret <vscale x 8 x i32> %a
-}
-
-declare <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64.nxv8i32(
-  <vscale x 8 x i32>,
-  <vscale x 8 x i64>,
-  <vscale x 8 x i32>,
-  <vscale x 8 x i1>,
-  i32);
-
-define <vscale x 8 x i32> @intrinsic_vnsra_mask_wv_nxv8i32_nxv8i64_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv8i32_nxv8i64_nxv8i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
-; CHECK-NEXT:    vnsra.wv v8, v16, v12, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64.nxv8i32(
-    <vscale x 8 x i32> %0,
-    <vscale x 8 x i64> %1,
-    <vscale x 8 x i32> %2,
-    <vscale x 8 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 8 x i32> %a
-}
-
-declare <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16(
   <vscale x 1 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vnsra_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsra_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16(
   <vscale x 1 x i8>,
   <vscale x 1 x i16>,
-  i8,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vnsra_mask_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsra_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16(
   <vscale x 2 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vnsra_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsra_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16(
   <vscale x 2 x i8>,
   <vscale x 2 x i16>,
-  i8,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vnsra_mask_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsra_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16(
   <vscale x 4 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vnsra_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsra_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16(
   <vscale x 4 x i8>,
   <vscale x 4 x i16>,
-  i8,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vnsra_mask_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsra_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16(
   <vscale x 8 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vnsra_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsra_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16(
   <vscale x 8 x i8>,
   <vscale x 8 x i16>,
-  i8,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vnsra_mask_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsra_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16(
   <vscale x 16 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vnsra_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsra_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16(
   <vscale x 16 x i8>,
   <vscale x 16 x i16>,
-  i8,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vnsra_mask_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsra_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16(
   <vscale x 32 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vnsra_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsra_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16(
   <vscale x 32 x i8>,
   <vscale x 32 x i16>,
-  i8,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vnsra_mask_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsra_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32(
   <vscale x 1 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vnsra_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsra_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32(
   <vscale x 1 x i16>,
   <vscale x 1 x i32>,
-  i16,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vnsra_mask_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsra_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32(
   <vscale x 2 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vnsra_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsra_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32(
   <vscale x 2 x i16>,
   <vscale x 2 x i32>,
-  i16,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vnsra_mask_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsra_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32(
   <vscale x 4 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vnsra_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsra_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32(
   <vscale x 4 x i16>,
   <vscale x 4 x i32>,
-  i16,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vnsra_mask_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsra_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32(
   <vscale x 8 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vnsra_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsra_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32(
   <vscale x 8 x i16>,
   <vscale x 8 x i32>,
-  i16,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vnsra_mask_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsra_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32(
   <vscale x 16 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vnsra_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsra_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32(
   <vscale x 16 x i16>,
   <vscale x 16 x i32>,
-  i16,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vnsra_mask_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsra_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64.i32(
-  <vscale x 1 x i64>,
-  i32,
-  i32);
-
-define <vscale x 1 x i32> @intrinsic_vnsra_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv1i32_nxv1i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vnsra.wx v25, v8, a0
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64.i32(
-    <vscale x 1 x i64> %0,
-    i32 %1,
-    i32 %2)
-
-  ret <vscale x 1 x i32> %a
-}
-
-declare <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64.i32(
-  <vscale x 1 x i32>,
-  <vscale x 1 x i64>,
-  i32,
-  <vscale x 1 x i1>,
-  i32);
-
-define <vscale x 1 x i32> @intrinsic_vnsra_mask_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv1i32_nxv1i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
-; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64.i32(
-    <vscale x 1 x i32> %0,
-    <vscale x 1 x i64> %1,
-    i32 %2,
-    <vscale x 1 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 1 x i32> %a
-}
-
-declare <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64.i32(
-  <vscale x 2 x i64>,
-  i32,
-  i32);
-
-define <vscale x 2 x i32> @intrinsic_vnsra_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv2i32_nxv2i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vnsra.wx v25, v8, a0
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64.i32(
-    <vscale x 2 x i64> %0,
-    i32 %1,
-    i32 %2)
-
-  ret <vscale x 2 x i32> %a
-}
-
-declare <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64.i32(
-  <vscale x 2 x i32>,
-  <vscale x 2 x i64>,
-  i32,
-  <vscale x 2 x i1>,
-  i32);
-
-define <vscale x 2 x i32> @intrinsic_vnsra_mask_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv2i32_nxv2i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
-; CHECK-NEXT:    vnsra.wx v8, v10, a0, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64.i32(
-    <vscale x 2 x i32> %0,
-    <vscale x 2 x i64> %1,
-    i32 %2,
-    <vscale x 2 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 2 x i32> %a
-}
-
-declare <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64.i32(
-  <vscale x 4 x i64>,
-  i32,
-  i32);
-
-define <vscale x 4 x i32> @intrinsic_vnsra_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv4i32_nxv4i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vnsra.wx v26, v8, a0
-; CHECK-NEXT:    vmv2r.v v8, v26
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64.i32(
-    <vscale x 4 x i64> %0,
-    i32 %1,
-    i32 %2)
-
-  ret <vscale x 4 x i32> %a
-}
-
-declare <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64.i32(
-  <vscale x 4 x i32>,
-  <vscale x 4 x i64>,
-  i32,
-  <vscale x 4 x i1>,
-  i32);
-
-define <vscale x 4 x i32> @intrinsic_vnsra_mask_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv4i32_nxv4i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
-; CHECK-NEXT:    vnsra.wx v8, v12, a0, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64.i32(
-    <vscale x 4 x i32> %0,
-    <vscale x 4 x i64> %1,
-    i32 %2,
-    <vscale x 4 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 4 x i32> %a
-}
-
-declare <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64.i32(
-  <vscale x 8 x i64>,
-  i32,
-  i32);
-
-define <vscale x 8 x i32> @intrinsic_vnsra_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv8i32_nxv8i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vnsra.wx v28, v8, a0
-; CHECK-NEXT:    vmv4r.v v8, v28
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64.i32(
-    <vscale x 8 x i64> %0,
-    i32 %1,
-    i32 %2)
-
-  ret <vscale x 8 x i32> %a
-}
-
-declare <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64.i32(
-  <vscale x 8 x i32>,
-  <vscale x 8 x i64>,
-  i32,
-  <vscale x 8 x i1>,
-  i32);
-
-define <vscale x 8 x i32> @intrinsic_vnsra_mask_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv8i32_nxv8i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
-; CHECK-NEXT:    vnsra.wx v8, v16, a0, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64.i32(
-    <vscale x 8 x i32> %0,
-    <vscale x 8 x i64> %1,
-    i32 %2,
-    <vscale x 8 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 8 x i32> %a
-}
-
-define <vscale x 1 x i8> @intrinsic_vnsra_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsra_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnsra_mask_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsra_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnsra_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsra_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnsra_mask_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsra_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnsra_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsra_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnsra_mask_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsra_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnsra_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsra_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnsra_mask_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsra_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnsra_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsra_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnsra_mask_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsra_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnsra_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsra_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnsra_mask_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsra_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnsra_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsra_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnsra_mask_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsra_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnsra_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsra_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnsra_mask_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsra_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnsra_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsra_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnsra_mask_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsra_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnsra_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsra_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnsra_mask_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsra_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnsra_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsra_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnsra_mask_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsra_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
   ret <vscale x 16 x i16> %a
 }
-
-define <vscale x 1 x i32> @intrinsic_vnsra_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv1i32_nxv1i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,ta,mu
-; CHECK-NEXT:    vnsra.wi v25, v8, 9
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64.i32(
-    <vscale x 1 x i64> %0,
-    i32 9,
-    i32 %1)
-
-  ret <vscale x 1 x i32> %a
-}
-
-define <vscale x 1 x i32> @intrinsic_vnsra_mask_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv1i32_nxv1i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
-; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64.i32(
-    <vscale x 1 x i32> %0,
-    <vscale x 1 x i64> %1,
-    i32 9,
-    <vscale x 1 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 1 x i32> %a
-}
-
-define <vscale x 2 x i32> @intrinsic_vnsra_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv2i32_nxv2i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
-; CHECK-NEXT:    vnsra.wi v25, v8, 9
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64.i32(
-    <vscale x 2 x i64> %0,
-    i32 9,
-    i32 %1)
-
-  ret <vscale x 2 x i32> %a
-}
-
-define <vscale x 2 x i32> @intrinsic_vnsra_mask_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv2i32_nxv2i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-NEXT:    vnsra.wi v8, v10, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64.i32(
-    <vscale x 2 x i32> %0,
-    <vscale x 2 x i64> %1,
-    i32 9,
-    <vscale x 2 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 2 x i32> %a
-}
-
-define <vscale x 4 x i32> @intrinsic_vnsra_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv4i32_nxv4i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
-; CHECK-NEXT:    vnsra.wi v26, v8, 9
-; CHECK-NEXT:    vmv2r.v v8, v26
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64.i32(
-    <vscale x 4 x i64> %0,
-    i32 9,
-    i32 %1)
-
-  ret <vscale x 4 x i32> %a
-}
-
-define <vscale x 4 x i32> @intrinsic_vnsra_mask_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv4i32_nxv4i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
-; CHECK-NEXT:    vnsra.wi v8, v12, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64.i32(
-    <vscale x 4 x i32> %0,
-    <vscale x 4 x i64> %1,
-    i32 9,
-    <vscale x 4 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 4 x i32> %a
-}
-
-define <vscale x 8 x i32> @intrinsic_vnsra_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv8i32_nxv8i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m4,ta,mu
-; CHECK-NEXT:    vnsra.wi v28, v8, 9
-; CHECK-NEXT:    vmv4r.v v8, v28
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64.i32(
-    <vscale x 8 x i64> %0,
-    i32 9,
-    i32 %1)
-
-  ret <vscale x 8 x i32> %a
-}
-
-define <vscale x 8 x i32> @intrinsic_vnsra_mask_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv8i32_nxv8i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
-; CHECK-NEXT:    vnsra.wi v8, v16, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64.i32(
-    <vscale x 8 x i32> %0,
-    <vscale x 8 x i64> %1,
-    i32 9,
-    <vscale x 8 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 8 x i32> %a
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnsra-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vnsra-rv64.ll
index df274711cabbdc566a69a67193fb62f69d4f627e..30f11f8f7cc2707f69f240e59fa7f8145f536b43 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnsra-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnsra-rv64.ll
@@ -676,1170 +676,1170 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16(
   <vscale x 1 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vnsra_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsra_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16(
   <vscale x 1 x i8>,
   <vscale x 1 x i16>,
-  i8,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vnsra_mask_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsra_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16(
   <vscale x 2 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vnsra_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsra_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16(
   <vscale x 2 x i8>,
   <vscale x 2 x i16>,
-  i8,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vnsra_mask_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsra_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16(
   <vscale x 4 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vnsra_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsra_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16(
   <vscale x 4 x i8>,
   <vscale x 4 x i16>,
-  i8,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vnsra_mask_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsra_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16(
   <vscale x 8 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vnsra_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsra_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16(
   <vscale x 8 x i8>,
   <vscale x 8 x i16>,
-  i8,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vnsra_mask_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsra_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16(
   <vscale x 16 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vnsra_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsra_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16(
   <vscale x 16 x i8>,
   <vscale x 16 x i16>,
-  i8,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vnsra_mask_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsra_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16(
   <vscale x 32 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vnsra_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsra_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16(
   <vscale x 32 x i8>,
   <vscale x 32 x i16>,
-  i8,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vnsra_mask_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsra_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32(
   <vscale x 1 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vnsra_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsra_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32(
   <vscale x 1 x i16>,
   <vscale x 1 x i32>,
-  i16,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vnsra_mask_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsra_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32(
   <vscale x 2 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vnsra_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsra_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32(
   <vscale x 2 x i16>,
   <vscale x 2 x i32>,
-  i16,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vnsra_mask_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsra_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32(
   <vscale x 4 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vnsra_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsra_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32(
   <vscale x 4 x i16>,
   <vscale x 4 x i32>,
-  i16,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vnsra_mask_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsra_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32(
   <vscale x 8 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vnsra_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsra_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32(
   <vscale x 8 x i16>,
   <vscale x 8 x i32>,
-  i16,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vnsra_mask_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsra_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32(
   <vscale x 16 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vnsra_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsra_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32(
   <vscale x 16 x i16>,
   <vscale x 16 x i32>,
-  i16,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vnsra_mask_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsra_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64(
   <vscale x 1 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vnsra_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnsra_vx_nxv1i32_nxv1i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64(
     <vscale x 1 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64(
   <vscale x 1 x i32>,
   <vscale x 1 x i64>,
-  i32,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vnsra_mask_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnsra_mask_vx_nxv1i32_nxv1i64(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64(
   <vscale x 2 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vnsra_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnsra_vx_nxv2i32_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64(
     <vscale x 2 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64(
   <vscale x 2 x i32>,
   <vscale x 2 x i64>,
-  i32,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vnsra_mask_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnsra_mask_vx_nxv2i32_nxv2i64(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64(
   <vscale x 4 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vnsra_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnsra_vx_nxv4i32_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64(
     <vscale x 4 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64(
   <vscale x 4 x i32>,
   <vscale x 4 x i64>,
-  i32,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vnsra_mask_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnsra_mask_vx_nxv4i32_nxv4i64(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64(
   <vscale x 8 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vnsra_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wx_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnsra_vx_nxv8i32_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64(
     <vscale x 8 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64(
   <vscale x 8 x i32>,
   <vscale x 8 x i64>,
-  i32,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vnsra_mask_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnsra_mask_vx_nxv8i32_nxv8i64(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i32> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnsra_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsra_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnsra_mask_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsra_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsra.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnsra_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsra_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnsra_mask_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsra_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsra.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnsra_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsra_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnsra_mask_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsra_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsra.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnsra_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsra_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnsra_mask_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsra_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsra.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnsra_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsra_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnsra_mask_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsra_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsra.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnsra_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsra_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnsra_mask_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsra_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsra.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnsra_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsra_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnsra_mask_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsra_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsra.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnsra_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsra_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnsra_mask_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsra_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsra.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnsra_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsra_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnsra_mask_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsra_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsra.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnsra_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsra_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnsra_mask_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsra_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsra.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnsra_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsra_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnsra_mask_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsra_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsra.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 1 x i32> @intrinsic_vnsra_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnsra_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.nxv1i32.nxv1i64(
     <vscale x 1 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i32> %a
 }
 
-define <vscale x 1 x i32> @intrinsic_vnsra_mask_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnsra_mask_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnsra.mask.nxv1i32.nxv1i64(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i32> %a
 }
 
-define <vscale x 2 x i32> @intrinsic_vnsra_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnsra_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vnsra.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.nxv2i32.nxv2i64(
     <vscale x 2 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i32> %a
 }
 
-define <vscale x 2 x i32> @intrinsic_vnsra_mask_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnsra_mask_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnsra.mask.nxv2i32.nxv2i64(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i32> %a
 }
 
-define <vscale x 4 x i32> @intrinsic_vnsra_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnsra_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
 ; CHECK-NEXT:    vnsra.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.nxv4i32.nxv4i64(
     <vscale x 4 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i32> %a
 }
 
-define <vscale x 4 x i32> @intrinsic_vnsra_mask_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnsra_mask_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnsra.mask.nxv4i32.nxv4i64(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i32> %a
 }
 
-define <vscale x 8 x i32> @intrinsic_vnsra_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_wi_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnsra_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,ta,mu
 ; CHECK-NEXT:    vnsra.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.nxv8i32.nxv8i64(
     <vscale x 8 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i32> %a
 }
 
-define <vscale x 8 x i32> @intrinsic_vnsra_mask_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnsra_mask_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsra_mask_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
 ; CHECK-NEXT:    vnsra.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnsra.mask.nxv8i32.nxv8i64(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnsrl-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vnsrl-rv32.ll
index ceeb0bf80ba6702e69d914ee2b1c9041d7b14aa0..8774a8949122a8d14e171e024f5c8209c59a07db 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnsrl-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnsrl-rv32.ll
@@ -496,1352 +496,860 @@ entry:
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64.nxv1i32(
-  <vscale x 1 x i64>,
-  <vscale x 1 x i32>,
-  i32);
-
-define <vscale x 1 x i32> @intrinsic_vnsrl_wv_nxv1i32_nxv1i64_nxv1i32(<vscale x 1 x i64> %0, <vscale x 1 x i32> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wv_nxv1i32_nxv1i64_nxv1i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,ta,mu
-; CHECK-NEXT:    vnsrl.wv v25, v8, v9
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64.nxv1i32(
-    <vscale x 1 x i64> %0,
-    <vscale x 1 x i32> %1,
-    i32 %2)
-
-  ret <vscale x 1 x i32> %a
-}
-
-declare <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64.nxv1i32(
-  <vscale x 1 x i32>,
-  <vscale x 1 x i64>,
-  <vscale x 1 x i32>,
-  <vscale x 1 x i1>,
-  i32);
-
-define <vscale x 1 x i32> @intrinsic_vnsrl_mask_wv_nxv1i32_nxv1i64_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv1i32_nxv1i64_nxv1i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
-; CHECK-NEXT:    vnsrl.wv v8, v9, v10, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64.nxv1i32(
-    <vscale x 1 x i32> %0,
-    <vscale x 1 x i64> %1,
-    <vscale x 1 x i32> %2,
-    <vscale x 1 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 1 x i32> %a
-}
-
-declare <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64.nxv2i32(
-  <vscale x 2 x i64>,
-  <vscale x 2 x i32>,
-  i32);
-
-define <vscale x 2 x i32> @intrinsic_vnsrl_wv_nxv2i32_nxv2i64_nxv2i32(<vscale x 2 x i64> %0, <vscale x 2 x i32> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wv_nxv2i32_nxv2i64_nxv2i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
-; CHECK-NEXT:    vnsrl.wv v25, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64.nxv2i32(
-    <vscale x 2 x i64> %0,
-    <vscale x 2 x i32> %1,
-    i32 %2)
-
-  ret <vscale x 2 x i32> %a
-}
-
-declare <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64.nxv2i32(
-  <vscale x 2 x i32>,
-  <vscale x 2 x i64>,
-  <vscale x 2 x i32>,
-  <vscale x 2 x i1>,
-  i32);
-
-define <vscale x 2 x i32> @intrinsic_vnsrl_mask_wv_nxv2i32_nxv2i64_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv2i32_nxv2i64_nxv2i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-NEXT:    vnsrl.wv v8, v10, v9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64.nxv2i32(
-    <vscale x 2 x i32> %0,
-    <vscale x 2 x i64> %1,
-    <vscale x 2 x i32> %2,
-    <vscale x 2 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 2 x i32> %a
-}
-
-declare <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64.nxv4i32(
-  <vscale x 4 x i64>,
-  <vscale x 4 x i32>,
-  i32);
-
-define <vscale x 4 x i32> @intrinsic_vnsrl_wv_nxv4i32_nxv4i64_nxv4i32(<vscale x 4 x i64> %0, <vscale x 4 x i32> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wv_nxv4i32_nxv4i64_nxv4i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
-; CHECK-NEXT:    vnsrl.wv v26, v8, v12
-; CHECK-NEXT:    vmv2r.v v8, v26
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64.nxv4i32(
-    <vscale x 4 x i64> %0,
-    <vscale x 4 x i32> %1,
-    i32 %2)
-
-  ret <vscale x 4 x i32> %a
-}
-
-declare <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64.nxv4i32(
-  <vscale x 4 x i32>,
-  <vscale x 4 x i64>,
-  <vscale x 4 x i32>,
-  <vscale x 4 x i1>,
-  i32);
-
-define <vscale x 4 x i32> @intrinsic_vnsrl_mask_wv_nxv4i32_nxv4i64_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv4i32_nxv4i64_nxv4i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
-; CHECK-NEXT:    vnsrl.wv v8, v12, v10, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64.nxv4i32(
-    <vscale x 4 x i32> %0,
-    <vscale x 4 x i64> %1,
-    <vscale x 4 x i32> %2,
-    <vscale x 4 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 4 x i32> %a
-}
-
-declare <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64.nxv8i32(
-  <vscale x 8 x i64>,
-  <vscale x 8 x i32>,
-  i32);
-
-define <vscale x 8 x i32> @intrinsic_vnsrl_wv_nxv8i32_nxv8i64_nxv8i32(<vscale x 8 x i64> %0, <vscale x 8 x i32> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wv_nxv8i32_nxv8i64_nxv8i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m4,ta,mu
-; CHECK-NEXT:    vnsrl.wv v28, v8, v16
-; CHECK-NEXT:    vmv4r.v v8, v28
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64.nxv8i32(
-    <vscale x 8 x i64> %0,
-    <vscale x 8 x i32> %1,
-    i32 %2)
-
-  ret <vscale x 8 x i32> %a
-}
-
-declare <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64.nxv8i32(
-  <vscale x 8 x i32>,
-  <vscale x 8 x i64>,
-  <vscale x 8 x i32>,
-  <vscale x 8 x i1>,
-  i32);
-
-define <vscale x 8 x i32> @intrinsic_vnsrl_mask_wv_nxv8i32_nxv8i64_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv8i32_nxv8i64_nxv8i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
-; CHECK-NEXT:    vnsrl.wv v8, v16, v12, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64.nxv8i32(
-    <vscale x 8 x i32> %0,
-    <vscale x 8 x i64> %1,
-    <vscale x 8 x i32> %2,
-    <vscale x 8 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 8 x i32> %a
-}
-
-declare <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16(
   <vscale x 1 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vnsrl_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsrl_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16(
   <vscale x 1 x i8>,
   <vscale x 1 x i16>,
-  i8,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vnsrl_mask_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsrl_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16(
   <vscale x 2 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vnsrl_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsrl_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16(
   <vscale x 2 x i8>,
   <vscale x 2 x i16>,
-  i8,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vnsrl_mask_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsrl_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16(
   <vscale x 4 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vnsrl_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsrl_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16(
   <vscale x 4 x i8>,
   <vscale x 4 x i16>,
-  i8,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vnsrl_mask_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsrl_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16(
   <vscale x 8 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vnsrl_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsrl_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16(
   <vscale x 8 x i8>,
   <vscale x 8 x i16>,
-  i8,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vnsrl_mask_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsrl_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16(
   <vscale x 16 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vnsrl_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsrl_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16(
   <vscale x 16 x i8>,
   <vscale x 16 x i16>,
-  i8,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vnsrl_mask_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsrl_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16(
   <vscale x 32 x i16>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vnsrl_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsrl_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16(
   <vscale x 32 x i8>,
   <vscale x 32 x i16>,
-  i8,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vnsrl_mask_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsrl_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32(
   <vscale x 1 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vnsrl_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsrl_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32(
   <vscale x 1 x i16>,
   <vscale x 1 x i32>,
-  i16,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vnsrl_mask_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsrl_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32(
   <vscale x 2 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vnsrl_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsrl_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32(
   <vscale x 2 x i16>,
   <vscale x 2 x i32>,
-  i16,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vnsrl_mask_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsrl_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32(
   <vscale x 4 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vnsrl_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsrl_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32(
   <vscale x 4 x i16>,
   <vscale x 4 x i32>,
-  i16,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vnsrl_mask_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsrl_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32(
   <vscale x 8 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vnsrl_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsrl_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32(
   <vscale x 8 x i16>,
   <vscale x 8 x i32>,
-  i16,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vnsrl_mask_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsrl_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32(
   <vscale x 16 x i32>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vnsrl_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsrl_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32(
   <vscale x 16 x i16>,
   <vscale x 16 x i32>,
-  i16,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vnsrl_mask_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsrl_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64.i32(
-  <vscale x 1 x i64>,
-  i32,
-  i32);
-
-define <vscale x 1 x i32> @intrinsic_vnsrl_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv1i32_nxv1i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
-; CHECK-NEXT:    vnsrl.wx v25, v8, a0
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64.i32(
-    <vscale x 1 x i64> %0,
-    i32 %1,
-    i32 %2)
-
-  ret <vscale x 1 x i32> %a
-}
-
-declare <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64.i32(
-  <vscale x 1 x i32>,
-  <vscale x 1 x i64>,
-  i32,
-  <vscale x 1 x i1>,
-  i32);
-
-define <vscale x 1 x i32> @intrinsic_vnsrl_mask_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv1i32_nxv1i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
-; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64.i32(
-    <vscale x 1 x i32> %0,
-    <vscale x 1 x i64> %1,
-    i32 %2,
-    <vscale x 1 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 1 x i32> %a
-}
-
-declare <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64.i32(
-  <vscale x 2 x i64>,
-  i32,
-  i32);
-
-define <vscale x 2 x i32> @intrinsic_vnsrl_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv2i32_nxv2i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
-; CHECK-NEXT:    vnsrl.wx v25, v8, a0
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64.i32(
-    <vscale x 2 x i64> %0,
-    i32 %1,
-    i32 %2)
-
-  ret <vscale x 2 x i32> %a
-}
-
-declare <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64.i32(
-  <vscale x 2 x i32>,
-  <vscale x 2 x i64>,
-  i32,
-  <vscale x 2 x i1>,
-  i32);
-
-define <vscale x 2 x i32> @intrinsic_vnsrl_mask_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv2i32_nxv2i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
-; CHECK-NEXT:    vnsrl.wx v8, v10, a0, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64.i32(
-    <vscale x 2 x i32> %0,
-    <vscale x 2 x i64> %1,
-    i32 %2,
-    <vscale x 2 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 2 x i32> %a
-}
-
-declare <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64.i32(
-  <vscale x 4 x i64>,
-  i32,
-  i32);
-
-define <vscale x 4 x i32> @intrinsic_vnsrl_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv4i32_nxv4i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
-; CHECK-NEXT:    vnsrl.wx v26, v8, a0
-; CHECK-NEXT:    vmv2r.v v8, v26
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64.i32(
-    <vscale x 4 x i64> %0,
-    i32 %1,
-    i32 %2)
-
-  ret <vscale x 4 x i32> %a
-}
-
-declare <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64.i32(
-  <vscale x 4 x i32>,
-  <vscale x 4 x i64>,
-  i32,
-  <vscale x 4 x i1>,
-  i32);
-
-define <vscale x 4 x i32> @intrinsic_vnsrl_mask_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv4i32_nxv4i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
-; CHECK-NEXT:    vnsrl.wx v8, v12, a0, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64.i32(
-    <vscale x 4 x i32> %0,
-    <vscale x 4 x i64> %1,
-    i32 %2,
-    <vscale x 4 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 4 x i32> %a
-}
-
-declare <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64.i32(
-  <vscale x 8 x i64>,
-  i32,
-  i32);
-
-define <vscale x 8 x i32> @intrinsic_vnsrl_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv8i32_nxv8i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
-; CHECK-NEXT:    vnsrl.wx v28, v8, a0
-; CHECK-NEXT:    vmv4r.v v8, v28
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64.i32(
-    <vscale x 8 x i64> %0,
-    i32 %1,
-    i32 %2)
-
-  ret <vscale x 8 x i32> %a
-}
-
-declare <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64.i32(
-  <vscale x 8 x i32>,
-  <vscale x 8 x i64>,
-  i32,
-  <vscale x 8 x i1>,
-  i32);
-
-define <vscale x 8 x i32> @intrinsic_vnsrl_mask_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv8i32_nxv8i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
-; CHECK-NEXT:    vnsrl.wx v8, v16, a0, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64.i32(
-    <vscale x 8 x i32> %0,
-    <vscale x 8 x i64> %1,
-    i32 %2,
-    <vscale x 8 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 8 x i32> %a
-}
-
-define <vscale x 1 x i8> @intrinsic_vnsrl_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsrl_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnsrl_mask_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsrl_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnsrl_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsrl_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnsrl_mask_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsrl_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnsrl_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsrl_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnsrl_mask_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsrl_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnsrl_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsrl_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnsrl_mask_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsrl_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnsrl_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsrl_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnsrl_mask_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsrl_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnsrl_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsrl_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnsrl_mask_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsrl_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnsrl_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsrl_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnsrl_mask_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsrl_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnsrl_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsrl_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnsrl_mask_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsrl_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnsrl_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsrl_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnsrl_mask_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsrl_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnsrl_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsrl_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnsrl_mask_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsrl_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnsrl_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsrl_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i32 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnsrl_mask_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsrl_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i32 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
   ret <vscale x 16 x i16> %a
 }
-
-define <vscale x 1 x i32> @intrinsic_vnsrl_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv1i32_nxv1i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,ta,mu
-; CHECK-NEXT:    vnsrl.wi v25, v8, 9
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64.i32(
-    <vscale x 1 x i64> %0,
-    i32 9,
-    i32 %1)
-
-  ret <vscale x 1 x i32> %a
-}
-
-define <vscale x 1 x i32> @intrinsic_vnsrl_mask_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv1i32_nxv1i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
-; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64.i32(
-    <vscale x 1 x i32> %0,
-    <vscale x 1 x i64> %1,
-    i32 9,
-    <vscale x 1 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 1 x i32> %a
-}
-
-define <vscale x 2 x i32> @intrinsic_vnsrl_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv2i32_nxv2i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
-; CHECK-NEXT:    vnsrl.wi v25, v8, 9
-; CHECK-NEXT:    vmv1r.v v8, v25
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64.i32(
-    <vscale x 2 x i64> %0,
-    i32 9,
-    i32 %1)
-
-  ret <vscale x 2 x i32> %a
-}
-
-define <vscale x 2 x i32> @intrinsic_vnsrl_mask_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv2i32_nxv2i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-NEXT:    vnsrl.wi v8, v10, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64.i32(
-    <vscale x 2 x i32> %0,
-    <vscale x 2 x i64> %1,
-    i32 9,
-    <vscale x 2 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 2 x i32> %a
-}
-
-define <vscale x 4 x i32> @intrinsic_vnsrl_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv4i32_nxv4i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
-; CHECK-NEXT:    vnsrl.wi v26, v8, 9
-; CHECK-NEXT:    vmv2r.v v8, v26
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64.i32(
-    <vscale x 4 x i64> %0,
-    i32 9,
-    i32 %1)
-
-  ret <vscale x 4 x i32> %a
-}
-
-define <vscale x 4 x i32> @intrinsic_vnsrl_mask_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv4i32_nxv4i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
-; CHECK-NEXT:    vnsrl.wi v8, v12, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64.i32(
-    <vscale x 4 x i32> %0,
-    <vscale x 4 x i64> %1,
-    i32 9,
-    <vscale x 4 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 4 x i32> %a
-}
-
-define <vscale x 8 x i32> @intrinsic_vnsrl_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv8i32_nxv8i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m4,ta,mu
-; CHECK-NEXT:    vnsrl.wi v28, v8, 9
-; CHECK-NEXT:    vmv4r.v v8, v28
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64.i32(
-    <vscale x 8 x i64> %0,
-    i32 9,
-    i32 %1)
-
-  ret <vscale x 8 x i32> %a
-}
-
-define <vscale x 8 x i32> @intrinsic_vnsrl_mask_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv8i32_nxv8i64_i32:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
-; CHECK-NEXT:    vnsrl.wi v8, v16, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64.i32(
-    <vscale x 8 x i32> %0,
-    <vscale x 8 x i64> %1,
-    i32 9,
-    <vscale x 8 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 8 x i32> %a
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnsrl-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vnsrl-rv64.ll
index 27a1b59df81e677cf769fa3c2af529c7e294d4a3..d049ca6c59b3d398be222a4fbd36dea6c80df303 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnsrl-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnsrl-rv64.ll
@@ -676,1170 +676,1170 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16(
   <vscale x 1 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vnsrl_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsrl_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16(
   <vscale x 1 x i8>,
   <vscale x 1 x i16>,
-  i8,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vnsrl_mask_wx_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsrl_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16(
   <vscale x 2 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vnsrl_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsrl_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16(
   <vscale x 2 x i8>,
   <vscale x 2 x i16>,
-  i8,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vnsrl_mask_wx_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsrl_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16(
   <vscale x 4 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vnsrl_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsrl_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16(
   <vscale x 4 x i8>,
   <vscale x 4 x i16>,
-  i8,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vnsrl_mask_wx_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsrl_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16(
   <vscale x 8 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vnsrl_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsrl_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16(
   <vscale x 8 x i8>,
   <vscale x 8 x i16>,
-  i8,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vnsrl_mask_wx_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsrl_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16(
   <vscale x 16 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vnsrl_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsrl_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16(
   <vscale x 16 x i8>,
   <vscale x 16 x i16>,
-  i8,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vnsrl_mask_wx_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsrl_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16(
   <vscale x 32 x i16>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vnsrl_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsrl_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16(
   <vscale x 32 x i8>,
   <vscale x 32 x i16>,
-  i8,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vnsrl_mask_wx_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsrl_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32(
   <vscale x 1 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vnsrl_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsrl_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32(
   <vscale x 1 x i16>,
   <vscale x 1 x i32>,
-  i16,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vnsrl_mask_wx_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsrl_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32(
   <vscale x 2 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vnsrl_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsrl_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32(
   <vscale x 2 x i16>,
   <vscale x 2 x i32>,
-  i16,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vnsrl_mask_wx_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsrl_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32(
   <vscale x 4 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vnsrl_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsrl_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32(
   <vscale x 4 x i16>,
   <vscale x 4 x i32>,
-  i16,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vnsrl_mask_wx_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsrl_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32(
   <vscale x 8 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vnsrl_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsrl_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32(
   <vscale x 8 x i16>,
   <vscale x 8 x i32>,
-  i16,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vnsrl_mask_wx_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsrl_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32(
   <vscale x 16 x i32>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vnsrl_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsrl_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32(
   <vscale x 16 x i16>,
   <vscale x 16 x i32>,
-  i16,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vnsrl_mask_wx_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsrl_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64(
   <vscale x 1 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vnsrl_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnsrl_vx_nxv1i32_nxv1i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64(
     <vscale x 1 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64(
   <vscale x 1 x i32>,
   <vscale x 1 x i64>,
-  i32,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vnsrl_mask_wx_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnsrl_mask_vx_nxv1i32_nxv1i64(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64(
   <vscale x 2 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vnsrl_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnsrl_vx_nxv2i32_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v25, v8, a0
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64(
     <vscale x 2 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64(
   <vscale x 2 x i32>,
   <vscale x 2 x i64>,
-  i32,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vnsrl_mask_wx_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnsrl_mask_vx_nxv2i32_nxv2i64(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64(
   <vscale x 4 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vnsrl_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnsrl_vx_nxv4i32_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v26, v8, a0
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64(
     <vscale x 4 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64(
   <vscale x 4 x i32>,
   <vscale x 4 x i64>,
-  i32,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vnsrl_mask_wx_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnsrl_mask_vx_nxv4i32_nxv4i64(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64(
   <vscale x 8 x i64>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vnsrl_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wx_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnsrl_vx_nxv8i32_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wx v28, v8, a0
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64(
     <vscale x 8 x i64> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64(
   <vscale x 8 x i32>,
   <vscale x 8 x i64>,
-  i32,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vnsrl_mask_wx_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnsrl_mask_vx_nxv8i32_nxv8i64(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i64> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i32> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnsrl_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsrl_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.nxv1i8.nxv1i16(
     <vscale x 1 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 1 x i8> @intrinsic_vnsrl_mask_wi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv1i8_nxv1i16_i8:
+define <vscale x 1 x i8> @intrinsic_vnsrl_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf8,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vnsrl.mask.nxv1i8.nxv1i16(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnsrl_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsrl_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.nxv2i8.nxv2i16(
     <vscale x 2 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 2 x i8> @intrinsic_vnsrl_mask_wi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv2i8_nxv2i16_i8:
+define <vscale x 2 x i8> @intrinsic_vnsrl_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf4,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vnsrl.mask.nxv2i8.nxv2i16(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnsrl_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsrl_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.nxv4i8.nxv4i16(
     <vscale x 4 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 4 x i8> @intrinsic_vnsrl_mask_wi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv4i8_nxv4i16_i8:
+define <vscale x 4 x i8> @intrinsic_vnsrl_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vnsrl.mask.nxv4i8.nxv4i16(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnsrl_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsrl_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.nxv8i8.nxv8i16(
     <vscale x 8 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 8 x i8> @intrinsic_vnsrl_mask_wi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv8i8_nxv8i16_i8:
+define <vscale x 8 x i8> @intrinsic_vnsrl_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vnsrl.mask.nxv8i8.nxv8i16(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
   ret <vscale x 8 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnsrl_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsrl_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.nxv16i8.nxv16i16(
     <vscale x 16 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 16 x i8> @intrinsic_vnsrl_mask_wi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv16i8_nxv16i16_i8:
+define <vscale x 16 x i8> @intrinsic_vnsrl_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vnsrl.mask.nxv16i8.nxv16i16(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
   ret <vscale x 16 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnsrl_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsrl_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.nxv32i8.nxv32i16(
     <vscale x 32 x i16> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 32 x i8> @intrinsic_vnsrl_mask_wi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv32i8_nxv32i16_i8:
+define <vscale x 32 x i8> @intrinsic_vnsrl_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vnsrl.mask.nxv32i8.nxv32i16(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i16> %1,
-    i8 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
   ret <vscale x 32 x i8> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnsrl_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsrl_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.nxv1i16.nxv1i32(
     <vscale x 1 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 1 x i16> @intrinsic_vnsrl_mask_wi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv1i16_nxv1i32_i16:
+define <vscale x 1 x i16> @intrinsic_vnsrl_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf4,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vnsrl.mask.nxv1i16.nxv1i32(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnsrl_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsrl_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.nxv2i16.nxv2i32(
     <vscale x 2 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 2 x i16> @intrinsic_vnsrl_mask_wi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv2i16_nxv2i32_i16:
+define <vscale x 2 x i16> @intrinsic_vnsrl_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vnsrl.mask.nxv2i16.nxv2i32(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnsrl_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsrl_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.nxv4i16.nxv4i32(
     <vscale x 4 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 4 x i16> @intrinsic_vnsrl_mask_wi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv4i16_nxv4i32_i16:
+define <vscale x 4 x i16> @intrinsic_vnsrl_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vnsrl.mask.nxv4i16.nxv4i32(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnsrl_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsrl_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.nxv8i16.nxv8i32(
     <vscale x 8 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 8 x i16> @intrinsic_vnsrl_mask_wi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv8i16_nxv8i32_i16:
+define <vscale x 8 x i16> @intrinsic_vnsrl_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vnsrl.mask.nxv8i16.nxv8i32(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
   ret <vscale x 8 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnsrl_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsrl_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.nxv16i16.nxv16i32(
     <vscale x 16 x i32> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 16 x i16> @intrinsic_vnsrl_mask_wi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv16i16_nxv16i32_i16:
+define <vscale x 16 x i16> @intrinsic_vnsrl_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e16,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vnsrl.mask.nxv16i16.nxv16i32(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i32> %1,
-    i16 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
   ret <vscale x 16 x i16> %a
 }
 
-define <vscale x 1 x i32> @intrinsic_vnsrl_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnsrl_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.nxv1i32.nxv1i64(
     <vscale x 1 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i32> %a
 }
 
-define <vscale x 1 x i32> @intrinsic_vnsrl_mask_wi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv1i32_nxv1i64_i32:
+define <vscale x 1 x i32> @intrinsic_vnsrl_mask_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,mf2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vnsrl.mask.nxv1i32.nxv1i64(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
   ret <vscale x 1 x i32> %a
 }
 
-define <vscale x 2 x i32> @intrinsic_vnsrl_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnsrl_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v25, v8, 9
 ; CHECK-NEXT:    vmv1r.v v8, v25
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.nxv2i32.nxv2i64(
     <vscale x 2 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i32> %a
 }
 
-define <vscale x 2 x i32> @intrinsic_vnsrl_mask_wi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv2i32_nxv2i64_i32:
+define <vscale x 2 x i32> @intrinsic_vnsrl_mask_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vnsrl.mask.nxv2i32.nxv2i64(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
   ret <vscale x 2 x i32> %a
 }
 
-define <vscale x 4 x i32> @intrinsic_vnsrl_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnsrl_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v26, v8, 9
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.nxv4i32.nxv4i64(
     <vscale x 4 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i32> %a
 }
 
-define <vscale x 4 x i32> @intrinsic_vnsrl_mask_wi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv4i32_nxv4i64_i32:
+define <vscale x 4 x i32> @intrinsic_vnsrl_mask_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vnsrl.mask.nxv4i32.nxv4i64(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
   ret <vscale x 4 x i32> %a
 }
 
-define <vscale x 8 x i32> @intrinsic_vnsrl_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_wi_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnsrl_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, i64 %1) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v28, v8, 9
 ; CHECK-NEXT:    vmv4r.v v8, v28
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.nxv8i32.nxv8i64(
     <vscale x 8 x i64> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i32> %a
 }
 
-define <vscale x 8 x i32> @intrinsic_vnsrl_mask_wi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
-; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv8i32_nxv8i64_i32:
+define <vscale x 8 x i32> @intrinsic_vnsrl_mask_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i64 %3) nounwind {
+; CHECK-LABEL: intrinsic_vnsrl_mask_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
 ; CHECK-NEXT:    vnsrl.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vnsrl.mask.nxv8i32.nxv8i64(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i64> %1,
-    i32 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsll-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vsll-rv32.ll
index 0e2be1ad194f726b888c29a8ad0451cf0098f167..a4427aab309fdf3e680a73fb649fff40bb616487 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsll-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsll-rv32.ll
@@ -796,591 +796,591 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vsll_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsll_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vsll_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsll_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vsll_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsll_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vsll_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsll_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vsll_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsll_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vsll_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsll_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vsll_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsll_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vsll_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsll_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vsll_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsll_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vsll_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsll_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vsll_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsll_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vsll_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsll_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vsll_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsll_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i32,
   <vscale x 64 x i1>,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vsll_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsll_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i32 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 64 x i1> %3,
     i32 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vsll_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsll_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vsll_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsll_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vsll_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsll_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vsll_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsll_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vsll_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsll_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vsll_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsll_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vsll_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsll_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vsll_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsll_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vsll_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsll_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vsll_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsll_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vsll_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsll_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vsll_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsll_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32(
   <vscale x 1 x i32>,
   i32,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vsll_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsll_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1388,21 +1388,21 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
   i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vsll_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsll_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 %2,
@@ -1412,19 +1412,19 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32(
   <vscale x 2 x i32>,
   i32,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vsll_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsll_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1432,21 +1432,21 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
   i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vsll_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsll_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 %2,
@@ -1456,19 +1456,19 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32(
   <vscale x 4 x i32>,
   i32,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vsll_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsll_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1476,21 +1476,21 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
   i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vsll_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsll_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 %2,
@@ -1500,19 +1500,19 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32(
   <vscale x 8 x i32>,
   i32,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vsll_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsll_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1520,21 +1520,21 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
   i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vsll_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsll_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 %2,
@@ -1544,19 +1544,19 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32(
   <vscale x 16 x i32>,
   i32,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vsll_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsll_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1564,21 +1564,21 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
   i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vsll_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsll_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 %2,
@@ -1595,9 +1595,9 @@ define <vscale x 1 x i8> @intrinsic_vsll_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i8> %a
@@ -1610,10 +1610,10 @@ define <vscale x 1 x i8> @intrinsic_vsll_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -1627,9 +1627,9 @@ define <vscale x 2 x i8> @intrinsic_vsll_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i8> %a
@@ -1642,10 +1642,10 @@ define <vscale x 2 x i8> @intrinsic_vsll_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -1659,9 +1659,9 @@ define <vscale x 4 x i8> @intrinsic_vsll_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i8> %a
@@ -1674,10 +1674,10 @@ define <vscale x 4 x i8> @intrinsic_vsll_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -1691,9 +1691,9 @@ define <vscale x 8 x i8> @intrinsic_vsll_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i8> %a
@@ -1706,10 +1706,10 @@ define <vscale x 8 x i8> @intrinsic_vsll_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -1723,9 +1723,9 @@ define <vscale x 16 x i8> @intrinsic_vsll_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i8> %a
@@ -1738,10 +1738,10 @@ define <vscale x 16 x i8> @intrinsic_vsll_mask_vi_nxv16i8_nxv16i8_i8(<vscale x 1
 ; CHECK-NEXT:    vsll.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -1755,9 +1755,9 @@ define <vscale x 32 x i8> @intrinsic_vsll_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i8> %a
@@ -1770,10 +1770,10 @@ define <vscale x 32 x i8> @intrinsic_vsll_mask_vi_nxv32i8_nxv32i8_i8(<vscale x 3
 ; CHECK-NEXT:    vsll.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -1787,9 +1787,9 @@ define <vscale x 64 x i8> @intrinsic_vsll_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 64 x i8> %a
@@ -1802,10 +1802,10 @@ define <vscale x 64 x i8> @intrinsic_vsll_mask_vi_nxv64i8_nxv64i8_i8(<vscale x 6
 ; CHECK-NEXT:    vsll.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 64 x i1> %2,
     i32 %3)
 
@@ -1819,9 +1819,9 @@ define <vscale x 1 x i16> @intrinsic_vsll_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i16> %a
@@ -1834,10 +1834,10 @@ define <vscale x 1 x i16> @intrinsic_vsll_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -1851,9 +1851,9 @@ define <vscale x 2 x i16> @intrinsic_vsll_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i16> %a
@@ -1866,10 +1866,10 @@ define <vscale x 2 x i16> @intrinsic_vsll_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -1883,9 +1883,9 @@ define <vscale x 4 x i16> @intrinsic_vsll_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i16> %a
@@ -1898,10 +1898,10 @@ define <vscale x 4 x i16> @intrinsic_vsll_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -1915,9 +1915,9 @@ define <vscale x 8 x i16> @intrinsic_vsll_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i16> %a
@@ -1930,10 +1930,10 @@ define <vscale x 8 x i16> @intrinsic_vsll_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -1947,9 +1947,9 @@ define <vscale x 16 x i16> @intrinsic_vsll_vi_nxv16i16_nxv16i16_i16(<vscale x 16
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i16> %a
@@ -1962,10 +1962,10 @@ define <vscale x 16 x i16> @intrinsic_vsll_mask_vi_nxv16i16_nxv16i16_i16(<vscale
 ; CHECK-NEXT:    vsll.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -1979,9 +1979,9 @@ define <vscale x 32 x i16> @intrinsic_vsll_vi_nxv32i16_nxv32i16_i16(<vscale x 32
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i16> %a
@@ -1994,10 +1994,10 @@ define <vscale x 32 x i16> @intrinsic_vsll_mask_vi_nxv32i16_nxv32i16_i16(<vscale
 ; CHECK-NEXT:    vsll.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -2011,7 +2011,7 @@ define <vscale x 1 x i32> @intrinsic_vsll_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 9,
     i32 %1)
@@ -2026,7 +2026,7 @@ define <vscale x 1 x i32> @intrinsic_vsll_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 9,
@@ -2043,7 +2043,7 @@ define <vscale x 2 x i32> @intrinsic_vsll_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 9,
     i32 %1)
@@ -2058,7 +2058,7 @@ define <vscale x 2 x i32> @intrinsic_vsll_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 9,
@@ -2075,7 +2075,7 @@ define <vscale x 4 x i32> @intrinsic_vsll_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 9,
     i32 %1)
@@ -2090,7 +2090,7 @@ define <vscale x 4 x i32> @intrinsic_vsll_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 9,
@@ -2107,7 +2107,7 @@ define <vscale x 8 x i32> @intrinsic_vsll_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 9,
     i32 %1)
@@ -2122,7 +2122,7 @@ define <vscale x 8 x i32> @intrinsic_vsll_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 9,
@@ -2139,7 +2139,7 @@ define <vscale x 16 x i32> @intrinsic_vsll_vi_nxv16i32_nxv16i32_i32(<vscale x 16
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 9,
     i32 %1)
@@ -2154,7 +2154,7 @@ define <vscale x 16 x i32> @intrinsic_vsll_mask_vi_nxv16i32_nxv16i32_i32(<vscale
 ; CHECK-NEXT:    vsll.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 9,
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsll-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vsll-rv64.ll
index f0356fb8100c180cb5af04019038ba4af7b2b546..9e6c6255f9a2a5f964fe16e6ce27f11130743e15 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsll-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsll-rv64.ll
@@ -973,811 +973,811 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vsll_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsll_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vsll_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsll_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vsll_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsll_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vsll_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsll_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vsll_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsll_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vsll_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsll_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vsll_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsll_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vsll_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsll_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vsll_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsll_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vsll_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsll_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vsll_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsll_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vsll_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsll_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vsll_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsll_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i64,
   <vscale x 64 x i1>,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vsll_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsll_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i64 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 64 x i1> %3,
     i64 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vsll_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsll_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vsll_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsll_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vsll_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsll_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vsll_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsll_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vsll_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsll_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vsll_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsll_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vsll_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsll_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vsll_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsll_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vsll_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsll_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vsll_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsll_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vsll_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsll_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vsll_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsll_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32(
   <vscale x 1 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vsll_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsll_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
-  i32,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vsll_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsll_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32(
   <vscale x 2 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vsll_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsll_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
-  i32,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vsll_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsll_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32(
   <vscale x 4 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vsll_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsll_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
-  i32,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vsll_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsll_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32(
   <vscale x 8 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vsll_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsll_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
-  i32,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vsll_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsll_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32(
   <vscale x 16 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vsll_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsll_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
-  i32,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vsll_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsll_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vsll.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vsll.nxv1i64(
   <vscale x 1 x i64>,
   i64,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vsll_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vsll_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsll.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsll.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1785,21 +1785,21 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vsll.mask.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vsll.mask.nxv1i64(
   <vscale x 1 x i64>,
   <vscale x 1 x i64>,
   i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vsll_mask_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vsll_mask_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsll.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsll.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 %2,
@@ -1809,19 +1809,19 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vsll.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vsll.nxv2i64(
   <vscale x 2 x i64>,
   i64,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vsll_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vsll_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsll.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsll.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1829,21 +1829,21 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vsll.mask.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vsll.mask.nxv2i64(
   <vscale x 2 x i64>,
   <vscale x 2 x i64>,
   i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vsll_mask_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vsll_mask_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsll.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsll.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 %2,
@@ -1853,19 +1853,19 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vsll.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vsll.nxv4i64(
   <vscale x 4 x i64>,
   i64,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vsll_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vsll_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsll.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsll.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1873,21 +1873,21 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vsll.mask.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vsll.mask.nxv4i64(
   <vscale x 4 x i64>,
   <vscale x 4 x i64>,
   i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vsll_mask_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vsll_mask_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsll.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsll.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 %2,
@@ -1897,19 +1897,19 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vsll.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vsll.nxv8i64(
   <vscale x 8 x i64>,
   i64,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vsll_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsll_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vsll_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsll_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
 ; CHECK-NEXT:    vsll.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsll.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsll.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1917,21 +1917,21 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vsll.mask.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vsll.mask.nxv8i64(
   <vscale x 8 x i64>,
   <vscale x 8 x i64>,
   i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vsll_mask_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vsll_mask_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,tu,mu
 ; CHECK-NEXT:    vsll.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsll.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsll.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 %2,
@@ -1948,9 +1948,9 @@ define <vscale x 1 x i8> @intrinsic_vsll_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i8> %a
@@ -1963,10 +1963,10 @@ define <vscale x 1 x i8> @intrinsic_vsll_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsll.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -1980,9 +1980,9 @@ define <vscale x 2 x i8> @intrinsic_vsll_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i8> %a
@@ -1995,10 +1995,10 @@ define <vscale x 2 x i8> @intrinsic_vsll_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsll.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2012,9 +2012,9 @@ define <vscale x 4 x i8> @intrinsic_vsll_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i8> %a
@@ -2027,10 +2027,10 @@ define <vscale x 4 x i8> @intrinsic_vsll_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsll.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2044,9 +2044,9 @@ define <vscale x 8 x i8> @intrinsic_vsll_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i8> %a
@@ -2059,10 +2059,10 @@ define <vscale x 8 x i8> @intrinsic_vsll_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsll.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2076,9 +2076,9 @@ define <vscale x 16 x i8> @intrinsic_vsll_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i8> %a
@@ -2091,10 +2091,10 @@ define <vscale x 16 x i8> @intrinsic_vsll_mask_vi_nxv16i8_nxv16i8_i8(<vscale x 1
 ; CHECK-NEXT:    vsll.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsll.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2108,9 +2108,9 @@ define <vscale x 32 x i8> @intrinsic_vsll_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i8> %a
@@ -2123,10 +2123,10 @@ define <vscale x 32 x i8> @intrinsic_vsll_mask_vi_nxv32i8_nxv32i8_i8(<vscale x 3
 ; CHECK-NEXT:    vsll.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsll.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2140,9 +2140,9 @@ define <vscale x 64 x i8> @intrinsic_vsll_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 64 x i8> %a
@@ -2155,10 +2155,10 @@ define <vscale x 64 x i8> @intrinsic_vsll_mask_vi_nxv64i8_nxv64i8_i8(<vscale x 6
 ; CHECK-NEXT:    vsll.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsll.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 64 x i1> %2,
     i64 %3)
 
@@ -2172,9 +2172,9 @@ define <vscale x 1 x i16> @intrinsic_vsll_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i16> %a
@@ -2187,10 +2187,10 @@ define <vscale x 1 x i16> @intrinsic_vsll_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsll.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2204,9 +2204,9 @@ define <vscale x 2 x i16> @intrinsic_vsll_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i16> %a
@@ -2219,10 +2219,10 @@ define <vscale x 2 x i16> @intrinsic_vsll_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsll.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2236,9 +2236,9 @@ define <vscale x 4 x i16> @intrinsic_vsll_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i16> %a
@@ -2251,10 +2251,10 @@ define <vscale x 4 x i16> @intrinsic_vsll_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsll.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2268,9 +2268,9 @@ define <vscale x 8 x i16> @intrinsic_vsll_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i16> %a
@@ -2283,10 +2283,10 @@ define <vscale x 8 x i16> @intrinsic_vsll_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsll.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2300,9 +2300,9 @@ define <vscale x 16 x i16> @intrinsic_vsll_vi_nxv16i16_nxv16i16_i16(<vscale x 16
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i16> %a
@@ -2315,10 +2315,10 @@ define <vscale x 16 x i16> @intrinsic_vsll_mask_vi_nxv16i16_nxv16i16_i16(<vscale
 ; CHECK-NEXT:    vsll.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsll.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2332,9 +2332,9 @@ define <vscale x 32 x i16> @intrinsic_vsll_vi_nxv32i16_nxv32i16_i16(<vscale x 32
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i16> %a
@@ -2347,10 +2347,10 @@ define <vscale x 32 x i16> @intrinsic_vsll_mask_vi_nxv32i16_nxv32i16_i16(<vscale
 ; CHECK-NEXT:    vsll.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsll.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2364,9 +2364,9 @@ define <vscale x 1 x i32> @intrinsic_vsll_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i32> %a
@@ -2379,10 +2379,10 @@ define <vscale x 1 x i32> @intrinsic_vsll_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsll.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2396,9 +2396,9 @@ define <vscale x 2 x i32> @intrinsic_vsll_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i32> %a
@@ -2411,10 +2411,10 @@ define <vscale x 2 x i32> @intrinsic_vsll_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsll.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2428,9 +2428,9 @@ define <vscale x 4 x i32> @intrinsic_vsll_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i32> %a
@@ -2443,10 +2443,10 @@ define <vscale x 4 x i32> @intrinsic_vsll_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsll.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2460,9 +2460,9 @@ define <vscale x 8 x i32> @intrinsic_vsll_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i32> %a
@@ -2475,10 +2475,10 @@ define <vscale x 8 x i32> @intrinsic_vsll_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsll.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2492,9 +2492,9 @@ define <vscale x 16 x i32> @intrinsic_vsll_vi_nxv16i32_nxv16i32_i32(<vscale x 16
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i32> %a
@@ -2507,10 +2507,10 @@ define <vscale x 16 x i32> @intrinsic_vsll_mask_vi_nxv16i32_nxv16i32_i32(<vscale
 ; CHECK-NEXT:    vsll.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsll.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2524,7 +2524,7 @@ define <vscale x 1 x i64> @intrinsic_vsll_vi_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsll.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsll.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 9,
     i64 %1)
@@ -2539,7 +2539,7 @@ define <vscale x 1 x i64> @intrinsic_vsll_mask_vi_nxv1i64_nxv1i64_i64(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsll.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsll.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 9,
@@ -2556,7 +2556,7 @@ define <vscale x 2 x i64> @intrinsic_vsll_vi_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsll.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsll.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 9,
     i64 %1)
@@ -2571,7 +2571,7 @@ define <vscale x 2 x i64> @intrinsic_vsll_mask_vi_nxv2i64_nxv2i64_i64(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsll.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsll.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 9,
@@ -2588,7 +2588,7 @@ define <vscale x 4 x i64> @intrinsic_vsll_vi_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsll.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsll.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 9,
     i64 %1)
@@ -2603,7 +2603,7 @@ define <vscale x 4 x i64> @intrinsic_vsll_mask_vi_nxv4i64_nxv4i64_i64(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsll.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsll.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 9,
@@ -2620,7 +2620,7 @@ define <vscale x 8 x i64> @intrinsic_vsll_vi_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; CHECK-NEXT:    vsll.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsll.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsll.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 9,
     i64 %1)
@@ -2635,7 +2635,7 @@ define <vscale x 8 x i64> @intrinsic_vsll_mask_vi_nxv8i64_nxv8i64_i64(<vscale x
 ; CHECK-NEXT:    vsll.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsll.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsll.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 9,
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsra-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vsra-rv32.ll
index 1a48652e1863be3fde4f99c97ddb6da80be618a2..1fe52e37a9840b073992bdbffbe461e802afd14c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsra-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsra-rv32.ll
@@ -796,591 +796,591 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vsra_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsra_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vsra_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsra_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vsra_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsra_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vsra_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsra_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vsra_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsra_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vsra_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsra_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vsra_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsra_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vsra_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsra_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vsra_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsra_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vsra_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsra_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vsra_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsra_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vsra_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsra_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vsra_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsra_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i32,
   <vscale x 64 x i1>,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vsra_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsra_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i32 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 64 x i1> %3,
     i32 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vsra_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsra_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vsra_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsra_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vsra_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsra_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vsra_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsra_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vsra_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsra_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vsra_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsra_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vsra_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsra_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vsra_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsra_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vsra_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsra_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vsra_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsra_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vsra_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsra_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vsra_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsra_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32(
   <vscale x 1 x i32>,
   i32,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vsra_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsra_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1388,21 +1388,21 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
   i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vsra_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsra_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 %2,
@@ -1412,19 +1412,19 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32(
   <vscale x 2 x i32>,
   i32,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vsra_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsra_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1432,21 +1432,21 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
   i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vsra_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsra_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 %2,
@@ -1456,19 +1456,19 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32(
   <vscale x 4 x i32>,
   i32,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vsra_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsra_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1476,21 +1476,21 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
   i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vsra_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsra_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 %2,
@@ -1500,19 +1500,19 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32(
   <vscale x 8 x i32>,
   i32,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vsra_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsra_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1520,21 +1520,21 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
   i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vsra_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsra_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 %2,
@@ -1544,19 +1544,19 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32(
   <vscale x 16 x i32>,
   i32,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vsra_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsra_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1564,21 +1564,21 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
   i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vsra_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsra_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 %2,
@@ -1595,9 +1595,9 @@ define <vscale x 1 x i8> @intrinsic_vsra_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i8> %a
@@ -1610,10 +1610,10 @@ define <vscale x 1 x i8> @intrinsic_vsra_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -1627,9 +1627,9 @@ define <vscale x 2 x i8> @intrinsic_vsra_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i8> %a
@@ -1642,10 +1642,10 @@ define <vscale x 2 x i8> @intrinsic_vsra_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -1659,9 +1659,9 @@ define <vscale x 4 x i8> @intrinsic_vsra_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i8> %a
@@ -1674,10 +1674,10 @@ define <vscale x 4 x i8> @intrinsic_vsra_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -1691,9 +1691,9 @@ define <vscale x 8 x i8> @intrinsic_vsra_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i8> %a
@@ -1706,10 +1706,10 @@ define <vscale x 8 x i8> @intrinsic_vsra_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -1723,9 +1723,9 @@ define <vscale x 16 x i8> @intrinsic_vsra_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i8> %a
@@ -1738,10 +1738,10 @@ define <vscale x 16 x i8> @intrinsic_vsra_mask_vi_nxv16i8_nxv16i8_i8(<vscale x 1
 ; CHECK-NEXT:    vsra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -1755,9 +1755,9 @@ define <vscale x 32 x i8> @intrinsic_vsra_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i8> %a
@@ -1770,10 +1770,10 @@ define <vscale x 32 x i8> @intrinsic_vsra_mask_vi_nxv32i8_nxv32i8_i8(<vscale x 3
 ; CHECK-NEXT:    vsra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -1787,9 +1787,9 @@ define <vscale x 64 x i8> @intrinsic_vsra_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 64 x i8> %a
@@ -1802,10 +1802,10 @@ define <vscale x 64 x i8> @intrinsic_vsra_mask_vi_nxv64i8_nxv64i8_i8(<vscale x 6
 ; CHECK-NEXT:    vsra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 64 x i1> %2,
     i32 %3)
 
@@ -1819,9 +1819,9 @@ define <vscale x 1 x i16> @intrinsic_vsra_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i16> %a
@@ -1834,10 +1834,10 @@ define <vscale x 1 x i16> @intrinsic_vsra_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -1851,9 +1851,9 @@ define <vscale x 2 x i16> @intrinsic_vsra_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i16> %a
@@ -1866,10 +1866,10 @@ define <vscale x 2 x i16> @intrinsic_vsra_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -1883,9 +1883,9 @@ define <vscale x 4 x i16> @intrinsic_vsra_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i16> %a
@@ -1898,10 +1898,10 @@ define <vscale x 4 x i16> @intrinsic_vsra_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -1915,9 +1915,9 @@ define <vscale x 8 x i16> @intrinsic_vsra_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i16> %a
@@ -1930,10 +1930,10 @@ define <vscale x 8 x i16> @intrinsic_vsra_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -1947,9 +1947,9 @@ define <vscale x 16 x i16> @intrinsic_vsra_vi_nxv16i16_nxv16i16_i16(<vscale x 16
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i16> %a
@@ -1962,10 +1962,10 @@ define <vscale x 16 x i16> @intrinsic_vsra_mask_vi_nxv16i16_nxv16i16_i16(<vscale
 ; CHECK-NEXT:    vsra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -1979,9 +1979,9 @@ define <vscale x 32 x i16> @intrinsic_vsra_vi_nxv32i16_nxv32i16_i16(<vscale x 32
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i16> %a
@@ -1994,10 +1994,10 @@ define <vscale x 32 x i16> @intrinsic_vsra_mask_vi_nxv32i16_nxv32i16_i16(<vscale
 ; CHECK-NEXT:    vsra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -2011,7 +2011,7 @@ define <vscale x 1 x i32> @intrinsic_vsra_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 9,
     i32 %1)
@@ -2026,7 +2026,7 @@ define <vscale x 1 x i32> @intrinsic_vsra_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 9,
@@ -2043,7 +2043,7 @@ define <vscale x 2 x i32> @intrinsic_vsra_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 9,
     i32 %1)
@@ -2058,7 +2058,7 @@ define <vscale x 2 x i32> @intrinsic_vsra_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 9,
@@ -2075,7 +2075,7 @@ define <vscale x 4 x i32> @intrinsic_vsra_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 9,
     i32 %1)
@@ -2090,7 +2090,7 @@ define <vscale x 4 x i32> @intrinsic_vsra_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 9,
@@ -2107,7 +2107,7 @@ define <vscale x 8 x i32> @intrinsic_vsra_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 9,
     i32 %1)
@@ -2122,7 +2122,7 @@ define <vscale x 8 x i32> @intrinsic_vsra_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 9,
@@ -2139,7 +2139,7 @@ define <vscale x 16 x i32> @intrinsic_vsra_vi_nxv16i32_nxv16i32_i32(<vscale x 16
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 9,
     i32 %1)
@@ -2154,7 +2154,7 @@ define <vscale x 16 x i32> @intrinsic_vsra_mask_vi_nxv16i32_nxv16i32_i32(<vscale
 ; CHECK-NEXT:    vsra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 9,
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsra-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vsra-rv64.ll
index 7c1f4cf787768424f145bad15a7759754b128022..3e078673e18c4b4693c945d5bc416fa95b556f73 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsra-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsra-rv64.ll
@@ -973,811 +973,811 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vsra_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsra_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vsra_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsra_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vsra_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsra_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vsra_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsra_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vsra_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsra_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vsra_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsra_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vsra_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsra_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vsra_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsra_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vsra_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsra_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vsra_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsra_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vsra_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsra_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vsra_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsra_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vsra_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsra_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i64,
   <vscale x 64 x i1>,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vsra_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsra_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i64 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 64 x i1> %3,
     i64 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vsra_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsra_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vsra_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsra_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vsra_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsra_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vsra_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsra_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vsra_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsra_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vsra_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsra_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vsra_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsra_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vsra_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsra_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vsra_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsra_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vsra_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsra_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vsra_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsra_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vsra_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsra_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32(
   <vscale x 1 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vsra_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsra_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
-  i32,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vsra_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsra_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32(
   <vscale x 2 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vsra_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsra_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
-  i32,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vsra_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsra_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32(
   <vscale x 4 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vsra_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsra_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
-  i32,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vsra_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsra_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32(
   <vscale x 8 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vsra_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsra_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
-  i32,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vsra_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsra_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32(
   <vscale x 16 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vsra_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsra_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
-  i32,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vsra_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsra_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vsra.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vsra.nxv1i64(
   <vscale x 1 x i64>,
   i64,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vsra_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vsra_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsra.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsra.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1785,21 +1785,21 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vsra.mask.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vsra.mask.nxv1i64(
   <vscale x 1 x i64>,
   <vscale x 1 x i64>,
   i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vsra_mask_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vsra_mask_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsra.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsra.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 %2,
@@ -1809,19 +1809,19 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vsra.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vsra.nxv2i64(
   <vscale x 2 x i64>,
   i64,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vsra_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vsra_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsra.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsra.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1829,21 +1829,21 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vsra.mask.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vsra.mask.nxv2i64(
   <vscale x 2 x i64>,
   <vscale x 2 x i64>,
   i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vsra_mask_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vsra_mask_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsra.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsra.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 %2,
@@ -1853,19 +1853,19 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vsra.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vsra.nxv4i64(
   <vscale x 4 x i64>,
   i64,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vsra_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vsra_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsra.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsra.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1873,21 +1873,21 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vsra.mask.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vsra.mask.nxv4i64(
   <vscale x 4 x i64>,
   <vscale x 4 x i64>,
   i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vsra_mask_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vsra_mask_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsra.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsra.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 %2,
@@ -1897,19 +1897,19 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vsra.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vsra.nxv8i64(
   <vscale x 8 x i64>,
   i64,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vsra_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsra_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vsra_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsra_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
 ; CHECK-NEXT:    vsra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsra.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsra.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1917,21 +1917,21 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vsra.mask.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vsra.mask.nxv8i64(
   <vscale x 8 x i64>,
   <vscale x 8 x i64>,
   i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vsra_mask_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vsra_mask_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,tu,mu
 ; CHECK-NEXT:    vsra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsra.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsra.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 %2,
@@ -1948,9 +1948,9 @@ define <vscale x 1 x i8> @intrinsic_vsra_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i8> %a
@@ -1963,10 +1963,10 @@ define <vscale x 1 x i8> @intrinsic_vsra_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsra.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -1980,9 +1980,9 @@ define <vscale x 2 x i8> @intrinsic_vsra_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i8> %a
@@ -1995,10 +1995,10 @@ define <vscale x 2 x i8> @intrinsic_vsra_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsra.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2012,9 +2012,9 @@ define <vscale x 4 x i8> @intrinsic_vsra_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i8> %a
@@ -2027,10 +2027,10 @@ define <vscale x 4 x i8> @intrinsic_vsra_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsra.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2044,9 +2044,9 @@ define <vscale x 8 x i8> @intrinsic_vsra_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i8> %a
@@ -2059,10 +2059,10 @@ define <vscale x 8 x i8> @intrinsic_vsra_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsra.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2076,9 +2076,9 @@ define <vscale x 16 x i8> @intrinsic_vsra_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i8> %a
@@ -2091,10 +2091,10 @@ define <vscale x 16 x i8> @intrinsic_vsra_mask_vi_nxv16i8_nxv16i8_i8(<vscale x 1
 ; CHECK-NEXT:    vsra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsra.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2108,9 +2108,9 @@ define <vscale x 32 x i8> @intrinsic_vsra_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i8> %a
@@ -2123,10 +2123,10 @@ define <vscale x 32 x i8> @intrinsic_vsra_mask_vi_nxv32i8_nxv32i8_i8(<vscale x 3
 ; CHECK-NEXT:    vsra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsra.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2140,9 +2140,9 @@ define <vscale x 64 x i8> @intrinsic_vsra_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 64 x i8> %a
@@ -2155,10 +2155,10 @@ define <vscale x 64 x i8> @intrinsic_vsra_mask_vi_nxv64i8_nxv64i8_i8(<vscale x 6
 ; CHECK-NEXT:    vsra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsra.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 64 x i1> %2,
     i64 %3)
 
@@ -2172,9 +2172,9 @@ define <vscale x 1 x i16> @intrinsic_vsra_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i16> %a
@@ -2187,10 +2187,10 @@ define <vscale x 1 x i16> @intrinsic_vsra_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsra.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2204,9 +2204,9 @@ define <vscale x 2 x i16> @intrinsic_vsra_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i16> %a
@@ -2219,10 +2219,10 @@ define <vscale x 2 x i16> @intrinsic_vsra_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsra.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2236,9 +2236,9 @@ define <vscale x 4 x i16> @intrinsic_vsra_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i16> %a
@@ -2251,10 +2251,10 @@ define <vscale x 4 x i16> @intrinsic_vsra_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsra.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2268,9 +2268,9 @@ define <vscale x 8 x i16> @intrinsic_vsra_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i16> %a
@@ -2283,10 +2283,10 @@ define <vscale x 8 x i16> @intrinsic_vsra_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsra.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2300,9 +2300,9 @@ define <vscale x 16 x i16> @intrinsic_vsra_vi_nxv16i16_nxv16i16_i16(<vscale x 16
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i16> %a
@@ -2315,10 +2315,10 @@ define <vscale x 16 x i16> @intrinsic_vsra_mask_vi_nxv16i16_nxv16i16_i16(<vscale
 ; CHECK-NEXT:    vsra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsra.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2332,9 +2332,9 @@ define <vscale x 32 x i16> @intrinsic_vsra_vi_nxv32i16_nxv32i16_i16(<vscale x 32
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i16> %a
@@ -2347,10 +2347,10 @@ define <vscale x 32 x i16> @intrinsic_vsra_mask_vi_nxv32i16_nxv32i16_i16(<vscale
 ; CHECK-NEXT:    vsra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsra.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2364,9 +2364,9 @@ define <vscale x 1 x i32> @intrinsic_vsra_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i32> %a
@@ -2379,10 +2379,10 @@ define <vscale x 1 x i32> @intrinsic_vsra_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsra.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2396,9 +2396,9 @@ define <vscale x 2 x i32> @intrinsic_vsra_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i32> %a
@@ -2411,10 +2411,10 @@ define <vscale x 2 x i32> @intrinsic_vsra_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsra.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2428,9 +2428,9 @@ define <vscale x 4 x i32> @intrinsic_vsra_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i32> %a
@@ -2443,10 +2443,10 @@ define <vscale x 4 x i32> @intrinsic_vsra_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsra.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2460,9 +2460,9 @@ define <vscale x 8 x i32> @intrinsic_vsra_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i32> %a
@@ -2475,10 +2475,10 @@ define <vscale x 8 x i32> @intrinsic_vsra_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsra.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2492,9 +2492,9 @@ define <vscale x 16 x i32> @intrinsic_vsra_vi_nxv16i32_nxv16i32_i32(<vscale x 16
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i32> %a
@@ -2507,10 +2507,10 @@ define <vscale x 16 x i32> @intrinsic_vsra_mask_vi_nxv16i32_nxv16i32_i32(<vscale
 ; CHECK-NEXT:    vsra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsra.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2524,7 +2524,7 @@ define <vscale x 1 x i64> @intrinsic_vsra_vi_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsra.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsra.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 9,
     i64 %1)
@@ -2539,7 +2539,7 @@ define <vscale x 1 x i64> @intrinsic_vsra_mask_vi_nxv1i64_nxv1i64_i64(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsra.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsra.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 9,
@@ -2556,7 +2556,7 @@ define <vscale x 2 x i64> @intrinsic_vsra_vi_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsra.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsra.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 9,
     i64 %1)
@@ -2571,7 +2571,7 @@ define <vscale x 2 x i64> @intrinsic_vsra_mask_vi_nxv2i64_nxv2i64_i64(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsra.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsra.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 9,
@@ -2588,7 +2588,7 @@ define <vscale x 4 x i64> @intrinsic_vsra_vi_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsra.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsra.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 9,
     i64 %1)
@@ -2603,7 +2603,7 @@ define <vscale x 4 x i64> @intrinsic_vsra_mask_vi_nxv4i64_nxv4i64_i64(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsra.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsra.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 9,
@@ -2620,7 +2620,7 @@ define <vscale x 8 x i64> @intrinsic_vsra_vi_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; CHECK-NEXT:    vsra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsra.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsra.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 9,
     i64 %1)
@@ -2635,7 +2635,7 @@ define <vscale x 8 x i64> @intrinsic_vsra_mask_vi_nxv8i64_nxv8i64_i64(<vscale x
 ; CHECK-NEXT:    vsra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsra.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsra.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 9,
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsrl-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vsrl-rv32.ll
index eba2d4bd383b7db496b30ba33896f514c766ac8e..5ad8dea25c01567bafac9ee48ed875bb8914c139 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsrl-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsrl-rv32.ll
@@ -796,591 +796,591 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vsrl_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsrl_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vsrl_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsrl_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vsrl_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsrl_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vsrl_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsrl_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vsrl_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsrl_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vsrl_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsrl_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vsrl_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsrl_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vsrl_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsrl_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vsrl_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsrl_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vsrl_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsrl_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i32,
   <vscale x 64 x i1>,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vsrl_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsrl_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i32 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 64 x i1> %3,
     i32 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vsrl_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsrl_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vsrl_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsrl_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vsrl_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsrl_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vsrl_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsrl_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vsrl_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsrl_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vsrl_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsrl_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vsrl_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsrl_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vsrl_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsrl_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32(
   <vscale x 1 x i32>,
   i32,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vsrl_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsrl_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1388,21 +1388,21 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
   i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vsrl_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsrl_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 %2,
@@ -1412,19 +1412,19 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32(
   <vscale x 2 x i32>,
   i32,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vsrl_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsrl_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1432,21 +1432,21 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
   i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 %2,
@@ -1456,19 +1456,19 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32(
   <vscale x 4 x i32>,
   i32,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vsrl_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsrl_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1476,21 +1476,21 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
   i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 %2,
@@ -1500,19 +1500,19 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32(
   <vscale x 8 x i32>,
   i32,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vsrl_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsrl_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1520,21 +1520,21 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
   i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 %2,
@@ -1544,19 +1544,19 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32(
   <vscale x 16 x i32>,
   i32,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vsrl_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsrl_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1564,21 +1564,21 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
   i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 %2,
@@ -1595,9 +1595,9 @@ define <vscale x 1 x i8> @intrinsic_vsrl_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i8> %a
@@ -1610,10 +1610,10 @@ define <vscale x 1 x i8> @intrinsic_vsrl_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -1627,9 +1627,9 @@ define <vscale x 2 x i8> @intrinsic_vsrl_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i8> %a
@@ -1642,10 +1642,10 @@ define <vscale x 2 x i8> @intrinsic_vsrl_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -1659,9 +1659,9 @@ define <vscale x 4 x i8> @intrinsic_vsrl_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i8> %a
@@ -1674,10 +1674,10 @@ define <vscale x 4 x i8> @intrinsic_vsrl_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -1691,9 +1691,9 @@ define <vscale x 8 x i8> @intrinsic_vsrl_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i8> %a
@@ -1706,10 +1706,10 @@ define <vscale x 8 x i8> @intrinsic_vsrl_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -1723,9 +1723,9 @@ define <vscale x 16 x i8> @intrinsic_vsrl_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i8> %a
@@ -1738,10 +1738,10 @@ define <vscale x 16 x i8> @intrinsic_vsrl_mask_vi_nxv16i8_nxv16i8_i8(<vscale x 1
 ; CHECK-NEXT:    vsrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -1755,9 +1755,9 @@ define <vscale x 32 x i8> @intrinsic_vsrl_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i8> %a
@@ -1770,10 +1770,10 @@ define <vscale x 32 x i8> @intrinsic_vsrl_mask_vi_nxv32i8_nxv32i8_i8(<vscale x 3
 ; CHECK-NEXT:    vsrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -1787,9 +1787,9 @@ define <vscale x 64 x i8> @intrinsic_vsrl_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 64 x i8> %a
@@ -1802,10 +1802,10 @@ define <vscale x 64 x i8> @intrinsic_vsrl_mask_vi_nxv64i8_nxv64i8_i8(<vscale x 6
 ; CHECK-NEXT:    vsrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 64 x i1> %2,
     i32 %3)
 
@@ -1819,9 +1819,9 @@ define <vscale x 1 x i16> @intrinsic_vsrl_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i16> %a
@@ -1834,10 +1834,10 @@ define <vscale x 1 x i16> @intrinsic_vsrl_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -1851,9 +1851,9 @@ define <vscale x 2 x i16> @intrinsic_vsrl_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i16> %a
@@ -1866,10 +1866,10 @@ define <vscale x 2 x i16> @intrinsic_vsrl_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -1883,9 +1883,9 @@ define <vscale x 4 x i16> @intrinsic_vsrl_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i16> %a
@@ -1898,10 +1898,10 @@ define <vscale x 4 x i16> @intrinsic_vsrl_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -1915,9 +1915,9 @@ define <vscale x 8 x i16> @intrinsic_vsrl_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i16> %a
@@ -1930,10 +1930,10 @@ define <vscale x 8 x i16> @intrinsic_vsrl_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -1947,9 +1947,9 @@ define <vscale x 16 x i16> @intrinsic_vsrl_vi_nxv16i16_nxv16i16_i16(<vscale x 16
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i16> %a
@@ -1962,10 +1962,10 @@ define <vscale x 16 x i16> @intrinsic_vsrl_mask_vi_nxv16i16_nxv16i16_i16(<vscale
 ; CHECK-NEXT:    vsrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -1979,9 +1979,9 @@ define <vscale x 32 x i16> @intrinsic_vsrl_vi_nxv32i16_nxv32i16_i16(<vscale x 32
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i16> %a
@@ -1994,10 +1994,10 @@ define <vscale x 32 x i16> @intrinsic_vsrl_mask_vi_nxv32i16_nxv32i16_i16(<vscale
 ; CHECK-NEXT:    vsrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -2011,7 +2011,7 @@ define <vscale x 1 x i32> @intrinsic_vsrl_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 9,
     i32 %1)
@@ -2026,7 +2026,7 @@ define <vscale x 1 x i32> @intrinsic_vsrl_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 9,
@@ -2043,7 +2043,7 @@ define <vscale x 2 x i32> @intrinsic_vsrl_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 9,
     i32 %1)
@@ -2058,7 +2058,7 @@ define <vscale x 2 x i32> @intrinsic_vsrl_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 9,
@@ -2075,7 +2075,7 @@ define <vscale x 4 x i32> @intrinsic_vsrl_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 9,
     i32 %1)
@@ -2090,7 +2090,7 @@ define <vscale x 4 x i32> @intrinsic_vsrl_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 9,
@@ -2107,7 +2107,7 @@ define <vscale x 8 x i32> @intrinsic_vsrl_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 9,
     i32 %1)
@@ -2122,7 +2122,7 @@ define <vscale x 8 x i32> @intrinsic_vsrl_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 9,
@@ -2139,7 +2139,7 @@ define <vscale x 16 x i32> @intrinsic_vsrl_vi_nxv16i32_nxv16i32_i32(<vscale x 16
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 9,
     i32 %1)
@@ -2154,7 +2154,7 @@ define <vscale x 16 x i32> @intrinsic_vsrl_mask_vi_nxv16i32_nxv16i32_i32(<vscale
 ; CHECK-NEXT:    vsrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 9,
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsrl-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vsrl-rv64.ll
index 58e6ef3d730cccd8020c5438378c2fb6bf7521f6..e32a30a1225454b8e0d68fef9ec269b7173353a7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsrl-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsrl-rv64.ll
@@ -973,811 +973,811 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vsrl_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsrl_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vsrl_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vsrl_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vsrl_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsrl_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vsrl_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vsrl_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vsrl_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsrl_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vsrl_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vsrl_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vsrl_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsrl_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vsrl_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsrl_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vsrl_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsrl_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vsrl_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsrl_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i64,
   <vscale x 64 x i1>,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vsrl_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vsrl_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i64 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 64 x i1> %3,
     i64 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vsrl_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsrl_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vsrl_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vsrl_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vsrl_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsrl_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vsrl_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vsrl_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vsrl_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsrl_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vsrl_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsrl_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vsrl_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsrl_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vsrl_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsrl_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32(
   <vscale x 1 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vsrl_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsrl_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
-  i32,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vsrl_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vsrl_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32(
   <vscale x 2 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vsrl_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsrl_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
-  i32,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32(
   <vscale x 4 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vsrl_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsrl_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
-  i32,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32(
   <vscale x 8 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vsrl_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsrl_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
-  i32,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32(
   <vscale x 16 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vsrl_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsrl_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
-  i32,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vsrl.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vsrl.nxv1i64(
   <vscale x 1 x i64>,
   i64,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vsrl_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vsrl_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsrl.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsrl.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1785,21 +1785,21 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vsrl.mask.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vsrl.mask.nxv1i64(
   <vscale x 1 x i64>,
   <vscale x 1 x i64>,
   i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vsrl_mask_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vsrl_mask_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsrl.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsrl.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 %2,
@@ -1809,19 +1809,19 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vsrl.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vsrl.nxv2i64(
   <vscale x 2 x i64>,
   i64,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vsrl_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vsrl_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsrl.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsrl.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1829,21 +1829,21 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vsrl.mask.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vsrl.mask.nxv2i64(
   <vscale x 2 x i64>,
   <vscale x 2 x i64>,
   i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vsrl_mask_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vsrl_mask_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsrl.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsrl.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 %2,
@@ -1853,19 +1853,19 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vsrl.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vsrl.nxv4i64(
   <vscale x 4 x i64>,
   i64,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vsrl_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vsrl_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsrl.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsrl.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1873,21 +1873,21 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vsrl.mask.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vsrl.mask.nxv4i64(
   <vscale x 4 x i64>,
   <vscale x 4 x i64>,
   i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vsrl_mask_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vsrl_mask_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsrl.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsrl.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 %2,
@@ -1897,19 +1897,19 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vsrl.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vsrl.nxv8i64(
   <vscale x 8 x i64>,
   i64,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vsrl_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vsrl_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
 ; CHECK-NEXT:    vsrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsrl.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsrl.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1917,21 +1917,21 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vsrl.mask.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vsrl.mask.nxv8i64(
   <vscale x 8 x i64>,
   <vscale x 8 x i64>,
   i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vsrl_mask_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vsrl_mask_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,tu,mu
 ; CHECK-NEXT:    vsrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsrl.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsrl.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 %2,
@@ -1948,9 +1948,9 @@ define <vscale x 1 x i8> @intrinsic_vsrl_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i8> %a
@@ -1963,10 +1963,10 @@ define <vscale x 1 x i8> @intrinsic_vsrl_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vsrl.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -1980,9 +1980,9 @@ define <vscale x 2 x i8> @intrinsic_vsrl_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i8> %a
@@ -1995,10 +1995,10 @@ define <vscale x 2 x i8> @intrinsic_vsrl_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vsrl.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2012,9 +2012,9 @@ define <vscale x 4 x i8> @intrinsic_vsrl_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i8> %a
@@ -2027,10 +2027,10 @@ define <vscale x 4 x i8> @intrinsic_vsrl_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vsrl.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2044,9 +2044,9 @@ define <vscale x 8 x i8> @intrinsic_vsrl_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i8> %a
@@ -2059,10 +2059,10 @@ define <vscale x 8 x i8> @intrinsic_vsrl_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vsrl.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2076,9 +2076,9 @@ define <vscale x 16 x i8> @intrinsic_vsrl_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i8> %a
@@ -2091,10 +2091,10 @@ define <vscale x 16 x i8> @intrinsic_vsrl_mask_vi_nxv16i8_nxv16i8_i8(<vscale x 1
 ; CHECK-NEXT:    vsrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vsrl.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2108,9 +2108,9 @@ define <vscale x 32 x i8> @intrinsic_vsrl_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i8> %a
@@ -2123,10 +2123,10 @@ define <vscale x 32 x i8> @intrinsic_vsrl_mask_vi_nxv32i8_nxv32i8_i8(<vscale x 3
 ; CHECK-NEXT:    vsrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vsrl.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2140,9 +2140,9 @@ define <vscale x 64 x i8> @intrinsic_vsrl_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 64 x i8> %a
@@ -2155,10 +2155,10 @@ define <vscale x 64 x i8> @intrinsic_vsrl_mask_vi_nxv64i8_nxv64i8_i8(<vscale x 6
 ; CHECK-NEXT:    vsrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vsrl.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 64 x i1> %2,
     i64 %3)
 
@@ -2172,9 +2172,9 @@ define <vscale x 1 x i16> @intrinsic_vsrl_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i16> %a
@@ -2187,10 +2187,10 @@ define <vscale x 1 x i16> @intrinsic_vsrl_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vsrl.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2204,9 +2204,9 @@ define <vscale x 2 x i16> @intrinsic_vsrl_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i16> %a
@@ -2219,10 +2219,10 @@ define <vscale x 2 x i16> @intrinsic_vsrl_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vsrl.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2236,9 +2236,9 @@ define <vscale x 4 x i16> @intrinsic_vsrl_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i16> %a
@@ -2251,10 +2251,10 @@ define <vscale x 4 x i16> @intrinsic_vsrl_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vsrl.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2268,9 +2268,9 @@ define <vscale x 8 x i16> @intrinsic_vsrl_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i16> %a
@@ -2283,10 +2283,10 @@ define <vscale x 8 x i16> @intrinsic_vsrl_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vsrl.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2300,9 +2300,9 @@ define <vscale x 16 x i16> @intrinsic_vsrl_vi_nxv16i16_nxv16i16_i16(<vscale x 16
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i16> %a
@@ -2315,10 +2315,10 @@ define <vscale x 16 x i16> @intrinsic_vsrl_mask_vi_nxv16i16_nxv16i16_i16(<vscale
 ; CHECK-NEXT:    vsrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vsrl.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2332,9 +2332,9 @@ define <vscale x 32 x i16> @intrinsic_vsrl_vi_nxv32i16_nxv32i16_i16(<vscale x 32
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i16> %a
@@ -2347,10 +2347,10 @@ define <vscale x 32 x i16> @intrinsic_vsrl_mask_vi_nxv32i16_nxv32i16_i16(<vscale
 ; CHECK-NEXT:    vsrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vsrl.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2364,9 +2364,9 @@ define <vscale x 1 x i32> @intrinsic_vsrl_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i32> %a
@@ -2379,10 +2379,10 @@ define <vscale x 1 x i32> @intrinsic_vsrl_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vsrl.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2396,9 +2396,9 @@ define <vscale x 2 x i32> @intrinsic_vsrl_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i32> %a
@@ -2411,10 +2411,10 @@ define <vscale x 2 x i32> @intrinsic_vsrl_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vsrl.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2428,9 +2428,9 @@ define <vscale x 4 x i32> @intrinsic_vsrl_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i32> %a
@@ -2443,10 +2443,10 @@ define <vscale x 4 x i32> @intrinsic_vsrl_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vsrl.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2460,9 +2460,9 @@ define <vscale x 8 x i32> @intrinsic_vsrl_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i32> %a
@@ -2475,10 +2475,10 @@ define <vscale x 8 x i32> @intrinsic_vsrl_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vsrl.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2492,9 +2492,9 @@ define <vscale x 16 x i32> @intrinsic_vsrl_vi_nxv16i32_nxv16i32_i32(<vscale x 16
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i32> %a
@@ -2507,10 +2507,10 @@ define <vscale x 16 x i32> @intrinsic_vsrl_mask_vi_nxv16i32_nxv16i32_i32(<vscale
 ; CHECK-NEXT:    vsrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vsrl.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2524,7 +2524,7 @@ define <vscale x 1 x i64> @intrinsic_vsrl_vi_nxv1i64_nxv1i64_i64(<vscale x 1 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsrl.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsrl.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 9,
     i64 %1)
@@ -2539,7 +2539,7 @@ define <vscale x 1 x i64> @intrinsic_vsrl_mask_vi_nxv1i64_nxv1i64_i64(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vsrl.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vsrl.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 9,
@@ -2556,7 +2556,7 @@ define <vscale x 2 x i64> @intrinsic_vsrl_vi_nxv2i64_nxv2i64_i64(<vscale x 2 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsrl.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsrl.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 9,
     i64 %1)
@@ -2571,7 +2571,7 @@ define <vscale x 2 x i64> @intrinsic_vsrl_mask_vi_nxv2i64_nxv2i64_i64(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vsrl.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vsrl.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 9,
@@ -2588,7 +2588,7 @@ define <vscale x 4 x i64> @intrinsic_vsrl_vi_nxv4i64_nxv4i64_i64(<vscale x 4 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsrl.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsrl.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 9,
     i64 %1)
@@ -2603,7 +2603,7 @@ define <vscale x 4 x i64> @intrinsic_vsrl_mask_vi_nxv4i64_nxv4i64_i64(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vsrl.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vsrl.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 9,
@@ -2620,7 +2620,7 @@ define <vscale x 8 x i64> @intrinsic_vsrl_vi_nxv8i64_nxv8i64_i64(<vscale x 8 x i
 ; CHECK-NEXT:    vsrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsrl.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsrl.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 9,
     i64 %1)
@@ -2635,7 +2635,7 @@ define <vscale x 8 x i64> @intrinsic_vsrl_mask_vi_nxv8i64_nxv8i64_i64(<vscale x
 ; CHECK-NEXT:    vsrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vsrl.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vsrl.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 9,
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssra-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vssra-rv32.ll
index 549e1cfd46917e2429105c020a53b73852d3c0c1..945b72f957e322dcb4bbab72c96d1b8b7dc5d992 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssra-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssra-rv32.ll
@@ -796,768 +796,591 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.nxv1i64(
-  <vscale x 1 x i64>,
-  <vscale x 1 x i64>,
-  i32);
-
-define <vscale x 1 x i64> @intrinsic_vssra_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vv_nxv1i64_nxv1i64_nxv1i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
-; CHECK-NEXT:    vssra.vv v8, v8, v9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.nxv1i64(
-    <vscale x 1 x i64> %0,
-    <vscale x 1 x i64> %1,
-    i32 %2)
-
-  ret <vscale x 1 x i64> %a
-}
-
-declare <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.nxv1i64(
-  <vscale x 1 x i64>,
-  <vscale x 1 x i64>,
-  <vscale x 1 x i64>,
-  <vscale x 1 x i1>,
-  i32);
-
-define <vscale x 1 x i64> @intrinsic_vssra_mask_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i64> %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv1i64_nxv1i64_nxv1i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-NEXT:    vssra.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.nxv1i64(
-    <vscale x 1 x i64> %0,
-    <vscale x 1 x i64> %1,
-    <vscale x 1 x i64> %2,
-    <vscale x 1 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 1 x i64> %a
-}
-
-declare <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.nxv2i64(
-  <vscale x 2 x i64>,
-  <vscale x 2 x i64>,
-  i32);
-
-define <vscale x 2 x i64> @intrinsic_vssra_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vv_nxv2i64_nxv2i64_nxv2i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m2,ta,mu
-; CHECK-NEXT:    vssra.vv v8, v8, v10
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.nxv2i64(
-    <vscale x 2 x i64> %0,
-    <vscale x 2 x i64> %1,
-    i32 %2)
-
-  ret <vscale x 2 x i64> %a
-}
-
-declare <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.nxv2i64(
-  <vscale x 2 x i64>,
-  <vscale x 2 x i64>,
-  <vscale x 2 x i64>,
-  <vscale x 2 x i1>,
-  i32);
-
-define <vscale x 2 x i64> @intrinsic_vssra_mask_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv2i64_nxv2i64_nxv2i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
-; CHECK-NEXT:    vssra.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.nxv2i64(
-    <vscale x 2 x i64> %0,
-    <vscale x 2 x i64> %1,
-    <vscale x 2 x i64> %2,
-    <vscale x 2 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 2 x i64> %a
-}
-
-declare <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.nxv4i64(
-  <vscale x 4 x i64>,
-  <vscale x 4 x i64>,
-  i32);
-
-define <vscale x 4 x i64> @intrinsic_vssra_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vv_nxv4i64_nxv4i64_nxv4i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m4,ta,mu
-; CHECK-NEXT:    vssra.vv v8, v8, v12
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.nxv4i64(
-    <vscale x 4 x i64> %0,
-    <vscale x 4 x i64> %1,
-    i32 %2)
-
-  ret <vscale x 4 x i64> %a
-}
-
-declare <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.nxv4i64(
-  <vscale x 4 x i64>,
-  <vscale x 4 x i64>,
-  <vscale x 4 x i64>,
-  <vscale x 4 x i1>,
-  i32);
-
-define <vscale x 4 x i64> @intrinsic_vssra_mask_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i64> %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv4i64_nxv4i64_nxv4i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m4,tu,mu
-; CHECK-NEXT:    vssra.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.nxv4i64(
-    <vscale x 4 x i64> %0,
-    <vscale x 4 x i64> %1,
-    <vscale x 4 x i64> %2,
-    <vscale x 4 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 4 x i64> %a
-}
-
-declare <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.nxv8i64(
-  <vscale x 8 x i64>,
-  <vscale x 8 x i64>,
-  i32);
-
-define <vscale x 8 x i64> @intrinsic_vssra_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vv_nxv8i64_nxv8i64_nxv8i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m8,ta,mu
-; CHECK-NEXT:    vssra.vv v8, v8, v16
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.nxv8i64(
-    <vscale x 8 x i64> %0,
-    <vscale x 8 x i64> %1,
-    i32 %2)
-
-  ret <vscale x 8 x i64> %a
-}
-
-declare <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.nxv8i64(
-  <vscale x 8 x i64>,
-  <vscale x 8 x i64>,
-  <vscale x 8 x i64>,
-  <vscale x 8 x i1>,
-  i32);
-
-define <vscale x 8 x i64> @intrinsic_vssra_mask_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, <vscale x 8 x i64> %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv8i64_nxv8i64_nxv8i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli a0, a1, e64,m8,tu,mu
-; CHECK-NEXT:    vssra.vv v8, v16, v24, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.nxv8i64(
-    <vscale x 8 x i64> %0,
-    <vscale x 8 x i64> %1,
-    <vscale x 8 x i64> %2,
-    <vscale x 8 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 8 x i64> %a
-}
-
-declare <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vssra_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vssra_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vssra_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vssra_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vssra_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vssra_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vssra_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vssra_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vssra_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vssra_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vssra_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vssra_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vssra_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vssra_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vssra_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vssra_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vssra_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vssra_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vssra_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vssra_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vssra_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vssra_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vssra_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vssra_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vssra_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vssra_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i32,
   <vscale x 64 x i1>,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vssra_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vssra_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i32 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 64 x i1> %3,
     i32 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vssra_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vssra_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vssra_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vssra_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vssra_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vssra_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vssra_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vssra_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vssra_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vssra_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vssra_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vssra_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vssra_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vssra_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vssra_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vssra_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vssra_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vssra_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vssra_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vssra_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vssra_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vssra_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vssra_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vssra_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32(
   <vscale x 1 x i32>,
   i32,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vssra_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vssra_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1565,21 +1388,21 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
   i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vssra_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vssra_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 %2,
@@ -1589,19 +1412,19 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32(
   <vscale x 2 x i32>,
   i32,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vssra_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vssra_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1609,21 +1432,21 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
   i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vssra_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vssra_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 %2,
@@ -1633,19 +1456,19 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32(
   <vscale x 4 x i32>,
   i32,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vssra_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vssra_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1653,21 +1476,21 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
   i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vssra_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vssra_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 %2,
@@ -1677,19 +1500,19 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32(
   <vscale x 8 x i32>,
   i32,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vssra_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vssra_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1697,21 +1520,21 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
   i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vssra_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vssra_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 %2,
@@ -1721,19 +1544,19 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32(
   <vscale x 16 x i32>,
   i32,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vssra_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vssra_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1741,21 +1564,21 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
   i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vssra_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vssra_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 %2,
@@ -1765,246 +1588,176 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64(
   <vscale x 1 x i64>,
-  i64,
+  i32,
   i32);
 
-define <vscale x 1 x i64> @intrinsic_vssra_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vssra_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v25, v25, a1
-; CHECK-NEXT:    vmv.v.x v26, a0
-; CHECK-NEXT:    vsll.vx v26, v26, a1
-; CHECK-NEXT:    vsrl.vx v26, v26, a1
-; CHECK-NEXT:    vor.vv v25, v26, v25
-; CHECK-NEXT:    vssra.vv v8, v8, v25
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
+; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64(
     <vscale x 1 x i64> %0,
-    i64 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64(
   <vscale x 1 x i64>,
   <vscale x 1 x i64>,
-  i64,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i64> @intrinsic_vssra_mask_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vssra_mask_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v25, v25, a1
-; CHECK-NEXT:    vmv.v.x v26, a0
-; CHECK-NEXT:    vsll.vx v26, v26, a1
-; CHECK-NEXT:    vsrl.vx v26, v26, a1
-; CHECK-NEXT:    vor.vv v25, v26, v25
-; CHECK-NEXT:    vsetvli a0, a2, e64,m1,tu,mu
-; CHECK-NEXT:    vssra.vv v8, v9, v25, v0.t
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
-    i64 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64(
   <vscale x 2 x i64>,
-  i64,
+  i32,
   i32);
 
-define <vscale x 2 x i64> @intrinsic_vssra_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i64 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vssra_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, a2, e64,m2,ta,mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v26, v26, a1
-; CHECK-NEXT:    vmv.v.x v28, a0
-; CHECK-NEXT:    vsll.vx v28, v28, a1
-; CHECK-NEXT:    vsrl.vx v28, v28, a1
-; CHECK-NEXT:    vor.vv v26, v28, v26
-; CHECK-NEXT:    vssra.vv v8, v8, v26
+; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
+; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64(
     <vscale x 2 x i64> %0,
-    i64 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64(
   <vscale x 2 x i64>,
   <vscale x 2 x i64>,
-  i64,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i64> @intrinsic_vssra_mask_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vssra_mask_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a3, a2, e64,m2,ta,mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v26, v26, a1
-; CHECK-NEXT:    vmv.v.x v28, a0
-; CHECK-NEXT:    vsll.vx v28, v28, a1
-; CHECK-NEXT:    vsrl.vx v28, v28, a1
-; CHECK-NEXT:    vor.vv v26, v28, v26
-; CHECK-NEXT:    vsetvli a0, a2, e64,m2,tu,mu
-; CHECK-NEXT:    vssra.vv v8, v10, v26, v0.t
+; CHECK-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
+; CHECK-NEXT:    vssra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
-    i64 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64(
   <vscale x 4 x i64>,
-  i64,
+  i32,
   i32);
 
-define <vscale x 4 x i64> @intrinsic_vssra_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i64 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vssra_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, a2, e64,m4,ta,mu
-; CHECK-NEXT:    vmv.v.x v28, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v28, v28, a1
-; CHECK-NEXT:    vmv.v.x v12, a0
-; CHECK-NEXT:    vsll.vx v12, v12, a1
-; CHECK-NEXT:    vsrl.vx v12, v12, a1
-; CHECK-NEXT:    vor.vv v28, v12, v28
-; CHECK-NEXT:    vssra.vv v8, v8, v28
+; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
+; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64(
     <vscale x 4 x i64> %0,
-    i64 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64(
   <vscale x 4 x i64>,
   <vscale x 4 x i64>,
-  i64,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i64> @intrinsic_vssra_mask_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vssra_mask_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a3, a2, e64,m4,ta,mu
-; CHECK-NEXT:    vmv.v.x v28, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v28, v28, a1
-; CHECK-NEXT:    vmv.v.x v16, a0
-; CHECK-NEXT:    vsll.vx v16, v16, a1
-; CHECK-NEXT:    vsrl.vx v16, v16, a1
-; CHECK-NEXT:    vor.vv v28, v16, v28
-; CHECK-NEXT:    vsetvli a0, a2, e64,m4,tu,mu
-; CHECK-NEXT:    vssra.vv v8, v12, v28, v0.t
+; CHECK-NEXT:    vsetvli a1, a1, e64,m4,tu,mu
+; CHECK-NEXT:    vssra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
-    i64 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64(
   <vscale x 8 x i64>,
-  i64,
+  i32,
   i32);
 
-define <vscale x 8 x i64> @intrinsic_vssra_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i64 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vssra_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, a2, e64,m8,ta,mu
-; CHECK-NEXT:    vmv.v.x v16, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v16, v16, a1
-; CHECK-NEXT:    vmv.v.x v24, a0
-; CHECK-NEXT:    vsll.vx v24, v24, a1
-; CHECK-NEXT:    vsrl.vx v24, v24, a1
-; CHECK-NEXT:    vor.vv v16, v24, v16
-; CHECK-NEXT:    vssra.vv v8, v8, v16
+; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
+; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64(
     <vscale x 8 x i64> %0,
-    i64 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64(
   <vscale x 8 x i64>,
   <vscale x 8 x i64>,
-  i64,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i64> @intrinsic_vssra_mask_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i64_nxv8i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    csrrs a3, vlenb, zero
-; CHECK-NEXT:    sub sp, sp, a3
-; CHECK-NEXT:    addi a3, sp, 16
-; CHECK-NEXT:    vs1r.v v0, (a3) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a3, a2, e64,m8,ta,mu
-; CHECK-NEXT:    vmv.v.x v24, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v0, v24, a1
-; CHECK-NEXT:    vmv.v.x v24, a0
-; CHECK-NEXT:    vsll.vx v24, v24, a1
-; CHECK-NEXT:    vsrl.vx v24, v24, a1
-; CHECK-NEXT:    vor.vv v24, v24, v0
-; CHECK-NEXT:    vsetvli a0, a2, e64,m8,tu,mu
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl1re8.v v0, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vssra.vv v8, v16, v24, v0.t
-; CHECK-NEXT:    csrrs a0, vlenb, zero
-; CHECK-NEXT:    add sp, sp, a0
-; CHECK-NEXT:    addi sp, sp, 16
+define <vscale x 8 x i64> @intrinsic_vssra_mask_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i64_nxv8i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a1, a1, e64,m8,tu,mu
+; CHECK-NEXT:    vssra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
-    i64 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
@@ -2018,9 +1771,9 @@ define <vscale x 1 x i8> @intrinsic_vssra_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8>
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i8> %a
@@ -2033,10 +1786,10 @@ define <vscale x 1 x i8> @intrinsic_vssra_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -2050,9 +1803,9 @@ define <vscale x 2 x i8> @intrinsic_vssra_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8>
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i8> %a
@@ -2065,10 +1818,10 @@ define <vscale x 2 x i8> @intrinsic_vssra_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -2082,9 +1835,9 @@ define <vscale x 4 x i8> @intrinsic_vssra_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8>
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i8> %a
@@ -2097,10 +1850,10 @@ define <vscale x 4 x i8> @intrinsic_vssra_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -2114,9 +1867,9 @@ define <vscale x 8 x i8> @intrinsic_vssra_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8>
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i8> %a
@@ -2129,10 +1882,10 @@ define <vscale x 8 x i8> @intrinsic_vssra_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -2146,9 +1899,9 @@ define <vscale x 16 x i8> @intrinsic_vssra_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i8> %a
@@ -2161,10 +1914,10 @@ define <vscale x 16 x i8> @intrinsic_vssra_mask_vi_nxv16i8_nxv16i8_i8(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -2178,9 +1931,9 @@ define <vscale x 32 x i8> @intrinsic_vssra_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i8> %a
@@ -2193,10 +1946,10 @@ define <vscale x 32 x i8> @intrinsic_vssra_mask_vi_nxv32i8_nxv32i8_i8(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -2210,9 +1963,9 @@ define <vscale x 64 x i8> @intrinsic_vssra_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 64 x i8> %a
@@ -2225,10 +1978,10 @@ define <vscale x 64 x i8> @intrinsic_vssra_mask_vi_nxv64i8_nxv64i8_i8(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 64 x i1> %2,
     i32 %3)
 
@@ -2242,9 +1995,9 @@ define <vscale x 1 x i16> @intrinsic_vssra_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i16> %a
@@ -2257,10 +2010,10 @@ define <vscale x 1 x i16> @intrinsic_vssra_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -2274,9 +2027,9 @@ define <vscale x 2 x i16> @intrinsic_vssra_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i16> %a
@@ -2289,10 +2042,10 @@ define <vscale x 2 x i16> @intrinsic_vssra_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -2306,9 +2059,9 @@ define <vscale x 4 x i16> @intrinsic_vssra_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i16> %a
@@ -2321,10 +2074,10 @@ define <vscale x 4 x i16> @intrinsic_vssra_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -2338,9 +2091,9 @@ define <vscale x 8 x i16> @intrinsic_vssra_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i16> %a
@@ -2353,10 +2106,10 @@ define <vscale x 8 x i16> @intrinsic_vssra_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -2370,9 +2123,9 @@ define <vscale x 16 x i16> @intrinsic_vssra_vi_nxv16i16_nxv16i16_i16(<vscale x 1
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i16> %a
@@ -2385,10 +2138,10 @@ define <vscale x 16 x i16> @intrinsic_vssra_mask_vi_nxv16i16_nxv16i16_i16(<vscal
 ; CHECK-NEXT:    vssra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -2402,9 +2155,9 @@ define <vscale x 32 x i16> @intrinsic_vssra_vi_nxv32i16_nxv32i16_i16(<vscale x 3
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i16> %a
@@ -2417,10 +2170,10 @@ define <vscale x 32 x i16> @intrinsic_vssra_mask_vi_nxv32i16_nxv32i16_i16(<vscal
 ; CHECK-NEXT:    vssra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -2434,7 +2187,7 @@ define <vscale x 1 x i32> @intrinsic_vssra_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 9,
     i32 %1)
@@ -2449,7 +2202,7 @@ define <vscale x 1 x i32> @intrinsic_vssra_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 9,
@@ -2466,7 +2219,7 @@ define <vscale x 2 x i32> @intrinsic_vssra_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 9,
     i32 %1)
@@ -2481,7 +2234,7 @@ define <vscale x 2 x i32> @intrinsic_vssra_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 9,
@@ -2498,7 +2251,7 @@ define <vscale x 4 x i32> @intrinsic_vssra_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 9,
     i32 %1)
@@ -2513,7 +2266,7 @@ define <vscale x 4 x i32> @intrinsic_vssra_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 9,
@@ -2530,7 +2283,7 @@ define <vscale x 8 x i32> @intrinsic_vssra_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 9,
     i32 %1)
@@ -2545,7 +2298,7 @@ define <vscale x 8 x i32> @intrinsic_vssra_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 9,
@@ -2562,7 +2315,7 @@ define <vscale x 16 x i32> @intrinsic_vssra_vi_nxv16i32_nxv16i32_i32(<vscale x 1
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 9,
     i32 %1)
@@ -2577,7 +2330,7 @@ define <vscale x 16 x i32> @intrinsic_vssra_mask_vi_nxv16i32_nxv16i32_i32(<vscal
 ; CHECK-NEXT:    vssra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 9,
@@ -2586,131 +2339,3 @@ entry:
 
   ret <vscale x 16 x i32> %a
 }
-
-define <vscale x 1 x i64> @intrinsic_vssra_vi_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vi_nxv1i64_nxv1i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
-; CHECK-NEXT:    vssra.vi v8, v8, 9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.i64(
-    <vscale x 1 x i64> %0,
-    i64 9,
-    i32 %1)
-
-  ret <vscale x 1 x i64> %a
-}
-
-define <vscale x 1 x i64> @intrinsic_vssra_mask_vi_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv1i64_nxv1i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.i64(
-    <vscale x 1 x i64> %0,
-    <vscale x 1 x i64> %1,
-    i64 9,
-    <vscale x 1 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 1 x i64> %a
-}
-
-define <vscale x 2 x i64> @intrinsic_vssra_vi_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vi_nxv2i64_nxv2i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m2,ta,mu
-; CHECK-NEXT:    vssra.vi v8, v8, 9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.i64(
-    <vscale x 2 x i64> %0,
-    i64 9,
-    i32 %1)
-
-  ret <vscale x 2 x i64> %a
-}
-
-define <vscale x 2 x i64> @intrinsic_vssra_mask_vi_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv2i64_nxv2i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
-; CHECK-NEXT:    vssra.vi v8, v10, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.i64(
-    <vscale x 2 x i64> %0,
-    <vscale x 2 x i64> %1,
-    i64 9,
-    <vscale x 2 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 2 x i64> %a
-}
-
-define <vscale x 4 x i64> @intrinsic_vssra_vi_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vi_nxv4i64_nxv4i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m4,ta,mu
-; CHECK-NEXT:    vssra.vi v8, v8, 9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.i64(
-    <vscale x 4 x i64> %0,
-    i64 9,
-    i32 %1)
-
-  ret <vscale x 4 x i64> %a
-}
-
-define <vscale x 4 x i64> @intrinsic_vssra_mask_vi_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv4i64_nxv4i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m4,tu,mu
-; CHECK-NEXT:    vssra.vi v8, v12, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.i64(
-    <vscale x 4 x i64> %0,
-    <vscale x 4 x i64> %1,
-    i64 9,
-    <vscale x 4 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 4 x i64> %a
-}
-
-define <vscale x 8 x i64> @intrinsic_vssra_vi_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vi_nxv8i64_nxv8i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m8,ta,mu
-; CHECK-NEXT:    vssra.vi v8, v8, 9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.i64(
-    <vscale x 8 x i64> %0,
-    i64 9,
-    i32 %1)
-
-  ret <vscale x 8 x i64> %a
-}
-
-define <vscale x 8 x i64> @intrinsic_vssra_mask_vi_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv8i64_nxv8i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m8,tu,mu
-; CHECK-NEXT:    vssra.vi v8, v16, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.i64(
-    <vscale x 8 x i64> %0,
-    <vscale x 8 x i64> %1,
-    i64 9,
-    <vscale x 8 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 8 x i64> %a
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssra-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vssra-rv64.ll
index 6aaef72bc7fa63d8b31660d47bf439b3a9e69c67..a7ac6f16fd806630cf5be96c772d26d49cb1170e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssra-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssra-rv64.ll
@@ -973,811 +973,811 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vssra_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vssra_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vssra_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vssra_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vssra_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vssra_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vssra_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vssra_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vssra_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vssra_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vssra_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vssra_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vssra_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vssra_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vssra_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vssra_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vssra_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vssra_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vssra_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vssra_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vssra_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vssra_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vssra_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vssra_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vssra_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vssra_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i64,
   <vscale x 64 x i1>,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vssra_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vssra_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i64 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 64 x i1> %3,
     i64 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vssra_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vssra_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vssra_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vssra_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vssra_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vssra_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vssra_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vssra_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vssra_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vssra_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vssra_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vssra_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vssra_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vssra_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vssra_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vssra_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vssra_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vssra_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vssra_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vssra_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vssra_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vssra_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vssra_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vssra_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32(
   <vscale x 1 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vssra_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vssra_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
-  i32,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vssra_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vssra_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32(
   <vscale x 2 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vssra_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vssra_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
-  i32,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vssra_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vssra_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32(
   <vscale x 4 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vssra_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vssra_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
-  i32,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vssra_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vssra_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32(
   <vscale x 8 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vssra_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vssra_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
-  i32,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vssra_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vssra_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32(
   <vscale x 16 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vssra_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vssra_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
-  i32,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vssra_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vssra_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64(
   <vscale x 1 x i64>,
   i64,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vssra_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vssra_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1785,21 +1785,21 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64(
   <vscale x 1 x i64>,
   <vscale x 1 x i64>,
   i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vssra_mask_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vssra_mask_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 %2,
@@ -1809,19 +1809,19 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64(
   <vscale x 2 x i64>,
   i64,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vssra_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vssra_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1829,21 +1829,21 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64(
   <vscale x 2 x i64>,
   <vscale x 2 x i64>,
   i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vssra_mask_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vssra_mask_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 %2,
@@ -1853,19 +1853,19 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64(
   <vscale x 4 x i64>,
   i64,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vssra_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vssra_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1873,21 +1873,21 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64(
   <vscale x 4 x i64>,
   <vscale x 4 x i64>,
   i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vssra_mask_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vssra_mask_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 %2,
@@ -1897,19 +1897,19 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64(
   <vscale x 8 x i64>,
   i64,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vssra_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssra_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vssra_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssra_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1917,21 +1917,21 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64(
   <vscale x 8 x i64>,
   <vscale x 8 x i64>,
   i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vssra_mask_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vssra_mask_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,tu,mu
 ; CHECK-NEXT:    vssra.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 %2,
@@ -1948,9 +1948,9 @@ define <vscale x 1 x i8> @intrinsic_vssra_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8>
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i8> %a
@@ -1963,10 +1963,10 @@ define <vscale x 1 x i8> @intrinsic_vssra_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -1980,9 +1980,9 @@ define <vscale x 2 x i8> @intrinsic_vssra_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8>
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i8> %a
@@ -1995,10 +1995,10 @@ define <vscale x 2 x i8> @intrinsic_vssra_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2012,9 +2012,9 @@ define <vscale x 4 x i8> @intrinsic_vssra_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8>
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i8> %a
@@ -2027,10 +2027,10 @@ define <vscale x 4 x i8> @intrinsic_vssra_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2044,9 +2044,9 @@ define <vscale x 8 x i8> @intrinsic_vssra_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8>
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i8> %a
@@ -2059,10 +2059,10 @@ define <vscale x 8 x i8> @intrinsic_vssra_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2076,9 +2076,9 @@ define <vscale x 16 x i8> @intrinsic_vssra_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i8> %a
@@ -2091,10 +2091,10 @@ define <vscale x 16 x i8> @intrinsic_vssra_mask_vi_nxv16i8_nxv16i8_i8(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2108,9 +2108,9 @@ define <vscale x 32 x i8> @intrinsic_vssra_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i8> %a
@@ -2123,10 +2123,10 @@ define <vscale x 32 x i8> @intrinsic_vssra_mask_vi_nxv32i8_nxv32i8_i8(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2140,9 +2140,9 @@ define <vscale x 64 x i8> @intrinsic_vssra_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 64 x i8> %a
@@ -2155,10 +2155,10 @@ define <vscale x 64 x i8> @intrinsic_vssra_mask_vi_nxv64i8_nxv64i8_i8(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 64 x i1> %2,
     i64 %3)
 
@@ -2172,9 +2172,9 @@ define <vscale x 1 x i16> @intrinsic_vssra_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i16> %a
@@ -2187,10 +2187,10 @@ define <vscale x 1 x i16> @intrinsic_vssra_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2204,9 +2204,9 @@ define <vscale x 2 x i16> @intrinsic_vssra_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i16> %a
@@ -2219,10 +2219,10 @@ define <vscale x 2 x i16> @intrinsic_vssra_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2236,9 +2236,9 @@ define <vscale x 4 x i16> @intrinsic_vssra_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i16> %a
@@ -2251,10 +2251,10 @@ define <vscale x 4 x i16> @intrinsic_vssra_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2268,9 +2268,9 @@ define <vscale x 8 x i16> @intrinsic_vssra_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i16> %a
@@ -2283,10 +2283,10 @@ define <vscale x 8 x i16> @intrinsic_vssra_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2300,9 +2300,9 @@ define <vscale x 16 x i16> @intrinsic_vssra_vi_nxv16i16_nxv16i16_i16(<vscale x 1
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i16> %a
@@ -2315,10 +2315,10 @@ define <vscale x 16 x i16> @intrinsic_vssra_mask_vi_nxv16i16_nxv16i16_i16(<vscal
 ; CHECK-NEXT:    vssra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2332,9 +2332,9 @@ define <vscale x 32 x i16> @intrinsic_vssra_vi_nxv32i16_nxv32i16_i16(<vscale x 3
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i16> %a
@@ -2347,10 +2347,10 @@ define <vscale x 32 x i16> @intrinsic_vssra_mask_vi_nxv32i16_nxv32i16_i16(<vscal
 ; CHECK-NEXT:    vssra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2364,9 +2364,9 @@ define <vscale x 1 x i32> @intrinsic_vssra_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i32> %a
@@ -2379,10 +2379,10 @@ define <vscale x 1 x i32> @intrinsic_vssra_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2396,9 +2396,9 @@ define <vscale x 2 x i32> @intrinsic_vssra_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i32> %a
@@ -2411,10 +2411,10 @@ define <vscale x 2 x i32> @intrinsic_vssra_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2428,9 +2428,9 @@ define <vscale x 4 x i32> @intrinsic_vssra_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i32> %a
@@ -2443,10 +2443,10 @@ define <vscale x 4 x i32> @intrinsic_vssra_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2460,9 +2460,9 @@ define <vscale x 8 x i32> @intrinsic_vssra_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i32> %a
@@ -2475,10 +2475,10 @@ define <vscale x 8 x i32> @intrinsic_vssra_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2492,9 +2492,9 @@ define <vscale x 16 x i32> @intrinsic_vssra_vi_nxv16i32_nxv16i32_i32(<vscale x 1
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i32> %a
@@ -2507,10 +2507,10 @@ define <vscale x 16 x i32> @intrinsic_vssra_mask_vi_nxv16i32_nxv16i32_i32(<vscal
 ; CHECK-NEXT:    vssra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2524,7 +2524,7 @@ define <vscale x 1 x i64> @intrinsic_vssra_vi_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 9,
     i64 %1)
@@ -2539,7 +2539,7 @@ define <vscale x 1 x i64> @intrinsic_vssra_mask_vi_nxv1i64_nxv1i64_i64(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 9,
@@ -2556,7 +2556,7 @@ define <vscale x 2 x i64> @intrinsic_vssra_vi_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 9,
     i64 %1)
@@ -2571,7 +2571,7 @@ define <vscale x 2 x i64> @intrinsic_vssra_mask_vi_nxv2i64_nxv2i64_i64(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 9,
@@ -2588,7 +2588,7 @@ define <vscale x 4 x i64> @intrinsic_vssra_vi_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 9,
     i64 %1)
@@ -2603,7 +2603,7 @@ define <vscale x 4 x i64> @intrinsic_vssra_mask_vi_nxv4i64_nxv4i64_i64(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 9,
@@ -2620,7 +2620,7 @@ define <vscale x 8 x i64> @intrinsic_vssra_vi_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; CHECK-NEXT:    vssra.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 9,
     i64 %1)
@@ -2635,7 +2635,7 @@ define <vscale x 8 x i64> @intrinsic_vssra_mask_vi_nxv8i64_nxv8i64_i64(<vscale x
 ; CHECK-NEXT:    vssra.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 9,
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssrl-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vssrl-rv32.ll
index e3fb735a064d88a988eddba8c73784e8fcc03852..67b0e77e0ca6897b6afd2da9e133010f09344277 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssrl-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssrl-rv32.ll
@@ -796,768 +796,591 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.nxv1i64(
-  <vscale x 1 x i64>,
-  <vscale x 1 x i64>,
-  i32);
-
-define <vscale x 1 x i64> @intrinsic_vssrl_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vv_nxv1i64_nxv1i64_nxv1i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
-; CHECK-NEXT:    vssrl.vv v8, v8, v9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.nxv1i64(
-    <vscale x 1 x i64> %0,
-    <vscale x 1 x i64> %1,
-    i32 %2)
-
-  ret <vscale x 1 x i64> %a
-}
-
-declare <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.nxv1i64(
-  <vscale x 1 x i64>,
-  <vscale x 1 x i64>,
-  <vscale x 1 x i64>,
-  <vscale x 1 x i1>,
-  i32);
-
-define <vscale x 1 x i64> @intrinsic_vssrl_mask_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i64> %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv1i64_nxv1i64_nxv1i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-NEXT:    vssrl.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.nxv1i64(
-    <vscale x 1 x i64> %0,
-    <vscale x 1 x i64> %1,
-    <vscale x 1 x i64> %2,
-    <vscale x 1 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 1 x i64> %a
-}
-
-declare <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.nxv2i64(
-  <vscale x 2 x i64>,
-  <vscale x 2 x i64>,
-  i32);
-
-define <vscale x 2 x i64> @intrinsic_vssrl_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vv_nxv2i64_nxv2i64_nxv2i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m2,ta,mu
-; CHECK-NEXT:    vssrl.vv v8, v8, v10
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.nxv2i64(
-    <vscale x 2 x i64> %0,
-    <vscale x 2 x i64> %1,
-    i32 %2)
-
-  ret <vscale x 2 x i64> %a
-}
-
-declare <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.nxv2i64(
-  <vscale x 2 x i64>,
-  <vscale x 2 x i64>,
-  <vscale x 2 x i64>,
-  <vscale x 2 x i1>,
-  i32);
-
-define <vscale x 2 x i64> @intrinsic_vssrl_mask_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv2i64_nxv2i64_nxv2i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
-; CHECK-NEXT:    vssrl.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.nxv2i64(
-    <vscale x 2 x i64> %0,
-    <vscale x 2 x i64> %1,
-    <vscale x 2 x i64> %2,
-    <vscale x 2 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 2 x i64> %a
-}
-
-declare <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.nxv4i64(
-  <vscale x 4 x i64>,
-  <vscale x 4 x i64>,
-  i32);
-
-define <vscale x 4 x i64> @intrinsic_vssrl_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vv_nxv4i64_nxv4i64_nxv4i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m4,ta,mu
-; CHECK-NEXT:    vssrl.vv v8, v8, v12
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.nxv4i64(
-    <vscale x 4 x i64> %0,
-    <vscale x 4 x i64> %1,
-    i32 %2)
-
-  ret <vscale x 4 x i64> %a
-}
-
-declare <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.nxv4i64(
-  <vscale x 4 x i64>,
-  <vscale x 4 x i64>,
-  <vscale x 4 x i64>,
-  <vscale x 4 x i1>,
-  i32);
-
-define <vscale x 4 x i64> @intrinsic_vssrl_mask_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i64> %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv4i64_nxv4i64_nxv4i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m4,tu,mu
-; CHECK-NEXT:    vssrl.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.nxv4i64(
-    <vscale x 4 x i64> %0,
-    <vscale x 4 x i64> %1,
-    <vscale x 4 x i64> %2,
-    <vscale x 4 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 4 x i64> %a
-}
-
-declare <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.nxv8i64(
-  <vscale x 8 x i64>,
-  <vscale x 8 x i64>,
-  i32);
-
-define <vscale x 8 x i64> @intrinsic_vssrl_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vv_nxv8i64_nxv8i64_nxv8i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m8,ta,mu
-; CHECK-NEXT:    vssrl.vv v8, v8, v16
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.nxv8i64(
-    <vscale x 8 x i64> %0,
-    <vscale x 8 x i64> %1,
-    i32 %2)
-
-  ret <vscale x 8 x i64> %a
-}
-
-declare <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.nxv8i64(
-  <vscale x 8 x i64>,
-  <vscale x 8 x i64>,
-  <vscale x 8 x i64>,
-  <vscale x 8 x i1>,
-  i32);
-
-define <vscale x 8 x i64> @intrinsic_vssrl_mask_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, <vscale x 8 x i64> %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv8i64_nxv8i64_nxv8i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli a0, a1, e64,m8,tu,mu
-; CHECK-NEXT:    vssrl.vv v8, v16, v24, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.nxv8i64(
-    <vscale x 8 x i64> %0,
-    <vscale x 8 x i64> %1,
-    <vscale x 8 x i64> %2,
-    <vscale x 8 x i1> %3,
-    i32 %4)
-
-  ret <vscale x 8 x i64> %a
-}
-
-declare <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vssrl_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vssrl_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i8> @intrinsic_vssrl_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vssrl_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vssrl_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vssrl_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i8> @intrinsic_vssrl_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vssrl_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vssrl_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vssrl_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i8> @intrinsic_vssrl_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vssrl_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vssrl_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vssrl_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i8> @intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vssrl_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vssrl_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i8> @intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vssrl_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vssrl_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i8> @intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i32,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vssrl_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vssrl_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i32,
   <vscale x 64 x i1>,
   i32);
 
-define <vscale x 64 x i8> @intrinsic_vssrl_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vssrl_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i32 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i32 %2,
     <vscale x 64 x i1> %3,
     i32 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vssrl_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vssrl_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i16> @intrinsic_vssrl_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vssrl_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vssrl_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vssrl_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i16> @intrinsic_vssrl_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vssrl_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vssrl_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vssrl_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i16> @intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vssrl_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vssrl_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i16> @intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vssrl_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vssrl_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i16> @intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 16 x i1> %3,
     i32 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i32,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vssrl_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vssrl_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i32,
   <vscale x 32 x i1>,
   i32);
 
-define <vscale x 32 x i16> @intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i32 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i32 %2,
     <vscale x 32 x i1> %3,
     i32 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32(
   <vscale x 1 x i32>,
   i32,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vssrl_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vssrl_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1565,21 +1388,21 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
   i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i32> @intrinsic_vssrl_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vssrl_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 %2,
@@ -1589,19 +1412,19 @@ entry:
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32(
   <vscale x 2 x i32>,
   i32,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vssrl_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vssrl_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1609,21 +1432,21 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
   i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i32> @intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 %2,
@@ -1633,19 +1456,19 @@ entry:
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32(
   <vscale x 4 x i32>,
   i32,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vssrl_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vssrl_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1653,21 +1476,21 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
   i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i32> @intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 %2,
@@ -1677,19 +1500,19 @@ entry:
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32(
   <vscale x 8 x i32>,
   i32,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vssrl_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vssrl_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1697,21 +1520,21 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
   i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i32> @intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 %2,
@@ -1721,19 +1544,19 @@ entry:
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32(
   <vscale x 16 x i32>,
   i32,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vssrl_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vssrl_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 %1,
     i32 %2)
@@ -1741,21 +1564,21 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
   i32,
   <vscale x 16 x i1>,
   i32);
 
-define <vscale x 16 x i32> @intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 %2,
@@ -1765,246 +1588,176 @@ entry:
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64(
   <vscale x 1 x i64>,
-  i64,
+  i32,
   i32);
 
-define <vscale x 1 x i64> @intrinsic_vssrl_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vssrl_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v25, v25, a1
-; CHECK-NEXT:    vmv.v.x v26, a0
-; CHECK-NEXT:    vsll.vx v26, v26, a1
-; CHECK-NEXT:    vsrl.vx v26, v26, a1
-; CHECK-NEXT:    vor.vv v25, v26, v25
-; CHECK-NEXT:    vssrl.vv v8, v8, v25
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
+; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64(
     <vscale x 1 x i64> %0,
-    i64 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64(
   <vscale x 1 x i64>,
   <vscale x 1 x i64>,
-  i64,
+  i32,
   <vscale x 1 x i1>,
   i32);
 
-define <vscale x 1 x i64> @intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
-; CHECK-NEXT:    vmv.v.x v25, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v25, v25, a1
-; CHECK-NEXT:    vmv.v.x v26, a0
-; CHECK-NEXT:    vsll.vx v26, v26, a1
-; CHECK-NEXT:    vsrl.vx v26, v26, a1
-; CHECK-NEXT:    vor.vv v25, v26, v25
-; CHECK-NEXT:    vsetvli a0, a2, e64,m1,tu,mu
-; CHECK-NEXT:    vssrl.vv v8, v9, v25, v0.t
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
-    i64 %2,
+    i32 %2,
     <vscale x 1 x i1> %3,
     i32 %4)
 
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64(
   <vscale x 2 x i64>,
-  i64,
+  i32,
   i32);
 
-define <vscale x 2 x i64> @intrinsic_vssrl_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i64 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vssrl_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, a2, e64,m2,ta,mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v26, v26, a1
-; CHECK-NEXT:    vmv.v.x v28, a0
-; CHECK-NEXT:    vsll.vx v28, v28, a1
-; CHECK-NEXT:    vsrl.vx v28, v28, a1
-; CHECK-NEXT:    vor.vv v26, v28, v26
-; CHECK-NEXT:    vssrl.vv v8, v8, v26
+; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
+; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64(
     <vscale x 2 x i64> %0,
-    i64 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64(
   <vscale x 2 x i64>,
   <vscale x 2 x i64>,
-  i64,
+  i32,
   <vscale x 2 x i1>,
   i32);
 
-define <vscale x 2 x i64> @intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a3, a2, e64,m2,ta,mu
-; CHECK-NEXT:    vmv.v.x v26, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v26, v26, a1
-; CHECK-NEXT:    vmv.v.x v28, a0
-; CHECK-NEXT:    vsll.vx v28, v28, a1
-; CHECK-NEXT:    vsrl.vx v28, v28, a1
-; CHECK-NEXT:    vor.vv v26, v28, v26
-; CHECK-NEXT:    vsetvli a0, a2, e64,m2,tu,mu
-; CHECK-NEXT:    vssrl.vv v8, v10, v26, v0.t
+; CHECK-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
+; CHECK-NEXT:    vssrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
-    i64 %2,
+    i32 %2,
     <vscale x 2 x i1> %3,
     i32 %4)
 
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64(
   <vscale x 4 x i64>,
-  i64,
+  i32,
   i32);
 
-define <vscale x 4 x i64> @intrinsic_vssrl_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i64 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vssrl_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, a2, e64,m4,ta,mu
-; CHECK-NEXT:    vmv.v.x v28, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v28, v28, a1
-; CHECK-NEXT:    vmv.v.x v12, a0
-; CHECK-NEXT:    vsll.vx v12, v12, a1
-; CHECK-NEXT:    vsrl.vx v12, v12, a1
-; CHECK-NEXT:    vor.vv v28, v12, v28
-; CHECK-NEXT:    vssrl.vv v8, v8, v28
+; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
+; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64(
     <vscale x 4 x i64> %0,
-    i64 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64(
   <vscale x 4 x i64>,
   <vscale x 4 x i64>,
-  i64,
+  i32,
   <vscale x 4 x i1>,
   i32);
 
-define <vscale x 4 x i64> @intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a3, a2, e64,m4,ta,mu
-; CHECK-NEXT:    vmv.v.x v28, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v28, v28, a1
-; CHECK-NEXT:    vmv.v.x v16, a0
-; CHECK-NEXT:    vsll.vx v16, v16, a1
-; CHECK-NEXT:    vsrl.vx v16, v16, a1
-; CHECK-NEXT:    vor.vv v28, v16, v28
-; CHECK-NEXT:    vsetvli a0, a2, e64,m4,tu,mu
-; CHECK-NEXT:    vssrl.vv v8, v12, v28, v0.t
+; CHECK-NEXT:    vsetvli a1, a1, e64,m4,tu,mu
+; CHECK-NEXT:    vssrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
-    i64 %2,
+    i32 %2,
     <vscale x 4 x i1> %3,
     i32 %4)
 
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64(
   <vscale x 8 x i64>,
-  i64,
+  i32,
   i32);
 
-define <vscale x 8 x i64> @intrinsic_vssrl_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i64 %1, i32 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vssrl_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, i32 %1, i32 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, a2, e64,m8,ta,mu
-; CHECK-NEXT:    vmv.v.x v16, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v16, v16, a1
-; CHECK-NEXT:    vmv.v.x v24, a0
-; CHECK-NEXT:    vsll.vx v24, v24, a1
-; CHECK-NEXT:    vsrl.vx v24, v24, a1
-; CHECK-NEXT:    vor.vv v16, v24, v16
-; CHECK-NEXT:    vssrl.vv v8, v8, v16
+; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
+; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64(
     <vscale x 8 x i64> %0,
-    i64 %1,
+    i32 %1,
     i32 %2)
 
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64(
   <vscale x 8 x i64>,
   <vscale x 8 x i64>,
-  i64,
+  i32,
   <vscale x 8 x i1>,
   i32);
 
-define <vscale x 8 x i64> @intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    csrrs a3, vlenb, zero
-; CHECK-NEXT:    sub sp, sp, a3
-; CHECK-NEXT:    addi a3, sp, 16
-; CHECK-NEXT:    vs1r.v v0, (a3) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a3, a2, e64,m8,ta,mu
-; CHECK-NEXT:    vmv.v.x v24, a1
-; CHECK-NEXT:    addi a1, zero, 32
-; CHECK-NEXT:    vsll.vx v0, v24, a1
-; CHECK-NEXT:    vmv.v.x v24, a0
-; CHECK-NEXT:    vsll.vx v24, v24, a1
-; CHECK-NEXT:    vsrl.vx v24, v24, a1
-; CHECK-NEXT:    vor.vv v24, v24, v0
-; CHECK-NEXT:    vsetvli a0, a2, e64,m8,tu,mu
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl1re8.v v0, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vssrl.vv v8, v16, v24, v0.t
-; CHECK-NEXT:    csrrs a0, vlenb, zero
-; CHECK-NEXT:    add sp, sp, a0
-; CHECK-NEXT:    addi sp, sp, 16
+define <vscale x 8 x i64> @intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a1, a1, e64,m8,tu,mu
+; CHECK-NEXT:    vssrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
-    i64 %2,
+    i32 %2,
     <vscale x 8 x i1> %3,
     i32 %4)
 
@@ -2018,9 +1771,9 @@ define <vscale x 1 x i8> @intrinsic_vssrl_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8>
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i8> %a
@@ -2033,10 +1786,10 @@ define <vscale x 1 x i8> @intrinsic_vssrl_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -2050,9 +1803,9 @@ define <vscale x 2 x i8> @intrinsic_vssrl_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8>
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i8> %a
@@ -2065,10 +1818,10 @@ define <vscale x 2 x i8> @intrinsic_vssrl_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -2082,9 +1835,9 @@ define <vscale x 4 x i8> @intrinsic_vssrl_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8>
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i8> %a
@@ -2097,10 +1850,10 @@ define <vscale x 4 x i8> @intrinsic_vssrl_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -2114,9 +1867,9 @@ define <vscale x 8 x i8> @intrinsic_vssrl_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8>
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i8> %a
@@ -2129,10 +1882,10 @@ define <vscale x 8 x i8> @intrinsic_vssrl_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -2146,9 +1899,9 @@ define <vscale x 16 x i8> @intrinsic_vssrl_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i8> %a
@@ -2161,10 +1914,10 @@ define <vscale x 16 x i8> @intrinsic_vssrl_mask_vi_nxv16i8_nxv16i8_i8(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -2178,9 +1931,9 @@ define <vscale x 32 x i8> @intrinsic_vssrl_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i8> %a
@@ -2193,10 +1946,10 @@ define <vscale x 32 x i8> @intrinsic_vssrl_mask_vi_nxv32i8_nxv32i8_i8(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -2210,9 +1963,9 @@ define <vscale x 64 x i8> @intrinsic_vssrl_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 64 x i8> %a
@@ -2225,10 +1978,10 @@ define <vscale x 64 x i8> @intrinsic_vssrl_mask_vi_nxv64i8_nxv64i8_i8(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i32 9,
     <vscale x 64 x i1> %2,
     i32 %3)
 
@@ -2242,9 +1995,9 @@ define <vscale x 1 x i16> @intrinsic_vssrl_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 1 x i16> %a
@@ -2257,10 +2010,10 @@ define <vscale x 1 x i16> @intrinsic_vssrl_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 1 x i1> %2,
     i32 %3)
 
@@ -2274,9 +2027,9 @@ define <vscale x 2 x i16> @intrinsic_vssrl_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 2 x i16> %a
@@ -2289,10 +2042,10 @@ define <vscale x 2 x i16> @intrinsic_vssrl_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 2 x i1> %2,
     i32 %3)
 
@@ -2306,9 +2059,9 @@ define <vscale x 4 x i16> @intrinsic_vssrl_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 4 x i16> %a
@@ -2321,10 +2074,10 @@ define <vscale x 4 x i16> @intrinsic_vssrl_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 4 x i1> %2,
     i32 %3)
 
@@ -2338,9 +2091,9 @@ define <vscale x 8 x i16> @intrinsic_vssrl_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 8 x i16> %a
@@ -2353,10 +2106,10 @@ define <vscale x 8 x i16> @intrinsic_vssrl_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 8 x i1> %2,
     i32 %3)
 
@@ -2370,9 +2123,9 @@ define <vscale x 16 x i16> @intrinsic_vssrl_vi_nxv16i16_nxv16i16_i16(<vscale x 1
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 16 x i16> %a
@@ -2385,10 +2138,10 @@ define <vscale x 16 x i16> @intrinsic_vssrl_mask_vi_nxv16i16_nxv16i16_i16(<vscal
 ; CHECK-NEXT:    vssrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 16 x i1> %2,
     i32 %3)
 
@@ -2402,9 +2155,9 @@ define <vscale x 32 x i16> @intrinsic_vssrl_vi_nxv32i16_nxv32i16_i16(<vscale x 3
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i32 9,
     i32 %1)
 
   ret <vscale x 32 x i16> %a
@@ -2417,10 +2170,10 @@ define <vscale x 32 x i16> @intrinsic_vssrl_mask_vi_nxv32i16_nxv32i16_i16(<vscal
 ; CHECK-NEXT:    vssrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i32 9,
     <vscale x 32 x i1> %2,
     i32 %3)
 
@@ -2434,7 +2187,7 @@ define <vscale x 1 x i32> @intrinsic_vssrl_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32(
     <vscale x 1 x i32> %0,
     i32 9,
     i32 %1)
@@ -2449,7 +2202,7 @@ define <vscale x 1 x i32> @intrinsic_vssrl_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
     i32 9,
@@ -2466,7 +2219,7 @@ define <vscale x 2 x i32> @intrinsic_vssrl_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32(
     <vscale x 2 x i32> %0,
     i32 9,
     i32 %1)
@@ -2481,7 +2234,7 @@ define <vscale x 2 x i32> @intrinsic_vssrl_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
     i32 9,
@@ -2498,7 +2251,7 @@ define <vscale x 4 x i32> @intrinsic_vssrl_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32(
     <vscale x 4 x i32> %0,
     i32 9,
     i32 %1)
@@ -2513,7 +2266,7 @@ define <vscale x 4 x i32> @intrinsic_vssrl_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
     i32 9,
@@ -2530,7 +2283,7 @@ define <vscale x 8 x i32> @intrinsic_vssrl_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32(
     <vscale x 8 x i32> %0,
     i32 9,
     i32 %1)
@@ -2545,7 +2298,7 @@ define <vscale x 8 x i32> @intrinsic_vssrl_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
     i32 9,
@@ -2562,7 +2315,7 @@ define <vscale x 16 x i32> @intrinsic_vssrl_vi_nxv16i32_nxv16i32_i32(<vscale x 1
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32(
     <vscale x 16 x i32> %0,
     i32 9,
     i32 %1)
@@ -2577,7 +2330,7 @@ define <vscale x 16 x i32> @intrinsic_vssrl_mask_vi_nxv16i32_nxv16i32_i32(<vscal
 ; CHECK-NEXT:    vssrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
     i32 9,
@@ -2586,131 +2339,3 @@ entry:
 
   ret <vscale x 16 x i32> %a
 }
-
-define <vscale x 1 x i64> @intrinsic_vssrl_vi_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vi_nxv1i64_nxv1i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
-; CHECK-NEXT:    vssrl.vi v8, v8, 9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.i64(
-    <vscale x 1 x i64> %0,
-    i64 9,
-    i32 %1)
-
-  ret <vscale x 1 x i64> %a
-}
-
-define <vscale x 1 x i64> @intrinsic_vssrl_mask_vi_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv1i64_nxv1i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.i64(
-    <vscale x 1 x i64> %0,
-    <vscale x 1 x i64> %1,
-    i64 9,
-    <vscale x 1 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 1 x i64> %a
-}
-
-define <vscale x 2 x i64> @intrinsic_vssrl_vi_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vi_nxv2i64_nxv2i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m2,ta,mu
-; CHECK-NEXT:    vssrl.vi v8, v8, 9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.i64(
-    <vscale x 2 x i64> %0,
-    i64 9,
-    i32 %1)
-
-  ret <vscale x 2 x i64> %a
-}
-
-define <vscale x 2 x i64> @intrinsic_vssrl_mask_vi_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv2i64_nxv2i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
-; CHECK-NEXT:    vssrl.vi v8, v10, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.i64(
-    <vscale x 2 x i64> %0,
-    <vscale x 2 x i64> %1,
-    i64 9,
-    <vscale x 2 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 2 x i64> %a
-}
-
-define <vscale x 4 x i64> @intrinsic_vssrl_vi_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vi_nxv4i64_nxv4i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m4,ta,mu
-; CHECK-NEXT:    vssrl.vi v8, v8, 9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.i64(
-    <vscale x 4 x i64> %0,
-    i64 9,
-    i32 %1)
-
-  ret <vscale x 4 x i64> %a
-}
-
-define <vscale x 4 x i64> @intrinsic_vssrl_mask_vi_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv4i64_nxv4i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m4,tu,mu
-; CHECK-NEXT:    vssrl.vi v8, v12, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.i64(
-    <vscale x 4 x i64> %0,
-    <vscale x 4 x i64> %1,
-    i64 9,
-    <vscale x 4 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 4 x i64> %a
-}
-
-define <vscale x 8 x i64> @intrinsic_vssrl_vi_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i32 %1) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vi_nxv8i64_nxv8i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m8,ta,mu
-; CHECK-NEXT:    vssrl.vi v8, v8, 9
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.i64(
-    <vscale x 8 x i64> %0,
-    i64 9,
-    i32 %1)
-
-  ret <vscale x 8 x i64> %a
-}
-
-define <vscale x 8 x i64> @intrinsic_vssrl_mask_vi_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, i32 %3) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv8i64_nxv8i64_i64:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, a0, e64,m8,tu,mu
-; CHECK-NEXT:    vssrl.vi v8, v16, 9, v0.t
-; CHECK-NEXT:    jalr zero, 0(ra)
-entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.i64(
-    <vscale x 8 x i64> %0,
-    <vscale x 8 x i64> %1,
-    i64 9,
-    <vscale x 8 x i1> %2,
-    i32 %3)
-
-  ret <vscale x 8 x i64> %a
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssrl-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vssrl-rv64.ll
index 369519f72caed1c817bc59a47ea4ee0a556f18df..d6305809cb54b64ea8d02dce9ad55cdcd0742a49 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssrl-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssrl-rv64.ll
@@ -973,811 +973,811 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8(
   <vscale x 1 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vssrl_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vssrl_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.i8(
+declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8(
   <vscale x 1 x i8>,
   <vscale x 1 x i8>,
-  i8,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i8> @intrinsic_vssrl_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i8_nxv1i8_i8:
+define <vscale x 1 x i8> @intrinsic_vssrl_mask_vx_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf8,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8(
   <vscale x 2 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vssrl_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vssrl_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.i8(
+declare <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8(
   <vscale x 2 x i8>,
   <vscale x 2 x i8>,
-  i8,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i8> @intrinsic_vssrl_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i8_nxv2i8_i8:
+define <vscale x 2 x i8> @intrinsic_vssrl_mask_vx_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8(
   <vscale x 4 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vssrl_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vssrl_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.i8(
+declare <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8(
   <vscale x 4 x i8>,
   <vscale x 4 x i8>,
-  i8,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i8> @intrinsic_vssrl_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i8_nxv4i8_i8:
+define <vscale x 4 x i8> @intrinsic_vssrl_mask_vx_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,mf2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8(
   <vscale x 8 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vssrl_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vssrl_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.i8(
+declare <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8(
   <vscale x 8 x i8>,
   <vscale x 8 x i8>,
-  i8,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i8> @intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8_i8:
+define <vscale x 8 x i8> @intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8(
   <vscale x 16 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vssrl_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vssrl_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.i8(
+declare <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8(
   <vscale x 16 x i8>,
   <vscale x 16 x i8>,
-  i8,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i8> @intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8_i8:
+define <vscale x 16 x i8> @intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8(
   <vscale x 32 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vssrl_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vssrl_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.i8(
+declare <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8(
   <vscale x 32 x i8>,
   <vscale x 32 x i8>,
-  i8,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i8> @intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8_i8:
+define <vscale x 32 x i8> @intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8(
   <vscale x 64 x i8>,
-  i8,
+  i64,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vssrl_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vssrl_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.i8(
+declare <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8(
   <vscale x 64 x i8>,
   <vscale x 64 x i8>,
-  i8,
+  i64,
   <vscale x 64 x i1>,
   i64);
 
-define <vscale x 64 x i8> @intrinsic_vssrl_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv64i8_nxv64i8_i8:
+define <vscale x 64 x i8> @intrinsic_vssrl_mask_vx_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i64 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 %2,
+    i64 %2,
     <vscale x 64 x i1> %3,
     i64 %4)
 
   ret <vscale x 64 x i8> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16(
   <vscale x 1 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vssrl_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vssrl_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.i16(
+declare <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16(
   <vscale x 1 x i16>,
   <vscale x 1 x i16>,
-  i16,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i16> @intrinsic_vssrl_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i16_nxv1i16_i16:
+define <vscale x 1 x i16> @intrinsic_vssrl_mask_vx_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16(
   <vscale x 2 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vssrl_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vssrl_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.i16(
+declare <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16(
   <vscale x 2 x i16>,
   <vscale x 2 x i16>,
-  i16,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i16> @intrinsic_vssrl_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i16_nxv2i16_i16:
+define <vscale x 2 x i16> @intrinsic_vssrl_mask_vx_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,mf2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16(
   <vscale x 4 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vssrl_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vssrl_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.i16(
+declare <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16(
   <vscale x 4 x i16>,
   <vscale x 4 x i16>,
-  i16,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i16> @intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16_i16:
+define <vscale x 4 x i16> @intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16(
   <vscale x 8 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vssrl_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vssrl_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.i16(
+declare <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16(
   <vscale x 8 x i16>,
   <vscale x 8 x i16>,
-  i16,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i16> @intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16_i16:
+define <vscale x 8 x i16> @intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16(
   <vscale x 16 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vssrl_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vssrl_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.i16(
+declare <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16(
   <vscale x 16 x i16>,
   <vscale x 16 x i16>,
-  i16,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i16> @intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16_i16:
+define <vscale x 16 x i16> @intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16(
   <vscale x 32 x i16>,
-  i16,
+  i64,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vssrl_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vssrl_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.i16(
+declare <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16(
   <vscale x 32 x i16>,
   <vscale x 32 x i16>,
-  i16,
+  i64,
   <vscale x 32 x i1>,
   i64);
 
-define <vscale x 32 x i16> @intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16_i16:
+define <vscale x 32 x i16> @intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i64 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 %2,
+    i64 %2,
     <vscale x 32 x i1> %3,
     i64 %4)
 
   ret <vscale x 32 x i16> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32(
   <vscale x 1 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vssrl_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vssrl_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.i32(
+declare <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32(
   <vscale x 1 x i32>,
   <vscale x 1 x i32>,
-  i32,
+  i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i32> @intrinsic_vssrl_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i32_nxv1i32_i32:
+define <vscale x 1 x i32> @intrinsic_vssrl_mask_vx_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,mf2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 1 x i1> %3,
     i64 %4)
 
   ret <vscale x 1 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32(
   <vscale x 2 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vssrl_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vssrl_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.i32(
+declare <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32(
   <vscale x 2 x i32>,
   <vscale x 2 x i32>,
-  i32,
+  i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i32> @intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32_i32:
+define <vscale x 2 x i32> @intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 2 x i1> %3,
     i64 %4)
 
   ret <vscale x 2 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32(
   <vscale x 4 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vssrl_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vssrl_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.i32(
+declare <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32(
   <vscale x 4 x i32>,
   <vscale x 4 x i32>,
-  i32,
+  i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i32> @intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32_i32:
+define <vscale x 4 x i32> @intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 4 x i1> %3,
     i64 %4)
 
   ret <vscale x 4 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32(
   <vscale x 8 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vssrl_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vssrl_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.i32(
+declare <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32(
   <vscale x 8 x i32>,
   <vscale x 8 x i32>,
-  i32,
+  i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i32> @intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32_i32:
+define <vscale x 8 x i32> @intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 8 x i1> %3,
     i64 %4)
 
   ret <vscale x 8 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32(
   <vscale x 16 x i32>,
-  i32,
+  i64,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vssrl_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vssrl_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 %1,
+    i64 %1,
     i64 %2)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.i32(
+declare <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32(
   <vscale x 16 x i32>,
   <vscale x 16 x i32>,
-  i32,
+  i64,
   <vscale x 16 x i1>,
   i64);
 
-define <vscale x 16 x i32> @intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32_i32:
+define <vscale x 16 x i32> @intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i64 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 %2,
+    i64 %2,
     <vscale x 16 x i1> %3,
     i64 %4)
 
   ret <vscale x 16 x i32> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64(
   <vscale x 1 x i64>,
   i64,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vssrl_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vssrl_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1785,21 +1785,21 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.i64(
+declare <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64(
   <vscale x 1 x i64>,
   <vscale x 1 x i64>,
   i64,
   <vscale x 1 x i1>,
   i64);
 
-define <vscale x 1 x i64> @intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64_i64:
+define <vscale x 1 x i64> @intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 %2,
@@ -1809,19 +1809,19 @@ entry:
   ret <vscale x 1 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64(
   <vscale x 2 x i64>,
   i64,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vssrl_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vssrl_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1829,21 +1829,21 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.i64(
+declare <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64(
   <vscale x 2 x i64>,
   <vscale x 2 x i64>,
   i64,
   <vscale x 2 x i1>,
   i64);
 
-define <vscale x 2 x i64> @intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64_i64:
+define <vscale x 2 x i64> @intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 %2,
@@ -1853,19 +1853,19 @@ entry:
   ret <vscale x 2 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64(
   <vscale x 4 x i64>,
   i64,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vssrl_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vssrl_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1873,21 +1873,21 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.i64(
+declare <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64(
   <vscale x 4 x i64>,
   <vscale x 4 x i64>,
   i64,
   <vscale x 4 x i1>,
   i64);
 
-define <vscale x 4 x i64> @intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64_i64:
+define <vscale x 4 x i64> @intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m4,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 %2,
@@ -1897,19 +1897,19 @@ entry:
   ret <vscale x 4 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64(
   <vscale x 8 x i64>,
   i64,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vssrl_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vssrl_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,ta,mu
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 %1,
     i64 %2)
@@ -1917,21 +1917,21 @@ entry:
   ret <vscale x 8 x i64> %a
 }
 
-declare <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.i64(
+declare <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64(
   <vscale x 8 x i64>,
   <vscale x 8 x i64>,
   i64,
   <vscale x 8 x i1>,
   i64);
 
-define <vscale x 8 x i64> @intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
-; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64_i64:
+define <vscale x 8 x i64> @intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
+; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a1, a1, e64,m8,tu,mu
 ; CHECK-NEXT:    vssrl.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 %2,
@@ -1948,9 +1948,9 @@ define <vscale x 1 x i8> @intrinsic_vssrl_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x i8>
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8(
     <vscale x 1 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i8> %a
@@ -1963,10 +1963,10 @@ define <vscale x 1 x i8> @intrinsic_vssrl_mask_vi_nxv1i8_nxv1i8_i8(<vscale x 1 x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.i8(
+  %a = call <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8(
     <vscale x 1 x i8> %0,
     <vscale x 1 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -1980,9 +1980,9 @@ define <vscale x 2 x i8> @intrinsic_vssrl_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x i8>
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8(
     <vscale x 2 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i8> %a
@@ -1995,10 +1995,10 @@ define <vscale x 2 x i8> @intrinsic_vssrl_mask_vi_nxv2i8_nxv2i8_i8(<vscale x 2 x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.i8(
+  %a = call <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8(
     <vscale x 2 x i8> %0,
     <vscale x 2 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2012,9 +2012,9 @@ define <vscale x 4 x i8> @intrinsic_vssrl_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x i8>
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8(
     <vscale x 4 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i8> %a
@@ -2027,10 +2027,10 @@ define <vscale x 4 x i8> @intrinsic_vssrl_mask_vi_nxv4i8_nxv4i8_i8(<vscale x 4 x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.i8(
+  %a = call <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8(
     <vscale x 4 x i8> %0,
     <vscale x 4 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2044,9 +2044,9 @@ define <vscale x 8 x i8> @intrinsic_vssrl_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x i8>
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8(
     <vscale x 8 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i8> %a
@@ -2059,10 +2059,10 @@ define <vscale x 8 x i8> @intrinsic_vssrl_mask_vi_nxv8i8_nxv8i8_i8(<vscale x 8 x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.i8(
+  %a = call <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8(
     <vscale x 8 x i8> %0,
     <vscale x 8 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2076,9 +2076,9 @@ define <vscale x 16 x i8> @intrinsic_vssrl_vi_nxv16i8_nxv16i8_i8(<vscale x 16 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8(
     <vscale x 16 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i8> %a
@@ -2091,10 +2091,10 @@ define <vscale x 16 x i8> @intrinsic_vssrl_mask_vi_nxv16i8_nxv16i8_i8(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.i8(
+  %a = call <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8(
     <vscale x 16 x i8> %0,
     <vscale x 16 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2108,9 +2108,9 @@ define <vscale x 32 x i8> @intrinsic_vssrl_vi_nxv32i8_nxv32i8_i8(<vscale x 32 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8(
     <vscale x 32 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i8> %a
@@ -2123,10 +2123,10 @@ define <vscale x 32 x i8> @intrinsic_vssrl_mask_vi_nxv32i8_nxv32i8_i8(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.i8(
+  %a = call <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8(
     <vscale x 32 x i8> %0,
     <vscale x 32 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2140,9 +2140,9 @@ define <vscale x 64 x i8> @intrinsic_vssrl_vi_nxv64i8_nxv64i8_i8(<vscale x 64 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8(
     <vscale x 64 x i8> %0,
-    i8 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 64 x i8> %a
@@ -2155,10 +2155,10 @@ define <vscale x 64 x i8> @intrinsic_vssrl_mask_vi_nxv64i8_nxv64i8_i8(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.i8(
+  %a = call <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8(
     <vscale x 64 x i8> %0,
     <vscale x 64 x i8> %1,
-    i8 9,
+    i64 9,
     <vscale x 64 x i1> %2,
     i64 %3)
 
@@ -2172,9 +2172,9 @@ define <vscale x 1 x i16> @intrinsic_vssrl_vi_nxv1i16_nxv1i16_i16(<vscale x 1 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16(
     <vscale x 1 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i16> %a
@@ -2187,10 +2187,10 @@ define <vscale x 1 x i16> @intrinsic_vssrl_mask_vi_nxv1i16_nxv1i16_i16(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.i16(
+  %a = call <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16(
     <vscale x 1 x i16> %0,
     <vscale x 1 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2204,9 +2204,9 @@ define <vscale x 2 x i16> @intrinsic_vssrl_vi_nxv2i16_nxv2i16_i16(<vscale x 2 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16(
     <vscale x 2 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i16> %a
@@ -2219,10 +2219,10 @@ define <vscale x 2 x i16> @intrinsic_vssrl_mask_vi_nxv2i16_nxv2i16_i16(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.i16(
+  %a = call <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16(
     <vscale x 2 x i16> %0,
     <vscale x 2 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2236,9 +2236,9 @@ define <vscale x 4 x i16> @intrinsic_vssrl_vi_nxv4i16_nxv4i16_i16(<vscale x 4 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16(
     <vscale x 4 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i16> %a
@@ -2251,10 +2251,10 @@ define <vscale x 4 x i16> @intrinsic_vssrl_mask_vi_nxv4i16_nxv4i16_i16(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.i16(
+  %a = call <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16(
     <vscale x 4 x i16> %0,
     <vscale x 4 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2268,9 +2268,9 @@ define <vscale x 8 x i16> @intrinsic_vssrl_vi_nxv8i16_nxv8i16_i16(<vscale x 8 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16(
     <vscale x 8 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i16> %a
@@ -2283,10 +2283,10 @@ define <vscale x 8 x i16> @intrinsic_vssrl_mask_vi_nxv8i16_nxv8i16_i16(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.i16(
+  %a = call <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16(
     <vscale x 8 x i16> %0,
     <vscale x 8 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2300,9 +2300,9 @@ define <vscale x 16 x i16> @intrinsic_vssrl_vi_nxv16i16_nxv16i16_i16(<vscale x 1
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16(
     <vscale x 16 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i16> %a
@@ -2315,10 +2315,10 @@ define <vscale x 16 x i16> @intrinsic_vssrl_mask_vi_nxv16i16_nxv16i16_i16(<vscal
 ; CHECK-NEXT:    vssrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.i16(
+  %a = call <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16(
     <vscale x 16 x i16> %0,
     <vscale x 16 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2332,9 +2332,9 @@ define <vscale x 32 x i16> @intrinsic_vssrl_vi_nxv32i16_nxv32i16_i16(<vscale x 3
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16(
     <vscale x 32 x i16> %0,
-    i16 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 32 x i16> %a
@@ -2347,10 +2347,10 @@ define <vscale x 32 x i16> @intrinsic_vssrl_mask_vi_nxv32i16_nxv32i16_i16(<vscal
 ; CHECK-NEXT:    vssrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.i16(
+  %a = call <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16(
     <vscale x 32 x i16> %0,
     <vscale x 32 x i16> %1,
-    i16 9,
+    i64 9,
     <vscale x 32 x i1> %2,
     i64 %3)
 
@@ -2364,9 +2364,9 @@ define <vscale x 1 x i32> @intrinsic_vssrl_vi_nxv1i32_nxv1i32_i32(<vscale x 1 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32(
     <vscale x 1 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 1 x i32> %a
@@ -2379,10 +2379,10 @@ define <vscale x 1 x i32> @intrinsic_vssrl_mask_vi_nxv1i32_nxv1i32_i32(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.i32(
+  %a = call <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32(
     <vscale x 1 x i32> %0,
     <vscale x 1 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 1 x i1> %2,
     i64 %3)
 
@@ -2396,9 +2396,9 @@ define <vscale x 2 x i32> @intrinsic_vssrl_vi_nxv2i32_nxv2i32_i32(<vscale x 2 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32(
     <vscale x 2 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 2 x i32> %a
@@ -2411,10 +2411,10 @@ define <vscale x 2 x i32> @intrinsic_vssrl_mask_vi_nxv2i32_nxv2i32_i32(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.i32(
+  %a = call <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32(
     <vscale x 2 x i32> %0,
     <vscale x 2 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 2 x i1> %2,
     i64 %3)
 
@@ -2428,9 +2428,9 @@ define <vscale x 4 x i32> @intrinsic_vssrl_vi_nxv4i32_nxv4i32_i32(<vscale x 4 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32(
     <vscale x 4 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 4 x i32> %a
@@ -2443,10 +2443,10 @@ define <vscale x 4 x i32> @intrinsic_vssrl_mask_vi_nxv4i32_nxv4i32_i32(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.i32(
+  %a = call <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32(
     <vscale x 4 x i32> %0,
     <vscale x 4 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 4 x i1> %2,
     i64 %3)
 
@@ -2460,9 +2460,9 @@ define <vscale x 8 x i32> @intrinsic_vssrl_vi_nxv8i32_nxv8i32_i32(<vscale x 8 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32(
     <vscale x 8 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 8 x i32> %a
@@ -2475,10 +2475,10 @@ define <vscale x 8 x i32> @intrinsic_vssrl_mask_vi_nxv8i32_nxv8i32_i32(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.i32(
+  %a = call <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32(
     <vscale x 8 x i32> %0,
     <vscale x 8 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 8 x i1> %2,
     i64 %3)
 
@@ -2492,9 +2492,9 @@ define <vscale x 16 x i32> @intrinsic_vssrl_vi_nxv16i32_nxv16i32_i32(<vscale x 1
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32(
     <vscale x 16 x i32> %0,
-    i32 9,
+    i64 9,
     i64 %1)
 
   ret <vscale x 16 x i32> %a
@@ -2507,10 +2507,10 @@ define <vscale x 16 x i32> @intrinsic_vssrl_mask_vi_nxv16i32_nxv16i32_i32(<vscal
 ; CHECK-NEXT:    vssrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.i32(
+  %a = call <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32(
     <vscale x 16 x i32> %0,
     <vscale x 16 x i32> %1,
-    i32 9,
+    i64 9,
     <vscale x 16 x i1> %2,
     i64 %3)
 
@@ -2524,7 +2524,7 @@ define <vscale x 1 x i64> @intrinsic_vssrl_vi_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64(
     <vscale x 1 x i64> %0,
     i64 9,
     i64 %1)
@@ -2539,7 +2539,7 @@ define <vscale x 1 x i64> @intrinsic_vssrl_mask_vi_nxv1i64_nxv1i64_i64(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v9, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.i64(
+  %a = call <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64(
     <vscale x 1 x i64> %0,
     <vscale x 1 x i64> %1,
     i64 9,
@@ -2556,7 +2556,7 @@ define <vscale x 2 x i64> @intrinsic_vssrl_vi_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64(
     <vscale x 2 x i64> %0,
     i64 9,
     i64 %1)
@@ -2571,7 +2571,7 @@ define <vscale x 2 x i64> @intrinsic_vssrl_mask_vi_nxv2i64_nxv2i64_i64(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v10, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.i64(
+  %a = call <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64(
     <vscale x 2 x i64> %0,
     <vscale x 2 x i64> %1,
     i64 9,
@@ -2588,7 +2588,7 @@ define <vscale x 4 x i64> @intrinsic_vssrl_vi_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64(
     <vscale x 4 x i64> %0,
     i64 9,
     i64 %1)
@@ -2603,7 +2603,7 @@ define <vscale x 4 x i64> @intrinsic_vssrl_mask_vi_nxv4i64_nxv4i64_i64(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v12, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.i64(
+  %a = call <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64(
     <vscale x 4 x i64> %0,
     <vscale x 4 x i64> %1,
     i64 9,
@@ -2620,7 +2620,7 @@ define <vscale x 8 x i64> @intrinsic_vssrl_vi_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ; CHECK-NEXT:    vssrl.vi v8, v8, 9
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64(
     <vscale x 8 x i64> %0,
     i64 9,
     i64 %1)
@@ -2635,7 +2635,7 @@ define <vscale x 8 x i64> @intrinsic_vssrl_mask_vi_nxv8i64_nxv8i64_i64(<vscale x
 ; CHECK-NEXT:    vssrl.vi v8, v16, 9, v0.t
 ; CHECK-NEXT:    jalr zero, 0(ra)
 entry:
-  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.i64(
+  %a = call <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64(
     <vscale x 8 x i64> %0,
     <vscale x 8 x i64> %1,
     i64 9,
diff --git a/llvm/test/CodeGen/RISCV/sadd_sat.ll b/llvm/test/CodeGen/RISCV/sadd_sat.ll
index d35e23c0a1cc69adce2e34747e7f429cbeb74b28..98b366939dab8ca25205f84391b5135daf665b0a 100644
--- a/llvm/test/CodeGen/RISCV/sadd_sat.ll
+++ b/llvm/test/CodeGen/RISCV/sadd_sat.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=riscv32 -mattr=+m | FileCheck %s --check-prefix=RV32I
 ; RUN: llc < %s -mtriple=riscv64 -mattr=+m | FileCheck %s --check-prefix=RV64I
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefix=RV32IZbb
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefix=RV64IZbb
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefixes=RV32IZbb,RV32IZbbNOZbt
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefixes=RV64IZbb,RV64IZbbNOZbt
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb,+experimental-zbt | FileCheck %s --check-prefixes=RV32IZbb,RV32IZbbZbt
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb,+experimental-zbt | FileCheck %s --check-prefixes=RV64IZbb,RV64IZbbZbt
 
 declare i4 @llvm.sadd.sat.i4(i4, i4)
 declare i8 @llvm.sadd.sat.i8(i8, i8)
@@ -45,22 +47,22 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 ; RV64I-NEXT:    lui a0, 524288
 ; RV64I-NEXT:    ret
 ;
-; RV32IZbb-LABEL: func:
-; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a2, a0
-; RV32IZbb-NEXT:    add a3, a0, a1
-; RV32IZbb-NEXT:    lui a0, 524288
-; RV32IZbb-NEXT:    bgez a3, .LBB0_2
-; RV32IZbb-NEXT:  # %bb.1:
-; RV32IZbb-NEXT:    addi a0, a0, -1
-; RV32IZbb-NEXT:  .LBB0_2:
-; RV32IZbb-NEXT:    slt a2, a3, a2
-; RV32IZbb-NEXT:    slti a1, a1, 0
-; RV32IZbb-NEXT:    bne a1, a2, .LBB0_4
-; RV32IZbb-NEXT:  # %bb.3:
-; RV32IZbb-NEXT:    mv a0, a3
-; RV32IZbb-NEXT:  .LBB0_4:
-; RV32IZbb-NEXT:    ret
+; RV32IZbbNOZbt-LABEL: func:
+; RV32IZbbNOZbt:       # %bb.0:
+; RV32IZbbNOZbt-NEXT:    mv a2, a0
+; RV32IZbbNOZbt-NEXT:    add a3, a0, a1
+; RV32IZbbNOZbt-NEXT:    lui a0, 524288
+; RV32IZbbNOZbt-NEXT:    bgez a3, .LBB0_2
+; RV32IZbbNOZbt-NEXT:  # %bb.1:
+; RV32IZbbNOZbt-NEXT:    addi a0, a0, -1
+; RV32IZbbNOZbt-NEXT:  .LBB0_2:
+; RV32IZbbNOZbt-NEXT:    slt a2, a3, a2
+; RV32IZbbNOZbt-NEXT:    slti a1, a1, 0
+; RV32IZbbNOZbt-NEXT:    bne a1, a2, .LBB0_4
+; RV32IZbbNOZbt-NEXT:  # %bb.3:
+; RV32IZbbNOZbt-NEXT:    mv a0, a3
+; RV32IZbbNOZbt-NEXT:  .LBB0_4:
+; RV32IZbbNOZbt-NEXT:    ret
 ;
 ; RV64IZbb-LABEL: func:
 ; RV64IZbb:       # %bb.0:
@@ -70,6 +72,19 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 ; RV64IZbb-NEXT:    min a0, a0, a2
 ; RV64IZbb-NEXT:    max a0, a0, a1
 ; RV64IZbb-NEXT:    ret
+;
+; RV32IZbbZbt-LABEL: func:
+; RV32IZbbZbt:       # %bb.0:
+; RV32IZbbZbt-NEXT:    add a2, a0, a1
+; RV32IZbbZbt-NEXT:    slti a3, a2, 0
+; RV32IZbbZbt-NEXT:    lui a4, 524288
+; RV32IZbbZbt-NEXT:    addi a5, a4, -1
+; RV32IZbbZbt-NEXT:    cmov a3, a3, a5, a4
+; RV32IZbbZbt-NEXT:    slt a0, a2, a0
+; RV32IZbbZbt-NEXT:    slti a1, a1, 0
+; RV32IZbbZbt-NEXT:    xor a0, a1, a0
+; RV32IZbbZbt-NEXT:    cmov a0, a0, a3, a2
+; RV32IZbbZbt-NEXT:    ret
   %tmp = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %y);
   ret i32 %tmp;
 }
@@ -118,47 +133,80 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; RV64I-NEXT:  .LBB1_4:
 ; RV64I-NEXT:    ret
 ;
-; RV32IZbb-LABEL: func2:
-; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a4, a1
-; RV32IZbb-NEXT:    mv a1, a0
-; RV32IZbb-NEXT:    add a5, a4, a3
-; RV32IZbb-NEXT:    add a0, a0, a2
-; RV32IZbb-NEXT:    sltu a1, a0, a1
-; RV32IZbb-NEXT:    add a2, a5, a1
-; RV32IZbb-NEXT:    lui a1, 524288
-; RV32IZbb-NEXT:    bgez a2, .LBB1_2
-; RV32IZbb-NEXT:  # %bb.1:
-; RV32IZbb-NEXT:    addi a1, a1, -1
-; RV32IZbb-NEXT:  .LBB1_2:
-; RV32IZbb-NEXT:    xor a5, a4, a2
-; RV32IZbb-NEXT:    xor a3, a4, a3
-; RV32IZbb-NEXT:    andn a3, a5, a3
-; RV32IZbb-NEXT:    bltz a3, .LBB1_4
-; RV32IZbb-NEXT:  # %bb.3:
-; RV32IZbb-NEXT:    mv a1, a2
-; RV32IZbb-NEXT:    ret
-; RV32IZbb-NEXT:  .LBB1_4:
-; RV32IZbb-NEXT:    srai a0, a2, 31
-; RV32IZbb-NEXT:    ret
+; RV32IZbbNOZbt-LABEL: func2:
+; RV32IZbbNOZbt:       # %bb.0:
+; RV32IZbbNOZbt-NEXT:    mv a4, a1
+; RV32IZbbNOZbt-NEXT:    mv a1, a0
+; RV32IZbbNOZbt-NEXT:    add a5, a4, a3
+; RV32IZbbNOZbt-NEXT:    add a0, a0, a2
+; RV32IZbbNOZbt-NEXT:    sltu a1, a0, a1
+; RV32IZbbNOZbt-NEXT:    add a2, a5, a1
+; RV32IZbbNOZbt-NEXT:    lui a1, 524288
+; RV32IZbbNOZbt-NEXT:    bgez a2, .LBB1_2
+; RV32IZbbNOZbt-NEXT:  # %bb.1:
+; RV32IZbbNOZbt-NEXT:    addi a1, a1, -1
+; RV32IZbbNOZbt-NEXT:  .LBB1_2:
+; RV32IZbbNOZbt-NEXT:    xor a5, a4, a2
+; RV32IZbbNOZbt-NEXT:    xor a3, a4, a3
+; RV32IZbbNOZbt-NEXT:    andn a3, a5, a3
+; RV32IZbbNOZbt-NEXT:    bltz a3, .LBB1_4
+; RV32IZbbNOZbt-NEXT:  # %bb.3:
+; RV32IZbbNOZbt-NEXT:    mv a1, a2
+; RV32IZbbNOZbt-NEXT:    ret
+; RV32IZbbNOZbt-NEXT:  .LBB1_4:
+; RV32IZbbNOZbt-NEXT:    srai a0, a2, 31
+; RV32IZbbNOZbt-NEXT:    ret
 ;
-; RV64IZbb-LABEL: func2:
-; RV64IZbb:       # %bb.0:
-; RV64IZbb-NEXT:    mv a2, a0
-; RV64IZbb-NEXT:    add a3, a0, a1
-; RV64IZbb-NEXT:    addi a0, zero, -1
-; RV64IZbb-NEXT:    slli a0, a0, 63
-; RV64IZbb-NEXT:    bgez a3, .LBB1_2
-; RV64IZbb-NEXT:  # %bb.1:
-; RV64IZbb-NEXT:    addi a0, a0, -1
-; RV64IZbb-NEXT:  .LBB1_2:
-; RV64IZbb-NEXT:    slt a2, a3, a2
-; RV64IZbb-NEXT:    slti a1, a1, 0
-; RV64IZbb-NEXT:    bne a1, a2, .LBB1_4
-; RV64IZbb-NEXT:  # %bb.3:
-; RV64IZbb-NEXT:    mv a0, a3
-; RV64IZbb-NEXT:  .LBB1_4:
-; RV64IZbb-NEXT:    ret
+; RV64IZbbNOZbt-LABEL: func2:
+; RV64IZbbNOZbt:       # %bb.0:
+; RV64IZbbNOZbt-NEXT:    mv a2, a0
+; RV64IZbbNOZbt-NEXT:    add a3, a0, a1
+; RV64IZbbNOZbt-NEXT:    addi a0, zero, -1
+; RV64IZbbNOZbt-NEXT:    slli a0, a0, 63
+; RV64IZbbNOZbt-NEXT:    bgez a3, .LBB1_2
+; RV64IZbbNOZbt-NEXT:  # %bb.1:
+; RV64IZbbNOZbt-NEXT:    addi a0, a0, -1
+; RV64IZbbNOZbt-NEXT:  .LBB1_2:
+; RV64IZbbNOZbt-NEXT:    slt a2, a3, a2
+; RV64IZbbNOZbt-NEXT:    slti a1, a1, 0
+; RV64IZbbNOZbt-NEXT:    bne a1, a2, .LBB1_4
+; RV64IZbbNOZbt-NEXT:  # %bb.3:
+; RV64IZbbNOZbt-NEXT:    mv a0, a3
+; RV64IZbbNOZbt-NEXT:  .LBB1_4:
+; RV64IZbbNOZbt-NEXT:    ret
+;
+; RV32IZbbZbt-LABEL: func2:
+; RV32IZbbZbt:       # %bb.0:
+; RV32IZbbZbt-NEXT:    add a4, a1, a3
+; RV32IZbbZbt-NEXT:    add a2, a0, a2
+; RV32IZbbZbt-NEXT:    sltu a0, a2, a0
+; RV32IZbbZbt-NEXT:    add a0, a4, a0
+; RV32IZbbZbt-NEXT:    slti a4, a0, 0
+; RV32IZbbZbt-NEXT:    lui a6, 524288
+; RV32IZbbZbt-NEXT:    addi a5, a6, -1
+; RV32IZbbZbt-NEXT:    cmov a4, a4, a5, a6
+; RV32IZbbZbt-NEXT:    xor a5, a1, a0
+; RV32IZbbZbt-NEXT:    xor a1, a1, a3
+; RV32IZbbZbt-NEXT:    andn a1, a5, a1
+; RV32IZbbZbt-NEXT:    slti a3, a1, 0
+; RV32IZbbZbt-NEXT:    cmov a1, a3, a4, a0
+; RV32IZbbZbt-NEXT:    srai a0, a0, 31
+; RV32IZbbZbt-NEXT:    cmov a0, a3, a0, a2
+; RV32IZbbZbt-NEXT:    ret
+;
+; RV64IZbbZbt-LABEL: func2:
+; RV64IZbbZbt:       # %bb.0:
+; RV64IZbbZbt-NEXT:    add a2, a0, a1
+; RV64IZbbZbt-NEXT:    slti a3, a2, 0
+; RV64IZbbZbt-NEXT:    addi a4, zero, -1
+; RV64IZbbZbt-NEXT:    slli a4, a4, 63
+; RV64IZbbZbt-NEXT:    addi a5, a4, -1
+; RV64IZbbZbt-NEXT:    cmov a3, a3, a5, a4
+; RV64IZbbZbt-NEXT:    slt a0, a2, a0
+; RV64IZbbZbt-NEXT:    slti a1, a1, 0
+; RV64IZbbZbt-NEXT:    xor a0, a1, a0
+; RV64IZbbZbt-NEXT:    cmov a0, a0, a3, a2
+; RV64IZbbZbt-NEXT:    ret
   %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
diff --git a/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll b/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
index 1767701e6eaf264fdd283bc49dce541026a4fd6d..69240024025b372894a157a1b88a26f1bfba9c51 100644
--- a/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
+++ b/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=riscv32 -mattr=+m | FileCheck %s --check-prefix=RV32I
 ; RUN: llc < %s -mtriple=riscv64 -mattr=+m | FileCheck %s --check-prefix=RV64I
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefix=RV32IZbb
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefix=RV64IZbb
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefixes=RV32IZbb,RV32IZbbNOZbt
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefixes=RV64IZbb,RV64IZbbNOZbt
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb,+experimental-zbt | FileCheck %s --check-prefixes=RV32IZbb,RV32IZbbZbt
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb,+experimental-zbt | FileCheck %s --check-prefixes=RV64IZbb,RV64IZbbZbt
 
 declare i4 @llvm.sadd.sat.i4(i4, i4)
 declare i8 @llvm.sadd.sat.i8(i8, i8)
@@ -48,23 +50,23 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ; RV64I-NEXT:    lui a0, 524288
 ; RV64I-NEXT:    ret
 ;
-; RV32IZbb-LABEL: func32:
-; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a3, a0
-; RV32IZbb-NEXT:    mul a2, a1, a2
-; RV32IZbb-NEXT:    add a1, a0, a2
-; RV32IZbb-NEXT:    lui a0, 524288
-; RV32IZbb-NEXT:    bgez a1, .LBB0_2
-; RV32IZbb-NEXT:  # %bb.1:
-; RV32IZbb-NEXT:    addi a0, a0, -1
-; RV32IZbb-NEXT:  .LBB0_2:
-; RV32IZbb-NEXT:    slt a3, a1, a3
-; RV32IZbb-NEXT:    slti a2, a2, 0
-; RV32IZbb-NEXT:    bne a2, a3, .LBB0_4
-; RV32IZbb-NEXT:  # %bb.3:
-; RV32IZbb-NEXT:    mv a0, a1
-; RV32IZbb-NEXT:  .LBB0_4:
-; RV32IZbb-NEXT:    ret
+; RV32IZbbNOZbt-LABEL: func32:
+; RV32IZbbNOZbt:       # %bb.0:
+; RV32IZbbNOZbt-NEXT:    mv a3, a0
+; RV32IZbbNOZbt-NEXT:    mul a2, a1, a2
+; RV32IZbbNOZbt-NEXT:    add a1, a0, a2
+; RV32IZbbNOZbt-NEXT:    lui a0, 524288
+; RV32IZbbNOZbt-NEXT:    bgez a1, .LBB0_2
+; RV32IZbbNOZbt-NEXT:  # %bb.1:
+; RV32IZbbNOZbt-NEXT:    addi a0, a0, -1
+; RV32IZbbNOZbt-NEXT:  .LBB0_2:
+; RV32IZbbNOZbt-NEXT:    slt a3, a1, a3
+; RV32IZbbNOZbt-NEXT:    slti a2, a2, 0
+; RV32IZbbNOZbt-NEXT:    bne a2, a3, .LBB0_4
+; RV32IZbbNOZbt-NEXT:  # %bb.3:
+; RV32IZbbNOZbt-NEXT:    mv a0, a1
+; RV32IZbbNOZbt-NEXT:  .LBB0_4:
+; RV32IZbbNOZbt-NEXT:    ret
 ;
 ; RV64IZbb-LABEL: func32:
 ; RV64IZbb:       # %bb.0:
@@ -76,6 +78,20 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ; RV64IZbb-NEXT:    min a0, a0, a2
 ; RV64IZbb-NEXT:    max a0, a0, a1
 ; RV64IZbb-NEXT:    ret
+;
+; RV32IZbbZbt-LABEL: func32:
+; RV32IZbbZbt:       # %bb.0:
+; RV32IZbbZbt-NEXT:    mul a1, a1, a2
+; RV32IZbbZbt-NEXT:    add a2, a0, a1
+; RV32IZbbZbt-NEXT:    slt a0, a2, a0
+; RV32IZbbZbt-NEXT:    slti a1, a1, 0
+; RV32IZbbZbt-NEXT:    xor a0, a1, a0
+; RV32IZbbZbt-NEXT:    slti a1, a2, 0
+; RV32IZbbZbt-NEXT:    lui a3, 524288
+; RV32IZbbZbt-NEXT:    addi a4, a3, -1
+; RV32IZbbZbt-NEXT:    cmov a1, a1, a4, a3
+; RV32IZbbZbt-NEXT:    cmov a0, a0, a1, a2
+; RV32IZbbZbt-NEXT:    ret
   %a = mul i32 %y, %z
   %tmp = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %a)
   ret i32 %tmp
@@ -125,47 +141,80 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; RV64I-NEXT:  .LBB1_4:
 ; RV64I-NEXT:    ret
 ;
-; RV32IZbb-LABEL: func64:
-; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a2, a1
-; RV32IZbb-NEXT:    mv a1, a0
-; RV32IZbb-NEXT:    add a3, a2, a5
-; RV32IZbb-NEXT:    add a0, a0, a4
-; RV32IZbb-NEXT:    sltu a1, a0, a1
-; RV32IZbb-NEXT:    add a3, a3, a1
-; RV32IZbb-NEXT:    lui a1, 524288
-; RV32IZbb-NEXT:    bgez a3, .LBB1_2
-; RV32IZbb-NEXT:  # %bb.1:
-; RV32IZbb-NEXT:    addi a1, a1, -1
-; RV32IZbb-NEXT:  .LBB1_2:
-; RV32IZbb-NEXT:    xor a4, a2, a3
-; RV32IZbb-NEXT:    xor a2, a2, a5
-; RV32IZbb-NEXT:    andn a2, a4, a2
-; RV32IZbb-NEXT:    bltz a2, .LBB1_4
-; RV32IZbb-NEXT:  # %bb.3:
-; RV32IZbb-NEXT:    mv a1, a3
-; RV32IZbb-NEXT:    ret
-; RV32IZbb-NEXT:  .LBB1_4:
-; RV32IZbb-NEXT:    srai a0, a3, 31
-; RV32IZbb-NEXT:    ret
+; RV32IZbbNOZbt-LABEL: func64:
+; RV32IZbbNOZbt:       # %bb.0:
+; RV32IZbbNOZbt-NEXT:    mv a2, a1
+; RV32IZbbNOZbt-NEXT:    mv a1, a0
+; RV32IZbbNOZbt-NEXT:    add a3, a2, a5
+; RV32IZbbNOZbt-NEXT:    add a0, a0, a4
+; RV32IZbbNOZbt-NEXT:    sltu a1, a0, a1
+; RV32IZbbNOZbt-NEXT:    add a3, a3, a1
+; RV32IZbbNOZbt-NEXT:    lui a1, 524288
+; RV32IZbbNOZbt-NEXT:    bgez a3, .LBB1_2
+; RV32IZbbNOZbt-NEXT:  # %bb.1:
+; RV32IZbbNOZbt-NEXT:    addi a1, a1, -1
+; RV32IZbbNOZbt-NEXT:  .LBB1_2:
+; RV32IZbbNOZbt-NEXT:    xor a4, a2, a3
+; RV32IZbbNOZbt-NEXT:    xor a2, a2, a5
+; RV32IZbbNOZbt-NEXT:    andn a2, a4, a2
+; RV32IZbbNOZbt-NEXT:    bltz a2, .LBB1_4
+; RV32IZbbNOZbt-NEXT:  # %bb.3:
+; RV32IZbbNOZbt-NEXT:    mv a1, a3
+; RV32IZbbNOZbt-NEXT:    ret
+; RV32IZbbNOZbt-NEXT:  .LBB1_4:
+; RV32IZbbNOZbt-NEXT:    srai a0, a3, 31
+; RV32IZbbNOZbt-NEXT:    ret
 ;
-; RV64IZbb-LABEL: func64:
-; RV64IZbb:       # %bb.0:
-; RV64IZbb-NEXT:    mv a1, a0
-; RV64IZbb-NEXT:    add a3, a0, a2
-; RV64IZbb-NEXT:    addi a0, zero, -1
-; RV64IZbb-NEXT:    slli a0, a0, 63
-; RV64IZbb-NEXT:    bgez a3, .LBB1_2
-; RV64IZbb-NEXT:  # %bb.1:
-; RV64IZbb-NEXT:    addi a0, a0, -1
-; RV64IZbb-NEXT:  .LBB1_2:
-; RV64IZbb-NEXT:    slt a1, a3, a1
-; RV64IZbb-NEXT:    slti a2, a2, 0
-; RV64IZbb-NEXT:    bne a2, a1, .LBB1_4
-; RV64IZbb-NEXT:  # %bb.3:
-; RV64IZbb-NEXT:    mv a0, a3
-; RV64IZbb-NEXT:  .LBB1_4:
-; RV64IZbb-NEXT:    ret
+; RV64IZbbNOZbt-LABEL: func64:
+; RV64IZbbNOZbt:       # %bb.0:
+; RV64IZbbNOZbt-NEXT:    mv a1, a0
+; RV64IZbbNOZbt-NEXT:    add a3, a0, a2
+; RV64IZbbNOZbt-NEXT:    addi a0, zero, -1
+; RV64IZbbNOZbt-NEXT:    slli a0, a0, 63
+; RV64IZbbNOZbt-NEXT:    bgez a3, .LBB1_2
+; RV64IZbbNOZbt-NEXT:  # %bb.1:
+; RV64IZbbNOZbt-NEXT:    addi a0, a0, -1
+; RV64IZbbNOZbt-NEXT:  .LBB1_2:
+; RV64IZbbNOZbt-NEXT:    slt a1, a3, a1
+; RV64IZbbNOZbt-NEXT:    slti a2, a2, 0
+; RV64IZbbNOZbt-NEXT:    bne a2, a1, .LBB1_4
+; RV64IZbbNOZbt-NEXT:  # %bb.3:
+; RV64IZbbNOZbt-NEXT:    mv a0, a3
+; RV64IZbbNOZbt-NEXT:  .LBB1_4:
+; RV64IZbbNOZbt-NEXT:    ret
+;
+; RV32IZbbZbt-LABEL: func64:
+; RV32IZbbZbt:       # %bb.0:
+; RV32IZbbZbt-NEXT:    add a2, a1, a5
+; RV32IZbbZbt-NEXT:    add a3, a0, a4
+; RV32IZbbZbt-NEXT:    sltu a0, a3, a0
+; RV32IZbbZbt-NEXT:    add a0, a2, a0
+; RV32IZbbZbt-NEXT:    slti a2, a0, 0
+; RV32IZbbZbt-NEXT:    lui a6, 524288
+; RV32IZbbZbt-NEXT:    addi a4, a6, -1
+; RV32IZbbZbt-NEXT:    cmov a2, a2, a4, a6
+; RV32IZbbZbt-NEXT:    xor a4, a1, a0
+; RV32IZbbZbt-NEXT:    xor a1, a1, a5
+; RV32IZbbZbt-NEXT:    andn a1, a4, a1
+; RV32IZbbZbt-NEXT:    slti a4, a1, 0
+; RV32IZbbZbt-NEXT:    cmov a1, a4, a2, a0
+; RV32IZbbZbt-NEXT:    srai a0, a0, 31
+; RV32IZbbZbt-NEXT:    cmov a0, a4, a0, a3
+; RV32IZbbZbt-NEXT:    ret
+;
+; RV64IZbbZbt-LABEL: func64:
+; RV64IZbbZbt:       # %bb.0:
+; RV64IZbbZbt-NEXT:    add a1, a0, a2
+; RV64IZbbZbt-NEXT:    slti a3, a1, 0
+; RV64IZbbZbt-NEXT:    addi a4, zero, -1
+; RV64IZbbZbt-NEXT:    slli a4, a4, 63
+; RV64IZbbZbt-NEXT:    addi a5, a4, -1
+; RV64IZbbZbt-NEXT:    cmov a3, a3, a5, a4
+; RV64IZbbZbt-NEXT:    slt a0, a1, a0
+; RV64IZbbZbt-NEXT:    slti a2, a2, 0
+; RV64IZbbZbt-NEXT:    xor a0, a2, a0
+; RV64IZbbZbt-NEXT:    cmov a0, a0, a3, a1
+; RV64IZbbZbt-NEXT:    ret
   %a = mul i64 %y, %z
   %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %z)
   ret i64 %tmp
diff --git a/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll b/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
index 8b03ea25e39b1803a2d05c1063181a24c3c1cd87..fe386e65434cde02447792b459e9f26404e044bc 100644
--- a/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
+++ b/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
@@ -699,14 +699,16 @@ define void @test_srem_vec(<3 x i33>* %X) nounwind {
 ; RV32MV-NEXT:    call __moddi3@plt
 ; RV32MV-NEXT:    sw a1, 12(sp)
 ; RV32MV-NEXT:    sw a0, 8(sp)
+; RV32MV-NEXT:    addi a0, zero, 85
+; RV32MV-NEXT:    vsetivli a1, 1, e8,m1,ta,mu
+; RV32MV-NEXT:    vmv.s.x v0, a0
+; RV32MV-NEXT:    vsetivli a0, 8, e32,m2,ta,mu
+; RV32MV-NEXT:    vmv.v.i v26, 1
+; RV32MV-NEXT:    vle32.v v28, (sp)
 ; RV32MV-NEXT:    lui a0, %hi(.LCPI3_0)
 ; RV32MV-NEXT:    addi a0, a0, %lo(.LCPI3_0)
-; RV32MV-NEXT:    vsetivli a1, 8, e32,m2,ta,mu
-; RV32MV-NEXT:    vle32.v v26, (a0)
-; RV32MV-NEXT:    vle32.v v28, (sp)
-; RV32MV-NEXT:    lui a0, %hi(.LCPI3_1)
-; RV32MV-NEXT:    addi a0, a0, %lo(.LCPI3_1)
 ; RV32MV-NEXT:    vle32.v v30, (a0)
+; RV32MV-NEXT:    vmerge.vim v26, v26, -1, v0
 ; RV32MV-NEXT:    vand.vv v26, v28, v26
 ; RV32MV-NEXT:    vsetivli a0, 4, e64,m2,ta,mu
 ; RV32MV-NEXT:    vmsne.vv v0, v26, v30
diff --git a/llvm/test/CodeGen/RISCV/ssub_sat.ll b/llvm/test/CodeGen/RISCV/ssub_sat.ll
index 61d5a4833ba911c0ea26151501488bc7d91fc5ea..008a3d0cf8add6694521ab9731afd29b89562f7f 100644
--- a/llvm/test/CodeGen/RISCV/ssub_sat.ll
+++ b/llvm/test/CodeGen/RISCV/ssub_sat.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=riscv32 -mattr=+m | FileCheck %s --check-prefix=RV32I
 ; RUN: llc < %s -mtriple=riscv64 -mattr=+m | FileCheck %s --check-prefix=RV64I
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefix=RV32IZbb
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefix=RV64IZbb
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefixes=RV32IZbb,RV32IZbbNOZbt
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefixes=RV64IZbb,RV64IZbbNOZbt
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb,+experimental-zbt | FileCheck %s --check-prefixes=RV32IZbb,RV32IZbbZbt
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb,+experimental-zbt | FileCheck %s --check-prefixes=RV64IZbb,RV64IZbbZbt
 
 declare i4 @llvm.ssub.sat.i4(i4, i4)
 declare i8 @llvm.ssub.sat.i8(i8, i8)
@@ -45,22 +47,22 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 ; RV64I-NEXT:    lui a0, 524288
 ; RV64I-NEXT:    ret
 ;
-; RV32IZbb-LABEL: func:
-; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a2, a0
-; RV32IZbb-NEXT:    sub a3, a0, a1
-; RV32IZbb-NEXT:    lui a0, 524288
-; RV32IZbb-NEXT:    bgez a3, .LBB0_2
-; RV32IZbb-NEXT:  # %bb.1:
-; RV32IZbb-NEXT:    addi a0, a0, -1
-; RV32IZbb-NEXT:  .LBB0_2:
-; RV32IZbb-NEXT:    sgtz a1, a1
-; RV32IZbb-NEXT:    slt a2, a3, a2
-; RV32IZbb-NEXT:    bne a1, a2, .LBB0_4
-; RV32IZbb-NEXT:  # %bb.3:
-; RV32IZbb-NEXT:    mv a0, a3
-; RV32IZbb-NEXT:  .LBB0_4:
-; RV32IZbb-NEXT:    ret
+; RV32IZbbNOZbt-LABEL: func:
+; RV32IZbbNOZbt:       # %bb.0:
+; RV32IZbbNOZbt-NEXT:    mv a2, a0
+; RV32IZbbNOZbt-NEXT:    sub a3, a0, a1
+; RV32IZbbNOZbt-NEXT:    lui a0, 524288
+; RV32IZbbNOZbt-NEXT:    bgez a3, .LBB0_2
+; RV32IZbbNOZbt-NEXT:  # %bb.1:
+; RV32IZbbNOZbt-NEXT:    addi a0, a0, -1
+; RV32IZbbNOZbt-NEXT:  .LBB0_2:
+; RV32IZbbNOZbt-NEXT:    sgtz a1, a1
+; RV32IZbbNOZbt-NEXT:    slt a2, a3, a2
+; RV32IZbbNOZbt-NEXT:    bne a1, a2, .LBB0_4
+; RV32IZbbNOZbt-NEXT:  # %bb.3:
+; RV32IZbbNOZbt-NEXT:    mv a0, a3
+; RV32IZbbNOZbt-NEXT:  .LBB0_4:
+; RV32IZbbNOZbt-NEXT:    ret
 ;
 ; RV64IZbb-LABEL: func:
 ; RV64IZbb:       # %bb.0:
@@ -70,6 +72,19 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 ; RV64IZbb-NEXT:    min a0, a0, a2
 ; RV64IZbb-NEXT:    max a0, a0, a1
 ; RV64IZbb-NEXT:    ret
+;
+; RV32IZbbZbt-LABEL: func:
+; RV32IZbbZbt:       # %bb.0:
+; RV32IZbbZbt-NEXT:    sgtz a2, a1
+; RV32IZbbZbt-NEXT:    sub a1, a0, a1
+; RV32IZbbZbt-NEXT:    slt a0, a1, a0
+; RV32IZbbZbt-NEXT:    xor a0, a2, a0
+; RV32IZbbZbt-NEXT:    slti a2, a1, 0
+; RV32IZbbZbt-NEXT:    lui a3, 524288
+; RV32IZbbZbt-NEXT:    addi a4, a3, -1
+; RV32IZbbZbt-NEXT:    cmov a2, a2, a4, a3
+; RV32IZbbZbt-NEXT:    cmov a0, a0, a2, a1
+; RV32IZbbZbt-NEXT:    ret
   %tmp = call i32 @llvm.ssub.sat.i32(i32 %x, i32 %y);
   ret i32 %tmp;
 }
@@ -116,46 +131,79 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; RV64I-NEXT:  .LBB1_4:
 ; RV64I-NEXT:    ret
 ;
-; RV32IZbb-LABEL: func2:
-; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a4, a1
-; RV32IZbb-NEXT:    sltu a1, a0, a2
-; RV32IZbb-NEXT:    sub a5, a4, a3
-; RV32IZbb-NEXT:    sub a5, a5, a1
-; RV32IZbb-NEXT:    lui a1, 524288
-; RV32IZbb-NEXT:    bgez a5, .LBB1_2
-; RV32IZbb-NEXT:  # %bb.1:
-; RV32IZbb-NEXT:    addi a1, a1, -1
-; RV32IZbb-NEXT:  .LBB1_2:
-; RV32IZbb-NEXT:    xor a6, a4, a5
-; RV32IZbb-NEXT:    xor a3, a4, a3
-; RV32IZbb-NEXT:    and a3, a3, a6
-; RV32IZbb-NEXT:    bltz a3, .LBB1_4
-; RV32IZbb-NEXT:  # %bb.3:
-; RV32IZbb-NEXT:    sub a0, a0, a2
-; RV32IZbb-NEXT:    mv a1, a5
-; RV32IZbb-NEXT:    ret
-; RV32IZbb-NEXT:  .LBB1_4:
-; RV32IZbb-NEXT:    srai a0, a5, 31
-; RV32IZbb-NEXT:    ret
+; RV32IZbbNOZbt-LABEL: func2:
+; RV32IZbbNOZbt:       # %bb.0:
+; RV32IZbbNOZbt-NEXT:    mv a4, a1
+; RV32IZbbNOZbt-NEXT:    sltu a1, a0, a2
+; RV32IZbbNOZbt-NEXT:    sub a5, a4, a3
+; RV32IZbbNOZbt-NEXT:    sub a5, a5, a1
+; RV32IZbbNOZbt-NEXT:    lui a1, 524288
+; RV32IZbbNOZbt-NEXT:    bgez a5, .LBB1_2
+; RV32IZbbNOZbt-NEXT:  # %bb.1:
+; RV32IZbbNOZbt-NEXT:    addi a1, a1, -1
+; RV32IZbbNOZbt-NEXT:  .LBB1_2:
+; RV32IZbbNOZbt-NEXT:    xor a6, a4, a5
+; RV32IZbbNOZbt-NEXT:    xor a3, a4, a3
+; RV32IZbbNOZbt-NEXT:    and a3, a3, a6
+; RV32IZbbNOZbt-NEXT:    bltz a3, .LBB1_4
+; RV32IZbbNOZbt-NEXT:  # %bb.3:
+; RV32IZbbNOZbt-NEXT:    sub a0, a0, a2
+; RV32IZbbNOZbt-NEXT:    mv a1, a5
+; RV32IZbbNOZbt-NEXT:    ret
+; RV32IZbbNOZbt-NEXT:  .LBB1_4:
+; RV32IZbbNOZbt-NEXT:    srai a0, a5, 31
+; RV32IZbbNOZbt-NEXT:    ret
 ;
-; RV64IZbb-LABEL: func2:
-; RV64IZbb:       # %bb.0:
-; RV64IZbb-NEXT:    mv a2, a0
-; RV64IZbb-NEXT:    sub a3, a0, a1
-; RV64IZbb-NEXT:    addi a0, zero, -1
-; RV64IZbb-NEXT:    slli a0, a0, 63
-; RV64IZbb-NEXT:    bgez a3, .LBB1_2
-; RV64IZbb-NEXT:  # %bb.1:
-; RV64IZbb-NEXT:    addi a0, a0, -1
-; RV64IZbb-NEXT:  .LBB1_2:
-; RV64IZbb-NEXT:    sgtz a1, a1
-; RV64IZbb-NEXT:    slt a2, a3, a2
-; RV64IZbb-NEXT:    bne a1, a2, .LBB1_4
-; RV64IZbb-NEXT:  # %bb.3:
-; RV64IZbb-NEXT:    mv a0, a3
-; RV64IZbb-NEXT:  .LBB1_4:
-; RV64IZbb-NEXT:    ret
+; RV64IZbbNOZbt-LABEL: func2:
+; RV64IZbbNOZbt:       # %bb.0:
+; RV64IZbbNOZbt-NEXT:    mv a2, a0
+; RV64IZbbNOZbt-NEXT:    sub a3, a0, a1
+; RV64IZbbNOZbt-NEXT:    addi a0, zero, -1
+; RV64IZbbNOZbt-NEXT:    slli a0, a0, 63
+; RV64IZbbNOZbt-NEXT:    bgez a3, .LBB1_2
+; RV64IZbbNOZbt-NEXT:  # %bb.1:
+; RV64IZbbNOZbt-NEXT:    addi a0, a0, -1
+; RV64IZbbNOZbt-NEXT:  .LBB1_2:
+; RV64IZbbNOZbt-NEXT:    sgtz a1, a1
+; RV64IZbbNOZbt-NEXT:    slt a2, a3, a2
+; RV64IZbbNOZbt-NEXT:    bne a1, a2, .LBB1_4
+; RV64IZbbNOZbt-NEXT:  # %bb.3:
+; RV64IZbbNOZbt-NEXT:    mv a0, a3
+; RV64IZbbNOZbt-NEXT:  .LBB1_4:
+; RV64IZbbNOZbt-NEXT:    ret
+;
+; RV32IZbbZbt-LABEL: func2:
+; RV32IZbbZbt:       # %bb.0:
+; RV32IZbbZbt-NEXT:    sltu a4, a0, a2
+; RV32IZbbZbt-NEXT:    sub a5, a1, a3
+; RV32IZbbZbt-NEXT:    sub a4, a5, a4
+; RV32IZbbZbt-NEXT:    slti a7, a4, 0
+; RV32IZbbZbt-NEXT:    lui a6, 524288
+; RV32IZbbZbt-NEXT:    addi a5, a6, -1
+; RV32IZbbZbt-NEXT:    cmov a6, a7, a5, a6
+; RV32IZbbZbt-NEXT:    xor a5, a1, a4
+; RV32IZbbZbt-NEXT:    xor a1, a1, a3
+; RV32IZbbZbt-NEXT:    and a1, a1, a5
+; RV32IZbbZbt-NEXT:    slti a3, a1, 0
+; RV32IZbbZbt-NEXT:    cmov a1, a3, a6, a4
+; RV32IZbbZbt-NEXT:    srai a4, a4, 31
+; RV32IZbbZbt-NEXT:    sub a0, a0, a2
+; RV32IZbbZbt-NEXT:    cmov a0, a3, a4, a0
+; RV32IZbbZbt-NEXT:    ret
+;
+; RV64IZbbZbt-LABEL: func2:
+; RV64IZbbZbt:       # %bb.0:
+; RV64IZbbZbt-NEXT:    sgtz a2, a1
+; RV64IZbbZbt-NEXT:    sub a1, a0, a1
+; RV64IZbbZbt-NEXT:    slt a0, a1, a0
+; RV64IZbbZbt-NEXT:    xor a0, a2, a0
+; RV64IZbbZbt-NEXT:    slti a2, a1, 0
+; RV64IZbbZbt-NEXT:    addi a3, zero, -1
+; RV64IZbbZbt-NEXT:    slli a3, a3, 63
+; RV64IZbbZbt-NEXT:    addi a4, a3, -1
+; RV64IZbbZbt-NEXT:    cmov a2, a2, a4, a3
+; RV64IZbbZbt-NEXT:    cmov a0, a0, a2, a1
+; RV64IZbbZbt-NEXT:    ret
   %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
diff --git a/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll b/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll
index 7f2e1595a98d845c0cbc74bafbd06f0900c79e4c..a30ad68631c47a356e64080e5b9f75c7f5419261 100644
--- a/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll
+++ b/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=riscv32 -mattr=+m | FileCheck %s --check-prefix=RV32I
 ; RUN: llc < %s -mtriple=riscv64 -mattr=+m | FileCheck %s --check-prefix=RV64I
-; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefix=RV32IZbb
-; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefix=RV64IZbb
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefixes=RV32IZbb,RV32IZbbNOZbt
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb | FileCheck %s --check-prefixes=RV64IZbb,RV64IZbbNOZbt
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zbb,+experimental-zbt | FileCheck %s --check-prefixes=RV32IZbb,RV32IZbbZbt
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zbb,+experimental-zbt | FileCheck %s --check-prefixes=RV64IZbb,RV64IZbbZbt
 
 declare i4 @llvm.ssub.sat.i4(i4, i4)
 declare i8 @llvm.ssub.sat.i8(i8, i8)
@@ -48,23 +50,23 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ; RV64I-NEXT:    lui a0, 524288
 ; RV64I-NEXT:    ret
 ;
-; RV32IZbb-LABEL: func32:
-; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a3, a0
-; RV32IZbb-NEXT:    mul a2, a1, a2
-; RV32IZbb-NEXT:    sub a1, a0, a2
-; RV32IZbb-NEXT:    lui a0, 524288
-; RV32IZbb-NEXT:    bgez a1, .LBB0_2
-; RV32IZbb-NEXT:  # %bb.1:
-; RV32IZbb-NEXT:    addi a0, a0, -1
-; RV32IZbb-NEXT:  .LBB0_2:
-; RV32IZbb-NEXT:    sgtz a2, a2
-; RV32IZbb-NEXT:    slt a3, a1, a3
-; RV32IZbb-NEXT:    bne a2, a3, .LBB0_4
-; RV32IZbb-NEXT:  # %bb.3:
-; RV32IZbb-NEXT:    mv a0, a1
-; RV32IZbb-NEXT:  .LBB0_4:
-; RV32IZbb-NEXT:    ret
+; RV32IZbbNOZbt-LABEL: func32:
+; RV32IZbbNOZbt:       # %bb.0:
+; RV32IZbbNOZbt-NEXT:    mv a3, a0
+; RV32IZbbNOZbt-NEXT:    mul a2, a1, a2
+; RV32IZbbNOZbt-NEXT:    sub a1, a0, a2
+; RV32IZbbNOZbt-NEXT:    lui a0, 524288
+; RV32IZbbNOZbt-NEXT:    bgez a1, .LBB0_2
+; RV32IZbbNOZbt-NEXT:  # %bb.1:
+; RV32IZbbNOZbt-NEXT:    addi a0, a0, -1
+; RV32IZbbNOZbt-NEXT:  .LBB0_2:
+; RV32IZbbNOZbt-NEXT:    sgtz a2, a2
+; RV32IZbbNOZbt-NEXT:    slt a3, a1, a3
+; RV32IZbbNOZbt-NEXT:    bne a2, a3, .LBB0_4
+; RV32IZbbNOZbt-NEXT:  # %bb.3:
+; RV32IZbbNOZbt-NEXT:    mv a0, a1
+; RV32IZbbNOZbt-NEXT:  .LBB0_4:
+; RV32IZbbNOZbt-NEXT:    ret
 ;
 ; RV64IZbb-LABEL: func32:
 ; RV64IZbb:       # %bb.0:
@@ -76,6 +78,20 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ; RV64IZbb-NEXT:    min a0, a0, a2
 ; RV64IZbb-NEXT:    max a0, a0, a1
 ; RV64IZbb-NEXT:    ret
+;
+; RV32IZbbZbt-LABEL: func32:
+; RV32IZbbZbt:       # %bb.0:
+; RV32IZbbZbt-NEXT:    mul a1, a1, a2
+; RV32IZbbZbt-NEXT:    sgtz a2, a1
+; RV32IZbbZbt-NEXT:    sub a1, a0, a1
+; RV32IZbbZbt-NEXT:    slt a0, a1, a0
+; RV32IZbbZbt-NEXT:    xor a0, a2, a0
+; RV32IZbbZbt-NEXT:    slti a2, a1, 0
+; RV32IZbbZbt-NEXT:    lui a3, 524288
+; RV32IZbbZbt-NEXT:    addi a4, a3, -1
+; RV32IZbbZbt-NEXT:    cmov a2, a2, a4, a3
+; RV32IZbbZbt-NEXT:    cmov a0, a0, a2, a1
+; RV32IZbbZbt-NEXT:    ret
   %a = mul i32 %y, %z
   %tmp = call i32 @llvm.ssub.sat.i32(i32 %x, i32 %a)
   ret i32 %tmp
@@ -123,46 +139,79 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; RV64I-NEXT:  .LBB1_4:
 ; RV64I-NEXT:    ret
 ;
-; RV32IZbb-LABEL: func64:
-; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a2, a1
-; RV32IZbb-NEXT:    sltu a1, a0, a4
-; RV32IZbb-NEXT:    sub a3, a2, a5
-; RV32IZbb-NEXT:    sub a3, a3, a1
-; RV32IZbb-NEXT:    lui a1, 524288
-; RV32IZbb-NEXT:    bgez a3, .LBB1_2
-; RV32IZbb-NEXT:  # %bb.1:
-; RV32IZbb-NEXT:    addi a1, a1, -1
-; RV32IZbb-NEXT:  .LBB1_2:
-; RV32IZbb-NEXT:    xor a6, a2, a3
-; RV32IZbb-NEXT:    xor a2, a2, a5
-; RV32IZbb-NEXT:    and a2, a2, a6
-; RV32IZbb-NEXT:    bltz a2, .LBB1_4
-; RV32IZbb-NEXT:  # %bb.3:
-; RV32IZbb-NEXT:    sub a0, a0, a4
-; RV32IZbb-NEXT:    mv a1, a3
-; RV32IZbb-NEXT:    ret
-; RV32IZbb-NEXT:  .LBB1_4:
-; RV32IZbb-NEXT:    srai a0, a3, 31
-; RV32IZbb-NEXT:    ret
+; RV32IZbbNOZbt-LABEL: func64:
+; RV32IZbbNOZbt:       # %bb.0:
+; RV32IZbbNOZbt-NEXT:    mv a2, a1
+; RV32IZbbNOZbt-NEXT:    sltu a1, a0, a4
+; RV32IZbbNOZbt-NEXT:    sub a3, a2, a5
+; RV32IZbbNOZbt-NEXT:    sub a3, a3, a1
+; RV32IZbbNOZbt-NEXT:    lui a1, 524288
+; RV32IZbbNOZbt-NEXT:    bgez a3, .LBB1_2
+; RV32IZbbNOZbt-NEXT:  # %bb.1:
+; RV32IZbbNOZbt-NEXT:    addi a1, a1, -1
+; RV32IZbbNOZbt-NEXT:  .LBB1_2:
+; RV32IZbbNOZbt-NEXT:    xor a6, a2, a3
+; RV32IZbbNOZbt-NEXT:    xor a2, a2, a5
+; RV32IZbbNOZbt-NEXT:    and a2, a2, a6
+; RV32IZbbNOZbt-NEXT:    bltz a2, .LBB1_4
+; RV32IZbbNOZbt-NEXT:  # %bb.3:
+; RV32IZbbNOZbt-NEXT:    sub a0, a0, a4
+; RV32IZbbNOZbt-NEXT:    mv a1, a3
+; RV32IZbbNOZbt-NEXT:    ret
+; RV32IZbbNOZbt-NEXT:  .LBB1_4:
+; RV32IZbbNOZbt-NEXT:    srai a0, a3, 31
+; RV32IZbbNOZbt-NEXT:    ret
 ;
-; RV64IZbb-LABEL: func64:
-; RV64IZbb:       # %bb.0:
-; RV64IZbb-NEXT:    mv a1, a0
-; RV64IZbb-NEXT:    sub a3, a0, a2
-; RV64IZbb-NEXT:    addi a0, zero, -1
-; RV64IZbb-NEXT:    slli a0, a0, 63
-; RV64IZbb-NEXT:    bgez a3, .LBB1_2
-; RV64IZbb-NEXT:  # %bb.1:
-; RV64IZbb-NEXT:    addi a0, a0, -1
-; RV64IZbb-NEXT:  .LBB1_2:
-; RV64IZbb-NEXT:    sgtz a2, a2
-; RV64IZbb-NEXT:    slt a1, a3, a1
-; RV64IZbb-NEXT:    bne a2, a1, .LBB1_4
-; RV64IZbb-NEXT:  # %bb.3:
-; RV64IZbb-NEXT:    mv a0, a3
-; RV64IZbb-NEXT:  .LBB1_4:
-; RV64IZbb-NEXT:    ret
+; RV64IZbbNOZbt-LABEL: func64:
+; RV64IZbbNOZbt:       # %bb.0:
+; RV64IZbbNOZbt-NEXT:    mv a1, a0
+; RV64IZbbNOZbt-NEXT:    sub a3, a0, a2
+; RV64IZbbNOZbt-NEXT:    addi a0, zero, -1
+; RV64IZbbNOZbt-NEXT:    slli a0, a0, 63
+; RV64IZbbNOZbt-NEXT:    bgez a3, .LBB1_2
+; RV64IZbbNOZbt-NEXT:  # %bb.1:
+; RV64IZbbNOZbt-NEXT:    addi a0, a0, -1
+; RV64IZbbNOZbt-NEXT:  .LBB1_2:
+; RV64IZbbNOZbt-NEXT:    sgtz a2, a2
+; RV64IZbbNOZbt-NEXT:    slt a1, a3, a1
+; RV64IZbbNOZbt-NEXT:    bne a2, a1, .LBB1_4
+; RV64IZbbNOZbt-NEXT:  # %bb.3:
+; RV64IZbbNOZbt-NEXT:    mv a0, a3
+; RV64IZbbNOZbt-NEXT:  .LBB1_4:
+; RV64IZbbNOZbt-NEXT:    ret
+;
+; RV32IZbbZbt-LABEL: func64:
+; RV32IZbbZbt:       # %bb.0:
+; RV32IZbbZbt-NEXT:    sltu a2, a0, a4
+; RV32IZbbZbt-NEXT:    sub a3, a1, a5
+; RV32IZbbZbt-NEXT:    sub a2, a3, a2
+; RV32IZbbZbt-NEXT:    slti a7, a2, 0
+; RV32IZbbZbt-NEXT:    lui a6, 524288
+; RV32IZbbZbt-NEXT:    addi a3, a6, -1
+; RV32IZbbZbt-NEXT:    cmov a6, a7, a3, a6
+; RV32IZbbZbt-NEXT:    xor a3, a1, a2
+; RV32IZbbZbt-NEXT:    xor a1, a1, a5
+; RV32IZbbZbt-NEXT:    and a1, a1, a3
+; RV32IZbbZbt-NEXT:    slti a3, a1, 0
+; RV32IZbbZbt-NEXT:    cmov a1, a3, a6, a2
+; RV32IZbbZbt-NEXT:    srai a2, a2, 31
+; RV32IZbbZbt-NEXT:    sub a0, a0, a4
+; RV32IZbbZbt-NEXT:    cmov a0, a3, a2, a0
+; RV32IZbbZbt-NEXT:    ret
+;
+; RV64IZbbZbt-LABEL: func64:
+; RV64IZbbZbt:       # %bb.0:
+; RV64IZbbZbt-NEXT:    sgtz a1, a2
+; RV64IZbbZbt-NEXT:    sub a2, a0, a2
+; RV64IZbbZbt-NEXT:    slt a0, a2, a0
+; RV64IZbbZbt-NEXT:    xor a0, a1, a0
+; RV64IZbbZbt-NEXT:    slti a1, a2, 0
+; RV64IZbbZbt-NEXT:    addi a3, zero, -1
+; RV64IZbbZbt-NEXT:    slli a3, a3, 63
+; RV64IZbbZbt-NEXT:    addi a4, a3, -1
+; RV64IZbbZbt-NEXT:    cmov a1, a1, a4, a3
+; RV64IZbbZbt-NEXT:    cmov a0, a0, a1, a2
+; RV64IZbbZbt-NEXT:    ret
   %a = mul i64 %y, %z
   %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %z)
   ret i64 %tmp
diff --git a/llvm/test/CodeGen/RISCV/stack-realignment.ll b/llvm/test/CodeGen/RISCV/stack-realignment.ll
index 6f72a2488c273a1d1327be64c59ced0c8e26e8d2..681ed762346c303b329922cca4cb671d7eb45e3e 100644
--- a/llvm/test/CodeGen/RISCV/stack-realignment.ll
+++ b/llvm/test/CodeGen/RISCV/stack-realignment.ll
@@ -460,7 +460,6 @@ define void @caller2048() {
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a0, a0, -2048
 ; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    mv a0, a0
 ; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    sub sp, s0, a0
@@ -489,7 +488,6 @@ define void @caller2048() {
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    addiw a0, a0, -2048
 ; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    mv a0, a0
 ; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    sub sp, s0, a0
@@ -552,7 +550,6 @@ define void @caller4096() {
 ; RV32I-NEXT:    slli sp, a0, 12
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    mv a0, a0
 ; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    lui a0, 2
 ; RV32I-NEXT:    sub sp, s0, a0
@@ -581,7 +578,6 @@ define void @caller4096() {
 ; RV64I-NEXT:    slli sp, a0, 12
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    mv a0, a0
 ; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    lui a0, 2
 ; RV64I-NEXT:    sub sp, s0, a0
diff --git a/llvm/test/CodeGen/RISCV/stack-slot-size.ll b/llvm/test/CodeGen/RISCV/stack-slot-size.ll
new file mode 100644
index 0000000000000000000000000000000000000000..14dd2b8cbcdb5bf020edf81ec56d773ad25c7111
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/stack-slot-size.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+; When passing a function argument with a size that isn't a multiple of XLEN,
+; and the argument is split and passed indirectly, we must ensure that the stack
+; slot size appropriately reflects the total size of the parts the argument is
+; split into. Otherwise, stack writes can clobber neighboring values.
+
+declare void @callee129(i129)
+declare void @callee160(i160)
+declare void @callee161(i161)
+
+; FIXME: Stack write clobbers the spilled value (on RV64).
+define i32 @caller129() nounwind {
+; RV32I-LABEL: caller129:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    addi a0, zero, 42
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw zero, 16(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    mv a0, sp
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    call callee129@plt
+; RV32I-NEXT:    lw a0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: caller129:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi a0, zero, 42
+; RV64I-NEXT:    sw a0, 20(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    mv a0, sp
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    call callee129@plt
+; RV64I-NEXT:    lw a0, 20(sp)
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+  %1 = alloca i32
+  store i32 42, i32* %1
+  call void @callee129(i129 0)
+  %2 = load i32, i32* %1
+  ret i32 %2
+}
+
+; FIXME: Stack write clobbers the spilled value (on RV64).
+define i32 @caller160() nounwind {
+; RV32I-LABEL: caller160:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    addi a0, zero, 42
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw zero, 16(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    mv a0, sp
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    call callee160@plt
+; RV32I-NEXT:    lw a0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: caller160:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi a0, zero, 42
+; RV64I-NEXT:    sw a0, 20(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    mv a0, sp
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    call callee160@plt
+; RV64I-NEXT:    lw a0, 20(sp)
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+  %1 = alloca i32
+  store i32 42, i32* %1
+  call void @callee160(i160 0)
+  %2 = load i32, i32* %1
+  ret i32 %2
+}
+
+define i32 @caller161() nounwind {
+; RV32I-LABEL: caller161:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    addi a0, zero, 42
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 16(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    mv a0, sp
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    call callee161@plt
+; RV32I-NEXT:    lw a0, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: caller161:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi a0, zero, 42
+; RV64I-NEXT:    sw a0, 36(sp)
+; RV64I-NEXT:    sd zero, 16(sp)
+; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    mv a0, sp
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    call callee161@plt
+; RV64I-NEXT:    lw a0, 36(sp)
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+  %1 = alloca i32
+  store i32 42, i32* %1
+  call void @callee161(i161 0)
+  %2 = load i32, i32* %1
+  ret i32 %2
+}
diff --git a/llvm/test/CodeGen/RISCV/uadd_sat.ll b/llvm/test/CodeGen/RISCV/uadd_sat.ll
index bac2a1915344a3e0a0dd9d66a69cf079ddb18c95..8f817b3e4972f44cac2e479e7e9adae1e4ca739c 100644
--- a/llvm/test/CodeGen/RISCV/uadd_sat.ll
+++ b/llvm/test/CodeGen/RISCV/uadd_sat.ll
@@ -24,19 +24,13 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 ;
 ; RV64I-LABEL: func:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    srli a1, a1, 32
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    addi a1, zero, 1
-; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    addi a1, a1, -1
-; RV64I-NEXT:    bltu a0, a1, .LBB0_2
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    addw a1, a0, a1
+; RV64I-NEXT:    addi a0, zero, -1
+; RV64I-NEXT:    bltu a1, a2, .LBB0_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:  .LBB0_2:
-; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    ret
 ;
 ; RV32IZbb-LABEL: func:
@@ -48,16 +42,9 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 ;
 ; RV64IZbb-LABEL: func:
 ; RV64IZbb:       # %bb.0:
-; RV64IZbb-NEXT:    slli a1, a1, 32
-; RV64IZbb-NEXT:    srli a1, a1, 32
-; RV64IZbb-NEXT:    slli a0, a0, 32
-; RV64IZbb-NEXT:    srli a0, a0, 32
-; RV64IZbb-NEXT:    add a0, a0, a1
-; RV64IZbb-NEXT:    addi a1, zero, 1
-; RV64IZbb-NEXT:    slli a1, a1, 32
-; RV64IZbb-NEXT:    addi a1, a1, -1
-; RV64IZbb-NEXT:    minu a0, a0, a1
-; RV64IZbb-NEXT:    sext.w a0, a0
+; RV64IZbb-NEXT:    not a2, a1
+; RV64IZbb-NEXT:    minu a0, a0, a2
+; RV64IZbb-NEXT:    addw a0, a0, a1
 ; RV64IZbb-NEXT:    ret
   %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y);
   ret i32 %tmp;
diff --git a/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll b/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll
index 20bb7a24d754f2c7e0a542124d871bceb2954092..589374493d71fe607efceaa7d4a3bcb0d9e2a27e 100644
--- a/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll
+++ b/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll
@@ -25,16 +25,11 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ;
 ; RV64I-LABEL: func32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    mul a1, a1, a2
-; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    srli a1, a1, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    addi a1, zero, 1
-; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    addi a1, a1, -1
-; RV64I-NEXT:    bltu a0, a1, .LBB0_2
+; RV64I-NEXT:    addw a1, a0, a1
+; RV64I-NEXT:    sext.w a2, a0
+; RV64I-NEXT:    addi a0, zero, -1
+; RV64I-NEXT:    bltu a1, a2, .LBB0_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:  .LBB0_2:
@@ -50,16 +45,11 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ;
 ; RV64IZbb-LABEL: func32:
 ; RV64IZbb:       # %bb.0:
-; RV64IZbb-NEXT:    slli a0, a0, 32
-; RV64IZbb-NEXT:    srli a0, a0, 32
-; RV64IZbb-NEXT:    mul a1, a1, a2
-; RV64IZbb-NEXT:    slli a1, a1, 32
-; RV64IZbb-NEXT:    srli a1, a1, 32
+; RV64IZbb-NEXT:    mulw a1, a1, a2
+; RV64IZbb-NEXT:    not a2, a1
+; RV64IZbb-NEXT:    sext.w a0, a0
+; RV64IZbb-NEXT:    minu a0, a0, a2
 ; RV64IZbb-NEXT:    add a0, a0, a1
-; RV64IZbb-NEXT:    addi a1, zero, 1
-; RV64IZbb-NEXT:    slli a1, a1, 32
-; RV64IZbb-NEXT:    addi a1, a1, -1
-; RV64IZbb-NEXT:    minu a0, a0, a1
 ; RV64IZbb-NEXT:    ret
   %a = mul i32 %y, %z
   %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %a)
diff --git a/llvm/test/CodeGen/RISCV/usub_sat.ll b/llvm/test/CodeGen/RISCV/usub_sat.ll
index 52657effaa9782cf05db486908e55c5be02d6430..353e8eaf838e1303ac5a45bfdfb7b251ac39ded9 100644
--- a/llvm/test/CodeGen/RISCV/usub_sat.ll
+++ b/llvm/test/CodeGen/RISCV/usub_sat.ll
@@ -24,17 +24,13 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 ;
 ; RV64I-LABEL: func:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    srli a1, a1, 32
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a2, a0, 32
-; RV64I-NEXT:    sub a0, a2, a1
-; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    bltu a2, a0, .LBB0_2
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    subw a1, a0, a1
+; RV64I-NEXT:    mv a0, zero
+; RV64I-NEXT:    bltu a2, a1, .LBB0_2
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:  .LBB0_2:
-; RV64I-NEXT:    sext.w a0, a1
 ; RV64I-NEXT:    ret
 ;
 ; RV32IZbb-LABEL: func:
@@ -45,11 +41,7 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 ;
 ; RV64IZbb-LABEL: func:
 ; RV64IZbb:       # %bb.0:
-; RV64IZbb-NEXT:    slli a2, a1, 32
-; RV64IZbb-NEXT:    srli a2, a2, 32
-; RV64IZbb-NEXT:    slli a0, a0, 32
-; RV64IZbb-NEXT:    srli a0, a0, 32
-; RV64IZbb-NEXT:    maxu a0, a0, a2
+; RV64IZbb-NEXT:    maxu a0, a0, a1
 ; RV64IZbb-NEXT:    subw a0, a0, a1
 ; RV64IZbb-NEXT:    ret
   %tmp = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y);
diff --git a/llvm/test/CodeGen/RISCV/usub_sat_plus.ll b/llvm/test/CodeGen/RISCV/usub_sat_plus.ll
index 590df5e65a725f4d5687a77bcfa183fc49f75d82..beeaf54e4238b419e43938676f801fe35d201196 100644
--- a/llvm/test/CodeGen/RISCV/usub_sat_plus.ll
+++ b/llvm/test/CodeGen/RISCV/usub_sat_plus.ll
@@ -25,14 +25,11 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ;
 ; RV64I-LABEL: func32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a3, a0, 32
-; RV64I-NEXT:    mul a0, a1, a2
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    sub a1, a3, a0
+; RV64I-NEXT:    mul a1, a1, a2
+; RV64I-NEXT:    subw a1, a0, a1
+; RV64I-NEXT:    sext.w a2, a0
 ; RV64I-NEXT:    mv a0, zero
-; RV64I-NEXT:    bltu a3, a1, .LBB0_2
+; RV64I-NEXT:    bltu a2, a1, .LBB0_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:  .LBB0_2:
@@ -47,11 +44,8 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ;
 ; RV64IZbb-LABEL: func32:
 ; RV64IZbb:       # %bb.0:
-; RV64IZbb-NEXT:    slli a0, a0, 32
-; RV64IZbb-NEXT:    srli a0, a0, 32
-; RV64IZbb-NEXT:    mul a1, a1, a2
-; RV64IZbb-NEXT:    slli a1, a1, 32
-; RV64IZbb-NEXT:    srli a1, a1, 32
+; RV64IZbb-NEXT:    mulw a1, a1, a2
+; RV64IZbb-NEXT:    sext.w a0, a0
 ; RV64IZbb-NEXT:    maxu a0, a0, a1
 ; RV64IZbb-NEXT:    sub a0, a0, a1
 ; RV64IZbb-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/vararg.ll b/llvm/test/CodeGen/RISCV/vararg.ll
index 0f2001b2746a13f6d4d6544d902b5291edc1a517..7efa1a3726031e3857898a3c24342e298ee48ee9 100644
--- a/llvm/test/CodeGen/RISCV/vararg.ll
+++ b/llvm/test/CodeGen/RISCV/vararg.ll
@@ -1773,7 +1773,6 @@ define i32 @va_large_stack(i8* %fmt, ...) {
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a1, 24414
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, a1, 280
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, sp, a1
-; ILP32-ILP32F-FPELIM-NEXT:    mv a1, a1
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a1, 24414
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, a1, 304
@@ -1852,7 +1851,6 @@ define i32 @va_large_stack(i8* %fmt, ...) {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a1, 24414
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, a1, 280
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, sp, a1
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a1, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a1, 24414
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, a1, 304
@@ -1896,7 +1894,6 @@ define i32 @va_large_stack(i8* %fmt, ...) {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 284
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, sp, a0
-; LP64-LP64F-LP64D-FPELIM-NEXT:    mv a0, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 24414
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 280
diff --git a/llvm/test/CodeGen/RISCV/xaluo.ll b/llvm/test/CodeGen/RISCV/xaluo.ll
index e29cfc9156bc3534b2ac5f74ac9bf0004e171570..707d1621d6ef819e13bb98015258d62fb2936ac5 100644
--- a/llvm/test/CodeGen/RISCV/xaluo.ll
+++ b/llvm/test/CodeGen/RISCV/xaluo.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=riscv32 -mattr=+m -verify-machineinstrs | FileCheck %s --check-prefix=RV32
 ; RUN: llc < %s -mtriple=riscv64 -mattr=+m -verify-machineinstrs | FileCheck %s --check-prefix=RV64
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+experimental-zba -verify-machineinstrs | FileCheck %s --check-prefix=RV32ZBA
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+experimental-zba -verify-machineinstrs | FileCheck %s --check-prefix=RV64ZBA
 
 ;
 ; Get the actual value of the overflow bit.
@@ -25,6 +27,26 @@ define zeroext i1 @saddo1.i32(i32 %v1, i32 %v2, i32* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a3, 0(a2)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo1.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a3, a0, a1
+; RV32ZBA-NEXT:    slt a0, a3, a0
+; RV32ZBA-NEXT:    slti a1, a1, 0
+; RV32ZBA-NEXT:    xor a0, a1, a0
+; RV32ZBA-NEXT:    sw a3, 0(a2)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo1.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a1, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    add a3, a0, a1
+; RV64ZBA-NEXT:    addw a0, a0, a1
+; RV64ZBA-NEXT:    xor a0, a0, a3
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a3, 0(a2)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -51,6 +73,23 @@ define zeroext i1 @saddo2.i32(i32 %v1, i32* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a2, 0(a1)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo2.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi a2, a0, 4
+; RV32ZBA-NEXT:    slt a0, a2, a0
+; RV32ZBA-NEXT:    sw a2, 0(a1)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo2.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    addi a2, a0, 4
+; RV64ZBA-NEXT:    addiw a0, a0, 4
+; RV64ZBA-NEXT:    xor a0, a0, a2
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a2, 0(a1)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 4)
   %val = extractvalue {i32, i1} %t, 0
@@ -78,6 +117,24 @@ define zeroext i1 @saddo3.i32(i32 %v1, i32* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a2, 0(a1)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo3.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi a2, a0, -4
+; RV32ZBA-NEXT:    slt a0, a2, a0
+; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    sw a2, 0(a1)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo3.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    addi a2, a0, -4
+; RV64ZBA-NEXT:    addiw a0, a0, -4
+; RV64ZBA-NEXT:    xor a0, a0, a2
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a2, 0(a1)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 -4)
   %val = extractvalue {i32, i1} %t, 0
@@ -108,6 +165,27 @@ define zeroext i1 @saddo4.i32(i32 %v1, i32* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a3, 0(a1)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo4.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    lui a2, 4096
+; RV32ZBA-NEXT:    addi a2, a2, -1
+; RV32ZBA-NEXT:    add a2, a0, a2
+; RV32ZBA-NEXT:    slt a0, a2, a0
+; RV32ZBA-NEXT:    sw a2, 0(a1)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo4.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    lui a2, 4096
+; RV64ZBA-NEXT:    addiw a2, a2, -1
+; RV64ZBA-NEXT:    add a3, a0, a2
+; RV64ZBA-NEXT:    addw a0, a0, a2
+; RV64ZBA-NEXT:    xor a0, a0, a3
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a3, 0(a1)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 16777215)
   %val = extractvalue {i32, i1} %t, 0
@@ -140,6 +218,30 @@ define zeroext i1 @saddo1.i64(i64 %v1, i64 %v2, i64* %res) {
 ; RV64-NEXT:    xor a0, a1, a0
 ; RV64-NEXT:    sd a3, 0(a2)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo1.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a5, a1, a3
+; RV32ZBA-NEXT:    add a2, a0, a2
+; RV32ZBA-NEXT:    sltu a0, a2, a0
+; RV32ZBA-NEXT:    add a5, a5, a0
+; RV32ZBA-NEXT:    xor a0, a1, a5
+; RV32ZBA-NEXT:    xor a1, a1, a3
+; RV32ZBA-NEXT:    not a1, a1
+; RV32ZBA-NEXT:    and a0, a1, a0
+; RV32ZBA-NEXT:    slti a0, a0, 0
+; RV32ZBA-NEXT:    sw a2, 0(a4)
+; RV32ZBA-NEXT:    sw a5, 4(a4)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo1.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    add a3, a0, a1
+; RV64ZBA-NEXT:    slt a0, a3, a0
+; RV64ZBA-NEXT:    slti a1, a1, 0
+; RV64ZBA-NEXT:    xor a0, a1, a0
+; RV64ZBA-NEXT:    sd a3, 0(a2)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -168,6 +270,26 @@ define zeroext i1 @saddo2.i64(i64 %v1, i64* %res) {
 ; RV64-NEXT:    slt a0, a2, a0
 ; RV64-NEXT:    sd a2, 0(a1)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo2.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi a3, a0, 4
+; RV32ZBA-NEXT:    sltu a0, a3, a0
+; RV32ZBA-NEXT:    add a4, a1, a0
+; RV32ZBA-NEXT:    xor a0, a1, a4
+; RV32ZBA-NEXT:    not a1, a1
+; RV32ZBA-NEXT:    and a0, a1, a0
+; RV32ZBA-NEXT:    slti a0, a0, 0
+; RV32ZBA-NEXT:    sw a3, 0(a2)
+; RV32ZBA-NEXT:    sw a4, 4(a2)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo2.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    addi a2, a0, 4
+; RV64ZBA-NEXT:    slt a0, a2, a0
+; RV64ZBA-NEXT:    sd a2, 0(a1)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 4)
   %val = extractvalue {i64, i1} %t, 0
@@ -197,6 +319,27 @@ define zeroext i1 @saddo3.i64(i64 %v1, i64* %res) {
 ; RV64-NEXT:    xori a0, a0, 1
 ; RV64-NEXT:    sd a2, 0(a1)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo3.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi a3, a0, -4
+; RV32ZBA-NEXT:    sltu a0, a3, a0
+; RV32ZBA-NEXT:    add a0, a1, a0
+; RV32ZBA-NEXT:    addi a4, a0, -1
+; RV32ZBA-NEXT:    xor a0, a1, a4
+; RV32ZBA-NEXT:    and a0, a1, a0
+; RV32ZBA-NEXT:    slti a0, a0, 0
+; RV32ZBA-NEXT:    sw a3, 0(a2)
+; RV32ZBA-NEXT:    sw a4, 4(a2)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo3.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    addi a2, a0, -4
+; RV64ZBA-NEXT:    slt a0, a2, a0
+; RV64ZBA-NEXT:    xori a0, a0, 1
+; RV64ZBA-NEXT:    sd a2, 0(a1)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 -4)
   %val = extractvalue {i64, i1} %t, 0
@@ -222,6 +365,23 @@ define zeroext i1 @uaddo.i32(i32 %v1, i32 %v2, i32* %res) {
 ; RV64-NEXT:    sw a0, 0(a2)
 ; RV64-NEXT:    mv a0, a3
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: uaddo.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a1, a0, a1
+; RV32ZBA-NEXT:    sltu a0, a1, a0
+; RV32ZBA-NEXT:    sw a1, 0(a2)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: uaddo.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    addw a3, a0, a1
+; RV64ZBA-NEXT:    sext.w a4, a0
+; RV64ZBA-NEXT:    sltu a3, a3, a4
+; RV64ZBA-NEXT:    add a0, a0, a1
+; RV64ZBA-NEXT:    sw a0, 0(a2)
+; RV64ZBA-NEXT:    mv a0, a3
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -251,6 +411,27 @@ define zeroext i1 @uaddo.i64(i64 %v1, i64 %v2, i64* %res) {
 ; RV64-NEXT:    sltu a0, a1, a0
 ; RV64-NEXT:    sd a1, 0(a2)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: uaddo.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a3, a1, a3
+; RV32ZBA-NEXT:    add a2, a0, a2
+; RV32ZBA-NEXT:    sltu a0, a2, a0
+; RV32ZBA-NEXT:    add a3, a3, a0
+; RV32ZBA-NEXT:    beq a3, a1, .LBB8_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    sltu a0, a3, a1
+; RV32ZBA-NEXT:  .LBB8_2: # %entry
+; RV32ZBA-NEXT:    sw a2, 0(a4)
+; RV32ZBA-NEXT:    sw a3, 4(a4)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: uaddo.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    add a1, a0, a1
+; RV64ZBA-NEXT:    sltu a0, a1, a0
+; RV64ZBA-NEXT:    sd a1, 0(a2)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -279,6 +460,26 @@ define zeroext i1 @ssubo1.i32(i32 %v1, i32 %v2, i32* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a3, 0(a2)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: ssubo1.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sgtz a3, a1
+; RV32ZBA-NEXT:    sub a1, a0, a1
+; RV32ZBA-NEXT:    slt a0, a1, a0
+; RV32ZBA-NEXT:    xor a0, a3, a0
+; RV32ZBA-NEXT:    sw a1, 0(a2)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: ssubo1.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a1, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    sub a3, a0, a1
+; RV64ZBA-NEXT:    subw a0, a0, a1
+; RV64ZBA-NEXT:    xor a0, a0, a3
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a3, 0(a2)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -304,6 +505,23 @@ define zeroext i1 @ssubo2.i32(i32 %v1, i32* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a2, 0(a1)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: ssubo2.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi a2, a0, 4
+; RV32ZBA-NEXT:    slt a0, a2, a0
+; RV32ZBA-NEXT:    sw a2, 0(a1)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: ssubo2.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    addi a2, a0, 4
+; RV64ZBA-NEXT:    addiw a0, a0, 4
+; RV64ZBA-NEXT:    xor a0, a0, a2
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a2, 0(a1)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 -4)
   %val = extractvalue {i32, i1} %t, 0
@@ -336,6 +554,30 @@ define zeroext i1 @ssubo.i64(i64 %v1, i64 %v2, i64* %res) {
 ; RV64-NEXT:    xor a0, a3, a0
 ; RV64-NEXT:    sd a1, 0(a2)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: ssubo.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sltu a6, a0, a2
+; RV32ZBA-NEXT:    sub a5, a1, a3
+; RV32ZBA-NEXT:    sub a5, a5, a6
+; RV32ZBA-NEXT:    xor a6, a1, a5
+; RV32ZBA-NEXT:    xor a1, a1, a3
+; RV32ZBA-NEXT:    and a1, a1, a6
+; RV32ZBA-NEXT:    slti a1, a1, 0
+; RV32ZBA-NEXT:    sub a0, a0, a2
+; RV32ZBA-NEXT:    sw a0, 0(a4)
+; RV32ZBA-NEXT:    sw a5, 4(a4)
+; RV32ZBA-NEXT:    mv a0, a1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: ssubo.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sgtz a3, a1
+; RV64ZBA-NEXT:    sub a1, a0, a1
+; RV64ZBA-NEXT:    slt a0, a1, a0
+; RV64ZBA-NEXT:    xor a0, a3, a0
+; RV64ZBA-NEXT:    sd a1, 0(a2)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -361,6 +603,23 @@ define zeroext i1 @usubo.i32(i32 %v1, i32 %v2, i32* %res) {
 ; RV64-NEXT:    sw a0, 0(a2)
 ; RV64-NEXT:    mv a0, a3
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: usubo.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sub a1, a0, a1
+; RV32ZBA-NEXT:    sltu a0, a0, a1
+; RV32ZBA-NEXT:    sw a1, 0(a2)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: usubo.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    subw a3, a0, a1
+; RV64ZBA-NEXT:    sext.w a4, a0
+; RV64ZBA-NEXT:    sltu a3, a4, a3
+; RV64ZBA-NEXT:    sub a0, a0, a1
+; RV64ZBA-NEXT:    sw a0, 0(a2)
+; RV64ZBA-NEXT:    mv a0, a3
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -393,6 +652,30 @@ define zeroext i1 @usubo.i64(i64 %v1, i64 %v2, i64* %res) {
 ; RV64-NEXT:    sltu a0, a0, a1
 ; RV64-NEXT:    sd a1, 0(a2)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: usubo.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sltu a5, a0, a2
+; RV32ZBA-NEXT:    sub a3, a1, a3
+; RV32ZBA-NEXT:    sub a3, a3, a5
+; RV32ZBA-NEXT:    sub a2, a0, a2
+; RV32ZBA-NEXT:    beq a3, a1, .LBB13_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    sltu a0, a1, a3
+; RV32ZBA-NEXT:    j .LBB13_3
+; RV32ZBA-NEXT:  .LBB13_2:
+; RV32ZBA-NEXT:    sltu a0, a0, a2
+; RV32ZBA-NEXT:  .LBB13_3: # %entry
+; RV32ZBA-NEXT:    sw a2, 0(a4)
+; RV32ZBA-NEXT:    sw a3, 4(a4)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: usubo.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sub a1, a0, a1
+; RV64ZBA-NEXT:    sltu a0, a0, a1
+; RV64ZBA-NEXT:    sd a1, 0(a2)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -422,6 +705,27 @@ define zeroext i1 @smulo.i32(i32 %v1, i32 %v2, i32* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a3, 0(a2)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mulh a3, a0, a1
+; RV32ZBA-NEXT:    mul a1, a0, a1
+; RV32ZBA-NEXT:    srai a0, a1, 31
+; RV32ZBA-NEXT:    xor a0, a3, a0
+; RV32ZBA-NEXT:    snez a0, a0
+; RV32ZBA-NEXT:    sw a1, 0(a2)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a1, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    mul a3, a0, a1
+; RV64ZBA-NEXT:    mulw a0, a0, a1
+; RV64ZBA-NEXT:    xor a0, a0, a3
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a3, 0(a2)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -452,6 +756,28 @@ define zeroext i1 @smulo2.i32(i32 %v1, i32* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a3, 0(a1)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo2.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi a2, zero, 13
+; RV32ZBA-NEXT:    mulh a3, a0, a2
+; RV32ZBA-NEXT:    mul a2, a0, a2
+; RV32ZBA-NEXT:    srai a0, a2, 31
+; RV32ZBA-NEXT:    xor a0, a3, a0
+; RV32ZBA-NEXT:    snez a0, a0
+; RV32ZBA-NEXT:    sw a2, 0(a1)
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo2.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    addi a2, zero, 13
+; RV64ZBA-NEXT:    mul a3, a0, a2
+; RV64ZBA-NEXT:    mulw a0, a0, a2
+; RV64ZBA-NEXT:    xor a0, a0, a3
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a3, 0(a1)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 13)
   %val = extractvalue {i32, i1} %t, 0
@@ -492,6 +818,38 @@ define zeroext i1 @smulo.i64(i64 %v1, i64 %v2, i64* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sd a1, 0(a2)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi sp, sp, -16
+; RV32ZBA-NEXT:    .cfi_def_cfa_offset 16
+; RV32ZBA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    .cfi_offset ra, -4
+; RV32ZBA-NEXT:    .cfi_offset s0, -8
+; RV32ZBA-NEXT:    mv s0, a4
+; RV32ZBA-NEXT:    sw zero, 4(sp)
+; RV32ZBA-NEXT:    addi a4, sp, 4
+; RV32ZBA-NEXT:    call __mulodi4@plt
+; RV32ZBA-NEXT:    lw a2, 4(sp)
+; RV32ZBA-NEXT:    snez a2, a2
+; RV32ZBA-NEXT:    sw a1, 4(s0)
+; RV32ZBA-NEXT:    sw a0, 0(s0)
+; RV32ZBA-NEXT:    mv a0, a2
+; RV32ZBA-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    addi sp, sp, 16
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    mulh a3, a0, a1
+; RV64ZBA-NEXT:    mul a1, a0, a1
+; RV64ZBA-NEXT:    srai a0, a1, 63
+; RV64ZBA-NEXT:    xor a0, a3, a0
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sd a1, 0(a2)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -535,6 +893,41 @@ define zeroext i1 @smulo2.i64(i64 %v1, i64* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sd a2, 0(a1)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo2.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi sp, sp, -16
+; RV32ZBA-NEXT:    .cfi_def_cfa_offset 16
+; RV32ZBA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    .cfi_offset ra, -4
+; RV32ZBA-NEXT:    .cfi_offset s0, -8
+; RV32ZBA-NEXT:    mv s0, a2
+; RV32ZBA-NEXT:    sw zero, 4(sp)
+; RV32ZBA-NEXT:    addi a2, zero, 13
+; RV32ZBA-NEXT:    addi a4, sp, 4
+; RV32ZBA-NEXT:    mv a3, zero
+; RV32ZBA-NEXT:    call __mulodi4@plt
+; RV32ZBA-NEXT:    lw a2, 4(sp)
+; RV32ZBA-NEXT:    snez a2, a2
+; RV32ZBA-NEXT:    sw a1, 4(s0)
+; RV32ZBA-NEXT:    sw a0, 0(s0)
+; RV32ZBA-NEXT:    mv a0, a2
+; RV32ZBA-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    addi sp, sp, 16
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo2.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    addi a2, zero, 13
+; RV64ZBA-NEXT:    mulh a3, a0, a2
+; RV64ZBA-NEXT:    mul a2, a0, a2
+; RV64ZBA-NEXT:    srai a0, a2, 63
+; RV64ZBA-NEXT:    xor a0, a3, a0
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sd a2, 0(a1)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 13)
   %val = extractvalue {i64, i1} %t, 0
@@ -556,14 +949,31 @@ define zeroext i1 @umulo.i32(i32 %v1, i32 %v2, i32* %res) {
 ; RV64-LABEL: umulo.i32:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    slli a1, a1, 32
-; RV64-NEXT:    srli a1, a1, 32
 ; RV64-NEXT:    slli a0, a0, 32
-; RV64-NEXT:    srli a0, a0, 32
-; RV64-NEXT:    mul a1, a0, a1
+; RV64-NEXT:    mulhu a1, a0, a1
 ; RV64-NEXT:    srli a0, a1, 32
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a1, 0(a2)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mulhu a3, a0, a1
+; RV32ZBA-NEXT:    snez a3, a3
+; RV32ZBA-NEXT:    mul a0, a0, a1
+; RV32ZBA-NEXT:    sw a0, 0(a2)
+; RV32ZBA-NEXT:    mv a0, a3
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    zext.w a1, a1
+; RV64ZBA-NEXT:    zext.w a0, a0
+; RV64ZBA-NEXT:    mul a1, a0, a1
+; RV64ZBA-NEXT:    srli a0, a1, 32
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a1, 0(a2)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -593,6 +1003,26 @@ define zeroext i1 @umulo2.i32(i32 %v1, i32* %res) {
 ; RV64-NEXT:    snez a0, a0
 ; RV64-NEXT:    sw a2, 0(a1)
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo2.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi a3, zero, 13
+; RV32ZBA-NEXT:    mulhu a2, a0, a3
+; RV32ZBA-NEXT:    snez a2, a2
+; RV32ZBA-NEXT:    mul a0, a0, a3
+; RV32ZBA-NEXT:    sw a0, 0(a1)
+; RV32ZBA-NEXT:    mv a0, a2
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo2.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    zext.w a0, a0
+; RV64ZBA-NEXT:    addi a2, zero, 13
+; RV64ZBA-NEXT:    mul a2, a0, a2
+; RV64ZBA-NEXT:    srli a0, a2, 32
+; RV64ZBA-NEXT:    snez a0, a0
+; RV64ZBA-NEXT:    sw a2, 0(a1)
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 13)
   %val = extractvalue {i32, i1} %t, 0
@@ -601,6 +1031,55 @@ entry:
   ret i1 %obit
 }
 
+; Similar to umulo.i32, but storing the overflow and returning the result.
+define signext i32 @umulo3.i32(i32 signext %0, i32 signext %1, i32* %2) {
+; RV32-LABEL: umulo3.i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    mul a3, a0, a1
+; RV32-NEXT:    mulhu a0, a0, a1
+; RV32-NEXT:    snez a0, a0
+; RV32-NEXT:    sw a0, 0(a2)
+; RV32-NEXT:    mv a0, a3
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: umulo3.i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    slli a1, a1, 32
+; RV64-NEXT:    slli a0, a0, 32
+; RV64-NEXT:    mulhu a0, a0, a1
+; RV64-NEXT:    srli a1, a0, 32
+; RV64-NEXT:    snez a1, a1
+; RV64-NEXT:    sext.w a0, a0
+; RV64-NEXT:    sw a1, 0(a2)
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo3.i32:
+; RV32ZBA:       # %bb.0:
+; RV32ZBA-NEXT:    mul a3, a0, a1
+; RV32ZBA-NEXT:    mulhu a0, a0, a1
+; RV32ZBA-NEXT:    snez a0, a0
+; RV32ZBA-NEXT:    sw a0, 0(a2)
+; RV32ZBA-NEXT:    mv a0, a3
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo3.i32:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    zext.w a1, a1
+; RV64ZBA-NEXT:    zext.w a0, a0
+; RV64ZBA-NEXT:    mul a3, a0, a1
+; RV64ZBA-NEXT:    srli a3, a3, 32
+; RV64ZBA-NEXT:    snez a3, a3
+; RV64ZBA-NEXT:    mulw a0, a0, a1
+; RV64ZBA-NEXT:    sw a3, 0(a2)
+; RV64ZBA-NEXT:    ret
+  %4 = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %0, i32 %1)
+  %5 = extractvalue { i32, i1 } %4, 1
+  %6 = extractvalue { i32, i1 } %4, 0
+  %7 = zext i1 %5 to i32
+  store i32 %7, i32* %2, align 4
+  ret i32 %6
+}
+
 define zeroext i1 @umulo.i64(i64 %v1, i64 %v2, i64* %res) {
 ; RV32-LABEL: umulo.i64:
 ; RV32:       # %bb.0: # %entry
@@ -634,6 +1113,39 @@ define zeroext i1 @umulo.i64(i64 %v1, i64 %v2, i64* %res) {
 ; RV64-NEXT:    sd a0, 0(a2)
 ; RV64-NEXT:    mv a0, a3
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mul a6, a3, a0
+; RV32ZBA-NEXT:    mul a5, a1, a2
+; RV32ZBA-NEXT:    add a6, a5, a6
+; RV32ZBA-NEXT:    mulhu a5, a0, a2
+; RV32ZBA-NEXT:    add a6, a5, a6
+; RV32ZBA-NEXT:    sltu a7, a6, a5
+; RV32ZBA-NEXT:    snez t0, a3
+; RV32ZBA-NEXT:    snez a5, a1
+; RV32ZBA-NEXT:    and a5, a5, t0
+; RV32ZBA-NEXT:    mulhu a1, a1, a2
+; RV32ZBA-NEXT:    snez a1, a1
+; RV32ZBA-NEXT:    or a1, a5, a1
+; RV32ZBA-NEXT:    mulhu a3, a3, a0
+; RV32ZBA-NEXT:    snez a3, a3
+; RV32ZBA-NEXT:    or a1, a1, a3
+; RV32ZBA-NEXT:    or a1, a1, a7
+; RV32ZBA-NEXT:    mul a0, a0, a2
+; RV32ZBA-NEXT:    sw a0, 0(a4)
+; RV32ZBA-NEXT:    sw a6, 4(a4)
+; RV32ZBA-NEXT:    mv a0, a1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    mulhu a3, a0, a1
+; RV64ZBA-NEXT:    snez a3, a3
+; RV64ZBA-NEXT:    mul a0, a0, a1
+; RV64ZBA-NEXT:    sd a0, 0(a2)
+; RV64ZBA-NEXT:    mv a0, a3
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -668,6 +1180,32 @@ define zeroext i1 @umulo2.i64(i64 %v1, i64* %res) {
 ; RV64-NEXT:    sd a0, 0(a1)
 ; RV64-NEXT:    mv a0, a2
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo2.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi a3, zero, 13
+; RV32ZBA-NEXT:    mul a4, a1, a3
+; RV32ZBA-NEXT:    mulhu a5, a0, a3
+; RV32ZBA-NEXT:    add a4, a5, a4
+; RV32ZBA-NEXT:    sltu a5, a4, a5
+; RV32ZBA-NEXT:    mulhu a1, a1, a3
+; RV32ZBA-NEXT:    snez a1, a1
+; RV32ZBA-NEXT:    or a1, a1, a5
+; RV32ZBA-NEXT:    mul a0, a0, a3
+; RV32ZBA-NEXT:    sw a0, 0(a2)
+; RV32ZBA-NEXT:    sw a4, 4(a2)
+; RV32ZBA-NEXT:    mv a0, a1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo2.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    addi a3, zero, 13
+; RV64ZBA-NEXT:    mulhu a2, a0, a3
+; RV64ZBA-NEXT:    snez a2, a2
+; RV64ZBA-NEXT:    mul a0, a0, a3
+; RV64ZBA-NEXT:    sd a0, 0(a1)
+; RV64ZBA-NEXT:    mv a0, a2
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 13)
   %val = extractvalue {i64, i1} %t, 0
@@ -686,10 +1224,10 @@ define i32 @saddo.select.i32(i32 %v1, i32 %v2) {
 ; RV32-NEXT:    add a2, a0, a1
 ; RV32-NEXT:    slt a2, a2, a0
 ; RV32-NEXT:    slti a3, a1, 0
-; RV32-NEXT:    bne a3, a2, .LBB22_2
+; RV32-NEXT:    bne a3, a2, .LBB23_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a1
-; RV32-NEXT:  .LBB22_2: # %entry
+; RV32-NEXT:  .LBB23_2: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: saddo.select.i32:
@@ -698,11 +1236,34 @@ define i32 @saddo.select.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    sext.w a3, a0
 ; RV64-NEXT:    add a4, a3, a2
 ; RV64-NEXT:    addw a2, a3, a2
-; RV64-NEXT:    bne a2, a4, .LBB22_2
+; RV64-NEXT:    bne a2, a4, .LBB23_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB22_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB23_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo.select.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a2, a0, a1
+; RV32ZBA-NEXT:    slt a2, a2, a0
+; RV32ZBA-NEXT:    slti a3, a1, 0
+; RV32ZBA-NEXT:    bne a3, a2, .LBB23_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv a0, a1
+; RV32ZBA-NEXT:  .LBB23_2: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo.select.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a2, a1
+; RV64ZBA-NEXT:    sext.w a3, a0
+; RV64ZBA-NEXT:    add a4, a3, a2
+; RV64ZBA-NEXT:    addw a2, a3, a2
+; RV64ZBA-NEXT:    bne a2, a4, .LBB23_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB23_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -729,6 +1290,25 @@ define i1 @saddo.not.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    xor a0, a0, a2
 ; RV64-NEXT:    seqz a0, a0
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo.not.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a2, a0, a1
+; RV32ZBA-NEXT:    slt a0, a2, a0
+; RV32ZBA-NEXT:    slti a1, a1, 0
+; RV32ZBA-NEXT:    xor a0, a1, a0
+; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo.not.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a1, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    add a2, a0, a1
+; RV64ZBA-NEXT:    addw a0, a0, a1
+; RV64ZBA-NEXT:    xor a0, a0, a2
+; RV64ZBA-NEXT:    seqz a0, a0
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -747,11 +1327,11 @@ define i64 @saddo.select.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    xor a5, a1, a3
 ; RV32-NEXT:    not a5, a5
 ; RV32-NEXT:    and a4, a5, a4
-; RV32-NEXT:    bltz a4, .LBB24_2
+; RV32-NEXT:    bltz a4, .LBB25_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a2
 ; RV32-NEXT:    mv a1, a3
-; RV32-NEXT:  .LBB24_2: # %entry
+; RV32-NEXT:  .LBB25_2: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: saddo.select.i64:
@@ -759,11 +1339,39 @@ define i64 @saddo.select.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    add a2, a0, a1
 ; RV64-NEXT:    slt a2, a2, a0
 ; RV64-NEXT:    slti a3, a1, 0
-; RV64-NEXT:    bne a3, a2, .LBB24_2
+; RV64-NEXT:    bne a3, a2, .LBB25_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB24_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB25_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo.select.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a4, a1, a3
+; RV32ZBA-NEXT:    add a5, a0, a2
+; RV32ZBA-NEXT:    sltu a5, a5, a0
+; RV32ZBA-NEXT:    add a4, a4, a5
+; RV32ZBA-NEXT:    xor a4, a1, a4
+; RV32ZBA-NEXT:    xor a5, a1, a3
+; RV32ZBA-NEXT:    not a5, a5
+; RV32ZBA-NEXT:    and a4, a5, a4
+; RV32ZBA-NEXT:    bltz a4, .LBB25_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv a0, a2
+; RV32ZBA-NEXT:    mv a1, a3
+; RV32ZBA-NEXT:  .LBB25_2: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo.select.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    add a2, a0, a1
+; RV64ZBA-NEXT:    slt a2, a2, a0
+; RV64ZBA-NEXT:    slti a3, a1, 0
+; RV64ZBA-NEXT:    bne a3, a2, .LBB25_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB25_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -794,6 +1402,29 @@ define i1 @saddo.not.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    xor a0, a1, a0
 ; RV64-NEXT:    xori a0, a0, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo.not.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a4, a1, a3
+; RV32ZBA-NEXT:    add a2, a0, a2
+; RV32ZBA-NEXT:    sltu a0, a2, a0
+; RV32ZBA-NEXT:    add a0, a4, a0
+; RV32ZBA-NEXT:    xor a0, a1, a0
+; RV32ZBA-NEXT:    xor a1, a1, a3
+; RV32ZBA-NEXT:    not a1, a1
+; RV32ZBA-NEXT:    and a0, a1, a0
+; RV32ZBA-NEXT:    addi a1, zero, -1
+; RV32ZBA-NEXT:    slt a0, a1, a0
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo.not.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    add a2, a0, a1
+; RV64ZBA-NEXT:    slt a0, a2, a0
+; RV64ZBA-NEXT:    slti a1, a1, 0
+; RV64ZBA-NEXT:    xor a0, a1, a0
+; RV64ZBA-NEXT:    xori a0, a0, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -805,21 +1436,40 @@ define i32 @uaddo.select.i32(i32 %v1, i32 %v2) {
 ; RV32-LABEL: uaddo.select.i32:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    add a2, a0, a1
-; RV32-NEXT:    bltu a2, a0, .LBB26_2
+; RV32-NEXT:    bltu a2, a0, .LBB27_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a1
-; RV32-NEXT:  .LBB26_2: # %entry
+; RV32-NEXT:  .LBB27_2: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: uaddo.select.i32:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    addw a2, a0, a1
 ; RV64-NEXT:    sext.w a3, a0
-; RV64-NEXT:    bltu a2, a3, .LBB26_2
+; RV64-NEXT:    bltu a2, a3, .LBB27_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB26_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB27_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: uaddo.select.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a2, a0, a1
+; RV32ZBA-NEXT:    bltu a2, a0, .LBB27_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv a0, a1
+; RV32ZBA-NEXT:  .LBB27_2: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: uaddo.select.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    addw a2, a0, a1
+; RV64ZBA-NEXT:    sext.w a3, a0
+; RV64ZBA-NEXT:    bltu a2, a3, .LBB27_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB27_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -842,6 +1492,21 @@ define i1 @uaddo.not.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    sltu a0, a1, a0
 ; RV64-NEXT:    xori a0, a0, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: uaddo.not.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a1, a0, a1
+; RV32ZBA-NEXT:    sltu a0, a1, a0
+; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: uaddo.not.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    addw a1, a0, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    sltu a0, a1, a0
+; RV64ZBA-NEXT:    xori a0, a0, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -856,15 +1521,15 @@ define i64 @uaddo.select.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    add a4, a0, a2
 ; RV32-NEXT:    sltu a4, a4, a0
 ; RV32-NEXT:    add a5, a5, a4
-; RV32-NEXT:    bne a5, a1, .LBB28_3
+; RV32-NEXT:    bne a5, a1, .LBB29_3
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    beqz a4, .LBB28_4
-; RV32-NEXT:  .LBB28_2: # %entry
+; RV32-NEXT:    beqz a4, .LBB29_4
+; RV32-NEXT:  .LBB29_2: # %entry
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB28_3: # %entry
+; RV32-NEXT:  .LBB29_3: # %entry
 ; RV32-NEXT:    sltu a4, a5, a1
-; RV32-NEXT:    bnez a4, .LBB28_2
-; RV32-NEXT:  .LBB28_4: # %entry
+; RV32-NEXT:    bnez a4, .LBB29_2
+; RV32-NEXT:  .LBB29_4: # %entry
 ; RV32-NEXT:    mv a0, a2
 ; RV32-NEXT:    mv a1, a3
 ; RV32-NEXT:    ret
@@ -872,11 +1537,39 @@ define i64 @uaddo.select.i64(i64 %v1, i64 %v2) {
 ; RV64-LABEL: uaddo.select.i64:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    add a2, a0, a1
-; RV64-NEXT:    bltu a2, a0, .LBB28_2
+; RV64-NEXT:    bltu a2, a0, .LBB29_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB28_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB29_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: uaddo.select.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a5, a1, a3
+; RV32ZBA-NEXT:    add a4, a0, a2
+; RV32ZBA-NEXT:    sltu a4, a4, a0
+; RV32ZBA-NEXT:    add a5, a5, a4
+; RV32ZBA-NEXT:    bne a5, a1, .LBB29_3
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    beqz a4, .LBB29_4
+; RV32ZBA-NEXT:  .LBB29_2: # %entry
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB29_3: # %entry
+; RV32ZBA-NEXT:    sltu a4, a5, a1
+; RV32ZBA-NEXT:    bnez a4, .LBB29_2
+; RV32ZBA-NEXT:  .LBB29_4: # %entry
+; RV32ZBA-NEXT:    mv a0, a2
+; RV32ZBA-NEXT:    mv a1, a3
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: uaddo.select.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    add a2, a0, a1
+; RV64ZBA-NEXT:    bltu a2, a0, .LBB29_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB29_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -891,10 +1584,10 @@ define i1 @uaddo.not.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    add a2, a0, a2
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a2, a3, a0
-; RV32-NEXT:    beq a2, a1, .LBB29_2
+; RV32-NEXT:    beq a2, a1, .LBB30_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    sltu a0, a2, a1
-; RV32-NEXT:  .LBB29_2: # %entry
+; RV32-NEXT:  .LBB30_2: # %entry
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    ret
 ;
@@ -904,6 +1597,26 @@ define i1 @uaddo.not.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    sltu a0, a1, a0
 ; RV64-NEXT:    xori a0, a0, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: uaddo.not.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a3, a1, a3
+; RV32ZBA-NEXT:    add a2, a0, a2
+; RV32ZBA-NEXT:    sltu a0, a2, a0
+; RV32ZBA-NEXT:    add a2, a3, a0
+; RV32ZBA-NEXT:    beq a2, a1, .LBB30_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    sltu a0, a2, a1
+; RV32ZBA-NEXT:  .LBB30_2: # %entry
+; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: uaddo.not.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    add a1, a0, a1
+; RV64ZBA-NEXT:    sltu a0, a1, a0
+; RV64ZBA-NEXT:    xori a0, a0, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -917,10 +1630,10 @@ define i32 @ssubo.select.i32(i32 %v1, i32 %v2) {
 ; RV32-NEXT:    sgtz a2, a1
 ; RV32-NEXT:    sub a3, a0, a1
 ; RV32-NEXT:    slt a3, a3, a0
-; RV32-NEXT:    bne a2, a3, .LBB30_2
+; RV32-NEXT:    bne a2, a3, .LBB31_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a1
-; RV32-NEXT:  .LBB30_2: # %entry
+; RV32-NEXT:  .LBB31_2: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ssubo.select.i32:
@@ -929,11 +1642,34 @@ define i32 @ssubo.select.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    sext.w a3, a0
 ; RV64-NEXT:    sub a4, a3, a2
 ; RV64-NEXT:    subw a2, a3, a2
-; RV64-NEXT:    bne a2, a4, .LBB30_2
+; RV64-NEXT:    bne a2, a4, .LBB31_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB30_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB31_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: ssubo.select.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sgtz a2, a1
+; RV32ZBA-NEXT:    sub a3, a0, a1
+; RV32ZBA-NEXT:    slt a3, a3, a0
+; RV32ZBA-NEXT:    bne a2, a3, .LBB31_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv a0, a1
+; RV32ZBA-NEXT:  .LBB31_2: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: ssubo.select.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a2, a1
+; RV64ZBA-NEXT:    sext.w a3, a0
+; RV64ZBA-NEXT:    sub a4, a3, a2
+; RV64ZBA-NEXT:    subw a2, a3, a2
+; RV64ZBA-NEXT:    bne a2, a4, .LBB31_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB31_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -960,6 +1696,25 @@ define i1 @ssubo.not.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    xor a0, a0, a2
 ; RV64-NEXT:    seqz a0, a0
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: ssubo.not.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sgtz a2, a1
+; RV32ZBA-NEXT:    sub a1, a0, a1
+; RV32ZBA-NEXT:    slt a0, a1, a0
+; RV32ZBA-NEXT:    xor a0, a2, a0
+; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: ssubo.not.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a1, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    sub a2, a0, a1
+; RV64ZBA-NEXT:    subw a0, a0, a1
+; RV64ZBA-NEXT:    xor a0, a0, a2
+; RV64ZBA-NEXT:    seqz a0, a0
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -976,11 +1731,11 @@ define i64 @ssubo.select.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    xor a4, a1, a4
 ; RV32-NEXT:    xor a5, a1, a3
 ; RV32-NEXT:    and a4, a5, a4
-; RV32-NEXT:    bltz a4, .LBB32_2
+; RV32-NEXT:    bltz a4, .LBB33_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a2
 ; RV32-NEXT:    mv a1, a3
-; RV32-NEXT:  .LBB32_2: # %entry
+; RV32-NEXT:  .LBB33_2: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ssubo.select.i64:
@@ -988,11 +1743,37 @@ define i64 @ssubo.select.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    sgtz a2, a1
 ; RV64-NEXT:    sub a3, a0, a1
 ; RV64-NEXT:    slt a3, a3, a0
-; RV64-NEXT:    bne a2, a3, .LBB32_2
+; RV64-NEXT:    bne a2, a3, .LBB33_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB32_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB33_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: ssubo.select.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sltu a4, a0, a2
+; RV32ZBA-NEXT:    sub a5, a1, a3
+; RV32ZBA-NEXT:    sub a4, a5, a4
+; RV32ZBA-NEXT:    xor a4, a1, a4
+; RV32ZBA-NEXT:    xor a5, a1, a3
+; RV32ZBA-NEXT:    and a4, a5, a4
+; RV32ZBA-NEXT:    bltz a4, .LBB33_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv a0, a2
+; RV32ZBA-NEXT:    mv a1, a3
+; RV32ZBA-NEXT:  .LBB33_2: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: ssubo.select.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sgtz a2, a1
+; RV64ZBA-NEXT:    sub a3, a0, a1
+; RV64ZBA-NEXT:    slt a3, a3, a0
+; RV64ZBA-NEXT:    bne a2, a3, .LBB33_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB33_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -1021,6 +1802,27 @@ define i1 @ssub.not.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    xor a0, a2, a0
 ; RV64-NEXT:    xori a0, a0, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: ssub.not.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sltu a0, a0, a2
+; RV32ZBA-NEXT:    sub a2, a1, a3
+; RV32ZBA-NEXT:    sub a0, a2, a0
+; RV32ZBA-NEXT:    xor a0, a1, a0
+; RV32ZBA-NEXT:    xor a1, a1, a3
+; RV32ZBA-NEXT:    and a0, a1, a0
+; RV32ZBA-NEXT:    addi a1, zero, -1
+; RV32ZBA-NEXT:    slt a0, a1, a0
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: ssub.not.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sgtz a2, a1
+; RV64ZBA-NEXT:    sub a1, a0, a1
+; RV64ZBA-NEXT:    slt a0, a1, a0
+; RV64ZBA-NEXT:    xor a0, a2, a0
+; RV64ZBA-NEXT:    xori a0, a0, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -1032,21 +1834,40 @@ define i32 @usubo.select.i32(i32 %v1, i32 %v2) {
 ; RV32-LABEL: usubo.select.i32:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    sub a2, a0, a1
-; RV32-NEXT:    bltu a0, a2, .LBB34_2
+; RV32-NEXT:    bltu a0, a2, .LBB35_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a1
-; RV32-NEXT:  .LBB34_2: # %entry
+; RV32-NEXT:  .LBB35_2: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: usubo.select.i32:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    subw a2, a0, a1
 ; RV64-NEXT:    sext.w a3, a0
-; RV64-NEXT:    bltu a3, a2, .LBB34_2
+; RV64-NEXT:    bltu a3, a2, .LBB35_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB34_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB35_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: usubo.select.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sub a2, a0, a1
+; RV32ZBA-NEXT:    bltu a0, a2, .LBB35_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv a0, a1
+; RV32ZBA-NEXT:  .LBB35_2: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: usubo.select.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    subw a2, a0, a1
+; RV64ZBA-NEXT:    sext.w a3, a0
+; RV64ZBA-NEXT:    bltu a3, a2, .LBB35_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB35_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -1069,6 +1890,21 @@ define i1 @usubo.not.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    sltu a0, a0, a1
 ; RV64-NEXT:    xori a0, a0, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: usubo.not.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sub a1, a0, a1
+; RV32ZBA-NEXT:    sltu a0, a0, a1
+; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: usubo.not.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    subw a1, a0, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    sltu a0, a0, a1
+; RV64ZBA-NEXT:    xori a0, a0, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -1082,29 +1918,58 @@ define i64 @usubo.select.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    sltu a4, a0, a2
 ; RV32-NEXT:    sub a5, a1, a3
 ; RV32-NEXT:    sub a4, a5, a4
-; RV32-NEXT:    beq a4, a1, .LBB36_2
+; RV32-NEXT:    beq a4, a1, .LBB37_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    sltu a4, a1, a4
-; RV32-NEXT:    beqz a4, .LBB36_3
-; RV32-NEXT:    j .LBB36_4
-; RV32-NEXT:  .LBB36_2:
+; RV32-NEXT:    beqz a4, .LBB37_3
+; RV32-NEXT:    j .LBB37_4
+; RV32-NEXT:  .LBB37_2:
 ; RV32-NEXT:    sub a4, a0, a2
 ; RV32-NEXT:    sltu a4, a0, a4
-; RV32-NEXT:    bnez a4, .LBB36_4
-; RV32-NEXT:  .LBB36_3: # %entry
+; RV32-NEXT:    bnez a4, .LBB37_4
+; RV32-NEXT:  .LBB37_3: # %entry
 ; RV32-NEXT:    mv a0, a2
 ; RV32-NEXT:    mv a1, a3
-; RV32-NEXT:  .LBB36_4: # %entry
+; RV32-NEXT:  .LBB37_4: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: usubo.select.i64:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    sub a2, a0, a1
-; RV64-NEXT:    bltu a0, a2, .LBB36_2
+; RV64-NEXT:    bltu a0, a2, .LBB37_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB36_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB37_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: usubo.select.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sltu a4, a0, a2
+; RV32ZBA-NEXT:    sub a5, a1, a3
+; RV32ZBA-NEXT:    sub a4, a5, a4
+; RV32ZBA-NEXT:    beq a4, a1, .LBB37_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    sltu a4, a1, a4
+; RV32ZBA-NEXT:    beqz a4, .LBB37_3
+; RV32ZBA-NEXT:    j .LBB37_4
+; RV32ZBA-NEXT:  .LBB37_2:
+; RV32ZBA-NEXT:    sub a4, a0, a2
+; RV32ZBA-NEXT:    sltu a4, a0, a4
+; RV32ZBA-NEXT:    bnez a4, .LBB37_4
+; RV32ZBA-NEXT:  .LBB37_3: # %entry
+; RV32ZBA-NEXT:    mv a0, a2
+; RV32ZBA-NEXT:    mv a1, a3
+; RV32ZBA-NEXT:  .LBB37_4: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: usubo.select.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sub a2, a0, a1
+; RV64ZBA-NEXT:    bltu a0, a2, .LBB37_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB37_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -1118,12 +1983,12 @@ define i1 @usubo.not.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    sltu a4, a0, a2
 ; RV32-NEXT:    sub a3, a1, a3
 ; RV32-NEXT:    sub a3, a3, a4
-; RV32-NEXT:    beq a3, a1, .LBB37_2
+; RV32-NEXT:    beq a3, a1, .LBB38_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    sltu a0, a1, a3
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB37_2:
+; RV32-NEXT:  .LBB38_2:
 ; RV32-NEXT:    sub a1, a0, a2
 ; RV32-NEXT:    sltu a0, a0, a1
 ; RV32-NEXT:    xori a0, a0, 1
@@ -1135,6 +2000,29 @@ define i1 @usubo.not.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    sltu a0, a0, a1
 ; RV64-NEXT:    xori a0, a0, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: usubo.not.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sltu a4, a0, a2
+; RV32ZBA-NEXT:    sub a3, a1, a3
+; RV32ZBA-NEXT:    sub a3, a3, a4
+; RV32ZBA-NEXT:    beq a3, a1, .LBB38_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    sltu a0, a1, a3
+; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB38_2:
+; RV32ZBA-NEXT:    sub a1, a0, a2
+; RV32ZBA-NEXT:    sltu a0, a0, a1
+; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: usubo.not.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sub a1, a0, a1
+; RV64ZBA-NEXT:    sltu a0, a0, a1
+; RV64ZBA-NEXT:    xori a0, a0, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -1148,10 +2036,10 @@ define i32 @smulo.select.i32(i32 %v1, i32 %v2) {
 ; RV32-NEXT:    mulh a2, a0, a1
 ; RV32-NEXT:    mul a3, a0, a1
 ; RV32-NEXT:    srai a3, a3, 31
-; RV32-NEXT:    bne a2, a3, .LBB38_2
+; RV32-NEXT:    bne a2, a3, .LBB39_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a1
-; RV32-NEXT:  .LBB38_2: # %entry
+; RV32-NEXT:  .LBB39_2: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: smulo.select.i32:
@@ -1160,11 +2048,34 @@ define i32 @smulo.select.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    sext.w a3, a0
 ; RV64-NEXT:    mul a4, a3, a2
 ; RV64-NEXT:    mulw a2, a3, a2
-; RV64-NEXT:    bne a2, a4, .LBB38_2
+; RV64-NEXT:    bne a2, a4, .LBB39_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB38_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB39_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo.select.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mulh a2, a0, a1
+; RV32ZBA-NEXT:    mul a3, a0, a1
+; RV32ZBA-NEXT:    srai a3, a3, 31
+; RV32ZBA-NEXT:    bne a2, a3, .LBB39_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv a0, a1
+; RV32ZBA-NEXT:  .LBB39_2: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo.select.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a2, a1
+; RV64ZBA-NEXT:    sext.w a3, a0
+; RV64ZBA-NEXT:    mul a4, a3, a2
+; RV64ZBA-NEXT:    mulw a2, a3, a2
+; RV64ZBA-NEXT:    bne a2, a4, .LBB39_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB39_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -1191,6 +2102,25 @@ define i1 @smulo.not.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    xor a0, a0, a2
 ; RV64-NEXT:    seqz a0, a0
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo.not.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mulh a2, a0, a1
+; RV32ZBA-NEXT:    mul a0, a0, a1
+; RV32ZBA-NEXT:    srai a0, a0, 31
+; RV32ZBA-NEXT:    xor a0, a2, a0
+; RV32ZBA-NEXT:    seqz a0, a0
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo.not.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a1, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    mul a2, a0, a1
+; RV64ZBA-NEXT:    mulw a0, a0, a1
+; RV64ZBA-NEXT:    xor a0, a0, a2
+; RV64ZBA-NEXT:    seqz a0, a0
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -1221,11 +2151,11 @@ define i64 @smulo.select.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    addi a4, sp, 8
 ; RV32-NEXT:    call __mulodi4@plt
 ; RV32-NEXT:    lw a0, 8(sp)
-; RV32-NEXT:    bnez a0, .LBB40_2
+; RV32-NEXT:    bnez a0, .LBB41_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv s1, s3
 ; RV32-NEXT:    mv s0, s2
-; RV32-NEXT:  .LBB40_2: # %entry
+; RV32-NEXT:  .LBB41_2: # %entry
 ; RV32-NEXT:    mv a0, s1
 ; RV32-NEXT:    mv a1, s0
 ; RV32-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
@@ -1241,11 +2171,59 @@ define i64 @smulo.select.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    mulh a2, a0, a1
 ; RV64-NEXT:    mul a3, a0, a1
 ; RV64-NEXT:    srai a3, a3, 63
-; RV64-NEXT:    bne a2, a3, .LBB40_2
+; RV64-NEXT:    bne a2, a3, .LBB41_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB40_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB41_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo.select.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi sp, sp, -32
+; RV32ZBA-NEXT:    .cfi_def_cfa_offset 32
+; RV32ZBA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    .cfi_offset ra, -4
+; RV32ZBA-NEXT:    .cfi_offset s0, -8
+; RV32ZBA-NEXT:    .cfi_offset s1, -12
+; RV32ZBA-NEXT:    .cfi_offset s2, -16
+; RV32ZBA-NEXT:    .cfi_offset s3, -20
+; RV32ZBA-NEXT:    mv s2, a3
+; RV32ZBA-NEXT:    mv s3, a2
+; RV32ZBA-NEXT:    mv s0, a1
+; RV32ZBA-NEXT:    mv s1, a0
+; RV32ZBA-NEXT:    sw zero, 8(sp)
+; RV32ZBA-NEXT:    addi a4, sp, 8
+; RV32ZBA-NEXT:    call __mulodi4@plt
+; RV32ZBA-NEXT:    lw a0, 8(sp)
+; RV32ZBA-NEXT:    bnez a0, .LBB41_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv s1, s3
+; RV32ZBA-NEXT:    mv s0, s2
+; RV32ZBA-NEXT:  .LBB41_2: # %entry
+; RV32ZBA-NEXT:    mv a0, s1
+; RV32ZBA-NEXT:    mv a1, s0
+; RV32ZBA-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    addi sp, sp, 32
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo.select.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    mulh a2, a0, a1
+; RV64ZBA-NEXT:    mul a3, a0, a1
+; RV64ZBA-NEXT:    srai a3, a3, 63
+; RV64ZBA-NEXT:    bne a2, a3, .LBB41_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB41_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -1277,6 +2255,30 @@ define i1 @smulo.not.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    xor a0, a2, a0
 ; RV64-NEXT:    seqz a0, a0
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo.not.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi sp, sp, -16
+; RV32ZBA-NEXT:    .cfi_def_cfa_offset 16
+; RV32ZBA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    .cfi_offset ra, -4
+; RV32ZBA-NEXT:    sw zero, 8(sp)
+; RV32ZBA-NEXT:    addi a4, sp, 8
+; RV32ZBA-NEXT:    call __mulodi4@plt
+; RV32ZBA-NEXT:    lw a0, 8(sp)
+; RV32ZBA-NEXT:    seqz a0, a0
+; RV32ZBA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    addi sp, sp, 16
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo.not.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    mulh a2, a0, a1
+; RV64ZBA-NEXT:    mul a0, a0, a1
+; RV64ZBA-NEXT:    srai a0, a0, 63
+; RV64ZBA-NEXT:    xor a0, a2, a0
+; RV64ZBA-NEXT:    seqz a0, a0
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -1288,25 +2290,44 @@ define i32 @umulo.select.i32(i32 %v1, i32 %v2) {
 ; RV32-LABEL: umulo.select.i32:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    mulhu a2, a0, a1
-; RV32-NEXT:    bnez a2, .LBB42_2
+; RV32-NEXT:    bnez a2, .LBB43_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a1
-; RV32-NEXT:  .LBB42_2: # %entry
+; RV32-NEXT:  .LBB43_2: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: umulo.select.i32:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    slli a2, a1, 32
-; RV64-NEXT:    srli a2, a2, 32
 ; RV64-NEXT:    slli a3, a0, 32
-; RV64-NEXT:    srli a3, a3, 32
-; RV64-NEXT:    mul a2, a3, a2
+; RV64-NEXT:    mulhu a2, a3, a2
 ; RV64-NEXT:    srli a2, a2, 32
-; RV64-NEXT:    bnez a2, .LBB42_2
+; RV64-NEXT:    bnez a2, .LBB43_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB42_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB43_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo.select.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mulhu a2, a0, a1
+; RV32ZBA-NEXT:    bnez a2, .LBB43_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv a0, a1
+; RV32ZBA-NEXT:  .LBB43_2: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo.select.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    zext.w a2, a1
+; RV64ZBA-NEXT:    zext.w a3, a0
+; RV64ZBA-NEXT:    mul a2, a3, a2
+; RV64ZBA-NEXT:    srli a2, a2, 32
+; RV64ZBA-NEXT:    bnez a2, .LBB43_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB43_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -1324,13 +2345,26 @@ define i1 @umulo.not.i32(i32 %v1, i32 %v2) {
 ; RV64-LABEL: umulo.not.i32:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    slli a1, a1, 32
-; RV64-NEXT:    srli a1, a1, 32
 ; RV64-NEXT:    slli a0, a0, 32
-; RV64-NEXT:    srli a0, a0, 32
-; RV64-NEXT:    mul a0, a0, a1
+; RV64-NEXT:    mulhu a0, a0, a1
 ; RV64-NEXT:    srli a0, a0, 32
 ; RV64-NEXT:    seqz a0, a0
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo.not.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mulhu a0, a0, a1
+; RV32ZBA-NEXT:    seqz a0, a0
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo.not.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    zext.w a1, a1
+; RV64ZBA-NEXT:    zext.w a0, a0
+; RV64ZBA-NEXT:    mul a0, a0, a1
+; RV64ZBA-NEXT:    srli a0, a0, 32
+; RV64ZBA-NEXT:    seqz a0, a0
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
@@ -1357,21 +2391,55 @@ define i64 @umulo.select.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    snez a5, a5
 ; RV32-NEXT:    or a4, a4, a5
 ; RV32-NEXT:    or a4, a4, a6
-; RV32-NEXT:    bnez a4, .LBB44_2
+; RV32-NEXT:    bnez a4, .LBB45_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a2
 ; RV32-NEXT:    mv a1, a3
-; RV32-NEXT:  .LBB44_2: # %entry
+; RV32-NEXT:  .LBB45_2: # %entry
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: umulo.select.i64:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    mulhu a2, a0, a1
-; RV64-NEXT:    bnez a2, .LBB44_2
+; RV64-NEXT:    bnez a2, .LBB45_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB44_2: # %entry
-; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB45_2: # %entry
+; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo.select.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mul a4, a3, a0
+; RV32ZBA-NEXT:    mul a5, a1, a2
+; RV32ZBA-NEXT:    add a4, a5, a4
+; RV32ZBA-NEXT:    mulhu a5, a0, a2
+; RV32ZBA-NEXT:    add a4, a5, a4
+; RV32ZBA-NEXT:    sltu a6, a4, a5
+; RV32ZBA-NEXT:    snez a5, a3
+; RV32ZBA-NEXT:    snez a4, a1
+; RV32ZBA-NEXT:    and a4, a4, a5
+; RV32ZBA-NEXT:    mulhu a5, a1, a2
+; RV32ZBA-NEXT:    snez a5, a5
+; RV32ZBA-NEXT:    or a4, a4, a5
+; RV32ZBA-NEXT:    mulhu a5, a3, a0
+; RV32ZBA-NEXT:    snez a5, a5
+; RV32ZBA-NEXT:    or a4, a4, a5
+; RV32ZBA-NEXT:    or a4, a4, a6
+; RV32ZBA-NEXT:    bnez a4, .LBB45_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    mv a0, a2
+; RV32ZBA-NEXT:    mv a1, a3
+; RV32ZBA-NEXT:  .LBB45_2: # %entry
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo.select.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    mulhu a2, a0, a1
+; RV64ZBA-NEXT:    bnez a2, .LBB45_2
+; RV64ZBA-NEXT:  # %bb.1: # %entry
+; RV64ZBA-NEXT:    mv a0, a1
+; RV64ZBA-NEXT:  .LBB45_2: # %entry
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -1406,6 +2474,33 @@ define i1 @umulo.not.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    mulhu a0, a0, a1
 ; RV64-NEXT:    seqz a0, a0
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo.not.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mul a4, a3, a0
+; RV32ZBA-NEXT:    mul a5, a1, a2
+; RV32ZBA-NEXT:    add a4, a5, a4
+; RV32ZBA-NEXT:    mulhu a5, a0, a2
+; RV32ZBA-NEXT:    add a4, a5, a4
+; RV32ZBA-NEXT:    sltu a6, a4, a5
+; RV32ZBA-NEXT:    snez a5, a3
+; RV32ZBA-NEXT:    snez a4, a1
+; RV32ZBA-NEXT:    and a4, a4, a5
+; RV32ZBA-NEXT:    mulhu a1, a1, a2
+; RV32ZBA-NEXT:    snez a1, a1
+; RV32ZBA-NEXT:    or a1, a4, a1
+; RV32ZBA-NEXT:    mulhu a0, a3, a0
+; RV32ZBA-NEXT:    snez a0, a0
+; RV32ZBA-NEXT:    or a0, a1, a0
+; RV32ZBA-NEXT:    or a0, a0, a6
+; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo.not.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    mulhu a0, a0, a1
+; RV64ZBA-NEXT:    seqz a0, a0
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
@@ -1423,11 +2518,11 @@ define zeroext i1 @saddo.br.i32(i32 %v1, i32 %v2) {
 ; RV32-NEXT:    add a2, a0, a1
 ; RV32-NEXT:    slt a0, a2, a0
 ; RV32-NEXT:    slti a1, a1, 0
-; RV32-NEXT:    beq a1, a0, .LBB46_2
+; RV32-NEXT:    beq a1, a0, .LBB47_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB46_2: # %continue
+; RV32-NEXT:  .LBB47_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
@@ -1437,13 +2532,40 @@ define zeroext i1 @saddo.br.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    sext.w a0, a0
 ; RV64-NEXT:    add a2, a0, a1
 ; RV64-NEXT:    addw a0, a0, a1
-; RV64-NEXT:    beq a0, a2, .LBB46_2
+; RV64-NEXT:    beq a0, a2, .LBB47_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB46_2: # %continue
+; RV64-NEXT:  .LBB47_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo.br.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a2, a0, a1
+; RV32ZBA-NEXT:    slt a0, a2, a0
+; RV32ZBA-NEXT:    slti a1, a1, 0
+; RV32ZBA-NEXT:    beq a1, a0, .LBB47_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB47_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo.br.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a1, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    add a2, a0, a1
+; RV64ZBA-NEXT:    addw a0, a0, a1
+; RV64ZBA-NEXT:    beq a0, a2, .LBB47_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB47_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -1468,11 +2590,11 @@ define zeroext i1 @saddo.br.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    xor a1, a1, a3
 ; RV32-NEXT:    not a1, a1
 ; RV32-NEXT:    and a0, a1, a0
-; RV32-NEXT:    bgez a0, .LBB47_2
+; RV32-NEXT:    bgez a0, .LBB48_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB47_2: # %continue
+; RV32-NEXT:  .LBB48_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
@@ -1481,13 +2603,44 @@ define zeroext i1 @saddo.br.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    add a2, a0, a1
 ; RV64-NEXT:    slt a0, a2, a0
 ; RV64-NEXT:    slti a1, a1, 0
-; RV64-NEXT:    beq a1, a0, .LBB47_2
+; RV64-NEXT:    beq a1, a0, .LBB48_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB47_2: # %continue
+; RV64-NEXT:  .LBB48_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: saddo.br.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a4, a1, a3
+; RV32ZBA-NEXT:    add a2, a0, a2
+; RV32ZBA-NEXT:    sltu a0, a2, a0
+; RV32ZBA-NEXT:    add a0, a4, a0
+; RV32ZBA-NEXT:    xor a0, a1, a0
+; RV32ZBA-NEXT:    xor a1, a1, a3
+; RV32ZBA-NEXT:    not a1, a1
+; RV32ZBA-NEXT:    and a0, a1, a0
+; RV32ZBA-NEXT:    bgez a0, .LBB48_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB48_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: saddo.br.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    add a2, a0, a1
+; RV64ZBA-NEXT:    slt a0, a2, a0
+; RV64ZBA-NEXT:    slti a1, a1, 0
+; RV64ZBA-NEXT:    beq a1, a0, .LBB48_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB48_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -1505,11 +2658,11 @@ define zeroext i1 @uaddo.br.i32(i32 %v1, i32 %v2) {
 ; RV32-LABEL: uaddo.br.i32:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    add a1, a0, a1
-; RV32-NEXT:    bgeu a1, a0, .LBB48_2
+; RV32-NEXT:    bgeu a1, a0, .LBB49_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB48_2: # %continue
+; RV32-NEXT:  .LBB49_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
@@ -1517,13 +2670,36 @@ define zeroext i1 @uaddo.br.i32(i32 %v1, i32 %v2) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    addw a1, a0, a1
 ; RV64-NEXT:    sext.w a0, a0
-; RV64-NEXT:    bgeu a1, a0, .LBB48_2
+; RV64-NEXT:    bgeu a1, a0, .LBB49_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB48_2: # %continue
+; RV64-NEXT:  .LBB49_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: uaddo.br.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a1, a0, a1
+; RV32ZBA-NEXT:    bgeu a1, a0, .LBB49_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB49_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: uaddo.br.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    addw a1, a0, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    bgeu a1, a0, .LBB49_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB49_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -1544,28 +2720,57 @@ define zeroext i1 @uaddo.br.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    add a2, a0, a2
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a2, a3, a0
-; RV32-NEXT:    beq a2, a1, .LBB49_2
+; RV32-NEXT:    beq a2, a1, .LBB50_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    sltu a0, a2, a1
-; RV32-NEXT:  .LBB49_2: # %entry
-; RV32-NEXT:    beqz a0, .LBB49_4
+; RV32-NEXT:  .LBB50_2: # %entry
+; RV32-NEXT:    beqz a0, .LBB50_4
 ; RV32-NEXT:  # %bb.3: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB49_4: # %continue
+; RV32-NEXT:  .LBB50_4: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: uaddo.br.i64:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    add a1, a0, a1
-; RV64-NEXT:    bgeu a1, a0, .LBB49_2
+; RV64-NEXT:    bgeu a1, a0, .LBB50_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB49_2: # %continue
+; RV64-NEXT:  .LBB50_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: uaddo.br.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a3, a1, a3
+; RV32ZBA-NEXT:    add a2, a0, a2
+; RV32ZBA-NEXT:    sltu a0, a2, a0
+; RV32ZBA-NEXT:    add a2, a3, a0
+; RV32ZBA-NEXT:    beq a2, a1, .LBB50_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    sltu a0, a2, a1
+; RV32ZBA-NEXT:  .LBB50_2: # %entry
+; RV32ZBA-NEXT:    beqz a0, .LBB50_4
+; RV32ZBA-NEXT:  # %bb.3: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB50_4: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: uaddo.br.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    add a1, a0, a1
+; RV64ZBA-NEXT:    bgeu a1, a0, .LBB50_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB50_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -1585,11 +2790,11 @@ define zeroext i1 @ssubo.br.i32(i32 %v1, i32 %v2) {
 ; RV32-NEXT:    sgtz a2, a1
 ; RV32-NEXT:    sub a1, a0, a1
 ; RV32-NEXT:    slt a0, a1, a0
-; RV32-NEXT:    beq a2, a0, .LBB50_2
+; RV32-NEXT:    beq a2, a0, .LBB51_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB50_2: # %continue
+; RV32-NEXT:  .LBB51_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
@@ -1599,13 +2804,40 @@ define zeroext i1 @ssubo.br.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    sext.w a0, a0
 ; RV64-NEXT:    sub a2, a0, a1
 ; RV64-NEXT:    subw a0, a0, a1
-; RV64-NEXT:    beq a0, a2, .LBB50_2
+; RV64-NEXT:    beq a0, a2, .LBB51_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB50_2: # %continue
+; RV64-NEXT:  .LBB51_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: ssubo.br.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sgtz a2, a1
+; RV32ZBA-NEXT:    sub a1, a0, a1
+; RV32ZBA-NEXT:    slt a0, a1, a0
+; RV32ZBA-NEXT:    beq a2, a0, .LBB51_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB51_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: ssubo.br.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a1, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    sub a2, a0, a1
+; RV64ZBA-NEXT:    subw a0, a0, a1
+; RV64ZBA-NEXT:    beq a0, a2, .LBB51_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB51_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -1628,11 +2860,11 @@ define zeroext i1 @ssubo.br.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    xor a0, a1, a0
 ; RV32-NEXT:    xor a1, a1, a3
 ; RV32-NEXT:    and a0, a1, a0
-; RV32-NEXT:    bgez a0, .LBB51_2
+; RV32-NEXT:    bgez a0, .LBB52_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB51_2: # %continue
+; RV32-NEXT:  .LBB52_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
@@ -1641,13 +2873,42 @@ define zeroext i1 @ssubo.br.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    sgtz a2, a1
 ; RV64-NEXT:    sub a1, a0, a1
 ; RV64-NEXT:    slt a0, a1, a0
-; RV64-NEXT:    beq a2, a0, .LBB51_2
+; RV64-NEXT:    beq a2, a0, .LBB52_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB51_2: # %continue
+; RV64-NEXT:  .LBB52_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: ssubo.br.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sltu a0, a0, a2
+; RV32ZBA-NEXT:    sub a2, a1, a3
+; RV32ZBA-NEXT:    sub a0, a2, a0
+; RV32ZBA-NEXT:    xor a0, a1, a0
+; RV32ZBA-NEXT:    xor a1, a1, a3
+; RV32ZBA-NEXT:    and a0, a1, a0
+; RV32ZBA-NEXT:    bgez a0, .LBB52_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB52_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: ssubo.br.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sgtz a2, a1
+; RV64ZBA-NEXT:    sub a1, a0, a1
+; RV64ZBA-NEXT:    slt a0, a1, a0
+; RV64ZBA-NEXT:    beq a2, a0, .LBB52_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB52_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -1665,11 +2926,11 @@ define zeroext i1 @usubo.br.i32(i32 %v1, i32 %v2) {
 ; RV32-LABEL: usubo.br.i32:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    sub a1, a0, a1
-; RV32-NEXT:    bgeu a0, a1, .LBB52_2
+; RV32-NEXT:    bgeu a0, a1, .LBB53_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB52_2: # %continue
+; RV32-NEXT:  .LBB53_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
@@ -1677,13 +2938,36 @@ define zeroext i1 @usubo.br.i32(i32 %v1, i32 %v2) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    subw a1, a0, a1
 ; RV64-NEXT:    sext.w a0, a0
-; RV64-NEXT:    bgeu a0, a1, .LBB52_2
+; RV64-NEXT:    bgeu a0, a1, .LBB53_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB52_2: # %continue
+; RV64-NEXT:  .LBB53_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: usubo.br.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sub a1, a0, a1
+; RV32ZBA-NEXT:    bgeu a0, a1, .LBB53_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB53_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: usubo.br.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    subw a1, a0, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    bgeu a0, a1, .LBB53_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB53_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -1703,31 +2987,62 @@ define zeroext i1 @usubo.br.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    sltu a4, a0, a2
 ; RV32-NEXT:    sub a3, a1, a3
 ; RV32-NEXT:    sub a3, a3, a4
-; RV32-NEXT:    beq a3, a1, .LBB53_3
+; RV32-NEXT:    beq a3, a1, .LBB54_3
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    sltu a0, a1, a3
-; RV32-NEXT:    bnez a0, .LBB53_4
-; RV32-NEXT:  .LBB53_2: # %continue
+; RV32-NEXT:    bnez a0, .LBB54_4
+; RV32-NEXT:  .LBB54_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB53_3:
+; RV32-NEXT:  .LBB54_3:
 ; RV32-NEXT:    sub a1, a0, a2
 ; RV32-NEXT:    sltu a0, a0, a1
-; RV32-NEXT:    beqz a0, .LBB53_2
-; RV32-NEXT:  .LBB53_4: # %overflow
+; RV32-NEXT:    beqz a0, .LBB54_2
+; RV32-NEXT:  .LBB54_4: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: usubo.br.i64:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    sub a1, a0, a1
-; RV64-NEXT:    bgeu a0, a1, .LBB53_2
+; RV64-NEXT:    bgeu a0, a1, .LBB54_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB53_2: # %continue
+; RV64-NEXT:  .LBB54_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: usubo.br.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    sltu a4, a0, a2
+; RV32ZBA-NEXT:    sub a3, a1, a3
+; RV32ZBA-NEXT:    sub a3, a3, a4
+; RV32ZBA-NEXT:    beq a3, a1, .LBB54_3
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    sltu a0, a1, a3
+; RV32ZBA-NEXT:    bnez a0, .LBB54_4
+; RV32ZBA-NEXT:  .LBB54_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB54_3:
+; RV32ZBA-NEXT:    sub a1, a0, a2
+; RV32ZBA-NEXT:    sltu a0, a0, a1
+; RV32ZBA-NEXT:    beqz a0, .LBB54_2
+; RV32ZBA-NEXT:  .LBB54_4: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: usubo.br.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sub a1, a0, a1
+; RV64ZBA-NEXT:    bgeu a0, a1, .LBB54_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB54_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -1747,11 +3062,11 @@ define zeroext i1 @smulo.br.i32(i32 %v1, i32 %v2) {
 ; RV32-NEXT:    mulh a2, a0, a1
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    srai a0, a0, 31
-; RV32-NEXT:    beq a2, a0, .LBB54_2
+; RV32-NEXT:    beq a2, a0, .LBB55_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB54_2: # %continue
+; RV32-NEXT:  .LBB55_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
@@ -1761,13 +3076,40 @@ define zeroext i1 @smulo.br.i32(i32 %v1, i32 %v2) {
 ; RV64-NEXT:    sext.w a0, a0
 ; RV64-NEXT:    mul a2, a0, a1
 ; RV64-NEXT:    mulw a0, a0, a1
-; RV64-NEXT:    beq a0, a2, .LBB54_2
+; RV64-NEXT:    beq a0, a2, .LBB55_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB54_2: # %continue
+; RV64-NEXT:  .LBB55_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo.br.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mulh a2, a0, a1
+; RV32ZBA-NEXT:    mul a0, a0, a1
+; RV32ZBA-NEXT:    srai a0, a0, 31
+; RV32ZBA-NEXT:    beq a2, a0, .LBB55_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB55_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo.br.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    sext.w a1, a1
+; RV64ZBA-NEXT:    sext.w a0, a0
+; RV64ZBA-NEXT:    mul a2, a0, a1
+; RV64ZBA-NEXT:    mulw a0, a0, a1
+; RV64ZBA-NEXT:    beq a0, a2, .LBB55_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB55_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -1792,13 +3134,13 @@ define zeroext i1 @smulo.br.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    addi a4, sp, 8
 ; RV32-NEXT:    call __mulodi4@plt
 ; RV32-NEXT:    lw a0, 8(sp)
-; RV32-NEXT:    beqz a0, .LBB55_2
+; RV32-NEXT:    beqz a0, .LBB56_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
-; RV32-NEXT:    j .LBB55_3
-; RV32-NEXT:  .LBB55_2: # %continue
+; RV32-NEXT:    j .LBB56_3
+; RV32-NEXT:  .LBB56_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
-; RV32-NEXT:  .LBB55_3: # %overflow
+; RV32-NEXT:  .LBB56_3: # %overflow
 ; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
@@ -1808,13 +3150,47 @@ define zeroext i1 @smulo.br.i64(i64 %v1, i64 %v2) {
 ; RV64-NEXT:    mulh a2, a0, a1
 ; RV64-NEXT:    mul a0, a0, a1
 ; RV64-NEXT:    srai a0, a0, 63
-; RV64-NEXT:    beq a2, a0, .LBB55_2
+; RV64-NEXT:    beq a2, a0, .LBB56_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB55_2: # %continue
+; RV64-NEXT:  .LBB56_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo.br.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi sp, sp, -16
+; RV32ZBA-NEXT:    .cfi_def_cfa_offset 16
+; RV32ZBA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    .cfi_offset ra, -4
+; RV32ZBA-NEXT:    sw zero, 8(sp)
+; RV32ZBA-NEXT:    addi a4, sp, 8
+; RV32ZBA-NEXT:    call __mulodi4@plt
+; RV32ZBA-NEXT:    lw a0, 8(sp)
+; RV32ZBA-NEXT:    beqz a0, .LBB56_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    j .LBB56_3
+; RV32ZBA-NEXT:  .LBB56_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:  .LBB56_3: # %overflow
+; RV32ZBA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    addi sp, sp, 16
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo.br.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    mulh a2, a0, a1
+; RV64ZBA-NEXT:    mul a0, a0, a1
+; RV64ZBA-NEXT:    srai a0, a0, 63
+; RV64ZBA-NEXT:    beq a2, a0, .LBB56_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB56_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -1841,13 +3217,13 @@ define zeroext i1 @smulo2.br.i64(i64 %v1) {
 ; RV32-NEXT:    addi a4, sp, 8
 ; RV32-NEXT:    call __mulodi4@plt
 ; RV32-NEXT:    lw a0, 8(sp)
-; RV32-NEXT:    beqz a0, .LBB56_2
+; RV32-NEXT:    beqz a0, .LBB57_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
-; RV32-NEXT:    j .LBB56_3
-; RV32-NEXT:  .LBB56_2: # %continue
+; RV32-NEXT:    j .LBB57_3
+; RV32-NEXT:  .LBB57_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
-; RV32-NEXT:  .LBB56_3: # %overflow
+; RV32-NEXT:  .LBB57_3: # %overflow
 ; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
@@ -1858,13 +3234,50 @@ define zeroext i1 @smulo2.br.i64(i64 %v1) {
 ; RV64-NEXT:    mulh a2, a0, a1
 ; RV64-NEXT:    mul a0, a0, a1
 ; RV64-NEXT:    srai a0, a0, 63
-; RV64-NEXT:    beq a2, a0, .LBB56_2
+; RV64-NEXT:    beq a2, a0, .LBB57_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB56_2: # %continue
+; RV64-NEXT:  .LBB57_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: smulo2.br.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    addi sp, sp, -16
+; RV32ZBA-NEXT:    .cfi_def_cfa_offset 16
+; RV32ZBA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZBA-NEXT:    .cfi_offset ra, -4
+; RV32ZBA-NEXT:    sw zero, 8(sp)
+; RV32ZBA-NEXT:    addi a2, zero, -13
+; RV32ZBA-NEXT:    addi a3, zero, -1
+; RV32ZBA-NEXT:    addi a4, sp, 8
+; RV32ZBA-NEXT:    call __mulodi4@plt
+; RV32ZBA-NEXT:    lw a0, 8(sp)
+; RV32ZBA-NEXT:    beqz a0, .LBB57_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    j .LBB57_3
+; RV32ZBA-NEXT:  .LBB57_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:  .LBB57_3: # %overflow
+; RV32ZBA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZBA-NEXT:    addi sp, sp, 16
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: smulo2.br.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    addi a1, zero, -13
+; RV64ZBA-NEXT:    mulh a2, a0, a1
+; RV64ZBA-NEXT:    mul a0, a0, a1
+; RV64ZBA-NEXT:    srai a0, a0, 63
+; RV64ZBA-NEXT:    beq a2, a0, .LBB57_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB57_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 -13)
   %val = extractvalue {i64, i1} %t, 0
@@ -1882,29 +3295,52 @@ define zeroext i1 @umulo.br.i32(i32 %v1, i32 %v2) {
 ; RV32-LABEL: umulo.br.i32:
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    mulhu a0, a0, a1
-; RV32-NEXT:    beqz a0, .LBB57_2
+; RV32-NEXT:    beqz a0, .LBB58_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB57_2: # %continue
+; RV32-NEXT:  .LBB58_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: umulo.br.i32:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    slli a1, a1, 32
-; RV64-NEXT:    srli a1, a1, 32
 ; RV64-NEXT:    slli a0, a0, 32
+; RV64-NEXT:    mulhu a0, a0, a1
 ; RV64-NEXT:    srli a0, a0, 32
-; RV64-NEXT:    mul a0, a0, a1
-; RV64-NEXT:    srli a0, a0, 32
-; RV64-NEXT:    beqz a0, .LBB57_2
+; RV64-NEXT:    beqz a0, .LBB58_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB57_2: # %continue
+; RV64-NEXT:  .LBB58_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo.br.i32:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mulhu a0, a0, a1
+; RV32ZBA-NEXT:    beqz a0, .LBB58_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB58_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo.br.i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    zext.w a1, a1
+; RV64ZBA-NEXT:    zext.w a0, a0
+; RV64ZBA-NEXT:    mul a0, a0, a1
+; RV64ZBA-NEXT:    srli a0, a0, 32
+; RV64ZBA-NEXT:    beqz a0, .LBB58_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB58_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
@@ -1937,24 +3373,61 @@ define zeroext i1 @umulo.br.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    snez a0, a0
 ; RV32-NEXT:    or a0, a1, a0
 ; RV32-NEXT:    or a0, a0, a6
-; RV32-NEXT:    beqz a0, .LBB58_2
+; RV32-NEXT:    beqz a0, .LBB59_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB58_2: # %continue
+; RV32-NEXT:  .LBB59_2: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: umulo.br.i64:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    mulhu a0, a0, a1
-; RV64-NEXT:    beqz a0, .LBB58_2
+; RV64-NEXT:    beqz a0, .LBB59_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB58_2: # %continue
+; RV64-NEXT:  .LBB59_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo.br.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    mul a4, a3, a0
+; RV32ZBA-NEXT:    mul a5, a1, a2
+; RV32ZBA-NEXT:    add a4, a5, a4
+; RV32ZBA-NEXT:    mulhu a5, a0, a2
+; RV32ZBA-NEXT:    add a4, a5, a4
+; RV32ZBA-NEXT:    sltu a6, a4, a5
+; RV32ZBA-NEXT:    snez a5, a3
+; RV32ZBA-NEXT:    snez a4, a1
+; RV32ZBA-NEXT:    and a4, a4, a5
+; RV32ZBA-NEXT:    mulhu a1, a1, a2
+; RV32ZBA-NEXT:    snez a1, a1
+; RV32ZBA-NEXT:    or a1, a4, a1
+; RV32ZBA-NEXT:    mulhu a0, a3, a0
+; RV32ZBA-NEXT:    snez a0, a0
+; RV32ZBA-NEXT:    or a0, a1, a0
+; RV32ZBA-NEXT:    or a0, a0, a6
+; RV32ZBA-NEXT:    beqz a0, .LBB59_2
+; RV32ZBA-NEXT:  # %bb.1: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB59_2: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo.br.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    mulhu a0, a0, a1
+; RV64ZBA-NEXT:    beqz a0, .LBB59_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB59_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
@@ -1975,28 +3448,57 @@ define zeroext i1 @umulo2.br.i64(i64 %v1) {
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a2, a1, a1
 ; RV32-NEXT:    add a2, a2, a0
-; RV32-NEXT:    beq a2, a1, .LBB59_2
+; RV32-NEXT:    beq a2, a1, .LBB60_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    sltu a0, a2, a1
-; RV32-NEXT:  .LBB59_2: # %entry
-; RV32-NEXT:    beqz a0, .LBB59_4
+; RV32-NEXT:  .LBB60_2: # %entry
+; RV32-NEXT:    beqz a0, .LBB60_4
 ; RV32-NEXT:  # %bb.3: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB59_4: # %continue
+; RV32-NEXT:  .LBB60_4: # %continue
 ; RV32-NEXT:    addi a0, zero, 1
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: umulo2.br.i64:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    add a1, a0, a0
-; RV64-NEXT:    bgeu a1, a0, .LBB59_2
+; RV64-NEXT:    bgeu a1, a0, .LBB60_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    mv a0, zero
 ; RV64-NEXT:    ret
-; RV64-NEXT:  .LBB59_2: # %continue
+; RV64-NEXT:  .LBB60_2: # %continue
 ; RV64-NEXT:    addi a0, zero, 1
 ; RV64-NEXT:    ret
+;
+; RV32ZBA-LABEL: umulo2.br.i64:
+; RV32ZBA:       # %bb.0: # %entry
+; RV32ZBA-NEXT:    add a2, a0, a0
+; RV32ZBA-NEXT:    sltu a0, a2, a0
+; RV32ZBA-NEXT:    add a2, a1, a1
+; RV32ZBA-NEXT:    add a2, a2, a0
+; RV32ZBA-NEXT:    beq a2, a1, .LBB60_2
+; RV32ZBA-NEXT:  # %bb.1: # %entry
+; RV32ZBA-NEXT:    sltu a0, a2, a1
+; RV32ZBA-NEXT:  .LBB60_2: # %entry
+; RV32ZBA-NEXT:    beqz a0, .LBB60_4
+; RV32ZBA-NEXT:  # %bb.3: # %overflow
+; RV32ZBA-NEXT:    mv a0, zero
+; RV32ZBA-NEXT:    ret
+; RV32ZBA-NEXT:  .LBB60_4: # %continue
+; RV32ZBA-NEXT:    addi a0, zero, 1
+; RV32ZBA-NEXT:    ret
+;
+; RV64ZBA-LABEL: umulo2.br.i64:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    add a1, a0, a0
+; RV64ZBA-NEXT:    bgeu a1, a0, .LBB60_2
+; RV64ZBA-NEXT:  # %bb.1: # %overflow
+; RV64ZBA-NEXT:    mv a0, zero
+; RV64ZBA-NEXT:    ret
+; RV64ZBA-NEXT:  .LBB60_2: # %continue
+; RV64ZBA-NEXT:    addi a0, zero, 1
+; RV64ZBA-NEXT:    ret
 entry:
   %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 2)
   %val = extractvalue {i64, i1} %t, 0
diff --git a/llvm/test/CodeGen/Thumb/cmp-add-fold.ll b/llvm/test/CodeGen/Thumb/cmp-add-fold.ll
index aa61b0825b0ccb61041d9a9781a594ad032af1a1..4dc0bc70440dd177a758444995685f26c6db3ae0 100644
--- a/llvm/test/CodeGen/Thumb/cmp-add-fold.ll
+++ b/llvm/test/CodeGen/Thumb/cmp-add-fold.ll
@@ -1,33 +1,65 @@
-; RUN: llc -mtriple=thumbv6m-eabi -verify-machineinstrs < %s | FileCheck --check-prefix=CHECK --check-prefix=T1 %s
-; RUN: llc -mtriple=thumbv7m-eabi -verify-machineinstrs < %s | FileCheck --check-prefix=CHECK --check-prefix=T2 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv6m-eabi -verify-machineinstrs < %s | FileCheck --check-prefix=T1 %s
+; RUN: llc -mtriple=thumbv7m-eabi -verify-machineinstrs < %s | FileCheck --check-prefix=T2 %s
 
-; CHECK-LABEL: addri1:
-; T1: adds r0, r0, #3
-; T1-NEXT: b{{eq|ne}}
-; T2: adds r0, #3
-; T2-NOT: cmp
 define i32 @addri1(i32 %a, i32 %b) {
+; T1-LABEL: addri1:
+; T1:       @ %bb.0: @ %entry
+; T1-NEXT:    adds r0, r0, #3
+; T1-NEXT:    beq .LBB0_2
+; T1-NEXT:  @ %bb.1: @ %false
+; T1-NEXT:    movs r0, #5
+; T1-NEXT:    bx lr
+; T1-NEXT:  .LBB0_2: @ %true
+; T1-NEXT:    movs r0, #4
+; T1-NEXT:    bx lr
+;
+; T2-LABEL: addri1:
+; T2:       @ %bb.0: @ %entry
+; T2-NEXT:    adds r0, #3
+; T2-NEXT:    mov.w r0, #5
+; T2-NEXT:    it eq
+; T2-NEXT:    moveq r0, #4
+; T2-NEXT:    bx lr
+entry:
   %c = add i32 %a, 3
   %d = icmp eq i32 %c, 0
   br i1 %d, label %true, label %false
 
 true:
   ret i32 4
+
 false:
   ret i32 5
 }
 
-; CHECK-LABEL: addri2:
-; CHECK: adds r0, #254
-; T1-NEXT: b{{eq|ne}}
-; T2-NOT: cmp
 define i32 @addri2(i32 %a, i32 %b) {
+; T1-LABEL: addri2:
+; T1:       @ %bb.0: @ %entry
+; T1-NEXT:    adds r0, #254
+; T1-NEXT:    beq .LBB1_2
+; T1-NEXT:  @ %bb.1: @ %false
+; T1-NEXT:    movs r0, #5
+; T1-NEXT:    bx lr
+; T1-NEXT:  .LBB1_2: @ %true
+; T1-NEXT:    movs r0, #4
+; T1-NEXT:    bx lr
+;
+; T2-LABEL: addri2:
+; T2:       @ %bb.0: @ %entry
+; T2-NEXT:    adds r0, #254
+; T2-NEXT:    mov.w r0, #5
+; T2-NEXT:    it eq
+; T2-NEXT:    moveq r0, #4
+; T2-NEXT:    bx lr
+entry:
   %c = add i32 %a, 254
   %d = icmp eq i32 %c, 0
   br i1 %d, label %true, label %false
 
 true:
   ret i32 4
+
 false:
   ret i32 5
 }
diff --git a/llvm/test/CodeGen/Thumb/cmp-fold.ll b/llvm/test/CodeGen/Thumb/cmp-fold.ll
index a278e4321d030855c83f2b6f4cc46c1a6f274b04..213dfa30631a81140cc8f79e9e3c0a378ae5f1f4 100644
--- a/llvm/test/CodeGen/Thumb/cmp-fold.ll
+++ b/llvm/test/CodeGen/Thumb/cmp-fold.ll
@@ -1,57 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv6m-eabi -verify-machineinstrs < %s | FileCheck %s
 
-; CHECK-LABEL: subs:
-; CHECK: subs
-; CHECK-NEXT: b{{eq|ne}}
 define i32 @subs(i32 %a, i32 %b) {
+; CHECK-LABEL: subs:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    beq .LBB0_2
+; CHECK-NEXT:  @ %bb.1: @ %false
+; CHECK-NEXT:    movs r0, #5
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:  .LBB0_2: @ %true
+; CHECK-NEXT:    movs r0, #4
+; CHECK-NEXT:    bx lr
+entry:
   %c = sub i32 %a, %b
   %d = icmp eq i32 %c, 0
   br i1 %d, label %true, label %false
 
 true:
   ret i32 4
+
 false:
   ret i32 5
 }
 
-; CHECK-LABEL: addsrr:
-; CHECK: adds
-; CHECK-NEXT: b{{eq|ne}}
 define i32 @addsrr(i32 %a, i32 %b) {
+; CHECK-LABEL: addsrr:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r0, r0, r1
+; CHECK-NEXT:    beq .LBB1_2
+; CHECK-NEXT:  @ %bb.1: @ %false
+; CHECK-NEXT:    movs r0, #5
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:  .LBB1_2: @ %true
+; CHECK-NEXT:    movs r0, #4
+; CHECK-NEXT:    bx lr
+entry:
   %c = add i32 %a, %b
   %d = icmp eq i32 %c, 0
   br i1 %d, label %true, label %false
 
 true:
   ret i32 4
+
 false:
   ret i32 5
 }
 
-; CHECK-LABEL: lslri:
-; CHECK: lsls
-; CHECK-NEXT: b{{eq|ne}}
 define i32 @lslri(i32 %a, i32 %b) {
+; CHECK-LABEL: lslri:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsls r0, r0, #3
+; CHECK-NEXT:    beq .LBB2_2
+; CHECK-NEXT:  @ %bb.1: @ %false
+; CHECK-NEXT:    movs r0, #5
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:  .LBB2_2: @ %true
+; CHECK-NEXT:    movs r0, #4
+; CHECK-NEXT:    bx lr
+entry:
   %c = shl i32 %a, 3
   %d = icmp eq i32 %c, 0
   br i1 %d, label %true, label %false
 
 true:
   ret i32 4
+
 false:
   ret i32 5
 }
 
-; CHECK-LABEL: lslrr:
-; CHECK: lsls
-; CHECK-NEXT: b{{eq|ne}}
 define i32 @lslrr(i32 %a, i32 %b) {
+; CHECK-LABEL: lslrr:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsls r0, r1
+; CHECK-NEXT:    beq .LBB3_2
+; CHECK-NEXT:  @ %bb.1: @ %false
+; CHECK-NEXT:    movs r0, #5
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:  .LBB3_2: @ %true
+; CHECK-NEXT:    movs r0, #4
+; CHECK-NEXT:    bx lr
+entry:
   %c = shl i32 %a, %b
   %d = icmp eq i32 %c, 0
   br i1 %d, label %true, label %false
 
 true:
   ret i32 4
+
 false:
   ret i32 5
 }
diff --git a/llvm/test/CodeGen/Thumb/ispositive.ll b/llvm/test/CodeGen/Thumb/ispositive.ll
index a9b2c139797eb94f30566b378c85eeba66a238ed..950cfac8caf14e82ac8aecb1c46ec3ad57ae163b 100644
--- a/llvm/test/CodeGen/Thumb/ispositive.ll
+++ b/llvm/test/CodeGen/Thumb/ispositive.ll
@@ -1,20 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumb-eabi %s -o - | FileCheck %s
 
 define i32 @test1(i32 %X) {
-entry:
 ; CHECK-LABEL: test1:
-; CHECK: lsrs r0, r0, #31
-        icmp slt i32 %X, 0              ; <i1>:0 [#uses=1]
-        zext i1 %0 to i32               ; <i32>:1 [#uses=1]
-        ret i32 %1
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsrs r0, r0, #31
+; CHECK-NEXT:    bx lr
+entry:
+  icmp slt i32 %X, 0              ; <i1>:0 [#uses=1]
+  zext i1 %0 to i32               ; <i32>:1 [#uses=1]
+  ret i32 %1
 }
 
 define i32 @test2(i32 %X) {
-entry:
 ; CHECK-LABEL: test2:
-; CHECK: lsls r1, r1, #31
-; CHECK-NEXT: adds
-        %tmp1 = sub i32 %X, 2147483648
-        ret i32 %tmp1
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r1, #1
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    adds r0, r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %tmp1 = sub i32 %X, 2147483648
+  ret i32 %tmp1
 }
 
diff --git a/llvm/test/CodeGen/Thumb/ldr_ext.ll b/llvm/test/CodeGen/Thumb/ldr_ext.ll
index 90194aecec97e4ddebe1bbce3ce7af17aec1f738..314c176bd346827695a3a7773c5c6c6772e10509 100644
--- a/llvm/test/CodeGen/Thumb/ldr_ext.ll
+++ b/llvm/test/CodeGen/Thumb/ldr_ext.ll
@@ -26,8 +26,8 @@ define i32 @test3(i8* %t0) nounwind {
 ; V5: lsls
 ; V5: asrs
 
-; V6: ldrb
-; V6: sxtb
+; V6: mov
+; V6: ldrsb
     %tmp.s = load i8, i8* %t0
     %tmp1.s = sext i8 %tmp.s to i32
     ret i32 %tmp1.s
@@ -38,8 +38,8 @@ define i32 @test4(i16* %t0) nounwind {
 ; V5: lsls
 ; V5: asrs
 
-; V6: ldrh
-; V6: sxth
+; V6: mov
+; V6: ldrsh
     %tmp.s = load i16, i16* %t0
     %tmp1.s = sext i16 %tmp.s to i32
     ret i32 %tmp1.s
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-vpt-debug.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-vpt-debug.mir
new file mode 100644
index 0000000000000000000000000000000000000000..8637ab3f18565c157e247d43b2cb56380ce7f042
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-vpt-debug.mir
@@ -0,0 +1,330 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -run-pass=arm-low-overhead-loops %s -o - --verify-machineinstrs | FileCheck %s
+--- |
+  ; ModuleID = 'skip-vpt-debug.ll'
+  source_filename = "skip-vpt-debug.c"
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main-arm-none-eabihf"
+
+  ; Function Attrs: nofree norecurse nounwind optsize
+  define hidden void @arm_max_no_idx_f32(float* nocapture readonly %pSrc, i32 %blockSize, float* nocapture %pResult) local_unnamed_addr #0 !dbg !13 {
+  entry:
+    call void @llvm.dbg.value(metadata float* %pSrc, metadata !24, metadata !DIExpression()), !dbg !29
+    call void @llvm.dbg.value(metadata i32 %blockSize, metadata !25, metadata !DIExpression()), !dbg !29
+    call void @llvm.dbg.value(metadata float* %pResult, metadata !26, metadata !DIExpression()), !dbg !29
+    call void @llvm.dbg.value(metadata float 0x3810000000000000, metadata !27, metadata !DIExpression()), !dbg !29
+    %cmp.not7 = icmp eq i32 %blockSize, 0, !dbg !30
+    br i1 %cmp.not7, label %while.end, label %vector.ph, !dbg !31
+
+  vector.ph:                                        ; preds = %entry
+    %n.rnd.up = add i32 %blockSize, 3, !dbg !31
+    %n.vec = and i32 %n.rnd.up, -4, !dbg !31
+    %0 = add i32 %n.vec, -4, !dbg !31
+    %1 = lshr i32 %0, 2, !dbg !31
+    %2 = add nuw nsw i32 %1, 1, !dbg !31
+    %3 = call i32 @llvm.start.loop.iterations.i32(i32 %2), !dbg !31
+    br label %vector.body, !dbg !31
+
+  vector.body:                                      ; preds = %vector.body, %vector.ph
+    %lsr.iv1 = phi float* [ %scevgep, %vector.body ], [ %pSrc, %vector.ph ]
+    %vec.phi = phi <4 x float> [ <float 0x3810000000000000, float 0x3810000000000000, float 0x3810000000000000, float 0x3810000000000000>, %vector.ph ], [ %10, %vector.body ]
+    %4 = phi i32 [ %3, %vector.ph ], [ %11, %vector.body ]
+    %5 = phi i32 [ %blockSize, %vector.ph ], [ %7, %vector.body ]
+    %lsr.iv12 = bitcast float* %lsr.iv1 to <4 x float>*
+    %6 = call <4 x i1> @llvm.arm.mve.vctp32(i32 %5)
+    %7 = sub i32 %5, 4
+    %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %lsr.iv12, i32 4, <4 x i1> %6, <4 x float> poison), !dbg !32, !tbaa !34
+    %8 = fcmp nnan ninf nsz olt <4 x float> %vec.phi, %wide.masked.load, !dbg !38
+    %9 = and <4 x i1> %6, %8, !dbg !40
+    %10 = select <4 x i1> %9, <4 x float> %wide.masked.load, <4 x float> %vec.phi, !dbg !40
+    %scevgep = getelementptr float, float* %lsr.iv1, i32 4
+    %11 = call i32 @llvm.loop.decrement.reg.i32(i32 %4, i32 1)
+    %12 = icmp ne i32 %11, 0
+    br i1 %12, label %vector.body, label %middle.block, !llvm.loop !41
+
+  middle.block:                                     ; preds = %vector.body
+    %13 = call nnan ninf nsz float @llvm.vector.reduce.fmax.v4f32(<4 x float> %10), !dbg !31
+    br label %while.end, !dbg !45
+
+  while.end:                                        ; preds = %middle.block, %entry
+    %maxValue.0.lcssa = phi float [ 0x3810000000000000, %entry ], [ %13, %middle.block ], !dbg !29
+    store float %maxValue.0.lcssa, float* %pResult, align 4, !dbg !45, !tbaa !34
+    ret void, !dbg !46
+  }
+
+  ; Function Attrs: nofree nosync nounwind readnone speculatable willreturn
+  declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+  ; Function Attrs: nofree nosync nounwind readnone willreturn
+  declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) #2
+
+  ; Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+  declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>) #3
+
+  ; Function Attrs: nofree nosync nounwind readnone willreturn
+  declare float @llvm.vector.reduce.fmax.v4f32(<4 x float>) #2
+
+  ; Function Attrs: noduplicate nofree nosync nounwind willreturn
+  declare i32 @llvm.start.loop.iterations.i32(i32) #4
+
+  ; Function Attrs: noduplicate nofree nosync nounwind willreturn
+  declare i32 @llvm.loop.decrement.reg.i32(i32, i32) #4
+
+  ; Function Attrs: nounwind readnone
+  declare <4 x i1> @llvm.arm.mve.vctp32(i32) #5
+
+  attributes #0 = { nofree norecurse nounwind optsize "denormal-fp-math"="preserve-sign,preserve-sign" "denormal-fp-math-f32"="ieee,ieee" "frame-pointer"="none" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+ras,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp,-aes,-bf16,-cdecp0,-cdecp1,-cdecp2,-cdecp3,-cdecp4,-cdecp5,-cdecp6,-cdecp7,-crc,-crypto,-dotprod,-fp16fml,-hwdiv-arm,-i8mm,-sb,-sha2" }
+  attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
+  attributes #2 = { nofree nosync nounwind readnone willreturn }
+  attributes #3 = { argmemonly nofree nosync nounwind readonly willreturn }
+  attributes #4 = { noduplicate nofree nosync nounwind willreturn }
+  attributes #5 = { nounwind readnone }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4, !5, !6, !7, !8, !9, !10, !11}
+  !llvm.ident = !{!12}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "Component: ARM Compiler 6.17.0.0 (permissive) Tool: armclang [00000000]", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+  !1 = !DIFile(filename: "skip-vpt-debug.c", directory: "/home/vicspe01")
+  !2 = !{}
+  !3 = !{i32 7, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = !{i32 1, !"wchar_size", i32 4}
+  !6 = !{i32 1, !"static_rwdata", i32 1}
+  !7 = !{i32 1, !"enumsize_buildattr", i32 2}
+  !8 = !{i32 1, !"armlib_unavailable", i32 0}
+  !9 = !{i32 1, !"branch-target-enforcement", i32 0}
+  !10 = !{i32 1, !"sign-return-address", i32 0}
+  !11 = !{i32 1, !"sign-return-address-all", i32 0}
+  !12 = !{!"Component: ARM Compiler 6.17.0.0 (permissive) Tool: armclang [00000000]"}
+  !13 = distinct !DISubprogram(name: "arm_max_no_idx_f32", scope: !1, file: !1, line: 5, type: !14, scopeLine: 6, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !23)
+  !14 = !DISubroutineType(types: !15)
+  !15 = !{null, !16, !20, !22}
+  !16 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !17, size: 32)
+  !17 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !18)
+  !18 = !DIDerivedType(tag: DW_TAG_typedef, name: "float32_t", file: !1, line: 1, baseType: !19)
+  !19 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
+  !20 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint32_t", file: !1, line: 2, baseType: !21)
+  !21 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+  !22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !18, size: 32)
+  !23 = !{!24, !25, !26, !27, !28}
+  !24 = !DILocalVariable(name: "pSrc", arg: 1, scope: !13, file: !1, line: 5, type: !16)
+  !25 = !DILocalVariable(name: "blockSize", arg: 2, scope: !13, file: !1, line: 5, type: !20)
+  !26 = !DILocalVariable(name: "pResult", arg: 3, scope: !13, file: !1, line: 6, type: !22)
+  !27 = !DILocalVariable(name: "maxValue", scope: !13, file: !1, line: 7, type: !18)
+  !28 = !DILocalVariable(name: "newVal", scope: !13, file: !1, line: 8, type: !18)
+  !29 = !DILocation(line: 0, scope: !13)
+  !30 = !DILocation(line: 10, column: 20, scope: !13)
+  !31 = !DILocation(line: 10, column: 3, scope: !13)
+  !32 = !DILocation(line: 11, column: 14, scope: !33)
+  !33 = distinct !DILexicalBlock(scope: !13, file: !1, line: 10, column: 26)
+  !34 = !{!35, !35, i64 0}
+  !35 = !{!"float", !36, i64 0}
+  !36 = !{!"omnipotent char", !37, i64 0}
+  !37 = !{!"Simple C/C++ TBAA"}
+  !38 = !DILocation(line: 12, column: 18, scope: !39)
+  !39 = distinct !DILexicalBlock(scope: !33, file: !1, line: 12, column: 9)
+  !40 = !DILocation(line: 12, column: 9, scope: !33)
+  !41 = distinct !{!41, !31, !42, !43, !44}
+  !42 = !DILocation(line: 15, column: 3, scope: !13)
+  !43 = !{!"llvm.loop.mustprogress"}
+  !44 = !{!"llvm.loop.isvectorized", i32 1}
+  !45 = !DILocation(line: 16, column: 12, scope: !13)
+  !46 = !DILocation(line: 17, column: 1, scope: !13)
+
+...
+---
+name:            arm_max_no_idx_f32
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r2', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       8
+  offsetAdjustment: 0
+  maxAlignment:    4
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:       []
+debugValueSubstitutions: []
+constants:
+  - id:              0
+    value:           float 0x3810000000000000
+    alignment:       4
+    isTargetSpecific: false
+machineFunctionInfo: {}
+body:             |
+  ; CHECK-LABEL: name: arm_max_no_idx_f32
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.4(0x30000000), %bb.1(0x50000000)
+  ; CHECK:   liveins: $lr, $r0, $r1, $r2, $r7
+  ; CHECK:   DBG_VALUE $r0, $noreg, !24, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r1, $noreg, !25, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r1, $noreg, !25, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+  ; CHECK:   frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $r7, -8
+  ; CHECK:   DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r0, $noreg, !24, !DIExpression(), debug-location !29
+  ; CHECK:   tCBZ renamable $r1, %bb.4, debug-location !31
+  ; CHECK: bb.1.vector.ph:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $r0, $r1, $r2
+  ; CHECK:   DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r1, $noreg, !25, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r0, $noreg, !24, !DIExpression(), debug-location !29
+  ; CHECK:   renamable $q0 = MVE_VMOVimmi32 1152, 0, $noreg, undef renamable $q0
+  ; CHECK:   $lr = MVE_DLSTP_32 killed renamable $r1, debug-location !31
+  ; CHECK: bb.2.vector.body (align 4):
+  ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+  ; CHECK:   liveins: $lr, $q0, $r0, $r2
+  ; CHECK:   DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+  ; CHECK:   renamable $r0, renamable $q1 = MVE_VLDRWU32_post killed renamable $r0, 16, 0, $noreg, debug-location !32 :: (load 16 from %ir.lsr.iv12, align 4, !tbaa !34)
+  ; CHECK:   DBG_VALUE $r0, $noreg, !24, !DIExpression(DW_OP_LLVM_entry_value, 1), debug-location !29
+  ; CHECK:   MVE_VPTv4f32 8, renamable $q1, renamable $q0, 12, implicit-def $vpr, debug-location !40
+  ; CHECK:   renamable $q0 = MVE_VORR killed renamable $q1, killed renamable $q1, 1, killed renamable $vpr, killed renamable $q0, debug-location !40
+  ; CHECK:   DBG_VALUE $r1, $noreg, !25, !DIExpression(DW_OP_LLVM_entry_value, 1), debug-location !29
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.2
+  ; CHECK: bb.3.middle.block:
+  ; CHECK:   successors: %bb.5(0x80000000)
+  ; CHECK:   liveins: $q0, $r2
+  ; CHECK:   DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+  ; CHECK:   renamable $s4 = nnan ninf nsz VFP_VMAXNMS renamable $s2, renamable $s3, debug-location !31
+  ; CHECK:   renamable $s0 = nnan ninf nsz VFP_VMAXNMS killed renamable $s0, killed renamable $s1, implicit killed $q0, debug-location !31
+  ; CHECK:   renamable $s0 = nnan ninf nsz VFP_VMAXNMS killed renamable $s0, killed renamable $s4, debug-location !31
+  ; CHECK:   tB %bb.5, 14 /* CC::al */, $noreg
+  ; CHECK: bb.4:
+  ; CHECK:   successors: %bb.5(0x80000000)
+  ; CHECK:   liveins: $r2
+  ; CHECK:   DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r1, $noreg, !25, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r0, $noreg, !24, !DIExpression(), debug-location !29
+  ; CHECK:   renamable $s0 = VLDRS %const.0, 0, 14 /* CC::al */, $noreg :: (load 4 from constant-pool)
+  ; CHECK: bb.5.while.end:
+  ; CHECK:   liveins: $r2, $s0
+  ; CHECK:   DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+  ; CHECK:   DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+  ; CHECK:   VSTRS killed renamable $s0, killed renamable $r2, 0, 14 /* CC::al */, $noreg, debug-location !45 :: (store 4 into %ir.pResult, !tbaa !34)
+  ; CHECK:   frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, debug-location !46
+  ; CHECK: bb.6 (align 4):
+  ; CHECK:   CONSTPOOL_ENTRY 0, %const.0, 4
+  bb.0.entry:
+    successors: %bb.4(0x30000000), %bb.1(0x50000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
+
+    DBG_VALUE $r0, $noreg, !24, !DIExpression(), debug-location !29
+    DBG_VALUE $r1, $noreg, !25, !DIExpression(), debug-location !29
+    DBG_VALUE $r1, $noreg, !25, !DIExpression(), debug-location !29
+    DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+    DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+    frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+    DBG_VALUE $r0, $noreg, !24, !DIExpression(), debug-location !29
+    tCBZ renamable $r1, %bb.4, debug-location !31
+
+  bb.1.vector.ph:
+    successors: %bb.2(0x80000000)
+    liveins: $r0, $r1, $r2
+
+    DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+    DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+    DBG_VALUE $r1, $noreg, !25, !DIExpression(), debug-location !29
+    DBG_VALUE $r0, $noreg, !24, !DIExpression(), debug-location !29
+    renamable $r3, dead $cpsr = tADDi3 renamable $r1, 3, 14 /* CC::al */, $noreg, debug-location !31
+    renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg, debug-location !31
+    renamable $r12 = t2SUBri killed renamable $r3, 4, 14 /* CC::al */, $noreg, $noreg, debug-location !31
+    renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
+    renamable $r3 = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg, debug-location !31
+    renamable $q0 = MVE_VMOVimmi32 1152, 0, $noreg, undef renamable $q0
+    renamable $lr = t2DoLoopStartTP killed renamable $r3, renamable $r1, debug-location !31
+
+  bb.2.vector.body (align 4):
+    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $q0, $r0, $r1, $r2
+
+    DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+    DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+    renamable $vpr = MVE_VCTP32 renamable $r1, 0, $noreg
+    MVE_VPST 2, implicit $vpr, debug-location !32
+    renamable $r0, renamable $q1 = MVE_VLDRWU32_post killed renamable $r0, 16, 1, renamable $vpr, debug-location !32 :: (load 16 from %ir.lsr.iv12, align 4, !tbaa !34)
+    DBG_VALUE $r0, $noreg, !24, !DIExpression(DW_OP_LLVM_entry_value, 1), debug-location !29
+    renamable $vpr = MVE_VCMPf32 renamable $q1, renamable $q0, 12, 1, killed renamable $vpr, debug-location !40
+    renamable $q0 = MVE_VORR killed renamable $q1, renamable $q1, 1, killed renamable $vpr, killed renamable $q0, debug-location !40
+    renamable $r1, dead $cpsr = tSUBi8 killed renamable $r1, 4, 14 /* CC::al */, $noreg
+    DBG_VALUE $r1, $noreg, !25, !DIExpression(DW_OP_LLVM_entry_value, 1), debug-location !29
+    renamable $lr = t2LoopEndDec killed renamable $lr, %bb.2, implicit-def dead $cpsr
+    tB %bb.3, 14 /* CC::al */, $noreg
+
+  bb.3.middle.block:
+    successors: %bb.5(0x80000000)
+    liveins: $q0, $r2
+
+    DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+    DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+    renamable $s4 = nnan ninf nsz VFP_VMAXNMS renamable $s2, renamable $s3, debug-location !31
+    renamable $s0 = nnan ninf nsz VFP_VMAXNMS killed renamable $s0, killed renamable $s1, implicit $q0, debug-location !31
+    renamable $s0 = nnan ninf nsz VFP_VMAXNMS killed renamable $s0, killed renamable $s4, debug-location !31
+    tB %bb.5, 14 /* CC::al */, $noreg
+
+  bb.4:
+    successors: %bb.5(0x80000000)
+    liveins: $r2
+
+    DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+    DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+    DBG_VALUE $r1, $noreg, !25, !DIExpression(), debug-location !29
+    DBG_VALUE $r0, $noreg, !24, !DIExpression(), debug-location !29
+    renamable $s0 = VLDRS %const.0, 0, 14 /* CC::al */, $noreg :: (load 4 from constant-pool)
+
+  bb.5.while.end:
+    liveins: $r2, $s0
+
+    DBG_VALUE float 0x3810000000000000, $noreg, !27, !DIExpression(), debug-location !29
+    DBG_VALUE $r2, $noreg, !26, !DIExpression(), debug-location !29
+    VSTRS killed renamable $s0, killed renamable $r2, 0, 14 /* CC::al */, $noreg, debug-location !45 :: (store 4 into %ir.pResult, !tbaa !34)
+    frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, debug-location !46
+
+  bb.6 (align 4):
+    CONSTPOOL_ENTRY 0, %const.0, 4
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
index 52690fc11f333b7ad9b9a0a679bcc300aca963bf..cfa5a34da74b9e757b7f30d97f0f1be11041fb37 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
@@ -322,21 +322,20 @@ define void @callinpreheader(i32* noalias nocapture readonly %pAngle, i32* nocap
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    subs r6, r2, #0
 ; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    beq .LBB3_3
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    cbz r2, .LBB3_3
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.ph
+; CHECK-NEXT:    mov r6, r2
 ; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    dls lr, r6
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:  .LBB3_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r1, [r5], #4
-; CHECK-NEXT:    subs r6, #1
 ; CHECK-NEXT:    add r0, r1
-; CHECK-NEXT:    cbz r6, .LBB3_3
-; CHECK-NEXT:    le .LBB3_2
+; CHECK-NEXT:    le lr, .LBB3_2
 ; CHECK-NEXT:  .LBB3_3: @ %for.cond.cleanup
 ; CHECK-NEXT:    str r0, [r4]
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
diff --git a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
index 6b053b8fd104c2ea6637c3d7fed1c4a06413ae25..dd8c4f110691dc90563276cd18f4fa45b6e50e2b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
@@ -1468,19 +1468,19 @@ define void @arm_biquad_cascade_df2T_f16(%struct.arm_biquad_cascade_df2T_instanc
 ; CHECK-NEXT:    @ Parent Loop BB17_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    ldrh r7, [r1], #4
-; CHECK-NEXT:    vmov r4, s4
+; CHECK-NEXT:    vmov r3, s4
 ; CHECK-NEXT:    vfma.f16 q2, q3, r7
-; CHECK-NEXT:    ldrh r3, [r1, #-2]
+; CHECK-NEXT:    ldrh r4, [r1, #-2]
 ; CHECK-NEXT:    vmov.u16 r7, q2[0]
 ; CHECK-NEXT:    vfma.f16 q2, q4, r7
-; CHECK-NEXT:    vmov.16 q2[3], r4
-; CHECK-NEXT:    vfma.f16 q2, q5, r3
-; CHECK-NEXT:    vmov.u16 r3, q2[1]
-; CHECK-NEXT:    vfma.f16 q2, q6, r3
-; CHECK-NEXT:    strh r3, [r5, #2]
+; CHECK-NEXT:    vins.f16 s9, s4
+; CHECK-NEXT:    vfma.f16 q2, q5, r4
+; CHECK-NEXT:    vmov.u16 r4, q2[1]
+; CHECK-NEXT:    vfma.f16 q2, q6, r4
+; CHECK-NEXT:    strh r4, [r5, #2]
 ; CHECK-NEXT:    vmov.f32 s8, s9
 ; CHECK-NEXT:    strh r7, [r5], #4
-; CHECK-NEXT:    vmov.16 q2[2], r4
+; CHECK-NEXT:    vmov.16 q2[2], r3
 ; CHECK-NEXT:    le lr, .LBB17_5
 ; CHECK-NEXT:  .LBB17_6: @ %while.end
 ; CHECK-NEXT:    @ in Loop: Header=BB17_3 Depth=1
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
index f894f2acd188b724d6d5ecbf2432d9499032aaf9..883a0781f31355a2b15d9be2d64a5193016e0188 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
@@ -76,34 +76,22 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @loads_i16(<8 x i16> *%A, <8 x i16> *%B, <8 x i16> *%C) {
 ; CHECK-LABEL: loads_i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrh.s32 q0, [r1]
-; CHECK-NEXT:    vldrh.s32 q1, [r0]
-; CHECK-NEXT:    vldrh.s32 q2, [r0, #8]
-; CHECK-NEXT:    vadd.i32 q0, q1, q0
-; CHECK-NEXT:    vldrh.u32 q1, [r2]
-; CHECK-NEXT:    vneg.s32 q1, q1
-; CHECK-NEXT:    vshl.s32 q1, q0, q1
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q0[0], r3
-; CHECK-NEXT:    vmov r3, s5
-; CHECK-NEXT:    vmov.16 q0[1], r3
-; CHECK-NEXT:    vmov r3, s6
-; CHECK-NEXT:    vmov.16 q0[2], r3
-; CHECK-NEXT:    vmov r3, s7
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
-; CHECK-NEXT:    vmov.16 q0[3], r3
-; CHECK-NEXT:    vadd.i32 q1, q2, q1
-; CHECK-NEXT:    vldrh.u32 q2, [r2, #8]
-; CHECK-NEXT:    vneg.s32 q2, q2
-; CHECK-NEXT:    vshl.s32 q1, q1, q2
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s5
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s6
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s7
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vmovlb.s16 q1, q0
+; CHECK-NEXT:    vmovlb.s16 q3, q2
+; CHECK-NEXT:    vmovlt.s16 q0, q0
+; CHECK-NEXT:    vmovlt.s16 q2, q2
+; CHECK-NEXT:    vadd.i32 q0, q2, q0
+; CHECK-NEXT:    vldrw.u32 q2, [r2]
+; CHECK-NEXT:    vadd.i32 q1, q3, q1
+; CHECK-NEXT:    vmovlt.u16 q3, q2
+; CHECK-NEXT:    vneg.s32 q3, q3
+; CHECK-NEXT:    vshl.s32 q3, q0, q3
+; CHECK-NEXT:    vmovlb.u16 q0, q2
+; CHECK-NEXT:    vneg.s32 q0, q0
+; CHECK-NEXT:    vshl.s32 q0, q1, q0
+; CHECK-NEXT:    vmovnt.i32 q0, q3
 ; CHECK-NEXT:    bx lr
 entry:
   %a = load <8 x i16>, <8 x i16> *%A, align 4
@@ -121,50 +109,22 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @loads_i8(<16 x i8> *%A, <16 x i8> *%B, <16 x i8> *%C) {
 ; CHECK-LABEL: loads_i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrb.s16 q0, [r1]
-; CHECK-NEXT:    vldrb.s16 q1, [r0]
-; CHECK-NEXT:    vldrb.s16 q2, [r0, #8]
-; CHECK-NEXT:    vadd.i16 q0, q1, q0
-; CHECK-NEXT:    vldrb.u16 q1, [r2]
-; CHECK-NEXT:    vneg.s16 q1, q1
-; CHECK-NEXT:    vshl.s16 q1, q0, q1
-; CHECK-NEXT:    vmov.u16 r3, q1[0]
-; CHECK-NEXT:    vmov.8 q0[0], r3
-; CHECK-NEXT:    vmov.u16 r3, q1[1]
-; CHECK-NEXT:    vmov.8 q0[1], r3
-; CHECK-NEXT:    vmov.u16 r3, q1[2]
-; CHECK-NEXT:    vmov.8 q0[2], r3
-; CHECK-NEXT:    vmov.u16 r3, q1[3]
-; CHECK-NEXT:    vmov.8 q0[3], r3
-; CHECK-NEXT:    vmov.u16 r3, q1[4]
-; CHECK-NEXT:    vmov.8 q0[4], r3
-; CHECK-NEXT:    vmov.u16 r3, q1[5]
-; CHECK-NEXT:    vmov.8 q0[5], r3
-; CHECK-NEXT:    vmov.u16 r3, q1[6]
-; CHECK-NEXT:    vmov.8 q0[6], r3
-; CHECK-NEXT:    vmov.u16 r3, q1[7]
-; CHECK-NEXT:    vldrb.s16 q1, [r1, #8]
-; CHECK-NEXT:    vmov.8 q0[7], r3
-; CHECK-NEXT:    vadd.i16 q1, q2, q1
-; CHECK-NEXT:    vldrb.u16 q2, [r2, #8]
-; CHECK-NEXT:    vneg.s16 q2, q2
-; CHECK-NEXT:    vshl.s16 q1, q1, q2
-; CHECK-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-NEXT:    vmov.8 q0[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[1]
-; CHECK-NEXT:    vmov.8 q0[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.8 q0[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.8 q0[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-NEXT:    vmov.8 q0[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-NEXT:    vmov.8 q0[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov.8 q0[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vmovlb.s8 q1, q0
+; CHECK-NEXT:    vmovlb.s8 q3, q2
+; CHECK-NEXT:    vmovlt.s8 q0, q0
+; CHECK-NEXT:    vmovlt.s8 q2, q2
+; CHECK-NEXT:    vadd.i16 q0, q2, q0
+; CHECK-NEXT:    vldrw.u32 q2, [r2]
+; CHECK-NEXT:    vadd.i16 q1, q3, q1
+; CHECK-NEXT:    vmovlt.u8 q3, q2
+; CHECK-NEXT:    vneg.s16 q3, q3
+; CHECK-NEXT:    vshl.s16 q3, q0, q3
+; CHECK-NEXT:    vmovlb.u8 q0, q2
+; CHECK-NEXT:    vneg.s16 q0, q0
+; CHECK-NEXT:    vshl.s16 q0, q1, q0
+; CHECK-NEXT:    vmovnt.i16 q0, q3
 ; CHECK-NEXT:    bx lr
 entry:
   %a = load <16 x i8>, <16 x i8> *%A, align 4
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
index c14efc566aed08de8737e732a5911219e321a0f8..ed5b86b9ecb5c51c6f5ba2df8eaacfb21a82cf97 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
@@ -96,52 +96,11 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @ext_add_trunc_v8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: ext_add_trunc_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.u16 r1, q1[0]
-; CHECK-NEXT:    vmov q2[2], q2[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.u16 r1, q1[1]
-; CHECK-NEXT:    vmov q2[3], q2[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q0[4]
-; CHECK-NEXT:    vadd.i32 q3, q3, q2
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov r0, s13
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov r0, s14
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s15
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov.u16 r1, q1[4]
-; CHECK-NEXT:    vmov q4[2], q4[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.u16 r1, q1[5]
-; CHECK-NEXT:    vmov q4[3], q4[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vadd.i32 q0, q3, q4
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s2
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s3
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q0, q2
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vrev32.16 q3, q0
+; CHECK-NEXT:    vrev32.16 q2, q1
+; CHECK-NEXT:    vadd.i32 q2, q3, q2
+; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vmovnt.i32 q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
   %sa = sext <8 x i16> %a to <8 x i32>
@@ -154,108 +113,11 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @ext_add_trunc_v16i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: ext_add_trunc_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vmov.u8 r0, q1[0]
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[3]
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[4]
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[5]
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[6]
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[7]
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vadd.i16 q3, q3, q2
-; CHECK-NEXT:    vmov.u16 r0, q3[0]
-; CHECK-NEXT:    vmov.8 q2[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[1]
-; CHECK-NEXT:    vmov.8 q2[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[2]
-; CHECK-NEXT:    vmov.8 q2[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[3]
-; CHECK-NEXT:    vmov.8 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[4]
-; CHECK-NEXT:    vmov.8 q2[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[5]
-; CHECK-NEXT:    vmov.8 q2[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[6]
-; CHECK-NEXT:    vmov.8 q2[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[7]
-; CHECK-NEXT:    vmov.8 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[8]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[8]
-; CHECK-NEXT:    vmov.16 q4[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[9]
-; CHECK-NEXT:    vmov.16 q4[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[10]
-; CHECK-NEXT:    vmov.16 q4[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[11]
-; CHECK-NEXT:    vmov.16 q4[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[12]
-; CHECK-NEXT:    vmov.16 q4[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[13]
-; CHECK-NEXT:    vmov.16 q4[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[14]
-; CHECK-NEXT:    vmov.16 q4[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[15]
-; CHECK-NEXT:    vmov.16 q4[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vadd.i16 q0, q3, q4
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov.8 q2[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.8 q2[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.8 q2[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.8 q2[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov.8 q2[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov.8 q2[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.8 q2[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.8 q2[15], r0
-; CHECK-NEXT:    vmov q0, q2
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vrev16.8 q3, q0
+; CHECK-NEXT:    vrev16.8 q2, q1
+; CHECK-NEXT:    vadd.i16 q2, q3, q2
+; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vmovnt.i16 q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
   %sa = sext <16 x i8> %a to <16 x i16>
@@ -268,95 +130,19 @@ entry:
 define arm_aapcs_vfpcc <16 x i16> @ext_add_trunc_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-LABEL: ext_add_trunc_v16i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    vmov.u16 r0, q2[2]
-; CHECK-NEXT:    vmov.u16 r1, q2[0]
-; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov q0[2], q0[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q2[3]
-; CHECK-NEXT:    vmov.u16 r1, q2[1]
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov.u16 r1, q4[0]
-; CHECK-NEXT:    vmov q5[2], q5[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov.u16 r1, q4[1]
-; CHECK-NEXT:    vmov q5[3], q5[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q4[4]
-; CHECK-NEXT:    vadd.i32 q5, q5, q0
-; CHECK-NEXT:    vmov r0, s20
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vmov r0, s21
-; CHECK-NEXT:    vmov.16 q0[1], r0
-; CHECK-NEXT:    vmov r0, s22
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov r0, s23
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov q5[2], q5[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q2[6]
-; CHECK-NEXT:    vmov.u16 r1, q2[4]
-; CHECK-NEXT:    vmov q6[2], q6[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q2[7]
-; CHECK-NEXT:    vmov.u16 r1, q2[5]
-; CHECK-NEXT:    vmov q6[3], q6[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov.u16 r1, q4[5]
-; CHECK-NEXT:    vmov q5[3], q5[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q3[0]
-; CHECK-NEXT:    vadd.i32 q2, q5, q6
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s9
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s10
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s11
-; CHECK-NEXT:    vmov.16 q0[7], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[2]
-; CHECK-NEXT:    vmov q2[2], q2[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q3[3]
-; CHECK-NEXT:    vmov.u16 r1, q3[1]
-; CHECK-NEXT:    vmov q2[3], q2[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.u16 r1, q1[0]
-; CHECK-NEXT:    vmov q4[2], q4[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.u16 r1, q1[1]
-; CHECK-NEXT:    vmov q4[3], q4[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q1[4]
-; CHECK-NEXT:    vadd.i32 q4, q4, q2
-; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov r0, s17
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov r0, s18
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s19
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov q4[2], q4[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q3[6]
-; CHECK-NEXT:    vmov.u16 r1, q3[4]
-; CHECK-NEXT:    vmov q5[2], q5[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q3[7]
-; CHECK-NEXT:    vmov.u16 r1, q3[5]
-; CHECK-NEXT:    vmov q5[3], q5[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.u16 r1, q1[5]
-; CHECK-NEXT:    vmov q4[3], q4[1], r1, r0
-; CHECK-NEXT:    vadd.i32 q1, q4, q5
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s5
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s6
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s7
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q1, q2
-; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vrev32.16 q5, q0
+; CHECK-NEXT:    vrev32.16 q4, q2
+; CHECK-NEXT:    vadd.i32 q0, q0, q2
+; CHECK-NEXT:    vadd.i32 q4, q5, q4
+; CHECK-NEXT:    vmovnt.i32 q0, q4
+; CHECK-NEXT:    vrev32.16 q4, q1
+; CHECK-NEXT:    vrev32.16 q2, q3
+; CHECK-NEXT:    vadd.i32 q1, q1, q3
+; CHECK-NEXT:    vadd.i32 q2, q4, q2
+; CHECK-NEXT:    vmovnt.i32 q1, q2
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    bx lr
 entry:
   %sa = sext <16 x i16> %a to <16 x i32>
@@ -369,207 +155,19 @@ entry:
 define arm_aapcs_vfpcc <32 x i8> @ext_add_trunc_v32i8(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK-LABEL: ext_add_trunc_v32i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    vmov.u8 r0, q2[0]
-; CHECK-NEXT:    vmov q4, q0
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[1]
-; CHECK-NEXT:    vmov.16 q0[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[2]
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[3]
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[4]
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[5]
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[6]
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[7]
-; CHECK-NEXT:    vmov.16 q0[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[0]
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[1]
-; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[2]
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[3]
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[4]
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[5]
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[6]
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[7]
-; CHECK-NEXT:    vmov.16 q5[7], r0
-; CHECK-NEXT:    vadd.i16 q5, q5, q0
-; CHECK-NEXT:    vmov.u16 r0, q5[0]
-; CHECK-NEXT:    vmov.8 q0[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[1]
-; CHECK-NEXT:    vmov.8 q0[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[2]
-; CHECK-NEXT:    vmov.8 q0[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[3]
-; CHECK-NEXT:    vmov.8 q0[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[4]
-; CHECK-NEXT:    vmov.8 q0[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[5]
-; CHECK-NEXT:    vmov.8 q0[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[6]
-; CHECK-NEXT:    vmov.8 q0[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q5[7]
-; CHECK-NEXT:    vmov.8 q0[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[8]
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[9]
-; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[10]
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[11]
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[12]
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[13]
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[14]
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[8]
-; CHECK-NEXT:    vmov.16 q6[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[9]
-; CHECK-NEXT:    vmov.16 q6[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[10]
-; CHECK-NEXT:    vmov.16 q6[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[11]
-; CHECK-NEXT:    vmov.16 q6[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[12]
-; CHECK-NEXT:    vmov.16 q6[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[13]
-; CHECK-NEXT:    vmov.16 q6[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[14]
-; CHECK-NEXT:    vmov.16 q6[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q2[15]
-; CHECK-NEXT:    vmov.16 q6[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q4[15]
-; CHECK-NEXT:    vmov.16 q5[7], r0
-; CHECK-NEXT:    vadd.i16 q2, q5, q6
-; CHECK-NEXT:    vmov.u16 r0, q2[0]
-; CHECK-NEXT:    vmov.8 q0[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[1]
-; CHECK-NEXT:    vmov.8 q0[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[2]
-; CHECK-NEXT:    vmov.8 q0[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[3]
-; CHECK-NEXT:    vmov.8 q0[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[4]
-; CHECK-NEXT:    vmov.8 q0[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[5]
-; CHECK-NEXT:    vmov.8 q0[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[6]
-; CHECK-NEXT:    vmov.8 q0[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[7]
-; CHECK-NEXT:    vmov.8 q0[15], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[0]
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[1]
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[2]
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[3]
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[4]
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[5]
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[6]
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[7]
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[0]
-; CHECK-NEXT:    vmov.16 q4[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.16 q4[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov.16 q4[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[3]
-; CHECK-NEXT:    vmov.16 q4[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[4]
-; CHECK-NEXT:    vmov.16 q4[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[5]
-; CHECK-NEXT:    vmov.16 q4[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[6]
-; CHECK-NEXT:    vmov.16 q4[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[7]
-; CHECK-NEXT:    vmov.16 q4[7], r0
-; CHECK-NEXT:    vadd.i16 q4, q4, q2
-; CHECK-NEXT:    vmov.u16 r0, q4[0]
-; CHECK-NEXT:    vmov.8 q2[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[1]
-; CHECK-NEXT:    vmov.8 q2[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[2]
-; CHECK-NEXT:    vmov.8 q2[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[3]
-; CHECK-NEXT:    vmov.8 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[4]
-; CHECK-NEXT:    vmov.8 q2[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
-; CHECK-NEXT:    vmov.8 q2[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[6]
-; CHECK-NEXT:    vmov.8 q2[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q4[7]
-; CHECK-NEXT:    vmov.8 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[8]
-; CHECK-NEXT:    vmov.16 q4[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[9]
-; CHECK-NEXT:    vmov.16 q4[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[10]
-; CHECK-NEXT:    vmov.16 q4[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[11]
-; CHECK-NEXT:    vmov.16 q4[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[12]
-; CHECK-NEXT:    vmov.16 q4[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[13]
-; CHECK-NEXT:    vmov.16 q4[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[14]
-; CHECK-NEXT:    vmov.16 q4[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[8]
-; CHECK-NEXT:    vmov.16 q5[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[9]
-; CHECK-NEXT:    vmov.16 q5[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[10]
-; CHECK-NEXT:    vmov.16 q5[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[11]
-; CHECK-NEXT:    vmov.16 q5[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[12]
-; CHECK-NEXT:    vmov.16 q5[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[13]
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[14]
-; CHECK-NEXT:    vmov.16 q5[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q3[15]
-; CHECK-NEXT:    vmov.16 q5[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[15]
-; CHECK-NEXT:    vmov.16 q4[7], r0
-; CHECK-NEXT:    vadd.i16 q1, q4, q5
-; CHECK-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-NEXT:    vmov.8 q2[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[1]
-; CHECK-NEXT:    vmov.8 q2[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.8 q2[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.8 q2[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-NEXT:    vmov.8 q2[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-NEXT:    vmov.8 q2[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov.8 q2[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.8 q2[15], r0
-; CHECK-NEXT:    vmov q1, q2
-; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vrev16.8 q5, q0
+; CHECK-NEXT:    vrev16.8 q4, q2
+; CHECK-NEXT:    vadd.i16 q0, q0, q2
+; CHECK-NEXT:    vadd.i16 q4, q5, q4
+; CHECK-NEXT:    vmovnt.i16 q0, q4
+; CHECK-NEXT:    vrev16.8 q4, q1
+; CHECK-NEXT:    vrev16.8 q2, q3
+; CHECK-NEXT:    vadd.i16 q1, q1, q3
+; CHECK-NEXT:    vadd.i16 q2, q4, q2
+; CHECK-NEXT:    vmovnt.i16 q1, q2
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    bx lr
 entry:
   %sa = sext <32 x i8> %a to <32 x i16>
@@ -640,55 +238,15 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @ext_add_ashr_trunc_i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: ext_add_ashr_trunc_i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.u16 r1, q1[0]
-; CHECK-NEXT:    vmov q2[2], q2[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.u16 r1, q1[1]
-; CHECK-NEXT:    vmov q2[3], q2[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    vmovlb.u16 q2, q2
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q1[4]
-; CHECK-NEXT:    vmovlb.s16 q3, q3
-; CHECK-NEXT:    vadd.i32 q2, q3, q2
-; CHECK-NEXT:    vshr.u32 q3, q2, #1
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov r0, s13
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov r0, s14
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s15
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.u16 r1, q1[5]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.u16 r1, q0[4]
-; CHECK-NEXT:    vmovlb.u16 q1, q3
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmovlb.s16 q0, q3
+; CHECK-NEXT:    vmovlb.u16 q2, q1
+; CHECK-NEXT:    vmovlb.s16 q3, q0
+; CHECK-NEXT:    vmovlt.u16 q1, q1
+; CHECK-NEXT:    vmovlt.s16 q0, q0
 ; CHECK-NEXT:    vadd.i32 q0, q0, q1
-; CHECK-NEXT:    vshr.u32 q0, q0, #1
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s2
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s3
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vadd.i32 q2, q3, q2
+; CHECK-NEXT:    vshr.u32 q1, q0, #1
+; CHECK-NEXT:    vshr.u32 q0, q2, #1
+; CHECK-NEXT:    vmovnt.i32 q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %sa = sext <8 x i16> %a to <8 x i32>
@@ -702,111 +260,15 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @ext_add_ashr_trunc_i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: ext_add_ashr_trunc_i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r0, q1[0]
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[3]
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[4]
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[5]
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[6]
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[7]
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.u8 q2, q2
-; CHECK-NEXT:    vmovlb.s8 q3, q3
-; CHECK-NEXT:    vadd.i16 q2, q3, q2
-; CHECK-NEXT:    vshr.u16 q3, q2, #1
-; CHECK-NEXT:    vmov.u16 r0, q3[0]
-; CHECK-NEXT:    vmov.8 q2[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[1]
-; CHECK-NEXT:    vmov.8 q2[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[2]
-; CHECK-NEXT:    vmov.8 q2[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[3]
-; CHECK-NEXT:    vmov.8 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[4]
-; CHECK-NEXT:    vmov.8 q2[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[5]
-; CHECK-NEXT:    vmov.8 q2[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[6]
-; CHECK-NEXT:    vmov.8 q2[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[7]
-; CHECK-NEXT:    vmov.8 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[8]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[8]
-; CHECK-NEXT:    vmovlb.u8 q1, q3
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.s8 q0, q3
+; CHECK-NEXT:    vmovlb.u8 q2, q1
+; CHECK-NEXT:    vmovlb.s8 q3, q0
+; CHECK-NEXT:    vmovlt.u8 q1, q1
+; CHECK-NEXT:    vmovlt.s8 q0, q0
 ; CHECK-NEXT:    vadd.i16 q0, q0, q1
-; CHECK-NEXT:    vshr.u16 q0, q0, #1
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov.8 q2[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.8 q2[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.8 q2[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.8 q2[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov.8 q2[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov.8 q2[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.8 q2[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.8 q2[15], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vadd.i16 q2, q3, q2
+; CHECK-NEXT:    vshr.u16 q1, q0, #1
+; CHECK-NEXT:    vshr.u16 q0, q2, #1
+; CHECK-NEXT:    vmovnt.i16 q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %sa = sext <16 x i8> %a to <16 x i16>
@@ -1075,70 +537,31 @@ define arm_aapcs_vfpcc <8 x i16> @ext_ops_trunc_i16(<8 x i16> %a, <8 x i16> %b)
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov.u16 r1, q1[4]
-; CHECK-NEXT:    vmov q2[2], q2[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.u16 r1, q1[5]
-; CHECK-NEXT:    vmov q2[3], q2[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.u16 r1, q0[4]
-; CHECK-NEXT:    vmovlb.u16 q2, q2
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    vneg.s32 q5, q2
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmovlb.s16 q3, q3
-; CHECK-NEXT:    vmov.u16 r1, q1[0]
+; CHECK-NEXT:    vmovlt.u16 q2, q1
+; CHECK-NEXT:    vmovlt.s16 q3, q0
 ; CHECK-NEXT:    vadd.i32 q4, q3, q2
-; CHECK-NEXT:    vcmp.i32 eq, q3, q2
+; CHECK-NEXT:    vneg.s32 q5, q2
 ; CHECK-NEXT:    vshl.s32 q4, q4, q5
 ; CHECK-NEXT:    vneg.s32 q5, q3
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.u16 r1, q1[1]
 ; CHECK-NEXT:    vsub.i32 q4, q4, q2
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
+; CHECK-NEXT:    vcmp.i32 eq, q3, q2
 ; CHECK-NEXT:    vmul.i32 q4, q4, q2
-; CHECK-NEXT:    vmovlb.u16 q1, q3
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-NEXT:    vmovlb.u16 q1, q1
 ; CHECK-NEXT:    vshl.u32 q4, q4, q5
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
 ; CHECK-NEXT:    vshl.u32 q4, q4, q2
-; CHECK-NEXT:    vmovlb.s16 q0, q3
-; CHECK-NEXT:    vpsel q2, q4, q2
 ; CHECK-NEXT:    vadd.i32 q3, q0, q1
+; CHECK-NEXT:    vpsel q2, q4, q2
 ; CHECK-NEXT:    vneg.s32 q4, q1
-; CHECK-NEXT:    vcmp.i32 eq, q0, q1
 ; CHECK-NEXT:    vshl.s32 q3, q3, q4
 ; CHECK-NEXT:    vneg.s32 q4, q0
 ; CHECK-NEXT:    vsub.i32 q3, q3, q1
+; CHECK-NEXT:    vcmp.i32 eq, q0, q1
 ; CHECK-NEXT:    vmul.i32 q3, q3, q1
 ; CHECK-NEXT:    vshl.u32 q3, q3, q4
 ; CHECK-NEXT:    vshl.u32 q3, q3, q1
-; CHECK-NEXT:    vpsel q1, q3, q1
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vmov r0, s5
-; CHECK-NEXT:    vmov.16 q0[1], r0
-; CHECK-NEXT:    vmov r0, s6
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    vmov r0, s7
-; CHECK-NEXT:    vmov.16 q0[3], r0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov r0, s9
-; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vmov r0, s10
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    vmov r0, s11
-; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    vpsel q0, q3, q1
+; CHECK-NEXT:    vmovnt.i32 q0, q2
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    bx lr
 entry:
@@ -1161,126 +584,31 @@ define arm_aapcs_vfpcc <16 x i8> @ext_ops_trunc_i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    vmov.u8 r0, q1[8]
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[9]
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[10]
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[11]
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[12]
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[13]
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[14]
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[15]
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[8]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.u8 q2, q2
-; CHECK-NEXT:    vmovlb.s8 q3, q3
-; CHECK-NEXT:    vneg.s16 q5, q2
+; CHECK-NEXT:    vmovlt.u8 q2, q1
+; CHECK-NEXT:    vmovlt.s8 q3, q0
 ; CHECK-NEXT:    vadd.i16 q4, q3, q2
-; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vneg.s16 q5, q2
 ; CHECK-NEXT:    vshl.s16 q4, q4, q5
 ; CHECK-NEXT:    vneg.s16 q5, q3
-; CHECK-NEXT:    vcmp.i16 eq, q3, q2
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
 ; CHECK-NEXT:    vsub.i16 q4, q4, q2
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[7]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmovlb.u8 q1, q3
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vcmp.i16 eq, q3, q2
 ; CHECK-NEXT:    vmul.i16 q4, q4, q2
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmovlb.u8 q1, q1
 ; CHECK-NEXT:    vshl.u16 q4, q4, q5
-; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmovlb.s8 q0, q0
 ; CHECK-NEXT:    vshl.u16 q4, q4, q2
-; CHECK-NEXT:    vmovlb.s8 q0, q3
-; CHECK-NEXT:    vpsel q2, q4, q2
 ; CHECK-NEXT:    vadd.i16 q3, q0, q1
+; CHECK-NEXT:    vpsel q2, q4, q2
 ; CHECK-NEXT:    vneg.s16 q4, q1
-; CHECK-NEXT:    vcmp.i16 eq, q0, q1
 ; CHECK-NEXT:    vshl.s16 q3, q3, q4
 ; CHECK-NEXT:    vneg.s16 q4, q0
 ; CHECK-NEXT:    vsub.i16 q3, q3, q1
+; CHECK-NEXT:    vcmp.i16 eq, q0, q1
 ; CHECK-NEXT:    vmul.i16 q3, q3, q1
 ; CHECK-NEXT:    vshl.u16 q3, q3, q4
 ; CHECK-NEXT:    vshl.u16 q3, q3, q1
-; CHECK-NEXT:    vpsel q1, q3, q1
-; CHECK-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-NEXT:    vmov.8 q0[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[1]
-; CHECK-NEXT:    vmov.8 q0[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.8 q0[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.8 q0[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-NEXT:    vmov.8 q0[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-NEXT:    vmov.8 q0[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov.8 q0[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.8 q0[7], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[0]
-; CHECK-NEXT:    vmov.8 q0[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[1]
-; CHECK-NEXT:    vmov.8 q0[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[2]
-; CHECK-NEXT:    vmov.8 q0[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[3]
-; CHECK-NEXT:    vmov.8 q0[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[4]
-; CHECK-NEXT:    vmov.8 q0[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[5]
-; CHECK-NEXT:    vmov.8 q0[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[6]
-; CHECK-NEXT:    vmov.8 q0[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q2[7]
-; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vpsel q0, q3, q1
+; CHECK-NEXT:    vmovnt.i16 q0, q2
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-load.ll b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
index 9d5e3412946abf0e3cad5c291340932ce0073cfc..02895b0a214ca08ea8ba252095510d145b6cb0f4 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
@@ -1500,8 +1500,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-LE-NEXT:    ldrh r2, [r0, #2]
 ; CHECK-LE-NEXT:    strh.w r2, [sp, #24]
 ; CHECK-LE-NEXT:    vldr.16 s4, [sp, #24]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[1], r2
+; CHECK-LE-NEXT:    vins.f16 s0, s4
 ; CHECK-LE-NEXT:    lsls r2, r1, #29
 ; CHECK-LE-NEXT:    bpl .LBB45_3
 ; CHECK-LE-NEXT:  .LBB45_11: @ %cond.load4
@@ -1516,8 +1515,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-LE-NEXT:    ldrh r2, [r0, #6]
 ; CHECK-LE-NEXT:    strh.w r2, [sp, #16]
 ; CHECK-LE-NEXT:    vldr.16 s4, [sp, #16]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[3], r2
+; CHECK-LE-NEXT:    vins.f16 s1, s4
 ; CHECK-LE-NEXT:    lsls r2, r1, #27
 ; CHECK-LE-NEXT:    bpl .LBB45_5
 ; CHECK-LE-NEXT:  .LBB45_13: @ %cond.load10
@@ -1532,8 +1530,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-LE-NEXT:    ldrh r2, [r0, #10]
 ; CHECK-LE-NEXT:    strh.w r2, [sp, #8]
 ; CHECK-LE-NEXT:    vldr.16 s4, [sp, #8]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[5], r2
+; CHECK-LE-NEXT:    vins.f16 s2, s4
 ; CHECK-LE-NEXT:    lsls r2, r1, #25
 ; CHECK-LE-NEXT:    bpl .LBB45_7
 ; CHECK-LE-NEXT:  .LBB45_15: @ %cond.load16
@@ -1548,8 +1545,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-LE-NEXT:    ldrh r0, [r0, #14]
 ; CHECK-LE-NEXT:    strh.w r0, [sp]
 ; CHECK-LE-NEXT:    vldr.16 s4, [sp]
-; CHECK-LE-NEXT:    vmov r0, s4
-; CHECK-LE-NEXT:    vmov.16 q0[7], r0
+; CHECK-LE-NEXT:    vins.f16 s3, s4
 ; CHECK-LE-NEXT:    add sp, #40
 ; CHECK-LE-NEXT:    bx lr
 ;
@@ -1614,8 +1610,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    ldrh r0, [r0, #14]
 ; CHECK-BE-NEXT:    strh.w r0, [sp]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp]
-; CHECK-BE-NEXT:    vmov r0, s0
-; CHECK-BE-NEXT:    vmov.16 q1[7], r0
+; CHECK-BE-NEXT:    vins.f16 s7, s0
 ; CHECK-BE-NEXT:  .LBB45_9: @ %else20
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
 ; CHECK-BE-NEXT:    add sp, #40
@@ -1630,8 +1625,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #2]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #24]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #24]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[1], r2
+; CHECK-BE-NEXT:    vins.f16 s4, s0
 ; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    bpl .LBB45_3
 ; CHECK-BE-NEXT:  .LBB45_12: @ %cond.load4
@@ -1646,8 +1640,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #6]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #16]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #16]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[3], r2
+; CHECK-BE-NEXT:    vins.f16 s5, s0
 ; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    bpl .LBB45_5
 ; CHECK-BE-NEXT:  .LBB45_14: @ %cond.load10
@@ -1662,8 +1655,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #10]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #8]
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #8]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[5], r2
+; CHECK-BE-NEXT:    vins.f16 s6, s0
 ; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB45_7
 ; CHECK-BE-NEXT:  .LBB45_16: @ %cond.load16
diff --git a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
index f69eeb773a9f0b4e82db1b504678827eb1fbba02..180fc78db665db60a157bc634de899045899203f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
@@ -1186,8 +1186,7 @@ define arm_aapcs_vfpcc void @ssatmul_8_q15(i16* nocapture readonly %pSrcA, i16*
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq .LBB6_8
+; CHECK-NEXT:    cbz r3, .LBB6_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #7
 ; CHECK-NEXT:    bhi .LBB6_3
@@ -1207,16 +1206,13 @@ define arm_aapcs_vfpcc void @ssatmul_8_q15(i16* nocapture readonly %pSrcA, i16*
 ; CHECK-NEXT:    add.w r6, r1, r5, lsl #1
 ; CHECK-NEXT:  .LBB6_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
-; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
-; CHECK-NEXT:    vldrh.s32 q1, [r1], #16
-; CHECK-NEXT:    vqshrnb.s32 q0, q0, #15
-; CHECK-NEXT:    vstrh.32 q0, [r2, #8]
-; CHECK-NEXT:    vldrh.s32 q0, [r0], #16
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
+; CHECK-NEXT:    vmullt.s16 q2, q1, q0
+; CHECK-NEXT:    vmullb.s16 q0, q1, q0
 ; CHECK-NEXT:    vqshrnb.s32 q0, q0, #15
-; CHECK-NEXT:    vstrh.32 q0, [r2], #16
+; CHECK-NEXT:    vqshrnt.s32 q0, q2, #15
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB6_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r5, r3
@@ -1540,14 +1536,12 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 define arm_aapcs_vfpcc void @ssatmul_8t_q15(i16* nocapture readonly %pSrcA, i16* nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %N) {
 ; CHECK-LABEL: ssatmul_8t_q15:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq.w .LBB9_3
+; CHECK-NEXT:    beq .LBB9_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    add.w r12, r3, #7
 ; CHECK-NEXT:    adr r4, .LCPI9_0
@@ -1556,19 +1550,17 @@ define arm_aapcs_vfpcc void @ssatmul_8t_q15(i16* nocapture readonly %pSrcA, i16*
 ; CHECK-NEXT:    sub.w r12, r12, #8
 ; CHECK-NEXT:    vldrw.u32 q0, [r4]
 ; CHECK-NEXT:    adr r4, .LCPI9_1
-; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    vmov.i8 q2, #0x0
 ; CHECK-NEXT:    add.w lr, lr, r12, lsr #3
 ; CHECK-NEXT:    sub.w r12, r3, #1
 ; CHECK-NEXT:    vldrw.u32 q4, [r4]
+; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vdup.32 q1, r12
-; CHECK-NEXT:    vmov.i8 q2, #0x0
 ; CHECK-NEXT:    vmov.i8 q3, #0xff
-; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
 ; CHECK-NEXT:  .LBB9_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vdup.32 q6, r5
-; CHECK-NEXT:    adds r5, #8
+; CHECK-NEXT:    vdup.32 q6, r3
+; CHECK-NEXT:    adds r3, #8
 ; CHECK-NEXT:    vorr q5, q6, q0
 ; CHECK-NEXT:    vorr q6, q6, q4
 ; CHECK-NEXT:    vcmp.u32 cs, q1, q5
@@ -1591,63 +1583,19 @@ define arm_aapcs_vfpcc void @ssatmul_8t_q15(i16* nocapture readonly %pSrcA, i16*
 ; CHECK-NEXT:    vmov.16 q5[6], r4
 ; CHECK-NEXT:    vmov r4, s27
 ; CHECK-NEXT:    vmov.16 q5[7], r4
-; CHECK-NEXT:    vpt.i16 ne, q5, zr
-; CHECK-NEXT:    vldrht.u16 q6, [r0], #16
-; CHECK-NEXT:    vmov.u16 r4, q6[2]
-; CHECK-NEXT:    vmov.u16 r3, q6[0]
-; CHECK-NEXT:    vmov q5[2], q5[0], r3, r4
-; CHECK-NEXT:    vmov.u16 r3, q6[3]
-; CHECK-NEXT:    vmov.u16 r4, q6[1]
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrht.u16 q7, [r1], #16
-; CHECK-NEXT:    vmov q5[3], q5[1], r4, r3
-; CHECK-NEXT:    vmov.u16 r3, q7[2]
-; CHECK-NEXT:    vmov.u16 r4, q7[0]
-; CHECK-NEXT:    vmov q0[2], q0[0], r4, r3
-; CHECK-NEXT:    vmov.u16 r3, q7[3]
-; CHECK-NEXT:    vmov.u16 r4, q7[1]
-; CHECK-NEXT:    vmov q0[3], q0[1], r4, r3
-; CHECK-NEXT:    vmov.u16 r4, q6[4]
-; CHECK-NEXT:    vmullb.s16 q0, q0, q5
-; CHECK-NEXT:    vqshrnb.s32 q0, q0, #15
-; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    vmov.16 q5[0], r3
-; CHECK-NEXT:    vmov r3, s1
-; CHECK-NEXT:    vmov.16 q5[1], r3
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    vmov.16 q5[2], r3
-; CHECK-NEXT:    vmov r3, s3
-; CHECK-NEXT:    vmov.16 q5[3], r3
-; CHECK-NEXT:    vmov.u16 r3, q6[6]
-; CHECK-NEXT:    vmov q0[2], q0[0], r4, r3
-; CHECK-NEXT:    vmov.u16 r3, q6[7]
-; CHECK-NEXT:    vmov.u16 r4, q6[5]
-; CHECK-NEXT:    vmov q0[3], q0[1], r4, r3
-; CHECK-NEXT:    vmov.u16 r3, q7[6]
-; CHECK-NEXT:    vmov.u16 r4, q7[4]
-; CHECK-NEXT:    vmov q6[2], q6[0], r4, r3
-; CHECK-NEXT:    vmov.u16 r3, q7[7]
-; CHECK-NEXT:    vmov.u16 r4, q7[5]
-; CHECK-NEXT:    vmov q6[3], q6[1], r4, r3
-; CHECK-NEXT:    vmullb.s16 q0, q6, q0
-; CHECK-NEXT:    vqshrnb.s32 q0, q0, #15
-; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    vmov.16 q5[4], r3
-; CHECK-NEXT:    vmov r3, s1
-; CHECK-NEXT:    vmov.16 q5[5], r3
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    vmov.16 q5[6], r3
-; CHECK-NEXT:    vmov r3, s3
-; CHECK-NEXT:    vmov.16 q5[7], r3
+; CHECK-NEXT:    vptt.i16 ne, q5, zr
+; CHECK-NEXT:    vldrht.u16 q5, [r0], #16
+; CHECK-NEXT:    vldrht.u16 q6, [r1], #16
+; CHECK-NEXT:    vmullt.s16 q7, q6, q5
+; CHECK-NEXT:    vmullb.s16 q5, q6, q5
+; CHECK-NEXT:    vqshrnb.s32 q5, q5, #15
+; CHECK-NEXT:    vqshrnt.s32 q5, q7, #15
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrht.16 q5, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB9_2
 ; CHECK-NEXT:  .LBB9_3: @ %for.cond.cleanup
-; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.4:
 ; CHECK-NEXT:  .LCPI9_0:
@@ -1989,16 +1937,16 @@ define arm_aapcs_vfpcc void @usatmul_8_q15(i16* nocapture readonly %pSrcA, i16*
 ; CHECK-NEXT:    add.w r6, r1, r5, lsl #1
 ; CHECK-NEXT:  .LBB12_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u32 q0, [r0, #8]
-; CHECK-NEXT:    vldrh.u32 q1, [r1, #8]
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
-; CHECK-NEXT:    vldrh.u32 q1, [r1], #16
-; CHECK-NEXT:    vqshrnb.u32 q0, q0, #15
-; CHECK-NEXT:    vstrh.32 q0, [r2, #8]
-; CHECK-NEXT:    vldrh.u32 q0, [r0], #16
-; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
+; CHECK-NEXT:    vmullt.u16 q2, q1, q0
+; CHECK-NEXT:    vmullb.u16 q0, q1, q0
+; CHECK-NEXT:    vqshrnb.u32 q2, q2, #15
 ; CHECK-NEXT:    vqshrnb.u32 q0, q0, #15
-; CHECK-NEXT:    vstrh.32 q0, [r2], #16
+; CHECK-NEXT:    vmovlb.u16 q2, q2
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    vmovnt.i32 q0, q2
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB12_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r5, r3
@@ -2358,8 +2306,7 @@ define arm_aapcs_vfpcc void @ssatmul_16_q7(i8* nocapture readonly %pSrcA, i8* no
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq .LBB15_8
+; CHECK-NEXT:    cbz r3, .LBB15_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #15
 ; CHECK-NEXT:    bhi .LBB15_3
@@ -2379,16 +2326,13 @@ define arm_aapcs_vfpcc void @ssatmul_16_q7(i8* nocapture readonly %pSrcA, i8* no
 ; CHECK-NEXT:    adds r6, r1, r5
 ; CHECK-NEXT:  .LBB15_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrb.s16 q0, [r0, #8]
-; CHECK-NEXT:    vldrb.s16 q1, [r1, #8]
-; CHECK-NEXT:    vmul.i16 q0, q1, q0
-; CHECK-NEXT:    vldrb.s16 q1, [r1], #16
-; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
-; CHECK-NEXT:    vstrb.16 q0, [r2, #8]
-; CHECK-NEXT:    vldrb.s16 q0, [r0], #16
-; CHECK-NEXT:    vmul.i16 q0, q1, q0
+; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
+; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
+; CHECK-NEXT:    vmullt.s8 q2, q1, q0
+; CHECK-NEXT:    vmullb.s8 q0, q1, q0
 ; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
-; CHECK-NEXT:    vstrb.16 q0, [r2], #16
+; CHECK-NEXT:    vqshrnt.s16 q0, q2, #7
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB15_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r5, r3
@@ -2776,181 +2720,75 @@ define arm_aapcs_vfpcc void @ssatmul_16t_q7(i8* nocapture readonly %pSrcA, i8* n
 ; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
 ; CHECK-NEXT:  .LBB18_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vdup.32 q4, r3
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vdup.32 q0, r3
 ; CHECK-NEXT:    adds r3, #16
-; CHECK-NEXT:    vorr q0, q4, q0
-; CHECK-NEXT:    vcmp.u32 cs, q1, q0
-; CHECK-NEXT:    vpsel q5, q3, q2
-; CHECK-NEXT:    vmov r4, s20
-; CHECK-NEXT:    vmov.16 q0[0], r4
-; CHECK-NEXT:    vmov r4, s21
-; CHECK-NEXT:    vmov.16 q0[1], r4
-; CHECK-NEXT:    vmov r4, s22
-; CHECK-NEXT:    vmov.16 q0[2], r4
-; CHECK-NEXT:    vmov r4, s23
-; CHECK-NEXT:    vldrw.u32 q5, [sp, #16] @ 16-byte Reload
-; CHECK-NEXT:    vmov.16 q0[3], r4
-; CHECK-NEXT:    vorr q5, q4, q5
-; CHECK-NEXT:    vcmp.u32 cs, q1, q5
-; CHECK-NEXT:    vpsel q5, q3, q2
-; CHECK-NEXT:    vmov r4, s20
-; CHECK-NEXT:    vmov.16 q0[4], r4
-; CHECK-NEXT:    vmov r4, s21
-; CHECK-NEXT:    vmov.16 q0[5], r4
-; CHECK-NEXT:    vmov r4, s22
-; CHECK-NEXT:    vmov.16 q0[6], r4
-; CHECK-NEXT:    vmov r4, s23
-; CHECK-NEXT:    vmov.16 q0[7], r4
-; CHECK-NEXT:    vcmp.i16 ne, q0, zr
-; CHECK-NEXT:    vpsel q5, q3, q2
-; CHECK-NEXT:    vmov.u16 r4, q5[0]
-; CHECK-NEXT:    vmov.8 q0[0], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[1]
-; CHECK-NEXT:    vmov.8 q0[1], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[2]
-; CHECK-NEXT:    vmov.8 q0[2], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[3]
-; CHECK-NEXT:    vmov.8 q0[3], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[4]
-; CHECK-NEXT:    vmov.8 q0[4], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[5]
-; CHECK-NEXT:    vmov.8 q0[5], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[6]
-; CHECK-NEXT:    vmov.8 q0[6], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[7]
-; CHECK-NEXT:    vldrw.u32 q5, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vmov.8 q0[7], r4
-; CHECK-NEXT:    vorr q5, q4, q5
-; CHECK-NEXT:    vorr q4, q4, q6
-; CHECK-NEXT:    vcmp.u32 cs, q1, q5
-; CHECK-NEXT:    vpsel q7, q3, q2
+; CHECK-NEXT:    vorr q4, q0, q4
 ; CHECK-NEXT:    vcmp.u32 cs, q1, q4
-; CHECK-NEXT:    vmov r4, s28
 ; CHECK-NEXT:    vpsel q4, q3, q2
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vmov r4, s29
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vmov r4, s30
-; CHECK-NEXT:    vmov.16 q5[2], r4
-; CHECK-NEXT:    vmov r4, s31
-; CHECK-NEXT:    vmov.16 q5[3], r4
 ; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    vmov.16 q5[4], r4
-; CHECK-NEXT:    vmov r4, s17
-; CHECK-NEXT:    vmov.16 q5[5], r4
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    vmov.16 q5[6], r4
-; CHECK-NEXT:    vmov r4, s19
-; CHECK-NEXT:    vmov.16 q5[7], r4
-; CHECK-NEXT:    vcmp.i16 ne, q5, zr
-; CHECK-NEXT:    vpsel q4, q3, q2
-; CHECK-NEXT:    vmov.u16 r4, q4[0]
-; CHECK-NEXT:    vmov.8 q0[8], r4
-; CHECK-NEXT:    vmov.u16 r4, q4[1]
-; CHECK-NEXT:    vmov.8 q0[9], r4
-; CHECK-NEXT:    vmov.u16 r4, q4[2]
-; CHECK-NEXT:    vmov.8 q0[10], r4
-; CHECK-NEXT:    vmov.u16 r4, q4[3]
-; CHECK-NEXT:    vmov.8 q0[11], r4
-; CHECK-NEXT:    vmov.u16 r4, q4[4]
-; CHECK-NEXT:    vmov.8 q0[12], r4
-; CHECK-NEXT:    vmov.u16 r4, q4[5]
-; CHECK-NEXT:    vmov.8 q0[13], r4
-; CHECK-NEXT:    vmov.u16 r4, q4[6]
-; CHECK-NEXT:    vmov.8 q0[14], r4
-; CHECK-NEXT:    vmov.u16 r4, q4[7]
-; CHECK-NEXT:    vmov.8 q0[15], r4
-; CHECK-NEXT:    vpt.i8 ne, q0, zr
-; CHECK-NEXT:    vldrbt.u8 q0, [r0], #16
-; CHECK-NEXT:    vmov.u8 r4, q0[0]
 ; CHECK-NEXT:    vmov.16 q7[0], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[1]
+; CHECK-NEXT:    vmov r4, s17
 ; CHECK-NEXT:    vmov.16 q7[1], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[2]
+; CHECK-NEXT:    vmov r4, s18
 ; CHECK-NEXT:    vmov.16 q7[2], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[3]
+; CHECK-NEXT:    vmov r4, s19
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #16] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.16 q7[3], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[4]
+; CHECK-NEXT:    vorr q4, q0, q4
+; CHECK-NEXT:    vcmp.u32 cs, q1, q4
+; CHECK-NEXT:    vpsel q4, q3, q2
+; CHECK-NEXT:    vmov r4, s16
 ; CHECK-NEXT:    vmov.16 q7[4], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[5]
+; CHECK-NEXT:    vmov r4, s17
 ; CHECK-NEXT:    vmov.16 q7[5], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[6]
+; CHECK-NEXT:    vmov r4, s18
 ; CHECK-NEXT:    vmov.16 q7[6], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[7]
+; CHECK-NEXT:    vmov r4, s19
 ; CHECK-NEXT:    vmov.16 q7[7], r4
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrbt.u8 q4, [r1], #16
-; CHECK-NEXT:    vmov.u8 r4, q4[0]
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[1]
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[2]
-; CHECK-NEXT:    vmov.16 q5[2], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[3]
-; CHECK-NEXT:    vmov.16 q5[3], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[4]
-; CHECK-NEXT:    vmov.16 q5[4], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[5]
-; CHECK-NEXT:    vmov.16 q5[5], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[6]
-; CHECK-NEXT:    vmov.16 q5[6], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[7]
-; CHECK-NEXT:    vmov.16 q5[7], r4
-; CHECK-NEXT:    vmullb.s8 q5, q5, q7
-; CHECK-NEXT:    vqshrnb.s16 q5, q5, #7
-; CHECK-NEXT:    vmovlb.s8 q5, q5
-; CHECK-NEXT:    vmov.u16 r4, q5[0]
+; CHECK-NEXT:    vcmp.i16 ne, q7, zr
+; CHECK-NEXT:    vpsel q4, q3, q2
+; CHECK-NEXT:    vmov.u16 r4, q4[0]
 ; CHECK-NEXT:    vmov.8 q7[0], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[1]
+; CHECK-NEXT:    vmov.u16 r4, q4[1]
 ; CHECK-NEXT:    vmov.8 q7[1], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[2]
+; CHECK-NEXT:    vmov.u16 r4, q4[2]
 ; CHECK-NEXT:    vmov.8 q7[2], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[3]
+; CHECK-NEXT:    vmov.u16 r4, q4[3]
 ; CHECK-NEXT:    vmov.8 q7[3], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[4]
+; CHECK-NEXT:    vmov.u16 r4, q4[4]
 ; CHECK-NEXT:    vmov.8 q7[4], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[5]
+; CHECK-NEXT:    vmov.u16 r4, q4[5]
 ; CHECK-NEXT:    vmov.8 q7[5], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[6]
+; CHECK-NEXT:    vmov.u16 r4, q4[6]
 ; CHECK-NEXT:    vmov.8 q7[6], r4
-; CHECK-NEXT:    vmov.u16 r4, q5[7]
+; CHECK-NEXT:    vmov.u16 r4, q4[7]
+; CHECK-NEXT:    vldrw.u32 q4, [sp] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.8 q7[7], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[8]
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[9]
-; CHECK-NEXT:    vmov.16 q5[1], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[10]
-; CHECK-NEXT:    vmov.16 q5[2], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[11]
-; CHECK-NEXT:    vmov.16 q5[3], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[12]
-; CHECK-NEXT:    vmov.16 q5[4], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[13]
-; CHECK-NEXT:    vmov.16 q5[5], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[14]
-; CHECK-NEXT:    vmov.16 q5[6], r4
-; CHECK-NEXT:    vmov.u8 r4, q0[15]
-; CHECK-NEXT:    vmov.16 q5[7], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[8]
-; CHECK-NEXT:    vmov.16 q0[0], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[9]
-; CHECK-NEXT:    vmov.16 q0[1], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[10]
-; CHECK-NEXT:    vmov.16 q0[2], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[11]
-; CHECK-NEXT:    vmov.16 q0[3], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[12]
-; CHECK-NEXT:    vmov.16 q0[4], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[13]
-; CHECK-NEXT:    vmov.16 q0[5], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[14]
-; CHECK-NEXT:    vmov.16 q0[6], r4
-; CHECK-NEXT:    vmov.u8 r4, q4[15]
-; CHECK-NEXT:    vmov.16 q0[7], r4
-; CHECK-NEXT:    vmullb.s8 q0, q0, q5
-; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
-; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vorr q4, q0, q4
+; CHECK-NEXT:    vorr q0, q0, q6
+; CHECK-NEXT:    vcmp.u32 cs, q1, q4
+; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vcmp.u32 cs, q1, q0
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vmov.16 q4[0], r4
+; CHECK-NEXT:    vmov r4, s21
+; CHECK-NEXT:    vmov.16 q4[1], r4
+; CHECK-NEXT:    vmov r4, s22
+; CHECK-NEXT:    vmov.16 q4[2], r4
+; CHECK-NEXT:    vmov r4, s23
+; CHECK-NEXT:    vmov.16 q4[3], r4
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov.16 q4[4], r4
+; CHECK-NEXT:    vmov r4, s1
+; CHECK-NEXT:    vmov.16 q4[5], r4
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    vmov.16 q4[6], r4
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    vmov.16 q4[7], r4
+; CHECK-NEXT:    vcmp.i16 ne, q4, zr
+; CHECK-NEXT:    vpsel q0, q3, q2
 ; CHECK-NEXT:    vmov.u16 r4, q0[0]
 ; CHECK-NEXT:    vmov.8 q7[8], r4
 ; CHECK-NEXT:    vmov.u16 r4, q0[1]
@@ -2967,8 +2805,15 @@ define arm_aapcs_vfpcc void @ssatmul_16t_q7(i8* nocapture readonly %pSrcA, i8* n
 ; CHECK-NEXT:    vmov.8 q7[14], r4
 ; CHECK-NEXT:    vmov.u16 r4, q0[7]
 ; CHECK-NEXT:    vmov.8 q7[15], r4
+; CHECK-NEXT:    vptt.i8 ne, q7, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r0], #16
+; CHECK-NEXT:    vldrbt.u8 q4, [r1], #16
+; CHECK-NEXT:    vmullt.s8 q5, q4, q0
+; CHECK-NEXT:    vmullb.s8 q0, q4, q0
+; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
+; CHECK-NEXT:    vqshrnt.s16 q0, q5, #7
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrbt.8 q7, [r2], #16
+; CHECK-NEXT:    vstrbt.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB18_2
 ; CHECK-NEXT:  .LBB18_3: @ %for.cond.cleanup
 ; CHECK-NEXT:    add sp, #56
@@ -3404,18 +3249,16 @@ define arm_aapcs_vfpcc void @usatmul_16_q7(i8* nocapture readonly %pSrcA, i8* no
 ; CHECK-NEXT:    adds r6, r1, r5
 ; CHECK-NEXT:  .LBB21_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrb.u16 q0, [r0, #8]
-; CHECK-NEXT:    vldrb.u16 q1, [r1, #8]
-; CHECK-NEXT:    vmul.i16 q0, q1, q0
-; CHECK-NEXT:    vldrb.u16 q1, [r1], #16
-; CHECK-NEXT:    vqshrnb.u16 q0, q0, #7
-; CHECK-NEXT:    vmovlb.u8 q0, q0
-; CHECK-NEXT:    vstrb.16 q0, [r2, #8]
-; CHECK-NEXT:    vldrb.u16 q0, [r0], #16
-; CHECK-NEXT:    vmul.i16 q0, q1, q0
+; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
+; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
+; CHECK-NEXT:    vmullt.u8 q2, q1, q0
+; CHECK-NEXT:    vmullb.u8 q0, q1, q0
+; CHECK-NEXT:    vqshrnb.u16 q2, q2, #7
 ; CHECK-NEXT:    vqshrnb.u16 q0, q0, #7
+; CHECK-NEXT:    vmovlb.u8 q2, q2
 ; CHECK-NEXT:    vmovlb.u8 q0, q0
-; CHECK-NEXT:    vstrb.16 q0, [r2], #16
+; CHECK-NEXT:    vmovnt.i16 q0, q2
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB21_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
 ; CHECK-NEXT:    cmp r5, r3
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index db8f7018ba5512ba810c689350cf07e6e1cae574..415ce651b5caa263ff589231e49eefcaa2c65b10 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -1319,8 +1319,7 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @oneoff21_f16(<8 x half> %src1, <8 x half> %src2) {
 ; CHECK-LABEL: oneoff21_f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vins.f16 s5, s0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-vabdus.ll b/llvm/test/CodeGen/Thumb2/mve-vabdus.ll
index cb509378778828d79ba673303ad436c82302d4b8..55ec20bb60974d19f0b5791373fc06308cf691b2 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vabdus.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vabdus.ll
@@ -4,111 +4,15 @@
 define arm_aapcs_vfpcc <16 x i8> @vabd_s8(<16 x i8> %src1, <16 x i8> %src2) {
 ; CHECK-LABEL: vabd_s8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.u8 r0, q1[0]
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[3]
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[4]
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[5]
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[6]
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[7]
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.s8 q2, q2
-; CHECK-NEXT:    vmovlb.s8 q3, q3
+; CHECK-NEXT:    vmovlt.s8 q2, q1
+; CHECK-NEXT:    vmovlt.s8 q3, q0
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s8 q0, q0
 ; CHECK-NEXT:    vsub.i16 q2, q3, q2
-; CHECK-NEXT:    vabs.s16 q3, q2
-; CHECK-NEXT:    vmov.u16 r0, q3[0]
-; CHECK-NEXT:    vmov.8 q2[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[1]
-; CHECK-NEXT:    vmov.8 q2[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[2]
-; CHECK-NEXT:    vmov.8 q2[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[3]
-; CHECK-NEXT:    vmov.8 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[4]
-; CHECK-NEXT:    vmov.8 q2[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[5]
-; CHECK-NEXT:    vmov.8 q2[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[6]
-; CHECK-NEXT:    vmov.8 q2[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[7]
-; CHECK-NEXT:    vmov.8 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[8]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[8]
-; CHECK-NEXT:    vmovlb.s8 q1, q3
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.s8 q0, q3
 ; CHECK-NEXT:    vsub.i16 q0, q0, q1
+; CHECK-NEXT:    vabs.s16 q2, q2
 ; CHECK-NEXT:    vabs.s16 q0, q0
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov.8 q2[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.8 q2[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.8 q2[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.8 q2[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov.8 q2[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov.8 q2[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.8 q2[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.8 q2[15], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovnt.i16 q0, q2
 ; CHECK-NEXT:    bx lr
   %sextsrc1 = sext <16 x i8> %src1 to <16 x i16>
   %sextsrc2 = sext <16 x i8> %src2 to <16 x i16>
@@ -123,55 +27,15 @@ define arm_aapcs_vfpcc <16 x i8> @vabd_s8(<16 x i8> %src1, <16 x i8> %src2) {
 define arm_aapcs_vfpcc <8 x i16> @vabd_s16(<8 x i16> %src1, <8 x i16> %src2) {
 ; CHECK-LABEL: vabd_s16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.u16 r1, q1[0]
-; CHECK-NEXT:    vmov q2[2], q2[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.u16 r1, q1[1]
-; CHECK-NEXT:    vmov q2[3], q2[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    vmovlb.s16 q2, q2
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q1[4]
-; CHECK-NEXT:    vmovlb.s16 q3, q3
+; CHECK-NEXT:    vmovlt.s16 q2, q1
+; CHECK-NEXT:    vmovlt.s16 q3, q0
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
 ; CHECK-NEXT:    vsub.i32 q2, q3, q2
-; CHECK-NEXT:    vabs.s32 q3, q2
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov r0, s13
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov r0, s14
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s15
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.u16 r1, q1[5]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.u16 r1, q0[4]
-; CHECK-NEXT:    vmovlb.s16 q1, q3
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmovlb.s16 q0, q3
 ; CHECK-NEXT:    vsub.i32 q0, q0, q1
+; CHECK-NEXT:    vabs.s32 q2, q2
 ; CHECK-NEXT:    vabs.s32 q0, q0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s2
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s3
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovnt.i32 q0, q2
 ; CHECK-NEXT:    bx lr
   %sextsrc1 = sext <8 x i16> %src1 to <8 x i32>
   %sextsrc2 = sext <8 x i16> %src2 to <8 x i32>
@@ -240,111 +104,15 @@ define arm_aapcs_vfpcc <4 x i32> @vabd_s32(<4 x i32> %src1, <4 x i32> %src2) {
 define arm_aapcs_vfpcc <16 x i8> @vabd_u8(<16 x i8> %src1, <16 x i8> %src2) {
 ; CHECK-LABEL: vabd_u8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.u8 r0, q1[0]
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[3]
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[4]
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[5]
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[6]
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[7]
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.u8 q2, q2
-; CHECK-NEXT:    vmovlb.u8 q3, q3
+; CHECK-NEXT:    vmovlt.u8 q2, q1
+; CHECK-NEXT:    vmovlt.u8 q3, q0
+; CHECK-NEXT:    vmovlb.u8 q1, q1
+; CHECK-NEXT:    vmovlb.u8 q0, q0
 ; CHECK-NEXT:    vsub.i16 q2, q3, q2
-; CHECK-NEXT:    vabs.s16 q3, q2
-; CHECK-NEXT:    vmov.u16 r0, q3[0]
-; CHECK-NEXT:    vmov.8 q2[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[1]
-; CHECK-NEXT:    vmov.8 q2[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[2]
-; CHECK-NEXT:    vmov.8 q2[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[3]
-; CHECK-NEXT:    vmov.8 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[4]
-; CHECK-NEXT:    vmov.8 q2[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[5]
-; CHECK-NEXT:    vmov.8 q2[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[6]
-; CHECK-NEXT:    vmov.8 q2[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[7]
-; CHECK-NEXT:    vmov.8 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[8]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[8]
-; CHECK-NEXT:    vmovlb.u8 q1, q3
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.u8 q0, q3
 ; CHECK-NEXT:    vsub.i16 q0, q0, q1
+; CHECK-NEXT:    vabs.s16 q2, q2
 ; CHECK-NEXT:    vabs.s16 q0, q0
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov.8 q2[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.8 q2[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.8 q2[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.8 q2[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov.8 q2[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov.8 q2[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.8 q2[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.8 q2[15], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovnt.i16 q0, q2
 ; CHECK-NEXT:    bx lr
   %zextsrc1 = zext <16 x i8> %src1 to <16 x i16>
   %zextsrc2 = zext <16 x i8> %src2 to <16 x i16>
@@ -359,55 +127,15 @@ define arm_aapcs_vfpcc <16 x i8> @vabd_u8(<16 x i8> %src1, <16 x i8> %src2) {
 define arm_aapcs_vfpcc <8 x i16> @vabd_u16(<8 x i16> %src1, <8 x i16> %src2) {
 ; CHECK-LABEL: vabd_u16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.u16 r1, q1[0]
-; CHECK-NEXT:    vmov q2[2], q2[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.u16 r1, q1[1]
-; CHECK-NEXT:    vmov q2[3], q2[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    vmovlb.u16 q2, q2
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q1[4]
-; CHECK-NEXT:    vmovlb.u16 q3, q3
+; CHECK-NEXT:    vmovlt.u16 q2, q1
+; CHECK-NEXT:    vmovlt.u16 q3, q0
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
 ; CHECK-NEXT:    vsub.i32 q2, q3, q2
-; CHECK-NEXT:    vabs.s32 q3, q2
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov r0, s13
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov r0, s14
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s15
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.u16 r1, q1[5]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.u16 r1, q0[4]
-; CHECK-NEXT:    vmovlb.u16 q1, q3
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmovlb.u16 q0, q3
 ; CHECK-NEXT:    vsub.i32 q0, q0, q1
+; CHECK-NEXT:    vabs.s32 q2, q2
 ; CHECK-NEXT:    vabs.s32 q0, q0
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s2
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s3
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovnt.i32 q0, q2
 ; CHECK-NEXT:    bx lr
   %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
   %zextsrc2 = zext <8 x i16> %src2 to <8 x i32>
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
index 1514fdafcfda6edc167b0df1c940079706b2cd12..3b3471476dfad0fdfe321c2525d5c926fa4d190e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
@@ -551,13 +551,13 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.u8 r1, q1[0]
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.u8 r1, q1[2]
 ; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    vmov.u8 r1, q0[3]
+; CHECK-NEXT:    vmov.u8 r2, q0[2]
 ; CHECK-NEXT:    vmov.i64 q2, #0xff
 ; CHECK-NEXT:    vmov q4[2], q4[0], r2, r1
 ; CHECK-NEXT:    vand q3, q3, q2
@@ -566,55 +566,53 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-NEXT:    vmov r1, s18
 ; CHECK-NEXT:    vmov r2, s12
 ; CHECK-NEXT:    vmov r3, s16
-; CHECK-NEXT:    umull r12, r1, r1, r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    umull r0, r1, r1, r0
 ; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    orr.w lr, r3, r1
-; CHECK-NEXT:    vmov.u8 r3, q1[3]
-; CHECK-NEXT:    vmov q3[2], q3[0], r0, r3
-; CHECK-NEXT:    vmov.u8 r3, q0[3]
-; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    vand q3, q3, q2
-; CHECK-NEXT:    vmov q4[2], q4[0], r1, r3
-; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov q3[2], q3[0], r2, r0
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vmov q3[3], q3[1], r3, r1
+; CHECK-NEXT:    vmov.u8 r3, q1[1]
+; CHECK-NEXT:    vmov q4[2], q4[0], r0, r3
+; CHECK-NEXT:    vmov.u8 r3, q0[1]
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
 ; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r3, s12
-; CHECK-NEXT:    vmov r1, s18
-; CHECK-NEXT:    add r2, r12
-; CHECK-NEXT:    vmov r4, s16
-; CHECK-NEXT:    umull r0, r1, r1, r0
-; CHECK-NEXT:    umull r3, r4, r4, r3
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r0
-; CHECK-NEXT:    vmov q3[3], q3[1], r4, r1
-; CHECK-NEXT:    vmov.u8 r4, q0[4]
-; CHECK-NEXT:    vmov r3, s12
-; CHECK-NEXT:    vmov r0, s13
-; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov q5[2], q5[0], r2, r3
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vand q5, q5, q2
+; CHECK-NEXT:    vmov r3, s18
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov r4, s22
+; CHECK-NEXT:    vmov lr, s12
+; CHECK-NEXT:    vmov r12, s13
+; CHECK-NEXT:    umull r0, r2, r2, r0
+; CHECK-NEXT:    smlabb r0, r4, r3, r0
 ; CHECK-NEXT:    vmov r3, s14
-; CHECK-NEXT:    adc.w r0, r0, lr
-; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov.u8 r4, q0[4]
+; CHECK-NEXT:    adds.w r0, r0, lr
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r0, r3
 ; CHECK-NEXT:    vmov.u8 r3, q1[4]
-; CHECK-NEXT:    adc.w r12, r0, r1
-; CHECK-NEXT:    vmov.u8 r1, q1[5]
-; CHECK-NEXT:    vmov q3[2], q3[0], r3, r1
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov.u8 r2, q1[5]
+; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
 ; CHECK-NEXT:    vmov.u8 r3, q0[5]
 ; CHECK-NEXT:    vmov q4[2], q4[0], r4, r3
 ; CHECK-NEXT:    vand q3, q3, q2
 ; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r1, s14
+; CHECK-NEXT:    vmov r2, s14
 ; CHECK-NEXT:    vmov r3, s18
 ; CHECK-NEXT:    vmov r4, s12
 ; CHECK-NEXT:    vmov r0, s16
-; CHECK-NEXT:    umull r1, r3, r3, r1
+; CHECK-NEXT:    umull r2, r3, r3, r2
 ; CHECK-NEXT:    umull r0, r4, r0, r4
-; CHECK-NEXT:    vmov q3[2], q3[0], r0, r1
+; CHECK-NEXT:    vmov q3[2], q3[0], r0, r2
 ; CHECK-NEXT:    vmov q3[3], q3[1], r4, r3
 ; CHECK-NEXT:    vmov.u8 r4, q0[6]
-; CHECK-NEXT:    vmov r1, s12
+; CHECK-NEXT:    vmov r2, s12
 ; CHECK-NEXT:    vmov r0, s13
-; CHECK-NEXT:    adds r1, r1, r2
-; CHECK-NEXT:    vmov r2, s14
-; CHECK-NEXT:    adc.w r0, r0, r12
+; CHECK-NEXT:    adds.w r2, r2, r12
+; CHECK-NEXT:    adcs r0, r1
+; CHECK-NEXT:    vmov r1, s14
 ; CHECK-NEXT:    adds r1, r1, r2
 ; CHECK-NEXT:    vmov.u8 r2, q1[7]
 ; CHECK-NEXT:    adc.w r12, r0, r3
@@ -722,7 +720,7 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-NEXT:    vmov r2, s6
 ; CHECK-NEXT:    vmov r3, s2
 ; CHECK-NEXT:    umlal r0, r1, r3, r2
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, pc}
 entry:
   %xx = zext <16 x i8> %x to <16 x i64>
@@ -1466,58 +1464,56 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vmov.u8 r2, q1[1]
-; CHECK-NEXT:    vmov.u8 r3, q1[0]
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov.u8 r2, q1[3]
+; CHECK-NEXT:    vmov.u8 r3, q1[2]
 ; CHECK-NEXT:    vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT:    vmov.u8 r3, q0[1]
-; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    vmov.u8 r3, q0[3]
+; CHECK-NEXT:    vmov.u8 r2, q0[2]
 ; CHECK-NEXT:    vmov.i64 q2, #0xff
 ; CHECK-NEXT:    vmov q4[2], q4[0], r2, r3
 ; CHECK-NEXT:    vand q3, q3, q2
 ; CHECK-NEXT:    vand q4, q4, q2
 ; CHECK-NEXT:    vmov r12, s14
 ; CHECK-NEXT:    vmov r2, s18
-; CHECK-NEXT:    vmov.u8 r4, q1[2]
-; CHECK-NEXT:    vmov r3, s16
-; CHECK-NEXT:    vmov.u8 r5, q0[2]
-; CHECK-NEXT:    umull r12, lr, r2, r12
-; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    umull r2, r3, r3, r2
-; CHECK-NEXT:    orr.w lr, lr, r3
-; CHECK-NEXT:    vmov.u8 r3, q1[3]
-; CHECK-NEXT:    vmov q3[2], q3[0], r4, r3
-; CHECK-NEXT:    vmov.u8 r4, q0[3]
-; CHECK-NEXT:    vmov q4[2], q4[0], r5, r4
-; CHECK-NEXT:    vand q3, q3, q2
+; CHECK-NEXT:    vmov.u8 r4, q1[0]
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov.u8 r5, q0[0]
+; CHECK-NEXT:    umull lr, r12, r2, r12
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    umull r2, r3, r2, r3
+; CHECK-NEXT:    vmov q3[2], q3[0], r2, lr
+; CHECK-NEXT:    vmov.u8 r2, q1[1]
+; CHECK-NEXT:    vmov q4[2], q4[0], r4, r2
+; CHECK-NEXT:    vmov.u8 r4, q0[1]
+; CHECK-NEXT:    vmov q5[2], q5[0], r5, r4
 ; CHECK-NEXT:    vand q4, q4, q2
-; CHECK-NEXT:    vmov r3, s14
-; CHECK-NEXT:    vmov r4, s18
-; CHECK-NEXT:    add r2, r12
-; CHECK-NEXT:    vmov r5, s12
-; CHECK-NEXT:    vmov r6, s16
-; CHECK-NEXT:    umull r3, r4, r4, r3
-; CHECK-NEXT:    umull r5, r6, r6, r5
-; CHECK-NEXT:    vmov q3[2], q3[0], r5, r3
-; CHECK-NEXT:    vmov.u8 r5, q1[4]
-; CHECK-NEXT:    vmov q3[3], q3[1], r6, r4
-; CHECK-NEXT:    vmov r6, s12
-; CHECK-NEXT:    vmov r3, s13
-; CHECK-NEXT:    adds r2, r2, r6
+; CHECK-NEXT:    vand q5, q5, q2
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vmov q3[3], q3[1], r3, r12
+; CHECK-NEXT:    vmov r5, s18
+; CHECK-NEXT:    vmov r6, s22
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov lr, s13
+; CHECK-NEXT:    umull r2, r4, r4, r2
+; CHECK-NEXT:    smlabb r2, r6, r5, r2
 ; CHECK-NEXT:    vmov r6, s14
-; CHECK-NEXT:    adc.w r3, r3, lr
-; CHECK-NEXT:    adds.w r12, r2, r6
+; CHECK-NEXT:    vmov.u8 r5, q1[4]
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, r4, lr
+; CHECK-NEXT:    vmov.u8 r4, q0[4]
+; CHECK-NEXT:    adds.w lr, r2, r6
 ; CHECK-NEXT:    vmov.u8 r6, q1[5]
 ; CHECK-NEXT:    vmov q3[2], q3[0], r5, r6
-; CHECK-NEXT:    adcs r3, r4
 ; CHECK-NEXT:    vmov.u8 r5, q0[5]
-; CHECK-NEXT:    vmov.u8 r4, q0[4]
 ; CHECK-NEXT:    vmov q4[2], q4[0], r4, r5
 ; CHECK-NEXT:    vand q3, q3, q2
 ; CHECK-NEXT:    vand q4, q4, q2
 ; CHECK-NEXT:    vmov r6, s14
 ; CHECK-NEXT:    vmov r5, s18
+; CHECK-NEXT:    adc.w r3, r3, r12
 ; CHECK-NEXT:    vmov r4, s12
 ; CHECK-NEXT:    vmov r2, s16
 ; CHECK-NEXT:    umull r6, r5, r5, r6
@@ -1527,7 +1523,7 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y
 ; CHECK-NEXT:    vmov.u8 r4, q0[6]
 ; CHECK-NEXT:    vmov r6, s12
 ; CHECK-NEXT:    vmov r2, s13
-; CHECK-NEXT:    adds.w r6, r6, r12
+; CHECK-NEXT:    adds.w r6, r6, lr
 ; CHECK-NEXT:    adcs r2, r3
 ; CHECK-NEXT:    vmov r3, s14
 ; CHECK-NEXT:    adds r3, r3, r6
@@ -1639,7 +1635,7 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y
 ; CHECK-NEXT:    umlal r3, r2, r5, r6
 ; CHECK-NEXT:    adds r0, r0, r3
 ; CHECK-NEXT:    adcs r1, r2
-; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %xx = zext <16 x i8> %x to <16 x i64>
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmulh.ll b/llvm/test/CodeGen/Thumb2/mve-vmulh.ll
index 6f7cef66809b7e8273af46a49ffdcfe5ef75839a..057f3f24d0eaee08d2432ab5bcdcfa67a97f069a 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vmulh.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vmulh.ll
@@ -139,51 +139,11 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @vmulhs_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
 ; CHECK-LABEL: vmulhs_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.u16 r1, q1[0]
-; CHECK-NEXT:    vmov q2[2], q2[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.u16 r1, q1[1]
-; CHECK-NEXT:    vmov q2[3], q2[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q1[4]
-; CHECK-NEXT:    vmullb.s16 q2, q3, q2
-; CHECK-NEXT:    vshr.u32 q3, q2, #16
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov r0, s13
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov r0, s14
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s15
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.u16 r1, q1[5]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.u16 r1, q0[4]
-; CHECK-NEXT:    vmov q1[2], q1[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    vmov q1[3], q1[1], r1, r0
-; CHECK-NEXT:    vmullb.s16 q0, q1, q3
+; CHECK-NEXT:    vmullt.s16 q2, q0, q1
+; CHECK-NEXT:    vmullb.s16 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q2, q2, #16
 ; CHECK-NEXT:    vshr.u32 q0, q0, #16
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s2
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s3
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovnt.i32 q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = sext <8 x i16> %s0 to <8 x i32>
@@ -197,51 +157,11 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @vmulhu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
 ; CHECK-LABEL: vmulhu_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.u16 r1, q1[0]
-; CHECK-NEXT:    vmov q2[2], q2[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.u16 r1, q1[1]
-; CHECK-NEXT:    vmov q2[3], q2[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.u16 r1, q0[0]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.u16 r1, q0[1]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r1, q1[4]
-; CHECK-NEXT:    vmullb.u16 q2, q3, q2
-; CHECK-NEXT:    vshr.u32 q3, q2, #16
-; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov r0, s13
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov r0, s14
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov r0, s15
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.u16 r1, q1[5]
-; CHECK-NEXT:    vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.u16 r1, q0[4]
-; CHECK-NEXT:    vmov q1[2], q1[0], r1, r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.u16 r1, q0[5]
-; CHECK-NEXT:    vmov q1[3], q1[1], r1, r0
-; CHECK-NEXT:    vmullb.u16 q0, q1, q3
+; CHECK-NEXT:    vmullt.u16 q2, q0, q1
+; CHECK-NEXT:    vmullb.u16 q0, q0, q1
+; CHECK-NEXT:    vshr.u32 q2, q2, #16
 ; CHECK-NEXT:    vshr.u32 q0, q0, #16
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov r0, s2
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov r0, s3
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovnt.i32 q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <8 x i16> %s0 to <8 x i32>
@@ -285,107 +205,11 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @vmulhs_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
 ; CHECK-LABEL: vmulhs_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r0, q1[0]
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[3]
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[4]
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[5]
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[6]
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[7]
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmullb.s8 q2, q3, q2
-; CHECK-NEXT:    vshr.u16 q3, q2, #8
-; CHECK-NEXT:    vmov.u16 r0, q3[0]
-; CHECK-NEXT:    vmov.8 q2[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[1]
-; CHECK-NEXT:    vmov.8 q2[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[2]
-; CHECK-NEXT:    vmov.8 q2[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[3]
-; CHECK-NEXT:    vmov.8 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[4]
-; CHECK-NEXT:    vmov.8 q2[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[5]
-; CHECK-NEXT:    vmov.8 q2[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[6]
-; CHECK-NEXT:    vmov.8 q2[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[7]
-; CHECK-NEXT:    vmov.8 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[8]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[8]
-; CHECK-NEXT:    vmov.16 q1[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[9]
-; CHECK-NEXT:    vmov.16 q1[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[10]
-; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[11]
-; CHECK-NEXT:    vmov.16 q1[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[12]
-; CHECK-NEXT:    vmov.16 q1[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[13]
-; CHECK-NEXT:    vmov.16 q1[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[14]
-; CHECK-NEXT:    vmov.16 q1[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[15]
-; CHECK-NEXT:    vmov.16 q1[7], r0
-; CHECK-NEXT:    vmullb.s8 q0, q1, q3
+; CHECK-NEXT:    vmullt.s8 q2, q0, q1
+; CHECK-NEXT:    vmullb.s8 q0, q0, q1
+; CHECK-NEXT:    vshr.u16 q2, q2, #8
 ; CHECK-NEXT:    vshr.u16 q0, q0, #8
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov.8 q2[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.8 q2[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.8 q2[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.8 q2[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov.8 q2[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov.8 q2[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.8 q2[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.8 q2[15], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovnt.i16 q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = sext <16 x i8> %s0 to <16 x i16>
@@ -399,107 +223,11 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @vmulhu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
 ; CHECK-LABEL: vmulhu_v16i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.u8 r0, q1[0]
-; CHECK-NEXT:    vmov.16 q2[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.16 q2[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[2]
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[3]
-; CHECK-NEXT:    vmov.16 q2[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[4]
-; CHECK-NEXT:    vmov.16 q2[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[5]
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[6]
-; CHECK-NEXT:    vmov.16 q2[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[7]
-; CHECK-NEXT:    vmov.16 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmullb.u8 q2, q3, q2
-; CHECK-NEXT:    vshr.u16 q3, q2, #8
-; CHECK-NEXT:    vmov.u16 r0, q3[0]
-; CHECK-NEXT:    vmov.8 q2[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[1]
-; CHECK-NEXT:    vmov.8 q2[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[2]
-; CHECK-NEXT:    vmov.8 q2[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[3]
-; CHECK-NEXT:    vmov.8 q2[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[4]
-; CHECK-NEXT:    vmov.8 q2[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[5]
-; CHECK-NEXT:    vmov.8 q2[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[6]
-; CHECK-NEXT:    vmov.8 q2[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[7]
-; CHECK-NEXT:    vmov.8 q2[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[8]
-; CHECK-NEXT:    vmov.16 q3[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[9]
-; CHECK-NEXT:    vmov.16 q3[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[10]
-; CHECK-NEXT:    vmov.16 q3[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[11]
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[12]
-; CHECK-NEXT:    vmov.16 q3[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[13]
-; CHECK-NEXT:    vmov.16 q3[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[14]
-; CHECK-NEXT:    vmov.16 q3[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q1[15]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[8]
-; CHECK-NEXT:    vmov.16 q1[0], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[9]
-; CHECK-NEXT:    vmov.16 q1[1], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[10]
-; CHECK-NEXT:    vmov.16 q1[2], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[11]
-; CHECK-NEXT:    vmov.16 q1[3], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[12]
-; CHECK-NEXT:    vmov.16 q1[4], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[13]
-; CHECK-NEXT:    vmov.16 q1[5], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[14]
-; CHECK-NEXT:    vmov.16 q1[6], r0
-; CHECK-NEXT:    vmov.u8 r0, q0[15]
-; CHECK-NEXT:    vmov.16 q1[7], r0
-; CHECK-NEXT:    vmullb.u8 q0, q1, q3
+; CHECK-NEXT:    vmullt.u8 q2, q0, q1
+; CHECK-NEXT:    vmullb.u8 q0, q0, q1
+; CHECK-NEXT:    vshr.u16 q2, q2, #8
 ; CHECK-NEXT:    vshr.u16 q0, q0, #8
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov.8 q2[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.8 q2[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.8 q2[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.8 q2[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov.8 q2[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov.8 q2[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.8 q2[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.8 q2[15], r0
-; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovnt.i16 q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
   %s0s = zext <16 x i8> %s0 to <16 x i16>
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst3.ll b/llvm/test/CodeGen/Thumb2/mve-vst3.ll
index c1367ea819a976ff1c82a4a2b5f97f6f658c161d..f569ddb2de91294850592279bd303ca9b7a847b5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vst3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vst3.ll
@@ -1392,63 +1392,61 @@ define void @vst3_v8f16(<8 x half> *%src, <24 x half> *%dst) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14}
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
 ; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
-; CHECK-NEXT:    vmov.f64 d0, d4
+; CHECK-NEXT:    vmov.f64 d0, d6
 ; CHECK-NEXT:    vmovx.f16 s6, s20
-; CHECK-NEXT:    vmovx.f16 s12, s8
-; CHECK-NEXT:    vmovx.f16 s24, s23
-; CHECK-NEXT:    vmov.f32 s4, s9
+; CHECK-NEXT:    vmovx.f16 s8, s12
+; CHECK-NEXT:    vmov.f32 s4, s13
 ; CHECK-NEXT:    vins.f16 s0, s20
 ; CHECK-NEXT:    vmov r2, s6
 ; CHECK-NEXT:    vins.f16 s4, s21
 ; CHECK-NEXT:    vmov.16 q0[4], r2
 ; CHECK-NEXT:    vmov.f32 s3, s4
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
-; CHECK-NEXT:    vmov.f32 s1, s8
+; CHECK-NEXT:    vmov.f32 s1, s12
 ; CHECK-NEXT:    vmov.f32 s17, s4
-; CHECK-NEXT:    vmovx.f16 s26, s6
+; CHECK-NEXT:    vmovx.f16 s24, s7
 ; CHECK-NEXT:    vmov.f32 s18, s4
-; CHECK-NEXT:    vins.f16 s17, s12
-; CHECK-NEXT:    vmovx.f16 s12, s18
-; CHECK-NEXT:    vins.f16 s2, s12
-; CHECK-NEXT:    vmovx.f16 s12, s7
-; CHECK-NEXT:    vins.f16 s24, s12
-; CHECK-NEXT:    vmovx.f16 s12, s22
-; CHECK-NEXT:    vmov r0, s23
-; CHECK-NEXT:    vins.f16 s12, s26
-; CHECK-NEXT:    vmov.16 q3[3], r0
-; CHECK-NEXT:    vrev32.16 q5, q5
-; CHECK-NEXT:    vmov.f32 s15, s24
-; CHECK-NEXT:    vmov.f32 s25, s11
-; CHECK-NEXT:    vmov.f32 s14, s7
-; CHECK-NEXT:    vmovx.f16 s28, s13
-; CHECK-NEXT:    vmov.f32 s26, s11
+; CHECK-NEXT:    vins.f16 s17, s8
+; CHECK-NEXT:    vmovx.f16 s8, s18
+; CHECK-NEXT:    vins.f16 s2, s8
+; CHECK-NEXT:    vmovx.f16 s11, s23
+; CHECK-NEXT:    vins.f16 s11, s24
+; CHECK-NEXT:    vmovx.f16 s24, s6
+; CHECK-NEXT:    vmovx.f16 s8, s22
+; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vins.f16 s8, s24
+; CHECK-NEXT:    vmov.f32 s25, s15
+; CHECK-NEXT:    vins.f16 s9, s23
+; CHECK-NEXT:    vmov.f32 s26, s15
+; CHECK-NEXT:    vmov.f32 s10, s7
+; CHECK-NEXT:    vmovx.f16 s28, s9
 ; CHECK-NEXT:    vins.f16 s25, s28
 ; CHECK-NEXT:    vmovx.f16 s28, s26
-; CHECK-NEXT:    vins.f16 s14, s28
-; CHECK-NEXT:    vmovx.f16 s28, s9
+; CHECK-NEXT:    vins.f16 s10, s28
+; CHECK-NEXT:    vmovx.f16 s28, s13
 ; CHECK-NEXT:    vmov.f32 s4, s5
+; CHECK-NEXT:    vrev32.16 q5, q5
 ; CHECK-NEXT:    vins.f16 s4, s28
-; CHECK-NEXT:    vmovx.f16 s28, s10
+; CHECK-NEXT:    vmovx.f16 s28, s14
 ; CHECK-NEXT:    vins.f16 s6, s28
-; CHECK-NEXT:    vmov.f32 s18, s2
+; CHECK-NEXT:    vmov.f32 s26, s10
 ; CHECK-NEXT:    vmov.f32 s7, s6
-; CHECK-NEXT:    vmov.f32 s6, s10
-; CHECK-NEXT:    vmovx.f16 s8, s5
-; CHECK-NEXT:    vins.f16 s21, s8
-; CHECK-NEXT:    vmovx.f16 s8, s22
-; CHECK-NEXT:    vins.f16 s6, s8
-; CHECK-NEXT:    vmov.f32 s26, s14
-; CHECK-NEXT:    vmov.f32 s22, s6
+; CHECK-NEXT:    vmov.f32 s6, s14
+; CHECK-NEXT:    vmovx.f16 s12, s5
+; CHECK-NEXT:    vins.f16 s21, s12
+; CHECK-NEXT:    vmovx.f16 s12, s22
+; CHECK-NEXT:    vins.f16 s6, s12
 ; CHECK-NEXT:    vmov.f32 s1, s17
-; CHECK-NEXT:    vmov.f32 s13, s25
+; CHECK-NEXT:    vmov.f32 s22, s6
+; CHECK-NEXT:    vmov.f32 s9, s25
 ; CHECK-NEXT:    vmov.f32 s5, s21
 ; CHECK-NEXT:    vmov.f32 s2, s18
+; CHECK-NEXT:    vmov.f32 s10, s26
 ; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    vmov.f32 s14, s26
+; CHECK-NEXT:    vstrw.32 q2, [r1, #32]
 ; CHECK-NEXT:    vmov.f32 s6, s22
-; CHECK-NEXT:    vstrw.32 q3, [r1, #32]
 ; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14}
 ; CHECK-NEXT:    bx lr
@@ -1473,146 +1471,142 @@ define void @vst3_v16f16(<16 x half> *%src, <48 x half> *%dst) {
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    .pad #128
 ; CHECK-NEXT:    sub sp, #128
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
 ; CHECK-NEXT:    vldrw.u32 q7, [r0, #64]
-; CHECK-NEXT:    vldrw.u32 q4, [r0]
-; CHECK-NEXT:    vldrw.u32 q5, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #80]
 ; CHECK-NEXT:    vmovx.f16 s0, s31
-; CHECK-NEXT:    vmovx.f16 s2, s15
-; CHECK-NEXT:    vins.f16 s2, s0
+; CHECK-NEXT:    vmovx.f16 s11, s7
+; CHECK-NEXT:    vins.f16 s11, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s30
-; CHECK-NEXT:    vmovx.f16 s4, s14
-; CHECK-NEXT:    vmov r2, s15
-; CHECK-NEXT:    vins.f16 s4, s0
-; CHECK-NEXT:    vmov q6, q5
-; CHECK-NEXT:    vmov.16 q1[3], r2
-; CHECK-NEXT:    vstrw.32 q3, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vmov.f32 s7, s2
-; CHECK-NEXT:    vmovx.f16 s2, s20
-; CHECK-NEXT:    vmov.f32 s6, s31
-; CHECK-NEXT:    vmovx.f16 s0, s5
-; CHECK-NEXT:    vmov q2, q1
-; CHECK-NEXT:    vmov.f32 s5, s19
-; CHECK-NEXT:    vmov.f32 s6, s19
-; CHECK-NEXT:    vstrw.32 q4, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vmovx.f16 s8, s6
+; CHECK-NEXT:    vmov q4, q1
+; CHECK-NEXT:    vins.f16 s8, s0
+; CHECK-NEXT:    vstrw.32 q4, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vins.f16 s9, s7
+; CHECK-NEXT:    vmov.f32 s10, s31
+; CHECK-NEXT:    vmovx.f16 s0, s9
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vmov.f32 s5, s11
+; CHECK-NEXT:    vmov q6, q2
+; CHECK-NEXT:    vmov.f32 s6, s11
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #48]
 ; CHECK-NEXT:    vins.f16 s5, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s6
 ; CHECK-NEXT:    vstrw.32 q1, [sp, #64] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-NEXT:    vins.f16 s10, s0
-; CHECK-NEXT:    vmov r2, s2
-; CHECK-NEXT:    vstrw.32 q2, [sp, #48] @ 16-byte Spill
-; CHECK-NEXT:    vmov.f64 d4, d2
-; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
-; CHECK-NEXT:    vmovx.f16 s2, s12
-; CHECK-NEXT:    vins.f16 s8, s20
+; CHECK-NEXT:    vins.f16 s14, s0
+; CHECK-NEXT:    vmovx.f16 s2, s8
+; CHECK-NEXT:    vstrw.32 q3, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.f64 d6, d2
+; CHECK-NEXT:    vstrw.32 q1, [sp, #80] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q6, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q2, [sp] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f32 s0, s5
-; CHECK-NEXT:    vins.f16 s0, s21
-; CHECK-NEXT:    vmov.16 q2[4], r2
-; CHECK-NEXT:    vldrw.u32 q5, [r0, #80]
-; CHECK-NEXT:    vmov.f32 s11, s0
-; CHECK-NEXT:    vmov.f32 s9, s4
+; CHECK-NEXT:    vins.f16 s12, s8
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vins.f16 s0, s9
+; CHECK-NEXT:    vmov.16 q3[4], r2
+; CHECK-NEXT:    vmovx.f16 s2, s16
+; CHECK-NEXT:    vmov.f32 s15, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s4
+; CHECK-NEXT:    vmov.f32 s13, s4
 ; CHECK-NEXT:    vmov.f32 s5, s20
 ; CHECK-NEXT:    vmov.f32 s6, s20
 ; CHECK-NEXT:    vins.f16 s5, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s6
 ; CHECK-NEXT:    vstrw.32 q1, [sp, #112] @ 16-byte Spill
-; CHECK-NEXT:    vmov.f64 d2, d8
-; CHECK-NEXT:    vins.f16 s10, s0
-; CHECK-NEXT:    vstrw.32 q2, [sp, #96] @ 16-byte Spill
-; CHECK-NEXT:    vmov q2, q6
-; CHECK-NEXT:    vmovx.f16 s24, s10
-; CHECK-NEXT:    vmov.f32 s0, s17
-; CHECK-NEXT:    vins.f16 s4, s12
+; CHECK-NEXT:    vmov q1, q6
+; CHECK-NEXT:    vins.f16 s14, s0
+; CHECK-NEXT:    vmov.f32 s0, s5
+; CHECK-NEXT:    vins.f16 s24, s16
 ; CHECK-NEXT:    vmov r0, s2
-; CHECK-NEXT:    vins.f16 s0, s13
-; CHECK-NEXT:    vmov.16 q1[4], r0
-; CHECK-NEXT:    vmov.f32 s13, s28
-; CHECK-NEXT:    vmov.f32 s7, s0
-; CHECK-NEXT:    vmovx.f16 s0, s16
-; CHECK-NEXT:    vmov.f32 s14, s28
-; CHECK-NEXT:    vmovx.f16 s2, s11
-; CHECK-NEXT:    vins.f16 s13, s0
-; CHECK-NEXT:    vmov.f32 s5, s16
-; CHECK-NEXT:    vmovx.f16 s0, s14
-; CHECK-NEXT:    vmov r0, s11
-; CHECK-NEXT:    vins.f16 s6, s0
+; CHECK-NEXT:    vins.f16 s0, s17
+; CHECK-NEXT:    vmov.16 q6[4], r0
+; CHECK-NEXT:    vmov.f32 s27, s0
+; CHECK-NEXT:    vmovx.f16 s0, s4
+; CHECK-NEXT:    vmov.f32 s25, s4
+; CHECK-NEXT:    vmov.f32 s5, s28
+; CHECK-NEXT:    vmov.f32 s6, s28
+; CHECK-NEXT:    vins.f16 s5, s0
+; CHECK-NEXT:    vmovx.f16 s0, s6
+; CHECK-NEXT:    vstrw.32 q1, [sp, #96] @ 16-byte Spill
+; CHECK-NEXT:    vins.f16 s26, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s22
-; CHECK-NEXT:    vins.f16 s24, s0
-; CHECK-NEXT:    vstrw.32 q1, [sp, #80] @ 16-byte Spill
-; CHECK-NEXT:    vmov q1, q2
+; CHECK-NEXT:    vmovx.f16 s4, s10
+; CHECK-NEXT:    vins.f16 s4, s0
 ; CHECK-NEXT:    vmovx.f16 s0, s23
-; CHECK-NEXT:    vldrw.u32 q2, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vins.f16 s2, s0
-; CHECK-NEXT:    vmov.16 q6[3], r0
-; CHECK-NEXT:    vrev32.16 q1, q1
-; CHECK-NEXT:    vmov.f32 s27, s2
+; CHECK-NEXT:    vmovx.f16 s7, s11
+; CHECK-NEXT:    vmov.f32 s28, s29
+; CHECK-NEXT:    vins.f16 s7, s0
+; CHECK-NEXT:    vins.f16 s5, s11
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s6, s23
+; CHECK-NEXT:    vmovx.f16 s16, s5
 ; CHECK-NEXT:    vmov.f32 s1, s11
-; CHECK-NEXT:    vmov.f32 s26, s23
-; CHECK-NEXT:    vmovx.f16 s16, s25
 ; CHECK-NEXT:    vmov.f32 s2, s11
 ; CHECK-NEXT:    vins.f16 s1, s16
 ; CHECK-NEXT:    vmovx.f16 s16, s2
-; CHECK-NEXT:    vins.f16 s26, s16
+; CHECK-NEXT:    vins.f16 s6, s16
 ; CHECK-NEXT:    vmovx.f16 s16, s9
 ; CHECK-NEXT:    vmov.f32 s20, s21
 ; CHECK-NEXT:    vins.f16 s20, s16
 ; CHECK-NEXT:    vmovx.f16 s16, s10
 ; CHECK-NEXT:    vins.f16 s22, s16
-; CHECK-NEXT:    vmov.f32 s2, s26
+; CHECK-NEXT:    vldrw.u32 q2, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #80] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s23, s22
-; CHECK-NEXT:    vmov.f32 s22, s10
+; CHECK-NEXT:    vrev32.16 q2, q2
+; CHECK-NEXT:    vmov.f32 s2, s6
+; CHECK-NEXT:    vmov.f32 s22, s18
 ; CHECK-NEXT:    vmovx.f16 s16, s21
-; CHECK-NEXT:    vldrw.u32 q2, [sp, #96] @ 16-byte Reload
-; CHECK-NEXT:    vins.f16 s5, s16
+; CHECK-NEXT:    vins.f16 s9, s16
 ; CHECK-NEXT:    vldrw.u32 q4, [sp, #112] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s25, s1
-; CHECK-NEXT:    vmov.f32 s18, s10
-; CHECK-NEXT:    vldrw.u32 q2, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s18, s14
+; CHECK-NEXT:    vstrw.32 q2, [sp, #80] @ 16-byte Spill
 ; CHECK-NEXT:    vstrw.32 q4, [sp, #112] @ 16-byte Spill
-; CHECK-NEXT:    vmovx.f16 s16, s6
-; CHECK-NEXT:    vmov.f32 s14, s10
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s18, s26
+; CHECK-NEXT:    vstrw.32 q4, [sp, #96] @ 16-byte Spill
+; CHECK-NEXT:    vmovx.f16 s16, s10
 ; CHECK-NEXT:    vins.f16 s22, s16
 ; CHECK-NEXT:    vldrw.u32 q4, [sp, #16] @ 16-byte Reload
-; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q1, [sp, #80] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s28, s29
+; CHECK-NEXT:    vmov.f32 s5, s1
 ; CHECK-NEXT:    vmovx.f16 s8, s17
-; CHECK-NEXT:    vmov.f32 s26, s2
-; CHECK-NEXT:    vmov.f32 s5, s13
+; CHECK-NEXT:    vmov.f32 s6, s2
 ; CHECK-NEXT:    vins.f16 s28, s8
 ; CHECK-NEXT:    vmovx.f16 s0, s18
-; CHECK-NEXT:    vldrw.u32 q2, [sp, #96] @ 16-byte Reload
 ; CHECK-NEXT:    vins.f16 s30, s0
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s6, s14
-; CHECK-NEXT:    vldrw.u32 q3, [sp, #112] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #48] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s31, s30
-; CHECK-NEXT:    vrev32.16 q0, q0
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #32] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s30, s18
 ; CHECK-NEXT:    vmovx.f16 s16, s29
-; CHECK-NEXT:    vmov.f32 s9, s13
+; CHECK-NEXT:    vrev32.16 q0, q0
+; CHECK-NEXT:    vstrw.32 q1, [r1, #80]
 ; CHECK-NEXT:    vins.f16 s1, s16
-; CHECK-NEXT:    vmov.f32 s10, s14
 ; CHECK-NEXT:    vmovx.f16 s16, s2
-; CHECK-NEXT:    vldrw.u32 q3, [sp, #48] @ 16-byte Reload
 ; CHECK-NEXT:    vins.f16 s30, s16
-; CHECK-NEXT:    vldrw.u32 q4, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #96] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s2, s30
-; CHECK-NEXT:    vmov.f32 s18, s14
-; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
-; CHECK-NEXT:    vmov.f32 s13, s17
-; CHECK-NEXT:    vstrw.32 q6, [r1, #80]
+; CHECK-NEXT:    vmov.f32 s25, s17
+; CHECK-NEXT:    vmov.f32 s26, s18
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #112] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s29, s1
-; CHECK-NEXT:    vstrw.32 q1, [r1]
-; CHECK-NEXT:    vmov.f32 s30, s2
+; CHECK-NEXT:    vstrw.32 q6, [r1]
+; CHECK-NEXT:    vmov.f32 s13, s17
 ; CHECK-NEXT:    vmov.f32 s14, s18
-; CHECK-NEXT:    vldrw.u32 q4, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s18, s22
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s18, s10
+; CHECK-NEXT:    vstrw.32 q3, [r1, #48]
+; CHECK-NEXT:    vmov.f32 s9, s17
+; CHECK-NEXT:    vmov.f32 s30, s2
 ; CHECK-NEXT:    vstrw.32 q7, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s10, s18
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s18, s22
+; CHECK-NEXT:    vstrw.32 q2, [r1, #32]
 ; CHECK-NEXT:    vmov.f32 s21, s17
-; CHECK-NEXT:    vstrw.32 q3, [r1, #32]
 ; CHECK-NEXT:    vmov.f32 s22, s18
 ; CHECK-NEXT:    vstrw.32 q5, [r1, #64]
 ; CHECK-NEXT:    add sp, #128
diff --git a/llvm/test/CodeGen/WebAssembly/call-indirect.ll b/llvm/test/CodeGen/WebAssembly/call-indirect.ll
index 3ce74ddfd30eac945cf6bf89f2511b1a1b8248a2..57e754ccb3be7cbd2e89a3a6cb845693621cdd74 100644
--- a/llvm/test/CodeGen/WebAssembly/call-indirect.ll
+++ b/llvm/test/CodeGen/WebAssembly/call-indirect.ll
@@ -22,7 +22,7 @@ define void @call_indirect_void(void ()* %callee) {
 ; OBJ-NEXT:        Field:           __linear_memory
 ; OBJ-NEXT:        Kind:            MEMORY
 ; OBJ-NEXT:        Memory:
-; OBJ-NEXT:          Initial:         0x0
+; OBJ-NEXT:          Minimum:         0x0
 ; OBJ-NEXT:      - Module:          env
 ; OBJ-NEXT:        Field:           __indirect_function_table
 ; OBJ-NEXT:        Kind:            TABLE
diff --git a/llvm/test/CodeGen/WebAssembly/function-pointer64.ll b/llvm/test/CodeGen/WebAssembly/function-pointer64.ll
index 155e4807d2973febfee298d9e1b26e2937e1a23a..21b2a8e17147bb43b0aee6be227ef9502ac5c23c 100644
--- a/llvm/test/CodeGen/WebAssembly/function-pointer64.ll
+++ b/llvm/test/CodeGen/WebAssembly/function-pointer64.ll
@@ -48,7 +48,7 @@ entry:
 
 ; YAML:      Memory:
 ; YAML-NEXT:   Flags:   [ IS_64 ]
-; YAML-NEXT:   Initial: 0x1
+; YAML-NEXT:   Minimum: 0x1
 
 ; YAML:      - Type:   CODE
 ; YAML:      - Type:   R_WASM_TABLE_INDEX_SLEB64
diff --git a/llvm/test/CodeGen/WebAssembly/simd-arith.ll b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
index fca4710b582f6427f5b59af011714b9c9e725e11..0268e8eb50c990fa5b89af90c5e53594d3bb2f44 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-arith.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
@@ -1,13 +1,14 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-SLOW
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-FAST
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 -fast-isel | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-SLOW
+
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-FAST
+
 ; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+
 ; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -fast-isel | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; check that a non-test run (including explicit locals pass) at least finishes
-; RUN: llc < %s -O0 -mattr=+unimplemented-simd128
-; RUN: llc < %s -O2 -mattr=+unimplemented-simd128
+; RUN: llc < %s -O0 -mattr=+simd128
+; RUN: llc < %s -O2 -mattr=+simd128
 
 ; Test that basic SIMD128 arithmetic operations assemble as expected.
 
diff --git a/llvm/test/CodeGen/WebAssembly/simd-build-vector.ll b/llvm/test/CodeGen/WebAssembly/simd-build-vector.ll
index 1360e0172d3fd2f8839234749129f3ec8702cca6..c1060ea1101fe7501eac19b939e2fb15426bd71c 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-build-vector.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-build-vector.ll
@@ -1,5 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes=CHECK,UNIMP
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s --check-prefixes=CHECK,SIMD-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
 
 ; Test that the logic to choose between v128.const vector
 ; initialization and splat vector initialization and to optimize the
@@ -8,95 +7,11 @@
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-; CHECK-LABEL:  emulated_const_trivial_splat:
-; CHECK-NEXT:   .functype       emulated_const_trivial_splat () -> (v128)
-; SIMD-VM-NEXT: i64.const       $push0=, 8589934593
-; SIMD-VM-NEXT: i64x2.splat     $push1=, $pop0
-; SIMD-VM-NEXT: return  $pop1
-; UNIMP: v128.const
-define <4 x i32> @emulated_const_trivial_splat() {
-  ret <4 x i32> <i32 1, i32 2, i32 1, i32 2>
-}
-
-; CHECK-LABEL:  emulated_const_first_sufficient:
-; CHECK-NEXT:   .functype       emulated_const_first_sufficient () -> (v128)
-; SIMD-VM-NEXT: i64.const       $push0=, 8589934593
-; SIMD-VM-NEXT: i64x2.splat     $push1=, $pop0
-; SIMD-VM-NEXT: return  $pop1
-; UNIMP: v128.const
-define <4 x i32> @emulated_const_first_sufficient() {
-  ret <4 x i32> <i32 1, i32 2, i32 undef, i32 2>
-}
-
-; CHECK-LABEL:  emulated_const_second_sufficient:
-; CHECK-NEXT:   .functype       emulated_const_second_sufficient () -> (v128)
-; SIMD-VM-NEXT: i64.const       $push0=, 8589934593
-; SIMD-VM-NEXT: i64x2.splat     $push1=, $pop0
-; SIMD-VM-NEXT: return  $pop1
-; UNIMP: v128.const
-define <4 x i32> @emulated_const_second_sufficient() {
-  ret <4 x i32> <i32 1, i32 undef, i32 1, i32 2>
-}
-
-; CHECK-LABEL:  emulated_const_combined_sufficient:
-; CHECK-NEXT:   .functype       emulated_const_combined_sufficient () -> (v128)
-; SIMD-VM-NEXT: i64.const       $push0=, 8589934593
-; SIMD-VM-NEXT: i64x2.splat     $push1=, $pop0
-; SIMD-VM-NEXT: return  $pop1
-; UNIMP: v128.const
-define <4 x i32> @emulated_const_combined_sufficient() {
-  ret <4 x i32> <i32 1, i32 undef, i32 undef, i32 2>
-}
-
-; CHECK-LABEL:  emulated_const_either_sufficient:
-; CHECK-NEXT:   .functype       emulated_const_either_sufficient () -> (v128)
-; SIMD-VM-NEXT: i64.const       $push0=, 1
-; SIMD-VM-NEXT: i64x2.splat     $push1=, $pop0
-; SIMD-VM-NEXT: return  $pop1
-; UNIMP: v128.const
-define <4 x i32> @emulated_const_either_sufficient() {
-  ret <4 x i32> <i32 1, i32 undef, i32 1, i32 undef>
-}
-
-; CHECK-LABEL: emulated_const_neither_sufficient:
-; CHECK-NEXT:   .functype       emulated_const_neither_sufficient () -> (v128)
-; SIMD-VM-NEXT: i64.const       $push0=, 8589934593
-; SIMD-VM-NEXT: i64x2.splat     $push1=, $pop0
-; SIMD-VM-NEXT: i64.const       $push2=, 17179869184
-; SIMD-VM-NEXT: i64x2.replace_lane      $push3=, $pop1, 1, $pop2
-; SIMD-VM-NEXT: return  $pop3
-define <4 x i32> @emulated_const_neither_sufficient() {
-  ret <4 x i32> <i32 1, i32 2, i32 undef, i32 4>
-}
-
-; CHECK-LABEL:  emulated_const_combined_sufficient_large:
-; CHECK-NEXT:   .functype       emulated_const_combined_sufficient_large () -> (v128)
-; SIMD-VM-NEXT: i64.const       $push0=, 506097522914230528
-; SIMD-VM-NEXT: i64x2.splat     $push1=, $pop0
-; SIMD-VM-NEXT: return  $pop1
-define <16 x i8> @emulated_const_combined_sufficient_large() {
-  ret <16 x i8> <i8 0, i8 undef, i8 2, i8 undef, i8 4, i8 undef, i8 6, i8 undef,
-                 i8 undef, i8 1, i8 undef, i8 3, i8 undef, i8 5, i8 undef, i8 7>
-}
-
-; CHECK-LABEL: emulated_const_neither_sufficient_large:
-; CHECK-NEXT:   .functype       emulated_const_neither_sufficient_large () -> (v128)
-; SIMD-VM-NEXT: i64.const       $push0=, -70368726997663744
-; SIMD-VM-NEXT: i64x2.splat     $push1=, $pop0
-; SIMD-VM-NEXT: i64.const       $push2=, 504408655873966336
-; SIMD-VM-NEXT: i64x2.replace_lane      $push3=, $pop1, 1, $pop2
-; SIMD-VM-NEXT: return  $pop3
-define <16 x i8> @emulated_const_neither_sufficient_large() {
-  ret <16 x i8> <i8 0, i8 undef, i8 2, i8 undef, i8 4, i8 undef, i8 6, i8 255,
-                 i8 undef, i8 1, i8 undef, i8 3, i8 undef, i8 5, i8 undef, i8 7>
-}
-
 ; CHECK-LABEL: same_const_one_replaced_i16x8:
 ; CHECK-NEXT:  .functype       same_const_one_replaced_i16x8 (i32) -> (v128)
-; UNIMP-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 42, 42, 42, 42, 42, 0, 42, 42
-; UNIMP-NEXT:  i16x8.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 5, $0
-; UNIMP-NEXT:  return          $pop[[L1]]
-; SIMD-VM: i64x2.splat
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 42, 42, 42, 42, 42, 0, 42, 42
+; CHECK-NEXT:  i16x8.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 5, $0
+; CHECK-NEXT:  return          $pop[[L1]]
 define <8 x i16> @same_const_one_replaced_i16x8(i16 %x) {
   %v = insertelement
     <8 x i16> <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>,
@@ -107,10 +22,9 @@ define <8 x i16> @same_const_one_replaced_i16x8(i16 %x) {
 
 ; CHECK-LABEL: different_const_one_replaced_i16x8:
 ; CHECK-NEXT:  .functype       different_const_one_replaced_i16x8 (i32) -> (v128)
-; UNIMP-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 1, -2, 3, -4, 5, 0, 7, -8
-; UNIMP-NEXT:  i16x8.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 5, $0
-; UNIMP-NEXT:  return          $pop[[L1]]
-; SIMD-VM: i64x2.splat
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 1, -2, 3, -4, 5, 0, 7, -8
+; CHECK-NEXT:  i16x8.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 5, $0
+; CHECK-NEXT:  return          $pop[[L1]]
 define <8 x i16> @different_const_one_replaced_i16x8(i16 %x) {
   %v = insertelement
     <8 x i16> <i16 1, i16 -2, i16 3, i16 -4, i16 5, i16 -6, i16 7, i16 -8>,
@@ -121,10 +35,9 @@ define <8 x i16> @different_const_one_replaced_i16x8(i16 %x) {
 
 ; CHECK-LABEL: same_const_one_replaced_f32x4:
 ; CHECK-NEXT:  .functype       same_const_one_replaced_f32x4 (f32) -> (v128)
-; UNIMP-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0x1.5p5, 0x1.5p5, 0x0p0, 0x1.5p5
-; UNIMP-NEXT:  f32x4.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 2, $0
-; UNIMP-NEXT:  return          $pop[[L1]]
-; SIMD-VM: f32x4.splat
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0x1.5p5, 0x1.5p5, 0x0p0, 0x1.5p5
+; CHECK-NEXT:  f32x4.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 2, $0
+; CHECK-NEXT:  return          $pop[[L1]]
 define <4 x float> @same_const_one_replaced_f32x4(float %x) {
   %v = insertelement
     <4 x float> <float 42., float 42., float 42., float 42.>,
@@ -135,10 +48,9 @@ define <4 x float> @same_const_one_replaced_f32x4(float %x) {
 
 ; CHECK-LABEL: different_const_one_replaced_f32x4:
 ; CHECK-NEXT:  .functype       different_const_one_replaced_f32x4 (f32) -> (v128)
-; UNIMP-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0x1p0, 0x1p1, 0x0p0, 0x1p2
-; UNIMP-NEXT:  f32x4.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 2, $0
-; UNIMP-NEXT:  return          $pop[[L1]]
-; SIMD-VM: f32x4.splat
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0x1p0, 0x1p1, 0x0p0, 0x1p2
+; CHECK-NEXT:  f32x4.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 2, $0
+; CHECK-NEXT:  return          $pop[[L1]]
 define <4 x float> @different_const_one_replaced_f32x4(float %x) {
   %v = insertelement
     <4 x float> <float 1., float 2., float 3., float 4.>,
@@ -149,9 +61,8 @@ define <4 x float> @different_const_one_replaced_f32x4(float %x) {
 
 ; CHECK-LABEL: splat_common_const_i32x4:
 ; CHECK-NEXT:  .functype       splat_common_const_i32x4 () -> (v128)
-; UNIMP-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0, 3, 3, 1
-; UNIMP-NEXT:  return          $pop[[L0]]
-; SIMD-VM: i64x2.splat
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0, 3, 3, 1
+; CHECK-NEXT:  return          $pop[[L0]]
 define <4 x i32> @splat_common_const_i32x4() {
   ret <4 x i32> <i32 undef, i32 3, i32 3, i32 1>
 }
@@ -284,12 +195,11 @@ define <16 x i8> @mashup_swizzle_i8x16(<16 x i8> %src, <16 x i8> %mask, i8 %spla
 
 ; CHECK-LABEL: mashup_const_i8x16:
 ; CHECK-NEXT:  .functype       mashup_const_i8x16 (v128, v128, i32) -> (v128)
-; UNIMP:       v128.const      $push[[L0:[0-9]+]]=, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0
-; UNIMP:       i8x16.replace_lane
-; UNIMP:       i8x16.replace_lane
-; UNIMP:       i8x16.replace_lane
-; UNIMP:       return
-; SIMD-VM: i64x2.splat
+; CHECK:       v128.const      $push[[L0:[0-9]+]]=, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0
+; CHECK:       i8x16.replace_lane
+; CHECK:       i8x16.replace_lane
+; CHECK:       i8x16.replace_lane
+; CHECK:       return
 define <16 x i8> @mashup_const_i8x16(<16 x i8> %src, <16 x i8> %mask, i8 %splatted) {
   ; swizzle 0
   %m0 = extractelement <16 x i8> %mask, i32 0
@@ -328,8 +238,8 @@ define <16 x i8> @mashup_splat_i8x16(<16 x i8> %src, <16 x i8> %mask, i8 %splatt
 
 ; CHECK-LABEL: undef_const_insert_f32x4:
 ; CHECK-NEXT:  .functype       undef_const_insert_f32x4 () -> (v128)
-; UNIMP-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0x0p0, 0x1.5p5, 0x0p0, 0x0p0
-; UNIMP-NEXT:  return          $pop[[L0]]
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0x0p0, 0x1.5p5, 0x0p0, 0x0p0
+; CHECK-NEXT:  return          $pop[[L0]]
 ; SIMD-VM: f32x4.splat
 define <4 x float> @undef_const_insert_f32x4() {
   %v = insertelement <4 x float> undef, float 42., i32 1
diff --git a/llvm/test/CodeGen/WebAssembly/simd-comparisons.ll b/llvm/test/CodeGen/WebAssembly/simd-comparisons.ll
index a77f9e1fa5818c44d4aa3eb99d9b5f3a0fe7b53b..475bfc5110fe16a043df63b8ac8bf7e919ddab9b 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-comparisons.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-comparisons.ll
@@ -1,5 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
 ; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test SIMD comparison operators
diff --git a/llvm/test/CodeGen/WebAssembly/simd-conversions.ll b/llvm/test/CodeGen/WebAssembly/simd-conversions.ll
index 53731b0f7c16d3b051b59eee6c586c03c8243ead..36856336e65e5211a074730f717320dbb95a3124 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-conversions.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-conversions.ll
@@ -1,5 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
 ; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test that vector float-to-int and int-to-float instructions lower correctly
@@ -29,7 +28,7 @@ define <4 x float> @convert_u_v4f32(<4 x i32> %x) {
 
 ; CHECK-LABEL: convert_s_v2f64:
 ; NO-SIMD128-NOT: i64x2
-; SIMD128-VM-NOT: f64x2.convert_i64x2_s
+; SIMD128-NOT: f64x2.convert_i64x2_s
 ; SIMD128-NEXT: .functype convert_s_v2f64 (v128) -> (v128){{$}}
 define <2 x double> @convert_s_v2f64(<2 x i64> %x) {
   %a = sitofp <2 x i64> %x to <2 x double>
@@ -38,7 +37,7 @@ define <2 x double> @convert_s_v2f64(<2 x i64> %x) {
 
 ; CHECK-LABEL: convert_u_v2f64:
 ; NO-SIMD128-NOT: i64x2
-; SIMD128-VM-NOT: f64x2.convert_i64x2_u
+; SIMD128-NOT: f64x2.convert_i64x2_u
 ; SIMD128-NEXT: .functype convert_u_v2f64 (v128) -> (v128){{$}}
 define <2 x double> @convert_u_v2f64(<2 x i64> %x) {
   %a = uitofp <2 x i64> %x to <2 x double>
@@ -67,7 +66,7 @@ define <4 x i32> @trunc_sat_u_v4i32(<4 x float> %x) {
 
 ; CHECK-LABEL: trunc_sat_s_v2i64:
 ; NO-SIMD128-NOT: f64x2
-; SIMD128-VM-NOT: i64x2.trunc_sat_f64x2_s
+; SIMD128-NOT: i64x2.trunc_sat_f64x2_s
 ; SIMD128-NEXT: .functype trunc_sat_s_v2i64 (v128) -> (v128){{$}}
 define <2 x i64> @trunc_sat_s_v2i64(<2 x double> %x) {
   %a = fptosi <2 x double> %x to <2 x i64>
@@ -76,7 +75,7 @@ define <2 x i64> @trunc_sat_s_v2i64(<2 x double> %x) {
 
 ; CHECK-LABEL: trunc_sat_u_v2i64:
 ; NO-SIMD128-NOT: f64x2
-; SIMD128-VM-NOT: i64x2.trunc_sat_f64x2_u
+; SIMD128-NOT: i64x2.trunc_sat_f64x2_u
 ; SIMD128-NEXT: .functype trunc_sat_u_v2i64 (v128) -> (v128){{$}}
 define <2 x i64> @trunc_sat_u_v2i64(<2 x double> %x) {
   %a = fptoui <2 x double> %x to <2 x i64>
diff --git a/llvm/test/CodeGen/WebAssembly/simd-widening.ll b/llvm/test/CodeGen/WebAssembly/simd-extending.ll
similarity index 59%
rename from llvm/test/CodeGen/WebAssembly/simd-widening.ll
rename to llvm/test/CodeGen/WebAssembly/simd-extending.ll
index c9a7ffbbfcaf5523f2de55d3773fd6454d2f3b4c..3f512cd2678e0eb13856fbd08d1fb84ef3884c34 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-widening.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-extending.ll
@@ -1,121 +1,121 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mattr=+simd128 | FileCheck %s
 
-;; Test that SIMD widening operations can be successfully selected
+;; Test that SIMD extending operations can be successfully selected
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-define <8 x i16> @widen_low_i8x16_s(<16 x i8> %v) {
-; CHECK-LABEL: widen_low_i8x16_s:
-; CHECK:         .functype widen_low_i8x16_s (v128) -> (v128)
+define <8 x i16> @extend_low_i8x16_s(<16 x i8> %v) {
+; CHECK-LABEL: extend_low_i8x16_s:
+; CHECK:         .functype extend_low_i8x16_s (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i16x8.widen_low_i8x16_s
+; CHECK-NEXT:    i16x8.extend_low_i8x16_s
 ; CHECK-NEXT:    # fallthrough-return
   %low = shufflevector <16 x i8> %v, <16 x i8> undef,
            <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %widened = sext <8 x i8> %low to <8 x i16>
-  ret <8 x i16> %widened
+  %extended = sext <8 x i8> %low to <8 x i16>
+  ret <8 x i16> %extended
 }
 
-define <8 x i16> @widen_low_i8x16_u(<16 x i8> %v) {
-; CHECK-LABEL: widen_low_i8x16_u:
-; CHECK:         .functype widen_low_i8x16_u (v128) -> (v128)
+define <8 x i16> @extend_low_i8x16_u(<16 x i8> %v) {
+; CHECK-LABEL: extend_low_i8x16_u:
+; CHECK:         .functype extend_low_i8x16_u (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i16x8.widen_low_i8x16_u
+; CHECK-NEXT:    i16x8.extend_low_i8x16_u
 ; CHECK-NEXT:    # fallthrough-return
   %low = shufflevector <16 x i8> %v, <16 x i8> undef,
            <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %widened = zext <8 x i8> %low to <8 x i16>
-  ret <8 x i16> %widened
+  %extended = zext <8 x i8> %low to <8 x i16>
+  ret <8 x i16> %extended
 }
 
-define <8 x i16> @widen_high_i8x16_s(<16 x i8> %v) {
-; CHECK-LABEL: widen_high_i8x16_s:
-; CHECK:         .functype widen_high_i8x16_s (v128) -> (v128)
+define <8 x i16> @extend_high_i8x16_s(<16 x i8> %v) {
+; CHECK-LABEL: extend_high_i8x16_s:
+; CHECK:         .functype extend_high_i8x16_s (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i16x8.widen_high_i8x16_s
+; CHECK-NEXT:    i16x8.extend_high_i8x16_s
 ; CHECK-NEXT:    # fallthrough-return
   %low = shufflevector <16 x i8> %v, <16 x i8> undef,
            <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-  %widened = sext <8 x i8> %low to <8 x i16>
-  ret <8 x i16> %widened
+  %extended = sext <8 x i8> %low to <8 x i16>
+  ret <8 x i16> %extended
 }
 
-define <8 x i16> @widen_high_i8x16_u(<16 x i8> %v) {
-; CHECK-LABEL: widen_high_i8x16_u:
-; CHECK:         .functype widen_high_i8x16_u (v128) -> (v128)
+define <8 x i16> @extend_high_i8x16_u(<16 x i8> %v) {
+; CHECK-LABEL: extend_high_i8x16_u:
+; CHECK:         .functype extend_high_i8x16_u (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i16x8.widen_high_i8x16_u
+; CHECK-NEXT:    i16x8.extend_high_i8x16_u
 ; CHECK-NEXT:    # fallthrough-return
   %low = shufflevector <16 x i8> %v, <16 x i8> undef,
            <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-  %widened = zext <8 x i8> %low to <8 x i16>
-  ret <8 x i16> %widened
+  %extended = zext <8 x i8> %low to <8 x i16>
+  ret <8 x i16> %extended
 }
 
-define <4 x i32> @widen_low_i16x8_s(<8 x i16> %v) {
-; CHECK-LABEL: widen_low_i16x8_s:
-; CHECK:         .functype widen_low_i16x8_s (v128) -> (v128)
+define <4 x i32> @extend_low_i16x8_s(<8 x i16> %v) {
+; CHECK-LABEL: extend_low_i16x8_s:
+; CHECK:         .functype extend_low_i16x8_s (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32x4.widen_low_i16x8_s
+; CHECK-NEXT:    i32x4.extend_low_i16x8_s
 ; CHECK-NEXT:    # fallthrough-return
   %low = shufflevector <8 x i16> %v, <8 x i16> undef,
            <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %widened = sext <4 x i16> %low to <4 x i32>
-  ret <4 x i32> %widened
+  %extended = sext <4 x i16> %low to <4 x i32>
+  ret <4 x i32> %extended
 }
 
-define <4 x i32> @widen_low_i16x8_u(<8 x i16> %v) {
-; CHECK-LABEL: widen_low_i16x8_u:
-; CHECK:         .functype widen_low_i16x8_u (v128) -> (v128)
+define <4 x i32> @extend_low_i16x8_u(<8 x i16> %v) {
+; CHECK-LABEL: extend_low_i16x8_u:
+; CHECK:         .functype extend_low_i16x8_u (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32x4.widen_low_i16x8_u
+; CHECK-NEXT:    i32x4.extend_low_i16x8_u
 ; CHECK-NEXT:    # fallthrough-return
   %low = shufflevector <8 x i16> %v, <8 x i16> undef,
            <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %widened = zext <4 x i16> %low to <4 x i32>
-  ret <4 x i32> %widened
+  %extended = zext <4 x i16> %low to <4 x i32>
+  ret <4 x i32> %extended
 }
 
-define <4 x i32> @widen_high_i16x8_s(<8 x i16> %v) {
-; CHECK-LABEL: widen_high_i16x8_s:
-; CHECK:         .functype widen_high_i16x8_s (v128) -> (v128)
+define <4 x i32> @extend_high_i16x8_s(<8 x i16> %v) {
+; CHECK-LABEL: extend_high_i16x8_s:
+; CHECK:         .functype extend_high_i16x8_s (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32x4.widen_high_i16x8_s
+; CHECK-NEXT:    i32x4.extend_high_i16x8_s
 ; CHECK-NEXT:    # fallthrough-return
   %low = shufflevector <8 x i16> %v, <8 x i16> undef,
            <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-  %widened = sext <4 x i16> %low to <4 x i32>
-  ret <4 x i32> %widened
+  %extended = sext <4 x i16> %low to <4 x i32>
+  ret <4 x i32> %extended
 }
 
-define <4 x i32> @widen_high_i16x8_u(<8 x i16> %v) {
-; CHECK-LABEL: widen_high_i16x8_u:
-; CHECK:         .functype widen_high_i16x8_u (v128) -> (v128)
+define <4 x i32> @extend_high_i16x8_u(<8 x i16> %v) {
+; CHECK-LABEL: extend_high_i16x8_u:
+; CHECK:         .functype extend_high_i16x8_u (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32x4.widen_high_i16x8_u
+; CHECK-NEXT:    i32x4.extend_high_i16x8_u
 ; CHECK-NEXT:    # fallthrough-return
   %low = shufflevector <8 x i16> %v, <8 x i16> undef,
            <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-  %widened = zext <4 x i16> %low to <4 x i32>
-  ret <4 x i32> %widened
+  %extended = zext <4 x i16> %low to <4 x i32>
+  ret <4 x i32> %extended
 }
 
 ;; Also test that similar patterns with offsets not corresponding to
 ;; the low or high half are correctly expanded.
 
-define <8 x i16> @widen_lowish_i8x16_s(<16 x i8> %v) {
-; CHECK-LABEL: widen_lowish_i8x16_s:
-; CHECK:         .functype widen_lowish_i8x16_s (v128) -> (v128)
+define <8 x i16> @extend_lowish_i8x16_s(<16 x i8> %v) {
+; CHECK-LABEL: extend_lowish_i8x16_s:
+; CHECK:         .functype extend_lowish_i8x16_s (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    i8x16.extract_lane_u 1
@@ -148,13 +148,13 @@ define <8 x i16> @widen_lowish_i8x16_s(<16 x i8> %v) {
 ; CHECK-NEXT:    # fallthrough-return
   %lowish = shufflevector <16 x i8> %v, <16 x i8> undef,
            <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
-  %widened = sext <8 x i8> %lowish to <8 x i16>
-  ret <8 x i16> %widened
+  %extended = sext <8 x i8> %lowish to <8 x i16>
+  ret <8 x i16> %extended
 }
 
-define <4 x i32> @widen_lowish_i16x8_s(<8 x i16> %v) {
-; CHECK-LABEL: widen_lowish_i16x8_s:
-; CHECK:         .functype widen_lowish_i16x8_s (v128) -> (v128)
+define <4 x i32> @extend_lowish_i16x8_s(<8 x i16> %v) {
+; CHECK-LABEL: extend_lowish_i16x8_s:
+; CHECK:         .functype extend_lowish_i16x8_s (v128) -> (v128)
 ; CHECK-NEXT:  # %bb.0:
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    i16x8.extract_lane_u 1
@@ -175,6 +175,6 @@ define <4 x i32> @widen_lowish_i16x8_s(<8 x i16> %v) {
 ; CHECK-NEXT:    # fallthrough-return
   %lowish = shufflevector <8 x i16> %v, <8 x i16> undef,
            <4 x i32> <i32 1, i32 2, i32 3, i32 4>
-  %widened = sext <4 x i16> %lowish to <4 x i32>
-  ret <4 x i32> %widened
+  %extended = sext <4 x i16> %lowish to <4 x i32>
+  ret <4 x i32> %extended
 }
diff --git a/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll b/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
index f223615c57a15e8519bfa4a47324c670e247e233..d2fdde3fcb3c6483900c493948abc9c5ce7556c5 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 -fast-isel | FileCheck %s
 
 ; Test that SIMD128 intrinsics lower as expected. These intrinsics are
 ; only expected to lower successfully if the simd128 attribute is
@@ -12,9 +12,9 @@ target triple = "wasm32-unknown-unknown"
 ; 16 x i8
 ; ==============================================================================
 ; CHECK-LABEL: swizzle_v16i8:
-; SIMD128-NEXT: .functype swizzle_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.swizzle $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype swizzle_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.swizzle $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.wasm.swizzle(<16 x i8>, <16 x i8>)
 define <16 x i8> @swizzle_v16i8(<16 x i8> %x, <16 x i8> %y) {
   %a = call <16 x i8> @llvm.wasm.swizzle(<16 x i8> %x, <16 x i8> %y)
@@ -22,9 +22,9 @@ define <16 x i8> @swizzle_v16i8(<16 x i8> %x, <16 x i8> %y) {
 }
 
 ; CHECK-LABEL: add_sat_s_v16i8:
-; SIMD128-NEXT: .functype add_sat_s_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.add_saturate_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype add_sat_s_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.add_sat_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>)
 define <16 x i8> @add_sat_s_v16i8(<16 x i8> %x, <16 x i8> %y) {
   %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
@@ -32,9 +32,9 @@ define <16 x i8> @add_sat_s_v16i8(<16 x i8> %x, <16 x i8> %y) {
 }
 
 ; CHECK-LABEL: add_sat_u_v16i8:
-; SIMD128-NEXT: .functype add_sat_u_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.add_saturate_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype add_sat_u_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.add_sat_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>)
 define <16 x i8> @add_sat_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
   %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
@@ -42,33 +42,33 @@ define <16 x i8> @add_sat_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
 }
 
 ; CHECK-LABEL: sub_sat_s_v16i8:
-; SIMD128-NEXT: .functype sub_sat_s_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.sub_saturate_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <16 x i8> @llvm.wasm.sub.saturate.signed.v16i8(<16 x i8>, <16 x i8>)
+; CHECK-NEXT: .functype sub_sat_s_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.sub_sat_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.sub.sat.signed.v16i8(<16 x i8>, <16 x i8>)
 define <16 x i8> @sub_sat_s_v16i8(<16 x i8> %x, <16 x i8> %y) {
-  %a = call <16 x i8> @llvm.wasm.sub.saturate.signed.v16i8(
+  %a = call <16 x i8> @llvm.wasm.sub.sat.signed.v16i8(
     <16 x i8> %x, <16 x i8> %y
   )
   ret <16 x i8> %a
 }
 
 ; CHECK-LABEL: sub_sat_u_v16i8:
-; SIMD128-NEXT: .functype sub_sat_u_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.sub_saturate_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <16 x i8> @llvm.wasm.sub.saturate.unsigned.v16i8(<16 x i8>, <16 x i8>)
+; CHECK-NEXT: .functype sub_sat_u_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.sub_sat_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.sub.sat.unsigned.v16i8(<16 x i8>, <16 x i8>)
 define <16 x i8> @sub_sat_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
-  %a = call <16 x i8> @llvm.wasm.sub.saturate.unsigned.v16i8(
+  %a = call <16 x i8> @llvm.wasm.sub.sat.unsigned.v16i8(
     <16 x i8> %x, <16 x i8> %y
   )
   ret <16 x i8> %a
 }
 
 ; CHECK-LABEL: avgr_u_v16i8:
-; SIMD128-NEXT: .functype avgr_u_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.avgr_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype avgr_u_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.avgr_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.wasm.avgr.unsigned.v16i8(<16 x i8>, <16 x i8>)
 define <16 x i8> @avgr_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
   %a = call <16 x i8> @llvm.wasm.avgr.unsigned.v16i8(<16 x i8> %x, <16 x i8> %y)
@@ -76,9 +76,9 @@ define <16 x i8> @avgr_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
 }
 
 ; CHECK-LABEL: popcnt_v16i8:
-; SIMD128-NEXT: .functype popcnt_v16i8 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.popcnt $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype popcnt_v16i8 (v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.popcnt $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.wasm.popcnt(<16 x i8>)
 define <16 x i8> @popcnt_v16i8(<16 x i8> %x) {
  %a = call <16 x i8> @llvm.wasm.popcnt(<16 x i8> %x)
@@ -86,9 +86,9 @@ define <16 x i8> @popcnt_v16i8(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: any_v16i8:
-; SIMD128-NEXT: .functype any_v16i8 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v16i8 (v128) -> (i32){{$}}
+; CHECK-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.anytrue.v16i8(<16 x i8>)
 define i32 @any_v16i8(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.anytrue.v16i8(<16 x i8> %x)
@@ -96,9 +96,9 @@ define i32 @any_v16i8(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: all_v16i8:
-; SIMD128-NEXT: .functype all_v16i8 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v16i8 (v128) -> (i32){{$}}
+; CHECK-NEXT: i8x16.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.alltrue.v16i8(<16 x i8>)
 define i32 @all_v16i8(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.alltrue.v16i8(<16 x i8> %x)
@@ -106,9 +106,9 @@ define i32 @all_v16i8(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: bitmask_v16i8:
-; SIMD128-NEXT: .functype bitmask_v16i8 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.bitmask $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitmask_v16i8 (v128) -> (i32){{$}}
+; CHECK-NEXT: i8x16.bitmask $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.bitmask.v16i8(<16 x i8>)
 define i32 @bitmask_v16i8(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.bitmask.v16i8(<16 x i8> %x)
@@ -116,9 +116,9 @@ define i32 @bitmask_v16i8(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: bitselect_v16i8:
-; SIMD128-NEXT: .functype bitselect_v16i8 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitselect_v16i8 (v128, v128, v128) -> (v128){{$}}
+; CHECK-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.wasm.bitselect.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
 define <16 x i8> @bitselect_v16i8(<16 x i8> %v1, <16 x i8> %v2, <16 x i8> %c) {
   %a = call <16 x i8> @llvm.wasm.bitselect.v16i8(
@@ -127,22 +127,10 @@ define <16 x i8> @bitselect_v16i8(<16 x i8> %v1, <16 x i8> %v2, <16 x i8> %c) {
   ret <16 x i8> %a
 }
 
-; CHECK-LABEL: signselect_v16i8:
-; SIMD128-NEXT: .functype signselect_v16i8 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.signselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <16 x i8> @llvm.wasm.signselect.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
-define <16 x i8> @signselect_v16i8(<16 x i8> %v1, <16 x i8> %v2, <16 x i8> %c) {
-  %a = call <16 x i8> @llvm.wasm.signselect.v16i8(
-     <16 x i8> %v1, <16 x i8> %v2, <16 x i8> %c
-  )
-  ret <16 x i8> %a
-}
-
 ; CHECK-LABEL: narrow_signed_v16i8:
-; SIMD128-NEXT: .functype narrow_signed_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.narrow_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype narrow_signed_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.narrow_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.wasm.narrow.signed.v16i8.v8i16(<8 x i16>, <8 x i16>)
 define <16 x i8> @narrow_signed_v16i8(<8 x i16> %low, <8 x i16> %high) {
   %a = call <16 x i8> @llvm.wasm.narrow.signed.v16i8.v8i16(
@@ -152,9 +140,9 @@ define <16 x i8> @narrow_signed_v16i8(<8 x i16> %low, <8 x i16> %high) {
 }
 
 ; CHECK-LABEL: narrow_unsigned_v16i8:
-; SIMD128-NEXT: .functype narrow_unsigned_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.narrow_i16x8_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype narrow_unsigned_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.narrow_i16x8_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.wasm.narrow.unsigned.v16i8.v8i16(<8 x i16>, <8 x i16>)
 define <16 x i8> @narrow_unsigned_v16i8(<8 x i16> %low, <8 x i16> %high) {
   %a = call <16 x i8> @llvm.wasm.narrow.unsigned.v16i8.v8i16(
@@ -164,11 +152,11 @@ define <16 x i8> @narrow_unsigned_v16i8(<8 x i16> %low, <8 x i16> %high) {
 }
 
 ; CHECK-LABEL: shuffle_v16i8:
-; NO-SIMD128-NOT: i8x16
-; SIMD128-NEXT: .functype shuffle_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
-; SIMD128-SAME: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; NO-CHECK-NOT: i8x16
+; CHECK-NEXT: .functype shuffle_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
+; CHECK-SAME: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.wasm.shuffle(
   <16 x i8>, <16 x i8>, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
   i32, i32, i32, i32, i32)
@@ -180,11 +168,11 @@ define <16 x i8> @shuffle_v16i8(<16 x i8> %x, <16 x i8> %y) {
 }
 
 ; CHECK-LABEL: shuffle_undef_v16i8:
-; NO-SIMD128-NOT: i8x16
-; SIMD128-NEXT: .functype shuffle_undef_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
-; SIMD128-SAME: 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; NO-CHECK-NOT: i8x16
+; CHECK-NEXT: .functype shuffle_undef_v16i8 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i8x16.shuffle $push[[R:[0-9]+]]=, $0, $1,
+; CHECK-SAME: 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @shuffle_undef_v16i8(<16 x i8> %x, <16 x i8> %y) {
   %res = call <16 x i8> @llvm.wasm.shuffle(<16 x i8> %x, <16 x i8> %y,
       i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef,
@@ -197,9 +185,9 @@ define <16 x i8> @shuffle_undef_v16i8(<16 x i8> %x, <16 x i8> %y) {
 ; 8 x i16
 ; ==============================================================================
 ; CHECK-LABEL: add_sat_s_v8i16:
-; SIMD128-NEXT: .functype add_sat_s_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.add_saturate_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype add_sat_s_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.add_sat_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
 define <8 x i16> @add_sat_s_v8i16(<8 x i16> %x, <8 x i16> %y) {
   %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
@@ -207,9 +195,9 @@ define <8 x i16> @add_sat_s_v8i16(<8 x i16> %x, <8 x i16> %y) {
 }
 
 ; CHECK-LABEL: add_sat_u_v8i16:
-; SIMD128-NEXT: .functype add_sat_u_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.add_saturate_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype add_sat_u_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.add_sat_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
 define <8 x i16> @add_sat_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
   %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
@@ -217,33 +205,33 @@ define <8 x i16> @add_sat_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
 }
 
 ; CHECK-LABEL: sub_sat_s_v8i16:
-; SIMD128-NEXT: .functype sub_sat_s_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.sub_saturate_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <8 x i16> @llvm.wasm.sub.saturate.signed.v8i16(<8 x i16>, <8 x i16>)
+; CHECK-NEXT: .functype sub_sat_s_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.sub_sat_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.sub.sat.signed.v8i16(<8 x i16>, <8 x i16>)
 define <8 x i16> @sub_sat_s_v8i16(<8 x i16> %x, <8 x i16> %y) {
-  %a = call <8 x i16> @llvm.wasm.sub.saturate.signed.v8i16(
+  %a = call <8 x i16> @llvm.wasm.sub.sat.signed.v8i16(
     <8 x i16> %x, <8 x i16> %y
   )
   ret <8 x i16> %a
 }
 
 ; CHECK-LABEL: sub_sat_u_v8i16:
-; SIMD128-NEXT: .functype sub_sat_u_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.sub_saturate_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <8 x i16> @llvm.wasm.sub.saturate.unsigned.v8i16(<8 x i16>, <8 x i16>)
+; CHECK-NEXT: .functype sub_sat_u_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.sub_sat_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.sub.sat.unsigned.v8i16(<8 x i16>, <8 x i16>)
 define <8 x i16> @sub_sat_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
-  %a = call <8 x i16> @llvm.wasm.sub.saturate.unsigned.v8i16(
+  %a = call <8 x i16> @llvm.wasm.sub.sat.unsigned.v8i16(
     <8 x i16> %x, <8 x i16> %y
   )
   ret <8 x i16> %a
 }
 
 ; CHECK-LABEL: avgr_u_v8i16:
-; SIMD128-NEXT: .functype avgr_u_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.avgr_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype avgr_u_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.avgr_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.avgr.unsigned.v8i16(<8 x i16>, <8 x i16>)
 define <8 x i16> @avgr_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
   %a = call <8 x i16> @llvm.wasm.avgr.unsigned.v8i16(<8 x i16> %x, <8 x i16> %y)
@@ -251,20 +239,20 @@ define <8 x i16> @avgr_u_v8i16(<8 x i16> %x, <8 x i16> %y) {
 }
 
 ; CHECK-LABEL: q15mulr_sat_s_v8i16:
-; SIMD128-NEXT: .functype q15mulr_sat_s_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.q15mulr_sat_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <8 x i16> @llvm.wasm.q15mulr.saturate.signed(<8 x i16>, <8 x i16>)
+; CHECK-NEXT: .functype q15mulr_sat_s_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.q15mulr_sat_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.q15mulr.sat.signed(<8 x i16>, <8 x i16>)
 define <8 x i16> @q15mulr_sat_s_v8i16(<8 x i16> %x, <8 x i16> %y) {
-  %a = call <8 x i16> @llvm.wasm.q15mulr.saturate.signed(<8 x i16> %x,
+  %a = call <8 x i16> @llvm.wasm.q15mulr.sat.signed(<8 x i16> %x,
                                                          <8 x i16> %y)
   ret <8 x i16> %a
 }
 
 ; CHECK-LABEL: extmul_low_s_v8i16:
-; SIMD128-NEXT: .functype extmul_low_s_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.extmul_low_i8x16_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_low_s_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.extmul_low_i8x16_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.extmul.low.signed.v8i16(<16 x i8>, <16 x i8>)
 define <8 x i16> @extmul_low_s_v8i16(<16 x i8> %x, <16 x i8> %y) {
   %a = call <8 x i16> @llvm.wasm.extmul.low.signed.v8i16(
@@ -274,9 +262,9 @@ define <8 x i16> @extmul_low_s_v8i16(<16 x i8> %x, <16 x i8> %y) {
 }
 
 ; CHECK-LABEL: extmul_high_s_v8i16:
-; SIMD128-NEXT: .functype extmul_high_s_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.extmul_high_i8x16_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_high_s_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.extmul_high_i8x16_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.extmul.high.signed.v8i16(<16 x i8>, <16 x i8>)
 define <8 x i16> @extmul_high_s_v8i16(<16 x i8> %x, <16 x i8> %y) {
   %a = call <8 x i16> @llvm.wasm.extmul.high.signed.v8i16(
@@ -286,9 +274,9 @@ define <8 x i16> @extmul_high_s_v8i16(<16 x i8> %x, <16 x i8> %y) {
 }
 
 ; CHECK-LABEL: extmul_low_u_v8i16:
-; SIMD128-NEXT: .functype extmul_low_u_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.extmul_low_i8x16_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_low_u_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.extmul_low_i8x16_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.extmul.low.unsigned.v8i16(<16 x i8>, <16 x i8>)
 define <8 x i16> @extmul_low_u_v8i16(<16 x i8> %x, <16 x i8> %y) {
   %a = call <8 x i16> @llvm.wasm.extmul.low.unsigned.v8i16(
@@ -298,9 +286,9 @@ define <8 x i16> @extmul_low_u_v8i16(<16 x i8> %x, <16 x i8> %y) {
 }
 
 ; CHECK-LABEL: extmul_high_u_v8i16:
-; SIMD128-NEXT: .functype extmul_high_u_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.extmul_high_i8x16_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_high_u_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.extmul_high_i8x16_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.extmul.high.unsigned.v8i16(<16 x i8>, <16 x i8>)
 define <8 x i16> @extmul_high_u_v8i16(<16 x i8> %x, <16 x i8> %y) {
   %a = call <8 x i16> @llvm.wasm.extmul.high.unsigned.v8i16(
@@ -310,9 +298,9 @@ define <8 x i16> @extmul_high_u_v8i16(<16 x i8> %x, <16 x i8> %y) {
 }
 
 ; CHECK-LABEL: extadd_pairwise_s_v8i16:
-; SIMD128-NEXT: .functype extadd_pairwise_s_v8i16 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.extadd_pairwise_i8x16_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extadd_pairwise_s_v8i16 (v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.extadd_pairwise_i8x16_s $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.extadd.pairwise.signed.v8i16(<16 x i8>)
 define <8 x i16> @extadd_pairwise_s_v8i16(<16 x i8> %x) {
   %a = call <8 x i16> @llvm.wasm.extadd.pairwise.signed.v8i16(<16 x i8> %x)
@@ -320,9 +308,9 @@ define <8 x i16> @extadd_pairwise_s_v8i16(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: extadd_pairwise_u_v8i16:
-; SIMD128-NEXT: .functype extadd_pairwise_u_v8i16 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.extadd_pairwise_i8x16_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extadd_pairwise_u_v8i16 (v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.extadd_pairwise_i8x16_u $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.extadd.pairwise.unsigned.v8i16(<16 x i8>)
 define <8 x i16> @extadd_pairwise_u_v8i16(<16 x i8> %x) {
   %a = call <8 x i16> @llvm.wasm.extadd.pairwise.unsigned.v8i16(<16 x i8> %x)
@@ -330,9 +318,9 @@ define <8 x i16> @extadd_pairwise_u_v8i16(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: any_v8i16:
-; SIMD128-NEXT: .functype any_v8i16 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v8i16 (v128) -> (i32){{$}}
+; CHECK-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.anytrue.v8i16(<8 x i16>)
 define i32 @any_v8i16(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.anytrue.v8i16(<8 x i16> %x)
@@ -340,9 +328,9 @@ define i32 @any_v8i16(<8 x i16> %x) {
 }
 
 ; CHECK-LABEL: all_v8i16:
-; SIMD128-NEXT: .functype all_v8i16 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v8i16 (v128) -> (i32){{$}}
+; CHECK-NEXT: i16x8.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.alltrue.v8i16(<8 x i16>)
 define i32 @all_v8i16(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.alltrue.v8i16(<8 x i16> %x)
@@ -350,9 +338,9 @@ define i32 @all_v8i16(<8 x i16> %x) {
 }
 
 ; CHECK-LABEL: bitmask_v8i16:
-; SIMD128-NEXT: .functype bitmask_v8i16 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.bitmask $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitmask_v8i16 (v128) -> (i32){{$}}
+; CHECK-NEXT: i16x8.bitmask $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.bitmask.v8i16(<8 x i16>)
 define i32 @bitmask_v8i16(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.bitmask.v8i16(<8 x i16> %x)
@@ -360,9 +348,9 @@ define i32 @bitmask_v8i16(<8 x i16> %x) {
 }
 
 ; CHECK-LABEL: bitselect_v8i16:
-; SIMD128-NEXT: .functype bitselect_v8i16 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitselect_v8i16 (v128, v128, v128) -> (v128){{$}}
+; CHECK-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.bitselect.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
 define <8 x i16> @bitselect_v8i16(<8 x i16> %v1, <8 x i16> %v2, <8 x i16> %c) {
   %a = call <8 x i16> @llvm.wasm.bitselect.v8i16(
@@ -371,22 +359,10 @@ define <8 x i16> @bitselect_v8i16(<8 x i16> %v1, <8 x i16> %v2, <8 x i16> %c) {
   ret <8 x i16> %a
 }
 
-; CHECK-LABEL: signselect_v8i16:
-; SIMD128-NEXT: .functype signselect_v8i16 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.signselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <8 x i16> @llvm.wasm.signselect.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
-define <8 x i16> @signselect_v8i16(<8 x i16> %v1, <8 x i16> %v2, <8 x i16> %c) {
-  %a = call <8 x i16> @llvm.wasm.signselect.v8i16(
-    <8 x i16> %v1, <8 x i16> %v2, <8 x i16> %c
-  )
-  ret <8 x i16> %a
-}
-
 ; CHECK-LABEL: narrow_signed_v8i16:
-; SIMD128-NEXT: .functype narrow_signed_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.narrow_i32x4_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype narrow_signed_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.narrow_i32x4_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.narrow.signed.v8i16.v4i32(<4 x i32>, <4 x i32>)
 define <8 x i16> @narrow_signed_v8i16(<4 x i32> %low, <4 x i32> %high) {
   %a = call <8 x i16> @llvm.wasm.narrow.signed.v8i16.v4i32(
@@ -396,9 +372,9 @@ define <8 x i16> @narrow_signed_v8i16(<4 x i32> %low, <4 x i32> %high) {
 }
 
 ; CHECK-LABEL: narrow_unsigned_v8i16:
-; SIMD128-NEXT: .functype narrow_unsigned_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.narrow_i32x4_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype narrow_unsigned_v8i16 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i16x8.narrow_i32x4_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.narrow.unsigned.v8i16.v4i32(<4 x i32>, <4 x i32>)
 define <8 x i16> @narrow_unsigned_v8i16(<4 x i32> %low, <4 x i32> %high) {
   %a = call <8 x i16> @llvm.wasm.narrow.unsigned.v8i16.v4i32(
@@ -411,9 +387,9 @@ define <8 x i16> @narrow_unsigned_v8i16(<4 x i32> %low, <4 x i32> %high) {
 ; 4 x i32
 ; ==============================================================================
 ; CHECK-LABEL: dot:
-; SIMD128-NEXT: .functype dot (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.dot_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype dot (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.dot_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x i32> @llvm.wasm.dot(<8 x i16>, <8 x i16>)
 define <4 x i32> @dot(<8 x i16> %x, <8 x i16> %y) {
   %a = call <4 x i32> @llvm.wasm.dot(<8 x i16> %x, <8 x i16> %y)
@@ -422,9 +398,9 @@ define <4 x i32> @dot(<8 x i16> %x, <8 x i16> %y) {
 
 
 ; CHECK-LABEL: extmul_low_s_v4i32:
-; SIMD128-NEXT: .functype extmul_low_s_v4i32 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.extmul_low_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_low_s_v4i32 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.extmul_low_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x i32> @llvm.wasm.extmul.low.signed.v4i32(<8 x i16>, <8 x i16>)
 define <4 x i32> @extmul_low_s_v4i32(<8 x i16> %x, <8 x i16> %y) {
   %a = call <4 x i32> @llvm.wasm.extmul.low.signed.v4i32(
@@ -434,9 +410,9 @@ define <4 x i32> @extmul_low_s_v4i32(<8 x i16> %x, <8 x i16> %y) {
 }
 
 ; CHECK-LABEL: extmul_high_s_v4i32:
-; SIMD128-NEXT: .functype extmul_high_s_v4i32 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.extmul_high_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_high_s_v4i32 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.extmul_high_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x i32> @llvm.wasm.extmul.high.signed.v4i32(<8 x i16>, <8 x i16>)
 define <4 x i32> @extmul_high_s_v4i32(<8 x i16> %x, <8 x i16> %y) {
   %a = call <4 x i32> @llvm.wasm.extmul.high.signed.v4i32(
@@ -446,9 +422,9 @@ define <4 x i32> @extmul_high_s_v4i32(<8 x i16> %x, <8 x i16> %y) {
 }
 
 ; CHECK-LABEL: extmul_low_u_v4i32:
-; SIMD128-NEXT: .functype extmul_low_u_v4i32 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.extmul_low_i16x8_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_low_u_v4i32 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.extmul_low_i16x8_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x i32> @llvm.wasm.extmul.low.unsigned.v4i32(<8 x i16>, <8 x i16>)
 define <4 x i32> @extmul_low_u_v4i32(<8 x i16> %x, <8 x i16> %y) {
   %a = call <4 x i32> @llvm.wasm.extmul.low.unsigned.v4i32(
@@ -458,9 +434,9 @@ define <4 x i32> @extmul_low_u_v4i32(<8 x i16> %x, <8 x i16> %y) {
 }
 
 ; CHECK-LABEL: extmul_high_u_v4i32:
-; SIMD128-NEXT: .functype extmul_high_u_v4i32 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.extmul_high_i16x8_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_high_u_v4i32 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.extmul_high_i16x8_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x i32> @llvm.wasm.extmul.high.unsigned.v4i32(<8 x i16>, <8 x i16>)
 define <4 x i32> @extmul_high_u_v4i32(<8 x i16> %x, <8 x i16> %y) {
   %a = call <4 x i32> @llvm.wasm.extmul.high.unsigned.v4i32(
@@ -470,9 +446,9 @@ define <4 x i32> @extmul_high_u_v4i32(<8 x i16> %x, <8 x i16> %y) {
 }
 
 ; CHECK-LABEL: extadd_pairwise_s_v4i32:
-; SIMD128-NEXT: .functype extadd_pairwise_s_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.extadd_pairwise_i16x8_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extadd_pairwise_s_v4i32 (v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.extadd_pairwise_i16x8_s $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x i32> @llvm.wasm.extadd.pairwise.signed.v4i32(<8 x i16>)
 define <4 x i32> @extadd_pairwise_s_v4i32(<8 x i16> %x) {
   %a = call <4 x i32> @llvm.wasm.extadd.pairwise.signed.v4i32(<8 x i16> %x)
@@ -480,9 +456,9 @@ define <4 x i32> @extadd_pairwise_s_v4i32(<8 x i16> %x) {
 }
 
 ; CHECK-LABEL: extadd_pairwise_u_v4i32:
-; SIMD128-NEXT: .functype extadd_pairwise_u_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.extadd_pairwise_i16x8_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extadd_pairwise_u_v4i32 (v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.extadd_pairwise_i16x8_u $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x i32> @llvm.wasm.extadd.pairwise.unsigned.v4i32(<8 x i16>)
 define <4 x i32> @extadd_pairwise_u_v4i32(<8 x i16> %x) {
   %a = call <4 x i32> @llvm.wasm.extadd.pairwise.unsigned.v4i32(<8 x i16> %x)
@@ -491,9 +467,9 @@ define <4 x i32> @extadd_pairwise_u_v4i32(<8 x i16> %x) {
 
 
 ; CHECK-LABEL: any_v4i32:
-; SIMD128-NEXT: .functype any_v4i32 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v4i32 (v128) -> (i32){{$}}
+; CHECK-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.anytrue.v4i32(<4 x i32>)
 define i32 @any_v4i32(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.anytrue.v4i32(<4 x i32> %x)
@@ -501,9 +477,9 @@ define i32 @any_v4i32(<4 x i32> %x) {
 }
 
 ; CHECK-LABEL: all_v4i32:
-; SIMD128-NEXT: .functype all_v4i32 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i32x4.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v4i32 (v128) -> (i32){{$}}
+; CHECK-NEXT: i32x4.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.alltrue.v4i32(<4 x i32>)
 define i32 @all_v4i32(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.alltrue.v4i32(<4 x i32> %x)
@@ -511,9 +487,9 @@ define i32 @all_v4i32(<4 x i32> %x) {
 }
 
 ; CHECK-LABEL: bitmask_v4i32:
-; SIMD128-NEXT: .functype bitmask_v4i32 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i32x4.bitmask $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitmask_v4i32 (v128) -> (i32){{$}}
+; CHECK-NEXT: i32x4.bitmask $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.bitmask.v4i32(<4 x i32>)
 define i32 @bitmask_v4i32(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.bitmask.v4i32(<4 x i32> %x)
@@ -521,9 +497,9 @@ define i32 @bitmask_v4i32(<4 x i32> %x) {
 }
 
 ; CHECK-LABEL: bitselect_v4i32:
-; SIMD128-NEXT: .functype bitselect_v4i32 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitselect_v4i32 (v128, v128, v128) -> (v128){{$}}
+; CHECK-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x i32> @llvm.wasm.bitselect.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
 define <4 x i32> @bitselect_v4i32(<4 x i32> %v1, <4 x i32> %v2, <4 x i32> %c) {
   %a = call <4 x i32> @llvm.wasm.bitselect.v4i32(
@@ -532,23 +508,11 @@ define <4 x i32> @bitselect_v4i32(<4 x i32> %v1, <4 x i32> %v2, <4 x i32> %c) {
   ret <4 x i32> %a
 }
 
-; CHECK-LABEL: signselect_v4i32:
-; SIMD128-NEXT: .functype signselect_v4i32 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.signselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x i32> @llvm.wasm.signselect.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
-define <4 x i32> @signselect_v4i32(<4 x i32> %v1, <4 x i32> %v2, <4 x i32> %c) {
-  %a = call <4 x i32> @llvm.wasm.signselect.v4i32(
-    <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %c
-  )
-  ret <4 x i32> %a
-}
-
 ; CHECK-LABEL: trunc_sat_s_v4i32:
-; NO-SIMD128-NOT: f32x4
-; SIMD128-NEXT: .functype trunc_sat_s_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.trunc_sat_f32x4_s $push[[R:[0-9]+]]=, $0
-; SIMD128-NEXT: return $pop[[R]]
+; NO-CHECK-NOT: f32x4
+; CHECK-NEXT: .functype trunc_sat_s_v4i32 (v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.trunc_sat_f32x4_s $push[[R:[0-9]+]]=, $0
+; CHECK-NEXT: return $pop[[R]]
 declare <4 x i32> @llvm.wasm.trunc.saturate.signed.v4i32.v4f32(<4 x float>)
 define <4 x i32> @trunc_sat_s_v4i32(<4 x float> %x) {
   %a = call <4 x i32> @llvm.wasm.trunc.saturate.signed.v4i32.v4f32(<4 x float> %x)
@@ -556,10 +520,10 @@ define <4 x i32> @trunc_sat_s_v4i32(<4 x float> %x) {
 }
 
 ; CHECK-LABEL: trunc_sat_u_v4i32:
-; NO-SIMD128-NOT: f32x4
-; SIMD128-NEXT: .functype trunc_sat_u_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.trunc_sat_f32x4_u $push[[R:[0-9]+]]=, $0
-; SIMD128-NEXT: return $pop[[R]]
+; NO-CHECK-NOT: f32x4
+; CHECK-NEXT: .functype trunc_sat_u_v4i32 (v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.trunc_sat_f32x4_u $push[[R:[0-9]+]]=, $0
+; CHECK-NEXT: return $pop[[R]]
 declare <4 x i32> @llvm.wasm.trunc.saturate.unsigned.v4i32.v4f32(<4 x float>)
 define <4 x i32> @trunc_sat_u_v4i32(<4 x float> %x) {
   %a = call <4 x i32> @llvm.wasm.trunc.saturate.unsigned.v4i32.v4f32(<4 x float> %x)
@@ -567,43 +531,22 @@ define <4 x i32> @trunc_sat_u_v4i32(<4 x float> %x) {
 }
 
 ; CHECK-LABEL: trunc_sat_zero_signed_v4i32:
-; SIMD128-NEXT: .functype trunc_sat_zero_signed_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.trunc_sat_zero_f64x2_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x i32> @llvm.wasm.trunc.saturate.zero.signed(<2 x double>)
+; CHECK-NEXT: .functype trunc_sat_zero_signed_v4i32 (v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.trunc_sat_zero_f64x2_s $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.trunc.sat.zero.signed(<2 x double>)
 define <4 x i32> @trunc_sat_zero_signed_v4i32(<2 x double> %a) {
-  %v = call <4 x i32> @llvm.wasm.trunc.saturate.zero.signed(<2 x double> %a)
+  %v = call <4 x i32> @llvm.wasm.trunc.sat.zero.signed(<2 x double> %a)
   ret <4 x i32> %v
 }
 
 ; CHECK-LABEL: trunc_sat_zero_unsigned_v4i32:
-; SIMD128-NEXT: .functype trunc_sat_zero_unsigned_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.trunc_sat_zero_f64x2_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x i32> @llvm.wasm.trunc.saturate.zero.unsigned(<2 x double>)
+; CHECK-NEXT: .functype trunc_sat_zero_unsigned_v4i32 (v128) -> (v128){{$}}
+; CHECK-NEXT: i32x4.trunc_sat_zero_f64x2_u $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.trunc.sat.zero.unsigned(<2 x double>)
 define <4 x i32> @trunc_sat_zero_unsigned_v4i32(<2 x double> %a) {
-  %v = call <4 x i32> @llvm.wasm.trunc.saturate.zero.unsigned(<2 x double> %a)
-  ret <4 x i32> %v
-}
-
-
-; CHECK-LABEL: widen_signed_v4i32:
-; SIMD128-NEXT: .functype widen_signed_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.widen_i8x16_s $push[[R:[0-9]+]]=, $0, 1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x i32> @llvm.wasm.widen.signed(<16 x i8>, i32 immarg)
-define <4 x i32> @widen_signed_v4i32(<16 x i8> %x) {
-  %v = call <4 x i32> @llvm.wasm.widen.signed(<16 x i8> %x, i32 1)
-  ret <4 x i32> %v
-}
-
-; CHECK-LABEL: widen_unsigned_v4i32:
-; SIMD128-NEXT: .functype widen_unsigned_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.widen_i8x16_u $push[[R:[0-9]+]]=, $0, 1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x i32> @llvm.wasm.widen.unsigned(<16 x i8>, i32 immarg)
-define <4 x i32> @widen_unsigned_v4i32(<16 x i8> %x) {
-  %v = call <4 x i32> @llvm.wasm.widen.unsigned(<16 x i8> %x, i32 1)
+  %v = call <4 x i32> @llvm.wasm.trunc.sat.zero.unsigned(<2 x double> %a)
   ret <4 x i32> %v
 }
 
@@ -611,59 +554,59 @@ define <4 x i32> @widen_unsigned_v4i32(<16 x i8> %x) {
 ; 2 x i64
 ; ==============================================================================
 ; CHECK-LABEL: eq_v2i64:
-; SIMD128-NEXT: .functype eq_v2i64 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.eq $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype eq_v2i64 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i64x2.eq $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x i64> @llvm.wasm.eq(<2 x i64>, <2 x i64>)
 define <2 x i64> @eq_v2i64(<2 x i64> %x, <2 x i64> %y) {
   %a = call <2 x i64> @llvm.wasm.eq(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %a
 }
 
-; CHECK-LABEL: widen_low_s_v2i64:
-; SIMD128-NEXT: .functype widen_low_s_v2i64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.widen_low_i32x4_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <2 x i64> @llvm.wasm.widen.low.signed(<4 x i32>)
-define <2 x i64> @widen_low_s_v2i64(<4 x i32> %x) {
-  %a = call <2 x i64> @llvm.wasm.widen.low.signed(<4 x i32> %x)
+; CHECK-LABEL: extend_low_s_v2i64:
+; CHECK-NEXT: .functype extend_low_s_v2i64 (v128) -> (v128){{$}}
+; CHECK-NEXT: i64x2.extend_low_i32x4_s $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <2 x i64> @llvm.wasm.extend.low.signed(<4 x i32>)
+define <2 x i64> @extend_low_s_v2i64(<4 x i32> %x) {
+  %a = call <2 x i64> @llvm.wasm.extend.low.signed(<4 x i32> %x)
   ret <2 x i64> %a
 }
 
-; CHECK-LABEL: widen_high_s_v2i64:
-; SIMD128-NEXT: .functype widen_high_s_v2i64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.widen_high_i32x4_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <2 x i64> @llvm.wasm.widen.high.signed(<4 x i32>)
-define <2 x i64> @widen_high_s_v2i64(<4 x i32> %x) {
-  %a = call <2 x i64> @llvm.wasm.widen.high.signed(<4 x i32> %x)
+; CHECK-LABEL: extend_high_s_v2i64:
+; CHECK-NEXT: .functype extend_high_s_v2i64 (v128) -> (v128){{$}}
+; CHECK-NEXT: i64x2.extend_high_i32x4_s $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <2 x i64> @llvm.wasm.extend.high.signed(<4 x i32>)
+define <2 x i64> @extend_high_s_v2i64(<4 x i32> %x) {
+  %a = call <2 x i64> @llvm.wasm.extend.high.signed(<4 x i32> %x)
   ret <2 x i64> %a
 }
 
-; CHECK-LABEL: widen_low_u_v2i64:
-; SIMD128-NEXT: .functype widen_low_u_v2i64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.widen_low_i32x4_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <2 x i64> @llvm.wasm.widen.low.unsigned(<4 x i32>)
-define <2 x i64> @widen_low_u_v2i64(<4 x i32> %x) {
-  %a = call <2 x i64> @llvm.wasm.widen.low.unsigned(<4 x i32> %x)
+; CHECK-LABEL: extend_low_u_v2i64:
+; CHECK-NEXT: .functype extend_low_u_v2i64 (v128) -> (v128){{$}}
+; CHECK-NEXT: i64x2.extend_low_i32x4_u $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <2 x i64> @llvm.wasm.extend.low.unsigned(<4 x i32>)
+define <2 x i64> @extend_low_u_v2i64(<4 x i32> %x) {
+  %a = call <2 x i64> @llvm.wasm.extend.low.unsigned(<4 x i32> %x)
   ret <2 x i64> %a
 }
 
-; CHECK-LABEL: widen_high_u_v2i64:
-; SIMD128-NEXT: .functype widen_high_u_v2i64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.widen_high_i32x4_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <2 x i64> @llvm.wasm.widen.high.unsigned(<4 x i32>)
-define <2 x i64> @widen_high_u_v2i64(<4 x i32> %x) {
-  %a = call <2 x i64> @llvm.wasm.widen.high.unsigned(<4 x i32> %x)
+; CHECK-LABEL: extend_high_u_v2i64:
+; CHECK-NEXT: .functype extend_high_u_v2i64 (v128) -> (v128){{$}}
+; CHECK-NEXT: i64x2.extend_high_i32x4_u $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
+declare <2 x i64> @llvm.wasm.extend.high.unsigned(<4 x i32>)
+define <2 x i64> @extend_high_u_v2i64(<4 x i32> %x) {
+  %a = call <2 x i64> @llvm.wasm.extend.high.unsigned(<4 x i32> %x)
   ret <2 x i64> %a
 }
 
 ; CHECK-LABEL: extmul_low_s_v2i64:
-; SIMD128-NEXT: .functype extmul_low_s_v2i64 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.extmul_low_i32x4_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_low_s_v2i64 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i64x2.extmul_low_i32x4_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x i64> @llvm.wasm.extmul.low.signed.v2i64(<4 x i32>, <4 x i32>)
 define <2 x i64> @extmul_low_s_v2i64(<4 x i32> %x, <4 x i32> %y) {
   %a = call <2 x i64> @llvm.wasm.extmul.low.signed.v2i64(
@@ -673,9 +616,9 @@ define <2 x i64> @extmul_low_s_v2i64(<4 x i32> %x, <4 x i32> %y) {
 }
 
 ; CHECK-LABEL: extmul_high_s_v2i64:
-; SIMD128-NEXT: .functype extmul_high_s_v2i64 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.extmul_high_i32x4_s $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_high_s_v2i64 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i64x2.extmul_high_i32x4_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x i64> @llvm.wasm.extmul.high.signed.v2i64(<4 x i32>, <4 x i32>)
 define <2 x i64> @extmul_high_s_v2i64(<4 x i32> %x, <4 x i32> %y) {
   %a = call <2 x i64> @llvm.wasm.extmul.high.signed.v2i64(
@@ -685,9 +628,9 @@ define <2 x i64> @extmul_high_s_v2i64(<4 x i32> %x, <4 x i32> %y) {
 }
 
 ; CHECK-LABEL: extmul_low_u_v2i64:
-; SIMD128-NEXT: .functype extmul_low_u_v2i64 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.extmul_low_i32x4_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_low_u_v2i64 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i64x2.extmul_low_i32x4_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x i64> @llvm.wasm.extmul.low.unsigned.v2i64(<4 x i32>, <4 x i32>)
 define <2 x i64> @extmul_low_u_v2i64(<4 x i32> %x, <4 x i32> %y) {
   %a = call <2 x i64> @llvm.wasm.extmul.low.unsigned.v2i64(
@@ -697,9 +640,9 @@ define <2 x i64> @extmul_low_u_v2i64(<4 x i32> %x, <4 x i32> %y) {
 }
 
 ; CHECK-LABEL: extmul_high_u_v2i64:
-; SIMD128-NEXT: .functype extmul_high_u_v2i64 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.extmul_high_i32x4_u $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype extmul_high_u_v2i64 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: i64x2.extmul_high_i32x4_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x i64> @llvm.wasm.extmul.high.unsigned.v2i64(<4 x i32>, <4 x i32>)
 define <2 x i64> @extmul_high_u_v2i64(<4 x i32> %x, <4 x i32> %y) {
   %a = call <2 x i64> @llvm.wasm.extmul.high.unsigned.v2i64(
@@ -709,9 +652,9 @@ define <2 x i64> @extmul_high_u_v2i64(<4 x i32> %x, <4 x i32> %y) {
 }
 
 ; CHECK-LABEL: any_v2i64:
-; SIMD128-NEXT: .functype any_v2i64 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i64x2.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v2i64 (v128) -> (i32){{$}}
+; CHECK-NEXT: i64x2.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.anytrue.v2i64(<2 x i64>)
 define i32 @any_v2i64(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.anytrue.v2i64(<2 x i64> %x)
@@ -719,9 +662,9 @@ define i32 @any_v2i64(<2 x i64> %x) {
 }
 
 ; CHECK-LABEL: all_v2i64:
-; SIMD128-NEXT: .functype all_v2i64 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i64x2.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v2i64 (v128) -> (i32){{$}}
+; CHECK-NEXT: i64x2.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.alltrue.v2i64(<2 x i64>)
 define i32 @all_v2i64(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.alltrue.v2i64(<2 x i64> %x)
@@ -729,9 +672,9 @@ define i32 @all_v2i64(<2 x i64> %x) {
 }
 
 ; CHECK-LABEL: bitmask_v2i64:
-; SIMD128-NEXT: .functype bitmask_v2i64 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i64x2.bitmask $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitmask_v2i64 (v128) -> (i32){{$}}
+; CHECK-NEXT: i64x2.bitmask $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare i32 @llvm.wasm.bitmask.v2i64(<2 x i64>)
 define i32 @bitmask_v2i64(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.bitmask.v2i64(<2 x i64> %x)
@@ -739,9 +682,9 @@ define i32 @bitmask_v2i64(<2 x i64> %x) {
 }
 
 ; CHECK-LABEL: bitselect_v2i64:
-; SIMD128-NEXT: .functype bitselect_v2i64 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitselect_v2i64 (v128, v128, v128) -> (v128){{$}}
+; CHECK-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x i64> @llvm.wasm.bitselect.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
 define <2 x i64> @bitselect_v2i64(<2 x i64> %v1, <2 x i64> %v2, <2 x i64> %c) {
   %a = call <2 x i64> @llvm.wasm.bitselect.v2i64(
@@ -750,25 +693,13 @@ define <2 x i64> @bitselect_v2i64(<2 x i64> %v1, <2 x i64> %v2, <2 x i64> %c) {
   ret <2 x i64> %a
 }
 
-; CHECK-LABEL: signselect_v2i64:
-; SIMD128-NEXT: .functype signselect_v2i64 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.signselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <2 x i64> @llvm.wasm.signselect.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
-define <2 x i64> @signselect_v2i64(<2 x i64> %v1, <2 x i64> %v2, <2 x i64> %c) {
-  %a = call <2 x i64> @llvm.wasm.signselect.v2i64(
-    <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %c
-  )
-  ret <2 x i64> %a
-}
-
 ; ==============================================================================
 ; 4 x f32
 ; ==============================================================================
 ; CHECK-LABEL: bitselect_v4f32:
-; SIMD128-NEXT: .functype bitselect_v4f32 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitselect_v4f32 (v128, v128, v128) -> (v128){{$}}
+; CHECK-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x float> @llvm.wasm.bitselect.v4f32(<4 x float>, <4 x float>, <4 x float>)
 define <4 x float> @bitselect_v4f32(<4 x float> %v1, <4 x float> %v2, <4 x float> %c) {
   %a = call <4 x float> @llvm.wasm.bitselect.v4f32(
@@ -778,9 +709,9 @@ define <4 x float> @bitselect_v4f32(<4 x float> %v1, <4 x float> %v2, <4 x float
 }
 
 ; CHECK-LABEL: pmin_v4f32:
-; SIMD128-NEXT: .functype pmin_v4f32 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.pmin $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype pmin_v4f32 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: f32x4.pmin $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x float> @llvm.wasm.pmin.v4f32(<4 x float>, <4 x float>)
 define <4 x float> @pmin_v4f32(<4 x float> %a, <4 x float> %b) {
   %v = call <4 x float> @llvm.wasm.pmin.v4f32(<4 x float> %a, <4 x float> %b)
@@ -788,9 +719,9 @@ define <4 x float> @pmin_v4f32(<4 x float> %a, <4 x float> %b) {
 }
 
 ; CHECK-LABEL: pmax_v4f32:
-; SIMD128-NEXT: .functype pmax_v4f32 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.pmax $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype pmax_v4f32 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: f32x4.pmax $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x float> @llvm.wasm.pmax.v4f32(<4 x float>, <4 x float>)
 define <4 x float> @pmax_v4f32(<4 x float> %a, <4 x float> %b) {
   %v = call <4 x float> @llvm.wasm.pmax.v4f32(<4 x float> %a, <4 x float> %b)
@@ -798,9 +729,9 @@ define <4 x float> @pmax_v4f32(<4 x float> %a, <4 x float> %b) {
 }
 
 ; CHECK-LABEL: ceil_v4f32:
-; SIMD128-NEXT: .functype ceil_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.ceil $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype ceil_v4f32 (v128) -> (v128){{$}}
+; CHECK-NEXT: f32x4.ceil $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x float> @llvm.wasm.ceil.v4f32(<4 x float>)
 define <4 x float> @ceil_v4f32(<4 x float> %a) {
   %v = call <4 x float> @llvm.wasm.ceil.v4f32(<4 x float> %a)
@@ -808,9 +739,9 @@ define <4 x float> @ceil_v4f32(<4 x float> %a) {
 }
 
 ; CHECK-LABEL: floor_v4f32:
-; SIMD128-NEXT: .functype floor_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.floor $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype floor_v4f32 (v128) -> (v128){{$}}
+; CHECK-NEXT: f32x4.floor $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x float> @llvm.wasm.floor.v4f32(<4 x float>)
 define <4 x float> @floor_v4f32(<4 x float> %a) {
   %v = call <4 x float> @llvm.wasm.floor.v4f32(<4 x float> %a)
@@ -818,9 +749,9 @@ define <4 x float> @floor_v4f32(<4 x float> %a) {
 }
 
 ; CHECK-LABEL: trunc_v4f32:
-; SIMD128-NEXT: .functype trunc_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.trunc $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype trunc_v4f32 (v128) -> (v128){{$}}
+; CHECK-NEXT: f32x4.trunc $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x float> @llvm.wasm.trunc.v4f32(<4 x float>)
 define <4 x float> @trunc_v4f32(<4 x float> %a) {
   %v = call <4 x float> @llvm.wasm.trunc.v4f32(<4 x float> %a)
@@ -828,43 +759,19 @@ define <4 x float> @trunc_v4f32(<4 x float> %a) {
 }
 
 ; CHECK-LABEL: nearest_v4f32:
-; SIMD128-NEXT: .functype nearest_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.nearest $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype nearest_v4f32 (v128) -> (v128){{$}}
+; CHECK-NEXT: f32x4.nearest $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x float> @llvm.wasm.nearest.v4f32(<4 x float>)
 define <4 x float> @nearest_v4f32(<4 x float> %a) {
   %v = call <4 x float> @llvm.wasm.nearest.v4f32(<4 x float> %a)
   ret <4 x float> %v
 }
 
-; CHECK-LABEL: qfma_v4f32:
-; SIMD128-NEXT: .functype qfma_v4f32 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.qfma $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x float> @llvm.wasm.qfma.v4f32(<4 x float>, <4 x float>, <4 x float>)
-define <4 x float> @qfma_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
-  %v = call <4 x float> @llvm.wasm.qfma.v4f32(
-    <4 x float> %a, <4 x float> %b, <4 x float> %c
-  )
-  ret <4 x float> %v
-}
-
-; CHECK-LABEL: qfms_v4f32:
-; SIMD128-NEXT: .functype qfms_v4f32 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.qfms $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <4 x float> @llvm.wasm.qfms.v4f32(<4 x float>, <4 x float>, <4 x float>)
-define <4 x float> @qfms_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
-  %v = call <4 x float> @llvm.wasm.qfms.v4f32(
-    <4 x float> %a, <4 x float> %b, <4 x float> %c
-  )
-  ret <4 x float> %v
-}
-
 ; CHECK-LABEL: demote_zero_v4f32:
-; SIMD128-NEXT: .functype demote_zero_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.demote_zero_f64x2 $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype demote_zero_v4f32 (v128) -> (v128){{$}}
+; CHECK-NEXT: f32x4.demote_zero_f64x2 $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <4 x float> @llvm.wasm.demote.zero(<2 x double>)
 define <4 x float> @demote_zero_v4f32(<2 x double> %a) {
   %v = call <4 x float> @llvm.wasm.demote.zero(<2 x double> %a)
@@ -875,9 +782,9 @@ define <4 x float> @demote_zero_v4f32(<2 x double> %a) {
 ; 2 x f64
 ; ==============================================================================
 ; CHECK-LABEL: bitselect_v2f64:
-; SIMD128-NEXT: .functype bitselect_v2f64 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype bitselect_v2f64 (v128, v128, v128) -> (v128){{$}}
+; CHECK-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.bitselect.v2f64(<2 x double>, <2 x double>, <2 x double>)
 define <2 x double> @bitselect_v2f64(<2 x double> %v1, <2 x double> %v2, <2 x double> %c) {
   %a = call <2 x double> @llvm.wasm.bitselect.v2f64(
@@ -887,9 +794,9 @@ define <2 x double> @bitselect_v2f64(<2 x double> %v1, <2 x double> %v2, <2 x do
 }
 
 ; CHECK-LABEL: pmin_v2f64:
-; SIMD128-NEXT: .functype pmin_v2f64 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.pmin $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype pmin_v2f64 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: f64x2.pmin $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.pmin.v2f64(<2 x double>, <2 x double>)
 define <2 x double> @pmin_v2f64(<2 x double> %a, <2 x double> %b) {
   %v = call <2 x double> @llvm.wasm.pmin.v2f64(<2 x double> %a, <2 x double> %b)
@@ -897,9 +804,9 @@ define <2 x double> @pmin_v2f64(<2 x double> %a, <2 x double> %b) {
 }
 
 ; CHECK-LABEL: pmax_v2f64:
-; SIMD128-NEXT: .functype pmax_v2f64 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.pmax $push[[R:[0-9]+]]=, $0, $1{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype pmax_v2f64 (v128, v128) -> (v128){{$}}
+; CHECK-NEXT: f64x2.pmax $push[[R:[0-9]+]]=, $0, $1{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.pmax.v2f64(<2 x double>, <2 x double>)
 define <2 x double> @pmax_v2f64(<2 x double> %a, <2 x double> %b) {
   %v = call <2 x double> @llvm.wasm.pmax.v2f64(<2 x double> %a, <2 x double> %b)
@@ -907,9 +814,9 @@ define <2 x double> @pmax_v2f64(<2 x double> %a, <2 x double> %b) {
 }
 
 ; CHECK-LABEL: ceil_v2f64:
-; SIMD128-NEXT: .functype ceil_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.ceil $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype ceil_v2f64 (v128) -> (v128){{$}}
+; CHECK-NEXT: f64x2.ceil $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.ceil.v2f64(<2 x double>)
 define <2 x double> @ceil_v2f64(<2 x double> %a) {
   %v = call <2 x double> @llvm.wasm.ceil.v2f64(<2 x double> %a)
@@ -917,9 +824,9 @@ define <2 x double> @ceil_v2f64(<2 x double> %a) {
 }
 
 ; CHECK-LABEL: floor_v2f64:
-; SIMD128-NEXT: .functype floor_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.floor $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype floor_v2f64 (v128) -> (v128){{$}}
+; CHECK-NEXT: f64x2.floor $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.floor.v2f64(<2 x double>)
 define <2 x double> @floor_v2f64(<2 x double> %a) {
   %v = call <2 x double> @llvm.wasm.floor.v2f64(<2 x double> %a)
@@ -927,9 +834,9 @@ define <2 x double> @floor_v2f64(<2 x double> %a) {
 }
 
 ; CHECK-LABEL: trunc_v2f64:
-; SIMD128-NEXT: .functype trunc_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.trunc $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype trunc_v2f64 (v128) -> (v128){{$}}
+; CHECK-NEXT: f64x2.trunc $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.trunc.v2f64(<2 x double>)
 define <2 x double> @trunc_v2f64(<2 x double> %a) {
   %v = call <2 x double> @llvm.wasm.trunc.v2f64(<2 x double> %a)
@@ -937,43 +844,19 @@ define <2 x double> @trunc_v2f64(<2 x double> %a) {
 }
 
 ; CHECK-LABEL: nearest_v2f64:
-; SIMD128-NEXT: .functype nearest_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.nearest $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype nearest_v2f64 (v128) -> (v128){{$}}
+; CHECK-NEXT: f64x2.nearest $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.nearest.v2f64(<2 x double>)
 define <2 x double> @nearest_v2f64(<2 x double> %a) {
   %v = call <2 x double> @llvm.wasm.nearest.v2f64(<2 x double> %a)
   ret <2 x double> %v
 }
 
-; CHECK-LABEL: qfma_v2f64:
-; SIMD128-NEXT: .functype qfma_v2f64 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.qfma $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <2 x double> @llvm.wasm.qfma.v2f64(<2 x double>, <2 x double>, <2 x double>)
-define <2 x double> @qfma_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
-  %v = call <2 x double> @llvm.wasm.qfma.v2f64(
-    <2 x double> %a, <2 x double> %b, <2 x double> %c
-  )
-  ret <2 x double> %v
-}
-
-; CHECK-LABEL: qfms_v2f64:
-; SIMD128-NEXT: .functype qfms_v2f64 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.qfms $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
-declare <2 x double> @llvm.wasm.qfms.v2f64(<2 x double>, <2 x double>, <2 x double>)
-define <2 x double> @qfms_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
-  %v = call <2 x double> @llvm.wasm.qfms.v2f64(
-    <2 x double> %a, <2 x double> %b, <2 x double> %c
-  )
-  ret <2 x double> %v
-}
-
 ; CHECK-LABEL: convert_low_signed_v2f64:
-; SIMD128-NEXT: .functype convert_low_signed_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.convert_low_i32x4_s $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype convert_low_signed_v2f64 (v128) -> (v128){{$}}
+; CHECK-NEXT: f64x2.convert_low_i32x4_s $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.convert.low.signed(<4 x i32>)
 define <2 x double> @convert_low_signed_v2f64(<4 x i32> %a) {
   %v = call <2 x double> @llvm.wasm.convert.low.signed(<4 x i32> %a)
@@ -981,9 +864,9 @@ define <2 x double> @convert_low_signed_v2f64(<4 x i32> %a) {
 }
 
 ; CHECK-LABEL: convert_low_unsigned_v2f64:
-; SIMD128-NEXT: .functype convert_low_unsigned_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.convert_low_i32x4_u $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype convert_low_unsigned_v2f64 (v128) -> (v128){{$}}
+; CHECK-NEXT: f64x2.convert_low_i32x4_u $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.convert.low.unsigned(<4 x i32>)
 define <2 x double> @convert_low_unsigned_v2f64(<4 x i32> %a) {
   %v = call <2 x double> @llvm.wasm.convert.low.unsigned(<4 x i32> %a)
@@ -991,9 +874,9 @@ define <2 x double> @convert_low_unsigned_v2f64(<4 x i32> %a) {
 }
 
 ; CHECK-LABEL: promote_low_v2f64:
-; SIMD128-NEXT: .functype promote_low_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.promote_low_f32x4 $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype promote_low_v2f64 (v128) -> (v128){{$}}
+; CHECK-NEXT: f64x2.promote_low_f32x4 $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.promote.low(<4 x float>)
 define <2 x double> @promote_low_v2f64(<4 x float> %a) {
   %v = call <2 x double> @llvm.wasm.promote.low(<4 x float> %a)
diff --git a/llvm/test/CodeGen/WebAssembly/simd-load-splat.ll b/llvm/test/CodeGen/WebAssembly/simd-load-splat.ll
index 3d08a586edb5a0c1180196e471846c350ff51c33..f976ac2630d5d86f494df655338b1b1b1985c2f0 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-load-splat.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-load-splat.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s
 
 ; Regression test for an ISel failure when a splatted load had more
 ; than one use. The main tests for load_splat are in simd-offset.ll.
diff --git a/llvm/test/CodeGen/WebAssembly/simd-load-store-alignment.ll b/llvm/test/CodeGen/WebAssembly/simd-load-store-alignment.ll
index 000b7730e3bf2a9b4e63e865c98c77c075497fda..9e16392c396e7f7c9681278c66cd261986be48fe 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-load-store-alignment.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-load-store-alignment.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
 
 ; Test loads and stores with custom alignment values.
 
diff --git a/llvm/test/CodeGen/WebAssembly/simd-noopt.ll b/llvm/test/CodeGen/WebAssembly/simd-noopt.ll
deleted file mode 100644
index 1ec259ccca73127afe4a57e91fe6d1945e9f1833..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/WebAssembly/simd-noopt.ll
+++ /dev/null
@@ -1,20 +0,0 @@
-; RUN: llc < %s -fast-isel -mattr=+simd128,+sign-ext -verify-machineinstrs
-
-;; Ensures fastisel produces valid code when storing and loading split
-;; up v2i64 values. Lowering away v2i64s is a temporary measure while
-;; V8 does not have support for i64x2.* operations, and is done when
-;; -wasm-enable-unimplemented-simd is not present. This is a
-;; regression test for a bug that crashed llc after fastisel produced
-;; machineinstrs that used registers that had never been defined.
-
-target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-target triple = "wasm32-unknown-unknown"
-
-define i64 @foo(<2 x i64> %vec) {
-entry:
-  %vec.addr = alloca <2 x i64>, align 16
-  store <2 x i64> %vec, <2 x i64>* %vec.addr, align 16
-  %0 = load <2 x i64>, <2 x i64>* %vec.addr, align 16
-  %1 = extractelement <2 x i64> %0, i32 0
-  ret i64 %1
-}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-prefetch-offset.ll b/llvm/test/CodeGen/WebAssembly/simd-prefetch-offset.ll
deleted file mode 100644
index f3b54481c0e476a9a69a232109e5d357bda6d47d..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/WebAssembly/simd-prefetch-offset.ll
+++ /dev/null
@@ -1,235 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -verify-machineinstrs -mattr=+simd128 | FileCheck %s
-
-; Test experimental prefetch instructions
-
-target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-target triple = "wasm32-unknown-unknown"
-
-declare void @llvm.wasm.prefetch.t(i8*)
-declare void @llvm.wasm.prefetch.nt(i8*)
-@gv = global i8 0
-
-;===----------------------------------------------------------------------------
-; prefetch.t
-;===----------------------------------------------------------------------------
-
-define void @prefetch_t_no_offset(i8* %p) {
-; CHECK-LABEL: prefetch_t_no_offset:
-; CHECK:         .functype prefetch_t_no_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    prefetch.t 0
-; CHECK-NEXT:    # fallthrough-return
-  tail call void @llvm.wasm.prefetch.t(i8* %p)
-  ret void
-}
-
-define void @prefetch_t_with_folded_offset(i8* %p) {
-; CHECK-LABEL: prefetch_t_with_folded_offset:
-; CHECK:         .functype prefetch_t_with_folded_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.t 0
-; CHECK-NEXT:    # fallthrough-return
-  %q = ptrtoint i8* %p to i32
-  %r = add nuw i32 %q, 24
-  %s = inttoptr i32 %r to i8*
-  tail call void @llvm.wasm.prefetch.t(i8* %s)
-  ret void
-}
-
-define void @prefetch_t_with_folded_gep_offset(i8* %p) {
-; CHECK-LABEL: prefetch_t_with_folded_gep_offset:
-; CHECK:         .functype prefetch_t_with_folded_gep_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const 6
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.t 0
-; CHECK-NEXT:    # fallthrough-return
-  %s = getelementptr inbounds i8, i8* %p, i32 6
-  tail call void @llvm.wasm.prefetch.t(i8* %s)
-  ret void
-}
-
-define void @prefetch_t_with_unfolded_gep_negative_offset(i8* %p) {
-; CHECK-LABEL: prefetch_t_with_unfolded_gep_negative_offset:
-; CHECK:         .functype prefetch_t_with_unfolded_gep_negative_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const -6
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.t 0
-; CHECK-NEXT:    # fallthrough-return
-  %s = getelementptr inbounds i8, i8* %p, i32 -6
-  tail call void @llvm.wasm.prefetch.t(i8* %s)
-  ret void
-}
-
-define void @prefetch_t_with_unfolded_offset(i8* %p) {
-; CHECK-LABEL: prefetch_t_with_unfolded_offset:
-; CHECK:         .functype prefetch_t_with_unfolded_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.t 0
-; CHECK-NEXT:    # fallthrough-return
-  %q = ptrtoint i8* %p to i32
-  %r = add nsw i32 %q, 24
-  %s = inttoptr i32 %r to i8*
-  tail call void @llvm.wasm.prefetch.t(i8* %s)
-  ret void
-}
-
-define void @prefetch_t_with_unfolded_gep_offset(i8* %p) {
-; CHECK-LABEL: prefetch_t_with_unfolded_gep_offset:
-; CHECK:         .functype prefetch_t_with_unfolded_gep_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const 6
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.t 0
-; CHECK-NEXT:    # fallthrough-return
-  %s = getelementptr i8, i8* %p, i32 6
-  tail call void @llvm.wasm.prefetch.t(i8* %s)
-  ret void
-}
-
-define void @prefetch_t_from_numeric_address() {
-; CHECK-LABEL: prefetch_t_from_numeric_address:
-; CHECK:         .functype prefetch_t_from_numeric_address () -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const 42
-; CHECK-NEXT:    prefetch.t 0
-; CHECK-NEXT:    # fallthrough-return
-  %s = inttoptr i32 42 to i8*
-  tail call void @llvm.wasm.prefetch.t(i8* %s)
-  ret void
-}
-
-define void @prefetch_t_from_global_address() {
-; CHECK-LABEL: prefetch_t_from_global_address:
-; CHECK:         .functype prefetch_t_from_global_address () -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const gv
-; CHECK-NEXT:    prefetch.t 0
-; CHECK-NEXT:    # fallthrough-return
-  tail call void @llvm.wasm.prefetch.t(i8* @gv)
-  ret void
-}
-
-;===----------------------------------------------------------------------------
-; prefetch.nt
-;===----------------------------------------------------------------------------
-
-define void @prefetch_nt_no_offset(i8* %p) {
-; CHECK-LABEL: prefetch_nt_no_offset:
-; CHECK:         .functype prefetch_nt_no_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    prefetch.nt 0
-; CHECK-NEXT:    # fallthrough-return
-  tail call void @llvm.wasm.prefetch.nt(i8* %p)
-  ret void
-}
-
-define void @prefetch_nt_with_folded_offset(i8* %p) {
-; CHECK-LABEL: prefetch_nt_with_folded_offset:
-; CHECK:         .functype prefetch_nt_with_folded_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.nt 0
-; CHECK-NEXT:    # fallthrough-return
-  %q = ptrtoint i8* %p to i32
-  %r = add nuw i32 %q, 24
-  %s = inttoptr i32 %r to i8*
-  tail call void @llvm.wasm.prefetch.nt(i8* %s)
-  ret void
-}
-
-define void @prefetch_nt_with_folded_gep_offset(i8* %p) {
-; CHECK-LABEL: prefetch_nt_with_folded_gep_offset:
-; CHECK:         .functype prefetch_nt_with_folded_gep_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const 6
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.nt 0
-; CHECK-NEXT:    # fallthrough-return
-  %s = getelementptr inbounds i8, i8* %p, i64 6
-  tail call void @llvm.wasm.prefetch.nt(i8* %s)
-  ret void
-}
-
-define void @prefetch_nt_with_unfolded_gep_negative_offset(i8* %p) {
-; CHECK-LABEL: prefetch_nt_with_unfolded_gep_negative_offset:
-; CHECK:         .functype prefetch_nt_with_unfolded_gep_negative_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const -6
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.nt 0
-; CHECK-NEXT:    # fallthrough-return
-  %s = getelementptr inbounds i8, i8* %p, i64 -6
-  tail call void @llvm.wasm.prefetch.nt(i8* %s)
-  ret void
-}
-
-define void @prefetch_nt_with_unfolded_offset(i8* %p) {
-; CHECK-LABEL: prefetch_nt_with_unfolded_offset:
-; CHECK:         .functype prefetch_nt_with_unfolded_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.nt 0
-; CHECK-NEXT:    # fallthrough-return
-  %q = ptrtoint i8* %p to i32
-  %r = add nsw i32 %q, 24
-  %s = inttoptr i32 %r to i8*
-  tail call void @llvm.wasm.prefetch.nt(i8* %s)
-  ret void
-}
-
-define void @prefetch_nt_with_unfolded_gep_offset(i8* %p) {
-; CHECK-LABEL: prefetch_nt_with_unfolded_gep_offset:
-; CHECK:         .functype prefetch_nt_with_unfolded_gep_offset (i32) -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i32.const 6
-; CHECK-NEXT:    i32.add
-; CHECK-NEXT:    prefetch.nt 0
-; CHECK-NEXT:    # fallthrough-return
-  %s = getelementptr i8, i8* %p, i64 6
-  tail call void @llvm.wasm.prefetch.nt(i8* %s)
-  ret void
-}
-
-define void @prefetch_nt_from_numeric_address() {
-; CHECK-LABEL: prefetch_nt_from_numeric_address:
-; CHECK:         .functype prefetch_nt_from_numeric_address () -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const 42
-; CHECK-NEXT:    prefetch.nt 0
-; CHECK-NEXT:    # fallthrough-return
-  %s = inttoptr i32 42 to i8*
-  tail call void @llvm.wasm.prefetch.nt(i8* %s)
-  ret void
-}
-
-define void @prefetch_nt_from_global_address() {
-; CHECK-LABEL: prefetch_nt_from_global_address:
-; CHECK:         .functype prefetch_nt_from_global_address () -> ()
-; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i32.const gv
-; CHECK-NEXT:    prefetch.nt 0
-; CHECK-NEXT:    # fallthrough-return
-  tail call void @llvm.wasm.prefetch.nt(i8* @gv)
-  ret void
-}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-reductions.ll b/llvm/test/CodeGen/WebAssembly/simd-reductions.ll
index 037cac514b03f30fc37fa555ea2c57ec6fa2645f..259ef3b3a81fef9902a8c90a4289dd1c3e0e86cf 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-reductions.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-reductions.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
 
 ; Tests that redundant masking and conversions are folded out
 ; following SIMD reduction instructions.
@@ -13,9 +13,9 @@ declare i32 @llvm.wasm.anytrue.v16i8(<16 x i8>)
 declare i32 @llvm.wasm.alltrue.v16i8(<16 x i8>)
 
 ; CHECK-LABEL: any_v16i8_trunc:
-; SIMD128-NEXT: .functype any_v16i8_trunc (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v16i8_trunc (v128) -> (i32){{$}}
+; CHECK-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v16i8_trunc(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.anytrue.v16i8(<16 x i8> %x)
   %b = trunc i32 %a to i1
@@ -24,9 +24,9 @@ define i32 @any_v16i8_trunc(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: any_v16i8_ne:
-; SIMD128-NEXT: .functype any_v16i8_ne (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v16i8_ne (v128) -> (i32){{$}}
+; CHECK-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v16i8_ne(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.anytrue.v16i8(<16 x i8> %x)
   %b = icmp ne i32 %a, 0
@@ -35,9 +35,9 @@ define i32 @any_v16i8_ne(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: any_v16i8_eq:
-; SIMD128-NEXT: .functype any_v16i8_eq (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v16i8_eq (v128) -> (i32){{$}}
+; CHECK-NEXT: i8x16.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v16i8_eq(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.anytrue.v16i8(<16 x i8> %x)
   %b = icmp eq i32 %a, 1
@@ -46,9 +46,9 @@ define i32 @any_v16i8_eq(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: all_v16i8_trunc:
-; SIMD128-NEXT: .functype all_v16i8_trunc (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v16i8_trunc (v128) -> (i32){{$}}
+; CHECK-NEXT: i8x16.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v16i8_trunc(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.alltrue.v16i8(<16 x i8> %x)
   %b = trunc i32 %a to i1
@@ -57,9 +57,9 @@ define i32 @all_v16i8_trunc(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: all_v16i8_ne:
-; SIMD128-NEXT: .functype all_v16i8_ne (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v16i8_ne (v128) -> (i32){{$}}
+; CHECK-NEXT: i8x16.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v16i8_ne(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.alltrue.v16i8(<16 x i8> %x)
   %b = icmp ne i32 %a, 0
@@ -68,9 +68,9 @@ define i32 @all_v16i8_ne(<16 x i8> %x) {
 }
 
 ; CHECK-LABEL: all_v16i8_eq:
-; SIMD128-NEXT: .functype all_v16i8_eq (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v16i8_eq (v128) -> (i32){{$}}
+; CHECK-NEXT: i8x16.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v16i8_eq(<16 x i8> %x) {
   %a = call i32 @llvm.wasm.alltrue.v16i8(<16 x i8> %x)
   %b = icmp eq i32 %a, 1
@@ -85,9 +85,9 @@ declare i32 @llvm.wasm.anytrue.v8i16(<8 x i16>)
 declare i32 @llvm.wasm.alltrue.v8i16(<8 x i16>)
 
 ; CHECK-LABEL: any_v8i16_trunc:
-; SIMD128-NEXT: .functype any_v8i16_trunc (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v8i16_trunc (v128) -> (i32){{$}}
+; CHECK-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v8i16_trunc(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.anytrue.v8i16(<8 x i16> %x)
   %b = trunc i32 %a to i1
@@ -96,9 +96,9 @@ define i32 @any_v8i16_trunc(<8 x i16> %x) {
 }
 
 ; CHECK-LABEL: any_v8i16_ne:
-; SIMD128-NEXT: .functype any_v8i16_ne (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v8i16_ne (v128) -> (i32){{$}}
+; CHECK-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v8i16_ne(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.anytrue.v8i16(<8 x i16> %x)
   %b = icmp ne i32 %a, 0
@@ -107,9 +107,9 @@ define i32 @any_v8i16_ne(<8 x i16> %x) {
 }
 
 ; CHECK-LABEL: any_v8i16_eq:
-; SIMD128-NEXT: .functype any_v8i16_eq (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v8i16_eq (v128) -> (i32){{$}}
+; CHECK-NEXT: i16x8.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v8i16_eq(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.anytrue.v8i16(<8 x i16> %x)
   %b = icmp eq i32 %a, 1
@@ -118,9 +118,9 @@ define i32 @any_v8i16_eq(<8 x i16> %x) {
 }
 
 ; CHECK-LABEL: all_v8i16_trunc:
-; SIMD128-NEXT: .functype all_v8i16_trunc (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v8i16_trunc (v128) -> (i32){{$}}
+; CHECK-NEXT: i16x8.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v8i16_trunc(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.alltrue.v8i16(<8 x i16> %x)
   %b = trunc i32 %a to i1
@@ -129,9 +129,9 @@ define i32 @all_v8i16_trunc(<8 x i16> %x) {
 }
 
 ; CHECK-LABEL: all_v8i16_ne:
-; SIMD128-NEXT: .functype all_v8i16_ne (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v8i16_ne (v128) -> (i32){{$}}
+; CHECK-NEXT: i16x8.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v8i16_ne(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.alltrue.v8i16(<8 x i16> %x)
   %b = icmp ne i32 %a, 0
@@ -140,9 +140,9 @@ define i32 @all_v8i16_ne(<8 x i16> %x) {
 }
 
 ; CHECK-LABEL: all_v8i16_eq:
-; SIMD128-NEXT: .functype all_v8i16_eq (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v8i16_eq (v128) -> (i32){{$}}
+; CHECK-NEXT: i16x8.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v8i16_eq(<8 x i16> %x) {
   %a = call i32 @llvm.wasm.alltrue.v8i16(<8 x i16> %x)
   %b = icmp eq i32 %a, 1
@@ -157,9 +157,9 @@ declare i32 @llvm.wasm.anytrue.v4i32(<4 x i32>)
 declare i32 @llvm.wasm.alltrue.v4i32(<4 x i32>)
 
 ; CHECK-LABEL: any_v4i32_trunc:
-; SIMD128-NEXT: .functype any_v4i32_trunc (v128) -> (i32){{$}}
-; SIMD128-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v4i32_trunc (v128) -> (i32){{$}}
+; CHECK-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v4i32_trunc(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.anytrue.v4i32(<4 x i32> %x)
   %b = trunc i32 %a to i1
@@ -168,9 +168,9 @@ define i32 @any_v4i32_trunc(<4 x i32> %x) {
 }
 
 ; CHECK-LABEL: any_v4i32_ne:
-; SIMD128-NEXT: .functype any_v4i32_ne (v128) -> (i32){{$}}
-; SIMD128-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v4i32_ne (v128) -> (i32){{$}}
+; CHECK-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v4i32_ne(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.anytrue.v4i32(<4 x i32> %x)
   %b = icmp ne i32 %a, 0
@@ -179,9 +179,9 @@ define i32 @any_v4i32_ne(<4 x i32> %x) {
 }
 
 ; CHECK-LABEL: any_v4i32_eq:
-; SIMD128-NEXT: .functype any_v4i32_eq (v128) -> (i32){{$}}
-; SIMD128-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v4i32_eq (v128) -> (i32){{$}}
+; CHECK-NEXT: i32x4.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v4i32_eq(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.anytrue.v4i32(<4 x i32> %x)
   %b = icmp eq i32 %a, 1
@@ -190,9 +190,9 @@ define i32 @any_v4i32_eq(<4 x i32> %x) {
 }
 
 ; CHECK-LABEL: all_v4i32_trunc:
-; SIMD128-NEXT: .functype all_v4i32_trunc (v128) -> (i32){{$}}
-; SIMD128-NEXT: i32x4.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v4i32_trunc (v128) -> (i32){{$}}
+; CHECK-NEXT: i32x4.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v4i32_trunc(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.alltrue.v4i32(<4 x i32> %x)
   %b = trunc i32 %a to i1
@@ -201,9 +201,9 @@ define i32 @all_v4i32_trunc(<4 x i32> %x) {
 }
 
 ; CHECK-LABEL: all_v4i32_ne:
-; SIMD128-NEXT: .functype all_v4i32_ne (v128) -> (i32){{$}}
-; SIMD128-NEXT: i32x4.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v4i32_ne (v128) -> (i32){{$}}
+; CHECK-NEXT: i32x4.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v4i32_ne(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.alltrue.v4i32(<4 x i32> %x)
   %b = icmp ne i32 %a, 0
@@ -212,9 +212,9 @@ define i32 @all_v4i32_ne(<4 x i32> %x) {
 }
 
 ; CHECK-LABEL: all_v4i32_eq:
-; SIMD128-NEXT: .functype all_v4i32_eq (v128) -> (i32){{$}}
-; SIMD128-NEXT: i32x4.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v4i32_eq (v128) -> (i32){{$}}
+; CHECK-NEXT: i32x4.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v4i32_eq(<4 x i32> %x) {
   %a = call i32 @llvm.wasm.alltrue.v4i32(<4 x i32> %x)
   %b = icmp eq i32 %a, 1
@@ -229,9 +229,9 @@ declare i32 @llvm.wasm.anytrue.v2i64(<2 x i64>)
 declare i32 @llvm.wasm.alltrue.v2i64(<2 x i64>)
 
 ; CHECK-LABEL: any_v2i64_trunc:
-; SIMD128-NEXT: .functype any_v2i64_trunc (v128) -> (i32){{$}}
-; SIMD128-NEXT: i64x2.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v2i64_trunc (v128) -> (i32){{$}}
+; CHECK-NEXT: i64x2.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v2i64_trunc(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.anytrue.v2i64(<2 x i64> %x)
   %b = trunc i32 %a to i1
@@ -240,9 +240,9 @@ define i32 @any_v2i64_trunc(<2 x i64> %x) {
 }
 
 ; CHECK-LABEL: any_v2i64_ne:
-; SIMD128-NEXT: .functype any_v2i64_ne (v128) -> (i32){{$}}
-; SIMD128-NEXT: i64x2.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v2i64_ne (v128) -> (i32){{$}}
+; CHECK-NEXT: i64x2.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v2i64_ne(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.anytrue.v2i64(<2 x i64> %x)
   %b = icmp ne i32 %a, 0
@@ -251,9 +251,9 @@ define i32 @any_v2i64_ne(<2 x i64> %x) {
 }
 
 ; CHECK-LABEL: any_v2i64_eq:
-; SIMD128-NEXT: .functype any_v2i64_eq (v128) -> (i32){{$}}
-; SIMD128-NEXT: i64x2.any_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype any_v2i64_eq (v128) -> (i32){{$}}
+; CHECK-NEXT: i64x2.any_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @any_v2i64_eq(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.anytrue.v2i64(<2 x i64> %x)
   %b = icmp eq i32 %a, 1
@@ -262,9 +262,9 @@ define i32 @any_v2i64_eq(<2 x i64> %x) {
 }
 
 ; CHECK-LABEL: all_v2i64_trunc:
-; SIMD128-NEXT: .functype all_v2i64_trunc (v128) -> (i32){{$}}
-; SIMD128-NEXT: i64x2.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v2i64_trunc (v128) -> (i32){{$}}
+; CHECK-NEXT: i64x2.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v2i64_trunc(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.alltrue.v2i64(<2 x i64> %x)
   %b = trunc i32 %a to i1
@@ -273,9 +273,9 @@ define i32 @all_v2i64_trunc(<2 x i64> %x) {
 }
 
 ; CHECK-LABEL: all_v2i64_ne:
-; SIMD128-NEXT: .functype all_v2i64_ne (v128) -> (i32){{$}}
-; SIMD128-NEXT: i64x2.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v2i64_ne (v128) -> (i32){{$}}
+; CHECK-NEXT: i64x2.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v2i64_ne(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.alltrue.v2i64(<2 x i64> %x)
   %b = icmp ne i32 %a, 0
@@ -284,9 +284,9 @@ define i32 @all_v2i64_ne(<2 x i64> %x) {
 }
 
 ; CHECK-LABEL: all_v2i64_eq:
-; SIMD128-NEXT: .functype all_v2i64_eq (v128) -> (i32){{$}}
-; SIMD128-NEXT: i64x2.all_true $push[[R:[0-9]+]]=, $0{{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
+; CHECK-NEXT: .functype all_v2i64_eq (v128) -> (i32){{$}}
+; CHECK-NEXT: i64x2.all_true $push[[R:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: return $pop[[R]]{{$}}
 define i32 @all_v2i64_eq(<2 x i64> %x) {
   %a = call i32 @llvm.wasm.alltrue.v2i64(<2 x i64> %x)
   %b = icmp eq i32 %a, 1
diff --git a/llvm/test/CodeGen/WebAssembly/simd-select.ll b/llvm/test/CodeGen/WebAssembly/simd-select.ll
index be36f94cf5a6da9f8124c8328dad591f587f341d..19d11328154ef190b3e050dfc81dfe3e378d131e 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-select.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-select.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -verify-machineinstrs -mattr=+unimplemented-simd128 | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mattr=+simd128 | FileCheck %s
 
 ; Test that vector selects of various varieties lower correctly.
 
diff --git a/llvm/test/CodeGen/WebAssembly/simd-sext-inreg.ll b/llvm/test/CodeGen/WebAssembly/simd-sext-inreg.ll
index 13f9ca14812de1f246bedf909bc640e8b2860e4d..c2f00b349489b6a5d9af34d13af68f0fc7eefc83 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-sext-inreg.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-sext-inreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
 ; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test that vector sign extensions lower to shifts
diff --git a/llvm/test/CodeGen/WebAssembly/simd-shift-complex-splats.ll b/llvm/test/CodeGen/WebAssembly/simd-shift-complex-splats.ll
index 4582bc62216a7c1a2eef642de3da2994751637b3..00a963b959ed69349e9e61cc213aea8e486771c0 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-shift-complex-splats.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-shift-complex-splats.ll
@@ -28,29 +28,11 @@ define <16 x i8> @shl_add(<16 x i8> %v, i8 %a, i8 %b) {
 
 ; CHECK-LABEL: shl_abs:
 ; CHECK-NEXT: .functype shl_abs (v128, i32) -> (v128)
-; CHECK-NEXT: i8x16.extract_lane_u $push8=, $0, 0
 ; CHECK-NEXT: i8x16.splat $push0=, $1
-; CHECK-NEXT: i8x16.abs $push98=, $pop0
-; CHECK-NEXT: local.tee $push97=, $2=, $pop98
-; CHECK-NEXT: i8x16.extract_lane_u $push6=, $pop97, 0
-; CHECK-NEXT: i32.const $push2=, 7
-; CHECK-NEXT: i32.and $push7=, $pop6, $pop2
-; CHECK-NEXT: i32.shl $push9=, $pop8, $pop7
-; CHECK-NEXT: i8x16.splat $push10=, $pop9
-; CHECK-NEXT: i8x16.extract_lane_u $push4=, $0, 1
-; CHECK-NEXT: i8x16.extract_lane_u $push1=, $2, 1
-; CHECK-NEXT: i32.const $push96=, 7
-; CHECK-NEXT: i32.and $push3=, $pop1, $pop96
-; CHECK-NEXT: i32.shl $push5=, $pop4, $pop3
-; CHECK-NEXT: i8x16.replace_lane $push11=, $pop10, 1, $pop5
-; ...
-; CHECK:      i8x16.extract_lane_u $push79=, $0, 15
-; CHECK-NEXT: i8x16.extract_lane_u $push77=, $2, 15
-; CHECK-NEXT: i32.const $push82=, 7
-; CHECK-NEXT: i32.and $push78=, $pop77, $pop82
-; CHECK-NEXT: i32.shl $push80=, $pop79, $pop78
-; CHECK-NEXT: i8x16.replace_lane $push81=, $pop76, 15, $pop80
-; CHECK-NEXT: return $pop81
+; CHECK-NEXT: i8x16.abs $push1=, $pop0
+; CHECK-NEXT: i8x16.extract_lane_u $push2=, $pop1, 0
+; CHECK-NEXT: i8x16.shl $push3=, $0, $pop2
+; CHECK-NEXT: return $pop3
 define <16 x i8> @shl_abs(<16 x i8> %v, i8 %a) {
   %t1 = insertelement <16 x i8> undef, i8 %a, i32 0
   %va = shufflevector <16 x i8> %t1, <16 x i8> undef, <16 x i32> zeroinitializer
@@ -63,32 +45,14 @@ define <16 x i8> @shl_abs(<16 x i8> %v, i8 %a) {
 
 ; CHECK-LABEL: shl_abs_add:
 ; CHECK-NEXT: .functype shl_abs_add (v128, i32, i32) -> (v128)
-; CHECK-NEXT: i8x16.extract_lane_u $push11=, $0, 0
 ; CHECK-NEXT: i8x16.splat $push1=, $1
 ; CHECK-NEXT: i8x16.splat $push0=, $2
 ; CHECK-NEXT: i8x16.add $push2=, $pop1, $pop0
 ; CHECK-NEXT: i8x16.shuffle $push3=, $pop2, $0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT: i8x16.abs $push101=, $pop3
-; CHECK-NEXT: local.tee $push100=, $3=, $pop101
-; CHECK-NEXT: i8x16.extract_lane_u $push9=, $pop100, 0
-; CHECK-NEXT: i32.const $push5=, 7
-; CHECK-NEXT: i32.and $push10=, $pop9, $pop5
-; CHECK-NEXT: i32.shl $push12=, $pop11, $pop10
-; CHECK-NEXT: i8x16.splat $push13=, $pop12
-; CHECK-NEXT: i8x16.extract_lane_u $push7=, $0, 1
-; CHECK-NEXT: i8x16.extract_lane_u $push4=, $3, 1
-; CHECK-NEXT: i32.const $push99=, 7
-; CHECK-NEXT: i32.and $push6=, $pop4, $pop99
-; CHECK-NEXT: i32.shl $push8=, $pop7, $pop6
-; CHECK-NEXT: i8x16.replace_lane $push14=, $pop13, 1, $pop8
-; ...
-; CHECK:      i8x16.extract_lane_u $push82=, $0, 15
-; CHECK-NEXT: i8x16.extract_lane_u $push80=, $3, 15
-; CHECK-NEXT: i32.const $push85=, 7
-; CHECK-NEXT: i32.and $push81=, $pop80, $pop85
-; CHECK-NEXT: i32.shl $push83=, $pop82, $pop81
-; CHECK-NEXT: i8x16.replace_lane $push84=, $pop79, 15, $pop83
-; CHECK-NEXT: return $pop84
+; CHECK-NEXT: i8x16.abs $push4=, $pop3
+; CHECK-NEXT: i8x16.extract_lane_u $push5=, $pop4, 0
+; CHECK-NEXT: i8x16.shl $push6=, $0, $pop5
+; CHECK-NEXT: return $pop6
 define <16 x i8> @shl_abs_add(<16 x i8> %v, i8 %a, i8 %b) {
   %t1 = insertelement <16 x i8> undef, i8 %a, i32 0
   %va = shufflevector <16 x i8> %t1, <16 x i8> undef, <16 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/WebAssembly/simd-unsupported.ll b/llvm/test/CodeGen/WebAssembly/simd-unsupported.ll
index dbe426b075154123c00939660fcf4a98482bf696..1fc0a92b9032d5b22ac542f54700717ef1f5ace0 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-unsupported.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-unsupported.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
 
 ; Test that operations that are not supported by SIMD are properly
 ; unrolled.
diff --git a/llvm/test/CodeGen/WebAssembly/simd.ll b/llvm/test/CodeGen/WebAssembly/simd.ll
index c8053293ebac0f08fce57caa182f19783559165d..b915b83cc132949c1a564267f86f00242aa5b528 100644
--- a/llvm/test/CodeGen/WebAssembly/simd.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd.ll
@@ -1,5 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128
 ; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test that basic SIMD128 vector manipulation operations assemble as expected.
@@ -12,7 +11,6 @@ target triple = "wasm32-unknown-unknown"
 ; ==============================================================================
 ; CHECK-LABEL: const_v16i8:
 ; NO-SIMD128-NOT: i8x16
-; SIMD128-VM-NOT: v128.const
 ; SIMD128-NEXT: .functype const_v16i8 () -> (v128){{$}}
 ; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=,
 ; SIMD128-SAME: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
@@ -277,7 +275,6 @@ define <16 x i8> @build_v16i8(i8 %x0, i8 %x1, i8 %x2, i8 %x3,
 ; ==============================================================================
 ; CHECK-LABEL: const_v8i16:
 ; NO-SIMD128-NOT: i16x8
-; SIMD128-VM-NOT: v128.const
 ; SIMD128-NEXT: .functype const_v8i16 () -> (v128){{$}}
 ; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=, 256, 770, 1284, 1798, 2312, 2826, 3340, 3854{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
@@ -526,7 +523,6 @@ define <8 x i16> @build_v8i16(i16 %x0, i16 %x1, i16 %x2, i16 %x3,
 ; ==============================================================================
 ; CHECK-LABEL: const_v4i32:
 ; NO-SIMD128-NOT: i32x4
-; SIMD128-VM-NOT: v128.const
 ; SIMD128-NEXT: .functype const_v4i32 () -> (v128){{$}}
 ; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=, 50462976, 117835012, 185207048, 252579084{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
@@ -678,7 +674,6 @@ define <4 x i32> @build_v4i32(i32 %x0, i32 %x1, i32 %x2, i32 %x3) {
 ; ==============================================================================
 ; CHECK-LABEL: const_v2i64:
 ; NO-SIMD128-NOT: i64x2
-; SIMD128-VM-NOT: v128.const
 ; SIMD128-NEXT: .functype const_v2i64 () -> (v128){{$}}
 ; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=, 506097522914230528, 1084818905618843912{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
@@ -824,7 +819,6 @@ define <2 x i64> @build_v2i64(i64 %x0, i64 %x1) {
 ; ==============================================================================
 ; CHECK-LABEL: const_v4f32:
 ; NO-SIMD128-NOT: f32x4
-; SIMD128-VM-NOT: v128.const
 ; SIMD128-NEXT: .functype const_v4f32 () -> (v128){{$}}
 ; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=,
 ; SIMD128-SAME: 0x1.0402p-121, 0x1.0c0a08p-113, 0x1.14121p-105, 0x1.1c1a18p-97{{$}}
@@ -978,7 +972,6 @@ define <4 x float> @build_v4f32(float %x0, float %x1, float %x2, float %x3) {
 ; ==============================================================================
 ; CHECK-LABEL: const_v2f64:
 ; NO-SIMD128-NOT: f64x2
-; SIMD128-VM-NOT: v128.const
 ; SIMD128-NEXT: .functype const_v2f64 () -> (v128){{$}}
 ; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=, 0x1.60504030201p-911, 0x1.e0d0c0b0a0908p-783{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
diff --git a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
index 2ba3cf23774a63afede5019be3160c21e2b54c02..b8729a200a329d9a58bce0afcdfb22cf2eeb8436 100644
--- a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll
@@ -34,20 +34,20 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalia
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; CHECK-NEXT:    movb {{[0-9]+}}(%esp), %bl
 ; CHECK-NEXT:    testb $1, %bl
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.1: ## %bb116.i
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.2: ## %bb52.i.i
 ; CHECK-NEXT:    testb $1, %bl
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.3: ## %bb142.i
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.4:
 ; CHECK-NEXT:    movl L_.str89$non_lazy_ptr, %edi
 ; CHECK-NEXT:    movb $1, %bh
-; CHECK-NEXT:    movl $274877907, %ebp ## imm = 0x10624DD3
+; CHECK-NEXT:    movl L_.str$non_lazy_ptr, %ebp
 ; CHECK-NEXT:    jmp LBB0_5
-; CHECK-NEXT:  LBB0_23: ## %bb7806
+; CHECK-NEXT:  LBB0_21: ## %bb7806
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:  Ltmp16:
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
@@ -58,7 +58,7 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalia
 ; CHECK-NEXT:  LBB0_5: ## %bb3261
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpl $37, 0
-; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:    jne LBB0_25
 ; CHECK-NEXT:  ## %bb.6: ## %bb3306
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:  Ltmp0:
@@ -70,7 +70,7 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalia
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    movl 0, %eax
 ; CHECK-NEXT:    cmpl $121, %eax
-; CHECK-NEXT:    ja LBB0_27
+; CHECK-NEXT:    ja LBB0_25
 ; CHECK-NEXT:  ## %bb.8: ## %bb3314
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    jmpl *LJTI0_0(,%eax,4)
@@ -78,11 +78,11 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalia
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne LBB0_27
+; CHECK-NEXT:    jne LBB0_25
 ; CHECK-NEXT:  ## %bb.11: ## %bb5809
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    testb %bh, %bh
-; CHECK-NEXT:    je LBB0_27
+; CHECK-NEXT:    je LBB0_25
 ; CHECK-NEXT:  ## %bb.12: ## %bb91.i8504
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    testb $1, %bl
@@ -98,10 +98,10 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalia
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    testb $1, %bl
 ; CHECK-NEXT:    je LBB0_15
-; CHECK-NEXT:  ## %bb.17: ## %bb278.i8617
+; CHECK-NEXT:  ## %bb.16: ## %bb278.i8617
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    je LBB0_19
-; CHECK-NEXT:  ## %bb.18: ## %bb440.i8663
+; CHECK-NEXT:    je LBB0_18
+; CHECK-NEXT:  ## %bb.17: ## %bb440.i8663
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:  Ltmp6:
 ; CHECK-NEXT:    movl L_.str4$non_lazy_ptr, %eax
@@ -110,39 +110,24 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalia
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl L__ZZNK10wxDateTime5GetTmERKNS_8TimeZoneEE12__FUNCTION__$non_lazy_ptr, %eax
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; CHECK-NEXT:    movl L_.str$non_lazy_ptr, %eax
-; CHECK-NEXT:    movl %eax, (%esp)
+; CHECK-NEXT:    movl %ebp, (%esp)
 ; CHECK-NEXT:    movl $1717, {{[0-9]+}}(%esp) ## imm = 0x6B5
 ; CHECK-NEXT:    calll __Z10wxOnAssertPKwiPKcS0_S0_
 ; CHECK-NEXT:  Ltmp7:
-; CHECK-NEXT:  LBB0_19: ## %bb448.i8694
-; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    jmp LBB0_20
+; CHECK-NEXT:    jmp LBB0_18
 ; CHECK-NEXT:  LBB0_15: ## %bb187.i8591
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    jne LBB0_27
-; CHECK-NEXT:  ## %bb.16: ## %bb265.i8606
+; CHECK-NEXT:    jne LBB0_25
+; CHECK-NEXT:  LBB0_18: ## %invcont5814
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    imull %ebp
-; CHECK-NEXT:    movl %edx, %eax
-; CHECK-NEXT:    shrl $31, %eax
-; CHECK-NEXT:    shrl $6, %edx
-; CHECK-NEXT:    addl %eax, %edx
-; CHECK-NEXT:    imull $1000, %edx, %eax ## imm = 0x3E8
-; CHECK-NEXT:    negl %eax
-; CHECK-NEXT:  LBB0_20: ## %invcont5814
-; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    movzwl %ax, %eax
 ; CHECK-NEXT:  Ltmp8:
-; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    calll __ZN8wxString6FormatEPKwz
 ; CHECK-NEXT:    subl $4, %esp
 ; CHECK-NEXT:  Ltmp9:
-; CHECK-NEXT:  ## %bb.21: ## %invcont5831
+; CHECK-NEXT:  ## %bb.19: ## %invcont5831
 ; CHECK-NEXT:    ## in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:  Ltmp10:
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
@@ -160,8 +145,8 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalia
 ; CHECK-NEXT:    movl %eax, (%esp)
 ; CHECK-NEXT:    calll __ZNK10wxDateTime12GetDayOfYearERKNS_8TimeZoneE
 ; CHECK-NEXT:  Ltmp14:
-; CHECK-NEXT:    jmp LBB0_27
-; CHECK-NEXT:  LBB0_22: ## %bb5968
+; CHECK-NEXT:    jmp LBB0_25
+; CHECK-NEXT:  LBB0_20: ## %bb5968
 ; CHECK-NEXT:  Ltmp2:
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
@@ -169,7 +154,7 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalia
 ; CHECK-NEXT:    calll __ZN8wxString6FormatEPKwz
 ; CHECK-NEXT:    subl $4, %esp
 ; CHECK-NEXT:  Ltmp3:
-; CHECK-NEXT:  LBB0_27: ## %bb115.critedge.i
+; CHECK-NEXT:  LBB0_25: ## %bb115.critedge.i
 ; CHECK-NEXT:    movl %esi, %eax
 ; CHECK-NEXT:    addl $28, %esp
 ; CHECK-NEXT:    popl %esi
@@ -177,15 +162,15 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(%struct.wxString* noalia
 ; CHECK-NEXT:    popl %ebx
 ; CHECK-NEXT:    popl %ebp
 ; CHECK-NEXT:    retl $4
-; CHECK-NEXT:  LBB0_25: ## %lpad.loopexit.split-lp
+; CHECK-NEXT:  LBB0_23: ## %lpad.loopexit.split-lp
 ; CHECK-NEXT:  Ltmp15:
-; CHECK-NEXT:    jmp LBB0_27
-; CHECK-NEXT:  LBB0_26: ## %lpad8185
+; CHECK-NEXT:    jmp LBB0_25
+; CHECK-NEXT:  LBB0_24: ## %lpad8185
 ; CHECK-NEXT:  Ltmp12:
-; CHECK-NEXT:    jmp LBB0_27
-; CHECK-NEXT:  LBB0_24: ## %lpad.loopexit
+; CHECK-NEXT:    jmp LBB0_25
+; CHECK-NEXT:  LBB0_22: ## %lpad.loopexit
 ; CHECK-NEXT:  Ltmp18:
-; CHECK-NEXT:    jmp LBB0_27
+; CHECK-NEXT:    jmp LBB0_25
 ; CHECK-NEXT:  Lfunc_end0:
 entry:
 	br i1 %foo, label %bb116.i, label %bb115.critedge.i
diff --git a/llvm/test/CodeGen/X86/AMX/amx-across-func.ll b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
index b687d03f92ba86f57762980699d5d4f897e6674f..d8d18a74961bd7a1e41a01b81535567b0f8a618c 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
@@ -263,6 +263,99 @@ define dso_local i32 @test_loop(i32 %0) nounwind {
   ret i32 %20
 }
 
+define dso_local void @test_loop2(i32 %0) nounwind {
+; CHECK-LABEL: test_loop2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $3024, %rsp # imm = 0xBD0
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    movl $buf, %r14d
+; CHECK-NEXT:    movl $32, %r15d
+; CHECK-NEXT:    movw $8, %bp
+; CHECK-NEXT:    movl $buf+2048, %r12d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    callq foo
+; CHECK-NEXT:    testl %ebx, %ebx
+; CHECK-NEXT:    jle .LBB3_3
+; CHECK-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
+; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
+; CHECK-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    tileloadd (%r14,%r15), %tmm0
+; CHECK-NEXT:    movabsq $64, %rax
+; CHECK-NEXT:    tilestored %tmm0, 1024(%rsp,%rax) # 1024-byte Folded Spill
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq foo
+; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movabsq $64, %rax
+; CHECK-NEXT:    tileloadd 1024(%rsp,%rax), %tmm0 # 1024-byte Folded Reload
+; CHECK-NEXT:    tilestored %tmm0, (%r12,%r15)
+; CHECK-NEXT:    callq foo
+; CHECK-NEXT:    jmp .LBB3_1
+; CHECK-NEXT:  .LBB3_3:
+; CHECK-NEXT:    addq $3024, %rsp # imm = 0xBD0
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    tilerelease
+; CHECK-NEXT:    retq
+;
+; IPRA-LABEL: test_loop2:
+; IPRA:       # %bb.0:
+; IPRA-NEXT:    subq $72, %rsp
+; IPRA-NEXT:    movl $buf, %eax
+; IPRA-NEXT:    movl $32, %ecx
+; IPRA-NEXT:    movw $8, %dx
+; IPRA-NEXT:    movl $buf+2048, %esi
+; IPRA-NEXT:    .p2align 4, 0x90
+; IPRA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
+; IPRA-NEXT:    callq foo
+; IPRA-NEXT:    testl %edi, %edi
+; IPRA-NEXT:    jle .LBB3_3
+; IPRA-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
+; IPRA-NEXT:    vpxord %zmm0, %zmm0, %zmm0
+; IPRA-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
+; IPRA-NEXT:    movb $1, {{[0-9]+}}(%rsp)
+; IPRA-NEXT:    movb $8, {{[0-9]+}}(%rsp)
+; IPRA-NEXT:    movw $8, {{[0-9]+}}(%rsp)
+; IPRA-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
+; IPRA-NEXT:    tileloadd (%rax,%rcx), %tmm0
+; IPRA-NEXT:    callq foo
+; IPRA-NEXT:    tilestored %tmm0, (%rsi,%rcx)
+; IPRA-NEXT:    callq foo
+; IPRA-NEXT:    jmp .LBB3_1
+; IPRA-NEXT:  .LBB3_3:
+; IPRA-NEXT:    addq $72, %rsp
+; IPRA-NEXT:    tilerelease
+; IPRA-NEXT:    vzeroupper
+; IPRA-NEXT:    retq
+  br label %2
+2:
+  %3 = phi i32 [ 0, %1 ], [ %7, %5 ]
+  call void @foo()
+  %4 = icmp sgt i32 %0, 0
+  br i1 %4, label %5, label %8
+5:
+  %6 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 0), i64 32)
+  call void @foo()
+  tail call void @llvm.x86.tilestored64.internal(i16 8, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 2048), i64 32, x86_amx %6)
+  call void @foo()
+  %7 = add i32 %3, 1
+  br label %2
+8:
+  ret void
+}
+
 declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64)
 declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
 declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, x86_amx)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll b/llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll
index 54201232f375eb6dcc231e452f25d8b5feca5194..1063b94484962966f0c3665612495ba74b31f651 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-low-intrinsics.ll
@@ -97,8 +97,8 @@ entry:
   ret void
 }
 
-define dso_local void @test_amx_dp(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 {
-; CHECK-LABEL: @test_amx_dp(
+define dso_local void @test_amx_dpbssd(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 {
+; CHECK-LABEL: @test_amx_dpbssd(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_AMX:%.*]] = bitcast <256 x i32> [[A:%.*]] to x86_amx
 ; CHECK-NEXT:    [[B_AMX:%.*]] = bitcast <256 x i32> [[B:%.*]] to x86_amx
@@ -172,6 +172,84 @@ entry:
   ret void
 }
 
+define dso_local void @test_amx_dpbf16ps(i16 signext %row, i16 signext %col, i16 signext %k, <256 x i32> %c, <256 x i32> %a, <256 x i32> %b, <256 x i32>* %vptr) #0 {
+; CHECK-LABEL: @test_amx_dpbf16ps(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_AMX:%.*]] = bitcast <256 x i32> [[A:%.*]] to x86_amx
+; CHECK-NEXT:    [[B_AMX:%.*]] = bitcast <256 x i32> [[B:%.*]] to x86_amx
+; CHECK-NEXT:    [[C_AMX:%.*]] = bitcast <256 x i32> [[C:%.*]] to x86_amx
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i16 [[COL:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[K:%.*]], 2
+; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_ROWS_HEADER:%.*]]
+; CHECK:       tdpbf16ps.scalarize.rows.header:
+; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_ROWS_IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TDPBF16PS_SCALARIZE_ROWS_STEP:%.*]], [[TDPBF16PS_SCALARIZE_ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_ROW:%.*]] = phi <256 x i32> [ [[C]], [[ENTRY]] ], [ [[TMP21:%.*]], [[TDPBF16PS_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_ROW:%.*]] = phi <256 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[TMP23:%.*]], [[TDPBF16PS_SCALARIZE_ROWS_LATCH]] ]
+; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_ROWS_BODY:%.*]]
+; CHECK:       tdpbf16ps.scalarize.rows.body:
+; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_COLS_HEADER:%.*]]
+; CHECK:       tdpbf16ps.scalarize.cols.header:
+; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_COLS_IV:%.*]] = phi i16 [ 0, [[TDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TDPBF16PS_SCALARIZE_COLS_STEP:%.*]], [[TDPBF16PS_SCALARIZE_COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_ROW]], [[TDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP21]], [[TDPBF16PS_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[VEC_D_PHI_COL:%.*]] = phi <256 x i32> [ [[VEC_D_PHI_ROW]], [[TDPBF16PS_SCALARIZE_ROWS_BODY]] ], [ [[TMP23]], [[TDPBF16PS_SCALARIZE_COLS_LATCH]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i16 [[TDPBF16PS_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[TMP2]], [[TDPBF16PS_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_COLS_BODY:%.*]]
+; CHECK:       tdpbf16ps.scalarize.cols.body:
+; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_INNER_HEADER:%.*]]
+; CHECK:       tdpbf16ps.scalarize.inner.header:
+; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_INNER_IV:%.*]] = phi i16 [ 0, [[TDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TDPBF16PS_SCALARIZE_INNER_STEP:%.*]], [[TDPBF16PS_SCALARIZE_INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[VEC_C_INNER_PHI:%.*]] = phi <256 x i32> [ [[VEC_C_PHI_COL]], [[TDPBF16PS_SCALARIZE_COLS_BODY]] ], [ [[TMP21]], [[TDPBF16PS_SCALARIZE_INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_INNER_BODY:%.*]]
+; CHECK:       tdpbf16ps.scalarize.inner.body:
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i16 [[TDPBF16PS_SCALARIZE_ROWS_IV]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = add i16 [[TMP4]], [[TDPBF16PS_SCALARIZE_INNER_IV]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i16 [[TDPBF16PS_SCALARIZE_INNER_IV]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[TMP6]], [[TDPBF16PS_SCALARIZE_COLS_IV]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <256 x i32> [[VEC_C_INNER_PHI]], i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32 [[TMP8]] to float
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <256 x i32> [[A]], i16 [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP10]] to <2 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <256 x i32> [[B]], i16 [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32 [[TMP12]] to <2 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i16> [[TMP11]], <2 x i16> zeroinitializer, <4 x i32> <i32 2, i32 0, i32 3, i32 1>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i16> [[TMP14]] to <2 x float>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x i16> [[TMP13]], <2 x i16> zeroinitializer, <4 x i32> <i32 2, i32 0, i32 3, i32 1>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <4 x i16> [[TMP16]] to <2 x float>
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul <2 x float> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.v2f32(float [[TMP9]], <2 x float> [[TMP18]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast float [[TMP19]] to i32
+; CHECK-NEXT:    [[TMP21]] = insertelement <256 x i32> [[VEC_C_INNER_PHI]], i32 [[TMP20]], i16 [[TMP3]]
+; CHECK-NEXT:    br label [[TDPBF16PS_SCALARIZE_INNER_LATCH]]
+; CHECK:       tdpbf16ps.scalarize.inner.latch:
+; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_INNER_STEP]] = add i16 [[TDPBF16PS_SCALARIZE_INNER_IV]], 1
+; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_INNER_COND:%.*]] = icmp ne i16 [[TDPBF16PS_SCALARIZE_INNER_STEP]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TDPBF16PS_SCALARIZE_INNER_COND]], label [[TDPBF16PS_SCALARIZE_INNER_HEADER]], label [[TDPBF16PS_SCALARIZE_COLS_LATCH]]
+; CHECK:       tdpbf16ps.scalarize.cols.latch:
+; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_COLS_STEP]] = add i16 [[TDPBF16PS_SCALARIZE_COLS_IV]], 1
+; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_COLS_COND:%.*]] = icmp ne i16 [[TDPBF16PS_SCALARIZE_COLS_STEP]], [[TMP0]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <256 x i32> [[TMP21]], i16 [[TMP3]]
+; CHECK-NEXT:    [[TMP23]] = insertelement <256 x i32> [[VEC_D_PHI_COL]], i32 [[TMP22]], i16 [[TMP3]]
+; CHECK-NEXT:    br i1 [[TDPBF16PS_SCALARIZE_COLS_COND]], label [[TDPBF16PS_SCALARIZE_COLS_HEADER]], label [[TDPBF16PS_SCALARIZE_ROWS_LATCH]]
+; CHECK:       tdpbf16ps.scalarize.rows.latch:
+; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_ROWS_STEP]] = add i16 [[TDPBF16PS_SCALARIZE_ROWS_IV]], 1
+; CHECK-NEXT:    [[TDPBF16PS_SCALARIZE_ROWS_COND:%.*]] = icmp ne i16 [[TDPBF16PS_SCALARIZE_ROWS_STEP]], [[ROW:%.*]]
+; CHECK-NEXT:    br i1 [[TDPBF16PS_SCALARIZE_ROWS_COND]], label [[TDPBF16PS_SCALARIZE_ROWS_HEADER]], label [[CONTINUE:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <256 x i32> [[TMP23]] to x86_amx
+; CHECK-NEXT:    store <256 x i32> [[TMP23]], <256 x i32>* [[VPTR:%.*]], align 64
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.amx = bitcast <256 x i32> %a to x86_amx
+  %b.amx = bitcast <256 x i32> %b to x86_amx
+  %c.amx = bitcast <256 x i32> %c to x86_amx
+  %acc = call x86_amx @llvm.x86.tdpbf16ps.internal(i16 %row, i16 %col, i16 %k, x86_amx %c.amx, x86_amx %a.amx, x86_amx %b.amx)
+  %vec = bitcast x86_amx %acc to <256 x i32>
+  store <256 x i32> %vec, <256 x i32>* %vptr, align 64
+  ret void
+}
+
 define dso_local void @test_amx_store(i16 signext %row, i16 signext %col, i8 *%ptr, i64 %stride, <256 x i32>* %vptr, <256 x i32> %vec) #0 {
 ; CHECK-LABEL: @test_amx_store(
 ; CHECK-NEXT:  entry:
@@ -232,6 +310,7 @@ entry:
 declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
 declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64)
 declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
+declare x86_amx @llvm.x86.tdpbf16ps.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
 declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, x86_amx)
 
 attributes #0 = { noinline nounwind optnone }
diff --git a/llvm/test/CodeGen/X86/AMX/amx-type.ll b/llvm/test/CodeGen/X86/AMX/amx-type.ll
index b730f3de16a8c0cbc9dfcbd06330dc3ad80a89aa..989a1076ce7a6d47989d242db4e66d74679a3ea4 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-type.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-type.ll
@@ -155,8 +155,8 @@ define dso_local void @__tile_loadd(%struct.__tile_str* nocapture %0, i8* %1, i6
   ret void
 }
 
-define dso_local void @__tile_dpbsud(%struct.__tile_str* nocapture %0, %struct.__tile_str* nocapture readonly byval(%struct.__tile_str) align 64 %1, %struct.__tile_str* nocapture readonly byval(%struct.__tile_str) align 64 %2) local_unnamed_addr {
-; CHECK-LABEL: @__tile_dpbsud(
+define dso_local void @__tile_dpbssd(%struct.__tile_str* nocapture %0, %struct.__tile_str* nocapture readonly byval(%struct.__tile_str) align 64 %1, %struct.__tile_str* nocapture readonly byval(%struct.__tile_str) align 64 %2) local_unnamed_addr {
+; CHECK-LABEL: @__tile_dpbssd(
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR:%.*]], %struct.__tile_str* [[TMP1:%.*]], i64 0, i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i16, i16* [[TMP4]], align 64
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP2:%.*]], i64 0, i32 1
@@ -198,6 +198,106 @@ define dso_local void @__tile_dpbsud(%struct.__tile_str* nocapture %0, %struct._
   ret void
 }
 
+define dso_local void @__tile_dpbsud(i16 %m, i16 %n, i16 %k, <256 x i32>* %pc, <256 x i32>* %pa, <256 x i32>* %pb) {
+; CHECK-LABEL: @__tile_dpbsud(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <256 x i32>* [[PA:%.*]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K:%.*]], i8* [[TMP1]], i64 64)
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <256 x i32>* [[PB:%.*]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[K]], i16 [[N:%.*]], i8* [[TMP3]], i64 64)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <256 x i32>* [[PC:%.*]] to i8*
+; CHECK-NEXT:    [[TMP6:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], i8* [[TMP5]], i64 64)
+; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbsud.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP6]], x86_amx [[TMP2]], x86_amx [[TMP4]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <256 x i32>* [[PC]] to i8*
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], i8* [[TMP7]], i64 64, x86_amx [[T6]])
+; CHECK-NEXT:    ret void
+;
+  %t0 = load <256 x i32>, <256 x i32>* %pa, align 64
+  %t1 = bitcast <256 x i32> %t0 to x86_amx
+  %t2 = load <256 x i32>, <256 x i32>* %pb, align 64
+  %t3 = bitcast <256 x i32> %t2 to x86_amx
+  %t4 = load <256 x i32>, <256 x i32>* %pc, align 64
+  %t5 = bitcast <256 x i32> %t4 to x86_amx
+  %t6 = tail call x86_amx @llvm.x86.tdpbsud.internal(i16 %m, i16 %n, i16 %k, x86_amx %t5, x86_amx %t1, x86_amx %t3)
+  %t7 = bitcast x86_amx %t6 to <256 x i32>
+  store <256 x i32> %t7, <256 x i32>* %pc, align 64
+  ret void
+}
+
+define dso_local void @__tile_dpbusd(i16 %m, i16 %n, i16 %k, <256 x i32>* %pc, <256 x i32>* %pa, <256 x i32>* %pb) {
+; CHECK-LABEL: @__tile_dpbusd(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <256 x i32>* [[PA:%.*]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K:%.*]], i8* [[TMP1]], i64 64)
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <256 x i32>* [[PB:%.*]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[K]], i16 [[N:%.*]], i8* [[TMP3]], i64 64)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <256 x i32>* [[PC:%.*]] to i8*
+; CHECK-NEXT:    [[TMP6:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], i8* [[TMP5]], i64 64)
+; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbusd.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP6]], x86_amx [[TMP2]], x86_amx [[TMP4]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <256 x i32>* [[PC]] to i8*
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], i8* [[TMP7]], i64 64, x86_amx [[T6]])
+; CHECK-NEXT:    ret void
+;
+  %t0 = load <256 x i32>, <256 x i32>* %pa, align 64
+  %t1 = bitcast <256 x i32> %t0 to x86_amx
+  %t2 = load <256 x i32>, <256 x i32>* %pb, align 64
+  %t3 = bitcast <256 x i32> %t2 to x86_amx
+  %t4 = load <256 x i32>, <256 x i32>* %pc, align 64
+  %t5 = bitcast <256 x i32> %t4 to x86_amx
+  %t6 = tail call x86_amx @llvm.x86.tdpbusd.internal(i16 %m, i16 %n, i16 %k, x86_amx %t5, x86_amx %t1, x86_amx %t3)
+  %t7 = bitcast x86_amx %t6 to <256 x i32>
+  store <256 x i32> %t7, <256 x i32>* %pc, align 64
+  ret void
+}
+
+define dso_local void @__tile_dpbuud(i16 %m, i16 %n, i16 %k, <256 x i32>* %pc, <256 x i32>* %pa, <256 x i32>* %pb) {
+; CHECK-LABEL: @__tile_dpbuud(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <256 x i32>* [[PA:%.*]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K:%.*]], i8* [[TMP1]], i64 64)
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <256 x i32>* [[PB:%.*]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[K]], i16 [[N:%.*]], i8* [[TMP3]], i64 64)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <256 x i32>* [[PC:%.*]] to i8*
+; CHECK-NEXT:    [[TMP6:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], i8* [[TMP5]], i64 64)
+; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbuud.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP6]], x86_amx [[TMP2]], x86_amx [[TMP4]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <256 x i32>* [[PC]] to i8*
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], i8* [[TMP7]], i64 64, x86_amx [[T6]])
+; CHECK-NEXT:    ret void
+;
+  %t0 = load <256 x i32>, <256 x i32>* %pa, align 64
+  %t1 = bitcast <256 x i32> %t0 to x86_amx
+  %t2 = load <256 x i32>, <256 x i32>* %pb, align 64
+  %t3 = bitcast <256 x i32> %t2 to x86_amx
+  %t4 = load <256 x i32>, <256 x i32>* %pc, align 64
+  %t5 = bitcast <256 x i32> %t4 to x86_amx
+  %t6 = tail call x86_amx @llvm.x86.tdpbuud.internal(i16 %m, i16 %n, i16 %k, x86_amx %t5, x86_amx %t1, x86_amx %t3)
+  %t7 = bitcast x86_amx %t6 to <256 x i32>
+  store <256 x i32> %t7, <256 x i32>* %pc, align 64
+  ret void
+}
+
+define dso_local void @__tile_dpbf16ps(i16 %m, i16 %n, i16 %k, <256 x i32>* %pc, <256 x i32>* %pa, <256 x i32>* %pb) {
+; CHECK-LABEL: @__tile_dpbf16ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <256 x i32>* [[PA:%.*]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K:%.*]], i8* [[TMP1]], i64 64)
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <256 x i32>* [[PB:%.*]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[K]], i16 [[N:%.*]], i8* [[TMP3]], i64 64)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <256 x i32>* [[PC:%.*]] to i8*
+; CHECK-NEXT:    [[TMP6:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], i8* [[TMP5]], i64 64)
+; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbf16ps.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP6]], x86_amx [[TMP2]], x86_amx [[TMP4]])
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <256 x i32>* [[PC]] to i8*
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], i8* [[TMP7]], i64 64, x86_amx [[T6]])
+; CHECK-NEXT:    ret void
+;
+  %t0 = load <256 x i32>, <256 x i32>* %pa, align 64
+  %t1 = bitcast <256 x i32> %t0 to x86_amx
+  %t2 = load <256 x i32>, <256 x i32>* %pb, align 64
+  %t3 = bitcast <256 x i32> %t2 to x86_amx
+  %t4 = load <256 x i32>, <256 x i32>* %pc, align 64
+  %t5 = bitcast <256 x i32> %t4 to x86_amx
+  %t6 = tail call x86_amx @llvm.x86.tdpbf16ps.internal(i16 %m, i16 %n, i16 %k, x86_amx %t5, x86_amx %t1, x86_amx %t3)
+  %t7 = bitcast x86_amx %t6 to <256 x i32>
+  store <256 x i32> %t7, <256 x i32>* %pc, align 64
+  ret void
+}
+
 define dso_local void @__tile_stored(i8* %0, i64 %1, %struct.__tile_str* nocapture readonly byval(%struct.__tile_str) align 64 %2) local_unnamed_addr {
 ; CHECK-LABEL: @__tile_stored(
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR:%.*]], %struct.__tile_str* [[TMP2:%.*]], i64 0, i32 0
@@ -227,4 +327,8 @@ define dso_local void @__tile_stored(i8* %0, i64 %1, %struct.__tile_str* nocaptu
 
 declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64)
 declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
+declare x86_amx @llvm.x86.tdpbsud.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
+declare x86_amx @llvm.x86.tdpbusd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
+declare x86_amx @llvm.x86.tdpbuud.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
+declare x86_amx @llvm.x86.tdpbf16ps.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
 declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, x86_amx)
diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X86_64.mir b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X86_64.mir
index 7b8b48178b120ec0e7bec74408a33d0c410ed094..a5c5d1c1f4c7aaa0433ef737f68b99fba20ba0cc 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X86_64.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X86_64.mir
@@ -625,29 +625,29 @@ body:             |
   bb.1 (%ir-block.0):
 
     ; FAST-LABEL: name: test_mul_gpr
-    ; FAST: [[DEF:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
-    ; FAST: [[DEF1:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
-    ; FAST: [[DEF2:%[0-9]+]]:gpr(s16) = IMPLICIT_DEF
-    ; FAST: [[DEF3:%[0-9]+]]:gpr(s8) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
+    ; FAST: [[DEF1:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
+    ; FAST: [[DEF2:%[0-9]+]]:gpr(s16) = G_IMPLICIT_DEF
+    ; FAST: [[DEF3:%[0-9]+]]:gpr(s8) = G_IMPLICIT_DEF
     ; FAST: [[MUL:%[0-9]+]]:gpr(s64) = G_MUL [[DEF]], [[DEF]]
     ; FAST: [[MUL1:%[0-9]+]]:gpr(s32) = G_MUL [[DEF1]], [[DEF1]]
     ; FAST: [[MUL2:%[0-9]+]]:gpr(s16) = G_MUL [[DEF2]], [[DEF2]]
     ; FAST: [[MUL3:%[0-9]+]]:gpr(s8) = G_MUL [[DEF3]], [[DEF3]]
     ; FAST: RET 0
     ; GREEDY-LABEL: name: test_mul_gpr
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
-    ; GREEDY: [[DEF1:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
-    ; GREEDY: [[DEF2:%[0-9]+]]:gpr(s16) = IMPLICIT_DEF
-    ; GREEDY: [[DEF3:%[0-9]+]]:gpr(s8) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
+    ; GREEDY: [[DEF1:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
+    ; GREEDY: [[DEF2:%[0-9]+]]:gpr(s16) = G_IMPLICIT_DEF
+    ; GREEDY: [[DEF3:%[0-9]+]]:gpr(s8) = G_IMPLICIT_DEF
     ; GREEDY: [[MUL:%[0-9]+]]:gpr(s64) = G_MUL [[DEF]], [[DEF]]
     ; GREEDY: [[MUL1:%[0-9]+]]:gpr(s32) = G_MUL [[DEF1]], [[DEF1]]
     ; GREEDY: [[MUL2:%[0-9]+]]:gpr(s16) = G_MUL [[DEF2]], [[DEF2]]
     ; GREEDY: [[MUL3:%[0-9]+]]:gpr(s8) = G_MUL [[DEF3]], [[DEF3]]
     ; GREEDY: RET 0
-    %0(s64) = IMPLICIT_DEF
-    %1(s32) = IMPLICIT_DEF
-    %2(s16) = IMPLICIT_DEF
-    %3(s8)  = IMPLICIT_DEF
+    %0(s64) = G_IMPLICIT_DEF
+    %1(s32) = G_IMPLICIT_DEF
+    %2(s16) = G_IMPLICIT_DEF
+    %3(s8)  = G_IMPLICIT_DEF
     %4(s64) = G_MUL %0, %0
     %5(s32) = G_MUL %1, %1
     %6(s16) = G_MUL %2, %2
@@ -767,8 +767,8 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; FAST-LABEL: name: test_fsub_float
-    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
-    ; FAST: [[DEF1:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
+    ; FAST: [[DEF1:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
     ; FAST: [[COPY:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; FAST: [[COPY1:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; FAST: [[FSUB:%[0-9]+]]:vecr(s32) = G_FSUB [[COPY]], [[COPY1]]
@@ -777,8 +777,8 @@ body:             |
     ; FAST: [[FSUB1:%[0-9]+]]:vecr(s64) = G_FSUB [[COPY2]], [[COPY3]]
     ; FAST: RET 0
     ; GREEDY-LABEL: name: test_fsub_float
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
-    ; GREEDY: [[DEF1:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
+    ; GREEDY: [[DEF1:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
     ; GREEDY: [[COPY:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; GREEDY: [[COPY1:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; GREEDY: [[FSUB:%[0-9]+]]:vecr(s32) = G_FSUB [[COPY]], [[COPY1]]
@@ -786,8 +786,8 @@ body:             |
     ; GREEDY: [[COPY3:%[0-9]+]]:vecr(s64) = COPY [[DEF1]](s64)
     ; GREEDY: [[FSUB1:%[0-9]+]]:vecr(s64) = G_FSUB [[COPY2]], [[COPY3]]
     ; GREEDY: RET 0
-    %0(s32) = IMPLICIT_DEF
-    %2(s64) = IMPLICIT_DEF
+    %0(s32) = G_IMPLICIT_DEF
+    %2(s64) = G_IMPLICIT_DEF
     %1(s32) = G_FSUB %0, %0
     %3(s64) = G_FSUB %2, %2
     RET 0
@@ -810,8 +810,8 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; FAST-LABEL: name: test_fmul_float
-    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
-    ; FAST: [[DEF1:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
+    ; FAST: [[DEF1:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
     ; FAST: [[COPY:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; FAST: [[COPY1:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; FAST: [[FMUL:%[0-9]+]]:vecr(s32) = G_FMUL [[COPY]], [[COPY1]]
@@ -820,8 +820,8 @@ body:             |
     ; FAST: [[FMUL1:%[0-9]+]]:vecr(s64) = G_FMUL [[COPY2]], [[COPY3]]
     ; FAST: RET 0
     ; GREEDY-LABEL: name: test_fmul_float
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
-    ; GREEDY: [[DEF1:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
+    ; GREEDY: [[DEF1:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
     ; GREEDY: [[COPY:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; GREEDY: [[COPY1:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; GREEDY: [[FMUL:%[0-9]+]]:vecr(s32) = G_FMUL [[COPY]], [[COPY1]]
@@ -829,8 +829,8 @@ body:             |
     ; GREEDY: [[COPY3:%[0-9]+]]:vecr(s64) = COPY [[DEF1]](s64)
     ; GREEDY: [[FMUL1:%[0-9]+]]:vecr(s64) = G_FMUL [[COPY2]], [[COPY3]]
     ; GREEDY: RET 0
-    %0(s32) = IMPLICIT_DEF
-    %2(s64) = IMPLICIT_DEF
+    %0(s32) = G_IMPLICIT_DEF
+    %2(s64) = G_IMPLICIT_DEF
     %1(s32) = G_FMUL %0, %0
     %3(s64) = G_FMUL %2, %2
     RET 0
@@ -853,8 +853,8 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; FAST-LABEL: name: test_fdiv_float
-    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
-    ; FAST: [[DEF1:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
+    ; FAST: [[DEF1:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
     ; FAST: [[COPY:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; FAST: [[COPY1:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; FAST: [[FDIV:%[0-9]+]]:vecr(s32) = G_FDIV [[COPY]], [[COPY1]]
@@ -863,8 +863,8 @@ body:             |
     ; FAST: [[FDIV1:%[0-9]+]]:vecr(s64) = G_FDIV [[COPY2]], [[COPY3]]
     ; FAST: RET 0
     ; GREEDY-LABEL: name: test_fdiv_float
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
-    ; GREEDY: [[DEF1:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
+    ; GREEDY: [[DEF1:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
     ; GREEDY: [[COPY:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; GREEDY: [[COPY1:%[0-9]+]]:vecr(s32) = COPY [[DEF]](s32)
     ; GREEDY: [[FDIV:%[0-9]+]]:vecr(s32) = G_FDIV [[COPY]], [[COPY1]]
@@ -872,8 +872,8 @@ body:             |
     ; GREEDY: [[COPY3:%[0-9]+]]:vecr(s64) = COPY [[DEF1]](s64)
     ; GREEDY: [[FDIV1:%[0-9]+]]:vecr(s64) = G_FDIV [[COPY2]], [[COPY3]]
     ; GREEDY: RET 0
-    %0(s32) = IMPLICIT_DEF
-    %2(s64) = IMPLICIT_DEF
+    %0(s32) = G_IMPLICIT_DEF
+    %2(s64) = G_IMPLICIT_DEF
     %1(s32) = G_FDIV %0, %0
     %3(s64) = G_FDIV %2, %2
     RET 0
@@ -1349,18 +1349,18 @@ registers:
 body:             |
   bb.0 (%ir-block.0):
     ; FAST-LABEL: name: trunc_check
-    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
     ; FAST: [[TRUNC:%[0-9]+]]:gpr(s1) = G_TRUNC [[DEF]](s32)
     ; FAST: [[TRUNC1:%[0-9]+]]:gpr(s8) = G_TRUNC [[DEF]](s32)
     ; FAST: [[TRUNC2:%[0-9]+]]:gpr(s16) = G_TRUNC [[DEF]](s32)
     ; FAST: RET 0
     ; GREEDY-LABEL: name: trunc_check
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
     ; GREEDY: [[TRUNC:%[0-9]+]]:gpr(s1) = G_TRUNC [[DEF]](s32)
     ; GREEDY: [[TRUNC1:%[0-9]+]]:gpr(s8) = G_TRUNC [[DEF]](s32)
     ; GREEDY: [[TRUNC2:%[0-9]+]]:gpr(s16) = G_TRUNC [[DEF]](s32)
     ; GREEDY: RET 0
-    %0(s32) = IMPLICIT_DEF
+    %0(s32) = G_IMPLICIT_DEF
     %1(s1) = G_TRUNC %0(s32)
     %2(s8) = G_TRUNC %0(s32)
     %3(s16) = G_TRUNC %0(s32)
@@ -1379,20 +1379,20 @@ registers:
 body:             |
   bb.0 (%ir-block.0):
     ; FAST-LABEL: name: test_gep
-    ; FAST: [[DEF:%[0-9]+]]:gpr(p0) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(p0) = G_IMPLICIT_DEF
     ; FAST: [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 20
     ; FAST: [[PTR_ADD:%[0-9]+]]:gpr(p0) = G_PTR_ADD [[DEF]], [[C]](s32)
     ; FAST: [[C1:%[0-9]+]]:gpr(s64) = G_CONSTANT i64 20
     ; FAST: [[PTR_ADD1:%[0-9]+]]:gpr(p0) = G_PTR_ADD [[DEF]], [[C1]](s64)
     ; FAST: RET 0
     ; GREEDY-LABEL: name: test_gep
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(p0) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(p0) = G_IMPLICIT_DEF
     ; GREEDY: [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 20
     ; GREEDY: [[PTR_ADD:%[0-9]+]]:gpr(p0) = G_PTR_ADD [[DEF]], [[C]](s32)
     ; GREEDY: [[C1:%[0-9]+]]:gpr(s64) = G_CONSTANT i64 20
     ; GREEDY: [[PTR_ADD1:%[0-9]+]]:gpr(p0) = G_PTR_ADD [[DEF]], [[C1]](s64)
     ; GREEDY: RET 0
-    %0(p0) = IMPLICIT_DEF
+    %0(p0) = G_IMPLICIT_DEF
     %1(s32) = G_CONSTANT i32 20
     %2(p0) = G_PTR_ADD %0, %1(s32)
     %3(s64) = G_CONSTANT i64 20
@@ -1567,16 +1567,16 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; FAST-LABEL: name: test_xor_i8
-    ; FAST: [[DEF:%[0-9]+]]:gpr(s8) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(s8) = G_IMPLICIT_DEF
     ; FAST: [[XOR:%[0-9]+]]:gpr(s8) = G_XOR [[DEF]], [[DEF]]
     ; FAST: $al = COPY [[XOR]](s8)
     ; FAST: RET 0, implicit $al
     ; GREEDY-LABEL: name: test_xor_i8
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s8) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s8) = G_IMPLICIT_DEF
     ; GREEDY: [[XOR:%[0-9]+]]:gpr(s8) = G_XOR [[DEF]], [[DEF]]
     ; GREEDY: $al = COPY [[XOR]](s8)
     ; GREEDY: RET 0, implicit $al
-    %0(s8) = IMPLICIT_DEF
+    %0(s8) = G_IMPLICIT_DEF
     %1(s8) = G_XOR %0, %0
     $al = COPY %1(s8)
     RET 0, implicit $al
@@ -1597,16 +1597,16 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; FAST-LABEL: name: test_or_i16
-    ; FAST: [[DEF:%[0-9]+]]:gpr(s16) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(s16) = G_IMPLICIT_DEF
     ; FAST: [[OR:%[0-9]+]]:gpr(s16) = G_OR [[DEF]], [[DEF]]
     ; FAST: $ax = COPY [[OR]](s16)
     ; FAST: RET 0, implicit $ax
     ; GREEDY-LABEL: name: test_or_i16
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s16) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s16) = G_IMPLICIT_DEF
     ; GREEDY: [[OR:%[0-9]+]]:gpr(s16) = G_OR [[DEF]], [[DEF]]
     ; GREEDY: $ax = COPY [[OR]](s16)
     ; GREEDY: RET 0, implicit $ax
-    %0(s16) = IMPLICIT_DEF
+    %0(s16) = G_IMPLICIT_DEF
     %1(s16) = G_OR %0, %0
     $ax = COPY %1(s16)
     RET 0, implicit $ax
@@ -1627,16 +1627,16 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; FAST-LABEL: name: test_and_i32
-    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
     ; FAST: [[AND:%[0-9]+]]:gpr(s32) = G_AND [[DEF]], [[DEF]]
     ; FAST: $eax = COPY [[AND]](s32)
     ; FAST: RET 0, implicit $eax
     ; GREEDY-LABEL: name: test_and_i32
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s32) = G_IMPLICIT_DEF
     ; GREEDY: [[AND:%[0-9]+]]:gpr(s32) = G_AND [[DEF]], [[DEF]]
     ; GREEDY: $eax = COPY [[AND]](s32)
     ; GREEDY: RET 0, implicit $eax
-    %0(s32) = IMPLICIT_DEF
+    %0(s32) = G_IMPLICIT_DEF
     %1(s32) = G_AND %0, %0
     $eax = COPY %1(s32)
     RET 0, implicit $eax
@@ -1657,16 +1657,16 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; FAST-LABEL: name: test_and_i64
-    ; FAST: [[DEF:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
+    ; FAST: [[DEF:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
     ; FAST: [[AND:%[0-9]+]]:gpr(s64) = G_AND [[DEF]], [[DEF]]
     ; FAST: $rax = COPY [[AND]](s64)
     ; FAST: RET 0, implicit $rax
     ; GREEDY-LABEL: name: test_and_i64
-    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s64) = IMPLICIT_DEF
+    ; GREEDY: [[DEF:%[0-9]+]]:gpr(s64) = G_IMPLICIT_DEF
     ; GREEDY: [[AND:%[0-9]+]]:gpr(s64) = G_AND [[DEF]], [[DEF]]
     ; GREEDY: $rax = COPY [[AND]](s64)
     ; GREEDY: RET 0, implicit $rax
-    %0(s64) = IMPLICIT_DEF
+    %0(s64) = G_IMPLICIT_DEF
     %1(s64) = G_AND %0, %0
     $rax = COPY %1(s64)
     RET 0, implicit $rax
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
index 2e1cbacf4584676172c2a35e0caa0f18ff7d15eb..e5b3584a0339e76958a6decb9e1c0ff6dd7d4e4a 100644
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -18,8 +18,6 @@
 ; CHECK-NEXT:     Pre-ISel Intrinsic Lowering
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Expand Atomic instructions
-; CHECK-NEXT:       Dominator Tree Construction
-; CHECK-NEXT:       Natural Loop Information
 ; CHECK-NEXT:       Lower AMX intrinsics
 ; CHECK-NEXT:       Lower AMX type for load/store
 ; CHECK-NEXT:       Module Verifier
diff --git a/llvm/test/CodeGen/X86/WidenArith.ll b/llvm/test/CodeGen/X86/WidenArith.ll
index cb9bf03b64c24453bbedb66ffe609a531b594778..8d1eedc96a6e53107f0d5a7475d2690315e9e1cc 100644
--- a/llvm/test/CodeGen/X86/WidenArith.ll
+++ b/llvm/test/CodeGen/X86/WidenArith.ll
@@ -11,7 +11,7 @@ define <8 x i32> @test(<8 x float> %a, <8 x float> %b) {
 ; X86-NEXT:    vcmpltps %ymm1, %ymm0, %ymm0
 ; X86-NEXT:    vcmpltps %ymm3, %ymm2, %ymm1
 ; X86-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; X86-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test:
diff --git a/llvm/test/CodeGen/X86/addsub-constant-folding.ll b/llvm/test/CodeGen/X86/addsub-constant-folding.ll
index 4e8a80a0c8ff9719e0c19fafc0562e86a196dc15..e6af950dd2737e5d77a7ac1115253fe76014f115 100644
--- a/llvm/test/CodeGen/X86/addsub-constant-folding.ll
+++ b/llvm/test/CodeGen/X86/addsub-constant-folding.ll
@@ -34,7 +34,7 @@ define i32 @add_const_add_const_extrause(i32 %arg) {
 ; X86-NEXT:    leal 8(%esi), %eax
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll use
+; X86-NEXT:    calll use@PLT
 ; X86-NEXT:    addl $4, %esp
 ; X86-NEXT:    .cfi_adjust_cfa_offset -4
 ; X86-NEXT:    addl $10, %esi
@@ -50,7 +50,7 @@ define i32 @add_const_add_const_extrause(i32 %arg) {
 ; X64-NEXT:    .cfi_offset %rbx, -16
 ; X64-NEXT:    movl %edi, %ebx
 ; X64-NEXT:    leal 8(%rbx), %edi
-; X64-NEXT:    callq use
+; X64-NEXT:    callq use@PLT
 ; X64-NEXT:    leal 10(%rbx), %eax
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    .cfi_def_cfa_offset 8
@@ -64,7 +64,7 @@ define i32 @add_const_add_const_extrause(i32 %arg) {
 define <4 x i32> @vec_add_const_add_const(<4 x i32> %arg) {
 ; X86-LABEL: vec_add_const_add_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_add_const_add_const:
@@ -85,9 +85,9 @@ define <4 x i32> @vec_add_const_add_const_extrause(<4 x i32> %arg) {
 ; X86-NEXT:    movdqu %xmm0, (%esp) # 16-byte Spill
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X86-NEXT:    paddd %xmm1, %xmm0
-; X86-NEXT:    calll vec_use
+; X86-NEXT:    calll vec_use@PLT
 ; X86-NEXT:    movdqu (%esp), %xmm0 # 16-byte Reload
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
@@ -100,7 +100,7 @@ define <4 x i32> @vec_add_const_add_const_extrause(<4 x i32> %arg) {
 ; X64-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X64-NEXT:    paddd %xmm1, %xmm0
-; X64-NEXT:    callq vec_use
+; X64-NEXT:    callq vec_use@PLT
 ; X64-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
 ; X64-NEXT:    paddd {{.*}}(%rip), %xmm0
 ; X64-NEXT:    addq $24, %rsp
@@ -115,7 +115,7 @@ define <4 x i32> @vec_add_const_add_const_extrause(<4 x i32> %arg) {
 define <4 x i32> @vec_add_const_add_const_nonsplat(<4 x i32> %arg) {
 ; X86-LABEL: vec_add_const_add_const_nonsplat:
 ; X86:       # %bb.0:
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_add_const_add_const_nonsplat:
@@ -156,7 +156,7 @@ define i32 @add_const_sub_const_extrause(i32 %arg) {
 ; X86-NEXT:    leal 8(%esi), %eax
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll use
+; X86-NEXT:    calll use@PLT
 ; X86-NEXT:    addl $4, %esp
 ; X86-NEXT:    .cfi_adjust_cfa_offset -4
 ; X86-NEXT:    addl $6, %esi
@@ -172,7 +172,7 @@ define i32 @add_const_sub_const_extrause(i32 %arg) {
 ; X64-NEXT:    .cfi_offset %rbx, -16
 ; X64-NEXT:    movl %edi, %ebx
 ; X64-NEXT:    leal 8(%rbx), %edi
-; X64-NEXT:    callq use
+; X64-NEXT:    callq use@PLT
 ; X64-NEXT:    leal 6(%rbx), %eax
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    .cfi_def_cfa_offset 8
@@ -186,7 +186,7 @@ define i32 @add_const_sub_const_extrause(i32 %arg) {
 define <4 x i32> @vec_add_const_sub_const(<4 x i32> %arg) {
 ; X86-LABEL: vec_add_const_sub_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_add_const_sub_const:
@@ -207,9 +207,9 @@ define <4 x i32> @vec_add_const_sub_const_extrause(<4 x i32> %arg) {
 ; X86-NEXT:    movdqu %xmm0, (%esp) # 16-byte Spill
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X86-NEXT:    paddd %xmm1, %xmm0
-; X86-NEXT:    calll vec_use
+; X86-NEXT:    calll vec_use@PLT
 ; X86-NEXT:    movdqu (%esp), %xmm0 # 16-byte Reload
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
@@ -222,7 +222,7 @@ define <4 x i32> @vec_add_const_sub_const_extrause(<4 x i32> %arg) {
 ; X64-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X64-NEXT:    paddd %xmm1, %xmm0
-; X64-NEXT:    callq vec_use
+; X64-NEXT:    callq vec_use@PLT
 ; X64-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
 ; X64-NEXT:    paddd {{.*}}(%rip), %xmm0
 ; X64-NEXT:    addq $24, %rsp
@@ -237,7 +237,7 @@ define <4 x i32> @vec_add_const_sub_const_extrause(<4 x i32> %arg) {
 define <4 x i32> @vec_add_const_sub_const_nonsplat(<4 x i32> %arg) {
 ; X86-LABEL: vec_add_const_sub_const_nonsplat:
 ; X86:       # %bb.0:
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_add_const_sub_const_nonsplat:
@@ -278,7 +278,7 @@ define i32 @add_const_const_sub_extrause(i32 %arg) {
 ; X86-NEXT:    leal 8(%esi), %eax
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll use
+; X86-NEXT:    calll use@PLT
 ; X86-NEXT:    addl $4, %esp
 ; X86-NEXT:    .cfi_adjust_cfa_offset -4
 ; X86-NEXT:    movl $-6, %eax
@@ -294,7 +294,7 @@ define i32 @add_const_const_sub_extrause(i32 %arg) {
 ; X64-NEXT:    .cfi_offset %rbx, -16
 ; X64-NEXT:    movl %edi, %ebx
 ; X64-NEXT:    leal 8(%rbx), %edi
-; X64-NEXT:    callq use
+; X64-NEXT:    callq use@PLT
 ; X64-NEXT:    movl $-6, %eax
 ; X64-NEXT:    subl %ebx, %eax
 ; X64-NEXT:    popq %rbx
@@ -334,7 +334,7 @@ define <4 x i32> @vec_add_const_const_sub_extrause(<4 x i32> %arg) {
 ; X86-NEXT:    movdqu %xmm0, (%esp) # 16-byte Spill
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X86-NEXT:    paddd %xmm1, %xmm0
-; X86-NEXT:    calll vec_use
+; X86-NEXT:    calll vec_use@PLT
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [4294967290,4294967290,4294967290,4294967290]
 ; X86-NEXT:    movdqu (%esp), %xmm1 # 16-byte Reload
 ; X86-NEXT:    psubd %xmm1, %xmm0
@@ -350,7 +350,7 @@ define <4 x i32> @vec_add_const_const_sub_extrause(<4 x i32> %arg) {
 ; X64-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X64-NEXT:    paddd %xmm1, %xmm0
-; X64-NEXT:    callq vec_use
+; X64-NEXT:    callq vec_use@PLT
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [4294967290,4294967290,4294967290,4294967290]
 ; X64-NEXT:    psubd (%rsp), %xmm0 # 16-byte Folded Reload
 ; X64-NEXT:    addq $24, %rsp
@@ -410,7 +410,7 @@ define i32 @sub_const_add_const_extrause(i32 %arg) {
 ; X86-NEXT:    leal -8(%esi), %eax
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll use
+; X86-NEXT:    calll use@PLT
 ; X86-NEXT:    addl $4, %esp
 ; X86-NEXT:    .cfi_adjust_cfa_offset -4
 ; X86-NEXT:    addl $-6, %esi
@@ -426,7 +426,7 @@ define i32 @sub_const_add_const_extrause(i32 %arg) {
 ; X64-NEXT:    .cfi_offset %rbx, -16
 ; X64-NEXT:    movl %edi, %ebx
 ; X64-NEXT:    leal -8(%rbx), %edi
-; X64-NEXT:    callq use
+; X64-NEXT:    callq use@PLT
 ; X64-NEXT:    leal -6(%rbx), %eax
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    .cfi_def_cfa_offset 8
@@ -440,7 +440,7 @@ define i32 @sub_const_add_const_extrause(i32 %arg) {
 define <4 x i32> @vec_sub_const_add_const(<4 x i32> %arg) {
 ; X86-LABEL: vec_sub_const_add_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_sub_const_add_const:
@@ -458,10 +458,10 @@ define <4 x i32> @vec_sub_const_add_const_extrause(<4 x i32> %arg) {
 ; X86-NEXT:    subl $28, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 32
 ; X86-NEXT:    movdqu %xmm0, (%esp) # 16-byte Spill
-; X86-NEXT:    psubd {{\.LCPI.*}}, %xmm0
-; X86-NEXT:    calll vec_use
+; X86-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-NEXT:    calll vec_use@PLT
 ; X86-NEXT:    movdqu (%esp), %xmm0 # 16-byte Reload
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
@@ -472,7 +472,7 @@ define <4 x i32> @vec_sub_const_add_const_extrause(<4 x i32> %arg) {
 ; X64-NEXT:    .cfi_def_cfa_offset 32
 ; X64-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; X64-NEXT:    psubd {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq vec_use
+; X64-NEXT:    callq vec_use@PLT
 ; X64-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
 ; X64-NEXT:    paddd {{.*}}(%rip), %xmm0
 ; X64-NEXT:    addq $24, %rsp
@@ -487,7 +487,7 @@ define <4 x i32> @vec_sub_const_add_const_extrause(<4 x i32> %arg) {
 define <4 x i32> @vec_sub_const_add_const_nonsplat(<4 x i32> %arg) {
 ; X86-LABEL: vec_sub_const_add_const_nonsplat:
 ; X86:       # %bb.0:
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_sub_const_add_const_nonsplat:
@@ -528,7 +528,7 @@ define i32 @sub_const_sub_const_extrause(i32 %arg) {
 ; X86-NEXT:    leal -8(%esi), %eax
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll use
+; X86-NEXT:    calll use@PLT
 ; X86-NEXT:    addl $4, %esp
 ; X86-NEXT:    .cfi_adjust_cfa_offset -4
 ; X86-NEXT:    addl $-10, %esi
@@ -544,7 +544,7 @@ define i32 @sub_const_sub_const_extrause(i32 %arg) {
 ; X64-NEXT:    .cfi_offset %rbx, -16
 ; X64-NEXT:    movl %edi, %ebx
 ; X64-NEXT:    leal -8(%rbx), %edi
-; X64-NEXT:    callq use
+; X64-NEXT:    callq use@PLT
 ; X64-NEXT:    leal -10(%rbx), %eax
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    .cfi_def_cfa_offset 8
@@ -558,7 +558,7 @@ define i32 @sub_const_sub_const_extrause(i32 %arg) {
 define <4 x i32> @vec_sub_const_sub_const(<4 x i32> %arg) {
 ; X86-LABEL: vec_sub_const_sub_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    psubd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_sub_const_sub_const:
@@ -576,10 +576,10 @@ define <4 x i32> @vec_sub_const_sub_const_extrause(<4 x i32> %arg) {
 ; X86-NEXT:    subl $28, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 32
 ; X86-NEXT:    movdqu %xmm0, (%esp) # 16-byte Spill
-; X86-NEXT:    psubd {{\.LCPI.*}}, %xmm0
-; X86-NEXT:    calll vec_use
+; X86-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-NEXT:    calll vec_use@PLT
 ; X86-NEXT:    movdqu (%esp), %xmm0 # 16-byte Reload
-; X86-NEXT:    psubd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
@@ -590,7 +590,7 @@ define <4 x i32> @vec_sub_const_sub_const_extrause(<4 x i32> %arg) {
 ; X64-NEXT:    .cfi_def_cfa_offset 32
 ; X64-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; X64-NEXT:    psubd {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq vec_use
+; X64-NEXT:    callq vec_use@PLT
 ; X64-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
 ; X64-NEXT:    psubd {{.*}}(%rip), %xmm0
 ; X64-NEXT:    addq $24, %rsp
@@ -605,7 +605,7 @@ define <4 x i32> @vec_sub_const_sub_const_extrause(<4 x i32> %arg) {
 define <4 x i32> @vec_sub_const_sub_const_nonsplat(<4 x i32> %arg) {
 ; X86-LABEL: vec_sub_const_sub_const_nonsplat:
 ; X86:       # %bb.0:
-; X86-NEXT:    psubd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_sub_const_sub_const_nonsplat:
@@ -646,7 +646,7 @@ define i32 @sub_const_const_sub_extrause(i32 %arg) {
 ; X86-NEXT:    leal -8(%esi), %eax
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll use
+; X86-NEXT:    calll use@PLT
 ; X86-NEXT:    addl $4, %esp
 ; X86-NEXT:    .cfi_adjust_cfa_offset -4
 ; X86-NEXT:    movl $10, %eax
@@ -662,7 +662,7 @@ define i32 @sub_const_const_sub_extrause(i32 %arg) {
 ; X64-NEXT:    .cfi_offset %rbx, -16
 ; X64-NEXT:    movl %edi, %ebx
 ; X64-NEXT:    leal -8(%rbx), %edi
-; X64-NEXT:    callq use
+; X64-NEXT:    callq use@PLT
 ; X64-NEXT:    movl $10, %eax
 ; X64-NEXT:    subl %ebx, %eax
 ; X64-NEXT:    popq %rbx
@@ -698,9 +698,9 @@ define <4 x i32> @vec_sub_const_const_sub_extrause(<4 x i32> %arg) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    subl $28, %esp
 ; X86-NEXT:    .cfi_def_cfa_offset 32
-; X86-NEXT:    psubd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    movdqu %xmm0, (%esp) # 16-byte Spill
-; X86-NEXT:    calll vec_use
+; X86-NEXT:    calll vec_use@PLT
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [2,2,2,2]
 ; X86-NEXT:    movdqu (%esp), %xmm1 # 16-byte Reload
 ; X86-NEXT:    psubd %xmm1, %xmm0
@@ -714,7 +714,7 @@ define <4 x i32> @vec_sub_const_const_sub_extrause(<4 x i32> %arg) {
 ; X64-NEXT:    .cfi_def_cfa_offset 32
 ; X64-NEXT:    psubd {{.*}}(%rip), %xmm0
 ; X64-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
-; X64-NEXT:    callq vec_use
+; X64-NEXT:    callq vec_use@PLT
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [2,2,2,2]
 ; X64-NEXT:    psubd (%rsp), %xmm0 # 16-byte Folded Reload
 ; X64-NEXT:    addq $24, %rsp
@@ -775,7 +775,7 @@ define i32 @const_sub_add_const_extrause(i32 %arg) {
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll use
+; X86-NEXT:    calll use@PLT
 ; X86-NEXT:    addl $4, %esp
 ; X86-NEXT:    .cfi_adjust_cfa_offset -4
 ; X86-NEXT:    movl $10, %eax
@@ -792,7 +792,7 @@ define i32 @const_sub_add_const_extrause(i32 %arg) {
 ; X64-NEXT:    movl %edi, %ebx
 ; X64-NEXT:    movl $8, %edi
 ; X64-NEXT:    subl %ebx, %edi
-; X64-NEXT:    callq use
+; X64-NEXT:    callq use@PLT
 ; X64-NEXT:    movl $10, %eax
 ; X64-NEXT:    subl %ebx, %eax
 ; X64-NEXT:    popq %rbx
@@ -832,7 +832,7 @@ define <4 x i32> @vec_const_sub_add_const_extrause(<4 x i32> %arg) {
 ; X86-NEXT:    movdqu %xmm0, (%esp) # 16-byte Spill
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X86-NEXT:    psubd %xmm1, %xmm0
-; X86-NEXT:    calll vec_use
+; X86-NEXT:    calll vec_use@PLT
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [10,10,10,10]
 ; X86-NEXT:    movdqu (%esp), %xmm1 # 16-byte Reload
 ; X86-NEXT:    psubd %xmm1, %xmm0
@@ -848,7 +848,7 @@ define <4 x i32> @vec_const_sub_add_const_extrause(<4 x i32> %arg) {
 ; X64-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X64-NEXT:    psubd %xmm1, %xmm0
-; X64-NEXT:    callq vec_use
+; X64-NEXT:    callq vec_use@PLT
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [10,10,10,10]
 ; X64-NEXT:    psubd (%rsp), %xmm0 # 16-byte Folded Reload
 ; X64-NEXT:    addq $24, %rsp
@@ -909,7 +909,7 @@ define i32 @const_sub_sub_const_extrause(i32 %arg) {
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll use
+; X86-NEXT:    calll use@PLT
 ; X86-NEXT:    addl $4, %esp
 ; X86-NEXT:    .cfi_adjust_cfa_offset -4
 ; X86-NEXT:    movl $6, %eax
@@ -926,7 +926,7 @@ define i32 @const_sub_sub_const_extrause(i32 %arg) {
 ; X64-NEXT:    movl %edi, %ebx
 ; X64-NEXT:    movl $8, %edi
 ; X64-NEXT:    subl %ebx, %edi
-; X64-NEXT:    callq use
+; X64-NEXT:    callq use@PLT
 ; X64-NEXT:    movl $6, %eax
 ; X64-NEXT:    subl %ebx, %eax
 ; X64-NEXT:    popq %rbx
@@ -966,7 +966,7 @@ define <4 x i32> @vec_const_sub_sub_const_extrause(<4 x i32> %arg) {
 ; X86-NEXT:    movdqu %xmm0, (%esp) # 16-byte Spill
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X86-NEXT:    psubd %xmm1, %xmm0
-; X86-NEXT:    calll vec_use
+; X86-NEXT:    calll vec_use@PLT
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [6,6,6,6]
 ; X86-NEXT:    movdqu (%esp), %xmm1 # 16-byte Reload
 ; X86-NEXT:    psubd %xmm1, %xmm0
@@ -982,7 +982,7 @@ define <4 x i32> @vec_const_sub_sub_const_extrause(<4 x i32> %arg) {
 ; X64-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [8,8,8,8]
 ; X64-NEXT:    psubd %xmm1, %xmm0
-; X64-NEXT:    callq vec_use
+; X64-NEXT:    callq vec_use@PLT
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [6,6,6,6]
 ; X64-NEXT:    psubd (%rsp), %xmm0 # 16-byte Folded Reload
 ; X64-NEXT:    addq $24, %rsp
@@ -1042,7 +1042,7 @@ define i32 @const_sub_const_sub_extrause(i32 %arg) {
 ; X86-NEXT:    subl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-NEXT:    calll use
+; X86-NEXT:    calll use@PLT
 ; X86-NEXT:    addl $4, %esp
 ; X86-NEXT:    .cfi_adjust_cfa_offset -4
 ; X86-NEXT:    movl $2, %eax
@@ -1059,7 +1059,7 @@ define i32 @const_sub_const_sub_extrause(i32 %arg) {
 ; X64-NEXT:    movl $8, %ebx
 ; X64-NEXT:    subl %edi, %ebx
 ; X64-NEXT:    movl %ebx, %edi
-; X64-NEXT:    callq use
+; X64-NEXT:    callq use@PLT
 ; X64-NEXT:    movl $2, %eax
 ; X64-NEXT:    subl %ebx, %eax
 ; X64-NEXT:    popq %rbx
@@ -1074,7 +1074,7 @@ define i32 @const_sub_const_sub_extrause(i32 %arg) {
 define <4 x i32> @vec_const_sub_const_sub(<4 x i32> %arg) {
 ; X86-LABEL: vec_const_sub_const_sub:
 ; X86:       # %bb.0:
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_const_sub_const_sub:
@@ -1095,7 +1095,7 @@ define <4 x i32> @vec_const_sub_const_sub_extrause(<4 x i32> %arg) {
 ; X86-NEXT:    psubd %xmm0, %xmm1
 ; X86-NEXT:    movdqu %xmm1, (%esp) # 16-byte Spill
 ; X86-NEXT:    movdqa %xmm1, %xmm0
-; X86-NEXT:    calll vec_use
+; X86-NEXT:    calll vec_use@PLT
 ; X86-NEXT:    movdqa {{.*#+}} xmm0 = [2,2,2,2]
 ; X86-NEXT:    movdqu (%esp), %xmm1 # 16-byte Reload
 ; X86-NEXT:    psubd %xmm1, %xmm0
@@ -1111,7 +1111,7 @@ define <4 x i32> @vec_const_sub_const_sub_extrause(<4 x i32> %arg) {
 ; X64-NEXT:    psubd %xmm0, %xmm1
 ; X64-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
 ; X64-NEXT:    movdqa %xmm1, %xmm0
-; X64-NEXT:    callq vec_use
+; X64-NEXT:    callq vec_use@PLT
 ; X64-NEXT:    movdqa {{.*#+}} xmm0 = [2,2,2,2]
 ; X64-NEXT:    psubd (%rsp), %xmm0 # 16-byte Folded Reload
 ; X64-NEXT:    addq $24, %rsp
@@ -1126,7 +1126,7 @@ define <4 x i32> @vec_const_sub_const_sub_extrause(<4 x i32> %arg) {
 define <4 x i32> @vec_const_sub_const_sub_nonsplat(<4 x i32> %arg) {
 ; X86-LABEL: vec_const_sub_const_sub_nonsplat:
 ; X86:       # %bb.0:
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_const_sub_const_sub_nonsplat:
diff --git a/llvm/test/CodeGen/X86/arg-copy-elide.ll b/llvm/test/CodeGen/X86/arg-copy-elide.ll
index f8761bd0ac9b4509d9f88d314ac01ad850ae6cd4..36510e09beba2924e5d4e901052a826e57bc356a 100644
--- a/llvm/test/CodeGen/X86/arg-copy-elide.ll
+++ b/llvm/test/CodeGen/X86/arg-copy-elide.ll
@@ -232,7 +232,7 @@ entry:
 ; CHECK: retl
 
 
-define void @avoid_inalloca(i32* inalloca %x) {
+define void @avoid_inalloca(i32* inalloca(i32) %x) {
 entry:
   %x.p.p = alloca i32*
   store i32* %x, i32** %x.p.p
diff --git a/llvm/test/CodeGen/X86/atomic-fp.ll b/llvm/test/CodeGen/X86/atomic-fp.ll
index 62296169e06d0bb1d03c1595186024821ae7d37b..a87da8428e0ae9173dbf63feed5675d41a91bd5c 100644
--- a/llvm/test/CodeGen/X86/atomic-fp.ll
+++ b/llvm/test/CodeGen/X86/atomic-fp.ll
@@ -200,7 +200,7 @@ define dso_local void @fadd_32g() nounwind {
 ; X86-SSE1-NEXT:    movl glob32, %eax
 ; X86-SSE1-NEXT:    movl %eax, (%esp)
 ; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SSE1-NEXT:    addss {{\.LCPI.*}}, %xmm0
+; X86-SSE1-NEXT:    addss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE1-NEXT:    movl %eax, glob32
@@ -296,7 +296,7 @@ define dso_local void @fadd_64g() nounwind {
 ; X86-SSE2-NEXT:    andl $-8, %esp
 ; X86-SSE2-NEXT:    subl $8, %esp
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X86-SSE2-NEXT:    addsd {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    addsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    movlps %xmm0, glob64
@@ -311,7 +311,7 @@ define dso_local void @fadd_64g() nounwind {
 ; X86-AVX-NEXT:    andl $-8, %esp
 ; X86-AVX-NEXT:    subl $8, %esp
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-AVX-NEXT:    vaddsd {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vaddsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX-NEXT:    vmovlps %xmm0, glob64
@@ -361,7 +361,7 @@ define dso_local void @fadd_32imm() nounwind {
 ; X86-SSE1-NEXT:    movl -559038737, %eax
 ; X86-SSE1-NEXT:    movl %eax, (%esp)
 ; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SSE1-NEXT:    addss {{\.LCPI.*}}, %xmm0
+; X86-SSE1-NEXT:    addss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE1-NEXT:    movl %eax, -559038737
@@ -459,7 +459,7 @@ define dso_local void @fadd_64imm() nounwind {
 ; X86-SSE2-NEXT:    andl $-8, %esp
 ; X86-SSE2-NEXT:    subl $8, %esp
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X86-SSE2-NEXT:    addsd {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    addsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    movlps %xmm0, -559038737
@@ -474,7 +474,7 @@ define dso_local void @fadd_64imm() nounwind {
 ; X86-AVX-NEXT:    andl $-8, %esp
 ; X86-AVX-NEXT:    subl $8, %esp
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-AVX-NEXT:    vaddsd {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vaddsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX-NEXT:    vmovlps %xmm0, -559038737
@@ -526,7 +526,7 @@ define dso_local void @fadd_32stack() nounwind {
 ; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE1-NEXT:    movl %eax, (%esp)
 ; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SSE1-NEXT:    addss {{\.LCPI.*}}, %xmm0
+; X86-SSE1-NEXT:    addss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE1-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE1-NEXT:    movl %eax, {{[0-9]+}}(%esp)
@@ -628,7 +628,7 @@ define dso_local void @fadd_64stack() nounwind {
 ; X86-SSE2-NEXT:    andl $-8, %esp
 ; X86-SSE2-NEXT:    subl $16, %esp
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X86-SSE2-NEXT:    addsd {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    addsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
@@ -643,7 +643,7 @@ define dso_local void @fadd_64stack() nounwind {
 ; X86-AVX-NEXT:    andl $-8, %esp
 ; X86-AVX-NEXT:    subl $16, %esp
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-AVX-NEXT:    vaddsd {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vaddsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
diff --git a/llvm/test/CodeGen/X86/avx-cmp.ll b/llvm/test/CodeGen/X86/avx-cmp.ll
index e564cf162ace9686688e709c8101e2e50aed9622..e22ebc82dda0068c80f1941aa75b81924774361d 100644
--- a/llvm/test/CodeGen/X86/avx-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx-cmp.ll
@@ -49,12 +49,12 @@ define void @render(double %a0) nounwind {
 ; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
 ; CHECK-NEXT:    vmovsd (%rsp), %xmm0 # 8-byte Reload
 ; CHECK-NEXT:    # xmm0 = mem[0],zero
-; CHECK-NEXT:    vucomisd {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    vucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK-NEXT:    jne .LBB2_5
 ; CHECK-NEXT:    jnp .LBB2_2
 ; CHECK-NEXT:  .LBB2_5: # %if.then
 ; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
-; CHECK-NEXT:    callq scale
+; CHECK-NEXT:    callq scale@PLT
 ; CHECK-NEXT:    jmp .LBB2_2
 ; CHECK-NEXT:  .LBB2_6: # %for.end52
 ; CHECK-NEXT:    addq $8, %rsp
diff --git a/llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
index 35c449e813c0947a59597e6c194c67211f264d38..7e9a727e7230be225328b834887256cac5a00ee3 100644
--- a/llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
@@ -1965,7 +1965,7 @@ define <4 x i64> @test_mm256_set1_epi64x(i64 %a0) nounwind {
 ; X64-LABEL: test_mm256_set1_epi64x:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovq %rdi, %xmm0
-; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X64-NEXT:    retq
   %res0 = insertelement <4 x i64> undef, i64 %a0, i32 0
diff --git a/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll b/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll
index 709507e1ed0cfe9274362c37bbc3e7ac8edf7475..7fff7e117b83318f1a321aea249da47c6756c29f 100644
--- a/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll
@@ -645,8 +645,8 @@ define void @test_x86_sse2_storeu_pd(i8* %a0, <2 x double> %a1) {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x57,0xc9]
-; X86-AVX-NEXT:    vmovhpd {{\.LCPI.*}}, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vmovhpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    # xmm1 = xmm1[0],mem[0]
 ; X86-AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x58,0xc1]
 ; X86-AVX-NEXT:    vmovupd %xmm0, (%eax) # encoding: [0xc5,0xf9,0x11,0x00]
@@ -656,8 +656,8 @@ define void @test_x86_sse2_storeu_pd(i8* %a0, <2 x double> %a1) {
 ; X86-AVX512VL:       # %bb.0:
 ; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-AVX512VL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0x57,0xc9]
-; X86-AVX512VL-NEXT:    vmovhpd {{\.LCPI.*}}, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vmovhpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    # xmm1 = xmm1[0],mem[0]
 ; X86-AVX512VL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x58,0xc1]
 ; X86-AVX512VL-NEXT:    vmovupd %xmm0, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x11,0x00]
@@ -667,7 +667,7 @@ define void @test_x86_sse2_storeu_pd(i8* %a0, <2 x double> %a1) {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x57,0xc9]
 ; X64-AVX-NEXT:    vmovhpd {{.*}}(%rip), %xmm1, %xmm1 # encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    # xmm1 = xmm1[0],mem[0]
 ; X64-AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x58,0xc1]
 ; X64-AVX-NEXT:    vmovupd %xmm0, (%rdi) # encoding: [0xc5,0xf9,0x11,0x07]
@@ -677,7 +677,7 @@ define void @test_x86_sse2_storeu_pd(i8* %a0, <2 x double> %a1) {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0x57,0xc9]
 ; X64-AVX512VL-NEXT:    vmovhpd {{.*}}(%rip), %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0x16,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    # xmm1 = xmm1[0],mem[0]
 ; X64-AVX512VL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x58,0xc1]
 ; X64-AVX512VL-NEXT:    vmovupd %xmm0, (%rdi) # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x11,0x07]
diff --git a/llvm/test/CodeGen/X86/avx-splat.ll b/llvm/test/CodeGen/X86/avx-splat.ll
index 7602975c8872de03c9e5897b0ec8e55ab29443a9..1890b44eb0750cb85765b771cc6226601b6f1b67 100644
--- a/llvm/test/CodeGen/X86/avx-splat.ll
+++ b/llvm/test/CodeGen/X86/avx-splat.ll
@@ -34,7 +34,7 @@ define <4 x i64> @funcC(i64 %q) nounwind uwtable readnone ssp {
 ; X64-LABEL: funcC:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    vmovq %rdi, %xmm0
-; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X64-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/avx2-arith.ll b/llvm/test/CodeGen/X86/avx2-arith.ll
index b694b98d04c47d970a67c521aff5c24be233427c..960fbf069bad0df1100f3b9d4b74512a02861722 100644
--- a/llvm/test/CodeGen/X86/avx2-arith.ll
+++ b/llvm/test/CodeGen/X86/avx2-arith.ll
@@ -148,7 +148,7 @@ define <16 x i8> @mul_v16i8(<16 x i8> %i, <16 x i8> %j) nounwind readnone {
 ; X32-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
 ; X32-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
 ; X32-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X32-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vzeroupper
@@ -303,7 +303,7 @@ define <8 x i32> @mul_const5(<8 x i32> %x) {
 define <8 x i32> @mul_const6(<8 x i32> %x) {
 ; X32-LABEL: mul_const6:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpmulld {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: mul_const6:
diff --git a/llvm/test/CodeGen/X86/avx2-conversions.ll b/llvm/test/CodeGen/X86/avx2-conversions.ll
index 0c9c552d0f76039f8520790923f32f1ce10d6eb3..68ea4deb9454377eccfa02e28ab7ab512c9ca8a8 100644
--- a/llvm/test/CodeGen/X86/avx2-conversions.ll
+++ b/llvm/test/CodeGen/X86/avx2-conversions.ll
@@ -159,7 +159,7 @@ define <16 x i16> @sext_16i8_16i16(<16 x i8> %z) {
 define <16 x i8> @trunc_16i16_16i8(<16 x i16> %z) {
 ; X32-LABEL: trunc_16i16_16i8:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X32-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll b/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
index ec9517508ec4ff65826dc1e6814c4bf1217b57f0..c0f080e8fb9b2877324482f7fd7bbd7b09f59ede 100644
--- a/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
+++ b/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
@@ -25,28 +25,28 @@ define <16 x i16> @test_x86_avx2_packssdw_fold() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_packssdw_fold:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
+; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_packssdw_fold:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packssdw_fold:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
   ret <16 x i16> %res
@@ -74,28 +74,28 @@ define <32 x i8> @test_x86_avx2_packsswb_fold() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_packsswb_fold:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
+; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_packsswb_fold:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packsswb_fold:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
   ret <32 x i8> %res
@@ -123,28 +123,28 @@ define <32 x i8> @test_x86_avx2_packuswb_fold() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_packuswb_fold:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_packuswb_fold:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packuswb_fold:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
   ret <32 x i8> %res
@@ -753,28 +753,28 @@ define <16 x i16> @test_x86_avx2_packusdw_fold() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_packusdw_fold:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
+; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_packusdw_fold:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packusdw_fold:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
   ret <16 x i16> %res
@@ -1025,26 +1025,26 @@ define <4 x i32> @test_x86_avx2_psllv_d_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,0,4294967295]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsllvd {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,1,4294967295]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x47,0xc9]
 ; X86-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psllv_d_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsllvd {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm1 # EVEX TO VEX Compression xmm1 = [1,1,1,4294967295]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm1 # EVEX TO VEX Compression xmm1 = [1,1,1,4294967295]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x47,0xc9]
 ; X86-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
@@ -1053,12 +1053,12 @@ define <4 x i32> @test_x86_avx2_psllv_d_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,0,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,1,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x47,0xc9]
 ; X64-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
@@ -1067,12 +1067,12 @@ define <4 x i32> @test_x86_avx2_psllv_d_const() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm1 # EVEX TO VEX Compression xmm1 = [1,1,1,4294967295]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x47,0xc9]
 ; X64-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
@@ -1103,29 +1103,29 @@ define <8 x i32> @test_x86_avx2_psllv_d_256_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psllv_d_256_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1133,14 +1133,14 @@ define <8 x i32> @test_x86_avx2_psllv_d_256_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvd {{.*}}(%rip), %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1148,14 +1148,14 @@ define <8 x i32> @test_x86_avx2_psllv_d_256_const() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res0 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2>)
@@ -1184,36 +1184,36 @@ define <2 x i64> @test_x86_avx2_psllv_q_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,0,4294967295,4294967295]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsllvq {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psllv_q_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,0,4294967295,4294967295]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,0,4294967295,4294967295]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsllvq {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_psllv_q_const:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,18446744073709551615]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psllv_q_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [4,18446744073709551615]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> <i64 4, i64 -1>, <2 x i64> <i64 1, i64 -1>)
   ret <2 x i64> %res
@@ -1240,36 +1240,36 @@ define <4 x i64> @test_x86_avx2_psllv_q_256_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [4,0,4,0,4,0,4294967295,4294967295]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsllvq {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvq {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psllv_q_256_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,0,4,0,4,0,4294967295,4294967295]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,0,4,0,4,0,4294967295,4294967295]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsllvq {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvq {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_psllv_q_256_const:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [4,4,4,18446744073709551615]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psllv_q_256_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,18446744073709551615]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
   ret <4 x i64> %res
@@ -1296,29 +1296,29 @@ define <4 x i32> @test_x86_avx2_psrlv_d_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,0,4294967295]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsrlvd {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,4,4,4294967295]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsrlvd {{\.LCPI.*}}, %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psrlv_d_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm1 # EVEX TO VEX Compression xmm1 = [4,4,4,4294967295]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm1 # EVEX TO VEX Compression xmm1 = [4,4,4,4294967295]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI.*}}, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1326,14 +1326,14 @@ define <4 x i32> @test_x86_avx2_psrlv_d_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,0,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,4,4,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1341,14 +1341,14 @@ define <4 x i32> @test_x86_avx2_psrlv_d_const() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm1 # EVEX TO VEX Compression xmm1 = [4,4,4,4294967295]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res0 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> <i32 2, i32 9, i32 0, i32 -1>, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
@@ -1378,29 +1378,29 @@ define <8 x i32> @test_x86_avx2_psrlv_d_256_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psrlv_d_256_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1408,14 +1408,14 @@ define <8 x i32> @test_x86_avx2_psrlv_d_256_const() {
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
@@ -1423,14 +1423,14 @@ define <8 x i32> @test_x86_avx2_psrlv_d_256_const() {
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res0 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2>)
@@ -1460,36 +1460,36 @@ define <2 x i64> @test_x86_avx2_psrlv_q_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,0,4,0]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsrlvq {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psrlv_q_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,0,4,0]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,0,4,0]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsrlvq {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_psrlv_q_const:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,4]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrlv_q_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [4,4]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> <i64 4, i64 4>, <2 x i64> <i64 1, i64 -1>)
   ret <2 x i64> %res
@@ -1517,36 +1517,36 @@ define <4 x i64> @test_x86_avx2_psrlv_q_256_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [4,0,4,0,4,0,4,0]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsrlvq {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvq {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psrlv_q_256_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,0,4,0,4,0,4,0]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,0,4,0,4,0,4,0]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsrlvq {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvq {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_psrlv_q_256_const:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [4,4,4,4]
 ; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x59,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrlv_q_256_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x59,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 4>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
   ret <4 x i64> %res
@@ -1573,36 +1573,36 @@ define <4 x i32> @test_x86_avx2_psrav_d_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,4294967284,23]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsravd {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsravd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psrav_d_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,4294967284,23]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,4294967284,23]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsravd {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsravd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_psrav_d_const:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,4294967284,23]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrav_d_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,4294967284,23]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> <i32 2, i32 9, i32 -12, i32 23>, <4 x i32> <i32 1, i32 18, i32 35, i32 52>)
   ret <4 x i32> %res
@@ -1628,36 +1628,36 @@ define <8 x i32> @test_x86_avx2_psrav_d_256_const() {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
 ; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsravd {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsravd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psrav_d_256_const:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
 ; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsravd {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsravd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_psrav_d_256_const:
 ; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrav_d_256_const:
 ; X64-AVX512VL:       # %bb.0:
 ; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> <i32 2, i32 9, i32 -12, i32 23, i32 -26, i32 37, i32 -40, i32 51>, <8 x i32> <i32 1, i32 18, i32 35, i32 52, i32 69, i32 15, i32 32, i32 49>)
   ret <8 x i32> %res
diff --git a/llvm/test/CodeGen/X86/avx2-nontemporal.ll b/llvm/test/CodeGen/X86/avx2-nontemporal.ll
index dac8b0e704effa7f02ccf476b5c1d3f7bef4c4e9..f0d8f9c6931a5d746eede62aa4a08f8858a74c2c 100644
--- a/llvm/test/CodeGen/X86/avx2-nontemporal.ll
+++ b/llvm/test/CodeGen/X86/avx2-nontemporal.ll
@@ -15,21 +15,21 @@ define i32 @f(<8 x float> %A, i8* %B, <4 x double> %C, <4 x i64> %E, <8 x i32> %
 ; X32-NEXT:    movl 8(%ebp), %ecx
 ; X32-NEXT:    movl 136(%ebp), %edx
 ; X32-NEXT:    movl (%edx), %eax
-; X32-NEXT:    vaddps {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vaddps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    vmovntps %ymm0, (%ecx)
-; X32-NEXT:    vpaddq {{\.LCPI.*}}, %ymm2, %ymm0
+; X32-NEXT:    vpaddq {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm0
 ; X32-NEXT:    addl (%edx), %eax
 ; X32-NEXT:    vmovntdq %ymm0, (%ecx)
-; X32-NEXT:    vaddpd {{\.LCPI.*}}, %ymm1, %ymm0
+; X32-NEXT:    vaddpd {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X32-NEXT:    addl (%edx), %eax
 ; X32-NEXT:    vmovntpd %ymm0, (%ecx)
-; X32-NEXT:    vpaddd {{\.LCPI.*}}, %ymm5, %ymm0
+; X32-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}, %ymm5, %ymm0
 ; X32-NEXT:    addl (%edx), %eax
 ; X32-NEXT:    vmovntdq %ymm0, (%ecx)
-; X32-NEXT:    vpaddw {{\.LCPI.*}}, %ymm4, %ymm0
+; X32-NEXT:    vpaddw {{\.LCPI[0-9]+_[0-9]+}}, %ymm4, %ymm0
 ; X32-NEXT:    addl (%edx), %eax
 ; X32-NEXT:    vmovntdq %ymm0, (%ecx)
-; X32-NEXT:    vpaddb {{\.LCPI.*}}, %ymm3, %ymm0
+; X32-NEXT:    vpaddb {{\.LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm0
 ; X32-NEXT:    addl (%edx), %eax
 ; X32-NEXT:    vmovntdq %ymm0, (%ecx)
 ; X32-NEXT:    movl %ebp, %esp
diff --git a/llvm/test/CodeGen/X86/avx2-shift.ll b/llvm/test/CodeGen/X86/avx2-shift.ll
index a9338007ececcd4730dd90a577b73919d67bc35a..bb6eceff081c8491116499d22b4b7db277140a46 100644
--- a/llvm/test/CodeGen/X86/avx2-shift.ll
+++ b/llvm/test/CodeGen/X86/avx2-shift.ll
@@ -424,7 +424,7 @@ define <32 x i8> @shl9(<32 x i8> %A) nounwind {
 ; X32-LABEL: shl9:
 ; X32:       # %bb.0:
 ; X32-NEXT:    vpsllw $3, %ymm0, %ymm0
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: shl9:
@@ -440,7 +440,7 @@ define <32 x i8> @shr9(<32 x i8> %A) nounwind {
 ; X32-LABEL: shr9:
 ; X32:       # %bb.0:
 ; X32-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: shr9:
@@ -472,7 +472,7 @@ define <32 x i8> @sra_v32i8(<32 x i8> %A) nounwind {
 ; X32-LABEL: sra_v32i8:
 ; X32:       # %bb.0:
 ; X32-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    vmovdqa {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; X32-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; X32-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/avx2-vbroadcasti128.ll b/llvm/test/CodeGen/X86/avx2-vbroadcasti128.ll
index 02e0b96e9d997351186d2ac8dfedd64b86bac5d8..d88d62dfd322fb7a2582ab4be745132243fd5b09 100644
--- a/llvm/test/CodeGen/X86/avx2-vbroadcasti128.ll
+++ b/llvm/test/CodeGen/X86/avx2-vbroadcasti128.ll
@@ -7,7 +7,7 @@ define <4 x double> @test_broadcast_2f64_4f64(<2 x double> *%p) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-NEXT:    vaddpd {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vaddpd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_broadcast_2f64_4f64:
@@ -26,7 +26,7 @@ define <4 x i64> @test_broadcast_2i64_4i64(<2 x i64> *%p) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-NEXT:    vpaddq {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpaddq {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_broadcast_2i64_4i64:
@@ -45,7 +45,7 @@ define <8 x float> @test_broadcast_4f32_8f32(<4 x float> *%p) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-NEXT:    vaddps {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vaddps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_broadcast_4f32_8f32:
@@ -64,7 +64,7 @@ define <8 x i32> @test_broadcast_4i32_8i32(<4 x i32> *%p) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-NEXT:    vpaddd {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_broadcast_4i32_8i32:
@@ -83,7 +83,7 @@ define <16 x i16> @test_broadcast_8i16_16i16(<8 x i16> *%p) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-NEXT:    vpaddw {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpaddw {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_broadcast_8i16_16i16:
@@ -102,7 +102,7 @@ define <32 x i8> @test_broadcast_16i8_32i8(<16 x i8> *%p) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; X32-NEXT:    vpaddb {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vpaddb {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_broadcast_16i8_32i8:
@@ -122,7 +122,7 @@ define <4 x double> @test_broadcast_2f64_4f64_reuse(<2 x double>* %p0, <2 x doub
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X32-NEXT:    vaddpd {{\.LCPI.*}}, %ymm1, %ymm0
+; X32-NEXT:    vaddpd {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X32-NEXT:    vmovapd %xmm1, (%eax)
 ; X32-NEXT:    retl
 ;
@@ -145,7 +145,7 @@ define <4 x i64> @test_broadcast_2i64_4i64_reuse(<2 x i64>* %p0, <2 x i64>* %p1)
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X32-NEXT:    vpaddq {{\.LCPI.*}}, %ymm1, %ymm0
+; X32-NEXT:    vpaddq {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X32-NEXT:    vmovdqa %xmm1, (%eax)
 ; X32-NEXT:    retl
 ;
@@ -168,7 +168,7 @@ define <8 x float> @test_broadcast_4f32_8f32_reuse(<4 x float>* %p0, <4 x float>
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X32-NEXT:    vaddps {{\.LCPI.*}}, %ymm1, %ymm0
+; X32-NEXT:    vaddps {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X32-NEXT:    vmovaps %xmm1, (%eax)
 ; X32-NEXT:    retl
 ;
@@ -191,7 +191,7 @@ define <8 x i32> @test_broadcast_4i32_8i32_reuse(<4 x i32>* %p0, <4 x i32>* %p1)
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X32-NEXT:    vpaddd {{\.LCPI.*}}, %ymm1, %ymm0
+; X32-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X32-NEXT:    vmovdqa %xmm1, (%eax)
 ; X32-NEXT:    retl
 ;
@@ -214,7 +214,7 @@ define <16 x i16> @test_broadcast_8i16_16i16_reuse(<8 x i16> *%p0, <8 x i16> *%p
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X32-NEXT:    vpaddw {{\.LCPI.*}}, %ymm1, %ymm0
+; X32-NEXT:    vpaddw {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X32-NEXT:    vmovdqa %xmm1, (%eax)
 ; X32-NEXT:    retl
 ;
@@ -237,7 +237,7 @@ define <32 x i8> @test_broadcast_16i8_32i8_reuse(<16 x i8> *%p0, <16 x i8> *%p1)
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1]
-; X32-NEXT:    vpaddb {{\.LCPI.*}}, %ymm1, %ymm0
+; X32-NEXT:    vpaddb {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X32-NEXT:    vmovdqa %xmm1, (%eax)
 ; X32-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/avx2-vector-shifts.ll b/llvm/test/CodeGen/X86/avx2-vector-shifts.ll
index d357150cde4e121658d4d2658388ebb96ef6a627..c1de8cd0627bf1107d9ad3abddc469f05234af83 100644
--- a/llvm/test/CodeGen/X86/avx2-vector-shifts.ll
+++ b/llvm/test/CodeGen/X86/avx2-vector-shifts.ll
@@ -486,10 +486,10 @@ define <32 x i8> @shl_32i8(<32 x i8> %r, <32 x i8> %a) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; X32-NEXT:    vpsllw $4, %ymm0, %ymm2
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X32-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X32-NEXT:    vpsllw $2, %ymm0, %ymm2
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X32-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; X32-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X32-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
@@ -692,14 +692,14 @@ define <32 x i8> @lshr_32i8(<32 x i8> %r, <32 x i8> %a) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; X32-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X32-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X32-NEXT:    vpsrlw $2, %ymm0, %ymm2
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X32-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; X32-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X32-NEXT:    vpsrlw $1, %ymm0, %ymm2
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X32-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; X32-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X32-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index 120d0ec83c4a0833ebe74b24672017e5214ef37c..e78ba264d7cc5dfd2f1a93d772c7322dd9839adf 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -1840,7 +1840,7 @@ define <2 x double> @test_mm_cvtu64_sd(<2 x double> %__A, i64 %__B) {
 ; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
 ; X86-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; X86-NEXT:    vsubpd {{\.LCPI.*}}, %xmm1, %xmm1
+; X86-NEXT:    vsubpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
 ; X86-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm1[1,0]
 ; X86-NEXT:    vaddsd %xmm1, %xmm2, %xmm1
 ; X86-NEXT:    vblendpd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
@@ -1888,7 +1888,7 @@ define <4 x float> @test_mm_cvtu64_ss(<4 x float> %__A, i64 %__B) {
 ; X86-NEXT:    vmovq %xmm1, {{[0-9]+}}(%esp)
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    fildll {{[0-9]+}}(%esp)
-; X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; X86-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
@@ -3118,7 +3118,7 @@ entry:
 define <8 x double> @test_mm512_fmsub_round_pd(<8 x double> %__A, <8 x double> %__B, <8 x double> %__C) {
 ; X86-LABEL: test_mm512_fmsub_round_pd:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxorq {{\.LCPI.*}}{1to8}, %zmm2, %zmm2
+; X86-NEXT:    vpxorq {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %zmm2, %zmm2
 ; X86-NEXT:    vfmadd213pd {rn-sae}, %zmm2, %zmm1, %zmm0
 ; X86-NEXT:    retl
 ;
@@ -3178,7 +3178,7 @@ entry:
 define <8 x double> @test_mm512_fnmadd_round_pd(<8 x double> %__A, <8 x double> %__B, <8 x double> %__C) {
 ; X86-LABEL: test_mm512_fnmadd_round_pd:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxorq {{\.LCPI.*}}{1to8}, %zmm0, %zmm0
+; X86-NEXT:    vpxorq {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %zmm0, %zmm0
 ; X86-NEXT:    vfmadd213pd {rn-sae}, %zmm2, %zmm1, %zmm0
 ; X86-NEXT:    retl
 ;
@@ -3349,7 +3349,7 @@ entry:
 define <8 x double> @test_mm512_fmsub_pd(<8 x double> %__A, <8 x double> %__B, <8 x double> %__C) {
 ; X86-LABEL: test_mm512_fmsub_pd:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxorq {{\.LCPI.*}}{1to8}, %zmm2, %zmm2
+; X86-NEXT:    vpxorq {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %zmm2, %zmm2
 ; X86-NEXT:    vfmadd213pd {{.*#+}} zmm0 = (zmm1 * zmm0) + zmm2
 ; X86-NEXT:    retl
 ;
@@ -3409,7 +3409,7 @@ entry:
 define <8 x double> @test_mm512_fnmadd_pd(<8 x double> %__A, <8 x double> %__B, <8 x double> %__C) {
 ; X86-LABEL: test_mm512_fnmadd_pd:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxorq {{\.LCPI.*}}{1to8}, %zmm0, %zmm0
+; X86-NEXT:    vpxorq {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %zmm0, %zmm0
 ; X86-NEXT:    vfmadd213pd {{.*#+}} zmm0 = (zmm1 * zmm0) + zmm2
 ; X86-NEXT:    retl
 ;
@@ -3582,7 +3582,7 @@ entry:
 define <16 x float> @test_mm512_fmsub_round_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_fmsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxord {{\.LCPI.*}}{1to16}, %zmm2, %zmm2
+; X86-NEXT:    vpxord {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm2, %zmm2
 ; X86-NEXT:    vfmadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0
 ; X86-NEXT:    retl
 ;
@@ -3642,7 +3642,7 @@ entry:
 define <16 x float> @test_mm512_fnmadd_round_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_fnmadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxord {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; X86-NEXT:    vpxord {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; X86-NEXT:    vfmadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0
 ; X86-NEXT:    retl
 ;
@@ -3813,7 +3813,7 @@ entry:
 define <16 x float> @test_mm512_fmsub_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_fmsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxord {{\.LCPI.*}}{1to16}, %zmm2, %zmm2
+; X86-NEXT:    vpxord {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm2, %zmm2
 ; X86-NEXT:    vfmadd213ps {{.*#+}} zmm0 = (zmm1 * zmm0) + zmm2
 ; X86-NEXT:    retl
 ;
@@ -3873,7 +3873,7 @@ entry:
 define <16 x float> @test_mm512_fnmadd_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_fnmadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxord {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; X86-NEXT:    vpxord {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; X86-NEXT:    vfmadd213ps {{.*#+}} zmm0 = (zmm1 * zmm0) + zmm2
 ; X86-NEXT:    retl
 ;
@@ -4046,7 +4046,7 @@ entry:
 define <8 x double> @test_mm512_fmsubadd_round_pd(<8 x double> %__A, <8 x double> %__B, <8 x double> %__C) {
 ; X86-LABEL: test_mm512_fmsubadd_round_pd:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxorq {{\.LCPI.*}}{1to8}, %zmm2, %zmm2
+; X86-NEXT:    vpxorq {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %zmm2, %zmm2
 ; X86-NEXT:    vfmaddsub213pd {rn-sae}, %zmm2, %zmm1, %zmm0
 ; X86-NEXT:    retl
 ;
@@ -4323,7 +4323,7 @@ entry:
 define <16 x float> @test_mm512_fmsubadd_round_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_fmsubadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    vpxord {{\.LCPI.*}}{1to16}, %zmm2, %zmm2
+; X86-NEXT:    vpxord {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm2, %zmm2
 ; X86-NEXT:    vfmaddsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0
 ; X86-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics.ll b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
index c008cc094adab3880444d9e6b62fe302a4511e94..818cd76946dbce2b3445260b8e2f514dc8347ea1 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
@@ -7118,9 +7118,9 @@ define <16 x i32> @test_x86_avx512_psllv_d_512_const() {
 ; X86-LABEL: test_x86_avx512_psllv_d_512_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,0,4294967295,3,7,4294967295,0,4,5,4294967294,0,5,3,4294967293,0]
-; X86-NEXT:    vpsllvd {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vpsllvd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4294967295]
-; X86-NEXT:    vpsllvd {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vpsllvd {{\.LCPI[0-9]+_[0-9]+}}, %zmm1, %zmm1
 ; X86-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %res0 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0, i32 4, i32 5, i32 -2, i32 0, i32 5, i32 3, i32 -3, i32 0>, <16 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
@@ -7191,9 +7191,9 @@ define <8 x i64> @test_x86_avx512_psllv_q_512_const() {
 ; X86-LABEL: test_x86_avx512_psllv_q_512_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,0,9,0,0,0,4294967295,4294967295,3,0,7,0,4294967295,4294967295,0,0]
-; X86-NEXT:    vpsllvq {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vpsllvq {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,0,4,0,4,0,4,0,4,0,4,0,4,0,4294967295,4294967295]
-; X86-NEXT:    vpsllvq {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vpsllvq {{\.LCPI[0-9]+_[0-9]+}}, %zmm1, %zmm1
 ; X86-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %res0 = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> <i64 2, i64 9, i64 0, i64 -1, i64 3, i64 7, i64 -1, i64 0>, <8 x i64> <i64 1, i64 0, i64 33, i64 -1,i64 2, i64 0, i64 34, i64 -2>)
@@ -7366,9 +7366,9 @@ define <16 x i32> @test_x86_avx512_psrlv_d_512_const() {
 ; X86-LABEL: test_x86_avx512_psrlv_d_512_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,0,4294967295,3,7,4294967295,0,4,5,4294967294,0,5,3,4294967293,0]
-; X86-NEXT:    vpsrlvd {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4294967295]
-; X86-NEXT:    vpsrlvd {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %zmm1, %zmm1
 ; X86-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %res0 = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0, i32 4, i32 5, i32 -2, i32 0, i32 5, i32 3, i32 -3, i32 0>, <16 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
@@ -7439,9 +7439,9 @@ define <8 x i64> @test_x86_avx512_psrlv_q_512_const() {
 ; X86-LABEL: test_x86_avx512_psrlv_q_512_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,0,9,0,0,0,4294967295,4294967295,3,0,7,0,4294967295,4294967295,0,0]
-; X86-NEXT:    vpsrlvq {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vpsrlvq {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,0,4,0,4,0,4,0,4,0,4,0,4,0,4294967295,4294967295]
-; X86-NEXT:    vpsrlvq {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vpsrlvq {{\.LCPI[0-9]+_[0-9]+}}, %zmm1, %zmm1
 ; X86-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %res0 = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> <i64 2, i64 9, i64 0, i64 -1, i64 3, i64 7, i64 -1, i64 0>, <8 x i64> <i64 1, i64 0, i64 33, i64 -1,i64 2, i64 0, i64 34, i64 -2>)
diff --git a/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll b/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
index 0ed73f142a84645e5869d1d5e805f991cbab0c06..462730a8f6c8941d29f86e60b5689752c2705462 100644
--- a/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
@@ -1292,18 +1292,18 @@ define <32 x i16> @test_x86_avx512_psrlv_w_512_const() optsize {
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X86-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-NEXT:    vpsrlvw {{\.LCPI.*}}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-NEXT:    vpsrlvw {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx512_psrlv_w_512_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <32 x i16> %res1
@@ -1410,18 +1410,18 @@ define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi_const(<32 x i16> %x0, <32
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,65524,23,65510,37,65496,51,2,9,65524,23,65510,37,65496,51,2,9,65524,23,65510,37,65496,51,2,9,65524,23,65510,37,65496,51]
 ; X86-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-NEXT:    vpsravw {{\.LCPI.*}}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-NEXT:    vpsravw {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav32_hi_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,65524,23,65510,37,65496,51,2,9,65524,23,65510,37,65496,51,2,9,65524,23,65510,37,65496,51,2,9,65524,23,65510,37,65496,51]
 ; X64-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsravw {{.*}}(%rip), %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> <i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51>, <32 x i16> <i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49>)
   ret <32 x i16> %1
@@ -1575,18 +1575,18 @@ define <32 x i16> @test_x86_avx512_psllv_w_512_const() optsize {
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X86-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-NEXT:    vpsllvw {{\.LCPI.*}}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-NEXT:    vpsllvw {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx512_psllv_w_512_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4,  i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 -1>)
   ret <32 x i16> %res1
diff --git a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll
index ab1af2c44a5d1e06c4bf9b9f87308332de46e757..03f1a2d51330f1f8be65d222cfb1709dd82239e8 100644
--- a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll
@@ -2153,20 +2153,20 @@ define <8 x i16>@test_int_x86_avx512_maskz_psrlv8_hi(<8 x i16> %x0, <8 x i16> %x
 define <8 x i16> @test_int_x86_avx512_psrlv_w_128_const() optsize {
 ; X86-LABEL: test_int_x86_avx512_psrlv_w_128_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
+; X86-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
 ; X86-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-NEXT:    vpsrlvw {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-NEXT:    vpsrlvw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_psrlv_w_128_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsrlvw {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <8 x i16> %res
@@ -2177,20 +2177,20 @@ declare <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16>, <8 x i16>)
 define <16 x i16> @test_int_x86_avx512_psrlv_w_256_const() optsize {
 ; X86-LABEL: test_int_x86_avx512_psrlv_w_256_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X86-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X86-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-NEXT:    vpsrlvw {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-NEXT:    vpsrlvw {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_psrlv_w_256_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <16 x i16> %res
@@ -2397,20 +2397,20 @@ define <8 x i16>@test_int_x86_avx512_maskz_psllv8_hi(<8 x i16> %x0, <8 x i16> %x
 define <8 x i16> @test_int_x86_avx512_psllv_w_128_const() optsize {
 ; X86-LABEL: test_int_x86_avx512_psllv_w_128_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
+; X86-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
 ; X86-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-NEXT:    vpsllvw {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-NEXT:    vpsllvw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_psllv_w_128_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsllvw {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <8 x i16> %res
@@ -2422,20 +2422,20 @@ declare <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16>, <8 x i16>)
 define <16 x i16> @test_int_x86_avx512_psllv_w_256_const() optsize {
 ; X86-LABEL: test_int_x86_avx512_psllv_w_256_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X86-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X86-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-NEXT:    vpsllvw {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-NEXT:    vpsllvw {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_psllv_w_256_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
 ; X64-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
   ret <16 x i16> %res
diff --git a/llvm/test/CodeGen/X86/avx512vbmi2-funnel-shifts.ll b/llvm/test/CodeGen/X86/avx512vbmi2-funnel-shifts.ll
index fe0d264e370be648cba8f5fd2da585632d45f06a..d00297a408a68c2b1ab981d3fe0910fe3545534b 100644
--- a/llvm/test/CodeGen/X86/avx512vbmi2-funnel-shifts.ll
+++ b/llvm/test/CodeGen/X86/avx512vbmi2-funnel-shifts.ll
@@ -5,7 +5,7 @@
 define <8 x i64> @avx512_funnel_shift_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 ; X86-LABEL: avx512_funnel_shift_q_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldvq {{\.LCPI.*}}, %zmm1, %zmm0
+; X86-NEXT:    vpshldvq {{\.LCPI[0-9]+_[0-9]+}}, %zmm1, %zmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: avx512_funnel_shift_q_512:
@@ -32,7 +32,7 @@ define <8 x i64> @avx512_funnel_shift_q_512_splat(<8 x i64> %a0, <8 x i64> %a1)
 define <16 x i32> @avx512_funnel_shift_d_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; X86-LABEL: avx512_funnel_shift_d_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldvd {{\.LCPI.*}}, %zmm1, %zmm0
+; X86-NEXT:    vpshldvd {{\.LCPI[0-9]+_[0-9]+}}, %zmm1, %zmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: avx512_funnel_shift_d_512:
@@ -59,7 +59,7 @@ define <16 x i32> @avx512_funnel_shift_d_512_splat(<16 x i32> %a0, <16 x i32> %a
 define <32 x i16> @avx512_funnel_shift_w_512(<32 x i16> %a0, <32 x i16> %a1) {
 ; X86-LABEL: avx512_funnel_shift_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldvw {{\.LCPI.*}}, %zmm1, %zmm0
+; X86-NEXT:    vpshldvw {{\.LCPI[0-9]+_[0-9]+}}, %zmm1, %zmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: avx512_funnel_shift_w_512:
diff --git a/llvm/test/CodeGen/X86/avx512vbmi2vl-funnel-shifts.ll b/llvm/test/CodeGen/X86/avx512vbmi2vl-funnel-shifts.ll
index 465abad18020d3569b9045163ec7e7361e666b81..588039ea78bf8b10104cf4c4ee7fb7692220a76a 100644
--- a/llvm/test/CodeGen/X86/avx512vbmi2vl-funnel-shifts.ll
+++ b/llvm/test/CodeGen/X86/avx512vbmi2vl-funnel-shifts.ll
@@ -5,7 +5,7 @@
 define <2 x i64> @avx512_funnel_shift_q_128(<2 x i64> %a0, <2 x i64> %a1) {
 ; X86-LABEL: avx512_funnel_shift_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldvq {{\.LCPI.*}}, %xmm1, %xmm0
+; X86-NEXT:    vpshldvq {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: avx512_funnel_shift_q_128:
@@ -21,7 +21,7 @@ define <2 x i64> @avx512_funnel_shift_q_128(<2 x i64> %a0, <2 x i64> %a1) {
 define <4 x i64> @avx512_funnel_shift_q_256(<4 x i64> %a0, <4 x i64> %a1) {
 ; X86-LABEL: avx512_funnel_shift_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldvq {{\.LCPI.*}}, %ymm1, %ymm0
+; X86-NEXT:    vpshldvq {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: avx512_funnel_shift_q_256:
@@ -59,7 +59,7 @@ define <4 x i64> @avx512_funnel_shift_q_256_splat(<4 x i64> %a0, <4 x i64> %a1)
 define <4 x i32> @avx512_funnel_shift_d_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86-LABEL: avx512_funnel_shift_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldvd {{\.LCPI.*}}, %xmm1, %xmm0
+; X86-NEXT:    vpshldvd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: avx512_funnel_shift_d_128:
@@ -75,7 +75,7 @@ define <4 x i32> @avx512_funnel_shift_d_128(<4 x i32> %a0, <4 x i32> %a1) {
 define <8 x i32> @avx512_funnel_shift_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; X86-LABEL: avx512_funnel_shift_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldvd {{\.LCPI.*}}, %ymm1, %ymm0
+; X86-NEXT:    vpshldvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: avx512_funnel_shift_d_256:
@@ -113,7 +113,7 @@ define <8 x i32> @avx512_funnel_shift_d_256_splat(<8 x i32> %a0, <8 x i32> %a1)
 define <8 x i16> @avx512_funnel_shift_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 ; X86-LABEL: avx512_funnel_shift_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldvw {{\.LCPI.*}}, %xmm1, %xmm0
+; X86-NEXT:    vpshldvw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: avx512_funnel_shift_w_128:
@@ -129,7 +129,7 @@ define <8 x i16> @avx512_funnel_shift_w_128(<8 x i16> %a0, <8 x i16> %a1) {
 define <16 x i16> @avx512_funnel_shift_w_256(<16 x i16> %a0, <16 x i16> %a1) {
 ; X86-LABEL: avx512_funnel_shift_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldvw {{\.LCPI.*}}, %ymm1, %ymm0
+; X86-NEXT:    vpshldvw {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: avx512_funnel_shift_w_256:
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
index 83f39c1dd7da9fc44dab2349cc1486363e5d795c..fd90a9fd1baa3c906a263232549a2d9265ceefc2 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
@@ -1905,7 +1905,7 @@ define <2 x i64> @test_mm_mask_set1_epi32(<2 x i64> %__O, i8 zeroext %__M)  {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vpbroadcastd {{\.LCPI.*}}, %xmm0 {%k1}
+; X86-NEXT:    vpbroadcastd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_mask_set1_epi32:
@@ -1927,7 +1927,7 @@ define <2 x i64> @test_mm_maskz_set1_epi32(i8 zeroext %__M) {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vpbroadcastd {{\.LCPI.*}}, %xmm0 {%k1} {z}
+; X86-NEXT:    vpbroadcastd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_maskz_set1_epi32:
@@ -1948,7 +1948,7 @@ define <4 x i64> @test_mm256_mask_set1_epi32(<4 x i64> %__O, i8 zeroext %__M)  {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vpbroadcastd {{\.LCPI.*}}, %ymm0 {%k1}
+; X86-NEXT:    vpbroadcastd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_mask_set1_epi32:
@@ -1969,7 +1969,7 @@ define <4 x i64> @test_mm256_maskz_set1_epi32(i8 zeroext %__M)  {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vpbroadcastd {{\.LCPI.*}}, %ymm0 {%k1} {z}
+; X86-NEXT:    vpbroadcastd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_maskz_set1_epi32:
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
index 15054e49b8ca747b8292365b2617c17bf5c93e48..1fef7ad034e67b4e3eac9f335f86e38822570421 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
@@ -7321,20 +7321,20 @@ define <8 x i32>@test_int_x86_avx512_maskz_psrav8_si(<8 x i32> %x0, <8 x i32> %x
 define <8 x i32>@test_int_x86_avx512_mask_psrav8_si_const() {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav8_si_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
+; X86-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
 ; X86-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-NEXT:    vpsravd {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-NEXT:    vpsravd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav8_si_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
 ; X64-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psrav8.si(<8 x i32> <i32 2, i32 9, i32 -12, i32 23, i32 -26, i32 37, i32 -40, i32 51>, <8 x i32> <i32 1, i32 18, i32 35, i32 52, i32 69, i32 15, i32 32, i32 49>, <8 x i32> zeroinitializer, i8 -1)
   ret <8 x i32> %res
@@ -8632,20 +8632,20 @@ define <2 x i64>@test_int_x86_avx512_maskz_psrav_q_128(<2 x i64> %x0, <2 x i64>
 define <2 x i64>@test_int_x86_avx512_mask_psrav_q_128_const(i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav_q_128_const:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,0,4294967287,4294967295]
+; X86-NEXT:    vmovdqa {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,0,4294967287,4294967295]
 ; X86-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-NEXT:    vpsravq {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x46,0x05,A,A,A,A]
-; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
+; X86-NEXT:    vpsravq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x46,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav_q_128_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [2,18446744073709551607]
 ; X64-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpsravq {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x46,0x05,A,A,A,A]
-; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.psrav.q.128(<2 x i64> <i64 2, i64 -9>, <2 x i64> <i64 1, i64 90>, <2 x i64> zeroinitializer, i8 -1)
   ret <2 x i64> %res
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-labels.ll b/llvm/test/CodeGen/X86/basic-block-sections-labels.ll
index f447834de8236fc69d839c9037a8be5a3e1f78bf..b9bcdc5258ada3b718e5475389b71679e36a3d15 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-labels.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-labels.ll
@@ -1,7 +1,7 @@
 ; Check the basic block sections labels option
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels | FileCheck %s --check-prefix=UNIQ
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=false -basic-block-sections=labels | FileCheck %s --check-prefix=NOUNIQ
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels -split-machine-functions | FileCheck %s --check-prefix=UNIQ
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels | FileCheck %s --check-prefixes=CHECK,UNIQ
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=false -basic-block-sections=labels | FileCheck %s --check-prefixes=CHECK,NOUNIQ
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels -split-machine-functions | FileCheck %s --check-prefixes=CHECK,UNIQ
 
 define void @_Z3bazb(i1 zeroext) personality i32 (...)* @__gxx_personality_v0 {
   br i1 %0, label %2, label %7
@@ -50,10 +50,10 @@ declare i32 @__gxx_personality_v0(...)
 ; CHECK-NEXT:	.byte	4
 ; CHECK-NEXT:	.uleb128 .Lfunc_begin0-.Lfunc_begin0
 ; CHECK-NEXT:	.uleb128 .LBB_END0_0-.Lfunc_begin0
-; CHECK-NEXT:	.byte	0
+; CHECK-NEXT:	.byte	8
 ; CHECK-NEXT:	.uleb128 .LBB0_1-.Lfunc_begin0
 ; CHECK-NEXT:	.uleb128 .LBB_END0_1-.LBB0_1
-; CHECK-NEXT:	.byte	0
+; CHECK-NEXT:	.byte	8
 ; CHECK-NEXT:	.uleb128 .LBB0_2-.Lfunc_begin0
 ; CHECK-NEXT:	.uleb128 .LBB_END0_2-.LBB0_2
 ; CHECK-NEXT:	.byte	1
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
index 1ce9f0113ca17df5ee34e87000dd4ff779658936..841782708cebd04281ac0e85809bf8591d2fb6cb 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
@@ -205,24 +205,19 @@ define <4 x i64> @ext_i4_4i64(i4 %a0) {
 ; AVX1-LABEL: ext_i4_4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i4_4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    # kill: def $edi killed $edi def $rdi
-; AVX2-NEXT:    vmovq %rdi, %xmm0
-; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm0
+; AVX2-NEXT:    vmovd %edi, %xmm0
+; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8]
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0
@@ -259,14 +254,10 @@ define <8 x i32> @ext_i8_8i32(i8 %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i8_8i32:
@@ -311,14 +302,10 @@ define <16 x i16> @ext_i16_16i16(i16 %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpeqw {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i16_16i16:
@@ -366,12 +353,10 @@ define <32 x i8> @ext_i32_32i8(i32 %a0) {
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,1,1,4,4,5,5]
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [9241421688590303745,9241421688590303745]
+; AVX1-NEXT:    # xmm2 = mem[0,0]
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -431,25 +416,20 @@ define <8 x i64> @ext_i8_8i64(i8 %a0) {
 ;
 ; AVX1-LABEL: ext_i8_8i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd %edi, %xmm0
+; AVX1-NEXT:    # kill: def $edi killed $edi def $rdi
+; AVX1-NEXT:    vmovq %rdi, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i8_8i64:
@@ -502,21 +482,15 @@ define <16 x i32> @ext_i16_16i32(i16 %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i16_16i32:
@@ -572,12 +546,10 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
 ; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [256,512,1024,2048,4096,8192,16384,32768]
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1,2,4,8,16,32,64,128]
+; AVX1-NEXT:    vpcmpeqw %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,1,1,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
@@ -585,9 +557,7 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqw %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqw %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
@@ -650,24 +620,18 @@ define <64 x i8> @ext_i64_64i8(i64 %a0) {
 ; AVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
 ; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,4,5,5]
 ; AVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,6,7,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[2,2,3,3,6,6,7,7]
 ; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i64_64i8:
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
index bbc1c148a6ab711baeb7e3fac81b0c486cdd3bb7..7c7db1678e7bebb9d282a411a4e0d5bd6a7e1c8c 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
@@ -261,26 +261,21 @@ define <4 x i64> @ext_i4_4i64(i4 %a0) {
 ; AVX1-LABEL: ext_i4_4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovd %edi, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlq $63, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlq $63, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlq $63, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i4_4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    # kill: def $edi killed $edi def $rdi
-; AVX2-NEXT:    vmovq %rdi, %xmm0
-; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm0
+; AVX2-NEXT:    vmovd %edi, %xmm0
+; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8]
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0
@@ -328,16 +323,12 @@ define <8 x i32> @ext_i8_8i32(i8 %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i8_8i32:
@@ -393,16 +384,12 @@ define <16 x i16> @ext_i16_16i16(i16 %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $15, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqw {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vpsrlw $15, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i16_16i16:
@@ -465,17 +452,15 @@ define <32 x i8> @ext_i32_32i8(i32 %a0) {
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,1,1,4,4,5,5]
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [9241421688590303745,9241421688590303745]
+; AVX1-NEXT:    # xmm2 = mem[0,0]
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -554,29 +539,24 @@ define <8 x i64> @ext_i8_8i64(i8 %a0) {
 ;
 ; AVX1-LABEL: ext_i8_8i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd %edi, %xmm0
+; AVX1-NEXT:    # kill: def $edi killed $edi def $rdi
+; AVX1-NEXT:    vmovq %rdi, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm0
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlq $63, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $63, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlq $63, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlq $63, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT:    vpsrlq $63, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlq $63, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i8_8i64:
@@ -643,25 +623,19 @@ define <16 x i32> @ext_i16_16i32(i16 %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm0
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrld $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrld $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i16_16i32:
@@ -730,14 +704,12 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
 ; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128]
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $15, %xmm4, %xmm4
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [256,512,1024,2048,4096,8192,16384,32768]
+; AVX1-NEXT:    vpcmpeqw %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $15, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,1,1,4,5,6,7]
@@ -745,11 +717,9 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $15, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqw %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $15, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
@@ -839,33 +809,27 @@ define <64 x i8> @ext_i64_64i8(i64 %a0) {
 ; AVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [9241421688590303745,9241421688590303745,9241421688590303745,9241421688590303745]
 ; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $7, %xmm3, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,4,5,5]
 ; AVX1-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,6,7,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[2,2,3,3,6,6,7,7]
 ; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm2
-; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $7, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ext_i64_64i8:
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool.ll
index 65bf43ff2ed9ea0e687e606fcd121b48fa2af53a..b4fb48ade63bbdf884cf9f88513c754f1dc2de7a 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool.ll
@@ -205,17 +205,15 @@ define <32 x i1> @bitcast_i32_32i1(i32 %a0) {
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,1,1,4,4,5,5]
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [9241421688590303745,9241421688590303745]
+; AVX1-NEXT:    # xmm2 = mem[0,0]
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index 8e2f6f9b463b0d23f6b72c900c65d6814a4f3d7f..e10f67d146397dd987fd936402faef7e8410c084 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -79,7 +79,7 @@ define <2 x i16> @test_bitreverse_v2i16(<2 x i16> %a) nounwind {
 ;
 ; X86XOP-LABEL: test_bitreverse_v2i16:
 ; X86XOP:       # %bb.0:
-; X86XOP-NEXT:    vpperm {{\.LCPI.*}}, %xmm0, %xmm0, %xmm0
+; X86XOP-NEXT:    vpperm {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    retl
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %a)
   ret <2 x i16> %b
@@ -155,7 +155,7 @@ define i64 @test_bitreverse_i64(i64 %a) nounwind {
 ; X86XOP-LABEL: test_bitreverse_i64:
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
-; X86XOP-NEXT:    vpperm {{\.LCPI.*}}, %xmm0, %xmm0, %xmm0
+; X86XOP-NEXT:    vpperm {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    vpextrd $1, %xmm0, %edx
 ; X86XOP-NEXT:    retl
@@ -213,7 +213,7 @@ define i32 @test_bitreverse_i32(i32 %a) nounwind {
 ; X86XOP-LABEL: test_bitreverse_i32:
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86XOP-NEXT:    vpperm {{\.LCPI.*}}, %xmm0, %xmm0, %xmm0
+; X86XOP-NEXT:    vpperm {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    retl
   %b = call i32 @llvm.bitreverse.i32(i32 %a)
@@ -272,7 +272,7 @@ define i24 @test_bitreverse_i24(i24 %a) nounwind {
 ; X86XOP-LABEL: test_bitreverse_i24:
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86XOP-NEXT:    vpperm {{\.LCPI.*}}, %xmm0, %xmm0, %xmm0
+; X86XOP-NEXT:    vpperm {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    shrl $8, %eax
 ; X86XOP-NEXT:    retl
@@ -332,7 +332,7 @@ define i16 @test_bitreverse_i16(i16 %a) nounwind {
 ; X86XOP-LABEL: test_bitreverse_i16:
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86XOP-NEXT:    vpperm {{\.LCPI.*}}, %xmm0, %xmm0, %xmm0
+; X86XOP-NEXT:    vpperm {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86XOP-NEXT:    retl
@@ -383,7 +383,7 @@ define i8 @test_bitreverse_i8(i8 %a) {
 ; X86XOP-LABEL: test_bitreverse_i8:
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86XOP-NEXT:    vpperm {{\.LCPI.*}}, %xmm0, %xmm0, %xmm0
+; X86XOP-NEXT:    vpperm {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    # kill: def $al killed $al killed $eax
 ; X86XOP-NEXT:    retl
@@ -436,7 +436,7 @@ define i4 @test_bitreverse_i4(i4 %a) {
 ; X86XOP-LABEL: test_bitreverse_i4:
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86XOP-NEXT:    vpperm {{\.LCPI.*}}, %xmm0, %xmm0, %xmm0
+; X86XOP-NEXT:    vpperm {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    shrb $4, %al
 ; X86XOP-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll b/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
index 4840369f7d93489631fd49abbb7d12f7ad4bac19..6ce20f18d44367c08b9fcdbb352711dc6caa8e68 100644
--- a/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
+++ b/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
@@ -128,7 +128,7 @@ define <32 x i8> @f32xi8_i16(<32 x i8> %a) {
 ; AVX-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f32xi8_i16:
@@ -168,7 +168,7 @@ define <32 x i8> @f32xi8_i32(<32 x i8> %a) {
 ; AVX-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f32xi8_i32:
@@ -209,7 +209,7 @@ define <32 x i8> @f32xi8_i64(<32 x i8> %a) {
 ; AVX-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f32xi8_i64:
@@ -250,7 +250,7 @@ define <32 x i8> @f32xi8_i128(<32 x i8> %a) {
 ; AVX-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f32xi8_i128:
@@ -716,7 +716,7 @@ define <16 x i16> @f16xi16_i32(<16 x i16> %a) {
 ; AVX-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f16xi16_i32:
@@ -757,7 +757,7 @@ define <16 x i16> @f16xi16_i64(<16 x i16> %a) {
 ; AVX-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f16xi16_i64:
@@ -798,7 +798,7 @@ define <16 x i16> @f16xi16_i128(<16 x i16> %a) {
 ; AVX-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f16xi16_i128:
@@ -1161,7 +1161,7 @@ define <8 x i32> @f8xi32_i64(<8 x i32> %a) {
 ; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f8xi32_i64:
@@ -1202,7 +1202,7 @@ define <8 x i32> @f8xi32_i128(<8 x i32> %a) {
 ; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f8xi32_i128:
@@ -1386,7 +1386,7 @@ define <4 x i64> @f4xi64_i128(<4 x i64> %a) {
 ; AVX-NEXT:    vpaddq %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX-NEXT:    retl
 ;
 ; ALL32-LABEL: f4xi64_i128:
diff --git a/llvm/test/CodeGen/X86/cleanuppad-inalloca.ll b/llvm/test/CodeGen/X86/cleanuppad-inalloca.ll
index 4e28b5f1a3e928e70b4de3a241a6ef8f02e43b66..927956d44129570a77254d4c76b1e1be6f9813f4 100644
--- a/llvm/test/CodeGen/X86/cleanuppad-inalloca.ll
+++ b/llvm/test/CodeGen/X86/cleanuppad-inalloca.ll
@@ -25,7 +25,7 @@ entry:
           to label %invoke.cont unwind label %ehcleanup
 
 invoke.cont:                                      ; preds = %entry
-  call void @takes_two(<{ %struct.A, %struct.A }>* inalloca nonnull %argmem)
+  call void @takes_two(<{ %struct.A, %struct.A }>* inalloca(<{ %struct.A, %struct.A }>) nonnull %argmem)
   ret void
 
 ehcleanup:                                        ; preds = %entry
@@ -57,7 +57,7 @@ ehcleanup:                                        ; preds = %entry
 ; CHECK: addl $8, %esp
 ; CHECK: retl
 
-declare void @takes_two(<{ %struct.A, %struct.A }>* inalloca) #0
+declare void @takes_two(<{ %struct.A, %struct.A }>* inalloca(<{ %struct.A, %struct.A }>)) #0
 
 declare x86_thiscallcc %struct.A* @"\01??0A@@QAE@XZ"(%struct.A* returned) #0
 
diff --git a/llvm/test/CodeGen/X86/cmov-fp.ll b/llvm/test/CodeGen/X86/cmov-fp.ll
index 756324bbdfdc9c24aefb2b2b88aeda4875b77b22..9af5483b95c060ef18adebc4ad135f47ac3c5e75 100644
--- a/llvm/test/CodeGen/X86/cmov-fp.ll
+++ b/llvm/test/CodeGen/X86/cmov-fp.ll
@@ -32,7 +32,7 @@ define double @test1(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE2-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOSSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovnbe %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -43,7 +43,7 @@ define double @test1(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE1-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovnbe %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -54,7 +54,7 @@ define double @test1(i32 %a, i32 %b, double %x) nounwind {
 ; NOCMOV-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    ja .LBB0_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -95,7 +95,7 @@ define double @test2(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE2-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOSSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovnb %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -106,7 +106,7 @@ define double @test2(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE1-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovnb %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -117,7 +117,7 @@ define double @test2(i32 %a, i32 %b, double %x) nounwind {
 ; NOCMOV-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jae .LBB1_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -158,7 +158,7 @@ define double @test3(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE2-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOSSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovb %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -169,7 +169,7 @@ define double @test3(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE1-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovb %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -180,7 +180,7 @@ define double @test3(i32 %a, i32 %b, double %x) nounwind {
 ; NOCMOV-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jb .LBB2_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -221,7 +221,7 @@ define double @test4(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE2-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOSSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovbe %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -232,7 +232,7 @@ define double @test4(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE1-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovbe %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -243,7 +243,7 @@ define double @test4(i32 %a, i32 %b, double %x) nounwind {
 ; NOCMOV-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jbe .LBB3_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -286,7 +286,7 @@ define double @test5(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    setg %al
 ; NOSSE2-NEXT:    testb %al, %al
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovne %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -299,7 +299,7 @@ define double @test5(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setg %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -310,7 +310,7 @@ define double @test5(i32 %a, i32 %b, double %x) nounwind {
 ; NOCMOV-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jg .LBB4_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -353,7 +353,7 @@ define double @test6(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    setge %al
 ; NOSSE2-NEXT:    testb %al, %al
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovne %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -366,7 +366,7 @@ define double @test6(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setge %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -377,7 +377,7 @@ define double @test6(i32 %a, i32 %b, double %x) nounwind {
 ; NOCMOV-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jge .LBB5_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -420,7 +420,7 @@ define double @test7(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    setl %al
 ; NOSSE2-NEXT:    testb %al, %al
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovne %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -433,7 +433,7 @@ define double @test7(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setl %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -444,7 +444,7 @@ define double @test7(i32 %a, i32 %b, double %x) nounwind {
 ; NOCMOV-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jl .LBB6_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -487,7 +487,7 @@ define double @test8(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    setle %al
 ; NOSSE2-NEXT:    testb %al, %al
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovne %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -500,7 +500,7 @@ define double @test8(i32 %a, i32 %b, double %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setle %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -511,7 +511,7 @@ define double @test8(i32 %a, i32 %b, double %x) nounwind {
 ; NOCMOV-NEXT:    fldl {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jle .LBB7_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -565,7 +565,7 @@ define float @test9(i32 %a, i32 %b, float %x) nounwind {
 ; NOSSE1-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovnbe %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -576,7 +576,7 @@ define float @test9(i32 %a, i32 %b, float %x) nounwind {
 ; NOCMOV-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    ja .LBB8_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -630,7 +630,7 @@ define float @test10(i32 %a, i32 %b, float %x) nounwind {
 ; NOSSE1-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovnb %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -641,7 +641,7 @@ define float @test10(i32 %a, i32 %b, float %x) nounwind {
 ; NOCMOV-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jae .LBB9_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -695,7 +695,7 @@ define float @test11(i32 %a, i32 %b, float %x) nounwind {
 ; NOSSE1-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovb %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -706,7 +706,7 @@ define float @test11(i32 %a, i32 %b, float %x) nounwind {
 ; NOCMOV-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jb .LBB10_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -760,7 +760,7 @@ define float @test12(i32 %a, i32 %b, float %x) nounwind {
 ; NOSSE1-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovbe %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -771,7 +771,7 @@ define float @test12(i32 %a, i32 %b, float %x) nounwind {
 ; NOCMOV-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jbe .LBB11_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -827,7 +827,7 @@ define float @test13(i32 %a, i32 %b, float %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setg %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -838,7 +838,7 @@ define float @test13(i32 %a, i32 %b, float %x) nounwind {
 ; NOCMOV-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jg .LBB12_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -894,7 +894,7 @@ define float @test14(i32 %a, i32 %b, float %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setge %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -905,7 +905,7 @@ define float @test14(i32 %a, i32 %b, float %x) nounwind {
 ; NOCMOV-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jge .LBB13_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -961,7 +961,7 @@ define float @test15(i32 %a, i32 %b, float %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setl %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -972,7 +972,7 @@ define float @test15(i32 %a, i32 %b, float %x) nounwind {
 ; NOCMOV-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jl .LBB14_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -1028,7 +1028,7 @@ define float @test16(i32 %a, i32 %b, float %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setle %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -1039,7 +1039,7 @@ define float @test16(i32 %a, i32 %b, float %x) nounwind {
 ; NOCMOV-NEXT:    flds {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jle .LBB15_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -1058,7 +1058,7 @@ define x86_fp80 @test17(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; SSE-NEXT:    flds {{\.LCPI.*}}
+; SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; SSE-NEXT:    fxch %st(1)
 ; SSE-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fcmovnbe %st(1), %st
@@ -1070,7 +1070,7 @@ define x86_fp80 @test17(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE2-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOSSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovnbe %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -1081,7 +1081,7 @@ define x86_fp80 @test17(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE1-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovnbe %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -1092,7 +1092,7 @@ define x86_fp80 @test17(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOCMOV-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    ja .LBB16_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -1111,7 +1111,7 @@ define x86_fp80 @test18(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; SSE-NEXT:    flds {{\.LCPI.*}}
+; SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; SSE-NEXT:    fxch %st(1)
 ; SSE-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fcmovnb %st(1), %st
@@ -1123,7 +1123,7 @@ define x86_fp80 @test18(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE2-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOSSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovnb %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -1134,7 +1134,7 @@ define x86_fp80 @test18(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE1-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovnb %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -1145,7 +1145,7 @@ define x86_fp80 @test18(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOCMOV-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jae .LBB17_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -1164,7 +1164,7 @@ define x86_fp80 @test19(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; SSE-NEXT:    flds {{\.LCPI.*}}
+; SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; SSE-NEXT:    fxch %st(1)
 ; SSE-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fcmovb %st(1), %st
@@ -1176,7 +1176,7 @@ define x86_fp80 @test19(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE2-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOSSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovb %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -1187,7 +1187,7 @@ define x86_fp80 @test19(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE1-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovb %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -1198,7 +1198,7 @@ define x86_fp80 @test19(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOCMOV-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jb .LBB18_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -1217,7 +1217,7 @@ define x86_fp80 @test20(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; SSE-NEXT:    flds {{\.LCPI.*}}
+; SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; SSE-NEXT:    fxch %st(1)
 ; SSE-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fcmovbe %st(1), %st
@@ -1229,7 +1229,7 @@ define x86_fp80 @test20(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE2-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOSSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovbe %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -1240,7 +1240,7 @@ define x86_fp80 @test20(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE1-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOSSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovbe %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -1251,7 +1251,7 @@ define x86_fp80 @test20(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOCMOV-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jbe .LBB19_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -1270,7 +1270,7 @@ define x86_fp80 @test21(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; SSE-NEXT:    flds {{\.LCPI.*}}
+; SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; SSE-NEXT:    fxch %st(1)
 ; SSE-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    setg %al
@@ -1286,7 +1286,7 @@ define x86_fp80 @test21(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    setg %al
 ; NOSSE2-NEXT:    testb %al, %al
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovne %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -1299,7 +1299,7 @@ define x86_fp80 @test21(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setg %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -1310,7 +1310,7 @@ define x86_fp80 @test21(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOCMOV-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jg .LBB20_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -1330,7 +1330,7 @@ define x86_fp80 @test22(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; SSE-NEXT:    flds {{\.LCPI.*}}
+; SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; SSE-NEXT:    fxch %st(1)
 ; SSE-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    setge %al
@@ -1346,7 +1346,7 @@ define x86_fp80 @test22(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    setge %al
 ; NOSSE2-NEXT:    testb %al, %al
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovne %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -1359,7 +1359,7 @@ define x86_fp80 @test22(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setge %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -1370,7 +1370,7 @@ define x86_fp80 @test22(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOCMOV-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jge .LBB21_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -1389,7 +1389,7 @@ define x86_fp80 @test23(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; SSE-NEXT:    flds {{\.LCPI.*}}
+; SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; SSE-NEXT:    fxch %st(1)
 ; SSE-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    setl %al
@@ -1405,7 +1405,7 @@ define x86_fp80 @test23(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    setl %al
 ; NOSSE2-NEXT:    testb %al, %al
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovne %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -1418,7 +1418,7 @@ define x86_fp80 @test23(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setl %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -1429,7 +1429,7 @@ define x86_fp80 @test23(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOCMOV-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jl .LBB22_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
@@ -1448,7 +1448,7 @@ define x86_fp80 @test24(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; SSE-NEXT:    flds {{\.LCPI.*}}
+; SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; SSE-NEXT:    fxch %st(1)
 ; SSE-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; SSE-NEXT:    setle %al
@@ -1464,7 +1464,7 @@ define x86_fp80 @test24(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE2-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE2-NEXT:    setle %al
 ; NOSSE2-NEXT:    testb %al, %al
-; NOSSE2-NEXT:    flds {{\.LCPI.*}}
+; NOSSE2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE2-NEXT:    fxch %st(1)
 ; NOSSE2-NEXT:    fcmovne %st(1), %st
 ; NOSSE2-NEXT:    fstp %st(1)
@@ -1477,7 +1477,7 @@ define x86_fp80 @test24(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOSSE1-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; NOSSE1-NEXT:    setle %al
 ; NOSSE1-NEXT:    testb %al, %al
-; NOSSE1-NEXT:    flds {{\.LCPI.*}}
+; NOSSE1-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOSSE1-NEXT:    fxch %st(1)
 ; NOSSE1-NEXT:    fcmovne %st(1), %st
 ; NOSSE1-NEXT:    fstp %st(1)
@@ -1488,7 +1488,7 @@ define x86_fp80 @test24(i32 %a, i32 %b, x86_fp80 %x) nounwind {
 ; NOCMOV-NEXT:    fldt {{[0-9]+}}(%esp)
 ; NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; NOCMOV-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; NOCMOV-NEXT:    flds {{\.LCPI.*}}
+; NOCMOV-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; NOCMOV-NEXT:    jle .LBB23_2
 ; NOCMOV-NEXT:  # %bb.1:
 ; NOCMOV-NEXT:    fstp %st(0)
diff --git a/llvm/test/CodeGen/X86/cmp.ll b/llvm/test/CodeGen/X86/cmp.ll
index ce441bbcc388360851d53789a7406d7324663dcc..bda450137d025b98796346f586ca4bedaa084a34 100644
--- a/llvm/test/CodeGen/X86/cmp.ll
+++ b/llvm/test/CodeGen/X86/cmp.ll
@@ -108,12 +108,12 @@ define dso_local i32 @test5(double %A) nounwind {
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    ucomisd {{.*}}(%rip), %xmm0 # encoding: [0x66,0x0f,0x2e,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    ja .LBB5_3 # encoding: [0x77,A]
 ; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB5_3-1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.1: # %entry
 ; CHECK-NEXT:    ucomisd {{.*}}(%rip), %xmm0 # encoding: [0x66,0x0f,0x2e,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    jb .LBB5_3 # encoding: [0x72,A]
 ; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB5_3-1, kind: FK_PCRel_1
 ; CHECK-NEXT:  # %bb.2: # %bb12
diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll
index ab25b64960a7389ade78053711d13ef9acfe02e8..dee7605e541e783a0af3e83a653fe80dbd1558c9 100644
--- a/llvm/test/CodeGen/X86/code-model-elf.ll
+++ b/llvm/test/CodeGen/X86/code-model-elf.ll
@@ -417,13 +417,13 @@ define dso_local float @load_constant_pool(float %x) #0 {
 ;
 ; MEDIUM-STATIC-LABEL: load_constant_pool:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; MEDIUM-STATIC-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; MEDIUM-STATIC-NEXT:    addss (%rax), %xmm0
 ; MEDIUM-STATIC-NEXT:    retq
 ;
 ; LARGE-STATIC-LABEL: load_constant_pool:
 ; LARGE-STATIC:       # %bb.0:
-; LARGE-STATIC-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; LARGE-STATIC-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; LARGE-STATIC-NEXT:    addss (%rax), %xmm0
 ; LARGE-STATIC-NEXT:    retq
 ;
@@ -435,7 +435,7 @@ define dso_local float @load_constant_pool(float %x) #0 {
 ; MEDIUM-PIC-LABEL: load_constant_pool:
 ; MEDIUM-PIC:       # %bb.0:
 ; MEDIUM-PIC-NEXT:    leaq {{.*}}(%rip), %rax
-; MEDIUM-PIC-NEXT:    movabsq ${{\.LCPI.*}}@GOTOFF, %rcx
+; MEDIUM-PIC-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}@GOTOFF, %rcx
 ; MEDIUM-PIC-NEXT:    addss (%rax,%rcx), %xmm0
 ; MEDIUM-PIC-NEXT:    retq
 ;
@@ -445,7 +445,7 @@ define dso_local float @load_constant_pool(float %x) #0 {
 ; LARGE-PIC-NEXT:    leaq .L11${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L11$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
-; LARGE-PIC-NEXT:    movabsq ${{\.LCPI.*}}@GOTOFF, %rax
+; LARGE-PIC-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addss (%rcx,%rax), %xmm0
 ; LARGE-PIC-NEXT:    retq
   %a = fadd float %x, 1.0
diff --git a/llvm/test/CodeGen/X86/combine-bextr.ll b/llvm/test/CodeGen/X86/combine-bextr.ll
index a6cf651a399297f552b8189cbbad7aac43af8ca1..8744beb1e20ec81598936c7319110a68d46339f5 100644
--- a/llvm/test/CodeGen/X86/combine-bextr.ll
+++ b/llvm/test/CodeGen/X86/combine-bextr.ll
@@ -40,8 +40,8 @@ define float @bextr_uitofp(i32 %x, i32 %y) {
 ; X32-NEXT:    movl $3855, %eax # imm = 0xF0F
 ; X32-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movd %eax, %xmm0
-; X32-NEXT:    por {{\.LCPI.*}}, %xmm0
-; X32-NEXT:    subsd {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    por {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X32-NEXT:    subsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    cvtsd2ss %xmm0, %xmm0
 ; X32-NEXT:    movss %xmm0, (%esp)
 ; X32-NEXT:    flds (%esp)
diff --git a/llvm/test/CodeGen/X86/combine-bitreverse.ll b/llvm/test/CodeGen/X86/combine-bitreverse.ll
index 8d268ddd75ee37b52920a004710068a30a88c251..1875c3f889f07b34174347866b2a61125d2d6640 100644
--- a/llvm/test/CodeGen/X86/combine-bitreverse.ll
+++ b/llvm/test/CodeGen/X86/combine-bitreverse.ll
@@ -51,23 +51,23 @@ define <4 x i32> @test_demandedbits_bitreverse(<4 x i32> %a0) nounwind {
 ; X86-NEXT:    packuswb %xmm2, %xmm0
 ; X86-NEXT:    movdqa %xmm0, %xmm1
 ; X86-NEXT:    psllw $4, %xmm1
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-NEXT:    psrlw $4, %xmm0
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    por %xmm1, %xmm0
 ; X86-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; X86-NEXT:    pand %xmm0, %xmm1
 ; X86-NEXT:    psllw $2, %xmm1
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    psrlw $2, %xmm0
 ; X86-NEXT:    por %xmm1, %xmm0
 ; X86-NEXT:    movdqa {{.*#+}} xmm1 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
 ; X86-NEXT:    pand %xmm0, %xmm1
 ; X86-NEXT:    paddb %xmm1, %xmm1
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    psrlw $1, %xmm0
 ; X86-NEXT:    por %xmm1, %xmm0
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_demandedbits_bitreverse:
diff --git a/llvm/test/CodeGen/X86/combine-bitselect.ll b/llvm/test/CodeGen/X86/combine-bitselect.ll
index d57bd877500c7ab4a5e9146c8653f17d94d88401..614d134173e7ba88661c1689b45807464363ffd4 100644
--- a/llvm/test/CodeGen/X86/combine-bitselect.ll
+++ b/llvm/test/CodeGen/X86/combine-bitselect.ll
@@ -505,26 +505,18 @@ define <4 x i64> @bitselect_v4i64_broadcast_rrr(<4 x i64> %a0, <4 x i64> %a1, i6
 ; XOP-LABEL: bitselect_v4i64_broadcast_rrr:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vmovq %rdi, %xmm2
-; XOP-NEXT:    vmovq %rdi, %xmm3
-; XOP-NEXT:    vmovddup {{.*#+}} xmm2 = xmm2[0,0]
+; XOP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
 ; XOP-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
-; XOP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
-; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
-; XOP-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; XOP-NEXT:    vandnps %ymm1, %ymm3, %ymm1
-; XOP-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; XOP-NEXT:    vpcmov %ymm2, %ymm1, %ymm0, %ymm0
 ; XOP-NEXT:    retq
 ;
 ; AVX1-LABEL: bitselect_v4i64_broadcast_rrr:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq %rdi, %xmm2
-; AVX1-NEXT:    vmovq %rdi, %xmm3
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = xmm2[0,0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm3
 ; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vandnps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -881,32 +873,22 @@ define <8 x i64> @bitselect_v8i64_broadcast_rrr(<8 x i64> %a0, <8 x i64> %a1, i6
 ; XOP-LABEL: bitselect_v8i64_broadcast_rrr:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vmovq %rdi, %xmm4
-; XOP-NEXT:    vmovq %rdi, %xmm5
-; XOP-NEXT:    vmovddup {{.*#+}} xmm4 = xmm4[0,0]
+; XOP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
 ; XOP-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; XOP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[0,1,0,1]
-; XOP-NEXT:    vinsertf128 $1, %xmm5, %ymm5, %ymm5
-; XOP-NEXT:    vandps %ymm4, %ymm1, %ymm1
-; XOP-NEXT:    vandps %ymm4, %ymm0, %ymm0
-; XOP-NEXT:    vandnps %ymm3, %ymm5, %ymm3
-; XOP-NEXT:    vorps %ymm3, %ymm1, %ymm1
-; XOP-NEXT:    vandnps %ymm2, %ymm5, %ymm2
-; XOP-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; XOP-NEXT:    vpcmov %ymm4, %ymm2, %ymm0, %ymm0
+; XOP-NEXT:    vpcmov %ymm4, %ymm3, %ymm1, %ymm1
 ; XOP-NEXT:    retq
 ;
 ; AVX1-LABEL: bitselect_v8i64_broadcast_rrr:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq %rdi, %xmm4
-; AVX1-NEXT:    vmovq %rdi, %xmm5
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm4 = xmm4[0,0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm5, %ymm5
 ; AVX1-NEXT:    vandps %ymm4, %ymm1, %ymm1
 ; AVX1-NEXT:    vandps %ymm4, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm3, %ymm5, %ymm3
+; AVX1-NEXT:    vandnps %ymm3, %ymm4, %ymm3
 ; AVX1-NEXT:    vorps %ymm3, %ymm1, %ymm1
-; AVX1-NEXT:    vandnps %ymm2, %ymm5, %ymm2
+; AVX1-NEXT:    vandnps %ymm2, %ymm4, %ymm2
 ; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/combine-multiplies.ll b/llvm/test/CodeGen/X86/combine-multiplies.ll
index 052c96fd2159382a38d0c8d61e054db78cc1c087..73f9642a2197f38d5d494045403e68e48b874449 100644
--- a/llvm/test/CodeGen/X86/combine-multiplies.ll
+++ b/llvm/test/CodeGen/X86/combine-multiplies.ll
@@ -116,7 +116,7 @@ define void @testCombineMultiplies_splat(<4 x i32> %v1) nounwind {
 ; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [242,242,242,242]
 ; CHECK-NEXT:    paddd %xmm0, %xmm2
-; CHECK-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK-NEXT:    movdqa %xmm2, v2
 ; CHECK-NEXT:    movdqa %xmm0, v3
 ; CHECK-NEXT:    movdqa %xmm1, x
@@ -151,7 +151,7 @@ define void @testCombineMultiplies_non_splat(<4 x i32> %v1) nounwind {
 ; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [242,726,1452,2420]
 ; CHECK-NEXT:    paddd %xmm0, %xmm2
-; CHECK-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK-NEXT:    movdqa %xmm2, v2
 ; CHECK-NEXT:    movdqa %xmm0, v3
 ; CHECK-NEXT:    movdqa %xmm1, x
diff --git a/llvm/test/CodeGen/X86/combine-pmuldq.ll b/llvm/test/CodeGen/X86/combine-pmuldq.ll
index 27823cf5fe8c5e95f5ece7f2ed176adc54bded9e..ae619ab590ecc2aca89960b1f5a127e2b5fc2736 100644
--- a/llvm/test/CodeGen/X86/combine-pmuldq.ll
+++ b/llvm/test/CodeGen/X86/combine-pmuldq.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefix=AVX --check-prefix=AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefix=AVX --check-prefix=AVX512DQVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512DQVL
 
 define <2 x i64> @combine_shuffle_sext_pmuldq(<4 x i32> %a0, <4 x i32> %a1) {
 ; SSE-LABEL: combine_shuffle_sext_pmuldq:
@@ -66,6 +66,15 @@ define <4 x i64> @combine_shuffle_zero_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1)
 ; SSE-NEXT:    pmuludq %xmm3, %xmm1
 ; SSE-NEXT:    retq
 ;
+; AVX1-LABEL: combine_shuffle_zero_pmuludq_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
 ; AVX2-LABEL: combine_shuffle_zero_pmuludq_256:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
@@ -102,6 +111,22 @@ define <8 x i64> @combine_zext_pmuludq_256(<8 x i32> %a) {
 ; SSE-NEXT:    pmuludq %xmm4, %xmm3
 ; SSE-NEXT:    retq
 ;
+; AVX1-LABEL: combine_zext_pmuludq_256:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [715827883,715827883]
+; AVX1-NEXT:    vpmuludq %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT:    vpmuludq %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpmuludq %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
 ; AVX2-LABEL: combine_zext_pmuludq_256:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
@@ -202,6 +227,32 @@ define i32 @PR43159(<4 x i32>* %a0) {
 ; SSE-NEXT:    pextrd $3, %xmm0, %ecx
 ; SSE-NEXT:    jmp foo # TAILCALL
 ;
+; AVX1-LABEL: PR43159:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [1645975491,344322273,2164392969,1916962805]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsrld $1, %xmm0, %xmm3
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm0[0,1,2,3],xmm3[4,5],xmm0[6,7]
+; AVX1-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3],xmm2[4,5],xmm0[6,7]
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrld $7, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrld $6, %xmm0, %xmm0
+; AVX1-NEXT:    vmovd %xmm0, %edi
+; AVX1-NEXT:    vpextrd $1, %xmm1, %esi
+; AVX1-NEXT:    vpextrd $2, %xmm0, %edx
+; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
+; AVX1-NEXT:    jmp foo # TAILCALL
+;
 ; AVX2-LABEL: PR43159:
 ; AVX2:       # %bb.0: # %entry
 ; AVX2-NEXT:    vmovdqa (%rdi), %xmm0
@@ -287,3 +338,323 @@ entry:
   ret i32 %call
 }
 declare dso_local i32 @foo(i32, i32, i32, i32)
+
+define <8 x i32> @PR49658_zext(i32* %ptr, i32 %mul) {
+; SSE-LABEL: PR49658_zext:
+; SSE:       # %bb.0: # %start
+; SSE-NEXT:    movl %esi, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]
+; SSE-NEXT:    pxor %xmm0, %xmm0
+; SSE-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:  .LBB7_1: # %loop
+; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
+; SSE-NEXT:    pmovzxdq {{.*#+}} xmm3 = mem[0],zero,mem[1],zero
+; SSE-NEXT:    pmovzxdq {{.*#+}} xmm4 = mem[0],zero,mem[1],zero
+; SSE-NEXT:    pmovzxdq {{.*#+}} xmm5 = mem[0],zero,mem[1],zero
+; SSE-NEXT:    pmovzxdq {{.*#+}} xmm6 = mem[0],zero,mem[1],zero
+; SSE-NEXT:    pmuludq %xmm2, %xmm6
+; SSE-NEXT:    pmuludq %xmm2, %xmm5
+; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,3],xmm6[1,3]
+; SSE-NEXT:    paddd %xmm5, %xmm0
+; SSE-NEXT:    pmuludq %xmm2, %xmm4
+; SSE-NEXT:    pmuludq %xmm2, %xmm3
+; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,3],xmm3[1,3]
+; SSE-NEXT:    paddd %xmm4, %xmm1
+; SSE-NEXT:    subq $-128, %rax
+; SSE-NEXT:    jne .LBB7_1
+; SSE-NEXT:  # %bb.2: # %end
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: PR49658_zext:
+; AVX1:       # %bb.0: # %start
+; AVX1-NEXT:    movl %esi, %eax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:  .LBB7_1: # %loop
+; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = mem[0],zero,mem[1],zero
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = mem[0],zero,mem[1],zero
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm5 = mem[0],zero,mem[1],zero
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm6 = mem[0],zero,mem[1],zero
+; AVX1-NEXT:    vpmuludq %xmm6, %xmm2, %xmm6
+; AVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm5
+; AVX1-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[1,3],xmm6[1,3]
+; AVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,3],xmm4[1,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    subq $-128, %rax
+; AVX1-NEXT:    jne .LBB7_1
+; AVX1-NEXT:  # %bb.2: # %end
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: PR49658_zext:
+; AVX2:       # %bb.0: # %start
+; AVX2-NEXT:    movl %esi, %eax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm1
+; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:  .LBB7_1: # %loop
+; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpmuludq %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm3[2,3],ymm2[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[1,3],ymm4[1,3],ymm2[5,7],ymm4[5,7]
+; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    subq $-128, %rax
+; AVX2-NEXT:    jne .LBB7_1
+; AVX2-NEXT:  # %bb.2: # %end
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: PR49658_zext:
+; AVX512VL:       # %bb.0: # %start
+; AVX512VL-NEXT:    movl %esi, %eax
+; AVX512VL-NEXT:    vpbroadcastq %rax, %zmm1
+; AVX512VL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; AVX512VL-NEXT:    .p2align 4, 0x90
+; AVX512VL-NEXT:  .LBB7_1: # %loop
+; AVX512VL-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX512VL-NEXT:    vpmuludq %zmm2, %zmm1, %zmm2
+; AVX512VL-NEXT:    vpsrlq $32, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpmovqd %zmm2, %ymm2
+; AVX512VL-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    subq $-128, %rax
+; AVX512VL-NEXT:    jne .LBB7_1
+; AVX512VL-NEXT:  # %bb.2: # %end
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQVL-LABEL: PR49658_zext:
+; AVX512DQVL:       # %bb.0: # %start
+; AVX512DQVL-NEXT:    movl %esi, %eax
+; AVX512DQVL-NEXT:    vpbroadcastq %rax, %zmm1
+; AVX512DQVL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; AVX512DQVL-NEXT:    .p2align 4, 0x90
+; AVX512DQVL-NEXT:  .LBB7_1: # %loop
+; AVX512DQVL-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX512DQVL-NEXT:    vpmovzxdq {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX512DQVL-NEXT:    vpmuludq %zmm2, %zmm1, %zmm2
+; AVX512DQVL-NEXT:    vpsrlq $32, %zmm2, %zmm2
+; AVX512DQVL-NEXT:    vpmovqd %zmm2, %ymm2
+; AVX512DQVL-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    subq $-128, %rax
+; AVX512DQVL-NEXT:    jne .LBB7_1
+; AVX512DQVL-NEXT:  # %bb.2: # %end
+; AVX512DQVL-NEXT:    retq
+start:
+  %t1 = zext i32 %mul to i64
+  %t2 = insertelement <8 x i64> undef, i64 %t1, i32 0
+  %mulvec = shufflevector <8 x i64> %t2, <8 x i64> undef, <8 x i32> zeroinitializer
+  br label %loop
+loop:
+  %loopcnt = phi i64 [ 0, %start ], [ %nextcnt, %loop ]
+  %sum = phi <8 x i32> [ zeroinitializer, %start ], [ %nextsum, %loop ]
+  %ptroff = getelementptr inbounds i32, i32* %ptr, i64 %loopcnt
+  %vptroff = bitcast i32* %ptroff to <8 x i32>*
+  %v = load <8 x i32>, <8 x i32>* %vptroff, align 4
+  %v64 = zext <8 x i32> %v to <8 x i64>
+  %vmul = mul nuw <8 x i64> %mulvec, %v64
+  %vmulhi = lshr <8 x i64> %vmul, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
+  %vtrunc = trunc <8 x i64> %vmulhi to <8 x i32>
+  %nextsum = add <8 x i32> %vtrunc, %sum
+  %nextcnt = add i64 %loopcnt, 32
+  %isdone = icmp eq i64 %nextcnt, 524288
+  br i1 %isdone, label %end, label %loop
+end:
+  ret <8 x i32> %nextsum
+}
+
+define <8 x i32> @PR49658_sext(i32* %ptr, i32 %mul) {
+; SSE-LABEL: PR49658_sext:
+; SSE:       # %bb.0: # %start
+; SSE-NEXT:    movslq %esi, %rax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[0,1,0,1]
+; SSE-NEXT:    pxor %xmm0, %xmm0
+; SSE-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; SSE-NEXT:    movdqa %xmm9, %xmm8
+; SSE-NEXT:    psrlq $32, %xmm8
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:  .LBB8_1: # %loop
+; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
+; SSE-NEXT:    pmovsxdq 2097176(%rdi,%rax), %xmm5
+; SSE-NEXT:    pmovsxdq 2097168(%rdi,%rax), %xmm4
+; SSE-NEXT:    pmovsxdq 2097152(%rdi,%rax), %xmm6
+; SSE-NEXT:    pmovsxdq 2097160(%rdi,%rax), %xmm7
+; SSE-NEXT:    movdqa %xmm8, %xmm3
+; SSE-NEXT:    pmuludq %xmm7, %xmm3
+; SSE-NEXT:    movdqa %xmm9, %xmm2
+; SSE-NEXT:    pmuludq %xmm7, %xmm2
+; SSE-NEXT:    psrlq $32, %xmm7
+; SSE-NEXT:    pmuludq %xmm9, %xmm7
+; SSE-NEXT:    paddq %xmm3, %xmm7
+; SSE-NEXT:    psllq $32, %xmm7
+; SSE-NEXT:    paddq %xmm2, %xmm7
+; SSE-NEXT:    movdqa %xmm8, %xmm2
+; SSE-NEXT:    pmuludq %xmm6, %xmm2
+; SSE-NEXT:    movdqa %xmm9, %xmm3
+; SSE-NEXT:    pmuludq %xmm6, %xmm3
+; SSE-NEXT:    psrlq $32, %xmm6
+; SSE-NEXT:    pmuludq %xmm9, %xmm6
+; SSE-NEXT:    paddq %xmm2, %xmm6
+; SSE-NEXT:    psllq $32, %xmm6
+; SSE-NEXT:    paddq %xmm3, %xmm6
+; SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[1,3],xmm7[1,3]
+; SSE-NEXT:    paddd %xmm6, %xmm0
+; SSE-NEXT:    movdqa %xmm4, %xmm2
+; SSE-NEXT:    psrlq $32, %xmm2
+; SSE-NEXT:    pmuludq %xmm9, %xmm2
+; SSE-NEXT:    movdqa %xmm8, %xmm3
+; SSE-NEXT:    pmuludq %xmm4, %xmm3
+; SSE-NEXT:    paddq %xmm2, %xmm3
+; SSE-NEXT:    psllq $32, %xmm3
+; SSE-NEXT:    pmuludq %xmm9, %xmm4
+; SSE-NEXT:    paddq %xmm3, %xmm4
+; SSE-NEXT:    movdqa %xmm5, %xmm2
+; SSE-NEXT:    psrlq $32, %xmm2
+; SSE-NEXT:    pmuludq %xmm9, %xmm2
+; SSE-NEXT:    movdqa %xmm8, %xmm3
+; SSE-NEXT:    pmuludq %xmm5, %xmm3
+; SSE-NEXT:    paddq %xmm2, %xmm3
+; SSE-NEXT:    psllq $32, %xmm3
+; SSE-NEXT:    pmuludq %xmm9, %xmm5
+; SSE-NEXT:    paddq %xmm3, %xmm5
+; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,3],xmm5[1,3]
+; SSE-NEXT:    paddd %xmm4, %xmm1
+; SSE-NEXT:    subq $-128, %rax
+; SSE-NEXT:    jne .LBB8_1
+; SSE-NEXT:  # %bb.2: # %end
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: PR49658_sext:
+; AVX1:       # %bb.0: # %start
+; AVX1-NEXT:    movslq %esi, %rax
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:  .LBB8_1: # %loop
+; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX1-NEXT:    vpmovsxdq 2097152(%rdi,%rax), %xmm3
+; AVX1-NEXT:    vpmovsxdq 2097160(%rdi,%rax), %xmm4
+; AVX1-NEXT:    vpmovsxdq 2097168(%rdi,%rax), %xmm5
+; AVX1-NEXT:    vpmovsxdq 2097176(%rdi,%rax), %xmm6
+; AVX1-NEXT:    vpmuldq %xmm6, %xmm2, %xmm6
+; AVX1-NEXT:    vpmuldq %xmm5, %xmm1, %xmm5
+; AVX1-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[1,3],xmm6[1,3]
+; AVX1-NEXT:    vpmuldq %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpmuldq %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,3],xmm4[1,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    subq $-128, %rax
+; AVX1-NEXT:    jne .LBB8_1
+; AVX1-NEXT:  # %bb.2: # %end
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: PR49658_sext:
+; AVX2:       # %bb.0: # %start
+; AVX2-NEXT:    movslq %esi, %rax
+; AVX2-NEXT:    vmovq %rax, %xmm0
+; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm1
+; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:  .LBB8_1: # %loop
+; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT:    vpmovsxdq 2097168(%rdi,%rax), %ymm2
+; AVX2-NEXT:    vpmovsxdq 2097152(%rdi,%rax), %ymm3
+; AVX2-NEXT:    vpmuldq %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpmuldq %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm3[2,3],ymm2[2,3]
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2
+; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[1,3],ymm4[1,3],ymm2[5,7],ymm4[5,7]
+; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    subq $-128, %rax
+; AVX2-NEXT:    jne .LBB8_1
+; AVX2-NEXT:  # %bb.2: # %end
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: PR49658_sext:
+; AVX512VL:       # %bb.0: # %start
+; AVX512VL-NEXT:    movslq %esi, %rax
+; AVX512VL-NEXT:    vpbroadcastq %rax, %zmm1
+; AVX512VL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; AVX512VL-NEXT:    .p2align 4, 0x90
+; AVX512VL-NEXT:  .LBB8_1: # %loop
+; AVX512VL-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX512VL-NEXT:    vpmuldq %zmm2, %zmm1, %zmm2
+; AVX512VL-NEXT:    vpsrlq $32, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpmovqd %zmm2, %ymm2
+; AVX512VL-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    subq $-128, %rax
+; AVX512VL-NEXT:    jne .LBB8_1
+; AVX512VL-NEXT:  # %bb.2: # %end
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQVL-LABEL: PR49658_sext:
+; AVX512DQVL:       # %bb.0: # %start
+; AVX512DQVL-NEXT:    movslq %esi, %rax
+; AVX512DQVL-NEXT:    vpbroadcastq %rax, %zmm1
+; AVX512DQVL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
+; AVX512DQVL-NEXT:    .p2align 4, 0x90
+; AVX512DQVL-NEXT:  .LBB8_1: # %loop
+; AVX512DQVL-NEXT:    # =>This Inner Loop Header: Depth=1
+; AVX512DQVL-NEXT:    vpmovzxdq {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX512DQVL-NEXT:    vpmuldq %zmm2, %zmm1, %zmm2
+; AVX512DQVL-NEXT:    vpsrlq $32, %zmm2, %zmm2
+; AVX512DQVL-NEXT:    vpmovqd %zmm2, %ymm2
+; AVX512DQVL-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    subq $-128, %rax
+; AVX512DQVL-NEXT:    jne .LBB8_1
+; AVX512DQVL-NEXT:  # %bb.2: # %end
+; AVX512DQVL-NEXT:    retq
+start:
+	%t1 = sext i32 %mul to i64
+	%t2 = insertelement <8 x i64> undef, i64 %t1, i32 0
+	%mulvec = shufflevector <8 x i64> %t2, <8 x i64> undef, <8 x i32> zeroinitializer
+	br label %loop
+loop:
+	%loopcnt = phi i64 [ 0, %start ], [ %nextcnt, %loop ]
+	%sum = phi <8 x i32> [ zeroinitializer, %start ], [ %nextsum, %loop ]
+	%ptroff = getelementptr inbounds i32, i32* %ptr, i64 %loopcnt
+	%vptroff = bitcast i32* %ptroff to <8 x i32>*
+	%v = load <8 x i32>, <8 x i32>* %vptroff, align 4
+	%v64 = sext <8 x i32> %v to <8 x i64>
+	%vmul = mul <8 x i64> %mulvec, %v64
+	%vmulhi = ashr <8 x i64> %vmul, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
+	%vtrunc = trunc <8 x i64> %vmulhi to <8 x i32>
+	%nextsum = add <8 x i32> %vtrunc, %sum
+	%nextcnt = add i64 %loopcnt, 32
+	%isdone = icmp eq i64 %nextcnt, 524288
+	br i1 %isdone, label %end, label %loop
+end:
+	ret <8 x i32> %nextsum
+}
diff --git a/llvm/test/CodeGen/X86/combine-sdiv.ll b/llvm/test/CodeGen/X86/combine-sdiv.ll
index 877dcbc6e4d294b0dd057404fbe8195d3aef2b36..b0bcfd9823979935c4cd3da4477c99a5565c2744 100644
--- a/llvm/test/CodeGen/X86/combine-sdiv.ll
+++ b/llvm/test/CodeGen/X86/combine-sdiv.ll
@@ -2936,16 +2936,16 @@ define <8 x i16> @combine_vec_sdiv_nonuniform7(<8 x i16> %x) {
 define <16 x i8> @pr38658(<16 x i8> %x) {
 ; SSE2-LABEL: pr38658:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
+; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSE2-NEXT:    pmulhw {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    psrlw $8, %xmm3
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    packuswb %xmm2, %xmm1
+; SSE2-NEXT:    packuswb %xmm3, %xmm1
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; SSE2-NEXT:    psraw $8, %xmm2
@@ -2960,11 +2960,11 @@ define <16 x i8> @pr38658(<16 x i8> %x) {
 ;
 ; SSE41-LABEL: pr38658:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE41-NEXT:    pmulhw {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    psrlw $8, %xmm2
 ; SSE41-NEXT:    packuswb %xmm2, %xmm1
 ; SSE41-NEXT:    paddb %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
@@ -2985,12 +2985,11 @@ define <16 x i8> @pr38658(<16 x i8> %x) {
 ;
 ; AVX1-LABEL: pr38658:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpsraw $8, %xmm1, %xmm1
@@ -3059,11 +3058,10 @@ define <16 x i8> @pr38658(<16 x i8> %x) {
 ;
 ; XOP-LABEL: pr38658:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; XOP-NEXT:    vpmovsxbw %xmm1, %xmm1
-; XOP-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
-; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOP-NEXT:    vpperm {{.*#+}} xmm1 = xmm2[1,3,5,7,9,11,13,15],xmm1[1,3,5,7,9,11,13,15]
+; XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; XOP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; XOP-NEXT:    vpmulhw {{.*}}(%rip), %xmm2, %xmm2
+; XOP-NEXT:    vpperm {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15],xmm2[1,3,5,7,9,11,13,15]
 ; XOP-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
 ; XOP-NEXT:    vpshab {{.*}}(%rip), %xmm0, %xmm1
 ; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/combine-udiv.ll b/llvm/test/CodeGen/X86/combine-udiv.ll
index c44342d00357a715014de3fe91f21350b44c3402..c6e741540999d2b07aae16ebdae1870eb1555446 100644
--- a/llvm/test/CodeGen/X86/combine-udiv.ll
+++ b/llvm/test/CodeGen/X86/combine-udiv.ll
@@ -693,23 +693,20 @@ define <16 x i8> @combine_vec_udiv_nonuniform4(<16 x i8> %x) {
 define <8 x i16> @pr38477(<8 x i16> %a0) {
 ; SSE2-LABEL: pr38477:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,4957,57457,4103,16385,35545,2048,2115]
-; SSE2-NEXT:    pmulhuw %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psubw %xmm2, %xmm1
-; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    paddw %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535,65535,65535,0,65535]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm1, %xmm3
-; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,4957,57457,4103,16385,35545,2048,2115]
+; SSE2-NEXT:    pmulhuw %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,65535,65535,65535,65535,65535,65535]
-; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psubw %xmm1, %xmm0
+; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,65535,0,65535]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    pmulhuw {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: pr38477:
diff --git a/llvm/test/CodeGen/X86/extractelement-fp.ll b/llvm/test/CodeGen/X86/extractelement-fp.ll
index 137b98db28a3bcce3976c1673b04d92983f903c8..89b3f4f7c0ec8f5adcda74586a106dd288bd8389 100644
--- a/llvm/test/CodeGen/X86/extractelement-fp.ll
+++ b/llvm/test/CodeGen/X86/extractelement-fp.ll
@@ -328,7 +328,7 @@ define <3 x double> @extvselectsetcc_crash(<2 x double> %x) {
 ;
 ; X86-LABEL: extvselectsetcc_crash:
 ; X86:       # %bb.0:
-; X86-NEXT:    vcmpeqpd {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-NEXT:    vcmpeqpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1
 ; X86-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
 ; X86-NEXT:    vandpd %xmm2, %xmm1, %xmm1
 ; X86-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
@@ -556,7 +556,7 @@ define double @fabs_v4f64(<4 x double> %x) nounwind {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    vmovlps %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
@@ -830,8 +830,8 @@ define double @copysign_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    vandps {{\.LCPI.*}}, %xmm1, %xmm1
-; X86-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
+; X86-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vmovlps %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
@@ -1111,7 +1111,7 @@ define double @round_v4f64(<4 x double> %x) nounwind {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    vandpd {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-NEXT:    vandpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1
 ; X86-NEXT:    vmovddup {{.*#+}} xmm2 = [4.9999999999999994E-1,4.9999999999999994E-1]
 ; X86-NEXT:    # xmm2 = mem[0,0]
 ; X86-NEXT:    vorpd %xmm1, %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/fast-isel-fneg.ll b/llvm/test/CodeGen/X86/fast-isel-fneg.ll
index d575a277cf0f91d1a39879ac0957ee9324437b68..28aabc8fadf5eb042735b6af6a34d6d175e7df39 100644
--- a/llvm/test/CodeGen/X86/fast-isel-fneg.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-fneg.ll
@@ -18,7 +18,7 @@ define double @fneg_f64(double %x) nounwind {
 ; SSE2-NEXT:    andl $-8, %esp
 ; SSE2-NEXT:    subl $8, %esp
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; SSE2-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE2-NEXT:    movlps %xmm0, (%esp)
 ; SSE2-NEXT:    fldl (%esp)
 ; SSE2-NEXT:    movl %ebp, %esp
@@ -40,7 +40,7 @@ define float @fneg_f32(float %x) nounwind {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pushl %eax
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; SSE2-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE2-NEXT:    movss %xmm0, (%esp)
 ; SSE2-NEXT:    flds (%esp)
 ; SSE2-NEXT:    popl %eax
@@ -65,7 +65,7 @@ define void @fneg_f64_mem(double* %x, double* %y) nounwind {
 ; SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; SSE2-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE2-NEXT:    movsd %xmm0, (%eax)
 ; SSE2-NEXT:    retl
   %a = load double, double* %x
diff --git a/llvm/test/CodeGen/X86/fildll.ll b/llvm/test/CodeGen/X86/fildll.ll
index a91974fbdfa3161757aa9a051895c892de12aaf2..8499397fed2d0f8a758b683d744b109aa09d18b2 100644
--- a/llvm/test/CodeGen/X86/fildll.ll
+++ b/llvm/test/CodeGen/X86/fildll.ll
@@ -36,7 +36,7 @@ define fastcc double @uint64_to_fp(i64 %X) {
 ; CHECK-NEXT:    movl %ecx, (%esp)
 ; CHECK-NEXT:    shrl $31, %edx
 ; CHECK-NEXT:    fildll (%esp)
-; CHECK-NEXT:    fadds {{\.LCPI.*}}(,%edx,4)
+; CHECK-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%edx,4)
 ; CHECK-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl %ebp, %esp
diff --git a/llvm/test/CodeGen/X86/fma-scalar-combine.ll b/llvm/test/CodeGen/X86/fma-scalar-combine.ll
index 7d9169ca2552ba5a00ee12e5f769313ff76dcd28..02e96ea73cebcf28845d1acd49a04142de81c465 100644
--- a/llvm/test/CodeGen/X86/fma-scalar-combine.ll
+++ b/llvm/test/CodeGen/X86/fma-scalar-combine.ll
@@ -548,9 +548,9 @@ define float @fma_const_fmul(float %x) {
 ; CHECK-LABEL: fma_const_fmul:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x59,0x0d,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    vfmadd132ss {{.*}}(%rip), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x99,0x05,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    # fixup A - offset: 5, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    # xmm0 = (xmm0 * mem) + xmm1
 ; CHECK-NEXT:    retq # encoding: [0xc3]
   %mul1 = fmul contract float %x, 10.0
diff --git a/llvm/test/CodeGen/X86/fmf-flags.ll b/llvm/test/CodeGen/X86/fmf-flags.ll
index 835ec72ff5919fce14b96e6aa752bcfac64ca492..a4285ebc888be5c7c8544b9b05cf2e347af3f6d9 100644
--- a/llvm/test/CodeGen/X86/fmf-flags.ll
+++ b/llvm/test/CodeGen/X86/fmf-flags.ll
@@ -38,7 +38,7 @@ define dso_local float @fast_fmuladd_opts(float %a , float %b , float %c) {
 ; X86-LABEL: fast_fmuladd_opts:
 ; X86:       # %bb.0:
 ; X86-NEXT:    flds {{[0-9]+}}(%esp)
-; X86-NEXT:    fmuls {{\.LCPI.*}}
+; X86-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-NEXT:    retl
   %res = call fast float @llvm.fmuladd.f32(float %a, float 2.0, float %a)
   ret float %res
@@ -61,9 +61,9 @@ define dso_local double @not_so_fast_mul_add(double %x) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    fldl {{[0-9]+}}(%esp)
 ; X86-NEXT:    fld %st(0)
-; X86-NEXT:    fmull {{\.LCPI.*}}
+; X86-NEXT:    fmull {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-NEXT:    fxch %st(1)
-; X86-NEXT:    fmull {{\.LCPI.*}}
+; X86-NEXT:    fmull {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-NEXT:    fxch %st(1)
 ; X86-NEXT:    fstpl mul1
 ; X86-NEXT:    retl
@@ -112,9 +112,9 @@ define dso_local float @div_arcp_by_const(half %x) {
 ; X64-NEXT:    pushq %rax
 ; X64-NEXT:    .cfi_def_cfa_offset 16
 ; X64-NEXT:    movzwl %di, %edi
-; X64-NEXT:    callq __gnu_h2f_ieee
+; X64-NEXT:    callq __gnu_h2f_ieee@PLT
 ; X64-NEXT:    mulss {{.*}}(%rip), %xmm0
-; X64-NEXT:    callq __gnu_f2h_ieee
+; X64-NEXT:    callq __gnu_f2h_ieee@PLT
 ; X64-NEXT:    movzwl %ax, %edi
 ; X64-NEXT:    popq %rax
 ; X64-NEXT:    .cfi_def_cfa_offset 8
@@ -127,7 +127,7 @@ define dso_local float @div_arcp_by_const(half %x) {
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl %eax, (%esp)
 ; X86-NEXT:    calll __gnu_h2f_ieee
-; X86-NEXT:    fmuls {{\.LCPI.*}}
+; X86-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-NEXT:    fstps (%esp)
 ; X86-NEXT:    calll __gnu_f2h_ieee
 ; X86-NEXT:    movzwl %ax, %eax
diff --git a/llvm/test/CodeGen/X86/fp-cvt.ll b/llvm/test/CodeGen/X86/fp-cvt.ll
index c8e6a95bcbac2d85438867fb06be939a1aad0229..facb84d6f104c5570d43f08210d426bd35a84a8b 100644
--- a/llvm/test/CodeGen/X86/fp-cvt.ll
+++ b/llvm/test/CodeGen/X86/fp-cvt.ll
@@ -443,7 +443,7 @@ define i64 @fptoui_i64_fp80(x86_fp80 %a0) nounwind {
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $16, %esp
 ; X86-NEXT:    fldt 8(%ebp)
-; X86-NEXT:    flds {{\.LCPI.*}}
+; X86-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-NEXT:    fucom %st(1)
 ; X86-NEXT:    fnstsw %ax
 ; X86-NEXT:    xorl %edx, %edx
@@ -523,7 +523,7 @@ define i64 @fptoui_i64_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X86-NEXT:    subl $16, %esp
 ; X86-NEXT:    movl 8(%ebp), %eax
 ; X86-NEXT:    fldt (%eax)
-; X86-NEXT:    flds {{\.LCPI.*}}
+; X86-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-NEXT:    fucom %st(1)
 ; X86-NEXT:    fnstsw %ax
 ; X86-NEXT:    xorl %edx, %edx
@@ -825,7 +825,7 @@ define x86_fp80 @uitofp_fp80_i64(i64 %a0) nounwind {
 ; X86-NEXT:    movl %eax, (%esp)
 ; X86-NEXT:    shrl $31, %ecx
 ; X86-NEXT:    fildll (%esp)
-; X86-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X86-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
@@ -837,7 +837,7 @@ define x86_fp80 @uitofp_fp80_i64(i64 %a0) nounwind {
 ; X64-NEXT:    testq %rdi, %rdi
 ; X64-NEXT:    sets %al
 ; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fadds {{\.LCPI.*}}(,%rax,4)
+; X64-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%rax,4)
 ; X64-NEXT:    retq
   %1 = uitofp i64 %a0 to x86_fp80
   ret x86_fp80 %1
@@ -857,7 +857,7 @@ define x86_fp80 @uitofp_fp80_i64_ld(i64 *%a0) nounwind {
 ; X86-NEXT:    movl %ecx, (%esp)
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    fildll (%esp)
-; X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; X86-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
@@ -870,7 +870,7 @@ define x86_fp80 @uitofp_fp80_i64_ld(i64 *%a0) nounwind {
 ; X64-NEXT:    testq %rax, %rax
 ; X64-NEXT:    sets %cl
 ; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fadds {{\.LCPI.*}}(,%rcx,4)
+; X64-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%rcx,4)
 ; X64-NEXT:    retq
   %1 = load i64, i64 *%a0
   %2 = uitofp i64 %1 to x86_fp80
@@ -896,7 +896,7 @@ define x86_fp80 @floor_fp80(x86_fp80 %a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq floorl
+; X64-NEXT:    callq floorl@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = call x86_fp80 @llvm.floor.f80(x86_fp80 %a0)
@@ -919,7 +919,7 @@ define x86_fp80 @floor_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt (%rdi)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq floorl
+; X64-NEXT:    callq floorl@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = load x86_fp80, x86_fp80 *%a0
@@ -948,7 +948,7 @@ define x86_fp80 @ceil_fp80(x86_fp80 %a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq ceill
+; X64-NEXT:    callq ceill@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = call x86_fp80 @llvm.ceil.f80(x86_fp80 %a0)
@@ -971,7 +971,7 @@ define x86_fp80 @ceil_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt (%rdi)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq ceill
+; X64-NEXT:    callq ceill@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = load x86_fp80, x86_fp80 *%a0
@@ -1000,7 +1000,7 @@ define x86_fp80 @trunc_fp80(x86_fp80 %a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq truncl
+; X64-NEXT:    callq truncl@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = call x86_fp80 @llvm.trunc.f80(x86_fp80 %a0)
@@ -1023,7 +1023,7 @@ define x86_fp80 @trunc_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt (%rdi)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq truncl
+; X64-NEXT:    callq truncl@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = load x86_fp80, x86_fp80 *%a0
@@ -1052,7 +1052,7 @@ define x86_fp80 @rint_fp80(x86_fp80 %a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq rintl
+; X64-NEXT:    callq rintl@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = call x86_fp80 @llvm.rint.f80(x86_fp80 %a0)
@@ -1075,7 +1075,7 @@ define x86_fp80 @rint_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt (%rdi)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq rintl
+; X64-NEXT:    callq rintl@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = load x86_fp80, x86_fp80 *%a0
@@ -1104,7 +1104,7 @@ define x86_fp80 @roundeven_fp80(x86_fp80 %a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq roundevenl
+; X64-NEXT:    callq roundevenl@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = call x86_fp80 @llvm.roundeven.f80(x86_fp80 %a0)
@@ -1127,7 +1127,7 @@ define x86_fp80 @roundeven_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X64-NEXT:    subq $24, %rsp
 ; X64-NEXT:    fldt (%rdi)
 ; X64-NEXT:    fstpt (%rsp)
-; X64-NEXT:    callq roundevenl
+; X64-NEXT:    callq roundevenl@PLT
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
   %1 = load x86_fp80, x86_fp80 *%a0
diff --git a/llvm/test/CodeGen/X86/fp-intrinsics.ll b/llvm/test/CodeGen/X86/fp-intrinsics.ll
index abe88f1ca233b080048a0718e2a5f6232016773a..e2fdb904dabcbed1f60fbcf7a6a31f0454408d96 100644
--- a/llvm/test/CodeGen/X86/fp-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/fp-intrinsics.ll
@@ -18,7 +18,7 @@ define double @f1() #0 {
 ; X87-LABEL: f1:
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    fld1
-; X87-NEXT:    fdivs {{\.LCPI.*}}
+; X87-NEXT:    fdivs {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    wait
 ; X87-NEXT:    retl
 ;
@@ -27,7 +27,7 @@ define double @f1() #0 {
 ; X86-SSE-NEXT:    subl $12, %esp
 ; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X86-SSE-NEXT:    divsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    divsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movsd %xmm0, (%esp)
 ; X86-SSE-NEXT:    fldl (%esp)
 ; X86-SSE-NEXT:    wait
@@ -209,7 +209,7 @@ define double @f4(i32 %n, double %a) #0 {
 ; X86-SSE-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    jle .LBB3_2
 ; X86-SSE-NEXT:  # %bb.1: # %if.then
-; X86-SSE-NEXT:    addsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    addsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:  .LBB3_2: # %if.end
 ; X86-SSE-NEXT:    movsd %xmm0, (%esp)
 ; X86-SSE-NEXT:    fldl (%esp)
@@ -255,7 +255,7 @@ if.end:
 define double @f5() #0 {
 ; X87-LABEL: f5:
 ; X87:       # %bb.0: # %entry
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fsqrt
 ; X87-NEXT:    wait
 ; X87-NEXT:    retl
@@ -297,9 +297,9 @@ define double @f6() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $28, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 32
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
-; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll pow
@@ -326,7 +326,7 @@ define double @f6() #0 {
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; SSE-NEXT:    callq pow
+; SSE-NEXT:    callq pow@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -337,7 +337,7 @@ define double @f6() #0 {
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    callq pow
+; AVX-NEXT:    callq pow@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -355,7 +355,7 @@ define double @f7() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    movl $3, {{[0-9]+}}(%esp)
@@ -382,7 +382,7 @@ define double @f7() #0 {
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE-NEXT:    movl $3, %edi
-; SSE-NEXT:    callq __powidf2
+; SSE-NEXT:    callq __powidf2@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -393,7 +393,7 @@ define double @f7() #0 {
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    movl $3, %edi
-; AVX-NEXT:    callq __powidf2
+; AVX-NEXT:    callq __powidf2@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -411,7 +411,7 @@ define double @f8() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll sin
@@ -435,7 +435,7 @@ define double @f8() #0 {
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    callq sin
+; SSE-NEXT:    callq sin@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -445,7 +445,7 @@ define double @f8() #0 {
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq sin
+; AVX-NEXT:    callq sin@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -462,7 +462,7 @@ define double @f9() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll cos
@@ -486,7 +486,7 @@ define double @f9() #0 {
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    callq cos
+; SSE-NEXT:    callq cos@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -496,7 +496,7 @@ define double @f9() #0 {
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq cos
+; AVX-NEXT:    callq cos@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -513,7 +513,7 @@ define double @f10() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll exp
@@ -537,7 +537,7 @@ define double @f10() #0 {
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    callq exp
+; SSE-NEXT:    callq exp@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -547,7 +547,7 @@ define double @f10() #0 {
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq exp
+; AVX-NEXT:    callq exp@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -564,7 +564,7 @@ define double @f11() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll exp2
@@ -588,7 +588,7 @@ define double @f11() #0 {
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    callq exp2
+; SSE-NEXT:    callq exp2@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -598,7 +598,7 @@ define double @f11() #0 {
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq exp2
+; AVX-NEXT:    callq exp2@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -615,7 +615,7 @@ define double @f12() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll log
@@ -639,7 +639,7 @@ define double @f12() #0 {
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    callq log
+; SSE-NEXT:    callq log@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -649,7 +649,7 @@ define double @f12() #0 {
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq log
+; AVX-NEXT:    callq log@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -666,7 +666,7 @@ define double @f13() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll log10
@@ -690,7 +690,7 @@ define double @f13() #0 {
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    callq log10
+; SSE-NEXT:    callq log10@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -700,7 +700,7 @@ define double @f13() #0 {
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq log10
+; AVX-NEXT:    callq log10@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -717,7 +717,7 @@ define double @f14() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll log2
@@ -741,7 +741,7 @@ define double @f14() #0 {
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    callq log2
+; SSE-NEXT:    callq log2@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -751,7 +751,7 @@ define double @f14() #0 {
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX-NEXT:    callq log2
+; AVX-NEXT:    callq log2@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -768,7 +768,7 @@ define double @f15() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll rint
@@ -792,7 +792,7 @@ define double @f15() #0 {
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    callq rint
+; SSE-NEXT:    callq rint@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -816,7 +816,7 @@ define double @f16() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $12, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 16
-; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl (%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    calll nearbyint
@@ -840,7 +840,7 @@ define double @f16() #0 {
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    callq nearbyint
+; SSE-NEXT:    callq nearbyint@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -863,7 +863,7 @@ define double @f19() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    subl $28, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 32
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; X87-NEXT:    wait
 ; X87-NEXT:    movl $1072693248, {{[0-9]+}}(%esp) # imm = 0x3FF00000
@@ -892,7 +892,7 @@ define double @f19() #0 {
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; SSE-NEXT:    callq fmod
+; SSE-NEXT:    callq fmod@PLT
 ; SSE-NEXT:    popq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -903,7 +903,7 @@ define double @f19() #0 {
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    callq fmod
+; AVX-NEXT:    callq fmod@PLT
 ; AVX-NEXT:    popq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -1172,14 +1172,14 @@ define i128 @f20s128(double %x) nounwind strictfp {
 ; SSE-LABEL: f20s128:
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
-; SSE-NEXT:    callq __fixdfti
+; SSE-NEXT:    callq __fixdfti@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: f20s128:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
-; AVX-NEXT:    callq __fixdfti
+; AVX-NEXT:    callq __fixdfti@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    retq
 entry:
@@ -1356,7 +1356,7 @@ define i64 @f20u64(double %x) #0 {
 ; X87-NEXT:    subl $20, %esp
 ; X87-NEXT:    .cfi_def_cfa_offset 24
 ; X87-NEXT:    fldl {{[0-9]+}}(%esp)
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    wait
 ; X87-NEXT:    xorl %edx, %edx
 ; X87-NEXT:    fcomi %st(1), %st
@@ -1517,14 +1517,14 @@ define i128 @f20u128(double %x) nounwind strictfp {
 ; SSE-LABEL: f20u128:
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
-; SSE-NEXT:    callq __fixunsdfti
+; SSE-NEXT:    callq __fixunsdfti@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: f20u128:
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
-; AVX-NEXT:    callq __fixunsdfti
+; AVX-NEXT:    callq __fixunsdfti@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    retq
 entry:
@@ -1541,7 +1541,7 @@ define float @f21() #0 {
 ; X87:       # %bb.0: # %entry
 ; X87-NEXT:    pushl %eax
 ; X87-NEXT:    .cfi_def_cfa_offset 8
-; X87-NEXT:    fldl {{\.LCPI.*}}
+; X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fstps (%esp)
 ; X87-NEXT:    flds (%esp)
 ; X87-NEXT:    wait
@@ -1644,7 +1644,7 @@ define i32 @f23(double %x) #0 {
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
-; SSE-NEXT:    callq lrint
+; SSE-NEXT:    callq lrint@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -1653,7 +1653,7 @@ define i32 @f23(double %x) #0 {
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
-; AVX-NEXT:    callq lrint
+; AVX-NEXT:    callq lrint@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -1692,7 +1692,7 @@ define i32 @f24(float %x) #0 {
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
-; SSE-NEXT:    callq lrintf
+; SSE-NEXT:    callq lrintf@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -1701,7 +1701,7 @@ define i32 @f24(float %x) #0 {
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
-; AVX-NEXT:    callq lrintf
+; AVX-NEXT:    callq lrintf@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -1740,7 +1740,7 @@ define i64 @f25(double %x) #0 {
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
-; SSE-NEXT:    callq llrint
+; SSE-NEXT:    callq llrint@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -1749,7 +1749,7 @@ define i64 @f25(double %x) #0 {
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
-; AVX-NEXT:    callq llrint
+; AVX-NEXT:    callq llrint@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -1788,7 +1788,7 @@ define i64 @f26(float %x) #0 {
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
-; SSE-NEXT:    callq llrintf
+; SSE-NEXT:    callq llrintf@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -1797,7 +1797,7 @@ define i64 @f26(float %x) #0 {
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
-; AVX-NEXT:    callq llrintf
+; AVX-NEXT:    callq llrintf@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -1836,7 +1836,7 @@ define i32 @f27(double %x) #0 {
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
-; SSE-NEXT:    callq lround
+; SSE-NEXT:    callq lround@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -1845,7 +1845,7 @@ define i32 @f27(double %x) #0 {
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
-; AVX-NEXT:    callq lround
+; AVX-NEXT:    callq lround@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -1883,7 +1883,7 @@ define i32 @f28(float %x) #0 {
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
-; SSE-NEXT:    callq lroundf
+; SSE-NEXT:    callq lroundf@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -1892,7 +1892,7 @@ define i32 @f28(float %x) #0 {
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
-; AVX-NEXT:    callq lroundf
+; AVX-NEXT:    callq lroundf@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -1930,7 +1930,7 @@ define i64 @f29(double %x) #0 {
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
-; SSE-NEXT:    callq llround
+; SSE-NEXT:    callq llround@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -1939,7 +1939,7 @@ define i64 @f29(double %x) #0 {
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
-; AVX-NEXT:    callq llround
+; AVX-NEXT:    callq llround@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -1977,7 +1977,7 @@ define i64 @f30(float %x) #0 {
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    pushq %rax
 ; SSE-NEXT:    .cfi_def_cfa_offset 16
-; SSE-NEXT:    callq llroundf
+; SSE-NEXT:    callq llroundf@PLT
 ; SSE-NEXT:    popq %rcx
 ; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
@@ -1986,7 +1986,7 @@ define i64 @f30(float %x) #0 {
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    pushq %rax
 ; AVX-NEXT:    .cfi_def_cfa_offset 16
-; AVX-NEXT:    callq llroundf
+; AVX-NEXT:    callq llroundf@PLT
 ; AVX-NEXT:    popq %rcx
 ; AVX-NEXT:    .cfi_def_cfa_offset 8
 ; AVX-NEXT:    retq
@@ -2437,8 +2437,8 @@ define double @uifdi(i32 %x) #0 {
 ; X86-SSE-NEXT:    subl $12, %esp
 ; X86-SSE-NEXT:    .cfi_def_cfa_offset 16
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SSE-NEXT:    orpd {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    subsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    orpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    subsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movsd %xmm0, (%esp)
 ; X86-SSE-NEXT:    fldl (%esp)
 ; X86-SSE-NEXT:    wait
@@ -2480,7 +2480,7 @@ define double @uifdl(i64 %x) #0 {
 ; X87-NEXT:    movl %eax, (%esp)
 ; X87-NEXT:    shrl $31, %ecx
 ; X87-NEXT:    fildll (%esp)
-; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; X87-NEXT:    fldl {{[0-9]+}}(%esp)
 ; X87-NEXT:    wait
@@ -2497,7 +2497,7 @@ define double @uifdl(i64 %x) #0 {
 ; X86-SSE-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    shrl $31, %eax
 ; X86-SSE-NEXT:    fildll {{[0-9]+}}(%esp)
-; X86-SSE-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; X86-SSE-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-SSE-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    wait
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
@@ -2658,8 +2658,8 @@ define float @uiffi(i32 %x) #0 {
 ; X86-SSE-NEXT:    pushl %eax
 ; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SSE-NEXT:    orpd {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    subsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    orpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    subsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0
 ; X86-SSE-NEXT:    movss %xmm0, (%esp)
 ; X86-SSE-NEXT:    flds (%esp)
@@ -2702,7 +2702,7 @@ define float @uiffl(i64 %x) #0 {
 ; X87-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; X87-NEXT:    shrl $31, %ecx
 ; X87-NEXT:    fildll {{[0-9]+}}(%esp)
-; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; X87-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X87-NEXT:    flds {{[0-9]+}}(%esp)
 ; X87-NEXT:    wait
@@ -2719,7 +2719,7 @@ define float @uiffl(i64 %x) #0 {
 ; X86-SSE-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    shrl $31, %eax
 ; X86-SSE-NEXT:    fildll {{[0-9]+}}(%esp)
-; X86-SSE-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; X86-SSE-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-SSE-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    wait
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/fp-stack-set-st1.ll b/llvm/test/CodeGen/X86/fp-stack-set-st1.ll
index 065f84d34b8a02e1b312c6b7fe206b00019b7cb0..1e41c869332b50f2b582db81c4246fb88479e2be 100644
--- a/llvm/test/CodeGen/X86/fp-stack-set-st1.ll
+++ b/llvm/test/CodeGen/X86/fp-stack-set-st1.ll
@@ -4,8 +4,8 @@
 define i32 @main() nounwind {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    fldl {{\.LCPI.*}}
-; CHECK-NEXT:    fldl {{\.LCPI.*}}
+; CHECK-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
+; CHECK-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; CHECK-NEXT:    fxch %st(1)
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    fmul %st(1), %st
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint.ll
index 156ee617e72a81ba406e49a850a5e78dba86eba9..478f2796c5a9837ee06e1b5c67d31a6c07f6fabe 100644
--- a/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint.ll
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-fptoint.ll
@@ -679,7 +679,7 @@ define i64 @fptoui_f32toi64(float %x) #0 {
 ; X87-NEXT:    andl $-8, %esp
 ; X87-NEXT:    subl $16, %esp
 ; X87-NEXT:    flds 8(%ebp)
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fcom %st(1)
 ; X87-NEXT:    wait
 ; X87-NEXT:    fnstsw %ax
@@ -1319,7 +1319,7 @@ define i64 @fptoui_f64toi64(double %x) #0 {
 ; X87-NEXT:    andl $-8, %esp
 ; X87-NEXT:    subl $16, %esp
 ; X87-NEXT:    fldl 8(%ebp)
-; X87-NEXT:    flds {{\.LCPI.*}}
+; X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-NEXT:    fcom %st(1)
 ; X87-NEXT:    wait
 ; X87-NEXT:    fnstsw %ax
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll
index 49e238df4aa2a57b2cb988f2d216e1e444d2c03d..b060f3cc8067486abe4f1c1716adf174599ef2f5 100644
--- a/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-inttofp.ll
@@ -488,8 +488,8 @@ define float @uitofp_i32tof32(i32 %x) #0 {
 ; SSE-X86-NEXT:    pushl %eax
 ; SSE-X86-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-X86-NEXT:    orpd {{\.LCPI.*}}, %xmm0
-; SSE-X86-NEXT:    subsd {{\.LCPI.*}}, %xmm0
+; SSE-X86-NEXT:    orpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; SSE-X86-NEXT:    subsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-X86-NEXT:    cvtsd2ss %xmm0, %xmm0
 ; SSE-X86-NEXT:    movss %xmm0, (%esp)
 ; SSE-X86-NEXT:    flds (%esp)
@@ -509,8 +509,8 @@ define float @uitofp_i32tof32(i32 %x) #0 {
 ; AVX1-X86-NEXT:    pushl %eax
 ; AVX1-X86-NEXT:    .cfi_def_cfa_offset 8
 ; AVX1-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-X86-NEXT:    vorpd {{\.LCPI.*}}, %xmm0, %xmm0
-; AVX1-X86-NEXT:    vsubsd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-X86-NEXT:    vorpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; AVX1-X86-NEXT:    vsubsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX1-X86-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
 ; AVX1-X86-NEXT:    vmovss %xmm0, (%esp)
 ; AVX1-X86-NEXT:    flds (%esp)
@@ -581,7 +581,7 @@ define float @uitofp_i64tof32(i64 %x) #0 {
 ; SSE-X86-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
 ; SSE-X86-NEXT:    shrl $31, %eax
 ; SSE-X86-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE-X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE-X86-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE-X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; SSE-X86-NEXT:    wait
 ; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -623,7 +623,7 @@ define float @uitofp_i64tof32(i64 %x) #0 {
 ; AVX-X86-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
 ; AVX-X86-NEXT:    shrl $31, %eax
 ; AVX-X86-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-X86-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; AVX-X86-NEXT:    wait
 ; AVX-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -671,7 +671,7 @@ define float @uitofp_i64tof32(i64 %x) #0 {
 ; X87-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; X87-NEXT:    shrl $31, %ecx
 ; X87-NEXT:    fildll {{[0-9]+}}(%esp)
-; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; X87-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X87-NEXT:    flds {{[0-9]+}}(%esp)
 ; X87-NEXT:    wait
@@ -1164,8 +1164,8 @@ define double @uitofp_i32tof64(i32 %x) #0 {
 ; SSE-X86-NEXT:    andl $-8, %esp
 ; SSE-X86-NEXT:    subl $8, %esp
 ; SSE-X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE-X86-NEXT:    orpd {{\.LCPI.*}}, %xmm0
-; SSE-X86-NEXT:    subsd {{\.LCPI.*}}, %xmm0
+; SSE-X86-NEXT:    orpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; SSE-X86-NEXT:    subsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-X86-NEXT:    movsd %xmm0, (%esp)
 ; SSE-X86-NEXT:    fldl (%esp)
 ; SSE-X86-NEXT:    wait
@@ -1190,8 +1190,8 @@ define double @uitofp_i32tof64(i32 %x) #0 {
 ; AVX1-X86-NEXT:    andl $-8, %esp
 ; AVX1-X86-NEXT:    subl $8, %esp
 ; AVX1-X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-X86-NEXT:    vorpd {{\.LCPI.*}}, %xmm0, %xmm0
-; AVX1-X86-NEXT:    vsubsd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-X86-NEXT:    vorpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; AVX1-X86-NEXT:    vsubsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX1-X86-NEXT:    vmovsd %xmm0, (%esp)
 ; AVX1-X86-NEXT:    fldl (%esp)
 ; AVX1-X86-NEXT:    wait
@@ -1268,7 +1268,7 @@ define double @uitofp_i64tof64(i64 %x) #0 {
 ; SSE-X86-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
 ; SSE-X86-NEXT:    shrl $31, %eax
 ; SSE-X86-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE-X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE-X86-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE-X86-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; SSE-X86-NEXT:    wait
 ; SSE-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
@@ -1310,7 +1310,7 @@ define double @uitofp_i64tof64(i64 %x) #0 {
 ; AVX-X86-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
 ; AVX-X86-NEXT:    shrl $31, %eax
 ; AVX-X86-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-X86-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-X86-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; AVX-X86-NEXT:    wait
 ; AVX-X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
@@ -1358,7 +1358,7 @@ define double @uitofp_i64tof64(i64 %x) #0 {
 ; X87-NEXT:    movl %eax, (%esp)
 ; X87-NEXT:    shrl $31, %ecx
 ; X87-NEXT:    fildll (%esp)
-; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; X87-NEXT:    fldl {{[0-9]+}}(%esp)
 ; X87-NEXT:    wait
diff --git a/llvm/test/CodeGen/X86/fp128-cast.ll b/llvm/test/CodeGen/X86/fp128-cast.ll
index 6093095d51d0e5950b7f6ce34b9af38a512fcbfb..ef5c7711aaca81fadf152f39041da71ba63185b6 100644
--- a/llvm/test/CodeGen/X86/fp128-cast.ll
+++ b/llvm/test/CodeGen/X86/fp128-cast.ll
@@ -26,7 +26,7 @@ define dso_local void @TestFPExtF32_F128() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-SSE-NEXT:    callq __extendsftf2
+; X64-SSE-NEXT:    callq __extendsftf2@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -57,7 +57,7 @@ define dso_local void @TestFPExtF32_F128() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-AVX-NEXT:    callq __extendsftf2
+; X64-AVX-NEXT:    callq __extendsftf2@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -73,7 +73,7 @@ define dso_local void @TestFPExtF64_F128() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X64-SSE-NEXT:    callq __extenddftf2
+; X64-SSE-NEXT:    callq __extenddftf2@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -104,7 +104,7 @@ define dso_local void @TestFPExtF64_F128() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X64-AVX-NEXT:    callq __extenddftf2
+; X64-AVX-NEXT:    callq __extenddftf2@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -121,7 +121,7 @@ define dso_local void @TestFPExtF80_F128() nounwind {
 ; X64-SSE-NEXT:    subq $24, %rsp
 ; X64-SSE-NEXT:    fldt {{.*}}(%rip)
 ; X64-SSE-NEXT:    fstpt (%rsp)
-; X64-SSE-NEXT:    callq __extendxftf2
+; X64-SSE-NEXT:    callq __extendxftf2@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    addq $24, %rsp
 ; X64-SSE-NEXT:    retq
@@ -153,7 +153,7 @@ define dso_local void @TestFPExtF80_F128() nounwind {
 ; X64-AVX-NEXT:    subq $24, %rsp
 ; X64-AVX-NEXT:    fldt {{.*}}(%rip)
 ; X64-AVX-NEXT:    fstpt (%rsp)
-; X64-AVX-NEXT:    callq __extendxftf2
+; X64-AVX-NEXT:    callq __extendxftf2@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    addq $24, %rsp
 ; X64-AVX-NEXT:    retq
@@ -169,7 +169,7 @@ define dso_local void @TestFPToSIF128_I16() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __fixtfsi
+; X64-SSE-NEXT:    callq __fixtfsi@PLT
 ; X64-SSE-NEXT:    movw %ax, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -191,7 +191,7 @@ define dso_local void @TestFPToSIF128_I16() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __fixtfsi
+; X64-AVX-NEXT:    callq __fixtfsi@PLT
 ; X64-AVX-NEXT:    movw %ax, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -207,7 +207,7 @@ define dso_local void @TestFPToUIF128_I16() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __fixtfsi
+; X64-SSE-NEXT:    callq __fixtfsi@PLT
 ; X64-SSE-NEXT:    movw %ax, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -229,7 +229,7 @@ define dso_local void @TestFPToUIF128_I16() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __fixtfsi
+; X64-AVX-NEXT:    callq __fixtfsi@PLT
 ; X64-AVX-NEXT:    movw %ax, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -245,7 +245,7 @@ define dso_local void @TestFPToSIF128_I32() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __fixtfsi
+; X64-SSE-NEXT:    callq __fixtfsi@PLT
 ; X64-SSE-NEXT:    movl %eax, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -267,7 +267,7 @@ define dso_local void @TestFPToSIF128_I32() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __fixtfsi
+; X64-AVX-NEXT:    callq __fixtfsi@PLT
 ; X64-AVX-NEXT:    movl %eax, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -283,7 +283,7 @@ define dso_local void @TestFPToUIF128_U32() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __fixunstfsi
+; X64-SSE-NEXT:    callq __fixunstfsi@PLT
 ; X64-SSE-NEXT:    movl %eax, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -305,7 +305,7 @@ define dso_local void @TestFPToUIF128_U32() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __fixunstfsi
+; X64-AVX-NEXT:    callq __fixunstfsi@PLT
 ; X64-AVX-NEXT:    movl %eax, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -321,7 +321,7 @@ define dso_local void @TestFPToSIF128_I64() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __fixtfsi
+; X64-SSE-NEXT:    callq __fixtfsi@PLT
 ; X64-SSE-NEXT:    cltq
 ; X64-SSE-NEXT:    movq %rax, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
@@ -346,7 +346,7 @@ define dso_local void @TestFPToSIF128_I64() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __fixtfsi
+; X64-AVX-NEXT:    callq __fixtfsi@PLT
 ; X64-AVX-NEXT:    cltq
 ; X64-AVX-NEXT:    movq %rax, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
@@ -364,7 +364,7 @@ define dso_local void @TestFPToUIF128_U64() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __fixunstfsi
+; X64-SSE-NEXT:    callq __fixunstfsi@PLT
 ; X64-SSE-NEXT:    movl %eax, %eax
 ; X64-SSE-NEXT:    movq %rax, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
@@ -388,7 +388,7 @@ define dso_local void @TestFPToUIF128_U64() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __fixunstfsi
+; X64-AVX-NEXT:    callq __fixunstfsi@PLT
 ; X64-AVX-NEXT:    movl %eax, %eax
 ; X64-AVX-NEXT:    movq %rax, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
@@ -406,7 +406,7 @@ define dso_local void @TestFPToSIF128_I128() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __fixtfti
+; X64-SSE-NEXT:    callq __fixtfti@PLT
 ; X64-SSE-NEXT:    movq %rdx, vi128+{{.*}}(%rip)
 ; X64-SSE-NEXT:    movq %rax, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
@@ -440,7 +440,7 @@ define dso_local void @TestFPToSIF128_I128() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __fixtfti
+; X64-AVX-NEXT:    callq __fixtfti@PLT
 ; X64-AVX-NEXT:    movq %rdx, vi128+{{.*}}(%rip)
 ; X64-AVX-NEXT:    movq %rax, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
@@ -457,7 +457,7 @@ define dso_local void @TestFPToUIF128_U128() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __fixunstfti
+; X64-SSE-NEXT:    callq __fixunstfti@PLT
 ; X64-SSE-NEXT:    movq %rdx, vu128+{{.*}}(%rip)
 ; X64-SSE-NEXT:    movq %rax, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
@@ -491,7 +491,7 @@ define dso_local void @TestFPToUIF128_U128() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __fixunstfti
+; X64-AVX-NEXT:    callq __fixunstfti@PLT
 ; X64-AVX-NEXT:    movq %rdx, vu128+{{.*}}(%rip)
 ; X64-AVX-NEXT:    movq %rax, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
@@ -508,7 +508,7 @@ define dso_local void @TestFPTruncF128_F32() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __trunctfsf2
+; X64-SSE-NEXT:    callq __trunctfsf2@PLT
 ; X64-SSE-NEXT:    movss %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -530,7 +530,7 @@ define dso_local void @TestFPTruncF128_F32() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __trunctfsf2
+; X64-AVX-NEXT:    callq __trunctfsf2@PLT
 ; X64-AVX-NEXT:    vmovss %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -546,7 +546,7 @@ define dso_local void @TestFPTruncF128_F64() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __trunctfdf2
+; X64-SSE-NEXT:    callq __trunctfdf2@PLT
 ; X64-SSE-NEXT:    movsd %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -568,7 +568,7 @@ define dso_local void @TestFPTruncF128_F64() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __trunctfdf2
+; X64-AVX-NEXT:    callq __trunctfdf2@PLT
 ; X64-AVX-NEXT:    vmovsd %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -584,7 +584,7 @@ define dso_local void @TestFPTruncF128_F80() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __trunctfxf2
+; X64-SSE-NEXT:    callq __trunctfxf2@PLT
 ; X64-SSE-NEXT:    fstpt {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -606,7 +606,7 @@ define dso_local void @TestFPTruncF128_F80() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm0
-; X64-AVX-NEXT:    callq __trunctfxf2
+; X64-AVX-NEXT:    callq __trunctfxf2@PLT
 ; X64-AVX-NEXT:    fstpt {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -622,7 +622,7 @@ define dso_local void @TestSIToFPI16_F128() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movswl {{.*}}(%rip), %edi
-; X64-SSE-NEXT:    callq __floatsitf
+; X64-SSE-NEXT:    callq __floatsitf@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -654,7 +654,7 @@ define dso_local void @TestSIToFPI16_F128() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    movswl {{.*}}(%rip), %edi
-; X64-AVX-NEXT:    callq __floatsitf
+; X64-AVX-NEXT:    callq __floatsitf@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -670,7 +670,7 @@ define dso_local void @TestSIToFPU16_F128() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movzwl {{.*}}(%rip), %edi
-; X64-SSE-NEXT:    callq __floatsitf
+; X64-SSE-NEXT:    callq __floatsitf@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -702,7 +702,7 @@ define dso_local void @TestSIToFPU16_F128() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    movzwl {{.*}}(%rip), %edi
-; X64-AVX-NEXT:    callq __floatsitf
+; X64-AVX-NEXT:    callq __floatsitf@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -718,7 +718,7 @@ define dso_local void @TestSIToFPI32_F128() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movl {{.*}}(%rip), %edi
-; X64-SSE-NEXT:    callq __floatsitf
+; X64-SSE-NEXT:    callq __floatsitf@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -748,7 +748,7 @@ define dso_local void @TestSIToFPI32_F128() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    movl {{.*}}(%rip), %edi
-; X64-AVX-NEXT:    callq __floatsitf
+; X64-AVX-NEXT:    callq __floatsitf@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -764,7 +764,7 @@ define dso_local void @TestUIToFPU32_F128() #2 {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movl {{.*}}(%rip), %edi
-; X64-SSE-NEXT:    callq __floatunsitf
+; X64-SSE-NEXT:    callq __floatunsitf@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -794,7 +794,7 @@ define dso_local void @TestUIToFPU32_F128() #2 {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    movl {{.*}}(%rip), %edi
-; X64-AVX-NEXT:    callq __floatunsitf
+; X64-AVX-NEXT:    callq __floatunsitf@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -810,7 +810,7 @@ define dso_local void @TestSIToFPI64_F128() nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movq {{.*}}(%rip), %rdi
-; X64-SSE-NEXT:    callq __floatditf
+; X64-SSE-NEXT:    callq __floatditf@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -841,7 +841,7 @@ define dso_local void @TestSIToFPI64_F128() nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64-AVX-NEXT:    callq __floatditf
+; X64-AVX-NEXT:    callq __floatditf@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -857,7 +857,7 @@ define dso_local void @TestUIToFPU64_F128() #2 {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movq {{.*}}(%rip), %rdi
-; X64-SSE-NEXT:    callq __floatunditf
+; X64-SSE-NEXT:    callq __floatunditf@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -888,7 +888,7 @@ define dso_local void @TestUIToFPU64_F128() #2 {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rdi
-; X64-AVX-NEXT:    callq __floatunditf
+; X64-AVX-NEXT:    callq __floatunditf@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -905,7 +905,7 @@ define dso_local void @TestSIToFPI128_F128() nounwind {
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movq {{.*}}(%rip), %rdi
 ; X64-SSE-NEXT:    movq vi128+{{.*}}(%rip), %rsi
-; X64-SSE-NEXT:    callq __floattitf
+; X64-SSE-NEXT:    callq __floattitf@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -939,7 +939,7 @@ define dso_local void @TestSIToFPI128_F128() nounwind {
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rdi
 ; X64-AVX-NEXT:    movq vi128+{{.*}}(%rip), %rsi
-; X64-AVX-NEXT:    callq __floattitf
+; X64-AVX-NEXT:    callq __floattitf@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -956,7 +956,7 @@ define dso_local void @TestUIToFPU128_F128() #2 {
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movq {{.*}}(%rip), %rdi
 ; X64-SSE-NEXT:    movq vu128+{{.*}}(%rip), %rsi
-; X64-SSE-NEXT:    callq __floatuntitf
+; X64-SSE-NEXT:    callq __floatuntitf@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, {{.*}}(%rip)
 ; X64-SSE-NEXT:    popq %rax
 ; X64-SSE-NEXT:    retq
@@ -990,7 +990,7 @@ define dso_local void @TestUIToFPU128_F128() #2 {
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rdi
 ; X64-AVX-NEXT:    movq vu128+{{.*}}(%rip), %rsi
-; X64-AVX-NEXT:    callq __floatuntitf
+; X64-AVX-NEXT:    callq __floatuntitf@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, {{.*}}(%rip)
 ; X64-AVX-NEXT:    popq %rax
 ; X64-AVX-NEXT:    retq
@@ -1006,7 +1006,7 @@ define dso_local i32 @TestConst128(fp128 %v) nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    movaps {{.*}}(%rip), %xmm1
-; X64-SSE-NEXT:    callq __gttf2
+; X64-SSE-NEXT:    callq __gttf2@PLT
 ; X64-SSE-NEXT:    xorl %ecx, %ecx
 ; X64-SSE-NEXT:    testl %eax, %eax
 ; X64-SSE-NEXT:    setg %cl
@@ -1038,7 +1038,7 @@ define dso_local i32 @TestConst128(fp128 %v) nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm1
-; X64-AVX-NEXT:    callq __gttf2
+; X64-AVX-NEXT:    callq __gttf2@PLT
 ; X64-AVX-NEXT:    xorl %ecx, %ecx
 ; X64-AVX-NEXT:    testl %eax, %eax
 ; X64-AVX-NEXT:    setg %cl
@@ -1057,7 +1057,7 @@ define dso_local i32 @TestConst128Zero(fp128 %v) nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    pushq %rax
 ; X64-SSE-NEXT:    xorps %xmm1, %xmm1
-; X64-SSE-NEXT:    callq __gttf2
+; X64-SSE-NEXT:    callq __gttf2@PLT
 ; X64-SSE-NEXT:    xorl %ecx, %ecx
 ; X64-SSE-NEXT:    testl %eax, %eax
 ; X64-SSE-NEXT:    setg %cl
@@ -1089,7 +1089,7 @@ define dso_local i32 @TestConst128Zero(fp128 %v) nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    pushq %rax
 ; X64-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-AVX-NEXT:    callq __gttf2
+; X64-AVX-NEXT:    callq __gttf2@PLT
 ; X64-AVX-NEXT:    xorl %ecx, %ecx
 ; X64-AVX-NEXT:    testl %eax, %eax
 ; X64-AVX-NEXT:    setg %cl
@@ -1121,7 +1121,7 @@ define dso_local i32 @TestBits128(fp128 %ld) nounwind {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    subq $24, %rsp
 ; X64-SSE-NEXT:    movaps %xmm0, %xmm1
-; X64-SSE-NEXT:    callq __multf3
+; X64-SSE-NEXT:    callq __multf3@PLT
 ; X64-SSE-NEXT:    movaps %xmm0, (%rsp)
 ; X64-SSE-NEXT:    movq (%rsp), %rcx
 ; X64-SSE-NEXT:    movq %rcx, %rdx
@@ -1167,7 +1167,7 @@ define dso_local i32 @TestBits128(fp128 %ld) nounwind {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    subq $24, %rsp
 ; X64-AVX-NEXT:    vmovaps %xmm0, %xmm1
-; X64-AVX-NEXT:    callq __multf3
+; X64-AVX-NEXT:    callq __multf3@PLT
 ; X64-AVX-NEXT:    vmovaps %xmm0, (%rsp)
 ; X64-AVX-NEXT:    movq (%rsp), %rcx
 ; X64-AVX-NEXT:    movq %rcx, %rdx
@@ -1258,10 +1258,10 @@ define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind {
 ; X64-SSE-NEXT:    jl .LBB26_2
 ; X64-SSE-NEXT:  # %bb.1: # %if.then
 ; X64-SSE-NEXT:    pushq %rax
-; X64-SSE-NEXT:    callq __trunctfdf2
+; X64-SSE-NEXT:    callq __trunctfdf2@PLT
 ; X64-SSE-NEXT:    andps {{.*}}(%rip), %xmm0
 ; X64-SSE-NEXT:    orps {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    callq __extenddftf2
+; X64-SSE-NEXT:    callq __extenddftf2@PLT
 ; X64-SSE-NEXT:    addq $8, %rsp
 ; X64-SSE-NEXT:  .LBB26_2: # %cleanup
 ; X64-SSE-NEXT:    retq
@@ -1287,8 +1287,8 @@ define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind {
 ; X32-NEXT:    addl $16, %esp
 ; X32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; X32-NEXT:    testb $-128, {{[0-9]+}}(%esp)
-; X32-NEXT:    flds {{\.LCPI.*}}
-; X32-NEXT:    flds {{\.LCPI.*}}
+; X32-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
+; X32-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X32-NEXT:    jne .LBB26_3
 ; X32-NEXT:  # %bb.2: # %if.then
 ; X32-NEXT:    fstp %st(1)
@@ -1322,12 +1322,12 @@ define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind {
 ; X64-AVX-NEXT:    jl .LBB26_2
 ; X64-AVX-NEXT:  # %bb.1: # %if.then
 ; X64-AVX-NEXT:    pushq %rax
-; X64-AVX-NEXT:    callq __trunctfdf2
+; X64-AVX-NEXT:    callq __trunctfdf2@PLT
 ; X64-AVX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovddup {{.*#+}} xmm1 = [+Inf,+Inf]
 ; X64-AVX-NEXT:    # xmm1 = mem[0,0]
 ; X64-AVX-NEXT:    vorps %xmm0, %xmm1, %xmm0
-; X64-AVX-NEXT:    callq __extenddftf2
+; X64-AVX-NEXT:    callq __extenddftf2@PLT
 ; X64-AVX-NEXT:    addq $8, %rsp
 ; X64-AVX-NEXT:  .LBB26_2: # %cleanup
 ; X64-AVX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/fp128-i128.ll b/llvm/test/CodeGen/X86/fp128-i128.ll
index 90224de50338719322396f11bd925ee6273eec3a..a9e932304dadb2f01d37deae1978626461fd161c 100644
--- a/llvm/test/CodeGen/X86/fp128-i128.ll
+++ b/llvm/test/CodeGen/X86/fp128-i128.ll
@@ -139,12 +139,12 @@ define fp128 @TestI128_1(fp128 %x) #0 {
 ; SSE-NEXT:    movq %rcx, (%rsp)
 ; SSE-NEXT:    movaps (%rsp), %xmm0
 ; SSE-NEXT:    movaps {{.*}}(%rip), %xmm1
-; SSE-NEXT:    callq __lttf2
+; SSE-NEXT:    callq __lttf2@PLT
 ; SSE-NEXT:    xorl %ecx, %ecx
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    sets %cl
 ; SSE-NEXT:    shlq $4, %rcx
-; SSE-NEXT:    movaps {{\.LCPI.*}}(%rcx), %xmm0
+; SSE-NEXT:    movaps {{\.LCPI[0-9]+_[0-9]+}}(%rcx), %xmm0
 ; SSE-NEXT:    addq $40, %rsp
 ; SSE-NEXT:    retq
 ;
@@ -159,12 +159,12 @@ define fp128 @TestI128_1(fp128 %x) #0 {
 ; AVX-NEXT:    movq %rcx, (%rsp)
 ; AVX-NEXT:    vmovaps (%rsp), %xmm0
 ; AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm1
-; AVX-NEXT:    callq __lttf2
+; AVX-NEXT:    callq __lttf2@PLT
 ; AVX-NEXT:    xorl %ecx, %ecx
 ; AVX-NEXT:    testl %eax, %eax
 ; AVX-NEXT:    sets %cl
 ; AVX-NEXT:    shlq $4, %rcx
-; AVX-NEXT:    vmovaps {{\.LCPI.*}}(%rcx), %xmm0
+; AVX-NEXT:    vmovaps {{\.LCPI[0-9]+_[0-9]+}}(%rcx), %xmm0
 ; AVX-NEXT:    addq $40, %rsp
 ; AVX-NEXT:    retq
 entry:
@@ -237,7 +237,7 @@ define fp128 @TestI128_3(fp128 %x, i32* nocapture readnone %ex) #0 {
 ; SSE-NEXT:    jmp .LBB4_3
 ; SSE-NEXT:  .LBB4_2: # %if.then
 ; SSE-NEXT:    movaps {{.*}}(%rip), %xmm1
-; SSE-NEXT:    callq __multf3
+; SSE-NEXT:    callq __multf3@PLT
 ; SSE-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; SSE-NEXT:    movabsq $-9223090561878065153, %rdx # imm = 0x8000FFFFFFFFFFFF
@@ -264,7 +264,7 @@ define fp128 @TestI128_3(fp128 %x, i32* nocapture readnone %ex) #0 {
 ; AVX-NEXT:    jmp .LBB4_3
 ; AVX-NEXT:  .LBB4_2: # %if.then
 ; AVX-NEXT:    vmovaps {{.*}}(%rip), %xmm1
-; AVX-NEXT:    callq __multf3
+; AVX-NEXT:    callq __multf3@PLT
 ; AVX-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%rsp)
 ; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; AVX-NEXT:    movabsq $-9223090561878065153, %rdx # imm = 0x8000FFFFFFFFFFFF
@@ -451,11 +451,11 @@ define dso_local void @TestCopySign({ fp128, fp128 }* noalias nocapture sret({ f
 ; SSE-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
 ; SSE-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
-; SSE-NEXT:    callq __gttf2
+; SSE-NEXT:    callq __gttf2@PLT
 ; SSE-NEXT:    movl %eax, %ebp
 ; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    callq __subtf3
+; SSE-NEXT:    callq __subtf3@PLT
 ; SSE-NEXT:    testl %ebp, %ebp
 ; SSE-NEXT:    jle .LBB10_1
 ; SSE-NEXT:  # %bb.2: # %if.then
@@ -488,11 +488,11 @@ define dso_local void @TestCopySign({ fp128, fp128 }* noalias nocapture sret({ f
 ; AVX-NEXT:    vmovaps {{[0-9]+}}(%rsp), %xmm1
 ; AVX-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; AVX-NEXT:    callq __gttf2
+; AVX-NEXT:    callq __gttf2@PLT
 ; AVX-NEXT:    movl %eax, %ebp
 ; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX-NEXT:    vmovaps %xmm0, %xmm1
-; AVX-NEXT:    callq __subtf3
+; AVX-NEXT:    callq __subtf3@PLT
 ; AVX-NEXT:    testl %ebp, %ebp
 ; AVX-NEXT:    jle .LBB10_1
 ; AVX-NEXT:  # %bb.2: # %if.then
diff --git a/llvm/test/CodeGen/X86/fp80-strict-scalar.ll b/llvm/test/CodeGen/X86/fp80-strict-scalar.ll
index e55e3903c0dc4412a15087e69dcacdad41ae6131..da638c2e4ff2a8deba028bfe7a92a91e3420001b 100644
--- a/llvm/test/CodeGen/X86/fp80-strict-scalar.ll
+++ b/llvm/test/CodeGen/X86/fp80-strict-scalar.ll
@@ -588,7 +588,7 @@ define i64 @fp80_to_uint64(x86_fp80 %x) #0 {
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $16, %esp
 ; X86-NEXT:    fldt 8(%ebp)
-; X86-NEXT:    flds {{\.LCPI.*}}
+; X86-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-NEXT:    fcom %st(1)
 ; X86-NEXT:    wait
 ; X86-NEXT:    fnstsw %ax
@@ -905,7 +905,7 @@ define x86_fp80 @uint64_to_fp80(i64 %x) #0 {
 ; X86-NEXT:    movl %eax, (%esp)
 ; X86-NEXT:    shrl $31, %ecx
 ; X86-NEXT:    fildll (%esp)
-; X86-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X86-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; X86-NEXT:    wait
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
@@ -919,7 +919,7 @@ define x86_fp80 @uint64_to_fp80(i64 %x) #0 {
 ; X64-NEXT:    testq %rdi, %rdi
 ; X64-NEXT:    sets %al
 ; X64-NEXT:    fildll -{{[0-9]+}}(%rsp)
-; X64-NEXT:    fadds {{\.LCPI.*}}(,%rax,4)
+; X64-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%rax,4)
 ; X64-NEXT:    wait
 ; X64-NEXT:    retq
   %result = call x86_fp80 @llvm.experimental.constrained.uitofp.f80.i64(i64 %x,
diff --git a/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll b/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll
index 23035a2f7e4092a15d160d8aaccbec91fa705913..37d8ace804e5910ab53c0de2a118100df089f85e 100644
--- a/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll
@@ -105,7 +105,7 @@ define i8 @test_signed_i8_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -117,7 +117,7 @@ define i8 @test_signed_i8_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movb {{[0-9]+}}(%esp), %dl
 ; X86-X87-NEXT:  .LBB1_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -176,7 +176,7 @@ define i13 @test_signed_i13_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -188,7 +188,7 @@ define i13 @test_signed_i13_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB2_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -248,7 +248,7 @@ define i16 @test_signed_i16_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -260,7 +260,7 @@ define i16 @test_signed_i16_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB3_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -320,7 +320,7 @@ define i19 @test_signed_i19_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -332,7 +332,7 @@ define i19 @test_signed_i19_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB4_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -362,8 +362,8 @@ define i19 @test_signed_i19_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    xorl %eax, %eax
 ; X86-SSE-NEXT:    ucomiss %xmm0, %xmm0
-; X86-SSE-NEXT:    maxss {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    minss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    maxss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    minss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvttss2si %xmm0, %ecx
 ; X86-SSE-NEXT:    cmovnpl %ecx, %eax
 ; X86-SSE-NEXT:    retl
@@ -393,7 +393,7 @@ define i32 @test_signed_i32_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -405,7 +405,7 @@ define i32 @test_signed_i32_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB5_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -434,7 +434,7 @@ define i32 @test_signed_i32_f32(float %f) nounwind {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    cvttss2si %xmm0, %eax
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $2147483647, %ecx # imm = 0x7FFFFFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %ecx
 ; X86-SSE-NEXT:    xorl %eax, %eax
@@ -471,7 +471,7 @@ define i50 @test_signed_i50_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fld %st(0)
 ; X86-X87-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -489,7 +489,7 @@ define i50 @test_signed_i50_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.3:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB6_4:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -537,12 +537,12 @@ define i50 @test_signed_i50_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    cmovbl %ecx, %esi
 ; X86-SSE-NEXT:    movl $-131072, %eax # imm = 0xFFFE0000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $131071, %edx # imm = 0x1FFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %edx
 ; X86-SSE-NEXT:    movl $-1, %eax
@@ -586,7 +586,7 @@ define i64 @test_signed_i64_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fld %st(0)
 ; X86-X87-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -604,7 +604,7 @@ define i64 @test_signed_i64_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.3:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB7_4:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -652,12 +652,12 @@ define i64 @test_signed_i64_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    cmovbl %ecx, %esi
 ; X86-SSE-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $2147483647, %edx # imm = 0x7FFFFFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %edx
 ; X86-SSE-NEXT:    movl $-1, %eax
@@ -695,7 +695,7 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fsts {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fsts {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-X87-NEXT:    fucompp
@@ -723,7 +723,7 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB8_6:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -789,7 +789,7 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    subl $4, %esp
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    xorl %ebp, %ebp
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-8, %ebx
 ; X86-SSE-NEXT:    movl $0, %ecx
 ; X86-SSE-NEXT:    movl $0, %edx
@@ -801,7 +801,7 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB8_2:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %eax
 ; X86-SSE-NEXT:    cmoval %eax, %edi
 ; X86-SSE-NEXT:    cmoval %eax, %edx
@@ -864,7 +864,7 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fsts {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fsts {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-X87-NEXT:    fucompp
@@ -888,7 +888,7 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB9_6:
 ; X86-X87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -964,7 +964,7 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    subl $4, %esp
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    cmovbl %ecx, %eax
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
@@ -973,7 +973,7 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    cmovbl %ecx, %edi
 ; X86-SSE-NEXT:    movl $-2147483648, %ebx # imm = 0x80000000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %ebx
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $2147483647, %ebp # imm = 0x7FFFFFFF
 ; X86-SSE-NEXT:    cmovbel %ebx, %ebp
 ; X86-SSE-NEXT:    movl $-1, %ebx
@@ -1125,7 +1125,7 @@ define i8 @test_signed_i8_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1137,7 +1137,7 @@ define i8 @test_signed_i8_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movb {{[0-9]+}}(%esp), %dl
 ; X86-X87-NEXT:  .LBB11_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1196,7 +1196,7 @@ define i13 @test_signed_i13_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1208,7 +1208,7 @@ define i13 @test_signed_i13_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB12_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1268,7 +1268,7 @@ define i16 @test_signed_i16_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1280,7 +1280,7 @@ define i16 @test_signed_i16_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB13_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1340,7 +1340,7 @@ define i19 @test_signed_i19_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1352,7 +1352,7 @@ define i19 @test_signed_i19_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB14_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1382,8 +1382,8 @@ define i19 @test_signed_i19_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE-NEXT:    xorl %eax, %eax
 ; X86-SSE-NEXT:    ucomisd %xmm0, %xmm0
-; X86-SSE-NEXT:    maxsd {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    minsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    maxsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    minsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvttsd2si %xmm0, %ecx
 ; X86-SSE-NEXT:    cmovnpl %ecx, %eax
 ; X86-SSE-NEXT:    retl
@@ -1413,7 +1413,7 @@ define i32 @test_signed_i32_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1425,7 +1425,7 @@ define i32 @test_signed_i32_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB15_2:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1455,8 +1455,8 @@ define i32 @test_signed_i32_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE-NEXT:    xorl %eax, %eax
 ; X86-SSE-NEXT:    ucomisd %xmm0, %xmm0
-; X86-SSE-NEXT:    maxsd {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    minsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    maxsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    minsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvttsd2si %xmm0, %ecx
 ; X86-SSE-NEXT:    cmovnpl %ecx, %eax
 ; X86-SSE-NEXT:    retl
@@ -1489,7 +1489,7 @@ define i50 @test_signed_i50_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fld %st(0)
 ; X86-X87-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1507,7 +1507,7 @@ define i50 @test_signed_i50_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.3:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB16_4:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1555,12 +1555,12 @@ define i50 @test_signed_i50_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    cmovbl %ecx, %esi
 ; X86-SSE-NEXT:    movl $-131072, %eax # imm = 0xFFFE0000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $131071, %edx # imm = 0x1FFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %edx
 ; X86-SSE-NEXT:    movl $-1, %eax
@@ -1600,7 +1600,7 @@ define i64 @test_signed_i64_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fld %st(0)
 ; X86-X87-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1618,7 +1618,7 @@ define i64 @test_signed_i64_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.3:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB17_4:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1666,12 +1666,12 @@ define i64 @test_signed_i64_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    cmovbl %ecx, %esi
 ; X86-SSE-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $2147483647, %edx # imm = 0x7FFFFFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %edx
 ; X86-SSE-NEXT:    movl $-1, %eax
@@ -1709,7 +1709,7 @@ define i100 @test_signed_i100_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fstl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fstl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
 ; X86-X87-NEXT:    fucompp
@@ -1737,7 +1737,7 @@ define i100 @test_signed_i100_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB18_6:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1803,7 +1803,7 @@ define i100 @test_signed_i100_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    subl $4, %esp
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE-NEXT:    xorl %ebp, %ebp
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-8, %ebx
 ; X86-SSE-NEXT:    movl $0, %ecx
 ; X86-SSE-NEXT:    movl $0, %edx
@@ -1815,7 +1815,7 @@ define i100 @test_signed_i100_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB18_2:
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %eax
 ; X86-SSE-NEXT:    cmoval %eax, %edi
 ; X86-SSE-NEXT:    cmoval %eax, %edx
@@ -1878,7 +1878,7 @@ define i128 @test_signed_i128_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fstl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fstl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
 ; X86-X87-NEXT:    fucompp
@@ -1902,7 +1902,7 @@ define i128 @test_signed_i128_f64(double %f) nounwind {
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB19_6:
 ; X86-X87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -1978,7 +1978,7 @@ define i128 @test_signed_i128_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    subl $4, %esp
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    cmovbl %ecx, %eax
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
@@ -1987,7 +1987,7 @@ define i128 @test_signed_i128_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    cmovbl %ecx, %edi
 ; X86-SSE-NEXT:    movl $-2147483648, %ebx # imm = 0x80000000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %ebx
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $2147483647, %ebp # imm = 0x7FFFFFFF
 ; X86-SSE-NEXT:    cmovbel %ebx, %ebp
 ; X86-SSE-NEXT:    movl $-1, %ebx
@@ -2153,7 +2153,7 @@ define i8 @test_signed_i8_f16(half %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2165,7 +2165,7 @@ define i8 @test_signed_i8_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movb {{[0-9]+}}(%esp), %dl
 ; X86-X87-NEXT:  .LBB21_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2236,7 +2236,7 @@ define i13 @test_signed_i13_f16(half %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2248,7 +2248,7 @@ define i13 @test_signed_i13_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB22_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2320,7 +2320,7 @@ define i16 @test_signed_i16_f16(half %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2332,7 +2332,7 @@ define i16 @test_signed_i16_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB23_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2404,7 +2404,7 @@ define i19 @test_signed_i19_f16(half %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2416,7 +2416,7 @@ define i19 @test_signed_i19_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB24_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2451,8 +2451,8 @@ define i19 @test_signed_i19_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    xorl %eax, %eax
 ; X86-SSE-NEXT:    ucomiss %xmm0, %xmm0
-; X86-SSE-NEXT:    maxss {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    minss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    maxss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    minss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvttss2si %xmm0, %ecx
 ; X86-SSE-NEXT:    cmovnpl %ecx, %eax
 ; X86-SSE-NEXT:    addl $12, %esp
@@ -2489,7 +2489,7 @@ define i32 @test_signed_i32_f16(half %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2501,7 +2501,7 @@ define i32 @test_signed_i32_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB25_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2535,7 +2535,7 @@ define i32 @test_signed_i32_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    cvttss2si %xmm0, %eax
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $2147483647, %ecx # imm = 0x7FFFFFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %ecx
 ; X86-SSE-NEXT:    xorl %eax, %eax
@@ -2579,7 +2579,7 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 ; X86-X87-NEXT:    fld %st(0)
 ; X86-X87-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2597,7 +2597,7 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.3:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB26_4:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2649,12 +2649,12 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    cmovbl %ecx, %esi
 ; X86-SSE-NEXT:    movl $-131072, %eax # imm = 0xFFFE0000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $131071, %edx # imm = 0x1FFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %edx
 ; X86-SSE-NEXT:    movl $-1, %eax
@@ -2704,7 +2704,7 @@ define i64 @test_signed_i64_f16(half %f) nounwind {
 ; X86-X87-NEXT:    fld %st(0)
 ; X86-X87-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2722,7 +2722,7 @@ define i64 @test_signed_i64_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.3:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB27_4:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2774,12 +2774,12 @@ define i64 @test_signed_i64_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    cmovbl %ecx, %esi
 ; X86-SSE-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $2147483647, %edx # imm = 0x7FFFFFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %edx
 ; X86-SSE-NEXT:    movl $-1, %eax
@@ -2823,7 +2823,7 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
 ; X86-X87-NEXT:    fsts {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fsts {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-X87-NEXT:    fucompp
@@ -2851,7 +2851,7 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB28_6:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -2923,7 +2923,7 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
 ; X86-SSE-NEXT:    # xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    xorl %ebp, %ebp
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-8, %ebx
 ; X86-SSE-NEXT:    movl $0, %ecx
 ; X86-SSE-NEXT:    movl $0, %edx
@@ -2935,7 +2935,7 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB28_2:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %eax
 ; X86-SSE-NEXT:    cmoval %eax, %edi
 ; X86-SSE-NEXT:    cmoval %eax, %edx
@@ -3002,7 +3002,7 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
 ; X86-X87-NEXT:    fsts {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fsts {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-X87-NEXT:    fucompp
@@ -3026,7 +3026,7 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB29_6:
 ; X86-X87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3108,7 +3108,7 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
 ; X86-SSE-NEXT:    movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
 ; X86-SSE-NEXT:    # xmm0 = mem[0],zero,zero,zero
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    cmovbl %ecx, %eax
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
@@ -3117,7 +3117,7 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    cmovbl %ecx, %edi
 ; X86-SSE-NEXT:    movl $-2147483648, %ebx # imm = 0x80000000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %ebx
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $2147483647, %ebp # imm = 0x7FFFFFFF
 ; X86-SSE-NEXT:    cmovbel %ebx, %ebp
 ; X86-SSE-NEXT:    movl $-1, %ebx
@@ -3311,7 +3311,7 @@ define i8 @test_signed_i8_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3323,7 +3323,7 @@ define i8 @test_signed_i8_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movb {{[0-9]+}}(%esp), %dl
 ; X86-X87-NEXT:  .LBB31_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3359,13 +3359,13 @@ define i8 @test_signed_i8_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
 ; X86-SSE-NEXT:    movl $128, %ecx
 ; X86-SSE-NEXT:    cmovael %eax, %ecx
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3422,7 +3422,7 @@ define i13 @test_signed_i13_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3434,7 +3434,7 @@ define i13 @test_signed_i13_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB32_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3470,7 +3470,7 @@ define i13 @test_signed_i13_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3479,7 +3479,7 @@ define i13 @test_signed_i13_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:  # %bb.1:
 ; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:  .LBB32_2:
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3538,7 +3538,7 @@ define i16 @test_signed_i16_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3550,7 +3550,7 @@ define i16 @test_signed_i16_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB33_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3586,7 +3586,7 @@ define i16 @test_signed_i16_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3595,7 +3595,7 @@ define i16 @test_signed_i16_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:  # %bb.1:
 ; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:  .LBB33_2:
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3654,7 +3654,7 @@ define i19 @test_signed_i19_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3666,7 +3666,7 @@ define i19 @test_signed_i19_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB34_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3702,7 +3702,7 @@ define i19 @test_signed_i19_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw (%esp)
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3711,7 +3711,7 @@ define i19 @test_signed_i19_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:  # %bb.1:
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:  .LBB34_2:
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3768,7 +3768,7 @@ define i32 @test_signed_i32_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3780,7 +3780,7 @@ define i32 @test_signed_i32_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB35_2:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3816,7 +3816,7 @@ define i32 @test_signed_i32_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fistl {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw (%esp)
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3825,7 +3825,7 @@ define i32 @test_signed_i32_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:  # %bb.1:
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:  .LBB35_2:
-; X86-SSE-NEXT:    fldl {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3885,7 +3885,7 @@ define i50 @test_signed_i50_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fld %st(0)
 ; X86-X87-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3903,7 +3903,7 @@ define i50 @test_signed_i50_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.3:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB36_4:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -3950,7 +3950,7 @@ define i50 @test_signed_i50_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -3958,7 +3958,7 @@ define i50 @test_signed_i50_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    cmovbl %ecx, %esi
 ; X86-SSE-NEXT:    movl $-131072, %eax # imm = 0xFFFE0000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    fldl {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -4024,7 +4024,7 @@ define i64 @test_signed_i64_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fld %st(0)
 ; X86-X87-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fldcw {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -4042,7 +4042,7 @@ define i64 @test_signed_i64_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.3:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB37_4:
-; X86-X87-NEXT:    fldt {{\.LCPI.*}}
+; X86-X87-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -4089,7 +4089,7 @@ define i64 @test_signed_i64_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fistpll {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -4097,7 +4097,7 @@ define i64 @test_signed_i64_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    cmovbl %ecx, %esi
 ; X86-SSE-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    fldt {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -4161,7 +4161,7 @@ define i100 @test_signed_i100_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fstpt {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fld %st(1)
 ; X86-X87-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
 ; X86-X87-NEXT:    fxch %st(1)
@@ -4190,7 +4190,7 @@ define i100 @test_signed_i100_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-X87-NEXT:  .LBB38_6:
-; X86-X87-NEXT:    fldt {{\.LCPI.*}}
+; X86-X87-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -4258,7 +4258,7 @@ define i100 @test_signed_i100_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    subl $4, %esp
 ; X86-SSE-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
 ; X86-SSE-NEXT:    xorl %ebp, %ebp
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -4273,7 +4273,7 @@ define i100 @test_signed_i100_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB38_2:
-; X86-SSE-NEXT:    fldt {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -4347,7 +4347,7 @@ define i128 @test_signed_i128_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fstpt {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fld %st(1)
 ; X86-X87-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
 ; X86-X87-NEXT:    fxch %st(1)
@@ -4372,7 +4372,7 @@ define i128 @test_signed_i128_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB39_6:
 ; X86-X87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-X87-NEXT:    fldt {{\.LCPI.*}}
+; X86-X87-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fstp %st(1)
@@ -4449,7 +4449,7 @@ define i128 @test_signed_i128_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    calll __fixxfti
 ; X86-SSE-NEXT:    subl $4, %esp
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
@@ -4461,7 +4461,7 @@ define i128 @test_signed_i128_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    cmovbl %ecx, %edi
 ; X86-SSE-NEXT:    movl $-2147483648, %ebx # imm = 0x80000000
 ; X86-SSE-NEXT:    cmovael {{[0-9]+}}(%esp), %ebx
-; X86-SSE-NEXT:    fldt {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
diff --git a/llvm/test/CodeGen/X86/fptoui-sat-scalar.ll b/llvm/test/CodeGen/X86/fptoui-sat-scalar.ll
index 294189815c490a8fda18d61ddb639da535a9baa4..18bc6400c583419349daaca34b14bf0a764f0afe 100644
--- a/llvm/test/CodeGen/X86/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/X86/fptoui-sat-scalar.ll
@@ -107,7 +107,7 @@ define i8 @test_unsigned_i8_f32(float %f) nounwind {
 ; X86-X87-NEXT:  .LBB1_1:
 ; X86-X87-NEXT:    xorl %ecx, %ecx
 ; X86-X87-NEXT:  .LBB1_3:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -168,7 +168,7 @@ define i13 @test_unsigned_i13_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB2_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -230,7 +230,7 @@ define i16 @test_unsigned_i16_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB3_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -293,7 +293,7 @@ define i19 @test_unsigned_i19_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB4_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -312,7 +312,7 @@ define i19 @test_unsigned_i19_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    xorps %xmm1, %xmm1
 ; X86-SSE-NEXT:    maxss %xmm1, %xmm0
-; X86-SSE-NEXT:    minss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    minss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvttss2si %xmm0, %eax
 ; X86-SSE-NEXT:    retl
 ;
@@ -352,7 +352,7 @@ define i32 @test_unsigned_i32_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB5_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -381,7 +381,7 @@ define i32 @test_unsigned_i32_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    xorps %xmm1, %xmm1
 ; X86-SSE-NEXT:    ucomiss %xmm1, %xmm0
 ; X86-SSE-NEXT:    cmovael %ecx, %edx
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %eax
 ; X86-SSE-NEXT:    cmovbel %edx, %eax
 ; X86-SSE-NEXT:    retl
@@ -407,7 +407,7 @@ define i50 @test_unsigned_i50_f32(float %f) nounwind {
 ; X86-X87-NEXT:    pushl %esi
 ; X86-X87-NEXT:    subl $16, %esp
 ; X86-X87-NEXT:    flds {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    xorl %ecx, %ecx
@@ -449,7 +449,7 @@ define i50 @test_unsigned_i50_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-X87-NEXT:  .LBB6_6:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -503,7 +503,7 @@ define i50 @test_unsigned_i50_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    xorl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:  .LBB6_4:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $262143, %edx # imm = 0x3FFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %edx
 ; X86-SSE-NEXT:    movl $-1, %eax
@@ -534,7 +534,7 @@ define i64 @test_unsigned_i64_f32(float %f) nounwind {
 ; X86-X87-NEXT:    pushl %esi
 ; X86-X87-NEXT:    subl $20, %esp
 ; X86-X87-NEXT:    flds {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    xorl %ecx, %ecx
@@ -576,7 +576,7 @@ define i64 @test_unsigned_i64_f32(float %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-X87-NEXT:  .LBB7_6:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -627,7 +627,7 @@ define i64 @test_unsigned_i64_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:  .LBB7_4:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %ecx
 ; X86-SSE-NEXT:    cmoval %ecx, %edx
 ; X86-SSE-NEXT:    cmoval %ecx, %eax
@@ -698,7 +698,7 @@ define i100 @test_unsigned_i100_f32(float %f) nounwind {
 ; X86-X87-NEXT:  .LBB8_6:
 ; X86-X87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -759,7 +759,7 @@ define i100 @test_unsigned_i100_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB8_2:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $15, %ebx
 ; X86-SSE-NEXT:    cmovbel %edi, %ebx
 ; X86-SSE-NEXT:    movl $-1, %edi
@@ -843,7 +843,7 @@ define i128 @test_unsigned_i128_f32(float %f) nounwind {
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-X87-NEXT:  .LBB9_6:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -900,7 +900,7 @@ define i128 @test_unsigned_i128_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB9_2:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %ebx
 ; X86-SSE-NEXT:    cmoval %ebx, %edi
 ; X86-SSE-NEXT:    cmoval %ebx, %edx
@@ -1043,7 +1043,7 @@ define i8 @test_unsigned_i8_f64(double %f) nounwind {
 ; X86-X87-NEXT:  .LBB11_1:
 ; X86-X87-NEXT:    xorl %ecx, %ecx
 ; X86-X87-NEXT:  .LBB11_3:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -1104,7 +1104,7 @@ define i13 @test_unsigned_i13_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB12_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -1166,7 +1166,7 @@ define i16 @test_unsigned_i16_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB13_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -1229,7 +1229,7 @@ define i19 @test_unsigned_i19_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB14_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -1248,7 +1248,7 @@ define i19 @test_unsigned_i19_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE-NEXT:    xorpd %xmm1, %xmm1
 ; X86-SSE-NEXT:    maxsd %xmm1, %xmm0
-; X86-SSE-NEXT:    minsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    minsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvttsd2si %xmm0, %eax
 ; X86-SSE-NEXT:    retl
 ;
@@ -1288,7 +1288,7 @@ define i32 @test_unsigned_i32_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB15_2:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -1307,7 +1307,7 @@ define i32 @test_unsigned_i32_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE-NEXT:    xorpd %xmm1, %xmm1
 ; X86-SSE-NEXT:    maxsd %xmm1, %xmm0
-; X86-SSE-NEXT:    minsd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    minsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvttsd2si %xmm0, %ecx
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; X86-SSE-NEXT:    movapd %xmm0, %xmm2
@@ -1337,7 +1337,7 @@ define i50 @test_unsigned_i50_f64(double %f) nounwind {
 ; X86-X87-NEXT:    pushl %esi
 ; X86-X87-NEXT:    subl $16, %esp
 ; X86-X87-NEXT:    fldl {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    xorl %ecx, %ecx
@@ -1379,7 +1379,7 @@ define i50 @test_unsigned_i50_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-X87-NEXT:  .LBB16_6:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -1433,7 +1433,7 @@ define i50 @test_unsigned_i50_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    xorl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:  .LBB16_4:
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $262143, %edx # imm = 0x3FFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %edx
 ; X86-SSE-NEXT:    movl $-1, %eax
@@ -1460,7 +1460,7 @@ define i64 @test_unsigned_i64_f64(double %f) nounwind {
 ; X86-X87-NEXT:    pushl %esi
 ; X86-X87-NEXT:    subl $20, %esp
 ; X86-X87-NEXT:    fldl {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    xorl %ecx, %ecx
@@ -1502,7 +1502,7 @@ define i64 @test_unsigned_i64_f64(double %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-X87-NEXT:  .LBB17_6:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -1553,7 +1553,7 @@ define i64 @test_unsigned_i64_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:  .LBB17_4:
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %ecx
 ; X86-SSE-NEXT:    cmoval %ecx, %edx
 ; X86-SSE-NEXT:    cmoval %ecx, %eax
@@ -1624,7 +1624,7 @@ define i100 @test_unsigned_i100_f64(double %f) nounwind {
 ; X86-X87-NEXT:  .LBB18_6:
 ; X86-X87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -1685,7 +1685,7 @@ define i100 @test_unsigned_i100_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB18_2:
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $15, %ebx
 ; X86-SSE-NEXT:    cmovbel %edi, %ebx
 ; X86-SSE-NEXT:    movl $-1, %edi
@@ -1769,7 +1769,7 @@ define i128 @test_unsigned_i128_f64(double %f) nounwind {
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-X87-NEXT:  .LBB19_6:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -1826,7 +1826,7 @@ define i128 @test_unsigned_i128_f64(double %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB19_2:
-; X86-SSE-NEXT:    ucomisd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomisd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %ebx
 ; X86-SSE-NEXT:    cmoval %ebx, %edi
 ; X86-SSE-NEXT:    cmoval %ebx, %edx
@@ -1983,7 +1983,7 @@ define i8 @test_unsigned_i8_f16(half %f) nounwind {
 ; X86-X87-NEXT:  .LBB21_1:
 ; X86-X87-NEXT:    xorl %ecx, %ecx
 ; X86-X87-NEXT:  .LBB21_3:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2056,7 +2056,7 @@ define i13 @test_unsigned_i13_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB22_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2130,7 +2130,7 @@ define i16 @test_unsigned_i16_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB23_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2205,7 +2205,7 @@ define i19 @test_unsigned_i19_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB24_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2229,7 +2229,7 @@ define i19 @test_unsigned_i19_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    xorps %xmm1, %xmm1
 ; X86-SSE-NEXT:    maxss %xmm1, %xmm0
-; X86-SSE-NEXT:    minss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    minss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvttss2si %xmm0, %eax
 ; X86-SSE-NEXT:    addl $12, %esp
 ; X86-SSE-NEXT:    retl
@@ -2276,7 +2276,7 @@ define i32 @test_unsigned_i32_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB25_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2310,7 +2310,7 @@ define i32 @test_unsigned_i32_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    xorps %xmm1, %xmm1
 ; X86-SSE-NEXT:    ucomiss %xmm1, %xmm0
 ; X86-SSE-NEXT:    cmovael %ecx, %edx
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %eax
 ; X86-SSE-NEXT:    cmovbel %edx, %eax
 ; X86-SSE-NEXT:    addl $12, %esp
@@ -2343,7 +2343,7 @@ define i50 @test_unsigned_i50_f16(half %f) nounwind {
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
 ; X86-X87-NEXT:    calll __gnu_h2f_ieee
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2385,7 +2385,7 @@ define i50 @test_unsigned_i50_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-X87-NEXT:  .LBB26_6:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2443,7 +2443,7 @@ define i50 @test_unsigned_i50_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    xorl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:  .LBB26_4:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $262143, %edx # imm = 0x3FFFF
 ; X86-SSE-NEXT:    cmovbel %eax, %edx
 ; X86-SSE-NEXT:    movl $-1, %eax
@@ -2480,7 +2480,7 @@ define i64 @test_unsigned_i64_f16(half %f) nounwind {
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
 ; X86-X87-NEXT:    calll __gnu_h2f_ieee
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2522,7 +2522,7 @@ define i64 @test_unsigned_i64_f16(half %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-X87-NEXT:  .LBB27_6:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2577,7 +2577,7 @@ define i64 @test_unsigned_i64_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    xorl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:  .LBB27_4:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %ecx
 ; X86-SSE-NEXT:    cmoval %ecx, %edx
 ; X86-SSE-NEXT:    cmoval %ecx, %eax
@@ -2654,7 +2654,7 @@ define i100 @test_unsigned_i100_f16(half %f) nounwind {
 ; X86-X87-NEXT:  .LBB28_6:
 ; X86-X87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2721,7 +2721,7 @@ define i100 @test_unsigned_i100_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB28_2:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $15, %ebx
 ; X86-SSE-NEXT:    cmovbel %edi, %ebx
 ; X86-SSE-NEXT:    movl $-1, %edi
@@ -2809,7 +2809,7 @@ define i128 @test_unsigned_i128_f16(half %f) nounwind {
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-X87-NEXT:  .LBB29_6:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -2872,7 +2872,7 @@ define i128 @test_unsigned_i128_f16(half %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB29_2:
-; X86-SSE-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movl $-1, %ebx
 ; X86-SSE-NEXT:    cmoval %ebx, %edi
 ; X86-SSE-NEXT:    cmoval %ebx, %edx
@@ -3051,7 +3051,7 @@ define i8 @test_unsigned_i8_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  .LBB31_1:
 ; X86-X87-NEXT:    xorl %ecx, %ecx
 ; X86-X87-NEXT:  .LBB31_3:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -3083,7 +3083,7 @@ define i8 @test_unsigned_i8_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
 ; X86-SSE-NEXT:    cmovael %eax, %ecx
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucompi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(0)
@@ -3146,7 +3146,7 @@ define i13 @test_unsigned_i13_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB32_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -3181,7 +3181,7 @@ define i13 @test_unsigned_i13_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:  # %bb.1:
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE-NEXT:  .LBB32_2:
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucompi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(0)
@@ -3246,7 +3246,7 @@ define i16 @test_unsigned_i16_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB33_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -3281,7 +3281,7 @@ define i16 @test_unsigned_i16_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:  # %bb.1:
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE-NEXT:  .LBB33_2:
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucompi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(0)
@@ -3347,7 +3347,7 @@ define i19 @test_unsigned_i19_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB34_2:
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -3382,7 +3382,7 @@ define i19 @test_unsigned_i19_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:  # %bb.1:
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE-NEXT:  .LBB34_2:
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucompi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(0)
@@ -3447,7 +3447,7 @@ define i32 @test_unsigned_i32_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-X87-NEXT:  .LBB35_2:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -3482,7 +3482,7 @@ define i32 @test_unsigned_i32_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:  # %bb.1:
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE-NEXT:  .LBB35_2:
-; X86-SSE-NEXT:    fldl {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucompi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(0)
@@ -3528,7 +3528,7 @@ define i50 @test_unsigned_i50_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    pushl %esi
 ; X86-X87-NEXT:    subl $16, %esp
 ; X86-X87-NEXT:    fldt {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    xorl %ecx, %ecx
@@ -3570,7 +3570,7 @@ define i50 @test_unsigned_i50_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-X87-NEXT:  .LBB36_6:
-; X86-X87-NEXT:    fldl {{\.LCPI.*}}
+; X86-X87-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -3595,7 +3595,7 @@ define i50 @test_unsigned_i50_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    pushl %esi
 ; X86-SSE-NEXT:    subl $16, %esp
 ; X86-SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    xorl %eax, %eax
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    setbe %cl
@@ -3625,7 +3625,7 @@ define i50 @test_unsigned_i50_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl %eax, %esi
 ; X86-SSE-NEXT:  .LBB36_2:
-; X86-SSE-NEXT:    fldl {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucompi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(0)
@@ -3682,7 +3682,7 @@ define i64 @test_unsigned_i64_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    pushl %esi
 ; X86-X87-NEXT:    subl $20, %esp
 ; X86-X87-NEXT:    fldt {{[0-9]+}}(%esp)
-; X86-X87-NEXT:    flds {{\.LCPI.*}}
+; X86-X87-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fucom %st(1)
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    xorl %ecx, %ecx
@@ -3724,7 +3724,7 @@ define i64 @test_unsigned_i64_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  # %bb.5:
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-X87-NEXT:  .LBB37_6:
-; X86-X87-NEXT:    fldt {{\.LCPI.*}}
+; X86-X87-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fxch %st(1)
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -3747,7 +3747,7 @@ define i64 @test_unsigned_i64_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    pushl %ebx
 ; X86-SSE-NEXT:    subl $16, %esp
 ; X86-SSE-NEXT:    fldt {{[0-9]+}}(%esp)
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    xorl %ecx, %ecx
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    setbe %bl
@@ -3777,7 +3777,7 @@ define i64 @test_unsigned_i64_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    movl %ecx, %edx
 ; X86-SSE-NEXT:  .LBB37_2:
-; X86-SSE-NEXT:    fldt {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucompi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(0)
@@ -3869,7 +3869,7 @@ define i100 @test_unsigned_i100_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:  .LBB38_6:
 ; X86-X87-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-X87-NEXT:    fldt {{\.LCPI.*}}
+; X86-X87-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -3933,7 +3933,7 @@ define i100 @test_unsigned_i100_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB38_2:
-; X86-SSE-NEXT:    fldt {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucompi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(0)
@@ -4028,7 +4028,7 @@ define i128 @test_unsigned_i128_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-X87-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-X87-NEXT:  .LBB39_6:
-; X86-X87-NEXT:    fldt {{\.LCPI.*}}
+; X86-X87-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-X87-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
 ; X86-X87-NEXT:    fucompp
 ; X86-X87-NEXT:    fnstsw %ax
@@ -4088,7 +4088,7 @@ define i128 @test_unsigned_i128_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-SSE-NEXT:  .LBB39_2:
-; X86-SSE-NEXT:    fldt {{\.LCPI.*}}
+; X86-SSE-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucompi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(0)
diff --git a/llvm/test/CodeGen/X86/funnel-shift-rot.ll b/llvm/test/CodeGen/X86/funnel-shift-rot.ll
index b5e17ab80208f1ab29107d43c37340f2c6dfcd8b..a777a0d0adc9b70489a5b0c0f36f71b5b10fb379 100644
--- a/llvm/test/CodeGen/X86/funnel-shift-rot.ll
+++ b/llvm/test/CodeGen/X86/funnel-shift-rot.ll
@@ -130,9 +130,9 @@ define i32 @rotl_i32(i32 %x, i32 %z) nounwind {
 define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %z) nounwind {
 ; X32-SSE2-LABEL: rotl_v4i32:
 ; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-SSE2-NEXT:    pslld $23, %xmm1
-; X32-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X32-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X32-SSE2-NEXT:    pmuludq %xmm1, %xmm0
@@ -324,9 +324,9 @@ define <4 x i32> @rotr_v4i32(<4 x i32> %x, <4 x i32> %z) nounwind {
 ; X32-SSE2:       # %bb.0:
 ; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X32-SSE2-NEXT:    psubd %xmm1, %xmm2
-; X32-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X32-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X32-SSE2-NEXT:    pslld $23, %xmm2
-; X32-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm2
+; X32-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X32-SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
 ; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X32-SSE2-NEXT:    pmuludq %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/haddsub-4.ll b/llvm/test/CodeGen/X86/haddsub-4.ll
index 2e077d6247ba9ee469ed3aa6bb62493241d9f5ec..3784400e30862063b751e32522e5f7e7bd0b1c11 100644
--- a/llvm/test/CodeGen/X86/haddsub-4.ll
+++ b/llvm/test/CodeGen/X86/haddsub-4.ll
@@ -123,26 +123,25 @@ define <8 x float> @hadd_reverse2_v8f32(<8 x float> %a0, <8 x float> %a1) {
 define <8 x float> @hadd_reverse3_v8f32(<8 x float> %a0, <8 x float> %a1) {
 ; SSE-LABEL: hadd_reverse3_v8f32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movaps %xmm0, %xmm4
-; SSE-NEXT:    haddps %xmm2, %xmm4
-; SSE-NEXT:    haddps %xmm3, %xmm1
-; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,2,1,0]
-; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,2,1,0]
-; SSE-NEXT:    movaps %xmm1, %xmm0
-; SSE-NEXT:    movaps %xmm4, %xmm1
+; SSE-NEXT:    haddps %xmm1, %xmm3
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,0,3,2]
+; SSE-NEXT:    haddps %xmm0, %xmm2
+; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,0,3,2]
+; SSE-NEXT:    movaps %xmm3, %xmm0
+; SSE-NEXT:    movaps %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: hadd_reverse3_v8f32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vhaddps %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4]
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,0,3,2,5,4,7,6]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: hadd_reverse3_v8f32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4]
+; AVX2-NEXT:    vhaddps %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,0,3,2,5,4,7,6]
 ; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
 ; AVX2-NEXT:    retq
   %shuf0 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
diff --git a/llvm/test/CodeGen/X86/haddsub-broadcast.ll b/llvm/test/CodeGen/X86/haddsub-broadcast.ll
index ec617bb2b03af1ded8a1670d719b476841cc5af9..8290d63cdccf711862fbdec5ddf904f57b9b5e57 100644
--- a/llvm/test/CodeGen/X86/haddsub-broadcast.ll
+++ b/llvm/test/CodeGen/X86/haddsub-broadcast.ll
@@ -8,7 +8,7 @@ define <4 x double> @PR43402(i64 %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; CHECK-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; CHECK-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; CHECK-NEXT:    vsubpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; CHECK-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; CHECK-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
 ; CHECK-NEXT:    vbroadcastsd %xmm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/haddsub-shuf.ll b/llvm/test/CodeGen/X86/haddsub-shuf.ll
index 22007df8320a1d567e344ca5b1fe7a974555b2d6..429175a10818b346c62cd0598245d72c6e16f2a7 100644
--- a/llvm/test/CodeGen/X86/haddsub-shuf.ll
+++ b/llvm/test/CodeGen/X86/haddsub-shuf.ll
@@ -525,7 +525,6 @@ define <8 x i32> @hadd_v8i32b(<8 x i32> %a) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: hadd_v8i32b:
@@ -615,7 +614,6 @@ define <8 x i32> @hsub_v8i32b(<8 x i32> %a) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vphsubd %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: hsub_v8i32b:
@@ -705,7 +703,6 @@ define <16 x i16> @hadd_v16i16b(<16 x i16> %a) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: hadd_v16i16b:
@@ -795,7 +792,6 @@ define <16 x i16> @hsub_v16i16b(<16 x i16> %a) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vphsubw %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: hsub_v16i16b:
diff --git a/llvm/test/CodeGen/X86/haddsub-undef.ll b/llvm/test/CodeGen/X86/haddsub-undef.ll
index 1be27fa3846dd40f5ed1b1ac0e50a16e087de762..1c06749440ee2fa0aee4f06bf5ec01501d8fd7a8 100644
--- a/llvm/test/CodeGen/X86/haddsub-undef.ll
+++ b/llvm/test/CodeGen/X86/haddsub-undef.ll
@@ -475,8 +475,8 @@ define <2 x double> @add_pd_010(<2 x double> %x) {
 ; AVX-SLOW-LABEL: add_pd_010:
 ; AVX-SLOW:       # %bb.0:
 ; AVX-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
-; AVX-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
 ; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: add_pd_010:
@@ -607,9 +607,9 @@ define <4 x float> @add_ps_017(<4 x float> %x) {
 ;
 ; AVX-SLOW-LABEL: add_ps_017:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: add_ps_017:
@@ -931,9 +931,9 @@ define <4 x float> @PR45747_1(<4 x float> %a, <4 x float> %b) nounwind {
 ;
 ; AVX-SLOW-LABEL: PR45747_1:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,2,2,2]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: PR45747_1:
@@ -963,9 +963,9 @@ define <4 x float> @PR45747_2(<4 x float> %a, <4 x float> %b) nounwind {
 ;
 ; AVX-SLOW-LABEL: PR45747_2:
 ; AVX-SLOW:       # %bb.0:
-; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm1[1,0]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,1,1]
+; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: PR45747_2:
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index 7789e7926041836d957d29b486ff67c6b17715e8..9f59af1ec6d537df7c815170794e544587aa6f30 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -103,7 +103,7 @@ define double @test_extend64(half* %addr) #0 {
 ; CHECK-LIBCALL:       # %bb.0:
 ; CHECK-LIBCALL-NEXT:    pushq %rax
 ; CHECK-LIBCALL-NEXT:    movzwl (%rdi), %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    cvtss2sd %xmm0, %xmm0
 ; CHECK-LIBCALL-NEXT:    popq %rax
 ; CHECK-LIBCALL-NEXT:    retq
@@ -135,7 +135,7 @@ define void @test_trunc32(float %in, half* %addr) #0 {
 ; CHECK-LIBCALL:       # %bb.0:
 ; CHECK-LIBCALL-NEXT:    pushq %rbx
 ; CHECK-LIBCALL-NEXT:    movq %rdi, %rbx
-; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    movw %ax, (%rbx)
 ; CHECK-LIBCALL-NEXT:    popq %rbx
 ; CHECK-LIBCALL-NEXT:    retq
@@ -168,7 +168,7 @@ define void @test_trunc64(double %in, half* %addr) #0 {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movq %rdi, %rbx
-; CHECK-NEXT:    callq __truncdfhf2
+; CHECK-NEXT:    callq __truncdfhf2@PLT
 ; CHECK-NEXT:    movw %ax, (%rbx)
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
@@ -195,7 +195,7 @@ define i64 @test_fptosi_i64(half* %p) #0 {
 ; CHECK-LIBCALL:       # %bb.0:
 ; CHECK-LIBCALL-NEXT:    pushq %rax
 ; CHECK-LIBCALL-NEXT:    movzwl (%rdi), %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    cvttss2si %xmm0, %rax
 ; CHECK-LIBCALL-NEXT:    popq %rcx
 ; CHECK-LIBCALL-NEXT:    retq
@@ -230,7 +230,7 @@ define void @test_sitofp_i64(i64 %a, half* %p) #0 {
 ; CHECK-LIBCALL-NEXT:    pushq %rbx
 ; CHECK-LIBCALL-NEXT:    movq %rsi, %rbx
 ; CHECK-LIBCALL-NEXT:    cvtsi2ss %rdi, %xmm0
-; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    movw %ax, (%rbx)
 ; CHECK-LIBCALL-NEXT:    popq %rbx
 ; CHECK-LIBCALL-NEXT:    retq
@@ -268,7 +268,7 @@ define i64 @test_fptoui_i64(half* %p) #0 {
 ; CHECK-LIBCALL:       # %bb.0:
 ; CHECK-LIBCALL-NEXT:    pushq %rax
 ; CHECK-LIBCALL-NEXT:    movzwl (%rdi), %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; CHECK-LIBCALL-NEXT:    movaps %xmm0, %xmm2
 ; CHECK-LIBCALL-NEXT:    subss %xmm1, %xmm2
@@ -330,7 +330,7 @@ define void @test_uitofp_i64(i64 %a, half* %p) #0 {
 ; CHECK-LIBCALL-NEXT:    cvtsi2ss %rdi, %xmm0
 ; CHECK-LIBCALL-NEXT:    addss %xmm0, %xmm0
 ; CHECK-LIBCALL-NEXT:  .LBB10_3:
-; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    movw %ax, (%rbx)
 ; CHECK-LIBCALL-NEXT:    popq %rbx
 ; CHECK-LIBCALL-NEXT:    retq
@@ -364,7 +364,7 @@ define void @test_uitofp_i64(i64 %a, half* %p) #0 {
 ; CHECK-I686-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
 ; CHECK-I686-NEXT:    shrl $31, %eax
 ; CHECK-I686-NEXT:    fildll {{[0-9]+}}(%esp)
-; CHECK-I686-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; CHECK-I686-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; CHECK-I686-NEXT:    fstps (%esp)
 ; CHECK-I686-NEXT:    calll __gnu_f2h_ieee
 ; CHECK-I686-NEXT:    movw %ax, (%esi)
@@ -389,21 +389,21 @@ define <4 x float> @test_extend32_vec4(<4 x half>* %p) #0 {
 ; CHECK-LIBCALL-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
 ; CHECK-LIBCALL-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-LIBCALL-NEXT:    pextrw $1, %xmm0, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-LIBCALL-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-LIBCALL-NEXT:    pextrw $0, %xmm0, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    punpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; CHECK-LIBCALL-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
 ; CHECK-LIBCALL-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-LIBCALL-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-LIBCALL-NEXT:    pextrw $1, %xmm0, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-LIBCALL-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-LIBCALL-NEXT:    pextrw $0, %xmm0, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    punpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; CHECK-LIBCALL-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
 ; CHECK-LIBCALL-NEXT:    punpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
@@ -476,21 +476,21 @@ define <4 x double> @test_extend64_vec4(<4 x half>* %p) #0 {
 ; CHECK-LIBCALL-NEXT:    movzwl 6(%rdi), %ebp
 ; CHECK-LIBCALL-NEXT:    movzwl (%rdi), %ebx
 ; CHECK-LIBCALL-NEXT:    movzwl 2(%rdi), %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    cvtss2sd %xmm0, %xmm0
 ; CHECK-LIBCALL-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; CHECK-LIBCALL-NEXT:    movl %ebx, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    cvtss2sd %xmm0, %xmm0
 ; CHECK-LIBCALL-NEXT:    unpcklpd (%rsp), %xmm0 # 16-byte Folded Reload
 ; CHECK-LIBCALL-NEXT:    # xmm0 = xmm0[0],mem[0]
 ; CHECK-LIBCALL-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; CHECK-LIBCALL-NEXT:    movl %ebp, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    cvtss2sd %xmm0, %xmm0
 ; CHECK-LIBCALL-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-LIBCALL-NEXT:    movl %r14d, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    cvtss2sd %xmm0, %xmm1
 ; CHECK-LIBCALL-NEXT:    unpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
 ; CHECK-LIBCALL-NEXT:    # xmm1 = xmm1[0],mem[0]
@@ -559,18 +559,18 @@ define void @test_trunc32_vec4(<4 x float> %a, <4 x half>* %p) #0 {
 ; BWON-NOF16C-NEXT:    movq %rdi, %rbx
 ; BWON-NOF16C-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; BWON-NOF16C-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee
+; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee@PLT
 ; BWON-NOF16C-NEXT:    movl %eax, %r14d
 ; BWON-NOF16C-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
 ; BWON-NOF16C-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee
+; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee@PLT
 ; BWON-NOF16C-NEXT:    movl %eax, %r15d
 ; BWON-NOF16C-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
 ; BWON-NOF16C-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee
+; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee@PLT
 ; BWON-NOF16C-NEXT:    movl %eax, %ebp
 ; BWON-NOF16C-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
-; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee
+; BWON-NOF16C-NEXT:    callq __gnu_f2h_ieee@PLT
 ; BWON-NOF16C-NEXT:    movw %ax, (%rbx)
 ; BWON-NOF16C-NEXT:    movw %bp, 6(%rbx)
 ; BWON-NOF16C-NEXT:    movw %r15w, 4(%rbx)
@@ -592,18 +592,18 @@ define void @test_trunc32_vec4(<4 x float> %a, <4 x half>* %p) #0 {
 ; BWOFF-NEXT:    movq %rdi, %rbx
 ; BWOFF-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; BWOFF-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; BWOFF-NEXT:    callq __gnu_f2h_ieee
+; BWOFF-NEXT:    callq __gnu_f2h_ieee@PLT
 ; BWOFF-NEXT:    movw %ax, %r14w
 ; BWOFF-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
 ; BWOFF-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; BWOFF-NEXT:    callq __gnu_f2h_ieee
+; BWOFF-NEXT:    callq __gnu_f2h_ieee@PLT
 ; BWOFF-NEXT:    movw %ax, %r15w
 ; BWOFF-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
 ; BWOFF-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; BWOFF-NEXT:    callq __gnu_f2h_ieee
+; BWOFF-NEXT:    callq __gnu_f2h_ieee@PLT
 ; BWOFF-NEXT:    movw %ax, %bp
 ; BWOFF-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
-; BWOFF-NEXT:    callq __gnu_f2h_ieee
+; BWOFF-NEXT:    callq __gnu_f2h_ieee@PLT
 ; BWOFF-NEXT:    movw %ax, (%rbx)
 ; BWOFF-NEXT:    movw %bp, 6(%rbx)
 ; BWOFF-NEXT:    movw %r15w, 4(%rbx)
@@ -674,17 +674,17 @@ define void @test_trunc64_vec4(<4 x double> %a, <4 x half>* %p) #0 {
 ; BWON-NOF16C-NEXT:    movaps %xmm1, (%rsp) # 16-byte Spill
 ; BWON-NOF16C-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; BWON-NOF16C-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; BWON-NOF16C-NEXT:    callq __truncdfhf2
+; BWON-NOF16C-NEXT:    callq __truncdfhf2@PLT
 ; BWON-NOF16C-NEXT:    movl %eax, %r14d
 ; BWON-NOF16C-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
 ; BWON-NOF16C-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; BWON-NOF16C-NEXT:    callq __truncdfhf2
+; BWON-NOF16C-NEXT:    callq __truncdfhf2@PLT
 ; BWON-NOF16C-NEXT:    movl %eax, %r15d
 ; BWON-NOF16C-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; BWON-NOF16C-NEXT:    callq __truncdfhf2
+; BWON-NOF16C-NEXT:    callq __truncdfhf2@PLT
 ; BWON-NOF16C-NEXT:    movl %eax, %ebp
 ; BWON-NOF16C-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
-; BWON-NOF16C-NEXT:    callq __truncdfhf2
+; BWON-NOF16C-NEXT:    callq __truncdfhf2@PLT
 ; BWON-NOF16C-NEXT:    movw %ax, 4(%rbx)
 ; BWON-NOF16C-NEXT:    movw %bp, (%rbx)
 ; BWON-NOF16C-NEXT:    movw %r15w, 6(%rbx)
@@ -707,17 +707,17 @@ define void @test_trunc64_vec4(<4 x double> %a, <4 x half>* %p) #0 {
 ; BWOFF-NEXT:    movaps %xmm1, (%rsp) # 16-byte Spill
 ; BWOFF-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; BWOFF-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; BWOFF-NEXT:    callq __truncdfhf2
+; BWOFF-NEXT:    callq __truncdfhf2@PLT
 ; BWOFF-NEXT:    movw %ax, %r14w
 ; BWOFF-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
 ; BWOFF-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; BWOFF-NEXT:    callq __truncdfhf2
+; BWOFF-NEXT:    callq __truncdfhf2@PLT
 ; BWOFF-NEXT:    movw %ax, %r15w
 ; BWOFF-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; BWOFF-NEXT:    callq __truncdfhf2
+; BWOFF-NEXT:    callq __truncdfhf2@PLT
 ; BWOFF-NEXT:    movw %ax, %bp
 ; BWOFF-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
-; BWOFF-NEXT:    callq __truncdfhf2
+; BWOFF-NEXT:    callq __truncdfhf2@PLT
 ; BWOFF-NEXT:    movw %ax, 4(%rbx)
 ; BWOFF-NEXT:    movw %bp, (%rbx)
 ; BWOFF-NEXT:    movw %r15w, 6(%rbx)
@@ -740,22 +740,22 @@ define void @test_trunc64_vec4(<4 x double> %a, <4 x half>* %p) #0 {
 ; BWON-F16C-NEXT:    vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; BWON-F16C-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
 ; BWON-F16C-NEXT:    vzeroupper
-; BWON-F16C-NEXT:    callq __truncdfhf2
+; BWON-F16C-NEXT:    callq __truncdfhf2@PLT
 ; BWON-F16C-NEXT:    movl %eax, %r14d
 ; BWON-F16C-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; BWON-F16C-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; BWON-F16C-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; BWON-F16C-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
 ; BWON-F16C-NEXT:    vzeroupper
-; BWON-F16C-NEXT:    callq __truncdfhf2
+; BWON-F16C-NEXT:    callq __truncdfhf2@PLT
 ; BWON-F16C-NEXT:    movl %eax, %r15d
 ; BWON-F16C-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; BWON-F16C-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; BWON-F16C-NEXT:    vzeroupper
-; BWON-F16C-NEXT:    callq __truncdfhf2
+; BWON-F16C-NEXT:    callq __truncdfhf2@PLT
 ; BWON-F16C-NEXT:    movl %eax, %ebp
 ; BWON-F16C-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; BWON-F16C-NEXT:    callq __truncdfhf2
+; BWON-F16C-NEXT:    callq __truncdfhf2@PLT
 ; BWON-F16C-NEXT:    movw %ax, 4(%rbx)
 ; BWON-F16C-NEXT:    movw %bp, (%rbx)
 ; BWON-F16C-NEXT:    movw %r15w, 6(%rbx)
@@ -815,15 +815,15 @@ define half @test_f80trunc_nodagcombine() #0 {
 ; CHECK-LIBCALL-LABEL: test_f80trunc_nodagcombine:
 ; CHECK-LIBCALL:       # %bb.0:
 ; CHECK-LIBCALL-NEXT:    pushq %rax
-; CHECK-LIBCALL-NEXT:    callq test_floatret
-; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee
+; CHECK-LIBCALL-NEXT:    callq test_floatret@PLT
+; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    popq %rcx
 ; CHECK-LIBCALL-NEXT:    retq
 ;
 ; BWON-F16C-LABEL: test_f80trunc_nodagcombine:
 ; BWON-F16C:       # %bb.0:
 ; BWON-F16C-NEXT:    pushq %rax
-; BWON-F16C-NEXT:    callq test_floatret
+; BWON-F16C-NEXT:    callq test_floatret@PLT
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; BWON-F16C-NEXT:    vmovd %xmm0, %eax
 ; BWON-F16C-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -833,7 +833,7 @@ define half @test_f80trunc_nodagcombine() #0 {
 ; CHECK-I686-LABEL: test_f80trunc_nodagcombine:
 ; CHECK-I686:       # %bb.0:
 ; CHECK-I686-NEXT:    subl $12, %esp
-; CHECK-I686-NEXT:    calll test_floatret
+; CHECK-I686-NEXT:    calll test_floatret@PLT
 ; CHECK-I686-NEXT:    fstps (%esp)
 ; CHECK-I686-NEXT:    calll __gnu_f2h_ieee
 ; CHECK-I686-NEXT:    addl $12, %esp
@@ -853,14 +853,14 @@ define float @test_sitofp_fadd_i32(i32 %a, half* %b) #0 {
 ; CHECK-LIBCALL-NEXT:    subq $16, %rsp
 ; CHECK-LIBCALL-NEXT:    movzwl (%rsi), %ebx
 ; CHECK-LIBCALL-NEXT:    cvtsi2ss %edi, %xmm0
-; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    movzwl %ax, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-LIBCALL-NEXT:    movl %ebx, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    addss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    movzwl %ax, %edi
 ; CHECK-LIBCALL-NEXT:    addq $16, %rsp
 ; CHECK-LIBCALL-NEXT:    popq %rbx
@@ -920,7 +920,7 @@ define half @PR40273(half) #0 {
 ; CHECK-LIBCALL:       # %bb.0:
 ; CHECK-LIBCALL-NEXT:    pushq %rax
 ; CHECK-LIBCALL-NEXT:    movzwl %di, %edi
-; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee@PLT
 ; CHECK-LIBCALL-NEXT:    xorl %eax, %eax
 ; CHECK-LIBCALL-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-LIBCALL-NEXT:    ucomiss %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
index 29994c41f0b9fb927282a2316bf654b6caf38c26..4752e63c9f4a5a0536a5ecd06b3df0861e15c315 100644
--- a/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/X86/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll
@@ -499,7 +499,7 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
@@ -508,7 +508,7 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
@@ -586,7 +586,7 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
@@ -595,7 +595,7 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; X86-SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll b/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
index 5175ccadd9e757793ea788bf01b5edf0141d951d..29667b4f728c7839f81c2b6fcb7224e1902769da 100644
--- a/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
+++ b/llvm/test/CodeGen/X86/hoist-and-by-const-from-shl-in-eqcmp-zero.ll
@@ -461,7 +461,7 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-SSE2-LABEL: vec_4xi32_splat_eq:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
@@ -510,7 +510,7 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-SSE2-LABEL: vec_4xi32_nonsplat_eq:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,16776960,2147483648]
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
@@ -563,12 +563,12 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; X86-SSE2-NEXT:    movl $1, %eax
 ; X86-SSE2-NEXT:    movd %eax, %xmm2
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X86-SSE2-NEXT:    pmuludq {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pmuludq {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
@@ -611,7 +611,7 @@ define <4 x i1> @vec_4xi32_nonsplat_undef1_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; X86-SSE2-LABEL: vec_4xi32_nonsplat_undef1_eq:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
@@ -661,12 +661,12 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi
 ; X86-SSE2-NEXT:    movl $1, %eax
 ; X86-SSE2-NEXT:    movd %eax, %xmm2
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X86-SSE2-NEXT:    pmuludq {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pmuludq {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/horizontal-sum.ll b/llvm/test/CodeGen/X86/horizontal-sum.ll
index 156a423970bc9f995bb2dc8f2c0fa7a004003fca..6f3c7828c4e67dc0f37dcf95b97de966b049fe77 100644
--- a/llvm/test/CodeGen/X86/horizontal-sum.ll
+++ b/llvm/test/CodeGen/X86/horizontal-sum.ll
@@ -26,13 +26,12 @@ define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl
 ; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    haddps %xmm2, %xmm2
 ; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSSE3-SLOW-NEXT:    movddup {{.*#+}} xmm2 = xmm2[0,0]
-; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm2
+; SSSE3-SLOW-NEXT:    addps %xmm2, %xmm1
 ; SSSE3-SLOW-NEXT:    haddps %xmm3, %xmm3
-; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm1
-; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3]
-; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0]
+; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm2
+; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3]
+; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
 ; SSSE3-SLOW-NEXT:    retq
 ;
 ; SSSE3-FAST-LABEL: pair_sum_v4f32_v4f32:
@@ -46,13 +45,12 @@ define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl
 ; AVX1-SLOW:       # %bb.0:
 ; AVX1-SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX1-SLOW-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
-; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[1,3],xmm1[1,1]
-; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,1]
-; AVX1-SLOW-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,1]
+; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,1]
 ; AVX1-SLOW-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
-; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; AVX1-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
-; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[1]
+; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
+; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX1-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: pair_sum_v4f32_v4f32:
@@ -66,13 +64,12 @@ define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
 ; AVX2-SLOW-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[1,3],xmm1[1,1]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
-; AVX2-SLOW-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,3]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,1]
 ; AVX2-SLOW-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
-; AVX2-SLOW-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; AVX2-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[1]
+; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]
+; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX2-SLOW-NEXT:    retq
   %5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
   %6 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
@@ -111,13 +108,12 @@ define <4 x i32> @pair_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2,
 ; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
 ; SSSE3-SLOW-NEXT:    phaddd %xmm2, %xmm2
 ; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
-; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm2
+; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm1
 ; SSSE3-SLOW-NEXT:    phaddd %xmm3, %xmm3
-; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
-; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm1
-; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3]
-; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0]
+; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
+; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm2
+; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3]
+; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
 ; SSSE3-SLOW-NEXT:    retq
 ;
 ; SSSE3-FAST-LABEL: pair_sum_v4i32_v4i32:
@@ -138,9 +134,8 @@ define <4 x i32> @pair_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2,
 ; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX1-SLOW-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
-; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]
+; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
 ; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
 ; AVX1-SLOW-NEXT:    retq
 ;
@@ -163,9 +158,8 @@ define <4 x i32> @pair_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2,
 ; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
 ; AVX2-SLOW-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
-; AVX2-SLOW-NEXT:    vpbroadcastq %xmm1, %xmm2
-; AVX2-SLOW-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT:    vpbroadcastd %xmm1, %xmm2
+; AVX2-SLOW-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX2-SLOW-NEXT:    retq
   %5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
@@ -443,7 +437,7 @@ define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2,
 ; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5],xmm5[6,7]
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,3,1,1]
 ; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[1],zero
-; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
+; AVX1-SLOW-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]
 ; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
 ; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
@@ -495,7 +489,7 @@ define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2,
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
 ; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
-; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]
 ; AVX2-SLOW-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
 ; AVX2-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
@@ -648,17 +642,15 @@ define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <
 ; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
 ; AVX-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
 ; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1]
-; AVX-SLOW-NEXT:    vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm2[2,3]
-; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm4, %xmm1
 ; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3]
-; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; AVX-SLOW-NEXT:    vaddps %xmm3, %xmm1, %xmm1
-; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
+; AVX-SLOW-NEXT:    vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddps %xmm3, %xmm4, %xmm4
+; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
+; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2]
 ; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
-; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
-; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
-; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX-SLOW-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
+; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-SLOW-NEXT:    retq
 ;
 ; AVX-FAST-LABEL: sequential_sum_v4f32_v4f32:
@@ -669,16 +661,14 @@ define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <
 ; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
 ; AVX-FAST-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
 ; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1]
-; AVX-FAST-NEXT:    vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm2[2,3]
-; AVX-FAST-NEXT:    vaddps %xmm1, %xmm4, %xmm1
 ; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3]
-; AVX-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
-; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
+; AVX-FAST-NEXT:    vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm4
+; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
+; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2]
 ; AVX-FAST-NEXT:    vaddps %xmm1, %xmm2, %xmm1
-; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
-; AVX-FAST-NEXT:    vaddps %xmm1, %xmm2, %xmm1
-; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX-FAST-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]
+; AVX-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    retq
   %5 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 0, i32 4>
   %6 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 1, i32 5>
diff --git a/llvm/test/CodeGen/X86/i64-to-float.ll b/llvm/test/CodeGen/X86/i64-to-float.ll
index c181f71e0d816adc9569195e60448148c5983837..f7a2d330206e6a4660f8d6e19fee2758348f1a5c 100644
--- a/llvm/test/CodeGen/X86/i64-to-float.ll
+++ b/llvm/test/CodeGen/X86/i64-to-float.ll
@@ -14,7 +14,7 @@ define <2 x double> @mask_sitofp_2i64_2f64(<2 x i64> %a) nounwind {
 ; X86-SSE-LABEL: mask_sitofp_2i64_2f64:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
@@ -32,7 +32,7 @@ define <2 x double> @mask_sitofp_2i64_2f64(<2 x i64> %a) nounwind {
 ;
 ; X86-AVX512DQ-LABEL: mask_sitofp_2i64_2f64:
 ; X86-AVX512DQ:       # %bb.0:
-; X86-AVX512DQ-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX512DQ-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX512DQ-NEXT:    vcvtqq2pd %xmm0, %xmm0
 ; X86-AVX512DQ-NEXT:    retl
 ;
@@ -69,7 +69,7 @@ define <2 x double> @mask_uitofp_2i64_2f64(<2 x i64> %a) nounwind {
 ; X86-SSE-LABEL: mask_uitofp_2i64_2f64:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
@@ -87,7 +87,7 @@ define <2 x double> @mask_uitofp_2i64_2f64(<2 x i64> %a) nounwind {
 ;
 ; X86-AVX512DQ-LABEL: mask_uitofp_2i64_2f64:
 ; X86-AVX512DQ:       # %bb.0:
-; X86-AVX512DQ-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX512DQ-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX512DQ-NEXT:    vcvtqq2pd %xmm0, %xmm0
 ; X86-AVX512DQ-NEXT:    retl
 ;
@@ -124,7 +124,7 @@ define <4 x float> @mask_sitofp_4i64_4f32(<4 x i64> %a) nounwind {
 ; X86-SSE-LABEL: mask_sitofp_4i64_4f32:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X86-SSE-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
@@ -132,7 +132,7 @@ define <4 x float> @mask_sitofp_4i64_4f32(<4 x i64> %a) nounwind {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X86-AVX-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X86-AVX-NEXT:    vzeroupper
 ; X86-AVX-NEXT:    retl
@@ -140,14 +140,14 @@ define <4 x float> @mask_sitofp_4i64_4f32(<4 x i64> %a) nounwind {
 ; X86-AVX512F-LABEL: mask_sitofp_4i64_4f32:
 ; X86-AVX512F:       # %bb.0:
 ; X86-AVX512F-NEXT:    vpmovqd %ymm0, %xmm0
-; X86-AVX512F-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX512F-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX512F-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X86-AVX512F-NEXT:    vzeroupper
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X86-AVX512DQ-LABEL: mask_sitofp_4i64_4f32:
 ; X86-AVX512DQ:       # %bb.0:
-; X86-AVX512DQ-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX512DQ-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX512DQ-NEXT:    vcvtqq2ps %ymm0, %xmm0
 ; X86-AVX512DQ-NEXT:    vzeroupper
 ; X86-AVX512DQ-NEXT:    retl
@@ -191,7 +191,7 @@ define <4 x float> @mask_uitofp_4i64_4f32(<4 x i64> %a) nounwind {
 ; X86-SSE-LABEL: mask_uitofp_4i64_4f32:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X86-SSE-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
@@ -199,7 +199,7 @@ define <4 x float> @mask_uitofp_4i64_4f32(<4 x i64> %a) nounwind {
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X86-AVX-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X86-AVX-NEXT:    vzeroupper
 ; X86-AVX-NEXT:    retl
@@ -207,14 +207,14 @@ define <4 x float> @mask_uitofp_4i64_4f32(<4 x i64> %a) nounwind {
 ; X86-AVX512F-LABEL: mask_uitofp_4i64_4f32:
 ; X86-AVX512F:       # %bb.0:
 ; X86-AVX512F-NEXT:    vpmovqd %ymm0, %xmm0
-; X86-AVX512F-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX512F-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX512F-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X86-AVX512F-NEXT:    vzeroupper
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X86-AVX512DQ-LABEL: mask_uitofp_4i64_4f32:
 ; X86-AVX512DQ:       # %bb.0:
-; X86-AVX512DQ-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX512DQ-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX512DQ-NEXT:    vcvtqq2ps %ymm0, %xmm0
 ; X86-AVX512DQ-NEXT:    vzeroupper
 ; X86-AVX512DQ-NEXT:    retl
@@ -270,7 +270,7 @@ define <2 x double> @clamp_sitofp_2i64_2f64(<2 x i64> %a) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
 ; X86-SSE-NEXT:    por %xmm2, %xmm3
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pandn {{\.LCPI.*}}, %xmm3
+; X86-SSE-NEXT:    pandn {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-SSE-NEXT:    por %xmm0, %xmm3
 ; X86-SSE-NEXT:    pxor %xmm3, %xmm1
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm0 = [2147483903,0,2147483903,0]
@@ -283,7 +283,7 @@ define <2 x double> @clamp_sitofp_2i64_2f64(<2 x i64> %a) nounwind {
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
 ; X86-SSE-NEXT:    por %xmm0, %xmm1
 ; X86-SSE-NEXT:    pand %xmm1, %xmm3
-; X86-SSE-NEXT:    pandn {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    pandn {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    por %xmm3, %xmm1
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; X86-SSE-NEXT:    cvtdq2pd %xmm0, %xmm0
@@ -305,16 +305,16 @@ define <2 x double> @clamp_sitofp_2i64_2f64(<2 x i64> %a) nounwind {
 ;
 ; X86-AVX512F-LABEL: clamp_sitofp_2i64_2f64:
 ; X86-AVX512F:       # %bb.0:
-; X86-AVX512F-NEXT:    vpmaxsq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX512F-NEXT:    vpminsq {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX512F-NEXT:    vpmaxsq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX512F-NEXT:    vpminsq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX512F-NEXT:    vcvtdq2pd %xmm0, %xmm0
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X86-AVX512DQ-LABEL: clamp_sitofp_2i64_2f64:
 ; X86-AVX512DQ:       # %bb.0:
-; X86-AVX512DQ-NEXT:    vpmaxsq {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX512DQ-NEXT:    vpminsq {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX512DQ-NEXT:    vpmaxsq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX512DQ-NEXT:    vpminsq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX512DQ-NEXT:    vcvtqq2pd %xmm0, %xmm0
 ; X86-AVX512DQ-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/inalloca-ctor.ll b/llvm/test/CodeGen/X86/inalloca-ctor.ll
index f13d537d90b8f1932feb0c6b8e27b19c99312044..740c61a3e7d3ae624819e2d57a32b7ac6fc43aa9 100644
--- a/llvm/test/CodeGen/X86/inalloca-ctor.ll
+++ b/llvm/test/CodeGen/X86/inalloca-ctor.ll
@@ -4,7 +4,7 @@
 
 %frame = type { %Foo, i32, %Foo }
 
-declare void @f(%frame* inalloca %a)
+declare void @f(%frame* inalloca(%frame) %a)
 
 declare void @Foo_ctor(%Foo* %this)
 
@@ -28,7 +28,7 @@ entry:
 ; CHECK-NEXT: pushl
 ; CHECK-NEXT: calll _Foo_ctor
 ; CHECK: addl $4, %esp
-  call void @f(%frame* inalloca %args)
+  call void @f(%frame* inalloca(%frame) %args)
 ; CHECK: calll   _f
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/inalloca-invoke.ll b/llvm/test/CodeGen/X86/inalloca-invoke.ll
index 4623c58210a3e38faee14e80b7ee9162fa3d1153..39a9ac5751f2992b0d1a764a2da1a958ee5f941b 100644
--- a/llvm/test/CodeGen/X86/inalloca-invoke.ll
+++ b/llvm/test/CodeGen/X86/inalloca-invoke.ll
@@ -9,7 +9,7 @@ declare void @llvm.stackrestore(i8*)
 declare i8* @llvm.stacksave()
 declare void @begin(%Iter* sret(%Iter))
 declare void @plus(%Iter* sret(%Iter), %Iter*, i32)
-declare void @reverse(%frame.reverse* inalloca align 4)
+declare void @reverse(%frame.reverse* inalloca(%frame.reverse) align 4)
 
 define i32 @main() personality i32 (...)* @pers {
   %temp.lvalue = alloca %Iter
@@ -42,7 +42,7 @@ invoke.cont:
 ; CHECK:  pushl %[[beg]]
 ; CHECK:  calll _begin
 
-  invoke void @reverse(%frame.reverse* inalloca align 4 %rev_args)
+  invoke void @reverse(%frame.reverse* inalloca(%frame.reverse) align 4 %rev_args)
           to label %invoke.cont5 unwind label %lpad
 
 invoke.cont5:                                     ; preds = %invoke.cont
diff --git a/llvm/test/CodeGen/X86/inalloca-regparm.ll b/llvm/test/CodeGen/X86/inalloca-regparm.ll
index d379333a962f603d44d8a289615246f316b28ac8..24a7d17d4b4aae441c7c8def78e8d15d800c279b 100644
--- a/llvm/test/CodeGen/X86/inalloca-regparm.ll
+++ b/llvm/test/CodeGen/X86/inalloca-regparm.ll
@@ -4,11 +4,11 @@
 ; This will compile successfully on x86 but not x86_64, because %b will become a
 ; register parameter.
 
-declare x86_thiscallcc i32 @f(i32 %a, i32* inalloca %b)
+declare x86_thiscallcc i32 @f(i32 %a, i32* inalloca(i32) %b)
 define void @g() {
   %b = alloca inalloca i32
   store i32 2, i32* %b
-  call x86_thiscallcc i32 @f(i32 0, i32* inalloca %b)
+  call x86_thiscallcc i32 @f(i32 0, i32* inalloca(i32) %b)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/X86/inalloca-stdcall.ll b/llvm/test/CodeGen/X86/inalloca-stdcall.ll
index 69d94d8bfa744881d5000f69ed0ba7561aefea85..e0a292b866f2cd14ad7e17cd4c5a65c22e75d781 100644
--- a/llvm/test/CodeGen/X86/inalloca-stdcall.ll
+++ b/llvm/test/CodeGen/X86/inalloca-stdcall.ll
@@ -2,7 +2,7 @@
 
 %Foo = type { i32, i32 }
 
-declare x86_stdcallcc void @f(%Foo* inalloca %a)
+declare x86_stdcallcc void @f(%Foo* inalloca(%Foo) %a)
 declare x86_stdcallcc void @i(i32 %a)
 
 define void @g() {
@@ -17,7 +17,7 @@ define void @g() {
 ; CHECK: movl %esp, %eax
 ; CHECK: movl    $13, (%eax)
 ; CHECK: movl    $42, 4(%eax)
-  call x86_stdcallcc void @f(%Foo* inalloca %b)
+  call x86_stdcallcc void @f(%Foo* inalloca(%Foo) %b)
 ; CHECK: calll   _f@8
 ; CHECK-NOT: %esp
 ; CHECK: pushl
diff --git a/llvm/test/CodeGen/X86/inalloca.ll b/llvm/test/CodeGen/X86/inalloca.ll
index 134de2f58ddad8e7863fc7467603b1fbdd764ed6..ed85c79f6d44cdfa873a0aaf72b601803ca51d4a 100644
--- a/llvm/test/CodeGen/X86/inalloca.ll
+++ b/llvm/test/CodeGen/X86/inalloca.ll
@@ -2,7 +2,7 @@
 
 %Foo = type { i32, i32 }
 
-declare void @f(%Foo* inalloca %b)
+declare void @f(%Foo* inalloca(%Foo) %b)
 
 define void @a() {
 ; CHECK-LABEL: _a:
@@ -17,12 +17,12 @@ entry:
 ; CHECK: movl %esp, %eax
 ; CHECK: movl    $13, (%eax)
 ; CHECK: movl    $42, 4(%eax)
-  call void @f(%Foo* inalloca %b)
+  call void @f(%Foo* inalloca(%Foo) %b)
 ; CHECK: calll   _f
   ret void
 }
 
-declare void @inreg_with_inalloca(i32 inreg %a, %Foo* inalloca %b)
+declare void @inreg_with_inalloca(i32 inreg %a, %Foo* inalloca(%Foo) %b)
 
 define void @b() {
 ; CHECK-LABEL: _b:
@@ -37,13 +37,13 @@ entry:
 ; CHECK: movl %esp, %eax
 ; CHECK: movl    $13, (%eax)
 ; CHECK: movl    $42, 4(%eax)
-  call void @inreg_with_inalloca(i32 inreg 1, %Foo* inalloca %b)
+  call void @inreg_with_inalloca(i32 inreg 1, %Foo* inalloca(%Foo) %b)
 ; CHECK: movl    $1, %eax
 ; CHECK: calll   _inreg_with_inalloca
   ret void
 }
 
-declare x86_thiscallcc void @thiscall_with_inalloca(i8* %a, %Foo* inalloca %b)
+declare x86_thiscallcc void @thiscall_with_inalloca(i8* %a, %Foo* inalloca(%Foo) %b)
 
 define void @c() {
 ; CHECK-LABEL: _c:
@@ -58,7 +58,7 @@ entry:
 ; CHECK: movl %esp, %eax
 ; CHECK-DAG: movl    $13, (%eax)
 ; CHECK-DAG: movl    $42, 4(%eax)
-  call x86_thiscallcc void @thiscall_with_inalloca(i8* null, %Foo* inalloca %b)
+  call x86_thiscallcc void @thiscall_with_inalloca(i8* null, %Foo* inalloca(%Foo) %b)
 ; CHECK-DAG: xorl    %ecx, %ecx
 ; CHECK: calll   _thiscall_with_inalloca
   ret void
diff --git a/llvm/test/CodeGen/X86/insert-into-constant-vector.ll b/llvm/test/CodeGen/X86/insert-into-constant-vector.ll
index 37c73e07a17bcfe647b05ce50b39e2a193cb831a..3aa1abae99f3201dc2a04e17e4b52574fba22092 100644
--- a/llvm/test/CodeGen/X86/insert-into-constant-vector.ll
+++ b/llvm/test/CodeGen/X86/insert-into-constant-vector.ll
@@ -16,7 +16,7 @@ define <16 x i8> @elt0_v16i8(i8 %x) {
 ; X86-SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-SSE2-NEXT:    movaps {{.*#+}} xmm0 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; X86-SSE2-NEXT:    andnps %xmm1, %xmm0
-; X86-SSE2-NEXT:    orps {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    orps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: elt0_v16i8:
@@ -393,7 +393,7 @@ define <8 x i64> @elt5_v8i64(i64 %x) {
 ; X86-AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX1-NEXT:    vmovaps {{.*#+}} xmm1 = [4,0,0,0]
 ; X86-AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; X86-AVX1-NEXT:    vinsertf128 $1, {{\.LCPI.*}}, %ymm0, %ymm1
+; X86-AVX1-NEXT:    vinsertf128 $1, {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm1
 ; X86-AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,0,1,0,2,0,3,0]
 ; X86-AVX1-NEXT:    retl
 ;
@@ -410,7 +410,7 @@ define <8 x i64> @elt5_v8i64(i64 %x) {
 ; X86-AVX2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-AVX2-NEXT:    vmovaps {{.*#+}} xmm1 = [4,0,0,0]
 ; X86-AVX2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; X86-AVX2-NEXT:    vinsertf128 $1, {{\.LCPI.*}}, %ymm0, %ymm1
+; X86-AVX2-NEXT:    vinsertf128 $1, {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm1
 ; X86-AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,0,1,0,2,0,3,0]
 ; X86-AVX2-NEXT:    retl
 ;
@@ -428,7 +428,7 @@ define <8 x i64> @elt5_v8i64(i64 %x) {
 ; X86-AVX512F-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; X86-AVX512F-NEXT:    vmovaps {{.*#+}} xmm2 = [4,0,0,0]
 ; X86-AVX512F-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; X86-AVX512F-NEXT:    vinsertf128 $1, {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-AVX512F-NEXT:    vinsertf128 $1, {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1
 ; X86-AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; X86-AVX512F-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/insertelement-var-index.ll b/llvm/test/CodeGen/X86/insertelement-var-index.ll
index c88f294926a217f7bfffdf7825bd1264ec5fc86d..133d6c75029ca2a9640789f045660f8211379309 100644
--- a/llvm/test/CodeGen/X86/insertelement-var-index.ll
+++ b/llvm/test/CodeGen/X86/insertelement-var-index.ll
@@ -461,7 +461,7 @@ define <4 x i64> @arg_i64_v4i64_undef(i64 %x, i32 %y) nounwind {
 ; AVX1-LABEL: arg_i64_v4i64_undef:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovq %rdi, %xmm0
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -1380,7 +1380,7 @@ define <8 x float> @arg_f32_v8f32(<8 x float> %v, float %x, i32 %y) nounwind {
 ; AVX1-NEXT:    vmovd %edi, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd {{\.LCPI.*}}+{{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -1422,9 +1422,9 @@ define <4 x double> @arg_f64_v4f64(<4 x double> %v, double %x, i32 %y) nounwind
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; AVX1-NEXT:    movslq %edi, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm2
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = xmm2[0,0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
 ; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqq {{\.LCPI.*}}+{{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqq {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -1661,7 +1661,7 @@ define <8 x float> @load_f32_v8f32(<8 x float> %v, float* %p, i32 %y) nounwind {
 ; AVX1-NEXT:    vmovd %esi, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vbroadcastss (%rdi), %ymm2
 ; AVX1-NEXT:    vblendvps %ymm1, %ymm2, %ymm0, %ymm0
@@ -1704,9 +1704,9 @@ define <4 x double> @load_f64_v4f64(<4 x double> %v, double* %p, i32 %y) nounwin
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movslq %esi, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm1
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
 ; AVX1-NEXT:    vpcmpeqq {{.*}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqq {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqq {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vbroadcastsd (%rdi), %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm1, %ymm2, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/known-bits-vector.ll b/llvm/test/CodeGen/X86/known-bits-vector.ll
index 79c56b10afcee44d4f7c22fafc17d8d5e956af33..2a8ea368edb03194ede4c55896628944a48496d0 100644
--- a/llvm/test/CodeGen/X86/known-bits-vector.ll
+++ b/llvm/test/CodeGen/X86/known-bits-vector.ll
@@ -74,7 +74,7 @@ define <4 x float> @knownbits_insert_uitofp(<4 x i32> %a0, i16 %a1, i16 %a2) nou
 define <4 x i32> @knownbits_mask_shuffle_sext(<8 x i16> %a0) nounwind {
 ; X32-LABEL: knownbits_mask_shuffle_sext:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; X32-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; X32-NEXT:    retl
@@ -94,7 +94,7 @@ define <4 x i32> @knownbits_mask_shuffle_sext(<8 x i16> %a0) nounwind {
 define <4 x i32> @knownbits_mask_shuffle_shuffle_sext(<8 x i16> %a0) nounwind {
 ; X32-LABEL: knownbits_mask_shuffle_shuffle_sext:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; X32-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; X32-NEXT:    retl
@@ -115,7 +115,7 @@ define <4 x i32> @knownbits_mask_shuffle_shuffle_sext(<8 x i16> %a0) nounwind {
 define <4 x i32> @knownbits_mask_shuffle_shuffle_undef_sext(<8 x i16> %a0) nounwind {
 ; X32-LABEL: knownbits_mask_shuffle_shuffle_undef_sext:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-NEXT:    vpmovsxwd %xmm0, %xmm0
 ; X32-NEXT:    retl
@@ -136,7 +136,7 @@ define <4 x i32> @knownbits_mask_shuffle_shuffle_undef_sext(<8 x i16> %a0) nounw
 define <4 x float> @knownbits_mask_shuffle_uitofp(<4 x i32> %a0) nounwind {
 ; X32-LABEL: knownbits_mask_shuffle_uitofp:
 ; X32:       # %bb.0:
-; X32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
 ; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    retl
@@ -173,8 +173,8 @@ define <4 x float> @knownbits_mask_or_shuffle_uitofp(<4 x i32> %a0) nounwind {
 define <4 x float> @knownbits_mask_xor_shuffle_uitofp(<4 x i32> %a0) nounwind {
 ; X32-LABEL: knownbits_mask_xor_shuffle_uitofp:
 ; X32:       # %bb.0:
-; X32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-NEXT:    vxorps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X32-NEXT:    vxorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
 ; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    retl
@@ -384,8 +384,8 @@ declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
 define <8 x float> @knownbits_mask_concat_uitofp(<4 x i32> %a0, <4 x i32> %a1) nounwind {
 ; X32-LABEL: knownbits_mask_concat_uitofp:
 ; X32:       # %bb.0:
-; X32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-NEXT:    vandps {{\.LCPI.*}}, %xmm1, %xmm1
+; X32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
 ; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,0,2]
 ; X32-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[1,3,1,3]
 ; X32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
@@ -432,8 +432,8 @@ define <4 x float> @knownbits_lshr_bitcast_shuffle_uitofp(<2 x i64> %a0, <4 x i3
 define <4 x float> @knownbits_smax_smin_shuffle_uitofp(<4 x i32> %a0) {
 ; X32-LABEL: knownbits_smax_smin_shuffle_uitofp:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpminsd {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-NEXT:    vpmaxsd {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vpminsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X32-NEXT:    vpmaxsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,3,3]
 ; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    retl
@@ -457,7 +457,7 @@ declare <4 x i32> @llvm.smax.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 define <4 x float> @knownbits_umin_shuffle_uitofp(<4 x i32> %a0) {
 ; X32-LABEL: knownbits_umin_shuffle_uitofp:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpminud {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vpminud {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,3,3]
 ; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    retl
@@ -495,8 +495,8 @@ declare <4 x i32> @llvm.umax.v4i32(<4 x i32>, <4 x i32>) nounwind readnone
 define <4 x float> @knownbits_mask_umax_shuffle_uitofp(<4 x i32> %a0) {
 ; X32-LABEL: knownbits_mask_umax_shuffle_uitofp:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-NEXT:    vpmaxud {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X32-NEXT:    vpmaxud {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,3,3]
 ; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    retl
@@ -540,7 +540,7 @@ define <4 x float> @knownbits_abs_uitofp(<4 x i32> %a0) {
 ; X32-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
 ; X32-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; X32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; X32-NEXT:    vsubps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vsubps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
@@ -563,7 +563,7 @@ define <4 x float> @knownbits_abs_uitofp(<4 x i32> %a0) {
 define <4 x float> @knownbits_or_abs_uitofp(<4 x i32> %a0) {
 ; X32-LABEL: knownbits_or_abs_uitofp:
 ; X32:       # %bb.0:
-; X32-NEXT:    vpor {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-NEXT:    vpor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,0,2]
 ; X32-NEXT:    vpabsd %xmm0, %xmm0
 ; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
@@ -593,8 +593,8 @@ define <4 x float> @knownbits_and_select_shuffle_uitofp(<4 x i32> %a0, <4 x i32>
 ; X32-NEXT:    andl $-16, %esp
 ; X32-NEXT:    subl $16, %esp
 ; X32-NEXT:    vmovaps 8(%ebp), %xmm3
-; X32-NEXT:    vandps {{\.LCPI.*}}, %xmm2, %xmm2
-; X32-NEXT:    vandps {{\.LCPI.*}}, %xmm3, %xmm3
+; X32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm2, %xmm2
+; X32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm3, %xmm3
 ; X32-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vblendvps %xmm0, %xmm2, %xmm3, %xmm0
 ; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,2,2]
@@ -630,7 +630,7 @@ define <4 x float> @knownbits_lshr_and_select_shuffle_uitofp(<4 x i32> %a0, <4 x
 ; X32-NEXT:    subl $16, %esp
 ; X32-NEXT:    vmovaps 8(%ebp), %xmm3
 ; X32-NEXT:    vpsrld $5, %xmm2, %xmm2
-; X32-NEXT:    vandps {{\.LCPI.*}}, %xmm3, %xmm3
+; X32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm3, %xmm3
 ; X32-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vblendvps %xmm0, %xmm2, %xmm3, %xmm0
 ; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,2,2]
diff --git a/llvm/test/CodeGen/X86/known-signbits-vector.ll b/llvm/test/CodeGen/X86/known-signbits-vector.ll
index 18cd42c8c1de33f79e21364b673d5e74acafd356..3f22dbe316e7434f3772c70d9bd5332d4aa13d91 100644
--- a/llvm/test/CodeGen/X86/known-signbits-vector.ll
+++ b/llvm/test/CodeGen/X86/known-signbits-vector.ll
@@ -513,19 +513,17 @@ define <4 x i32> @signbits_mask_ashr_smax(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpsrad $25, %xmm0, %xmm0
 ; X86-NEXT:    vpsrad $25, %xmm1, %xmm1
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X86-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; X86-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-AVX1-LABEL: signbits_mask_ashr_smax:
 ; X64-AVX1:       # %bb.0:
 ; X64-AVX1-NEXT:    vpsrad $25, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpsrad $25, %xmm1, %xmm1
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX1-NEXT:    retq
 ;
@@ -553,19 +551,17 @@ define <4 x i32> @signbits_mask_ashr_smin(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpsrad $25, %xmm0, %xmm0
 ; X86-NEXT:    vpsrad $25, %xmm1, %xmm1
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X86-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; X86-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-AVX1-LABEL: signbits_mask_ashr_smin:
 ; X64-AVX1:       # %bb.0:
 ; X64-AVX1-NEXT:    vpsrad $25, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpsrad $25, %xmm1, %xmm1
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX1-NEXT:    retq
 ;
@@ -593,19 +589,17 @@ define <4 x i32> @signbits_mask_ashr_umax(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpsrad $25, %xmm0, %xmm0
 ; X86-NEXT:    vpsrad $25, %xmm1, %xmm1
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X86-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; X86-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-AVX1-LABEL: signbits_mask_ashr_umax:
 ; X64-AVX1:       # %bb.0:
 ; X64-AVX1-NEXT:    vpsrad $25, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpsrad $25, %xmm1, %xmm1
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX1-NEXT:    retq
 ;
@@ -633,19 +627,17 @@ define <4 x i32> @signbits_mask_ashr_umin(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpsrad $25, %xmm0, %xmm0
 ; X86-NEXT:    vpsrad $25, %xmm1, %xmm1
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X86-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; X86-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-AVX1-LABEL: signbits_mask_ashr_umin:
 ; X64-AVX1:       # %bb.0:
 ; X64-AVX1-NEXT:    vpsrad $25, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpsrad $25, %xmm1, %xmm1
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX1-NEXT:    retq
 ;
@@ -682,7 +674,7 @@ define void @cross_bb_signbits_insert_subvec(<32 x i8>* %ptr, <32 x i8> %x, <32
 ; X86-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
 ; X86-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; X86-NEXT:    vandnps %ymm1, %ymm0, %ymm1
-; X86-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; X86-NEXT:    vmovaps %ymm0, (%eax)
 ; X86-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/licm-dominance.ll b/llvm/test/CodeGen/X86/licm-dominance.ll
index f6f563c9bcb64f07b66d257bb818d226a269075f..0bcf413cdf1668d6b946d4637e684a2c9a4a6939 100644
--- a/llvm/test/CodeGen/X86/licm-dominance.ll
+++ b/llvm/test/CodeGen/X86/licm-dominance.ll
@@ -1,24 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --force-update
 ; RUN: llc -asm-verbose=true < %s | FileCheck %s
 
 ; MachineLICM should check dominance before hoisting instructions.
 ; only the load of a0 is guaranteed to execute, so only it can be hoisted.
-; CHECK: movb (%rdi), [[a0reg:%[a-z0-9]+]]
-; CHECK: ## %for.body.i
-; CHECK: testb [[a0reg]], [[a0reg]]
-; CHECK: ## in Loop:
-; CHECK: cmpb $1, ({{%[a-z0-9]+}})
-; CHECK: cmpb $2, ({{%[a-z0-9]+}})
-; CHECK: cmpb $3, ({{%[a-z0-9]+}})
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.2"
 
 define void @CMSColorWorldCreateParametricData(
-    i8* dereferenceable(1) %a0,
-    i8* dereferenceable(1) %a1,
-    i8* dereferenceable(1) %a2,
-    i8* dereferenceable(1) %a3,
-    i64 %count) nounwind uwtable optsize ssp readonly {
+; CHECK-LABEL: CMSColorWorldCreateParametricData:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movb (%rdi), %dil
+; CHECK-NEXT:  LBB0_1: ## %for.body.i
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    je LBB0_5
+; CHECK-NEXT:  ## %bb.2: ## %if.then26.i
+; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    cmpb $1, (%rsi)
+; CHECK-NEXT:    jne LBB0_4
+; CHECK-NEXT:  ## %bb.3: ## %if.else.i.i
+; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    cmpb $2, (%rdx)
+; CHECK-NEXT:    jne LBB0_5
+; CHECK-NEXT:  LBB0_4: ## %lor.lhs.false.i.i
+; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    cmpb $3, (%rcx)
+; CHECK-NEXT:    jne LBB0_6
+; CHECK-NEXT:  LBB0_5: ## %for.inc.i
+; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    incq %rax
+; CHECK-NEXT:    cmpq %r8, %rax
+; CHECK-NEXT:    jb LBB0_1
+; CHECK-NEXT:  LBB0_6: ## %if.end28.i
+; CHECK-NEXT:    retq
+  i8* dereferenceable(1) %a0,
+  i8* dereferenceable(1) %a1,
+  i8* dereferenceable(1) %a2,
+  i8* dereferenceable(1) %a3,
+  i64 %count) nounwind uwtable optsize ssp readonly {
 entry:
   br label %for.body.i
 
diff --git a/llvm/test/CodeGen/X86/limited-prec.ll b/llvm/test/CodeGen/X86/limited-prec.ll
index 97882f113b69730aaf549a6ddbe240c11bda5845..68ba8e21bb5305487cbad44dc6947ad0535867b1 100644
--- a/llvm/test/CodeGen/X86/limited-prec.ll
+++ b/llvm/test/CodeGen/X86/limited-prec.ll
@@ -8,7 +8,7 @@ define float @f1(float %x) nounwind noinline {
 ; precision6:       # %bb.0: # %entry
 ; precision6-NEXT:    subl $20, %esp
 ; precision6-NEXT:    flds {{[0-9]+}}(%esp)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fnstcw (%esp)
 ; precision6-NEXT:    movzwl (%esp), %eax
 ; precision6-NEXT:    orl $3072, %eax # imm = 0xC00
@@ -20,10 +20,10 @@ define float @f1(float %x) nounwind noinline {
 ; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision6-NEXT:    fisubl {{[0-9]+}}(%esp)
 ; precision6-NEXT:    fld %st(0)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fmulp %st, %st(1)
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fstps {{[0-9]+}}(%esp)
 ; precision6-NEXT:    shll $23, %eax
 ; precision6-NEXT:    addl {{[0-9]+}}(%esp), %eax
@@ -36,7 +36,7 @@ define float @f1(float %x) nounwind noinline {
 ; precision12:       # %bb.0: # %entry
 ; precision12-NEXT:    subl $20, %esp
 ; precision12-NEXT:    flds {{[0-9]+}}(%esp)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fnstcw (%esp)
 ; precision12-NEXT:    movzwl (%esp), %eax
 ; precision12-NEXT:    orl $3072, %eax # imm = 0xC00
@@ -48,12 +48,12 @@ define float @f1(float %x) nounwind noinline {
 ; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision12-NEXT:    fisubl {{[0-9]+}}(%esp)
 ; precision12-NEXT:    fld %st(0)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmul %st(1), %st
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmulp %st, %st(1)
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fstps {{[0-9]+}}(%esp)
 ; precision12-NEXT:    shll $23, %eax
 ; precision12-NEXT:    addl {{[0-9]+}}(%esp), %eax
@@ -66,7 +66,7 @@ define float @f1(float %x) nounwind noinline {
 ; precision18:       # %bb.0: # %entry
 ; precision18-NEXT:    subl $20, %esp
 ; precision18-NEXT:    flds {{[0-9]+}}(%esp)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fnstcw (%esp)
 ; precision18-NEXT:    movzwl (%esp), %eax
 ; precision18-NEXT:    orl $3072, %eax # imm = 0xC00
@@ -78,16 +78,16 @@ define float @f1(float %x) nounwind noinline {
 ; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision18-NEXT:    fisubl {{[0-9]+}}(%esp)
 ; precision18-NEXT:    fld %st(0)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmulp %st, %st(1)
 ; precision18-NEXT:    fld1
 ; precision18-NEXT:    faddp %st, %st(1)
@@ -122,10 +122,10 @@ define float @f2(float %x) nounwind noinline {
 ; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision6-NEXT:    fisubl {{[0-9]+}}(%esp)
 ; precision6-NEXT:    fld %st(0)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fmulp %st, %st(1)
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fstps {{[0-9]+}}(%esp)
 ; precision6-NEXT:    shll $23, %eax
 ; precision6-NEXT:    addl {{[0-9]+}}(%esp), %eax
@@ -149,12 +149,12 @@ define float @f2(float %x) nounwind noinline {
 ; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision12-NEXT:    fisubl {{[0-9]+}}(%esp)
 ; precision12-NEXT:    fld %st(0)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmul %st(1), %st
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmulp %st, %st(1)
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fstps {{[0-9]+}}(%esp)
 ; precision12-NEXT:    shll $23, %eax
 ; precision12-NEXT:    addl {{[0-9]+}}(%esp), %eax
@@ -178,16 +178,16 @@ define float @f2(float %x) nounwind noinline {
 ; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision18-NEXT:    fisubl {{[0-9]+}}(%esp)
 ; precision18-NEXT:    fld %st(0)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmulp %st, %st(1)
 ; precision18-NEXT:    fld1
 ; precision18-NEXT:    faddp %st, %st(1)
@@ -211,7 +211,7 @@ define float @f3(float %x) nounwind noinline {
 ; precision6:       # %bb.0: # %entry
 ; precision6-NEXT:    subl $20, %esp
 ; precision6-NEXT:    flds {{[0-9]+}}(%esp)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fnstcw (%esp)
 ; precision6-NEXT:    movzwl (%esp), %eax
 ; precision6-NEXT:    orl $3072, %eax # imm = 0xC00
@@ -223,10 +223,10 @@ define float @f3(float %x) nounwind noinline {
 ; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision6-NEXT:    fisubl {{[0-9]+}}(%esp)
 ; precision6-NEXT:    fld %st(0)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fmulp %st, %st(1)
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fstps {{[0-9]+}}(%esp)
 ; precision6-NEXT:    shll $23, %eax
 ; precision6-NEXT:    addl {{[0-9]+}}(%esp), %eax
@@ -239,7 +239,7 @@ define float @f3(float %x) nounwind noinline {
 ; precision12:       # %bb.0: # %entry
 ; precision12-NEXT:    subl $20, %esp
 ; precision12-NEXT:    flds {{[0-9]+}}(%esp)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fnstcw (%esp)
 ; precision12-NEXT:    movzwl (%esp), %eax
 ; precision12-NEXT:    orl $3072, %eax # imm = 0xC00
@@ -251,12 +251,12 @@ define float @f3(float %x) nounwind noinline {
 ; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision12-NEXT:    fisubl {{[0-9]+}}(%esp)
 ; precision12-NEXT:    fld %st(0)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmul %st(1), %st
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmulp %st, %st(1)
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fstps {{[0-9]+}}(%esp)
 ; precision12-NEXT:    shll $23, %eax
 ; precision12-NEXT:    addl {{[0-9]+}}(%esp), %eax
@@ -269,7 +269,7 @@ define float @f3(float %x) nounwind noinline {
 ; precision18:       # %bb.0: # %entry
 ; precision18-NEXT:    subl $20, %esp
 ; precision18-NEXT:    flds {{[0-9]+}}(%esp)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fnstcw (%esp)
 ; precision18-NEXT:    movzwl (%esp), %eax
 ; precision18-NEXT:    orl $3072, %eax # imm = 0xC00
@@ -281,16 +281,16 @@ define float @f3(float %x) nounwind noinline {
 ; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision18-NEXT:    fisubl {{[0-9]+}}(%esp)
 ; precision18-NEXT:    fld %st(0)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmulp %st, %st(1)
 ; precision18-NEXT:    fld1
 ; precision18-NEXT:    faddp %st, %st(1)
@@ -324,12 +324,12 @@ define float @f4(float %x) nounwind noinline {
 ; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision6-NEXT:    flds (%esp)
 ; precision6-NEXT:    fld %st(0)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fmulp %st, %st(1)
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fildl {{[0-9]+}}(%esp)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    faddp %st, %st(1)
 ; precision6-NEXT:    addl $8, %esp
 ; precision6-NEXT:    retl
@@ -348,16 +348,16 @@ define float @f4(float %x) nounwind noinline {
 ; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision12-NEXT:    flds (%esp)
 ; precision12-NEXT:    fld %st(0)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmul %st(1), %st
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmul %st(1), %st
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmulp %st, %st(1)
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fildl {{[0-9]+}}(%esp)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    faddp %st, %st(1)
 ; precision12-NEXT:    addl $8, %esp
 ; precision12-NEXT:    retl
@@ -376,20 +376,20 @@ define float @f4(float %x) nounwind noinline {
 ; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision18-NEXT:    flds (%esp)
 ; precision18-NEXT:    fld %st(0)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmulp %st, %st(1)
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fildl {{[0-9]+}}(%esp)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    faddp %st, %st(1)
 ; precision18-NEXT:    addl $8, %esp
 ; precision18-NEXT:    retl
@@ -416,10 +416,10 @@ define float @f5(float %x) nounwind noinline {
 ; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision6-NEXT:    flds (%esp)
 ; precision6-NEXT:    fld %st(0)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fmulp %st, %st(1)
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fiaddl {{[0-9]+}}(%esp)
 ; precision6-NEXT:    addl $8, %esp
 ; precision6-NEXT:    retl
@@ -438,14 +438,14 @@ define float @f5(float %x) nounwind noinline {
 ; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision12-NEXT:    flds (%esp)
 ; precision12-NEXT:    fld %st(0)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmul %st(1), %st
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmul %st(1), %st
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmulp %st, %st(1)
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fiaddl {{[0-9]+}}(%esp)
 ; precision12-NEXT:    addl $8, %esp
 ; precision12-NEXT:    retl
@@ -464,18 +464,18 @@ define float @f5(float %x) nounwind noinline {
 ; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision18-NEXT:    flds (%esp)
 ; precision18-NEXT:    fld %st(0)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmulp %st, %st(1)
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fiaddl {{[0-9]+}}(%esp)
 ; precision18-NEXT:    addl $8, %esp
 ; precision18-NEXT:    retl
@@ -502,12 +502,12 @@ define float @f6(float %x) nounwind noinline {
 ; precision6-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision6-NEXT:    flds (%esp)
 ; precision6-NEXT:    fld %st(0)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fmulp %st, %st(1)
-; precision6-NEXT:    fadds {{\.LCPI.*}}
+; precision6-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    fildl {{[0-9]+}}(%esp)
-; precision6-NEXT:    fmuls {{\.LCPI.*}}
+; precision6-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision6-NEXT:    faddp %st, %st(1)
 ; precision6-NEXT:    addl $8, %esp
 ; precision6-NEXT:    retl
@@ -526,14 +526,14 @@ define float @f6(float %x) nounwind noinline {
 ; precision12-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision12-NEXT:    flds (%esp)
 ; precision12-NEXT:    fld %st(0)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmul %st(1), %st
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fmulp %st, %st(1)
-; precision12-NEXT:    fadds {{\.LCPI.*}}
+; precision12-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    fildl {{[0-9]+}}(%esp)
-; precision12-NEXT:    fmuls {{\.LCPI.*}}
+; precision12-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision12-NEXT:    faddp %st, %st(1)
 ; precision12-NEXT:    addl $8, %esp
 ; precision12-NEXT:    retl
@@ -552,18 +552,18 @@ define float @f6(float %x) nounwind noinline {
 ; precision18-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; precision18-NEXT:    flds (%esp)
 ; precision18-NEXT:    fld %st(0)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmul %st(1), %st
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fmulp %st, %st(1)
-; precision18-NEXT:    fadds {{\.LCPI.*}}
+; precision18-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    fildl {{[0-9]+}}(%esp)
-; precision18-NEXT:    fmuls {{\.LCPI.*}}
+; precision18-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; precision18-NEXT:    faddp %st, %st(1)
 ; precision18-NEXT:    addl $8, %esp
 ; precision18-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index a024a04fa37f37f912d70d5828d0bb2ae6da4692..c0ff6a79ef18a76f3b7c671256d7da477f2a9056 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -3046,3 +3046,104 @@ middle.block:
   %11 = extractelement <8 x i32> %bin.rdx34, i32 0
   ret i32 %11
 }
+
+; PR49716 - https://llvm.org/PR49716
+
+define <4 x i32> @input_size_mismatch(<16 x i16> %x, <16 x i16>* %p) {
+; SSE2-LABEL: input_size_mismatch:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa (%rdi), %xmm1
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pmulhw %xmm3, %xmm4
+; SSE2-NEXT:    pmullw %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pmulhw %xmm1, %xmm3
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: input_size_mismatch:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX-NEXT:    vpshufb %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; AVX-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
+; AVX-NEXT:    vpmovsxwd %xmm2, %xmm2
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpmovsxwd %xmm3, %xmm2
+; AVX-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %y = load <16 x i16>, <16 x i16>* %p, align 32
+  %x0 = shufflevector <16 x i16> %x, <16 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %x1 = shufflevector <16 x i16> %x, <16 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %y0 = shufflevector <16 x i16> %y, <16 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %y1 = shufflevector <16 x i16> %y, <16 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %sx0 = sext <4 x i16> %x0 to <4 x i32>
+  %sx1 = sext <4 x i16> %x1 to <4 x i32>
+  %sy0 = sext <4 x i16> %y0 to <4 x i32>
+  %sy1 = sext <4 x i16> %y1 to <4 x i32>
+  %m0 = mul <4 x i32> %sx0, %sy0
+  %m1 = mul <4 x i32> %sx1, %sy1
+  %r = add <4 x i32> %m0, %m1
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @output_size_mismatch(<16 x i16> %x, <16 x i16> %y) {
+; SSE2-LABEL: output_size_mismatch:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pmaddwd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: output_size_mismatch:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX-NEXT:    vpshufb %xmm2, %xmm0, %xmm3
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm4 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15]
+; AVX-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm2, %xmm1, %xmm2
+; AVX-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
+; AVX-NEXT:    vpmovsxwd %xmm3, %xmm3
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vpmovsxwd %xmm2, %xmm2
+; AVX-NEXT:    vpmulld %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x0 = shufflevector <16 x i16> %x, <16 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %x1 = shufflevector <16 x i16> %x, <16 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %y0 = shufflevector <16 x i16> %y, <16 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %y1 = shufflevector <16 x i16> %y, <16 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %sx0 = sext <4 x i16> %x0 to <4 x i32>
+  %sx1 = sext <4 x i16> %x1 to <4 x i32>
+  %sy0 = sext <4 x i16> %y0 to <4 x i32>
+  %sy1 = sext <4 x i16> %y1 to <4 x i32>
+  %m0 = mul <4 x i32> %sx0, %sy0
+  %m1 = mul <4 x i32> %sx1, %sy1
+  %r = add <4 x i32> %m0, %m1
+  ret <4 x i32> %r
+}
diff --git a/llvm/test/CodeGen/X86/masked_gather.ll b/llvm/test/CodeGen/X86/masked_gather.ll
index bf4baf6d4bd64ff797a0f42ea086dde55a5fefe3..3bb06d7c731f351cfa5782331678101692478ab9 100644
--- a/llvm/test/CodeGen/X86/masked_gather.ll
+++ b/llvm/test/CodeGen/X86/masked_gather.ll
@@ -1025,11 +1025,6 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movdqa %xmm1, %xmm3
 ; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    movl $c, %eax
-; SSE-NEXT:    movq %rax, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,1,0,1]
-; SSE-NEXT:    movdqa {{.*#+}} xmm5 = [12,12]
-; SSE-NEXT:    paddq %xmm4, %xmm5
 ; SSE-NEXT:    pxor %xmm0, %xmm0
 ; SSE-NEXT:    pcmpeqd %xmm0, %xmm1
 ; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
@@ -1039,7 +1034,6 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE-NEXT:    testb $1, %al
 ; SSE-NEXT:    je .LBB4_1
 ; SSE-NEXT:  # %bb.2: # %cond.load
-; SSE-NEXT:    movq %xmm5, %rcx
 ; SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    jne .LBB4_4
@@ -1049,8 +1043,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    je .LBB4_5
 ; SSE-NEXT:  .LBB4_4: # %cond.load1
-; SSE-NEXT:    pextrq $1, %xmm5, %rcx
-; SSE-NEXT:    pinsrd $1, (%rcx), %xmm0
+; SSE-NEXT:    pinsrd $1, c+{{.*}}(%rip), %xmm0
 ; SSE-NEXT:  .LBB4_5: # %else2
 ; SSE-NEXT:    testb $4, %al
 ; SSE-NEXT:    jne .LBB4_6
@@ -1061,19 +1054,16 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE-NEXT:    testb $16, %al
 ; SSE-NEXT:    je .LBB4_10
 ; SSE-NEXT:  .LBB4_11: # %cond.load10
-; SSE-NEXT:    movq %xmm5, %rcx
-; SSE-NEXT:    pinsrd $0, (%rcx), %xmm1
+; SSE-NEXT:    pinsrd $0, c+{{.*}}(%rip), %xmm1
 ; SSE-NEXT:    testb $32, %al
 ; SSE-NEXT:    jne .LBB4_13
 ; SSE-NEXT:    jmp .LBB4_14
 ; SSE-NEXT:  .LBB4_6: # %cond.load4
-; SSE-NEXT:    movq %xmm5, %rcx
-; SSE-NEXT:    pinsrd $2, (%rcx), %xmm0
+; SSE-NEXT:    pinsrd $2, c+{{.*}}(%rip), %xmm0
 ; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB4_9
 ; SSE-NEXT:  .LBB4_8: # %cond.load7
-; SSE-NEXT:    pextrq $1, %xmm5, %rcx
-; SSE-NEXT:    pinsrd $3, (%rcx), %xmm0
+; SSE-NEXT:    pinsrd $3, c+{{.*}}(%rip), %xmm0
 ; SSE-NEXT:    testb $16, %al
 ; SSE-NEXT:    jne .LBB4_11
 ; SSE-NEXT:  .LBB4_10:
@@ -1081,8 +1071,7 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE-NEXT:    testb $32, %al
 ; SSE-NEXT:    je .LBB4_14
 ; SSE-NEXT:  .LBB4_13: # %cond.load13
-; SSE-NEXT:    pextrq $1, %xmm5, %rcx
-; SSE-NEXT:    pinsrd $1, (%rcx), %xmm1
+; SSE-NEXT:    pinsrd $1, c+{{.*}}(%rip), %xmm1
 ; SSE-NEXT:  .LBB4_14: # %else14
 ; SSE-NEXT:    testb $64, %al
 ; SSE-NEXT:    jne .LBB4_15
@@ -1090,97 +1079,85 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE-NEXT:    testb $-128, %al
 ; SSE-NEXT:    je .LBB4_18
 ; SSE-NEXT:  .LBB4_17: # %cond.load19
-; SSE-NEXT:    pextrq $1, %xmm5, %rax
-; SSE-NEXT:    pinsrd $3, (%rax), %xmm1
+; SSE-NEXT:    pinsrd $3, c+{{.*}}(%rip), %xmm1
 ; SSE-NEXT:  .LBB4_18: # %else20
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    paddq {{.*}}(%rip), %xmm4
-; SSE-NEXT:    movdqa %xmm2, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE-NEXT:    packssdw %xmm5, %xmm6
-; SSE-NEXT:    packsswb %xmm6, %xmm6
-; SSE-NEXT:    pmovmskb %xmm6, %eax
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    movdqa %xmm2, %xmm5
+; SSE-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE-NEXT:    packssdw %xmm4, %xmm5
+; SSE-NEXT:    packsswb %xmm5, %xmm5
+; SSE-NEXT:    pmovmskb %xmm5, %eax
 ; SSE-NEXT:    testb $1, %al
 ; SSE-NEXT:    je .LBB4_19
 ; SSE-NEXT:  # %bb.20: # %cond.load23
-; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    jne .LBB4_22
 ; SSE-NEXT:    jmp .LBB4_23
 ; SSE-NEXT:  .LBB4_15: # %cond.load16
-; SSE-NEXT:    movq %xmm5, %rcx
-; SSE-NEXT:    pinsrd $2, (%rcx), %xmm1
+; SSE-NEXT:    pinsrd $2, c+{{.*}}(%rip), %xmm1
 ; SSE-NEXT:    testb $-128, %al
 ; SSE-NEXT:    jne .LBB4_17
 ; SSE-NEXT:    jmp .LBB4_18
 ; SSE-NEXT:  .LBB4_19:
-; SSE-NEXT:    # implicit-def: $xmm5
+; SSE-NEXT:    # implicit-def: $xmm4
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    je .LBB4_23
-; SSE-NEXT:  .LBB4_22: # %cond.load29
-; SSE-NEXT:    pextrq $1, %xmm4, %rcx
-; SSE-NEXT:    pinsrd $1, (%rcx), %xmm5
-; SSE-NEXT:  .LBB4_23: # %else33
+; SSE-NEXT:  .LBB4_22: # %cond.load28
+; SSE-NEXT:    pinsrd $1, c+{{.*}}(%rip), %xmm4
+; SSE-NEXT:  .LBB4_23: # %else31
 ; SSE-NEXT:    testb $4, %al
 ; SSE-NEXT:    jne .LBB4_24
-; SSE-NEXT:  # %bb.25: # %else39
+; SSE-NEXT:  # %bb.25: # %else36
 ; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    jne .LBB4_26
-; SSE-NEXT:  .LBB4_27: # %else45
+; SSE-NEXT:  .LBB4_27: # %else41
 ; SSE-NEXT:    testb $16, %al
 ; SSE-NEXT:    je .LBB4_28
-; SSE-NEXT:  .LBB4_29: # %cond.load47
-; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    pinsrd $0, (%rcx), %xmm6
+; SSE-NEXT:  .LBB4_29: # %cond.load43
+; SSE-NEXT:    pinsrd $0, c+{{.*}}(%rip), %xmm5
 ; SSE-NEXT:    testb $32, %al
 ; SSE-NEXT:    jne .LBB4_31
 ; SSE-NEXT:    jmp .LBB4_32
-; SSE-NEXT:  .LBB4_24: # %cond.load35
-; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    pinsrd $2, (%rcx), %xmm5
+; SSE-NEXT:  .LBB4_24: # %cond.load33
+; SSE-NEXT:    pinsrd $2, c+{{.*}}(%rip), %xmm4
 ; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB4_27
-; SSE-NEXT:  .LBB4_26: # %cond.load41
-; SSE-NEXT:    pextrq $1, %xmm4, %rcx
-; SSE-NEXT:    pinsrd $3, (%rcx), %xmm5
+; SSE-NEXT:  .LBB4_26: # %cond.load38
+; SSE-NEXT:    pinsrd $3, c+{{.*}}(%rip), %xmm4
 ; SSE-NEXT:    testb $16, %al
 ; SSE-NEXT:    jne .LBB4_29
 ; SSE-NEXT:  .LBB4_28:
-; SSE-NEXT:    # implicit-def: $xmm6
+; SSE-NEXT:    # implicit-def: $xmm5
 ; SSE-NEXT:    testb $32, %al
 ; SSE-NEXT:    je .LBB4_32
-; SSE-NEXT:  .LBB4_31: # %cond.load53
-; SSE-NEXT:    pextrq $1, %xmm4, %rcx
-; SSE-NEXT:    pinsrd $1, (%rcx), %xmm6
-; SSE-NEXT:  .LBB4_32: # %else57
+; SSE-NEXT:  .LBB4_31: # %cond.load48
+; SSE-NEXT:    pinsrd $1, c+{{.*}}(%rip), %xmm5
+; SSE-NEXT:  .LBB4_32: # %else51
 ; SSE-NEXT:    testb $64, %al
 ; SSE-NEXT:    jne .LBB4_33
-; SSE-NEXT:  # %bb.34: # %else63
+; SSE-NEXT:  # %bb.34: # %else56
 ; SSE-NEXT:    testb $-128, %al
 ; SSE-NEXT:    je .LBB4_36
-; SSE-NEXT:  .LBB4_35: # %cond.load65
-; SSE-NEXT:    pextrq $1, %xmm4, %rax
-; SSE-NEXT:    pinsrd $3, (%rax), %xmm6
-; SSE-NEXT:  .LBB4_36: # %else69
-; SSE-NEXT:    pxor %xmm7, %xmm7
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm3
+; SSE-NEXT:  .LBB4_35: # %cond.load58
+; SSE-NEXT:    pinsrd $3, c+{{.*}}(%rip), %xmm5
+; SSE-NEXT:  .LBB4_36: # %else61
+; SSE-NEXT:    pxor %xmm6, %xmm6
+; SSE-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm6, %xmm3
 ; SSE-NEXT:    packssdw %xmm3, %xmm2
 ; SSE-NEXT:    packsswb %xmm2, %xmm2
 ; SSE-NEXT:    pmovmskb %xmm2, %eax
 ; SSE-NEXT:    testb $1, %al
 ; SSE-NEXT:    je .LBB4_37
-; SSE-NEXT:  # %bb.38: # %cond.load72
-; SSE-NEXT:    movq %xmm4, %rcx
+; SSE-NEXT:  # %bb.38: # %cond.load64
 ; SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    jne .LBB4_40
 ; SSE-NEXT:    jmp .LBB4_41
-; SSE-NEXT:  .LBB4_33: # %cond.load59
-; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    pinsrd $2, (%rcx), %xmm6
+; SSE-NEXT:  .LBB4_33: # %cond.load53
+; SSE-NEXT:    pinsrd $2, c+{{.*}}(%rip), %xmm5
 ; SSE-NEXT:    testb $-128, %al
 ; SSE-NEXT:    jne .LBB4_35
 ; SSE-NEXT:    jmp .LBB4_36
@@ -1188,322 +1165,293 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; SSE-NEXT:    # implicit-def: $xmm2
 ; SSE-NEXT:    testb $2, %al
 ; SSE-NEXT:    je .LBB4_41
-; SSE-NEXT:  .LBB4_40: # %cond.load78
-; SSE-NEXT:    pextrq $1, %xmm4, %rcx
-; SSE-NEXT:    pinsrd $1, (%rcx), %xmm2
-; SSE-NEXT:  .LBB4_41: # %else82
+; SSE-NEXT:  .LBB4_40: # %cond.load69
+; SSE-NEXT:    pinsrd $1, c+{{.*}}(%rip), %xmm2
+; SSE-NEXT:  .LBB4_41: # %else72
 ; SSE-NEXT:    testb $4, %al
 ; SSE-NEXT:    jne .LBB4_42
-; SSE-NEXT:  # %bb.43: # %else88
+; SSE-NEXT:  # %bb.43: # %else77
 ; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    jne .LBB4_44
-; SSE-NEXT:  .LBB4_45: # %else94
+; SSE-NEXT:  .LBB4_45: # %else82
 ; SSE-NEXT:    testb $16, %al
 ; SSE-NEXT:    je .LBB4_46
-; SSE-NEXT:  .LBB4_47: # %cond.load96
-; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    pinsrd $0, (%rcx), %xmm3
+; SSE-NEXT:  .LBB4_47: # %cond.load84
+; SSE-NEXT:    pinsrd $0, c+{{.*}}(%rip), %xmm3
 ; SSE-NEXT:    testb $32, %al
 ; SSE-NEXT:    jne .LBB4_49
 ; SSE-NEXT:    jmp .LBB4_50
-; SSE-NEXT:  .LBB4_42: # %cond.load84
-; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    pinsrd $2, (%rcx), %xmm2
+; SSE-NEXT:  .LBB4_42: # %cond.load74
+; SSE-NEXT:    pinsrd $2, c+{{.*}}(%rip), %xmm2
 ; SSE-NEXT:    testb $8, %al
 ; SSE-NEXT:    je .LBB4_45
-; SSE-NEXT:  .LBB4_44: # %cond.load90
-; SSE-NEXT:    pextrq $1, %xmm4, %rcx
-; SSE-NEXT:    pinsrd $3, (%rcx), %xmm2
+; SSE-NEXT:  .LBB4_44: # %cond.load79
+; SSE-NEXT:    pinsrd $3, c+{{.*}}(%rip), %xmm2
 ; SSE-NEXT:    testb $16, %al
 ; SSE-NEXT:    jne .LBB4_47
 ; SSE-NEXT:  .LBB4_46:
 ; SSE-NEXT:    # implicit-def: $xmm3
 ; SSE-NEXT:    testb $32, %al
 ; SSE-NEXT:    je .LBB4_50
-; SSE-NEXT:  .LBB4_49: # %cond.load102
-; SSE-NEXT:    pextrq $1, %xmm4, %rcx
-; SSE-NEXT:    pinsrd $1, (%rcx), %xmm3
-; SSE-NEXT:  .LBB4_50: # %else106
+; SSE-NEXT:  .LBB4_49: # %cond.load89
+; SSE-NEXT:    pinsrd $1, c+{{.*}}(%rip), %xmm3
+; SSE-NEXT:  .LBB4_50: # %else92
 ; SSE-NEXT:    testb $64, %al
 ; SSE-NEXT:    je .LBB4_52
-; SSE-NEXT:  # %bb.51: # %cond.load108
-; SSE-NEXT:    movq %xmm4, %rcx
-; SSE-NEXT:    pinsrd $2, (%rcx), %xmm3
-; SSE-NEXT:  .LBB4_52: # %else112
-; SSE-NEXT:    paddd %xmm5, %xmm0
-; SSE-NEXT:    paddd %xmm6, %xmm1
+; SSE-NEXT:  # %bb.51: # %cond.load94
+; SSE-NEXT:    pinsrd $2, c+{{.*}}(%rip), %xmm3
+; SSE-NEXT:  .LBB4_52: # %else97
+; SSE-NEXT:    paddd %xmm4, %xmm0
+; SSE-NEXT:    paddd %xmm5, %xmm1
 ; SSE-NEXT:    testb $-128, %al
 ; SSE-NEXT:    je .LBB4_54
-; SSE-NEXT:  # %bb.53: # %cond.load114
-; SSE-NEXT:    pextrq $1, %xmm4, %rax
-; SSE-NEXT:    pinsrd $3, (%rax), %xmm3
-; SSE-NEXT:  .LBB4_54: # %else118
+; SSE-NEXT:  # %bb.53: # %cond.load99
+; SSE-NEXT:    pinsrd $3, c+{{.*}}(%rip), %xmm3
+; SSE-NEXT:  .LBB4_54: # %else102
 ; SSE-NEXT:    paddd %xmm3, %xmm1
 ; SSE-NEXT:    paddd %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: gather_v8i32_v8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    movl $c, %ecx
-; AVX1-NEXT:    vmovq %rcx, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vpaddq {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm3
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm9
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm9, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm3
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX1-NEXT:    vmovmskps %ymm1, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    # implicit-def: $ymm1
-; AVX1-NEXT:    je .LBB4_2
-; AVX1-NEXT:  # %bb.1: # %cond.load
-; AVX1-NEXT:    vmovq %xmm3, %rdx
-; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-NEXT:  .LBB4_2: # %else
+; AVX1-NEXT:    jne .LBB4_1
+; AVX1-NEXT:  # %bb.2: # %else
 ; AVX1-NEXT:    testb $2, %al
-; AVX1-NEXT:    je .LBB4_4
-; AVX1-NEXT:  # %bb.3: # %cond.load1
-; AVX1-NEXT:    vpextrq $1, %xmm3, %rdx
-; AVX1-NEXT:    vpinsrd $1, (%rdx), %xmm1, %xmm4
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    jne .LBB4_3
 ; AVX1-NEXT:  .LBB4_4: # %else2
 ; AVX1-NEXT:    testb $4, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX1-NEXT:    je .LBB4_6
-; AVX1-NEXT:  # %bb.5: # %cond.load4
-; AVX1-NEXT:    vmovq %xmm4, %rdx
-; AVX1-NEXT:    vpinsrd $2, (%rdx), %xmm1, %xmm5
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    jne .LBB4_5
 ; AVX1-NEXT:  .LBB4_6: # %else5
-; AVX1-NEXT:    vmovq %rcx, %xmm5
 ; AVX1-NEXT:    testb $8, %al
-; AVX1-NEXT:    je .LBB4_8
-; AVX1-NEXT:  # %bb.7: # %cond.load7
-; AVX1-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX1-NEXT:    vpinsrd $3, (%rcx), %xmm1, %xmm6
-; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    jne .LBB4_7
 ; AVX1-NEXT:  .LBB4_8: # %else8
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = xmm5[0,0]
 ; AVX1-NEXT:    testb $16, %al
-; AVX1-NEXT:    je .LBB4_10
-; AVX1-NEXT:  # %bb.9: # %cond.load10
-; AVX1-NEXT:    vmovq %xmm3, %rcx
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX1-NEXT:    vpinsrd $0, (%rcx), %xmm6, %xmm6
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
+; AVX1-NEXT:    jne .LBB4_9
 ; AVX1-NEXT:  .LBB4_10: # %else11
 ; AVX1-NEXT:    testb $32, %al
-; AVX1-NEXT:    je .LBB4_12
-; AVX1-NEXT:  # %bb.11: # %cond.load13
-; AVX1-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    jne .LBB4_11
 ; AVX1-NEXT:  .LBB4_12: # %else14
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm5, %ymm5
 ; AVX1-NEXT:    testb $64, %al
-; AVX1-NEXT:    je .LBB4_14
-; AVX1-NEXT:  # %bb.13: # %cond.load16
-; AVX1-NEXT:    vmovq %xmm4, %rcx
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    jne .LBB4_13
 ; AVX1-NEXT:  .LBB4_14: # %else17
 ; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB4_16
-; AVX1-NEXT:  # %bb.15: # %cond.load19
-; AVX1-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX1-NEXT:  .LBB4_15: # %cond.load19
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpinsrd $3, (%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX1-NEXT:  .LBB4_16: # %else20
-; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [28,28]
-; AVX1-NEXT:    vpaddq %xmm3, %xmm10, %xmm3
-; AVX1-NEXT:    vpaddq %xmm5, %xmm10, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm9, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
-; AVX1-NEXT:    vmovmskps %ymm2, %eax
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vmovmskps %ymm3, %eax
 ; AVX1-NEXT:    testb $1, %al
-; AVX1-NEXT:    # implicit-def: $ymm4
-; AVX1-NEXT:    je .LBB4_18
-; AVX1-NEXT:  # %bb.17: # %cond.load23
-; AVX1-NEXT:    vmovq %xmm7, %rcx
-; AVX1-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX1-NEXT:  .LBB4_18: # %else27
+; AVX1-NEXT:    # implicit-def: $ymm3
+; AVX1-NEXT:    jne .LBB4_17
+; AVX1-NEXT:  # %bb.18: # %else26
 ; AVX1-NEXT:    testb $2, %al
-; AVX1-NEXT:    je .LBB4_20
-; AVX1-NEXT:  # %bb.19: # %cond.load29
-; AVX1-NEXT:    vpextrq $1, %xmm7, %rcx
-; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm4, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX1-NEXT:  .LBB4_20: # %else33
-; AVX1-NEXT:    vpaddq %xmm5, %xmm10, %xmm8
+; AVX1-NEXT:    jne .LBB4_19
+; AVX1-NEXT:  .LBB4_20: # %else31
 ; AVX1-NEXT:    testb $4, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm7
-; AVX1-NEXT:    je .LBB4_22
-; AVX1-NEXT:  # %bb.21: # %cond.load35
-; AVX1-NEXT:    vmovq %xmm7, %rcx
-; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm4, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX1-NEXT:  .LBB4_22: # %else39
+; AVX1-NEXT:    jne .LBB4_21
+; AVX1-NEXT:  .LBB4_22: # %else36
 ; AVX1-NEXT:    testb $8, %al
-; AVX1-NEXT:    je .LBB4_24
-; AVX1-NEXT:  # %bb.23: # %cond.load41
-; AVX1-NEXT:    vpextrq $1, %xmm7, %rcx
-; AVX1-NEXT:    vpinsrd $3, (%rcx), %xmm4, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX1-NEXT:  .LBB4_24: # %else45
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm8, %ymm7
+; AVX1-NEXT:    jne .LBB4_23
+; AVX1-NEXT:  .LBB4_24: # %else41
 ; AVX1-NEXT:    testb $16, %al
-; AVX1-NEXT:    je .LBB4_26
-; AVX1-NEXT:  # %bb.25: # %cond.load47
-; AVX1-NEXT:    vmovq %xmm7, %rcx
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
-; AVX1-NEXT:    vpinsrd $0, (%rcx), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm4
-; AVX1-NEXT:  .LBB4_26: # %else51
+; AVX1-NEXT:    jne .LBB4_25
+; AVX1-NEXT:  .LBB4_26: # %else46
 ; AVX1-NEXT:    testb $32, %al
-; AVX1-NEXT:    je .LBB4_28
-; AVX1-NEXT:  # %bb.27: # %cond.load53
-; AVX1-NEXT:    vpextrq $1, %xmm7, %rcx
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
-; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm4
-; AVX1-NEXT:  .LBB4_28: # %else57
+; AVX1-NEXT:    jne .LBB4_27
+; AVX1-NEXT:  .LBB4_28: # %else51
 ; AVX1-NEXT:    testb $64, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm7
-; AVX1-NEXT:    je .LBB4_30
-; AVX1-NEXT:  # %bb.29: # %cond.load59
-; AVX1-NEXT:    vmovq %xmm7, %rcx
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
-; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm4
-; AVX1-NEXT:  .LBB4_30: # %else63
+; AVX1-NEXT:    jne .LBB4_29
+; AVX1-NEXT:  .LBB4_30: # %else56
 ; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB4_32
-; AVX1-NEXT:  # %bb.31: # %cond.load65
-; AVX1-NEXT:    vpextrq $1, %xmm7, %rax
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
-; AVX1-NEXT:    vpinsrd $3, (%rax), %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm4
-; AVX1-NEXT:  .LBB4_32: # %else69
-; AVX1-NEXT:    vpaddq %xmm5, %xmm10, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm7
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm9, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:  .LBB4_31: # %cond.load58
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:  .LBB4_32: # %else61
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vmovmskps %ymm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    # implicit-def: $ymm0
-; AVX1-NEXT:    je .LBB4_34
-; AVX1-NEXT:  # %bb.33: # %cond.load72
-; AVX1-NEXT:    vmovq %xmm7, %rcx
-; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-NEXT:  .LBB4_34: # %else76
+; AVX1-NEXT:    jne .LBB4_33
+; AVX1-NEXT:  # %bb.34: # %else67
 ; AVX1-NEXT:    testb $2, %al
-; AVX1-NEXT:    je .LBB4_36
-; AVX1-NEXT:  # %bb.35: # %cond.load78
-; AVX1-NEXT:    vpextrq $1, %xmm7, %rcx
-; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm0, %xmm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX1-NEXT:  .LBB4_36: # %else82
-; AVX1-NEXT:    vpaddq %xmm5, %xmm10, %xmm2
+; AVX1-NEXT:    jne .LBB4_35
+; AVX1-NEXT:  .LBB4_36: # %else72
 ; AVX1-NEXT:    testb $4, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm5
-; AVX1-NEXT:    je .LBB4_38
-; AVX1-NEXT:  # %bb.37: # %cond.load84
-; AVX1-NEXT:    vmovq %xmm5, %rcx
-; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm0, %xmm6
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
-; AVX1-NEXT:  .LBB4_38: # %else88
+; AVX1-NEXT:    jne .LBB4_37
+; AVX1-NEXT:  .LBB4_38: # %else77
 ; AVX1-NEXT:    testb $8, %al
-; AVX1-NEXT:    je .LBB4_40
-; AVX1-NEXT:  # %bb.39: # %cond.load90
-; AVX1-NEXT:    vpextrq $1, %xmm5, %rcx
-; AVX1-NEXT:    vpinsrd $3, (%rcx), %xmm0, %xmm5
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
-; AVX1-NEXT:  .LBB4_40: # %else94
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    jne .LBB4_39
+; AVX1-NEXT:  .LBB4_40: # %else82
 ; AVX1-NEXT:    testb $16, %al
 ; AVX1-NEXT:    je .LBB4_42
-; AVX1-NEXT:  # %bb.41: # %cond.load96
-; AVX1-NEXT:    vmovq %xmm2, %rcx
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpinsrd $0, (%rcx), %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:  .LBB4_42: # %else100
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:  .LBB4_41: # %cond.load84
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpinsrd $0, c+{{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:  .LBB4_42: # %else87
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    testb $32, %al
 ; AVX1-NEXT:    je .LBB4_44
-; AVX1-NEXT:  # %bb.43: # %cond.load102
-; AVX1-NEXT:    vpextrq $1, %xmm2, %rcx
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpinsrd $1, (%rcx), %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; AVX1-NEXT:  .LBB4_44: # %else106
-; AVX1-NEXT:    vpaddd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:  # %bb.43: # %cond.load89
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:  .LBB4_44: # %else92
+; AVX1-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
 ; AVX1-NEXT:    testb $64, %al
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
 ; AVX1-NEXT:    je .LBB4_46
-; AVX1-NEXT:  # %bb.45: # %cond.load108
-; AVX1-NEXT:    vmovq %xmm2, %rcx
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpinsrd $2, (%rcx), %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; AVX1-NEXT:  .LBB4_46: # %else112
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:  # %bb.45: # %cond.load94
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:  .LBB4_46: # %else97
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX1-NEXT:    testb $-128, %al
 ; AVX1-NEXT:    je .LBB4_48
-; AVX1-NEXT:  # %bb.47: # %cond.load114
-; AVX1-NEXT:    vpextrq $1, %xmm2, %rax
+; AVX1-NEXT:  # %bb.47: # %cond.load99
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpinsrd $3, (%rax), %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:  .LBB4_48: # %else118
+; AVX1-NEXT:  .LBB4_48: # %else102
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB4_1: # %cond.load
+; AVX1-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB4_4
+; AVX1-NEXT:  .LBB4_3: # %cond.load1
+; AVX1-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je .LBB4_6
+; AVX1-NEXT:  .LBB4_5: # %cond.load4
+; AVX1-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je .LBB4_8
+; AVX1-NEXT:  .LBB4_7: # %cond.load7
+; AVX1-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je .LBB4_10
+; AVX1-NEXT:  .LBB4_9: # %cond.load10
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpinsrd $0, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je .LBB4_12
+; AVX1-NEXT:  .LBB4_11: # %cond.load13
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    je .LBB4_14
+; AVX1-NEXT:  .LBB4_13: # %cond.load16
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB4_15
+; AVX1-NEXT:    jmp .LBB4_16
+; AVX1-NEXT:  .LBB4_17: # %cond.load23
+; AVX1-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB4_20
+; AVX1-NEXT:  .LBB4_19: # %cond.load28
+; AVX1-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm3, %xmm4
+; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je .LBB4_22
+; AVX1-NEXT:  .LBB4_21: # %cond.load33
+; AVX1-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm3, %xmm4
+; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je .LBB4_24
+; AVX1-NEXT:  .LBB4_23: # %cond.load38
+; AVX1-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm3, %xmm4
+; AVX1-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    je .LBB4_26
+; AVX1-NEXT:  .LBB4_25: # %cond.load43
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpinsrd $0, c+{{.*}}(%rip), %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    testb $32, %al
+; AVX1-NEXT:    je .LBB4_28
+; AVX1-NEXT:  .LBB4_27: # %cond.load48
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    testb $64, %al
+; AVX1-NEXT:    je .LBB4_30
+; AVX1-NEXT:  .LBB4_29: # %cond.load53
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    testb $-128, %al
+; AVX1-NEXT:    jne .LBB4_31
+; AVX1-NEXT:    jmp .LBB4_32
+; AVX1-NEXT:  .LBB4_33: # %cond.load64
+; AVX1-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-NEXT:    testb $2, %al
+; AVX1-NEXT:    je .LBB4_36
+; AVX1-NEXT:  .LBB4_35: # %cond.load69
+; AVX1-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:    testb $4, %al
+; AVX1-NEXT:    je .LBB4_38
+; AVX1-NEXT:  .LBB4_37: # %cond.load74
+; AVX1-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:    testb $8, %al
+; AVX1-NEXT:    je .LBB4_40
+; AVX1-NEXT:  .LBB4_39: # %cond.load79
+; AVX1-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX1-NEXT:    testb $16, %al
+; AVX1-NEXT:    jne .LBB4_41
+; AVX1-NEXT:    jmp .LBB4_42
 ;
 ; AVX2-LABEL: gather_v8i32_v8i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    movl $c, %eax
-; AVX2-NEXT:    vmovq %rax, %xmm1
-; AVX2-NEXT:    vpbroadcastq %xmm1, %ymm2
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [12,12,12,12]
-; AVX2-NEXT:    vpaddq %ymm1, %ymm2, %ymm3
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm1
 ; AVX2-NEXT:    vmovmskps %ymm1, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    # implicit-def: $ymm1
-; AVX2-NEXT:    je .LBB4_2
-; AVX2-NEXT:  # %bb.1: # %cond.load
-; AVX2-NEXT:    vmovq %xmm3, %rcx
-; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-NEXT:  .LBB4_2: # %else
+; AVX2-NEXT:    jne .LBB4_1
+; AVX2-NEXT:  # %bb.2: # %else
 ; AVX2-NEXT:    testb $2, %al
-; AVX2-NEXT:    je .LBB4_4
-; AVX2-NEXT:  # %bb.3: # %cond.load1
-; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm1, %xmm4
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    jne .LBB4_3
 ; AVX2-NEXT:  .LBB4_4: # %else2
 ; AVX2-NEXT:    testb $4, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
 ; AVX2-NEXT:    jne .LBB4_5
-; AVX2-NEXT:  # %bb.6: # %else5
+; AVX2-NEXT:  .LBB4_6: # %else5
 ; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    jne .LBB4_7
 ; AVX2-NEXT:  .LBB4_8: # %else8
@@ -1519,197 +1467,185 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB4_16
 ; AVX2-NEXT:  .LBB4_15: # %cond.load19
-; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrd $3, (%rax), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
 ; AVX2-NEXT:  .LBB4_16: # %else20
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [28,28,28,28]
-; AVX2-NEXT:    vpaddq %ymm3, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm0, %ymm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm0, %ymm2
 ; AVX2-NEXT:    vmovmskps %ymm2, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    # implicit-def: $ymm2
-; AVX2-NEXT:    je .LBB4_18
-; AVX2-NEXT:  # %bb.17: # %cond.load23
-; AVX2-NEXT:    vmovq %xmm3, %rcx
-; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX2-NEXT:  .LBB4_18: # %else27
+; AVX2-NEXT:    jne .LBB4_17
+; AVX2-NEXT:  # %bb.18: # %else26
 ; AVX2-NEXT:    testb $2, %al
-; AVX2-NEXT:    je .LBB4_20
-; AVX2-NEXT:  # %bb.19: # %cond.load29
-; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm2, %xmm4
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-NEXT:  .LBB4_20: # %else33
+; AVX2-NEXT:    jne .LBB4_19
+; AVX2-NEXT:  .LBB4_20: # %else31
 ; AVX2-NEXT:    testb $4, %al
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
 ; AVX2-NEXT:    jne .LBB4_21
-; AVX2-NEXT:  # %bb.22: # %else39
+; AVX2-NEXT:  .LBB4_22: # %else36
 ; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    jne .LBB4_23
-; AVX2-NEXT:  .LBB4_24: # %else45
+; AVX2-NEXT:  .LBB4_24: # %else41
 ; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    jne .LBB4_25
-; AVX2-NEXT:  .LBB4_26: # %else51
+; AVX2-NEXT:  .LBB4_26: # %else46
 ; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    jne .LBB4_27
-; AVX2-NEXT:  .LBB4_28: # %else57
+; AVX2-NEXT:  .LBB4_28: # %else51
 ; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    jne .LBB4_29
-; AVX2-NEXT:  .LBB4_30: # %else63
+; AVX2-NEXT:  .LBB4_30: # %else56
 ; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB4_32
-; AVX2-NEXT:  .LBB4_31: # %cond.load65
-; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
-; AVX2-NEXT:    vpinsrd $3, (%rax), %xmm5, %xmm5
-; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
-; AVX2-NEXT:  .LBB4_32: # %else69
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:  .LBB4_31: # %cond.load58
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX2-NEXT:  .LBB4_32: # %else61
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vmovmskps %ymm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    # implicit-def: $ymm0
 ; AVX2-NEXT:    jne .LBB4_33
-; AVX2-NEXT:  # %bb.34: # %else76
+; AVX2-NEXT:  # %bb.34: # %else67
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    jne .LBB4_35
-; AVX2-NEXT:  .LBB4_36: # %else82
+; AVX2-NEXT:  .LBB4_36: # %else72
 ; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    jne .LBB4_37
-; AVX2-NEXT:  .LBB4_38: # %else88
+; AVX2-NEXT:  .LBB4_38: # %else77
 ; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    jne .LBB4_39
-; AVX2-NEXT:  .LBB4_40: # %else94
+; AVX2-NEXT:  .LBB4_40: # %else82
 ; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    jne .LBB4_41
-; AVX2-NEXT:  .LBB4_42: # %else100
+; AVX2-NEXT:  .LBB4_42: # %else87
 ; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    jne .LBB4_43
-; AVX2-NEXT:  .LBB4_44: # %else106
+; AVX2-NEXT:  .LBB4_44: # %else92
 ; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB4_46
-; AVX2-NEXT:  .LBB4_45: # %cond.load108
-; AVX2-NEXT:    vmovq %xmm4, %rcx
+; AVX2-NEXT:  .LBB4_45: # %cond.load94
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm3, %xmm3
+; AVX2-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm3, %xmm3
 ; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX2-NEXT:  .LBB4_46: # %else112
+; AVX2-NEXT:  .LBB4_46: # %else97
 ; AVX2-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    je .LBB4_48
-; AVX2-NEXT:  # %bb.47: # %cond.load114
-; AVX2-NEXT:    vpextrq $1, %xmm4, %rax
+; AVX2-NEXT:  # %bb.47: # %cond.load99
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vpinsrd $3, (%rax), %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm2, %xmm2
 ; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT:  .LBB4_48: # %else118
+; AVX2-NEXT:  .LBB4_48: # %else102
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
+; AVX2-NEXT:  .LBB4_1: # %cond.load
+; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je .LBB4_4
+; AVX2-NEXT:  .LBB4_3: # %cond.load1
+; AVX2-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm1, %xmm2
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je .LBB4_6
 ; AVX2-NEXT:  .LBB4_5: # %cond.load4
-; AVX2-NEXT:    vmovq %xmm4, %rcx
-; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm1, %xmm5
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm1, %xmm2
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB4_8
 ; AVX2-NEXT:  .LBB4_7: # %cond.load7
-; AVX2-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX2-NEXT:    vpinsrd $3, (%rcx), %xmm1, %xmm5
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm1, %xmm2
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB4_10
 ; AVX2-NEXT:  .LBB4_9: # %cond.load10
-; AVX2-NEXT:    vmovq %xmm3, %rcx
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
-; AVX2-NEXT:    vpinsrd $0, (%rcx), %xmm5, %xmm5
-; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpinsrd $0, c+{{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
 ; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB4_12
 ; AVX2-NEXT:  .LBB4_11: # %cond.load13
-; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
 ; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB4_14
 ; AVX2-NEXT:  .LBB4_13: # %cond.load16
-; AVX2-NEXT:    vmovq %xmm4, %rcx
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm3, %xmm3
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
 ; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    jne .LBB4_15
 ; AVX2-NEXT:    jmp .LBB4_16
-; AVX2-NEXT:  .LBB4_21: # %cond.load35
-; AVX2-NEXT:    vmovq %xmm4, %rcx
-; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm2, %xmm5
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NEXT:  .LBB4_17: # %cond.load23
+; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT:    testb $2, %al
+; AVX2-NEXT:    je .LBB4_20
+; AVX2-NEXT:  .LBB4_19: # %cond.load28
+; AVX2-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm2, %xmm3
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NEXT:    testb $4, %al
+; AVX2-NEXT:    je .LBB4_22
+; AVX2-NEXT:  .LBB4_21: # %cond.load33
+; AVX2-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm2, %xmm3
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB4_24
-; AVX2-NEXT:  .LBB4_23: # %cond.load41
-; AVX2-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX2-NEXT:    vpinsrd $3, (%rcx), %xmm2, %xmm5
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-NEXT:  .LBB4_23: # %cond.load38
+; AVX2-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm2, %xmm3
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB4_26
-; AVX2-NEXT:  .LBB4_25: # %cond.load47
-; AVX2-NEXT:    vmovq %xmm3, %rcx
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
-; AVX2-NEXT:    vpinsrd $0, (%rcx), %xmm5, %xmm5
-; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-NEXT:  .LBB4_25: # %cond.load43
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpinsrd $0, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
 ; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB4_28
-; AVX2-NEXT:  .LBB4_27: # %cond.load53
-; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
-; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm5, %xmm5
-; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-NEXT:  .LBB4_27: # %cond.load48
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
 ; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    je .LBB4_30
-; AVX2-NEXT:  .LBB4_29: # %cond.load59
-; AVX2-NEXT:    vmovq %xmm4, %rcx
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
-; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm5, %xmm5
-; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm2, %ymm2
+; AVX2-NEXT:  .LBB4_29: # %cond.load53
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
 ; AVX2-NEXT:    testb $-128, %al
 ; AVX2-NEXT:    jne .LBB4_31
 ; AVX2-NEXT:    jmp .LBB4_32
-; AVX2-NEXT:  .LBB4_33: # %cond.load72
-; AVX2-NEXT:    vmovq %xmm3, %rcx
+; AVX2-NEXT:  .LBB4_33: # %cond.load64
 ; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX2-NEXT:    testb $2, %al
 ; AVX2-NEXT:    je .LBB4_36
-; AVX2-NEXT:  .LBB4_35: # %cond.load78
-; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
-; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm0, %xmm5
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:  .LBB4_35: # %cond.load69
+; AVX2-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm0, %xmm3
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-NEXT:    testb $4, %al
 ; AVX2-NEXT:    je .LBB4_38
-; AVX2-NEXT:  .LBB4_37: # %cond.load84
-; AVX2-NEXT:    vmovq %xmm4, %rcx
-; AVX2-NEXT:    vpinsrd $2, (%rcx), %xmm0, %xmm5
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:  .LBB4_37: # %cond.load74
+; AVX2-NEXT:    vpinsrd $2, c+{{.*}}(%rip), %xmm0, %xmm3
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-NEXT:    testb $8, %al
 ; AVX2-NEXT:    je .LBB4_40
-; AVX2-NEXT:  .LBB4_39: # %cond.load90
-; AVX2-NEXT:    vpextrq $1, %xmm4, %rcx
-; AVX2-NEXT:    vpinsrd $3, (%rcx), %xmm0, %xmm5
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:  .LBB4_39: # %cond.load79
+; AVX2-NEXT:    vpinsrd $3, c+{{.*}}(%rip), %xmm0, %xmm3
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-NEXT:    testb $16, %al
 ; AVX2-NEXT:    je .LBB4_42
-; AVX2-NEXT:  .LBB4_41: # %cond.load96
-; AVX2-NEXT:    vmovq %xmm3, %rcx
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX2-NEXT:    vpinsrd $0, (%rcx), %xmm5, %xmm5
-; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm0
+; AVX2-NEXT:  .LBB4_41: # %cond.load84
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vpinsrd $0, c+{{.*}}(%rip), %xmm3, %xmm3
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
 ; AVX2-NEXT:    testb $32, %al
 ; AVX2-NEXT:    je .LBB4_44
-; AVX2-NEXT:  .LBB4_43: # %cond.load102
-; AVX2-NEXT:    vpextrq $1, %xmm3, %rcx
+; AVX2-NEXT:  .LBB4_43: # %cond.load89
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpinsrd $1, (%rcx), %xmm3, %xmm3
+; AVX2-NEXT:    vpinsrd $1, c+{{.*}}(%rip), %xmm3, %xmm3
 ; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
 ; AVX2-NEXT:    testb $64, %al
 ; AVX2-NEXT:    jne .LBB4_45
@@ -1721,25 +1657,21 @@ define <8 x i32> @gather_v8i32_v8i32(<8 x i32> %trigger) {
 ; AVX512-NEXT:    vptestnmd %zmm0, %zmm0, %k0
 ; AVX512-NEXT:    kshiftlw $8, %k0, %k0
 ; AVX512-NEXT:    kshiftrw $8, %k0, %k1
-; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12]
 ; AVX512-NEXT:    kmovw %k1, %k2
-; AVX512-NEXT:    vpgatherdd c+12(,%zmm0), %zmm1 {%k2}
-; AVX512-NEXT:    vpgatherdd c+28(,%zmm0), %zmm2 {%k1}
+; AVX512-NEXT:    vpgatherdd c(,%zmm0), %zmm1 {%k2}
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28]
+; AVX512-NEXT:    vpgatherdd c(,%zmm0), %zmm2 {%k1}
 ; AVX512-NEXT:    vpaddd %ymm2, %ymm2, %ymm0
 ; AVX512-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    retq
-  %1 = insertelement <8 x %struct.a*> undef, %struct.a* @c, i32 0
-  %2 = shufflevector <8 x %struct.a*> %1, <8 x %struct.a*> undef, <8 x i32> zeroinitializer
-  %3 = getelementptr %struct.a, <8 x %struct.a*> %2, <8 x i32> zeroinitializer, i32 0, i32 3
-  %4 = icmp eq <8 x i32> %trigger, zeroinitializer
-  %5 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %3, i32 4, <8 x i1> %4, <8 x i32> undef)
-  %6 = getelementptr %struct.a, <8 x %struct.a*> %2, <8 x i32> zeroinitializer, i32 3
-  %7 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %6, i32 4, <8 x i1> %4, <8 x i32> undef)
-  %8 = add <8 x i32> %5, %7
-  %9 = getelementptr %struct.a, <8 x %struct.a*> %2, i32 0, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-  %10 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %9, i32 4, <8 x i1> %4, <8 x i32> undef)
-  %11 = add <8 x i32> %8, %10
-  ret <8 x i32> %11
+  %1 = icmp eq <8 x i32> %trigger, zeroinitializer
+  %2 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> getelementptr (%struct.a, <8 x %struct.a*> <%struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c>, <8 x i64> zeroinitializer, i32 0, <8 x i64> <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>), i32 4, <8 x i1> %1, <8 x i32> undef)
+  %3 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> getelementptr (%struct.a, <8 x %struct.a*> <%struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c>, <8 x i64> zeroinitializer, i32 3), i32 4, <8 x i1> %1, <8 x i32> undef)
+  %4 = add <8 x i32> %2, %3
+  %5 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> getelementptr (%struct.a, <8 x %struct.a*> <%struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c, %struct.a* @c>, <8 x i64> zeroinitializer, i32 3), i32 4, <8 x i1> %1, <8 x i32> undef)
+  %6 = add <8 x i32> %4, %5
+  ret <8 x i32> %6
 }
 
 declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*>, i32, <2 x i1>, <2 x double>)
diff --git a/llvm/test/CodeGen/X86/masked_gather_scatter.ll b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
index 0b6e3c9d830dc68d9e5bd3d4bf56b832c99ae487..ae736031a9cba43eaef051207d8f8d1df21cdfbe 100644
--- a/llvm/test/CodeGen/X86/masked_gather_scatter.ll
+++ b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
@@ -517,13 +517,13 @@ define <8 x i32> @test9(%struct.ST* %base, <8 x i64> %ind1, <8 x i32>%ind5) {
 ; SKX_LARGE:       # %bb.0: # %entry
 ; SKX_LARGE-NEXT:    vpbroadcastq %rdi, %zmm2
 ; SKX_LARGE-NEXT:    vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vpmuldq (%rax){1to8}, %zmm1, %zmm1
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vpmullq (%rax){1to8}, %zmm0, %zmm0
 ; SKX_LARGE-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; SKX_LARGE-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vpaddq (%rax){1to8}, %zmm0, %zmm1
 ; SKX_LARGE-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX_LARGE-NEXT:    vpgatherqd (,%zmm1), %ymm0 {%k1}
@@ -531,12 +531,12 @@ define <8 x i32> @test9(%struct.ST* %base, <8 x i64> %ind1, <8 x i32>%ind5) {
 ;
 ; SKX_32-LABEL: test9:
 ; SKX_32:       # %bb.0: # %entry
-; SKX_32-NEXT:    vpmulld {{\.LCPI.*}}{1to8}, %ymm1, %ymm1
+; SKX_32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %ymm1, %ymm1
 ; SKX_32-NEXT:    vpmovqd %zmm0, %ymm0
-; SKX_32-NEXT:    vpmulld {{\.LCPI.*}}{1to8}, %ymm0, %ymm0
+; SKX_32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
 ; SKX_32-NEXT:    vpaddd {{[0-9]+}}(%esp){1to8}, %ymm0, %ymm0
 ; SKX_32-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; SKX_32-NEXT:    vpaddd {{\.LCPI.*}}{1to8}, %ymm0, %ymm1
+; SKX_32-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm1
 ; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX_32-NEXT:    vpgatherdd (,%ymm1), %ymm0 {%k1}
 ; SKX_32-NEXT:    retl
@@ -603,13 +603,13 @@ define <8 x i32> @test10(%struct.ST* %base, <8 x i64> %i1, <8 x i32>%ind5) {
 ; SKX_LARGE:       # %bb.0: # %entry
 ; SKX_LARGE-NEXT:    vpbroadcastq %rdi, %zmm2
 ; SKX_LARGE-NEXT:    vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vpmuldq (%rax){1to8}, %zmm1, %zmm1
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vpmullq (%rax){1to8}, %zmm0, %zmm0
 ; SKX_LARGE-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; SKX_LARGE-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vpaddq (%rax){1to8}, %zmm0, %zmm1
 ; SKX_LARGE-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX_LARGE-NEXT:    vpgatherqd (,%zmm1), %ymm0 {%k1}
@@ -617,12 +617,12 @@ define <8 x i32> @test10(%struct.ST* %base, <8 x i64> %i1, <8 x i32>%ind5) {
 ;
 ; SKX_32-LABEL: test10:
 ; SKX_32:       # %bb.0: # %entry
-; SKX_32-NEXT:    vpmulld {{\.LCPI.*}}{1to8}, %ymm1, %ymm1
+; SKX_32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %ymm1, %ymm1
 ; SKX_32-NEXT:    vpmovqd %zmm0, %ymm0
-; SKX_32-NEXT:    vpmulld {{\.LCPI.*}}{1to8}, %ymm0, %ymm0
+; SKX_32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
 ; SKX_32-NEXT:    vpaddd {{[0-9]+}}(%esp){1to8}, %ymm0, %ymm0
 ; SKX_32-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; SKX_32-NEXT:    vpaddd {{\.LCPI.*}}{1to8}, %ymm0, %ymm1
+; SKX_32-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm1
 ; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX_32-NEXT:    vpgatherdd (,%ymm1), %ymm0 {%k1}
 ; SKX_32-NEXT:    retl
@@ -2893,7 +2893,7 @@ define <16 x float> @zext_index(float* %base, <16 x i32> %ind) {
 ; KNL_32-LABEL: zext_index:
 ; KNL_32:       # %bb.0:
 ; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; KNL_32-NEXT:    vpandd {{\.LCPI.*}}{1to16}, %zmm0, %zmm1
+; KNL_32-NEXT:    vpandd {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm1
 ; KNL_32-NEXT:    kxnorw %k0, %k0, %k1
 ; KNL_32-NEXT:    vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
 ; KNL_32-NEXT:    retl
@@ -2907,7 +2907,7 @@ define <16 x float> @zext_index(float* %base, <16 x i32> %ind) {
 ;
 ; SKX_LARGE-LABEL: zext_index:
 ; SKX_LARGE:       # %bb.0:
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vandps (%rax){1to16}, %zmm0, %zmm1
 ; SKX_LARGE-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX_LARGE-NEXT:    vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
@@ -2916,7 +2916,7 @@ define <16 x float> @zext_index(float* %base, <16 x i32> %ind) {
 ; SKX_32-LABEL: zext_index:
 ; SKX_32:       # %bb.0:
 ; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SKX_32-NEXT:    vandps {{\.LCPI.*}}{1to16}, %zmm0, %zmm1
+; SKX_32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm1
 ; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX_32-NEXT:    vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
 ; SKX_32-NEXT:    retl
@@ -3184,7 +3184,7 @@ define <2 x i64> @gather_2i64_constant_indices(i64* %ptr, <2 x i1> %mask) {
 ; SKX_LARGE:       # %bb.0:
 ; SKX_LARGE-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; SKX_LARGE-NEXT:    vpmovq2m %xmm0, %k1
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vmovdqa (%rax), %xmm1
 ; SKX_LARGE-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; SKX_LARGE-NEXT:    vpgatherdq (%rdi,%xmm1,8), %xmm0 {%k1}
@@ -3241,7 +3241,7 @@ define <16 x i32> @gather_16i64_constant_indices(i32* %ptr, <16 x i1> %mask) {
 ; SKX_LARGE-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; SKX_LARGE-NEXT:    vpslld $31, %zmm0, %zmm0
 ; SKX_LARGE-NEXT:    vpmovd2m %zmm0, %k1
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vmovdqa64 (%rax), %zmm1
 ; SKX_LARGE-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; SKX_LARGE-NEXT:    vpgatherdd (%rdi,%zmm1,4), %zmm0 {%k1}
@@ -3300,7 +3300,7 @@ define void @scatter_2i64_constant_indices(i32* %ptr, <2 x i1> %mask, <2 x i32>
 ; SKX_LARGE:       # %bb.0:
 ; SKX_LARGE-NEXT:    vpsllq $63, %xmm0, %xmm0
 ; SKX_LARGE-NEXT:    vpmovq2m %xmm0, %k1
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vmovdqa (%rax), %xmm0
 ; SKX_LARGE-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1}
 ; SKX_LARGE-NEXT:    retq
@@ -3355,7 +3355,7 @@ define void @scatter_16i64_constant_indices(i32* %ptr, <16 x i1> %mask, <16 x i3
 ; SKX_LARGE-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; SKX_LARGE-NEXT:    vpslld $31, %zmm0, %zmm0
 ; SKX_LARGE-NEXT:    vpmovd2m %zmm0, %k1
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vmovdqa64 (%rax), %zmm0
 ; SKX_LARGE-NEXT:    vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1}
 ; SKX_LARGE-NEXT:    vzeroupper
@@ -3506,7 +3506,7 @@ define <8 x i64> @pr45906(<8 x %struct.foo*> %ptr) {
 ;
 ; SKX_LARGE-LABEL: pr45906:
 ; SKX_LARGE:       # %bb.0: # %bb
-; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI[0-9]+_[0-9]+}}, %rax
 ; SKX_LARGE-NEXT:    vpaddq (%rax){1to8}, %zmm0, %zmm1
 ; SKX_LARGE-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX_LARGE-NEXT:    vpgatherqq (,%zmm1), %zmm0 {%k1}
@@ -3514,7 +3514,7 @@ define <8 x i64> @pr45906(<8 x %struct.foo*> %ptr) {
 ;
 ; SKX_32-LABEL: pr45906:
 ; SKX_32:       # %bb.0: # %bb
-; SKX_32-NEXT:    vpaddd {{\.LCPI.*}}{1to8}, %ymm0, %ymm1
+; SKX_32-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm1
 ; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
 ; SKX_32-NEXT:    vpgatherdq (,%ymm1), %zmm0 {%k1}
 ; SKX_32-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc.ll b/llvm/test/CodeGen/X86/masked_store_trunc.ll
index 6107e8c0ca5ffac9a4dd4281c87990ace55b0e9e..53873481a30e54306e98483628d8c85186c64ec3 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc.ll
@@ -453,11 +453,12 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask
 ; AVX2-LABEL: truncstore_v8i64_v8i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm4[0,2],ymm0[4,6],ymm4[4,6]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1,2,3],ymm1[4],ymm3[5,6,7],ymm1[8],ymm3[9,10,11],ymm1[12],ymm3[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm3[1,2,3],ymm0[4],ymm3[5,6,7],ymm0[8],ymm3[9,10,11],ymm0[12],ymm3[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
 ; AVX2-NEXT:    vmovmskps %ymm1, %eax
 ; AVX2-NEXT:    notl %eax
@@ -841,17 +842,14 @@ define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX2-LABEL: truncstore_v8i64_v8i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT:    vpshufb %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
 ; AVX2-NEXT:    vmovmskps %ymm1, %eax
 ; AVX2-NEXT:    notl %eax
@@ -2365,12 +2363,10 @@ define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32>
 ; AVX2-LABEL: truncstore_v16i32_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm5, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpshufb %ymm5, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2],ymm4[3],ymm1[4],ymm4[5],ymm1[6],ymm4[7],ymm1[8],ymm4[9],ymm1[10],ymm4[11],ymm1[12],ymm4[13],ymm1[14],ymm4[15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2],ymm4[3],ymm0[4],ymm4[5],ymm0[6],ymm4[7],ymm0[8],ymm4[9],ymm0[10],ymm4[11],ymm0[12],ymm4[13],ymm0[14],ymm4[15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
 ; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
@@ -3043,15 +3039,13 @@ define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %m
 ; AVX2-LABEL: truncstore_v16i32_v16i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm5, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm5, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm6, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm1
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
 ; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
@@ -5135,7 +5129,7 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; AVX2-LABEL: truncstore_v32i16_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
index 55df7b78745e7b28af65b240e25edc929da83e19..9bf23917b375f714968660713d8b21a87c69143a 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
@@ -1333,9 +1333,9 @@ define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm0, %ymm5
 ; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
 ; AVX2-NEXT:    vmovmskps %ymm1, %eax
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
index bbe832d93d916a7db6fe0364f189384a7db3c3a4..21ad6259a4633fbc56f5444d098b057ecc89cd37 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
@@ -1137,9 +1137,9 @@ define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm7, %ymm5
 ; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm1
 ; AVX2-NEXT:    vmovmskps %ymm1, %eax
diff --git a/llvm/test/CodeGen/X86/memcmp-minsize.ll b/llvm/test/CodeGen/X86/memcmp-minsize.ll
index f6b74ec7737821788f6ff08e544cc8eee583e159..455c07d093dad82fbccdef191b7f6ceefa2e7481 100644
--- a/llvm/test/CodeGen/X86/memcmp-minsize.ll
+++ b/llvm/test/CodeGen/X86/memcmp-minsize.ll
@@ -456,7 +456,7 @@ define i1 @length16_eq_const(i8* %X) nounwind minsize {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
 ; X86-SSE2-NEXT:    sete %al
diff --git a/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
index 0ba4f7b6d88430be66b8517f7aaa5a2719509513..878e0cd256c14200f95ff07c9f5f63e40f988395 100644
--- a/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
+++ b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
@@ -1480,7 +1480,7 @@ define i1 @length16_eq_const(i8* %X) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
 ; X86-SSE2-NEXT:    sete %al
@@ -1490,7 +1490,7 @@ define i1 @length16_eq_const(i8* %X) nounwind {
 ; X86-SSE41:       # %bb.0:
 ; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    sete %al
 ; X86-SSE41-NEXT:    retl
@@ -1823,8 +1823,8 @@ define i1 @length24_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -1836,8 +1836,8 @@ define i1 @length24_eq_const(i8* %X) nounwind {
 ; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE41-NEXT:    movdqu 8(%eax), %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    por %xmm1, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    setne %al
@@ -2312,8 +2312,8 @@ define i1 @length31_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 15(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -2325,8 +2325,8 @@ define i1 @length31_eq_const(i8* %X) nounwind {
 ; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE41-NEXT:    movdqu 15(%eax), %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    por %xmm1, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    setne %al
@@ -2816,8 +2816,8 @@ define i1 @length32_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -2829,8 +2829,8 @@ define i1 @length32_eq_const(i8* %X) nounwind {
 ; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    por %xmm1, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    setne %al
@@ -3293,9 +3293,9 @@ define i1 @length48_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
 ; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm2
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm0, %xmm2
 ; X86-SSE2-NEXT:    pmovmskb %xmm2, %eax
@@ -3309,9 +3309,9 @@ define i1 @length48_eq_const(i8* %X) nounwind {
 ; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
 ; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm2
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm2
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE41-NEXT:    por %xmm1, %xmm2
 ; X86-SSE41-NEXT:    por %xmm0, %xmm2
 ; X86-SSE41-NEXT:    ptest %xmm2, %xmm2
@@ -3673,12 +3673,12 @@ define i1 @length63_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
 ; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm2
 ; X86-SSE2-NEXT:    movdqu 47(%eax), %xmm3
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm3
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm2
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -3692,12 +3692,12 @@ define i1 @length63_eq_const(i8* %X) nounwind {
 ; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
 ; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm2
 ; X86-SSE41-NEXT:    movdqu 47(%eax), %xmm3
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm3
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm2
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE41-NEXT:    por %xmm3, %xmm2
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE41-NEXT:    por %xmm2, %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    por %xmm1, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    sete %al
@@ -4079,12 +4079,12 @@ define i1 @length64_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
 ; X86-SSE2-NEXT:    movdqu 32(%eax), %xmm2
 ; X86-SSE2-NEXT:    movdqu 48(%eax), %xmm3
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm3
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm2
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -4098,12 +4098,12 @@ define i1 @length64_eq_const(i8* %X) nounwind {
 ; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
 ; X86-SSE41-NEXT:    movdqu 32(%eax), %xmm2
 ; X86-SSE41-NEXT:    movdqu 48(%eax), %xmm3
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm3
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm2
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE41-NEXT:    por %xmm3, %xmm2
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE41-NEXT:    por %xmm2, %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    por %xmm1, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    sete %al
diff --git a/llvm/test/CodeGen/X86/memcmp-optsize.ll b/llvm/test/CodeGen/X86/memcmp-optsize.ll
index 1e56b40bc90cd6ba7523fa143f14a32fba897ee1..1d67355d6ff436fc5fd642bcd96c339aa6cdacb3 100644
--- a/llvm/test/CodeGen/X86/memcmp-optsize.ll
+++ b/llvm/test/CodeGen/X86/memcmp-optsize.ll
@@ -590,7 +590,7 @@ define i1 @length16_eq_const(i8* %X) nounwind optsize {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
 ; X86-SSE2-NEXT:    sete %al
@@ -715,8 +715,8 @@ define i1 @length24_eq_const(i8* %X) nounwind optsize {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -854,8 +854,8 @@ define i1 @length32_eq_const(i8* %X) nounwind optsize {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
diff --git a/llvm/test/CodeGen/X86/memcmp-pgso.ll b/llvm/test/CodeGen/X86/memcmp-pgso.ll
index 469d2be91b4544f22266cf02c3402d0c49fbb53d..7ea412e893ca9c56325f83b1f6bd2317001a6e36 100644
--- a/llvm/test/CodeGen/X86/memcmp-pgso.ll
+++ b/llvm/test/CodeGen/X86/memcmp-pgso.ll
@@ -590,7 +590,7 @@ define i1 @length16_eq_const(i8* %X) nounwind !prof !14 {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
 ; X86-SSE2-NEXT:    sete %al
@@ -715,8 +715,8 @@ define i1 @length24_eq_const(i8* %X) nounwind !prof !14 {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -854,8 +854,8 @@ define i1 @length32_eq_const(i8* %X) nounwind !prof !14 {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
diff --git a/llvm/test/CodeGen/X86/memcmp.ll b/llvm/test/CodeGen/X86/memcmp.ll
index 18ee773d14eaa6224df2312c47f2760c062b00f8..e05573544335f2d3010bf2c885bae2ea453ab0d0 100644
--- a/llvm/test/CodeGen/X86/memcmp.ll
+++ b/llvm/test/CodeGen/X86/memcmp.ll
@@ -1485,7 +1485,7 @@ define i1 @length16_eq_const(i8* %X) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
 ; X86-SSE2-NEXT:    sete %al
@@ -1495,7 +1495,7 @@ define i1 @length16_eq_const(i8* %X) nounwind {
 ; X86-SSE41:       # %bb.0:
 ; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    sete %al
 ; X86-SSE41-NEXT:    retl
@@ -1756,8 +1756,8 @@ define i1 @length24_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -1769,8 +1769,8 @@ define i1 @length24_eq_const(i8* %X) nounwind {
 ; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE41-NEXT:    movdqu 8(%eax), %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    por %xmm1, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    setne %al
@@ -2152,8 +2152,8 @@ define i1 @length31_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 15(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -2165,8 +2165,8 @@ define i1 @length31_eq_const(i8* %X) nounwind {
 ; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE41-NEXT:    movdqu 15(%eax), %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    por %xmm1, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    setne %al
@@ -2563,8 +2563,8 @@ define i1 @length32_eq_const(i8* %X) nounwind {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE2-NEXT:    movdqu 16(%eax), %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
-; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
@@ -2576,8 +2576,8 @@ define i1 @length32_eq_const(i8* %X) nounwind {
 ; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE41-NEXT:    movdqu (%eax), %xmm0
 ; X86-SSE41-NEXT:    movdqu 16(%eax), %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm1
-; X86-SSE41-NEXT:    pxor {{\.LCPI.*}}, %xmm0
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE41-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE41-NEXT:    por %xmm1, %xmm0
 ; X86-SSE41-NEXT:    ptest %xmm0, %xmm0
 ; X86-SSE41-NEXT:    setne %al
diff --git a/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll b/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll
index 210ad51b0a18b6783568c44f766b82923465e383..7e0f3b7cbfb009b98fb6be06457b131cea38fb0e 100644
--- a/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll
+++ b/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll
@@ -501,7 +501,7 @@ define <16 x i16> @merge_16i16_i16_0uu3zzuuuuuzCuEF(i16* %ptr) nounwind uwtable
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    vmovups (%eax), %ymm0
-; X86-AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 0
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 3
diff --git a/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll b/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll
index 3eb837d16e0ccd016ce70a131343f57cf0d4655a..d24a710746d3537931100d227cb53f0a1b617ad7 100644
--- a/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll
+++ b/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll
@@ -138,7 +138,7 @@ define <8 x double> @merge_8f64_f64_1u3u5zu8(double* %ptr) nounwind uwtable noin
 ; X86-AVX512F:       # %bb.0:
 ; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512F-NEXT:    vmovdqu64 8(%eax), %zmm0
-; X86-AVX512F-NEXT:    vpandq {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    vpandq {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 1
   %ptr2 = getelementptr inbounds double, double* %ptr, i64 3
@@ -217,7 +217,7 @@ define <8 x i64> @merge_8i64_i64_1u3u5zu8(i64* %ptr) nounwind uwtable noinline s
 ; X86-AVX512F:       # %bb.0:
 ; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512F-NEXT:    vmovdqu64 8(%eax), %zmm0
-; X86-AVX512F-NEXT:    vpandd {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    vpandd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 1
   %ptr2 = getelementptr inbounds i64, i64* %ptr, i64 3
@@ -436,7 +436,7 @@ define <16 x i32> @merge_16i32_i32_0uu3zzuuuuuzCuEF(i32* %ptr) nounwind uwtable
 ; X86-AVX512F:       # %bb.0:
 ; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512F-NEXT:    vmovdqu64 (%eax), %zmm0
-; X86-AVX512F-NEXT:    vpandd {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    vpandd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 0
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 3
diff --git a/llvm/test/CodeGen/X86/mmx-arith.ll b/llvm/test/CodeGen/X86/mmx-arith.ll
index c5c78938c4442d2a6b19d78c618537fb4f45421b..c81520b98cdb275d65467ed5b9e9451210e6c78b 100644
--- a/llvm/test/CodeGen/X86/mmx-arith.ll
+++ b/llvm/test/CodeGen/X86/mmx-arith.ll
@@ -33,7 +33,7 @@ define void @test0(x86_mmx* %A, x86_mmx* %B) {
 ; X32-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-NEXT:    pmullw %xmm0, %xmm1
-; X32-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-NEXT:    packuswb %xmm1, %xmm1
 ; X32-NEXT:    movq %xmm1, (%eax)
 ; X32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
@@ -658,7 +658,7 @@ define i64 @pr43922() {
 ; X32-NEXT:    .cfi_def_cfa_register %ebp
 ; X32-NEXT:    andl $-8, %esp
 ; X32-NEXT:    subl $8, %esp
-; X32-NEXT:    movq {{\.LCPI.*}}, %mm0 # mm0 = 0x7AAAAAAA7AAAAAAA
+; X32-NEXT:    movq {{\.LCPI[0-9]+_[0-9]+}}, %mm0 # mm0 = 0x7AAAAAAA7AAAAAAA
 ; X32-NEXT:    psrad $255, %mm0
 ; X32-NEXT:    movq %mm0, (%esp)
 ; X32-NEXT:    movl (%esp), %eax
diff --git a/llvm/test/CodeGen/X86/mmx-fold-zero.ll b/llvm/test/CodeGen/X86/mmx-fold-zero.ll
index 73dc8a81a807879dcf53b281b02ce9fb2257421e..7f7d71612eea34fa5227814d800194f260bbcc80 100644
--- a/llvm/test/CodeGen/X86/mmx-fold-zero.ll
+++ b/llvm/test/CodeGen/X86/mmx-fold-zero.ll
@@ -32,7 +32,7 @@ define double @mmx_zero(double, double, double, double) nounwind {
 ; X86-NEXT:    paddw %mm2, %mm0
 ; X86-NEXT:    paddw %mm6, %mm0
 ; X86-NEXT:    pmuludq %mm3, %mm0
-; X86-NEXT:    paddw {{\.LCPI.*}}, %mm0
+; X86-NEXT:    paddw {{\.LCPI[0-9]+_[0-9]+}}, %mm0
 ; X86-NEXT:    paddw %mm1, %mm0
 ; X86-NEXT:    pmuludq %mm7, %mm0
 ; X86-NEXT:    pmuludq (%esp), %mm0 # 8-byte Folded Reload
@@ -70,7 +70,7 @@ define double @mmx_zero(double, double, double, double) nounwind {
 ; X64-NEXT:    paddw %mm2, %mm0
 ; X64-NEXT:    paddw %mm6, %mm0
 ; X64-NEXT:    pmuludq %mm3, %mm0
-; X64-NEXT:    paddw {{\.LCPI.*}}, %mm0
+; X64-NEXT:    paddw {{\.LCPI[0-9]+_[0-9]+}}, %mm0
 ; X64-NEXT:    paddw %mm1, %mm0
 ; X64-NEXT:    pmuludq %mm7, %mm0
 ; X64-NEXT:    pmuludq {{[-0-9]+}}(%r{{[sb]}}p), %mm0 # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/X86/movtopush.ll b/llvm/test/CodeGen/X86/movtopush.ll
index fa200c2253b94dd21cff8272194998eec729a8d2..33e11235fe83b88293dc0b062d1ef126f87cd2e4 100644
--- a/llvm/test/CodeGen/X86/movtopush.ll
+++ b/llvm/test/CodeGen/X86/movtopush.ll
@@ -15,7 +15,7 @@ declare void @eightparams(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g
 declare void @eightparams16(i16 %a, i16 %b, i16 %c, i16 %d, i16 %e, i16 %f, i16 %g, i16 %h)
 declare void @eightparams64(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i64 %f, i64 %g, i64 %h)
 declare void @struct(%struct.s* byval(%struct.s) %a, i32 %b, i32 %c, i32 %d)
-declare void @inalloca(<{ %struct.s }>* inalloca)
+declare void @inalloca(<{ %struct.s }>* inalloca(<{ %struct.s }>))
 
 declare i8* @llvm.stacksave()
 declare void @llvm.stackrestore(i8*)
diff --git a/llvm/test/CodeGen/X86/musttail-inalloca.ll b/llvm/test/CodeGen/X86/musttail-inalloca.ll
index c0e571a7213b4e0238b6d62064161b046b88f2f3..f3e27fac3a9d824abc9198346c8620d3d860ddf7 100644
--- a/llvm/test/CodeGen/X86/musttail-inalloca.ll
+++ b/llvm/test/CodeGen/X86/musttail-inalloca.ll
@@ -11,10 +11,10 @@ target triple = "i386-pc-windows-msvc19.16.0"
 ; 20 bytes of memory.
 %struct.Args = type { i32, i32, i32, i32, i32 }
 
-declare dso_local x86_thiscallcc void @methodWithVtorDisp(i8* nocapture readonly, <{ %struct.Args }>* inalloca)
+declare dso_local x86_thiscallcc void @methodWithVtorDisp(i8* nocapture readonly, <{ %struct.Args }>* inalloca(<{ %struct.Args }>))
 
 ; Function Attrs: nounwind optsize
-define dso_local x86_thiscallcc void @methodWithVtorDisp_thunk(i8* %0, <{ %struct.Args }>* inalloca %1) #0 {
+define dso_local x86_thiscallcc void @methodWithVtorDisp_thunk(i8* %0, <{ %struct.Args }>* inalloca(<{ %struct.Args }>) %1) #0 {
 ; CHECK-LABEL: methodWithVtorDisp_thunk:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pushl %esi
@@ -34,7 +34,7 @@ define dso_local x86_thiscallcc void @methodWithVtorDisp_thunk(i8* %0, <{ %struc
   %7 = getelementptr i8, i8* %0, i32 %6
   %8 = call i8* @llvm.returnaddress(i32 0)
   call void @__cyg_profile_func_exit(i8* bitcast (void (i8*, <{ %struct.Args }>*)* @methodWithVtorDisp_thunk to i8*), i8* %8)
-  musttail call x86_thiscallcc void @methodWithVtorDisp(i8* %7, <{ %struct.Args }>* inalloca nonnull %1)
+  musttail call x86_thiscallcc void @methodWithVtorDisp(i8* %7, <{ %struct.Args }>* inalloca(<{ %struct.Args }>) nonnull %1)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/X86/musttail-indirect.ll b/llvm/test/CodeGen/X86/musttail-indirect.ll
index f30d775a343b383861492c1eaccc8ba8bf7d1455..cb7a31433e275807949f39e9d1cba78a87046e4b 100644
--- a/llvm/test/CodeGen/X86/musttail-indirect.ll
+++ b/llvm/test/CodeGen/X86/musttail-indirect.ll
@@ -42,13 +42,13 @@ entry:
 ; CHECK-NOT: mov %{{.*}}, {{.*(.*esp.*)}}
 ; CHECK: jmpl
 ; CHECK-NOT: ret
-define x86_thiscallcc i32 @g_thunk(%struct.B* %this, <{ %struct.A, i32, %struct.A }>* inalloca) {
+define x86_thiscallcc i32 @g_thunk(%struct.B* %this, <{ %struct.A, i32, %struct.A }>* inalloca(<{ %struct.A, i32, %struct.A }>)) {
 entry:
   %1 = bitcast %struct.B* %this to i32 (%struct.B*, <{ %struct.A, i32, %struct.A }>*)***
   %vtable = load i32 (%struct.B*, <{ %struct.A, i32, %struct.A }>*)**, i32 (%struct.B*, <{ %struct.A, i32, %struct.A }>*)*** %1
   %vfn = getelementptr inbounds i32 (%struct.B*, <{ %struct.A, i32, %struct.A }>*)*, i32 (%struct.B*, <{ %struct.A, i32, %struct.A }>*)** %vtable, i32 1
   %2 = load i32 (%struct.B*, <{ %struct.A, i32, %struct.A }>*)*, i32 (%struct.B*, <{ %struct.A, i32, %struct.A }>*)** %vfn
-  %3 = musttail call x86_thiscallcc i32 %2(%struct.B* %this, <{ %struct.A, i32, %struct.A }>* inalloca %0)
+  %3 = musttail call x86_thiscallcc i32 %2(%struct.B* %this, <{ %struct.A, i32, %struct.A }>* inalloca(<{ %struct.A, i32, %struct.A }>) %0)
   ret i32 %3
 }
 
@@ -71,13 +71,13 @@ entry:
 ; CHECK: jmpl
 ; CHECK-NOT: mov %{{.*}}, {{.*(.*esp.*)}}
 ; CHECK-NOT: ret
-define x86_thiscallcc void @h_thunk(%struct.B* %this, <{ %struct.A, i32, %struct.A }>* inalloca) {
+define x86_thiscallcc void @h_thunk(%struct.B* %this, <{ %struct.A, i32, %struct.A }>* inalloca(<{ %struct.A, i32, %struct.A }>)) {
 entry:
   %1 = bitcast %struct.B* %this to void (%struct.B*, <{ %struct.A, i32, %struct.A }>*)***
   %vtable = load void (%struct.B*, <{ %struct.A, i32, %struct.A }>*)**, void (%struct.B*, <{ %struct.A, i32, %struct.A }>*)*** %1
   %vfn = getelementptr inbounds void (%struct.B*, <{ %struct.A, i32, %struct.A }>*)*, void (%struct.B*, <{ %struct.A, i32, %struct.A }>*)** %vtable, i32 2
   %2 = load void (%struct.B*, <{ %struct.A, i32, %struct.A }>*)*, void (%struct.B*, <{ %struct.A, i32, %struct.A }>*)** %vfn
-  musttail call x86_thiscallcc void %2(%struct.B* %this, <{ %struct.A, i32, %struct.A }>* inalloca %0)
+  musttail call x86_thiscallcc void %2(%struct.B* %this, <{ %struct.A, i32, %struct.A }>* inalloca(<{ %struct.A, i32, %struct.A }>) %0)
   ret void
 }
 
@@ -99,13 +99,13 @@ entry:
 ; CHECK-NOT: mov %{{.*}}, {{.*(.*esp.*)}}
 ; CHECK: jmpl
 ; CHECK-NOT: ret
-define x86_thiscallcc %struct.A* @i_thunk(%struct.B* %this, <{ %struct.A*, %struct.A, i32, %struct.A }>* inalloca) {
+define x86_thiscallcc %struct.A* @i_thunk(%struct.B* %this, <{ %struct.A*, %struct.A, i32, %struct.A }>* inalloca(<{ %struct.A*, %struct.A, i32, %struct.A }>)) {
 entry:
   %1 = bitcast %struct.B* %this to %struct.A* (%struct.B*, <{ %struct.A*, %struct.A, i32, %struct.A }>*)***
   %vtable = load %struct.A* (%struct.B*, <{ %struct.A*, %struct.A, i32, %struct.A }>*)**, %struct.A* (%struct.B*, <{ %struct.A*, %struct.A, i32, %struct.A }>*)*** %1
   %vfn = getelementptr inbounds %struct.A* (%struct.B*, <{ %struct.A*, %struct.A, i32, %struct.A }>*)*, %struct.A* (%struct.B*, <{ %struct.A*, %struct.A, i32, %struct.A }>*)** %vtable, i32 3
   %2 = load %struct.A* (%struct.B*, <{ %struct.A*, %struct.A, i32, %struct.A }>*)*, %struct.A* (%struct.B*, <{ %struct.A*, %struct.A, i32, %struct.A }>*)** %vfn
-  %3 = musttail call x86_thiscallcc %struct.A* %2(%struct.B* %this, <{ %struct.A*, %struct.A, i32, %struct.A }>* inalloca %0)
+  %3 = musttail call x86_thiscallcc %struct.A* %2(%struct.B* %this, <{ %struct.A*, %struct.A, i32, %struct.A }>* inalloca(<{ %struct.A*, %struct.A, i32, %struct.A }>) %0)
   ret %struct.A* %3
 }
 
@@ -140,7 +140,7 @@ entry:
 ; CHECK-NOT: mov %{{.*}}, {{.*(.*esp.*)}}
 ; CHECK: jmpl
 ; CHECK-NOT: ret
-define x86_stdcallcc i32 @stdcall_thunk(<{ %struct.B*, %struct.A }>* inalloca) {
+define x86_stdcallcc i32 @stdcall_thunk(<{ %struct.B*, %struct.A }>* inalloca(<{ %struct.B*, %struct.A }>)) {
 entry:
   %this_ptr = getelementptr inbounds <{ %struct.B*, %struct.A }>, <{ %struct.B*, %struct.A }>* %0, i32 0, i32 0
   %this = load %struct.B*, %struct.B** %this_ptr
@@ -148,7 +148,7 @@ entry:
   %vtable = load i32 (<{ %struct.B*, %struct.A }>*)**, i32 (<{ %struct.B*, %struct.A }>*)*** %1
   %vfn = getelementptr inbounds i32 (<{ %struct.B*, %struct.A }>*)*, i32 (<{ %struct.B*, %struct.A }>*)** %vtable, i32 1
   %2 = load i32 (<{ %struct.B*, %struct.A }>*)*, i32 (<{ %struct.B*, %struct.A }>*)** %vfn
-  %3 = musttail call x86_stdcallcc i32 %2(<{ %struct.B*, %struct.A }>* inalloca %0)
+  %3 = musttail call x86_stdcallcc i32 %2(<{ %struct.B*, %struct.A }>* inalloca(<{ %struct.B*, %struct.A }>) %0)
   ret i32 %3
 }
 
@@ -172,13 +172,13 @@ entry:
 ; CHECK-NOT: mov %{{.*}}, {{.*(.*esp.*)}}
 ; CHECK: jmpl
 ; CHECK-NOT: ret
-define x86_fastcallcc i32 @fastcall_thunk(%struct.B* inreg %this, <{ %struct.A }>* inalloca) {
+define x86_fastcallcc i32 @fastcall_thunk(%struct.B* inreg %this, <{ %struct.A }>* inalloca(<{ %struct.A }>)) {
 entry:
   %1 = bitcast %struct.B* %this to i32 (%struct.B*, <{ %struct.A }>*)***
   %vtable = load i32 (%struct.B*, <{ %struct.A }>*)**, i32 (%struct.B*, <{ %struct.A }>*)*** %1
   %vfn = getelementptr inbounds i32 (%struct.B*, <{ %struct.A }>*)*, i32 (%struct.B*, <{ %struct.A }>*)** %vtable, i32 1
   %2 = load i32 (%struct.B*, <{ %struct.A }>*)*, i32 (%struct.B*, <{ %struct.A }>*)** %vfn
-  %3 = musttail call x86_fastcallcc i32 %2(%struct.B* inreg %this, <{ %struct.A }>* inalloca %0)
+  %3 = musttail call x86_fastcallcc i32 %2(%struct.B* inreg %this, <{ %struct.A }>* inalloca(<{ %struct.A }>) %0)
   ret i32 %3
 }
 
diff --git a/llvm/test/CodeGen/X86/musttail-thiscall.ll b/llvm/test/CodeGen/X86/musttail-thiscall.ll
index 682f85e1eb85226bc36a20dd792d9502e74c0d88..b9994963132c78a99d2623dd151bcb50e202d8c1 100644
--- a/llvm/test/CodeGen/X86/musttail-thiscall.ll
+++ b/llvm/test/CodeGen/X86/musttail-thiscall.ll
@@ -21,14 +21,14 @@ declare x86_thiscallcc i32 @t2_callee(i8* %this, i32 %a)
 
 ; CHECK-LABEL: t3:
 ; CHECK: jmp {{_?}}t3_callee
-define x86_thiscallcc i8* @t3(i8* %this, <{ i8*, i32 }>* inalloca %args) {
+define x86_thiscallcc i8* @t3(i8* %this, <{ i8*, i32 }>* inalloca(<{ i8*, i32 }>) %args) {
   %adj = getelementptr i8, i8* %this, i32 4
   %a_ptr = getelementptr <{ i8*, i32 }>, <{ i8*, i32 }>* %args, i32 0, i32 1
   store i32 0, i32* %a_ptr
-  %rv = musttail call x86_thiscallcc i8* @t3_callee(i8* %adj, <{ i8*, i32 }>* inalloca %args)
+  %rv = musttail call x86_thiscallcc i8* @t3_callee(i8* %adj, <{ i8*, i32 }>* inalloca(<{ i8*, i32 }>) %args)
   ret i8* %rv
 }
-declare x86_thiscallcc i8* @t3_callee(i8* %this, <{ i8*, i32 }>* inalloca %args);
+declare x86_thiscallcc i8* @t3_callee(i8* %this, <{ i8*, i32 }>* inalloca(<{ i8*, i32 }>) %args);
 
 ; CHECK-LABEL: t4:
 ; CHECK: jmp {{_?}}t4_callee
diff --git a/llvm/test/CodeGen/X86/neg_fp.ll b/llvm/test/CodeGen/X86/neg_fp.ll
index e07157c952f7c8284971c42a2e76ca1a945a5492..a82d51a236f527f9532f7904a19ea5e8d9d505d3 100644
--- a/llvm/test/CodeGen/X86/neg_fp.ll
+++ b/llvm/test/CodeGen/X86/neg_fp.ll
@@ -10,7 +10,7 @@ define float @negfp(float %a, float %b) nounwind {
 ; CHECK-NEXT:    pushl %eax
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    subss {{[0-9]+}}(%esp), %xmm0
-; CHECK-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK-NEXT:    movss %xmm0, (%esp)
 ; CHECK-NEXT:    flds (%esp)
 ; CHECK-NEXT:    popl %eax
diff --git a/llvm/test/CodeGen/X86/nontemporal.ll b/llvm/test/CodeGen/X86/nontemporal.ll
index 104a90a6283f7d1dbc90c426c4128fc76e540ce8..ae80496bcdc249491539ab4d354f5c44bcca224b 100644
--- a/llvm/test/CodeGen/X86/nontemporal.ll
+++ b/llvm/test/CodeGen/X86/nontemporal.ll
@@ -20,21 +20,21 @@ define i32 @f(<4 x float> %A, i8* %B, <2 x double> %C, i32 %D, <2 x i64> %E, <4
 ; X86-SSE-NEXT:    movl 8(%ebp), %esi
 ; X86-SSE-NEXT:    movl 80(%ebp), %edx
 ; X86-SSE-NEXT:    movl (%edx), %eax
-; X86-SSE-NEXT:    addps {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    addps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movntps %xmm0, (%esi)
-; X86-SSE-NEXT:    paddq {{\.LCPI.*}}, %xmm2
+; X86-SSE-NEXT:    paddq {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE-NEXT:    addl (%edx), %eax
 ; X86-SSE-NEXT:    movntdq %xmm2, (%esi)
-; X86-SSE-NEXT:    addpd {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    addpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    addl (%edx), %eax
 ; X86-SSE-NEXT:    movntpd %xmm1, (%esi)
-; X86-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm6
+; X86-SSE-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm6
 ; X86-SSE-NEXT:    addl (%edx), %eax
 ; X86-SSE-NEXT:    movntdq %xmm6, (%esi)
-; X86-SSE-NEXT:    paddw {{\.LCPI.*}}, %xmm5
+; X86-SSE-NEXT:    paddw {{\.LCPI[0-9]+_[0-9]+}}, %xmm5
 ; X86-SSE-NEXT:    addl (%edx), %eax
 ; X86-SSE-NEXT:    movntdq %xmm5, (%esi)
-; X86-SSE-NEXT:    paddb {{\.LCPI.*}}, %xmm4
+; X86-SSE-NEXT:    paddb {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE-NEXT:    addl (%edx), %eax
 ; X86-SSE-NEXT:    movntdq %xmm4, (%esi)
 ; X86-SSE-NEXT:    addl (%edx), %eax
@@ -62,21 +62,21 @@ define i32 @f(<4 x float> %A, i8* %B, <2 x double> %C, i32 %D, <2 x i64> %E, <4
 ; X86-AVX-NEXT:    movl 8(%ebp), %edx
 ; X86-AVX-NEXT:    movl 80(%ebp), %esi
 ; X86-AVX-NEXT:    movl (%esi), %eax
-; X86-AVX-NEXT:    vaddps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vaddps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovntps %xmm0, (%edx)
-; X86-AVX-NEXT:    vpaddq {{\.LCPI.*}}, %xmm2, %xmm0
+; X86-AVX-NEXT:    vpaddq {{\.LCPI[0-9]+_[0-9]+}}, %xmm2, %xmm0
 ; X86-AVX-NEXT:    addl (%esi), %eax
 ; X86-AVX-NEXT:    vmovntdq %xmm0, (%edx)
-; X86-AVX-NEXT:    vaddpd {{\.LCPI.*}}, %xmm1, %xmm0
+; X86-AVX-NEXT:    vaddpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm0
 ; X86-AVX-NEXT:    addl (%esi), %eax
 ; X86-AVX-NEXT:    vmovntpd %xmm0, (%edx)
-; X86-AVX-NEXT:    vpaddd {{\.LCPI.*}}, %xmm6, %xmm0
+; X86-AVX-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm6, %xmm0
 ; X86-AVX-NEXT:    addl (%esi), %eax
 ; X86-AVX-NEXT:    vmovntdq %xmm0, (%edx)
-; X86-AVX-NEXT:    vpaddw {{\.LCPI.*}}, %xmm5, %xmm0
+; X86-AVX-NEXT:    vpaddw {{\.LCPI[0-9]+_[0-9]+}}, %xmm5, %xmm0
 ; X86-AVX-NEXT:    addl (%esi), %eax
 ; X86-AVX-NEXT:    vmovntdq %xmm0, (%edx)
-; X86-AVX-NEXT:    vpaddb {{\.LCPI.*}}, %xmm4, %xmm0
+; X86-AVX-NEXT:    vpaddb {{\.LCPI[0-9]+_[0-9]+}}, %xmm4, %xmm0
 ; X86-AVX-NEXT:    addl (%esi), %eax
 ; X86-AVX-NEXT:    vmovntdq %xmm0, (%edx)
 ; X86-AVX-NEXT:    addl (%esi), %eax
diff --git a/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll b/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll
index c160807cf053eb6ef4b725f47c00eb659f2f27e1..534c7121ffcb3557a473ad31ae2d7025d86b81e2 100644
--- a/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll
+++ b/llvm/test/CodeGen/X86/omit-urem-of-power-of-two-or-zero-when-comparing-with-zero.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse,+sse2,+avx,+avx2 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2   | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2   | FileCheck %s --check-prefixes=CHECK,AVX2
 
 ; Given:
 ;   icmp eq/ne (urem %x, C), 0
@@ -77,23 +79,56 @@ define i1 @p3_scalar_shifted2_urem_by_const(i32 %x, i32 %y) {
 ;------------------------------------------------------------------------------;
 
 define <4 x i1> @p4_vector_urem_by_const__splat(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: p4_vector_urem_by_const__splat:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
-; CHECK-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
-; CHECK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
-; CHECK-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; SSE2-LABEL: p4_vector_urem_by_const__splat:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psrld $2, %xmm2
+; SSE2-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    psubd %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: p4_vector_urem_by_const__splat:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; SSE4-NEXT:    pmuludq %xmm2, %xmm1
+; SSE4-NEXT:    pmuludq %xmm0, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; SSE4-NEXT:    psrld $2, %xmm2
+; SSE4-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    psubd %xmm2, %xmm0
+; SSE4-NEXT:    pxor %xmm1, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX2-LABEL: p4_vector_urem_by_const__splat:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmaddwd {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
   %t0 = and <4 x i32> %x, <i32 128, i32 128, i32 128, i32 128> ; clearly a power-of-two or zero
   %t1 = urem <4 x i32> %t0, <i32 6, i32 6, i32 6, i32 6> ; '6' is clearly not a power of two
   %t2 = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 0, i32 0>
@@ -101,22 +136,65 @@ define <4 x i1> @p4_vector_urem_by_const__splat(<4 x i32> %x, <4 x i32> %y) {
 }
 
 define <4 x i1> @p5_vector_urem_by_const__nonsplat(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: p5_vector_urem_by_const__nonsplat:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; CHECK-NEXT:    vmovdqa {{.*#+}} xmm1 = [2863311531,3435973837,2863311531,954437177]
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
-; CHECK-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; SSE2-LABEL: p5_vector_urem_by_const__nonsplat:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,3435973837,2863311531,954437177]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    psrld $2, %xmm1
+; SSE2-NEXT:    psrld $1, %xmm2
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[1,2]
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2,3,1]
+; SSE2-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    psubd %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: p5_vector_urem_by_const__nonsplat:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,3435973837,2863311531,954437177]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE4-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE4-NEXT:    pmuludq %xmm2, %xmm3
+; SSE4-NEXT:    pmuludq %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE4-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; SSE4-NEXT:    movdqa %xmm1, %xmm2
+; SSE4-NEXT:    psrld $2, %xmm2
+; SSE4-NEXT:    psrld $1, %xmm1
+; SSE4-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5],xmm1[6,7]
+; SSE4-NEXT:    pmaddwd {{.*}}(%rip), %xmm1
+; SSE4-NEXT:    psubd %xmm1, %xmm0
+; SSE4-NEXT:    pxor %xmm1, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX2-LABEL: p5_vector_urem_by_const__nonsplat:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [2863311531,3435973837,2863311531,954437177]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmaddwd {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
   %t0 = and <4 x i32> %x, <i32 128, i32 2, i32 4, i32 8>
   %t1 = urem <4 x i32> %t0, <i32 3, i32 5, i32 6, i32 9>
   %t2 = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 0, i32 0>
@@ -124,23 +202,63 @@ define <4 x i1> @p5_vector_urem_by_const__nonsplat(<4 x i32> %x, <4 x i32> %y) {
 }
 
 define <4 x i1> @p6_vector_urem_by_const__nonsplat_undef0(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: p6_vector_urem_by_const__nonsplat_undef0:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
-; CHECK-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
-; CHECK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
-; CHECK-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; SSE2-LABEL: p6_vector_urem_by_const__nonsplat_undef0:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psrld $2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [6,6,6,6]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psubd %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: p6_vector_urem_by_const__nonsplat_undef0:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; SSE4-NEXT:    pmuludq %xmm2, %xmm1
+; SSE4-NEXT:    pmuludq %xmm0, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; SSE4-NEXT:    psrld $2, %xmm2
+; SSE4-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    psubd %xmm2, %xmm0
+; SSE4-NEXT:    pxor %xmm1, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX2-LABEL: p6_vector_urem_by_const__nonsplat_undef0:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
+; AVX2-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
   %t0 = and <4 x i32> %x, <i32 128, i32 128, i32 undef, i32 128>
   %t1 = urem <4 x i32> %t0, <i32 6, i32 6, i32 6, i32 6> ; '6' is clearly not a power of two
   %t2 = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 0, i32 0>
@@ -148,23 +266,56 @@ define <4 x i1> @p6_vector_urem_by_const__nonsplat_undef0(<4 x i32> %x, <4 x i32
 }
 
 define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: p7_vector_urem_by_const__nonsplat_undef2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
-; CHECK-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
-; CHECK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
-; CHECK-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; SSE2-LABEL: p7_vector_urem_by_const__nonsplat_undef2:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psrld $2, %xmm2
+; SSE2-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    psubd %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: p7_vector_urem_by_const__nonsplat_undef2:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; SSE4-NEXT:    pmuludq %xmm2, %xmm1
+; SSE4-NEXT:    pmuludq %xmm0, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; SSE4-NEXT:    psrld $2, %xmm2
+; SSE4-NEXT:    pmaddwd {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    psubd %xmm2, %xmm0
+; SSE4-NEXT:    pxor %xmm1, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX2-LABEL: p7_vector_urem_by_const__nonsplat_undef2:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmaddwd {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
   %t0 = and <4 x i32> %x, <i32 128, i32 128, i32 128, i32 128> ; clearly a power-of-two or zero
   %t1 = urem <4 x i32> %t0, <i32 6, i32 6, i32 6, i32 6> ; '6' is clearly not a power of two
   %t2 = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 undef, i32 0>
@@ -172,23 +323,63 @@ define <4 x i1> @p7_vector_urem_by_const__nonsplat_undef2(<4 x i32> %x, <4 x i32
 }
 
 define <4 x i1> @p8_vector_urem_by_const__nonsplat_undef3(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: p8_vector_urem_by_const__nonsplat_undef3:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
-; CHECK-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
-; CHECK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
-; CHECK-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
-; CHECK-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; SSE2-LABEL: p8_vector_urem_by_const__nonsplat_undef3:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2863311531,2863311531,2863311531,2863311531]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psrld $2, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [6,6,6,6]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psubd %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE4-LABEL: p8_vector_urem_by_const__nonsplat_undef3:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; SSE4-NEXT:    pmuludq %xmm2, %xmm1
+; SSE4-NEXT:    pmuludq %xmm0, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE4-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; SSE4-NEXT:    psrld $2, %xmm2
+; SSE4-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    psubd %xmm2, %xmm0
+; SSE4-NEXT:    pxor %xmm1, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE4-NEXT:    retq
+;
+; AVX2-LABEL: p8_vector_urem_by_const__nonsplat_undef3:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [128,128,128,128]
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2863311531,2863311531,2863311531,2863311531]
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3]
+; AVX2-NEXT:    vpsrld $2, %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,6,6,6]
+; AVX2-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
   %t0 = and <4 x i32> %x, <i32 128, i32 128, i32 undef, i32 128>
   %t1 = urem <4 x i32> %t0, <i32 6, i32 6, i32 6, i32 6> ; '6' is clearly not a power of two
   %t2 = icmp eq <4 x i32> %t1, <i32 0, i32 0, i32 undef, i32 0>
diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll
index 0f92e5a7be061b196b9c4330519ceb7807bb4e66..9df12b7a3fd36e2d0c01ec03d46eabebd49c395e 100644
--- a/llvm/test/CodeGen/X86/opt-pipeline.ll
+++ b/llvm/test/CodeGen/X86/opt-pipeline.ll
@@ -24,12 +24,12 @@
 ; CHECK-NEXT:     Pre-ISel Intrinsic Lowering
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Expand Atomic instructions
-; CHECK-NEXT:       Dominator Tree Construction
-; CHECK-NEXT:       Natural Loop Information
 ; CHECK-NEXT:       Lower AMX intrinsics
 ; CHECK-NEXT:       Lower AMX type for load/store
 ; CHECK-NEXT:       Module Verifier
+; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
+; CHECK-NEXT:       Natural Loop Information
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       Scalar Evolution Analysis
 ; CHECK-NEXT:       Loop Pass Manager
diff --git a/llvm/test/CodeGen/X86/packss.ll b/llvm/test/CodeGen/X86/packss.ll
index 16349ae2c7f9b5f88afeb58c5a33781673f0c54d..c395cf84ce34e83eb5933c68835ebd3dbb4cef13 100644
--- a/llvm/test/CodeGen/X86/packss.ll
+++ b/llvm/test/CodeGen/X86/packss.ll
@@ -121,14 +121,14 @@ define <8 x i16> @trunc_ashr_v4i32_icmp_v4i32(<4 x i32> %a, <4 x i32> %b) nounwi
 ; X86-SSE-LABEL: trunc_ashr_v4i32_icmp_v4i32:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrad $31, %xmm0
-; X86-SSE-NEXT:    pcmpgtd {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    pcmpgtd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    packssdw %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X86-AVX-LABEL: trunc_ashr_v4i32_icmp_v4i32:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vpsrad $31, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpcmpgtd {{\.LCPI.*}}, %xmm1, %xmm1
+; X86-AVX-NEXT:    vpcmpgtd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
 ; X86-AVX-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/peep-test-1.ll b/llvm/test/CodeGen/X86/peep-test-1.ll
index 88762cac19b50eec960edb96007b5b505c29fccb..7a356c1b361f10893d13fc8a3b5ad0e2ab3b9530 100644
--- a/llvm/test/CodeGen/X86/peep-test-1.ll
+++ b/llvm/test/CodeGen/X86/peep-test-1.ll
@@ -10,7 +10,7 @@ define void @foo(i32 %n, double* nocapture %p) nounwind {
 ; CHECK-NEXT:  .LBB0_1: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    fldl (%eax,%ecx,8)
-; CHECK-NEXT:    fmull {{\.LCPI.*}}
+; CHECK-NEXT:    fmull {{\.LCPI[0-9]+_[0-9]+}}
 ; CHECK-NEXT:    fstpl (%eax,%ecx,8)
 ; CHECK-NEXT:    decl %ecx
 ; CHECK-NEXT:    js .LBB0_1
diff --git a/llvm/test/CodeGen/X86/pmulh.ll b/llvm/test/CodeGen/X86/pmulh.ll
index 12d46d912a4501ca9080295ac9b88075d387e24f..4c932f13595a67d8c9a3a77bbf30b4598e44dbe6 100644
--- a/llvm/test/CodeGen/X86/pmulh.ll
+++ b/llvm/test/CodeGen/X86/pmulh.ll
@@ -5,13 +5,13 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
 
-define <4 x i16> @mulhuw_v4i16(<4 x i16> %a, <4 x i16> %b) {
-; SSE-LABEL: mulhuw_v4i16:
+define <4 x i16> @zext_mulhuw_v4i16(<4 x i16> %a, <4 x i16> %b) {
+; SSE-LABEL: zext_mulhuw_v4i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhuw %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: mulhuw_v4i16:
+; AVX-LABEL: zext_mulhuw_v4i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -23,13 +23,78 @@ define <4 x i16> @mulhuw_v4i16(<4 x i16> %a, <4 x i16> %b) {
   ret <4 x i16> %e
 }
 
-define <4 x i16> @mulhw_v4i16(<4 x i16> %a, <4 x i16> %b) {
-; SSE-LABEL: mulhw_v4i16:
+define <4 x i16> @and_mulhuw_v4i16(<4 x i64> %a, <4 x i64> %b) {
+; SSE2-LABEL: and_mulhuw_v4i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [65535,0,0,0,65535,0,0,0]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pmuludq %xmm2, %xmm0
+; SSE2-NEXT:    psrlq $16, %xmm0
+; SSE2-NEXT:    psrlq $16, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: and_mulhuw_v4i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3,4],xmm4[5],xmm2[6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1],xmm0[2,3,4],xmm4[5],xmm0[6,7]
+; SSE41-NEXT:    pmuldq %xmm2, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3,4],xmm4[5],xmm3[6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2,3,4],xmm4[5],xmm1[6,7]
+; SSE41-NEXT:    pmuldq %xmm3, %xmm1
+; SSE41-NEXT:    psrlq $16, %xmm1
+; SSE41-NEXT:    psrlq $16, %xmm0
+; SSE41-NEXT:    packusdw %xmm1, %xmm0
+; SSE41-NEXT:    packusdw %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX2-LABEL: and_mulhuw_v4i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3,4],ymm2[5],ymm0[6,7,8],ymm2[9],ymm0[10,11,12],ymm2[13],ymm0[14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4],ymm2[5],ymm1[6,7,8],ymm2[9],ymm1[10,11,12],ymm2[13],ymm1[14,15]
+; AVX2-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlq $16, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: and_mulhuw_v4i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3,4],ymm2[5],ymm0[6,7,8],ymm2[9],ymm0[10,11,12],ymm2[13],ymm0[14,15]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4],ymm2[5],ymm1[6,7,8],ymm2[9],ymm1[10,11,12],ymm2[13],ymm1[14,15]
+; AVX512-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpsrlq $16, %ymm0, %ymm0
+; AVX512-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %a1 = and <4 x i64> %a, <i64 65535, i64 65535, i64 65535, i64 65535>
+  %b1 = and <4 x i64> %b, <i64 65535, i64 65535, i64 65535, i64 65535>
+  %c = mul <4 x i64> %a1, %b1
+  %d = lshr <4 x i64> %c, <i64 16, i64 16, i64 16, i64 16>
+  %e = trunc <4 x i64> %d to <4 x i16>
+  ret <4 x i16> %e
+}
+
+define <4 x i16> @sext_mulhw_v4i16(<4 x i16> %a, <4 x i16> %b) {
+; SSE-LABEL: sext_mulhw_v4i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhw %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: mulhw_v4i16:
+; AVX-LABEL: sext_mulhw_v4i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -41,13 +106,48 @@ define <4 x i16> @mulhw_v4i16(<4 x i16> %a, <4 x i16> %b) {
   ret <4 x i16> %e
 }
 
-define <8 x i16> @mulhuw_v8i16(<8 x i16> %a, <8 x i16> %b) {
-; SSE-LABEL: mulhuw_v8i16:
+define <4 x i16> @ashr_mulhw_v4i16(<4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: ashr_mulhw_v4i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    packssdw %xmm1, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    packssdw %xmm0, %xmm0
+; SSE2-NEXT:    pmulhw %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: ashr_mulhw_v4i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    psrad $16, %xmm0
+; SSE41-NEXT:    psrad $16, %xmm1
+; SSE41-NEXT:    pmulld %xmm1, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    packusdw %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: ashr_mulhw_v4i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrad $16, %xmm0, %xmm0
+; AVX-NEXT:    vpsrad $16, %xmm1, %xmm1
+; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %a1 = ashr <4 x i32> %a, <i32 16, i32 16, i32 16, i32 16>
+  %b1 = ashr <4 x i32> %b, <i32 16, i32 16, i32 16, i32 16>
+  %c = mul <4 x i32> %a1, %b1
+  %d = lshr <4 x i32> %c, <i32 16, i32 16, i32 16, i32 16>
+  %e = trunc <4 x i32> %d to <4 x i16>
+  ret <4 x i16> %e
+}
+
+define <8 x i16> @zext_mulhuw_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; SSE-LABEL: zext_mulhuw_v8i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhuw %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: mulhuw_v8i16:
+; AVX-LABEL: zext_mulhuw_v8i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -59,13 +159,67 @@ define <8 x i16> @mulhuw_v8i16(<8 x i16> %a, <8 x i16> %b) {
   ret <8 x i16> %e
 }
 
-define <8 x i16> @mulhw_v8i16(<8 x i16> %a, <8 x i16> %b) {
-; SSE-LABEL: mulhw_v8i16:
+define <8 x i16> @lshr_mulhuw_v8i16(<8 x i32> %a, <8 x i32> %b) {
+; SSE2-LABEL: lshr_mulhuw_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    pmulhuw %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: lshr_mulhuw_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm3
+; SSE41-NEXT:    pmulld %xmm1, %xmm3
+; SSE41-NEXT:    psrld $16, %xmm2
+; SSE41-NEXT:    pmulld %xmm2, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm3
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    packusdw %xmm3, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX2-LABEL: lshr_mulhuw_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm1
+; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: lshr_mulhuw_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX512-NEXT:    vpsrld $16, %ymm1, %ymm1
+; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %a1 = lshr <8 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %b1 = lshr <8 x i32> %b, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %c = mul <8 x i32> %a1, %b1
+  %d = lshr <8 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %e = trunc <8 x i32> %d to <8 x i16>
+  ret <8 x i16> %e
+}
+
+define <8 x i16> @sext_mulhw_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; SSE-LABEL: sext_mulhw_v8i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhw %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: mulhw_v8i16:
+; AVX-LABEL: sext_mulhw_v8i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -77,14 +231,87 @@ define <8 x i16> @mulhw_v8i16(<8 x i16> %a, <8 x i16> %b) {
   ret <8 x i16> %e
 }
 
-define <16 x i16> @mulhuw_v16i16(<16 x i16> %a, <16 x i16> %b) {
-; SSE-LABEL: mulhuw_v16i16:
+define <8 x i16> @sextinreg_mulhw_v8i16(<8 x i32> %a, <8 x i32> %b) {
+; SSE2-LABEL: sextinreg_mulhw_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslld $24, %xmm1
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    pslld $24, %xmm0
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    pslld $25, %xmm3
+; SSE2-NEXT:    psrad $25, %xmm3
+; SSE2-NEXT:    pslld $25, %xmm2
+; SSE2-NEXT:    psrad $25, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    pmullw %xmm0, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: sextinreg_mulhw_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pslld $24, %xmm1
+; SSE41-NEXT:    psrad $24, %xmm1
+; SSE41-NEXT:    pslld $24, %xmm0
+; SSE41-NEXT:    psrad $24, %xmm0
+; SSE41-NEXT:    pslld $25, %xmm3
+; SSE41-NEXT:    psrad $25, %xmm3
+; SSE41-NEXT:    pmulld %xmm1, %xmm3
+; SSE41-NEXT:    pslld $25, %xmm2
+; SSE41-NEXT:    psrad $25, %xmm2
+; SSE41-NEXT:    pmulld %xmm2, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm3
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    packusdw %xmm3, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX2-LABEL: sextinreg_mulhw_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpslld $24, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $24, %ymm0, %ymm0
+; AVX2-NEXT:    vpslld $25, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $25, %ymm1, %ymm1
+; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: sextinreg_mulhw_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpslld $24, %ymm0, %ymm0
+; AVX512-NEXT:    vpsrad $24, %ymm0, %ymm0
+; AVX512-NEXT:    vpslld $25, %ymm1, %ymm1
+; AVX512-NEXT:    vpsrad $25, %ymm1, %ymm1
+; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %a1 = shl <8 x i32> %a, <i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>
+  %b1 = shl <8 x i32> %b, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %a2 = ashr <8 x i32> %a1, <i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>
+  %b2 = ashr <8 x i32> %b1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %c = mul <8 x i32> %a2, %b2
+  %d = lshr <8 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %e = trunc <8 x i32> %d to <8 x i16>
+  ret <8 x i16> %e
+}
+
+define <16 x i16> @zext_mulhuw_v16i16(<16 x i16> %a, <16 x i16> %b) {
+; SSE-LABEL: zext_mulhuw_v16i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhuw %xmm2, %xmm0
 ; SSE-NEXT:    pmulhuw %xmm3, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: mulhuw_v16i16:
+; AVX-LABEL: zext_mulhuw_v16i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
@@ -96,14 +323,111 @@ define <16 x i16> @mulhuw_v16i16(<16 x i16> %a, <16 x i16> %b) {
   ret <16 x i16> %e
 }
 
-define <16 x i16> @mulhw_v16i16(<16 x i16> %a, <16 x i16> %b) {
-; SSE-LABEL: mulhw_v16i16:
+define <16 x i16> @and_mulhuw_v16i16(<16 x i32> %a, <16 x i32> %b) {
+; SSE2-LABEL: and_mulhuw_v16i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767,32767,32767]
+; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm8, %xmm1
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm7
+; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-NEXT:    pmulhw %xmm2, %xmm6
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    pmulhw %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    movdqa %xmm6, %xmm1
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
+; SSE2-NEXT:    packssdw %xmm1, %xmm6
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSE2-NEXT:    packssdw %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm6, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: and_mulhuw_v16i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767,32767,32767]
+; SSE41-NEXT:    pand %xmm8, %xmm3
+; SSE41-NEXT:    pand %xmm8, %xmm2
+; SSE41-NEXT:    pand %xmm8, %xmm1
+; SSE41-NEXT:    pand %xmm8, %xmm0
+; SSE41-NEXT:    pand %xmm8, %xmm7
+; SSE41-NEXT:    pmaddwd %xmm3, %xmm7
+; SSE41-NEXT:    pand %xmm8, %xmm6
+; SSE41-NEXT:    pmaddwd %xmm2, %xmm6
+; SSE41-NEXT:    pand %xmm8, %xmm5
+; SSE41-NEXT:    pmaddwd %xmm1, %xmm5
+; SSE41-NEXT:    pand %xmm8, %xmm4
+; SSE41-NEXT:    pmaddwd %xmm4, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm7
+; SSE41-NEXT:    psrld $16, %xmm6
+; SSE41-NEXT:    packusdw %xmm7, %xmm6
+; SSE41-NEXT:    psrld $16, %xmm5
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    packusdw %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX2-LABEL: and_mulhuw_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpmaddwd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpmaddwd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: and_mulhuw_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm2 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX512F-NEXT:    vpandd %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpandd %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: and_mulhuw_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm2 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX512BW-NEXT:    vpandd %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandd %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpmaddwd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $16, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    retq
+  %a1 = and <16 x i32> %a, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %b1 = and <16 x i32> %b, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %c = mul <16 x i32> %a1, %b1
+  %d = lshr <16 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %e = trunc <16 x i32> %d to <16 x i16>
+  ret <16 x i16> %e
+}
+
+define <16 x i16> @sext_mulhuw_v16i16(<16 x i16> %a, <16 x i16> %b) {
+; SSE-LABEL: sext_mulhuw_v16i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhw %xmm2, %xmm0
 ; SSE-NEXT:    pmulhw %xmm3, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: mulhw_v16i16:
+; AVX-LABEL: sext_mulhuw_v16i16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
@@ -115,8 +439,81 @@ define <16 x i16> @mulhw_v16i16(<16 x i16> %a, <16 x i16> %b) {
   ret <16 x i16> %e
 }
 
-define <32 x i16> @mulhuw_v32i16(<32 x i16> %a, <32 x i16> %b) {
-; SSE-LABEL: mulhuw_v32i16:
+define <16 x i16> @ashr_mulhuw_v16i16(<16 x i32> %a, <16 x i32> %b) {
+; SSE2-LABEL: ashr_mulhuw_v16i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psrad $16, %xmm5
+; SSE2-NEXT:    psrad $16, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    pmulhw %xmm4, %xmm0
+; SSE2-NEXT:    psrad $16, %xmm7
+; SSE2-NEXT:    psrad $16, %xmm6
+; SSE2-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    pmulhw %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: ashr_mulhuw_v16i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    psrad $16, %xmm3
+; SSE41-NEXT:    psrad $16, %xmm2
+; SSE41-NEXT:    psrad $16, %xmm1
+; SSE41-NEXT:    psrad $16, %xmm0
+; SSE41-NEXT:    psrad $16, %xmm7
+; SSE41-NEXT:    pmulld %xmm3, %xmm7
+; SSE41-NEXT:    psrad $16, %xmm6
+; SSE41-NEXT:    pmulld %xmm2, %xmm6
+; SSE41-NEXT:    psrad $16, %xmm5
+; SSE41-NEXT:    pmulld %xmm1, %xmm5
+; SSE41-NEXT:    psrad $16, %xmm4
+; SSE41-NEXT:    pmulld %xmm4, %xmm0
+; SSE41-NEXT:    psrld $16, %xmm7
+; SSE41-NEXT:    psrld $16, %xmm6
+; SSE41-NEXT:    packusdw %xmm7, %xmm6
+; SSE41-NEXT:    psrld $16, %xmm5
+; SSE41-NEXT:    psrld $16, %xmm0
+; SSE41-NEXT:    packusdw %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX2-LABEL: ashr_mulhuw_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrad $16, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $16, %ymm3, %ymm3
+; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $16, %ymm2, %ymm2
+; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ashr_mulhuw_v16i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsrad $16, %zmm0, %zmm0
+; AVX512-NEXT:    vpsrad $16, %zmm1, %zmm1
+; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpsrld $16, %zmm0, %zmm0
+; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512-NEXT:    retq
+  %a1 = ashr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %b1 = ashr <16 x i32> %b, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %c = mul <16 x i32> %a1, %b1
+  %d = lshr <16 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %e = trunc <16 x i32> %d to <16 x i16>
+  ret <16 x i16> %e
+}
+
+define <32 x i16> @zext_mulhuw_v32i16(<32 x i16> %a, <32 x i16> %b) {
+; SSE-LABEL: zext_mulhuw_v32i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhuw %xmm4, %xmm0
 ; SSE-NEXT:    pmulhuw %xmm5, %xmm1
@@ -124,13 +521,13 @@ define <32 x i16> @mulhuw_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ; SSE-NEXT:    pmulhuw %xmm7, %xmm3
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: mulhuw_v32i16:
+; AVX2-LABEL: zext_mulhuw_v32i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: mulhuw_v32i16:
+; AVX512F-LABEL: zext_mulhuw_v32i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
@@ -139,7 +536,7 @@ define <32 x i16> @mulhuw_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
-; AVX512BW-LABEL: mulhuw_v32i16:
+; AVX512BW-LABEL: zext_mulhuw_v32i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
@@ -151,8 +548,8 @@ define <32 x i16> @mulhuw_v32i16(<32 x i16> %a, <32 x i16> %b) {
   ret <32 x i16> %e
 }
 
-define <32 x i16> @mulhw_v32i16(<32 x i16> %a, <32 x i16> %b) {
-; SSE-LABEL: mulhw_v32i16:
+define <32 x i16> @sext_mulhuw_v32i16(<32 x i16> %a, <32 x i16> %b) {
+; SSE-LABEL: sext_mulhuw_v32i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhw %xmm4, %xmm0
 ; SSE-NEXT:    pmulhw %xmm5, %xmm1
@@ -160,13 +557,13 @@ define <32 x i16> @mulhw_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ; SSE-NEXT:    pmulhw %xmm7, %xmm3
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: mulhw_v32i16:
+; AVX2-LABEL: sext_mulhuw_v32i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmulhw %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: mulhw_v32i16:
+; AVX512F-LABEL: sext_mulhuw_v32i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
@@ -175,7 +572,7 @@ define <32 x i16> @mulhw_v32i16(<32 x i16> %a, <32 x i16> %b) {
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
-; AVX512BW-LABEL: mulhw_v32i16:
+; AVX512BW-LABEL: sext_mulhuw_v32i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpmulhw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
@@ -187,8 +584,8 @@ define <32 x i16> @mulhw_v32i16(<32 x i16> %a, <32 x i16> %b) {
   ret <32 x i16> %e
 }
 
-define <64 x i16> @mulhuw_v64i16(<64 x i16> %a, <64 x i16> %b) {
-; SSE-LABEL: mulhuw_v64i16:
+define <64 x i16> @zext_mulhuw_v64i16(<64 x i16> %a, <64 x i16> %b) {
+; SSE-LABEL: zext_mulhuw_v64i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq %rdi, %rax
 ; SSE-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm0
@@ -209,7 +606,7 @@ define <64 x i16> @mulhuw_v64i16(<64 x i16> %a, <64 x i16> %b) {
 ; SSE-NEXT:    movdqa %xmm0, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: mulhuw_v64i16:
+; AVX2-LABEL: zext_mulhuw_v64i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhuw %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmulhuw %ymm5, %ymm1, %ymm1
@@ -217,7 +614,7 @@ define <64 x i16> @mulhuw_v64i16(<64 x i16> %a, <64 x i16> %b) {
 ; AVX2-NEXT:    vpmulhuw %ymm7, %ymm3, %ymm3
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: mulhuw_v64i16:
+; AVX512F-LABEL: zext_mulhuw_v64i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
@@ -231,7 +628,7 @@ define <64 x i16> @mulhuw_v64i16(<64 x i16> %a, <64 x i16> %b) {
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
 ; AVX512F-NEXT:    retq
 ;
-; AVX512BW-LABEL: mulhuw_v64i16:
+; AVX512BW-LABEL: zext_mulhuw_v64i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpmulhuw %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmulhuw %zmm3, %zmm1, %zmm1
@@ -244,8 +641,8 @@ define <64 x i16> @mulhuw_v64i16(<64 x i16> %a, <64 x i16> %b) {
   ret <64 x i16> %e
 }
 
-define <64 x i16> @mulhw_v64i16(<64 x i16> %a, <64 x i16> %b) {
-; SSE-LABEL: mulhw_v64i16:
+define <64 x i16> @sext_mulhuw_v64i16(<64 x i16> %a, <64 x i16> %b) {
+; SSE-LABEL: sext_mulhuw_v64i16:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq %rdi, %rax
 ; SSE-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm0
@@ -266,7 +663,7 @@ define <64 x i16> @mulhw_v64i16(<64 x i16> %a, <64 x i16> %b) {
 ; SSE-NEXT:    movdqa %xmm0, (%rdi)
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: mulhw_v64i16:
+; AVX2-LABEL: sext_mulhuw_v64i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhw %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmulhw %ymm5, %ymm1, %ymm1
@@ -274,7 +671,7 @@ define <64 x i16> @mulhw_v64i16(<64 x i16> %a, <64 x i16> %b) {
 ; AVX2-NEXT:    vpmulhw %ymm7, %ymm3, %ymm3
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: mulhw_v64i16:
+; AVX512F-LABEL: sext_mulhuw_v64i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
@@ -288,7 +685,7 @@ define <64 x i16> @mulhw_v64i16(<64 x i16> %a, <64 x i16> %b) {
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
 ; AVX512F-NEXT:    retq
 ;
-; AVX512BW-LABEL: mulhw_v64i16:
+; AVX512BW-LABEL: sext_mulhuw_v64i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpmulhw %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmulhw %zmm3, %zmm1, %zmm1
@@ -301,13 +698,13 @@ define <64 x i16> @mulhw_v64i16(<64 x i16> %a, <64 x i16> %b) {
   ret <64 x i16> %e
 }
 
-define <8 x i16> @mulhuw_v8i16_i64(<8 x i16> %a, <8 x i16> %b) {
-; SSE-LABEL: mulhuw_v8i16_i64:
+define <8 x i16> @zext_mulhuw_v8i16_i64(<8 x i16> %a, <8 x i16> %b) {
+; SSE-LABEL: zext_mulhuw_v8i16_i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhuw %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: mulhuw_v8i16_i64:
+; AVX-LABEL: zext_mulhuw_v8i16_i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -319,13 +716,13 @@ define <8 x i16> @mulhuw_v8i16_i64(<8 x i16> %a, <8 x i16> %b) {
   ret <8 x i16> %e
 }
 
-define <8 x i16> @mulhw_v8i16_i64(<8 x i16> %a, <8 x i16> %b) {
-; SSE-LABEL: mulhw_v8i16_i64:
+define <8 x i16> @sext_mulhuw_v8i16_i64(<8 x i16> %a, <8 x i16> %b) {
+; SSE-LABEL: sext_mulhuw_v8i16_i64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmulhw %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: mulhw_v8i16_i64:
+; AVX-LABEL: sext_mulhuw_v8i16_i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -337,21 +734,21 @@ define <8 x i16> @mulhw_v8i16_i64(<8 x i16> %a, <8 x i16> %b) {
   ret <8 x i16> %e
 }
 
-define <4 x i32> @mulhuw_v4i16_lshr(<4 x i16> %a, <4 x i16> %b) {
-; SSE2-LABEL: mulhuw_v4i16_lshr:
+define <4 x i32> @zext_mulhuw_v4i16_lshr(<4 x i16> %a, <4 x i16> %b) {
+; SSE2-LABEL: zext_mulhuw_v4i16_lshr:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pmulhuw %xmm1, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: mulhuw_v4i16_lshr:
+; SSE41-LABEL: zext_mulhuw_v4i16_lshr:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhuw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: mulhuw_v4i16_lshr:
+; AVX-LABEL: zext_mulhuw_v4i16_lshr:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
@@ -417,8 +814,8 @@ define <4 x i32> @mulhsw_v4i16_ashr(<4 x i16> %a, <4 x i16> %b) {
   ret <4 x i32> %d
 }
 
-define <8 x i32> @mulhuw_v8i16_lshr(<8 x i16> %a, <8 x i16> %b) {
-; SSE2-LABEL: mulhuw_v8i16_lshr:
+define <8 x i32> @zext_mulhuw_v8i16_lshr(<8 x i16> %a, <8 x i16> %b) {
+; SSE2-LABEL: zext_mulhuw_v8i16_lshr:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pmulhuw %xmm1, %xmm2
@@ -429,7 +826,7 @@ define <8 x i32> @mulhuw_v8i16_lshr(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: mulhuw_v8i16_lshr:
+; SSE41-LABEL: zext_mulhuw_v8i16_lshr:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    pmulhuw %xmm1, %xmm2
@@ -439,7 +836,7 @@ define <8 x i32> @mulhuw_v8i16_lshr(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41-NEXT:    movdqa %xmm2, %xmm1
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: mulhuw_v8i16_lshr:
+; AVX-LABEL: zext_mulhuw_v8i16_lshr:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
@@ -517,8 +914,8 @@ define <8 x i32> @mulhsw_v8i16_ashr(<8 x i16> %a, <8 x i16> %b) {
   ret <8 x i32> %d
 }
 
-define <16 x i32> @mulhuw_v16i16_lshr(<16 x i16> %a, <16 x i16> %b) {
-; SSE2-LABEL: mulhuw_v16i16_lshr:
+define <16 x i32> @zext_mulhuw_v16i16_lshr(<16 x i16> %a, <16 x i16> %b) {
+; SSE2-LABEL: zext_mulhuw_v16i16_lshr:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
@@ -534,7 +931,7 @@ define <16 x i32> @mulhuw_v16i16_lshr(<16 x i16> %a, <16 x i16> %b) {
 ; SSE2-NEXT:    movdqa %xmm4, %xmm3
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: mulhuw_v16i16_lshr:
+; SSE41-LABEL: zext_mulhuw_v16i16_lshr:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm1, %xmm4
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
@@ -548,7 +945,7 @@ define <16 x i32> @mulhuw_v16i16_lshr(<16 x i16> %a, <16 x i16> %b) {
 ; SSE41-NEXT:    movdqa %xmm4, %xmm3
 ; SSE41-NEXT:    retq
 ;
-; AVX2-LABEL: mulhuw_v16i16_lshr:
+; AVX2-LABEL: zext_mulhuw_v16i16_lshr:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm1
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
@@ -556,7 +953,7 @@ define <16 x i32> @mulhuw_v16i16_lshr(<16 x i16> %a, <16 x i16> %b) {
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: mulhuw_v16i16_lshr:
+; AVX512-LABEL: zext_mulhuw_v16i16_lshr:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
@@ -669,8 +1066,8 @@ define <16 x i32> @mulhsw_v16i16_ashr(<16 x i16> %a, <16 x i16> %b) {
   ret <16 x i32> %d
 }
 
-define <32 x i32> @mulhuw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {
-; SSE2-LABEL: mulhuw_v32i16_lshr:
+define <32 x i32> @zext_mulhuw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {
+; SSE2-LABEL: zext_mulhuw_v32i16_lshr:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %rax
 ; SSE2-NEXT:    pmulhuw %xmm7, %xmm3
@@ -700,7 +1097,7 @@ define <32 x i32> @mulhuw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {
 ; SSE2-NEXT:    movdqa %xmm8, (%rdi)
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: mulhuw_v32i16_lshr:
+; SSE41-LABEL: zext_mulhuw_v32i16_lshr:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
 ; SSE41-NEXT:    pmulhuw %xmm4, %xmm0
@@ -726,7 +1123,7 @@ define <32 x i32> @mulhuw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {
 ; SSE41-NEXT:    movdqa %xmm8, (%rdi)
 ; SSE41-NEXT:    retq
 ;
-; AVX2-LABEL: mulhuw_v32i16_lshr:
+; AVX2-LABEL: zext_mulhuw_v32i16_lshr:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm2
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
@@ -739,7 +1136,7 @@ define <32 x i32> @mulhuw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {
 ; AVX2-NEXT:    vmovdqa %ymm4, %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: mulhuw_v32i16_lshr:
+; AVX512F-LABEL: zext_mulhuw_v32i16_lshr:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm2
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
@@ -750,7 +1147,7 @@ define <32 x i32> @mulhuw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
-; AVX512BW-LABEL: mulhuw_v32i16_lshr:
+; AVX512BW-LABEL: zext_mulhuw_v32i16_lshr:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
@@ -960,8 +1357,8 @@ define <32 x i32> @mulhsw_v32i16_ashr(<32 x i16> %a, <32 x i16> %b) {
   ret <32 x i32> %d
 }
 
-define <64 x i32> @mulhuw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {
-; SSE2-LABEL: mulhuw_v64i16_lshr:
+define <64 x i32> @zext_mulhuw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {
+; SSE2-LABEL: zext_mulhuw_v64i16_lshr:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
 ; SSE2-NEXT:    movq %rdi, %rax
@@ -1018,7 +1415,7 @@ define <64 x i32> @mulhuw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {
 ; SSE2-NEXT:    movaps %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: mulhuw_v64i16_lshr:
+; SSE41-LABEL: zext_mulhuw_v64i16_lshr:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
 ; SSE41-NEXT:    movq %rdi, %rax
@@ -1074,7 +1471,7 @@ define <64 x i32> @mulhuw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {
 ; SSE41-NEXT:    movdqa %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
 ;
-; AVX2-LABEL: mulhuw_v64i16_lshr:
+; AVX2-LABEL: zext_mulhuw_v64i16_lshr:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    movq %rdi, %rax
 ; AVX2-NEXT:    vpmulhuw %ymm4, %ymm0, %ymm0
@@ -1104,7 +1501,7 @@ define <64 x i32> @mulhuw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: mulhuw_v64i16_lshr:
+; AVX512F-LABEL: zext_mulhuw_v64i16_lshr:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm4
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
@@ -1122,7 +1519,7 @@ define <64 x i32> @mulhuw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {
 ; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
 ; AVX512F-NEXT:    retq
 ;
-; AVX512BW-LABEL: mulhuw_v64i16_lshr:
+; AVX512BW-LABEL: zext_mulhuw_v64i16_lshr:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpmulhuw %zmm2, %zmm0, %zmm2
 ; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
@@ -1504,8 +1901,8 @@ define <64 x i32> @mulhsw_v64i16_ashr(<64 x i16> %a, <64 x i16> %b) {
   ret <64 x i32> %d
 }
 
-define <8 x i64> @mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
-; SSE2-LABEL: mulhuw_v8i16_lshr_i64:
+define <8 x i64> @zext_mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
+; SSE2-LABEL: zext_mulhuw_v8i16_lshr_i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1533,7 +1930,7 @@ define <8 x i64> @mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: mulhuw_v8i16_lshr_i64:
+; SSE41-LABEL: zext_mulhuw_v8i16_lshr_i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhuw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
@@ -1546,7 +1943,7 @@ define <8 x i64> @mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX2-LABEL: mulhuw_v8i16_lshr_i64:
+; AVX2-LABEL: zext_mulhuw_v8i16_lshr_i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
@@ -1554,7 +1951,7 @@ define <8 x i64> @mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: mulhuw_v8i16_lshr_i64:
+; AVX512-LABEL: zext_mulhuw_v8i16_lshr_i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
@@ -1566,8 +1963,8 @@ define <8 x i64> @mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
   ret <8 x i64> %d
 }
 
-define <8 x i64> @mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
-; SSE2-LABEL: mulhsw_v8i16_lshr_i64:
+define <8 x i64> @sext_mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
+; SSE2-LABEL: sext_mulhsw_v8i16_lshr_i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
 ; SSE2-NEXT:    psrad $16, %xmm6
@@ -1635,7 +2032,7 @@ define <8 x i64> @mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-NEXT:    psrlq $16, %xmm3
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: mulhsw_v8i16_lshr_i64:
+; SSE41-LABEL: sext_mulhsw_v8i16_lshr_i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
@@ -1648,7 +2045,7 @@ define <8 x i64> @mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX2-LABEL: mulhsw_v8i16_lshr_i64:
+; AVX2-LABEL: sext_mulhsw_v8i16_lshr_i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
@@ -1656,7 +2053,7 @@ define <8 x i64> @mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: mulhsw_v8i16_lshr_i64:
+; AVX512-LABEL: sext_mulhsw_v8i16_lshr_i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
@@ -1668,8 +2065,8 @@ define <8 x i64> @mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {
   ret <8 x i64> %d
 }
 
-define <8 x i64> @mulhsw_v8i16_ashr_i64(<8 x i16> %a, <8 x i16> %b) {
-; SSE2-LABEL: mulhsw_v8i16_ashr_i64:
+define <8 x i64> @sext_mulhsw_v8i16_ashr_i64(<8 x i16> %a, <8 x i16> %b) {
+; SSE2-LABEL: sext_mulhsw_v8i16_ashr_i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
 ; SSE2-NEXT:    psrad $16, %xmm5
@@ -1757,7 +2154,7 @@ define <8 x i64> @mulhsw_v8i16_ashr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: mulhsw_v8i16_ashr_i64:
+; SSE41-LABEL: sext_mulhsw_v8i16_ashr_i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmulhw %xmm1, %xmm0
 ; SSE41-NEXT:    pmovsxwq %xmm0, %xmm4
@@ -1770,7 +2167,7 @@ define <8 x i64> @mulhsw_v8i16_ashr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX2-LABEL: mulhsw_v8i16_ashr_i64:
+; AVX2-LABEL: sext_mulhsw_v8i16_ashr_i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm0
@@ -1778,7 +2175,7 @@ define <8 x i64> @mulhsw_v8i16_ashr_i64(<8 x i16> %a, <8 x i16> %b) {
 ; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: mulhsw_v8i16_ashr_i64:
+; AVX512-LABEL: sext_mulhsw_v8i16_ashr_i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpmovsxwq %xmm0, %zmm0
diff --git a/llvm/test/CodeGen/X86/pointer-vector.ll b/llvm/test/CodeGen/X86/pointer-vector.ll
index 67b389d2e5b6625e337e30fe4ef7e26ac6e84b8b..04e2a2a350c45535c6a2d6c74268ff0fb70339d6 100644
--- a/llvm/test/CodeGen/X86/pointer-vector.ll
+++ b/llvm/test/CodeGen/X86/pointer-vector.ll
@@ -133,7 +133,7 @@ define <4 x i32> @ICMP0(<4 x i8*>* %p0, <4 x i8*>* %p1) nounwind {
 ; CHECK-NEXT:    movdqa (%ecx), %xmm0
 ; CHECK-NEXT:    pcmpgtd (%eax), %xmm0
 ; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [9,8,7,6]
-; CHECK-NEXT:    blendvps %xmm0, {{\.LCPI.*}}, %xmm1
+; CHECK-NEXT:    blendvps %xmm0, {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; CHECK-NEXT:    movaps %xmm1, %xmm0
 ; CHECK-NEXT:    retl
 entry:
@@ -152,7 +152,7 @@ define <4 x i32> @ICMP1(<4 x i8*>* %p0, <4 x i8*>* %p1) nounwind {
 ; CHECK-NEXT:    movdqa (%ecx), %xmm0
 ; CHECK-NEXT:    pcmpeqd (%eax), %xmm0
 ; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [9,8,7,6]
-; CHECK-NEXT:    blendvps %xmm0, {{\.LCPI.*}}, %xmm1
+; CHECK-NEXT:    blendvps %xmm0, {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; CHECK-NEXT:    movaps %xmm1, %xmm0
 ; CHECK-NEXT:    retl
 entry:
diff --git a/llvm/test/CodeGen/X86/popcnt.ll b/llvm/test/CodeGen/X86/popcnt.ll
index 3fe9871bae65b2bb1520fc8e35834fff278f0da6..fc35ac4780714b6b95837c51197631847598c2e2 100644
--- a/llvm/test/CodeGen/X86/popcnt.ll
+++ b/llvm/test/CodeGen/X86/popcnt.ll
@@ -253,7 +253,7 @@ define i64 @cnt64(i64 %x) nounwind readnone {
 ; X86-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $1, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psubb %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
@@ -264,7 +264,7 @@ define i64 @cnt64(i64 %x) nounwind readnone {
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
 ; X86-SSE2-NEXT:    paddb %xmm0, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm0
 ; X86-SSE2-NEXT:    psadbw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movd %xmm0, %eax
@@ -749,7 +749,7 @@ define i64 @cnt64_optsize(i64 %x) nounwind readnone optsize {
 ; X86-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $1, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psubb %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
@@ -760,7 +760,7 @@ define i64 @cnt64_optsize(i64 %x) nounwind readnone optsize {
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
 ; X86-SSE2-NEXT:    paddb %xmm0, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm0
 ; X86-SSE2-NEXT:    psadbw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movd %xmm0, %eax
@@ -1178,7 +1178,7 @@ define i64 @cnt64_pgso(i64 %x) nounwind readnone !prof !14 {
 ; X86-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $1, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psubb %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
@@ -1189,7 +1189,7 @@ define i64 @cnt64_pgso(i64 %x) nounwind readnone !prof !14 {
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
 ; X86-SSE2-NEXT:    paddb %xmm0, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm0, %xmm0
 ; X86-SSE2-NEXT:    psadbw %xmm1, %xmm0
 ; X86-SSE2-NEXT:    movd %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/pr15309.ll b/llvm/test/CodeGen/X86/pr15309.ll
index e154af43b56856123728caad963291ccc4c5b9ee..91dfdf5bdf0ba15a13488cd50b767db452af1e06 100644
--- a/llvm/test/CodeGen/X86/pr15309.ll
+++ b/llvm/test/CodeGen/X86/pr15309.ll
@@ -19,10 +19,10 @@ define void @test_convert_float2_ulong2(<2 x i64>* nocapture %src, <2 x float>*
 ; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    shrl $31, %ecx
 ; CHECK-NEXT:    fildll (%esp)
-; CHECK-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; CHECK-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; CHECK-NEXT:    shrl $31, %esi
 ; CHECK-NEXT:    fildll {{[0-9]+}}(%esp)
-; CHECK-NEXT:    fadds {{\.LCPI.*}}(,%esi,4)
+; CHECK-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%esi,4)
 ; CHECK-NEXT:    fstps 84(%eax)
 ; CHECK-NEXT:    fstps 80(%eax)
 ; CHECK-NEXT:    addl $20, %esp
diff --git a/llvm/test/CodeGen/X86/pr18054.ll b/llvm/test/CodeGen/X86/pr18054.ll
index b7af516180479ddb08216f29854fd94b3036d3e0..0c5079a931700193dd870a0e50fb66852850ccd3 100644
--- a/llvm/test/CodeGen/X86/pr18054.ll
+++ b/llvm/test/CodeGen/X86/pr18054.ll
@@ -1,10 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mcpu=penryn | FileCheck %s
 
 define void @foo(<16 x i32>* %p, <16 x i1> %x) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; CHECK-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; CHECK-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; CHECK-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; CHECK-NEXT:    pslld $31, %xmm0
+; CHECK-NEXT:    psrad $31, %xmm0
+; CHECK-NEXT:    pslld $31, %xmm3
+; CHECK-NEXT:    psrad $31, %xmm3
+; CHECK-NEXT:    pslld $31, %xmm2
+; CHECK-NEXT:    psrad $31, %xmm2
+; CHECK-NEXT:    pslld $31, %xmm1
+; CHECK-NEXT:    psrad $31, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, (%rdi)
+; CHECK-NEXT:    movdqa %xmm2, 48(%rdi)
+; CHECK-NEXT:    movdqa %xmm3, 32(%rdi)
+; CHECK-NEXT:    movdqa %xmm0, 16(%rdi)
+; CHECK-NEXT:    retq
   %ret = sext <16 x i1> %x to <16 x i32>
   store <16 x i32> %ret, <16 x i32>* %p
   ret void
-; CHECK: foo
-; CHECK-NOT: pmovsxbd
-; CHECK: ret
 }
diff --git a/llvm/test/CodeGen/X86/pr34080-2.ll b/llvm/test/CodeGen/X86/pr34080-2.ll
index b09f2a274bb04512ff2c8b1850c4233b38950f2c..dee28859924736e24ac981725f0dcb9dd6b611ef 100644
--- a/llvm/test/CodeGen/X86/pr34080-2.ll
+++ b/llvm/test/CodeGen/X86/pr34080-2.ll
@@ -48,8 +48,8 @@ define void @computeJD(%struct.DateTime*) nounwind {
 ; CHECK-NEXT:    leal 257(%ecx,%edx), %eax
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    fildl {{[0-9]+}}(%esp)
-; CHECK-NEXT:    fadds {{\.LCPI.*}}
-; CHECK-NEXT:    fmuls {{\.LCPI.*}}
+; CHECK-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}
+; CHECK-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
@@ -62,7 +62,7 @@ define void @computeJD(%struct.DateTime*) nounwind {
 ; CHECK-NEXT:    imull $60000, 24(%ebx), %ecx # imm = 0xEA60
 ; CHECK-NEXT:    addl %eax, %ecx
 ; CHECK-NEXT:    fldl 28(%ebx)
-; CHECK-NEXT:    fmuls {{\.LCPI.*}}
+; CHECK-NEXT:    fmuls {{\.LCPI[0-9]+_[0-9]+}}
 ; CHECK-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    orl $3072, %eax # imm = 0xC00
diff --git a/llvm/test/CodeGen/X86/pr34605.ll b/llvm/test/CodeGen/X86/pr34605.ll
index 4c1a3d7781e25137797f37b3c8ea4083e2bfab20..2bd9c03be46c87972086f1b770dcd69851397698 100644
--- a/llvm/test/CodeGen/X86/pr34605.ll
+++ b/llvm/test/CodeGen/X86/pr34605.ll
@@ -6,18 +6,18 @@ define void @pr34605(i8* nocapture %s, i32 %p) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    vpbroadcastd {{[0-9]+}}(%esp), %zmm0
-; CHECK-NEXT:    vpcmpeqd {{\.LCPI.*}}, %zmm0, %k0
-; CHECK-NEXT:    vpcmpeqd {{\.LCPI.*}}, %zmm0, %k1
+; CHECK-NEXT:    vpcmpeqd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %k0
+; CHECK-NEXT:    vpcmpeqd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %k1
 ; CHECK-NEXT:    kunpckwd %k0, %k1, %k0
-; CHECK-NEXT:    vpcmpeqd {{\.LCPI.*}}, %zmm0, %k1
-; CHECK-NEXT:    vpcmpeqd {{\.LCPI.*}}, %zmm0, %k2
+; CHECK-NEXT:    vpcmpeqd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %k1
+; CHECK-NEXT:    vpcmpeqd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %k2
 ; CHECK-NEXT:    kunpckwd %k1, %k2, %k1
 ; CHECK-NEXT:    kunpckdq %k0, %k1, %k0
 ; CHECK-NEXT:    movl $1, %ecx
 ; CHECK-NEXT:    kmovd %ecx, %k1
 ; CHECK-NEXT:    kmovd %k1, %k1
 ; CHECK-NEXT:    kandq %k1, %k0, %k1
-; CHECK-NEXT:    vmovdqu8 {{\.LCPI.*}}, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vmovdqu8 {{\.LCPI[0-9]+_[0-9]+}}, %zmm0 {%k1} {z}
 ; CHECK-NEXT:    vmovdqu64 %zmm0, (%eax)
 ; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovdqu64 %zmm0, 64(%eax)
diff --git a/llvm/test/CodeGen/X86/pr40539.ll b/llvm/test/CodeGen/X86/pr40539.ll
index a727e83cdca737b41a3541220f7d5f226e864d46..8f98a7a35fa28c73f8021fe3c3d7f4a6254b9547 100644
--- a/llvm/test/CodeGen/X86/pr40539.ll
+++ b/llvm/test/CodeGen/X86/pr40539.ll
@@ -41,7 +41,7 @@ define zeroext i1 @_Z8test_cosv() {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 12
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT:    divss {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    divss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    #APP
@@ -51,7 +51,7 @@ define zeroext i1 @_Z8test_cosv() {
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    ucomiss %xmm0, %xmm1
 ; CHECK-NEXT:    setae %cl
-; CHECK-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    ucomiss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK-NEXT:    setae %al
 ; CHECK-NEXT:    andb %cl, %al
 ; CHECK-NEXT:    addl $8, %esp
diff --git a/llvm/test/CodeGen/X86/pr40891.ll b/llvm/test/CodeGen/X86/pr40891.ll
index d67739767b2182c0e27a12ba638e910549d471e3..1455f72e810aaf1599fb2f0e30f915e2417f6972 100644
--- a/llvm/test/CodeGen/X86/pr40891.ll
+++ b/llvm/test/CodeGen/X86/pr40891.ll
@@ -7,7 +7,7 @@ define <8 x i32> @foo(<8 x i64> %x, <4 x i64> %y) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; CHECK-NEXT:    vandps {{\.LCPI.*}}, %ymm1, %ymm1
+; CHECK-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1
 ; CHECK-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; CHECK-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
diff --git a/llvm/test/CodeGen/X86/pr46527.ll b/llvm/test/CodeGen/X86/pr46527.ll
index 48b1095a6329a75e0ccb91fefec4e566ac881092..076d8137120f6885b51d368886c9a021f909aef8 100644
--- a/llvm/test/CodeGen/X86/pr46527.ll
+++ b/llvm/test/CodeGen/X86/pr46527.ll
@@ -22,7 +22,7 @@ define void @f(<16 x i8>* %out, <16 x i8> %in, i1 %flag) {
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; CHECK-NEXT:    paddb %xmm1, %xmm1
 ; CHECK-NEXT:    pxor %xmm0, %xmm1
-; CHECK-NEXT:    pxor {{\.LCPI.*}}@GOTOFF(%eax), %xmm1
+; CHECK-NEXT:    pxor {{\.LCPI[0-9]+_[0-9]+}}@GOTOFF(%eax), %xmm1
 ; CHECK-NEXT:    movdqa %xmm1, (%ecx)
 ; CHECK-NEXT:    retl
 entry:
diff --git a/llvm/test/CodeGen/X86/pr47299.ll b/llvm/test/CodeGen/X86/pr47299.ll
index 2f5d07802c7c18f5f11fe192a3fd84281521ab15..93710b87751e0a60c4382536abd6326bfb4950d0 100644
--- a/llvm/test/CodeGen/X86/pr47299.ll
+++ b/llvm/test/CodeGen/X86/pr47299.ll
@@ -13,7 +13,7 @@ define <7 x i1> @create_mask7(i64 %0) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mov rax, rdi
 ; CHECK-NEXT:    vpbroadcastq zmm0, rsi
-; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kshiftrb k1, k0, 6
 ; CHECK-NEXT:    kmovd r8d, k1
 ; CHECK-NEXT:    kshiftrb k1, k0, 5
@@ -57,8 +57,8 @@ define <16 x i1> @create_mask16(i64 %0) {
 ; CHECK-LABEL: create_mask16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpbroadcastq zmm0, rdi
-; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckbw k0, k1, k0
 ; CHECK-NEXT:    vpmovm2b xmm0, k0
 ; CHECK-NEXT:    vzeroupper
@@ -71,11 +71,11 @@ define <32 x i1> @create_mask32(i64 %0) {
 ; CHECK-LABEL: create_mask32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpbroadcastq zmm0, rdi
-; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
-; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckbw k0, k1, k0
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckbw k1, k1, k2
 ; CHECK-NEXT:    kunpckwd k0, k1, k0
 ; CHECK-NEXT:    vpmovm2b ymm0, k0
@@ -88,18 +88,18 @@ define <64 x i1> @create_mask64(i64 %0) {
 ; CHECK-LABEL: create_mask64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpbroadcastq zmm0, rdi
-; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
-; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckbw k0, k1, k0
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckbw k1, k1, k2
-; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckwd k0, k1, k0
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckbw k1, k1, k2
-; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
-; CHECK-NEXT:    vpcmpnleuq k3, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT:    vpcmpnleuq k3, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckbw k2, k3, k2
 ; CHECK-NEXT:    kunpckwd k1, k2, k1
 ; CHECK-NEXT:    kunpckdq k0, k1, k0
@@ -113,7 +113,7 @@ define <16 x i1> @create_mask16_i32(i32 %0) {
 ; CHECK-LABEL: create_mask16_i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpbroadcastd zmm0, edi
-; CHECK-NEXT:    vpcmpnleud k0, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleud k0, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    vpmovm2b xmm0, k0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret
@@ -125,11 +125,11 @@ define <64 x i1> @create_mask64_i32(i32 %0) {
 ; CHECK-LABEL: create_mask64_i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpbroadcastd zmm0, edi
-; CHECK-NEXT:    vpcmpnleud k0, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
-; CHECK-NEXT:    vpcmpnleud k1, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
-; CHECK-NEXT:    vpcmpnleud k2, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleud k0, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT:    vpcmpnleud k1, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT:    vpcmpnleud k2, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckwd k0, k1, k0
-; CHECK-NEXT:    vpcmpnleud k1, zmm0, zmmword ptr [rip + {{\.LCPI.*}}]
+; CHECK-NEXT:    vpcmpnleud k1, zmm0, zmmword ptr [rip + {{\.LCPI[0-9]+_[0-9]+}}]
 ; CHECK-NEXT:    kunpckwd k1, k1, k2
 ; CHECK-NEXT:    kunpckdq k0, k1, k0
 ; CHECK-NEXT:    vpmovm2b zmm0, k0
diff --git a/llvm/test/CodeGen/X86/pr49076.ll b/llvm/test/CodeGen/X86/pr49076.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c430ec4ff95450f4c17bef030d0a7fa2305f47c3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr49076.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -O0 -mattr=avx | FileCheck %s
+
+define void @foo() {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %BB
+; CHECK-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:  # %bb.1: # %BB0
+; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vpslld $31, %xmm0, %xmm0
+; CHECK-NEXT:    vpsrad $31, %xmm0, %xmm1
+; CHECK-NEXT:    vpmovsxdq %xmm1, %xmm2
+; CHECK-NEXT:    # implicit-def: $ymm0
+; CHECK-NEXT:    vmovaps %xmm2, %xmm0
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; CHECK-NEXT:    vpmovsxdq %xmm1, %xmm1
+; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-NEXT:  # %bb.2: # %BB1
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+BB:
+  %A1 = alloca i1, align 1
+  %L1 = load i1, i1* %A1, align 1
+  %Cmp = icmp ugt <4 x i64> zeroinitializer, zeroinitializer
+  br label %BB0
+
+BB0:
+  %Se = sext <4 x i1> %Cmp to <4 x i64>
+  br label %BB1
+
+BB1:
+  %Sl = select i1 %L1, <4 x i64> %Se, <4 x i64> zeroinitializer
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/psubus.ll b/llvm/test/CodeGen/X86/psubus.ll
index ea3a89fdef4cb37d4f714f3ff9048951aefa793f..58970d49583f08b911f4e624baac76b5bd249d4f 100644
--- a/llvm/test/CodeGen/X86/psubus.ll
+++ b/llvm/test/CodeGen/X86/psubus.ll
@@ -702,15 +702,13 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm5
 ; AVX2-NEXT:    vpackssdw %xmm5, %xmm3, %xmm3
 ; AVX2-NEXT:    vpacksswb %xmm3, %xmm4, %xmm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm5, %xmm1, %xmm1
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
 ; AVX2-NEXT:    vpsubb %xmm0, %xmm1, %xmm0
 ; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
 ; AVX2-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/rotate-extract-vector.ll b/llvm/test/CodeGen/X86/rotate-extract-vector.ll
index 7b7feb3372f2ae01aec4dfe05114349c8f42fd9e..cc3664176e3513b58b5904ad21a21f9f6d8fae6b 100644
--- a/llvm/test/CodeGen/X86/rotate-extract-vector.ll
+++ b/llvm/test/CodeGen/X86/rotate-extract-vector.ll
@@ -109,7 +109,7 @@ define <4 x i32> @vrolw_extract_mul_with_mask(<4 x i32> %i) nounwind {
 ; X86-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [9,9,9,9]
 ; X86-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vprold $7, %zmm0, %zmm0
-; X86-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ;
@@ -132,8 +132,8 @@ define <4 x i32> @vrolw_extract_mul_with_mask(<4 x i32> %i) nounwind {
 define <32 x i16> @illegal_no_extract_mul(<32 x i16> %i) nounwind {
 ; X86-LABEL: illegal_no_extract_mul:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmullw {{\.LCPI.*}}, %zmm0, %zmm1
-; X86-NEXT:    vpmullw {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vpmullw {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm1
+; X86-NEXT:    vpmullw {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; X86-NEXT:    vpsrlw $10, %zmm0, %zmm0
 ; X86-NEXT:    vporq %zmm0, %zmm1, %zmm0
 ; X86-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll b/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll
index 0ee1cac1fa55f8e4d8fd1d44a9cb9776d12318cb..f4d08960be0a339705a5fd3aa7e080d3a3849968 100644
--- a/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll
+++ b/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll
@@ -267,7 +267,7 @@ define i64 @f_to_u64(float %a) nounwind {
 ; X87-LIN:       # %bb.0:
 ; X87-LIN-NEXT:    subl $20, %esp
 ; X87-LIN-NEXT:    flds {{[0-9]+}}(%esp)
-; X87-LIN-NEXT:    flds {{\.LCPI.*}}
+; X87-LIN-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-LIN-NEXT:    fucom %st(1)
 ; X87-LIN-NEXT:    fnstsw %ax
 ; X87-LIN-NEXT:    xorl %edx, %edx
@@ -691,7 +691,7 @@ define i64 @d_to_u64(double %a) nounwind {
 ; X87-LIN:       # %bb.0:
 ; X87-LIN-NEXT:    subl $20, %esp
 ; X87-LIN-NEXT:    fldl {{[0-9]+}}(%esp)
-; X87-LIN-NEXT:    flds {{\.LCPI.*}}
+; X87-LIN-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-LIN-NEXT:    fucom %st(1)
 ; X87-LIN-NEXT:    fnstsw %ax
 ; X87-LIN-NEXT:    xorl %edx, %edx
@@ -914,7 +914,7 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X86-AVX512-LIN:       # %bb.0:
 ; X86-AVX512-LIN-NEXT:    subl $12, %esp
 ; X86-AVX512-LIN-NEXT:    fldt {{[0-9]+}}(%esp)
-; X86-AVX512-LIN-NEXT:    flds {{\.LCPI.*}}
+; X86-AVX512-LIN-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-AVX512-LIN-NEXT:    xorl %edx, %edx
 ; X86-AVX512-LIN-NEXT:    fucomi %st(1), %st
 ; X86-AVX512-LIN-NEXT:    fldz
@@ -990,7 +990,7 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X86-SSE3-LIN:       # %bb.0:
 ; X86-SSE3-LIN-NEXT:    subl $12, %esp
 ; X86-SSE3-LIN-NEXT:    fldt {{[0-9]+}}(%esp)
-; X86-SSE3-LIN-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE3-LIN-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE3-LIN-NEXT:    xorl %edx, %edx
 ; X86-SSE3-LIN-NEXT:    fucomi %st(1), %st
 ; X86-SSE3-LIN-NEXT:    fldz
@@ -1072,7 +1072,7 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X86-SSE2-LIN:       # %bb.0:
 ; X86-SSE2-LIN-NEXT:    subl $20, %esp
 ; X86-SSE2-LIN-NEXT:    fldt {{[0-9]+}}(%esp)
-; X86-SSE2-LIN-NEXT:    flds {{\.LCPI.*}}
+; X86-SSE2-LIN-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X86-SSE2-LIN-NEXT:    xorl %edx, %edx
 ; X86-SSE2-LIN-NEXT:    fucomi %st(1), %st
 ; X86-SSE2-LIN-NEXT:    setbe %dl
@@ -1180,7 +1180,7 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X87-LIN:       # %bb.0:
 ; X87-LIN-NEXT:    subl $20, %esp
 ; X87-LIN-NEXT:    fldt {{[0-9]+}}(%esp)
-; X87-LIN-NEXT:    flds {{\.LCPI.*}}
+; X87-LIN-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; X87-LIN-NEXT:    fucom %st(1)
 ; X87-LIN-NEXT:    fnstsw %ax
 ; X87-LIN-NEXT:    xorl %edx, %edx
diff --git a/llvm/test/CodeGen/X86/scalar-int-to-fp.ll b/llvm/test/CodeGen/X86/scalar-int-to-fp.ll
index 9f74ad0599e4a1f69a7ab24cb066afc60d842bac..eedf744f159254354f1558a95b65b048d7f7643f 100644
--- a/llvm/test/CodeGen/X86/scalar-int-to-fp.ll
+++ b/llvm/test/CodeGen/X86/scalar-int-to-fp.ll
@@ -33,8 +33,8 @@ define float @u32_to_f(i32 %a) nounwind {
 ; SSE2_32:       # %bb.0:
 ; SSE2_32-NEXT:    pushl %eax
 ; SSE2_32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2_32-NEXT:    orpd {{\.LCPI.*}}, %xmm0
-; SSE2_32-NEXT:    subsd {{\.LCPI.*}}, %xmm0
+; SSE2_32-NEXT:    orpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; SSE2_32-NEXT:    subsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE2_32-NEXT:    cvtsd2ss %xmm0, %xmm0
 ; SSE2_32-NEXT:    movss %xmm0, (%esp)
 ; SSE2_32-NEXT:    flds (%esp)
@@ -147,8 +147,8 @@ define double @u32_to_d(i32 %a) nounwind {
 ; SSE2_32-NEXT:    andl $-8, %esp
 ; SSE2_32-NEXT:    subl $8, %esp
 ; SSE2_32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2_32-NEXT:    orpd {{\.LCPI.*}}, %xmm0
-; SSE2_32-NEXT:    subsd {{\.LCPI.*}}, %xmm0
+; SSE2_32-NEXT:    orpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; SSE2_32-NEXT:    subsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE2_32-NEXT:    movsd %xmm0, (%esp)
 ; SSE2_32-NEXT:    fldl (%esp)
 ; SSE2_32-NEXT:    movl %ebp, %esp
@@ -333,7 +333,7 @@ define float @u64_to_f(i64 %a) nounwind {
 ; AVX512F_32-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
 ; AVX512F_32-NEXT:    shrl $31, %eax
 ; AVX512F_32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX512F_32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX512F_32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX512F_32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; AVX512F_32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX512F_32-NEXT:    vmovss %xmm0, (%esp)
@@ -353,7 +353,7 @@ define float @u64_to_f(i64 %a) nounwind {
 ; SSE2_32-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
 ; SSE2_32-NEXT:    shrl $31, %eax
 ; SSE2_32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE2_32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE2_32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE2_32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; SSE2_32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2_32-NEXT:    movss %xmm0, (%esp)
@@ -392,7 +392,7 @@ define float @u64_to_f(i64 %a) nounwind {
 ; SSE1_32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; SSE1_32-NEXT:    shrl $31, %ecx
 ; SSE1_32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE1_32-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; SSE1_32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; SSE1_32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; SSE1_32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE1_32-NEXT:    movss %xmm0, (%esp)
@@ -413,7 +413,7 @@ define float @u64_to_f(i64 %a) nounwind {
 ; X87-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; X87-NEXT:    shrl $31, %ecx
 ; X87-NEXT:    fildll {{[0-9]+}}(%esp)
-; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; X87-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X87-NEXT:    flds {{[0-9]+}}(%esp)
 ; X87-NEXT:    movl %ebp, %esp
@@ -652,7 +652,7 @@ define double @u64_to_d(i64 %a) nounwind {
 ; AVX512F_32-NEXT:    subl $8, %esp
 ; AVX512F_32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX512F_32-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; AVX512F_32-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512F_32-NEXT:    vsubpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512F_32-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512F_32-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
 ; AVX512F_32-NEXT:    vmovsd %xmm0, (%esp)
@@ -669,7 +669,7 @@ define double @u64_to_d(i64 %a) nounwind {
 ; SSE2_32-NEXT:    subl $8, %esp
 ; SSE2_32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE2_32-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; SSE2_32-NEXT:    subpd {{\.LCPI.*}}, %xmm0
+; SSE2_32-NEXT:    subpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE2_32-NEXT:    movapd %xmm0, %xmm1
 ; SSE2_32-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2_32-NEXT:    addsd %xmm0, %xmm1
@@ -701,7 +701,7 @@ define double @u64_to_d(i64 %a) nounwind {
 ; SSE1_32-NEXT:    movl %eax, (%esp)
 ; SSE1_32-NEXT:    shrl $31, %ecx
 ; SSE1_32-NEXT:    fildll (%esp)
-; SSE1_32-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; SSE1_32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; SSE1_32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; SSE1_32-NEXT:    fldl {{[0-9]+}}(%esp)
 ; SSE1_32-NEXT:    movl %ebp, %esp
@@ -720,7 +720,7 @@ define double @u64_to_d(i64 %a) nounwind {
 ; X87-NEXT:    movl %eax, (%esp)
 ; X87-NEXT:    shrl $31, %ecx
 ; X87-NEXT:    fildll (%esp)
-; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; X87-NEXT:    fldl {{[0-9]+}}(%esp)
 ; X87-NEXT:    movl %ebp, %esp
@@ -774,7 +774,7 @@ define double @u64_to_d_optsize(i64 %a) nounwind optsize {
 ; AVX512F_32-NEXT:    subl $8, %esp
 ; AVX512F_32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX512F_32-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; AVX512F_32-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512F_32-NEXT:    vsubpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512F_32-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
 ; AVX512F_32-NEXT:    vmovlpd %xmm0, (%esp)
 ; AVX512F_32-NEXT:    fldl (%esp)
@@ -790,7 +790,7 @@ define double @u64_to_d_optsize(i64 %a) nounwind optsize {
 ; SSE2_32-NEXT:    subl $8, %esp
 ; SSE2_32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE2_32-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; SSE2_32-NEXT:    subpd {{\.LCPI.*}}, %xmm0
+; SSE2_32-NEXT:    subpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE2_32-NEXT:    movapd %xmm0, %xmm1
 ; SSE2_32-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
 ; SSE2_32-NEXT:    addsd %xmm0, %xmm1
@@ -822,7 +822,7 @@ define double @u64_to_d_optsize(i64 %a) nounwind optsize {
 ; SSE1_32-NEXT:    movl %eax, (%esp)
 ; SSE1_32-NEXT:    shrl $31, %ecx
 ; SSE1_32-NEXT:    fildll (%esp)
-; SSE1_32-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; SSE1_32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; SSE1_32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; SSE1_32-NEXT:    fldl {{[0-9]+}}(%esp)
 ; SSE1_32-NEXT:    movl %ebp, %esp
@@ -841,7 +841,7 @@ define double @u64_to_d_optsize(i64 %a) nounwind optsize {
 ; X87-NEXT:    movl %eax, (%esp)
 ; X87-NEXT:    shrl $31, %ecx
 ; X87-NEXT:    fildll (%esp)
-; X87-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; X87-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; X87-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; X87-NEXT:    fldl {{[0-9]+}}(%esp)
 ; X87-NEXT:    movl %ebp, %esp
@@ -1076,7 +1076,7 @@ define x86_fp80 @u64_to_x(i64 %a) nounwind {
 ; CHECK32-NEXT:    movl %eax, (%esp)
 ; CHECK32-NEXT:    shrl $31, %ecx
 ; CHECK32-NEXT:    fildll (%esp)
-; CHECK32-NEXT:    fadds {{\.LCPI.*}}(,%ecx,4)
+; CHECK32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; CHECK32-NEXT:    movl %ebp, %esp
 ; CHECK32-NEXT:    popl %ebp
 ; CHECK32-NEXT:    retl
@@ -1088,7 +1088,7 @@ define x86_fp80 @u64_to_x(i64 %a) nounwind {
 ; CHECK64-NEXT:    testq %rdi, %rdi
 ; CHECK64-NEXT:    sets %al
 ; CHECK64-NEXT:    fildll -{{[0-9]+}}(%rsp)
-; CHECK64-NEXT:    fadds {{\.LCPI.*}}(,%rax,4)
+; CHECK64-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%rax,4)
 ; CHECK64-NEXT:    retq
   %r = uitofp i64 %a to x86_fp80
   ret x86_fp80 %r
diff --git a/llvm/test/CodeGen/X86/select-of-fp-constants.ll b/llvm/test/CodeGen/X86/select-of-fp-constants.ll
index 0a3e9f93c33e8e92a9705e08c741cf32fd50ba70..934f0f8b9465e8abb6b47e55a0f4f1373dde338a 100644
--- a/llvm/test/CodeGen/X86/select-of-fp-constants.ll
+++ b/llvm/test/CodeGen/X86/select-of-fp-constants.ll
@@ -16,7 +16,7 @@ define float @icmp_select_fp_constants(i32 %x) nounwind readnone {
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    sete %al
-; X86-NEXT:    flds {{\.LCPI.*}}(,%eax,4)
+; X86-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-NEXT:    retl
 ;
 ; X64-SSE-LABEL: icmp_select_fp_constants:
@@ -46,7 +46,7 @@ define float @fcmp_select_fp_constants(float %x) nounwind readnone {
 ; X86-SSE-NEXT:    cmpneqss {{[0-9]+}}(%esp), %xmm0
 ; X86-SSE-NEXT:    movd %xmm0, %eax
 ; X86-SSE-NEXT:    andl $1, %eax
-; X86-SSE-NEXT:    flds {{\.LCPI.*}}(,%eax,4)
+; X86-SSE-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-SSE-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: fcmp_select_fp_constants:
@@ -55,15 +55,15 @@ define float @fcmp_select_fp_constants(float %x) nounwind readnone {
 ; X86-AVX2-NEXT:    vcmpneqss {{[0-9]+}}(%esp), %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    andl $1, %eax
-; X86-AVX2-NEXT:    flds {{\.LCPI.*}}(,%eax,4)
+; X86-AVX2-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-AVX2-NEXT:    retl
 ;
 ; X86-AVX512F-LABEL: fcmp_select_fp_constants:
 ; X86-AVX512F:       # %bb.0:
 ; X86-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-AVX512F-NEXT:    vcmpneqss {{\.LCPI.*}}, %xmm0, %k0
+; X86-AVX512F-NEXT:    vcmpneqss {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %k0
 ; X86-AVX512F-NEXT:    kmovw %k0, %eax
-; X86-AVX512F-NEXT:    flds {{\.LCPI.*}}(,%eax,4)
+; X86-AVX512F-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-AVX512F-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fcmp_select_fp_constants:
diff --git a/llvm/test/CodeGen/X86/select.ll b/llvm/test/CodeGen/X86/select.ll
index ebd5c5495a57e6abc207e4fb687b3cb718133e4b..34c7911220c424e0583f59caead16b8fbb2b109b 100644
--- a/llvm/test/CodeGen/X86/select.ll
+++ b/llvm/test/CodeGen/X86/select.ll
@@ -102,7 +102,7 @@ define i32 @test2() nounwind {
 ;
 ; MCU-LABEL: test2:
 ; MCU:       # %bb.0: # %entry
-; MCU-NEXT:    calll return_false
+; MCU-NEXT:    calll return_false@PLT
 ; MCU-NEXT:    xorl %ecx, %ecx
 ; MCU-NEXT:    testb $1, %al
 ; MCU-NEXT:    jne .LBB1_2
@@ -163,7 +163,7 @@ define float @test3(i32 %x) nounwind readnone {
 ; MCU-NEXT:    xorl %ecx, %ecx
 ; MCU-NEXT:    testl %eax, %eax
 ; MCU-NEXT:    sete %cl
-; MCU-NEXT:    flds {{\.LCPI.*}}(,%ecx,4)
+; MCU-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}(,%ecx,4)
 ; MCU-NEXT:    retl
 entry:
   %0 = icmp eq i32 %x, 0
@@ -197,7 +197,7 @@ define signext i8 @test4(i8* nocapture %P, double %F) nounwind readonly {
 ; MCU:       # %bb.0: # %entry
 ; MCU-NEXT:    movl %eax, %ecx
 ; MCU-NEXT:    fldl {{[0-9]+}}(%esp)
-; MCU-NEXT:    flds {{\.LCPI.*}}
+; MCU-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; MCU-NEXT:    fucompp
 ; MCU-NEXT:    fnstsw %ax
 ; MCU-NEXT:    xorl %edx, %edx
@@ -422,7 +422,7 @@ define x86_fp80 @test7(i32 %tmp8) nounwind {
 ; MCU-NEXT:    notl %eax
 ; MCU-NEXT:    shrl $27, %eax
 ; MCU-NEXT:    andl $-16, %eax
-; MCU-NEXT:    fldt {{\.LCPI.*}}(%eax)
+; MCU-NEXT:    fldt {{\.LCPI[0-9]+_[0-9]+}}(%eax)
 ; MCU-NEXT:    retl
   %tmp9 = icmp sgt i32 %tmp8, -1
   %retval = select i1 %tmp9, x86_fp80 0xK4005B400000000000000, x86_fp80 0xK40078700000000000000
diff --git a/llvm/test/CodeGen/X86/setcc-lowering.ll b/llvm/test/CodeGen/X86/setcc-lowering.ll
index 34f3ad697ad52c30d7c39a39c03735aa8fd0e899..e6c5bffe2dfcfff234d5a59a299ddfbda9f4a86a 100644
--- a/llvm/test/CodeGen/X86/setcc-lowering.ll
+++ b/llvm/test/CodeGen/X86/setcc-lowering.ll
@@ -22,7 +22,7 @@ define <8 x i16> @pr25080(<8 x i32> %a) {
 ; KNL-32-LABEL: pr25080:
 ; KNL-32:       # %bb.0: # %entry
 ; KNL-32-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; KNL-32-NEXT:    vptestnmd {{\.LCPI.*}}{1to16}, %zmm0, %k0
+; KNL-32-NEXT:    vptestnmd {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %k0
 ; KNL-32-NEXT:    movb $15, %al
 ; KNL-32-NEXT:    kmovw %eax, %k1
 ; KNL-32-NEXT:    korw %k1, %k0, %k1
diff --git a/llvm/test/CodeGen/X86/shrink-fp-const2.ll b/llvm/test/CodeGen/X86/shrink-fp-const2.ll
index 670f268b6fa2dad539e97073ecc69a2f6558319e..01d799b589825692b89a01eec0b175d1decca96b 100644
--- a/llvm/test/CodeGen/X86/shrink-fp-const2.ll
+++ b/llvm/test/CodeGen/X86/shrink-fp-const2.ll
@@ -4,7 +4,7 @@
 define x86_fp80 @test2() nounwind  {
 ; CHECK-LABEL: test2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    flds {{\.LCPI.*}}
+; CHECK-NEXT:    flds {{\.LCPI[0-9]+_[0-9]+}}
 ; CHECK-NEXT:    retl
 entry:
 	ret x86_fp80 0xK3FFFC000000000000000
diff --git a/llvm/test/CodeGen/X86/shrink-wrap-chkstk.ll b/llvm/test/CodeGen/X86/shrink-wrap-chkstk.ll
index 32e046df6bb0558db3231c6444bc84b3d263a984..241d42c3227b243d62797223d8ebca50ebf36776 100644
--- a/llvm/test/CodeGen/X86/shrink-wrap-chkstk.ll
+++ b/llvm/test/CodeGen/X86/shrink-wrap-chkstk.ll
@@ -25,7 +25,7 @@ bb1:
   br label %bb2
 
 bb2:
-  call void @inalloca_params(<{ %struct.S }>* inalloca nonnull %argmem)
+  call void @inalloca_params(<{ %struct.S }>* inalloca(<{ %struct.S }>) nonnull %argmem)
   ret void
 }
 
@@ -39,7 +39,7 @@ bb2:
 ; CHECK: popl %ebp
 ; CHECK: retl
 
-declare void @inalloca_params(<{ %struct.S }>* inalloca)
+declare void @inalloca_params(<{ %struct.S }>* inalloca(<{ %struct.S }>))
 
 declare i32 @doSomething(i32, i32*)
 
diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll
index e6660f9e49572ca34a91187b68f7441b24787a9f..177d1206c960810cb633cc95294405eb3caaf90c 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul.ll
@@ -1406,7 +1406,7 @@ define void @mul_2xi8_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; X86-SSE-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movq %xmm0, (%edx,%eax,4)
 ; X86-SSE-NEXT:    retl
 ;
@@ -1418,7 +1418,7 @@ define void @mul_2xi8_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
 ; X86-AVX-NEXT:    vmovd %ecx, %xmm0
 ; X86-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X86-AVX-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1471,7 +1471,7 @@ define void @mul_2xi8_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X86-SSE-NEXT:    movd %ecx, %xmm0
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X86-SSE-NEXT:    psraw $8, %xmm0
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
 ; X86-SSE-NEXT:    psrad $16, %xmm0
 ; X86-SSE-NEXT:    movq %xmm0, (%edx,%eax,4)
@@ -1485,7 +1485,7 @@ define void @mul_2xi8_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
 ; X86-AVX-NEXT:    vmovd %ecx, %xmm0
 ; X86-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1540,7 +1540,7 @@ define void @mul_2xi8_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; X86-SSE-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movq %xmm0, (%edx,%eax,4)
 ; X86-SSE-NEXT:    retl
 ;
@@ -1552,7 +1552,7 @@ define void @mul_2xi8_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
 ; X86-AVX-NEXT:    vmovd %ecx, %xmm0
 ; X86-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X86-AVX-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1621,7 +1621,7 @@ define void @mul_2xi8_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
 ; X86-AVX-NEXT:    vmovd %ecx, %xmm0
 ; X86-AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1693,7 +1693,7 @@ define void @mul_2xi8_varconst5(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
 ; X86-AVX-NEXT:    vmovd %ecx, %xmm0
 ; X86-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1765,7 +1765,7 @@ define void @mul_2xi8_varconst6(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movzwl (%ecx,%eax), %ecx
 ; X86-AVX-NEXT:    vmovd %ecx, %xmm0
 ; X86-AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1833,7 +1833,7 @@ define void @mul_2xi16_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1897,7 +1897,7 @@ define void @mul_2xi16_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1946,12 +1946,9 @@ define void @mul_2xi16_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE-NEXT:    movl c, %edx
 ; X86-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SSE-NEXT:    pxor %xmm1, %xmm1
-; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; X86-SSE-NEXT:    pmuludq {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    pmuludq {{\.LCPI.*}}, %xmm1
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE-NEXT:    psrld $16, %xmm0
+; X86-SSE-NEXT:    pmuludq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    psllq $32, %xmm0
 ; X86-SSE-NEXT:    movq %xmm0, (%edx,%eax,4)
 ; X86-SSE-NEXT:    retl
 ;
@@ -1962,7 +1959,7 @@ define void @mul_2xi16_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1970,12 +1967,9 @@ define void @mul_2xi16_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-SSE-NEXT:    pxor %xmm1, %xmm1
-; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; X64-SSE-NEXT:    psrld $16, %xmm0
 ; X64-SSE-NEXT:    pmuludq {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    pmuludq {{.*}}(%rip), %xmm1
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE-NEXT:    psllq $32, %xmm0
 ; X64-SSE-NEXT:    movq %xmm0, (%rax,%rsi,4)
 ; X64-SSE-NEXT:    retq
 ;
@@ -2012,12 +2006,11 @@ define void @mul_2xi16_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE-NEXT:    movl c, %edx
 ; X86-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X86-SSE-NEXT:    psrad $16, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; X86-SSE-NEXT:    pmuludq {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    pmuludq {{\.LCPI.*}}, %xmm1
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    psllq $32, %xmm0
 ; X86-SSE-NEXT:    movq %xmm0, (%edx,%eax,4)
 ; X86-SSE-NEXT:    retl
 ;
@@ -2028,7 +2021,7 @@ define void @mul_2xi16_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -2036,12 +2029,11 @@ define void @mul_2xi16_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-SSE-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-SSE-NEXT:    psrad $16, %xmm0
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; X64-SSE-NEXT:    pmuludq {{.*}}(%rip), %xmm0
-; X64-SSE-NEXT:    pmuludq {{.*}}(%rip), %xmm1
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE-NEXT:    psllq $32, %xmm0
 ; X64-SSE-NEXT:    movq %xmm0, (%rax,%rsi,4)
 ; X64-SSE-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index a2a2f1a43881dcc660a267d8da62edf7dc11e44d..123fba437141ff036d66c019d8fd0a38681b5ca5 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -1277,10 +1277,8 @@ define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
 ; AVX512VL-LABEL: negative:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm0[2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512VL-NEXT:    vpternlogq $206, %ymm1, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm2[0,3,2,3]
+; AVX512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
+; AVX512VL-NEXT:    vpternlogq $248, {{.*}}(%rip), %xmm1, %xmm0
 ; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/sink-addsub-of-const.ll b/llvm/test/CodeGen/X86/sink-addsub-of-const.ll
index 5c7d4e0717a26d0455635f8cebff62d9345302dd..75241d9ea1b5d016c18007a722c34a5a9fb4d752 100644
--- a/llvm/test/CodeGen/X86/sink-addsub-of-const.ll
+++ b/llvm/test/CodeGen/X86/sink-addsub-of-const.ll
@@ -261,7 +261,7 @@ define <4 x i32> @vec_sink_add_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_add_of_const_to_add0:
 ; X32:       # %bb.0:
 ; X32-NEXT:    paddd %xmm1, %xmm0
-; X32-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: vec_sink_add_of_const_to_add0:
@@ -277,7 +277,7 @@ define <4 x i32> @vec_sink_add_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_add_of_const_to_add1:
 ; X32:       # %bb.0:
 ; X32-NEXT:    paddd %xmm1, %xmm0
-; X32-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: vec_sink_add_of_const_to_add1:
@@ -297,7 +297,7 @@ define <4 x i32> @vec_sink_sub_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_sub_of_const_to_add0:
 ; X32:       # %bb.0:
 ; X32-NEXT:    paddd %xmm1, %xmm0
-; X32-NEXT:    psubd {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: vec_sink_sub_of_const_to_add0:
@@ -313,7 +313,7 @@ define <4 x i32> @vec_sink_sub_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_sub_of_const_to_add1:
 ; X32:       # %bb.0:
 ; X32-NEXT:    paddd %xmm1, %xmm0
-; X32-NEXT:    psubd {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: vec_sink_sub_of_const_to_add1:
@@ -333,7 +333,7 @@ define <4 x i32> @vec_sink_sub_from_const_to_add0(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_sub_from_const_to_add0:
 ; X32:       # %bb.0:
 ; X32-NEXT:    psubd %xmm0, %xmm1
-; X32-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X32-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-NEXT:    movdqa %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
@@ -351,7 +351,7 @@ define <4 x i32> @vec_sink_sub_from_const_to_add1(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_sub_from_const_to_add1:
 ; X32:       # %bb.0:
 ; X32-NEXT:    psubd %xmm0, %xmm1
-; X32-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X32-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-NEXT:    movdqa %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
@@ -373,7 +373,7 @@ define <4 x i32> @vec_sink_add_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_add_of_const_to_sub:
 ; X32:       # %bb.0:
 ; X32-NEXT:    psubd %xmm1, %xmm0
-; X32-NEXT:    paddd {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: vec_sink_add_of_const_to_sub:
@@ -389,7 +389,7 @@ define <4 x i32> @vec_sink_add_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_add_of_const_to_sub2:
 ; X32:       # %bb.0:
 ; X32-NEXT:    psubd %xmm0, %xmm1
-; X32-NEXT:    psubd {{\.LCPI.*}}, %xmm1
+; X32-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-NEXT:    movdqa %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
@@ -411,7 +411,7 @@ define <4 x i32> @vec_sink_sub_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_sub_of_const_to_sub:
 ; X32:       # %bb.0:
 ; X32-NEXT:    psubd %xmm1, %xmm0
-; X32-NEXT:    psubd {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: vec_sink_sub_of_const_to_sub:
@@ -427,7 +427,7 @@ define <4 x i32> @vec_sink_sub_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_sub_of_const_to_sub2:
 ; X32:       # %bb.0:
 ; X32-NEXT:    psubd %xmm0, %xmm1
-; X32-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X32-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-NEXT:    movdqa %xmm1, %xmm0
 ; X32-NEXT:    retl
 ;
@@ -461,7 +461,7 @@ define <4 x i32> @vec_sink_sub_from_const_to_sub2(<4 x i32> %a, <4 x i32> %b) {
 ; X32-LABEL: vec_sink_sub_from_const_to_sub2:
 ; X32:       # %bb.0:
 ; X32-NEXT:    paddd %xmm1, %xmm0
-; X32-NEXT:    psubd {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    psubd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: vec_sink_sub_from_const_to_sub2:
diff --git a/llvm/test/CodeGen/X86/slow-pmulld.ll b/llvm/test/CodeGen/X86/slow-pmulld.ll
index 4eece0d1135e77fa04c21e7933f987f30dc35cad..55834ab90a67ef4b29ddb7793c11283ef071c5f9 100644
--- a/llvm/test/CodeGen/X86/slow-pmulld.ll
+++ b/llvm/test/CodeGen/X86/slow-pmulld.ll
@@ -21,7 +21,7 @@ define <4 x i32> @test_mul_v4i32_v4i8(<4 x i8> %A) {
 ; CHECK32-LABEL: test_mul_v4i32_v4i8:
 ; CHECK32:       # %bb.0:
 ; CHECK32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; CHECK32-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
+; CHECK32-NEXT:    pmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK32-NEXT:    retl
 ;
 ; CHECK64-LABEL: test_mul_v4i32_v4i8:
@@ -33,7 +33,7 @@ define <4 x i32> @test_mul_v4i32_v4i8(<4 x i8> %A) {
 ; SSE4-32-LABEL: test_mul_v4i32_v4i8:
 ; SSE4-32:       # %bb.0:
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE4-32-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
+; SSE4-32-NEXT:    pmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE4-32-NEXT:    retl
 ;
 ; SSE4-64-LABEL: test_mul_v4i32_v4i8:
@@ -45,7 +45,7 @@ define <4 x i32> @test_mul_v4i32_v4i8(<4 x i8> %A) {
 ; AVX2-32-LABEL: test_mul_v4i32_v4i8:
 ; AVX2-32:       # %bb.0:
 ; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX2-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX2-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX2-32-NEXT:    retl
 ;
 ; AVX2-64-LABEL: test_mul_v4i32_v4i8:
@@ -57,7 +57,7 @@ define <4 x i32> @test_mul_v4i32_v4i8(<4 x i8> %A) {
 ; AVX512DQ-32-LABEL: test_mul_v4i32_v4i8:
 ; AVX512DQ-32:       # %bb.0:
 ; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v4i32_v4i8:
@@ -69,7 +69,7 @@ define <4 x i32> @test_mul_v4i32_v4i8(<4 x i8> %A) {
 ; AVX512BW-32-LABEL: test_mul_v4i32_v4i8:
 ; AVX512BW-32:       # %bb.0:
 ; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v4i32_v4i8:
@@ -168,7 +168,7 @@ define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ; AVX2-32-LABEL: test_mul_v8i32_v8i8:
 ; AVX2-32:       # %bb.0:
 ; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX2-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX2-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX2-32-NEXT:    retl
 ;
 ; AVX2-64-LABEL: test_mul_v8i32_v8i8:
@@ -180,7 +180,7 @@ define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ; AVX512DQ-32-LABEL: test_mul_v8i32_v8i8:
 ; AVX512DQ-32:       # %bb.0:
 ; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v8i32_v8i8:
@@ -192,7 +192,7 @@ define <8 x i32> @test_mul_v8i32_v8i8(<8 x i8> %A) {
 ; AVX512BW-32-LABEL: test_mul_v8i32_v8i8:
 ; AVX512BW-32:       # %bb.0:
 ; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v8i32_v8i8:
@@ -359,7 +359,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ; AVX512DQ-32-LABEL: test_mul_v16i32_v16i8:
 ; AVX512DQ-32:       # %bb.0:
 ; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-32-NEXT:    vpmulld {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; AVX512DQ-32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v16i32_v16i8:
@@ -371,7 +371,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ; AVX512BW-32-LABEL: test_mul_v16i32_v16i8:
 ; AVX512BW-32:       # %bb.0:
 ; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %zmm0, %zmm0
+; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v16i32_v16i8:
@@ -383,7 +383,7 @@ define <16 x i32> @test_mul_v16i32_v16i8(<16 x i8> %A) {
 ; KNL-32-LABEL: test_mul_v16i32_v16i8:
 ; KNL-32:       # %bb.0:
 ; KNL-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; KNL-32-NEXT:    vpmulld {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; KNL-32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; KNL-32-NEXT:    retl
 ;
 ; KNL-64-LABEL: test_mul_v16i32_v16i8:
@@ -418,7 +418,7 @@ define <4 x i32> @test_mul_v4i32_v4i16(<4 x i16> %A) {
 ; SSE4-32-LABEL: test_mul_v4i32_v4i16:
 ; SSE4-32:       # %bb.0:
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE4-32-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; SSE4-32-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE4-32-NEXT:    retl
 ;
 ; SSE4-64-LABEL: test_mul_v4i32_v4i16:
@@ -666,7 +666,7 @@ define <16 x i32> @test_mul_v16i32_v16i16(<16 x i16> %A) {
 ; AVX512-32-LABEL: test_mul_v16i32_v16i16:
 ; AVX512-32:       # %bb.0:
 ; AVX512-32-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512-32-NEXT:    vpmulld {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; AVX512-32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; AVX512-32-NEXT:    retl
 ;
 ; AVX512-64-LABEL: test_mul_v16i32_v16i16:
@@ -687,7 +687,7 @@ define <4 x i32> @test_mul_v4i32_v4i8_minsize(<4 x i8> %A) minsize {
 ; CHECK32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; CHECK32:       # %bb.0:
 ; CHECK32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; CHECK32-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
+; CHECK32-NEXT:    pmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK32-NEXT:    retl
 ;
 ; CHECK64-LABEL: test_mul_v4i32_v4i8_minsize:
@@ -699,7 +699,7 @@ define <4 x i32> @test_mul_v4i32_v4i8_minsize(<4 x i8> %A) minsize {
 ; SSE4-32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; SSE4-32:       # %bb.0:
 ; SSE4-32-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE4-32-NEXT:    pmaddwd {{\.LCPI.*}}, %xmm0
+; SSE4-32-NEXT:    pmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE4-32-NEXT:    retl
 ;
 ; SSE4-64-LABEL: test_mul_v4i32_v4i8_minsize:
@@ -711,7 +711,7 @@ define <4 x i32> @test_mul_v4i32_v4i8_minsize(<4 x i8> %A) minsize {
 ; AVX2-32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; AVX2-32:       # %bb.0:
 ; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX2-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX2-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX2-32-NEXT:    retl
 ;
 ; AVX2-64-LABEL: test_mul_v4i32_v4i8_minsize:
@@ -723,7 +723,7 @@ define <4 x i32> @test_mul_v4i32_v4i8_minsize(<4 x i8> %A) minsize {
 ; AVX512DQ-32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; AVX512DQ-32:       # %bb.0:
 ; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v4i32_v4i8_minsize:
@@ -735,7 +735,7 @@ define <4 x i32> @test_mul_v4i32_v4i8_minsize(<4 x i8> %A) minsize {
 ; AVX512BW-32-LABEL: test_mul_v4i32_v4i8_minsize:
 ; AVX512BW-32:       # %bb.0:
 ; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v4i32_v4i8_minsize:
@@ -826,7 +826,7 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ; AVX2-32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; AVX2-32:       # %bb.0:
 ; AVX2-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX2-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX2-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX2-32-NEXT:    retl
 ;
 ; AVX2-64-LABEL: test_mul_v8i32_v8i8_minsize:
@@ -838,7 +838,7 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ; AVX512DQ-32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; AVX512DQ-32:       # %bb.0:
 ; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX512DQ-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v8i32_v8i8_minsize:
@@ -850,7 +850,7 @@ define <8 x i32> @test_mul_v8i32_v8i8_minsize(<8 x i8> %A) minsize {
 ; AVX512BW-32-LABEL: test_mul_v8i32_v8i8_minsize:
 ; AVX512BW-32:       # %bb.0:
 ; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v8i32_v8i8_minsize:
@@ -997,7 +997,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; AVX512DQ-32-LABEL: test_mul_v16i32_v16i8_minsize:
 ; AVX512DQ-32:       # %bb.0:
 ; AVX512DQ-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQ-32-NEXT:    vpmulld {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; AVX512DQ-32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; AVX512DQ-32-NEXT:    retl
 ;
 ; AVX512DQ-64-LABEL: test_mul_v16i32_v16i8_minsize:
@@ -1009,7 +1009,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; AVX512BW-32-LABEL: test_mul_v16i32_v16i8_minsize:
 ; AVX512BW-32:       # %bb.0:
 ; AVX512BW-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI.*}}, %zmm0, %zmm0
+; AVX512BW-32-NEXT:    vpmaddwd {{\.LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0
 ; AVX512BW-32-NEXT:    retl
 ;
 ; AVX512BW-64-LABEL: test_mul_v16i32_v16i8_minsize:
@@ -1021,7 +1021,7 @@ define <16 x i32> @test_mul_v16i32_v16i8_minsize(<16 x i8> %A) minsize {
 ; KNL-32-LABEL: test_mul_v16i32_v16i8_minsize:
 ; KNL-32:       # %bb.0:
 ; KNL-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; KNL-32-NEXT:    vpmulld {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; KNL-32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; KNL-32-NEXT:    retl
 ;
 ; KNL-64-LABEL: test_mul_v16i32_v16i8_minsize:
@@ -1038,7 +1038,7 @@ define <4 x i32> @test_mul_v4i32_v4i16_minsize(<4 x i16> %A) minsize {
 ; CHECK32-LABEL: test_mul_v4i32_v4i16_minsize:
 ; CHECK32:       # %bb.0:
 ; CHECK32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; CHECK32-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; CHECK32-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK32-NEXT:    retl
 ;
 ; CHECK64-LABEL: test_mul_v4i32_v4i16_minsize:
@@ -1050,7 +1050,7 @@ define <4 x i32> @test_mul_v4i32_v4i16_minsize(<4 x i16> %A) minsize {
 ; SSE4-32-LABEL: test_mul_v4i32_v4i16_minsize:
 ; SSE4-32:       # %bb.0:
 ; SSE4-32-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE4-32-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; SSE4-32-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE4-32-NEXT:    retl
 ;
 ; SSE4-64-LABEL: test_mul_v4i32_v4i16_minsize:
@@ -1260,7 +1260,7 @@ define <16 x i32> @test_mul_v16i32_v16i16_minsize(<16 x i16> %A) minsize {
 ; AVX512-32-LABEL: test_mul_v16i32_v16i16_minsize:
 ; AVX512-32:       # %bb.0:
 ; AVX512-32-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512-32-NEXT:    vpmulld {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; AVX512-32-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; AVX512-32-NEXT:    retl
 ;
 ; AVX512-64-LABEL: test_mul_v16i32_v16i16_minsize:
diff --git a/llvm/test/CodeGen/X86/smul_fix_sat_constants.ll b/llvm/test/CodeGen/X86/smul_fix_sat_constants.ll
index e92b8410e038622792412882bbd4b52f192a5b07..b8a46567e75b6c9fe54f4f64689b275ed605d634 100644
--- a/llvm/test/CodeGen/X86/smul_fix_sat_constants.ll
+++ b/llvm/test/CodeGen/X86/smul_fix_sat_constants.ll
@@ -15,13 +15,13 @@ define i64 @func() nounwind {
 ; X64-NEXT:    movl $2, %ecx
 ; X64-NEXT:    movl $3, %eax
 ; X64-NEXT:    imulq %rcx
-; X64-NEXT:    shrdq $2, %rdx, %rax
 ; X64-NEXT:    cmpq $1, %rdx
-; X64-NEXT:    movabsq $9223372036854775807, %rcx # imm = 0x7FFFFFFFFFFFFFFF
-; X64-NEXT:    cmovgq %rcx, %rax
+; X64-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    cmovgq %rax, %rcx
 ; X64-NEXT:    cmpq $-2, %rdx
-; X64-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; X64-NEXT:    cmovlq %rcx, %rax
+; X64-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; X64-NEXT:    cmovgeq %rcx, %rax
 ; X64-NEXT:    retq
   %tmp = call i64 @llvm.smul.fix.sat.i64(i64 3, i64 2, i32 2)
   ret i64 %tmp
@@ -51,12 +51,12 @@ define i64 @func3() nounwind {
 ; X64-NEXT:    movl $2, %edx
 ; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    imulq %rdx
-; X64-NEXT:    shrdq $2, %rdx, %rax
 ; X64-NEXT:    cmpq $1, %rdx
-; X64-NEXT:    cmovgq %rcx, %rax
+; X64-NEXT:    movabsq $4611686018427387903, %rsi # imm = 0x3FFFFFFFFFFFFFFF
+; X64-NEXT:    cmovgq %rcx, %rsi
 ; X64-NEXT:    cmpq $-2, %rdx
-; X64-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; X64-NEXT:    cmovlq %rcx, %rax
+; X64-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    retq
   %tmp = call i64 @llvm.smul.fix.sat.i64(i64 9223372036854775807, i64 2, i32 2)
   ret i64 %tmp
@@ -69,12 +69,12 @@ define i64 @func4() nounwind {
 ; X64-NEXT:    movl $2, %edx
 ; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    imulq %rdx
-; X64-NEXT:    shrdq $32, %rdx, %rax
 ; X64-NEXT:    cmpq $2147483647, %rdx # imm = 0x7FFFFFFF
-; X64-NEXT:    cmovgq %rcx, %rax
+; X64-NEXT:    movl $4294967295, %esi # imm = 0xFFFFFFFF
+; X64-NEXT:    cmovgq %rcx, %rsi
 ; X64-NEXT:    cmpq $-2147483648, %rdx # imm = 0x80000000
-; X64-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; X64-NEXT:    cmovlq %rcx, %rax
+; X64-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    retq
   %tmp = call i64 @llvm.smul.fix.sat.i64(i64 9223372036854775807, i64 2, i32 32)
   ret i64 %tmp
@@ -87,14 +87,14 @@ define i64 @func5() nounwind {
 ; X64-NEXT:    movl $2, %edx
 ; X64-NEXT:    movq %rcx, %rax
 ; X64-NEXT:    imulq %rdx
-; X64-NEXT:    shrdq $63, %rdx, %rax
-; X64-NEXT:    movabsq $4611686018427387903, %rsi # imm = 0x3FFFFFFFFFFFFFFF
-; X64-NEXT:    cmpq %rsi, %rdx
-; X64-NEXT:    cmovgq %rcx, %rax
-; X64-NEXT:    movabsq $-4611686018427387904, %rcx # imm = 0xC000000000000000
-; X64-NEXT:    cmpq %rcx, %rdx
-; X64-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; X64-NEXT:    cmovlq %rcx, %rax
+; X64-NEXT:    movabsq $4611686018427387903, %rax # imm = 0x3FFFFFFFFFFFFFFF
+; X64-NEXT:    cmpq %rax, %rdx
+; X64-NEXT:    movl $1, %esi
+; X64-NEXT:    cmovgq %rcx, %rsi
+; X64-NEXT:    movabsq $-4611686018427387904, %rax # imm = 0xC000000000000000
+; X64-NEXT:    cmpq %rax, %rdx
+; X64-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    retq
   %tmp = call i64 @llvm.smul.fix.sat.i64(i64 9223372036854775807, i64 2, i32 63)
   ret i64 %tmp
diff --git a/llvm/test/CodeGen/X86/sse-fcopysign.ll b/llvm/test/CodeGen/X86/sse-fcopysign.ll
index fc49538e80b7da1fbc1efe62c92d710339f26f99..896516c272543c1c8ce5d0332eece7e5b9a48f9e 100644
--- a/llvm/test/CodeGen/X86/sse-fcopysign.ll
+++ b/llvm/test/CodeGen/X86/sse-fcopysign.ll
@@ -65,9 +65,9 @@ define float @int1(float %a, float %b) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-NEXT:    andps {{\.LCPI.*}}, %xmm1
+; X32-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-NEXT:    orps %xmm0, %xmm1
 ; X32-NEXT:    movss %xmm1, (%esp)
 ; X32-NEXT:    flds (%esp)
@@ -94,9 +94,9 @@ define double @int2(double %a, float %b, float %c) nounwind {
 ; X32-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X32-NEXT:    addss 20(%ebp), %xmm0
 ; X32-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; X32-NEXT:    andps {{\.LCPI.*}}, %xmm1
+; X32-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-NEXT:    cvtss2sd %xmm0, %xmm0
-; X32-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X32-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-NEXT:    orps %xmm1, %xmm0
 ; X32-NEXT:    movlps %xmm0, (%esp)
 ; X32-NEXT:    fldl (%esp)
diff --git a/llvm/test/CodeGen/X86/sse-load-ret.ll b/llvm/test/CodeGen/X86/sse-load-ret.ll
index 841410bf86351d9ad7f65aee9a09bd2911e1d7ba..510c9e2bd8bace6a4c7594ac45d8a26aa204a249 100644
--- a/llvm/test/CodeGen/X86/sse-load-ret.ll
+++ b/llvm/test/CodeGen/X86/sse-load-ret.ll
@@ -14,7 +14,7 @@ define double @test1(double* %P) {
 define double @test2() {
 ; CHECK-LABEL: test2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    fldl {{\.LCPI.*}}
+; CHECK-NEXT:    fldl {{\.LCPI[0-9]+_[0-9]+}}
 ; CHECK-NEXT:    retl
         ret double 1.234560e+03
 }
diff --git a/llvm/test/CodeGen/X86/sse1-fcopysign.ll b/llvm/test/CodeGen/X86/sse1-fcopysign.ll
index 529ac546ef15e80cf0fd5554cf639645cd4ba7f4..28c7a3b80a03c6b1d0e0642717065658f06aa655 100644
--- a/llvm/test/CodeGen/X86/sse1-fcopysign.ll
+++ b/llvm/test/CodeGen/X86/sse1-fcopysign.ll
@@ -7,7 +7,7 @@ define float @f32_pos(float %a, float %b) nounwind {
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    movss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
 ; X86-NEXT:    popl %eax
@@ -26,7 +26,7 @@ define float @f32_neg(float %a, float %b) nounwind {
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-NEXT:    orps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    orps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    movss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
 ; X86-NEXT:    popl %eax
@@ -43,7 +43,7 @@ define float @f32_neg(float %a, float %b) nounwind {
 define <4 x float> @v4f32_pos(<4 x float> %a, <4 x float> %b) nounwind {
 ; X86-LABEL: v4f32_pos:
 ; X86:       # %bb.0:
-; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: v4f32_pos:
@@ -57,7 +57,7 @@ define <4 x float> @v4f32_pos(<4 x float> %a, <4 x float> %b) nounwind {
 define <4 x float> @v4f32_neg(<4 x float> %a, <4 x float> %b) nounwind {
 ; X86-LABEL: v4f32_neg:
 ; X86:       # %bb.0:
-; X86-NEXT:    orps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    orps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: v4f32_neg:
@@ -72,8 +72,8 @@ define <4 x float> @v4f32_const_mag(<4 x float> %a, <4 x float> %b) nounwind {
 ; X86-LABEL: v4f32_const_mag:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movaps %xmm1, %xmm0
-; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
-; X86-NEXT:    orps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-NEXT:    orps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: v4f32_const_mag:
diff --git a/llvm/test/CodeGen/X86/sse1.ll b/llvm/test/CodeGen/X86/sse1.ll
index e2b95eb7b93ab6a1d4c285bccb8c3d804a9f695b..7a5654c6ab1de9aacee36c598340c787facdf2a9 100644
--- a/llvm/test/CodeGen/X86/sse1.ll
+++ b/llvm/test/CodeGen/X86/sse1.ll
@@ -180,7 +180,7 @@ define <4 x i32> @PR30512(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; X86-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
 ; X86-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm1[0]
-; X86-NEXT:    andps {{\.LCPI.*}}, %xmm2
+; X86-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-NEXT:    movaps %xmm2, (%eax)
 ; X86-NEXT:    addl $16, %esp
 ; X86-NEXT:    popl %esi
@@ -238,7 +238,7 @@ define <4 x i32> @PR30512(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <2 x float> @PR31672() #0 {
 ; X86-LABEL: PR31672:
 ; X86:       # %bb.0:
-; X86-NEXT:    sqrtps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    sqrtps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: PR31672:
diff --git a/llvm/test/CodeGen/X86/sse2.ll b/llvm/test/CodeGen/X86/sse2.ll
index eeb55d861205c446949215484b6737bd6410ddcf..49aefe0ec5e9a7139268a6385655064631a585d6 100644
--- a/llvm/test/CodeGen/X86/sse2.ll
+++ b/llvm/test/CodeGen/X86/sse2.ll
@@ -675,7 +675,7 @@ define <2 x i64> @test_insert_64_zext(<2 x i64> %i) {
 define <4 x i32> @PR19721(<4 x i32> %i) {
 ; X86-SSE-LABEL: PR19721:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; AVX-LABEL: PR19721:
diff --git a/llvm/test/CodeGen/X86/sse3.ll b/llvm/test/CodeGen/X86/sse3.ll
index 750abbabb545a0a7d7f2a99510bffd4d7cfd19fb..7a82e681220814c093a5fb3eddc4e3098fce7804 100644
--- a/llvm/test/CodeGen/X86/sse3.ll
+++ b/llvm/test/CodeGen/X86/sse3.ll
@@ -397,7 +397,7 @@ define <4 x i32> @t17() nounwind {
 ; X86-LABEL: t17:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pshufd {{.*#+}} xmm0 = mem[0,1,0,1]
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: t17:
diff --git a/llvm/test/CodeGen/X86/stack-clash-dynamic-alloca.ll b/llvm/test/CodeGen/X86/stack-clash-dynamic-alloca.ll
index 6dd8b6ab58975572a57bf78ac4e0e23614d6b078..3af4bf72b4d831b08ea93f396e72226fd8ba6e4d 100644
--- a/llvm/test/CodeGen/X86/stack-clash-dynamic-alloca.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-dynamic-alloca.ll
@@ -1,71 +1,69 @@
-; RUN: llc -mtriple=x86_64-linux-android < %s | FileCheck -check-prefix=CHECK-X86-64 %s 
-; RUN: llc -mtriple=i686-linux-android < %s | FileCheck -check-prefix=CHECK-X86-32 %s 
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
+; RUN: llc -mtriple=x86_64-linux-android < %s | FileCheck -check-prefix=CHECK-X86-64 %s
+; RUN: llc -mtriple=i686-linux-android < %s | FileCheck -check-prefix=CHECK-X86-32 %s
 
 define i32 @foo(i32 %n) local_unnamed_addr #0 {
-  %a = alloca i32, i32 %n, align 16
-  %b = getelementptr inbounds i32, i32* %a, i64 1198
-  store volatile i32 1, i32* %b
-  %c = load volatile i32, i32* %a
-  ret i32 %c
-}
-
-attributes #0 =  {"probe-stack"="inline-asm"}
-
 ; CHECK-X86-64-LABEL: foo:
 ; CHECK-X86-64:       # %bb.0:
-; CHECK-X86-64-NEXT:  	pushq	%rbp
-; CHECK-X86-64-NEXT:  	.cfi_def_cfa_offset 16
-; CHECK-X86-64-NEXT:  	.cfi_offset %rbp, -16
-; CHECK-X86-64-NEXT:  	movq	%rsp, %rbp
-; CHECK-X86-64-NEXT:  	.cfi_def_cfa_register %rbp
-; CHECK-X86-64-NEXT:  	movq    %rsp, %rax
-; CHECK-X86-64-NEXT:    movl    %edi, %ecx
-; CHECK-X86-64-NEXT:  	leaq 15(,%rcx,4), %rcx
-; CHECK-X86-64-NEXT:  	andq	$-16, %rcx
-; CHECK-X86-64-NEXT:  	subq	%rcx, %rax
-; CHECK-X86-64-NEXT:  	cmpq	%rsp, %rax
-; CHECK-X86-64-NEXT:  	jge	.LBB0_3
+; CHECK-X86-64-NEXT:    pushq %rbp
+; CHECK-X86-64-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-X86-64-NEXT:    .cfi_offset %rbp, -16
+; CHECK-X86-64-NEXT:    movq %rsp, %rbp
+; CHECK-X86-64-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-X86-64-NEXT:    movq %rsp, %rax
+; CHECK-X86-64-NEXT:    movl %edi, %ecx
+; CHECK-X86-64-NEXT:    leaq 15(,%rcx,4), %rcx
+; CHECK-X86-64-NEXT:    andq $-16, %rcx
+; CHECK-X86-64-NEXT:    subq %rcx, %rax
+; CHECK-X86-64-NEXT:    cmpq %rsp, %rax
+; CHECK-X86-64-NEXT:    jge .LBB0_3
 ; CHECK-X86-64-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
-; CHECK-X86-64-NEXT:  	xorq	$0, (%rsp)
-; CHECK-X86-64-NEXT:  	subq	$4096, %rsp # imm = 0x1000
-; CHECK-X86-64-NEXT:  	cmpq	%rsp, %rax
-; CHECK-X86-64-NEXT:  	jl	.LBB0_2
+; CHECK-X86-64-NEXT:    xorq $0, (%rsp)
+; CHECK-X86-64-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-X86-64-NEXT:    cmpq %rsp, %rax
+; CHECK-X86-64-NEXT:    jl .LBB0_2
 ; CHECK-X86-64-NEXT:  .LBB0_3:
-; CHECK-X86-64-NEXT:  	movq	%rax, %rsp
-; CHECK-X86-64-NEXT:  	movl	$1, 4792(%rax)
-; CHECK-X86-64-NEXT:  	movl	(%rax), %eax
-; CHECK-X86-64-NEXT:  	movq	%rbp, %rsp
-; CHECK-X86-64-NEXT:  	popq	%rbp
-; CHECK-X86-64-NEXT:  .cfi_def_cfa %rsp, 8
-; CHECK-X86-64-NEXT:   retq
-
-
+; CHECK-X86-64-NEXT:    movq %rax, %rsp
+; CHECK-X86-64-NEXT:    movl $1, 4792(%rax)
+; CHECK-X86-64-NEXT:    movl (%rax), %eax
+; CHECK-X86-64-NEXT:    movq %rbp, %rsp
+; CHECK-X86-64-NEXT:    popq %rbp
+; CHECK-X86-64-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-X86-64-NEXT:    retq
+;
 ; CHECK-X86-32-LABEL: foo:
 ; CHECK-X86-32:       # %bb.0:
-; CHECK-X86-32-NEXT:    pushl   %ebp
+; CHECK-X86-32-NEXT:    pushl %ebp
 ; CHECK-X86-32-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-X86-32-NEXT:    .cfi_offset %ebp, -8
-; CHECK-X86-32-NEXT:    movl    %esp, %ebp
+; CHECK-X86-32-NEXT:    movl %esp, %ebp
 ; CHECK-X86-32-NEXT:    .cfi_def_cfa_register %ebp
-; CHECK-X86-32-NEXT:    subl    $8, %esp
-; CHECK-X86-32-NEXT:    movl    8(%ebp), %ecx
-; CHECK-X86-32-NEXT:    movl    %esp, %eax
-; CHECK-X86-32-NEXT:    leal    15(,%ecx,4), %ecx
-; CHECK-X86-32-NEXT:    andl    $-16, %ecx
-; CHECK-X86-32-NEXT:    subl    %ecx, %eax
-; CHECK-X86-32-NEXT:    cmpl    %esp, %eax
-; CHECK-X86-32-NEXT:    jge  .LBB0_3
+; CHECK-X86-32-NEXT:    subl $8, %esp
+; CHECK-X86-32-NEXT:    movl 8(%ebp), %ecx
+; CHECK-X86-32-NEXT:    movl %esp, %eax
+; CHECK-X86-32-NEXT:    leal 15(,%ecx,4), %ecx
+; CHECK-X86-32-NEXT:    andl $-16, %ecx
+; CHECK-X86-32-NEXT:    subl %ecx, %eax
+; CHECK-X86-32-NEXT:    cmpl %esp, %eax
+; CHECK-X86-32-NEXT:    jge .LBB0_3
 ; CHECK-X86-32-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
-; CHECK-X86-32-NEXT:    xorl    $0, (%esp)
-; CHECK-X86-32-NEXT:    subl    $4096, %esp # imm = 0x1000
-; CHECK-X86-32-NEXT:    cmpl    %esp, %eax
+; CHECK-X86-32-NEXT:    xorl $0, (%esp)
+; CHECK-X86-32-NEXT:    subl $4096, %esp # imm = 0x1000
+; CHECK-X86-32-NEXT:    cmpl %esp, %eax
 ; CHECK-X86-32-NEXT:    jl .LBB0_2
 ; CHECK-X86-32-NEXT:  .LBB0_3:
-; CHECK-X86-32-NEXT:    movl    %eax, %esp
-; CHECK-X86-32-NEXT:    movl    $1, 4792(%eax)
-; CHECK-X86-32-NEXT:    movl    (%eax), %eax
-; CHECK-X86-32-NEXT:    movl    %ebp, %esp
-; CHECK-X86-32-NEXT:    popl    %ebp
+; CHECK-X86-32-NEXT:    movl %eax, %esp
+; CHECK-X86-32-NEXT:    movl $1, 4792(%eax)
+; CHECK-X86-32-NEXT:    movl (%eax), %eax
+; CHECK-X86-32-NEXT:    movl %ebp, %esp
+; CHECK-X86-32-NEXT:    popl %ebp
 ; CHECK-X86-32-NEXT:    .cfi_def_cfa %esp, 4
 ; CHECK-X86-32-NEXT:    retl
+  %a = alloca i32, i32 %n, align 16
+  %b = getelementptr inbounds i32, i32* %a, i64 1198
+  store volatile i32 1, i32* %b
+  %c = load volatile i32, i32* %a
+  ret i32 %c
+}
 
+attributes #0 =  {"probe-stack"="inline-asm"}
diff --git a/llvm/test/CodeGen/X86/stack-clash-large-large-align.ll b/llvm/test/CodeGen/X86/stack-clash-large-large-align.ll
index 6c981cb4ac910367882b7c589176ad7006cef74a..5710252f6c7df8d68cb0b8d3baef293b3a2500ce 100644
--- a/llvm/test/CodeGen/X86/stack-clash-large-large-align.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-large-large-align.ll
@@ -1,28 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
 ; RUN: llc < %s | FileCheck %s
 
-
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo_noprotect() local_unnamed_addr {
 ; CHECK-LABEL: foo_noprotect:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:	pushq	%rbp
-; CHECK-NEXT:   .cfi_def_cfa_offset 16
-; CHECK-NEXT:   .cfi_offset %rbp, -16
-; CHECK-NEXT:   movq	%rsp, %rbp
-; CHECK-NEXT:   .cfi_def_cfa_register %rbp
-; CHECK-NEXT:   andq	$-4096, %rsp                    # imm = 0xF000
-; CHECK-NEXT:   subq	$73728, %rsp                    # imm = 0x12000
-; CHECK-NEXT:   movl	$1, 392(%rsp)
-; CHECK-NEXT:   movl	$1, 28792(%rsp)
-; CHECK-NEXT:   movl	(%rsp), %eax
-; CHECK-NEXT:   movq	%rbp, %rsp
-; CHECK-NEXT:   popq	%rbp
-; CHECK-NEXT:   .cfi_def_cfa %rsp, 8
-; CHECK-NEXT:   retq
-
-
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    andq $-4096, %rsp # imm = 0xF000
+; CHECK-NEXT:    subq $73728, %rsp # imm = 0x12000
+; CHECK-NEXT:    movl $1, 392(%rsp)
+; CHECK-NEXT:    movl $1, 28792(%rsp)
+; CHECK-NEXT:    movl (%rsp), %eax
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i64 18000, align 4096
   %b0 = getelementptr inbounds i32, i32* %a, i64 98
   %b1 = getelementptr inbounds i32, i32* %a, i64 7198
@@ -35,47 +33,43 @@ define i32 @foo_noprotect() local_unnamed_addr {
 define i32 @foo_protect() local_unnamed_addr #0 {
 ; CHECK-LABEL: foo_protect:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:	pushq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-NEXT:	.cfi_offset %rbp, -16
-; CHECK-NEXT:	movq	%rsp, %rbp
-; CHECK-NEXT:	.cfi_def_cfa_register %rbp
-; CHECK-NEXT:	movq	%rsp, %r11
-; CHECK-NEXT:	andq	$-4096, %r11                    # imm = 0xF000
-; CHECK-NEXT:	cmpq	%rsp, %r11
-; CHECK-NEXT:	je	.LBB1_4
-; CHECK-NEXT:# %bb.1:
-; CHECK-NEXT:	subq	$4096, %rsp                     # imm = 0x1000
-; CHECK-NEXT:	cmpq	%rsp, %r11
-; CHECK-NEXT:	jb	.LBB1_3
-; CHECK-NEXT:.LBB1_2:                                # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:	movq	$0, (%rsp)
-; CHECK-NEXT:	subq	$4096, %rsp                     # imm = 0x1000
-; CHECK-NEXT:	cmpq	%rsp, %r11
-; CHECK-NEXT:	jb	.LBB1_2
-; CHECK-NEXT:.LBB1_3:
-; CHECK-NEXT:	movq	%r11, %rsp
-; CHECK-NEXT:	movq	$0, (%rsp)
-; CHECK-NEXT:.LBB1_4:
-; CHECK-NEXT:	movq	%rsp, %r11
-; CHECK-NEXT:	subq	$73728, %r11                    # imm = 0x12000
-; CHECK-NEXT:.LBB1_5:                                # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:	subq	$4096, %rsp                     # imm = 0x1000
-; CHECK-NEXT:	movq	$0, (%rsp)
-; CHECK-NEXT:	cmpq	%r11, %rsp
-; CHECK-NEXT:	jne	.LBB1_5
-; CHECK-NEXT:# %bb.6:
-; CHECK-NEXT:	movl	$1, 392(%rsp)
-; CHECK-NEXT:	movl	$1, 28792(%rsp)
-; CHECK-NEXT:	movl	(%rsp), %eax
-; CHECK-NEXT:	movq	%rbp, %rsp
-; CHECK-NEXT:	popq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa %rsp, 8
-; CHECK-NEXT:	retq
-
-
-
-
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    movq %rsp, %r11
+; CHECK-NEXT:    andq $-4096, %r11 # imm = 0xF000
+; CHECK-NEXT:    cmpq %rsp, %r11
+; CHECK-NEXT:    je .LBB1_4
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    cmpq %rsp, %r11
+; CHECK-NEXT:    jb .LBB1_3
+; CHECK-NEXT:  .LBB1_2: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    cmpq %rsp, %r11
+; CHECK-NEXT:    jb .LBB1_2
+; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:    movq %r11, %rsp
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:    movq %rsp, %r11
+; CHECK-NEXT:    subq $73728, %r11 # imm = 0x12000
+; CHECK-NEXT:  .LBB1_5: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    cmpq %r11, %rsp
+; CHECK-NEXT:    jne .LBB1_5
+; CHECK-NEXT:  # %bb.6:
+; CHECK-NEXT:    movl $1, 392(%rsp)
+; CHECK-NEXT:    movl $1, 28792(%rsp)
+; CHECK-NEXT:    movl (%rsp), %eax
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i64 18000, align 4096
   %b0 = getelementptr inbounds i32, i32* %a, i64 98
   %b1 = getelementptr inbounds i32, i32* %a, i64 7198
diff --git a/llvm/test/CodeGen/X86/stack-clash-large.ll b/llvm/test/CodeGen/X86/stack-clash-large.ll
index dd53cd8f6964617209f12df8358d2758b2336ebb..9129e4ed40fda2bf950111200c8d6b745e23e49d 100644
--- a/llvm/test/CodeGen/X86/stack-clash-large.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-large.ll
@@ -1,8 +1,45 @@
-; RUN: llc -mtriple=x86_64-linux-android < %s | FileCheck -check-prefix=CHECK-X86-64 %s 
-; RUN: llc -mtriple=i686-linux-android < %s | FileCheck -check-prefix=CHECK-X86-32 %s 
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
+; RUN: llc -mtriple=x86_64-linux-android < %s | FileCheck -check-prefix=CHECK-X86-64 %s
+; RUN: llc -mtriple=i686-linux-android < %s | FileCheck -check-prefix=CHECK-X86-32 %s
 
 define i32 @foo() local_unnamed_addr #0 {
-
+; CHECK-X86-64-LABEL: foo:
+; CHECK-X86-64:       # %bb.0:
+; CHECK-X86-64-NEXT:    movq %rsp, %r11
+; CHECK-X86-64-NEXT:    subq $69632, %r11 # imm = 0x11000
+; CHECK-X86-64-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; CHECK-X86-64-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-X86-64-NEXT:    movq $0, (%rsp)
+; CHECK-X86-64-NEXT:    cmpq %r11, %rsp
+; CHECK-X86-64-NEXT:    jne .LBB0_1
+; CHECK-X86-64-NEXT:  # %bb.2:
+; CHECK-X86-64-NEXT:    subq $2248, %rsp # imm = 0x8C8
+; CHECK-X86-64-NEXT:    .cfi_def_cfa_offset 71888
+; CHECK-X86-64-NEXT:    movl $1, 264(%rsp)
+; CHECK-X86-64-NEXT:    movl $1, 28664(%rsp)
+; CHECK-X86-64-NEXT:    movl -128(%rsp), %eax
+; CHECK-X86-64-NEXT:    addq $71880, %rsp # imm = 0x118C8
+; CHECK-X86-64-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-X86-64-NEXT:    retq
+;
+; CHECK-X86-32-LABEL: foo:
+; CHECK-X86-32:       # %bb.0:
+; CHECK-X86-32-NEXT:    movl %esp, %r11d
+; CHECK-X86-32-NEXT:    subl $69632, %r11d # imm = 0x11000
+; CHECK-X86-32-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; CHECK-X86-32-NEXT:    subl $4096, %esp # imm = 0x1000
+; CHECK-X86-32-NEXT:    movl $0, (%esp)
+; CHECK-X86-32-NEXT:    cmpl %r11d, %esp
+; CHECK-X86-32-NEXT:    jne .LBB0_1
+; CHECK-X86-32-NEXT:  # %bb.2:
+; CHECK-X86-32-NEXT:    subl $2380, %esp # imm = 0x94C
+; CHECK-X86-32-NEXT:    .cfi_def_cfa_offset 72016
+; CHECK-X86-32-NEXT:    movl $1, 392(%esp)
+; CHECK-X86-32-NEXT:    movl $1, 28792(%esp)
+; CHECK-X86-32-NEXT:    movl (%esp), %eax
+; CHECK-X86-32-NEXT:    addl $72012, %esp # imm = 0x1194C
+; CHECK-X86-32-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-X86-32-NEXT:    retl
   %a = alloca i32, i64 18000, align 16
   %b0 = getelementptr inbounds i32, i32* %a, i64 98
   %b1 = getelementptr inbounds i32, i32* %a, i64 7198
@@ -13,41 +50,3 @@ define i32 @foo() local_unnamed_addr #0 {
 }
 
 attributes #0 =  {"probe-stack"="inline-asm"}
-
-; CHECK-X86-64-LABEL: foo:
-; CHECK-X86-64:        # %bb.0:
-; CHECK-X86-64-NEXT:	movq	%rsp, %r11
-; CHECK-X86-64-NEXT:	subq	$69632, %r11 # imm = 0x11000
-; CHECK-X86-64-NEXT:   .LBB0_1:
-; CHECK-X86-64-NEXT:	subq	$4096, %rsp # imm = 0x1000
-; CHECK-X86-64-NEXT:	movq	$0, (%rsp)
-; CHECK-X86-64-NEXT:	cmpq	%r11, %rsp
-; CHECK-X86-64-NEXT:	jne	.LBB0_1
-; CHECK-X86-64-NEXT:# %bb.2:
-; CHECK-X86-64-NEXT:	subq    $2248, %rsp
-; CHECK-X86-64-NEXT:	.cfi_def_cfa_offset 71888
-; CHECK-X86-64-NEXT:	movl	$1, 264(%rsp)
-; CHECK-X86-64-NEXT:	movl	$1, 28664(%rsp)
-; CHECK-X86-64-NEXT:	movl	-128(%rsp), %eax
-; CHECK-X86-64-NEXT:	addq	$71880, %rsp # imm = 0x118C8
-; CHECK-X86-64-NEXT:	.cfi_def_cfa_offset 8
-; CHECK-X86-64-NEXT:	retq
-
-; CHECK-X86-32-LABEL: foo:
-; CHECK-X86-32:      # %bb.0:
-; CHECK-X86-32-NEXT:    movl    %esp, %r11d
-; CHECK-X86-32-NEXT:    subl    $69632, %r11d # imm = 0x11000
-; CHECK-X86-32-NEXT:    .LBB0_1: # =>This Inner Loop Header: Depth=1
-; CHECK-X86-32-NEXT:    subl    $4096, %esp # imm = 0x1000
-; CHECK-X86-32-NEXT:    movl    $0, (%esp)
-; CHECK-X86-32-NEXT:    cmpl    %r11d, %esp
-; CHECK-X86-32-NEXT:    jne  .LBB0_1
-; CHECK-X86-32-NEXT:# %bb.2:
-; CHECK-X86-32-NEXT:    subl    $2380, %esp
-; CHECK-X86-32-NEXT:    .cfi_def_cfa_offset 72016
-; CHECK-X86-32-NEXT:    movl    $1, 392(%esp)
-; CHECK-X86-32-NEXT:    movl    $1, 28792(%esp)
-; CHECK-X86-32-NEXT:    movl    (%esp), %eax
-; CHECK-X86-32-NEXT:    addl    $72012, %esp # imm = 0x1194C
-; CHECK-X86-32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK-X86-32-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes-mutliple-objects.ll b/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes-mutliple-objects.ll
index 0fe492a93d0e1bcd51815ffb5a866baf2018a9fe..ecb30dfeb36eeca1222432013733559909dab7b4 100644
--- a/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes-mutliple-objects.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes-mutliple-objects.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
 ; RUN: llc < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -11,9 +11,9 @@ define i32 @foo() local_unnamed_addr #0 {
 ; CHECK-NEXT:    movq $0, (%rsp)
 ; CHECK-NEXT:    subq $1784, %rsp # imm = 0x6F8
 ; CHECK-NEXT:    .cfi_def_cfa_offset 5888
-; CHECK-NEXT:    movl $1, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movl $2, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movl $1, 3872(%rsp)
+; CHECK-NEXT:    movl $2, 672(%rsp)
+; CHECK-NEXT:    movl 1872(%rsp), %eax
 ; CHECK-NEXT:    addq $5880, %rsp # imm = 0x16F8
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes.ll b/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes.ll
index bb2be8846ec2f6bea77be945e6f6a31f4f362241..b682cf8ac9650b25e29b5e00fbbe7a24c7f5da4a 100644
--- a/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes.ll
@@ -1,26 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
 ; RUN: llc < %s | FileCheck %s
 
-
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo() local_unnamed_addr #0 {
-
 ; CHECK-LABEL: foo:
-; CHECK:         # %bb.0:
-; CHECK-NEXT:	subq	$4096, %rsp             # imm = 0x1000
-; CHECK-NEXT:	movq	$0, (%rsp)
-; CHECK-NEXT:	subq	$3784, %rsp             # imm = 0xEC8
-; CHECK-NEXT:	.cfi_def_cfa_offset 7888
-; CHECK-NEXT:	movl	$1, 264(%rsp)
-; CHECK-NEXT:	movl	$1, 4664(%rsp)
-; CHECK-NEXT:	movl	-128(%rsp), %eax
-; CHECK-NEXT:	addq	$7880, %rsp             # imm = 0x1EC8
-; CHECK-NEXT:	.cfi_def_cfa_offset 8
-; CHECK-NEXT:	retq
-
-
-
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    subq $3784, %rsp # imm = 0xEC8
+; CHECK-NEXT:    .cfi_def_cfa_offset 7888
+; CHECK-NEXT:    movl $1, 264(%rsp)
+; CHECK-NEXT:    movl $1, 4664(%rsp)
+; CHECK-NEXT:    movl -128(%rsp), %eax
+; CHECK-NEXT:    addq $7880, %rsp # imm = 0x1EC8
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i64 2000, align 16
   %b0 = getelementptr inbounds i32, i32* %a, i64 98
   %b1 = getelementptr inbounds i32, i32* %a, i64 1198
diff --git a/llvm/test/CodeGen/X86/stack-clash-medium.ll b/llvm/test/CodeGen/X86/stack-clash-medium.ll
index 5a97074025f1fa87085fd54297827464842dff9a..c40396fcead9c4b3aab53d19c474b83863c1f6ca 100644
--- a/llvm/test/CodeGen/X86/stack-clash-medium.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-medium.ll
@@ -1,7 +1,31 @@
-; RUN: llc -mtriple=x86_64-linux-android < %s | FileCheck -check-prefix=CHECK-X86-64 %s 
-; RUN: llc -mtriple=i686-linux-android < %s | FileCheck -check-prefix=CHECK-X86-32 %s 
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
+; RUN: llc -mtriple=x86_64-linux-android < %s | FileCheck -check-prefix=CHECK-X86-64 %s
+; RUN: llc -mtriple=i686-linux-android < %s | FileCheck -check-prefix=CHECK-X86-32 %s
 
 define i32 @foo() local_unnamed_addr #0 {
+; CHECK-X86-64-LABEL: foo:
+; CHECK-X86-64:       # %bb.0:
+; CHECK-X86-64-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-X86-64-NEXT:    movq $0, (%rsp)
+; CHECK-X86-64-NEXT:    subq $3784, %rsp # imm = 0xEC8
+; CHECK-X86-64-NEXT:    .cfi_def_cfa_offset 7888
+; CHECK-X86-64-NEXT:    movl $1, 672(%rsp)
+; CHECK-X86-64-NEXT:    movl -128(%rsp), %eax
+; CHECK-X86-64-NEXT:    addq $7880, %rsp # imm = 0x1EC8
+; CHECK-X86-64-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-X86-64-NEXT:    retq
+;
+; CHECK-X86-32-LABEL: foo:
+; CHECK-X86-32:       # %bb.0:
+; CHECK-X86-32-NEXT:    subl $4096, %esp # imm = 0x1000
+; CHECK-X86-32-NEXT:    movl $0, (%esp)
+; CHECK-X86-32-NEXT:    subl $3916, %esp # imm = 0xF4C
+; CHECK-X86-32-NEXT:    .cfi_def_cfa_offset 8016
+; CHECK-X86-32-NEXT:    movl $1, 800(%esp)
+; CHECK-X86-32-NEXT:    movl (%esp), %eax
+; CHECK-X86-32-NEXT:    addl $8012, %esp # imm = 0x1F4C
+; CHECK-X86-32-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-X86-32-NEXT:    retl
   %a = alloca i32, i64 2000, align 16
   %b = getelementptr inbounds i32, i32* %a, i64 200
   store volatile i32 1, i32* %b
@@ -10,28 +34,3 @@ define i32 @foo() local_unnamed_addr #0 {
 }
 
 attributes #0 =  {"probe-stack"="inline-asm"}
-
-; CHECK-X86-64-LABEL: foo:
-; CHECK-X86-64:      # %bb.0:
-; CHECK-X86-64-NEXT: subq	$4096, %rsp             # imm = 0x1000
-; CHECK-X86-64-NEXT: movq	$0, (%rsp)
-; CHECK-X86-64-NEXT: subq	$3784, %rsp             # imm = 0xEC8
-; CHECK-X86-64-NEXT: .cfi_def_cfa_offset 7888
-; CHECK-X86-64-NEXT: movl	$1, 672(%rsp)
-; CHECK-X86-64-NEXT: movl	-128(%rsp), %eax
-; CHECK-X86-64-NEXT: addq	$7880, %rsp             # imm = 0x1EC8
-; CHECK-X86-64-NEXT: .cfi_def_cfa_offset 8
-; CHECK-X86-64-NEXT: retq
-
-
-; CHECK-X86-32-LABEL: foo:
-; CHECK-X86-32:      # %bb.0:
-; CHECK-X86-32-NEXT: subl	$4096, %esp # imm = 0x1000
-; CHECK-X86-32-NEXT: movl	$0, (%esp)
-; CHECK-X86-32-NEXT: subl	$3916, %esp # imm = 0xF4C
-; CHECK-X86-32-NEXT: .cfi_def_cfa_offset 8016
-; CHECK-X86-32-NEXT: movl	$1, 800(%esp)
-; CHECK-X86-32-NEXT: movl	(%esp), %eax
-; CHECK-X86-32-NEXT: addl	$8012, %esp # imm = 0x1F4C
-; CHECK-X86-32-NEXT: .cfi_def_cfa_offset 4
-; CHECK-X86-32-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/stack-clash-small-alloc-medium-align.ll b/llvm/test/CodeGen/X86/stack-clash-small-alloc-medium-align.ll
index 39b6c3640a60a8d73e76113bb11cb3d1472f80a7..221a2e36947ebeea7747252771761bee231e54c5 100644
--- a/llvm/test/CodeGen/X86/stack-clash-small-alloc-medium-align.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-small-alloc-medium-align.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
 ; RUN: llc < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -6,21 +7,20 @@ target triple = "x86_64-unknown-linux-gnu"
 ; | case1 | alloca + align < probe_size
 define i32 @foo1(i64 %i) local_unnamed_addr #0 {
 ; CHECK-LABEL: foo1:
-; CHECK:        # %bb.0:
-; CHECK-NEXT:	pushq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-NEXT:	.cfi_offset %rbp, -16
-; CHECK-NEXT:	movq	%rsp, %rbp
-; CHECK-NEXT:	.cfi_def_cfa_register %rbp
-; CHECK-NEXT:   andq    $-64, %rsp
-; CHECK-NEXT:   subq    $832, %rsp                      # imm = 0x340
-; CHECK-NEXT:   movl    $1, (%rsp,%rdi,4)
-; CHECK-NEXT:   movl    (%rsp), %eax
-; CHECK-NEXT:   movq    %rbp, %rsp
-; CHECK-NEXT:   popq    %rbp
-; CHECK-NEXT:	.cfi_def_cfa %rsp, 8
-; CHECK-NEXT:	retq
-
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    andq $-64, %rsp
+; CHECK-NEXT:    subq $832, %rsp # imm = 0x340
+; CHECK-NEXT:    movl $1, (%rsp,%rdi,4)
+; CHECK-NEXT:    movl (%rsp), %eax
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i32 200, align 64
   %b = getelementptr inbounds i32, i32* %a, i64 %i
   store volatile i32 1, i32* %b
@@ -31,25 +31,24 @@ define i32 @foo1(i64 %i) local_unnamed_addr #0 {
 ; | case2 | alloca > probe_size, align > probe_size
 define i32 @foo2(i64 %i) local_unnamed_addr #0 {
 ; CHECK-LABEL: foo2:
-; CHECK:        # %bb.0:
-; CHECK-NEXT:	pushq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-NEXT:	.cfi_offset %rbp, -16
-; CHECK-NEXT:	movq	%rsp, %rbp
-; CHECK-NEXT:	.cfi_def_cfa_register %rbp
-; CHECK-NEXT:   andq    $-2048, %rsp                    # imm = 0xF800
-; CHECK-NEXT:   subq    $2048, %rsp                     # imm = 0x800
-; CHECK-NEXT:   movq    $0, (%rsp)
-; CHECK-NEXT:   subq    $4096, %rsp                     # imm = 0x1000
-; CHECK-NEXT:   movq    $0, (%rsp)
-; CHECK-NEXT:   subq    $2048, %rsp                     # imm = 0x800
-; CHECK-NEXT:   movl    $1, (%rsp,%rdi,4)
-; CHECK-NEXT:   movl    (%rsp), %eax
-; CHECK-NEXT:   movq    %rbp, %rsp
-; CHECK-NEXT:   popq    %rbp
-; CHECK-NEXT:   .cfi_def_cfa %rsp, 8
-; CHECK-NEXT:   retq
-
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    andq $-2048, %rsp # imm = 0xF800
+; CHECK-NEXT:    subq $2048, %rsp # imm = 0x800
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    subq $2048, %rsp # imm = 0x800
+; CHECK-NEXT:    movl $1, (%rsp,%rdi,4)
+; CHECK-NEXT:    movl (%rsp), %eax
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i32 2000, align 2048
   %b = getelementptr inbounds i32, i32* %a, i64 %i
   store volatile i32 1, i32* %b
@@ -60,24 +59,22 @@ define i32 @foo2(i64 %i) local_unnamed_addr #0 {
 ; | case3 | alloca < probe_size, align < probe_size, alloca + align > probe_size
 define i32 @foo3(i64 %i) local_unnamed_addr #0 {
 ; CHECK-LABEL: foo3:
-; CHECK:        # %bb.0:
-; CHECK-NEXT:   pushq   %rbp
-; CHECK-NEXT:   .cfi_def_cfa_offset 16
-; CHECK-NEXT:   .cfi_offset %rbp, -16
-; CHECK-NEXT:   movq    %rsp, %rbp
-; CHECK-NEXT:   .cfi_def_cfa_register %rbp
-; CHECK-NEXT:   andq    $-1024, %rsp                    # imm = 0xFC00
-; CHECK-NEXT:   subq    $3072, %rsp                     # imm = 0xC00
-; CHECK-NEXT:   movq    $0, (%rsp)
-; CHECK-NEXT:   subq    $1024, %rsp                     # imm = 0x400
-; CHECK-NEXT:   movl    $1, (%rsp,%rdi,4)
-; CHECK-NEXT:   movl    (%rsp), %eax
-; CHECK-NEXT:   movq    %rbp, %rsp
-; CHECK-NEXT:   popq    %rbp
-; CHECK-NEXT:   .cfi_def_cfa %rsp, 8
-; CHECK-NEXT:   retq
-
-
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    andq $-1024, %rsp # imm = 0xFC00
+; CHECK-NEXT:    subq $3072, %rsp # imm = 0xC00
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    subq $1024, %rsp # imm = 0x400
+; CHECK-NEXT:    movl $1, (%rsp,%rdi,4)
+; CHECK-NEXT:    movl (%rsp), %eax
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i32 1000, align 1024
   %b = getelementptr inbounds i32, i32* %a, i64 %i
   store volatile i32 1, i32* %b
@@ -88,40 +85,39 @@ define i32 @foo3(i64 %i) local_unnamed_addr #0 {
 ; | case4 | alloca + probe_size < probe_size, followed by dynamic alloca
 define i32 @foo4(i64 %i) local_unnamed_addr #0 {
 ; CHECK-LABEL: foo4:
-; CHECK:        # %bb.0:
-; CHECK-NEXT:	pushq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-NEXT:	.cfi_offset %rbp, -16
-; CHECK-NEXT:	movq	%rsp, %rbp
-; CHECK-NEXT:	.cfi_def_cfa_register %rbp
-; CHECK-NEXT:	pushq	%rbx
-; CHECK-NEXT:	andq	$-64, %rsp
-; CHECK-NEXT:	subq	$896, %rsp                      # imm = 0x380
-; CHECK-NEXT:	movq	%rsp, %rbx
-; CHECK-NEXT:	.cfi_offset %rbx, -24
-; CHECK-NEXT:	movl	$1, (%rbx,%rdi,4)
-; CHECK-NEXT:	movl	(%rbx), %ecx
-; CHECK-NEXT:	movq	%rsp, %rax
-; CHECK-NEXT:	leaq	15(,%rcx,4), %rcx
-; CHECK-NEXT:	andq	$-16, %rcx
-; CHECK-NEXT:	subq	%rcx, %rax
-; CHECK-NEXT:	cmpq	%rsp, %rax
-; CHECK-NEXT:	jge	.LBB3_3
-; CHECK-NEXT:.LBB3_2:                                # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:	xorq	$0, (%rsp)
-; CHECK-NEXT:	subq	$4096, %rsp                     # imm = 0x1000
-; CHECK-NEXT:	cmpq	%rsp, %rax
-; CHECK-NEXT:	jl	.LBB3_2
-; CHECK-NEXT:.LBB3_3:
-; CHECK-NEXT:	andq	$-64, %rax
-; CHECK-NEXT:	movq	%rax, %rsp
-; CHECK-NEXT:	movl	(%rax), %eax
-; CHECK-NEXT:	leaq	-8(%rbp), %rsp
-; CHECK-NEXT:	popq	%rbx
-; CHECK-NEXT:	popq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa %rsp, 8
-; CHECK-NEXT:	retq
-
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    andq $-64, %rsp
+; CHECK-NEXT:    subq $896, %rsp # imm = 0x380
+; CHECK-NEXT:    movq %rsp, %rbx
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    movl $1, (%rbx,%rdi,4)
+; CHECK-NEXT:    movl (%rbx), %ecx
+; CHECK-NEXT:    movq %rsp, %rax
+; CHECK-NEXT:    leaq 15(,%rcx,4), %rcx
+; CHECK-NEXT:    andq $-16, %rcx
+; CHECK-NEXT:    subq %rcx, %rax
+; CHECK-NEXT:    cmpq %rsp, %rax
+; CHECK-NEXT:    jge .LBB3_3
+; CHECK-NEXT:  .LBB3_2: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    xorq $0, (%rsp)
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    cmpq %rsp, %rax
+; CHECK-NEXT:    jl .LBB3_2
+; CHECK-NEXT:  .LBB3_3:
+; CHECK-NEXT:    andq $-64, %rax
+; CHECK-NEXT:    movq %rax, %rsp
+; CHECK-NEXT:    movl (%rax), %eax
+; CHECK-NEXT:    leaq -8(%rbp), %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i32 200, align 64
   %b = getelementptr inbounds i32, i32* %a, i64 %i
   store volatile i32 1, i32* %b
@@ -132,4 +128,3 @@ define i32 @foo4(i64 %i) local_unnamed_addr #0 {
 }
 
 attributes #0 =  {"probe-stack"="inline-asm"}
-
diff --git a/llvm/test/CodeGen/X86/stack-clash-small-large-align.ll b/llvm/test/CodeGen/X86/stack-clash-small-large-align.ll
index e608bab9041564af72d13b3ab8625083697e5a26..c0541a8077ba0f572e9a9da36eec08b498414418 100644
--- a/llvm/test/CodeGen/X86/stack-clash-small-large-align.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-small-large-align.ll
@@ -1,28 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
 ; RUN: llc < %s | FileCheck %s
-
-
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo_noprotect() local_unnamed_addr {
 ; CHECK-LABEL: foo_noprotect:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:	pushq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-NEXT:	.cfi_offset %rbp, -16
-; CHECK-NEXT:	movq	%rsp, %rbp
-; CHECK-NEXT:	.cfi_def_cfa_register %rbp
-; CHECK-NEXT:	andq	$-65536, %rsp
-; CHECK-NEXT:	subq	$65536, %rsp
-; CHECK-NEXT:	movl	$1, 392(%rsp)
-; CHECK-NEXT:	movl	(%rsp), %eax
-; CHECK-NEXT:	movq	%rbp, %rsp
-; CHECK-NEXT:	popq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa %rsp, 8
-; CHECK-NEXT:	retq
-
-
-
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    andq $-65536, %rsp # imm = 0xFFFF0000
+; CHECK-NEXT:    subq $65536, %rsp # imm = 0x10000
+; CHECK-NEXT:    movl $1, 392(%rsp)
+; CHECK-NEXT:    movl (%rsp), %eax
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i64 100, align 65536
   %b = getelementptr inbounds i32, i32* %a, i64 98
   store volatile i32 1, i32* %b
@@ -33,46 +29,42 @@ define i32 @foo_noprotect() local_unnamed_addr {
 define i32 @foo_protect() local_unnamed_addr #0 {
 ; CHECK-LABEL: foo_protect:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:	pushq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa_offset 16
-; CHECK-NEXT:	.cfi_offset %rbp, -16
-; CHECK-NEXT:	movq	%rsp, %rbp
-; CHECK-NEXT:	.cfi_def_cfa_register %rbp
-; CHECK-NEXT:	movq	%rsp, %r11
-; CHECK-NEXT:	andq	$-65536, %r11                   # imm = 0xFFFF0000
-; CHECK-NEXT:	cmpq	%rsp, %r11
-; CHECK-NEXT:	je	.LBB1_4
-; CHECK-NEXT:# %bb.1:
-; CHECK-NEXT:	subq	$4096, %rsp                     # imm = 0x1000
-; CHECK-NEXT:	cmpq	%rsp, %r11
-; CHECK-NEXT:	jb	.LBB1_3
-; CHECK-NEXT:.LBB1_2:                                # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:	movq	$0, (%rsp)
-; CHECK-NEXT:	subq	$4096, %rsp                     # imm = 0x1000
-; CHECK-NEXT:	cmpq	%rsp, %r11
-; CHECK-NEXT:	jb	.LBB1_2
-; CHECK-NEXT:.LBB1_3:
-; CHECK-NEXT:	movq	%r11, %rsp
-; CHECK-NEXT:	movq	$0, (%rsp)
-; CHECK-NEXT:.LBB1_4:
-; CHECK-NEXT:	movq	%rsp, %r11
-; CHECK-NEXT:	subq	$65536, %r11                    # imm = 0x10000
-; CHECK-NEXT:.LBB1_5:                                # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:	subq	$4096, %rsp                     # imm = 0x1000
-; CHECK-NEXT:	movq	$0, (%rsp)
-; CHECK-NEXT:	cmpq	%r11, %rsp
-; CHECK-NEXT:	jne	.LBB1_5
-; CHECK-NEXT:# %bb.6:
-; CHECK-NEXT:	movl	$1, 392(%rsp)
-; CHECK-NEXT:	movl	(%rsp), %eax
-; CHECK-NEXT:	movq	%rbp, %rsp
-; CHECK-NEXT:	popq	%rbp
-; CHECK-NEXT:	.cfi_def_cfa %rsp, 8
-; CHECK-NEXT:	retq
-
-
-
-
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    movq %rsp, %r11
+; CHECK-NEXT:    andq $-65536, %r11 # imm = 0xFFFF0000
+; CHECK-NEXT:    cmpq %rsp, %r11
+; CHECK-NEXT:    je .LBB1_4
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    cmpq %rsp, %r11
+; CHECK-NEXT:    jb .LBB1_3
+; CHECK-NEXT:  .LBB1_2: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    cmpq %rsp, %r11
+; CHECK-NEXT:    jb .LBB1_2
+; CHECK-NEXT:  .LBB1_3:
+; CHECK-NEXT:    movq %r11, %rsp
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:    movq %rsp, %r11
+; CHECK-NEXT:    subq $65536, %r11 # imm = 0x10000
+; CHECK-NEXT:  .LBB1_5: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    cmpq %r11, %rsp
+; CHECK-NEXT:    jne .LBB1_5
+; CHECK-NEXT:  # %bb.6:
+; CHECK-NEXT:    movl $1, 392(%rsp)
+; CHECK-NEXT:    movl (%rsp), %eax
+; CHECK-NEXT:    movq %rbp, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i64 100, align 65536
   %b = getelementptr inbounds i32, i32* %a, i64 98
   store volatile i32 1, i32* %b
diff --git a/llvm/test/CodeGen/X86/stack-clash-small.ll b/llvm/test/CodeGen/X86/stack-clash-small.ll
index bf40fe907dc2f8a914bc603b76138c31f90c326b..ecfaf7a1c4f165171172c8beadce61cec4d9c1d5 100644
--- a/llvm/test/CodeGen/X86/stack-clash-small.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-small.ll
@@ -1,20 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
 ; RUN: llc < %s | FileCheck %s
-
-
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo() local_unnamed_addr #0 {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:  subq	$280, %rsp # imm = 0x118
-; CHECK-NEXT:  .cfi_def_cfa_offset 288
-; CHECK-NEXT:  movl	$1, 264(%rsp)
-; CHECK-NEXT:  movl	-128(%rsp), %eax
-; CHECK-NEXT:  addq	$280, %rsp # imm = 0x118
-; CHECK-NEXT:  .cfi_def_cfa_offset 8
-; CHECK-NEXT:  retq
-
+; CHECK-NEXT:    subq $280, %rsp # imm = 0x118
+; CHECK-NEXT:    .cfi_def_cfa_offset 288
+; CHECK-NEXT:    movl $1, 264(%rsp)
+; CHECK-NEXT:    movl -128(%rsp), %eax
+; CHECK-NEXT:    addq $280, %rsp # imm = 0x118
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i64 100, align 16
   %b = getelementptr inbounds i32, i32* %a, i64 98
   store volatile i32 1, i32* %b
diff --git a/llvm/test/CodeGen/X86/stack-clash-unknown-call.ll b/llvm/test/CodeGen/X86/stack-clash-unknown-call.ll
index 9294d70528fa2c7f9a286d85b2897dc2ef6b1705..2df3eca6546010e729229c75095bb6ac27a30e1f 100644
--- a/llvm/test/CodeGen/X86/stack-clash-unknown-call.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-unknown-call.ll
@@ -1,28 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
 ; RUN: llc < %s | FileCheck %s
-
-
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg);
 
-define void @foo() local_unnamed_addr #0 {
-
-;CHECK-LABEL: foo:
-;CHECK:         # %bb.0:
-;CHECK-NEXT:	subq	$4096, %rsp # imm = 0x1000
 ; it's important that we don't use the call as a probe here
-;CHECK-NEXT:	movq	$0, (%rsp)
-;CHECK-NEXT:	subq	$3912, %rsp # imm = 0xF48
-;CHECK-NEXT:	.cfi_def_cfa_offset 8016
-;CHECK-NEXT:	movq	%rsp, %rdi
-;CHECK-NEXT:	movl	$8000, %edx # imm = 0x1F40
-;CHECK-NEXT:	xorl	%esi, %esi
-;CHECK-NEXT:	callq	memset
-;CHECK-NEXT:	addq	$8008, %rsp # imm = 0x1F48
-;CHECK-NEXT:	.cfi_def_cfa_offset 8
-;CHECK-NEXT:	retq
-
+define void @foo() local_unnamed_addr #0 {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    subq $3912, %rsp # imm = 0xF48
+; CHECK-NEXT:    .cfi_def_cfa_offset 8016
+; CHECK-NEXT:    movq %rsp, %rdi
+; CHECK-NEXT:    movl $8000, %edx # imm = 0x1F40
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    callq memset@PLT
+; CHECK-NEXT:    addq $8008, %rsp # imm = 0x1F48
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %a = alloca i8, i64 8000, align 16
   call void @llvm.memset.p0i8.i64(i8* align 16 %a, i8 0, i64 8000, i1 false)
   ret void
diff --git a/llvm/test/CodeGen/X86/statepoint-ra.ll b/llvm/test/CodeGen/X86/statepoint-ra.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b5a2c81a02425232715a16bf21f09e1aefbc16f6
--- /dev/null
+++ b/llvm/test/CodeGen/X86/statepoint-ra.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -O3 -use-registers-for-deopt-values -restrict-statepoint-remat=true < %s 2>&1 | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-NOT: error: ran out of registers during register allocation
+
+define void @barney(i8 addrspace(1)* %arg, double %arg1, double %arg2, double %arg3, double %arg4, double %arg5, double %arg6, double %arg7, double %arg8, double %arg9, double %arg10, double %arg11, double %arg12) gc "statepoint-example" personality i32* ()* @widget {
+bb:
+  %tmp = call coldcc token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* nonnull @blam, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 1, i32 0, i32 0, i32 0, i32 26, i32 0, i32 0, i8 addrspace(1)* %arg, i32 4, double %arg1, i32 7, i8* null, i32 4, double %arg2, i32 7, i8* null, i32 4, double %arg3, i32 7, i8* null, i32 4, double %arg4, i32 7, i8* null, i32 4, double %arg5, i32 7, i8* null, i32 4, double %arg6, i32 7, i8* null, i32 4, double %arg7, i32 7, i8* null, i32 4, double %arg8, i32 7, i8* null, i32 4, double %arg9, i32 7, i8* null, i32 4, double %arg10, i32 7, i8* null, i32 4, double %arg11, i32 7, i8* null, i32 4, double %arg12, i32 7, i8* null, i32 7, i8* null), "gc-live"(i8 addrspace(1)* %arg) ]
+  br i1 undef, label %bb13, label %bb15
+
+bb13:                                             ; preds = %bb
+  %tmp14 = call token (i64, i32, i32 (i8 addrspace(1)*, double, double, double, double, double, double, double, double, double)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32p1i8f64f64f64f64f64f64f64f64f64f(i64 2, i32 5, i32 (i8 addrspace(1)*, double, double, double, double, double, double, double, double, double)* nonnull @quux, i32 10, i32 0, i8 addrspace(1)* nonnull null, double %arg1, double %arg2, double %arg3, double %arg5, double %arg6, double %arg7, double %arg9, double %arg10, double %arg11, i32 0, i32 0) [ "deopt"(i32 0, i32 2, i32 0, i32 70, i32 0, i32 26, i32 0, i32 0, i8 addrspace(1)* null, i32 4, double %arg1, i32 7, i8* null, i32 4, double %arg2, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 4, double %arg4, i32 7, i8* null, i32 4, double %arg5, i32 7, i8* null, i32 4, double %arg6, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 4, double %arg8, i32 7, i8* null, i32 4, double %arg9, i32 7, i8* null, i32 4, double %arg10, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 4, double %arg12, i32 7, i8* null, i32 7, i8* null), "gc-live"(i8 addrspace(1)* null) ]
+  br label %bb15
+
+bb15:                                             ; preds = %bb13, %bb
+  %tmp16 = phi double [ %arg4, %bb13 ], [ 1.000000e+00, %bb ]
+  %tmp17 = phi double [ %arg8, %bb13 ], [ 1.000000e+00, %bb ]
+  %tmp18 = phi double [ %arg12, %bb13 ], [ 1.000000e+00, %bb ]
+  br i1 undef, label %bb25, label %bb19
+
+bb19:                                             ; preds = %bb15
+  %tmp20 = invoke token (i64, i32, i32 (i32, i8 addrspace(1)*, i32)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32i32p1i8i32f(i64 1, i32 16, i32 (i32, i8 addrspace(1)*, i32)* nonnull @eggs, i32 3, i32 0, i32 undef, i8 addrspace(1)* nonnull undef, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 2, i32 0, i32 97, i32 0, i32 26, i32 0, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 0, i32 2, i32 3, i32 0, i32 20, i32 0, i32 0, i8 addrspace(1)* undef, i32 4, double %arg1, i32 7, i8* null, i32 4, double %arg2, i32 7, i8* null, i32 4, double %tmp16, i32 7, i8* null, i32 4, double %arg5, i32 7, i8* null, i32 4, double %arg6, i32 7, i8* null, i32 4, double %tmp17, i32 7, i8* null, i32 4, double %arg9, i32 7, i8* null, i32 4, double %arg10, i32 7, i8* null, i32 4, double %tmp18, i32 7, i8* null, i32 7, i8* null), "gc-live"(i8 addrspace(1)* undef) ]
+          to label %bb21 unwind label %bb23
+
+bb21:                                             ; preds = %bb19
+  %tmp22 = call token (i64, i32, void (i8 addrspace(1)*, double, double, double, double, double, double, double, double, double, i32)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i8f64f64f64f64f64f64f64f64f64i32f(i64 2, i32 5, void (i8 addrspace(1)*, double, double, double, double, double, double, double, double, double, i32)* nonnull @ham, i32 11, i32 0, i8 addrspace(1)* nonnull undef, double %arg1, double %arg2, double %tmp16, double %arg5, double %arg6, double %tmp17, double %arg9, double %arg10, double %tmp18, i32 51, i32 0, i32 0) [ "deopt"(i32 0, i32 2, i32 0, i32 97, i32 0, i32 26, i32 0, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 2, i32 2, i32 46, i32 0, i32 20, i32 0, i32 0, i8 addrspace(1)* undef, i32 4, double %arg1, i32 7, i8* null, i32 4, double %arg2, i32 7, i8* null, i32 4, double %tmp16, i32 7, i8* null, i32 4, double %arg5, i32 7, i8* null, i32 4, double %arg6, i32 7, i8* null, i32 4, double %tmp17, i32 7, i8* null, i32 4, double %arg9, i32 7, i8* null, i32 4, double %arg10, i32 7, i8* null, i32 4, double %tmp18, i32 7, i8* null, i32 3, i32 51), "gc-live"(i8 addrspace(1)* undef) ]
+  unreachable
+
+bb23:                                             ; preds = %bb19
+  %tmp24 = landingpad token
+          cleanup
+  ret void
+
+bb25:                                             ; preds = %bb15
+  ret void
+}
+
+declare i32* @widget()
+declare i32 @quux(i8 addrspace(1)*, double, double, double, double, double, double, double, double, double)
+declare void @blam()
+declare i32 @eggs(i32, i8 addrspace(1)*, i32)
+declare void @ham(i8 addrspace(1)*, double, double, double, double, double, double, double, double, double, i32)
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 , i32 , void ()*, i32 , i32 , ...)
+declare token @llvm.experimental.gc.statepoint.p0f_i32p1i8f64f64f64f64f64f64f64f64f64f(i64 , i32 , i32 (i8 addrspace(1)*, double, double, double, double, double, double, double, double, double)*, i32 , i32 , ...)
+declare token @llvm.experimental.gc.statepoint.p0f_i32i32p1i8i32f(i64 , i32 , i32 (i32, i8 addrspace(1)*, i32)*, i32 , i32 , ...)
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidp1i8f64f64f64f64f64f64f64f64f64i32f(i64 , i32 , void (i8 addrspace(1)*, double, double, double, double, double, double, double, double, double, i32)*, i32 , i32 , ...)
diff --git a/llvm/test/CodeGen/X86/tail-call-mutable-memarg.ll b/llvm/test/CodeGen/X86/tail-call-mutable-memarg.ll
index 34db632ec2050c646e92bce17d59cc19a7d4a7f7..e4b9dada399d7de584dc1d7c069d45a111f898f9 100644
--- a/llvm/test/CodeGen/X86/tail-call-mutable-memarg.ll
+++ b/llvm/test/CodeGen/X86/tail-call-mutable-memarg.ll
@@ -24,7 +24,7 @@ entry:
 ; CHECK:         calll   _tail_std@4
 ; CHECK:         retl    $4
 
-define x86_thiscallcc void @inalloca(i32* %this, i32* inalloca %args) {
+define x86_thiscallcc void @inalloca(i32* %this, i32* inalloca(i32) %args) {
 entry:
   %val = load i32, i32* %args
   store i32 0, i32* %args
diff --git a/llvm/test/CodeGen/X86/uint64-to-float.ll b/llvm/test/CodeGen/X86/uint64-to-float.ll
index fdab443921c31169c6da54398adcfa66a27cc197..ab230c65e7dcf79f75e570d144ac01c3d4b02823 100644
--- a/llvm/test/CodeGen/X86/uint64-to-float.ll
+++ b/llvm/test/CodeGen/X86/uint64-to-float.ll
@@ -18,7 +18,7 @@ define float @test(i64 %a) nounwind {
 ; X86-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    fildll {{[0-9]+}}(%esp)
-; X86-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; X86-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X86-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-NEXT:    movss %xmm0, (%esp)
diff --git a/llvm/test/CodeGen/X86/uint_to_fp-2.ll b/llvm/test/CodeGen/X86/uint_to_fp-2.ll
index c9211540329b1d26e77f07be477748b71abec0b5..da27f04e2df187155288aa56c4bf94efb97156e3 100644
--- a/llvm/test/CodeGen/X86/uint_to_fp-2.ll
+++ b/llvm/test/CodeGen/X86/uint_to_fp-2.ll
@@ -7,8 +7,8 @@ define float @test1(i32 %x) nounwind readnone {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushl %eax
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    orpd {{\.LCPI.*}}, %xmm0
-; CHECK-NEXT:    subsd {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    orpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; CHECK-NEXT:    subsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; CHECK-NEXT:    cvtsd2ss %xmm0, %xmm0
 ; CHECK-NEXT:    movss %xmm0, (%esp)
 ; CHECK-NEXT:    flds (%esp)
@@ -26,8 +26,8 @@ define float @test2(<4 x i32> %x) nounwind readnone ssp {
 ; CHECK-NEXT:    pushl %eax
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
-; CHECK-NEXT:    orps {{\.LCPI.*}}, %xmm1
-; CHECK-NEXT:    subsd {{\.LCPI.*}}, %xmm1
+; CHECK-NEXT:    orps {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; CHECK-NEXT:    subsd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsd2ss %xmm1, %xmm0
 ; CHECK-NEXT:    movss %xmm0, (%esp)
diff --git a/llvm/test/CodeGen/X86/uint_to_fp-3.ll b/llvm/test/CodeGen/X86/uint_to_fp-3.ll
index e3ac60cf77efc14a315cf63d25aeea913ab2ed37..b6846a68f23b802183f9d10c255745371d3eefa5 100644
--- a/llvm/test/CodeGen/X86/uint_to_fp-3.ll
+++ b/llvm/test/CodeGen/X86/uint_to_fp-3.ll
@@ -9,13 +9,13 @@
 define <4 x float> @mask_ucvt_4i32_4f32(<4 x i32> %a) {
 ; X32-SSE-LABEL: mask_ucvt_4i32_4f32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; X32-SSE-NEXT:    retl
 ;
 ; X32-AVX-LABEL: mask_ucvt_4i32_4f32:
 ; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-AVX-NEXT:    retl
 ;
@@ -38,7 +38,7 @@ define <4 x float> @mask_ucvt_4i32_4f32(<4 x i32> %a) {
 define <4 x double> @mask_ucvt_4i32_4f64(<4 x i32> %a) {
 ; X32-SSE-LABEL: mask_ucvt_4i32_4f64:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-SSE-NEXT:    cvtdq2pd %xmm0, %xmm2
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X32-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1
@@ -47,7 +47,7 @@ define <4 x double> @mask_ucvt_4i32_4f64(<4 x i32> %a) {
 ;
 ; X32-AVX-LABEL: mask_ucvt_4i32_4f64:
 ; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-AVX-NEXT:    vcvtdq2pd %xmm0, %ymm0
 ; X32-AVX-NEXT:    retl
 ;
@@ -80,7 +80,7 @@ define <4 x float> @lshr_truncate_mask_ucvt_4i64_4f32(<4 x i64> *%p0) {
 ; X32-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
 ; X32-SSE-NEXT:    psrld $16, %xmm0
 ; X32-SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
-; X32-SSE-NEXT:    mulps {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    mulps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-SSE-NEXT:    retl
 ;
 ; X32-AVX-LABEL: lshr_truncate_mask_ucvt_4i64_4f32:
@@ -90,7 +90,7 @@ define <4 x float> @lshr_truncate_mask_ucvt_4i64_4f32(<4 x i64> *%p0) {
 ; X32-AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
 ; X32-AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; X32-AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
-; X32-AVX-NEXT:    vmulps {{\.LCPI.*}}, %xmm0, %xmm0
+; X32-AVX-NEXT:    vmulps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X32-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: lshr_truncate_mask_ucvt_4i64_4f32:
diff --git a/llvm/test/CodeGen/X86/umul_fix.ll b/llvm/test/CodeGen/X86/umul_fix.ll
index 8481fe4ac6b4220aa0311ee59f18d30a17458aad..37cdc49bdfad6a092452c6c0674ee4d10d1c4b76 100644
--- a/llvm/test/CodeGen/X86/umul_fix.ll
+++ b/llvm/test/CodeGen/X86/umul_fix.ll
@@ -77,11 +77,11 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X64-LABEL: func3:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
-; X64-NEXT:    andl $15, %esi
-; X64-NEXT:    andl $15, %eax
-; X64-NEXT:    imull %esi, %eax
-; X64-NEXT:    shrb $2, %al
+; X64-NEXT:    andb $15, %al
+; X64-NEXT:    andb $15, %sil
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %sil
+; X64-NEXT:    shrb $2, %al
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: func3:
@@ -90,11 +90,8 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 ; X86-NEXT:    andb $15, %al
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NEXT:    andb $15, %cl
-; X86-NEXT:    movzbl %cl, %ecx
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    imull %ecx, %eax
+; X86-NEXT:    mulb %cl
 ; X86-NEXT:    shrb $2, %al
-; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
   %tmp = call i4 @llvm.umul.fix.i4(i4 %x, i4 %y, i32 2)
   ret i4 %tmp
diff --git a/llvm/test/CodeGen/X86/urem-power-of-two.ll b/llvm/test/CodeGen/X86/urem-power-of-two.ll
index ba9b552df18733364448a0131a1885d3c061be8c..89555a61ea6b0b889bb9e191dc12ee7d0a989cb3 100644
--- a/llvm/test/CodeGen/X86/urem-power-of-two.ll
+++ b/llvm/test/CodeGen/X86/urem-power-of-two.ll
@@ -106,7 +106,7 @@ define i8 @and_pow_2(i8 %x, i8 %y) {
 define <4 x i32> @vec_const_uniform_pow_2(<4 x i32> %x) {
 ; X86-LABEL: vec_const_uniform_pow_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_const_uniform_pow_2:
@@ -120,7 +120,7 @@ define <4 x i32> @vec_const_uniform_pow_2(<4 x i32> %x) {
 define <4 x i32> @vec_const_nonuniform_pow_2(<4 x i32> %x) {
 ; X86-LABEL: vec_const_nonuniform_pow_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vec_const_nonuniform_pow_2:
diff --git a/llvm/test/CodeGen/X86/var-permute-256.ll b/llvm/test/CodeGen/X86/var-permute-256.ll
index 98335d362f95ca48d3a8e89f391c593a4418a429..99e1382e6e91e47538c6cfca7207a794d8c4a91b 100644
--- a/llvm/test/CodeGen/X86/var-permute-256.ll
+++ b/llvm/test/CodeGen/X86/var-permute-256.ll
@@ -34,7 +34,7 @@ define <4 x i64> @var_shuffle_v4i64(<4 x i64> %v, <4 x i64> %indices) nounwind {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vpermilpd %ymm4, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtq {{.*}}(%rip), %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vblendvpd %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -94,7 +94,7 @@ define <8 x i32> @var_shuffle_v8i32(<8 x i32> %v, <8 x i32> %indices) nounwind {
 ; AVX1-NEXT:    vpermilps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtd {{.*}}(%rip), %xmm1, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vblendvps %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -454,7 +454,7 @@ define <4 x double> @var_shuffle_v4f64(<4 x double> %v, <4 x i64> %indices) noun
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vpermilpd %ymm4, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtq {{.*}}(%rip), %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vblendvpd %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -514,7 +514,7 @@ define <8 x float> @var_shuffle_v8f32(<8 x float> %v, <8 x i32> %indices) nounwi
 ; AVX1-NEXT:    vpermilps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtd {{.*}}(%rip), %xmm1, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vblendvps %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -576,7 +576,7 @@ define <4 x i64> @var_shuffle_v4i64_from_v2i64(<2 x i64> %v, <4 x i64> %indices)
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm3
 ; AVX1-NEXT:    vpermilpd %ymm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtq {{.*}}(%rip), %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vpermilpd %ymm3, %ymm0, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm1, %ymm2, %ymm0, %ymm0
@@ -638,7 +638,7 @@ define <8 x i32> @var_shuffle_v8i32_from_v4i32(<4 x i32> %v, <8 x i32> %indices)
 ; AVX1-NEXT:    vpermilps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtd {{.*}}(%rip), %xmm1, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vblendvps %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -997,7 +997,7 @@ define <4 x double> @var_shuffle_v4f64_from_v2f64(<2 x double> %v, <4 x i64> %in
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm3
 ; AVX1-NEXT:    vpermilpd %ymm3, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtq {{.*}}(%rip), %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vpermilpd %ymm3, %ymm0, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm1, %ymm2, %ymm0, %ymm0
@@ -1059,7 +1059,7 @@ define <8 x float> @var_shuffle_v8f32_from_v4f32(<4 x float> %v, <8 x i32> %indi
 ; AVX1-NEXT:    vpermilps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtd {{.*}}(%rip), %xmm1, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd {{\.LCPI.*}}+{{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vblendvps %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
index 4569e69b7e501e79996ffd2c608926b7f1faa391..72fcb412481975faaf2a85c4dabbc87623ab34cb 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
@@ -2031,7 +2031,7 @@ define <2 x i8> @strict_vector_fptosi_v2f64_to_v2i8(<2 x double> %a) #0 {
 ; SSE-32-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
 ; SSE-32:       # %bb.0:
 ; SSE-32-NEXT:    cvttpd2dq %xmm0, %xmm0
-; SSE-32-NEXT:    andpd {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    andpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    retl
@@ -2082,7 +2082,7 @@ define <2 x i8> @strict_vector_fptoui_v2f64_to_v2i8(<2 x double> %a) #0 {
 ; SSE-32-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
 ; SSE-32:       # %bb.0:
 ; SSE-32-NEXT:    cvttpd2dq %xmm0, %xmm0
-; SSE-32-NEXT:    andpd {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    andpd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    retl
@@ -2134,7 +2134,7 @@ define <2 x i8> @strict_vector_fptosi_v2f32_to_v2i8(<2 x float> %a) #0 {
 ; SSE-32:       # %bb.0:
 ; SSE-32-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
 ; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    retl
@@ -2192,7 +2192,7 @@ define <2 x i8> @strict_vector_fptoui_v2f32_to_v2i8(<2 x float> %a) #0 {
 ; SSE-32:       # %bb.0:
 ; SSE-32-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero
 ; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    retl
@@ -3037,7 +3037,7 @@ define <4 x i32> @strict_vector_fptoui_v4f32_to_v4i32(<4 x float> %a) #0 {
 ; SSE-32-NEXT:    movaps %xmm0, %xmm3
 ; SSE-32-NEXT:    cmpltps %xmm2, %xmm3
 ; SSE-32-NEXT:    movaps %xmm3, %xmm1
-; SSE-32-NEXT:    andnps {{\.LCPI.*}}, %xmm1
+; SSE-32-NEXT:    andnps {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; SSE-32-NEXT:    andnps %xmm2, %xmm3
 ; SSE-32-NEXT:    subps %xmm3, %xmm0
 ; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
index 52639987f54742c3d83b06029a5a9d95b2ef7eb1..5c92db796933f50f8e46edc73de6c501bdcfe13e 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
@@ -278,14 +278,14 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE-32-NEXT:    movd %xmm1, %eax
 ; SSE-32-NEXT:    shrl $31, %eax
 ; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE-32-NEXT:    fstps (%esp)
 ; SSE-32-NEXT:    wait
 ; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE-32-NEXT:    movd %xmm0, %eax
 ; SSE-32-NEXT:    shrl $31, %eax
 ; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    wait
 ; SSE-32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
@@ -347,14 +347,14 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; SSE41-32-NEXT:    movd %xmm1, %eax
 ; SSE41-32-NEXT:    shrl $31, %eax
 ; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE41-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE41-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE41-32-NEXT:    fstps (%esp)
 ; SSE41-32-NEXT:    wait
 ; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-32-NEXT:    movd %xmm0, %eax
 ; SSE41-32-NEXT:    shrl $31, %eax
 ; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE41-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE41-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE41-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    wait
 ; SSE41-32-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
@@ -415,13 +415,13 @@ define <2 x float> @uitofp_v2i64_v2f32(<2 x i64> %x) #0 {
 ; AVX-32-NEXT:    vextractps $1, %xmm0, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vextractps $3, %xmm0, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstps (%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -526,7 +526,7 @@ define <4 x float> @sitofp_v4i1_v4f32(<4 x i1> %x) #0 {
 define <4 x float> @uitofp_v4i1_v4f32(<4 x i1> %x) #0 {
 ; SSE-32-LABEL: uitofp_v4i1_v4f32:
 ; SSE-32:       # %bb.0:
-; SSE-32-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-32-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE-32-NEXT:    retl
 ;
@@ -538,7 +538,7 @@ define <4 x float> @uitofp_v4i1_v4f32(<4 x i1> %x) #0 {
 ;
 ; SSE41-32-LABEL: uitofp_v4i1_v4f32:
 ; SSE41-32:       # %bb.0:
-; SSE41-32-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; SSE41-32-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE41-32-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE41-32-NEXT:    retl
 ;
@@ -550,7 +550,7 @@ define <4 x float> @uitofp_v4i1_v4f32(<4 x i1> %x) #0 {
 ;
 ; AVX1-32-LABEL: uitofp_v4i1_v4f32:
 ; AVX1-32:       # %bb.0:
-; AVX1-32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX1-32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; AVX1-32-NEXT:    retl
 ;
@@ -569,7 +569,7 @@ define <4 x float> @uitofp_v4i1_v4f32(<4 x i1> %x) #0 {
 ;
 ; AVX512VL-32-LABEL: uitofp_v4i1_v4f32:
 ; AVX512VL-32:       # %bb.0:
-; AVX512VL-32-NEXT:    vpandd {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpandd {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
 ; AVX512VL-32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; AVX512VL-32-NEXT:    retl
 ;
@@ -588,7 +588,7 @@ define <4 x float> @uitofp_v4i1_v4f32(<4 x i1> %x) #0 {
 ;
 ; AVX512DQVL-32-LABEL: uitofp_v4i1_v4f32:
 ; AVX512DQVL-32:       # %bb.0:
-; AVX512DQVL-32-NEXT:    vandps {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
 ; AVX512DQVL-32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; AVX512DQVL-32-NEXT:    retl
 ;
@@ -737,10 +737,10 @@ define <4 x float> @uitofp_v4i32_v4f32(<4 x i32> %x) #0 {
 ; SSE-32:       # %bb.0:
 ; SSE-32-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
 ; SSE-32-NEXT:    pand %xmm0, %xmm1
-; SSE-32-NEXT:    por {{\.LCPI.*}}, %xmm1
+; SSE-32-NEXT:    por {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; SSE-32-NEXT:    psrld $16, %xmm0
-; SSE-32-NEXT:    por {{\.LCPI.*}}, %xmm0
-; SSE-32-NEXT:    subps {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    por {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; SSE-32-NEXT:    subps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-32-NEXT:    addps %xmm1, %xmm0
 ; SSE-32-NEXT:    retl
 ;
@@ -759,10 +759,10 @@ define <4 x float> @uitofp_v4i32_v4f32(<4 x i32> %x) #0 {
 ; SSE41-32:       # %bb.0:
 ; SSE41-32-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535]
 ; SSE41-32-NEXT:    pand %xmm0, %xmm1
-; SSE41-32-NEXT:    por {{\.LCPI.*}}, %xmm1
+; SSE41-32-NEXT:    por {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; SSE41-32-NEXT:    psrld $16, %xmm0
-; SSE41-32-NEXT:    por {{\.LCPI.*}}, %xmm0
-; SSE41-32-NEXT:    subps {{\.LCPI.*}}, %xmm0
+; SSE41-32-NEXT:    por {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; SSE41-32-NEXT:    subps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE41-32-NEXT:    addps %xmm1, %xmm0
 ; SSE41-32-NEXT:    retl
 ;
@@ -782,7 +782,7 @@ define <4 x float> @uitofp_v4i32_v4f32(<4 x i32> %x) #0 {
 ; AVX1-32-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
 ; AVX1-32-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; AVX1-32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
-; AVX1-32-NEXT:    vsubps {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vsubps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX1-32-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX1-32-NEXT:    retl
 ;
@@ -860,7 +860,7 @@ define <2 x double> @uitofp_v2i1_v2f64(<2 x i1> %x) #0 {
 ; SSE-32-LABEL: uitofp_v2i1_v2f64:
 ; SSE-32:       # %bb.0:
 ; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE-32-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-32-NEXT:    cvtdq2pd %xmm0, %xmm0
 ; SSE-32-NEXT:    retl
 ;
@@ -874,7 +874,7 @@ define <2 x double> @uitofp_v2i1_v2f64(<2 x i1> %x) #0 {
 ; SSE41-32-LABEL: uitofp_v2i1_v2f64:
 ; SSE41-32:       # %bb.0:
 ; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE41-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; SSE41-32-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE41-32-NEXT:    cvtdq2pd %xmm0, %xmm0
 ; SSE41-32-NEXT:    retl
 ;
@@ -888,7 +888,7 @@ define <2 x double> @uitofp_v2i1_v2f64(<2 x i1> %x) #0 {
 ; AVX1-32-LABEL: uitofp_v2i1_v2f64:
 ; AVX1-32:       # %bb.0:
 ; AVX1-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX1-32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX1-32-NEXT:    vcvtdq2pd %xmm0, %xmm0
 ; AVX1-32-NEXT:    retl
 ;
@@ -910,7 +910,7 @@ define <2 x double> @uitofp_v2i1_v2f64(<2 x i1> %x) #0 {
 ; AVX512VL-32-LABEL: uitofp_v2i1_v2f64:
 ; AVX512VL-32:       # %bb.0:
 ; AVX512VL-32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX512VL-32-NEXT:    vpandd {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpandd {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
 ; AVX512VL-32-NEXT:    vcvtdq2pd %xmm0, %xmm0
 ; AVX512VL-32-NEXT:    retl
 ;
@@ -932,7 +932,7 @@ define <2 x double> @uitofp_v2i1_v2f64(<2 x i1> %x) #0 {
 ; AVX512DQVL-32-LABEL: uitofp_v2i1_v2f64:
 ; AVX512DQVL-32:       # %bb.0:
 ; AVX512DQVL-32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX512DQVL-32-NEXT:    vandps {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
 ; AVX512DQVL-32-NEXT:    vcvtdq2pd %xmm0, %xmm0
 ; AVX512DQVL-32-NEXT:    retl
 ;
@@ -1276,14 +1276,14 @@ define <2 x double> @uitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; SSE-32-NEXT:    movd %xmm1, %eax
 ; SSE-32-NEXT:    shrl $31, %eax
 ; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; SSE-32-NEXT:    wait
 ; SSE-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE-32-NEXT:    movd %xmm0, %eax
 ; SSE-32-NEXT:    shrl $31, %eax
 ; SSE-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE-32-NEXT:    fstpl (%esp)
 ; SSE-32-NEXT:    wait
 ; SSE-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
@@ -1344,14 +1344,14 @@ define <2 x double> @uitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; SSE41-32-NEXT:    movd %xmm1, %eax
 ; SSE41-32-NEXT:    shrl $31, %eax
 ; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE41-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE41-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE41-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; SSE41-32-NEXT:    wait
 ; SSE41-32-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
 ; SSE41-32-NEXT:    movd %xmm0, %eax
 ; SSE41-32-NEXT:    shrl $31, %eax
 ; SSE41-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; SSE41-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; SSE41-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; SSE41-32-NEXT:    fstpl (%esp)
 ; SSE41-32-NEXT:    wait
 ; SSE41-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
@@ -1411,13 +1411,13 @@ define <2 x double> @uitofp_v2i64_v2f64(<2 x i64> %x) #0 {
 ; AVX-32-NEXT:    vextractps $1, %xmm0, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vextractps $3, %xmm0, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstpl (%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
index 456e317a213810611f4858760300f5e8ee45fc5d..3d1f87f3fc0694afd8d2013c63ee2cf780a64501 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
@@ -94,7 +94,7 @@ define <8 x float> @sitofp_v8i1_v8f32(<8 x i1> %x) #0 {
 define <8 x float> @uitofp_v8i1_v8f32(<8 x i1> %x) #0 {
 ; AVX1-32-LABEL: uitofp_v8i1_v8f32:
 ; AVX1-32:       # %bb.0:
-; AVX1-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX1-32-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-32-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX1-32-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
@@ -114,7 +114,7 @@ define <8 x float> @uitofp_v8i1_v8f32(<8 x i1> %x) #0 {
 ;
 ; AVX2-32-LABEL: uitofp_v8i1_v8f32:
 ; AVX2-32:       # %bb.0:
-; AVX2-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX2-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX2-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; AVX2-32-NEXT:    retl
@@ -128,7 +128,7 @@ define <8 x float> @uitofp_v8i1_v8f32(<8 x i1> %x) #0 {
 ;
 ; AVX512F-32-LABEL: uitofp_v8i1_v8f32:
 ; AVX512F-32:       # %bb.0:
-; AVX512F-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512F-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512F-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512F-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; AVX512F-32-NEXT:    retl
@@ -142,7 +142,7 @@ define <8 x float> @uitofp_v8i1_v8f32(<8 x i1> %x) #0 {
 ;
 ; AVX512VL-32-LABEL: uitofp_v8i1_v8f32:
 ; AVX512VL-32:       # %bb.0:
-; AVX512VL-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512VL-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512VL-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; AVX512VL-32-NEXT:    retl
@@ -156,7 +156,7 @@ define <8 x float> @uitofp_v8i1_v8f32(<8 x i1> %x) #0 {
 ;
 ; AVX512DQ-32-LABEL: uitofp_v8i1_v8f32:
 ; AVX512DQ-32:       # %bb.0:
-; AVX512DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512DQ-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512DQ-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQ-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; AVX512DQ-32-NEXT:    retl
@@ -170,7 +170,7 @@ define <8 x float> @uitofp_v8i1_v8f32(<8 x i1> %x) #0 {
 ;
 ; AVX512DQVL-32-LABEL: uitofp_v8i1_v8f32:
 ; AVX512DQVL-32:       # %bb.0:
-; AVX512DQVL-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX512DQVL-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX512DQVL-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; AVX512DQVL-32-NEXT:    retl
@@ -386,8 +386,8 @@ define <8 x float> @uitofp_v8i32_v8f32(<8 x i32> %x) #0 {
 ; AVX1-32-NEXT:    vpsrld $16, %xmm2, %xmm2
 ; AVX1-32-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX1-32-NEXT:    vcvtdq2ps %ymm1, %ymm1
-; AVX1-32-NEXT:    vmulps {{\.LCPI.*}}, %ymm1, %ymm1
-; AVX1-32-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; AVX1-32-NEXT:    vmulps {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1
+; AVX1-32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; AVX1-32-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; AVX1-32-NEXT:    vaddps %ymm0, %ymm1, %ymm0
 ; AVX1-32-NEXT:    retl
@@ -462,7 +462,7 @@ define <4 x double> @sitofp_v4i1_v4f64(<4 x i1> %x) #0 {
 define <4 x double> @uitofp_v4i1_v4f64(<4 x i1> %x) #0 {
 ; AVX1-32-LABEL: uitofp_v4i1_v4f64:
 ; AVX1-32:       # %bb.0:
-; AVX1-32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX1-32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX1-32-NEXT:    vcvtdq2pd %xmm0, %ymm0
 ; AVX1-32-NEXT:    retl
 ;
@@ -488,7 +488,7 @@ define <4 x double> @uitofp_v4i1_v4f64(<4 x i1> %x) #0 {
 ;
 ; AVX512VL-32-LABEL: uitofp_v4i1_v4f64:
 ; AVX512VL-32:       # %bb.0:
-; AVX512VL-32-NEXT:    vpandd {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512VL-32-NEXT:    vpandd {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
 ; AVX512VL-32-NEXT:    vcvtdq2pd %xmm0, %ymm0
 ; AVX512VL-32-NEXT:    retl
 ;
@@ -507,7 +507,7 @@ define <4 x double> @uitofp_v4i1_v4f64(<4 x i1> %x) #0 {
 ;
 ; AVX512DQVL-32-LABEL: uitofp_v4i1_v4f64:
 ; AVX512DQVL-32:       # %bb.0:
-; AVX512DQVL-32-NEXT:    vandps {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; AVX512DQVL-32-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
 ; AVX512DQVL-32-NEXT:    vcvtdq2pd %xmm0, %ymm0
 ; AVX512DQVL-32-NEXT:    retl
 ;
@@ -767,25 +767,25 @@ define <4 x double> @uitofp_v4i64_v4f64(<4 x i64> %x) #0 {
 ; AVX-32-NEXT:    vextractps $1, %xmm0, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstpl (%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vextractps $3, %xmm0, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vextractps $1, %xmm1, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vextractps $3, %xmm1, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
@@ -1051,25 +1051,25 @@ define <4 x float> @uitofp_v4i64_v4f32(<4 x i64> %x) #0 {
 ; AVX-32-NEXT:    vextractps $1, %xmm0, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstps (%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vextractps $3, %xmm0, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vextractps $1, %xmm1, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vextractps $3, %xmm1, %eax
 ; AVX-32-NEXT:    shrl $31, %eax
 ; AVX-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; AVX-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; AVX-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; AVX-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; AVX-32-NEXT:    wait
 ; AVX-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll b/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
index 00be17a4d162c70a68858e58f386b0541d994a42..4d54e96a22522a0f2613a2e66514fd5ae472c68a 100644
--- a/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-inttofp-512.ll
@@ -42,7 +42,7 @@ define <16 x float> @sitofp_v16i1_v16f32(<16 x i1> %x) #0 {
 define <16 x float> @uitofp_v16i1_v16f32(<16 x i1> %x) #0 {
 ; NODQ-32-LABEL: uitofp_v16i1_v16f32:
 ; NODQ-32:       # %bb.0:
-; NODQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; NODQ-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; NODQ-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
 ; NODQ-32-NEXT:    vcvtdq2ps %zmm0, %zmm0
 ; NODQ-32-NEXT:    retl
@@ -56,7 +56,7 @@ define <16 x float> @uitofp_v16i1_v16f32(<16 x i1> %x) #0 {
 ;
 ; DQ-32-LABEL: uitofp_v16i1_v16f32:
 ; DQ-32:       # %bb.0:
-; DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; DQ-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; DQ-32-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
 ; DQ-32-NEXT:    vcvtdq2ps %zmm0, %zmm0
 ; DQ-32-NEXT:    retl
@@ -160,7 +160,7 @@ define <8 x double> @sitofp_v8i1_v8f64(<8 x i1> %x) #0 {
 define <8 x double> @uitofp_v8i1_v8f64(<8 x i1> %x) #0 {
 ; NODQ-32-LABEL: uitofp_v8i1_v8f64:
 ; NODQ-32:       # %bb.0:
-; NODQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; NODQ-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; NODQ-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; NODQ-32-NEXT:    vcvtdq2pd %ymm0, %zmm0
 ; NODQ-32-NEXT:    retl
@@ -174,7 +174,7 @@ define <8 x double> @uitofp_v8i1_v8f64(<8 x i1> %x) #0 {
 ;
 ; DQ-32-LABEL: uitofp_v8i1_v8f64:
 ; DQ-32:       # %bb.0:
-; DQ-32-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; DQ-32-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; DQ-32-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; DQ-32-NEXT:    vcvtdq2pd %ymm0, %zmm0
 ; DQ-32-NEXT:    retl
@@ -387,49 +387,49 @@ define <8 x double> @uitofp_v8i64_v8f64(<8 x i64> %x) #0 {
 ; NODQ-32-NEXT:    vextractps $1, %xmm2, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $3, %xmm2, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $1, %xmm3, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $3, %xmm3, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $1, %xmm0, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstpl (%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $3, %xmm0, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $1, %xmm1, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $3, %xmm1, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
@@ -612,49 +612,49 @@ define <8 x float> @uitofp_v8i64_v8f32(<8 x i64> %x) #0 {
 ; NODQ-32-NEXT:    vextractps $1, %xmm0, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstps (%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $3, %xmm0, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $1, %xmm3, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $3, %xmm3, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $1, %xmm2, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $3, %xmm2, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $1, %xmm1, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vextractps $3, %xmm1, %eax
 ; NODQ-32-NEXT:    shrl $31, %eax
 ; NODQ-32-NEXT:    fildll {{[0-9]+}}(%esp)
-; NODQ-32-NEXT:    fadds {{\.LCPI.*}}(,%eax,4)
+; NODQ-32-NEXT:    fadds {{\.LCPI[0-9]+_[0-9]+}}(,%eax,4)
 ; NODQ-32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; NODQ-32-NEXT:    wait
 ; NODQ-32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/vec_fabs.ll b/llvm/test/CodeGen/X86/vec_fabs.ll
index e866ef23a5b991deb297f0ee1e285e34200cf157..8ce675054f413a27c32e7aebecdbb0ff78b196b2 100644
--- a/llvm/test/CodeGen/X86/vec_fabs.ll
+++ b/llvm/test/CodeGen/X86/vec_fabs.ll
@@ -9,7 +9,7 @@
 define <2 x double> @fabs_v2f64(<2 x double> %p) {
 ; X86-LABEL: fabs_v2f64:
 ; X86:       # %bb.0:
-; X86-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: fabs_v2f64:
@@ -24,17 +24,17 @@ declare <2 x double> @llvm.fabs.v2f64(<2 x double> %p)
 define <4 x float> @fabs_v4f32(<4 x float> %p) {
 ; X86-AVX-LABEL: fabs_v4f32:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X86-AVX512VL-LABEL: fabs_v4f32:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vpandd {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; X86-AVX512VL-NEXT:    vpandd {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
 ; X86-AVX512VL-NEXT:    retl
 ;
 ; X86-AVX512VLDQ-LABEL: fabs_v4f32:
 ; X86-AVX512VLDQ:       # %bb.0:
-; X86-AVX512VLDQ-NEXT:    vandps {{\.LCPI.*}}{1to4}, %xmm0, %xmm0
+; X86-AVX512VLDQ-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0
 ; X86-AVX512VLDQ-NEXT:    retl
 ;
 ; X64-AVX-LABEL: fabs_v4f32:
@@ -59,17 +59,17 @@ declare <4 x float> @llvm.fabs.v4f32(<4 x float> %p)
 define <4 x double> @fabs_v4f64(<4 x double> %p) {
 ; X86-AVX-LABEL: fabs_v4f64:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X86-AVX512VL-LABEL: fabs_v4f64:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vpandq {{\.LCPI.*}}{1to4}, %ymm0, %ymm0
+; X86-AVX512VL-NEXT:    vpandq {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %ymm0, %ymm0
 ; X86-AVX512VL-NEXT:    retl
 ;
 ; X86-AVX512VLDQ-LABEL: fabs_v4f64:
 ; X86-AVX512VLDQ:       # %bb.0:
-; X86-AVX512VLDQ-NEXT:    vandpd {{\.LCPI.*}}{1to4}, %ymm0, %ymm0
+; X86-AVX512VLDQ-NEXT:    vandpd {{\.LCPI[0-9]+_[0-9]+}}{1to4}, %ymm0, %ymm0
 ; X86-AVX512VLDQ-NEXT:    retl
 ;
 ; X64-AVX-LABEL: fabs_v4f64:
@@ -94,17 +94,17 @@ declare <4 x double> @llvm.fabs.v4f64(<4 x double> %p)
 define <8 x float> @fabs_v8f32(<8 x float> %p) {
 ; X86-AVX-LABEL: fabs_v8f32:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X86-AVX512VL-LABEL: fabs_v8f32:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vpandd {{\.LCPI.*}}{1to8}, %ymm0, %ymm0
+; X86-AVX512VL-NEXT:    vpandd {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
 ; X86-AVX512VL-NEXT:    retl
 ;
 ; X86-AVX512VLDQ-LABEL: fabs_v8f32:
 ; X86-AVX512VLDQ:       # %bb.0:
-; X86-AVX512VLDQ-NEXT:    vandps {{\.LCPI.*}}{1to8}, %ymm0, %ymm0
+; X86-AVX512VLDQ-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
 ; X86-AVX512VLDQ-NEXT:    retl
 ;
 ; X64-AVX-LABEL: fabs_v8f32:
@@ -136,12 +136,12 @@ define <8 x double> @fabs_v8f64(<8 x double> %p) {
 ;
 ; X86-AVX512VL-LABEL: fabs_v8f64:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vpandq {{\.LCPI.*}}{1to8}, %zmm0, %zmm0
+; X86-AVX512VL-NEXT:    vpandq {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %zmm0, %zmm0
 ; X86-AVX512VL-NEXT:    retl
 ;
 ; X86-AVX512VLDQ-LABEL: fabs_v8f64:
 ; X86-AVX512VLDQ:       # %bb.0:
-; X86-AVX512VLDQ-NEXT:    vandpd {{\.LCPI.*}}{1to8}, %zmm0, %zmm0
+; X86-AVX512VLDQ-NEXT:    vandpd {{\.LCPI[0-9]+_[0-9]+}}{1to8}, %zmm0, %zmm0
 ; X86-AVX512VLDQ-NEXT:    retl
 ;
 ; X64-AVX-LABEL: fabs_v8f64:
@@ -175,12 +175,12 @@ define <16 x float> @fabs_v16f32(<16 x float> %p) {
 ;
 ; X86-AVX512VL-LABEL: fabs_v16f32:
 ; X86-AVX512VL:       # %bb.0:
-; X86-AVX512VL-NEXT:    vpandd {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; X86-AVX512VL-NEXT:    vpandd {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; X86-AVX512VL-NEXT:    retl
 ;
 ; X86-AVX512VLDQ-LABEL: fabs_v16f32:
 ; X86-AVX512VLDQ:       # %bb.0:
-; X86-AVX512VLDQ-NEXT:    vandps {{\.LCPI.*}}{1to16}, %zmm0, %zmm0
+; X86-AVX512VLDQ-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm0
 ; X86-AVX512VLDQ-NEXT:    retl
 ;
 ; X64-AVX-LABEL: fabs_v16f32:
diff --git a/llvm/test/CodeGen/X86/vec_fneg.ll b/llvm/test/CodeGen/X86/vec_fneg.ll
index 3794bd2ce94bf7175336884b6496b79babf7389a..9c162d2123c6f333b66f81ee64e6399a45ea8ebd 100644
--- a/llvm/test/CodeGen/X86/vec_fneg.ll
+++ b/llvm/test/CodeGen/X86/vec_fneg.ll
@@ -10,7 +10,7 @@
 define <4 x float> @t1(<4 x float> %Q) nounwind {
 ; X32-SSE-LABEL: t1:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: t1:
@@ -166,7 +166,7 @@ define <2 x float> @fneg_bitcast(i64 %i) nounwind {
 define <4 x float> @fneg_undef_elts_v4f32(<4 x float> %x) {
 ; X32-SSE-LABEL: fneg_undef_elts_v4f32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fneg_undef_elts_v4f32:
@@ -194,7 +194,7 @@ define <4 x float> @fsub0_undef_elts_v4f32(<4 x float> %x) {
 define <4 x float> @fneg(<4 x float> %Q) nounwind {
 ; X32-SSE-LABEL: fneg:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fneg:
diff --git a/llvm/test/CodeGen/X86/vec_fpext.ll b/llvm/test/CodeGen/X86/vec_fpext.ll
index 46ad1f16f3c08a5250aac9fb579c0aa1f0f56f37..84b6039dc1ad84694504655a46489f7c8e19c507 100644
--- a/llvm/test/CodeGen/X86/vec_fpext.ll
+++ b/llvm/test/CodeGen/X86/vec_fpext.ll
@@ -255,42 +255,42 @@ define <2 x double> @fpext_fromconst() {
 ; X32-SSE:       # %bb.0: # %entry
 ; X32-SSE-NEXT:    movaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0]
 ; X32-SSE-NEXT:    # encoding: [0x0f,0x28,0x05,A,A,A,A]
-; X32-SSE-NEXT:    # fixup A - offset: 3, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X32-SSE-NEXT:    # fixup A - offset: 3, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X32-SSE-NEXT:    retl # encoding: [0xc3]
 ;
 ; X32-AVX-LABEL: fpext_fromconst:
 ; X32-AVX:       # %bb.0: # %entry
 ; X32-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0]
 ; X32-AVX-NEXT:    # encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; X32-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X32-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X32-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X32-AVX512VL-LABEL: fpext_fromconst:
 ; X32-AVX512VL:       # %bb.0: # %entry
-; X32-AVX512VL-NEXT:    vmovaps {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [1.0E+0,-2.0E+0]
+; X32-AVX512VL-NEXT:    vmovaps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [1.0E+0,-2.0E+0]
 ; X32-AVX512VL-NEXT:    # encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; X32-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X32-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
 ; X32-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-SSE-LABEL: fpext_fromconst:
 ; X64-SSE:       # %bb.0: # %entry
 ; X64-SSE-NEXT:    movaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0]
 ; X64-SSE-NEXT:    # encoding: [0x0f,0x28,0x05,A,A,A,A]
-; X64-SSE-NEXT:    # fixup A - offset: 3, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-SSE-NEXT:    # fixup A - offset: 3, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-SSE-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: fpext_fromconst:
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0]
 ; X64-AVX-NEXT:    # encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: fpext_fromconst:
 ; X64-AVX512VL:       # %bb.0: # %entry
 ; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [1.0E+0,-2.0E+0]
 ; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte
 ; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
 entry:
   %0  = insertelement <2 x float> undef, float 1.0, i32 0
diff --git a/llvm/test/CodeGen/X86/vec_fptrunc.ll b/llvm/test/CodeGen/X86/vec_fptrunc.ll
index e7318d9d69723c0c32f95b378caf23496a66ca54..56da56204799a5fa42eca94c200c63e3c8adf9b8 100644
--- a/llvm/test/CodeGen/X86/vec_fptrunc.ll
+++ b/llvm/test/CodeGen/X86/vec_fptrunc.ll
@@ -186,14 +186,14 @@ define <4 x float> @fptrunc_fromreg2_zext(<2 x double> %arg) {
 define <4 x float> @fptrunc_fromconst() {
 ; X32-SSE-LABEL: fptrunc_fromconst:
 ; X32-SSE:       # %bb.0: # %entry
-; X32-SSE-NEXT:    cvtpd2ps {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    cvtpd2ps {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    cvtpd2ps {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
+; X32-SSE-NEXT:    cvtpd2ps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; X32-SSE-NEXT:    retl
 ;
 ; X32-AVX-LABEL: fptrunc_fromconst:
 ; X32-AVX:       # %bb.0: # %entry
-; X32-AVX-NEXT:    vcvtpd2psy {{\.LCPI.*}}, %xmm0
+; X32-AVX-NEXT:    vcvtpd2psy {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: fptrunc_fromconst:
diff --git a/llvm/test/CodeGen/X86/vec_logical.ll b/llvm/test/CodeGen/X86/vec_logical.ll
index ec29d4886a2bd4e857086aa906f731e10647f3bc..12ee3a7336faa80a19eeabb0ced47e97c41928f9 100644
--- a/llvm/test/CodeGen/X86/vec_logical.ll
+++ b/llvm/test/CodeGen/X86/vec_logical.ll
@@ -5,13 +5,13 @@
 define void @t(<4 x float> %A) {
 ; SSE-LABEL: t:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; SSE-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; SSE-NEXT:    movaps %xmm0, 0
 ; SSE-NEXT:    retl
 ;
 ; AVX-LABEL: t:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vxorps {{\.LCPI.*}}, %xmm0, %xmm0
+; AVX-NEXT:    vxorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; AVX-NEXT:    vmovaps %xmm0, 0
 ; AVX-NEXT:    retl
   %tmp1277 = fsub <4 x float> < float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00 >, %A
diff --git a/llvm/test/CodeGen/X86/vec_partial.ll b/llvm/test/CodeGen/X86/vec_partial.ll
index a9044c6ffb50130ad294963de0a2d07a98877f09..f3c8369e783487a2ae32829ae07f33538cc04a8c 100644
--- a/llvm/test/CodeGen/X86/vec_partial.ll
+++ b/llvm/test/CodeGen/X86/vec_partial.ll
@@ -6,7 +6,7 @@
 define <3 x float> @addf3(<3 x float> %x) {
 ; X86-LABEL: addf3:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    addps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    addps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: addf3:
diff --git a/llvm/test/CodeGen/X86/vec_reassociate.ll b/llvm/test/CodeGen/X86/vec_reassociate.ll
index c8b61809d31ad20f16cd6a46460b0171723f3fcd..277f877cacf0c9703f2c7acacd19041c31e43d17 100644
--- a/llvm/test/CodeGen/X86/vec_reassociate.ll
+++ b/llvm/test/CodeGen/X86/vec_reassociate.ll
@@ -38,7 +38,7 @@ define <4 x i32> @mul_4i32(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86-LABEL: mul_4i32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pmulld %xmm1, %xmm0
-; X86-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: mul_4i32:
@@ -56,7 +56,7 @@ define <4 x i32> @mul_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86-LABEL: mul_4i32_commute:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pmulld %xmm1, %xmm0
-; X86-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: mul_4i32_commute:
@@ -74,7 +74,7 @@ define <4 x i32> @and_4i32(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86-LABEL: and_4i32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    andps %xmm1, %xmm0
-; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: and_4i32:
@@ -92,7 +92,7 @@ define <4 x i32> @and_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86-LABEL: and_4i32_commute:
 ; X86:       # %bb.0:
 ; X86-NEXT:    andps %xmm1, %xmm0
-; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: and_4i32_commute:
@@ -110,7 +110,7 @@ define <4 x i32> @or_4i32(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86-LABEL: or_4i32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    orps %xmm1, %xmm0
-; X86-NEXT:    orps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    orps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: or_4i32:
@@ -128,7 +128,7 @@ define <4 x i32> @or_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86-LABEL: or_4i32_commute:
 ; X86:       # %bb.0:
 ; X86-NEXT:    orps %xmm1, %xmm0
-; X86-NEXT:    orps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    orps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: or_4i32_commute:
@@ -146,7 +146,7 @@ define <4 x i32> @xor_4i32(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86-LABEL: xor_4i32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    xorps %xmm1, %xmm0
-; X86-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: xor_4i32:
@@ -164,7 +164,7 @@ define <4 x i32> @xor_4i32_commute(<4 x i32> %a0, <4 x i32> %a1) {
 ; X86-LABEL: xor_4i32_commute:
 ; X86:       # %bb.0:
 ; X86-NEXT:    xorps %xmm1, %xmm0
-; X86-NEXT:    xorps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    xorps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: xor_4i32_commute:
diff --git a/llvm/test/CodeGen/X86/vec_saddo.ll b/llvm/test/CodeGen/X86/vec_saddo.ll
index ede201903a47c9bf52bb5b86c4f7fd728e88c3a8..f07b4db1388ff78762dba007b807753a26046246 100644
--- a/llvm/test/CodeGen/X86/vec_saddo.ll
+++ b/llvm/test/CodeGen/X86/vec_saddo.ll
@@ -539,12 +539,11 @@ define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT:    psrad $24, %xmm4
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm4
-; SSE2-NEXT:    psrad $31, %xmm4
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -568,12 +567,11 @@ define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSSE3-NEXT:    pcmpeqb %xmm0, %xmm2
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSSE3-NEXT:    psrad $24, %xmm4
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm1
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm4
-; SSSE3-NEXT:    psrad $31, %xmm4
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -597,9 +595,7 @@ define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pcmpeqb %xmm0, %xmm2
 ; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm4
-; SSE41-NEXT:    psrad $31, %xmm4
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm4
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
@@ -673,10 +669,8 @@ define <8 x i32> @saddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE2-NEXT:    pcmpeqw %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    psrad $16, %xmm2
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -692,10 +686,8 @@ define <8 x i32> @saddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSSE3-NEXT:    pcmpeqw %xmm0, %xmm2
 ; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm2
-; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    psrad $16, %xmm2
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -711,9 +703,7 @@ define <8 x i32> @saddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE41-NEXT:    pcmpeqw %xmm0, %xmm2
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    pslld $31, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
 ; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
diff --git a/llvm/test/CodeGen/X86/vec_shift4.ll b/llvm/test/CodeGen/X86/vec_shift4.ll
index 789796ed4a416fcb63ffa8bd478ad75c7a590027..e0428b185e5fdbf60dd5fe10f7f7add7b4516f0f 100644
--- a/llvm/test/CodeGen/X86/vec_shift4.ll
+++ b/llvm/test/CodeGen/X86/vec_shift4.ll
@@ -6,7 +6,7 @@ define <2 x i64> @shl1(<4 x i32> %r, <4 x i32> %a) nounwind readnone ssp {
 ; X86-LABEL: shl1:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pslld $23, %xmm1
-; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-NEXT:    pmulld %xmm1, %xmm0
 ; X86-NEXT:    retl
@@ -31,12 +31,12 @@ define <2 x i64> @shl2(<16 x i8> %r, <16 x i8> %a) nounwind readnone ssp {
 ; X86-NEXT:    psllw $5, %xmm1
 ; X86-NEXT:    movdqa %xmm0, %xmm3
 ; X86-NEXT:    psllw $4, %xmm3
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm3
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-NEXT:    movdqa %xmm1, %xmm0
 ; X86-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
 ; X86-NEXT:    movdqa %xmm2, %xmm3
 ; X86-NEXT:    psllw $2, %xmm3
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm3
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-NEXT:    paddb %xmm1, %xmm1
 ; X86-NEXT:    movdqa %xmm1, %xmm0
 ; X86-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
diff --git a/llvm/test/CodeGen/X86/vec_smulo.ll b/llvm/test/CodeGen/X86/vec_smulo.ll
index 4fa3367521a5d2a25c6f7909df3fb66a6b30332e..6562518f2c59f898e71fb7e3fdbcb228cb1650ba 100644
--- a/llvm/test/CodeGen/X86/vec_smulo.ll
+++ b/llvm/test/CodeGen/X86/vec_smulo.ll
@@ -1262,42 +1262,41 @@ define <16 x i32> @smulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
 ; SSE2-LABEL: smulo_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSE2-NEXT:    pmulhw %xmm3, %xmm4
+; SSE2-NEXT:    psrlw $8, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSE2-NEXT:    pmulhw %xmm3, %xmm5
+; SSE2-NEXT:    psrlw $8, %xmm5
+; SSE2-NEXT:    packuswb %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm3, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm2, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    packuswb %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtb %xmm4, %xmm0
-; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    psraw $8, %xmm6
-; SSE2-NEXT:    pmullw %xmm3, %xmm6
-; SSE2-NEXT:    psrlw $8, %xmm6
-; SSE2-NEXT:    psraw $8, %xmm5
-; SSE2-NEXT:    psraw $8, %xmm7
-; SSE2-NEXT:    pmullw %xmm5, %xmm7
-; SSE2-NEXT:    psrlw $8, %xmm7
-; SSE2-NEXT:    packuswb %xmm6, %xmm7
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm7
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    packuswb %xmm4, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT:    pcmpeqb %xmm5, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT:    psrad $24, %xmm4
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -1309,47 +1308,47 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm3
 ; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    movdqa %xmm4, (%rdi)
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: smulo_v16i8:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSSE3-NEXT:    pmulhw %xmm3, %xmm4
+; SSSE3-NEXT:    psrlw $8, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSSE3-NEXT:    pmulhw %xmm3, %xmm5
+; SSSE3-NEXT:    psrlw $8, %xmm5
+; SSSE3-NEXT:    packuswb %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    pmullw %xmm3, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; SSSE3-NEXT:    pand %xmm3, %xmm4
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pmullw %xmm1, %xmm0
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pmullw %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSSE3-NEXT:    pand %xmm1, %xmm0
-; SSSE3-NEXT:    pand %xmm1, %xmm4
-; SSSE3-NEXT:    packuswb %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtb %xmm4, %xmm0
-; SSSE3-NEXT:    psraw $8, %xmm3
-; SSSE3-NEXT:    psraw $8, %xmm6
-; SSSE3-NEXT:    pmullw %xmm3, %xmm6
-; SSSE3-NEXT:    psrlw $8, %xmm6
-; SSSE3-NEXT:    psraw $8, %xmm5
-; SSSE3-NEXT:    psraw $8, %xmm7
-; SSSE3-NEXT:    pmullw %xmm5, %xmm7
-; SSSE3-NEXT:    psrlw $8, %xmm7
-; SSSE3-NEXT:    packuswb %xmm6, %xmm7
-; SSSE3-NEXT:    pcmpeqb %xmm0, %xmm7
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    packuswb %xmm4, %xmm0
+; SSSE3-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpeqb %xmm5, %xmm2
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSSE3-NEXT:    psrad $24, %xmm4
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm1
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -1361,40 +1360,41 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm3
 ; SSSE3-NEXT:    psrad $31, %xmm3
-; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: smulo_v16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pmovsxbw %xmm1, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSE41-NEXT:    pmulhw %xmm3, %xmm4
+; SSE41-NEXT:    psrlw $8, %xmm4
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSE41-NEXT:    pmulhw %xmm3, %xmm5
+; SSE41-NEXT:    psrlw $8, %xmm5
+; SSE41-NEXT:    packuswb %xmm4, %xmm5
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pmovsxbw %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    pmullw %xmm2, %xmm4
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
 ; SSE41-NEXT:    pand %xmm1, %xmm0
+; SSE41-NEXT:    pmullw %xmm3, %xmm4
 ; SSE41-NEXT:    pand %xmm1, %xmm4
 ; SSE41-NEXT:    packuswb %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtb %xmm4, %xmm0
-; SSE41-NEXT:    pmullw %xmm3, %xmm6
-; SSE41-NEXT:    psrlw $8, %xmm6
-; SSE41-NEXT:    pmovsxbw %xmm5, %xmm1
-; SSE41-NEXT:    pmovsxbw %xmm7, %xmm2
-; SSE41-NEXT:    pmullw %xmm1, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    packuswb %xmm2, %xmm6
-; SSE41-NEXT:    pcmpeqb %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtb %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqb %xmm5, %xmm2
 ; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE41-NEXT:    pxor %xmm6, %xmm3
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
@@ -1412,42 +1412,40 @@ define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ;
 ; AVX1-LABEL: smulo_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm3
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm4
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm5
-; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm0, %xmm4
+; AVX1-NEXT:    vpcmpgtb %xmm4, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
-; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
-; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovdqa %xmm4, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: smulo_v16i8:
@@ -1527,368 +1525,357 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE2-LABEL: smulo_v32i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movdqa %xmm3, %xmm10
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm1, %xmm9
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; SSE2-NEXT:    pxor %xmm6, %xmm6
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmulhw %xmm5, %xmm6
+; SSE2-NEXT:    psrlw $8, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3],xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
+; SSE2-NEXT:    pmulhw %xmm5, %xmm7
+; SSE2-NEXT:    psrlw $8, %xmm7
+; SSE2-NEXT:    packuswb %xmm6, %xmm7
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm5, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm3, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm8, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm10, %xmm9
-; SSE2-NEXT:    pand %xmm8, %xmm9
-; SSE2-NEXT:    packuswb %xmm1, %xmm9
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtb %xmm9, %xmm1
-; SSE2-NEXT:    psraw $8, %xmm7
-; SSE2-NEXT:    psraw $8, %xmm6
-; SSE2-NEXT:    pmullw %xmm7, %xmm6
+; SSE2-NEXT:    pand %xmm9, %xmm1
+; SSE2-NEXT:    packuswb %xmm6, %xmm1
+; SSE2-NEXT:    pxor %xmm10, %xmm10
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm10
+; SSE2-NEXT:    pcmpeqb %xmm7, %xmm10
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSE2-NEXT:    pxor %xmm8, %xmm10
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm2[8],xmm7[9],xmm2[9],xmm7[10],xmm2[10],xmm7[11],xmm2[11],xmm7[12],xmm2[12],xmm7[13],xmm2[13],xmm7[14],xmm2[14],xmm7[15],xmm2[15]
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
+; SSE2-NEXT:    pmulhw %xmm7, %xmm6
 ; SSE2-NEXT:    psrlw $8, %xmm6
-; SSE2-NEXT:    psraw $8, %xmm4
-; SSE2-NEXT:    psraw $8, %xmm5
-; SSE2-NEXT:    pmullw %xmm4, %xmm5
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSE2-NEXT:    pmulhw %xmm7, %xmm5
 ; SSE2-NEXT:    psrlw $8, %xmm5
 ; SSE2-NEXT:    packuswb %xmm6, %xmm5
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm2, %xmm10
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm7, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm6, %xmm7
+; SSE2-NEXT:    pand %xmm9, %xmm7
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm10, %xmm2
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pand %xmm8, %xmm0
-; SSE2-NEXT:    pand %xmm8, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE2-NEXT:    pxor %xmm8, %xmm5
-; SSE2-NEXT:    packuswb %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm2, %xmm7
-; SSE2-NEXT:    psraw $8, %xmm4
-; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    pmullw %xmm4, %xmm3
-; SSE2-NEXT:    psrlw $8, %xmm3
-; SSE2-NEXT:    psraw $8, %xmm6
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pmullw %xmm6, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    packuswb %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm7, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm8
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm8
-; SSE2-NEXT:    psrad $31, %xmm8
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    pmullw %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm9, %xmm0
+; SSE2-NEXT:    packuswb %xmm7, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
+; SSE2-NEXT:    pcmpeqb %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm11
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    movdqa %xmm11, %xmm6
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
 ; SSE2-NEXT:    pslld $31, %xmm6
 ; SSE2-NEXT:    psrad $31, %xmm6
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $31, %xmm1
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm3, %xmm7
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm11
+; SSE2-NEXT:    psrad $31, %xmm11
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm7
 ; SSE2-NEXT:    psrad $31, %xmm7
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3],xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
 ; SSE2-NEXT:    pslld $31, %xmm3
 ; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm4
-; SSE2-NEXT:    psrad $31, %xmm4
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $31, %xmm5
-; SSE2-NEXT:    psrad $31, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, 16(%rsi)
-; SSE2-NEXT:    movdqa %xmm2, (%rsi)
-; SSE2-NEXT:    movdqa %xmm5, 112(%rdi)
-; SSE2-NEXT:    movdqa %xmm4, 96(%rdi)
-; SSE2-NEXT:    movdqa %xmm3, 80(%rdi)
-; SSE2-NEXT:    movdqa %xmm7, 64(%rdi)
-; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm10
+; SSE2-NEXT:    psrad $31, %xmm10
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1],xmm5[2],xmm8[2],xmm5[3],xmm8[3]
+; SSE2-NEXT:    psrad $24, %xmm5
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
+; SSE2-NEXT:    psrad $24, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, 16(%rsi)
+; SSE2-NEXT:    movdqa %xmm0, (%rsi)
+; SSE2-NEXT:    movdqa %xmm2, 64(%rdi)
+; SSE2-NEXT:    movdqa %xmm5, (%rdi)
+; SSE2-NEXT:    movdqa %xmm10, 112(%rdi)
+; SSE2-NEXT:    movdqa %xmm3, 96(%rdi)
+; SSE2-NEXT:    movdqa %xmm7, 80(%rdi)
+; SSE2-NEXT:    movdqa %xmm11, 48(%rdi)
 ; SSE2-NEXT:    movdqa %xmm6, 32(%rdi)
-; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
-; SSE2-NEXT:    movdqa %xmm8, (%rdi)
+; SSE2-NEXT:    movdqa %xmm4, 16(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: smulo_v32i8:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq %rdi, %rax
-; SSSE3-NEXT:    movdqa %xmm3, %xmm10
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm3[8],xmm7[9],xmm3[9],xmm7[10],xmm3[10],xmm7[11],xmm3[11],xmm7[12],xmm3[12],xmm7[13],xmm3[13],xmm7[14],xmm3[14],xmm7[15],xmm3[15]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm9
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    pmulhw %xmm5, %xmm6
+; SSSE3-NEXT:    psrlw $8, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3],xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
+; SSSE3-NEXT:    pmulhw %xmm5, %xmm7
+; SSSE3-NEXT:    psrlw $8, %xmm7
+; SSSE3-NEXT:    packuswb %xmm6, %xmm7
+; SSSE3-NEXT:    movdqa %xmm3, %xmm5
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm6
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    pmullw %xmm5, %xmm6
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
+; SSSE3-NEXT:    pand %xmm9, %xmm6
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pmullw %xmm3, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
-; SSSE3-NEXT:    pand %xmm8, %xmm1
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pmullw %xmm10, %xmm9
-; SSSE3-NEXT:    pand %xmm8, %xmm9
-; SSSE3-NEXT:    packuswb %xmm1, %xmm9
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtb %xmm9, %xmm1
-; SSSE3-NEXT:    psraw $8, %xmm7
-; SSSE3-NEXT:    psraw $8, %xmm6
-; SSSE3-NEXT:    pmullw %xmm7, %xmm6
+; SSSE3-NEXT:    pand %xmm9, %xmm1
+; SSSE3-NEXT:    packuswb %xmm6, %xmm1
+; SSSE3-NEXT:    pxor %xmm10, %xmm10
+; SSSE3-NEXT:    pcmpgtb %xmm1, %xmm10
+; SSSE3-NEXT:    pcmpeqb %xmm7, %xmm10
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSSE3-NEXT:    pxor %xmm8, %xmm10
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm2[8],xmm7[9],xmm2[9],xmm7[10],xmm2[10],xmm7[11],xmm2[11],xmm7[12],xmm2[12],xmm7[13],xmm2[13],xmm7[14],xmm2[14],xmm7[15],xmm2[15]
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
+; SSSE3-NEXT:    pmulhw %xmm7, %xmm6
 ; SSSE3-NEXT:    psrlw $8, %xmm6
-; SSSE3-NEXT:    psraw $8, %xmm4
-; SSSE3-NEXT:    psraw $8, %xmm5
-; SSSE3-NEXT:    pmullw %xmm4, %xmm5
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSSE3-NEXT:    pmulhw %xmm7, %xmm5
 ; SSSE3-NEXT:    psrlw $8, %xmm5
 ; SSSE3-NEXT:    packuswb %xmm6, %xmm5
-; SSSE3-NEXT:    pcmpeqb %xmm1, %xmm5
-; SSSE3-NEXT:    movdqa %xmm2, %xmm10
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm7
+; SSSE3-NEXT:    movdqa %xmm2, %xmm6
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm7
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    pmullw %xmm7, %xmm0
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pmullw %xmm6, %xmm7
+; SSSE3-NEXT:    pand %xmm9, %xmm7
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pmullw %xmm10, %xmm2
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pand %xmm8, %xmm0
-; SSSE3-NEXT:    pand %xmm8, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSSE3-NEXT:    pxor %xmm8, %xmm5
-; SSSE3-NEXT:    packuswb %xmm0, %xmm2
-; SSSE3-NEXT:    pcmpgtb %xmm2, %xmm7
-; SSSE3-NEXT:    psraw $8, %xmm4
-; SSSE3-NEXT:    psraw $8, %xmm3
-; SSSE3-NEXT:    pmullw %xmm4, %xmm3
-; SSSE3-NEXT:    psrlw $8, %xmm3
-; SSSE3-NEXT:    psraw $8, %xmm6
-; SSSE3-NEXT:    psraw $8, %xmm1
-; SSSE3-NEXT:    pmullw %xmm6, %xmm1
-; SSSE3-NEXT:    psrlw $8, %xmm1
-; SSSE3-NEXT:    packuswb %xmm3, %xmm1
-; SSSE3-NEXT:    pcmpeqb %xmm7, %xmm1
-; SSSE3-NEXT:    pxor %xmm8, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm8
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm8
-; SSSE3-NEXT:    psrad $31, %xmm8
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm6
+; SSSE3-NEXT:    pmullw %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm9, %xmm0
+; SSSE3-NEXT:    packuswb %xmm7, %xmm0
+; SSSE3-NEXT:    pcmpgtb %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpeqb %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm11
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    movdqa %xmm11, %xmm6
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
 ; SSSE3-NEXT:    pslld $31, %xmm6
 ; SSSE3-NEXT:    psrad $31, %xmm6
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm1
-; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm3, %xmm7
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm11
+; SSSE3-NEXT:    psrad $31, %xmm11
+; SSSE3-NEXT:    movdqa %xmm10, %xmm7
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm7
 ; SSSE3-NEXT:    psrad $31, %xmm7
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3],xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
 ; SSSE3-NEXT:    pslld $31, %xmm3
 ; SSSE3-NEXT:    psrad $31, %xmm3
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm4
-; SSSE3-NEXT:    psrad $31, %xmm4
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm5
-; SSSE3-NEXT:    psrad $31, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, 16(%rsi)
-; SSSE3-NEXT:    movdqa %xmm2, (%rsi)
-; SSSE3-NEXT:    movdqa %xmm5, 112(%rdi)
-; SSSE3-NEXT:    movdqa %xmm4, 96(%rdi)
-; SSSE3-NEXT:    movdqa %xmm3, 80(%rdi)
-; SSSE3-NEXT:    movdqa %xmm7, 64(%rdi)
-; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm10
+; SSSE3-NEXT:    psrad $31, %xmm10
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1],xmm5[2],xmm8[2],xmm5[3],xmm8[3]
+; SSSE3-NEXT:    psrad $24, %xmm5
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
+; SSSE3-NEXT:    psrad $24, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, 16(%rsi)
+; SSSE3-NEXT:    movdqa %xmm0, (%rsi)
+; SSSE3-NEXT:    movdqa %xmm2, 64(%rdi)
+; SSSE3-NEXT:    movdqa %xmm5, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm10, 112(%rdi)
+; SSSE3-NEXT:    movdqa %xmm3, 96(%rdi)
+; SSSE3-NEXT:    movdqa %xmm7, 80(%rdi)
+; SSSE3-NEXT:    movdqa %xmm11, 48(%rdi)
 ; SSSE3-NEXT:    movdqa %xmm6, 32(%rdi)
-; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
-; SSSE3-NEXT:    movdqa %xmm8, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, 16(%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: smulo_v32i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    pmovsxbw %xmm3, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
+; SSE41-NEXT:    pxor %xmm11, %xmm11
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
+; SSE41-NEXT:    pmulhw %xmm5, %xmm6
+; SSE41-NEXT:    psrlw $8, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3],xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
+; SSE41-NEXT:    pmulhw %xmm5, %xmm7
+; SSE41-NEXT:    psrlw $8, %xmm7
+; SSE41-NEXT:    packuswb %xmm6, %xmm7
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pmovsxbw %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm3, %xmm1
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
 ; SSE41-NEXT:    pand %xmm8, %xmm1
-; SSE41-NEXT:    pmullw %xmm10, %xmm9
-; SSE41-NEXT:    pand %xmm8, %xmm9
-; SSE41-NEXT:    packuswb %xmm1, %xmm9
+; SSE41-NEXT:    pmullw %xmm6, %xmm10
+; SSE41-NEXT:    pand %xmm8, %xmm10
+; SSE41-NEXT:    packuswb %xmm1, %xmm10
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pcmpgtb %xmm9, %xmm1
-; SSE41-NEXT:    pmullw %xmm7, %xmm4
-; SSE41-NEXT:    psrlw $8, %xmm4
-; SSE41-NEXT:    pmovsxbw %xmm5, %xmm3
-; SSE41-NEXT:    pmovsxbw %xmm6, %xmm5
-; SSE41-NEXT:    pmullw %xmm3, %xmm5
-; SSE41-NEXT:    psrlw $8, %xmm5
-; SSE41-NEXT:    packuswb %xmm5, %xmm4
-; SSE41-NEXT:    pcmpeqb %xmm1, %xmm4
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmovsxbw %xmm2, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
-; SSE41-NEXT:    movdqa %xmm2, %xmm7
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm7, %xmm0
-; SSE41-NEXT:    pmullw %xmm10, %xmm1
+; SSE41-NEXT:    pcmpgtb %xmm10, %xmm1
+; SSE41-NEXT:    pcmpeqb %xmm7, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm9
+; SSE41-NEXT:    pxor %xmm9, %xmm1
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
+; SSE41-NEXT:    pmulhw %xmm3, %xmm6
+; SSE41-NEXT:    psrlw $8, %xmm6
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
 ; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
+; SSE41-NEXT:    pmulhw %xmm3, %xmm7
+; SSE41-NEXT:    psrlw $8, %xmm7
+; SSE41-NEXT:    packuswb %xmm6, %xmm7
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
 ; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    pand %xmm8, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE41-NEXT:    pxor %xmm8, %xmm4
-; SSE41-NEXT:    packuswb %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtb %xmm1, %xmm7
-; SSE41-NEXT:    pmullw %xmm5, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pmovsxbw %xmm6, %xmm0
-; SSE41-NEXT:    pmovsxbw %xmm3, %xmm3
-; SSE41-NEXT:    pmullw %xmm0, %xmm3
-; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    packuswb %xmm3, %xmm2
-; SSE41-NEXT:    pcmpeqb %xmm7, %xmm2
-; SSE41-NEXT:    pxor %xmm8, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm8
-; SSE41-NEXT:    psrad $31, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm5
-; SSE41-NEXT:    psrad $31, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm6
-; SSE41-NEXT:    psrad $31, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm7
-; SSE41-NEXT:    psrad $31, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[2,3,2,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm3
-; SSE41-NEXT:    psrad $31, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[3,3,3,3]
+; SSE41-NEXT:    pmullw %xmm6, %xmm3
+; SSE41-NEXT:    pand %xmm8, %xmm3
+; SSE41-NEXT:    packuswb %xmm0, %xmm3
+; SSE41-NEXT:    pcmpgtb %xmm3, %xmm11
+; SSE41-NEXT:    pcmpeqb %xmm7, %xmm11
+; SSE41-NEXT:    pxor %xmm9, %xmm11
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm11[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm11[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm6
+; SSE41-NEXT:    psrad $31, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,1,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm7
+; SSE41-NEXT:    psrad $31, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm5
+; SSE41-NEXT:    psrad $31, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm4
 ; SSE41-NEXT:    psrad $31, %xmm4
-; SSE41-NEXT:    movdqa %xmm9, 16(%rsi)
-; SSE41-NEXT:    movdqa %xmm1, (%rsi)
-; SSE41-NEXT:    movdqa %xmm4, 64(%rdi)
-; SSE41-NEXT:    movdqa %xmm2, (%rdi)
-; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
-; SSE41-NEXT:    movdqa %xmm3, 96(%rdi)
+; SSE41-NEXT:    pmovsxbd %xmm11, %xmm8
+; SSE41-NEXT:    pmovsxbd %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm10, 16(%rsi)
+; SSE41-NEXT:    movdqa %xmm3, (%rsi)
+; SSE41-NEXT:    movdqa %xmm1, 64(%rdi)
+; SSE41-NEXT:    movdqa %xmm8, (%rdi)
+; SSE41-NEXT:    movdqa %xmm4, 112(%rdi)
+; SSE41-NEXT:    movdqa %xmm5, 96(%rdi)
 ; SSE41-NEXT:    movdqa %xmm7, 80(%rdi)
 ; SSE41-NEXT:    movdqa %xmm6, 48(%rdi)
-; SSE41-NEXT:    movdqa %xmm5, 32(%rdi)
-; SSE41-NEXT:    movdqa %xmm8, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)
+; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: smulo_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
-; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm4, %xmm9
-; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
-; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm8, %xmm3
-; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm7
-; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm4
-; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm6, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
-; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm3
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
-; AVX1-NEXT:    vpand %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpgtb %xmm5, %xmm8, %xmm2
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm6
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm7
-; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
+; AVX1-NEXT:    vpmulhw %xmm6, %xmm7, %xmm6
 ; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm3, %xmm9
+; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm8, %xmm8
+; AVX1-NEXT:    vpxor %xmm3, %xmm8, %xmm3
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX1-NEXT:    vpackuswb %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpand %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm6, %xmm0
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm7, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm5, %xmm0, %xmm5
+; AVX1-NEXT:    vpcmpgtb %xmm5, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm8, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm2
@@ -1913,32 +1900,28 @@ define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ;
 ; AVX2-LABEL: smulo_v32i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vpackuswb %ymm2, %ymm3, %ymm4
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtb %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpsraw $8, %ymm3, %ymm3
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpsraw $8, %ymm5, %ymm5
-; AVX2-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX2-NEXT:    vpmulhw %ymm3, %ymm4, %ymm3
 ; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX2-NEXT:    vpmulhw %ymm4, %ymm5, %ymm4
+; AVX2-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX2-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm4, %ymm0, %ymm4
+; AVX2-NEXT:    vpcmpgtb %ymm4, %ymm2, %ymm0
+; AVX2-NEXT:    vpcmpeqb %ymm0, %ymm3, %ymm0
 ; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm1
 ; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm0
@@ -2025,618 +2008,586 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE2-LABEL: smulo_v64i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movdqa %xmm7, %xmm9
-; SSE2-NEXT:    movdqa %xmm7, %xmm10
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm3, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm11
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm10, %xmm11
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm9, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm9, %xmm11
-; SSE2-NEXT:    pand %xmm9, %xmm8
-; SSE2-NEXT:    packuswb %xmm11, %xmm8
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm3[8],xmm9[9],xmm3[9],xmm9[10],xmm3[10],xmm9[11],xmm3[11],xmm9[12],xmm3[12],xmm9[13],xmm3[13],xmm9[14],xmm3[14],xmm9[15],xmm3[15]
+; SSE2-NEXT:    pmulhw %xmm8, %xmm9
+; SSE2-NEXT:    psrlw $8, %xmm9
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; SSE2-NEXT:    pxor %xmm10, %xmm10
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm3[0],xmm10[1],xmm3[1],xmm10[2],xmm3[2],xmm10[3],xmm3[3],xmm10[4],xmm3[4],xmm10[5],xmm3[5],xmm10[6],xmm3[6],xmm10[7],xmm3[7]
+; SSE2-NEXT:    pmulhw %xmm8, %xmm10
+; SSE2-NEXT:    psrlw $8, %xmm10
+; SSE2-NEXT:    packuswb %xmm9, %xmm10
+; SSE2-NEXT:    movdqa %xmm7, %xmm8
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    movdqa %xmm3, %xmm9
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm8, %xmm9
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
+; SSE2-NEXT:    pand %xmm8, %xmm9
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3],xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
-; SSE2-NEXT:    psraw $8, %xmm10
-; SSE2-NEXT:    psraw $8, %xmm11
-; SSE2-NEXT:    pmullw %xmm10, %xmm11
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpgtb %xmm8, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm7, %xmm3
+; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    packuswb %xmm9, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm9
+; SSE2-NEXT:    pcmpeqb %xmm10, %xmm9
+; SSE2-NEXT:    pxor %xmm10, %xmm10
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm6[8],xmm10[9],xmm6[9],xmm10[10],xmm6[10],xmm10[11],xmm6[11],xmm10[12],xmm6[12],xmm10[13],xmm6[13],xmm10[14],xmm6[14],xmm10[15],xmm6[15]
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm2[8],xmm7[9],xmm2[9],xmm7[10],xmm2[10],xmm7[11],xmm2[11],xmm7[12],xmm2[12],xmm7[13],xmm2[13],xmm7[14],xmm2[14],xmm7[15],xmm2[15]
+; SSE2-NEXT:    pmulhw %xmm10, %xmm7
+; SSE2-NEXT:    psrlw $8, %xmm7
+; SSE2-NEXT:    pxor %xmm10, %xmm10
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3],xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; SSE2-NEXT:    pxor %xmm11, %xmm11
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3],xmm11[4],xmm2[4],xmm11[5],xmm2[5],xmm11[6],xmm2[6],xmm11[7],xmm2[7]
+; SSE2-NEXT:    pmulhw %xmm10, %xmm11
 ; SSE2-NEXT:    psrlw $8, %xmm11
-; SSE2-NEXT:    psraw $8, %xmm7
-; SSE2-NEXT:    psraw $8, %xmm12
-; SSE2-NEXT:    pmullw %xmm7, %xmm12
-; SSE2-NEXT:    psrlw $8, %xmm12
-; SSE2-NEXT:    packuswb %xmm11, %xmm12
-; SSE2-NEXT:    pcmpeqb %xmm3, %xmm12
+; SSE2-NEXT:    packuswb %xmm7, %xmm11
 ; SSE2-NEXT:    movdqa %xmm6, %xmm10
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm7, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm11
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm10, %xmm11
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
-; SSE2-NEXT:    pand %xmm9, %xmm3
-; SSE2-NEXT:    pand %xmm9, %xmm11
-; SSE2-NEXT:    packuswb %xmm3, %xmm11
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE2-NEXT:    pmullw %xmm10, %xmm7
+; SSE2-NEXT:    pand %xmm8, %xmm7
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm7
-; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    pmullw %xmm7, %xmm3
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtb %xmm11, %xmm7
-; SSE2-NEXT:    psrlw $8, %xmm3
-; SSE2-NEXT:    psraw $8, %xmm6
-; SSE2-NEXT:    psraw $8, %xmm2
 ; SSE2-NEXT:    pmullw %xmm6, %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm7, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm6, %xmm7
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    packuswb %xmm7, %xmm2
+; SSE2-NEXT:    pxor %xmm10, %xmm10
+; SSE2-NEXT:    pcmpgtb %xmm2, %xmm10
+; SSE2-NEXT:    pcmpeqb %xmm11, %xmm10
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15]
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
+; SSE2-NEXT:    pmulhw %xmm7, %xmm6
+; SSE2-NEXT:    psrlw $8, %xmm6
+; SSE2-NEXT:    pxor %xmm11, %xmm11
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1],xmm11[2],xmm5[2],xmm11[3],xmm5[3],xmm11[4],xmm5[4],xmm11[5],xmm5[5],xmm11[6],xmm5[6],xmm11[7],xmm5[7]
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3],xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
+; SSE2-NEXT:    pmulhw %xmm11, %xmm7
+; SSE2-NEXT:    psrlw $8, %xmm7
+; SSE2-NEXT:    packuswb %xmm6, %xmm7
+; SSE2-NEXT:    movdqa %xmm5, %xmm11
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm3, %xmm6
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pand %xmm9, %xmm6
-; SSE2-NEXT:    packuswb %xmm7, %xmm6
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm11, %xmm6
+; SSE2-NEXT:    pand %xmm8, %xmm6
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    psraw $8, %xmm7
-; SSE2-NEXT:    pmullw %xmm3, %xmm7
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpgtb %xmm6, %xmm3
-; SSE2-NEXT:    psrlw $8, %xmm7
-; SSE2-NEXT:    psraw $8, %xmm5
-; SSE2-NEXT:    psraw $8, %xmm1
 ; SSE2-NEXT:    pmullw %xmm5, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    packuswb %xmm7, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pand %xmm8, %xmm1
+; SSE2-NEXT:    packuswb %xmm6, %xmm1
+; SSE2-NEXT:    pxor %xmm11, %xmm11
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm11
+; SSE2-NEXT:    pcmpeqb %xmm7, %xmm11
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
+; SSE2-NEXT:    pmulhw %xmm6, %xmm7
+; SSE2-NEXT:    psrlw $8, %xmm7
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSE2-NEXT:    pmulhw %xmm6, %xmm5
+; SSE2-NEXT:    psrlw $8, %xmm5
+; SSE2-NEXT:    packuswb %xmm7, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm7
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm5, %xmm7
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm3, %xmm5
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pand %xmm9, %xmm5
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
-; SSE2-NEXT:    packuswb %xmm7, %xmm5
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
-; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    psraw $8, %xmm7
-; SSE2-NEXT:    pmullw %xmm3, %xmm7
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; SSE2-NEXT:    pmullw %xmm6, %xmm7
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    pmullw %xmm3, %xmm0
-; SSE2-NEXT:    psrlw $8, %xmm7
-; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    pmullw %xmm4, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm7
+; SSE2-NEXT:    pand %xmm8, %xmm0
 ; SSE2-NEXT:    packuswb %xmm7, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpgtb %xmm5, %xmm3
-; SSE2-NEXT:    pcmpeqb %xmm3, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm3, %xmm12
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm3, %xmm0
-; SSE2-NEXT:    movdqa %xmm8, 48(%rsi)
-; SSE2-NEXT:    movdqa %xmm11, 32(%rsi)
-; SSE2-NEXT:    movdqa %xmm6, 16(%rsi)
-; SSE2-NEXT:    movdqa %xmm12, %xmm3
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm5, (%rsi)
-; SSE2-NEXT:    movdqa %xmm12, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm12
-; SSE2-NEXT:    psrad $31, %xmm12
-; SSE2-NEXT:    movdqa %xmm12, 224(%rdi)
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $31, %xmm4
-; SSE2-NEXT:    psrad $31, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, 240(%rdi)
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm3
-; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, 192(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $31, %xmm4
-; SSE2-NEXT:    psrad $31, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, 208(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, 160(%rdi)
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $31, %xmm4
-; SSE2-NEXT:    psrad $31, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, 176(%rdi)
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm3
-; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, 128(%rdi)
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm4
+; SSE2-NEXT:    pcmpeqb %xmm5, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm9
+; SSE2-NEXT:    pxor %xmm5, %xmm10
+; SSE2-NEXT:    pxor %xmm5, %xmm11
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, 48(%rsi)
+; SSE2-NEXT:    movdqa %xmm2, 32(%rsi)
+; SSE2-NEXT:    movdqa %xmm1, 16(%rsi)
+; SSE2-NEXT:    movdqa %xmm9, %xmm1
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, 144(%rdi)
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, (%rsi)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, 192(%rdi)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, 128(%rdi)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1],xmm0[2],xmm11[2],xmm0[3],xmm11[3],xmm0[4],xmm11[4],xmm0[5],xmm11[5],xmm0[6],xmm11[6],xmm0[7],xmm11[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, 64(%rdi)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, 96(%rdi)
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, 112(%rdi)
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm3
-; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, 64(%rdi)
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, 224(%rdi)
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, 240(%rdi)
+; SSE2-NEXT:    movdqa %xmm10, %xmm0
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm9 = xmm9[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm9
+; SSE2-NEXT:    psrad $31, %xmm9
+; SSE2-NEXT:    movdqa %xmm9, 208(%rdi)
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pslld $31, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, 160(%rdi)
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, 80(%rdi)
+; SSE2-NEXT:    movdqa %xmm1, 176(%rdi)
+; SSE2-NEXT:    movdqa %xmm11, %xmm0
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm10
+; SSE2-NEXT:    psrad $31, %xmm10
+; SSE2-NEXT:    movdqa %xmm10, 144(%rdi)
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSE2-NEXT:    pslld $31, %xmm0
 ; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, 32(%rdi)
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movdqa %xmm0, 96(%rdi)
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, (%rdi)
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movdqa %xmm1, 112(%rdi)
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm11
+; SSE2-NEXT:    psrad $31, %xmm11
+; SSE2-NEXT:    movdqa %xmm11, 80(%rdi)
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSE2-NEXT:    pslld $31, %xmm0
 ; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm0, 32(%rdi)
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, 16(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: smulo_v64i8:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq %rdi, %rax
-; SSSE3-NEXT:    movdqa %xmm7, %xmm9
-; SSSE3-NEXT:    movdqa %xmm7, %xmm10
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm3, %xmm8
-; SSSE3-NEXT:    movdqa %xmm3, %xmm11
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    pmullw %xmm10, %xmm11
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm7[8],xmm10[9],xmm7[9],xmm10[10],xmm7[10],xmm10[11],xmm7[11],xmm10[12],xmm7[12],xmm10[13],xmm7[13],xmm10[14],xmm7[14],xmm10[15],xmm7[15]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pmullw %xmm9, %xmm8
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
-; SSSE3-NEXT:    pand %xmm9, %xmm11
-; SSSE3-NEXT:    pand %xmm9, %xmm8
-; SSSE3-NEXT:    packuswb %xmm11, %xmm8
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm3[8],xmm11[9],xmm3[9],xmm11[10],xmm3[10],xmm11[11],xmm3[11],xmm11[12],xmm3[12],xmm11[13],xmm3[13],xmm11[14],xmm3[14],xmm11[15],xmm3[15]
+; SSSE3-NEXT:    pxor %xmm8, %xmm8
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
+; SSSE3-NEXT:    pxor %xmm9, %xmm9
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm3[8],xmm9[9],xmm3[9],xmm9[10],xmm3[10],xmm9[11],xmm3[11],xmm9[12],xmm3[12],xmm9[13],xmm3[13],xmm9[14],xmm3[14],xmm9[15],xmm3[15]
+; SSSE3-NEXT:    pmulhw %xmm8, %xmm9
+; SSSE3-NEXT:    psrlw $8, %xmm9
+; SSSE3-NEXT:    pxor %xmm8, %xmm8
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; SSSE3-NEXT:    pxor %xmm10, %xmm10
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm3[0],xmm10[1],xmm3[1],xmm10[2],xmm3[2],xmm10[3],xmm3[3],xmm10[4],xmm3[4],xmm10[5],xmm3[5],xmm10[6],xmm3[6],xmm10[7],xmm3[7]
+; SSSE3-NEXT:    pmulhw %xmm8, %xmm10
+; SSSE3-NEXT:    psrlw $8, %xmm10
+; SSSE3-NEXT:    packuswb %xmm9, %xmm10
+; SSSE3-NEXT:    movdqa %xmm7, %xmm8
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm9
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    pmullw %xmm8, %xmm9
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]
+; SSSE3-NEXT:    pand %xmm8, %xmm9
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3],xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
-; SSSE3-NEXT:    psraw $8, %xmm10
-; SSSE3-NEXT:    psraw $8, %xmm11
-; SSSE3-NEXT:    pmullw %xmm10, %xmm11
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pcmpgtb %xmm8, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pmullw %xmm7, %xmm3
+; SSSE3-NEXT:    pand %xmm8, %xmm3
+; SSSE3-NEXT:    packuswb %xmm9, %xmm3
+; SSSE3-NEXT:    pxor %xmm9, %xmm9
+; SSSE3-NEXT:    pcmpgtb %xmm3, %xmm9
+; SSSE3-NEXT:    pcmpeqb %xmm10, %xmm9
+; SSSE3-NEXT:    pxor %xmm10, %xmm10
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm6[8],xmm10[9],xmm6[9],xmm10[10],xmm6[10],xmm10[11],xmm6[11],xmm10[12],xmm6[12],xmm10[13],xmm6[13],xmm10[14],xmm6[14],xmm10[15],xmm6[15]
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm2[8],xmm7[9],xmm2[9],xmm7[10],xmm2[10],xmm7[11],xmm2[11],xmm7[12],xmm2[12],xmm7[13],xmm2[13],xmm7[14],xmm2[14],xmm7[15],xmm2[15]
+; SSSE3-NEXT:    pmulhw %xmm10, %xmm7
+; SSSE3-NEXT:    psrlw $8, %xmm7
+; SSSE3-NEXT:    pxor %xmm10, %xmm10
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3],xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; SSSE3-NEXT:    pxor %xmm11, %xmm11
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3],xmm11[4],xmm2[4],xmm11[5],xmm2[5],xmm11[6],xmm2[6],xmm11[7],xmm2[7]
+; SSSE3-NEXT:    pmulhw %xmm10, %xmm11
 ; SSSE3-NEXT:    psrlw $8, %xmm11
-; SSSE3-NEXT:    psraw $8, %xmm7
-; SSSE3-NEXT:    psraw $8, %xmm12
-; SSSE3-NEXT:    pmullw %xmm7, %xmm12
-; SSSE3-NEXT:    psrlw $8, %xmm12
-; SSSE3-NEXT:    packuswb %xmm11, %xmm12
-; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm12
+; SSSE3-NEXT:    packuswb %xmm7, %xmm11
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm10
-; SSSE3-NEXT:    movdqa %xmm6, %xmm7
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm7
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    pmullw %xmm7, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm11
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pmullw %xmm10, %xmm11
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
-; SSSE3-NEXT:    pand %xmm9, %xmm3
-; SSSE3-NEXT:    pand %xmm9, %xmm11
-; SSSE3-NEXT:    packuswb %xmm3, %xmm11
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSSE3-NEXT:    pmullw %xmm10, %xmm7
+; SSSE3-NEXT:    pand %xmm8, %xmm7
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    psraw $8, %xmm7
-; SSSE3-NEXT:    psraw $8, %xmm3
-; SSSE3-NEXT:    pmullw %xmm7, %xmm3
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtb %xmm11, %xmm7
-; SSSE3-NEXT:    psrlw $8, %xmm3
-; SSSE3-NEXT:    psraw $8, %xmm6
-; SSSE3-NEXT:    psraw $8, %xmm2
 ; SSSE3-NEXT:    pmullw %xmm6, %xmm2
-; SSSE3-NEXT:    psrlw $8, %xmm2
-; SSSE3-NEXT:    packuswb %xmm3, %xmm2
-; SSSE3-NEXT:    pcmpeqb %xmm7, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    movdqa %xmm5, %xmm6
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm7
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    pmullw %xmm6, %xmm7
+; SSSE3-NEXT:    pand %xmm8, %xmm2
+; SSSE3-NEXT:    packuswb %xmm7, %xmm2
+; SSSE3-NEXT:    pxor %xmm10, %xmm10
+; SSSE3-NEXT:    pcmpgtb %xmm2, %xmm10
+; SSSE3-NEXT:    pcmpeqb %xmm11, %xmm10
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15]
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
+; SSSE3-NEXT:    pmulhw %xmm7, %xmm6
+; SSSE3-NEXT:    psrlw $8, %xmm6
+; SSSE3-NEXT:    pxor %xmm11, %xmm11
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1],xmm11[2],xmm5[2],xmm11[3],xmm5[3],xmm11[4],xmm5[4],xmm11[5],xmm5[5],xmm11[6],xmm5[6],xmm11[7],xmm5[7]
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3],xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
+; SSSE3-NEXT:    pmulhw %xmm11, %xmm7
+; SSSE3-NEXT:    psrlw $8, %xmm7
+; SSSE3-NEXT:    packuswb %xmm6, %xmm7
+; SSSE3-NEXT:    movdqa %xmm5, %xmm11
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm6
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pmullw %xmm3, %xmm6
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
-; SSSE3-NEXT:    pand %xmm9, %xmm7
-; SSSE3-NEXT:    pand %xmm9, %xmm6
-; SSSE3-NEXT:    packuswb %xmm7, %xmm6
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    pmullw %xmm11, %xmm6
+; SSSE3-NEXT:    pand %xmm8, %xmm6
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    psraw $8, %xmm3
-; SSSE3-NEXT:    psraw $8, %xmm7
-; SSSE3-NEXT:    pmullw %xmm3, %xmm7
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pcmpgtb %xmm6, %xmm3
-; SSSE3-NEXT:    psrlw $8, %xmm7
-; SSSE3-NEXT:    psraw $8, %xmm5
-; SSSE3-NEXT:    psraw $8, %xmm1
 ; SSSE3-NEXT:    pmullw %xmm5, %xmm1
-; SSSE3-NEXT:    psrlw $8, %xmm1
-; SSSE3-NEXT:    packuswb %xmm7, %xmm1
-; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm1
-; SSSE3-NEXT:    movdqa %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    pand %xmm8, %xmm1
+; SSSE3-NEXT:    packuswb %xmm6, %xmm1
+; SSSE3-NEXT:    pxor %xmm11, %xmm11
+; SSSE3-NEXT:    pcmpgtb %xmm1, %xmm11
+; SSSE3-NEXT:    pcmpeqb %xmm7, %xmm11
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
+; SSSE3-NEXT:    pmulhw %xmm6, %xmm7
+; SSSE3-NEXT:    psrlw $8, %xmm7
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSSE3-NEXT:    pmulhw %xmm6, %xmm5
+; SSSE3-NEXT:    psrlw $8, %xmm5
+; SSSE3-NEXT:    packuswb %xmm7, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm6
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm7
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    pmullw %xmm5, %xmm7
-; SSSE3-NEXT:    movdqa %xmm0, %xmm5
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pmullw %xmm3, %xmm5
-; SSSE3-NEXT:    pand %xmm9, %xmm7
-; SSSE3-NEXT:    pand %xmm9, %xmm5
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
-; SSSE3-NEXT:    packuswb %xmm7, %xmm5
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
-; SSSE3-NEXT:    psraw $8, %xmm3
-; SSSE3-NEXT:    psraw $8, %xmm7
-; SSSE3-NEXT:    pmullw %xmm3, %xmm7
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; SSSE3-NEXT:    pmullw %xmm6, %xmm7
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    psraw $8, %xmm3
-; SSSE3-NEXT:    psraw $8, %xmm0
-; SSSE3-NEXT:    pmullw %xmm3, %xmm0
-; SSSE3-NEXT:    psrlw $8, %xmm7
-; SSSE3-NEXT:    psrlw $8, %xmm0
+; SSSE3-NEXT:    pmullw %xmm4, %xmm0
+; SSSE3-NEXT:    pand %xmm8, %xmm7
+; SSSE3-NEXT:    pand %xmm8, %xmm0
 ; SSSE3-NEXT:    packuswb %xmm7, %xmm0
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pcmpgtb %xmm5, %xmm3
-; SSSE3-NEXT:    pcmpeqb %xmm3, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm3, %xmm12
-; SSSE3-NEXT:    pxor %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm3, %xmm0
-; SSSE3-NEXT:    movdqa %xmm8, 48(%rsi)
-; SSSE3-NEXT:    movdqa %xmm11, 32(%rsi)
-; SSSE3-NEXT:    movdqa %xmm6, 16(%rsi)
-; SSSE3-NEXT:    movdqa %xmm12, %xmm3
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm5, (%rsi)
-; SSSE3-NEXT:    movdqa %xmm12, %xmm4
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm12
-; SSSE3-NEXT:    psrad $31, %xmm12
-; SSSE3-NEXT:    movdqa %xmm12, 224(%rdi)
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm4
-; SSSE3-NEXT:    psrad $31, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, 240(%rdi)
-; SSSE3-NEXT:    movdqa %xmm3, %xmm4
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm3
-; SSSE3-NEXT:    psrad $31, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, 192(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm4
-; SSSE3-NEXT:    psrad $31, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, 208(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm2
-; SSSE3-NEXT:    psrad $31, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, 160(%rdi)
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm4
-; SSSE3-NEXT:    psrad $31, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, 176(%rdi)
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm3
-; SSSE3-NEXT:    psrad $31, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, 128(%rdi)
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtb %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpeqb %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm9
+; SSSE3-NEXT:    pxor %xmm5, %xmm10
+; SSSE3-NEXT:    pxor %xmm5, %xmm11
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    movdqa %xmm3, 48(%rsi)
+; SSSE3-NEXT:    movdqa %xmm2, 32(%rsi)
+; SSSE3-NEXT:    movdqa %xmm1, 16(%rsi)
+; SSSE3-NEXT:    movdqa %xmm9, %xmm1
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm2
-; SSSE3-NEXT:    psrad $31, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, 144(%rdi)
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm0, (%rsi)
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, 192(%rdi)
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, 128(%rdi)
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1],xmm0[2],xmm11[2],xmm0[3],xmm11[3],xmm0[4],xmm11[4],xmm0[5],xmm11[5],xmm0[6],xmm11[6],xmm0[7],xmm11[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, 64(%rdi)
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, 96(%rdi)
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm2
-; SSSE3-NEXT:    psrad $31, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, 112(%rdi)
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm3
-; SSSE3-NEXT:    psrad $31, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, 64(%rdi)
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, 224(%rdi)
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, 240(%rdi)
+; SSSE3-NEXT:    movdqa %xmm10, %xmm0
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm9 = xmm9[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm9
+; SSSE3-NEXT:    psrad $31, %xmm9
+; SSSE3-NEXT:    movdqa %xmm9, 208(%rdi)
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, 160(%rdi)
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, 176(%rdi)
+; SSSE3-NEXT:    movdqa %xmm11, %xmm0
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm10
+; SSSE3-NEXT:    psrad $31, %xmm10
+; SSSE3-NEXT:    movdqa %xmm10, 144(%rdi)
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, 96(%rdi)
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, 80(%rdi)
+; SSSE3-NEXT:    movdqa %xmm1, 112(%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm11
+; SSSE3-NEXT:    psrad $31, %xmm11
+; SSSE3-NEXT:    movdqa %xmm11, 80(%rdi)
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm1
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
 ; SSSE3-NEXT:    pslld $31, %xmm0
 ; SSSE3-NEXT:    psrad $31, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm0, 32(%rdi)
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm2
-; SSSE3-NEXT:    psrad $31, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, 16(%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: smulo_v64i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
-; SSE41-NEXT:    movdqa %xmm7, %xmm10
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; SSE41-NEXT:    movdqa %xmm3, %xmm11
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm10, %xmm11
-; SSE41-NEXT:    pmovsxbw %xmm7, %xmm12
-; SSE41-NEXT:    pmullw %xmm8, %xmm9
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pand %xmm10, %xmm11
-; SSE41-NEXT:    pand %xmm10, %xmm9
-; SSE41-NEXT:    packuswb %xmm11, %xmm9
-; SSE41-NEXT:    pmovsxbw %xmm3, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[2,3,2,3]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm3[2,3,2,3]
-; SSE41-NEXT:    pmullw %xmm12, %xmm8
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtb %xmm9, %xmm7
-; SSE41-NEXT:    psrlw $8, %xmm8
-; SSE41-NEXT:    pmovsxbw %xmm11, %xmm11
-; SSE41-NEXT:    pmovsxbw %xmm13, %xmm3
-; SSE41-NEXT:    pmullw %xmm11, %xmm3
-; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    packuswb %xmm3, %xmm8
-; SSE41-NEXT:    pcmpeqb %xmm7, %xmm8
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pxor %xmm8, %xmm8
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
+; SSE41-NEXT:    pxor %xmm9, %xmm9
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm3[8],xmm9[9],xmm3[9],xmm9[10],xmm3[10],xmm9[11],xmm3[11],xmm9[12],xmm3[12],xmm9[13],xmm3[13],xmm9[14],xmm3[14],xmm9[15],xmm3[15]
+; SSE41-NEXT:    pmulhw %xmm8, %xmm9
+; SSE41-NEXT:    psrlw $8, %xmm9
+; SSE41-NEXT:    pxor %xmm8, %xmm8
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; SSE41-NEXT:    pxor %xmm10, %xmm10
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm3[0],xmm10[1],xmm3[1],xmm10[2],xmm3[2],xmm10[3],xmm3[3],xmm10[4],xmm3[4],xmm10[5],xmm3[5],xmm10[6],xmm3[6],xmm10[7],xmm3[7]
+; SSE41-NEXT:    pmulhw %xmm8, %xmm10
+; SSE41-NEXT:    psrlw $8, %xmm10
+; SSE41-NEXT:    packuswb %xmm9, %xmm10
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm7, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    pmullw %xmm11, %xmm12
-; SSE41-NEXT:    pmovsxbw %xmm6, %xmm7
-; SSE41-NEXT:    pand %xmm10, %xmm3
-; SSE41-NEXT:    pand %xmm10, %xmm12
-; SSE41-NEXT:    packuswb %xmm3, %xmm12
-; SSE41-NEXT:    pmovsxbw %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
-; SSE41-NEXT:    pmullw %xmm7, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pand %xmm9, %xmm3
+; SSE41-NEXT:    pmullw %xmm11, %xmm8
+; SSE41-NEXT:    pand %xmm9, %xmm8
+; SSE41-NEXT:    packuswb %xmm3, %xmm8
+; SSE41-NEXT:    pxor %xmm11, %xmm11
+; SSE41-NEXT:    pcmpgtb %xmm8, %xmm11
+; SSE41-NEXT:    pcmpeqb %xmm10, %xmm11
 ; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtb %xmm12, %xmm7
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE41-NEXT:    pmulhw %xmm7, %xmm3
 ; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    pmovsxbw %xmm6, %xmm6
-; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
-; SSE41-NEXT:    pmullw %xmm6, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    packuswb %xmm2, %xmm3
-; SSE41-NEXT:    pcmpeqb %xmm7, %xmm3
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pxor %xmm10, %xmm10
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3],xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; SSE41-NEXT:    pmulhw %xmm10, %xmm7
+; SSE41-NEXT:    psrlw $8, %xmm7
+; SSE41-NEXT:    packuswb %xmm3, %xmm7
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    movdqa %xmm1, %xmm7
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm6, %xmm7
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pmullw %xmm2, %xmm11
-; SSE41-NEXT:    pmovsxbw %xmm5, %xmm6
-; SSE41-NEXT:    pand %xmm10, %xmm7
-; SSE41-NEXT:    pand %xmm10, %xmm11
-; SSE41-NEXT:    packuswb %xmm7, %xmm11
-; SSE41-NEXT:    pmovsxbw %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm6, %xmm2
+; SSE41-NEXT:    pand %xmm9, %xmm2
+; SSE41-NEXT:    pmullw %xmm3, %xmm10
+; SSE41-NEXT:    pand %xmm9, %xmm10
+; SSE41-NEXT:    packuswb %xmm2, %xmm10
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pcmpgtb %xmm10, %xmm2
+; SSE41-NEXT:    pcmpeqb %xmm7, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
 ; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtb %xmm11, %xmm6
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pmovsxbw %xmm5, %xmm5
-; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
-; SSE41-NEXT:    pmullw %xmm5, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    packuswb %xmm1, %xmm2
-; SSE41-NEXT:    pcmpeqb %xmm6, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
-; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
+; SSE41-NEXT:    pmulhw %xmm3, %xmm6
+; SSE41-NEXT:    psrlw $8, %xmm6
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3],xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
+; SSE41-NEXT:    pmulhw %xmm3, %xmm7
+; SSE41-NEXT:    psrlw $8, %xmm7
+; SSE41-NEXT:    packuswb %xmm6, %xmm7
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm5, %xmm6
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE41-NEXT:    pmullw %xmm5, %xmm1
+; SSE41-NEXT:    pand %xmm9, %xmm1
+; SSE41-NEXT:    pmullw %xmm3, %xmm6
+; SSE41-NEXT:    pand %xmm9, %xmm6
+; SSE41-NEXT:    packuswb %xmm1, %xmm6
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pcmpgtb %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqb %xmm7, %xmm1
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; SSE41-NEXT:    pmulhw %xmm3, %xmm5
+; SSE41-NEXT:    psrlw $8, %xmm5
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
+; SSE41-NEXT:    pmulhw %xmm3, %xmm7
+; SSE41-NEXT:    psrlw $8, %xmm7
+; SSE41-NEXT:    packuswb %xmm5, %xmm7
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    pmullw %xmm1, %xmm5
-; SSE41-NEXT:    pand %xmm10, %xmm6
-; SSE41-NEXT:    pand %xmm10, %xmm5
-; SSE41-NEXT:    pmovsxbw %xmm4, %xmm7
-; SSE41-NEXT:    packuswb %xmm6, %xmm5
-; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
-; SSE41-NEXT:    pmullw %xmm7, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; SSE41-NEXT:    pmovsxbw %xmm4, %xmm4
-; SSE41-NEXT:    pmovsxbw %xmm0, %xmm0
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm4, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    packuswb %xmm0, %xmm1
+; SSE41-NEXT:    pmullw %xmm3, %xmm5
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    pand %xmm9, %xmm5
+; SSE41-NEXT:    packuswb %xmm0, %xmm5
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    pcmpgtb %xmm5, %xmm0
-; SSE41-NEXT:    pcmpeqb %xmm0, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm8
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm9, 48(%rsi)
-; SSE41-NEXT:    movdqa %xmm12, 32(%rsi)
-; SSE41-NEXT:    movdqa %xmm11, 16(%rsi)
+; SSE41-NEXT:    pcmpeqb %xmm7, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm3, %xmm11
+; SSE41-NEXT:    pxor %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm1
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm8, 48(%rsi)
+; SSE41-NEXT:    movdqa %xmm10, 32(%rsi)
+; SSE41-NEXT:    movdqa %xmm6, 16(%rsi)
 ; SSE41-NEXT:    movdqa %xmm5, (%rsi)
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, (%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,2,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,3,3,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 208(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 176(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 144(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 80(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, 48(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE41-NEXT:    pmovsxbd %xmm11, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, 192(%rdi)
+; SSE41-NEXT:    pmovsxbd %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, 128(%rdi)
+; SSE41-NEXT:    pmovsxbd %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, 64(%rdi)
+; SSE41-NEXT:    pmovsxbd %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, (%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[2,3,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm3
+; SSE41-NEXT:    psrad $31, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, 224(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm3
+; SSE41-NEXT:    psrad $31, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, 240(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[1,1,1,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm3
+; SSE41-NEXT:    psrad $31, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, 208(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm3
+; SSE41-NEXT:    psrad $31, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, 160(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm3
+; SSE41-NEXT:    psrad $31, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, 176(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, 144(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, 96(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, 112(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, 80(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, 32(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, 48(%rdi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2646,112 +2597,104 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-LABEL: smulo_v64i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movq %rdi, %rax
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
+; AVX1-NEXT:    vpmulhw %xmm6, %xmm8, %xmm6
+; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm8
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
+; AVX1-NEXT:    vpmulhw %xmm6, %xmm9, %xmm6
+; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
+; AVX1-NEXT:    vpackuswb %xmm8, %xmm6, %xmm8
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm6, %xmm9, %xmm6
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm8
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
-; AVX1-NEXT:    vpackuswb %xmm8, %xmm4, %xmm9
-; AVX1-NEXT:    vpxor %xmm11, %xmm11, %xmm11
-; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm11, %xmm8
-; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm4
-; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm7
+; AVX1-NEXT:    vpand %xmm6, %xmm10, %xmm6
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
-; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm5, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
-; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpackuswb %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm8, %xmm8
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
-; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm12
-; AVX1-NEXT:    vpcmpgtb %xmm12, %xmm11, %xmm4
-; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm6
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm7
-; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpackuswb %xmm6, %xmm4, %xmm9
+; AVX1-NEXT:    vpcmpgtb %xmm9, %xmm5, %xmm4
+; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm8, %xmm8
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; AVX1-NEXT:    vpmulhw %xmm6, %xmm7, %xmm6
 ; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpand %xmm6, %xmm10, %xmm6
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm6, %xmm1
-; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm1, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpand %xmm1, %xmm10, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm6, %xmm1, %xmm12
+; AVX1-NEXT:    vpcmpgtb %xmm12, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm4, %xmm11
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
+; AVX1-NEXT:    vpmulhw %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm1, %xmm7, %xmm1
-; AVX1-NEXT:    vpand %xmm1, %xmm10, %xmm1
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm5
-; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm7
-; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
-; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm11, %xmm7
-; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
-; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX1-NEXT:    vpmullw %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vpand %xmm3, %xmm10, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtb %xmm3, %xmm5, %xmm4
+; AVX1-NEXT:    vpcmpeqb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm6, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vpcmpeqb %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpand %xmm5, %xmm10, %xmm5
-; AVX1-NEXT:    vpand %xmm6, %xmm10, %xmm6
-; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm7
-; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm5
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm6
-; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm7, %xmm11, %xmm2
-; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm8, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm4, %xmm10, %xmm2
+; AVX1-NEXT:    vpand %xmm0, %xmm10, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm4
+; AVX1-NEXT:    vpcmpgtb %xmm4, %xmm5, %xmm0
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm6, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm7, %xmm7
+; AVX1-NEXT:    vpxor %xmm7, %xmm8, %xmm6
+; AVX1-NEXT:    vpxor %xmm7, %xmm11, %xmm5
+; AVX1-NEXT:    vpxor %xmm7, %xmm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm7, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovdqa %xmm9, 48(%rsi)
 ; AVX1-NEXT:    vmovdqa %xmm12, 32(%rsi)
-; AVX1-NEXT:    vmovdqa %xmm1, 16(%rsi)
-; AVX1-NEXT:    vmovdqa %xmm7, (%rsi)
+; AVX1-NEXT:    vmovdqa %xmm3, 16(%rsi)
+; AVX1-NEXT:    vmovdqa %xmm4, (%rsi)
 ; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 192(%rdi)
-; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm1
+; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 128(%rdi)
 ; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 64(%rdi)
@@ -2766,13 +2709,13 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm6[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 208(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 160(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[3,3,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[3,3,3,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 176(%rdi)
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqa %xmm1, 144(%rdi)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
@@ -2799,80 +2742,72 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; AVX2-LABEL: smulo_v64i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    movq %rdi, %rax
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15],ymm4[24],ymm3[24],ymm4[25],ymm3[25],ymm4[26],ymm3[26],ymm4[27],ymm3[27],ymm4[28],ymm3[28],ymm4[29],ymm3[29],ymm4[30],ymm3[30],ymm4[31],ymm3[31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11],ymm4[12],ymm1[12],ymm4[13],ymm1[13],ymm4[14],ymm1[14],ymm4[15],ymm1[15],ymm4[24],ymm1[24],ymm4[25],ymm1[25],ymm4[26],ymm1[26],ymm4[27],ymm1[27],ymm4[28],ymm1[28],ymm4[29],ymm1[29],ymm4[30],ymm1[30],ymm4[31],ymm1[31]
+; AVX2-NEXT:    vpmulhw %ymm5, %ymm6, %ymm5
+; AVX2-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[4],ymm1[4],ymm4[5],ymm1[5],ymm4[6],ymm1[6],ymm4[7],ymm1[7],ymm4[16],ymm1[16],ymm4[17],ymm1[17],ymm4[18],ymm1[18],ymm4[19],ymm1[19],ymm4[20],ymm1[20],ymm4[21],ymm1[21],ymm4[22],ymm1[22],ymm4[23],ymm1[23]
+; AVX2-NEXT:    vpmulhw %ymm6, %ymm7, %ymm6
+; AVX2-NEXT:    vpsrlw $8, %ymm6, %ymm6
+; AVX2-NEXT:    vpackuswb %ymm5, %ymm6, %ymm5
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX2-NEXT:    vpmullw %ymm6, %ymm7, %ymm6
-; AVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
-; AVX2-NEXT:    vpackuswb %ymm4, %ymm6, %ymm10
-; AVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT:    vpcmpgtb %ymm10, %ymm6, %ymm7
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpsraw $8, %ymm8, %ymm8
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpsraw $8, %ymm9, %ymm9
-; AVX2-NEXT:    vpmullw %ymm8, %ymm9, %ymm8
-; AVX2-NEXT:    vpsrlw $8, %ymm8, %ymm8
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm7 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm7, %ymm6, %ymm6
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpsraw $8, %ymm3, %ymm3
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
 ; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2-NEXT:    vpackuswb %ymm8, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpeqb %ymm7, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpeqd %ymm7, %ymm7, %ymm7
-; AVX2-NEXT:    vpxor %ymm7, %ymm1, %ymm1
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpmullw %ymm3, %ymm8, %ymm3
-; AVX2-NEXT:    vpand %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpmullw %ymm8, %ymm9, %ymm8
-; AVX2-NEXT:    vpand %ymm5, %ymm8, %ymm5
-; AVX2-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpcmpgtb %ymm3, %ymm6, %ymm5
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpsraw $8, %ymm6, %ymm6
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpsraw $8, %ymm8, %ymm8
-; AVX2-NEXT:    vpmullw %ymm6, %ymm8, %ymm6
+; AVX2-NEXT:    vpand %ymm7, %ymm1, %ymm1
+; AVX2-NEXT:    vpackuswb %ymm6, %ymm1, %ymm10
+; AVX2-NEXT:    vpcmpgtb %ymm10, %ymm4, %ymm3
+; AVX2-NEXT:    vpcmpeqb %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm5, %ymm5
+; AVX2-NEXT:    vpxor %ymm5, %ymm3, %ymm3
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11],ymm4[12],ymm2[12],ymm4[13],ymm2[13],ymm4[14],ymm2[14],ymm4[15],ymm2[15],ymm4[24],ymm2[24],ymm4[25],ymm2[25],ymm4[26],ymm2[26],ymm4[27],ymm2[27],ymm4[28],ymm2[28],ymm4[29],ymm2[29],ymm4[30],ymm2[30],ymm4[31],ymm2[31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]
+; AVX2-NEXT:    vpmulhw %ymm6, %ymm8, %ymm6
 ; AVX2-NEXT:    vpsrlw $8, %ymm6, %ymm6
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[16],ymm2[16],ymm4[17],ymm2[17],ymm4[18],ymm2[18],ymm4[19],ymm2[19],ymm4[20],ymm2[20],ymm4[21],ymm2[21],ymm4[22],ymm2[22],ymm4[23],ymm2[23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]
+; AVX2-NEXT:    vpmulhw %ymm8, %ymm9, %ymm8
+; AVX2-NEXT:    vpsrlw $8, %ymm8, %ymm8
+; AVX2-NEXT:    vpackuswb %ymm6, %ymm8, %ymm6
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpmullw %ymm8, %ymm9, %ymm8
+; AVX2-NEXT:    vpand %ymm7, %ymm8, %ymm8
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpsraw $8, %ymm2, %ymm2
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX2-NEXT:    vpackuswb %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm0
-; AVX2-NEXT:    vpxor %ymm7, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm8
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
+; AVX2-NEXT:    vpand %ymm7, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm8, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtb %ymm0, %ymm4, %ymm2
+; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm6, %ymm2
+; AVX2-NEXT:    vpxor %ymm5, %ymm2, %ymm2
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX2-NEXT:    vpmovsxbd %xmm4, %ymm8
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm6, %ymm6
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm3[2,3,2,3]
 ; AVX2-NEXT:    vpmovsxbd %xmm7, %ymm7
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
+; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[2,3,2,3]
+; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
 ; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm2
-; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
 ; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm5
-; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm1
+; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm3
 ; AVX2-NEXT:    vpmovsxbd %xmm4, %ymm4
 ; AVX2-NEXT:    vmovdqa %ymm10, 32(%rsi)
-; AVX2-NEXT:    vmovdqa %ymm3, (%rsi)
+; AVX2-NEXT:    vmovdqa %ymm0, (%rsi)
 ; AVX2-NEXT:    vmovdqa %ymm4, 192(%rdi)
-; AVX2-NEXT:    vmovdqa %ymm1, 128(%rdi)
+; AVX2-NEXT:    vmovdqa %ymm3, 128(%rdi)
 ; AVX2-NEXT:    vmovdqa %ymm5, 64(%rdi)
-; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
-; AVX2-NEXT:    vmovdqa %ymm2, 224(%rdi)
+; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
+; AVX2-NEXT:    vmovdqa %ymm1, 224(%rdi)
 ; AVX2-NEXT:    vmovdqa %ymm7, 160(%rdi)
 ; AVX2-NEXT:    vmovdqa %ymm6, 96(%rdi)
 ; AVX2-NEXT:    vmovdqa %ymm8, 32(%rdi)
@@ -2964,19 +2899,16 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ;
 ; AVX512BW-LABEL: smulo_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpsraw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpsraw $8, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpsraw $8, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm4 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpsraw $8, %zmm4, %zmm4
-; AVX512BW-NEXT:    vpmullw %zmm3, %zmm4, %zmm3
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm2[8],zmm1[8],zmm2[9],zmm1[9],zmm2[10],zmm1[10],zmm2[11],zmm1[11],zmm2[12],zmm1[12],zmm2[13],zmm1[13],zmm2[14],zmm1[14],zmm2[15],zmm1[15],zmm2[24],zmm1[24],zmm2[25],zmm1[25],zmm2[26],zmm1[26],zmm2[27],zmm1[27],zmm2[28],zmm1[28],zmm2[29],zmm1[29],zmm2[30],zmm1[30],zmm2[31],zmm1[31],zmm2[40],zmm1[40],zmm2[41],zmm1[41],zmm2[42],zmm1[42],zmm2[43],zmm1[43],zmm2[44],zmm1[44],zmm2[45],zmm1[45],zmm2[46],zmm1[46],zmm2[47],zmm1[47],zmm2[56],zmm1[56],zmm2[57],zmm1[57],zmm2[58],zmm1[58],zmm2[59],zmm1[59],zmm2[60],zmm1[60],zmm2[61],zmm1[61],zmm2[62],zmm1[62],zmm2[63],zmm1[63]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm2[8],zmm0[8],zmm2[9],zmm0[9],zmm2[10],zmm0[10],zmm2[11],zmm0[11],zmm2[12],zmm0[12],zmm2[13],zmm0[13],zmm2[14],zmm0[14],zmm2[15],zmm0[15],zmm2[24],zmm0[24],zmm2[25],zmm0[25],zmm2[26],zmm0[26],zmm2[27],zmm0[27],zmm2[28],zmm0[28],zmm2[29],zmm0[29],zmm2[30],zmm0[30],zmm2[31],zmm0[31],zmm2[40],zmm0[40],zmm2[41],zmm0[41],zmm2[42],zmm0[42],zmm2[43],zmm0[43],zmm2[44],zmm0[44],zmm2[45],zmm0[45],zmm2[46],zmm0[46],zmm2[47],zmm0[47],zmm2[56],zmm0[56],zmm2[57],zmm0[57],zmm2[58],zmm0[58],zmm2[59],zmm0[59],zmm2[60],zmm0[60],zmm2[61],zmm0[61],zmm2[62],zmm0[62],zmm2[63],zmm0[63]
+; AVX512BW-NEXT:    vpmulhw %zmm3, %zmm4, %zmm3
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm4 = zmm2[0],zmm1[0],zmm2[1],zmm1[1],zmm2[2],zmm1[2],zmm2[3],zmm1[3],zmm2[4],zmm1[4],zmm2[5],zmm1[5],zmm2[6],zmm1[6],zmm2[7],zmm1[7],zmm2[16],zmm1[16],zmm2[17],zmm1[17],zmm2[18],zmm1[18],zmm2[19],zmm1[19],zmm2[20],zmm1[20],zmm2[21],zmm1[21],zmm2[22],zmm1[22],zmm2[23],zmm1[23],zmm2[32],zmm1[32],zmm2[33],zmm1[33],zmm2[34],zmm1[34],zmm2[35],zmm1[35],zmm2[36],zmm1[36],zmm2[37],zmm1[37],zmm2[38],zmm1[38],zmm2[39],zmm1[39],zmm2[48],zmm1[48],zmm2[49],zmm1[49],zmm2[50],zmm1[50],zmm2[51],zmm1[51],zmm2[52],zmm1[52],zmm2[53],zmm1[53],zmm2[54],zmm1[54],zmm2[55],zmm1[55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmulhw %zmm4, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BW-NEXT:    vpmullw %zmm3, %zmm4, %zmm3
@@ -3018,10 +2950,8 @@ define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE2-NEXT:    pcmpeqw %xmm2, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -3037,10 +2967,8 @@ define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSSE3-NEXT:    pcmpeqw %xmm2, %xmm0
 ; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSSE3-NEXT:    pxor %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    psrad $16, %xmm0
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -3056,9 +2984,7 @@ define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
 ; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
diff --git a/llvm/test/CodeGen/X86/vec_ssubo.ll b/llvm/test/CodeGen/X86/vec_ssubo.ll
index 20fca27ab2d491ddfa942ec2e8bc5072a577458d..b591f3b4b94e833adde78def7f94b18c3ecb4e0c 100644
--- a/llvm/test/CodeGen/X86/vec_ssubo.ll
+++ b/llvm/test/CodeGen/X86/vec_ssubo.ll
@@ -544,12 +544,11 @@ define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT:    psrad $24, %xmm4
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm4
-; SSE2-NEXT:    psrad $31, %xmm4
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -573,12 +572,11 @@ define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSSE3-NEXT:    pcmpeqb %xmm0, %xmm2
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSSE3-NEXT:    psrad $24, %xmm4
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm1
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm4
-; SSSE3-NEXT:    psrad $31, %xmm4
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -602,9 +600,7 @@ define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pcmpeqb %xmm0, %xmm2
 ; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm4
-; SSE41-NEXT:    psrad $31, %xmm4
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm4
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
@@ -678,10 +674,8 @@ define <8 x i32> @ssubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE2-NEXT:    pcmpeqw %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    psrad $16, %xmm2
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -697,10 +691,8 @@ define <8 x i32> @ssubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSSE3-NEXT:    pcmpeqw %xmm0, %xmm2
 ; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm2
-; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT:    psrad $16, %xmm2
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -716,9 +708,7 @@ define <8 x i32> @ssubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE41-NEXT:    pcmpeqw %xmm0, %xmm2
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    pslld $31, %xmm2
-; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pmovsxwd %xmm1, %xmm2
 ; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
diff --git a/llvm/test/CodeGen/X86/vec_uaddo.ll b/llvm/test/CodeGen/X86/vec_uaddo.ll
index 7cf566f7b3a1d9e7e9f921540f92a4219de6d1cc..836cfaf2d39b67c5794bd2d83c813d2b4d6df889 100644
--- a/llvm/test/CodeGen/X86/vec_uaddo.ll
+++ b/llvm/test/CodeGen/X86/vec_uaddo.ll
@@ -626,12 +626,11 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm4
 ; SSE2-NEXT:    psrad $31, %xmm4
@@ -654,12 +653,11 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSSE3-NEXT:    pcmpeqb %xmm1, %xmm0
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm4
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm0
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm4
 ; SSSE3-NEXT:    psrad $31, %xmm4
@@ -682,9 +680,7 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pcmpeqb %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm4
@@ -757,10 +753,8 @@ define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE2-NEXT:    pxor %xmm3, %xmm2
 ; SSE2-NEXT:    pxor %xmm1, %xmm3
 ; SSE2-NEXT:    pcmpgtw %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm2
 ; SSE2-NEXT:    psrad $31, %xmm2
@@ -776,10 +770,8 @@ define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSSE3-NEXT:    pxor %xmm3, %xmm2
 ; SSSE3-NEXT:    pxor %xmm1, %xmm3
 ; SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    psrad $16, %xmm0
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm2
 ; SSSE3-NEXT:    psrad $31, %xmm2
@@ -794,9 +786,7 @@ define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE41-NEXT:    pcmpeqw %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
 ; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxwd %xmm2, %xmm0
 ; SSE41-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; SSE41-NEXT:    pslld $31, %xmm2
 ; SSE41-NEXT:    psrad $31, %xmm2
diff --git a/llvm/test/CodeGen/X86/vec_umulo.ll b/llvm/test/CodeGen/X86/vec_umulo.ll
index 5d29e20888a07612c40186339e2f96d25b35447e..9075b83275a9e84f69be320cef003138a8371935 100644
--- a/llvm/test/CodeGen/X86/vec_umulo.ll
+++ b/llvm/test/CodeGen/X86/vec_umulo.ll
@@ -1104,12 +1104,11 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -1155,12 +1154,11 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSSE3-NEXT:    pcmpeqb %xmm2, %xmm0
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm1
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -1199,9 +1197,7 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pcmpeqb %xmm2, %xmm5
 ; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE41-NEXT:    pxor %xmm5, %xmm3
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
@@ -1335,11 +1331,11 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE2-NEXT:    pand %xmm8, %xmm7
 ; SSE2-NEXT:    movdqa %xmm3, %xmm6
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm1, %xmm10
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm6, %xmm10
-; SSE2-NEXT:    pand %xmm8, %xmm10
-; SSE2-NEXT:    packuswb %xmm7, %xmm10
+; SSE2-NEXT:    movdqa %xmm1, %xmm11
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm6, %xmm11
+; SSE2-NEXT:    pand %xmm8, %xmm11
+; SSE2-NEXT:    packuswb %xmm7, %xmm11
 ; SSE2-NEXT:    pxor %xmm6, %xmm6
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
@@ -1369,49 +1365,47 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE2-NEXT:    pcmpeqb %xmm6, %xmm0
 ; SSE2-NEXT:    pxor %xmm3, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm3
-; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm6
-; SSE2-NEXT:    psrad $31, %xmm6
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3],xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm0
 ; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm4, %xmm7
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm7
-; SSE2-NEXT:    psrad $31, %xmm7
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
 ; SSE2-NEXT:    pslld $31, %xmm4
 ; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm6
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm6
+; SSE2-NEXT:    psrad $31, %xmm6
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm1[0],xmm10[1],xmm1[1],xmm10[2],xmm1[2],xmm10[3],xmm1[3],xmm10[4],xmm1[4],xmm10[5],xmm1[5],xmm10[6],xmm1[6],xmm10[7],xmm1[7]
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm1, %xmm5
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm5
-; SSE2-NEXT:    psrad $31, %xmm5
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pslld $31, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    movdqa %xmm10, 16(%rsi)
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
+; SSE2-NEXT:    psrad $24, %xmm7
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3]
+; SSE2-NEXT:    psrad $24, %xmm5
+; SSE2-NEXT:    movdqa %xmm11, 16(%rsi)
 ; SSE2-NEXT:    movdqa %xmm9, (%rsi)
+; SSE2-NEXT:    movdqa %xmm5, 64(%rdi)
+; SSE2-NEXT:    movdqa %xmm7, (%rdi)
 ; SSE2-NEXT:    movdqa %xmm1, 112(%rdi)
-; SSE2-NEXT:    movdqa %xmm5, 96(%rdi)
-; SSE2-NEXT:    movdqa %xmm4, 80(%rdi)
-; SSE2-NEXT:    movdqa %xmm7, 64(%rdi)
-; SSE2-NEXT:    movdqa %xmm0, 48(%rdi)
-; SSE2-NEXT:    movdqa %xmm6, 32(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, 16(%rdi)
-; SSE2-NEXT:    movdqa %xmm3, (%rdi)
+; SSE2-NEXT:    movdqa %xmm3, 96(%rdi)
+; SSE2-NEXT:    movdqa %xmm6, 80(%rdi)
+; SSE2-NEXT:    movdqa %xmm2, 48(%rdi)
+; SSE2-NEXT:    movdqa %xmm4, 32(%rdi)
+; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: umulo_v32i8:
@@ -1439,11 +1433,11 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSSE3-NEXT:    pand %xmm8, %xmm7
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm6
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm10
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pmullw %xmm6, %xmm10
-; SSSE3-NEXT:    pand %xmm8, %xmm10
-; SSSE3-NEXT:    packuswb %xmm7, %xmm10
+; SSSE3-NEXT:    movdqa %xmm1, %xmm11
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pmullw %xmm6, %xmm11
+; SSSE3-NEXT:    pand %xmm8, %xmm11
+; SSSE3-NEXT:    packuswb %xmm7, %xmm11
 ; SSSE3-NEXT:    pxor %xmm6, %xmm6
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm7
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
@@ -1473,49 +1467,47 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSSE3-NEXT:    pcmpeqb %xmm6, %xmm0
 ; SSSE3-NEXT:    pxor %xmm3, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm3
-; SSSE3-NEXT:    psrad $31, %xmm3
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    pslld $31, %xmm2
-; SSSE3-NEXT:    psrad $31, %xmm2
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm6
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm6
-; SSSE3-NEXT:    psrad $31, %xmm6
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3],xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm0
 ; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm7
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm7
-; SSSE3-NEXT:    psrad $31, %xmm7
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
 ; SSSE3-NEXT:    pslld $31, %xmm4
 ; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm6
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    pslld $31, %xmm6
+; SSSE3-NEXT:    psrad $31, %xmm6
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm1[0],xmm10[1],xmm1[1],xmm10[2],xmm1[2],xmm10[3],xmm1[3],xmm10[4],xmm1[4],xmm10[5],xmm1[5],xmm10[6],xmm1[6],xmm10[7],xmm1[7]
 ; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm5
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm5
-; SSSE3-NEXT:    psrad $31, %xmm5
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm3
+; SSSE3-NEXT:    psrad $31, %xmm3
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    movdqa %xmm10, 16(%rsi)
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
+; SSSE3-NEXT:    psrad $24, %xmm7
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3]
+; SSSE3-NEXT:    psrad $24, %xmm5
+; SSSE3-NEXT:    movdqa %xmm11, 16(%rsi)
 ; SSSE3-NEXT:    movdqa %xmm9, (%rsi)
+; SSSE3-NEXT:    movdqa %xmm5, 64(%rdi)
+; SSSE3-NEXT:    movdqa %xmm7, (%rdi)
 ; SSSE3-NEXT:    movdqa %xmm1, 112(%rdi)
-; SSSE3-NEXT:    movdqa %xmm5, 96(%rdi)
-; SSSE3-NEXT:    movdqa %xmm4, 80(%rdi)
-; SSSE3-NEXT:    movdqa %xmm7, 64(%rdi)
-; SSSE3-NEXT:    movdqa %xmm0, 48(%rdi)
-; SSSE3-NEXT:    movdqa %xmm6, 32(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, 16(%rdi)
-; SSSE3-NEXT:    movdqa %xmm3, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm3, 96(%rdi)
+; SSSE3-NEXT:    movdqa %xmm6, 80(%rdi)
+; SSSE3-NEXT:    movdqa %xmm2, 48(%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, 32(%rdi)
+; SSSE3-NEXT:    movdqa %xmm0, 16(%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: umulo_v32i8:
@@ -1526,42 +1518,42 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    movdqa %xmm0, %xmm7
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm4, %xmm7
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT:    pand %xmm9, %xmm7
+; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT:    pand %xmm10, %xmm7
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    pmullw %xmm5, %xmm6
 ; SSE41-NEXT:    movdqa %xmm6, %xmm8
-; SSE41-NEXT:    pand %xmm9, %xmm8
+; SSE41-NEXT:    pand %xmm10, %xmm8
 ; SSE41-NEXT:    packuswb %xmm7, %xmm8
 ; SSE41-NEXT:    movdqa %xmm3, %xmm7
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm7, %xmm5
-; SSE41-NEXT:    pand %xmm9, %xmm5
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; SSE41-NEXT:    pand %xmm10, %xmm5
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; SSE41-NEXT:    pmullw %xmm4, %xmm7
-; SSE41-NEXT:    pand %xmm7, %xmm9
-; SSE41-NEXT:    packuswb %xmm5, %xmm9
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; SSE41-NEXT:    pmullw %xmm9, %xmm7
+; SSE41-NEXT:    pand %xmm7, %xmm10
+; SSE41-NEXT:    packuswb %xmm5, %xmm10
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]
 ; SSE41-NEXT:    pmullw %xmm3, %xmm1
 ; SSE41-NEXT:    psrlw $8, %xmm1
 ; SSE41-NEXT:    psrlw $8, %xmm7
 ; SSE41-NEXT:    packuswb %xmm1, %xmm7
-; SSE41-NEXT:    pcmpeqb %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqb %xmm5, %xmm7
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    pxor %xmm1, %xmm7
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm5[8],xmm0[9],xmm5[9],xmm0[10],xmm5[10],xmm0[11],xmm5[11],xmm0[12],xmm5[12],xmm0[13],xmm5[13],xmm0[14],xmm5[14],xmm0[15],xmm5[15]
 ; SSE41-NEXT:    pmullw %xmm2, %xmm0
 ; SSE41-NEXT:    psrlw $8, %xmm0
 ; SSE41-NEXT:    psrlw $8, %xmm6
 ; SSE41-NEXT:    packuswb %xmm0, %xmm6
-; SSE41-NEXT:    pcmpeqb %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqb %xmm5, %xmm6
 ; SSE41-NEXT:    pxor %xmm1, %xmm6
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
@@ -1579,26 +1571,22 @@ define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, <32 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm3
 ; SSE41-NEXT:    psrad $31, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[2,3,2,3]
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm4
-; SSE41-NEXT:    psrad $31, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[3,3,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm5
 ; SSE41-NEXT:    psrad $31, %xmm5
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm6
-; SSE41-NEXT:    psrad $31, %xmm6
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm7
-; SSE41-NEXT:    psrad $31, %xmm7
-; SSE41-NEXT:    movdqa %xmm9, 16(%rsi)
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[3,3,3,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm4
+; SSE41-NEXT:    psrad $31, %xmm4
+; SSE41-NEXT:    pmovsxbd %xmm6, %xmm6
+; SSE41-NEXT:    pmovsxbd %xmm7, %xmm7
+; SSE41-NEXT:    movdqa %xmm10, 16(%rsi)
 ; SSE41-NEXT:    movdqa %xmm8, (%rsi)
 ; SSE41-NEXT:    movdqa %xmm7, 64(%rdi)
 ; SSE41-NEXT:    movdqa %xmm6, (%rdi)
-; SSE41-NEXT:    movdqa %xmm5, 112(%rdi)
-; SSE41-NEXT:    movdqa %xmm4, 96(%rdi)
+; SSE41-NEXT:    movdqa %xmm4, 112(%rdi)
+; SSE41-NEXT:    movdqa %xmm5, 96(%rdi)
 ; SSE41-NEXT:    movdqa %xmm3, 80(%rdi)
 ; SSE41-NEXT:    movdqa %xmm2, 48(%rdi)
 ; SSE41-NEXT:    movdqa %xmm1, 32(%rdi)
@@ -1872,86 +1860,82 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE2-NEXT:    movdqa %xmm10, 32(%rsi)
 ; SSE2-NEXT:    movdqa %xmm9, 16(%rsi)
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    movdqa %xmm8, (%rsi)
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm3
-; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, 224(%rdi)
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, 192(%rdi)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, 128(%rdi)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, 64(%rdi)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, (%rdi)
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pslld $31, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, 224(%rdi)
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm5
 ; SSE2-NEXT:    psrad $31, %xmm5
 ; SSE2-NEXT:    movdqa %xmm5, 240(%rdi)
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm4
-; SSE2-NEXT:    psrad $31, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, 192(%rdi)
 ; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm3
 ; SSE2-NEXT:    psrad $31, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, 208(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, 160(%rdi)
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pslld $31, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, 160(%rdi)
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm3
 ; SSE2-NEXT:    psrad $31, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, 176(%rdi)
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm4
-; SSE2-NEXT:    psrad $31, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, 128(%rdi)
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm2
 ; SSE2-NEXT:    psrad $31, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, 144(%rdi)
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm1
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, 96(%rdi)
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pslld $31, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, 96(%rdi)
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm2
 ; SSE2-NEXT:    psrad $31, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, 112(%rdi)
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm3
-; SSE2-NEXT:    psrad $31, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, 64(%rdi)
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, 80(%rdi)
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, 32(%rdi)
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    psrad $31, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, (%rdi)
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm0
 ; SSE2-NEXT:    psrad $31, %xmm0
@@ -2072,86 +2056,82 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSSE3-NEXT:    movdqa %xmm10, 32(%rsi)
 ; SSSE3-NEXT:    movdqa %xmm9, 16(%rsi)
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm4
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSSE3-NEXT:    movdqa %xmm8, (%rsi)
-; SSSE3-NEXT:    movdqa %xmm3, %xmm5
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm3
-; SSSE3-NEXT:    psrad $31, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, 224(%rdi)
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, 192(%rdi)
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, 128(%rdi)
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, 64(%rdi)
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, 224(%rdi)
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm5
 ; SSSE3-NEXT:    psrad $31, %xmm5
 ; SSSE3-NEXT:    movdqa %xmm5, 240(%rdi)
-; SSSE3-NEXT:    movdqa %xmm4, %xmm3
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm4
-; SSSE3-NEXT:    psrad $31, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, 192(%rdi)
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm3
 ; SSSE3-NEXT:    psrad $31, %xmm3
 ; SSSE3-NEXT:    movdqa %xmm3, 208(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, %xmm3
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm2
-; SSSE3-NEXT:    psrad $31, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, 160(%rdi)
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, 160(%rdi)
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm3
 ; SSSE3-NEXT:    psrad $31, %xmm3
 ; SSSE3-NEXT:    movdqa %xmm3, 176(%rdi)
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm4
-; SSSE3-NEXT:    psrad $31, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, 128(%rdi)
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm2
 ; SSSE3-NEXT:    psrad $31, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm2, 144(%rdi)
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm1
-; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, 96(%rdi)
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm3
+; SSSE3-NEXT:    psrad $31, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, 96(%rdi)
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm2
 ; SSSE3-NEXT:    psrad $31, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm2, 112(%rdi)
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm3
-; SSSE3-NEXT:    psrad $31, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, 64(%rdi)
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, 80(%rdi)
-; SSSE3-NEXT:    movdqa %xmm0, %xmm1
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
-; SSSE3-NEXT:    movdqa %xmm0, 32(%rdi)
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, 48(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm2
-; SSSE3-NEXT:    psrad $31, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm0
 ; SSSE3-NEXT:    psrad $31, %xmm0
@@ -2200,27 +2180,27 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm14 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    pmullw %xmm14, %xmm12
-; SSE41-NEXT:    movdqa %xmm12, %xmm15
-; SSE41-NEXT:    pand %xmm10, %xmm15
-; SSE41-NEXT:    packuswb %xmm0, %xmm15
+; SSE41-NEXT:    movdqa %xmm12, %xmm14
+; SSE41-NEXT:    pand %xmm10, %xmm14
+; SSE41-NEXT:    packuswb %xmm0, %xmm14
 ; SSE41-NEXT:    movdqa %xmm7, %xmm0
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    movdqa %xmm4, %xmm3
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm3
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm14 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
-; SSE41-NEXT:    pmullw %xmm0, %xmm14
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm15 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; SSE41-NEXT:    pmullw %xmm0, %xmm15
 ; SSE41-NEXT:    pand %xmm10, %xmm3
-; SSE41-NEXT:    pand %xmm14, %xmm10
+; SSE41-NEXT:    pand %xmm15, %xmm10
 ; SSE41-NEXT:    packuswb %xmm3, %xmm10
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm0[8],xmm7[9],xmm0[9],xmm7[10],xmm0[10],xmm7[11],xmm0[11],xmm7[12],xmm0[12],xmm7[13],xmm0[13],xmm7[14],xmm0[14],xmm7[15],xmm0[15]
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
 ; SSE41-NEXT:    pmullw %xmm7, %xmm4
 ; SSE41-NEXT:    psrlw $8, %xmm4
-; SSE41-NEXT:    psrlw $8, %xmm14
-; SSE41-NEXT:    packuswb %xmm4, %xmm14
+; SSE41-NEXT:    psrlw $8, %xmm15
+; SSE41-NEXT:    packuswb %xmm4, %xmm15
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
 ; SSE41-NEXT:    pmullw %xmm6, %xmm2
@@ -2239,48 +2219,40 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, <64 x i8>* %p2) nou
 ; SSE41-NEXT:    psrlw $8, %xmm5
 ; SSE41-NEXT:    psrlw $8, %xmm8
 ; SSE41-NEXT:    packuswb %xmm5, %xmm8
-; SSE41-NEXT:    pcmpeqb %xmm0, %xmm14
+; SSE41-NEXT:    pcmpeqb %xmm0, %xmm15
 ; SSE41-NEXT:    pcmpeqb %xmm0, %xmm12
 ; SSE41-NEXT:    pcmpeqb %xmm0, %xmm9
 ; SSE41-NEXT:    pcmpeqb %xmm0, %xmm8
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm14
+; SSE41-NEXT:    pxor %xmm0, %xmm15
 ; SSE41-NEXT:    pxor %xmm0, %xmm12
 ; SSE41-NEXT:    pxor %xmm0, %xmm9
 ; SSE41-NEXT:    pxor %xmm0, %xmm8
 ; SSE41-NEXT:    movdqa %xmm10, 48(%rsi)
-; SSE41-NEXT:    movdqa %xmm15, 32(%rsi)
+; SSE41-NEXT:    movdqa %xmm14, 32(%rsi)
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; SSE41-NEXT:    movaps %xmm0, 16(%rsi)
 ; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; SSE41-NEXT:    movaps %xmm0, (%rsi)
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm14[0],zero,zero,zero,xmm14[1],zero,zero,zero,xmm14[2],zero,zero,zero,xmm14[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm15, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero,xmm12[2],zero,zero,zero,xmm12[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm12, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm9, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm8, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, (%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[2,3,2,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm15[2,3,2,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[3,3,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm15[3,3,3,3]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[1,1,1,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm15[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
@@ -2655,10 +2627,8 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE2-NEXT:    pcmpeqw %xmm0, %xmm3
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -2674,10 +2644,8 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSSE3-NEXT:    pcmpeqw %xmm0, %xmm3
 ; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSSE3-NEXT:    pxor %xmm3, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    psrad $16, %xmm0
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -2693,9 +2661,7 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE41-NEXT:    pcmpeqw %xmm0, %xmm3
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    pxor %xmm3, %xmm1
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
 ; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
diff --git a/llvm/test/CodeGen/X86/vec_usubo.ll b/llvm/test/CodeGen/X86/vec_usubo.ll
index dde6832d648221262068cfa937c02af6b03774aa..4e2906c3d5e3a626ea13a86da0f31249dedf546b 100644
--- a/llvm/test/CodeGen/X86/vec_usubo.ll
+++ b/llvm/test/CodeGen/X86/vec_usubo.ll
@@ -670,12 +670,11 @@ define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE2-NEXT:    pcmpeqb %xmm4, %xmm0
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
 ; SSE2-NEXT:    movdqa %xmm3, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    psrad $31, %xmm0
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
@@ -698,12 +697,11 @@ define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSSE3-NEXT:    pcmpeqb %xmm4, %xmm0
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm3
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm1
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm0
-; SSSE3-NEXT:    psrad $31, %xmm0
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
@@ -726,9 +724,7 @@ define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nou
 ; SSE41-NEXT:    pcmpeqb %xmm4, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxbd %xmm3, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
 ; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
 ; SSE41-NEXT:    pslld $31, %xmm1
@@ -800,10 +796,8 @@ define <8 x i32> @usubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE2-NEXT:    psubw %xmm1, %xmm0
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtw %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pslld $31, %xmm1
-; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pslld $31, %xmm2
 ; SSE2-NEXT:    psrad $31, %xmm2
@@ -820,10 +814,8 @@ define <8 x i32> @usubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSSE3-NEXT:    psubw %xmm1, %xmm0
 ; SSSE3-NEXT:    pxor %xmm0, %xmm2
 ; SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    pslld $31, %xmm1
-; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    psrad $16, %xmm1
 ; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    pslld $31, %xmm2
 ; SSSE3-NEXT:    psrad $31, %xmm2
@@ -840,9 +832,7 @@ define <8 x i32> @usubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 ; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pmovsxwd %xmm1, %xmm0
 ; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE41-NEXT:    pslld $31, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index d74db60f78658077e6d3423003f5cd456a620a3a..c73edfbc525c29e6545293500e3122c82f40a393 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -406,7 +406,7 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; X86-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm1[0,3]
 ; X86-SSE2-NEXT:    pand %xmm4, %xmm2
 ; X86-SSE2-NEXT:    pslld $23, %xmm2
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
@@ -703,7 +703,7 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm3, %xmm5
 ; X86-SSE2-NEXT:    psrlw $1, %xmm3
 ; X86-SSE2-NEXT:    pand %xmm4, %xmm3
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
@@ -1017,7 +1017,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm7
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    por %xmm7, %xmm1
 ; X86-SSE2-NEXT:    paddb %xmm6, %xmm6
 ; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
@@ -1026,7 +1026,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm7
 ; X86-SSE2-NEXT:    psrlw $2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    por %xmm7, %xmm1
 ; X86-SSE2-NEXT:    paddb %xmm6, %xmm6
 ; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
@@ -1045,7 +1045,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm4, %xmm0
 ; X86-SSE2-NEXT:    paddb %xmm2, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -1054,7 +1054,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE2-NEXT:    psllw $2, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm4, %xmm0
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE2-NEXT:    paddb %xmm2, %xmm2
@@ -1766,7 +1766,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; X86-SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
 ; X86-SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; X86-SSE2-NEXT:    psrlw $1, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psrlw %xmm2, %xmm1
 ; X86-SSE2-NEXT:    psrlw %xmm2, %xmm5
 ; X86-SSE2-NEXT:    psrlw $8, %xmm5
@@ -2407,9 +2407,9 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ;
 ; X86-SSE2-LABEL: constant_funnnel_v8i16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm1
-; X86-SSE2-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pmulhuw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
 ; X86-SSE2-NEXT:    retl
   %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
@@ -2603,22 +2603,22 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm2
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm0
 ; X86-SSE2-NEXT:    packuswb %xmm2, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm3
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-SSE2-NEXT:    psrlw $8, %xmm3
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psrlw $8, %xmm1
 ; X86-SSE2-NEXT:    packuswb %xmm3, %xmm1
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
@@ -2932,9 +2932,9 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; X86-SSE2-LABEL: splatconstant_funnnel_v16i8:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
 ; X86-SSE2-NEXT:    retl
   %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
index 762672bc446c5d73091085b00b7491cb23d3bfb2..2beaaea11de996f550533e22778fd32dcb78a53b 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -282,9 +282,9 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind {
 ;
 ; X86-SSE2-LABEL: var_funnnel_v4i32:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
@@ -458,7 +458,7 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ;
 ; X86-SSE2-LABEL: var_funnnel_v8i16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; X86-SSE2-NEXT:    pslld $23, %xmm2
@@ -706,20 +706,20 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; X86-SSE2-NEXT:    psrlw $4, %xmm4
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm5
 ; X86-SSE2-NEXT:    psllw $4, %xmm5
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm5
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm5
 ; X86-SSE2-NEXT:    por %xmm4, %xmm5
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm5
 ; X86-SSE2-NEXT:    pandn %xmm2, %xmm3
 ; X86-SSE2-NEXT:    por %xmm5, %xmm3
 ; X86-SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; X86-SSE2-NEXT:    psrlw $6, %xmm2
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; X86-SSE2-NEXT:    psllw $2, %xmm4
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE2-NEXT:    por %xmm2, %xmm4
 ; X86-SSE2-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
@@ -731,7 +731,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ; X86-SSE2-NEXT:    paddb %xmm2, %xmm3
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; X86-SSE2-NEXT:    psrlw $7, %xmm4
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE2-NEXT:    por %xmm3, %xmm4
 ; X86-SSE2-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
@@ -1103,7 +1103,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind
 ;
 ; X86-SSE2-LABEL: splatvar_funnnel_v8i16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,0,0]
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1333,7 +1333,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; X86-SSE2-LABEL: splatvar_funnnel_v16i8:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; X86-SSE2-NEXT:    psubb %xmm1, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm3
@@ -1848,20 +1848,20 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    psrlw $8, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm3
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-SSE2-NEXT:    psrlw $8, %xmm3
 ; X86-SSE2-NEXT:    packuswb %xmm2, %xmm3
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    packuswb %xmm1, %xmm0
 ; X86-SSE2-NEXT:    por %xmm3, %xmm0
@@ -2168,9 +2168,9 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
 ; X86-SSE2-NEXT:    retl
   %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %x, <16 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll
index ced1eab5b343195cfe862aeebd7d6a1607c7d15d..5bb7cacf6a9ec7b68e333dc45709c9603e7931e9 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll
@@ -135,9 +135,9 @@ define <2 x i32> @var_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind {
 ;
 ; X86-SSE2-LABEL: var_funnnel_v2i32:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
@@ -290,9 +290,9 @@ define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind
 ; X86-SSE2-LABEL: splatvar_funnnel_v2i32:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index 6e7d036d926724380906a9345e0d4dc5ee769739..f36a871d5e41d1a7c89c039dc4dc42dcee50f5bf 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -407,7 +407,7 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; X86-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm1[0,3]
 ; X86-SSE2-NEXT:    pandn %xmm4, %xmm2
 ; X86-SSE2-NEXT:    pslld $23, %xmm2
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pslld $1, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
@@ -702,7 +702,7 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm4
 ; X86-SSE2-NEXT:    psrlw $1, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm1
-; X86-SSE2-NEXT:    pandn {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pandn {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; X86-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
 ; X86-SSE2-NEXT:    pslld $23, %xmm3
@@ -1005,7 +1005,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm7
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm6, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm7, %xmm0
 ; X86-SSE2-NEXT:    paddb %xmm4, %xmm4
 ; X86-SSE2-NEXT:    pxor %xmm6, %xmm6
@@ -1014,7 +1014,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm0, %xmm7
 ; X86-SSE2-NEXT:    psllw $2, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm6, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm7, %xmm0
 ; X86-SSE2-NEXT:    paddb %xmm4, %xmm4
 ; X86-SSE2-NEXT:    pxor %xmm6, %xmm6
@@ -1031,7 +1031,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm6
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm5, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    por %xmm6, %xmm1
 ; X86-SSE2-NEXT:    paddb %xmm2, %xmm2
 ; X86-SSE2-NEXT:    pxor %xmm5, %xmm5
@@ -1040,7 +1040,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm6
 ; X86-SSE2-NEXT:    psrlw $2, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm5, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    por %xmm6, %xmm1
 ; X86-SSE2-NEXT:    paddb %xmm2, %xmm2
 ; X86-SSE2-NEXT:    pcmpgtb %xmm2, %xmm3
@@ -1048,7 +1048,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm2
 ; X86-SSE2-NEXT:    psrlw $1, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    por %xmm2, %xmm1
 ; X86-SSE2-NEXT:    por %xmm4, %xmm1
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
@@ -2133,10 +2133,10 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,65535,65535,65535,65535,65535,65535]
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; X86-SSE2-NEXT:    pandn %xmm1, %xmm3
-; X86-SSE2-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pmulhuw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X86-SSE2-NEXT:    psllw $1, %xmm0
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm3, %xmm0
 ; X86-SSE2-NEXT:    por %xmm2, %xmm0
 ; X86-SSE2-NEXT:    retl
@@ -2321,20 +2321,20 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm3
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-SSE2-NEXT:    psrlw $8, %xmm3
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psrlw $8, %xmm1
 ; X86-SSE2-NEXT:    packuswb %xmm3, %xmm1
 ; X86-SSE2-NEXT:    paddb %xmm0, %xmm0
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm2
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm0
 ; X86-SSE2-NEXT:    packuswb %xmm2, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
@@ -2648,9 +2648,9 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; X86-SSE2-LABEL: splatconstant_funnnel_v16i8:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
 ; X86-SSE2-NEXT:    retl
   %res = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index 11753627a571de089eb557e76bde54d25111502c..6adf34e0dcf5e047a1eb9d7949aec2df1cbdf964 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -296,9 +296,9 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    psubd %xmm1, %xmm2
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    pslld $23, %xmm2
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
@@ -492,7 +492,7 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    psubw %xmm1, %xmm2
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; X86-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
@@ -746,20 +746,20 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ; X86-SSE2-NEXT:    pcmpgtb %xmm3, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; X86-SSE2-NEXT:    psrlw $4, %xmm4
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm5
 ; X86-SSE2-NEXT:    psllw $4, %xmm5
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm5
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm5
 ; X86-SSE2-NEXT:    por %xmm4, %xmm5
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm5
 ; X86-SSE2-NEXT:    pandn %xmm2, %xmm1
 ; X86-SSE2-NEXT:    por %xmm5, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    psrlw $6, %xmm2
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm4
 ; X86-SSE2-NEXT:    psllw $2, %xmm4
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE2-NEXT:    por %xmm2, %xmm4
 ; X86-SSE2-NEXT:    paddb %xmm3, %xmm3
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
@@ -771,7 +771,7 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ; X86-SSE2-NEXT:    paddb %xmm2, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; X86-SSE2-NEXT:    psrlw $7, %xmm4
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE2-NEXT:    por %xmm1, %xmm4
 ; X86-SSE2-NEXT:    paddb %xmm3, %xmm3
 ; X86-SSE2-NEXT:    pcmpgtb %xmm3, %xmm0
@@ -1179,7 +1179,7 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    psubw %xmm1, %xmm2
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,0,0]
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1420,7 +1420,7 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    psubb %xmm1, %xmm2
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; X86-SSE2-NEXT:    psubb %xmm2, %xmm3
 ; X86-SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
@@ -1934,20 +1934,20 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    psrlw $8, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm3
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-SSE2-NEXT:    psrlw $8, %xmm3
 ; X86-SSE2-NEXT:    packuswb %xmm2, %xmm3
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    packuswb %xmm1, %xmm0
 ; X86-SSE2-NEXT:    por %xmm3, %xmm0
@@ -2254,9 +2254,9 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
 ; X86-SSE2-NEXT:    retl
   %res = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %x, <16 x i8> %x, <16 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll
index a89efc635070380b4bdaba06fae4ad0c128ec432..c25f277dfbac06160be0a6e7b9af713c4d446a27 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll
@@ -147,9 +147,9 @@ define <2 x i32> @var_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    psubd %xmm1, %xmm2
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    pslld $23, %xmm2
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
@@ -316,9 +316,9 @@ define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
 ; X86-SSE2-NEXT:    psubd %xmm1, %xmm2
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    pslld $23, %xmm2
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-gep.ll b/llvm/test/CodeGen/X86/vector-gep.ll
index f8c5a4fc253f4e305e8747f9edb1ed8e5f44919b..f0673bdd1aed2fcb211f4e79706462d7ade2bc6b 100644
--- a/llvm/test/CodeGen/X86/vector-gep.ll
+++ b/llvm/test/CodeGen/X86/vector-gep.ll
@@ -6,7 +6,7 @@ define <4 x i32*> @AGEP0(i32* %ptr) nounwind {
 ; CHECK-LABEL: AGEP0:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vbroadcastss {{[0-9]+}}(%esp), %xmm0
-; CHECK-NEXT:    vpaddd {{\.LCPI.*}}, %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; CHECK-NEXT:    retl
   %vecinit.i = insertelement <4 x i32*> undef, i32* %ptr, i32 0
   %vecinit2.i = insertelement <4 x i32*> %vecinit.i, i32* %ptr, i32 1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
index 267ba0a0dd87245e6102657bb99a1cd4ff8a20c8..d82003dfcd2ec64fc5f8539e09f8b90b48e89eea 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-128.ll
@@ -168,65 +168,41 @@ define <8 x i16> @test_div7_8i16(<8 x i16> %a) nounwind {
 }
 
 define <16 x i8> @test_div7_16i8(<16 x i8> %a) nounwind {
-; SSE2-LABEL: test_div7_16i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65427,65427,65427,65427,65427,65427,65427,65427]
-; SSE2-NEXT:    pmullw %xmm3, %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pmullw %xmm3, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    packuswb %xmm2, %xmm1
-; SSE2-NEXT:    paddb %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrlw $2, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    psrlw $7, %xmm1
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    paddb %xmm0, %xmm1
-; SSE2-NEXT:    psubb %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test_div7_16i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [65427,65427,65427,65427,65427,65427,65427,65427]
-; SSE41-NEXT:    pmullw %xmm2, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE41-NEXT:    pmovsxbw %xmm3, %xmm3
-; SSE41-NEXT:    pmullw %xmm2, %xmm3
-; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    packuswb %xmm3, %xmm1
-; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $2, %xmm0
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    psrlw $7, %xmm1
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    psubb %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_div7_16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    pxor %xmm0, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [37632,37632,37632,37632,37632,37632,37632,37632]
+; SSE-NEXT:    pmulhw %xmm3, %xmm2
+; SSE-NEXT:    psrlw $8, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE-NEXT:    pmulhw %xmm3, %xmm0
+; SSE-NEXT:    psrlw $8, %xmm0
+; SSE-NEXT:    packuswb %xmm2, %xmm0
+; SSE-NEXT:    paddb %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrlw $2, %xmm1
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    psrlw $7, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    paddb %xmm1, %xmm0
+; SSE-NEXT:    psubb %xmm2, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: test_div7_16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm3
-; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX1-NEXT:    vpmulhw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
@@ -284,72 +260,44 @@ define <16 x i8> @test_div7_16i8(<16 x i8> %a) nounwind {
 ;
 
 define <16 x i8> @test_divconstant_16i8(<16 x i8> %a) nounwind {
-; SSE2-LABEL: test_divconstant_16i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    packuswb %xmm1, %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    paddb %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    packuswb %xmm1, %xmm2
-; SSE2-NEXT:    psrlw $7, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    paddb %xmm2, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test_divconstant_16i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE41-NEXT:    pmovsxbw %xmm2, %xmm2
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    packuswb %xmm2, %xmm1
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    paddb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; SSE41-NEXT:    psraw $8, %xmm1
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE41-NEXT:    psraw $8, %xmm2
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    packuswb %xmm1, %xmm2
-; SSE41-NEXT:    psrlw $7, %xmm0
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    paddb %xmm2, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_divconstant_16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE-NEXT:    pmulhw {{.*}}(%rip), %xmm2
+; SSE-NEXT:    psrlw $8, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE-NEXT:    pmulhw {{.*}}(%rip), %xmm1
+; SSE-NEXT:    psrlw $8, %xmm1
+; SSE-NEXT:    packuswb %xmm2, %xmm1
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    paddb %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; SSE-NEXT:    psraw $8, %xmm1
+; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; SSE-NEXT:    psrlw $8, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE-NEXT:    psraw $8, %xmm2
+; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm2
+; SSE-NEXT:    psrlw $8, %xmm2
+; SSE-NEXT:    packuswb %xmm1, %xmm2
+; SSE-NEXT:    psrlw $7, %xmm0
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE-NEXT:    paddb %xmm2, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: test_divconstant_16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
@@ -609,73 +557,45 @@ define <8 x i16> @test_rem7_8i16(<8 x i16> %a) nounwind {
 }
 
 define <16 x i8> @test_rem7_16i8(<16 x i8> %a) nounwind {
-; SSE2-LABEL: test_rem7_16i8:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65427,65427,65427,65427,65427,65427,65427,65427]
-; SSE2-NEXT:    pmullw %xmm3, %xmm2
-; SSE2-NEXT:    psrlw $8, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pmullw %xmm3, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
-; SSE2-NEXT:    packuswb %xmm2, %xmm1
-; SSE2-NEXT:    paddb %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psrlw $2, %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    psrlw $7, %xmm1
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    paddb %xmm2, %xmm1
-; SSE2-NEXT:    psubb %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    psllw $3, %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    psubb %xmm2, %xmm1
-; SSE2-NEXT:    paddb %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: test_rem7_16i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovsxbw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [65427,65427,65427,65427,65427,65427,65427,65427]
-; SSE41-NEXT:    pmullw %xmm2, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; SSE41-NEXT:    pmovsxbw %xmm3, %xmm3
-; SSE41-NEXT:    pmullw %xmm2, %xmm3
-; SSE41-NEXT:    psrlw $8, %xmm3
-; SSE41-NEXT:    packuswb %xmm3, %xmm1
-; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psrlw $2, %xmm2
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; SSE41-NEXT:    pxor %xmm3, %xmm2
-; SSE41-NEXT:    psrlw $7, %xmm1
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    paddb %xmm2, %xmm1
-; SSE41-NEXT:    psubb %xmm3, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    psllw $3, %xmm2
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    psubb %xmm2, %xmm1
-; SSE41-NEXT:    paddb %xmm1, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: test_rem7_16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [37632,37632,37632,37632,37632,37632,37632,37632]
+; SSE-NEXT:    pmulhw %xmm3, %xmm2
+; SSE-NEXT:    psrlw $8, %xmm2
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE-NEXT:    pmulhw %xmm3, %xmm1
+; SSE-NEXT:    psrlw $8, %xmm1
+; SSE-NEXT:    packuswb %xmm2, %xmm1
+; SSE-NEXT:    paddb %xmm0, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    psrlw $2, %xmm2
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; SSE-NEXT:    pxor %xmm3, %xmm2
+; SSE-NEXT:    psrlw $7, %xmm1
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE-NEXT:    paddb %xmm2, %xmm1
+; SSE-NEXT:    psubb %xmm3, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    psllw $3, %xmm2
+; SSE-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE-NEXT:    psubb %xmm2, %xmm1
+; SSE-NEXT:    paddb %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: test_rem7_16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm3
-; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX1-NEXT:    vpmulhw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm2
@@ -747,13 +667,13 @@ define <16 x i8> @test_rem7_16i8(<16 x i8> %a) nounwind {
 define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ; SSE2-LABEL: test_remconstant_16i8:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; SSE2-NEXT:    psraw $8, %xmm1
-; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    pmulhw {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    psrlw $8, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    pmulhw {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    psrlw $8, %xmm2
 ; SSE2-NEXT:    packuswb %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,0,255,0,0,255,255,0,0,255,0,0,0,255]
@@ -787,13 +707,14 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ;
 ; SSE41-LABEL: test_remconstant_16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovsxbw %xmm0, %xmm2
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE41-NEXT:    pmovsxbw %xmm1, %xmm1
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; SSE41-NEXT:    pmulhw {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE41-NEXT:    pmulhw {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    psrlw $8, %xmm2
 ; SSE41-NEXT:    packuswb %xmm1, %xmm2
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,0,255,0,0,255,255,0,0,255,0,0,0,255]
 ; SSE41-NEXT:    pand %xmm0, %xmm1
@@ -825,13 +746,13 @@ define <16 x i8> @test_remconstant_16i8(<16 x i8> %a) nounwind {
 ;
 ; AVX1-LABEL: test_remconstant_16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm1
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm2
 ; AVX1-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
index d177f0f2a417305146c388607c7ec2f5a7e35ddb..30e79495357008b568127eb55f7a4913d51acdac 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-256.ll
@@ -159,57 +159,55 @@ define <32 x i8> @test_div7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: test_div7_32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
-; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpackuswb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpsubb %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm2
-; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm7, %xmm7
-; AVX1-NEXT:    vpmullw %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX1-NEXT:    vpxor %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm2
-; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm7, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubb %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm7, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2NOBW-LABEL: test_div7_32i8:
 ; AVX2NOBW:       # %bb.0:
-; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX2NOBW-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm3, %ymm3
-; AVX2NOBW-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX2NOBW-NEXT:    vmovdqa {{.*#+}} ymm3 = [37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpackuswb %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
 ; AVX2NOBW-NEXT:    vpsrlw $2, %ymm0, %ymm1
 ; AVX2NOBW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
@@ -249,39 +247,38 @@ define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: test_divconstant_32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm2
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm1
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm2
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpsraw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpaddb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpsraw $8, %xmm4, %xmm4
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm5, %xmm5
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm5, %xmm5
+; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm3
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm0
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm4, %xmm4
+; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
@@ -292,22 +289,21 @@ define <32 x i8> @test_divconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2NOBW-LABEL: test_divconstant_32i8:
 ; AVX2NOBW:       # %bb.0:
-; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX2NOBW-NEXT:    vpmulhw {{.*}}(%rip), %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpackuswb %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2NOBW-NEXT:    vpmulhw {{.*}}(%rip), %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX2NOBW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
 ; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
@@ -548,49 +544,48 @@ define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: test_rem7_32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpmovsxbw %xmm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
-; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpackuswb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm4
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddb %xmm1, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $7, %xmm3, %xmm5
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm4, %xmm8, %xmm4
-; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX1-NEXT:    vpxor %xmm7, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubb %xmm7, %xmm2, %xmm2
-; AVX1-NEXT:    vpsllw $3, %xmm2, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; AVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm2
-; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm4, %xmm4
-; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpand %xmm5, %xmm8, %xmm5
+; AVX1-NEXT:    vpsrlw $2, %xmm3, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX1-NEXT:    vpand %xmm3, %xmm9, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubb %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpsllw $3, %xmm3, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
+; AVX1-NEXT:    vpand %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vpsubb %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
 ; AVX1-NEXT:    vpand %xmm3, %xmm8, %xmm3
 ; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm2
-; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm7, %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm2, %xmm9, %xmm2
+; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubb %xmm7, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubb %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsllw $3, %xmm2, %xmm3
-; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsubb %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
@@ -598,16 +593,15 @@ define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 ;
 ; AVX2NOBW-LABEL: test_rem7_32i8:
 ; AVX2NOBW:       # %bb.0:
-; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX2NOBW-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm3, %ymm3
-; AVX2NOBW-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
+; AVX2NOBW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX2NOBW-NEXT:    vmovdqa {{.*#+}} ymm3 = [37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpackuswb %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2NOBW-NEXT:    vpmulhw %ymm3, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX2NOBW-NEXT:    vpaddb %ymm0, %ymm1, %ymm1
 ; AVX2NOBW-NEXT:    vpsrlw $2, %ymm1, %ymm2
 ; AVX2NOBW-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
@@ -654,83 +648,81 @@ define <32 x i8> @test_rem7_32i8(<32 x i8> %a) nounwind {
 define <32 x i8> @test_remconstant_32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: test_remconstant_32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpmovsxbw %xmm3, %xmm1
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm2, %xmm2
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm2
-; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpaddb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpsraw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX1-NEXT:    vpaddb %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpsraw $8, %xmm5, %xmm5
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm5, %xmm5
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpackuswb %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm2, %xmm5, %xmm5
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm6
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm2, %xmm6, %xmm6
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpsraw $8, %xmm6, %xmm6
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm6, %xmm6
+; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
+; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpsrlw $7, %xmm3, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm3, %xmm5, %xmm5
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm3, %xmm7
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm3, %xmm7, %xmm7
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm5, %xmm5
-; AVX1-NEXT:    vpand %xmm2, %xmm5, %xmm5
-; AVX1-NEXT:    vpackuswb %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpsubb %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpmovsxbw %xmm0, %xmm5
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm3, %xmm5, %xmm5
+; AVX1-NEXT:    vpackuswb %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vpsubb %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm5, %xmm5
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxbw %xmm6, %xmm6
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm6, %xmm6
-; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpackuswb %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpaddb %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpmulhw {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; AVX1-NEXT:    vpsraw $8, %xmm5, %xmm5
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm5, %xmm5
 ; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX1-NEXT:    vpsraw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm6, %xmm6
-; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpackuswb %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm5, %xmm2
 ; AVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm4, %xmm4
-; AVX1-NEXT:    vpand %xmm2, %xmm4, %xmm4
+; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2NOBW-LABEL: test_remconstant_32i8:
 ; AVX2NOBW:       # %bb.0:
-; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2NOBW-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX2NOBW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX2NOBW-NEXT:    vpmulhw {{.*}}(%rip), %ymm2, %ymm2
 ; AVX2NOBW-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX2NOBW-NEXT:    vpackuswb %ymm1, %ymm2, %ymm1
+; AVX2NOBW-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2NOBW-NEXT:    vpmulhw {{.*}}(%rip), %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX2NOBW-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX2NOBW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm2
 ; AVX2NOBW-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
 ; AVX2NOBW-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
diff --git a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
index 4aca1fa22091384c29a97861fd52ea8748fd64af..c5d45e1258e6dc17b5d2d637393c670a946bb7a9 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-sdiv-512.ll
@@ -130,59 +130,55 @@ define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_div7_64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
-; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm5, %ymm5
+; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $2, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512F-NEXT:    vpxor %ymm6, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsubb %ymm6, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm7 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm7, %ymm7
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm7, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX512F-NEXT:    vpand %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512F-NEXT:    vpxor %ymm7, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubb %ymm7, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm2
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpxor %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vpxor %ymm7, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsubb %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsubb %ymm7, %ymm0, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_div7_64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpsraw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX512BW-NEXT:    vpmullw %zmm2, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpsraw $8, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX512BW-NEXT:    vpmulhw %zmm3, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; AVX512BW-NEXT:    vpmulhw %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
@@ -204,41 +200,38 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_divconstant_64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm2
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
+; AVX512F-NEXT:    vpmulhw {{.*}}(%rip), %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
+; AVX512F-NEXT:    vpmulhw {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm3
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpsraw $8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpsraw $8, %ymm5, %ymm5
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
+; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm5, %ymm4
 ; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmulhw {{.*}}(%rip), %ymm4, %ymm4
+; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512F-NEXT:    vpmulhw {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
@@ -249,22 +242,21 @@ define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm0
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_divconstant_64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpsraw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpsraw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; AVX512BW-NEXT:    vpmulhw {{.*}}(%rip), %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; AVX512BW-NEXT:    vpmulhw {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
@@ -454,51 +446,48 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_rem7_64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $7, %ymm2, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsrlw $2, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512F-NEXT:    vpxor %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsllw $3, %ymm2, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; AVX512F-NEXT:    vpand %ymm4, %ymm8, %ymm4
-; AVX512F-NEXT:    vpsubb %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpackuswb %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm5, %ymm5
+; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm5, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vpand %ymm6, %ymm5, %ymm5
+; AVX512F-NEXT:    vpsrlw $2, %ymm3, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512F-NEXT:    vpxor %ymm3, %ymm8, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsubb %ymm8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsllw $3, %ymm3, %ymm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
+; AVX512F-NEXT:    vpand %ymm5, %ymm9, %ymm5
+; AVX512F-NEXT:    vpsubb %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512F-NEXT:    vpmulhw %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm0, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $7, %ymm2, %ymm3
-; AVX512F-NEXT:    vpand %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsrlw $2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vpxor %ymm7, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
+; AVX512F-NEXT:    vpxor %ymm2, %ymm8, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsubb %ymm7, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsubb %ymm8, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $3, %ymm2, %ymm3
-; AVX512F-NEXT:    vpand %ymm3, %ymm8, %ymm3
+; AVX512F-NEXT:    vpand %ymm3, %ymm9, %ymm3
 ; AVX512F-NEXT:    vpsubb %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
@@ -506,16 +495,15 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 ;
 ; AVX512BW-LABEL: test_rem7_64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpsraw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427,65427]
-; AVX512BW-NEXT:    vpmullw %zmm2, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpsraw $8, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632,37632]
+; AVX512BW-NEXT:    vpmulhw %zmm3, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; AVX512BW-NEXT:    vpmulhw %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpaddb %zmm0, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $2, %zmm1, %zmm2
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
@@ -540,85 +528,81 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: test_remconstant_64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15],ymm1[24],ymm4[24],ymm1[25],ymm4[25],ymm1[26],ymm4[26],ymm1[27],ymm4[27],ymm1[28],ymm4[28],ymm1[29],ymm4[29],ymm1[30],ymm4[30],ymm1[31],ymm4[31]
+; AVX512F-NEXT:    vpmulhw {{.*}}(%rip), %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT:    vpackuswb %ymm1, %ymm2, %ymm2
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm1
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpsraw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm5, %ymm5
-; AVX512F-NEXT:    vpsrlw $7, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm6
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpand %ymm2, %ymm6, %ymm6
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpand %ymm2, %ymm5, %ymm5
-; AVX512F-NEXT:    vpackuswb %ymm6, %ymm5, %ymm5
-; AVX512F-NEXT:    vpsubb %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[16],ymm4[16],ymm1[17],ymm4[17],ymm1[18],ymm4[18],ymm1[19],ymm4[19],ymm1[20],ymm4[20],ymm1[21],ymm4[21],ymm1[22],ymm4[22],ymm1[23],ymm4[23]
+; AVX512F-NEXT:    vpmulhw {{.*}}(%rip), %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpackuswb %ymm2, %ymm3, %ymm3
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm5
+; AVX512F-NEXT:    vpaddb %ymm5, %ymm3, %ymm3
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512F-NEXT:    vpsraw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512F-NEXT:    vpsraw $8, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm6, %ymm6
 ; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpackuswb %ymm5, %ymm6, %ymm5
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm5, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpsrlw $7, %ymm3, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT:    vpaddb %ymm3, %ymm5, %ymm5
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm3, %ymm7
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpand %ymm3, %ymm7, %ymm7
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
+; AVX512F-NEXT:    vpand %ymm3, %ymm5, %ymm5
+; AVX512F-NEXT:    vpackuswb %ymm7, %ymm5, %ymm5
+; AVX512F-NEXT:    vpsubb %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512F-NEXT:    vpmulhw {{.*}}(%rip), %ymm5, %ymm5
+; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512F-NEXT:    vpmulhw {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpsraw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512F-NEXT:    vpsraw $8, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512F-NEXT:    vpsraw $8, %ymm6, %ymm6
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm6, %ymm6
-; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm6
-; AVX512F-NEXT:    vpackuswb %ymm5, %ymm6, %ymm5
+; AVX512F-NEXT:    vpackuswb %ymm2, %ymm5, %ymm2
 ; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm5, %ymm1
-; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm2, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm6, %ymm1, %ymm1
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_remconstant_64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpsraw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpsraw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpmullw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; AVX512BW-NEXT:    vpmulhw {{.*}}(%rip), %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; AVX512BW-NEXT:    vpmulhw {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm2
 ; AVX512BW-NEXT:    vpaddb %zmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
diff --git a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
index 8ce651e9cd15af59c1760ea0c059b7283bdc591c..85157d11e4b4a1ca270c0a5cf94e5a905dd02674 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-v2i32.ll
@@ -259,7 +259,7 @@ define void @test_urem_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    andps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    andps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    movlps %xmm0, (%eax)
 ; X86-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-128.ll b/llvm/test/CodeGen/X86/vector-lzcnt-128.ll
index cc8b39107064a1d2ccf9d1d3ce8ccdfee949b4b5..9576a0d7a8dc141ad7570904b3a6cdaf84da795c 100644
--- a/llvm/test/CodeGen/X86/vector-lzcnt-128.ll
+++ b/llvm/test/CodeGen/X86/vector-lzcnt-128.ll
@@ -236,7 +236,7 @@ define <2 x i64> @testv2i64(<2 x i64> %in) nounwind {
 ; X32-SSE-NEXT:    pshufb %xmm0, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlw $4, %xmm1
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-SSE-NEXT:    pxor %xmm4, %xmm4
 ; X32-SSE-NEXT:    pshufb %xmm1, %xmm2
 ; X32-SSE-NEXT:    pcmpeqb %xmm4, %xmm1
@@ -489,7 +489,7 @@ define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind {
 ; X32-SSE-NEXT:    pshufb %xmm0, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlw $4, %xmm1
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-SSE-NEXT:    pxor %xmm4, %xmm4
 ; X32-SSE-NEXT:    pshufb %xmm1, %xmm2
 ; X32-SSE-NEXT:    pcmpeqb %xmm4, %xmm1
@@ -724,7 +724,7 @@ define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
 ; X32-SSE-NEXT:    pshufb %xmm0, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlw $4, %xmm1
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-SSE-NEXT:    pxor %xmm4, %xmm4
 ; X32-SSE-NEXT:    pshufb %xmm1, %xmm2
 ; X32-SSE-NEXT:    pcmpeqb %xmm4, %xmm1
@@ -953,7 +953,7 @@ define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind {
 ; X32-SSE-NEXT:    pshufb %xmm0, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlw $4, %xmm1
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-SSE-NEXT:    pxor %xmm4, %xmm4
 ; X32-SSE-NEXT:    pshufb %xmm1, %xmm2
 ; X32-SSE-NEXT:    pcmpeqb %xmm4, %xmm1
@@ -1153,7 +1153,7 @@ define <8 x i16> @testv8i16(<8 x i16> %in) nounwind {
 ; X32-SSE-NEXT:    pshufb %xmm0, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlw $4, %xmm1
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-SSE-NEXT:    pxor %xmm4, %xmm4
 ; X32-SSE-NEXT:    pshufb %xmm1, %xmm2
 ; X32-SSE-NEXT:    pcmpeqb %xmm4, %xmm1
@@ -1346,7 +1346,7 @@ define <8 x i16> @testv8i16u(<8 x i16> %in) nounwind {
 ; X32-SSE-NEXT:    pshufb %xmm0, %xmm3
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X32-SSE-NEXT:    psrlw $4, %xmm1
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X32-SSE-NEXT:    pxor %xmm4, %xmm4
 ; X32-SSE-NEXT:    pshufb %xmm1, %xmm2
 ; X32-SSE-NEXT:    pcmpeqb %xmm4, %xmm1
@@ -1501,7 +1501,7 @@ define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
 ; X32-SSE-NEXT:    pshufb %xmm0, %xmm2
 ; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-SSE-NEXT:    pxor %xmm3, %xmm3
 ; X32-SSE-NEXT:    pcmpeqb %xmm0, %xmm3
 ; X32-SSE-NEXT:    pand %xmm2, %xmm3
@@ -1651,7 +1651,7 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
 ; X32-SSE-NEXT:    pshufb %xmm0, %xmm2
 ; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X32-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X32-SSE-NEXT:    pxor %xmm3, %xmm3
 ; X32-SSE-NEXT:    pcmpeqb %xmm0, %xmm3
 ; X32-SSE-NEXT:    pand %xmm2, %xmm3
diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-256.ll b/llvm/test/CodeGen/X86/vector-lzcnt-256.ll
index b7632aea6d2b35a3aa601c2d92bb7d64d042e15b..9624b3601a95f596d872e59fae337f274390a32d 100644
--- a/llvm/test/CodeGen/X86/vector-lzcnt-256.ll
+++ b/llvm/test/CodeGen/X86/vector-lzcnt-256.ll
@@ -164,7 +164,7 @@ define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
 ; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; X32-AVX-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; X32-AVX-NEXT:    vpand {{\.LCPI.*}}, %ymm3, %ymm3
+; X32-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
 ; X32-AVX-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; X32-AVX-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm5
 ; X32-AVX-NEXT:    vpand %ymm5, %ymm2, %ymm2
@@ -346,7 +346,7 @@ define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
 ; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; X32-AVX-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; X32-AVX-NEXT:    vpand {{\.LCPI.*}}, %ymm3, %ymm3
+; X32-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
 ; X32-AVX-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; X32-AVX-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm5
 ; X32-AVX-NEXT:    vpand %ymm5, %ymm2, %ymm2
@@ -503,7 +503,7 @@ define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
 ; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; X32-AVX-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; X32-AVX-NEXT:    vpand {{\.LCPI.*}}, %ymm3, %ymm3
+; X32-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
 ; X32-AVX-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; X32-AVX-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm5
 ; X32-AVX-NEXT:    vpand %ymm5, %ymm2, %ymm2
@@ -655,7 +655,7 @@ define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {
 ; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; X32-AVX-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; X32-AVX-NEXT:    vpand {{\.LCPI.*}}, %ymm3, %ymm3
+; X32-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
 ; X32-AVX-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; X32-AVX-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm5
 ; X32-AVX-NEXT:    vpand %ymm5, %ymm2, %ymm2
@@ -778,7 +778,7 @@ define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {
 ; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; X32-AVX-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; X32-AVX-NEXT:    vpand {{\.LCPI.*}}, %ymm3, %ymm3
+; X32-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
 ; X32-AVX-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; X32-AVX-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm5
 ; X32-AVX-NEXT:    vpand %ymm5, %ymm2, %ymm2
@@ -895,7 +895,7 @@ define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {
 ; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; X32-AVX-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; X32-AVX-NEXT:    vpand {{\.LCPI.*}}, %ymm3, %ymm3
+; X32-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
 ; X32-AVX-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; X32-AVX-NEXT:    vpcmpeqb %ymm4, %ymm3, %ymm5
 ; X32-AVX-NEXT:    vpand %ymm5, %ymm2, %ymm2
@@ -994,7 +994,7 @@ define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
 ; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; X32-AVX-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; X32-AVX-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; X32-AVX-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
 ; X32-AVX-NEXT:    vpand %ymm3, %ymm2, %ymm2
@@ -1088,7 +1088,7 @@ define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
 ; X32-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; X32-AVX-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
 ; X32-AVX-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; X32-AVX-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X32-AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; X32-AVX-NEXT:    vpcmpeqb %ymm3, %ymm0, %ymm3
 ; X32-AVX-NEXT:    vpand %ymm3, %ymm2, %ymm2
diff --git a/llvm/test/CodeGen/X86/vector-mul.ll b/llvm/test/CodeGen/X86/vector-mul.ll
index 20e6e05440d8ae09208bd236b3469ab333308615..837bf8a27f3aa089ad8323e4dc556fa12d13f8a7 100644
--- a/llvm/test/CodeGen/X86/vector-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-mul.ll
@@ -55,7 +55,7 @@ define <16 x i8> @mul_v16i8_32(<16 x i8> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v16i8_32:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psllw $5, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v16i8_32:
@@ -118,7 +118,7 @@ define <2 x i64> @mul_v2i64_32_8(<2 x i64> %a0) nounwind {
 define <4 x i32> @mul_v4i32_1_2_4_8(<4 x i32> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v4i32_1_2_4_8:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v4i32_1_2_4_8:
@@ -147,7 +147,7 @@ define <4 x i32> @mul_v4i32_1_2_4_8(<4 x i32> %a0) nounwind {
 define <8 x i16> @mul_v8i16_1_2_4_8_16_32_64_128(<8 x i16> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v8i16_1_2_4_8_16_32_64_128:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v8i16_1_2_4_8_16_32_64_128:
@@ -250,7 +250,7 @@ define <2 x i64> @mul_v2i64_17(<2 x i64> %a0) nounwind {
 define <4 x i32> @mul_v4i32_17(<4 x i32> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v4i32_17:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v4i32_17:
@@ -280,7 +280,7 @@ define <4 x i32> @mul_v4i32_17(<4 x i32> %a0) nounwind {
 define <8 x i16> @mul_v8i16_17(<8 x i16> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v8i16_17:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v8i16_17:
@@ -301,7 +301,7 @@ define <16 x i8> @mul_v16i8_17(<16 x i8> %a0) nounwind {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE-NEXT:    psllw $4, %xmm1
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    paddb %xmm0, %xmm1
 ; X86-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -523,7 +523,7 @@ define <2 x i64> @mul_v2i64_neg1025(<2 x i64> %a0) nounwind {
 define <4 x i32> @mul_v4i32_neg33(<4 x i32> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v4i32_neg33:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v4i32_neg33:
@@ -553,7 +553,7 @@ define <4 x i32> @mul_v4i32_neg33(<4 x i32> %a0) nounwind {
 define <8 x i16> @mul_v8i16_neg9(<8 x i16> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v8i16_neg9:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v8i16_neg9:
@@ -574,7 +574,7 @@ define <16 x i8> @mul_v16i8_neg5(<16 x i8> %a0) nounwind {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE-NEXT:    psllw $2, %xmm1
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    paddb %xmm0, %xmm1
 ; X86-SSE-NEXT:    pxor %xmm0, %xmm0
 ; X86-SSE-NEXT:    psubb %xmm1, %xmm0
@@ -845,7 +845,7 @@ define <2 x i64> @mul_v2i64_17_65(<2 x i64> %a0) nounwind {
 define <4 x i32> @mul_v4i32_5_17_33_65(<4 x i32> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v4i32_5_17_33_65:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v4i32_5_17_33_65:
@@ -864,7 +864,7 @@ define <4 x i32> @mul_v4i32_5_17_33_65(<4 x i32> %a0) nounwind {
 define <8 x i16> @mul_v8i16_2_3_9_17_33_65_129_257(<8 x i16> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v8i16_2_3_9_17_33_65_129_257:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v8i16_2_3_9_17_33_65_129_257:
@@ -885,10 +885,10 @@ define <16 x i8> @mul_v16i8_2_3_9_17_33_65_129_2_3_9_17_33_65_129_2_3(<16 x i8>
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; X86-SSE-NEXT:    pand %xmm2, %xmm0
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    pand %xmm2, %xmm1
 ; X86-SSE-NEXT:    packuswb %xmm0, %xmm1
 ; X86-SSE-NEXT:    movdqa %xmm1, %xmm0
@@ -974,7 +974,7 @@ define <2 x i64> @mul_v2i64_7(<2 x i64> %a0) nounwind {
 define <4 x i32> @mul_v4i32_7(<4 x i32> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v4i32_7:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v4i32_7:
@@ -1004,7 +1004,7 @@ define <4 x i32> @mul_v4i32_7(<4 x i32> %a0) nounwind {
 define <8 x i16> @mul_v8i16_7(<8 x i16> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v8i16_7:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v8i16_7:
@@ -1025,7 +1025,7 @@ define <16 x i8> @mul_v16i8_31(<16 x i8> %a0) nounwind {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE-NEXT:    psllw $5, %xmm1
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    psubb %xmm0, %xmm1
 ; X86-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1097,7 +1097,7 @@ define <2 x i64> @mul_v2i64_neg7(<2 x i64> %a0) nounwind {
 define <4 x i32> @mul_v4i32_neg63(<4 x i32> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v4i32_neg63:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v4i32_neg63:
@@ -1127,7 +1127,7 @@ define <4 x i32> @mul_v4i32_neg63(<4 x i32> %a0) nounwind {
 define <8 x i16> @mul_v8i16_neg31(<8 x i16> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v8i16_neg31:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v8i16_neg31:
@@ -1148,7 +1148,7 @@ define <16 x i8> @mul_v16i8_neg15(<16 x i8> %a0) nounwind {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE-NEXT:    psllw $4, %xmm1
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    psubb %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
@@ -1503,7 +1503,7 @@ define <2 x i64> @mul_v2i64_15_neg_63(<2 x i64> %a0) nounwind {
 define <4 x i32> @mul_v4i32_0_15_31_7(<4 x i32> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v4i32_0_15_31_7:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmulld {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v4i32_0_15_31_7:
@@ -1522,7 +1522,7 @@ define <4 x i32> @mul_v4i32_0_15_31_7(<4 x i32> %a0) nounwind {
 define <8 x i16> @mul_v8i16_0_1_7_15_31_63_127_255(<8 x i16> %a0) nounwind {
 ; X86-SSE-LABEL: mul_v8i16_0_1_7_15_31_63_127_255:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-SSE-LABEL: mul_v8i16_0_1_7_15_31_63_127_255:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
index 2958f7e96cb5c1295720f72c44e61bb388db7f15..e558660e4de8589c403ac7de62fc865a376b682f 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
@@ -219,16 +219,25 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; SSE-NEXT:    sete %al
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: trunc_v4i64_v4i1:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
-; AVX-NEXT:    vmovmskps %xmm0, %eax
-; AVX-NEXT:    cmpb $15, %al
-; AVX-NEXT:    sete %al
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: trunc_v4i64_v4i1:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vmovmskps %xmm0, %eax
+; AVX1-NEXT:    cmpb $15, %al
+; AVX1-NEXT:    sete %al
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_v4i64_v4i1:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $63, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskpd %ymm0, %eax
+; AVX2-NEXT:    cmpb $15, %al
+; AVX2-NEXT:    sete %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v4i64_v4i1:
 ; AVX512F:       # %bb.0:
@@ -296,14 +305,11 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ;
 ; AVX1-LABEL: trunc_v8i32_v8i1:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
 ; AVX1-NEXT:    cmpb $-1, %al
 ; AVX1-NEXT:    sete %al
 ; AVX1-NEXT:    vzeroupper
@@ -311,11 +317,8 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ;
 ; AVX2-LABEL: trunc_v8i32_v8i1:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
 ; AVX2-NEXT:    cmpb $-1, %al
 ; AVX2-NEXT:    sete %al
 ; AVX2-NEXT:    vzeroupper
@@ -536,17 +539,14 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ;
 ; AVX1-LABEL: trunc_v8i64_v8i1:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [65535,65535,65535,65535]
-; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
 ; AVX1-NEXT:    cmpb $-1, %al
 ; AVX1-NEXT:    sete %al
 ; AVX1-NEXT:    vzeroupper
@@ -557,11 +557,8 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; AVX2-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
 ; AVX2-NEXT:    cmpb $-1, %al
 ; AVX2-NEXT:    sete %al
 ; AVX2-NEXT:    vzeroupper
@@ -653,15 +650,13 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 ;
 ; AVX2-LABEL: trunc_v16i32_v16i1:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX2-NEXT:    cmpw $-1, %ax
@@ -714,7 +709,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ;
 ; AVX2-LABEL: trunc_v32i16_v32i1:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
index 283939f86a9451bbb5da6a403ca1a064ea138565..7239b8314d3a507b97f406b1065fef99484d9505 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
@@ -212,16 +212,25 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; SSE-NEXT:    setne %al
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: trunc_v4i64_v4i1:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
-; AVX-NEXT:    vmovmskps %xmm0, %eax
-; AVX-NEXT:    testb %al, %al
-; AVX-NEXT:    setne %al
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: trunc_v4i64_v4i1:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vmovmskps %xmm0, %eax
+; AVX1-NEXT:    testb %al, %al
+; AVX1-NEXT:    setne %al
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_v4i64_v4i1:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $63, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskpd %ymm0, %eax
+; AVX2-NEXT:    testb %al, %al
+; AVX2-NEXT:    setne %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v4i64_v4i1:
 ; AVX512F:       # %bb.0:
@@ -285,25 +294,21 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ;
 ; AVX1-LABEL: trunc_v8i32_v8i1:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
+; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    setne %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_v8i32_v8i1:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; AVX2-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
+; AVX2-NEXT:    testb %al, %al
 ; AVX2-NEXT:    setne %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -521,17 +526,15 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ;
 ; AVX1-LABEL: trunc_v8i64_v8i1:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [65535,65535,65535,65535]
-; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
-; AVX1-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
+; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    setne %al
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -541,11 +544,9 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; AVX2-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
-; AVX2-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
+; AVX2-NEXT:    testb %al, %al
 ; AVX2-NEXT:    setne %al
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -636,15 +637,13 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 ;
 ; AVX2-LABEL: trunc_v16i32_v16i1:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX2-NEXT:    testw %ax, %ax
@@ -697,7 +696,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ;
 ; AVX2-LABEL: trunc_v32i16_v32i1:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index a60dbcd7480af4d1eb5e84127ace255d23156dac..b5263772a6af19ea69ec4bb764cb3a4f5bdb7d4e 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -215,16 +215,25 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; SSE-NEXT:    setnp %al
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: trunc_v4i64_v4i1:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
-; AVX-NEXT:    vmovmskps %xmm0, %eax
-; AVX-NEXT:    testb %al, %al
-; AVX-NEXT:    setnp %al
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: trunc_v4i64_v4i1:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vmovmskps %xmm0, %eax
+; AVX1-NEXT:    testb %al, %al
+; AVX1-NEXT:    setnp %al
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: trunc_v4i64_v4i1:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $63, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskpd %ymm0, %eax
+; AVX2-NEXT:    testb %al, %al
+; AVX2-NEXT:    setnp %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_v4i64_v4i1:
 ; AVX512F:       # %bb.0:
@@ -290,14 +299,11 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ;
 ; AVX1-LABEL: trunc_v8i32_v8i1:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
 ; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
@@ -305,11 +311,8 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ;
 ; AVX2-LABEL: trunc_v8i32_v8i1:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
 ; AVX2-NEXT:    testb %al, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
@@ -548,17 +551,14 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ;
 ; AVX1-LABEL: trunc_v8i64_v8i1:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [65535,65535,65535,65535]
-; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovmskps %ymm0, %eax
 ; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    setnp %al
 ; AVX1-NEXT:    vzeroupper
@@ -569,11 +569,8 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
 ; AVX2-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
 ; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpsllw $15, %xmm0, %xmm0
-; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vmovmskps %ymm0, %eax
 ; AVX2-NEXT:    testb %al, %al
 ; AVX2-NEXT:    setnp %al
 ; AVX2-NEXT:    vzeroupper
@@ -665,15 +662,13 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 ;
 ; AVX2-LABEL: trunc_v16i32_v16i1:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpsllw $7, %xmm0, %xmm0
 ; AVX2-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX2-NEXT:    xorb %ah, %al
@@ -753,7 +748,7 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ;
 ; AVX2-LABEL: trunc_v32i16_v32i1:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index a00df716657b963e419e8f10373e6a6385639174..9db77246915c8fcbbb632b29c12bdd900508d839 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -265,9 +265,9 @@ define <4 x i32> @var_rotate_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ;
 ; X86-SSE2-LABEL: var_rotate_v4i32:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    pslld $23, %xmm1
-; X86-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE2-NEXT:    pmuludq %xmm1, %xmm0
@@ -444,7 +444,7 @@ define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ;
 ; X86-SSE2-LABEL: var_rotate_v8i16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
 ; X86-SSE2-NEXT:    pslld $23, %xmm2
@@ -677,20 +677,20 @@ define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; X86-SSE2-NEXT:    psrlw $4, %xmm4
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm5
 ; X86-SSE2-NEXT:    psllw $4, %xmm5
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm5
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm5
 ; X86-SSE2-NEXT:    por %xmm4, %xmm5
 ; X86-SSE2-NEXT:    pand %xmm3, %xmm5
 ; X86-SSE2-NEXT:    pandn %xmm2, %xmm3
 ; X86-SSE2-NEXT:    por %xmm5, %xmm3
 ; X86-SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; X86-SSE2-NEXT:    psrlw $6, %xmm2
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; X86-SSE2-NEXT:    psllw $2, %xmm4
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE2-NEXT:    por %xmm2, %xmm4
 ; X86-SSE2-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm2
@@ -702,7 +702,7 @@ define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE2-NEXT:    paddb %xmm2, %xmm3
 ; X86-SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; X86-SSE2-NEXT:    psrlw $7, %xmm4
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm4
 ; X86-SSE2-NEXT:    por %xmm3, %xmm4
 ; X86-SSE2-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
@@ -1070,7 +1070,7 @@ define <8 x i16> @splatvar_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ;
 ; X86-SSE2-LABEL: splatvar_rotate_v8i16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,0,0]
 ; X86-SSE2-NEXT:    pand %xmm1, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
@@ -1285,7 +1285,7 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; X86-SSE2-LABEL: splatvar_rotate_v16i8:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; X86-SSE2-NEXT:    psubb %xmm1, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm1, %xmm3
@@ -1801,20 +1801,20 @@ define <16 x i8> @constant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm2
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE2-NEXT:    psrlw $8, %xmm2
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm3
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-SSE2-NEXT:    psrlw $8, %xmm3
 ; X86-SSE2-NEXT:    packuswb %xmm2, %xmm3
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm1
 ; X86-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE2-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE2-NEXT:    packuswb %xmm1, %xmm0
 ; X86-SSE2-NEXT:    por %xmm3, %xmm0
@@ -2127,9 +2127,9 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
 ; X86-SSE2-NEXT:    retl
   %shl = shl <16 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
@@ -2206,7 +2206,7 @@ define <2 x i64> @splatconstant_rotate_mask_v2i64(<2 x i64> %a) nounwind {
 ; X86-SSE2-LABEL: splatconstant_rotate_mask_v2i64:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    psrlq $49, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
   %shl = shl <2 x i64> %a, <i64 15, i64 15>
   %lshr = lshr <2 x i64> %a, <i64 49, i64 49>
@@ -2288,7 +2288,7 @@ define <4 x i32> @splatconstant_rotate_mask_v4i32(<4 x i32> %a) nounwind {
 ; X86-SSE2-NEXT:    psrld $28, %xmm1
 ; X86-SSE2-NEXT:    pslld $4, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
   %shl = shl <4 x i32> %a, <i32 4, i32 4, i32 4, i32 4>
   %lshr = lshr <4 x i32> %a, <i32 28, i32 28, i32 28, i32 28>
@@ -2372,7 +2372,7 @@ define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind {
 ; X86-SSE2-NEXT:    psrlw $11, %xmm1
 ; X86-SSE2-NEXT:    psllw $5, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
   %shl = shl <8 x i16> %a, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
   %lshr = lshr <8 x i16> %a, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
@@ -2465,11 +2465,11 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind {
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE2-NEXT:    psrlw $4, %xmm1
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
   %shl = shl <16 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %lshr = lshr <16 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
@@ -2551,7 +2551,7 @@ define <4 x i32> @rot16_demandedbits(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-SSE2-NEXT:    psrld $11, %xmm1
 ; X86-SSE2-NEXT:    pslld $11, %xmm0
 ; X86-SSE2-NEXT:    por %xmm1, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
   %t0 = lshr <4 x i32> %x, <i32 11, i32 11, i32 11, i32 11>
   %t1 = shl <4 x i32> %x, <i32 11, i32 11, i32 11, i32 11>
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index 212b552254722c4d66ddbb7d1bba7eddd7b38544..c2769e7a9cc8f34d6877e50fcef5345a5db7a92f 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -2333,14 +2333,10 @@ define <8 x i32> @load_sext_8i1_to_8i32(<8 x i1> *%ptr) {
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: load_sext_8i1_to_8i32:
@@ -2558,14 +2554,10 @@ define <16 x i16> @load_sext_16i1_to_16i16(<16 x i1> *%ptr) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpeqw {{.*}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpeqw {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: load_sext_16i1_to_16i16:
@@ -2636,12 +2628,10 @@ define <32 x i8> @load_sext_32i1_to_32i8(<32 x i1> *%ptr) nounwind readnone {
 ; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,0,1,1,4,4,5,5]
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [9241421688590303745,9241421688590303745]
+; AVX1-NEXT:    # xmm2 = mem[0,0]
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -3801,7 +3791,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X86-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
-; X86-SSE2-NEXT:    paddw {{\.LCPI.*}}, %xmm3
+; X86-SSE2-NEXT:    paddw {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
 ; X86-SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]
 ; X86-SSE2-NEXT:    psllq $58, %xmm0
@@ -3845,7 +3835,7 @@ define <8 x i64> @sext_8i6_to_8i64(i32 %x) nounwind uwtable readnone ssp {
 ; X86-SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; X86-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]
-; X86-SSE41-NEXT:    paddw {{\.LCPI.*}}, %xmm3
+; X86-SSE41-NEXT:    paddw {{\.LCPI[0-9]+_[0-9]+}}, %xmm3
 ; X86-SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
 ; X86-SSE41-NEXT:    psllq $58, %xmm0
 ; X86-SSE41-NEXT:    movdqa %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
index cb3382cf20d6074f969b190ae96fe2e13fe572f0..1ad32daeddc834adb03ba45b6875f1abecda0ff4 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -1255,11 +1255,11 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind {
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; X86-SSE-NEXT:    psraw $8, %xmm1
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    psrlw $8, %xmm1
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X86-SSE-NEXT:    psraw $8, %xmm0
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    psrlw $8, %xmm0
 ; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1447,7 +1447,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v16i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrlw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE-NEXT:    psubb %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index a182335f06d15bc3cd64fc7c03fad34213747232..81cadf70f4460f998a71c76971c151d3901f1ba8 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -1158,7 +1158,7 @@ define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
 ;
 ; X86-AVX2-LABEL: constant_shift_v8i32:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpsravd {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsravd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %shift = ashr <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
   ret <8 x i32> %shift
@@ -1230,18 +1230,18 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
 ;
 ; X86-AVX1-LABEL: constant_shift_v16i16:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpmulhw {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpmulhw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1
 ; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; X86-AVX1-NEXT:    vpsraw $1, %xmm0, %xmm2
 ; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3,4,5,6,7]
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X86-AVX1-NEXT:    vpmulhw {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpmulhw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: constant_shift_v16i16:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpmulhw {{\.LCPI.*}}, %ymm0, %ymm1
+; X86-AVX2-NEXT:    vpmulhw {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm1
 ; X86-AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; X86-AVX2-NEXT:    vpsraw $1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3,4,5,6,7]
@@ -1379,11 +1379,11 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; X86-AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
-; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1
 ; X86-AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; X86-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; X86-AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
-; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
@@ -1651,7 +1651,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; X86-AVX2-LABEL: splatconstant_shift_v32i8:
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; X86-AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index a55dc2e16185e53ab8180aac8c9263206298a9a1..d9d5802da26a5c4dcf08ca7e2aa48915b0a61f3c 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -2016,7 +2016,7 @@ define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X86-SSE-NEXT:    psraw $8, %xmm0
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    psrlw $8, %xmm0
 ; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -2104,7 +2104,7 @@ define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X86-SSE-NEXT:    psraw $8, %xmm0
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    psrlw $8, %xmm0
 ; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -2192,7 +2192,7 @@ define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X86-SSE-NEXT:    psraw $8, %xmm0
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    psrlw $8, %xmm0
 ; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -2350,7 +2350,7 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v8i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrlw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE-NEXT:    psubb %xmm1, %xmm0
@@ -2403,7 +2403,7 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v4i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrlw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE-NEXT:    psubb %xmm1, %xmm0
@@ -2456,7 +2456,7 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v2i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrlw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE-NEXT:    psubb %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
index 530ebbe60920eb2bf19915cccdcc41ff8f0e4423..0466721999ffa34d05b6c52776c40bcbb9f8e441 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
@@ -488,7 +488,7 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psrlw $4, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pxor %xmm3, %xmm3
@@ -497,7 +497,7 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psrlw $2, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
@@ -505,7 +505,7 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE-NEXT:    psrlw $1, %xmm0
 ; X86-SSE-NEXT:    pand %xmm2, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = lshr <16 x i8> %a, %b
@@ -972,7 +972,7 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind {
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
 ; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm2
-; X86-SSE-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmulhuw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    por %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1073,10 +1073,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind {
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm2
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm2
 ; X86-SSE-NEXT:    psrlw $8, %xmm2
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    psrlw $8, %xmm0
 ; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1223,7 +1223,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v16i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrlw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
index c995af7c430d37f936a459de1e9132a9c0fd7dc3..36d2470fac9d1fb0ad394919ffbe6135edfd9371 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -504,14 +504,14 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; X86-AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X86-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X86-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
@@ -875,7 +875,7 @@ define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
 ;
 ; X86-AVX2-LABEL: constant_shift_v4i64:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpsrlvq {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrlvq {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %shift = lshr <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
   ret <4 x i64> %shift
@@ -948,7 +948,7 @@ define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
 ;
 ; X86-AVX2-LABEL: constant_shift_v8i32:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrlvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %shift = lshr <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
   ret <8 x i32> %shift
@@ -1015,16 +1015,16 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
 ;
 ; X86-AVX1-LABEL: constant_shift_v16i16:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpmulhuw {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpmulhuw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1
 ; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X86-AVX1-NEXT:    vpmulhuw {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpmulhuw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: constant_shift_v16i16:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpmulhuw {{\.LCPI.*}}, %ymm0, %ymm1
+; X86-AVX2-NEXT:    vpmulhuw {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm1
 ; X86-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; X86-AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; X86-AVX2-NEXT:    retl
@@ -1151,10 +1151,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; X86-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X86-AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
 ; X86-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
@@ -1384,7 +1384,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; X86-AVX2-LABEL: splatconstant_shift_v32i8:
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
@@ -1454,7 +1454,7 @@ define <4 x i32> @sh_trunc_sh_vec(<4 x i64> %x) {
 ; X86-AVX1-NEXT:    vpsrlq $36, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpsrlq $36, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X86-AVX1-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
index 49cf4c0793d150433de5dbc14fb6f58ce92bb808..3e821a2d88e0fe414a299ae4393d5a0a0ac67824 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
@@ -595,7 +595,7 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psrlw $4, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pxor %xmm3, %xmm3
@@ -604,7 +604,7 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psrlw $2, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
@@ -612,7 +612,7 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE-NEXT:    psrlw $1, %xmm0
 ; X86-SSE-NEXT:    pand %xmm2, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = lshr <8 x i8> %a, %b
@@ -745,7 +745,7 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psrlw $4, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pxor %xmm3, %xmm3
@@ -754,7 +754,7 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psrlw $2, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
@@ -762,7 +762,7 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE-NEXT:    psrlw $1, %xmm0
 ; X86-SSE-NEXT:    pand %xmm2, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i8> %a, %b
@@ -895,7 +895,7 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psrlw $4, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pxor %xmm3, %xmm3
@@ -904,7 +904,7 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psrlw $2, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
@@ -912,7 +912,7 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm1
 ; X86-SSE-NEXT:    psrlw $1, %xmm0
 ; X86-SSE-NEXT:    pand %xmm2, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i8> %a, %b
@@ -1543,7 +1543,7 @@ define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
 ; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm2
-; X86-SSE-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmulhuw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    pand %xmm1, %xmm0
 ; X86-SSE-NEXT:    por %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1713,7 +1713,7 @@ define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    psrlw $8, %xmm0
 ; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1809,7 +1809,7 @@ define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    psrlw $8, %xmm0
 ; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1905,7 +1905,7 @@ define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    psrlw $8, %xmm0
 ; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -2052,7 +2052,7 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v8i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrlw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = lshr <8 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <8 x i8> %shift
@@ -2091,7 +2091,7 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v4i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrlw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i8> %a, <i8 3, i8 3, i8 3, i8 3>
   ret <4 x i8> %shift
@@ -2130,7 +2130,7 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v2i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psrlw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i8> %a, <i8 3, i8 3>
   ret <2 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
index d2d958bf328663281457ad7ef4a70c3781cee363..0c7a6fa03cfecfa5050df90ff5b2288dd55afb44 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
@@ -140,7 +140,7 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; X86-SSE-LABEL: var_shift_v4i32:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    pslld $23, %xmm1
-; X86-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE-NEXT:    pmuludq %xmm1, %xmm0
@@ -402,7 +402,7 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psllw $4, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pxor %xmm3, %xmm3
@@ -411,7 +411,7 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psllw $2, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
@@ -844,7 +844,7 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind {
 ;
 ; X86-SSE-LABEL: constant_shift_v8i16:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = shl <8 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
   ret <8 x i16> %shift
@@ -942,11 +942,11 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind {
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
 ; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; X86-SSE-NEXT:    pand %xmm2, %xmm1
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    pand %xmm2, %xmm0
 ; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1093,7 +1093,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v16i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psllw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index 758de4e64f7356379218388df449e12e59eabd4c..da3cebc4758639775fd7362d502bdf8e3b141444 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -435,10 +435,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; X86-AVX2-NEXT:    vpsllw $4, %ymm0, %ymm2
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpsllw $2, %ymm0, %ymm2
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X86-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
@@ -800,7 +800,7 @@ define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
 ;
 ; X86-AVX2-LABEL: constant_shift_v4i64:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpsllvq {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsllvq {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %shift = shl <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
   ret <4 x i64> %shift
@@ -845,15 +845,15 @@ define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
 ;
 ; X86-AVX1-LABEL: constant_shift_v8i32:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X86-AVX1-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpmulld {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: constant_shift_v8i32:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsllvd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %shift = shl <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
   ret <8 x i32> %shift
@@ -911,15 +911,15 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
 ;
 ; X86-AVX1-LABEL: constant_shift_v16i16:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpmullw {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X86-AVX1-NEXT:    vpmullw {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: constant_shift_v16i16:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %shift = shl <16 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
   ret <16 x i16> %shift
@@ -1055,12 +1055,12 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
 ; X86-AVX2-LABEL: constant_shift_v32i8:
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpsllw $4, %ymm0, %ymm1
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1
 ; X86-AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [8192,24640,41088,57536,49376,32928,16480,32,8192,24640,41088,57536,49376,32928,16480,32]
 ; X86-AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
 ; X86-AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpsllw $2, %ymm0, %ymm1
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm1
 ; X86-AVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; X86-AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm1
@@ -1293,7 +1293,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; X86-AVX2-LABEL: splatconstant_shift_v32i8:
 ; X86-AVX2:       # %bb.0:
 ; X86-AVX2-NEXT:    vpsllw $3, %ymm0, %ymm0
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    retl
   %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
index 0861a5f4e4029a0d05098e4eb77cafb15e0fef7e..60f21f3fb50c41431aa5b8032cdd96c7919f1ad4 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
@@ -76,7 +76,7 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; X86-SSE-LABEL: var_shift_v2i32:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    pslld $23, %xmm1
-; X86-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    paddd {{\.LCPI[0-9]+_[0-9]+}}, %xmm1
 ; X86-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X86-SSE-NEXT:    pmuludq %xmm1, %xmm0
@@ -465,7 +465,7 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psllw $4, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pxor %xmm3, %xmm3
@@ -474,7 +474,7 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psllw $2, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
@@ -609,7 +609,7 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psllw $4, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pxor %xmm3, %xmm3
@@ -618,7 +618,7 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psllw $2, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
@@ -753,7 +753,7 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psllw $4, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pxor %xmm3, %xmm3
@@ -762,7 +762,7 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; X86-SSE-NEXT:    pandn %xmm0, %xmm4
 ; X86-SSE-NEXT:    psllw $2, %xmm0
 ; X86-SSE-NEXT:    pand %xmm3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    por %xmm4, %xmm0
 ; X86-SSE-NEXT:    paddb %xmm1, %xmm1
 ; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
@@ -1369,7 +1369,7 @@ define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 ;
 ; X86-SSE-LABEL: constant_shift_v4i16:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = shl <4 x i16> %a, <i16 0, i16 1, i16 2, i16 3>
   ret <4 x i16> %shift
@@ -1431,7 +1431,7 @@ define <2 x i16> @constant_shift_v2i16(<2 x i16> %a) nounwind {
 ;
 ; X86-SSE-LABEL: constant_shift_v2i16:
 ; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = shl <2 x i16> %a, <i16 2, i16 3>
   ret <2 x i16> %shift
@@ -1517,8 +1517,8 @@ define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
 ; X86-SSE-LABEL: constant_shift_v8i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1606,8 +1606,8 @@ define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 ; X86-SSE-LABEL: constant_shift_v4i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1695,8 +1695,8 @@ define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ; X86-SSE-LABEL: constant_shift_v2i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    pxor %xmm1, %xmm1
 ; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
 ; X86-SSE-NEXT:    retl
@@ -1843,7 +1843,7 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v8i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psllw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = shl <8 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <8 x i8> %shift
@@ -1882,7 +1882,7 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v4i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psllw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = shl <4 x i8> %a, <i8 3, i8 3, i8 3, i8 3>
   ret <4 x i8> %shift
@@ -1921,7 +1921,7 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; X86-SSE-LABEL: splatconstant_shift_v2i8:
 ; X86-SSE:       # %bb.0:
 ; X86-SSE-NEXT:    psllw $3, %xmm0
-; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE-NEXT:    retl
   %shift = shl <2 x i8> %a, <i8 3, i8 3>
   ret <2 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
index 422f64d982bfb30bcfbce2bb6e0997dde7907611..1e1224ee5eab18a3aef3390a17ef3fade274dd9a 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
@@ -345,7 +345,7 @@ define <32 x i16> @test_mm512_mask_blend_epi16(<32 x i16> %A, <32 x i16> %W){
 ;
 ; KNL32-LABEL: test_mm512_mask_blend_epi16:
 ; KNL32:       # %bb.0: # %entry
-; KNL32-NEXT:    vpternlogd $216, {{\.LCPI.*}}{1to16}, %zmm1, %zmm0
+; KNL32-NEXT:    vpternlogd $216, {{\.LCPI[0-9]+_[0-9]+}}{1to16}, %zmm1, %zmm0
 ; KNL32-NEXT:    retl
 entry:
   %0 = shufflevector <32 x i16> %A, <32 x i16> %W, <32 x i32>  <i32 32, i32 1, i32 34, i32 3, i32 36, i32 5, i32 38, i32 7, i32 40, i32 9, i32 42, i32 11, i32 44, i32 13, i32 46, i32 15, i32 48, i32 17, i32 50, i32 19, i32 52, i32 21, i32 54, i32 23, i32 56, i32 25, i32 58, i32 27, i32 60, i32 29, i32 62, i32 31>
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
index 7187425aee0fa2548c38807de9f4d86aa53cd21c..32303f867e4bf9d46971864d42cb6fd9e96eba6c 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
@@ -77,7 +77,7 @@ define <4 x i64> @combine_permq_pshufb_as_vextracti128(<4 x i64> %a0) {
 ; X86-LABEL: combine_permq_pshufb_as_vextracti128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; X86-NEXT:    vpaddq {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vpaddq {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permq_pshufb_as_vextracti128:
@@ -97,7 +97,7 @@ define <4 x i64> @combine_permq_pshufb_as_vmovdqa(<4 x i64> %a0) {
 ; X86-LABEL: combine_permq_pshufb_as_vmovdqa:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovdqa %xmm0, %xmm0
-; X86-NEXT:    vpaddq {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vpaddq {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permq_pshufb_as_vmovdqa:
@@ -210,7 +210,7 @@ define <16 x i8> @combine_pshufb_as_vpbroadcastd128(<16 x i8> %a) {
 ; X86-LABEL: combine_pshufb_as_vpbroadcastd128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpbroadcastd %xmm0, %xmm0
-; X86-NEXT:    vpaddb {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpaddb {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_pshufb_as_vpbroadcastd128:
@@ -227,7 +227,7 @@ define <8 x i32> @combine_permd_as_vpbroadcastd256(<4 x i32> %a) {
 ; X86-LABEL: combine_permd_as_vpbroadcastd256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpbroadcastd %xmm0, %ymm0
-; X86-NEXT:    vpaddd {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permd_as_vpbroadcastd256:
@@ -254,7 +254,7 @@ define <8 x i32> @combine_permd_as_vpbroadcastq256(<4 x i32> %a) {
 ; X86-LABEL: combine_permd_as_vpbroadcastq256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpbroadcastq %xmm0, %ymm0
-; X86-NEXT:    vpaddd {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vpaddd {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permd_as_vpbroadcastq256:
@@ -543,7 +543,7 @@ define <32 x i8> @combine_pshufb_as_unpackhi_zero(<32 x i8> %a0) {
 define <32 x i8> @combine_psrlw_pshufb(<16 x i16> %a0) {
 ; X86-LABEL: combine_psrlw_pshufb:
 ; X86:       # %bb.0:
-; X86-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_psrlw_pshufb:
@@ -559,7 +559,7 @@ define <32 x i8> @combine_psrlw_pshufb(<16 x i16> %a0) {
 define <32 x i8> @combine_pslld_pshufb(<8 x i32> %a0) {
 ; X86-LABEL: combine_pslld_pshufb:
 ; X86:       # %bb.0:
-; X86-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    vandps {{\.LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_pslld_pshufb:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
index 56834cc2311657065592a67ce70ff49e7b4b2913..9d61fd2e8ecd51a6ad75e5c761c1b7000c917130 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
@@ -901,8 +901,8 @@ define <8 x double> @combine_vpermi2var_8f64_as_permpd(<8 x double> %x0, <8 x do
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
 ; X86-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],mem[0]
-; X86-NEXT:    vinsertf128 $1, {{\.LCPI.*}}, %ymm2, %ymm2
-; X86-NEXT:    vinsertf64x4 $1, {{\.LCPI.*}}, %zmm2, %zmm2
+; X86-NEXT:    vinsertf128 $1, {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
+; X86-NEXT:    vinsertf64x4 $1, {{\.LCPI[0-9]+_[0-9]+}}, %zmm2, %zmm2
 ; X86-NEXT:    vpermi2pd %zmm1, %zmm0, %zmm2
 ; X86-NEXT:    vpermpd {{.*#+}} zmm0 = zmm2[2,3,1,1,6,7,5,5]
 ; X86-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-xop.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-xop.ll
index f758a1d32c13b742aca579c348e893b84b987dd8..b453517c2885d1cabd499506df04aeb011e885c0 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-xop.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-xop.ll
@@ -135,7 +135,7 @@ define <4 x double> @demandedelts_vpermil2pd256_as_shufpd(<4 x double> %a0, <4 x
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
 ; X86-NEXT:    vunpcklpd {{.*#+}} xmm2 = xmm2[0],mem[0]
-; X86-NEXT:    vinsertf128 $1, {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-NEXT:    vinsertf128 $1, {{\.LCPI[0-9]+_[0-9]+}}, %ymm2, %ymm2
 ; X86-NEXT:    vpermil2pd $0, %ymm2, %ymm1, %ymm0, %ymm0
 ; X86-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,1,2,3]
 ; X86-NEXT:    retl
@@ -174,7 +174,7 @@ define <16 x i8> @combine_vpperm_zero(<16 x i8> %a0, <16 x i8> %a1) {
 define <16 x i8> @combine_vpperm_identity_bitcast(<16 x i8> %a0, <16 x i8> %a1) {
 ; X86-LABEL: combine_vpperm_identity_bitcast:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpaddq {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    vpaddq {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpperm_identity_bitcast:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
index baa0da8fabfd57558a833513e2de09cd025adf8c..18cba47834db207b6b08e048d34900c47fb385db 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
@@ -3112,10 +3112,10 @@ define void @PR43024() {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [NaN,NaN,0.0E+0,0.0E+0]
 ; AVX-NEXT:    vmovaps %xmm0, (%rax)
-; AVX-NEXT:    vaddss {{\.LCPI.*}}+{{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vaddss {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vaddss {{\.LCPI.*}}+{{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vaddss {{\.LCPI[0-9]+_[0-9]+}}+{{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vmovss %xmm0, (%rax)
 ; AVX-NEXT:    retq
   store <4 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x0, float 0x0>, <4 x float>* undef, align 16
diff --git a/llvm/test/CodeGen/X86/vector-trunc-math.ll b/llvm/test/CodeGen/X86/vector-trunc-math.ll
index 8820d5569bb495a202cf580aae326ca11d993025..5a8ecf519d093dbe9255d1058934692694e3dc8c 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-math.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-math.ll
@@ -99,32 +99,19 @@ define <8 x i16> @trunc_add_v8i64_v8i16(<8 x i64> %a0, <8 x i64> %a1) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_add_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpaddq %ymm3, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_add_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpaddq %ymm3, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_add_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_add_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -249,54 +236,27 @@ define <16 x i8> @trunc_add_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_add_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpaddq %ymm5, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vpaddq %ymm4, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpaddq %ymm7, %ymm3, %ymm3
-; AVX2-SLOW-NEXT:    vpaddq %ymm6, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_add_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpaddq %ymm5, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpaddq %ymm4, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpaddq %ymm7, %ymm3, %ymm3
-; AVX2-FAST-NEXT:    vpaddq %ymm6, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_add_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddq %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddq %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpaddq %ymm7, %ymm3, %ymm3
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_add_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -354,15 +314,13 @@ define <16 x i8> @trunc_add_v16i32_v16i8(<16 x i32> %a0, <16 x i32> %a1) nounwin
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -565,28 +523,18 @@ define <8 x i16> @trunc_add_const_v8i64_v8i16(<8 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_add_const_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_add_const_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_add_const_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_add_const_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -685,48 +633,24 @@ define <16 x i8> @trunc_add_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_add_const_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpaddb {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_add_const_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpaddb {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_add_const_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpaddb {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_add_const_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -771,15 +695,13 @@ define <16 x i8> @trunc_add_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 ;
 ; AVX2-LABEL: trunc_add_const_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpaddb {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -943,32 +865,19 @@ define <8 x i16> @trunc_sub_v8i64_v8i16(<8 x i64> %a0, <8 x i64> %a1) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_sub_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpsubq %ymm3, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_sub_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpsubq %ymm3, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_sub_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubq %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_sub_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -1093,54 +1002,27 @@ define <16 x i8> @trunc_sub_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_sub_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpsubq %ymm5, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vpsubq %ymm4, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpsubq %ymm7, %ymm3, %ymm3
-; AVX2-SLOW-NEXT:    vpsubq %ymm6, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_sub_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpsubq %ymm5, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpsubq %ymm4, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpsubq %ymm7, %ymm3, %ymm3
-; AVX2-FAST-NEXT:    vpsubq %ymm6, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_sub_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsubq %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubq %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpsubq %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpsubq %ymm7, %ymm3, %ymm3
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_sub_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -1198,15 +1080,13 @@ define <16 x i8> @trunc_sub_v16i32_v16i8(<16 x i32> %a0, <16 x i32> %a1) nounwin
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -1377,28 +1257,18 @@ define <8 x i16> @trunc_sub_const_v8i64_v8i16(<8 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_sub_const_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_sub_const_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_sub_const_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpsubw {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_sub_const_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -1497,48 +1367,24 @@ define <16 x i8> @trunc_sub_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_sub_const_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_sub_const_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_sub_const_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_sub_const_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -1583,15 +1429,13 @@ define <16 x i8> @trunc_sub_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 ;
 ; AVX2-LABEL: trunc_sub_const_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpsubb {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1822,40 +1666,23 @@ define <8 x i16> @trunc_mul_v8i64_v8i16(<8 x i64> %a0, <8 x i64> %a1) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_mul_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm4[0,2],ymm0[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_mul_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_mul_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1,2,3],ymm3[4],ymm4[5,6,7],ymm3[8],ymm4[9,10,11],ymm3[12],ymm4[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1,2,3],ymm2[4],ymm4[5,6,7],ymm2[8],ymm4[9,10,11],ymm2[12],ymm4[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm4[1,2,3],ymm1[4],ymm4[5,6,7],ymm1[8],ymm4[9,10,11],ymm1[12],ymm4[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm4[1,2,3],ymm0[4],ymm4[5,6,7],ymm0[8],ymm4[9,10,11],ymm0[12],ymm4[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_mul_v8i64_v8i16:
 ; AVX512F:       # %bb.0:
@@ -2008,70 +1835,27 @@ define <16 x i8> @trunc_mul_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_mul_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm7, %xmm8
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm8 = xmm7[0,2],xmm8[0,2]
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm3, %xmm7
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,2],xmm7[0,2]
-; AVX2-SLOW-NEXT:    vpmulld %xmm8, %xmm3, %xmm3
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm6, %xmm7
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[0,2],xmm7[0,2]
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm2, %xmm7
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm7[0,2]
-; AVX2-SLOW-NEXT:    vpmulld %xmm6, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm5, %xmm7
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[0,2],xmm7[0,2]
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm7[0,2]
-; AVX2-SLOW-NEXT:    vpmulld %xmm5, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm4, %xmm5
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,2],xmm5[0,2]
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm5[0,2]
-; AVX2-SLOW-NEXT:    vpmulld %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm6, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_mul_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm8 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm7, %ymm8, %ymm7
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm8, %ymm3
-; AVX2-FAST-NEXT:    vpmulld %xmm7, %xmm3, %xmm3
-; AVX2-FAST-NEXT:    vpermd %ymm6, %ymm8, %ymm6
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm8, %ymm2
-; AVX2-FAST-NEXT:    vpmulld %xmm6, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm5, %ymm8, %ymm5
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm8, %ymm1
-; AVX2-FAST-NEXT:    vpmulld %xmm5, %xmm1, %xmm1
-; AVX2-FAST-NEXT:    vpermd %ymm4, %ymm8, %ymm4
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm8, %ymm0
-; AVX2-FAST-NEXT:    vpmulld %xmm4, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm6, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_mul_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmuludq %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpmuludq %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpmuludq %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpmuludq %ymm7, %ymm3, %ymm3
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_mul_v16i64_v16i8:
 ; AVX512F:       # %bb.0:
@@ -2173,15 +1957,13 @@ define <16 x i8> @trunc_mul_v16i32_v16i8(<16 x i32> %a0, <16 x i32> %a1) nounwin
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2387,28 +2169,18 @@ define <8 x i16> @trunc_mul_const_v8i64_v8i16(<8 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_mul_const_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_mul_const_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_mul_const_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_mul_const_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -2524,58 +2296,27 @@ define <16 x i8> @trunc_mul_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_mul_const_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm4[0,2]
-; AVX2-SLOW-NEXT:    vpmulld {{.*}}(%rip), %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,2],xmm4[0,2]
-; AVX2-SLOW-NEXT:    vpmulld {{.*}}(%rip), %xmm3, %xmm3
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm5[0,2]
-; AVX2-SLOW-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[0,2]
-; AVX2-SLOW-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_mul_const_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpmulld {{.*}}(%rip), %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vpmulld {{.*}}(%rip), %xmm3, %xmm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_mul_const_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmuludq {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpmuludq {{.*}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT:    vpmuludq {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpmuludq {{.*}}(%rip), %ymm3, %ymm3
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_mul_const_v16i64_v16i8:
 ; AVX512F:       # %bb.0:
@@ -2677,17 +2418,15 @@ define <16 x i8> @trunc_mul_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 ;
 ; AVX2-LABEL: trunc_mul_const_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpmulld {{.*}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld {{.*}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -2846,32 +2585,19 @@ define <8 x i16> @trunc_and_v8i64_v8i16(<8 x i64> %a0, <8 x i64> %a1) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_and_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vandps %ymm3, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_and_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_and_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_and_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -2982,54 +2708,27 @@ define <16 x i8> @trunc_and_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_and_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vandps %ymm5, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vandps %ymm4, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vandps %ymm7, %ymm3, %ymm3
-; AVX2-SLOW-NEXT:    vandps %ymm6, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_and_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpand %ymm7, %ymm3, %ymm3
-; AVX2-FAST-NEXT:    vpand %ymm6, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_and_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm7, %ymm8, %ymm7
+; AVX2-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm6, %ymm8, %ymm6
+; AVX2-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm5, %ymm8, %ymm3
+; AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm8, %ymm3
+; AVX2-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_and_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -3079,17 +2778,15 @@ define <16 x i8> @trunc_and_v16i32_v16i8(<16 x i32> %a0, <16 x i32> %a1) nounwin
 ;
 ; AVX2-LABEL: trunc_and_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
 ; AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -3239,28 +2936,18 @@ define <8 x i16> @trunc_and_const_v8i64_v8i16(<8 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_and_const_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_and_const_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_and_const_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_and_const_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -3359,48 +3046,24 @@ define <16 x i8> @trunc_and_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_and_const_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_and_const_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_and_const_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_and_const_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -3445,15 +3108,13 @@ define <16 x i8> @trunc_and_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 ;
 ; AVX2-LABEL: trunc_and_const_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -3609,32 +3270,19 @@ define <8 x i16> @trunc_xor_v8i64_v8i16(<8 x i64> %a0, <8 x i64> %a1) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_xor_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vxorps %ymm3, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vxorps %ymm2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_xor_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpxor %ymm3, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpxor %ymm2, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_xor_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_xor_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -3745,54 +3393,27 @@ define <16 x i8> @trunc_xor_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_xor_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vxorps %ymm5, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vxorps %ymm4, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vxorps %ymm7, %ymm3, %ymm3
-; AVX2-SLOW-NEXT:    vxorps %ymm6, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_xor_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpxor %ymm5, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpxor %ymm7, %ymm3, %ymm3
-; AVX2-FAST-NEXT:    vpxor %ymm6, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_xor_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm7, %ymm3, %ymm3
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_xor_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -3844,15 +3465,13 @@ define <16 x i8> @trunc_xor_v16i32_v16i8(<16 x i32> %a0, <16 x i32> %a1) nounwin
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4002,28 +3621,18 @@ define <8 x i16> @trunc_xor_const_v8i64_v8i16(<8 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_xor_const_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_xor_const_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_xor_const_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_xor_const_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -4122,48 +3731,24 @@ define <16 x i8> @trunc_xor_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_xor_const_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_xor_const_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_xor_const_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_xor_const_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -4208,15 +3793,13 @@ define <16 x i8> @trunc_xor_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 ;
 ; AVX2-LABEL: trunc_xor_const_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -4372,32 +3955,19 @@ define <8 x i16> @trunc_or_v8i64_v8i16(<8 x i64> %a0, <8 x i64> %a1) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_or_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vorps %ymm3, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vorps %ymm2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_or_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpor %ymm3, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_or_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_or_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -4508,54 +4078,27 @@ define <16 x i8> @trunc_or_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwind
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_or_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vorps %ymm5, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vorps %ymm4, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vorps %ymm7, %ymm3, %ymm3
-; AVX2-SLOW-NEXT:    vorps %ymm6, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_or_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpor %ymm5, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vpor %ymm4, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpor %ymm7, %ymm3, %ymm3
-; AVX2-FAST-NEXT:    vpor %ymm6, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_or_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpor %ymm7, %ymm3, %ymm3
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_or_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -4607,15 +4150,13 @@ define <16 x i8> @trunc_or_v16i32_v16i8(<16 x i32> %a0, <16 x i32> %a1) nounwind
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpor %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4765,28 +4306,18 @@ define <8 x i16> @trunc_or_const_v8i64_v8i16(<8 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_or_const_v8i64_v8i16:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_or_const_v8i64_v8i16:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_or_const_v8i64_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_or_const_v8i64_v8i16:
 ; AVX512:       # %bb.0:
@@ -4885,48 +4416,24 @@ define <16 x i8> @trunc_or_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc_or_const_v16i64_v16i8:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm2[2,3],ymm3[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,2],ymm4[0,2],ymm2[4,6],ymm4[4,6]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-SLOW-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
-; AVX2-SLOW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc_or_const_v16i64_v16i8:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm4, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc_or_const_v16i64_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc_or_const_v16i64_v16i8:
 ; AVX512:       # %bb.0:
@@ -4971,15 +4478,13 @@ define <16 x i8> @trunc_or_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 ;
 ; AVX2-LABEL: trunc_or_const_v16i32_v16i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-trunc-packus.ll b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
index 30867e21914bfdb6be860399159aa8c9d26ce528..a3146c58eac184cbe66e465d2771e3f9c728acaa 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-packus.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
@@ -3871,9 +3871,9 @@ define <8 x i8> @trunc_packus_v8i64_v8i8(<8 x i64>* %p0) "min-legal-vector-width
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm2
 ; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -4274,9 +4274,9 @@ define void @trunc_packus_v8i64_v8i8_store(<8 x i64>* %p0, <8 x i8> *%p1) "min-l
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm2
 ; AVX2-NEXT:    vpand %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
index 442d903fe039a91f33dfac3253849f3571be2634..34f05c161a082c5601408b9d0a52ee0e524a813f 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
@@ -3639,9 +3639,9 @@ define <8 x i8> @trunc_ssat_v8i64_v8i8(<8 x i64>* %p0) "min-legal-vector-width"=
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -4060,9 +4060,9 @@ define void @trunc_ssat_v8i64_v8i8_store(<8 x i64>* %p0, <8 x i8> *%p1) "min-leg
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-trunc-usat.ll b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
index 54d29c644a4257af9b2a181052a66df5b1fb760c..720e8185b1ea74ca6b84d64a9a7d063c4d9f834e 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-usat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
@@ -2812,9 +2812,9 @@ define <8 x i8> @trunc_usat_v8i64_v8i8(<8 x i64>* %p0) {
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm5, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -3073,9 +3073,9 @@ define void @trunc_usat_v8i64_v8i8_store(<8 x i64>* %p0, <8 x i8> *%p1) {
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm5, %ymm3
 ; AVX2-NEXT:    vblendvpd %ymm3, %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX2-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-trunc.ll b/llvm/test/CodeGen/X86/vector-trunc.ll
index 6a103f642c76f794b1152d91168a687b54bc713c..c2b1bf1f04c93536f36cce9e1e48847e2fc60ce2 100644
--- a/llvm/test/CodeGen/X86/vector-trunc.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc.ll
@@ -187,28 +187,17 @@ define <8 x i16> @trunc8i64_8i16(<8 x i64> %a) {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-SLOW-LABEL: trunc8i64_8i16:
-; AVX2-SLOW:       # %bb.0: # %entry
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
-; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-SLOW-NEXT:    vzeroupper
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: trunc8i64_8i16:
-; AVX2-FAST:       # %bb.0: # %entry
-; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
-; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT:    vpermd %ymm1, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-FAST-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-FAST-NEXT:    vzeroupper
-; AVX2-FAST-NEXT:    retq
+; AVX2-LABEL: trunc8i64_8i16:
+; AVX2:       # %bb.0: # %entry
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7],ymm1[8],ymm2[9,10,11],ymm1[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7],ymm0[8],ymm2[9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: trunc8i64_8i16:
 ; AVX512:       # %bb.0: # %entry
@@ -280,17 +269,14 @@ define void @trunc8i64_8i8(<8 x i64> %a) {
 ;
 ; AVX2-LABEL: trunc8i64_8i8:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,0,8,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, (%rax)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -676,13 +662,12 @@ define void @trunc16i32_16i16(<16 x i32> %a) {
 ;
 ; AVX2-LABEL: trunc16i32_16i16:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqu %xmm1, (%rax)
-; AVX2-NEXT:    vmovdqu %xmm0, (%rax)
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7],ymm1[8],ymm2[9],ymm1[10],ymm2[11],ymm1[12],ymm2[13],ymm1[14],ymm2[15]
+; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vmovdqu %ymm0, (%rax)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -903,15 +888,13 @@ define void @trunc16i32_16i8(<16 x i32> %a) {
 ;
 ; AVX2-LABEL: trunc16i32_16i8:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
 ; AVX2-NEXT:    vmovdqu %xmm0, (%rax)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1303,7 +1286,7 @@ define void @trunc32i16_32i8(<32 x i16> %a) {
 ;
 ; AVX2-LABEL: trunc32i16_32i8:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll b/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll
index 137ce2d618834f09d8e2a1c66fe970b417285412..cf42ed3c2612c82c905e80637d3505053f7cf708 100644
--- a/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll
+++ b/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll
@@ -17,16 +17,16 @@
 define <16 x i8> @test_128_i8_x_16_7_mask_lshr_1(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_7_mask_lshr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_7_mask_lshr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_7_mask_lshr_1:
@@ -50,13 +50,13 @@ define <16 x i8> @test_128_i8_x_16_7_mask_lshr_1(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_lshr_1(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_lshr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_lshr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -78,13 +78,13 @@ define <16 x i8> @test_128_i8_x_16_28_mask_lshr_1(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_lshr_2(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_lshr_2:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $2, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_lshr_2:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $2, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -106,16 +106,16 @@ define <16 x i8> @test_128_i8_x_16_28_mask_lshr_2(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_lshr_3(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_lshr_3:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $3, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_lshr_3:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_28_mask_lshr_3:
@@ -138,16 +138,16 @@ define <16 x i8> @test_128_i8_x_16_28_mask_lshr_3(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_lshr_4(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_lshr_4:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $4, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_lshr_4:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_28_mask_lshr_4:
@@ -171,13 +171,13 @@ define <16 x i8> @test_128_i8_x_16_28_mask_lshr_4(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_224_mask_lshr_1(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_lshr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_224_mask_lshr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -199,13 +199,13 @@ define <16 x i8> @test_128_i8_x_16_224_mask_lshr_1(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_224_mask_lshr_4(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_lshr_4:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $4, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_224_mask_lshr_4:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -228,13 +228,13 @@ define <16 x i8> @test_128_i8_x_16_224_mask_lshr_5(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_lshr_5:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    psrlw $5, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_224_mask_lshr_5:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vpsrlw $5, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_224_mask_lshr_5:
@@ -256,13 +256,13 @@ define <16 x i8> @test_128_i8_x_16_224_mask_lshr_6(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_lshr_6:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    psrlw $6, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_224_mask_lshr_6:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vpsrlw $6, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_224_mask_lshr_6:
@@ -286,16 +286,16 @@ define <16 x i8> @test_128_i8_x_16_224_mask_lshr_6(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_7_mask_ashr_1(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_7_mask_ashr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_7_mask_ashr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_7_mask_ashr_1:
@@ -319,13 +319,13 @@ define <16 x i8> @test_128_i8_x_16_7_mask_ashr_1(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_ashr_1(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_ashr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_ashr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -347,13 +347,13 @@ define <16 x i8> @test_128_i8_x_16_28_mask_ashr_1(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_ashr_2(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_ashr_2:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $2, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_ashr_2:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $2, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -375,16 +375,16 @@ define <16 x i8> @test_128_i8_x_16_28_mask_ashr_2(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_ashr_3(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_ashr_3:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $3, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_ashr_3:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_28_mask_ashr_3:
@@ -407,16 +407,16 @@ define <16 x i8> @test_128_i8_x_16_28_mask_ashr_3(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_ashr_4(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_ashr_4:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $4, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_ashr_4:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_28_mask_ashr_4:
@@ -440,7 +440,7 @@ define <16 x i8> @test_128_i8_x_16_28_mask_ashr_4(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_224_mask_ashr_1(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_ashr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm0
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm0
@@ -449,7 +449,7 @@ define <16 x i8> @test_128_i8_x_16_224_mask_ashr_1(<16 x i8> %a0) {
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_224_mask_ashr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
 ; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -480,7 +480,7 @@ define <16 x i8> @test_128_i8_x_16_224_mask_ashr_1(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_224_mask_ashr_4(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_ashr_4:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $4, %xmm0
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm0
@@ -489,7 +489,7 @@ define <16 x i8> @test_128_i8_x_16_224_mask_ashr_4(<16 x i8> %a0) {
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_224_mask_ashr_4:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
@@ -521,7 +521,7 @@ define <16 x i8> @test_128_i8_x_16_224_mask_ashr_5(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_ashr_5:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    psrlw $5, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE2-NEXT:    psubb %xmm1, %xmm0
@@ -530,7 +530,7 @@ define <16 x i8> @test_128_i8_x_16_224_mask_ashr_5(<16 x i8> %a0) {
 ; X86-AVX-LABEL: test_128_i8_x_16_224_mask_ashr_5:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vpsrlw $5, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
 ; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
@@ -561,7 +561,7 @@ define <16 x i8> @test_128_i8_x_16_224_mask_ashr_6(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_ashr_6:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    psrlw $6, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
 ; X86-SSE2-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE2-NEXT:    psubb %xmm1, %xmm0
@@ -570,7 +570,7 @@ define <16 x i8> @test_128_i8_x_16_224_mask_ashr_6(<16 x i8> %a0) {
 ; X86-AVX-LABEL: test_128_i8_x_16_224_mask_ashr_6:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vpsrlw $6, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
 ; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
@@ -603,13 +603,13 @@ define <16 x i8> @test_128_i8_x_16_224_mask_ashr_6(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_7_mask_shl_1(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_7_mask_shl_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    paddb %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_7_mask_shl_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -631,13 +631,13 @@ define <16 x i8> @test_128_i8_x_16_7_mask_shl_1(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_7_mask_shl_4(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_7_mask_shl_4:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_7_mask_shl_4:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllw $4, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -660,13 +660,13 @@ define <16 x i8> @test_128_i8_x_16_7_mask_shl_5(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_7_mask_shl_5:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    psllw $5, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_7_mask_shl_5:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vpsllw $5, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_7_mask_shl_5:
@@ -688,13 +688,13 @@ define <16 x i8> @test_128_i8_x_16_7_mask_shl_6(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_7_mask_shl_6:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    psllw $6, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_7_mask_shl_6:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vpsllw $6, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_7_mask_shl_6:
@@ -716,13 +716,13 @@ define <16 x i8> @test_128_i8_x_16_7_mask_shl_6(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_shl_1(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_shl_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    paddb %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_shl_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -744,13 +744,13 @@ define <16 x i8> @test_128_i8_x_16_28_mask_shl_1(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_shl_2(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_shl_2:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllw $2, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_shl_2:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllw $2, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -772,13 +772,13 @@ define <16 x i8> @test_128_i8_x_16_28_mask_shl_2(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_shl_3(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_shl_3:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllw $3, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_shl_3:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -800,16 +800,16 @@ define <16 x i8> @test_128_i8_x_16_28_mask_shl_3(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_28_mask_shl_4(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_28_mask_shl_4:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_28_mask_shl_4:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllw $4, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: test_128_i8_x_16_28_mask_shl_4:
@@ -833,13 +833,13 @@ define <16 x i8> @test_128_i8_x_16_28_mask_shl_4(<16 x i8> %a0) {
 define <16 x i8> @test_128_i8_x_16_224_mask_shl_1(<16 x i8> %a0) {
 ; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_shl_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    paddb %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i8_x_16_224_mask_shl_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -868,13 +868,13 @@ define <16 x i8> @test_128_i8_x_16_224_mask_shl_1(<16 x i8> %a0) {
 define <8 x i16> @test_128_i16_x_8_127_mask_lshr_1(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_127_mask_lshr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_127_mask_lshr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -897,13 +897,13 @@ define <8 x i16> @test_128_i16_x_8_127_mask_lshr_1(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_lshr_3(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_lshr_3:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $3, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_lshr_3:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -925,13 +925,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_lshr_3(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_lshr_4(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_lshr_4:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $4, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_lshr_4:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -953,13 +953,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_lshr_4(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_lshr_5(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_lshr_5:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $5, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_lshr_5:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $5, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -981,13 +981,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_lshr_5(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_lshr_6(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_lshr_6:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $6, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_lshr_6:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $6, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1010,13 +1010,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_lshr_6(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_65024_mask_lshr_1(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_65024_mask_lshr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_65024_mask_lshr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1038,13 +1038,13 @@ define <8 x i16> @test_128_i16_x_8_65024_mask_lshr_1(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_65024_mask_lshr_8(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_65024_mask_lshr_8:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $8, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_65024_mask_lshr_8:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1117,13 +1117,13 @@ define <8 x i16> @test_128_i16_x_8_65024_mask_lshr_10(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_127_mask_ashr_1(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_127_mask_ashr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_127_mask_ashr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1146,13 +1146,13 @@ define <8 x i16> @test_128_i16_x_8_127_mask_ashr_1(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_ashr_3(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_ashr_3:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $3, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_ashr_3:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1174,13 +1174,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_ashr_3(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_ashr_4(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_ashr_4:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $4, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_ashr_4:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1202,13 +1202,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_ashr_4(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_ashr_5(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_ashr_5:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $5, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_ashr_5:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $5, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1230,13 +1230,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_ashr_5(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_ashr_6(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_ashr_6:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlw $6, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_ashr_6:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlw $6, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1259,13 +1259,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_ashr_6(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_65024_mask_ashr_1(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_65024_mask_ashr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psraw $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_65024_mask_ashr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsraw $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1287,13 +1287,13 @@ define <8 x i16> @test_128_i16_x_8_65024_mask_ashr_1(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_65024_mask_ashr_8(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_65024_mask_ashr_8:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psraw $8, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_65024_mask_ashr_8:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsraw $8, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1366,13 +1366,13 @@ define <8 x i16> @test_128_i16_x_8_65024_mask_ashr_10(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_127_mask_shl_1(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_127_mask_shl_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    paddw %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_127_mask_shl_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpaddw %xmm0, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1394,13 +1394,13 @@ define <8 x i16> @test_128_i16_x_8_127_mask_shl_1(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_127_mask_shl_8(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_127_mask_shl_8:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllw $8, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_127_mask_shl_8:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllw $8, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1471,13 +1471,13 @@ define <8 x i16> @test_128_i16_x_8_127_mask_shl_10(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_shl_3(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_shl_3:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllw $3, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_shl_3:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1499,13 +1499,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_shl_3(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_shl_4(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_shl_4:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllw $4, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_shl_4:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllw $4, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1527,13 +1527,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_shl_4(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_shl_5(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_shl_5:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllw $5, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_shl_5:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllw $5, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1555,13 +1555,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_shl_5(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_2032_mask_shl_6(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_2032_mask_shl_6:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllw $6, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_2032_mask_shl_6:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllw $6, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1584,13 +1584,13 @@ define <8 x i16> @test_128_i16_x_8_2032_mask_shl_6(<8 x i16> %a0) {
 define <8 x i16> @test_128_i16_x_8_65024_mask_shl_1(<8 x i16> %a0) {
 ; X86-SSE2-LABEL: test_128_i16_x_8_65024_mask_shl_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    paddw %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i16_x_8_65024_mask_shl_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpaddw %xmm0, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -1619,13 +1619,13 @@ define <8 x i16> @test_128_i16_x_8_65024_mask_shl_1(<8 x i16> %a0) {
 define <4 x i32> @test_128_i32_x_4_32767_mask_lshr_1(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_32767_mask_lshr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_32767_mask_lshr_1:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -1662,13 +1662,13 @@ define <4 x i32> @test_128_i32_x_4_32767_mask_lshr_1(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_lshr_7(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_lshr_7:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $7, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_lshr_7:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $7, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -1704,13 +1704,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_lshr_7(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_lshr_8(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_lshr_8:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $8, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_lshr_8:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $8, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -1746,13 +1746,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_lshr_8(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_lshr_9(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_lshr_9:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $9, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_lshr_9:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $9, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -1788,13 +1788,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_lshr_9(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_lshr_10(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_lshr_10:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $10, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_lshr_10:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $10, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -1831,13 +1831,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_lshr_10(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_4294836224_mask_lshr_1(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_4294836224_mask_lshr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_4294836224_mask_lshr_1:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -1873,13 +1873,13 @@ define <4 x i32> @test_128_i32_x_4_4294836224_mask_lshr_1(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_4294836224_mask_lshr_16(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_4294836224_mask_lshr_16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $16, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_4294836224_mask_lshr_16:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -1966,13 +1966,13 @@ define <4 x i32> @test_128_i32_x_4_4294836224_mask_lshr_18(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_32767_mask_ashr_1(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_32767_mask_ashr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_32767_mask_ashr_1:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2009,13 +2009,13 @@ define <4 x i32> @test_128_i32_x_4_32767_mask_ashr_1(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_ashr_7(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_ashr_7:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $7, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_ashr_7:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $7, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2051,13 +2051,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_ashr_7(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_ashr_8(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_ashr_8:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $8, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_ashr_8:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $8, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2093,13 +2093,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_ashr_8(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_ashr_9(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_ashr_9:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $9, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_ashr_9:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $9, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2135,13 +2135,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_ashr_9(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_ashr_10(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_ashr_10:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrld $10, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_ashr_10:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrld $10, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2178,13 +2178,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_ashr_10(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_4294836224_mask_ashr_1(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_4294836224_mask_ashr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrad $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_4294836224_mask_ashr_1:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrad $1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2220,13 +2220,13 @@ define <4 x i32> @test_128_i32_x_4_4294836224_mask_ashr_1(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_4294836224_mask_ashr_16(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_4294836224_mask_ashr_16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrad $16, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_4294836224_mask_ashr_16:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrad $16, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2313,13 +2313,13 @@ define <4 x i32> @test_128_i32_x_4_4294836224_mask_ashr_18(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_32767_mask_shl_1(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_32767_mask_shl_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    paddd %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_32767_mask_shl_1:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2355,13 +2355,13 @@ define <4 x i32> @test_128_i32_x_4_32767_mask_shl_1(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_32767_mask_shl_16(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_32767_mask_shl_16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pslld $16, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_32767_mask_shl_16:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpslld $16, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2446,13 +2446,13 @@ define <4 x i32> @test_128_i32_x_4_32767_mask_shl_18(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_shl_7(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_shl_7:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pslld $7, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_shl_7:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpslld $7, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2488,13 +2488,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_shl_7(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_shl_8(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_shl_8:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pslld $8, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_shl_8:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpslld $8, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2530,13 +2530,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_shl_8(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_shl_9(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_shl_9:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pslld $9, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_shl_9:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpslld $9, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2572,13 +2572,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_shl_9(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_8388352_mask_shl_10(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_8388352_mask_shl_10:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pslld $10, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_8388352_mask_shl_10:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpslld $10, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2615,13 +2615,13 @@ define <4 x i32> @test_128_i32_x_4_8388352_mask_shl_10(<4 x i32> %a0) {
 define <4 x i32> @test_128_i32_x_4_4294836224_mask_shl_1(<4 x i32> %a0) {
 ; X86-SSE2-LABEL: test_128_i32_x_4_4294836224_mask_shl_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    paddd %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i32_x_4_4294836224_mask_shl_1:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2664,13 +2664,13 @@ define <4 x i32> @test_128_i32_x_4_4294836224_mask_shl_1(<4 x i32> %a0) {
 define <2 x i64> @test_128_i64_x_2_2147483647_mask_lshr_1(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_2147483647_mask_lshr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_2147483647_mask_lshr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -2693,13 +2693,13 @@ define <2 x i64> @test_128_i64_x_2_2147483647_mask_lshr_1(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_lshr_15(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_lshr_15:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $15, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_lshr_15:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $15, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -2721,13 +2721,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_lshr_15(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_lshr_16(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_lshr_16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $16, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_lshr_16:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $16, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -2749,13 +2749,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_lshr_16(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_lshr_17(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_lshr_17:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $17, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_lshr_17:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $17, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -2777,13 +2777,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_lshr_17(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_lshr_18(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_lshr_18:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $18, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_lshr_18:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $18, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -2806,13 +2806,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_lshr_18(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_lshr_1(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_18446744065119617024_mask_lshr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_18446744065119617024_mask_lshr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -2834,13 +2834,13 @@ define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_lshr_1(<2 x i64> %a
 define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_lshr_32(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_18446744065119617024_mask_lshr_32:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $32, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i64_x_2_18446744065119617024_mask_lshr_32:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -2920,13 +2920,13 @@ define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_lshr_34(<2 x i64> %
 define <2 x i64> @test_128_i64_x_2_2147483647_mask_ashr_1(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_2147483647_mask_ashr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $1, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_2147483647_mask_ashr_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -2949,13 +2949,13 @@ define <2 x i64> @test_128_i64_x_2_2147483647_mask_ashr_1(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_ashr_15(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_ashr_15:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $15, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_ashr_15:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $15, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -2977,13 +2977,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_ashr_15(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_ashr_16(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_ashr_16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $16, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_ashr_16:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $16, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -3005,13 +3005,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_ashr_16(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_ashr_17(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_ashr_17:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $17, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_ashr_17:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $17, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -3033,13 +3033,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_ashr_17(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_ashr_18(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_ashr_18:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrlq $18, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_ashr_18:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsrlq $18, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -3062,14 +3062,14 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_ashr_18(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_ashr_1(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_18446744065119617024_mask_ashr_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psrad $1, %xmm0
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i64_x_2_18446744065119617024_mask_ashr_1:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrad $1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
@@ -3113,7 +3113,7 @@ define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_ashr_1(<2 x i64> %a
 define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_ashr_32(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_18446744065119617024_mask_ashr_32:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
 ; X86-SSE2-NEXT:    psrad $31, %xmm0
 ; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
@@ -3123,7 +3123,7 @@ define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_ashr_32(<2 x i64> %
 ;
 ; X86-AVX1-LABEL: test_128_i64_x_2_18446744065119617024_mask_ashr_32:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
 ; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
@@ -3131,7 +3131,7 @@ define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_ashr_32(<2 x i64> %
 ;
 ; X86-AVX2-LABEL: test_128_i64_x_2_18446744065119617024_mask_ashr_32:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; X86-AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
@@ -3284,13 +3284,13 @@ define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_ashr_34(<2 x i64> %
 define <2 x i64> @test_128_i64_x_2_2147483647_mask_shl_1(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_2147483647_mask_shl_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    paddq %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_2147483647_mask_shl_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -3312,13 +3312,13 @@ define <2 x i64> @test_128_i64_x_2_2147483647_mask_shl_1(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_2147483647_mask_shl_32(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_2147483647_mask_shl_32:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllq $32, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: test_128_i64_x_2_2147483647_mask_shl_32:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    retl
 ;
@@ -3396,13 +3396,13 @@ define <2 x i64> @test_128_i64_x_2_2147483647_mask_shl_34(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_shl_15(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_shl_15:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllq $15, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_shl_15:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllq $15, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -3424,13 +3424,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_shl_15(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_shl_16(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_shl_16:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllq $16, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_shl_16:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllq $16, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -3452,13 +3452,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_shl_16(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_shl_17(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_shl_17:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllq $17, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_shl_17:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllq $17, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -3480,13 +3480,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_shl_17(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_140737488289792_mask_shl_18(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_140737488289792_mask_shl_18:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    psllq $18, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_140737488289792_mask_shl_18:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpsllq $18, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
@@ -3509,13 +3509,13 @@ define <2 x i64> @test_128_i64_x_2_140737488289792_mask_shl_18(<2 x i64> %a0) {
 define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_shl_1(<2 x i64> %a0) {
 ; X86-SSE2-LABEL: test_128_i64_x_2_18446744065119617024_mask_shl_1:
 ; X86-SSE2:       # %bb.0:
-; X86-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    paddq %xmm0, %xmm0
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_128_i64_x_2_18446744065119617024_mask_shl_1:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    vpand {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
 ; X86-AVX-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/vshift-6.ll b/llvm/test/CodeGen/X86/vshift-6.ll
index 471ea5ad5c9372e60570da729141c22a8375ce62..3c0081d049f4623f3911bdff1406381c9ac7a319 100644
--- a/llvm/test/CodeGen/X86/vshift-6.ll
+++ b/llvm/test/CodeGen/X86/vshift-6.ll
@@ -42,7 +42,7 @@ define <16 x i8> @do_not_crash(i8*, i32*, i64*, i32, i64, i8) {
 ; X86-NEXT:    pxor %xmm0, %xmm0
 ; X86-NEXT:    pcmpgtb %xmm1, %xmm0
 ; X86-NEXT:    pxor %xmm0, %xmm2
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    por %xmm2, %xmm0
 ; X86-NEXT:    paddb %xmm1, %xmm1
 ; X86-NEXT:    pxor %xmm2, %xmm2
@@ -51,7 +51,7 @@ define <16 x i8> @do_not_crash(i8*, i32*, i64*, i32, i64, i8) {
 ; X86-NEXT:    pandn %xmm0, %xmm4
 ; X86-NEXT:    psllw $2, %xmm0
 ; X86-NEXT:    pand %xmm2, %xmm0
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    por %xmm4, %xmm0
 ; X86-NEXT:    paddb %xmm1, %xmm1
 ; X86-NEXT:    pcmpgtb %xmm1, %xmm3
diff --git a/llvm/test/CodeGen/X86/vsplit-and.ll b/llvm/test/CodeGen/X86/vsplit-and.ll
index aa043ed67f693118618fedbf41c6623e4c0b4dca..e1a99d209acf9a411023dae1796f434cdc623d2c 100644
--- a/llvm/test/CodeGen/X86/vsplit-and.ll
+++ b/llvm/test/CodeGen/X86/vsplit-and.ll
@@ -41,16 +41,13 @@ define void @t2(<3 x i64>* %dst, <3 x i64> %src1, <3 x i64> %src2) nounwind read
 ; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
 ; CHECK-NEXT:    xorps %xmm0, %xmm1
 ; CHECK-NEXT:    andnps %xmm1, %xmm2
-; CHECK-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,3,3]
 ; CHECK-NEXT:    psllq $63, %xmm0
 ; CHECK-NEXT:    psrad $31, %xmm0
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]
-; CHECK-NEXT:    psllq $63, %xmm1
-; CHECK-NEXT:    psrad $31, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-NEXT:    movq %xmm1, 16(%rdi)
-; CHECK-NEXT:    movdqa %xmm0, (%rdi)
+; CHECK-NEXT:    pmovsxdq %xmm2, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, (%rdi)
+; CHECK-NEXT:    movq %xmm0, 16(%rdi)
 ; CHECK-NEXT:    retq
   %cmp1 = icmp ne <3 x i64> %src1, zeroinitializer
   %cmp2 = icmp ne <3 x i64> %src2, zeroinitializer
diff --git a/llvm/test/CodeGen/X86/widen_load-2.ll b/llvm/test/CodeGen/X86/widen_load-2.ll
index c9531f2c9ced3c2b63b7ac68146c092725f016bb..a14736bd683747a18dc6466d558f51124f9e62d4 100644
--- a/llvm/test/CodeGen/X86/widen_load-2.ll
+++ b/llvm/test/CodeGen/X86/widen_load-2.ll
@@ -359,7 +359,7 @@ define void @rot(%i8vec3pack* nocapture sret(%i8vec3pack) %result, %i8vec3pack*
 ; X86-NEXT:    movw $257, (%ecx) # imm = 0x101
 ; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-NEXT:    psrlw $1, %xmm0
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    pextrb $2, %xmm0, 2(%eax)
 ; X86-NEXT:    pextrw $0, %xmm0, (%eax)
 ; X86-NEXT:    retl $4
diff --git a/llvm/test/CodeGen/X86/x86-repmov-copy-eflags.ll b/llvm/test/CodeGen/X86/x86-repmov-copy-eflags.ll
index 91bce1610c8b818179e3423b8006578283728365..110691504655b493960cbb9583513d560f943989 100644
--- a/llvm/test/CodeGen/X86/x86-repmov-copy-eflags.ll
+++ b/llvm/test/CodeGen/X86/x86-repmov-copy-eflags.ll
@@ -5,7 +5,7 @@ target triple = "i686-pc-windows-msvc18.0.0"
 %struct.T = type { i64, [3 x i32] }
 
 ; Function Attrs: nounwind optsize
-define void @f(i8* %p, i8* %q, i32* inalloca nocapture %unused) #0 {
+define void @f(i8* %p, i8* %q, i32* inalloca(i32) nocapture %unused) #0 {
 entry:
   %g = alloca %struct.T, align 8
   %r = alloca i32, align 8
@@ -25,7 +25,7 @@ while.end:                                        ; preds = %while.body
   ret void
 }
 
-define void @f_pgso(i8* %p, i8* %q, i32* inalloca nocapture %unused) !prof !14 {
+define void @f_pgso(i8* %p, i8* %q, i32* inalloca(i32) nocapture %unused) !prof !14 {
 entry:
   %g = alloca %struct.T, align 8
   %r = alloca i32, align 8
diff --git a/llvm/test/CodeGen/X86/x86-shifts.ll b/llvm/test/CodeGen/X86/x86-shifts.ll
index 6a7089967cf6213421151d9560a83b502e314184..3063666dabeb2390f630d4c7d69d81a54da575e4 100644
--- a/llvm/test/CodeGen/X86/x86-shifts.ll
+++ b/llvm/test/CodeGen/X86/x86-shifts.ll
@@ -131,7 +131,7 @@ define <8 x i16> @sll8_nosplat(<8 x i16> %A) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movdqa {{.*#+}} xmm1 = [2,4,8,64,4,4,4,4]
 ; X86-NEXT:    pmullw %xmm0, %xmm1
-; X86-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pmullw {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    pxor %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
@@ -204,7 +204,7 @@ define <16 x i8> @shl9(<16 x i8> %A) nounwind {
 ; X86-LABEL: shl9:
 ; X86:       # %bb.0:
 ; X86-NEXT:    psllw $3, %xmm0
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: shl9:
@@ -220,7 +220,7 @@ define <16 x i8> @shr9(<16 x i8> %A) nounwind {
 ; X86-LABEL: shr9:
 ; X86:       # %bb.0:
 ; X86-NEXT:    psrlw $3, %xmm0
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: shr9:
@@ -247,7 +247,7 @@ define <16 x i8> @sra_v16i8(<16 x i8> %A) nounwind {
 ; X86-LABEL: sra_v16i8:
 ; X86:       # %bb.0:
 ; X86-NEXT:    psrlw $3, %xmm0
-; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pand {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
 ; X86-NEXT:    pxor %xmm1, %xmm0
 ; X86-NEXT:    psubb %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/xop-mask-comments.ll b/llvm/test/CodeGen/X86/xop-mask-comments.ll
index 3e5bb351c5d118cb3c998bec180337975e56f98a..418a6479810d00c558b69506068e4b580e7148df 100644
--- a/llvm/test/CodeGen/X86/xop-mask-comments.ll
+++ b/llvm/test/CodeGen/X86/xop-mask-comments.ll
@@ -55,7 +55,7 @@ define <16 x i8> @vpperm_shuffle_binary_zero(<16 x i8> %a0, <16 x i8> %a1) {
 define <16 x i8> @vpperm_shuffle_general(<16 x i8> %a0, <16 x i8> %a1) {
 ; X86-LABEL: vpperm_shuffle_general:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpperm {{\.LCPI.*}}, %xmm0, %xmm0, %xmm0
+; X86-NEXT:    vpperm {{\.LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: vpperm_shuffle_general:
diff --git a/llvm/test/CodeGen/X86/xor.ll b/llvm/test/CodeGen/X86/xor.ll
index 8867a4d8b54fd9a1d24b0f5a34816b393dcd3c2f..9b77c56c3a81bbe45463a04d67ccddc95a980637 100644
--- a/llvm/test/CodeGen/X86/xor.ll
+++ b/llvm/test/CodeGen/X86/xor.ll
@@ -373,7 +373,7 @@ define i32 @test9(i32 %a) nounwind {
 define <4 x i32> @test10(<4 x i32> %a) nounwind {
 ; X86-LABEL: test10:
 ; X86:       # %bb.0:
-; X86-NEXT:    andnps {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    andnps {{\.LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LIN-LABEL: test10:
diff --git a/llvm/test/DebugInfo/Generic/2010-05-03-OriginDIE.ll b/llvm/test/DebugInfo/Generic/2010-05-03-OriginDIE.ll
index c72cffde5282ba0bb12de0e2b1c0c418dfab9fc7..7619297d2b29dac50578ae70247fc56dcb542591 100644
--- a/llvm/test/DebugInfo/Generic/2010-05-03-OriginDIE.ll
+++ b/llvm/test/DebugInfo/Generic/2010-05-03-OriginDIE.ll
@@ -2,6 +2,9 @@
 ;RUN: llc < %s -o /dev/null
 ;Radar 7937109
 
+;XCore default subtarget does not support 8-byte alignment on stack.
+;XFAIL: xcore
+
 %struct.anon = type { i64, i32, i32, i32, [1 x i32] }
 %struct.gpm_t = type { i32, i8*, [16 x i8], i32, i64, i64, i64, i64, i64, i64, i32, i16, i16, [8 x %struct.gpmr_t] }
 %struct.gpmr_t = type { [48 x i8], [48 x i8], [16 x i8], i64, i64, i64, i64, i16 }
diff --git a/llvm/test/DebugInfo/Symbolize/ELF/aarch64-mapping-symbol.s b/llvm/test/DebugInfo/Symbolize/ELF/aarch64-mapping-symbol.s
new file mode 100644
index 0000000000000000000000000000000000000000..2f1140cf629104b08924e281a61d0a01ecfc56ce
--- /dev/null
+++ b/llvm/test/DebugInfo/Symbolize/ELF/aarch64-mapping-symbol.s
@@ -0,0 +1,31 @@
+# REQUIRES: aarch64-registered-target
+## Ignore AArch64 mapping symbols (with a prefix of $d or $x).
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t
+
+## Verify that mapping symbols are actually present in the object at expected
+## addresses.
+# RUN: llvm-nm --special-syms %t | FileCheck %s -check-prefix MAPPING_SYM
+
+# MAPPING_SYM:      0000000000000000 t $d.0
+# MAPPING_SYM-NEXT: 000000000000000c t $d.2
+# MAPPING_SYM-NEXT: 0000000000000004 t $x.1
+# MAPPING_SYM-NEXT: 0000000000000000 T foo
+
+# RUN: llvm-symbolizer --obj=%t 0 4 0xc | FileCheck %s -check-prefix SYMBOL
+
+# SYMBOL:      foo
+# SYMBOL-NEXT: ??:0:0
+# SYMBOL-EMPTY:
+# SYMBOL:      foo
+# SYMBOL-NEXT: ??:0:0
+# SYMBOL-EMPTY:
+# SYMBOL:      foo
+# SYMBOL-NEXT: ??:0:0
+
+    .global foo
+foo:
+    .word 32
+    nop
+    nop
+    .word 42
diff --git a/llvm/test/DebugInfo/Symbolize/ELF/arm-mapping-symbol.s b/llvm/test/DebugInfo/Symbolize/ELF/arm-mapping-symbol.s
index 810ae4f5b8bd91d02c4af086a63840aedf48e820..6e17ef29ae577bd57bad289d909ea26266ee310d 100644
--- a/llvm/test/DebugInfo/Symbolize/ELF/arm-mapping-symbol.s
+++ b/llvm/test/DebugInfo/Symbolize/ELF/arm-mapping-symbol.s
@@ -2,15 +2,32 @@
 ## Ignore ARM mapping symbols (with a prefix of $a, $d or $t).
 
 # RUN: llvm-mc -filetype=obj -triple=armv7-none-linux %s -o %t
-# RUN: llvm-symbolizer --obj=%t 4 8 | FileCheck %s
+
+## Verify that mapping symbols are actually present in the object at expected
+## addresses.
+# RUN: llvm-nm --special-syms %t | FileCheck %s -check-prefix MAPPING_A
+
+# MAPPING_A:      00000004 t $a.1
+# MAPPING_A-NEXT: 00000000 t $d.0
+# MAPPING_A-NEXT: 00000008 t $d.2
+# MAPPING_A-NEXT: 00000000 T foo
+
 # RUN: llvm-mc -filetype=obj -triple=thumbv7-none-linux %s -o %tthumb
-# RUN: llvm-symbolizer --obj=%tthumb 4 8 | FileCheck %s
+# RUN: llvm-nm --special-syms %tthumb | FileCheck %s -check-prefix MAPPING_T
+
+# MAPPING_T:      00000000 t $d.0
+# MAPPING_T-NEXT: 00000006 t $d.2
+# MAPPING_T-NEXT: 00000004 t $t.1
+# MAPPING_T-NEXT: 00000000 T foo
+
+# RUN: llvm-symbolizer --obj=%t 4 8 | FileCheck %s -check-prefix SYMBOL
+# RUN: llvm-symbolizer --obj=%tthumb 4 8 | FileCheck %s -check-prefix SYMBOL
 
-# CHECK:       foo
-# CHECK-NEXT:  ??:0:0
-# CHECK-EMPTY:
-# CHECK-NEXT:  foo
-# CHECK-NEXT:  ??:0:0
+# SYMBOL:       foo
+# SYMBOL-NEXT:  ??:0:0
+# SYMBOL-EMPTY:
+# SYMBOL-NEXT:  foo
+# SYMBOL-NEXT:  ??:0:0
 
 .globl foo
 foo:
diff --git a/llvm/test/DebugInfo/Symbolize/ELF/riscv-empty-name-symbol.s b/llvm/test/DebugInfo/Symbolize/ELF/riscv-empty-name-symbol.s
new file mode 100644
index 0000000000000000000000000000000000000000..1e0fa8a3061830cad60f06edf716a5cda98759c5
--- /dev/null
+++ b/llvm/test/DebugInfo/Symbolize/ELF/riscv-empty-name-symbol.s
@@ -0,0 +1,26 @@
+# REQUIRES: riscv-registered-target
+## Ignore empty name symbols.
+
+# RUN: llvm-mc -filetype=obj -triple=riscv64 %s -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYM
+
+# SYM: 0000000000000004  0 NOTYPE LOCAL  DEFAULT [[#]] {{$}}
+# SYM: 0000000000000000  0 NOTYPE GLOBAL DEFAULT [[#]] foo
+
+## Make sure we test at an address larger than or equal to an empty name symbol.
+# RUN: llvm-symbolizer --obj=%t 0 4 | FileCheck %s
+
+# CHECK:       foo
+# CHECK-NEXT:  ??:0:0
+# CHECK-EMPTY:
+# CHECK-NEXT:  foo
+# CHECK-NEXT:  ??:0:0
+
+.globl foo
+foo:
+  nop
+  .file 1 "/tmp" "a.s"
+  .loc 1 1 0
+  nop
+
+.section .debug_line,"",@progbits
diff --git a/llvm/test/DebugInfo/X86/dbg-declare-inalloca.ll b/llvm/test/DebugInfo/X86/dbg-declare-inalloca.ll
index ce5e583f911543004b253a8e122f88e04b256397..d6920a7f9a59f66f501487aed9f192bc0041fd15 100644
--- a/llvm/test/DebugInfo/X86/dbg-declare-inalloca.ll
+++ b/llvm/test/DebugInfo/X86/dbg-declare-inalloca.ll
@@ -109,7 +109,7 @@ target triple = "i386-pc-windows-msvc19.10.24728"
 %struct.NonTrivial = type { i32 }
 
 ; Function Attrs: nounwind
-define void @f(<{ %struct.NonTrivial, i32, i32, i32 }>* inalloca) local_unnamed_addr #0 !dbg !7 {
+define void @f(<{ %struct.NonTrivial, i32, i32, i32 }>* inalloca(<{ %struct.NonTrivial, i32, i32, i32 }>)) local_unnamed_addr #0 !dbg !7 {
 entry:
   %a = getelementptr inbounds <{ %struct.NonTrivial, i32, i32, i32 }>, <{ %struct.NonTrivial, i32, i32, i32 }>* %0, i32 0, i32 0
   %b = getelementptr inbounds <{ %struct.NonTrivial, i32, i32, i32 }>, <{ %struct.NonTrivial, i32, i32, i32 }>* %0, i32 0, i32 1
diff --git a/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1.c b/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1.c
new file mode 100644
index 0000000000000000000000000000000000000000..3a5c2bcefee14568b0abaf72bb6e9d2716b49d1b
--- /dev/null
+++ b/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1.c
@@ -0,0 +1,2 @@
+int sub1(int x) { return x - 1; }
+int main(int argc, char **argv) { return sub1(argc); }
diff --git a/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll b/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll
new file mode 100644
index 0000000000000000000000000000000000000000..659dbe109ec691a6ce77deec87924ef705c6ff9c
--- /dev/null
+++ b/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll
@@ -0,0 +1,52 @@
+; ModuleID = 'argc_sub1.c'
+target triple = "x86_64-unknown-unknown-elf"
+
+define i32 @sub1(i32) !dbg !8 {
+  call void @llvm.dbg.value(metadata i32 %0, metadata !13, metadata !DIExpression()), !dbg !14
+  %2 = add nsw i32 %0, -1, !dbg !15
+  ret i32 %2, !dbg !16
+}
+
+define i32 @main(i32, i8** nocapture readnone) !dbg !17 {
+  call void @llvm.dbg.value(metadata i32 %0, metadata !24, metadata !DIExpression()), !dbg !26
+  call void @llvm.dbg.value(metadata i8** %1, metadata !25, metadata !DIExpression()), !dbg !27
+  %3 = tail call i32 @sub1(i32 %0), !dbg !28
+  ret i32 %3, !dbg !29
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.1-8+deb10u2 (tags/RELEASE_701/final)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "argc_sub1.c", directory: "Inputs/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 7.0.1-8+deb10u2 (tags/RELEASE_701/final)"}
+!8 = distinct !DISubprogram(name: "sub1", scope: !1, file: !1, line: 1, type: !9, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !{!13}
+!13 = !DILocalVariable(name: "x", arg: 1, scope: !8, file: !1, line: 1, type: !11)
+!14 = !DILocation(line: 1, column: 14, scope: !8)
+!15 = !DILocation(line: 1, column: 28, scope: !8)
+!16 = !DILocation(line: 1, column: 19, scope: !8)
+!17 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 2, type: !18, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !23)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!11, !11, !20}
+!20 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !21, size: 64)
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !22, size: 64)
+!22 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!23 = !{!24, !25}
+!24 = !DILocalVariable(name: "argc", arg: 1, scope: !17, file: !1, line: 2, type: !11)
+!25 = !DILocalVariable(name: "argv", arg: 2, scope: !17, file: !1, line: 2, type: !20)
+!26 = !DILocation(line: 2, column: 14, scope: !17)
+!27 = !DILocation(line: 2, column: 27, scope: !17)
+!28 = !DILocation(line: 2, column: 42, scope: !17)
+!29 = !DILocation(line: 2, column: 35, scope: !17)
diff --git a/llvm/test/Examples/OrcV2Examples/lljit-with-remote-debugging.test b/llvm/test/Examples/OrcV2Examples/lljit-with-remote-debugging.test
new file mode 100644
index 0000000000000000000000000000000000000000..a9c3946b55e2cff2230af0ab92556fcfb145eb96
--- /dev/null
+++ b/llvm/test/Examples/OrcV2Examples/lljit-with-remote-debugging.test
@@ -0,0 +1,10 @@
+# This test makes sure that the example builds and executes as expected.
+# Instructions for debugging can be found in LLJITWithRemoteDebugging.cpp
+
+# RUN: LLJITWithRemoteDebugging %p/Inputs/argc_sub1_elf.ll
+# CHECK: Running: {{.*}}/Inputs/argc_sub1_elf.ll
+# CHECK: Exit code: 0
+
+# RUN: LLJITWithRemoteDebugging %p/Inputs/argc_sub1_elf.ll --args 2nd 3rd 4th
+# CHECK: Running: {{.*}}/Inputs/argc_sub1_elf.ll 2nd 3rd 4th
+# CHECK: Exit code: 3
diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_relocations.s
index 8cdc91500005562bf223756fc44c44ee593bdc1c..165e53e40cf3d52f45b4390f830a7c1fafcfeb7b 100644
--- a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_relocations.s
+++ b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_relocations.s
@@ -147,13 +147,15 @@ Lanon_data:
 # anonymous.
 #
 # Note: +8 offset in expression below to accounts for sizeof(Lanon_data).
-# jitlink-check: *{8}(section_addr(macho_reloc.o, __data) + 8) = (section_addr(macho_reloc.o, __data) + 8) - named_data + 2
+# jitlink-check: *{8}(section_addr(macho_reloc.o, __DATA,__data) + 8) = \
+# jitlink-check:     (section_addr(macho_reloc.o, __DATA,__data) + 8) - named_data + 2
         .p2align  3
 Lanon_minuend_quad:
         .quad Lanon_minuend_quad - named_data + 2
 
 # Note: +16 offset in expression below to accounts for sizeof(Lanon_data) + sizeof(Lanon_minuend_long).
-# jitlink-check: *{4}(section_addr(macho_reloc.o, __data) + 16) = ((section_addr(macho_reloc.o, __data) + 16) - named_data + 2)[31:0]
+# jitlink-check: *{4}(section_addr(macho_reloc.o, __DATA,__data) + 16) = \
+# jitlink-check:     ((section_addr(macho_reloc.o, __DATA,__data) + 16) - named_data + 2)[31:0]
         .p2align  2
 Lanon_minuend_long:
         .long Lanon_minuend_long - named_data + 2
@@ -185,7 +187,8 @@ named_func_addr_quad:
 # Check ARM64_RELOC_UNSIGNED / quad / non-extern handling by putting the
 # address of a local anonymous function into a quad symbol.
 #
-# jitlink-check: *{8}anon_func_addr_quad = section_addr(macho_reloc.o, __text)
+# jitlink-check: *{8}anon_func_addr_quad = \
+# jitlink-check:     section_addr(macho_reloc.o, __TEXT,__text)
         .globl  anon_func_addr_quad
         .p2align  3
 anon_func_addr_quad:
@@ -193,7 +196,8 @@ anon_func_addr_quad:
 
 # ARM64_RELOC_SUBTRACTOR Quad/Long in named storage with anonymous minuend
 #
-# jitlink-check: *{8}anon_minuend_quad1 = section_addr(macho_reloc.o, __data) - anon_minuend_quad1 + 2
+# jitlink-check: *{8}anon_minuend_quad1 = \
+# jitlink-check:     section_addr(macho_reloc.o, __DATA,__data) - anon_minuend_quad1 + 2
 # Only the form "B: .quad LA - B + C" is tested. The form "B: .quad B - LA + C" is
 # invalid because the subtrahend can not be local.
         .globl  anon_minuend_quad1
@@ -201,7 +205,8 @@ anon_func_addr_quad:
 anon_minuend_quad1:
         .quad Lanon_data - anon_minuend_quad1 + 2
 
-# jitlink-check: *{4}anon_minuend_long1 = (section_addr(macho_reloc.o, __data) - anon_minuend_long1 + 2)[31:0]
+# jitlink-check: *{4}anon_minuend_long1 = \
+# jitlink-check:     (section_addr(macho_reloc.o, __DATA,__data) - anon_minuend_long1 + 2)[31:0]
         .globl  anon_minuend_long1
         .p2align  2
 anon_minuend_long1:
@@ -308,14 +313,14 @@ test_got:
 # ORC responsibility set, which is automatically marked live and would couse
 # spurious passes.
 #
-# jitlink-check: *{8}section_addr(macho_reloc.o, __nds_test_sect) = 0
+# jitlink-check: *{8}section_addr(macho_reloc.o, __DATA,__nds_test_sect) = 0
         .section        __DATA,__nds_test_sect,regular,no_dead_strip
         .quad 0
 
 # Check that unreferenced local symbols that have been marked no-dead-strip are
 # not dead-striped.
 #
-# jitlink-check: *{8}section_addr(macho_reloc.o, __nds_test_nlst) = 0
+# jitlink-check: *{8}section_addr(macho_reloc.o, __DATA,__nds_test_nlst) = 0
         .section       __DATA,__nds_test_nlst,regular
         .no_dead_strip no_dead_strip_test_symbol
 no_dead_strip_test_symbol:
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_large_pic_relocations.s b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_large_pic_relocations.s
new file mode 100644
index 0000000000000000000000000000000000000000..b70debd8565ba2133c8e889a73e2639e0f75b90c
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_large_pic_relocations.s
@@ -0,0 +1,58 @@
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj \
+# RUN:         -large-code-model -o %t/elf_lg_pic_reloc.o %s
+# RUN: llvm-jitlink -noexec -slab-allocate 100Kb -slab-address 0xfff00000 \
+# RUN:              -check %s %t/elf_lg_pic_reloc.o
+#
+# Test ELF large/PIC relocations.
+
+        .text
+        .file   "testcase.c"
+
+        # Empty main entry point.
+        .globl  main
+        .p2align        4, 0x90
+        .type   main,@function
+main:
+        retq
+
+        .size   main, .-main
+
+# Test R_X86_64_GOTPC64 handling. We want to check that the offset of the
+# operand is the 64-bit delta to the start of the GOT.
+# jitlink-check: decode_operand(test_gotpc64, 1) = \
+# jitlink-check:   _GLOBAL_OFFSET_TABLE_ - test_lg_pic_GOT
+# jitlink-check: decode_operand(test_got64, 1) = \
+# jitlink-check:   got_addr(elf_lg_pic_reloc.o, named_data) - \
+# jitlink-check:     _GLOBAL_OFFSET_TABLE_
+        .globl test_lg_pic_GOT
+        .p2align    4, 0x90
+        .type   test_lg_pic_GOT,@function
+test_lg_pic_GOT:
+.L0$pb:
+        leaq    .L0$pb(%rip), %rax
+
+        .globl test_gotpc64
+test_gotpc64:
+        movabsq $_GLOBAL_OFFSET_TABLE_-.L0$pb, %rcx
+        .size   test_gotpc64, .-test_gotpc64
+
+        addq    %rax, %rcx
+        .globl test_got64
+test_got64:
+        movabsq $named_data@GOT, %rax
+        .size   test_got64, .-test_got64
+
+        .size   test_lg_pic_GOT, .-test_lg_pic_GOT
+
+        .data
+
+        .type   named_data,@object
+        .p2align        3
+named_data:
+        .quad   42
+        .size   named_data, 8
+
+        .ident  "clang version 10.0.0-4ubuntu1 "
+        .section        ".note.GNU-stack","",@progbits
+        .addrsig
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_small_pic_relocations.s
similarity index 78%
rename from llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s
rename to llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_small_pic_relocations.s
index c2039bd64264f42edfcfbe09043324922d9566c0..147f2b48bbb7e01044bb4f60fd041d1be31b3c4d 100644
--- a/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s
+++ b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_small_pic_relocations.s
@@ -1,12 +1,13 @@
 # RUN: rm -rf %t && mkdir -p %t
-# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj -o %t/elf_reloc.o %s
+# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj \
+# RUN:         -o %t/elf_sm_pic_reloc.o %s
 # RUN: llvm-jitlink -noexec -slab-allocate 100Kb -slab-address 0xfff00000 \
 # RUN:              -define-abs external_data=0x1 \
 # RUN:              -define-abs extern_in_range32=0xffe00000 \
 # RUN:              -define-abs extern_out_of_range32=0x7fff00000000 \
-# RUN:              -check %s %t/elf_reloc.o
+# RUN:              -check %s %t/elf_sm_pic_reloc.o
 #
-# Test standard ELF relocations.
+# Test ELF small/PIC relocations.
 
         .text
         .file   "testcase.c"
@@ -55,7 +56,8 @@ test_call_local:
 # resolution, the target turns out to be in-range from the callsite and so the
 # edge is relaxed in post-allocation optimization.
 #
-# jitlink-check: decode_operand(test_call_extern, 0) = extern_in_range32 - next_pc(test_call_extern)
+# jitlink-check: decode_operand(test_call_extern, 0) = \
+# jitlink-check:     extern_in_range32 - next_pc(test_call_extern)
         .globl  test_call_extern
         .p2align       4, 0x90
         .type   test_call_extern,@function
@@ -70,8 +72,10 @@ test_call_extern:
 # entry.
 #
 # jitlink-check: decode_operand(test_call_extern_plt, 0) = \
-# jitlink-check:     stub_addr(elf_reloc.o, extern_out_of_range32) - next_pc(test_call_extern_plt)
-# jitlink-check: *{8}(got_addr(elf_reloc.o, extern_out_of_range32)) = extern_out_of_range32
+# jitlink-check:     stub_addr(elf_sm_pic_reloc.o, extern_out_of_range32) - \
+# jitlink-check:        next_pc(test_call_extern_plt)
+# jitlink-check: *{8}(got_addr(elf_sm_pic_reloc.o, extern_out_of_range32)) = \
+# jitlink-check:     extern_out_of_range32
         .globl  test_call_extern_plt
         .p2align       4, 0x90
         .type   test_call_extern_plt,@function
@@ -82,8 +86,9 @@ test_call_extern_plt:
 
 # Test GOTPCREL handling. We want to check both the offset to the GOT entry and its
 # contents.
-# jitlink-check: decode_operand(test_gotpcrel, 4) = got_addr(elf_reloc.o, named_data) - next_pc(test_gotpcrel)
-# jitlink-check: *{8}(got_addr(elf_reloc.o, named_data)) = named_data
+# jitlink-check: decode_operand(test_gotpcrel, 4) = \
+# jitlink-check:     got_addr(elf_sm_pic_reloc.o, named_data) - next_pc(test_gotpcrel)
+# jitlink-check: *{8}(got_addr(elf_sm_pic_reloc.o, named_data)) = named_data
 
         .globl test_gotpcrel
         .p2align      4, 0x90
@@ -96,7 +101,7 @@ test_gotpcrel:
 # Test REX_GOTPCRELX handling. We want to check both the offset to the GOT entry and its
 # contents.
 # jitlink-check: decode_operand(test_rex_gotpcrelx, 4) = \
-# jitlink-check:   got_addr(elf_reloc.o, external_data) - next_pc(test_rex_gotpcrelx)
+# jitlink-check:   got_addr(elf_sm_pic_reloc.o, external_data) - next_pc(test_rex_gotpcrelx)
 
         .globl test_rex_gotpcrelx
         .p2align      4, 0x90
@@ -106,6 +111,16 @@ test_rex_gotpcrelx:
 
         .size   test_rex_gotpcrelx, .-test_rex_gotpcrelx
 
+# Test GOTOFF64 handling.
+# jitlink-check: decode_operand(test_gotoff64, 1) = named_func - _GLOBAL_OFFSET_TABLE_
+        .globl test_gotoff64
+        .p2align     4, 0x90
+        .type  test_gotoff64,@function
+test_gotoff64:
+        movabsq $named_func@GOTOFF, %rax
+
+        .size   test_gotoff64, .-test_gotoff64
+
 # Test that relocations to anonymous constant pool entries work.
         .globl  test_anchor_LCPI
         .p2align        4, 0x90
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/MachO_x86-64_relocations.s b/llvm/test/ExecutionEngine/JITLink/X86/MachO_x86-64_relocations.s
index dd3cc455bd349410bc45f29f4413613bfd5cad55..3e823364294060c7d8ea96481069c3b0043dc8a2 100644
--- a/llvm/test/ExecutionEngine/JITLink/X86/MachO_x86-64_relocations.s
+++ b/llvm/test/ExecutionEngine/JITLink/X86/MachO_x86-64_relocations.s
@@ -106,22 +106,26 @@ signed4:
         movl $0xAAAAAAAA, named_data(%rip)
 
         .globl signedanon
-# jitlink-check: decode_operand(signedanon, 4) = section_addr(macho_reloc.o, __data) - next_pc(signedanon)
+# jitlink-check: decode_operand(signedanon, 4) = \
+# jitlink-check:     section_addr(macho_reloc.o, __DATA,__data) - next_pc(signedanon)
 signedanon:
         movq Lanon_data(%rip), %rax
 
         .globl signed1anon
-# jitlink-check: decode_operand(signed1anon, 3) = section_addr(macho_reloc.o, __data) - next_pc(signed1anon)
+# jitlink-check: decode_operand(signed1anon, 3) = \
+# jitlink-check:     section_addr(macho_reloc.o, __DATA,__data) - next_pc(signed1anon)
 signed1anon:
         movb $0xAA, Lanon_data(%rip)
 
         .globl signed2anon
-# jitlink-check: decode_operand(signed2anon, 3) = section_addr(macho_reloc.o, __data) - next_pc(signed2anon)
+# jitlink-check: decode_operand(signed2anon, 3) = \
+# jitlink-check:     section_addr(macho_reloc.o, __DATA,__data) - next_pc(signed2anon)
 signed2anon:
         movw $0xAAAA, Lanon_data(%rip)
 
         .globl signed4anon
-# jitlink-check: decode_operand(signed4anon, 3) = section_addr(macho_reloc.o, __data) - next_pc(signed4anon)
+# jitlink-check: decode_operand(signed4anon, 3) = \
+# jitlink-check:     section_addr(macho_reloc.o, __DATA,__data) - next_pc(signed4anon)
 signed4anon:
         movl $0xAAAAAAAA, Lanon_data(%rip)
 
@@ -140,13 +144,15 @@ Lanon_data:
 # anonymous.
 #
 # Note: +8 offset in expression below to accounts for sizeof(Lanon_data).
-# jitlink-check: *{8}(section_addr(macho_reloc.o, __data) + 8) = (section_addr(macho_reloc.o, __data) + 8) - named_data - 2
+# jitlink-check: *{8}(section_addr(macho_reloc.o, __DATA,__data) + 8) = \
+# jitlink-check:     (section_addr(macho_reloc.o, __DATA,__data) + 8) - named_data - 2
         .p2align  3
 Lanon_minuend_quad:
         .quad Lanon_minuend_quad - named_data - 2
 
 # Note: +16 offset in expression below to accounts for sizeof(Lanon_data) + sizeof(Lanon_minuend_long).
-# jitlink-check: *{4}(section_addr(macho_reloc.o, __data) + 16) = ((section_addr(macho_reloc.o, __data) + 16) - named_data - 2)[31:0]
+# jitlink-check: *{4}(section_addr(macho_reloc.o, __DATA,__data) + 16) = \
+# jitlink-check:     ((section_addr(macho_reloc.o, __DATA,__data) + 16) - named_data - 2)[31:0]
         .p2align  2
 Lanon_minuend_long:
         .long Lanon_minuend_long - named_data - 2
@@ -185,7 +191,7 @@ named_func_addr_long:
 # Check X86_64_RELOC_UNSIGNED / quad / non-extern handling by putting the
 # address of a local anonymous function into a quad symbol.
 #
-# jitlink-check: *{8}anon_func_addr_quad = section_addr(macho_reloc.o, __text)
+# jitlink-check: *{8}anon_func_addr_quad = section_addr(macho_reloc.o, __TEXT,__text)
         .globl  anon_func_addr_quad
         .p2align  3
 anon_func_addr_quad:
@@ -193,7 +199,8 @@ anon_func_addr_quad:
 
 # X86_64_RELOC_SUBTRACTOR Quad/Long in named storage with anonymous minuend
 #
-# jitlink-check: *{8}anon_minuend_quad1 = section_addr(macho_reloc.o, __data) - anon_minuend_quad1 - 2
+# jitlink-check: *{8}anon_minuend_quad1 = \
+# jitlink-check:     section_addr(macho_reloc.o, __DATA,__data) - anon_minuend_quad1 - 2
 # Only the form "B: .quad LA - B + C" is tested. The form "B: .quad B - LA + C" is
 # invalid because the subtrahend can not be local.
         .globl  anon_minuend_quad1
@@ -201,7 +208,8 @@ anon_func_addr_quad:
 anon_minuend_quad1:
         .quad Lanon_data - anon_minuend_quad1 - 2
 
-# jitlink-check: *{4}anon_minuend_long1 = (section_addr(macho_reloc.o, __data) - anon_minuend_long1 - 2)[31:0]
+# jitlink-check: *{4}anon_minuend_long1 = \
+# jitlink-check:     (section_addr(macho_reloc.o, __DATA,__data) - anon_minuend_long1 - 2)[31:0]
         .globl  anon_minuend_long1
         .p2align  2
 anon_minuend_long1:
@@ -311,14 +319,14 @@ test_got:
 # ORC responsibility set, which is automatically marked live and would couse
 # spurious passes.
 #
-# jitlink-check: *{8}section_addr(macho_reloc.o, __nds_test_sect) = 0
+# jitlink-check: *{8}section_addr(macho_reloc.o, __DATA,__nds_test_sect) = 0
         .section        __DATA,__nds_test_sect,regular,no_dead_strip
         .quad 0
 
 # Check that unreferenced local symbols that have been marked no-dead-strip are
 # not dead-striped.
 #
-# jitlink-check: *{8}section_addr(macho_reloc.o, __nds_test_nlst) = 0
+# jitlink-check: *{8}section_addr(macho_reloc.o, __DATA,__nds_test_nlst) = 0
         .section       __DATA,__nds_test_nlst,regular
         .no_dead_strip no_dead_strip_test_symbol
 no_dead_strip_test_symbol:
diff --git a/llvm/test/FileCheck/dump-input/annotations.txt b/llvm/test/FileCheck/dump-input/annotations.txt
index 0c911906ebb93a5ba194f688cc4019f2a24049d4..3118ef67e4d6065f7aee3db062178919f313a95a 100644
--- a/llvm/test/FileCheck/dump-input/annotations.txt
+++ b/llvm/test/FileCheck/dump-input/annotations.txt
@@ -504,52 +504,104 @@
 ;--------------------------------------------------
 ; CHECK-LABEL
 ;
-; FIXME: Labels sometimes produce redundant diagnostics for good matches.
-; That bug is independent of but affects -dump-input.
+; Each CHECK-LABEL is processed twice: once before other patterns in the
+; preceding section, and once afterward.
+;
+; As expected, the search range for a negative pattern preceding a CHECK-LABEL
+; ends at the start of the CHECK-LABEL match.  not:7 and not:11 below
+; demonstrate this behavior.
+;
+; The search range for a positive pattern preceding a CHECK-LABEL ends at the
+; end of the CHECK-LABEL match.  check:3 and check:5 below demonstrate this
+; behavior.  As in the case of check:5, an effect of this behavior is that the
+; second CHECK-LABEL match might fail even though the first succeeded.
+;
+; FIXME: It seems like the search range for such a positive pattern should be
+; the same as in the case of a negative pattern.  Note that -dump-input is
+; correct here.  It's the matching behavior that's strange.
 ;--------------------------------------------------
 
-; Good match and no match.
-
-; RUN: echo 'lab0' > %t.in
-; RUN: echo 'foo' >> %t.in
-; RUN: echo 'lab1' >> %t.in
-; RUN: echo 'bar' >> %t.in
-
-; RUN: echo 'CHECK-LABEL: lab0' > %t.chk
-; RUN: echo 'CHECK: foo' >> %t.chk
-; RUN: echo 'CHECK-LABEL: lab2' >> %t.chk
+; RUN: echo 'text'   >  %t.in
+; RUN: echo 'labelA' >> %t.in
+; RUN: echo 'textA'  >> %t.in
+; RUN: echo 'labelB' >> %t.in
+; RUN: echo 'textB'  >> %t.in
+; RUN: echo 'labelC' >> %t.in
+; RUN: echo 'textC'  >> %t.in
+; RUN: echo 'labelD' >> %t.in
+; RUN: echo 'textD'  >> %t.in
+; RUN: echo 'labelE' >> %t.in
+; RUN: echo 'textE'  >> %t.in
+; RUN: echo 'labelF' >> %t.in
+
+; RUN: echo 'CHECK: text'         >  %t.chk
+; RUN: echo 'CHECK-LABEL: labelA' >> %t.chk
+; RUN: echo 'CHECK: foobar'       >> %t.chk
+; RUN: echo 'CHECK-LABEL: labelB' >> %t.chk
+; RUN: echo 'CHECK: labelC'       >> %t.chk
+; RUN: echo 'CHECK-LABEL: labelC' >> %t.chk
+; RUN: echo 'CHECK-NOT: foobar'   >> %t.chk
+; RUN: echo 'CHECK-LABEL: labelD' >> %t.chk
+; RUN: echo 'CHECK-NOT: textD'    >> %t.chk
+; RUN: echo 'CHECK-LABEL: labelE' >> %t.chk
+; RUN: echo 'CHECK-NOT: labelF'   >> %t.chk
+; RUN: echo 'CHECK-LABEL: labelF' >> %t.chk
 
 ; RUN: %ProtectFileCheckOutput \
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB \
-; RUN:             -implicit-check-not='remark:'
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB,LAB-Q \
+; RUN:             -implicit-check-not='{{remark:|error:}}'
 ; RUN: %ProtectFileCheckOutput \
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
 ; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB,LAB-V \
-; RUN:             -implicit-check-not='remark:'
+; RUN:             -implicit-check-not='{{remark:|error:}}'
 ; RUN: %ProtectFileCheckOutput \
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
 ; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB,LAB-V,LAB-VV \
-; RUN:             -implicit-check-not='remark:' -allow-unused-prefixes
+; RUN:             -implicit-check-not='{{remark:|error:}}'
 
 ; Verbose diagnostics are suppressed but not errors.
-; LAB: {{.*}}error:{{.*}}
-; LAB: {{.*}}possible intended match{{.*}}
-
-; LAB:         <<<<<<
-; LAB-NEXT:               1: lab0 
-; LAB-V-NEXT:  label:1'0     ^~~~
-; LAB-V-NEXT:  label:1'1     ^~~~
-; LAB-NEXT:    label:3'0         X error: no match found
-; LAB-NEXT:               2: foo 
-; LAB-NEXT:    label:3'0     ~~~~
-; LAB-NEXT:               3: lab1 
-; LAB-NEXT:    label:3'0     ~~~~~
-; LAB-NEXT:    label:3'1     ?     possible intended match
-; LAB-NEXT:               4: bar 
-; LAB-NEXT:    label:3'0     ~~~~
-; LAB-NEXT:    >>>>>>
-; LAB-NOT:     {{.}}
+; LAB:{{.*}}.chk:3:8: error: CHECK: expected string not found in input
+; LAB:{{.*}}.chk:6:14: error: CHECK-LABEL: expected string not found in input
+; LAB:{{.*}}.chk:9:12: error: CHECK-NOT: excluded string found in input
+
+;         LAB:<<<<<<
+;    LAB-NEXT:            1: text 
+;  LAB-V-NEXT:check:1        ^~~~
+;    LAB-NEXT:            2: labelA 
+;  LAB-V-NEXT:label:2'0      ^~~~~~
+;  LAB-V-NEXT:label:2'1      ^~~~~~
+;    LAB-NEXT:check:3              X error: no match found
+;    LAB-NEXT:            3: textA 
+;    LAB-NEXT:check:3        ~~~~~~
+;    LAB-NEXT:            4: labelB 
+;  LAB-V-NEXT:label:4        ^~~~~~
+;    LAB-NEXT:check:3        ~~~~~~
+;    LAB-NEXT:            5: textB 
+;    LAB-NEXT:            6: labelC 
+;  LAB-V-NEXT:label:6'0      ^~~~~~
+;  LAB-V-NEXT:check:5        ^~~~~~
+;  LAB-Q-NEXT:label:6              X error: no match found
+;  LAB-V-NEXT:label:6'1            X error: no match found
+; LAB-VV-NEXT:not:7                X
+;    LAB-NEXT:            7: textC 
+; LAB-VV-NEXT:not:7          ~~~~~~
+;    LAB-NEXT:            8: labelD 
+;  LAB-V-NEXT:label:8'0      ^~~~~~
+;  LAB-V-NEXT:label:8'1      ^~~~~~
+;    LAB-NEXT:            9: textD 
+;    LAB-NEXT:not:9          !~~~~  error: no match expected
+;    LAB-NEXT:           10: labelE 
+;  LAB-V-NEXT:label:10'0     ^~~~~~
+;  LAB-V-NEXT:label:10'1     ^~~~~~
+; LAB-VV-NEXT:not:11               X
+;    LAB-NEXT:           11: textE 
+; LAB-VV-NEXT:not:11         ~~~~~~
+;    LAB-NEXT:           12: labelF 
+;  LAB-V-NEXT:label:12'0     ^~~~~~
+;  LAB-V-NEXT:label:12'1     ^~~~~~
+;    LAB-NEXT:>>>>>>
+;     LAB-NOT:{{.}}
 
 ;--------------------------------------------------
 ; --implicit-check-not
@@ -566,20 +618,28 @@
 ; RUN: echo 'CHECK: wor' >> %t.chk
 ; RUN: echo 'CHECK: !' >> %t.chk
 
+; Prefixes used here:
+; IMPNOT    = quiet, -v, or -vv
+; IMPNOT-Q  = quiet
+; IMPNOT-V  = -v or -vv (-vv implies -v)
+; IMPNOT-VQ = -v and not -vv
+; IMPNOT-VV = -vv
+
 ; RUN: %ProtectFileCheckOutput \
 ; RUN: not FileCheck -dump-input=always -input-file=%t.in %t.chk 2>&1 \
 ; RUN:               --implicit-check-not='goodbye' \
 ; RUN:               --implicit-check-not='world' \
 ; RUN:               --implicit-check-not='again' \
-; RUN: | FileCheck -match-full-lines %s -check-prefix=IMPNOT \
-; RUN:             -implicit-check-not='remark:'
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=IMPNOT,IMPNOT-Q \
+; RUN:             -implicit-check-not='{{remark:|error:}}'
 ; RUN: %ProtectFileCheckOutput \
 ; RUN: not FileCheck -dump-input=always -input-file=%t.in %t.chk -v 2>&1 \
 ; RUN:               --implicit-check-not='goodbye' \
 ; RUN:               --implicit-check-not='world' \
 ; RUN:               --implicit-check-not='again' \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=IMPNOT,IMPNOT-V \
-; RUN:             -implicit-check-not='remark:'
+; RUN: | FileCheck -match-full-lines %s \
+; RUN:             -check-prefixes=IMPNOT,IMPNOT-V,IMPNOT-VQ \
+; RUN:             -implicit-check-not='{{remark:|error:}}'
 ; RUN: %ProtectFileCheckOutput \
 ; RUN: not FileCheck -dump-input=always -input-file=%t.in %t.chk -vv 2>&1 \
 ; RUN:               --implicit-check-not='goodbye' \
@@ -587,25 +647,27 @@
 ; RUN:               --implicit-check-not='again' \
 ; RUN: | FileCheck -match-full-lines %s \
 ; RUN:             -check-prefixes=IMPNOT,IMPNOT-V,IMPNOT-VV \
-; RUN:             -implicit-check-not='remark:'
+; RUN:             -implicit-check-not='{{remark:|error:}}'
 
 ; Verbose diagnostics are suppressed but not errors.
-; IMPNOT:{{.*}}error:{{.*}}
+; IMPNOT:{{.*}}command line:1:22: error: CHECK-NOT: excluded string found in input
 
 ;         IMPNOT:<<<<<<
-;    IMPNOT-NEXT:          1: hello world again! 
-;  IMPNOT-V-NEXT:check:1      ^~~
-; IMPNOT-VV-NEXT:not:imp1     X
-; IMPNOT-VV-NEXT:not:imp2     X
-; IMPNOT-VV-NEXT:not:imp3     X
-;  IMPNOT-V-NEXT:check:2            ^~~
-; IMPNOT-VV-NEXT:not:imp1        X~~
-; IMPNOT-VV-NEXT:not:imp2        X~~
-; IMPNOT-VV-NEXT:not:imp3        X~~
-;  IMPNOT-V-NEXT:check:3                       ^
-; IMPNOT-VV-NEXT:not:imp1              X~~~~~~~
-; IMPNOT-VV-NEXT:not:imp2              X~~~~~~~
-;    IMPNOT-NEXT:not:imp3                 !~~~~   error: no match expected
+;    IMPNOT-NEXT:            1: hello world again! 
+;  IMPNOT-V-NEXT:check:1        ^~~
+; IMPNOT-VV-NEXT:not:imp1'0     X
+; IMPNOT-VV-NEXT:not:imp2'0     X
+; IMPNOT-VV-NEXT:not:imp3'0     X
+;  IMPNOT-V-NEXT:check:2              ^~~
+; IMPNOT-VV-NEXT:not:imp1'1        X~~
+; IMPNOT-VV-NEXT:not:imp2'1        X~~
+; IMPNOT-VV-NEXT:not:imp3'1        X~~
+;  IMPNOT-V-NEXT:check:3                         ^
+; IMPNOT-VV-NEXT:not:imp1'2              X~~~~~~~
+; IMPNOT-VV-NEXT:not:imp2'2              X~~~~~~~
+;  IMPNOT-Q-NEXT:not:imp3                   !~~~~   error: no match expected
+; IMPNOT-VQ-NEXT:not:imp3                   !~~~~   error: no match expected
+; IMPNOT-VV-NEXT:not:imp3'2                 !~~~~   error: no match expected
 ;    IMPNOT-NEXT:>>>>>>
 ;     IMPNOT-NOT:{{.}}
 
diff --git a/llvm/test/FileCheck/match-time-error-propagation/invalid-excluded-pattern.txt b/llvm/test/FileCheck/match-time-error-propagation/invalid-excluded-pattern.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4513fdc4f8fc6d3c22478e3edc65f3158ab03f69
--- /dev/null
+++ b/llvm/test/FileCheck/match-time-error-propagation/invalid-excluded-pattern.txt
@@ -0,0 +1,67 @@
+; Check handling of match-time diagnostics for invalid patterns (e.g.,
+; substitution overflow) in the case of excluded patterns (e.g., CHECK-NOT).
+;
+; At one time, FileCheck's exit status was zero for this case.  Moreover, it
+; printed the error diagnostic only if -vv was specified and input dumps were
+; disabled.  Test every combination as the logic is hard to get right.
+
+RUN: echo > %t.chk \
+RUN:      'CHECK-NOT: [[#0x8000000000000000+0x8000000000000000]] [[UNDEFVAR]]'
+RUN: echo > %t.in '10000000000000000'
+
+     ERR-NOT:{{.}}
+      ERR-VV:{{.*}}: remark: implicit EOF: expected string found in input
+ ERR-VV-NEXT:CHECK-NOT: {{.*}}
+ ERR-VV-NEXT:{{ *}}^
+ ERR-VV-NEXT:{{.*}}: note: found here
+ERR-VV-EMPTY:
+ ERR-VV-NEXT:^
+     ERR-NOT:{{.}}
+         ERR:{{.*}}: error: unable to substitute variable or numeric expression: overflow error
+    ERR-NEXT:CHECK-NOT: {{.*}}
+    ERR-NEXT:{{ *}}^
+    ERR-NEXT:<stdin>:1:1: note: uses undefined variable(s): "UNDEFVAR"
+    ERR-NEXT:10000000000000000
+    ERR-NEXT:^
+     ERR-NOT:{{error|note|remark}}:
+
+        DUMP:<<<<<<
+   DUMP-NEXT:         1: 10000000000000000 
+   DUMP-NEXT:not:1'0     X~~~~~~~~~~~~~~~~~ error: match failed for invalid pattern
+   DUMP-NEXT:not:1'1                        unable to substitute variable or numeric expression: overflow error
+   DUMP-NEXT:not:1'2                        uses undefined variable(s): "UNDEFVAR"
+DUMP-VV-NEXT:         2: 
+DUMP-VV-NEXT:eof:1       ^
+   DUMP-NEXT:>>>>>>
+
+;--------------------------------------------------
+; Check -dump-input=never cases.
+;--------------------------------------------------
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never -v %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never -vv %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,ERR-VV
+
+;--------------------------------------------------
+; Check -dump-input=fail cases.
+;--------------------------------------------------
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail -v %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP,DUMP-VV
diff --git a/llvm/test/FileCheck/match-time-error-propagation/invalid-expected-pattern.txt b/llvm/test/FileCheck/match-time-error-propagation/invalid-expected-pattern.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6c55b545c031ea66ec170517e7f51731a6e464c4
--- /dev/null
+++ b/llvm/test/FileCheck/match-time-error-propagation/invalid-expected-pattern.txt
@@ -0,0 +1,54 @@
+; Check handling of match-time diagnostics for invalid patterns (e.g.,
+; substitution overflow) in the case of expected patterns (e.g., CHECK).
+
+RUN: echo > %t.chk \
+RUN:      'CHECK: [[#0x8000000000000000+0x8000000000000000]] [[UNDEFVAR]]'
+RUN: echo > %t.in '10000000000000000'
+
+  ERR-NOT:{{.}}
+      ERR:{{.*}}: error: unable to substitute variable or numeric expression: overflow error
+ ERR-NEXT:CHECK: {{.*}}
+ ERR-NEXT:{{ *}}^
+ ERR-NEXT:<stdin>:1:1: note: uses undefined variable(s): "UNDEFVAR"
+ ERR-NEXT:10000000000000000
+ ERR-NEXT:^
+  ERR-NOT:{{error|note|remark}}:
+
+     DUMP:<<<<<<
+DUMP-NEXT:           1: 10000000000000000 
+DUMP-NEXT:check:1'0     X~~~~~~~~~~~~~~~~~ error: match failed for invalid pattern
+DUMP-NEXT:check:1'1                        unable to substitute variable or numeric expression: overflow error
+DUMP-NEXT:check:1'2                        uses undefined variable(s): "UNDEFVAR"
+DUMP-NEXT:>>>>>>
+
+;--------------------------------------------------
+; Check -dump-input=never cases.
+;--------------------------------------------------
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never -v %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never -vv %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+;--------------------------------------------------
+; Check -dump-input=fail cases.
+;--------------------------------------------------
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail -v %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
diff --git a/llvm/test/FileCheck/match-time-error-propagation/matched-excluded-pattern.txt b/llvm/test/FileCheck/match-time-error-propagation/matched-excluded-pattern.txt
new file mode 100644
index 0000000000000000000000000000000000000000..32ffbf2e8f211e11fcefeb05f5777454e3c8f10c
--- /dev/null
+++ b/llvm/test/FileCheck/match-time-error-propagation/matched-excluded-pattern.txt
@@ -0,0 +1,88 @@
+; Check handling of diagnostics for problematic matches (e.g., variable capture
+; overflow) in the case of excluded patterns (e.g., CHECK-NOT).
+;
+; At one time, FileCheck's exit status for the following example was zero even
+; though the excluded pattern does match (it's just capturing that fails).
+; Moreover, it printed the error diagnostic only if -vv was specified and input
+; dumps were disabled.  Test every combination as the logic is hard to get
+; right. 
+;
+; TODO: Capturing from an excluded pattern probably shouldn't be permitted
+; because it seems useless: it's captured only if the pattern matches, but then
+; FileCheck fails.  The helpfulness of reporting overflow from that capture is
+; perhaps questionable then, but it doesn't seem harmful either.  Anyway, the
+; goal of this test is simply to exercise the error propagation mechanism for a
+; matched excluded pattern.  In the future, if we have a more interesting error
+; to exercise in that case, we should instead use it in this test, and then we
+; might want to think more about where that error should be presented in the
+; list of diagnostics.
+
+RUN: echo > %t.chk 'CHECK-NOT: [[#122+1]] [[STR:abc]] [[#NUM:]]'
+RUN: echo > %t.in '123 abc 1000000000000000000000000000000000000000000000000000'
+
+     ERR-NOT:{{.}}
+      ERR-VV:{{.*}}: remark: implicit EOF: expected string found in input
+ ERR-VV-NEXT:CHECK-NOT: {{.*}}
+ ERR-VV-NEXT:{{ *}}^
+ ERR-VV-NEXT:{{.*}}: note: found here
+ERR-VV-EMPTY:
+ ERR-VV-NEXT:^
+     ERR-NOT:{{.}}
+         ERR:{{.*}}: error: CHECK-NOT: excluded string found in input
+    ERR-NEXT:CHECK-NOT: {{.*}}
+    ERR-NEXT:{{ *}}^
+    ERR-NEXT:<stdin>:1:1: note: found here
+    ERR-NEXT:123 abc 10{{0*}}
+    ERR-NEXT:^~~~~~~~~{{~*}}
+    ERR-NEXT:<stdin>:1:1: note: with "122+1" equal to "123"
+    ERR-NEXT:123 abc 10{{0*}}
+    ERR-NEXT:^
+    ERR-NEXT:<stdin>:1:5: note: captured var "STR"
+    ERR-NEXT:123 abc 10{{0*}}
+    ERR-NEXT:    ^~~
+    ERR-NEXT:<stdin>:1:9: error: unable to represent numeric value
+    ERR-NEXT:123 abc 10{{0*}}
+    ERR-NEXT:        ^
+     ERR-NOT:{{error|note|remark}}:
+
+        DUMP:<<<<<<
+   DUMP-NEXT:         1: 123 abc 10{{0*}} 
+   DUMP-NEXT:not:1'0     !~~~~~~~~~{{~*}}  error: no match expected
+   DUMP-NEXT:not:1'1                       with "122+1" equal to "123"
+   DUMP-NEXT:not:1'2         !~~           captured var "STR"
+   DUMP-NEXT:not:1'3             !~{{~*}}  error: unable to represent numeric value
+DUMP-VV-NEXT:         2: 
+DUMP-VV-NEXT:eof:1       ^
+   DUMP-NEXT:>>>>>>
+
+;--------------------------------------------------
+; Check -dump-input=never cases.
+;--------------------------------------------------
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never -v %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never -vv %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,ERR-VV
+
+;--------------------------------------------------
+; Check -dump-input=fail cases.
+;--------------------------------------------------
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail -v %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP,DUMP-VV
diff --git a/llvm/test/FileCheck/match-time-error-propagation/matched-expected-pattern.txt b/llvm/test/FileCheck/match-time-error-propagation/matched-expected-pattern.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d376f73bfdc0f3ff7beae27490c6ef88243709ec
--- /dev/null
+++ b/llvm/test/FileCheck/match-time-error-propagation/matched-expected-pattern.txt
@@ -0,0 +1,63 @@
+; Check handling of diagnostics for problematic matches (e.g., variable capture
+; overflow) in the case of expected patterns (e.g., CHECK).
+
+RUN: echo > %t.chk 'CHECK: [[#122+1]] [[STR:abc]] [[#NUM:]]'
+RUN: echo > %t.in '123 abc 1000000000000000000000000000000000000000000000000000'
+
+  ERR-NOT:{{.}}
+      ERR:{{.*}}: remark: CHECK: expected string found in input
+ ERR-NEXT:CHECK: {{.*}}
+ ERR-NEXT:{{ *}}^
+ ERR-NEXT:<stdin>:1:1: note: found here
+ ERR-NEXT:123 abc 10{{0*}}
+ ERR-NEXT:^~~~~~~~~{{~*}}
+ ERR-NEXT:<stdin>:1:1: note: with "122+1" equal to "123"
+ ERR-NEXT:123 abc 10{{0*}}
+ ERR-NEXT:^
+ ERR-NEXT:<stdin>:1:5: note: captured var "STR"
+ ERR-NEXT:123 abc 10{{0*}}
+ ERR-NEXT:    ^~~
+ ERR-NEXT:<stdin>:1:9: error: unable to represent numeric value
+ ERR-NEXT:123 abc 10{{0*}}
+ ERR-NEXT:        ^
+  ERR-NOT:{{error|note|remark}}:
+
+     DUMP:<<<<<<
+DUMP-NEXT:           1: 123 abc 10{{0*}} 
+DUMP-NEXT:check:1'0     ^~~~~~~~~~{{~*}}
+DUMP-NEXT:check:1'1                       with "122+1" equal to "123"
+DUMP-NEXT:check:1'2         ^~~           captured var "STR"
+DUMP-NEXT:check:1'3             !~{{~*}}  error: unable to represent numeric value
+DUMP-NEXT:>>>>>>
+
+;--------------------------------------------------
+; Check -dump-input=never cases.
+;--------------------------------------------------
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never -v %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=never -vv %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR
+
+;--------------------------------------------------
+; Check -dump-input=fail cases.
+;--------------------------------------------------
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail -v %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
+
+RUN: %ProtectFileCheckOutput \
+RUN: not FileCheck -dump-input=fail -vv %t.chk < %t.in 2>&1 \
+RUN: | FileCheck %s -match-full-lines -check-prefixes=ERR,DUMP
diff --git a/llvm/test/FileCheck/simple-var-capture.txt b/llvm/test/FileCheck/simple-var-capture.txt
index a487baaa531cb4eeba03bf0048051d0615b52935..d9f456130e023df479d34218615f90bcae05b3cc 100644
--- a/llvm/test/FileCheck/simple-var-capture.txt
+++ b/llvm/test/FileCheck/simple-var-capture.txt
@@ -11,3 +11,15 @@ op4 r30, r18, r21
 ; CHECK-NEXT:   op4 {{r[0-9]+}}, [[REGa]], [[REGb]]
 
 
+// RUN: %ProtectFileCheckOutput \
+// RUN: not FileCheck --check-prefixes INVALID-VARNAME --input-file %s %s 2>&1 \
+// RUN:   | FileCheck --check-prefix INVALID-VARNAME-MSG --strict-whitespace %s
+
+5
+4
+; INVALID-VARNAME: [[X:]]
+; INVALID-VARNAME-NEXT: [[Y:]]
+; INVALID-VARNAME-NEXT: [[X-Y]]
+; INVALID-VARNAME-MSG: simple-var-capture.txt:[[#@LINE-1]]:27: error: invalid name in string variable use
+; INVALID-VARNAME-MSG-NEXT: ; {{I}}NVALID-VARNAME-NEXT: {{\[\[X-Y\]\]}}
+; INVALID-VARNAME-MSG-NEXT:    {{^}}                          ^{{$}}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument-dynamic-allocas.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument-dynamic-allocas.ll
index 434e4be4e8e6d40fb8ff290e833dfdd3d2d51ab9..257c0cbf0c7e9436c4d399954573fbdb66980ecb 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/instrument-dynamic-allocas.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument-dynamic-allocas.ll
@@ -31,8 +31,8 @@ define void @has_inalloca() uwtable sanitize_address {
 entry:
   %t = alloca inalloca i32
   store i32 42, i32* %t
-  call void @pass_inalloca(i32* inalloca %t)
+  call void @pass_inalloca(i32* inalloca(i32) %t)
   ret void
 }
 
-declare void @pass_inalloca(i32* inalloca)
+declare void @pass_inalloca(i32* inalloca(i32))
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/abilist.ll b/llvm/test/Instrumentation/DataFlowSanitizer/abilist.ll
index 3750ce346586b3685516d6733ecd2a55b54dd8b9..11a09af8dc00e83f296622a13ae572f7b9e2ffd0 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/abilist.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/abilist.ll
@@ -1,4 +1,6 @@
 ; RUN: opt < %s -dfsan -dfsan-args-abi -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-args-abi -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-args-abi -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/abilist_aggregate.ll b/llvm/test/Instrumentation/DataFlowSanitizer/abilist_aggregate.ll
index 1bcc78517ca9128a340f7a44d5d0f7e881486719..6d59e96a94019d4282161aa712bfdabc4c968688 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/abilist_aggregate.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/abilist_aggregate.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI
 ; RUN: opt < %s -dfsan -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefixes=CHECK,LEGACY
 ; RUN: opt < %s -dfsan -dfsan-args-abi -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefixes=CHECK,ARGS_ABI
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/array.ll b/llvm/test/Instrumentation/DataFlowSanitizer/array.ll
index 939a7f81ba9626d4bf748c110c75b45092376df8..fc88061480d5c835008702bf6b3473157e645ca1 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/array.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/array.ll
@@ -1,10 +1,15 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,LEGACY
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-event-callbacks=true -S | FileCheck %s --check-prefixes=CHECK,EVENT_CALLBACKS
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-event-callbacks=true -S | FileCheck %s --check-prefixes=CHECK,EVENT_CALLBACKS
 ; RUN: opt < %s -dfsan -dfsan-args-abi -S | FileCheck %s --check-prefixes=CHECK,ARGS_ABI
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST16
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -90,12 +95,12 @@ define [1 x i1] @load_array1([1 x i1]* %p) {
   ; EVENT_CALLBACKS: [[L:%.*]] = or i[[#SBITS]]
   ; EVENT_CALLBACKS: call void @__dfsan_load_callback(i[[#SBITS]] [[L]], i8* {{.*}})
 
-  ; FAST16: @"dfs$load_array1"
-  ; FAST16: [[P:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; FAST16: [[L:%.*]] = load i[[#SBITS]], i[[#SBITS]]* {{.*}}, align [[#SBYTES]]
-  ; FAST16: [[U:%.*]] = or i[[#SBITS]] [[L]], [[P]]
-  ; FAST16: [[S1:%.*]] = insertvalue [1 x i[[#SBITS]]] undef, i[[#SBITS]] [[U]], 0
-  ; FAST16: store [1 x i[[#SBITS]]] [[S1]], [1 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [1 x i[[#SBITS]]]*), align [[ALIGN]]
+  ; FAST: @"dfs$load_array1"
+  ; FAST: [[P:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
+  ; FAST: [[L:%.*]] = load i[[#SBITS]], i[[#SBITS]]* {{.*}}, align [[#SBYTES]]
+  ; FAST: [[U:%.*]] = or i[[#SBITS]] [[L]], [[P]]
+  ; FAST: [[S1:%.*]] = insertvalue [1 x i[[#SBITS]]] undef, i[[#SBITS]] [[U]], 0
+  ; FAST: store [1 x i[[#SBITS]]] [[S1]], [1 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [1 x i[[#SBITS]]]*), align [[ALIGN]]
 
   ; LEGACY: @"dfs$load_array1"
   ; LEGACY: [[P:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
@@ -123,13 +128,13 @@ define [2 x i1] @load_array2([2 x i1]* %p) {
   ; EVENT_CALLBACKS: [[O2:%.*]] = or i[[#SBITS]] [[O1]]
   ; EVENT_CALLBACKS: call void @__dfsan_load_callback(i[[#SBITS]] [[O2]], i8* {{.*}})
 
-  ; FAST16: @"dfs$load_array2"
-  ; FAST16: [[P:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; FAST16: [[O:%.*]] = or i[[#SBITS]]
-  ; FAST16: [[U:%.*]] = or i[[#SBITS]] [[O]], [[P]]
-  ; FAST16: [[S:%.*]] = insertvalue [2 x i[[#SBITS]]] undef, i[[#SBITS]] [[U]], 0
-  ; FAST16: [[S1:%.*]] = insertvalue [2 x i[[#SBITS]]] [[S]], i[[#SBITS]] [[U]], 1
-  ; FAST16: store [2 x i[[#SBITS]]] [[S1]], [2 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [2 x i[[#SBITS]]]*), align [[ALIGN]]
+  ; FAST: @"dfs$load_array2"
+  ; FAST: [[P:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
+  ; FAST: [[O:%.*]] = or i[[#SBITS]]
+  ; FAST: [[U:%.*]] = or i[[#SBITS]] [[O]], [[P]]
+  ; FAST: [[S:%.*]] = insertvalue [2 x i[[#SBITS]]] undef, i[[#SBITS]] [[U]], 0
+  ; FAST: [[S1:%.*]] = insertvalue [2 x i[[#SBITS]]] [[S]], i[[#SBITS]] [[U]], 1
+  ; FAST: store [2 x i[[#SBITS]]] [[S1]], [2 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [2 x i[[#SBITS]]]*), align [[ALIGN]]
   %a = load [2 x i1], [2 x i1]* %p
   ret [2 x i1] %a
 }
@@ -150,14 +155,14 @@ define [4 x i1] @load_array4([4 x i1]* %p) {
   ; EVENT_CALLBACKS: [[O3:%.*]] = or i[[#SBITS]] [[O2]]
   ; EVENT_CALLBACKS: call void @__dfsan_load_callback(i[[#SBITS]] [[O3]], i8* {{.*}})
 
-  ; FAST16: @"dfs$load_array4"
-  ; FAST16: [[T:%.*]] = trunc i[[#mul(4, SBITS)]] {{.*}} to i[[#SBITS]]
-  ; FAST16: [[O:%.*]] = or i[[#SBITS]] [[T]]
-  ; FAST16: [[S1:%.*]] = insertvalue [4 x i[[#SBITS]]] undef, i[[#SBITS]] [[O]], 0
-  ; FAST16: [[S2:%.*]] = insertvalue [4 x i[[#SBITS]]] [[S1]], i[[#SBITS]] [[O]], 1
-  ; FAST16: [[S3:%.*]] = insertvalue [4 x i[[#SBITS]]] [[S2]], i[[#SBITS]] [[O]], 2
-  ; FAST16: [[S4:%.*]] = insertvalue [4 x i[[#SBITS]]] [[S3]], i[[#SBITS]] [[O]], 3
-  ; FAST16: store [4 x i[[#SBITS]]] [[S4]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [4 x i[[#SBITS]]]*), align 2
+  ; FAST: @"dfs$load_array4"
+  ; FAST: [[T:%.*]] = trunc i[[#mul(4, SBITS)]] {{.*}} to i[[#SBITS]]
+  ; FAST: [[O:%.*]] = or i[[#SBITS]] [[T]]
+  ; FAST: [[S1:%.*]] = insertvalue [4 x i[[#SBITS]]] undef, i[[#SBITS]] [[O]], 0
+  ; FAST: [[S2:%.*]] = insertvalue [4 x i[[#SBITS]]] [[S1]], i[[#SBITS]] [[O]], 1
+  ; FAST: [[S3:%.*]] = insertvalue [4 x i[[#SBITS]]] [[S2]], i[[#SBITS]] [[O]], 2
+  ; FAST: [[S4:%.*]] = insertvalue [4 x i[[#SBITS]]] [[S3]], i[[#SBITS]] [[O]], 3
+  ; FAST: store [4 x i[[#SBITS]]] [[S4]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [4 x i[[#SBITS]]]*), align 2
 
   ; LEGACY: @"dfs$load_array4"
   ; LEGACY: [[P:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
@@ -191,25 +196,25 @@ define [4 x i1] @insert_array([4 x i1] %a, i1 %e2) {
 }
 
 define void @store_alloca_array([4 x i1] %a) {
-  ; FAST16: @"dfs$store_alloca_array"
-  ; FAST16: [[S:%.*]] = load [4 x i[[#SBITS]]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [4 x i[[#SBITS]]]*), align [[ALIGN:2]]
-  ; FAST16: [[SP:%.*]] = alloca i[[#SBITS]], align [[#SBYTES]]
-  ; FAST16: [[E0:%.*]] = extractvalue [4 x i[[#SBITS]]] [[S]], 0
-  ; FAST16: [[E1:%.*]] = extractvalue [4 x i[[#SBITS]]] [[S]], 1
-  ; FAST16: [[E01:%.*]] = or i[[#SBITS]] [[E0]], [[E1]]
-  ; FAST16: [[E2:%.*]] = extractvalue [4 x i[[#SBITS]]] [[S]], 2
-  ; FAST16: [[E012:%.*]] = or i[[#SBITS]] [[E01]], [[E2]]
-  ; FAST16: [[E3:%.*]] = extractvalue [4 x i[[#SBITS]]] [[S]], 3
-  ; FAST16: [[E0123:%.*]] = or i[[#SBITS]] [[E012]], [[E3]]
-  ; FAST16: store i[[#SBITS]] [[E0123]], i[[#SBITS]]* [[SP]], align [[#SBYTES]]
+  ; FAST: @"dfs$store_alloca_array"
+  ; FAST: [[S:%.*]] = load [4 x i[[#SBITS]]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [4 x i[[#SBITS]]]*), align [[ALIGN:2]]
+  ; FAST: [[SP:%.*]] = alloca i[[#SBITS]], align [[#SBYTES]]
+  ; FAST: [[E0:%.*]] = extractvalue [4 x i[[#SBITS]]] [[S]], 0
+  ; FAST: [[E1:%.*]] = extractvalue [4 x i[[#SBITS]]] [[S]], 1
+  ; FAST: [[E01:%.*]] = or i[[#SBITS]] [[E0]], [[E1]]
+  ; FAST: [[E2:%.*]] = extractvalue [4 x i[[#SBITS]]] [[S]], 2
+  ; FAST: [[E012:%.*]] = or i[[#SBITS]] [[E01]], [[E2]]
+  ; FAST: [[E3:%.*]] = extractvalue [4 x i[[#SBITS]]] [[S]], 3
+  ; FAST: [[E0123:%.*]] = or i[[#SBITS]] [[E012]], [[E3]]
+  ; FAST: store i[[#SBITS]] [[E0123]], i[[#SBITS]]* [[SP]], align [[#SBYTES]]
   %p = alloca [4 x i1]
   store [4 x i1] %a, [4 x i1]* %p
   ret void
 }
 
 define void @store_zero_array([4 x i1]* %p) {
-  ; FAST16: @"dfs$store_zero_array"
-  ; FAST16: store i[[#mul(4, SBITS)]] 0, i[[#mul(4, SBITS)]]* {{.*}}
+  ; FAST: @"dfs$store_zero_array"
+  ; FAST: store i[[#mul(4, SBITS)]] 0, i[[#mul(4, SBITS)]]* {{.*}}
   store [4 x i1] zeroinitializer, [4 x i1]* %p
   ret void
 }
@@ -227,15 +232,15 @@ define void @store_array2([2 x i1] %a, [2 x i1]* %p) {
   ; EVENT_CALLBACKS: [[P:%.*]] = bitcast [2 x i1]* %p to i8*
   ; EVENT_CALLBACKS: call void @__dfsan_store_callback(i[[#SBITS]] [[E12]], i8* [[P]])
 
-  ; FAST16: @"dfs$store_array2"
-  ; FAST16: [[S:%.*]] = load [2 x i[[#SBITS]]], [2 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [2 x i[[#SBITS]]]*), align [[ALIGN:2]]
-  ; FAST16: [[E1:%.*]] = extractvalue [2 x i[[#SBITS]]] [[S]], 0
-  ; FAST16: [[E2:%.*]] = extractvalue [2 x i[[#SBITS]]] [[S]], 1
-  ; FAST16: [[E12:%.*]] = or i[[#SBITS]] [[E1]], [[E2]]
-  ; FAST16: [[SP0:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[SP:%.*]], i32 0
-  ; FAST16: store i[[#SBITS]] [[E12]], i[[#SBITS]]* [[SP0]], align [[#SBYTES]]
-  ; FAST16: [[SP1:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[SP]], i32 1
-  ; FAST16: store i[[#SBITS]] [[E12]], i[[#SBITS]]* [[SP1]], align [[#SBYTES]]
+  ; FAST: @"dfs$store_array2"
+  ; FAST: [[S:%.*]] = load [2 x i[[#SBITS]]], [2 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [2 x i[[#SBITS]]]*), align [[ALIGN:2]]
+  ; FAST: [[E1:%.*]] = extractvalue [2 x i[[#SBITS]]] [[S]], 0
+  ; FAST: [[E2:%.*]] = extractvalue [2 x i[[#SBITS]]] [[S]], 1
+  ; FAST: [[E12:%.*]] = or i[[#SBITS]] [[E1]], [[E2]]
+  ; FAST: [[SP0:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[SP:%.*]], i32 0
+  ; FAST: store i[[#SBITS]] [[E12]], i[[#SBITS]]* [[SP0]], align [[#SBYTES]]
+  ; FAST: [[SP1:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[SP]], i32 1
+  ; FAST: store i[[#SBITS]] [[E12]], i[[#SBITS]]* [[SP1]], align [[#SBYTES]]
 
   ; COMBINE_STORE_PTR: @"dfs$store_array2"
   ; COMBINE_STORE_PTR: [[O:%.*]] = or i[[#SBITS]]
@@ -250,72 +255,72 @@ define void @store_array2([2 x i1] %a, [2 x i1]* %p) {
 }
 
 define void @store_array17([17 x i1] %a, [17 x i1]* %p) {
-  ; FAST16: @"dfs$store_array17"
-  ; FAST16: %[[#R:]]   = load [17 x i[[#SBITS]]], [17 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [17 x i[[#SBITS]]]*), align 2
-  ; FAST16: %[[#R+1]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 0
-  ; FAST16: %[[#R+2]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 1
-  ; FAST16: %[[#R+3]]  = or i[[#SBITS]] %[[#R+1]], %[[#R+2]]
-  ; FAST16: %[[#R+4]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 2
-  ; FAST16: %[[#R+5]]  = or i[[#SBITS]] %[[#R+3]], %[[#R+4]]
-  ; FAST16: %[[#R+6]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 3
-  ; FAST16: %[[#R+7]]  = or i[[#SBITS]] %[[#R+5]], %[[#R+6]]
-  ; FAST16: %[[#R+8]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 4
-  ; FAST16: %[[#R+9]]  = or i[[#SBITS]] %[[#R+7]], %[[#R+8]]
-  ; FAST16: %[[#R+10]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 5
-  ; FAST16: %[[#R+11]] = or i[[#SBITS]] %[[#R+9]], %[[#R+10]]
-  ; FAST16: %[[#R+12]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 6
-  ; FAST16: %[[#R+13]] = or i[[#SBITS]] %[[#R+11]], %[[#R+12]]
-  ; FAST16: %[[#R+14]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 7
-  ; FAST16: %[[#R+15]] = or i[[#SBITS]] %[[#R+13]], %[[#R+14]]
-  ; FAST16: %[[#R+16]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 8
-  ; FAST16: %[[#R+17]] = or i[[#SBITS]] %[[#R+15]], %[[#R+16]]
-  ; FAST16: %[[#R+18]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 9
-  ; FAST16: %[[#R+19]] = or i[[#SBITS]] %[[#R+17]], %[[#R+18]]
-  ; FAST16: %[[#R+20]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 10
-  ; FAST16: %[[#R+21]] = or i[[#SBITS]] %[[#R+19]], %[[#R+20]]
-  ; FAST16: %[[#R+22]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 11
-  ; FAST16: %[[#R+23]] = or i[[#SBITS]] %[[#R+21]], %[[#R+22]]
-  ; FAST16: %[[#R+24]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 12
-  ; FAST16: %[[#R+25]] = or i[[#SBITS]] %[[#R+23]], %[[#R+24]]
-  ; FAST16: %[[#R+26]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 13
-  ; FAST16: %[[#R+27]] = or i[[#SBITS]] %[[#R+25]], %[[#R+26]]
-  ; FAST16: %[[#R+28]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 14
-  ; FAST16: %[[#R+29]] = or i[[#SBITS]] %[[#R+27]], %[[#R+28]]
-  ; FAST16: %[[#R+30]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 15
-  ; FAST16: %[[#R+31]] = or i[[#SBITS]] %[[#R+29]], %[[#R+30]]
-  ; FAST16: %[[#R+32]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 16
-  ; FAST16: %[[#R+33]] = or i[[#SBITS]] %[[#R+31]], %[[#R+32]]
-  ; FAST16: %[[#VREG:]]  = insertelement <8 x i[[#SBITS]]> undef, i[[#SBITS]] %[[#R+33]], i32 0
-  ; FAST16: %[[#VREG+1]] = insertelement <8 x i[[#SBITS]]> %[[#VREG]], i[[#SBITS]] %[[#R+33]], i32 1
-  ; FAST16: %[[#VREG+2]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+1]], i[[#SBITS]] %[[#R+33]], i32 2
-  ; FAST16: %[[#VREG+3]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+2]], i[[#SBITS]] %[[#R+33]], i32 3
-  ; FAST16: %[[#VREG+4]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+3]], i[[#SBITS]] %[[#R+33]], i32 4
-  ; FAST16: %[[#VREG+5]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+4]], i[[#SBITS]] %[[#R+33]], i32 5
-  ; FAST16: %[[#VREG+6]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+5]], i[[#SBITS]] %[[#R+33]], i32 6
-  ; FAST16: %[[#VREG+7]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+6]], i[[#SBITS]] %[[#R+33]], i32 7
-  ; FAST16: %[[#VREG+8]] = bitcast i[[#SBITS]]* %[[P:.*]] to <8 x i[[#SBITS]]>*
-  ; FAST16: %[[#VREG+9]]  = getelementptr <8 x i[[#SBITS]]>, <8 x i[[#SBITS]]>* %[[#VREG+8]], i32 0
-  ; FAST16: store <8 x i[[#SBITS]]> %[[#VREG+7]], <8 x i[[#SBITS]]>* %[[#VREG+9]], align [[#SBYTES]]
-  ; FAST16: %[[#VREG+10]] = getelementptr <8 x i[[#SBITS]]>, <8 x i[[#SBITS]]>* %[[#VREG+8]], i32 1
-  ; FAST16: store <8 x i[[#SBITS]]> %[[#VREG+7]], <8 x i[[#SBITS]]>* %[[#VREG+10]], align [[#SBYTES]]
-  ; FAST16: %[[#VREG+11]] = getelementptr i[[#SBITS]], i[[#SBITS]]* %[[P]], i32 16
-  ; FAST16: store i[[#SBITS]] %[[#R+33]], i[[#SBITS]]* %[[#VREG+11]], align [[#SBYTES]]
+  ; FAST: @"dfs$store_array17"
+  ; FAST: %[[#R:]]   = load [17 x i[[#SBITS]]], [17 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [17 x i[[#SBITS]]]*), align 2
+  ; FAST: %[[#R+1]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 0
+  ; FAST: %[[#R+2]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 1
+  ; FAST: %[[#R+3]]  = or i[[#SBITS]] %[[#R+1]], %[[#R+2]]
+  ; FAST: %[[#R+4]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 2
+  ; FAST: %[[#R+5]]  = or i[[#SBITS]] %[[#R+3]], %[[#R+4]]
+  ; FAST: %[[#R+6]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 3
+  ; FAST: %[[#R+7]]  = or i[[#SBITS]] %[[#R+5]], %[[#R+6]]
+  ; FAST: %[[#R+8]]  = extractvalue [17 x i[[#SBITS]]] %[[#R]], 4
+  ; FAST: %[[#R+9]]  = or i[[#SBITS]] %[[#R+7]], %[[#R+8]]
+  ; FAST: %[[#R+10]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 5
+  ; FAST: %[[#R+11]] = or i[[#SBITS]] %[[#R+9]], %[[#R+10]]
+  ; FAST: %[[#R+12]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 6
+  ; FAST: %[[#R+13]] = or i[[#SBITS]] %[[#R+11]], %[[#R+12]]
+  ; FAST: %[[#R+14]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 7
+  ; FAST: %[[#R+15]] = or i[[#SBITS]] %[[#R+13]], %[[#R+14]]
+  ; FAST: %[[#R+16]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 8
+  ; FAST: %[[#R+17]] = or i[[#SBITS]] %[[#R+15]], %[[#R+16]]
+  ; FAST: %[[#R+18]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 9
+  ; FAST: %[[#R+19]] = or i[[#SBITS]] %[[#R+17]], %[[#R+18]]
+  ; FAST: %[[#R+20]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 10
+  ; FAST: %[[#R+21]] = or i[[#SBITS]] %[[#R+19]], %[[#R+20]]
+  ; FAST: %[[#R+22]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 11
+  ; FAST: %[[#R+23]] = or i[[#SBITS]] %[[#R+21]], %[[#R+22]]
+  ; FAST: %[[#R+24]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 12
+  ; FAST: %[[#R+25]] = or i[[#SBITS]] %[[#R+23]], %[[#R+24]]
+  ; FAST: %[[#R+26]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 13
+  ; FAST: %[[#R+27]] = or i[[#SBITS]] %[[#R+25]], %[[#R+26]]
+  ; FAST: %[[#R+28]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 14
+  ; FAST: %[[#R+29]] = or i[[#SBITS]] %[[#R+27]], %[[#R+28]]
+  ; FAST: %[[#R+30]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 15
+  ; FAST: %[[#R+31]] = or i[[#SBITS]] %[[#R+29]], %[[#R+30]]
+  ; FAST: %[[#R+32]] = extractvalue [17 x i[[#SBITS]]] %[[#R]], 16
+  ; FAST: %[[#R+33]] = or i[[#SBITS]] %[[#R+31]], %[[#R+32]]
+  ; FAST: %[[#VREG:]]  = insertelement <8 x i[[#SBITS]]> undef, i[[#SBITS]] %[[#R+33]], i32 0
+  ; FAST: %[[#VREG+1]] = insertelement <8 x i[[#SBITS]]> %[[#VREG]], i[[#SBITS]] %[[#R+33]], i32 1
+  ; FAST: %[[#VREG+2]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+1]], i[[#SBITS]] %[[#R+33]], i32 2
+  ; FAST: %[[#VREG+3]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+2]], i[[#SBITS]] %[[#R+33]], i32 3
+  ; FAST: %[[#VREG+4]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+3]], i[[#SBITS]] %[[#R+33]], i32 4
+  ; FAST: %[[#VREG+5]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+4]], i[[#SBITS]] %[[#R+33]], i32 5
+  ; FAST: %[[#VREG+6]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+5]], i[[#SBITS]] %[[#R+33]], i32 6
+  ; FAST: %[[#VREG+7]] = insertelement <8 x i[[#SBITS]]> %[[#VREG+6]], i[[#SBITS]] %[[#R+33]], i32 7
+  ; FAST: %[[#VREG+8]] = bitcast i[[#SBITS]]* %[[P:.*]] to <8 x i[[#SBITS]]>*
+  ; FAST: %[[#VREG+9]]  = getelementptr <8 x i[[#SBITS]]>, <8 x i[[#SBITS]]>* %[[#VREG+8]], i32 0
+  ; FAST: store <8 x i[[#SBITS]]> %[[#VREG+7]], <8 x i[[#SBITS]]>* %[[#VREG+9]], align [[#SBYTES]]
+  ; FAST: %[[#VREG+10]] = getelementptr <8 x i[[#SBITS]]>, <8 x i[[#SBITS]]>* %[[#VREG+8]], i32 1
+  ; FAST: store <8 x i[[#SBITS]]> %[[#VREG+7]], <8 x i[[#SBITS]]>* %[[#VREG+10]], align [[#SBYTES]]
+  ; FAST: %[[#VREG+11]] = getelementptr i[[#SBITS]], i[[#SBITS]]* %[[P]], i32 16
+  ; FAST: store i[[#SBITS]] %[[#R+33]], i[[#SBITS]]* %[[#VREG+11]], align [[#SBYTES]]
   store [17 x i1] %a, [17 x i1]* %p
   ret void
 }
 
 define [2 x i32] @const_array() {
-  ; FAST16: @"dfs$const_array"
-  ; FAST16: store [2 x i[[#SBITS]]] zeroinitializer, [2 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [2 x i[[#SBITS]]]*), align 2
+  ; FAST: @"dfs$const_array"
+  ; FAST: store [2 x i[[#SBITS]]] zeroinitializer, [2 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [2 x i[[#SBITS]]]*), align 2
   ret [2 x i32] [ i32 42, i32 11 ]
 }
 
 define [4 x i8] @call_array([4 x i8] %a) {
-  ; FAST16-LABEL: @"dfs$call_array"
-  ; FAST16: %[[#R:]] = load [4 x i[[#SBITS]]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [4 x i[[#SBITS]]]*), align [[ALIGN:2]]
-  ; FAST16: store [4 x i[[#SBITS]]] %[[#R]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [4 x i[[#SBITS]]]*), align [[ALIGN]]
-  ; FAST16: %_dfsret = load [4 x i[[#SBITS]]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [4 x i[[#SBITS]]]*), align [[ALIGN]]
-  ; FAST16: store [4 x i[[#SBITS]]] %_dfsret, [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [4 x i[[#SBITS]]]*), align [[ALIGN]]
+  ; FAST-LABEL: @"dfs$call_array"
+  ; FAST: %[[#R:]] = load [4 x i[[#SBITS]]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [4 x i[[#SBITS]]]*), align [[ALIGN:2]]
+  ; FAST: store [4 x i[[#SBITS]]] %[[#R]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to [4 x i[[#SBITS]]]*), align [[ALIGN]]
+  ; FAST: %_dfsret = load [4 x i[[#SBITS]]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [4 x i[[#SBITS]]]*), align [[ALIGN]]
+  ; FAST: store [4 x i[[#SBITS]]] %_dfsret, [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [4 x i[[#SBITS]]]*), align [[ALIGN]]
 
   %r = call [4 x i8] @pass_array([4 x i8] %a)
   ret [4 x i8] %r
@@ -324,31 +329,31 @@ define [4 x i8] @call_array([4 x i8] %a) {
 %LargeArr = type [1000 x i8]
 
 define i8 @fun_with_large_args(i1 %i, %LargeArr %a) {
-  ; FAST16: @"dfs$fun_with_large_args"
-  ; FAST16: store i[[#SBITS]] 0, i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align 2
+  ; FAST: @"dfs$fun_with_large_args"
+  ; FAST: store i[[#SBITS]] 0, i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align 2
   %r = extractvalue %LargeArr %a, 0
   ret i8 %r
 }
 
 define %LargeArr @fun_with_large_ret() {
-  ; FAST16: @"dfs$fun_with_large_ret"
-  ; FAST16-NEXT: ret  [1000 x i8] zeroinitializer
+  ; FAST: @"dfs$fun_with_large_ret"
+  ; FAST-NEXT: ret  [1000 x i8] zeroinitializer
   ret %LargeArr zeroinitializer
 }
 
 define i8 @call_fun_with_large_ret() {
-  ; FAST16: @"dfs$call_fun_with_large_ret"
-  ; FAST16: store i[[#SBITS]] 0, i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align 2
+  ; FAST: @"dfs$call_fun_with_large_ret"
+  ; FAST: store i[[#SBITS]] 0, i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align 2
   %r = call %LargeArr @fun_with_large_ret()
   %e = extractvalue %LargeArr %r, 0
   ret i8 %e
 }
 
 define i8 @call_fun_with_large_args(i1 %i, %LargeArr %a) {
-  ; FAST16: @"dfs$call_fun_with_large_args"
-  ; FAST16: [[I:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; FAST16: store i[[#SBITS]] [[I]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; FAST16: %r = call i8 @"dfs$fun_with_large_args"(i1 %i, [1000 x i8] %a)
+  ; FAST: @"dfs$call_fun_with_large_args"
+  ; FAST: [[I:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
+  ; FAST: store i[[#SBITS]] [[I]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; FAST: %r = call i8 @"dfs$fun_with_large_args"(i1 %i, [1000 x i8] %a)
 
   %r = call i8 @fun_with_large_args(i1 %i, %LargeArr %a)
   ret i8 %r
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/atomics.ll b/llvm/test/Instrumentation/DataFlowSanitizer/atomics.ll
index f5a225b0cb040af499ce96088b137c539964a4b2..0075170410fe2e8ca97400018be56d710182f7e5 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/atomics.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/atomics.ll
@@ -1,6 +1,7 @@
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK16
-;
-; The patterns about origins cannot be tested until the origin tracking feature is complete.
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK16,CHECK_ORIGIN
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -dfsan-instrument-with-call-threshold=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK16,CHECK_ORIGIN
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/basic.ll b/llvm/test/Instrumentation/DataFlowSanitizer/basic.ll
index 98fb755992bf9b7ccf2132bb3a96fc3b48caf689..e45a8b4fc3da77f2f04a0115490af3e60bf92b9d 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/basic.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/basic.ll
@@ -1,5 +1,6 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,CHECK_NO_ORIGIN -DSHADOW_MASK=-123145302310913
 ; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN -DSHADOW_MASK=-123145302310913
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -S | FileCheck %s --check-prefixes=CHECK,CHECK_NO_ORIGIN -DSHADOW_MASK=-105553116266497
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/call.ll b/llvm/test/Instrumentation/DataFlowSanitizer/call.ll
index cbe62fa37e3e0424da5c9eaf93742980941642fa..bb47d8c6ba30aa7a8031937a046bacf35258d296 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/call.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/call.ll
@@ -1,5 +1,6 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels -S | FileCheck %s
 ; RUN: opt < %s -passes=dfsan -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/external_mask.ll b/llvm/test/Instrumentation/DataFlowSanitizer/external_mask.ll
index 5b538febb3e3d327702e6bbd3ee6f1ffaebebf18..2b5cd5e3065c41eff41664a965d16c7ad1bdaf5f 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/external_mask.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/external_mask.ll
@@ -1,5 +1,6 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,CHECK16
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK16
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
 
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
index 00f05d0c8fb1a6dcd012fa7e9dc794948832faef..c6b998fb28e65531f6e76b24546f9b31ed3e2ce3 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
@@ -1,6 +1,7 @@
 ; Test that -dfsan-fast-16-labels mode uses inline ORs rather than calling
 ; __dfsan_union or __dfsan_union_load.
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels -S | FileCheck %s --implicit-check-not="call{{.*}}__dfsan_union" --check-prefixes=CHECK,CHECK16
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels -S | FileCheck %s --implicit-check-not="call{{.*}}__dfsan_union" --check-prefixes=CHECK,CHECK8
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -13,7 +14,7 @@ define i8 @add(i8 %a, i8 %b) {
   ; CHECK-LABEL: define i8 @"dfs$add"
   ; CHECK-DAG: %[[ALABEL:.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
   ; CHECK-DAG: %[[BLABEL:.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK: %[[ADDLABEL:.*]] = or i16 %[[ALABEL]], %[[BLABEL]]
+  ; CHECK: %[[ADDLABEL:.*]] = or i[[#SBITS]] %[[ALABEL]], %[[BLABEL]]
   ; CHECK: %c = add i8 %a, %b
   ; CHECK: store i[[#SBITS]] %[[ADDLABEL]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; CHECK: ret i8 %c
@@ -24,7 +25,7 @@ define i8 @add(i8 %a, i8 %b) {
 define i8 @load8(i8* %p) {
   ; CHECK-LABEL:  define i8 @"dfs$load8"
   ; CHECK-SAME:   (i8* %[[PADDR:.*]])
-  ; CHECK-NEXT:   %[[#ARG:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; CHECK-NEXT:   %[[#ARG:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; CHECK-NEXT:   %[[#R:]] = ptrtoint i8* %[[PADDR]] to i64
   ; CHECK-NEXT:   %[[#PS:R+1]] = and i64 %[[#R]], [[#%.10d,MASK:]]
   ; CHECK16-NEXT: %[[#PS:R+2]] = mul i64 %[[#R+1]], 2
@@ -106,6 +107,16 @@ define i64 @load64(i64* %p) {
   ; CHECK16-NEXT: %[[#WS+5]]        = trunc i64 %[[#WS+4]] to i[[#SBITS]]
   ; CHECK16-NEXT: %[[#S_OUT:]]      = or i[[#SBITS]] %[[#WS+5]], %[[#ARG]]
 
+  ; COMM: On fast8, no need to OR the wide shadow but one more shift is needed.
+  ; CHECK8-NEXT: %[[#WS+1]]         = lshr i64 %[[#WS]], 32
+  ; CHECK8-NEXT: %[[#WS+2]]         = or i64 %[[#WS]], %[[#WS+1]]
+  ; CHECK8-NEXT: %[[#WS+3]]         = lshr i64 %[[#WS+2]], 16
+  ; CHECK8-NEXT: %[[#WS+4]]         = or i64 %[[#WS+2]], %[[#WS+3]]
+  ; CHECK8-NEXT: %[[#WS+5]]         = lshr i64 %[[#WS+4]], 8
+  ; CHECK8-NEXT: %[[#WS+6]]         = or i64 %[[#WS+4]], %[[#WS+5]]
+  ; CHECK8-NEXT: %[[#WS+7]]         = trunc i64 %[[#WS+6]] to i[[#SBITS]]
+  ; CHECK8-NEXT: %[[#S_OUT:]]       = or i[[#SBITS]] %[[#WS+7]], %[[#ARG]]
+
   ; CHECK-NEXT:   %a = load i64, i64* %p
   ; CHECK-NEXT:   store i[[#SBITS]] %[[#S_OUT]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; CHECK-NEXT:   ret i64 %a
@@ -142,6 +153,16 @@ define i128 @load128(i128* %p) {
   ; CHECK16-NEXT: %[[#WS+5]]    = trunc i64 %[[#WS+4]] to i[[#SBITS]]
   ; CHECK16-NEXT: %[[#S_OUT:]]  = or i[[#SBITS]] %[[#WS+5]], %[[#ARG]]
 
+  ; COMM: On fast8, we need to OR 2x64bits for the wide shadow, before ORing its bytes (one more shift).
+  ; CHECK8-NEXT: %[[#WS+1]]     = lshr i64 %[[#WS]], 32
+  ; CHECK8-NEXT: %[[#WS+2]]     = or i64 %[[#WS]], %[[#WS+1]]
+  ; CHECK8-NEXT: %[[#WS+3]]     = lshr i64 %[[#WS+2]], 16
+  ; CHECK8-NEXT: %[[#WS+4]]     = or i64 %[[#WS+2]], %[[#WS+3]]
+  ; CHECK8-NEXT: %[[#WS+5]]     = lshr i64 %[[#WS+4]], 8
+  ; CHECK8-NEXT: %[[#WS+6]]     = or i64 %[[#WS+4]], %[[#WS+5]]
+  ; CHECK8-NEXT: %[[#WS+7]]     = trunc i64 %[[#WS+6]] to i[[#SBITS]]
+  ; CHECK8-NEXT: %[[#S_OUT:]]   = or i[[#SBITS]] %[[#WS+7]], %[[#ARG]]
+
   ; CHECK-NEXT: %a = load i128, i128* %p
   ; CHECK-NEXT: store i[[#SBITS]] %[[#S_OUT]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; CHECK-NEXT: ret i128 %a
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/phi.ll b/llvm/test/Instrumentation/DataFlowSanitizer/phi.ll
index e9ef73ecca802ee6fe6d380de8d482057cb2442c..fe2b0ba3b47fc43ca1c7e497e6f72716fecd3fb8 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/phi.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/phi.ll
@@ -1,5 +1,6 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,LEGACY
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -11,7 +12,7 @@ define {i32, i32} @test({i32, i32} %a, i1 %c) {
   ; LEGACY: [[PL:%.*]] = phi i[[#SBITS]] [ [[AL]], %T ], [ [[AL]], %F ]
   ; LEGACY: store i[[#SBITS]] [[PL]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
 
-  ; FAST: [[AL:%.*]] = load { [[ST:i[0-9]+]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
+  ; FAST: [[AL:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
   ; FAST: [[AL0:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[AL]], i[[#SBITS]] 0, 0
   ; FAST: [[AL1:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[AL]], i[[#SBITS]] 0, 1
   ; FAST: [[PL:%.*]] = phi { i[[#SBITS]], i[[#SBITS]] } [ [[AL0]], %T ], [ [[AL1]], %F ]
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/select.ll b/llvm/test/Instrumentation/DataFlowSanitizer/select.ll
index e15e9932ec7b5a70869fecc7acb144713a13edbf..6a3355b4a649a32404e754544937dea43f5aa2e4 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/select.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/select.ll
@@ -2,6 +2,8 @@
 ; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=0 -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CF,NO_TRACK_CF_LEGACY
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels -dfsan-track-select-control-flow=1 -S | FileCheck %s --check-prefixes=CHECK,TRACK_CF,TRACK_CF_FAST
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels -dfsan-track-select-control-flow=0 -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CF,NO_TRACK_CF_FAST
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels -dfsan-track-select-control-flow=1 -S | FileCheck %s --check-prefixes=CHECK,TRACK_CF,TRACK_CF_FAST
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels -dfsan-track-select-control-flow=0 -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CF,NO_TRACK_CF_FAST
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/shadow-args-zext.ll b/llvm/test/Instrumentation/DataFlowSanitizer/shadow-args-zext.ll
index a10edd5792d70af4f3e54192e0764462715db205..9c7440e5ebaf767d0f50e8b1dd95c2e2ddbd7f65 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/shadow-args-zext.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/shadow-args-zext.ll
@@ -1,5 +1,6 @@
 ; RUN: opt -mtriple=x86_64-unknown-linux-gnu < %s -dfsan -S --dfsan-abilist=%S/Inputs/shadow-args-abilist.txt | FileCheck %s
 ; RUN: opt -mtriple=x86_64-unknown-linux-gnu < %s -dfsan -S --dfsan-abilist=%S/Inputs/shadow-args-abilist.txt -dfsan-fast-16-labels | FileCheck %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu < %s -dfsan -S --dfsan-abilist=%S/Inputs/shadow-args-abilist.txt -dfsan-fast-8-labels | FileCheck %s
 
 ; REQUIRES: x86-registered-target
 
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/store.ll b/llvm/test/Instrumentation/DataFlowSanitizer/store.ll
index a66b6c86dfde5d885046496394ff09d4b33fa5f8..660656f57d2619c73c896dac22611445bb406ca4 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/store.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/store.ll
@@ -2,6 +2,8 @@
 ; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-store=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK16,NO_COMBINE_PTR_LABEL
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels -dfsan-combine-pointer-labels-on-store=1 -S | FileCheck %s --check-prefixes=CHECK,CHECK16,COMBINE_PTR_LABEL_FAST
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels -dfsan-combine-pointer-labels-on-store=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK16,NO_COMBINE_PTR_LABEL
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels -dfsan-combine-pointer-labels-on-store=1 -S | FileCheck %s --check-prefixes=CHECK,COMBINE_PTR_LABEL_FAST
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels -dfsan-combine-pointer-labels-on-store=0 -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_PTR_LABEL
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/struct.ll b/llvm/test/Instrumentation/DataFlowSanitizer/struct.ll
index f45a1597cec429ceca38e92845d8bb4d7ad53a7b..4d68a2c0577c72de89ac72e6765e640cb03b3cb4 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/struct.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/struct.ll
@@ -1,11 +1,16 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,LEGACY
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-event-callbacks=true -S | FileCheck %s --check-prefixes=CHECK,EVENT_CALLBACKS
 ; RUN: opt < %s -dfsan -dfsan-args-abi -S | FileCheck %s --check-prefixes=CHECK,ARGS_ABI
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST16
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-track-select-control-flow=false -S | FileCheck %s --check-prefixes=CHECK,NO_SELECT_CONTROL
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-track-select-control-flow=false -S | FileCheck %s --check-prefixes=CHECK,NO_SELECT_CONTROL
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -73,18 +78,18 @@ define {i1, i32} @select_struct(i1 %c, {i1, i32} %a, {i1, i32} %b) {
   ; NO_SELECT_CONTROL: [[S:%.*]] = select i1 %c, { i[[#SBITS]], i[[#SBITS]] } [[A]], { i[[#SBITS]], i[[#SBITS]] } [[B]]
   ; NO_SELECT_CONTROL: store { i[[#SBITS]], i[[#SBITS]] } [[S]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
 
-  ; FAST16: @"dfs$select_struct"
-  ; FAST16: %[[#R:]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(2, SBYTES) + 2]]) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
-  ; FAST16: %[[#R+1]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
-  ; FAST16: %[[#R+2]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; FAST16: %[[#R+3]] = select i1 %c, { i[[#SBITS]], i[[#SBITS]] } %[[#R+1]], { i[[#SBITS]], i[[#SBITS]] } %[[#R]]
-  ; FAST16: %[[#R+4]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } %[[#R+3]], 0
-  ; FAST16: %[[#R+5]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } %[[#R+3]], 1
-  ; FAST16: %[[#R+6]] = or i[[#SBITS]] %[[#R+4]], %[[#R+5]]
-  ; FAST16: %[[#R+7]] = or i[[#SBITS]] %[[#R+2]], %[[#R+6]]
-  ; FAST16: %[[#R+8]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } undef, i[[#SBITS]] %[[#R+7]], 0
-  ; FAST16: %[[#R+9]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } %[[#R+8]], i[[#SBITS]] %[[#R+7]], 1
-  ; FAST16: store { i[[#SBITS]], i[[#SBITS]] } %[[#R+9]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: @"dfs$select_struct"
+  ; FAST: %[[#R:]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(2, SBYTES) + 2]]) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
+  ; FAST: %[[#R+1]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: %[[#R+2]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; FAST: %[[#R+3]] = select i1 %c, { i[[#SBITS]], i[[#SBITS]] } %[[#R+1]], { i[[#SBITS]], i[[#SBITS]] } %[[#R]]
+  ; FAST: %[[#R+4]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } %[[#R+3]], 0
+  ; FAST: %[[#R+5]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } %[[#R+3]], 1
+  ; FAST: %[[#R+6]] = or i[[#SBITS]] %[[#R+4]], %[[#R+5]]
+  ; FAST: %[[#R+7]] = or i[[#SBITS]] %[[#R+2]], %[[#R+6]]
+  ; FAST: %[[#R+8]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } undef, i[[#SBITS]] %[[#R+7]], 0
+  ; FAST: %[[#R+9]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } %[[#R+8]], i[[#SBITS]] %[[#R+7]], 1
+  ; FAST: store { i[[#SBITS]], i[[#SBITS]] } %[[#R+9]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
 
   ; LEGACY: @"dfs$select_struct"
   ; LEGACY: [[U:%.*]] = call zeroext i[[#SBITS]] @__dfsan_union
@@ -96,13 +101,13 @@ define {i1, i32} @select_struct(i1 %c, {i1, i32} %a, {i1, i32} %b) {
 }
 
 define { i32, i32 } @asm_struct(i32 %0, i32 %1) {
-  ; FAST16: @"dfs$asm_struct"
-  ; FAST16: [[E1:%.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; FAST16: [[E0:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; FAST16: [[E01:%.*]] = or i[[#SBITS]] [[E0]], [[E1]]
-  ; FAST16: [[S0:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } undef, i[[#SBITS]] [[E01]], 0
-  ; FAST16: [[S1:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[S0]], i[[#SBITS]] [[E01]], 1
-  ; FAST16: store { i[[#SBITS]], i[[#SBITS]] } [[S1]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: @"dfs$asm_struct"
+  ; FAST: [[E1:%.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN:2]]
+  ; FAST: [[E0:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; FAST: [[E01:%.*]] = or i[[#SBITS]] [[E0]], [[E1]]
+  ; FAST: [[S0:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } undef, i[[#SBITS]] [[E01]], 0
+  ; FAST: [[S1:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[S0]], i[[#SBITS]] [[E01]], 1
+  ; FAST: store { i[[#SBITS]], i[[#SBITS]] } [[S1]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
 
   ; LEGACY: @"dfs$asm_struct"
   ; LEGACY: [[E1:%.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN:2]]
@@ -117,8 +122,8 @@ entry:
 }
 
 define {i32, i32} @const_struct() {
-  ; FAST16: @"dfs$const_struct"
-  ; FAST16: store { i[[#SBITS]], i[[#SBITS]] } zeroinitializer, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align 2
+  ; FAST: @"dfs$const_struct"
+  ; FAST: store { i[[#SBITS]], i[[#SBITS]] } zeroinitializer, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align 2
 
   ; LEGACY: @"dfs$const_struct"
   ; LEGACY: store i[[#SBITS]] 0, i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align 2
@@ -126,10 +131,10 @@ define {i32, i32} @const_struct() {
 }
 
 define i1 @extract_struct({i1, i5} %s) {
-  ; FAST16: @"dfs$extract_struct"
-  ; FAST16: [[SM:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
-  ; FAST16: [[EM:%.*]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } [[SM]], 0
-  ; FAST16: store i[[#SBITS]] [[EM]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; FAST: @"dfs$extract_struct"
+  ; FAST: [[SM:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
+  ; FAST: [[EM:%.*]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } [[SM]], 0
+  ; FAST: store i[[#SBITS]] [[EM]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
 
   ; LEGACY: @"dfs$extract_struct"
   ; LEGACY: [[SM:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
@@ -139,11 +144,11 @@ define i1 @extract_struct({i1, i5} %s) {
 }
 
 define {i1, i5} @insert_struct({i1, i5} %s, i5 %e1) {
-  ; FAST16: @"dfs$insert_struct"
-  ; FAST16: [[EM:%.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(2, SBYTES)]]) to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; FAST16: [[SM:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
-  ; FAST16: [[SM1:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[SM]], i[[#SBITS]] [[EM]], 1
-  ; FAST16: store { i[[#SBITS]], i[[#SBITS]] } [[SM1]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: @"dfs$insert_struct"
+  ; FAST: [[EM:%.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(2, SBYTES)]]) to i[[#SBITS]]*), align [[ALIGN:2]]
+  ; FAST: [[SM:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: [[SM1:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[SM]], i[[#SBITS]] [[EM]], 1
+  ; FAST: store { i[[#SBITS]], i[[#SBITS]] } [[SM1]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
 
   ; LEGACY: @"dfs$insert_struct"
   ; LEGACY: [[EM:%.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN:2]]
@@ -173,15 +178,15 @@ define {i1, i1} @load_struct({i1, i1}* %p) {
 }
 
 define void @store_struct({i1, i1}* %p, {i1, i1} %s) {
-  ; FAST16: @"dfs$store_struct"
-  ; FAST16: [[S:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
-  ; FAST16: [[E0:%.*]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } [[S]], 0
-  ; FAST16: [[E1:%.*]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } [[S]], 1
-  ; FAST16: [[E:%.*]] = or i[[#SBITS]] [[E0]], [[E1]]
-  ; FAST16: [[P0:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[P:%.*]], i32 0
-  ; FAST16: store i[[#SBITS]] [[E]], i[[#SBITS]]* [[P0]], align [[#SBYTES]]
-  ; FAST16: [[P1:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[P]], i32 1
-  ; FAST16: store i[[#SBITS]] [[E]], i[[#SBITS]]* [[P1]], align [[#SBYTES]]
+  ; FAST: @"dfs$store_struct"
+  ; FAST: [[S:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
+  ; FAST: [[E0:%.*]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } [[S]], 0
+  ; FAST: [[E1:%.*]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } [[S]], 1
+  ; FAST: [[E:%.*]] = or i[[#SBITS]] [[E0]], [[E1]]
+  ; FAST: [[P0:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[P:%.*]], i32 0
+  ; FAST: store i[[#SBITS]] [[E]], i[[#SBITS]]* [[P0]], align [[#SBYTES]]
+  ; FAST: [[P1:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[P]], i32 1
+  ; FAST: store i[[#SBITS]] [[E]], i[[#SBITS]]* [[P1]], align [[#SBYTES]]
 
   ; EVENT_CALLBACKS: @"dfs$store_struct"
   ; EVENT_CALLBACKS: [[OL:%.*]] = or i[[#SBITS]]
@@ -204,68 +209,68 @@ define void @store_struct({i1, i1}* %p, {i1, i1} %s) {
 }
 
 define i2 @extract_struct_of_aggregate11(%StructOfAggr %s) {
-  ; FAST16: @"dfs$extract_struct_of_aggregate11"
-  ; FAST16: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
-  ; FAST16: [[E11:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 1, 1
-  ; FAST16: store i[[#SBITS]] [[E11]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; FAST: @"dfs$extract_struct_of_aggregate11"
+  ; FAST: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
+  ; FAST: [[E11:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 1, 1
+  ; FAST: store i[[#SBITS]] [[E11]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
 
   %e11 = extractvalue %StructOfAggr %s, 1, 1
   ret i2 %e11
 }
 
 define [4 x i2] @extract_struct_of_aggregate1(%StructOfAggr %s) {
-  ; FAST16: @"dfs$extract_struct_of_aggregate1"
-  ; FAST16: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
-  ; FAST16: [[E1:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 1
-  ; FAST16: store [4 x i[[#SBITS]]] [[E1]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [4 x i[[#SBITS]]]*), align [[ALIGN]]
+  ; FAST: @"dfs$extract_struct_of_aggregate1"
+  ; FAST: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
+  ; FAST: [[E1:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 1
+  ; FAST: store [4 x i[[#SBITS]]] [[E1]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [4 x i[[#SBITS]]]*), align [[ALIGN]]
   %e1 = extractvalue %StructOfAggr %s, 1
   ret [4 x i2] %e1
 }
 
 define <4 x i3> @extract_struct_of_aggregate2(%StructOfAggr %s) {
-  ; FAST16: @"dfs$extract_struct_of_aggregate2"
-  ; FAST16: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
-  ; FAST16: [[E2:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 2
-  ; FAST16: store i[[#SBITS]] [[E2]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; FAST: @"dfs$extract_struct_of_aggregate2"
+  ; FAST: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
+  ; FAST: [[E2:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 2
+  ; FAST: store i[[#SBITS]] [[E2]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   %e2 = extractvalue %StructOfAggr %s, 2
   ret <4 x i3> %e2
 }
 
 define { i1, i1 } @extract_struct_of_aggregate3(%StructOfAggr %s) {
-  ; FAST16: @"dfs$extract_struct_of_aggregate3"
-  ; FAST16: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
-  ; FAST16: [[E3:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 3
-  ; FAST16: store { i[[#SBITS]], i[[#SBITS]] } [[E3]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: @"dfs$extract_struct_of_aggregate3"
+  ; FAST: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
+  ; FAST: [[E3:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 3
+  ; FAST: store { i[[#SBITS]], i[[#SBITS]] } [[E3]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
   %e3 = extractvalue %StructOfAggr %s, 3
   ret { i1, i1 } %e3
 }
 
 define i1 @extract_struct_of_aggregate31(%StructOfAggr %s) {
-  ; FAST16: @"dfs$extract_struct_of_aggregate31"
-  ; FAST16: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
-  ; FAST16: [[E31:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 3, 1
-  ; FAST16: store i[[#SBITS]] [[E31]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; FAST: @"dfs$extract_struct_of_aggregate31"
+  ; FAST: [[E:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN:2]]
+  ; FAST: [[E31:%.*]] = extractvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[E]], 3, 1
+  ; FAST: store i[[#SBITS]] [[E31]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   %e31 = extractvalue %StructOfAggr %s, 3, 1
   ret i1 %e31
 }
 
 define %StructOfAggr @insert_struct_of_aggregate11(%StructOfAggr %s, i2 %e11) {
-  ; FAST16: @"dfs$insert_struct_of_aggregate11"
-  ; FAST16: [[E11:%.*]]  = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(8, SBYTES)]]) to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; FAST16: [[S:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN]]
-  ; FAST16: [[S1:%.*]] = insertvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[S]], i[[#SBITS]] [[E11]], 1, 1
-  ; FAST16: store { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[S1]], { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN]]
+  ; FAST: @"dfs$insert_struct_of_aggregate11"
+  ; FAST: [[E11:%.*]]  = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(8, SBYTES)]]) to i[[#SBITS]]*), align [[ALIGN:2]]
+  ; FAST: [[S:%.*]] = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN]]
+  ; FAST: [[S1:%.*]] = insertvalue { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[S]], i[[#SBITS]] [[E11]], 1, 1
+  ; FAST: store { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } [[S1]], { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN]]
 
   %s1 = insertvalue %StructOfAggr %s, i2 %e11, 1, 1
   ret %StructOfAggr %s1
 }
 
 define {i8*, i32} @call_struct({i8*, i32} %s) {
-  ; FAST16: @"dfs$call_struct"
-  ; FAST16: [[S:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
-  ; FAST16: store { i[[#SBITS]], i[[#SBITS]] } [[S]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
-  ; FAST16: %_dfsret = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
-  ; FAST16: store { i[[#SBITS]], i[[#SBITS]] } %_dfsret, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: @"dfs$call_struct"
+  ; FAST: [[S:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
+  ; FAST: store { i[[#SBITS]], i[[#SBITS]] } [[S]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: %_dfsret = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: store { i[[#SBITS]], i[[#SBITS]] } %_dfsret, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
 
   %r = call {i8*, i32} @pass_struct({i8*, i32} %s)
   ret {i8*, i32} %r
@@ -274,15 +279,15 @@ define {i8*, i32} @call_struct({i8*, i32} %s) {
 declare %StructOfAggr @fun_with_many_aggr_args(<2 x i7> %v, [2 x i5] %a, {i3, i3} %s)
 
 define %StructOfAggr @call_many_aggr_args(<2 x i7> %v, [2 x i5] %a, {i3, i3} %s) {
-  ; FAST16: @"dfs$call_many_aggr_args"
-  ; FAST16: [[S:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(2, SBYTES) + 2]]) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
-  ; FAST16: [[A:%.*]] = load [2 x i[[#SBITS]]], [2 x i[[#SBITS]]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to [2 x i[[#SBITS]]]*), align [[ALIGN]]
-  ; FAST16: [[V:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; FAST16: store i[[#SBITS]] [[V]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; FAST16: store [2 x i[[#SBITS]]] [[A]], [2 x i[[#SBITS]]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to [2 x i[[#SBITS]]]*), align [[ALIGN]]
-  ; FAST16: store { i[[#SBITS]], i[[#SBITS]] } [[S]], { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(2, SBYTES) + 2]]) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
-  ; FAST16: %_dfsret = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN]]
-  ; FAST16: store { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } %_dfsret, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN]]
+  ; FAST: @"dfs$call_many_aggr_args"
+  ; FAST: [[S:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(2, SBYTES) + 2]]) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
+  ; FAST: [[A:%.*]] = load [2 x i[[#SBITS]]], [2 x i[[#SBITS]]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to [2 x i[[#SBITS]]]*), align [[ALIGN]]
+  ; FAST: [[V:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; FAST: store i[[#SBITS]] [[V]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; FAST: store [2 x i[[#SBITS]]] [[A]], [2 x i[[#SBITS]]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to [2 x i[[#SBITS]]]*), align [[ALIGN]]
+  ; FAST: store { i[[#SBITS]], i[[#SBITS]] } [[S]], { i[[#SBITS]], i[[#SBITS]] }* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 [[#mul(2, SBYTES) + 2]]) to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
+  ; FAST: %_dfsret = load { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN]]
+  ; FAST: store { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } } %_dfsret, { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], [4 x i[[#SBITS]]], i[[#SBITS]], { i[[#SBITS]], i[[#SBITS]] } }*), align [[ALIGN]]
 
   %r = call %StructOfAggr @fun_with_many_aggr_args(<2 x i7> %v, [2 x i5] %a, {i3, i3} %s)
   ret %StructOfAggr %r
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/vector.ll b/llvm/test/Instrumentation/DataFlowSanitizer/vector.ll
index 0a2d29c68bddc76ed4a08ab4946cd274a6725e04..51de5f620c7160d73c5dda86932fb1d27ab51a88 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/vector.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/vector.ll
@@ -1,6 +1,7 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI,TLS_ABI_LEGACY
 ; RUN: opt < %s -dfsan -dfsan-args-abi -S | FileCheck %s --check-prefixes=CHECK,ARGS_ABI
 ; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI,TLS_ABI_FAST
+; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI,TLS_ABI_FAST
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll
index 56676b6fd432b3f029d44fd409d4d92e37354a99..e85fc70fecfac9556c8b4c46d20be91694689e6e 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll
@@ -9,12 +9,9 @@ define void @atomicrmw(i64* %ptr) sanitize_hwaddress {
 ; CHECK-LABEL: @atomicrmw(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %ptr to i64
 
-; CHECK: call void asm sideeffect "int3\0Anopl 83(%rax)", "{rdi}"(i64 %[[A]])
+; CHECK: call void @__hwasan_store8(i64 %[[A]])
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %ptr to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i64*
-; CHECK: atomicrmw add i64* %[[UNTAGGED_PTR]], i64 1 seq_cst
+; CHECK: atomicrmw add i64* %ptr, i64 1 seq_cst
 ; CHECK: ret void
 
 entry:
@@ -26,12 +23,9 @@ define void @cmpxchg(i64* %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddr
 ; CHECK-LABEL: @cmpxchg(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %ptr to i64
 
-; CHECK: call void asm sideeffect "int3\0Anopl 83(%rax)", "{rdi}"(i64 %[[A]])
+; CHECK: call void @__hwasan_store8(i64 %[[A]])
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %ptr to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i64*
-; CHECK: cmpxchg i64* %[[UNTAGGED_PTR]], i64 %compare_to, i64 %new_value seq_cst seq_cst
+; CHECK: cmpxchg i64* %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
 ; CHECK: ret void
 
 entry:
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll
index a35dcff87381a2d59b5bf3ff4511f50f194a124e..59e73c5f208195bb41d82ac30f5a508723785f0e 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll
@@ -12,15 +12,10 @@ define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
 
-; ABORT: call void asm sideeffect "int3\0Anopl 64(%rax)", "{rdi}"(i64 %[[A]])
-; ABORT: unreachable
-; RECOVER: call void asm sideeffect "int3\0Anopl 96(%rax)", "{rdi}"(i64 %[[A]])
-; RECOVER: br label
+; ABORT: call void @__hwasan_load1(i64 %[[A]])
+; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]])
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i8*
-; CHECK: %[[G:[^ ]*]] = load i8, i8* %[[UNTAGGED_PTR]], align 4
+; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
 
 entry:
@@ -35,10 +30,7 @@ define i40 @test_load40(i40* %a) sanitize_hwaddress {
 ; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
 ; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i40*
-; CHECK: %[[B:[^ ]*]] = load i40, i40* %[[UNTAGGED_PTR]]
+; CHECK: %[[B:[^ ]*]] = load i40, i40* %a
 ; CHECK: ret i40 %[[B]]
 
 entry:
@@ -50,15 +42,10 @@ define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
 
-; ABORT: call void asm sideeffect "int3\0Anopl 80(%rax)", "{rdi}"(i64 %[[A]])
-; ABORT: unreachable
-; RECOVER: call void asm sideeffect "int3\0Anopl 112(%rax)", "{rdi}"(i64 %[[A]])
-; RECOVER: br label
+; ABORT: call void @__hwasan_store1(i64 %[[A]])
+; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]])
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i8*
-; CHECK: store i8 %b, i8* %[[UNTAGGED_PTR]], align 4
+; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
 
 entry:
@@ -73,10 +60,7 @@ define void @test_store40(i40* %a, i40 %b) sanitize_hwaddress {
 ; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
 ; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i40*
-; CHECK: store i40 %b, i40* %[[UNTAGGED_PTR]]
+; CHECK: store i40 %b, i40* %a
 ; CHECK: ret void
 
 entry:
@@ -91,10 +75,7 @@ define void @test_store_unaligned(i64* %a, i64 %b) sanitize_hwaddress {
 ; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 8)
 ; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 8)
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i64*
-; CHECK: store i64 %b, i64* %[[UNTAGGED_PTR]], align 4
+; CHECK: store i64 %b, i64* %a, align 4
 ; CHECK: ret void
 
 entry:
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll
index 06ec63c859b148c9731087832379b0abacff160d..7cea081f6dee111c5c4c1b912bc76701cc1c648d 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/kernel.ll
@@ -15,15 +15,10 @@ define i8 @test_load(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
 
-; ABORT: call void asm sideeffect "int3\0Anopl 64(%rax)", "{rdi}"(i64 %[[A]])
-; ABORT: unreachable
-; RECOVER: call void asm sideeffect "int3\0Anopl 96(%rax)", "{rdi}"(i64 %[[A]])
-; RECOVER: br label
+; ABORT: call void @__hwasan_load1(i64 %[[A]])
+; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]])
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = or i64 %[[A]], -72057594037927936
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i8*
-; CHECK: %[[G:[^ ]*]] = load i8, i8* %[[UNTAGGED_PTR]], align 4
+; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
 
 entry:
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/with-calls.ll
index c6fce2fe2cacdb17e60a7b79716aebf30651df3c..60d2f047b7f1b4d2deeaa8c38563dd30f274d571 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/with-calls.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/with-calls.ll
@@ -13,10 +13,7 @@ define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; ABORT: call void @__hwasan_load1(i64 %[[A]])
 ; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]])
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i8*
-; CHECK: %[[B:[^ ]*]] = load i8, i8* %[[UNTAGGED_PTR]]
+; CHECK: %[[B:[^ ]*]] = load i8, i8* %a
 ; CHECK: ret i8 %[[B]]
 
 entry:
@@ -31,10 +28,7 @@ define i40 @test_load40(i40* %a) sanitize_hwaddress {
 ; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
 ; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i40*
-; CHECK: %[[B:[^ ]*]] = load i40, i40* %[[UNTAGGED_PTR]]
+; CHECK: %[[B:[^ ]*]] = load i40, i40* %a
 ; CHECK: ret i40 %[[B]]
 
 entry:
@@ -49,10 +43,7 @@ define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; ABORT: call void @__hwasan_store1(i64 %[[A]])
 ; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]])
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i8*
-; CHECK: store i8 %b, i8* %[[UNTAGGED_PTR]]
+; CHECK: store i8 %b, i8* %a
 ; CHECK: ret void
 
 entry:
@@ -67,10 +58,7 @@ define void @test_store40(i40* %a, i40 %b) sanitize_hwaddress {
 ; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
 ; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
 
-; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
-; CHECK: %[[UNTAGGED:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[UNTAGGED_PTR:[^ ]*]] = inttoptr i64 %[[UNTAGGED]] to i40*
-; CHECK: store i40 %b, i40* %[[UNTAGGED_PTR]]
+; CHECK: store i40 %b, i40* %a
 ; CHECK: ret void
 
 entry:
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/fuchsia.ll b/llvm/test/Instrumentation/HWAddressSanitizer/fuchsia.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9005830008359e34049f04e78e4e898344cf2ed2
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/fuchsia.ll
@@ -0,0 +1,9 @@
+; Check HWASan shadow mapping on Fuchsia.
+; RUN: opt -hwasan -S -mtriple=aarch64-unknown-fuchsia < %s | FileCheck %s
+
+define i32 @test_load(i32* %a) sanitize_hwaddress {
+; CHECK: %.hwasan.shadow = call i8* asm "", "=r,0"(i8* null)
+entry:
+  %x = load i32, i32* %a, align 4
+  ret i32 %x
+}
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
index 0134c7ea92c145e4ef76f0ec170ea70fafce1938..445a96d1e8c3c690bb25176f15c9b11985ea5755 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
@@ -11,3 +11,5 @@ entry:
 
 ; CHECK-DAG: section ".SCOV{{\$}}CM",
 ; CHECK-DAG: section ".SCOVP{{\$}}M",
+; CHECK:     @__start___sancov_cntrs = external hidden global i8
+; CHECK:     @__stop___sancov_cntrs = external hidden global i8
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-bool-flag.ll b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-bool-flag.ll
index d91714ba8fcf6ce147a8b2d0aa8dec31be141906..2f046f9587b802b8577ea53fdf0fc1f154fc7b44 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-bool-flag.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coff-pc-table-inline-bool-flag.ll
@@ -11,3 +11,5 @@ entry:
 
 ; CHECK-DAG: section ".SCOV{{\$}}BM",
 ; CHECK-DAG: section ".SCOVP{{\$}}M",
+; CHECK:     @__start___sancov_bools = external hidden global i1
+; CHECK:     @__stop___sancov_bools = external hidden global i1
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll b/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
index 4f905428769a426f243624676e81a28473f3ab47..3611cba34ce5e0617530e3acce1b456c5acaebdc 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
@@ -2,11 +2,14 @@
 ; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1  -S -enable-new-pm=0 | FileCheck %s
 ; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1  -S | FileCheck %s
 
+; CHECK:      @__sancov_gen_ = private global [1 x i8] zeroinitializer, section "__sancov_cntrs", comdat($foo), align 1
+; CHECK:      @__start___sancov_cntrs = extern_weak hidden global i8
+; CHECK-NEXT: @__stop___sancov_cntrs = extern_weak hidden global i8
+
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 define void @foo() {
 entry:
-; CHECK: section "__sancov_cntrs", comdat($foo), align 1
 ; CHECK:  %0 = load i8, i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__sancov_gen_, i64 0, i64 0), align 1, !nosanitize
 ; CHECK:  %1 = add i8 %0, 1
 ; CHECK:  store i8 %1, i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__sancov_gen_, i64 0, i64 0), align 1, !nosanitize
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/inline-bool-flag.ll b/llvm/test/Instrumentation/SanitizerCoverage/inline-bool-flag.ll
index e711d96a5a438c1c97d50ff7a2c64930a5cb8766..48a4c60d98b2b3d0301e27a5ae3b0d52d3ea6dab 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/inline-bool-flag.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/inline-bool-flag.ll
@@ -4,6 +4,8 @@
 
 ; CHECK:      $foo = comdat noduplicates
 ; CHECK:      @__sancov_gen_ = private global [1 x i1] zeroinitializer, section "__sancov_bools", comdat($foo), align 1{{$}}
+; CHECK:      @__start___sancov_bools = extern_weak hidden global i1
+; CHECK-NEXT: @__stop___sancov_bools = extern_weak hidden global i1
 ; CHECK-NOT:  @llvm.used =
 ; CHECK:      @llvm.compiler.used = appending global [1 x i8*] [i8* bitcast ([1 x i1]* @__sancov_gen_ to i8*)], section "llvm.metadata"
 
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll b/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll
index eeeb56bfb2abd3d47d0670ecc15550bb3b6202dc..8ebb6a05ca7c2e4123d8847672cdc117dd9fa7f8 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/pc-table.ll
@@ -22,6 +22,8 @@ entry:
 }
 
 ; CHECK: private constant [6 x i64*] [{{.*}}@foo{{.*}}blockaddress{{.*}}blockaddress{{.*}}], section "__sancov_pcs", comdat($foo), align 8
+; CHECK:      @__start___sancov_pcs = extern_weak hidden global i64
+; CHECK-NEXT: @__stop___sancov_pcs = extern_weak hidden global i64
 ; CHECK: define internal void @sancov.module_ctor
 ; CHECK: call void @__sanitizer_cov
 ; CHECK: call void @__sanitizer_cov_pcs_init
diff --git a/llvm/test/LTO/X86/inline-asm-lto-discard.ll b/llvm/test/LTO/X86/inline-asm-lto-discard.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4893eb186cfbb6c7b56fd0a4119d47d7799466b7
--- /dev/null
+++ b/llvm/test/LTO/X86/inline-asm-lto-discard.ll
@@ -0,0 +1,87 @@
+; Check that non-prevailing symbols in module inline assembly are discarded
+; during regular LTO otherwise the final symbol binding could be wrong.
+
+; RUN: split-file %s %t
+; RUN: opt %t/t1.ll -o %t1
+; RUN: opt %t/t2.ll -o %t2
+; RUN: opt %t/t3.ll -o %t3
+; RUN: opt %t/t4.ll -o %t4
+
+; RUN: llvm-lto2 run -o %to1 -save-temps %t1 %t2 \
+; RUN:  -r %t1,foo,px \
+; RUN:  -r %t2,foo, \
+; RUN:  -r %t2,bar,pl
+; RUN: llvm-dis < %to1.0.0.preopt.bc | FileCheck %s --check-prefix=ASM1
+; RUN: llvm-nm %to1.0 | FileCheck %s --check-prefix=SYM
+; RUN: llvm-objdump -d --disassemble-symbols=foo %to1.0 \
+; RUN:   | FileCheck %s --check-prefix=DEF
+
+; RUN: llvm-lto2 run -o %to2 -save-temps %t2 %t3 \
+; RUN:  -r %t2,foo, \
+; RUN:  -r %t2,bar,pl \
+; RUN:  -r %t3,foo,px
+; RUN: llvm-dis < %to2.0.0.preopt.bc | FileCheck %s --check-prefix=ASM2
+; RUN: llvm-nm %to2.0 | FileCheck %s --check-prefix=SYM
+; RUN: llvm-objdump -d --disassemble-symbols=foo %to2.0 \
+; RUN:   | FileCheck %s --check-prefix=DEF
+
+; Check that ".symver" is properly handled.
+; RUN: llvm-lto2 run -o %to3 -save-temps %t4 \
+; RUN:  -r %t4,bar, \
+; RUN:  -r %t4,foo, \
+; RUN:  -r %t4,foo@@VER1,px
+; RUN: llvm-dis < %to3.0.0.preopt.bc | FileCheck %s --check-prefix=ASM3
+
+; ASM1:      module asm ".lto_discard foo"
+; ASM1-NEXT: module asm ".weak foo"
+; ASM1-NEXT: module asm ".equ foo,bar"
+
+; ASM2:      module asm ".lto_discard foo"
+; ASM2-NEXT: module asm ".weak foo"
+; ASM2-NEXT: module asm ".equ foo,bar"
+; ASM2-NEXT: module asm ".lto_discard"
+; ASM2-NEXT: module asm " .global foo ; foo: leal    2(%rdi), %eax"
+
+; ASM3-NOT:  module asm ".lto_discard foo"
+
+; SYM: T foo
+
+; DEF: leal    2(%rdi), %eax
+
+;--- t1.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local i32 @foo(i32 %0) {
+  %2 = add nsw i32 %0, 2
+  ret i32 %2
+}
+
+;--- t2.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+module asm ".weak foo"
+module asm ".equ foo,bar"
+
+@llvm.compiler.used = appending global [1 x i8*] [i8* bitcast (i32 (i32)* @bar to i8*)], section "llvm.metadata"
+
+define internal i32 @bar(i32 %0) {
+  %2 = add nsw i32 %0, 1
+  ret i32 %2
+}
+
+;--- t3.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+module asm " .global foo ; foo: leal    2(%rdi), %eax"
+
+;--- t4.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+module asm ".global foo"
+module asm "foo: call bar"
+module asm ".symver foo,foo@@@VER1"
+module asm ".symver bar,bar@@@VER1"
diff --git a/llvm/test/LTO/X86/inline-asm-lto-discard2.ll b/llvm/test/LTO/X86/inline-asm-lto-discard2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5d111d0a52e3f899ab3f1e62d358495344a26fc6
--- /dev/null
+++ b/llvm/test/LTO/X86/inline-asm-lto-discard2.ll
@@ -0,0 +1,29 @@
+; Check that
+; 1. ".lto_discard" works as module inlineasm marker and its argument symbols
+;    are discarded.
+; 2. there is no reassignment error in the presence of ".lto_discard"
+; RUN: llc < %s | FileCheck %s
+
+; CHECK:    .data
+; CHECK-NOT:  .weak  foo
+; CHECK-NOT:  .set   foo, bar
+; CHECK:      .globl foo
+; CHECK:      foo:
+; CHECK:        .byte 1
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+module asm ".lto_discard foo"
+module asm "	.text"
+module asm "bar:"
+module asm "	.data"
+module asm ".weak foo"
+module asm ".set   foo, bar"
+module asm ".weak foo"
+module asm ".set   foo, bar"
+
+module asm ".lto_discard"
+module asm ".globl foo"
+module asm "foo:"
+module asm "   .byte 1"
diff --git a/llvm/test/Linker/Inputs/inalloca-type-input.ll b/llvm/test/Linker/Inputs/inalloca-type-input.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7fa2d8fdb3f3dbd5a067cdb6b5675856bce59674
--- /dev/null
+++ b/llvm/test/Linker/Inputs/inalloca-type-input.ll
@@ -0,0 +1,13 @@
+%a = type { i64 }
+%struct = type { i32, i8 }
+
+define void @g(%a* inalloca(%a)) {
+  ret void
+}
+
+declare void @baz(%struct* inalloca(%struct))
+
+define void @foo(%struct* inalloca(%struct) %a) {
+  call void @baz(%struct* inalloca(%struct) %a)
+  ret void
+}
diff --git a/llvm/test/Linker/inalloca-types.ll b/llvm/test/Linker/inalloca-types.ll
new file mode 100644
index 0000000000000000000000000000000000000000..36cc9c3f7ef4a2262edc7d529dc0c1efd8707103
--- /dev/null
+++ b/llvm/test/Linker/inalloca-types.ll
@@ -0,0 +1,25 @@
+; RUN: llvm-link %s %p/Inputs/inalloca-type-input.ll -S | FileCheck %s
+
+%a = type { i64 }
+%struct = type { i32, i8 }
+
+; CHECK-LABEL: define void @f(%a* inalloca(%a) %0)
+define void @f(%a* inalloca(%a)) {
+  ret void
+}
+
+; CHECK-LABEL: define void @bar(
+; CHECK: call void @foo(%struct* inalloca(%struct) %ptr)
+define void @bar() {
+  %ptr = alloca inalloca %struct
+  call void @foo(%struct* inalloca(%struct) %ptr)
+  ret void
+}
+
+; CHECK-LABEL: define void @g(%a* inalloca(%a) %0)
+
+; CHECK-LABEL: define void @foo(%struct* inalloca(%struct) %a)
+; CHECK-NEXT:   call void @baz(%struct* inalloca(%struct) %a)
+declare void @foo(%struct* inalloca(%struct) %a)
+
+; CHECK: declare void @baz(%struct* inalloca(%struct))
diff --git a/llvm/test/Linker/intrinsics-with-unnamed-types.ll b/llvm/test/Linker/intrinsics-with-unnamed-types.ll
new file mode 100644
index 0000000000000000000000000000000000000000..76287f698df9c075cef4bf529e7629291b7f1118
--- /dev/null
+++ b/llvm/test/Linker/intrinsics-with-unnamed-types.ll
@@ -0,0 +1,101 @@
+; RUN: split-file %s %t
+; RUN: llvm-as -o %t1.bc %t/f01.ll
+; RUN: llvm-as -o %t2.bc %t/f02.ll
+; RUN: llvm-link %t1.bc %t2.bc -o %t3.bc
+; RUN: llvm-dis -o - %t3.bc | FileCheck %s
+
+; Make sure we can link files with clashing intrinsic names using unnamed types.
+
+;--- f01.ll
+%1 = type opaque
+%0 = type opaque
+
+; CHECK-LABEL: @test01(
+; CHECK:       %cmp1 = icmp ne %0* %arg, null
+; CHECK-NEXT:  %c1 = call %0* @llvm.ssa.copy.p0s_s.0(%0* %arg)
+; CHECK-NEXT:  %c2 = call %1* @llvm.ssa.copy.p0s_s.1(%1* %tmp)
+; CHECK-NEXT:  %c3a = call %0** @llvm.ssa.copy.p0p0s_s.0(%0** %arg2)
+; CHECK-NEXT:  %c3b = call %0** @llvm.ssa.copy.p0p0s_s.0(%0** %arg2)
+; CHECK-NEXT:  %c4a = call %1** @llvm.ssa.copy.p0p0s_s.1(%1** %tmp2)
+; CHECK-NEXT:  %c4ba = call %1** @llvm.ssa.copy.p0p0s_s.1(%1** %tmp2)
+; CHECK-NEXT:  %c5 = call %0*** @llvm.ssa.copy.p0p0p0s_s.0(%0*** %arg3)
+; CHECK-NEXT:  %c6 = call %1*** @llvm.ssa.copy.p0p0p0s_s.1(%1*** %tmp3)
+
+define void @test01(%0* %arg, %1* %tmp, %1** %tmp2, %0** %arg2, %1*** %tmp3, %0*** %arg3) {
+bb:
+  %cmp1 = icmp ne %0* %arg, null
+  %c1 = call %0* @llvm.ssa.copy.p0s_s.0(%0* %arg)
+  %c2 = call %1* @llvm.ssa.copy.p0s_s.1(%1* %tmp)
+  %c3a = call %0** @llvm.ssa.copy.p0p0s_s.1(%0** %arg2)
+  %c3b = call %0** @llvm.ssa.copy.p0p0s_s.1(%0** %arg2)
+  %c4a = call %1** @llvm.ssa.copy.p0p0s_s.0(%1** %tmp2)
+  %c4ba = call %1** @llvm.ssa.copy.p0p0s_s.0(%1** %tmp2)
+  %c5 = call %0*** @llvm.ssa.copy.p0p0p0s_s.1(%0*** %arg3)
+  %c6 = call %1*** @llvm.ssa.copy.p0p0p0s_s.0(%1*** %tmp3)
+  ret void
+}
+
+declare %0* @llvm.ssa.copy.p0s_s.0(%0* returned)
+
+declare %1* @llvm.ssa.copy.p0s_s.1(%1* returned)
+
+declare %0** @llvm.ssa.copy.p0p0s_s.1(%0** returned)
+
+declare %1** @llvm.ssa.copy.p0p0s_s.0(%1** returned)
+
+declare %0*** @llvm.ssa.copy.p0p0p0s_s.1(%0*** returned)
+
+declare %1*** @llvm.ssa.copy.p0p0p0s_s.0(%1*** returned)
+
+; now with recycling of previous declarations:
+; CHECK-LABEL: @test02(
+; CHECK:       %cmp1 = icmp ne %0* %arg, null
+; CHECK-NEXT:  %c4a = call %1** @llvm.ssa.copy.p0p0s_s.1(%1** %tmp2)
+; CHECK-NEXT:  %c6 = call %1*** @llvm.ssa.copy.p0p0p0s_s.1(%1*** %tmp3)
+; CHECK-NEXT:  %c1 = call %0* @llvm.ssa.copy.p0s_s.0(%0* %arg)
+; CHECK-NEXT:  %c2 = call %1* @llvm.ssa.copy.p0s_s.1(%1* %tmp)
+; CHECK-NEXT:  %c3b = call %0** @llvm.ssa.copy.p0p0s_s.0(%0** %arg2)
+; CHECK-NEXT:  %c4ba = call %1** @llvm.ssa.copy.p0p0s_s.1(%1** %tmp2)
+; CHECK-NEXT:  %c5 = call %0*** @llvm.ssa.copy.p0p0p0s_s.0(%0*** %arg3)
+
+define void @test02(%0* %arg, %1* %tmp, %1** %tmp2, %0** %arg2, %1*** %tmp3, %0*** %arg3) {
+bb:
+  %cmp1 = icmp ne %0* %arg, null
+  %c4a = call %1** @llvm.ssa.copy.p0p0s_s.0(%1** %tmp2)
+  %c6 = call %1*** @llvm.ssa.copy.p0p0p0s_s.0(%1*** %tmp3)
+  %c1 = call %0* @llvm.ssa.copy.p0s_s.0(%0* %arg)
+  %c2 = call %1* @llvm.ssa.copy.p0s_s.1(%1* %tmp)
+  %c3b = call %0** @llvm.ssa.copy.p0p0s_s.1(%0** %arg2)
+  %c4ba = call %1** @llvm.ssa.copy.p0p0s_s.0(%1** %tmp2)
+  %c5 = call %0*** @llvm.ssa.copy.p0p0p0s_s.1(%0*** %arg3)
+  ret void
+}
+
+;--- f02.ll
+%1 = type opaque
+%2 = type opaque
+
+; CHECK-LABEL: @test03(
+; CHECK:      %cmp1 = icmp ne %3* %arg, null
+; CHECK-NEXT: %c1 = call %3* @llvm.ssa.copy.p0s_s.2(%3* %arg)
+; CHECK-NEXT: %c2 = call %2* @llvm.ssa.copy.p0s_s.3(%2* %tmp)
+; CHECK-NEXT: %c3 = call %3** @llvm.ssa.copy.p0p0s_s.2(%3** %arg2)
+; CHECK-NEXT: %c4 = call %2** @llvm.ssa.copy.p0p0s_s.3(%2** %tmp2)
+
+define void @test03(%1* %tmp, %2* %arg, %1** %tmp2, %2** %arg2) {
+bb:
+  %cmp1 = icmp ne %2* %arg, null
+  %c1 = call %2* @llvm.ssa.copy.p0s_s.0(%2* %arg)
+  %c2 = call %1* @llvm.ssa.copy.p0s_s.1(%1* %tmp)
+  %c3 = call %2** @llvm.ssa.copy.p0p0s_s.1(%2** %arg2)
+  %c4 = call %1** @llvm.ssa.copy.p0p0s_s.0(%1** %tmp2)
+  ret void
+}
+
+declare %2* @llvm.ssa.copy.p0s_s.0(%2* returned)
+
+declare %1* @llvm.ssa.copy.p0s_s.1(%1* returned)
+
+declare %2** @llvm.ssa.copy.p0p0s_s.1(%2** returned)
+
+declare %1** @llvm.ssa.copy.p0p0s_s.0(%1** returned)
diff --git a/llvm/test/MC/AArch64/directive-arch.s b/llvm/test/MC/AArch64/directive-arch.s
index 7dfcdb4598f0bf314dea945571cdccd14dbe32a3..2aa2bcbf5665ee5f4771b075cef2b16a226727e3 100644
--- a/llvm/test/MC/AArch64/directive-arch.s
+++ b/llvm/test/MC/AArch64/directive-arch.s
@@ -17,3 +17,9 @@
 	.arch armv8-a+lse
 	casa  w5, w7, [x20]
 # CHECK:        casa    w5, w7, [x20]
+
+	.arch armv8.5-a+rng
+	mrs   x0, rndr
+	mrs   x0, rndrrs
+# CHECK:        mrs     x0, RNDR
+# CHECK:        mrs     x0, RNDRRS
diff --git a/llvm/test/MC/AArch64/directive-arch_extension.s b/llvm/test/MC/AArch64/directive-arch_extension.s
index 8cf2acd509b1b75a2ee2bcb0668e7b89d6da37c5..064c89ace6529a7cb7a6fd822d77cda147d067f3 100644
--- a/llvm/test/MC/AArch64/directive-arch_extension.s
+++ b/llvm/test/MC/AArch64/directive-arch_extension.s
@@ -56,6 +56,10 @@ irg x0, x1
 tlbi vmalle1os
 // CHECK: tlbi vmalle1os
 
+.arch_extension pan
+mrs x0, pan
+// CHECK: mrs x0, PAN
+
 .arch_extension pan-rwv
 at s1e1wp, x2
 // CHECK: at s1e1wp, x2
diff --git a/llvm/test/MC/AMDGPU/gfx10_err_pos.s b/llvm/test/MC/AMDGPU/gfx10_err_pos.s
index e9e2add95d884f701b3e08969704095dce9e67d9..ebd314455c1a9566e209bc2e4a0da2d41ec748c5 100644
--- a/llvm/test/MC/AMDGPU/gfx10_err_pos.s
+++ b/llvm/test/MC/AMDGPU/gfx10_err_pos.s
@@ -935,20 +935,43 @@ v_ceil_f32 v0, --1
 // CHECK-NEXT:{{^}}               ^
 
 //==============================================================================
-// invalid use of lds_direct
+// lane id must be in the interval [0,group size - 1]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,2,-1)
+// CHECK: error: lane id must be in the interval [0,group size - 1]
+// CHECK-NEXT:{{^}}ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,2,-1)
+// CHECK-NEXT:{{^}}                                                 ^
+
+//==============================================================================
+// lds_direct cannot be used with this instruction
 
 v_ashrrev_i16 v0, lds_direct, v0
-// CHECK: error: invalid use of lds_direct
+// CHECK: error: lds_direct cannot be used with this instruction
 // CHECK-NEXT:{{^}}v_ashrrev_i16 v0, lds_direct, v0
 // CHECK-NEXT:{{^}}                  ^
 
+v_ashrrev_i16 v0, v1, lds_direct
+// CHECK: error: lds_direct cannot be used with this instruction
+// CHECK-NEXT:{{^}}v_ashrrev_i16 v0, v1, lds_direct
+// CHECK-NEXT:{{^}}                      ^
+
+v_mov_b32_sdwa v1, src_lds_direct dst_sel:DWORD
+// CHECK: error: lds_direct cannot be used with this instruction
+// CHECK-NEXT:{{^}}v_mov_b32_sdwa v1, src_lds_direct dst_sel:DWORD
+// CHECK-NEXT:{{^}}                   ^
+
+v_add_f32_sdwa v5, v1, lds_direct dst_sel:DWORD
+// CHECK: error: lds_direct cannot be used with this instruction
+// CHECK-NEXT:{{^}}v_add_f32_sdwa v5, v1, lds_direct dst_sel:DWORD
+// CHECK-NEXT:{{^}}                       ^
+
 //==============================================================================
-// lane id must be in the interval [0,group size - 1]
+// lds_direct may be used as src0 only
 
-ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,2,-1)
-// CHECK: error: lane id must be in the interval [0,group size - 1]
-// CHECK-NEXT:{{^}}ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,2,-1)
-// CHECK-NEXT:{{^}}                                                 ^
+v_add_f32 v5, v1, lds_direct
+// CHECK: error: lds_direct may be used as src0 only
+// CHECK-NEXT:{{^}}v_add_f32 v5, v1, lds_direct
+// CHECK-NEXT:{{^}}                  ^
 
 //==============================================================================
 // message does not support operations
diff --git a/llvm/test/MC/AMDGPU/gfx90a_asm_features.s b/llvm/test/MC/AMDGPU/gfx90a_asm_features.s
index 38fa212175d8486785ac745eff8d161e7a2066f9..fac42fd900ced6ddafa54a430ad0ce4b0764dd70 100644
--- a/llvm/test/MC/AMDGPU/gfx90a_asm_features.s
+++ b/llvm/test/MC/AMDGPU/gfx90a_asm_features.s
@@ -1046,3 +1046,7 @@ global_atomic_add_f32 v1, v0, v2, s[0:1] glc ; encoding: [0x00,0x80,0x35,0xdd,0x
 // GFX1010: error: instruction not supported on this GPU
 // GFX90A: global_atomic_pk_add_f16  v0, v[0:1], v2, off glc ; encoding: [0x00,0x80,0x39,0xdd,0x00,0x02,0x7f,0x00]
 global_atomic_pk_add_f16 v0, v[0:1], v2, off glc
+
+// NOT-GFX90A: error: scc modifier is not supported on this GPU
+// GFX90A: buffer_atomic_add v4, off, s[8:11], s3 scc ; encoding: [0x00,0x80,0x08,0xe1,0x00,0x04,0x02,0x03]
+buffer_atomic_add v4, off, s[8:11], s3 scc
diff --git a/llvm/test/MC/AMDGPU/gfx90a_err.s b/llvm/test/MC/AMDGPU/gfx90a_err.s
index 246291961599e73907c3d63fc4497ff031a91861..44c48595ca17c39cd8035448993540f0ba23ec5d 100644
--- a/llvm/test/MC/AMDGPU/gfx90a_err.s
+++ b/llvm/test/MC/AMDGPU/gfx90a_err.s
@@ -230,3 +230,30 @@ global_atomic_min_f64 v[0:1], v[2:3], off scc
 
 global_atomic_max_f64 v[0:1], v[2:3], off scc
 // GFX90A: error: instruction must not use scc
+
+buffer_atomic_add_f32 v4, off, s[8:11], s3 scc
+// GFX90A: error: instruction must not use scc
+
+buffer_atomic_pk_add_f16 v4, off, s[8:11], s3 scc
+// GFX90A: error: instruction must not use scc
+
+buffer_atomic_add_f64 v[4:5], off, s[8:11], s3 scc
+// GFX90A: error: instruction must not use scc
+
+buffer_atomic_max_f64 v[4:5], off, s[8:11], s3 scc
+// GFX90A: error: instruction must not use scc
+
+buffer_atomic_min_f64 v[4:5], off, s[8:11], s3 scc
+// GFX90A: error: instruction must not use scc
+
+v_mov_b32_sdwa v1, src_lds_direct dst_sel:DWORD
+// GFX90A: error: lds_direct is not supported on this GPU
+
+v_add_f32_sdwa v5, v1, lds_direct dst_sel:DWORD
+// GFX90A: error: lds_direct is not supported on this GPU
+
+v_ashrrev_i16 v0, lds_direct, v0
+// GFX90A: error: lds_direct is not supported on this GPU
+
+v_add_f32 v5, v1, lds_direct
+// GFX90A: error: lds_direct is not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/hsa-diag-v3.s b/llvm/test/MC/AMDGPU/hsa-diag-v3.s
index e255e6eb2050ac122129b8504c9ecbc2923977a4..ee4543c9fb50ba30f5f11ff671ac67c02a489165 100644
--- a/llvm/test/MC/AMDGPU/hsa-diag-v3.s
+++ b/llvm/test/MC/AMDGPU/hsa-diag-v3.s
@@ -1,16 +1,16 @@
-// RUN: not llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=GCN,GFX8,NONGFX10,AMDHSA
-// RUN: not llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=GCN,GFX10,AMDHSA
-// RUN: not llvm-mc -triple amdgcn-amd- -mcpu=gfx803 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=GCN,NONAMDHSA
+// RUN: not llvm-mc --amdhsa-code-object-version=3 -triple amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=GCN,GFX8,NONGFX10,AMDHSA
+// RUN: not llvm-mc --amdhsa-code-object-version=3 -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=GCN,GFX10,AMDHSA
+// RUN: not llvm-mc --amdhsa-code-object-version=3 -triple amdgcn-amd- -mcpu=gfx810 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=GCN,NONAMDHSA
 // RUN: not llvm-mc --amdhsa-code-object-version=3 -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=GFX90A,NONGFX10,AMDHSA,ALL
 
 .text
 
 // GCN-LABEL: warning: test_target
 // GFX8-NOT: error:
-// GFX10: error: target must match options
-// NONAMDHSA: error: unknown directive
+// GFX10: error: .amdgcn_target directive's target id amdgcn-amd-amdhsa--gfx810+xnack does not match the specified target id amdgcn-amd-amdhsa--gfx1010+xnack
+// NONAMDHSA: error: .amdgcn_target directive's target id amdgcn-amd-amdhsa--gfx810+xnack does not match the specified target id amdgcn-amd-unknown--gfx810
 .warning "test_target"
-.amdgcn_target "amdgcn-amd-amdhsa--gfx803+xnack"
+.amdgcn_target "amdgcn-amd-amdhsa--gfx810+xnack"
 
 // GCN-LABEL: warning: test_amdhsa_kernel_no_name
 // GCN: error: unknown directive
diff --git a/llvm/test/MC/AMDGPU/hsa-gfx10-v3.s b/llvm/test/MC/AMDGPU/hsa-gfx10-v3.s
index dbf3c96023da928715c3d6971fd5145097a52984..6de81f57bf710b152756980b955683efa64a734d 100644
--- a/llvm/test/MC/AMDGPU/hsa-gfx10-v3.s
+++ b/llvm/test/MC/AMDGPU/hsa-gfx10-v3.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
-// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+xnack -filetype=obj < %s > %t
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 --amdhsa-code-object-version=3 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 --amdhsa-code-object-version=3 -mattr=+xnack -filetype=obj < %s > %t
 // RUN: llvm-readobj -elf-output-style=GNU -sections -symbols -relocations %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
@@ -28,7 +28,7 @@
 // OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0030 0000ac60 80000000 00000000 00000000
 // complete
-// OBJDUMP-NEXT: 0040 01000000 01000000 00000000 00000000
+// OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0070 015001e4 1f0f007f 7f040000 00000000
@@ -80,6 +80,7 @@ special_sgpr:
 .amdhsa_kernel complete
   .amdhsa_group_segment_fixed_size 1
   .amdhsa_private_segment_fixed_size 1
+  .amdhsa_kernarg_size 8
   .amdhsa_user_sgpr_private_segment_buffer 1
   .amdhsa_user_sgpr_dispatch_ptr 1
   .amdhsa_user_sgpr_queue_ptr 1
@@ -98,7 +99,7 @@ special_sgpr:
   .amdhsa_next_free_sgpr 27
   .amdhsa_reserve_vcc 0
   .amdhsa_reserve_flat_scratch 0
-  .amdhsa_reserve_xnack_mask 0
+  .amdhsa_reserve_xnack_mask 1
   .amdhsa_float_round_mode_32 1
   .amdhsa_float_round_mode_16_64 1
   .amdhsa_float_denorm_mode_32 1
@@ -121,6 +122,7 @@ special_sgpr:
 // ASM: .amdhsa_kernel complete
 // ASM-NEXT: .amdhsa_group_segment_fixed_size 1
 // ASM-NEXT: .amdhsa_private_segment_fixed_size 1
+// ASM-NEXT: .amdhsa_kernarg_size 8
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
@@ -139,7 +141,7 @@ special_sgpr:
 // ASM-NEXT: .amdhsa_next_free_sgpr 27
 // ASM-NEXT: .amdhsa_reserve_vcc 0
 // ASM-NEXT: .amdhsa_reserve_flat_scratch 0
-// ASM-NEXT: .amdhsa_reserve_xnack_mask 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
 // ASM-NEXT: .amdhsa_float_round_mode_32 1
 // ASM-NEXT: .amdhsa_float_round_mode_16_64 1
 // ASM-NEXT: .amdhsa_float_denorm_mode_32 1
@@ -169,7 +171,7 @@ special_sgpr:
   .amdhsa_reserve_flat_scratch 1
 
   .amdhsa_reserve_vcc 0
-  .amdhsa_reserve_xnack_mask 0
+  .amdhsa_reserve_xnack_mask 1
 
   .amdhsa_float_denorm_mode_16_64 0
   .amdhsa_dx10_clamp 0
@@ -181,7 +183,7 @@ special_sgpr:
 // ASM: .amdhsa_next_free_vgpr 0
 // ASM-NEXT: .amdhsa_next_free_sgpr 27
 // ASM-NEXT: .amdhsa_reserve_vcc 0
-// ASM-NEXT: .amdhsa_reserve_xnack_mask 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
 // ASM: .amdhsa_float_denorm_mode_16_64 0
 // ASM-NEXT: .amdhsa_dx10_clamp 0
 // ASM-NEXT: .amdhsa_ieee_mode 0
diff --git a/llvm/test/MC/AMDGPU/hsa-v3.s b/llvm/test/MC/AMDGPU/hsa-v3.s
index 92c4d59a2e8d86edbada961db178774538786ce0..c98a7da36e623a8c71a32d7f80f9fd59dee28c73 100644
--- a/llvm/test/MC/AMDGPU/hsa-v3.s
+++ b/llvm/test/MC/AMDGPU/hsa-v3.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
-// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 -mattr=+xnack -filetype=obj < %s > %t
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=3 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=3 -mattr=+xnack -filetype=obj < %s > %t
 // RUN: llvm-readelf -sections -symbols -relocations %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
@@ -31,7 +31,7 @@
 // OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000
 // complete
-// OBJDUMP-NEXT: 0040 01000000 01000000 00000000 00000000
+// OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0070 c2500104 1f0f007f 7f000000 00000000
@@ -93,6 +93,7 @@ disabled_user_sgpr:
 .amdhsa_kernel complete
   .amdhsa_group_segment_fixed_size 1
   .amdhsa_private_segment_fixed_size 1
+  .amdhsa_kernarg_size 8
   .amdhsa_user_sgpr_private_segment_buffer 1
   .amdhsa_user_sgpr_dispatch_ptr 1
   .amdhsa_user_sgpr_queue_ptr 1
@@ -110,7 +111,7 @@ disabled_user_sgpr:
   .amdhsa_next_free_sgpr 27
   .amdhsa_reserve_vcc 0
   .amdhsa_reserve_flat_scratch 0
-  .amdhsa_reserve_xnack_mask 0
+  .amdhsa_reserve_xnack_mask 1
   .amdhsa_float_round_mode_32 1
   .amdhsa_float_round_mode_16_64 1
   .amdhsa_float_denorm_mode_32 1
@@ -130,6 +131,7 @@ disabled_user_sgpr:
 // ASM: .amdhsa_kernel complete
 // ASM-NEXT: .amdhsa_group_segment_fixed_size 1
 // ASM-NEXT: .amdhsa_private_segment_fixed_size 1
+// ASM-NEXT: .amdhsa_kernarg_size 8
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
@@ -147,7 +149,7 @@ disabled_user_sgpr:
 // ASM-NEXT: .amdhsa_next_free_sgpr 27
 // ASM-NEXT: .amdhsa_reserve_vcc 0
 // ASM-NEXT: .amdhsa_reserve_flat_scratch 0
-// ASM-NEXT: .amdhsa_reserve_xnack_mask 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
 // ASM-NEXT: .amdhsa_float_round_mode_32 1
 // ASM-NEXT: .amdhsa_float_round_mode_16_64 1
 // ASM-NEXT: .amdhsa_float_denorm_mode_32 1
@@ -174,7 +176,7 @@ disabled_user_sgpr:
   .amdhsa_reserve_flat_scratch 1
 
   .amdhsa_reserve_vcc 0
-  .amdhsa_reserve_xnack_mask 0
+  .amdhsa_reserve_xnack_mask 1
 
   .amdhsa_float_denorm_mode_16_64 0
   .amdhsa_dx10_clamp 0
@@ -186,7 +188,7 @@ disabled_user_sgpr:
 // ASM: .amdhsa_next_free_vgpr 0
 // ASM-NEXT: .amdhsa_next_free_sgpr 27
 // ASM-NEXT: .amdhsa_reserve_vcc 0
-// ASM-NEXT: .amdhsa_reserve_xnack_mask 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
 // ASM: .amdhsa_float_denorm_mode_16_64 0
 // ASM-NEXT: .amdhsa_dx10_clamp 0
 // ASM-NEXT: .amdhsa_ieee_mode 0
diff --git a/llvm/test/MC/AMDGPU/hsa-v4.s b/llvm/test/MC/AMDGPU/hsa-v4.s
new file mode 100644
index 0000000000000000000000000000000000000000..b1941184150edfd7c8cf6cab1343c159f0d6e558
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-v4.s
@@ -0,0 +1,303 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=4 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=4 -mattr=+xnack -filetype=obj < %s > %t
+// RUN: llvm-readelf -sections -symbols -relocations %t | FileCheck --check-prefix=READOBJ %s
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// READOBJ: Section Headers
+// READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
+// READOBJ: .rodata PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}}        000100 {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
+
+// READOBJ: Relocation section '.rela.rodata' at offset
+// READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000090 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 210
+// READOBJ: 00000000000000d0 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 310
+
+// READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
+// READOBJ:      0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
+// READOBJ-NEXT: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
+// READOBJ-NEXT: 0000000000000200  0 FUNC    LOCAL  PROTECTED 2 special_sgpr
+// READOBJ-NEXT: 0000000000000300  0 FUNC    LOCAL  PROTECTED 2 disabled_user_sgpr
+// READOBJ-NEXT: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
+// READOBJ-NEXT: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
+// READOBJ-NEXT: 0000000000000080 64 OBJECT  LOCAL  DEFAULT   3 special_sgpr.kd
+// READOBJ-NEXT: 00000000000000c0 64 OBJECT  LOCAL  DEFAULT   3 disabled_user_sgpr.kd
+
+// OBJDUMP: Contents of section .rodata
+// Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
+// minimal
+// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000
+// complete
+// OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
+// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0070 c2500104 1f0f007f 7f000000 00000000
+// special_sgpr
+// OBJDUMP-NEXT: 0080 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0090 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00a0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00b0 00010000 80000000 00000000 00000000
+// disabled_user_sgpr
+// OBJDUMP-NEXT: 00c0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00d0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00e0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00f0 0000ac00 80000000 00000000 00000000
+
+.text
+// ASM: .text
+
+.amdgcn_target "amdgcn-amd-amdhsa--gfx904:xnack+"
+// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx904:xnack+"
+
+.p2align 8
+.type minimal,@function
+minimal:
+  s_endpgm
+
+.p2align 8
+.type complete,@function
+complete:
+  s_endpgm
+
+.p2align 8
+.type special_sgpr,@function
+special_sgpr:
+  s_endpgm
+
+.p2align 8
+.type disabled_user_sgpr,@function
+disabled_user_sgpr:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+// Test that only specifying required directives is allowed, and that defaulted
+// values are omitted.
+.p2align 6
+.amdhsa_kernel minimal
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel minimal
+// ASM: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM: .end_amdhsa_kernel
+
+// Test that we can specify all available directives with non-default values.
+.p2align 6
+.amdhsa_kernel complete
+  .amdhsa_group_segment_fixed_size 1
+  .amdhsa_private_segment_fixed_size 1
+  .amdhsa_kernarg_size 8
+  .amdhsa_user_sgpr_private_segment_buffer 1
+  .amdhsa_user_sgpr_dispatch_ptr 1
+  .amdhsa_user_sgpr_queue_ptr 1
+  .amdhsa_user_sgpr_kernarg_segment_ptr 1
+  .amdhsa_user_sgpr_dispatch_id 1
+  .amdhsa_user_sgpr_flat_scratch_init 1
+  .amdhsa_user_sgpr_private_segment_size 1
+  .amdhsa_system_sgpr_private_segment_wavefront_offset 1
+  .amdhsa_system_sgpr_workgroup_id_x 0
+  .amdhsa_system_sgpr_workgroup_id_y 1
+  .amdhsa_system_sgpr_workgroup_id_z 1
+  .amdhsa_system_sgpr_workgroup_info 1
+  .amdhsa_system_vgpr_workitem_id 1
+  .amdhsa_next_free_vgpr 9
+  .amdhsa_next_free_sgpr 27
+  .amdhsa_reserve_vcc 0
+  .amdhsa_reserve_flat_scratch 0
+  .amdhsa_reserve_xnack_mask 1
+  .amdhsa_float_round_mode_32 1
+  .amdhsa_float_round_mode_16_64 1
+  .amdhsa_float_denorm_mode_32 1
+  .amdhsa_float_denorm_mode_16_64 0
+  .amdhsa_dx10_clamp 0
+  .amdhsa_ieee_mode 0
+  .amdhsa_fp16_overflow 1
+  .amdhsa_exception_fp_ieee_invalid_op 1
+  .amdhsa_exception_fp_denorm_src 1
+  .amdhsa_exception_fp_ieee_div_zero 1
+  .amdhsa_exception_fp_ieee_overflow 1
+  .amdhsa_exception_fp_ieee_underflow 1
+  .amdhsa_exception_fp_ieee_inexact 1
+  .amdhsa_exception_int_div_zero 1
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel complete
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 1
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 1
+// ASM-NEXT: .amdhsa_kernarg_size 8
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 1
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 1
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1
+// ASM-NEXT: .amdhsa_next_free_vgpr 9
+// ASM-NEXT: .amdhsa_next_free_sgpr 27
+// ASM-NEXT: .amdhsa_reserve_vcc 0
+// ASM-NEXT: .amdhsa_reserve_flat_scratch 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
+// ASM-NEXT: .amdhsa_float_round_mode_32 1
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 1
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 1
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 0
+// ASM-NEXT: .amdhsa_dx10_clamp 0
+// ASM-NEXT: .amdhsa_ieee_mode 0
+// ASM-NEXT: .amdhsa_fp16_overflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1
+// ASM-NEXT: .amdhsa_exception_int_div_zero 1
+// ASM-NEXT: .end_amdhsa_kernel
+
+// Test that we are including special SGPR usage in the granulated count.
+.p2align 6
+.amdhsa_kernel special_sgpr
+  // Same next_free_sgpr as "complete", but...
+  .amdhsa_next_free_sgpr 27
+  // ...on GFX9 this should require an additional 6 SGPRs, pushing us from
+  // 3 granules to 4
+  .amdhsa_reserve_flat_scratch 1
+
+  .amdhsa_reserve_vcc 0
+  .amdhsa_reserve_xnack_mask 1
+
+  .amdhsa_float_denorm_mode_16_64 0
+  .amdhsa_dx10_clamp 0
+  .amdhsa_ieee_mode 0
+  .amdhsa_next_free_vgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel special_sgpr
+// ASM: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 27
+// ASM-NEXT: .amdhsa_reserve_vcc 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
+// ASM: .amdhsa_float_denorm_mode_16_64 0
+// ASM-NEXT: .amdhsa_dx10_clamp 0
+// ASM-NEXT: .amdhsa_ieee_mode 0
+// ASM: .end_amdhsa_kernel
+
+// Test that explicitly disabling user_sgpr's does not affect the user_sgpr
+// count, i.e. this should produce the same descriptor as minimal.
+.p2align 6
+.amdhsa_kernel disabled_user_sgpr
+  .amdhsa_user_sgpr_private_segment_buffer 0
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel disabled_user_sgpr
+// ASM: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM: .end_amdhsa_kernel
+
+.section .foo
+
+.byte .amdgcn.gfx_generation_number
+// ASM: .byte 9
+
+.byte .amdgcn.gfx_generation_minor
+// ASM: .byte 0
+
+.byte .amdgcn.gfx_generation_stepping
+// ASM: .byte 4
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 0
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 0
+
+v_mov_b32_e32 v7, s10
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 8
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 11
+
+.set .amdgcn.next_free_vgpr, 0
+.set .amdgcn.next_free_sgpr, 0
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 0
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 0
+
+v_mov_b32_e32 v16, s3
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 17
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 4
+
+// Metadata
+
+.amdgpu_metadata
+  amdhsa.version:
+    - 3
+    - 0
+  amdhsa.kernels:
+    - .name:       amd_kernel_code_t_test_all
+      .symbol: amd_kernel_code_t_test_all@kd
+      .kernarg_segment_size: 8
+      .group_segment_fixed_size: 16
+      .private_segment_fixed_size: 32
+      .kernarg_segment_align: 64
+      .wavefront_size: 128
+      .sgpr_count: 14
+      .vgpr_count: 40
+      .max_flat_workgroup_size: 256
+    - .name:       amd_kernel_code_t_minimal
+      .symbol: amd_kernel_code_t_minimal@kd
+      .kernarg_segment_size: 8
+      .group_segment_fixed_size: 16
+      .private_segment_fixed_size: 32
+      .kernarg_segment_align: 64
+      .wavefront_size: 128
+      .sgpr_count: 14
+      .vgpr_count: 40
+      .max_flat_workgroup_size: 256
+.end_amdgpu_metadata
+
+// ASM:      	.amdgpu_metadata
+// ASM:      amdhsa.kernels:  
+// ASM:        - .group_segment_fixed_size: 16
+// ASM:          .kernarg_segment_align: 64
+// ASM:          .kernarg_segment_size: 8
+// ASM:          .max_flat_workgroup_size: 256
+// ASM:          .name:           amd_kernel_code_t_test_all
+// ASM:          .private_segment_fixed_size: 32
+// ASM:          .sgpr_count:     14
+// ASM:          .symbol:         'amd_kernel_code_t_test_all@kd'
+// ASM:          .vgpr_count:     40
+// ASM:          .wavefront_size: 128
+// ASM:        - .group_segment_fixed_size: 16
+// ASM:          .kernarg_segment_align: 64
+// ASM:          .kernarg_segment_size: 8
+// ASM:          .max_flat_workgroup_size: 256
+// ASM:          .name:           amd_kernel_code_t_minimal
+// ASM:          .private_segment_fixed_size: 32
+// ASM:          .sgpr_count:     14
+// ASM:          .symbol:         'amd_kernel_code_t_minimal@kd'
+// ASM:          .vgpr_count:     40
+// ASM:          .wavefront_size: 128
+// ASM:      amdhsa.version:  
+// ASM-NEXT:   - 3
+// ASM-NEXT:   - 0
+// ASM:      	.end_amdgpu_metadata
diff --git a/llvm/test/MC/AMDGPU/hsa_isa_version_attrs.s b/llvm/test/MC/AMDGPU/hsa_isa_version_attrs.s
index a4039869458e34cdbd12af576b68a4a917c11969..ed5b1f27396a02fad456c575ae0b8630f8f5f8ee 100644
--- a/llvm/test/MC/AMDGPU/hsa_isa_version_attrs.s
+++ b/llvm/test/MC/AMDGPU/hsa_isa_version_attrs.s
@@ -1,8 +1,6 @@
 // RUN: llvm-mc -arch=amdgcn -mcpu=gfx801 -mattr=-fast-fmaf -show-encoding %s | FileCheck --check-prefix=GFX8 %s
-// RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -mattr=-mad-mix-insts -show-encoding %s | FileCheck --check-prefix=GFX9 %s
-// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32 -show-encoding %s | FileCheck --check-prefix=GFX10 %s
+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -mattr=-mad-mix-insts,-xnack -show-encoding %s | FileCheck --check-prefix=GFX9 %s
 
 .hsa_code_object_isa
 // GFX8:  .hsa_code_object_isa 8,0,1,"AMD","AMDGPU"
 // GFX9:  .hsa_code_object_isa 9,0,0,"AMD","AMDGPU"
-// GFX10: .hsa_code_object_isa 10,1,0,"AMD","AMDGPU"
diff --git a/llvm/test/MC/AMDGPU/isa-version-hsa.s b/llvm/test/MC/AMDGPU/isa-version-hsa.s
index 7281487b0517db8fbc3cadc7073b05b94543544a..ce550b057d867b1a8872a188af1cfbd3fd6c180f 100644
--- a/llvm/test/MC/AMDGPU/isa-version-hsa.s
+++ b/llvm/test/MC/AMDGPU/isa-version-hsa.s
@@ -7,7 +7,7 @@
 // RUN: not llvm-mc -triple amdgcn-amd-amdpal -mcpu=iceland %s 2>&1 | FileCheck --check-prefix=OSABI-PAL-ERR %s
 
 // OSABI-HSA: .amd_amdgpu_isa "amdgcn-amd-amdhsa--gfx802"
-// OSABI-UNK-ERR: error: .amd_amdgpu_isa directive does not match triple and/or mcpu arguments specified through the command line
-// OSABI-HSA-ERR: error: .amd_amdgpu_isa directive does not match triple and/or mcpu arguments specified through the command line
-// OSABI-PAL-ERR: error: .amd_amdgpu_isa directive does not match triple and/or mcpu arguments specified through the command line
+// OSABI-UNK-ERR: error: target id must match options
+// OSABI-HSA-ERR: error: target id must match options
+// OSABI-PAL-ERR: error: target id must match options
 .amd_amdgpu_isa "amdgcn-amd-amdhsa--gfx802"
diff --git a/llvm/test/MC/AMDGPU/isa-version-pal.s b/llvm/test/MC/AMDGPU/isa-version-pal.s
index 98b91ad8bda94e6093cf93b125ef280dc33a39b9..5eab6b94403172158000dce1cafb1c13ec154ff1 100644
--- a/llvm/test/MC/AMDGPU/isa-version-pal.s
+++ b/llvm/test/MC/AMDGPU/isa-version-pal.s
@@ -7,7 +7,7 @@
 // RUN: not llvm-mc -triple amdgcn-amd-unknown -mcpu=gfx802 %s 2>&1 | FileCheck --check-prefix=OSABI-UNK-ERR %s
 
 // OSABI-PAL: .amd_amdgpu_isa "amdgcn-amd-amdpal--gfx802"
-// OSABI-UNK-ERR: error: .amd_amdgpu_isa directive does not match triple and/or mcpu arguments specified through the command line
-// OSABI-HSA-ERR: error: .amd_amdgpu_isa directive does not match triple and/or mcpu arguments specified through the command line
-// OSABI-PAL-ERR: error: .amd_amdgpu_isa directive does not match triple and/or mcpu arguments specified through the command line
+// OSABI-UNK-ERR: error: target id must match options
+// OSABI-HSA-ERR: error: target id must match options
+// OSABI-PAL-ERR: error: target id must match options
 .amd_amdgpu_isa "amdgcn-amd-amdpal--gfx802"
diff --git a/llvm/test/MC/AMDGPU/isa-version-unk.s b/llvm/test/MC/AMDGPU/isa-version-unk.s
index ef18895487dae9de7895a4b9ead3aef2a6e7d963..a5e5206678388883d7aa42c4859a586a6930be52 100644
--- a/llvm/test/MC/AMDGPU/isa-version-unk.s
+++ b/llvm/test/MC/AMDGPU/isa-version-unk.s
@@ -7,7 +7,7 @@
 // RUN: not llvm-mc -triple amdgcn-amd-amdpal -mcpu=iceland %s 2>&1 | FileCheck --check-prefix=OSABI-PAL-ERR %s
 
 // OSABI-UNK: .amd_amdgpu_isa "amdgcn-amd-unknown--gfx802"
-// OSABI-UNK-ERR: error: .amd_amdgpu_isa directive does not match triple and/or mcpu arguments specified through the command line
-// OSABI-HSA-ERR: error: .amd_amdgpu_isa directive does not match triple and/or mcpu arguments specified through the command line
-// OSABI-PAL-ERR: error: .amd_amdgpu_isa directive does not match triple and/or mcpu arguments specified through the command line
+// OSABI-UNK-ERR: error: target id must match options
+// OSABI-HSA-ERR: error: target id must match options
+// OSABI-PAL-ERR: error: target id must match options
 .amd_amdgpu_isa "amdgcn-amd-unknown--gfx802"
diff --git a/llvm/test/MC/AMDGPU/lds_direct-err.s b/llvm/test/MC/AMDGPU/lds_direct-err.s
index 48314613b040c00214b407563c3fa0a62e94f476..854efad84ec0035dc5158520f8cff74cf2f1acf3 100644
--- a/llvm/test/MC/AMDGPU/lds_direct-err.s
+++ b/llvm/test/MC/AMDGPU/lds_direct-err.s
@@ -12,46 +12,46 @@ s_and_b32 s2, lds_direct, s1
 //---------------------------------------------------------------------------//
 
 v_ashrrev_i16 v0, lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_ashrrev_i32 v0, lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_lshlrev_b16 v0, lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_lshlrev_b32 v0, lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_lshrrev_b16 v0, lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_lshrrev_b32 v0, lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_pk_ashrrev_i16 v0, lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_pk_lshlrev_b16 v0, lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_pk_lshrrev_b16 v0, lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_subbrev_co_u32 v0, vcc, src_lds_direct, v0, vcc
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_subrev_co_u32 v0, vcc, src_lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_subrev_f16 v0, src_lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_subrev_u16 v0, src_lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 v_subrev_u32 v0, src_lds_direct, v0
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct cannot be used with this instruction
 
 //---------------------------------------------------------------------------//
 // lds_direct may not be used with v_writelane_b32 for VI/GFX9
@@ -72,10 +72,10 @@ v_add_f64 v[0:1], lds_direct, v[0:1]
 //---------------------------------------------------------------------------//
 
 v_add_i32 v0, v0, lds_direct
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct may be used as src0 only
 
 v_add_i32 lds_direct, v0, v0
 // NOGFX9: error: invalid operand for instruction
 
 v_fma_f32 v0, v0, v0, lds_direct
-// NOGFX9: error: invalid use of lds_direct
+// NOGFX9: error: lds_direct may be used as src0 only
diff --git a/llvm/test/MC/AMDGPU/lds_direct-gfx10.s b/llvm/test/MC/AMDGPU/lds_direct-gfx10.s
index 61e4de3e4691460f5aa69d62d105e59e88f50a23..df83471ed6da43ca34acb9b6af904328acbf2bb8 100644
--- a/llvm/test/MC/AMDGPU/lds_direct-gfx10.s
+++ b/llvm/test/MC/AMDGPU/lds_direct-gfx10.s
@@ -17,43 +17,43 @@ v_permlanex16_b32 v0, lds_direct, s0, s0
 // NOGFX10: error: invalid operand for instruction
 
 v_ashrrev_i16 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_ashrrev_i32 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_lshlrev_b16 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_lshlrev_b32 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_lshrrev_b16 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_lshrrev_b32 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_pk_ashrrev_i16 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_pk_lshlrev_b16 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_pk_lshrrev_b16 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_subrev_co_ci_u32 v0, vcc_lo, src_lds_direct, v0, vcc_lo
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_subrev_co_u32 v0, s0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_subrev_f16 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_subrev_f32 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
 
 v_subrev_nc_u32 v0, src_lds_direct, v0
-// NOGFX10: error: invalid use of lds_direct
+// NOGFX10: error: lds_direct cannot be used with this instruction
diff --git a/llvm/test/MC/AMDGPU/round-trip.s b/llvm/test/MC/AMDGPU/round-trip.s
index eb355ea544e02bc7a1c89724454c2c1cc52cd0b9..aa817d399c002d5ad263ba3394525bcc351e2559 100644
--- a/llvm/test/MC/AMDGPU/round-trip.s
+++ b/llvm/test/MC/AMDGPU/round-trip.s
@@ -1,5 +1,5 @@
-# RUN: llvm-mc -preserve-comments -triple amdgcn-amd-amdhsa %s >%t-1.s
-# RUN: llvm-mc -preserve-comments -triple amdgcn-amd-amdhsa %t-1.s >%t-2.s
+# RUN: llvm-mc -preserve-comments -triple amdgcn-amd- %s >%t-1.s
+# RUN: llvm-mc -preserve-comments -triple amdgcn-amd- %t-1.s >%t-2.s
 # RUN: diff %t-1.s %t-2.s
 
 # Test that AMDGPU assembly round-trips when run through MC; the first
diff --git a/llvm/test/MC/AMDGPU/sop1-err.s b/llvm/test/MC/AMDGPU/sop1-err.s
index be88b1077f832c25d0157428b046e250835e3a8d..1593488adbcb46fb26553cb59ac801e41c521deb 100644
--- a/llvm/test/MC/AMDGPU/sop1-err.s
+++ b/llvm/test/MC/AMDGPU/sop1-err.s
@@ -39,3 +39,6 @@ s_mov_b32 s103, 1
 
 s_mov_b64 s[102:103], -1
 // VI: error: register not available on this GPU
+
+s_setpc_b64 0
+// GCN: error: invalid operand for instruction
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx90a_dasm_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx90a_dasm_features.txt
index cc007a6cd4ca669c6c62f36ece6b9e01e301179e..bc5c6509d738c9279ff4cfe729209088d66135de 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx90a_dasm_features.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx90a_dasm_features.txt
@@ -793,3 +793,6 @@
 
 # GFX90A: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc ; encoding: [0x00,0x00,0x41,0xdd,0x00,0x02,0x00,0x00]
 0x00,0x00,0x41,0xdd,0x00,0x02,0x00,0x00
+
+# GFX90A: buffer_atomic_add v4, off, s[8:11], s3 scc ; encoding: [0x00,0x80,0x08,0xe1,0x00,0x04,0x02,0x03]
+0x00,0x80,0x08,0xe1,0x00,0x04,0x02,0x03
diff --git a/llvm/test/MC/ELF/lto-discard.s b/llvm/test/MC/ELF/lto-discard.s
new file mode 100644
index 0000000000000000000000000000000000000000..8ea8ab5775cdda808794b2446d5c52d784025c66
--- /dev/null
+++ b/llvm/test/MC/ELF/lto-discard.s
@@ -0,0 +1,30 @@
+// Check that ".lto_discard" ignores symbol assignments and attribute changes
+// for the specified symbols.
+// RUN: llvm-mc -triple x86_64-pc-linux-gnu < %s | FileCheck %s
+
+// Check that ".lto_discard" only accepts identifiers.
+// RUN: not llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu --defsym ERR=1 %s 2>&1 |\
+// RUN:         FileCheck %s --check-prefix=ERR
+
+// CHECK-NOT:   .weak foo
+// CHECK-NOT:       foo:
+// CHECK:       .weak bar
+// CHECK:           bar:
+// CHECK:               .byte 2
+
+.lto_discard foo
+.weak foo
+foo:
+    .byte 1
+
+.lto_discard
+.weak bar
+bar:
+    .byte 2
+
+
+.ifdef ERR
+.text
+# ERR: {{.*}}.s:[[#@LINE+1]]:14: error: expected identifier in directive
+.lto_discard 1
+.endif
diff --git a/llvm/test/MC/RISCV/rvv/aliases.s b/llvm/test/MC/RISCV/rvv/aliases.s
index ebe9e79399a697ea205f421302bb3d7865e0b2e7..ba41594baa735ffafc853e2666df942fedd1a7b8 100644
--- a/llvm/test/MC/RISCV/rvv/aliases.s
+++ b/llvm/test/MC/RISCV/rvv/aliases.s
@@ -75,3 +75,6 @@ vncvt.x.x.w v2, v1, v0.t
 # ALIAS:    vfneg.v         v2, v1, v0.t     # encoding: [0x57,0x91,0x10,0x24]
 # NO-ALIAS: vfsgnjn.vv      v2, v1, v1, v0.t # encoding: [0x57,0x91,0x10,0x24]
 vfneg.v v2, v1, v0.t 
+# ALIAS:    vfabs.v         v2, v1, v0.t     # encoding: [0x57,0x91,0x10,0x28]
+# NO-ALIAS: vfsgnjx.vv      v2, v1, v1, v0.t # encoding: [0x57,0x91,0x10,0x28]
+vfabs.v v2, v1, v0.t
diff --git a/llvm/test/MC/WebAssembly/assembler-binary.ll b/llvm/test/MC/WebAssembly/assembler-binary.ll
index e3a9a1b76a231eb10d5c4380f62a9973a4132699..7b64f11cd68f37c3c3b0a17c5df3ecc44af4f205 100644
--- a/llvm/test/MC/WebAssembly/assembler-binary.ll
+++ b/llvm/test/MC/WebAssembly/assembler-binary.ll
@@ -50,7 +50,7 @@ entry:
 ; CHECK-NEXT:         Field:           __linear_memory
 ; CHECK-NEXT:         Kind:            MEMORY
 ; CHECK-NEXT:         Memory:
-; CHECK-NEXT:           Initial:         0x0
+; CHECK-NEXT:           Minimum:         0x0
 ; CHECK-NEXT:       - Module:          env
 ; CHECK-NEXT:         Field:           bar
 ; CHECK-NEXT:         Kind:            FUNCTION
diff --git a/llvm/test/MC/WebAssembly/basic-assembly.s b/llvm/test/MC/WebAssembly/basic-assembly.s
index 2faddd215433a7913bb89097beac5bb47f33eadf..a9ebe225f9652d3a75029297223f474f8ae0630f 100644
--- a/llvm/test/MC/WebAssembly/basic-assembly.s
+++ b/llvm/test/MC/WebAssembly/basic-assembly.s
@@ -1,6 +1,6 @@
-# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+reference-types,atomics,+unimplemented-simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+reference-types,atomics,+simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
 # Check that it converts to .o without errors, but don't check any output:
-# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -mattr=+reference-types,+atomics,+unimplemented-simd128,+nontrapping-fptoint,+exception-handling -o %t.o < %s
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -mattr=+reference-types,+atomics,+simd128,+nontrapping-fptoint,+exception-handling -o %t.o < %s
 
 
 empty_func:
diff --git a/llvm/test/MC/WebAssembly/call-indirect-relocs.s b/llvm/test/MC/WebAssembly/call-indirect-relocs.s
index be4202f7d494a4b486bfa6679f21c5677e4fe0d0..8a1b6451982faaef727110d191c80e9ee4e13cd0 100644
--- a/llvm/test/MC/WebAssembly/call-indirect-relocs.s
+++ b/llvm/test/MC/WebAssembly/call-indirect-relocs.s
@@ -45,7 +45,7 @@ empty_fref_table:
 # BIN-NEXT:         Field:           __linear_memory
 # BIN-NEXT:         Kind:            MEMORY
 # BIN-NEXT:         Memory:
-# BIN-NEXT:           Initial:         0x0
+# BIN-NEXT:           Minimum:         0x0
 # BIN-NEXT:   - Type:            FUNCTION
 # BIN-NEXT:     FunctionTypes:   [ 0 ]
 # BIN-NEXT:   - Type:            TABLE
@@ -53,7 +53,7 @@ empty_fref_table:
 # BIN-NEXT:       - Index:           0
 # BIN-NEXT:         ElemType:        FUNCREF
 # BIN-NEXT:         Limits:
-# BIN-NEXT:           Initial:         0x0
+# BIN-NEXT:           Minimum:         0x0
 # BIN-NEXT:   - Type:            CODE
 # BIN-NEXT:     Relocations:
 # BIN-NEXT:       - Type:            R_WASM_TYPE_INDEX_LEB
diff --git a/llvm/test/MC/WebAssembly/comdat.ll b/llvm/test/MC/WebAssembly/comdat.ll
index 5af940d1ad519c5f9b037dd545460ce98acb09f0..6f8c1403be749478ae3ea72926d9aaba4eec77e3 100644
--- a/llvm/test/MC/WebAssembly/comdat.ll
+++ b/llvm/test/MC/WebAssembly/comdat.ll
@@ -41,7 +41,7 @@ define linkonce_odr i32 @sharedFn() #1 comdat($sharedComdat) {
 ; CHECK-NEXT:         Field:           __linear_memory
 ; CHECK-NEXT:         Kind:            MEMORY
 ; CHECK-NEXT:         Memory:
-; CHECK-NEXT:           Initial:         0x1
+; CHECK-NEXT:           Minimum:         0x1
 ; CHECK-NEXT:       - Module:          env
 ; CHECK-NEXT:         Field:           funcImport
 ; CHECK-NEXT:         Kind:            FUNCTION
diff --git a/llvm/test/MC/WebAssembly/data-section.s b/llvm/test/MC/WebAssembly/data-section.s
index 71c62e5f0b163b8a4b24ec0c2a478f11a08fd7f3..72c735fec0f35bfbc64ae61ff21890388161597e 100644
--- a/llvm/test/MC/WebAssembly/data-section.s
+++ b/llvm/test/MC/WebAssembly/data-section.s
@@ -1,12 +1,12 @@
-# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+unimplemented-simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
 # Check that it converts to .o without errors:
-# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -mattr=+unimplemented-simd128,+nontrapping-fptoint,+exception-handling < %s | obj2yaml | FileCheck -check-prefixes=BIN,BIN32 %s
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -mattr=+simd128,+nontrapping-fptoint,+exception-handling < %s | obj2yaml | FileCheck -check-prefixes=BIN,BIN32 %s
 
 # Same again for wasm64
 
-# RUN: llvm-mc -triple=wasm64-unknown-unknown -mattr=+unimplemented-simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
+# RUN: llvm-mc -triple=wasm64-unknown-unknown -mattr=+simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
 # Check that it converts to .o without errors:
-# RUN: llvm-mc -triple=wasm64-unknown-unknown -filetype=obj -mattr=+unimplemented-simd128,+nontrapping-fptoint,+exception-handling < %s | obj2yaml | FileCheck -check-prefixes=BIN,BIN64 %s
+# RUN: llvm-mc -triple=wasm64-unknown-unknown -filetype=obj -mattr=+simd128,+nontrapping-fptoint,+exception-handling < %s | obj2yaml | FileCheck -check-prefixes=BIN,BIN64 %s
 
 # Minimal test for data sections.
 
@@ -51,7 +51,7 @@ test0:
 # BIN-NEXT:         Kind:            MEMORY
 # BIN-NEXT:         Memory:
 # BIN64-NEXT:         Flags:           [ IS_64 ]
-# BIN-NEXT:           Initial:         0x1
+# BIN-NEXT:           Minimum:         0x1
 # BIN-NEXT:   - Type:            FUNCTION
 # BIN-NEXT:     FunctionTypes:   [ 0 ]
 # BIN-NEXT:   - Type:            DATACOUNT
diff --git a/llvm/test/MC/WebAssembly/global-ctor-dtor.ll b/llvm/test/MC/WebAssembly/global-ctor-dtor.ll
index 5f56cb6e280dc5ce8a588b69fa527d5ea5288d4a..df1252ef9ce154f58ff2fcc980e6e18d45278179 100644
--- a/llvm/test/MC/WebAssembly/global-ctor-dtor.ll
+++ b/llvm/test/MC/WebAssembly/global-ctor-dtor.ll
@@ -25,7 +25,7 @@ declare void @func3()
 ; CHECK-NEXT:         Field:           __linear_memory
 ; CHECK-NEXT:         Kind:            MEMORY
 ; CHECK-NEXT:         Memory:
-; CHECK-NEXT:           Initial:         0x1
+; CHECK-NEXT:           Minimum:         0x1
 ; CHECK-NEXT:       - Module:          env
 ; CHECK-NEXT:         Field:           func3
 ; CHECK-NEXT:         Kind:            FUNCTION
@@ -53,7 +53,7 @@ declare void @func3()
 ; CHECK-NEXT:           Index:           0
 ; CHECK-NEXT:           ElemType:        FUNCREF
 ; CHECK-NEXT:           Limits:
-; CHECK-NEXT:             Initial:         0x2
+; CHECK-NEXT:             Minimum:         0x2
 ; CHECK-NEXT:   - Type:            FUNCTION
 ; CHECK-NEXT:     FunctionTypes:   [ 0, 1, 0, 1 ]
 ; CHECK-NEXT:   - Type:            ELEM
diff --git a/llvm/test/MC/WebAssembly/reloc-pic.s b/llvm/test/MC/WebAssembly/reloc-pic.s
index 477b2cb8ac5e6975d3a649569201660509f49351..4036665ec932c356fc921c7612611d02fff227fa 100644
--- a/llvm/test/MC/WebAssembly/reloc-pic.s
+++ b/llvm/test/MC/WebAssembly/reloc-pic.s
@@ -62,7 +62,7 @@ hidden_func:
 # CHECK-NEXT:         Field:           __linear_memory
 # CHECK-NEXT:         Kind:            MEMORY
 # CHECK-NEXT:         Memory:
-# CHECK-NEXT:           Initial:         0x1
+# CHECK-NEXT:           Minimum:         0x1
 # CHECK-NEXT:       - Module:          env
 # CHECK-NEXT:         Field:           default_func
 # CHECK-NEXT:         Kind:            FUNCTION
@@ -74,7 +74,7 @@ hidden_func:
 # CHECK-NEXT:           Index:           0
 # CHECK-NEXT:           ElemType:        FUNCREF
 # CHECK-NEXT:           Limits:
-# CHECK-NEXT:             Initial:         0x1
+# CHECK-NEXT:             Minimum:         0x1
 # CHECK-NEXT:       - Module:          GOT.mem
 # CHECK-NEXT:         Field:           default_data
 # CHECK-NEXT:         Kind:            GLOBAL
diff --git a/llvm/test/MC/WebAssembly/simd-encodings.s b/llvm/test/MC/WebAssembly/simd-encodings.s
index 099c6489e703f2f6db3689c3c7dcfffe3b694fb8..1a687468487aa20995194cbc8038887deccab4ef 100644
--- a/llvm/test/MC/WebAssembly/simd-encodings.s
+++ b/llvm/test/MC/WebAssembly/simd-encodings.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -show-encoding -triple=wasm32-unknown-unknown -mattr=+unimplemented-simd128 < %s | FileCheck %s
+# RUN: llvm-mc -show-encoding -triple=wasm32-unknown-unknown -mattr=+simd128 < %s | FileCheck %s
 
 main:
     .functype main () -> ()
@@ -280,38 +280,51 @@ main:
     # CHECK: v128.bitselect # encoding: [0xfd,0x52]
     v128.bitselect
 
-    # CHECK: v128.load8_lane 32, 1 # encoding: [0xfd,0x58,0x00,0x20,0x01]
+    # TODO: v128.any_true # encoding: [0xfd,0x53]
+
+    # CHECK: v128.load8_lane 32, 1 # encoding: [0xfd,0x54,0x00,0x20,0x01]
     v128.load8_lane 32, 1
 
-    # CHECK: v128.load16_lane 32, 1 # encoding: [0xfd,0x59,0x01,0x20,0x01]
+    # CHECK: v128.load16_lane 32, 1 # encoding: [0xfd,0x55,0x01,0x20,0x01]
     v128.load16_lane 32, 1
 
-    # CHECK: v128.load32_lane 32, 1 # encoding: [0xfd,0x5a,0x02,0x20,0x01]
+    # CHECK: v128.load32_lane 32, 1 # encoding: [0xfd,0x56,0x02,0x20,0x01]
     v128.load32_lane 32, 1
 
-    # CHECK: v128.load64_lane 32, 1 # encoding: [0xfd,0x5b,0x03,0x20,0x01]
+    # CHECK: v128.load64_lane 32, 1 # encoding: [0xfd,0x57,0x03,0x20,0x01]
     v128.load64_lane 32, 1
 
-    # CHECK: v128.store8_lane 32, 1 # encoding: [0xfd,0x5c,0x00,0x20,0x01]
+    # CHECK: v128.store8_lane 32, 1 # encoding: [0xfd,0x58,0x00,0x20,0x01]
     v128.store8_lane 32, 1
 
-    # CHECK: v128.store16_lane 32, 1 # encoding: [0xfd,0x5d,0x01,0x20,0x01]
+    # CHECK: v128.store16_lane 32, 1 # encoding: [0xfd,0x59,0x01,0x20,0x01]
     v128.store16_lane 32, 1
 
-    # CHECK: v128.store32_lane 32, 1 # encoding: [0xfd,0x5e,0x02,0x20,0x01]
+    # CHECK: v128.store32_lane 32, 1 # encoding: [0xfd,0x5a,0x02,0x20,0x01]
     v128.store32_lane 32, 1
 
-    # CHECK: v128.store64_lane 32, 1 # encoding: [0xfd,0x5f,0x03,0x20,0x01]
+    # CHECK: v128.store64_lane 32, 1 # encoding: [0xfd,0x5b,0x03,0x20,0x01]
     v128.store64_lane 32, 1
 
+    # CHECK: v128.load32_zero 32 # encoding: [0xfd,0x5c,0x02,0x20]
+    v128.load32_zero 32
+
+    # CHECK: v128.load64_zero 32 # encoding: [0xfd,0x5d,0x03,0x20]
+    v128.load64_zero 32
+
+    # CHECK: f32x4.demote_zero_f64x2 # encoding: [0xfd,0x5e]
+    f32x4.demote_zero_f64x2
+
+    # CHECK: f64x2.promote_low_f32x4 # encoding: [0xfd,0x5f]
+    f64x2.promote_low_f32x4
+
     # CHECK: i8x16.abs # encoding: [0xfd,0x60]
     i8x16.abs
 
     # CHECK: i8x16.neg # encoding: [0xfd,0x61]
     i8x16.neg
 
-    # CHECK: i8x16.any_true # encoding: [0xfd,0x62]
-    i8x16.any_true
+    # TODO: i8x16.popcnt # encoding: [0xfd,0x62]
 
     # CHECK: i8x16.all_true # encoding: [0xfd,0x63]
     i8x16.all_true
@@ -325,6 +338,18 @@ main:
     # CHECK: i8x16.narrow_i16x8_u # encoding: [0xfd,0x66]
     i8x16.narrow_i16x8_u
 
+    # CHECK: f32x4.ceil # encoding: [0xfd,0x67]
+    f32x4.ceil
+
+    # CHECK: f32x4.floor # encoding: [0xfd,0x68]
+    f32x4.floor
+
+    # CHECK: f32x4.trunc # encoding: [0xfd,0x69]
+    f32x4.trunc
+
+    # CHECK: f32x4.nearest # encoding: [0xfd,0x6a]
+    f32x4.nearest
+
     # CHECK: i8x16.shl # encoding: [0xfd,0x6b]
     i8x16.shl
 
@@ -337,20 +362,26 @@ main:
     # CHECK: i8x16.add # encoding: [0xfd,0x6e]
     i8x16.add
 
-    # CHECK: i8x16.add_saturate_s # encoding: [0xfd,0x6f]
-    i8x16.add_saturate_s
+    # CHECK: i8x16.add_sat_s # encoding: [0xfd,0x6f]
+    i8x16.add_sat_s
 
-    # CHECK: i8x16.add_saturate_u # encoding: [0xfd,0x70]
-    i8x16.add_saturate_u
+    # CHECK: i8x16.add_sat_u # encoding: [0xfd,0x70]
+    i8x16.add_sat_u
 
     # CHECK: i8x16.sub # encoding: [0xfd,0x71]
     i8x16.sub
 
-    # CHECK: i8x16.sub_saturate_s # encoding: [0xfd,0x72]
-    i8x16.sub_saturate_s
+    # CHECK: i8x16.sub_sat_s # encoding: [0xfd,0x72]
+    i8x16.sub_sat_s
+
+    # CHECK: i8x16.sub_sat_u # encoding: [0xfd,0x73]
+    i8x16.sub_sat_u
+
+    # CHECK: f64x2.ceil # encoding: [0xfd,0x74]
+    f64x2.ceil
 
-    # CHECK: i8x16.sub_saturate_u # encoding: [0xfd,0x73]
-    i8x16.sub_saturate_u
+    # CHECK: f64x2.floor # encoding: [0xfd,0x75]
+    f64x2.floor
 
     # CHECK: i8x16.min_s # encoding: [0xfd,0x76]
     i8x16.min_s
@@ -364,11 +395,23 @@ main:
     # CHECK: i8x16.max_u # encoding: [0xfd,0x79]
     i8x16.max_u
 
+    # CHECK: f64x2.trunc # encoding: [0xfd,0x7a]
+    f64x2.trunc
+
     # CHECK: i8x16.avgr_u # encoding: [0xfd,0x7b]
     i8x16.avgr_u
 
-    # CHECK: i8x16.popcnt # encoding: [0xfd,0x7c]
-    i8x16.popcnt
+    # CHECK: i16x8.extadd_pairwise_i8x16_s # encoding: [0xfd,0x7c]
+    i16x8.extadd_pairwise_i8x16_s
+
+    # CHECK: i16x8.extadd_pairwise_i8x16_u # encoding: [0xfd,0x7d]
+    i16x8.extadd_pairwise_i8x16_u
+
+    # CHECK: i32x4.extadd_pairwise_i16x8_s # encoding: [0xfd,0x7e]
+    i32x4.extadd_pairwise_i16x8_s
+
+    # CHECK: i32x4.extadd_pairwise_i16x8_u # encoding: [0xfd,0x7f]
+    i32x4.extadd_pairwise_i16x8_u
 
     # CHECK: i16x8.abs # encoding: [0xfd,0x80,0x01]
     i16x8.abs
@@ -376,8 +419,8 @@ main:
     # CHECK: i16x8.neg # encoding: [0xfd,0x81,0x01]
     i16x8.neg
 
-    # CHECK: i16x8.any_true # encoding: [0xfd,0x82,0x01]
-    i16x8.any_true
+    # CHECK: i16x8.q15mulr_sat_s # encoding: [0xfd,0x82,0x01]
+    i16x8.q15mulr_sat_s
 
     # CHECK: i16x8.all_true # encoding: [0xfd,0x83,0x01]
     i16x8.all_true
@@ -391,17 +434,17 @@ main:
     # CHECK: i16x8.narrow_i32x4_u # encoding: [0xfd,0x86,0x01]
     i16x8.narrow_i32x4_u
 
-    # CHECK: i16x8.widen_low_i8x16_s # encoding: [0xfd,0x87,0x01]
-    i16x8.widen_low_i8x16_s
+    # CHECK: i16x8.extend_low_i8x16_s # encoding: [0xfd,0x87,0x01]
+    i16x8.extend_low_i8x16_s
 
-    # CHECK: i16x8.widen_high_i8x16_s # encoding: [0xfd,0x88,0x01]
-    i16x8.widen_high_i8x16_s
+    # CHECK: i16x8.extend_high_i8x16_s # encoding: [0xfd,0x88,0x01]
+    i16x8.extend_high_i8x16_s
 
-    # CHECK: i16x8.widen_low_i8x16_u # encoding: [0xfd,0x89,0x01]
-    i16x8.widen_low_i8x16_u
+    # CHECK: i16x8.extend_low_i8x16_u # encoding: [0xfd,0x89,0x01]
+    i16x8.extend_low_i8x16_u
 
-    # CHECK: i16x8.widen_high_i8x16_u # encoding: [0xfd,0x8a,0x01]
-    i16x8.widen_high_i8x16_u
+    # CHECK: i16x8.extend_high_i8x16_u # encoding: [0xfd,0x8a,0x01]
+    i16x8.extend_high_i8x16_u
 
     # CHECK: i16x8.shl # encoding: [0xfd,0x8b,0x01]
     i16x8.shl
@@ -415,20 +458,23 @@ main:
     # CHECK: i16x8.add # encoding: [0xfd,0x8e,0x01]
     i16x8.add
 
-    # CHECK: i16x8.add_saturate_s # encoding: [0xfd,0x8f,0x01]
-    i16x8.add_saturate_s
+    # CHECK: i16x8.add_sat_s # encoding: [0xfd,0x8f,0x01]
+    i16x8.add_sat_s
 
-    # CHECK: i16x8.add_saturate_u # encoding: [0xfd,0x90,0x01]
-    i16x8.add_saturate_u
+    # CHECK: i16x8.add_sat_u # encoding: [0xfd,0x90,0x01]
+    i16x8.add_sat_u
 
     # CHECK: i16x8.sub # encoding: [0xfd,0x91,0x01]
     i16x8.sub
 
-    # CHECK: i16x8.sub_saturate_s # encoding: [0xfd,0x92,0x01]
-    i16x8.sub_saturate_s
+    # CHECK: i16x8.sub_sat_s # encoding: [0xfd,0x92,0x01]
+    i16x8.sub_sat_s
 
-    # CHECK: i16x8.sub_saturate_u # encoding: [0xfd,0x93,0x01]
-    i16x8.sub_saturate_u
+    # CHECK: i16x8.sub_sat_u # encoding: [0xfd,0x93,0x01]
+    i16x8.sub_sat_u
+
+    # CHECK: f64x2.nearest # encoding: [0xfd,0x94,0x01]
+    f64x2.nearest
 
     # CHECK: i16x8.mul # encoding: [0xfd,0x95,0x01]
     i16x8.mul
@@ -445,11 +491,22 @@ main:
     # CHECK: i16x8.max_u # encoding: [0xfd,0x99,0x01]
     i16x8.max_u
 
+    # 0x0a unused
+
     # CHECK: i16x8.avgr_u # encoding: [0xfd,0x9b,0x01]
     i16x8.avgr_u
 
-    # CHECK: i16x8.q15mulr_sat_s # encoding: [0xfd,0x9c,0x01]
-    i16x8.q15mulr_sat_s
+    # CHECK: i16x8.extmul_low_i8x16_s # encoding: [0xfd,0x9c,0x01]
+    i16x8.extmul_low_i8x16_s
+
+    # CHECK: i16x8.extmul_high_i8x16_s # encoding: [0xfd,0x9d,0x01]
+    i16x8.extmul_high_i8x16_s
+
+    # CHECK: i16x8.extmul_low_i8x16_u # encoding: [0xfd,0x9e,0x01]
+    i16x8.extmul_low_i8x16_u
+
+    # CHECK: i16x8.extmul_high_i8x16_u # encoding: [0xfd,0x9f,0x01]
+    i16x8.extmul_high_i8x16_u
 
     # CHECK: i32x4.abs # encoding: [0xfd,0xa0,0x01]
     i32x4.abs
@@ -457,8 +514,7 @@ main:
     # CHECK: i32x4.neg # encoding: [0xfd,0xa1,0x01]
     i32x4.neg
 
-    # CHECK: i32x4.any_true # encoding: [0xfd,0xa2,0x01]
-    i32x4.any_true
+    # 0xa2 unused
 
     # CHECK: i32x4.all_true # encoding: [0xfd,0xa3,0x01]
     i32x4.all_true
@@ -466,17 +522,21 @@ main:
     # CHECK: i32x4.bitmask # encoding: [0xfd,0xa4,0x01]
     i32x4.bitmask
 
-    # CHECK: i32x4.widen_low_i16x8_s # encoding: [0xfd,0xa7,0x01]
-    i32x4.widen_low_i16x8_s
+    # 0xa5 unused
+
+    # 0xa6 unused
 
-    # CHECK: i32x4.widen_high_i16x8_s # encoding: [0xfd,0xa8,0x01]
-    i32x4.widen_high_i16x8_s
+    # CHECK: i32x4.extend_low_i16x8_s # encoding: [0xfd,0xa7,0x01]
+    i32x4.extend_low_i16x8_s
 
-    # CHECK: i32x4.widen_low_i16x8_u # encoding: [0xfd,0xa9,0x01]
-    i32x4.widen_low_i16x8_u
+    # CHECK: i32x4.extend_high_i16x8_s # encoding: [0xfd,0xa8,0x01]
+    i32x4.extend_high_i16x8_s
 
-    # CHECK: i32x4.widen_high_i16x8_u # encoding: [0xfd,0xaa,0x01]
-    i32x4.widen_high_i16x8_u
+    # CHECK: i32x4.extend_low_i16x8_u # encoding: [0xfd,0xa9,0x01]
+    i32x4.extend_low_i16x8_u
+
+    # CHECK: i32x4.extend_high_i16x8_u # encoding: [0xfd,0xaa,0x01]
+    i32x4.extend_high_i16x8_u
 
     # CHECK: i32x4.shl # encoding: [0xfd,0xab,0x01]
     i32x4.shl
@@ -490,9 +550,19 @@ main:
     # CHECK: i32x4.add # encoding: [0xfd,0xae,0x01]
     i32x4.add
 
+    # 0xaf unused
+
+    # 0xb0 unused
+
     # CHECK: i32x4.sub # encoding: [0xfd,0xb1,0x01]
     i32x4.sub
 
+    # 0xb2 unused
+
+    # 0xb3 unused
+
+    # 0xb4 unused
+
     # CHECK: i32x4.mul # encoding: [0xfd,0xb5,0x01]
     i32x4.mul
 
@@ -511,14 +581,26 @@ main:
     # CHECK: i32x4.dot_i16x8_s # encoding: [0xfd,0xba,0x01]
     i32x4.dot_i16x8_s
 
-    # CHECK: i64x2.eq # encoding: [0xfd,0xc0,0x01]
-    i64x2.eq
+    # 0xbb unused
+
+    # CHECK: i32x4.extmul_low_i16x8_s # encoding: [0xfd,0xbc,0x01]
+    i32x4.extmul_low_i16x8_s
+
+    # CHECK: i32x4.extmul_high_i16x8_s # encoding: [0xfd,0xbd,0x01]
+    i32x4.extmul_high_i16x8_s
+
+    # CHECK: i32x4.extmul_low_i16x8_u # encoding: [0xfd,0xbe,0x01]
+    i32x4.extmul_low_i16x8_u
+
+    # CHECK: i32x4.extmul_high_i16x8_u # encoding: [0xfd,0xbf,0x01]
+    i32x4.extmul_high_i16x8_u
+
+    # TODO: i64x2.abs # encoding: [0xfd,0xc0,0x01]
 
     # CHECK: i64x2.neg # encoding: [0xfd,0xc1,0x01]
     i64x2.neg
 
-    # CHECK: i64x2.any_true # encoding: [0xfd,0xc2,0x01]
-    i64x2.any_true
+    # 0xc2 unused
 
     # CHECK: i64x2.all_true # encoding: [0xfd,0xc3,0x01]
     i64x2.all_true
@@ -526,17 +608,21 @@ main:
     # CHECK: i64x2.bitmask # encoding: [0xfd,0xc4,0x01]
     i64x2.bitmask
 
-    # CHECK: i64x2.widen_low_i32x4_s # encoding: [0xfd,0xc7,0x01]
-    i64x2.widen_low_i32x4_s
+    # 0xc5 unused
+
+    # 0xc6 unused
+
+    # CHECK: i64x2.extend_low_i32x4_s # encoding: [0xfd,0xc7,0x01]
+    i64x2.extend_low_i32x4_s
 
-    # CHECK: i64x2.widen_high_i32x4_s # encoding: [0xfd,0xc8,0x01]
-    i64x2.widen_high_i32x4_s
+    # CHECK: i64x2.extend_high_i32x4_s # encoding: [0xfd,0xc8,0x01]
+    i64x2.extend_high_i32x4_s
 
-    # CHECK: i64x2.widen_low_i32x4_u # encoding: [0xfd,0xc9,0x01]
-    i64x2.widen_low_i32x4_u
+    # CHECK: i64x2.extend_low_i32x4_u # encoding: [0xfd,0xc9,0x01]
+    i64x2.extend_low_i32x4_u
 
-    # CHECK: i64x2.widen_high_i32x4_u # encoding: [0xfd,0xca,0x01]
-    i64x2.widen_high_i32x4_u
+    # CHECK: i64x2.extend_high_i32x4_u # encoding: [0xfd,0xca,0x01]
+    i64x2.extend_high_i32x4_u
 
     # CHECK: i64x2.shl # encoding: [0xfd,0xcb,0x01]
     i64x2.shl
@@ -550,35 +636,45 @@ main:
     # CHECK: i64x2.add # encoding: [0xfd,0xce,0x01]
     i64x2.add
 
+    # 0xcf unused
+
+    # 0xd0 unused
+
     # CHECK: i64x2.sub # encoding: [0xfd,0xd1,0x01]
     i64x2.sub
 
+    # 0xd2 unused
+
+    # 0xd3 unused
+
+    # 0xd4 unused
+
     # CHECK: i64x2.mul # encoding: [0xfd,0xd5,0x01]
     i64x2.mul
 
-    # CHECK: f32x4.ceil # encoding: [0xfd,0xd8,0x01]
-    f32x4.ceil
+    # TODO: i64x2.eq # encoding: [0xfd,0xd6,0x01]
 
-    # CHECK: f32x4.floor # encoding: [0xfd,0xd9,0x01]
-    f32x4.floor
+    # TODO: i64x2.ne # encoding: [0xfd,0xd7,0x01]
 
-    # CHECK: f32x4.trunc # encoding: [0xfd,0xda,0x01]
-    f32x4.trunc
+    # TODO: i64x2.lt_s # encoding: [0xfd,0xd8,0x01]
 
-    # CHECK: f32x4.nearest # encoding: [0xfd,0xdb,0x01]
-    f32x4.nearest
+    # TODO: i64x2.gt_s # encoding: [0xfd,0xd9,0x01]
 
-    # CHECK: f64x2.ceil # encoding: [0xfd,0xdc,0x01]
-    f64x2.ceil
+    # TODO: i64x2.le_s # encoding: [0xfd,0xda,0x01]
 
-    # CHECK: f64x2.floor # encoding: [0xfd,0xdd,0x01]
-    f64x2.floor
+    # TODO: i64x2.ge_s # encoding: [0xfd,0xdb,0x01]
 
-    # CHECK: f64x2.trunc # encoding: [0xfd,0xde,0x01]
-    f64x2.trunc
+    # CHECK: i64x2.extmul_low_i32x4_s # encoding: [0xfd,0xdc,0x01]
+    i64x2.extmul_low_i32x4_s
 
-    # CHECK: f64x2.nearest # encoding: [0xfd,0xdf,0x01]
-    f64x2.nearest
+    # CHECK: i64x2.extmul_high_i32x4_s # encoding: [0xfd,0xdd,0x01]
+    i64x2.extmul_high_i32x4_s
+
+    # CHECK: i64x2.extmul_low_i32x4_u # encoding: [0xfd,0xde,0x01]
+    i64x2.extmul_low_i32x4_u
+
+    # CHECK: i64x2.extmul_high_i32x4_u # encoding: [0xfd,0xdf,0x01]
+    i64x2.extmul_high_i32x4_u
 
     # CHECK: f32x4.abs # encoding: [0xfd,0xe0,0x01]
     f32x4.abs
@@ -586,6 +682,8 @@ main:
     # CHECK: f32x4.neg # encoding: [0xfd,0xe1,0x01]
     f32x4.neg
 
+    # 0xe2 unused
+
     # CHECK: f32x4.sqrt # encoding: [0xfd,0xe3,0x01]
     f32x4.sqrt
 
@@ -619,6 +717,8 @@ main:
     # CHECK: f64x2.neg # encoding: [0xfd,0xed,0x01]
     f64x2.neg
 
+    # 0xee unused
+
     # CHECK: f64x2.sqrt # encoding: [0xfd,0xef,0x01]
     f64x2.sqrt
 
@@ -658,112 +758,16 @@ main:
     # CHECK: f32x4.convert_i32x4_u # encoding: [0xfd,0xfb,0x01]
     f32x4.convert_i32x4_u
 
-    # CHECK: v128.load32_zero 32 # encoding: [0xfd,0xfc,0x01,0x02,0x20]
-    v128.load32_zero 32
-
-    # CHECK: v128.load64_zero 32 # encoding: [0xfd,0xfd,0x01,0x03,0x20]
-    v128.load64_zero 32
-
-    # CHECK: f32x4.qfma # encoding: [0xfd,0xb4,0x01]
-    f32x4.qfma
-
-    # CHECK: f32x4.qfms # encoding: [0xfd,0xd4,0x01]
-    f32x4.qfms
-
-    # CHECK: f64x2.qfma # encoding: [0xfd,0xfe,0x01]
-    f64x2.qfma
-
-    # CHECK: f64x2.qfms # encoding: [0xfd,0xff,0x01]
-    f64x2.qfms
-
-    # CHECK: i16x8.extmul_low_i8x16_s # encoding: [0xfd,0x9a,0x01]
-    i16x8.extmul_low_i8x16_s
-
-    # CHECK: i16x8.extmul_high_i8x16_s # encoding: [0xfd,0x9d,0x01]
-    i16x8.extmul_high_i8x16_s
-
-    # CHECK: i16x8.extmul_low_i8x16_u # encoding: [0xfd,0x9e,0x01]
-    i16x8.extmul_low_i8x16_u
-
-    # CHECK: i16x8.extmul_high_i8x16_u # encoding: [0xfd,0x9f,0x01]
-    i16x8.extmul_high_i8x16_u
-
-    # CHECK: i32x4.extmul_low_i16x8_s # encoding: [0xfd,0xbb,0x01]
-    i32x4.extmul_low_i16x8_s
-
-    # CHECK: i32x4.extmul_high_i16x8_s # encoding: [0xfd,0xbd,0x01]
-    i32x4.extmul_high_i16x8_s
-
-    # CHECK: i32x4.extmul_low_i16x8_u # encoding: [0xfd,0xbe,0x01]
-    i32x4.extmul_low_i16x8_u
-
-    # CHECK: i32x4.extmul_high_i16x8_u # encoding: [0xfd,0xbf,0x01]
-    i32x4.extmul_high_i16x8_u
-
-    # CHECK: i64x2.extmul_low_i32x4_s # encoding: [0xfd,0xd2,0x01]
-    i64x2.extmul_low_i32x4_s
-
-    # CHECK: i64x2.extmul_high_i32x4_s # encoding: [0xfd,0xd3,0x01]
-    i64x2.extmul_high_i32x4_s
-
-    # CHECK: i64x2.extmul_low_i32x4_u # encoding: [0xfd,0xd6,0x01]
-    i64x2.extmul_low_i32x4_u
-
-    # CHECK: i64x2.extmul_high_i32x4_u # encoding: [0xfd,0xd7,0x01]
-    i64x2.extmul_high_i32x4_u
-
-    # CHECK: i8x16.signselect # encoding: [0xfd,0x7d]
-    i8x16.signselect
-
-    # CHECK: i16x8.signselect # encoding: [0xfd,0x7e]
-    i16x8.signselect
-
-    # CHECK: i32x4.signselect # encoding: [0xfd,0x7f]
-    i32x4.signselect
-
-    # CHECK: i64x2.signselect # encoding: [0xfd,0x94,0x01]
-    i64x2.signselect
-
-    # CHECK: i16x8.extadd_pairwise_i8x16_s # encoding: [0xfd,0xc2,0x01]
-    i16x8.extadd_pairwise_i8x16_s
-
-    # CHECK: i16x8.extadd_pairwise_i8x16_u # encoding: [0xfd,0xc3,0x01]
-    i16x8.extadd_pairwise_i8x16_u
-
-    # CHECK: i32x4.extadd_pairwise_i16x8_s # encoding: [0xfd,0xa5,0x01]
-    i32x4.extadd_pairwise_i16x8_s
-
-    # CHECK: i32x4.extadd_pairwise_i16x8_u # encoding: [0xfd,0xa6,0x01]
-    i32x4.extadd_pairwise_i16x8_u
-
-    # CHECK: prefetch.t 16 # encoding: [0xfd,0xc5,0x01,0x00,0x10]
-    prefetch.t 16
-
-    # CHECK: prefetch.nt 16 # encoding: [0xfd,0xc6,0x01,0x00,0x10]
-    prefetch.nt 16
-
-    # CHECK: f64x2.convert_low_i32x4_s # encoding: [0xfd,0x53]
-    f64x2.convert_low_i32x4_s
-
-    # CHECK: f64x2.convert_low_i32x4_u # encoding: [0xfd,0x54]
-    f64x2.convert_low_i32x4_u
-
-    # CHECK: i32x4.trunc_sat_zero_f64x2_s # encoding: [0xfd,0x55]
+    # CHECK: i32x4.trunc_sat_zero_f64x2_s # encoding: [0xfd,0xfc,0x01]
     i32x4.trunc_sat_zero_f64x2_s
 
-    # CHECK: i32x4.trunc_sat_zero_f64x2_u # encoding: [0xfd,0x56]
+    # CHECK: i32x4.trunc_sat_zero_f64x2_u # encoding: [0xfd,0xfd,0x01]
     i32x4.trunc_sat_zero_f64x2_u
 
-    # CHECK: f32x4.demote_zero_f64x2 # encoding: [0xfd,0x57]
-    f32x4.demote_zero_f64x2
-
-    # CHECK: f64x2.promote_low_f32x4 # encoding: [0xfd,0x69]
-    f64x2.promote_low_f32x4
-
-    # CHECK: i32x4.widen_i8x16_s 3 # encoding: [0xfd,0x67,0x03]
-    i32x4.widen_i8x16_s 3
+    # CHECK: f64x2.convert_low_i32x4_s # encoding: [0xfd,0xfe,0x01]
+    f64x2.convert_low_i32x4_s
 
-    # CHECK: i32x4.widen_i8x16_u 3 # encoding: [0xfd,0x68,0x03]
-    i32x4.widen_i8x16_u 3
+    # CHECK: f64x2.convert_low_i32x4_u # encoding: [0xfd,0xff,0x01]
+    f64x2.convert_low_i32x4_u
 
     end_function
diff --git a/llvm/test/MC/WebAssembly/tables.s b/llvm/test/MC/WebAssembly/tables.s
index a023b9970a8386075ecac21edc861086c5a6f9a5..a682a738445b79a30002e847e2e8efbab935a9a8 100644
--- a/llvm/test/MC/WebAssembly/tables.s
+++ b/llvm/test/MC/WebAssembly/tables.s
@@ -16,9 +16,9 @@ bar:
     .tabletype bar, funcref
 
 table1:
-    .tabletype table1, funcref
+    .tabletype table1, funcref, 42
 table2:
-    .tabletype table2, funcref
+    .tabletype table2, funcref, 42, 100
 
 # Table instructions
 
@@ -124,19 +124,21 @@ table_fill:
 # BIN-NEXT:      - Index:           0
 # BIN-NEXT:        ElemType:        EXTERNREF
 # BIN-NEXT:        Limits:
-# BIN-NEXT:          Initial:         0x0
+# BIN-NEXT:          Minimum:         0x0
 # BIN-NEXT:      - Index:           1
 # BIN-NEXT:        ElemType:        FUNCREF
 # BIN-NEXT:        Limits:
-# BIN-NEXT:          Initial:         0x0
+# BIN-NEXT:          Minimum:         0x0
 # BIN-NEXT:      - Index:           2
 # BIN-NEXT:        ElemType:        FUNCREF
 # BIN-NEXT:        Limits:
-# BIN-NEXT:          Initial:         0x0
+# BIN-NEXT:          Minimum:         0x2A
 # BIN-NEXT:      - Index:           3
 # BIN-NEXT:        ElemType:        FUNCREF
 # BIN-NEXT:        Limits:
-# BIN-NEXT:          Initial:         0x0
+# BIN-NEXT:          Flags:           [ HAS_MAX ]
+# BIN-NEXT:          Minimum:         0x2A
+# BIN-NEXT:          Maximum:         0x64
 
 #      BIN:  - Type:            CODE
 # BIN-NEXT:    Relocations:
diff --git a/llvm/test/MC/WebAssembly/type-index.s b/llvm/test/MC/WebAssembly/type-index.s
index eef1a10124664c1bdd0809219038c3a78fe6d07b..906509e9d9af762969510543aedf2e6a5acedb23 100644
--- a/llvm/test/MC/WebAssembly/type-index.s
+++ b/llvm/test/MC/WebAssembly/type-index.s
@@ -1,6 +1,6 @@
-# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+reference-types,+unimplemented-simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+reference-types,+simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
 # Check that it converts to .o without errors:
-# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -mattr=+reference-types,+unimplemented-simd128,+nontrapping-fptoint,+exception-handling < %s | obj2yaml | FileCheck -check-prefix=BIN %s
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -mattr=+reference-types,+simd128,+nontrapping-fptoint,+exception-handling < %s | obj2yaml | FileCheck -check-prefix=BIN %s
 
 # Minimal test for type indices and table references in call_indirect.
 
@@ -37,7 +37,7 @@ test0:
 # BIN-NEXT:         Field:           __linear_memory
 # BIN-NEXT:         Kind:            MEMORY
 # BIN-NEXT:         Memory:
-# BIN-NEXT:           Initial:         0x0
+# BIN-NEXT:           Minimum:         0x0
 # BIN-NEXT:       - Module:          env
 # BIN-NEXT:         Field:           __indirect_function_table
 # BIN-NEXT:         Kind:            TABLE
@@ -45,7 +45,7 @@ test0:
 # BIN-NEXT:           Index:           0
 # BIN-NEXT:           ElemType:        FUNCREF
 # BIN-NEXT:           Limits:
-# BIN-NEXT:             Initial:         0x0
+# BIN-NEXT:             Minimum:         0x0
 # BIN-NEXT:   - Type:            FUNCTION
 # BIN-NEXT:     FunctionTypes:   [ 0 ]
 # BIN-NEXT:   - Type:            CODE
diff --git a/llvm/test/MC/WebAssembly/types.ll b/llvm/test/MC/WebAssembly/types.ll
index c049d3ce0e82d7661df3586a1b28ad890390366e..0c49f888fee4ba541d3f1679ea97b3bf44f8f5ba 100644
--- a/llvm/test/MC/WebAssembly/types.ll
+++ b/llvm/test/MC/WebAssembly/types.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mattr=+unimplemented-simd128 -filetype=obj %s -o - | obj2yaml | FileCheck %s
+; RUN: llc -mattr=+simd128 -filetype=obj %s -o - | obj2yaml | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/MC/WebAssembly/wasm64.s b/llvm/test/MC/WebAssembly/wasm64.s
index 793f91f11af9df300a2c0ab0cf7127ba81c13df7..402d5020caba6970e8f0214f7ae8cfe82dd14f3b 100644
--- a/llvm/test/MC/WebAssembly/wasm64.s
+++ b/llvm/test/MC/WebAssembly/wasm64.s
@@ -1,5 +1,5 @@
-# RUN: llvm-mc -triple=wasm64-unknown-unknown -mattr=+atomics,+unimplemented-simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
-# RUN: llvm-mc -triple=wasm64-unknown-unknown -filetype=obj -mattr=+atomics,+unimplemented-simd128,+nontrapping-fptoint,+exception-handling -o - < %s | obj2yaml | FileCheck %s -check-prefix=BIN
+# RUN: llvm-mc -triple=wasm64-unknown-unknown -mattr=+atomics,+simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
+# RUN: llvm-mc -triple=wasm64-unknown-unknown -filetype=obj -mattr=+atomics,+simd128,+nontrapping-fptoint,+exception-handling -o - < %s | obj2yaml | FileCheck %s -check-prefix=BIN
 
 # Most of our other tests are for wasm32, this one adds some wasm64 specific tests.
 
@@ -148,7 +148,7 @@ test:
 # BIN-NEXT:         Kind:            MEMORY
 # BIN-NEXT:         Memory:
 # BIN-NEXT:           Flags:           [ IS_64 ]
-# BIN-NEXT:           Initial:         0x1
+# BIN-NEXT:           Minimum:         0x1
 # BIN-NEXT:       - Module:          env
 # BIN-NEXT:         Field:           myglob64
 # BIN-NEXT:         Kind:            GLOBAL
diff --git a/llvm/test/MC/WebAssembly/weak-alias.s b/llvm/test/MC/WebAssembly/weak-alias.s
index 48bfc0b7a58d811618e786104fd56ca1153346fe..7e6a9ffc36d31a3d35ba3eba4866a8044ea52a1c 100644
--- a/llvm/test/MC/WebAssembly/weak-alias.s
+++ b/llvm/test/MC/WebAssembly/weak-alias.s
@@ -87,7 +87,7 @@ alias_address:
 # CHECK-NEXT:         Field:           __linear_memory
 # CHECK-NEXT:         Kind:            MEMORY
 # CHECK-NEXT:         Memory:
-# CHECK-NEXT:           Initial:         0x1
+# CHECK-NEXT:           Minimum:         0x1
 # CHECK-NEXT:       - Module:          env
 # CHECK-NEXT:         Field:           __indirect_function_table
 # CHECK-NEXT:         Kind:            TABLE
@@ -95,7 +95,7 @@ alias_address:
 # CHECK-NEXT:           Index:           0
 # CHECK-NEXT:           ElemType:        FUNCREF
 # CHECK-NEXT:           Limits:
-# CHECK-NEXT:             Initial:         0x1
+# CHECK-NEXT:             Minimum:         0x1
 # CHECK-NEXT:   - Type:            FUNCTION
 # CHECK-NEXT:     FunctionTypes:   [ 0, 0, 0, 0, 0 ]
 # CHECK-NEXT:   - Type:            ELEM
@@ -254,7 +254,7 @@ alias_address:
 # REF-NEXT:         Field:           __linear_memory
 # REF-NEXT:         Kind:            MEMORY
 # REF-NEXT:         Memory:
-# REF-NEXT:           Initial:         0x1
+# REF-NEXT:           Minimum:         0x1
 # REF-NEXT:       - Module:          env
 # REF-NEXT:         Field:           __indirect_function_table
 # REF-NEXT:         Kind:            TABLE
@@ -262,7 +262,7 @@ alias_address:
 # REF-NEXT:           Index:           0
 # REF-NEXT:           ElemType:        FUNCREF
 # REF-NEXT:           Limits:
-# REF-NEXT:             Initial:         0x1
+# REF-NEXT:             Minimum:         0x1
 # REF-NEXT:   - Type:            FUNCTION
 # REF-NEXT:     FunctionTypes:   [ 0, 0, 0, 0, 0 ]
 # REF-NEXT:   - Type:            ELEM
diff --git a/llvm/test/MachineVerifier/test_g_bzero.mir b/llvm/test/MachineVerifier/test_g_bzero.mir
new file mode 100644
index 0000000000000000000000000000000000000000..6e0212115aa6afbaf35bdba7696461e4202d3b90
--- /dev/null
+++ b/llvm/test/MachineVerifier/test_g_bzero.mir
@@ -0,0 +1,33 @@
+#RUN: not --crash llc -o - -march=arm64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+---
+name:            test_bzero
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %ptr:_(p0) = G_IMPLICIT_DEF
+    %cst1:_(s64) = G_CONSTANT i64 4
+    %cst2:_(s8) = G_CONSTANT i8 7
+
+    ; CHECK: *** Bad machine code: bzero must have 1 memory operand ***
+    G_BZERO %ptr, %cst2, 0
+
+    ; CHECK: *** Bad machine code: bzero memory operand must be a store ***
+    G_BZERO %ptr, %cst2, 0 :: (load 4)
+
+    ; CHECK: *** Bad machine code: Missing mayLoad flag ***
+    ; CHECK: *** Bad machine code: bzero memory operand must be a store ***
+    G_BZERO %ptr, %cst2, 0 :: (load store 4)
+
+    ; CHECK: *** Bad machine code: inconsistent bzero address space ***
+    G_BZERO %ptr, %cst2, 0 :: (store 4, addrspace 1)
+
+   ; CHECK: *** Bad machine code: bzero operand must be a pointer ***
+    G_BZERO %cst1, %cst2, 0 :: (store 4)
+
+...
diff --git a/llvm/test/MachineVerifier/test_g_rotr_rotl.mir b/llvm/test/MachineVerifier/test_g_rotr_rotl.mir
new file mode 100644
index 0000000000000000000000000000000000000000..0d545662765ca30acf82765d200583f12b4ed8ab
--- /dev/null
+++ b/llvm/test/MachineVerifier/test_g_rotr_rotl.mir
@@ -0,0 +1,13 @@
+# RUN: not --crash llc -march=arm64 -verify-machineinstrs -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+---
+name:            test_uniform
+body: |
+  bb.0:
+    %src:_(<2 x s64>) = G_IMPLICIT_DEF
+    %amt:_(s64) = G_IMPLICIT_DEF
+
+    ; CHECK: Rotate requires operands to be either all scalars or all vectors
+    %rotr:_(<2 x s64>) = G_ROTR %src, %amt
+
+...
diff --git a/llvm/test/MachineVerifier/test_g_ubfx_sbfx.mir b/llvm/test/MachineVerifier/test_g_ubfx_sbfx.mir
new file mode 100644
index 0000000000000000000000000000000000000000..6f177e1d51fa107258d0710917347c9c3cf35bd8
--- /dev/null
+++ b/llvm/test/MachineVerifier/test_g_ubfx_sbfx.mir
@@ -0,0 +1,15 @@
+# RUN: not --crash llc -march=arm64 -verify-machineinstrs -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+
+name:            test
+body: |
+  bb.0:
+    %v1:_(<2 x s64>) = G_IMPLICIT_DEF
+    %v2:_(<2 x s64>) = G_IMPLICIT_DEF
+    %v3:_(<2 x s64>) = G_IMPLICIT_DEF
+
+    ; CHECK: *** Bad machine code: Bitfield extraction is not supported on vectors ***
+    %ubfx_vector:_(<2 x s64>) = G_UBFX %v1, %v2, %v3
+    ; CHECK: *** Bad machine code: Bitfield extraction is not supported on vectors ***
+    %sbfx_vector:_(<2 x s64>) = G_SBFX %v1, %v2, %v3
+...
diff --git a/llvm/test/Object/AMDGPU/elf-header-flags-sram-ecc.yaml b/llvm/test/Object/AMDGPU/elf-header-flags-sramecc.yaml
similarity index 73%
rename from llvm/test/Object/AMDGPU/elf-header-flags-sram-ecc.yaml
rename to llvm/test/Object/AMDGPU/elf-header-flags-sramecc.yaml
index c75339ad9954a70f1983af1bb3ebb412c512065d..4d4d95c812e7bff34878158c7e755cc5ec1028b3 100644
--- a/llvm/test/Object/AMDGPU/elf-header-flags-sram-ecc.yaml
+++ b/llvm/test/Object/AMDGPU/elf-header-flags-sramecc.yaml
@@ -9,23 +9,23 @@
 # RUN: obj2yaml %t.o.3 | FileCheck --check-prefixes=YAML-SRAM-ECC-XNACK-GFX900 %s
 
 # ELF-SRAM-ECC-NONE:      Flags [
-# ELF-SRAM-ECC-NONE-NEXT:   EF_AMDGPU_SRAM_ECC (0x200)
+# ELF-SRAM-ECC-NONE-NEXT:   EF_AMDGPU_FEATURE_SRAMECC_V3 (0x200)
 # ELF-SRAM-ECC-NONE-NEXT: ]
 
 # ELF-SRAM-ECC-GFX900:      Flags [
+# ELF-SRAM-ECC-GFX900-NEXT:   EF_AMDGPU_FEATURE_SRAMECC_V3 (0x200)
 # ELF-SRAM-ECC-GFX900-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C)
-# ELF-SRAM-ECC-GFX900-NEXT:   EF_AMDGPU_SRAM_ECC           (0x200)
 # ELF-SRAM-ECC-GFX900-NEXT: ]
 
 # ELF-SRAM-ECC-XNACK-GFX900:      Flags [
+# ELF-SRAM-ECC-XNACK-GFX900-NEXT:   EF_AMDGPU_FEATURE_SRAMECC_V3 (0x200)
+# ELF-SRAM-ECC-XNACK-GFX900-NEXT:   EF_AMDGPU_FEATURE_XNACK_V3   (0x100)
 # ELF-SRAM-ECC-XNACK-GFX900-NEXT:   EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C)
-# ELF-SRAM-ECC-XNACK-GFX900-NEXT:   EF_AMDGPU_SRAM_ECC           (0x200)
-# ELF-SRAM-ECC-XNACK-GFX900-NEXT:   EF_AMDGPU_XNACK              (0x100)
 # ELF-SRAM-ECC-XNACK-GFX900-NEXT: ]
 
-# YAML-SRAM-ECC-NONE:         Flags: [ EF_AMDGPU_MACH_NONE, EF_AMDGPU_SRAM_ECC ]
-# YAML-SRAM-ECC-GFX900:       Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX900, EF_AMDGPU_SRAM_ECC ]
-# YAML-SRAM-ECC-XNACK-GFX900: Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX900, EF_AMDGPU_XNACK, EF_AMDGPU_SRAM_ECC ]
+# YAML-SRAM-ECC-NONE:         Flags: [ EF_AMDGPU_MACH_NONE, EF_AMDGPU_FEATURE_SRAMECC_V3 ]
+# YAML-SRAM-ECC-GFX900:       Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX900, EF_AMDGPU_FEATURE_SRAMECC_V3 ]
+# YAML-SRAM-ECC-XNACK-GFX900: Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX900, EF_AMDGPU_FEATURE_XNACK_V3, EF_AMDGPU_FEATURE_SRAMECC_V3 ]
 
 # Doc1
 --- !ELF
@@ -35,7 +35,7 @@ FileHeader:
   OSABI:   ELFOSABI_NONE
   Type:    ET_REL
   Machine: EM_AMDGPU
-  Flags:   [ EF_AMDGPU_SRAM_ECC ]
+  Flags:   [ EF_AMDGPU_FEATURE_SRAMECC_V3 ]
 ...
 
 # Doc2
@@ -46,7 +46,7 @@ FileHeader:
   OSABI:   ELFOSABI_NONE
   Type:    ET_REL
   Machine: EM_AMDGPU
-  Flags:   [ EF_AMDGPU_MACH_AMDGCN_GFX900, EF_AMDGPU_SRAM_ECC ]
+  Flags:   [ EF_AMDGPU_MACH_AMDGCN_GFX900, EF_AMDGPU_FEATURE_SRAMECC_V3 ]
 ...
 
 # Doc3
@@ -57,5 +57,5 @@ FileHeader:
   OSABI:   ELFOSABI_NONE
   Type:    ET_REL
   Machine: EM_AMDGPU
-  Flags:   [ EF_AMDGPU_MACH_AMDGCN_GFX900, EF_AMDGPU_XNACK, EF_AMDGPU_SRAM_ECC ]
+  Flags:   [ EF_AMDGPU_MACH_AMDGCN_GFX900, EF_AMDGPU_FEATURE_XNACK_V3, EF_AMDGPU_FEATURE_SRAMECC_V3 ]
 ...
diff --git a/llvm/test/Object/AMDGPU/elf-header-flags-xnack.yaml b/llvm/test/Object/AMDGPU/elf-header-flags-xnack.yaml
index c0e32d84b7eb80bc657f5696e6b8b6efd953b983..e9c834edb975adc191d524ddd2dd6025ad0f931b 100644
--- a/llvm/test/Object/AMDGPU/elf-header-flags-xnack.yaml
+++ b/llvm/test/Object/AMDGPU/elf-header-flags-xnack.yaml
@@ -6,13 +6,13 @@
 # RUN: obj2yaml %t.o.2 | FileCheck --check-prefixes=YAML-XNACK-GFX801 %s
 
 # ELF-ALL:          Flags [
-# ELF-XNACK-NONE:     EF_AMDGPU_XNACK              (0x100)
+# ELF-XNACK-NONE:     EF_AMDGPU_FEATURE_XNACK_V3   (0x100)
+# ELF-XNACK-GFX801:   EF_AMDGPU_FEATURE_XNACK_V3   (0x100)
 # ELF-XNACK-GFX801:   EF_AMDGPU_MACH_AMDGCN_GFX801 (0x28)
-# ELF-XNACK-GFX801:   EF_AMDGPU_XNACK              (0x100)
 # ELF-ALL:          ]
 
-# YAML-XNACK-NONE:   Flags: [ EF_AMDGPU_MACH_NONE, EF_AMDGPU_XNACK ]
-# YAML-XNACK-GFX801: Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX801, EF_AMDGPU_XNACK ]
+# YAML-XNACK-NONE:   Flags: [ EF_AMDGPU_MACH_NONE, EF_AMDGPU_FEATURE_XNACK_V3 ]
+# YAML-XNACK-GFX801: Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX801, EF_AMDGPU_FEATURE_XNACK_V3 ]
 
 # Doc1
 --- !ELF
@@ -22,7 +22,7 @@ FileHeader:
   OSABI:   ELFOSABI_NONE
   Type:    ET_REL
   Machine: EM_AMDGPU
-  Flags:   [ EF_AMDGPU_XNACK ]
+  Flags:   [ EF_AMDGPU_FEATURE_XNACK_V3 ]
 ...
 
 # Doc2
@@ -33,5 +33,5 @@ FileHeader:
   OSABI:   ELFOSABI_NONE
   Type:    ET_REL
   Machine: EM_AMDGPU
-  Flags:   [ EF_AMDGPU_MACH_AMDGCN_GFX801, EF_AMDGPU_XNACK ]
+  Flags:   [ EF_AMDGPU_MACH_AMDGCN_GFX801, EF_AMDGPU_FEATURE_XNACK_V3 ]
 ...
diff --git a/llvm/test/Object/wasm-relocs-and-producers.yaml b/llvm/test/Object/wasm-relocs-and-producers.yaml
index 9bbb0a1d084db7b6dbe143be5ceb0deec77dd339..a96aa1fece1416e295ba797a8ce50da14f0d46c6 100644
--- a/llvm/test/Object/wasm-relocs-and-producers.yaml
+++ b/llvm/test/Object/wasm-relocs-and-producers.yaml
@@ -24,7 +24,7 @@ Sections:
         Field:           __linear_memory
         Kind:            MEMORY
         Memory:
-          Initial:         0x00000000
+          Minimum:         0x00000000
       - Module:          env
         Field:           __indirect_function_table
         Kind:            TABLE
@@ -32,7 +32,7 @@ Sections:
           Index:           0
           ElemType:        FUNCREF
           Limits:
-            Initial:         0x00000000
+            Minimum:         0x00000000
   - Type:            FUNCTION
     FunctionTypes:   [ 0 ]
   - Type:            CODE
diff --git a/llvm/test/ObjectYAML/wasm/data_section.yaml b/llvm/test/ObjectYAML/wasm/data_section.yaml
index 957687e2465f19a552f1f16078b9cf85ab061705..2289e8a728570f381cd1ced4c647f8f18eaa58fa 100644
--- a/llvm/test/ObjectYAML/wasm/data_section.yaml
+++ b/llvm/test/ObjectYAML/wasm/data_section.yaml
@@ -5,7 +5,7 @@ FileHeader:
 Sections:
   - Type:            MEMORY
     Memories:
-      - Initial:         0x00000003
+      - Minimum:         0x00000003
   - Type:            DATA
     Segments:
       - InitFlags:        0
diff --git a/llvm/test/ObjectYAML/wasm/elem_section.yaml b/llvm/test/ObjectYAML/wasm/elem_section.yaml
index e60db7ddc129d292c4ec2406d71d44edabb0c69e..c2563e6d76b498ea279ccec3e55f23d6510c5eff 100644
--- a/llvm/test/ObjectYAML/wasm/elem_section.yaml
+++ b/llvm/test/ObjectYAML/wasm/elem_section.yaml
@@ -9,7 +9,7 @@ Sections:
         ElemType:          FUNCREF
         Limits:
           Flags:           [ HAS_MAX ]
-          Initial:         0x00000010
+          Minimum:         0x00000010
           Maximum:         0x00000011
   - Type:            ELEM
     Segments:
diff --git a/llvm/test/ObjectYAML/wasm/import_memory_shared.yaml b/llvm/test/ObjectYAML/wasm/import_memory_shared.yaml
index 087ce843508722cebb3b064597363349e05e5a88..123181011e0c3dd5f4ef9742fe5c2d97f8fd91ba 100644
--- a/llvm/test/ObjectYAML/wasm/import_memory_shared.yaml
+++ b/llvm/test/ObjectYAML/wasm/import_memory_shared.yaml
@@ -17,7 +17,7 @@ Sections:
         Kind:            MEMORY
         Memory:
           Flags:           [ HAS_MAX, IS_SHARED ]
-          Initial:         0x00000010
+          Minimum:         0x00000010
           Maximum:         0x00000011
 
 ...
@@ -32,6 +32,6 @@ Sections:
 # CHECK:         Kind:            MEMORY
 # CHECK:         Memory:
 # CHECK:           Flags:           [ HAS_MAX, IS_SHARED ]
-# CHECK:           Initial:         0x10
+# CHECK:           Minimum:         0x10
 # CHECK:           Maximum:         0x11
 # CHECK: ...
diff --git a/llvm/test/ObjectYAML/wasm/import_section.yaml b/llvm/test/ObjectYAML/wasm/import_section.yaml
index bc71e5f2ab20fecee75d6a3b4f4b2c02d245dc97..e4c06368f6250f92669afb22ce70c22ddc8c3c29 100644
--- a/llvm/test/ObjectYAML/wasm/import_section.yaml
+++ b/llvm/test/ObjectYAML/wasm/import_section.yaml
@@ -26,7 +26,7 @@ Sections:
         Kind:            MEMORY
         Memory:
           Flags:           [ HAS_MAX ]
-          Initial:         0x00000010
+          Minimum:         0x00000010
           Maximum:         0x00000011
       - Module:          foo
         Field:           imported_table
@@ -36,7 +36,7 @@ Sections:
           ElemType:      FUNCREF
           Limits:
             Flags:           [ HAS_MAX ]
-            Initial:         0x00000020
+            Minimum:         0x00000020
             Maximum:         0x00000022
 ...
 # CHECK: --- !WASM
@@ -59,7 +59,7 @@ Sections:
 # CHECK:         Kind:            MEMORY
 # CHECK:         Memory:
 # CHECK:           Flags:           [ HAS_MAX ]
-# CHECK:           Initial:         0x10
+# CHECK:           Minimum:         0x10
 # CHECK:           Maximum:         0x11
 # CHECK:       - Module:          foo
 # CHECK:         Field:           imported_table
@@ -68,6 +68,6 @@ Sections:
 # CHECK:           ElemType:      FUNCREF
 # CHECK:           Limits:
 # CHECK:             Flags:           [ HAS_MAX ]
-# CHECK:             Initial:         0x20
+# CHECK:             Minimum:         0x20
 # CHECK:             Maximum:         0x22
 # CHECK: ...
diff --git a/llvm/test/ObjectYAML/wasm/memory_section.yaml b/llvm/test/ObjectYAML/wasm/memory_section.yaml
index 22ca72419af8869c01f75e4bcbfbc77bac33a9ee..6cd9db3ffd1d2beb48a1b84d8dc7cd6f2bc1ef5d 100644
--- a/llvm/test/ObjectYAML/wasm/memory_section.yaml
+++ b/llvm/test/ObjectYAML/wasm/memory_section.yaml
@@ -6,9 +6,9 @@ Sections:
   - Type:            MEMORY
     Memories:         
       - Flags:           [ HAS_MAX ]
-        Initial:         0x00000002
+        Minimum:         0x00000002
         Maximum:         0x000000FF
-      - Initial:         0x00000003
+      - Minimum:         0x00000003
 ...
 # CHECK: --- !WASM
 # CHECK: FileHeader:
@@ -17,7 +17,7 @@ Sections:
 # CHECK:   - Type:            MEMORY
 # CHECK:     Memories:         
 # CHECK:       - Flags:            [ HAS_MAX ]
-# CHECK:         Initial:          0x2
+# CHECK:         Minimum:          0x2
 # CHECK:         Maximum:          0xFF
-# CHECK:       - Initial:          0x3
+# CHECK:       - Minimum:          0x3
 # CHECK: ...
diff --git a/llvm/test/ObjectYAML/wasm/multiple-tables.yaml b/llvm/test/ObjectYAML/wasm/multiple-tables.yaml
index 50f03bc90641802d35c03cc677a5d1684124655e..b16a5396e01225c471cb193666b8ce09d35d0188 100644
--- a/llvm/test/ObjectYAML/wasm/multiple-tables.yaml
+++ b/llvm/test/ObjectYAML/wasm/multiple-tables.yaml
@@ -18,7 +18,7 @@ Sections:
           Index:           0
           ElemType:        FUNCREF
           Limits:
-            Initial:         0x0
+            Minimum:         0x0
   - Type:            FUNCTION
     FunctionTypes:   [ 0 ]
   - Type:            TABLE
@@ -26,16 +26,16 @@ Sections:
       - Index:           1
         ElemType:        FUNCREF
         Limits:
-          Initial:         0x0
+          Minimum:         0x0
       - Index:           2
         ElemType:        EXTERNREF
         Limits:
-          Initial:         0x0
+          Minimum:         0x0
       - Index:           3
         ElemType:        FUNCREF
         Limits:
           Flags:           [ HAS_MAX ]
-          Initial:         0x3
+          Minimum:         0x3
           Maximum:         0x3
   - Type:            EXPORT
     Exports:
@@ -79,7 +79,7 @@ Sections:
 # CHECK-NEXT:           Index:           0
 # CHECK-NEXT:           ElemType:        FUNCREF
 # CHECK-NEXT:           Limits:
-# CHECK-NEXT:             Initial:         0x0
+# CHECK-NEXT:             Minimum:         0x0
 # CHECK-NEXT:   - Type:            FUNCTION
 # CHECK-NEXT:     FunctionTypes:   [ 0 ]
 # CHECK-NEXT:   - Type:            TABLE
@@ -87,16 +87,16 @@ Sections:
 # CHECK-NEXT:       - Index:           1
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
-# CHECK-NEXT:           Initial:         0x0
+# CHECK-NEXT:           Minimum:         0x0
 # CHECK-NEXT:       - Index:           2
 # CHECK-NEXT:         ElemType:        EXTERNREF
 # CHECK-NEXT:         Limits:
-# CHECK-NEXT:           Initial:         0x0
+# CHECK-NEXT:           Minimum:         0x0
 # CHECK-NEXT:       - Index:           3
 # CHECK-NEXT:         ElemType:        FUNCREF
 # CHECK-NEXT:         Limits:
 # CHECK-NEXT:           Flags:           [ HAS_MAX ]
-# CHECK-NEXT:           Initial:         0x3
+# CHECK-NEXT:           Minimum:         0x3
 # CHECK-NEXT:           Maximum:         0x3
 # CHECK-NEXT:   - Type:            EXPORT
 # CHECK-NEXT:     Exports:
diff --git a/llvm/test/ObjectYAML/wasm/table_section.yaml b/llvm/test/ObjectYAML/wasm/table_section.yaml
index 18cff6de4166da7d1d6187a19a0781b2fce97555..e81b76626ab1294d1002aba1a3ecc4f86de16fd7 100644
--- a/llvm/test/ObjectYAML/wasm/table_section.yaml
+++ b/llvm/test/ObjectYAML/wasm/table_section.yaml
@@ -9,7 +9,7 @@ Sections:
         ElemType:        FUNCREF
         Limits:
           Flags:           [ HAS_MAX ]
-          Initial:         0x00000010
+          Minimum:         0x00000010
           Maximum:         0x00000011
 ...
 # CHECK: --- !WASM
@@ -22,6 +22,6 @@ Sections:
 # CHECK:        ElemType:        FUNCREF
 # CHECK:        Limits:
 # CHECK:          Flags:           [ HAS_MAX ]
-# CHECK:          Initial:         0x10
+# CHECK:          Minimum:         0x10
 # CHECK:          Maximum:         0x11
 # CHECK: ...
diff --git a/llvm/test/Other/ChangePrinters/print-changed-diff.ll b/llvm/test/Other/ChangePrinters/print-changed-diff.ll
index 45322c29508e8a962ba2908625fa4cd0539c18c2..b5d408fcfed0ac8aa9c88c5413f32c4b15e2d239 100644
--- a/llvm/test/Other/ChangePrinters/print-changed-diff.ll
+++ b/llvm/test/Other/ChangePrinters/print-changed-diff.ll
@@ -62,6 +62,70 @@
 ; instsimplify is run on f will result in changes
 ; RUN: opt -S -print-changed=diff-quiet -passes="instsimplify,instsimplify" -filter-print-funcs=f  2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-DIFF-QUIET-MULT-PASSES-FILTER-FUNC
 
+; Simple checks of -print-changed=cdiff
+;
+; Note that (mostly) only the banners are checked.
+;
+; Simple functionality check.
+; RUN: opt -S -print-changed=cdiff -passes=instsimplify 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-SIMPLE
+;
+; Check that only the passes that change the IR are printed and that the
+; others (including g) are filtered out.
+; RUN: opt -S -print-changed=cdiff -passes=instsimplify -filter-print-funcs=f  2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-FUNC-FILTER
+;
+; Check that the reporting of IRs respects is not affected by
+; -print-module-scope
+; RUN: opt -S -print-changed=cdiff -passes=instsimplify -print-module-scope 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-PRINT-MOD-SCOPE
+;
+; Check that reporting of multiple functions happens
+; RUN: opt -S -print-changed=cdiff -passes=instsimplify -filter-print-funcs="f,g" 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-FILTER-MULT-FUNC
+;
+; Check that the reporting of IRs respects -filter-passes
+; RUN: opt -S -print-changed=cdiff -passes="instsimplify,no-op-function" -filter-passes="NoOpFunctionPass" 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-FILTER-PASSES
+;
+; Check that the reporting of IRs respects -filter-passes with multiple passes
+; RUN: opt -S -print-changed=cdiff -passes="instsimplify,no-op-function" -filter-passes="NoOpFunctionPass,InstSimplifyPass" 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-FILTER-MULT-PASSES
+;
+; Check that the reporting of IRs respects both -filter-passes and -filter-print-funcs
+; RUN: opt -S -print-changed=cdiff -passes="instsimplify,no-op-function" -filter-passes="NoOpFunctionPass,InstSimplifyPass" -filter-print-funcs=f 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-FILTER-FUNC-PASSES
+;
+; Check that repeated passes that change the IR are printed and that the
+; others (including g) are filtered out.  Note that only the first time
+; instsimplify is run on f will result in changes
+; RUN: opt -S -print-changed=cdiff -passes="instsimplify,instsimplify" -filter-print-funcs=f  2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-MULT-PASSES-FILTER-FUNC
+;
+; Simple checks of -print-changed=cdiff-quiet
+;
+; Note that (mostly) only the banners are checked.
+;
+; Simple functionality check.
+; RUN: opt -S -print-changed=cdiff-quiet -passes=instsimplify 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-QUIET-SIMPLE --allow-empty
+;
+; Check that only the passes that change the IR are printed and that the
+; others (including g) are filtered out.
+; RUN: opt -S -print-changed=cdiff-quiet -passes=instsimplify -filter-print-funcs=f  2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-QUIET-FUNC-FILTER
+;
+; Check that the reporting of IRs respects is not affected by
+; -print-module-scope
+; RUN: opt -S -print-changed=cdiff-quiet -passes=instsimplify -print-module-scope 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE
+;
+; Check that reporting of multiple functions happens
+; RUN: opt -S -print-changed=cdiff-quiet -passes=instsimplify -filter-print-funcs="f,g" 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-QUIET-FILTER-MULT-FUNC
+;
+; Check that the reporting of IRs respects -filter-passes
+; RUN: opt -S -print-changed=cdiff-quiet -passes="instsimplify,no-op-function" -filter-passes="NoOpFunctionPass" 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-QUIET-FILTER-PASSES-NONE --allow-empty
+;
+; Check that the reporting of IRs respects -filter-passes with multiple passes
+; RUN: opt -S -print-changed=cdiff-quiet -passes="instsimplify,no-op-function" -filter-passes="NoOpFunctionPass,InstSimplifyPass" 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-QUIET-FILTER-MULT-PASSES
+;
+; Check that the reporting of IRs respects both -filter-passes and -filter-print-funcs
+; RUN: opt -S -print-changed=cdiff-quiet -passes="instsimplify,no-op-function" -filter-passes="NoOpFunctionPass,InstSimplifyPass" -filter-print-funcs=f 2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-QUIET-FILTER-FUNC-PASSES
+;
+; Check that repeated passes that change the IR are printed and that the
+; others (including g) are filtered out.  Note that only the first time
+; instsimplify is run on f will result in changes
+; RUN: opt -S -print-changed=cdiff-quiet -passes="instsimplify,instsimplify" -filter-print-funcs=f  2>&1 -o /dev/null < %s | FileCheck %s --check-prefix=CHECK-CDIFF-QUIET-MULT-PASSES-FILTER-FUNC
+
 define i32 @g() {
 entry:
   %a = add i32 2, 3
@@ -286,3 +350,216 @@ entry:
 ; CHECK-DIFF-QUIET-MULT-PASSES-FILTER-FUNC:-  ret i32 %a
 ; CHECK-DIFF-QUIET-MULT-PASSES-FILTER-FUNC:+  ret i32 5
 ; CHECK-DIFF-QUIET-MULT-PASSES-FILTER-FUNC-NOT: *** IR
+
+; CHECK-CDIFF-SIMPLE: *** IR Dump At Start: ***
+; CHECK-CDIFF-SIMPLE: ModuleID = {{.+}}
+; CHECK-CDIFF-SIMPLE: *** IR Dump After VerifierPass (module) omitted because no change ***
+; CHECK-CDIFF-SIMPLE: *** IR Dump After InstSimplifyPass *** (function: g)
+; CHECK-CDIFF-SIMPLE-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-SIMPLE-NOT: *** IR{{.*}}
+; CHECK-CDIFF-SIMPLE: entry:
+; CHECK-CDIFF-SIMPLE-NEXT:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-SIMPLE-NEXT:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-SIMPLE-NEXT:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-SIMPLE: *** IR Pass PassManager{{.*}} (function: g) ignored ***
+; CHECK-CDIFF-SIMPLE: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-SIMPLE-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-SIMPLE-NOT: *** IR{{.*}}
+; CHECK-CDIFF-SIMPLE: entry:
+; CHECK-CDIFF-SIMPLE-NEXT:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-SIMPLE-NEXT:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-SIMPLE-NEXT:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-SIMPLE: *** IR Pass PassManager{{.*}} (function: f) ignored ***
+; CHECK-CDIFF-SIMPLE: *** IR Pass ModuleToFunctionPassAdaptor (module) ignored ***
+; CHECK-CDIFF-SIMPLE: *** IR Dump After VerifierPass (module) omitted because no change ***
+; CHECK-CDIFF-SIMPLE: *** IR Dump After PrintModulePass (module) omitted because no change ***
+
+; CHECK-CDIFF-FUNC-FILTER: *** IR Dump At Start: ***
+; CHECK-CDIFF-FUNC-FILTER-NEXT: ; ModuleID = {{.+}}
+; CHECK-CDIFF-FUNC-FILTER: *** IR Dump After InstSimplifyPass (function: g) filtered out ***
+; CHECK-CDIFF-FUNC-FILTER: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-FUNC-FILTER-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-FUNC-FILTER: entry:
+; CHECK-CDIFF-FUNC-FILTER:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-FUNC-FILTER:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-FUNC-FILTER:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-FUNC-FILTER: *** IR Pass PassManager{{.*}} (function: f) ignored ***
+; CHECK-CDIFF-FUNC-FILTER: *** IR Pass ModuleToFunctionPassAdaptor (module) ignored ***
+; CHECK-CDIFF-FUNC-FILTER: *** IR Dump After VerifierPass (module) omitted because no change ***
+; CHECK-CDIFF-FUNC-FILTER: *** IR Dump After PrintModulePass (module) omitted because no change ***
+
+; CHECK-CDIFF-PRINT-MOD-SCOPE: *** IR Dump At Start: ***
+; CHECK-CDIFF-PRINT-MOD-SCOPE: ModuleID = {{.+}}
+; CHECK-CDIFF-PRINT-MOD-SCOPE: *** IR Dump After InstSimplifyPass *** (function: g)
+; CHECK-CDIFF-PRINT-MOD-SCOPE-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-PRINT-MOD-SCOPE: entry:
+; CHECK-CDIFF-PRINT-MOD-SCOPE:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-PRINT-MOD-SCOPE:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-PRINT-MOD-SCOPE:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-PRINT-MOD-SCOPE: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-PRINT-MOD-SCOPE-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-PRINT-MOD-SCOPE: entry:
+; CHECK-CDIFF-PRINT-MOD-SCOPE:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-PRINT-MOD-SCOPE:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-PRINT-MOD-SCOPE:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-PRINT-MOD-SCOPE: *** IR Pass PassManager{{.*}} (function: f) ignored ***
+; CHECK-CDIFF-PRINT-MOD-SCOPE: *** IR Pass ModuleToFunctionPassAdaptor (module) ignored ***
+; CHECK-CDIFF-PRINT-MOD-SCOPE: *** IR Dump After VerifierPass (module) omitted because no change ***
+; CHECK-CDIFF-PRINT-MOD-SCOPE: *** IR Dump After PrintModulePass (module) omitted because no change ***
+
+; CHECK-CDIFF-FILTER-MULT-FUNC: *** IR Dump At Start: ***
+; CHECK-CDIFF-FILTER-MULT-FUNC: *** IR Dump After InstSimplifyPass *** (function: g)
+; CHECK-CDIFF-FILTER-MULT-FUNC-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-FILTER-MULT-FUNC: entry:
+; CHECK-CDIFF-FILTER-MULT-FUNC:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-FUNC:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-FUNC:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-FUNC: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-FILTER-MULT-FUNC-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-FILTER-MULT-FUNC: entry:
+; CHECK-CDIFF-FILTER-MULT-FUNC:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-FUNC:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-FUNC:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-FUNC: *** IR Pass PassManager{{.*}} (function: f) ignored ***
+; CHECK-CDIFF-FILTER-MULT-FUNC: *** IR Pass ModuleToFunctionPassAdaptor (module) ignored ***
+; CHECK-CDIFF-FILTER-MULT-FUNC: *** IR Dump After VerifierPass (module) omitted because no change ***
+; CHECK-CDIFF-FILTER-MULT-FUNC: *** IR Dump After PrintModulePass (module) omitted because no change ***
+
+; CHECK-CDIFF-FILTER-PASSES: *** IR Dump After InstSimplifyPass (function: g) filtered out ***
+; CHECK-CDIFF-FILTER-PASSES: *** IR Dump At Start: *** (function: g)
+; CHECK-CDIFF-FILTER-PASSES: *** IR Dump After NoOpFunctionPass (function: g) omitted because no change ***
+; CHECK-CDIFF-FILTER-PASSES: *** IR Dump After InstSimplifyPass (function: f) filtered out ***
+; CHECK-CDIFF-FILTER-PASSES: *** IR Dump After NoOpFunctionPass (function: f) omitted because no change ***
+
+; CHECK-CDIFF-FILTER-MULT-PASSES: *** IR Dump At Start: *** (function: g)
+; CHECK-CDIFF-FILTER-MULT-PASSES: *** IR Dump After InstSimplifyPass *** (function: g)
+; CHECK-CDIFF-FILTER-MULT-PASSES-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-FILTER-MULT-PASSES: entry:
+; CHECK-CDIFF-FILTER-MULT-PASSES:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-PASSES:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-PASSES:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-PASSES: *** IR Dump After NoOpFunctionPass (function: g) omitted because no change ***
+; CHECK-CDIFF-FILTER-MULT-PASSES: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-FILTER-MULT-PASSES-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-FILTER-MULT-PASSES: entry:
+; CHECK-CDIFF-FILTER-MULT-PASSES:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-PASSES:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-PASSES:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-FILTER-MULT-PASSES: *** IR Dump After NoOpFunctionPass (function: f) omitted because no change ***
+
+; CHECK-CDIFF-FILTER-FUNC-PASSES: *** IR Dump After InstSimplifyPass (function: g) filtered out ***
+; CHECK-CDIFF-FILTER-FUNC-PASSES: *** IR Dump After NoOpFunctionPass (function: g) filtered out ***
+; CHECK-CDIFF-FILTER-FUNC-PASSES: *** IR Dump At Start: *** (function: f)
+; CHECK-CDIFF-FILTER-FUNC-PASSES: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-FILTER-FUNC-PASSES-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-FILTER-FUNC-PASSES: entry:
+; CHECK-CDIFF-FILTER-FUNC-PASSES:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-FILTER-FUNC-PASSES:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-FILTER-FUNC-PASSES:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-FILTER-FUNC-PASSES: *** IR Dump After NoOpFunctionPass (function: f) omitted because no change ***
+
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC: *** IR Dump At Start: ***
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC: *** IR Dump After InstSimplifyPass (function: g) filtered out ***
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC: *** IR Dump After InstSimplifyPass (function: g) filtered out ***
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC: entry:
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-MULT-PASSES-FILTER-FUNC: *** IR Dump After InstSimplifyPass (function: f) omitted because no change ***
+
+; CHECK-CDIFF-QUIET-SIMPLE-NOT: *** IR Dump {{.*(At Start:|no change|ignored|filtered out)}} ***
+; CHECK-CDIFF-QUIET-SIMPLE: *** IR Dump After InstSimplifyPass *** (function: g)
+; CHECK-CDIFF-QUIET-SIMPLE-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-SIMPLE-NOT: *** IR{{.*}}
+; CHECK-CDIFF-QUIET-SIMPLE: entry:
+; CHECK-CDIFF-QUIET-SIMPLE-NEXT:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-SIMPLE-NEXT:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-SIMPLE-NEXT:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-SIMPLE-EMPTY:
+; CHECK-CDIFF-QUIET-SIMPLE-NEXT: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-QUIET-SIMPLE-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-SIMPLE-NOT: *** IR{{.*}}
+; CHECK-CDIFF-QUIET-SIMPLE: entry:
+; CHECK-CDIFF-QUIET-SIMPLE-NEXT:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-SIMPLE-NEXT:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-SIMPLE-NEXT:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-SIMPLE-NOT: *** IR{{.*}}
+
+; CHECK-CDIFF-QUIET-FUNC-FILTER-NOT: *** IR Dump {{.*(At Start:|no change|ignored|filtered out)}} ***
+; CHECK-CDIFF-QUIET-FUNC-FILTER: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-QUIET-FUNC-FILTER-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-FUNC-FILTER: entry:
+; CHECK-CDIFF-QUIET-FUNC-FILTER:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-FUNC-FILTER:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-FUNC-FILTER:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-FUNC-FILTER-NOT: *** IR{{.*}}
+
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE-NOT: *** IR Dump {{.*(At Start:|no change|ignored|filtered out)}} ***
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE: *** IR Dump After InstSimplifyPass *** (function: g)
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE: entry:
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE-EMPTY:
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE-NEXT: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE: entry:
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-PRINT-MOD-SCOPE-NOT: *** IR{{.*}}
+
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC-NOT: *** IR Dump {{.*(At Start:|no change|ignored|filtered out)}} ***
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC: *** IR Dump After InstSimplifyPass *** (function: g)
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC: entry:
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC-EMPTY:
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC-NEXT: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC: entry:
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-FUNC-NOT: *** IR{{.*}}
+
+; CHECK-CDIFF-QUIET-FILTER-PASSES-NONE-NOT: *** IR
+
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES-NOT: *** IR Dump {{.*(At Start:|no change|ignored|filtered out)}} ***
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES: *** IR Dump After InstSimplifyPass *** (function: g)
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES: entry:
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES-EMPTY:
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES: entry:
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-MULT-PASSES-NOT: *** IR
+
+; CHECK-CDIFF-QUIET-FILTER-FUNC-PASSES-NOT: *** IR Dump {{.*(At Start:|no change|ignored|filtered out)}} ***
+; CHECK-CDIFF-QUIET-FILTER-FUNC-PASSES: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-QUIET-FILTER-FUNC-PASSES-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-FILTER-FUNC-PASSES: entry:
+; CHECK-CDIFF-QUIET-FILTER-FUNC-PASSES:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-FUNC-PASSES:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-FUNC-PASSES:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-FILTER-FUNC-PASSES-NOT: *** IR
+
+; CHECK-CDIFF-QUIET-MULT-PASSES-FILTER-FUNC-NOT: *** IR Dump {{.*(At Start:|no change|ignored|filtered out)}} ***
+; CHECK-CDIFF-QUIET-MULT-PASSES-FILTER-FUNC: *** IR Dump After InstSimplifyPass *** (function: f)
+; CHECK-CDIFF-QUIET-MULT-PASSES-FILTER-FUNC-NOT: ModuleID = {{.+}}
+; CHECK-CDIFF-QUIET-MULT-PASSES-FILTER-FUNC: entry:
+; CHECK-CDIFF-QUIET-MULT-PASSES-FILTER-FUNC:{{.\[31m-}}  %a = add i32 2, 3{{.\[0m}}
+; CHECK-CDIFF-QUIET-MULT-PASSES-FILTER-FUNC:{{.\[31m-}}  ret i32 %a{{.\[0m}}
+; CHECK-CDIFF-QUIET-MULT-PASSES-FILTER-FUNC:{{.\[32m\+}}  ret i32 5{{.\[0m}}
+; CHECK-CDIFF-QUIET-MULT-PASSES-FILTER-FUNC-NOT: *** IR
diff --git a/llvm/test/Other/debugcounter-predicateinfo.ll b/llvm/test/Other/debugcounter-predicateinfo.ll
index bbc7a0f712711dfe74798d718037bbc4c642f2a9..90303ab0dabcbf368ddbcba5c5807fbb4c413f14 100644
--- a/llvm/test/Other/debugcounter-predicateinfo.ll
+++ b/llvm/test/Other/debugcounter-predicateinfo.ll
@@ -8,10 +8,10 @@ define fastcc void @barney() {
 ; CHECK-NEXT:    br label [[BB22:%.*]]
 ; CHECK:       bb22:
 ; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i32 undef, 2
-; CHECK:         [[TMP23_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP23]])
+; CHECK:         [[TMP23_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP23]])
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[BB29:%.*]], label [[BB35:%.*]]
 ; CHECK:       bb29:
-; CHECK:         [[TMP23_0_1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP23_0]])
+; CHECK:         [[TMP23_0_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP23_0]])
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[BB33:%.*]], label [[BB35]]
 ; CHECK:       bb33:
 ; CHECK-NEXT:    br i1 [[TMP23_0_1]], label [[BB35]], label [[BB35]]
diff --git a/llvm/test/Other/optimization-remarks-auto.ll b/llvm/test/Other/optimization-remarks-auto.ll
index c534167364952b2928e92ef955be09a0f4094636..cdedefce78a83a689ff6e0300403e9988d8ebc95 100644
--- a/llvm/test/Other/optimization-remarks-auto.ll
+++ b/llvm/test/Other/optimization-remarks-auto.ll
@@ -10,14 +10,14 @@
 ; RUN: FileCheck %s -check-prefix=YAML-MISS < %t.yaml
 
 ;; test 'auto' threshold
-; RUN: opt < %s --disable-output --enable-new-pm \
+; RUN: opt < %s --disable-output --enable-new-pm --inline-enable-cost-benefit-analysis=0 \
 ; RUN: --passes='module(print-profile-summary,cgscc(inline))' \
 ; RUN: --pass-remarks-output=%t.hot.yaml --pass-remarks-filter='inline' \
 ; RUN: --pass-remarks-with-hotness --pass-remarks-hotness-threshold=auto 2>&1 | FileCheck %s
 ; RUN: FileCheck %s -check-prefix=YAML-PASS < %t.hot.yaml
 ; RUN: not FileCheck %s -check-prefix=YAML-MISS < %t.hot.yaml
 
-; RUN: opt < %s --disable-output --enable-new-pm \
+; RUN: opt < %s --disable-output --enable-new-pm --inline-enable-cost-benefit-analysis=0  \
 ; RUN: --passes='module(print-profile-summary,cgscc(inline))' \
 ; RUN: --pass-remarks=inline --pass-remarks-missed=inline --pass-remarks-analysis=inline \
 ; RUN: --pass-remarks-with-hotness --pass-remarks-hotness-threshold=auto 2>&1 | FileCheck %s -check-prefix=CHECK-RPASS
@@ -82,4 +82,3 @@ entry:
 !12 = !{i32 10000, i64 100, i32 1}
 !13 = !{i32 999000, i64 100, i32 1}
 !14 = !{i32 999999, i64 1, i32 2}
-
diff --git a/llvm/test/TableGen/BitOffsetDecoder.td b/llvm/test/TableGen/BitOffsetDecoder.td
index f94e8d4f09789f34b72a84329723704bba6d6958..04d6e164d0eee3eb41760d4c4ddeccc3a9d4ee7c 100644
--- a/llvm/test/TableGen/BitOffsetDecoder.td
+++ b/llvm/test/TableGen/BitOffsetDecoder.td
@@ -59,6 +59,6 @@ def baz : Instruction {
 
 // CHECK: tmp = fieldFromInstruction(insn, 8, 7);
 // CHECK: tmp = fieldFromInstruction(insn, 8, 8) << 3;
-// CHECK: tmp |= fieldFromInstruction(insn, 8, 4) << 7;
-// CHECK: tmp |= fieldFromInstruction(insn, 12, 4) << 3;
+// CHECK: insertBits(tmp, fieldFromInstruction(insn, 8, 4), 7, 4);
+// CHECK: insertBits(tmp, fieldFromInstruction(insn, 12, 4), 3, 4);
 // CHECK: tmp = fieldFromInstruction(insn, 8, 8) << 4;
diff --git a/llvm/test/TableGen/FixedLenDecoderEmitter/InitValue.td b/llvm/test/TableGen/FixedLenDecoderEmitter/InitValue.td
index f0f7b00af825b1be4f3574d43a700e4b6a10cedb..03847439ffc2e690cfb9fec6dd7b3eeb6b753e78 100644
--- a/llvm/test/TableGen/FixedLenDecoderEmitter/InitValue.td
+++ b/llvm/test/TableGen/FixedLenDecoderEmitter/InitValue.td
@@ -41,6 +41,6 @@ def bax : Instruction {
 
 // CHECK: tmp = fieldFromInstruction(insn, 9, 7) << 1;
 // CHECK: tmp = 0x1;
-// CHECK: tmp |= fieldFromInstruction(insn, 9, 7) << 1;
+// CHECK: insertBits(tmp, fieldFromInstruction(insn, 9, 7), 1, 7);
 // CHECK: tmp = 0x100000000;
-// CHECK: tmp |= fieldFromInstruction(insn, 8, 7) << 25;
+// CHECK: insertBits(tmp, fieldFromInstruction(insn, 8, 7), 25, 7);
diff --git a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td
index 6f6320f6389d0161ce7eaa02f9522738ec36da55..408055da34c985648c8232812ba4d94c4bc56b1a 100644
--- a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td
+++ b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td
@@ -1,5 +1,27 @@
 // RUN: llvm-tblgen %s -gen-global-isel -optimize-match-table=false -I %p/../../include -I %p/Common -o - | FileCheck %s
 
+// Verify that all MI predicates are enumerated.
+//
+// CHECK: // PatFrag predicates.
+// CHECK-NEXT: enum {
+// CHECK-NEXT:   GIPFP_MI_Predicate_and_or_pat = GIPFP_MI_Invalid + 1,
+// CHECK-NEXT:   GIPFP_MI_Predicate_or_oneuse,
+// CHECK-NEXT:   GIPFP_MI_Predicate_patfrags_test_pat,
+// CHECK-NEXT:   GIPFP_MI_Predicate_sub3_pat,
+// CHECK-NEXT: };
+
+// Verify that we emit cases for all MI predicates.
+//
+// CHECK: bool MyTargetInstructionSelector::testMIPredicate_MI(
+// CHECK:    case GIPFP_MI_Predicate_and_or_pat: {
+// CHECK:      llvm_unreachable("GISelPredicateCode should have returned");
+// CHECK:    case GIPFP_MI_Predicate_or_oneuse: {
+// CHECK:      llvm_unreachable("GISelPredicateCode should have returned");
+// CHECK:    case GIPFP_MI_Predicate_patfrags_test_pat: {
+// CHECK:      llvm_unreachable("GISelPredicateCode should have returned");
+// CHECK:    case GIPFP_MI_Predicate_sub3_pat: {
+// CHECK:      llvm_unreachable("GISelPredicateCode should have returned");
+
 include "llvm/Target/Target.td"
 include "GlobalISelEmitterCommon.td"
 
@@ -28,7 +50,6 @@ def DRegs : MyClass<32, [i32], (sequence "D%u", 0, 0)>;
 def DOP : RegisterOperand<DRegs>;
 def AND_OR : I<(outs DRegs:$dst), (ins DOP:$src0, DOP:$src1, DOP:$src2), []>;
 
-
 def or_oneuse : PatFrag<
   (ops node:$x, node:$y),
   (or node:$x, node:$y), [{ return foo(); }]> {
@@ -48,7 +69,7 @@ def and_or_pat : PatFrag<
   let PredicateCodeUsesOperands = 1;
 }
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ 99, // Rule ID 2 //
+// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ 99, // Rule ID 6 //
 // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
 // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND,
 // CHECK-NEXT: // MIs[0] dst
@@ -56,7 +77,7 @@ def and_or_pat : PatFrag<
 // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID,
 // CHECK-NEXT: // MIs[0] src2
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/2, // Name : pred:2:z
+// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/2, // Name : pred:3:z
 // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/Test::DRegsRegClassID,
 // CHECK-NEXT: // MIs[0] Operand 2
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
@@ -67,18 +88,18 @@ def and_or_pat : PatFrag<
 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
 // CHECK-NEXT: // MIs[1] src0
 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:2:x
+// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:3:x
 // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID,
 // CHECK-NEXT: // MIs[1] src1
 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:2:y
+// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:3:y
 // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID,
 // CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIPFP_MI_Predicate_and_or_pat,
 // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
-// CHECK-NEXT: // (and:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:2:z, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:2:x, DOP:{ *:[i32] }:$src1:$pred:2:y))<<P:2:Predicate_and_or_pat>>  =>  (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
+// CHECK-NEXT: // (and:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:3:z, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:3:x, DOP:{ *:[i32] }:$src1:$pred:3:y))<<P:3:Predicate_and_or_pat>>  =>  (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
 // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR,
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ 198, // Rule ID 1 //
+// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ 198, // Rule ID 3 //
 // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
 // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND,
 // CHECK-NEXT: // MIs[0] dst
@@ -93,19 +114,19 @@ def and_or_pat : PatFrag<
 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
 // CHECK-NEXT: // MIs[1] src0
 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:2:x
+// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:3:x
 // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID,
 // CHECK-NEXT: // MIs[1] src1
 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:2:y
+// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:3:y
 // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID,
 // CHECK-NEXT: // MIs[0] src2
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:2:z
+// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:3:z
 // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/Test::DRegsRegClassID,
 // CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIPFP_MI_Predicate_and_or_pat,
 // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
-// CHECK-NEXT: // (and:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:2:x, DOP:{ *:[i32] }:$src1:$pred:2:y), DOP:{ *:[i32] }:$src2:$pred:2:z)<<P:2:Predicate_and_or_pat>>  =>  (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
+// CHECK-NEXT: // (and:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:3:x, DOP:{ *:[i32] }:$src1:$pred:3:y), DOP:{ *:[i32] }:$src2:$pred:3:z)<<P:3:Predicate_and_or_pat>>  =>  (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
 // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR,
 
 // Test commutative, standalone pattern.
@@ -157,3 +178,35 @@ def SUB3 : I<(outs DRegs:$dst),
   (ins DOP:$src0, DOP:$src1, DOP:$src2),
   [(set DRegs:$dst, (sub3_pat i32:$src0, i32:$src1, i32:$src2))]
 >;
+
+
+def patfrags_test_pat : PatFrags<
+  (ops node:$x, node:$y, node:$z),
+  [ (xor (add node:$x, node:$y), node:$z),
+    (xor (sub node:$x, node:$y), node:$z)
+  ], [{ return foo(); }]> {
+  let GISelPredicateCode = [{
+    return doesComplexCheck(MI);
+  }];
+
+  let PredicateCodeUsesOperands = 1;
+}
+
+// CHECK: GIM_Try, /*On fail goto*//*Label 3*/ 372, // Rule ID 1 //
+// CHECK: // (xor:{ *:[i32] } (add:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y), i32:{ *:[i32] }:$src2:$pred:2:z)<<P:2:Predicate_patfrags_test_pat>>  =>  (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
+
+// CHECK: GIM_Try, /*On fail goto*//*Label 4*/ 459, // Rule ID 2 //
+// CHECK: // (xor:{ *:[i32] } (sub:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y), i32:{ *:[i32] }:$src2:$pred:2:z)<<P:2:Predicate_patfrags_test_pat>>  =>  (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
+
+// CHECK: GIM_Try, /*On fail goto*//*Label 5*/ 546, // Rule ID 4 //
+// CHECK: // (xor:{ *:[i32] } i32:{ *:[i32] }:$src2:$pred:2:z, (add:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y))<<P:2:Predicate_patfrags_test_pat>>  =>  (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
+
+// CHECK: GIM_Try, /*On fail goto*//*Label 6*/ 633, // Rule ID 5 //
+// CHECK: // (xor:{ *:[i32] } i32:{ *:[i32] }:$src2:$pred:2:z, (sub:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y))<<P:2:Predicate_patfrags_test_pat>>  =>  (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
+
+
+// Test a commutative pattern using multiple patterns using PatFrags.
+def PATFRAGS : I<(outs DRegs:$dst),
+  (ins DOP:$src0, DOP:$src1, DOP:$src2),
+  [(set DRegs:$dst, (patfrags_test_pat i32:$src0, i32:$src1, i32:$src2))]
+>;
diff --git a/llvm/test/TableGen/directive1.td b/llvm/test/TableGen/directive1.td
index dbf9b6c03d3a29a8db9d9ad56b4c623b2c161f65..b5fc08c15a5d99c0b7ed6e8a84226a5244a19640 100644
--- a/llvm/test/TableGen/directive1.td
+++ b/llvm/test/TableGen/directive1.td
@@ -1,6 +1,5 @@
 // RUN: llvm-tblgen -gen-directive-decl -I %p/../../include %s | FileCheck -match-full-lines %s
 // RUN: llvm-tblgen -gen-directive-impl -I %p/../../include %s | FileCheck -match-full-lines %s -check-prefix=IMPL
-// RUN: llvm-tblgen -gen-directive-gen -I %p/../../include %s | FileCheck -match-full-lines %s -check-prefix=GEN
 
 include "llvm/Frontend/Directive/DirectiveBase.td"
 
@@ -101,12 +100,143 @@ def TDL_DirA : Directive<"dira"> {
 // CHECK-NEXT:  #endif // LLVM_Tdl_INC
 
 
-// IMPL:       #include "llvm/ADT/StringRef.h"
-// IMPL-NEXT:  #include "llvm/ADT/StringSwitch.h"
-// IMPL-NEXT:  #include "llvm/Support/ErrorHandling.h"
+// IMPL:       #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// IMPL-NEXT:  #undef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
 // IMPL-EMPTY:
-// IMPL-NEXT:  using namespace llvm;
-// IMPL-NEXT:  using namespace tdl;
+// IMPL-NEXT:  namespace llvm {
+// IMPL-NEXT:  namespace tdl {
+// IMPL-EMPTY:
+// IMPL-NEXT:    // Sets for dira
+// IMPL-EMPTY:
+// IMPL-NEXT:    static  allowedClauses_TDLD_dira {
+// IMPL-NEXT:      llvm::tdl::Clause::TDLC_clausea,
+// IMPL-NEXT:      llvm::tdl::Clause::TDLC_clauseb,
+// IMPL-NEXT:    };
+// IMPL-EMPTY:
+// IMPL-NEXT:    static  allowedOnceClauses_TDLD_dira {
+// IMPL-NEXT:    };
+// IMPL-EMPTY:
+// IMPL-NEXT:    static  allowedExclusiveClauses_TDLD_dira {
+// IMPL-NEXT:    };
+// IMPL-EMPTY:
+// IMPL-NEXT:    static  requiredClauses_TDLD_dira {
+// IMPL-NEXT:    };
+// IMPL-NEXT:  } // namespace tdl
+// IMPL-NEXT:  } // namespace llvm
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// IMPL-NEXT:  #undef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// IMPL-EMPTY:
+// IMPL-NEXT:  {
+// IMPL-NEXT:    {llvm::tdl::Directive::TDLD_dira,
+// IMPL-NEXT:      {
+// IMPL-NEXT:        llvm::tdl::allowedClauses_TDLD_dira,
+// IMPL-NEXT:        llvm::tdl::allowedOnceClauses_TDLD_dira,
+// IMPL-NEXT:        llvm::tdl::allowedExclusiveClauses_TDLD_dira,
+// IMPL-NEXT:        llvm::tdl::requiredClauses_TDLD_dira,
+// IMPL-NEXT:      }
+// IMPL-NEXT:    },
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_CLAUSE_PARSER_CLASSES
+// IMPL-NEXT:  #undef GEN_FLANG_CLAUSE_PARSER_CLASSES
+// IMPL-EMPTY:
+// IMPL-NEXT:  EMPTY_CLASS(Clausea);
+// IMPL-NEXT:  WRAPPER_CLASS(Clauseb, std::optional<IntExpr>);
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_CLAUSE_PARSER_CLASSES
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
+// IMPL-NEXT:  #undef GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
+// IMPL-EMPTY:
+// IMPL-NEXT:  Clausea
+// IMPL-NEXT:  , Clauseb
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
+// IMPL-NEXT:  #undef GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
+// IMPL-EMPTY:
+// IMPL-NEXT:  NODE(TdlClause, Clausea)
+// IMPL-NEXT:  NODE(TdlClause, Clauseb)
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_CLAUSE_UNPARSE
+// IMPL-NEXT:  #undef GEN_FLANG_CLAUSE_UNPARSE
+// IMPL-EMPTY:
+// IMPL-NEXT:  void Before(const TdlClause::Clausea &) { Word("CLAUSEA"); }
+// IMPL-NEXT:  void Unparse(const TdlClause::Clauseb &x) {
+// IMPL-NEXT:    Word("CLAUSEB");
+// IMPL-NEXT:    Walk("(", x.v, ")");
+// IMPL-NEXT:  }
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_CLAUSE_UNPARSE
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_CLAUSE_CHECK_ENTER
+// IMPL-NEXT:  #undef GEN_FLANG_CLAUSE_CHECK_ENTER
+// IMPL-EMPTY:
+// IMPL-NEXT:  void Enter(const parser::TdlClause::Clausea &);
+// IMPL-NEXT:  void Enter(const parser::TdlClause::Clauseb &);
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_CLAUSE_CHECK_ENTER
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_CLAUSE_PARSER_KIND_MAP
+// IMPL-NEXT:  #undef GEN_FLANG_CLAUSE_PARSER_KIND_MAP
+// IMPL-EMPTY:
+// IMPL-NEXT:  if constexpr (std::is_same_v<A, parser::TdlClause::Clausea>)
+// IMPL-NEXT:    return llvm::tdl::Clause::TDLC_clausea;
+// IMPL-NEXT:  if constexpr (std::is_same_v<A, parser::TdlClause::Clauseb>)
+// IMPL-NEXT:    return llvm::tdl::Clause::TDLC_clauseb;
+// IMPL-NEXT:  llvm_unreachable("Invalid Tdl Parser clause");
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_CLAUSE_PARSER_KIND_MAP
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_CLANG_CLAUSE_CLASS
+// IMPL-NEXT:  #undef GEN_CLANG_CLAUSE_CLASS
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifndef CLAUSE
+// IMPL-NEXT:  #define CLAUSE(Enum, Str, Implicit)
+// IMPL-NEXT:  #endif
+// IMPL-NEXT:  #ifndef CLAUSE_CLASS
+// IMPL-NEXT:  #define CLAUSE_CLASS(Enum, Str, Class)
+// IMPL-NEXT:  #endif
+// IMPL-NEXT:  #ifndef CLAUSE_NO_CLASS
+// IMPL-NEXT:  #define CLAUSE_NO_CLASS(Enum, Str)
+// IMPL-NEXT:  #endif
+// IMPL-EMPTY:
+// IMPL-NEXT:  #define __CLAUSE(Name, Class)                      \
+// IMPL-NEXT:    CLAUSE(TDLC_##Name, #Name, /* Implicit */ false) \
+// IMPL-NEXT:    CLAUSE_CLASS(TDLC_##Name, #Name, Class)
+// IMPL-NEXT:  #define __CLAUSE_NO_CLASS(Name)                    \
+// IMPL-NEXT:    CLAUSE(TDLC_##Name, #Name, /* Implicit */ false) \
+// IMPL-NEXT:    CLAUSE_NO_CLASS(TDLC_##Name, #Name)
+// IMPL-NEXT:  #define __IMPLICIT_CLAUSE_CLASS(Name, Str, Class)  \
+// IMPL-NEXT:    CLAUSE(TDLC_##Name, Str, /* Implicit */ true)    \
+// IMPL-NEXT:    CLAUSE_CLASS(TDLC_##Name, Str, Class)
+// IMPL-NEXT:  #define __IMPLICIT_CLAUSE_NO_CLASS(Name, Str)      \
+// IMPL-NEXT:    CLAUSE(TDLC_##Name, Str, /* Implicit */ true)    \
+// IMPL-NEXT:    CLAUSE_NO_CLASS(TDLC_##Name, Str)
+// IMPL-EMPTY:
+// IMPL-NEXT:  __CLAUSE_NO_CLASS(clausea)
+// IMPL-NEXT:  __CLAUSE_NO_CLASS(clauseb)
+// IMPL-EMPTY:
+// IMPL-NEXT:  #undef __IMPLICIT_CLAUSE_NO_CLASS
+// IMPL-NEXT:  #undef __IMPLICIT_CLAUSE_CLASS
+// IMPL-NEXT:  #undef __CLAUSE
+// IMPL-NEXT:  #undef CLAUSE_NO_CLASS
+// IMPL-NEXT:  #undef CLAUSE_CLASS
+// IMPL-NEXT:  #undef CLAUSE
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_CLANG_CLAUSE_CLASS
+// IMPL-EMPTY:
+
+// IMPL:       #ifdef GEN_DIRECTIVES_IMPL
+// IMPL-NEXT:  #undef GEN_DIRECTIVES_IMPL
 // IMPL-EMPTY:
 // IMPL-NEXT:  Directive llvm::tdl::getTdlDirectiveKind(llvm::StringRef Str) {
 // IMPL-NEXT:    return llvm::StringSwitch<Directive>(Str)
@@ -177,82 +307,4 @@ def TDL_DirA : Directive<"dira"> {
 // IMPL-NEXT:    llvm_unreachable("Invalid Tdl Directive kind");
 // IMPL-NEXT:  }
 // IMPL-EMPTY:
-
-
-
-// GEN:       #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
-// GEN-NEXT:  #undef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
-// GEN-EMPTY:
-// GEN-NEXT:  namespace llvm {
-// GEN-NEXT:  namespace tdl {
-// GEN-EMPTY:
-// GEN-NEXT:    // Sets for dira
-// GEN-EMPTY:
-// GEN-NEXT:    static  allowedClauses_TDLD_dira {
-// GEN-NEXT:      llvm::tdl::Clause::TDLC_clausea,
-// GEN-NEXT:      llvm::tdl::Clause::TDLC_clauseb,
-// GEN-NEXT:    };
-// GEN-EMPTY:
-// GEN-NEXT:    static  allowedOnceClauses_TDLD_dira {
-// GEN-NEXT:    };
-// GEN-EMPTY:
-// GEN-NEXT:    static  allowedExclusiveClauses_TDLD_dira {
-// GEN-NEXT:    };
-// GEN-EMPTY:
-// GEN-NEXT:    static  requiredClauses_TDLD_dira {
-// GEN-NEXT:    };
-// GEN-NEXT:  } // namespace tdl
-// GEN-NEXT:  } // namespace llvm
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_DIRECTIVE_CLAUSE_SETS
-// GEN-EMPTY:
-// GEN-NEXT:  #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
-// GEN-NEXT:  #undef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
-// GEN-EMPTY:
-// GEN-NEXT:  {
-// GEN-NEXT:    {llvm::tdl::Directive::TDLD_dira,
-// GEN-NEXT:      {
-// GEN-NEXT:        llvm::tdl::allowedClauses_TDLD_dira,
-// GEN-NEXT:        llvm::tdl::allowedOnceClauses_TDLD_dira,
-// GEN-NEXT:        llvm::tdl::allowedExclusiveClauses_TDLD_dira,
-// GEN-NEXT:        llvm::tdl::requiredClauses_TDLD_dira,
-// GEN-NEXT:      }
-// GEN-NEXT:    },
-// GEN-NEXT:  }
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_DIRECTIVE_CLAUSE_MAP
-// GEN-EMPTY:
-// GEN-NEXT:  #ifdef GEN_FLANG_CLAUSE_PARSER_CLASSES
-// GEN-NEXT:  #undef GEN_FLANG_CLAUSE_PARSER_CLASSES
-// GEN-EMPTY:
-// GEN-NEXT:  EMPTY_CLASS(Clausea);
-// GEN-NEXT:  WRAPPER_CLASS(Clauseb, std::optional<IntExpr>);
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_CLAUSE_PARSER_CLASSES
-// GEN-EMPTY:
-// GEN-NEXT:  #ifdef GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
-// GEN-NEXT:  #undef GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
-// GEN-EMPTY:
-// GEN-NEXT:  Clausea
-// GEN-NEXT:  , Clauseb
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
-// GEN-EMPTY:
-// GEN-NEXT:  #ifdef GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
-// GEN-NEXT:  #undef GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
-// GEN-EMPTY:
-// GEN-NEXT:  NODE(TdlClause, Clausea)
-// GEN-NEXT:  NODE(TdlClause, Clauseb)
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
-// GEN-EMPTY:
-// GEN-NEXT:  #ifdef GEN_FLANG_CLAUSE_UNPARSE
-// GEN-NEXT:  #undef GEN_FLANG_CLAUSE_UNPARSE
-// GEN-EMPTY:
-// GEN-NEXT:  void Before(const TdlClause::Clausea &) { Word("CLAUSEA"); }
-// GEN-NEXT:  void Unparse(const TdlClause::Clauseb &x) {
-// GEN-NEXT:    Word("CLAUSEB");
-// GEN-NEXT:    Walk("(", x.v, ")");
-// GEN-NEXT:  }
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_CLAUSE_UNPARSE
+// IMPL-NEXT:  #endif // GEN_DIRECTIVES_IMPL
diff --git a/llvm/test/TableGen/directive2.td b/llvm/test/TableGen/directive2.td
index 1e02cbd01dad041b613654cad274fb6ebb666ecd..154d1e86ffb1d66413e5cbcd1228a992c4649b39 100644
--- a/llvm/test/TableGen/directive2.td
+++ b/llvm/test/TableGen/directive2.td
@@ -1,6 +1,5 @@
 // RUN: llvm-tblgen -gen-directive-decl -I %p/../../include %s | FileCheck -match-full-lines %s
 // RUN: llvm-tblgen -gen-directive-impl -I %p/../../include %s | FileCheck -match-full-lines %s -check-prefix=IMPL
-// RUN: llvm-tblgen -gen-directive-gen -I %p/../../include %s | FileCheck -match-full-lines %s -check-prefix=GEN
 
 include "llvm/Frontend/Directive/DirectiveBase.td"
 
@@ -79,14 +78,143 @@ def TDL_DirA : Directive<"dira"> {
 // CHECK-NEXT:  } // namespace llvm
 // CHECK-NEXT:  #endif // LLVM_Tdl_INC
 
-// IMPL:       #include "tdl.h.inc"
+// IMPL:      #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// IMPL-NEXT: #undef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
 // IMPL-EMPTY:
-// IMPL-NEXT:  #include "llvm/ADT/StringRef.h"
-// IMPL-NEXT:  #include "llvm/ADT/StringSwitch.h"
-// IMPL-NEXT:  #include "llvm/Support/ErrorHandling.h"
+// IMPL-NEXT: namespace llvm {
+// IMPL-NEXT: namespace tdl {
 // IMPL-EMPTY:
-// IMPL-NEXT:  using namespace llvm;
-// IMPL-NEXT:  using namespace tdl;
+// IMPL-NEXT:   // Sets for dira
+// IMPL-EMPTY:
+// IMPL-NEXT:   static  allowedClauses_TDLD_dira {
+// IMPL-NEXT:     llvm::tdl::Clause::TDLC_clausea,
+// IMPL-NEXT:     llvm::tdl::Clause::TDLC_clauseb,
+// IMPL-NEXT:   };
+// IMPL-EMPTY:
+// IMPL-NEXT:   static  allowedOnceClauses_TDLD_dira {
+// IMPL-NEXT:   };
+// IMPL-EMPTY:
+// IMPL-NEXT:   static  allowedExclusiveClauses_TDLD_dira {
+// IMPL-NEXT:   };
+// IMPL-EMPTY:
+// IMPL-NEXT:   static  requiredClauses_TDLD_dira {
+// IMPL-NEXT:   };
+// IMPL-NEXT: } // namespace tdl
+// IMPL-NEXT: } // namespace llvm
+// IMPL-EMPTY:
+// IMPL-NEXT: #endif // GEN_FLANG_DIRECTIVE_CLAUSE_SETS
+// IMPL-EMPTY:
+// IMPL-NEXT: #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// IMPL-NEXT: #undef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// IMPL-EMPTY:
+// IMPL-NEXT: {
+// IMPL-NEXT:   {llvm::tdl::Directive::TDLD_dira,
+// IMPL-NEXT:     {
+// IMPL-NEXT:       llvm::tdl::allowedClauses_TDLD_dira,
+// IMPL-NEXT:       llvm::tdl::allowedOnceClauses_TDLD_dira,
+// IMPL-NEXT:       llvm::tdl::allowedExclusiveClauses_TDLD_dira,
+// IMPL-NEXT:       llvm::tdl::requiredClauses_TDLD_dira,
+// IMPL-NEXT:     }
+// IMPL-NEXT:   },
+// IMPL-NEXT: }
+// IMPL-EMPTY:
+// IMPL-NEXT: #endif // GEN_FLANG_DIRECTIVE_CLAUSE_MAP
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_CLAUSE_PARSER_CLASSES
+// IMPL-NEXT:  #undef GEN_FLANG_CLAUSE_PARSER_CLASSES
+// IMPL-EMPTY:
+// IMPL-NEXT:  EMPTY_CLASS(Clausea);
+// IMPL-NEXT:  WRAPPER_CLASS(Clauseb, std::list<IntExpr>);
+// IMPL-NEXT:  WRAPPER_CLASS(Clausec, std::optional<Name>);
+// IMPL-NEXT:  EMPTY_CLASS(Claused);
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_CLAUSE_PARSER_CLASSES
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
+// IMPL-NEXT:  #undef GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
+// IMPL-EMPTY:
+// IMPL-NEXT:  Clausea
+// IMPL-NEXT:  , Clauseb
+// IMPL-NEXT:  , Clausec
+// IMPL-NEXT:  , Claused
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
+// IMPL-NEXT:  #undef GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
+// IMPL-EMPTY:
+// IMPL-NEXT:  NODE(TdlClause, Clausea)
+// IMPL-NEXT:  NODE(TdlClause, Clauseb)
+// IMPL-NEXT:  NODE(TdlClause, Clausec)
+// IMPL-NEXT:  NODE(TdlClause, Claused)
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifdef GEN_FLANG_CLAUSE_UNPARSE
+// IMPL-NEXT:  #undef GEN_FLANG_CLAUSE_UNPARSE
+// IMPL-EMPTY:
+// IMPL-NEXT:  void Before(const TdlClause::Clausea &) { Word("CLAUSEA"); }
+// IMPL-NEXT:  void Unparse(const TdlClause::Clauseb &x) {
+// IMPL-NEXT:    Word("CLAUSEB");
+// IMPL-NEXT:    Put("(");
+// IMPL-NEXT:    Walk(x.v, ",");
+// IMPL-NEXT:    Put(")");
+// IMPL-NEXT:  }
+// IMPL-NEXT:  void Unparse(const TdlClause::Clausec &x) {
+// IMPL-NEXT:  Word("CLAUSEC");
+// IMPL-NEXT:    Put("(");
+// IMPL-NEXT:    if (x.v.has_value())
+// IMPL-NEXT:    Walk(x.v);
+// IMPL-NEXT:    else
+// IMPL-NEXT:    Put("*");
+// IMPL-NEXT:    Put(")");
+// IMPL-NEXT:  }
+// IMPL-NEXT:  void Before(const TdlClause::Claused &) { Word("CLAUSED"); }
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_FLANG_CLAUSE_UNPARSE
+
+// IMPL:       #ifdef GEN_CLANG_CLAUSE_CLASS
+// IMPL-NEXT:  #undef GEN_CLANG_CLAUSE_CLASS
+// IMPL-EMPTY:
+// IMPL-NEXT:  #ifndef CLAUSE
+// IMPL-NEXT:  #define CLAUSE(Enum, Str, Implicit)
+// IMPL-NEXT:  #endif
+// IMPL-NEXT:  #ifndef CLAUSE_CLASS
+// IMPL-NEXT:  #define CLAUSE_CLASS(Enum, Str, Class)
+// IMPL-NEXT:  #endif
+// IMPL-NEXT:  #ifndef CLAUSE_NO_CLASS
+// IMPL-NEXT:  #define CLAUSE_NO_CLASS(Enum, Str)
+// IMPL-NEXT:  #endif
+// IMPL-EMPTY:
+// IMPL-NEXT:  #define __CLAUSE(Name, Class)                      \
+// IMPL-NEXT:    CLAUSE(TDLC_##Name, #Name, /* Implicit */ false) \
+// IMPL-NEXT:    CLAUSE_CLASS(TDLC_##Name, #Name, Class)
+// IMPL-NEXT:  #define __CLAUSE_NO_CLASS(Name)                    \
+// IMPL-NEXT:    CLAUSE(TDLC_##Name, #Name, /* Implicit */ false) \
+// IMPL-NEXT:    CLAUSE_NO_CLASS(TDLC_##Name, #Name)
+// IMPL-NEXT:  #define __IMPLICIT_CLAUSE_CLASS(Name, Str, Class)  \
+// IMPL-NEXT:    CLAUSE(TDLC_##Name, Str, /* Implicit */ true)    \
+// IMPL-NEXT:  CLAUSE_CLASS(TDLC_##Name, Str, Class)
+// IMPL-NEXT:  #define __IMPLICIT_CLAUSE_NO_CLASS(Name, Str)      \
+// IMPL-NEXT:    CLAUSE(TDLC_##Name, Str, /* Implicit */ true)    \
+// IMPL-NEXT:  CLAUSE_NO_CLASS(TDLC_##Name, Str)
+// IMPL-EMPTY:
+// IMPL-NEXT:  __IMPLICIT_CLAUSE_NO_CLASS(clausea, "clausea")
+// IMPL-NEXT:  __CLAUSE_NO_CLASS(clauseb)
+// IMPL-NEXT:  __CLAUSE(clausec, ClauseC)
+// IMPL-NEXT:  __IMPLICIT_CLAUSE_CLASS(claused, "claused", ClauseD)
+// IMPL-EMPTY:
+// IMPL-NEXT:  #undef __IMPLICIT_CLAUSE_NO_CLASS
+// IMPL-NEXT:  #undef __IMPLICIT_CLAUSE_CLASS
+// IMPL-NEXT:  #undef __CLAUSE
+// IMPL-NEXT:  #undef CLAUSE_NO_CLASS
+// IMPL-NEXT:  #undef CLAUSE_CLASS
+// IMPL-NEXT:  #undef CLAUSE
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_CLANG_CLAUSE_CLASS
+
+// IMPL:       #ifdef GEN_DIRECTIVES_IMPL
+// IMPL-NEXT:  #undef GEN_DIRECTIVES_IMPL
 // IMPL-EMPTY:
 // IMPL-NEXT:  Directive llvm::tdl::getTdlDirectiveKind(llvm::StringRef Str) {
 // IMPL-NEXT:    return llvm::StringSwitch<Directive>(Str)
@@ -142,140 +270,5 @@ def TDL_DirA : Directive<"dira"> {
 // IMPL-NEXT:    }
 // IMPL-NEXT:    llvm_unreachable("Invalid Tdl Directive kind");
 // IMPL-NEXT:  }
-
-
-// GEN:      #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
-// GEN-NEXT: #undef GEN_FLANG_DIRECTIVE_CLAUSE_SETS
-// GEN-EMPTY:
-// GEN-NEXT: namespace llvm {
-// GEN-NEXT: namespace tdl {
-// GEN-EMPTY:
-// GEN-NEXT:   // Sets for dira
-// GEN-EMPTY:
-// GEN-NEXT:   static  allowedClauses_TDLD_dira {
-// GEN-NEXT:     llvm::tdl::Clause::TDLC_clausea,
-// GEN-NEXT:     llvm::tdl::Clause::TDLC_clauseb,
-// GEN-NEXT:   };
-// GEN-EMPTY:
-// GEN-NEXT:   static  allowedOnceClauses_TDLD_dira {
-// GEN-NEXT:   };
-// GEN-EMPTY:
-// GEN-NEXT:   static  allowedExclusiveClauses_TDLD_dira {
-// GEN-NEXT:   };
-// GEN-EMPTY:
-// GEN-NEXT:   static  requiredClauses_TDLD_dira {
-// GEN-NEXT:   };
-// GEN-NEXT: } // namespace tdl
-// GEN-NEXT: } // namespace llvm
-// GEN-EMPTY:
-// GEN-NEXT: #endif // GEN_FLANG_DIRECTIVE_CLAUSE_SETS
-// GEN-EMPTY:
-// GEN-NEXT: #ifdef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
-// GEN-NEXT: #undef GEN_FLANG_DIRECTIVE_CLAUSE_MAP
-// GEN-EMPTY:
-// GEN-NEXT: {
-// GEN-NEXT:   {llvm::tdl::Directive::TDLD_dira,
-// GEN-NEXT:     {
-// GEN-NEXT:       llvm::tdl::allowedClauses_TDLD_dira,
-// GEN-NEXT:       llvm::tdl::allowedOnceClauses_TDLD_dira,
-// GEN-NEXT:       llvm::tdl::allowedExclusiveClauses_TDLD_dira,
-// GEN-NEXT:       llvm::tdl::requiredClauses_TDLD_dira,
-// GEN-NEXT:     }
-// GEN-NEXT:   },
-// GEN-NEXT: }
-// GEN-EMPTY:
-// GEN-NEXT: #endif // GEN_FLANG_DIRECTIVE_CLAUSE_MAP
-// GEN-EMPTY:
-// GEN-NEXT:  #ifdef GEN_FLANG_CLAUSE_PARSER_CLASSES
-// GEN-NEXT:  #undef GEN_FLANG_CLAUSE_PARSER_CLASSES
-// GEN-EMPTY:
-// GEN-NEXT:  EMPTY_CLASS(Clausea);
-// GEN-NEXT:  WRAPPER_CLASS(Clauseb, std::list<IntExpr>);
-// GEN-NEXT:  WRAPPER_CLASS(Clausec, std::optional<Name>);
-// GEN-NEXT:  EMPTY_CLASS(Claused);
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_CLAUSE_PARSER_CLASSES
-// GEN-EMPTY:
-// GEN-NEXT:  #ifdef GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
-// GEN-NEXT:  #undef GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
-// GEN-EMPTY:
-// GEN-NEXT:  Clausea
-// GEN-NEXT:  , Clauseb
-// GEN-NEXT:  , Clausec
-// GEN-NEXT:  , Claused
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST
-// GEN-EMPTY:
-// GEN-NEXT:  #ifdef GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
-// GEN-NEXT:  #undef GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
-// GEN-EMPTY:
-// GEN-NEXT:  NODE(TdlClause, Clausea)
-// GEN-NEXT:  NODE(TdlClause, Clauseb)
-// GEN-NEXT:  NODE(TdlClause, Clausec)
-// GEN-NEXT:  NODE(TdlClause, Claused)
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
-// GEN-EMPTY:
-// GEN-NEXT:  #ifdef GEN_FLANG_CLAUSE_UNPARSE
-// GEN-NEXT:  #undef GEN_FLANG_CLAUSE_UNPARSE
-// GEN-EMPTY:
-// GEN-NEXT:  void Before(const TdlClause::Clausea &) { Word("CLAUSEA"); }
-// GEN-NEXT:  void Unparse(const TdlClause::Clauseb &x) {
-// GEN-NEXT:    Word("CLAUSEB");
-// GEN-NEXT:    Put("(");
-// GEN-NEXT:    Walk(x.v, ",");
-// GEN-NEXT:    Put(")");
-// GEN-NEXT:  }
-// GEN-NEXT:  void Unparse(const TdlClause::Clausec &x) {
-// GEN-NEXT:  Word("CLAUSEC");
-// GEN-NEXT:    Put("(");
-// GEN-NEXT:    if (x.v.has_value())
-// GEN-NEXT:    Walk(x.v);
-// GEN-NEXT:    else
-// GEN-NEXT:    Put("*");
-// GEN-NEXT:    Put(")");
-// GEN-NEXT:  }
-// GEN-NEXT:  void Before(const TdlClause::Claused &) { Word("CLAUSED"); }
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_FLANG_CLAUSE_UNPARSE
-
-// GEN:       #ifdef GEN_CLANG_CLAUSE_CLASS
-// GEN-NEXT:  #undef GEN_CLANG_CLAUSE_CLASS
-// GEN-EMPTY:
-// GEN-NEXT:  #ifndef CLAUSE
-// GEN-NEXT:  #define CLAUSE(Enum, Str, Implicit)
-// GEN-NEXT:  #endif
-// GEN-NEXT:  #ifndef CLAUSE_CLASS
-// GEN-NEXT:  #define CLAUSE_CLASS(Enum, Str, Class)
-// GEN-NEXT:  #endif
-// GEN-NEXT:  #ifndef CLAUSE_NO_CLASS
-// GEN-NEXT:  #define CLAUSE_NO_CLASS(Enum, Str)
-// GEN-NEXT:  #endif
-// GEN-EMPTY:
-// GEN-NEXT:  #define __CLAUSE(Name, Class)                      \
-// GEN-NEXT:    CLAUSE(TDLC_##Name, #Name, /* Implicit */ false) \
-// GEN-NEXT:    CLAUSE_CLASS(TDLC_##Name, #Name, Class)
-// GEN-NEXT:  #define __CLAUSE_NO_CLASS(Name)                    \
-// GEN-NEXT:    CLAUSE(TDLC_##Name, #Name, /* Implicit */ false) \
-// GEN-NEXT:    CLAUSE_NO_CLASS(TDLC_##Name, #Name)
-// GEN-NEXT:  #define __IMPLICIT_CLAUSE_CLASS(Name, Str, Class)  \
-// GEN-NEXT:    CLAUSE(TDLC_##Name, Str, /* Implicit */ true)    \
-// GEN-NEXT:  CLAUSE_CLASS(TDLC_##Name, Str, Class)
-// GEN-NEXT:  #define __IMPLICIT_CLAUSE_NO_CLASS(Name, Str)      \
-// GEN-NEXT:    CLAUSE(TDLC_##Name, Str, /* Implicit */ true)    \
-// GEN-NEXT:  CLAUSE_NO_CLASS(TDLC_##Name, Str)
-// GEN-EMPTY:
-// GEN-NEXT:  __IMPLICIT_CLAUSE_NO_CLASS(clausea, "clausea")
-// GEN-NEXT:  __CLAUSE_NO_CLASS(clauseb)
-// GEN-NEXT:  __CLAUSE(clausec, ClauseC)
-// GEN-NEXT:  __IMPLICIT_CLAUSE_CLASS(claused, "claused", ClauseD)
-// GEN-EMPTY:
-// GEN-NEXT:  #undef __IMPLICIT_CLAUSE_NO_CLASS
-// GEN-NEXT:  #undef __IMPLICIT_CLAUSE_CLASS
-// GEN-NEXT:  #undef __CLAUSE
-// GEN-NEXT:  #undef CLAUSE_NO_CLASS
-// GEN-NEXT:  #undef CLAUSE_CLASS
-// GEN-NEXT:  #undef CLAUSE
-// GEN-EMPTY:
-// GEN-NEXT:  #endif // GEN_CLANG_CLAUSE_CLASS
-
+// IMPL-EMPTY:
+// IMPL-NEXT:  #endif // GEN_DIRECTIVES_IMPL
diff --git a/llvm/test/TableGen/directive3.td b/llvm/test/TableGen/directive3.td
index 8af4594f299414ebfb7c2b97bb3445d25fc3488a..2a82b5ebcb493a4380e37d2812b2c7fac389f460 100644
--- a/llvm/test/TableGen/directive3.td
+++ b/llvm/test/TableGen/directive3.td
@@ -1,6 +1,5 @@
 // RUN: not llvm-tblgen -gen-directive-decl -I %p/../../include %s 2>&1 | FileCheck -match-full-lines %s
 // RUN: not llvm-tblgen -gen-directive-impl -I %p/../../include %s 2>&1 | FileCheck -match-full-lines %s
-// RUN: not llvm-tblgen -gen-directive-gen -I %p/../../include %s 2>&1 | FileCheck -match-full-lines %s
 
 include "llvm/Frontend/Directive/DirectiveBase.td"
 
diff --git a/llvm/test/TableGen/pseudo-inst-expansion.td b/llvm/test/TableGen/pseudo-inst-expansion.td
new file mode 100644
index 0000000000000000000000000000000000000000..538d754eaf6232f6e00587010d8760e460bfc308
--- /dev/null
+++ b/llvm/test/TableGen/pseudo-inst-expansion.td
@@ -0,0 +1,37 @@
+// RUN: llvm-tblgen -gen-pseudo-lowering -I %p/../../include %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+def TestTargetInstrInfo : InstrInfo;
+
+def TestTarget : Target {
+  let InstructionSet = TestTargetInstrInfo;
+}
+
+def REG : Register<"REG">;
+def GPR : RegisterClass<"TestTarget", [i32], 32, (add REG)>;
+
+class SysReg<bits<12> op> {
+  bits<12> Encoding = op;
+}
+def SR : SysReg<0b111100001111>;
+
+class Pseudo<dag outs, dag ins, list<dag> pattern>
+    : Instruction {
+  dag OutOperandList = outs;
+  dag InOperandList = ins;
+  let Pattern = pattern;
+  let isPseudo = 1;
+}
+
+def INSTR : Instruction {
+  let OutOperandList = (outs GPR:$rd);
+  let InOperandList = (ins i32imm:$val);
+  let Pattern = [];
+}
+
+def PSEUDO : Pseudo<(outs GPR:$rd), (ins),
+                    [(set GPR:$rd, (i32 SR.Encoding))]>,
+             PseudoInstExpansion<(INSTR GPR:$rd, SR.Encoding)>;
+
+// CHECK: .addOperand(MCOperand::createImm(3855));
diff --git a/llvm/test/TableGen/self-reference-typeerror.td b/llvm/test/TableGen/self-reference-typeerror.td
index 35c6131fa2c983023da2d72514a15e5d08036af3..6f8da4dae13596a7a6fadd97b0ef48063de228aa 100644
--- a/llvm/test/TableGen/self-reference-typeerror.td
+++ b/llvm/test/TableGen/self-reference-typeerror.td
@@ -1,13 +1,14 @@
 // RUN: not llvm-tblgen %s 2>&1 | FileCheck %s
 // XFAIL: vg_leak
 
-class A<A x> {
-  A a = x;
+class Cl<Cl rec> {
+  Cl Arec = rec;
 }
 
 // At the time A0 is referenced, A has not yet been established as a superclass.
 // This kind of self-reference is discourage, but if you *really* want it, you
 // can force it with !cast.
 //
-// CHECK: Field 'A:x' of type 'A' is incompatible with value
-def A0 : A<A0>;
+// CHECK: alue specified for template argument 'Cl:rec'
+
+def Rec0 : Cl<Rec0>;
diff --git a/llvm/test/TableGen/template-args.td b/llvm/test/TableGen/template-args.td
new file mode 100644
index 0000000000000000000000000000000000000000..2a931adffe9ad5d30595748cbb8c67f8a071c665
--- /dev/null
+++ b/llvm/test/TableGen/template-args.td
@@ -0,0 +1,142 @@
+// RUN: llvm-tblgen %s | FileCheck %s
+// RUN: not llvm-tblgen -DERROR1 %s 2>&1 | FileCheck --check-prefix=ERROR1 %s
+// RUN: not llvm-tblgen -DERROR2 %s 2>&1 | FileCheck --check-prefix=ERROR2 %s
+// RUN: not llvm-tblgen -DERROR3 %s 2>&1 | FileCheck --check-prefix=ERROR3 %s
+// RUN: not llvm-tblgen -DERROR4 %s 2>&1 | FileCheck --check-prefix=ERROR4 %s
+// RUN: not llvm-tblgen -DERROR5 %s 2>&1 | FileCheck --check-prefix=ERROR5 %s
+// RUN: not llvm-tblgen -DERROR6 %s 2>&1 | FileCheck --check-prefix=ERROR6 %s
+
+// This file tests that template arguments are type-checked and cast
+// if necessary.
+
+// Class template arguments.
+
+class Class1<string nm> {
+  string Name = nm;
+}
+
+// CHECK: def Rec1
+// CHECK:   string Name = "Alice"
+// CHECK:   string NameName = "AliceAlice"
+
+def Rec1 : Class1<"Alice"> {
+  string NameName = Name # Name;
+}
+
+#ifdef ERROR1
+// ERROR1: Value specified for template argument 'Class1:nm' (#0) is of type int
+
+def Rec2 : Class1<42> {
+}
+#endif
+
+class Class2<bits<8> cd> {
+  int Code = cd;
+}
+
+// CHECK: def Rec3
+// CHECK:   int Code = 42
+// CHECK:   list<int> CodeList = [42]
+
+def Rec3 : Class2<0b00101010> {
+  list<int> CodeList = [Code];
+}
+
+// CHECK: def Rec4
+// CHECK:   int Code = 42
+// CHECK:   list<int> CodeList = [42]
+
+def Rec4 : Class2<42> {
+  list<int> CodeList = [Code];
+}
+
+#ifdef ERROR2
+// ERROR2: Value specified for template argument 'Class2:cd' (#0) is of type string
+
+def Rec5 : Class2<"oops"> {
+  list<int> CodeList = [Code];
+}
+#endif
+
+// Anonymous class instantiation template arguments.
+
+// CHECK: def Rec6
+// CHECK:   string Name = "Ted"
+
+def Rec6 {
+  string Name = Class1<"Ted">.Name;
+}
+
+#ifdef ERROR3
+// ERROR3: Value specified for template argument 'Class1:nm' (#0) is of type int
+
+def Rec7 {
+  string Name = Class1<42>.Name;
+}
+#endif
+
+// CHECK: def Rec8
+// CHECK:   list<int> CodeList = [42]
+
+def Rec8 {
+  list<int> CodeList = [Class2<42>.Code];
+}
+
+#ifdef ERROR4
+// ERROR4: Value specified for template argument 'Class2:cd' (#0) is of type string
+
+def Rec9 {
+  list<int> CodeList = [Class2<"huh?">.Code];
+}
+#endif
+
+// Multiclass template arguments.
+
+multiclass MC1<string nm> {
+  def _1 {
+    string Name = nm;
+  }
+  def _2 {
+    string NameNmae = nm # nm;
+  }
+}
+
+// CHECK: def RecMC1_1
+// CHECK:   string Name = "Carol"
+// CHECK: def RecMC1_2
+// CHECK:   string NameNmae = "CarolCarol"
+
+defm RecMC1 : MC1<"Carol">;
+
+#ifdef ERROR5
+// ERROR5: Value specified for template argument 'MC1::nm' (#0) is of type int
+
+defm RecMC2 : MC1<42>;
+#endif
+
+multiclass MC2<bits<8> cd> {
+  def _1 {
+    bits<8> Code = cd;
+  }
+  def _2 {
+    int Code = cd;
+  }
+  def _3 {
+    list<int> CodeList = [cd];
+  }
+}
+
+// CHECK: def RecMC3_1
+// CHECK:   bits<8> Code = { 0, 0, 1, 0, 1, 0, 1, 0 }
+// CHECK: def RecMC3_2
+// CHECK:   int Code = 42
+// CHECK: def RecMC3_3
+// CHECK:   list<int> CodeList = [42]
+
+defm RecMC3 : MC2<42>;
+
+#ifdef ERROR6
+// ERROR6: Value specified for template argument 'MC2::cd' (#0) is of type string
+
+defm RecMC4 : MC2<"Bob">;
+#endif
diff --git a/llvm/test/Transforms/ArgumentPromotion/X86/thiscall.ll b/llvm/test/Transforms/ArgumentPromotion/X86/thiscall.ll
index d9f3681ba4ab229caca697cf0fcbde10e508c061..0643397be09929393b0b1280c274ae4c1986e8e0 100644
--- a/llvm/test/Transforms/ArgumentPromotion/X86/thiscall.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/X86/thiscall.ll
@@ -12,25 +12,25 @@ target triple = "i386-pc-windows-msvc19.11.0"
 
 %struct.a = type { i8 }
 
-define internal x86_thiscallcc void @internalfun(%struct.a* %this, <{ %struct.a }>* inalloca) {
+define internal x86_thiscallcc void @internalfun(%struct.a* %this, <{ %struct.a }>* inalloca(<{ %struct.a }>)) {
 ; ARGPROMOTION-LABEL: define {{[^@]+}}@internalfun
-; ARGPROMOTION-SAME: (%struct.a* [[THIS:%.*]], <{ [[STRUCT_A:%.*]] }>* inalloca [[TMP0:%.*]])
+; ARGPROMOTION-SAME: (%struct.a* [[THIS:%.*]], <{ [[STRUCT_A:%.*]] }>* inalloca(<{ [[STRUCT_A]] }>) [[TMP0:%.*]]) {
 ; ARGPROMOTION-NEXT:  entry:
 ; ARGPROMOTION-NEXT:    [[A:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[TMP0]], i32 0, i32 0
 ; ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A]] }>, align 4
 ; ARGPROMOTION-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[ARGMEM]], i32 0, i32 0
 ; ARGPROMOTION-NEXT:    [[CALL:%.*]] = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* [[TMP1]], %struct.a* dereferenceable(1) [[A]])
-; ARGPROMOTION-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* inalloca [[ARGMEM]])
+; ARGPROMOTION-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* inalloca(<{ [[STRUCT_A]] }>) [[ARGMEM]])
 ; ARGPROMOTION-NEXT:    ret void
 ;
 ; GLOBALOPT_ARGPROMOTION-LABEL: define {{[^@]+}}@internalfun
-; GLOBALOPT_ARGPROMOTION-SAME: (<{ [[STRUCT_A:%.*]] }>* [[TMP0:%.*]]) unnamed_addr
+; GLOBALOPT_ARGPROMOTION-SAME: (<{ [[STRUCT_A:%.*]] }>* [[TMP0:%.*]]) unnamed_addr {
 ; GLOBALOPT_ARGPROMOTION-NEXT:  entry:
 ; GLOBALOPT_ARGPROMOTION-NEXT:    [[A:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[TMP0]], i32 0, i32 0
 ; GLOBALOPT_ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A]] }>, align 4
 ; GLOBALOPT_ARGPROMOTION-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[ARGMEM]], i32 0, i32 0
 ; GLOBALOPT_ARGPROMOTION-NEXT:    [[CALL:%.*]] = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* [[TMP1]], %struct.a* dereferenceable(1) [[A]])
-; GLOBALOPT_ARGPROMOTION-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* inalloca [[ARGMEM]])
+; GLOBALOPT_ARGPROMOTION-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* inalloca(<{ [[STRUCT_A]] }>) [[ARGMEM]])
 ; GLOBALOPT_ARGPROMOTION-NEXT:    ret void
 ;
 entry:
@@ -38,22 +38,22 @@ entry:
   %argmem = alloca inalloca <{ %struct.a }>, align 4
   %1 = getelementptr inbounds <{ %struct.a }>, <{ %struct.a }>* %argmem, i32 0, i32 0
   %call = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* %1, %struct.a* dereferenceable(1) %a)
-  call void @ext(<{ %struct.a }>* inalloca %argmem)
+  call void @ext(<{ %struct.a }>* inalloca(<{ %struct.a }>) %argmem)
   ret void
 }
 
 ; This is here to ensure @internalfun is live.
 define void @exportedfun(%struct.a* %a) {
 ; ARGPROMOTION-LABEL: define {{[^@]+}}@exportedfun
-; ARGPROMOTION-SAME: (%struct.a* [[A:%.*]])
+; ARGPROMOTION-SAME: (%struct.a* [[A:%.*]]) {
 ; ARGPROMOTION-NEXT:    [[INALLOCA_SAVE:%.*]] = tail call i8* @llvm.stacksave()
 ; ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A:%.*]] }>, align 4
-; ARGPROMOTION-NEXT:    call x86_thiscallcc void @internalfun(%struct.a* [[A]], <{ [[STRUCT_A]] }>* inalloca [[ARGMEM]])
+; ARGPROMOTION-NEXT:    call x86_thiscallcc void @internalfun(%struct.a* [[A]], <{ [[STRUCT_A]] }>* inalloca(<{ [[STRUCT_A]] }>) [[ARGMEM]])
 ; ARGPROMOTION-NEXT:    call void @llvm.stackrestore(i8* [[INALLOCA_SAVE]])
 ; ARGPROMOTION-NEXT:    ret void
 ;
 ; GLOBALOPT_ARGPROMOTION-LABEL: define {{[^@]+}}@exportedfun
-; GLOBALOPT_ARGPROMOTION-SAME: (%struct.a* [[A:%.*]]) local_unnamed_addr
+; GLOBALOPT_ARGPROMOTION-SAME: (%struct.a* [[A:%.*]]) local_unnamed_addr {
 ; GLOBALOPT_ARGPROMOTION-NEXT:    [[INALLOCA_SAVE:%.*]] = tail call i8* @llvm.stacksave()
 ; GLOBALOPT_ARGPROMOTION-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A:%.*]] }>, align 4
 ; GLOBALOPT_ARGPROMOTION-NEXT:    call fastcc void @internalfun(<{ [[STRUCT_A]] }>* [[ARGMEM]])
@@ -62,12 +62,12 @@ define void @exportedfun(%struct.a* %a) {
 ;
   %inalloca.save = tail call i8* @llvm.stacksave()
   %argmem = alloca inalloca <{ %struct.a }>, align 4
-  call x86_thiscallcc void @internalfun(%struct.a* %a, <{ %struct.a }>* inalloca %argmem)
+  call x86_thiscallcc void @internalfun(%struct.a* %a, <{ %struct.a }>* inalloca(<{ %struct.a }>) %argmem)
   call void @llvm.stackrestore(i8* %inalloca.save)
   ret void
 }
 
 declare x86_thiscallcc %struct.a* @copy_ctor(%struct.a* returned, %struct.a* dereferenceable(1))
-declare void @ext(<{ %struct.a }>* inalloca)
+declare void @ext(<{ %struct.a }>* inalloca(<{ %struct.a }>))
 declare i8* @llvm.stacksave()
 declare void @llvm.stackrestore(i8*)
diff --git a/llvm/test/Transforms/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/ArgumentPromotion/inalloca.ll
index ebf3d18f2fc5cd0efd8ed9d561df756e3f138142..7eaa7499af6b40f46813808277ada38696f443c7 100644
--- a/llvm/test/Transforms/ArgumentPromotion/inalloca.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/inalloca.ll
@@ -7,9 +7,9 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 %struct.ss = type { i32, i32 }
 
 ; Argpromote + sroa should change this to passing the two integers by value.
-define internal i32 @f(%struct.ss* inalloca  %s) {
+define internal i32 @f(%struct.ss* inalloca(%struct.ss)  %s) {
 ; CHECK-LABEL: define {{[^@]+}}@f
-; CHECK-SAME: (i32 [[S_0_0_VAL:%.*]], i32 [[S_0_1_VAL:%.*]]) unnamed_addr
+; CHECK-SAME: (i32 [[S_0_0_VAL:%.*]], i32 [[S_0_1_VAL:%.*]]) unnamed_addr {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[R:%.*]] = add i32 [[S_0_0_VAL]], [[S_0_1_VAL]]
 ; CHECK-NEXT:    ret i32 [[R]]
@@ -24,7 +24,7 @@ entry:
 }
 
 define i32 @main() {
-; CHECK-LABEL: define {{[^@]+}}@main() local_unnamed_addr
+; CHECK-LABEL: define {{[^@]+}}@main() local_unnamed_addr {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[R:%.*]] = call fastcc i32 @f(i32 1, i32 2)
 ; CHECK-NEXT:    ret i32 [[R]]
@@ -35,14 +35,14 @@ entry:
   %f1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
   store i32 1, i32* %f0, align 4
   store i32 2, i32* %f1, align 4
-  %r = call i32 @f(%struct.ss* inalloca %S)
+  %r = call i32 @f(%struct.ss* inalloca(%struct.ss) %S)
   ret i32 %r
 }
 
 ; Argpromote can't promote %a because of the icmp use.
-define internal i1 @g(%struct.ss* %a, %struct.ss* inalloca %b) nounwind  {
+define internal i1 @g(%struct.ss* %a, %struct.ss* inalloca(%struct.ss) %b) nounwind  {
 ; CHECK-LABEL: define {{[^@]+}}@g
-; CHECK-SAME: (%struct.ss* [[A:%.*]], %struct.ss* [[B:%.*]]) unnamed_addr
+; CHECK-SAME: (%struct.ss* [[A:%.*]], %struct.ss* [[B:%.*]]) unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq %struct.ss* [[A]], [[B]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -53,14 +53,14 @@ entry:
 }
 
 define i32 @test() {
-; CHECK-LABEL: define {{[^@]+}}@test() local_unnamed_addr
+; CHECK-LABEL: define {{[^@]+}}@test() local_unnamed_addr {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[S:%.*]] = alloca inalloca [[STRUCT_SS:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = alloca inalloca [[STRUCT_SS:%.*]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = call fastcc i1 @g(%struct.ss* [[S]], %struct.ss* [[S]])
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
   %S = alloca inalloca %struct.ss
-  %c = call i1 @g(%struct.ss* %S, %struct.ss* inalloca %S)
+  %c = call i1 @g(%struct.ss* %S, %struct.ss* inalloca(%struct.ss) %S)
   ret i32 0
 }
diff --git a/llvm/test/Transforms/ArgumentPromotion/metadata.ll b/llvm/test/Transforms/ArgumentPromotion/metadata.ll
deleted file mode 100644
index c00bda735116653c4a5e5917c4fb2833d4fff584..0000000000000000000000000000000000000000
--- a/llvm/test/Transforms/ArgumentPromotion/metadata.ll
+++ /dev/null
@@ -1,70 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
-; RUN: opt < %s -argpromotion -S | FileCheck %s
-
-define i32 @should_copy_range(i32* %x) {
-; CHECK-LABEL: define {{[^@]+}}@should_copy_range
-; CHECK-SAME: (i32* [[X:%.*]]) {
-; CHECK-NEXT:    [[X_VAL:%.*]] = load i32, i32* [[X]], align 4, !range !0
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @f_load_range(i32 [[X_VAL]])
-; CHECK-NEXT:    ret i32 [[TMP1]]
-;
-  %1 = call i32 @f_load_range(i32* %x)
-  ret i32 %1
-}
-
-define internal i32 @f_load_range(i32* %v) {
-; CHECK-LABEL: define {{[^@]+}}@f_load_range
-; CHECK-SAME: (i32 [[V_VAL:%.*]]) {
-; CHECK-NEXT:    ret i32 [[V_VAL]]
-;
-  %1 = load i32, i32* %v, align 4, !range !0
-  ret i32 %1
-}
-
-define i32* @should_copy_nonnull(i32** %x) {
-; CHECK-LABEL: define {{[^@]+}}@should_copy_nonnull
-; CHECK-SAME: (i32** [[X:%.*]]) {
-; CHECK-NEXT:    [[X_VAL:%.*]] = load i32*, i32** [[X]], align 4, !nonnull !1
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32* @f_load_nonnull(i32* [[X_VAL]])
-; CHECK-NEXT:    ret i32* [[TMP1]]
-;
-  %1 = call i32* @f_load_nonnull(i32** %x)
-  ret i32* %1
-}
-
-define internal i32* @f_load_nonnull(i32** %v) {
-; CHECK-LABEL: define {{[^@]+}}@f_load_nonnull
-; CHECK-SAME: (i32* [[V_VAL:%.*]]) {
-; CHECK-NEXT:    ret i32* [[V_VAL]]
-;
-  %1 = load i32*, i32** %v, align 4, !nonnull !1
-  ret i32* %1
-}
-
-define i32* @should_copy_dereferenceable(i32** %x) {
-; CHECK-LABEL: define {{[^@]+}}@should_copy_dereferenceable
-; CHECK-SAME: (i32** [[X:%.*]]) {
-; CHECK-NEXT:    [[X_VAL:%.*]] = load i32*, i32** [[X]], align 4, !dereferenceable !2
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32* @f_load_dereferenceable(i32* [[X_VAL]])
-; CHECK-NEXT:    ret i32* [[TMP1]]
-;
-  %1 = call i32* @f_load_dereferenceable(i32** %x)
-  ret i32* %1
-}
-
-define internal i32* @f_load_dereferenceable(i32** %v) {
-; CHECK-LABEL: define {{[^@]+}}@f_load_dereferenceable
-; CHECK-SAME: (i32* [[V_VAL:%.*]]) {
-; CHECK-NEXT:    ret i32* [[V_VAL]]
-;
-  %1 = load i32*, i32** %v, align 4, !dereferenceable !2
-  ret i32* %1
-}
-
-; CHECK:      !0 = !{i32 0, i32 4}
-; CHECK-NEXT: !1 = !{}
-; CHECK-NEXT: !2 = !{i64 42}
-;
-!0 = !{i32 0, i32 4}
-!1 = !{}
-!2 = !{i64 42}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
index 9937e6cf20729cbbce92e8ceab5840b5c30c3f91..ad452d68acbdcc0f59bba67c731336ba51ea98f0 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/thiscall.ll
@@ -14,15 +14,15 @@ target triple = "i386-pc-windows-msvc19.11.0"
 
 %struct.a = type { i8 }
 
-define internal x86_thiscallcc void @internalfun(%struct.a* %this, <{ %struct.a }>* inalloca) {
+define internal x86_thiscallcc void @internalfun(%struct.a* %this, <{ %struct.a }>* inalloca(<{ %struct.a }>)) {
 ; CHECK-LABEL: define {{[^@]+}}@internalfun
-; CHECK-SAME: (%struct.a* noalias nocapture nofree readnone [[THIS:%.*]], <{ [[STRUCT_A:%.*]] }>* inalloca noundef nonnull align 4 dereferenceable(1) [[TMP0:%.*]]) {
+; CHECK-SAME: (%struct.a* noalias nocapture nofree readnone [[THIS:%.*]], <{ [[STRUCT_A:%.*]] }>* noundef nonnull inalloca(<{ [[STRUCT_A]] }>) align 4 dereferenceable(1) [[TMP0:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[TMP0]], i32 0, i32 0
 ; CHECK-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A]] }>, align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <{ [[STRUCT_A]] }>, <{ [[STRUCT_A]] }>* [[ARGMEM]], i32 0, i32 0
 ; CHECK-NEXT:    [[CALL:%.*]] = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* noundef nonnull align 4 dereferenceable(1) [[TMP1]], %struct.a* noundef nonnull align 4 dereferenceable(1) [[A]])
-; CHECK-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* inalloca noundef nonnull align 4 dereferenceable(1) [[ARGMEM]])
+; CHECK-NEXT:    call void @ext(<{ [[STRUCT_A]] }>* noundef nonnull inalloca(<{ [[STRUCT_A]] }>) align 4 dereferenceable(1) [[ARGMEM]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -30,7 +30,7 @@ entry:
   %argmem = alloca inalloca <{ %struct.a }>, align 4
   %1 = getelementptr inbounds <{ %struct.a }>, <{ %struct.a }>* %argmem, i32 0, i32 0
   %call = call x86_thiscallcc %struct.a* @copy_ctor(%struct.a* %1, %struct.a* dereferenceable(1) %a)
-  call void @ext(<{ %struct.a }>* inalloca %argmem)
+  call void @ext(<{ %struct.a }>* inalloca(<{ %struct.a }>) %argmem)
   ret void
 }
 
@@ -40,19 +40,19 @@ define void @exportedfun(%struct.a* %a) {
 ; CHECK-SAME: (%struct.a* nocapture nofree readnone [[A:%.*]]) {
 ; CHECK-NEXT:    [[INALLOCA_SAVE:%.*]] = tail call i8* @llvm.stacksave() #[[ATTR1:[0-9]+]]
 ; CHECK-NEXT:    [[ARGMEM:%.*]] = alloca inalloca <{ [[STRUCT_A:%.*]] }>, align 4
-; CHECK-NEXT:    call x86_thiscallcc void @internalfun(%struct.a* noalias nocapture nofree readnone undef, <{ [[STRUCT_A]] }>* inalloca noundef nonnull align 4 dereferenceable(1) [[ARGMEM]])
+; CHECK-NEXT:    call x86_thiscallcc void @internalfun(%struct.a* noalias nocapture nofree readnone undef, <{ [[STRUCT_A]] }>* noundef nonnull inalloca(<{ [[STRUCT_A]] }>) align 4 dereferenceable(1) [[ARGMEM]])
 ; CHECK-NEXT:    call void @llvm.stackrestore(i8* nofree [[INALLOCA_SAVE]])
 ; CHECK-NEXT:    ret void
 ;
   %inalloca.save = tail call i8* @llvm.stacksave()
   %argmem = alloca inalloca <{ %struct.a }>, align 4
-  call x86_thiscallcc void @internalfun(%struct.a* %a, <{ %struct.a }>* inalloca %argmem)
+  call x86_thiscallcc void @internalfun(%struct.a* %a, <{ %struct.a }>* inalloca(<{ %struct.a }>) %argmem)
   call void @llvm.stackrestore(i8* %inalloca.save)
   ret void
 }
 
 declare x86_thiscallcc %struct.a* @copy_ctor(%struct.a* returned, %struct.a* dereferenceable(1))
-declare void @ext(<{ %struct.a }>* inalloca)
+declare void @ext(<{ %struct.a }>* inalloca(<{ %struct.a }>))
 declare i8* @llvm.stacksave()
 declare void @llvm.stackrestore(i8*)
 ;.
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
index 30575fe8bde93c6d9b823b9308e0e9bb79de1dad..0b82eac6b982ae72eed919a9ee6430bbcc01285b 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
@@ -9,12 +9,12 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 %struct.ss = type { i32, i32 }
 
 ; Argpromote + sroa should change this to passing the two integers by value.
-define internal i32 @f(%struct.ss* inalloca  %s) {
+define internal i32 @f(%struct.ss* inalloca(%struct.ss) %s) {
 ; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@f
-; IS__TUNIT____-SAME: (%struct.ss* inalloca noalias nocapture nofree noundef nonnull align 4 dereferenceable(8) [[S:%.*]]) #[[ATTR0:[0-9]+]] {
+; IS__TUNIT____-SAME: (%struct.ss* noalias nocapture nofree noundef nonnull inalloca([[STRUCT_SS:%.*]]) align 4 dereferenceable(8) [[S:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IS__TUNIT____-NEXT:  entry:
-; IS__TUNIT____-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[S]], i32 0, i32 0
+; IS__TUNIT____-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
 ; IS__TUNIT____-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
 ; IS__TUNIT____-NEXT:    [[A:%.*]] = load i32, i32* [[F0]], align 4
 ; IS__TUNIT____-NEXT:    [[B:%.*]] = load i32, i32* [[F1]], align 4
@@ -23,9 +23,9 @@ define internal i32 @f(%struct.ss* inalloca  %s) {
 ;
 ; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@f
-; IS__CGSCC____-SAME: (%struct.ss* inalloca noalias nocapture nofree noundef nonnull align 4 dereferenceable(8) [[S:%.*]]) #[[ATTR0:[0-9]+]] {
+; IS__CGSCC____-SAME: (%struct.ss* noalias nocapture nofree noundef nonnull inalloca([[STRUCT_SS:%.*]]) align 4 dereferenceable(8) [[S:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IS__CGSCC____-NEXT:  entry:
-; IS__CGSCC____-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[S]], i32 0, i32 0
+; IS__CGSCC____-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
 ; IS__CGSCC____-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
 ; IS__CGSCC____-NEXT:    [[A:%.*]] = load i32, i32* [[F0]], align 4
 ; IS__CGSCC____-NEXT:    [[B:%.*]] = load i32, i32* [[F1]], align 4
@@ -51,7 +51,7 @@ define i32 @main() {
 ; IS__TUNIT____-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
 ; IS__TUNIT____-NEXT:    store i32 1, i32* [[F0]], align 4
 ; IS__TUNIT____-NEXT:    store i32 2, i32* [[F1]], align 4
-; IS__TUNIT____-NEXT:    [[R:%.*]] = call i32 @f(%struct.ss* inalloca noalias nocapture nofree noundef nonnull align 4 dereferenceable(8) [[S]]) #[[ATTR2:[0-9]+]]
+; IS__TUNIT____-NEXT:    [[R:%.*]] = call i32 @f(%struct.ss* noalias nocapture nofree noundef nonnull inalloca([[STRUCT_SS]]) align 4 dereferenceable(8) [[S]]) #[[ATTR2:[0-9]+]]
 ; IS__TUNIT____-NEXT:    ret i32 [[R]]
 ;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
@@ -63,7 +63,7 @@ define i32 @main() {
 ; IS__CGSCC____-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
 ; IS__CGSCC____-NEXT:    store i32 1, i32* [[F0]], align 4
 ; IS__CGSCC____-NEXT:    store i32 2, i32* [[F1]], align 4
-; IS__CGSCC____-NEXT:    [[R:%.*]] = call i32 @f(%struct.ss* inalloca noalias nocapture nofree noundef nonnull align 4 dereferenceable(8) [[S]]) #[[ATTR2:[0-9]+]]
+; IS__CGSCC____-NEXT:    [[R:%.*]] = call i32 @f(%struct.ss* noalias nocapture nofree noundef nonnull inalloca([[STRUCT_SS]]) align 4 dereferenceable(8) [[S]]) #[[ATTR2:[0-9]+]]
 ; IS__CGSCC____-NEXT:    ret i32 [[R]]
 ;
 entry:
@@ -72,15 +72,15 @@ entry:
   %f1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
   store i32 1, i32* %f0, align 4
   store i32 2, i32* %f1, align 4
-  %r = call i32 @f(%struct.ss* inalloca %S)
+  %r = call i32 @f(%struct.ss* inalloca(%struct.ss) %S)
   ret i32 %r
 }
 
 ; Argpromote can't promote %a because of the icmp use.
-define internal i1 @g(%struct.ss* %a, %struct.ss* inalloca %b) nounwind  {
+define internal i1 @g(%struct.ss* %a, %struct.ss* inalloca(%struct.ss) %b) nounwind  {
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@g
-; IS__CGSCC____-SAME: (%struct.ss* noalias nocapture nofree nonnull readnone align 4 dereferenceable(8) [[A:%.*]], %struct.ss* inalloca noalias nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[B:%.*]]) #[[ATTR1]] {
+; IS__CGSCC____-SAME: (%struct.ss* noalias nocapture nofree nonnull readnone align 4 dereferenceable(8) [[A:%.*]], %struct.ss* noalias nocapture nofree nonnull writeonly inalloca([[STRUCT_SS:%.*]]) align 4 dereferenceable(8) [[B:%.*]]) #[[ATTR1]] {
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    ret i1 undef
 ;
@@ -104,7 +104,7 @@ define i32 @test() {
 ;
 entry:
   %S = alloca inalloca %struct.ss
-  %c = call i1 @g(%struct.ss* %S, %struct.ss* inalloca %S)
+  %c = call i1 @g(%struct.ss* %S, %struct.ss* inalloca(%struct.ss) %S)
   ret i32 0
 }
 ;.
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index a0b958395964b99c8079508f59c801b67b4c0592..2cc1094deb225f72f4ca46f5ad1ea622c2e92b5e 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -1154,7 +1154,7 @@ declare void @llvm.assume(i1)
 ; IS__TUNIT_OPM: attributes #[[ATTR2]] = { argmemonly nofree nosync nounwind willreturn writeonly }
 ; IS__TUNIT_OPM: attributes #[[ATTR3]] = { argmemonly nofree nosync nounwind writeonly }
 ; IS__TUNIT_OPM: attributes #[[ATTR4]] = { argmemonly nofree nosync nounwind }
-; IS__TUNIT_OPM: attributes #[[ATTR5:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__TUNIT_OPM: attributes #[[ATTR5:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__TUNIT_OPM: attributes #[[ATTR6]] = { nofree nosync nounwind writeonly }
 ; IS__TUNIT_OPM: attributes #[[ATTR7]] = { willreturn }
 ; IS__TUNIT_OPM: attributes #[[ATTR8]] = { nounwind }
@@ -1164,7 +1164,7 @@ declare void @llvm.assume(i1)
 ; IS__TUNIT_NPM: attributes #[[ATTR2]] = { argmemonly nofree nosync nounwind willreturn writeonly }
 ; IS__TUNIT_NPM: attributes #[[ATTR3]] = { argmemonly nofree nosync nounwind willreturn }
 ; IS__TUNIT_NPM: attributes #[[ATTR4]] = { argmemonly nofree nosync nounwind writeonly }
-; IS__TUNIT_NPM: attributes #[[ATTR5:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__TUNIT_NPM: attributes #[[ATTR5:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__TUNIT_NPM: attributes #[[ATTR6]] = { nofree nosync nounwind willreturn writeonly }
 ; IS__TUNIT_NPM: attributes #[[ATTR7]] = { nofree nosync nounwind writeonly }
 ; IS__TUNIT_NPM: attributes #[[ATTR8]] = { willreturn }
@@ -1176,7 +1176,7 @@ declare void @llvm.assume(i1)
 ; IS__CGSCC_OPM: attributes #[[ATTR3]] = { argmemonly nofree norecurse nosync nounwind writeonly }
 ; IS__CGSCC_OPM: attributes #[[ATTR4]] = { argmemonly nofree norecurse nosync nounwind }
 ; IS__CGSCC_OPM: attributes #[[ATTR5]] = { argmemonly nofree nosync nounwind writeonly }
-; IS__CGSCC_OPM: attributes #[[ATTR6:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__CGSCC_OPM: attributes #[[ATTR6:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__CGSCC_OPM: attributes #[[ATTR7]] = { nosync nounwind writeonly }
 ; IS__CGSCC_OPM: attributes #[[ATTR8]] = { nofree nosync nounwind writeonly }
 ; IS__CGSCC_OPM: attributes #[[ATTR9]] = { willreturn }
@@ -1187,7 +1187,7 @@ declare void @llvm.assume(i1)
 ; IS__CGSCC_NPM: attributes #[[ATTR2]] = { argmemonly nofree norecurse nosync nounwind willreturn writeonly }
 ; IS__CGSCC_NPM: attributes #[[ATTR3]] = { argmemonly nofree norecurse nosync nounwind willreturn }
 ; IS__CGSCC_NPM: attributes #[[ATTR4]] = { argmemonly nofree nosync nounwind writeonly }
-; IS__CGSCC_NPM: attributes #[[ATTR5:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__CGSCC_NPM: attributes #[[ATTR5:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__CGSCC_NPM: attributes #[[ATTR6]] = { nosync nounwind willreturn writeonly }
 ; IS__CGSCC_NPM: attributes #[[ATTR7]] = { nofree nosync nounwind writeonly }
 ; IS__CGSCC_NPM: attributes #[[ATTR8]] = { willreturn }
diff --git a/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
index cddeebf18bfa77ac7cade8fa62922a12f0753545..8c55a0391be97b6bf8fee61a90997bd9220cb2e6 100644
--- a/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
+++ b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
@@ -320,22 +320,22 @@ declare void @dummy(i1) nounwind
 declare void @llvm.experimental.guard(i1, ...)
 ;.
 ; IS__TUNIT_OPM: attributes #[[ATTR0]] = { nofree nosync nounwind readnone }
-; IS__TUNIT_OPM: attributes #[[ATTR1:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__TUNIT_OPM: attributes #[[ATTR1:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__TUNIT_OPM: attributes #[[ATTR2]] = { nounwind }
 ; IS__TUNIT_OPM: attributes #[[ATTR3:[0-9]+]] = { nofree nosync willreturn }
 ;.
 ; IS__TUNIT_NPM: attributes #[[ATTR0]] = { nofree nosync nounwind readnone willreturn }
-; IS__TUNIT_NPM: attributes #[[ATTR1:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__TUNIT_NPM: attributes #[[ATTR1:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__TUNIT_NPM: attributes #[[ATTR2]] = { nounwind }
 ; IS__TUNIT_NPM: attributes #[[ATTR3:[0-9]+]] = { nofree nosync willreturn }
 ;.
 ; IS__CGSCC_OPM: attributes #[[ATTR0]] = { nofree norecurse nosync nounwind readnone }
-; IS__CGSCC_OPM: attributes #[[ATTR1:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__CGSCC_OPM: attributes #[[ATTR1:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__CGSCC_OPM: attributes #[[ATTR2]] = { nounwind }
 ; IS__CGSCC_OPM: attributes #[[ATTR3:[0-9]+]] = { nofree nosync willreturn }
 ;.
 ; IS__CGSCC_NPM: attributes #[[ATTR0]] = { nofree norecurse nosync nounwind readnone willreturn }
-; IS__CGSCC_NPM: attributes #[[ATTR1:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__CGSCC_NPM: attributes #[[ATTR1:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__CGSCC_NPM: attributes #[[ATTR2]] = { nounwind }
 ; IS__CGSCC_NPM: attributes #[[ATTR3:[0-9]+]] = { nofree nosync willreturn }
 ;.
diff --git a/llvm/test/Transforms/Attributor/nofree.ll b/llvm/test/Transforms/Attributor/nofree.ll
index 4273c69e7eafd631f3565fe145713c2a245db40a..5bf96a2864978b811c1229bf2c83d12e2b0c1eb9 100644
--- a/llvm/test/Transforms/Attributor/nofree.ll
+++ b/llvm/test/Transforms/Attributor/nofree.ll
@@ -478,7 +478,7 @@ attributes #2 = { nobuiltin nounwind }
 ; IS__TUNIT____: attributes #[[ATTR6:[0-9]+]] = { nofree nosync nounwind readnone speculatable willreturn }
 ; IS__TUNIT____: attributes #[[ATTR7]] = { nofree nounwind }
 ; IS__TUNIT____: attributes #[[ATTR8:[0-9]+]] = { nobuiltin nofree nounwind }
-; IS__TUNIT____: attributes #[[ATTR9:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__TUNIT____: attributes #[[ATTR9:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__TUNIT____: attributes #[[ATTR10:[0-9]+]] = { nounwind willreturn }
 ; IS__TUNIT____: attributes #[[ATTR11]] = { readnone willreturn }
 ; IS__TUNIT____: attributes #[[ATTR12]] = { willreturn }
@@ -493,7 +493,7 @@ attributes #2 = { nobuiltin nounwind }
 ; IS__CGSCC_OPM: attributes #[[ATTR7]] = { nofree noinline nosync nounwind readnone uwtable willreturn }
 ; IS__CGSCC_OPM: attributes #[[ATTR8]] = { nofree nounwind }
 ; IS__CGSCC_OPM: attributes #[[ATTR9:[0-9]+]] = { nobuiltin nofree nounwind }
-; IS__CGSCC_OPM: attributes #[[ATTR10:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__CGSCC_OPM: attributes #[[ATTR10:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__CGSCC_OPM: attributes #[[ATTR11:[0-9]+]] = { nounwind willreturn }
 ; IS__CGSCC_OPM: attributes #[[ATTR12]] = { readnone willreturn }
 ; IS__CGSCC_OPM: attributes #[[ATTR13]] = { willreturn }
@@ -508,7 +508,7 @@ attributes #2 = { nobuiltin nounwind }
 ; IS__CGSCC_NPM: attributes #[[ATTR7]] = { nofree noinline nosync nounwind readnone uwtable willreturn }
 ; IS__CGSCC_NPM: attributes #[[ATTR8]] = { nofree nounwind }
 ; IS__CGSCC_NPM: attributes #[[ATTR9:[0-9]+]] = { nobuiltin nofree nounwind }
-; IS__CGSCC_NPM: attributes #[[ATTR10:[0-9]+]] = { nofree nosync nounwind willreturn }
+; IS__CGSCC_NPM: attributes #[[ATTR10:[0-9]+]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__CGSCC_NPM: attributes #[[ATTR11:[0-9]+]] = { nounwind willreturn }
 ; IS__CGSCC_NPM: attributes #[[ATTR12]] = { readnone willreturn }
 ; IS__CGSCC_NPM: attributes #[[ATTR13]] = { willreturn }
diff --git a/llvm/test/Transforms/Attributor/nonnull.ll b/llvm/test/Transforms/Attributor/nonnull.ll
index af165baf3c8e156b2efdae6f377ee20cccb570d6..9a7a98bea5c65020882bc3a6517329c4461e207e 100644
--- a/llvm/test/Transforms/Attributor/nonnull.ll
+++ b/llvm/test/Transforms/Attributor/nonnull.ll
@@ -37,7 +37,7 @@ define i8* @test2(i8* nonnull %p) {
 
 define i8* @test2A(i1 %c, i8* %ret) {
 ; ATTRIBUTOR: define nonnull i8* @test2A(i1 %c, i8* nofree nonnull readnone returned %ret)
-; NOT_CGSCC_OPM: Function Attrs: nofree nosync nounwind willreturn
+; NOT_CGSCC_OPM: Function Attrs: inaccessiblememonly nofree nosync nounwind willreturn
 ; NOT_CGSCC_OPM-LABEL: define {{[^@]+}}@test2A
 ; NOT_CGSCC_OPM-SAME: (i1 [[C:%.*]], i8* nofree nonnull readnone returned "no-capture-maybe-returned" [[RET:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NOT_CGSCC_OPM-NEXT:    br i1 [[C]], label [[A:%.*]], label [[B:%.*]]
@@ -48,7 +48,7 @@ define i8* @test2A(i1 %c, i8* %ret) {
 ; NOT_CGSCC_OPM-NEXT:    call void @llvm.assume(i1 noundef true) #[[ATTR12]] [ "nonnull"(i8* [[RET]]) ]
 ; NOT_CGSCC_OPM-NEXT:    ret i8* [[RET]]
 ;
-; IS__CGSCC_OPM: Function Attrs: nofree nosync nounwind willreturn
+; IS__CGSCC_OPM: Function Attrs: inaccessiblememonly nofree nosync nounwind willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test2A
 ; IS__CGSCC_OPM-SAME: (i1 [[C:%.*]], i8* nofree nonnull readnone returned "no-capture-maybe-returned" [[RET:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IS__CGSCC_OPM-NEXT:    br i1 [[C]], label [[A:%.*]], label [[B:%.*]]
@@ -70,7 +70,7 @@ B:
 
 define i8* @test2B(i1 %c, i8* %ret) {
 ; ATTRIBUTOR: define nonnull dereferenceable(4) i8* @test2B(i1 %c, i8* nofree nonnull readnone returned dereferenceable(4) %ret)
-; NOT_CGSCC_OPM: Function Attrs: nofree nosync nounwind willreturn
+; NOT_CGSCC_OPM: Function Attrs: inaccessiblememonly nofree nosync nounwind willreturn
 ; NOT_CGSCC_OPM-LABEL: define {{[^@]+}}@test2B
 ; NOT_CGSCC_OPM-SAME: (i1 [[C:%.*]], i8* nofree nonnull readnone returned dereferenceable(4) "no-capture-maybe-returned" [[RET:%.*]]) #[[ATTR0]] {
 ; NOT_CGSCC_OPM-NEXT:    br i1 [[C]], label [[A:%.*]], label [[B:%.*]]
@@ -81,7 +81,7 @@ define i8* @test2B(i1 %c, i8* %ret) {
 ; NOT_CGSCC_OPM-NEXT:    call void @llvm.assume(i1 noundef true) #[[ATTR12]] [ "dereferenceable"(i8* [[RET]], i32 4) ]
 ; NOT_CGSCC_OPM-NEXT:    ret i8* [[RET]]
 ;
-; IS__CGSCC_OPM: Function Attrs: nofree nosync nounwind willreturn
+; IS__CGSCC_OPM: Function Attrs: inaccessiblememonly nofree nosync nounwind willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test2B
 ; IS__CGSCC_OPM-SAME: (i1 [[C:%.*]], i8* nofree nonnull readnone returned dereferenceable(4) "no-capture-maybe-returned" [[RET:%.*]]) #[[ATTR0]] {
 ; IS__CGSCC_OPM-NEXT:    br i1 [[C]], label [[A:%.*]], label [[B:%.*]]
@@ -335,14 +335,14 @@ define i8* @test9(i8* %a, i64 %n) {
 ; ATTRIBUTOR_OPM: define i8* @test10
 ; ATTRIBUTOR_NPM: define nonnull i8* @test10
 define i8* @test10(i8* %a, i64 %n) {
-; NOT_CGSCC_OPM: Function Attrs: nofree nosync nounwind willreturn
+; NOT_CGSCC_OPM: Function Attrs: inaccessiblememonly nofree nosync nounwind willreturn
 ; NOT_CGSCC_OPM-LABEL: define {{[^@]+}}@test10
 ; NOT_CGSCC_OPM-SAME: (i8* nofree readnone "no-capture-maybe-returned" [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; NOT_CGSCC_OPM-NEXT:    call void @llvm.assume(i1 noundef true) #[[ATTR12]]
 ; NOT_CGSCC_OPM-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[N]]
 ; NOT_CGSCC_OPM-NEXT:    ret i8* [[B]]
 ;
-; IS__CGSCC_OPM: Function Attrs: nofree nosync nounwind willreturn
+; IS__CGSCC_OPM: Function Attrs: inaccessiblememonly nofree nosync nounwind willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test10
 ; IS__CGSCC_OPM-SAME: (i8* nofree readnone "no-capture-maybe-returned" [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; IS__CGSCC_OPM-NEXT:    call void @llvm.assume(i1 noundef true) #[[ATTR13]]
@@ -1718,7 +1718,7 @@ declare void @nonnull_callee(i8* nonnull %p)
 attributes #0 = { null_pointer_is_valid }
 attributes #1 = { nounwind willreturn}
 ;.
-; IS__TUNIT____: attributes #[[ATTR0]] = { nofree nosync nounwind willreturn }
+; IS__TUNIT____: attributes #[[ATTR0]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__TUNIT____: attributes #[[ATTR1]] = { nofree nosync nounwind readnone willreturn }
 ; IS__TUNIT____: attributes #[[ATTR2]] = { nofree noreturn nosync nounwind readnone }
 ; IS__TUNIT____: attributes #[[ATTR3]] = { noreturn }
@@ -1734,7 +1734,7 @@ attributes #1 = { nounwind willreturn}
 ; IS__TUNIT____: attributes #[[ATTR13]] = { nofree nosync nounwind readonly }
 ; IS__TUNIT____: attributes #[[ATTR14]] = { readonly willreturn }
 ;.
-; IS__CGSCC_OPM: attributes #[[ATTR0]] = { nofree nosync nounwind willreturn }
+; IS__CGSCC_OPM: attributes #[[ATTR0]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__CGSCC_OPM: attributes #[[ATTR1]] = { nofree norecurse nosync nounwind readnone willreturn }
 ; IS__CGSCC_OPM: attributes #[[ATTR2]] = { nofree noreturn nosync nounwind readnone }
 ; IS__CGSCC_OPM: attributes #[[ATTR3]] = { nofree nosync nounwind readnone willreturn }
@@ -1751,7 +1751,7 @@ attributes #1 = { nounwind willreturn}
 ; IS__CGSCC_OPM: attributes #[[ATTR14]] = { nofree nosync nounwind readonly }
 ; IS__CGSCC_OPM: attributes #[[ATTR15]] = { readonly willreturn }
 ;.
-; IS__CGSCC_NPM: attributes #[[ATTR0]] = { nofree nosync nounwind willreturn }
+; IS__CGSCC_NPM: attributes #[[ATTR0]] = { inaccessiblememonly nofree nosync nounwind willreturn }
 ; IS__CGSCC_NPM: attributes #[[ATTR1]] = { nofree norecurse nosync nounwind readnone willreturn }
 ; IS__CGSCC_NPM: attributes #[[ATTR2]] = { nofree noreturn nosync nounwind readnone }
 ; IS__CGSCC_NPM: attributes #[[ATTR3]] = { noreturn }
diff --git a/llvm/test/Transforms/Attributor/readattrs.ll b/llvm/test/Transforms/Attributor/readattrs.ll
index 9f28407c76c4723cdeb9a3fc0a02e47953546878..c440e12ddfce44fa5f2c9e7c46811e927271f4e7 100644
--- a/llvm/test/Transforms/Attributor/readattrs.ll
+++ b/llvm/test/Transforms/Attributor/readattrs.ll
@@ -107,15 +107,15 @@ define void @test6_2(i8** %p, i8* %q) {
 }
 
 ; inalloca parameters are always considered written
-define void @test7_1(i32* inalloca %a) {
+define void @test7_1(i32* inalloca(i32) %a) {
 ; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test7_1
-; IS__TUNIT____-SAME: (i32* inalloca nocapture nofree nonnull writeonly dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly inalloca(i32) dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
 ; IS__TUNIT____-NEXT:    ret void
 ;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test7_1
-; IS__CGSCC____-SAME: (i32* inalloca nocapture nofree nonnull writeonly dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly inalloca(i32) dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
 ; IS__CGSCC____-NEXT:    ret void
 ;
   ret void
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index 41fa16c0284a88b3d7e5d0e71f9e8e8bc2a63a20..0ed93a912789bc7dd7805ce1cd159ffa459e53a9 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -332,15 +332,15 @@ define i32 @ipccp3() {
 
 ; Do not touch complicated arguments (for now)
 %struct.X = type { i8* }
-define internal i32* @test_inalloca(i32* inalloca %a) {
+define internal i32* @test_inalloca(i32* inalloca(i32) %a) {
 ; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test_inalloca
-; IS__TUNIT____-SAME: (i32* inalloca noalias nofree nonnull returned writeonly dereferenceable(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR1]] {
+; IS__TUNIT____-SAME: (i32* noalias nofree nonnull returned writeonly inalloca(i32) dereferenceable(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR1]] {
 ; IS__TUNIT____-NEXT:    ret i32* [[A]]
 ;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test_inalloca
-; IS__CGSCC____-SAME: (i32* inalloca noalias nofree noundef nonnull returned writeonly dereferenceable(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR1]] {
+; IS__CGSCC____-SAME: (i32* noalias nofree noundef nonnull returned writeonly inalloca(i32) dereferenceable(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR1]] {
 ; IS__CGSCC____-NEXT:    ret i32* [[A]]
 ;
   ret i32* %a
diff --git a/llvm/test/Transforms/ConstraintElimination/geps-inbounds-precondition.ll b/llvm/test/Transforms/ConstraintElimination/geps-inbounds-precondition.ll
new file mode 100644
index 0000000000000000000000000000000000000000..63e32d29d8fca526a23af714c44b380747ef27c3
--- /dev/null
+++ b/llvm/test/Transforms/ConstraintElimination/geps-inbounds-precondition.ll
@@ -0,0 +1,344 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -constraint-elimination -S %s | FileCheck %s
+
+; Tests for using inbounds information from GEPs.
+
+declare void @noundef(i32* noundef)
+
+define i1 @inbounds_poison_is_ub1(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @inbounds_poison_is_ub1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[UPPER:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 5
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER]])
+; CHECK-NEXT:    [[CMP_IDX:%.*]] = icmp ult i32 [[IDX:%.*]], [[N:%.*]]
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[SRC_IDX_4:%.*]] = getelementptr i32, i32* [[SRC]], i64 4
+; CHECK-NEXT:    [[CMP_UPPER_4:%.*]] = icmp ule i32* [[SRC_IDX_4]], [[UPPER]]
+; CHECK-NEXT:    [[SRC_IDX_5:%.*]] = getelementptr i32, i32* [[SRC]], i64 5
+; CHECK-NEXT:    [[CMP_UPPER_5:%.*]] = icmp ule i32* [[SRC_IDX_5]], [[UPPER]]
+; CHECK-NEXT:    [[RES_0:%.*]] = xor i1 [[CMP_UPPER_4]], [[CMP_UPPER_5]]
+; CHECK-NEXT:    [[SRC_IDX_6:%.*]] = getelementptr i32, i32* [[SRC]], i64 6
+; CHECK-NEXT:    [[CMP_UPPER_6:%.*]] = icmp ule i32* [[SRC_IDX_6]], [[UPPER]]
+; CHECK-NEXT:    [[RES_1:%.*]] = xor i1 [[RES_0]], [[CMP_UPPER_6]]
+; CHECK-NEXT:    [[SRC_IDX_NEG_1:%.*]] = getelementptr i32, i32* [[SRC]], i64 -1
+; CHECK-NEXT:    [[CMP_UPPER_NEG_1:%.*]] = icmp ule i32* [[SRC_IDX_NEG_1]], [[UPPER]]
+; CHECK-NEXT:    [[RES_2:%.*]] = xor i1 [[RES_1]], [[CMP_UPPER_NEG_1]]
+; CHECK-NEXT:    ret i1 [[RES_2]]
+;
+entry:
+  %upper = getelementptr inbounds i32, i32* %src, i64 5
+  call void @noundef(i32* %upper)
+  %cmp.idx = icmp ult i32 %idx, %n
+  %idx.ext = zext i32 %idx to i64
+  %src.idx.4 = getelementptr i32, i32* %src, i64 4
+  %cmp.upper.4 = icmp ule i32* %src.idx.4, %upper
+  %src.idx.5 = getelementptr i32, i32* %src, i64 5
+  %cmp.upper.5 = icmp ule i32* %src.idx.5, %upper
+  %res.0 = xor i1 %cmp.upper.4, %cmp.upper.5
+
+  %src.idx.6 = getelementptr i32, i32* %src, i64 6
+  %cmp.upper.6 = icmp ule i32* %src.idx.6, %upper
+  %res.1 = xor i1 %res.0, %cmp.upper.6
+
+  %src.idx.neg.1 = getelementptr i32, i32* %src, i64 -1
+  %cmp.upper.neg.1 = icmp ule i32* %src.idx.neg.1, %upper
+  %res.2 = xor i1 %res.1, %cmp.upper.neg.1
+  ret i1 %res.2
+}
+
+; %start + %n.ext is guaranteed to not overflow (due to inbounds).
+; %start + %idx.ext does not overflow if %idx.ext <= %n.ext.
+define i1 @inbounds_poison_is_ub2(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @inbounds_poison_is_ub2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[UPPER:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 [[N_EXT]]
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER]])
+; CHECK-NEXT:    [[CMP_IDX:%.*]] = icmp ult i32 [[IDX:%.*]], [[N]]
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[SRC_IDX:%.*]] = getelementptr i32, i32* [[SRC]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    br i1 [[CMP_IDX]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[CMP_UPPER_1:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_1]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP_UPPER_2:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_2]]
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %upper = getelementptr inbounds i32, i32* %src, i64 %n.ext
+  call void @noundef(i32* %upper)
+  %cmp.idx = icmp ult i32 %idx, %n
+  %idx.ext = zext i32 %idx to i64
+  %src.idx = getelementptr i32, i32* %src, i64 %idx.ext
+  br i1 %cmp.idx, label %then, label %else
+
+then:
+  %cmp.upper.1 = icmp ule i32* %src.idx, %upper
+  ret i1 %cmp.upper.1
+
+else:
+  %cmp.upper.2 = icmp ule i32* %src.idx, %upper
+  ret i1 %cmp.upper.2
+}
+
+; Same as inbounds_poison_is_ub2, but with individual GEPs in the %then and
+; %else blocks.
+define i1 @inbounds_poison_is_ub3(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @inbounds_poison_is_ub3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX:%.*]] to i64
+; CHECK-NEXT:    [[CMP_IDX:%.*]] = icmp ult i64 [[IDX_EXT]], [[N_EXT]]
+; CHECK-NEXT:    br i1 [[CMP_IDX]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[UPPER_1:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 [[N_EXT]]
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_1]])
+; CHECK-NEXT:    [[SRC_IDX_1:%.*]] = getelementptr i32, i32* [[SRC]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[CMP_UPPER_1:%.*]] = icmp ule i32* [[SRC_IDX_1]], [[UPPER_1]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_1]]
+; CHECK:       else:
+; CHECK-NEXT:    [[UPPER_2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 [[N_EXT]]
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_2]])
+; CHECK-NEXT:    [[SRC_IDX_2:%.*]] = getelementptr i32, i32* [[SRC]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[CMP_UPPER_2:%.*]] = icmp ule i32* [[SRC_IDX_2]], [[UPPER_2]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_2]]
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %idx.ext = zext i32 %idx to i64
+  %cmp.idx = icmp ult i64 %idx.ext, %n.ext
+  br i1 %cmp.idx, label %then, label %else
+
+then:
+  %upper.1 = getelementptr inbounds i32, i32* %src, i64 %n.ext
+  call void @noundef(i32* %upper.1)
+  %src.idx.1 = getelementptr i32, i32* %src, i64 %idx.ext
+  %cmp.upper.1 = icmp ule i32* %src.idx.1, %upper.1
+  ret i1 %cmp.upper.1
+
+else:
+  %upper.2 = getelementptr inbounds i32, i32* %src, i64 %n.ext
+  call void @noundef(i32* %upper.2)
+  %src.idx.2 = getelementptr i32, i32* %src, i64 %idx.ext
+  %cmp.upper.2 = icmp ule i32* %src.idx.2, %upper.2
+  ret i1 %cmp.upper.2
+}
+
+; The function does not have UB if %upper is poison because of an overflow. Do
+; not simplify anything. In this particular case, the returned result will be
+; poison in this case, so it could be simplified, but currently we cannot
+; distinguish that case.
+define i1 @inbounds_poison_does_not_cause_ub(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @inbounds_poison_does_not_cause_ub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX:%.*]] to i64
+; CHECK-NEXT:    [[UPPER:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 [[N_EXT]]
+; CHECK-NEXT:    [[SRC_IDX:%.*]] = getelementptr i32, i32* [[SRC]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[CMP_IDX:%.*]] = icmp ult i64 [[IDX_EXT]], [[N_EXT]]
+; CHECK-NEXT:    br i1 [[CMP_IDX]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[CMP_UPPER_1:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_1]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP_UPPER_2:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_2]]
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %idx.ext = zext i32 %idx to i64
+  %upper = getelementptr inbounds i32, i32* %src, i64 %n.ext
+  %src.idx = getelementptr i32, i32* %src, i64 %idx.ext
+  %cmp.idx = icmp ult i64 %idx.ext, %n.ext
+  br i1 %cmp.idx, label %then, label %else
+
+then:
+  %cmp.upper.1 = icmp ule i32* %src.idx, %upper
+  ret i1 %cmp.upper.1
+
+else:
+  %cmp.upper.2 = icmp ule i32* %src.idx, %upper
+  ret i1 %cmp.upper.2
+}
+
+; Same as @inbounds_poison_does_not_cause_ub, but with separate GEPs in the
+; %then and %else blocks.
+define i1 @inbounds_poison_does_not_cause_ub2(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @inbounds_poison_does_not_cause_ub2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX:%.*]] to i64
+; CHECK-NEXT:    [[CMP_IDX:%.*]] = icmp ult i64 [[IDX_EXT]], [[N_EXT]]
+; CHECK-NEXT:    br i1 [[CMP_IDX]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[UPPER_1:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 [[N_EXT]]
+; CHECK-NEXT:    [[SRC_IDX_1:%.*]] = getelementptr i32, i32* [[SRC]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[CMP_UPPER_1:%.*]] = icmp ule i32* [[SRC_IDX_1]], [[UPPER_1]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_1]]
+; CHECK:       else:
+; CHECK-NEXT:    [[SRC_IDX_2:%.*]] = getelementptr i32, i32* [[SRC]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[UPPER_2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 [[N_EXT]]
+; CHECK-NEXT:    [[CMP_UPPER_2:%.*]] = icmp ule i32* [[SRC_IDX_2]], [[UPPER_2]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_2]]
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %idx.ext = zext i32 %idx to i64
+  %cmp.idx = icmp ult i64 %idx.ext, %n.ext
+  br i1 %cmp.idx, label %then, label %else
+
+then:
+  %upper.1 = getelementptr inbounds i32, i32* %src, i64 %n.ext
+  %src.idx.1 = getelementptr i32, i32* %src, i64 %idx.ext
+  %cmp.upper.1 = icmp ule i32* %src.idx.1, %upper.1
+  ret i1 %cmp.upper.1
+
+else:
+  %src.idx.2 = getelementptr i32, i32* %src, i64 %idx.ext
+  %upper.2 = getelementptr inbounds i32, i32* %src, i64 %n.ext
+  %cmp.upper.2 = icmp ule i32* %src.idx.2, %upper.2
+  ret i1 %cmp.upper.2
+}
+
+define i1 @no_zexts_indices_may_be_negative(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @no_zexts_indices_may_be_negative(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[UPPER:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i32 [[N:%.*]]
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER]])
+; CHECK-NEXT:    [[SRC_IDX:%.*]] = getelementptr i32, i32* [[SRC]], i32 [[IDX:%.*]]
+; CHECK-NEXT:    [[CMP_IDX:%.*]] = icmp ult i32 [[IDX]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP_IDX]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[CMP_UPPER_1:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_1]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP_UPPER_2:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_2]]
+;
+entry:
+  %upper = getelementptr inbounds i32, i32* %src, i32 %n
+  call void @noundef(i32* %upper)
+  %src.idx = getelementptr i32, i32* %src, i32 %idx
+  %cmp.idx = icmp ult i32 %idx, %n
+  br i1 %cmp.idx, label %then, label %else
+
+then:
+  %cmp.upper.1 = icmp ule i32* %src.idx, %upper
+  ret i1 %cmp.upper.1
+
+else:
+  %cmp.upper.2 = icmp ule i32* %src.idx, %upper
+  ret i1 %cmp.upper.2
+}
+
+; Tests for multiple inbound GEPs, make sure the largest upper bound is used.
+define i1 @multiple_upper_bounds(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @multiple_upper_bounds(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[UPPER_1:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_1]])
+; CHECK-NEXT:    [[UPPER_2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 [[N_EXT]]
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_2]])
+; CHECK-NEXT:    [[CMP_IDX:%.*]] = icmp ult i32 [[IDX:%.*]], [[N]]
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[SRC_IDX:%.*]] = getelementptr i32, i32* [[SRC]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    br i1 [[CMP_IDX]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[CMP_UPPER_1:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER_2]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_1]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP_UPPER_2:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER_2]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_2]]
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %upper.1 = getelementptr inbounds i32, i32* %src, i64 1
+  call void @noundef(i32* %upper.1)
+  %upper.2 = getelementptr inbounds i32, i32* %src, i64 %n.ext
+  call void @noundef(i32* %upper.2)
+  %cmp.idx = icmp ult i32 %idx, %n
+  %idx.ext = zext i32 %idx to i64
+  %src.idx = getelementptr i32, i32* %src, i64 %idx.ext
+  br i1 %cmp.idx, label %then, label %else
+
+then:
+  %cmp.upper.1 = icmp ule i32* %src.idx, %upper.2
+  ret i1 %cmp.upper.1
+
+else:
+  %cmp.upper.2 = icmp ule i32* %src.idx, %upper.2
+  ret i1 %cmp.upper.2
+}
+
+define i1 @multiple_upper_bounds2(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @multiple_upper_bounds2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[UPPER_1:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_1]])
+; CHECK-NEXT:    [[UPPER_2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 4
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_2]])
+; CHECK-NEXT:    [[SRC_IDX:%.*]] = getelementptr i32, i32* [[SRC]], i64 4
+; CHECK-NEXT:    [[CMP_UPPER_1:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER_2]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_1]]
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %upper.1 = getelementptr inbounds i32, i32* %src, i64 1
+  call void @noundef(i32* %upper.1)
+  %upper.2 = getelementptr inbounds i32, i32* %src, i64 4
+  call void @noundef(i32* %upper.2)
+  %src.idx = getelementptr i32, i32* %src, i64 4
+  %cmp.upper.1 = icmp ule i32* %src.idx, %upper.2
+  ret i1 %cmp.upper.1
+}
+
+define i1 @multiple_upper_bounds3(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @multiple_upper_bounds3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[UPPER_1:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 4
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_1]])
+; CHECK-NEXT:    [[UPPER_2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 1
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_2]])
+; CHECK-NEXT:    [[SRC_IDX:%.*]] = getelementptr i32, i32* [[SRC]], i64 4
+; CHECK-NEXT:    [[CMP_UPPER_1:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER_1]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_1]]
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %upper.1 = getelementptr inbounds i32, i32* %src, i64 4
+  call void @noundef(i32* %upper.1)
+  %upper.2 = getelementptr inbounds i32, i32* %src, i64 1
+  call void @noundef(i32* %upper.2)
+  %src.idx = getelementptr i32, i32* %src, i64 4
+  %cmp.upper.1 = icmp ule i32* %src.idx, %upper.1
+  ret i1 %cmp.upper.1
+}
+
+; %src.idx + 5 may overflow.
+define i1 @multiple_upper_bounds4(i32* %src, i32 %n, i32 %idx) {
+; CHECK-LABEL: @multiple_upper_bounds4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[UPPER_1:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_1]])
+; CHECK-NEXT:    [[UPPER_2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 4
+; CHECK-NEXT:    call void @noundef(i32* [[UPPER_2]])
+; CHECK-NEXT:    [[SRC_IDX:%.*]] = getelementptr i32, i32* [[SRC]], i64 5
+; CHECK-NEXT:    [[CMP_UPPER_1:%.*]] = icmp ule i32* [[SRC_IDX]], [[UPPER_2]]
+; CHECK-NEXT:    ret i1 [[CMP_UPPER_1]]
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %upper.1 = getelementptr inbounds i32, i32* %src, i64 1
+  call void @noundef(i32* %upper.1)
+  %upper.2 = getelementptr inbounds i32, i32* %src, i64 4
+  call void @noundef(i32* %upper.2)
+  %src.idx = getelementptr i32, i32* %src, i64 5
+  %cmp.upper.1 = icmp ule i32* %src.idx, %upper.2
+  ret i1 %cmp.upper.1
+}
diff --git a/llvm/test/Transforms/ConstraintElimination/geps-pointers-to-arrays.ll b/llvm/test/Transforms/ConstraintElimination/geps-pointers-to-arrays.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fe4d3b83a6d8659ad32e5e291cb728e551a26565
--- /dev/null
+++ b/llvm/test/Transforms/ConstraintElimination/geps-pointers-to-arrays.ll
@@ -0,0 +1,254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -constraint-elimination -S %s | FileCheck %s
+
+define void @pointer.to.array.test.ult.true.due.to.first.dimension([10 x i8]* %start, i8* %high) {
+; CHECK-LABEL: @pointer.to.array.test.ult.true.due.to.first.dimension(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 9, i64 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_I]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 5, i64 0
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_0]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 9, i64 3
+  %c.1 = icmp ule i8* %add.ptr.i, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 5, i64 0
+  %c.0 = icmp ult i8* %start.0, %high
+  call void @use(i1 %c.0)
+
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+define void @pointer.to.array.test.ult.unknown.due.to.first.dimension([10 x i8]* %start, i8* %high) {
+; CHECK-LABEL: @pointer.to.array.test.ult.unknown.due.to.first.dimension(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 5, i64 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_I]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 10, i64 0
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_0]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 5, i64 3
+  %c.1 = icmp ule i8* %add.ptr.i, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 10, i64 0
+  %c.0 = icmp ult i8* %start.0, %high
+  call void @use(i1 %c.0)
+
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+define void @pointer.to.array.test.ult.true.due.to.second.dimension([10 x i8]* %start, i8* %high) {
+; CHECK-LABEL: @pointer.to.array.test.ult.true.due.to.second.dimension(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 5, i64 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_I]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 5, i64 0
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_0]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 5, i64 1
+  %c.1 = icmp ule i8* %add.ptr.i, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 5, i64 0
+  %c.0 = icmp ult i8* %start.0, %high
+  call void @use(i1 %c.0)
+
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+define void @pointer.to.array.test.ult.unknown.to.second.dimension([10 x i8]* %start, i8* %high) {
+; CHECK-LABEL: @pointer.to.array.test.ult.unknown.to.second.dimension(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 5, i64 0
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_I]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 5, i64 1
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_0]])
+; CHECK-NEXT:    ret void
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 5, i64 0
+  %c.1 = icmp ule i8* %add.ptr.i, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 5, i64 1
+  %c.0 = icmp ult i8* %start.0, %high
+  call void @use(i1 %c.0)
+
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+define void @pointer.to.array.test.not.uge.ult([10 x i8]* %start, i8* %high) {
+; CHECK-LABEL: @pointer.to.array.test.not.uge.ult(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 1, i64 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp uge i8* [[ADD_PTR_I]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret void
+; CHECK:       if.end:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 10, i64 0
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_0]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 1, i64 3
+  %c.1 = icmp uge i8* %add.ptr.i, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.end:                                           ; preds = %entry
+  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 10, i64 0
+  %c.0 = icmp ult i8* %start.0, %high
+  call void @use(i1 %c.0)
+  ret void
+}
+
+define void @pointer.to.array.test.not.uge.ule([10 x i8]* %start, i8* %high) {
+; CHECK-LABEL: @pointer.to.array.test.not.uge.ule(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 1, i64 3
+; CHECK-NEXT:    [[C:%.*]] = icmp uge i8* [[ADD_PTR_I]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret void
+; CHECK:       if.end:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 10, i64 0
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ule i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_0]])
+; CHECK-NEXT:    [[START_1:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 2, i64 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_1]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 1, i64 3
+  %c = icmp uge i8* %add.ptr.i, %high
+  br i1 %c, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.end:                                           ; preds = %entry
+  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 10, i64 0
+  %c.0 = icmp ule i8* %start.0, %high
+  call void @use(i1 %c.0)
+  %start.1 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 2, i64 1
+  %c.1 = icmp ule i8* %start.1, %high
+  call void @use(i1 %c.1)
+  ret void
+}
+
+define void @pointer.to.array.test.not.uge.ugt([10 x i8]* %start, i8* %high) {
+; CHECK-LABEL: @pointer.to.array.test.not.uge.ugt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 1, i64 3
+; CHECK-NEXT:    [[C:%.*]] = icmp uge i8* [[ADD_PTR_I]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret void
+; CHECK:       if.end:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 3, i64 0
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ugt i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_0]])
+; CHECK-NEXT:    [[START_1:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 3, i64 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ugt i8* [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_1]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 1, i64 3
+  %c = icmp uge i8* %add.ptr.i, %high
+  br i1 %c, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.end:                                           ; preds = %entry
+  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 3, i64 0
+  %c.0 = icmp ugt i8* %start.0, %high
+  call void @use(i1 %c.0)
+
+  %start.1 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 3, i64 1
+  %c.1 = icmp ugt i8* %start.1, %high
+  call void @use(i1 %c.1)
+  ret void
+}
+
+define void @pointer.to.array.test.not.uge.uge([10 x i8]* %start, i8* %high) {
+; CHECK-LABEL: @pointer.to.array.test.not.uge.uge(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 1, i64 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp uge i8* [[ADD_PTR_I]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret void
+; CHECK:       if.end:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 3, i64 0
+; CHECK-NEXT:    [[C_0:%.*]] = icmp uge i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_0]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 1, i64 3
+  %c.1 = icmp uge i8* %add.ptr.i, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.end:                                           ; preds = %entry
+  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 3, i64 0
+  %c.0 = icmp uge i8* %start.0, %high
+  call void @use(i1 %c.0)
+
+  ret void
+}
+
+declare void @use(i1)
diff --git a/llvm/test/Transforms/ConstraintElimination/geps-pointers-to-structs.ll b/llvm/test/Transforms/ConstraintElimination/geps-pointers-to-structs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..750293b8c26a15c5d2521d52b251538f07024c3b
--- /dev/null
+++ b/llvm/test/Transforms/ConstraintElimination/geps-pointers-to-structs.ll
@@ -0,0 +1,437 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -constraint-elimination -S %s | FileCheck %s
+
+%struct.1 = type { i32, i64, i8 }
+
+define i1 @test.ult.true.due.to.first.dimension(%struct.1* %start, i8* %high) {
+; CHECK-LABEL: @test.ult.true.due.to.first.dimension(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_1:%.*]], %struct.1* [[START:%.*]], i64 6, i32 0
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_1]], %struct.1* [[START]], i64 5, i32 0
+; CHECK-NEXT:    [[START_0_CAST:%.*]] = bitcast i32* [[START_0]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0_CAST]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %add.ptr = getelementptr inbounds %struct.1, %struct.1* %start, i64 6, i32 0
+  %add.ptr.cast = bitcast i32* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds %struct.1, %struct.1* %start, i64 5, i32 0
+  %start.0.cast = bitcast i32* %start.0 to i8*
+  %c.0 = icmp ult i8* %start.0.cast, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @test.ult.true.due.to.first.dimension.var.index.0(%struct.1* %start, i8* %high, i32 %idx) {
+; CHECK-LABEL: @test.ult.true.due.to.first.dimension.var.index.0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1:%.*]] = add nuw nsw i32 [[IDX:%.*]], 1
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1_EXT:%.*]] = zext i32 [[IDX_EXT_PLUS_1]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_1:%.*]], %struct.1* [[START:%.*]], i64 [[IDX_EXT_PLUS_1_EXT]], i32 0
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_1]], %struct.1* [[START]], i64 [[IDX_EXT]], i32 0
+; CHECK-NEXT:    [[START_0_CAST:%.*]] = bitcast i32* [[START_0]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0_CAST]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %idx.ext.plus.1 = add nuw nsw i32 %idx, 1
+  %idx.ext.plus.1.ext = zext i32 %idx.ext.plus.1 to i64
+  %add.ptr = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext.plus.1.ext, i32 0
+  %add.ptr.cast = bitcast i32* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %idx.ext = zext i32 %idx to i64
+  %start.0 = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext, i32 0
+  %start.0.cast = bitcast i32* %start.0 to i8*
+  %c.0 = icmp ult i8* %start.0.cast, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @test.ult.true.due.to.first.dimension.var.index.1(%struct.1* %start, i8* %high, i32 %idx) {
+; CHECK-LABEL: @test.ult.true.due.to.first.dimension.var.index.1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1:%.*]] = add nuw nsw i32 [[IDX:%.*]], 1
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1_EXT:%.*]] = zext i32 [[IDX_EXT_PLUS_1]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_1:%.*]], %struct.1* [[START:%.*]], i64 [[IDX_EXT_PLUS_1_EXT]], i32 0
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_1]], %struct.1* [[START]], i64 [[IDX_EXT]], i32 1
+; CHECK-NEXT:    [[START_0_CAST:%.*]] = bitcast i64* [[START_0]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0_CAST]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %idx.ext.plus.1 = add nuw nsw i32 %idx, 1
+  %idx.ext.plus.1.ext = zext i32 %idx.ext.plus.1 to i64
+  %add.ptr = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext.plus.1.ext, i32 0
+  %add.ptr.cast = bitcast i32* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %idx.ext = zext i32 %idx to i64
+  %start.0 = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext, i32 1
+  %start.0.cast = bitcast i64* %start.0 to i8*
+  %c.0 = icmp ult i8* %start.0.cast, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @test.ult.true.due.to.first.dimension.var.index.2(%struct.1* %start, i8* %high, i32 %idx) {
+; CHECK-LABEL: @test.ult.true.due.to.first.dimension.var.index.2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1:%.*]] = add nuw nsw i32 [[IDX:%.*]], 1
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1_EXT:%.*]] = zext i32 [[IDX_EXT_PLUS_1]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_1:%.*]], %struct.1* [[START:%.*]], i64 [[IDX_EXT_PLUS_1_EXT]], i32 0
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_1]], %struct.1* [[START]], i64 [[IDX_EXT]], i32 2
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %idx.ext.plus.1 = add nuw nsw i32 %idx, 1
+  %idx.ext.plus.1.ext = zext i32 %idx.ext.plus.1 to i64
+  %add.ptr = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext.plus.1.ext, i32 0
+  %add.ptr.cast = bitcast i32* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %idx.ext = zext i32 %idx to i64
+  %start.0 = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext, i32 2
+  %c.0 = icmp ult i8* %start.0, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @test.ult.unknown.due.to.first.dimension(%struct.1* %start, i8* %high) {
+; CHECK-LABEL: @test.ult.unknown.due.to.first.dimension(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_1:%.*]], %struct.1* [[START:%.*]], i64 5, i32 0
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_1]], %struct.1* [[START]], i64 6, i32 0
+; CHECK-NEXT:    [[START_0_CAST:%.*]] = bitcast i32* [[START_0]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0_CAST]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %add.ptr = getelementptr inbounds %struct.1, %struct.1* %start, i64 5, i32 0
+  %add.ptr.cast = bitcast i32* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds %struct.1, %struct.1* %start, i64 6, i32 0
+  %start.0.cast = bitcast i32* %start.0 to i8*
+  %c.0 = icmp ult i8* %start.0.cast, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @test.ult.true.due.to.second.dimension(%struct.1* %start, i8* %high) {
+; CHECK-LABEL: @test.ult.true.due.to.second.dimension(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_1:%.*]], %struct.1* [[START:%.*]], i64 5, i32 2
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_1]], %struct.1* [[START]], i64 5, i32 1
+; CHECK-NEXT:    [[START_0_CAST:%.*]] = bitcast i64* [[START_0]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0_CAST]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %add.ptr = getelementptr inbounds %struct.1, %struct.1* %start, i64 5, i32 2
+  %c.1 = icmp ule i8* %add.ptr, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds %struct.1, %struct.1* %start, i64 5, i32 1
+  %start.0.cast = bitcast i64 * %start.0 to i8*
+  %c.0 = icmp ult i8* %start.0.cast, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @test.ult.unknown.due.to.second.dimension(%struct.1* %start, i8* %high) {
+; CHECK-LABEL: @test.ult.unknown.due.to.second.dimension(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_1:%.*]], %struct.1* [[START:%.*]], i64 5, i32 2
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_1]], %struct.1* [[START]], i64 5, i32 2
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %add.ptr = getelementptr inbounds %struct.1, %struct.1* %start, i64 5, i32 2
+  %c.1 = icmp ule i8* %add.ptr, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds %struct.1, %struct.1* %start, i64 5, i32 2
+  %c.0 = icmp ult i8* %start.0, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @test.ult.unknown.due.to.second.dimension.var.index.0(%struct.1* %start, i8* %high, i32 %idx) {
+; CHECK-LABEL: @test.ult.unknown.due.to.second.dimension.var.index.0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX:%.*]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_1:%.*]], %struct.1* [[START:%.*]], i64 [[IDX_EXT]], i32 0
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_1]], %struct.1* [[START]], i64 [[IDX_EXT]], i32 1
+; CHECK-NEXT:    [[START_0_CAST:%.*]] = bitcast i64* [[START_0]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0_CAST]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %idx.ext = zext i32 %idx to i64
+  %add.ptr = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext, i32 0
+  %add.ptr.cast = bitcast i32* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext, i32 1
+  %start.0.cast = bitcast i64* %start.0 to i8*
+  %c.0 = icmp ult i8* %start.0.cast, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @test.ult.unknown.due.to.second.dimension.var.index.1(%struct.1* %start, i8* %high, i32 %idx) {
+; CHECK-LABEL: @test.ult.unknown.due.to.second.dimension.var.index.1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX:%.*]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_1:%.*]], %struct.1* [[START:%.*]], i64 [[IDX_EXT]], i32 1
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i64* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_1]], %struct.1* [[START]], i64 [[IDX_EXT]], i32 2
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %idx.ext = zext i32 %idx to i64
+  %add.ptr = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext, i32 1
+  %add.ptr.cast = bitcast i64* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds %struct.1, %struct.1* %start, i64 %idx.ext, i32 2
+  %c.0 = icmp ult i8* %start.0, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+%struct.2 = type { i32, [20 x i64], i8 }
+
+define i1 @ptr.int.struct.test.ult.true.due.to.first.dimension(%struct.2* %start, i8* %high) {
+; CHECK-LABEL: @ptr.int.struct.test.ult.true.due.to.first.dimension(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_2:%.*]], %struct.2* [[START:%.*]], i64 6, i32 1, i32 5
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i64* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_2]], %struct.2* [[START]], i64 6, i32 0
+; CHECK-NEXT:    [[START_0_CAST:%.*]] = bitcast i32* [[START_0]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0_CAST]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %add.ptr = getelementptr inbounds %struct.2, %struct.2* %start, i64 6, i32 1, i32 5
+  %add.ptr.cast = bitcast i64* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %start.0 = getelementptr inbounds %struct.2, %struct.2* %start, i64 6, i32 0
+  %start.0.cast = bitcast i32* %start.0 to i8*
+  %c.0 = icmp ult i8* %start.0.cast, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @ptr.int.struct.test.ult.true.due.to.third.dimension.var.index(%struct.2* %start, i8* %high, i32 %idx) {
+; CHECK-LABEL: @ptr.int.struct.test.ult.true.due.to.third.dimension.var.index(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1:%.*]] = add nuw nsw i32 [[IDX:%.*]], 1
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1_EXT:%.*]] = zext i32 [[IDX_EXT_PLUS_1]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_2:%.*]], %struct.2* [[START:%.*]], i64 6, i32 1, i64 [[IDX_EXT_PLUS_1_EXT]]
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i64* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_2]], %struct.2* [[START]], i64 6, i32 1, i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[START_0_CAST:%.*]] = bitcast i64* [[START_0]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0_CAST]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %idx.ext.plus.1 = add nuw nsw i32 %idx, 1
+  %idx.ext.plus.1.ext = zext i32 %idx.ext.plus.1 to i64
+  %add.ptr = getelementptr inbounds %struct.2, %struct.2* %start, i64 6, i32 1, i64 %idx.ext.plus.1.ext
+  %add.ptr.cast = bitcast i64* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %idx.ext = zext i32 %idx to i64
+  %start.0 = getelementptr inbounds %struct.2, %struct.2* %start, i64 6, i32 1, i64 %idx.ext
+  %start.0.cast = bitcast i64* %start.0 to i8*
+  %c.0 = icmp ult i8* %start.0.cast, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @ptr.int.struct.test.ult.due.to.second.dimension.var.index(%struct.2* %start, i8* %high, i32 %idx) {
+; CHECK-LABEL: @ptr.int.struct.test.ult.due.to.second.dimension.var.index(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1:%.*]] = add nuw nsw i32 [[IDX:%.*]], 1
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1_EXT:%.*]] = zext i32 [[IDX_EXT_PLUS_1]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_2:%.*]], %struct.2* [[START:%.*]], i64 6, i32 1, i64 [[IDX_EXT_PLUS_1_EXT]]
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i64* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_2]], %struct.2* [[START]], i64 6, i32 0
+; CHECK-NEXT:    [[START_0_CAST:%.*]] = bitcast i32* [[START_0]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0_CAST]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %idx.ext.plus.1 = add nuw nsw i32 %idx, 1
+  %idx.ext.plus.1.ext = zext i32 %idx.ext.plus.1 to i64
+  %add.ptr = getelementptr inbounds %struct.2, %struct.2* %start, i64 6, i32 1, i64 %idx.ext.plus.1.ext
+  %add.ptr.cast = bitcast i64* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %idx.ext = zext i32 %idx to i64
+  %start.0 = getelementptr inbounds %struct.2, %struct.2* %start, i64 6, i32 0
+  %start.0.cast = bitcast i32* %start.0 to i8*
+  %c.0 = icmp ult i8* %start.0.cast, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
+
+define i1 @ptr.int.struct.test.ult.unknown.due.to.second.dimension.var.index(%struct.2* %start, i8* %high, i32 %idx) {
+; CHECK-LABEL: @ptr.int.struct.test.ult.unknown.due.to.second.dimension.var.index(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1:%.*]] = add nuw nsw i32 [[IDX:%.*]], 1
+; CHECK-NEXT:    [[IDX_EXT_PLUS_1_EXT:%.*]] = zext i32 [[IDX_EXT_PLUS_1]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_2:%.*]], %struct.2* [[START:%.*]], i64 6, i32 1, i64 [[IDX_EXT_PLUS_1_EXT]]
+; CHECK-NEXT:    [[ADD_PTR_CAST:%.*]] = bitcast i64* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[ADD_PTR_CAST]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [[STRUCT_2]], %struct.2* [[START]], i64 6, i32 2
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
+; CHECK-NEXT:    ret i1 [[C_0]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %idx.ext.plus.1 = add nuw nsw i32 %idx, 1
+  %idx.ext.plus.1.ext = zext i32 %idx.ext.plus.1 to i64
+  %add.ptr = getelementptr inbounds %struct.2, %struct.2* %start, i64 6, i32 1, i64 %idx.ext.plus.1.ext
+  %add.ptr.cast = bitcast i64* %add.ptr to i8*
+  %c.1 = icmp ule i8* %add.ptr.cast, %high
+  br i1 %c.1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %idx.ext = zext i32 %idx to i64
+  %start.0 = getelementptr inbounds %struct.2, %struct.2* %start, i64 6, i32 2
+  %c.0 = icmp ult i8* %start.0, %high
+  ret i1 %c.0
+
+if.end:                                           ; preds = %entry
+  ret i1 1
+}
diff --git a/llvm/test/Transforms/ConstraintElimination/geps-precondition-overflow-check.ll b/llvm/test/Transforms/ConstraintElimination/geps-precondition-overflow-check.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9f4e98f738d0e7d9e1273deb0436b177f7cb68a3
--- /dev/null
+++ b/llvm/test/Transforms/ConstraintElimination/geps-precondition-overflow-check.ll
@@ -0,0 +1,333 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -constraint-elimination -S %s | FileCheck %s
+
+; Tests for cases with explicit checks that %ptr + x >= %ptr. The information can
+; be used to determine that certain GEPs do not overflow.
+
+define i1 @overflow_check_1(i32* %dst) {
+; CHECK-LABEL: @overflow_check_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DST_5:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 5
+; CHECK-NEXT:    [[DST_5_UGE:%.*]] = icmp uge i32* [[DST_5]], [[DST]]
+; CHECK-NEXT:    br i1 [[DST_5_UGE]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[DST_4:%.*]] = getelementptr i32, i32* [[DST]], i64 4
+; CHECK-NEXT:    [[TRUE_DST_4_UGE:%.*]] = icmp uge i32* [[DST_4]], [[DST]]
+; CHECK-NEXT:    ret i1 [[TRUE_DST_4_UGE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %dst.5 = getelementptr i32, i32* %dst, i64 5
+  %dst.5.uge = icmp uge i32* %dst.5, %dst
+  br i1 %dst.5.uge, label %then, label %else
+
+then:
+  %dst.4 = getelementptr i32, i32* %dst, i64 4
+  %true.dst.4.uge = icmp uge i32* %dst.4, %dst
+  ret i1 %true.dst.4.uge
+
+else:
+  ret i1 0
+}
+
+define i1 @overflow_check_2_and(i32* %dst) {
+; CHECK-LABEL: @overflow_check_2_and(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DST_5:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 5
+; CHECK-NEXT:    [[DST_5_UGE:%.*]] = icmp uge i32* [[DST_5]], [[DST]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[DST_5_UGE]], [[DST_5_UGE]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[DST_4:%.*]] = getelementptr i32, i32* [[DST]], i64 4
+; CHECK-NEXT:    [[TRUE_DST_4_UGE:%.*]] = icmp uge i32* [[DST_4]], [[DST]]
+; CHECK-NEXT:    ret i1 [[TRUE_DST_4_UGE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %dst.5 = getelementptr i32, i32* %dst, i64 5
+  %dst.5.uge = icmp uge i32* %dst.5, %dst
+  %and = and i1 %dst.5.uge, %dst.5.uge
+  br i1 %and, label %then, label %else
+
+then:
+  %dst.4 = getelementptr i32, i32* %dst, i64 4
+  %true.dst.4.uge = icmp uge i32* %dst.4, %dst
+  ret i1 %true.dst.4.uge
+
+else:
+  ret i1 true
+}
+
+define i1 @overflow_check_3_and(i32* %dst) {
+; CHECK-LABEL: @overflow_check_3_and(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DST_5:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 5
+; CHECK-NEXT:    [[DST_5_UGE:%.*]] = icmp uge i32* [[DST_5]], [[DST]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[DST_5_UGE]], [[DST_5_UGE]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[DST_4:%.*]] = getelementptr i32, i32* [[DST]], i64 4
+; CHECK-NEXT:    [[DST_4_UGE:%.*]] = icmp uge i32* [[DST_4]], [[DST]]
+; CHECK-NEXT:    ret i1 [[DST_4_UGE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[ELSE_DST_4:%.*]] = getelementptr i32, i32* [[DST]], i64 4
+; CHECK-NEXT:    [[ELSE_DST_4_UGE:%.*]] = icmp uge i32* [[ELSE_DST_4]], [[DST]]
+; CHECK-NEXT:    ret i1 [[ELSE_DST_4_UGE]]
+;
+entry:
+  %dst.5 = getelementptr i32, i32* %dst, i64 5
+  %dst.5.uge = icmp uge i32* %dst.5, %dst
+  %and = and i1 %dst.5.uge, %dst.5.uge
+  br i1 %and, label %then, label %else
+
+then:
+  %dst.4 = getelementptr i32, i32* %dst, i64 4
+  %dst.4.uge = icmp uge i32* %dst.4, %dst
+  ret i1 %dst.4.uge
+
+else:
+  %else.dst.4 = getelementptr i32, i32* %dst, i64 4
+  %else.dst.4.uge = icmp uge i32* %else.dst.4, %dst
+  ret i1 %else.dst.4.uge
+}
+
+define i1 @overflow_check_4_and(i32* %dst) {
+; CHECK-LABEL: @overflow_check_4_and(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DST_5:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 5
+; CHECK-NEXT:    [[DST_5_UGE:%.*]] = icmp uge i32* [[DST_5]], [[DST]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[DST_5_UGE]], [[DST_5_UGE]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[DST_4:%.*]] = getelementptr i32, i32* [[DST]], i64 4
+; CHECK-NEXT:    [[TRUE_DST_4_UGE:%.*]] = icmp uge i32* [[DST_4]], [[DST]]
+; CHECK-NEXT:    [[DST_5_2:%.*]] = getelementptr i32, i32* [[DST]], i64 5
+; CHECK-NEXT:    [[TRUE_DST_5_UGE:%.*]] = icmp uge i32* [[DST_5_2]], [[DST]]
+; CHECK-NEXT:    [[RES_0:%.*]] = xor i1 [[TRUE_DST_4_UGE]], [[TRUE_DST_5_UGE]]
+; CHECK-NEXT:    [[DST_6:%.*]] = getelementptr i32, i32* [[DST]], i64 6
+; CHECK-NEXT:    [[C_DST_6_UGE:%.*]] = icmp uge i32* [[DST_6]], [[DST]]
+; CHECK-NEXT:    [[RES_1:%.*]] = xor i1 [[RES_0]], [[C_DST_6_UGE]]
+; CHECK-NEXT:    ret i1 [[RES_1]]
+; CHECK:       else:
+; CHECK-NEXT:    [[ELSE_DST_4:%.*]] = getelementptr i32, i32* [[DST]], i64 4
+; CHECK-NEXT:    [[ELSE_DST_4_UGE:%.*]] = icmp uge i32* [[ELSE_DST_4]], [[DST]]
+; CHECK-NEXT:    [[ELSE_DST_6:%.*]] = getelementptr i32, i32* [[DST]], i64 6
+; CHECK-NEXT:    [[ELSE_DST_6_UGE:%.*]] = icmp uge i32* [[ELSE_DST_6]], [[DST]]
+; CHECK-NEXT:    [[ELSE_RES_0:%.*]] = xor i1 [[ELSE_DST_4_UGE]], [[ELSE_DST_6_UGE]]
+; CHECK-NEXT:    ret i1 [[ELSE_RES_0]]
+;
+entry:
+  %dst.5 = getelementptr i32, i32* %dst, i64 5
+  %dst.5.uge = icmp uge i32* %dst.5, %dst
+  %and = and i1 %dst.5.uge, %dst.5.uge
+  br i1 %and, label %then, label %else
+
+then:
+  %dst.4 = getelementptr i32, i32* %dst, i64 4
+  %true.dst.4.uge = icmp uge i32* %dst.4, %dst
+  %dst.5.2 = getelementptr i32, i32* %dst, i64 5
+  %true.dst.5.uge = icmp uge i32* %dst.5.2, %dst
+  %res.0 = xor i1 %true.dst.4.uge, %true.dst.5.uge
+
+  %dst.6 = getelementptr i32, i32* %dst, i64 6
+  %c.dst.6.uge = icmp uge i32* %dst.6, %dst
+  %res.1 = xor i1 %res.0, %c.dst.6.uge
+
+  ret i1 %res.1
+
+else:
+  %else.dst.4 = getelementptr i32, i32* %dst, i64 4
+  %else.dst.4.uge = icmp uge i32* %else.dst.4, %dst
+  %else.dst.6 = getelementptr i32, i32* %dst, i64 6
+  %else.dst.6.uge = icmp uge i32* %else.dst.6, %dst
+  %else.res.0 = xor i1 %else.dst.4.uge, %else.dst.6.uge
+
+  ret i1 %else.res.0
+}
+
+define i1 @overflow_check_3_or(i32* %dst) {
+; CHECK-LABEL: @overflow_check_3_or(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DST_5:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 5
+; CHECK-NEXT:    [[DST_5_UGE:%.*]] = icmp uge i32* [[DST_5]], [[DST]]
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[DST_5_UGE]], [[DST_5_UGE]]
+; CHECK-NEXT:    br i1 [[OR]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[DST_4:%.*]] = getelementptr i32, i32* [[DST]], i64 4
+; CHECK-NEXT:    [[TRUE_DST_4_UGE:%.*]] = icmp uge i32* [[DST_4]], [[DST]]
+; CHECK-NEXT:    ret i1 [[TRUE_DST_4_UGE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %dst.5 = getelementptr i32, i32* %dst, i64 5
+  %dst.5.uge = icmp uge i32* %dst.5, %dst
+  %or = or i1 %dst.5.uge, %dst.5.uge
+  br i1 %or, label %then, label %else
+
+then:
+  %dst.4 = getelementptr i32, i32* %dst, i64 4
+  %true.dst.4.uge = icmp uge i32* %dst.4, %dst
+  ret i1 %true.dst.4.uge
+
+else:
+  ret i1 0
+}
+
+define i1 @upper_and_lower_checks_1(i32* %dst, i32 %n) {
+; CHECK-LABEL: @upper_and_lower_checks_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[UPPER:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 [[N_EXT]]
+; CHECK-NEXT:    [[DST_5:%.*]] = getelementptr i32, i32* [[DST]], i64 5
+; CHECK-NEXT:    [[DST_5_ULT:%.*]] = icmp ult i32* [[DST_5]], [[UPPER]]
+; CHECK-NEXT:    [[DST_5_UGE:%.*]] = icmp uge i32* [[DST_5]], [[DST]]
+; CHECK-NEXT:    [[AND_1:%.*]] = and i1 [[DST_5_ULT]], [[DST_5_UGE]]
+; CHECK-NEXT:    br i1 [[AND_1]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[DST_4:%.*]] = getelementptr i32, i32* [[DST]], i64 4
+; CHECK-NEXT:    [[TRUE_DST_4_ULT:%.*]] = icmp ult i32* [[DST_4]], [[UPPER]]
+; CHECK-NEXT:    [[TRUE_DST_4_UGE:%.*]] = icmp uge i32* [[DST_4]], [[DST]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[TRUE_DST_4_ULT]], [[TRUE_DST_4_UGE]]
+; CHECK-NEXT:    ret i1 [[AND]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %upper = getelementptr inbounds i32, i32* %dst, i64 %n.ext
+  %dst.5 = getelementptr i32, i32* %dst, i64 5
+  %dst.5.ult = icmp ult i32* %dst.5, %upper
+  %dst.5.uge = icmp uge i32* %dst.5, %dst
+  %and.1 = and i1 %dst.5.ult, %dst.5.uge
+  br i1 %and.1, label %then, label %else
+
+then:
+  %dst.4 = getelementptr i32, i32* %dst, i64 4
+  %true.dst.4.ult = icmp ult i32* %dst.4, %upper
+  %true.dst.4.uge = icmp uge i32* %dst.4, %dst
+  %and = and i1 %true.dst.4.ult, %true.dst.4.uge
+  ret i1 %and
+
+else:
+  ret i1 0
+}
+
+define i1 @upper_and_lower_checks_2_dst6(i32* %dst, i32 %n) {
+; CHECK-LABEL: @upper_and_lower_checks_2_dst6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[UPPER:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 [[N_EXT]]
+; CHECK-NEXT:    [[DST_5:%.*]] = getelementptr i32, i32* [[DST]], i64 5
+; CHECK-NEXT:    [[DST_5_ULT:%.*]] = icmp ult i32* [[DST_5]], [[UPPER]]
+; CHECK-NEXT:    [[DST_5_UGE:%.*]] = icmp uge i32* [[DST_5]], [[DST]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[DST_5_ULT]], [[DST_5_UGE]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[DST_6:%.*]] = getelementptr i32, i32* [[DST]], i64 6
+; CHECK-NEXT:    [[C_DST_6_ULT:%.*]] = icmp ult i32* [[DST_6]], [[UPPER]]
+; CHECK-NEXT:    [[TRUE_DST_6_UGE:%.*]] = icmp uge i32* [[DST_6]], [[DST]]
+; CHECK-NEXT:    [[RES:%.*]] = and i1 [[C_DST_6_ULT]], [[TRUE_DST_6_UGE]]
+; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %upper = getelementptr inbounds i32, i32* %dst, i64 %n.ext
+  %dst.5 = getelementptr i32, i32* %dst, i64 5
+  %dst.5.ult = icmp ult i32* %dst.5, %upper
+  %dst.5.uge = icmp uge i32* %dst.5, %dst
+  %and = and i1 %dst.5.ult, %dst.5.uge
+  br i1 %and, label %then, label %else
+
+then:
+  %dst.6 = getelementptr i32, i32* %dst, i64 6
+  %c.dst.6.ult = icmp ult i32* %dst.6, %upper
+  %true.dst.6.uge = icmp uge i32* %dst.6, %dst
+  %res = and i1 %c.dst.6.ult, %true.dst.6.uge
+  ret i1 %res
+
+else:
+  ret i1 0
+}
+
+define i1 @upper_and_lower_checks_2_dst7(i32* %dst, i32 %n) {
+; CHECK-LABEL: @upper_and_lower_checks_2_dst7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[UPPER:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 [[N_EXT]]
+; CHECK-NEXT:    [[DST_5:%.*]] = getelementptr i32, i32* [[DST]], i64 5
+; CHECK-NEXT:    [[DST_5_ULT:%.*]] = icmp ult i32* [[DST_5]], [[UPPER]]
+; CHECK-NEXT:    [[DST_5_UGE:%.*]] = icmp uge i32* [[DST_5]], [[DST]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[DST_5_ULT]], [[DST_5_UGE]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[DST_7:%.*]] = getelementptr i32, i32* [[DST]], i64 7
+; CHECK-NEXT:    [[C_DST_7_ULT:%.*]] = icmp ult i32* [[DST_7]], [[UPPER]]
+; CHECK-NEXT:    [[C_DST_7_UGE:%.*]] = icmp uge i32* [[DST_7]], [[DST]]
+; CHECK-NEXT:    [[RES:%.*]] = and i1 [[C_DST_7_ULT]], [[C_DST_7_UGE]]
+; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %upper = getelementptr inbounds i32, i32* %dst, i64 %n.ext
+  %dst.5 = getelementptr i32, i32* %dst, i64 5
+  %dst.5.ult = icmp ult i32* %dst.5, %upper
+  %dst.5.uge = icmp uge i32* %dst.5, %dst
+  %or.cond = and i1 %dst.5.ult, %dst.5.uge
+  br i1 %or.cond, label %then, label %else
+
+then:
+  %dst.7 = getelementptr i32, i32* %dst, i64 7
+  %c.dst.7.ult = icmp ult i32* %dst.7, %upper
+  %c.dst.7.uge = icmp uge i32* %dst.7, %dst
+  %res = and i1 %c.dst.7.ult, %c.dst.7.uge
+  ret i1 %res
+
+else:
+  ret i1 0
+}
+
+define i1 @upper_and_lower_checks_lt(i32* %dst, i32 %n) {
+; CHECK-LABEL: @upper_and_lower_checks_lt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[DST_5:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[N_EXT]]
+; CHECK-NEXT:    [[DST_5_UGE:%.*]] = icmp uge i32* [[DST_5]], [[DST]]
+; CHECK-NEXT:    [[N_EXT_UGE:%.*]] = icmp uge i64 [[N_EXT]], 3
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[DST_5_UGE]], [[N_EXT_UGE]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[DST_3:%.*]] = getelementptr i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    [[TRUE_DST_3_UGE:%.*]] = icmp uge i32* [[DST_3]], [[DST]]
+; CHECK-NEXT:    [[DST_4:%.*]] = getelementptr i32, i32* [[DST]], i64 4
+; CHECK-NEXT:    [[C_DST_4_UGE:%.*]] = icmp uge i32* [[DST_4]], [[DST]]
+; CHECK-NEXT:    [[RES_0:%.*]] = xor i1 [[TRUE_DST_3_UGE]], [[C_DST_4_UGE]]
+; CHECK-NEXT:    ret i1 [[RES_0]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %n.ext = zext i32 %n to i64
+  %dst.5 = getelementptr i32, i32* %dst, i64 %n.ext
+  %dst.5.uge = icmp uge i32* %dst.5, %dst
+  %n.ext.uge = icmp uge i64 %n.ext, 3
+  %or.cond = and i1 %dst.5.uge, %n.ext.uge
+  br i1 %or.cond, label %then, label %else
+
+then:
+  %dst.3 = getelementptr i32, i32* %dst, i64 3
+  %true.dst.3.uge = icmp uge i32* %dst.3, %dst
+  %dst.4 = getelementptr i32, i32* %dst, i64 4
+  %c.dst.4.uge = icmp uge i32* %dst.4, %dst
+  %res.0 = xor i1 %true.dst.3.uge, %c.dst.4.uge
+  ret i1 %res.0
+
+else:
+  ret i1 0
+}
diff --git a/llvm/test/Transforms/ConstraintElimination/geps.2d.ll b/llvm/test/Transforms/ConstraintElimination/geps.2d.ll
deleted file mode 100644
index 35ffadbd85ea11e28fc5c695888da206fe193490..0000000000000000000000000000000000000000
--- a/llvm/test/Transforms/ConstraintElimination/geps.2d.ll
+++ /dev/null
@@ -1,134 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -constraint-elimination -S %s | FileCheck %s
-
-define void @test.not.uge.ult([10 x i8]* %start, i8* %low, i8* %high) {
-; CHECK-LABEL: @test.not.uge.ult(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 1, i64 3
-; CHECK-NEXT:    [[C_1:%.*]] = icmp uge i8* [[ADD_PTR_I]], [[HIGH:%.*]]
-; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    ret void
-; CHECK:       if.end:
-; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 10, i64 0
-; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i8* [[START_0]], [[HIGH]]
-; CHECK-NEXT:    call void @use(i1 [[C_0]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 1, i64 3
-  %c.1 = icmp uge i8* %add.ptr.i, %high
-  br i1 %c.1, label %if.then, label %if.end
-
-if.then:                                          ; preds = %entry
-  ret void
-
-if.end:                                           ; preds = %entry
-  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 10, i64 0
-  %c.0 = icmp ult i8* %start.0, %high
-  call void @use(i1 %c.0)
-  ret void
-}
-
-define void @test.not.uge.ule([10 x i8]* %start, i8* %low, i8* %high) {
-; CHECK-LABEL: @test.not.uge.ule(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 1, i64 3
-; CHECK-NEXT:    [[C:%.*]] = icmp uge i8* [[ADD_PTR_I]], [[HIGH:%.*]]
-; CHECK-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    ret void
-; CHECK:       if.end:
-; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 10, i64 0
-; CHECK-NEXT:    [[C_0:%.*]] = icmp ule i8* [[START_0]], [[HIGH]]
-; CHECK-NEXT:    call void @use(i1 [[C_0]])
-; CHECK-NEXT:    [[START_1:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 2, i64 1
-; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i8* [[START_1]], [[HIGH]]
-; CHECK-NEXT:    call void @use(i1 [[C_1]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 1, i64 3
-  %c = icmp uge i8* %add.ptr.i, %high
-  br i1 %c, label %if.then, label %if.end
-
-if.then:                                          ; preds = %entry
-  ret void
-
-if.end:                                           ; preds = %entry
-  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 10, i64 0
-  %c.0 = icmp ule i8* %start.0, %high
-  call void @use(i1 %c.0)
-  %start.1 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 2, i64 1
-  %c.1 = icmp ule i8* %start.1, %high
-  call void @use(i1 %c.1)
-  ret void
-}
-
-define void @test.not.uge.ugt([10 x i8]* %start, i8* %low, i8* %high) {
-; CHECK-LABEL: @test.not.uge.ugt(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 1, i64 3
-; CHECK-NEXT:    [[C:%.*]] = icmp uge i8* [[ADD_PTR_I]], [[HIGH:%.*]]
-; CHECK-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    ret void
-; CHECK:       if.end:
-; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 3, i64 0
-; CHECK-NEXT:    [[C_0:%.*]] = icmp ugt i8* [[START_0]], [[HIGH]]
-; CHECK-NEXT:    call void @use(i1 [[C_0]])
-; CHECK-NEXT:    [[START_1:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 3, i64 1
-; CHECK-NEXT:    [[C_1:%.*]] = icmp ugt i8* [[START_1]], [[HIGH]]
-; CHECK-NEXT:    call void @use(i1 [[C_1]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 1, i64 3
-  %c = icmp uge i8* %add.ptr.i, %high
-  br i1 %c, label %if.then, label %if.end
-
-if.then:                                          ; preds = %entry
-  ret void
-
-if.end:                                           ; preds = %entry
-  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 3, i64 0
-  %c.0 = icmp ugt i8* %start.0, %high
-  call void @use(i1 %c.0)
-
-  %start.1 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 3, i64 1
-  %c.1 = icmp ugt i8* %start.1, %high
-  call void @use(i1 %c.1)
-  ret void
-}
-
-define void @test.not.uge.uge([10 x i8]* %start, i8* %low, i8* %high) {
-; CHECK-LABEL: @test.not.uge.uge(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START:%.*]], i64 1, i64 3
-; CHECK-NEXT:    [[C_1:%.*]] = icmp uge i8* [[ADD_PTR_I]], [[HIGH:%.*]]
-; CHECK-NEXT:    br i1 [[C_1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    ret void
-; CHECK:       if.end:
-; CHECK-NEXT:    [[START_0:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[START]], i64 3, i64 0
-; CHECK-NEXT:    [[C_0:%.*]] = icmp uge i8* [[START_0]], [[HIGH]]
-; CHECK-NEXT:    call void @use(i1 [[C_0]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %add.ptr.i = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 1, i64 3
-  %c.1 = icmp uge i8* %add.ptr.i, %high
-  br i1 %c.1, label %if.then, label %if.end
-
-if.then:                                          ; preds = %entry
-  ret void
-
-if.end:                                           ; preds = %entry
-  %start.0 = getelementptr inbounds [10 x i8], [10 x i8]* %start, i64 3, i64 0
-  %c.0 = icmp uge i8* %start.0, %high
-  call void @use(i1 %c.0)
-
-  ret void
-}
-
-declare void @use(i1)
diff --git a/llvm/test/Transforms/ConstraintElimination/pointercast.ll b/llvm/test/Transforms/ConstraintElimination/pointercast.ll
index 83e245663cda764b8e36552c394ce5da56e95e79..acd0fbcdcf9bb1926a0630bd86e6611d5a1f92e6 100644
--- a/llvm/test/Transforms/ConstraintElimination/pointercast.ll
+++ b/llvm/test/Transforms/ConstraintElimination/pointercast.ll
@@ -2,7 +2,6 @@
 ; RUN: opt -constraint-elimination -S %s | FileCheck %s
 
 define i1 @bitcast_and_cmp(i32* readonly %src, i32* readnone %min, i32* readnone %max) {
-;
 ; CHECK-LABEL: @bitcast_and_cmp(
 ; CHECK-NEXT:  check.0.min:
 ; CHECK-NEXT:    [[SRC_C:%.*]] = bitcast i32* [[SRC:%.*]] to i8*
@@ -79,3 +78,159 @@ checks:
 
   ret i1 %res.7
 }
+
+define i1 @gep0_and_cmp(i32* readonly %src, i32* readnone %min, i32* readnone %max) {
+; CHECK-LABEL: @gep0_and_cmp(
+; CHECK-NEXT:  check.0.min:
+; CHECK-NEXT:    [[SRC_C:%.*]] = getelementptr i32, i32* [[SRC:%.*]], i64 0
+; CHECK-NEXT:    [[MIN_C:%.*]] = getelementptr i32, i32* [[MIN:%.*]], i64 0
+; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[GEP_3_C:%.*]] = getelementptr i32, i32* [[GEP_3]], i32 0
+; CHECK-NEXT:    [[C_MIN_0:%.*]] = icmp ult i32* [[SRC_C]], [[MIN_C]]
+; CHECK-NEXT:    [[C_MAX_3:%.*]] = icmp ugt i32* [[GEP_3_C]], [[MAX:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[C_MIN_0]], [[C_MAX_3]]
+; CHECK-NEXT:    br i1 [[OR]], label [[TRAP:%.*]], label [[CHECKS:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       checks:
+; CHECK-NEXT:    [[C_3_MIN:%.*]] = icmp ult i32* [[GEP_3]], [[MIN]]
+; CHECK-NEXT:    [[C_3_MAX:%.*]] = icmp ult i32* [[GEP_3]], [[MAX]]
+; CHECK-NEXT:    [[RES_1:%.*]] = xor i1 false, [[C_3_MAX]]
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 1
+; CHECK-NEXT:    [[C_1_MIN:%.*]] = icmp ult i32* [[GEP_1]], [[MIN]]
+; CHECK-NEXT:    [[C_1_MAX:%.*]] = icmp ult i32* [[GEP_1]], [[MAX]]
+; CHECK-NEXT:    [[RES_2:%.*]] = xor i1 false, true
+; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[C_2_MIN:%.*]] = icmp ult i32* [[GEP_2]], [[MIN]]
+; CHECK-NEXT:    [[C_2_MAX:%.*]] = icmp ult i32* [[GEP_2]], [[MAX]]
+; CHECK-NEXT:    [[RES_3:%.*]] = xor i1 false, true
+; CHECK-NEXT:    [[GEP_4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 4
+; CHECK-NEXT:    [[C_4_MIN:%.*]] = icmp ult i32* [[GEP_4]], [[MIN]]
+; CHECK-NEXT:    [[C_4_MAX:%.*]] = icmp ult i32* [[GEP_4]], [[MAX]]
+; CHECK-NEXT:    [[RES_4:%.*]] = xor i1 false, [[C_4_MAX]]
+; CHECK-NEXT:    [[RES_5:%.*]] = xor i1 [[RES_1]], [[RES_2]]
+; CHECK-NEXT:    [[RES_6:%.*]] = xor i1 [[RES_5]], [[RES_3]]
+; CHECK-NEXT:    [[RES_7:%.*]] = xor i1 [[RES_6]], [[RES_4]]
+; CHECK-NEXT:    ret i1 [[RES_7]]
+;
+check.0.min:
+  %src.c = getelementptr i32, i32* %src, i64 0
+  %min.c = getelementptr i32, i32* %min, i64 0
+
+  %gep.3 = getelementptr inbounds i32, i32* %src, i64 3
+  %gep.3.c = getelementptr i32, i32* %gep.3, i32 0
+  %c.min.0 = icmp ult i32* %src.c, %min.c
+  %c.max.3 = icmp ugt i32* %gep.3.c, %max
+
+  %or = or i1 %c.min.0, %c.max.3
+  br i1 %or, label %trap, label %checks
+
+trap:
+  ret i1 0
+
+checks:
+  %c.3.min = icmp ult i32* %gep.3, %min
+  %c.3.max = icmp ult i32* %gep.3, %max
+  %res.1 = xor i1 %c.3.min, %c.3.max
+
+  %gep.1 = getelementptr inbounds i32, i32* %src, i64 1
+  %c.1.min = icmp ult i32* %gep.1, %min
+  %c.1.max = icmp ult i32* %gep.1, %max
+  %res.2 = xor i1 %c.1.min, %c.1.max
+
+  %gep.2 = getelementptr inbounds i32, i32* %src, i64 2
+  %c.2.min = icmp ult i32* %gep.2, %min
+  %c.2.max = icmp ult i32* %gep.2, %max
+  %res.3 = xor i1 %c.2.min, %c.2.max
+
+  %gep.4 = getelementptr inbounds i32, i32* %src, i64 4
+  %c.4.min = icmp ult i32* %gep.4, %min
+  %c.4.max = icmp ult i32* %gep.4, %max
+  %res.4 = xor i1 %c.4.min, %c.4.max
+
+  %res.5 = xor i1 %res.1, %res.2
+  %res.6 = xor i1 %res.5, %res.3
+  %res.7 = xor i1 %res.6, %res.4
+
+  ret i1 %res.7
+}
+
+; Should not look through addresspacecast, because it may change the pointer
+; value.
+define i1 @addrspacecast_and_cmp(i32* readonly %src, i32* readnone %min, i32* readnone %max) {
+; CHECK-LABEL: @addrspacecast_and_cmp(
+; CHECK-NEXT:  check.0.min:
+; CHECK-NEXT:    [[SRC_C:%.*]] = addrspacecast i32* [[SRC:%.*]] to i8 addrspace(1)*
+; CHECK-NEXT:    [[MIN_C:%.*]] = addrspacecast i32* [[MIN:%.*]] to i8 addrspace(1)*
+; CHECK-NEXT:    [[MAX_C:%.*]] = addrspacecast i32* [[MAX:%.*]] to i16 addrspace(1)*
+; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[GEP_3_C:%.*]] = addrspacecast i32* [[GEP_3]] to i16 addrspace(1)*
+; CHECK-NEXT:    [[C_MIN_0:%.*]] = icmp ult i8 addrspace(1)* [[SRC_C]], [[MIN_C]]
+; CHECK-NEXT:    [[C_MAX_3:%.*]] = icmp ugt i16 addrspace(1)* [[GEP_3_C]], [[MAX_C]]
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[C_MIN_0]], [[C_MAX_3]]
+; CHECK-NEXT:    br i1 [[OR]], label [[TRAP:%.*]], label [[CHECKS:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       checks:
+; CHECK-NEXT:    [[C_3_MIN:%.*]] = icmp ult i32* [[GEP_3]], [[MIN]]
+; CHECK-NEXT:    [[C_3_MAX:%.*]] = icmp ult i32* [[GEP_3]], [[MAX]]
+; CHECK-NEXT:    [[RES_1:%.*]] = xor i1 [[C_3_MIN]], [[C_3_MAX]]
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 1
+; CHECK-NEXT:    [[C_1_MIN:%.*]] = icmp ult i32* [[GEP_1]], [[MIN]]
+; CHECK-NEXT:    [[C_1_MAX:%.*]] = icmp ult i32* [[GEP_1]], [[MAX]]
+; CHECK-NEXT:    [[RES_2:%.*]] = xor i1 [[C_1_MIN]], [[C_1_MAX]]
+; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[C_2_MIN:%.*]] = icmp ult i32* [[GEP_2]], [[MIN]]
+; CHECK-NEXT:    [[C_2_MAX:%.*]] = icmp ult i32* [[GEP_2]], [[MAX]]
+; CHECK-NEXT:    [[RES_3:%.*]] = xor i1 [[C_2_MIN]], [[C_2_MAX]]
+; CHECK-NEXT:    [[GEP_4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 4
+; CHECK-NEXT:    [[C_4_MIN:%.*]] = icmp ult i32* [[GEP_4]], [[MIN]]
+; CHECK-NEXT:    [[C_4_MAX:%.*]] = icmp ult i32* [[GEP_4]], [[MAX]]
+; CHECK-NEXT:    [[RES_4:%.*]] = xor i1 [[C_4_MIN]], [[C_4_MAX]]
+; CHECK-NEXT:    [[RES_5:%.*]] = xor i1 [[RES_1]], [[RES_2]]
+; CHECK-NEXT:    [[RES_6:%.*]] = xor i1 [[RES_5]], [[RES_3]]
+; CHECK-NEXT:    [[RES_7:%.*]] = xor i1 [[RES_6]], [[RES_4]]
+; CHECK-NEXT:    ret i1 [[RES_7]]
+;
+check.0.min:
+  %src.c = addrspacecast i32* %src to i8 addrspace(1)*
+  %min.c = addrspacecast i32* %min to i8 addrspace(1)*
+  %max.c = addrspacecast i32* %max to i16 addrspace(1)*
+
+  %gep.3 = getelementptr inbounds i32, i32* %src, i64 3
+  %gep.3.c = addrspacecast i32* %gep.3 to i16 addrspace(1) *
+  %c.min.0 = icmp ult i8 addrspace(1)* %src.c, %min.c
+  %c.max.3 = icmp ugt i16 addrspace(1)* %gep.3.c, %max.c
+
+  %or = or i1 %c.min.0, %c.max.3
+  br i1 %or, label %trap, label %checks
+
+trap:
+  ret i1 0
+
+checks:
+  %c.3.min = icmp ult i32* %gep.3, %min
+  %c.3.max = icmp ult i32* %gep.3, %max
+  %res.1 = xor i1 %c.3.min, %c.3.max
+
+  %gep.1 = getelementptr inbounds i32, i32* %src, i64 1
+  %c.1.min = icmp ult i32* %gep.1, %min
+  %c.1.max = icmp ult i32* %gep.1, %max
+  %res.2 = xor i1 %c.1.min, %c.1.max
+
+  %gep.2 = getelementptr inbounds i32, i32* %src, i64 2
+  %c.2.min = icmp ult i32* %gep.2, %min
+  %c.2.max = icmp ult i32* %gep.2, %max
+  %res.3 = xor i1 %c.2.min, %c.2.max
+
+  %gep.4 = getelementptr inbounds i32, i32* %src, i64 4
+  %c.4.min = icmp ult i32* %gep.4, %min
+  %c.4.max = icmp ult i32* %gep.4, %max
+  %res.4 = xor i1 %c.4.min, %c.4.max
+
+  %res.5 = xor i1 %res.1, %res.2
+  %res.6 = xor i1 %res.5, %res.3
+  %res.7 = xor i1 %res.6, %res.4
+
+  ret i1 %res.7
+}
diff --git a/llvm/test/Transforms/DeadArgElim/NoundefAttrs.ll b/llvm/test/Transforms/DeadArgElim/NoundefAttrs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..576f972fc87a8dc0e85bfa5d4e68ca0811d0701d
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/NoundefAttrs.ll
@@ -0,0 +1,18 @@
+; RUN: opt -deadargelim -S < %s | FileCheck %s
+
+; If caller is changed to pass in undef, noundef and related attributes
+; should be deleted.
+
+
+; CHECK:   define i64 @bar(i64* %0, i64 %1)
+define i64 @bar(i64* nonnull dereferenceable(8) %0, i64 %1) {
+entry:
+  %2 = add i64 %1, 8
+  ret i64 %2
+}
+
+define i64 @foo(i64* %p, i64 %v) {
+; CHECK:   %retval = call i64 @bar(i64* undef, i64 %v)
+  %retval = call i64 @bar(i64* nonnull dereferenceable(8) %p, i64 %v)
+  ret i64 %retval
+}
diff --git a/llvm/test/Transforms/DeadArgElim/keepalive.ll b/llvm/test/Transforms/DeadArgElim/keepalive.ll
index fff14a7f52a8520362a3ba8f8043628ec1f40865..b9be83e5cdf94c8c99bc5de331471a5c3ae61377 100644
--- a/llvm/test/Transforms/DeadArgElim/keepalive.ll
+++ b/llvm/test/Transforms/DeadArgElim/keepalive.ll
@@ -39,13 +39,13 @@ define void @caller() {
 
 ; We can't remove 'this' here, as that would put argmem in ecx instead of
 ; memory.
-define internal x86_thiscallcc i32 @unused_this(i32* %this, i32* inalloca %argmem) {
+define internal x86_thiscallcc i32 @unused_this(i32* %this, i32* inalloca(i32) %argmem) {
 ;
 ;
   %v = load i32, i32* %argmem
   ret i32 %v
 }
-; CHECK-LABEL: define internal x86_thiscallcc i32 @unused_this(i32* %this, i32* inalloca %argmem)
+; CHECK-LABEL: define internal x86_thiscallcc i32 @unused_this(i32* %this, i32* inalloca(i32) %argmem)
 
 define i32 @caller2() {
 ;
@@ -53,7 +53,7 @@ define i32 @caller2() {
   %t = alloca i32
   %m = alloca inalloca i32
   store i32 42, i32* %m
-  %v = call x86_thiscallcc i32 @unused_this(i32* %t, i32* inalloca %m)
+  %v = call x86_thiscallcc i32 @unused_this(i32* %t, i32* inalloca(i32) %m)
   ret i32 %v
 }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/simple.ll b/llvm/test/Transforms/DeadStoreElimination/simple.ll
index 48a939c1228f9be47e97543d046580ba71b0d08c..361b243f5121012487d8630316b1761e49d2c3f5 100644
--- a/llvm/test/Transforms/DeadStoreElimination/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/simple.ll
@@ -148,7 +148,7 @@ define void @test9(%struct.x* byval(%struct.x)  %a) nounwind  {
 }
 
 ; Test for inalloca handling.
-define void @test9_2(%struct.x* inalloca  %a) nounwind  {
+define void @test9_2(%struct.x* inalloca(%struct.x) %a) nounwind {
 ; CHECK-LABEL: @test9_2(
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/EarlyCSE/gc_relocate.ll b/llvm/test/Transforms/EarlyCSE/gc_relocate.ll
index df32d3f85b1e84914e05d18ae896b388aa3a1fe3..ae9001c9db66993f3b772f4fd7c6339695507954 100644
--- a/llvm/test/Transforms/EarlyCSE/gc_relocate.ll
+++ b/llvm/test/Transforms/EarlyCSE/gc_relocate.ll
@@ -30,11 +30,8 @@ define i1 @test_readnone(i32 addrspace(1)* %in) gc "statepoint-example" {
 ; CHECK-NEXT:    [[SAFEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[IN:%.*]]) ]
 ; CHECK-NEXT:    [[A:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    store i32 0, i32* @G, align 4
-; CHECK-NEXT:    [[B:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 addrspace(1)* [[A]], null
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 addrspace(1)* [[B]], null
-; CHECK-NEXT:    [[CMP:%.*]] = and i1 [[CMP1]], [[CMP2]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
 ;
 entry:
   %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %in)]
diff --git a/llvm/test/Transforms/FunctionAttrs/readattrs.ll b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
index ae34219bd011eaec967c42a5e9ad3a570e31c197..6585ee4d24482566adcafd2d1d6b5149902af154 100644
--- a/llvm/test/Transforms/FunctionAttrs/readattrs.ll
+++ b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
@@ -50,9 +50,9 @@ define void @test6_2(i8** %p, i8* %q) {
   ret void
 }
 
-; CHECK: define void @test7_1(i32* inalloca nocapture %a)
+; CHECK: define void @test7_1(i32* nocapture inalloca(i32) %a)
 ; inalloca parameters are always considered written
-define void @test7_1(i32* inalloca %a) {
+define void @test7_1(i32* inalloca(i32) %a) {
   ret void
 }
 
diff --git a/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll b/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
index 6d16587ed152879a2f3a3dcdf305b54d4d2ec2e0..d4cbda17b51d83a0ef479b5ce0eccdf2594eb023 100644
--- a/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
+++ b/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -basic-aa -gvn < %s | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin11.0.0"
@@ -5,14 +6,34 @@ target triple = "i386-apple-darwin11.0.0"
 @sortlist = external global [5001 x i32], align 4
 
 define void @Bubble() nounwind noinline {
-; CHECK: entry:
-; CHECK-NEXT: %tmp7.pre = load i32
+; CHECK-LABEL: @Bubble(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7_PRE:%.*]] = load i32, i32* getelementptr inbounds ([5001 x i32], [5001 x i32]* @sortlist, i32 0, i32 1), align 4
+; CHECK-NEXT:    br label [[WHILE_BODY5:%.*]]
+; CHECK:       while.body5:
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP7_PRE]], [[ENTRY:%.*]] ], [ [[TMP71:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP6:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[INDVAR]], 2
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr [5001 x i32], [5001 x i32]* @sortlist, i32 0, i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP6]] = add i32 [[INDVAR]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr [5001 x i32], [5001 x i32]* @sortlist, i32 0, i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[CMP11]], label [[IF_THEN:%.*]], label [[IF_END]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 [[TMP10]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    store i32 [[TMP7]], i32* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP71]] = phi i32 [ [[TMP7]], [[IF_THEN]] ], [ [[TMP10]], [[WHILE_BODY5]] ]
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP6]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY5]]
+; CHECK:       while.end.loopexit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %while.body5
 
-; CHECK: while.body5:
-; CHECK: %tmp7 = phi i32
-; CHECK-NOT: %tmp7 = load i32
 while.body5:
   %indvar = phi i32 [ 0, %entry ], [ %tmp6, %if.end ]
   %tmp5 = add i32 %indvar, 2
@@ -24,7 +45,6 @@ while.body5:
   %cmp11 = icmp sgt i32 %tmp7, %tmp10
   br i1 %cmp11, label %if.then, label %if.end
 
-; CHECK: if.then:
 if.then:
   store i32 %tmp10, i32* %arrayidx, align 4
   store i32 %tmp7, i32* %arrayidx9, align 4
@@ -43,14 +63,18 @@ declare void @clobber()
 
 ; This is a classic LICM case
 define i32 @test1(i1 %cnd, i32* %p) {
-; CHECK-LABEL: @test1
-entry: 
-; CHECK-LABEL: entry
-; CHECK-NEXT: %v1.pre = load i32, i32* %p
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V1_PRE:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    call void @hold(i32 [[V1_PRE]])
+; CHECK-NEXT:    br label [[HEADER]]
+;
+entry:
   br label %header
 
 header:
-; CHECK-LABEL: header
   %v1 = load i32, i32* %p
   call void @hold(i32 %v1)
   br label %header
@@ -62,14 +86,24 @@ header:
 ; the value is fully available across the backedge, we only need to establish
 ; anticipation for the preheader block (which is trivial in this case.)
 define i32 @test2(i1 %cnd, i32* %p) {
-; CHECK-LABEL: @test2
-entry: 
-; CHECK-LABEL: entry
-; CHECK-NEXT: %v1.pre = load i32, i32* %p
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V1_PRE:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    call void @hold(i32 [[V1_PRE]])
+; CHECK-NEXT:    br i1 [[CND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    br label [[HEADER]]
+;
+entry:
   br label %header
 
 header:
-; CHECK-LABEL: header
   %v1 = load i32, i32* %p
   call void @hold(i32 %v1)
   br i1 %cnd, label %bb1, label %bb2
@@ -90,9 +124,21 @@ merge:
 ; case could be solved through either a backwards anticipation walk or
 ; use of the "safe to speculate" status (if we annotate the param)
 define i32 @test3(i1 %cnd, i32* %p) {
-entry: 
-; CHECK-LABEL: @test3
-; CHECK-LABEL: entry
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    br i1 [[CND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    call void @hold(i32 [[V1]])
+; CHECK-NEXT:    br label [[HEADER]]
+;
+entry:
   br label %header
 
 header:
@@ -105,8 +151,6 @@ bb2:
   br label %merge
 
 merge:
-; CHECK-LABEL: merge
-; CHECK: load i32, i32* %p
   %v1 = load i32, i32* %p
   call void @hold(i32 %v1)
   br label %header
@@ -115,15 +159,28 @@ merge:
 ; Highlight that we can PRE into a latch block when there are multiple
 ; latches only one of which clobbers an otherwise invariant value.
 define i32 @test4(i1 %cnd, i32* %p) {
-; CHECK-LABEL: @test4
-entry: 
-; CHECK-LABEL: entry
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    call void @hold(i32 [[V1]])
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[V2:%.*]] = phi i32 [ [[V2_PRE:%.*]], [[BB2:%.*]] ], [ [[V2]], [[BB1:%.*]] ], [ [[V1]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    call void @hold(i32 [[V2]])
+; CHECK-NEXT:    br i1 [[CND:%.*]], label [[BB1]], label [[BB2]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    [[V2_PRE]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    br label [[HEADER]]
+;
+entry:
   %v1 = load i32, i32* %p
   call void @hold(i32 %v1)
   br label %header
 
 header:
-; CHECK-LABEL: header
   %v2 = load i32, i32* %p
   call void @hold(i32 %v2)
   br i1 %cnd, label %bb1, label %bb2
@@ -132,10 +189,6 @@ bb1:
   br label %header
 
 bb2:
-; CHECK-LABEL: bb2
-; CHECK:       call void @clobber()
-; CHECK-NEXT:  %v2.pre = load i32, i32* %p
-; CHECK-NEXT:  br label %header
 
   call void @clobber()
   br label %header
@@ -145,15 +198,31 @@ bb2:
 ; even in loop simplify form (though multiple applications of the same
 ; transformation).
 define i32 @test5(i1 %cnd, i32* %p) {
-; CHECK-LABEL: @test5
-entry: 
-; CHECK-LABEL: entry
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    call void @hold(i32 [[V1]])
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[V2_PRE2:%.*]] = phi i32 [ [[V2_PRE:%.*]], [[MERGE:%.*]] ], [ [[V1]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    call void @hold(i32 [[V2_PRE2]])
+; CHECK-NEXT:    br i1 [[CND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    [[V2_PRE_PRE:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[V2_PRE]] = phi i32 [ [[V2_PRE_PRE]], [[BB2]] ], [ [[V2_PRE2]], [[BB1]] ]
+; CHECK-NEXT:    br label [[HEADER]]
+;
+entry:
   %v1 = load i32, i32* %p
   call void @hold(i32 %v1)
   br label %header
 
 header:
-; CHECK-LABEL: header
   %v2 = load i32, i32* %p
   call void @hold(i32 %v2)
   br i1 %cnd, label %bb1, label %bb2
@@ -162,10 +231,6 @@ bb1:
   br label %merge
 
 bb2:
-; CHECK-LABEL: bb2
-; CHECK:       call void @clobber()
-; CHECK-NEXT:  %v2.pre.pre = load i32, i32* %p
-; CHECK-NEXT:  br label %merge
 
   call void @clobber()
   br label %merge
@@ -179,13 +244,19 @@ declare void @llvm.experimental.guard(i1 %cnd, ...)
 ; These two tests highlight speculation safety when we can not establish
 ; anticipation (since the original load might actually not execcute)
 define i32 @test6a(i1 %cnd, i32* %p) {
-entry: 
-; CHECK-LABEL: @test6a
+; CHECK-LABEL: @test6a(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CND:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    call void @hold(i32 [[V1]])
+; CHECK-NEXT:    br label [[HEADER]]
+;
+entry:
   br label %header
 
 header:
-; CHECK-LABEL: header
-; CHECK: load i32, i32* %p
   call void (i1, ...) @llvm.experimental.guard(i1 %cnd) ["deopt"()]
   %v1 = load i32, i32* %p
   call void @hold(i32 %v1)
@@ -193,13 +264,19 @@ header:
 }
 
 define i32 @test6b(i1 %cnd, i32* dereferenceable(8) align 4 %p) {
-entry: 
-; CHECK-LABEL: @test6b
-; CHECK: load i32, i32* %p
+; CHECK-LABEL: @test6b(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V1_PRE:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CND:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    call void @hold(i32 [[V1_PRE]])
+; CHECK-NEXT:    br label [[HEADER]]
+;
+entry:
   br label %header
 
 header:
-; CHECK-LABEL: header
   call void (i1, ...) @llvm.experimental.guard(i1 %cnd) ["deopt"()]
   %v1 = load i32, i32* %p
   call void @hold(i32 %v1)
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load.ll b/llvm/test/Transforms/GVN/PRE/pre-load.ll
index 831086d0f68a49ee73e8fb5cd1e04f53570a3190..161053d4fd8576a3de66465850b49333b257125b 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-load.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-load.ll
@@ -1,16 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basic-aa -gvn -enable-load-pre -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes="gvn<load-pre>" -enable-load-pre=false -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define i32 @test1(i32* %p, i1 %C) {
 ; CHECK-LABEL: @test1(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; CHECK:       block2:
+; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[BLOCK4:%.*]]
+; CHECK:       block3:
+; CHECK-NEXT:    store i32 0, i32* [[P]], align 4
+; CHECK-NEXT:    br label [[BLOCK4]]
+; CHECK:       block4:
+; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; CHECK-NEXT:    ret i32 [[PRE]]
+;
 block1:
-	br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
- br label %block4
-; CHECK: block2:
-; CHECK-NEXT: load i32, i32* %p
+  br label %block4
 
 block3:
   store i32 0, i32* %p
@@ -19,21 +30,29 @@ block3:
 block4:
   %PRE = load i32, i32* %p
   ret i32 %PRE
-; CHECK: block4:
-; CHECK-NEXT: phi i32
-; CHECK-NEXT: ret i32
 }
 
 ; This is a simple phi translation case.
 define i32 @test2(i32* %p, i32* %q, i1 %C) {
 ; CHECK-LABEL: @test2(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; CHECK:       block2:
+; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, i32* [[Q:%.*]], align 4
+; CHECK-NEXT:    br label [[BLOCK4:%.*]]
+; CHECK:       block3:
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[BLOCK4]]
+; CHECK:       block4:
+; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; CHECK-NEXT:    [[P2:%.*]] = phi i32* [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; CHECK-NEXT:    ret i32 [[PRE]]
+;
 block1:
-	br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
- br label %block4
-; CHECK: block2:
-; CHECK-NEXT: load i32, i32* %q
+  br label %block4
 
 block3:
   store i32 0, i32* %p
@@ -43,24 +62,35 @@ block4:
   %P2 = phi i32* [%p, %block3], [%q, %block2]
   %PRE = load i32, i32* %P2
   ret i32 %PRE
-; CHECK: block4:
-; CHECK-NEXT: phi i32 [
-; CHECK-NOT: load
-; CHECK: ret i32
 }
 
 ; This is a PRE case that requires phi translation through a GEP.
 define i32 @test3(i32* %p, i32* %q, i32** %Hack, i1 %C) {
 ; CHECK-LABEL: @test3(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, i32* [[Q:%.*]], i32 1
+; CHECK-NEXT:    store i32* [[B]], i32** [[HACK:%.*]], align 8
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; CHECK:       block2:
+; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, i32* [[B]], align 4
+; CHECK-NEXT:    br label [[BLOCK4:%.*]]
+; CHECK:       block3:
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, i32* [[P:%.*]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[A]], align 4
+; CHECK-NEXT:    br label [[BLOCK4]]
+; CHECK:       block4:
+; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; CHECK-NEXT:    [[P2:%.*]] = phi i32* [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr i32, i32* [[P2]], i32 1
+; CHECK-NEXT:    ret i32 [[PRE]]
+;
 block1:
   %B = getelementptr i32, i32* %q, i32 1
   store i32* %B, i32** %Hack
-	br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
- br label %block4
-; CHECK: block2:
-; CHECK-NEXT: load i32, i32* %B
+  br label %block4
 
 block3:
   %A = getelementptr i32, i32* %p, i32 1
@@ -72,24 +102,35 @@ block4:
   %P3 = getelementptr i32, i32* %P2, i32 1
   %PRE = load i32, i32* %P3
   ret i32 %PRE
-; CHECK: block4:
-; CHECK: phi i32 [
-; CHECK-NOT: load
-; CHECK: ret i32
 }
 
 ;; Here the loaded address is available, but the computation is in 'block3'
 ;; which does not dominate 'block2'.
 define i32 @test4(i32* %p, i32* %q, i32** %Hack, i1 %C) {
 ; CHECK-LABEL: @test4(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; CHECK:       block2:
+; CHECK-NEXT:    [[P3_PHI_TRANS_INSERT:%.*]] = getelementptr i32, i32* [[Q:%.*]], i32 1
+; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, i32* [[P3_PHI_TRANS_INSERT]], align 4
+; CHECK-NEXT:    br label [[BLOCK4:%.*]]
+; CHECK:       block3:
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, i32* [[Q]], i32 1
+; CHECK-NEXT:    store i32* [[B]], i32** [[HACK:%.*]], align 8
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, i32* [[P:%.*]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[A]], align 4
+; CHECK-NEXT:    br label [[BLOCK4]]
+; CHECK:       block4:
+; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; CHECK-NEXT:    [[P2:%.*]] = phi i32* [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr i32, i32* [[P2]], i32 1
+; CHECK-NEXT:    ret i32 [[PRE]]
+;
 block1:
-	br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
- br label %block4
-; CHECK: block2:
-; CHECK:   load i32, i32*
-; CHECK:   br label %block4
+  br label %block4
 
 block3:
   %B = getelementptr i32, i32* %q, i32 1
@@ -104,10 +145,6 @@ block4:
   %P3 = getelementptr i32, i32* %P2, i32 1
   %PRE = load i32, i32* %P3
   ret i32 %PRE
-; CHECK: block4:
-; CHECK: phi i32 [
-; CHECK-NOT: load
-; CHECK: ret i32
 }
 
 ;void test5(int N, double *G) {
@@ -118,38 +155,53 @@ block4:
 
 define void @test5(i32 %N, double* nocapture %G) nounwind ssp {
 ; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; CHECK:       bb.nph:
+; CHECK-NEXT:    [[TMP:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[DOTPRE:%.*]] = load double, double* [[G:%.*]], align 8
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP3:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[TMP6]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[G]], i64 [[TMP6]]
+; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, double* [[G]], i64 [[INDVAR]]
+; CHECK-NEXT:    [[TMP3]] = load double, double* [[SCEVGEP]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd double [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    store double [[TMP4]], double* [[SCEVGEP7]], align 8
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP6]], [[TMP]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
-  %0 = add i32 %N, -1           
-  %1 = icmp sgt i32 %0, 0       
+  %0 = add i32 %N, -1
+  %1 = icmp sgt i32 %0, 0
   br i1 %1, label %bb.nph, label %return
 
-bb.nph:                         
-  %tmp = zext i32 %0 to i64     
+bb.nph:
+  %tmp = zext i32 %0 to i64
   br label %bb
 
-; CHECK: bb.nph:
-; CHECK: load double, double*
-; CHECK: br label %bb
 
-bb:             
+bb:
   %indvar = phi i64 [ 0, %bb.nph ], [ %tmp6, %bb ]
-  %tmp6 = add i64 %indvar, 1                    
+  %tmp6 = add i64 %indvar, 1
   %scevgep = getelementptr double, double* %G, i64 %tmp6
   %scevgep7 = getelementptr double, double* %G, i64 %indvar
   %2 = load double, double* %scevgep7, align 8
-  %3 = load double, double* %scevgep, align 8 
-  %4 = fadd double %2, %3             
+  %3 = load double, double* %scevgep, align 8
+  %4 = fadd double %2, %3
   store double %4, double* %scevgep7, align 8
-  %exitcond = icmp eq i64 %tmp6, %tmp 
+  %exitcond = icmp eq i64 %tmp6, %tmp
   br i1 %exitcond, label %return, label %bb
 
 ; Should only be one load in the loop.
-; CHECK: bb:
-; CHECK: load double, double*
-; CHECK-NOT: load double, double*
-; CHECK: br i1 %exitcond
 
-return:                               
+return:
   ret void
 }
 
@@ -161,38 +213,53 @@ return:
 
 define void @test6(i32 %N, double* nocapture %G) nounwind ssp {
 ; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; CHECK:       bb.nph:
+; CHECK-NEXT:    [[TMP:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[DOTPRE:%.*]] = load double, double* [[G:%.*]], align 8
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP4:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[TMP6]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[G]], i64 [[TMP6]]
+; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, double* [[G]], i64 [[INDVAR]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, double* [[SCEVGEP]], align 8
+; CHECK-NEXT:    [[TMP4]] = fadd double [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    store double [[TMP4]], double* [[SCEVGEP]], align 8
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP6]], [[TMP]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
-  %0 = add i32 %N, -1           
-  %1 = icmp sgt i32 %0, 0       
+  %0 = add i32 %N, -1
+  %1 = icmp sgt i32 %0, 0
   br i1 %1, label %bb.nph, label %return
 
-bb.nph:                         
-  %tmp = zext i32 %0 to i64     
+bb.nph:
+  %tmp = zext i32 %0 to i64
   br label %bb
 
-; CHECK: bb.nph:
-; CHECK: load double, double*
-; CHECK: br label %bb
 
-bb:             
+bb:
   %indvar = phi i64 [ 0, %bb.nph ], [ %tmp6, %bb ]
-  %tmp6 = add i64 %indvar, 1                    
+  %tmp6 = add i64 %indvar, 1
   %scevgep = getelementptr double, double* %G, i64 %tmp6
   %scevgep7 = getelementptr double, double* %G, i64 %indvar
   %2 = load double, double* %scevgep7, align 8
-  %3 = load double, double* %scevgep, align 8 
-  %4 = fadd double %2, %3             
+  %3 = load double, double* %scevgep, align 8
+  %4 = fadd double %2, %3
   store double %4, double* %scevgep, align 8
-  %exitcond = icmp eq i64 %tmp6, %tmp 
+  %exitcond = icmp eq i64 %tmp6, %tmp
   br i1 %exitcond, label %return, label %bb
 
 ; Should only be one load in the loop.
-; CHECK: bb:
-; CHECK: load double, double*
-; CHECK-NOT: load double, double*
-; CHECK: br i1 %exitcond
 
-return:                               
+return:
   ret void
 }
 
@@ -205,38 +272,60 @@ return:
 
 ; This requires phi translation of the adds.
 define void @test7(i32 %N, double* nocapture %G) nounwind ssp {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, double* [[G:%.*]], i64 1
+; CHECK-NEXT:    store double 1.000000e+00, double* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; CHECK:       bb.nph:
+; CHECK-NEXT:    [[TMP:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP]], -1
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi double [ 1.000000e+00, [[BB_NPH]] ], [ [[TMP5:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP9:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDVAR]], 2
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[G]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, double* [[G]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load double, double* [[SCEVGEP]], align 8
+; CHECK-NEXT:    [[TMP5]] = fadd double [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    store double [[TMP5]], double* [[SCEVGEP]], align 8
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP9]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
-  %0 = getelementptr inbounds double, double* %G, i64 1   
+  %0 = getelementptr inbounds double, double* %G, i64 1
   store double 1.000000e+00, double* %0, align 8
-  %1 = add i32 %N, -1                             
-  %2 = icmp sgt i32 %1, 1                         
+  %1 = add i32 %N, -1
+  %2 = icmp sgt i32 %1, 1
   br i1 %2, label %bb.nph, label %return
 
-bb.nph:                                           
-  %tmp = sext i32 %1 to i64                       
-  %tmp7 = add i64 %tmp, -1                        
+bb.nph:
+  %tmp = sext i32 %1 to i64
+  %tmp7 = add i64 %tmp, -1
   br label %bb
 
-bb:                                               
-  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp9, %bb ] 
-  %tmp8 = add i64 %indvar, 2                      
-  %scevgep = getelementptr double, double* %G, i64 %tmp8  
-  %tmp9 = add i64 %indvar, 1                      
-  %scevgep10 = getelementptr double, double* %G, i64 %tmp9 
-  %3 = load double, double* %scevgep10, align 8           
-  %4 = load double, double* %scevgep, align 8             
-  %5 = fadd double %3, %4                         
+bb:
+  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp9, %bb ]
+  %tmp8 = add i64 %indvar, 2
+  %scevgep = getelementptr double, double* %G, i64 %tmp8
+  %tmp9 = add i64 %indvar, 1
+  %scevgep10 = getelementptr double, double* %G, i64 %tmp9
+  %3 = load double, double* %scevgep10, align 8
+  %4 = load double, double* %scevgep, align 8
+  %5 = fadd double %3, %4
   store double %5, double* %scevgep, align 8
-  %exitcond = icmp eq i64 %tmp9, %tmp7            
+  %exitcond = icmp eq i64 %tmp9, %tmp7
   br i1 %exitcond, label %return, label %bb
 
 ; Should only be one load in the loop.
-; CHECK: bb:
-; CHECK: load double, double*
-; CHECK-NOT: load double, double*
-; CHECK: br i1 %exitcond
 
-return:                                           
+return:
   ret void
 }
 
@@ -244,14 +333,27 @@ return:
 ;; GEP to be inserted into it.
 define i32 @test8(i32* %p, i32* %q, i32** %Hack, i1 %C) {
 ; CHECK-LABEL: @test8(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; CHECK:       block2:
+; CHECK-NEXT:    [[P3_PHI_TRANS_INSERT:%.*]] = getelementptr i32, i32* [[Q:%.*]], i32 1
+; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, i32* [[P3_PHI_TRANS_INSERT]], align 4
+; CHECK-NEXT:    br label [[BLOCK4:%.*]]
+; CHECK:       block3:
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, i32* [[P:%.*]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[A]], align 4
+; CHECK-NEXT:    br label [[BLOCK4]]
+; CHECK:       block4:
+; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; CHECK-NEXT:    [[P2:%.*]] = phi i32* [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr i32, i32* [[P2]], i32 1
+; CHECK-NEXT:    ret i32 [[PRE]]
+;
 block1:
-	br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
- br label %block4
-; CHECK: block2:
-; CHECK:   load i32, i32*
-; CHECK:   br label %block4
+  br label %block4
 
 block3:
   %A = getelementptr i32, i32* %p, i32 1
@@ -263,10 +365,6 @@ block4:
   %P3 = getelementptr i32, i32* %P2, i32 1
   %PRE = load i32, i32* %P3
   ret i32 %PRE
-; CHECK: block4:
-; CHECK: phi i32 [
-; CHECK-NOT: load
-; CHECK: ret i32
 }
 
 ;void test9(int N, double* G) {
@@ -277,41 +375,60 @@ block4:
 
 ; This requires phi translation of the adds.
 define void @test9(i32 %N, double* nocapture %G) nounwind ssp {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 1
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; CHECK:       bb.nph:
+; CHECK-NEXT:    [[TMP:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP]], -1
+; CHECK-NEXT:    [[SCEVGEP10_PHI_TRANS_INSERT:%.*]] = getelementptr double, double* [[G:%.*]], i64 1
+; CHECK-NEXT:    [[DOTPRE:%.*]] = load double, double* [[SCEVGEP10_PHI_TRANS_INSERT]], align 8
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP4:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP9:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDVAR]], 2
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[G]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, double* [[G]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, double* [[SCEVGEP]], align 8
+; CHECK-NEXT:    [[TMP4]] = fadd double [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    store double [[TMP4]], double* [[SCEVGEP]], align 8
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP9]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
   add i32 0, 0
-  %1 = add i32 %N, -1                             
-  %2 = icmp sgt i32 %1, 1                         
+  %1 = add i32 %N, -1
+  %2 = icmp sgt i32 %1, 1
   br i1 %2, label %bb.nph, label %return
 
-bb.nph:                                           
-  %tmp = sext i32 %1 to i64                       
-  %tmp7 = add i64 %tmp, -1                        
+bb.nph:
+  %tmp = sext i32 %1 to i64
+  %tmp7 = add i64 %tmp, -1
   br label %bb
 
-; CHECK: bb.nph:
-; CHECK:   load double, double*
-; CHECK:   br label %bb
-
-bb:                                               
-  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp9, %bb ] 
-  %tmp8 = add i64 %indvar, 2                      
-  %scevgep = getelementptr double, double* %G, i64 %tmp8  
-  %tmp9 = add i64 %indvar, 1                      
-  %scevgep10 = getelementptr double, double* %G, i64 %tmp9 
-  %3 = load double, double* %scevgep10, align 8           
-  %4 = load double, double* %scevgep, align 8             
-  %5 = fadd double %3, %4                         
+
+bb:
+  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp9, %bb ]
+  %tmp8 = add i64 %indvar, 2
+  %scevgep = getelementptr double, double* %G, i64 %tmp8
+  %tmp9 = add i64 %indvar, 1
+  %scevgep10 = getelementptr double, double* %G, i64 %tmp9
+  %3 = load double, double* %scevgep10, align 8
+  %4 = load double, double* %scevgep, align 8
+  %5 = fadd double %3, %4
   store double %5, double* %scevgep, align 8
-  %exitcond = icmp eq i64 %tmp9, %tmp7            
+  %exitcond = icmp eq i64 %tmp9, %tmp7
   br i1 %exitcond, label %return, label %bb
 
 ; Should only be one load in the loop.
-; CHECK: bb:
-; CHECK: load double, double*
-; CHECK-NOT: load double, double*
-; CHECK: br i1 %exitcond
 
-return:                                           
+return:
   ret void
 }
 
@@ -323,6 +440,36 @@ return:
 
 ; PR5501
 define void @test10(i32 %N, double* nocapture %G) nounwind ssp {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 1
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; CHECK:       bb.nph:
+; CHECK-NEXT:    [[TMP:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP]], -1
+; CHECK-NEXT:    [[SCEVGEP12_PHI_TRANS_INSERT:%.*]] = getelementptr double, double* [[G:%.*]], i64 1
+; CHECK-NEXT:    [[DOTPRE:%.*]] = load double, double* [[SCEVGEP12_PHI_TRANS_INSERT]], align 8
+; CHECK-NEXT:    [[DOTPRE1:%.*]] = load double, double* [[G]], align 8
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE1]], [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP4:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP11:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[G]], i64 [[INDVAR]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDVAR]], 2
+; CHECK-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, double* [[G]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[SCEVGEP12:%.*]] = getelementptr double, double* [[G]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP4]] = load double, double* [[SCEVGEP10]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd double [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6]] = fadd double [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    store double [[TMP6]], double* [[SCEVGEP12]], align 8
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = add i32 %N, -1
   %1 = icmp sgt i32 %0, 1
@@ -332,10 +479,6 @@ bb.nph:
   %tmp = sext i32 %0 to i64
   %tmp8 = add i64 %tmp, -1
   br label %bb
-; CHECK: bb.nph:
-; CHECK:   load double, double*
-; CHECK:   load double, double*
-; CHECK:   br label %bb
 
 
 bb:
@@ -355,10 +498,6 @@ bb:
   br i1 %exitcond, label %return, label %bb
 
 ; Should only be one load in the loop.
-; CHECK: bb:
-; CHECK: load double, double*
-; CHECK-NOT: load double, double*
-; CHECK: br i1 %exitcond
 
 return:
   ret void
@@ -367,14 +506,30 @@ return:
 ; Test critical edge splitting.
 define i32 @test11(i32* %p, i1 %C, i32 %N) {
 ; CHECK-LABEL: @test11(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; CHECK:       block2:
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[N:%.*]], 1
+; CHECK-NEXT:    br i1 [[COND]], label [[BLOCK2_BLOCK4_CRIT_EDGE:%.*]], label [[BLOCK5:%.*]]
+; CHECK:       block2.block4_crit_edge:
+; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[BLOCK4:%.*]]
+; CHECK:       block3:
+; CHECK-NEXT:    store i32 0, i32* [[P]], align 4
+; CHECK-NEXT:    br label [[BLOCK4]]
+; CHECK:       block4:
+; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ [[PRE_PRE]], [[BLOCK2_BLOCK4_CRIT_EDGE]] ], [ 0, [[BLOCK3]] ]
+; CHECK-NEXT:    br label [[BLOCK5]]
+; CHECK:       block5:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ 0, [[BLOCK2]] ], [ [[PRE]], [[BLOCK4]] ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
 block1:
-        br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
- %cond = icmp sgt i32 %N, 1
- br i1 %cond, label %block4, label %block5
-; CHECK: load i32, i32* %p
-; CHECK-NEXT: br label %block4
+  %cond = icmp sgt i32 %N, 1
+  br i1 %cond, label %block4, label %block5
 
 block3:
   store i32 0, i32* %p
@@ -387,8 +542,6 @@ block4:
 block5:
   %ret = phi i32 [ 0, %block2 ], [ %PRE, %block4 ]
   ret i32 %ret
-; CHECK: block4:
-; CHECK-NEXT: phi i32
 }
 
 declare void @f()
@@ -398,13 +551,36 @@ declare i32 @__CxxFrameHandler3(...)
 ; Test that loads aren't PRE'd into EH pads.
 define void @test12(i32* %p) personality i32 (...)* @__CxxFrameHandler3 {
 ; CHECK-LABEL: @test12(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    invoke void @f()
+; CHECK-NEXT:    to label [[BLOCK2:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       block2:
+; CHECK-NEXT:    invoke void @f()
+; CHECK-NEXT:    to label [[BLOCK3:%.*]] unwind label [[CLEANUP:%.*]]
+; CHECK:       block3:
+; CHECK-NEXT:    ret void
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[CS1:%.*]] = catchswitch within none [label %catch] unwind label [[CLEANUP2:%.*]]
+; CHECK:       catch:
+; CHECK-NEXT:    [[C:%.*]] = catchpad within [[CS1]] []
+; CHECK-NEXT:    catchret from [[C]] to label [[BLOCK2]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    [[C1:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    cleanupret from [[C1]] unwind label [[CLEANUP2]]
+; CHECK:       cleanup2:
+; CHECK-NEXT:    [[C2:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    [[NOTPRE:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    call void @g(i32 [[NOTPRE]])
+; CHECK-NEXT:    cleanupret from [[C2]] unwind to caller
+;
 block1:
   invoke void @f()
-          to label %block2 unwind label %catch.dispatch
+  to label %block2 unwind label %catch.dispatch
 
 block2:
   invoke void @f()
-          to label %block3 unwind label %cleanup
+  to label %block3 unwind label %cleanup
 
 block3:
   ret void
@@ -421,10 +597,6 @@ cleanup:
   store i32 0, i32* %p
   cleanupret from %c1 unwind label %cleanup2
 
-; CHECK: cleanup2:
-; CHECK-NOT: phi
-; CHECK-NEXT: %c2 = cleanuppad within none []
-; CHECK-NEXT: %NOTPRE = load i32, i32* %p
 cleanup2:
   %c2 = cleanuppad within none []
   %NOTPRE = load i32, i32* %p
@@ -435,28 +607,30 @@ cleanup2:
 ; Don't PRE load across potentially throwing calls.
 
 define i32 @test13(i32* noalias nocapture readonly %x, i32* noalias nocapture %r, i32 %a) {
-
 ; CHECK-LABEL: @test13(
-; CHECK: entry:
-; CHECK-NEXT: icmp eq
-; CHECK-NEXT: br i1
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[UU:%.*]] = load i32, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    store i32 [[UU]], i32* [[R:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    [[VV:%.*]] = load i32, i32* [[X]], align 4
+; CHECK-NEXT:    ret i32 [[VV]]
+;
 
 entry:
   %tobool = icmp eq i32 %a, 0
   br i1 %tobool, label %if.end, label %if.then
 
-; CHECK: if.then:
-; CHECK-NEXT: load i32
-; CHECK-NEXT: store i32
 
 if.then:
   %uu = load i32, i32* %x, align 4
   store i32 %uu, i32* %r, align 4
   br label %if.end
 
-; CHECK: if.end:
-; CHECK-NEXT: call void @f()
-; CHECK-NEXT: load i32
 
 if.end:
   call void @f()
@@ -468,28 +642,30 @@ if.end:
 ; block.
 
 define i32 @test14(i32* noalias nocapture readonly %x, i32* noalias nocapture %r, i32 %a) {
-
 ; CHECK-LABEL: @test14(
-; CHECK: entry:
-; CHECK-NEXT: icmp eq
-; CHECK-NEXT: br i1
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[UU:%.*]] = load i32, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    store i32 [[UU]], i32* [[R:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    [[VV:%.*]] = load i32, i32* [[X]], align 4
+; CHECK-NEXT:    ret i32 [[VV]]
+;
 
 entry:
   %tobool = icmp eq i32 %a, 0
   br i1 %tobool, label %if.end, label %if.then
 
-; CHECK: if.then:
-; CHECK-NEXT: load i32
-; CHECK-NEXT: store i32
 
 if.then:
   %uu = load i32, i32* %x, align 4
   store i32 %uu, i32* %r, align 4
   br label %if.end
 
-; CHECK: if.end:
-; CHECK-NEXT: call void @f()
-; CHECK-NEXT: load i32
 
 if.end:
   call void @f()
@@ -508,39 +684,39 @@ follow_2:
 ; Since it is OK to speculate, PRE is allowed.
 
 define i32 @test15(i32* noalias nocapture readonly dereferenceable(8) align 4 %x, i32* noalias nocapture %r, i32 %a) {
-
-; CHECK-LABEL: @test15
-; CHECK: entry:
-; CHECK-NEXT: icmp eq
-; CHECK-NEXT: br i1
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[ENTRY_IF_END_CRIT_EDGE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       entry.if.end_crit_edge:
+; CHECK-NEXT:    [[VV_PRE:%.*]] = load i32, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[UU:%.*]] = load i32, i32* [[X]], align 4
+; CHECK-NEXT:    store i32 [[UU]], i32* [[R:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[VV:%.*]] = phi i32 [ [[VV_PRE]], [[ENTRY_IF_END_CRIT_EDGE]] ], [ [[UU]], [[IF_THEN]] ]
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 [[VV]]
+;
 
 entry:
   %tobool = icmp eq i32 %a, 0
   br i1 %tobool, label %if.end, label %if.then
 
-; CHECK: entry.if.end_crit_edge:
-; CHECK-NEXT: %vv.pre = load i32, i32* %x, align 4
-; CHECK-NEXT: br label %if.end
 
 if.then:
   %uu = load i32, i32* %x, align 4
   store i32 %uu, i32* %r, align 4
   br label %if.end
 
-; CHECK: if.then:
-; CHECK-NEXT: %uu = load i32, i32* %x, align 4
-; CHECK-NEXT: store i32 %uu, i32* %r, align 4
-; CHECK-NEXT: br label %if.end
 
 if.end:
   call void @f()
   %vv = load i32, i32* %x, align 4
   ret i32 %vv
 
-; CHECK: if.end:
-; CHECK-NEXT: %vv = phi i32 [ %vv.pre, %entry.if.end_crit_edge ], [ %uu, %if.then ]
-; CHECK-NEXT: call void @f()
-; CHECK-NEXT: ret i32 %vv
 
 }
 
@@ -549,38 +725,38 @@ if.end:
 ; Since it is OK to speculate, PRE is allowed.
 
 define i32 @test16(i32* noalias nocapture readonly dereferenceable(8) align 4 %x, i32* noalias nocapture %r, i32 %a) {
-
 ; CHECK-LABEL: @test16(
-; CHECK: entry:
-; CHECK-NEXT: icmp eq
-; CHECK-NEXT: br i1
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[ENTRY_IF_END_CRIT_EDGE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       entry.if.end_crit_edge:
+; CHECK-NEXT:    [[VV_PRE:%.*]] = load i32, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[UU:%.*]] = load i32, i32* [[X]], align 4
+; CHECK-NEXT:    store i32 [[UU]], i32* [[R:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[VV:%.*]] = phi i32 [ [[VV_PRE]], [[ENTRY_IF_END_CRIT_EDGE]] ], [ [[UU]], [[IF_THEN]] ]
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 [[VV]]
+;
 
 entry:
   %tobool = icmp eq i32 %a, 0
   br i1 %tobool, label %if.end, label %if.then
 
-; CHECK: entry.if.end_crit_edge:
-; CHECK-NEXT: %vv.pre = load i32, i32* %x, align 4
-; CHECK-NEXT: br label %if.end
 
 if.then:
   %uu = load i32, i32* %x, align 4
   store i32 %uu, i32* %r, align 4
   br label %if.end
 
-; CHECK: if.then:
-; CHECK-NEXT: %uu = load i32, i32* %x, align 4
-; CHECK-NEXT: store i32 %uu, i32* %r, align 4
-; CHECK-NEXT: br label %if.end
 
 if.end:
   call void @f()
   br label %follow_1
 
-; CHECK: if.end:
-; CHECK-NEXT: %vv = phi i32 [ %vv.pre, %entry.if.end_crit_edge ], [ %uu, %if.then ]
-; CHECK-NEXT: call void @f()
-; CHECK-NEXT: ret i32 %vv
 
 follow_1:
   br label %follow_2
diff --git a/llvm/test/Transforms/GVN/gc_relocate.ll b/llvm/test/Transforms/GVN/gc_relocate.ll
index 53b9e5f300fc55b24a90b2904713f330b3860a4c..bd279bdf42bbfbe5ca16b9efe1f139fb7ae5df55 100644
--- a/llvm/test/Transforms/GVN/gc_relocate.ll
+++ b/llvm/test/Transforms/GVN/gc_relocate.ll
@@ -30,11 +30,8 @@ define i1 @test_readnone(i32 addrspace(1)* %in) gc "statepoint-example" {
 ; CHECK-NEXT:    [[SAFEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[IN:%.*]]) ]
 ; CHECK-NEXT:    [[A:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    store i32 0, i32* @G, align 4
-; CHECK-NEXT:    [[B:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 addrspace(1)* [[A]], null
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 addrspace(1)* [[B]], null
-; CHECK-NEXT:    [[CMP:%.*]] = and i1 [[CMP1]], [[CMP2]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
 ;
 entry:
   %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %in)]
@@ -52,14 +49,10 @@ define i1 @test_call(i32 addrspace(1)* %in) gc "statepoint-example" {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SAFEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[IN:%.*]], i32 addrspace(1)* [[IN]]) ]
 ; CHECK-NEXT:    [[BASE:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0)
-; CHECK-NEXT:    [[DERIVED:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 1)
-; CHECK-NEXT:    [[SAFEPOINT_TOKEN2:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[BASE]], i32 addrspace(1)* [[DERIVED]]) ]
+; CHECK-NEXT:    [[SAFEPOINT_TOKEN2:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[BASE]], i32 addrspace(1)* [[BASE]]) ]
 ; CHECK-NEXT:    [[BASE_RELOC:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN2]], i32 0, i32 0)
-; CHECK-NEXT:    [[DERIVED_RELOC:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN2]], i32 0, i32 1)
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 addrspace(1)* [[BASE_RELOC]], null
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 addrspace(1)* [[DERIVED_RELOC]], null
-; CHECK-NEXT:    [[CMP:%.*]] = and i1 [[CMP1]], [[CMP2]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
 ;
 entry:
   %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %in, i32 addrspace(1)* %in)]
diff --git a/llvm/test/Transforms/GVN/loadpre-context.ll b/llvm/test/Transforms/GVN/loadpre-context.ll
index 50a43b0e91f8546021b3ee0f3687f4a0fab326c3..1a178e71cc81d0427cc1672302aedd633a56a8da 100644
--- a/llvm/test/Transforms/GVN/loadpre-context.ll
+++ b/llvm/test/Transforms/GVN/loadpre-context.ll
@@ -14,7 +14,7 @@ define i32 @loadpre_critical_edge(i32* align 8 dereferenceable_or_null(48) %arg,
 ; CHECK:       header:
 ; CHECK-NEXT:    [[V:%.*]] = phi i32 [ [[V_PRE]], [[ENTRY_HEADER_CRIT_EDGE]] ], [ [[SUM:%.*]], [[HEADER]] ]
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY_HEADER_CRIT_EDGE]] ], [ [[IV_NEXT:%.*]], [[HEADER]] ]
-; CHECK-NEXT:    [[NEW_V:%.*]] = call i32 @ro_foo(i32 [[IV]])
+; CHECK-NEXT:    [[NEW_V:%.*]] = call i32 @ro_foo(i32 [[IV]]) #[[ATTR0:[0-9]+]]
 ; CHECK-NEXT:    [[SUM]] = add i32 [[NEW_V]], [[V]]
 ; CHECK-NEXT:    store i32 [[SUM]], i32* [[ARG]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
@@ -59,7 +59,7 @@ define i32 @loadpre_basic(i32* align 8 dereferenceable_or_null(48) %arg, i32 %N)
 ; CHECK:       header:
 ; CHECK-NEXT:    [[V:%.*]] = phi i32 [ [[V_PRE]], [[PREHEADER]] ], [ [[SUM:%.*]], [[HEADER]] ]
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[IV_NEXT:%.*]], [[HEADER]] ]
-; CHECK-NEXT:    [[NEW_V:%.*]] = call i32 @ro_foo(i32 [[IV]])
+; CHECK-NEXT:    [[NEW_V:%.*]] = call i32 @ro_foo(i32 [[IV]]) #[[ATTR0]]
 ; CHECK-NEXT:    [[SUM]] = add i32 [[NEW_V]], [[V]]
 ; CHECK-NEXT:    store i32 [[SUM]], i32* [[ARG]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
@@ -107,7 +107,7 @@ define i32 @loadpre_maybe_null(i32* align 8 dereferenceable_or_null(48) %arg, i3
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[IV_NEXT:%.*]], [[HEADER]] ]
-; CHECK-NEXT:    [[NEW_V:%.*]] = call i32 @ro_foo(i32 [[IV]])
+; CHECK-NEXT:    [[NEW_V:%.*]] = call i32 @ro_foo(i32 [[IV]]) #[[ATTR0]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[ARG]], align 4
 ; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[NEW_V]], [[V]]
 ; CHECK-NEXT:    store i32 [[SUM]], i32* [[ARG]], align 4
diff --git a/llvm/test/Transforms/GVNHoist/hoist-pr28606.ll b/llvm/test/Transforms/GVNHoist/hoist-pr28606.ll
index 2c588283ea914f92f5e2c0f760757caea18af818..d8bad7ae45638a6d9b9de2992aa62fde2ba2be5d 100644
--- a/llvm/test/Transforms/GVNHoist/hoist-pr28606.ll
+++ b/llvm/test/Transforms/GVNHoist/hoist-pr28606.ll
@@ -5,7 +5,7 @@ target triple = "i686-pc-windows-msvc18.0.0"
 
 %struct.S = type { i8* }
 
-declare void @f(<{ %struct.S }>* inalloca)
+declare void @f(<{ %struct.S }>* inalloca(<{ %struct.S }>))
 
 
 ; Check that we don't clone the %x alloca and insert it in the live range of
@@ -41,7 +41,7 @@ false:
   br label %exit
 
 exit:
-  call void @f(<{ %struct.S }>* inalloca %argmem)
+  call void @f(<{ %struct.S }>* inalloca(<{ %struct.S }>) %argmem)
   call void @llvm.stackrestore(i8* %inalloca.save)
   ret void
 }
diff --git a/llvm/test/Transforms/GlobalOpt/fastcc.ll b/llvm/test/Transforms/GlobalOpt/fastcc.ll
index edd0688ea92b7ff007c4771e80ada259203376e0..0278d83d209f3dd172c5fbb293eec7b5ada5ecea 100644
--- a/llvm/test/Transforms/GlobalOpt/fastcc.ll
+++ b/llvm/test/Transforms/GlobalOpt/fastcc.ll
@@ -29,19 +29,19 @@ define internal i32 @j(i32* %m) {
   ret i32 %v
 }
 
-define internal i32 @inalloca(i32* inalloca %p) {
+define internal i32 @inalloca(i32* inalloca(i32) %p) {
 ; CHECK-LABEL: define internal fastcc i32 @inalloca(i32* %p)
   %rv = load i32, i32* %p
   ret i32 %rv
 }
 
-define i32 @inalloca2_caller(i32* inalloca %p) {
-  %rv = musttail call i32 @inalloca2(i32* inalloca %p)
+define i32 @inalloca2_caller(i32* inalloca(i32) %p) {
+  %rv = musttail call i32 @inalloca2(i32* inalloca(i32) %p)
   ret i32 %rv
 }
-define internal i32 @inalloca2(i32* inalloca %p) {
+define internal i32 @inalloca2(i32* inalloca(i32) %p) {
 ; Because of the musttail caller, this inalloca cannot be dropped.
-; CHECK-LABEL: define internal i32 @inalloca2(i32* inalloca %p)
+; CHECK-LABEL: define internal i32 @inalloca2(i32* inalloca(i32) %p)
   %rv = load i32, i32* %p
   ret i32 %rv
 }
@@ -59,7 +59,7 @@ define void @call_things() {
   call coldcc i32 @h(i32* %m)
   call i32 @j(i32* %m)
   %args = alloca inalloca i32
-  call i32 @inalloca(i32* inalloca %args)
+  call i32 @inalloca(i32* inalloca(i32) %args)
   %c = call token @llvm.call.preallocated.setup(i32 1)
   %N = call i8* @llvm.call.preallocated.arg(token %c, i32 0) preallocated(i32)
   %n = bitcast i8* %N to i32*
diff --git a/llvm/test/Transforms/GlobalSplit/basic.ll b/llvm/test/Transforms/GlobalSplit/basic.ll
index aa5b331a96ba36368003ba7de07405f93467d792..0a3724ffe3eef310490025b7c7d941314bc96dac 100644
--- a/llvm/test/Transforms/GlobalSplit/basic.ll
+++ b/llvm/test/Transforms/GlobalSplit/basic.ll
@@ -12,8 +12,8 @@ target triple = "x86_64-unknown-linux-gnu"
 ]
 
 ; CHECK-NOT: @global =
-; CHECK: @global.0 = private constant [2 x i8* ()*] [i8* ()* @f1, i8* ()* @f2], !type [[T1:![0-9]+]], !type [[T2:![0-9]+]], !type [[T3:![0-9]+]], !vcall_visibility [[VIS:![0-9]+$]]
-; CHECK: @global.1 = private constant [1 x i8* ()*] [i8* ()* @f3], !type [[T4:![0-9]+]], !type [[T5:![0-9]+]], !vcall_visibility [[VIS$]]
+; CHECK: @global.0 = private constant [2 x i8* ()*] [i8* ()* @f1, i8* ()* @f2], !type [[T1:![0-9]+]], !type [[T2:![0-9]+]], !type [[T3:![0-9]+]], !vcall_visibility [[VIS:![0-9]+]]{{$}}
+; CHECK: @global.1 = private constant [1 x i8* ()*] [i8* ()* @f3], !type [[T4:![0-9]+]], !type [[T5:![0-9]+]], !vcall_visibility [[VIS]]{{$}}
 ; CHECK-NOT: @global =
 @global = internal constant { [2 x i8* ()*], [1 x i8* ()*] } {
   [2 x i8* ()*] [i8* ()* @f1, i8* ()* @f2],
diff --git a/llvm/test/Transforms/IRCE/bad_expander.ll b/llvm/test/Transforms/IRCE/bad_expander.ll
index 8eb769526aaa4f39a53c454e5d215a46be69b9b8..a3d6a054ef05438005b83efd340f7951cfc8e704 100644
--- a/llvm/test/Transforms/IRCE/bad_expander.ll
+++ b/llvm/test/Transforms/IRCE/bad_expander.ll
@@ -98,7 +98,7 @@ define void @test_03(i64* %p1, i64* %p2, i1 %maybe_exit) {
 ; CHECK-NEXT:    %iv = phi i64 [ %iv.next, %guarded ], [ 0, %loop.preheader ]
 ; CHECK-NEXT:    %iv.next = add nuw nsw i64 %iv, 1
 ; CHECK-NEXT:    %rc = icmp slt i64 %iv.next, %div_result
-; CHECK-NEXT:    %or.cond = and i1 %maybe_exit, true
+; CHECK-NEXT:    %or.cond = select i1 %maybe_exit, i1 true, i1 false
 ; CHECK-NEXT:    br i1 %or.cond, label %guarded, label %exit.loopexit1
 ; CHECK:       guarded:
 ; CHECK-NEXT:    %gep = getelementptr i64, i64* %p1, i64 %iv.next
diff --git a/llvm/test/Transforms/IndVarSimplify/2011-10-27-lftrnull.ll b/llvm/test/Transforms/IndVarSimplify/2011-10-27-lftrnull.ll
index d56e985ce993c89c24e8c70e301c019eae3a02e3..ed2b8743a79d2520c97821ce5519d8552ba8a17b 100644
--- a/llvm/test/Transforms/IndVarSimplify/2011-10-27-lftrnull.ll
+++ b/llvm/test/Transforms/IndVarSimplify/2011-10-27-lftrnull.ll
@@ -28,16 +28,13 @@ define void @test() nounwind {
 ; CHECK-NEXT:    br label [[FOR_BODY21_I:%.*]]
 ; CHECK:       for.body21.i:
 ; CHECK-NEXT:    [[DESTYPIXELPTR_010_I:%.*]] = phi i8* [ null, [[FOR_BODY21_LR_PH_I]] ], [ [[INCDEC_PTR_I:%.*]], [[IF_END_I126:%.*]] ]
-; CHECK-NEXT:    [[X_09_I:%.*]] = phi i32 [ 0, [[FOR_BODY21_LR_PH_I]] ], [ [[INC_I125:%.*]], [[IF_END_I126]] ]
 ; CHECK-NEXT:    br i1 undef, label [[IF_END_I126]], label [[IF_ELSE_I124:%.*]]
 ; CHECK:       if.else.i124:
 ; CHECK-NEXT:    store i8 undef, i8* [[DESTYPIXELPTR_010_I]], align 1
 ; CHECK-NEXT:    br label [[IF_END_I126]]
 ; CHECK:       if.end.i126:
 ; CHECK-NEXT:    [[INCDEC_PTR_I]] = getelementptr inbounds i8, i8* [[DESTYPIXELPTR_010_I]], i32 1
-; CHECK-NEXT:    [[INC_I125]] = add nuw i32 [[X_09_I]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[INC_I125]], undef
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY21_I]], label [[FOR_END_I129_LOOPEXIT:%.*]]
+; CHECK-NEXT:    br i1 true, label [[FOR_BODY21_I]], label [[FOR_END_I129_LOOPEXIT:%.*]]
 ; CHECK:       for.end.i129.loopexit:
 ; CHECK-NEXT:    br label [[FOR_END_I129]]
 ; CHECK:       for.end.i129:
diff --git a/llvm/test/Transforms/IndVarSimplify/X86/pr35406.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr35406.ll
index e51ee24cd343b66cf04ad06bf1eacd22818526b3..cd5615a1bc67fd4d5c6b7a2bb3e68b4c0f480cf7 100644
--- a/llvm/test/Transforms/IndVarSimplify/X86/pr35406.ll
+++ b/llvm/test/Transforms/IndVarSimplify/X86/pr35406.ll
@@ -9,8 +9,8 @@ define i32 @testDiv(i8* %p, i64* %p1) {
 ; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop1:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP2_EXIT:%.*]] ], [ 8, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[EXITCOND3:%.*]] = icmp eq i64 [[INDVARS_IV]], 15
-; CHECK-NEXT:    br i1 [[EXITCOND3]], label [[EXIT:%.*]], label [[GENERAL_CASE24:%.*]]
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV]], 15
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[GENERAL_CASE24:%.*]]
 ; CHECK:       general_case24:
 ; CHECK-NEXT:    br i1 false, label [[LOOP2_PREHEADER:%.*]], label [[LOOP2_EXIT]]
 ; CHECK:       loop2.preheader:
@@ -19,14 +19,11 @@ define i32 @testDiv(i8* %p, i64* %p1) {
 ; CHECK-NEXT:    br label [[LOOP2:%.*]]
 ; CHECK:       loop2:
 ; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ [[TMP1]], [[LOOP2_PREHEADER]] ], [ [[INDVARS_IV_NEXT2:%.*]], [[LOOP2]] ]
-; CHECK-NEXT:    [[LOCAL_2_57:%.*]] = phi i32 [ [[I7:%.*]], [[LOOP2]] ], [ 1, [[LOOP2_PREHEADER]] ]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT2]] = add nsw i64 [[INDVARS_IV1]], -1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT2]] = add nuw nsw i64 [[INDVARS_IV1]], -1
 ; CHECK-NEXT:    [[I4:%.*]] = load atomic i64, i64* [[P1:%.*]] unordered, align 8
 ; CHECK-NEXT:    [[I6:%.*]] = sub i64 [[I4]], [[INDVARS_IV_NEXT2]]
 ; CHECK-NEXT:    store atomic i64 [[I6]], i64* [[P1]] unordered, align 8
-; CHECK-NEXT:    [[I7]] = add nuw nsw i32 [[LOCAL_2_57]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I7]], 9
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP2_EXIT_LOOPEXIT:%.*]], label [[LOOP2]]
+; CHECK-NEXT:    br i1 true, label [[LOOP2_EXIT_LOOPEXIT:%.*]], label [[LOOP2]]
 ; CHECK:       loop2.exit.loopexit:
 ; CHECK-NEXT:    br label [[LOOP2_EXIT]]
 ; CHECK:       loop2.exit:
@@ -79,8 +76,8 @@ define i32 @testRem(i8* %p, i64* %p1) {
 ; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop1:
 ; CHECK-NEXT:    [[LOCAL_0_:%.*]] = phi i32 [ 8, [[ENTRY:%.*]] ], [ [[I9:%.*]], [[LOOP2_EXIT:%.*]] ]
-; CHECK-NEXT:    [[EXITCOND1:%.*]] = icmp eq i32 [[LOCAL_0_]], 15
-; CHECK-NEXT:    br i1 [[EXITCOND1]], label [[EXIT:%.*]], label [[GENERAL_CASE24:%.*]]
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LOCAL_0_]], 15
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[GENERAL_CASE24:%.*]]
 ; CHECK:       general_case24:
 ; CHECK-NEXT:    br i1 false, label [[LOOP2_PREHEADER:%.*]], label [[LOOP2_EXIT]]
 ; CHECK:       loop2.preheader:
@@ -93,14 +90,11 @@ define i32 @testRem(i8* %p, i64* %p1) {
 ; CHECK-NEXT:    br label [[LOOP2:%.*]]
 ; CHECK:       loop2:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP5]], [[LOOP2_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LOOP2]] ]
-; CHECK-NEXT:    [[LOCAL_2_57:%.*]] = phi i32 [ [[I7:%.*]], [[LOOP2]] ], [ 1, [[LOOP2_PREHEADER]] ]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
 ; CHECK-NEXT:    [[I4:%.*]] = load atomic i64, i64* [[P1:%.*]] unordered, align 8
 ; CHECK-NEXT:    [[I6:%.*]] = sub i64 [[I4]], [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    store atomic i64 [[I6]], i64* [[P1]] unordered, align 8
-; CHECK-NEXT:    [[I7]] = add nuw nsw i32 [[LOCAL_2_57]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I7]], 9
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP2_EXIT_LOOPEXIT:%.*]], label [[LOOP2]]
+; CHECK-NEXT:    br i1 true, label [[LOOP2_EXIT_LOOPEXIT:%.*]], label [[LOOP2]]
 ; CHECK:       loop2.exit.loopexit:
 ; CHECK-NEXT:    br label [[LOOP2_EXIT]]
 ; CHECK:       loop2.exit:
diff --git a/llvm/test/Transforms/IndVarSimplify/checks_against_min_value.ll b/llvm/test/Transforms/IndVarSimplify/checks_against_min_value.ll
index 4575a4547faccb2e17eded1f4a1e5bbdd9ac5c3f..2b25daf9573be29a00c19a9bdfe5c3d0e91dcd12 100644
--- a/llvm/test/Transforms/IndVarSimplify/checks_against_min_value.ll
+++ b/llvm/test/Transforms/IndVarSimplify/checks_against_min_value.ll
@@ -15,8 +15,7 @@ define void @test_signed(i32 %start) {
 ; CHECK-NEXT:    [[CHECK:%.*]] = icmp slt i32 [[IV_NEXT]], [[IV]]
 ; CHECK-NEXT:    br i1 [[CHECK]], label [[GUARDED]], label [[FAIL:%.*]]
 ; CHECK:       guarded:
-; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ne i32 [[IV]], -2147483648
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK-NEXT:    br i1 true, label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -58,8 +57,7 @@ define void @test_unsigned(i32 %start) {
 ; CHECK-NEXT:    [[CHECK:%.*]] = icmp ult i32 [[IV_NEXT]], [[IV]]
 ; CHECK-NEXT:    br i1 [[CHECK]], label [[GUARDED]], label [[FAIL:%.*]]
 ; CHECK:       guarded:
-; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ne i32 [[IV]], 0
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK-NEXT:    br i1 true, label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll
index 0bbcd3fa3ae0aa300d72a36e2df81595b6f3c842..ee864b77ba2e102d6a375917951003b3f27c4633 100644
--- a/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll
@@ -380,7 +380,7 @@ declare void @side_effect()
 define void @func_13(i32* %len.ptr) {
 ; CHECK-LABEL: @func_13(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, [[RNG0:!range !.*]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, !range [[RNG0:![0-9]+]]
 ; CHECK-NEXT:    [[LEN_IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[LEN_IS_ZERO]], label [[LEAVE:%.*]], label [[LOOP_PREHEADER:%.*]]
 ; CHECK:       loop.preheader:
@@ -424,7 +424,7 @@ leave:
 define void @func_14(i32* %len.ptr) {
 ; CHECK-LABEL: @func_14(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[LEN_IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
 ; CHECK-NEXT:    [[LEN_IS_INT_MIN:%.*]] = icmp eq i32 [[LEN]], -2147483648
 ; CHECK-NEXT:    [[NO_ENTRY:%.*]] = or i1 [[LEN_IS_ZERO]], [[LEN_IS_INT_MIN]]
@@ -472,7 +472,7 @@ leave:
 define void @func_15(i32* %len.ptr) {
 ; CHECK-LABEL: @func_15(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[LEN_ADD_1:%.*]] = add i32 [[LEN]], 1
 ; CHECK-NEXT:    [[LEN_ADD_1_IS_ZERO:%.*]] = icmp eq i32 [[LEN_ADD_1]], 0
 ; CHECK-NEXT:    br i1 [[LEN_ADD_1_IS_ZERO]], label [[LEAVE:%.*]], label [[LOOP_PREHEADER:%.*]]
@@ -517,7 +517,7 @@ leave:
 define void @func_16(i32* %len.ptr) {
 ; CHECK-LABEL: @func_16(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[LEN_ADD_5:%.*]] = add i32 [[LEN]], 5
 ; CHECK-NEXT:    [[ENTRY_COND_0:%.*]] = icmp slt i32 [[LEN]], 2147483643
 ; CHECK-NEXT:    [[ENTRY_COND_1:%.*]] = icmp slt i32 4, [[LEN_ADD_5]]
@@ -624,7 +624,7 @@ leave:
 define i1 @func_18(i16* %tmp20, i32* %len.addr) {
 ; CHECK-LABEL: @func_18(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_ADDR:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LEN_ADDR:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[BB2:%.*]], label [[BB0_PREHEADER:%.*]]
 ; CHECK:       bb0.preheader:
@@ -686,7 +686,7 @@ bb3:
 define void @func_19(i32* %length.ptr) {
 ; CHECK-LABEL: @func_19(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[LENGTH_IS_NONZERO:%.*]] = icmp ne i32 [[LENGTH]], 0
 ; CHECK-NEXT:    br i1 [[LENGTH_IS_NONZERO]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]]
 ; CHECK:       loop.preheader:
@@ -774,7 +774,7 @@ leave:
 define void @func_21(i32* %length.ptr) {
 ; CHECK-LABEL: @func_21(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[LIM:%.*]] = sub i32 [[LENGTH]], 1
 ; CHECK-NEXT:    [[ENTRY_COND:%.*]] = icmp sgt i32 [[LENGTH]], 1
 ; CHECK-NEXT:    br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]]
@@ -819,7 +819,7 @@ leave:
 define void @func_22(i32* %length.ptr) {
 ; CHECK-LABEL: @func_22(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[ENTRY_COND:%.*]] = icmp sgt i32 [[LENGTH]], 1
 ; CHECK-NEXT:    br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]]
 ; CHECK:       loop.preheader:
@@ -861,7 +861,7 @@ leave:
 define void @func_23(i32* %length.ptr) {
 ; CHECK-LABEL: @func_23(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[ENTRY_COND:%.*]] = icmp ult i32 4, [[LENGTH]]
 ; CHECK-NEXT:    br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]]
 ; CHECK:       loop.preheader:
@@ -902,7 +902,7 @@ leave:
 define void @func_24(i32* %init.ptr) {
 ; CHECK-LABEL: @func_24(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INIT:%.*]] = load i32, i32* [[INIT_PTR:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[INIT:%.*]] = load i32, i32* [[INIT_PTR:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[ENTRY_COND:%.*]] = icmp ugt i32 [[INIT]], 4
 ; CHECK-NEXT:    br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]]
 ; CHECK:       loop.preheader:
@@ -951,11 +951,9 @@ define i32 @func_25(i32 %start) {
 ; CHECK-NEXT:    [[C1:%.*]] = icmp ne i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[C1]], label [[CHECKED_1:%.*]], label [[FAIL:%.*]]
 ; CHECK:       checked.1:
-; CHECK-NEXT:    [[C2:%.*]] = icmp ne i32 [[IV]], 0
-; CHECK-NEXT:    br i1 [[C2]], label [[CHECKED_2:%.*]], label [[FAIL]]
+; CHECK-NEXT:    br i1 true, label [[CHECKED_2:%.*]], label [[FAIL]]
 ; CHECK:       checked.2:
-; CHECK-NEXT:    [[C3:%.*]] = icmp ne i32 [[IV]], 0
-; CHECK-NEXT:    br i1 [[C3]], label [[BACKEDGE]], label [[FAIL]]
+; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[FAIL]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 758394
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = call i1 @cond_func()
@@ -1003,11 +1001,9 @@ define i32 @func_26(i32 %start) {
 ; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[C1]], label [[CHECKED_1:%.*]], label [[FAIL:%.*]]
 ; CHECK:       checked.1:
-; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C2]], label [[CHECKED_2:%.*]], label [[FAIL]]
+; CHECK-NEXT:    br i1 true, label [[CHECKED_2:%.*]], label [[FAIL]]
 ; CHECK:       checked.2:
-; CHECK-NEXT:    [[C3:%.*]] = icmp slt i32 [[IV]], 2
-; CHECK-NEXT:    br i1 [[C3]], label [[BACKEDGE]], label [[FAIL]]
+; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[FAIL]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 758394
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = call i1 @cond_func()
@@ -1046,5 +1042,106 @@ exit:
   ret i32 %iv
 }
 
+define i32 @func_27(i32 %start) {
+; CHECK-LABEL: @func_27(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[IV]], 2
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[IV]], 1
+; CHECK-NEXT:    [[C3:%.*]] = icmp slt i32 [[IV]], 0
+; CHECK-NEXT:    br i1 [[C1]], label [[CHECKED_1:%.*]], label [[FAIL:%.*]]
+; CHECK:       checked.1:
+; CHECK-NEXT:    br i1 [[C2]], label [[CHECKED_2:%.*]], label [[FAIL]]
+; CHECK:       checked.2:
+; CHECK-NEXT:    br i1 [[C3]], label [[BACKEDGE]], label [[FAIL]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 758394
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = call i1 @cond_func()
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       fail:
+; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV_LCSSA1:%.*]] = phi i32 [ [[IV]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[IV_LCSSA1]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [%start, %entry], [%iv.next, %backedge]
+  %c1 = icmp slt i32 %iv, 2
+  %c2 = icmp slt i32 %iv, 1
+  %c3 = icmp slt i32 %iv, 0
+  br i1 %c1, label %checked.1, label %fail
+
+checked.1:
+  br i1 %c2, label %checked.2, label %fail
+
+checked.2:
+  br i1 %c3, label %backedge, label %fail
+
+backedge:
+  %iv.next = add i32 %iv, 758394
+  %loop.cond = call i1 @cond_func()
+  br i1 %loop.cond, label %loop, label %exit
+
+fail:
+  unreachable
+
+exit:
+  ret i32 %iv
+}
+
+define i32 @func_28(i32 %start) {
+; CHECK-LABEL: @func_28(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[C1:%.*]] = icmp ne i32 [[IV]], 0
+; CHECK-NEXT:    br i1 [[C1]], label [[CHECKED_1:%.*]], label [[FAIL:%.*]]
+; CHECK:       checked.1:
+; CHECK-NEXT:    br i1 true, label [[CHECKED_2:%.*]], label [[FAIL]]
+; CHECK:       checked.2:
+; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[FAIL]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 758394
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = call i1 @cond_func()
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       fail:
+; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV_LCSSA1:%.*]] = phi i32 [ [[IV]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[IV_LCSSA1]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [%start, %entry], [%iv.next, %backedge]
+  %c1 = icmp ne i32 %iv, 0
+  %c2 = icmp ne i32 %iv, 0
+  %c3 = icmp ne i32 %iv, 0
+  br i1 %c1, label %checked.1, label %fail
+
+checked.1:
+  br i1 %c2, label %checked.2, label %fail
+
+checked.2:
+  br i1 %c3, label %backedge, label %fail
+
+backedge:
+  %iv.next = add i32 %iv, 758394
+  %loop.cond = call i1 @cond_func()
+  br i1 %loop.cond, label %loop, label %exit
+
+fail:
+  unreachable
+
+exit:
+  ret i32 %iv
+}
 
 !0 = !{i32 0, i32 2147483647}
diff --git a/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll b/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll
new file mode 100644
index 0000000000000000000000000000000000000000..60a0c2a8526cb2dbdb8e447753525f9178d4567e
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -indvars -S < %s | FileCheck %s
+; RUN: opt -passes=indvars -S < %s | FileCheck %s
+
+declare i1 @cond()
+
+define void @test_01(i32 %x) {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 true, label [[LOOP_1_PREHEADER:%.*]], label [[LOOP_2_PREHEADER:%.*]]
+; CHECK:       loop.2.preheader:
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop.1.preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop.1:
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i32 [ [[IV_NEXT_1:%.*]], [[GUARDED_1:%.*]] ], [ 0, [[LOOP_1_PREHEADER]] ]
+; CHECK-NEXT:    [[CHECK_1:%.*]] = icmp slt i32 [[IV_1]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CHECK_1]], label [[GUARDED_1]], label [[FAIL_LOOPEXIT:%.*]]
+; CHECK:       guarded.1:
+; CHECK-NEXT:    [[IV_NEXT_1]] = add nuw i32 [[IV_1]], 1
+; CHECK-NEXT:    [[LOOP_COND_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[LOOP_COND_1]], label [[LOOP_1]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       loop.2:
+; CHECK-NEXT:    br i1 true, label [[GUARDED_2:%.*]], label [[FAIL_LOOPEXIT1:%.*]]
+; CHECK:       guarded.2:
+; CHECK-NEXT:    [[LOOP_COND_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[LOOP_COND_2]], label [[LOOP_2]], label [[EXIT_LOOPEXIT2:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit.loopexit2:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       fail.loopexit:
+; CHECK-NEXT:    br label [[FAIL:%.*]]
+; CHECK:       fail.loopexit1:
+; CHECK-NEXT:    br label [[FAIL]]
+; CHECK:       fail:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br i1 true, label %loop.1, label %loop.2
+
+loop.1:
+  %iv.1 = phi i32 [0, %entry], [%iv.next.1, %guarded.1]
+  %check.1 = icmp slt i32 %iv.1, %x
+  br i1 %check.1, label %guarded.1, label %fail
+
+guarded.1:
+  %iv.next.1 = add i32 %iv.1, 1
+  %loop.cond.1 = call i1 @cond()
+  br i1 %loop.cond.1, label %loop.1, label %exit
+
+loop.2:
+  %iv.2 = phi i32 [0, %entry], [%iv.next.2, %guarded.2]
+  %check.2 = icmp slt i32 %iv.2, %x
+  br i1 %check.2, label %guarded.2, label %fail
+
+guarded.2:
+  %iv.next.2 = add i32 %iv.2, 1
+  %loop.cond.2 = call i1 @cond()
+  br i1 %loop.cond.2, label %loop.2, label %exit
+
+exit:
+  ret void
+
+fail:
+  unreachable
+}
+
+define void @test_02(i32 %x) {
+; CHECK-LABEL: @test_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[LOOP_1_PREHEADER:%.*]], label [[LOOP_2_PREHEADER:%.*]]
+; CHECK:       loop.2.preheader:
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop.1.preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop.1:
+; CHECK-NEXT:    br i1 true, label [[GUARDED_1:%.*]], label [[FAIL_LOOPEXIT:%.*]]
+; CHECK:       guarded.1:
+; CHECK-NEXT:    [[LOOP_COND_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[LOOP_COND_1]], label [[LOOP_1]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       loop.2:
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i32 [ [[IV_NEXT_2:%.*]], [[GUARDED_2:%.*]] ], [ 0, [[LOOP_2_PREHEADER]] ]
+; CHECK-NEXT:    [[CHECK_2:%.*]] = icmp slt i32 [[IV_2]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CHECK_2]], label [[GUARDED_2]], label [[FAIL_LOOPEXIT1:%.*]]
+; CHECK:       guarded.2:
+; CHECK-NEXT:    [[IV_NEXT_2]] = add nuw i32 [[IV_2]], 1
+; CHECK-NEXT:    [[LOOP_COND_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[LOOP_COND_2]], label [[LOOP_2]], label [[EXIT_LOOPEXIT2:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit.loopexit2:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       fail.loopexit:
+; CHECK-NEXT:    br label [[FAIL:%.*]]
+; CHECK:       fail.loopexit1:
+; CHECK-NEXT:    br label [[FAIL]]
+; CHECK:       fail:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br i1 false, label %loop.1, label %loop.2
+
+loop.1:
+  %iv.1 = phi i32 [0, %entry], [%iv.next.1, %guarded.1]
+  %check.1 = icmp slt i32 %iv.1, %x
+  br i1 %check.1, label %guarded.1, label %fail
+
+guarded.1:
+  %iv.next.1 = add i32 %iv.1, 1
+  %loop.cond.1 = call i1 @cond()
+  br i1 %loop.cond.1, label %loop.1, label %exit
+
+loop.2:
+  %iv.2 = phi i32 [0, %entry], [%iv.next.2, %guarded.2]
+  %check.2 = icmp slt i32 %iv.2, %x
+  br i1 %check.2, label %guarded.2, label %fail
+
+guarded.2:
+  %iv.next.2 = add i32 %iv.2, 1
+  %loop.cond.2 = call i1 @cond()
+  br i1 %loop.cond.2, label %loop.2, label %exit
+
+exit:
+  ret void
+
+fail:
+  unreachable
+}
diff --git a/llvm/test/Transforms/Inline/inalloca-not-static.ll b/llvm/test/Transforms/Inline/inalloca-not-static.ll
index 74b5ecf420ce98eaa83b044aeaa2558a2f8b592f..1a6dd75a0178127b7810d4bae1f62f3a8f428c3f 100644
--- a/llvm/test/Transforms/Inline/inalloca-not-static.ll
+++ b/llvm/test/Transforms/Inline/inalloca-not-static.ll
@@ -41,13 +41,13 @@ entry:
   %argmem = alloca inalloca <{ %struct.Foo }>, align 4
   %0 = getelementptr inbounds <{ %struct.Foo }>, <{ %struct.Foo }>* %argmem, i32 0, i32 0
   %call = call x86_thiscallcc %struct.Foo* @"\01??0Foo@@QAE@XZ"(%struct.Foo* %0)
-  call void @h(<{ %struct.Foo }>* inalloca %argmem)
+  call void @h(<{ %struct.Foo }>* inalloca(<{ %struct.Foo }>) %argmem)
   call void @llvm.stackrestore(i8* %inalloca.save)
   ret void
 }
 
 ; Function Attrs: alwaysinline inlinehint nounwind
-define internal void @h(<{ %struct.Foo }>* inalloca) alwaysinline {
+define internal void @h(<{ %struct.Foo }>* inalloca(<{ %struct.Foo }>)) alwaysinline {
 entry:
   %o = getelementptr inbounds <{ %struct.Foo }>, <{ %struct.Foo }>* %0, i32 0, i32 0
   call x86_thiscallcc void @"\01??1Foo@@QAE@XZ"(%struct.Foo* %o)
diff --git a/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll b/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll
index d8c8e1e02025b540468289ee204262703f101f19..86b9bc57fe509fef36b2285fa70c77715eaa5200 100644
--- a/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll
+++ b/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll
@@ -1,4 +1,7 @@
-; RUN: opt < %s -instcombine -sroa -S | not grep " = alloca"
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck --check-prefix=IC %s
+; RUN: opt < %s -instcombine -sroa -S | FileCheck --check-prefix=IC_SROA %s
+
 ; rdar://6417724
 ; Instcombine shouldn't do anything to this function that prevents promoting the allocas inside it.
 
@@ -15,6 +18,262 @@ target triple = "i386-apple-darwin9.6"
 %"struct.std::vector<int,std::allocator<int> >" = type { %"struct.std::_Vector_base<int,std::allocator<int> >" }
 
 define i32* @_Z3fooRSt6vectorIiSaIiEE(%"struct.std::vector<int,std::allocator<int> >"* %X) {
+; IC-LABEL: @_Z3fooRSt6vectorIiSaIiEE(
+; IC-NEXT:  entry:
+; IC-NEXT:    [[__FIRST_ADDR_I_I:%.*]] = alloca %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", align 8
+; IC-NEXT:    [[__LAST_ADDR_I_I:%.*]] = alloca %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", align 8
+; IC-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
+; IC-NEXT:    store i32 42, i32* [[TMP0]], align 4
+; IC-NEXT:    [[TMP1:%.*]] = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* [[X:%.*]], i32 0, i32 0, i32 0, i32 1
+; IC-NEXT:    [[TMP2:%.*]] = load i32*, i32** [[TMP1]], align 4
+; IC-NEXT:    [[TMP3:%.*]] = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* [[X]], i32 0, i32 0, i32 0, i32 0
+; IC-NEXT:    [[TMP4:%.*]] = load i32*, i32** [[TMP3]], align 4
+; IC-NEXT:    [[TMP5:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    store i32* [[TMP4]], i32** [[TMP5]], align 8
+; IC-NEXT:    [[TMP6:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__LAST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    store i32* [[TMP2]], i32** [[TMP6]], align 8
+; IC-NEXT:    [[TMP7:%.*]] = ptrtoint i32* [[TMP2]] to i32
+; IC-NEXT:    [[TMP8:%.*]] = ptrtoint i32* [[TMP4]] to i32
+; IC-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP7]], [[TMP8]]
+; IC-NEXT:    [[TMP10:%.*]] = ashr i32 [[TMP9]], 4
+; IC-NEXT:    br label [[BB12_I_I:%.*]]
+; IC:       bb.i.i:
+; IC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP12:%.*]] = load i32*, i32** [[TMP11]], align 8
+; IC-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
+; IC-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP0]], align 4
+; IC-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP13]], [[TMP14]]
+; IC-NEXT:    br i1 [[TMP15]], label [[BB1_I_I:%.*]], label [[BB2_I_I:%.*]]
+; IC:       bb1.i.i:
+; IC-NEXT:    [[TMP16:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP17:%.*]] = load i32*, i32** [[TMP16]], align 8
+; IC-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT:%.*]]
+; IC:       bb2.i.i:
+; IC-NEXT:    [[TMP18:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP19:%.*]] = load i32*, i32** [[TMP18]], align 8
+; IC-NEXT:    [[TMP20:%.*]] = getelementptr i32, i32* [[TMP19]], i32 1
+; IC-NEXT:    [[TMP21:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    store i32* [[TMP20]], i32** [[TMP21]], align 8
+; IC-NEXT:    [[TMP22:%.*]] = load i32, i32* [[TMP20]], align 4
+; IC-NEXT:    [[TMP23:%.*]] = load i32, i32* [[TMP0]], align 4
+; IC-NEXT:    [[TMP24:%.*]] = icmp eq i32 [[TMP22]], [[TMP23]]
+; IC-NEXT:    br i1 [[TMP24]], label [[BB4_I_I:%.*]], label [[BB5_I_I:%.*]]
+; IC:       bb4.i.i:
+; IC-NEXT:    [[TMP25:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP26:%.*]] = load i32*, i32** [[TMP25]], align 8
+; IC-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC:       bb5.i.i:
+; IC-NEXT:    [[TMP27:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP28:%.*]] = load i32*, i32** [[TMP27]], align 8
+; IC-NEXT:    [[TMP29:%.*]] = getelementptr i32, i32* [[TMP28]], i32 1
+; IC-NEXT:    [[TMP30:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    store i32* [[TMP29]], i32** [[TMP30]], align 8
+; IC-NEXT:    [[TMP31:%.*]] = load i32, i32* [[TMP29]], align 4
+; IC-NEXT:    [[TMP32:%.*]] = load i32, i32* [[TMP0]], align 4
+; IC-NEXT:    [[TMP33:%.*]] = icmp eq i32 [[TMP31]], [[TMP32]]
+; IC-NEXT:    br i1 [[TMP33]], label [[BB7_I_I:%.*]], label [[BB8_I_I:%.*]]
+; IC:       bb7.i.i:
+; IC-NEXT:    [[TMP34:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP35:%.*]] = load i32*, i32** [[TMP34]], align 8
+; IC-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC:       bb8.i.i:
+; IC-NEXT:    [[TMP36:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP37:%.*]] = load i32*, i32** [[TMP36]], align 8
+; IC-NEXT:    [[TMP38:%.*]] = getelementptr i32, i32* [[TMP37]], i32 1
+; IC-NEXT:    [[TMP39:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    store i32* [[TMP38]], i32** [[TMP39]], align 8
+; IC-NEXT:    [[TMP40:%.*]] = load i32, i32* [[TMP38]], align 4
+; IC-NEXT:    [[TMP41:%.*]] = load i32, i32* [[TMP0]], align 4
+; IC-NEXT:    [[TMP42:%.*]] = icmp eq i32 [[TMP40]], [[TMP41]]
+; IC-NEXT:    br i1 [[TMP42]], label [[BB10_I_I:%.*]], label [[BB11_I_I:%.*]]
+; IC:       bb10.i.i:
+; IC-NEXT:    [[TMP43:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP44:%.*]] = load i32*, i32** [[TMP43]], align 8
+; IC-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC:       bb11.i.i:
+; IC-NEXT:    [[TMP45:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP46:%.*]] = load i32*, i32** [[TMP45]], align 8
+; IC-NEXT:    [[TMP47:%.*]] = getelementptr i32, i32* [[TMP46]], i32 1
+; IC-NEXT:    [[TMP48:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    store i32* [[TMP47]], i32** [[TMP48]], align 8
+; IC-NEXT:    [[TMP49:%.*]] = add i32 [[__TRIP_COUNT_0_I_I:%.*]], -1
+; IC-NEXT:    br label [[BB12_I_I]]
+; IC:       bb12.i.i:
+; IC-NEXT:    [[__TRIP_COUNT_0_I_I]] = phi i32 [ [[TMP10]], [[ENTRY:%.*]] ], [ [[TMP49]], [[BB11_I_I]] ]
+; IC-NEXT:    [[TMP50:%.*]] = icmp sgt i32 [[__TRIP_COUNT_0_I_I]], 0
+; IC-NEXT:    br i1 [[TMP50]], label [[BB_I_I:%.*]], label [[BB13_I_I:%.*]]
+; IC:       bb13.i.i:
+; IC-NEXT:    [[TMP51:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__LAST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP52:%.*]] = load i32*, i32** [[TMP51]], align 8
+; IC-NEXT:    [[TMP53:%.*]] = ptrtoint i32* [[TMP52]] to i32
+; IC-NEXT:    [[TMP54:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP55:%.*]] = load i32*, i32** [[TMP54]], align 8
+; IC-NEXT:    [[TMP56:%.*]] = ptrtoint i32* [[TMP55]] to i32
+; IC-NEXT:    [[TMP57:%.*]] = sub i32 [[TMP53]], [[TMP56]]
+; IC-NEXT:    [[TMP58:%.*]] = ashr i32 [[TMP57]], 2
+; IC-NEXT:    switch i32 [[TMP58]], label [[BB26_I_I:%.*]] [
+; IC-NEXT:    i32 1, label [[BB22_I_I:%.*]]
+; IC-NEXT:    i32 2, label [[BB18_I_I:%.*]]
+; IC-NEXT:    i32 3, label [[BB14_I_I:%.*]]
+; IC-NEXT:    ]
+; IC:       bb14.i.i:
+; IC-NEXT:    [[TMP59:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP60:%.*]] = load i32*, i32** [[TMP59]], align 8
+; IC-NEXT:    [[TMP61:%.*]] = load i32, i32* [[TMP60]], align 4
+; IC-NEXT:    [[TMP62:%.*]] = load i32, i32* [[TMP0]], align 4
+; IC-NEXT:    [[TMP63:%.*]] = icmp eq i32 [[TMP61]], [[TMP62]]
+; IC-NEXT:    br i1 [[TMP63]], label [[BB16_I_I:%.*]], label [[BB17_I_I:%.*]]
+; IC:       bb16.i.i:
+; IC-NEXT:    [[TMP64:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP65:%.*]] = load i32*, i32** [[TMP64]], align 8
+; IC-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC:       bb17.i.i:
+; IC-NEXT:    [[TMP66:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP67:%.*]] = load i32*, i32** [[TMP66]], align 8
+; IC-NEXT:    [[TMP68:%.*]] = getelementptr i32, i32* [[TMP67]], i32 1
+; IC-NEXT:    [[TMP69:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    store i32* [[TMP68]], i32** [[TMP69]], align 8
+; IC-NEXT:    br label [[BB18_I_I]]
+; IC:       bb18.i.i:
+; IC-NEXT:    [[TMP70:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP71:%.*]] = load i32*, i32** [[TMP70]], align 8
+; IC-NEXT:    [[TMP72:%.*]] = load i32, i32* [[TMP71]], align 4
+; IC-NEXT:    [[TMP73:%.*]] = load i32, i32* [[TMP0]], align 4
+; IC-NEXT:    [[TMP74:%.*]] = icmp eq i32 [[TMP72]], [[TMP73]]
+; IC-NEXT:    br i1 [[TMP74]], label [[BB20_I_I:%.*]], label [[BB21_I_I:%.*]]
+; IC:       bb20.i.i:
+; IC-NEXT:    [[TMP75:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP76:%.*]] = load i32*, i32** [[TMP75]], align 8
+; IC-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC:       bb21.i.i:
+; IC-NEXT:    [[TMP77:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP78:%.*]] = load i32*, i32** [[TMP77]], align 8
+; IC-NEXT:    [[TMP79:%.*]] = getelementptr i32, i32* [[TMP78]], i32 1
+; IC-NEXT:    [[TMP80:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    store i32* [[TMP79]], i32** [[TMP80]], align 8
+; IC-NEXT:    br label [[BB22_I_I]]
+; IC:       bb22.i.i:
+; IC-NEXT:    [[TMP81:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP82:%.*]] = load i32*, i32** [[TMP81]], align 8
+; IC-NEXT:    [[TMP83:%.*]] = load i32, i32* [[TMP82]], align 4
+; IC-NEXT:    [[TMP84:%.*]] = load i32, i32* [[TMP0]], align 4
+; IC-NEXT:    [[TMP85:%.*]] = icmp eq i32 [[TMP83]], [[TMP84]]
+; IC-NEXT:    br i1 [[TMP85]], label [[BB24_I_I:%.*]], label [[BB25_I_I:%.*]]
+; IC:       bb24.i.i:
+; IC-NEXT:    [[TMP86:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP87:%.*]] = load i32*, i32** [[TMP86]], align 8
+; IC-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC:       bb25.i.i:
+; IC-NEXT:    [[TMP88:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP89:%.*]] = load i32*, i32** [[TMP88]], align 8
+; IC-NEXT:    [[TMP90:%.*]] = getelementptr i32, i32* [[TMP89]], i32 1
+; IC-NEXT:    [[TMP91:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__FIRST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    store i32* [[TMP90]], i32** [[TMP91]], align 8
+; IC-NEXT:    br label [[BB26_I_I]]
+; IC:       bb26.i.i:
+; IC-NEXT:    [[TMP92:%.*]] = getelementptr inbounds %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* [[__LAST_ADDR_I_I]], i32 0, i32 0
+; IC-NEXT:    [[TMP93:%.*]] = load i32*, i32** [[TMP92]], align 8
+; IC-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC:       _ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit:
+; IC-NEXT:    [[DOT0_0_I_I:%.*]] = phi i32* [ [[TMP93]], [[BB26_I_I]] ], [ [[TMP87]], [[BB24_I_I]] ], [ [[TMP76]], [[BB20_I_I]] ], [ [[TMP65]], [[BB16_I_I]] ], [ [[TMP44]], [[BB10_I_I]] ], [ [[TMP35]], [[BB7_I_I]] ], [ [[TMP26]], [[BB4_I_I]] ], [ [[TMP17]], [[BB1_I_I]] ]
+; IC-NEXT:    br label [[RETURN:%.*]]
+; IC:       return:
+; IC-NEXT:    ret i32* [[DOT0_0_I_I]]
+;
+; IC_SROA-LABEL: @_Z3fooRSt6vectorIiSaIiEE(
+; IC_SROA-NEXT:  entry:
+; IC_SROA-NEXT:    [[TMP0:%.*]] = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* [[X:%.*]], i32 0, i32 0, i32 0, i32 1
+; IC_SROA-NEXT:    [[TMP1:%.*]] = load i32*, i32** [[TMP0]], align 4
+; IC_SROA-NEXT:    [[TMP2:%.*]] = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* [[X]], i32 0, i32 0, i32 0, i32 0
+; IC_SROA-NEXT:    [[TMP3:%.*]] = load i32*, i32** [[TMP2]], align 4
+; IC_SROA-NEXT:    [[TMP4:%.*]] = ptrtoint i32* [[TMP1]] to i32
+; IC_SROA-NEXT:    [[TMP5:%.*]] = ptrtoint i32* [[TMP3]] to i32
+; IC_SROA-NEXT:    [[TMP6:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+; IC_SROA-NEXT:    [[TMP7:%.*]] = ashr i32 [[TMP6]], 4
+; IC_SROA-NEXT:    br label [[BB12_I_I:%.*]]
+; IC_SROA:       bb.i.i:
+; IC_SROA-NEXT:    [[TMP8:%.*]] = load i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0:%.*]], align 4
+; IC_SROA-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 42
+; IC_SROA-NEXT:    br i1 [[TMP9]], label [[BB1_I_I:%.*]], label [[BB2_I_I:%.*]]
+; IC_SROA:       bb1.i.i:
+; IC_SROA-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT:%.*]]
+; IC_SROA:       bb2.i.i:
+; IC_SROA-NEXT:    [[TMP10:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0]], i32 1
+; IC_SROA-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP10]], align 4
+; IC_SROA-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 42
+; IC_SROA-NEXT:    br i1 [[TMP12]], label [[BB4_I_I:%.*]], label [[BB5_I_I:%.*]]
+; IC_SROA:       bb4.i.i:
+; IC_SROA-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC_SROA:       bb5.i.i:
+; IC_SROA-NEXT:    [[TMP13:%.*]] = getelementptr i32, i32* [[TMP10]], i32 1
+; IC_SROA-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4
+; IC_SROA-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP14]], 42
+; IC_SROA-NEXT:    br i1 [[TMP15]], label [[BB7_I_I:%.*]], label [[BB8_I_I:%.*]]
+; IC_SROA:       bb7.i.i:
+; IC_SROA-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC_SROA:       bb8.i.i:
+; IC_SROA-NEXT:    [[TMP16:%.*]] = getelementptr i32, i32* [[TMP13]], i32 1
+; IC_SROA-NEXT:    [[TMP17:%.*]] = load i32, i32* [[TMP16]], align 4
+; IC_SROA-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[TMP17]], 42
+; IC_SROA-NEXT:    br i1 [[TMP18]], label [[BB10_I_I:%.*]], label [[BB11_I_I:%.*]]
+; IC_SROA:       bb10.i.i:
+; IC_SROA-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC_SROA:       bb11.i.i:
+; IC_SROA-NEXT:    [[TMP19:%.*]] = getelementptr i32, i32* [[TMP16]], i32 1
+; IC_SROA-NEXT:    [[TMP20:%.*]] = add i32 [[__TRIP_COUNT_0_I_I:%.*]], -1
+; IC_SROA-NEXT:    br label [[BB12_I_I]]
+; IC_SROA:       bb12.i.i:
+; IC_SROA-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0]] = phi i32* [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TMP19]], [[BB11_I_I]] ]
+; IC_SROA-NEXT:    [[__TRIP_COUNT_0_I_I]] = phi i32 [ [[TMP7]], [[ENTRY]] ], [ [[TMP20]], [[BB11_I_I]] ]
+; IC_SROA-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[__TRIP_COUNT_0_I_I]], 0
+; IC_SROA-NEXT:    br i1 [[TMP21]], label [[BB_I_I:%.*]], label [[BB13_I_I:%.*]]
+; IC_SROA:       bb13.i.i:
+; IC_SROA-NEXT:    [[TMP22:%.*]] = ptrtoint i32* [[TMP1]] to i32
+; IC_SROA-NEXT:    [[TMP23:%.*]] = ptrtoint i32* [[__FIRST_ADDR_I_I_SROA_0_0]] to i32
+; IC_SROA-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+; IC_SROA-NEXT:    [[TMP25:%.*]] = ashr i32 [[TMP24]], 2
+; IC_SROA-NEXT:    switch i32 [[TMP25]], label [[BB26_I_I:%.*]] [
+; IC_SROA-NEXT:    i32 1, label [[BB22_I_I:%.*]]
+; IC_SROA-NEXT:    i32 2, label [[BB18_I_I:%.*]]
+; IC_SROA-NEXT:    i32 3, label [[BB14_I_I:%.*]]
+; IC_SROA-NEXT:    ]
+; IC_SROA:       bb14.i.i:
+; IC_SROA-NEXT:    [[TMP26:%.*]] = load i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0]], align 4
+; IC_SROA-NEXT:    [[TMP27:%.*]] = icmp eq i32 [[TMP26]], 42
+; IC_SROA-NEXT:    br i1 [[TMP27]], label [[BB16_I_I:%.*]], label [[BB17_I_I:%.*]]
+; IC_SROA:       bb16.i.i:
+; IC_SROA-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC_SROA:       bb17.i.i:
+; IC_SROA-NEXT:    [[TMP28:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0]], i32 1
+; IC_SROA-NEXT:    br label [[BB18_I_I]]
+; IC_SROA:       bb18.i.i:
+; IC_SROA-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_1:%.*]] = phi i32* [ [[TMP28]], [[BB17_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0_0]], [[BB13_I_I]] ]
+; IC_SROA-NEXT:    [[TMP29:%.*]] = load i32, i32* [[__FIRST_ADDR_I_I_SROA_0_1]], align 4
+; IC_SROA-NEXT:    [[TMP30:%.*]] = icmp eq i32 [[TMP29]], 42
+; IC_SROA-NEXT:    br i1 [[TMP30]], label [[BB20_I_I:%.*]], label [[BB21_I_I:%.*]]
+; IC_SROA:       bb20.i.i:
+; IC_SROA-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC_SROA:       bb21.i.i:
+; IC_SROA-NEXT:    [[TMP31:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_1]], i32 1
+; IC_SROA-NEXT:    br label [[BB22_I_I]]
+; IC_SROA:       bb22.i.i:
+; IC_SROA-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_2:%.*]] = phi i32* [ [[TMP31]], [[BB21_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0_0]], [[BB13_I_I]] ]
+; IC_SROA-NEXT:    [[TMP32:%.*]] = load i32, i32* [[__FIRST_ADDR_I_I_SROA_0_2]], align 4
+; IC_SROA-NEXT:    [[TMP33:%.*]] = icmp eq i32 [[TMP32]], 42
+; IC_SROA-NEXT:    br i1 [[TMP33]], label [[BB24_I_I:%.*]], label [[BB25_I_I:%.*]]
+; IC_SROA:       bb24.i.i:
+; IC_SROA-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC_SROA:       bb25.i.i:
+; IC_SROA-NEXT:    [[TMP34:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_2]], i32 1
+; IC_SROA-NEXT:    br label [[BB26_I_I]]
+; IC_SROA:       bb26.i.i:
+; IC_SROA-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; IC_SROA:       _ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit:
+; IC_SROA-NEXT:    [[DOT0_0_I_I:%.*]] = phi i32* [ [[TMP1]], [[BB26_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0_2]], [[BB24_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0_1]], [[BB20_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0_0]], [[BB16_I_I]] ], [ [[TMP16]], [[BB10_I_I]] ], [ [[TMP13]], [[BB7_I_I]] ], [ [[TMP10]], [[BB4_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0_0]], [[BB1_I_I]] ]
+; IC_SROA-NEXT:    br label [[RETURN:%.*]]
+; IC_SROA:       return:
+; IC_SROA-NEXT:    ret i32* [[DOT0_0_I_I]]
+;
 entry:
   %0 = alloca %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"
   %__first_addr.i.i = alloca %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"
@@ -179,9 +438,9 @@ bb13.i.i:                                         ; preds = %bb12.i.i
   %93 = sub i32 %89, %92
   %94 = ashr i32 %93, 2
   switch i32 %94, label %bb26.i.i [
-    i32 1, label %bb22.i.i
-    i32 2, label %bb18.i.i
-    i32 3, label %bb14.i.i
+  i32 1, label %bb22.i.i
+  i32 2, label %bb18.i.i
+  i32 3, label %bb14.i.i
   ]
 
 bb14.i.i:                                         ; preds = %bb13.i.i
diff --git a/llvm/test/Transforms/InstCombine/PR30597.ll b/llvm/test/Transforms/InstCombine/PR30597.ll
index a54fd68fd07ed5ab5e3cff22bcc879c82e846ccc..33b3695697de78190db9b8e3a1afa33a403fe314 100644
--- a/llvm/test/Transforms/InstCombine/PR30597.ll
+++ b/llvm/test/Transforms/InstCombine/PR30597.ll
@@ -23,7 +23,7 @@ entry-block:
 define i64* @function(i64* noalias nocapture readonly dereferenceable(8)) {
 ; CHECK-LABEL: @function(
 ; CHECK-NEXT:  entry-block:
-; CHECK-NEXT:    [[LOADED:%.*]] = load i64, i64* [[TMP0:%.*]], align 8, [[RNG0:!range !.*]]
+; CHECK-NEXT:    [[LOADED:%.*]] = load i64, i64* [[TMP0:%.*]], align 8, !range [[RNG0:![0-9]+]]
 ; CHECK-NEXT:    [[INTTOPTR:%.*]] = inttoptr i64 [[LOADED]] to i64*
 ; CHECK-NEXT:    ret i64* [[INTTOPTR]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/alloca.ll b/llvm/test/Transforms/InstCombine/alloca.ll
index 072d5f2fe9f94a7b46b70dd01d3e4bd899c1a532..fee8e0ef70f14bf19c3f03d7616b8cfdd9a3d7f8 100644
--- a/llvm/test/Transforms/InstCombine/alloca.ll
+++ b/llvm/test/Transforms/InstCombine/alloca.ll
@@ -207,7 +207,7 @@ define void @test8() {
 
 ; PR19569
 %struct_type = type { i32, i32 }
-declare void @test9_aux(<{ %struct_type }>* inalloca)
+declare void @test9_aux(<{ %struct_type }>* inalloca(<{ %struct_type }>))
 declare i8* @llvm.stacksave()
 declare void @llvm.stackrestore(i8*)
 
@@ -219,7 +219,7 @@ define void @test9(%struct_type* %a) {
 ; ALL-NEXT:    [[TMP0:%.*]] = bitcast %struct_type* [[A:%.*]] to i64*
 ; ALL-NEXT:    [[TMP1:%.*]] = load i64, i64* [[TMP0]], align 4
 ; ALL-NEXT:    store i64 [[TMP1]], i64* [[ARGMEM]], align 8
-; ALL-NEXT:    call void @test9_aux(<{ [[STRUCT_TYPE]] }>* inalloca nonnull [[TMPCAST]])
+; ALL-NEXT:    call void @test9_aux(<{ [[STRUCT_TYPE]] }>* nonnull inalloca(<{ [[STRUCT_TYPE]] }>) [[TMPCAST]])
 ; ALL-NEXT:    ret void
 ;
 entry:
@@ -229,7 +229,7 @@ entry:
   %1 = bitcast %struct_type* %0 to i8*
   %2 = bitcast %struct_type* %a to i8*
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %1, i8* align 4 %2, i32 8, i1 false)
-  call void @test9_aux(<{ %struct_type }>* inalloca %argmem)
+  call void @test9_aux(<{ %struct_type }>* inalloca(<{ %struct_type }>) %argmem)
   call void @llvm.stackrestore(i8* %inalloca.save)
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/annotation-intrinsic.ll b/llvm/test/Transforms/InstCombine/annotation-intrinsic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..aef68c9558be4457f92257e8087d6d8937723caa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/annotation-intrinsic.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine %s -S -o - | FileCheck %s
+
+; This tests that llvm.annotation does not prevent load combining.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+declare i32 @llvm.annotation.i32(i32, i8*, i8*, i32) #1
+
+define dso_local i32 @annotated(i32* %c) local_unnamed_addr #0 {
+; CHECK-LABEL: @annotated(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.annotation.i32(i32 [[TMP0]], i8* undef, i8* undef, i32 undef)
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+entry:
+  %0 = load i32, i32* %c, align 4
+  %1 = call i32 @llvm.annotation.i32(i32 %0, i8* undef, i8* undef, i32 undef)
+  %2 = load i32, i32* %c, align 4
+  %add = add nsw i32 %1, %2
+  ret i32 %add
+}
+
+attributes #0 = { nofree nounwind uwtable willreturn mustprogress }
diff --git a/llvm/test/Transforms/InstCombine/apint-call-cast-target.ll b/llvm/test/Transforms/InstCombine/apint-call-cast-target.ll
index f3a66c32414706b82518c36262a5a468ad2081e1..85a9f0982dfa17e3e970db2419008f6cb0d45830 100644
--- a/llvm/test/Transforms/InstCombine/apint-call-cast-target.ll
+++ b/llvm/test/Transforms/InstCombine/apint-call-cast-target.ll
@@ -1,23 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 target datalayout = "e-p:32:32"
 target triple = "i686-pc-linux-gnu"
 
-define i32 @main() {
-; CHECK-LABEL: @main(
-; CHECK: %[[call:.*]] = call i7* @ctime(i999* null)
-; CHECK: %[[cast:.*]] = ptrtoint i7* %[[call]] to i32
-; CHECK: ret i32 %[[cast]]
+declare i32 @main2()
+declare i7* @ctime2(i999*)
+
+define i7* @ctime(i999*) {
+; CHECK-LABEL: @ctime(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = call i32 @main2()
+; CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i32 [[I0]] to i7*
+; CHECK-NEXT:    ret i7* [[TMP1]]
+;
 entry:
-	%tmp = call i32 bitcast (i7* (i999*)* @ctime to i32 (i99*)*)( i99* null )
-	ret i32 %tmp
+  %i0 = call i7* bitcast (i32 ()* @main2 to i7* ()*)( )
+  ret i7* %i0
 }
 
-define i7* @ctime(i999*) {
-; CHECK-LABEL: define i7* @ctime(
-; CHECK: %[[call:.*]] = call i32 @main()
-; CHECK: %[[cast:.*]] = inttoptr i32 %[[call]] to i7*
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1:%.*]] = call i7* @ctime2(i999* null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint i7* [[I1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
 entry:
-	%tmp = call i7* bitcast (i32 ()* @main to i7* ()*)( )
-	ret i7* %tmp
+  %i1 = call i32 bitcast (i7* (i999*)* @ctime2 to i32 (i99*)*)( i99* null )
+  ret i32 %i1
 }
diff --git a/llvm/test/Transforms/InstCombine/call-cast-target-inalloca.ll b/llvm/test/Transforms/InstCombine/call-cast-target-inalloca.ll
index 90289e2468f85b161b012713cdbfbc40893232ca..bbf2008d585449843bf7edba2c777bc85afb2679 100644
--- a/llvm/test/Transforms/InstCombine/call-cast-target-inalloca.ll
+++ b/llvm/test/Transforms/InstCombine/call-cast-target-inalloca.ll
@@ -4,12 +4,12 @@ target datalayout = "e-p:32:32"
 target triple = "i686-pc-linux-gnu"
 
 declare void @takes_i32(i32)
-declare void @takes_i32_inalloca(i32* inalloca)
+declare void @takes_i32_inalloca(i32* inalloca(i32))
 
 define void @f() {
 ; CHECK-LABEL: define void @f()
   %args = alloca inalloca i32
-  call void bitcast (void (i32)* @takes_i32 to void (i32*)*)(i32* inalloca %args)
+  call void bitcast (void (i32)* @takes_i32 to void (i32*)*)(i32* inalloca(i32) %args)
 ; CHECK: call void bitcast
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/call-guard.ll b/llvm/test/Transforms/InstCombine/call-guard.ll
index e9ff9fa68155f9028401e894ccafdb717fbd07c6..c219300fdaf11865bc1626104590cec3b2c38c6f 100644
--- a/llvm/test/Transforms/InstCombine/call-guard.ll
+++ b/llvm/test/Transforms/InstCombine/call-guard.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -instcombine-infinite-loop-threshold=2 -S | FileCheck %s
 ; RUN: opt < %s -instcombine -S -debugify-each | FileCheck %s
 ; RUN: opt < %s -passes=instcombine -S -debugify-each | FileCheck %s
@@ -6,8 +7,9 @@ declare void @llvm.experimental.guard(i1, ...)
 
 define void @test_guard_adjacent_same_cond(i1 %A) {
 ; CHECK-LABEL: @test_guard_adjacent_same_cond(
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %A) [ "deopt"() ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[A:%.*]]) [ "deopt"() ]
 ; CHECK-NEXT:    ret void
+;
   call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
   call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
   call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
@@ -23,10 +25,11 @@ define void @test_guard_adjacent_same_cond(i1 %A) {
 
 define void @test_guard_adjacent_diff_cond(i1 %A, i1 %B, i1 %C) {
 ; CHECK-LABEL: @test_guard_adjacent_diff_cond(
-; CHECK-NEXT:    %1 = and i1 %A, %B
-; CHECK-NEXT:    %2 = and i1 %1, %C
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %2, i32 123) [ "deopt"() ]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 123) [ "deopt"() ]
 ; CHECK-NEXT:    ret void
+;
   call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
   call void(i1, ...) @llvm.experimental.guard( i1 %B, i32 456 )[ "deopt"() ]
   call void(i1, ...) @llvm.experimental.guard( i1 %C, i32 789 )[ "deopt"() ]
@@ -37,13 +40,14 @@ define void @test_guard_adjacent_diff_cond(i1 %A, i1 %B, i1 %C) {
 ; between the guards
 define void @test_guard_adjacent_diff_cond2(i32 %V1, i32 %V2) {
 ; CHECK-LABEL: @test_guard_adjacent_diff_cond2(
-; CHECK-NEXT:    %1 = and i32 %V1, %V2
-; CHECK-NEXT:    %2 = icmp slt i32 %1, 0
-; CHECK-NEXT:    %and = and i32 %V1, 255
-; CHECK-NEXT:    %C = icmp ult i32 %and, 129
-; CHECK-NEXT:    %3 = and i1 %2, %C
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %3, i32 123) [ "deopt"() ]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[V1]], 255
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[AND]], 129
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[C]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 123) [ "deopt"() ]
 ; CHECK-NEXT:    ret void
+;
   %A = icmp slt i32 %V1, 0
   call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
   %B = icmp slt i32 %V2, 0
@@ -57,9 +61,14 @@ define void @test_guard_adjacent_diff_cond2(i32 %V1, i32 %V2) {
 ; Might not be legal to hoist the load above the first guard since the
 ; guard might control dereferenceability
 define void @negative_load(i32 %V1, i32* %P) {
-; CHECK-LABEL: @negative_load
-; CHECK:    @llvm.experimental.guard
-; CHECK:    @llvm.experimental.guard
+; CHECK-LABEL: @negative_load(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i32 [[V1:%.*]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[A]], i32 123) [ "deopt"() ]
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[V2]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[B]], i32 456) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+;
   %A = icmp slt i32 %V1, 0
   call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
   %V2 = load i32, i32* %P
@@ -69,11 +78,13 @@ define void @negative_load(i32 %V1, i32* %P) {
 }
 
 define void @deref_load(i32 %V1, i32* dereferenceable(4) align 4 %P) {
-; CHECK-LABEL: @deref_load
-; CHECK-NEXT:  %V2 = load i32, i32* %P, align 4
-; CHECK-NEXT:  %1 = and i32 %V2, %V1
-; CHECK-NEXT:  %2 = icmp slt i32 %1, 0
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %2, i32 123) [ "deopt"() ]
+; CHECK-LABEL: @deref_load(
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[V2]], [[V1:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 123) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+;
   %A = icmp slt i32 %V1, 0
   call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
   %V2 = load i32, i32* %P
@@ -84,12 +95,17 @@ define void @deref_load(i32 %V1, i32* dereferenceable(4) align 4 %P) {
 
 ; The divide might fault above the guard
 define void @negative_div(i32 %V1, i32 %D) {
-; CHECK-LABEL: @negative_div
-; CHECK:    @llvm.experimental.guard
-; CHECK:    @llvm.experimental.guard
+; CHECK-LABEL: @negative_div(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i32 [[V1:%.*]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[A]], i32 123) [ "deopt"() ]
+; CHECK-NEXT:    [[V2:%.*]] = udiv i32 [[V1]], [[D:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[V2]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[B]], i32 456) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+;
   %A = icmp slt i32 %V1, 0
   call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
-  %V2 = udiv i32 %V1, %D 
+  %V2 = udiv i32 %V1, %D
   %B = icmp slt i32 %V2, 0
   call void(i1, ...) @llvm.experimental.guard( i1 %B, i32 456 )[ "deopt"() ]
   ret void
@@ -97,9 +113,16 @@ define void @negative_div(i32 %V1, i32 %D) {
 
 ; Highlight the limit of the window in a case which would otherwise be mergable
 define void @negative_window(i32 %V1, i32 %a, i32 %b, i32 %c, i32 %d) {
-; CHECK-LABEL: @negative_window
-; CHECK:    @llvm.experimental.guard
-; CHECK:    @llvm.experimental.guard
+; CHECK-LABEL: @negative_window(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i32 [[V1:%.*]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[A]], i32 123) [ "deopt"() ]
+; CHECK-NEXT:    [[V2:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[V3:%.*]] = add i32 [[V2]], [[C:%.*]]
+; CHECK-NEXT:    [[V4:%.*]] = add i32 [[V3]], [[D:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[V4]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[B]], i32 456) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+;
   %A = icmp slt i32 %V1, 0
   call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
   %V2 = add i32 %a, %b
diff --git a/llvm/test/Transforms/InstCombine/cast_ptr.ll b/llvm/test/Transforms/InstCombine/cast_ptr.ll
index 831d1b4de2f5543f0ace604ca398de17907ec3c5..1b46e3666c9d05879a154e462f3f488d094b6635 100644
--- a/llvm/test/Transforms/InstCombine/cast_ptr.ll
+++ b/llvm/test/Transforms/InstCombine/cast_ptr.ll
@@ -127,8 +127,8 @@ define i1 @test4_as2(i16 %A) {
 ; Pulling the cast out of the load allows us to eliminate the load, and then
 ; the whole array.
 
-        %op = type { float }
-        %unop = type { i32 }
+  %op = type { float }
+  %unop = type { i32 }
 @Array = internal constant [1 x %op* (%op*)*] [ %op* (%op*)* @foo ]
 
 declare %op* @foo(%op* %X)
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index 8a0035c8135fb7b24d0dfbfc5639da7531928a89..f3bdcf39bbd88a03635a8d6d33adf76328ad6f42 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -1305,4 +1305,23 @@ define i8* @D98588(i8* %c1, i64 %offset) {
   ret i8* %gep
 }
 
+declare noalias i8* @malloc(i64) nounwind
+
+define i32 @test_gep_bitcast_malloc(%struct.A* %a) {
+; CHECK-LABEL: @test_gep_bitcast_malloc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call noalias dereferenceable_or_null(16) i8* @malloc(i64 16)
+; CHECK-NEXT:    [[B:%.*]] = bitcast i8* [[CALL]] to %struct.A*
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr [[STRUCT_A:%.*]], %struct.A* [[B]], i64 0, i32 2
+; CHECK-NEXT:    [[A_C:%.*]] = load i32, i32* [[G3]], align 4
+; CHECK-NEXT:    ret i32 [[A_C]]
+;
+entry:
+  %call = call noalias i8* @malloc(i64 16) #2
+  %B = bitcast i8* %call to %struct.A*
+  %g3 = getelementptr %struct.A, %struct.A* %B, i32 0, i32 2
+  %a_c = load i32, i32* %g3, align 4
+  ret i32 %a_c
+}
+
 !0 = !{!"branch_weights", i32 2, i32 10}
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index 5e6bed4e280fa673bd252f0e997b331648ef1f0e..5f48684d04a4f78dbd5d17e71b8b38b9569671cb 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -3053,7 +3053,7 @@ define i1 @icmp_sle_zero_add_nsw(i32 %a) {
 
 define zeroext i1 @icmp_cmpxchg_strong(i32* %sc, i32 %old_val, i32 %new_val) {
 ; CHECK-LABEL: @icmp_cmpxchg_strong(
-; CHECK-NEXT:    [[XCHG:%.*]] = cmpxchg i32* [[SC:%.*]], i32 [[OLD_VAL:%.*]], i32 [[NEW_VAL:%.*]] seq_cst seq_cst
+; CHECK-NEXT:    [[XCHG:%.*]] = cmpxchg i32* [[SC:%.*]], i32 [[OLD_VAL:%.*]], i32 [[NEW_VAL:%.*]] seq_cst seq_cst, align 4
 ; CHECK-NEXT:    [[ICMP:%.*]] = extractvalue { i32, i1 } [[XCHG]], 1
 ; CHECK-NEXT:    ret i1 [[ICMP]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll b/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
index 859fd6437bb3063a248cc3eb11b4a5b3eb312f77..2ca2a45d5c3ff79d4f1baca7a66d134a5b28b13b 100644
--- a/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
+++ b/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
@@ -1,8 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instcombine -S  < %s | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64"
 
 define i32 *@test1(i32* %A, i32 %Offset) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[RHS_IDX:%.*]] = phi i32 [ [[RHS_ADD:%.*]], [[BB]] ], [ [[OFFSET:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[RHS_ADD]] = add nsw i32 [[RHS_IDX]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[RHS_IDX]], 100
+; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[RHS_PTR:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i32 [[RHS_IDX]]
+; CHECK-NEXT:    ret i32* [[RHS_PTR]]
+;
 entry:
   %tmp = getelementptr inbounds i32, i32* %A, i32 %Offset
   br label %bb
@@ -17,16 +30,22 @@ bb:
 bb2:
   ret i32* %RHS
 
-; CHECK-LABEL: @test1(
-; CHECK:  %[[INDEX:[0-9A-Za-z.]+]] = phi i32 [ %[[ADD:[0-9A-Za-z.]+]], %bb ], [ %Offset, %entry ]
-; CHECK:  %[[ADD]] = add nsw i32 %[[INDEX]], 1
-; CHECK:  %cond = icmp sgt i32 %[[INDEX]], 100
-; CHECK:  br i1 %cond, label %bb2, label %bb
-; CHECK:  %[[PTR:[0-9A-Za-z.]+]] = getelementptr inbounds i32, i32* %A, i32 %[[INDEX]]
-; CHECK:  ret i32* %[[PTR]]
 }
 
 define i32 *@test2(i32 %A, i32 %Offset) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[RHS_IDX:%.*]] = phi i32 [ [[RHS_ADD:%.*]], [[BB]] ], [ [[OFFSET:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[RHS_ADD]] = add nsw i32 [[RHS_IDX]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[RHS_IDX]], 100
+; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[RHSTO_PTR:%.*]] = inttoptr i32 [[A:%.*]] to i32*
+; CHECK-NEXT:    [[RHS_PTR:%.*]] = getelementptr inbounds i32, i32* [[RHSTO_PTR]], i32 [[RHS_IDX]]
+; CHECK-NEXT:    ret i32* [[RHS_PTR]]
+;
 entry:
   %A.ptr = inttoptr i32 %A to i32*
   %tmp = getelementptr inbounds i32, i32* %A.ptr, i32 %Offset
@@ -44,18 +63,23 @@ bb:
 bb2:
   ret i32* %RHS
 
-; CHECK-LABEL: @test2(
-; CHECK:  %[[INDEX:[0-9A-Za-z.]+]] = phi i32 [ %[[ADD:[0-9A-Za-z.]+]], %bb ], [ %Offset, %entry ]
-; CHECK:  %[[ADD]] = add nsw i32 %[[INDEX]], 1
-; CHECK:  %cond = icmp sgt i32 %[[INDEX]], 100
-; CHECK:  br i1 %cond, label %bb2, label %bb
-; CHECK:  %[[TOPTR:[0-9A-Za-z.]+]] = inttoptr i32 %[[ADD:[0-9A-Za-z.]+]] to i32*
-; CHECK:  %[[PTR:[0-9A-Za-z.]+]] = getelementptr inbounds i32, i32* %[[TOPTR]], i32 %[[INDEX]]
-; CHECK:  ret i32* %[[PTR]]
 }
 
 ; Perform the transformation only if we know that the GEPs used are inbounds.
 define i32 *@test3(i32* %A, i32 %Offset) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr i32, i32* [[A:%.*]], i32 [[OFFSET:%.*]]
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[RHS:%.*]] = phi i32* [ [[RHS_NEXT:%.*]], [[BB]] ], [ [[TMP]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LHS:%.*]] = getelementptr i32, i32* [[A]], i32 100
+; CHECK-NEXT:    [[RHS_NEXT]] = getelementptr i32, i32* [[RHS]], i32 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32* [[LHS]], [[RHS]]
+; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret i32* [[RHS]]
+;
 entry:
   %tmp = getelementptr i32, i32* %A, i32 %Offset
   br label %bb
@@ -70,14 +94,26 @@ bb:
 bb2:
   ret i32* %RHS
 
-; CHECK-LABEL: @test3(
-; CHECK-NOT:  %cond = icmp sgt i32 %{{[0-9A-Za-z.]+}}, 100
 }
 
 ; An inttoptr that requires an extension or truncation will be opaque when determining
 ; the base pointer. In this case we can still perform the transformation by considering
 ; A.ptr as being the base pointer.
 define i32 *@test4(i16 %A, i32 %Offset) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[RHS_IDX:%.*]] = phi i32 [ [[RHS_ADD:%.*]], [[BB]] ], [ [[OFFSET:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[RHS_ADD]] = add nsw i32 [[RHS_IDX]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[RHS_IDX]], 100
+; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[RHSTO_PTR:%.*]] = inttoptr i32 [[TMP0]] to i32*
+; CHECK-NEXT:    [[RHS_PTR:%.*]] = getelementptr inbounds i32, i32* [[RHSTO_PTR]], i32 [[RHS_IDX]]
+; CHECK-NEXT:    ret i32* [[RHS_PTR]]
+;
 entry:
   %A.ptr = inttoptr i16 %A to i32*
   %tmp = getelementptr inbounds i32, i32* %A.ptr, i32 %Offset
@@ -95,15 +131,32 @@ bb:
 bb2:
   ret i32* %RHS
 
-; CHECK-LABEL: @test4(
-; CHECK:  %cond = icmp sgt i32 %{{[0-9A-Za-z.]+}}, 100
 }
 
 declare i32* @fun_ptr()
 
 define i32 *@test5(i32 %Offset) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = invoke i32* @fun_ptr()
+; CHECK-NEXT:    to label [[CONT:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[RHS_IDX:%.*]] = phi i32 [ [[RHS_ADD:%.*]], [[BB]] ], [ [[OFFSET:%.*]], [[CONT]] ]
+; CHECK-NEXT:    [[RHS_ADD]] = add nsw i32 [[RHS_IDX]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[RHS_IDX]], 100
+; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[RHS_PTR:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[RHS_IDX]]
+; CHECK-NEXT:    ret i32* [[RHS_PTR]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[L:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    ret i32* null
+;
 entry:
- %A = invoke i32 *@fun_ptr() to label %cont unwind label %lpad
+  %A = invoke i32 *@fun_ptr() to label %cont unwind label %lpad
 
 cont:
   %tmp = getelementptr inbounds i32, i32* %A, i32 %Offset
@@ -123,20 +176,33 @@ lpad:
   %l = landingpad { i8*, i32 } cleanup
   ret i32* null
 
-; CHECK-LABEL: @test5(
-; CHECK:  %[[INDEX:[0-9A-Za-z.]+]] = phi i32 [ %[[ADD:[0-9A-Za-z.]+]], %bb ], [ %Offset, %cont ]
-; CHECK:  %[[ADD]] = add nsw i32 %[[INDEX]], 1
-; CHECK:  %cond = icmp sgt i32 %[[INDEX]], 100
-; CHECK:  br i1 %cond, label %bb2, label %bb
-; CHECK:  %[[PTR:[0-9A-Za-z.]+]] = getelementptr inbounds i32, i32* %A, i32 %[[INDEX]]
-; CHECK:  ret i32* %[[PTR]]
 }
 
 declare i32 @fun_i32()
 
 define i32 *@test6(i32 %Offset) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = invoke i32 @fun_i32()
+; CHECK-NEXT:    to label [[CONT:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[RHS_IDX:%.*]] = phi i32 [ [[RHS_ADD:%.*]], [[BB]] ], [ [[OFFSET:%.*]], [[CONT]] ]
+; CHECK-NEXT:    [[RHS_ADD]] = add nsw i32 [[RHS_IDX]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[RHS_IDX]], 100
+; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[RHSTO_PTR:%.*]] = inttoptr i32 [[A]] to i32*
+; CHECK-NEXT:    [[RHS_PTR:%.*]] = getelementptr inbounds i32, i32* [[RHSTO_PTR]], i32 [[RHS_IDX]]
+; CHECK-NEXT:    ret i32* [[RHS_PTR]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[L:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    ret i32* null
+;
 entry:
- %A = invoke i32 @fun_i32() to label %cont unwind label %lpad
+  %A = invoke i32 @fun_i32() to label %cont unwind label %lpad
 
 cont:
   %A.ptr = inttoptr i32 %A to i32*
@@ -157,19 +223,20 @@ lpad:
   %l = landingpad { i8*, i32 } cleanup
   ret i32* null
 
-; CHECK-LABEL: @test6(
-; CHECK:  %[[INDEX:[0-9A-Za-z.]+]] = phi i32 [ %[[ADD:[0-9A-Za-z.]+]], %bb ], [ %Offset, %cont ]
-; CHECK:  %[[ADD]] = add nsw i32 %[[INDEX]], 1
-; CHECK:  %cond = icmp sgt i32 %[[INDEX]], 100
-; CHECK:  br i1 %cond, label %bb2, label %bb
-; CHECK:  %[[TOPTR:[0-9A-Za-z.]+]] = inttoptr i32 %[[ADD:[0-9A-Za-z.]+]] to i32*
-; CHECK:  %[[PTR:[0-9A-Za-z.]+]] = getelementptr inbounds i32, i32* %[[TOPTR]], i32 %[[INDEX]]
-; CHECK:  ret i32* %[[PTR]]
 }
 
 
 @pr30402 = constant i64 3
 define i1 @test7() {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB7:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[CMP:%.*]] = phi i1 [ true, [[BB7]] ], [ false, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB10:%.*]], label [[BB7]]
+; CHECK:       bb10:
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
 entry:
   br label %bb7
 
@@ -181,26 +248,35 @@ bb7:                                              ; preds = %bb10, %entry-block
 bb10:
   ret i1 %cmp
 }
-; CHECK-LABEL: @test7(
-; CHECK:  %[[cmp:.*]] = phi i1 [ true, %bb7 ], [ false, %entry ]
-; CHECK: ret i1 %[[cmp]]
 
 
 declare i32 @__gxx_personality_v0(...)
 
 define i1 @test8(i64* %in, i64 %offset) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LD:%.*]] = load i64, i64* [[IN:%.*]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[LD]] to i32
+; CHECK-NEXT:    [[CASTI8:%.*]] = inttoptr i32 [[TMP0]] to i8*
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[OFFSET:%.*]] to i32
+; CHECK-NEXT:    [[GEPI8:%.*]] = getelementptr inbounds i8, i8* [[CASTI8]], i32 [[TMP1]]
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8* [[GEPI8]] to i32**
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[LD]] to i32
+; CHECK-NEXT:    [[PTRCAST:%.*]] = inttoptr i32 [[TMP2]] to i32**
+; CHECK-NEXT:    [[GEPI32:%.*]] = getelementptr inbounds i32*, i32** [[PTRCAST]], i32 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32** [[GEPI32]], [[CAST]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
 entry:
 
- %ld = load i64, i64* %in, align 8
- %casti8 = inttoptr i64 %ld to i8*
- %gepi8 = getelementptr inbounds i8, i8* %casti8, i64 %offset
- %cast = bitcast i8* %gepi8 to i32**
- %ptrcast = inttoptr i64 %ld to i32**
- %gepi32 = getelementptr inbounds i32*, i32** %ptrcast, i64 1
- %cmp = icmp eq i32** %gepi32, %cast
- ret i1 %cmp
+  %ld = load i64, i64* %in, align 8
+  %casti8 = inttoptr i64 %ld to i8*
+  %gepi8 = getelementptr inbounds i8, i8* %casti8, i64 %offset
+  %cast = bitcast i8* %gepi8 to i32**
+  %ptrcast = inttoptr i64 %ld to i32**
+  %gepi32 = getelementptr inbounds i32*, i32** %ptrcast, i64 1
+  %cmp = icmp eq i32** %gepi32, %cast
+  ret i1 %cmp
 
 
-; CHECK-LABEL: @test8(
-; CHECK-NOT: icmp eq i32 %{{[0-9A-Za-z.]+}}, 1
 }
diff --git a/llvm/test/Transforms/InstCombine/intptr1.ll b/llvm/test/Transforms/InstCombine/intptr1.ll
index 71cfb36bb328978e506f054104563f0395c41398..42fa9bf8b44dbc4ed5d8156fc39ec163437698ab 100644
--- a/llvm/test/Transforms/InstCombine/intptr1.ll
+++ b/llvm/test/Transforms/InstCombine/intptr1.ll
@@ -13,8 +13,8 @@ define void @test1(float* %a, float* readnone %a_end, i64* %b.i64) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[A_ADDR_03:%.*]] = phi float* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[B_ADDR_02_PTR:%.*]] = phi float* [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[B_PTR]], [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[B_ADDR_02_PTR]], align 4
-; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[TMP1]], 4.200000e+01
+; CHECK-NEXT:    [[I1:%.*]] = load float, float* [[B_ADDR_02_PTR]], align 4
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], float* [[A_ADDR_03]], align 4
 ; CHECK-NEXT:    [[ADD]] = getelementptr inbounds float, float* [[B_ADDR_02_PTR]], i64 1
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[A_ADDR_03]], i64 1
@@ -37,8 +37,8 @@ for.body:                                         ; preds = %for.body, %for.body
 
 
   %tmp = inttoptr i64 %b.addr.02 to float*
-  %tmp1 = load float, float* %tmp, align 4
-  %mul.i = fmul float %tmp1, 4.200000e+01
+  %i1 = load float, float* %tmp, align 4
+  %mul.i = fmul float %i1, 4.200000e+01
   store float %mul.i, float* %a.addr.03, align 4
   %add = getelementptr inbounds float, float* %tmp, i64 1
   %add.int = ptrtoint float* %add to i64
@@ -65,8 +65,8 @@ define void @test1_neg(float* %a, float* readnone %a_end, i64* %b.i64) {
 ; CHECK-NEXT:    [[PTRCMP:%.*]] = icmp ult float* [[TMP]], [[A_END]]
 ; CHECK-NEXT:    br i1 [[PTRCMP]], label [[FOR_END]], label [[BB]]
 ; CHECK:       bb:
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[A]], align 4
-; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[TMP1]], 4.200000e+01
+; CHECK-NEXT:    [[I1:%.*]] = load float, float* [[A]], align 4
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], float* [[A_ADDR_03]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = getelementptr inbounds float, float* [[A]], i64 1
 ; CHECK-NEXT:    [[ADD_INT]] = ptrtoint float* [[ADD]] to i64
@@ -94,8 +94,8 @@ for.body:                                         ; preds = %for.body, %for.body
   br i1 %ptrcmp, label %for.end, label %bb
 
 bb:
-  %tmp1 = load float, float* %a, align 4
-  %mul.i = fmul float %tmp1, 4.200000e+01
+  %i1 = load float, float* %a, align 4
+  %mul.i = fmul float %i1, 4.200000e+01
   store float %mul.i, float* %a.addr.03, align 4
   %add = getelementptr inbounds float, float* %a, i64 1
   %add.int = ptrtoint float* %add to i64
@@ -121,8 +121,8 @@ define void @test2(float* %a, float* readnone %a_end, float** %b.float) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[A_ADDR_03:%.*]] = phi float* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[B_ADDR_02_PTR:%.*]] = phi float* [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[B_PTR]], [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[B_ADDR_02_PTR]], align 4
-; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[TMP1]], 4.200000e+01
+; CHECK-NEXT:    [[I1:%.*]] = load float, float* [[B_ADDR_02_PTR]], align 4
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], float* [[A_ADDR_03]], align 4
 ; CHECK-NEXT:    [[ADD]] = getelementptr inbounds float, float* [[B_ADDR_02_PTR]], i64 1
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[A_ADDR_03]], i64 1
@@ -146,8 +146,8 @@ for.body:                                         ; preds = %for.body, %for.body
 
 
   %tmp = inttoptr i64 %b.addr.02 to float*
-  %tmp1 = load float, float* %tmp, align 4
-  %mul.i = fmul float %tmp1, 4.200000e+01
+  %i1 = load float, float* %tmp, align 4
+  %mul.i = fmul float %i1, 4.200000e+01
   store float %mul.i, float* %a.addr.03, align 4
   %add = getelementptr inbounds float, float* %tmp, i64 1
   %add.int = ptrtoint float* %add to i64
@@ -173,8 +173,8 @@ define void @test3(float* %a, float* readnone %a_end, i8** %b.i8p) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[A_ADDR_03:%.*]] = phi float* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[B_ADDR_02_PTR:%.*]] = phi float* [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[B_PTR]], [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[B_ADDR_02_PTR]], align 4
-; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[TMP1]], 4.200000e+01
+; CHECK-NEXT:    [[I1:%.*]] = load float, float* [[B_ADDR_02_PTR]], align 4
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], float* [[A_ADDR_03]], align 4
 ; CHECK-NEXT:    [[ADD]] = getelementptr inbounds float, float* [[B_ADDR_02_PTR]], i64 1
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[A_ADDR_03]], i64 1
@@ -198,8 +198,8 @@ for.body:                                         ; preds = %for.body, %for.body
 
 
   %tmp = inttoptr i64 %b.addr.02 to float*
-  %tmp1 = load float, float* %tmp, align 4
-  %mul.i = fmul float %tmp1, 4.200000e+01
+  %i1 = load float, float* %tmp, align 4
+  %mul.i = fmul float %i1, 4.200000e+01
   store float %mul.i, float* %a.addr.03, align 4
   %add = getelementptr inbounds float, float* %tmp, i64 1
   %add.int = ptrtoint float* %add to i64
@@ -223,8 +223,8 @@ define void @test4(float* %a, float* readnone %a_end, float** %b.float) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[A_ADDR_03:%.*]] = phi float* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[B_ADDR_02_IN:%.*]] = phi float* [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[B_F]], [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[B_ADDR_02_IN]], align 4
-; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[TMP1]], 4.200000e+01
+; CHECK-NEXT:    [[I1:%.*]] = load float, float* [[B_ADDR_02_IN]], align 4
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
 ; CHECK-NEXT:    store float [[MUL_I]], float* [[A_ADDR_03]], align 4
 ; CHECK-NEXT:    [[ADD]] = getelementptr inbounds float, float* [[B_ADDR_02_IN]], i64 1
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[A_ADDR_03]], i64 1
@@ -246,8 +246,8 @@ for.body:                                         ; preds = %for.body, %for.body
   %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
   %b.addr.02 = phi i64 [ %add.int, %for.body ], [ %b, %for.body.preheader ]
   %tmp = inttoptr i64 %b.addr.02 to float*
-  %tmp1 = load float, float* %tmp, align 4
-  %mul.i = fmul float %tmp1, 4.200000e+01
+  %i1 = load float, float* %tmp, align 4
+  %mul.i = fmul float %i1, 4.200000e+01
   store float %mul.i, float* %a.addr.03, align 4
   %add = getelementptr inbounds float, float* %tmp, i64 1
   %add.int = ptrtoint float* %add to i64
diff --git a/llvm/test/Transforms/InstCombine/intptr3.ll b/llvm/test/Transforms/InstCombine/intptr3.ll
index 72b81ce350f704a1e78afde0856a3655cfda67c4..84a0cb9229790a56fb72b6f3dbfc050f5ab6f56e 100644
--- a/llvm/test/Transforms/InstCombine/intptr3.ll
+++ b/llvm/test/Transforms/InstCombine/intptr3.ll
@@ -1,6 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s  -instcombine -S | FileCheck %s
 
 define  void @test(float* %a, float* readnone %a_end, i64 %b) unnamed_addr  {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult float* [[A:%.*]], [[A_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[B_FLOAT:%.*]] = inttoptr i64 [[B:%.*]] to float*
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[A_ADDR_03:%.*]] = phi float* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[A]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[B_ADDR_FLOAT:%.*]] = phi float* [ [[B_ADDR_FLOAT_INC:%.*]], [[FOR_BODY]] ], [ [[B_FLOAT]], [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[L:%.*]] = load float, float* [[B_ADDR_FLOAT]], align 4
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[L]], 4.200000e+01
+; CHECK-NEXT:    store float [[MUL_I]], float* [[A_ADDR_03]], align 4
+; CHECK-NEXT:    [[B_ADDR_FLOAT_INC]] = getelementptr inbounds float, float* [[B_ADDR_FLOAT]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[A_ADDR_03]], i64 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult float* [[INCDEC_PTR]], [[A_END]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp1 = icmp ult float* %a, %a_end
   br i1 %cmp1, label %for.body.preheader, label %for.end
@@ -13,21 +34,13 @@ for.body:                                         ; preds = %for.body.preheader,
   %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
   %b.addr.float = phi float* [ %b.addr.float.inc, %for.body ], [ %b.float, %for.body.preheader ]
   %b.addr.i64 = phi i64 [ %b.addr.i64.inc, %for.body ], [ %b, %for.body.preheader ]
-; CHECK: %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
-; CHECK-NEXT:  %b.addr.float = phi float* [ %b.addr.float.inc, %for.body ], [ %b.float, %for.body.preheader ]
-; CHECK-NEXT: = load float
-  %l = load float, float* %b.addr.float, align 4 
+  %l = load float, float* %b.addr.float, align 4
   %mul.i = fmul float %l, 4.200000e+01
   store float %mul.i, float* %a.addr.03, align 4
-; CHECK: store float
   %b.addr.float.2 = inttoptr i64 %b.addr.i64 to float*
-; CHECK-NOT: inttoptr
   %b.addr.float.inc = getelementptr inbounds float, float* %b.addr.float.2, i64 1
-; CHECK: %b.addr.float.inc = 
   %b.addr.i64.inc = ptrtoint float* %b.addr.float.inc to i64
-; CHECK-NOT: ptrtoint
   %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
-; CHECK: %incdec.ptr = 
   %cmp = icmp ult float* %incdec.ptr, %a_end
   br i1 %cmp, label %for.body, label %for.end
 
diff --git a/llvm/test/Transforms/InstCombine/intptr7.ll b/llvm/test/Transforms/InstCombine/intptr7.ll
index 1e83bacd65bb2b74ce57ae4398fc6ebe1338bc6e..ee5825f2f9885c256ef46a95abbfbf49132d720b 100644
--- a/llvm/test/Transforms/InstCombine/intptr7.ll
+++ b/llvm/test/Transforms/InstCombine/intptr7.ll
@@ -1,7 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s  -instcombine -S | FileCheck %s
 
 define void @matching_phi(i64 %a, float* %b, i1 %cond) {
-; CHECK-LABEL: @matching_phi
+; CHECK-LABEL: @matching_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_INT:%.*]] = add i64 [[A:%.*]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = inttoptr i64 [[ADD_INT]] to float*
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[BBB:%.*]], label [[A:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    [[ADDB:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 2
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       Bbb:
+; CHECK-NEXT:    store float 1.000000e+01, float* [[ADD]], align 4
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       C:
+; CHECK-NEXT:    [[A_ADDR_03:%.*]] = phi float* [ [[ADDB]], [[A]] ], [ [[ADD]], [[BBB]] ]
+; CHECK-NEXT:    [[I1:%.*]] = load float, float* [[A_ADDR_03]], align 4
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
+; CHECK-NEXT:    store float [[MUL_I]], float* [[A_ADDR_03]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp1 = icmp  eq i1 %cond, 0
   %add.int = add i64 %a, 1
@@ -9,27 +27,43 @@ entry:
 
   %addb = getelementptr inbounds float, float* %b, i64 2
   %addb.int = ptrtoint float* %addb to i64
-  br i1 %cmp1, label %A, label %B
+  br i1 %cmp1, label %A, label %Bbb
 A:
   br label %C
-B:
+Bbb:
   store float 1.0e+01, float* %add, align 4
   br label %C
 
 C:
-  %a.addr.03 = phi float* [ %addb, %A ], [ %add, %B ]
-  %b.addr.02 = phi i64 [ %addb.int, %A ], [ %add.int, %B ]
-  %tmp = inttoptr i64 %b.addr.02 to float*
-; CHECK: %a.addr.03 = phi
-; CHECK-NEXT: = load
-  %tmp1 = load float, float* %tmp, align 4
-  %mul.i = fmul float %tmp1, 4.200000e+01
+  %a.addr.03 = phi float* [ %addb, %A ], [ %add, %Bbb ]
+  %b.addr.02 = phi i64 [ %addb.int, %A ], [ %add.int, %Bbb ]
+  %i0 = inttoptr i64 %b.addr.02 to float*
+  %i1 = load float, float* %i0, align 4
+  %mul.i = fmul float %i1, 4.200000e+01
   store float %mul.i, float* %a.addr.03, align 4
   ret void
 }
 
 define void @no_matching_phi(i64 %a, float* %b, i1 %cond) {
-; CHECK-LABEL: @no_matching_phi
+; CHECK-LABEL: @no_matching_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD_INT:%.*]] = add i64 [[A:%.*]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = inttoptr i64 [[ADD_INT]] to float*
+; CHECK-NEXT:    [[ADDB:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 2
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[B:%.*]], label [[A:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    store float 1.000000e+01, float* [[ADD]], align 4
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       C:
+; CHECK-NEXT:    [[A_ADDR_03:%.*]] = phi float* [ [[ADDB]], [[A]] ], [ [[ADD]], [[B]] ]
+; CHECK-NEXT:    [[B_ADDR_02_PTR:%.*]] = phi float* [ [[ADD]], [[A]] ], [ [[ADDB]], [[B]] ]
+; CHECK-NEXT:    [[I1:%.*]] = load float, float* [[B_ADDR_02_PTR]], align 4
+; CHECK-NEXT:    [[MUL_I:%.*]] = fmul float [[I1]], 4.200000e+01
+; CHECK-NEXT:    store float [[MUL_I]], float* [[A_ADDR_03]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp1 = icmp  eq i1 %cond, 0
   %add.int = add i64 %a, 1
@@ -47,12 +81,9 @@ B:
 C:
   %a.addr.03 = phi float* [ %addb, %A ], [ %add, %B ]
   %b.addr.02 = phi i64 [ %addb.int, %B ], [ %add.int, %A ]
-  %tmp = inttoptr i64 %b.addr.02 to float*
-  %tmp1 = load float, float* %tmp, align 4
-; CHECK: %a.addr.03 = phi
-; CHECK-NEXT: %b.addr.02.ptr = phi
-; CHECK-NEXT: = load
-  %mul.i = fmul float %tmp1, 4.200000e+01
+  %i0 = inttoptr i64 %b.addr.02 to float*
+  %i1 = load float, float* %i0, align 4
+  %mul.i = fmul float %i1, 4.200000e+01
   store float %mul.i, float* %a.addr.03, align 4
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/load-no-aliasing.ll b/llvm/test/Transforms/InstCombine/load-no-aliasing.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c4e0bc002bec273caaf4101ade752ff9b48316be
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-no-aliasing.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -tbaa -instcombine -S < %s | FileCheck %s
+
+; Check that load to load forwarding works with non aliasing store inbetween.
+define i32 @test_load_store_load_combine(i32*, float*) {
+; CHECK-LABEL: @test_load_store_load_combine(
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[TMP0:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT:    [[F:%.*]] = sitofp i32 [[A]] to float
+; CHECK-NEXT:    store float [[F]], float* [[TMP1:%.*]], align 4, !tbaa [[TBAA4:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = load i32, i32* %0, align 4, !tbaa !0
+  %f = sitofp i32 %a to float
+  store float %f, float* %1, align 4, !tbaa !4
+  %b = load i32, i32* %0, align 4, !tbaa !0
+  ret i32 %b
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"int", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C++ TBAA"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"float", !2, i64 0}
diff --git a/llvm/test/Transforms/InstCombine/minmax-fold.ll b/llvm/test/Transforms/InstCombine/minmax-fold.ll
index 78126302f9b9ffde7b0a89701047682314a76756..7aa3a3992449a08d66d33fb2d99cfc747231db87 100644
--- a/llvm/test/Transforms/InstCombine/minmax-fold.ll
+++ b/llvm/test/Transforms/InstCombine/minmax-fold.ll
@@ -1080,7 +1080,7 @@ define i37 @add_umax_constant_limit(i37 %x) {
 
 define i37 @add_umax_simplify(i37 %x) {
 ; CHECK-LABEL: @add_umax_simplify(
-; CHECK-NEXT:    [[A:%.*]] = add i37 [[X:%.*]], 42
+; CHECK-NEXT:    [[A:%.*]] = add nuw i37 [[X:%.*]], 42
 ; CHECK-NEXT:    ret i37 [[A]]
 ;
   %a = add nuw i37 %x, 42
diff --git a/llvm/test/Transforms/InstCombine/multi-size-address-space-pointer.ll b/llvm/test/Transforms/InstCombine/multi-size-address-space-pointer.ll
index 4e5b2100f34ab06c95fbaf3674795219da0d2e98..b93e19ac258d5b662bd213e4e7ba3983b45890af 100644
--- a/llvm/test/Transforms/InstCombine/multi-size-address-space-pointer.ll
+++ b/llvm/test/Transforms/InstCombine/multi-size-address-space-pointer.ll
@@ -1,10 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -instcombine %s -o - | FileCheck %s
 target datalayout = "e-p:32:32:32-p1:64:64:64-p2:8:8:8-p3:16:16:16-p4:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32"
 
 
 define i32 @test_as0(i32 addrspace(0)* %a) {
 ; CHECK-LABEL: @test_as0(
-; CHECK: %arrayidx = getelementptr i32, i32* %a, i32 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32* [[A:%.*]], i32 1
+; CHECK-NEXT:    [[Y:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret i32 [[Y]]
+;
   %arrayidx = getelementptr i32, i32 addrspace(0)* %a, i64 1
   %y = load i32, i32 addrspace(0)* %arrayidx, align 4
   ret i32 %y
@@ -12,7 +16,10 @@ define i32 @test_as0(i32 addrspace(0)* %a) {
 
 define i32 @test_as1(i32 addrspace(1)* %a) {
 ; CHECK-LABEL: @test_as1(
-; CHECK: %arrayidx = getelementptr i32, i32 addrspace(1)* %a, i64 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32 addrspace(1)* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[Y:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret i32 [[Y]]
+;
   %arrayidx = getelementptr i32, i32 addrspace(1)* %a, i32 1
   %y = load i32, i32 addrspace(1)* %arrayidx, align 4
   ret i32 %y
@@ -20,7 +27,10 @@ define i32 @test_as1(i32 addrspace(1)* %a) {
 
 define i32 @test_as2(i32 addrspace(2)* %a) {
 ; CHECK-LABEL: @test_as2(
-; CHECK: %arrayidx = getelementptr i32, i32 addrspace(2)* %a, i8 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32 addrspace(2)* [[A:%.*]], i8 1
+; CHECK-NEXT:    [[Y:%.*]] = load i32, i32 addrspace(2)* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret i32 [[Y]]
+;
   %arrayidx = getelementptr i32, i32 addrspace(2)* %a, i32 1
   %y = load i32, i32 addrspace(2)* %arrayidx, align 4
   ret i32 %y
@@ -28,7 +38,10 @@ define i32 @test_as2(i32 addrspace(2)* %a) {
 
 define i32 @test_as3(i32 addrspace(3)* %a) {
 ; CHECK-LABEL: @test_as3(
-; CHECK: %arrayidx = getelementptr i32, i32 addrspace(3)* %a, i16 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32 addrspace(3)* [[A:%.*]], i16 1
+; CHECK-NEXT:    [[Y:%.*]] = load i32, i32 addrspace(3)* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret i32 [[Y]]
+;
   %arrayidx = getelementptr i32, i32 addrspace(3)* %a, i32 1
   %y = load i32, i32 addrspace(3)* %arrayidx, align 4
   ret i32 %y
@@ -36,8 +49,9 @@ define i32 @test_as3(i32 addrspace(3)* %a) {
 
 define i32 @test_combine_ptrtoint(i32 addrspace(2)* %a) {
 ; CHECK-LABEL: @test_combine_ptrtoint(
-; CHECK-NEXT: %y = load i32, i32 addrspace(2)* %a
-; CHECK-NEXT: ret i32 %y
+; CHECK-NEXT:    [[Y:%.*]] = load i32, i32 addrspace(2)* [[A:%.*]], align 4
+; CHECK-NEXT:    ret i32 [[Y]]
+;
   %cast = ptrtoint i32 addrspace(2)* %a to i8
   %castback = inttoptr i8 %cast to i32 addrspace(2)*
   %y = load i32, i32 addrspace(2)* %castback, align 4
@@ -46,7 +60,8 @@ define i32 @test_combine_ptrtoint(i32 addrspace(2)* %a) {
 
 define i8 @test_combine_inttoptr(i8 %a) {
 ; CHECK-LABEL: @test_combine_inttoptr(
-; CHECK-NEXT: ret i8 %a
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
   %cast = inttoptr i8 %a to i32 addrspace(2)*
   %castback = ptrtoint i32 addrspace(2)* %cast to i8
   ret i8 %castback
@@ -54,9 +69,10 @@ define i8 @test_combine_inttoptr(i8 %a) {
 
 define i32 @test_combine_vector_ptrtoint(<2 x i32 addrspace(2)*> %a) {
 ; CHECK-LABEL: @test_combine_vector_ptrtoint(
-; CHECK-NEXT: %p = extractelement <2 x i32 addrspace(2)*> %a, i32 0
-; CHECK-NEXT: %y = load i32, i32 addrspace(2)* %p, align 4
-; CHECK-NEXT: ret i32 %y
+; CHECK-NEXT:    [[P:%.*]] = extractelement <2 x i32 addrspace(2)*> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[Y:%.*]] = load i32, i32 addrspace(2)* [[P]], align 4
+; CHECK-NEXT:    ret i32 [[Y]]
+;
   %cast = ptrtoint <2 x i32 addrspace(2)*> %a to <2 x i8>
   %castback = inttoptr <2 x i8> %cast to <2 x i32 addrspace(2)*>
   %p = extractelement <2 x i32 addrspace(2)*> %castback, i32 0
@@ -66,7 +82,8 @@ define i32 @test_combine_vector_ptrtoint(<2 x i32 addrspace(2)*> %a) {
 
 define <2 x i8> @test_combine_vector_inttoptr(<2 x i8> %a) {
 ; CHECK-LABEL: @test_combine_vector_inttoptr(
-; CHECK-NEXT: ret <2 x i8> %a
+; CHECK-NEXT:    ret <2 x i8> [[A:%.*]]
+;
   %cast = inttoptr <2 x i8> %a to <2 x i32 addrspace(2)*>
   %castback = ptrtoint <2 x i32 addrspace(2)*> %cast to <2 x i8>
   ret <2 x i8> %castback
@@ -75,37 +92,47 @@ define <2 x i8> @test_combine_vector_inttoptr(<2 x i8> %a) {
 ; Check that the GEP index is changed to the address space integer type (i64 -> i8)
 define i32 addrspace(2)* @shrink_gep_constant_index_64_as2(i32 addrspace(2)* %p) {
 ; CHECK-LABEL: @shrink_gep_constant_index_64_as2(
-; CHECK-NEXT: getelementptr i32, i32 addrspace(2)* %p, i8 1
+; CHECK-NEXT:    [[RET:%.*]] = getelementptr i32, i32 addrspace(2)* [[P:%.*]], i8 1
+; CHECK-NEXT:    ret i32 addrspace(2)* [[RET]]
+;
   %ret = getelementptr i32, i32 addrspace(2)* %p, i64 1
   ret i32 addrspace(2)* %ret
 }
 
 define i32 addrspace(2)* @shrink_gep_constant_index_32_as2(i32 addrspace(2)* %p) {
 ; CHECK-LABEL: @shrink_gep_constant_index_32_as2(
-; CHECK-NEXT: getelementptr i32, i32 addrspace(2)* %p, i8 1
+; CHECK-NEXT:    [[RET:%.*]] = getelementptr i32, i32 addrspace(2)* [[P:%.*]], i8 1
+; CHECK-NEXT:    ret i32 addrspace(2)* [[RET]]
+;
   %ret = getelementptr i32, i32 addrspace(2)* %p, i32 1
   ret i32 addrspace(2)* %ret
 }
 
 define i32 addrspace(3)* @shrink_gep_constant_index_64_as3(i32 addrspace(3)* %p) {
 ; CHECK-LABEL: @shrink_gep_constant_index_64_as3(
-; CHECK-NEXT: getelementptr i32, i32 addrspace(3)* %p, i16 1
+; CHECK-NEXT:    [[RET:%.*]] = getelementptr i32, i32 addrspace(3)* [[P:%.*]], i16 1
+; CHECK-NEXT:    ret i32 addrspace(3)* [[RET]]
+;
   %ret = getelementptr i32, i32 addrspace(3)* %p, i64 1
   ret i32 addrspace(3)* %ret
 }
 
 define i32 addrspace(2)* @shrink_gep_variable_index_64_as2(i32 addrspace(2)* %p, i64 %idx) {
 ; CHECK-LABEL: @shrink_gep_variable_index_64_as2(
-; CHECK-NEXT: %1 = trunc i64 %idx to i8
-; CHECK-NEXT: getelementptr i32, i32 addrspace(2)* %p, i8 %1
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[IDX:%.*]] to i8
+; CHECK-NEXT:    [[RET:%.*]] = getelementptr i32, i32 addrspace(2)* [[P:%.*]], i8 [[TMP1]]
+; CHECK-NEXT:    ret i32 addrspace(2)* [[RET]]
+;
   %ret = getelementptr i32, i32 addrspace(2)* %p, i64 %idx
   ret i32 addrspace(2)* %ret
 }
 
 define i32 addrspace(1)* @grow_gep_variable_index_8_as1(i32 addrspace(1)* %p, i8 %idx) {
 ; CHECK-LABEL: @grow_gep_variable_index_8_as1(
-; CHECK-NEXT: %1 = sext i8 %idx to i64
-; CHECK-NEXT: getelementptr i32, i32 addrspace(1)* %p, i64 %1
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i8 [[IDX:%.*]] to i64
+; CHECK-NEXT:    [[RET:%.*]] = getelementptr i32, i32 addrspace(1)* [[P:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    ret i32 addrspace(1)* [[RET]]
+;
   %ret = getelementptr i32, i32 addrspace(1)* %p, i8 %idx
   ret i32 addrspace(1)* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-bitcast-inseltpoison.ll b/llvm/test/Transforms/InstCombine/scalable-bitcast-inseltpoison.ll
similarity index 85%
rename from llvm/test/Transforms/InstCombine/AArch64/sve-bitcast-inseltpoison.ll
rename to llvm/test/Transforms/InstCombine/scalable-bitcast-inseltpoison.ll
index d0cc892b0e692d6df7637f472e335cc259cd6fcd..6c6f770add1dd8ab01c70e5e637acbba7fe06fa7 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-bitcast-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/scalable-bitcast-inseltpoison.ll
@@ -1,4 +1,4 @@
-; RUN: opt -instcombine -mtriple=aarch64-linux-gnu -mattr=+sve -S < %s | FileCheck %s
+; RUN: opt -instcombine -S < %s | FileCheck %s
 
 ; We shouldn't fold bitcast(insert <vscale x 1 x iX> .., iX %val, i32 0)
 ; into bitcast(iX %val) for scalable vectors.
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-bitcast.ll b/llvm/test/Transforms/InstCombine/scalable-bitcast.ll
similarity index 85%
rename from llvm/test/Transforms/InstCombine/AArch64/sve-bitcast.ll
rename to llvm/test/Transforms/InstCombine/scalable-bitcast.ll
index 8049cad596b501cfceddc3665bd97fad0db1c7cc..d6980b9d74060d80e9965524f49bcbace5005239 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-bitcast.ll
+++ b/llvm/test/Transforms/InstCombine/scalable-bitcast.ll
@@ -1,4 +1,4 @@
-; RUN: opt -instcombine -mtriple=aarch64-linux-gnu -mattr=+sve -S < %s | FileCheck %s
+; RUN: opt -instcombine -S < %s | FileCheck %s
 
 ; We shouldn't fold bitcast(insert <vscale x 1 x iX> .., iX %val, i32 0)
 ; into bitcast(iX %val) for scalable vectors.
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-cast-of-alloc.ll b/llvm/test/Transforms/InstCombine/scalable-cast-of-alloc.ll
similarity index 98%
rename from llvm/test/Transforms/InstCombine/AArch64/sve-cast-of-alloc.ll
rename to llvm/test/Transforms/InstCombine/scalable-cast-of-alloc.ll
index b593770c1ccbfafaac0a0169d5f2ba2a4a8d523c..e264b69698fde2eb6c756ebfc8f07a8008ef6132 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-cast-of-alloc.ll
+++ b/llvm/test/Transforms/InstCombine/scalable-cast-of-alloc.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -instcombine -mtriple aarch64-linux-gnu -mattr=+sve -S < %s 2>%t | FileCheck %s
+; RUN: opt -instcombine -S < %s 2>%t | FileCheck %s
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
 ; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
diff --git a/llvm/test/Transforms/InstCombine/scalable-const-fp-splat.ll b/llvm/test/Transforms/InstCombine/scalable-const-fp-splat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..09156d614bb29648929cb5fb7b239f94db57339a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/scalable-const-fp-splat.ll
@@ -0,0 +1,17 @@
+; RUN: opt -instcombine -S -o - < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+define <vscale x 2 x float> @shrink_splat_scalable_extend(<vscale x 2 x float> %a) {
+  ; CHECK-LABEL: @shrink_splat_scalable_extend
+  ; CHECK-NEXT:  %[[FADD:.*]] = fadd <vscale x 2 x float> %a, shufflevector (<vscale x 2 x float> insertelement (<vscale x 2 x float> undef, float -1.000000e+00, i32 0), <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer)
+  ; CHECK-NEXT:  ret <vscale x 2 x float> %[[FADD]]
+  %1 = shufflevector <vscale x 2 x float> insertelement (<vscale x 2 x float> undef, float -1.000000e+00, i32 0), <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
+  %2 = fpext <vscale x 2 x float> %a to <vscale x 2 x double>
+  %3 = fpext <vscale x 2 x float> %1 to <vscale x 2 x double>
+  %4 = fadd <vscale x 2 x double> %2, %3
+  %5 = fptrunc <vscale x 2 x double> %4 to <vscale x 2 x float>
+  ret <vscale x 2 x float> %5
+}
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-trunc.ll b/llvm/test/Transforms/InstCombine/scalable-trunc.ll
similarity index 94%
rename from llvm/test/Transforms/InstCombine/AArch64/sve-trunc.ll
rename to llvm/test/Transforms/InstCombine/scalable-trunc.ll
index d18beb5dbdf40594afbc33cf89cab1288a7904c1..c64b2997d273911a5e7157155d6604115a0b5174 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-trunc.ll
+++ b/llvm/test/Transforms/InstCombine/scalable-trunc.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple aarch64-linux-gnu -mattr=+sve -instcombine -S < %s 2>%t | FileCheck %s
+; RUN: opt -instcombine -S < %s 2>%t | FileCheck %s
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
 ; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
diff --git a/llvm/test/Transforms/InstCombine/select-min-max.ll b/llvm/test/Transforms/InstCombine/select-min-max.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d4e6ff92a0c4f7fb09097b1974da1139eb0fe80b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-min-max.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare i5 @llvm.smin.i5(i5, i5)
+declare <2 x i8> @llvm.smax.v2i8(<2 x i8>, <2 x i8>)
+declare i5 @llvm.umin.i5(i5, i5)
+declare <3 x i5> @llvm.umax.v3i5(<3 x i5>, <3 x i5>)
+
+define i5 @smin_smin_common_op_00(i1 %cond, i5 %x, i5 %y, i5 %z) {
+; CHECK-LABEL: @smin_smin_common_op_00(
+; CHECK-NEXT:    [[MINMAXOP:%.*]] = select i1 [[COND:%.*]], i5 [[X:%.*]], i5 [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = call i5 @llvm.smin.i5(i5 [[MINMAXOP]], i5 [[Z:%.*]])
+; CHECK-NEXT:    ret i5 [[SEL]]
+;
+  %m1 = call i5 @llvm.smin.i5(i5 %z, i5 %x)
+  %m2 = call i5 @llvm.smin.i5(i5 %z, i5 %y)
+  %sel = select i1 %cond, i5 %m1, i5 %m2
+  ret i5 %sel
+}
+
+define <2 x i8> @smax_smax_common_op_01(<2 x i1> %cond, <2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @smax_smax_common_op_01(
+; CHECK-NEXT:    [[MINMAXOP:%.*]] = select <2 x i1> [[COND:%.*]], <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = call <2 x i8> @llvm.smax.v2i8(<2 x i8> [[MINMAXOP]], <2 x i8> [[Z:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[SEL]]
+;
+  %m1 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %z, <2 x i8> %x)
+  %m2 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> %y, <2 x i8> %z)
+  %sel = select <2 x i1> %cond, <2 x i8> %m1, <2 x i8> %m2
+  ret <2 x i8> %sel
+}
+
+define i5 @umin_umin_common_op_10(i1 %cond, i5 %x, i5 %y, i5 %z, i5* %p) {
+; CHECK-LABEL: @umin_umin_common_op_10(
+; CHECK-NEXT:    [[M1:%.*]] = call i5 @llvm.umin.i5(i5 [[X:%.*]], i5 [[Z:%.*]])
+; CHECK-NEXT:    store i5 [[M1]], i5* [[P:%.*]], align 1
+; CHECK-NEXT:    [[MINMAXOP:%.*]] = select i1 [[COND:%.*]], i5 [[X]], i5 [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = call i5 @llvm.umin.i5(i5 [[MINMAXOP]], i5 [[Z]])
+; CHECK-NEXT:    ret i5 [[SEL]]
+;
+  %m1 = call i5 @llvm.umin.i5(i5 %x, i5 %z)
+  store i5 %m1, i5* %p
+  %m2 = call i5 @llvm.umin.i5(i5 %z, i5 %y)
+  %sel = select i1 %cond, i5 %m1, i5 %m2
+  ret i5 %sel
+}
+
+define <3 x i5> @umax_umax_common_op_11(i1 %cond, <3 x i5> %x, <3 x i5> %y, <3 x i5> %z, <3 x i5>* %p) {
+; CHECK-LABEL: @umax_umax_common_op_11(
+; CHECK-NEXT:    [[M2:%.*]] = call <3 x i5> @llvm.umax.v3i5(<3 x i5> [[Y:%.*]], <3 x i5> [[Z:%.*]])
+; CHECK-NEXT:    store <3 x i5> [[M2]], <3 x i5>* [[P:%.*]], align 4
+; CHECK-NEXT:    [[MINMAXOP:%.*]] = select i1 [[COND:%.*]], <3 x i5> [[X:%.*]], <3 x i5> [[Y]]
+; CHECK-NEXT:    [[SEL:%.*]] = call <3 x i5> @llvm.umax.v3i5(<3 x i5> [[MINMAXOP]], <3 x i5> [[Z]])
+; CHECK-NEXT:    ret <3 x i5> [[SEL]]
+;
+  %m1 = call <3 x i5> @llvm.umax.v3i5(<3 x i5> %x, <3 x i5> %z)
+  %m2 = call <3 x i5> @llvm.umax.v3i5(<3 x i5> %y, <3 x i5> %z)
+  store <3 x i5> %m2, <3 x i5>* %p
+  %sel = select i1 %cond, <3 x i5> %m1, <3 x i5> %m2
+  ret <3 x i5> %sel
+}
+
+; negative test - intrinsic mismatch
+
+define i5 @smin_umin_common_op_11(i1 %cond, i5 %x, i5 %y, i5 %z) {
+; CHECK-LABEL: @smin_umin_common_op_11(
+; CHECK-NEXT:    [[M1:%.*]] = call i5 @llvm.smin.i5(i5 [[X:%.*]], i5 [[Z:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i5 @llvm.umin.i5(i5 [[Y:%.*]], i5 [[Z]])
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i5 [[M1]], i5 [[M2]]
+; CHECK-NEXT:    ret i5 [[SEL]]
+;
+  %m1 = call i5 @llvm.smin.i5(i5 %x, i5 %z)
+  %m2 = call i5 @llvm.umin.i5(i5 %y, i5 %z)
+  %sel = select i1 %cond, i5 %m1, i5 %m2
+  ret i5 %sel
+}
+
+; negative test - require shared operand
+
+define i5 @smin_smin_no_common_op(i1 %cond, i5 %x, i5 %y, i5 %z, i5 %w) {
+; CHECK-LABEL: @smin_smin_no_common_op(
+; CHECK-NEXT:    [[M1:%.*]] = call i5 @llvm.smin.i5(i5 [[Z:%.*]], i5 [[X:%.*]])
+; CHECK-NEXT:    [[M2:%.*]] = call i5 @llvm.smin.i5(i5 [[W:%.*]], i5 [[Y:%.*]])
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i5 [[M1]], i5 [[M2]]
+; CHECK-NEXT:    ret i5 [[SEL]]
+;
+  %m1 = call i5 @llvm.smin.i5(i5 %z, i5 %x)
+  %m2 = call i5 @llvm.smin.i5(i5 %w, i5 %y)
+  %sel = select i1 %cond, i5 %m1, i5 %m2
+  ret i5 %sel
+}
+
+; negative test - too many uses
+
+define i5 @umin_umin_common_op_10_uses(i1 %cond, i5 %x, i5 %y, i5 %z, i5* %p1, i5* %p2) {
+; CHECK-LABEL: @umin_umin_common_op_10_uses(
+; CHECK-NEXT:    [[M1:%.*]] = call i5 @llvm.umin.i5(i5 [[X:%.*]], i5 [[Z:%.*]])
+; CHECK-NEXT:    store i5 [[M1]], i5* [[P1:%.*]], align 1
+; CHECK-NEXT:    [[M2:%.*]] = call i5 @llvm.umin.i5(i5 [[Z]], i5 [[Y:%.*]])
+; CHECK-NEXT:    store i5 [[M2]], i5* [[P2:%.*]], align 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i5 [[M1]], i5 [[M2]]
+; CHECK-NEXT:    ret i5 [[SEL]]
+;
+  %m1 = call i5 @llvm.umin.i5(i5 %x, i5 %z)
+  store i5 %m1, i5* %p1
+  %m2 = call i5 @llvm.umin.i5(i5 %z, i5 %y)
+  store i5 %m2, i5* %p2
+  %sel = select i1 %cond, i5 %m1, i5 %m2
+  ret i5 %sel
+}
diff --git a/llvm/test/Transforms/InstCombine/select-safe-bool-transforms.ll b/llvm/test/Transforms/InstCombine/select-safe-bool-transforms.ll
index 83275ec21c3c28bccbd665dba9bf9a147092ef08..f86febaf95ee6f3d0334a8b1ea9f74607629be39 100644
--- a/llvm/test/Transforms/InstCombine/select-safe-bool-transforms.ll
+++ b/llvm/test/Transforms/InstCombine/select-safe-bool-transforms.ll
@@ -18,8 +18,7 @@ define i1 @land_land_left1(i1 %A, i1 %B) {
 }
 define i1 @land_land_left2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @land_land_left2(
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[B:%.*]], i1 [[A:%.*]], i1 false
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C]], i1 [[A]], i1 false
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[B:%.*]], i1 [[A:%.*]], i1 false
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %c = select i1 %B, i1 %A, i1 false
@@ -181,8 +180,7 @@ define i1 @lor_lor_left1(i1 %A, i1 %B) {
 }
 define i1 @lor_lor_left2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @lor_lor_left2(
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[B:%.*]], i1 true, i1 [[A:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C]], i1 true, i1 [[A]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[B:%.*]], i1 true, i1 [[A:%.*]]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %c = select i1 %B, i1 true, i1 %A
@@ -262,8 +260,8 @@ define i1 @land_land_right1(i1 %A, i1 %B) {
 }
 define i1 @land_land_right2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @land_land_right2(
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[A:%.*]], i1 [[B:%.*]], i1 false
-; CHECK-NEXT:    ret i1 [[TMP1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A:%.*]], i1 [[B:%.*]], i1 false
+; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %c = select i1 %B, i1 %A, i1 false
   %res = select i1 %A, i1 %c, i1 false
@@ -301,9 +299,7 @@ define i1 @land_lor_right1(i1 %A, i1 %B) {
 }
 define i1 @land_lor_right2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @land_lor_right2(
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[B:%.*]], i1 [[A:%.*]], i1 false
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 true, i1 [[C]]
-; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK-NEXT:    ret i1 [[A:%.*]]
 ;
   %c = select i1 %B, i1 %A, i1 false
   %res = select i1 %A, i1 true, i1 %c
@@ -333,8 +329,7 @@ define i1 @land_bor_right2(i1 %A, i1 %B) {
 ; A land (A band B)
 define i1 @band_land_right1(i1 %A, i1 %B) {
 ; CHECK-LABEL: @band_land_right1(
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 [[C]], i1 false
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A:%.*]], i1 [[B:%.*]], i1 false
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %c = and i1 %A, %B
@@ -343,8 +338,7 @@ define i1 @band_land_right1(i1 %A, i1 %B) {
 }
 define i1 @band_land_right2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @band_land_right2(
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[B:%.*]], [[A:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 [[C]], i1 false
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A:%.*]], i1 [[B:%.*]], i1 false
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %c = and i1 %B, %A
@@ -355,9 +349,7 @@ define i1 @band_land_right2(i1 %A, i1 %B) {
 ; A lor (A band B)
 define i1 @band_lor_right1(i1 %A, i1 %B) {
 ; CHECK-LABEL: @band_lor_right1(
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 true, i1 [[C]]
-; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK-NEXT:    ret i1 [[A:%.*]]
 ;
   %c = and i1 %A, %B
   %res = select i1 %A, i1 true, i1 %c
@@ -365,9 +357,7 @@ define i1 @band_lor_right1(i1 %A, i1 %B) {
 }
 define i1 @band_lor_right2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @band_lor_right2(
-; CHECK-NEXT:    [[C:%.*]] = and i1 [[B:%.*]], [[A:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 true, i1 [[C]]
-; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK-NEXT:    ret i1 [[A:%.*]]
 ;
   %c = and i1 %B, %A
   %res = select i1 %A, i1 true, i1 %c
@@ -385,9 +375,7 @@ define i1 @lor_land_right1(i1 %A, i1 %B) {
 }
 define i1 @lor_land_right2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @lor_land_right2(
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[B:%.*]], i1 true, i1 [[A:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 [[C]], i1 false
-; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK-NEXT:    ret i1 [[A:%.*]]
 ;
   %c = select i1 %B, i1 true, i1 %A
   %res = select i1 %A, i1 %c, i1 false
@@ -426,8 +414,8 @@ define i1 @lor_lor_right1(i1 %A, i1 %B) {
 }
 define i1 @lor_lor_right2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @lor_lor_right2(
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[B:%.*]]
-; CHECK-NEXT:    ret i1 [[TMP1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %c = select i1 %B, i1 true, i1 %A
   %res = select i1 %A, i1 true, i1 %c
@@ -457,9 +445,7 @@ define i1 @lor_bor_right2(i1 %A, i1 %B) {
 ; A land (A bor B)
 define i1 @bor_land_right1(i1 %A, i1 %B) {
 ; CHECK-LABEL: @bor_land_right1(
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 [[C]], i1 false
-; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK-NEXT:    ret i1 [[A:%.*]]
 ;
   %c = or i1 %A, %B
   %res = select i1 %A, i1 %c, i1 false
@@ -467,9 +453,7 @@ define i1 @bor_land_right1(i1 %A, i1 %B) {
 }
 define i1 @bor_land_right2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @bor_land_right2(
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], [[A:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 [[C]], i1 false
-; CHECK-NEXT:    ret i1 [[RES]]
+; CHECK-NEXT:    ret i1 [[A:%.*]]
 ;
   %c = or i1 %B, %A
   %res = select i1 %A, i1 %c, i1 false
@@ -479,8 +463,7 @@ define i1 @bor_land_right2(i1 %A, i1 %B) {
 ; A lor (A bor B)
 define i1 @bor_lor_right1(i1 %A, i1 %B) {
 ; CHECK-LABEL: @bor_lor_right1(
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 true, i1 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[B:%.*]]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %c = or i1 %A, %B
@@ -489,8 +472,7 @@ define i1 @bor_lor_right1(i1 %A, i1 %B) {
 }
 define i1 @bor_lor_right2(i1 %A, i1 %B) {
 ; CHECK-LABEL: @bor_lor_right2(
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], [[A:%.*]]
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A]], i1 true, i1 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[B:%.*]]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %c = or i1 %B, %A
diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
index d603de371ba020721ef8fc53e8c28be435ce21cf..f98a369c144ba02d958be09c93cdd6c62b072f6f 100644
--- a/llvm/test/Transforms/InstCombine/select.ll
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -902,7 +902,9 @@ define i32 @test56(i16 %x) {
 define i32 @test57(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test57(
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret i32 [[AND]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    [[DOTAND:%.*]] = select i1 [[TOBOOL]], i32 0, i32 [[AND]]
+; CHECK-NEXT:    ret i32 [[DOTAND]]
 ;
   %and = and i32 %x, %y
   %tobool = icmp eq i32 %x, 0
@@ -2680,7 +2682,7 @@ define void @select_freeze_icmp_multuses(i32 %x, i32 %y) {
 define i32 @pr47322_more_poisonous_replacement(i32 %arg) {
 ; CHECK-LABEL: @pr47322_more_poisonous_replacement(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[ARG:%.*]], 0
-; CHECK-NEXT:    [[TRAILING:%.*]] = call i32 @llvm.cttz.i32(i32 [[ARG]], i1 immarg true), [[RNG0:!range !.*]]
+; CHECK-NEXT:    [[TRAILING:%.*]] = call i32 @llvm.cttz.i32(i32 [[ARG]], i1 immarg true), !range [[RNG0:![0-9]+]]
 ; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[ARG]], [[TRAILING]]
 ; CHECK-NEXT:    [[R1_SROA_0_1:%.*]] = select i1 [[CMP]], i32 0, i32 [[SHIFTED]]
 ; CHECK-NEXT:    ret i32 [[R1_SROA_0_1]]
diff --git a/llvm/test/Transforms/InstCombine/stacksaverestore.ll b/llvm/test/Transforms/InstCombine/stacksaverestore.ll
index 9eb0efb1911ba6a41933279b98ecbf86f5e84ca0..cbc353afe61909e160bfe35cae36c46060fa3cc6 100644
--- a/llvm/test/Transforms/InstCombine/stacksaverestore.ll
+++ b/llvm/test/Transforms/InstCombine/stacksaverestore.ll
@@ -9,7 +9,7 @@ declare void @llvm.stackrestore(i8*)
 define i32* @test1(i32 %P) {
 	%tmp = call i8* @llvm.stacksave( )
 	call void @llvm.stackrestore( i8* %tmp ) ;; not restoring anything
-	%A = alloca i32, i32 %P		
+	%A = alloca i32, i32 %P
 	ret i32* %A
 }
 
@@ -49,7 +49,7 @@ bb:		; preds = %bb, %bb.preheader
 	%tmp77 = alloca i8, i32 %size		; <i8*> [#uses=1]
 	%tmp78 = call i8* @llvm.stacksave( )		; <i8*> [#uses=1]
 	%tmp102 = alloca i8, i32 %size		; <i8*> [#uses=1]
-	call void @bar( i32 %i.0.reg2mem.0, i8* %tmp23, i8* %tmp52, i8* %tmp77, i8* %tmp102, i32 %size ) nounwind 
+	call void @bar( i32 %i.0.reg2mem.0, i8* %tmp23, i8* %tmp52, i8* %tmp77, i8* %tmp102, i32 %size ) nounwind
 	call void @llvm.stackrestore( i8* %tmp78 )
 	call void @llvm.stackrestore( i8* %tmp53 )
 	call void @llvm.stackrestore( i8* %tmp28 )
@@ -72,7 +72,7 @@ return:		; preds = %bb, %entry
 
 declare void @bar(i32, i8*, i8*, i8*, i8*, i32)
 
-declare void @inalloca_callee(i32* inalloca)
+declare void @inalloca_callee(i32* inalloca(i32))
 
 define void @test3(i32 %c) {
 entry:
@@ -83,7 +83,7 @@ loop:
   %save1 = call i8* @llvm.stacksave()
   %argmem = alloca inalloca i32
   store i32 0, i32* %argmem
-  call void @inalloca_callee(i32* inalloca %argmem)
+  call void @inalloca_callee(i32* inalloca(i32) %argmem)
 
   ; This restore cannot be deleted, the restore below does not make it dead.
   call void @llvm.stackrestore(i8* %save1)
@@ -106,7 +106,7 @@ return:
 ; CHECK: %save1 = call i8* @llvm.stacksave()
 ; CHECK: %argmem = alloca inalloca i32
 ; CHECK: store i32 0, i32* %argmem
-; CHECK: call void @inalloca_callee(i32* inalloca {{.*}} %argmem)
+; CHECK: call void @inalloca_callee(i32* {{.*}} inalloca(i32) %argmem)
 ; CHECK: call void @llvm.stackrestore(i8* %save1)
 ; CHECK: br i1 %done, label %loop, label %return
 ; CHECK: ret void
diff --git a/llvm/test/Transforms/InstCombine/sub-minmax.ll b/llvm/test/Transforms/InstCombine/sub-minmax.ll
index 63a884aba5c3a0552061deb39daa68547ebda77b..83b210929ec1aa43ebc69f74b91add75d4904e2e 100644
--- a/llvm/test/Transforms/InstCombine/sub-minmax.ll
+++ b/llvm/test/Transforms/InstCombine/sub-minmax.ll
@@ -1,6 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
+declare i5 @llvm.umin.i5(i5, i5)
+declare <2 x i8> @llvm.umin.v2i8(<2 x i8>, <2 x i8>)
+
 define i32 @max_na_b_minux_na(i32 %A, i32 %B) {
 ; CHECK-LABEL: @max_na_b_minux_na(
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
@@ -28,6 +31,52 @@ define i32 @na_minus_max_na_b(i32 %A, i32 %B) {
   ret i32 %x
 }
 
+define i5 @sub_umin(i5 %a, i5 %b) {
+; CHECK-LABEL: @sub_umin(
+; CHECK-NEXT:    [[UMIN:%.*]] = call i5 @llvm.umin.i5(i5 [[A:%.*]], i5 [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = sub i5 [[A]], [[UMIN]]
+; CHECK-NEXT:    ret i5 [[R]]
+;
+  %umin = call i5 @llvm.umin.i5(i5 %a, i5 %b)
+  %r = sub i5 %a, %umin
+  ret i5 %r
+}
+
+define <2 x i8> @sub_umin_commute_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @sub_umin_commute_vec(
+; CHECK-NEXT:    [[UMIN:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[B:%.*]], <2 x i8> [[A:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i8> [[B]], [[UMIN]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %umin = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %b, <2 x i8> %a)
+  %r = sub <2 x i8> %b, %umin
+  ret <2 x i8> %r
+}
+
+define i5 @sub_umin_uses(i5 %a, i5 %b, i5* %p) {
+; CHECK-LABEL: @sub_umin_uses(
+; CHECK-NEXT:    [[UMIN:%.*]] = call i5 @llvm.umin.i5(i5 [[A:%.*]], i5 [[B:%.*]])
+; CHECK-NEXT:    store i5 [[UMIN]], i5* [[P:%.*]], align 1
+; CHECK-NEXT:    [[R:%.*]] = sub i5 [[A]], [[UMIN]]
+; CHECK-NEXT:    ret i5 [[R]]
+;
+  %umin = call i5 @llvm.umin.i5(i5 %a, i5 %b)
+  store i5 %umin, i5* %p
+  %r = sub i5 %a, %umin
+  ret i5 %r
+}
+
+define i5 @sub_umin_no_common_op(i5 %a, i5 %b, i5 %c) {
+; CHECK-LABEL: @sub_umin_no_common_op(
+; CHECK-NEXT:    [[UMIN:%.*]] = call i5 @llvm.umin.i5(i5 [[A:%.*]], i5 [[B:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = sub i5 [[C:%.*]], [[UMIN]]
+; CHECK-NEXT:    ret i5 [[R]]
+;
+  %umin = call i5 @llvm.umin.i5(i5 %a, i5 %b)
+  %r = sub i5 %c, %umin
+  ret i5 %r
+}
+
 define i32 @max_b_na_minus_na(i32 %A, i32 %B) {
 ; CHECK-LABEL: @max_b_na_minus_na(
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
diff --git a/llvm/test/Transforms/InstCombine/unused-nonnull.ll b/llvm/test/Transforms/InstCombine/unused-nonnull.ll
index 74173ad4592d232dc322c35786ebc11829b74d3e..0f6ef72999803b98e64b5477574e8fbd7909d168 100644
--- a/llvm/test/Transforms/InstCombine/unused-nonnull.ll
+++ b/llvm/test/Transforms/InstCombine/unused-nonnull.ll
@@ -37,7 +37,7 @@ done:
 
 define i32 @compute(i8* noundef nonnull %ptr, i32 %x) #1 {
 ; CHECK-LABEL: define {{[^@]+}}@compute
-; CHECK-SAME: (i8* nocapture noundef nonnull readnone [[PTR:%.*]], i32 returned [[X:%.*]]) local_unnamed_addr #1
+; CHECK-SAME: (i8* nocapture readnone [[PTR:%.*]], i32 returned [[X:%.*]]) local_unnamed_addr #1
 ; CHECK-NEXT:    ret i32 [[X]]
 ;
   ret i32 %x
diff --git a/llvm/test/Transforms/InstSimplify/AndOrXor.ll b/llvm/test/Transforms/InstSimplify/AndOrXor.ll
index e23262835c3cf3b36ce2b4425d4a17b5c59b27db..2e364a789675c6339c77f6d2030617b3e45e1a7e 100644
--- a/llvm/test/Transforms/InstSimplify/AndOrXor.ll
+++ b/llvm/test/Transforms/InstSimplify/AndOrXor.ll
@@ -1,6 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instsimplify -S | FileCheck %s
 
+declare i32 @llvm.smax.i32(i32, i32)
+declare <2 x i32> @llvm.umin.v2i32(<2 x i32>, <2 x i32>)
+
 define i8 @and0(i8 %x) {
 ; CHECK-LABEL: @and0(
 ; CHECK-NEXT:    ret i8 0
@@ -89,6 +92,21 @@ define i64 @pow2b(i32 %x) {
   ret i64 %e2
 }
 
+define i32 @pow2b_max(i32 %x, i32 %y) {
+; CHECK-LABEL: @pow2b_max(
+; CHECK-NEXT:    [[SHX:%.*]] = shl i32 2, [[X:%.*]]
+; CHECK-NEXT:    [[SHY:%.*]] = shl i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[M:%.*]] = call i32 @llvm.smax.i32(i32 [[SHX]], i32 [[SHY]])
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %shx = shl i32 2, %x
+  %shy = shl i32 32, %y
+  %m = call i32 @llvm.smax.i32(i32 %shx, i32 %shy)
+  %neg = sub i32 0, %m
+  %r = and i32 %m, %neg
+  ret i32 %r
+}
+
 ; Power-of-2-or-zero value has no bits in common with its decrement.
 
 define i32 @pow2_decrement(i32 %p) {
@@ -111,6 +129,18 @@ define <2 x i32> @pow2_decrement_commute_vec(<2 x i32> %p) {
   ret <2 x i32> %r
 }
 
+define <2 x i32> @pow2_decrement_min_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @pow2_decrement_min_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %p1 = and <2 x i32> %x, <i32 2048, i32 2048>
+  %p2 = shl <2 x i32> <i32 1, i32 1>, %y
+  %m = call <2 x i32> @llvm.umin.v2i32(<2 x i32> %p1, <2 x i32> %p2)
+  %a = add <2 x i32> %m, <i32 -1, i32 -1>
+  %r = and <2 x i32> %m, %a
+  ret <2 x i32> %r
+}
+
 define i1 @and_of_icmps0(i32 %b) {
 ; CHECK-LABEL: @and_of_icmps0(
 ; CHECK-NEXT:    ret i1 false
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll b/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll
index 68d7390b0e6bccec803ef3d22afc4da97329f111..6f25b9f8d10cdfb375a4c2532d33b95fa9aaac32 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll
@@ -1,5 +1,6 @@
-; RUN: opt < %s -data-layout="e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64" -instcombine -S | FileCheck %s --check-prefix=LE
-; RUN: opt < %s -data-layout="E-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64" -instcombine -S | FileCheck %s --check-prefix=BE
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -data-layout="e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64" -instcombine -S | FileCheck %s --check-prefixes=CHECK,LE
+; RUN: opt < %s -data-layout="E-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64" -instcombine -S | FileCheck %s --check-prefixes=CHECK,BE
 
 ; {{ 0xDEADBEEF, 0xBA }, 0xCAFEBABE}
 @g1 = constant {{i32,i8},i32} {{i32,i8} { i32 -559038737, i8 186 }, i32 -889275714 }
@@ -9,142 +10,113 @@
 
 ; Simple load
 define i32 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 -559038737
+;
   %r = load i32, i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0)
   ret i32 %r
-
-; 0xDEADBEEF
-; LE-LABEL: @test1(
-; LE: ret i32 -559038737
-
-; 0xDEADBEEF
-; BE-LABEL: @test1(
-; BE: ret i32 -559038737
 }
 
 ; PR3152
 ; Load of first 16 bits of 32-bit value.
 define i16 @test2() {
-  %r = load i16, i16* bitcast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16*)
-  ret i16 %r
-
-; 0xBEEF
 ; LE-LABEL: @test2(
-; LE: ret i16 -16657
-
-; 0xDEAD
+; LE-NEXT:    ret i16 -16657
+;
 ; BE-LABEL: @test2(
-; BE: ret i16 -8531
+; BE-NEXT:    ret i16 -8531
+;
+  %r = load i16, i16* bitcast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16*)
+  ret i16 %r
 }
 
+; FIXME: Should be able to load through a constant addrspacecast.
 define i16 @test2_addrspacecast() {
+; CHECK-LABEL: @test2_addrspacecast(
+; CHECK-NEXT:    [[R:%.*]] = load i16, i16 addrspace(1)* addrspacecast (i16* bitcast ({ { i32, i8 }, i32 }* @g1 to i16*) to i16 addrspace(1)*), align 8
+; CHECK-NEXT:    ret i16 [[R]]
+;
   %r = load i16, i16 addrspace(1)* addrspacecast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16 addrspace(1)*)
   ret i16 %r
-
-; FIXME: Should be able to load through a constant addrspacecast.
-; 0xBEEF
-; LE-LABEL: @test2_addrspacecast(
-; XLE: ret i16 -16657
-; LE: load i16, i16 addrspace(1)* addrspacecast
-
-; 0xDEAD
-; BE-LABEL: @test2_addrspacecast(
-; XBE: ret i16 -8531
-; BE: load i16, i16 addrspace(1)* addrspacecast
 }
 
 ; Load of second 16 bits of 32-bit value.
 define i16 @test3() {
-  %r = load i16, i16* getelementptr(i16, i16* bitcast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16*), i32 1)
-  ret i16 %r
-
-; 0xDEAD
 ; LE-LABEL: @test3(
-; LE: ret i16 -8531
-
-; 0xBEEF
+; LE-NEXT:    ret i16 -8531
+;
 ; BE-LABEL: @test3(
-; BE: ret i16 -16657
+; BE-NEXT:    ret i16 -16657
+;
+  %r = load i16, i16* getelementptr(i16, i16* bitcast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16*), i32 1)
+  ret i16 %r
 }
 
 ; Load of 8 bit field + tail padding.
 define i16 @test4() {
-  %r = load i16, i16* getelementptr(i16, i16* bitcast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16*), i32 2)
-  ret i16 %r
-
-; 0x00BA
 ; LE-LABEL: @test4(
-; LE: ret i16 186
-
-; 0xBA00
+; LE-NEXT:    ret i16 186
+;
 ; BE-LABEL: @test4(
-; BE: ret i16 -17920
+; BE-NEXT:    ret i16 -17920
+;
+  %r = load i16, i16* getelementptr(i16, i16* bitcast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16*), i32 2)
+  ret i16 %r
 }
 
 ; Load of double bits.
 define i64 @test6() {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i64 4607182418800017408
+;
   %r = load i64, i64* bitcast(double* @g2 to i64*)
   ret i64 %r
-
-; 0x3FF_0000000000000
-; LE-LABEL: @test6(
-; LE: ret i64 4607182418800017408
-
-; 0x3FF_0000000000000
-; BE-LABEL: @test6(
-; BE: ret i64 4607182418800017408
 }
 
 ; Load of double bits.
 define i16 @test7() {
-  %r = load i16, i16* bitcast(double* @g2 to i16*)
-  ret i16 %r
-
-; 0x0000
 ; LE-LABEL: @test7(
-; LE: ret i16 0
-
-; 0x3FF0
+; LE-NEXT:    ret i16 0
+;
 ; BE-LABEL: @test7(
-; BE: ret i16 16368
+; BE-NEXT:    ret i16 16368
+;
+  %r = load i16, i16* bitcast(double* @g2 to i16*)
+  ret i16 %r
 }
 
 ; Double load.
 define double @test8() {
-  %r = load double, double* bitcast({{i32,i8},i32}* @g1 to double*)
-  ret double %r
-
 ; LE-LABEL: @test8(
-; LE: ret double 0xBADEADBEEF
-
+; LE-NEXT:    ret double 0xBADEADBEEF
+;
 ; BE-LABEL: @test8(
-; BE: ret double 0xDEADBEEFBA000000
+; BE-NEXT:    ret double 0xDEADBEEFBA000000
+;
+  %r = load double, double* bitcast({{i32,i8},i32}* @g1 to double*)
+  ret double %r
 }
 
 
 ; i128 load.
 define i128 @test9() {
-  %r = load i128, i128* bitcast({i64, i64}* @g3 to i128*)
-  ret i128 %r
-
-; 0x00000000_06B1BFF8_00000000_0000007B
 ; LE-LABEL: @test9(
-; LE: ret i128 2071796475790618158476296315
-
-; 0x00000000_0000007B_00000000_06B1BFF8
+; LE-NEXT:    ret i128 2071796475790618158476296315
+;
 ; BE-LABEL: @test9(
-; BE: ret i128 2268949521066387161080
+; BE-NEXT:    ret i128 2268949521066387161080
+;
+  %r = load i128, i128* bitcast({i64, i64}* @g3 to i128*)
+  ret i128 %r
 }
 
 ; vector load.
 define <2 x i64> @test10() {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret <2 x i64> <i64 123, i64 112312312>
+;
   %r = load <2 x i64>, <2 x i64>* bitcast({i64, i64}* @g3 to <2 x i64>*)
   ret <2 x i64> %r
-
-; LE-LABEL: @test10(
-; LE: ret <2 x i64> <i64 123, i64 112312312>
-
-; BE-LABEL: @test10(
-; BE: ret <2 x i64> <i64 123, i64 112312312>
 }
 
 
@@ -153,17 +125,17 @@ define <2 x i64> @test10() {
 @g4 = internal constant { i8, i8 } { i8 -95, i8 8 }
 
 define i16 @test11() nounwind {
+; LE-LABEL: @test11(
+; LE-NEXT:  entry:
+; LE-NEXT:    ret i16 2209
+;
+; BE-LABEL: @test11(
+; BE-NEXT:  entry:
+; BE-NEXT:    ret i16 -24312
+;
 entry:
   %a = load i16, i16* bitcast ({ i8, i8 }* @g4 to i16*)
   ret i16 %a
-
-; 0x08A1
-; LE-LABEL: @test11(
-; LE: ret i16 2209
-
-; 0xA108
-; BE-LABEL: @test11(
-; BE: ret i16 -24312
 }
 
 
@@ -171,140 +143,121 @@ entry:
 @test12g = private constant [6 x i8] c"a\00b\00\00\00"
 
 define i16 @test12() {
-  %a = load i16, i16* getelementptr inbounds ([3 x i16], [3 x i16]* bitcast ([6 x i8]* @test12g to [3 x i16]*), i32 0, i64 1)
-  ret i16 %a
-
-; 0x0062
 ; LE-LABEL: @test12(
-; LE: ret i16 98
-
-; 0x6200
+; LE-NEXT:    ret i16 98
+;
 ; BE-LABEL: @test12(
-; BE: ret i16 25088
+; BE-NEXT:    ret i16 25088
+;
+  %a = load i16, i16* getelementptr inbounds ([3 x i16], [3 x i16]* bitcast ([6 x i8]* @test12g to [3 x i16]*), i32 0, i64 1)
+  ret i16 %a
 }
 
 
 ; PR5978
 @g5 = constant i8 4
 define i1 @test13() {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    ret i1 false
+;
   %A = load i1, i1* bitcast (i8* @g5 to i1*)
   ret i1 %A
-
-; LE-LABEL: @test13(
-; LE: ret i1 false
-
-; BE-LABEL: @test13(
-; BE: ret i1 false
 }
 
 @g6 = constant [2 x i8*] [i8* inttoptr (i64 1 to i8*), i8* inttoptr (i64 2 to i8*)]
 define i64 @test14() nounwind {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i64 1
+;
 entry:
   %tmp = load i64, i64* bitcast ([2 x i8*]* @g6 to i64*)
   ret i64 %tmp
-
-; LE-LABEL: @test14(
-; LE: ret i64 1
-
-; BE-LABEL: @test14(
-; BE: ret i64 1
 }
 
 ; Check with address space pointers
 @g6_as1 = constant [2 x i8 addrspace(1)*] [i8 addrspace(1)* inttoptr (i16 1 to i8 addrspace(1)*), i8 addrspace(1)* inttoptr (i16 2 to i8 addrspace(1)*)]
 define i16 @test14_as1() nounwind {
+; CHECK-LABEL: @test14_as1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i16 1
+;
 entry:
   %tmp = load i16, i16* bitcast ([2 x i8 addrspace(1)*]* @g6_as1 to i16*)
   ret i16 %tmp
-
-; LE: @test14_as1
-; LE: ret i16 1
-
-; BE: @test14_as1
-; BE: ret i16 1
 }
 
 define i64 @test15() nounwind {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i64 2
+;
 entry:
   %tmp = load i64, i64* bitcast (i8** getelementptr inbounds ([2 x i8*], [2 x i8*]* @g6, i32 0, i64 1) to i64*)
   ret i64 %tmp
-
-; LE-LABEL: @test15(
-; LE: ret i64 2
-
-; BE-LABEL: @test15(
-; BE: ret i64 2
 }
 
 @gv7 = constant [4 x i8*] [i8* null, i8* inttoptr (i64 -14 to i8*), i8* null, i8* null]
 define i64 @test16.1() {
+; CHECK-LABEL: @test16.1(
+; CHECK-NEXT:    ret i64 0
+;
   %v = load i64, i64* bitcast ([4 x i8*]* @gv7 to i64*), align 8
   ret i64 %v
-
-; LE-LABEL: @test16.1(
-; LE: ret i64 0
-
-; BE-LABEL: @test16.1(
-; BE: ret i64 0
 }
 
 define i64 @test16.2() {
+; CHECK-LABEL: @test16.2(
+; CHECK-NEXT:    ret i64 -14
+;
   %v = load i64, i64* bitcast (i8** getelementptr inbounds ([4 x i8*], [4 x i8*]* @gv7, i64 0, i64 1) to i64*), align 8
   ret i64 %v
-
-; LE-LABEL: @test16.2(
-; LE: ret i64 -14
-
-; BE-LABEL: @test16.2(
-; BE: ret i64 -14
 }
 
 define i64 @test16.3() {
+; CHECK-LABEL: @test16.3(
+; CHECK-NEXT:    ret i64 0
+;
   %v = load i64, i64* bitcast (i8** getelementptr inbounds ([4 x i8*], [4 x i8*]* @gv7, i64 0, i64 2) to i64*), align 8
   ret i64 %v
-
-; LE-LABEL: @test16.3(
-; LE: ret i64 0
-
-; BE-LABEL: @test16.3(
-; BE: ret i64 0
 }
 
 @g7 = constant {[0 x i32], [0 x i8], {}*} { [0 x i32] undef, [0 x i8] undef, {}* null }
 
 define i64* @test_leading_zero_size_elems() {
+; CHECK-LABEL: @test_leading_zero_size_elems(
+; CHECK-NEXT:    ret i64* null
+;
   %v = load i64*, i64** bitcast ({[0 x i32], [0 x i8], {}*}* @g7 to i64**)
   ret i64* %v
-
-; LE-LABEL: @test_leading_zero_size_elems(
-; LE: ret i64* null
-
-; BE-LABEL: @test_leading_zero_size_elems(
-; BE: ret i64* null
 }
 
 @g8 = constant {[4294967295 x [0 x i32]], i64} { [4294967295 x [0 x i32]] undef, i64 123 }
 
 define i64 @test_leading_zero_size_elems_big() {
+; CHECK-LABEL: @test_leading_zero_size_elems_big(
+; CHECK-NEXT:    ret i64 123
+;
   %v = load i64, i64* bitcast ({[4294967295 x [0 x i32]], i64}* @g8 to i64*)
   ret i64 %v
-
-; LE-LABEL: @test_leading_zero_size_elems_big(
-; LE: ret i64 123
-
-; BE-LABEL: @test_leading_zero_size_elems_big(
-; BE: ret i64 123
 }
 
 @g9 = constant [4294967295 x [0 x i32]] zeroinitializer
 
 define i64 @test_array_of_zero_size_array() {
+; CHECK-LABEL: @test_array_of_zero_size_array(
+; CHECK-NEXT:    ret i64 0
+;
   %v = load i64, i64* bitcast ([4294967295 x [0 x i32]]* @g9 to i64*)
   ret i64 %v
+}
 
-; LE-LABEL: @test_array_of_zero_size_array(
-; LE: ret i64 0
+@g10 = constant {i128} {i128 undef}
 
-; BE-LABEL: @test_array_of_zero_size_array(
-; BE: ret i64 0
+define i32* @test_undef_aggregate() {
+; CHECK-LABEL: @test_undef_aggregate(
+; CHECK-NEXT:    ret i32* undef
+;
+  %v = load i32*, i32** bitcast ({i128}* @g10 to i32**)
+  ret i32* %v
 }
diff --git a/llvm/test/Transforms/InstSimplify/gep.ll b/llvm/test/Transforms/InstSimplify/gep.ll
index 3c460ecc4a67769adba983131bccf7160a4d09f4..e1da60ee566844e64763c18211392149e817006f 100644
--- a/llvm/test/Transforms/InstSimplify/gep.ll
+++ b/llvm/test/Transforms/InstSimplify/gep.ll
@@ -7,7 +7,12 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 define %struct.A* @test1(%struct.A* %b, %struct.A* %e) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    ret %struct.A* [[E:%.*]]
+; CHECK-NEXT:    [[E_PTR:%.*]] = ptrtoint %struct.A* [[E:%.*]] to i64
+; CHECK-NEXT:    [[B_PTR:%.*]] = ptrtoint %struct.A* [[B:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[E_PTR]], [[B_PTR]]
+; CHECK-NEXT:    [[SDIV:%.*]] = sdiv exact i64 [[SUB]], 7
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [[STRUCT_A:%.*]], %struct.A* [[B]], i64 [[SDIV]]
+; CHECK-NEXT:    ret %struct.A* [[GEP]]
 ;
   %e_ptr = ptrtoint %struct.A* %e to i64
   %b_ptr = ptrtoint %struct.A* %b to i64
@@ -19,7 +24,11 @@ define %struct.A* @test1(%struct.A* %b, %struct.A* %e) {
 
 define i8* @test2(i8* %b, i8* %e) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    ret i8* [[E:%.*]]
+; CHECK-NEXT:    [[E_PTR:%.*]] = ptrtoint i8* [[E:%.*]] to i64
+; CHECK-NEXT:    [[B_PTR:%.*]] = ptrtoint i8* [[B:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[E_PTR]], [[B_PTR]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[B]], i64 [[SUB]]
+; CHECK-NEXT:    ret i8* [[GEP]]
 ;
   %e_ptr = ptrtoint i8* %e to i64
   %b_ptr = ptrtoint i8* %b to i64
@@ -30,7 +39,12 @@ define i8* @test2(i8* %b, i8* %e) {
 
 define i64* @test3(i64* %b, i64* %e) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    ret i64* [[E:%.*]]
+; CHECK-NEXT:    [[E_PTR:%.*]] = ptrtoint i64* [[E:%.*]] to i64
+; CHECK-NEXT:    [[B_PTR:%.*]] = ptrtoint i64* [[B:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[E_PTR]], [[B_PTR]]
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr exact i64 [[SUB]], 3
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 [[ASHR]]
+; CHECK-NEXT:    ret i64* [[GEP]]
 ;
   %e_ptr = ptrtoint i64* %e to i64
   %b_ptr = ptrtoint i64* %b to i64
diff --git a/llvm/test/Transforms/InstSimplify/pr49495.ll b/llvm/test/Transforms/InstSimplify/pr49495.ll
index f085de3b4a227ab1dc56f744b929c6cb1642dbc1..b3eca968c0603b94f4787b227c0049efe8f072e0 100644
--- a/llvm/test/Transforms/InstSimplify/pr49495.ll
+++ b/llvm/test/Transforms/InstSimplify/pr49495.ll
@@ -4,9 +4,11 @@
 ; The first comparison (a != b) should not be dropped
 define i1 @test1(i8* %a, i8* %b) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i8, i8* [[A:%.*]], i64 -1
-; CHECK-NEXT:    [[COND2:%.*]] = icmp ugt i8* [[A2]], [[B:%.*]]
-; CHECK-NEXT:    ret i1 [[COND2]]
+; CHECK-NEXT:    [[COND1:%.*]] = icmp ne i8* [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 -1
+; CHECK-NEXT:    [[COND2:%.*]] = icmp ugt i8* [[A2]], [[B]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[COND1]], i1 [[COND2]], i1 false
+; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %cond1 = icmp ne i8* %a, %b
   %a2 = getelementptr inbounds i8, i8* %a, i64 -1
@@ -18,9 +20,11 @@ define i1 @test1(i8* %a, i8* %b) {
 ; The first comparison (a != b) should not be dropped
 define i1 @test2(i32 %a, i32 %b) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[A2:%.*]] = add nuw i32 [[A:%.*]], 1
-; CHECK-NEXT:    [[COND2:%.*]] = icmp ult i32 [[A2]], [[B:%.*]]
-; CHECK-NEXT:    ret i1 [[COND2]]
+; CHECK-NEXT:    [[COND1:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[A2:%.*]] = add nuw i32 [[A]], 1
+; CHECK-NEXT:    [[COND2:%.*]] = icmp ult i32 [[A2]], [[B]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[COND1]], i1 [[COND2]], i1 false
+; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %cond1 = icmp ne i32 %a, %b
   %a2 = add nuw i32 %a, 1
diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll
index 6460b42d63c1e68ee3707c0c752a4070a0d9179a..f106b909a8ac0981dff7f61ec4d41744eae7f389 100644
--- a/llvm/test/Transforms/InstSimplify/select.ll
+++ b/llvm/test/Transforms/InstSimplify/select.ll
@@ -969,6 +969,66 @@ define <vscale x 2 x i1> @ignore_scalable_undef(<vscale x 2 x i1> %cond) {
   ret <vscale x 2 x i1> %s
 }
 
+define i32 @select_neutral_add_rhs(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_neutral_add_rhs(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %cmp = icmp ne i32 %y, 0
+  %add = add i32 %x, %y
+  %sel = select i1 %cmp, i32 %add, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_neutral_add_lhs(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_neutral_add_lhs(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %cmp = icmp ne i32 %y, 0
+  %add = add i32 %y, %x
+  %sel = select i1 %cmp, i32 %add, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_neutral_sub_rhs(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_neutral_sub_rhs(
+; CHECK-NEXT:    [[ADD:%.*]] = sub i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %cmp = icmp ne i32 %y, 0
+  %add = sub i32 %x, %y
+  %sel = select i1 %cmp, i32 %add, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_neutral_sub_lhs(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_neutral_sub_lhs(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[ADD:%.*]] = sub i32 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp ne i32 %y, 0
+  %add = sub i32 %y, %x
+  %sel = select i1 %cmp, i32 %add, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_ctpop_zero(i32 %x) {
+; CHECK-LABEL: @select_ctpop_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.ctpop.i32(i32 [[X:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %0 = icmp eq i32 %x, 0
+  %1 = call i32 @llvm.ctpop.i32(i32 %x)
+  %sel = select i1 %0, i32 0, i32 %1
+  ret i32 %sel
+}
+declare i32 @llvm.ctpop.i32(i32)
+
 ; TODO: these can be optimized more
 
 define i32 @poison(i32 %x, i32 %y) {
diff --git a/llvm/test/Transforms/JumpThreading/guards.ll b/llvm/test/Transforms/JumpThreading/guards.ll
index 911eec8dad2785175d86e23ea636788ff3255d8e..f487efa045b0f59924b3e70f4e356cab6823b0e8 100644
--- a/llvm/test/Transforms/JumpThreading/guards.ll
+++ b/llvm/test/Transforms/JumpThreading/guards.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -jump-threading -dce -S | FileCheck %s
 
 declare void @llvm.experimental.guard(i1, ...)
@@ -7,30 +8,34 @@ declare i32 @f2()
 
 define i32 @branch_implies_guard(i32 %a) {
 ; CHECK-LABEL: @branch_implies_guard(
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], 10
+; CHECK-NEXT:    br i1 [[COND]], label [[T1_SPLIT:%.*]], label [[F1_SPLIT:%.*]]
+; CHECK:       T1.split:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    [[RETVAL3:%.*]] = add i32 [[V1]], 10
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1.split:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    [[RETVAL1:%.*]] = add i32 [[V2]], 10
+; CHECK-NEXT:    [[CONDGUARD2:%.*]] = icmp slt i32 [[A]], 20
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CONDGUARD2]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[RETVAL3]], [[T1_SPLIT]] ], [ [[RETVAL1]], [[F1_SPLIT]] ]
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
   %cond = icmp slt i32 %a, 10
   br i1 %cond, label %T1, label %F1
 
 T1:
-; CHECK:       T1.split
-; CHECK:         %v1 = call i32 @f1()
-; CHECK-NEXT:    %retVal
-; CHECK-NEXT:    br label %Merge
   %v1 = call i32 @f1()
   br label %Merge
 
 F1:
-; CHECK:       F1.split
-; CHECK:         %v2 = call i32 @f2()
-; CHECK-NEXT:    %retVal
-; CHECK-NEXT:    %condGuard
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %condGuard
-; CHECK-NEXT:    br label %Merge
   %v2 = call i32 @f2()
   br label %Merge
 
 Merge:
-; CHECK:       Merge
-; CHECK-NOT:     call void(i1, ...) @llvm.experimental.guard(
   %retPhi = phi i32 [ %v1, %T1 ], [ %v2, %F1 ]
   %retVal = add i32 %retPhi, 10
   %condGuard = icmp slt i32 %a, 20
@@ -40,30 +45,34 @@ Merge:
 
 define i32 @not_branch_implies_guard(i32 %a) {
 ; CHECK-LABEL: @not_branch_implies_guard(
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], 20
+; CHECK-NEXT:    br i1 [[COND]], label [[T1_SPLIT:%.*]], label [[F1_SPLIT:%.*]]
+; CHECK:       T1.split:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    [[RETVAL1:%.*]] = add i32 [[V1]], 10
+; CHECK-NEXT:    [[CONDGUARD2:%.*]] = icmp sgt i32 [[A]], 10
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CONDGUARD2]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1.split:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    [[RETVAL3:%.*]] = add i32 [[V2]], 10
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[RETVAL3]], [[F1_SPLIT]] ], [ [[RETVAL1]], [[T1_SPLIT]] ]
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
   %cond = icmp slt i32 %a, 20
   br i1 %cond, label %T1, label %F1
 
 T1:
-; CHECK:       T1.split:
-; CHECK-NEXT:    %v1 = call i32 @f1()
-; CHECK-NEXT:    %retVal
-; CHECK-NEXT:    %condGuard
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %condGuard
-; CHECK-NEXT:    br label %Merge
   %v1 = call i32 @f1()
   br label %Merge
 
 F1:
-; CHECK:       F1.split:
-; CHECK-NEXT:   %v2 = call i32 @f2()
-; CHECK-NEXT:   %retVal
-; CHECK-NEXT:   br label %Merge
   %v2 = call i32 @f2()
   br label %Merge
 
 Merge:
-; CHECK:       Merge
-; CHECK-NOT:     call void(i1, ...) @llvm.experimental.guard(
   %retPhi = phi i32 [ %v1, %T1 ], [ %v2, %F1 ]
   %retVal = add i32 %retPhi, 10
   %condGuard = icmp sgt i32 %a, 10
@@ -73,27 +82,33 @@ Merge:
 
 define i32 @branch_overlaps_guard(i32 %a) {
 ; CHECK-LABEL: @branch_overlaps_guard(
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], 20
+; CHECK-NEXT:    br i1 [[COND]], label [[T1:%.*]], label [[F1:%.*]]
+; CHECK:       T1:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[RETPHI:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = add i32 [[RETPHI]], 10
+; CHECK-NEXT:    [[CONDGUARD:%.*]] = icmp slt i32 [[A]], 10
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CONDGUARD]]) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
   %cond = icmp slt i32 %a, 20
   br i1 %cond, label %T1, label %F1
 
 T1:
-; CHECK:        T1:
-; CHECK-NEXT:      %v1 = call i32 @f1()
-; CHECK-NEXT:      br label %Merge
   %v1 = call i32 @f1()
   br label %Merge
 
 F1:
-; CHECK:        F1:
-; CHECK-NEXT:     %v2 = call i32 @f2()
-; CHECK-NEXT:     br label %Merge
   %v2 = call i32 @f2()
   br label %Merge
 
 Merge:
-; CHECK:       Merge
-; CHECK:         %condGuard = icmp slt i32 %a, 10
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %condGuard) [ "deopt"() ]
   %retPhi = phi i32 [ %v1, %T1 ], [ %v2, %F1 ]
   %retVal = add i32 %retPhi, 10
   %condGuard = icmp slt i32 %a, 10
@@ -103,27 +118,33 @@ Merge:
 
 define i32 @branch_doesnt_overlap_guard(i32 %a) {
 ; CHECK-LABEL: @branch_doesnt_overlap_guard(
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], 10
+; CHECK-NEXT:    br i1 [[COND]], label [[T1:%.*]], label [[F1:%.*]]
+; CHECK:       T1:
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       F1:
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       Merge:
+; CHECK-NEXT:    [[RETPHI:%.*]] = phi i32 [ [[V1]], [[T1]] ], [ [[V2]], [[F1]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = add i32 [[RETPHI]], 10
+; CHECK-NEXT:    [[CONDGUARD:%.*]] = icmp sgt i32 [[A]], 20
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CONDGUARD]]) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
   %cond = icmp slt i32 %a, 10
   br i1 %cond, label %T1, label %F1
 
 T1:
-; CHECK:        T1:
-; CHECK-NEXT:      %v1 = call i32 @f1()
-; CHECK-NEXT:      br label %Merge
   %v1 = call i32 @f1()
   br label %Merge
 
 F1:
-; CHECK:        F1:
-; CHECK-NEXT:     %v2 = call i32 @f2()
-; CHECK-NEXT:     br label %Merge
   %v2 = call i32 @f2()
   br label %Merge
 
 Merge:
-; CHECK:       Merge
-; CHECK:         %condGuard = icmp sgt i32 %a, 20
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %condGuard) [ "deopt"() ]
   %retPhi = phi i32 [ %v1, %T1 ], [ %v2, %F1 ]
   %retVal = add i32 %retPhi, 10
   %condGuard = icmp sgt i32 %a, 20
@@ -133,31 +154,47 @@ Merge:
 
 define i32 @not_a_diamond1(i32 %a, i1 %cond1) {
 ; CHECK-LABEL: @not_a_diamond1(
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[PRED:%.*]], label [[EXIT:%.*]]
+; CHECK:       Pred:
+; CHECK-NEXT:    switch i32 [[A:%.*]], label [[EXIT]] [
+; CHECK-NEXT:    i32 10, label [[MERGE:%.*]]
+; CHECK-NEXT:    i32 20, label [[MERGE]]
+; CHECK-NEXT:    ]
+; CHECK:       Merge:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND1]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       Exit:
+; CHECK-NEXT:    ret i32 [[A]]
+;
   br i1 %cond1, label %Pred, label %Exit
 
 Pred:
-; CHECK:       Pred:
-; CHECK-NEXT:    switch i32 %a, label %Exit
   switch i32 %a, label %Exit [
-    i32 10, label %Merge
-    i32 20, label %Merge
+  i32 10, label %Merge
+  i32 20, label %Merge
   ]
 
 Merge:
-; CHECK:       Merge:
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
-; CHECK-NEXT:    br label %Exit
   call void(i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
   br label %Exit
 
 Exit:
-; CHECK:       Exit:
-; CHECK-NEXT:    ret i32 %a
   ret i32 %a
 }
 
 define void @not_a_diamond2(i32 %a, i1 %cond1) {
 ; CHECK-LABEL: @not_a_diamond2(
+; CHECK-NEXT:  Pred:
+; CHECK-NEXT:    switch i32 [[A:%.*]], label [[EXIT:%.*]] [
+; CHECK-NEXT:    i32 10, label [[MERGE:%.*]]
+; CHECK-NEXT:    i32 20, label [[MERGE]]
+; CHECK-NEXT:    ]
+; CHECK:       Merge:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND1:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       Exit:
+; CHECK-NEXT:    ret void
+;
   br label %Parent
 
 Merge:
@@ -165,20 +202,15 @@ Merge:
   ret void
 
 Pred:
-; CHECK-NEXT:  Pred:
-; CHECK-NEXT:    switch i32 %a, label %Exit
   switch i32 %a, label %Exit [
-    i32 10, label %Merge
-    i32 20, label %Merge
+  i32 10, label %Merge
+  i32 20, label %Merge
   ]
 
 Parent:
   br label %Pred
 
 Exit:
-; CHECK:       Merge:
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
-; CHECK-NEXT:    ret void
   ret void
 }
 
@@ -187,11 +219,15 @@ declare void @never_called(i1)
 ; LVI uses guard to identify value of %c2 in branch as true, we cannot replace that
 ; guard with guard(true & c1).
 define void @dont_fold_guard(i8* %addr, i32 %i, i32 %length) {
-; CHECK-LABEL: dont_fold_guard
-; CHECK: %wide.chk = and i1 %c1, %c2
-; CHECK-NEXT: experimental.guard(i1 %wide.chk)
-; CHECK-NEXT: call void @never_called(i1 true)
-; CHECK-NEXT: ret void
+; CHECK-LABEL: @dont_fold_guard(
+; CHECK-NEXT:  BB1:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[I:%.*]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32 [[I]], 0
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[C1]], [[C2]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    call void @never_called(i1 true)
+; CHECK-NEXT:    ret void
+;
   %c1 = icmp ult i32 %i, %length
   %c2 = icmp eq i32 %i, 0
   %wide.chk = and i1 %c1, %c2
@@ -210,12 +246,16 @@ declare void @dummy(i1) nounwind willreturn
 ; same as dont_fold_guard1 but there's a use immediately after guard and before
 ; branch. We can fold that use.
 define void @dont_fold_guard2(i8* %addr, i32 %i, i32 %length) {
-; CHECK-LABEL: dont_fold_guard2
-; CHECK: %wide.chk = and i1 %c1, %c2
-; CHECK-NEXT: experimental.guard(i1 %wide.chk)
-; CHECK-NEXT: dummy(i1 true)
-; CHECK-NEXT: call void @never_called(i1 true)
-; CHECK-NEXT: ret void
+; CHECK-LABEL: @dont_fold_guard2(
+; CHECK-NEXT:  BB1:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[I:%.*]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32 [[I]], 0
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[C1]], [[C2]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    call void @dummy(i1 true)
+; CHECK-NEXT:    call void @never_called(i1 true)
+; CHECK-NEXT:    ret void
+;
   %c1 = icmp ult i32 %i, %length
   %c2 = icmp eq i32 %i, 0
   %wide.chk = and i1 %c1, %c2
@@ -235,8 +275,15 @@ BB2:
 ; We cannot fold the guard under any circumstance.
 ; FIXME: We can merge unreachableBB2 into not_zero.
 define void @dont_fold_guard3(i8* %addr, i1 %cmp, i32 %i, i32 %length) {
-; CHECK-LABEL: dont_fold_guard3
-; CHECK: guard(i1 %cmp)
+; CHECK-LABEL: @dont_fold_guard3(
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       BB1:
+; CHECK-NEXT:    call void @never_called(i1 [[CMP]])
+; CHECK-NEXT:    ret void
+; CHECK:       BB2:
+; CHECK-NEXT:    ret void
+;
   call void(i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   br i1 %cmp, label %BB1, label %BB2
 
@@ -252,23 +299,28 @@ declare void @f(i1)
 ; Same as dont_fold_guard1 but use switch instead of branch.
 ; triggers source code `ProcessThreadableEdges`.
 define void @dont_fold_guard4(i1 %cmp1, i32 %i) nounwind {
-; CHECK-LABEL: dont_fold_guard4 
-; CHECK-LABEL: L2:
-; CHECK-NEXT: %cmp = icmp eq i32 %i, 0 
-; CHECK-NEXT: guard(i1 %cmp)
-; CHECK-NEXT: dummy(i1 true)
-; CHECK-NEXT: @f(i1 true)
-; CHECK-NEXT: ret void
+; CHECK-LABEL: @dont_fold_guard4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CMP1:%.*]], label [[L2:%.*]], label [[L3:%.*]]
+; CHECK:       L2:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; CHECK-NEXT:    call void @dummy(i1 true)
+; CHECK-NEXT:    call void @f(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       L3:
+; CHECK-NEXT:    ret void
+;
 entry:
-  br i1 %cmp1, label %L0, label %L3 
+  br i1 %cmp1, label %L0, label %L3
 L0:
   %cmp = icmp eq i32 %i, 0
   call void(i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   call void @dummy(i1 %cmp)
   switch i1 %cmp, label %L3 [
-    i1 false, label %L1
-    i1 true, label %L2
-    ]
+  i1 false, label %L1
+  i1 true, label %L2
+  ]
 
 L1:
   ret void
@@ -281,16 +333,17 @@ L3:
 
 ; Make sure that we don't PRE a non-speculable load across a guard.
 define void @unsafe_pre_across_guard(i8* %p, i1 %load.is.valid) {
-
 ; CHECK-LABEL: @unsafe_pre_across_guard(
-; CHECK-NOT:   loaded.pr
-; CHECK:       entry:
-; CHECK-NEXT:    br label %loop
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %load.is.valid) [ "deopt"() ]
-; CHECK-NEXT:    %loaded = load i8, i8* %p
-; CHECK-NEXT:    %continue = icmp eq i8 %loaded, 0
-; CHECK-NEXT:    br i1 %continue, label %exit, label %loop
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[LOAD_IS_VALID:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    [[LOADED:%.*]] = load i8, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp eq i8 [[LOADED]], 0
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -306,16 +359,18 @@ exit:                                             ; preds = %loop
 
 ; Make sure that we can safely PRE a speculable load across a guard.
 define void @safe_pre_across_guard(i8* noalias nocapture readonly dereferenceable(8) %p, i1 %load.is.valid) {
-
 ; CHECK-LABEL: @safe_pre_across_guard(
-; CHECK:       entry:
-; CHECK-NEXT:    %loaded.pr = load i8, i8* %p
-; CHECK-NEXT:    br label %loop
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOADED_PR:%.*]] = load i8, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    %loaded = phi i8 [ %loaded, %loop ], [ %loaded.pr, %entry ]
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %load.is.valid) [ "deopt"() ]
-; CHECK-NEXT:    %continue = icmp eq i8 %loaded, 0
-; CHECK-NEXT:    br i1 %continue, label %exit, label %loop
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[LOADED]], [[LOOP]] ], [ [[LOADED_PR]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[LOAD_IS_VALID:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp eq i8 [[LOADED]], 0
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -333,16 +388,17 @@ exit:                                             ; preds = %loop
 ; Make sure that we don't PRE a non-speculable load across a call which may
 ; alias with the load.
 define void @unsafe_pre_across_call(i8* %p) {
-
 ; CHECK-LABEL: @unsafe_pre_across_call(
-; CHECK-NOT:   loaded.pr
-; CHECK:       entry:
-; CHECK-NEXT:    br label %loop
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    call i32 @f1()
-; CHECK-NEXT:    %loaded = load i8, i8* %p
-; CHECK-NEXT:    %continue = icmp eq i8 %loaded, 0
-; CHECK-NEXT:    br i1 %continue, label %exit, label %loop
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @f1()
+; CHECK-NEXT:    [[LOADED:%.*]] = load i8, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp eq i8 [[LOADED]], 0
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
@@ -358,16 +414,18 @@ exit:                                             ; preds = %loop
 
 ; Make sure that we can safely PRE a speculable load across a call.
 define void @safe_pre_across_call(i8* noalias nocapture readonly dereferenceable(8) %p) {
-
 ; CHECK-LABEL: @safe_pre_across_call(
-; CHECK:       entry:
-; CHECK-NEXT:    %loaded.pr = load i8, i8* %p
-; CHECK-NEXT:    br label %loop
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOADED_PR:%.*]] = load i8, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    %loaded = phi i8 [ %loaded, %loop ], [ %loaded.pr, %entry ]
-; CHECK-NEXT:    call i32 @f1()
-; CHECK-NEXT:    %continue = icmp eq i8 %loaded, 0
-; CHECK-NEXT:    br i1 %continue, label %exit, label %loop
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[LOADED]], [[LOOP]] ], [ [[LOADED_PR]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @f1()
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp eq i8 [[LOADED]], 0
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/JumpThreading/pr43276.ll b/llvm/test/Transforms/JumpThreading/pr43276.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3d933867e7da39e4841fb47eff77079b63176bba
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr43276.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -aa-pipeline=basic-aa -passes='require<loops>,jump-threading' | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+@global = external global i8*
+
+define i32 @wibble() {
+; CHECK-LABEL: @wibble(
+; CHECK-NEXT:  bb19:
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i8, i8* undef, i64 16
+; CHECK-NEXT:    [[TMP21:%.*]] = load atomic i8*, i8** @global unordered, align 8
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, i8* [[TMP21]], i64 936
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i8* [ [[TMP22]], [[BB19:%.*]] ], [ undef, [[BB2:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i8* [ [[TMP21]], [[BB19]] ], [ undef, [[BB2]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[TMP]] to i64*
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[TMP4]], i64 848
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i8**
+; CHECK-NEXT:    br label [[BB11:%.*]]
+; CHECK:       bb11:
+; CHECK-NEXT:    [[TMP12:%.*]] = load atomic i8*, i8** [[TMP7]] unordered, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i8* [[TMP12]], null
+; CHECK-NEXT:    br i1 [[TMP13]], label [[BB17:%.*]], label [[BB16:%.*]]
+; CHECK:       bb16:
+; CHECK-NEXT:    store atomic i64 undef, i64* [[TMP5]] unordered, align 8
+; CHECK-NEXT:    br label [[BB11]]
+; CHECK:       bb17:
+; CHECK-NEXT:    ret i32 undef
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb
+  br label %bb18
+
+bb2:                                              ; No predecessors!
+  br label %bb3
+
+bb3:                                              ; preds = %bb19, %bb2
+  %tmp = phi i8* [ %tmp22, %bb19 ], [ undef, %bb2 ]
+  %tmp4 = phi i8* [ %tmp21, %bb19 ], [ undef, %bb2 ]
+  %tmp5 = bitcast i8* %tmp to i64*
+  %tmp6 = getelementptr inbounds i8, i8* %tmp4, i64 848
+  %tmp7 = bitcast i8* %tmp6 to i8**
+  br label %bb11
+
+bb11:                                             ; preds = %bb16, %bb3
+  %tmp12 = load atomic i8*, i8** %tmp7 unordered, align 8
+  %tmp13 = icmp eq i8* %tmp12, null
+  br i1 %tmp13, label %bb17, label %bb14
+
+bb14:                                             ; preds = %bb11
+  br label %bb15
+
+bb15:                                             ; preds = %bb14
+  br label %bb16
+
+bb16:                                             ; preds = %bb15
+  store atomic i64 undef, i64* %tmp5 unordered, align 8
+  br label %bb11
+
+bb17:                                             ; preds = %bb11
+  ret i32 undef
+
+bb18:                                             ; preds = %bb1
+  br label %bb19
+
+bb19:                                             ; preds = %bb18
+  %tmp20 = getelementptr i8, i8* undef, i64 16
+  %tmp21 = load atomic i8*, i8** @global unordered, align 8
+  %tmp22 = getelementptr inbounds i8, i8* %tmp21, i64 936
+  br label %bb3
+}
+
+define void @zot(i8* align 8 dereferenceable_or_null(16) %arg, i32 %arg1) {
+; CHECK-LABEL: @zot(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    ret void
+;
+bb:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/gc-relocate.ll b/llvm/test/Transforms/LICM/gc-relocate.ll
index ac5c8df47beed1291b243525d3e3a87eba2cfba4..e7498fb51e37fb07cb94aff63ca44e076cb817e1 100644
--- a/llvm/test/Transforms/LICM/gc-relocate.ll
+++ b/llvm/test/Transforms/LICM/gc-relocate.ll
@@ -10,18 +10,17 @@ define i8 addrspace(1)* @test(i8 addrspace(1)* %arg) #0 gc "statepoint-example"
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP:%.*]] = call token (i64, i32, i32 (i32, i8 addrspace(1)*, i32, i32, i32)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32i32p1i8i32i32i32f(i64 1, i32 16, i32 (i32, i8 addrspace(1)*, i32, i32, i32)* nonnull @zot, i32 5, i32 0, i32 undef, i8 addrspace(1)* undef, i32 undef, i32 undef, i32 undef, i32 0, i32 0) [ "deopt"(i32 0, i32 0, i32 0, i32 235, i32 3, i32 32, i32 0, i32 0, i8 addrspace(1)* undef, i32 3, i32 undef, i32 3, float undef, i32 0, i8 addrspace(1)* undef, i32 7, i8* null, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 0, i8 addrspace(1)* undef, i32 4, double undef, i32 7, i8* null, i32 0, i8 addrspace(1)* undef, i32 3, float undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* null, i32 3, i32 -15108, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null), "gc-live"(i8 addrspace(1)* [[ARG:%.*]]) ]
-; CHECK-NEXT:    [[RES:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[TMP]], i32 0, i32 0)
 ; CHECK-NEXT:    br i1 false, label [[BB1]], label [[BB2:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i8 addrspace(1)* [ [[RES]], [[BB1]] ]
-; CHECK-NEXT:    ret i8 addrspace(1)* [[RES_LCSSA]]
+; CHECK-NEXT:    [[RES_LE:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[TMP]], i32 0, i32 0) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret i8 addrspace(1)* [[RES_LE]]
 ;
 bb:
   br label %bb1
 
 bb1:
   %tmp = call token (i64, i32, i32 (i32, i8 addrspace(1)*, i32, i32, i32)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32i32p1i8i32i32i32f(i64 1, i32 16, i32 (i32, i8 addrspace(1)*, i32, i32, i32)* nonnull @zot, i32 5, i32 0, i32 undef, i8 addrspace(1)* undef, i32 undef, i32 undef, i32 undef, i32 0, i32 0) [ "deopt"(i32 0, i32 0, i32 0, i32 235, i32 3, i32 32, i32 0, i32 0, i8 addrspace(1)* undef, i32 3, i32 undef, i32 3, float undef, i32 0, i8 addrspace(1)* undef, i32 7, i8* null, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 3, i32 undef, i32 0, i8 addrspace(1)* undef, i32 4, double undef, i32 7, i8* null, i32 0, i8 addrspace(1)* undef, i32 3, float undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* undef, i32 0, i8 addrspace(1)* null, i32 3, i32 -15108, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null, i32 7, i8* null), "gc-live"(i8 addrspace(1)* %arg) ]
-  %res = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tmp, i32 0, i32 0)
+  %res = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tmp, i32 0, i32 0) readnone
   br i1 undef, label %bb1, label %bb2
 
 bb2:
diff --git a/llvm/test/Transforms/LICM/hoist-alloc.ll b/llvm/test/Transforms/LICM/hoist-alloc.ll
index 2ea64a519861c195387b99d0edc6b5c1410fa3d7..78196f57d4698e5a2bcc40f75813c53b51d01608 100644
--- a/llvm/test/Transforms/LICM/hoist-alloc.ll
+++ b/llvm/test/Transforms/LICM/hoist-alloc.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm -use-dereferenceable-at-point-semantics=0 < %s | FileCheck %s
+; RUN: opt -S -basic-aa -licm -use-dereferenceable-at-point-semantics=1 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -321,3 +322,86 @@ for.end:
   ret i8 %phi
 }
 
+declare noalias i8* @my_alloc(i64) allocsize(0)
+
+; FIXME: While the result shown here is correct for the test case as written,
+; it would require context sensitive reasoning about frees which we don't
+; currently have to reach this result.  So, this is effectively demonstrating
+; a miscompile which needs investigated further.
+define i8 @test_hoist_allocsize() {
+; CHECK-LABEL: @test_hoist_allocsize(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_RAW:%.*]] = call nonnull i8* @my_alloc(i64 32)
+; CHECK-NEXT:    call void @init(i8* [[A_RAW]])
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i8, i8* [[A_RAW]], i32 31
+; CHECK-NEXT:    [[RES:%.*]] = load i8, i8* [[ADDR]], align 1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @use(i8 [[RES]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 200
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i8 [ [[RES]], [[FOR_BODY]] ]
+; CHECK-NEXT:    call void @free(i8* [[A_RAW]])
+; CHECK-NEXT:    ret i8 [[RES_LCSSA]]
+;
+entry:
+  %a.raw = call nonnull i8* @my_alloc(i64 32)
+  call void @init(i8* %a.raw)
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  call void @unknown() ;; may throw
+  %addr = getelementptr i8, i8* %a.raw, i32 31
+  %res = load i8, i8* %addr
+  call void @use(i8 %res)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 200
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  call void @free(i8* %a.raw)
+  ret i8 %res
+}
+
+define i8 @test_hoist_allocsize_leak() {
+; CHECK-LABEL: @test_hoist_allocsize_leak(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_RAW:%.*]] = call nonnull i8* @my_alloc(i64 32)
+; CHECK-NEXT:    call void @init(i8* [[A_RAW]])
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i8, i8* [[A_RAW]], i32 31
+; CHECK-NEXT:    [[RES:%.*]] = load i8, i8* [[ADDR]], align 1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @use(i8 [[RES]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 200
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i8 [ [[RES]], [[FOR_BODY]] ]
+; CHECK-NEXT:    ret i8 [[RES_LCSSA]]
+;
+entry:
+  %a.raw = call nonnull i8* @my_alloc(i64 32)
+  call void @init(i8* %a.raw)
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  call void @unknown() ;; may throw
+  %addr = getelementptr i8, i8* %a.raw, i32 31
+  %res = load i8, i8* %addr
+  call void @use(i8 %res)
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 200
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i8 %res
+}
diff --git a/llvm/test/Transforms/LICM/hoist-deref-load.ll b/llvm/test/Transforms/LICM/hoist-deref-load.ll
index a5b4aed681246cbd9436bfd283438fe363b143af..e198394bc92a9f768167fdfc3d50a8fd97c344fd 100644
--- a/llvm/test/Transforms/LICM/hoist-deref-load.ll
+++ b/llvm/test/Transforms/LICM/hoist-deref-load.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -basic-aa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(loop-simplifycfg,licm)' -S < %s | FileCheck %s
 ; RUN: opt -S -basic-aa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
@@ -15,11 +16,36 @@ target triple = "x86_64-unknown-linux-gnu"
 ; and we want to hoist the load of %c out of the loop. This can be done only
 ; because the dereferenceable attribute is on %c.
 
-; CHECK-LABEL: @test1
-; CHECK: load i32, i32* %c, align 4
-; CHECK: for.body:
-
 define void @test1(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* nocapture readonly nonnull dereferenceable(4) align 4 %c, i32 %n) #0 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C:%.*]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp11 = icmp sgt i32 %n, 0
   br i1 %cmp11, label %for.body, label %for.end
@@ -51,12 +77,36 @@ for.end:                                          ; preds = %for.inc, %entry
 
 ; This is the same as @test1, but without the dereferenceable attribute on %c.
 ; Without this attribute, we should not hoist the load of %c.
-
-; CHECK-LABEL: @test2
-; CHECK: if.then:
-; CHECK: load i32, i32* %c, align 4
-
 define void @test2(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* nocapture readonly nonnull %c, i32 %n) #0 {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp11 = icmp sgt i32 %n, 0
   br i1 %cmp11, label %for.body, label %for.end
@@ -94,12 +144,37 @@ for.end:                                          ; preds = %for.inc, %entry
 ; }
 ; and we want to hoist the load of c[2] out of the loop. This can be done only
 ; because the dereferenceable attribute is on %c.
-
-; CHECK-LABEL: @test3
-; CHECK: load i32, i32* %c2, align 4
-; CHECK: for.body:
-
 define void @test3(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* nocapture readonly dereferenceable(12) align 4 %c, i32 %n) #0 {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[C2:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 2
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C2]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp11 = icmp sgt i32 %n, 0
   br i1 %cmp11, label %for.body, label %for.end
@@ -132,12 +207,37 @@ for.end:                                          ; preds = %for.inc, %entry
 
 ; This is the same as @test3, but with a dereferenceable attribute on %c with a
 ; size too small to cover c[2] (and so we should not hoist it).
-
-; CHECK-LABEL: @test4
-; CHECK: if.then:
-; CHECK: load i32, i32* %c2, align 4
-
 define void @test4(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* nocapture readonly dereferenceable(11) %c, i32 %n) #0 {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[C2:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 2
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp11 = icmp sgt i32 %n, 0
   br i1 %cmp11, label %for.body, label %for.end
@@ -178,12 +278,39 @@ for.end:                                          ; preds = %for.inc, %entry
 ; and we want to hoist the load of %c out of the loop. This can be done only
 ; because the dereferenceable_or_null attribute is on %c and there is a null
 ; check on %c.
-
-; CHECK-LABEL: @test5
-; CHECK: load i32, i32* %c, align 4
-; CHECK: for.body:
-
 define void @test5(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) align 4 %c, i32 %n) #0 {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i32* [[C:%.*]], null
+; CHECK-NEXT:    br i1 [[NOT_NULL]], label [[NOT_NULL:%.*]], label [[FOR_END:%.*]]
+; CHECK:       not.null:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %not_null = icmp ne i32* %c, null
   br i1 %not_null, label %not.null, label %for.end
@@ -221,13 +348,38 @@ for.end:                                          ; preds = %for.inc, %entry, %n
 ; Without this check, we should not hoist the load of %c.
 
 ; This test case has an icmp on c but the use of this comparison is
-; not a branch. 
-
-; CHECK-LABEL: @test6
-; CHECK: if.then:
-; CHECK: load i32, i32* %c, align 4
-
+; not a branch.
 define i1 @test6(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n) #0 {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i32* [[C:%.*]], null
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
 entry:
   %not_null = icmp ne i32* %c, null
   %cmp11 = icmp sgt i32 %n, 0
@@ -267,12 +419,37 @@ for.end:                                          ; preds = %for.inc, %entry
 ; }
 ; and we want to hoist the load of %c out of the loop. This can be done only
 ; because the dereferenceable meatdata on the c = *cptr load.
-
-; CHECK-LABEL: @test7
-; CHECK: load i32, i32* %c, align 4
-; CHECK: for.body:
-
 define void @test7(i32* noalias %a, i32* %b, i32** %cptr, i32 %n) #0 {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = load i32*, i32** [[CPTR:%.*]], align 8, !dereferenceable !0, !align !0
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %c = load i32*, i32** %cptr, !dereferenceable !0, !align !{i64 4}
   %cmp11 = icmp sgt i32 %n, 0
@@ -312,14 +489,42 @@ for.end:                                          ; preds = %for.inc, %entry
 ;         a[i] = (*c)*b[i];
 ; }
 ; and we want to hoist the load of %c out of the loop. This can be done only
-; because the dereferenceable_or_null meatdata on the c = *cptr load and there 
+; because the dereferenceable_or_null meatdata on the c = *cptr load and there
 ; is a null check on %c.
-
-; CHECK-LABEL: @test8
-; CHECK: load i32, i32* %c, align 4
-; CHECK: for.body:
-
 define void @test8(i32* noalias %a, i32* %b, i32** %cptr, i32 %n) #0 {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = load i32*, i32** [[CPTR:%.*]], align 8, !dereferenceable_or_null !0, !align !0
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i32* [[C]], null
+; CHECK-NEXT:    br i1 [[NOT_NULL]], label [[NOT_NULL:%.*]], label [[FOR_END:%.*]]
+; CHECK:       not.null:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %c = load i32*, i32** %cptr, !dereferenceable_or_null !0, !align !{i64 4}
   %not_null = icmp ne i32* %c, null
@@ -356,12 +561,37 @@ for.end:                                          ; preds = %for.inc, %entry, %n
 
 ; This is the same as @test8, but without the null check on %c.
 ; Without this check, we should not hoist the load of %c.
-
-; CHECK-LABEL: @test9
-; CHECK: if.then:
-; CHECK: load i32, i32* %c, align 4
-
 define void @test9(i32* noalias %a, i32* %b, i32** %cptr, i32 %n) #0 {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = load i32*, i32** [[CPTR:%.*]], align 8, !dereferenceable_or_null !0
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %c = load i32*, i32** %cptr, !dereferenceable_or_null !0
   %cmp11 = icmp sgt i32 %n, 0
@@ -398,14 +628,37 @@ for.end:                                          ; preds = %for.inc, %entry
 ; on the dereferenceability anymore.
 ; In other words this test checks that we strip dereferenceability  metadata
 ; after hoisting an instruction.
-
-; CHECK-LABEL: @test10
-; CHECK: %c = load i32*, i32** %cptr
-; CHECK-NOT: dereferenceable
-; CHECK: if.then:
-; CHECK: load i32, i32* %c, align 4
-
 define void @test10(i32* noalias %a, i32* %b, i32** dereferenceable(8) align 8 %cptr, i32 %n) #0 {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[C:%.*]] = load i32*, i32** [[CPTR:%.*]], align 8
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp11 = icmp sgt i32 %n, 0
   br i1 %cmp11, label %for.body, label %for.end
@@ -438,13 +691,39 @@ for.end:                                          ; preds = %for.inc, %entry
 
 define void @test11(i32* noalias %a, i32* %b, i32** dereferenceable(8) %cptr, i32 %n) #0 {
 ; CHECK-LABEL: @test11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[C:%.*]] = load i32*, i32** [[CPTR:%.*]], align 8, !dereferenceable !0
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[D:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[E:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[E]], [[D]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp11 = icmp sgt i32 %n, 0
   br i1 %cmp11, label %for.body, label %for.end
 
-; CHECK: for.body.preheader:
-; CHECK:  %c = load i32*, i32** %cptr, align 8, !dereferenceable !0
-; CHECK:  %d = load i32, i32* %c, align 4
 
 
 for.body:                                         ; preds = %entry, %for.inc
@@ -475,19 +754,45 @@ for.end:                                          ; preds = %for.inc, %entry
 
 declare void @llvm.experimental.guard(i1, ...)
 
-define void @test12(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) align 4 %c, i32 %n) #0 {
 ; Prove non-null ness of %c via a guard, not a branch.
-
+define void @test12(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) align 4 %c, i32 %n) #0 {
 ; CHECK-LABEL: @test12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i32* [[C:%.*]], null
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[NOT_NULL]]) [ "deopt"() ]
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %not_null = icmp ne i32* %c, null
   call void(i1, ...) @llvm.experimental.guard(i1 %not_null) [ "deopt"() ]
   %cmp11 = icmp sgt i32 %n, 0
   br i1 %cmp11, label %for.body, label %for.end
 
-; CHECK: for.body.preheader:
-; CHECK-NEXT:  [[VAL:%[^ ]]] = load i32, i32* %c, align 4
-; CHECK-NEXT:  br label %for.body
 
 
 for.body:                                         ; preds = %entry, %for.inc
@@ -515,19 +820,45 @@ for.end:                                          ; preds = %for.inc, %entry, %e
   ret void
 }
 
-define void @test13(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n) #0 {
 ; Like @test12, but has a post-dominating guard, which cannot be used
 ; to prove %c is nonnull at the point of the load.
-
+define void @test13(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n) #0 {
 ; CHECK-LABEL: @test13(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i32* [[C:%.*]], null
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[NOT_NULL]]) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+;
 entry:
   %not_null = icmp ne i32* %c, null
   %cmp11 = icmp sgt i32 %n, 0
   br i1 %cmp11, label %for.body, label %for.end
 
-; CHECK: for.body.preheader:
-; CHECK-NOT:  load i32, i32* %c
-; CHECK:  br label %for.body
 
 for.body:                                         ; preds = %entry, %for.inc
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
@@ -537,9 +868,6 @@ for.body:                                         ; preds = %entry, %for.inc
   br i1 %cmp1, label %if.then, label %for.inc
 
 if.then:                                          ; preds = %for.body
-; CHECK: if.then:
-; CHECK:  load i32, i32* %c
-; CHECK:  br label %for.inc
   %1 = load i32, i32* %c, align 4
   %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
   %2 = load i32, i32* %arrayidx3, align 4
@@ -561,10 +889,39 @@ for.end:                                          ; preds = %for.inc, %entry, %e
 ; Check that branch by condition "null check AND something" allows to hoist the
 ; load.
 define void @test14(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) align 4 %c, i32 %n, i1 %dummy_cond) #0 {
-
-; CHECK-LABEL: @test14
-; CHECK: load i32, i32* %c, align 4
-; CHECK: for.body:
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i32* [[C:%.*]], null
+; CHECK-NEXT:    [[DUMMY_AND:%.*]] = and i1 [[NOT_NULL]], [[DUMMY_COND:%.*]]
+; CHECK-NEXT:    br i1 [[DUMMY_AND]], label [[NOT_NULL:%.*]], label [[FOR_END:%.*]]
+; CHECK:       not.null:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %not_null = icmp ne i32* %c, null
@@ -603,10 +960,38 @@ for.end:                                          ; preds = %for.inc, %entry, %n
 ; Check that guard by condition "null check AND something" allows to hoist the
 ; load.
 define void @test15(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) align 4 %c, i32 %n, i1 %dummy_cond) #0 {
-
-; CHECK-LABEL: @test15
-; CHECK: load i32, i32* %c, align 4
-; CHECK: for.body:
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i32* [[C:%.*]], null
+; CHECK-NEXT:    [[DUMMY_AND:%.*]] = and i1 [[NOT_NULL]], [[DUMMY_COND:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[DUMMY_AND]]) [ "deopt"() ]
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %not_null = icmp ne i32* %c, null
@@ -644,10 +1029,39 @@ for.end:                                          ; preds = %for.inc, %entry
 ; non-null in false branch. So the condition ((c == null && other_cond) == false)
 ; is not sufficient to conclude that c != null.
 define void @test16(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n, i1 %dummy_cond) #0 {
-
-; CHECK-LABEL: @test16
-; CHECK: for.body:
-; CHECK: load i32, i32* %c, align 4
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i32* [[C:%.*]], null
+; CHECK-NEXT:    [[DUMMY_AND:%.*]] = and i1 [[NOT_NULL]], [[DUMMY_COND:%.*]]
+; CHECK-NEXT:    br i1 [[DUMMY_AND]], label [[FOR_END:%.*]], label [[NOT_NULL:%.*]]
+; CHECK:       not.null:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %not_null = icmp eq i32* %c, null
@@ -687,10 +1101,38 @@ for.end:                                          ; preds = %for.inc, %entry, %n
 ; non-null in false branch. So the condition ((c == null && other_cond) == false)
 ; is not sufficient to conclude that c != null.
 define void @test17(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n, i1 %dummy_cond) #0 {
-
-; CHECK-LABEL: @test17
-; CHECK: for.body:
-; CHECK: load i32, i32* %c, align 4
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i32* [[C:%.*]], null
+; CHECK-NEXT:    [[DUMMY_AND:%.*]] = and i1 [[NOT_NULL]], [[DUMMY_COND:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[DUMMY_AND]]) [ "deopt"() ]
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %not_null = icmp eq i32* %c, null
diff --git a/llvm/test/Transforms/LoopFlatten/pr49571.ll b/llvm/test/Transforms/LoopFlatten/pr49571.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0037064245d0008ed14fb5796871029621b9499d
--- /dev/null
+++ b/llvm/test/Transforms/LoopFlatten/pr49571.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -loop-flatten -verify-loop-info -verify-dom-info -verify-scev -verify | FileCheck %s
+
+; Testcase of PR49571
+; Previously we had an assertion that the incoming value from the
+; loop latch (%inc) is the same as the operator in the compare
+; statement (%0). This does not necessarily hold as demonstrated
+; in the following case.
+
+define dso_local void @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[A_03:%.*]] = phi i32 [ 0, [[FOR_COND]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[A_03]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[A_03]], 1
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    br label [[FOR_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.end, %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %for.cond
+  %a.03 = phi i32 [ 0, %for.cond ], [ %inc, %for.inc ]
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %0 = add i32 %a.03, 1
+  %cmp = icmp slt i32 %0, 10
+  %inc = add nsw i32 %a.03, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/LoopIdiom/RISCV/lit.local.cfg b/llvm/test/Transforms/LoopIdiom/RISCV/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..c63820126f8ca6ce0511f50de23b37b5e484bb70
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/RISCV/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'RISCV' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopIdiom/RISCV/popcnt.ll b/llvm/test/Transforms/LoopIdiom/RISCV/popcnt.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b8c00eaf721637e73a0a5b684b4c55a48c11fb61
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/RISCV/popcnt.ll
@@ -0,0 +1,322 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-idiom -mtriple=riscv32 -mattr=+experimental-zbb -S < %s | FileCheck %s --check-prefixes=CPOP
+; RUN: opt -loop-idiom -mtriple=riscv64 -mattr=+experimental-zbb -S < %s | FileCheck %s --check-prefixes=CPOP
+; RUN: opt -loop-idiom -mtriple=riscv32 -mattr=+experimental-b -S < %s | FileCheck %s --check-prefixes=CPOP
+; RUN: opt -loop-idiom -mtriple=riscv64 -mattr=+experimental-b -S < %s | FileCheck %s --check-prefixes=CPOP
+; RUN: opt -loop-idiom -mtriple=riscv32 -S < %s | FileCheck %s --check-prefixes=NOCPOP
+; RUN: opt -loop-idiom -mtriple=riscv64 -S < %s | FileCheck %s --check-prefixes=NOCPOP
+
+; Mostly copied from AMDGPU version.
+
+;To recognize this pattern:
+;int popcount(unsigned long long a) {
+;    int c = 0;
+;    while (a) {
+;        c++;
+;        a &= a - 1;
+;    }
+;    return c;
+;}
+;
+
+define i32 @popcount_i64(i64 %a) nounwind uwtable readnone ssp {
+; CPOP-LABEL: @popcount_i64(
+; CPOP-NEXT:  entry:
+; CPOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[A:%.*]])
+; CPOP-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CPOP-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CPOP-NEXT:    br i1 [[TMP2]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CPOP:       while.body.preheader:
+; CPOP-NEXT:    br label [[WHILE_BODY:%.*]]
+; CPOP:       while.body:
+; CPOP-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY_PREHEADER]] ], [ [[TCDEC:%.*]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    [[C_05:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[A_ADDR_04:%.*]] = phi i64 [ [[AND:%.*]], [[WHILE_BODY]] ], [ [[A]], [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[INC]] = add nsw i32 [[C_05]], 1
+; CPOP-NEXT:    [[SUB:%.*]] = add i64 [[A_ADDR_04]], -1
+; CPOP-NEXT:    [[AND]] = and i64 [[SUB]], [[A_ADDR_04]]
+; CPOP-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; CPOP-NEXT:    [[TOBOOL:%.*]] = icmp sle i32 [[TCDEC]], 0
+; CPOP-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; CPOP:       while.end.loopexit:
+; CPOP-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    br label [[WHILE_END]]
+; CPOP:       while.end:
+; CPOP-NEXT:    [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; CPOP-NEXT:    ret i32 [[C_0_LCSSA]]
+;
+; NOCPOP-LABEL: @popcount_i64(
+; NOCPOP-NEXT:  entry:
+; NOCPOP-NEXT:    [[TOBOOL3:%.*]] = icmp eq i64 [[A:%.*]], 0
+; NOCPOP-NEXT:    br i1 [[TOBOOL3]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; NOCPOP:       while.body.preheader:
+; NOCPOP-NEXT:    br label [[WHILE_BODY:%.*]]
+; NOCPOP:       while.body:
+; NOCPOP-NEXT:    [[C_05:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[A_ADDR_04:%.*]] = phi i64 [ [[AND:%.*]], [[WHILE_BODY]] ], [ [[A]], [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[INC]] = add nsw i32 [[C_05]], 1
+; NOCPOP-NEXT:    [[SUB:%.*]] = add i64 [[A_ADDR_04]], -1
+; NOCPOP-NEXT:    [[AND]] = and i64 [[SUB]], [[A_ADDR_04]]
+; NOCPOP-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[AND]], 0
+; NOCPOP-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; NOCPOP:       while.end.loopexit:
+; NOCPOP-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; NOCPOP-NEXT:    br label [[WHILE_END]]
+; NOCPOP:       while.end:
+; NOCPOP-NEXT:    [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; NOCPOP-NEXT:    ret i32 [[C_0_LCSSA]]
+;
+entry:
+  %tobool3 = icmp eq i64 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i64 %a.addr.04, -1
+  %and = and i64 %sub, %a.addr.04
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
+define i32 @popcount_i32(i32 %a) nounwind uwtable readnone ssp {
+; CPOP-LABEL: @popcount_i32(
+; CPOP-NEXT:  entry:
+; CPOP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.ctpop.i32(i32 [[A:%.*]])
+; CPOP-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
+; CPOP-NEXT:    br i1 [[TMP1]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CPOP:       while.body.preheader:
+; CPOP-NEXT:    br label [[WHILE_BODY:%.*]]
+; CPOP:       while.body:
+; CPOP-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP0]], [[WHILE_BODY_PREHEADER]] ], [ [[TCDEC:%.*]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    [[C_05:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[A_ADDR_04:%.*]] = phi i32 [ [[AND:%.*]], [[WHILE_BODY]] ], [ [[A]], [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[INC]] = add nsw i32 [[C_05]], 1
+; CPOP-NEXT:    [[SUB:%.*]] = add i32 [[A_ADDR_04]], -1
+; CPOP-NEXT:    [[AND]] = and i32 [[SUB]], [[A_ADDR_04]]
+; CPOP-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; CPOP-NEXT:    [[TOBOOL:%.*]] = icmp sle i32 [[TCDEC]], 0
+; CPOP-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; CPOP:       while.end.loopexit:
+; CPOP-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[TMP0]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    br label [[WHILE_END]]
+; CPOP:       while.end:
+; CPOP-NEXT:    [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; CPOP-NEXT:    ret i32 [[C_0_LCSSA]]
+;
+; NOCPOP-LABEL: @popcount_i32(
+; NOCPOP-NEXT:  entry:
+; NOCPOP-NEXT:    [[TOBOOL3:%.*]] = icmp eq i32 [[A:%.*]], 0
+; NOCPOP-NEXT:    br i1 [[TOBOOL3]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; NOCPOP:       while.body.preheader:
+; NOCPOP-NEXT:    br label [[WHILE_BODY:%.*]]
+; NOCPOP:       while.body:
+; NOCPOP-NEXT:    [[C_05:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[A_ADDR_04:%.*]] = phi i32 [ [[AND:%.*]], [[WHILE_BODY]] ], [ [[A]], [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[INC]] = add nsw i32 [[C_05]], 1
+; NOCPOP-NEXT:    [[SUB:%.*]] = add i32 [[A_ADDR_04]], -1
+; NOCPOP-NEXT:    [[AND]] = and i32 [[SUB]], [[A_ADDR_04]]
+; NOCPOP-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[AND]], 0
+; NOCPOP-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; NOCPOP:       while.end.loopexit:
+; NOCPOP-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; NOCPOP-NEXT:    br label [[WHILE_END]]
+; NOCPOP:       while.end:
+; NOCPOP-NEXT:    [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; NOCPOP-NEXT:    ret i32 [[C_0_LCSSA]]
+;
+entry:
+  %tobool3 = icmp eq i32 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i32 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i32 %a.addr.04, -1
+  %and = and i32 %sub, %a.addr.04
+  %tobool = icmp eq i32 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
+define i32 @popcount_i128(i128 %a) nounwind uwtable readnone ssp {
+; CPOP-LABEL: @popcount_i128(
+; CPOP-NEXT:  entry:
+; CPOP-NEXT:    [[TMP0:%.*]] = call i128 @llvm.ctpop.i128(i128 [[A:%.*]])
+; CPOP-NEXT:    [[TMP1:%.*]] = trunc i128 [[TMP0]] to i32
+; CPOP-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CPOP-NEXT:    br i1 [[TMP2]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CPOP:       while.body.preheader:
+; CPOP-NEXT:    br label [[WHILE_BODY:%.*]]
+; CPOP:       while.body:
+; CPOP-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY_PREHEADER]] ], [ [[TCDEC:%.*]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    [[C_05:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[A_ADDR_04:%.*]] = phi i128 [ [[AND:%.*]], [[WHILE_BODY]] ], [ [[A]], [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[INC]] = add nsw i32 [[C_05]], 1
+; CPOP-NEXT:    [[SUB:%.*]] = add i128 [[A_ADDR_04]], -1
+; CPOP-NEXT:    [[AND]] = and i128 [[SUB]], [[A_ADDR_04]]
+; CPOP-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; CPOP-NEXT:    [[TOBOOL:%.*]] = icmp sle i32 [[TCDEC]], 0
+; CPOP-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; CPOP:       while.end.loopexit:
+; CPOP-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    br label [[WHILE_END]]
+; CPOP:       while.end:
+; CPOP-NEXT:    [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; CPOP-NEXT:    ret i32 [[C_0_LCSSA]]
+;
+; NOCPOP-LABEL: @popcount_i128(
+; NOCPOP-NEXT:  entry:
+; NOCPOP-NEXT:    [[TOBOOL3:%.*]] = icmp eq i128 [[A:%.*]], 0
+; NOCPOP-NEXT:    br i1 [[TOBOOL3]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; NOCPOP:       while.body.preheader:
+; NOCPOP-NEXT:    br label [[WHILE_BODY:%.*]]
+; NOCPOP:       while.body:
+; NOCPOP-NEXT:    [[C_05:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[A_ADDR_04:%.*]] = phi i128 [ [[AND:%.*]], [[WHILE_BODY]] ], [ [[A]], [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[INC]] = add nsw i32 [[C_05]], 1
+; NOCPOP-NEXT:    [[SUB:%.*]] = add i128 [[A_ADDR_04]], -1
+; NOCPOP-NEXT:    [[AND]] = and i128 [[SUB]], [[A_ADDR_04]]
+; NOCPOP-NEXT:    [[TOBOOL:%.*]] = icmp eq i128 [[AND]], 0
+; NOCPOP-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; NOCPOP:       while.end.loopexit:
+; NOCPOP-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; NOCPOP-NEXT:    br label [[WHILE_END]]
+; NOCPOP:       while.end:
+; NOCPOP-NEXT:    [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; NOCPOP-NEXT:    ret i32 [[C_0_LCSSA]]
+;
+entry:
+  %tobool3 = icmp eq i128 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i128 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i128 %a.addr.04, -1
+  %and = and i128 %sub, %a.addr.04
+  %tobool = icmp eq i128 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
+; To recognize this pattern:
+;int popcount(unsigned long long a, int mydata1, int mydata2) {
+;    int c = 0;
+;    while (a) {
+;        c++;
+;        a &= a - 1;
+;        mydata1 *= c;
+;        mydata2 *= (int)a;
+;    }
+;    return c + mydata1 + mydata2;
+;}
+
+define i32 @popcount2(i64 %a, i32 %mydata1, i32 %mydata2) nounwind uwtable readnone ssp {
+; CPOP-LABEL: @popcount2(
+; CPOP-NEXT:  entry:
+; CPOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[A:%.*]])
+; CPOP-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CPOP-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CPOP-NEXT:    br i1 [[TMP2]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CPOP:       while.body.preheader:
+; CPOP-NEXT:    br label [[WHILE_BODY:%.*]]
+; CPOP:       while.body:
+; CPOP-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY_PREHEADER]] ], [ [[TCDEC:%.*]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    [[C_013:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[MYDATA2_ADDR_012:%.*]] = phi i32 [ [[MUL1:%.*]], [[WHILE_BODY]] ], [ [[MYDATA2:%.*]], [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[MYDATA1_ADDR_011:%.*]] = phi i32 [ [[MUL:%.*]], [[WHILE_BODY]] ], [ [[MYDATA1:%.*]], [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[A_ADDR_010:%.*]] = phi i64 [ [[AND:%.*]], [[WHILE_BODY]] ], [ [[A]], [[WHILE_BODY_PREHEADER]] ]
+; CPOP-NEXT:    [[INC]] = add nsw i32 [[C_013]], 1
+; CPOP-NEXT:    [[SUB:%.*]] = add i64 [[A_ADDR_010]], -1
+; CPOP-NEXT:    [[AND]] = and i64 [[SUB]], [[A_ADDR_010]]
+; CPOP-NEXT:    [[MUL]] = mul nsw i32 [[INC]], [[MYDATA1_ADDR_011]]
+; CPOP-NEXT:    [[CONV:%.*]] = trunc i64 [[AND]] to i32
+; CPOP-NEXT:    [[MUL1]] = mul nsw i32 [[CONV]], [[MYDATA2_ADDR_012]]
+; CPOP-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; CPOP-NEXT:    [[TOBOOL:%.*]] = icmp sle i32 [[TCDEC]], 0
+; CPOP-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; CPOP:       while.end.loopexit:
+; CPOP-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    [[MUL1_LCSSA:%.*]] = phi i32 [ [[MUL1]], [[WHILE_BODY]] ]
+; CPOP-NEXT:    br label [[WHILE_END]]
+; CPOP:       while.end:
+; CPOP-NEXT:    [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; CPOP-NEXT:    [[MYDATA2_ADDR_0_LCSSA:%.*]] = phi i32 [ [[MYDATA2]], [[ENTRY]] ], [ [[MUL1_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; CPOP-NEXT:    [[MYDATA1_ADDR_0_LCSSA:%.*]] = phi i32 [ [[MYDATA1]], [[ENTRY]] ], [ [[MUL_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; CPOP-NEXT:    [[ADD:%.*]] = add i32 [[MYDATA2_ADDR_0_LCSSA]], [[MYDATA1_ADDR_0_LCSSA]]
+; CPOP-NEXT:    [[ADD2:%.*]] = add i32 [[ADD]], [[C_0_LCSSA]]
+; CPOP-NEXT:    ret i32 [[ADD2]]
+;
+; NOCPOP-LABEL: @popcount2(
+; NOCPOP-NEXT:  entry:
+; NOCPOP-NEXT:    [[TOBOOL9:%.*]] = icmp eq i64 [[A:%.*]], 0
+; NOCPOP-NEXT:    br i1 [[TOBOOL9]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; NOCPOP:       while.body.preheader:
+; NOCPOP-NEXT:    br label [[WHILE_BODY:%.*]]
+; NOCPOP:       while.body:
+; NOCPOP-NEXT:    [[C_013:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[MYDATA2_ADDR_012:%.*]] = phi i32 [ [[MUL1:%.*]], [[WHILE_BODY]] ], [ [[MYDATA2:%.*]], [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[MYDATA1_ADDR_011:%.*]] = phi i32 [ [[MUL:%.*]], [[WHILE_BODY]] ], [ [[MYDATA1:%.*]], [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[A_ADDR_010:%.*]] = phi i64 [ [[AND:%.*]], [[WHILE_BODY]] ], [ [[A]], [[WHILE_BODY_PREHEADER]] ]
+; NOCPOP-NEXT:    [[INC]] = add nsw i32 [[C_013]], 1
+; NOCPOP-NEXT:    [[SUB:%.*]] = add i64 [[A_ADDR_010]], -1
+; NOCPOP-NEXT:    [[AND]] = and i64 [[SUB]], [[A_ADDR_010]]
+; NOCPOP-NEXT:    [[MUL]] = mul nsw i32 [[INC]], [[MYDATA1_ADDR_011]]
+; NOCPOP-NEXT:    [[CONV:%.*]] = trunc i64 [[AND]] to i32
+; NOCPOP-NEXT:    [[MUL1]] = mul nsw i32 [[CONV]], [[MYDATA2_ADDR_012]]
+; NOCPOP-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[AND]], 0
+; NOCPOP-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; NOCPOP:       while.end.loopexit:
+; NOCPOP-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; NOCPOP-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[WHILE_BODY]] ]
+; NOCPOP-NEXT:    [[MUL1_LCSSA:%.*]] = phi i32 [ [[MUL1]], [[WHILE_BODY]] ]
+; NOCPOP-NEXT:    br label [[WHILE_END]]
+; NOCPOP:       while.end:
+; NOCPOP-NEXT:    [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; NOCPOP-NEXT:    [[MYDATA2_ADDR_0_LCSSA:%.*]] = phi i32 [ [[MYDATA2]], [[ENTRY]] ], [ [[MUL1_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; NOCPOP-NEXT:    [[MYDATA1_ADDR_0_LCSSA:%.*]] = phi i32 [ [[MYDATA1]], [[ENTRY]] ], [ [[MUL_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; NOCPOP-NEXT:    [[ADD:%.*]] = add i32 [[MYDATA2_ADDR_0_LCSSA]], [[MYDATA1_ADDR_0_LCSSA]]
+; NOCPOP-NEXT:    [[ADD2:%.*]] = add i32 [[ADD]], [[C_0_LCSSA]]
+; NOCPOP-NEXT:    ret i32 [[ADD2]]
+;
+entry:
+  %tobool9 = icmp eq i64 %a, 0
+  br i1 %tobool9, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.013 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %mydata2.addr.012 = phi i32 [ %mul1, %while.body ], [ %mydata2, %entry ]
+  %mydata1.addr.011 = phi i32 [ %mul, %while.body ], [ %mydata1, %entry ]
+  %a.addr.010 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.013, 1
+  %sub = add i64 %a.addr.010, -1
+  %and = and i64 %sub, %a.addr.010
+  %mul = mul nsw i32 %inc, %mydata1.addr.011
+  %conv = trunc i64 %and to i32
+  %mul1 = mul nsw i32 %conv, %mydata2.addr.012
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  %mydata2.addr.0.lcssa = phi i32 [ %mydata2, %entry ], [ %mul1, %while.body ]
+  %mydata1.addr.0.lcssa = phi i32 [ %mydata1, %entry ], [ %mul, %while.body ]
+  %add = add i32 %mydata2.addr.0.lcssa, %mydata1.addr.0.lcssa
+  %add2 = add i32 %add, %c.0.lcssa
+  ret i32 %add2
+}
diff --git a/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll b/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll
index 167d4bfdc8bb563c27b06087e402dce00d0a7b15..82f661502f5736f35d60777c3adf22ad72b7339c 100644
--- a/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll
+++ b/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll
@@ -103,3 +103,41 @@ for.inc10:
 for.end12:
   ret void
 }
+
+;; The following Loop is not considered tightly nested and is not interchanged.
+;; The outer loop header does not branch to the inner loop preheader, or the
+;; inner loop header, or the outer loop latch.
+; CHECK: Not interchanging loops. Cannot prove legality.
+define void @interchange_07(i32 %k, i32 %N, i32 %ny) {
+entry:
+  br label %for1.header
+
+for1.header:
+  %j23 = phi i32 [ 0, %entry ], [ %j.next24, %for1.inc10 ]
+  %cmp21 = icmp slt i32 0, %ny
+  br label %singleSucc
+
+singleSucc:
+  br i1 %cmp21, label %preheader.j, label %for1.inc10
+
+preheader.j:
+  br label %for2
+
+for2:
+  %j = phi i32 [ %j.next, %for2 ], [ 0, %preheader.j ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i32 0, i32 %j, i32 %j23
+  %lv = load i32, i32* %arrayidx5
+  %add = add nsw i32 %lv, %k
+  store i32 %add, i32* %arrayidx5
+  %j.next = add nuw nsw i32 %j, 1
+  %exitcond = icmp eq i32 %j, 99
+  br i1 %exitcond, label %for1.inc10, label %for2
+
+for1.inc10:
+  %j.next24 = add nuw nsw i32 %j23, 1
+  %exitcond26 = icmp eq i32 %j23, 99
+  br i1 %exitcond26, label %for.end12, label %for1.header
+
+for.end12:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/pr-49141.ll b/llvm/test/Transforms/LoopLoadElim/pr-49141.ll
index df7a97581d6a5cf0e13fe44a1b7b1c4af6e994ab..8c7b5f163419eec0142050bb0991d0d2933c37c7 100644
--- a/llvm/test/Transforms/LoopLoadElim/pr-49141.ll
+++ b/llvm/test/Transforms/LoopLoadElim/pr-49141.ll
@@ -8,12 +8,10 @@ define void @test() {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br i1 true, label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[A_01:%.*]] = phi i16 [ undef, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[INC]] = add nsw i16 [[A_01]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i16 [[INC]], 2
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 false, [[CMP]]
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK-NEXT:    br i1 false, label [[FOR_COND:%.*]], label [[FOR_END]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopSimplify/merge-exits.ll b/llvm/test/Transforms/LoopSimplify/merge-exits.ll
index 5cdf814877842c929cebe63556c4db19cfa8b7b2..4efc7e963048ad89f926fe4b4377468ea22e9878 100644
--- a/llvm/test/Transforms/LoopSimplify/merge-exits.ll
+++ b/llvm/test/Transforms/LoopSimplify/merge-exits.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -loop-simplify -loop-rotate -instcombine -indvars -S -verify-loop-info -verify-dom-info | FileCheck %s
 
 ; Loopsimplify should be able to merge the two loop exits
@@ -7,42 +8,143 @@
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n32:64"
 
-; CHECK-LABEL: @test1
-; CHECK: bb:
-; CHECK: phi i64
-; CHECK-NOT: phi i64
-; CHECK-NOT: sext
-
 define float @test1(float* %pTmp1, float* %peakWeight, i32 %bandEdgeIndex) nounwind {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = load float, float* [[PEAKWEIGHT:%.*]], align 4
+; CHECK-NEXT:    [[T11:%.*]] = add i32 [[BANDEDGEINDEX:%.*]], -1
+; CHECK-NEXT:    [[T121:%.*]] = icmp sgt i32 [[T11]], 0
+; CHECK-NEXT:    br i1 [[T121]], label [[BB_LR_PH:%.*]], label [[BB3:%.*]]
+; CHECK:       bb.lr.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[T11]] to i64
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[BB_LR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[DISTERBHI_04:%.*]] = phi float [ 0.000000e+00, [[BB_LR_PH]] ], [ [[T4:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[PEAKCOUNT_02:%.*]] = phi float [ [[T0]], [[BB_LR_PH]] ], [ [[T9:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[T2:%.*]] = getelementptr float, float* [[PTMP1:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[T3:%.*]] = load float, float* [[T2]], align 4
+; CHECK-NEXT:    [[T4]] = fadd float [[T3]], [[DISTERBHI_04]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[T7:%.*]] = getelementptr float, float* [[PEAKWEIGHT]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[T8:%.*]] = load float, float* [[T7]], align 4
+; CHECK-NEXT:    [[T9]] = fadd float [[T8]], [[PEAKCOUNT_02]]
+; CHECK-NEXT:    [[T10:%.*]] = fcmp olt float [[T4]], 2.500000e+00
+; CHECK-NEXT:    [[T12:%.*]] = icmp sgt i64 [[TMP0]], [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[T10]], [[T12]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[BB]], label [[BB1_BB3_CRIT_EDGE:%.*]]
+; CHECK:       bb1.bb3_crit_edge:
+; CHECK-NEXT:    [[T4_LCSSA:%.*]] = phi float [ [[T4]], [[BB]] ]
+; CHECK-NEXT:    [[T9_LCSSA:%.*]] = phi float [ [[T9]], [[BB]] ]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[PEAKCOUNT_0_LCSSA:%.*]] = phi float [ [[T9_LCSSA]], [[BB1_BB3_CRIT_EDGE]] ], [ [[T0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[DISTERBHI_0_LCSSA:%.*]] = phi float [ [[T4_LCSSA]], [[BB1_BB3_CRIT_EDGE]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-NEXT:    [[T13:%.*]] = fdiv float [[PEAKCOUNT_0_LCSSA]], [[DISTERBHI_0_LCSSA]]
+; CHECK-NEXT:    ret float [[T13]]
+;
+entry:
+  %t0 = load float, float* %peakWeight, align 4
+  br label %bb1
+
+bb:		; preds = %bb2
+  %t1 = sext i32 %hiPart.0 to i64
+  %t2 = getelementptr float, float* %pTmp1, i64 %t1
+  %t3 = load float, float* %t2, align 4
+  %t4 = fadd float %t3, %distERBhi.0
+  %t5 = add i32 %hiPart.0, 1
+  %t6 = sext i32 %t5 to i64
+  %t7 = getelementptr float, float* %peakWeight, i64 %t6
+  %t8 = load float, float* %t7, align 4
+  %t9 = fadd float %t8, %peakCount.0
+  br label %bb1
+
+bb1:		; preds = %bb, %entry
+  %peakCount.0 = phi float [ %t0, %entry ], [ %t9, %bb ]
+  %hiPart.0 = phi i32 [ 0, %entry ], [ %t5, %bb ]
+  %distERBhi.0 = phi float [ 0.000000e+00, %entry ], [ %t4, %bb ]
+  %t10 = fcmp uge float %distERBhi.0, 2.500000e+00
+  br i1 %t10, label %bb3, label %bb2
+
+bb2:		; preds = %bb1
+  %t11 = add i32 %bandEdgeIndex, -1
+  %t12 = icmp sgt i32 %t11, %hiPart.0
+  br i1 %t12, label %bb, label %bb3
+
+bb3:		; preds = %bb2, %bb1
+  %t13 = fdiv float %peakCount.0, %distERBhi.0
+  ret float %t13
+}
+
+; Same test as above.
+; This would crash because we assumed TTI was available to process the metadata.
+
+define float @merge_branches_profile_metadata(float* %pTmp1, float* %peakWeight, i32 %bandEdgeIndex) nounwind {
+; CHECK-LABEL: @merge_branches_profile_metadata(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = load float, float* [[PEAKWEIGHT:%.*]], align 4
+; CHECK-NEXT:    [[T11:%.*]] = add i32 [[BANDEDGEINDEX:%.*]], -1
+; CHECK-NEXT:    [[T121:%.*]] = icmp sgt i32 [[T11]], 0
+; CHECK-NEXT:    br i1 [[T121]], label [[BB_LR_PH:%.*]], label [[BB3:%.*]], !prof !0
+; CHECK:       bb.lr.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[T11]] to i64
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[BB_LR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[DISTERBHI_04:%.*]] = phi float [ 0.000000e+00, [[BB_LR_PH]] ], [ [[T4:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[PEAKCOUNT_02:%.*]] = phi float [ [[T0]], [[BB_LR_PH]] ], [ [[T9:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[T2:%.*]] = getelementptr float, float* [[PTMP1:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[T3:%.*]] = load float, float* [[T2]], align 4
+; CHECK-NEXT:    [[T4]] = fadd float [[T3]], [[DISTERBHI_04]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[T7:%.*]] = getelementptr float, float* [[PEAKWEIGHT]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[T8:%.*]] = load float, float* [[T7]], align 4
+; CHECK-NEXT:    [[T9]] = fadd float [[T8]], [[PEAKCOUNT_02]]
+; CHECK-NEXT:    [[T10:%.*]] = fcmp olt float [[T4]], 2.500000e+00
+; CHECK-NEXT:    [[T12:%.*]] = icmp sgt i64 [[TMP0]], [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[T10]], [[T12]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[BB]], label [[BB1_BB3_CRIT_EDGE:%.*]], !prof !0
+; CHECK:       bb1.bb3_crit_edge:
+; CHECK-NEXT:    [[T4_LCSSA:%.*]] = phi float [ [[T4]], [[BB]] ]
+; CHECK-NEXT:    [[T9_LCSSA:%.*]] = phi float [ [[T9]], [[BB]] ]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[PEAKCOUNT_0_LCSSA:%.*]] = phi float [ [[T9_LCSSA]], [[BB1_BB3_CRIT_EDGE]] ], [ [[T0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[DISTERBHI_0_LCSSA:%.*]] = phi float [ [[T4_LCSSA]], [[BB1_BB3_CRIT_EDGE]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-NEXT:    [[T13:%.*]] = fdiv float [[PEAKCOUNT_0_LCSSA]], [[DISTERBHI_0_LCSSA]]
+; CHECK-NEXT:    ret float [[T13]]
+;
 entry:
-	%t0 = load float, float* %peakWeight, align 4
-	br label %bb1
+  %t0 = load float, float* %peakWeight, align 4
+  br label %bb1
 
 bb:		; preds = %bb2
-	%t1 = sext i32 %hiPart.0 to i64
-	%t2 = getelementptr float, float* %pTmp1, i64 %t1
-	%t3 = load float, float* %t2, align 4
-	%t4 = fadd float %t3, %distERBhi.0
-	%t5 = add i32 %hiPart.0, 1
-	%t6 = sext i32 %t5 to i64
-	%t7 = getelementptr float, float* %peakWeight, i64 %t6
-	%t8 = load float, float* %t7, align 4
-	%t9 = fadd float %t8, %peakCount.0
-	br label %bb1
+  %t1 = sext i32 %hiPart.0 to i64
+  %t2 = getelementptr float, float* %pTmp1, i64 %t1
+  %t3 = load float, float* %t2, align 4
+  %t4 = fadd float %t3, %distERBhi.0
+  %t5 = add i32 %hiPart.0, 1
+  %t6 = sext i32 %t5 to i64
+  %t7 = getelementptr float, float* %peakWeight, i64 %t6
+  %t8 = load float, float* %t7, align 4
+  %t9 = fadd float %t8, %peakCount.0
+  br label %bb1
 
 bb1:		; preds = %bb, %entry
-	%peakCount.0 = phi float [ %t0, %entry ], [ %t9, %bb ]
-	%hiPart.0 = phi i32 [ 0, %entry ], [ %t5, %bb ]
-	%distERBhi.0 = phi float [ 0.000000e+00, %entry ], [ %t4, %bb ]
-	%t10 = fcmp uge float %distERBhi.0, 2.500000e+00
-	br i1 %t10, label %bb3, label %bb2
+  %peakCount.0 = phi float [ %t0, %entry ], [ %t9, %bb ]
+  %hiPart.0 = phi i32 [ 0, %entry ], [ %t5, %bb ]
+  %distERBhi.0 = phi float [ 0.000000e+00, %entry ], [ %t4, %bb ]
+  %t10 = fcmp uge float %distERBhi.0, 2.500000e+00
+  br i1 %t10, label %bb3, label %bb2, !prof !0
 
 bb2:		; preds = %bb1
-	%t11 = add i32 %bandEdgeIndex, -1
-	%t12 = icmp sgt i32 %t11, %hiPart.0
-	br i1 %t12, label %bb, label %bb3
+  %t11 = add i32 %bandEdgeIndex, -1
+  %t12 = icmp sgt i32 %t11, %hiPart.0
+  br i1 %t12, label %bb, label %bb3
 
 bb3:		; preds = %bb2, %bb1
-	%t13 = fdiv float %peakCount.0, %distERBhi.0
-	ret float %t13
+  %t13 = fdiv float %peakCount.0, %distERBhi.0
+  ret float %t13
 }
+
+!0 = !{!"branch_weights", i32 2000, i32 1}
diff --git a/llvm/test/Transforms/LoopSimplify/pr26682.ll b/llvm/test/Transforms/LoopSimplify/pr26682.ll
index 092c0c3f0b049387d562d84de88884dab7d39649..4016b4e89b521a9cace6a9cafc129e885ec08a18 100644
--- a/llvm/test/Transforms/LoopSimplify/pr26682.ll
+++ b/llvm/test/Transforms/LoopSimplify/pr26682.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-unknown-unknown"
 ; Check that loop-simplify merges two loop exits, but preserves LCSSA form.
 ; CHECK-LABEL: @foo
 ; CHECK: for:
-; CHECK: %or.cond = and i1 %cmp1, %cmp2
+; CHECK: %or.cond = select i1 %cmp1, i1 %cmp2, i1 false
 ; CHECK-NOT: for.cond:
 ; CHECK: for.end:
 ; CHECK: %a.lcssa = phi i32 [ %a, %for ]
diff --git a/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll b/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll
index b47bdc92cdca1290dd419f20fbad74e047cbe4a1..a95f1bd63fea5678146cdeb01411acf4ffc26a61 100644
--- a/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll
+++ b/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll
@@ -1,14 +1,46 @@
-; RUN: opt -loop-unroll -S -mtriple arm-none-eabi -mcpu=cortex-m7 %s | FileCheck %s -check-prefix=UNROLL
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-unroll -simplifycfg -instcombine -simplifycfg -S -mtriple arm-none-eabi -mcpu=cortex-m7 %s | FileCheck %s
 
 ; This test is meant to check that this loop is unrolled into three iterations.
-
-; UNROLL-LABEL: @test
-; UNROLL: load i32, i32*
-; UNROLL: load i32, i32*
-; UNROLL: load i32, i32*
-; UNROLL-NOT: load i32, i32*
-
 define void @test(i32* %x, i32 %n) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[SUB]], 4
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[REM]], 0
+; CHECK-NEXT:    br i1 [[CMP7]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP0]], 10
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 0, i32* [[X]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[REM]], 1
+; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_BODY_1:%.*]], label [[WHILE_END]]
+; CHECK:       while.end:
+; CHECK-NEXT:    ret void
+; CHECK:       while.body.1:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP1]], 10
+; CHECK-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[IF_END_1:%.*]]
+; CHECK:       if.then.1:
+; CHECK-NEXT:    store i32 0, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    br label [[IF_END_1]]
+; CHECK:       if.end.1:
+; CHECK-NEXT:    [[INCDEC_PTR_1:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp sgt i32 [[REM]], 2
+; CHECK-NEXT:    br i1 [[CMP_1]], label [[WHILE_BODY_2:%.*]], label [[WHILE_END]]
+; CHECK:       while.body.2:
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR_1]], align 4
+; CHECK-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP2]], 10
+; CHECK-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[WHILE_END]]
+; CHECK:       if.then.2:
+; CHECK-NEXT:    store i32 0, i32* [[INCDEC_PTR_1]], align 4
+; CHECK-NEXT:    br label [[WHILE_END]]
+;
 entry:
   %sub = add nsw i32 %n, -1
   %rem = srem i32 %sub, 4
@@ -36,3 +68,96 @@ while.end:                                        ; preds = %if.end, %entry
   ret void
 }
 
+; Larger test that is still fully unrolled, thanks in part to the constant data and the max upper bound.
+@data = internal constant [50 x i32] [i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50], align 4
+define i32 @test2(i32 %l86) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L86_OFF:%.*]] = add i32 [[L86:%.*]], -1
+; CHECK-NEXT:    [[SWITCH:%.*]] = icmp ult i32 [[L86_OFF]], 24
+; CHECK-NEXT:    br i1 [[SWITCH]], label [[FOR_END_I_IF_END8_I_CRIT_EDGE_I:%.*]], label [[FOR_INC_I_3_I_5:%.*]]
+; CHECK:       for.end.i.if.end8.i_crit_edge.i:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       for.inc.i.3.i.5:
+; CHECK-NEXT:    [[DOTNOT30:%.*]] = icmp ne i32 [[L86]], 25
+; CHECK-NEXT:    [[SPEC_SELECT24:%.*]] = zext i1 [[DOTNOT30]] to i32
+; CHECK-NEXT:    ret i32 [[SPEC_SELECT24]]
+;
+entry:
+  br label %for.body.i.i
+
+for.body.i.i:                                     ; preds = %for.body.i.i.preheader, %for.inc.i.3.i
+  %i.0137.i.i = phi i32 [ %add.i.3.i, %for.inc.i.3.i ], [ 0, %entry ]
+  %add.i.i = or i32 %i.0137.i.i, 1
+  %arrayidx.i.i = getelementptr inbounds i32, i32* getelementptr inbounds ([50 x i32], [50 x i32]* @data, i32 0, i32 0), i32 %add.i.i
+  %l93 = load i32, i32* %arrayidx.i.i, align 4
+  %cmp1.i.i = icmp sgt i32 %l93, %l86
+  br i1 %cmp1.i.i, label %land.lhs.true.i.i, label %for.inc.i.i
+
+land.lhs.true.i.i:                                ; preds = %for.body.i.i
+  %arrayidx2.i.i = getelementptr inbounds i32, i32* getelementptr inbounds ([50 x i32], [50 x i32]* @data, i32 0, i32 0), i32 %i.0137.i.i
+  %l94 = load i32, i32* %arrayidx2.i.i, align 4
+  %cmp3.not.i.i = icmp sgt i32 %l94, %l86
+  br i1 %cmp3.not.i.i, label %for.inc.i.i, label %for.end.i.if.end8.i_crit_edge.i.loopexit
+
+for.inc.i.i:                                      ; preds = %land.lhs.true.i.i, %for.body.i.i
+  %exitcond.not.i.i = icmp eq i32 %add.i.i, 25
+  br i1 %exitcond.not.i.i, label %if.then6.i.i.loopexitsplit, label %for.body.i.1.i
+
+for.body.i.1.i:                                   ; preds = %for.inc.i.i
+  %add.i.1.i = or i32 %i.0137.i.i, 2
+  %arrayidx.i.1.i = getelementptr inbounds i32, i32* getelementptr inbounds ([50 x i32], [50 x i32]* @data, i32 0, i32 0), i32 %add.i.1.i
+  %l345 = load i32, i32* %arrayidx.i.1.i, align 4
+  %cmp1.i.1.i = icmp sgt i32 %l345, %l86
+  br i1 %cmp1.i.1.i, label %land.lhs.true.i.1.i, label %for.inc.i.1.i
+
+land.lhs.true.i.1.i:                              ; preds = %for.body.i.1.i
+  br i1 %cmp1.i.i, label %for.inc.i.1.i, label %for.end.i.i
+
+for.inc.i.1.i:                                    ; preds = %land.lhs.true.i.1.i, %for.body.i.1.i
+  %add.i.2.i = or i32 %i.0137.i.i, 3
+  %arrayidx.i.2.i = getelementptr inbounds i32, i32* getelementptr inbounds ([50 x i32], [50 x i32]* @data, i32 0, i32 0), i32 %add.i.2.i
+  %l346 = load i32, i32* %arrayidx.i.2.i, align 4
+  %cmp1.i.2.i = icmp sgt i32 %l346, %l86
+  br i1 %cmp1.i.2.i, label %land.lhs.true.i.2.i, label %for.inc.i.2.i
+
+land.lhs.true.i.2.i:                              ; preds = %for.inc.i.1.i
+  br i1 %cmp1.i.1.i, label %for.inc.i.2.i, label %for.end.i.if.end8.i_crit_edge.i.loopexit
+
+for.inc.i.2.i:                                    ; preds = %land.lhs.true.i.2.i, %for.inc.i.1.i
+  br label %for.body.i.3.i
+
+for.body.i.3.i:                                   ; preds = %for.inc.i.2.i
+  %add.i.3.i = add nuw nsw i32 %i.0137.i.i, 4
+  %arrayidx.i.3.i = getelementptr inbounds i32, i32* getelementptr inbounds ([50 x i32], [50 x i32]* @data, i32 0, i32 0), i32 %add.i.3.i
+  %l347 = load i32, i32* %arrayidx.i.3.i, align 4
+  %cmp1.i.3.i = icmp sgt i32 %l347, %l86
+  br i1 %cmp1.i.3.i, label %land.lhs.true.i.3.i, label %for.inc.i.3.i
+
+land.lhs.true.i.3.i:                              ; preds = %for.body.i.3.i
+  br i1 %cmp1.i.2.i, label %for.inc.i.3.i, label %for.end.i.i
+
+for.inc.i.3.i:                                    ; preds = %land.lhs.true.i.3.i, %for.body.i.3.i
+  br label %for.body.i.i
+
+for.end.i.if.end8.i_crit_edge.i.loopexit:         ; preds = %land.lhs.true.i.i, %land.lhs.true.i.2.i
+  %i.0.lcssa.i.i3.ph = phi i32 [ %i.0137.i.i, %land.lhs.true.i.i ], [ %add.i.1.i, %land.lhs.true.i.2.i ]
+  br label %for.end.i.if.end8.i_crit_edge.i
+
+if.then6.i.i.loopexitsplit:                       ; preds = %for.inc.i.i
+  br label %if.then6.i.i.loopexit
+
+for.end.i.i:                                      ; preds = %land.lhs.true.i.3.i, %land.lhs.true.i.1.i
+  %i.0.lcssa.i.i = phi i32 [ %add.i.i, %land.lhs.true.i.1.i ], [ %add.i.2.i, %land.lhs.true.i.3.i ]
+  %cmp5.i.i = icmp eq i32 %i.0.lcssa.i.i, 25
+  br i1 %cmp5.i.i, label %if.then6.i.i, label %for.end.i.if.end8.i_crit_edge.i
+
+for.end.i.if.end8.i_crit_edge.i:
+  ret i32 0
+
+if.then6.i.i.loopexit:
+  ret i32 1
+
+if.then6.i.i:
+  ret i32 2
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/AMDGPU/divergent-unswitch.ll b/llvm/test/Transforms/LoopUnswitch/AMDGPU/divergent-unswitch.ll
index 873a7653973d7e17b5722a88e626d6456699ad4b..40146648a39ae33613c55b3e5aa604ae494752e6 100644
--- a/llvm/test/Transforms/LoopUnswitch/AMDGPU/divergent-unswitch.ll
+++ b/llvm/test/Transforms/LoopUnswitch/AMDGPU/divergent-unswitch.ll
@@ -1,47 +1,4 @@
-; RUN: opt -mtriple=amdgcn-- -O3 -S -enable-new-pm=0 %s | FileCheck %s
-
-; This fails with the new pass manager:
-; https://bugs.llvm.org/show_bug.cgi?id=48819
-
-; Check that loop unswitch happened and condition hoisted out of the loop.
-; Condition is uniform so all targets should perform unswitching.
-
-; CHECK-LABEL: {{^}}define amdgpu_kernel void @uniform_unswitch
-; CHECK: entry:
-; CHECK-NEXT: [[LOOP_COND:%[a-z0-9]+]] = icmp
-; CHECK-NEXT: [[IF_COND:%[a-z0-9]+]] = icmp eq i32 %x, 123456
-; CHECK-NEXT: and i1 [[LOOP_COND]], [[IF_COND]]
-; CHECK-NEXT: br i1
-
-define amdgpu_kernel void @uniform_unswitch(i32 * nocapture %out, i32 %n, i32 %x) {
-entry:
-  %cmp6 = icmp sgt i32 %n, 0
-  br i1 %cmp6, label %for.body.lr.ph, label %for.cond.cleanup
-
-for.body.lr.ph:                                   ; preds = %entry
-  %cmp1 = icmp eq i32 %x, 123456
-  br label %for.body
-
-for.cond.cleanup.loopexit:                        ; preds = %for.inc
-  br label %for.cond.cleanup
-
-for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
-  ret void
-
-for.body:                                         ; preds = %for.inc, %for.body.lr.ph
-  %i.07 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
-  br i1 %cmp1, label %if.then, label %for.inc
-
-if.then:                                          ; preds = %for.body
-  %arrayidx = getelementptr inbounds i32, i32 * %out, i32 %i.07
-  store i32 %i.07, i32 * %arrayidx, align 4
-  br label %for.inc
-
-for.inc:                                          ; preds = %for.body, %if.then
-  %inc = add nuw nsw i32 %i.07, 1
-  %exitcond = icmp eq i32 %inc, %n
-  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
-}
+; RUN: opt -mtriple=amdgcn-- -O3 -S %s | FileCheck %s
 
 ; Check that loop unswitch does not happen if condition is divergent.
 
diff --git a/llvm/test/Transforms/LoopUnswitch/AMDGPU/uniform-unswitch.ll b/llvm/test/Transforms/LoopUnswitch/AMDGPU/uniform-unswitch.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b9feba0cfda3485835f574b86727fd34c09f5d27
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/AMDGPU/uniform-unswitch.ll
@@ -0,0 +1,53 @@
+; RUN: opt -mtriple=amdgcn-- -O3 -S -enable-new-pm=1 %s | FileCheck %s
+; XFAIL: *
+
+; Check that loop unswitch happened and condition hoisted out of the loop.
+; Condition is uniform so even targets with divergence should perform unswitching.
+
+; This fails with the new pass manager:
+; https://bugs.llvm.org/show_bug.cgi?id=48819
+; The correct behaviour (allow uniform non-trivial branches to be
+; unswitched on all targets) requires access to the function-level
+; divergence analysis from a loop transform, which is currently not
+; supported in the new pass manager.
+
+; CHECK-LABEL: {{^}}define amdgpu_kernel void @uniform_unswitch
+; CHECK: entry:
+; CHECK-NEXT: [[LOOP_COND:%[a-z0-9]+]] = icmp
+; CHECK-NEXT: [[IF_COND:%[a-z0-9]+]] = icmp eq i32 %x, 123456
+; CHECK-NEXT: and i1 [[LOOP_COND]], [[IF_COND]]
+; CHECK-NEXT: br i1
+
+define amdgpu_kernel void @uniform_unswitch(i32 * nocapture %out, i32 %n, i32 %x) {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %cmp1 = icmp eq i32 %x, 123456
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %i.07 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %arrayidx = getelementptr inbounds i32, i32 * %out, i32 %i.07
+  store i32 %i.07, i32 * %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
index 37e70adf64f8e71c6dc5a4f56237ae30e795ca3f..2536c6c85deabf78a91508ccc7eab2b43499370b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
@@ -76,6 +76,47 @@ exit:                        ; preds = %for.inc
   ret void
 }
 
+define void @invariant_load_cond(i32* noalias nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %cond, i64 %n) {
+; CHECK-LABEL: @invariant_load_cond
+; CHECK: vector.body
+; CHECK: %[[GEP:.*]] = getelementptr inbounds i32, i32* %b, i64 42
+; CHECK-NEXT: %[[SPLATINS:.*]] = insertelement <vscale x 4 x i32*> poison, i32* %[[GEP]], i32 0
+; CHECK-NEXT: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32*> %[[SPLATINS]], <vscale x 4 x i32*> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK: %[[LOAD:.*]] = load <vscale x 4 x i32>, <vscale x 4 x i32>*
+; CHECK-NEXT: %[[ICMP:.*]] = icmp ne <vscale x 4 x i32> %[[LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 0, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK: %[[MASKED_LOAD:.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* %[[BITCAST:.*]], i32 4, <vscale x 4 x i1> %[[ICMP]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: %[[MASKED_GATHER:.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %[[SPLAT]], i32 4, <vscale x 4 x i1> %[[ICMP]], <vscale x 4 x i32> undef)
+; CHECK-NEXT: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[MASKED_GATHER]], %[[MASKED_LOAD]]
+; CHECK: call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[ADD]], <vscale x 4 x i32>* %[[BITCAST1:.*]], i32 4, <vscale x 4 x i1> %[[ICMP]])
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i64 42
+  %arrayidx2 = getelementptr inbounds i32, i32* %cond, i64 %iv
+  %0 = load i32, i32* %arrayidx2, align 4
+  %tobool.not = icmp eq i32 %0, 0
+  br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %iv
+  %1 = load i32, i32* %arrayidx3, align 4
+  %2 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %2, %1
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i64 %iv
+  store i32 %add, i32* %arrayidx4, align 4
+  br label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
 !0 = distinct !{!0, !1, !2, !3, !4, !5}
 !1 = !{!"llvm.loop.mustprogress"}
 !2 = !{!"llvm.loop.vectorize.width", i32 4}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-loads.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-loads.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2514fddc3e38cce84a8d0b5d419f24b2b74f3498
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-loads.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -loop-vectorize -mattr=+sve -mtriple aarch64-linux-gnu < %s 2>%t | FileCheck %s
+
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+define void @invariant_load(i64 %n, i32* noalias nocapture %a, i32* nocapture readonly %b) {
+; CHECK-LABEL: @invariant_load
+; CHECK: vector.body:
+; CHECK: %[[GEP:.*]] = getelementptr inbounds i32, i32* %b, i64 42
+; CHECK-NEXT: %[[INVLOAD:.*]] = load i32, i32* %[[GEP]]
+; CHECK-NEXT: %[[SPLATINS:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[INVLOAD]], i32 0
+; CHECK-NEXT: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[SPLATINS]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK: %[[LOAD:.*]] = load <vscale x 4 x i32>, <vscale x 4 x i32>*
+; CHECK-NEXT: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[SPLAT]], %[[LOAD]]
+; CHECK: store <vscale x 4 x i32> %[[ADD]], <vscale x 4 x i32>*
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 42
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i64 %iv
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %0, %1
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!1 = distinct !{!1, !2, !3, !4}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.interleave.count", i32 1}
+!4 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d803f6b75ed925b1d7bb651a7a41ee0f9b258854
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll
@@ -0,0 +1,68 @@
+; This is the loop in c++ being vectorize in this file with 
+; experimental.vector.reverse
+
+;#pragma clang loop vectorize_width(4, scalable)
+;  for (long int i = N - 1; i >= 0; i--)
+;  {
+;    if (cond[i])
+;      a[i] += 1;
+;  }
+
+; The test checks if the mask is being correctly created, reverted and used
+
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S < %s 2>%t | FileCheck %s
+
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @vector_reverse_mask_nxv4i1(double* %a, double* %cond, i64 %N) #0 {
+; CHECK-LABEL: vector.body:
+; CHECK: %[[REVERSE6:.*]] = call <vscale x 4 x i1> @llvm.experimental.vector.reverse.nxv4i1(<vscale x 4 x i1> %{{.*}})
+; CHECK: %[[WIDEMSKLOAD:.*]] = call <vscale x 4 x double> @llvm.masked.load.nxv4f64.p0nxv4f64(<vscale x 4 x double>* nonnull %{{.*}}, i32 8, <vscale x 4 x i1> %[[REVERSE6]], <vscale x 4 x double> poison)
+; CHECK-NEXT: %[[REVERSE7:.*]] = call <vscale x 4 x double> @llvm.experimental.vector.reverse.nxv4f64(<vscale x 4 x double> %[[WIDEMSKLOAD]])
+; CHECK-NEXT: %[[FADD:.*]] = fadd <vscale x 4 x double> %[[REVERSE7]]
+; CHECK-NEXT: %[[REVERSE8:.*]] = call <vscale x 4 x double> @llvm.experimental.vector.reverse.nxv4f64(<vscale x 4 x double> %[[FADD]])
+; CHECK:  %[[REVERSE9:.*]] = call <vscale x 4 x i1> @llvm.experimental.vector.reverse.nxv4i1(<vscale x 4 x i1> %{{.*}})
+; CHECK: call void @llvm.masked.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> %[[REVERSE8]], <vscale x 4 x double>* %{{.*}}, i32 8, <vscale x 4 x i1> %[[REVERSE9]]
+
+entry:
+  %cmp7 = icmp sgt i64 %N, 0
+  br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08.in = phi i64 [ %i.08, %for.inc ], [ %N, %entry ]
+  %i.08 = add nsw i64 %i.08.in, -1
+  %arrayidx = getelementptr inbounds double, double* %cond, i64 %i.08
+  %0 = load double, double* %arrayidx, align 8
+  %tobool = fcmp une double %0, 0.000000e+00
+  br i1 %tobool, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %arrayidx1 = getelementptr inbounds double, double* %a, i64 %i.08
+  %1 = load double, double* %arrayidx1, align 8
+  %add = fadd double %1, 1.000000e+00
+  store double %add, double* %arrayidx1, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %cmp = icmp sgt i64 %i.08.in, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+}
+
+attributes #0 = {"target-cpu"="generic" "target-features"="+neon,+sve"}
+
+
+!0 = distinct !{!0, !1, !2, !3, !4}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!4 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
new file mode 100644
index 0000000000000000000000000000000000000000..aef5efe030f547ad27d6ec724b87532cf2ee88f3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
@@ -0,0 +1,108 @@
+; This is the loop in c++ being vectorize in this file with
+;experimental.vector.reverse
+;  #pragma clang loop vectorize_width(8, scalable)
+;  for (int i = N-1; i >= 0; --i)
+;    a[i] = b[i] + 1.0;
+
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S < %s 2>%t | FileCheck %s
+
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+define void @vector_reverse_f64(i64 %N, double* %a, double* %b) #0{
+; CHECK-LABEL: @vector_reverse_f64
+; CHECK-LABEL: vector.body:
+; CHECK: %[[ADD:.*]] = add i64 %{{.*}}, %N
+; CHECK-NEXT: %[[GEP:.*]] = getelementptr inbounds double, double* %b, i64 %[[ADD]]
+; CHECK-NEXT: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: %[[MUL:.*]] = mul i32 %[[VSCALE]], -8
+; CHECK-NEXT: %[[OR:.*]] = or i32 %[[MUL]], 1
+; CHECK-NEXT: %[[SEXT:.*]] = sext i32 %[[OR]] to i64
+; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds double, double* %[[GEP]], i64 %[[SEXT]]
+; CHECK-NEXT: %[[CAST:.*]] = bitcast double* %[[GEP1]] to <vscale x 8 x double>*
+; CHECK-NEXT: %[[WIDE:.*]] = load <vscale x 8 x double>, <vscale x 8 x double>* %[[CAST]], align 8
+; CHECK-NEXT: %[[REVERSE:.*]] = call <vscale x 8 x double> @llvm.experimental.vector.reverse.nxv8f64(<vscale x 8 x double> %[[WIDE]])
+; CHECK-NEXT: %[[FADD:.*]] = fadd <vscale x 8 x double> %[[REVERSE]], shufflevector
+; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds double, double* %a, i64 %[[ADD]]
+; CHECK-NEXT: %[[REVERSE6:.*]] = call <vscale x 8 x double> @llvm.experimental.vector.reverse.nxv8f64(<vscale x 8 x double> %[[FADD]])
+; CHECK-NEXT: %[[VSCALE1:.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: %[[MUL1:.*]] = mul i32 %[[VSCALE1]], -8
+; CHECK-NEXT: %[[OR1:.*]] = or i32 %[[MUL1]], 1
+; CHECK-NEXT: %[[SEXT1:.*]] = sext i32 %[[OR1]] to i64
+; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds double, double* %[[GEP2]], i64 %[[SEXT1]]
+; CHECK-NEXT: %[[CAST1:.*]] = bitcast double* %[[GEP3]] to <vscale x 8 x double>*
+; CHECK-NEXT: store <vscale x 8 x double> %[[REVERSE6]], <vscale x 8 x double>* %[[CAST1]], align 8
+
+entry:
+  %cmp7 = icmp sgt i64 %N, 0
+  br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.08.in = phi i64 [ %i.08, %for.body ], [ %N, %entry ]
+  %i.08 = add nsw i64 %i.08.in, -1
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %i.08
+  %0 = load double, double* %arrayidx, align 8
+  %add = fadd double %0, 1.000000e+00
+  %arrayidx1 = getelementptr inbounds double, double* %a, i64 %i.08
+  store double %add, double* %arrayidx1, align 8
+  %cmp = icmp sgt i64 %i.08.in, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+}
+
+
+define void @vector_reverse_i64(i64 %N, i64* %a, i64* %b) #0 {
+; CHECK-LABEL: vector_reverse_i64
+; CHECK-LABEL: vector.body:
+; CHECK: %[[ADD:.*]] = add i64 %{{.*}}, %N
+; CHECK-NEXT: %[[GEP:.*]] = getelementptr inbounds i64, i64* %b, i64 %[[ADD]]
+; CHECK-NEXT: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: %[[MUL:.*]] = mul i32 %[[VSCALE]], -8
+; CHECK-NEXT: %[[OR:.*]] = or i32 %[[MUL]], 1
+; CHECK-NEXT: %[[SEXT:.*]] = sext i32 %[[OR]] to i64
+; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds i64, i64* %[[GEP]], i64 %[[SEXT]]
+; CHECK-NEXT: %[[CAST:.*]] = bitcast i64* %[[GEP1]] to <vscale x 8 x i64>*
+; CHECK-NEXT: %[[WIDE:.*]] = load <vscale x 8 x i64>, <vscale x 8 x i64>* %[[CAST]], align 8
+; CHECK-NEXT: %[[REVERSE:.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.reverse.nxv8i64(<vscale x 8 x i64> %[[WIDE]])
+; CHECK-NEXT: %[[ADD1:.*]] = add <vscale x 8 x i64> %[[REVERSE]]
+; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds i64, i64* %a, i64 %[[ADD]]
+; CHECK-NEXT: %[[REVERSE6]] = call <vscale x 8 x i64> @llvm.experimental.vector.reverse.nxv8i64(<vscale x 8 x i64> %[[ADD1]])
+; CHECK-NEXT: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: %[[MUL1:.*]] = mul i32 %[[VSCALE]], -8
+; CHECK-NEXT: %[[OR1:.*]] = or i32 %[[MUL1]], 1
+; CHECK-NEXT: %[[SEXT1:.*]] = sext i32 %[[OR1]] to i64
+; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds i64, i64* %[[GEP2]], i64 %[[SEXT1]]
+; CHECK-NEXT: %[[CAST1:.*]] = bitcast i64* %[[GEP3]] to <vscale x 8 x i64>*
+; CHECK-NEXT:  store <vscale x 8 x i64> %[[REVERSE6]], <vscale x 8 x i64>* %[[CAST1]], align 8
+
+entry:
+  %cmp8 = icmp sgt i64 %N, 0
+  br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09.in = phi i64 [ %i.09, %for.body ], [ %N, %entry ]
+  %i.09 = add nsw i64 %i.09.in, -1
+  %arrayidx = getelementptr inbounds i64, i64* %b, i64 %i.09
+  %0 = load i64, i64* %arrayidx, align 8
+  %add = add i64 %0, 1
+  %arrayidx2 = getelementptr inbounds i64, i64* %a, i64 %i.09
+  store i64 %add, i64* %arrayidx2, align 8
+  %cmp = icmp sgt i64 %i.09.in, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+}
+
+attributes #0 = { "target-cpu"="generic" "target-features"="+neon,+sve" }
+
+!0 = distinct !{!0, !1, !2, !3, !4}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.width", i32 8}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!4 = !{!"llvm.loop.vectorize.enable", i1 true}
+
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll
new file mode 100644
index 0000000000000000000000000000000000000000..22de7c1e1ca827d25e535a0895832cf79f8ca70e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll
@@ -0,0 +1,65 @@
+; This is the loop in c++ being vectorize in this file with 
+; shuffle reverse
+
+;#pragma clang loop vectorize_width(4, fixed)
+;  for (long int i = N - 1; i >= 0; i--)
+;  {
+;    if (cond[i])
+;      a[i] += 1;
+;  }
+
+; The test checks if the mask is being correctly created, reverted  and used
+
+; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S < %s 2>%t | FileCheck %s
+
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @vector_reverse_mask_v4i1(double* %a, double* %cond, i64 %N) #0 {
+; CHECK-LABEL: vector.body:
+; CHECK: %[[REVERSE6:.*]] = shufflevector <4 x i1> %{{.*}}, <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK: %[[WIDEMSKLOAD:.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull %{{.*}}, i32 8, <4 x i1> %[[REVERSE6]], <4 x double> poison)
+; CHECK-NEXT: %[[FADD:.*]] = fadd <4 x double> %[[WIDEMSKLOAD]]
+; CHECK: call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> %[[FADD]], <4 x double>* %{{.*}}, i32 8, <4 x i1> %[[REVERSE6]])
+
+entry:
+  %cmp7 = icmp sgt i64 %N, 0
+  br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08.in = phi i64 [ %i.08, %for.inc ], [ %N, %entry ]
+  %i.08 = add nsw i64 %i.08.in, -1
+  %arrayidx = getelementptr inbounds double, double* %cond, i64 %i.08
+  %0 = load double, double* %arrayidx, align 8
+  %tobool = fcmp une double %0, 0.000000e+00
+  br i1 %tobool, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %arrayidx1 = getelementptr inbounds double, double* %a, i64 %i.08
+  %1 = load double, double* %arrayidx1, align 8
+  %add = fadd double %1, 1.000000e+00
+  store double %add, double* %arrayidx1, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %cmp = icmp sgt i64 %i.08.in, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+}
+
+attributes #0 = {"target-cpu"="generic" "target-features"="+neon,+sve"}
+
+
+!0 = distinct !{!0, !1, !2, !3, !4}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+!4 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ae3aad1e75a8b8a99eb6cb0df082bc4ae760adf9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
@@ -0,0 +1,91 @@
+; Test VLA for reverse with fixed size vector
+; This is the loop in c++ being vectorize in this file with
+; shuffle reverse
+;  #pragma clang loop vectorize_width(8, fixed)
+;  for (int i = N-1; i >= 0; --i)
+;    a[i] = b[i] + 1.0;
+
+; RUN: opt -loop-vectorize -dce  -mtriple aarch64-linux-gnu -S < %s 2>%t | FileCheck %s
+
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.$
+; WARN-NOT: warning
+
+define void @vector_reverse_f64(i64 %N, double* %a, double* %b) #0 {
+; CHECK-LABEL: vector_reverse_f64
+; CHECK-LABEL: vector.body
+; CHECK: %[[GEP:.*]] = getelementptr inbounds double, double* %{{.*}}, i32 0
+; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds double, double* %[[GEP]], i32 -7
+; CHECK-NEXT: %[[CAST:.*]] = bitcast double* %[[GEP1]] to <8 x double>*
+; CHECK-NEXT: %[[WIDE:.*]] = load <8 x double>, <8 x double>* %[[CAST]], align 8
+; CHECK-NEXT: %[[REVERSE:.*]] = shufflevector <8 x double> %[[WIDE]], <8 x double> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: %[[FADD:.*]] = fadd <8 x double> %[[REVERSE]]
+; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds double, double* {{.*}}, i64 {{.*}}
+; CHECK-NEXT: %[[REVERSE6:.*]] = shufflevector <8 x double> %[[FADD]], <8 x double> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds double, double* %[[GEP2]], i32 0
+; CHECK-NEXT: %[[GEP4:.*]] = getelementptr inbounds double, double* %[[GEP3]], i32 -7
+; CHECK-NEXT: %[[CAST:.*]] = bitcast double* %[[GEP4]] to <8 x double>*
+; CHECK-NEXT:  store <8 x double> %[[REVERSE6]], <8 x double>* %[[CAST]], align 8
+
+entry:
+  %cmp7 = icmp sgt i64 %N, 0
+  br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.08.in = phi i64 [ %i.08, %for.body ], [ %N, %entry ]
+  %i.08 = add nsw i64 %i.08.in, -1
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %i.08
+  %0 = load double, double* %arrayidx, align 8
+  %add = fadd double %0, 1.000000e+00
+  %arrayidx1 = getelementptr inbounds double, double* %a, i64 %i.08
+  store double %add, double* %arrayidx1, align 8
+  %cmp = icmp sgt i64 %i.08.in, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+}
+
+define void @vector_reverse_i64(i64 %N, i64* %a, i64* %b) #0 {
+; CHECK-LABEL: vector_reverse_i64
+; CHECK-LABEL: vector.body
+; CHECK: %[[GEP:.*]] = getelementptr inbounds i64, i64* %{{.*}}, i32 0
+; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds i64, i64* %[[GEP]], i32 -7
+; CHECK-NEXT: %[[CAST:.*]] = bitcast i64* %[[GEP1]] to <8 x i64>*
+; CHECK-NEXT: %[[WIDE:.*]] = load <8 x i64>, <8 x i64>* %[[CAST]], align 8
+; CHECK-NEXT: %[[REVERSE:.*]] = shufflevector <8 x i64> %[[WIDE]], <8 x i64> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: %[[FADD:.*]] = add <8 x i64> %[[REVERSE]]
+; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds i64, i64* {{.*}}, i64 {{.*}}
+; CHECK-NEXT: %[[REVERSE6:.*]] = shufflevector <8 x i64> %[[FADD]], <8 x i64> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds i64, i64* %[[GEP2]], i32 0
+; CHECK-NEXT: %[[GEP4:.*]] = getelementptr inbounds i64, i64* %[[GEP3]], i32 -7
+; CHECK-NEXT: %[[CAST1:.*]] = bitcast i64* %[[GEP4]] to <8 x i64>*
+; CHECK-NEXT:  store <8 x i64> %[[REVERSE6]], <8 x i64>* %[[CAST1]], align 8
+
+entry:
+  %cmp8 = icmp sgt i64 %N, 0
+  br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09.in = phi i64 [ %i.09, %for.body ], [ %N, %entry ]
+  %i.09 = add nsw i64 %i.09.in, -1
+  %arrayidx = getelementptr inbounds i64, i64* %b, i64 %i.09
+  %0 = load i64, i64* %arrayidx, align 8
+  %add = add i64 %0, 1
+  %arrayidx2 = getelementptr inbounds i64, i64* %a, i64 %i.09
+  store i64 %add, i64* %arrayidx2, align 8
+  %cmp = icmp sgt i64 %i.09.in, 1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+}
+
+attributes #0 = { "target-cpu"="generic" "target-features"="+neon,+sve" }
+
+!0 = distinct !{!0, !1, !2, !3, !4}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.width", i32 8}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+!4 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
index d2961326ba71202e726a41859ec22844bb6d903f..6d7555ae59be6e9cb214838aac4f5603984a3567 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
@@ -20,18 +20,18 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
 ; CHECK: LV: Scalar loop costs: 5.
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %i.016 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %arrayidx = getelementptr inbounds i16, i16* %s, i32 %i.016
-; CHECK: LV: Found an estimated cost of 10 for VF 2 For instruction:   %1 = load i16, i16* %arrayidx, align 2
+; CHECK: LV: Found an estimated cost of 18 for VF 2 For instruction:   %1 = load i16, i16* %arrayidx, align 2
 ; CHECK: LV: Found an estimated cost of 4 for VF 2 For instruction:   %conv = sext i16 %1 to i32
-; CHECK: LV: Found an estimated cost of 12 for VF 2 For instruction:   %cmp2 = icmp sgt i32 %conv, %conv1
+; CHECK: LV: Found an estimated cost of 20 for VF 2 For instruction:   %cmp2 = icmp sgt i32 %conv, %conv1
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   br i1 %cmp2, label %if.then, label %for.inc
-; CHECK: LV: Found an estimated cost of 14 for VF 2 For instruction:   %conv6 = add i16 %1, %0
+; CHECK: LV: Found an estimated cost of 26 for VF 2 For instruction:   %conv6 = add i16 %1, %0
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %arrayidx7 = getelementptr inbounds i16, i16* %d, i32 %i.016
 ; CHECK: LV: Found an estimated cost of 16 for VF 2 For instruction:   store i16 %conv6, i16* %arrayidx7, align 2
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   br label %for.inc
 ; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %inc = add nuw nsw i32 %i.016, 1
 ; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %exitcond.not = icmp eq i32 %inc, %n
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
-; CHECK: LV: Vector loop of width 2 costs: 29.
+; CHECK: LV: Vector loop of width 2 costs: 43.
 ; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %i.016 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
 ; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %arrayidx = getelementptr inbounds i16, i16* %s, i32 %i.016
 ; CHECK: LV: Found an estimated cost of 2 for VF 4 For instruction:   %1 = load i16, i16* %arrayidx, align 2
@@ -50,7 +50,7 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
 ; CHECK: LV: Found an estimated cost of 0 for VF 8 For instruction:   %arrayidx = getelementptr inbounds i16, i16* %s, i32 %i.016
 ; CHECK: LV: Found an estimated cost of 2 for VF 8 For instruction:   %1 = load i16, i16* %arrayidx, align 2
 ; CHECK: LV: Found an estimated cost of 2 for VF 8 For instruction:   %conv = sext i16 %1 to i32
-; CHECK: LV: Found an estimated cost of 68 for VF 8 For instruction:   %cmp2 = icmp sgt i32 %conv, %conv1
+; CHECK: LV: Found an estimated cost of 36 for VF 8 For instruction:   %cmp2 = icmp sgt i32 %conv, %conv1
 ; CHECK: LV: Found an estimated cost of 0 for VF 8 For instruction:   br i1 %cmp2, label %if.then, label %for.inc
 ; CHECK: LV: Found an estimated cost of 2 for VF 8 For instruction:   %conv6 = add i16 %1, %0
 ; CHECK: LV: Found an estimated cost of 0 for VF 8 For instruction:   %arrayidx7 = getelementptr inbounds i16, i16* %d, i32 %i.016
@@ -59,7 +59,7 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
 ; CHECK: LV: Found an estimated cost of 1 for VF 8 For instruction:   %inc = add nuw nsw i32 %i.016, 1
 ; CHECK: LV: Found an estimated cost of 1 for VF 8 For instruction:   %exitcond.not = icmp eq i32 %inc, %n
 ; CHECK: LV: Found an estimated cost of 0 for VF 8 For instruction:   br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
-; CHECK: LV: Vector loop of width 8 costs: 9.
+; CHECK: LV: Vector loop of width 8 costs: 5.
 ; CHECK: LV: Selecting VF: 4.
 define void @expensive_icmp(i16* noalias nocapture %d, i16* nocapture readonly %s, i32 %n, i16 zeroext %m) #0 {
 entry:
@@ -120,22 +120,22 @@ for.inc:                                          ; preds = %for.body, %if.then
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %pDst.addr.010 = phi i8* [ %incdec.ptr5, %while.body ], [ %pDst, %while.body.preheader ]
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %pSrcB.addr.09 = phi i8* [ %incdec.ptr2, %while.body ], [ %pSrcB, %while.body.preheader ]
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %incdec.ptr = getelementptr inbounds i8, i8* %pSrcA.addr.011, i32 1
-; CHECK: LV: Found an estimated cost of 10 for VF 2 For instruction:   %0 = load i8, i8* %pSrcA.addr.011, align 1
+; CHECK: LV: Found an estimated cost of 18 for VF 2 For instruction:   %0 = load i8, i8* %pSrcA.addr.011, align 1
 ; CHECK: LV: Found an estimated cost of 4 for VF 2 For instruction:   %conv1 = sext i8 %0 to i32
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %incdec.ptr2 = getelementptr inbounds i8, i8* %pSrcB.addr.09, i32 1
-; CHECK: LV: Found an estimated cost of 10 for VF 2 For instruction:   %1 = load i8, i8* %pSrcB.addr.09, align 1
+; CHECK: LV: Found an estimated cost of 18 for VF 2 For instruction:   %1 = load i8, i8* %pSrcB.addr.09, align 1
 ; CHECK: LV: Found an estimated cost of 4 for VF 2 For instruction:   %conv3 = sext i8 %1 to i32
-; CHECK: LV: Found an estimated cost of 14 for VF 2 For instruction:   %mul = mul nsw i32 %conv3, %conv1
-; CHECK: LV: Found an estimated cost of 10 for VF 2 For instruction:   %shr = ashr i32 %mul, 7
+; CHECK: LV: Found an estimated cost of 26 for VF 2 For instruction:   %mul = mul nsw i32 %conv3, %conv1
+; CHECK: LV: Found an estimated cost of 18 for VF 2 For instruction:   %shr = ashr i32 %mul, 7
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %2 = icmp slt i32 %shr, 127
-; CHECK: LV: Found an estimated cost of 24 for VF 2 For instruction:   %spec.select.i = select i1 %2, i32 %shr, i32 127
+; CHECK: LV: Found an estimated cost of 40 for VF 2 For instruction:   %spec.select.i = select i1 %2, i32 %shr, i32 127
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %conv4 = trunc i32 %spec.select.i to i8
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %incdec.ptr5 = getelementptr inbounds i8, i8* %pDst.addr.010, i32 1
-; CHECK: LV: Found an estimated cost of 10 for VF 2 For instruction:   store i8 %conv4, i8* %pDst.addr.010, align 1
+; CHECK: LV: Found an estimated cost of 18 for VF 2 For instruction:   store i8 %conv4, i8* %pDst.addr.010, align 1
 ; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %dec = add i32 %blkCnt.012, -1
 ; CHECK: LV: Found an estimated cost of 1 for VF 2 For instruction:   %cmp.not = icmp eq i32 %dec, 0
 ; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   br i1 %cmp.not, label %while.end.loopexit, label %while.body
-; CHECK: LV: Vector loop of width 2 costs: 44.
+; CHECK: LV: Vector loop of width 2 costs: 74.
 ; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %blkCnt.012 = phi i32 [ %dec, %while.body ], [ %blockSize, %while.body.preheader ]
 ; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %pSrcA.addr.011 = phi i8* [ %incdec.ptr, %while.body ], [ %pSrcA, %while.body.preheader ]
 ; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %pDst.addr.010 = phi i8* [ %incdec.ptr5, %while.body ], [ %pDst, %while.body.preheader ]
@@ -238,8 +238,8 @@ while.end:                                        ; preds = %while.end.loopexit,
 }
 
 ; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction:   %cmp1 = fcmp
-; CHECK: LV: Found an estimated cost of 10 for VF 2 For instruction:   %cmp1 = fcmp
-; CHECK: LV: Found an estimated cost of 36 for VF 4 For instruction:   %cmp1 = fcmp
+; CHECK: LV: Found an estimated cost of 12 for VF 2 For instruction:   %cmp1 = fcmp
+; CHECK: LV: Found an estimated cost of 24 for VF 4 For instruction:   %cmp1 = fcmp
 define void @floatcmp(float* nocapture readonly %pSrc, i32* nocapture %pDst, i32 %blockSize) #0 {
 entry:
   %cmp.not7 = icmp eq i32 %blockSize, 0
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
index f511a81c29157060f772d28144a5895d4f83df6d..006fc47e5c32a3b9e61514dd5fdfd275f08955f0 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-interleaved-cost.ll
@@ -15,10 +15,10 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i8_factor_2"
-; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
+; VF_2:          Found an estimated cost of 24 for VF 2 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp0, align 1
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store i8 0, i8* %tmp1, align 1
 ; VF_4-LABEL:  Checking a loop in "i8_factor_2"
 ; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
@@ -56,10 +56,10 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i16_factor_2"
-; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_2:          Found an estimated cost of 24 for VF 2 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp0, align 2
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store i16 0, i16* %tmp1, align 2
 ; VF_4-LABEL:  Checking a loop in "i16_factor_2"
 ; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
@@ -97,10 +97,10 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i32_factor_2"
-; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_2:          Found an estimated cost of 24 for VF 2 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store i32 0, i32* %tmp1, align 4
 ; VF_4-LABEL:  Checking a loop in "i32_factor_2"
 ; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
@@ -138,25 +138,25 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i64_factor_2"
-; VF_2:          Found an estimated cost of 24 for VF 2 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_2:          Found an estimated cost of 44 for VF 2 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp0, align 8
-; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i64 0, i64* %tmp1, align 8
 ; VF_4-LABEL:  Checking a loop in "i64_factor_2"
-; VF_4:          Found an estimated cost of 80 for VF 4 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_4:          Found an estimated cost of 88 for VF 4 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp0, align 8
-; VF_4-NEXT:     Found an estimated cost of 48 for VF 4 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store i64 0, i64* %tmp1, align 8
 ; VF_8-LABEL:  Checking a loop in "i64_factor_2"
-; VF_8:          Found an estimated cost of 288 for VF 8 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_8:          Found an estimated cost of 176 for VF 8 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp0, align 8
-; VF_8-NEXT:     Found an estimated cost of 160 for VF 8 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store i64 0, i64* %tmp1, align 8
 ; VF_16-LABEL: Checking a loop in "i64_factor_2"
-; VF_16:         Found an estimated cost of 1088 for VF 16 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_16:         Found an estimated cost of 352 for VF 16 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp0, align 8
-; VF_16-NEXT:    Found an estimated cost of 576 for VF 16 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store i64 0, i64* %tmp1, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i64.2, %i64.2* %data, i64 %i, i32 0
@@ -179,15 +179,15 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "f16_factor_2"
-; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load half, half* %tmp0, align 2
+; VF_2:          Found an estimated cost of 12 for VF 2 For instruction: %tmp2 = load half, half* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load half, half* %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp0, align 2
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store half 0xH0000, half* %tmp1, align 2
 ; VF_4-LABEL:  Checking a loop in "f16_factor_2"
-; VF_4:          Found an estimated cost of 72 for VF 4 For instruction: %tmp2 = load half, half* %tmp0, align 2
+; VF_4:          Found an estimated cost of 18 for VF 4 For instruction: %tmp2 = load half, half* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load half, half* %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp0, align 2
-; VF_4-NEXT:     Found an estimated cost of 40 for VF 4 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 16 for VF 4 For instruction: store half 0xH0000, half* %tmp1, align 2
 ; VF_8-LABEL:  Checking a loop in "f16_factor_2"
 ; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load half, half* %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load half, half* %tmp1, align 2
@@ -220,10 +220,10 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "f32_factor_2"
-; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load float, float* %tmp0, align 4
+; VF_2:          Found an estimated cost of 10 for VF 2 For instruction: %tmp2 = load float, float* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load float, float* %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp0, align 4
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store float 0.000000e+00, float* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store float 0.000000e+00, float* %tmp1, align 4
 ; VF_4-LABEL:  Checking a loop in "f32_factor_2"
 ; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load float, float* %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load float, float* %tmp1, align 4
@@ -261,25 +261,25 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "f64_factor_2"
-; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp2 = load double, double* %tmp0, align 8
+; VF_2:          Found an estimated cost of 12 for VF 2 For instruction: %tmp2 = load double, double* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load double, double* %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp0, align 8
-; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 8 for VF 2 For instruction: store double 0.000000e+00, double* %tmp1, align 8
 ; VF_4-LABEL:  Checking a loop in "f64_factor_2"
-; VF_4:          Found an estimated cost of 72 for VF 4 For instruction: %tmp2 = load double, double* %tmp0, align 8
+; VF_4:          Found an estimated cost of 24 for VF 4 For instruction: %tmp2 = load double, double* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load double, double* %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp0, align 8
-; VF_4-NEXT:     Found an estimated cost of 40 for VF 4 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 16 for VF 4 For instruction: store double 0.000000e+00, double* %tmp1, align 8
 ; VF_8-LABEL:  Checking a loop in "f64_factor_2"
-; VF_8:          Found an estimated cost of 272 for VF 8 For instruction: %tmp2 = load double, double* %tmp0, align 8
+; VF_8:          Found an estimated cost of 48 for VF 8 For instruction: %tmp2 = load double, double* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load double, double* %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp0, align 8
-; VF_8-NEXT:     Found an estimated cost of 144 for VF 8 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 32 for VF 8 For instruction: store double 0.000000e+00, double* %tmp1, align 8
 ; VF_16-LABEL: Checking a loop in "f64_factor_2"
-; VF_16:         Found an estimated cost of 1056 for VF 16 For instruction: %tmp2 = load double, double* %tmp0, align 8
+; VF_16:         Found an estimated cost of 96 for VF 16 For instruction: %tmp2 = load double, double* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load double, double* %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp0, align 8
-; VF_16-NEXT:    Found an estimated cost of 544 for VF 16 For instruction: store double 0.000000e+00, double* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 64 for VF 16 For instruction: store double 0.000000e+00, double* %tmp1, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f64.2, %f64.2* %data, i64 %i, i32 0
@@ -306,33 +306,33 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i8_factor_3"
-; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
+; VF_2:          Found an estimated cost of 36 for VF 2 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i8, i8* %tmp1, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i8, i8* %tmp2, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp0, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp1, align 1
-; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i8 0, i8* %tmp2, align 1
 ; VF_4-LABEL:  Checking a loop in "i8_factor_3"
-; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
+; VF_4:          Found an estimated cost of 72 for VF 4 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i8, i8* %tmp1, align 1
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i8, i8* %tmp2, align 1
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp0, align 1
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp1, align 1
-; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store i8 0, i8* %tmp2, align 1
 ; VF_8-LABEL:  Checking a loop in "i8_factor_3"
-; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
+; VF_8:          Found an estimated cost of 144 for VF 8 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i8, i8* %tmp1, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i8, i8* %tmp2, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp0, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp1, align 1
-; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store i8 0, i8* %tmp2, align 1
 ; VF_16-LABEL: Checking a loop in "i8_factor_3"
-; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
+; VF_16:         Found an estimated cost of 288 for VF 16 For instruction: %tmp3 = load i8, i8* %tmp0, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i8, i8* %tmp1, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i8, i8* %tmp2, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp0, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp1, align 1
-; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store i8 0, i8* %tmp2, align 1
+; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store i8 0, i8* %tmp2, align 1
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i8.3, %i8.3* %data, i64 %i, i32 0
@@ -358,33 +358,33 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i16_factor_3"
-; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
+; VF_2:          Found an estimated cost of 36 for VF 2 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i16, i16* %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i16, i16* %tmp2, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp1, align 2
-; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i16 0, i16* %tmp2, align 2
 ; VF_4-LABEL:  Checking a loop in "i16_factor_3"
-; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
+; VF_4:          Found an estimated cost of 72 for VF 4 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i16, i16* %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i16, i16* %tmp2, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store i16 0, i16* %tmp2, align 2
 ; VF_8-LABEL:  Checking a loop in "i16_factor_3"
-; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
+; VF_8:          Found an estimated cost of 144 for VF 8 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i16, i16* %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i16, i16* %tmp2, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store i16 0, i16* %tmp2, align 2
 ; VF_16-LABEL: Checking a loop in "i16_factor_3"
-; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
+; VF_16:         Found an estimated cost of 288 for VF 16 For instruction: %tmp3 = load i16, i16* %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i16, i16* %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i16, i16* %tmp2, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store i16 0, i16* %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store i16 0, i16* %tmp2, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i16.3, %i16.3* %data, i64 %i, i32 0
@@ -410,12 +410,12 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i32_factor_3"
-; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
+; VF_2:          Found an estimated cost of 36 for VF 2 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i32, i32* %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i32, i32* %tmp2, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store i32 0, i32* %tmp2, align 4
 ; VF_4-LABEL:  Checking a loop in "i32_factor_3"
 ; VF_4:          Found an estimated cost of 24 for VF 4 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i32, i32* %tmp1, align 4
@@ -424,19 +424,19 @@ entry:
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp1, align 4
 ; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store i32 0, i32* %tmp2, align 4
 ; VF_8-LABEL:  Checking a loop in "i32_factor_3"
-; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
+; VF_8:          Found an estimated cost of 144 for VF 8 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i32, i32* %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i32, i32* %tmp2, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store i32 0, i32* %tmp2, align 4
 ; VF_16-LABEL: Checking a loop in "i32_factor_3"
-; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
+; VF_16:         Found an estimated cost of 288 for VF 16 For instruction: %tmp3 = load i32, i32* %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i32, i32* %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i32, i32* %tmp2, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store i32 0, i32* %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store i32 0, i32* %tmp2, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i32.3, %i32.3* %data, i64 %i, i32 0
@@ -462,33 +462,33 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i64_factor_3"
-; VF_2:          Found an estimated cost of 36 for VF 2 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
+; VF_2:          Found an estimated cost of 66 for VF 2 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load i64, i64* %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i64, i64* %tmp2, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp1, align 8
-; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store i64 0, i64* %tmp2, align 8
 ; VF_4-LABEL:  Checking a loop in "i64_factor_3"
-; VF_4:          Found an estimated cost of 120 for VF 4 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
+; VF_4:          Found an estimated cost of 132 for VF 4 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load i64, i64* %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i64, i64* %tmp2, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp1, align 8
-; VF_4-NEXT:     Found an estimated cost of 72 for VF 4 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 36 for VF 4 For instruction: store i64 0, i64* %tmp2, align 8
 ; VF_8-LABEL:  Checking a loop in "i64_factor_3"
-; VF_8:          Found an estimated cost of 432 for VF 8 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
+; VF_8:          Found an estimated cost of 264 for VF 8 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load i64, i64* %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i64, i64* %tmp2, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp1, align 8
-; VF_8-NEXT:     Found an estimated cost of 240 for VF 8 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 72 for VF 8 For instruction: store i64 0, i64* %tmp2, align 8
 ; VF_16-LABEL: Checking a loop in "i64_factor_3"
-; VF_16:         Found an estimated cost of 1632 for VF 16 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
+; VF_16:         Found an estimated cost of 528 for VF 16 For instruction: %tmp3 = load i64, i64* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load i64, i64* %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i64, i64* %tmp2, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp1, align 8
-; VF_16-NEXT:    Found an estimated cost of 864 for VF 16 For instruction: store i64 0, i64* %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 144 for VF 16 For instruction: store i64 0, i64* %tmp2, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i64.3, %i64.3* %data, i64 %i, i32 0
@@ -514,33 +514,33 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "f16_factor_3"
-; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load half, half* %tmp0, align 2
+; VF_2:          Found an estimated cost of 18 for VF 2 For instruction: %tmp3 = load half, half* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load half, half* %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load half, half* %tmp2, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp1, align 2
-; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store half 0xH0000, half* %tmp2, align 2
 ; VF_4-LABEL:  Checking a loop in "f16_factor_3"
-; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load half, half* %tmp0, align 2
+; VF_4:          Found an estimated cost of 28 for VF 4 For instruction: %tmp3 = load half, half* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load half, half* %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load half, half* %tmp2, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp1, align 2
-; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store half 0xH0000, half* %tmp2, align 2
 ; VF_8-LABEL:  Checking a loop in "f16_factor_3"
-; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load half, half* %tmp0, align 2
+; VF_8:          Found an estimated cost of 56 for VF 8 For instruction: %tmp3 = load half, half* %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load half, half* %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load half, half* %tmp2, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp1, align 2
-; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store half 0xH0000, half* %tmp2, align 2
 ; VF_16-LABEL: Checking a loop in "f16_factor_3"
-; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load half, half* %tmp0, align 2
+; VF_16:         Found an estimated cost of 112 for VF 16 For instruction: %tmp3 = load half, half* %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load half, half* %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load half, half* %tmp2, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp1, align 2
-; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store half 0xH0000, half* %tmp2, align 2
+; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store half 0xH0000, half* %tmp2, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f16.3, %f16.3* %data, i64 %i, i32 0
@@ -566,12 +566,12 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "f32_factor_3"
-; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load float, float* %tmp0, align 4
+; VF_2:          Found an estimated cost of 16 for VF 2 For instruction: %tmp3 = load float, float* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load float, float* %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load float, float* %tmp2, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp1, align 4
-; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store float 0.000000e+00, float* %tmp2, align 4
 ; VF_4-LABEL:  Checking a loop in "f32_factor_3"
 ; VF_4:          Found an estimated cost of 24 for VF 4 For instruction: %tmp3 = load float, float* %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load float, float* %tmp1, align 4
@@ -580,19 +580,19 @@ entry:
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, float* %tmp1, align 4
 ; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store float 0.000000e+00, float* %tmp2, align 4
 ; VF_8-LABEL:  Checking a loop in "f32_factor_3"
-; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load float, float* %tmp0, align 4
+; VF_8:          Found an estimated cost of 64 for VF 8 For instruction: %tmp3 = load float, float* %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load float, float* %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load float, float* %tmp2, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp1, align 4
-; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store float 0.000000e+00, float* %tmp2, align 4
 ; VF_16-LABEL: Checking a loop in "f32_factor_3"
-; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load float, float* %tmp0, align 4
+; VF_16:         Found an estimated cost of 128 for VF 16 For instruction: %tmp3 = load float, float* %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load float, float* %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load float, float* %tmp2, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp1, align 4
-; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store float 0.000000e+00, float* %tmp2, align 4
+; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store float 0.000000e+00, float* %tmp2, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f32.3, %f32.3* %data, i64 %i, i32 0
@@ -618,33 +618,33 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "f64_factor_3"
-; VF_2:          Found an estimated cost of 30 for VF 2 For instruction: %tmp3 = load double, double* %tmp0, align 8
+; VF_2:          Found an estimated cost of 18 for VF 2 For instruction: %tmp3 = load double, double* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp4 = load double, double* %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load double, double* %tmp2, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp1, align 8
-; VF_2-NEXT:     Found an estimated cost of 18 for VF 2 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_2-NEXT:     Found an estimated cost of 12 for VF 2 For instruction: store double 0.000000e+00, double* %tmp2, align 8
 ; VF_4-LABEL:  Checking a loop in "f64_factor_3"
-; VF_4:          Found an estimated cost of 108 for VF 4 For instruction: %tmp3 = load double, double* %tmp0, align 8
+; VF_4:          Found an estimated cost of 36 for VF 4 For instruction: %tmp3 = load double, double* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp4 = load double, double* %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load double, double* %tmp2, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp1, align 8
-; VF_4-NEXT:     Found an estimated cost of 60 for VF 4 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_4-NEXT:     Found an estimated cost of 24 for VF 4 For instruction: store double 0.000000e+00, double* %tmp2, align 8
 ; VF_8-LABEL:  Checking a loop in "f64_factor_3"
-; VF_8:          Found an estimated cost of 408 for VF 8 For instruction: %tmp3 = load double, double* %tmp0, align 8
+; VF_8:          Found an estimated cost of 72 for VF 8 For instruction: %tmp3 = load double, double* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp4 = load double, double* %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load double, double* %tmp2, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp1, align 8
-; VF_8-NEXT:     Found an estimated cost of 216 for VF 8 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_8-NEXT:     Found an estimated cost of 48 for VF 8 For instruction: store double 0.000000e+00, double* %tmp2, align 8
 ; VF_16-LABEL: Checking a loop in "f64_factor_3"
-; VF_16:         Found an estimated cost of 1584 for VF 16 For instruction: %tmp3 = load double, double* %tmp0, align 8
+; VF_16:         Found an estimated cost of 144 for VF 16 For instruction: %tmp3 = load double, double* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp4 = load double, double* %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load double, double* %tmp2, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp1, align 8
-; VF_16-NEXT:    Found an estimated cost of 816 for VF 16 For instruction: store double 0.000000e+00, double* %tmp2, align 8
+; VF_16-NEXT:    Found an estimated cost of 96 for VF 16 For instruction: store double 0.000000e+00, double* %tmp2, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f64.3, %f64.3* %data, i64 %i, i32 0
@@ -673,41 +673,41 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i8_factor_4"
-; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
+; VF_2:          Found an estimated cost of 48 for VF 2 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i8, i8* %tmp1, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i8, i8* %tmp2, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i8, i8* %tmp3, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp0, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp1, align 1
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i8 0, i8* %tmp2, align 1
-; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i8 0, i8* %tmp3, align 1
+; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store i8 0, i8* %tmp3, align 1
 ; VF_4-LABEL: Checking a loop in "i8_factor_4"
-; VF_4:         Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
+; VF_4:         Found an estimated cost of 96 for VF 4 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i8, i8* %tmp1, align 1
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i8, i8* %tmp2, align 1
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i8, i8* %tmp3, align 1
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp0, align 1
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp1, align 1
 ; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store i8 0, i8* %tmp2, align 1
-; VF_4-NEXT:    Found an estimated cost of 80 for VF 4 For instruction: store i8 0, i8* %tmp3, align 1
+; VF_4-NEXT:    Found an estimated cost of 32 for VF 4 For instruction: store i8 0, i8* %tmp3, align 1
 ; VF_8-LABEL:  Checking a loop in "i8_factor_4"
-; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
+; VF_8:          Found an estimated cost of 192 for VF 8 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i8, i8* %tmp1, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i8, i8* %tmp2, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i8, i8* %tmp3, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp0, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp1, align 1
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp2, align 1
-; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store i8 0, i8* %tmp3, align 1
+; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store i8 0, i8* %tmp3, align 1
 ; VF_16-LABEL: Checking a loop in "i8_factor_4"
-; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
+; VF_16:         Found an estimated cost of 384 for VF 16 For instruction: %tmp4 = load i8, i8* %tmp0, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i8, i8* %tmp1, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i8, i8* %tmp2, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i8, i8* %tmp3, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp0, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp1, align 1
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp2, align 1
-; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store i8 0, i8* %tmp3, align 1
+; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store i8 0, i8* %tmp3, align 1
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i8.4, %i8.4* %data, i64 %i, i32 0
@@ -736,41 +736,41 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i16_factor_4"
-; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
+; VF_2:          Found an estimated cost of 48 for VF 2 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i16, i16* %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i16, i16* %tmp2, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i16, i16* %tmp3, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i16 0, i16* %tmp2, align 2
-; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i16 0, i16* %tmp3, align 2
+; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store i16 0, i16* %tmp3, align 2
 ; VF_4-LABEL:  Checking a loop in "i16_factor_4"
-; VF_4:          Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
+; VF_4:          Found an estimated cost of 96 for VF 4 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i16, i16* %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i16, i16* %tmp2, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i16, i16* %tmp3, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp2, align 2
-; VF_4-NEXT:     Found an estimated cost of 80 for VF 4 For instruction: store i16 0, i16* %tmp3, align 2
+; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store i16 0, i16* %tmp3, align 2
 ; VF_8-LABEL:  Checking a loop in "i16_factor_4"
-; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
+; VF_8:          Found an estimated cost of 192 for VF 8 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i16, i16* %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i16, i16* %tmp2, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i16, i16* %tmp3, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp2, align 2
-; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store i16 0, i16* %tmp3, align 2
+; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store i16 0, i16* %tmp3, align 2
 ; VF_16-LABEL: Checking a loop in "i16_factor_4"
-; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
+; VF_16:         Found an estimated cost of 384 for VF 16 For instruction: %tmp4 = load i16, i16* %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i16, i16* %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i16, i16* %tmp2, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i16, i16* %tmp3, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp2, align 2
-; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store i16 0, i16* %tmp3, align 2
+; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store i16 0, i16* %tmp3, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i16.4, %i16.4* %data, i64 %i, i32 0
@@ -799,14 +799,14 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i32_factor_4"
-; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
+; VF_2:          Found an estimated cost of 48 for VF 2 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i32, i32* %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i32, i32* %tmp2, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i32, i32* %tmp3, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp2, align 4
-; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i32 0, i32* %tmp3, align 4
+; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store i32 0, i32* %tmp3, align 4
 ; VF_4-LABEL:  Checking a loop in "i32_factor_4"
 ; VF_4:          Found an estimated cost of 32 for VF 4 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i32, i32* %tmp1, align 4
@@ -817,23 +817,23 @@ entry:
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp2, align 4
 ; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store i32 0, i32* %tmp3, align 4
 ; VF_8-LABEL:  Checking a loop in "i32_factor_4"
-; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
+; VF_8:          Found an estimated cost of 192 for VF 8 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i32, i32* %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i32, i32* %tmp2, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i32, i32* %tmp3, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp2, align 4
-; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store i32 0, i32* %tmp3, align 4
+; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store i32 0, i32* %tmp3, align 4
 ; VF_16-LABEL: Checking a loop in "i32_factor_4"
-; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
+; VF_16:         Found an estimated cost of 384 for VF 16 For instruction: %tmp4 = load i32, i32* %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i32, i32* %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i32, i32* %tmp2, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i32, i32* %tmp3, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp2, align 4
-; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store i32 0, i32* %tmp3, align 4
+; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store i32 0, i32* %tmp3, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i32.4, %i32.4* %data, i64 %i, i32 0
@@ -862,41 +862,41 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "i64_factor_4"
-; VF_2:          Found an estimated cost of 48 for VF 2 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
+; VF_2:          Found an estimated cost of 88 for VF 2 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load i64, i64* %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load i64, i64* %tmp2, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load i64, i64* %tmp3, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp2, align 8
-; VF_2-NEXT:     Found an estimated cost of 32 for VF 2 For instruction: store i64 0, i64* %tmp3, align 8
+; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store i64 0, i64* %tmp3, align 8
 ; VF_4-LABEL:  Checking a loop in "i64_factor_4"
-; VF_4:          Found an estimated cost of 160 for VF 4 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
+; VF_4:          Found an estimated cost of 176 for VF 4 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load i64, i64* %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load i64, i64* %tmp2, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load i64, i64* %tmp3, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp2, align 8
-; VF_4-NEXT:     Found an estimated cost of 96 for VF 4 For instruction: store i64 0, i64* %tmp3, align 8
+; VF_4-NEXT:     Found an estimated cost of 48 for VF 4 For instruction: store i64 0, i64* %tmp3, align 8
 ; VF_8-LABEL:  Checking a loop in "i64_factor_4"
-; VF_8:          Found an estimated cost of 576 for VF 8 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
+; VF_8:          Found an estimated cost of 352 for VF 8 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load i64, i64* %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load i64, i64* %tmp2, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load i64, i64* %tmp3, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp2, align 8
-; VF_8-NEXT:     Found an estimated cost of 320 for VF 8 For instruction: store i64 0, i64* %tmp3, align 8
+; VF_8-NEXT:     Found an estimated cost of 96 for VF 8 For instruction: store i64 0, i64* %tmp3, align 8
 ; VF_16-LABEL: Checking a loop in "i64_factor_4"
-; VF_16:         Found an estimated cost of 2176 for VF 16 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
+; VF_16:         Found an estimated cost of 704 for VF 16 For instruction: %tmp4 = load i64, i64* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load i64, i64* %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load i64, i64* %tmp2, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load i64, i64* %tmp3, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp2, align 8
-; VF_16-NEXT:    Found an estimated cost of 1152 for VF 16 For instruction: store i64 0, i64* %tmp3, align 8
+; VF_16-NEXT:    Found an estimated cost of 192 for VF 16 For instruction: store i64 0, i64* %tmp3, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %i64.4, %i64.4* %data, i64 %i, i32 0
@@ -925,41 +925,41 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "f16_factor_4"
-; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load half, half* %tmp0, align 2
+; VF_2:          Found an estimated cost of 18 for VF 2 For instruction: %tmp4 = load half, half* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load half, half* %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load half, half* %tmp2, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load half, half* %tmp3, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp0, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp1, align 2
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store half 0xH0000, half* %tmp2, align 2
-; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store half 0xH0000, half* %tmp3, align 2
+; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store half 0xH0000, half* %tmp3, align 2
 ; VF_4-LABEL:  Checking a loop in "f16_factor_4"
-; VF_4:          Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load half, half* %tmp0, align 2
+; VF_4:          Found an estimated cost of 36 for VF 4 For instruction: %tmp4 = load half, half* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load half, half* %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load half, half* %tmp2, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load half, half* %tmp3, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp0, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp1, align 2
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp2, align 2
-; VF_4-NEXT:     Found an estimated cost of 80 for VF 4 For instruction: store half 0xH0000, half* %tmp3, align 2
+; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store half 0xH0000, half* %tmp3, align 2
 ; VF_8-LABEL:  Checking a loop in "f16_factor_4"
-; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load half, half* %tmp0, align 2
+; VF_8:          Found an estimated cost of 72 for VF 8 For instruction: %tmp4 = load half, half* %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load half, half* %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load half, half* %tmp2, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load half, half* %tmp3, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp0, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp1, align 2
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp2, align 2
-; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store half 0xH0000, half* %tmp3, align 2
+; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store half 0xH0000, half* %tmp3, align 2
 ; VF_16-LABEL: Checking a loop in "f16_factor_4"
-; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load half, half* %tmp0, align 2
+; VF_16:         Found an estimated cost of 144 for VF 16 For instruction: %tmp4 = load half, half* %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load half, half* %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load half, half* %tmp2, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load half, half* %tmp3, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp0, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp1, align 2
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store half 0xH0000, half* %tmp2, align 2
-; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store half 0xH0000, half* %tmp3, align 2
+; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store half 0xH0000, half* %tmp3, align 2
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f16.4, %f16.4* %data, i64 %i, i32 0
@@ -988,14 +988,14 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "f32_factor_4"
-; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load float, float* %tmp0, align 4
+; VF_2:          Found an estimated cost of 20 for VF 2 For instruction: %tmp4 = load float, float* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load float, float* %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load float, float* %tmp2, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load float, float* %tmp3, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp0, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp1, align 4
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store float 0.000000e+00, float* %tmp2, align 4
-; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store float 0.000000e+00, float* %tmp3, align 4
+; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store float 0.000000e+00, float* %tmp3, align 4
 ; VF_4-LABEL:  Checking a loop in "f32_factor_4"
 ; VF_4:          Found an estimated cost of 32 for VF 4 For instruction: %tmp4 = load float, float* %tmp0, align 4
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load float, float* %tmp1, align 4
@@ -1006,23 +1006,23 @@ entry:
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store float 0.000000e+00, float* %tmp2, align 4
 ; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store float 0.000000e+00, float* %tmp3, align 4
 ; VF_8-LABEL:  Checking a loop in "f32_factor_4"
-; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load float, float* %tmp0, align 4
+; VF_8:          Found an estimated cost of 80 for VF 8 For instruction: %tmp4 = load float, float* %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load float, float* %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load float, float* %tmp2, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load float, float* %tmp3, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp0, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp1, align 4
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store float 0.000000e+00, float* %tmp2, align 4
-; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store float 0.000000e+00, float* %tmp3, align 4
+; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store float 0.000000e+00, float* %tmp3, align 4
 ; VF_16-LABEL: Checking a loop in "f32_factor_4"
-; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load float, float* %tmp0, align 4
+; VF_16:         Found an estimated cost of 160 for VF 16 For instruction: %tmp4 = load float, float* %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load float, float* %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load float, float* %tmp2, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load float, float* %tmp3, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp0, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp1, align 4
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store float 0.000000e+00, float* %tmp2, align 4
-; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store float 0.000000e+00, float* %tmp3, align 4
+; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store float 0.000000e+00, float* %tmp3, align 4
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f32.4, %f32.4* %data, i64 %i, i32 0
@@ -1051,41 +1051,41 @@ entry:
   br label %for.body
 
 ; VF_2-LABEL:  Checking a loop in "f64_factor_4"
-; VF_2:          Found an estimated cost of 40 for VF 2 For instruction: %tmp4 = load double, double* %tmp0, align 8
+; VF_2:          Found an estimated cost of 24 for VF 2 For instruction: %tmp4 = load double, double* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp5 = load double, double* %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp6 = load double, double* %tmp2, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp7 = load double, double* %tmp3, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp0, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp1, align 8
 ; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store double 0.000000e+00, double* %tmp2, align 8
-; VF_2-NEXT:     Found an estimated cost of 24 for VF 2 For instruction: store double 0.000000e+00, double* %tmp3, align 8
+; VF_2-NEXT:     Found an estimated cost of 16 for VF 2 For instruction: store double 0.000000e+00, double* %tmp3, align 8
 ; VF_4-LABEL:  Checking a loop in "f64_factor_4"
-; VF_4:          Found an estimated cost of 144 for VF 4 For instruction: %tmp4 = load double, double* %tmp0, align 8
+; VF_4:          Found an estimated cost of 48 for VF 4 For instruction: %tmp4 = load double, double* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp5 = load double, double* %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp6 = load double, double* %tmp2, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp7 = load double, double* %tmp3, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp0, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp1, align 8
 ; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store double 0.000000e+00, double* %tmp2, align 8
-; VF_4-NEXT:     Found an estimated cost of 80 for VF 4 For instruction: store double 0.000000e+00, double* %tmp3, align 8
+; VF_4-NEXT:     Found an estimated cost of 32 for VF 4 For instruction: store double 0.000000e+00, double* %tmp3, align 8
 ; VF_8-LABEL:  Checking a loop in "f64_factor_4"
-; VF_8:          Found an estimated cost of 544 for VF 8 For instruction: %tmp4 = load double, double* %tmp0, align 8
+; VF_8:          Found an estimated cost of 96 for VF 8 For instruction: %tmp4 = load double, double* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp5 = load double, double* %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp6 = load double, double* %tmp2, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp7 = load double, double* %tmp3, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp0, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp1, align 8
 ; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store double 0.000000e+00, double* %tmp2, align 8
-; VF_8-NEXT:     Found an estimated cost of 288 for VF 8 For instruction: store double 0.000000e+00, double* %tmp3, align 8
+; VF_8-NEXT:     Found an estimated cost of 64 for VF 8 For instruction: store double 0.000000e+00, double* %tmp3, align 8
 ; VF_16-LABEL: Checking a loop in "f64_factor_4"
-; VF_16:         Found an estimated cost of 2112 for VF 16 For instruction: %tmp4 = load double, double* %tmp0, align 8
+; VF_16:         Found an estimated cost of 192 for VF 16 For instruction: %tmp4 = load double, double* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp5 = load double, double* %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp6 = load double, double* %tmp2, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp7 = load double, double* %tmp3, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp0, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp1, align 8
 ; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store double 0.000000e+00, double* %tmp2, align 8
-; VF_16-NEXT:    Found an estimated cost of 1088 for VF 16 For instruction: store double 0.000000e+00, double* %tmp3, align 8
+; VF_16-NEXT:    Found an estimated cost of 128 for VF 16 For instruction: store double 0.000000e+00, double* %tmp3, align 8
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %tmp0 = getelementptr inbounds %f64.4, %f64.4* %data, i64 %i, i32 0
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll
index 3356db935eea6be097860278bbf181ef1ad362fb..30bc06a7a0b1aef97418d4f02c5b0fb448534aa1 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll
@@ -8,7 +8,7 @@ target triple = "thumbv8.1m.main-arm-none-eabi"
 
 ; CHECK-COST-LABEL: arm_offset_q15
 ; CHECK-COST: Found an estimated cost of 10 for VF 1 For instruction:   %1 = tail call i16 @llvm.sadd.sat.i16(i16 %0, i16 %offset)
-; CHECK-COST: Found an estimated cost of 28 for VF 2 For instruction:   %1 = tail call i16 @llvm.sadd.sat.i16(i16 %0, i16 %offset)
+; CHECK-COST: Found an estimated cost of 36 for VF 2 For instruction:   %1 = tail call i16 @llvm.sadd.sat.i16(i16 %0, i16 %offset)
 ; CHECK-COST: Found an estimated cost of 8 for VF 4 For instruction:   %1 = tail call i16 @llvm.sadd.sat.i16(i16 %0, i16 %offset)
 ; CHECK-COST: Found an estimated cost of 2 for VF 8 For instruction:   %1 = tail call i16 @llvm.sadd.sat.i16(i16 %0, i16 %offset)
 
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
index 730e26af0bb2646fd8c0b14f9937d3faac9df519..59422cb07d6a6182a20faf32c407bbe28c07589e 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
@@ -688,19 +688,37 @@ end:
 define hidden void @pointer_phi_v4half_add3(half* noalias nocapture readonly %A, half* noalias nocapture %B, half %y) {
 ; CHECK-LABEL: @pointer_phi_v4half_add3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr half, half* [[A:%.*]], i32 2976
+; CHECK-NEXT:    [[IND_END3:%.*]] = getelementptr half, half* [[B:%.*]], i32 992
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[Y:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x half> [[BROADCAST_SPLATINSERT]], <8 x half> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[INDEX]], 3
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr half, half* [[A]], i32 [[TMP0]]
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr half, half* [[B]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half* [[NEXT_GEP]] to <24 x half>*
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <24 x half>, <24 x half>* [[TMP1]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x half> [[WIDE_VEC]], <24 x half> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <8 x half> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast half* [[NEXT_GEP4]] to <8 x half>*
+; CHECK-NEXT:    store <8 x half> [[TMP2]], <8 x half>* [[TMP3]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
+; CHECK-NEXT:    br i1 [[TMP4]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], [[LOOP22:!llvm.loop !.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[A_ADDR_09:%.*]] = phi half* [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[B_ADDR_07:%.*]] = phi half* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[B:%.*]], [[ENTRY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = load half, half* [[A_ADDR_09]], align 4
+; CHECK-NEXT:    [[A_ADDR_09:%.*]] = phi half* [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 992, [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[B_ADDR_07:%.*]] = phi half* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[IND_END3]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = load half, half* [[A_ADDR_09]], align 4
 ; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds half, half* [[A_ADDR_09]], i32 3
-; CHECK-NEXT:    [[ADD:%.*]] = fadd fast half [[TMP0]], [[Y:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast half [[TMP5]], [[Y]]
 ; CHECK-NEXT:    store half [[ADD]], half* [[B_ADDR_07]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds half, half* [[B_ADDR_07]], i32 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[FOR_BODY]], [[LOOP23:!llvm.loop !.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void
 ;
@@ -753,7 +771,7 @@ define hidden void @pointer_phi_v4i32_uf2(i32* noalias nocapture readonly %A, i3
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 9992
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i32, i32* [[POINTER_PHI]], i32 48
-; CHECK-NEXT:    br i1 [[TMP7]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], [[LOOP22:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], [[LOOP24:!llvm.loop !.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
@@ -767,7 +785,7 @@ define hidden void @pointer_phi_v4i32_uf2(i32* noalias nocapture readonly %A, i3
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i32, i32* [[B_ADDR_06]], i32 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_07]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 10000
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], [[LOOP23:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], [[LOOP25:!llvm.loop !.*]]
 ;
 
 entry:
@@ -837,7 +855,7 @@ define hidden void @pointer_phi_v4i32_uf4(i32* noalias nocapture readonly %A, i3
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 9984
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i32, i32* [[POINTER_PHI]], i32 96
-; CHECK-NEXT:    br i1 [[TMP15]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], [[LOOP24:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY]], [[LOOP26:!llvm.loop !.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
@@ -851,7 +869,7 @@ define hidden void @pointer_phi_v4i32_uf4(i32* noalias nocapture readonly %A, i3
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i32, i32* [[B_ADDR_06]], i32 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_07]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 10000
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], [[LOOP25:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], [[LOOP27:!llvm.loop !.*]]
 ;
 entry:
   br label %for.body
@@ -893,23 +911,23 @@ define hidden void @mult_ptr_iv(i8* noalias nocapture readonly %x, i8* noalias n
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, i8* [[POINTER_PHI]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, i8* [[POINTER_PHI5]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, <4 x i8*> [[TMP0]], i32 1
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP0]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef), !alias.scope !26
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP0]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef), !alias.scope !28
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, <4 x i8*> [[TMP0]], i32 2
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP2]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef), !alias.scope !26
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER8:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP3]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef), !alias.scope !26
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP2]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef), !alias.scope !28
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER8:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP3]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef), !alias.scope !28
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], <i8 10, i8 10, i8 10, i8 10>
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER7]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER8]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, <4 x i8*> [[TMP1]], i32 1
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP4]], <4 x i8*> [[TMP1]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !alias.scope !29, !noalias !26
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP4]], <4 x i8*> [[TMP1]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !alias.scope !31, !noalias !28
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, <4 x i8*> [[TMP1]], i32 2
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP5]], <4 x i8*> [[TMP7]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !alias.scope !29, !noalias !26
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP6]], <4 x i8*> [[TMP8]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !alias.scope !29, !noalias !26
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP5]], <4 x i8*> [[TMP7]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !alias.scope !31, !noalias !28
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP6]], <4 x i8*> [[TMP8]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !alias.scope !31, !noalias !28
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, i8* [[POINTER_PHI]], i32 12
 ; CHECK-NEXT:    [[PTR_IND6]] = getelementptr i8, i8* [[POINTER_PHI5]], i32 12
-; CHECK-NEXT:    br i1 [[TMP9]], label [[END:%.*]], label [[VECTOR_BODY]], [[LOOP31:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[END:%.*]], label [[VECTOR_BODY]], [[LOOP33:!llvm.loop !.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[X_ADDR_050:%.*]] = phi i8* [ [[INCDEC_PTR2:%.*]], [[FOR_BODY]] ], [ [[X]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[Z_ADDR_049:%.*]] = phi i8* [ [[INCDEC_PTR34:%.*]], [[FOR_BODY]] ], [ [[Z]], [[ENTRY]] ]
@@ -931,7 +949,7 @@ define hidden void @mult_ptr_iv(i8* noalias nocapture readonly %x, i8* noalias n
 ; CHECK-NEXT:    store i8 [[MUL2]], i8* [[INCDEC_PTR33]], align 1
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_048]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END]], label [[FOR_BODY]], [[LOOP32:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END]], label [[FOR_BODY]], [[LOOP34:!llvm.loop !.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll
index b67475f19fd189d15b17ccb6b28b28b2e5dbb906..7e33b331aaa82ab6caa5e31a1aa964f03f49a858 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll
@@ -465,25 +465,58 @@ for.body:
 define void @fptrunc_not_allowed(float* noalias nocapture %A, float* noalias nocapture readonly %B, float* noalias nocapture readonly %C, half* noalias nocapture %D) #0 {
 ; CHECK-LABEL: @fptrunc_not_allowed(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, float* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <4 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[C:%.*]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <4 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x float>, <4 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast float* [[TMP9]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP7]], <4 x float>* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = fptrunc <4 x float> [[TMP7]] to <4 x half>
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast <4 x half> [[TMP11]], <half 0xH4000, half 0xH4000, half 0xH4000, half 0xH4000>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds half, half* [[D:%.*]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds half, half* [[TMP13]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast half* [[TMP14]] to <4 x half>*
+; CHECK-NEXT:    store <4 x half> [[TMP12]], <4 x half>* [[TMP15]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 428
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP12:!llvm.loop !.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 431, 428
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 428, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_017:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD6:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i32 [[I_017]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[C:%.*]], i32 [[I_017]]
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX1]], align 4
-; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i32 [[I_017]]
+; CHECK-NEXT:    [[I_017:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD6:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i32 [[I_017]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[C]], i32 [[I_017]]
+; CHECK-NEXT:    [[TMP18:%.*]] = load float, float* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i32 [[I_017]]
 ; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[CONV:%.*]] = fptrunc float [[ADD]] to half
 ; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast half [[CONV]], 0xH4000
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds half, half* [[D:%.*]], i32 [[I_017]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds half, half* [[D]], i32 [[I_017]]
 ; CHECK-NEXT:    store half [[FACTOR]], half* [[ARRAYIDX5]], align 2
 ; CHECK-NEXT:    [[ADD6]] = add nuw nsw i32 [[I_017]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[ADD6]], 431
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], [[LOOP13:!llvm.loop !.*]]
 ;
 entry:
   br label %for.body
@@ -591,7 +624,7 @@ define i32 @i32_smin_reduction(i32* nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP5]] = select <4 x i1> [[TMP4]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP12:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP14:!llvm.loop !.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP5]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
@@ -609,7 +642,7 @@ define i32 @i32_smin_reduction(i32* nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP8]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP13:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP15:!llvm.loop !.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -661,7 +694,7 @@ define i32 @i32_smax_reduction(i32* nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP5]] = select <4 x i1> [[TMP4]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP14:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP16:!llvm.loop !.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP5]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
@@ -679,7 +712,7 @@ define i32 @i32_smax_reduction(i32* nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP8]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP15:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP17:!llvm.loop !.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -731,7 +764,7 @@ define i32 @i32_umin_reduction(i32* nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP5]] = select <4 x i1> [[TMP4]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP16:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP18:!llvm.loop !.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP5]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
@@ -749,7 +782,7 @@ define i32 @i32_umin_reduction(i32* nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP8]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP17:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP19:!llvm.loop !.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -801,7 +834,7 @@ define i32 @i32_umax_reduction(i32* nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP5]] = select <4 x i1> [[TMP4]], <4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP18:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP20:!llvm.loop !.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP5]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
@@ -819,7 +852,7 @@ define i32 @i32_umax_reduction(i32* nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[ADD]] = select i1 [[C]], i32 [[R_07]], i32 [[TMP8]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP19:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP21:!llvm.loop !.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
index 13c03f01d1789ba74163a72c396b2e6f78edb5be..f63effd6cda10e106b55cf95ccf6c1730cf9e91d 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
@@ -234,7 +234,7 @@ for.body:                                         ; preds = %for.body, %entry
   %x.05 = phi ppc_fp128 [ %d, %entry ], [ %sub, %for.body ]
   %arrayidx = getelementptr inbounds ppc_fp128, ppc_fp128* %n, i32 %i.06
   %0 = load ppc_fp128, ppc_fp128* %arrayidx, align 8
-  %sub = fsub ppc_fp128 %x.05, %0
+  %sub = fsub fast ppc_fp128 %x.05, %0
   %inc = add nsw i32 %i.06, 1
   %exitcond = icmp eq i32 %inc, 2048
   br i1 %exitcond, label %for.end, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/cast-float-fptosi.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-cast-float-fptosi.ll
similarity index 98%
rename from llvm/test/Transforms/LoopVectorize/RISCV/cast-float-fptosi.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-cast-float-fptosi.ll
index 01d9e814688234fb96b77d10284a54dac26dfb22..4666b5350be12bb30c0c12f87d74a9881cc3c66b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/cast-float-fptosi.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-cast-float-fptosi.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './cast-float-fptosi.c'
 source_filename = "./cast-float-fptosi.c"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/cast-float-trunc.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-cast-float-trunc.ll
similarity index 97%
rename from llvm/test/Transforms/LoopVectorize/RISCV/cast-float-trunc.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-cast-float-trunc.ll
index 0fd7297575b7a0ad3217a56655fe3bfc50f2bc02..55635d09f717d6d3636eef3ed524a9695e01fbc7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/cast-float-trunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-cast-float-trunc.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './cast-float-trunc.c'
 source_filename = "./cast-float-trunc.c"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/cast-float-uitofp.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-cast-float-uitofp.ll
similarity index 97%
rename from llvm/test/Transforms/LoopVectorize/RISCV/cast-float-uitofp.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-cast-float-uitofp.ll
index 56d23ab14c8d832f4d23c3715fc63564c6ad02b6..a2b69dcc590f80ee4ea61fc9c0791014de4579cd 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/cast-float-uitofp.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-cast-float-uitofp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './ext-float.c'
 source_filename = "./ext-float.c"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/daxpy.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-daxpy.ll
similarity index 99%
rename from llvm/test/Transforms/LoopVectorize/RISCV/daxpy.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-daxpy.ll
index a747614f4accb4c3f8fe64f84f26033dd02f9754..450ac2ccf040157ae42b62fa18731f7cce2981d2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/daxpy.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-daxpy.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -vector-register-width-factor=8 < %s -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize < %s -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = 'daxpy.c'
 source_filename = "daxpy.c"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/ext-int-vpred.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-ext-int-vpred.ll
similarity index 97%
rename from llvm/test/Transforms/LoopVectorize/RISCV/ext-int-vpred.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-ext-int-vpred.ll
index f97d807bd623e8f9c0e884542f156331253197f2..7d3c4e9a2cc614f050a1566bd261510194b802b4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/ext-int-vpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-ext-int-vpred.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './ext-int.c'
 source_filename = "./ext-int.c"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-slideleft.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-first-order-recurrence-slideleft.ll
similarity index 99%
rename from llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-slideleft.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-first-order-recurrence-slideleft.ll
index d4c02efea9de973df16615963df0216425fb85d4..9b9776a84699dcc5dbb71d6841cb7072b2713fb5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-slideleft.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-first-order-recurrence-slideleft.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -dce \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -dce -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -vector-register-width-factor=8 -instcombine < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -dce \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -dce -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -instcombine < %s  -o - | FileCheck %s --check-prefix=CHECK1
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -dce \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -dce -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize \
 ; RUN:    -loop-vectorize -instcombine < %s  -o - | FileCheck %s --check-prefix=CHECKVP
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-vpred.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-gather-vpred.ll
similarity index 67%
rename from llvm/test/Transforms/LoopVectorize/RISCV/gather-vpred.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-gather-vpred.ll
index 08150cde92a4e2a47daf78c90393982fd235f09f..b0a61c65453d44e53a87bd93f41bc08cc414a0e6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-vpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-gather-vpred.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -prefer-predicate-over-epilogue=predicate-dont-vectorize -riscv-v-vector-bits-min=64 -S -O2 < %s  -o - | FileCheck %s
 ; ModuleID = './gather-vpred.c'
 source_filename = "./gather-vpred.c"
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
@@ -16,30 +16,30 @@ define dso_local void @simple_add(i32 signext %N, i8* noalias nocapture readonly
 ; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP2]], i64 3, i64 3)
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64* [[TMP1]] to <vscale x 8 x i64>*
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64.p0nxv8i64(<vscale x 8 x i64>* [[TMP4]], i32 8, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer), i32 [[TMP5]])
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[ISA:%.*]], <vscale x 8 x i64> [[VP_OP_LOAD]]
-; CHECK-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK-NEXT:    [[VP_GATHER:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0i8(<vscale x 8 x i8*> [[TMP6]], i32 1, <vscale x 8 x i1> [[TMP0]], i32 [[TMP7]])
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, i8* [[ISC:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to <vscale x 8 x i8>*
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv8i8.p0nxv8i8(<vscale x 8 x i8> [[VP_GATHER]], <vscale x 8 x i8>* [[TMP9]], i32 1, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer), i32 [[TMP10]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP11]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP3]], i64 3, i64 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[TMP2]] to <vscale x 8 x i64>*
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64.p0nxv8i64(<vscale x 8 x i64>* [[TMP5]], i32 8, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer), i32 [[TMP6]])
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, i8* [[ISA:%.*]], <vscale x 8 x i64> [[VP_OP_LOAD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    [[VP_GATHER:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0i8(<vscale x 8 x i8*> [[TMP7]], i32 1, <vscale x 8 x i1> [[TMP1]], i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8* [[ISC:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8* [[TMP9]] to <vscale x 8 x i8>*
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i8.p0nxv8i8(<vscale x 8 x i8> [[VP_GATHER]], <vscale x 8 x i8>* [[TMP10]], i32 1, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer), i32 [[TMP11]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-float.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-induction-float.ll
similarity index 58%
rename from llvm/test/Transforms/LoopVectorize/RISCV/induction-float.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-induction-float.ll
index 1a2101c623cee792039dd055988584bf968c4558..2d35e699e13308adfe87d5373a38711097e98f1c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-float.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-induction-float.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O3 < %s -o - \
 ; RUN:    | FileCheck %s
 ; ModuleID = 'induction-float.c'
@@ -15,35 +15,35 @@ define dso_local void @induction_float(i32 signext %N, float* noalias nocapture
 ; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    [[STEPVEC_BASE4:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.stepvector.nxv2i32()
-; CHECK-NEXT:    [[TMP0:%.*]] = uitofp <vscale x 2 x i32> [[STEPVEC_BASE4]] to <vscale x 2 x float>
-; CHECK-NEXT:    [[STEPVEC_SCALED:%.*]] = fmul fast <vscale x 2 x float> [[TMP0]], shufflevector (<vscale x 2 x float> insertelement (<vscale x 2 x float> poison, float 2.000000e+00, i32 0), <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[INDUCTION5:%.*]] = fadd fast <vscale x 2 x float> [[STEPVEC_SCALED]], shufflevector (<vscale x 2 x float> insertelement (<vscale x 2 x float> poison, float 1.000000e+00, i32 0), <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.experimental.stepvector.nxv2i32()
+; CHECK-NEXT:    [[TMP1:%.*]] = uitofp <vscale x 2 x i32> [[TMP0]] to <vscale x 2 x float>
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <vscale x 2 x float> [[TMP1]], shufflevector (<vscale x 2 x float> insertelement (<vscale x 2 x float> poison, float 2.000000e+00, i32 0), <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION12:%.*]] = fadd fast <vscale x 2 x float> [[TMP2]], shufflevector (<vscale x 2 x float> insertelement (<vscale x 2 x float> poison, float 1.000000e+00, i32 0), <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer)
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x float> [ [[INDUCTION5]], [[FOR_BODY_PREHEADER]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP1]], i64 2, i64 0)
-; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[VP_OP:%.*]] = call fast <vscale x 2 x float> @llvm.vp.fadd.nxv2f32(<vscale x 2 x float> [[VEC_IND]], <vscale x 2 x float> shufflevector (<vscale x 2 x float> insertelement (<vscale x 2 x float> poison, float 2.000000e+00, i32 0), <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer), metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP3]])
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[TMP4]] to <vscale x 2 x float>*
-; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x float> @llvm.vp.load.nxv2f32.p0nxv2f32(<vscale x 2 x float>* [[TMP5]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP6]])
-; CHECK-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[VP_OP16:%.*]] = call fast <vscale x 2 x float> @llvm.vp.fmul.nxv2f32(<vscale x 2 x float> [[VP_OP_LOAD]], <vscale x 2 x float> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP7]])
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast float* [[TMP4]] to <vscale x 2 x float>*
-; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv2f32.p0nxv2f32(<vscale x 2 x float> [[VP_OP16]], <vscale x 2 x float>* [[TMP8]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP9]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT:    [[TMP10:%.*]] = uitofp i64 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP11:%.*]] = fmul float [[TMP10]], 2.000000e+00
-; CHECK-NEXT:    [[DOTSPLATINSERT19:%.*]] = insertelement <vscale x 2 x float> poison, float [[TMP11]], i32 0
-; CHECK-NEXT:    [[DOTSPLAT20:%.*]] = shufflevector <vscale x 2 x float> [[DOTSPLATINSERT19]], <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = fadd fast <vscale x 2 x float> [[VEC_IND]], [[DOTSPLAT20]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x float> [ [[INDUCTION12]], [[FOR_BODY_PREHEADER]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP3]], i64 2, i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    [[VP_OP:%.*]] = call fast <vscale x 2 x float> @llvm.vp.fadd.nxv2f32(<vscale x 2 x float> [[VEC_IND]], <vscale x 2 x float> shufflevector (<vscale x 2 x float> insertelement (<vscale x 2 x float> poison, float 2.000000e+00, i32 0), <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer), metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <vscale x 2 x float>*
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x float> @llvm.vp.load.nxv2f32.p0nxv2f32(<vscale x 2 x float>* [[TMP7]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    [[VP_OP23:%.*]] = call fast <vscale x 2 x float> @llvm.vp.fmul.nxv2f32(<vscale x 2 x float> [[VP_OP_LOAD]], <vscale x 2 x float> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP9]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast float* [[TMP6]] to <vscale x 2 x float>*
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2f32.p0nxv2f32(<vscale x 2 x float> [[VP_OP23]], <vscale x 2 x float>* [[TMP10]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP11]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = uitofp i64 [[TMP4]] to float
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul float [[TMP12]], 2.000000e+00
+; CHECK-NEXT:    [[DOTSPLATINSERT26:%.*]] = insertelement <vscale x 2 x float> poison, float [[TMP13]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT27:%.*]] = shufflevector <vscale x 2 x float> [[DOTSPLATINSERT26]], <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = fadd fast <vscale x 2 x float> [[VEC_IND]], [[DOTSPLAT27]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-int.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-induction-int.ll
similarity index 53%
rename from llvm/test/Transforms/LoopVectorize/RISCV/induction-int.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-induction-int.ll
index 1238bc03ec5406a142849c380e7808fec955e03e..81654799dd35df549a2cc50d573ba565ba0a4b20 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-int.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-induction-int.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O3 < %s -o - \
 ; RUN:    | FileCheck %s
 ; ModuleID = 'induction-int.c'
@@ -15,37 +15,37 @@ define dso_local void @induction_float(i32 signext %N, i32* noalias nocapture %a
 ; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    [[STEPVEC_BASE4:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.stepvector.nxv2i32()
-; CHECK-NEXT:    [[SPLAT_STEP_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[X:%.*]], i32 0
-; CHECK-NEXT:    [[SPLAT_STEP_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[SPLAT_STEP_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_SCALED:%.*]] = mul <vscale x 2 x i32> [[STEPVEC_BASE4]], [[SPLAT_STEP_SPLAT]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[X]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT8]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.experimental.stepvector.nxv2i32()
+; CHECK-NEXT:    [[DOTSPLATINSERT10:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT11:%.*]] = shufflevector <vscale x 2 x i32> [[DOTSPLATINSERT10]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <vscale x 2 x i32> [[TMP0]], [[DOTSPLAT11]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[X]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT16:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT15]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i32> [ [[STEPVEC_SCALED]], [[FOR_BODY_PREHEADER]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP0]], i64 2, i64 0)
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[VEC_IND]], <vscale x 2 x i32> [[BROADCAST_SPLAT9]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP2]])
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <vscale x 2 x i32>*
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP4]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP5]])
-; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT:    [[VP_OP16:%.*]] = call <vscale x 2 x i32> @llvm.vp.mul.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP6]])
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP3]] to <vscale x 2 x i32>*
-; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv2i32.p0nxv2i32(<vscale x 2 x i32> [[VP_OP16]], <vscale x 2 x i32>* [[TMP7]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP8]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], [[X]]
-; CHECK-NEXT:    [[DOTSPLATINSERT19:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP10]], i32 0
-; CHECK-NEXT:    [[DOTSPLAT20:%.*]] = shufflevector <vscale x 2 x i32> [[DOTSPLATINSERT19]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[DOTSPLAT20]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP11]], label [[FOR_END]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i32> [ [[TMP1]], [[FOR_BODY_PREHEADER]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP2]], i64 2, i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[VEC_IND]], <vscale x 2 x i32> [[BROADCAST_SPLAT16]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <vscale x 2 x i32>*
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP6]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP7]])
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[VP_OP23:%.*]] = call <vscale x 2 x i32> @llvm.vp.mul.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP5]] to <vscale x 2 x i32>*
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i32.p0nxv2i32(<vscale x 2 x i32> [[VP_OP23]], <vscale x 2 x i32>* [[TMP9]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP10]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], [[X]]
+; CHECK-NEXT:    [[DOTSPLATINSERT26:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP12]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT27:%.*]] = shufflevector <vscale x 2 x i32> [[DOTSPLATINSERT26]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[DOTSPLAT27]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[FOR_END]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/epi-initialize-with-ind-int.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-initialize-with-ind-int.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f7d1d1b10548c8fd9e3f96ebd6cfdf673abd08cd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-initialize-with-ind-int.ll
@@ -0,0 +1,357 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
+; RUN:    -loop-vectorize -vector-register-width-factor=8 < %s -o - | FileCheck %s
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
+; RUN:    -loop-vectorize < %s -o - | FileCheck %s --check-prefix=CHECK1
+; ModuleID = 't2.i'
+source_filename = "t2.i"
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
+target triple = "riscv64-unknown-linux-gnu"
+
+; Function Attrs: nounwind readnone
+define dso_local signext i32 @main(i32 signext %argc, i8** nocapture readnone %argv) local_unnamed_addr #0 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = alloca [1024 x double], align 8
+; CHECK-NEXT:    [[B:%.*]] = alloca [1024 x double], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [1024 x double]* [[A]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast [1024 x double]* [[B]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP1]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = add <vscale x 8 x i64> [[TMP6]], [[DOTSPLAT2]]
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 8 x i64> poison, i64 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT3]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 8 x i64> [[TMP7]], [[DOTSPLAT4]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[DOTSPLAT]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 1, [[TMP10]]
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP11]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT5]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLATINSERT7:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT8:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT7]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = call <vscale x 8 x i32> @llvm.experimental.stepvector.nxv8i32()
+; CHECK-NEXT:    [[DOTSPLATINSERT9:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT10:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT9]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 8 x i32> [[TMP12]], [[DOTSPLAT10]]
+; CHECK-NEXT:    [[DOTSPLATINSERT11:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT12:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT11]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = mul <vscale x 8 x i32> [[TMP13]], [[DOTSPLAT12]]
+; CHECK-NEXT:    [[INDUCTION13:%.*]] = add <vscale x 8 x i32> [[DOTSPLAT8]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 8
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i32 1, [[TMP16]]
+; CHECK-NEXT:    [[DOTSPLATINSERT14:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP17]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT15:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT14]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLATINSERT18:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT19:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT18]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = call <vscale x 8 x i32> @llvm.experimental.stepvector.nxv8i32()
+; CHECK-NEXT:    [[DOTSPLATINSERT20:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT21:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT20]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = add <vscale x 8 x i32> [[TMP18]], [[DOTSPLAT21]]
+; CHECK-NEXT:    [[DOTSPLATINSERT22:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT23:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT22]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[DOTSPLAT23]]
+; CHECK-NEXT:    [[INDUCTION24:%.*]] = add <vscale x 8 x i32> [[DOTSPLAT19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP22:%.*]] = mul i32 [[TMP21]], 8
+; CHECK-NEXT:    [[TMP23:%.*]] = mul i32 1, [[TMP22]]
+; CHECK-NEXT:    [[DOTSPLATINSERT25:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP23]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT26:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT25]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1024, i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND16:%.*]] = phi <vscale x 8 x i32> [ [[INDUCTION13]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND27:%.*]] = phi <vscale x 8 x i32> [ [[INDUCTION24]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT28:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 0
+; CHECK-NEXT:    [[TMP26:%.*]] = add i64 [[TMP25]], 0
+; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 1
+; CHECK-NEXT:    [[TMP28:%.*]] = add i64 [[INDEX]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 0
+; CHECK-NEXT:    [[TMP31:%.*]] = add i64 [[TMP30]], 1
+; CHECK-NEXT:    [[TMP32:%.*]] = mul i64 [[TMP31]], 1
+; CHECK-NEXT:    [[TMP33:%.*]] = add i64 [[INDEX]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP35:%.*]] = mul i64 [[TMP34]], 0
+; CHECK-NEXT:    [[TMP36:%.*]] = add i64 [[TMP35]], 2
+; CHECK-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], 1
+; CHECK-NEXT:    [[TMP38:%.*]] = add i64 [[INDEX]], [[TMP37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP40:%.*]] = mul i64 [[TMP39]], 0
+; CHECK-NEXT:    [[TMP41:%.*]] = add i64 [[TMP40]], 3
+; CHECK-NEXT:    [[TMP42:%.*]] = mul i64 [[TMP41]], 1
+; CHECK-NEXT:    [[TMP43:%.*]] = add i64 [[INDEX]], [[TMP42]]
+; CHECK-NEXT:    [[TMP44:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP45:%.*]] = mul i64 [[TMP44]], 0
+; CHECK-NEXT:    [[TMP46:%.*]] = add i64 [[TMP45]], 4
+; CHECK-NEXT:    [[TMP47:%.*]] = mul i64 [[TMP46]], 1
+; CHECK-NEXT:    [[TMP48:%.*]] = add i64 [[INDEX]], [[TMP47]]
+; CHECK-NEXT:    [[TMP49:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP50:%.*]] = mul i64 [[TMP49]], 0
+; CHECK-NEXT:    [[TMP51:%.*]] = add i64 [[TMP50]], 5
+; CHECK-NEXT:    [[TMP52:%.*]] = mul i64 [[TMP51]], 1
+; CHECK-NEXT:    [[TMP53:%.*]] = add i64 [[INDEX]], [[TMP52]]
+; CHECK-NEXT:    [[TMP54:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP55:%.*]] = mul i64 [[TMP54]], 0
+; CHECK-NEXT:    [[TMP56:%.*]] = add i64 [[TMP55]], 6
+; CHECK-NEXT:    [[TMP57:%.*]] = mul i64 [[TMP56]], 1
+; CHECK-NEXT:    [[TMP58:%.*]] = add i64 [[INDEX]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP60:%.*]] = mul i64 [[TMP59]], 0
+; CHECK-NEXT:    [[TMP61:%.*]] = add i64 [[TMP60]], 7
+; CHECK-NEXT:    [[TMP62:%.*]] = mul i64 [[TMP61]], 1
+; CHECK-NEXT:    [[TMP63:%.*]] = add i64 [[INDEX]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = sitofp <vscale x 8 x i32> [[VEC_IND16]] to <vscale x 8 x double>
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[TMP28]]
+; CHECK-NEXT:    [[TMP66:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP67:%.*]] = mul i32 [[TMP66]], 0
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds double, double* [[TMP65]], i32 [[TMP67]]
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast double* [[TMP68]] to <vscale x 8 x double>*
+; CHECK-NEXT:    store <vscale x 8 x double> [[TMP64]], <vscale x 8 x double>* [[TMP69]], align 8, !tbaa [[TBAA2:![0-9]+]]
+; CHECK-NEXT:    [[TMP70:%.*]] = sub <vscale x 8 x i32> [[BROADCAST_SPLAT]], [[VEC_IND27]]
+; CHECK-NEXT:    [[TMP71:%.*]] = sitofp <vscale x 8 x i32> [[TMP70]] to <vscale x 8 x double>
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[TMP28]]
+; CHECK-NEXT:    [[TMP73:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP74:%.*]] = mul i32 [[TMP73]], 0
+; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds double, double* [[TMP72]], i32 [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = bitcast double* [[TMP75]] to <vscale x 8 x double>*
+; CHECK-NEXT:    store <vscale x 8 x double> [[TMP71]], <vscale x 8 x double>* [[TMP76]], align 8, !tbaa [[TBAA2]]
+; CHECK-NEXT:    [[TMP77:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP78:%.*]] = mul i64 [[TMP77]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP78]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT6]]
+; CHECK-NEXT:    [[VEC_IND_NEXT17]] = add <vscale x 8 x i32> [[VEC_IND16]], [[DOTSPLAT15]]
+; CHECK-NEXT:    [[VEC_IND_NEXT28]] = add <vscale x 8 x i32> [[VEC_IND27]], [[DOTSPLAT26]]
+; CHECK-NEXT:    [[TMP79:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP79]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[ARGC:%.*]] to i64
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[IDXPROM4]]
+; CHECK-NEXT:    [[TMP80:%.*]] = load double, double* [[ARRAYIDX5]], align 8, !tbaa [[TBAA2]]
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[IDXPROM4]]
+; CHECK-NEXT:    [[TMP81:%.*]] = load double, double* [[ARRAYIDX7]], align 8, !tbaa [[TBAA2]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[CONV8:%.*]] = fptosi double [[ADD]] to i32
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP1]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR3]]
+; CHECK-NEXT:    ret i32 [[CONV8]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP82:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP82]] to double
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store double [[CONV]], double* [[ARRAYIDX]], align 8, !tbaa [[TBAA2]]
+; CHECK-NEXT:    [[TMP83:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP84:%.*]] = sub i32 1024, [[TMP83]]
+; CHECK-NEXT:    [[CONV1:%.*]] = sitofp i32 [[TMP84]] to double
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store double [[CONV1]], double* [[ARRAYIDX3]], align 8, !tbaa [[TBAA2]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+;
+; CHECK1-LABEL: @main(
+; CHECK1-NEXT:  entry:
+; CHECK1-NEXT:    [[A:%.*]] = alloca [1024 x double], align 8
+; CHECK1-NEXT:    [[B:%.*]] = alloca [1024 x double], align 8
+; CHECK1-NEXT:    [[TMP0:%.*]] = bitcast [1024 x double]* [[A]] to i8*
+; CHECK1-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR3:[0-9]+]]
+; CHECK1-NEXT:    [[TMP1:%.*]] = bitcast [1024 x double]* [[B]] to i8*
+; CHECK1-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP1]]) #[[ATTR3]]
+; CHECK1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
+; CHECK1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK1:       vector.ph:
+; CHECK1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
+; CHECK1-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP5:%.*]] = add <vscale x 1 x i64> [[TMP4]], [[DOTSPLAT2]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 1 x i64> poison, i64 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT3]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP6:%.*]] = mul <vscale x 1 x i64> [[TMP5]], [[DOTSPLAT4]]
+; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[DOTSPLAT]], [[TMP6]]
+; CHECK1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP7]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK1-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT5]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[DOTSPLATINSERT7:%.*]] = insertelement <vscale x 1 x i32> poison, i32 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT8:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT7]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP9:%.*]] = call <vscale x 1 x i32> @llvm.experimental.stepvector.nxv1i32()
+; CHECK1-NEXT:    [[DOTSPLATINSERT9:%.*]] = insertelement <vscale x 1 x i32> poison, i32 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT10:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT9]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP10:%.*]] = add <vscale x 1 x i32> [[TMP9]], [[DOTSPLAT10]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT11:%.*]] = insertelement <vscale x 1 x i32> poison, i32 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT12:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT11]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP11:%.*]] = mul <vscale x 1 x i32> [[TMP10]], [[DOTSPLAT12]]
+; CHECK1-NEXT:    [[INDUCTION13:%.*]] = add <vscale x 1 x i32> [[DOTSPLAT8]], [[TMP11]]
+; CHECK1-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP13:%.*]] = mul i32 1, [[TMP12]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT14:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP13]], i32 0
+; CHECK1-NEXT:    [[DOTSPLAT15:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT14]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[DOTSPLATINSERT18:%.*]] = insertelement <vscale x 1 x i32> poison, i32 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT19:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT18]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP14:%.*]] = call <vscale x 1 x i32> @llvm.experimental.stepvector.nxv1i32()
+; CHECK1-NEXT:    [[DOTSPLATINSERT20:%.*]] = insertelement <vscale x 1 x i32> poison, i32 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT21:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT20]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i32> [[TMP14]], [[DOTSPLAT21]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT22:%.*]] = insertelement <vscale x 1 x i32> poison, i32 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT23:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT22]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP16:%.*]] = mul <vscale x 1 x i32> [[TMP15]], [[DOTSPLAT23]]
+; CHECK1-NEXT:    [[INDUCTION24:%.*]] = add <vscale x 1 x i32> [[DOTSPLAT19]], [[TMP16]]
+; CHECK1-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP18:%.*]] = mul i32 1, [[TMP17]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT25:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP18]], i32 0
+; CHECK1-NEXT:    [[DOTSPLAT26:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT25]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 1024, i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK1:       vector.body:
+; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK1-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK1-NEXT:    [[VEC_IND16:%.*]] = phi <vscale x 1 x i32> [ [[INDUCTION13]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT17:%.*]], [[VECTOR_BODY]] ]
+; CHECK1-NEXT:    [[VEC_IND27:%.*]] = phi <vscale x 1 x i32> [ [[INDUCTION24]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT28:%.*]], [[VECTOR_BODY]] ]
+; CHECK1-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 0
+; CHECK1-NEXT:    [[TMP21:%.*]] = add i64 [[TMP20]], 0
+; CHECK1-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 1
+; CHECK1-NEXT:    [[TMP23:%.*]] = add i64 [[INDEX]], [[TMP22]]
+; CHECK1-NEXT:    [[TMP24:%.*]] = sitofp <vscale x 1 x i32> [[VEC_IND16]] to <vscale x 1 x double>
+; CHECK1-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[TMP23]]
+; CHECK1-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP27:%.*]] = mul i32 [[TMP26]], 0
+; CHECK1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP25]], i32 [[TMP27]]
+; CHECK1-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    store <vscale x 1 x double> [[TMP24]], <vscale x 1 x double>* [[TMP29]], align 8, !tbaa [[TBAA2:![0-9]+]]
+; CHECK1-NEXT:    [[TMP30:%.*]] = sub <vscale x 1 x i32> [[BROADCAST_SPLAT]], [[VEC_IND27]]
+; CHECK1-NEXT:    [[TMP31:%.*]] = sitofp <vscale x 1 x i32> [[TMP30]] to <vscale x 1 x double>
+; CHECK1-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[TMP23]]
+; CHECK1-NEXT:    [[TMP33:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP34:%.*]] = mul i32 [[TMP33]], 0
+; CHECK1-NEXT:    [[TMP35:%.*]] = getelementptr inbounds double, double* [[TMP32]], i32 [[TMP34]]
+; CHECK1-NEXT:    [[TMP36:%.*]] = bitcast double* [[TMP35]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    store <vscale x 1 x double> [[TMP31]], <vscale x 1 x double>* [[TMP36]], align 8, !tbaa [[TBAA2]]
+; CHECK1-NEXT:    [[TMP37:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP37]]
+; CHECK1-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 1 x i64> [[VEC_IND]], [[DOTSPLAT6]]
+; CHECK1-NEXT:    [[VEC_IND_NEXT17]] = add <vscale x 1 x i32> [[VEC_IND16]], [[DOTSPLAT15]]
+; CHECK1-NEXT:    [[VEC_IND_NEXT28]] = add <vscale x 1 x i32> [[VEC_IND27]], [[DOTSPLAT26]]
+; CHECK1-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK1-NEXT:    br i1 [[TMP38]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK1:       middle.block:
+; CHECK1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK1-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK1:       scalar.ph:
+; CHECK1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK1:       for.cond.cleanup:
+; CHECK1-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[ARGC:%.*]] to i64
+; CHECK1-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[IDXPROM4]]
+; CHECK1-NEXT:    [[TMP39:%.*]] = load double, double* [[ARRAYIDX5]], align 8, !tbaa [[TBAA2]]
+; CHECK1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[IDXPROM4]]
+; CHECK1-NEXT:    [[TMP40:%.*]] = load double, double* [[ARRAYIDX7]], align 8, !tbaa [[TBAA2]]
+; CHECK1-NEXT:    [[ADD:%.*]] = fadd double [[TMP39]], [[TMP40]]
+; CHECK1-NEXT:    [[CONV8:%.*]] = fptosi double [[ADD]] to i32
+; CHECK1-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP1]]) #[[ATTR3]]
+; CHECK1-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR3]]
+; CHECK1-NEXT:    ret i32 [[CONV8]]
+; CHECK1:       for.body:
+; CHECK1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK1-NEXT:    [[TMP41:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP41]] to double
+; CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    store double [[CONV]], double* [[ARRAYIDX]], align 8, !tbaa [[TBAA2]]
+; CHECK1-NEXT:    [[TMP42:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK1-NEXT:    [[TMP43:%.*]] = sub i32 1024, [[TMP42]]
+; CHECK1-NEXT:    [[CONV1:%.*]] = sitofp i32 [[TMP43]] to double
+; CHECK1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    store double [[CONV1]], double* [[ARRAYIDX3]], align 8, !tbaa [[TBAA2]]
+; CHECK1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK1-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+;
+entry:
+  %a = alloca [1024 x double], align 8
+  %b = alloca [1024 x double], align 8
+  %0 = bitcast [1024 x double]* %a to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) #2
+  %1 = bitcast [1024 x double]* %b to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %1) #2
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %idxprom4 = sext i32 %argc to i64
+  %arrayidx5 = getelementptr inbounds [1024 x double], [1024 x double]* %a, i64 0, i64 %idxprom4
+  %2 = load double, double* %arrayidx5, align 8, !tbaa !2
+  %arrayidx7 = getelementptr inbounds [1024 x double], [1024 x double]* %b, i64 0, i64 %idxprom4
+  %3 = load double, double* %arrayidx7, align 8, !tbaa !2
+  %add = fadd double %2, %3
+  %conv8 = fptosi double %add to i32
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %1) #2
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) #2
+  ret i32 %conv8
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %4 = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %4 to double
+  %arrayidx = getelementptr inbounds [1024 x double], [1024 x double]* %a, i64 0, i64 %indvars.iv
+  store double %conv, double* %arrayidx, align 8, !tbaa !2
+  %5 = trunc i64 %indvars.iv to i32
+  %6 = sub i32 1024, %5
+  %conv1 = sitofp i32 %6 to double
+  %arrayidx3 = getelementptr inbounds [1024 x double], [1024 x double]* %b, i64 0, i64 %indvars.iv
+  store double %conv1, double* %arrayidx3, align 8, !tbaa !2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+a,+c,+d,+experimental-v,+f,+m,-relax" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 10.0.0 (git@repo.hca.bsc.es:EPI/System-Software/llvm-mono.git 823a818e0f44dc9b594f14a328ec52b247f3611a)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"double", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/initialize-with-ind-long.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-initialize-with-ind-long.ll
similarity index 58%
rename from llvm/test/Transforms/LoopVectorize/RISCV/initialize-with-ind-long.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-initialize-with-ind-long.ll
index b32f29e6c9fcabeb1482ec39159d65f7bb778de3..ed168d683c5efb7eb56a08a41bf9df799033d005 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/initialize-with-ind-long.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-initialize-with-ind-long.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -vector-register-width-factor=8 < %s -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize < %s -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = 't3.i'
 source_filename = "t3.i"
@@ -14,7 +14,7 @@ define dso_local signext i32 @main(i32 signext %argc, i8** nocapture readnone %a
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = alloca [1024 x i64], align 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [1024 x i64]* [[A]] to i8*
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
@@ -26,72 +26,78 @@ define dso_local signext i32 @main(i32 signext %argc, i8** nocapture readnone %a
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[DOTSPLAT]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 1, [[TMP6]]
-; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
 ; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = add <vscale x 8 x i64> [[TMP5]], [[DOTSPLAT2]]
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 8 x i64> poison, i64 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT3]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 8 x i64> [[TMP6]], [[DOTSPLAT4]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[DOTSPLAT]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 1, [[TMP9]]
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP10]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT5]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 42, i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 0
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 0
-; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], 1
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 1
-; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 0
-; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[TMP19]], 2
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 1
-; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 0
-; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], 3
-; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 1
-; CHECK-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX]], [[TMP26]]
-; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 0
-; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[TMP29]], 4
-; CHECK-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 1
-; CHECK-NEXT:    [[TMP32:%.*]] = add i64 [[INDEX]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP33]], 0
-; CHECK-NEXT:    [[TMP35:%.*]] = add i64 [[TMP34]], 5
-; CHECK-NEXT:    [[TMP36:%.*]] = mul i64 [[TMP35]], 1
-; CHECK-NEXT:    [[TMP37:%.*]] = add i64 [[INDEX]], [[TMP36]]
-; CHECK-NEXT:    [[TMP38:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP39:%.*]] = mul i64 [[TMP38]], 0
-; CHECK-NEXT:    [[TMP40:%.*]] = add i64 [[TMP39]], 6
-; CHECK-NEXT:    [[TMP41:%.*]] = mul i64 [[TMP40]], 1
-; CHECK-NEXT:    [[TMP42:%.*]] = add i64 [[INDEX]], [[TMP41]]
-; CHECK-NEXT:    [[TMP43:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP44:%.*]] = mul i64 [[TMP43]], 0
-; CHECK-NEXT:    [[TMP45:%.*]] = add i64 [[TMP44]], 7
-; CHECK-NEXT:    [[TMP46:%.*]] = mul i64 [[TMP45]], 1
-; CHECK-NEXT:    [[TMP47:%.*]] = add i64 [[INDEX]], [[TMP46]]
-; CHECK-NEXT:    [[TMP48:%.*]] = add nuw nsw <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds [1024 x i64], [1024 x i64]* [[A]], i64 0, i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP50:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP51:%.*]] = mul i32 [[TMP50]], 0
-; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds i64, i64* [[TMP49]], i32 [[TMP51]]
-; CHECK-NEXT:    [[TMP53:%.*]] = bitcast i64* [[TMP52]] to <vscale x 8 x i64>*
-; CHECK-NEXT:    store <vscale x 8 x i64> [[TMP48]], <vscale x 8 x i64>* [[TMP53]], align 8, !tbaa [[TBAA2:![0-9]+]]
-; CHECK-NEXT:    [[TMP54:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP55:%.*]] = mul i64 [[TMP54]], 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP55]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT2]]
-; CHECK-NEXT:    [[TMP56:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP56]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 1
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 1
+; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[INDEX]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 0
+; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[TMP22]], 2
+; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 1
+; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[INDEX]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = add i64 [[TMP27]], 3
+; CHECK-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 1
+; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[INDEX]], [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP32:%.*]] = mul i64 [[TMP31]], 0
+; CHECK-NEXT:    [[TMP33:%.*]] = add i64 [[TMP32]], 4
+; CHECK-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP33]], 1
+; CHECK-NEXT:    [[TMP35:%.*]] = add i64 [[INDEX]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], 0
+; CHECK-NEXT:    [[TMP38:%.*]] = add i64 [[TMP37]], 5
+; CHECK-NEXT:    [[TMP39:%.*]] = mul i64 [[TMP38]], 1
+; CHECK-NEXT:    [[TMP40:%.*]] = add i64 [[INDEX]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP42:%.*]] = mul i64 [[TMP41]], 0
+; CHECK-NEXT:    [[TMP43:%.*]] = add i64 [[TMP42]], 6
+; CHECK-NEXT:    [[TMP44:%.*]] = mul i64 [[TMP43]], 1
+; CHECK-NEXT:    [[TMP45:%.*]] = add i64 [[INDEX]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP47:%.*]] = mul i64 [[TMP46]], 0
+; CHECK-NEXT:    [[TMP48:%.*]] = add i64 [[TMP47]], 7
+; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[TMP48]], 1
+; CHECK-NEXT:    [[TMP50:%.*]] = add i64 [[INDEX]], [[TMP49]]
+; CHECK-NEXT:    [[TMP51:%.*]] = add nuw nsw <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds [1024 x i64], [1024 x i64]* [[A]], i64 0, i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP53:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP54:%.*]] = mul i32 [[TMP53]], 0
+; CHECK-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i64, i64* [[TMP52]], i32 [[TMP54]]
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast i64* [[TMP55]] to <vscale x 8 x i64>*
+; CHECK-NEXT:    store <vscale x 8 x i64> [[TMP51]], <vscale x 8 x i64>* [[TMP56]], align 8, !tbaa [[TBAA2:![0-9]+]]
+; CHECK-NEXT:    [[TMP57:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP57]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP58]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT6]]
+; CHECK-NEXT:    [[TMP59:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP59]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
@@ -101,9 +107,9 @@ define dso_local signext i32 @main(i32 signext %argc, i8** nocapture readnone %a
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[ARGC:%.*]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x i64], [1024 x i64]* [[A]], i64 0, i64 [[IDXPROM]]
-; CHECK-NEXT:    [[TMP57:%.*]] = load i64, i64* [[ARRAYIDX1]], align 8, !tbaa [[TBAA2]]
-; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP57]] to i32
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR4]]
+; CHECK-NEXT:    [[TMP60:%.*]] = load i64, i64* [[ARRAYIDX1]], align 8, !tbaa [[TBAA2]]
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP60]] to i32
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR3]]
 ; CHECK-NEXT:    ret i32 [[CONV]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_07:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
@@ -118,7 +124,7 @@ define dso_local signext i32 @main(i32 signext %argc, i8** nocapture readnone %a
 ; CHECK1-NEXT:  entry:
 ; CHECK1-NEXT:    [[A:%.*]] = alloca [1024 x i64], align 8
 ; CHECK1-NEXT:    [[TMP0:%.*]] = bitcast [1024 x i64]* [[A]] to i8*
-; CHECK1-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR4:[0-9]+]]
+; CHECK1-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR3:[0-9]+]]
 ; CHECK1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -128,35 +134,41 @@ define dso_local signext i32 @main(i32 signext %argc, i8** nocapture readnone %a
 ; CHECK1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 0
 ; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vector.stepvector.nxv1i64()
-; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[DOTSPLAT]], [[STEPVEC_BASE]]
-; CHECK1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
-; CHECK1-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP4]], i32 0
+; CHECK1-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
+; CHECK1-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 0
 ; CHECK1-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP4:%.*]] = add <vscale x 1 x i64> [[TMP3]], [[DOTSPLAT2]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 1 x i64> poison, i64 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT3]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP5:%.*]] = mul <vscale x 1 x i64> [[TMP4]], [[DOTSPLAT4]]
+; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[DOTSPLAT]], [[TMP5]]
+; CHECK1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP7:%.*]] = mul i64 1, [[TMP6]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK1-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT5]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
 ; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 42, i32 0
 ; CHECK1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
 ; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK1:       vector.body:
 ; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK1-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 0
-; CHECK1-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 0
-; CHECK1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 1
-; CHECK1-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]]
-; CHECK1-NEXT:    [[TMP10:%.*]] = add nuw nsw <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [1024 x i64], [1024 x i64]* [[A]], i64 0, i64 [[TMP9]]
-; CHECK1-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 0
-; CHECK1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, i64* [[TMP11]], i32 [[TMP13]]
-; CHECK1-NEXT:    [[TMP15:%.*]] = bitcast i64* [[TMP14]] to <vscale x 1 x i64>*
-; CHECK1-NEXT:    store <vscale x 1 x i64> [[TMP10]], <vscale x 1 x i64>* [[TMP15]], align 8, !tbaa [[TBAA2:![0-9]+]]
-; CHECK1-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP16]]
-; CHECK1-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 1 x i64> [[VEC_IND]], [[DOTSPLAT2]]
-; CHECK1-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK1-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK1-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 0
+; CHECK1-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 0
+; CHECK1-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 1
+; CHECK1-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]]
+; CHECK1-NEXT:    [[TMP13:%.*]] = add nuw nsw <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [1024 x i64], [1024 x i64]* [[A]], i64 0, i64 [[TMP12]]
+; CHECK1-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 0
+; CHECK1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, i64* [[TMP14]], i32 [[TMP16]]
+; CHECK1-NEXT:    [[TMP18:%.*]] = bitcast i64* [[TMP17]] to <vscale x 1 x i64>*
+; CHECK1-NEXT:    store <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64>* [[TMP18]], align 8, !tbaa [[TBAA2:![0-9]+]]
+; CHECK1-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP19]]
+; CHECK1-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 1 x i64> [[VEC_IND]], [[DOTSPLAT6]]
+; CHECK1-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK1-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK1:       middle.block:
 ; CHECK1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK1-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
@@ -166,9 +178,9 @@ define dso_local signext i32 @main(i32 signext %argc, i8** nocapture readnone %a
 ; CHECK1:       for.cond.cleanup:
 ; CHECK1-NEXT:    [[IDXPROM:%.*]] = sext i32 [[ARGC:%.*]] to i64
 ; CHECK1-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x i64], [1024 x i64]* [[A]], i64 0, i64 [[IDXPROM]]
-; CHECK1-NEXT:    [[TMP18:%.*]] = load i64, i64* [[ARRAYIDX1]], align 8, !tbaa [[TBAA2]]
-; CHECK1-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP18]] to i32
-; CHECK1-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR4]]
+; CHECK1-NEXT:    [[TMP21:%.*]] = load i64, i64* [[ARRAYIDX1]], align 8, !tbaa [[TBAA2]]
+; CHECK1-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP21]] to i32
+; CHECK1-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR3]]
 ; CHECK1-NEXT:    ret i32 [[CONV]]
 ; CHECK1:       for.body:
 ; CHECK1-NEXT:    [[I_07:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/intptr-vpred.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-intptr-vpred.ll
similarity index 97%
rename from llvm/test/Transforms/LoopVectorize/RISCV/intptr-vpred.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-intptr-vpred.ll
index 064d51d6cbc4988cebcd3c963a53bd5682f3936c..60bfe24fd2d70aac1214be442c056bfafe4c0473 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/intptr-vpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-intptr-vpred.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './intptr.c'
 source_filename = "./intptr.c"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/mul-reduce-with-epilog.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-mul-reduce-with-epilog.ll
similarity index 98%
rename from llvm/test/Transforms/LoopVectorize/RISCV/mul-reduce-with-epilog.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-mul-reduce-with-epilog.ll
index 68b002d2435dd50dc5aee7d8888e151cc1c544e2..a7e06740cc617b7e8e4b341a8246bdad4d1030ea 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/mul-reduce-with-epilog.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-mul-reduce-with-epilog.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -S \
 ; RUN:   -loop-vectorize -vector-register-width-factor=8 < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:   -prefer-predicate-over-epilogue=scalar-epilogue -S \
 ; RUN:   -loop-vectorize < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './mul-reduce.c'
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/mul-reduce.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-mul-reduce.ll
similarity index 60%
rename from llvm/test/Transforms/LoopVectorize/RISCV/mul-reduce.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-mul-reduce.ll
index 2762eeb468138276dc27cf0a193e65193020dd23..8f170a97edf83fd6cf5ef1139334a70e50f5f57f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/mul-reduce.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-mul-reduce.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:   -prefer-predicate-over-epilogue=predicate-dont-vectorize -S \
 ; RUN:   -loop-vectorize -vector-register-width-factor=8 < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:   -prefer-predicate-over-epilogue=predicate-dont-vectorize -S \
 ; RUN:   -loop-vectorize < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './mul-reduce.c'
@@ -27,58 +27,64 @@ define dso_local signext i32 @mul_reduce(i32 signext %N, i32 signext %c, i32* no
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 1
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP8]], i64 2, i64 2)
-; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i32 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP7]], i32 [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <vscale x 8 x i32>*
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP13]], i32 4, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP14]])
-; CHECK-NEXT:    [[TMP15:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x i32> @llvm.vp.mul.nxv8i32(<vscale x 8 x i32> [[VP_OP_LOAD]], <vscale x 8 x i32> [[VEC_PHI]], <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP15]])
-; CHECK-NEXT:    [[EVL_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = add <vscale x 8 x i64> [[TMP1]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 8 x i64> poison, i64 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT3]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <vscale x 8 x i64> [[TMP2]], [[DOTSPLAT4]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP11]], i64 2, i64 2)
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <vscale x 8 x i32>*
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
+; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT5]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP16]], i32 4, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP17]])
+; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x i32> @llvm.vp.mul.nxv8i32(<vscale x 8 x i32> [[VP_OP_LOAD]], <vscale x 8 x i32> [[VEC_PHI]], <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP18]])
+; CHECK-NEXT:    [[EVL_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP12]], i32 0
 ; CHECK-NEXT:    [[EVL_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[EVL_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
+; CHECK-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[EVL_MASK:%.*]] = icmp ult <vscale x 8 x i64> [[STEP_VEC]], [[EVL_SPLAT]]
-; CHECK-NEXT:    [[TMP16]] = select <vscale x 8 x i1> [[EVL_MASK]], <vscale x 8 x i32> [[VP_OP]], <vscale x 8 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-NEXT:    [[TMP19]] = select <vscale x 8 x i1> [[EVL_MASK]], <vscale x 8 x i32> [[VP_OP]], <vscale x 8 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP12]]
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vector.reduce.mul.nxv8i32(<vscale x 8 x i32> [[TMP16]])
+; CHECK-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.mul.nxv8i32(<vscale x 8 x i32> [[TMP19]])
 ; CHECK-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[WIDE_TRIP_COUNT]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[C]], [[FOR_BODY_PREHEADER]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[C]], [[FOR_BODY_PREHEADER]] ], [ [[TMP23]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[C_ADDR_06:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP21:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[MUL]] = mul nsw i32 [[TMP21]], [[C_ADDR_06]]
+; CHECK-NEXT:    [[TMP24:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[MUL]] = mul nsw i32 [[TMP24]], [[C_ADDR_06]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       for.end.loopexit:
-; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY]] ], [ [[TMP23]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    [[C_ADDR_0_LCSSA:%.*]] = phi i32 [ [[C]], [[ENTRY:%.*]] ], [ [[MUL_LCSSA]], [[FOR_END_LOOPEXIT]] ]
@@ -99,58 +105,64 @@ define dso_local signext i32 @mul_reduce(i32 signext %N, i32 signext %c, i32* no
 ; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK1:       vector.body:
 ; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
+; CHECK1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
 ; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vector.stepvector.nxv2i64()
-; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 0
-; CHECK1-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 0
-; CHECK1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 1
-; CHECK1-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], [[TMP4]]
-; CHECK1-NEXT:    [[TMP6:%.*]] = icmp ule <vscale x 2 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP5]]
-; CHECK1-NEXT:    [[TMP8:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK1-NEXT:    [[TMP9:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP8]], i64 2, i64 0)
-; CHECK1-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP11:%.*]] = mul i32 [[TMP10]], 0
-; CHECK1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP7]], i32 [[TMP11]]
-; CHECK1-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <vscale x 2 x i32>*
-; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 true, i32 0
-; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[DOTSPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP13]], i32 4, <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP14]])
-; CHECK1-NEXT:    [[TMP15:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.mul.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i32> [[VEC_PHI]], <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP15]])
-; CHECK1-NEXT:    [[EVL_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK1-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP2:%.*]] = add <vscale x 2 x i64> [[TMP1]], [[DOTSPLAT]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 2 x i64> poison, i64 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP3:%.*]] = mul <vscale x 2 x i64> [[TMP2]], [[DOTSPLAT4]]
+; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT2]], [[TMP3]]
+; CHECK1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 0
+; CHECK1-NEXT:    [[TMP6:%.*]] = add i64 [[TMP5]], 0
+; CHECK1-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 1
+; CHECK1-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], [[TMP7]]
+; CHECK1-NEXT:    [[TMP9:%.*]] = icmp ule <vscale x 2 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP8]]
+; CHECK1-NEXT:    [[TMP11:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK1-NEXT:    [[TMP12:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP11]], i64 2, i64 0)
+; CHECK1-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP13]], 0
+; CHECK1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP10]], i32 [[TMP14]]
+; CHECK1-NEXT:    [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <vscale x 2 x i32>*
+; CHECK1-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 2 x i1> poison, i1 true, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 2 x i1> [[DOTSPLATINSERT5]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP16]], i32 4, <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP17]])
+; CHECK1-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.mul.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i32> [[VEC_PHI]], <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP18]])
+; CHECK1-NEXT:    [[EVL_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i32 0
 ; CHECK1-NEXT:    [[EVL_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[EVL_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vector.stepvector.nxv2i64()
+; CHECK1-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
 ; CHECK1-NEXT:    [[EVL_MASK:%.*]] = icmp ult <vscale x 2 x i64> [[STEP_VEC]], [[EVL_SPLAT]]
-; CHECK1-NEXT:    [[TMP16]] = select <vscale x 2 x i1> [[EVL_MASK]], <vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i32> [[VEC_PHI]]
-; CHECK1-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 2
-; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
-; CHECK1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK1-NEXT:    [[TMP19]] = select <vscale x 2 x i1> [[EVL_MASK]], <vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i32> [[VEC_PHI]]
+; CHECK1-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 2
+; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP12]]
+; CHECK1-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK1-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK1:       middle.block:
-; CHECK1-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vector.reduce.mul.nxv2i32(<vscale x 2 x i32> [[TMP16]])
+; CHECK1-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.mul.nxv2i32(<vscale x 2 x i32> [[TMP19]])
 ; CHECK1-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK1:       scalar.ph:
 ; CHECK1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[WIDE_TRIP_COUNT]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[C]], [[FOR_BODY_PREHEADER]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[C]], [[FOR_BODY_PREHEADER]] ], [ [[TMP23]], [[MIDDLE_BLOCK]] ]
 ; CHECK1-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK1:       for.body:
 ; CHECK1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK1-NEXT:    [[C_ADDR_06:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MUL:%.*]], [[FOR_BODY]] ]
 ; CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP21:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
-; CHECK1-NEXT:    [[MUL]] = mul nsw i32 [[TMP21]], [[C_ADDR_06]]
+; CHECK1-NEXT:    [[TMP24:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK1-NEXT:    [[MUL]] = mul nsw i32 [[TMP24]], [[C_ADDR_06]]
 ; CHECK1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
+; CHECK1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK1:       for.end.loopexit:
-; CHECK1-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK1-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY]] ], [ [[TMP23]], [[MIDDLE_BLOCK]] ]
 ; CHECK1-NEXT:    br label [[FOR_END]]
 ; CHECK1:       for.end:
 ; CHECK1-NEXT:    [[C_ADDR_0_LCSSA:%.*]] = phi i32 [ [[C]], [[ENTRY:%.*]] ], [ [[MUL_LCSSA]], [[FOR_END_LOOPEXIT]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/no-mem-vec.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-no-mem-vec.ll
similarity index 98%
rename from llvm/test/Transforms/LoopVectorize/RISCV/no-mem-vec.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-no-mem-vec.ll
index aeb7f3d9a9699b9c8d0f4b1b0f8f5f4d7e446ceb..8ea0be41501727d8709f6332745fd49e9b8a145b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/no-mem-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-no-mem-vec.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:   -loop-vectorize -debug-only=loop-vectorize < %s -o - | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reduce-sum.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-reduce-sum.ll
similarity index 95%
rename from llvm/test/Transforms/LoopVectorize/RISCV/reduce-sum.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-reduce-sum.ll
index 0fae778aed674a79e0b3405c252d729364db5384..d10172791ff931a605ae33c49234d8b5f0dee60a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reduce-sum.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-reduce-sum.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './reduce-sum.c'
 source_filename = "./reduce-sum.c"
@@ -16,7 +16,7 @@ define dso_local signext i32 @vec_redsum(i32 signext %N, i32* noalias nocapture
 ; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
+; CHECK-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -35,7 +35,7 @@ define dso_local signext i32 @vec_redsum(i32 signext %N, i32* noalias nocapture
 ; CHECK-NEXT:    [[TMP6]] = select <vscale x 8 x i1> [[EVL_MASK]], <vscale x 8 x i32> [[VP_OP]], <vscale x 8 x i32> [[VEC_PHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP6]])
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -49,7 +49,7 @@ define dso_local signext i32 @vec_redsum(i32 signext %N, i32* noalias nocapture
 ; CHECK1-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK1:       for.body.preheader:
 ; CHECK1-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK1-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vector.stepvector.nxv2i64()
+; CHECK1-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
 ; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK1:       vector.body:
 ; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -68,7 +68,7 @@ define dso_local signext i32 @vec_redsum(i32 signext %N, i32* noalias nocapture
 ; CHECK1-NEXT:    [[TMP6]] = select <vscale x 2 x i1> [[EVL_MASK]], <vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i32> [[VEC_PHI]]
 ; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
 ; CHECK1-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK1-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK1:       middle.block:
 ; CHECK1-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[TMP6]])
 ; CHECK1-NEXT:    br label [[FOR_COND_CLEANUP]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reduction-order-vpred.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-reduction-order-vpred.ll
similarity index 52%
rename from llvm/test/Transforms/LoopVectorize/RISCV/reduction-order-vpred.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-reduction-order-vpred.ll
index 241c35725d8865883d6c17312d9cf1bd23aef9f9..c7d00b57165d35a4203a48546610a1e05947b429 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reduction-order-vpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-reduction-order-vpred.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=riscv64 -mattr=+m,+a,+f,+d,+experimental-v \
+; RUN: opt < %s -mtriple=riscv64 -mattr=+m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -vector-register-width-factor=8 -debug-only=loop-vectorize \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S 2>&1 \
 ; RUN:    | FileCheck %s -check-prefixes=CHECK
-; RUN: opt < %s -mtriple=riscv64 -mattr=+m,+a,+f,+d,+experimental-v \
+; RUN: opt < %s -mtriple=riscv64 -mattr=+m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -debug-only=loop-vectorize \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S 2>&1 \
 ; RUN:    | FileCheck %s -check-prefixes=CHECK1
@@ -20,66 +20,72 @@ define internal i64 @foo(i32* %t0) !prof !1 {
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <vscale x 8 x i32> zeroinitializer, i32 0, i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 8 x i32> zeroinitializer, i32 0, i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 8 x i32> poison, i32 3, i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT4]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 8 x i32> poison, i32 5, i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 8 x i32> poison, i32 3, i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT6]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <vscale x 8 x i32> poison, i32 5, i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT11:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT10]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 8 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 undef, [[INDEX]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP8]], i64 2, i64 2)
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> [[BROADCAST_SPLAT5]], <vscale x 8 x i32> [[VEC_PHI3]], <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP10]])
-; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT:    [[VP_OP10:%.*]] = call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[BROADCAST_SPLAT7]], <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP11]])
-; CHECK-NEXT:    [[EVL_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = add <vscale x 8 x i64> [[TMP2]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 8 x i64> poison, i64 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT3]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <vscale x 8 x i64> [[TMP3]], [[DOTSPLAT4]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sub i64 undef, [[INDEX]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP11]], i64 2, i64 2)
+; CHECK-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
+; CHECK-NEXT:    [[DOTSPLAT9:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT8]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> [[BROADCAST_SPLAT7]], <vscale x 8 x i32> [[VEC_PHI5]], <vscale x 8 x i1> [[DOTSPLAT9]], i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK-NEXT:    [[VP_OP14:%.*]] = call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[BROADCAST_SPLAT11]], <vscale x 8 x i1> [[DOTSPLAT9]], i32 [[TMP14]])
+; CHECK-NEXT:    [[EVL_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP12]], i32 0
 ; CHECK-NEXT:    [[EVL_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[EVL_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
+; CHECK-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[EVL_MASK:%.*]] = icmp ult <vscale x 8 x i64> [[STEP_VEC]], [[EVL_SPLAT]]
-; CHECK-NEXT:    [[TMP12]] = select <vscale x 8 x i1> [[EVL_MASK]], <vscale x 8 x i32> [[VP_OP10]], <vscale x 8 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP13]] = select <vscale x 8 x i1> [[EVL_MASK]], <vscale x 8 x i32> [[VP_OP]], <vscale x 8 x i32> [[VEC_PHI3]]
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
-; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef
-; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof !1, [[LOOP2:!llvm.loop !.*]]
+; CHECK-NEXT:    [[TMP15]] = select <vscale x 8 x i1> [[EVL_MASK]], <vscale x 8 x i32> [[VP_OP14]], <vscale x 8 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP16]] = select <vscale x 8 x i1> [[EVL_MASK]], <vscale x 8 x i32> [[VP_OP]], <vscale x 8 x i32> [[VEC_PHI5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof !1, !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP13]])
-; CHECK-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP12]])
+; CHECK-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP16]])
+; CHECK-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP15]])
 ; CHECK-NEXT:    br i1 true, label [[T17:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[T16:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[T16]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i32 [ 0, [[T16]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[T16]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX15:%.*]] = phi i32 [ 0, [[T16]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[T20:%.*]]
 ; CHECK:       t17:
-; CHECK-NEXT:    [[T18:%.*]] = phi i32 [ [[T24:%.*]], [[T20]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    [[T19:%.*]] = phi i32 [ [[T28:%.*]], [[T20]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[T18:%.*]] = phi i32 [ [[T24:%.*]], [[T20]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[T19:%.*]] = phi i32 [ [[T28:%.*]], [[T20]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[T31:%.*]]
 ; CHECK:       t20:
 ; CHECK-NEXT:    [[T21:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[T29:%.*]], [[T20]] ]
 ; CHECK-NEXT:    [[T22:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[T28]], [[T20]] ]
-; CHECK-NEXT:    [[T23:%.*]] = phi i32 [ [[BC_MERGE_RDX11]], [[SCALAR_PH]] ], [ [[T24]], [[T20]] ]
+; CHECK-NEXT:    [[T23:%.*]] = phi i32 [ [[BC_MERGE_RDX15]], [[SCALAR_PH]] ], [ [[T24]], [[T20]] ]
 ; CHECK-NEXT:    [[T24]] = add i32 3, [[T23]]
 ; CHECK-NEXT:    [[T28]] = add i32 [[T22]], 5
 ; CHECK-NEXT:    [[T29]] = add nuw nsw i64 [[T21]], 1
 ; CHECK-NEXT:    [[T30:%.*]] = icmp eq i64 [[T29]], undef
-; CHECK-NEXT:    br i1 [[T30]], label [[T17]], label [[T20]], !prof !4, [[LOOP5:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[T30]], label [[T17]], label [[T20]], !prof !4, !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       t31:
 ; CHECK-NEXT:    ret i64 undef
 ;
@@ -91,66 +97,72 @@ define internal i64 @foo(i32* %t0) !prof !1 {
 ; CHECK1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK1-NEXT:    [[TMP0:%.*]] = insertelement <vscale x 2 x i32> zeroinitializer, i32 0, i32 0
 ; CHECK1-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 2 x i32> zeroinitializer, i32 0, i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 2 x i32> poison, i32 3, i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT4]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 2 x i32> poison, i32 5, i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 2 x i32> poison, i32 3, i32 0
 ; CHECK1-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <vscale x 2 x i32> poison, i32 5, i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLAT11:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT10]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK1:       vector.body:
 ; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 2 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK1-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 2 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
 ; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vector.stepvector.nxv2i64()
-; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 0
-; CHECK1-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], 0
-; CHECK1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
-; CHECK1-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
-; CHECK1-NEXT:    [[TMP7:%.*]] = icmp ule <vscale x 2 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK1-NEXT:    [[TMP8:%.*]] = sub i64 undef, [[INDEX]]
-; CHECK1-NEXT:    [[TMP9:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP8]], i64 2, i64 0)
-; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 true, i32 0
-; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[DOTSPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[BROADCAST_SPLAT5]], <vscale x 2 x i32> [[VEC_PHI3]], <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP10]])
-; CHECK1-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK1-NEXT:    [[VP_OP10:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 2 x i32> [[BROADCAST_SPLAT7]], <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP11]])
-; CHECK1-NEXT:    [[EVL_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK1-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP3:%.*]] = add <vscale x 2 x i64> [[TMP2]], [[DOTSPLAT]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 2 x i64> poison, i64 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP4:%.*]] = mul <vscale x 2 x i64> [[TMP3]], [[DOTSPLAT4]]
+; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT2]], [[TMP4]]
+; CHECK1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 0
+; CHECK1-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 0
+; CHECK1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 1
+; CHECK1-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]]
+; CHECK1-NEXT:    [[TMP10:%.*]] = icmp ule <vscale x 2 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK1-NEXT:    [[TMP11:%.*]] = sub i64 undef, [[INDEX]]
+; CHECK1-NEXT:    [[TMP12:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP11]], i64 2, i64 0)
+; CHECK1-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <vscale x 2 x i1> poison, i1 true, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT9:%.*]] = shufflevector <vscale x 2 x i1> [[DOTSPLATINSERT8]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[BROADCAST_SPLAT7]], <vscale x 2 x i32> [[VEC_PHI5]], <vscale x 2 x i1> [[DOTSPLAT9]], i32 [[TMP13]])
+; CHECK1-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK1-NEXT:    [[VP_OP14:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 2 x i32> [[BROADCAST_SPLAT11]], <vscale x 2 x i1> [[DOTSPLAT9]], i32 [[TMP14]])
+; CHECK1-NEXT:    [[EVL_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i32 0
 ; CHECK1-NEXT:    [[EVL_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[EVL_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vector.stepvector.nxv2i64()
+; CHECK1-NEXT:    [[STEP_VEC:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
 ; CHECK1-NEXT:    [[EVL_MASK:%.*]] = icmp ult <vscale x 2 x i64> [[STEP_VEC]], [[EVL_SPLAT]]
-; CHECK1-NEXT:    [[TMP12]] = select <vscale x 2 x i1> [[EVL_MASK]], <vscale x 2 x i32> [[VP_OP10]], <vscale x 2 x i32> [[VEC_PHI]]
-; CHECK1-NEXT:    [[TMP13]] = select <vscale x 2 x i1> [[EVL_MASK]], <vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i32> [[VEC_PHI3]]
-; CHECK1-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 2
-; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
-; CHECK1-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef
-; CHECK1-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof !1, [[LOOP2:!llvm.loop !.*]]
+; CHECK1-NEXT:    [[TMP15]] = select <vscale x 2 x i1> [[EVL_MASK]], <vscale x 2 x i32> [[VP_OP14]], <vscale x 2 x i32> [[VEC_PHI]]
+; CHECK1-NEXT:    [[TMP16]] = select <vscale x 2 x i1> [[EVL_MASK]], <vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i32> [[VEC_PHI5]]
+; CHECK1-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 2
+; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP12]]
+; CHECK1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef
+; CHECK1-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof !1, !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK1:       middle.block:
-; CHECK1-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[TMP13]])
-; CHECK1-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[TMP12]])
+; CHECK1-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[TMP16]])
+; CHECK1-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[TMP15]])
 ; CHECK1-NEXT:    br i1 true, label [[T17:%.*]], label [[SCALAR_PH]]
 ; CHECK1:       scalar.ph:
 ; CHECK1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ undef, [[MIDDLE_BLOCK]] ], [ 0, [[T16:%.*]] ]
-; CHECK1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[T16]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
-; CHECK1-NEXT:    [[BC_MERGE_RDX11:%.*]] = phi i32 [ 0, [[T16]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
+; CHECK1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[T16]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
+; CHECK1-NEXT:    [[BC_MERGE_RDX15:%.*]] = phi i32 [ 0, [[T16]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
 ; CHECK1-NEXT:    br label [[T20:%.*]]
 ; CHECK1:       t17:
-; CHECK1-NEXT:    [[T18:%.*]] = phi i32 [ [[T24:%.*]], [[T20]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
-; CHECK1-NEXT:    [[T19:%.*]] = phi i32 [ [[T28:%.*]], [[T20]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK1-NEXT:    [[T18:%.*]] = phi i32 [ [[T24:%.*]], [[T20]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ]
+; CHECK1-NEXT:    [[T19:%.*]] = phi i32 [ [[T28:%.*]], [[T20]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
 ; CHECK1-NEXT:    br label [[T31:%.*]]
 ; CHECK1:       t20:
 ; CHECK1-NEXT:    [[T21:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[T29:%.*]], [[T20]] ]
 ; CHECK1-NEXT:    [[T22:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[T28]], [[T20]] ]
-; CHECK1-NEXT:    [[T23:%.*]] = phi i32 [ [[BC_MERGE_RDX11]], [[SCALAR_PH]] ], [ [[T24]], [[T20]] ]
+; CHECK1-NEXT:    [[T23:%.*]] = phi i32 [ [[BC_MERGE_RDX15]], [[SCALAR_PH]] ], [ [[T24]], [[T20]] ]
 ; CHECK1-NEXT:    [[T24]] = add i32 3, [[T23]]
 ; CHECK1-NEXT:    [[T28]] = add i32 [[T22]], 5
 ; CHECK1-NEXT:    [[T29]] = add nuw nsw i64 [[T21]], 1
 ; CHECK1-NEXT:    [[T30:%.*]] = icmp eq i64 [[T29]], undef
-; CHECK1-NEXT:    br i1 [[T30]], label [[T17]], label [[T20]], !prof !4, [[LOOP5:!llvm.loop !.*]]
+; CHECK1-NEXT:    br i1 [[T30]], label [[T17]], label [[T20]], !prof !4, !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK1:       t31:
 ; CHECK1-NEXT:    ret i64 undef
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-not-parallel-loop.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-scalable-not-parallel-loop.ll
similarity index 97%
rename from llvm/test/Transforms/LoopVectorize/RISCV/scalable-not-parallel-loop.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-scalable-not-parallel-loop.ll
index 57e94e42b239cf8da628484bd053673417fde8b6..d50673af62241a30b919b0015b200ee0eb1d91ae 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-not-parallel-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-scalable-not-parallel-loop.ll
@@ -1,13 +1,13 @@
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize < %s -o - | FileCheck %s --check-prefix=CHECK-IR
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize < %s -o - \
 ; RUN:    | FileCheck %s --check-prefix=CHECK-IR
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -debug-only=loop-vectorize \
 ; RUN:    < %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-DBG
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -debug-only=loop-vectorize \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize < %s -o - \
 ; RUN:    2>&1 | FileCheck %s --check-prefix=CHECK-DBG
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scatter-vpred-force-no-scalarize.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-scatter-vpred-force-no-scalarize.ll
similarity index 69%
rename from llvm/test/Transforms/LoopVectorize/RISCV/scatter-vpred-force-no-scalarize.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-scatter-vpred-force-no-scalarize.ll
index 7baeda111226208e878098296cba7d19422bebbc..d6e71eefe94e60045f02323a6b69443cc3552864 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scatter-vpred-force-no-scalarize.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-scatter-vpred-force-no-scalarize.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s  -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s --check-prefix=CHECK1
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 -riscv-v-vector-bits-min=64 < %s  -o - | FileCheck %s
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -riscv-v-vector-bits-min=64 < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = 'scatter-vpred-no-scalarize.c'
 source_filename = "scatter-vpred-no-scalarize.c"
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
@@ -17,26 +17,26 @@ define dso_local void @simple_scatter(i32 signext %N, double* noalias nocapture
 ; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP2]], i64 3, i64 3)
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64* [[TMP1]] to <vscale x 8 x i64>*
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64.p0nxv8i64(<vscale x 8 x i64>* [[TMP4]], i32 8, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer), i32 [[TMP5]])
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds double, double* [[ISA:%.*]], <vscale x 8 x i64> [[VP_OP_LOAD]]
-; CHECK-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> shufflevector (<vscale x 8 x double> insertelement (<vscale x 8 x double> poison, double 0.000000e+00, i32 0), <vscale x 8 x double> poison, <vscale x 8 x i32> zeroinitializer), <vscale x 8 x double*> [[TMP6]], i32 8, <vscale x 8 x i1> [[TMP0]], i32 [[TMP7]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP3]], i64 3, i64 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[TMP2]] to <vscale x 8 x i64>*
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64.p0nxv8i64(<vscale x 8 x i64>* [[TMP5]], i32 8, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer), i32 [[TMP6]])
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, double* [[ISA:%.*]], <vscale x 8 x i64> [[VP_OP_LOAD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8f64.nxv8p0f64(<vscale x 8 x double> shufflevector (<vscale x 8 x double> insertelement (<vscale x 8 x double> poison, double 0.000000e+00, i32 0), <vscale x 8 x double> poison, <vscale x 8 x i32> zeroinitializer), <vscale x 8 x double*> [[TMP7]], i32 8, <vscale x 8 x i1> [[TMP1]], i32 [[TMP8]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
@@ -49,26 +49,26 @@ define dso_local void @simple_scatter(i32 signext %N, double* noalias nocapture
 ; CHECK1-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
 ; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
 ; CHECK1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vector.stepvector.nxv1i64()
+; CHECK1-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
 ; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK1:       vector.body:
 ; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK1-NEXT:    [[TMP0:%.*]] = icmp ule <vscale x 1 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 [[INDEX]]
-; CHECK1-NEXT:    [[TMP2:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP2]], i64 3, i64 0)
-; CHECK1-NEXT:    [[TMP4:%.*]] = bitcast i64* [[TMP1]] to <vscale x 1 x i64>*
-; CHECK1-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 1 x i64> @llvm.vp.load.nxv1i64.p0nxv1i64(<vscale x 1 x i64>* [[TMP4]], i32 8, <vscale x 1 x i1> shufflevector (<vscale x 1 x i1> insertelement (<vscale x 1 x i1> poison, i1 true, i32 0), <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer), i32 [[TMP5]])
-; CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds double, double* [[ISA:%.*]], <vscale x 1 x i64> [[VP_OP_LOAD]]
-; CHECK1-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK1-NEXT:    call void @llvm.vp.scatter.nxv1f64.nxv1p0f64(<vscale x 1 x double> shufflevector (<vscale x 1 x double> insertelement (<vscale x 1 x double> poison, double 0.000000e+00, i32 0), <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x double*> [[TMP6]], i32 8, <vscale x 1 x i1> [[TMP0]], i32 [[TMP7]])
-; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK1-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[BROADCAST_SPLAT2]], [[TMP0]]
+; CHECK1-NEXT:    [[TMP1:%.*]] = icmp ule <vscale x 1 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 [[INDEX]]
+; CHECK1-NEXT:    [[TMP3:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP3]], i64 3, i64 0)
+; CHECK1-NEXT:    [[TMP5:%.*]] = bitcast i64* [[TMP2]] to <vscale x 1 x i64>*
+; CHECK1-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 1 x i64> @llvm.vp.load.nxv1i64.p0nxv1i64(<vscale x 1 x i64>* [[TMP5]], i32 8, <vscale x 1 x i1> shufflevector (<vscale x 1 x i1> insertelement (<vscale x 1 x i1> poison, i1 true, i32 0), <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer), i32 [[TMP6]])
+; CHECK1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, double* [[ISA:%.*]], <vscale x 1 x i64> [[VP_OP_LOAD]]
+; CHECK1-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK1-NEXT:    call void @llvm.vp.scatter.nxv1f64.nxv1p0f64(<vscale x 1 x double> shufflevector (<vscale x 1 x double> insertelement (<vscale x 1 x double> poison, double 0.000000e+00, i32 0), <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x double*> [[TMP7]], i32 8, <vscale x 1 x i1> [[TMP1]], i32 [[TMP8]])
+; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK1-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK1-NEXT:    br i1 [[TMP9]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK1:       for.cond.cleanup:
 ; CHECK1-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scatter-vpred.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-scatter-vpred.ll
similarity index 72%
rename from llvm/test/Transforms/LoopVectorize/RISCV/scatter-vpred.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-scatter-vpred.ll
index be5896cacc1dbe8fa7a8c29af7fe4e9485340036..bfd98abbdeae79448ce15dcc673557098826ea53 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scatter-vpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-scatter-vpred.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -riscv-v-vector-bits-min=64 < %s  -o - | FileCheck %s
 ; ModuleID = './scatter-vpred.c'
 source_filename = "./scatter-vpred.c"
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
@@ -16,26 +16,26 @@ define dso_local void @simple_add(i32 signext %N, i8* noalias nocapture %isA, i6
 ; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP2]], i64 3, i64 3)
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64* [[TMP1]] to <vscale x 8 x i64>*
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64.p0nxv8i64(<vscale x 8 x i64>* [[TMP4]], i32 8, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer), i32 [[TMP5]])
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[ISA:%.*]], <vscale x 8 x i64> [[VP_OP_LOAD]]
-; CHECK-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i32
-; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i8.nxv8p0i8(<vscale x 8 x i8> shufflevector (<vscale x 8 x i8> insertelement (<vscale x 8 x i8> poison, i8 0, i32 0), <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer), <vscale x 8 x i8*> [[TMP6]], i32 1, <vscale x 8 x i1> [[TMP0]], i32 [[TMP7]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP3]], i64 3, i64 3)
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[TMP2]] to <vscale x 8 x i64>*
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64.p0nxv8i64(<vscale x 8 x i64>* [[TMP5]], i32 8, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer), i32 [[TMP6]])
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, i8* [[ISA:%.*]], <vscale x 8 x i64> [[VP_OP_LOAD]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i8.nxv8p0i8(<vscale x 8 x i8> shufflevector (<vscale x 8 x i8> insertelement (<vscale x 8 x i8> poison, i8 0, i32 0), <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer), <vscale x 8 x i8*> [[TMP7]], i32 1, <vscale x 8 x i1> [[TMP1]], i32 [[TMP8]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-control-flow.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-control-flow.ll
new file mode 100644
index 0000000000000000000000000000000000000000..af6d7c703c3eef9ce8f3ca6c4fab93fd7f0a28c8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-control-flow.ll
@@ -0,0 +1,294 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize -vector-register-width-factor=8 -riscv-v-vector-bits-min=64 < %s -o - \
+; RUN:    | FileCheck %s
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize -riscv-v-vector-bits-min=64 < %s -o - \
+; RUN:    | FileCheck %s --check-prefix=CHECK1
+; ModuleID = './simple-add-control-flow.c'
+source_filename = "./simple-add-control-flow.c"
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
+target triple = "riscv64-unknown-linux-gnu"
+
+; Function Attrs: nofree norecurse nounwind
+define dso_local void @vec_add(i32 signext %N, double* noalias nocapture %c, double* noalias nocapture readonly %a, double* noalias nocapture readonly %b, double* noalias nocapture %d) local_unnamed_addr {
+; CHECK-LABEL: @vec_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP22:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP22]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], 1
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 8 x double> poison, double 3.000000e+00, i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 8 x double> [[BROADCAST_SPLATINSERT7]], <vscale x 8 x double> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <vscale x 8 x i1> poison, i1 false, i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT18:%.*]] = shufflevector <vscale x 8 x i1> [[BROADCAST_SPLATINSERT17]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = add <vscale x 8 x i64> [[TMP0]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 8 x i64> poison, i64 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT3]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <vscale x 8 x i64> [[TMP1]], [[DOTSPLAT4]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP10]], i64 3, i64 3)
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, double* [[TMP9]], i32 [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast double* [[TMP14]] to <vscale x 8 x double>*
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
+; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT5]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x double> @llvm.vp.load.nxv8f64.p0nxv8f64(<vscale x 8 x double>* [[TMP15]], i32 8, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP16]])
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP_FCMP:%.*]] = call <vscale x 8 x i1> @llvm.vp.fcmp.nxv8f64(<vscale x 8 x double> [[VP_OP_LOAD]], <vscale x 8 x double> [[BROADCAST_SPLAT8]], i8 4, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP17]])
+; CHECK-NEXT:    [[TMP18:%.*]] = select <vscale x 8 x i1> [[DOTSPLAT6]], <vscale x 8 x i1> [[VP_OP_FCMP]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP19]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <vscale x 8 x double>*
+; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD13:%.*]] = call <vscale x 8 x double> @llvm.vp.load.nxv8f64.p0nxv8f64(<vscale x 8 x double>* [[TMP23]], i32 8, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP24]])
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP27:%.*]] = mul i32 [[TMP26]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP25]], i32 [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <vscale x 8 x double>*
+; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD16:%.*]] = call <vscale x 8 x double> @llvm.vp.load.nxv8f64.p0nxv8f64(<vscale x 8 x double>* [[TMP29]], i32 8, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP30]])
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <vscale x 8 x i1> [[TMP18]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> undef, i1 true, i32 0), <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP32:%.*]] = select <vscale x 8 x i1> [[TMP8]], <vscale x 8 x i1> [[TMP31]], <vscale x 8 x i1> [[BROADCAST_SPLAT18]]
+; CHECK-NEXT:    [[TMP33:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x double> @llvm.vp.fmul.nxv8f64(<vscale x 8 x double> [[VP_OP_LOAD13]], <vscale x 8 x double> [[VP_OP_LOAD16]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 8 x i1> [[TMP32]], i32 [[TMP33]])
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[D:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP35:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP36:%.*]] = mul i32 [[TMP35]], 0
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds double, double* [[TMP34]], i32 [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast double* [[TMP37]] to <vscale x 8 x double>*
+; CHECK-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8f64.p0nxv8f64(<vscale x 8 x double> [[VP_OP]], <vscale x 8 x double>* [[TMP38]], i32 8, <vscale x 8 x i1> [[TMP32]], i32 [[TMP39]])
+; CHECK-NEXT:    [[TMP40:%.*]] = select <vscale x 8 x i1> [[TMP8]], <vscale x 8 x i1> [[TMP18]], <vscale x 8 x i1> [[BROADCAST_SPLAT18]]
+; CHECK-NEXT:    [[TMP41:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP19:%.*]] = call <vscale x 8 x double> @llvm.vp.fadd.nxv8f64(<vscale x 8 x double> [[VP_OP_LOAD13]], <vscale x 8 x double> [[VP_OP_LOAD16]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 8 x i1> [[TMP40]], i32 [[TMP41]])
+; CHECK-NEXT:    [[TMP42:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP43:%.*]] = mul i32 [[TMP42]], 0
+; CHECK-NEXT:    [[TMP44:%.*]] = getelementptr inbounds double, double* [[TMP9]], i32 [[TMP43]]
+; CHECK-NEXT:    [[TMP45:%.*]] = bitcast double* [[TMP44]] to <vscale x 8 x double>*
+; CHECK-NEXT:    [[TMP46:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8f64.p0nxv8f64(<vscale x 8 x double> [[VP_OP19]], <vscale x 8 x double>* [[TMP45]], i32 8, <vscale x 8 x i1> [[TMP40]], i32 [[TMP46]])
+; CHECK-NEXT:    [[TMP47:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP48:%.*]] = mul i64 [[TMP47]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT:    [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[TMP49]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[WIDE_TRIP_COUNT]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load double, double* [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt double [[TMP50]], 3.000000e+00
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP51:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP52:%.*]] = load double, double* [[ARRAYIDX5]], align 8
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    store double [[ADD]], double* [[ARRAYIDX]], align 8
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP51]], [[TMP52]]
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds double, double* [[D]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store double [[MUL]], double* [[ARRAYIDX13]], align 8
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+;
+; CHECK1-LABEL: @vec_add(
+; CHECK1-NEXT:  entry:
+; CHECK1-NEXT:    [[CMP22:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK1-NEXT:    br i1 [[CMP22]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK1:       for.body.preheader:
+; CHECK1-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK1:       vector.ph:
+; CHECK1-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], 1
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 1 x double> poison, double 3.000000e+00, i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 1 x double> [[BROADCAST_SPLATINSERT7]], <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <vscale x 1 x i1> poison, i1 false, i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLAT18:%.*]] = shufflevector <vscale x 1 x i1> [[BROADCAST_SPLATINSERT17]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK1:       vector.body:
+; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[INDEX]], i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
+; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP1:%.*]] = add <vscale x 1 x i64> [[TMP0]], [[DOTSPLAT]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 1 x i64> poison, i64 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT3]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP2:%.*]] = mul <vscale x 1 x i64> [[TMP1]], [[DOTSPLAT4]]
+; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[BROADCAST_SPLAT2]], [[TMP2]]
+; CHECK1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 0
+; CHECK1-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 0
+; CHECK1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 1
+; CHECK1-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK1-NEXT:    [[TMP8:%.*]] = icmp ule <vscale x 1 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP10:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP10]], i64 3, i64 0)
+; CHECK1-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 0
+; CHECK1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, double* [[TMP9]], i32 [[TMP13]]
+; CHECK1-NEXT:    [[TMP15:%.*]] = bitcast double* [[TMP14]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 1 x i1> [[DOTSPLATINSERT5]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP15]], i32 8, <vscale x 1 x i1> [[DOTSPLAT6]], i32 [[TMP16]])
+; CHECK1-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP_FCMP:%.*]] = call <vscale x 1 x i1> @llvm.vp.fcmp.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[BROADCAST_SPLAT8]], i8 4, <vscale x 1 x i1> [[DOTSPLAT6]], i32 [[TMP17]])
+; CHECK1-NEXT:    [[TMP18:%.*]] = select <vscale x 1 x i1> [[DOTSPLAT6]], <vscale x 1 x i1> [[VP_OP_FCMP]], <vscale x 1 x i1> zeroinitializer
+; CHECK1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 0
+; CHECK1-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP19]], i32 [[TMP21]]
+; CHECK1-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD13:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP23]], i32 8, <vscale x 1 x i1> [[DOTSPLAT6]], i32 [[TMP24]])
+; CHECK1-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP27:%.*]] = mul i32 [[TMP26]], 0
+; CHECK1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP25]], i32 [[TMP27]]
+; CHECK1-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD16:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP29]], i32 8, <vscale x 1 x i1> [[DOTSPLAT6]], i32 [[TMP30]])
+; CHECK1-NEXT:    [[TMP31:%.*]] = xor <vscale x 1 x i1> [[TMP18]], shufflevector (<vscale x 1 x i1> insertelement (<vscale x 1 x i1> undef, i1 true, i32 0), <vscale x 1 x i1> undef, <vscale x 1 x i32> zeroinitializer)
+; CHECK1-NEXT:    [[TMP32:%.*]] = select <vscale x 1 x i1> [[TMP8]], <vscale x 1 x i1> [[TMP31]], <vscale x 1 x i1> [[BROADCAST_SPLAT18]]
+; CHECK1-NEXT:    [[TMP33:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD13]], <vscale x 1 x double> [[VP_OP_LOAD16]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[TMP32]], i32 [[TMP33]])
+; CHECK1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[D:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP35:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP36:%.*]] = mul i32 [[TMP35]], 0
+; CHECK1-NEXT:    [[TMP37:%.*]] = getelementptr inbounds double, double* [[TMP34]], i32 [[TMP36]]
+; CHECK1-NEXT:    [[TMP38:%.*]] = bitcast double* [[TMP37]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP]], <vscale x 1 x double>* [[TMP38]], i32 8, <vscale x 1 x i1> [[TMP32]], i32 [[TMP39]])
+; CHECK1-NEXT:    [[TMP40:%.*]] = select <vscale x 1 x i1> [[TMP8]], <vscale x 1 x i1> [[TMP18]], <vscale x 1 x i1> [[BROADCAST_SPLAT18]]
+; CHECK1-NEXT:    [[TMP41:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP19:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD13]], <vscale x 1 x double> [[VP_OP_LOAD16]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[TMP40]], i32 [[TMP41]])
+; CHECK1-NEXT:    [[TMP42:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP43:%.*]] = mul i32 [[TMP42]], 0
+; CHECK1-NEXT:    [[TMP44:%.*]] = getelementptr inbounds double, double* [[TMP9]], i32 [[TMP43]]
+; CHECK1-NEXT:    [[TMP45:%.*]] = bitcast double* [[TMP44]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[TMP46:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP19]], <vscale x 1 x double>* [[TMP45]], i32 8, <vscale x 1 x i1> [[TMP40]], i32 [[TMP46]])
+; CHECK1-NEXT:    [[TMP47:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; CHECK1-NEXT:    [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK1-NEXT:    br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK1:       middle.block:
+; CHECK1-NEXT:    br i1 true, label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK1:       scalar.ph:
+; CHECK1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[WIDE_TRIP_COUNT]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK1:       for.cond.cleanup.loopexit:
+; CHECK1-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK1:       for.cond.cleanup:
+; CHECK1-NEXT:    ret void
+; CHECK1:       for.body:
+; CHECK1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[C]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    [[TMP49:%.*]] = load double, double* [[ARRAYIDX]], align 8
+; CHECK1-NEXT:    [[CMP1:%.*]] = fcmp olt double [[TMP49]], 3.000000e+00
+; CHECK1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    [[TMP50:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; CHECK1-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    [[TMP51:%.*]] = load double, double* [[ARRAYIDX5]], align 8
+; CHECK1-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK1:       if.then:
+; CHECK1-NEXT:    [[ADD:%.*]] = fadd double [[TMP50]], [[TMP51]]
+; CHECK1-NEXT:    store double [[ADD]], double* [[ARRAYIDX]], align 8
+; CHECK1-NEXT:    br label [[FOR_INC]]
+; CHECK1:       if.else:
+; CHECK1-NEXT:    [[MUL:%.*]] = fmul double [[TMP50]], [[TMP51]]
+; CHECK1-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds double, double* [[D]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    store double [[MUL]], double* [[ARRAYIDX13]], align 8
+; CHECK1-NEXT:    br label [[FOR_INC]]
+; CHECK1:       for.inc:
+; CHECK1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK1-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+;
+entry:
+  %cmp22 = icmp sgt i32 %N, 0
+  br i1 %cmp22, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.inc, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]
+  %arrayidx = getelementptr inbounds double, double* %c, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %cmp1 = fcmp olt double %0, 3.000000e+00
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  %1 = load double, double* %arrayidx3, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %b, i64 %indvars.iv
+  %2 = load double, double* %arrayidx5, align 8
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:                                          ; preds = %for.body
+  %add = fadd double %1, %2
+  store double %add, double* %arrayidx, align 8
+  br label %for.inc
+
+if.else:                                          ; preds = %for.body
+  %mul = fmul double %1, %2
+  %arrayidx13 = getelementptr inbounds double, double* %d, i64 %indvars.iv
+  store double %mul, double* %arrayidx13, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.else
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-double-vpred.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-double-vpred.ll
similarity index 97%
rename from llvm/test/Transforms/LoopVectorize/RISCV/simple-add-double-vpred.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-double-vpred.ll
index 751fa7951a9d87e5a3b2386690c4460d0a63d046..25a0e1ac5371fd14c4ccbc2869d2d00172f96679 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-double-vpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-double-vpred.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s -o - | FileCheck %s
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 -riscv-v-vector-bits-min=64 < %s -o - | FileCheck %s
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s -o - | FileCheck %s --check-prefix=CHECK1
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -riscv-v-vector-bits-min=64 < %s -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './simple-add-double.c'
 source_filename = "./simple-add-double.c"
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-float-vpred.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-float-vpred.ll
similarity index 97%
rename from llvm/test/Transforms/LoopVectorize/RISCV/simple-add-float-vpred.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-float-vpred.ll
index 0b9aae7b7d89940d0e1e4a74702b64a3b2255ed6..5d6f40c9f3d50988a80b7ed5cbec1cabadb18ce3 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-float-vpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-float-vpred.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s -o - | FileCheck %s
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 -riscv-v-vector-bits-min=64 < %s -o - | FileCheck %s
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s -o - | FileCheck %s --check-prefix=CHECK1
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -riscv-v-vector-bits-min=64 < %s -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './simple-add-float.c'
 source_filename = "./simple-add-float.c"
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-reverse.ll
similarity index 98%
rename from llvm/test/Transforms/LoopVectorize/RISCV/simple-add-reverse.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-reverse.ll
index 62e1e28c7fc70df1d920fbb0bd22115db904a5df..6d4923e93453a4623574a6ca84b0e79aa63a8d40 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-reverse.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -dce \
-; RUN:    -loop-vectorize -vector-register-width-factor=8 -instcombine | FileCheck %s
+; RUN:    -loop-vectorize -vector-register-width-factor=8 -instcombine -riscv-v-vector-bits-min=64 | FileCheck %s
 ; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -dce \
-; RUN:    -loop-vectorize -instcombine | FileCheck %s --check-prefix=CHECK1
+; RUN:    -loop-vectorize -instcombine -riscv-v-vector-bits-min=64 | FileCheck %s --check-prefix=CHECK1
 
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
 target triple = "riscv64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-vpred.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-vpred.ll
similarity index 52%
rename from llvm/test/Transforms/LoopVectorize/RISCV/simple-add-vpred.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-vpred.ll
index 14cbcbff09c428a35ac023da21460c103dd80be9..78c36ad2ed96b93cc1b83b3ad2f486a4db806837 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-vpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-add-vpred.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize -vector-register-width-factor=8 < %s -o - \
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize -vector-register-width-factor=8 -riscv-v-vector-bits-min=64 < %s -o - \
 ; RUN:    | FileCheck %s
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize < %s -o - \
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize -riscv-v-vector-bits-min=64 < %s -o - \
 ; RUN:    | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './simple-add.c'
 source_filename = "./simple-add.c"
@@ -28,46 +28,52 @@ define dso_local void @simple_add(i32 signext %N, i32* noalias nocapture %c, i32
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP7]], i64 2, i64 2)
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP6]], i32 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 8 x i32>*
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP12]], i32 4, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP13]])
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 0
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP14]], i32 [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32* [[TMP17]] to <vscale x 8 x i32>*
-; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD5:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP18]], i32 4, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP19]])
-; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> [[VP_OP_LOAD5]], <vscale x 8 x i32> [[VP_OP_LOAD]], <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP20]])
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 0
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <vscale x 8 x i32>*
-; CHECK-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv8i32.p0nxv8i32(<vscale x 8 x i32> [[VP_OP]], <vscale x 8 x i32>* [[TMP25]], i32 4, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP26]])
-; CHECK-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
-; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = add <vscale x 8 x i64> [[TMP0]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 8 x i64> poison, i64 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT3]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <vscale x 8 x i64> [[TMP1]], [[DOTSPLAT4]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP10]], i64 2, i64 2)
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <vscale x 8 x i32>*
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
+; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT5]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP15]], i32 4, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP16]])
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP19:%.*]] = mul i32 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP17]], i32 [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <vscale x 8 x i32>*
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD9:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP21]], i32 4, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP22]])
+; CHECK-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> [[VP_OP_LOAD9]], <vscale x 8 x i32> [[VP_OP_LOAD]], <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP23]])
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP26:%.*]] = mul i32 [[TMP25]], 0
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i32 [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i32* [[TMP27]] to <vscale x 8 x i32>*
+; CHECK-NEXT:    [[TMP29:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i32.p0nxv8i32(<vscale x 8 x i32> [[VP_OP]], <vscale x 8 x i32>* [[TMP28]], i32 4, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP29]])
+; CHECK-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -76,15 +82,15 @@ define dso_local void @simple_add(i32 signext %N, i32* noalias nocapture %c, i32
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP30:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP33:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP31:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP31]], [[TMP30]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP34]], [[TMP33]]
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX4]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       for.end.loopexit:
 ; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
@@ -106,46 +112,52 @@ define dso_local void @simple_add(i32 signext %N, i32* noalias nocapture %c, i32
 ; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vector.stepvector.nxv2i64()
-; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 0
-; CHECK1-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 0
-; CHECK1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 1
-; CHECK1-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK1-NEXT:    [[TMP5:%.*]] = icmp ule <vscale x 2 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP7:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK1-NEXT:    [[TMP8:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP7]], i64 2, i64 0)
-; CHECK1-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 0
-; CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP6]], i32 [[TMP10]]
-; CHECK1-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 2 x i32>*
-; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 true, i32 0
-; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[DOTSPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP12]], i32 4, <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP13]])
-; CHECK1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 0
-; CHECK1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP14]], i32 [[TMP16]]
-; CHECK1-NEXT:    [[TMP18:%.*]] = bitcast i32* [[TMP17]] to <vscale x 2 x i32>*
-; CHECK1-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD5:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP18]], i32 4, <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP19]])
-; CHECK1-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD5]], <vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP20]])
-; CHECK1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 0
-; CHECK1-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 [[TMP23]]
-; CHECK1-NEXT:    [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <vscale x 2 x i32>*
-; CHECK1-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    call void @llvm.vp.store.nxv2i32.p0nxv2i32(<vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i32>* [[TMP25]], i32 4, <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP26]])
-; CHECK1-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 2
-; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
-; CHECK1-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK1-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP1:%.*]] = add <vscale x 2 x i64> [[TMP0]], [[DOTSPLAT]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 2 x i64> poison, i64 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP2:%.*]] = mul <vscale x 2 x i64> [[TMP1]], [[DOTSPLAT4]]
+; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT2]], [[TMP2]]
+; CHECK1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 0
+; CHECK1-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 0
+; CHECK1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 1
+; CHECK1-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK1-NEXT:    [[TMP8:%.*]] = icmp ule <vscale x 2 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP10:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP10]], i64 2, i64 0)
+; CHECK1-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 0
+; CHECK1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 [[TMP13]]
+; CHECK1-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <vscale x 2 x i32>*
+; CHECK1-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 2 x i1> poison, i1 true, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 2 x i1> [[DOTSPLATINSERT5]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP15]], i32 4, <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP16]])
+; CHECK1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP19:%.*]] = mul i32 [[TMP18]], 0
+; CHECK1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP17]], i32 [[TMP19]]
+; CHECK1-NEXT:    [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <vscale x 2 x i32>*
+; CHECK1-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD9:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP21]], i32 4, <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP22]])
+; CHECK1-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD9]], <vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP23]])
+; CHECK1-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP26:%.*]] = mul i32 [[TMP25]], 0
+; CHECK1-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i32 [[TMP26]]
+; CHECK1-NEXT:    [[TMP28:%.*]] = bitcast i32* [[TMP27]] to <vscale x 2 x i32>*
+; CHECK1-NEXT:    [[TMP29:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    call void @llvm.vp.store.nxv2i32.p0nxv2i32(<vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i32>* [[TMP28]], i32 4, <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP29]])
+; CHECK1-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 2
+; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; CHECK1-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK1-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK1:       middle.block:
 ; CHECK1-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK1:       scalar.ph:
@@ -154,15 +166,15 @@ define dso_local void @simple_add(i32 signext %N, i32* noalias nocapture %c, i32
 ; CHECK1:       for.body:
 ; CHECK1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP30:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK1-NEXT:    [[TMP33:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
 ; CHECK1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP31:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
-; CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP31]], [[TMP30]]
+; CHECK1-NEXT:    [[TMP34:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP34]], [[TMP33]]
 ; CHECK1-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]]
 ; CHECK1-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX4]], align 4
 ; CHECK1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
+; CHECK1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK1:       for.end.loopexit:
 ; CHECK1-NEXT:    br label [[FOR_END]]
 ; CHECK1:       for.end:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/simple-compare-double-ri1.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-compare-double-ri1.ll
similarity index 98%
rename from llvm/test/Transforms/LoopVectorize/RISCV/simple-compare-double-ri1.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-compare-double-ri1.ll
index 02e88df4e6fb18d47a5630876ad0f40862b7eb2e..d4bde59632cc3e078b3f4adca6795207c2704389 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/simple-compare-double-ri1.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-compare-double-ri1.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s -o - | FileCheck %s
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 -riscv-v-vector-bits-min=64 < %s -o - | FileCheck %s
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s -o - | FileCheck %s --check-prefix=CHECK1
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -riscv-v-vector-bits-min=64 < %s -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './simple-compare-double-ri1.c'
 source_filename = "./simple-compare-double-ri1.c"
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/simple-compare-ri1.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-compare-ri1.ll
similarity index 98%
rename from llvm/test/Transforms/LoopVectorize/RISCV/simple-compare-ri1.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-compare-ri1.ll
index cb7a368575b679ecd2918b58f913e4d99714e96d..1b2192cc967eeb766d7180f039d50032c81f481e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/simple-compare-ri1.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-compare-ri1.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 < %s  -o - | FileCheck %s
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -vector-register-width-factor=8 -riscv-v-vector-bits-min=64 < %s  -o - | FileCheck %s
 ; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s  -o - | FileCheck %s --check-prefix=CHECK1
+; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 -riscv-v-vector-bits-min=64 < %s  -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './simple-compare-ri1.c'
 source_filename = "./simple-compare-ri1.c"
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/simple-vplan.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-vplan.ll
similarity index 50%
rename from llvm/test/Transforms/LoopVectorize/RISCV/simple-vplan.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-vplan.ll
index 5f742ead293e96279d5b2f9cf3224573ea3835fc..8fbe369ac9a2aecf22766b6fee59309b0d301d75 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/simple-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-simple-vplan.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize -vector-register-width-factor=8 \
 ; RUN: -debug-only=loop-vectorize -disable-output -print-after=loop-vectorize 2>&1 | FileCheck %s
-; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize \
 ; RUN: -debug-only=loop-vectorize -disable-output -print-after=loop-vectorize 2>&1 | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = './simple-add.c'
@@ -11,20 +11,20 @@ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:12
 target triple = "riscv64-unknown-linux-gnu"
 
 ; CHECK-LABEL: simple_add
-; CHECK:       N0 [label =
-; CHECK-NEXT:     "for.body:\n" +
-; CHECK-NEXT:       "WIDEN-INDUCTION %indvars.iv = phi 0, %indvars.iv.next\l" +
-; CHECK-NEXT:       "EMIT vp<%2> = icmp ule ir<%indvars.iv> vp<%0>\l" +
-; CHECK-NEXT:       "CLONE ir<%arrayidx> = getelementptr ir<%a>, ir<%indvars.iv>\l" +
-; CHECK-NEXT:       "EMIT vp<%4> = GENERATE-EXPLICIT-VECTOR-LENGTH\l" +
-; CHECK-NEXT:       "PREDICATED-WIDEN ir<%0> = load ir<%arrayidx>, vp<%4>, vp<%2> (ALL-ONES-MASK)\l" +
-; CHECK-NEXT:       "CLONE ir<%arrayidx2> = getelementptr ir<%b>, ir<%indvars.iv>\l" +
-; CHECK-NEXT:       "PREDICATED-WIDEN ir<%1> = load ir<%arrayidx2>, vp<%4>, vp<%2> (ALL-ONES-MASK)\l" +
-; CHECK-NEXT:       "PREDICATED-WIDEN ir<%add> = add ir<%1>, ir<%0>, vp<%4>, vp<%2> (ALL-ONES-MASK)\l" +
-; CHECK-NEXT:       "CLONE ir<%arrayidx4> = getelementptr ir<%c>, ir<%indvars.iv>\l" +
-; CHECK-NEXT:       "PREDICATED-WIDEN store ir<%arrayidx4>, ir<%add>, vp<%4>, vp<%2> (ALL-ONES-MASK)\l"
-; CHECK-NEXT:  ]
-
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4,vscale x 8,vscale x 16},UF>=1' {
+; CHECK-NEXT: for.body:
+; CHECK-NEXT:   WIDEN-INDUCTION %indvars.iv = phi 0, %indvars.iv.next
+; CHECK-NEXT:   EMIT vp<%2> = icmp ule ir<%indvars.iv> vp<%0>
+; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr ir<%a>, ir<%indvars.iv>
+; CHECK-NEXT: EMIT vp<%4> = GENERATE-EXPLICIT-VECTOR-LENGTH
+; CHECK-NEXT: PREDICATED-WIDEN ir<%0> = load ir<%arrayidx>, vp<%4>, vp<%2> (ALL-ONES-MASK)
+; CHECK-NEXT:   CLONE ir<%arrayidx2> = getelementptr ir<%b>, ir<%indvars.iv>
+; CHECK-NEXT: PREDICATED-WIDEN ir<%1> = load ir<%arrayidx2>, vp<%4>, vp<%2> (ALL-ONES-MASK)
+; CHECK-NEXT: PREDICATED-WIDEN ir<%add> = add ir<%1>, ir<%0>, vp<%4>, vp<%2> (ALL-ONES-MASK)
+; CHECK-NEXT:   CLONE ir<%arrayidx4> = getelementptr ir<%c>, ir<%indvars.iv>
+; CHECK-NEXT: PREDICATED-WIDEN store ir<%arrayidx4>, ir<%add>, vp<%4>, vp<%2> (ALL-ONES-MASK)
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
 
 ; Function Attrs: nofree norecurse nounwind
 define dso_local void @simple_add(i32 signext %N, i32* noalias nocapture %c, i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b) local_unnamed_addr {
@@ -44,46 +44,52 @@ define dso_local void @simple_add(i32 signext %N, i32* noalias nocapture %c, i32
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP7]], i64 2, i64 2)
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP6]], i32 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 8 x i32>*
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP12]], i32 4, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP13]])
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 0
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP14]], i32 [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32* [[TMP17]] to <vscale x 8 x i32>*
-; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD5:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP18]], i32 4, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP19]])
-; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> [[VP_OP_LOAD5]], <vscale x 8 x i32> [[VP_OP_LOAD]], <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP20]])
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 0
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <vscale x 8 x i32>*
-; CHECK-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv8i32.p0nxv8i32(<vscale x 8 x i32> [[VP_OP]], <vscale x 8 x i32>* [[TMP25]], i32 4, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP26]])
-; CHECK-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
-; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = add <vscale x 8 x i64> [[TMP0]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 8 x i64> poison, i64 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT3]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <vscale x 8 x i64> [[TMP1]], [[DOTSPLAT4]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP10]], i64 2, i64 2)
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <vscale x 8 x i32>*
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
+; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT5]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP15]], i32 4, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP16]])
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP19:%.*]] = mul i32 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP17]], i32 [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <vscale x 8 x i32>*
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD9:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0nxv8i32(<vscale x 8 x i32>* [[TMP21]], i32 4, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP22]])
+; CHECK-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> [[VP_OP_LOAD9]], <vscale x 8 x i32> [[VP_OP_LOAD]], <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP23]])
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP26:%.*]] = mul i32 [[TMP25]], 0
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i32 [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i32* [[TMP27]] to <vscale x 8 x i32>*
+; CHECK-NEXT:    [[TMP29:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i32.p0nxv8i32(<vscale x 8 x i32> [[VP_OP]], <vscale x 8 x i32>* [[TMP28]], i32 4, <vscale x 8 x i1> [[DOTSPLAT6]], i32 [[TMP29]])
+; CHECK-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -92,15 +98,15 @@ define dso_local void @simple_add(i32 signext %N, i32* noalias nocapture %c, i32
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP30:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP33:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP31:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP31]], [[TMP30]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP34]], [[TMP33]]
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX4]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       for.end.loopexit:
 ; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
@@ -122,46 +128,52 @@ define dso_local void @simple_add(i32 signext %N, i32* noalias nocapture %c, i32
 ; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[INDEX]], i32 0
 ; CHECK1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vector.stepvector.nxv2i64()
-; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 0
-; CHECK1-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 0
-; CHECK1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 1
-; CHECK1-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK1-NEXT:    [[TMP5:%.*]] = icmp ule <vscale x 2 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP7:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK1-NEXT:    [[TMP8:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP7]], i64 2, i64 0)
-; CHECK1-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 0
-; CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP6]], i32 [[TMP10]]
-; CHECK1-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <vscale x 2 x i32>*
-; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 true, i32 0
-; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[DOTSPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK1-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP12]], i32 4, <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP13]])
-; CHECK1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 0
-; CHECK1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP14]], i32 [[TMP16]]
-; CHECK1-NEXT:    [[TMP18:%.*]] = bitcast i32* [[TMP17]] to <vscale x 2 x i32>*
-; CHECK1-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD5:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP18]], i32 4, <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP19]])
-; CHECK1-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD5]], <vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP20]])
-; CHECK1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 0
-; CHECK1-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 [[TMP23]]
-; CHECK1-NEXT:    [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <vscale x 2 x i32>*
-; CHECK1-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    call void @llvm.vp.store.nxv2i32.p0nxv2i32(<vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i32>* [[TMP25]], i32 4, <vscale x 2 x i1> [[DOTSPLAT]], i32 [[TMP26]])
-; CHECK1-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 2
-; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
-; CHECK1-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK1-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP1:%.*]] = add <vscale x 2 x i64> [[TMP0]], [[DOTSPLAT]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 2 x i64> poison, i64 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP2:%.*]] = mul <vscale x 2 x i64> [[TMP1]], [[DOTSPLAT4]]
+; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT2]], [[TMP2]]
+; CHECK1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 0
+; CHECK1-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 0
+; CHECK1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 1
+; CHECK1-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK1-NEXT:    [[TMP8:%.*]] = icmp ule <vscale x 2 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
+; CHECK1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP10:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
+; CHECK1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP10]], i64 2, i64 0)
+; CHECK1-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 0
+; CHECK1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 [[TMP13]]
+; CHECK1-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <vscale x 2 x i32>*
+; CHECK1-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 2 x i1> poison, i1 true, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 2 x i1> [[DOTSPLATINSERT5]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP15]], i32 4, <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP16]])
+; CHECK1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP19:%.*]] = mul i32 [[TMP18]], 0
+; CHECK1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP17]], i32 [[TMP19]]
+; CHECK1-NEXT:    [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <vscale x 2 x i32>*
+; CHECK1-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD9:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0nxv2i32(<vscale x 2 x i32>* [[TMP21]], i32 4, <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP22]])
+; CHECK1-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 2 x i32> @llvm.vp.add.nxv2i32(<vscale x 2 x i32> [[VP_OP_LOAD9]], <vscale x 2 x i32> [[VP_OP_LOAD]], <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP23]])
+; CHECK1-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 [[TMP7]]
+; CHECK1-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP26:%.*]] = mul i32 [[TMP25]], 0
+; CHECK1-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i32 [[TMP26]]
+; CHECK1-NEXT:    [[TMP28:%.*]] = bitcast i32* [[TMP27]] to <vscale x 2 x i32>*
+; CHECK1-NEXT:    [[TMP29:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK1-NEXT:    call void @llvm.vp.store.nxv2i32.p0nxv2i32(<vscale x 2 x i32> [[VP_OP]], <vscale x 2 x i32>* [[TMP28]], i32 4, <vscale x 2 x i1> [[DOTSPLAT6]], i32 [[TMP29]])
+; CHECK1-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 2
+; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]]
+; CHECK1-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK1-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK1:       middle.block:
 ; CHECK1-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK1:       scalar.ph:
@@ -170,15 +182,15 @@ define dso_local void @simple_add(i32 signext %N, i32* noalias nocapture %c, i32
 ; CHECK1:       for.body:
 ; CHECK1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP30:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK1-NEXT:    [[TMP33:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
 ; CHECK1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP31:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
-; CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP31]], [[TMP30]]
+; CHECK1-NEXT:    [[TMP34:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP34]], [[TMP33]]
 ; CHECK1-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDVARS_IV]]
 ; CHECK1-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX4]], align 4
 ; CHECK1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
+; CHECK1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK1:       for.end.loopexit:
 ; CHECK1-NEXT:    br label [[FOR_END]]
 ; CHECK1:       for.end:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/spmv-no-prefer-predicate-over-epilog.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-spmv-no-prefer-predicate-over-epilog.ll
similarity index 99%
rename from llvm/test/Transforms/LoopVectorize/RISCV/spmv-no-prefer-predicate-over-epilog.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-spmv-no-prefer-predicate-over-epilog.ll
index e7867325c92e055b07414d939cdded824df85ef5..c8961eed7034bce36c1db92da8b295c34e510117 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/spmv-no-prefer-predicate-over-epilog.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-spmv-no-prefer-predicate-over-epilog.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -O3 < %s  -o - | FileCheck %s
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -O3 -riscv-v-vector-bits-min=64 < %s  -o - | FileCheck %s
 ; ModuleID = './spmv.c'
 source_filename = "./spmv.c"
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/epi-vec-reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-vec-reg-usage.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f511716860ff5f9c69e4d7d08c75e75d72ef857f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-vec-reg-usage.ll
@@ -0,0 +1,388 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
+; RUN:   -prefer-predicate-over-epilogue=predicate-dont-vectorize -S \
+; RUN:   -loop-vectorize -vector-register-width-factor=8 -debug-only=loop-vectorize 2>&1 \
+; RUN:   | FileCheck %s
+; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
+; RUN:   -prefer-predicate-over-epilogue=predicate-dont-vectorize -S \
+; RUN:   -loop-vectorize -debug-only=loop-vectorize 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=CHECK1
+
+;Check that register usage for VF=8 is >32, thus selecting MaxVF=4.
+;void highRegUage(int N, double *restrict c, double *restrict a,
+;                 double *restrict b, double *restrict e, double *restrict f,
+;                 double *restrict g) {
+;  int i;
+;  for (i = 1; i < N + 1; i++) {
+;    c[i] = a[i] + b[i];
+;    e[i] = a[i] - b[i] + c[i];
+;    f[i] = a[i] * b[i] + c[i] - e[i];
+;    g[i] = a[i] * c[i] + b[i] + e[i] - f[i];
+;  }
+;}
+
+; CHECK-LABEL: highRegUage
+; CHECK: LV: The Smallest and Widest types: 64 / 64 bits.
+; CHECK: LV: The Widest register safe to use is: 512 bits.
+; CHECK: LV(REG): Calculating max register usage:
+; CHECK: LV(REG): VF = vscale x 2
+; CHECK-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 10 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 1 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 2 registers
+; CHECK-NEXT: LV(REG): VF = vscale x 4
+; CHECK-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 20 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 1 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 4 registers
+; CHECK-NEXT: LV(REG): VF = vscale x 8
+; CHECK-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 40 registers
+; CHECK-NEXT: LV(REG): Found invariant usage: 1 item
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 8 registers
+; CHECK: LV: Selecting VF: vscale x 4.
+;
+; ModuleID = './vec-reg-usage.c'
+source_filename = "./vec-reg-usage.c"
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
+target triple = "riscv64-unknown-linux-gnu"
+
+; Function Attrs: nofree norecurse nounwind
+define dso_local void @highRegUage(i32 signext %N, double* noalias nocapture %c, double* noalias nocapture readonly %a, double* noalias nocapture readonly %b, double* noalias nocapture %e, double* noalias nocapture %f, double* noalias nocapture %g) local_unnamed_addr {
+; CHECK-LABEL: @highRegUage(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP_NOT75:%.*]] = icmp slt i32 [[N:%.*]], 1
+; CHECK-NEXT:    br i1 [[CMP_NOT75]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i32 [[N]], 1
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[IND_END:%.*]] = add i64 1, [[TMP1]]
+; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 0, i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = add <vscale x 4 x i64> [[TMP2]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 1, i32 0
+; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <vscale x 4 x i64> [[TMP3]], [[DOTSPLAT4]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], [[TMP8]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[STEPVECTOR:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[VEC_IV:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT6]], [[STEPVECTOR]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ule <vscale x 4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[TMP1]], [[INDEX]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP12]], i64 3, i64 2)
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[TMP11]], i32 [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <vscale x 4 x double>*
+; CHECK-NEXT:    [[DOTSPLATINSERT7:%.*]] = insertelement <vscale x 4 x i1> poison, i1 true, i32 0
+; CHECK-NEXT:    [[DOTSPLAT8:%.*]] = shufflevector <vscale x 4 x i1> [[DOTSPLATINSERT7]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x double> @llvm.vp.load.nxv4f64.p0nxv4f64(<vscale x 4 x double>* [[TMP17]], i32 8, <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP18]])
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP19]], i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <vscale x 4 x double>*
+; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP_LOAD11:%.*]] = call <vscale x 4 x double> @llvm.vp.load.nxv4f64.p0nxv4f64(<vscale x 4 x double>* [[TMP23]], i32 8, <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP24]])
+; CHECK-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD]], <vscale x 4 x double> [[VP_OP_LOAD11]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP25]])
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP28:%.*]] = mul i32 [[TMP27]], 0
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds double, double* [[TMP26]], i32 [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast double* [[TMP29]] to <vscale x 4 x double>*
+; CHECK-NEXT:    [[TMP31:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> [[VP_OP]], <vscale x 4 x double>* [[TMP30]], i32 8, <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP31]])
+; CHECK-NEXT:    [[TMP32:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP18:%.*]] = call <vscale x 4 x double> @llvm.vp.fsub.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD]], <vscale x 4 x double> [[VP_OP_LOAD11]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP32]])
+; CHECK-NEXT:    [[TMP33:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP21:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP18]], <vscale x 4 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP33]])
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[E:%.*]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP35:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP36:%.*]] = mul i32 [[TMP35]], 0
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds double, double* [[TMP34]], i32 [[TMP36]]
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast double* [[TMP37]] to <vscale x 4 x double>*
+; CHECK-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> [[VP_OP21]], <vscale x 4 x double>* [[TMP38]], i32 8, <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP39]])
+; CHECK-NEXT:    [[TMP40:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP26:%.*]] = call <vscale x 4 x double> @llvm.vp.fmul.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD]], <vscale x 4 x double> [[VP_OP_LOAD11]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP40]])
+; CHECK-NEXT:    [[TMP41:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP29:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP26]], <vscale x 4 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP41]])
+; CHECK-NEXT:    [[TMP42:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP32:%.*]] = call <vscale x 4 x double> @llvm.vp.fsub.nxv4f64(<vscale x 4 x double> [[VP_OP29]], <vscale x 4 x double> [[VP_OP21]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP42]])
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr inbounds double, double* [[F:%.*]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP45:%.*]] = mul i32 [[TMP44]], 0
+; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr inbounds double, double* [[TMP43]], i32 [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = bitcast double* [[TMP46]] to <vscale x 4 x double>*
+; CHECK-NEXT:    [[TMP48:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> [[VP_OP32]], <vscale x 4 x double>* [[TMP47]], i32 8, <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP48]])
+; CHECK-NEXT:    [[TMP49:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP37:%.*]] = call <vscale x 4 x double> @llvm.vp.fmul.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD]], <vscale x 4 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP49]])
+; CHECK-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP40:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD11]], <vscale x 4 x double> [[VP_OP37]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP50]])
+; CHECK-NEXT:    [[TMP51:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP43:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP21]], <vscale x 4 x double> [[VP_OP40]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP51]])
+; CHECK-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    [[VP_OP46:%.*]] = call <vscale x 4 x double> @llvm.vp.fsub.nxv4f64(<vscale x 4 x double> [[VP_OP43]], <vscale x 4 x double> [[VP_OP32]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP52]])
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds double, double* [[G:%.*]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP54:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP55:%.*]] = mul i32 [[TMP54]], 0
+; CHECK-NEXT:    [[TMP56:%.*]] = getelementptr inbounds double, double* [[TMP53]], i32 [[TMP55]]
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast double* [[TMP56]] to <vscale x 4 x double>*
+; CHECK-NEXT:    [[TMP58:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK-NEXT:    call void @llvm.vp.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> [[VP_OP46]], <vscale x 4 x double>* [[TMP57]], i32 8, <vscale x 4 x i1> [[DOTSPLAT8]], i32 [[TMP58]])
+; CHECK-NEXT:    [[TMP59:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP60:%.*]] = mul i64 [[TMP59]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]]
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TMP61]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP62:%.*]] = load double, double* [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP63:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd double [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store double [[ADD3]], double* [[ARRAYIDX5]], align 8
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[ADD12:%.*]] = fadd double [[SUB]], [[ADD3]]
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds double, double* [[E]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store double [[ADD12]], double* [[ARRAYIDX14]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP62]], [[TMP63]]
+; CHECK-NEXT:    [[ADD21:%.*]] = fadd double [[MUL]], [[ADD3]]
+; CHECK-NEXT:    [[SUB24:%.*]] = fsub double [[ADD21]], [[ADD12]]
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds double, double* [[F]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store double [[SUB24]], double* [[ARRAYIDX26]], align 8
+; CHECK-NEXT:    [[MUL31:%.*]] = fmul double [[TMP62]], [[ADD3]]
+; CHECK-NEXT:    [[ADD34:%.*]] = fadd double [[TMP63]], [[MUL31]]
+; CHECK-NEXT:    [[ADD37:%.*]] = fadd double [[ADD12]], [[ADD34]]
+; CHECK-NEXT:    [[SUB40:%.*]] = fsub double [[ADD37]], [[SUB24]]
+; CHECK-NEXT:    [[ARRAYIDX42:%.*]] = getelementptr inbounds double, double* [[G]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store double [[SUB40]], double* [[ARRAYIDX42]], align 8
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+; CHECK1-LABEL: @highRegUage(
+; CHECK1-NEXT:  entry:
+; CHECK1-NEXT:    [[CMP_NOT75:%.*]] = icmp slt i32 [[N:%.*]], 1
+; CHECK1-NEXT:    br i1 [[CMP_NOT75]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; CHECK1:       for.body.preheader:
+; CHECK1-NEXT:    [[TMP0:%.*]] = add nuw i32 [[N]], 1
+; CHECK1-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK1-NEXT:    [[TMP1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
+; CHECK1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK1:       vector.ph:
+; CHECK1-NEXT:    [[IND_END:%.*]] = add i64 1, [[TMP1]]
+; CHECK1-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK1:       vector.body:
+; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK1-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[OFFSET_IDX]], i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
+; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP3:%.*]] = add <vscale x 1 x i64> [[TMP2]], [[DOTSPLAT]]
+; CHECK1-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 1 x i64> poison, i64 1, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT3]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP4:%.*]] = mul <vscale x 1 x i64> [[TMP3]], [[DOTSPLAT4]]
+; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[BROADCAST_SPLAT2]], [[TMP4]]
+; CHECK1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 0
+; CHECK1-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 0
+; CHECK1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 1
+; CHECK1-NEXT:    [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], [[TMP8]]
+; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[INDEX]], i32 0
+; CHECK1-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[STEPVECTOR:%.*]] = call <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
+; CHECK1-NEXT:    [[VEC_IV:%.*]] = add <vscale x 1 x i64> [[BROADCAST_SPLAT6]], [[STEPVECTOR]]
+; CHECK1-NEXT:    [[TMP10:%.*]] = icmp ule <vscale x 1 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP9]]
+; CHECK1-NEXT:    [[TMP12:%.*]] = sub i64 [[TMP1]], [[INDEX]]
+; CHECK1-NEXT:    [[TMP13:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP12]], i64 3, i64 0)
+; CHECK1-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 0
+; CHECK1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[TMP11]], i32 [[TMP15]]
+; CHECK1-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[DOTSPLATINSERT7:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; CHECK1-NEXT:    [[DOTSPLAT8:%.*]] = shufflevector <vscale x 1 x i1> [[DOTSPLATINSERT7]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK1-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP17]], i32 8, <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP18]])
+; CHECK1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP9]]
+; CHECK1-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 0
+; CHECK1-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP19]], i32 [[TMP21]]
+; CHECK1-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP_LOAD11:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP23]], i32 8, <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP24]])
+; CHECK1-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[VP_OP_LOAD11]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP25]])
+; CHECK1-NEXT:    [[TMP26:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[TMP9]]
+; CHECK1-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP28:%.*]] = mul i32 [[TMP27]], 0
+; CHECK1-NEXT:    [[TMP29:%.*]] = getelementptr inbounds double, double* [[TMP26]], i32 [[TMP28]]
+; CHECK1-NEXT:    [[TMP30:%.*]] = bitcast double* [[TMP29]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[TMP31:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP]], <vscale x 1 x double>* [[TMP30]], i32 8, <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP31]])
+; CHECK1-NEXT:    [[TMP32:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP18:%.*]] = call <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[VP_OP_LOAD11]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP32]])
+; CHECK1-NEXT:    [[TMP33:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP21:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP18]], <vscale x 1 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP33]])
+; CHECK1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[E:%.*]], i64 [[TMP9]]
+; CHECK1-NEXT:    [[TMP35:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP36:%.*]] = mul i32 [[TMP35]], 0
+; CHECK1-NEXT:    [[TMP37:%.*]] = getelementptr inbounds double, double* [[TMP34]], i32 [[TMP36]]
+; CHECK1-NEXT:    [[TMP38:%.*]] = bitcast double* [[TMP37]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP21]], <vscale x 1 x double>* [[TMP38]], i32 8, <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP39]])
+; CHECK1-NEXT:    [[TMP40:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP26:%.*]] = call <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[VP_OP_LOAD11]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP40]])
+; CHECK1-NEXT:    [[TMP41:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP29:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP26]], <vscale x 1 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP41]])
+; CHECK1-NEXT:    [[TMP42:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP32:%.*]] = call <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> [[VP_OP29]], <vscale x 1 x double> [[VP_OP21]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP42]])
+; CHECK1-NEXT:    [[TMP43:%.*]] = getelementptr inbounds double, double* [[F:%.*]], i64 [[TMP9]]
+; CHECK1-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP45:%.*]] = mul i32 [[TMP44]], 0
+; CHECK1-NEXT:    [[TMP46:%.*]] = getelementptr inbounds double, double* [[TMP43]], i32 [[TMP45]]
+; CHECK1-NEXT:    [[TMP47:%.*]] = bitcast double* [[TMP46]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[TMP48:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP32]], <vscale x 1 x double>* [[TMP47]], i32 8, <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP48]])
+; CHECK1-NEXT:    [[TMP49:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP37:%.*]] = call <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP49]])
+; CHECK1-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP40:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD11]], <vscale x 1 x double> [[VP_OP37]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP50]])
+; CHECK1-NEXT:    [[TMP51:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP43:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP21]], <vscale x 1 x double> [[VP_OP40]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP51]])
+; CHECK1-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    [[VP_OP46:%.*]] = call <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> [[VP_OP43]], <vscale x 1 x double> [[VP_OP32]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP52]])
+; CHECK1-NEXT:    [[TMP53:%.*]] = getelementptr inbounds double, double* [[G:%.*]], i64 [[TMP9]]
+; CHECK1-NEXT:    [[TMP54:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK1-NEXT:    [[TMP55:%.*]] = mul i32 [[TMP54]], 0
+; CHECK1-NEXT:    [[TMP56:%.*]] = getelementptr inbounds double, double* [[TMP53]], i32 [[TMP55]]
+; CHECK1-NEXT:    [[TMP57:%.*]] = bitcast double* [[TMP56]] to <vscale x 1 x double>*
+; CHECK1-NEXT:    [[TMP58:%.*]] = trunc i64 [[TMP13]] to i32
+; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP46]], <vscale x 1 x double>* [[TMP57]], i32 8, <vscale x 1 x i1> [[DOTSPLAT8]], i32 [[TMP58]])
+; CHECK1-NEXT:    [[TMP59:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]]
+; CHECK1-NEXT:    [[TMP60:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP1]]
+; CHECK1-NEXT:    br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK1:       middle.block:
+; CHECK1-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK1:       scalar.ph:
+; CHECK1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_BODY_PREHEADER]] ]
+; CHECK1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK1:       for.body:
+; CHECK1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    [[TMP61:%.*]] = load double, double* [[ARRAYIDX]], align 8
+; CHECK1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    [[TMP62:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; CHECK1-NEXT:    [[ADD3:%.*]] = fadd double [[TMP61]], [[TMP62]]
+; CHECK1-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    store double [[ADD3]], double* [[ARRAYIDX5]], align 8
+; CHECK1-NEXT:    [[SUB:%.*]] = fsub double [[TMP61]], [[TMP62]]
+; CHECK1-NEXT:    [[ADD12:%.*]] = fadd double [[SUB]], [[ADD3]]
+; CHECK1-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds double, double* [[E]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    store double [[ADD12]], double* [[ARRAYIDX14]], align 8
+; CHECK1-NEXT:    [[MUL:%.*]] = fmul double [[TMP61]], [[TMP62]]
+; CHECK1-NEXT:    [[ADD21:%.*]] = fadd double [[MUL]], [[ADD3]]
+; CHECK1-NEXT:    [[SUB24:%.*]] = fsub double [[ADD21]], [[ADD12]]
+; CHECK1-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds double, double* [[F]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    store double [[SUB24]], double* [[ARRAYIDX26]], align 8
+; CHECK1-NEXT:    [[MUL31:%.*]] = fmul double [[TMP61]], [[ADD3]]
+; CHECK1-NEXT:    [[ADD34:%.*]] = fadd double [[TMP62]], [[MUL31]]
+; CHECK1-NEXT:    [[ADD37:%.*]] = fadd double [[ADD12]], [[ADD34]]
+; CHECK1-NEXT:    [[SUB40:%.*]] = fsub double [[ADD37]], [[SUB24]]
+; CHECK1-NEXT:    [[ARRAYIDX42:%.*]] = getelementptr inbounds double, double* [[G]], i64 [[INDVARS_IV]]
+; CHECK1-NEXT:    store double [[SUB40]], double* [[ARRAYIDX42]], align 8
+; CHECK1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK1:       for.end.loopexit:
+; CHECK1-NEXT:    br label [[FOR_END]]
+; CHECK1:       for.end:
+; CHECK1-NEXT:    ret void
+;
+entry:
+  %cmp.not75 = icmp slt i32 %N, 1
+  br i1 %cmp.not75, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = add nuw i32 %N, 1
+  %wide.trip.count = zext i32 %0 to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 1, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  %1 = load double, double* %arrayidx, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %b, i64 %indvars.iv
+  %2 = load double, double* %arrayidx2, align 8
+  %add3 = fadd double %1, %2
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 %indvars.iv
+  store double %add3, double* %arrayidx5, align 8
+  %sub = fsub double %1, %2
+  %add12 = fadd double %sub, %add3
+  %arrayidx14 = getelementptr inbounds double, double* %e, i64 %indvars.iv
+  store double %add12, double* %arrayidx14, align 8
+  %mul = fmul double %1, %2
+  %add21 = fadd double %mul, %add3
+  %sub24 = fsub double %add21, %add12
+  %arrayidx26 = getelementptr inbounds double, double* %f, i64 %indvars.iv
+  store double %sub24, double* %arrayidx26, align 8
+  %mul31 = fmul double %1, %add3
+  %add34 = fadd double %2, %mul31
+  %add37 = fadd double %add12, %add34
+  %sub40 = fsub double %add37, %sub24
+  %arrayidx42 = getelementptr inbounds double, double* %g, i64 %indvars.iv
+  store double %sub40, double* %arrayidx42, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vfadd32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-vfadd32.ll
similarity index 99%
rename from llvm/test/Transforms/LoopVectorize/RISCV/vfadd32.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-vfadd32.ll
index 0d352c2b9e78f5679d41c489803893d5792ffe77..ab3f89f8a19316a5d1abd08f6e85356cd727d346 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vfadd32.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-vfadd32.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -vector-register-width-factor=8 < %s -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize < %s -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = 'vfadd32.c'
 source_filename = "vfadd32.c"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vfadd64.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-vfadd64.ll
similarity index 99%
rename from llvm/test/Transforms/LoopVectorize/RISCV/vfadd64.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-vfadd64.ll
index 4d97abb00dc4d2e2693bd22b4ffdf960fbdc9967..7014bdcf56db53c537e46a601035d1c242f5b0a7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vfadd64.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-vfadd64.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize -vector-register-width-factor=8 < %s -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S -riscv-v-vector-bits-min=64 \
 ; RUN:    -loop-vectorize < %s -o - | FileCheck %s --check-prefix=CHECK1
 ; ModuleID = 'vfadd64.c'
 source_filename = "vfadd64.c"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vpred-fix-TC.ll b/llvm/test/Transforms/LoopVectorize/RISCV/epi-vpred-fix-TC.ll
similarity index 97%
rename from llvm/test/Transforms/LoopVectorize/RISCV/vpred-fix-TC.ll
rename to llvm/test/Transforms/LoopVectorize/RISCV/epi-vpred-fix-TC.ll
index 449086a72c8ac7ae5832828fd6648ee8e884ae86..b1980b01599848df9e7e3a54b781763245dc7ab2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vpred-fix-TC.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/epi-vpred-fix-TC.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
+; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -riscv-v-vector-bits-min=64 \
 ; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -O2 < %s -o - | FileCheck %s
 
 ; ModuleID = 'simple-add-notail.c'
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/initialize-with-ind-int.ll b/llvm/test/Transforms/LoopVectorize/RISCV/initialize-with-ind-int.ll
deleted file mode 100644
index c1b9c410da10cfa37a53756b9864aa748158b146..0000000000000000000000000000000000000000
--- a/llvm/test/Transforms/LoopVectorize/RISCV/initialize-with-ind-int.ll
+++ /dev/null
@@ -1,321 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
-; RUN:    -loop-vectorize -vector-register-width-factor=8 < %s -o - | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v -S \
-; RUN:    -loop-vectorize < %s -o - | FileCheck %s --check-prefix=CHECK1
-; ModuleID = 't2.i'
-source_filename = "t2.i"
-target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
-target triple = "riscv64-unknown-linux-gnu"
-
-; Function Attrs: nounwind readnone
-define dso_local signext i32 @main(i32 signext %argc, i8** nocapture readnone %argv) local_unnamed_addr #0 {
-; CHECK-LABEL: @main(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A:%.*]] = alloca [1024 x double], align 8
-; CHECK-NEXT:    [[B:%.*]] = alloca [1024 x double], align 8
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [1024 x double]* [[A]] to i8*
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR4:[0-9]+]]
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast [1024 x double]* [[B]] to i8*
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP1]]) #[[ATTR4]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP5]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[DOTSPLAT]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP7]]
-; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP8]], i32 0
-; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 0
-; CHECK-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT3]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE5:%.*]] = call <vscale x 8 x i32> @llvm.experimental.vector.stepvector.nxv8i32()
-; CHECK-NEXT:    [[INDUCTION6:%.*]] = add <vscale x 8 x i32> [[DOTSPLAT4]], [[STEPVEC_BASE5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 8
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i32 1, [[TMP10]]
-; CHECK-NEXT:    [[DOTSPLATINSERT7:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP11]], i32 0
-; CHECK-NEXT:    [[DOTSPLAT8:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT7]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[DOTSPLATINSERT11:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 0
-; CHECK-NEXT:    [[DOTSPLAT12:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT11]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE13:%.*]] = call <vscale x 8 x i32> @llvm.experimental.vector.stepvector.nxv8i32()
-; CHECK-NEXT:    [[INDUCTION14:%.*]] = add <vscale x 8 x i32> [[DOTSPLAT12]], [[STEPVEC_BASE13]]
-; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 8
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 1, [[TMP13]]
-; CHECK-NEXT:    [[DOTSPLATINSERT15:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP14]], i32 0
-; CHECK-NEXT:    [[DOTSPLAT16:%.*]] = shufflevector <vscale x 8 x i32> [[DOTSPLATINSERT15]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1024, i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND9:%.*]] = phi <vscale x 8 x i32> [ [[INDUCTION6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT10:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND17:%.*]] = phi <vscale x 8 x i32> [ [[INDUCTION14]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 0
-; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP16]], 0
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 1
-; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX]], [[TMP18]]
-; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 0
-; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[TMP21]], 1
-; CHECK-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 1
-; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[INDEX]], [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 0
-; CHECK-NEXT:    [[TMP27:%.*]] = add i64 [[TMP26]], 2
-; CHECK-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 1
-; CHECK-NEXT:    [[TMP29:%.*]] = add i64 [[INDEX]], [[TMP28]]
-; CHECK-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 0
-; CHECK-NEXT:    [[TMP32:%.*]] = add i64 [[TMP31]], 3
-; CHECK-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP32]], 1
-; CHECK-NEXT:    [[TMP34:%.*]] = add i64 [[INDEX]], [[TMP33]]
-; CHECK-NEXT:    [[TMP35:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP36:%.*]] = mul i64 [[TMP35]], 0
-; CHECK-NEXT:    [[TMP37:%.*]] = add i64 [[TMP36]], 4
-; CHECK-NEXT:    [[TMP38:%.*]] = mul i64 [[TMP37]], 1
-; CHECK-NEXT:    [[TMP39:%.*]] = add i64 [[INDEX]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP41:%.*]] = mul i64 [[TMP40]], 0
-; CHECK-NEXT:    [[TMP42:%.*]] = add i64 [[TMP41]], 5
-; CHECK-NEXT:    [[TMP43:%.*]] = mul i64 [[TMP42]], 1
-; CHECK-NEXT:    [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP43]]
-; CHECK-NEXT:    [[TMP45:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP46:%.*]] = mul i64 [[TMP45]], 0
-; CHECK-NEXT:    [[TMP47:%.*]] = add i64 [[TMP46]], 6
-; CHECK-NEXT:    [[TMP48:%.*]] = mul i64 [[TMP47]], 1
-; CHECK-NEXT:    [[TMP49:%.*]] = add i64 [[INDEX]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP51:%.*]] = mul i64 [[TMP50]], 0
-; CHECK-NEXT:    [[TMP52:%.*]] = add i64 [[TMP51]], 7
-; CHECK-NEXT:    [[TMP53:%.*]] = mul i64 [[TMP52]], 1
-; CHECK-NEXT:    [[TMP54:%.*]] = add i64 [[INDEX]], [[TMP53]]
-; CHECK-NEXT:    [[TMP55:%.*]] = sitofp <vscale x 8 x i32> [[VEC_IND9]] to <vscale x 8 x double>
-; CHECK-NEXT:    [[TMP56:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[TMP19]]
-; CHECK-NEXT:    [[TMP57:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP58:%.*]] = mul i32 [[TMP57]], 0
-; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds double, double* [[TMP56]], i32 [[TMP58]]
-; CHECK-NEXT:    [[TMP60:%.*]] = bitcast double* [[TMP59]] to <vscale x 8 x double>*
-; CHECK-NEXT:    store <vscale x 8 x double> [[TMP55]], <vscale x 8 x double>* [[TMP60]], align 8, !tbaa [[TBAA2:![0-9]+]]
-; CHECK-NEXT:    [[TMP61:%.*]] = sub <vscale x 8 x i32> [[BROADCAST_SPLAT]], [[VEC_IND17]]
-; CHECK-NEXT:    [[TMP62:%.*]] = sitofp <vscale x 8 x i32> [[TMP61]] to <vscale x 8 x double>
-; CHECK-NEXT:    [[TMP63:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[TMP19]]
-; CHECK-NEXT:    [[TMP64:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP65:%.*]] = mul i32 [[TMP64]], 0
-; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds double, double* [[TMP63]], i32 [[TMP65]]
-; CHECK-NEXT:    [[TMP67:%.*]] = bitcast double* [[TMP66]] to <vscale x 8 x double>*
-; CHECK-NEXT:    store <vscale x 8 x double> [[TMP62]], <vscale x 8 x double>* [[TMP67]], align 8, !tbaa [[TBAA2]]
-; CHECK-NEXT:    [[TMP68:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP69:%.*]] = mul i64 [[TMP68]], 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP69]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT2]]
-; CHECK-NEXT:    [[VEC_IND_NEXT10]] = add <vscale x 8 x i32> [[VEC_IND9]], [[DOTSPLAT8]]
-; CHECK-NEXT:    [[VEC_IND_NEXT18]] = add <vscale x 8 x i32> [[VEC_IND17]], [[DOTSPLAT16]]
-; CHECK-NEXT:    [[TMP70:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP70]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[ARGC:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[IDXPROM4]]
-; CHECK-NEXT:    [[TMP71:%.*]] = load double, double* [[ARRAYIDX5]], align 8, !tbaa [[TBAA2]]
-; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[IDXPROM4]]
-; CHECK-NEXT:    [[TMP72:%.*]] = load double, double* [[ARRAYIDX7]], align 8, !tbaa [[TBAA2]]
-; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP71]], [[TMP72]]
-; CHECK-NEXT:    [[CONV8:%.*]] = fptosi double [[ADD]] to i32
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP1]]) #[[ATTR4]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR4]]
-; CHECK-NEXT:    ret i32 [[CONV8]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP73:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP73]] to double
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store double [[CONV]], double* [[ARRAYIDX]], align 8, !tbaa [[TBAA2]]
-; CHECK-NEXT:    [[TMP74:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[TMP75:%.*]] = sub i32 1024, [[TMP74]]
-; CHECK-NEXT:    [[CONV1:%.*]] = sitofp i32 [[TMP75]] to double
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store double [[CONV1]], double* [[ARRAYIDX3]], align 8, !tbaa [[TBAA2]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-;
-; CHECK1-LABEL: @main(
-; CHECK1-NEXT:  entry:
-; CHECK1-NEXT:    [[A:%.*]] = alloca [1024 x double], align 8
-; CHECK1-NEXT:    [[B:%.*]] = alloca [1024 x double], align 8
-; CHECK1-NEXT:    [[TMP0:%.*]] = bitcast [1024 x double]* [[A]] to i8*
-; CHECK1-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR4:[0-9]+]]
-; CHECK1-NEXT:    [[TMP1:%.*]] = bitcast [1024 x double]* [[B]] to i8*
-; CHECK1-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull [[TMP1]]) #[[ATTR4]]
-; CHECK1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; CHECK1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK1:       vector.ph:
-; CHECK1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 0
-; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vector.stepvector.nxv1i64()
-; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[DOTSPLAT]], [[STEPVEC_BASE]]
-; CHECK1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP5:%.*]] = mul i64 1, [[TMP4]]
-; CHECK1-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP5]], i32 0
-; CHECK1-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 1 x i32> poison, i32 0, i32 0
-; CHECK1-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT3]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE5:%.*]] = call <vscale x 1 x i32> @llvm.experimental.vector.stepvector.nxv1i32()
-; CHECK1-NEXT:    [[INDUCTION6:%.*]] = add <vscale x 1 x i32> [[DOTSPLAT4]], [[STEPVEC_BASE5]]
-; CHECK1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP7:%.*]] = mul i32 1, [[TMP6]]
-; CHECK1-NEXT:    [[DOTSPLATINSERT7:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP7]], i32 0
-; CHECK1-NEXT:    [[DOTSPLAT8:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT7]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[DOTSPLATINSERT11:%.*]] = insertelement <vscale x 1 x i32> poison, i32 0, i32 0
-; CHECK1-NEXT:    [[DOTSPLAT12:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT11]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE13:%.*]] = call <vscale x 1 x i32> @llvm.experimental.vector.stepvector.nxv1i32()
-; CHECK1-NEXT:    [[INDUCTION14:%.*]] = add <vscale x 1 x i32> [[DOTSPLAT12]], [[STEPVEC_BASE13]]
-; CHECK1-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP9:%.*]] = mul i32 1, [[TMP8]]
-; CHECK1-NEXT:    [[DOTSPLATINSERT15:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP9]], i32 0
-; CHECK1-NEXT:    [[DOTSPLAT16:%.*]] = shufflevector <vscale x 1 x i32> [[DOTSPLATINSERT15]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 1024, i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK1:       vector.body:
-; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[VEC_IND9:%.*]] = phi <vscale x 1 x i32> [ [[INDUCTION6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT10:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[VEC_IND17:%.*]] = phi <vscale x 1 x i32> [ [[INDUCTION14]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT18:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 0
-; CHECK1-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], 0
-; CHECK1-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 1
-; CHECK1-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], [[TMP13]]
-; CHECK1-NEXT:    [[TMP15:%.*]] = sitofp <vscale x 1 x i32> [[VEC_IND9]] to <vscale x 1 x double>
-; CHECK1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[TMP14]]
-; CHECK1-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 0
-; CHECK1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP16]], i32 [[TMP18]]
-; CHECK1-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    store <vscale x 1 x double> [[TMP15]], <vscale x 1 x double>* [[TMP20]], align 8, !tbaa [[TBAA2:![0-9]+]]
-; CHECK1-NEXT:    [[TMP21:%.*]] = sub <vscale x 1 x i32> [[BROADCAST_SPLAT]], [[VEC_IND17]]
-; CHECK1-NEXT:    [[TMP22:%.*]] = sitofp <vscale x 1 x i32> [[TMP21]] to <vscale x 1 x double>
-; CHECK1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[TMP14]]
-; CHECK1-NEXT:    [[TMP24:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP25:%.*]] = mul i32 [[TMP24]], 0
-; CHECK1-NEXT:    [[TMP26:%.*]] = getelementptr inbounds double, double* [[TMP23]], i32 [[TMP25]]
-; CHECK1-NEXT:    [[TMP27:%.*]] = bitcast double* [[TMP26]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    store <vscale x 1 x double> [[TMP22]], <vscale x 1 x double>* [[TMP27]], align 8, !tbaa [[TBAA2]]
-; CHECK1-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP28]]
-; CHECK1-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 1 x i64> [[VEC_IND]], [[DOTSPLAT2]]
-; CHECK1-NEXT:    [[VEC_IND_NEXT10]] = add <vscale x 1 x i32> [[VEC_IND9]], [[DOTSPLAT8]]
-; CHECK1-NEXT:    [[VEC_IND_NEXT18]] = add <vscale x 1 x i32> [[VEC_IND17]], [[DOTSPLAT16]]
-; CHECK1-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK1-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK1:       middle.block:
-; CHECK1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK1-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; CHECK1:       scalar.ph:
-; CHECK1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK1-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK1:       for.cond.cleanup:
-; CHECK1-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[ARGC:%.*]] to i64
-; CHECK1-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[IDXPROM4]]
-; CHECK1-NEXT:    [[TMP30:%.*]] = load double, double* [[ARRAYIDX5]], align 8, !tbaa [[TBAA2]]
-; CHECK1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[IDXPROM4]]
-; CHECK1-NEXT:    [[TMP31:%.*]] = load double, double* [[ARRAYIDX7]], align 8, !tbaa [[TBAA2]]
-; CHECK1-NEXT:    [[ADD:%.*]] = fadd double [[TMP30]], [[TMP31]]
-; CHECK1-NEXT:    [[CONV8:%.*]] = fptosi double [[ADD]] to i32
-; CHECK1-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP1]]) #[[ATTR4]]
-; CHECK1-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull [[TMP0]]) #[[ATTR4]]
-; CHECK1-NEXT:    ret i32 [[CONV8]]
-; CHECK1:       for.body:
-; CHECK1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK1-NEXT:    [[TMP32:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP32]] to double
-; CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[A]], i64 0, i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    store double [[CONV]], double* [[ARRAYIDX]], align 8, !tbaa [[TBAA2]]
-; CHECK1-NEXT:    [[TMP33:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK1-NEXT:    [[TMP34:%.*]] = sub i32 1024, [[TMP33]]
-; CHECK1-NEXT:    [[CONV1:%.*]] = sitofp i32 [[TMP34]] to double
-; CHECK1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [1024 x double], [1024 x double]* [[B]], i64 0, i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    store double [[CONV1]], double* [[ARRAYIDX3]], align 8, !tbaa [[TBAA2]]
-; CHECK1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
-; CHECK1-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-;
-entry:
-  %a = alloca [1024 x double], align 8
-  %b = alloca [1024 x double], align 8
-  %0 = bitcast [1024 x double]* %a to i8*
-  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) #2
-  %1 = bitcast [1024 x double]* %b to i8*
-  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %1) #2
-  br label %for.body
-
-for.cond.cleanup:                                 ; preds = %for.body
-  %idxprom4 = sext i32 %argc to i64
-  %arrayidx5 = getelementptr inbounds [1024 x double], [1024 x double]* %a, i64 0, i64 %idxprom4
-  %2 = load double, double* %arrayidx5, align 8, !tbaa !2
-  %arrayidx7 = getelementptr inbounds [1024 x double], [1024 x double]* %b, i64 0, i64 %idxprom4
-  %3 = load double, double* %arrayidx7, align 8, !tbaa !2
-  %add = fadd double %2, %3
-  %conv8 = fptosi double %add to i32
-  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %1) #2
-  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) #2
-  ret i32 %conv8
-
-for.body:                                         ; preds = %for.body, %entry
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %4 = trunc i64 %indvars.iv to i32
-  %conv = sitofp i32 %4 to double
-  %arrayidx = getelementptr inbounds [1024 x double], [1024 x double]* %a, i64 0, i64 %indvars.iv
-  store double %conv, double* %arrayidx, align 8, !tbaa !2
-  %5 = trunc i64 %indvars.iv to i32
-  %6 = sub i32 1024, %5
-  %conv1 = sitofp i32 %6 to double
-  %arrayidx3 = getelementptr inbounds [1024 x double], [1024 x double]* %b, i64 0, i64 %indvars.iv
-  store double %conv1, double* %arrayidx3, align 8, !tbaa !2
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, 1024
-  br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
-
-; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
-
-attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+a,+c,+d,+experimental-v,+f,+m,-relax" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { argmemonly nounwind willreturn }
-attributes #2 = { nounwind }
-
-!llvm.module.flags = !{!0}
-!llvm.ident = !{!1}
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{!"clang version 10.0.0 (git@repo.hca.bsc.es:EPI/System-Software/llvm-mono.git 823a818e0f44dc9b594f14a328ec52b247f3611a)"}
-!2 = !{!3, !3, i64 0}
-!3 = !{!"double", !4, i64 0}
-!4 = !{!"omnipotent char", !5, i64 0}
-!5 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ef876ba6dd44891a70a4aa4bbad51aac26f16299
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -0,0 +1,240 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-vectorize -mtriple=riscv32 -mattr=+experimental-v,+d -riscv-v-vector-bits-min=256 -S | FileCheck %s -check-prefixes=RV32
+; RUN: opt < %s -loop-vectorize -mtriple=riscv64 -mattr=+experimental-v,+d -riscv-v-vector-bits-min=256 -S | FileCheck %s -check-prefixes=RV64
+
+; The source code:
+;
+;void foo4(double *A, double *B, int *trigger) {
+;
+;  for (int i=0; i<10000; i += 16) {
+;    if (trigger[i] < 100) {
+;          A[i] = B[i*2] + trigger[i]; << non-consecutive access
+;    }
+;  }
+;}
+
+define void @foo4(double* nocapture %A, double* nocapture readonly %B, i32* nocapture readonly %trigger) local_unnamed_addr #0 {
+; RV32-LABEL: @foo4(
+; RV32-NEXT:  entry:
+; RV32-NEXT:    [[A1:%.*]] = bitcast double* [[A:%.*]] to i8*
+; RV32-NEXT:    [[TRIGGER3:%.*]] = bitcast i32* [[TRIGGER:%.*]] to i8*
+; RV32-NEXT:    [[B6:%.*]] = bitcast double* [[B:%.*]] to i8*
+; RV32-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; RV32-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; RV32-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 625, [[TMP1]]
+; RV32-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; RV32:       vector.memcheck:
+; RV32-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[A]], i64 9985
+; RV32-NEXT:    [[SCEVGEP2:%.*]] = bitcast double* [[SCEVGEP]] to i8*
+; RV32-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[TRIGGER]], i64 9985
+; RV32-NEXT:    [[SCEVGEP45:%.*]] = bitcast i32* [[SCEVGEP4]] to i8*
+; RV32-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, double* [[B]], i64 19969
+; RV32-NEXT:    [[SCEVGEP78:%.*]] = bitcast double* [[SCEVGEP7]] to i8*
+; RV32-NEXT:    [[BOUND0:%.*]] = icmp ult i8* [[A1]], [[SCEVGEP45]]
+; RV32-NEXT:    [[BOUND1:%.*]] = icmp ult i8* [[TRIGGER3]], [[SCEVGEP2]]
+; RV32-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; RV32-NEXT:    [[BOUND09:%.*]] = icmp ult i8* [[A1]], [[SCEVGEP78]]
+; RV32-NEXT:    [[BOUND110:%.*]] = icmp ult i8* [[B6]], [[SCEVGEP2]]
+; RV32-NEXT:    [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]
+; RV32-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT11]]
+; RV32-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[CONFLICT_RDX]], true
+; RV32-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; RV32:       vector.ph:
+; RV32-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; RV32-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 625, [[TMP3]]
+; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]]
+; RV32-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 16
+; RV32-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 0
+; RV32-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV32-NEXT:    [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; RV32-NEXT:    [[DOTSPLATINSERT12:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 0
+; RV32-NEXT:    [[DOTSPLAT13:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT12]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV32-NEXT:    [[TMP5:%.*]] = add <vscale x 2 x i64> [[TMP4]], [[DOTSPLAT13]]
+; RV32-NEXT:    [[DOTSPLATINSERT14:%.*]] = insertelement <vscale x 2 x i64> poison, i64 16, i32 0
+; RV32-NEXT:    [[DOTSPLAT15:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT14]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV32-NEXT:    [[TMP6:%.*]] = mul <vscale x 2 x i64> [[TMP5]], [[DOTSPLAT15]]
+; RV32-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[TMP6]]
+; RV32-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; RV32-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; RV32-NEXT:    [[TMP9:%.*]] = mul i64 16, [[TMP8]]
+; RV32-NEXT:    [[DOTSPLATINSERT16:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP9]], i32 0
+; RV32-NEXT:    [[DOTSPLAT17:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT16]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV32-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 100, i32 0
+; RV32-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; RV32-NEXT:    [[BROADCAST_SPLATINSERT18:%.*]] = insertelement <vscale x 2 x i64> poison, i64 1, i32 0
+; RV32-NEXT:    [[BROADCAST_SPLAT19:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT18]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV32-NEXT:    br label [[VECTOR_BODY:%.*]]
+; RV32:       vector.body:
+; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RV32-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RV32-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
+; RV32-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0i32(<vscale x 2 x i32*> [[TMP10]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> undef, i1 true, i32 0), <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i32> undef), !alias.scope !0
+; RV32-NEXT:    [[TMP11:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], [[BROADCAST_SPLAT]]
+; RV32-NEXT:    [[TMP12:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT19]]
+; RV32-NEXT:    [[TMP13:%.*]] = getelementptr inbounds double, double* [[B]], <vscale x 2 x i64> [[TMP12]]
+; RV32-NEXT:    [[WIDE_MASKED_GATHER20:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0f64(<vscale x 2 x double*> [[TMP13]], i32 8, <vscale x 2 x i1> [[TMP11]], <vscale x 2 x double> undef), !alias.scope !3
+; RV32-NEXT:    [[TMP14:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
+; RV32-NEXT:    [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER20]], [[TMP14]]
+; RV32-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[A]], <vscale x 2 x i64> [[VEC_IND]]
+; RV32-NEXT:    call void @llvm.masked.scatter.nxv2f64.nxv2p0f64(<vscale x 2 x double> [[TMP15]], <vscale x 2 x double*> [[TMP16]], i32 8, <vscale x 2 x i1> [[TMP11]]), !alias.scope !5, !noalias !7
+; RV32-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
+; RV32-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 2
+; RV32-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP18]]
+; RV32-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT17]]
+; RV32-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV32-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; RV32:       middle.block:
+; RV32-NEXT:    [[CMP_N:%.*]] = icmp eq i64 625, [[N_VEC]]
+; RV32-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; RV32:       scalar.ph:
+; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; RV32-NEXT:    br label [[FOR_BODY:%.*]]
+; RV32:       for.body:
+; RV32-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; RV32-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; RV32-NEXT:    [[TMP20:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; RV32-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP20]], 100
+; RV32-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; RV32:       if.then:
+; RV32-NEXT:    [[TMP21:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 1
+; RV32-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP21]]
+; RV32-NEXT:    [[TMP22:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; RV32-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP20]] to double
+; RV32-NEXT:    [[ADD:%.*]] = fadd double [[TMP22]], [[CONV]]
+; RV32-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; RV32-NEXT:    store double [[ADD]], double* [[ARRAYIDX7]], align 8
+; RV32-NEXT:    br label [[FOR_INC]]
+; RV32:       for.inc:
+; RV32-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 16
+; RV32-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 10000
+; RV32-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP10:![0-9]+]]
+; RV32:       for.end:
+; RV32-NEXT:    ret void
+;
+; RV64-LABEL: @foo4(
+; RV64-NEXT:  entry:
+; RV64-NEXT:    [[A1:%.*]] = bitcast double* [[A:%.*]] to i8*
+; RV64-NEXT:    [[TRIGGER3:%.*]] = bitcast i32* [[TRIGGER:%.*]] to i8*
+; RV64-NEXT:    [[B6:%.*]] = bitcast double* [[B:%.*]] to i8*
+; RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; RV64-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 625, [[TMP1]]
+; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; RV64:       vector.memcheck:
+; RV64-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[A]], i64 9985
+; RV64-NEXT:    [[SCEVGEP2:%.*]] = bitcast double* [[SCEVGEP]] to i8*
+; RV64-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[TRIGGER]], i64 9985
+; RV64-NEXT:    [[SCEVGEP45:%.*]] = bitcast i32* [[SCEVGEP4]] to i8*
+; RV64-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, double* [[B]], i64 19969
+; RV64-NEXT:    [[SCEVGEP78:%.*]] = bitcast double* [[SCEVGEP7]] to i8*
+; RV64-NEXT:    [[BOUND0:%.*]] = icmp ult i8* [[A1]], [[SCEVGEP45]]
+; RV64-NEXT:    [[BOUND1:%.*]] = icmp ult i8* [[TRIGGER3]], [[SCEVGEP2]]
+; RV64-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; RV64-NEXT:    [[BOUND09:%.*]] = icmp ult i8* [[A1]], [[SCEVGEP78]]
+; RV64-NEXT:    [[BOUND110:%.*]] = icmp ult i8* [[B6]], [[SCEVGEP2]]
+; RV64-NEXT:    [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]
+; RV64-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT11]]
+; RV64-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[CONFLICT_RDX]], true
+; RV64-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; RV64:       vector.ph:
+; RV64-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; RV64-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 625, [[TMP3]]
+; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]]
+; RV64-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 16
+; RV64-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 0
+; RV64-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV64-NEXT:    [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; RV64-NEXT:    [[DOTSPLATINSERT12:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 0
+; RV64-NEXT:    [[DOTSPLAT13:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT12]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV64-NEXT:    [[TMP5:%.*]] = add <vscale x 2 x i64> [[TMP4]], [[DOTSPLAT13]]
+; RV64-NEXT:    [[DOTSPLATINSERT14:%.*]] = insertelement <vscale x 2 x i64> poison, i64 16, i32 0
+; RV64-NEXT:    [[DOTSPLAT15:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT14]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV64-NEXT:    [[TMP6:%.*]] = mul <vscale x 2 x i64> [[TMP5]], [[DOTSPLAT15]]
+; RV64-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[TMP6]]
+; RV64-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; RV64-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; RV64-NEXT:    [[TMP9:%.*]] = mul i64 16, [[TMP8]]
+; RV64-NEXT:    [[DOTSPLATINSERT16:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP9]], i32 0
+; RV64-NEXT:    [[DOTSPLAT17:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT16]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV64-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 100, i32 0
+; RV64-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; RV64-NEXT:    [[BROADCAST_SPLATINSERT18:%.*]] = insertelement <vscale x 2 x i64> poison, i64 1, i32 0
+; RV64-NEXT:    [[BROADCAST_SPLAT19:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT18]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; RV64-NEXT:    br label [[VECTOR_BODY:%.*]]
+; RV64:       vector.body:
+; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RV64-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RV64-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
+; RV64-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0i32(<vscale x 2 x i32*> [[TMP10]], i32 4, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> undef, i1 true, i32 0), <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i32> undef), !alias.scope !0
+; RV64-NEXT:    [[TMP11:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], [[BROADCAST_SPLAT]]
+; RV64-NEXT:    [[TMP12:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT19]]
+; RV64-NEXT:    [[TMP13:%.*]] = getelementptr inbounds double, double* [[B]], <vscale x 2 x i64> [[TMP12]]
+; RV64-NEXT:    [[WIDE_MASKED_GATHER20:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0f64(<vscale x 2 x double*> [[TMP13]], i32 8, <vscale x 2 x i1> [[TMP11]], <vscale x 2 x double> undef), !alias.scope !3
+; RV64-NEXT:    [[TMP14:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
+; RV64-NEXT:    [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER20]], [[TMP14]]
+; RV64-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[A]], <vscale x 2 x i64> [[VEC_IND]]
+; RV64-NEXT:    call void @llvm.masked.scatter.nxv2f64.nxv2p0f64(<vscale x 2 x double> [[TMP15]], <vscale x 2 x double*> [[TMP16]], i32 8, <vscale x 2 x i1> [[TMP11]]), !alias.scope !5, !noalias !7
+; RV64-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
+; RV64-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 2
+; RV64-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP18]]
+; RV64-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT17]]
+; RV64-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV64-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; RV64:       middle.block:
+; RV64-NEXT:    [[CMP_N:%.*]] = icmp eq i64 625, [[N_VEC]]
+; RV64-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; RV64:       scalar.ph:
+; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; RV64-NEXT:    br label [[FOR_BODY:%.*]]
+; RV64:       for.body:
+; RV64-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; RV64-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; RV64-NEXT:    [[TMP20:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; RV64-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP20]], 100
+; RV64-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; RV64:       if.then:
+; RV64-NEXT:    [[TMP21:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 1
+; RV64-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP21]]
+; RV64-NEXT:    [[TMP22:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; RV64-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP20]] to double
+; RV64-NEXT:    [[ADD:%.*]] = fadd double [[TMP22]], [[CONV]]
+; RV64-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; RV64-NEXT:    store double [[ADD]], double* [[ARRAYIDX7]], align 8
+; RV64-NEXT:    br label [[FOR_INC]]
+; RV64:       for.inc:
+; RV64-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 16
+; RV64-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 10000
+; RV64-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP10:![0-9]+]]
+; RV64:       for.end:
+; RV64-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %arrayidx = getelementptr inbounds i32, i32* %trigger, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp slt i32 %0, 100
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = shl nuw nsw i64 %indvars.iv, 1
+  %arrayidx3 = getelementptr inbounds double, double* %B, i64 %1
+  %2 = load double, double* %arrayidx3, align 8
+  %conv = sitofp i32 %0 to double
+  %add = fadd double %2, %conv
+  %arrayidx7 = getelementptr inbounds double, double* %A, i64 %indvars.iv
+  store double %add, double* %arrayidx7, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 16
+  %cmp = icmp ult i64 %indvars.iv.next, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-control-flow.ll b/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-control-flow.ll
deleted file mode 100644
index 898aefa1f1fdaab203c1b1e1520505ca12ac93d0..0000000000000000000000000000000000000000
--- a/llvm/test/Transforms/LoopVectorize/RISCV/simple-add-control-flow.ll
+++ /dev/null
@@ -1,282 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize -vector-register-width-factor=8 < %s -o - \
-; RUN:    | FileCheck %s
-; RUN: opt -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:    -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -loop-vectorize < %s -o - \
-; RUN:    | FileCheck %s --check-prefix=CHECK1
-; ModuleID = './simple-add-control-flow.c'
-source_filename = "./simple-add-control-flow.c"
-target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
-target triple = "riscv64-unknown-linux-gnu"
-
-; Function Attrs: nofree norecurse nounwind
-define dso_local void @vec_add(i32 signext %N, double* noalias nocapture %c, double* noalias nocapture readonly %a, double* noalias nocapture readonly %b, double* noalias nocapture %d) local_unnamed_addr {
-; CHECK-LABEL: @vec_add(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP22:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK-NEXT:    br i1 [[CMP22]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK:       for.body.preheader:
-; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], 1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x double> poison, double 3.000000e+00, i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x double> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x double> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <vscale x 8 x i1> poison, i1 false, i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT14:%.*]] = shufflevector <vscale x 8 x i1> [[BROADCAST_SPLATINSERT13]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[INDEX]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 8 x i64> @llvm.experimental.vector.stepvector.nxv8i64()
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ule <vscale x 8 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP7]], i64 3, i64 3)
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, double* [[TMP6]], i32 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast double* [[TMP11]] to <vscale x 8 x double>*
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i1> [[DOTSPLATINSERT]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x double> @llvm.vp.load.nxv8f64.p0nxv8f64(<vscale x 8 x double>* [[TMP12]], i32 8, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP13]])
-; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP_FCMP:%.*]] = call <vscale x 8 x i1> @llvm.vp.fcmp.nxv8f64(<vscale x 8 x double> [[VP_OP_LOAD]], <vscale x 8 x double> [[BROADCAST_SPLAT4]], i8 4, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP14]])
-; CHECK-NEXT:    [[TMP15:%.*]] = select <vscale x 8 x i1> [[DOTSPLAT]], <vscale x 8 x i1> [[VP_OP_FCMP]], <vscale x 8 x i1> zeroinitializer
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 0
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP16]], i32 [[TMP18]]
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <vscale x 8 x double>*
-; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD9:%.*]] = call <vscale x 8 x double> @llvm.vp.load.nxv8f64.p0nxv8f64(<vscale x 8 x double>* [[TMP20]], i32 8, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP21]])
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP24:%.*]] = mul i32 [[TMP23]], 0
-; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP22]], i32 [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = bitcast double* [[TMP25]] to <vscale x 8 x double>*
-; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD12:%.*]] = call <vscale x 8 x double> @llvm.vp.load.nxv8f64.p0nxv8f64(<vscale x 8 x double>* [[TMP26]], i32 8, <vscale x 8 x i1> [[DOTSPLAT]], i32 [[TMP27]])
-; CHECK-NEXT:    [[TMP28:%.*]] = xor <vscale x 8 x i1> [[TMP15]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> undef, i1 true, i32 0), <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP29:%.*]] = select <vscale x 8 x i1> [[TMP5]], <vscale x 8 x i1> [[TMP28]], <vscale x 8 x i1> [[BROADCAST_SPLAT14]]
-; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 8 x double> @llvm.vp.fmul.nxv8f64(<vscale x 8 x double> [[VP_OP_LOAD9]], <vscale x 8 x double> [[VP_OP_LOAD12]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 8 x i1> [[TMP29]], i32 [[TMP30]])
-; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds double, double* [[D:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP33:%.*]] = mul i32 [[TMP32]], 0
-; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP31]], i32 [[TMP33]]
-; CHECK-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <vscale x 8 x double>*
-; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv8f64.p0nxv8f64(<vscale x 8 x double> [[VP_OP]], <vscale x 8 x double>* [[TMP35]], i32 8, <vscale x 8 x i1> [[TMP29]], i32 [[TMP36]])
-; CHECK-NEXT:    [[TMP37:%.*]] = select <vscale x 8 x i1> [[TMP5]], <vscale x 8 x i1> [[TMP15]], <vscale x 8 x i1> [[BROADCAST_SPLAT14]]
-; CHECK-NEXT:    [[TMP38:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    [[VP_OP15:%.*]] = call <vscale x 8 x double> @llvm.vp.fadd.nxv8f64(<vscale x 8 x double> [[VP_OP_LOAD9]], <vscale x 8 x double> [[VP_OP_LOAD12]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 8 x i1> [[TMP37]], i32 [[TMP38]])
-; CHECK-NEXT:    [[TMP39:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP40:%.*]] = mul i32 [[TMP39]], 0
-; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds double, double* [[TMP6]], i32 [[TMP40]]
-; CHECK-NEXT:    [[TMP42:%.*]] = bitcast double* [[TMP41]] to <vscale x 8 x double>*
-; CHECK-NEXT:    [[TMP43:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv8f64.p0nxv8f64(<vscale x 8 x double> [[VP_OP15]], <vscale x 8 x double>* [[TMP42]], i32 8, <vscale x 8 x i1> [[TMP37]], i32 [[TMP43]])
-; CHECK-NEXT:    [[TMP44:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP45:%.*]] = mul i64 [[TMP44]], 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
-; CHECK-NEXT:    [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[TMP46]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[WIDE_TRIP_COUNT]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.cond.cleanup.loopexit:
-; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
-; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    ret void
-; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[C]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP47:%.*]] = load double, double* [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt double [[TMP47]], 3.000000e+00
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP48:%.*]] = load double, double* [[ARRAYIDX3]], align 8
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP49:%.*]] = load double, double* [[ARRAYIDX5]], align 8
-; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP48]], [[TMP49]]
-; CHECK-NEXT:    store double [[ADD]], double* [[ARRAYIDX]], align 8
-; CHECK-NEXT:    br label [[FOR_INC]]
-; CHECK:       if.else:
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP48]], [[TMP49]]
-; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds double, double* [[D]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store double [[MUL]], double* [[ARRAYIDX13]], align 8
-; CHECK-NEXT:    br label [[FOR_INC]]
-; CHECK:       for.inc:
-; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
-;
-; CHECK1-LABEL: @vec_add(
-; CHECK1-NEXT:  entry:
-; CHECK1-NEXT:    [[CMP22:%.*]] = icmp sgt i32 [[N:%.*]], 0
-; CHECK1-NEXT:    br i1 [[CMP22]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK1:       for.body.preheader:
-; CHECK1-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK1:       vector.ph:
-; CHECK1-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], 1
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 1 x double> poison, double 3.000000e+00, i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 1 x double> [[BROADCAST_SPLATINSERT3]], <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <vscale x 1 x i1> poison, i1 false, i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLAT14:%.*]] = shufflevector <vscale x 1 x i1> [[BROADCAST_SPLATINSERT13]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK1:       vector.body:
-; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[INDEX]], i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vector.stepvector.nxv1i64()
-; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 0
-; CHECK1-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 0
-; CHECK1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 1
-; CHECK1-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], [[TMP3]]
-; CHECK1-NEXT:    [[TMP5:%.*]] = icmp ule <vscale x 1 x i64> [[INDUCTION]], [[BROADCAST_SPLAT]]
-; CHECK1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP7:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDEX]]
-; CHECK1-NEXT:    [[TMP8:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP7]], i64 3, i64 0)
-; CHECK1-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 0
-; CHECK1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, double* [[TMP6]], i32 [[TMP10]]
-; CHECK1-NEXT:    [[TMP12:%.*]] = bitcast double* [[TMP11]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 1 x i1> [[DOTSPLATINSERT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP12]], i32 8, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP13]])
-; CHECK1-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP_FCMP:%.*]] = call <vscale x 1 x i1> @llvm.vp.fcmp.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[BROADCAST_SPLAT4]], i8 4, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP14]])
-; CHECK1-NEXT:    [[TMP15:%.*]] = select <vscale x 1 x i1> [[DOTSPLAT]], <vscale x 1 x i1> [[VP_OP_FCMP]], <vscale x 1 x i1> zeroinitializer
-; CHECK1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 0
-; CHECK1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP16]], i32 [[TMP18]]
-; CHECK1-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD9:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP20]], i32 8, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP21]])
-; CHECK1-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP24:%.*]] = mul i32 [[TMP23]], 0
-; CHECK1-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP22]], i32 [[TMP24]]
-; CHECK1-NEXT:    [[TMP26:%.*]] = bitcast double* [[TMP25]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[TMP27:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD12:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP26]], i32 8, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP27]])
-; CHECK1-NEXT:    [[TMP28:%.*]] = xor <vscale x 1 x i1> [[TMP15]], shufflevector (<vscale x 1 x i1> insertelement (<vscale x 1 x i1> undef, i1 true, i32 0), <vscale x 1 x i1> undef, <vscale x 1 x i32> zeroinitializer)
-; CHECK1-NEXT:    [[TMP29:%.*]] = select <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP28]], <vscale x 1 x i1> [[BROADCAST_SPLAT14]]
-; CHECK1-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD9]], <vscale x 1 x double> [[VP_OP_LOAD12]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[TMP29]], i32 [[TMP30]])
-; CHECK1-NEXT:    [[TMP31:%.*]] = getelementptr inbounds double, double* [[D:%.*]], i64 [[TMP4]]
-; CHECK1-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP33:%.*]] = mul i32 [[TMP32]], 0
-; CHECK1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP31]], i32 [[TMP33]]
-; CHECK1-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP]], <vscale x 1 x double>* [[TMP35]], i32 8, <vscale x 1 x i1> [[TMP29]], i32 [[TMP36]])
-; CHECK1-NEXT:    [[TMP37:%.*]] = select <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP15]], <vscale x 1 x i1> [[BROADCAST_SPLAT14]]
-; CHECK1-NEXT:    [[TMP38:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    [[VP_OP15:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD9]], <vscale x 1 x double> [[VP_OP_LOAD12]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[TMP37]], i32 [[TMP38]])
-; CHECK1-NEXT:    [[TMP39:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP40:%.*]] = mul i32 [[TMP39]], 0
-; CHECK1-NEXT:    [[TMP41:%.*]] = getelementptr inbounds double, double* [[TMP6]], i32 [[TMP40]]
-; CHECK1-NEXT:    [[TMP42:%.*]] = bitcast double* [[TMP41]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[TMP43:%.*]] = trunc i64 [[TMP8]] to i32
-; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP15]], <vscale x 1 x double>* [[TMP42]], i32 8, <vscale x 1 x i1> [[TMP37]], i32 [[TMP43]])
-; CHECK1-NEXT:    [[TMP44:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
-; CHECK1-NEXT:    [[TMP45:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[TMP45]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
-; CHECK1:       middle.block:
-; CHECK1-NEXT:    br i1 true, label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; CHECK1:       scalar.ph:
-; CHECK1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[WIDE_TRIP_COUNT]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK1-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK1:       for.cond.cleanup.loopexit:
-; CHECK1-NEXT:    br label [[FOR_COND_CLEANUP]]
-; CHECK1:       for.cond.cleanup:
-; CHECK1-NEXT:    ret void
-; CHECK1:       for.body:
-; CHECK1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[C]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP46:%.*]] = load double, double* [[ARRAYIDX]], align 8
-; CHECK1-NEXT:    [[CMP1:%.*]] = fcmp olt double [[TMP46]], 3.000000e+00
-; CHECK1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP47:%.*]] = load double, double* [[ARRAYIDX3]], align 8
-; CHECK1-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP48:%.*]] = load double, double* [[ARRAYIDX5]], align 8
-; CHECK1-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK1:       if.then:
-; CHECK1-NEXT:    [[ADD:%.*]] = fadd double [[TMP47]], [[TMP48]]
-; CHECK1-NEXT:    store double [[ADD]], double* [[ARRAYIDX]], align 8
-; CHECK1-NEXT:    br label [[FOR_INC]]
-; CHECK1:       if.else:
-; CHECK1-NEXT:    [[MUL:%.*]] = fmul double [[TMP47]], [[TMP48]]
-; CHECK1-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds double, double* [[D]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    store double [[MUL]], double* [[ARRAYIDX13]], align 8
-; CHECK1-NEXT:    br label [[FOR_INC]]
-; CHECK1:       for.inc:
-; CHECK1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
-;
-entry:
-  %cmp22 = icmp sgt i32 %N, 0
-  br i1 %cmp22, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:                               ; preds = %entry
-  %wide.trip.count = zext i32 %N to i64
-  br label %for.body
-
-for.cond.cleanup.loopexit:                        ; preds = %for.inc
-  br label %for.cond.cleanup
-
-for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
-  ret void
-
-for.body:                                         ; preds = %for.inc, %for.body.preheader
-  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]
-  %arrayidx = getelementptr inbounds double, double* %c, i64 %indvars.iv
-  %0 = load double, double* %arrayidx, align 8
-  %cmp1 = fcmp olt double %0, 3.000000e+00
-  %arrayidx3 = getelementptr inbounds double, double* %a, i64 %indvars.iv
-  %1 = load double, double* %arrayidx3, align 8
-  %arrayidx5 = getelementptr inbounds double, double* %b, i64 %indvars.iv
-  %2 = load double, double* %arrayidx5, align 8
-  br i1 %cmp1, label %if.then, label %if.else
-
-if.then:                                          ; preds = %for.body
-  %add = fadd double %1, %2
-  store double %add, double* %arrayidx, align 8
-  br label %for.inc
-
-if.else:                                          ; preds = %for.body
-  %mul = fmul double %1, %2
-  %arrayidx13 = getelementptr inbounds double, double* %d, i64 %indvars.iv
-  store double %mul, double* %arrayidx13, align 8
-  br label %for.inc
-
-for.inc:                                          ; preds = %if.then, %if.else
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
-  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
-}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vec-reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vec-reg-usage.ll
deleted file mode 100644
index 3483dc0b9da8ae046a67037e03ccf05e0026364d..0000000000000000000000000000000000000000
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vec-reg-usage.ll
+++ /dev/null
@@ -1,376 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:   -prefer-predicate-over-epilogue=predicate-dont-vectorize -S \
-; RUN:   -loop-vectorize -vector-register-width-factor=8 -debug-only=loop-vectorize 2>&1 \
-; RUN:   | FileCheck %s
-; RUN: opt < %s -mtriple riscv64 -mattr +m,+a,+f,+d,+experimental-v \
-; RUN:   -prefer-predicate-over-epilogue=predicate-dont-vectorize -S \
-; RUN:   -loop-vectorize -debug-only=loop-vectorize 2>&1 \
-; RUN:   | FileCheck %s --check-prefix=CHECK1
-
-;Check that register usage for VF=8 is >32, thus selecting MaxVF=4.
-;void highRegUage(int N, double *restrict c, double *restrict a,
-;                 double *restrict b, double *restrict e, double *restrict f,
-;                 double *restrict g) {
-;  int i;
-;  for (i = 1; i < N + 1; i++) {
-;    c[i] = a[i] + b[i];
-;    e[i] = a[i] - b[i] + c[i];
-;    f[i] = a[i] * b[i] + c[i] - e[i];
-;    g[i] = a[i] * c[i] + b[i] + e[i] - f[i];
-;  }
-;}
-
-; CHECK-LABEL: highRegUage
-; CHECK: LV: The Smallest and Widest types: 64 / 64 bits.
-; CHECK: LV: The Widest register safe to use is: 512 bits.
-; CHECK: LV(REG): Calculating max register usage:
-; CHECK: LV(REG): VF = vscale x 2
-; CHECK-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 10 registers
-; CHECK-NEXT: LV(REG): Found invariant usage: 1 item
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 2 registers
-; CHECK-NEXT: LV(REG): VF = vscale x 4
-; CHECK-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 20 registers
-; CHECK-NEXT: LV(REG): Found invariant usage: 1 item
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 4 registers
-; CHECK-NEXT: LV(REG): VF = vscale x 8
-; CHECK-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 40 registers
-; CHECK-NEXT: LV(REG): Found invariant usage: 1 item
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 8 registers
-; CHECK: LV: Selecting VF: vscale x 4.
-;
-; ModuleID = './vec-reg-usage.c'
-source_filename = "./vec-reg-usage.c"
-target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128-v128:128:128-v256:128:128-v512:128:128-v1024:128:128"
-target triple = "riscv64-unknown-linux-gnu"
-
-; Function Attrs: nofree norecurse nounwind
-define dso_local void @highRegUage(i32 signext %N, double* noalias nocapture %c, double* noalias nocapture readonly %a, double* noalias nocapture readonly %b, double* noalias nocapture %e, double* noalias nocapture %f, double* noalias nocapture %g) local_unnamed_addr {
-; CHECK-LABEL: @highRegUage(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP_NOT75:%.*]] = icmp slt i32 [[N:%.*]], 1
-; CHECK-NEXT:    br i1 [[CMP_NOT75]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
-; CHECK:       for.body.preheader:
-; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i32 [[N]], 1
-; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[IND_END:%.*]] = add i64 1, [[TMP1]]
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 4 x i64> @llvm.experimental.vector.stepvector.nxv4i64()
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[STEPVECTOR:%.*]] = call <vscale x 4 x i64> @llvm.experimental.vector.stepvector.nxv4i64()
-; CHECK-NEXT:    [[VEC_IV:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT4]], [[STEPVECTOR]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ule <vscale x 4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP1]], [[INDEX]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP9]], i64 3, i64 2)
-; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 0
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds double, double* [[TMP8]], i32 [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = bitcast double* [[TMP13]] to <vscale x 4 x double>*
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 true, i32 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[DOTSPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP15:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x double> @llvm.vp.load.nxv4f64.p0nxv4f64(<vscale x 4 x double>* [[TMP14]], i32 8, <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP15]])
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 0
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP16]], i32 [[TMP18]]
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <vscale x 4 x double>*
-; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP_LOAD7:%.*]] = call <vscale x 4 x double> @llvm.vp.load.nxv4f64.p0nxv4f64(<vscale x 4 x double>* [[TMP20]], i32 8, <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP21]])
-; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD]], <vscale x 4 x double> [[VP_OP_LOAD7]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP22]])
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP24:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP25:%.*]] = mul i32 [[TMP24]], 0
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds double, double* [[TMP23]], i32 [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = bitcast double* [[TMP26]] to <vscale x 4 x double>*
-; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> [[VP_OP]], <vscale x 4 x double>* [[TMP27]], i32 8, <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP28]])
-; CHECK-NEXT:    [[TMP29:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP14:%.*]] = call <vscale x 4 x double> @llvm.vp.fsub.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD]], <vscale x 4 x double> [[VP_OP_LOAD7]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP29]])
-; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP17:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP14]], <vscale x 4 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP30]])
-; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds double, double* [[E:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP33:%.*]] = mul i32 [[TMP32]], 0
-; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP31]], i32 [[TMP33]]
-; CHECK-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <vscale x 4 x double>*
-; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> [[VP_OP17]], <vscale x 4 x double>* [[TMP35]], i32 8, <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP36]])
-; CHECK-NEXT:    [[TMP37:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP22:%.*]] = call <vscale x 4 x double> @llvm.vp.fmul.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD]], <vscale x 4 x double> [[VP_OP_LOAD7]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP37]])
-; CHECK-NEXT:    [[TMP38:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP25:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP22]], <vscale x 4 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP38]])
-; CHECK-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP28:%.*]] = call <vscale x 4 x double> @llvm.vp.fsub.nxv4f64(<vscale x 4 x double> [[VP_OP25]], <vscale x 4 x double> [[VP_OP17]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP39]])
-; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds double, double* [[F:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP41:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP42:%.*]] = mul i32 [[TMP41]], 0
-; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr inbounds double, double* [[TMP40]], i32 [[TMP42]]
-; CHECK-NEXT:    [[TMP44:%.*]] = bitcast double* [[TMP43]] to <vscale x 4 x double>*
-; CHECK-NEXT:    [[TMP45:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> [[VP_OP28]], <vscale x 4 x double>* [[TMP44]], i32 8, <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP45]])
-; CHECK-NEXT:    [[TMP46:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP33:%.*]] = call <vscale x 4 x double> @llvm.vp.fmul.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD]], <vscale x 4 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP46]])
-; CHECK-NEXT:    [[TMP47:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP36:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP_LOAD7]], <vscale x 4 x double> [[VP_OP33]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP47]])
-; CHECK-NEXT:    [[TMP48:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP39:%.*]] = call <vscale x 4 x double> @llvm.vp.fadd.nxv4f64(<vscale x 4 x double> [[VP_OP17]], <vscale x 4 x double> [[VP_OP36]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP48]])
-; CHECK-NEXT:    [[TMP49:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    [[VP_OP42:%.*]] = call <vscale x 4 x double> @llvm.vp.fsub.nxv4f64(<vscale x 4 x double> [[VP_OP39]], <vscale x 4 x double> [[VP_OP28]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP49]])
-; CHECK-NEXT:    [[TMP50:%.*]] = getelementptr inbounds double, double* [[G:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP51:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP52:%.*]] = mul i32 [[TMP51]], 0
-; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds double, double* [[TMP50]], i32 [[TMP52]]
-; CHECK-NEXT:    [[TMP54:%.*]] = bitcast double* [[TMP53]] to <vscale x 4 x double>*
-; CHECK-NEXT:    [[TMP55:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT:    call void @llvm.vp.store.nxv4f64.p0nxv4f64(<vscale x 4 x double> [[VP_OP42]], <vscale x 4 x double>* [[TMP54]], i32 8, <vscale x 4 x i1> [[DOTSPLAT]], i32 [[TMP55]])
-; CHECK-NEXT:    [[TMP56:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP57:%.*]] = mul i64 [[TMP56]], 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]]
-; CHECK-NEXT:    [[TMP58:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[TMP58]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP59:%.*]] = load double, double* [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP60:%.*]] = load double, double* [[ARRAYIDX2]], align 8
-; CHECK-NEXT:    [[ADD3:%.*]] = fadd double [[TMP59]], [[TMP60]]
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store double [[ADD3]], double* [[ARRAYIDX5]], align 8
-; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[TMP59]], [[TMP60]]
-; CHECK-NEXT:    [[ADD12:%.*]] = fadd double [[SUB]], [[ADD3]]
-; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds double, double* [[E]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store double [[ADD12]], double* [[ARRAYIDX14]], align 8
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP59]], [[TMP60]]
-; CHECK-NEXT:    [[ADD21:%.*]] = fadd double [[MUL]], [[ADD3]]
-; CHECK-NEXT:    [[SUB24:%.*]] = fsub double [[ADD21]], [[ADD12]]
-; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds double, double* [[F]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store double [[SUB24]], double* [[ARRAYIDX26]], align 8
-; CHECK-NEXT:    [[MUL31:%.*]] = fmul double [[TMP59]], [[ADD3]]
-; CHECK-NEXT:    [[ADD34:%.*]] = fadd double [[TMP60]], [[MUL31]]
-; CHECK-NEXT:    [[ADD37:%.*]] = fadd double [[ADD12]], [[ADD34]]
-; CHECK-NEXT:    [[SUB40:%.*]] = fsub double [[ADD37]], [[SUB24]]
-; CHECK-NEXT:    [[ARRAYIDX42:%.*]] = getelementptr inbounds double, double* [[G]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    store double [[SUB40]], double* [[ARRAYIDX42]], align 8
-; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
-; CHECK:       for.end.loopexit:
-; CHECK-NEXT:    br label [[FOR_END]]
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
-;
-; CHECK1-LABEL: @highRegUage(
-; CHECK1-NEXT:  entry:
-; CHECK1-NEXT:    [[CMP_NOT75:%.*]] = icmp slt i32 [[N:%.*]], 1
-; CHECK1-NEXT:    br i1 [[CMP_NOT75]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
-; CHECK1:       for.body.preheader:
-; CHECK1-NEXT:    [[TMP0:%.*]] = add nuw i32 [[N]], 1
-; CHECK1-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK1-NEXT:    [[TMP1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
-; CHECK1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK1:       vector.ph:
-; CHECK1-NEXT:    [[IND_END:%.*]] = add i64 1, [[TMP1]]
-; CHECK1-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK1:       vector.body:
-; CHECK1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK1-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[OFFSET_IDX]], i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVEC_BASE:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vector.stepvector.nxv1i64()
-; CHECK1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 1 x i64> [[BROADCAST_SPLAT2]], [[STEPVEC_BASE]]
-; CHECK1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 0
-; CHECK1-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], 0
-; CHECK1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
-; CHECK1-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
-; CHECK1-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[INDEX]], i32 0
-; CHECK1-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[STEPVECTOR:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vector.stepvector.nxv1i64()
-; CHECK1-NEXT:    [[VEC_IV:%.*]] = add <vscale x 1 x i64> [[BROADCAST_SPLAT4]], [[STEPVECTOR]]
-; CHECK1-NEXT:    [[TMP7:%.*]] = icmp ule <vscale x 1 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP6]]
-; CHECK1-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP1]], [[INDEX]]
-; CHECK1-NEXT:    [[TMP10:%.*]] = call i64 @llvm.epi.vsetvl(i64 [[TMP9]], i64 3, i64 0)
-; CHECK1-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 0
-; CHECK1-NEXT:    [[TMP13:%.*]] = getelementptr inbounds double, double* [[TMP8]], i32 [[TMP12]]
-; CHECK1-NEXT:    [[TMP14:%.*]] = bitcast double* [[TMP13]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; CHECK1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 1 x i1> [[DOTSPLATINSERT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK1-NEXT:    [[TMP15:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP14]], i32 8, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP15]])
-; CHECK1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP6]]
-; CHECK1-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 0
-; CHECK1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP16]], i32 [[TMP18]]
-; CHECK1-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP_LOAD7:%.*]] = call <vscale x 1 x double> @llvm.vp.load.nxv1f64.p0nxv1f64(<vscale x 1 x double>* [[TMP20]], i32 8, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP21]])
-; CHECK1-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[VP_OP_LOAD7]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP22]])
-; CHECK1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[TMP6]]
-; CHECK1-NEXT:    [[TMP24:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP25:%.*]] = mul i32 [[TMP24]], 0
-; CHECK1-NEXT:    [[TMP26:%.*]] = getelementptr inbounds double, double* [[TMP23]], i32 [[TMP25]]
-; CHECK1-NEXT:    [[TMP27:%.*]] = bitcast double* [[TMP26]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP]], <vscale x 1 x double>* [[TMP27]], i32 8, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP28]])
-; CHECK1-NEXT:    [[TMP29:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP14:%.*]] = call <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[VP_OP_LOAD7]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP29]])
-; CHECK1-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP17:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP14]], <vscale x 1 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP30]])
-; CHECK1-NEXT:    [[TMP31:%.*]] = getelementptr inbounds double, double* [[E:%.*]], i64 [[TMP6]]
-; CHECK1-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP33:%.*]] = mul i32 [[TMP32]], 0
-; CHECK1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP31]], i32 [[TMP33]]
-; CHECK1-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP17]], <vscale x 1 x double>* [[TMP35]], i32 8, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP36]])
-; CHECK1-NEXT:    [[TMP37:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP22:%.*]] = call <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[VP_OP_LOAD7]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP37]])
-; CHECK1-NEXT:    [[TMP38:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP25:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP22]], <vscale x 1 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP38]])
-; CHECK1-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP28:%.*]] = call <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> [[VP_OP25]], <vscale x 1 x double> [[VP_OP17]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP39]])
-; CHECK1-NEXT:    [[TMP40:%.*]] = getelementptr inbounds double, double* [[F:%.*]], i64 [[TMP6]]
-; CHECK1-NEXT:    [[TMP41:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP42:%.*]] = mul i32 [[TMP41]], 0
-; CHECK1-NEXT:    [[TMP43:%.*]] = getelementptr inbounds double, double* [[TMP40]], i32 [[TMP42]]
-; CHECK1-NEXT:    [[TMP44:%.*]] = bitcast double* [[TMP43]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[TMP45:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP28]], <vscale x 1 x double>* [[TMP44]], i32 8, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP45]])
-; CHECK1-NEXT:    [[TMP46:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP33:%.*]] = call <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD]], <vscale x 1 x double> [[VP_OP]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP46]])
-; CHECK1-NEXT:    [[TMP47:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP36:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP_LOAD7]], <vscale x 1 x double> [[VP_OP33]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP47]])
-; CHECK1-NEXT:    [[TMP48:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP39:%.*]] = call <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> [[VP_OP17]], <vscale x 1 x double> [[VP_OP36]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP48]])
-; CHECK1-NEXT:    [[TMP49:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    [[VP_OP42:%.*]] = call <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> [[VP_OP39]], <vscale x 1 x double> [[VP_OP28]], metadata !"round.tonearest", metadata !"fpexcept.ignore", <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP49]])
-; CHECK1-NEXT:    [[TMP50:%.*]] = getelementptr inbounds double, double* [[G:%.*]], i64 [[TMP6]]
-; CHECK1-NEXT:    [[TMP51:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK1-NEXT:    [[TMP52:%.*]] = mul i32 [[TMP51]], 0
-; CHECK1-NEXT:    [[TMP53:%.*]] = getelementptr inbounds double, double* [[TMP50]], i32 [[TMP52]]
-; CHECK1-NEXT:    [[TMP54:%.*]] = bitcast double* [[TMP53]] to <vscale x 1 x double>*
-; CHECK1-NEXT:    [[TMP55:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK1-NEXT:    call void @llvm.vp.store.nxv1f64.p0nxv1f64(<vscale x 1 x double> [[VP_OP42]], <vscale x 1 x double>* [[TMP54]], i32 8, <vscale x 1 x i1> [[DOTSPLAT]], i32 [[TMP55]])
-; CHECK1-NEXT:    [[TMP56:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]]
-; CHECK1-NEXT:    [[TMP57:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP1]]
-; CHECK1-NEXT:    br i1 [[TMP57]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
-; CHECK1:       middle.block:
-; CHECK1-NEXT:    br i1 true, label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
-; CHECK1:       scalar.ph:
-; CHECK1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[FOR_BODY_PREHEADER]] ]
-; CHECK1-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK1:       for.body:
-; CHECK1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP58:%.*]] = load double, double* [[ARRAYIDX]], align 8
-; CHECK1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    [[TMP59:%.*]] = load double, double* [[ARRAYIDX2]], align 8
-; CHECK1-NEXT:    [[ADD3:%.*]] = fadd double [[TMP58]], [[TMP59]]
-; CHECK1-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    store double [[ADD3]], double* [[ARRAYIDX5]], align 8
-; CHECK1-NEXT:    [[SUB:%.*]] = fsub double [[TMP58]], [[TMP59]]
-; CHECK1-NEXT:    [[ADD12:%.*]] = fadd double [[SUB]], [[ADD3]]
-; CHECK1-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds double, double* [[E]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    store double [[ADD12]], double* [[ARRAYIDX14]], align 8
-; CHECK1-NEXT:    [[MUL:%.*]] = fmul double [[TMP58]], [[TMP59]]
-; CHECK1-NEXT:    [[ADD21:%.*]] = fadd double [[MUL]], [[ADD3]]
-; CHECK1-NEXT:    [[SUB24:%.*]] = fsub double [[ADD21]], [[ADD12]]
-; CHECK1-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds double, double* [[F]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    store double [[SUB24]], double* [[ARRAYIDX26]], align 8
-; CHECK1-NEXT:    [[MUL31:%.*]] = fmul double [[TMP58]], [[ADD3]]
-; CHECK1-NEXT:    [[ADD34:%.*]] = fadd double [[TMP59]], [[MUL31]]
-; CHECK1-NEXT:    [[ADD37:%.*]] = fadd double [[ADD12]], [[ADD34]]
-; CHECK1-NEXT:    [[SUB40:%.*]] = fsub double [[ADD37]], [[SUB24]]
-; CHECK1-NEXT:    [[ARRAYIDX42:%.*]] = getelementptr inbounds double, double* [[G]], i64 [[INDVARS_IV]]
-; CHECK1-NEXT:    store double [[SUB40]], double* [[ARRAYIDX42]], align 8
-; CHECK1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
-; CHECK1:       for.end.loopexit:
-; CHECK1-NEXT:    br label [[FOR_END]]
-; CHECK1:       for.end:
-; CHECK1-NEXT:    ret void
-;
-entry:
-  %cmp.not75 = icmp slt i32 %N, 1
-  br i1 %cmp.not75, label %for.end, label %for.body.preheader
-
-for.body.preheader:                               ; preds = %entry
-  %0 = add nuw i32 %N, 1
-  %wide.trip.count = zext i32 %0 to i64
-  br label %for.body
-
-for.body:                                         ; preds = %for.body.preheader, %for.body
-  %indvars.iv = phi i64 [ 1, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds double, double* %a, i64 %indvars.iv
-  %1 = load double, double* %arrayidx, align 8
-  %arrayidx2 = getelementptr inbounds double, double* %b, i64 %indvars.iv
-  %2 = load double, double* %arrayidx2, align 8
-  %add3 = fadd double %1, %2
-  %arrayidx5 = getelementptr inbounds double, double* %c, i64 %indvars.iv
-  store double %add3, double* %arrayidx5, align 8
-  %sub = fsub double %1, %2
-  %add12 = fadd double %sub, %add3
-  %arrayidx14 = getelementptr inbounds double, double* %e, i64 %indvars.iv
-  store double %add12, double* %arrayidx14, align 8
-  %mul = fmul double %1, %2
-  %add21 = fadd double %mul, %add3
-  %sub24 = fsub double %add21, %add12
-  %arrayidx26 = getelementptr inbounds double, double* %f, i64 %indvars.iv
-  store double %sub24, double* %arrayidx26, align 8
-  %mul31 = fmul double %1, %add3
-  %add34 = fadd double %2, %mul31
-  %add37 = fadd double %add12, %add34
-  %sub40 = fsub double %add37, %sub24
-  %arrayidx42 = getelementptr inbounds double, double* %g, i64 %indvars.iv
-  store double %sub40, double* %arrayidx42, align 8
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
-  br i1 %exitcond.not, label %for.end.loopexit, label %for.body
-
-for.end.loopexit:                                 ; preds = %for.body
-  br label %for.end
-
-for.end:                                          ; preds = %for.end.loopexit, %entry
-  ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
index 4f3fd288d710511725311ceb8aba9c3503a6774d..631b43c79340d3ba1179003c7acb3ada852ac860 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
@@ -111,7 +111,7 @@ define void @fp_iv_loop1(float* noalias nocapture %A, i32 %N) #0 {
 ; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_3]] = fadd fast <8 x float> [[VEC_IND]], <float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01>
 ; AUTO_VEC-NEXT:    [[NITER_NSUB_3]] = add i64 [[NITER]], -4
 ; AUTO_VEC-NEXT:    [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NSUB_3]], 0
-; AUTO_VEC-NEXT:    br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; AUTO_VEC-NEXT:    br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; AUTO_VEC:       middle.block.unr-lcssa:
 ; AUTO_VEC-NEXT:    [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT_3]], [[VECTOR_BODY]] ]
 ; AUTO_VEC-NEXT:    [[VEC_IND_UNR:%.*]] = phi <8 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00, float 3.000000e+00, float 3.500000e+00, float 4.000000e+00, float 4.500000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_3]], [[VECTOR_BODY]] ]
@@ -140,7 +140,7 @@ define void @fp_iv_loop1(float* noalias nocapture %A, i32 %N) #0 {
 ; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_EPIL]] = fadd fast <8 x float> [[VEC_IND_EPIL]], <float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01>
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0
-; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], [[LOOP2:!llvm.loop !.*]]
+; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], !llvm.loop [[LOOP2:![0-9]+]]
 ; AUTO_VEC:       middle.block:
 ; AUTO_VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[ZEXT]]
 ; AUTO_VEC-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY]]
@@ -152,7 +152,7 @@ define void @fp_iv_loop1(float* noalias nocapture %A, i32 %N) #0 {
 ; AUTO_VEC-NEXT:    [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AUTO_VEC-NEXT:    [[TMP45:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; AUTO_VEC-NEXT:    br i1 [[TMP45]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP4:!llvm.loop !.*]]
+; AUTO_VEC-NEXT:    br i1 [[TMP45]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; AUTO_VEC:       for.end:
 ; AUTO_VEC-NEXT:    ret void
 ;
@@ -259,7 +259,7 @@ define void @fp_iv_loop2(float* noalias nocapture %A, i32 %N) {
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_EPIL]] = add nuw nsw i64 [[INDVARS_IV_EPIL]], 1
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0
-; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY_EPIL]], [[LOOP6:!llvm.loop !.*]]
+; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY_EPIL]], !llvm.loop [[LOOP6:![0-9]+]]
 ; AUTO_VEC:       for.end:
 ; AUTO_VEC-NEXT:    ret void
 ;
@@ -381,7 +381,7 @@ define double @external_use_with_fast_math(double* %a, i64 %n) {
 ; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_3]] = fadd fast <4 x double> [[VEC_IND]], <double 1.920000e+02, double 1.920000e+02, double 1.920000e+02, double 1.920000e+02>
 ; AUTO_VEC-NEXT:    [[NITER_NSUB_3]] = add i64 [[NITER]], -4
 ; AUTO_VEC-NEXT:    [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NSUB_3]], 0
-; AUTO_VEC-NEXT:    br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], [[LOOP7:!llvm.loop !.*]]
+; AUTO_VEC-NEXT:    br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; AUTO_VEC:       middle.block.unr-lcssa:
 ; AUTO_VEC-NEXT:    [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT_3]], [[VECTOR_BODY]] ]
 ; AUTO_VEC-NEXT:    [[VEC_IND_UNR:%.*]] = phi <4 x double> [ <double 0.000000e+00, double 3.000000e+00, double 6.000000e+00, double 9.000000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_3]], [[VECTOR_BODY]] ]
@@ -410,7 +410,7 @@ define double @external_use_with_fast_math(double* %a, i64 %n) {
 ; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_EPIL]] = fadd fast <4 x double> [[VEC_IND_EPIL]], <double 4.800000e+01, double 4.800000e+01, double 4.800000e+01, double 4.800000e+01>
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0
-; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], [[LOOP8:!llvm.loop !.*]]
+; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], !llvm.loop [[LOOP8:![0-9]+]]
 ; AUTO_VEC:       middle.block:
 ; AUTO_VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
 ; AUTO_VEC-NEXT:    [[TMP45:%.*]] = add nsw i64 [[N_VEC]], -1
@@ -425,7 +425,7 @@ define double @external_use_with_fast_math(double* %a, i64 %n) {
 ; AUTO_VEC-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; AUTO_VEC-NEXT:    [[J_NEXT]] = fadd fast double [[J]], 3.000000e+00
 ; AUTO_VEC-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[I_NEXT]], [[SMAX]]
-; AUTO_VEC-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP9:!llvm.loop !.*]]
+; AUTO_VEC-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; AUTO_VEC:       for.end:
 ; AUTO_VEC-NEXT:    [[J_LCSSA:%.*]] = phi double [ [[TMP46]], [[MIDDLE_BLOCK]] ], [ [[J]], [[FOR_BODY]] ]
 ; AUTO_VEC-NEXT:    ret double [[J_LCSSA]]
@@ -514,7 +514,7 @@ define double @external_use_without_fast_math(double* %a, i64 %n) {
 ; AUTO_VEC-NEXT:    [[J_NEXT_EPIL]] = fadd double [[J_EPIL]], 3.000000e+00
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0
-; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY_EPIL]], [[LOOP10:!llvm.loop !.*]]
+; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY_EPIL]], !llvm.loop [[LOOP10:![0-9]+]]
 ; AUTO_VEC:       for.end:
 ; AUTO_VEC-NEXT:    [[J_LCSSA:%.*]] = phi double [ [[J_LCSSA_PH]], [[FOR_END_UNR_LCSSA]] ], [ [[J_EPIL]], [[FOR_BODY_EPIL]] ]
 ; AUTO_VEC-NEXT:    ret double [[J_LCSSA]]
@@ -536,3 +536,174 @@ for.end:
   %t1 = phi double [ %j, %for.body ]
   ret double %t1
 }
+
+;;  void fadd_induction(float *p, unsigned N) {
+;;    float x = 1.0f;
+;;    for (unsigned i=0; i!=N; ++i) {
+;;      p[i] = p[i] + x;
+;;      x += 42.0f;
+;;    }
+;;  }
+
+define void @fadd_reassoc_FMF(float* nocapture %p, i32 %N) {
+; AUTO_VEC-LABEL: @fadd_reassoc_FMF(
+; AUTO_VEC-NEXT:  entry:
+; AUTO_VEC-NEXT:    [[CMP_NOT11:%.*]] = icmp eq i32 [[N:%.*]], 0
+; AUTO_VEC-NEXT:    br i1 [[CMP_NOT11]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; AUTO_VEC:       for.body.preheader:
+; AUTO_VEC-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; AUTO_VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 32
+; AUTO_VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
+; AUTO_VEC:       vector.ph:
+; AUTO_VEC-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967264
+; AUTO_VEC-NEXT:    [[CAST_CRD:%.*]] = sitofp i64 [[N_VEC]] to float
+; AUTO_VEC-NEXT:    [[TMP1:%.*]] = fmul reassoc float [[CAST_CRD]], 4.200000e+01
+; AUTO_VEC-NEXT:    [[IND_END:%.*]] = fadd reassoc float [[TMP1]], 1.000000e+00
+; AUTO_VEC-NEXT:    [[TMP2:%.*]] = add nsw i64 [[N_VEC]], -32
+; AUTO_VEC-NEXT:    [[TMP3:%.*]] = lshr exact i64 [[TMP2]], 5
+; AUTO_VEC-NEXT:    [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
+; AUTO_VEC-NEXT:    [[XTRAITER:%.*]] = and i64 [[TMP4]], 1
+; AUTO_VEC-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[TMP2]], 0
+; AUTO_VEC-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK_UNR_LCSSA:%.*]], label [[VECTOR_PH_NEW:%.*]]
+; AUTO_VEC:       vector.ph.new:
+; AUTO_VEC-NEXT:    [[UNROLL_ITER:%.*]] = and i64 [[TMP4]], 1152921504606846974
+; AUTO_VEC-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AUTO_VEC:       vector.body:
+; AUTO_VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH_NEW]] ], [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[VEC_IND:%.*]] = phi <8 x float> [ <float 1.000000e+00, float 4.300000e+01, float 8.500000e+01, float 1.270000e+02, float 1.690000e+02, float 2.110000e+02, float 2.530000e+02, float 2.950000e+02>, [[VECTOR_PH_NEW]] ], [ [[VEC_IND_NEXT_1:%.*]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[NITER:%.*]] = phi i64 [ [[UNROLL_ITER]], [[VECTOR_PH_NEW]] ], [ [[NITER_NSUB_1:%.*]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[STEP_ADD:%.*]] = fadd reassoc <8 x float> [[VEC_IND]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD2:%.*]] = fadd reassoc <8 x float> [[STEP_ADD]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD3:%.*]] = fadd reassoc <8 x float> [[STEP_ADD2]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, float* [[P:%.*]], i64 [[INDEX]]
+; AUTO_VEC-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x float>, <8 x float>* [[TMP7]], align 4
+; AUTO_VEC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, float* [[TMP6]], i64 8
+; AUTO_VEC-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP8]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD5:%.*]] = load <8 x float>, <8 x float>* [[TMP9]], align 4
+; AUTO_VEC-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, float* [[TMP6]], i64 16
+; AUTO_VEC-NEXT:    [[TMP11:%.*]] = bitcast float* [[TMP10]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD6:%.*]] = load <8 x float>, <8 x float>* [[TMP11]], align 4
+; AUTO_VEC-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, float* [[TMP6]], i64 24
+; AUTO_VEC-NEXT:    [[TMP13:%.*]] = bitcast float* [[TMP12]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD7:%.*]] = load <8 x float>, <8 x float>* [[TMP13]], align 4
+; AUTO_VEC-NEXT:    [[TMP14:%.*]] = fadd reassoc <8 x float> [[VEC_IND]], [[WIDE_LOAD]]
+; AUTO_VEC-NEXT:    [[TMP15:%.*]] = fadd reassoc <8 x float> [[STEP_ADD]], [[WIDE_LOAD5]]
+; AUTO_VEC-NEXT:    [[TMP16:%.*]] = fadd reassoc <8 x float> [[STEP_ADD2]], [[WIDE_LOAD6]]
+; AUTO_VEC-NEXT:    [[TMP17:%.*]] = fadd reassoc <8 x float> [[STEP_ADD3]], [[WIDE_LOAD7]]
+; AUTO_VEC-NEXT:    [[TMP18:%.*]] = bitcast float* [[TMP6]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP14]], <8 x float>* [[TMP18]], align 4
+; AUTO_VEC-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP8]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP15]], <8 x float>* [[TMP19]], align 4
+; AUTO_VEC-NEXT:    [[TMP20:%.*]] = bitcast float* [[TMP10]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP16]], <8 x float>* [[TMP20]], align 4
+; AUTO_VEC-NEXT:    [[TMP21:%.*]] = bitcast float* [[TMP12]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP17]], <8 x float>* [[TMP21]], align 4
+; AUTO_VEC-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 32
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT:%.*]] = fadd reassoc <8 x float> [[STEP_ADD3]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD_1:%.*]] = fadd reassoc <8 x float> [[VEC_IND_NEXT]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD2_1:%.*]] = fadd reassoc <8 x float> [[STEP_ADD_1]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD3_1:%.*]] = fadd reassoc <8 x float> [[STEP_ADD2_1]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDEX_NEXT]]
+; AUTO_VEC-NEXT:    [[TMP23:%.*]] = bitcast float* [[TMP22]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD_1:%.*]] = load <8 x float>, <8 x float>* [[TMP23]], align 4
+; AUTO_VEC-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, float* [[TMP22]], i64 8
+; AUTO_VEC-NEXT:    [[TMP25:%.*]] = bitcast float* [[TMP24]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD5_1:%.*]] = load <8 x float>, <8 x float>* [[TMP25]], align 4
+; AUTO_VEC-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, float* [[TMP22]], i64 16
+; AUTO_VEC-NEXT:    [[TMP27:%.*]] = bitcast float* [[TMP26]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD6_1:%.*]] = load <8 x float>, <8 x float>* [[TMP27]], align 4
+; AUTO_VEC-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, float* [[TMP22]], i64 24
+; AUTO_VEC-NEXT:    [[TMP29:%.*]] = bitcast float* [[TMP28]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD7_1:%.*]] = load <8 x float>, <8 x float>* [[TMP29]], align 4
+; AUTO_VEC-NEXT:    [[TMP30:%.*]] = fadd reassoc <8 x float> [[VEC_IND_NEXT]], [[WIDE_LOAD_1]]
+; AUTO_VEC-NEXT:    [[TMP31:%.*]] = fadd reassoc <8 x float> [[STEP_ADD_1]], [[WIDE_LOAD5_1]]
+; AUTO_VEC-NEXT:    [[TMP32:%.*]] = fadd reassoc <8 x float> [[STEP_ADD2_1]], [[WIDE_LOAD6_1]]
+; AUTO_VEC-NEXT:    [[TMP33:%.*]] = fadd reassoc <8 x float> [[STEP_ADD3_1]], [[WIDE_LOAD7_1]]
+; AUTO_VEC-NEXT:    [[TMP34:%.*]] = bitcast float* [[TMP22]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP30]], <8 x float>* [[TMP34]], align 4
+; AUTO_VEC-NEXT:    [[TMP35:%.*]] = bitcast float* [[TMP24]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP31]], <8 x float>* [[TMP35]], align 4
+; AUTO_VEC-NEXT:    [[TMP36:%.*]] = bitcast float* [[TMP26]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP32]], <8 x float>* [[TMP36]], align 4
+; AUTO_VEC-NEXT:    [[TMP37:%.*]] = bitcast float* [[TMP28]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP33]], <8 x float>* [[TMP37]], align 4
+; AUTO_VEC-NEXT:    [[INDEX_NEXT_1]] = add i64 [[INDEX]], 64
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_1]] = fadd reassoc <8 x float> [[STEP_ADD3_1]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[NITER_NSUB_1]] = add i64 [[NITER]], -2
+; AUTO_VEC-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NSUB_1]], 0
+; AUTO_VEC-NEXT:    br i1 [[NITER_NCMP_1]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; AUTO_VEC:       middle.block.unr-lcssa:
+; AUTO_VEC-NEXT:    [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT_1]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[VEC_IND_UNR:%.*]] = phi <8 x float> [ <float 1.000000e+00, float 4.300000e+01, float 8.500000e+01, float 1.270000e+02, float 1.690000e+02, float 2.110000e+02, float 2.530000e+02, float 2.950000e+02>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_1]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0
+; AUTO_VEC-NEXT:    br i1 [[LCMP_MOD_NOT]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY_EPIL:%.*]]
+; AUTO_VEC:       vector.body.epil:
+; AUTO_VEC-NEXT:    [[STEP_ADD_EPIL:%.*]] = fadd reassoc <8 x float> [[VEC_IND_UNR]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD2_EPIL:%.*]] = fadd reassoc <8 x float> [[STEP_ADD_EPIL]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD3_EPIL:%.*]] = fadd reassoc <8 x float> [[STEP_ADD2_EPIL]], <float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02, float 3.360000e+02>
+; AUTO_VEC-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDEX_UNR]]
+; AUTO_VEC-NEXT:    [[TMP39:%.*]] = bitcast float* [[TMP38]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD_EPIL:%.*]] = load <8 x float>, <8 x float>* [[TMP39]], align 4
+; AUTO_VEC-NEXT:    [[TMP40:%.*]] = getelementptr inbounds float, float* [[TMP38]], i64 8
+; AUTO_VEC-NEXT:    [[TMP41:%.*]] = bitcast float* [[TMP40]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD5_EPIL:%.*]] = load <8 x float>, <8 x float>* [[TMP41]], align 4
+; AUTO_VEC-NEXT:    [[TMP42:%.*]] = getelementptr inbounds float, float* [[TMP38]], i64 16
+; AUTO_VEC-NEXT:    [[TMP43:%.*]] = bitcast float* [[TMP42]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD6_EPIL:%.*]] = load <8 x float>, <8 x float>* [[TMP43]], align 4
+; AUTO_VEC-NEXT:    [[TMP44:%.*]] = getelementptr inbounds float, float* [[TMP38]], i64 24
+; AUTO_VEC-NEXT:    [[TMP45:%.*]] = bitcast float* [[TMP44]] to <8 x float>*
+; AUTO_VEC-NEXT:    [[WIDE_LOAD7_EPIL:%.*]] = load <8 x float>, <8 x float>* [[TMP45]], align 4
+; AUTO_VEC-NEXT:    [[TMP46:%.*]] = fadd reassoc <8 x float> [[VEC_IND_UNR]], [[WIDE_LOAD_EPIL]]
+; AUTO_VEC-NEXT:    [[TMP47:%.*]] = fadd reassoc <8 x float> [[STEP_ADD_EPIL]], [[WIDE_LOAD5_EPIL]]
+; AUTO_VEC-NEXT:    [[TMP48:%.*]] = fadd reassoc <8 x float> [[STEP_ADD2_EPIL]], [[WIDE_LOAD6_EPIL]]
+; AUTO_VEC-NEXT:    [[TMP49:%.*]] = fadd reassoc <8 x float> [[STEP_ADD3_EPIL]], [[WIDE_LOAD7_EPIL]]
+; AUTO_VEC-NEXT:    [[TMP50:%.*]] = bitcast float* [[TMP38]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP46]], <8 x float>* [[TMP50]], align 4
+; AUTO_VEC-NEXT:    [[TMP51:%.*]] = bitcast float* [[TMP40]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP47]], <8 x float>* [[TMP51]], align 4
+; AUTO_VEC-NEXT:    [[TMP52:%.*]] = bitcast float* [[TMP42]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP48]], <8 x float>* [[TMP52]], align 4
+; AUTO_VEC-NEXT:    [[TMP53:%.*]] = bitcast float* [[TMP44]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[TMP49]], <8 x float>* [[TMP53]], align 4
+; AUTO_VEC-NEXT:    br label [[MIDDLE_BLOCK]]
+; AUTO_VEC:       middle.block:
+; AUTO_VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
+; AUTO_VEC-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
+; AUTO_VEC:       for.cond.cleanup:
+; AUTO_VEC-NEXT:    ret void
+; AUTO_VEC:       for.body:
+; AUTO_VEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; AUTO_VEC-NEXT:    [[X_012:%.*]] = phi float [ [[ADD3:%.*]], [[FOR_BODY]] ], [ 1.000000e+00, [[FOR_BODY_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
+; AUTO_VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDVARS_IV]]
+; AUTO_VEC-NEXT:    [[TMP54:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; AUTO_VEC-NEXT:    [[ADD:%.*]] = fadd reassoc float [[X_012]], [[TMP54]]
+; AUTO_VEC-NEXT:    store float [[ADD]], float* [[ARRAYIDX]], align 4
+; AUTO_VEC-NEXT:    [[ADD3]] = fadd reassoc float [[X_012]], 4.200000e+01
+; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AUTO_VEC-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[TMP0]]
+; AUTO_VEC-NEXT:    br i1 [[CMP_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+;
+entry:
+  %cmp.not11 = icmp eq i32 %N, 0
+  br i1 %cmp.not11, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = zext i32 %N to i64
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %x.012 = phi float [ 1.000000e+00, %for.body.preheader ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %p, i64 %indvars.iv
+  %1 = load float, float* %arrayidx, align 4
+  %add = fadd reassoc float %x.012, %1
+  store float %add, float* %arrayidx, align 4
+  %add3 = fadd reassoc float %x.012, 4.200000e+01
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp.not = icmp eq i64 %indvars.iv.next, %0
+  br i1 %cmp.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
index 7b9ba8a36a6f7d8ecbcec12d0310afd629a64b2f..d0ba0b8cbb6131c917725d6788ddf05a587f8c16 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
@@ -92,7 +92,7 @@ define i32 @test_explicit_pred(i64 %len) {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD2]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP40]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP40]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP37]], [[TMP36]]
 ; CHECK-NEXT:    [[BIN_RDX19:%.*]] = add <4 x i32> [[TMP38]], [[BIN_RDX]]
@@ -118,7 +118,7 @@ define i32 @test_explicit_pred(i64 %len) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP2:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP41]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -258,7 +258,7 @@ define i32 @test_explicit_pred_generic(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP4:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
@@ -285,7 +285,7 @@ define i32 @test_explicit_pred_generic(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP5:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -445,7 +445,7 @@ define i32 @test_invariant_address(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP103]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI6]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP104:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP104]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP6:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP104]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP101]], [[TMP100]]
 ; CHECK-NEXT:    [[BIN_RDX7:%.*]] = add <4 x i32> [[TMP102]], [[BIN_RDX]]
@@ -471,7 +471,7 @@ define i32 @test_invariant_address(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP7:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP105]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -771,7 +771,7 @@ define i32 @test_step_narrower_than_access(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP183]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI36]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP184:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP184]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP8:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP184]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP181]], [[TMP180]]
 ; CHECK-NEXT:    [[BIN_RDX37:%.*]] = add <4 x i32> [[TMP182]], [[BIN_RDX]]
@@ -800,7 +800,7 @@ define i32 @test_step_narrower_than_access(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP9:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP185]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -948,7 +948,7 @@ define i32 @test_max_trip_count(i64 %len, i1* %test_base, i64 %n) {
 ; CHECK-NEXT:    [[TMP84]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP85:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP85]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP10:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP85]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP82]], [[TMP81]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX]]
@@ -975,7 +975,7 @@ define i32 @test_max_trip_count(i64 %len, i1* %test_base, i64 %n) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], [[MIN_N]]
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP11:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP86]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -1121,7 +1121,7 @@ define i32 @test_non_zero_start(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 3072
-; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP12:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
@@ -1148,7 +1148,7 @@ define i32 @test_non_zero_start(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP13:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -1472,7 +1472,7 @@ define i32 @test_non_unit_stride(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP151]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI36]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP152:%.*]] = icmp eq i64 [[INDEX_NEXT]], 2048
-; CHECK-NEXT:    br i1 [[TMP152]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP14:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP152]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP149]], [[TMP148]]
 ; CHECK-NEXT:    [[BIN_RDX37:%.*]] = add <4 x i32> [[TMP150]], [[BIN_RDX]]
@@ -1499,7 +1499,7 @@ define i32 @test_non_unit_stride(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4093
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP15:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP153]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -1639,7 +1639,7 @@ define i32 @neg_off_by_many(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP16:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
@@ -1666,7 +1666,7 @@ define i32 @neg_off_by_many(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP17:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -1806,7 +1806,7 @@ define i32 @neg_off_by_one_iteration(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP18:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
@@ -1833,7 +1833,7 @@ define i32 @neg_off_by_one_iteration(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP19:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -1973,7 +1973,7 @@ define i32 @neg_off_by_one_byte(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP20:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
@@ -2000,7 +2000,7 @@ define i32 @neg_off_by_one_byte(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP21:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -2149,7 +2149,7 @@ define i32 @test_constant_max(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP84]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP85:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP85]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP22:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP85]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP82]], [[TMP81]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX]]
@@ -2176,7 +2176,7 @@ define i32 @test_constant_max(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], [[MIN]]
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP23:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP86]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -2323,7 +2323,7 @@ define i32 @test_allocsize(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP24:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
@@ -2350,7 +2350,7 @@ define i32 @test_allocsize(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP25:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -2491,7 +2491,7 @@ define i32 @test_allocsize_array(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP26:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
@@ -2518,7 +2518,7 @@ define i32 @test_allocsize_array(i64 %len, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP27:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
@@ -2669,7 +2669,7 @@ define i32 @test_allocsize_cond_deref(i1 %allzero, i1* %test_base) {
 ; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP28:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
 ; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
@@ -2696,7 +2696,7 @@ define i32 @test_allocsize_cond_deref(i1 %allzero, i1* %test_base) {
 ; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
 ; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
 ; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
-; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP29:!llvm.loop !.*]]
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK:       loop_exit:
 ; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr48340.ll b/llvm/test/Transforms/LoopVectorize/X86/pr48340.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9a63501e091f64eed26d1dd63901e15d8f99ad23
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr48340.ll
@@ -0,0 +1,54 @@
+; RUN: opt -loop-vectorize --force-vector-width=4 --force-vector-interleave=0 -S -o - < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%0 = type { i32 }
+%1 = type { i64 }
+
+define void @foo(i64* %p, i64* %p.last) unnamed_addr #0 {
+; CHECK-LABEL: @foo(
+; CHECK: vector.body:
+; CHECK:         [[WIDE_MASKED_GATHER0:%.*]] = call <4 x %0*> @llvm.masked.gather.v4p0s_s.v4p0p0s_s.0(<4 x %0**> [[TMP5:%.*]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x %0*> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <4 x %0*> @llvm.masked.gather.v4p0s_s.v4p0p0s_s.0(<4 x %0**> [[TMP6:%.*]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x %0*> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <4 x %0*> @llvm.masked.gather.v4p0s_s.v4p0p0s_s.0(<4 x %0**> [[TMP7:%.*]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x %0*> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <4 x %0*> @llvm.masked.gather.v4p0s_s.v4p0p0s_s.0(<4 x %0**> [[TMP8:%.*]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x %0*> undef)
+entry:
+  br label %loop
+
+loop:
+  %p2 = phi i64* [ %p, %entry ], [ %p.inc, %loop ]
+  %p.inc = getelementptr inbounds i64, i64* %p2, i64 2
+  %p3 = bitcast i64* %p2 to %0**
+  %v = load %0*, %0** %p3, align 8
+  %b = icmp eq i64* %p.inc, %p.last
+  br i1 %b, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bar(i64* %p, i64* %p.last) unnamed_addr #0 {
+; CHECK-LABEL: @bar(
+; CHECK: vector.body:
+; CHECK:         [[WIDE_MASKED_GATHER0:%.*]] = call <4 x %1*> @llvm.masked.gather.v4p0s_s.v4p0p0s_s.1(<4 x %1**> [[TMP5:%.*]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x %1*> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <4 x %1*> @llvm.masked.gather.v4p0s_s.v4p0p0s_s.1(<4 x %1**> [[TMP6:%.*]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x %1*> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <4 x %1*> @llvm.masked.gather.v4p0s_s.v4p0p0s_s.1(<4 x %1**> [[TMP7:%.*]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x %1*> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <4 x %1*> @llvm.masked.gather.v4p0s_s.v4p0p0s_s.1(<4 x %1**> [[TMP8:%.*]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x %1*> undef)
+entry:
+  br label %loop
+
+loop:
+  %p2 = phi i64* [ %p, %entry ], [ %p.inc, %loop ]
+  %p.inc = getelementptr inbounds i64, i64* %p2, i64 2
+  %p3 = bitcast i64* %p2 to %1**
+  %v = load %1*, %1** %p3, align 8
+  %b = icmp eq i64* %p.inc, %p.last
+  br i1 %b, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+attributes #0 = { "target-cpu"="skylake" }
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
index 9bebe7767414a647ef32d327de0a9e18abc0f4af..f2d93df76e741922e102b7ae0684a085cddb6f5e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
@@ -160,109 +160,100 @@ define void @example2(i32 %n, i32 %x) optsize {
 ; CHECK:       pred.load.if:
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = load i32, i32* [[TMP25]], align 4
-; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x i32> poison, i32 [[TMP26]], i32 0
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
-; CHECK-NEXT:    [[TMP28:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY9]] ], [ [[TMP27]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP23]], i32 1
-; CHECK-NEXT:    br i1 [[TMP29]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]]
+; CHECK-NEXT:    [[TMP27:%.*]] = phi i32 [ poison, [[VECTOR_BODY9]] ], [ [[TMP26]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x i1> [[TMP23]], i32 1
+; CHECK-NEXT:    br i1 [[TMP28]], label [[PRED_LOAD_IF30:%.*]], label [[PRED_LOAD_CONTINUE31:%.*]]
 ; CHECK:       pred.load.if30:
-; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP20]]
-; CHECK-NEXT:    [[TMP31:%.*]] = load i32, i32* [[TMP30]], align 4
-; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP31]], i32 1
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP30:%.*]] = load i32, i32* [[TMP29]], align 4
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE31]]
 ; CHECK:       pred.load.continue31:
-; CHECK-NEXT:    [[TMP33:%.*]] = phi <4 x i32> [ [[TMP28]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP32]], [[PRED_LOAD_IF30]] ]
-; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x i1> [[TMP23]], i32 2
-; CHECK-NEXT:    br i1 [[TMP34]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33:%.*]]
+; CHECK-NEXT:    [[TMP31:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP30]], [[PRED_LOAD_IF30]] ]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x i1> [[TMP23]], i32 2
+; CHECK-NEXT:    br i1 [[TMP32]], label [[PRED_LOAD_IF32:%.*]], label [[PRED_LOAD_CONTINUE33:%.*]]
 ; CHECK:       pred.load.if32:
-; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP21]]
-; CHECK-NEXT:    [[TMP36:%.*]] = load i32, i32* [[TMP35]], align 4
-; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP36]], i32 2
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i32, i32* [[TMP33]], align 4
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE33]]
 ; CHECK:       pred.load.continue33:
-; CHECK-NEXT:    [[TMP38:%.*]] = phi <4 x i32> [ [[TMP33]], [[PRED_LOAD_CONTINUE31]] ], [ [[TMP37]], [[PRED_LOAD_IF32]] ]
-; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <4 x i1> [[TMP23]], i32 3
-; CHECK-NEXT:    br i1 [[TMP39]], label [[PRED_LOAD_IF34:%.*]], label [[PRED_LOAD_CONTINUE35:%.*]]
+; CHECK-NEXT:    [[TMP35:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE31]] ], [ [[TMP34]], [[PRED_LOAD_IF32]] ]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <4 x i1> [[TMP23]], i32 3
+; CHECK-NEXT:    br i1 [[TMP36]], label [[PRED_LOAD_IF34:%.*]], label [[PRED_LOAD_CONTINUE35:%.*]]
 ; CHECK:       pred.load.if34:
-; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP22]]
-; CHECK-NEXT:    [[TMP41:%.*]] = load i32, i32* [[TMP40]], align 4
-; CHECK-NEXT:    [[TMP42:%.*]] = insertelement <4 x i32> [[TMP38]], i32 [[TMP41]], i32 3
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP38:%.*]] = load i32, i32* [[TMP37]], align 4
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE35]]
 ; CHECK:       pred.load.continue35:
-; CHECK-NEXT:    [[TMP43:%.*]] = phi <4 x i32> [ [[TMP38]], [[PRED_LOAD_CONTINUE33]] ], [ [[TMP42]], [[PRED_LOAD_IF34]] ]
-; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <4 x i1> [[TMP23]], i32 0
-; CHECK-NEXT:    br i1 [[TMP44]], label [[PRED_LOAD_IF36:%.*]], label [[PRED_LOAD_CONTINUE37:%.*]]
+; CHECK-NEXT:    [[TMP39:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE33]] ], [ [[TMP38]], [[PRED_LOAD_IF34]] ]
+; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <4 x i1> [[TMP23]], i32 0
+; CHECK-NEXT:    br i1 [[TMP40]], label [[PRED_LOAD_IF36:%.*]], label [[PRED_LOAD_CONTINUE37:%.*]]
 ; CHECK:       pred.load.if36:
-; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP46:%.*]] = load i32, i32* [[TMP45]], align 4
-; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i32> poison, i32 [[TMP46]], i32 0
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load i32, i32* [[TMP41]], align 4
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE37]]
 ; CHECK:       pred.load.continue37:
-; CHECK-NEXT:    [[TMP48:%.*]] = phi <4 x i32> [ poison, [[PRED_LOAD_CONTINUE35]] ], [ [[TMP47]], [[PRED_LOAD_IF36]] ]
-; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <4 x i1> [[TMP23]], i32 1
-; CHECK-NEXT:    br i1 [[TMP49]], label [[PRED_LOAD_IF38:%.*]], label [[PRED_LOAD_CONTINUE39:%.*]]
+; CHECK-NEXT:    [[TMP43:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE35]] ], [ [[TMP42]], [[PRED_LOAD_IF36]] ]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <4 x i1> [[TMP23]], i32 1
+; CHECK-NEXT:    br i1 [[TMP44]], label [[PRED_LOAD_IF38:%.*]], label [[PRED_LOAD_CONTINUE39:%.*]]
 ; CHECK:       pred.load.if38:
-; CHECK-NEXT:    [[TMP50:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP20]]
-; CHECK-NEXT:    [[TMP51:%.*]] = load i32, i32* [[TMP50]], align 4
-; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i32> [[TMP48]], i32 [[TMP51]], i32 1
+; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP46:%.*]] = load i32, i32* [[TMP45]], align 4
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE39]]
 ; CHECK:       pred.load.continue39:
-; CHECK-NEXT:    [[TMP53:%.*]] = phi <4 x i32> [ [[TMP48]], [[PRED_LOAD_CONTINUE37]] ], [ [[TMP52]], [[PRED_LOAD_IF38]] ]
-; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <4 x i1> [[TMP23]], i32 2
-; CHECK-NEXT:    br i1 [[TMP54]], label [[PRED_LOAD_IF40:%.*]], label [[PRED_LOAD_CONTINUE41:%.*]]
+; CHECK-NEXT:    [[TMP47:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE37]] ], [ [[TMP46]], [[PRED_LOAD_IF38]] ]
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <4 x i1> [[TMP23]], i32 2
+; CHECK-NEXT:    br i1 [[TMP48]], label [[PRED_LOAD_IF40:%.*]], label [[PRED_LOAD_CONTINUE41:%.*]]
 ; CHECK:       pred.load.if40:
-; CHECK-NEXT:    [[TMP55:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP21]]
-; CHECK-NEXT:    [[TMP56:%.*]] = load i32, i32* [[TMP55]], align 4
-; CHECK-NEXT:    [[TMP57:%.*]] = insertelement <4 x i32> [[TMP53]], i32 [[TMP56]], i32 2
+; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i32, i32* [[TMP49]], align 4
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE41]]
 ; CHECK:       pred.load.continue41:
-; CHECK-NEXT:    [[TMP58:%.*]] = phi <4 x i32> [ [[TMP53]], [[PRED_LOAD_CONTINUE39]] ], [ [[TMP57]], [[PRED_LOAD_IF40]] ]
-; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <4 x i1> [[TMP23]], i32 3
-; CHECK-NEXT:    br i1 [[TMP59]], label [[PRED_LOAD_IF42:%.*]], label [[PRED_LOAD_CONTINUE43:%.*]]
+; CHECK-NEXT:    [[TMP51:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE39]] ], [ [[TMP50]], [[PRED_LOAD_IF40]] ]
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <4 x i1> [[TMP23]], i32 3
+; CHECK-NEXT:    br i1 [[TMP52]], label [[PRED_LOAD_IF42:%.*]], label [[PRED_LOAD_CONTINUE43:%.*]]
 ; CHECK:       pred.load.if42:
-; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP22]]
-; CHECK-NEXT:    [[TMP61:%.*]] = load i32, i32* [[TMP60]], align 4
-; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i32> [[TMP58]], i32 [[TMP61]], i32 3
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP54:%.*]] = load i32, i32* [[TMP53]], align 4
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE43]]
 ; CHECK:       pred.load.continue43:
-; CHECK-NEXT:    [[TMP63:%.*]] = phi <4 x i32> [ [[TMP58]], [[PRED_LOAD_CONTINUE41]] ], [ [[TMP62]], [[PRED_LOAD_IF42]] ]
-; CHECK-NEXT:    [[TMP64:%.*]] = and <4 x i32> [[TMP63]], [[TMP43]]
-; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <4 x i1> [[TMP23]], i32 0
-; CHECK-NEXT:    br i1 [[TMP65]], label [[PRED_STORE_IF44:%.*]], label [[PRED_STORE_CONTINUE45:%.*]]
+; CHECK-NEXT:    [[TMP55:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE41]] ], [ [[TMP54]], [[PRED_LOAD_IF42]] ]
+; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <4 x i1> [[TMP23]], i32 0
+; CHECK-NEXT:    br i1 [[TMP56]], label [[PRED_STORE_IF44:%.*]], label [[PRED_STORE_CONTINUE45:%.*]]
 ; CHECK:       pred.store.if44:
-; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP67:%.*]] = extractelement <4 x i32> [[TMP64]], i32 0
-; CHECK-NEXT:    store i32 [[TMP67]], i32* [[TMP66]], align 4
+; CHECK-NEXT:    [[TMP57:%.*]] = and i32 [[TMP43]], [[TMP27]]
+; CHECK-NEXT:    [[TMP58:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    store i32 [[TMP57]], i32* [[TMP58]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE45]]
 ; CHECK:       pred.store.continue45:
-; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <4 x i1> [[TMP23]], i32 1
-; CHECK-NEXT:    br i1 [[TMP68]], label [[PRED_STORE_IF46:%.*]], label [[PRED_STORE_CONTINUE47:%.*]]
+; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <4 x i1> [[TMP23]], i32 1
+; CHECK-NEXT:    br i1 [[TMP59]], label [[PRED_STORE_IF46:%.*]], label [[PRED_STORE_CONTINUE47:%.*]]
 ; CHECK:       pred.store.if46:
-; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP20]]
-; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <4 x i32> [[TMP64]], i32 1
-; CHECK-NEXT:    store i32 [[TMP70]], i32* [[TMP69]], align 4
+; CHECK-NEXT:    [[TMP60:%.*]] = and i32 [[TMP47]], [[TMP31]]
+; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP20]]
+; CHECK-NEXT:    store i32 [[TMP60]], i32* [[TMP61]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE47]]
 ; CHECK:       pred.store.continue47:
-; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <4 x i1> [[TMP23]], i32 2
-; CHECK-NEXT:    br i1 [[TMP71]], label [[PRED_STORE_IF48:%.*]], label [[PRED_STORE_CONTINUE49:%.*]]
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <4 x i1> [[TMP23]], i32 2
+; CHECK-NEXT:    br i1 [[TMP62]], label [[PRED_STORE_IF48:%.*]], label [[PRED_STORE_CONTINUE49:%.*]]
 ; CHECK:       pred.store.if48:
-; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP21]]
-; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <4 x i32> [[TMP64]], i32 2
-; CHECK-NEXT:    store i32 [[TMP73]], i32* [[TMP72]], align 4
+; CHECK-NEXT:    [[TMP63:%.*]] = and i32 [[TMP51]], [[TMP35]]
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP21]]
+; CHECK-NEXT:    store i32 [[TMP63]], i32* [[TMP64]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE49]]
 ; CHECK:       pred.store.continue49:
-; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <4 x i1> [[TMP23]], i32 3
-; CHECK-NEXT:    br i1 [[TMP74]], label [[PRED_STORE_IF50:%.*]], label [[PRED_STORE_CONTINUE51]]
+; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <4 x i1> [[TMP23]], i32 3
+; CHECK-NEXT:    br i1 [[TMP65]], label [[PRED_STORE_IF50:%.*]], label [[PRED_STORE_CONTINUE51]]
 ; CHECK:       pred.store.if50:
-; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP22]]
-; CHECK-NEXT:    [[TMP76:%.*]] = extractelement <4 x i32> [[TMP64]], i32 3
-; CHECK-NEXT:    store i32 [[TMP76]], i32* [[TMP75]], align 4
+; CHECK-NEXT:    [[TMP66:%.*]] = and i32 [[TMP55]], [[TMP39]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[TMP22]]
+; CHECK-NEXT:    store i32 [[TMP66]], i32* [[TMP67]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE51]]
 ; CHECK:       pred.store.continue51:
 ; CHECK-NEXT:    [[INDEX_NEXT15]] = add i64 [[INDEX14]], 4
-; CHECK-NEXT:    [[TMP77:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC13]]
-; CHECK-NEXT:    br i1 [[TMP77]], label [[MIDDLE_BLOCK7:%.*]], label [[VECTOR_BODY9]], [[LOOP5:!llvm.loop !.*]]
+; CHECK-NEXT:    [[TMP68:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC13]]
+; CHECK-NEXT:    br i1 [[TMP68]], label [[MIDDLE_BLOCK7:%.*]], label [[VECTOR_BODY9]], [[LOOP5:!llvm.loop !.*]]
 ; CHECK:       middle.block7:
 ; CHECK-NEXT:    br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH8]]
 ; CHECK:       scalar.ph8:
diff --git a/llvm/test/Transforms/LoopVectorize/float-induction.ll b/llvm/test/Transforms/LoopVectorize/float-induction.ll
index 291c01efc3abec1a0cb5113638a295c5f4aa269b..bc4b4b02497ed2408afedc5497d0befe0366c061 100644
--- a/llvm/test/Transforms/LoopVectorize/float-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/float-induction.ll
@@ -235,10 +235,43 @@ define void @fp_iv_loop1_reassoc_FMF(float %init, float* noalias nocapture %A, i
 ; VEC1_INTERL2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
 ; VEC1_INTERL2:       for.body.lr.ph:
 ; VEC1_INTERL2-NEXT:    [[FPINC:%.*]] = load float, float* @fp_inc, align 4
+; VEC1_INTERL2-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; VEC1_INTERL2-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; VEC1_INTERL2-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; VEC1_INTERL2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[TMP0]], 0
+; VEC1_INTERL2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; VEC1_INTERL2:       vector.ph:
+; VEC1_INTERL2-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934590
+; VEC1_INTERL2-NEXT:    [[CAST_CRD:%.*]] = sitofp i64 [[N_VEC]] to float
+; VEC1_INTERL2-NEXT:    [[TMP3:%.*]] = fmul reassoc float [[FPINC]], [[CAST_CRD]]
+; VEC1_INTERL2-NEXT:    [[IND_END:%.*]] = fsub reassoc float [[INIT:%.*]], [[TMP3]]
+; VEC1_INTERL2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; VEC1_INTERL2:       vector.body:
+; VEC1_INTERL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VEC1_INTERL2-NEXT:    [[INDUCTION2:%.*]] = or i64 [[INDEX]], 1
+; VEC1_INTERL2-NEXT:    [[TMP4:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC1_INTERL2-NEXT:    [[TMP5:%.*]] = fmul reassoc float [[FPINC]], [[TMP4]]
+; VEC1_INTERL2-NEXT:    [[OFFSET_IDX:%.*]] = fsub reassoc float [[INIT]], [[TMP5]]
+; VEC1_INTERL2-NEXT:    [[TMP6:%.*]] = fmul reassoc float [[FPINC]], 0.000000e+00
+; VEC1_INTERL2-NEXT:    [[TMP7:%.*]] = fsub reassoc float [[OFFSET_IDX]], [[TMP6]]
+; VEC1_INTERL2-NEXT:    [[TMP8:%.*]] = fsub reassoc float [[OFFSET_IDX]], [[FPINC]]
+; VEC1_INTERL2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDUCTION2]]
+; VEC1_INTERL2-NEXT:    store float [[TMP7]], float* [[TMP9]], align 4
+; VEC1_INTERL2-NEXT:    store float [[TMP8]], float* [[TMP10]], align 4
+; VEC1_INTERL2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; VEC1_INTERL2-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VEC1_INTERL2-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VEC1_INTERL2:       middle.block:
+; VEC1_INTERL2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; VEC1_INTERL2-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; VEC1_INTERL2:       scalar.ph:
+; VEC1_INTERL2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH]] ]
+; VEC1_INTERL2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi float [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[INIT]], [[FOR_BODY_LR_PH]] ]
 ; VEC1_INTERL2-NEXT:    br label [[FOR_BODY:%.*]]
 ; VEC1_INTERL2:       for.body:
-; VEC1_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; VEC1_INTERL2-NEXT:    [[X_05:%.*]] = phi float [ [[INIT:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; VEC1_INTERL2-NEXT:    [[X_05:%.*]] = phi float [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; VEC1_INTERL2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[INDVARS_IV]]
 ; VEC1_INTERL2-NEXT:    store float [[X_05]], float* [[ARRAYIDX]], align 4
 ; VEC1_INTERL2-NEXT:    [[ADD]] = fsub reassoc float [[X_05]], [[FPINC]]
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index 6aa385d1df8d6cf549a500db86b07153c4db6047..6393c0babb70a1dcc6969c9fc99d1e9507a7ef97 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -36,12 +36,13 @@ for.end:
 }
 
 ; Check for crash exposed by D76992.
-; CHECK:       N0 [label =
-; CHECK-NEXT:    "loop:\n" +
-; CHECK-NEXT:      "WIDEN-INDUCTION %iv = phi 0, %iv.next\l" +
-; CHECK-NEXT:      "WIDEN ir<%cond0> = icmp ir<%iv>, ir<13>\l" +
-; CHECK-NEXT:      "WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20>\l"
-; CHECK-NEXT:  ]
+; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: loop:
+; CHECK-NEXT:   WIDEN-INDUCTION %iv = phi 0, %iv.next
+; CHECK-NEXT:   WIDEN ir<%cond0> = icmp ir<%iv>, ir<13>
+; CHECK-NEXT:   WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20>
+; CHECK-NEXT: No successor
+; CHECK-NEXT: }
 define void @test() {
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/irregular_type.ll b/llvm/test/Transforms/LoopVectorize/irregular_type.ll
new file mode 100644
index 0000000000000000000000000000000000000000..167a1a101e6f96a2c25ea291b2a33443f3beec8b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/irregular_type.ll
@@ -0,0 +1,27 @@
+; RUN: opt %s -loop-vectorize -force-vector-width=4 -S | FileCheck %s
+
+; Ensure the array loads/stores are not optimized into vector operations when
+; the element type has padding bits.
+
+; CHECK: foo
+; CHECK: vector.body
+; CHECK-NOT: load <4 x i7>
+; CHECK-NOT: store <4 x i7>
+; CHECK: for.body
+define void @foo(i7* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i7, i7* %a, i64 %indvars.iv
+  %0 = load i7, i7* %arrayidx, align 1
+  %sub = add nuw nsw i7 %0, 0
+  store i7 %sub, i7* %arrayidx, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, %n
+  br i1 %cmp, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7d8d18dcfdaa4ef415640cbda616be6b4d137158
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
@@ -0,0 +1,40 @@
+; REQUIRES: asserts
+
+; RUN: opt -loop-vectorize -debug-only=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -vplan-print-in-dot-format -disable-output %s 2>&1 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Verify that -vplan-print-in-dot-format option works.
+
+define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+; CHECK:       N0 [label =
+; CHECK-NEXT:    "for.body:\l" +
+; CHECK-NEXT:    "  WIDEN-INDUCTION %iv = phi %iv.next, 0\l" +
+; CHECK-NEXT:    "  CLONE ir\<%arrayidx\> = getelementptr ir\<%y\>, ir\<%iv\>\l" +
+; CHECK-NEXT:    "  WIDEN ir\<%lv\> = load ir\<%arrayidx\>\l" +
+; CHECK-NEXT:    "  WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>)\l" +
+; CHECK-NEXT:    "  CLONE ir\<%arrayidx2\> = getelementptr ir\<%x\>, ir\<%iv\>\l" +
+; CHECK-NEXT:    "  WIDEN store ir\<%arrayidx2\>, ir\<%call\>\l" +
+; CHECK-NEXT:    "No successors\l"
+; CHECK-NEXT:  ]
+;
+entry:
+  %cmp6 = icmp sgt i64 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %iv
+  %lv = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.sqrt.f32(float %lv) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %iv
+  store float %call, float* %arrayidx2, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.sqrt.f32(float) nounwind readnone
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 1f649f3dc206e6c0c8b298a2c787dff4c8620ad8..dcee58df8ad7a2ea835756be4029c30587a07550 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -7,16 +7,18 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; Tests for printing VPlans.
 
 define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
-; CHECK: N0 [label =
-; CHECK-NEXT: "for.body:\n" +
-; CHECK-NEXT:       "WIDEN-INDUCTION %iv = phi %iv.next, 0\l" +
-; CHECK-NEXT:       "CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>\l" +
-; CHECK-NEXT:       "WIDEN ir<%lv> = load ir<%arrayidx>\l" +
-; CHECK-NEXT:       "WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>)\l" +
-; CHECK-NEXT:       "CLONE ir<%arrayidx2> = getelementptr ir<%x>, ir<%iv>\l" +
-; CHECK-NEXT:       "WIDEN store ir<%arrayidx2>, ir<%call>\l"
-; CHECK-NEXT:   ]
-
+; CHECK-LABEL: Checking a loop in "print_call_and_memory"
+; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: for.body:
+; CHECK-NEXT:   WIDEN-INDUCTION %iv = phi %iv.next, 0
+; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
+; CHECK-NEXT:   WIDEN ir<%lv> = load ir<%arrayidx>
+; CHECK-NEXT:   WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>)
+; CHECK-NEXT:   CLONE ir<%arrayidx2> = getelementptr ir<%x>, ir<%iv>
+; CHECK-NEXT:   WIDEN store ir<%arrayidx2>, ir<%call>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
 entry:
   %cmp6 = icmp sgt i64 %n, 0
   br i1 %cmp6, label %for.body, label %for.end
@@ -37,18 +39,20 @@ for.end:                                          ; preds = %for.body, %entry
 }
 
 define void @print_widen_gep_and_select(i64 %n, float* noalias %y, float* noalias %x, float* %z) nounwind uwtable {
-; CHECK: N0 [label =
-; CHECK-NEXT: "for.body:\n" +
-; CHECK-NEXT:      "WIDEN-INDUCTION %iv = phi %iv.next, 0\l" +
-; CHECK-NEXT:      "WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>\l" +
-; CHECK-NEXT:      "WIDEN ir<%lv> = load ir<%arrayidx>\l" +
-; CHECK-NEXT:      "WIDEN ir<%cmp> = icmp ir<%arrayidx>, ir<%z>\l" +
-; CHECK-NEXT:      "WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>\l" +
-; CHECK-NEXT:      "WIDEN ir<%add> = fadd ir<%lv>, ir<%sel>\l" +
-; CHECK-NEXT:      "CLONE ir<%arrayidx2> = getelementptr ir<%x>, ir<%iv>\l" +
-; CHECK-NEXT:      "WIDEN store ir<%arrayidx2>, ir<%add>\l"
-; CHECK-NEXT:   ]
-
+; CHECK-LABEL: Checking a loop in "print_widen_gep_and_select"
+; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: for.body:
+; CHECK-NEXT:   WIDEN-INDUCTION %iv = phi %iv.next, 0
+; CHECK-NEXT:   WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
+; CHECK-NEXT:   WIDEN ir<%lv> = load ir<%arrayidx>
+; CHECK-NEXT:   WIDEN ir<%cmp> = icmp ir<%arrayidx>, ir<%z>
+; CHECK-NEXT:   WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>
+; CHECK-NEXT:   WIDEN ir<%add> = fadd ir<%lv>, ir<%sel>
+; CHECK-NEXT:   CLONE ir<%arrayidx2> = getelementptr ir<%x>, ir<%iv>
+; CHECK-NEXT:   WIDEN store ir<%arrayidx2>, ir<%add>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
 entry:
   %cmp6 = icmp sgt i64 %n, 0
   br i1 %cmp6, label %for.body, label %for.end
@@ -71,15 +75,17 @@ for.end:                                          ; preds = %for.body, %entry
 }
 
 define float @print_reduction(i64 %n, float* noalias %y) {
-; CHECK: N0 [label =
-; CHECK-NEXT: "for.body:\n" +
-; CHECK-NEXT:       "WIDEN-INDUCTION %iv = phi %iv.next, 0\l" +
-; CHECK-NEXT:       "WIDEN-PHI %red = phi %red.next, 0.000000e+00\l" +
-; CHECK-NEXT:       "CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>\l" +
-; CHECK-NEXT:       "WIDEN ir<%lv> = load ir<%arrayidx>\l" +
-; CHECK-NEXT:       "REDUCE ir<%red.next> = ir<%red> + reduce.fadd (ir<%lv>)\l"
-; CHECK-NEXT:   ]
-
+; CHECK-LABEL: Checking a loop in "print_reduction"
+; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: for.body:
+; CHECK-NEXT:   WIDEN-INDUCTION %iv = phi %iv.next, 0
+; CHECK-NEXT:   WIDEN-PHI %red = phi %red.next, 0.000000e+00
+; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
+; CHECK-NEXT:   WIDEN ir<%lv> = load ir<%arrayidx>
+; CHECK-NEXT:   REDUCE ir<%red.next> = ir<%red> + reduce.fadd (ir<%lv>)
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
 entry:
   br label %for.body
 
@@ -98,36 +104,41 @@ for.end:                                          ; preds = %for.body, %entry
 }
 
 define void @print_replicate_predicated_phi(i64 %n, i64* %x) {
-; CHECK:       N0 [label =
-; CHECK-NEXT:    "for.body:\n" +
-; CHECK-NEXT:      "WIDEN-INDUCTION %i = phi 0, %i.next\l" +
-; CHECK-NEXT:      "WIDEN ir<%cmp> = icmp ir<%i>, ir<5>\l"
-; CHECK-NEXT:  ]
-;
-; CHECK:       N2 [label =
-; CHECK-NEXT:    "pred.udiv.entry:\n" +
-; CHECK-NEXT:      +
-; CHECK-NEXT:      "BRANCH-ON-MASK ir<%cmp>\l"\l
-; CHECK-NEXT:         "CondBit: ir<%cmp>"
-; CHECK-NEXT:    ]
-;
-; CHECK:       N4 [label =
-; CHECK-NEXT:    "pred.udiv.if:\n" +
-; CHECK-NEXT:      "REPLICATE ir<%tmp4> = udiv ir<%n>, ir<%i> (S->V)\l"
-; CHECK-NEXT:  ]
-;
-; CHECK:       N5 [label =
-; CHECK-NEXT:    "pred.udiv.continue:\n" +
-; CHECK-NEXT:      "PHI-PREDICATED-INSTRUCTION vp<%3> = ir<%tmp4>\l"
-; CHECK-NEXT:  ]
-;
-; CHECK:       N7 [label =
-; CHECK-NEXT:    "for.inc:\n" +
-; CHECK-NEXT:      "EMIT vp<%4> = not ir<%cmp>\l" +
-; CHECK-NEXT:      "BLEND %d = ir<0>/vp<%4> vp<%3>/ir<%cmp>\l" +
-; CHECK-NEXT:      "CLONE ir<%idx> = getelementptr ir<%x>, ir<%i>\l" +
-; CHECK-NEXT:      "WIDEN store ir<%idx>, ir<%d>\l"
-; CHECK-NEXT:  ]
+; CHECK-LABEL: Checking a loop in "print_replicate_predicated_phi"
+; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: for.body:
+; CHECK-NEXT:   WIDEN-INDUCTION %i = phi 0, %i.next
+; CHECK-NEXT:   WIDEN ir<%cmp> = icmp ir<%i>, ir<5>
+; CHECK-NEXT: Successor(s): if.then
+; CHECK-EMPTY:
+; CHECK-NEXT: if.then:
+; CHECK-NEXT: Successor(s): pred.udiv
+; CHECK-EMPTY:
+; CHECK-NEXT: <xVFxUF> pred.udiv: {
+; CHECK-NEXT:   pred.udiv.entry:
+; CHECK-NEXT:     BRANCH-ON-MASK ir<%cmp>
+; CHECK-NEXT:   Successor(s): pred.udiv.if, pred.udiv.continue
+; CHECK-NEXT:   CondBit: ir<%cmp>
+; CHECK-EMPTY:
+; CHECK-NEXT:   pred.udiv.if:
+; CHECK-NEXT:     REPLICATE ir<%tmp4> = udiv ir<%n>, ir<%i> (S->V)
+; CHECK-NEXT:   Successor(s): pred.udiv.continue
+; CHECK-EMPTY:
+; CHECK-NEXT:   pred.udiv.continue:
+; CHECK-NEXT:     PHI-PREDICATED-INSTRUCTION vp<%3> = ir<%tmp4>
+; CHECK-NEXT:   No successors
+; CHECK-NEXT: }
+; CHECK-EMPTY:
+; CHECK-NEXT: if.then.0:
+; CHECK-NEXT: Successor(s): for.inc
+; CHECK-EMPTY:
+; CHECK-NEXT: for.inc:
+; CHECK-NEXT:   EMIT vp<%4> = not ir<%cmp>
+; CHECK-NEXT:   BLEND %d = ir<0>/vp<%4> vp<%3>/ir<%cmp>
+; CHECK-NEXT:   CLONE ir<%idx> = getelementptr ir<%x>, ir<%i>
+; CHECK-NEXT:   WIDEN store ir<%idx>, ir<%d>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy.ll b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
index 096d5d9ea7f8e3f9d3cbe83f1db3d76766382460..96f4faa94791f40ea9847e976c488e121b79a5b9 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
@@ -14,7 +14,7 @@ define void @test1(%0* sret(%0)  %agg.result, x86_fp80 %z.0, x86_fp80 %z.1) noun
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca [[TMP0:%.*]], align 16
 ; CHECK-NEXT:    [[MEMTMP:%.*]] = alloca [[TMP0]], align 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = fsub x86_fp80 0xK80000000000000000000, [[Z_1:%.*]]
-; CHECK-NEXT:    call void @ccoshl(%0* sret(%0) [[TMP2]], x86_fp80 [[TMP5]], x86_fp80 [[Z_0:%.*]]) [[ATTR0:#.*]]
+; CHECK-NEXT:    call void @ccoshl(%0* sret([[TMP0]]) [[TMP2]], x86_fp80 [[TMP5]], x86_fp80 [[Z_0:%.*]]) #[[ATTR0:[0-9]+]]
 ; CHECK-NEXT:    [[TMP219:%.*]] = bitcast %0* [[TMP2]] to i8*
 ; CHECK-NEXT:    [[MEMTMP20:%.*]] = bitcast %0* [[MEMTMP]] to i8*
 ; CHECK-NEXT:    [[AGG_RESULT21:%.*]] = bitcast %0* [[AGG_RESULT:%.*]] to i8*
@@ -222,7 +222,7 @@ define i32 @test5(i32 %x) nounwind ssp {
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 [[TMP]], i8* align 16 bitcast (%struct.S* @sS to i8*), i64 32, i1 false)
 ; CHECK-NEXT:    [[A:%.*]] = getelementptr [[STRUCT_S]], %struct.S* [[Y]], i64 0, i32 1, i64 0
 ; CHECK-NEXT:    store i8 4, i8* [[A]], align 1
-; CHECK-NEXT:    call void @test5a(%struct.S* byval(%struct.S) align 16 [[Y]])
+; CHECK-NEXT:    call void @test5a(%struct.S* byval([[STRUCT_S]]) align 16 [[Y]])
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -252,7 +252,7 @@ define void @test6(i8 *%P) {
 define i32 @test7(%struct.p* nocapture align 8 byval(%struct.p) %q) nounwind ssp {
 ; CHECK-LABEL: @test7(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @g(%struct.p* byval(%struct.p) align 8 [[Q:%.*]]) [[ATTR0]]
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @g(%struct.p* byval([[STRUCT_P:%.*]]) align 8 [[Q:%.*]]) #[[ATTR0]]
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
 entry:
@@ -295,7 +295,7 @@ define void @test9_addrspacecast() nounwind ssp uwtable {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = alloca [[STRUCT_BIG:%.*]], align 4
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca [[STRUCT_BIG]], align 4
-; CHECK-NEXT:    call void @f1(%struct.big* sret(%struct.big) [[B]])
+; CHECK-NEXT:    call void @f1(%struct.big* sret([[STRUCT_BIG]]) [[B]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast %struct.big* [[B]] to i8 addrspace(1)*
 ; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast %struct.big* [[TMP]] to i8 addrspace(1)*
 ; CHECK-NEXT:    call void @f2(%struct.big* [[B]])
@@ -317,7 +317,7 @@ define void @test9() nounwind ssp uwtable {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = alloca [[STRUCT_BIG:%.*]], align 4
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca [[STRUCT_BIG]], align 4
-; CHECK-NEXT:    call void @f1(%struct.big* sret(%struct.big) [[B]])
+; CHECK-NEXT:    call void @f1(%struct.big* sret([[STRUCT_BIG]]) [[B]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.big* [[B]] to i8*
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %struct.big* [[TMP]] to i8*
 ; CHECK-NEXT:    call void @f2(%struct.big* [[B]])
@@ -381,7 +381,6 @@ declare void @llvm.memcpy.p1i8.p0i8.i64(i8 addrspace(1)* nocapture, i8* nocaptur
 declare void @f1(%struct.big* nocapture sret(%struct.big))
 declare void @f2(%struct.big*)
 
-; CHECK: attributes [[ATTR0]] = { nounwind }
 ; CHECK: attributes #1 = { argmemonly nofree nosync nounwind willreturn }
 ; CHECK: attributes #2 = { nounwind ssp }
 ; CHECK: attributes #3 = { willreturn }
diff --git a/llvm/test/Transforms/MemCpyOpt/sret.ll b/llvm/test/Transforms/MemCpyOpt/sret.ll
index 12608edb84a7c85277c67dba8e7c1c01de44bf02..98f22ede58654698bf9a8853630575f1bdb9f4fc 100644
--- a/llvm/test/Transforms/MemCpyOpt/sret.ll
+++ b/llvm/test/Transforms/MemCpyOpt/sret.ll
@@ -21,7 +21,7 @@ define void @ccosl(%0* noalias sret(%0) %agg.result, %0* byval(%0) align 8 %z) n
 ; CHECK-NEXT:    [[TMP8:%.*]] = load x86_fp80, x86_fp80* [[TMP7]], align 16
 ; CHECK-NEXT:    store x86_fp80 [[TMP3]], x86_fp80* [[REAL]], align 16
 ; CHECK-NEXT:    store x86_fp80 [[TMP8]], x86_fp80* [[TMP4]], align 16
-; CHECK-NEXT:    call void @ccoshl(%0* noalias sret(%0) [[AGG_RESULT:%.*]], %0* byval(%0) align 8 [[IZ]]) [[ATTR0:#.*]]
+; CHECK-NEXT:    call void @ccoshl(%0* noalias sret([[TMP0]]) [[AGG_RESULT:%.*]], %0* byval([[TMP0]]) align 8 [[IZ]]) #[[ATTR0:[0-9]+]]
 ; CHECK-NEXT:    [[MEMTMP14:%.*]] = bitcast %0* [[MEMTMP]] to i8*
 ; CHECK-NEXT:    [[AGG_RESULT15:%.*]] = bitcast %0* [[AGG_RESULT]] to i8*
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/MergeICmps/X86/alias-merge-blocks.ll b/llvm/test/Transforms/MergeICmps/X86/alias-merge-blocks.ll
index 3bc9f83894a709df661ce6f1201ded8912f2d9a5..25a73fa04bc1d848976874034321f9143c364c3f 100644
--- a/llvm/test/Transforms/MergeICmps/X86/alias-merge-blocks.ll
+++ b/llvm/test/Transforms/MergeICmps/X86/alias-merge-blocks.ll
@@ -6,8 +6,8 @@
 define zeroext i1 @opeq1(
 ; X86-LABEL: @opeq1(
 ; X86-NEXT:  "entry+land.rhs.i+land.rhs.i.2+land.rhs.i.3":
-; X86-NEXT:    [[PTR:%.*]] = alloca i32
-; X86-NEXT:    store i32 42, i32* [[PTR]]
+; X86-NEXT:    [[PTR:%.*]] = alloca i32, align 4
+; X86-NEXT:    store i32 42, i32* [[PTR]], align 4
 ; X86-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[S:%.*]], %S* [[A:%.*]], i64 0, i32 0
 ; X86-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[S]], %S* [[B:%.*]], i64 0, i32 0
 ; X86-NEXT:    [[CSTR:%.*]] = bitcast i32* [[TMP0]] to i8*
diff --git a/llvm/test/Transforms/MergeICmps/X86/entry-block-shuffled.ll b/llvm/test/Transforms/MergeICmps/X86/entry-block-shuffled.ll
index c15fd8810f196caa4e13cfb43111e17a5adc4630..7a5a6e40910504abe6f8fea6d9f8677ea7d714b8 100644
--- a/llvm/test/Transforms/MergeICmps/X86/entry-block-shuffled.ll
+++ b/llvm/test/Transforms/MergeICmps/X86/entry-block-shuffled.ll
@@ -20,15 +20,15 @@ define zeroext i1 @opeq1(
 ; CHECK:       entry2:
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[S]], %S* [[A]], i64 0, i32 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[S]], %S* [[B]], i64 0, i32 2
-; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[TMP3]]
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[TMP4]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP5]], [[TMP6]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[LAND_RHS_I_31:%.*]], label [[OPEQ1_EXIT]]
 ; CHECK:       land.rhs.i.31:
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[S]], %S* [[A]], i64 0, i32 3
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[S]], %S* [[B]], i64 0, i32 3
-; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
-; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP10]], [[TMP11]]
 ; CHECK-NEXT:    br label [[OPEQ1_EXIT]]
 ; CHECK:       opeq1.exit:
diff --git a/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll b/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
index ff525e96d7f1f14e6324a131e961f8c28ad90ab6..ad19eec760fe4b0b98754a57baa2140e5894e258 100644
--- a/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
+++ b/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
@@ -1,14 +1,25 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown-unknown -mergeicmps -verify-dom-info -S | FileCheck %s --check-prefix=X86
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --force-update
+; RUN: opt < %s -mtriple=x86_64-unknown-unknown -mergeicmps -verify-dom-info -S 2>&1 | FileCheck %s
 
 ; 8-byte int and 8-byte pointer should merge into a 16-byte memcmp.
-; X86: memcmp(i8* {{.*}}, i8* {{.*}}, i64 16)
 
 %struct.outer = type { i64, %struct.inner* }
 %struct.inner = type { i32, i32, i32 }
 
 ; Function Attrs: nounwind uwtable
-define dso_local i1 @"?foo@@YAHAEAUouter@@0@Z"(%struct.outer* align 8 dereferenceable(16) %o1, %struct.outer* align 8 dereferenceable(116) %o2) local_unnamed_addr #0 {
+define i1 @test(%struct.outer* align 8 dereferenceable(16) %o1, %struct.outer* align 8 dereferenceable(116) %o2) local_unnamed_addr #0 {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  "entry+if.then":
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_OUTER:%.*]], %struct.outer* [[O1:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_OUTER]], %struct.outer* [[O2:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[CSTR:%.*]] = bitcast i64* [[TMP0]] to i8*
+; CHECK-NEXT:    [[CSTR1:%.*]] = bitcast i64* [[TMP1]] to i8*
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* [[CSTR]], i8* [[CSTR1]], i64 16)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    br label [[IF_END5:%.*]]
+; CHECK:       if.end5:
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
 entry:
   %p1 = getelementptr inbounds %struct.outer, %struct.outer* %o1, i64 0, i32 0
   %0 = load i64, i64* %p1, align 8
diff --git a/llvm/test/Transforms/MergeICmps/X86/pr41917.ll b/llvm/test/Transforms/MergeICmps/X86/pr41917.ll
index 22cba94c582cfd532c2ccfce16a422277a2f054b..0af00c7c2c5c0752cfbe8c11f86aa15d4570d991 100644
--- a/llvm/test/Transforms/MergeICmps/X86/pr41917.ll
+++ b/llvm/test/Transforms/MergeICmps/X86/pr41917.ll
@@ -10,10 +10,10 @@ target triple = "i386-pc-windows-msvc19.11.0"
 define dso_local zeroext i1 @pr41917(%class.a* byval(%class.a) nocapture readonly align 4 %g, %class.a* byval(%class.a) nocapture readonly align 4 %p2) local_unnamed_addr #0 {
 ; CHECK-LABEL: @pr41917(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = tail call zeroext i1 @f2() #3
+; CHECK-NEXT:    [[CALL:%.*]] = tail call zeroext i1 @f2() #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    br i1 [[CALL]], label [[LAND_RHS:%.*]], label %"land.end+land.rhs3"
 ; CHECK:       land.rhs:
-; CHECK-NEXT:    [[CALL1:%.*]] = tail call zeroext i1 @f2() #3
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call zeroext i1 @f2() #[[ATTR3]]
 ; CHECK-NEXT:    br label %"land.end+land.rhs3"
 ; CHECK:       "land.end+land.rhs3":
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[CLASS_A:%.*]], %class.a* [[G:%.*]], i32 0, i32 1
diff --git a/llvm/test/Transforms/OpenMP/add_attributes.ll b/llvm/test/Transforms/OpenMP/add_attributes.ll
index 8476f42dd5297e222ebd9be92c40336eaca6ea76..087fc572b02f09b682eaf5c0e547aab37a5d4dd4 100644
--- a/llvm/test/Transforms/OpenMP/add_attributes.ll
+++ b/llvm/test/Transforms/OpenMP/add_attributes.ll
@@ -2,9 +2,6 @@
 ; RUN: opt < %s -S -passes=openmpopt | FileCheck %s
 ; RUN: opt < %s -S -openmpopt        -openmp-ir-builder-optimistic-attributes | FileCheck %s --check-prefix=OPTIMISTIC
 ; RUN: opt < %s -S -passes=openmpopt -openmp-ir-builder-optimistic-attributes | FileCheck %s --check-prefix=OPTIMISTIC
-;
-; TODO: Not all omp_XXXX methods are known to the OpenMPIRBuilder/OpenMPOpt.
-;
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
 %struct.omp_lock_t = type { i8* }
@@ -669,6 +666,11 @@ declare i8* @__kmpc_task_reduction_modifier_init(i8*, i32, i32, i32, i8*)
 
 declare void @__kmpc_proxy_task_completed_ooo(i8*)
 
+; Function Attrs: noinline
+declare void @__kmpc_barrier_simple_spmd(%struct.ident_t* nocapture nofree readonly, i32) #0
+
+attributes #0 = { noinline }
+
 ; CHECK: ; Function Attrs: nounwind
 ; CHECK-NEXT: declare dso_local void @omp_set_num_threads(i32)
 
@@ -685,52 +687,52 @@ declare void @__kmpc_proxy_task_completed_ooo(i8*)
 ; CHECK-NEXT: declare dso_local void @omp_set_schedule(i32, i32)
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_num_threads() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_num_threads()
 
 ; CHECK-NOT: Function Attrs
 ; CHECK: declare dso_local void @use_int(i32)
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_dynamic() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_dynamic()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_nested() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_nested()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_max_threads() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_max_threads()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_thread_num() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_thread_num()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_num_procs() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_num_procs()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_in_parallel() #0
+; CHECK-NEXT: declare dso_local i32 @omp_in_parallel()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_in_final() #0
+; CHECK-NEXT: declare dso_local i32 @omp_in_final()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_active_level() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_active_level()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_level() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_level()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_ancestor_thread_num(i32) #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_ancestor_thread_num(i32)
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_team_size(i32) #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_team_size(i32)
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_thread_limit() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_thread_limit()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_max_active_levels() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_max_active_levels()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local void @omp_get_schedule(i32* nocapture writeonly, i32* nocapture writeonly) #0
+; CHECK-NEXT: declare dso_local void @omp_get_schedule(i32* nocapture writeonly, i32* nocapture writeonly)
 
 ; CHECK-NOT: Function Attrs
 ; CHECK: declare dso_local i32 @omp_get_max_task_priority()
@@ -799,7 +801,7 @@ declare void @__kmpc_proxy_task_completed_ooo(i8*)
 ; CHECK: declare dso_local i32 @omp_get_team_num()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_cancellation() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_cancellation()
 
 ; CHECK-NOT: Function Attrs
 ; CHECK: declare dso_local i32 @omp_get_initial_device()
@@ -829,25 +831,25 @@ declare void @__kmpc_proxy_task_completed_ooo(i8*)
 ; CHECK: declare dso_local i32 @omp_get_device_num()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_proc_bind() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_proc_bind()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_num_places() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_num_places()
 
 ; CHECK-NOT: Function Attrs
 ; CHECK: declare dso_local i32 @omp_get_place_num_procs(i32)
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local void @omp_get_place_proc_ids(i32, i32* nocapture writeonly) #0
+; CHECK-NEXT: declare dso_local void @omp_get_place_proc_ids(i32, i32* nocapture writeonly)
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_place_num() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_place_num()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local i32 @omp_get_partition_num_places() #0
+; CHECK-NEXT: declare dso_local i32 @omp_get_partition_num_places()
 
 ; CHECK: ; Function Attrs: nounwind
-; CHECK-NEXT: declare dso_local void @omp_get_partition_place_nums(i32*) #0
+; CHECK-NEXT: declare dso_local void @omp_get_partition_place_nums(i32*)
 
 ; CHECK-NOT: Function Attrs
 ; CHECK: declare dso_local i32 @omp_control_tool(i32, i32, i8*)
@@ -1206,6 +1208,9 @@ declare void @__kmpc_proxy_task_completed_ooo(i8*)
 ; CHECK: ; Function Attrs: nounwind
 ; CHECK-NEXT: declare void @__kmpc_proxy_task_completed_ooo(i8*)
 
+; CHECK: ; Function Attrs: convergent noinline nounwind
+; CHECK-NEXT: declare void @__kmpc_barrier_simple_spmd(%struct.ident_t* nocapture nofree readonly, i32)
+
 ; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind willreturn writeonly
 ; OPTIMISTIC-NEXT: declare dso_local void @omp_set_num_threads(i32)
 
@@ -1730,3 +1735,7 @@ declare void @__kmpc_proxy_task_completed_ooo(i8*)
 
 ; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind willreturn
 ; OPTIMISTIC-NEXT: declare void @__kmpc_proxy_task_completed_ooo(i8*)
+
+; OPTIMISTIC: ; Function Attrs: convergent noinline nounwind 
+; OPTIMISTIC-NEXT: declare void @__kmpc_barrier_simple_spmd(%struct.ident_t* nocapture nofree readonly, i32)
+
diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll
index 8a05a624209ef378dad43eb199f56da478dfa989..ddf4811a0363e37e07e6219fb7684681c9b31def 100644
--- a/llvm/test/Transforms/PGOProfile/chr.ll
+++ b/llvm/test/Transforms/PGOProfile/chr.ll
@@ -1277,11 +1277,12 @@ define i32 @test_chr_14(i32* %i, i32* %j, i32 %sum0, i1 %pred, i32 %z) !prof !14
 ; CHECK-LABEL: @test_chr_14(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[I:%.*]], align 4
-; CHECK-NEXT:    [[V1:%.*]] = icmp ne i32 [[Z:%.*]], 1
+; CHECK-NEXT:    [[V1:%.*]] = icmp eq i32 [[Z:%.*]], 1
+; CHECK-NEXT:    br i1 [[V1]], label [[BB1:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       entry.split.nonchr:
 ; CHECK-NEXT:    [[V0:%.*]] = icmp eq i32 [[Z]], 0
 ; CHECK-NEXT:    [[V3_NONCHR:%.*]] = and i1 [[V0]], [[PRED:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[V1]], [[V3_NONCHR]]
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[BB0_NONCHR:%.*]], label [[BB1:%.*]], !prof !19
+; CHECK-NEXT:    br i1 [[V3_NONCHR]], label [[BB0_NONCHR:%.*]], label [[BB1]], !prof !16
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB1]]
@@ -1912,7 +1913,7 @@ define i32 @test_chr_21(i64 %i, i64 %k, i64 %j) !prof !14 {
 ; CHECK-NEXT:    switch i64 [[I]], label [[BB2:%.*]] [
 ; CHECK-NEXT:    i64 2, label [[BB3_NONCHR2:%.*]]
 ; CHECK-NEXT:    i64 86, label [[BB2_NONCHR1:%.*]]
-; CHECK-NEXT:    ], !prof !20
+; CHECK-NEXT:    ], !prof !19
 ; CHECK:       bb2:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    call void @foo()
@@ -2489,14 +2490,14 @@ define void @test_chr_24(i32* %i) !prof !14 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[TMP2]], label [[BB1:%.*]], label [[BB0:%.*]], !prof !21
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB1:%.*]], label [[BB0:%.*]], !prof !20
 ; CHECK:       bb0:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB1]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
-; CHECK-NEXT:    br i1 [[TMP4]], label [[BB3:%.*]], label [[BB2:%.*]], !prof !21
+; CHECK-NEXT:    br i1 [[TMP4]], label [[BB3:%.*]], label [[BB2:%.*]], !prof !20
 ; CHECK:       bb2:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB3]]
@@ -2550,4 +2551,3 @@ bb3:
 ; CHECK: !16 = !{!"branch_weights", i32 0, i32 1}
 ; CHECK: !17 = !{!"branch_weights", i32 1, i32 1}
 ; CHECK: !18 = !{!"branch_weights", i32 1, i32 0}
-; CHECK: !19 = !{!"branch_weights", i32 0, i32 1000}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/ctlz-loop.ll b/llvm/test/Transforms/PhaseOrdering/X86/ctlz-loop.ll
new file mode 100644
index 0000000000000000000000000000000000000000..67aa30fabe7d114f84f46f1f5461f1fe06b0135e
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/ctlz-loop.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -O1 -S -mattr=+lzcnt | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; This test ensures we are able to optimize the following loop to an llvm.abs
+; followed by an llvm.ctlz.
+; FIXME: LoopIdiom recongize is not forming llvm.ctlz.
+
+; int ctlz_zero_check(int n)
+; {
+;   n = n >= 0 ? n : -n;
+;   int i = 0;
+;   while(n) {
+;     n >>= 1;
+;     i++;
+;   }
+;   return i;
+; }
+
+define i32 @ctlz_loop_with_abs(i32 %n) {
+; CHECK-LABEL: @ctlz_loop_with_abs(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL_NOT1:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT1]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CHECK:       while.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.abs.i32(i32 [[N]], i1 true)
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[N_ADDR_03:%.*]] = phi i32 [ [[TMP1:%.*]], [[WHILE_BODY]] ], [ [[TMP0]], [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_02:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP1]] = lshr i32 [[N_ADDR_03]], 1
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_02]], 1
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label [[WHILE_END]], label [[WHILE_BODY]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC]], [[WHILE_BODY]] ]
+; CHECK-NEXT:    ret i32 [[I_0_LCSSA]]
+;
+entry:
+  %cmp = icmp sge i32 %n, 0
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %entry
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  %sub = sub nsw i32 0, %n
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %n, %cond.true ], [ %sub, %cond.false ]
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %cond.end
+  %i.0 = phi i32 [ 0, %cond.end ], [ %inc, %while.body ]
+  %n.addr.0 = phi i32 [ %cond, %cond.end ], [ %shr, %while.body ]
+  %tobool = icmp ne i32 %n.addr.0, 0
+  br i1 %tobool, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %shr = ashr i32 %n.addr.0, 1
+  %inc = add nsw i32 %i.0, 1
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll b/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3e659414d9822ba2113139ffb2c69fe4ecd30ca1
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/spurious-peeling.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O3 -S -enable-new-pm=0 < %s   | FileCheck %s --check-prefixes=OLDPM
+; RUN: opt -passes='default<O3>' -S < %s  | FileCheck %s --check-prefixes=NEWPM
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%class.FloatVecPair = type { %class.HomemadeVector, %class.HomemadeVector }
+%class.HomemadeVector = type <{ %class.HomemadeVector.0*, i32, [4 x i8] }>
+%class.HomemadeVector.0 = type <{ float*, i32, [4 x i8] }>
+
+$_ZN12FloatVecPair6vecIncEv = comdat any
+
+define dso_local void @_Z13vecIncFromPtrP12FloatVecPair(%class.FloatVecPair* %FVP) {
+; OLDPM-LABEL: @_Z13vecIncFromPtrP12FloatVecPair(
+; OLDPM-NEXT:  entry:
+; OLDPM-NEXT:    [[BASE_I_I:%.*]] = getelementptr inbounds [[CLASS_FLOATVECPAIR:%.*]], %class.FloatVecPair* [[FVP:%.*]], i64 0, i32 1, i32 0
+; OLDPM-NEXT:    [[TMP0:%.*]] = load %class.HomemadeVector.0*, %class.HomemadeVector.0** [[BASE_I_I]], align 8, !tbaa [[TBAA0:![0-9]+]]
+; OLDPM-NEXT:    [[SIZE410_I:%.*]] = getelementptr inbounds [[CLASS_HOMEMADEVECTOR_0:%.*]], %class.HomemadeVector.0* [[TMP0]], i64 undef, i32 1
+; OLDPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[SIZE410_I]], align 8, !tbaa [[TBAA6:![0-9]+]]
+; OLDPM-NEXT:    [[CMP511_NOT_I:%.*]] = icmp eq i32 [[TMP1]], 0
+; OLDPM-NEXT:    br i1 [[CMP511_NOT_I]], label [[_ZN12FLOATVECPAIR6VECINCEV_EXIT:%.*]], label [[FOR_BODY7_LR_PH_I:%.*]]
+; OLDPM:       for.body7.lr.ph.i:
+; OLDPM-NEXT:    [[BASE_I4_I:%.*]] = getelementptr inbounds [[CLASS_FLOATVECPAIR]], %class.FloatVecPair* [[FVP]], i64 0, i32 0, i32 0
+; OLDPM-NEXT:    [[TMP2:%.*]] = load %class.HomemadeVector.0*, %class.HomemadeVector.0** [[BASE_I4_I]], align 8, !tbaa [[TBAA0]]
+; OLDPM-NEXT:    [[BASE_I2_I:%.*]] = getelementptr inbounds [[CLASS_HOMEMADEVECTOR_0]], %class.HomemadeVector.0* [[TMP2]], i64 undef, i32 0
+; OLDPM-NEXT:    [[TMP3:%.*]] = load float*, float** [[BASE_I2_I]], align 8, !tbaa [[TBAA8:![0-9]+]]
+; OLDPM-NEXT:    [[ARRAYIDX_I3_I:%.*]] = getelementptr inbounds float, float* [[TMP3]], i64 undef
+; OLDPM-NEXT:    [[BASE_I6_PEEL_I:%.*]] = getelementptr inbounds [[CLASS_HOMEMADEVECTOR_0]], %class.HomemadeVector.0* [[TMP0]], i64 undef, i32 0
+; OLDPM-NEXT:    [[TMP4:%.*]] = load float*, float** [[BASE_I6_PEEL_I]], align 8, !tbaa [[TBAA8]]
+; OLDPM-NEXT:    [[ARRAYIDX_I7_PEEL_I:%.*]] = getelementptr inbounds float, float* [[TMP4]], i64 undef
+; OLDPM-NEXT:    [[TMP5:%.*]] = load float, float* [[ARRAYIDX_I7_PEEL_I]], align 4, !tbaa [[TBAA9:![0-9]+]]
+; OLDPM-NEXT:    [[TMP6:%.*]] = load float, float* [[ARRAYIDX_I3_I]], align 4, !tbaa [[TBAA9]]
+; OLDPM-NEXT:    [[ADD_PEEL_I:%.*]] = fadd float [[TMP5]], [[TMP6]]
+; OLDPM-NEXT:    store float [[ADD_PEEL_I]], float* [[ARRAYIDX_I3_I]], align 4, !tbaa [[TBAA9]]
+; OLDPM-NEXT:    [[EXITCOND_PEEL_NOT_I:%.*]] = icmp eq i32 [[TMP1]], 1
+; OLDPM-NEXT:    br i1 [[EXITCOND_PEEL_NOT_I]], label [[_ZN12FLOATVECPAIR6VECINCEV_EXIT]], label [[FOR_BODY7_I:%.*]]
+; OLDPM:       for.body7.i:
+; OLDPM-NEXT:    [[TMP7:%.*]] = phi float [ [[ADD_I:%.*]], [[FOR_BODY7_I]] ], [ [[ADD_PEEL_I]], [[FOR_BODY7_LR_PH_I]] ]
+; OLDPM-NEXT:    [[J_012_I:%.*]] = phi i32 [ [[INC_I:%.*]], [[FOR_BODY7_I]] ], [ 1, [[FOR_BODY7_LR_PH_I]] ]
+; OLDPM-NEXT:    [[TMP8:%.*]] = load float, float* [[ARRAYIDX_I7_PEEL_I]], align 4, !tbaa [[TBAA9]]
+; OLDPM-NEXT:    [[ADD_I]] = fadd float [[TMP7]], [[TMP8]]
+; OLDPM-NEXT:    store float [[ADD_I]], float* [[ARRAYIDX_I3_I]], align 4, !tbaa [[TBAA9]]
+; OLDPM-NEXT:    [[INC_I]] = add nuw i32 [[J_012_I]], 1
+; OLDPM-NEXT:    [[EXITCOND_NOT_I:%.*]] = icmp eq i32 [[INC_I]], [[TMP1]]
+; OLDPM-NEXT:    br i1 [[EXITCOND_NOT_I]], label [[_ZN12FLOATVECPAIR6VECINCEV_EXIT]], label [[FOR_BODY7_I]], !llvm.loop [[LOOP11:![0-9]+]]
+; OLDPM:       _ZN12FloatVecPair6vecIncEv.exit:
+; OLDPM-NEXT:    ret void
+;
+; NEWPM-LABEL: @_Z13vecIncFromPtrP12FloatVecPair(
+; NEWPM-NEXT:  entry:
+; NEWPM-NEXT:    [[BASE_I_I:%.*]] = getelementptr inbounds [[CLASS_FLOATVECPAIR:%.*]], %class.FloatVecPair* [[FVP:%.*]], i64 0, i32 1, i32 0
+; NEWPM-NEXT:    [[TMP0:%.*]] = load %class.HomemadeVector.0*, %class.HomemadeVector.0** [[BASE_I_I]], align 8, !tbaa [[TBAA0:![0-9]+]]
+; NEWPM-NEXT:    [[SIZE410_I:%.*]] = getelementptr inbounds [[CLASS_HOMEMADEVECTOR_0:%.*]], %class.HomemadeVector.0* [[TMP0]], i64 undef, i32 1
+; NEWPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[SIZE410_I]], align 8, !tbaa [[TBAA6:![0-9]+]]
+; NEWPM-NEXT:    [[CMP511_NOT_I:%.*]] = icmp eq i32 [[TMP1]], 0
+; NEWPM-NEXT:    br i1 [[CMP511_NOT_I]], label [[_ZN12FLOATVECPAIR6VECINCEV_EXIT:%.*]], label [[FOR_BODY7_LR_PH_I:%.*]]
+; NEWPM:       for.body7.lr.ph.i:
+; NEWPM-NEXT:    [[BASE_I6_I:%.*]] = getelementptr inbounds [[CLASS_FLOATVECPAIR]], %class.FloatVecPair* [[FVP]], i64 0, i32 0, i32 0
+; NEWPM-NEXT:    [[TMP2:%.*]] = load %class.HomemadeVector.0*, %class.HomemadeVector.0** [[BASE_I6_I]], align 8, !tbaa [[TBAA0]]
+; NEWPM-NEXT:    [[BASE_I8_I:%.*]] = getelementptr inbounds [[CLASS_HOMEMADEVECTOR_0]], %class.HomemadeVector.0* [[TMP2]], i64 undef, i32 0
+; NEWPM-NEXT:    [[TMP3:%.*]] = load float*, float** [[BASE_I8_I]], align 8, !tbaa [[TBAA8:![0-9]+]]
+; NEWPM-NEXT:    [[ARRAYIDX_I9_I:%.*]] = getelementptr inbounds float, float* [[TMP3]], i64 undef
+; NEWPM-NEXT:    [[BASE_I4_PEEL_I:%.*]] = getelementptr inbounds [[CLASS_HOMEMADEVECTOR_0]], %class.HomemadeVector.0* [[TMP0]], i64 undef, i32 0
+; NEWPM-NEXT:    [[TMP4:%.*]] = load float*, float** [[BASE_I4_PEEL_I]], align 8, !tbaa [[TBAA8]]
+; NEWPM-NEXT:    [[ARRAYIDX_I5_PEEL_I:%.*]] = getelementptr inbounds float, float* [[TMP4]], i64 undef
+; NEWPM-NEXT:    [[TMP5:%.*]] = load float, float* [[ARRAYIDX_I5_PEEL_I]], align 4, !tbaa [[TBAA9:![0-9]+]]
+; NEWPM-NEXT:    [[TMP6:%.*]] = load float, float* [[ARRAYIDX_I9_I]], align 4, !tbaa [[TBAA9]]
+; NEWPM-NEXT:    [[ADD_PEEL_I:%.*]] = fadd float [[TMP5]], [[TMP6]]
+; NEWPM-NEXT:    store float [[ADD_PEEL_I]], float* [[ARRAYIDX_I9_I]], align 4, !tbaa [[TBAA9]]
+; NEWPM-NEXT:    [[EXITCOND_PEEL_NOT_I:%.*]] = icmp eq i32 [[TMP1]], 1
+; NEWPM-NEXT:    br i1 [[EXITCOND_PEEL_NOT_I]], label [[_ZN12FLOATVECPAIR6VECINCEV_EXIT]], label [[FOR_BODY7_LR_PH_I_FOR_BODY7_I_CRIT_EDGE:%.*]]
+; NEWPM:       for.body7.lr.ph.i.for.body7.i_crit_edge:
+; NEWPM-NEXT:    [[INC_I_1:%.*]] = add nuw i32 1, 1
+; NEWPM-NEXT:    br label [[FOR_BODY7_I:%.*]]
+; NEWPM:       for.body7.i:
+; NEWPM-NEXT:    [[TMP7:%.*]] = phi float [ [[ADD_I:%.*]], [[FOR_BODY7_I_FOR_BODY7_I_CRIT_EDGE:%.*]] ], [ [[ADD_PEEL_I]], [[FOR_BODY7_LR_PH_I_FOR_BODY7_I_CRIT_EDGE]] ]
+; NEWPM-NEXT:    [[INC_I_PHI:%.*]] = phi i32 [ [[INC_I_0:%.*]], [[FOR_BODY7_I_FOR_BODY7_I_CRIT_EDGE]] ], [ [[INC_I_1]], [[FOR_BODY7_LR_PH_I_FOR_BODY7_I_CRIT_EDGE]] ]
+; NEWPM-NEXT:    [[TMP8:%.*]] = load float, float* [[ARRAYIDX_I5_PEEL_I]], align 4, !tbaa [[TBAA9]]
+; NEWPM-NEXT:    [[ADD_I]] = fadd float [[TMP7]], [[TMP8]]
+; NEWPM-NEXT:    store float [[ADD_I]], float* [[ARRAYIDX_I9_I]], align 4, !tbaa [[TBAA9]]
+; NEWPM-NEXT:    [[EXITCOND_NOT_I:%.*]] = icmp eq i32 [[INC_I_PHI]], [[TMP1]]
+; NEWPM-NEXT:    br i1 [[EXITCOND_NOT_I]], label [[_ZN12FLOATVECPAIR6VECINCEV_EXIT]], label [[FOR_BODY7_I_FOR_BODY7_I_CRIT_EDGE]], !llvm.loop [[LOOP11:![0-9]+]]
+; NEWPM:       for.body7.i.for.body7.i_crit_edge:
+; NEWPM-NEXT:    [[INC_I_0]] = add nuw i32 [[INC_I_PHI]], 1
+; NEWPM-NEXT:    br label [[FOR_BODY7_I]]
+; NEWPM:       _ZN12FloatVecPair6vecIncEv.exit:
+; NEWPM-NEXT:    ret void
+;
+entry:
+  %FVP.addr = alloca %class.FloatVecPair*, align 8
+  store %class.FloatVecPair* %FVP, %class.FloatVecPair** %FVP.addr, align 8, !tbaa !0
+  %0 = load %class.FloatVecPair*, %class.FloatVecPair** %FVP.addr, align 8, !tbaa !0
+  call void @_ZN12FloatVecPair6vecIncEv(%class.FloatVecPair* %0)
+  ret void
+}
+
+define linkonce_odr dso_local void @_ZN12FloatVecPair6vecIncEv(%class.FloatVecPair* %this) comdat align 2 {
+entry:
+  %this.addr = alloca %class.FloatVecPair*, align 8
+  %j = alloca i32, align 4
+  store %class.FloatVecPair* %this, %class.FloatVecPair** %this.addr, align 8, !tbaa !0
+  %this1 = load %class.FloatVecPair*, %class.FloatVecPair** %this.addr, align 8
+  br label %for.cond
+
+for.cond:                                         ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond
+  store i32 0, i32* %j, align 4, !tbaa !4
+  br label %for.cond2
+
+for.cond2:                                        ; preds = %for.inc, %for.body
+  %0 = load i32, i32* %j, align 4, !tbaa !4
+  %Vsrc23 = getelementptr inbounds %class.FloatVecPair, %class.FloatVecPair* %this1, i32 0, i32 1
+  %call = call %class.HomemadeVector.0* @_ZN14HomemadeVectorIS_IfLj8EELj8EEixEj(%class.HomemadeVector* %Vsrc23)
+  %size4 = getelementptr inbounds %class.HomemadeVector.0, %class.HomemadeVector.0* %call, i32 0, i32 1
+  %1 = load i32, i32* %size4, align 8, !tbaa !6
+  %cmp5 = icmp ult i32 %0, %1
+  br i1 %cmp5, label %for.body7, label %for.cond.cleanup6
+
+for.cond.cleanup6:                                ; preds = %for.cond2
+  ret void
+
+for.body7:                                        ; preds = %for.cond2
+  %Vsrc28 = getelementptr inbounds %class.FloatVecPair, %class.FloatVecPair* %this1, i32 0, i32 1
+  %call9 = call %class.HomemadeVector.0* @_ZN14HomemadeVectorIS_IfLj8EELj8EEixEj(%class.HomemadeVector* %Vsrc28)
+  %call10 = call float* @_ZN14HomemadeVectorIfLj8EEixEj(%class.HomemadeVector.0* %call9)
+  %2 = load float, float* %call10, align 4, !tbaa !8
+  %Vsrcdst = getelementptr inbounds %class.FloatVecPair, %class.FloatVecPair* %this1, i32 0, i32 0
+  %call11 = call %class.HomemadeVector.0* @_ZN14HomemadeVectorIS_IfLj8EELj8EEixEj(%class.HomemadeVector* %Vsrcdst)
+  %call12 = call float* @_ZN14HomemadeVectorIfLj8EEixEj(%class.HomemadeVector.0* %call11)
+  %3 = load float, float* %call12, align 4, !tbaa !8
+  %add = fadd float %3, %2
+  store float %add, float* %call12, align 4, !tbaa !8
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body7
+  %4 = load i32, i32* %j, align 4, !tbaa !4
+  %inc = add i32 %4, 1
+  store i32 %inc, i32* %j, align 4, !tbaa !4
+  br label %for.cond2, !llvm.loop !10
+}
+
+define linkonce_odr dso_local %class.HomemadeVector.0* @_ZN14HomemadeVectorIS_IfLj8EELj8EEixEj(%class.HomemadeVector* %this) align 2 {
+entry:
+  %this.addr = alloca %class.HomemadeVector*, align 8
+  store %class.HomemadeVector* %this, %class.HomemadeVector** %this.addr, align 8, !tbaa !0
+  %this1 = load %class.HomemadeVector*, %class.HomemadeVector** %this.addr, align 8
+  %base = getelementptr inbounds %class.HomemadeVector, %class.HomemadeVector* %this1, i32 0, i32 0
+  %0 = load %class.HomemadeVector.0*, %class.HomemadeVector.0** %base, align 8, !tbaa !12
+  %1 = bitcast %class.HomemadeVector.0* %0 to i8*
+  %2 = bitcast i8* %1 to %class.HomemadeVector.0*
+  %arrayidx = getelementptr inbounds %class.HomemadeVector.0, %class.HomemadeVector.0* %2, i64 undef
+  ret %class.HomemadeVector.0* %arrayidx
+}
+
+define linkonce_odr dso_local float* @_ZN14HomemadeVectorIfLj8EEixEj(%class.HomemadeVector.0* %this) align 2 {
+entry:
+  %this.addr = alloca %class.HomemadeVector.0*, align 8
+  store %class.HomemadeVector.0* %this, %class.HomemadeVector.0** %this.addr, align 8, !tbaa !0
+  %this1 = load %class.HomemadeVector.0*, %class.HomemadeVector.0** %this.addr, align 8
+  %base = getelementptr inbounds %class.HomemadeVector.0, %class.HomemadeVector.0* %this1, i32 0, i32 0
+  %0 = load float*, float** %base, align 8, !tbaa !14
+  %1 = bitcast float* %0 to i8*
+  %2 = bitcast i8* %1 to float*
+  %arrayidx = getelementptr inbounds float, float* %2, i64 undef
+  ret float* %arrayidx
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"any pointer", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C++ TBAA"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !2, i64 0}
+!6 = !{!7, !5, i64 8}
+!7 = !{!"_ZTS14HomemadeVectorIfLj8EE", !1, i64 0, !5, i64 8}
+!8 = !{!9, !9, i64 0}
+!9 = !{!"float", !2, i64 0}
+!10 = distinct !{!10, !11}
+!11 = !{!"llvm.loop.mustprogress"}
+!12 = !{!13, !1, i64 0}
+!13 = !{!"_ZTS14HomemadeVectorIS_IfLj8EELj8EE", !1, i64 0, !5, i64 8}
+!14 = !{!7, !1, i64 0}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
index 3aefdb2dd6b1d8c70439404d4b353f52afc1b646..463be094596deb61be984f81afde3c5ef04b232f 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -O2 -expand-reductions -S < %s | FileCheck %s
+; RUN: opt -O2 -expand-reductions -mattr=avx -S < %s | FileCheck %s
 
 ; Test if SLP vector reduction patterns are recognized
 ; and optionally converted to reduction intrinsics and
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll
index b5177b9628c74bfb09b1486b11de882dbe208284..0df3043fd4cab081741f54b960909e4e8ef2eb2a 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll
@@ -18,11 +18,9 @@ define i8 addrspace(1)* @test(i8 addrspace(1)* %a) gc "statepoint-example" {
 
 define i8 addrspace(1)* @test_select(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2) gc "statepoint-example" {
 ; CHECK-LABEL: @test_select(
-; CHECK-NEXT:    [[SEL_BASE:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C]], i8 addrspace(1)* [[A1]], i8 addrspace(1)* [[A2]]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[SEL]], i8 addrspace(1)* [[SEL_BASE]]) ]
-; CHECK-NEXT:    [[SEL_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[SEL_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[SEL]]) ]
+; CHECK-NEXT:    [[SEL_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret i8 addrspace(1)* [[SEL_RELOCATED]]
 ;
   %sel = select i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2
@@ -39,11 +37,9 @@ define i8 addrspace(1)* @test_phi1(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)*
 ; CHECK:       untaken:
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[TAKEN]] ], [ [[A2:%.*]], [[UNTAKEN]] ], !is_base_value !0
-; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1]], [[TAKEN]] ], [ [[A2]], [[UNTAKEN]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[PHI_BASE]]) ]
-; CHECK-NEXT:    [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[PHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[TAKEN]] ], [ [[A2:%.*]], [[UNTAKEN]] ]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]]) ]
+; CHECK-NEXT:    [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret i8 addrspace(1)* [[PHI_RELOCATED]]
 ;
 entry:
@@ -81,13 +77,11 @@ define i8 addrspace(1)* @test_loop1(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[ENTRY:%.*]] ], [ [[A2:%.*]], [[LOOP]] ], !is_base_value !0
-; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1]], [[ENTRY]] ], [ [[A2]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[ENTRY:%.*]] ], [ [[A2:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[PHI_BASE]]) ]
-; CHECK-NEXT:    [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[PHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]]) ]
+; CHECK-NEXT:    [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret i8 addrspace(1)* [[PHI_RELOCATED]]
 ;
 entry:
@@ -106,15 +100,13 @@ define i8 addrspace(1)* @test_loop2(i1 %c, i8 addrspace(1)* %a1) gc "statepoint-
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[ENTRY:%.*]] ], [ [[O2:%.*]], [[LOOP]] ], !is_base_value !0
-; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1]], [[ENTRY]] ], [ [[O2]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[ENTRY:%.*]] ], [ [[O2:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[ADDR:%.*]] = bitcast i8 addrspace(1)* [[PHI]] to i8 addrspace(1)* addrspace(1)*
 ; CHECK-NEXT:    [[O2]] = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[ADDR]], align 8
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[PHI_BASE]]) ]
-; CHECK-NEXT:    [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[PHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]]) ]
+; CHECK-NEXT:    [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret i8 addrspace(1)* [[PHI_RELOCATED]]
 ;
 entry:
@@ -171,11 +163,9 @@ define <2 x i8 addrspace(1)*> @test_vec_passthrough(<2 x i8 addrspace(1)*> %a) g
 
 define <2 x i8 addrspace(1)*> @test_insert(i8 addrspace(1)* %a) gc "statepoint-example" {
 ; CHECK-LABEL: @test_insert(
-; CHECK-NEXT:    [[VEC_BASE:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[A:%.*]], i64 0, !is_base_value !0
-; CHECK-NEXT:    [[VEC:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[A]], i64 0
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[VEC]], <2 x i8 addrspace(1)*> [[VEC_BASE]]) ]
-; CHECK-NEXT:    [[VEC_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[VEC_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[VEC:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[A:%.*]], i64 0
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[VEC]]) ]
+; CHECK-NEXT:    [[VEC_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x i8 addrspace(1)*> [[VEC_RELOCATED]]
 ;
   %vec = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* %a, i64 0
@@ -185,11 +175,9 @@ define <2 x i8 addrspace(1)*> @test_insert(i8 addrspace(1)* %a) gc "statepoint-e
 
 define i8 addrspace(1)* @test_extract(<2 x i8 addrspace(1)*> %a) gc "statepoint-example" {
 ; CHECK-LABEL: @test_extract(
-; CHECK-NEXT:    [[BASE_EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[A:%.*]], i64 0, !is_base_value !0
-; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[A]], i64 0
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[EE]], i8 addrspace(1)* [[BASE_EE]]) ]
-; CHECK-NEXT:    [[EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[EE]]) ]
+; CHECK-NEXT:    [[EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret i8 addrspace(1)* [[EE_RELOCATED]]
 ;
   %ee = extractelement <2 x i8 addrspace(1)*> %a, i64 0
@@ -200,9 +188,8 @@ define i8 addrspace(1)* @test_extract(<2 x i8 addrspace(1)*> %a) gc "statepoint-
 define <2 x i8 addrspace(1)*> @test_shuffle(<2 x i8 addrspace(1)*> %a1) gc "statepoint-example" {
 ; CHECK-LABEL: @test_shuffle(
 ; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A1]], <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[RES]], <2 x i8 addrspace(1)*> [[A1]]) ]
-; CHECK-NEXT:    [[RES_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[A1_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[RES]]) ]
+; CHECK-NEXT:    [[RES_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x i8 addrspace(1)*> [[RES_RELOCATED]]
 ;
   %res = shufflevector <2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a1, <2 x i32> zeroinitializer
@@ -212,11 +199,9 @@ define <2 x i8 addrspace(1)*> @test_shuffle(<2 x i8 addrspace(1)*> %a1) gc "stat
 
 define <2 x i8 addrspace(1)*> @test_shuffle2(<2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a2) gc "statepoint-example" {
 ; CHECK-LABEL: @test_shuffle2(
-; CHECK-NEXT:    [[RES_BASE:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A2:%.*]], <2 x i32> zeroinitializer, !is_base_value !0
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1]], <2 x i8 addrspace(1)*> [[A2]], <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[RES]], <2 x i8 addrspace(1)*> [[RES_BASE]]) ]
-; CHECK-NEXT:    [[RES_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[RES_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A2:%.*]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[RES]]) ]
+; CHECK-NEXT:    [[RES_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x i8 addrspace(1)*> [[RES_RELOCATED]]
 ;
   %res = shufflevector <2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a2, <2 x i32> zeroinitializer
@@ -226,11 +211,9 @@ define <2 x i8 addrspace(1)*> @test_shuffle2(<2 x i8 addrspace(1)*> %a1, <2 x i8
 
 define <4 x i8 addrspace(1)*> @test_shuffle_concat(<2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a2) gc "statepoint-example" {
 ; CHECK-LABEL: @test_shuffle_concat(
-; CHECK-NEXT:    [[RES_BASE:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A2:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>, !is_base_value !0
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1]], <2 x i8 addrspace(1)*> [[A2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<4 x i8 addrspace(1)*> [[RES]], <4 x i8 addrspace(1)*> [[RES_BASE]]) ]
-; CHECK-NEXT:    [[RES_RELOCATED:%.*]] = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[RES_BASE_RELOCATED:%.*]] = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A2:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<4 x i8 addrspace(1)*> [[RES]]) ]
+; CHECK-NEXT:    [[RES_RELOCATED:%.*]] = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    ret <4 x i8 addrspace(1)*> [[RES_RELOCATED]]
 ;
   %res = shufflevector <2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -238,23 +221,38 @@ define <4 x i8 addrspace(1)*> @test_shuffle_concat(<2 x i8 addrspace(1)*> %a1, <
   ret <4 x i8 addrspace(1)*> %res
 }
 
+; TODO: Special case worth handling - we interpret the shuffle as if we need
+; to select the base pointers from either input when the mask is known.
+define <2 x i8 addrspace(1)*> @test_shuffle_broadcast(i8 addrspace(1)* %a) gc "statepoint-example" {
+; CHECK-LABEL: @test_shuffle_broadcast(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IE:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[A:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE]], <2 x i8 addrspace(1)*> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[BROADCAST]]) ]
+; CHECK-NEXT:    [[BROADCAST_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
+; CHECK-NEXT:    ret <2 x i8 addrspace(1)*> [[BROADCAST_RELOCATED]]
+;
+entry:
+  %ie = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* %a, i64 0
+  %broadcast = shufflevector <2 x i8 addrspace(1)*> %ie, <2 x i8 addrspace(1)*> undef, <2 x i32> zeroinitializer
+  call void @foo()
+  ret <2 x i8 addrspace(1)*> %broadcast
+}
 
 ; Show a case where only a portion of the sub-graph propagates base pointers.
 define i8 @test_subgraph(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2) gc "statepoint-example" {
 ; CHECK-LABEL: @test_subgraph(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SEL_BASE:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C]], i8 addrspace(1)* [[A1]], i8 addrspace(1)* [[A2]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]]
 ; CHECK-NEXT:    br i1 [[C]], label [[TAKEN:%.*]], label [[MERGE:%.*]]
 ; CHECK:       taken:
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, i8 addrspace(1)* [[SEL]], i64 8
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[SEL_BASE]], [[TAKEN]] ], [ [[SEL_BASE]], [[ENTRY:%.*]] ], !is_base_value !0
-; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[GEP]], [[TAKEN]] ], [ [[SEL]], [[ENTRY]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[PHI_BASE]]) ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[GEP]], [[TAKEN]] ], [ [[SEL]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[SEL]]) ]
 ; CHECK-NEXT:    [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[PHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[SEL_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
 ; CHECK-NEXT:    [[RES:%.*]] = load i8, i8 addrspace(1)* [[PHI_RELOCATED]], align 1
 ; CHECK-NEXT:    ret i8 [[RES]]
 ;
@@ -275,24 +273,19 @@ merge:
 define i8 @test_subgraph2(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2) gc "statepoint-example" {
 ; CHECK-LABEL: @test_subgraph2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SEL_BASE:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C]], i8 addrspace(1)* [[A1]], i8 addrspace(1)* [[A2]]
-; CHECK-NEXT:    [[IE_BASE:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[SEL_BASE]], i64 0, !is_base_value !0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]]
 ; CHECK-NEXT:    [[IE:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[SEL]], i64 0
-; CHECK-NEXT:    [[BROADCAST_BASE:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE_BASE]], <2 x i8 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
-; CHECK-NEXT:    [[BROADCAST:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE]], <2 x i8 addrspace(1)*> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[EE_BASE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[BROADCAST_BASE]], i32 1, !is_base_value !0
+; CHECK-NEXT:    [[BROADCAST:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE]], <2 x i8 addrspace(1)*> [[IE]], <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[BROADCAST]], i32 1
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[EE]], i8 addrspace(1)* [[EE_BASE]]) ]
-; CHECK-NEXT:    [[EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
-; CHECK-NEXT:    [[EE_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[EE]]) ]
+; CHECK-NEXT:    [[EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[RES:%.*]] = load i8, i8 addrspace(1)* [[EE_RELOCATED]], align 1
 ; CHECK-NEXT:    ret i8 [[RES]]
 ;
 entry:
   %sel = select i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2
   %ie = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* %sel, i64 0
-  %broadcast = shufflevector <2 x i8 addrspace(1)*> %ie, <2 x i8 addrspace(1)*> undef, <2 x i32> zeroinitializer
+  %broadcast = shufflevector <2 x i8 addrspace(1)*> %ie, <2 x i8 addrspace(1)*> %ie, <2 x i32> zeroinitializer
   %ee = extractelement <2 x i8 addrspace(1)*> %broadcast, i32 1
   call void @foo()
   %res = load i8, i8 addrspace(1)* %ee
@@ -300,5 +293,4 @@ entry:
 }
 
 
-
 declare void @foo()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
index 31a78ae861893a67103e8e55ee447c709ea8cbc7..2b75785bf9dbca0a0627d318e8000ef456b0d55d 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
@@ -27,12 +27,9 @@ define void @test(i32 %condition) gc "statepoint-example" {
 ; CHECK:       dest_c:
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[OBJ_TO_CONSUME_BASE:%.*]] = phi i64 addrspace(1)* [ [[TMP0]], [[DEST_A]] ], [ null, [[DEST_B]] ], [ null, [[DEST_C]] ], !is_base_value !0
 ; CHECK-NEXT:    [[OBJ_TO_CONSUME:%.*]] = phi i64 addrspace(1)* [ [[TMP0]], [[DEST_A]] ], [ null, [[DEST_B]] ], [ null, [[DEST_C]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN1:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @consume_obj, i32 1, i32 0, i64 addrspace(1)* [[OBJ_TO_CONSUME]], i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[OBJ_TO_CONSUME_BASE]], i64 addrspace(1)* [[OBJ_TO_CONSUME]]) ]
-; CHECK-NEXT:    [[OBJ_TO_CONSUME_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN1]], i32 0, i32 0)
-; CHECK-NEXT:    [[OBJ_TO_CONSUME_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_TO_CONSUME_BASE_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[OBJ_TO_CONSUME_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN1]], i32 0, i32 1)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN1:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @consume_obj, i32 1, i32 0, i64 addrspace(1)* [[OBJ_TO_CONSUME]], i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[OBJ_TO_CONSUME]]) ]
+; CHECK-NEXT:    [[OBJ_TO_CONSUME_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN1]], i32 0, i32 0)
 ; CHECK-NEXT:    [[OBJ_TO_CONSUME_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_TO_CONSUME_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    br label [[MERGE_SPLIT:%.*]]
 ; CHECK:       merge.split:
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
index 827bb1cccc73d17eebc6db4f30b61016cfb1031d..8c2fa51399502a2e65d5363bda234e962e93c706 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
@@ -41,7 +41,6 @@ define i64 addrspace(1)* @test1(i32 %caller, i8 addrspace(1)* %a, i8 addrspace(1
 ; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
 ; CHECK:       left:
 ; CHECK-NEXT:    [[A_CAST:%.*]] = bitcast i8 addrspace(1)* [[A:%.*]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[A]] to i64 addrspace(1)*
 ; CHECK-NEXT:    switch i32 [[UNKNOWN:%.*]], label [[RIGHT]] [
 ; CHECK-NEXT:    i32 0, label [[MERGE:%.*]]
 ; CHECK-NEXT:    i32 1, label [[MERGE]]
@@ -49,16 +48,12 @@ define i64 addrspace(1)* @test1(i32 %caller, i8 addrspace(1)* %a, i8 addrspace(1
 ; CHECK-NEXT:    ]
 ; CHECK:       right:
 ; CHECK-NEXT:    [[B_CAST:%.*]] = bitcast i8 addrspace(1)* [[B:%.*]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[CAST1:%.*]] = bitcast i8 addrspace(1)* [[B]] to i64 addrspace(1)*
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[VALUE_BASE:%.*]] = phi i64 addrspace(1)* [ [[CAST]], [[LEFT]] ], [ [[CAST]], [[LEFT]] ], [ [[CAST]], [[LEFT]] ], [ [[CAST1]], [[RIGHT]] ], !is_base_value !0
 ; CHECK-NEXT:    [[VALUE:%.*]] = phi i64 addrspace(1)* [ [[A_CAST]], [[LEFT]] ], [ [[A_CAST]], [[LEFT]] ], [ [[A_CAST]], [[LEFT]] ], [ [[B_CAST]], [[RIGHT]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @parse_point, i32 1, i32 0, i64 addrspace(1)* [[VALUE]], i32 0, i32 0) [ "deopt"(i32 0, i32 0, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[VALUE]], i64 addrspace(1)* [[VALUE_BASE]]) ]
-; CHECK-NEXT:    [[VALUE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @parse_point, i32 1, i32 0, i64 addrspace(1)* [[VALUE]], i32 0, i32 0) [ "deopt"(i32 0, i32 0, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[VALUE]]) ]
+; CHECK-NEXT:    [[VALUE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[VALUE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[VALUE_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[VALUE_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[VALUE_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[VALUE_BASE_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    ret i64 addrspace(1)* [[VALUE_RELOCATED_CASTED]]
 ;
 entry:
@@ -146,13 +141,10 @@ define i64 addrspace(1)* @test3(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1
 ; CHECK:       taken:
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[BDV_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[TAKEN]] ], !is_base_value !0
-; CHECK-NEXT:    [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ]], [[ENTRY]] ], [ [[OBJ2]], [[TAKEN]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BDV_BASE]]) ]
-; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[TAKEN]] ]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ]
+; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[BDV_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[BDV_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_BASE_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    ret i64 addrspace(1)* [[BDV_RELOCATED_CASTED]]
 ;
 entry:
@@ -175,11 +167,9 @@ define i64 addrspace(1)* @test4(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
 ; CHECK-NEXT:    [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ]], [[TAKEN]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[OBJ]]) ]
-; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ]
+; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    ret i64 addrspace(1)* [[BDV_RELOCATED_CASTED]]
 ;
 entry:
@@ -199,15 +189,12 @@ define i64 addrspace(1)* @test5(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[MERGE:%.*]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[BDV_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[MERGE]] ], !is_base_value !0
-; CHECK-NEXT:    [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ]], [[ENTRY]] ], [ [[OBJ2]], [[MERGE]] ]
+; CHECK-NEXT:    [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[MERGE]] ]
 ; CHECK-NEXT:    br i1 [[CND:%.*]], label [[MERGE]], label [[NEXT:%.*]]
 ; CHECK:       next:
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BDV_BASE]]) ]
-; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ]
+; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[BDV_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[BDV_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_BASE_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    ret i64 addrspace(1)* [[BDV_RELOCATED_CASTED]]
 ;
 entry:
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll
index 6e0a8eef901069f0e8e18d61c4f7aed26281de25..d7f14dec9853f071869c93aa0d0a248f903aef53 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll
@@ -6,13 +6,10 @@
 define i64 addrspace(1)* @test(<2 x i64 addrspace(1)*> %vec, i32 %idx) gc "statepoint-example" {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[BASE_EE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC:%.*]], i32 [[IDX:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[OBJ:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX]]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]], i64 addrspace(1)* [[BASE_EE]]) ]
-; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[OBJ:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC:%.*]], i32 [[IDX:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]]) ]
+; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[BASE_EE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASE_EE_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    ret i64 addrspace(1)* [[OBJ_RELOCATED_CASTED]]
 ;
 ; Note that the second extractelement is actually redundant here.  A correct output would
@@ -34,25 +31,19 @@ define i64 addrspace(1)* @test2(<2 x i64 addrspace(1)*>* %ptr, i1 %cnd, i32 %idx
 ; CHECK-NEXT:    [[OBJB:%.*]] = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* [[PTR]], align 16
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[VEC_BASE:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ], !is_base_value !0
 ; CHECK-NEXT:    [[VEC:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ]
 ; CHECK-NEXT:    br i1 [[CND]], label [[TAKEN2:%.*]], label [[UNTAKEN2:%.*]]
 ; CHECK:       taken2:
-; CHECK-NEXT:    [[OBJ0_BASE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC_BASE]], i32 [[IDX1:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[OBJ0:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX1]]
+; CHECK-NEXT:    [[OBJ0:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX1:%.*]]
 ; CHECK-NEXT:    br label [[MERGE2:%.*]]
 ; CHECK:       untaken2:
-; CHECK-NEXT:    [[OBJ1_BASE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC_BASE]], i32 [[IDX2:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[OBJ1:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX2]]
+; CHECK-NEXT:    [[OBJ1:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX2:%.*]]
 ; CHECK-NEXT:    br label [[MERGE2]]
 ; CHECK:       merge2:
-; CHECK-NEXT:    [[OBJ_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ0_BASE]], [[TAKEN2]] ], [ [[OBJ1_BASE]], [[UNTAKEN2]] ], !is_base_value !0
 ; CHECK-NEXT:    [[OBJ:%.*]] = phi i64 addrspace(1)* [ [[OBJ0]], [[TAKEN2]] ], [ [[OBJ1]], [[UNTAKEN2]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]], i64 addrspace(1)* [[OBJ_BASE]]) ]
-; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]]) ]
+; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[OBJ_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[OBJ_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_BASE_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    ret i64 addrspace(1)* [[OBJ_RELOCATED_CASTED]]
 ;
 entry:
@@ -282,13 +273,10 @@ define void @test9(<4 x i64 addrspace(1)*> %vec1, i64 %idx) gc "statepoint-examp
 ; CHECK-LABEL: @test9(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <4 x i64 addrspace(1)*> [[VEC1:%.*]], <4 x i64 addrspace(1)*> [[VEC1]], <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[BASE_EE:%.*]] = extractelement <4 x i64 addrspace(1)*> [[VEC1]], i64 [[IDX:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[BDV:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i64 [[IDX]]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BASE_EE]]) ]
-; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[BDV:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i64 [[IDX:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ]
+; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[BASE_EE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASE_EE_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    call void @use(i64 addrspace(1)* [[BDV_RELOCATED_CASTED]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll
index f377989e8118e31e7742d83f1935875ae360750e..f790ef248f6e43d9e3098dc46bfb89b45abc9754 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll
@@ -8,13 +8,10 @@ define i64 addrspace(1)* @test(<2 x i64 addrspace(1)*> %vec, i32 %idx) gc "state
 ; be to reuse the existing obj as a base since it is actually a base pointer.
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[BASE_EE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC:%.*]], i32 [[IDX:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[OBJ:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX]]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]], i64 addrspace(1)* [[BASE_EE]]) ]
-; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[OBJ:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC:%.*]], i32 [[IDX:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]]) ]
+; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[BASE_EE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASE_EE_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    ret i64 addrspace(1)* [[OBJ_RELOCATED_CASTED]]
 ;
 entry:
@@ -34,25 +31,19 @@ define i64 addrspace(1)* @test2(<2 x i64 addrspace(1)*>* %ptr, i1 %cnd, i32 %idx
 ; CHECK-NEXT:    [[OBJB:%.*]] = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* [[PTR]], align 16
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[VEC_BASE:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ], !is_base_value !0
 ; CHECK-NEXT:    [[VEC:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ]
 ; CHECK-NEXT:    br i1 [[CND]], label [[TAKEN2:%.*]], label [[UNTAKEN2:%.*]]
 ; CHECK:       taken2:
-; CHECK-NEXT:    [[OBJ0_BASE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC_BASE]], i32 [[IDX1:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[OBJ0:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX1]]
+; CHECK-NEXT:    [[OBJ0:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX1:%.*]]
 ; CHECK-NEXT:    br label [[MERGE2:%.*]]
 ; CHECK:       untaken2:
-; CHECK-NEXT:    [[OBJ1_BASE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC_BASE]], i32 [[IDX2:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[OBJ1:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX2]]
+; CHECK-NEXT:    [[OBJ1:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX2:%.*]]
 ; CHECK-NEXT:    br label [[MERGE2]]
 ; CHECK:       merge2:
-; CHECK-NEXT:    [[OBJ_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ0_BASE]], [[TAKEN2]] ], [ [[OBJ1_BASE]], [[UNTAKEN2]] ], !is_base_value !0
 ; CHECK-NEXT:    [[OBJ:%.*]] = phi i64 addrspace(1)* [ [[OBJ0]], [[TAKEN2]] ], [ [[OBJ1]], [[UNTAKEN2]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]], i64 addrspace(1)* [[OBJ_BASE]]) ]
-; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]]) ]
+; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[OBJ_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[OBJ_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_BASE_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    ret i64 addrspace(1)* [[OBJ_RELOCATED_CASTED]]
 ;
 entry:
@@ -282,13 +273,10 @@ define void @test9(<4 x i64 addrspace(1)*> %vec1, i64 %idx) gc "statepoint-examp
 ; CHECK-LABEL: @test9(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <4 x i64 addrspace(1)*> [[VEC1:%.*]], <4 x i64 addrspace(1)*> [[VEC1]], <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[BASE_EE:%.*]] = extractelement <4 x i64 addrspace(1)*> [[VEC1]], i64 [[IDX:%.*]], !is_base_value !0
-; CHECK-NEXT:    [[BDV:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i64 [[IDX]]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BASE_EE]]) ]
-; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[BDV:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i64 [[IDX:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ]
+; CHECK-NEXT:    [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[BASE_EE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASE_EE_RELOCATED]] to i64 addrspace(1)*
 ; CHECK-NEXT:    call void @use(i64 addrspace(1)* [[BDV_RELOCATED_CASTED]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll
index a38b556d0a37987eb6b1401afb973988d7358c05..31892b51c36089391bd79ff28ee782d564d039f7 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll
@@ -112,13 +112,10 @@ define <2 x i64 addrspace(1)*> @test6(i1 %cnd, <2 x i64 addrspace(1)*>* %ptr) gc
 ; CHECK-NEXT:    [[OBJB:%.*]] = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* [[PTR]], align 16
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[OBJ_BASE:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ], !is_base_value !0
 ; CHECK-NEXT:    [[OBJ:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(<2 x i64 addrspace(1)*> [[OBJ]], <2 x i64 addrspace(1)*> [[OBJ_BASE]]) ]
-; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(<2 x i64 addrspace(1)*> [[OBJ]]) ]
+; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[OBJ_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[OBJ_RELOCATED]] to <2 x i64 addrspace(1)*>
-; CHECK-NEXT:    [[OBJ_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[OBJ_BASE_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[OBJ_BASE_RELOCATED]] to <2 x i64 addrspace(1)*>
 ; CHECK-NEXT:    ret <2 x i64 addrspace(1)*> [[OBJ_RELOCATED_CASTED]]
 ;
 entry:
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll
index 6e6333f1dd89a6f392c9b2800d75c9edeaab28a4..76d06ac0517943fff5dd74c62bf7c9dcfc2f6e27 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll
@@ -112,13 +112,10 @@ define <2 x i64 addrspace(1)*> @test6(i1 %cnd, <2 x i64 addrspace(1)*>* %ptr) gc
 ; CHECK-NEXT:    [[OBJB:%.*]] = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* [[PTR]], align 16
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[OBJ_BASE:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ], !is_base_value !0
 ; CHECK-NEXT:    [[OBJ:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ]
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(<2 x i64 addrspace(1)*> [[OBJ]], <2 x i64 addrspace(1)*> [[OBJ_BASE]]) ]
-; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(<2 x i64 addrspace(1)*> [[OBJ]]) ]
+; CHECK-NEXT:    [[OBJ_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
 ; CHECK-NEXT:    [[OBJ_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[OBJ_RELOCATED]] to <2 x i64 addrspace(1)*>
-; CHECK-NEXT:    [[OBJ_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
-; CHECK-NEXT:    [[OBJ_BASE_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[OBJ_BASE_RELOCATED]] to <2 x i64 addrspace(1)*>
 ; CHECK-NEXT:    ret <2 x i64 addrspace(1)*> [[OBJ_RELOCATED_CASTED]]
 ;
 entry:
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/meetBDVState-hangs.ll b/llvm/test/Transforms/RewriteStatepointsForGC/meetBDVState-hangs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e70ebc4e3d7a6b63cc9f959beeffd41376b901de
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/meetBDVState-hangs.ll
@@ -0,0 +1,63 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+
+; Regression test to incorrectly testing fixed state causing infinite loop.
+; CHECK: test
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @bar(i8 addrspace(1)* nocapture readonly)
+declare noalias i8 addrspace(1)* @foo()
+
+define i8 addrspace(1)* @test(i1 %c, i1 %c1, i1 %c2, i1 %c3, i1 %c4, i1 %c5, i1 %c.exit) gc "statepoint-example" {
+entry:
+  br i1 %c, label %ph.L, label %ph.R
+ph.L:
+  %ph.L.p.b = call noalias nonnull i8 addrspace(1)* @foo()
+  %ph.L.p = getelementptr i8, i8 addrspace(1)* %ph.L.p.b, i64 8
+  br label %ph.M
+ph.R:
+  %ph.R.p = call noalias nonnull i8 addrspace(1)* @foo()
+  br label %ph.M
+ph.M:
+  %ph.M.p = phi i8 addrspace(1)* [ %ph.L.p, %ph.L ], [ %ph.R.p, %ph.R ]
+  br label %header
+  
+header:
+  %header.p = phi i8 addrspace(1)* [ %ph.M.p, %ph.M ], [ %backedge.p, %backedge]
+  br i1 %c1, label %loop.M, label %loop.R
+
+loop.R:
+  br i1 %c2, label %loop.R.M, label %loop.R.R
+
+loop.R.R:
+  %loop.R.R.p = call noalias nonnull i8 addrspace(1)* @foo()
+  br label %loop.R.M
+
+loop.R.M:
+  %loop.R.M.p = phi i8 addrspace(1)* [ %header.p, %loop.R ], [ %loop.R.R.p, %loop.R.R ]
+  br label %loop.M
+
+loop.M:
+  %loop.M.p = phi i8 addrspace(1)* [ %loop.R.M.p, %loop.R.M ], [ %header.p, %header ]
+  br i1 %c4, label %backedge, label %pre.backedge.R
+  
+pre.backedge.R:
+  br i1 %c5, label %pre.backedge.R.L, label %pre.backedge.R.R
+pre.backedge.R.L:
+  %pre.backedge.R.L.p.b = call noalias nonnull i8 addrspace(1)* @foo()
+  %pre.backedge.R.L.p = getelementptr i8, i8 addrspace(1)* %pre.backedge.R.L.p.b, i64 8
+  br label %pre.backedge.R.M
+pre.backedge.R.R:
+  %pre.backedge.R.R.p = call noalias nonnull i8 addrspace(1)* @foo()
+  br label %pre.backedge.R.M
+pre.backedge.R.M:
+  %pre.backedge.R.M.p = phi i8 addrspace(1)* [ %pre.backedge.R.L.p, %pre.backedge.R.L ], [ %pre.backedge.R.R.p, %pre.backedge.R.R ]
+  br label %backedge
+  
+backedge:
+  %backedge.p = phi i8 addrspace(1)* [ %pre.backedge.R.M.p, %pre.backedge.R.M ], [ %loop.M.p, %loop.M ]
+  br i1 %c.exit, label %header, label %exit
+  
+exit:                                                ; preds = %3, %1
+  call void @bar(i8 addrspace(1)* align 8 %header.p) [ "deopt"() ]
+  ret i8 addrspace(1)* %header.p
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll b/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll
index 6f674f443c94928940aed0215ab1e8270c78f7cb..77bedd875af03bc8d1abb59627888e92a1b0795f 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll
@@ -375,13 +375,12 @@ define void @contains_basephi(i1 %cond) gc "statepoint-example" {
 ; CHECK:       there:
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[BASEPHI_BASE:%.*]] = phi i32 addrspace(1)* [ [[BASE1]], [[HERE]] ], [ [[BASE2]], [[THERE]] ], !is_base_value !0
 ; CHECK-NEXT:    [[BASEPHI:%.*]] = phi i32 addrspace(1)* [ [[BASE1]], [[HERE]] ], [ [[BASE2]], [[THERE]] ]
 ; CHECK-NEXT:    [[PTR_GEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI]], i32 15
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i32 addrspace(1)* [[BASEPHI_BASE]]) ]
-; CHECK-NEXT:    [[BASEPHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
-; CHECK-NEXT:    [[BASEPHI_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASEPHI_BASE_RELOCATED]] to i32 addrspace(1)*
-; CHECK-NEXT:    [[PTR_GEP_REMAT:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_BASE_RELOCATED_CASTED]], i32 15
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i32 addrspace(1)* [[BASEPHI]]) ]
+; CHECK-NEXT:    [[BASEPHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
+; CHECK-NEXT:    [[BASEPHI_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASEPHI_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[PTR_GEP_REMAT:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_RELOCATED_CASTED]], i32 15
 ; CHECK-NEXT:    call void @use_obj32(i32 addrspace(1)* [[PTR_GEP_REMAT]])
 ; CHECK-NEXT:    ret void
 ;
@@ -419,17 +418,16 @@ define void @test_intersecting_chains_with_phi(i1 %cond) gc "statepoint-example"
 ; CHECK:       there:
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[BASEPHI_BASE:%.*]] = phi i32 addrspace(1)* [ [[BASE1]], [[HERE]] ], [ [[BASE2]], [[THERE]] ], !is_base_value !0
 ; CHECK-NEXT:    [[BASEPHI:%.*]] = phi i32 addrspace(1)* [ [[BASE1]], [[HERE]] ], [ [[BASE2]], [[THERE]] ]
 ; CHECK-NEXT:    [[PTR_GEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI]], i32 15
 ; CHECK-NEXT:    [[PTR_CAST:%.*]] = bitcast i32 addrspace(1)* [[PTR_GEP]] to i64 addrspace(1)*
 ; CHECK-NEXT:    [[PTR_CAST2:%.*]] = bitcast i32 addrspace(1)* [[PTR_GEP]] to i16 addrspace(1)*
-; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i32 addrspace(1)* [[BASEPHI_BASE]]) ]
-; CHECK-NEXT:    [[BASEPHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
-; CHECK-NEXT:    [[BASEPHI_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASEPHI_BASE_RELOCATED]] to i32 addrspace(1)*
-; CHECK-NEXT:    [[PTR_GEP_REMAT1:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_BASE_RELOCATED_CASTED]], i32 15
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i32 addrspace(1)* [[BASEPHI]]) ]
+; CHECK-NEXT:    [[BASEPHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
+; CHECK-NEXT:    [[BASEPHI_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASEPHI_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[PTR_GEP_REMAT1:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_RELOCATED_CASTED]], i32 15
 ; CHECK-NEXT:    [[PTR_CAST_REMAT:%.*]] = bitcast i32 addrspace(1)* [[PTR_GEP_REMAT1]] to i64 addrspace(1)*
-; CHECK-NEXT:    [[PTR_GEP_REMAT:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_BASE_RELOCATED_CASTED]], i32 15
+; CHECK-NEXT:    [[PTR_GEP_REMAT:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_RELOCATED_CASTED]], i32 15
 ; CHECK-NEXT:    [[PTR_CAST2_REMAT:%.*]] = bitcast i32 addrspace(1)* [[PTR_GEP_REMAT]] to i16 addrspace(1)*
 ; CHECK-NEXT:    call void @use_obj64(i64 addrspace(1)* [[PTR_CAST_REMAT]])
 ; CHECK-NEXT:    call void @use_obj16(i16 addrspace(1)* [[PTR_CAST2_REMAT]])
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll b/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll
index 82768f2b17a39af09b03487b34742b6aeb2435ca..2fb066608b4e0e2e8f83048e0b51b5fc35b00c28 100644
--- a/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll
@@ -14,13 +14,12 @@ define i32 addrspace(1)* @test1(i8 addrspace(1)* %base1, <2 x i64> %offsets) gc
 ; CHECK-NEXT:    [[BASE21:%.*]] = call i8 addrspace(1)* @llvm.experimental.gc.result.p1i8(token [[STATEPOINT_TOKEN]])
 ; CHECK-NEXT:    br label [[SECOND]]
 ; CHECK:       second:
-; CHECK-NEXT:    [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[BASE1:%.*]], [[ENTRY:%.*]] ], [ [[BASE21]], [[FIRST]] ], !is_base_value !0
-; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[BASE1]], [[ENTRY]] ], [ [[BASE21]], [[FIRST]] ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[BASE1:%.*]], [[ENTRY:%.*]] ], [ [[BASE21]], [[FIRST]] ]
 ; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast i8 addrspace(1)* [[PHI]] to i32 addrspace(1)*
-; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[PHI_BASE]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[PHI]] to i32 addrspace(1)*
 ; CHECK-NEXT:    [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> poison, i32 addrspace(1)* [[BASE_I32]], i32 0
-; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer, !is_base_value !0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]]
 ; CHECK-NEXT:    [[PTR_BASE:%.*]] = extractelement <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]], i32 1, !is_base_value !0
@@ -55,7 +54,7 @@ define i32 addrspace(1)* @test2(i8 addrspace(1)* %base, <2 x i64> %offsets) gc "
 ; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[BASE]] to i32 addrspace(1)*
 ; CHECK-NEXT:    [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> poison, i32 addrspace(1)* [[BASE_I32]], i32 0
-; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer, !is_base_value !0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]]
 ; CHECK-NEXT:    [[PTR_BASE:%.*]] = extractelement <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]], i32 1, !is_base_value !0
@@ -103,7 +102,7 @@ define i32 addrspace(1)* @test4(i8 addrspace(1)* %base, <2 x i64> %offsets) gc "
 ; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[BASE]] to i32 addrspace(1)*
 ; CHECK-NEXT:    [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> poison, i32 addrspace(1)* [[BASE_I32]], i32 0
-; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer, !is_base_value !0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]]
 ; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i32 addrspace(1)*> [[VEC]], <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]]) ]
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/trunc-insertion.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/trunc-insertion.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f6ab38bb3935e645018b985f00c01d7ddbd24779
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/trunc-insertion.ll
@@ -0,0 +1,100 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S | FileCheck %s
+target triple = "aarch64-unknown-linux-gnu"
+@d = internal unnamed_addr global i32 5, align 4
+
+define dso_local void @l() local_unnamed_addr {
+; CHECK-LABEL: @l(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i16> [ undef, [[BB:%.*]] ], [ [[TMP11:%.*]], [[BB25:%.*]] ]
+; CHECK-NEXT:    br i1 undef, label [[BB3:%.*]], label [[BB11:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[I4:%.*]] = zext i1 undef to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i16> [[TMP0]], undef
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt <2 x i16> [[TMP1]], <i16 8, i16 8>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <2 x i1> [[TMP2]] to <2 x i32>
+; CHECK-NEXT:    br label [[BB25]]
+; CHECK:       bb11:
+; CHECK-NEXT:    [[I12:%.*]] = zext i1 undef to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i16> [[TMP0]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <2 x i16> [[TMP4]] to <2 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule <2 x i64> undef, [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <2 x i1> [[TMP6]] to <2 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult <2 x i32> undef, [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = zext <2 x i1> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    br label [[BB25]]
+; CHECK:       bb25:
+; CHECK-NEXT:    [[I28:%.*]] = phi i32 [ [[I12]], [[BB11]] ], [ [[I4]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = phi <2 x i32> [ [[TMP9]], [[BB11]] ], [ [[TMP3]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP11]] = phi <2 x i16> [ [[TMP4]], [[BB11]] ], [ [[TMP1]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc <2 x i32> [[TMP10]] to <2 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i8> [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i8 [[TMP13]] to i32
+; CHECK-NEXT:    [[I31:%.*]] = and i32 undef, [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i8> [[TMP12]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i8 [[TMP15]] to i32
+; CHECK-NEXT:    [[I32:%.*]] = and i32 [[I31]], [[TMP16]]
+; CHECK-NEXT:    [[I33:%.*]] = and i32 [[I32]], [[I28]]
+; CHECK-NEXT:    br i1 undef, label [[BB34:%.*]], label [[BB1]]
+; CHECK:       bb34:
+; CHECK-NEXT:    [[I35:%.*]] = phi i32 [ [[I33]], [[BB25]] ]
+; CHECK-NEXT:    br label [[BB36:%.*]]
+; CHECK:       bb36:
+; CHECK-NEXT:    store i32 [[I35]], i32* @d, align 4
+; CHECK-NEXT:    ret void
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb25, %bb
+  %i = phi i16 [ undef, %bb ], [ %i29, %bb25 ]
+  %i2 = phi i16 [ undef, %bb ], [ %i30, %bb25 ]
+  br i1 undef, label %bb3, label %bb11
+
+bb3:                                              ; preds = %bb1
+  %i4 = zext i1 undef to i32
+  %i5 = xor i16 %i2, undef
+  %i6 = icmp ugt i16 %i5, 8
+  %i7 = zext i1 %i6 to i32
+  %i8 = xor i16 %i, undef
+  %i9 = icmp ugt i16 %i8, 8
+  %i10 = zext i1 %i9 to i32
+  br label %bb25
+
+bb11:                                             ; preds = %bb1
+  %i12 = zext i1 undef to i32
+  %i13 = xor i16 %i2, undef
+  %i14 = sext i16 %i13 to i64
+  %i15 = icmp ule i64 undef, %i14
+  %i16 = zext i1 %i15 to i32
+  %i17 = icmp ult i32 undef, %i16
+  %i18 = zext i1 %i17 to i32
+  %i19 = xor i16 %i, undef
+  %i20 = sext i16 %i19 to i64
+  %i21 = icmp ule i64 undef, %i20
+  %i22 = zext i1 %i21 to i32
+  %i23 = icmp ult i32 undef, %i22
+  %i24 = zext i1 %i23 to i32
+  br label %bb25
+
+bb25:                                             ; preds = %bb11, %bb3
+  %i26 = phi i32 [ %i24, %bb11 ], [ %i10, %bb3 ]
+  %i27 = phi i32 [ %i18, %bb11 ], [ %i7, %bb3 ]
+  %i28 = phi i32 [ %i12, %bb11 ], [ %i4, %bb3 ]
+  %i29 = phi i16 [ %i19, %bb11 ], [ %i8, %bb3 ]
+  %i30 = phi i16 [ %i13, %bb11 ], [ %i5, %bb3 ]
+  %i31 = and i32 undef, %i26
+  %i32 = and i32 %i31, %i27
+  %i33 = and i32 %i32, %i28
+  br i1 undef, label %bb34, label %bb1
+
+bb34:                                             ; preds = %bb25
+  %i35 = phi i32 [ %i33, %bb25 ]
+  br label %bb36
+
+bb36:                                             ; preds = %bb34
+  store i32 %i35, i32* @d, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_exceed_scheduling.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_exceed_scheduling.ll
new file mode 100644
index 0000000000000000000000000000000000000000..14d6920424e7cc8297c34209a65a142e3663fe3c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_exceed_scheduling.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -slp-min-tree-size=2 -slp-threshold=-1000 -slp-max-look-ahead-depth=1 -slp-look-ahead-users-budget=1 -slp-schedule-budget=27 -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define void @exceed(double %0, double %1) {
+; CHECK-LABEL: @exceed(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[TMP0:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> poison, double [[TMP1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> [[TMP4]], double [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = fdiv fast <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[TMP6]], i32 1
+; CHECK-NEXT:    [[IX:%.*]] = fmul double [[TMP7]], undef
+; CHECK-NEXT:    [[IXX0:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX1:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX2:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX3:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX4:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX5:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IX1:%.*]] = fmul double [[TMP7]], undef
+; CHECK-NEXT:    [[IXX10:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX11:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX12:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX13:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX14:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX15:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX20:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX21:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[IXX22:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP6]], i32 0
+; CHECK-NEXT:    [[IX2:%.*]] = fmul double [[TMP8]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x double> [[TMP2]], double [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd fast <2 x double> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd fast <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast <2 x double> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[IXX101:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x double> [[TMP13]], double undef, i32 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x double> <double undef, double poison>, double [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul fast <2 x double> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    switch i32 undef, label [[BB1:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[LABEL:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[LABEL]]
+; CHECK:       label:
+; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x double> [ [[TMP12]], [[BB1]] ], [ [[TMP16]], [[BB2]] ]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i10 = fdiv fast double %0, %1
+  %ix = fmul double %i10, undef
+  %ixx0 = fsub double undef, undef
+  %ixx1 = fsub double undef, undef
+  %ixx2 = fsub double undef, undef
+  %ixx3 = fsub double undef, undef
+  %ixx4 = fsub double undef, undef
+  %ixx5 = fsub double undef, undef
+  %ix1 = fmul double %i10, undef
+  %ixx10 = fsub double undef, undef
+  %ixx11 = fsub double undef, undef
+  %ixx12 = fsub double undef, undef
+  %ixx13 = fsub double undef, undef
+  %ixx14 = fsub double undef, undef
+  %ixx15 = fsub double undef, undef
+  %ixx20 = fsub double undef, undef
+  %ixx21 = fsub double undef, undef
+  %ixx22 = fsub double undef, undef
+  %i11 = fdiv fast double %0, %1
+  %ix2 = fmul double %i11, %i11
+  %tmp1 = fadd fast double %i11, %0
+  %tmp2 = fadd fast double %0, %1
+  %tmp5 = fmul fast double %tmp1, %tmp2
+  %tmp15 = fadd fast double %i10, %1
+  %tmp25 = fadd fast double %0, %1
+  %tmp6 = fmul fast double %tmp15, %tmp25
+  %tmp555 = fmul fast double %i10, undef
+  %ixx101 = fsub double undef, undef
+  %tmp666 = fmul fast double %1, undef
+  switch i32 undef, label %bb1 [
+  i32 0, label %bb2
+  ]
+
+bb1:                                              ; preds = %entry
+  br label %label
+
+bb2:                                              ; preds = %entry
+  br label %label
+
+label:                                            ; preds = %bb2, %bb1
+  %phi1 = phi double [ %tmp5, %bb1 ], [ %tmp555, %bb2 ]
+  %phi2 = phi double [ %tmp6, %bb1 ], [ %tmp666, %bb2 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
index 40962860b73177549f1637b3bcc3105109c75840..3d41293110f6ae40fc58e252279260f9efe25696 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
@@ -9,6 +9,13 @@
 @arrp = local_unnamed_addr global [32 x i32*] zeroinitializer, align 16
 @var = global i32 zeroinitializer, align 8
 
+declare i32 @llvm.smax.i32(i32, i32)
+declare i16 @llvm.smin.i16(i16, i16)
+declare i64 @llvm.umax.i64(i64, i64)
+declare i8 @llvm.umin.i8(i8, i8)
+declare i32 @llvm.smin.i32(i32, i32)
+declare i32 @llvm.umin.i32(i32, i32)
+
 define i32 @maxi8(i32) {
 ; CHECK-LABEL: @maxi8(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr to <8 x i32>*), align 16
@@ -198,8 +205,8 @@ define i32 @maxi32(i32) {
   ret i32 %95
 }
 
-; FIXME: Use fmaxnum intrinsics to match what InstCombine creates for fcmp+select
-; with fastmath on the select.
+; Note: legacy test - InstCombine creates maxnum intrinsics for fcmp+select with fastmath on the select.
+
 define float @maxf8(float) {
 ; DEFAULT-LABEL: @maxf8(
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
@@ -277,8 +284,8 @@ define float @maxf8(float) {
   ret float %23
 }
 
-; FIXME: Use fmaxnum intrinsics to match what InstCombine creates for fcmp+select
-; with fastmath on the select.
+; Note: legacy test - maxnum intrinsics match what InstCombine creates for fcmp+select with fastmath on the select.
+
 define float @maxf16(float) {
 ; DEFAULT-LABEL: @maxf16(
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
@@ -428,8 +435,8 @@ define float @maxf16(float) {
   ret float %47
 }
 
-; FIXME: Use fmaxnum intrinsics to match what InstCombine creates for fcmp+select
-; with fastmath on the select.
+; Note: legacy test - InstCombine creates maxnum intrinsics for fcmp+select with fastmath on the select.
+
 define float @maxf32(float) {
 ; DEFAULT-LABEL: @maxf32(
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
@@ -1001,3 +1008,270 @@ define i32* @maxp8(i32) {
   %23 = select i1 %22, i32* %20, i32* %21
   ret i32* %23
 }
+
+define i32 @smax_intrinsic_rdx_v8i32(i32* %p0) {
+; CHECK-LABEL: @smax_intrinsic_rdx_v8i32(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 7
+; CHECK-NEXT:    [[T0:%.*]] = load i32, i32* [[P0]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[P1]], align 4
+; CHECK-NEXT:    [[T2:%.*]] = load i32, i32* [[P2]], align 4
+; CHECK-NEXT:    [[T3:%.*]] = load i32, i32* [[P3]], align 4
+; CHECK-NEXT:    [[T4:%.*]] = load i32, i32* [[P4]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = load i32, i32* [[P5]], align 4
+; CHECK-NEXT:    [[T6:%.*]] = load i32, i32* [[P6]], align 4
+; CHECK-NEXT:    [[T7:%.*]] = load i32, i32* [[P7]], align 4
+; CHECK-NEXT:    [[M10:%.*]] = tail call i32 @llvm.smax.i32(i32 [[T1]], i32 [[T0]])
+; CHECK-NEXT:    [[M32:%.*]] = tail call i32 @llvm.smax.i32(i32 [[T3]], i32 [[T2]])
+; CHECK-NEXT:    [[M54:%.*]] = tail call i32 @llvm.smax.i32(i32 [[T5]], i32 [[T4]])
+; CHECK-NEXT:    [[M76:%.*]] = tail call i32 @llvm.smax.i32(i32 [[T7]], i32 [[T6]])
+; CHECK-NEXT:    [[M3210:%.*]] = tail call i32 @llvm.smax.i32(i32 [[M32]], i32 [[M10]])
+; CHECK-NEXT:    [[M7654:%.*]] = tail call i32 @llvm.smax.i32(i32 [[M76]], i32 [[M54]])
+; CHECK-NEXT:    [[M:%.*]] = tail call i32 @llvm.smax.i32(i32 [[M7654]], i32 [[M3210]])
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %p1 = getelementptr inbounds i32, i32* %p0, i64 1
+  %p2 = getelementptr inbounds i32, i32* %p0, i64 2
+  %p3 = getelementptr inbounds i32, i32* %p0, i64 3
+  %p4 = getelementptr inbounds i32, i32* %p0, i64 4
+  %p5 = getelementptr inbounds i32, i32* %p0, i64 5
+  %p6 = getelementptr inbounds i32, i32* %p0, i64 6
+  %p7 = getelementptr inbounds i32, i32* %p0, i64 7
+  %t0 = load i32, i32* %p0, align 4
+  %t1 = load i32, i32* %p1, align 4
+  %t2 = load i32, i32* %p2, align 4
+  %t3 = load i32, i32* %p3, align 4
+  %t4 = load i32, i32* %p4, align 4
+  %t5 = load i32, i32* %p5, align 4
+  %t6 = load i32, i32* %p6, align 4
+  %t7 = load i32, i32* %p7, align 4
+  %m10 = tail call i32 @llvm.smax.i32(i32 %t1, i32 %t0)
+  %m32 = tail call i32 @llvm.smax.i32(i32 %t3, i32 %t2)
+  %m54 = tail call i32 @llvm.smax.i32(i32 %t5, i32 %t4)
+  %m76 = tail call i32 @llvm.smax.i32(i32 %t7, i32 %t6)
+  %m3210 = tail call i32 @llvm.smax.i32(i32 %m32, i32 %m10)
+  %m7654 = tail call i32 @llvm.smax.i32(i32 %m76, i32 %m54)
+  %m = tail call i32 @llvm.smax.i32(i32 %m7654, i32 %m3210)
+  ret i32 %m
+}
+
+define i16 @smin_intrinsic_rdx_v8i16(i16* %p0) {
+; CHECK-LABEL: @smin_intrinsic_rdx_v8i16(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; CHECK-NEXT:    [[T0:%.*]] = load i16, i16* [[P0]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load i16, i16* [[P1]], align 4
+; CHECK-NEXT:    [[T2:%.*]] = load i16, i16* [[P2]], align 4
+; CHECK-NEXT:    [[T3:%.*]] = load i16, i16* [[P3]], align 4
+; CHECK-NEXT:    [[T4:%.*]] = load i16, i16* [[P4]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = load i16, i16* [[P5]], align 4
+; CHECK-NEXT:    [[T6:%.*]] = load i16, i16* [[P6]], align 4
+; CHECK-NEXT:    [[T7:%.*]] = load i16, i16* [[P7]], align 4
+; CHECK-NEXT:    [[M10:%.*]] = tail call i16 @llvm.smin.i16(i16 [[T1]], i16 [[T0]])
+; CHECK-NEXT:    [[M32:%.*]] = tail call i16 @llvm.smin.i16(i16 [[T3]], i16 [[T2]])
+; CHECK-NEXT:    [[M54:%.*]] = tail call i16 @llvm.smin.i16(i16 [[T5]], i16 [[T4]])
+; CHECK-NEXT:    [[M76:%.*]] = tail call i16 @llvm.smin.i16(i16 [[T7]], i16 [[T6]])
+; CHECK-NEXT:    [[M3210:%.*]] = tail call i16 @llvm.smin.i16(i16 [[M32]], i16 [[M10]])
+; CHECK-NEXT:    [[M7654:%.*]] = tail call i16 @llvm.smin.i16(i16 [[M76]], i16 [[M54]])
+; CHECK-NEXT:    [[M:%.*]] = tail call i16 @llvm.smin.i16(i16 [[M7654]], i16 [[M3210]])
+; CHECK-NEXT:    ret i16 [[M]]
+;
+  %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+  %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+  %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+  %p4 = getelementptr inbounds i16, i16* %p0, i64 4
+  %p5 = getelementptr inbounds i16, i16* %p0, i64 5
+  %p6 = getelementptr inbounds i16, i16* %p0, i64 6
+  %p7 = getelementptr inbounds i16, i16* %p0, i64 7
+  %t0 = load i16, i16* %p0, align 4
+  %t1 = load i16, i16* %p1, align 4
+  %t2 = load i16, i16* %p2, align 4
+  %t3 = load i16, i16* %p3, align 4
+  %t4 = load i16, i16* %p4, align 4
+  %t5 = load i16, i16* %p5, align 4
+  %t6 = load i16, i16* %p6, align 4
+  %t7 = load i16, i16* %p7, align 4
+  %m10 = tail call i16 @llvm.smin.i16(i16 %t1, i16 %t0)
+  %m32 = tail call i16 @llvm.smin.i16(i16 %t3, i16 %t2)
+  %m54 = tail call i16 @llvm.smin.i16(i16 %t5, i16 %t4)
+  %m76 = tail call i16 @llvm.smin.i16(i16 %t7, i16 %t6)
+  %m3210 = tail call i16 @llvm.smin.i16(i16 %m32, i16 %m10)
+  %m7654 = tail call i16 @llvm.smin.i16(i16 %m76, i16 %m54)
+  %m = tail call i16 @llvm.smin.i16(i16 %m7654, i16 %m3210)
+  ret i16 %m
+}
+
+define i64 @umax_intrinsic_rdx_v4i64(i64* %p0) {
+; CHECK-LABEL: @umax_intrinsic_rdx_v4i64(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i64, i64* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i64, i64* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i64, i64* [[P0]], i64 3
+; CHECK-NEXT:    [[T0:%.*]] = load i64, i64* [[P0]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load i64, i64* [[P1]], align 4
+; CHECK-NEXT:    [[T2:%.*]] = load i64, i64* [[P2]], align 4
+; CHECK-NEXT:    [[T3:%.*]] = load i64, i64* [[P3]], align 4
+; CHECK-NEXT:    [[M10:%.*]] = tail call i64 @llvm.umax.i64(i64 [[T1]], i64 [[T0]])
+; CHECK-NEXT:    [[M32:%.*]] = tail call i64 @llvm.umax.i64(i64 [[T3]], i64 [[T2]])
+; CHECK-NEXT:    [[M:%.*]] = tail call i64 @llvm.umax.i64(i64 [[M32]], i64 [[M10]])
+; CHECK-NEXT:    ret i64 [[M]]
+;
+  %p1 = getelementptr inbounds i64, i64* %p0, i64 1
+  %p2 = getelementptr inbounds i64, i64* %p0, i64 2
+  %p3 = getelementptr inbounds i64, i64* %p0, i64 3
+  %t0 = load i64, i64* %p0, align 4
+  %t1 = load i64, i64* %p1, align 4
+  %t2 = load i64, i64* %p2, align 4
+  %t3 = load i64, i64* %p3, align 4
+  %m10 = tail call i64 @llvm.umax.i64(i64 %t1, i64 %t0)
+  %m32 = tail call i64 @llvm.umax.i64(i64 %t3, i64 %t2)
+  %m = tail call i64 @llvm.umax.i64(i64 %m32, i64 %m10)
+  ret i64 %m
+}
+
+define i8 @umin_intrinsic_rdx_v16i8(i8* %p0) {
+; CHECK-LABEL: @umin_intrinsic_rdx_v16i8(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; CHECK-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; CHECK-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; CHECK-NEXT:    [[PA:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; CHECK-NEXT:    [[PB:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; CHECK-NEXT:    [[PC:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; CHECK-NEXT:    [[PD:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; CHECK-NEXT:    [[PE:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; CHECK-NEXT:    [[PF:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; CHECK-NEXT:    [[T0:%.*]] = load i8, i8* [[P0]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load i8, i8* [[P1]], align 4
+; CHECK-NEXT:    [[T2:%.*]] = load i8, i8* [[P2]], align 4
+; CHECK-NEXT:    [[T3:%.*]] = load i8, i8* [[P3]], align 4
+; CHECK-NEXT:    [[T4:%.*]] = load i8, i8* [[P4]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = load i8, i8* [[P5]], align 4
+; CHECK-NEXT:    [[T6:%.*]] = load i8, i8* [[P6]], align 4
+; CHECK-NEXT:    [[T7:%.*]] = load i8, i8* [[P7]], align 4
+; CHECK-NEXT:    [[T8:%.*]] = load i8, i8* [[P8]], align 4
+; CHECK-NEXT:    [[T9:%.*]] = load i8, i8* [[P9]], align 4
+; CHECK-NEXT:    [[TA:%.*]] = load i8, i8* [[PA]], align 4
+; CHECK-NEXT:    [[TB:%.*]] = load i8, i8* [[PB]], align 4
+; CHECK-NEXT:    [[TC:%.*]] = load i8, i8* [[PC]], align 4
+; CHECK-NEXT:    [[TD:%.*]] = load i8, i8* [[PD]], align 4
+; CHECK-NEXT:    [[TE:%.*]] = load i8, i8* [[PE]], align 4
+; CHECK-NEXT:    [[TF:%.*]] = load i8, i8* [[PF]], align 4
+; CHECK-NEXT:    [[M10:%.*]] = tail call i8 @llvm.umin.i8(i8 [[T1]], i8 [[T0]])
+; CHECK-NEXT:    [[M32:%.*]] = tail call i8 @llvm.umin.i8(i8 [[T3]], i8 [[T2]])
+; CHECK-NEXT:    [[M54:%.*]] = tail call i8 @llvm.umin.i8(i8 [[T5]], i8 [[T4]])
+; CHECK-NEXT:    [[M76:%.*]] = tail call i8 @llvm.umin.i8(i8 [[T7]], i8 [[T6]])
+; CHECK-NEXT:    [[M98:%.*]] = tail call i8 @llvm.umin.i8(i8 [[T9]], i8 [[T8]])
+; CHECK-NEXT:    [[MBA:%.*]] = tail call i8 @llvm.umin.i8(i8 [[TB]], i8 [[TA]])
+; CHECK-NEXT:    [[MDC:%.*]] = tail call i8 @llvm.umin.i8(i8 [[TD]], i8 [[TC]])
+; CHECK-NEXT:    [[MFE:%.*]] = tail call i8 @llvm.umin.i8(i8 [[TF]], i8 [[TE]])
+; CHECK-NEXT:    [[M3210:%.*]] = tail call i8 @llvm.umin.i8(i8 [[M32]], i8 [[M10]])
+; CHECK-NEXT:    [[M7654:%.*]] = tail call i8 @llvm.umin.i8(i8 [[M76]], i8 [[M54]])
+; CHECK-NEXT:    [[MDC98:%.*]] = tail call i8 @llvm.umin.i8(i8 [[MDC]], i8 [[M98]])
+; CHECK-NEXT:    [[MFEBA:%.*]] = tail call i8 @llvm.umin.i8(i8 [[MFE]], i8 [[MBA]])
+; CHECK-NEXT:    [[ML:%.*]] = tail call i8 @llvm.umin.i8(i8 [[M3210]], i8 [[M7654]])
+; CHECK-NEXT:    [[MH:%.*]] = tail call i8 @llvm.umin.i8(i8 [[MFEBA]], i8 [[MDC98]])
+; CHECK-NEXT:    [[M:%.*]] = tail call i8 @llvm.umin.i8(i8 [[MH]], i8 [[ML]])
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+  %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+  %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+  %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+  %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+  %p8 = getelementptr inbounds i8, i8* %p0, i64 8
+  %p9 = getelementptr inbounds i8, i8* %p0, i64 9
+  %pa = getelementptr inbounds i8, i8* %p0, i64 10
+  %pb = getelementptr inbounds i8, i8* %p0, i64 11
+  %pc = getelementptr inbounds i8, i8* %p0, i64 12
+  %pd = getelementptr inbounds i8, i8* %p0, i64 13
+  %pe = getelementptr inbounds i8, i8* %p0, i64 14
+  %pf = getelementptr inbounds i8, i8* %p0, i64 15
+  %t0 = load i8, i8* %p0, align 4
+  %t1 = load i8, i8* %p1, align 4
+  %t2 = load i8, i8* %p2, align 4
+  %t3 = load i8, i8* %p3, align 4
+  %t4 = load i8, i8* %p4, align 4
+  %t5 = load i8, i8* %p5, align 4
+  %t6 = load i8, i8* %p6, align 4
+  %t7 = load i8, i8* %p7, align 4
+  %t8 = load i8, i8* %p8, align 4
+  %t9 = load i8, i8* %p9, align 4
+  %ta = load i8, i8* %pa, align 4
+  %tb = load i8, i8* %pb, align 4
+  %tc = load i8, i8* %pc, align 4
+  %td = load i8, i8* %pd, align 4
+  %te = load i8, i8* %pe, align 4
+  %tf = load i8, i8* %pf, align 4
+  %m10 = tail call i8 @llvm.umin.i8(i8 %t1, i8 %t0)
+  %m32 = tail call i8 @llvm.umin.i8(i8 %t3, i8 %t2)
+  %m54 = tail call i8 @llvm.umin.i8(i8 %t5, i8 %t4)
+  %m76 = tail call i8 @llvm.umin.i8(i8 %t7, i8 %t6)
+  %m98 = tail call i8 @llvm.umin.i8(i8 %t9, i8 %t8)
+  %mba = tail call i8 @llvm.umin.i8(i8 %tb, i8 %ta)
+  %mdc = tail call i8 @llvm.umin.i8(i8 %td, i8 %tc)
+  %mfe = tail call i8 @llvm.umin.i8(i8 %tf, i8 %te)
+  %m3210 = tail call i8 @llvm.umin.i8(i8 %m32, i8 %m10)
+  %m7654 = tail call i8 @llvm.umin.i8(i8 %m76, i8 %m54)
+  %mdc98 = tail call i8 @llvm.umin.i8(i8 %mdc, i8 %m98)
+  %mfeba = tail call i8 @llvm.umin.i8(i8 %mfe, i8 %mba)
+  %ml = tail call i8 @llvm.umin.i8(i8 %m3210, i8 %m7654)
+  %mh = tail call i8 @llvm.umin.i8(i8 %mfeba, i8 %mdc98)
+  %m = tail call i8 @llvm.umin.i8(i8 %mh, i8 %ml)
+  ret i8 %m
+}
+
+; This should not crash.
+
+define void @PR49730() {
+; CHECK-LABEL: @PR49730(
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.smin.i32(i32 undef, i32 2)
+; CHECK-NEXT:    [[T1:%.*]] = sub nsw i32 undef, [[T]]
+; CHECK-NEXT:    [[T2:%.*]] = call i32 @llvm.umin.i32(i32 undef, i32 [[T1]])
+; CHECK-NEXT:    [[T3:%.*]] = call i32 @llvm.smin.i32(i32 undef, i32 2)
+; CHECK-NEXT:    [[T4:%.*]] = sub nsw i32 undef, [[T3]]
+; CHECK-NEXT:    [[T5:%.*]] = call i32 @llvm.umin.i32(i32 [[T2]], i32 [[T4]])
+; CHECK-NEXT:    [[T6:%.*]] = call i32 @llvm.smin.i32(i32 undef, i32 1)
+; CHECK-NEXT:    [[T7:%.*]] = sub nuw nsw i32 undef, [[T6]]
+; CHECK-NEXT:    [[T8:%.*]] = call i32 @llvm.umin.i32(i32 [[T5]], i32 [[T7]])
+; CHECK-NEXT:    [[T9:%.*]] = call i32 @llvm.smin.i32(i32 undef, i32 1)
+; CHECK-NEXT:    [[T10:%.*]] = sub nsw i32 undef, [[T9]]
+; CHECK-NEXT:    [[T11:%.*]] = call i32 @llvm.umin.i32(i32 [[T8]], i32 [[T10]])
+; CHECK-NEXT:    [[T12:%.*]] = sub nsw i32 undef, undef
+; CHECK-NEXT:    [[T13:%.*]] = call i32 @llvm.umin.i32(i32 [[T11]], i32 [[T12]])
+; CHECK-NEXT:    [[T14:%.*]] = call i32 @llvm.umin.i32(i32 [[T13]], i32 93)
+; CHECK-NEXT:    ret void
+;
+  %t = call i32 @llvm.smin.i32(i32 undef, i32 2)
+  %t1 = sub nsw i32 undef, %t
+  %t2 = call i32 @llvm.umin.i32(i32 undef, i32 %t1)
+  %t3 = call i32 @llvm.smin.i32(i32 undef, i32 2)
+  %t4 = sub nsw i32 undef, %t3
+  %t5 = call i32 @llvm.umin.i32(i32 %t2, i32 %t4)
+  %t6 = call i32 @llvm.smin.i32(i32 undef, i32 1)
+  %t7 = sub nuw nsw i32 undef, %t6
+  %t8 = call i32 @llvm.umin.i32(i32 %t5, i32 %t7)
+  %t9 = call i32 @llvm.smin.i32(i32 undef, i32 1)
+  %t10 = sub nsw i32 undef, %t9
+  %t11 = call i32 @llvm.umin.i32(i32 %t8, i32 %t10)
+  %t12 = sub nsw i32 undef, undef
+  %t13 = call i32 @llvm.umin.i32(i32 %t11, i32 %t12)
+  %t14 = call i32 @llvm.umin.i32(i32 %t13, i32 93)
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
index 267cf1a02c298f43108e1391841f3e40fe0ec496..e283628949107ee8cd239fc7559142bd4e7d81b4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -slp-vectorizer -slp-vectorizer -mattr=+sse2 -S | FileCheck %s --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -slp-vectorizer -slp-vectorizer -mattr=+avx  -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -slp-vectorizer -slp-vectorizer -mattr=+avx2 -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -slp-vectorizer -mattr=+sse2 -S | FileCheck %s --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -slp-vectorizer -mattr=+avx  -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -slp-vectorizer -mattr=+avx2 -S | FileCheck %s --check-prefix=AVX
 
 %class.1 = type { %class.2 }
 %class.2 = type { %"class.3" }
@@ -117,13 +117,10 @@ define void @pr35497() local_unnamed_addr #0 {
 ; AVX-NEXT:    [[ARRAYIDX2_6:%.*]] = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 0
 ; AVX-NEXT:    [[TMP10:%.*]] = bitcast i64* [[ARRAYIDX2_6]] to <2 x i64>*
 ; AVX-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP10]], align 1
-; AVX-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
-; AVX-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> poison, i64 [[TMP11]], i32 0
-; AVX-NEXT:    [[TMP13:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP5]], i32 1
-; AVX-NEXT:    [[TMP14:%.*]] = lshr <2 x i64> [[TMP13]], <i64 6, i64 6>
-; AVX-NEXT:    [[TMP15:%.*]] = add nuw nsw <2 x i64> [[TMP9]], [[TMP14]]
-; AVX-NEXT:    [[TMP16:%.*]] = bitcast i64* [[ARRAYIDX2_2]] to <2 x i64>*
-; AVX-NEXT:    store <2 x i64> [[TMP15]], <2 x i64>* [[TMP16]], align 1
+; AVX-NEXT:    [[TMP11:%.*]] = lshr <2 x i64> [[TMP4]], <i64 6, i64 6>
+; AVX-NEXT:    [[TMP12:%.*]] = add nuw nsw <2 x i64> [[TMP9]], [[TMP11]]
+; AVX-NEXT:    [[TMP13:%.*]] = bitcast i64* [[ARRAYIDX2_2]] to <2 x i64>*
+; AVX-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* [[TMP13]], align 1
 ; AVX-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/slp-max-reg-size.ll b/llvm/test/Transforms/SLPVectorizer/slp-max-reg-size.ll
new file mode 100644
index 0000000000000000000000000000000000000000..be03ab7813e4e8f31c20ddb1117b1981703cdfb0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/slp-max-reg-size.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -slp-max-vf=8 -slp-min-reg-size=32 -slp-max-reg-size=160 -S | FileCheck  -check-prefix CHECK-VF8-160 %s
+; RUN: opt < %s -slp-vectorizer -slp-max-vf=4 -slp-min-reg-size=32 -slp-max-reg-size=160 -S | FileCheck  -check-prefix CHECK-VF4-160 %s
+; RUN: opt < %s -slp-vectorizer -slp-max-vf=2 -slp-min-reg-size=32 -slp-max-reg-size=160 -S | FileCheck  -check-prefix CHECK-VF2-160 %s
+; RUN: opt < %s -slp-vectorizer -slp-max-vf=8 -slp-min-reg-size=32 -slp-max-reg-size=128 -S | FileCheck -check-prefix CHECK-VF8-128 %s
+; RUN: opt < %s -slp-vectorizer -slp-max-vf=4 -slp-min-reg-size=32 -slp-max-reg-size=128 -S | FileCheck -check-prefix CHECK-VF4-128 %s
+; RUN: opt < %s -slp-vectorizer -slp-max-vf=2 -slp-min-reg-size=32 -slp-max-reg-size=128 -S | FileCheck -check-prefix CHECK-VF2-128 %s
+; RUN: opt < %s -slp-vectorizer -slp-max-vf=8 -slp-min-reg-size=32 -slp-max-reg-size=256 -S | FileCheck -check-prefix CHECK-VF8-256 %s
+; RUN: opt < %s -slp-vectorizer -slp-max-vf=2 -slp-min-reg-size=128 -slp-max-reg-size=128 -S | FileCheck -check-prefix CHECK-VF2-128-128 %s
+
+; This test case is checking if max register size (and max VF) is honored when
+; vecorizing store chains in SLPVectorizer.
+
+@X = external global [8 x i32], align 1
+
+define void @foo() {
+; CHECK-VF8-160-LABEL: @foo(
+; CHECK-VF8-160-NEXT:    store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* bitcast ([8 x i32]* @X to <4 x i32>*), align 1
+; CHECK-VF8-160-NEXT:    store <4 x i32> <i32 5, i32 6, i32 7, i32 8>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 4) to <4 x i32>*), align 1
+; CHECK-VF8-160-NEXT:    ret void
+;
+; CHECK-VF4-160-LABEL: @foo(
+; CHECK-VF4-160-NEXT:    store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* bitcast ([8 x i32]* @X to <4 x i32>*), align 1
+; CHECK-VF4-160-NEXT:    store <4 x i32> <i32 5, i32 6, i32 7, i32 8>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 4) to <4 x i32>*), align 1
+; CHECK-VF4-160-NEXT:    ret void
+;
+; CHECK-VF2-160-LABEL: @foo(
+; CHECK-VF2-160-NEXT:    store <2 x i32> <i32 1, i32 2>, <2 x i32>* bitcast ([8 x i32]* @X to <2 x i32>*), align 1
+; CHECK-VF2-160-NEXT:    store <2 x i32> <i32 3, i32 4>, <2 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 2) to <2 x i32>*), align 1
+; CHECK-VF2-160-NEXT:    store <2 x i32> <i32 5, i32 6>, <2 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 4) to <2 x i32>*), align 1
+; CHECK-VF2-160-NEXT:    store <2 x i32> <i32 7, i32 8>, <2 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 6) to <2 x i32>*), align 1
+; CHECK-VF2-160-NEXT:    ret void
+;
+; CHECK-VF8-128-LABEL: @foo(
+; CHECK-VF8-128-NEXT:    store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* bitcast ([8 x i32]* @X to <4 x i32>*), align 1
+; CHECK-VF8-128-NEXT:    store <4 x i32> <i32 5, i32 6, i32 7, i32 8>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 4) to <4 x i32>*), align 1
+; CHECK-VF8-128-NEXT:    ret void
+;
+; CHECK-VF4-128-LABEL: @foo(
+; CHECK-VF4-128-NEXT:    store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* bitcast ([8 x i32]* @X to <4 x i32>*), align 1
+; CHECK-VF4-128-NEXT:    store <4 x i32> <i32 5, i32 6, i32 7, i32 8>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 4) to <4 x i32>*), align 1
+; CHECK-VF4-128-NEXT:    ret void
+;
+; CHECK-VF2-128-LABEL: @foo(
+; CHECK-VF2-128-NEXT:    store <2 x i32> <i32 1, i32 2>, <2 x i32>* bitcast ([8 x i32]* @X to <2 x i32>*), align 1
+; CHECK-VF2-128-NEXT:    store <2 x i32> <i32 3, i32 4>, <2 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 2) to <2 x i32>*), align 1
+; CHECK-VF2-128-NEXT:    store <2 x i32> <i32 5, i32 6>, <2 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 4) to <2 x i32>*), align 1
+; CHECK-VF2-128-NEXT:    store <2 x i32> <i32 7, i32 8>, <2 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 6) to <2 x i32>*), align 1
+; CHECK-VF2-128-NEXT:    ret void
+;
+; CHECK-VF8-256-LABEL: @foo(
+; CHECK-VF8-256-NEXT:    store <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>, <8 x i32>* bitcast ([8 x i32]* @X to <8 x i32>*), align 1
+; CHECK-VF8-256-NEXT:    ret void
+;
+; CHECK-VF2-128-128-LABEL: @foo(
+; CHECK-VF2-128-128-NEXT:    store i32 1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 0), align 1
+; CHECK-VF2-128-128-NEXT:    store i32 2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 1), align 1
+; CHECK-VF2-128-128-NEXT:    store i32 3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 2), align 1
+; CHECK-VF2-128-128-NEXT:    store i32 4, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 3), align 1
+; CHECK-VF2-128-128-NEXT:    store i32 5, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 4), align 1
+; CHECK-VF2-128-128-NEXT:    store i32 6, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 5), align 1
+; CHECK-VF2-128-128-NEXT:    store i32 7, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 6), align 1
+; CHECK-VF2-128-128-NEXT:    store i32 8, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 7), align 1
+; CHECK-VF2-128-128-NEXT:    ret void
+;
+  store i32 1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 0), align 1
+  store i32 2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 1), align 1
+  store i32 3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 2), align 1
+  store i32 4, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 3), align 1
+  store i32 5, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 4), align 1
+  store i32 6, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 5), align 1
+  store i32 7, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 6), align 1
+  store i32 8, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @X, i16 0, i16 7), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/slp-umax-rdx-matcher-crash.ll b/llvm/test/Transforms/SLPVectorizer/slp-umax-rdx-matcher-crash.ll
index 11adbd0c1439916a9c4d66eabb15d9f1a8826a0d..192f51b96dc65758f31a5a68a3c5252299ab7968 100644
--- a/llvm/test/Transforms/SLPVectorizer/slp-umax-rdx-matcher-crash.ll
+++ b/llvm/test/Transforms/SLPVectorizer/slp-umax-rdx-matcher-crash.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -slp-vectorizer -S < %s 2>&1 | FileCheck %s
-; REQUIRES: asserts
+; RUN: opt -slp-vectorizer -S < %s | FileCheck %s
 
 ; Given LLVM IR caused associative reduction matching routine crash in SLP.
 ; The routines begins with select as integer Umax reduction kind
@@ -37,3 +36,41 @@ next:
 
 declare i8 @llvm.umax.i8(i8, i8)
 
+declare i32 @llvm.smin.i32(i32, i32)
+declare i32 @llvm.umin.i32(i32, i32)
+
+; Given LLVM IR caused crash in SLP.
+define void @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMIN0:%.*]] = call i32 @llvm.smin.i32(i32 undef, i32 0)
+; CHECK-NEXT:    [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 undef, i32 1)
+; CHECK-NEXT:    [[SMIN2:%.*]] = call i32 @llvm.smin.i32(i32 undef, i32 2)
+; CHECK-NEXT:    [[SMIN3:%.*]] = call i32 @llvm.smin.i32(i32 undef, i32 3)
+; CHECK-NEXT:    [[A:%.*]] = sub nsw i32 undef, [[SMIN0]]
+; CHECK-NEXT:    [[B:%.*]] = sub nsw i32 undef, [[SMIN1]]
+; CHECK-NEXT:    [[C:%.*]] = sub nsw i32 undef, [[SMIN2]]
+; CHECK-NEXT:    [[D:%.*]] = sub nsw i32 undef, [[SMIN3]]
+; CHECK-NEXT:    [[UMIN0:%.*]] = call i32 @llvm.umin.i32(i32 [[D]], i32 [[C]])
+; CHECK-NEXT:    [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN0]], i32 [[B]])
+; CHECK-NEXT:    [[UMIN2:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN1]], i32 [[A]])
+; CHECK-NEXT:    [[UMIN3:%.*]] = call i32 @llvm.umin.i32(i32 [[UMIN2]], i32 77)
+; CHECK-NEXT:    [[E:%.*]] = icmp ugt i32 [[UMIN3]], 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %smin0 = call i32 @llvm.smin.i32(i32 undef, i32 0)
+  %smin1 = call i32 @llvm.smin.i32(i32 undef, i32 1)
+  %smin2 = call i32 @llvm.smin.i32(i32 undef, i32 2)
+  %smin3 = call i32 @llvm.smin.i32(i32 undef, i32 3)
+  %a = sub nsw i32 undef, %smin0
+  %b = sub nsw i32 undef, %smin1
+  %c = sub nsw i32 undef, %smin2
+  %d = sub nsw i32 undef, %smin3
+  %umin0 = call i32 @llvm.umin.i32(i32 %d, i32 %c)
+  %umin1 = call i32 @llvm.umin.i32(i32 %umin0, i32 %b)
+  %umin2 = call i32 @llvm.umin.i32(i32 %umin1, i32 %a)
+  %umin3 = call i32 @llvm.umin.i32(i32 %umin2, i32 77)
+  %e = icmp ugt i32 %umin3, 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SROA/2009-02-20-InstCombine-SROA.ll b/llvm/test/Transforms/SROA/2009-02-20-InstCombine-SROA.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e0c2290dcabad01997747b7af542e6e77426d6db
--- /dev/null
+++ b/llvm/test/Transforms/SROA/2009-02-20-InstCombine-SROA.ll
@@ -0,0 +1,272 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -sroa -S | FileCheck %s
+
+; rdar://6417724
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+
+%"struct.std::vector<int,std::allocator<int> >" = type { %"struct.std::_Vector_base<int,std::allocator<int> >" }
+%"struct.std::_Vector_base<int,std::allocator<int> >" = type { %"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl" }
+%"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl" = type { i32*, i32*, i32* }
+
+define i32* @_Z3fooRSt6vectorIiSaIiEE(%"struct.std::vector<int,std::allocator<int> >"* %X) {
+; CHECK-LABEL: @_Z3fooRSt6vectorIiSaIiEE(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0:%.*]] = alloca i32*, align 8
+; CHECK-NEXT:    [[__LAST_ADDR_I_I_SROA_0:%.*]] = alloca i32*, align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* [[X:%.*]], i32 0, i32 0, i32 0, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32*, i32** [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* [[X]], i32 0, i32 0, i32 0, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32*, i32** [[TMP2]], align 4
+; CHECK-NEXT:    store i32* [[TMP3]], i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    store i32* [[TMP1]], i32** [[__LAST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint i32* [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint i32* [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i32 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = ashr i32 [[TMP6]], 4
+; CHECK-NEXT:    br label [[BB12_I_I:%.*]]
+; CHECK:       bb.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_13:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_13]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 42
+; CHECK-NEXT:    br i1 [[TMP9]], label [[BB1_I_I:%.*]], label [[BB2_I_I:%.*]]
+; CHECK:       bb1.i.i:
+; CHECK-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT:%.*]]
+; CHECK:       bb2.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_14:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_14]], i32 1
+; CHECK-NEXT:    store i32* [[TMP10]], i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 42
+; CHECK-NEXT:    br i1 [[TMP12]], label [[BB4_I_I:%.*]], label [[BB5_I_I:%.*]]
+; CHECK:       bb4.i.i:
+; CHECK-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; CHECK:       bb5.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_15:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_15]], i32 1
+; CHECK-NEXT:    store i32* [[TMP13]], i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP14]], 42
+; CHECK-NEXT:    br i1 [[TMP15]], label [[BB7_I_I:%.*]], label [[BB8_I_I:%.*]]
+; CHECK:       bb7.i.i:
+; CHECK-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; CHECK:       bb8.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_16:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_16]], i32 1
+; CHECK-NEXT:    store i32* [[TMP16]], i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, i32* [[TMP16]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[TMP17]], 42
+; CHECK-NEXT:    br i1 [[TMP18]], label [[BB10_I_I:%.*]], label [[BB11_I_I:%.*]]
+; CHECK:       bb10.i.i:
+; CHECK-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; CHECK:       bb11.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_17:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_17]], i32 1
+; CHECK-NEXT:    store i32* [[TMP19]], i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = add i32 [[__TRIP_COUNT_0_I_I:%.*]], -1
+; CHECK-NEXT:    br label [[BB12_I_I]]
+; CHECK:       bb12.i.i:
+; CHECK-NEXT:    [[__TRIP_COUNT_0_I_I]] = phi i32 [ [[TMP7]], [[ENTRY:%.*]] ], [ [[TMP20]], [[BB11_I_I]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[__TRIP_COUNT_0_I_I]], 0
+; CHECK-NEXT:    br i1 [[TMP21]], label [[BB_I_I:%.*]], label [[BB13_I_I:%.*]]
+; CHECK:       bb13.i.i:
+; CHECK-NEXT:    [[__LAST_ADDR_I_I_SROA_0_0___LAST_ADDR_I_I_SROA_0_0___LAST_ADDR_I_I_SROA_0_0___LAST_ADDR_I_I_SROA_0_0_:%.*]] = load i32*, i32** [[__LAST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP22:%.*]] = ptrtoint i32* [[__LAST_ADDR_I_I_SROA_0_0___LAST_ADDR_I_I_SROA_0_0___LAST_ADDR_I_I_SROA_0_0___LAST_ADDR_I_I_SROA_0_0_]] to i32
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = ptrtoint i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_]] to i32
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = ashr i32 [[TMP24]], 2
+; CHECK-NEXT:    switch i32 [[TMP25]], label [[BB26_I_I:%.*]] [
+; CHECK-NEXT:    i32 1, label [[BB22_I_I:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB18_I_I:%.*]]
+; CHECK-NEXT:    i32 3, label [[BB14_I_I:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb14.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_7:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_7]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i32 [[TMP26]], 42
+; CHECK-NEXT:    br i1 [[TMP27]], label [[BB16_I_I:%.*]], label [[BB17_I_I:%.*]]
+; CHECK:       bb16.i.i:
+; CHECK-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; CHECK:       bb17.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_8:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_8]], i32 1
+; CHECK-NEXT:    store i32* [[TMP28]], i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    br label [[BB18_I_I]]
+; CHECK:       bb18.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_9:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP29:%.*]] = load i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_9]], align 4
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i32 [[TMP29]], 42
+; CHECK-NEXT:    br i1 [[TMP30]], label [[BB20_I_I:%.*]], label [[BB21_I_I:%.*]]
+; CHECK:       bb20.i.i:
+; CHECK-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; CHECK:       bb21.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_10:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_10]], i32 1
+; CHECK-NEXT:    store i32* [[TMP31]], i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    br label [[BB22_I_I]]
+; CHECK:       bb22.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_11:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP32:%.*]] = load i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_11]], align 4
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp eq i32 [[TMP32]], 42
+; CHECK-NEXT:    br i1 [[TMP33]], label [[BB24_I_I:%.*]], label [[BB25_I_I:%.*]]
+; CHECK:       bb24.i.i:
+; CHECK-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; CHECK:       bb25.i.i:
+; CHECK-NEXT:    [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_12:%.*]] = load i32*, i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i32, i32* [[__FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0___FIRST_ADDR_I_I_SROA_0_0_12]], i32 1
+; CHECK-NEXT:    store i32* [[TMP34]], i32** [[__FIRST_ADDR_I_I_SROA_0]], align 8
+; CHECK-NEXT:    br label [[BB26_I_I]]
+; CHECK:       bb26.i.i:
+; CHECK-NEXT:    br label [[_ZST4FINDIN9__GNU_CXX17__NORMAL_ITERATORIPIST6VECTORIISAIIEEEEIET_S7_S7_RKT0__EXIT]]
+; CHECK:       _ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit:
+; CHECK-NEXT:    [[DOTIN_IN:%.*]] = phi i32** [ [[__LAST_ADDR_I_I_SROA_0]], [[BB26_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0]], [[BB24_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0]], [[BB20_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0]], [[BB16_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0]], [[BB10_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0]], [[BB7_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0]], [[BB4_I_I]] ], [ [[__FIRST_ADDR_I_I_SROA_0]], [[BB1_I_I]] ]
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       return:
+; CHECK-NEXT:    [[TMP35:%.*]] = load i32*, i32** [[DOTIN_IN]], align 4
+; CHECK-NEXT:    ret i32* [[TMP35]]
+;
+entry:
+  %__first_addr.i.i.sroa.0 = alloca i32*, align 8
+  %__last_addr.i.i.sroa.0 = alloca i32*, align 8
+  %0 = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* %X, i32 0, i32 0, i32 0, i32 1
+  %1 = load i32*, i32** %0, align 4
+  %2 = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* %X, i32 0, i32 0, i32 0, i32 0
+  %3 = load i32*, i32** %2, align 4
+  store i32* %3, i32** %__first_addr.i.i.sroa.0, align 8
+  store i32* %1, i32** %__last_addr.i.i.sroa.0, align 8
+  %4 = ptrtoint i32* %1 to i32
+  %5 = ptrtoint i32* %3 to i32
+  %6 = sub i32 %4, %5
+  %7 = ashr i32 %6, 4
+  br label %bb12.i.i
+
+bb.i.i:                                           ; preds = %bb12.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.13 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %8 = load i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.13, align 4
+  %9 = icmp eq i32 %8, 42
+  br i1 %9, label %bb1.i.i, label %bb2.i.i
+
+bb1.i.i:                                          ; preds = %bb.i.i
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb2.i.i:                                          ; preds = %bb.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.14 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %10 = getelementptr i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.14, i32 1
+  store i32* %10, i32** %__first_addr.i.i.sroa.0, align 8
+  %11 = load i32, i32* %10, align 4
+  %12 = icmp eq i32 %11, 42
+  br i1 %12, label %bb4.i.i, label %bb5.i.i
+
+bb4.i.i:                                          ; preds = %bb2.i.i
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb5.i.i:                                          ; preds = %bb2.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.15 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %13 = getelementptr i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.15, i32 1
+  store i32* %13, i32** %__first_addr.i.i.sroa.0, align 8
+  %14 = load i32, i32* %13, align 4
+  %15 = icmp eq i32 %14, 42
+  br i1 %15, label %bb7.i.i, label %bb8.i.i
+
+bb7.i.i:                                          ; preds = %bb5.i.i
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb8.i.i:                                          ; preds = %bb5.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.16 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %16 = getelementptr i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.16, i32 1
+  store i32* %16, i32** %__first_addr.i.i.sroa.0, align 8
+  %17 = load i32, i32* %16, align 4
+  %18 = icmp eq i32 %17, 42
+  br i1 %18, label %bb10.i.i, label %bb11.i.i
+
+bb10.i.i:                                         ; preds = %bb8.i.i
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb11.i.i:                                         ; preds = %bb8.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.17 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %19 = getelementptr i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.17, i32 1
+  store i32* %19, i32** %__first_addr.i.i.sroa.0, align 8
+  %20 = add i32 %__trip_count.0.i.i, -1
+  br label %bb12.i.i
+
+bb12.i.i:                                         ; preds = %bb11.i.i, %entry
+  %__trip_count.0.i.i = phi i32 [ %7, %entry ], [ %20, %bb11.i.i ]
+  %21 = icmp sgt i32 %__trip_count.0.i.i, 0
+  br i1 %21, label %bb.i.i, label %bb13.i.i
+
+bb13.i.i:                                         ; preds = %bb12.i.i
+  %__last_addr.i.i.sroa.0.0.__last_addr.i.i.sroa.0.0.__last_addr.i.i.sroa.0.0. = load i32*, i32** %__last_addr.i.i.sroa.0, align 8
+  %22 = ptrtoint i32* %__last_addr.i.i.sroa.0.0.__last_addr.i.i.sroa.0.0.__last_addr.i.i.sroa.0.0. to i32
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0. = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %23 = ptrtoint i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0. to i32
+  %24 = sub i32 %22, %23
+  %25 = ashr i32 %24, 2
+  switch i32 %25, label %bb26.i.i [
+  i32 1, label %bb22.i.i
+  i32 2, label %bb18.i.i
+  i32 3, label %bb14.i.i
+  ]
+
+bb14.i.i:                                         ; preds = %bb13.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.7 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %26 = load i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.7, align 4
+  %27 = icmp eq i32 %26, 42
+  br i1 %27, label %bb16.i.i, label %bb17.i.i
+
+bb16.i.i:                                         ; preds = %bb14.i.i
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb17.i.i:                                         ; preds = %bb14.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.8 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %28 = getelementptr i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.8, i32 1
+  store i32* %28, i32** %__first_addr.i.i.sroa.0, align 8
+  br label %bb18.i.i
+
+bb18.i.i:                                         ; preds = %bb17.i.i, %bb13.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.9 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %29 = load i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.9, align 4
+  %30 = icmp eq i32 %29, 42
+  br i1 %30, label %bb20.i.i, label %bb21.i.i
+
+bb20.i.i:                                         ; preds = %bb18.i.i
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb21.i.i:                                         ; preds = %bb18.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.10 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %31 = getelementptr i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.10, i32 1
+  store i32* %31, i32** %__first_addr.i.i.sroa.0, align 8
+  br label %bb22.i.i
+
+bb22.i.i:                                         ; preds = %bb21.i.i, %bb13.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.11 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %32 = load i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.11, align 4
+  %33 = icmp eq i32 %32, 42
+  br i1 %33, label %bb24.i.i, label %bb25.i.i
+
+bb24.i.i:                                         ; preds = %bb22.i.i
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb25.i.i:                                         ; preds = %bb22.i.i
+  %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.12 = load i32*, i32** %__first_addr.i.i.sroa.0, align 8
+  %34 = getelementptr i32, i32* %__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.__first_addr.i.i.sroa.0.0.12, i32 1
+  store i32* %34, i32** %__first_addr.i.i.sroa.0, align 8
+  br label %bb26.i.i
+
+bb26.i.i:                                         ; preds = %bb25.i.i, %bb13.i.i
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit: ; preds = %bb26.i.i, %bb24.i.i, %bb20.i.i, %bb16.i.i, %bb10.i.i, %bb7.i.i, %bb4.i.i, %bb1.i.i
+  %.in.in = phi i32** [ %__last_addr.i.i.sroa.0, %bb26.i.i ], [ %__first_addr.i.i.sroa.0, %bb24.i.i ], [ %__first_addr.i.i.sroa.0, %bb20.i.i ], [ %__first_addr.i.i.sroa.0, %bb16.i.i ], [ %__first_addr.i.i.sroa.0, %bb10.i.i ], [ %__first_addr.i.i.sroa.0, %bb7.i.i ], [ %__first_addr.i.i.sroa.0, %bb4.i.i ], [ %__first_addr.i.i.sroa.0, %bb1.i.i ]
+  br label %return
+
+return:                                           ; preds = %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+  %35 = load i32*, i32** %.in.in, align 4
+  ret i32* %35
+}
+
+; Function Attrs: argmemonly nofree nosync nounwind willreturn
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) #0
+
+attributes #0 = { argmemonly nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/SROA/phi-with-duplicate-pred.ll b/llvm/test/Transforms/SROA/phi-with-duplicate-pred.ll
index 64963fd663467e191ce36fa3c8b216f3cb24e687..87226f7fe5743fe88ce7498851b9e45bd0a3e4d1 100644
--- a/llvm/test/Transforms/SROA/phi-with-duplicate-pred.ll
+++ b/llvm/test/Transforms/SROA/phi-with-duplicate-pred.ll
@@ -4,6 +4,8 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 
 @a = external global i16, align 1
 
+declare void @maybe_writes()
+
 define void @f2() {
 ; CHECK-LABEL: @f2(
 ; CHECK-NEXT:  entry:
@@ -49,3 +51,215 @@ cleanup7:                                         ; preds = %cleanup
   ret void
 }
 
+define void @f3() {
+; CHECK-LABEL: @f3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[E:%.*]] = alloca i16, align 1
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[CLEANUP:%.*]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    switch i32 2, label [[CLEANUP7:%.*]] [
+; CHECK-NEXT:    i32 0, label [[LBL1:%.*]]
+; CHECK-NEXT:    i32 2, label [[LBL1]]
+; CHECK-NEXT:    ]
+; CHECK:       if.else:
+; CHECK-NEXT:    br label [[LBL1]]
+; CHECK:       lbl1:
+; CHECK-NEXT:    [[G_0:%.*]] = phi i16* [ @a, [[CLEANUP]] ], [ @a, [[CLEANUP]] ], [ [[E]], [[IF_ELSE]] ]
+; CHECK-NEXT:    br label [[FINAL:%.*]]
+; CHECK:       final:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[G_0]], align 1
+; CHECK-NEXT:    unreachable
+; CHECK:       cleanup7:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %e = alloca i16, align 1
+  br i1 undef, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.then
+  switch i32 2, label %cleanup7 [
+  i32 0, label %lbl1
+  i32 2, label %lbl1
+  ]
+
+if.else:                                          ; preds = %entry
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.else, %cleanup, %cleanup
+  %g.0 = phi i16* [ @a, %cleanup ], [ @a, %cleanup ], [ %e, %if.else ]
+  br label %final
+
+final:
+  %0 = load i16, i16* %g.0, align 1
+  unreachable
+
+cleanup7:                                         ; preds = %cleanup
+  ret void
+}
+
+define void @f4() {
+; CHECK-LABEL: @f4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[E:%.*]] = alloca i16, align 1
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[CLEANUP:%.*]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    switch i32 2, label [[CLEANUP7:%.*]] [
+; CHECK-NEXT:    i32 0, label [[LBL1:%.*]]
+; CHECK-NEXT:    i32 2, label [[LBL1]]
+; CHECK-NEXT:    ]
+; CHECK:       if.else:
+; CHECK-NEXT:    br label [[LBL1]]
+; CHECK:       lbl1:
+; CHECK-NEXT:    [[G_0:%.*]] = phi i16* [ @a, [[CLEANUP]] ], [ @a, [[CLEANUP]] ], [ [[E]], [[IF_ELSE]] ]
+; CHECK-NEXT:    br label [[FINAL:%.*]]
+; CHECK:       final:
+; CHECK-NEXT:    call void @maybe_writes()
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[G_0]], align 1
+; CHECK-NEXT:    unreachable
+; CHECK:       cleanup7:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %e = alloca i16, align 1
+  br i1 undef, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.then
+  switch i32 2, label %cleanup7 [
+  i32 0, label %lbl1
+  i32 2, label %lbl1
+  ]
+
+if.else:                                          ; preds = %entry
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.else, %cleanup, %cleanup
+  %g.0 = phi i16* [ @a, %cleanup ], [ @a, %cleanup ], [ %e, %if.else ]
+  br label %final
+
+final:
+  call void @maybe_writes()
+  %0 = load i16, i16* %g.0, align 1
+  unreachable
+
+cleanup7:                                         ; preds = %cleanup
+  ret void
+}
+
+define void @f5() {
+; CHECK-LABEL: @f5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[E:%.*]] = alloca i16, align 1
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[CLEANUP:%.*]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    switch i32 2, label [[CLEANUP7:%.*]] [
+; CHECK-NEXT:    i32 0, label [[LBL1:%.*]]
+; CHECK-NEXT:    i32 2, label [[LBL1]]
+; CHECK-NEXT:    ]
+; CHECK:       if.else:
+; CHECK-NEXT:    br label [[LBL1]]
+; CHECK:       lbl1:
+; CHECK-NEXT:    [[G_0:%.*]] = phi i16* [ @a, [[CLEANUP]] ], [ @a, [[CLEANUP]] ], [ [[E]], [[IF_ELSE]] ]
+; CHECK-NEXT:    br i1 undef, label [[FINAL:%.*]], label [[CLEANUP7]]
+; CHECK:       final:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[G_0]], align 1
+; CHECK-NEXT:    unreachable
+; CHECK:       cleanup7:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %e = alloca i16, align 1
+  br i1 undef, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.then
+  switch i32 2, label %cleanup7 [
+  i32 0, label %lbl1
+  i32 2, label %lbl1
+  ]
+
+if.else:                                          ; preds = %entry
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.else, %cleanup, %cleanup
+  %g.0 = phi i16* [ @a, %cleanup ], [ @a, %cleanup ], [ %e, %if.else ]
+  br i1 undef, label %final, label %cleanup7
+
+final:
+  %0 = load i16, i16* %g.0, align 1
+  unreachable
+
+cleanup7:                                         ; preds = %cleanup
+  ret void
+}
+
+define void @f6() {
+; CHECK-LABEL: @f6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[E:%.*]] = alloca i16, align 1
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[CLEANUP:%.*]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    switch i32 2, label [[CLEANUP7:%.*]] [
+; CHECK-NEXT:    i32 0, label [[LBL1:%.*]]
+; CHECK-NEXT:    i32 2, label [[LBL1]]
+; CHECK-NEXT:    ]
+; CHECK:       if.else:
+; CHECK-NEXT:    br label [[LBL1]]
+; CHECK:       lbl1:
+; CHECK-NEXT:    [[G_0:%.*]] = phi i16* [ @a, [[CLEANUP]] ], [ @a, [[CLEANUP]] ], [ [[E]], [[IF_ELSE]] ]
+; CHECK-NEXT:    br label [[FINAL:%.*]]
+; CHECK:       unreachable_pred:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    call void @maybe_writes()
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[G_0]], align 1
+; CHECK-NEXT:    unreachable
+; CHECK:       cleanup7:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %e = alloca i16, align 1
+  br i1 undef, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.then
+  switch i32 2, label %cleanup7 [
+  i32 0, label %lbl1
+  i32 2, label %lbl1
+  ]
+
+if.else:                                          ; preds = %entry
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.else, %cleanup, %cleanup
+  %g.0 = phi i16* [ @a, %cleanup ], [ @a, %cleanup ], [ %e, %if.else ]
+  br label %final
+
+unreachable_pred:
+  br label %final
+
+final:
+  call void @maybe_writes()
+  %0 = load i16, i16* %g.0, align 1
+  unreachable
+
+cleanup7:                                         ; preds = %cleanup
+  ret void
+}
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/norepeated-icp-3.prof b/llvm/test/Transforms/SampleProfile/Inputs/norepeated-icp-3.prof
new file mode 100644
index 0000000000000000000000000000000000000000..a65c792bf070157625a4483288b1490955e11f39
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/norepeated-icp-3.prof
@@ -0,0 +1,6 @@
+_Z3foov:225715:1
+ 2: 5553
+ 3: 5391
+ 1: _Z3goov:5860
+  1: 5279 _Z3hoov:5860 _Z3moov:210
+  2: 5279
diff --git a/llvm/test/Transforms/SampleProfile/norepeated-icp-3.ll b/llvm/test/Transforms/SampleProfile/norepeated-icp-3.ll
new file mode 100644
index 0000000000000000000000000000000000000000..140a15f587478ee1b55ebb7f5da000278177bd85
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/norepeated-icp-3.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -passes=sample-profile -sample-profile-icp-max-prom=4 -sample-profile-file=%S/Inputs/norepeated-icp-3.prof -S | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [5 x i8] c"hoo\0A\00", align 1
+@p = dso_local global void ()* null, align 8
+@str = private unnamed_addr constant [4 x i8] c"hoo\00", align 1
+
+; Function Attrs: nofree nounwind
+declare dso_local noundef i32 @printf(i8* nocapture noundef readonly, ...) #1
+
+; Function Attrs: uwtable mustprogress
+define dso_local void @_Z3goov() #0 !dbg !11 {
+entry:
+  %0 = load void ()*, void ()** @p, align 8, !dbg !12, !tbaa !13
+  call void %0(), !dbg !17, !prof !22
+  ret void, !dbg !18
+}
+
+; After the indirect call in _Z3goov is inlined into _Z3foov, it will be
+; annotated with new inline instance profile. The existing value profile
+; associated with the indirect call should be dropped except those values
+; wth NOMORE_ICP_MAGICNUM magic number indicating promoted targets.
+; CHECK-LABEL: @_Z3foov(
+; CHECK: call void %0(), {{.*}} !prof ![[PROF_ID:[0-9]+]]
+; CHECK-NEXT: ret void
+
+; Function Attrs: uwtable mustprogress
+define dso_local void @_Z3foov() #0 !dbg !19 {
+entry:
+  call void @_Z3goov(), !dbg !20
+  ret void, !dbg !21
+}
+
+; Function Attrs: nofree nounwind
+declare noundef i32 @puts(i8* nocapture noundef readonly) #2
+
+attributes #0 = { uwtable mustprogress "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-sample-profile" "use-soft-float"="false" }
+attributes #1 = { nofree nounwind "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nofree nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, splitDebugInlining: false, debugInfoForProfiling: true, nameTableKind: None)
+!1 = !DIFile(filename: "1.cc", directory: "")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!""}
+!8 = !DISubroutineType(types: !2)
+!11 = distinct !DISubprogram(name: "goo", linkageName: "_Z3goov", scope: !1, file: !1, line: 6, type: !8, scopeLine: 6, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!12 = !DILocation(line: 7, column: 5, scope: !11)
+!13 = !{!14, !14, i64 0}
+!14 = !{!"any pointer", !15, i64 0}
+!15 = !{!"omnipotent char", !16, i64 0}
+!16 = !{!"Simple C++ TBAA"}
+!17 = !DILocation(line: 7, column: 3, scope: !11)
+!18 = !DILocation(line: 8, column: 1, scope: !11)
+!19 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 10, type: !8, scopeLine: 10, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!20 = !DILocation(line: 11, column: 3, scope: !19)
+!21 = !DILocation(line: 12, column: 3, scope: !19)
+; The original value 125292384912345234234 and its count 8000 should
+; be dropped when the indirect call is annotated with new profile.
+; The original value -7383239051784516332 and its count -1 should be kept
+; because -1 is NOMORE_ICP_MAGICNUM.
+; CHECK: ![[PROF_ID]] = !{!"VP", i32 0, i64 5860, i64 -7383239051784516332, i64 -1, i64 -7701940972712279918, i64 5860}
+!22 = !{!"VP", i32 0, i64 8000, i64 -7383239051784516332, i64 -1, i64 125292384912345234234, i64 8000}
diff --git a/llvm/test/Transforms/SimplifyCFG/ARM/branch-fold-threshold.ll b/llvm/test/Transforms/SimplifyCFG/ARM/branch-fold-threshold.ll
index a3d4129e0af83c27ebe684a75ac2caf891cef5df..f00b427193a69321803600c3fa198551687a4564 100644
--- a/llvm/test/Transforms/SimplifyCFG/ARM/branch-fold-threshold.ll
+++ b/llvm/test/Transforms/SimplifyCFG/ARM/branch-fold-threshold.ll
@@ -9,7 +9,7 @@ define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[D:%.*]], 3
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP]], [[CMP1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP]], i1 [[CMP1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[COND_FALSE:%.*]], label [[COND_END:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -42,7 +42,7 @@ define i32 @foo_minsize(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input) #0 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[D:%.*]], 3
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP]], [[CMP1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP]], i1 [[CMP1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[COND_FALSE:%.*]], label [[COND_END:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -75,7 +75,7 @@ define i32 @foo_minsize_i64(i64 %a, i64 %b, i64 %c, i64 %d, i32* %input) #0 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[D:%.*]], 3
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[C:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i64 [[ADD]], [[B:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP]], [[CMP1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP]], i1 [[CMP1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[COND_FALSE:%.*]], label [[COND_END:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -108,7 +108,7 @@ define i32 @or_predicate(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[D:%.*]], 3
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP]], [[CMP1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP]], i1 true, i1 [[CMP1]]
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -141,7 +141,7 @@ define i32 @or_invert_predicate(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[D:%.*]], 3
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP]], [[CMP1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP]], i1 true, i1 [[CMP1]]
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -189,7 +189,7 @@ define i32 @or_predicate_minsize(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input) #0
 ; ARM-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[D:%.*]], 3
 ; ARM-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; ARM-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; ARM-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP]], [[CMP1]]
+; ARM-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP]], i1 true, i1 [[CMP1]]
 ; ARM-NEXT:    br i1 [[OR_COND]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; ARM:       cond.false:
 ; ARM-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -237,7 +237,7 @@ define i32 @or_invert_predicate_minsize(i32 %a, i32 %b, i32 %c, i32 %d, i32* %in
 ; ARM-NEXT:    [[CMP:%.*]] = icmp sle i32 [[D:%.*]], 3
 ; ARM-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; ARM-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; ARM-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP]], [[CMP1]]
+; ARM-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP]], i1 true, i1 [[CMP1]]
 ; ARM-NEXT:    br i1 [[OR_COND]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; ARM:       cond.false:
 ; ARM-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -270,7 +270,7 @@ define i32 @or_xor_predicate(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input, i1 %cm
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = xor i1 [[CMP:%.*]], true
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP_NOT]], [[CMP1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP_NOT]], i1 true, i1 [[CMP1]]
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -316,7 +316,7 @@ define i32 @or_xor_predicate_minsize(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input
 ; ARM-NEXT:    [[CMP_NOT:%.*]] = xor i1 [[CMP:%.*]], true
 ; ARM-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; ARM-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; ARM-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP_NOT]], [[CMP1]]
+; ARM-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP_NOT]], i1 true, i1 [[CMP1]]
 ; ARM-NEXT:    br i1 [[OR_COND]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
 ; ARM:       cond.false:
 ; ARM-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -348,7 +348,7 @@ define i32 @and_xor(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input, i1 %cmp) {
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = xor i1 [[CMP:%.*]], true
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP_NOT]], [[CMP1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP_NOT]], i1 [[CMP1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[COND_FALSE:%.*]], label [[COND_END:%.*]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
@@ -394,7 +394,7 @@ define i32 @and_xor_minsize(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input, i1 %cmp
 ; ARM-NEXT:    [[CMP_NOT:%.*]] = xor i1 [[CMP:%.*]], true
 ; ARM-NEXT:    [[ADD:%.*]] = add nsw i32 [[C:%.*]], [[A:%.*]]
 ; ARM-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B:%.*]]
-; ARM-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP_NOT]], [[CMP1]]
+; ARM-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP_NOT]], i1 [[CMP1]], i1 false
 ; ARM-NEXT:    br i1 [[OR_COND]], label [[COND_FALSE:%.*]], label [[COND_END:%.*]]
 ; ARM:       cond.false:
 ; ARM-NEXT:    [[TMP0:%.*]] = load i32, i32* [[INPUT:%.*]], align 4
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll b/llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll
index 78b4f45c39b52089186ea0a435033a122fb54cae..bcac5f98316893d2975c9fe3cf7a254fe47d132e 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll
@@ -72,10 +72,10 @@ define i8* @test4(i1* %dummy, i8* %a, i8* %b) {
 ; expressions.
 ; CHECK-LABEL: @test4(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[COND1:%.*]] = load volatile i1, i1* [[DUMMY:%.*]]
+; CHECK-NEXT:    [[COND1:%.*]] = load volatile i1, i1* [[DUMMY:%.*]], align 1
 ; CHECK-NEXT:    br i1 [[COND1]], label [[IF:%.*]], label [[END:%.*]]
 ; CHECK:       if:
-; CHECK-NEXT:    [[COND2:%.*]] = load volatile i1, i1* [[DUMMY]]
+; CHECK-NEXT:    [[COND2:%.*]] = load volatile i1, i1* [[DUMMY]], align 1
 ; CHECK-NEXT:    br i1 [[COND2]], label [[THEN:%.*]], label [[END]]
 ; CHECK:       then:
 ; CHECK-NEXT:    br label [[END]]
@@ -126,7 +126,7 @@ define i32* @test5(i32 %a, i32 %b, i32 %c, i32* dereferenceable(10) %ptr1, i32*
 ; CHECK-NEXT:    br i1 [[T1]], label [[BB1:%.*]], label [[BB3:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[T2:%.*]] = icmp sgt i32 [[C:%.*]], 1
-; CHECK-NEXT:    [[T3:%.*]] = load i32*, i32** [[PTR3:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = load i32*, i32** [[PTR3:%.*]], align 8
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[T2]], i32* [[T3]], i32* [[PTR2:%.*]]
 ; CHECK-NEXT:    ret i32* [[SPEC_SELECT]]
 ; CHECK:       bb3:
diff --git a/llvm/test/Transforms/SimplifyCFG/annotations.ll b/llvm/test/Transforms/SimplifyCFG/annotations.ll
index c10a2a8bd1f35c22ed233bc75d3652182bcc714e..b70b9046bbbc1bbcf44618b0684456d9e637824a 100644
--- a/llvm/test/Transforms/SimplifyCFG/annotations.ll
+++ b/llvm/test/Transforms/SimplifyCFG/annotations.ll
@@ -7,7 +7,7 @@ define i32 @test_preserve_and(i8* %a, i8* %b, i8* %c, i8* %d) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp ult i8* [[A:%.*]], [[B:%.*]], !annotation !0
 ; CHECK-NEXT:    [[C_2:%.*]] = icmp uge i8* [[C:%.*]], [[D:%.*]], !annotation !0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C_1]], [[C_2]], !annotation !0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C_1]], i1 [[C_2]], i1 false, !annotation !0
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[CONT1:%.*]], label [[TRAP:%.*]], !annotation !0
 ; CHECK:       trap: ; preds = %entry
 ; CHECK-NEXT:    call void @fn1()
@@ -40,7 +40,7 @@ define i32 @test_preserve_or(i8* %a, i8* %b, i8* %c, i8* %d) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp uge i8* [[A:%.*]], [[B:%.*]], !annotation !0
 ; CHECK-NEXT:    [[C_2:%.*]] = icmp uge i8* [[C:%.*]], [[D:%.*]], !annotation !0
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[C_1]], [[C_2]], !annotation !0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C_1]], i1 true, i1 [[C_2]], !annotation !0
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[TRAP:%.*]], label [[CONT1:%.*]], !annotation !0
 ; CHECK:       trap: ; preds = %entry
 ; CHECK-NEXT:    call void @fn1()
@@ -75,7 +75,7 @@ define i32 @test_preserve_or_not(i8* %a, i8* %b, i8* %c, i8* %d) {
 ; CHECK-NEXT:    [[C_2:%.*]] = xor i1 [[C_1]], true
 ; CHECK-NEXT:    [[C_2_NOT:%.*]] = xor i1 [[C_2]], true, !annotation !0
 ; CHECK-NEXT:    [[C_3:%.*]] = icmp uge i8* [[C:%.*]], [[D:%.*]], !annotation !0
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[C_2_NOT]], [[C_3]], !annotation !0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C_2_NOT]], i1 true, i1 [[C_3]], !annotation !0
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[TRAP:%.*]], label [[CONT1:%.*]], !annotation !0
 ; CHECK:       trap: ; preds = %entry
 ; CHECK-NEXT:    call void @fn1()
@@ -112,7 +112,7 @@ define i32 @test_or_not_no_annotation(i8* %a, i8* %b, i8* %c, i8* %d) {
 ; CHECK-NEXT:    [[C_2:%.*]] = xor i1 [[C_1]], true
 ; CHECK-NEXT:    [[C_2_NOT:%.*]] = xor i1 [[C_2]], true
 ; CHECK-NEXT:    [[C_3:%.*]] = icmp uge i8* [[C:%.*]], [[D:%.*]], !annotation !0
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[C_2_NOT]], [[C_3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C_2_NOT]], i1 true, i1 [[C_3]]
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[TRAP:%.*]], label [[CONT1:%.*]], !annotation !0
 ; CHECK:       trap: ; preds = %entry
 ; CHECK-NEXT:    call void @fn1()
diff --git a/llvm/test/Transforms/SimplifyCFG/common-dest-folding.ll b/llvm/test/Transforms/SimplifyCFG/common-dest-folding.ll
index 0035a445cca1bd5e82ba6afb459de6e7d6106fb2..4f18f914cf998c9a89dd68c40b68ce5dce3b5263 100644
--- a/llvm/test/Transforms/SimplifyCFG/common-dest-folding.ll
+++ b/llvm/test/Transforms/SimplifyCFG/common-dest-folding.ll
@@ -5,7 +5,7 @@
 ;CHECK: icmp eq i32
 ;CHECK: and i32 %c2, %k
 ;CHECK: icmp eq i32
-;CHECK: or i1
+;CHECK: select i1 {{.*}}, i1 true,
 ;CHECK: ret
 define i32 @foo(i32 %k, i32 %c1, i32 %c2) {
   %1 = and i32 %c1, %k
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest-two-preds-cost.ll b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest-two-preds-cost.ll
new file mode 100644
index 0000000000000000000000000000000000000000..450ab5fcc7992a2d878875baa3f1bfd641a29539
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest-two-preds-cost.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -simplifycfg -simplifycfg-require-and-preserve-domtree=1 -bonus-inst-threshold=1 | FileCheck --check-prefixes=ALL,THR1 %s
+; RUN: opt < %s -S -simplifycfg -simplifycfg-require-and-preserve-domtree=1 -bonus-inst-threshold=2 | FileCheck --check-prefixes=ALL,THR2 %s
+
+declare void @sideeffect0()
+declare void @sideeffect1()
+declare void @sideeffect2()
+declare void @use8(i8)
+declare i1 @gen1()
+
+; Here we'd want to duplicate %v3_adj into two predecessors,
+; but -bonus-inst-threshold=1 says that we can only clone it into one.
+; With -bonus-inst-threshold=2 we can clone it into both though.
+define void @two_preds_with_extra_op(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
+; THR1-LABEL: @two_preds_with_extra_op(
+; THR1-NEXT:  entry:
+; THR1-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
+; THR1-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
+; THR1:       pred0:
+; THR1-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
+; THR1-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]]
+; THR1:       pred1:
+; THR1-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
+; THR1-NEXT:    br i1 [[C2]], label [[DISPATCH]], label [[FINAL_RIGHT:%.*]]
+; THR1:       dispatch:
+; THR1-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
+; THR1-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3_ADJ]], 0
+; THR1-NEXT:    br i1 [[C3]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; THR1:       final_left:
+; THR1-NEXT:    call void @sideeffect0()
+; THR1-NEXT:    ret void
+; THR1:       final_right:
+; THR1-NEXT:    call void @sideeffect1()
+; THR1-NEXT:    ret void
+;
+; THR2-LABEL: @two_preds_with_extra_op(
+; THR2-NEXT:  entry:
+; THR2-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
+; THR2-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
+; THR2:       pred0:
+; THR2-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
+; THR2-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2:%.*]]
+; THR2-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3_ADJ_OLD]], 0
+; THR2-NEXT:    [[OR_COND1:%.*]] = select i1 [[C1]], i1 true, i1 [[C3_OLD]]
+; THR2-NEXT:    br i1 [[OR_COND1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
+; THR2:       pred1:
+; THR2-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2]], 0
+; THR2-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
+; THR2-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3_ADJ]], 0
+; THR2-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
+; THR2-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; THR2:       final_left:
+; THR2-NEXT:    call void @sideeffect0()
+; THR2-NEXT:    ret void
+; THR2:       final_right:
+; THR2-NEXT:    call void @sideeffect1()
+; THR2-NEXT:    ret void
+;
+entry:
+  %c0 = icmp eq i8 %v0, 0
+  br i1 %c0, label %pred0, label %pred1
+pred0:
+  %c1 = icmp eq i8 %v1, 0
+  br i1 %c1, label %final_left, label %dispatch
+pred1:
+  %c2 = icmp eq i8 %v2, 0
+  br i1 %c2, label %dispatch, label %final_right
+dispatch:
+  %v3_adj = add i8 %v1, %v2
+  %c3 = icmp eq i8 %v3_adj, 0
+  br i1 %c3, label %final_left, label %final_right
+final_left:
+  call void @sideeffect0()
+  ret void
+final_right:
+  call void @sideeffect1()
+  ret void
+}
+
+; Here we'd want to duplicate %v3_adj into two predecessors,
+; but -bonus-inst-threshold=1 says that we can only clone it into one.
+; But, we aren't going to clone it into one of the predecessors,
+; because that isn't profitable. So we should not use it in cost calculation.
+define void @two_preds_with_extra_op_and_branchweights(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
+; ALL-LABEL: @two_preds_with_extra_op_and_branchweights(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
+; ALL-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
+; ALL:       pred0:
+; ALL-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
+; ALL-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]], !prof !0
+; ALL:       pred1:
+; ALL-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
+; ALL-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
+; ALL-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3_ADJ]], 0
+; ALL-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
+; ALL-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
+; ALL:       dispatch:
+; ALL-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2]]
+; ALL-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3_ADJ_OLD]], 0
+; ALL-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; ALL:       final_left:
+; ALL-NEXT:    call void @sideeffect0()
+; ALL-NEXT:    ret void
+; ALL:       final_right:
+; ALL-NEXT:    call void @sideeffect1()
+; ALL-NEXT:    ret void
+;
+entry:
+  %c0 = icmp eq i8 %v0, 0
+  br i1 %c0, label %pred0, label %pred1
+pred0:
+  %c1 = icmp eq i8 %v1, 0
+  br i1 %c1, label %final_left, label %dispatch, !prof !0 ; likely branches to %final_left
+pred1:
+  %c2 = icmp eq i8 %v2, 0
+  br i1 %c2, label %dispatch, label %final_right
+dispatch:
+  %v3_adj = add i8 %v1, %v2
+  %c3 = icmp eq i8 %v3_adj, 0
+  br i1 %c3, label %final_left, label %final_right
+final_left:
+  call void @sideeffect0()
+  ret void
+final_right:
+  call void @sideeffect1()
+  ret void
+}
+
+!0 = !{!"branch_weights", i32 99, i32 1}
+
+; CHECK: !0 = !{!"branch_weights", i32 99, i32 1}
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
index 002a0e57d99fa7ef0d389f2ad01a85359cc8eb7b..b578bb1bd45eebfc8b49be9ab671256c86a9a647 100644
--- a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
+++ b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
@@ -14,7 +14,7 @@ define void @one_pred(i8 %v0, i8 %v1) {
 ; CHECK-NEXT:  pred:
 ; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C0]], [[C1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -45,12 +45,12 @@ define void @two_preds(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
 ; CHECK:       pred0:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
 ; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3:%.*]], 0
-; CHECK-NEXT:    [[OR_COND1:%.*]] = or i1 [[C1]], [[C3_OLD]]
+; CHECK-NEXT:    [[OR_COND1:%.*]] = select i1 [[C1]], i1 true, i1 [[C3_OLD]]
 ; CHECK-NEXT:    br i1 [[OR_COND1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       pred1:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C2]], [[C3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -87,7 +87,7 @@ define void @one_pred_with_extra_op(i8 %v0, i8 %v1) {
 ; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; CHECK-NEXT:    [[V1_ADJ:%.*]] = add i8 [[V0]], [[V1:%.*]]
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1_ADJ]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C0]], [[C1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -120,13 +120,13 @@ define void @two_preds_with_extra_op(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
 ; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2:%.*]]
 ; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3_ADJ_OLD]], 0
-; CHECK-NEXT:    [[OR_COND1:%.*]] = or i1 [[C1]], [[C3_OLD]]
+; CHECK-NEXT:    [[OR_COND1:%.*]] = select i1 [[C1]], i1 true, i1 [[C3_OLD]]
 ; CHECK-NEXT:    br i1 [[OR_COND1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       pred1:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3_ADJ]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C2]], [[C3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -165,7 +165,7 @@ define void @one_pred_with_extra_op_multiuse(i8 %v0, i8 %v1) {
 ; CHECK-NEXT:    [[V1_ADJ:%.*]] = add i8 [[V0]], [[V1:%.*]]
 ; CHECK-NEXT:    [[V1_ADJ_ADJ:%.*]] = add i8 [[V1_ADJ]], [[V1_ADJ]]
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1_ADJ_ADJ]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C0]], [[C1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -200,14 +200,14 @@ define void @two_preds_with_extra_op_multiuse(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
 ; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2:%.*]]
 ; CHECK-NEXT:    [[V3_ADJ_ADJ_OLD:%.*]] = add i8 [[V3_ADJ_OLD]], [[V3_ADJ_OLD]]
 ; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3_ADJ_ADJ_OLD]], 0
-; CHECK-NEXT:    [[OR_COND1:%.*]] = or i1 [[C1]], [[C3_OLD]]
+; CHECK-NEXT:    [[OR_COND1:%.*]] = select i1 [[C1]], i1 true, i1 [[C3_OLD]]
 ; CHECK-NEXT:    br i1 [[OR_COND1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       pred1:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
 ; CHECK-NEXT:    [[V3_ADJ_ADJ:%.*]] = add i8 [[V3_ADJ]], [[V3_ADJ]]
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3_ADJ_ADJ]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C2]], [[C3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -247,7 +247,7 @@ define void @one_pred_with_extra_op_liveout(i8 %v0, i8 %v1) {
 ; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; CHECK-NEXT:    [[V1_ADJ:%.*]] = add i8 [[V0]], [[V1:%.*]]
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1_ADJ]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C0]], [[C1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -278,7 +278,7 @@ define void @one_pred_with_extra_op_liveout_multiuse(i8 %v0, i8 %v1) {
 ; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; CHECK-NEXT:    [[V1_ADJ:%.*]] = add i8 [[V0]], [[V1:%.*]]
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1_ADJ]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C0]], [[C1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -315,7 +315,7 @@ define void @one_pred_with_extra_op_liveout_distant_phi(i8 %v0, i8 %v1) {
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
 ; CHECK-NEXT:    [[V2_ADJ:%.*]] = add i8 [[V0]], [[V1]]
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2_ADJ]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C1]], [[C2]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C1]], i1 [[C2]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -366,7 +366,7 @@ define void @two_preds_with_extra_op_liveout(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3_ADJ]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C2]], [[C3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
 ; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2]]
@@ -416,7 +416,7 @@ define void @two_preds_with_extra_op_liveout_multiuse(i8 %v0, i8 %v1, i8 %v2, i8
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3_ADJ]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C2]], [[C3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
 ; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2]]
@@ -467,7 +467,7 @@ define void @one_pred_with_extra_op_eexternally_used_only(i8 %v0, i8 %v1) {
 ; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; CHECK-NEXT:    [[V1_ADJ:%.*]] = add i8 [[V0]], [[V1:%.*]]
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C0]], [[C1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -498,7 +498,7 @@ define void @one_pred_with_extra_op_externally_used_only_multiuse(i8 %v0, i8 %v1
 ; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; CHECK-NEXT:    [[V1_ADJ:%.*]] = add i8 [[V0]], [[V1:%.*]]
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C0]], [[C1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -538,7 +538,7 @@ define void @two_preds_with_extra_op_externally_used_only(i8 %v0, i8 %v1, i8 %v2
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3:%.*]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C2]], [[C3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
 ; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2]]
@@ -588,7 +588,7 @@ define void @two_preds_with_extra_op_externally_used_only_multiuse(i8 %v0, i8 %v
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3:%.*]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C2]], [[C3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
 ; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2]]
@@ -640,7 +640,7 @@ define void @one_pred_with_extra_op_externally_used_only_after_cond_distant_phi(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3:%.*]], 0
 ; CHECK-NEXT:    [[V2_ADJ:%.*]] = add i8 [[V4:%.*]], [[V5:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C1]], [[C3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C1]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -690,7 +690,7 @@ define void @two_preds_with_extra_op_externally_used_only_after_cond(i8 %v0, i8
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3:%.*]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V4:%.*]], [[V5:%.*]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C2]], [[C3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
 ; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3]], 0
@@ -745,7 +745,7 @@ define void @pr48450() {
 ; CHECK-NEXT:    [[C2_NOT:%.*]] = xor i1 [[C2]], true
 ; CHECK-NEXT:    [[DEC:%.*]] = add i8 [[COUNTDOWN]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i8 [[COUNTDOWN]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[C2_NOT]], [[CMP_NOT]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2_NOT]], i1 true, i1 [[CMP_NOT]]
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[IF_END_LOOPEXIT]], label [[FOR_BODYTHREAD_PRE_SPLIT]]
 ; CHECK:       for.bodythread-pre-split:
 ; CHECK-NEXT:    [[DEC_MERGE]] = phi i8 [ [[DEC]], [[IF_THEN]] ], [ [[DEC_OLD]], [[FOR_INC]] ]
@@ -796,7 +796,7 @@ define void @pr48450_2(i1 %enable_loopback) {
 ; CHECK-NEXT:    [[C2_NOT:%.*]] = xor i1 [[C2]], true
 ; CHECK-NEXT:    [[DEC:%.*]] = add i8 [[COUNTDOWN]], -1
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i8 [[COUNTDOWN]], 0
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[C2_NOT]], [[CMP_NOT]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2_NOT]], i1 true, i1 [[CMP_NOT]]
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[IF_END_LOOPEXIT]], label [[FOR_BODYTHREAD_PRE_SPLIT]]
 ; CHECK:       for.bodythread-pre-split:
 ; CHECK-NEXT:    [[DEC_MERGE]] = phi i8 [ [[DEC_OLD]], [[FOR_INC]] ], [ [[DEC_MERGE]], [[FOR_BODYTHREAD_PRE_SPLIT_LOOPBACK:%.*]] ], [ [[DEC]], [[IF_THEN]] ]
@@ -894,3 +894,37 @@ if.end7:
 cleanup:
   unreachable
 }
+
+@global_pr49510 = external global i16, align 1
+
+define void @pr49510() {
+; CHECK-LABEL: @pr49510(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTOLD:%.*]] = load i16, i16* @global_pr49510, align 1
+; CHECK-NEXT:    [[TOBOOL_OLD:%.*]] = icmp ne i16 [[DOTOLD]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_OLD]], label [[LAND_RHS:%.*]], label [[FOR_END:%.*]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    [[DOTMERGE:%.*]] = phi i16 [ [[TMP0:%.*]], [[LAND_RHS]] ], [ [[DOTOLD]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[DOTMERGE]], 0
+; CHECK-NEXT:    [[TMP0]] = load i16, i16* @global_pr49510, align 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i16 [[TMP0]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP]], i1 [[TOBOOL]], i1 false
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[LAND_RHS]], label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.cond
+
+for.cond:
+  %0 = load i16, i16* @global_pr49510, align 1
+  %tobool = icmp ne i16 %0, 0
+  br i1 %tobool, label %land.rhs, label %for.end
+
+land.rhs:
+  %cmp = icmp slt i16 %0, 0
+  br i1 %cmp, label %for.cond, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-debug-location.ll b/llvm/test/Transforms/SimplifyCFG/fold-debug-location.ll
index 1fe3faf5713bd53a4d6b79f6fb1ede14470812dd..c7834a9b2de868a3ba26c17b18bd912c38915d71 100644
--- a/llvm/test/Transforms/SimplifyCFG/fold-debug-location.ll
+++ b/llvm/test/Transforms/SimplifyCFG/fold-debug-location.ll
@@ -33,7 +33,7 @@ define void @dbg_all_equal(i32 %k, i32 %c1, i32 %c2) !dbg !17 {
 ; CHECK-NEXT:    [[C1:%[a-z0-9]+]] = icmp eq i32 [[A1]], 0, !dbg [[DBG]]
 ; CHECK-NEXT:    [[A2:%[a-z0-9]+]] = and i32 %c2, %k, !dbg [[DBG]]
 ; CHECK-NEXT:    [[C2:%[a-z0-9]+]] = icmp eq i32 [[A2]], 0, !dbg [[DBG]]
-; CHECK-NEXT:    [[OR:%[.a-z0-9]+]] = or i1 [[C1]], [[C2]], !dbg [[DBG]]
+; CHECK-NEXT:    [[OR:%[.a-z0-9]+]] = select i1 [[C1]], i1 true, i1 [[C2]], !dbg [[DBG]]
 ; CHECK-NEXT:    br i1 [[OR]], label {{.+}}, label {{.+}}, !dbg [[DBG]]
 ;
   %1 = and i32 %c1, %k, !dbg !18
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
index 921d26387aa29ace372be4d107139950222600b6..71aac1c4d48bbbf87f6d4b826480141288043e66 100644
--- a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
+++ b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
@@ -267,18 +267,17 @@ declare void @f()
 define i32 @test_diamond_simple(i32* %p, i32* %q, i32 %a, i32 %b) {
 ; CHECK-LABEL: @test_diamond_simple(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X1:%.*]] = icmp eq i32 [[A:%.*]], 0
-; CHECK-NEXT:    [[Z2:%.*]] = select i1 [[X1]], i32 [[B:%.*]], i32 0
-; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B]], 0
-; CHECK-NEXT:    [[TMP0:%.*]] = or i32 [[A]], [[B]]
-; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i32 [[TMP0]], 0
-; CHECK-NEXT:    br i1 [[DOTNOT]], label [[TMP2:%.*]], label [[TMP1:%.*]]
-; CHECK:       1:
-; CHECK-NEXT:    [[SIMPLIFYCFG_MERGE:%.*]] = select i1 [[X2]], i32 [[Z2]], i32 1
-; CHECK-NEXT:    store i32 [[SIMPLIFYCFG_MERGE]], i32* [[P:%.*]], align 4
-; CHECK-NEXT:    br label [[TMP2]]
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = or i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP3:%.*]], label [[TMP2:%.*]]
 ; CHECK:       2:
-; CHECK-NEXT:    [[Z4:%.*]] = select i1 [[X2]], i32 [[Z2]], i32 3
+; CHECK-NEXT:    [[NOT_X2:%.*]] = xor i1 [[X2]], true
+; CHECK-NEXT:    [[SIMPLIFYCFG_MERGE:%.*]] = zext i1 [[NOT_X2]] to i32
+; CHECK-NEXT:    store i32 [[SIMPLIFYCFG_MERGE]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[TMP3]]
+; CHECK:       3:
+; CHECK-NEXT:    [[Z4:%.*]] = select i1 [[X2]], i32 0, i32 3
 ; CHECK-NEXT:    ret i32 [[Z4]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
index 25106d435cc7f628d63b549deff4dcf23ddd29b8..38d82bb94481244acb26813e249969e6fa9755b0 100644
--- a/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
@@ -8,7 +8,7 @@ define void @test1(i1 %a, i1 %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_NOT:%.*]] = xor i1 [[A:%.*]], true
 ; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A_NOT]], [[C]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A_NOT]], i1 [[C]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !0
 ; CHECK:       Y:
 ; CHECK-NEXT:    call void @helper(i32 0)
@@ -40,7 +40,7 @@ define void @fake_weights(i1 %a, i1 %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_NOT:%.*]] = xor i1 [[A:%.*]], true
 ; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A_NOT]], [[C]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A_NOT]], i1 [[C]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !1
 ; CHECK:       Y:
 ; CHECK-NEXT:    call void @helper(i32 0)
@@ -68,7 +68,7 @@ define void @test2(i1 %a, i1 %b) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A:%.*]], [[C]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A:%.*]], i1 [[C]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !2
 ; CHECK:       Y:
 ; CHECK-NEXT:    call void @helper(i32 0)
@@ -97,7 +97,7 @@ define void @test3(i1 %a, i1 %b) {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A:%.*]], [[C]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A:%.*]], i1 [[C]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !1
 ; CHECK:       Y:
 ; CHECK-NEXT:    call void @helper(i32 0)
@@ -126,7 +126,7 @@ define void @test4(i1 %a, i1 %b) {
 ; CHECK-LABEL: @test4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
-; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A:%.*]], [[C]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A:%.*]], i1 [[C]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !1
 ; CHECK:       Y:
 ; CHECK-NEXT:    call void @helper(i32 0)
@@ -257,7 +257,7 @@ define void @test1_swap(i1 %a, i1 %b) {
 ; CHECK-LABEL: @test1_swap(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[A:%.*]], [[C]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[C]]
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[Y:%.*]], label [[Z:%.*]], !prof !5
 ; CHECK:       Y:
 ; CHECK-NEXT:    call void @helper(i32 0)
@@ -318,10 +318,10 @@ define void @test8(i64 %x, i64 %y) nounwind {
 ; CHECK-NEXT:    [[LT:%.*]] = icmp slt i64 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    br i1 [[LT]], label [[A:%.*]], label [[B:%.*]], !prof !7
 ; CHECK:       a:
-; CHECK-NEXT:    call void @helper(i32 0) [[ATTR1:#.*]]
+; CHECK-NEXT:    call void @helper(i32 0) #[[ATTR1:[0-9]+]]
 ; CHECK-NEXT:    ret void
 ; CHECK:       b:
-; CHECK-NEXT:    call void @helper(i32 1) [[ATTR1]]
+; CHECK-NEXT:    call void @helper(i32 1) #[[ATTR1]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -355,14 +355,14 @@ define i1 @test9(i32 %x, i32 %y) nounwind {
 ; CHECK-NEXT:    i32 92, label [[END]]
 ; CHECK-NEXT:    ], !prof !8
 ; CHECK:       a:
-; CHECK-NEXT:    call void @helper(i32 0) [[ATTR1]]
+; CHECK-NEXT:    call void @helper(i32 0) #[[ATTR1]]
 ; CHECK-NEXT:    [[RETA:%.*]] = icmp slt i32 [[X]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[RETA]]
 ; CHECK:       bees:
 ; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
 ; CHECK-NEXT:    [[RET:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ false, [[BEES]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ]
-; CHECK-NEXT:    call void @helper(i32 2) [[ATTR1]]
+; CHECK-NEXT:    call void @helper(i32 2) #[[ATTR1]]
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
 entry:
@@ -394,10 +394,10 @@ define void @test10(i32 %x) nounwind readnone ssp noredzone {
 ; CHECK-NEXT:    [[SWITCH:%.*]] = icmp ult i32 [[X_OFF]], 3
 ; CHECK-NEXT:    br i1 [[SWITCH]], label [[LOR_END:%.*]], label [[LOR_RHS:%.*]], !prof !9
 ; CHECK:       lor.rhs:
-; CHECK-NEXT:    call void @helper(i32 1) [[ATTR1]]
+; CHECK-NEXT:    call void @helper(i32 1) #[[ATTR1]]
 ; CHECK-NEXT:    ret void
 ; CHECK:       lor.end:
-; CHECK-NEXT:    call void @helper(i32 0) [[ATTR1]]
+; CHECK-NEXT:    call void @helper(i32 0) #[[ATTR1]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -424,10 +424,10 @@ define void @test11(i32 %x) nounwind {
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I]], 24
 ; CHECK-NEXT:    br i1 [[COND]], label [[C:%.*]], label [[A:%.*]], !prof !10
 ; CHECK:       a:
-; CHECK-NEXT:    call void @helper(i32 0) [[ATTR1]]
+; CHECK-NEXT:    call void @helper(i32 0) #[[ATTR1]]
 ; CHECK-NEXT:    ret void
 ; CHECK:       c:
-; CHECK-NEXT:    call void @helper(i32 2) [[ATTR1]]
+; CHECK-NEXT:    call void @helper(i32 2) #[[ATTR1]]
 ; CHECK-NEXT:    ret void
 ;
   %i = shl i32 %x, 1
@@ -472,7 +472,7 @@ sw.epilog:
 define void @test13(i32 %x) nounwind {
 ; CHECK-LABEL: @test13(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    call void @helper(i32 0) [[ATTR1]]
+; CHECK-NEXT:    call void @helper(i32 0) #[[ATTR1]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -507,7 +507,7 @@ define void @test14(i32* %old, i32 %final) {
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[BIT_0]], 0
 ; CHECK-NEXT:    [[V3:%.*]] = load i32, i32* @max_regno, align 4
 ; CHECK-NEXT:    [[CMP4:%.*]] = icmp eq i32 [[I_1]], [[V3]]
-; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[TOBOOL]], [[CMP4]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[TOBOOL]], i1 true, i1 [[CMP4]]
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FOR_EXIT:%.*]], label [[FOR_INC]], !prof !11
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[SHL]] = shl i32 [[BIT_0]], 1
@@ -636,6 +636,422 @@ exit:
   ret i32 %outval
 }
 
+; Merging the icmps with logic-op defeats the purpose of the metadata.
+; We can't tell which condition is expensive if they are combined.
+
+define void @or_icmps_harmful(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @or_icmps_harmful(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    br i1 [[EXPECTED_TRUE]], label [[EXIT:%.*]], label [[RARE:%.*]], !prof !19
+; CHECK:       rare:
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    br i1 [[EXPENSIVE]], label [[EXIT]], label [[FALSE:%.*]]
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_true = icmp sgt i32 %x, -1
+  br i1 %expected_true, label %exit, label %rare, !prof !15
+
+rare:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %exit, label %false
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Merging the icmps with logic-op defeats the purpose of the metadata.
+; We can't tell which condition is expensive if they are combined.
+
+define void @or_icmps_harmful_inverted(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @or_icmps_harmful_inverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_FALSE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    br i1 [[EXPECTED_FALSE]], label [[RARE:%.*]], label [[EXIT:%.*]], !prof !20
+; CHECK:       rare:
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    br i1 [[EXPENSIVE]], label [[EXIT]], label [[FALSE:%.*]]
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_false = icmp sgt i32 %x, -1
+  br i1 %expected_false, label %rare, label %exit, !prof !16
+
+rare:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %exit, label %false
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; The probability threshold is determined by a TTI setting.
+; In this example, we are just short of strongly expected, so speculate.
+
+define void @or_icmps_not_that_harmful(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @or_icmps_not_that_harmful(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 true, i1 [[EXPENSIVE]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof !21
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_true = icmp sgt i32 %x, -1
+  br i1 %expected_true, label %exit, label %rare, !prof !17
+
+rare:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %exit, label %false
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; The probability threshold is determined by a TTI setting.
+; In this example, we are just short of strongly expected, so speculate.
+
+define void @or_icmps_not_that_harmful_inverted(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @or_icmps_not_that_harmful_inverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 true, i1 [[EXPENSIVE]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof !22
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_true = icmp sgt i32 %x, -1
+  br i1 %expected_true, label %exit, label %rare, !prof !18
+
+rare:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %exit, label %false
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; The 1st cmp is probably true, so speculating the 2nd is probably a win.
+
+define void @or_icmps_useful(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @or_icmps_useful(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_TRUE:%.*]] = icmp sle i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 true, i1 [[EXPENSIVE]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof !23
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_true = icmp sgt i32 %x, -1
+  br i1 %expected_true, label %likely, label %exit, !prof !15
+
+likely:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %exit, label %false
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; The 1st cmp is probably false, so speculating the 2nd is probably a win.
+
+define void @or_icmps_useful_inverted(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @or_icmps_useful_inverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_FALSE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EXPECTED_FALSE]], i1 true, i1 [[EXPENSIVE]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof !23
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_false = icmp sgt i32 %x, -1
+  br i1 %expected_false, label %exit, label %likely, !prof !16
+
+likely:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %exit, label %false
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Don't crash processing degenerate metadata.
+
+define void @or_icmps_empty_metadata(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @or_icmps_empty_metadata(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 true, i1 [[EXPENSIVE]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[EXIT:%.*]], label [[MORE_RARE:%.*]]
+; CHECK:       more_rare:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_true = icmp sgt i32 %x, -1
+  br i1 %expected_true, label %exit, label %rare, !prof !19
+
+rare:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %exit, label %more_rare
+
+more_rare:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Merging the icmps with logic-op defeats the purpose of the metadata.
+; We can't tell which condition is expensive if they are combined.
+
+define void @and_icmps_harmful(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @and_icmps_harmful(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_FALSE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    br i1 [[EXPECTED_FALSE]], label [[RARE:%.*]], label [[EXIT:%.*]], !prof !20
+; CHECK:       rare:
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    br i1 [[EXPENSIVE]], label [[FALSE:%.*]], label [[EXIT]]
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_false = icmp sgt i32 %x, -1
+  br i1 %expected_false, label %rare, label %exit, !prof !16
+
+rare:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %false, label %exit
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Merging the icmps with logic-op defeats the purpose of the metadata.
+; We can't tell which condition is expensive if they are combined.
+
+define void @and_icmps_harmful_inverted(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @and_icmps_harmful_inverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    br i1 [[EXPECTED_TRUE]], label [[EXIT:%.*]], label [[RARE:%.*]], !prof !19
+; CHECK:       rare:
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    br i1 [[EXPENSIVE]], label [[FALSE:%.*]], label [[EXIT]]
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_true = icmp sgt i32 %x, -1
+  br i1 %expected_true, label %exit, label %rare, !prof !15
+
+rare:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %false, label %exit
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; The probability threshold is determined by a TTI setting.
+; In this example, we are just short of strongly expected, so speculate.
+
+define void @and_icmps_not_that_harmful(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @and_icmps_not_that_harmful(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_FALSE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EXPECTED_FALSE]], i1 [[EXPENSIVE]], i1 false
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof !24
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_false = icmp sgt i32 %x, -1
+  br i1 %expected_false, label %rare, label %exit, !prof !18
+
+rare:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %false, label %exit
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; The probability threshold is determined by a TTI setting.
+; In this example, we are just short of strongly expected, so speculate.
+
+define void @and_icmps_not_that_harmful_inverted(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @and_icmps_not_that_harmful_inverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_TRUE:%.*]] = icmp sle i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 [[EXPENSIVE]], i1 false
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof !24
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_true = icmp sgt i32 %x, -1
+  br i1 %expected_true, label %exit, label %rare, !prof !17
+
+rare:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %false, label %exit
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; The 1st cmp is probably true, so speculating the 2nd is probably a win.
+
+define void @and_icmps_useful(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @and_icmps_useful(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 [[EXPENSIVE]], i1 false
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof !25
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_true = icmp sgt i32 %x, -1
+  br i1 %expected_true, label %likely, label %exit, !prof !15
+
+likely:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %false, label %exit
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; The 1st cmp is probably false, so speculating the 2nd is probably a win.
+
+define void @and_icmps_useful_inverted(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @and_icmps_useful_inverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXPECTED_FALSE:%.*]] = icmp sle i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EXPECTED_FALSE]], i1 [[EXPENSIVE]], i1 false
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof !25
+; CHECK:       false:
+; CHECK-NEXT:    store i8 42, i8* [[P:%.*]], align 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %expected_false = icmp sgt i32 %x, -1
+  br i1 %expected_false, label %exit, label %likely, !prof !16
+
+likely:
+  %expensive = icmp eq i32 %y, 0
+  br i1 %expensive, label %false, label %exit
+
+false:
+  store i8 42, i8* %p, align 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+
 !0 = !{!"branch_weights", i32 3, i32 5}
 !1 = !{!"branch_weights", i32 1, i32 1}
 !2 = !{!"branch_weights", i32 1, i32 2}
@@ -651,6 +1067,11 @@ exit:
 !12 = !{!"these_are_not_the_branch_weights_you_are_looking_for", i32 3, i32 5}
 !13 = !{!"branch_weights", i32 2, i32 3}
 !14 = !{!"branch_weights", i32 4, i32 7}
+!15 = !{!"branch_weights", i32 99, i32 1}
+!16 = !{!"branch_weights", i32 1, i32 99}
+!17 = !{!"branch_weights", i32 98, i32 1}
+!18 = !{!"branch_weights", i32 1, i32 98}
+!19 = !{!"branch_weights", i32 0, i32 0}
 
 ; CHECK: !0 = !{!"branch_weights", i32 5, i32 11}
 ; CHECK: !1 = !{!"branch_weights", i32 1, i32 3}
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll b/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll
index a4f9e1dd29dd9d7214c47799c7c662173868dc53..e5c12cbc56f76a8e1cd276bcad94696f815105be 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll
@@ -465,8 +465,8 @@ return:                                           ; preds = %if.end, %if.then
 define void @test12() nounwind {
 ; CHECK-LABEL: @test12(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[DOTOLD:%.*]] = icmp eq i32 undef, undef
-; CHECK-NEXT:    br i1 [[DOTOLD]], label [[BB55_US_US:%.*]], label [[MALFORMED:%.*]]
+; CHECK-NEXT:    [[A_OLD:%.*]] = icmp eq i32 undef, undef
+; CHECK-NEXT:    br i1 [[A_OLD]], label [[BB55_US_US:%.*]], label [[MALFORMED:%.*]]
 ; CHECK:       bb55.us.us:
 ; CHECK-NEXT:    [[B:%.*]] = icmp ugt i32 undef, undef
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 undef, undef
diff --git a/llvm/test/Transforms/TailCallElim/reorder_load.ll b/llvm/test/Transforms/TailCallElim/reorder_load.ll
index 027cfe78bb4bb418f094548b0fb69e7a11f3ae3c..59d9703a80ac77aeac381a904900d09d77f76428 100644
--- a/llvm/test/Transforms/TailCallElim/reorder_load.ll
+++ b/llvm/test/Transforms/TailCallElim/reorder_load.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -tailcallelim -verify-dom-info -S | FileCheck %s
 ; PR4323
 
@@ -16,23 +17,35 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; and the call has no side effects.
 define fastcc i32 @raise_load_1(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) nounwind readonly {
 ; CHECK-LABEL: @raise_load_1(
-; CHECK-NOT: call
-; CHECK: load i32, i32*
-; CHECK-NOT: call
-; CHECK: }
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[TAILRECURSE:%.*]]
+; CHECK:       tailrecurse:
+; CHECK-NEXT:    [[ACCUMULATOR_TR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP10:%.*]], [[ELSE:%.*]] ]
+; CHECK-NEXT:    [[START_ARG_TR:%.*]] = phi i32 [ [[START_ARG:%.*]], [[ENTRY]] ], [ [[TMP7:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sge i32 [[START_ARG_TR]], [[A_LEN_ARG:%.*]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[IF:%.*]], label [[ELSE]]
+; CHECK:       if:
+; CHECK-NEXT:    [[ACCUMULATOR_RET_TR:%.*]] = add i32 0, [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    ret i32 [[ACCUMULATOR_RET_TR]]
+; CHECK:       else:
+; CHECK-NEXT:    [[TMP7]] = add i32 [[START_ARG_TR]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[A_ARG:%.*]], align 4
+; CHECK-NEXT:    [[TMP10]] = add i32 [[TMP9]], [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    br label [[TAILRECURSE]]
+;
 entry:
-	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
-	br i1 %tmp2, label %if, label %else
+  %tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+  br i1 %tmp2, label %if, label %else
 
 if:		; preds = %entry
-	ret i32 0
+  ret i32 0
 
 else:		; preds = %entry
-	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
-	%tmp8 = call fastcc i32 @raise_load_1(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
-	%tmp9 = load i32, i32* %a_arg		; <i32> [#uses=1]
-	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
-	ret i32 %tmp10
+  %tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+  %tmp8 = call fastcc i32 @raise_load_1(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+  %tmp9 = load i32, i32* %a_arg		; <i32> [#uses=1]
+  %tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+  ret i32 %tmp10
 }
 
 
@@ -40,30 +53,47 @@ else:		; preds = %entry
 ; and the load provably can't trap.
 define fastcc i32 @raise_load_2(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) readonly {
 ; CHECK-LABEL: @raise_load_2(
-; CHECK-NOT: call
-; CHECK: load i32, i32*
-; CHECK-NOT: call
-; CHECK: }
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[TAILRECURSE:%.*]]
+; CHECK:       tailrecurse:
+; CHECK-NEXT:    [[ACCUMULATOR_TR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP10:%.*]], [[RECURSE:%.*]] ]
+; CHECK-NEXT:    [[START_ARG_TR:%.*]] = phi i32 [ [[START_ARG:%.*]], [[ENTRY]] ], [ [[TMP7:%.*]], [[RECURSE]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sge i32 [[START_ARG_TR]], [[A_LEN_ARG:%.*]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[ACCUMULATOR_RET_TR:%.*]] = add i32 0, [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    ret i32 [[ACCUMULATOR_RET_TR]]
+; CHECK:       else:
+; CHECK-NEXT:    [[NULLCHECK:%.*]] = icmp eq i32* [[A_ARG:%.*]], null
+; CHECK-NEXT:    br i1 [[NULLCHECK]], label [[UNWIND:%.*]], label [[RECURSE]]
+; CHECK:       unwind:
+; CHECK-NEXT:    unreachable
+; CHECK:       recurse:
+; CHECK-NEXT:    [[TMP7]] = add i32 [[START_ARG_TR]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* @global, align 4
+; CHECK-NEXT:    [[TMP10]] = add i32 [[TMP9]], [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    br label [[TAILRECURSE]]
+;
 entry:
-	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
-	br i1 %tmp2, label %if, label %else
+  %tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+  br i1 %tmp2, label %if, label %else
 
 if:		; preds = %entry
-	ret i32 0
+  ret i32 0
 
 else:		; preds = %entry
-	%nullcheck = icmp eq i32* %a_arg, null		; <i1> [#uses=1]
-	br i1 %nullcheck, label %unwind, label %recurse
+  %nullcheck = icmp eq i32* %a_arg, null		; <i1> [#uses=1]
+  br i1 %nullcheck, label %unwind, label %recurse
 
 unwind:		; preds = %else
-	unreachable
+  unreachable
 
 recurse:		; preds = %else
-	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
-	%tmp8 = call fastcc i32 @raise_load_2(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
-	%tmp9 = load i32, i32* @global		; <i32> [#uses=1]
-	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
-	ret i32 %tmp10
+  %tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+  %tmp8 = call fastcc i32 @raise_load_2(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+  %tmp9 = load i32, i32* @global		; <i32> [#uses=1]
+  %tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+  ret i32 %tmp10
 }
 
 
@@ -71,23 +101,35 @@ recurse:		; preds = %else
 ; extern_weak global) because the call has no side effects.
 define fastcc i32 @raise_load_3(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) nounwind readonly {
 ; CHECK-LABEL: @raise_load_3(
-; CHECK-NOT: call
-; CHECK: load i32, i32*
-; CHECK-NOT: call
-; CHECK: }
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[TAILRECURSE:%.*]]
+; CHECK:       tailrecurse:
+; CHECK-NEXT:    [[ACCUMULATOR_TR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP10:%.*]], [[ELSE:%.*]] ]
+; CHECK-NEXT:    [[START_ARG_TR:%.*]] = phi i32 [ [[START_ARG:%.*]], [[ENTRY]] ], [ [[TMP7:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sge i32 [[START_ARG_TR]], [[A_LEN_ARG:%.*]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[IF:%.*]], label [[ELSE]]
+; CHECK:       if:
+; CHECK-NEXT:    [[ACCUMULATOR_RET_TR:%.*]] = add i32 0, [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    ret i32 [[ACCUMULATOR_RET_TR]]
+; CHECK:       else:
+; CHECK-NEXT:    [[TMP7]] = add i32 [[START_ARG_TR]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* @extern_weak_global, align 4
+; CHECK-NEXT:    [[TMP10]] = add i32 [[TMP9]], [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    br label [[TAILRECURSE]]
+;
 entry:
-	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
-	br i1 %tmp2, label %if, label %else
+  %tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+  br i1 %tmp2, label %if, label %else
 
 if:		; preds = %entry
-	ret i32 0
+  ret i32 0
 
 else:		; preds = %entry
-	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
-	%tmp8 = call fastcc i32 @raise_load_3(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
-	%tmp9 = load i32, i32* @extern_weak_global		; <i32> [#uses=1]
-	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
-	ret i32 %tmp10
+  %tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+  %tmp8 = call fastcc i32 @raise_load_3(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+  %tmp9 = load i32, i32* @extern_weak_global		; <i32> [#uses=1]
+  %tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+  ret i32 %tmp10
 }
 
 
@@ -96,79 +138,124 @@ else:		; preds = %entry
 ; proves it doesn't trap.
 define fastcc i32 @raise_load_4(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) readonly {
 ; CHECK-LABEL: @raise_load_4(
-; CHECK-NOT: call
-; CHECK: load i32, i32*
-; CHECK-NEXT: load i32, i32*
-; CHECK-NOT: call
-; CHECK: }
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[TAILRECURSE:%.*]]
+; CHECK:       tailrecurse:
+; CHECK-NEXT:    [[ACCUMULATOR_TR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP10:%.*]], [[RECURSE:%.*]] ]
+; CHECK-NEXT:    [[A_LEN_ARG_TR:%.*]] = phi i32 [ [[A_LEN_ARG:%.*]], [[ENTRY]] ], [ [[FIRST:%.*]], [[RECURSE]] ]
+; CHECK-NEXT:    [[START_ARG_TR:%.*]] = phi i32 [ [[START_ARG:%.*]], [[ENTRY]] ], [ [[TMP7:%.*]], [[RECURSE]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sge i32 [[START_ARG_TR]], [[A_LEN_ARG_TR]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[ACCUMULATOR_RET_TR:%.*]] = add i32 0, [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    ret i32 [[ACCUMULATOR_RET_TR]]
+; CHECK:       else:
+; CHECK-NEXT:    [[NULLCHECK:%.*]] = icmp eq i32* [[A_ARG:%.*]], null
+; CHECK-NEXT:    br i1 [[NULLCHECK]], label [[UNWIND:%.*]], label [[RECURSE]]
+; CHECK:       unwind:
+; CHECK-NEXT:    unreachable
+; CHECK:       recurse:
+; CHECK-NEXT:    [[TMP7]] = add i32 [[START_ARG_TR]], 1
+; CHECK-NEXT:    [[FIRST]] = load i32, i32* [[A_ARG]], align 4
+; CHECK-NEXT:    [[SECOND:%.*]] = load i32, i32* [[A_ARG]], align 4
+; CHECK-NEXT:    [[TMP10]] = add i32 [[SECOND]], [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    br label [[TAILRECURSE]]
+;
 entry:
-	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
-	br i1 %tmp2, label %if, label %else
+  %tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+  br i1 %tmp2, label %if, label %else
 
 if:		; preds = %entry
-	ret i32 0
+  ret i32 0
 
 else:		; preds = %entry
-	%nullcheck = icmp eq i32* %a_arg, null		; <i1> [#uses=1]
-	br i1 %nullcheck, label %unwind, label %recurse
+  %nullcheck = icmp eq i32* %a_arg, null		; <i1> [#uses=1]
+  br i1 %nullcheck, label %unwind, label %recurse
 
 unwind:		; preds = %else
-	unreachable
+  unreachable
 
 recurse:		; preds = %else
-	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
-	%first = load i32, i32* %a_arg		; <i32> [#uses=1]
-	%tmp8 = call fastcc i32 @raise_load_4(i32* %a_arg, i32 %first, i32 %tmp7)		; <i32> [#uses=1]
-	%second = load i32, i32* %a_arg		; <i32> [#uses=1]
-	%tmp10 = add i32 %second, %tmp8		; <i32> [#uses=1]
-	ret i32 %tmp10
+  %tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+  %first = load i32, i32* %a_arg		; <i32> [#uses=1]
+  %tmp8 = call fastcc i32 @raise_load_4(i32* %a_arg, i32 %first, i32 %tmp7)		; <i32> [#uses=1]
+  %second = load i32, i32* %a_arg		; <i32> [#uses=1]
+  %tmp10 = add i32 %second, %tmp8		; <i32> [#uses=1]
+  ret i32 %tmp10
 }
 
 ; This load can be moved above the call because the function won't write to it
 ; and the a_arg is dereferenceable.
 define fastcc i32 @raise_load_5(i32* dereferenceable(4) align 4 %a_arg, i32 %a_len_arg, i32 %start_arg) readonly {
 ; CHECK-LABEL: @raise_load_5(
-; CHECK-NOT: call
-; CHECK: load i32, i32*
-; CHECK-NOT: call
-; CHECK: }
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[TAILRECURSE:%.*]]
+; CHECK:       tailrecurse:
+; CHECK-NEXT:    [[ACCUMULATOR_TR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP10:%.*]], [[ELSE:%.*]] ]
+; CHECK-NEXT:    [[START_ARG_TR:%.*]] = phi i32 [ [[START_ARG:%.*]], [[ENTRY]] ], [ [[TMP7:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sge i32 [[START_ARG_TR]], [[A_LEN_ARG:%.*]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[IF:%.*]], label [[ELSE]]
+; CHECK:       if:
+; CHECK-NEXT:    [[ACCUMULATOR_RET_TR:%.*]] = add i32 0, [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    ret i32 [[ACCUMULATOR_RET_TR]]
+; CHECK:       else:
+; CHECK-NEXT:    [[TMP7]] = add i32 [[START_ARG_TR]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[A_ARG:%.*]], align 4
+; CHECK-NEXT:    [[TMP10]] = add i32 [[TMP9]], [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    br label [[TAILRECURSE]]
+;
 entry:
-	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
-	br i1 %tmp2, label %if, label %else
+  %tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+  br i1 %tmp2, label %if, label %else
 
 if:		; preds = %entry
-	ret i32 0
+  ret i32 0
 
 else:		; preds = %entry
-	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
-	%tmp8 = call fastcc i32 @raise_load_5(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
-	%tmp9 = load i32, i32* %a_arg		; <i32> [#uses=1]
-	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
-	ret i32 %tmp10
+  %tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+  %tmp8 = call fastcc i32 @raise_load_5(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+  %tmp9 = load i32, i32* %a_arg		; <i32> [#uses=1]
+  %tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+  ret i32 %tmp10
 }
 
 ; This load can be moved above the call because the function call does not write to the memory the load
 ; is accessing and the load is safe to speculate.
 define fastcc i32 @raise_load_6(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) nounwind  {
 ; CHECK-LABEL: @raise_load_6(
-; CHECK-NOT: call
-; CHECK: load i32, i32*
-; CHECK-NOT: call
-; CHECK: }
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    br label [[TAILRECURSE:%.*]]
+; CHECK:       tailrecurse:
+; CHECK-NEXT:    [[ACCUMULATOR_TR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP10:%.*]], [[ELSE:%.*]] ]
+; CHECK-NEXT:    [[START_ARG_TR:%.*]] = phi i32 [ [[START_ARG:%.*]], [[ENTRY]] ], [ [[TMP7:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    store i32 4, i32* [[S]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sge i32 [[START_ARG_TR]], [[A_LEN_ARG:%.*]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[IF:%.*]], label [[ELSE]]
+; CHECK:       if:
+; CHECK-NEXT:    store i32 1, i32* [[A_ARG:%.*]], align 4
+; CHECK-NEXT:    [[ACCUMULATOR_RET_TR:%.*]] = add i32 0, [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    ret i32 [[ACCUMULATOR_RET_TR]]
+; CHECK:       else:
+; CHECK-NEXT:    [[TMP7]] = add i32 [[START_ARG_TR]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[S]], align 4
+; CHECK-NEXT:    [[TMP10]] = add i32 [[TMP9]], [[ACCUMULATOR_TR]]
+; CHECK-NEXT:    br label [[TAILRECURSE]]
+;
 entry:
   %s = alloca i32
   store i32 4, i32* %s
-	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
-	br i1 %tmp2, label %if, label %else
+  %tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+  br i1 %tmp2, label %if, label %else
 
 if:		; preds = %entry
   store i32 1, i32* %a_arg
-	ret i32 0
+  ret i32 0
 
 else:		; preds = %entry
-	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
-	%tmp8 = call fastcc i32 @raise_load_6(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
-	%tmp9 = load i32, i32* %s		; <i32> [#uses=1]
-	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
-	ret i32 %tmp10
+  %tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+  %tmp8 = call fastcc i32 @raise_load_6(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+  %tmp9 = load i32, i32* %s		; <i32> [#uses=1]
+  %tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+  ret i32 %tmp10
 }
diff --git a/llvm/test/Transforms/Util/PredicateInfo/condprop.ll b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
index 689326b6ca978b3ef82aaf2c8db6ddba0d84a4a4..9400e60c81ffc639c59c963869ff894d4d506b38 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
@@ -186,10 +186,10 @@ case3:
 define i1 @test5(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
-; CHECK:         [[Y_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
+; CHECK:         [[Y_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
 ; CHECK:       same:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[X_0]], [[Y_0]]
@@ -259,10 +259,10 @@ different:
 define i1 @test7(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test7(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
-; CHECK:         [[Y_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
+; CHECK:         [[Y_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
 ; CHECK:       same:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[X_0]], [[Y_0]]
@@ -286,10 +286,10 @@ different:
 define i1 @test7_fp(float %x, float %y) {
 ; CHECK-LABEL: @test7_fp(
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[X:%.*]], [[Y:%.*]]
-; CHECK:         [[X_0:%.*]] = call float @llvm.ssa.copy.{{.+}}(float [[X]])
-; CHECK:         [[X_1:%.*]] = call float @llvm.ssa.copy.{{.+}}(float [[X]])
-; CHECK:         [[Y_0:%.*]] = call float @llvm.ssa.copy.{{.+}}(float [[Y]])
-; CHECK:         [[Y_1:%.*]] = call float @llvm.ssa.copy.{{.+}}(float [[Y]])
+; CHECK:         [[X_0:%.*]] = call float @llvm.ssa.copy.f32(float [[X]])
+; CHECK:         [[X_1:%.*]] = call float @llvm.ssa.copy.f32(float [[X]])
+; CHECK:         [[Y_0:%.*]] = call float @llvm.ssa.copy.f32(float [[Y]])
+; CHECK:         [[Y_1:%.*]] = call float @llvm.ssa.copy.f32(float [[Y]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
 ; CHECK:       same:
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ule float [[X_0]], [[Y_0]]
@@ -359,8 +359,8 @@ different:
 define i32 @test9(i32 %i, i32 %j) {
 ; CHECK-LABEL: @test9(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]]
-; CHECK:         [[I_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[I]])
-; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[J]])
+; CHECK:         [[I_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[I]])
+; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[J]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]]
 ; CHECK:       cond_true:
 ; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[I_0]], [[J_0]]
@@ -382,8 +382,8 @@ ret:
 define i32 @test10(i32 %j, i32 %i) {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]]
-; CHECK:         [[I_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[I]])
-; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[J]])
+; CHECK:         [[I_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[I]])
+; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[J]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]]
 ; CHECK:       cond_true:
 ; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[I_0]], [[J_0]]
@@ -409,14 +409,14 @@ define i32 @test11(i32 %x) {
 ; CHECK-NEXT:    [[V0:%.*]] = call i32 @yogibar()
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @yogibar()
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V0]], [[V1]]
-; CHECK:         [[V0_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[V0]])
-; CHECK:         [[V1_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[V1]])
+; CHECK:         [[V0_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[V0]])
+; CHECK:         [[V1_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[V1]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[NEXT:%.*]]
 ; CHECK:       cond_true:
 ; CHECK-NEXT:    ret i32 [[V1_0]]
 ; CHECK:       next:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[X:%.*]], [[V0_0]]
-; CHECK:         [[V0_0_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[V0_0]])
+; CHECK:         [[V0_0_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[V0_0]])
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[COND_TRUE2:%.*]], label [[NEXT2:%.*]]
 ; CHECK:       cond_true2:
 ; CHECK-NEXT:    ret i32 [[V0_0_1]]
@@ -445,8 +445,8 @@ next2:
 define i32 @test12(i32 %x) {
 ; CHECK-LABEL: @test12(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond_true:
 ; CHECK-NEXT:    br label [[RET:%.*]]
diff --git a/llvm/test/Transforms/Util/PredicateInfo/diamond.ll b/llvm/test/Transforms/Util/PredicateInfo/diamond.ll
index 8e3da687c1395f21814477d60c526019e9a5b1fd..e3f56d88caf0e7af5518507a298575e86da45f4d 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/diamond.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/diamond.ll
@@ -5,12 +5,12 @@ define i1 @f(i32 %x, i1 %y) {
 ; CHECK-NEXT:    br i1 [[Y:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[BB3:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[X2:%.*]] = add nuw nsw i32 [[X]], 1
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X2]], 2
-; CHECK:         [[X2_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X2]])
+; CHECK:         [[X2_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X2]])
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[BB2]], label [[BB3]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[X3:%.*]] = phi i32 [ [[X_0]], [[BB0]] ], [ [[X2_0]], [[BB1]] ]
@@ -38,12 +38,12 @@ define i1 @g(i32 %x, i1 %y) {
 ; CHECK-NEXT:    br i1 [[Y:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB3:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[X2:%.*]] = add nuw nsw i32 [[X]], 1
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X2]], 2
-; CHECK:         [[X2_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X2]])
+; CHECK:         [[X2_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X2]])
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[BB3]], label [[BB2]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[X3:%.*]] = phi i32 [ [[X_0]], [[BB0]] ], [ [[X2_0]], [[BB1]] ]
diff --git a/llvm/test/Transforms/Util/PredicateInfo/edge.ll b/llvm/test/Transforms/Util/PredicateInfo/edge.ll
index dbd15dc70ba528e06f4512f930cee0933bb6ed2c..2b88e32fd4503a90240dda6a7bafb0d8656c4e11 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/edge.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/edge.ll
@@ -5,7 +5,7 @@ define i32 @f1(i32 %x) {
 ; CHECK-LABEL: @f1(
 ; CHECK-NEXT:  bb0:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    br label [[BB2]]
@@ -29,7 +29,7 @@ define i32 @f2(i32 %x) {
 ; CHECK-LABEL: @f2(
 ; CHECK-NEXT:  bb0:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    br label [[BB2]]
@@ -52,7 +52,7 @@ bb2:
 define i32 @f3(i32 %x) {
 ; CHECK-LABEL: @f3(
 ; CHECK-NEXT:  bb0:
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X:%.*]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X:%.*]])
 ; CHECK-NEXT:    switch i32 [[X]], label [[BB1:%.*]] [
 ; CHECK-NEXT:    i32 0, label [[BB2:%.*]]
 ; CHECK-NEXT:    ]
@@ -78,7 +78,7 @@ define double @fcmp_oeq_not_zero(double %x, double %y) {
 ; CHECK-LABEL: @fcmp_oeq_not_zero(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y:%.*]], 2.000000e+00
-; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Y]])
+; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Y]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[RETURN:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
@@ -105,7 +105,7 @@ define double @fcmp_une_not_zero(double %x, double %y) {
 ; CHECK-LABEL: @fcmp_une_not_zero(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[Y:%.*]], 2.000000e+00
-; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Y]])
+; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Y]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
@@ -132,7 +132,7 @@ define double @fcmp_oeq_zero(double %x, double %y) {
 ; CHECK-LABEL: @fcmp_oeq_zero(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y:%.*]], 0.000000e+00
-; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Y]])
+; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Y]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[RETURN:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
@@ -159,7 +159,7 @@ define double @fcmp_une_zero(double %x, double %y) {
 ; CHECK-LABEL: @fcmp_une_zero(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[Y:%.*]], -0.000000e+00
-; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Y]])
+; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Y]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
@@ -188,7 +188,7 @@ define double @fcmp_oeq_maybe_zero(double %x, double %y, double %z1, double %z2)
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[Z:%.*]] = fadd double [[Z1:%.*]], [[Z2:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y:%.*]], [[Z]]
-; CHECK:         [[Z_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Z]])
+; CHECK:         [[Z_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Z]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[RETURN:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Z_0]]
@@ -217,7 +217,7 @@ define double @fcmp_une_maybe_zero(double %x, double %y, double %z1, double %z2)
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[Z:%.*]] = fadd double [[Z1:%.*]], [[Z2:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[Y:%.*]], [[Z]]
-; CHECK:         [[Z_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Z]])
+; CHECK:         [[Z_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Z]])
 ; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Z_0]]
diff --git a/llvm/test/Transforms/Util/PredicateInfo/testandor.ll b/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
index 64ca2664e4f84b109343ed1beece3cac734aa737..9c765fe72b89fec5422e9054ccba173c29033b4e 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
@@ -10,11 +10,11 @@ define void @test_or(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
-; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
+; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
 ; CHECK-NEXT:    br i1 [[Z]], label [[ONEOF:%.*]], label [[NEITHER:%.*]]
 ; CHECK:       oneof:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ]])
@@ -55,11 +55,11 @@ define void @test_or_logical(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = select i1 [[XZ]], i1 true, i1 [[YZ]]
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
-; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
+; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
 ; CHECK-NEXT:    br i1 [[Z]], label [[ONEOF:%.*]], label [[NEITHER:%.*]]
 ; CHECK:       oneof:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ]])
@@ -100,11 +100,11 @@ define void @test_and(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
-; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
+; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ_0]])
@@ -145,11 +145,11 @@ define void @test_and_logical(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = select i1 [[XZ]], i1 [[YZ]], i1 false
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
-; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
+; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ_0]])
@@ -190,11 +190,11 @@ define void @testandsame(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XGT:%.*]] = icmp sgt i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[XLT:%.*]] = icmp slt i32 [[X]], 100
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XGT]], [[XLT]]
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
-; CHECK:         [[XGT_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XGT]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X_0]])
-; CHECK:         [[XLT_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XLT]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
+; CHECK:         [[XGT_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XGT]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X_0]])
+; CHECK:         [[XLT_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XLT]])
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[XGT_0]])
@@ -229,16 +229,16 @@ define void @testandassume(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[Z]])
-; CHECK:         [[TMP1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
-; CHECK:         [[TMP2:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
-; CHECK:         [[TMP3:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
-; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
-; CHECK:         [[DOT0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP5]])
-; CHECK:         [[DOT01:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP4]])
-; CHECK:         [[DOT02:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[TMP3]])
-; CHECK:         [[DOT03:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP2]])
-; CHECK:         [[DOT04:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[TMP1]])
+; CHECK:         [[TMP1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
+; CHECK:         [[TMP2:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
+; CHECK:         [[TMP3:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
+; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
+; CHECK:         [[DOT0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP5]])
+; CHECK:         [[DOT01:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP4]])
+; CHECK:         [[DOT02:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[TMP3]])
+; CHECK:         [[DOT03:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP2]])
+; CHECK:         [[DOT04:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[TMP1]])
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[DOT01]])
@@ -274,8 +274,8 @@ define void @testorassume(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = or i1 [[XZ]], [[YZ]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[Z]])
-; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
-; CHECK:         [[DOT0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP1]])
+; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
+; CHECK:         [[DOT0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP1]])
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ]])
@@ -307,11 +307,11 @@ define void @test_and_one_unknown_cond(i32 %x, i1 %c1) {
 ; CHECK-LABEL: @test_and_one_unknown_cond(
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[A:%.*]] = and i1 [[C1:%.*]], [[C2]]
-; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A]])
-; CHECK:         [[A_1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A]])
-; CHECK:         [[C1_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[C1]])
-; CHECK:         [[C2_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[C2]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
+; CHECK:         [[A_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
+; CHECK:         [[C1_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C1]])
+; CHECK:         [[C2_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C2]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
 ; CHECK-NEXT:    br i1 [[A]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @bar(i32 [[X_0]])
@@ -349,11 +349,11 @@ define void @test_or_one_unknown_cond(i32 %x, i1 %c1) {
 ; CHECK-LABEL: @test_or_one_unknown_cond(
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[A:%.*]] = or i1 [[C1:%.*]], [[C2]]
-; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A]])
-; CHECK:         [[A_1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A]])
-; CHECK:         [[C1_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[C1]])
-; CHECK:         [[C2_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[C2]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
+; CHECK:         [[A_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
+; CHECK:         [[C1_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C1]])
+; CHECK:         [[C2_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C2]])
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
 ; CHECK-NEXT:    br i1 [[A]], label [[NOPE:%.*]], label [[BOTH_INVERTED:%.*]]
 ; CHECK:       both_inverted:
 ; CHECK-NEXT:    call void @bar(i32 [[X_0]])
@@ -391,12 +391,12 @@ define void @test_and_chain(i1 %a, i1 %b, i1 %c) {
 ; CHECK-LABEL: @test_and_chain(
 ; CHECK-NEXT:    [[AND1:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[AND1]], [[C:%.*]]
-; CHECK:         [[AND2_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[AND2]])
-; CHECK:         [[AND2_1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[AND2]])
-; CHECK:         [[AND1_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[AND1]])
-; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A]])
-; CHECK:         [[B_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[B]])
-; CHECK:         [[C_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[C]])
+; CHECK:         [[AND2_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND2]])
+; CHECK:         [[AND2_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND2]])
+; CHECK:         [[AND1_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND1]])
+; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
+; CHECK:         [[B_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[B]])
+; CHECK:         [[C_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C]])
 ; CHECK-NEXT:    br i1 [[AND2]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A_0]])
@@ -438,12 +438,12 @@ define void @test_or_chain(i1 %a, i1 %b, i1 %c) {
 ; CHECK-LABEL: @test_or_chain(
 ; CHECK-NEXT:    [[OR1:%.*]] = or i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[OR2:%.*]] = or i1 [[OR1]], [[C:%.*]]
-; CHECK:         [[OR2_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[OR2]])
-; CHECK:         [[OR2_1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[OR2]])
-; CHECK:         [[OR1_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[OR1]])
-; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A]])
-; CHECK:         [[B_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[B]])
-; CHECK:         [[C_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[C]])
+; CHECK:         [[OR2_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR2]])
+; CHECK:         [[OR2_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR2]])
+; CHECK:         [[OR1_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR1]])
+; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
+; CHECK:         [[B_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[B]])
+; CHECK:         [[C_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C]])
 ; CHECK-NEXT:    br i1 [[OR2]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A]])
@@ -485,10 +485,10 @@ define void @test_and_or_mixed(i1 %a, i1 %b, i1 %c) {
 ; CHECK-LABEL: @test_and_or_mixed(
 ; CHECK-NEXT:    [[OR:%.*]] = or i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i1 [[OR]], [[C:%.*]]
-; CHECK:         [[AND_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[AND]])
-; CHECK:         [[AND_1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[AND]])
-; CHECK:         [[OR_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[OR]])
-; CHECK:         [[C_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[C]])
+; CHECK:         [[AND_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND]])
+; CHECK:         [[AND_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND]])
+; CHECK:         [[OR_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR]])
+; CHECK:         [[C_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C]])
 ; CHECK-NEXT:    br i1 [[AND]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A]])
@@ -542,15 +542,15 @@ define void @test_deep_and_chain(i1 %a1) {
 ; CHECK-NEXT:    [[A13:%.*]] = and i1 [[A12]], true
 ; CHECK-NEXT:    [[A14:%.*]] = and i1 [[A13]], true
 ; CHECK-NEXT:    [[A15:%.*]] = and i1 [[A14]], true
-; CHECK:         [[A15_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A15]])
-; CHECK:         [[A15_1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A15]])
-; CHECK:         [[A14_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A14]])
-; CHECK:         [[A13_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A13]])
-; CHECK:         [[A12_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A12]])
-; CHECK:         [[A11_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A11]])
-; CHECK:         [[A10_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A10]])
-; CHECK:         [[A9_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A9]])
-; CHECK:         [[A8_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A8]])
+; CHECK:         [[A15_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
+; CHECK:         [[A15_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
+; CHECK:         [[A14_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A14]])
+; CHECK:         [[A13_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A13]])
+; CHECK:         [[A12_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A12]])
+; CHECK:         [[A11_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A11]])
+; CHECK:         [[A10_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A10]])
+; CHECK:         [[A9_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A9]])
+; CHECK:         [[A8_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A8]])
 ; CHECK-NEXT:    br i1 [[A15]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A1]])
@@ -656,15 +656,15 @@ define void @test_deep_and_tree(i1 %a1) {
 ; CHECK-NEXT:    [[A13:%.*]] = and i1 [[A12]], [[A12]]
 ; CHECK-NEXT:    [[A14:%.*]] = and i1 [[A13]], [[A13]]
 ; CHECK-NEXT:    [[A15:%.*]] = and i1 [[A14]], [[A14]]
-; CHECK:         [[A15_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A15]])
-; CHECK:         [[A15_1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A15]])
-; CHECK:         [[A14_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A14]])
-; CHECK:         [[A13_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A13]])
-; CHECK:         [[A12_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A12]])
-; CHECK:         [[A11_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A11]])
-; CHECK:         [[A10_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A10]])
-; CHECK:         [[A9_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A9]])
-; CHECK:         [[A8_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A8]])
+; CHECK:         [[A15_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
+; CHECK:         [[A15_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
+; CHECK:         [[A14_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A14]])
+; CHECK:         [[A13_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A13]])
+; CHECK:         [[A12_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A12]])
+; CHECK:         [[A11_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A11]])
+; CHECK:         [[A10_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A10]])
+; CHECK:         [[A9_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A9]])
+; CHECK:         [[A8_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A8]])
 ; CHECK-NEXT:    br i1 [[A15]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A1]])
@@ -770,15 +770,15 @@ define void @test_deep_or_tree(i1 %a1) {
 ; CHECK-NEXT:    [[A13:%.*]] = or i1 [[A12]], [[A12]]
 ; CHECK-NEXT:    [[A14:%.*]] = or i1 [[A13]], [[A13]]
 ; CHECK-NEXT:    [[A15:%.*]] = or i1 [[A14]], [[A14]]
-; CHECK:         [[A15_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A15]])
-; CHECK:         [[A15_1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A15]])
-; CHECK:         [[A14_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A14]])
-; CHECK:         [[A13_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A13]])
-; CHECK:         [[A12_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A12]])
-; CHECK:         [[A11_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A11]])
-; CHECK:         [[A10_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A10]])
-; CHECK:         [[A9_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A9]])
-; CHECK:         [[A8_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A8]])
+; CHECK:         [[A15_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
+; CHECK:         [[A15_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
+; CHECK:         [[A14_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A14]])
+; CHECK:         [[A13_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A13]])
+; CHECK:         [[A12_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A12]])
+; CHECK:         [[A11_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A11]])
+; CHECK:         [[A10_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A10]])
+; CHECK:         [[A9_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A9]])
+; CHECK:         [[A8_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A8]])
 ; CHECK-NEXT:    br i1 [[A15]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A1]])
@@ -873,11 +873,11 @@ define void @test_assume_and_chain(i1 %a, i1 %b, i1 %c) {
 ; CHECK-NEXT:    [[AND1:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[AND1]], [[C:%.*]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[AND2]])
-; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[C]])
-; CHECK:         [[TMP2:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[B]])
-; CHECK:         [[TMP3:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A]])
-; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[AND1]])
-; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[AND2]])
+; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C]])
+; CHECK:         [[TMP2:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[B]])
+; CHECK:         [[TMP3:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
+; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND1]])
+; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND2]])
 ; CHECK-NEXT:    call void @foo(i1 [[TMP3]])
 ; CHECK-NEXT:    call void @foo(i1 [[TMP2]])
 ; CHECK-NEXT:    call void @foo(i1 [[TMP1]])
@@ -901,7 +901,7 @@ define void @test_assume_or_chain(i1 %a, i1 %b, i1 %c) {
 ; CHECK-NEXT:    [[OR1:%.*]] = or i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[OR2:%.*]] = or i1 [[OR1]], [[C:%.*]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[OR2]])
-; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[OR2]])
+; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR2]])
 ; CHECK-NEXT:    call void @foo(i1 [[A]])
 ; CHECK-NEXT:    call void @foo(i1 [[B]])
 ; CHECK-NEXT:    call void @foo(i1 [[C]])
@@ -937,14 +937,14 @@ define void @test_assume_deep_and_tree(i1 %a1) {
 ; CHECK-NEXT:    [[A14:%.*]] = and i1 [[A13]], [[A13]]
 ; CHECK-NEXT:    [[A15:%.*]] = and i1 [[A14]], [[A14]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[A15]])
-; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A8]])
-; CHECK:         [[TMP2:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A9]])
-; CHECK:         [[TMP3:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A10]])
-; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A11]])
-; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A12]])
-; CHECK:         [[TMP6:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A13]])
-; CHECK:         [[TMP7:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A14]])
-; CHECK:         [[TMP8:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[A15]])
+; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A8]])
+; CHECK:         [[TMP2:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A9]])
+; CHECK:         [[TMP3:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A10]])
+; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A11]])
+; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A12]])
+; CHECK:         [[TMP6:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A13]])
+; CHECK:         [[TMP7:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A14]])
+; CHECK:         [[TMP8:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
 ; CHECK-NEXT:    call void @foo(i1 [[A1]])
 ; CHECK-NEXT:    call void @foo(i1 [[A2]])
 ; CHECK-NEXT:    call void @foo(i1 [[A3]])
diff --git a/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll b/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
index d1e0f358fc9fc1fb6b4f8cbc8cfea92fb1c7138d..13575e7caa66aeb3be72621a834e9038c63e10cc 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
@@ -8,12 +8,12 @@
 ; CHECK-LABEL: bb:
 ; CHECK: Has predicate info
 ; CHECK: branch predicate info { TrueEdge: 1 Comparison:  %cmp1 = icmp ne %0* %arg, null Edge: [label %bb,label %bb1], RenamedOp: %arg }
-; CHECK-NEXT:  %arg.0 = call %0* @llvm.ssa.copy.{{.+}}(%0* %arg)
+; CHECK-NEXT:  %arg.0 = call %0* @llvm.ssa.copy.p0s_s.{{.+}}(%0* %arg)
 
 ; CHECK-LABEL: bb1:
 ; CHECK: Has predicate info
 ; CHECK-NEXT: branch predicate info { TrueEdge: 0 Comparison:  %cmp2 = icmp ne %1* null, %tmp Edge: [label %bb1,label %bb3], RenamedOp: %tmp }
-; CHECK-NEXT: %tmp.0 = call %1* @llvm.ssa.copy.{{.+}}(%1* %tmp)
+; CHECK-NEXT: %tmp.0 = call %1* @llvm.ssa.copy.p0s_s.{{.+}}(%1* %tmp)
 
 define void @f0(%0* %arg, %1* %tmp) {
 bb:
diff --git a/llvm/test/Transforms/Util/annotation-remarks-dbg-info.ll b/llvm/test/Transforms/Util/annotation-remarks-dbg-info.ll
new file mode 100644
index 0000000000000000000000000000000000000000..25e5729dd70d5d18e805b741c5df630143da87a1
--- /dev/null
+++ b/llvm/test/Transforms/Util/annotation-remarks-dbg-info.ll
@@ -0,0 +1,78 @@
+; RUN: opt -annotation-remarks -pass-remarks-missed='annotation-remarks' -disable-output -pass-remarks-output=%t.opt.yaml %s
+; RUN: FileCheck --input-file=%t.opt.yaml %s
+; RUN: opt -passes='annotation-remarks' -pass-remarks-missed='annotation-remarks' -disable-output -pass-remarks-output=%t.opt.yaml %s
+; RUN: FileCheck --input-file=%t.opt.yaml %s
+
+; Make sure a suitable location is used for the function start when emitting
+; the annotation summary remarks.
+
+; CHECK:      --- !Analysis
+; CHECK-NEXT: Pass:            annotation-remarks
+; CHECK-NEXT: Name:            AnnotationSummary
+; CHECK-NEXT: DebugLoc: { File: test.c, Line: 10, Column: 0 }
+; CHECK-NEXT: Function:        test1
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          'Annotated '
+; CHECK-NEXT:   - count:           '4'
+; CHECK-NEXT:   - String:          ' instructions with '
+; CHECK-NEXT:   - type:            _remarks1
+; CHECK-NEXT: ...
+; CHECK-NEXT: --- !Analysis
+; CHECK-NEXT: Pass:            annotation-remarks
+; CHECK-NEXT: Name:            AnnotationSummary
+; CHECK-NEXT: DebugLoc: { File: test.c, Line: 10, Column: 0 }
+; CHECK-NEXT: Function:        test1
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          'Annotated '
+; CHECK-NEXT:   - count:           '3'
+; CHECK-NEXT:   - String:          ' instructions with '
+; CHECK-NEXT:   - type:            _remarks2
+; CHECK-NEXT: ...
+; CHECK-NEXT: --- !Analysis
+; CHECK-NEXT: Pass:            annotation-remarks
+; CHECK-NEXT: Name:            AnnotationSummary
+; CHECK-NEXT: DebugLoc: { File: test.c, Line: 20, Column: 0 }
+; CHECK-NEXT: Function:        test2
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          'Annotated '
+; CHECK-NEXT:   - count:           '2'
+; CHECK-NEXT:   - String:          ' instructions with '
+; CHECK-NEXT:   - type:            _remarks1
+; CHECK-NEXT: ...
+
+define void @test1(float* %a) !dbg !7 {
+entry:
+  %a.addr = alloca float*, align 8, !dbg !16, !annotation !5
+  store float* null, float** %a.addr, align 8, !annotation !6
+  store float* %a, float** %a.addr, align 8, !annotation !5
+  ret void, !annotation !5
+}
+
+define void @test2(float* %a) !dbg !17 {
+entry:
+  %a.addr = alloca float*, align 8, !annotation !6
+  ret void, !dbg !18, !annotation !6
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: "/test")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"_remarks1", !"_remarks2"}
+!6 = !{!"_remarks1"}
+!7 = distinct !DISubprogram(name: "test1", scope: !1, file: !1, line: 11, type: !8, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10, !10, !13}
+!10 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 32, align: 32)
+!12 = !DIBasicType(name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!13 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !{!15}
+!15 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!16 = !DILocation(line: 400, column: 3, scope: !7)
+!17 = distinct !DISubprogram(name: "test2", scope: !1, file: !1, line: 21, type: !8, scopeLine: 20, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14)
+!18 = !DILocation(line: 200, column: 3, scope: !17)
diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ceb18addace4ef214e249251c9d02c06e97d22b2
--- /dev/null
+++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -strip-nonlinetable-debuginfo %s -o - |  FileCheck %s
+; RUN: opt -S -strip-debug %s -o - |  FileCheck %s
+; int *get() { return new int[256]; }
+; ModuleID = '/tmp/heapallocsite.cpp'
+source_filename = "/tmp/heapallocsite.cpp"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx11.0.0"
+
+; Function Attrs: noinline optnone ssp uwtable mustprogress
+define dso_local i32* @_Z3getv() #0 !dbg !8 {
+entry:
+; CHECK-LABEL: entry:
+; CHECK-NOT: !heapallocsite
+  %call = call noalias nonnull i8* @_Znam(i64 1024) #2, !dbg !14, !heapallocsite !13
+  %0 = bitcast i8* %call to i32*, !dbg !14
+  ret i32* %0, !dbg !15
+}
+
+; Function Attrs: nobuiltin allocsize(0)
+declare nonnull i8* @_Znam(i64) #1
+
+attributes #0 = { noinline optnone ssp uwtable mustprogress }
+attributes #1 = { nobuiltin allocsize(0) "frame-pointer"="all" }
+attributes #2 = { builtin allocsize(0) }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+; CHECK-LABEL: !llvm.ident
+; CHECK-NOT: !DIBasicType(name: "int"
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 13.0.0 (git@github.com:llvm/llvm-project 6d4ce49dae17715de502acbd50ab4c9b3c18215b)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None, sysroot: "/")
+!1 = !DIFile(filename: "/tmp/heapallocsite.cpp", directory: "/Volumes/Data/llvm-project")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 13.0.0 (git@github.com:llvm/llvm-project 6d4ce49dae17715de502acbd50ab4c9b3c18215b)"}
+!8 = distinct !DISubprogram(name: "get", linkageName: "_Z3getv", scope: !9, file: !9, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!9 = !DIFile(filename: "/tmp/heapallocsite.cpp", directory: "")
+!10 = !DISubroutineType(types: !11)
+!11 = !{!12}
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !DILocation(line: 1, column: 21, scope: !8)
+!15 = !DILocation(line: 1, column: 14, scope: !8)
diff --git a/llvm/test/Unit/lit.cfg.py b/llvm/test/Unit/lit.cfg.py
index 3198ab2c9539b5c696404bc7d73b2b72e8355176..3a5c40dc14da63b3d62ca4864b0b6507aff27536 100644
--- a/llvm/test/Unit/lit.cfg.py
+++ b/llvm/test/Unit/lit.cfg.py
@@ -13,9 +13,6 @@ config.name = 'LLVM-Unit'
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = []
 
-# is_early; Request to run this suite early.
-config.is_early = True
-
 # test_source_root: The root path where tests are located.
 # test_exec_root: The root path where tests should be run.
 config.test_exec_root = os.path.join(config.llvm_obj_root, 'unittests')
diff --git a/llvm/test/Verifier/align.ll b/llvm/test/Verifier/align.ll
index 38ce3772e765257a26003fa7c92a976e64d4eed8..1f5c8da7654a44133ed88bcff562c68190999ae8 100644
--- a/llvm/test/Verifier/align.ll
+++ b/llvm/test/Verifier/align.ll
@@ -1,12 +1,12 @@
 ; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
 
-; CHECK: Wrong types for attribute: inalloca nest noalias nocapture nonnull readnone readonly byref(i32) byval(i32) preallocated(i32) sret(i32) align 1 dereferenceable(1) dereferenceable_or_null(1)
+; CHECK: Wrong types for attribute: nest noalias nocapture nonnull readnone readonly byref(i32) byval(i32) inalloca(i32) preallocated(i32) sret(i32) align 1 dereferenceable(1) dereferenceable_or_null(1)
 ; CHECK-NEXT: @align_non_pointer1
 define void @align_non_pointer1(i32 align 4 %a) {
   ret void
 }
 
-; CHECK: Wrong types for attribute: inalloca nest noalias nocapture noundef nonnull readnone readonly signext zeroext byref(void) byval(void) preallocated(void) sret(void) align 1 dereferenceable(1) dereferenceable_or_null(1)
+; CHECK: Wrong types for attribute: nest noalias nocapture noundef nonnull readnone readonly signext zeroext byref(void) byval(void) inalloca(void) preallocated(void) sret(void) align 1 dereferenceable(1) dereferenceable_or_null(1)
 ; CHECK-NEXT: @align_non_pointer2
 define align 4 void @align_non_pointer2(i32 %a) {
   ret void
diff --git a/llvm/test/Verifier/amdgpu-cc.ll b/llvm/test/Verifier/amdgpu-cc.ll
index b9b4c5027a1986da05d66b3a2f78e60a51cd1807..61f1c68cd5b3d4e63262cda70f835652db88aedf 100644
--- a/llvm/test/Verifier/amdgpu-cc.ll
+++ b/llvm/test/Verifier/amdgpu-cc.ll
@@ -118,7 +118,7 @@ define amdgpu_kernel void @preallocated_as0_cc_amdgpu_kernel(i32* preallocated(i
 
 ; CHECK: Calling convention disallows inalloca
 ; CHECK-NEXT: void (i32*)* @inalloca_as0_cc_amdgpu_kernel
-define amdgpu_kernel void @inalloca_as0_cc_amdgpu_kernel(i32* inalloca %ptr) {
+define amdgpu_kernel void @inalloca_as0_cc_amdgpu_kernel(i32* inalloca(i32) %ptr) {
   ret void
 }
 
diff --git a/llvm/test/Verifier/byref.ll b/llvm/test/Verifier/byref.ll
index 2f22ee37292e7106afe335102b0f54af03b9818e..d5921bf5b2614bdbc8949af99b74cfcbd09af6d3 100644
--- a/llvm/test/Verifier/byref.ll
+++ b/llvm/test/Verifier/byref.ll
@@ -28,7 +28,7 @@ define void @byref_byval(i32* byref(i32) byval(i32)) {
 
 ; CHECK: Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', 'byref', and 'sret' are incompatible!
 ; CHECK-NEXT: void (i32*)* @byref_inalloca
-define void @byref_inalloca(i32* byref(i32) inalloca) {
+define void @byref_inalloca(i32* byref(i32) inalloca(i32)) {
   ret void
 }
 
@@ -56,7 +56,7 @@ define void @byref_nest(i32* byref(i32) nest) {
   ret void
 }
 
-; CHECK: Wrong types for attribute: inalloca nest noalias nocapture nonnull readnone readonly byref(i32) byval(i32) preallocated(i32) sret(i32) align 1 dereferenceable(1) dereferenceable_or_null(1)
+; CHECK: Wrong types for attribute: nest noalias nocapture nonnull readnone readonly byref(i32) byval(i32) inalloca(i32) preallocated(i32) sret(i32) align 1 dereferenceable(1) dereferenceable_or_null(1)
 ; CHECK-NEXT: void (i32)* @byref_non_pointer
 define void @byref_non_pointer(i32 byref(i32)) {
   ret void
diff --git a/llvm/test/Verifier/byval-1.ll b/llvm/test/Verifier/byval-1.ll
index e2b4519b17cb98b1b4518d4665f83a372951cb74..6344371bba5e04ffa7a1f73b4f362954428b62a0 100644
--- a/llvm/test/Verifier/byval-1.ll
+++ b/llvm/test/Verifier/byval-1.ll
@@ -1,5 +1,5 @@
 ; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
 
-; CHECK: Wrong types for attribute: inalloca nest noalias nocapture nonnull readnone readonly byref(i32) byval(i32) preallocated(i32) sret(i32) align 1 dereferenceable(1) dereferenceable_or_null(1)
+; CHECK: Wrong types for attribute: nest noalias nocapture nonnull readnone readonly byref(i32) byval(i32) inalloca(i32) preallocated(i32) sret(i32) align 1 dereferenceable(1) dereferenceable_or_null(1)
 ; CHECK-NEXT: void (i32)* @h
 declare void @h(i32 byval(i32) %num)
diff --git a/llvm/test/Verifier/inalloca-vararg.ll b/llvm/test/Verifier/inalloca-vararg.ll
index 428f89ec88f10238279b14dc0a45d88ebb8cd933..de7622b638d88449ecedd8112afab1f98f79e7da 100644
--- a/llvm/test/Verifier/inalloca-vararg.ll
+++ b/llvm/test/Verifier/inalloca-vararg.ll
@@ -3,7 +3,7 @@
 declare void @h(i32, ...)
 define void @i() {
   %args = alloca inalloca i32
-  call void (i32, ...) @h(i32 1, i32* inalloca %args, i32 3)
+  call void (i32, ...) @h(i32 1, i32* inalloca(i32) %args, i32 3)
 ; CHECK: inalloca isn't on the last argument!
   ret void
 }
diff --git a/llvm/test/Verifier/inalloca1.ll b/llvm/test/Verifier/inalloca1.ll
index 7ee2cba5ac1749d75a45cf38aaf52853f6f1d1ec..76da66adc7983c77b36e900ae5cab87d07b6f9dd 100644
--- a/llvm/test/Verifier/inalloca1.ll
+++ b/llvm/test/Verifier/inalloca1.ll
@@ -1,22 +1,34 @@
 ; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
 
-declare void @a(i64* byval(i64) inalloca %p)
+declare void @a(i64* byval(i64) inalloca(i64) %p)
 ; CHECK: Attributes {{.*}} are incompatible
 
-declare void @b(i64* inreg inalloca %p)
+declare void @b(i64* inreg inalloca(i64) %p)
 ; CHECK: Attributes {{.*}} are incompatible
 
-declare void @c(i64* sret(i64) inalloca %p)
+declare void @c(i64* sret(i64) inalloca(i64) %p)
 ; CHECK: Attributes {{.*}} are incompatible
 
-declare void @d(i64* nest inalloca %p)
+declare void @d(i64* nest inalloca(i64) %p)
 ; CHECK: Attributes {{.*}} are incompatible
 
-declare void @e(i64* readonly inalloca %p)
+declare void @e(i64* readonly inalloca(i64) %p)
 ; CHECK: Attributes {{.*}} are incompatible
 
-declare void @f(void ()* inalloca %p)
+declare void @f(void ()* inalloca(void()) %p)
 ; CHECK: do not support unsized types
 
-declare void @g(i32* inalloca %p, i32 %p2)
+declare void @g(i32* inalloca(i32) %p, i32 %p2)
 ; CHECK: inalloca isn't on the last parameter!
+
+; CHECK: Attribute 'inalloca' type does not match parameter!
+; CHECK-NEXT: void (i32*)* @inalloca_mismatched_pointee_type0
+define void @inalloca_mismatched_pointee_type0(i32* inalloca(i8)) {
+  ret void
+}
+
+; CHECK: Wrong types for attribute:
+; CHECK-NEXT: void (i8)* @inalloca_not_pointer
+define void @inalloca_not_pointer(i8 byref(i8)) {
+  ret void
+}
diff --git a/llvm/test/Verifier/inalloca2.ll b/llvm/test/Verifier/inalloca2.ll
index 12a454999285737b3166fe94fe462d579e8d7377..21fc2517cd0a7b8f8df77ee38c3fed8b5c62c2ec 100644
--- a/llvm/test/Verifier/inalloca2.ll
+++ b/llvm/test/Verifier/inalloca2.ll
@@ -2,21 +2,21 @@
 ; doesn't reject it.
 ; RUN: llvm-as %s -o /dev/null
 
-declare void @doit(i64* inalloca %a)
+declare void @doit(i64* inalloca(i64) %a)
 
 define void @a() {
 entry:
   %a = alloca inalloca [2 x i32]
   %b = bitcast [2 x i32]* %a to i64*
-  call void @doit(i64* inalloca %b)
+  call void @doit(i64* inalloca(i64) %b)
   ret void
 }
 
 define void @b() {
 entry:
   %a = alloca inalloca i64
-  call void @doit(i64* inalloca %a)
-  call void @doit(i64* inalloca %a)
+  call void @doit(i64* inalloca(i64) %a)
+  call void @doit(i64* inalloca(i64) %a)
   ret void
 }
 
@@ -34,6 +34,6 @@ else:
 
 call:
   %args = phi i64* [ %a, %if ], [ %b, %else ]
-  call void @doit(i64* inalloca %args)
+  call void @doit(i64* inalloca(i64) %args)
   ret void
 }
diff --git a/llvm/test/Verifier/inalloca3.ll b/llvm/test/Verifier/inalloca3.ll
index c09ce100849b550009c17da48b13a66a263c2f4c..28cdbfef97857b67de3f65e61b55dbc70b523159 100644
--- a/llvm/test/Verifier/inalloca3.ll
+++ b/llvm/test/Verifier/inalloca3.ll
@@ -1,13 +1,13 @@
 ; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
 
 
-declare void @doit(i64* inalloca %a)
+declare void @doit(i64* inalloca(i64) %a)
 
 define void @a() {
 entry:
   %a = alloca [2 x i32]
   %b = bitcast [2 x i32]* %a to i64*
-  call void @doit(i64* inalloca %b)
+  call void @doit(i64* inalloca(i64) %b)
 ; CHECK: inalloca argument for call has mismatched alloca
   ret void
 }
diff --git a/llvm/test/Verifier/noundef.ll b/llvm/test/Verifier/noundef.ll
index 7b199cd6d2dee5701135e9933257ff8b430db6cf..2ece2dd1a9ac1ce48f7ca0fe38cd81b9e728a7f3 100644
--- a/llvm/test/Verifier/noundef.ll
+++ b/llvm/test/Verifier/noundef.ll
@@ -1,6 +1,6 @@
 ; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
 
-; CHECK: Wrong types for attribute: inalloca nest noalias nocapture noundef nonnull readnone readonly signext zeroext byref(void) byval(void) preallocated(void) sret(void) align 1 dereferenceable(1) dereferenceable_or_null(1)
+; CHECK: Wrong types for attribute: nest noalias nocapture noundef nonnull readnone readonly signext zeroext byref(void) byval(void) inalloca(void) preallocated(void) sret(void) align 1 dereferenceable(1) dereferenceable_or_null(1)
 ; CHECK-NEXT: @noundef_void
 define noundef void @noundef_void() {
   ret void
diff --git a/llvm/test/Verifier/stepvector-intrinsic.ll b/llvm/test/Verifier/stepvector-intrinsic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f6852dd75e7c7e514dbdc15f05dc767f8b66e296
--- /dev/null
+++ b/llvm/test/Verifier/stepvector-intrinsic.ll
@@ -0,0 +1,29 @@
+; RUN: not opt -S -verify < %s 2>&1 | FileCheck %s
+
+; Reject stepvector intrinsics that return a scalar
+
+define i32 @stepvector_i32() {
+; CHECK: Intrinsic has incorrect return type!
+  %1 = call i32 @llvm.experimental.stepvector.i32()
+  ret i32 %1
+}
+
+; Reject vectors with non-integer elements
+
+define <vscale x 4 x float> @stepvector_float() {
+; CHECK: experimental_stepvector only supported for vectors of integers with a bitwidth of at least 8
+  %1 = call <vscale x 4 x float> @llvm.experimental.stepvector.nxv4f32()
+  ret <vscale x 4 x float> %1
+}
+
+; Reject vectors of integers less than 8 bits in width
+
+define <vscale x 16 x i1> @stepvector_i1() {
+; CHECK: experimental_stepvector only supported for vectors of integers with a bitwidth of at least 8
+  %1 = call <vscale x 16 x i1> @llvm.experimental.stepvector.nxv16i1()
+  ret <vscale x 16 x i1> %1
+}
+
+declare i32 @llvm.experimental.stepvector.i32()
+declare <vscale x 4 x float> @llvm.experimental.stepvector.nxv4f32()
+declare <vscale x 16 x i1> @llvm.experimental.stepvector.nxv16i1()
diff --git a/llvm/test/Verifier/vscale_range.ll b/llvm/test/Verifier/vscale_range.ll
new file mode 100644
index 0000000000000000000000000000000000000000..58c1bc0127619dfe9f213c65038f679d988ddba4
--- /dev/null
+++ b/llvm/test/Verifier/vscale_range.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: 'vscale_range' minimum cannot be greater than maximum
+declare i8* @b(i32*) vscale_range(8, 1)
diff --git a/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/bar.o b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/bar.o
new file mode 100644
index 0000000000000000000000000000000000000000..26d795f44e1acdb0d9438f658a565a6de5c52365
Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/bar.o differ
diff --git a/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/foo.o b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/foo.o
new file mode 100644
index 0000000000000000000000000000000000000000..b7dcab5c27ba5ce31a4cdf6cd7519d5e4f73af17
Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/foo.o differ
diff --git a/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/foobar.dylib b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/foobar.dylib
new file mode 100755
index 0000000000000000000000000000000000000000..75b22f37723793126c9909319c67d05b6cae1964
Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/foobar.dylib differ
diff --git a/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/lto/0.x86_64.thinlto.o b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/lto/0.x86_64.thinlto.o
new file mode 100644
index 0000000000000000000000000000000000000000..119b9268e539debb14b15d3cd107969bf58b7952
Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/lto/0.x86_64.thinlto.o differ
diff --git a/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/lto/1.x86_64.thinlto.o b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/lto/1.x86_64.thinlto.o
new file mode 100644
index 0000000000000000000000000000000000000000..1c207a7e8515c14e06edcddea600e734e2c3ea78
Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/private/tmp/thinlto/lto/1.x86_64.thinlto.o differ
diff --git a/llvm/test/tools/dsymutil/X86/thinlto.test b/llvm/test/tools/dsymutil/X86/thinlto.test
new file mode 100644
index 0000000000000000000000000000000000000000..ebd4068a5c127515199410e61850e1a863853c29
--- /dev/null
+++ b/llvm/test/tools/dsymutil/X86/thinlto.test
@@ -0,0 +1,24 @@
+$ cat foo.cpp
+struct nontrivial {
+  nontrivial() { }
+};
+
+void function2()
+{
+  static const nontrivial magic_static;
+}
+
+$ cat bar.cpp
+void function2();
+
+void function1()
+{
+  function2();
+}
+
+$ xcrun clang++ -g -flto=thin -O2 foo.cpp bar.cpp -c
+$ xcrun clang++ -flto=thin foo.o bar.o -Xlinker -object_path_lto -Xlinker lto -shared -o foobar.dylib
+
+RUN: dsymutil -oso-prepend-path %p/../Inputs %p/../Inputs/private/tmp/thinlto/foobar.dylib -o %t.dSYM 2>&1 | FileCheck %s --allow-empty
+CHECK-NOT: could not find object file symbol for symbol __ZZ9function2vE12magic_static
+CHECK-NOT: could not find object file symbol for symbol __ZGVZ9function2vE12magic_static
diff --git a/llvm/test/tools/llvm-dwarfdump/XCOFF/Inputs/basic32.o b/llvm/test/tools/llvm-dwarfdump/XCOFF/Inputs/basic32.o
new file mode 100644
index 0000000000000000000000000000000000000000..3ab6a54b803d94593b775b9f69ab699447b2f157
Binary files /dev/null and b/llvm/test/tools/llvm-dwarfdump/XCOFF/Inputs/basic32.o differ
diff --git a/llvm/test/tools/llvm-dwarfdump/XCOFF/Inputs/basic64.o b/llvm/test/tools/llvm-dwarfdump/XCOFF/Inputs/basic64.o
new file mode 100644
index 0000000000000000000000000000000000000000..a7d2625a99f2d39bfb0164cee3e96a6858e2001b
Binary files /dev/null and b/llvm/test/tools/llvm-dwarfdump/XCOFF/Inputs/basic64.o differ
diff --git a/llvm/test/tools/llvm-dwarfdump/XCOFF/basic.test b/llvm/test/tools/llvm-dwarfdump/XCOFF/basic.test
new file mode 100644
index 0000000000000000000000000000000000000000..f1ff35a57df36f1149ac30993d0c1dc75ab29d90
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/XCOFF/basic.test
@@ -0,0 +1,274 @@
+# RUN: llvm-dwarfdump --all %p/Inputs/basic32.o | FileCheck %s --check-prefix=DWARF32 
+# RUN: llvm-dwarfdump --all %p/Inputs/basic64.o | FileCheck %s --check-prefix=DWARF64
+
+## basic32.o Compiled with IBM XL C/C++ for AIX, V16.1.1.0
+## Compiler command: xlc basic.c -c -qdbgfmt=dwarf -g -q32 -o basic32.o
+##
+## basic64.o Compiled with IBM XL C/C++ for AIX, V16.1.1.0
+## Compiler command: xlc basic.c -c -qdbgfmt=dwarf -g -q64 -o basic64.o
+##
+## basic.c:
+## int main(void)
+## {
+##   return 0;
+## }
+
+# DWARF32:      .debug_abbrev contents:
+# DWARF32-NEXT:  Abbrev table for offset: 0x00000000
+# DWARF32-NEXT:  [1] DW_TAG_compile_unit DW_CHILDREN_yes
+# DWARF32-NEXT:          DW_AT_name      DW_FORM_string
+# DWARF32-NEXT:          DW_AT_stmt_list DW_FORM_data4
+# DWARF32-NEXT:          DW_AT_low_pc    DW_FORM_addr
+# DWARF32-NEXT:          DW_AT_high_pc   DW_FORM_addr
+# DWARF32-NEXT:          DW_AT_language  DW_FORM_data1
+# DWARF32-NEXT:          DW_AT_comp_dir  DW_FORM_string
+# DWARF32-NEXT:          DW_AT_producer  DW_FORM_string
+# DWARF32:       [2] DW_TAG_base_type    DW_CHILDREN_no
+# DWARF32-NEXT:          DW_AT_name      DW_FORM_string
+# DWARF32-NEXT:          DW_AT_byte_size DW_FORM_data1
+# DWARF32-NEXT:          DW_AT_encoding  DW_FORM_data1
+# DWARF32:       [3] DW_TAG_const_type   DW_CHILDREN_no
+# DWARF32-NEXT:          DW_AT_type      DW_FORM_ref4
+# DWARF32:       [4] DW_TAG_array_type   DW_CHILDREN_yes
+# DWARF32-NEXT:          DW_AT_sibling   DW_FORM_ref4
+# DWARF32-NEXT:          DW_AT_type      DW_FORM_ref4
+# DWARF32:       [5] DW_TAG_subrange_type        DW_CHILDREN_no
+# DWARF32-NEXT:          DW_AT_lower_bound       DW_FORM_data1
+# DWARF32-NEXT:          DW_AT_upper_bound       DW_FORM_data1
+# DWARF32:       [6] DW_TAG_subprogram   DW_CHILDREN_yes
+# DWARF32-NEXT:          DW_AT_name      DW_FORM_string
+# DWARF32-NEXT:          DW_AT_low_pc    DW_FORM_addr
+# DWARF32-NEXT:          DW_AT_high_pc   DW_FORM_addr
+# DWARF32-NEXT:          DW_AT_prototyped        DW_FORM_flag
+# DWARF32-NEXT:          DW_AT_calling_convention        DW_FORM_data1
+# DWARF32-NEXT:          DW_AT_decl_file DW_FORM_data1
+# DWARF32-NEXT:          DW_AT_decl_line DW_FORM_data1
+# DWARF32-NEXT:          DW_AT_external  DW_FORM_flag
+# DWARF32-NEXT:          DW_AT_frame_base        DW_FORM_block1
+# DWARF32-NEXT:          DW_AT_type      DW_FORM_ref4
+# DWARF32:       [7] DW_TAG_variable     DW_CHILDREN_no
+# DWARF32-NEXT:          DW_AT_location  DW_FORM_block1
+# DWARF32-NEXT:          DW_AT_name      DW_FORM_string
+# DWARF32-NEXT:          DW_AT_decl_file DW_FORM_data1
+# DWARF32-NEXT:          DW_AT_decl_line DW_FORM_data1
+# DWARF32-NEXT:          DW_AT_type      DW_FORM_ref4
+# DWARF32:       .debug_info contents:
+# DWARF32-NEXT:  0x00000000: Compile Unit: length = 0x00000094, format = DWARF32, version = 0x0003, abbr_offset = 0x0000, addr_size = 0x04 (next unit at 0x00000098)
+# DWARF32:       0x0000000b: DW_TAG_compile_unit
+# DWARF32:                DW_AT_name        ("basic.c")
+# DWARF32-NEXT:                DW_AT_stmt_list   (0x00000000)
+# DWARF32-NEXT:                DW_AT_low_pc      (0x00000000)
+# DWARF32-NEXT:                DW_AT_high_pc     (0x00000080)
+# DWARF32-NEXT:                DW_AT_language    (DW_LANG_C89)
+# DWARF32-NEXT:                DW_AT_comp_dir    ("/")
+# DWARF32-NEXT:                DW_AT_producer    ("IBM XL C for AIX, Version 16.1.1.0")
+# DWARF32:       0x00000046:   DW_TAG_base_type
+# DWARF32-NEXT:                  DW_AT_name      ("char")
+# DWARF32-NEXT:                  DW_AT_byte_size (0x01)
+# DWARF32-NEXT:                  DW_AT_encoding  (DW_ATE_unsigned_char)
+# DWARF32:       0x0000004e:   DW_TAG_base_type
+# DWARF32-NEXT:                  DW_AT_name      ("int")
+# DWARF32-NEXT:                  DW_AT_byte_size (0x04)
+# DWARF32-NEXT:                  DW_AT_encoding  (DW_ATE_signed)
+# DWARF32:       0x00000055:   DW_TAG_const_type
+# DWARF32-NEXT:                  DW_AT_type      (0x00000046 "char")
+# DWARF32:       0x0000005a:   DW_TAG_array_type
+# DWARF32-NEXT:                  DW_AT_sibling   (0x00000067)
+# DWARF32-NEXT:                  DW_AT_type      (0x00000055 "const char")
+# DWARF32:       0x00000063:     DW_TAG_subrange_type
+# DWARF32-NEXT:                    DW_AT_lower_bound     (0x00)
+# DWARF32-NEXT:                    DW_AT_upper_bound     (0x04)
+# DWARF32:       0x00000066:     NULL
+# DWARF32:       0x00000067:   DW_TAG_subprogram
+# DWARF32-NEXT:                  DW_AT_name      ("main")
+# DWARF32-NEXT:                  DW_AT_low_pc    (0x00000000)
+# DWARF32-NEXT:                  DW_AT_high_pc   (0x00000024)
+# DWARF32-NEXT:                  DW_AT_prototyped        (0x01)
+# DWARF32-NEXT:                  DW_AT_calling_convention        (DW_CC_program)
+# DWARF32-NEXT:                  DW_AT_decl_file ("/basic.c")
+# DWARF32-NEXT:                  DW_AT_decl_line (1)
+# DWARF32-NEXT:                  DW_AT_external  (0x01)
+# DWARF32-NEXT:                  DW_AT_frame_base        (DW_OP_reg1 R1)
+# DWARF32-NEXT:                  DW_AT_type      (0x0000004e "int")
+# DWARF32:       0x00000080:     DW_TAG_variable
+# DWARF32-NEXT:                    DW_AT_location        (DW_OP_addr 0x0)
+# DWARF32-NEXT:                    DW_AT_name    ("__func__")
+# DWARF32-NEXT:                    DW_AT_decl_file       ("/basic.c")
+# DWARF32-NEXT:                    DW_AT_decl_line       (0)
+# DWARF32-NEXT:                    DW_AT_type    (0x0000005a "const char[5]")
+# DWARF32:       0x00000096:     NULL
+# DWARF32:       0x00000097:   NULL
+# DWARF32:       .debug_line contents:
+# DWARF32-NEXT:  debug_line[0x00000000]
+# DWARF32-NEXT:  Line table prologue:
+# DWARF32-NEXT:      total_length: 0x00000042
+# DWARF32-NEXT:            format: DWARF32
+# DWARF32-NEXT:           version: 3
+# DWARF32-NEXT:   prologue_length: 0x0000001e
+# DWARF32-NEXT:   min_inst_length: 1
+# DWARF32-NEXT:   default_is_stmt: 1
+# DWARF32-NEXT:         line_base: -100
+# DWARF32-NEXT:        line_range: 220
+# DWARF32-NEXT:       opcode_base: 13
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_copy] = 0
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_advance_pc] = 1
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_advance_line] = 1
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_set_file] = 1
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_set_column] = 1
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_negate_stmt] = 0
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_set_basic_block] = 0
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_const_add_pc] = 0
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_fixed_advance_pc] = 1
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_set_prologue_end] = 0
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_set_epilogue_begin] = 0
+# DWARF32-NEXT:  standard_opcode_lengths[DW_LNS_set_isa] = 1
+# DWARF32-NEXT:  file_names[  1]:
+# DWARF32-NEXT:             name: "basic.c"
+# DWARF32-NEXT:        dir_index: 0
+# DWARF32-NEXT:         mod_time: 0x00000000
+# DWARF32-NEXT:           length: 0x00000000
+# DWARF32:       Address            Line   Column File   ISA Discriminator Flags
+# DWARF32-NEXT:  ------------------ ------ ------ ------ --- ------------- -------------
+# DWARF32-NEXT:  0x0000000000000000      1      0      1   0             0  is_stmt
+# DWARF32-NEXT:  0x0000000000000004      3      0      1   0             0  is_stmt
+# DWARF32-NEXT:  0x0000000000000008      4      0      1   0             0  is_stmt
+# DWARF32-NEXT:  0x0000000000000080      4      0      1   0             0  is_stmt end_sequence
+# DWARF32:      .debug_pubnames contents:
+# DWARF32-NEXT:  length = 0x00000017, format = DWARF32, version = 0x0002, unit_offset = 0x00000000, unit_size = 0x00000098
+# DWARF32-NEXT:  Offset     Name
+# DWARF32-NEXT:  0x00000067 "main"
+# DWARF32:       .debug_pubtypes contents:
+# DWARF32-NEXT:  length = 0x0000001f, format = DWARF32, version = 0x0002, unit_offset = 0x00000000, unit_size = 0x00000098
+# DWARF32-NEXT:  Offset     Name
+# DWARF32-NEXT:  0x0000004e "int"
+# DWARF32-NEXT:  0x00000046 "char"
+
+# DWARF64:       .debug_abbrev contents:
+# DWARF64-NEXT:  Abbrev table for offset: 0x00000000
+# DWARF64-NEXT:  [1] DW_TAG_compile_unit DW_CHILDREN_yes
+# DWARF64-NEXT:          DW_AT_name      DW_FORM_string
+# DWARF64-NEXT:          DW_AT_stmt_list DW_FORM_data8
+# DWARF64-NEXT:          DW_AT_low_pc    DW_FORM_addr
+# DWARF64-NEXT:          DW_AT_high_pc   DW_FORM_addr
+# DWARF64-NEXT:          DW_AT_language  DW_FORM_data1
+# DWARF64-NEXT:          DW_AT_comp_dir  DW_FORM_string
+# DWARF64-NEXT:          DW_AT_producer  DW_FORM_string
+# DWARF64:       [2] DW_TAG_base_type    DW_CHILDREN_no
+# DWARF64-NEXT:          DW_AT_name      DW_FORM_string
+# DWARF64-NEXT:          DW_AT_byte_size DW_FORM_data1
+# DWARF64-NEXT:          DW_AT_encoding  DW_FORM_data1
+# DWARF64:       [3] DW_TAG_const_type   DW_CHILDREN_no
+# DWARF64-NEXT:          DW_AT_type      DW_FORM_ref8
+# DWARF64:       [4] DW_TAG_array_type   DW_CHILDREN_yes
+# DWARF64-NEXT:          DW_AT_sibling   DW_FORM_ref8
+# DWARF64-NEXT:          DW_AT_type      DW_FORM_ref8
+# DWARF64:       [5] DW_TAG_subrange_type        DW_CHILDREN_no
+# DWARF64-NEXT:          DW_AT_lower_bound       DW_FORM_data1
+# DWARF64-NEXT:          DW_AT_upper_bound       DW_FORM_data1
+# DWARF64:       [6] DW_TAG_subprogram   DW_CHILDREN_yes
+# DWARF64-NEXT:          DW_AT_name      DW_FORM_string
+# DWARF64-NEXT:          DW_AT_low_pc    DW_FORM_addr
+# DWARF64-NEXT:          DW_AT_high_pc   DW_FORM_addr
+# DWARF64-NEXT:          DW_AT_prototyped        DW_FORM_flag
+# DWARF64-NEXT:          DW_AT_calling_convention        DW_FORM_data1
+# DWARF64-NEXT:          DW_AT_decl_file DW_FORM_data1
+# DWARF64-NEXT:          DW_AT_decl_line DW_FORM_data1
+# DWARF64-NEXT:          DW_AT_external  DW_FORM_flag
+# DWARF64-NEXT:          DW_AT_frame_base        DW_FORM_block1
+# DWARF64-NEXT:          DW_AT_type      DW_FORM_ref8
+# DWARF64:       [7] DW_TAG_variable     DW_CHILDREN_no
+# DWARF64-NEXT:          DW_AT_location  DW_FORM_block1
+# DWARF64-NEXT:          DW_AT_name      DW_FORM_string
+# DWARF64-NEXT:          DW_AT_decl_file DW_FORM_data1
+# DWARF64-NEXT:          DW_AT_decl_line DW_FORM_data1
+# DWARF64-NEXT:          DW_AT_type      DW_FORM_ref8
+# DWARF64:       .debug_info contents:
+# DWARF64-NEXT:  0x00000000: Compile Unit: length = 0x00000000000000c4, format = DWARF64, version = 0x0003, abbr_offset = 0x0000, addr_size = 0x08 (next unit at 0x000000d0)
+# DWARF64:       0x00000017: DW_TAG_compile_unit
+# DWARF64-NEXT:                DW_AT_name        ("basic.c")
+# DWARF64-NEXT:                DW_AT_stmt_list   (0x0000000000000000)
+# DWARF64-NEXT:                DW_AT_low_pc      (0x0000000000000000)
+# DWARF64-NEXT:                DW_AT_high_pc     (0x0000000000000080)
+# DWARF64-NEXT:                DW_AT_language    (DW_LANG_C89)
+# DWARF64-NEXT:                DW_AT_comp_dir    ("/")
+# DWARF64-NEXT:                DW_AT_producer    ("IBM XL C for AIX, Version 16.1.1.0")
+# DWARF64:       0x0000005e:   DW_TAG_base_type
+# DWARF64-NEXT:                  DW_AT_name      ("char")
+# DWARF64-NEXT:                  DW_AT_byte_size (0x01)
+# DWARF64-NEXT:                  DW_AT_encoding  (DW_ATE_unsigned_char)
+# DWARF64:       0x00000066:   DW_TAG_base_type
+# DWARF64-NEXT:                  DW_AT_name      ("int")
+# DWARF64-NEXT:                  DW_AT_byte_size (0x04)
+# DWARF64-NEXT:                  DW_AT_encoding  (DW_ATE_signed)
+# DWARF64:       0x0000006d:   DW_TAG_const_type
+# DWARF64-NEXT:                  DW_AT_type      (0x0000005e "char")
+# DWARF64:       0x00000076:   DW_TAG_array_type
+# DWARF64-NEXT:                  DW_AT_sibling   (0x0000008b)
+# DWARF64-NEXT:                  DW_AT_type      (0x0000006d "const char")
+# DWARF64:       0x00000087:     DW_TAG_subrange_type
+# DWARF64-NEXT:                    DW_AT_lower_bound     (0x00)
+# DWARF64-NEXT:                    DW_AT_upper_bound     (0x04)
+# DWARF64:       0x0000008a:     NULL
+# DWARF64:       0x0000008b:   DW_TAG_subprogram
+# DWARF64-NEXT:                  DW_AT_name      ("main")
+# DWARF64-NEXT:                  DW_AT_low_pc    (0x0000000000000000)
+# DWARF64-NEXT:                  DW_AT_high_pc   (0x0000000000000024)
+# DWARF64-NEXT:                  DW_AT_prototyped        (0x01)
+# DWARF64-NEXT:                  DW_AT_calling_convention        (DW_CC_program)
+# DWARF64-NEXT:                  DW_AT_decl_file ("/basic.c")
+# DWARF64-NEXT:                  DW_AT_decl_line (1)
+# DWARF64-NEXT:                  DW_AT_external  (0x01)
+# DWARF64-NEXT:                  DW_AT_frame_base        (DW_OP_reg1 X1)
+# DWARF64-NEXT:                  DW_AT_type      (0x00000066 "int")
+# DWARF64:       0x000000b0:     DW_TAG_variable
+# DWARF64-NEXT:                    DW_AT_location        (DW_OP_addr 0x0)
+# DWARF64-NEXT:                    DW_AT_name    ("__func__")
+# DWARF64-NEXT:                    DW_AT_decl_file       ("/basic.c")
+# DWARF64-NEXT:                    DW_AT_decl_line       (0)
+# DWARF64-NEXT:                    DW_AT_type    (0x00000076 "const char[5]")
+# DWARF64:       0x000000ce:     NULL
+# DWARF64:       0x000000cf:   NULL
+# DWARF64:       .debug_line contents:
+# DWARF64-NEXT:  debug_line[0x00000000]
+# DWARF64-NEXT:  Line table prologue:
+# DWARF64-NEXT:      total_length: 0x000000000000004a
+# DWARF64-NEXT:            format: DWARF64
+# DWARF64-NEXT:           version: 3
+# DWARF64-NEXT:   prologue_length: 0x000000000000001e
+# DWARF64-NEXT:   min_inst_length: 1
+# DWARF64-NEXT:   default_is_stmt: 1
+# DWARF64-NEXT:         line_base: -100
+# DWARF64-NEXT:        line_range: 220
+# DWARF64-NEXT:       opcode_base: 13
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_copy] = 0
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_advance_pc] = 1
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_advance_line] = 1
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_set_file] = 1
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_set_column] = 1
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_negate_stmt] = 0
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_set_basic_block] = 0
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_const_add_pc] = 0
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_fixed_advance_pc] = 1
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_set_prologue_end] = 0
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_set_epilogue_begin] = 0
+# DWARF64-NEXT:  standard_opcode_lengths[DW_LNS_set_isa] = 1
+# DWARF64-NEXT:  file_names[  1]:
+# DWARF64-NEXT:             name: "basic.c"
+# DWARF64-NEXT:        dir_index: 0
+# DWARF64-NEXT:         mod_time: 0x00000000
+# DWARF64-NEXT:           length: 0x00000000
+# DWARF64:       Address            Line   Column File   ISA Discriminator Flags
+# DWARF64-NEXT:  ------------------ ------ ------ ------ --- ------------- -------------
+# DWARF64-NEXT:  0x0000000000000000      1      0      1   0             0  is_stmt
+# DWARF64-NEXT:  0x0000000000000004      3      0      1   0             0  is_stmt
+# DWARF64-NEXT:  0x0000000000000008      4      0      1   0             0  is_stmt
+# DWARF64-NEXT:  0x0000000000000080      4      0      1   0             0  is_stmt end_sequence
+# DWARF64:       .debug_pubnames contents:
+# DWARF64-NEXT:  length = 0x0000000000000027, format = DWARF64, version = 0x0002, unit_offset = 0x0000000000000000, unit_size = 0x00000000000000d0
+# DWARF64-NEXT:  Offset     Name
+# DWARF64-NEXT:  0x000000000000008b "main"
+# DWARF64:       .debug_pubtypes contents:
+# DWARF64-NEXT:  length = 0x0000000000000033, format = DWARF64, version = 0x0002, unit_offset = 0x0000000000000000, unit_size = 0x00000000000000d0
+# DWARF64-NEXT:  Offset     Name
+# DWARF64-NEXT:  0x0000000000000066 "int"
+# DWARF64-NEXT:  0x000000000000005e "char"
diff --git a/llvm/test/tools/llvm-dwarfdump/XCOFF/lit.local.cfg b/llvm/test/tools/llvm-dwarfdump/XCOFF/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..091332439b186719ce70be1df6973b9bd6f14c51
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/XCOFF/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/tools/llvm-exegesis/X86/latency-IN16rr.s b/llvm/test/tools/llvm-exegesis/X86/latency-IN16rr.s
index fcdaf6a403415cbbb4fd2c422c655102cd62b216..c57b61a55d5f05de01af66f291f84d8a05ae916e 100644
--- a/llvm/test/tools/llvm-exegesis/X86/latency-IN16rr.s
+++ b/llvm/test/tools/llvm-exegesis/X86/latency-IN16rr.s
@@ -1,5 +1,8 @@
 # RUN: llvm-exegesis -mode=latency -opcode-name=IN16rr -repetition-mode=duplicate | FileCheck %s
 
+# FIXME: Sometimes fails with: 'unimplemented operand type'
+# ALLOW_RETRIES: 2
+
 CHECK:      ---
 CHECK-NEXT: mode: latency
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A53-carry-over.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A53-carry-over.s
new file mode 100644
index 0000000000000000000000000000000000000000..a5715b965210982463a9f2787daac335dc7bb496
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A53-carry-over.s
@@ -0,0 +1,83 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-a53 --timeline --iterations=1 < %s | FileCheck %s
+
+ldp  w3, w5, [x10], #4 // 2uop + 1uop carry over
+add  w10, w11, w12
+add  w13, w14, w15
+ldp  w7, w8, [x11] // 2uop, no carry over
+add  w16, w17, w18
+add  w19, w20, w21
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      6
+# CHECK-NEXT: Total Cycles:      8
+# CHECK-NEXT: Total uOps:        9
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.13
+# CHECK-NEXT: IPC:               0.75
+# CHECK-NEXT: Block RThroughput: 4.5
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  3      4     2.00    *                   ldp	w3, w5, [x10], #4
+# CHECK-NEXT:  1      3     0.50                        add	w10, w11, w12
+# CHECK-NEXT:  1      3     0.50                        add	w13, w14, w15
+# CHECK-NEXT:  2      4     2.00    *                   ldp	w7, w8, [x11]
+# CHECK-NEXT:  1      3     0.50                        add	w16, w17, w18
+# CHECK-NEXT:  1      3     0.50                        add	w19, w20, w21
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - A53UnitALU
+# CHECK-NEXT: [0.1] - A53UnitALU
+# CHECK-NEXT: [1]   - A53UnitB
+# CHECK-NEXT: [2]   - A53UnitDiv
+# CHECK-NEXT: [3]   - A53UnitFPALU
+# CHECK-NEXT: [4]   - A53UnitFPMDS
+# CHECK-NEXT: [5]   - A53UnitLdSt
+# CHECK-NEXT: [6]   - A53UnitMAC
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]
+# CHECK-NEXT: 2.00   2.00    -      -      -      -     4.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     ldp	w3, w5, [x10], #4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -     add	w10, w11, w12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -     add	w13, w14, w15
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     ldp	w7, w8, [x11]
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -     add	w16, w17, w18
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -     add	w19, w20, w21
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     01234567
+
+# CHECK:      [0,0]     DeeeE. .   ldp	w3, w5, [x10], #4
+# CHECK-NEXT: [0,1]     .DeeE. .   add	w10, w11, w12
+# CHECK-NEXT: [0,2]     . DeeE .   add	w13, w14, w15
+# CHECK-NEXT: [0,3]     .  DeeeE   ldp	w7, w8, [x11]
+# CHECK-NEXT: [0,4]     .   DeeE   add	w16, w17, w18
+# CHECK-NEXT: [0,5]     .   DeeE   add	w19, w20, w21
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     0.0    0.0    0.0       ldp	w3, w5, [x10], #4
+# CHECK-NEXT: 1.     1     0.0    0.0    0.0       add	w10, w11, w12
+# CHECK-NEXT: 2.     1     0.0    0.0    0.0       add	w13, w14, w15
+# CHECK-NEXT: 3.     1     0.0    0.0    0.0       ldp	w7, w8, [x11]
+# CHECK-NEXT: 4.     1     0.0    0.0    0.0       add	w16, w17, w18
+# CHECK-NEXT: 5.     1     0.0    0.0    0.0       add	w19, w20, w21
+# CHECK-NEXT:        1     0.0    0.0    0.0       <total>
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-add-sequence.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-add-sequence.s
index be817b755f77433df105fc95492d62df77254cac..dbcb3c53b22f0685949cd29b201e1a7863949dee 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-add-sequence.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-add-sequence.s
@@ -8,12 +8,12 @@ add      w1, w0, #4
 
 # CHECK:      Iterations:        2
 # CHECK-NEXT: Instructions:      8
-# CHECK-NEXT: Total Cycles:      10
+# CHECK-NEXT: Total Cycles:      9
 # CHECK-NEXT: Total uOps:        8
 
 # CHECK:      Dispatch Width:    2
-# CHECK-NEXT: uOps Per Cycle:    0.80
-# CHECK-NEXT: IPC:               0.80
+# CHECK-NEXT: uOps Per Cycle:    0.89
+# CHECK-NEXT: IPC:               0.89
 # CHECK-NEXT: Block RThroughput: 2.0
 
 # CHECK:      Instruction Info:
@@ -56,16 +56,16 @@ add      w1, w0, #4
 # CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -     add	w1, w0, #4
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.   .   add	w2, w3, #1
-# CHECK-NEXT: [0,1]     DeeER.   .   add	w4, w3, #2, lsl #12
-# CHECK-NEXT: [0,2]     .DeeER   .   add	w0, w4, #3
-# CHECK-NEXT: [0,3]     . DeeER  .   add	w1, w0, #4
-# CHECK-NEXT: [1,0]     . DeeER  .   add	w2, w3, #1
-# CHECK-NEXT: [1,1]     .  DeeER .   add	w4, w3, #2, lsl #12
-# CHECK-NEXT: [1,2]     .   DeeER.   add	w0, w4, #3
-# CHECK-NEXT: [1,3]     .    DeeER   add	w1, w0, #4
+# CHECK:      [0,0]     DeeE .  .   add	w2, w3, #1
+# CHECK-NEXT: [0,1]     DeeE .  .   add	w4, w3, #2, lsl #12
+# CHECK-NEXT: [0,2]     .DeeE.  .   add	w0, w4, #3
+# CHECK-NEXT: [0,3]     . DeeE  .   add	w1, w0, #4
+# CHECK-NEXT: [1,0]     . DeeE  .   add	w2, w3, #1
+# CHECK-NEXT: [1,1]     .  DeeE .   add	w4, w3, #2, lsl #12
+# CHECK-NEXT: [1,2]     .   DeeE.   add	w0, w4, #3
+# CHECK-NEXT: [1,3]     .    DeeE   add	w1, w0, #4
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-all-stats.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-all-stats.s
index 35149b09f66f80e3d8a9bcbb76cdec97c10392f0..9081fb525ee22053d900224ff55085d782a2311d 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-all-stats.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-all-stats.s
@@ -10,12 +10,12 @@ str	w0, [x21, x18, lsl #2]
 
 # CHECK:      Iterations:        2
 # CHECK-NEXT: Instructions:      12
-# CHECK-NEXT: Total Cycles:      21
+# CHECK-NEXT: Total Cycles:      20
 # CHECK-NEXT: Total uOps:        14
 
 # CHECK:      Dispatch Width:    2
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.57
+# CHECK-NEXT: uOps Per Cycle:    0.70
+# CHECK-NEXT: IPC:               0.60
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Instruction Info:
@@ -35,7 +35,7 @@ str	w0, [x21, x18, lsl #2]
 # CHECK-NEXT:  1      4     1.00           *            str	w0, [x21, x18, lsl #2]
 
 # CHECK:      Dynamic Dispatch Stall Cycles:
-# CHECK-NEXT: RAT     - Register unavailable:                      10  (47.6%)
+# CHECK-NEXT: RAT     - Register unavailable:                      8  (40.0%)
 # CHECK-NEXT: RCU     - Retire tokens unavailable:                 0
 # CHECK-NEXT: SCHEDQ  - Scheduler full:                            0
 # CHECK-NEXT: LQ      - Load queue full:                           0
@@ -44,33 +44,22 @@ str	w0, [x21, x18, lsl #2]
 
 # CHECK:      Dispatch Logic - number of cycles where we saw N micro opcodes dispatched:
 # CHECK-NEXT: [# dispatched], [# cycles]
-# CHECK-NEXT:  0,              11  (52.4%)
-# CHECK-NEXT:  1,              6  (28.6%)
-# CHECK-NEXT:  2,              4  (19.0%)
+# CHECK-NEXT:  0,              10  (50.0%)
+# CHECK-NEXT:  1,              6  (30.0%)
+# CHECK-NEXT:  2,              4  (20.0%)
 
 # CHECK:      Schedulers - number of cycles where we saw N micro opcodes issued:
 # CHECK-NEXT: [# issued], [# cycles]
-# CHECK-NEXT:  0,          11  (52.4%)
-# CHECK-NEXT:  1,          6  (28.6%)
-# CHECK-NEXT:  2,          4  (19.0%)
+# CHECK-NEXT:  0,          10  (50.0%)
+# CHECK-NEXT:  1,          6  (30.0%)
+# CHECK-NEXT:  2,          4  (20.0%)
 
 # CHECK:      Scheduler's queue usage:
 # CHECK-NEXT: No scheduler resources used.
 
-# CHECK:      Retire Control Unit - number of cycles where we saw N instructions retired:
-# CHECK-NEXT: [# retired], [# cycles]
-# CHECK-NEXT:  0,           14  (66.7%)
-# CHECK-NEXT:  1,           4  (19.0%)
-# CHECK-NEXT:  2,           1  (4.8%)
-# CHECK-NEXT:  3,           2  (9.5%)
-
-# CHECK:      Total ROB Entries:                64
-# CHECK-NEXT: Max Used ROB Entries:             6  ( 9.4% )
-# CHECK-NEXT: Average Used ROB Entries per cy:  2  ( 3.1% )
-
 # CHECK:      Register File statistics:
 # CHECK-NEXT: Total number of mappings created:    14
-# CHECK-NEXT: Max number of mappings used:         6
+# CHECK-NEXT: Max number of mappings used:         4
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0.0] - CortexA55UnitALU
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-all-views.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-all-views.s
index f6b3f622a38b4b752e32b13ea587cf214743d4b3..d49e68adc1c3fe56b688232ec412e4e081551cfc 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-all-views.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-all-views.s
@@ -10,12 +10,12 @@ str	w0, [x21, x18, lsl #2]
 
 # CHECK:      Iterations:        2
 # CHECK-NEXT: Instructions:      12
-# CHECK-NEXT: Total Cycles:      21
+# CHECK-NEXT: Total Cycles:      20
 # CHECK-NEXT: Total uOps:        14
 
 # CHECK:      Dispatch Width:    2
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.57
+# CHECK-NEXT: uOps Per Cycle:    0.70
+# CHECK-NEXT: IPC:               0.60
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Instruction Info:
@@ -35,7 +35,7 @@ str	w0, [x21, x18, lsl #2]
 # CHECK-NEXT:  1      4     1.00           *            str	w0, [x21, x18, lsl #2]
 
 # CHECK:      Dynamic Dispatch Stall Cycles:
-# CHECK-NEXT: RAT     - Register unavailable:                      10  (47.6%)
+# CHECK-NEXT: RAT     - Register unavailable:                      8  (40.0%)
 # CHECK-NEXT: RCU     - Retire tokens unavailable:                 0
 # CHECK-NEXT: SCHEDQ  - Scheduler full:                            0
 # CHECK-NEXT: LQ      - Load queue full:                           0
@@ -44,33 +44,22 @@ str	w0, [x21, x18, lsl #2]
 
 # CHECK:      Dispatch Logic - number of cycles where we saw N micro opcodes dispatched:
 # CHECK-NEXT: [# dispatched], [# cycles]
-# CHECK-NEXT:  0,              11  (52.4%)
-# CHECK-NEXT:  1,              6  (28.6%)
-# CHECK-NEXT:  2,              4  (19.0%)
+# CHECK-NEXT:  0,              10  (50.0%)
+# CHECK-NEXT:  1,              6  (30.0%)
+# CHECK-NEXT:  2,              4  (20.0%)
 
 # CHECK:      Schedulers - number of cycles where we saw N micro opcodes issued:
 # CHECK-NEXT: [# issued], [# cycles]
-# CHECK-NEXT:  0,          11  (52.4%)
-# CHECK-NEXT:  1,          6  (28.6%)
-# CHECK-NEXT:  2,          4  (19.0%)
+# CHECK-NEXT:  0,          10  (50.0%)
+# CHECK-NEXT:  1,          6  (30.0%)
+# CHECK-NEXT:  2,          4  (20.0%)
 
 # CHECK:      Scheduler's queue usage:
 # CHECK-NEXT: No scheduler resources used.
 
-# CHECK:      Retire Control Unit - number of cycles where we saw N instructions retired:
-# CHECK-NEXT: [# retired], [# cycles]
-# CHECK-NEXT:  0,           14  (66.7%)
-# CHECK-NEXT:  1,           4  (19.0%)
-# CHECK-NEXT:  2,           1  (4.8%)
-# CHECK-NEXT:  3,           2  (9.5%)
-
-# CHECK:      Total ROB Entries:                64
-# CHECK-NEXT: Max Used ROB Entries:             6  ( 9.4% )
-# CHECK-NEXT: Average Used ROB Entries per cy:  2  ( 3.1% )
-
 # CHECK:      Register File statistics:
 # CHECK-NEXT: Total number of mappings created:    14
-# CHECK-NEXT: Max number of mappings used:         6
+# CHECK-NEXT: Max number of mappings used:         4
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0.0] - CortexA55UnitALU
@@ -101,20 +90,20 @@ str	w0, [x21, x18, lsl #2]
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
-
-# CHECK:      [0,0]     DeeER.    .    .    .   ldr	w4, [x2], #4
-# CHECK-NEXT: [0,1]     .DeeER    .    .    .   ldr	w5, [x3]
-# CHECK-NEXT: [0,2]     .   DeeeER.    .    .   madd	w0, w5, w4, w0
-# CHECK-NEXT: [0,3]     .   DeeE-R.    .    .   add	x3, x3, x13
-# CHECK-NEXT: [0,4]     .    DeeER.    .    .   subs	x1, x1, #1
-# CHECK-NEXT: [0,5]     .    . DeeeER  .    .   str	w0, [x21, x18, lsl #2]
-# CHECK-NEXT: [1,0]     .    .  DeeER  .    .   ldr	w4, [x2], #4
-# CHECK-NEXT: [1,1]     .    .   DeeER .    .   ldr	w5, [x3]
-# CHECK-NEXT: [1,2]     .    .    . DeeeER  .   madd	w0, w5, w4, w0
-# CHECK-NEXT: [1,3]     .    .    . DeeE-R  .   add	x3, x3, x13
-# CHECK-NEXT: [1,4]     .    .    .  DeeER  .   subs	x1, x1, #1
-# CHECK-NEXT: [1,5]     .    .    .    DeeeER   str	w0, [x21, x18, lsl #2]
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeE .    .    .   .   ldr	w4, [x2], #4
+# CHECK-NEXT: [0,1]     .DeeE.    .    .   .   ldr	w5, [x3]
+# CHECK-NEXT: [0,2]     .   DeeeE .    .   .   madd	w0, w5, w4, w0
+# CHECK-NEXT: [0,3]     .    DeeE .    .   .   add	x3, x3, x13
+# CHECK-NEXT: [0,4]     .    DeeE .    .   .   subs	x1, x1, #1
+# CHECK-NEXT: [0,5]     .    . DeeeE   .   .   str	w0, [x21, x18, lsl #2]
+# CHECK-NEXT: [1,0]     .    .  DeeE   .   .   ldr	w4, [x2], #4
+# CHECK-NEXT: [1,1]     .    .   DeeE  .   .   ldr	w5, [x3]
+# CHECK-NEXT: [1,2]     .    .    . DeeeE  .   madd	w0, w5, w4, w0
+# CHECK-NEXT: [1,3]     .    .    .  DeeE  .   add	x3, x3, x13
+# CHECK-NEXT: [1,4]     .    .    .  DeeE  .   subs	x1, x1, #1
+# CHECK-NEXT: [1,5]     .    .    .    DeeeE   str	w0, [x21, x18, lsl #2]
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -126,7 +115,7 @@ str	w0, [x21, x18, lsl #2]
 # CHECK-NEXT: 0.     2     0.0    0.0    0.0       ldr	w4, [x2], #4
 # CHECK-NEXT: 1.     2     0.0    0.0    0.0       ldr	w5, [x3]
 # CHECK-NEXT: 2.     2     0.0    0.0    0.0       madd	w0, w5, w4, w0
-# CHECK-NEXT: 3.     2     0.0    0.0    1.0       add	x3, x3, x13
+# CHECK-NEXT: 3.     2     0.0    0.0    0.0       add	x3, x3, x13
 # CHECK-NEXT: 4.     2     0.0    0.0    0.0       subs	x1, x1, #1
 # CHECK-NEXT: 5.     2     0.0    0.0    0.0       str	w0, [x21, x18, lsl #2]
-# CHECK-NEXT:        2     0.0    0.0    0.2       <total>
+# CHECK-NEXT:        2     0.0    0.0    0.0       <total>
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-in-order-retire.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-in-order-retire.s
index 71c1a06206072dcb28ba37f028a6dac2766040c9..c35332420549053daf06a221a9fb31afba99cd7a 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-in-order-retire.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-in-order-retire.s
@@ -10,12 +10,12 @@ add	w7, w9, w0
 
 # CHECK:      Iterations:        2
 # CHECK-NEXT: Instructions:      12
-# CHECK-NEXT: Total Cycles:      18
+# CHECK-NEXT: Total Cycles:      19
 # CHECK-NEXT: Total uOps:        12
 
 # CHECK:      Dispatch Width:    2
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.67
+# CHECK-NEXT: uOps Per Cycle:    0.63
+# CHECK-NEXT: IPC:               0.63
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Instruction Info:
@@ -40,33 +40,26 @@ add	w7, w9, w0
 # CHECK-NEXT: SCHEDQ  - Scheduler full:                            0
 # CHECK-NEXT: LQ      - Load queue full:                           0
 # CHECK-NEXT: SQ      - Store queue full:                          0
-# CHECK-NEXT: GROUP   - Static restrictions on the dispatch group: 5  (27.8%)
+# CHECK-NEXT: GROUP   - Static restrictions on the dispatch group: 1  (5.3%)
 
 # CHECK:      Dispatch Logic - number of cycles where we saw N micro opcodes dispatched:
 # CHECK-NEXT: [# dispatched], [# cycles]
-# CHECK-NEXT:  0,              12  (66.7%)
-# CHECK-NEXT:  2,              6  (33.3%)
+# CHECK-NEXT:  0,              11  (57.9%)
+# CHECK-NEXT:  1,              4  (21.1%)
+# CHECK-NEXT:  2,              4  (21.1%)
 
 # CHECK:      Schedulers - number of cycles where we saw N micro opcodes issued:
 # CHECK-NEXT: [# issued], [# cycles]
-# CHECK-NEXT:  0,          12  (66.7%)
-# CHECK-NEXT:  2,          6  (33.3%)
+# CHECK-NEXT:  0,          11  (57.9%)
+# CHECK-NEXT:  1,          4  (21.1%)
+# CHECK-NEXT:  2,          4  (21.1%)
 
 # CHECK:      Scheduler's queue usage:
 # CHECK-NEXT: No scheduler resources used.
 
-# CHECK:      Retire Control Unit - number of cycles where we saw N instructions retired:
-# CHECK-NEXT: [# retired], [# cycles]
-# CHECK-NEXT:  0,           16  (88.9%)
-# CHECK-NEXT:  6,           2  (11.1%)
-
-# CHECK:      Total ROB Entries:                64
-# CHECK-NEXT: Max Used ROB Entries:             8  ( 12.5% )
-# CHECK-NEXT: Average Used ROB Entries per cy:  5  ( 7.8% )
-
 # CHECK:      Register File statistics:
 # CHECK-NEXT: Total number of mappings created:    12
-# CHECK-NEXT: Max number of mappings used:         8
+# CHECK-NEXT: Max number of mappings used:         6
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0.0] - CortexA55UnitALU
@@ -96,21 +89,21 @@ add	w7, w9, w0
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w7, w9, w0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234567
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    . .   sdiv	w12, w21, w0
-# CHECK-NEXT: [0,1]     DeeE-----R.    . .   add	w8, w8, #1
-# CHECK-NEXT: [0,2]     .DeeE----R.    . .   add	w1, w2, w0
-# CHECK-NEXT: [0,3]     .DeeE----R.    . .   add	w3, w4, #1
-# CHECK-NEXT: [0,4]     . DeeE---R.    . .   add	w5, w6, w0
-# CHECK-NEXT: [0,5]     . DeeE---R.    . .   add	w7, w9, w0
-# CHECK-NEXT: [1,0]     .    .  DeeeeeeeER   sdiv	w12, w21, w0
-# CHECK-NEXT: [1,1]     .    .  DeeE-----R   add	w8, w8, #1
-# CHECK-NEXT: [1,2]     .    .   DeeE----R   add	w1, w2, w0
-# CHECK-NEXT: [1,3]     .    .   DeeE----R   add	w3, w4, #1
-# CHECK-NEXT: [1,4]     .    .    DeeE---R   add	w5, w6, w0
-# CHECK-NEXT: [1,5]     .    .    DeeE---R   add	w7, w9, w0
+# CHECK:      [0,0]     DeeeeeeeE .    .  .   sdiv	w12, w21, w0
+# CHECK-NEXT: [0,1]     .    DeeE .    .  .   add	w8, w8, #1
+# CHECK-NEXT: [0,2]     .    DeeE .    .  .   add	w1, w2, w0
+# CHECK-NEXT: [0,3]     .    .DeeE.    .  .   add	w3, w4, #1
+# CHECK-NEXT: [0,4]     .    .DeeE.    .  .   add	w5, w6, w0
+# CHECK-NEXT: [0,5]     .    . DeeE    .  .   add	w7, w9, w0
+# CHECK-NEXT: [1,0]     .    .  DeeeeeeeE .   sdiv	w12, w21, w0
+# CHECK-NEXT: [1,1]     .    .    .  DeeE .   add	w8, w8, #1
+# CHECK-NEXT: [1,2]     .    .    .  DeeE .   add	w1, w2, w0
+# CHECK-NEXT: [1,3]     .    .    .   DeeE.   add	w3, w4, #1
+# CHECK-NEXT: [1,4]     .    .    .   DeeE.   add	w5, w6, w0
+# CHECK-NEXT: [1,5]     .    .    .    DeeE   add	w7, w9, w0
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -120,9 +113,9 @@ add	w7, w9, w0
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     2     0.0    0.0    0.0       sdiv	w12, w21, w0
-# CHECK-NEXT: 1.     2     0.0    0.0    5.0       add	w8, w8, #1
-# CHECK-NEXT: 2.     2     0.0    0.0    4.0       add	w1, w2, w0
-# CHECK-NEXT: 3.     2     0.0    0.0    4.0       add	w3, w4, #1
-# CHECK-NEXT: 4.     2     0.0    0.0    3.0       add	w5, w6, w0
-# CHECK-NEXT: 5.     2     0.0    0.0    3.0       add	w7, w9, w0
-# CHECK-NEXT:        2     0.0    0.0    3.2       <total>
+# CHECK-NEXT: 1.     2     0.0    0.0    0.0       add	w8, w8, #1
+# CHECK-NEXT: 2.     2     0.0    0.0    0.0       add	w1, w2, w0
+# CHECK-NEXT: 3.     2     0.0    0.0    0.0       add	w3, w4, #1
+# CHECK-NEXT: 4.     2     0.0    0.0    0.0       add	w5, w6, w0
+# CHECK-NEXT: 5.     2     0.0    0.0    0.0       add	w7, w9, w0
+# CHECK-NEXT:        2     0.0    0.0    0.0       <total>
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-out-of-order-retire.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-out-of-order-retire.s
index 6231116f25ace78d12bfd4f261de651aa694c31b..8935d254cd98da5158e704304f094f1166b7298b 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-out-of-order-retire.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-out-of-order-retire.s
@@ -10,12 +10,12 @@ add	w7, w9, w0
 
 # CHECK:      Iterations:        2
 # CHECK-NEXT: Instructions:      12
-# CHECK-NEXT: Total Cycles:      25
+# CHECK-NEXT: Total Cycles:      24
 # CHECK-NEXT: Total uOps:        12
 
 # CHECK:      Dispatch Width:    2
-# CHECK-NEXT: uOps Per Cycle:    0.48
-# CHECK-NEXT: IPC:               0.48
+# CHECK-NEXT: uOps Per Cycle:    0.50
+# CHECK-NEXT: IPC:               0.50
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Instruction Info:
@@ -40,31 +40,21 @@ add	w7, w9, w0
 # CHECK-NEXT: SCHEDQ  - Scheduler full:                            0
 # CHECK-NEXT: LQ      - Load queue full:                           0
 # CHECK-NEXT: SQ      - Store queue full:                          0
-# CHECK-NEXT: GROUP   - Static restrictions on the dispatch group: 7  (28.0%)
+# CHECK-NEXT: GROUP   - Static restrictions on the dispatch group: 7  (29.2%)
 
 # CHECK:      Dispatch Logic - number of cycles where we saw N micro opcodes dispatched:
 # CHECK-NEXT: [# dispatched], [# cycles]
-# CHECK-NEXT:  0,              19  (76.0%)
-# CHECK-NEXT:  2,              6  (24.0%)
+# CHECK-NEXT:  0,              18  (75.0%)
+# CHECK-NEXT:  2,              6  (25.0%)
 
 # CHECK:      Schedulers - number of cycles where we saw N micro opcodes issued:
 # CHECK-NEXT: [# issued], [# cycles]
-# CHECK-NEXT:  0,          19  (76.0%)
-# CHECK-NEXT:  2,          6  (24.0%)
+# CHECK-NEXT:  0,          18  (75.0%)
+# CHECK-NEXT:  2,          6  (25.0%)
 
 # CHECK:      Scheduler's queue usage:
 # CHECK-NEXT: No scheduler resources used.
 
-# CHECK:      Retire Control Unit - number of cycles where we saw N instructions retired:
-# CHECK-NEXT: [# retired], [# cycles]
-# CHECK-NEXT:  0,           18  (72.0%)
-# CHECK-NEXT:  1,           2  (8.0%)
-# CHECK-NEXT:  2,           5  (20.0%)
-
-# CHECK:      Total ROB Entries:                64
-# CHECK-NEXT: Max Used ROB Entries:             7  ( 10.9% )
-# CHECK-NEXT: Average Used ROB Entries per cy:  2  ( 3.1% )
-
 # CHECK:      Register File statistics:
 # CHECK-NEXT: Total number of mappings created:    12
 # CHECK-NEXT: Max number of mappings used:         7
@@ -98,20 +88,20 @@ add	w7, w9, w0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234
-
-# CHECK:      [0,0]     DeeeeeeeeeeeeER.    .   .   fdiv	s1, s2, s3
-# CHECK-NEXT: [0,1]     DeeER.    .    .    .   .   add	w8, w8, #1
-# CHECK-NEXT: [0,2]     .DeeER    .    .    .   .   add	w1, w2, w0
-# CHECK-NEXT: [0,3]     .DeeER    .    .    .   .   add	w3, w4, #1
-# CHECK-NEXT: [0,4]     . DeeER   .    .    .   .   add	w5, w6, w0
-# CHECK-NEXT: [0,5]     . DeeER   .    .    .   .   add	w7, w9, w0
-# CHECK-NEXT: [1,0]     .    .    DeeeeeeeeeeeeER   fdiv	s1, s2, s3
-# CHECK-NEXT: [1,1]     .    .    DeeER.    .   .   add	w8, w8, #1
-# CHECK-NEXT: [1,2]     .    .    .DeeER    .   .   add	w1, w2, w0
-# CHECK-NEXT: [1,3]     .    .    .DeeER    .   .   add	w3, w4, #1
-# CHECK-NEXT: [1,4]     .    .    . DeeER   .   .   add	w5, w6, w0
-# CHECK-NEXT: [1,5]     .    .    . DeeER   .   .   add	w7, w9, w0
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeeeeeeeeeeE .    .  .   fdiv	s1, s2, s3
+# CHECK-NEXT: [0,1]     DeeE .    .    .    .  .   add	w8, w8, #1
+# CHECK-NEXT: [0,2]     .DeeE.    .    .    .  .   add	w1, w2, w0
+# CHECK-NEXT: [0,3]     .DeeE.    .    .    .  .   add	w3, w4, #1
+# CHECK-NEXT: [0,4]     . DeeE    .    .    .  .   add	w5, w6, w0
+# CHECK-NEXT: [0,5]     . DeeE    .    .    .  .   add	w7, w9, w0
+# CHECK-NEXT: [1,0]     .    .    DeeeeeeeeeeeeE   fdiv	s1, s2, s3
+# CHECK-NEXT: [1,1]     .    .    DeeE .    .  .   add	w8, w8, #1
+# CHECK-NEXT: [1,2]     .    .    .DeeE.    .  .   add	w1, w2, w0
+# CHECK-NEXT: [1,3]     .    .    .DeeE.    .  .   add	w3, w4, #1
+# CHECK-NEXT: [1,4]     .    .    . DeeE    .  .   add	w5, w6, w0
+# CHECK-NEXT: [1,5]     .    .    . DeeE    .  .   add	w7, w9, w0
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
diff --git a/llvm/test/tools/llvm-mca/AMDGPU/gfx10-add-sequence.s b/llvm/test/tools/llvm-mca/AMDGPU/gfx10-add-sequence.s
index 64ee3bdc8355f19678973d5c6739f8aeb1323fd4..be57731389ba3e769fd11d321a00d080c77c047d 100644
--- a/llvm/test/tools/llvm-mca/AMDGPU/gfx10-add-sequence.s
+++ b/llvm/test/tools/llvm-mca/AMDGPU/gfx10-add-sequence.s
@@ -7,12 +7,12 @@ v_add_f32 v2, v1, v0
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      3
-# CHECK-NEXT: Total Cycles:      13
+# CHECK-NEXT: Total Cycles:      12
 # CHECK-NEXT: Total uOps:        3
 
 # CHECK:      Dispatch Width:    1
-# CHECK-NEXT: uOps Per Cycle:    0.23
-# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: uOps Per Cycle:    0.25
+# CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Instruction Info:
@@ -48,12 +48,12 @@ v_add_f32 v2, v1, v0
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_add_f32_e32 v2, v1, v0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012
+# CHECK-NEXT:                     01
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   . .   v_add_f32_e32 v0, v0, v0
-# CHECK-NEXT: [0,1]     .DeeeeER  . .   v_add_f32_e32 v1, v1, v1
-# CHECK-NEXT: [0,2]     .    .DeeeeER   v_add_f32_e32 v2, v1, v0
+# CHECK:      [0,0]     DeeeeE    ..   v_add_f32_e32 v0, v0, v0
+# CHECK-NEXT: [0,1]     .DeeeeE   ..   v_add_f32_e32 v1, v1, v1
+# CHECK-NEXT: [0,2]     .    .DeeeeE   v_add_f32_e32 v2, v1, v0
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
diff --git a/llvm/test/tools/llvm-mca/AMDGPU/gfx10-double.s b/llvm/test/tools/llvm-mca/AMDGPU/gfx10-double.s
new file mode 100644
index 0000000000000000000000000000000000000000..3a0991d875f6a172cf7690c0f6ec7fa4c5a3e0af
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AMDGPU/gfx10-double.s
@@ -0,0 +1,187 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=amdgcn -mcpu=gfx1010 --timeline --iterations=1 < %s | FileCheck %s
+
+v_cvt_i32_f64 v0, v[0:1]
+v_cvt_f64_i32 v[2:3], v2
+v_cvt_f32_f64 v4, v[4:5]
+v_cvt_f64_f32 v[6:7], v6
+v_cvt_u32_f64 v8, v[8:9]
+v_cvt_f64_u32 v[10:11], v10
+
+v_frexp_exp_i32_f64 v0, v[0:1]
+v_frexp_mant_f64 v[2:3], v[2:3]
+v_fract_f64 v[4:5], v[4:5]
+
+v_trunc_f64 v[0:1], v[0:1]
+v_ceil_f64 v[2:3], v[2:3]
+v_rndne_f64 v[4:5], v[4:5]
+v_floor_f64 v[6:7], v[6:7]
+
+v_fma_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+v_add_f64 v[2:3], v[2:3], v[2:3]
+v_mul_f64 v[4:5], v[4:5], v[4:5]
+v_min_f64 v[6:7], v[6:7], v[6:7]
+v_max_f64 v[8:9], v[8:9], v[8:9]
+
+v_div_fmas_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+
+v_div_fixup_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+v_ldexp_f64 v[2:3], v[2:3], v0
+
+v_div_scale_f64 v[0:1], vcc_lo, v[0:1], v[0:1], v[0:1]
+
+v_trig_preop_f64 v[2:3], v[2:3], v0
+
+v_cmp_eq_f64 v[0:1], v[0:1]
+v_cmp_class_f64 vcc_lo, v[2:3], s0
+
+v_rcp_f64 v[0:1], v[0:1]
+v_rsq_f64 v[2:3], v[2:3]
+v_sqrt_f64 v[4:5], v[4:5]
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      28
+# CHECK-NEXT: Total Cycles:      224
+# CHECK-NEXT: Total uOps:        29
+
+# CHECK:      Dispatch Width:    1
+# CHECK-NEXT: uOps Per Cycle:    0.13
+# CHECK-NEXT: IPC:               0.13
+# CHECK-NEXT: Block RThroughput: 29.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      22    1.00                  U     v_cvt_i32_f64_e32 v0, v[0:1]
+# CHECK-NEXT:  1      22    1.00                  U     v_cvt_f64_i32_e32 v[2:3], v2
+# CHECK-NEXT:  1      22    1.00                  U     v_cvt_f32_f64_e32 v4, v[4:5]
+# CHECK-NEXT:  1      22    1.00                  U     v_cvt_f64_f32_e32 v[6:7], v6
+# CHECK-NEXT:  1      22    1.00                  U     v_cvt_u32_f64_e32 v8, v[8:9]
+# CHECK-NEXT:  1      22    1.00                  U     v_cvt_f64_u32_e32 v[10:11], v10
+# CHECK-NEXT:  1      22    1.00                  U     v_frexp_exp_i32_f64_e32 v0, v[0:1]
+# CHECK-NEXT:  1      22    1.00                  U     v_frexp_mant_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT:  1      22    1.00                  U     v_fract_f64_e32 v[4:5], v[4:5]
+# CHECK-NEXT:  1      22    1.00                  U     v_trunc_f64_e32 v[0:1], v[0:1]
+# CHECK-NEXT:  1      22    1.00                  U     v_ceil_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT:  1      22    1.00                  U     v_rndne_f64_e32 v[4:5], v[4:5]
+# CHECK-NEXT:  1      22    1.00                  U     v_floor_f64_e32 v[6:7], v[6:7]
+# CHECK-NEXT:  1      22    1.00                  U     v_fma_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT:  1      22    1.00                  U     v_add_f64 v[2:3], v[2:3], v[2:3]
+# CHECK-NEXT:  1      22    1.00                  U     v_mul_f64 v[4:5], v[4:5], v[4:5]
+# CHECK-NEXT:  1      22    1.00                  U     v_min_f64 v[6:7], v[6:7], v[6:7]
+# CHECK-NEXT:  1      22    1.00                  U     v_max_f64 v[8:9], v[8:9], v[8:9]
+# CHECK-NEXT:  1      22    1.00                  U     v_div_fmas_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT:  1      22    1.00                  U     v_div_fixup_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT:  1      22    1.00                  U     v_ldexp_f64 v[2:3], v[2:3], v0
+# CHECK-NEXT:  2      22    2.00                  U     v_div_scale_f64 v[0:1], vcc_lo, v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT:  1      22    1.00                  U     v_trig_preop_f64 v[2:3], v[2:3], v0
+# CHECK-NEXT:  1      22    1.00                  U     v_cmp_eq_f64_e32 vcc_lo, v[0:1], v[0:1]
+# CHECK-NEXT:  1      22    1.00                  U     v_cmp_class_f64_e64 vcc_lo, v[2:3], s0
+# CHECK-NEXT:  1      24    1.00                  U     v_rcp_f64_e32 v[0:1], v[0:1]
+# CHECK-NEXT:  1      24    1.00                  U     v_rsq_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT:  1      24    1.00                  U     v_sqrt_f64_e32 v[4:5], v[4:5]
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - HWBranch
+# CHECK-NEXT: [1]   - HWExport
+# CHECK-NEXT: [2]   - HWLGKM
+# CHECK-NEXT: [3]   - HWRC
+# CHECK-NEXT: [4]   - HWSALU
+# CHECK-NEXT: [5]   - HWVALU
+# CHECK-NEXT: [6]   - HWVMEM
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]
+# CHECK-NEXT:  -      -      -     29.00  1.00   28.00   -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_cvt_i32_f64_e32 v0, v[0:1]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_cvt_f64_i32_e32 v[2:3], v2
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_cvt_f32_f64_e32 v4, v[4:5]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_cvt_f64_f32_e32 v[6:7], v6
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_cvt_u32_f64_e32 v8, v[8:9]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_cvt_f64_u32_e32 v[10:11], v10
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_frexp_exp_i32_f64_e32 v0, v[0:1]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_frexp_mant_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_fract_f64_e32 v[4:5], v[4:5]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_trunc_f64_e32 v[0:1], v[0:1]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_ceil_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_rndne_f64_e32 v[4:5], v[4:5]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_floor_f64_e32 v[6:7], v[6:7]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_fma_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_add_f64 v[2:3], v[2:3], v[2:3]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_mul_f64 v[4:5], v[4:5], v[4:5]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_min_f64 v[6:7], v[6:7], v[6:7]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_max_f64 v[8:9], v[8:9], v[8:9]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_div_fmas_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_div_fixup_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_ldexp_f64 v[2:3], v[2:3], v0
+# CHECK-NEXT:  -      -      -     2.00   1.00   1.00    -     v_div_scale_f64 v[0:1], vcc_lo, v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_trig_preop_f64 v[2:3], v[2:3], v0
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_cmp_eq_f64_e32 vcc_lo, v[0:1], v[0:1]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_cmp_class_f64_e64 vcc_lo, v[2:3], s0
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_rcp_f64_e32 v[0:1], v[0:1]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_rsq_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -     v_sqrt_f64_e32 v[4:5], v[4:5]
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456789          0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789          0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeeeeeeeeeeeeeeE  .    .    .    .    .    .    .    .    .    .   v_cvt_i32_f64_e32 v0, v[0:1]
+# CHECK-NEXT: [0,1]     .DeeeeeeeeeeeeeeeeeeeeeE .    .    .    .    .    .    .    .    .    .   v_cvt_f64_i32_e32 v[2:3], v2
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeeeeeeeeeeeeeE.    .    .    .    .    .    .    .    .    .   v_cvt_f32_f64_e32 v4, v[4:5]
+# CHECK-NEXT: [0,3]     .  DeeeeeeeeeeeeeeeeeeeeeE    .    .    .    .    .    .    .    .    .   v_cvt_f64_f32_e32 v[6:7], v6
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeeeeeeeeeeeeeeE   .    .    .    .    .    .    .    .    .   v_cvt_u32_f64_e32 v8, v[8:9]
+# CHECK-NEXT: [0,5]     .    DeeeeeeeeeeeeeeeeeeeeeE  .    .    .    .    .    .    .    .    .   v_cvt_f64_u32_e32 v[10:11], v10
+# CHECK-NEXT: [0,6]     .    .    .    .    . DeeeeeeeeeeeeeeeeeeeeeE.    .    .    .    .    .   v_frexp_exp_i32_f64_e32 v0, v[0:1]
+# CHECK-NEXT: [0,7]     .    .    .    .    .  DeeeeeeeeeeeeeeeeeeeeeE    .    .    .    .    .   v_frexp_mant_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT: [0,8]     .    .    .    .    .   DeeeeeeeeeeeeeeeeeeeeeE   .    .    .    .    .   v_fract_f64_e32 v[4:5], v[4:5]
+# CHECK-NEXT: [0,9]     .    .    .    .    .    .    .    .    .   DeeeeeeeeeeeeeeeeeeeeeE   .   v_trunc_f64_e32 v[0:1], v[0:1]
+# CHECK-NEXT: [0,10]    .    .    .    .    .    .    .    .    .    DeeeeeeeeeeeeeeeeeeeeeE  .   v_ceil_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT: [0,11]    .    .    .    .    .    .    .    .    .    .DeeeeeeeeeeeeeeeeeeeeeE .   v_rndne_f64_e32 v[4:5], v[4:5]
+# CHECK-NEXT: [0,12]    .    .    .    .    .    .    .    .    .    . DeeeeeeeeeeeeeeeeeeeeeE.   v_floor_f64_e32 v[6:7], v[6:7]
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     0.0    0.0    0.0       v_cvt_i32_f64_e32 v0, v[0:1]
+# CHECK-NEXT: 1.     1     0.0    0.0    0.0       v_cvt_f64_i32_e32 v[2:3], v2
+# CHECK-NEXT: 2.     1     0.0    0.0    0.0       v_cvt_f32_f64_e32 v4, v[4:5]
+# CHECK-NEXT: 3.     1     0.0    0.0    0.0       v_cvt_f64_f32_e32 v[6:7], v6
+# CHECK-NEXT: 4.     1     0.0    0.0    0.0       v_cvt_u32_f64_e32 v8, v[8:9]
+# CHECK-NEXT: 5.     1     0.0    0.0    0.0       v_cvt_f64_u32_e32 v[10:11], v10
+# CHECK-NEXT: 6.     1     0.0    0.0    0.0       v_frexp_exp_i32_f64_e32 v0, v[0:1]
+# CHECK-NEXT: 7.     1     0.0    0.0    0.0       v_frexp_mant_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT: 8.     1     0.0    0.0    0.0       v_fract_f64_e32 v[4:5], v[4:5]
+# CHECK-NEXT: 9.     1     0.0    0.0    0.0       v_trunc_f64_e32 v[0:1], v[0:1]
+# CHECK-NEXT: 10.    1     0.0    0.0    0.0       v_ceil_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT: 11.    1     0.0    0.0    0.0       v_rndne_f64_e32 v[4:5], v[4:5]
+# CHECK-NEXT: 12.    1     0.0    0.0    0.0       v_floor_f64_e32 v[6:7], v[6:7]
+# CHECK-NEXT: 13.    1     0.0    0.0    0.0       v_fma_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT: 14.    1     0.0    0.0    0.0       v_add_f64 v[2:3], v[2:3], v[2:3]
+# CHECK-NEXT: 15.    1     0.0    0.0    0.0       v_mul_f64 v[4:5], v[4:5], v[4:5]
+# CHECK-NEXT: 16.    1     0.0    0.0    0.0       v_min_f64 v[6:7], v[6:7], v[6:7]
+# CHECK-NEXT: 17.    1     0.0    0.0    0.0       v_max_f64 v[8:9], v[8:9], v[8:9]
+# CHECK-NEXT: 18.    1     0.0    0.0    0.0       v_div_fmas_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT: 19.    1     0.0    0.0    0.0       v_div_fixup_f64 v[0:1], v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT: 20.    1     0.0    0.0    0.0       v_ldexp_f64 v[2:3], v[2:3], v0
+# CHECK-NEXT: 21.    1     0.0    0.0    0.0       v_div_scale_f64 v[0:1], vcc_lo, v[0:1], v[0:1], v[0:1]
+# CHECK-NEXT: 22.    1     0.0    0.0    0.0       v_trig_preop_f64 v[2:3], v[2:3], v0
+# CHECK-NEXT: 23.    1     0.0    0.0    0.0       v_cmp_eq_f64_e32 vcc_lo, v[0:1], v[0:1]
+# CHECK-NEXT: 24.    1     0.0    0.0    0.0       v_cmp_class_f64_e64 vcc_lo, v[2:3], s0
+# CHECK-NEXT: 25.    1     0.0    0.0    0.0       v_rcp_f64_e32 v[0:1], v[0:1]
+# CHECK-NEXT: 26.    1     0.0    0.0    0.0       v_rsq_f64_e32 v[2:3], v[2:3]
+# CHECK-NEXT: 27.    1     0.0    0.0    0.0       v_sqrt_f64_e32 v[4:5], v[4:5]
+# CHECK-NEXT:        1     0.0    0.0    0.0       <total>
diff --git a/llvm/test/tools/llvm-mca/ARM/m7-negative-readadvance.s b/llvm/test/tools/llvm-mca/ARM/m7-negative-readadvance.s
index f195c069ef15b35003251e19347cb9ae7a17da46..ad9a2b1ad0c2022353789030a2988a451d35545f 100644
--- a/llvm/test/tools/llvm-mca/ARM/m7-negative-readadvance.s
+++ b/llvm/test/tools/llvm-mca/ARM/m7-negative-readadvance.s
@@ -9,12 +9,12 @@ vldr d0, [r1]
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      3
-# CHECK-NEXT: Total Cycles:      7
+# CHECK-NEXT: Total Cycles:      6
 # CHECK-NEXT: Total uOps:        3
 
 # CHECK:      Dispatch Width:    2
-# CHECK-NEXT: uOps Per Cycle:    0.43
-# CHECK-NEXT: IPC:               0.43
+# CHECK-NEXT: uOps Per Cycle:    0.50
+# CHECK-NEXT: IPC:               0.50
 # CHECK-NEXT: Block RThroughput: 1.5
 
 # CHECK:      Instruction Info:
@@ -56,11 +56,11 @@ vldr d0, [r1]
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -     2.00   vldr	d0, [r1]
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456
+# CHECK-NEXT: Index     012345
 
-# CHECK:      [0,0]     DER  ..   add.w	r1, r1, #1
-# CHECK-NEXT: [0,1]     .DER ..   add.w	r1, r1, #2
-# CHECK-NEXT: [0,2]     .  DeER   vldr	d0, [r1]
+# CHECK:      [0,0]     DE   .   add.w	r1, r1, #1
+# CHECK-NEXT: [0,1]     .DE  .   add.w	r1, r1, #2
+# CHECK-NEXT: [0,2]     .  DeE   vldr	d0, [r1]
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
diff --git a/llvm/test/tools/llvm-mca/X86/BtVer2/negative-read-advance.s b/llvm/test/tools/llvm-mca/X86/BtVer2/negative-read-advance.s
new file mode 100644
index 0000000000000000000000000000000000000000..de409e1c28d04d7bb59ba0215ad13bfb59871059
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/BtVer2/negative-read-advance.s
@@ -0,0 +1,90 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -iterations=1 -timeline < %s | FileCheck %s
+
+add %ebx, %ebx
+vpinsrd $1, %ebx, %xmm0, %xmm1
+vpinsrd $1, %ebx, %xmm2, %xmm3
+vpinsrd $2, %ebx, %xmm4, %xmm5
+vpinsrd $2, %ebx, %xmm6, %xmm7
+vpinsrd $3, %ebx, %xmm8, %xmm10
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      6
+# CHECK-NEXT: Total Cycles:      13
+# CHECK-NEXT: Total uOps:        11
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.85
+# CHECK-NEXT: IPC:               0.46
+# CHECK-NEXT: Block RThroughput: 5.5
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        addl	%ebx, %ebx
+# CHECK-NEXT:  2      7     0.50                        vpinsrd	$1, %ebx, %xmm0, %xmm1
+# CHECK-NEXT:  2      7     0.50                        vpinsrd	$1, %ebx, %xmm2, %xmm3
+# CHECK-NEXT:  2      7     0.50                        vpinsrd	$2, %ebx, %xmm4, %xmm5
+# CHECK-NEXT:  2      7     0.50                        vpinsrd	$2, %ebx, %xmm6, %xmm7
+# CHECK-NEXT:  2      7     0.50                        vpinsrd	$3, %ebx, %xmm8, %xmm10
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -     1.00    -      -      -     2.00   3.00    -      -      -      -     2.00   3.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     addl	%ebx, %ebx
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -     vpinsrd	$1, %ebx, %xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -     vpinsrd	$1, %ebx, %xmm2, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -     vpinsrd	$2, %ebx, %xmm4, %xmm5
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -     vpinsrd	$2, %ebx, %xmm6, %xmm7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -     vpinsrd	$3, %ebx, %xmm8, %xmm10
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeER .    . .   addl	%ebx, %ebx
+# CHECK-NEXT: [0,1]     .D======eER .   vpinsrd	$1, %ebx, %xmm0, %xmm1
+# CHECK-NEXT: [0,2]     . D=====eER .   vpinsrd	$1, %ebx, %xmm2, %xmm3
+# CHECK-NEXT: [0,3]     .  D=====eER.   vpinsrd	$2, %ebx, %xmm4, %xmm5
+# CHECK-NEXT: [0,4]     .   D====eER.   vpinsrd	$2, %ebx, %xmm6, %xmm7
+# CHECK-NEXT: [0,5]     .    D====eER   vpinsrd	$3, %ebx, %xmm8, %xmm10
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       addl	%ebx, %ebx
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       vpinsrd	$1, %ebx, %xmm0, %xmm1
+# CHECK-NEXT: 2.     1     6.0    0.0    0.0       vpinsrd	$1, %ebx, %xmm2, %xmm3
+# CHECK-NEXT: 3.     1     6.0    1.0    0.0       vpinsrd	$2, %ebx, %xmm4, %xmm5
+# CHECK-NEXT: 4.     1     5.0    1.0    0.0       vpinsrd	$2, %ebx, %xmm6, %xmm7
+# CHECK-NEXT: 5.     1     5.0    2.0    0.0       vpinsrd	$3, %ebx, %xmm8, %xmm10
+# CHECK-NEXT:        1     5.0    0.8    0.0       <total>
diff --git a/llvm/test/tools/llvm-ml/parse_only.asm b/llvm/test/tools/llvm-ml/parse_only.asm
new file mode 100644
index 0000000000000000000000000000000000000000..0105d350fcf9b4c31008c456b66d710482e62abe
--- /dev/null
+++ b/llvm/test/tools/llvm-ml/parse_only.asm
@@ -0,0 +1,18 @@
+; RUN: llvm-ml %s /Zs /Fo - | FileCheck %s
+
+.code
+
+t1 PROC
+  ECHO Testing!
+  ret
+t1 ENDP
+
+; check for the .text symbol (appears in both object files & .s output)
+; CHECK-NOT: .text
+
+; CHECK: Testing!
+
+; check for the .text symbol (appears in both object files & .s output)
+; CHECK-NOT: .text
+
+end
diff --git a/llvm/test/tools/llvm-ml/parse_only_errors.asm b/llvm/test/tools/llvm-ml/parse_only_errors.asm
new file mode 100644
index 0000000000000000000000000000000000000000..fbaed93edea03cd21e58cc93881bca964410be76
--- /dev/null
+++ b/llvm/test/tools/llvm-ml/parse_only_errors.asm
@@ -0,0 +1,18 @@
+; RUN: not llvm-ml %s /Zs /Fo - 2>&1 | FileCheck %s
+
+.code
+
+t1 PROC
+  blah
+  ret
+t1 ENDP
+
+; check for the .text symbol (appears in both object files & .s output)
+; CHECK-NOT: .text
+
+; CHECK: error: invalid instruction mnemonic 'blah'
+
+; check for the .text symbol (appears in both object files & .s output)
+; CHECK-NOT: .text
+
+end
diff --git a/llvm/test/tools/llvm-nm/ARM/special-syms.test b/llvm/test/tools/llvm-nm/ARM/special-syms.test
new file mode 100644
index 0000000000000000000000000000000000000000..534be1773c5b36904cf5473463812d3259bb3582
--- /dev/null
+++ b/llvm/test/tools/llvm-nm/ARM/special-syms.test
@@ -0,0 +1,30 @@
+## Test --special-syms flag for ARM mapping symbols used to mark transitions
+## between ARM code, THUMB code and data ($a, $t, $t).
+#
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-nm %t | count 0
+# RUN: llvm-nm --special-syms %t | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_ARM
+  Flags:           [ EF_ARM_EABI_VER5 ]
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    0x4
+Symbols:
+  - Name:            '$d.0'
+    Section:         .text
+  - Name:            '$a.1'
+    Section:         .text
+  - Name:            '$t.1'
+    Section:         .text
+
+# CHECK: 00000000 t $a.1
+# CHECK: 00000000 t $d.0
+# CHECK: 00000000 t $t.1
diff --git a/llvm/test/tools/llvm-nm/debug-syms.test b/llvm/test/tools/llvm-nm/debug-syms.test
index 0f970916499fe35a93780a50264e97ff6ef8e82c..e58176588d3062d629e874ec9293df8e3cf23e43 100644
--- a/llvm/test/tools/llvm-nm/debug-syms.test
+++ b/llvm/test/tools/llvm-nm/debug-syms.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s -o %t.o
-# RUN: llvm-nm --special-syms --debug-syms %t.o | FileCheck %s --implicit-check-not {{.}} --check-prefix SYMBOL
-# RUN: llvm-nm --special-syms -a %t.o | FileCheck %s --implicit-check-not {{.}} --check-prefix SYMBOL
+# RUN: llvm-nm --debug-syms %t.o | FileCheck %s --implicit-check-not {{.}} --check-prefix SYMBOL
+# RUN: llvm-nm -a %t.o | FileCheck %s --implicit-check-not {{.}} --check-prefix SYMBOL
 
 # SYMBOL:      0000000000000000 n $a
 # SYMBOL-NEXT: 0000000000000000 n $d
diff --git a/llvm/test/tools/llvm-nm/just-symbol-name.test b/llvm/test/tools/llvm-nm/just-symbol-name.test
deleted file mode 100644
index 85be79a6a24092c2a09b4d761fb7a29f471ba240..0000000000000000000000000000000000000000
--- a/llvm/test/tools/llvm-nm/just-symbol-name.test
+++ /dev/null
@@ -1,38 +0,0 @@
-## Show that the -j/--just-symbol-name prints only the the symbol name (except
-## in posix output).
-
-# RUN: yaml2obj %s -o %t.o
-
-# RUN: llvm-nm --just-symbol-name %t.o > %t.bsd.txt
-# RUN: llvm-nm -j %t.o > %t.j.txt
-# RUN: cmp %t.bsd.txt %t.j.txt
-
-# RUN: FileCheck %s --input-file=%t.bsd.txt --implicit-check-not={{.}} --check-prefix=COMMON
-# RUN: llvm-nm -j %t.o --format=sysv | \
-# RUN:   FileCheck %s --implicit-check-not={{.}} --check-prefixes=COMMON,SYSV -DFILE=%t.o
-# RUN: llvm-nm -j %t.o --format=posix | FileCheck %s --implicit-check-not={{.}} --check-prefix=POSIX
-
-# SYSV:        Symbols from [[FILE]]:
-# SYSV-EMPTY:
-# SYSV-NEXT:   Name Value Class Type Size Line Section
-# COMMON:      {{^}}defined{{$}}
-# COMMON-NEXT: {{^}}undefined{{$}}
-
-# POSIX:      defined T 0 0
-# POSIX-NEXT: undefined U 0 0
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-Sections:
-  - Name:  .text
-    Type:  SHT_PROGBITS
-    Flags: [ SHF_ALLOC, SHF_EXECINSTR ]
-Symbols:
-  - Name:    defined
-    Section: .text
-    Binding: STB_GLOBAL
-  - Name:    undefined
-    Binding: STB_GLOBAL
diff --git a/llvm/test/tools/llvm-nm/just-symbols.test b/llvm/test/tools/llvm-nm/just-symbols.test
new file mode 100644
index 0000000000000000000000000000000000000000..53bd2e25a62ef3161cf7aab6fd42caee3ad3cbbb
--- /dev/null
+++ b/llvm/test/tools/llvm-nm/just-symbols.test
@@ -0,0 +1,53 @@
+## Show that the -j/--just-symbol-name/--format=just-symbols prints only the the
+## symbol name.
+
+# RUN: yaml2obj %s -o %t.o
+
+# RUN: llvm-nm -j %t.o > %t.txt
+# RUN: llvm-nm --just-symbol-name %t.o | diff %t.txt -
+# RUN: llvm-nm --format=just-symbols %t.o | diff %t.txt -
+# RUN: llvm-nm --format=sysv -j %t.o | diff %t.txt -
+# RUN: llvm-nm -j --format=posix %t.o | diff %t.txt -
+
+# RUN: FileCheck %s --input-file=%t.txt --implicit-check-not={{.}} --check-prefix=COMMON
+
+# COMMON:      {{^}}defined{{$}}
+# COMMON-NEXT: {{^}}undefined{{$}}
+
+# RUN: llvm-nm -j %t.o %t.o | FileCheck %s --check-prefix=MULTI1 -DFILE=%t.o
+
+# MULTI1-NOT:   {{.}}
+# MULTI1:       {{^$}}
+# MULTI1-NEXT:  [[FILE]]:
+# MULTI1-NEXT:  defined
+# MULTI1-NEXT:  undefined
+# MULTI1-EMPTY:
+# MULTI1-NEXT:  [[FILE]]:
+# MULTI1-NEXT:  defined
+# MULTI1-NEXT:  undefined
+# MULTI1-NOT:   {{.}}
+
+# RUN: llvm-nm -j --print-file-name %t.o %t.o | FileCheck %s --check-prefix=MULTI2 -DFILE=%t.o
+
+# MULTI2-NOT:   {{.}}
+# MULTI2:       [[FILE]]: defined
+# MULTI2-NEXT:  [[FILE]]: undefined
+# MULTI2-NEXT:  [[FILE]]: defined
+# MULTI2-NEXT:  [[FILE]]: undefined
+# MULTI2-NOT:   {{.}}
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+Sections:
+  - Name:  .text
+    Type:  SHT_PROGBITS
+    Flags: [ SHF_ALLOC, SHF_EXECINSTR ]
+Symbols:
+  - Name:    defined
+    Section: .text
+    Binding: STB_GLOBAL
+  - Name:    undefined
+    Binding: STB_GLOBAL
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-failure.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-failure.s
index ff0a2920c4f690d8f52c9234356fed6f56db87a7..98b8b9a4f0b60b434ae2eb1e30a5173cec9fce8d 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-failure.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-failure.s
@@ -19,11 +19,12 @@
 my_kernel.kd:
   .long 0x00000000           ;; group_segment_fixed_size
   .long 0x00000000           ;; private_segment_fixed_size
-  .quad 0x00FF000000000000   ;; reserved bytes.
+  .long 0x00000000           ;; kernarg_segment_size.
+  .long 0x00000000           ;; reserved bytes.
   .quad 0x0000000000000000   ;; kernel_code_entry_byte_offset, any value works.
 
   ;; 20 reserved bytes.
-  .quad 0x0000000000000000
+  .quad 0x00FF000000000000   ;; reserved bytes.
   .quad 0x0000000000000000
   .long 0x00000000
 
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-sgpr.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-sgpr.s
index 76ccc904b88393559a8035067bca174ee1c0da1e..e397932585bb3450d1295788d50cfe4be3f88605 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-sgpr.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-sgpr.s
@@ -2,19 +2,19 @@
 
 ; RUN: split-file %s %t.dir
 
-; RUN: llvm-mc %t.dir/1.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t1
+; RUN: llvm-mc %t.dir/1.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t1
 ; RUN: llvm-objdump --disassemble-symbols=my_kernel_1.kd %t1 | tail -n +8 \
-; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t1-re-assemble
+; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t1-re-assemble
 ; RUN: diff %t1 %t1-re-assemble
 
-; RUN: llvm-mc %t.dir/2.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t2
+; RUN: llvm-mc %t.dir/2.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t2
 ; RUN: llvm-objdump --disassemble-symbols=my_kernel_2.kd %t2 | tail -n +8 \
-; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t2-re-assemble
+; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t2-re-assemble
 ; RUN: diff %t2 %t2-re-assemble
 
-; RUN: llvm-mc %t.dir/3.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t3
+; RUN: llvm-mc %t.dir/3.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t3
 ; RUN: llvm-objdump --disassemble-symbols=my_kernel_3.kd %t3 | tail -n +8 \
-; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t3-re-assemble
+; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t3-re-assemble
 ; RUN: diff %t3 %t3-re-assemble
 
 
@@ -34,7 +34,7 @@
   .amdhsa_next_free_vgpr 0
   .amdhsa_next_free_sgpr 0
   .amdhsa_reserve_flat_scratch 1
-  .amdhsa_reserve_xnack_mask 1
+  .amdhsa_reserve_xnack_mask 0
   .amdhsa_reserve_vcc 1
 .end_amdhsa_kernel
 
@@ -44,6 +44,6 @@
   .amdhsa_next_free_vgpr 0
   .amdhsa_next_free_sgpr 35
   .amdhsa_reserve_flat_scratch 1
-  .amdhsa_reserve_xnack_mask 1
+  .amdhsa_reserve_xnack_mask 0
   .amdhsa_reserve_vcc 1
 .end_amdhsa_kernel
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-vgpr.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-vgpr.s
index 2176f5852dc1976cd6ee13c7b8938beb91adb700..ba6e1d60b207eb6c07945d0f10b9e24b79f57e9f 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-vgpr.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-vgpr.s
@@ -2,19 +2,19 @@
 
 ; RUN: split-file %s %t.dir
 
-; RUN: llvm-mc %t.dir/1.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t1
+; RUN: llvm-mc %t.dir/1.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t1
 ; RUN: llvm-objdump --disassemble-symbols=my_kernel_1.kd %t1 | tail -n +8 \
-; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t1-re-assemble
+; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t1-re-assemble
 ; RUN: diff %t1 %t1-re-assemble
 
-; RUN: llvm-mc %t.dir/2.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t2
+; RUN: llvm-mc %t.dir/2.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t2
 ; RUN: llvm-objdump --disassemble-symbols=my_kernel_2.kd %t2 | tail -n +8 \
-; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t2-re-assemble
+; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t2-re-assemble
 ; RUN: diff %t2 %t2-re-assemble
 
-; RUN: llvm-mc %t.dir/3.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t3
+; RUN: llvm-mc %t.dir/3.s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t3
 ; RUN: llvm-objdump --disassemble-symbols=my_kernel_3.kd %t3 | tail -n +8 \
-; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t3-re-assemble
+; RUN: | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t3-re-assemble
 ; RUN: diff %t3 %t3-re-assemble
 
 ;--- 1.s
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s
index 24e316877d3820918d5211c145a8aec9bc91fca2..761e0162c6b5d48786203ad5cd40112b50866fd3 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s
@@ -1,6 +1,6 @@
 ;; Entirely zeroed kernel descriptor (for GFX10).
 
-; RUN: llvm-mc %s --triple=amdgcn-amd-amdhsa -mcpu=gfx1010 -filetype=obj -o %t
+; RUN: llvm-mc %s --triple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=-xnack -filetype=obj -o %t
 ; RUN: llvm-objdump -s -j .text %t | FileCheck --check-prefix=OBJDUMP %s
 
 ;; TODO:
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx9.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx9.s
index 55a4e36a233d8af2cc69fed9581894901557514b..68a502492c50348f02494b39ef235492fdf1fcb7 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx9.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx9.s
@@ -1,8 +1,8 @@
 ;; Entirely zeroed kernel descriptor (for GFX9).
 
-; RUN: llvm-mc %s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t1
+; RUN: llvm-mc %s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t1
 ; RUN: llvm-objdump --disassemble-symbols=my_kernel.kd %t1 \
-; RUN: | tail -n +8 | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t2
+; RUN: | tail -n +8 | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t2
 ; RUN: diff %t1 %t2
 
 ; RUN: llvm-objdump -s -j .text %t1 | FileCheck --check-prefix=OBJDUMP %s
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-raw.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-raw.s
index 05b017975fffc31538b3f306c7774d5ee60bc661..7e98bd9f67c0357c9c8465fadad5aec0606bc36f 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-raw.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-raw.s
@@ -1,6 +1,6 @@
-; RUN: llvm-mc %s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t1
+; RUN: llvm-mc %s --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t1
 ; RUN: llvm-objdump --disassemble-symbols=my_kernel.kd %t1 \
-; RUN: | tail -n +8 | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=obj -o %t2
+; RUN: | tail -n +8 | llvm-mc --triple=amdgcn-amd-amdhsa -mcpu=gfx908 -mattr=-xnack -filetype=obj -o %t2
 ; RUN: llvm-objdump -s -j .text %t2 | FileCheck --check-prefix=OBJDUMP %s
 
 ;; Not running lit-test over gfx10 (see kd-zeroed-gfx10.s for details).
diff --git a/llvm/test/tools/llvm-objdump/X86/source-interleave-prefix.test b/llvm/test/tools/llvm-objdump/X86/source-interleave-prefix.test
index f82d403437b49323eebaf6f36e537e3c8cf9c2a0..c2fc95ed4f025fa0cc5e3ee5eb951b9252f491cd 100644
--- a/llvm/test/tools/llvm-objdump/X86/source-interleave-prefix.test
+++ b/llvm/test/tools/llvm-objdump/X86/source-interleave-prefix.test
@@ -12,8 +12,8 @@
 ; RUN: sed -e "s,SRC_COMPDIR,./Inputs,g" %p/Inputs/source-interleave.ll > %t-relative-path.ll
 ; RUN: llc -o %t-relative-path.o -filetype=obj -mtriple=x86_64-pc-linux %t-relative-path.ll
 ; RUN: llvm-objdump --prefix myprefix --source %t-relative-path.o 2>&1 | \
-; RUN:   FileCheck %s --check-prefix=CHECK-BROKEN-PREFIX -DFILE=%t-relative-path.o -DPREFIX=.
-; CHECK-BROKEN-PREFIX: warning: '[[FILE]]': failed to find source [[PREFIX]]/Inputs{{[/\\]}}source-interleave-x86_64.c
+; RUN:   FileCheck %s --check-prefix=CHECK-BROKEN-PREFIX -DFILE=%t-relative-path.o -DPREFIX=. -DCOMPDIR=/Inputs
+; CHECK-BROKEN-PREFIX: warning: '[[FILE]]': failed to find source [[PREFIX]][[COMPDIR]]{{[/\\]}}source-interleave-x86_64.c
 
 ;; Test invalid source interleave fixed by adding the correct prefix.
 
@@ -28,16 +28,64 @@
 ; RUN: sed -e "s,SRC_COMPDIR,%/p/Inputs,g" %p/Inputs/source-interleave.ll > %t-correct-prefix.ll
 ; RUN: llc -o %t-correct-prefix.o -filetype=obj -mtriple=x86_64-pc-linux %t-correct-prefix.ll
 ; RUN: llvm-objdump --prefix myprefix --source %t-correct-prefix.o 2>&1 | \
-; RUN:   FileCheck %s --check-prefix=CHECK-BROKEN-PREFIX -DFILE=%t-correct-prefix.o -DPREFIX=myprefix%/p
+; RUN:   FileCheck %s --check-prefix=CHECK-BROKEN-PREFIX -DFILE=%t-correct-prefix.o -DPREFIX=myprefix%/p -DCOMPDIR=/Inputs
 
 ;; Using only a prefix separator is the same as not using the `--prefix` option.
 
 ; RUN: llvm-objdump --prefix / --source %t-missing-prefix.o 2>&1 | \
-; RUN:   FileCheck %s --check-prefix=CHECK-BROKEN-PREFIX -DFILE=%t-missing-prefix.o -DPREFIX=''
+; RUN:   FileCheck %s --check-prefix=CHECK-BROKEN-PREFIX -DFILE=%t-missing-prefix.o -DPREFIX='' -DCOMPDIR=/Inputs
 
 ;; All trailing separators on the prefix are discarded.
 ;; The prefix 'myprefix//' is converted to 'myprefix'.
 
 ; RUN: llvm-objdump --prefix myprefix// --source %t-missing-prefix.o 2>&1 | \
-; RUN:   FileCheck %s --check-prefix=CHECK-BROKEN-PREFIX -DFILE=%t-missing-prefix.o -DPREFIX=myprefix
+; RUN:   FileCheck %s --check-prefix=CHECK-BROKEN-PREFIX -DFILE=%t-missing-prefix.o -DPREFIX=myprefix -DCOMPDIR=/Inputs
 
+;; Test invalid source interleave fixed by adding the correct prefix and
+;; stripping out an extra directory from the path.
+
+; RUN: sed -e "s,SRC_COMPDIR,/extra/Inputs,g" %p/Inputs/source-interleave.ll > %t-extra-path-prefix.ll
+; RUN: llc -o %t-extra-path-prefix.o -filetype=obj -mtriple=x86_64-pc-linux %t-extra-path-prefix.ll
+; RUN: llvm-objdump --prefix %p --prefix-strip 1 --source %t-extra-path-prefix.o 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-MISSING-PREFIX-FIX
+
+;; Test do not skip extra separators. The --prefix-strip should take into
+;; account each separator individually. Hence, to fix '/extra/Inputs'
+;; --prefix-strip needs to be 1. To fix '//extra/Inputs' --prefix-strip
+;; needs to be 2.
+
+; RUN: sed -e "s,SRC_COMPDIR,//extra/Inputs,g" %p/Inputs/source-interleave.ll > %t-extra-sep-path-prefix.ll
+; RUN: llc -o %t-extra-sep-path-prefix.o -filetype=obj -mtriple=x86_64-pc-linux %t-extra-sep-path-prefix.ll
+; RUN: llvm-objdump --prefix %p --prefix-strip 2 --source %t-extra-sep-path-prefix.o 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-MISSING-PREFIX-FIX
+
+;; Test --prefix-strip value of 0. No effect.
+;; SRC_COMPDIR is set to '/Inputs' before and after --prefix-strip 0.
+
+; RUN: llvm-objdump --prefix %p --prefix-strip 0 --source %t-missing-prefix.o 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-MISSING-PREFIX-FIX
+
+;; Test --prefix-strip value equal to the number of directory components.
+;; SRC_COMPDIR is set to '/Inputs' before --prefix-strip 1.
+;; SRC_COMPDIR becomes '' after --prefix-strip 1.
+
+; RUN: llvm-objdump --prefix %p/Inputs --prefix-strip 1 --source %t-missing-prefix.o 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-MISSING-PREFIX-FIX
+
+;; Test --prefix-strip value greater than the number of components.
+;; SRC_COMPDIR is set to '/Inputs' before --prefix-strip 2.
+;; SRC_COMPDIR becomes '' after --prefix-strip 2.
+
+; RUN: llvm-objdump --prefix %p/Inputs --prefix-strip 2 --source %t-missing-prefix.o 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-MISSING-PREFIX-FIX
+
+;; Test negative value --prefix-strip. Reports an error.
+
+; RUN: not llvm-objdump --prefix %p --prefix-strip '-1' --source %t-missing-prefix.o 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-INVALID-PREFIX-STRIP -DOPTION='-1'
+; CHECK-INVALID-PREFIX-STRIP: llvm-objdump{{.*}}: for the --prefix-strip option: '[[OPTION]]' value invalid for uint argument!
+
+;; Test text value --prefix-strip. Reports an error.
+
+; RUN: not llvm-objdump --prefix %p --prefix-strip foo --source %t-missing-prefix.o 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-INVALID-PREFIX-STRIP -DOPTION='foo'
diff --git a/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext b/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext
index eead4d4d62f05570623ee1fc566f4679ec654ead..e960dea02e6b211f244bf306b76746024c8d8e01 100644
--- a/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext
+++ b/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext
@@ -4,6 +4,7 @@
  3: 287884
  4: 287864 _Z3fibi:315608
  15: 23
+ !Attributes: 0
 [main:3.1 @ _Z5funcBi:1 @ _Z8funcLeafi]:500853:20
  0: 15
  1: 15
@@ -12,25 +13,32 @@
  10: 23324
  11: 23327 _Z3fibi:25228
  15: 11
+ !Attributes: 1
 [main]:154:0
  2: 12
  3: 18 _Z5funcAi:11
  3.1: 18 _Z5funcBi:19
+ !Attributes: 0
 [external:12 @ main]:154:12
  2: 12
  3: 10 _Z5funcAi:7
  3.1: 10 _Z5funcBi:11
+ !Attributes: 0
 [main:3.1 @ _Z5funcBi]:120:19
  0: 19
  1: 19 _Z8funcLeafi:20
  3: 12
+ !Attributes: 1
 [externalA:17 @ _Z5funcBi]:120:3
  0: 3
  1: 3
+ !Attributes: 0
 [external:10 @ _Z5funcBi]:120:10
  0: 10
  1: 10
+ !Attributes: 0
 [main:3 @ _Z5funcAi]:99:11
  0: 10
  1: 10 _Z8funcLeafi:11
  3: 24
+ !Attributes: 0
diff --git a/llvm/test/tools/llvm-profdata/Inputs/pseudo-probe-profile.proftext b/llvm/test/tools/llvm-profdata/Inputs/pseudo-probe-profile.proftext
index f4ae6d9197476e0f76c0ce80a6eed0ff24ecbd5d..82f57d6065f893769827fa49d993b8cb6202b8fe 100644
--- a/llvm/test/tools/llvm-profdata/Inputs/pseudo-probe-profile.proftext
+++ b/llvm/test/tools/llvm-profdata/Inputs/pseudo-probe-profile.proftext
@@ -6,3 +6,4 @@ foo:3200:13
  5: 7 _Z3foov:5 _Z3barv:2
  6: 6 _Z3barv:4 _Z3foov:2
  !CFGChecksum: 563022570642068
+ !Attributes: 0
diff --git a/llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test b/llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test
index cb414c2e6c06e75963eb83a2b6c931994ef96b70..5fc87475f505e529681608f8667f8891614036ed 100644
--- a/llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test
+++ b/llvm/test/tools/llvm-profgen/inline-cs-pseudoprobe.test
@@ -9,7 +9,7 @@
 ; CHECK-NEXT: 6: 15
 ; CHECK-NEXT: 8: 14 bar:14
 ; CHECK-NEXT: !CFGChecksum: 138950591924
-; CHECK-NEXT:[main:2 @ foo:8 @ bar]:28:14
+; CHECK:[main:2 @ foo:8 @ bar]:28:14
 ; CHECK-NEXT: 1: 14
 ; CHECK-NEXT: 2: 18446744073709551615
 ; CHECK-NEXT: 3: 18446744073709551615
diff --git a/llvm/test/tools/llvm-profgen/merge-cold-profile.test b/llvm/test/tools/llvm-profgen/merge-cold-profile.test
index e0c65ac44e2b9cca4b73c718e0338a9a40b4dac2..0549befd34e9d657349efef4f05af1fe2e815d13 100644
--- a/llvm/test/tools/llvm-profgen/merge-cold-profile.test
+++ b/llvm/test/tools/llvm-profgen/merge-cold-profile.test
@@ -2,10 +2,14 @@
 ; RUN: llvm-profgen --perfscript=%S/Inputs/recursion-compression-pseudoprobe.perfscript --binary=%S/Inputs/recursion-compression-pseudoprobe.perfbin --output=%t --compress-recursion=-1 --csprof-cold-thres=8
 ; RUN: FileCheck %s --input-file %t
 
-; Test --csprof-keep-cold
-; RUN: llvm-profgen --perfscript=%S/Inputs/recursion-compression-pseudoprobe.perfscript --binary=%S/Inputs/recursion-compression-pseudoprobe.perfbin --output=%t --compress-recursion=-1 --csprof-cold-thres=100 --csprof-keep-cold
+; Test --csprof-trim-cold-context=0
+; RUN: llvm-profgen --perfscript=%S/Inputs/recursion-compression-pseudoprobe.perfscript --binary=%S/Inputs/recursion-compression-pseudoprobe.perfbin --output=%t --compress-recursion=-1 --csprof-cold-thres=100 --csprof-trim-cold-context=0
 ; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-KEEP-COLD
 
+; Test --csprof-merge-cold-context=0
+; RUN: llvm-profgen --perfscript=%S/Inputs/recursion-compression-pseudoprobe.perfscript --binary=%S/Inputs/recursion-compression-pseudoprobe.perfbin --output=%t --compress-recursion=-1 --csprof-cold-thres=10 --csprof-merge-cold-context=0
+; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-UNMERGED
+
 ; CHECK:     [fa]:14:4
 ; CHECK-NEXT: 1: 4
 ; CHECK-NEXT: 3: 4
@@ -14,6 +18,7 @@
 ; CHECK-NEXT: 7: 2 fb:2
 ; CHECK-NEXT: 8: 1 fa:1
 ; CHECK-NEXT: !CFGChecksum: 120515930909
+; CHECK-NEXT: !Attributes: 0
 ; CHECK-NEXT:[main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb]:13:4
 ; CHECK-NEXT: 1: 4
 ; CHECK-NEXT: 2: 3
@@ -29,6 +34,7 @@
 ; CHECK-KEEP-COLD-NEXT: 5: 4 fb:4
 ; CHECK-KEEP-COLD-NEXT: 6: 3 fa:3
 ; CHECK-KEEP-COLD-NEXT: !CFGChecksum: 72617220756
+; CHECK-KEEP-COLD-NEXT: !Attributes: 0
 ; CHECK-KEEP-COLD-NEXT:[fa]:14:4
 ; CHECK-KEEP-COLD-NEXT: 1: 4
 ; CHECK-KEEP-COLD-NEXT: 3: 4
@@ -38,6 +44,16 @@
 ; CHECK-KEEP-COLD-NEXT: 8: 1 fa:1
 ; CHECK-KEEP-COLD-NEXT: !CFGChecksum: 120515930909
 
+; CHECK-UNMERGED:     [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb]:13:4
+; CHECK-UNMERGED-NEXT: 1: 4
+; CHECK-UNMERGED-NEXT: 2: 3
+; CHECK-UNMERGED-NEXT: 3: 1
+; CHECK-UNMERGED-NEXT: 5: 4 fb:4
+; CHECK-UNMERGED-NEXT: 6: 1 fa:1
+; CHECK-UNMERGED-NEXT: !CFGChecksum: 72617220756
+; CHECK-UNMERGED-NOT: [fa]
+; CHECK-UNMERGED-NOT: [fb]
+
 
 ; clang -O3 -fexperimental-new-pass-manager -fuse-ld=lld -fpseudo-probe-for-profiling
 ; -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -Xclang -mdisable-tail-calls
diff --git a/llvm/test/tools/llvm-profgen/noinline-cs-pseudoprobe.test b/llvm/test/tools/llvm-profgen/noinline-cs-pseudoprobe.test
index 64a8b052ab936af730ff0056b1f1ec36257e7aed..c4edb978bfcaf5bb2ecdafd8dba7d7cdac84c596 100644
--- a/llvm/test/tools/llvm-profgen/noinline-cs-pseudoprobe.test
+++ b/llvm/test/tools/llvm-profgen/noinline-cs-pseudoprobe.test
@@ -8,7 +8,7 @@
 ; CHECK-NEXT: 6: 15
 ; CHECK-NEXT: 8: 15 bar:15
 ; CHECK-NEXT: !CFGChecksum: 138950591924
-; CHECK-NEXT:[main:2 @ foo:8 @ bar]:30:15
+; CHECK:[main:2 @ foo:8 @ bar]:30:15
 ; CHECK-NEXT: 1: 15
 ; CHECK-NEXT: 2: 18446744073709551615
 ; CHECK-NEXT: 3: 18446744073709551615
diff --git a/llvm/test/tools/llvm-readobj/ELF/amdgpu-elf-headers.test b/llvm/test/tools/llvm-readobj/ELF/amdgpu-elf-headers.test
index 7a70933c340ca8dfedf33152ca87e8beec988efd..157ac6aa9d041158b495fa7b0c920b56ad054db6 100644
--- a/llvm/test/tools/llvm-readobj/ELF/amdgpu-elf-headers.test
+++ b/llvm/test/tools/llvm-readobj/ELF/amdgpu-elf-headers.test
@@ -1,127 +1,338 @@
-# RUN: yaml2obj %s -o %t -DCPU=GFX600
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX600 -DFLAGS=0x20
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX600
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX600 -DFLAG_VALUE=0x20
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX601
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX601 -DFLAGS=0x21
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX600
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX600 -DFLAG_VALUE=0x20
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX602
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX602 -DFLAGS=0x3A
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX600
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX600 -DFLAG_VALUE=0x20
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX700
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX700 -DFLAGS=0x22
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX601
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX601 -DFLAG_VALUE=0x21
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX701
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX701 -DFLAGS=0x23
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX601
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX601 -DFLAG_VALUE=0x21
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX702
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX702 -DFLAGS=0x24
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX601
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX601 -DFLAG_VALUE=0x21
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX703
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX703 -DFLAGS=0x25
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX602
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX602 -DFLAG_VALUE=0x3A
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX704
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX704 -DFLAGS=0x26
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX602
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX602 -DFLAG_VALUE=0x3A
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX705
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX705 -DFLAGS=0x3B
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX602
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX602 -DFLAG_VALUE=0x3A
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX801
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX801 -DFLAGS=0x28
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX700
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX700 -DFLAG_VALUE=0x22
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX802
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX802 -DFLAGS=0x29
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX700
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX700 -DFLAG_VALUE=0x22
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX803
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX803 -DFLAGS=0x2A
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX700
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX700 -DFLAG_VALUE=0x22
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX805
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX805 -DFLAGS=0x3C
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX701
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX701 -DFLAG_VALUE=0x23
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX810
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX810 -DFLAGS=0x2B
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX701
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX701 -DFLAG_VALUE=0x23
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX900
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX900 -DFLAGS=0x2C
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX701
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX701 -DFLAG_VALUE=0x23
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX902
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX902 -DFLAGS=0x2D
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX702
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX702 -DFLAG_VALUE=0x24
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX904
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX904 -DFLAGS=0x2E
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX702
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX702 -DFLAG_VALUE=0x24
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX906
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX906 -DFLAGS=0x2F
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX702
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX702 -DFLAG_VALUE=0x24
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX908
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX908 -DFLAGS=0x30
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX703
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX703 -DFLAG_VALUE=0x25
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX909
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX909 -DFLAGS=0x31
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX703
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX703 -DFLAG_VALUE=0x25
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX90A
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX90A -DFLAGS=0x3F
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX703
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX703 -DFLAG_VALUE=0x25
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX90C
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX90C -DFLAGS=0x32
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX704
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX704 -DFLAG_VALUE=0x26
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX1010
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX1010 -DFLAGS=0x33
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX704
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX704 -DFLAG_VALUE=0x26
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX1011
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX1011 -DFLAGS=0x34
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX704
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX704 -DFLAG_VALUE=0x26
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX1012
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX1012 -DFLAGS=0x35
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX705
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX705 -DFLAG_VALUE=0x3B
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX1030
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX1030 -DFLAGS=0x36
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX705
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX705 -DFLAG_VALUE=0x3B
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX1031
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX1031 -DFLAGS=0x37
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX705
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX705 -DFLAG_VALUE=0x3B
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX1032
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX1032 -DFLAGS=0x38
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX801
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX801 -DFLAG_VALUE=0x28
 
-# RUN: yaml2obj %s -o %t -DCPU=GFX1033
-# RUN: llvm-readobj -h %t | FileCheck %s --match-full-lines -DFILE=%t -DCPU=GFX1033 -DFLAGS=0x39
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX801
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX801 -DFLAG_VALUE=0x28
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX801
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX801 -DFLAG_VALUE=0x28
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX802
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX802 -DFLAG_VALUE=0x29
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX802
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX802 -DFLAG_VALUE=0x29
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX802
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX802 -DFLAG_VALUE=0x29
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX803
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX803 -DFLAG_VALUE=0x2A
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX803
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX803 -DFLAG_VALUE=0x2A
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX803
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX803 -DFLAG_VALUE=0x2A
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX805
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX805 -DFLAG_VALUE=0x3C
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX805
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX805 -DFLAG_VALUE=0x3C
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX805
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX805 -DFLAG_VALUE=0x3C
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX810
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX810 -DFLAG_VALUE=0x2B
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX810
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX810 -DFLAG_VALUE=0x2B
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX810
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX810 -DFLAG_VALUE=0x2B
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX900
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX900 -DFLAG_VALUE=0x2C
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX900
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX900 -DFLAG_VALUE=0x2C
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX900
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX900 -DFLAG_VALUE=0x2C
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX902
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX902 -DFLAG_VALUE=0x2D
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX902
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX902 -DFLAG_VALUE=0x2D
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX902
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX902 -DFLAG_VALUE=0x2D
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX904
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX904 -DFLAG_VALUE=0x2E
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX904
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX904 -DFLAG_VALUE=0x2E
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX904
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX904 -DFLAG_VALUE=0x2E
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX906
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX906 -DFLAG_VALUE=0x2F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX906
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX906 -DFLAG_VALUE=0x2F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX906
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX906 -DFLAG_VALUE=0x2F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX908
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX908 -DFLAG_VALUE=0x30
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX908
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX908 -DFLAG_VALUE=0x30
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX908
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX908 -DFLAG_VALUE=0x30
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX909
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX909 -DFLAG_VALUE=0x31
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX909
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX909 -DFLAG_VALUE=0x31
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX909
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX909 -DFLAG_VALUE=0x31
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90A
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90A -DFLAG_VALUE=0x3F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90A
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90A -DFLAG_VALUE=0x3F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90A
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90A -DFLAG_VALUE=0x3F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90C
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90C -DFLAG_VALUE=0x32
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90C
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90C -DFLAG_VALUE=0x32
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90C
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90C -DFLAG_VALUE=0x32
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1010
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1010 -DFLAG_VALUE=0x33
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1010
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1010 -DFLAG_VALUE=0x33
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1010
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1010 -DFLAG_VALUE=0x33
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1011
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1011 -DFLAG_VALUE=0x34
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1011
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1011 -DFLAG_VALUE=0x34
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1011
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1011 -DFLAG_VALUE=0x34
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1012
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1012 -DFLAG_VALUE=0x35
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1012
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1012 -DFLAG_VALUE=0x35
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1012
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1012 -DFLAG_VALUE=0x35
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1030
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1030 -DFLAG_VALUE=0x36
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1030
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1030 -DFLAG_VALUE=0x36
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1030
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1030 -DFLAG_VALUE=0x36
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1031
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1031 -DFLAG_VALUE=0x37
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1031
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1031 -DFLAG_VALUE=0x37
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1031
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1031 -DFLAG_VALUE=0x37
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1032
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1032 -DFLAG_VALUE=0x38
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1032
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1032 -DFLAG_VALUE=0x38
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1032
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1032 -DFLAG_VALUE=0x38
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1033
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1033 -DFLAG_VALUE=0x39
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1033
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1033 -DFLAG_VALUE=0x39
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1033
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1033 -DFLAG_VALUE=0x39
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_XNACK_V3"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_XNACK_V3 (0x100)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0x13F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_XNACK_V3"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_XNACK_V3 (0x100)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0x13F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_SRAMECC_V3"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_SRAMECC_V3 (0x200)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0x23F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_SRAMECC_V3"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_SRAMECC_V3 (0x200)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0x23F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_XNACK_ANY_V4"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_XNACK_ANY_V4 (0x100)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0x13F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_XNACK_OFF_V4"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0x23F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_XNACK_ON_V4"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0x33F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_SRAMECC_ANY_V4"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_SRAMECC_ANY_V4 (0x400)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0x43F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_SRAMECC_OFF_V4"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_SRAMECC_OFF_V4 (0x800)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0x83F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME="EF_AMDGPU_MACH_AMDGCN_GFX90A, EF_AMDGPU_FEATURE_SRAMECC_ON_V4"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,DOUBLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_0="EF_AMDGPU_FEATURE_SRAMECC_ON_V4 (0xC00)" -DFLAG_1="EF_AMDGPU_MACH_AMDGCN_GFX90A (0x3F)" -DFLAG_VALUE=0xC3F
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=16 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX90A
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,UNKNOWN-ABI-VERSION --match-full-lines -DABI_VERSION=16 -DFILE=%t -DFLAG_VALUE=0x3F
 
 --- !ELF
 FileHeader:
   Class:           ELFCLASS64
   Data:            ELFDATA2LSB
   OSABI:           ELFOSABI_AMDGPU_HSA
+  ABIVersion:      [[ABI_VERSION]]
   Type:            ET_REL
   Machine:         EM_AMDGPU
-  Flags:           [ EF_AMDGPU_MACH_AMDGCN_[[CPU]] ]
-
-# CHECK:      File: [[FILE]]
-# CHECK-NEXT: Format: elf64-amdgpu
-# CHECK-NEXT: Arch: amdgcn
-# CHECK-NEXT: AddressSize: 64bit
-# CHECK-NEXT: LoadName: <Not found>
-# CHECK-NEXT: ElfHeader {
-# CHECK-NEXT:   Ident {
-# CHECK-NEXT:     Magic: (7F 45 4C 46)
-# CHECK-NEXT:     Class: 64-bit (0x2)
-# CHECK-NEXT:     DataEncoding: LittleEndian (0x1)
-# CHECK-NEXT:     FileVersion: 1
-# CHECK-NEXT:     OS/ABI: AMDGPU_HSA (0x40)
-# CHECK-NEXT:     ABIVersion: 0
-# CHECK-NEXT:     Unused: (00 00 00 00 00 00 00)
-# CHECK-NEXT:   }
-# CHECK-NEXT:   Type: Relocatable (0x1)
-# CHECK-NEXT:   Machine: EM_AMDGPU (0xE0)
-# CHECK-NEXT:   Version: 1
-# CHECK-NEXT:   Entry: 0x0
-# CHECK-NEXT:   ProgramHeaderOffset: 0x0
-# CHECK-NEXT:   SectionHeaderOffset: 0x58
-# CHECK-NEXT:     Flags [ ([[FLAGS]])
-# CHECK-NEXT:     EF_AMDGPU_MACH_AMDGCN_[[CPU]] ([[FLAGS]])
-# CHECK-NEXT:   ]
-# CHECK-NEXT:   HeaderSize: 64
-# CHECK-NEXT:   ProgramHeaderEntrySize: 0
-# CHECK-NEXT:   ProgramHeaderCount: 0
-# CHECK-NEXT:   SectionHeaderEntrySize: 64
-# CHECK-NEXT:   SectionHeaderCount: 3
-# CHECK-NEXT:   StringTableSectionIndex: 2
-# CHECK-NEXT: }
+  Flags:           [ [[FLAG_NAME]] ]
+
+# ALL:                      File: [[FILE]]
+# ALL-NEXT:                 Format: elf64-amdgpu
+# ALL-NEXT:                 Arch: amdgcn
+# ALL-NEXT:                 AddressSize: 64bit
+# ALL-NEXT:                 LoadName: <Not found>
+# ALL-NEXT:                 ElfHeader {
+# ALL-NEXT:                   Ident {
+# ALL-NEXT:                     Magic: (7F 45 4C 46)
+# ALL-NEXT:                     Class: 64-bit (0x2)
+# ALL-NEXT:                     DataEncoding: LittleEndian (0x1)
+# ALL-NEXT:                     FileVersion: 1
+# ALL-NEXT:                     OS/ABI: AMDGPU_HSA (0x40)
+# ALL-NEXT:                     ABIVersion: [[ABI_VERSION]]
+# ALL-NEXT:                     Unused: (00 00 00 00 00 00 00)
+# ALL-NEXT:                   }
+# ALL-NEXT:                   Type: Relocatable (0x1)
+# ALL-NEXT:                   Machine: EM_AMDGPU (0xE0)
+# ALL-NEXT:                   Version: 1
+# ALL-NEXT:                   Entry: 0x0
+# ALL-NEXT:                   ProgramHeaderOffset: 0x0
+# ALL-NEXT:                   SectionHeaderOffset: 0x58
+# KNOWN-ABI-VERSION-NEXT:     Flags [ ([[FLAG_VALUE]])
+# SINGLE-FLAG-NEXT:             [[FLAG_NAME]] ([[FLAG_VALUE]])
+# DOUBLE-FLAG-NEXT:             [[FLAG_0]]
+# DOUBLE-FLAG-NEXT:             [[FLAG_1]]
+# KNOWN-ABI-VERSION-NEXT:     ]
+# UNKNOWN-ABI-VERSION-NEXT:   Flags: [[FLAG_VALUE]]
+# ALL-NEXT:                   HeaderSize: 64
+# ALL-NEXT:                   ProgramHeaderEntrySize: 0
+# ALL-NEXT:                   ProgramHeaderCount: 0
+# ALL-NEXT:                   SectionHeaderEntrySize: 64
+# ALL-NEXT:                   SectionHeaderCount: 3
+# ALL-NEXT:                   StringTableSectionIndex: 2
+# ALL-NEXT:                 }
diff --git a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test
index f9b9dc0980e5a560498e91218d22b335f4ca2250..0cebfee6872acfc05d21d5ad13f5f9da7f0d34a4 100644
--- a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test
+++ b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test
@@ -24,6 +24,7 @@
 # LLVM-NEXT:         HasReturn: No
 # LLVM-NEXT:         HasTailCall: Yes
 # LLVM-NEXT:         IsEHPad: No
+# LLVM-NEXT:         CanFallThrough: No
 # LLVM-NEXT:       }
 # LLVM-NEXT:       {
 # LLVM-NEXT:         Offset: 0x3
@@ -31,6 +32,7 @@
 # LLVM-NEXT:         HasReturn: Yes
 # LLVM-NEXT:         HasTailCall: No
 # LLVM-NEXT:         IsEHPad: Yes
+# LLVM-NEXT:         CanFallThrough: No
 # LLVM-NEXT:       }
 # LLVM-NEXT:     ]
 # LLVM-NEXT:   }
@@ -43,6 +45,7 @@
 # LLVM-NEXT:         HasReturn: No
 # LLVM-NEXT:         HasTailCall: No
 # LLVM-NEXT:         IsEHPad: No
+# LLVM-NEXT:         CanFallThrough: Yes
 # LLVM-NEXT:       }
 # LLVM-NEXT:     ]
 # LLVM-NEXT:   }
@@ -57,6 +60,7 @@
 # LLVM-NEXT:         HasReturn: Yes
 # LLVM-NEXT:         HasTailCall: Yes
 # LLVM-NEXT:         IsEHPad: No
+# LLVM-NEXT:         CanFallThrough: Yes
 # LLVM-NEXT:       }
 # LLVM-NEXT:     ]
 # LLVM-NEXT:   }
@@ -78,6 +82,7 @@
 # TRUNCATED-NEXT:         HasReturn: Yes
 # TRUNCATED-NEXT:         HasTailCall: Yes
 # TRUNCATED-NEXT:         IsEHPad: No
+# TRUNCATED-NEXT:         CanFallThrough: Yes
 # TRUNCATED-NEXT:       }
 # TRUNCATED-NEXT:     ]
 # TRUNCATED-NEXT:   }
diff --git a/llvm/test/tools/llvm-readobj/ELF/note-amd.s b/llvm/test/tools/llvm-readobj/ELF/note-amd.s
index c54ebd7aa32c355ea72cacaee862b068c050824b..260be3a725af761db6ef1f2d6dca7f117212a15f 100644
--- a/llvm/test/tools/llvm-readobj/ELF/note-amd.s
+++ b/llvm/test/tools/llvm-readobj/ELF/note-amd.s
@@ -6,25 +6,27 @@
 
 // GNU:      Displaying notes found in: .note.no.desc
 // GNU-NEXT:   Owner                Data size        Description
-// GNU-NEXT:   AMD                  0x00000000       NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
-// GNU-NEXT:     HSA Metadata:
+// GNU-NEXT:   AMD                  0x00000000       NT_AMD_HSA_METADATA (AMD HSA Metadata)
+// GNU-NEXT:     AMD HSA Metadata:
 // GNU-NEXT: {{^        $}}
-// GNU-NEXT:   AMD                  0x00000000       NT_AMD_AMDGPU_ISA (ISA Version)
-// GNU-NEXT:     ISA Version:
+// GNU-NEXT:   AMD                  0x00000000       NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)
+// GNU-NEXT:     AMD HSA ISA Name:
 // GNU-NEXT: {{^        $}}
 // GNU-EMPTY:
 // GNU-NEXT: Displaying notes found in: .note.desc
 // GNU-NEXT:   Owner                Data size        Description
-// GNU-NEXT:   AMD                  0x0000000a       NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
-// GNU-NEXT:     HSA Metadata:
+// GNU-NEXT:   AMD                  0x0000000a       NT_AMD_HSA_METADATA (AMD HSA Metadata)
+// GNU-NEXT:     AMD HSA Metadata:
 // GNU-NEXT:     meta_blah
-// GNU-NEXT:   AMD                  0x00000009       NT_AMD_AMDGPU_ISA (ISA Version)
-// GNU-NEXT:     ISA Version:
+// GNU-NEXT:   AMD                  0x00000009       NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)
+// GNU-NEXT:     AMD HSA ISA Name:
 // GNU-NEXT:     isa_blah
 // GNU-EMPTY:
 // GNU-NEXT: Displaying notes found in: .note.other
 // GNU-NEXT:   Owner                Data size        Description
-// GNU-NEXT:   AMD                  0x00000000       NT_AMD_AMDGPU_PAL_METADATA (PAL Metadata)
+// GNU-NEXT:   AMD                  0x00000000       NT_AMD_PAL_METADATA (AMD PAL Metadata)
+// GNU-NEXT:     AMD PAL Metadata:
+// GNU-NEXT: {{^        $}}
 // GNU-EMPTY:
 // GNU-NEXT: Displaying notes found in: .note.unknown
 // GNU-NEXT:   Owner                Data size 	Description
@@ -40,14 +42,14 @@
 // LLVM-NEXT:     Note {
 // LLVM-NEXT:       Owner: AMD
 // LLVM-NEXT:       Data size: 0x0
-// LLVM-NEXT:       Type: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
-// LLVM-NEXT:       HSA Metadata:
+// LLVM-NEXT:       Type: NT_AMD_HSA_METADATA (AMD HSA Metadata)
+// LLVM-NEXT:       AMD HSA Metadata:
 // LLVM-NEXT:     }
 // LLVM-NEXT:     Note {
 // LLVM-NEXT:       Owner: AMD
 // LLVM-NEXT:       Data size: 0x0
-// LLVM-NEXT:       Type: NT_AMD_AMDGPU_ISA (ISA Version)
-// LLVM-NEXT:       ISA Version:
+// LLVM-NEXT:       Type: NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)
+// LLVM-NEXT:       AMD HSA ISA Name:
 // LLVM-NEXT:     }
 // LLVM-NEXT:   }
 // LLVM-NEXT:   NoteSection {
@@ -57,14 +59,14 @@
 // LLVM-NEXT:     Note {
 // LLVM-NEXT:       Owner: AMD
 // LLVM-NEXT:       Data size: 0xA
-// LLVM-NEXT:       Type: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
-// LLVM-NEXT:       HSA Metadata: meta_blah
+// LLVM-NEXT:       Type: NT_AMD_HSA_METADATA (AMD HSA Metadata)
+// LLVM-NEXT:       AMD HSA Metadata: meta_blah
 // LLVM-NEXT:     }
 // LLVM-NEXT:     Note {
 // LLVM-NEXT:       Owner: AMD
 // LLVM-NEXT:       Data size: 0x9
-// LLVM-NEXT:       Type: NT_AMD_AMDGPU_ISA (ISA Version)
-// LLVM-NEXT:       ISA Version: isa_blah
+// LLVM-NEXT:       Type: NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)
+// LLVM-NEXT:       AMD HSA ISA Name: isa_blah
 // LLVM-NEXT:     }
 // LLVM-NEXT:   }
 // LLVM-NEXT:   NoteSection {
@@ -74,7 +76,8 @@
 // LLVM-NEXT:     Note {
 // LLVM-NEXT:       Owner: AMD
 // LLVM-NEXT:       Data size: 0x0
-// LLVM-NEXT:       Type: NT_AMD_AMDGPU_PAL_METADATA (PAL Metadata)
+// LLVM-NEXT:       Type: NT_AMD_PAL_METADATA (AMD PAL Metadata)
+// LLVM-NEXT:       AMD PAL Metadata:
 // LLVM-NEXT:     }
 // LLVM-NEXT:   }
 // LLVM-NEXT:   NoteSection {
@@ -96,17 +99,17 @@
 	.align 4
 	.long 4 /* namesz */
 	.long 0 /* descsz */
-	.long 10 /* type = NT_AMD_AMDGPU_HSA_METADATA */
+	.long 10 /* type = NT_AMD_HSA_METADATA */
 	.asciz "AMD"
 	.long 4 /* namesz */
 	.long 0 /* descsz */
-	.long 11 /* type = NT_AMD_AMDGPU_ISA */
+	.long 11 /* type = NT_AMD_HSA_ISA_NAME */
 	.asciz "AMD"
 .section ".note.desc", "a"
 	.align 4
 	.long 4 /* namesz */
 	.long end.meta - begin.meta /* descsz */
-	.long 10 /* type = NT_AMD_AMDGPU_HSA_METADATA */
+	.long 10 /* type = NT_AMD_HSA_METADATA */
 	.asciz "AMD"
 begin.meta:
 	.asciz "meta_blah"
@@ -114,7 +117,7 @@ end.meta:
 	.align 4
 	.long 4 /* namesz */
 	.long end.isa - begin.isa /* descsz */
-	.long 11 /* type = NT_AMD_AMDGPU_ISA */
+	.long 11 /* type = NT_AMD_HSA_ISA_NAME */
 	.asciz "AMD"
 begin.isa:
 	.asciz "isa_blah"
@@ -124,7 +127,7 @@ end.isa:
 	.align 4
 	.long 4 /* namesz */
 	.long 0 /* descsz */
-	.long 12 /* type = NT_AMD_AMDGPU_PAL_METADATA */
+	.long 12 /* type = NT_AMD_PAL_METADATA */
 	.asciz "AMD"
 .section ".note.unknown", "a"
 	.align 4
diff --git a/llvm/test/tools/llvm-readobj/ELF/note-gnu-property.s b/llvm/test/tools/llvm-readobj/ELF/note-gnu-property.s
index 8c3a40ad2f69b612408da255bf4d8bb90004a691..2d0d00f60639153573aff7d488823934dac671a6 100644
--- a/llvm/test/tools/llvm-readobj/ELF/note-gnu-property.s
+++ b/llvm/test/tools/llvm-readobj/ELF/note-gnu-property.s
@@ -12,10 +12,10 @@
 // GNU-NEXT:     x86 feature: SHSTK
 // GNU-NEXT:     x86 feature: IBT, SHSTK
 // GNU-NEXT:     x86 feature: <None>
-// GNU-NEXT:     x86 ISA needed: CMOV, SSE, SSE2, SSE3, SSSE3, SSE4_1, SSE4_2, AVX, AVX2, FMA, AVX512F, AVX512CD
-// GNU-NEXT:     x86 ISA used: AVX512ER, AVX512PF, AVX512VL, AVX512DQ, AVX512BW, AVX512_4FMAPS, AVX512_4VNNIW, AVX512_BITALG, AVX512_IFMA, AVX512_VBMI, AVX512_VBMI2, AVX512_VNNI
 // GNU-NEXT:     x86 feature needed: x86, x87, MMX, XMM, YMM
 // GNU-NEXT:     x86 feature used: ZMM, FXSR, XSAVE, XSAVEOPT, XSAVEC
+// GNU-NEXT:     x86 ISA needed: x86-64-baseline, x86-64-v2, x86-64-v3, x86-64-v4
+// GNU-NEXT:     x86 ISA used: x86-64-baseline, x86-64-v2, x86-64-v3, x86-64-v4
 // GNU-NEXT:     <application-specific type 0xfefefefe>
 // GNU-NEXT:     stack size: <corrupt length: 0x0>
 // GNU-NEXT:     stack size: <corrupt length: 0x4>
@@ -40,10 +40,10 @@
 // LLVM-NEXT:         x86 feature: SHSTK
 // LLVM-NEXT:         x86 feature: IBT, SHSTK
 // LLVM-NEXT:         x86 feature: <None>
-// LLVM-NEXT:         x86 ISA needed: CMOV, SSE, SSE2, SSE3, SSSE3, SSE4_1, SSE4_2, AVX, AVX2, FMA, AVX512F, AVX512CD
-// LLVM-NEXT:         x86 ISA used: AVX512ER, AVX512PF, AVX512VL, AVX512DQ, AVX512BW, AVX512_4FMAPS, AVX512_4VNNIW, AVX512_BITALG, AVX512_IFMA, AVX512_VBMI, AVX512_VBMI2, AVX512_VNNI
 // LLVM-NEXT:         x86 feature needed: x86, x87, MMX, XMM, YMM
 // LLVM-NEXT:         x86 feature used: ZMM, FXSR, XSAVE, XSAVEOPT, XSAVEC
+// LLVM-NEXT:         x86 ISA needed: x86-64-baseline, x86-64-v2, x86-64-v3, x86-64-v4
+// LLVM-NEXT:         x86 ISA used: x86-64-baseline, x86-64-v2, x86-64-v3, x86-64-v4
 // LLVM-NEXT:         <application-specific type 0xfefefefe>
 // LLVM-NEXT:         stack size: <corrupt length: 0x0>
 // LLVM-NEXT:         stack size: <corrupt length: 0x4>
@@ -96,24 +96,24 @@ begin:
   .long 0           /* Empty flags, not an error */
   .p2align 3        /* Align to 8 byte for 64 bit */
 
-  .long 0xc0008000         /* Type: GNU_PROPERTY_X86_ISA_1_NEEDED */
+  .long 0xc0008001         /* Type: GNU_PROPERTY_X86_FEATURE_2_NEEDED */
   .long 4                  /* Data size */
-  .long 0x00000fff         /* CMOV, ... */
+  .long 0x0000001f         /* X86, ... */
   .p2align 3               /* Align to 8 byte for 64 bit */
 
-  .long 0xc0010000         /* Type: GNU_PROPERTY_X86_ISA_1_USED */
+  .long 0xc0010001         /* Type: GNU_PROPERTY_X86_FEATURE_2_USED */
   .long 4                  /* Data size */
-  .long 0x00fff000         /* AVX512_ER, ... */
+  .long 0x000003e0         /* ZMM, ... */
   .p2align 3               /* Align to 8 byte for 64 bit */
 
-  .long 0xc0008001         /* Type: GNU_PROPERTY_X86_FEATURE_2_NEEDED */
+  .long 0xc0008002         /* Type: GNU_PROPERTY_X86_ISA_1_NEEDED */
   .long 4                  /* Data size */
-  .long 0x0000001f         /* X86, ... */
+  .long 0x0000000f         /* x86-64-baseline, ... */
   .p2align 3               /* Align to 8 byte for 64 bit */
 
-  .long 0xc0010001         /* Type: GNU_PROPERTY_X86_FEATURE_2_USED */
+  .long 0xc0010002         /* Type: GNU_PROPERTY_X86_ISA_1_USED */
   .long 4                  /* Data size */
-  .long 0x000003e0         /* ZMM, ... */
+  .long 0x0000000f         /* x86-64-baseline, ... */
   .p2align 3               /* Align to 8 byte for 64 bit */
 
   /* All notes below are broken. Test we are able to report them. */
diff --git a/llvm/test/tools/llvm-readobj/wasm/wasm-imports.test b/llvm/test/tools/llvm-readobj/wasm/wasm-imports.test
index dd93c14fbebe69fbb725ce9ea5ee920028ddbd25..e8f8c9d9154fc593717d9e7f806421e3447e1279 100644
--- a/llvm/test/tools/llvm-readobj/wasm/wasm-imports.test
+++ b/llvm/test/tools/llvm-readobj/wasm/wasm-imports.test
@@ -21,7 +21,7 @@ Sections:
         Field:           __linear_memory
         Kind:            MEMORY
         Memory:          
-          Initial:         0x00000000
+          Minimum:         0x00000000
       - Module:          env
         Field:           __indirect_function_table
         Kind:            TABLE
@@ -29,7 +29,7 @@ Sections:
           Index:           0
           ElemType:        FUNCREF
           Limits:          
-            Initial:         0x00000000
+            Minimum:         0x00000000
       - Module:          red
         Field:           foo
         Kind:            FUNCTION
diff --git a/llvm/tools/dsymutil/MachODebugMapParser.cpp b/llvm/tools/dsymutil/MachODebugMapParser.cpp
index fccf2f5406a9c4d424ea50a782b03215849e8c28..ba2f9a18e6837d91e8cc1f3b6bd0fac2f525640e 100644
--- a/llvm/tools/dsymutil/MachODebugMapParser.cpp
+++ b/llvm/tools/dsymutil/MachODebugMapParser.cpp
@@ -462,6 +462,19 @@ void MachODebugMapParser::handleStabSymbolTableEntry(uint32_t StringIndex,
     }
   }
 
+  // ThinLTO adds a unique suffix to exported private symbols.
+  if (ObjectSymIt == CurrentObjectAddresses.end()) {
+    for (auto Iter = CurrentObjectAddresses.begin();
+         Iter != CurrentObjectAddresses.end(); ++Iter) {
+      llvm::StringRef SymbolName = Iter->getKey();
+      auto Pos = SymbolName.rfind(".llvm.");
+      if (Pos != llvm::StringRef::npos && SymbolName.substr(0, Pos) == Name) {
+        ObjectSymIt = Iter;
+        break;
+      }
+    }
+  }
+
   if (ObjectSymIt == CurrentObjectAddresses.end()) {
     Warning("could not find object file symbol for symbol " + Twine(Name));
     return;
diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp
index 66b2c13c426f8c515f3c81cac5ad85852d1ef188..782bc745e4a94e6ea4229c92d661d843855c0b57 100644
--- a/llvm/tools/lli/lli.cpp
+++ b/llvm/tools/lli/lli.cpp
@@ -84,7 +84,7 @@ static codegen::RegisterCodeGenFlags CGF;
 
 namespace {
 
-  enum class JITKind { MCJIT, OrcLazy };
+  enum class JITKind { MCJIT, Orc, OrcLazy };
   enum class JITLinkerKind { Default, RuntimeDyld, JITLink };
 
   cl::opt<std::string>
@@ -101,6 +101,7 @@ namespace {
       "jit-kind", cl::desc("Choose underlying JIT kind."),
       cl::init(JITKind::MCJIT),
       cl::values(clEnumValN(JITKind::MCJIT, "mcjit", "MCJIT"),
+                 clEnumValN(JITKind::Orc, "orc", "Orc JIT"),
                  clEnumValN(JITKind::OrcLazy, "orc-lazy",
                             "Orc-based lazy JIT.")));
 
@@ -326,7 +327,8 @@ public:
       return nullptr;
     // Load the object from the cache filename
     ErrorOr<std::unique_ptr<MemoryBuffer>> IRObjectBuffer =
-        MemoryBuffer::getFile(CacheName, -1, false);
+        MemoryBuffer::getFile(CacheName, /*IsText=*/false,
+                              /*RequiresNullTerminator=*/false);
     // If the file isn't there, that's OK.
     if (!IRObjectBuffer)
       return nullptr;
@@ -416,7 +418,7 @@ static void reportError(SMDiagnostic Err, const char *ProgName) {
 }
 
 Error loadDylibs();
-int runOrcLazyJIT(const char *ProgName);
+int runOrcJIT(const char *ProgName);
 void disallowOrcOptions();
 
 //===----------------------------------------------------------------------===//
@@ -443,11 +445,12 @@ int main(int argc, char **argv, char * const *envp) {
 
   ExitOnErr(loadDylibs());
 
-  if (UseJITKind == JITKind::OrcLazy)
-    return runOrcLazyJIT(argv[0]);
-  else
+  if (UseJITKind == JITKind::MCJIT)
     disallowOrcOptions();
+  else
+    return runOrcJIT(argv[0]);
 
+  // Old lli implementation based on ExecutionEngine and MCJIT.
   LLVMContext Context;
 
   // Load the bitcode...
@@ -829,7 +832,7 @@ loadModule(StringRef Path, orc::ThreadSafeContext TSCtx) {
   return orc::ThreadSafeModule(std::move(M), std::move(TSCtx));
 }
 
-int runOrcLazyJIT(const char *ProgName) {
+int runOrcJIT(const char *ProgName) {
   // Start setting up the JIT environment.
 
   // Parse the main module.
@@ -867,6 +870,16 @@ int runOrcLazyJIT(const char *ProgName) {
       .setRelocationModel(codegen::getExplicitRelocModel())
       .setCodeModel(codegen::getExplicitCodeModel());
 
+  // FIXME: Setting a dummy call-through manager in non-lazy mode prevents the
+  // JIT builder to instantiate a default (which would fail with an error for
+  // unsupported architectures).
+  if (UseJITKind != JITKind::OrcLazy) {
+    auto ES = std::make_unique<orc::ExecutionSession>();
+    Builder.setLazyCallthroughManager(
+        std::make_unique<orc::LazyCallThroughManager>(*ES, 0, nullptr));
+    Builder.setExecutionSession(std::move(ES));
+  }
+
   Builder.setLazyCompileFailureAddr(
       pointerToJITTargetAddress(exitOnLazyCallThroughFailure));
   Builder.setNumCompileThreads(LazyJITCompileThreads);
@@ -975,8 +988,17 @@ int runOrcLazyJIT(const char *ProgName) {
         std::make_unique<LLIBuiltinFunctionGenerator>(GenerateBuiltinFunctions,
                                                       Mangle));
 
+  // Regular modules are greedy: They materialize as a whole and trigger
+  // materialization for all required symbols recursively. Lazy modules go
+  // through partitioning and they replace outgoing calls with reexport stubs
+  // that resolve on call-through.
+  auto AddModule = [&](orc::JITDylib &JD, orc::ThreadSafeModule M) {
+    return UseJITKind == JITKind::OrcLazy ? J->addLazyIRModule(JD, std::move(M))
+                                          : J->addIRModule(JD, std::move(M));
+  };
+
   // Add the main module.
-  ExitOnErr(J->addLazyIRModule(std::move(MainModule)));
+  ExitOnErr(AddModule(J->getMainJITDylib(), std::move(MainModule)));
 
   // Create JITDylibs and add any extra modules.
   {
@@ -1004,7 +1026,7 @@ int runOrcLazyJIT(const char *ProgName) {
       assert(EMIdx != 0 && "ExtraModule should have index > 0");
       auto JDItr = std::prev(IdxToDylib.lower_bound(EMIdx));
       auto &JD = *JDItr->second;
-      ExitOnErr(J->addLazyIRModule(JD, std::move(M)));
+      ExitOnErr(AddModule(JD, std::move(M)));
     }
 
     for (auto EAItr = ExtraArchives.begin(), EAEnd = ExtraArchives.end();
diff --git a/llvm/tools/llvm-ar/llvm-ar.cpp b/llvm/tools/llvm-ar/llvm-ar.cpp
index 4c26c8cad3fafa01e87640a9f6e3e92414d87827..4991e51a0f75be516f6dfcb105c73ce1e75e5c1c 100644
--- a/llvm/tools/llvm-ar/llvm-ar.cpp
+++ b/llvm/tools/llvm-ar/llvm-ar.cpp
@@ -270,7 +270,8 @@ static void getArchive() {
 }
 
 static object::Archive &readLibrary(const Twine &Library) {
-  auto BufOrErr = MemoryBuffer::getFile(Library, -1, false);
+  auto BufOrErr = MemoryBuffer::getFile(Library, /*IsText=*/false,
+                                        /*RequiresNullTerminator=*/false);
   failIfError(BufOrErr.getError(), "could not open library " + Library);
   ArchiveBuffers.push_back(std::move(*BufOrErr));
   auto LibOrErr =
@@ -995,8 +996,8 @@ static void performOperation(ArchiveOperation Operation,
 static int performOperation(ArchiveOperation Operation,
                             std::vector<NewArchiveMember> *NewMembers) {
   // Create or open the archive object.
-  ErrorOr<std::unique_ptr<MemoryBuffer>> Buf =
-      MemoryBuffer::getFile(ArchiveName, -1, false);
+  ErrorOr<std::unique_ptr<MemoryBuffer>> Buf = MemoryBuffer::getFile(
+      ArchiveName, /*IsText=*/false, /*RequiresNullTerminator=*/false);
   std::error_code EC = Buf.getError();
   if (EC && EC != errc::no_such_file_or_directory)
     fail("unable to open '" + ArchiveName + "': " + EC.message());
diff --git a/llvm/tools/llvm-cov/CodeCoverage.cpp b/llvm/tools/llvm-cov/CodeCoverage.cpp
index 712766ec992937ad2e6ed813c7fda8e3b1bdbf19..144f338b10ed97c5f57b5c30fa7dbd6390a96d1f 100644
--- a/llvm/tools/llvm-cov/CodeCoverage.cpp
+++ b/llvm/tools/llvm-cov/CodeCoverage.cpp
@@ -475,7 +475,7 @@ void CodeCoverageTool::remapPathNames(const CoverageMapping &Coverage) {
     SmallString<128> NativePath;
     sys::path::native(Path, NativePath);
     sys::path::remove_dots(NativePath, true);
-    if (!sys::path::is_separator(NativePath.back()))
+    if (!NativePath.empty() && !sys::path::is_separator(NativePath.back()))
       NativePath += sys::path::get_separator();
     return NativePath.c_str();
   };
diff --git a/llvm/tools/llvm-cov/gcov.cpp b/llvm/tools/llvm-cov/gcov.cpp
index d42e7cd3b551e460f248f9d0def6e11f1b3b394d..9a1ebebc87fc8a352ec1e0d1f717cbd46564d2a3 100644
--- a/llvm/tools/llvm-cov/gcov.cpp
+++ b/llvm/tools/llvm-cov/gcov.cpp
@@ -46,7 +46,8 @@ static void reportCoverage(StringRef SourceFile, StringRef ObjectDir,
   // Open .gcda and .gcda without requiring a NUL terminator. The concurrent
   // modification may nullify the NUL terminator condition.
   ErrorOr<std::unique_ptr<MemoryBuffer>> GCNO_Buff =
-      MemoryBuffer::getFileOrSTDIN(GCNO, -1, /*RequiresNullTerminator=*/false);
+      MemoryBuffer::getFileOrSTDIN(GCNO, /*IsText=*/false,
+                                   /*RequiresNullTerminator=*/false);
   if (std::error_code EC = GCNO_Buff.getError()) {
     errs() << GCNO << ": " << EC.message() << "\n";
     return;
@@ -58,7 +59,8 @@ static void reportCoverage(StringRef SourceFile, StringRef ObjectDir,
   }
 
   ErrorOr<std::unique_ptr<MemoryBuffer>> GCDA_Buff =
-      MemoryBuffer::getFileOrSTDIN(GCDA, -1, /*RequiresNullTerminator=*/false);
+      MemoryBuffer::getFileOrSTDIN(GCDA, /*IsText=*/false,
+                                   /*RequiresNullTerminator=*/false);
   if (std::error_code EC = GCDA_Buff.getError()) {
     if (EC != errc::no_such_file_or_directory) {
       errs() << GCDA << ": " << EC.message() << "\n";
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink-elf.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink-elf.cpp
index beb73fb8edf71280ccffb8eb4242ce5c2a4865d7..3f222af3222daa7963e58e0031d858a5ee033705 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink-elf.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink-elf.cpp
@@ -116,11 +116,15 @@ Error registerELFGraphInfo(Session &S, LinkGraph &G) {
           return make_error<StringError>("zero-fill atom in GOT section",
                                          inconvertibleErrorCode());
 
-        if (auto TS = getELFGOTTarget(G, Sym->getBlock()))
-          FileInfo.GOTEntryInfos[TS->getName()] = {Sym->getSymbolContent(),
-                                                   Sym->getAddress()};
-        else
-          return TS.takeError();
+        // If this is a GOT symbol with size (i.e. not the GOT start symbol)
+        // then add it to the GOT entry info table.
+        if (Sym->getSize() != 0) {
+          if (auto TS = getELFGOTTarget(G, Sym->getBlock()))
+            FileInfo.GOTEntryInfos[TS->getName()] = {Sym->getSymbolContent(),
+                                                     Sym->getAddress()};
+          else
+            return TS.takeError();
+        }
         SectionContainsContent = true;
       } else if (isStubsSection) {
         if (Sym->isSymbolZeroFill())
@@ -133,8 +137,9 @@ Error registerELFGraphInfo(Session &S, LinkGraph &G) {
         else
           return TS.takeError();
         SectionContainsContent = true;
-      } else if (Sym->hasName()) {
-        dbgs() << "Symbol: " << Sym->getName() << "\n";
+      }
+
+      if (Sym->hasName()) {
         if (Sym->isSymbolZeroFill()) {
           S.SymbolInfos[Sym->getName()] = {Sym->getSize(), Sym->getAddress()};
           SectionContainsZeroFill = true;
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
index f693b0268cd855be06a82c93906462f0a8b05ba0..9a92581157c5e45c1046638f5024ff8239a7915d 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
@@ -17,11 +17,14 @@
 #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include <cstring>
 #include <sstream>
 
 #ifdef LLVM_ON_UNIX
 
+#include <netdb.h>
 #include <netinet/in.h>
 #include <sys/socket.h>
 
@@ -46,37 +49,58 @@ void printErrorAndExit(Twine ErrMsg) {
   exit(1);
 }
 
-int openListener(std::string Host, int Port) {
+int openListener(std::string Host, std::string PortStr) {
 #ifndef LLVM_ON_UNIX
   // FIXME: Add TCP support for Windows.
   printErrorAndExit("listen option not supported");
   return 0;
 #else
-  int SockFD = socket(PF_INET, SOCK_STREAM, 0);
-  struct sockaddr_in ServerAddr, ClientAddr;
-  socklen_t ClientAddrLen = sizeof(ClientAddr);
-  memset(&ServerAddr, 0, sizeof(ServerAddr));
-  ServerAddr.sin_family = PF_INET;
-  ServerAddr.sin_family = INADDR_ANY;
-  ServerAddr.sin_port = htons(Port);
-
-  {
-    // lose the "Address already in use" error message
-    int Yes = 1;
-    if (setsockopt(SockFD, SOL_SOCKET, SO_REUSEADDR, &Yes, sizeof(int)) == -1) {
-      errs() << "Error calling setsockopt.\n";
-      exit(1);
-    }
+  addrinfo Hints{};
+  Hints.ai_family = AF_INET;
+  Hints.ai_socktype = SOCK_STREAM;
+  Hints.ai_flags = AI_PASSIVE;
+
+  addrinfo *AI;
+  if (int EC = getaddrinfo(nullptr, PortStr.c_str(), &Hints, &AI)) {
+    errs() << "Error setting up bind address: " << gai_strerror(EC) << "\n";
+    exit(1);
+  }
+
+  // Create a socket from first addrinfo structure returned by getaddrinfo.
+  int SockFD;
+  if ((SockFD = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol)) < 0) {
+    errs() << "Error creating socket: " << std::strerror(errno) << "\n";
+    exit(1);
   }
 
-  if (bind(SockFD, (struct sockaddr *)&ServerAddr, sizeof(ServerAddr)) < 0) {
-    errs() << "Error on binding.\n";
+  // Avoid "Address already in use" errors.
+  const int Yes = 1;
+  if (setsockopt(SockFD, SOL_SOCKET, SO_REUSEADDR, &Yes, sizeof(int)) == -1) {
+    errs() << "Error calling setsockopt: " << std::strerror(errno) << "\n";
     exit(1);
   }
 
-  listen(SockFD, 1);
-  return accept(SockFD, (struct sockaddr *)&ClientAddr, &ClientAddrLen);
+  // Bind the socket to the desired port.
+  if (bind(SockFD, AI->ai_addr, AI->ai_addrlen) < 0) {
+    errs() << "Error on binding: " << std::strerror(errno) << "\n";
+    exit(1);
+  }
+
+  // Listen for incomming connections.
+  static constexpr int ConnectionQueueLen = 1;
+  listen(SockFD, ConnectionQueueLen);
+
+  outs() << "Listening at " << Host << ":" << PortStr << "\n";
+
+#if defined(_AIX)
+  assert(Hi_32(AI->ai_addrlen) == 0 && "Field is a size_t on 64-bit AIX");
+  socklen_t AddrLen = Lo_32(AI->ai_addrlen);
+  return accept(SockFD, AI->ai_addr, &AddrLen);
+#else
+  return accept(SockFD, AI->ai_addr, &AI->ai_addrlen);
 #endif
+
+#endif // LLVM_ON_UNIX
 }
 
 int main(int argc, char *argv[]) {
@@ -105,9 +129,11 @@ int main(int argc, char *argv[]) {
 
       int Port = 0;
       if (PortStr.getAsInteger(10, Port))
-        printErrorAndExit("port" + PortStr + " is not a valid integer");
+        printErrorAndExit("port number '" + PortStr +
+                          "' is not a valid integer");
 
-      InFD = OutFD = openListener(Host.str(), Port);
+      InFD = OutFD = openListener(Host.str(), PortStr.str());
+      outs() << "Connection established. Running OrcRPCTPCServer...\n";
     } else
       printErrorAndExit("invalid specifier type \"" + SpecifierType + "\"");
   }
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index 473afc5f47e73920474af166684ba3ed6a77b60f..fb5fe4231f085e89fbecd0cddf6130cdd6fa48f5 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -42,6 +42,7 @@
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/Timer.h"
 
+#include <cstring>
 #include <list>
 #include <string>
 
@@ -668,73 +669,89 @@ LLVMJITLinkRemoteTargetProcessControl::LaunchExecutor() {
 #endif
 }
 
-Expected<std::unique_ptr<TargetProcessControl>>
-LLVMJITLinkRemoteTargetProcessControl::ConnectToExecutor() {
+static Error createTCPSocketError(Twine Details) {
+  return make_error<StringError>(
+      formatv("Failed to connect TCP socket '{0}': {1}",
+              OutOfProcessExecutorConnect, Details),
+      inconvertibleErrorCode());
+}
+
+static Expected<int> connectTCPSocket(std::string Host, std::string PortStr) {
 #ifndef LLVM_ON_UNIX
   // FIXME: Add TCP support for Windows.
   return make_error<StringError>("-" + OutOfProcessExecutorConnect.ArgStr +
                                      " not supported on non-unix platforms",
                                  inconvertibleErrorCode());
 #else
-
-  shared::registerStringError<LLVMJITLinkChannel>();
-
-  StringRef HostNameStr, PortStr;
-  std::tie(HostNameStr, PortStr) =
-      StringRef(OutOfProcessExecutorConnect).split(':');
-
-  if (HostNameStr.empty())
-    return make_error<StringError>("host name for -" +
-                                       OutOfProcessExecutorConnect.ArgStr +
-                                       " can not be empty",
-                                   inconvertibleErrorCode());
-  if (PortStr.empty())
-    return make_error<StringError>(
-        "port for -" + OutOfProcessExecutorConnect.ArgStr + " can not be empty",
-        inconvertibleErrorCode());
-
-  std::string HostName = HostNameStr.str();
-  int Port = 0;
-  if (PortStr.getAsInteger(10, Port))
-    return make_error<StringError>("port number " + PortStr +
-                                       " is not a valid integer",
-                                   inconvertibleErrorCode());
-
   addrinfo *AI;
   addrinfo Hints{};
   Hints.ai_family = AF_INET;
   Hints.ai_socktype = SOCK_STREAM;
-  Hints.ai_protocol = PF_INET;
   Hints.ai_flags = AI_NUMERICSERV;
-  if (getaddrinfo(HostName.c_str(), PortStr.str().c_str(), &Hints, &AI) != 0)
-    return make_error<StringError>("Failed to resolve " + HostName + ":" +
-                                       Twine(Port),
-                                   inconvertibleErrorCode());
 
-  int SockFD = socket(PF_INET, SOCK_STREAM, 0);
-  sockaddr_in ServAddr;
-  memset(&ServAddr, 0, sizeof(ServAddr));
-  ServAddr.sin_family = PF_INET;
-  ServAddr.sin_port = htons(Port);
-
-  // getaddrinfo returns a list of address structures.  Go through the list
-  // to find one we can connect to.
-  int ConnectRC = -1;
-  for (addrinfo *Server = AI; Server; Server = Server->ai_next) {
-    memmove(&Server->ai_addr, &ServAddr.sin_addr.s_addr, Server->ai_addrlen);
-    ConnectRC = connect(SockFD, reinterpret_cast<sockaddr *>(&ServAddr),
-                        sizeof(ServAddr));
-    if (ConnectRC == 0)
+  if (int EC = getaddrinfo(Host.c_str(), PortStr.c_str(), &Hints, &AI))
+    return createTCPSocketError("Address resolution failed (" +
+                                StringRef(gai_strerror(EC)) + ")");
+
+  // Cycle through the returned addrinfo structures and connect to the first
+  // reachable endpoint.
+  int SockFD;
+  addrinfo *Server;
+  for (Server = AI; Server != nullptr; Server = Server->ai_next) {
+    // socket might fail, e.g. if the address family is not supported. Skip to
+    // the next addrinfo structure in such a case.
+    if ((SockFD = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol)) < 0)
+      continue;
+
+    // If connect returns null, we exit the loop with a working socket.
+    if (connect(SockFD, Server->ai_addr, Server->ai_addrlen) == 0)
       break;
+
+    close(SockFD);
   }
   freeaddrinfo(AI);
-  if (ConnectRC == -1)
-    return make_error<StringError>("Failed to connect to " + HostName + ":" +
-                                       Twine(Port),
-                                   inconvertibleErrorCode());
+
+  // If we reached the end of the loop without connecting to a valid endpoint,
+  // dump the last error that was logged in socket() or connect().
+  if (Server == nullptr)
+    return createTCPSocketError(std::strerror(errno));
+
+  return SockFD;
+#endif
+}
+
+Expected<std::unique_ptr<TargetProcessControl>>
+LLVMJITLinkRemoteTargetProcessControl::ConnectToExecutor() {
+#ifndef LLVM_ON_UNIX
+  // FIXME: Add TCP support for Windows.
+  return make_error<StringError>("-" + OutOfProcessExecutorConnect.ArgStr +
+                                     " not supported on non-unix platforms",
+                                 inconvertibleErrorCode());
+#else
+
+  shared::registerStringError<LLVMJITLinkChannel>();
+
+  StringRef Host, PortStr;
+  std::tie(Host, PortStr) = StringRef(OutOfProcessExecutorConnect).split(':');
+  if (Host.empty())
+    return createTCPSocketError("Host name for -" +
+                                OutOfProcessExecutorConnect.ArgStr +
+                                " can not be empty");
+  if (PortStr.empty())
+    return createTCPSocketError("Port number in -" +
+                                OutOfProcessExecutorConnect.ArgStr +
+                                " can not be empty");
+  int Port = 0;
+  if (PortStr.getAsInteger(10, Port))
+    return createTCPSocketError("Port number '" + PortStr +
+                                "' is not a valid integer");
+
+  Expected<int> SockFD = connectTCPSocket(Host.str(), PortStr.str());
+  if (!SockFD)
+    return SockFD.takeError();
 
   auto SSP = std::make_shared<SymbolStringPool>();
-  auto Channel = std::make_unique<shared::FDRawByteChannel>(SockFD, SockFD);
+  auto Channel = std::make_unique<shared::FDRawByteChannel>(*SockFD, *SockFD);
   auto Endpoint = std::make_unique<LLVMJITLinkRPCEndpoint>(*Channel, true);
 
   auto ReportError = [](Error Err) {
@@ -1008,11 +1025,22 @@ Session::findSymbolInfo(StringRef SymbolName, Twine ErrorMsgStem) {
 static Triple getFirstFileTriple() {
   static Triple FirstTT = []() {
     assert(!InputFiles.empty() && "InputFiles can not be empty");
-    auto ObjBuffer =
-        ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFiles.front())));
-    auto Obj = ExitOnErr(
-        object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()));
-    return Obj->makeTriple();
+    for (auto InputFile : InputFiles) {
+      auto ObjBuffer =
+          ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFile)));
+      switch (identify_magic(ObjBuffer->getBuffer())) {
+      case file_magic::elf_relocatable:
+      case file_magic::macho_object:
+      case file_magic::coff_object: {
+        auto Obj = ExitOnErr(
+            object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()));
+        return Obj->makeTriple();
+      }
+      default:
+        break;
+      }
+    }
+    return Triple();
   }();
 
   return FirstTT;
diff --git a/llvm/tools/llvm-libtool-darwin/llvm-libtool-darwin.cpp b/llvm/tools/llvm-libtool-darwin/llvm-libtool-darwin.cpp
index 021736418419d80a5e5cc396734fa2bf9991cf69..f16b4a17105c648f97e9f5f75ea1065bfca5cc7a 100644
--- a/llvm/tools/llvm-libtool-darwin/llvm-libtool-darwin.cpp
+++ b/llvm/tools/llvm-libtool-darwin/llvm-libtool-darwin.cpp
@@ -147,7 +147,7 @@ static Error processFileList() {
   std::tie(FileName, DirName) = StringRef(FileList).rsplit(",");
 
   ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
-      MemoryBuffer::getFileOrSTDIN(FileName, /*FileSize=*/-1,
+      MemoryBuffer::getFileOrSTDIN(FileName, /*IsText=*/false,
                                    /*RequiresNullTerminator=*/false);
   if (std::error_code EC = FileOrErr.getError())
     return createFileError(FileName, errorCodeToError(EC));
diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp
index eed49c438335a85a9d362fac0e34aa7992340d02..b01270de727a5fdf010e7eb3d897a2cf9d217d80 100644
--- a/llvm/tools/llvm-link/llvm-link.cpp
+++ b/llvm/tools/llvm-link/llvm-link.cpp
@@ -246,8 +246,10 @@ public:
 Module &ModuleLazyLoaderCache::operator()(const char *argv0,
                                           const std::string &Identifier) {
   auto &Module = ModuleMap[Identifier];
-  if (!Module)
+  if (!Module) {
     Module = createLazyModule(argv0, Identifier);
+    assert(Module && "Failed to create lazy module!");
+  }
   return *Module;
 }
 } // anonymous namespace
diff --git a/llvm/tools/llvm-mca/Views/TimelineView.cpp b/llvm/tools/llvm-mca/Views/TimelineView.cpp
index c8b481bc7ce68bd80dcf743d9e26a8495a5e7198..ceeb267cf119fb7f3dd5de14ba0d8076e3468f08 100644
--- a/llvm/tools/llvm-mca/Views/TimelineView.cpp
+++ b/llvm/tools/llvm-mca/Views/TimelineView.cpp
@@ -77,8 +77,10 @@ void TimelineView::onEvent(const HWInstructionEvent &Event) {
            "Instruction cannot be ready if it hasn't been dispatched yet!");
     WTEntry.CyclesSpentInSQWhileReady +=
         TVEntry.CycleIssued - TVEntry.CycleReady;
-    WTEntry.CyclesSpentAfterWBAndBeforeRetire +=
-        (CurrentCycle - 1) - TVEntry.CycleExecuted;
+    if (CurrentCycle > TVEntry.CycleExecuted) {
+      WTEntry.CyclesSpentAfterWBAndBeforeRetire +=
+          (CurrentCycle - 1) - TVEntry.CycleExecuted;
+    }
     break;
   }
   case HWInstructionEvent::Ready:
@@ -243,7 +245,8 @@ void TimelineView::printTimelineViewEntry(formatted_raw_ostream &OS,
 
   for (unsigned I = Entry.CycleExecuted + 1, E = Entry.CycleRetired; I < E; ++I)
     OS << TimelineView::DisplayChar::RetireLag;
-  OS << TimelineView::DisplayChar::Retired;
+  if (Entry.CycleExecuted < Entry.CycleRetired)
+    OS << TimelineView::DisplayChar::Retired;
 
   // Skip other columns.
   for (unsigned I = Entry.CycleRetired + 1, E = LastCycle; I <= E; ++I)
diff --git a/llvm/tools/llvm-mca/llvm-mca.cpp b/llvm/tools/llvm-mca/llvm-mca.cpp
index 830a619d2e3205b18cd43f36bab92b677b63e617..0e0a39883f93a334d43b0f2c86f39bac443c11df 100644
--- a/llvm/tools/llvm-mca/llvm-mca.cpp
+++ b/llvm/tools/llvm-mca/llvm-mca.cpp
@@ -278,7 +278,8 @@ static void processViewOptions(bool IsOutOfOrder) {
   processOptionImpl(PrintRegisterFileStats, Default);
   processOptionImpl(PrintDispatchStats, Default);
   processOptionImpl(PrintSchedulerStats, Default);
-  processOptionImpl(PrintRetireStats, Default);
+  if (IsOutOfOrder)
+    processOptionImpl(PrintRetireStats, Default);
 }
 
 // Returns true on success.
diff --git a/llvm/tools/llvm-ml/Opts.td b/llvm/tools/llvm-ml/Opts.td
index 4c2757b05722a56b9f8efa57cb22f761528702be..a78072007e92778d5db6513b0305ac9f7921e16d 100644
--- a/llvm/tools/llvm-ml/Opts.td
+++ b/llvm/tools/llvm-ml/Opts.td
@@ -27,25 +27,6 @@ class UnsupportedJoinedOrSeparate<string name> :
 class UnsupportedSeparate<string name> : Separate<["/", "-"], name>,
                                          Group<unsupported_Group>;
 
-def help : MLFlag<"?">,
-           HelpText<"Display available options">;
-def help_long : MLFlag<"help">, Alias<help>;
-def assemble_only : MLFlag<"c">, HelpText<"Assemble only; do not link">;
-def define : MLJoinedOrSeparate<"D">, MetaVarName<"<macro>=<value>">,
-             HelpText<"Define <macro> to <value> (or blank if <value> "
-                      "omitted)">;
-def output_file : MLJoinedOrSeparate<"Fo">, HelpText<"Names the output file">;
-def include_path : MLJoinedOrSeparate<"I">,
-                   HelpText<"Sets path for include files">;
-def safeseh : MLFlag<"safeseh">,
-              HelpText<"Mark resulting object files as either containing no "
-                       "exception handlers or containing exception handlers "
-                       "that are all declared with .SAFESEH. Only available in "
-                       "32-bit.">;
-def assembly_file : MLJoinedOrSeparate<"Ta">,
-                    HelpText<"Assemble source file with name not ending with "
-                             "the .asm extension">;
-
 def bitness : LLVMJoined<"m">, Values<"32,64">,
               HelpText<"Target platform (x86 or x86-64)">;
 def as_lex : LLVMFlag<"as-lex">,
@@ -69,6 +50,27 @@ def preserve_comments : LLVMFlag<"preserve-comments">,
 def save_temp_labels : LLVMFlag<"save-temp-labels">,
                        HelpText<"Don't discard temporary labels">;
 
+def help : MLFlag<"?">,
+           HelpText<"Display available options">;
+def help_long : MLFlag<"help">, Alias<help>;
+def assemble_only : MLFlag<"c">, HelpText<"Assemble only; do not link">;
+def define : MLJoinedOrSeparate<"D">, MetaVarName<"<macro>=<value>">,
+             HelpText<"Define <macro> to <value> (or blank if <value> "
+                      "omitted)">;
+def output_file : MLJoinedOrSeparate<"Fo">, HelpText<"Names the output file">;
+def include_path : MLJoinedOrSeparate<"I">,
+                   HelpText<"Sets path for include files">;
+def safeseh : MLFlag<"safeseh">,
+              HelpText<"Mark resulting object files as either containing no "
+                       "exception handlers or containing exception handlers "
+                       "that are all declared with .SAFESEH. Only available in "
+                       "32-bit.">;
+def assembly_file : MLJoinedOrSeparate<"Ta">,
+                    HelpText<"Assemble source file with name not ending with "
+                             "the .asm extension">;
+def parse_only : MLFlag<"Zs">, HelpText<"Run a syntax-check only">,
+                 Alias<filetype>, AliasArgs<["null"]>;
+
 def tiny_model_support : UnsupportedFlag<"AT">, HelpText<"">;
 def alternate_linker : UnsupportedJoined<"Bl">, HelpText<"">;
 def coff_object_file : UnsupportedFlag<"coff">, HelpText<"">;
@@ -107,4 +109,3 @@ def export_all_symbols : UnsupportedFlag<"Zf">, HelpText<"">;
 def codeview_info : UnsupportedFlag<"Zi">, HelpText<"">;
 def enable_m510_option : UnsupportedFlag<"Zm">, HelpText<"">;
 def structure_packing : UnsupportedJoined<"Zp">, HelpText<"">;
-def parse_only : UnsupportedFlag<"Zs">, HelpText<"">;
diff --git a/llvm/tools/llvm-nm/llvm-nm.cpp b/llvm/tools/llvm-nm/llvm-nm.cpp
index b978eafcbda777f51ce330a7a3e4f5e95338c6a2..914b0a14bfedca538c3617aa3d7aecea7927b270 100644
--- a/llvm/tools/llvm-nm/llvm-nm.cpp
+++ b/llvm/tools/llvm-nm/llvm-nm.cpp
@@ -47,7 +47,7 @@ using namespace llvm;
 using namespace object;
 
 namespace {
-enum OutputFormatTy { bsd, sysv, posix, darwin };
+enum OutputFormatTy { bsd, sysv, posix, darwin, just_symbols };
 
 cl::OptionCategory NMCat("llvm-nm Options");
 
@@ -55,7 +55,9 @@ cl::opt<OutputFormatTy> OutputFormat(
     "format", cl::desc("Specify output format"),
     cl::values(clEnumVal(bsd, "BSD format"), clEnumVal(sysv, "System V format"),
                clEnumVal(posix, "POSIX.2 format"),
-               clEnumVal(darwin, "Darwin -m format")),
+               clEnumVal(darwin, "Darwin -m format"),
+               cl::OptionEnumValue{"just-symbols", int(just_symbols),
+                                   "just symbol names"}),
     cl::init(bsd), cl::cat(NMCat));
 cl::alias OutputFormat2("f", cl::desc("Alias for --format"),
                         cl::aliasopt(OutputFormat));
@@ -180,9 +182,9 @@ cl::alias RadixAlias("t", cl::desc("Alias for --radix"),
                      cl::aliasopt(AddressRadix));
 
 cl::opt<bool> JustSymbolName("just-symbol-name",
-                             cl::desc("Print just the symbol's name"),
+                             cl::desc("Alias for --format=just-symbols"),
                              cl::cat(NMCat));
-cl::alias JustSymbolNames("j", cl::desc("Alias for --just-symbol-name"),
+cl::alias JustSymbolNames("j", cl::desc("Alias for --format-just-symbols"),
                           cl::aliasopt(JustSymbolName), cl::Grouping);
 
 cl::opt<bool>
@@ -740,16 +742,6 @@ static void writeFileName(raw_ostream &S, StringRef ArchiveName,
   }
 }
 
-static bool isSpecialSym(SymbolicFile &Obj, StringRef Name) {
-  auto *ELFObj = dyn_cast<ELFObjectFileBase>(&Obj);
-  if (!ELFObj)
-    return false;
-  uint16_t EMachine = ELFObj->getEMachine();
-  if (EMachine != ELF::EM_ARM && EMachine != ELF::EM_AARCH64)
-    return false;
-  return !Name.empty() && Name[0] == '$';
-}
-
 static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
                                    StringRef ArchiveName,
                                    StringRef ArchitectureName) {
@@ -772,10 +764,10 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
   }
 
   if (!PrintFileName) {
-    if (OutputFormat == posix && MultipleFiles && printName) {
+    if ((OutputFormat == bsd || OutputFormat == posix ||
+         OutputFormat == just_symbols) &&
+        MultipleFiles && printName) {
       outs() << '\n' << CurrentFilename << ":\n";
-    } else if (OutputFormat == bsd && MultipleFiles && printName) {
-      outs() << "\n" << CurrentFilename << ":\n";
     } else if (OutputFormat == sysv) {
       outs() << "\n\nSymbols from " << CurrentFilename << ":\n\n";
       if (isSymbolList64Bit(Obj))
@@ -838,13 +830,14 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
     bool Undefined = SymFlags & SymbolRef::SF_Undefined;
     bool Global = SymFlags & SymbolRef::SF_Global;
     bool Weak = SymFlags & SymbolRef::SF_Weak;
+    bool FormatSpecific = SymFlags & SymbolRef::SF_FormatSpecific;
     if ((!Undefined && UndefinedOnly) || (Undefined && DefinedOnly) ||
         (!Global && ExternalOnly) || (Weak && NoWeakSymbols) ||
-        (!SpecialSyms && isSpecialSym(Obj, Name)))
+        (FormatSpecific && !(SpecialSyms || DebugSyms)))
       continue;
     if (PrintFileName)
       writeFileName(outs(), ArchiveName, ArchitectureName);
-    if ((JustSymbolName ||
+    if ((OutputFormat == just_symbols ||
          (UndefinedOnly && MachO && OutputFormat != darwin)) &&
         OutputFormat != posix) {
       outs() << Name << "\n";
@@ -1810,7 +1803,14 @@ static void dumpSymbolNamesFromObject(SymbolicFile &Obj, bool printName,
         error(SymFlagsOrErr.takeError(), Obj.getFileName());
         return;
       }
-      if (!DebugSyms && (*SymFlagsOrErr & SymbolRef::SF_FormatSpecific))
+
+      // Don't drop format specifc symbols for ARM and AArch64 ELF targets, they
+      // are used to repesent mapping symbols and needed to honor the
+      // --special-syms option.
+      auto *ELFObj = dyn_cast<ELFObjectFileBase>(&Obj);
+      if ((!ELFObj || (ELFObj->getEMachine() != ELF::EM_ARM &&
+                       ELFObj->getEMachine() != ELF::EM_AARCH64)) &&
+          !DebugSyms && (*SymFlagsOrErr & SymbolRef::SF_FormatSpecific))
         continue;
       if (WithoutAliases && (*SymFlagsOrErr & SymbolRef::SF_Indirect))
         continue;
@@ -2251,6 +2251,8 @@ int main(int argc, char **argv) {
     OutputFormat = posix;
   if (DarwinFormat)
     OutputFormat = darwin;
+  if (JustSymbolName)
+    OutputFormat = just_symbols;
 
   // The relative order of these is important. If you pass --size-sort it should
   // only print out the size. However, if you pass -S --size-sort, it should
diff --git a/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp b/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
index 0cf0172c35500afb73725ffc805d8e42add35369..d139814617b1423fd87164f5922b08b80a046a24 100644
--- a/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
+++ b/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
@@ -173,32 +173,6 @@ static Error makeStringError(std::error_code EC, const Twine &Msg,
   return createStringError(EC, FullMsg.c_str(), std::forward<Ts>(Args)...);
 }
 
-static Error splitDWOToFile(const CopyConfig &Config, const Reader &Reader,
-                            StringRef File, ElfType OutputElfType) {
-  Expected<std::unique_ptr<Object>> DWOFile = Reader.create(false);
-  if (!DWOFile)
-    return DWOFile.takeError();
-
-  auto OnlyKeepDWOPred = [&DWOFile](const SectionBase &Sec) {
-    return onlyKeepDWOPred(**DWOFile, Sec);
-  };
-  if (Error E =
-          (*DWOFile)->removeSections(Config.AllowBrokenLinks, OnlyKeepDWOPred))
-    return E;
-  if (Config.OutputArch) {
-    (*DWOFile)->Machine = Config.OutputArch.getValue().EMachine;
-    (*DWOFile)->OSABI = Config.OutputArch.getValue().OSABI;
-  }
-
-  return writeToFile(File, [&](raw_ostream &OutFile) -> Error {
-    std::unique_ptr<Writer> Writer =
-        createWriter(Config, **DWOFile, OutFile, OutputElfType);
-    if (Error E = Writer->finalize())
-      return E;
-    return Writer->write();
-  });
-}
-
 static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
                                Object &Obj) {
   for (auto &Sec : Obj.sections()) {
@@ -374,7 +348,7 @@ static Error replaceAndRemoveSections(const CopyConfig &Config, Object &Obj) {
     };
   }
 
-  if (Config.StripDWO || !Config.SplitDWO.empty())
+  if (Config.StripDWO)
     RemovePred = [RemovePred](const SectionBase &Sec) {
       return isDWOSection(Sec) || RemovePred(Sec);
     };
@@ -532,21 +506,22 @@ static Error replaceAndRemoveSections(const CopyConfig &Config, Object &Obj) {
 // any previous removals. Lastly whether or not something is removed shouldn't
 // depend a) on the order the options occur in or b) on some opaque priority
 // system. The only priority is that keeps/copies overrule removes.
-static Error handleArgs(const CopyConfig &Config, Object &Obj,
-                        const Reader &Reader, ElfType OutputElfType) {
+static Error handleArgs(const CopyConfig &Config, Object &Obj) {
   if (Config.StripSwiftSymbols || Config.KeepUndefined)
     return createStringError(llvm::errc::invalid_argument,
                              "option not supported by llvm-objcopy for ELF");
-  if (!Config.SplitDWO.empty())
-    if (Error E =
-            splitDWOToFile(Config, Reader, Config.SplitDWO, OutputElfType))
-      return E;
 
   if (Config.OutputArch) {
     Obj.Machine = Config.OutputArch.getValue().EMachine;
     Obj.OSABI = Config.OutputArch.getValue().OSABI;
   }
 
+  if (!Config.SplitDWO.empty() && Config.ExtractDWO) {
+    return Obj.removeSections(
+        Config.AllowBrokenLinks,
+        [&Obj](const SectionBase &Sec) { return onlyKeepDWOPred(Obj, Sec); });
+  }
+
   // Dump sections before add/remove for compatibility with GNU objcopy.
   for (StringRef Flag : Config.DumpSection) {
     StringRef SectionName;
@@ -706,7 +681,7 @@ Error executeObjcopyOnIHex(const CopyConfig &Config, MemoryBuffer &In,
 
   const ElfType OutputElfType =
       getOutputElfType(Config.OutputArch.getValueOr(MachineInfo()));
-  if (Error E = handleArgs(Config, **Obj, Reader, OutputElfType))
+  if (Error E = handleArgs(Config, **Obj))
     return E;
   return writeOutput(Config, **Obj, Out, OutputElfType);
 }
@@ -724,7 +699,7 @@ Error executeObjcopyOnRawBinary(const CopyConfig &Config, MemoryBuffer &In,
   // (-B<arch>).
   const ElfType OutputElfType =
       getOutputElfType(Config.OutputArch.getValueOr(MachineInfo()));
-  if (Error E = handleArgs(Config, **Obj, Reader, OutputElfType))
+  if (Error E = handleArgs(Config, **Obj))
     return E;
   return writeOutput(Config, **Obj, Out, OutputElfType);
 }
@@ -741,7 +716,7 @@ Error executeObjcopyOnBinary(const CopyConfig &Config,
       Config.OutputArch ? getOutputElfType(Config.OutputArch.getValue())
                         : getOutputElfType(In);
 
-  if (Error E = handleArgs(Config, **Obj, Reader, OutputElfType))
+  if (Error E = handleArgs(Config, **Obj))
     return createFileError(Config.InputFilename, std::move(E));
 
   if (Error E = writeOutput(Config, **Obj, Out, OutputElfType))
diff --git a/llvm/tools/llvm-objcopy/llvm-objcopy.cpp b/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
index bc3247358cd68e70eeb225d1e79cee6ee8a49789..6c6b26b3c32f92a4363340f82e5dafe80c0e9d19 100644
--- a/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
+++ b/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
@@ -57,44 +57,6 @@
 namespace llvm {
 namespace objcopy {
 
-Error writeToFile(StringRef OutputFileName,
-                  std::function<Error(raw_ostream &)> Write, bool KeepOwnership,
-                  unsigned UserID, unsigned GroupID) {
-  if (OutputFileName == "-")
-    return Write(outs());
-
-  if (OutputFileName == "/dev/null") {
-    raw_null_ostream Out;
-    return Write(Out);
-  }
-
-  unsigned Mode = sys::fs::all_read | sys::fs::all_write | sys::fs::all_exe;
-  Expected<sys::fs::TempFile> Temp =
-      sys::fs::TempFile::create(OutputFileName + ".temp-objcopy-%%%%%%", Mode);
-  if (!Temp)
-    return createFileError(OutputFileName, Temp.takeError());
-
-#ifndef _WIN32
-  // Try to preserve file ownership if requested.
-  if (KeepOwnership) {
-    sys::fs::file_status Stat;
-    if (!sys::fs::status(Temp->FD, Stat) && Stat.getUser() == 0)
-      sys::fs::changeFileOwnership(Temp->FD, UserID, GroupID);
-  }
-#endif
-
-  raw_fd_ostream Out(Temp->FD, false);
-
-  if (Error E = Write(Out)) {
-    if (Error DiscardError = Temp->discard())
-      return joinErrors(std::move(E), std::move(DiscardError));
-    return E;
-  }
-  Out.flush();
-
-  return Temp->keep(OutputFileName);
-}
-
 // The name this program was invoked as.
 StringRef ToolName;
 
@@ -156,9 +118,9 @@ static Error deepWriteArchive(StringRef ArcName,
     // now in-memory buffers can not be completely avoided since
     // NewArchiveMember still requires them even though writeArchive does not
     // write them on disk.
-    Expected<std::unique_ptr<FileOutputBuffer>> FB = FileOutputBuffer::create(
-        Member.MemberName, Member.Buf->getBufferSize(),
-        FileOutputBuffer::F_executable | FileOutputBuffer::F_keep_ownership);
+    Expected<std::unique_ptr<FileOutputBuffer>> FB =
+        FileOutputBuffer::create(Member.MemberName, Member.Buf->getBufferSize(),
+                                 FileOutputBuffer::F_executable);
     if (!FB)
       return FB.takeError();
     std::copy(Member.Buf->getBufferStart(), Member.Buf->getBufferEnd(),
@@ -306,6 +268,12 @@ static Error restoreStatOnFile(StringRef Filename,
     if (auto EC = sys::fs::setPermissions(FD, Perm))
 #endif
       return createFileError(Filename, EC);
+
+#ifndef _WIN32
+    // Keep ownership if llvm-objcopy is called under root.
+    if (Config.InputFilename == Config.OutputFilename && OStat.getUser() == 0)
+      sys::fs::changeFileOwnership(FD, Stat.getUser(), Stat.getGroup());
+#endif
   }
 
   if (auto EC = sys::Process::SafelyCloseFileDescriptor(FD))
@@ -326,48 +294,68 @@ static Error executeObjcopy(CopyConfig &Config) {
     Stat.permissions(static_cast<sys::fs::perms>(0777));
   }
 
-  using ProcessRawFn = Error (*)(CopyConfig &, MemoryBuffer &, raw_ostream &);
-  ProcessRawFn ProcessRaw;
-  switch (Config.InputFormat) {
-  case FileFormat::Binary:
-    ProcessRaw = executeObjcopyOnRawBinary;
-    break;
-  case FileFormat::IHex:
-    ProcessRaw = executeObjcopyOnIHex;
-    break;
-  default:
-    ProcessRaw = nullptr;
-  }
+  std::function<Error(raw_ostream & OutFile)> ObjcopyFunc;
+
+  OwningBinary<llvm::object::Binary> BinaryHolder;
+  std::unique_ptr<MemoryBuffer> MemoryBufferHolder;
 
-  if (ProcessRaw) {
-    auto BufOrErr = MemoryBuffer::getFileOrSTDIN(Config.InputFilename);
+  if (Config.InputFormat == FileFormat::Binary ||
+      Config.InputFormat == FileFormat::IHex) {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
+        MemoryBuffer::getFileOrSTDIN(Config.InputFilename);
     if (!BufOrErr)
       return createFileError(Config.InputFilename, BufOrErr.getError());
-
-    if (Error E = writeToFile(
-            Config.OutputFilename, [&](raw_ostream &OutFile) -> Error {
-              return ProcessRaw(Config, *BufOrErr->get(), OutFile);
-            }))
-      return E;
+    MemoryBufferHolder = std::move(*BufOrErr);
+
+    if (Config.InputFormat == FileFormat::Binary)
+      ObjcopyFunc = [&](raw_ostream &OutFile) -> Error {
+        // Handle FileFormat::Binary.
+        return executeObjcopyOnRawBinary(Config, *MemoryBufferHolder, OutFile);
+      };
+    else
+      ObjcopyFunc = [&](raw_ostream &OutFile) -> Error {
+        // Handle FileFormat::IHex.
+        return executeObjcopyOnIHex(Config, *MemoryBufferHolder, OutFile);
+      };
   } else {
     Expected<OwningBinary<llvm::object::Binary>> BinaryOrErr =
         createBinary(Config.InputFilename);
     if (!BinaryOrErr)
       return createFileError(Config.InputFilename, BinaryOrErr.takeError());
+    BinaryHolder = std::move(*BinaryOrErr);
 
-    if (Archive *Ar = dyn_cast<Archive>(BinaryOrErr.get().getBinary())) {
+    if (Archive *Ar = dyn_cast<Archive>(BinaryHolder.getBinary())) {
+      // Handle Archive.
       if (Error E = executeObjcopyOnArchive(Config, *Ar))
         return E;
     } else {
-      if (Error E = writeToFile(
-              Config.OutputFilename,
-              [&](raw_ostream &OutFile) -> Error {
-                return executeObjcopyOnBinary(
-                    Config, *BinaryOrErr.get().getBinary(), OutFile);
-              },
-              Config.InputFilename != "-" &&
-                  Config.InputFilename == Config.OutputFilename,
-              Stat.getUser(), Stat.getGroup()))
+      // Handle llvm::object::Binary.
+      ObjcopyFunc = [&](raw_ostream &OutFile) -> Error {
+        return executeObjcopyOnBinary(Config, *BinaryHolder.getBinary(),
+                                      OutFile);
+      };
+    }
+  }
+
+  if (ObjcopyFunc) {
+    if (Config.SplitDWO.empty()) {
+      // Apply transformations described by Config and store result into
+      // Config.OutputFilename using specified ObjcopyFunc function.
+      if (Error E = writeToOutput(Config.OutputFilename, ObjcopyFunc))
+        return E;
+    } else {
+      Config.ExtractDWO = true;
+      Config.StripDWO = false;
+      // Copy .dwo tables from the Config.InputFilename into Config.SplitDWO
+      // file using specified ObjcopyFunc function.
+      if (Error E = writeToOutput(Config.SplitDWO, ObjcopyFunc))
+        return E;
+      Config.ExtractDWO = false;
+      Config.StripDWO = true;
+      // Apply transformations described by Config, remove .dwo tables and
+      // store result into Config.OutputFilename using specified ObjcopyFunc
+      // function.
+      if (Error E = writeToOutput(Config.OutputFilename, ObjcopyFunc))
         return E;
     }
   }
diff --git a/llvm/tools/llvm-objcopy/llvm-objcopy.h b/llvm/tools/llvm-objcopy/llvm-objcopy.h
index 3d080a011baac2b684afccda2021b3ed5076aeea..64b554f5c5aa3611faa4df9b3d840aee85a354e1 100644
--- a/llvm/tools/llvm-objcopy/llvm-objcopy.h
+++ b/llvm/tools/llvm-objcopy/llvm-objcopy.h
@@ -27,18 +27,6 @@ struct CopyConfig;
 Expected<std::vector<NewArchiveMember>>
 createNewArchiveMembers(CopyConfig &Config, const object::Archive &Ar);
 
-/// A writeToFile helper creates an output stream, based on the specified
-/// \p OutputFileName: std::outs for the "-", raw_null_ostream for
-/// the "/dev/null", temporary file in the same directory as the final output
-/// file for other names. The final output file is atomically replaced with
-/// the temporary file after \p Write handler is finished. \p KeepOwnership
-/// used to setting specified \p UserID and \p GroupID for the resulting file
-/// if writeToFile is called under /root.
-Error writeToFile(StringRef OutputFileName,
-                  std::function<Error(raw_ostream &)> Write,
-                  bool KeepOwnership = false, unsigned UserID = 0,
-                  unsigned GroupID = 0);
-
 } // end namespace objcopy
 } // end namespace llvm
 
diff --git a/llvm/tools/llvm-objcopy/wasm/Writer.cpp b/llvm/tools/llvm-objcopy/wasm/Writer.cpp
index bce24b8595733a749a05c5f0f8379844ebd1f9ac..2fad9e60c50f456f5f8ef0bdb39aa568cf1b047e 100644
--- a/llvm/tools/llvm-objcopy/wasm/Writer.cpp
+++ b/llvm/tools/llvm-objcopy/wasm/Writer.cpp
@@ -56,29 +56,21 @@ size_t Writer::finalize() {
 
 Error Writer::write() {
   size_t TotalSize = finalize();
-  std::unique_ptr<WritableMemoryBuffer> Buf =
-      WritableMemoryBuffer::getNewMemBuffer(TotalSize);
-  if (!Buf)
-    return createStringError(errc::not_enough_memory,
-                             "failed to allocate memory buffer of " +
-                                 Twine::utohexstr(TotalSize) + " bytes");
+  Out.reserveExtraSpace(TotalSize);
 
   // Write the header.
-  uint8_t *Ptr = reinterpret_cast<uint8_t *>(Buf->getBufferStart());
-  Ptr = std::copy(Obj.Header.Magic.begin(), Obj.Header.Magic.end(), Ptr);
-  support::endian::write32le(Ptr, Obj.Header.Version);
-  Ptr += sizeof(Obj.Header.Version);
+  Out.write(Obj.Header.Magic.data(), Obj.Header.Magic.size());
+  uint32_t Version;
+  support::endian::write32le(&Version, Obj.Header.Version);
+  Out.write(reinterpret_cast<const char *>(&Version), sizeof(Version));
 
   // Write each section.
   for (size_t I = 0, S = SectionHeaders.size(); I < S; ++I) {
-    Ptr = std::copy(SectionHeaders[I].begin(), SectionHeaders[I].end(), Ptr);
-    ArrayRef<uint8_t> Contents = Obj.Sections[I].Contents;
-    Ptr = std::copy(Contents.begin(), Contents.end(), Ptr);
+    Out.write(SectionHeaders[I].data(), SectionHeaders[I].size());
+    Out.write(reinterpret_cast<const char *>(Obj.Sections[I].Contents.data()),
+              Obj.Sections[I].Contents.size());
   }
 
-  // TODO: Implement direct writing to the output stream (without intermediate
-  // memory buffer Buf).
-  Out.write(Buf->getBufferStart(), Buf->getBufferSize());
   return Error::success();
 }
 
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 6fccaa694afa69de33703426a1b718c7bf6b5905..13d60f721add7bb70d9fb91c3df8da8ccf07da42 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -358,6 +358,12 @@ cl::opt<std::string> objdump::Prefix("prefix",
                                      cl::desc("Add prefix to absolute paths"),
                                      cl::cat(ObjdumpCat));
 
+cl::opt<uint32_t>
+    objdump::PrefixStrip("prefix-strip",
+                         cl::desc("Strip out initial directories from absolute "
+                                  "paths. No effect without --prefix"),
+                         cl::init(0), cl::cat(ObjdumpCat));
+
 enum DebugVarsFormat {
   DVDisabled,
   DVUnicode,
@@ -1031,6 +1037,27 @@ void SourcePrinter::printSourceLine(formatted_raw_ostream &OS,
   }
 
   if (!Prefix.empty() && sys::path::is_absolute_gnu(LineInfo.FileName)) {
+    // FileName has at least one character since is_absolute_gnu is false for
+    // an empty string.
+    assert(!LineInfo.FileName.empty());
+    if (PrefixStrip > 0) {
+      uint32_t Level = 0;
+      auto StrippedNameStart = LineInfo.FileName.begin();
+
+      // Path.h iterator skips extra separators. Therefore it cannot be used
+      // here to keep compatibility with GNU Objdump.
+      for (auto Pos = StrippedNameStart + 1, End = LineInfo.FileName.end();
+           Pos != End && Level < PrefixStrip; ++Pos) {
+        if (sys::path::is_separator(*Pos)) {
+          StrippedNameStart = Pos;
+          ++Level;
+        }
+      }
+
+      LineInfo.FileName =
+          std::string(StrippedNameStart, LineInfo.FileName.end());
+    }
+
     SmallString<128> FilePath;
     sys::path::append(FilePath, Prefix, LineInfo.FileName);
 
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.h b/llvm/tools/llvm-objdump/llvm-objdump.h
index d15bc059b45019b6a1a45ed002514d0e324a65dc..884702d11f8591be4445688247678fcc779b4c9a 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.h
+++ b/llvm/tools/llvm-objdump/llvm-objdump.h
@@ -42,6 +42,7 @@ extern cl::opt<std::string> MCPU;
 extern cl::opt<bool> NoShowRawInsn;
 extern cl::opt<bool> NoLeadingAddr;
 extern cl::opt<std::string> Prefix;
+extern cl::opt<uint32_t> PrefixStrip;
 extern cl::opt<bool> PrintImmHex;
 extern cl::opt<bool> PrivateHeaders;
 extern cl::opt<bool> Relocations;
diff --git a/llvm/tools/llvm-pdbutil/InputFile.cpp b/llvm/tools/llvm-pdbutil/InputFile.cpp
index b316882de64d36d01d31f21e4fc7adc88a8b1f27..40b35625b6f8d3dce2b2a5da766e6208d575079f 100644
--- a/llvm/tools/llvm-pdbutil/InputFile.cpp
+++ b/llvm/tools/llvm-pdbutil/InputFile.cpp
@@ -288,7 +288,8 @@ Expected<InputFile> InputFile::open(StringRef Path, bool AllowUnknownFile) {
         formatv("File {0} is not a supported file type", Path),
         inconvertibleErrorCode());
 
-  auto Result = MemoryBuffer::getFile(Path, -1LL, false);
+  auto Result = MemoryBuffer::getFile(Path, /*IsText=*/false,
+                                      /*RequiresNullTerminator=*/false);
   if (!Result)
     return make_error<StringError>(
         formatv("File {0} could not be opened", Path), Result.getError());
diff --git a/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp b/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp
index 19f4880ab5eb716045d96dc928e43558a14d3c02..f70558a5ab74b9767bf7cf05add2272acb62c92d 100644
--- a/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp
+++ b/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp
@@ -748,7 +748,7 @@ static ExitOnError ExitOnErr;
 static void yamlToPdb(StringRef Path) {
   BumpPtrAllocator Allocator;
   ErrorOr<std::unique_ptr<MemoryBuffer>> ErrorOrBuffer =
-      MemoryBuffer::getFileOrSTDIN(Path, /*FileSize=*/-1,
+      MemoryBuffer::getFileOrSTDIN(Path, /*IsText=*/false,
                                    /*RequiresNullTerminator=*/false);
 
   if (ErrorOrBuffer.getError()) {
diff --git a/llvm/tools/llvm-profgen/PerfReader.cpp b/llvm/tools/llvm-profgen/PerfReader.cpp
index 1f842008db42e9214861c9843ecc4119b3d8fa7f..0d60fa3332b48f9886ffa43ba0aa1a3eef692621 100644
--- a/llvm/tools/llvm-profgen/PerfReader.cpp
+++ b/llvm/tools/llvm-profgen/PerfReader.cpp
@@ -92,7 +92,8 @@ void VirtualUnwinder::unwindBranchWithinFrame(UnwindState &State) {
 std::shared_ptr<StringBasedCtxKey> FrameStack::getContextKey() {
   std::shared_ptr<StringBasedCtxKey> KeyStr =
       std::make_shared<StringBasedCtxKey>();
-  KeyStr->Context = Binary->getExpandedContextStr(Stack);
+  KeyStr->Context =
+      Binary->getExpandedContextStr(Stack, KeyStr->WasLeafInlined);
   if (KeyStr->Context.empty())
     return nullptr;
   KeyStr->genHashCode();
diff --git a/llvm/tools/llvm-profgen/PerfReader.h b/llvm/tools/llvm-profgen/PerfReader.h
index b802c212eb469103947c62f24ff6ff556d38b732..a1d319226864ae87c687867b23129c3661be0254 100644
--- a/llvm/tools/llvm-profgen/PerfReader.h
+++ b/llvm/tools/llvm-profgen/PerfReader.h
@@ -311,7 +311,8 @@ struct ContextKey {
 // String based context id
 struct StringBasedCtxKey : public ContextKey {
   std::string Context;
-  StringBasedCtxKey() : ContextKey(CK_StringBased){};
+  bool WasLeafInlined;
+  StringBasedCtxKey() : ContextKey(CK_StringBased), WasLeafInlined(false){};
   static bool classof(const ContextKey *K) {
     return K->getKind() == CK_StringBased;
   }
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index 77416d2ff989cfacf7629f8a12aaa26a25a8be1a..b3fb015b6725c4c7d58ad465bb6935908dca030f 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ProfileGenerator.h"
+#include "llvm/ProfileData/ProfileCommon.h"
 
 static cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
                                            cl::Required,
@@ -31,18 +32,23 @@ static cl::opt<int32_t, true> RecursionCompression(
     cl::Hidden,
     cl::location(llvm::sampleprof::CSProfileGenerator::MaxCompressionSize));
 
-static cl::opt<uint64_t> CSProfColdThres(
+static cl::opt<uint64_t> CSProfColdThreshold(
     "csprof-cold-thres", cl::init(100), cl::ZeroOrMore,
     cl::desc("Specify the total samples threshold for a context profile to "
              "be considered cold, any cold profiles will be merged into "
              "context-less base profiles"));
 
-static cl::opt<bool> CSProfKeepCold(
-    "csprof-keep-cold", cl::init(false), cl::ZeroOrMore,
+static cl::opt<bool> CSProfMergeColdContext(
+    "csprof-merge-cold-context", cl::init(true), cl::ZeroOrMore,
     cl::desc("This works together with --csprof-cold-thres. If the total count "
-             "of the profile after all merge is done is still smaller than the "
-             "csprof-cold-thres, it will be trimmed unless csprof-keep-cold "
-             "flag is specified."));
+             "of context profile is smaller than the threshold, it will be "
+             "merged into context-less base profile."));
+
+static cl::opt<bool> CSProfTrimColdContext(
+    "csprof-trim-cold-context", cl::init(true), cl::ZeroOrMore,
+    cl::desc("This works together with --csprof-cold-thres. If the total count "
+             "of the profile after all merge is done is still smaller than "
+             "threshold, it will be trimmed."));
 
 using namespace llvm;
 using namespace sampleprof;
@@ -188,12 +194,16 @@ void ProfileGenerator::findDisjointRanges(RangeSample &DisjointRanges,
 }
 
 FunctionSamples &
-CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr) {
+CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr,
+                                                 bool WasLeafInlined) {
   auto Ret = ProfileMap.try_emplace(ContextStr, FunctionSamples());
   if (Ret.second) {
     SampleContext FContext(Ret.first->first(), RawContext);
+    if (WasLeafInlined)
+      FContext.setAttribute(ContextWasInlined);
     FunctionSamples &FProfile = Ret.first->second;
     FProfile.setContext(FContext);
+    FProfile.setName(FContext.getNameWithoutContext());
   }
   return Ret.first->second;
 }
@@ -208,7 +218,7 @@ void CSProfileGenerator::generateProfile() {
       StringRef ContextId(CtxKey->Context);
       // Get or create function profile for the range
       FunctionSamples &FunctionProfile =
-          getFunctionProfileForContext(ContextId);
+          getFunctionProfileForContext(ContextId, CtxKey->WasLeafInlined);
 
       // Fill in function body samples
       populateFunctionBodySamples(FunctionProfile, CI.second.RangeCounter,
@@ -223,6 +233,10 @@ void CSProfileGenerator::generateProfile() {
   // functions, we estimate it from inlinee's profile using the entry of the
   // body sample.
   populateInferredFunctionSamples();
+
+  // Compute hot/cold threshold based on profile. This will be used for cold
+  // context profile merging/trimming.
+  computeSummaryAndThreshold();
 }
 
 void CSProfileGenerator::updateBodySamplesforFunctionProfile(
@@ -378,36 +392,54 @@ void CSProfileGenerator::populateInferredFunctionSamples() {
   }
 }
 
+void CSProfileGenerator::computeSummaryAndThreshold() {
+  SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
+  auto Summary = Builder.computeSummaryForProfiles(ProfileMap);
+  PSI.reset(new ProfileSummaryInfo(std::move(Summary)));
+}
+
 void CSProfileGenerator::mergeAndTrimColdProfile(
     StringMap<FunctionSamples> &ProfileMap) {
+  if (!CSProfMergeColdContext && !CSProfTrimColdContext)
+    return;
+
+  // Use threshold calculated from profile summary unless specified
+  uint64_t ColdThreshold = PSI->getColdCountThreshold();
+  if (CSProfColdThreshold.getNumOccurrences()) {
+    ColdThreshold = CSProfColdThreshold;
+  }
+
   // Nothing to merge if sample threshold is zero
-  if (!CSProfColdThres)
+  if (ColdThreshold == 0)
     return;
 
   // Filter the cold profiles from ProfileMap and move them into a tmp
   // container
-  std::vector<std::pair<StringRef, const FunctionSamples *>> ToRemoveVec;
+  std::vector<std::pair<StringRef, const FunctionSamples *>> ColdProfiles;
   for (const auto &I : ProfileMap) {
     const FunctionSamples &FunctionProfile = I.second;
-    if (FunctionProfile.getTotalSamples() >= CSProfColdThres)
+    if (FunctionProfile.getTotalSamples() >= ColdThreshold)
       continue;
-    ToRemoveVec.emplace_back(I.getKey(), &I.second);
+    ColdProfiles.emplace_back(I.getKey(), &I.second);
   }
 
   // Remove the code profile from ProfileMap and merge them into BaseProileMap
   StringMap<FunctionSamples> BaseProfileMap;
-  for (const auto &I : ToRemoveVec) {
-    auto Ret = BaseProfileMap.try_emplace(
-        I.second->getContext().getNameWithoutContext(), FunctionSamples());
-    FunctionSamples &BaseProfile = Ret.first->second;
-    BaseProfile.merge(*I.second);
+  for (const auto &I : ColdProfiles) {
+    if (CSProfMergeColdContext) {
+      auto Ret = BaseProfileMap.try_emplace(
+          I.second->getContext().getNameWithoutContext(), FunctionSamples());
+      FunctionSamples &BaseProfile = Ret.first->second;
+      BaseProfile.merge(*I.second);
+    }
     ProfileMap.erase(I.first);
   }
 
   // Merge the base profiles into ProfileMap;
   for (const auto &I : BaseProfileMap) {
     // Filter the cold base profile
-    if (!CSProfKeepCold && I.second.getTotalSamples() < CSProfColdThres &&
+    if (CSProfTrimColdContext &&
+        I.second.getTotalSamples() < CSProfColdThreshold &&
         ProfileMap.find(I.getKey()) == ProfileMap.end())
       continue;
     // Merge the profile if the original profile exists, otherwise just insert
@@ -428,6 +460,7 @@ void CSProfileGenerator::write(std::unique_ptr<SampleProfileWriter> Writer,
     assert(Ret.second && "Must be a unique context");
     SampleContext FContext(Ret.first->first(), RawContext);
     FunctionSamples &FProfile = Ret.first->second;
+    FContext.setAllAttributes(FProfile.getContext().getAllAttributes());
     FProfile.setName(FContext.getNameWithContext(true));
     FProfile.setContext(FContext);
   }
@@ -466,6 +499,10 @@ void PseudoProbeCSProfileGenerator::generateProfile() {
                                         ContextStrStack, Binary);
     }
   }
+
+  // Compute hot/cold threshold based on profile. This will be used for cold
+  // context profile merging/trimming.
+  computeSummaryAndThreshold();
 }
 
 void PseudoProbeCSProfileGenerator::extractProbesFromRange(
@@ -587,7 +624,7 @@ void PseudoProbeCSProfileGenerator::populateBoundarySamplesWithProbes(
 
 FunctionSamples &PseudoProbeCSProfileGenerator::getFunctionProfileForLeafProbe(
     SmallVectorImpl<std::string> &ContextStrStack,
-    const PseudoProbeFuncDesc *LeafFuncDesc) {
+    const PseudoProbeFuncDesc *LeafFuncDesc, bool WasLeafInlined) {
   assert(ContextStrStack.size() && "Profile context must have the leaf frame");
   // Compress the context string except for the leaf frame
   std::string LeafFrame = ContextStrStack.back();
@@ -608,7 +645,7 @@ FunctionSamples &PseudoProbeCSProfileGenerator::getFunctionProfileForLeafProbe(
   OContextStr << StringRef(LeafFrame).split(":").first.str();
 
   FunctionSamples &FunctionProile =
-      getFunctionProfileForContext(OContextStr.str());
+      getFunctionProfileForContext(OContextStr.str(), WasLeafInlined);
   FunctionProile.setFunctionHash(LeafFuncDesc->FuncHash);
   return FunctionProile;
 }
@@ -619,13 +656,11 @@ FunctionSamples &PseudoProbeCSProfileGenerator::getFunctionProfileForLeafProbe(
   // Explicitly copy the context for appending the leaf context
   SmallVector<std::string, 16> ContextStrStackCopy(ContextStrStack.begin(),
                                                    ContextStrStack.end());
-  Binary->getInlineContextForProbe(LeafProbe, ContextStrStackCopy);
-  // Note that the context from probe doesn't include leaf frame,
-  // hence we need to retrieve and append the leaf frame.
+  Binary->getInlineContextForProbe(LeafProbe, ContextStrStackCopy, true);
   const auto *FuncDesc = Binary->getFuncDescForGUID(LeafProbe->GUID);
-  ContextStrStackCopy.emplace_back(FuncDesc->FuncName + ":" +
-                                   Twine(LeafProbe->Index).str());
-  return getFunctionProfileForLeafProbe(ContextStrStackCopy, FuncDesc);
+  bool WasLeafInlined = LeafProbe->InlineTree->hasInlineSite();
+  return getFunctionProfileForLeafProbe(ContextStrStackCopy, FuncDesc,
+                                        WasLeafInlined);
 }
 
 } // end namespace sampleprof
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h
index 4ea459e7dabb2f33ce89eb1aab6632a5ba472582..ff0116fb5c351dd91ea9d6fdc866fc3d9432ae8a 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.h
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.h
@@ -11,7 +11,9 @@
 #include "ErrorHandling.h"
 #include "PerfReader.h"
 #include "ProfiledBinary.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/ProfileData/SampleProfWriter.h"
+#include <memory>
 
 using namespace llvm;
 using namespace sampleprof;
@@ -174,10 +176,12 @@ public:
 
 protected:
   // Lookup or create FunctionSamples for the context
-  FunctionSamples &getFunctionProfileForContext(StringRef ContextId);
+  FunctionSamples &getFunctionProfileForContext(StringRef ContextId,
+                                                bool WasLeafInlined = false);
   // Merge cold context profile whose total sample is below threshold
   // into base profile.
   void mergeAndTrimColdProfile(StringMap<FunctionSamples> &ProfileMap);
+  void computeSummaryAndThreshold();
   void write(std::unique_ptr<SampleProfileWriter> Writer,
              StringMap<FunctionSamples> &ProfileMap) override;
 
@@ -196,6 +200,9 @@ private:
                                        ProfiledBinary *Binary);
   void populateInferredFunctionSamples();
 
+  // Profile summary to answer isHotCount and isColdCount queries.
+  std::unique_ptr<ProfileSummaryInfo> PSI;
+
 public:
   // Deduplicate adjacent repeated context sequences up to a given sequence
   // length. -1 means no size limit.
@@ -229,7 +236,8 @@ private:
   // Helper function to get FunctionSamples for the leaf inlined context
   FunctionSamples &
   getFunctionProfileForLeafProbe(SmallVectorImpl<std::string> &ContextStrStack,
-                                 const PseudoProbeFuncDesc *LeafFuncDesc);
+                                 const PseudoProbeFuncDesc *LeafFuncDesc,
+                                 bool WasLeafInlined);
   // Helper function to get FunctionSamples for the leaf probe
   FunctionSamples &
   getFunctionProfileForLeafProbe(SmallVectorImpl<std::string> &ContextStrStack,
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
index 2d6cbfe474fd3a4b9d9610cf4ec79e6f8620599b..9063f06f5579239e7aa8319bca52db9a6b9900ea 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
@@ -131,8 +131,9 @@ bool ProfiledBinary::inlineContextEqual(uint64_t Address1,
                     Context2.begin(), Context2.begin() + Context2.size() - 1);
 }
 
-std::string ProfiledBinary::getExpandedContextStr(
-    const SmallVectorImpl<uint64_t> &Stack) const {
+std::string
+ProfiledBinary::getExpandedContextStr(const SmallVectorImpl<uint64_t> &Stack,
+                                      bool &WasLeafInlined) const {
   std::string ContextStr;
   SmallVector<std::string, 16> ContextVec;
   // Process from frame root to leaf
@@ -143,6 +144,9 @@ std::string ProfiledBinary::getExpandedContextStr(
     // processing
     if (ExpandedContext.empty())
       return std::string();
+    // Set WasLeafInlined to the size of inlined frame count for the last
+    // address which is leaf
+    WasLeafInlined = (ExpandedContext.size() > 1);
     for (const auto &Loc : ExpandedContext) {
       ContextVec.push_back(getCallSite(Loc));
     }
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h
index 7ceca1c5995c9be14000612b6ff016fa4aa65c03..b56574e0bf6fc26efafdca9870a3facfb7c5a098 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.h
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.h
@@ -239,8 +239,8 @@ public:
   // Get the context string of the current stack with inline context filled in.
   // It will search the disassembling info stored in Offset2LocStackMap. This is
   // used as the key of function sample map
-  std::string
-  getExpandedContextStr(const SmallVectorImpl<uint64_t> &Stack) const;
+  std::string getExpandedContextStr(const SmallVectorImpl<uint64_t> &Stack,
+                                    bool &WasLeafInlined) const;
 
   const PseudoProbe *getCallProbeForAddr(uint64_t Address) const {
     return ProbeDecoder.getCallProbeForAddr(Address);
diff --git a/llvm/tools/llvm-rc/ResourceFileWriter.cpp b/llvm/tools/llvm-rc/ResourceFileWriter.cpp
index 553bb754aea0b3d3d8998e576b7c73f7ab5b3260..2856fa8fe08a8e4078633d6ac5b65b6773b2a9cf 100644
--- a/llvm/tools/llvm-rc/ResourceFileWriter.cpp
+++ b/llvm/tools/llvm-rc/ResourceFileWriter.cpp
@@ -1524,14 +1524,16 @@ ResourceFileWriter::loadFile(StringRef File) const {
   // properly though, so if using that to append paths below, this early
   // exception case could be removed.)
   if (sys::path::has_root_directory(File))
-    return errorOrToExpected(MemoryBuffer::getFile(File, -1, false));
+    return errorOrToExpected(MemoryBuffer::getFile(
+        File, /*IsText=*/false, /*RequiresNullTerminator=*/false));
 
   // 1. The current working directory.
   sys::fs::current_path(Cwd);
   Path.assign(Cwd.begin(), Cwd.end());
   sys::path::append(Path, File);
   if (sys::fs::exists(Path))
-    return errorOrToExpected(MemoryBuffer::getFile(Path, -1, false));
+    return errorOrToExpected(MemoryBuffer::getFile(
+        Path, /*IsText=*/false, /*RequiresNullTerminator=*/false));
 
   // 2. The directory of the input resource file, if it is different from the
   // current working directory.
@@ -1539,19 +1541,22 @@ ResourceFileWriter::loadFile(StringRef File) const {
   Path.assign(InputFileDir.begin(), InputFileDir.end());
   sys::path::append(Path, File);
   if (sys::fs::exists(Path))
-    return errorOrToExpected(MemoryBuffer::getFile(Path, -1, false));
+    return errorOrToExpected(MemoryBuffer::getFile(
+        Path, /*IsText=*/false, /*RequiresNullTerminator=*/false));
 
   // 3. All of the include directories specified on the command line.
   for (StringRef ForceInclude : Params.Include) {
     Path.assign(ForceInclude.begin(), ForceInclude.end());
     sys::path::append(Path, File);
     if (sys::fs::exists(Path))
-      return errorOrToExpected(MemoryBuffer::getFile(Path, -1, false));
+      return errorOrToExpected(MemoryBuffer::getFile(
+          Path, /*IsText=*/false, /*RequiresNullTerminator=*/false));
   }
 
   if (auto Result =
           llvm::sys::Process::FindInEnvPath("INCLUDE", File, Params.NoInclude))
-    return errorOrToExpected(MemoryBuffer::getFile(*Result, -1, false));
+    return errorOrToExpected(MemoryBuffer::getFile(
+        *Result, /*IsText=*/false, /*RequiresNullTerminator=*/false));
 
   return make_error<StringError>("error : file not found : " + Twine(File),
                                  inconvertibleErrorCode());
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 2535ae7830ae529992ec22fbcaaa7fd142fa74db..55162dd8f2df580fa9b4f3119d2ee1d337d6d1ae 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -1430,7 +1430,7 @@ static const EnumEntry<unsigned> ElfHeaderMipsFlags[] = {
   ENUM_ENT(EF_MIPS_ARCH_64R6, "mips64r6")
 };
 
-static const EnumEntry<unsigned> ElfHeaderAMDGPUFlags[] = {
+static const EnumEntry<unsigned> ElfHeaderAMDGPUFlagsABIVersion3[] = {
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_NONE),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_R600),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_R630),
@@ -1477,8 +1477,63 @@ static const EnumEntry<unsigned> ElfHeaderAMDGPUFlags[] = {
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1031),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1032),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1033),
-  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_XNACK),
-  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_SRAM_ECC)
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_XNACK_V3),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_SRAMECC_V3)
+};
+
+static const EnumEntry<unsigned> ElfHeaderAMDGPUFlagsABIVersion4[] = {
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_NONE),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_R600),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_R630),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RS880),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV670),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV710),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV730),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_RV770),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CEDAR),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CYPRESS),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_JUNIPER),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_REDWOOD),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_SUMO),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_BARTS),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CAICOS),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_CAYMAN),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_R600_TURKS),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX600),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX601),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX602),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX700),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX701),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX702),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX703),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX704),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX705),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX801),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX802),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX803),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX805),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX810),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX900),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX902),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX904),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX906),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX908),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX909),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX90A),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX90C),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1010),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1011),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1012),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1030),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1031),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1032),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_MACH_AMDGCN_GFX1033),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_XNACK_ANY_V4),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_XNACK_OFF_V4),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_XNACK_ON_V4),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_SRAMECC_ANY_V4),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_SRAMECC_OFF_V4),
+  LLVM_READOBJ_ENUM_ENT(ELF, EF_AMDGPU_FEATURE_SRAMECC_ON_V4)
 };
 
 static const EnumEntry<unsigned> ElfHeaderRISCVFlags[] = {
@@ -4736,47 +4791,6 @@ static std::string getGNUProperty(uint32_t Type, uint32_t DataSize,
     if (PrData)
       OS << format("<unknown flags: 0x%x>", PrData);
     return OS.str();
-  case GNU_PROPERTY_X86_ISA_1_NEEDED:
-  case GNU_PROPERTY_X86_ISA_1_USED:
-    OS << "x86 ISA "
-       << (Type == GNU_PROPERTY_X86_ISA_1_NEEDED ? "needed: " : "used: ");
-    if (DataSize != 4) {
-      OS << format("<corrupt length: 0x%x>", DataSize);
-      return OS.str();
-    }
-    PrData = support::endian::read32<ELFT::TargetEndianness>(Data.data());
-    if (PrData == 0) {
-      OS << "<None>";
-      return OS.str();
-    }
-    DumpBit(GNU_PROPERTY_X86_ISA_1_CMOV, "CMOV");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_SSE, "SSE");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_SSE2, "SSE2");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_SSE3, "SSE3");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_SSSE3, "SSSE3");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_SSE4_1, "SSE4_1");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_SSE4_2, "SSE4_2");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX, "AVX");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX2, "AVX2");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_FMA, "FMA");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512F, "AVX512F");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512CD, "AVX512CD");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512ER, "AVX512ER");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512PF, "AVX512PF");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512VL, "AVX512VL");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512DQ, "AVX512DQ");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512BW, "AVX512BW");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512_4FMAPS, "AVX512_4FMAPS");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512_4VNNIW, "AVX512_4VNNIW");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512_BITALG, "AVX512_BITALG");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512_IFMA, "AVX512_IFMA");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512_VBMI, "AVX512_VBMI");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512_VBMI2, "AVX512_VBMI2");
-    DumpBit(GNU_PROPERTY_X86_ISA_1_AVX512_VNNI, "AVX512_VNNI");
-    if (PrData)
-      OS << format("<unknown flags: 0x%x>", PrData);
-    return OS.str();
-    break;
   case GNU_PROPERTY_X86_FEATURE_2_NEEDED:
   case GNU_PROPERTY_X86_FEATURE_2_USED:
     OS << "x86 feature "
@@ -4803,6 +4817,26 @@ static std::string getGNUProperty(uint32_t Type, uint32_t DataSize,
     if (PrData)
       OS << format("<unknown flags: 0x%x>", PrData);
     return OS.str();
+  case GNU_PROPERTY_X86_ISA_1_NEEDED:
+  case GNU_PROPERTY_X86_ISA_1_USED:
+    OS << "x86 ISA "
+       << (Type == GNU_PROPERTY_X86_ISA_1_NEEDED ? "needed: " : "used: ");
+    if (DataSize != 4) {
+      OS << format("<corrupt length: 0x%x>", DataSize);
+      return OS.str();
+    }
+    PrData = support::endian::read32<ELFT::TargetEndianness>(Data.data());
+    if (PrData == 0) {
+      OS << "<None>";
+      return OS.str();
+    }
+    DumpBit(GNU_PROPERTY_X86_ISA_1_BASELINE, "x86-64-baseline");
+    DumpBit(GNU_PROPERTY_X86_ISA_1_V2, "x86-64-v2");
+    DumpBit(GNU_PROPERTY_X86_ISA_1_V3, "x86-64-v3");
+    DumpBit(GNU_PROPERTY_X86_ISA_1_V4, "x86-64-v4");
+    if (PrData)
+      OS << format("<unknown flags: 0x%x>", PrData);
+    return OS.str();
   }
 }
 
@@ -4965,15 +4999,95 @@ static AMDNote getAMDNote(uint32_t NoteType, ArrayRef<uint8_t> Desc) {
   switch (NoteType) {
   default:
     return {"", ""};
-  case ELF::NT_AMD_AMDGPU_HSA_METADATA:
+  case ELF::NT_AMD_HSA_CODE_OBJECT_VERSION: {
+    struct CodeObjectVersion {
+      uint32_t MajorVersion;
+      uint32_t MinorVersion;
+    };
+    if (Desc.size() != sizeof(CodeObjectVersion))
+      return {"AMD HSA Code Object Version",
+              "Invalid AMD HSA Code Object Version"};
+    std::string VersionString;
+    raw_string_ostream StrOS(VersionString);
+    auto Version = reinterpret_cast<const CodeObjectVersion *>(Desc.data());
+    StrOS << "[Major: " << Version->MajorVersion
+          << ", Minor: " << Version->MinorVersion << "]";
+    return {"AMD HSA Code Object Version", VersionString};
+  }
+  case ELF::NT_AMD_HSA_HSAIL: {
+    struct HSAILProperties {
+      uint32_t HSAILMajorVersion;
+      uint32_t HSAILMinorVersion;
+      uint8_t Profile;
+      uint8_t MachineModel;
+      uint8_t DefaultFloatRound;
+    };
+    if (Desc.size() != sizeof(HSAILProperties))
+      return {"AMD HSA HSAIL Properties", "Invalid AMD HSA HSAIL Properties"};
+    auto Properties = reinterpret_cast<const HSAILProperties *>(Desc.data());
+    std::string HSAILPropetiesString;
+    raw_string_ostream StrOS(HSAILPropetiesString);
+    StrOS << "[HSAIL Major: " << Properties->HSAILMajorVersion
+          << ", HSAIL Minor: " << Properties->HSAILMinorVersion
+          << ", Profile: " << Properties->Profile
+          << ", Machine Model: " << Properties->MachineModel
+          << ", Default Float Round: " << Properties->DefaultFloatRound << "]";
+    return {"AMD HSA HSAIL Properties", HSAILPropetiesString};
+  }
+  case ELF::NT_AMD_HSA_ISA_VERSION: {
+    struct IsaVersion {
+      uint16_t VendorNameSize;
+      uint16_t ArchitectureNameSize;
+      uint32_t Major;
+      uint32_t Minor;
+      uint32_t Stepping;
+    };
+    if (Desc.size() < sizeof(IsaVersion))
+      return {"AMD HSA ISA Version", "Invalid AMD HSA ISA Version"};
+    auto Isa = reinterpret_cast<const IsaVersion *>(Desc.data());
+    if (Desc.size() < sizeof(IsaVersion) +
+                          Isa->VendorNameSize + Isa->ArchitectureNameSize ||
+        Isa->VendorNameSize == 0 || Isa->ArchitectureNameSize == 0)
+      return {"AMD HSA ISA Version", "Invalid AMD HSA ISA Version"};
+    std::string IsaString;
+    raw_string_ostream StrOS(IsaString);
+    StrOS << "[Vendor: "
+          << StringRef((const char*)Desc.data() + sizeof(IsaVersion), Isa->VendorNameSize - 1)
+          << ", Architecture: "
+          << StringRef((const char*)Desc.data() + sizeof(IsaVersion) + Isa->VendorNameSize,
+                       Isa->ArchitectureNameSize - 1)
+          << ", Major: " << Isa->Major << ", Minor: " << Isa->Minor
+          << ", Stepping: " << Isa->Stepping << "]";
+    return {"AMD HSA ISA Version", IsaString};
+  }
+  case ELF::NT_AMD_HSA_METADATA: {
+    if (Desc.size() == 0)
+      return {"AMD HSA Metadata", ""};
     return {
-        "HSA Metadata",
-        std::string(reinterpret_cast<const char *>(Desc.data()), Desc.size())};
-  case ELF::NT_AMD_AMDGPU_ISA:
+        "AMD HSA Metadata",
+        std::string(reinterpret_cast<const char *>(Desc.data()), Desc.size() - 1)};
+  }
+  case ELF::NT_AMD_HSA_ISA_NAME: {
+    if (Desc.size() == 0)
+      return {"AMD HSA ISA Name", ""};
     return {
-        "ISA Version",
+        "AMD HSA ISA Name",
         std::string(reinterpret_cast<const char *>(Desc.data()), Desc.size())};
   }
+  case ELF::NT_AMD_PAL_METADATA: {
+    struct PALMetadata {
+      uint32_t Key;
+      uint32_t Value;
+    };
+    auto Isa = reinterpret_cast<const PALMetadata *>(Desc.data());
+    std::string MetadataString;
+    raw_string_ostream StrOS(MetadataString);
+    for (size_t I = 0, E = Desc.size() / sizeof(PALMetadata); I < E; ++E) {
+      StrOS << "[" << Isa[I].Key << ": " << Isa[I].Value << "]";
+    }
+    return {"AMD PAL Metadata", MetadataString};
+  }
+  }
 }
 
 struct AMDGPUNote {
@@ -4994,11 +5108,11 @@ static AMDGPUNote getAMDGPUNote(uint32_t NoteType, ArrayRef<uint8_t> Desc) {
       return {"", ""};
 
     AMDGPU::HSAMD::V3::MetadataVerifier Verifier(true);
-    std::string HSAMetadataString;
+    std::string MetadataString;
     if (!Verifier.verify(MsgPackDoc.getRoot()))
-      HSAMetadataString = "Invalid AMDGPU Metadata\n";
+      MetadataString = "Invalid AMDGPU Metadata\n";
 
-    raw_string_ostream StrOS(HSAMetadataString);
+    raw_string_ostream StrOS(MetadataString);
     if (MsgPackDoc.getRoot().isScalar()) {
       // TODO: passing a scalar root to toYAML() asserts:
       // (PolymorphicTraits<T>::getKind(Val) != NodeKind::Scalar &&
@@ -5127,11 +5241,13 @@ static const NoteType FreeBSDNoteTypes[] = {
 };
 
 static const NoteType AMDNoteTypes[] = {
-    {ELF::NT_AMD_AMDGPU_HSA_METADATA,
-     "NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)"},
-    {ELF::NT_AMD_AMDGPU_ISA, "NT_AMD_AMDGPU_ISA (ISA Version)"},
-    {ELF::NT_AMD_AMDGPU_PAL_METADATA,
-     "NT_AMD_AMDGPU_PAL_METADATA (PAL Metadata)"},
+    {ELF::NT_AMD_HSA_CODE_OBJECT_VERSION,
+     "NT_AMD_HSA_CODE_OBJECT_VERSION (AMD HSA Code Object Version)"},
+    {ELF::NT_AMD_HSA_HSAIL, "NT_AMD_HSA_HSAIL (AMD HSA HSAIL Properties)"},
+    {ELF::NT_AMD_HSA_ISA_VERSION, "NT_AMD_HSA_ISA_VERSION (AMD HSA ISA Version)"},
+    {ELF::NT_AMD_HSA_METADATA, "NT_AMD_HSA_METADATA (AMD HSA Metadata)"},
+    {ELF::NT_AMD_HSA_ISA_NAME, "NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)"},
+    {ELF::NT_AMD_PAL_METADATA, "NT_AMD_PAL_METADATA (AMD PAL Metadata)"},
 };
 
 static const NoteType AMDGPUNoteTypes[] = {
@@ -6071,10 +6187,28 @@ template <class ELFT> void LLVMELFDumper<ELFT>::printFileHeaders() {
       W.printFlags("Flags", E.e_flags, makeArrayRef(ElfHeaderMipsFlags),
                    unsigned(ELF::EF_MIPS_ARCH), unsigned(ELF::EF_MIPS_ABI),
                    unsigned(ELF::EF_MIPS_MACH));
-    else if (E.e_machine == EM_AMDGPU)
-      W.printFlags("Flags", E.e_flags, makeArrayRef(ElfHeaderAMDGPUFlags),
-                   unsigned(ELF::EF_AMDGPU_MACH));
-    else if (E.e_machine == EM_RISCV)
+    else if (E.e_machine == EM_AMDGPU) {
+      switch (E.e_ident[ELF::EI_ABIVERSION]) {
+      default:
+        W.printHex("Flags", E.e_flags);
+        break;
+      case 0:
+        // ELFOSABI_AMDGPU_PAL, ELFOSABI_AMDGPU_MESA3D support *_V3 flags.
+        LLVM_FALLTHROUGH;
+      case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
+        W.printFlags("Flags", E.e_flags,
+                     makeArrayRef(ElfHeaderAMDGPUFlagsABIVersion3),
+                     unsigned(ELF::EF_AMDGPU_MACH));
+        break;
+      case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
+        W.printFlags("Flags", E.e_flags,
+                     makeArrayRef(ElfHeaderAMDGPUFlagsABIVersion4),
+                     unsigned(ELF::EF_AMDGPU_MACH),
+                     unsigned(ELF::EF_AMDGPU_FEATURE_XNACK_V4),
+                     unsigned(ELF::EF_AMDGPU_FEATURE_SRAMECC_V4));
+        break;
+      }
+    } else if (E.e_machine == EM_RISCV)
       W.printFlags("Flags", E.e_flags, makeArrayRef(ElfHeaderRISCVFlags));
     else
       W.printFlags("Flags", E.e_flags);
@@ -6549,6 +6683,7 @@ template <class ELFT> void LLVMELFDumper<ELFT>::printBBAddrMaps() {
         W.printBoolean("HasReturn", BBE.HasReturn);
         W.printBoolean("HasTailCall", BBE.HasTailCall);
         W.printBoolean("IsEHPad", BBE.IsEHPad);
+        W.printBoolean("CanFallThrough", BBE.CanFallThrough);
       }
     }
   }
diff --git a/llvm/tools/llvm-readobj/WasmDumper.cpp b/llvm/tools/llvm-readobj/WasmDumper.cpp
index fb7134d20a8548e3d868de2de941067ee0077e4a..0ee8c69122e98e9c8d37506927477cf26e34e246 100644
--- a/llvm/tools/llvm-readobj/WasmDumper.cpp
+++ b/llvm/tools/llvm-readobj/WasmDumper.cpp
@@ -192,7 +192,7 @@ void WasmDumper::printSectionHeaders() {
       ListScope Group(W, "Memories");
       for (const wasm::WasmLimits &Memory : Obj->memories()) {
         DictScope Group(W, "Memory");
-        W.printNumber("InitialPages", Memory.Initial);
+        W.printNumber("MinPages", Memory.Minimum);
         if (Memory.Flags & wasm::WASM_LIMITS_FLAG_HAS_MAX) {
           W.printNumber("MaxPages", WasmSec.Offset);
         }
diff --git a/llvm/tools/llvm-readobj/llvm-readobj.cpp b/llvm/tools/llvm-readobj/llvm-readobj.cpp
index 5d6ee25961f275990f1e58ec08cd8ad08b5115c5..7db52e63da47888375cb76610faf51862ee7901a 100644
--- a/llvm/tools/llvm-readobj/llvm-readobj.cpp
+++ b/llvm/tools/llvm-readobj/llvm-readobj.cpp
@@ -653,7 +653,7 @@ static void dumpWindowsResourceFile(WindowsResource *WinRes,
 /// Opens \a File and dumps it.
 static void dumpInput(StringRef File, ScopedPrinter &Writer) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
-      MemoryBuffer::getFileOrSTDIN(File, /*FileSize=*/-1,
+      MemoryBuffer::getFileOrSTDIN(File, /*IsText=*/false,
                                    /*RequiresNullTerminator=*/false);
   if (std::error_code EC = FileOrErr.getError())
     return reportError(errorCodeToError(EC), File);
diff --git a/llvm/tools/obj2yaml/obj2yaml.cpp b/llvm/tools/obj2yaml/obj2yaml.cpp
index da70450503f33eddb9e2a987b8b29f9e1fa1701a..ff6b470f5244b85db4aaf24360edf77e102f7ed1 100644
--- a/llvm/tools/obj2yaml/obj2yaml.cpp
+++ b/llvm/tools/obj2yaml/obj2yaml.cpp
@@ -36,7 +36,7 @@ static Error dumpObject(const ObjectFile &Obj) {
 
 static Error dumpInput(StringRef File) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
-      MemoryBuffer::getFileOrSTDIN(File, /*FileSize=*/-1,
+      MemoryBuffer::getFileOrSTDIN(File, /*IsText=*/false,
                                    /*RequiresNullTerminator=*/false);
   if (std::error_code EC = FileOrErr.getError())
     return errorCodeToError(EC);
diff --git a/llvm/tools/obj2yaml/wasm2yaml.cpp b/llvm/tools/obj2yaml/wasm2yaml.cpp
index c421b14331c16cc81eeb12f89001445f2b87b579..5ddf2c486610f424afec1497ec8cac20a27e2749 100644
--- a/llvm/tools/obj2yaml/wasm2yaml.cpp
+++ b/llvm/tools/obj2yaml/wasm2yaml.cpp
@@ -34,7 +34,7 @@ public:
 static WasmYAML::Limits makeLimits(const wasm::WasmLimits &Limits) {
   WasmYAML::Limits L;
   L.Flags = Limits.Flags;
-  L.Initial = Limits.Initial;
+  L.Minimum = Limits.Minimum;
   L.Maximum = Limits.Maximum;
   return L;
 }
diff --git a/llvm/tools/sanstats/sanstats.cpp b/llvm/tools/sanstats/sanstats.cpp
index 1f154e08f2434c121ce209a48bc5b20857e00165..54ad35b2ace6626708ce264ff6573c9389f33821 100644
--- a/llvm/tools/sanstats/sanstats.cpp
+++ b/llvm/tools/sanstats/sanstats.cpp
@@ -125,8 +125,8 @@ int main(int argc, char **argv) {
   cl::ParseCommandLineOptions(argc, argv,
                               "Sanitizer Statistics Processing Tool");
 
-  ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr =
-      MemoryBuffer::getFile(ClInputFile, -1, false);
+  ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getFile(
+      ClInputFile, /*IsText=*/false, /*RequiresNullTerminator=*/false);
   if (!MBOrErr) {
     errs() << argv[0] << ": " << ClInputFile << ": "
            << MBOrErr.getError().message() << '\n';
diff --git a/llvm/unittests/ADT/BitVectorTest.cpp b/llvm/unittests/ADT/BitVectorTest.cpp
index 0f15a478e4523c575f4af3628999205954c0fc48..1779f2fcc9c2748c24ff94cbdb2c41e7d93d36f0 100644
--- a/llvm/unittests/ADT/BitVectorTest.cpp
+++ b/llvm/unittests/ADT/BitVectorTest.cpp
@@ -779,6 +779,7 @@ TYPED_TEST(BitVectorTest, ProxyIndex) {
   EXPECT_TRUE(Vec.none());
 }
 
+
 TYPED_TEST(BitVectorTest, PortableBitMask) {
   TypeParam A;
   const uint32_t Mask1[] = { 0x80000000, 6, 5 };
@@ -1142,10 +1143,12 @@ TYPED_TEST(BitVectorTest, Iterators) {
 
   TypeParam Empty;
   EXPECT_EQ(Empty.set_bits_begin(), Empty.set_bits_end());
+  int BitCount = 0;
   for (unsigned Bit : Empty.set_bits()) {
     (void)Bit;
-    EXPECT_TRUE(false);
+    BitCount++;
   }
+  ASSERT_EQ(BitCount, 0);
 
   TypeParam ToFill(100, false);
   ToFill.set(0);
@@ -1259,4 +1262,40 @@ TYPED_TEST(BitVectorTest, DenseMapHashing) {
   }
 }
 
+TEST(BitVectoryTest, Apply) {
+  for (int i = 0; i < 2; ++i) {
+    int j = i * 100 + 3;
+
+    const BitVector x =
+        createBitVector<BitVector>(j + 5, {{i, i + 1}, {j - 1, j}});
+    const BitVector y = createBitVector<BitVector>(j + 5, {{i, j}});
+    const BitVector z =
+        createBitVector<BitVector>(j + 5, {{i + 1, i + 2}, {j, j + 1}});
+
+    auto op0 = [](auto x) { return ~x; };
+    BitVector expected0 = x;
+    expected0.flip();
+    BitVector out0(j - 2);
+    BitVector::apply(op0, out0, x);
+    EXPECT_EQ(out0, expected0);
+
+    auto op1 = [](auto x, auto y) { return x & ~y; };
+    BitVector expected1 = x;
+    expected1.reset(y);
+    BitVector out1;
+    BitVector::apply(op1, out1, x, y);
+    EXPECT_EQ(out1, expected1);
+
+    auto op2 = [](auto x, auto y, auto z) { return (x ^ ~y) | z; };
+    BitVector expected2 = y;
+    expected2.flip();
+    expected2 ^= x;
+    expected2 |= z;
+    BitVector out2(j + 5);
+    BitVector::apply(op2, out2, x, y, z);
+    EXPECT_EQ(out2, expected2);
+  }
+}
+
+
 } // namespace
diff --git a/llvm/unittests/ADT/ImmutableListTest.cpp b/llvm/unittests/ADT/ImmutableListTest.cpp
index ab3b8b472b903c4dffd7374a767c829adea3b43c..28624c0d551d48e0eba74eb912021f1876343db5 100644
--- a/llvm/unittests/ADT/ImmutableListTest.cpp
+++ b/llvm/unittests/ADT/ImmutableListTest.cpp
@@ -245,7 +245,6 @@ TEST_F(ImmutableListTest, LongListOrderingTest) {
   int i = 0;
   for (ImmutableList<Wrapper<long>>::iterator I = L.begin(), E = L.end();
        I != E; ++I) {
-    ASSERT_EQ(i, *I);
     i++;
   }
   ASSERT_EQ(0, i);
diff --git a/llvm/unittests/ADT/StringRefTest.cpp b/llvm/unittests/ADT/StringRefTest.cpp
index 50e38c50f62125d4f4928cbbefc8172f626759e5..e3f943bdbf418dba3b3185711558c8260727a9de 100644
--- a/llvm/unittests/ADT/StringRefTest.cpp
+++ b/llvm/unittests/ADT/StringRefTest.cpp
@@ -646,12 +646,8 @@ TEST(StringRefTest, getAsInteger) {
       ASSERT_TRUE(U32Success);
     }
     bool U64Success = StringRef(Unsigned[i].Str).getAsInteger(0, U64);
-    if (static_cast<uint64_t>(Unsigned[i].Expected) == Unsigned[i].Expected) {
-      ASSERT_FALSE(U64Success);
-      EXPECT_EQ(U64, Unsigned[i].Expected);
-    } else {
-      ASSERT_TRUE(U64Success);
-    }
+    ASSERT_FALSE(U64Success);
+    EXPECT_EQ(U64, Unsigned[i].Expected);
   }
 
   int8_t S8;
@@ -682,12 +678,8 @@ TEST(StringRefTest, getAsInteger) {
       ASSERT_TRUE(S32Success);
     }
     bool S64Success = StringRef(Signed[i].Str).getAsInteger(0, S64);
-    if (static_cast<int64_t>(Signed[i].Expected) == Signed[i].Expected) {
-      ASSERT_FALSE(S64Success);
-      EXPECT_EQ(S64, Signed[i].Expected);
-    } else {
-      ASSERT_TRUE(S64Success);
-    }
+    ASSERT_FALSE(S64Success);
+    EXPECT_EQ(S64, Signed[i].Expected);
   }
 }
 
@@ -828,14 +820,9 @@ TEST(StringRefTest, consumeIntegerUnsigned) {
 
     Str = ConsumeUnsigned[i].Str;
     bool U64Success = Str.consumeInteger(0, U64);
-    if (static_cast<uint64_t>(ConsumeUnsigned[i].Expected) ==
-        ConsumeUnsigned[i].Expected) {
-      ASSERT_FALSE(U64Success);
-      EXPECT_EQ(U64, ConsumeUnsigned[i].Expected);
-      EXPECT_EQ(Str, ConsumeUnsigned[i].Leftover);
-    } else {
-      ASSERT_TRUE(U64Success);
-    }
+    ASSERT_FALSE(U64Success);
+    EXPECT_EQ(U64, ConsumeUnsigned[i].Expected);
+    EXPECT_EQ(Str, ConsumeUnsigned[i].Leftover);
   }
 }
 
@@ -881,14 +868,9 @@ TEST(StringRefTest, consumeIntegerSigned) {
 
     Str = ConsumeSigned[i].Str;
     bool S64Success = Str.consumeInteger(0, S64);
-    if (static_cast<int64_t>(ConsumeSigned[i].Expected) ==
-        ConsumeSigned[i].Expected) {
-      ASSERT_FALSE(S64Success);
-      EXPECT_EQ(S64, ConsumeSigned[i].Expected);
-      EXPECT_EQ(Str, ConsumeSigned[i].Leftover);
-    } else {
-      ASSERT_TRUE(S64Success);
-    }
+    ASSERT_FALSE(S64Success);
+    EXPECT_EQ(S64, ConsumeSigned[i].Expected);
+    EXPECT_EQ(Str, ConsumeSigned[i].Leftover);
   }
 }
 
diff --git a/llvm/unittests/ADT/TripleTest.cpp b/llvm/unittests/ADT/TripleTest.cpp
index 0e49a1aa143a011bd2b1a61e598379172142f555..2e3c78aa4f640bb27ddad27975d1126ca91794c4 100644
--- a/llvm/unittests/ADT/TripleTest.cpp
+++ b/llvm/unittests/ADT/TripleTest.cpp
@@ -111,6 +111,12 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::Linux, T.getOS());
   EXPECT_EQ(Triple::Musl, T.getEnvironment());
 
+  T = Triple("x86_64-pc-linux-muslx32");
+  EXPECT_EQ(Triple::x86_64, T.getArch());
+  EXPECT_EQ(Triple::PC, T.getVendor());
+  EXPECT_EQ(Triple::Linux, T.getOS());
+  EXPECT_EQ(Triple::MuslX32, T.getEnvironment());
+
   // PS4 has two spellings for the vendor.
   T = Triple("x86_64-scei-ps4");
   EXPECT_EQ(Triple::x86_64, T.getArch());
diff --git a/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp b/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp
index 59ff97d0fc1a5b9b93c934ec9a6f96a0c6474c86..ceaeaaf83e5d153e879dac60aae40c236b68b19a 100644
--- a/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp
+++ b/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp
@@ -1942,5 +1942,30 @@ TEST_F(CGSCCPassManagerTest, TestInsertionOfNewNonTrivialCallEdge) {
   ASSERT_TRUE(Ran);
 }
 
+TEST_F(CGSCCPassManagerTest, TestFunctionPassesAreQueriedForInvalidation) {
+  std::unique_ptr<Module> M = parseIR("define void @f() { ret void }");
+  CGSCCPassManager CGPM;
+  bool SCCCalled = false;
+  FunctionPassManager FPM;
+  int ImmRuns = 0;
+  FAM.registerPass([&] { return TestImmutableFunctionAnalysis(ImmRuns); });
+  FPM.addPass(RequireAnalysisPass<TestImmutableFunctionAnalysis, Function>());
+  CGPM.addPass(
+      LambdaSCCPass([&](LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
+                        LazyCallGraph &CG, CGSCCUpdateResult &UR) {
+        SCCCalled = true;
+        return PreservedAnalyses::none();
+      }));
+  CGPM.addPass(createCGSCCToFunctionPassAdaptor(
+      RequireAnalysisPass<TestImmutableFunctionAnalysis, Function>()));
+  ModulePassManager MPM;
+
+  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+  MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
+  MPM.run(*M, MAM);
+  ASSERT_EQ(ImmRuns, 1);
+  ASSERT_TRUE(SCCCalled);
+}
+
 #endif
 } // namespace
diff --git a/llvm/unittests/Analysis/MemorySSATest.cpp b/llvm/unittests/Analysis/MemorySSATest.cpp
index f34677997010b4daf3e7199c937e407993a1d844..f8854f8052f41ba16b1f44a4aeb702435a0302fe 100644
--- a/llvm/unittests/Analysis/MemorySSATest.cpp
+++ b/llvm/unittests/Analysis/MemorySSATest.cpp
@@ -11,12 +11,14 @@
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/MemorySSAUpdater.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
@@ -1669,3 +1671,52 @@ TEST_F(MemorySSATest, TestLoadClobber) {
   MemoryAccess *Load2Clobber = Walker->getClobberingMemoryAccess(Load2Access);
   EXPECT_EQ(Load2Clobber, Load1Access);
 }
+
+// We want to test if the location information are retained
+// when the IsGuaranteedLoopInvariant function handles a
+// memory access referring to a pointer defined in the entry
+// block, hence automatically guaranteed to be loop invariant.
+TEST_F(MemorySSATest, TestLoopInvariantEntryBlockPointer) {
+  SMDiagnostic E;
+  auto LocalM =
+      parseAssemblyString("define void @test(i64 %a0, i8* %a1, i1* %a2) {\n"
+                          "entry:\n"
+                          "%v0 = getelementptr i8, i8* %a1, i64 %a0\n"
+                          "%v1 = bitcast i8* %v0 to i64*\n"
+                          "%v2 = bitcast i8* %v0 to i32*\n"
+                          "%v3 = load i1, i1* %a2\n"
+                          "br i1 %v3, label %body, label %exit\n"
+                          "body:\n"
+                          "store i32 1, i32* %v2\n"
+                          "br label %exit\n"
+                          "exit:\n"
+                          "store i64 0, i64* %v1\n"
+                          "ret void\n"
+                          "}",
+                          E, C);
+  ASSERT_TRUE(LocalM);
+  F = LocalM->getFunction("test");
+  ASSERT_TRUE(F);
+  // Setup the analysis
+  setupAnalyses();
+  MemorySSA &MSSA = *Analyses->MSSA;
+  // Find the exit block
+  for (auto &BB : *F) {
+    if (BB.getName() == "exit") {
+      // Get the store instruction
+      auto *SI = BB.getFirstNonPHI();
+      // Get the memory access and location
+      MemoryAccess *MA = MSSA.getMemoryAccess(SI);
+      MemoryLocation ML = MemoryLocation::get(SI);
+      // Use the 'upward_defs_iterator' which internally calls
+      // IsGuaranteedLoopInvariant
+      auto ItA = upward_defs_begin({MA, ML}, MSSA.getDomTree());
+      auto ItB =
+          upward_defs_begin({ItA->first, ItA->second}, MSSA.getDomTree());
+      // Check if the location information have been retained
+      EXPECT_TRUE(ItB->second.Size.isPrecise());
+      EXPECT_TRUE(ItB->second.Size.hasValue());
+      EXPECT_TRUE(ItB->second.Size.getValue() == 8);
+    }
+  }
+}
\ No newline at end of file
diff --git a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
index 7fa588566c55ccaba721cd8f72f8050e0bdcef6d..3014fa4cb37926a8570b48bb6be72e6c0577c6b6 100644
--- a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
+++ b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
@@ -68,6 +68,13 @@ protected:
                         const SCEV *&RHS) {
     return SE.matchURem(Expr, LHS, RHS);
   }
+
+  static bool isImpliedCond(
+      ScalarEvolution &SE, ICmpInst::Predicate Pred, const SCEV *LHS,
+      const SCEV *RHS, ICmpInst::Predicate FoundPred, const SCEV *FoundLHS,
+      const SCEV *FoundRHS) {
+    return SE.isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS);
+  }
 };
 
 TEST_F(ScalarEvolutionsTest, SCEVUnknownRAUW) {
@@ -1368,6 +1375,45 @@ TEST_F(ScalarEvolutionsTest, ProveImplicationViaNarrowing) {
   });
 }
 
+TEST_F(ScalarEvolutionsTest, ImpliedCond) {
+  LLVMContext C;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M = parseAssemblyString(
+      "define void @foo(i32 %len) { "
+      "entry: "
+      "  br label %loop "
+      "loop: "
+      "  %iv = phi i32 [ 0, %entry], [%iv.next, %loop] "
+      "  %iv.next = add nsw i32 %iv, 1 "
+      "  %cmp = icmp slt i32 %iv, %len "
+      "  br i1 %cmp, label %loop, label %exit "
+      "exit:"
+      "  ret void "
+      "}",
+      Err, C);
+
+  ASSERT_TRUE(M && "Could not parse module?");
+  ASSERT_TRUE(!verifyModule(*M) && "Must have been well formed!");
+
+  runWithSE(*M, "foo", [](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+    Instruction *IV = getInstructionByName(F, "iv");
+    Type *Ty = IV->getType();
+    const SCEV *Zero = SE.getZero(Ty);
+    const SCEV *MinusOne = SE.getMinusOne(Ty);
+    // {0,+,1}<nuw><nsw>
+    const SCEV *AddRec_0_1 = SE.getSCEV(IV);
+    // {0,+,-1}<nw>
+    const SCEV *AddRec_0_N1 = SE.getNegativeSCEV(AddRec_0_1);
+
+    // {0,+,1}<nuw><nsw> > 0  ->  {0,+,-1}<nw> < 0
+    EXPECT_TRUE(isImpliedCond(SE, ICmpInst::ICMP_SLT, AddRec_0_N1, Zero,
+                                  ICmpInst::ICMP_SGT, AddRec_0_1, Zero));
+    // {0,+,-1}<nw> < -1  ->  {0,+,1}<nuw><nsw> > 0
+    EXPECT_TRUE(isImpliedCond(SE, ICmpInst::ICMP_SGT, AddRec_0_1, Zero,
+                                  ICmpInst::ICMP_SLT, AddRec_0_N1, MinusOne));
+  });
+}
+
 TEST_F(ScalarEvolutionsTest, MatchURem) {
   LLVMContext C;
   SMDiagnostic Err;
diff --git a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
index e9354dafd17bbb71b8fc273c6e3a2454064c4943..f7543d8ded8a336e8e3ccb09a4ea67ec9d37c62d 100644
--- a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
@@ -648,4 +648,18 @@ TEST_F(AArch64SelectionDAGTest, getTypeConversion_NoScalarizeEVT_nxv1f128) {
   EXPECT_DEATH(getTypeAction(FromVT), "Cannot legalize this vector");
 }
 
+TEST_F(AArch64SelectionDAGTest, TestFold_STEP_VECTOR) {
+  if (!TM)
+    return;
+
+  SDLoc Loc;
+  auto IntVT = EVT::getIntegerVT(Context, 8);
+  auto VecVT = EVT::getVectorVT(Context, MVT::i8, 16, true);
+
+  // Should create SPLAT_VECTOR
+  SDValue Zero = DAG->getConstant(0, Loc, IntVT);
+  SDValue Op = DAG->getNode(ISD::STEP_VECTOR, Loc, VecVT, Zero);
+  EXPECT_EQ(Op.getOpcode(), ISD::SPLAT_VECTOR);
+}
+
 } // end namespace llvm
diff --git a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
index 8128c3390aa041b9b147fde4a0aaf089158b89e4..daad2f78632bc99d10a9810c0290d1287e61fa15 100644
--- a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
@@ -398,3 +398,25 @@ TEST_F(AArch64GISelMITest, BuildAddoSubo) {
 
   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
+
+TEST_F(AArch64GISelMITest, BuildBitfieldExtract) {
+  setUp();
+  if (!TM)
+    return;
+  LLT S64 = LLT::scalar(64);
+  SmallVector<Register, 4> Copies;
+  collectCopies(Copies, MF);
+
+  auto Ubfx = B.buildUbfx(S64, Copies[0], Copies[1], Copies[2]);
+  B.buildSbfx(S64, Ubfx, Copies[0], Copies[2]);
+
+  const auto *CheckStr = R"(
+  ; CHECK: [[COPY0:%[0-9]+]]:_(s64) = COPY $x0
+  ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+  ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
+  ; CHECK: [[UBFX:%[0-9]+]]:_(s64) = G_UBFX [[COPY0]]:_, [[COPY1]]:_, [[COPY2]]:_
+  ; CHECK: [[SBFX:%[0-9]+]]:_(s64) = G_SBFX [[UBFX]]:_, [[COPY0]]:_, [[COPY2]]:_
+  )";
+
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
+}
diff --git a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
index c00b6c518e70020acf439b52850ddf4ac8c44a67..626176d4ba9a73ec25784927115313563a09152e 100644
--- a/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest.cpp
@@ -165,14 +165,12 @@ TEST_F(SelectionDAGAddressAnalysisTest, unknownSizeFrameObjects) {
   int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(*MF, FI);
   SDValue Value = DAG->getConstant(0, Loc, SubVecVT);
-  TypeSize Offset0 = TypeSize::Fixed(0);
   TypeSize Offset1 = SubVecVT.getStoreSize();
-  SDValue Index0 = DAG->getMemBasePlusOffset(FIPtr, Offset0, Loc);
   SDValue Index1 = DAG->getMemBasePlusOffset(FIPtr, Offset1, Loc);
-  SDValue Store0 = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index0,
-                                 PtrInfo.getWithOffset(Offset0));
+  SDValue Store0 =
+      DAG->getStore(DAG->getEntryNode(), Loc, Value, FIPtr, PtrInfo);
   SDValue Store1 = DAG->getStore(DAG->getEntryNode(), Loc, Value, Index1,
-                                 PtrInfo.getWithOffset(Offset1));
+                                 MachinePointerInfo(PtrInfo.getAddrSpace()));
   Optional<int64_t> NumBytes0 = MemoryLocation::getSizeOrUnknown(
       cast<StoreSDNode>(Store0)->getMemoryVT().getStoreSize());
   Optional<int64_t> NumBytes1 = MemoryLocation::getSizeOrUnknown(
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
index 1d0a4507426dedced72fceea79728849a8f9a7ba..5ccb349d1d7bd243c4fded4c8947e8d01770421f 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
@@ -42,7 +42,7 @@ namespace {
 template <uint16_t Version, class AddrType, class RefAddrType>
 void TestAllForms() {
   Triple Triple = getDefaultTargetTripleForAddrSize(sizeof(AddrType));
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   // Test that we can decode all DW_FORM values correctly.
@@ -464,7 +464,7 @@ TEST(DWARFDebugInfo, TestDWARF32Version5Addr8AllForms) {
 
 template <uint16_t Version, class AddrType> void TestChildren() {
   Triple Triple = getDefaultTargetTripleForAddrSize(sizeof(AddrType));
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   // Test that we can decode DW_FORM_ref_addr values correctly in DWARF 2 with
@@ -594,7 +594,7 @@ TEST(DWARFDebugInfo, TestDWARF32Version4Addr8Children) {
 
 template <uint16_t Version, class AddrType> void TestReferences() {
   Triple Triple = getDefaultTargetTripleForAddrSize(sizeof(AddrType));
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   // Test that we can decode DW_FORM_refXXX values correctly in DWARF.
@@ -844,7 +844,7 @@ TEST(DWARFDebugInfo, TestDWARF32Version4Addr8References) {
 
 template <uint16_t Version, class AddrType> void TestAddresses() {
   Triple Triple = getDefaultTargetTripleForAddrSize(sizeof(AddrType));
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   // Test the DWARF APIs related to accessing the DW_AT_low_pc and
@@ -1016,10 +1016,7 @@ TEST(DWARFDebugInfo, TestDWARF32Version4Addr8Addresses) {
 
 TEST(DWARFDebugInfo, TestStringOffsets) {
   Triple Triple = getNormalizedDefaultTargetTriple();
-  // RISC-V still does not fully support DWARF yet
-  if (Triple.getArch() == Triple::riscv64)
-    return;
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   const char *String1 = "Hello";
@@ -1083,7 +1080,7 @@ TEST(DWARFDebugInfo, TestStringOffsets) {
 
 TEST(DWARFDebugInfo, TestEmptyStringOffsets) {
   Triple Triple = getNormalizedDefaultTargetTriple();
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   const char *String1 = "Hello";
@@ -1112,7 +1109,7 @@ TEST(DWARFDebugInfo, TestEmptyStringOffsets) {
 
 TEST(DWARFDebugInfo, TestRelations) {
   Triple Triple = getNormalizedDefaultTargetTriple();
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   // Test the DWARF APIs related to accessing the DW_AT_low_pc and
@@ -1299,7 +1296,7 @@ TEST(DWARFDebugInfo, TestDWARFDie) {
 
 TEST(DWARFDebugInfo, TestChildIterators) {
   Triple Triple = getNormalizedDefaultTargetTriple();
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   // Test the DWARF APIs related to iterating across the children of a DIE using
@@ -1408,7 +1405,7 @@ TEST(DWARFDebugInfo, TestEmptyChildren) {
 
 TEST(DWARFDebugInfo, TestAttributeIterators) {
   Triple Triple = getNormalizedDefaultTargetTriple();
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   // Test the DWARF APIs related to iterating across all attribute values in a
@@ -1470,10 +1467,7 @@ TEST(DWARFDebugInfo, TestAttributeIterators) {
 
 TEST(DWARFDebugInfo, TestFindRecurse) {
   Triple Triple = getNormalizedDefaultTargetTriple();
-  // RISC-V still does not fully support DWARF
-  if (Triple.getArch() == Triple::riscv64)
-    return;
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   uint16_t Version = 4;
@@ -1687,7 +1681,7 @@ TEST(DWARFDebugInfo, TestDwarfToFunctions) {
 
 TEST(DWARFDebugInfo, TestFindAttrs) {
   Triple Triple = getNormalizedDefaultTargetTriple();
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   // Test the DWARFDie::find() and DWARFDie::findRecursively() that take an
@@ -1750,7 +1744,7 @@ TEST(DWARFDebugInfo, TestFindAttrs) {
 
 TEST(DWARFDebugInfo, TestImplicitConstAbbrevs) {
   Triple Triple = getNormalizedDefaultTargetTriple();
-  if (!isConfigurationSupported(Triple))
+  if (!isObjectEmissionSupported(Triple))
     return;
 
   uint16_t Version = 5;
diff --git a/llvm/unittests/DebugInfo/DWARF/DwarfUtils.cpp b/llvm/unittests/DebugInfo/DWARF/DwarfUtils.cpp
index 249cfb42271ae12bf96479fa430ba5a3fbfcfc5f..20dc7bc8ff12146e24d33d3c403c3b86daebc3c5 100644
--- a/llvm/unittests/DebugInfo/DWARF/DwarfUtils.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DwarfUtils.cpp
@@ -52,3 +52,10 @@ bool llvm::dwarf::utils::isConfigurationSupported(Triple &T) {
   std::string Err;
   return TargetRegistry::lookupTarget(T.getTriple(), Err);
 }
+
+bool llvm::dwarf::utils::isObjectEmissionSupported(Triple &T) {
+  initLLVMIfNeeded();
+  std::string Err;
+  const Target *TheTarget = TargetRegistry::lookupTarget(T.getTriple(), Err);
+  return TheTarget && TheTarget->hasMCAsmBackend();
+}
diff --git a/llvm/unittests/DebugInfo/DWARF/DwarfUtils.h b/llvm/unittests/DebugInfo/DWARF/DwarfUtils.h
index 036071e0b5670d3ac767345374c0cb6a244b49d1..00eaef25cfbac89ff56bade20d5cb356535a334b 100644
--- a/llvm/unittests/DebugInfo/DWARF/DwarfUtils.h
+++ b/llvm/unittests/DebugInfo/DWARF/DwarfUtils.h
@@ -21,6 +21,7 @@ namespace utils {
 Triple getDefaultTargetTripleForAddrSize(uint8_t AddrSize);
 Triple getNormalizedDefaultTargetTriple();
 bool isConfigurationSupported(Triple &T);
+bool isObjectEmissionSupported(Triple &T);
 
 } // end namespace utils
 } // end namespace dwarf
diff --git a/llvm/unittests/Demangle/DemangleTest.cpp b/llvm/unittests/Demangle/DemangleTest.cpp
index cb70a720cad9d35a6c8b42120c89788cade0b1a2..1216d2c88537f4e4467b02b7ca4ecaf1b349ebf4 100644
--- a/llvm/unittests/Demangle/DemangleTest.cpp
+++ b/llvm/unittests/Demangle/DemangleTest.cpp
@@ -21,4 +21,9 @@ TEST(Demangle, demangleTest) {
             "invocation function for block in foo(int)");
   EXPECT_EQ(demangle("?foo@@YAXH@Z"), "void __cdecl foo(int)");
   EXPECT_EQ(demangle("foo"), "foo");
+
+  // Regression test for demangling of optional template-args for vendor
+  // extended type qualifier (https://bugs.llvm.org/show_bug.cgi?id=48009)
+  EXPECT_EQ(demangle("_Z3fooILi79EEbU7_ExtIntIXT_EEi"),
+            "bool foo<79>(int _ExtInt<79>)");
 }
diff --git a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
index 810a2fd0e1f33a5a7d68d26a28523f3b89f371c6..6e00550cf24259b80895778e309883d5e10e9ef1 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
@@ -101,14 +101,97 @@ TEST(LinkGraphTest, BlockAndSymbolIteration) {
   EXPECT_TRUE(llvm::count(G.defined_symbols(), &S4));
 }
 
-TEST(LinkGraphTest, SplitBlock) {
-  // Check that the LinkGraph::splitBlock test works as expected.
+TEST(LinkGraphTest, MakeExternal) {
+  // Check that we can make a defined symbol external.
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+              getGenericEdgeKindName);
+  auto &Sec = G.createSection("__data", RWFlags);
+
+  // Create an initial block.
+  auto &B1 = G.createContentBlock(Sec, BlockContent, 0x1000, 8, 0);
+
+  // Add a symbol to the block.
+  auto &S1 = G.addDefinedSymbol(B1, 0, "S1", 4, Linkage::Strong, Scope::Default,
+                                false, false);
+
+  EXPECT_TRUE(S1.isDefined()) << "Symbol should be defined";
+  EXPECT_FALSE(S1.isExternal()) << "Symbol should not be external";
+  EXPECT_FALSE(S1.isAbsolute()) << "Symbol should not be absolute";
+  EXPECT_TRUE(&S1.getBlock()) << "Symbol should have a non-null block";
+  EXPECT_EQ(S1.getAddress(), 0x1000U) << "Unexpected symbol address";
+
+  EXPECT_EQ(
+      std::distance(G.defined_symbols().begin(), G.defined_symbols().end()), 1U)
+      << "Unexpected number of defined symbols";
+  EXPECT_EQ(
+      std::distance(G.external_symbols().begin(), G.external_symbols().end()),
+      0U)
+      << "Unexpected number of external symbols";
+
+  // Make S1 external, confirm that the its flags are updated and that it is
+  // moved from the defined symbols to the externals list.
+  G.makeExternal(S1);
+
+  EXPECT_FALSE(S1.isDefined()) << "Symbol should not be defined";
+  EXPECT_TRUE(S1.isExternal()) << "Symbol should be external";
+  EXPECT_FALSE(S1.isAbsolute()) << "Symbol should not be absolute";
+  EXPECT_EQ(S1.getAddress(), 0U) << "Unexpected symbol address";
+
+  EXPECT_EQ(
+      std::distance(G.defined_symbols().begin(), G.defined_symbols().end()), 0U)
+      << "Unexpected number of defined symbols";
+  EXPECT_EQ(
+      std::distance(G.external_symbols().begin(), G.external_symbols().end()),
+      1U)
+      << "Unexpected number of external symbols";
+}
+
+TEST(LinkGraphTest, MakeDefined) {
+  // Check that we can make an external symbol defined.
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+              getGenericEdgeKindName);
+  auto &Sec = G.createSection("__data", RWFlags);
 
-  const char BlockContentBytes[] = {0x10, 0x11, 0x12, 0x13, 0x14, 0x15,
-                                    0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B,
-                                    0x1C, 0x1D, 0x1E, 0x1F, 0x00};
-  StringRef BlockContent(BlockContentBytes);
+  // Create an initial block.
+  auto &B1 = G.createContentBlock(Sec, BlockContent, 0x1000, 8, 0);
+
+  // Add an external symbol.
+  auto &S1 = G.addExternalSymbol("S1", 4, Linkage::Strong);
+
+  EXPECT_FALSE(S1.isDefined()) << "Symbol should not be defined";
+  EXPECT_TRUE(S1.isExternal()) << "Symbol should be external";
+  EXPECT_FALSE(S1.isAbsolute()) << "Symbol should not be absolute";
+  EXPECT_EQ(S1.getAddress(), 0U) << "Unexpected symbol address";
 
+  EXPECT_EQ(
+      std::distance(G.defined_symbols().begin(), G.defined_symbols().end()), 0U)
+      << "Unexpected number of defined symbols";
+  EXPECT_EQ(
+      std::distance(G.external_symbols().begin(), G.external_symbols().end()),
+      1U)
+      << "Unexpected number of external symbols";
+
+  // Make S1 defined, confirm that its flags are updated and that it is
+  // moved from the defined symbols to the externals list.
+  G.makeDefined(S1, B1, 0, 4, Linkage::Strong, Scope::Default, false);
+
+  EXPECT_TRUE(S1.isDefined()) << "Symbol should be defined";
+  EXPECT_FALSE(S1.isExternal()) << "Symbol should not be external";
+  EXPECT_FALSE(S1.isAbsolute()) << "Symbol should not be absolute";
+  EXPECT_TRUE(&S1.getBlock()) << "Symbol should have a non-null block";
+  EXPECT_EQ(S1.getAddress(), 0x1000U) << "Unexpected symbol address";
+
+  EXPECT_EQ(
+      std::distance(G.defined_symbols().begin(), G.defined_symbols().end()), 1U)
+      << "Unexpected number of defined symbols";
+  EXPECT_EQ(
+      std::distance(G.external_symbols().begin(), G.external_symbols().end()),
+      0U)
+      << "Unexpected number of external symbols";
+}
+
+TEST(LinkGraphTest, SplitBlock) {
+  // Check that the LinkGraph::splitBlock test works as expected.
   LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
               getGenericEdgeKindName);
   auto &Sec = G.createSection("__data", RWFlags);
diff --git a/llvm/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp
index 1ad13e8b4e2068c2fc66d7c4149d1c341e1a2349..0f5cb14fec37cb7b5e5c77164d869a81b75e6be0 100644
--- a/llvm/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp
@@ -871,8 +871,7 @@ TEST(DummyRPC, TestRemoveHandler) {
 
   Server.addHandler<DummyRPCAPI::VoidBool>(
     [](bool B) {
-      EXPECT_EQ(B, true)
-        << "Server void(bool) received unexpected result";
+      llvm_unreachable("Server void(bool) received unexpected result");
     });
 
   Server.removeHandler<DummyRPCAPI::VoidBool>();
@@ -884,8 +883,7 @@ TEST(DummyRPC, TestClearHandlers) {
 
   Server.addHandler<DummyRPCAPI::VoidBool>(
     [](bool B) {
-      EXPECT_EQ(B, true)
-        << "Server void(bool) received unexpected result";
+      llvm_unreachable("Server void(bool) received unexpected result");
     });
 
   Server.clearHandlers();
diff --git a/llvm/unittests/FileCheck/FileCheckTest.cpp b/llvm/unittests/FileCheck/FileCheckTest.cpp
index 3ed51433943966e8dda36571d151ca6c2d402b0f..f54b940d68c4fb582def7d274849de63898ef836 100644
--- a/llvm/unittests/FileCheck/FileCheckTest.cpp
+++ b/llvm/unittests/FileCheck/FileCheckTest.cpp
@@ -1027,8 +1027,10 @@ public:
 
   Expected<size_t> match(StringRef Buffer) {
     StringRef BufferRef = bufferize(SM, Buffer);
-    size_t MatchLen;
-    return P.match(BufferRef, MatchLen, SM);
+    Pattern::MatchResult Res = P.match(BufferRef, SM);
+    if (Res.TheError)
+      return std::move(Res.TheError);
+    return Res.TheMatch->Pos;
   }
 
   void printVariableDefs(FileCheckDiag::MatchType MatchTy,
@@ -1341,6 +1343,9 @@ TEST_F(FileCheckTest, ParsePattern) {
   // Collision with numeric variable.
   EXPECT_TRUE(Tester.parsePattern("[[FOO:]]"));
 
+  // Invalid use of string variable.
+  EXPECT_TRUE(Tester.parsePattern("[[FOO-BAR]]"));
+
   // Valid use of string variable.
   EXPECT_FALSE(Tester.parsePattern("[[BAR]]"));
 
@@ -1640,8 +1645,8 @@ TEST_F(FileCheckTest, FileCheckContext) {
   FileCheckRequest Req;
   Cxt.createLineVariable();
   ASSERT_FALSE(P.parsePattern("[[@LINE]]", "CHECK", SM, Req));
-  size_t MatchLen;
-  ASSERT_THAT_EXPECTED(P.match("1", MatchLen, SM), Succeeded());
+  Pattern::MatchResult Res = P.match("1", SM);
+  ASSERT_THAT_ERROR(std::move(Res.TheError), Succeeded());
 
 #ifndef NDEBUG
   // Recreating @LINE pseudo numeric variable fails.
diff --git a/llvm/unittests/IR/AttributesTest.cpp b/llvm/unittests/IR/AttributesTest.cpp
index 2c191264a892e22fe35defaea516fcd9f44093f0..11b1598979896e70367c9c575cdb8122f3c2743c 100644
--- a/llvm/unittests/IR/AttributesTest.cpp
+++ b/llvm/unittests/IR/AttributesTest.cpp
@@ -180,9 +180,6 @@ TEST(Attributes, StringRepresentation) {
   Attribute A = Attribute::getWithByValType(C, Ty);
   EXPECT_EQ(A.getAsString(), "byval(%mystruct)");
 
-  A = Attribute::getWithByValType(C, nullptr);
-  EXPECT_EQ(A.getAsString(), "byval");
-
   A = Attribute::getWithByValType(C, Type::getInt32Ty(C));
   EXPECT_EQ(A.getAsString(), "byval(i32)");
 }
diff --git a/llvm/unittests/IR/BasicBlockTest.cpp b/llvm/unittests/IR/BasicBlockTest.cpp
index fa923c90c729e4834da9e7a76ee098daf9815a35..408275732058526dd3cf45e55345d0703f08de3f 100644
--- a/llvm/unittests/IR/BasicBlockTest.cpp
+++ b/llvm/unittests/IR/BasicBlockTest.cpp
@@ -37,10 +37,12 @@ TEST(BasicBlockTest, PhiRange) {
   BranchInst::Create(BB.get(), BB2.get());
 
   // Make sure this doesn't crash if there are no phis.
+  int PhiCount = 0;
   for (auto &PN : BB->phis()) {
     (void)PN;
-    EXPECT_TRUE(false) << "empty block should have no phis";
+    PhiCount++;
   }
+  ASSERT_EQ(PhiCount, 0) << "empty block should have no phis";
 
   // Make it a cycle.
   auto *BI = BranchInst::Create(BB.get(), BB.get());
diff --git a/llvm/unittests/IR/ConstantsTest.cpp b/llvm/unittests/IR/ConstantsTest.cpp
index 44dbb90758adfe22d97ea19ce41dbba4cd63fb28..50eb3e0df1f584165ff76eb2b3a3dcdc4634b601 100644
--- a/llvm/unittests/IR/ConstantsTest.cpp
+++ b/llvm/unittests/IR/ConstantsTest.cpp
@@ -418,45 +418,55 @@ static std::string getNameOfType(Type *T) {
 
 TEST(ConstantsTest, BuildConstantDataArrays) {
   LLVMContext Context;
-  std::unique_ptr<Module> M(new Module("MyModule", Context));
 
   for (Type *T : {Type::getInt8Ty(Context), Type::getInt16Ty(Context),
                   Type::getInt32Ty(Context), Type::getInt64Ty(Context)}) {
     ArrayType *ArrayTy = ArrayType::get(T, 2);
     Constant *Vals[] = {ConstantInt::get(T, 0), ConstantInt::get(T, 1)};
-    Constant *CDV = ConstantArray::get(ArrayTy, Vals);
-    ASSERT_TRUE(dyn_cast<ConstantDataArray>(CDV) != nullptr)
-        << " T = " << getNameOfType(T);
+    Constant *CA = ConstantArray::get(ArrayTy, Vals);
+    ASSERT_TRUE(isa<ConstantDataArray>(CA)) << " T = " << getNameOfType(T);
+    auto *CDA = cast<ConstantDataArray>(CA);
+    Constant *CA2 = ConstantDataArray::getRaw(
+        CDA->getRawDataValues(), CDA->getNumElements(), CDA->getElementType());
+    ASSERT_TRUE(CA == CA2) << " T = " << getNameOfType(T);
   }
 
-  for (Type *T : {Type::getHalfTy(Context), Type::getFloatTy(Context),
-                  Type::getDoubleTy(Context)}) {
+  for (Type *T : {Type::getHalfTy(Context), Type::getBFloatTy(Context),
+                  Type::getFloatTy(Context), Type::getDoubleTy(Context)}) {
     ArrayType *ArrayTy = ArrayType::get(T, 2);
     Constant *Vals[] = {ConstantFP::get(T, 0), ConstantFP::get(T, 1)};
-    Constant *CDV = ConstantArray::get(ArrayTy, Vals);
-    ASSERT_TRUE(dyn_cast<ConstantDataArray>(CDV) != nullptr)
-        << " T = " << getNameOfType(T);
+    Constant *CA = ConstantArray::get(ArrayTy, Vals);
+    ASSERT_TRUE(isa<ConstantDataArray>(CA)) << " T = " << getNameOfType(T);
+    auto *CDA = cast<ConstantDataArray>(CA);
+    Constant *CA2 = ConstantDataArray::getRaw(
+        CDA->getRawDataValues(), CDA->getNumElements(), CDA->getElementType());
+    ASSERT_TRUE(CA == CA2) << " T = " << getNameOfType(T);
   }
 }
 
 TEST(ConstantsTest, BuildConstantDataVectors) {
   LLVMContext Context;
-  std::unique_ptr<Module> M(new Module("MyModule", Context));
 
   for (Type *T : {Type::getInt8Ty(Context), Type::getInt16Ty(Context),
                   Type::getInt32Ty(Context), Type::getInt64Ty(Context)}) {
     Constant *Vals[] = {ConstantInt::get(T, 0), ConstantInt::get(T, 1)};
-    Constant *CDV = ConstantVector::get(Vals);
-    ASSERT_TRUE(dyn_cast<ConstantDataVector>(CDV) != nullptr)
-        << " T = " << getNameOfType(T);
+    Constant *CV = ConstantVector::get(Vals);
+    ASSERT_TRUE(isa<ConstantDataVector>(CV)) << " T = " << getNameOfType(T);
+    auto *CDV = cast<ConstantDataVector>(CV);
+    Constant *CV2 = ConstantDataVector::getRaw(
+        CDV->getRawDataValues(), CDV->getNumElements(), CDV->getElementType());
+    ASSERT_TRUE(CV == CV2) << " T = " << getNameOfType(T);
   }
 
-  for (Type *T : {Type::getHalfTy(Context), Type::getFloatTy(Context),
-                  Type::getDoubleTy(Context)}) {
+  for (Type *T : {Type::getHalfTy(Context), Type::getBFloatTy(Context),
+                  Type::getFloatTy(Context), Type::getDoubleTy(Context)}) {
     Constant *Vals[] = {ConstantFP::get(T, 0), ConstantFP::get(T, 1)};
-    Constant *CDV = ConstantVector::get(Vals);
-    ASSERT_TRUE(dyn_cast<ConstantDataVector>(CDV) != nullptr)
-        << " T = " << getNameOfType(T);
+    Constant *CV = ConstantVector::get(Vals);
+    ASSERT_TRUE(isa<ConstantDataVector>(CV)) << " T = " << getNameOfType(T);
+    auto *CDV = cast<ConstantDataVector>(CV);
+    Constant *CV2 = ConstantDataVector::getRaw(
+        CDV->getRawDataValues(), CDV->getNumElements(), CDV->getElementType());
+    ASSERT_TRUE(CV == CV2) << " T = " << getNameOfType(T);
   }
 }
 
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index 2e5b050ad5c0363098f39bb2f3bfa037d7669b77..ef3895d1aba9b4e8418547a3fc33972928d65079 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -180,6 +180,32 @@ TEST_F(IRBuilderTest, IntrinsicsWithScalableVectors) {
     EXPECT_EQ(FTy->getParamType(i), ArgTys[i]->getType());
 }
 
+TEST_F(IRBuilderTest, CreateStepVector) {
+  IRBuilder<> Builder(BB);
+
+  // Fixed width vectors
+  Type *DstVecTy = VectorType::get(Builder.getInt32Ty(), 4, false);
+  Value *StepVec = Builder.CreateStepVector(DstVecTy);
+  EXPECT_TRUE(isa<Constant>(StepVec));
+  EXPECT_EQ(StepVec->getType(), DstVecTy);
+
+  const auto *VectorValue = cast<Constant>(StepVec);
+  for (unsigned i = 0; i < 4; i++) {
+    EXPECT_TRUE(isa<ConstantInt>(VectorValue->getAggregateElement(i)));
+    ConstantInt *El = cast<ConstantInt>(VectorValue->getAggregateElement(i));
+    EXPECT_EQ(El->getValue(), i);
+  }
+
+  // Scalable vectors
+  DstVecTy = VectorType::get(Builder.getInt32Ty(), 4, true);
+  StepVec = Builder.CreateStepVector(DstVecTy);
+  EXPECT_TRUE(isa<CallInst>(StepVec));
+  CallInst *Call = cast<CallInst>(StepVec);
+  FunctionType *FTy = Call->getFunctionType();
+  EXPECT_EQ(FTy->getReturnType(), DstVecTy);
+  EXPECT_EQ(Call->getIntrinsicID(), Intrinsic::experimental_stepvector);
+}
+
 TEST_F(IRBuilderTest, ConstrainedFP) {
   IRBuilder<> Builder(BB);
   Value *V;
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 174e3eea3cce796c37a64e3edd48026a26ed03cf..6b1217feeac7a89510db22fefe9e6bcae6cd8b54 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -238,5 +238,20 @@ TEST(VerifierTest, DetectInvalidDebugInfo) {
   }
 }
 
+TEST(VerifierTest, MDNodeWrongContext) {
+  LLVMContext C1, C2;
+  auto *Node = MDNode::get(C1, None);
+
+  Module M("M", C2);
+  auto *NamedNode = M.getOrInsertNamedMetadata("test");
+  NamedNode->addOperand(Node);
+
+  std::string Error;
+  raw_string_ostream ErrorOS(Error);
+  EXPECT_TRUE(verifyModule(M, &ErrorOS));
+  EXPECT_TRUE(StringRef(ErrorOS.str())
+                  .startswith("MDNode context does not match Module context!"));
+}
+
 } // end anonymous namespace
 } // end namespace llvm
diff --git a/llvm/unittests/Linker/LinkModulesTest.cpp b/llvm/unittests/Linker/LinkModulesTest.cpp
index 05523c56cc2aa0295090733c4add08f2d0f5808a..793c744a2df53712e0d7b725b8ff39c118492a23 100644
--- a/llvm/unittests/Linker/LinkModulesTest.cpp
+++ b/llvm/unittests/Linker/LinkModulesTest.cpp
@@ -72,7 +72,7 @@ protected:
 };
 
 static void expectNoDiags(const DiagnosticInfo &DI, void *C) {
-  EXPECT_TRUE(false);
+  llvm_unreachable("expectNoDiags called!");
 }
 
 TEST_F(LinkModuleTest, BlockAddress) {
diff --git a/llvm/unittests/MC/SystemZ/CMakeLists.txt b/llvm/unittests/MC/SystemZ/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c50e7db265cece3d3f45929b61836c667a3bd705
--- /dev/null
+++ b/llvm/unittests/MC/SystemZ/CMakeLists.txt
@@ -0,0 +1,14 @@
+include_directories(
+  ${LLVM_MAIN_SRC_DIR}/lib/Target/SystemZ
+  )
+
+set(LLVM_LINK_COMPONENTS
+  SystemZ
+  MCParser
+  MC
+  Support
+  )
+
+add_llvm_unittest(SystemZAsmLexerTests
+  SystemZAsmLexerTest.cpp
+  )
diff --git a/llvm/unittests/MC/SystemZ/SystemZAsmLexerTest.cpp b/llvm/unittests/MC/SystemZ/SystemZAsmLexerTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..183b3ffc9352a505d7fbb950bcb4b310ad3e7b40
--- /dev/null
+++ b/llvm/unittests/MC/SystemZ/SystemZAsmLexerTest.cpp
@@ -0,0 +1,155 @@
+//===- llvm/unittests/MC/SystemZ/SystemZAsmLexerTest.cpp ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--------------------------------------------------------------------===//
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+// Come up with our hacked version of MCAsmInfo.
+// This hacked version derives from the main MCAsmInfo instance.
+// Here, we're free to override whatever we want, without polluting
+// the main MCAsmInfo interface.
+class MockedUpMCAsmInfo : public MCAsmInfo {
+public:
+  void setRestrictCommentStringToStartOfStatement(bool Value) {
+    RestrictCommentStringToStartOfStatement = Value;
+  }
+  void setCommentString(StringRef Value) { CommentString = Value; }
+};
+
+// Setup a testing class that the GTest framework can call.
+class SystemZAsmLexerTest : public ::testing::Test {
+protected:
+  static void SetUpTestCase() {
+    LLVMInitializeSystemZTargetInfo();
+    LLVMInitializeSystemZTargetMC();
+  }
+
+  std::unique_ptr<MCRegisterInfo> MRI;
+  std::unique_ptr<MockedUpMCAsmInfo> MUPMAI;
+  std::unique_ptr<const MCInstrInfo> MII;
+  std::unique_ptr<MCStreamer> Str;
+  std::unique_ptr<MCAsmParser> Parser;
+  std::unique_ptr<MCContext> Ctx;
+
+  SourceMgr SrcMgr;
+  std::string TripleName;
+  llvm::Triple Triple;
+  const Target *TheTarget;
+
+  const MCTargetOptions MCOptions;
+  MCObjectFileInfo MOFI;
+
+  SystemZAsmLexerTest() {
+    // We will use the SystemZ triple, because of missing
+    // Object File and Streamer support for the z/OS target.
+    TripleName = "s390x-ibm-linux";
+    Triple = llvm::Triple(TripleName);
+
+    std::string Error;
+    TheTarget = TargetRegistry::lookupTarget(TripleName, Error);
+    EXPECT_NE(TheTarget, nullptr);
+
+    MRI.reset(TheTarget->createMCRegInfo(TripleName));
+    EXPECT_NE(MRI, nullptr);
+
+    std::unique_ptr<MCAsmInfo> MAI;
+    MAI.reset(TheTarget->createMCAsmInfo(*MRI, TripleName, MCOptions));
+    EXPECT_NE(MAI, nullptr);
+
+    // Now we cast to our mocked up version of MCAsmInfo.
+    MUPMAI.reset(static_cast<MockedUpMCAsmInfo *>(MAI.release()));
+    // MUPMAI should "hold" MAI.
+    EXPECT_NE(MUPMAI, nullptr);
+    // After releasing, MAI should now be null.
+    EXPECT_EQ(MAI, nullptr);
+  }
+
+  void setupCallToAsmParser(StringRef AsmStr) {
+    std::unique_ptr<MemoryBuffer> Buffer(MemoryBuffer::getMemBuffer(AsmStr));
+    SrcMgr.AddNewSourceBuffer(std::move(Buffer), SMLoc());
+    EXPECT_EQ(Buffer, nullptr);
+
+    Ctx.reset(
+        new MCContext(MUPMAI.get(), MRI.get(), &MOFI, &SrcMgr, &MCOptions));
+    MOFI.InitMCObjectFileInfo(Triple, false, *Ctx, false);
+
+    Str.reset(TheTarget->createNullStreamer(*Ctx));
+
+    Parser.reset(createMCAsmParser(SrcMgr, *Ctx, *Str, *MUPMAI));
+    // Lex initially to get the string.
+    Parser->getLexer().Lex();
+  }
+
+  void lexAndCheckTokens(StringRef AsmStr,
+                         SmallVector<AsmToken::TokenKind> ExpectedTokens) {
+    // Get reference to AsmLexer.
+    MCAsmLexer &Lexer = Parser->getLexer();
+    // Loop through all expected tokens checking one by one.
+    for (size_t I = 0; I < ExpectedTokens.size(); ++I) {
+      EXPECT_EQ(Lexer.getTok().getKind(), ExpectedTokens[I]);
+      Lexer.Lex();
+    }
+  }
+};
+
+TEST_F(SystemZAsmLexerTest, CheckDontRestrictCommentStringToStartOfStatement) {
+  StringRef AsmStr = "jne #-4";
+
+  // Setup.
+  setupCallToAsmParser(AsmStr);
+
+  SmallVector<AsmToken::TokenKind> ExpectedTokens(
+      {AsmToken::Identifier, AsmToken::EndOfStatement});
+  lexAndCheckTokens(AsmStr /* "jne #-4" */, ExpectedTokens);
+}
+
+// Testing MCAsmInfo's RestrictCommentStringToStartOfStatement attribute.
+TEST_F(SystemZAsmLexerTest, CheckRestrictCommentStringToStartOfStatement) {
+  StringRef AsmStr = "jne #-4";
+
+  // Setup.
+  MUPMAI->setRestrictCommentStringToStartOfStatement(true);
+  setupCallToAsmParser(AsmStr);
+
+  // When we are restricting the comment string to only the start of the
+  // statement, The sequence of tokens we are expecting are: Identifier - "jne"
+  // Hash - '#'
+  // Minus - '-'
+  // Integer - '4'
+  SmallVector<AsmToken::TokenKind> ExpectedTokens(
+      {AsmToken::Identifier, AsmToken::Hash, AsmToken::Minus,
+       AsmToken::Integer});
+  lexAndCheckTokens(AsmStr /* "jne #-4" */, ExpectedTokens);
+}
+
+// Test HLASM Comment Syntax ('*')
+TEST_F(SystemZAsmLexerTest, CheckHLASMComment) {
+  StringRef AsmStr = "* lhi 1,10";
+
+  // Setup.
+  MUPMAI->setCommentString("*");
+  setupCallToAsmParser(AsmStr);
+
+  SmallVector<AsmToken::TokenKind> ExpectedTokens(
+      {AsmToken::EndOfStatement, AsmToken::Eof});
+  lexAndCheckTokens(AsmStr /* "* lhi 1,10" */, ExpectedTokens);
+}
+} // end anonymous namespace
diff --git a/llvm/unittests/Support/KnownBitsTest.cpp b/llvm/unittests/Support/KnownBitsTest.cpp
index 4e69df49837e1b800241c1a270e4bcb0393aaa30..5f2133165d24afad9771d3aa692a4ce85d469506 100644
--- a/llvm/unittests/Support/KnownBitsTest.cpp
+++ b/llvm/unittests/Support/KnownBitsTest.cpp
@@ -113,6 +113,8 @@ TEST(KnownBitsTest, BinaryExhaustive) {
       KnownBits KnownSMax(KnownAnd);
       KnownBits KnownSMin(KnownAnd);
       KnownBits KnownMul(KnownAnd);
+      KnownBits KnownMulHS(KnownAnd);
+      KnownBits KnownMulHU(KnownAnd);
       KnownBits KnownUDiv(KnownAnd);
       KnownBits KnownURem(KnownAnd);
       KnownBits KnownSRem(KnownAnd);
@@ -156,6 +158,14 @@ TEST(KnownBitsTest, BinaryExhaustive) {
           KnownMul.One &= Res;
           KnownMul.Zero &= ~Res;
 
+          Res = (N1.sext(2 * Bits) * N2.sext(2 * Bits)).extractBits(Bits, Bits);
+          KnownMulHS.One &= Res;
+          KnownMulHS.Zero &= ~Res;
+
+          Res = (N1.zext(2 * Bits) * N2.zext(2 * Bits)).extractBits(Bits, Bits);
+          KnownMulHU.One &= Res;
+          KnownMulHU.Zero &= ~Res;
+
           if (!N2.isNullValue()) {
             Res = N1.udiv(N2);
             KnownUDiv.One &= Res;
@@ -218,12 +228,20 @@ TEST(KnownBitsTest, BinaryExhaustive) {
       EXPECT_EQ(KnownSMin.Zero, ComputedSMin.Zero);
       EXPECT_EQ(KnownSMin.One, ComputedSMin.One);
 
-      // ComputedMul is conservatively correct, but not guaranteed to be
+      // The following are conservatively correct, but not guaranteed to be
       // precise.
       KnownBits ComputedMul = KnownBits::computeForMul(Known1, Known2);
       EXPECT_TRUE(ComputedMul.Zero.isSubsetOf(KnownMul.Zero));
       EXPECT_TRUE(ComputedMul.One.isSubsetOf(KnownMul.One));
 
+      KnownBits ComputedMulHS = KnownBits::mulhs(Known1, Known2);
+      EXPECT_TRUE(ComputedMulHS.Zero.isSubsetOf(KnownMulHS.Zero));
+      EXPECT_TRUE(ComputedMulHS.One.isSubsetOf(KnownMulHS.One));
+
+      KnownBits ComputedMulHU = KnownBits::mulhu(Known1, Known2);
+      EXPECT_TRUE(ComputedMulHU.Zero.isSubsetOf(KnownMulHU.Zero));
+      EXPECT_TRUE(ComputedMulHU.One.isSubsetOf(KnownMulHU.One));
+
       KnownBits ComputedUDiv = KnownBits::udiv(Known1, Known2);
       EXPECT_TRUE(ComputedUDiv.Zero.isSubsetOf(KnownUDiv.Zero));
       EXPECT_TRUE(ComputedUDiv.One.isSubsetOf(KnownUDiv.One));
diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp
index ef1877381b0492bdc574f9c80faf31cb46e835d7..73f0cbbaae38eb5d7eec52fe70f18d3aaa32433a 100644
--- a/llvm/unittests/Support/Path.cpp
+++ b/llvm/unittests/Support/Path.cpp
@@ -1088,6 +1088,11 @@ TEST_F(FileSystemTest, DirectoryIteration) {
   ASSERT_NO_ERROR(fs::remove(Twine(TestDirectory) + "/reclevel"));
 }
 
+TEST_F(FileSystemTest, DirectoryNotExecutable) {
+  ASSERT_EQ(fs::access(TestDirectory, sys::fs::AccessMode::Execute),
+            errc::permission_denied);
+}
+
 #ifdef LLVM_ON_UNIX
 TEST_F(FileSystemTest, BrokenSymlinkDirectoryIteration) {
   // Create a known hierarchy to recurse over.
diff --git a/llvm/unittests/Support/raw_ostream_test.cpp b/llvm/unittests/Support/raw_ostream_test.cpp
index 78fdb04bcdaa5771112ce83c8e7affa250654220..d8e17324ecd04e284ba42b19a7145b03f5117edc 100644
--- a/llvm/unittests/Support/raw_ostream_test.cpp
+++ b/llvm/unittests/Support/raw_ostream_test.cpp
@@ -8,8 +8,11 @@
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
@@ -469,4 +472,75 @@ TEST(raw_ostreamTest, reserve_stream) {
   OS.flush();
   EXPECT_EQ("11111111111111111111hello1world", Str);
 }
+
+static void checkFileData(StringRef FileName, StringRef GoldenData) {
+  ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
+      MemoryBuffer::getFileOrSTDIN(FileName);
+  EXPECT_FALSE(BufOrErr.getError());
+
+  EXPECT_EQ((*BufOrErr)->getBufferSize(), GoldenData.size());
+  EXPECT_EQ(memcmp((*BufOrErr)->getBufferStart(), GoldenData.data(),
+                   GoldenData.size()),
+            0);
+}
+
+TEST(raw_ostreamTest, writeToOutputFile) {
+  SmallString<64> Path;
+  int FD;
+  ASSERT_FALSE(sys::fs::createTemporaryFile("foo", "bar", FD, Path));
+  FileRemover Cleanup(Path);
+
+  ASSERT_THAT_ERROR(writeToOutput(Path,
+                                  [](raw_ostream &Out) -> Error {
+                                    Out << "HelloWorld";
+                                    return Error::success();
+                                  }),
+                    Succeeded());
+  checkFileData(Path, "HelloWorld");
+}
+
+TEST(raw_ostreamTest, writeToNonexistingPath) {
+  StringRef FileName = "/_bad/_path";
+  std::string ErrorMessage = toString(createFileError(
+      FileName, make_error_code(errc::no_such_file_or_directory)));
+
+  EXPECT_THAT_ERROR(writeToOutput(FileName,
+                                  [](raw_ostream &Out) -> Error {
+                                    Out << "HelloWorld";
+                                    return Error::success();
+                                  }),
+                    FailedWithMessage(ErrorMessage));
+}
+
+TEST(raw_ostreamTest, writeToDevNull) {
+  bool DevNullIsUsed = false;
+
+  EXPECT_THAT_ERROR(
+      writeToOutput("/dev/null",
+                    [&](raw_ostream &Out) -> Error {
+                      DevNullIsUsed =
+                          testing::internal::CheckedDowncastToActualType<
+                              raw_null_ostream, raw_ostream>(&Out);
+                      return Error::success();
+                    }),
+      Succeeded());
+
+  EXPECT_TRUE(DevNullIsUsed);
+}
+
+TEST(raw_ostreamTest, writeToStdOut) {
+  outs().flush();
+  testing::internal::CaptureStdout();
+
+  EXPECT_THAT_ERROR(writeToOutput("-",
+                                  [](raw_ostream &Out) -> Error {
+                                    Out << "HelloWorld";
+                                    return Error::success();
+                                  }),
+                    Succeeded());
+  outs().flush();
+
+  std::string CapturedStdOut = testing::internal::GetCapturedStdout();
+  EXPECT_EQ(CapturedStdOut, "HelloWorld");
+}
 }
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index 6bab80215c0b1cc4bb60d5ebafc7a3a0c2209d35..34802b63aae845536bf1d5a2d7d6dcf6781dbf22 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -718,10 +718,10 @@ TEST(CloneFunction, CloneEmptyFunction) {
 
 TEST(CloneFunction, CloneFunctionWithInalloca) {
   StringRef ImplAssembly = R"(
-    declare void @a(i32* inalloca)
+    declare void @a(i32* inalloca(i32))
     define void @foo() {
       %a = alloca inalloca i32
-      call void @a(i32* inalloca %a)
+      call void @a(i32* inalloca(i32) %a)
       ret void
     }
     declare void @bar()
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 880b8f7114627e6bc0d42274e71555ad6d024039..853be5757731c693f7ef3d3510c118a9010a29a5 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -89,11 +89,13 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
   EXPECT_EQ(IndvarAdd, ICmp->getOperand(0));
   EXPECT_EQ(VecBB->getCondBit(), ICmp);
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   // Add an external value to check we do not print the list of external values,
   // as this is not required with the new printing.
   Plan->addVPValue(&*F->arg_begin());
   std::string FullDump;
-  raw_string_ostream(FullDump) << *Plan;
+  raw_string_ostream OS(FullDump);
+  Plan->printDOT(OS);
   const char *ExpectedStr = R"(digraph VPlan {
 graph [labelloc=t, fontsize=30; label="Vectorization Plan"]
 node [shape=rect, fontname=Courier, fontsize=30]
@@ -103,30 +105,34 @@ compound=true
     fontname=Courier
     label="\<x1\> TopRegion"
     N1 [label =
-      "entry:\n"
+      "entry:\l" +
+      "Successor(s): for.body\l"
     ]
     N1 -> N2 [ label=""]
     N2 [label =
-      "for.body:\n" +
-        "WIDEN-PHI %indvars.iv = phi 0, %indvars.iv.next\l" +
-        "EMIT ir<%arr.idx> = getelementptr ir<%A> ir<%indvars.iv>\l" +
-        "EMIT ir<%l1> = load ir<%arr.idx>\l" +
-        "EMIT ir<%res> = add ir<%l1> ir<10>\l" +
-        "EMIT store ir<%res> ir<%arr.idx>\l" +
-        "EMIT ir<%indvars.iv.next> = add ir<%indvars.iv> ir<1>\l" +
-        "EMIT ir<%exitcond> = icmp ir<%indvars.iv.next> ir<%N>\l" +
-         "CondBit: ir<%exitcond> (for.body)\l"
+      "for.body:\l" +
+      "  WIDEN-PHI %indvars.iv = phi 0, %indvars.iv.next\l" +
+      "  EMIT ir\<%arr.idx\> = getelementptr ir\<%A\> ir\<%indvars.iv\>\l" +
+      "  EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" +
+      "  EMIT ir\<%res\> = add ir\<%l1\> ir\<10\>\l" +
+      "  EMIT store ir\<%res\> ir\<%arr.idx\>\l" +
+      "  EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\> ir\<1\>\l" +
+      "  EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\> ir\<%N\>\l" +
+      "Successor(s): for.body, for.end\l" +
+      "CondBit: ir\<%exitcond\> (for.body)\l"
     ]
     N2 -> N2 [ label="T"]
     N2 -> N3 [ label="F"]
     N3 [label =
-      "for.end:\n" +
-        "EMIT ret\l"
+      "for.end:\l" +
+      "  EMIT ret\l" +
+      "No successors\l"
     ]
   }
 }
 )";
   EXPECT_EQ(ExpectedStr, FullDump);
+#endif
 
   LoopVectorizationLegality::InductionList Inductions;
   SmallPtrSet<Instruction *, 1> DeadInstructions;
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index 268f43774d12f4562305b67e089c383ff64f9f7f..af0bef544e38467ff0b9ca61cb442d1e9bb12ffe 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -40,7 +40,7 @@ protected:
                                                    VPlan &Plan) {
     AC.reset(new AssumptionCache(F));
     SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI));
-    BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT, &*LI));
+    BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT));
     AARes.reset(new AAResults(TLI));
     AARes->addAAResult(*BasicAA);
     PSE.reset(new PredicatedScalarEvolution(*SE, *L));
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index f8f1562d548c20bf8f3374d6dd9c4e4894874ba1..2836e8199678a482bbd8cf7c9d58692dca056e82 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -324,6 +324,7 @@ TEST(VPBasicBlockTest, getPlan) {
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 TEST(VPBasicBlockTest, print) {
   VPInstruction *I1 = new VPInstruction(Instruction::Add, {});
   VPInstruction *I2 = new VPInstruction(Instruction::Sub, {I1});
@@ -333,12 +334,14 @@ TEST(VPBasicBlockTest, print) {
   VPBB1->appendRecipe(I1);
   VPBB1->appendRecipe(I2);
   VPBB1->appendRecipe(I3);
+  VPBB1->setName("bb1");
 
   VPInstruction *I4 = new VPInstruction(Instruction::Mul, {I2, I1});
   VPInstruction *I5 = new VPInstruction(Instruction::Ret, {I4});
   VPBasicBlock *VPBB2 = new VPBasicBlock();
   VPBB2->appendRecipe(I4);
   VPBB2->appendRecipe(I5);
+  VPBB2->setName("bb2");
 
   VPBlockUtils::connectBlocks(VPBB1, VPBB2);
 
@@ -355,7 +358,8 @@ TEST(VPBasicBlockTest, print) {
   VPlan Plan;
   Plan.setEntry(VPBB1);
   std::string FullDump;
-  raw_string_ostream(FullDump) << Plan;
+  raw_string_ostream OS(FullDump);
+  Plan.printDOT(OS);
 
   const char *ExpectedStr = R"(digraph VPlan {
 graph [labelloc=t, fontsize=30; label="Vectorization Plan"]
@@ -363,21 +367,45 @@ node [shape=rect, fontname=Courier, fontsize=30]
 edge [fontname=Courier, fontsize=30]
 compound=true
   N0 [label =
-    ":\n" +
-      "EMIT vp<%0> = add\l" +
-      "EMIT vp<%1> = sub vp<%0>\l" +
-      "EMIT br vp<%0> vp<%1>\l"
+    "bb1:\l" +
+    "  EMIT vp\<%0\> = add\l" +
+    "  EMIT vp\<%1\> = sub vp\<%0\>\l" +
+    "  EMIT br vp\<%0\> vp\<%1\>\l" +
+    "Successor(s): bb2\l"
   ]
   N0 -> N1 [ label=""]
   N1 [label =
-    ":\n" +
-      "EMIT vp<%3> = mul vp<%1> vp<%0>\l" +
-      "EMIT ret vp<%3>\l"
+    "bb2:\l" +
+    "  EMIT vp\<%3\> = mul vp\<%1\> vp\<%0\>\l" +
+    "  EMIT ret vp\<%3\>\l" +
+    "No successors\l"
   ]
 }
 )";
   EXPECT_EQ(ExpectedStr, FullDump);
 
+  const char *ExpectedBlock1Str = R"(bb1:
+  EMIT vp<%0> = add
+  EMIT vp<%1> = sub vp<%0>
+  EMIT br vp<%0> vp<%1>
+Successor(s): bb2
+)";
+  std::string Block1Dump;
+  raw_string_ostream OS1(Block1Dump);
+  VPBB1->print(OS1);
+  EXPECT_EQ(ExpectedBlock1Str, Block1Dump);
+
+  // Ensure that numbering is good when dumping the second block in isolation.
+  const char *ExpectedBlock2Str = R"(bb2:
+  EMIT vp<%3> = mul vp<%1> vp<%0>
+  EMIT ret vp<%3>
+No successors
+)";
+  std::string Block2Dump;
+  raw_string_ostream OS2(Block2Dump);
+  VPBB2->print(OS2);
+  EXPECT_EQ(ExpectedBlock2Str, Block2Dump);
+
   {
     std::string I3Dump;
     raw_string_ostream OS(I3Dump);
@@ -395,6 +423,7 @@ compound=true
     EXPECT_EQ("EMIT vp<%3> = mul vp<%1> vp<%0>", I4Dump);
   }
 }
+#endif
 
 TEST(VPRecipeTest, CastVPInstructionToVPUser) {
   VPValue Op1;
@@ -581,6 +610,7 @@ TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUserAndVPDef) {
   delete Load;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 TEST(VPRecipeTest, dump) {
   VPlan Plan;
   VPBasicBlock *VPBB1 = new VPBasicBlock();
@@ -636,6 +666,7 @@ TEST(VPRecipeTest, dump) {
 
   delete AI;
 }
+#endif
 
 TEST(VPRecipeTest, CastVPReductionRecipeToVPUser) {
   LLVMContext C;
@@ -657,8 +688,10 @@ struct VPDoubleValueDef : public VPRecipeBase {
   }
 
   void execute(struct VPTransformState &State) override{};
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override {}
+#endif
 };
 
 TEST(VPDoubleValueDefTest, traverseUseLists) {
diff --git a/llvm/utils/FileCheck/FileCheck.cpp b/llvm/utils/FileCheck/FileCheck.cpp
index ebee55b3599a42e7b686b451fbdf53cd41d4b858..4dcc4812502e1e8128fe13bcc6346a1e49ec916e 100644
--- a/llvm/utils/FileCheck/FileCheck.cpp
+++ b/llvm/utils/FileCheck/FileCheck.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cmath>
+#include <map>
 using namespace llvm;
 
 static cl::extrahelp FileCheckOptsEnv(
@@ -212,11 +213,20 @@ static MarkerStyle GetMarker(FileCheckDiag::MatchType MatchTy) {
   case FileCheckDiag::MatchFoundButDiscarded:
     return MarkerStyle('!', raw_ostream::CYAN,
                        "discard: overlaps earlier match");
+  case FileCheckDiag::MatchFoundErrorNote:
+    // Note should always be overridden within the FileCheckDiag.
+    return MarkerStyle('!', raw_ostream::RED,
+                       "error: unknown error after match",
+                       /*FiltersAsError=*/true);
   case FileCheckDiag::MatchNoneAndExcluded:
     return MarkerStyle('X', raw_ostream::GREEN);
   case FileCheckDiag::MatchNoneButExpected:
     return MarkerStyle('X', raw_ostream::RED, "error: no match found",
                        /*FiltersAsError=*/true);
+  case FileCheckDiag::MatchNoneForInvalidPattern:
+    return MarkerStyle('X', raw_ostream::RED,
+                       "error: match failed for invalid pattern",
+                       /*FiltersAsError=*/true);
   case FileCheckDiag::MatchFuzzy:
     return MarkerStyle('?', raw_ostream::MAGENTA, "possible intended match",
                        /*FiltersAsError=*/true);
@@ -369,16 +379,25 @@ BuildInputAnnotations(const SourceMgr &SM, unsigned CheckFileBufferID,
                       const std::vector<FileCheckDiag> &Diags,
                       std::vector<InputAnnotation> &Annotations,
                       unsigned &LabelWidth) {
-  // How many diagnostics have we seen so far?
-  unsigned DiagCount = 0;
-  // How many diagnostics has the current check seen so far?
-  unsigned CheckDiagCount = 0;
+  struct CompareSMLoc {
+    bool operator()(const SMLoc &LHS, const SMLoc &RHS) const {
+      return LHS.getPointer() < RHS.getPointer();
+    }
+  };
+  // How many diagnostics does each pattern have?
+  std::map<SMLoc, unsigned, CompareSMLoc> DiagCountPerPattern;
+  for (auto Diag : Diags)
+    ++DiagCountPerPattern[Diag.CheckLoc];
+  // How many diagnostics have we seen so far per pattern?
+  std::map<SMLoc, unsigned, CompareSMLoc> DiagIndexPerPattern;
+  // How many total diagnostics have we seen so far?
+  unsigned DiagIndex = 0;
   // What's the widest label?
   LabelWidth = 0;
   for (auto DiagItr = Diags.begin(), DiagEnd = Diags.end(); DiagItr != DiagEnd;
        ++DiagItr) {
     InputAnnotation A;
-    A.DiagIndex = DiagCount++;
+    A.DiagIndex = DiagIndex++;
 
     // Build label, which uniquely identifies this check result.
     unsigned CheckBufferID = SM.FindBufferContainingLoc(DiagItr->CheckLoc);
@@ -394,17 +413,8 @@ BuildInputAnnotations(const SourceMgr &SM, unsigned CheckFileBufferID,
     else
       llvm_unreachable("expected diagnostic's check location to be either in "
                        "the check file or for an implicit pattern");
-    unsigned CheckDiagIndex = UINT_MAX;
-    auto DiagNext = std::next(DiagItr);
-    if (DiagNext != DiagEnd && DiagItr->CheckTy == DiagNext->CheckTy &&
-        DiagItr->CheckLoc == DiagNext->CheckLoc)
-      CheckDiagIndex = CheckDiagCount++;
-    else if (CheckDiagCount) {
-      CheckDiagIndex = CheckDiagCount;
-      CheckDiagCount = 0;
-    }
-    if (CheckDiagIndex != UINT_MAX)
-      Label << "'" << CheckDiagIndex;
+    if (DiagCountPerPattern[DiagItr->CheckLoc] > 1)
+      Label << "'" << DiagIndexPerPattern[DiagItr->CheckLoc]++;
     Label.flush();
     LabelWidth = std::max((std::string::size_type)LabelWidth, A.Label.size());
 
@@ -421,6 +431,11 @@ BuildInputAnnotations(const SourceMgr &SM, unsigned CheckFileBufferID,
           DiagItr->InputStartCol == DiagItr->InputEndCol)
         A.Marker.Lead = ' ';
     }
+    if (DiagItr->MatchTy == FileCheckDiag::MatchFoundErrorNote) {
+      assert(!DiagItr->Note.empty() &&
+             "expected custom note for MatchFoundErrorNote");
+      A.Marker.Note = "error: " + A.Marker.Note;
+    }
     A.FoundAndExpectedMatch =
         DiagItr->MatchTy == FileCheckDiag::MatchFoundAndExpected;
 
@@ -807,7 +822,7 @@ int main(int argc, char **argv) {
 
   // Read the expected strings from the check file.
   ErrorOr<std::unique_ptr<MemoryBuffer>> CheckFileOrErr =
-      MemoryBuffer::getFileOrSTDIN(CheckFilename);
+      MemoryBuffer::getFileOrSTDIN(CheckFilename, /*IsText=*/true);
   if (std::error_code EC = CheckFileOrErr.getError()) {
     errs() << "Could not open check file '" << CheckFilename
            << "': " << EC.message() << '\n';
@@ -829,7 +844,7 @@ int main(int argc, char **argv) {
 
   // Open the file to check and add it to SourceMgr.
   ErrorOr<std::unique_ptr<MemoryBuffer>> InputFileOrErr =
-      MemoryBuffer::getFileOrSTDIN(InputFilename);
+      MemoryBuffer::getFileOrSTDIN(InputFilename, /*IsText=*/true);
   if (InputFilename == "-")
     InputFilename = "<stdin>"; // Overwrite for improved diagnostic messages
   if (std::error_code EC = InputFileOrErr.getError()) {
diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp
index deb51a082649f688b6a56b50d02504b267b6257d..b21bf369d18e5f985d982f76220a2ebc095e45c1 100644
--- a/llvm/utils/TableGen/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/DirectiveEmitter.cpp
@@ -647,6 +647,29 @@ void GenerateFlangClauseCheckPrototypes(const DirectiveLanguage &DirLang,
   }
 }
 
+// Generate the mapping for clauses between the parser class and the
+// corresponding clause Kind
+void GenerateFlangClauseParserKindMap(const DirectiveLanguage &DirLang,
+                                      raw_ostream &OS) {
+
+  IfDefScope Scope("GEN_FLANG_CLAUSE_PARSER_KIND_MAP", OS);
+
+  OS << "\n";
+  for (const auto &C : DirLang.getClauses()) {
+    Clause Clause{C};
+    OS << "if constexpr (std::is_same_v<A, parser::"
+       << DirLang.getFlangClauseBaseClass()
+       << "::" << Clause.getFormattedParserClassName();
+    OS << ">)\n";
+    OS << "  return llvm::" << DirLang.getCppNamespace()
+       << "::Clause::" << DirLang.getClausePrefix() << Clause.getFormattedName()
+       << ";\n";
+  }
+
+  OS << "llvm_unreachable(\"Invalid " << DirLang.getName()
+     << " Parser clause\");\n";
+}
+
 // Generate the implementation section for the enumeration in the directive
 // language
 void EmitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
@@ -665,6 +688,8 @@ void EmitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
   GenerateFlangClauseUnparse(DirLang, OS);
 
   GenerateFlangClauseCheckPrototypes(DirLang, OS);
+
+  GenerateFlangClauseParserKindMap(DirLang, OS);
 }
 
 void GenerateClauseClassMacro(const DirectiveLanguage &DirLang,
@@ -733,37 +758,11 @@ void GenerateClauseClassMacro(const DirectiveLanguage &DirLang,
   OS << "#undef CLAUSE\n";
 }
 
-// Generate the implementation section for the enumeration in the directive
-// language.
-void EmitDirectivesGen(RecordKeeper &Records, raw_ostream &OS) {
-  const auto DirLang = DirectiveLanguage{Records};
-  if (DirLang.HasValidityErrors())
-    return;
-
-  EmitDirectivesFlangImpl(DirLang, OS);
-
-  GenerateClauseClassMacro(DirLang, OS);
-}
-
-// Generate the implementation for the enumeration in the directive
+// Generate the implemenation for the enumeration in the directive
 // language. This code can be included in library.
-void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
-  const auto DirLang = DirectiveLanguage{Records};
-  if (DirLang.HasValidityErrors())
-    return;
-
-  if (!DirLang.getIncludeHeader().empty())
-    OS << "#include \"" << DirLang.getIncludeHeader() << "\"\n\n";
-
-  OS << "#include \"llvm/ADT/StringRef.h\"\n";
-  OS << "#include \"llvm/ADT/StringSwitch.h\"\n";
-  OS << "#include \"llvm/Support/ErrorHandling.h\"\n";
-  OS << "\n";
-  OS << "using namespace llvm;\n";
-  llvm::SmallVector<StringRef, 2> Namespaces;
-  llvm::SplitString(DirLang.getCppNamespace(), Namespaces, "::");
-  for (auto Ns : Namespaces)
-    OS << "using namespace " << Ns << ";\n";
+void EmitDirectivesBasicImpl(const DirectiveLanguage &DirLang,
+                             raw_ostream &OS) {
+  IfDefScope Scope("GEN_DIRECTIVES_IMPL", OS);
 
   // getDirectiveKind(StringRef Str)
   GenerateGetKind(DirLang.getDirectives(), OS, "Directive", DirLang,
@@ -789,4 +788,18 @@ void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
   GenerateIsAllowedClause(DirLang, OS);
 }
 
+// Generate the implemenation section for the enumeration in the directive
+// language.
+void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
+  const auto DirLang = DirectiveLanguage{Records};
+  if (DirLang.HasValidityErrors())
+    return;
+
+  EmitDirectivesFlangImpl(DirLang, OS);
+
+  GenerateClauseClassMacro(DirLang, OS);
+
+  EmitDirectivesBasicImpl(DirLang, OS);
+}
+
 } // namespace llvm
diff --git a/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp b/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
index 1c03296fe411aac1497c38c96a995e10cb5b17c8..a647a754fe7a2de811b70d97d614e6bb472670bc 100644
--- a/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
+++ b/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
@@ -973,7 +973,13 @@ emitDecoderFunction(formatted_raw_ostream &OS, DecoderSet &Decoders,
     << "Address, const void *Decoder, bool &DecodeComplete) {\n";
   Indentation += 2;
   OS.indent(Indentation) << "DecodeComplete = true;\n";
-  OS.indent(Indentation) << "InsnType tmp;\n";
+  // TODO: When InsnType is large, using uint64_t limits all fields to 64 bits
+  // It would be better for emitBinaryParser to use a 64-bit tmp whenever
+  // possible but fall back to an InsnType-sized tmp for truly large fields.
+  OS.indent(Indentation) << "using TmpType = "
+                            "std::conditional_t<std::is_integral<InsnType>::"
+                            "value, InsnType, uint64_t>;\n";
+  OS.indent(Indentation) << "TmpType tmp;\n";
   OS.indent(Indentation) << "switch (Idx) {\n";
   OS.indent(Indentation) << "default: llvm_unreachable(\"Invalid index!\");\n";
   unsigned Index = 0;
@@ -1107,18 +1113,24 @@ void FilterChooser::emitBinaryParser(raw_ostream &o, unsigned &Indentation,
                                      bool &OpHasCompleteDecoder) const {
   const std::string &Decoder = OpInfo.Decoder;
 
-  if (OpInfo.numFields() != 1 || OpInfo.InitValue != 0) {
+  bool UseInsertBits = OpInfo.numFields() != 1 || OpInfo.InitValue != 0;
+
+  if (UseInsertBits) {
     o.indent(Indentation) << "tmp = 0x";
     o.write_hex(OpInfo.InitValue);
     o << ";\n";
   }
 
   for (const EncodingField &EF : OpInfo) {
-    o.indent(Indentation) << "tmp ";
-    if (OpInfo.numFields() != 1 || OpInfo.InitValue != 0) o << '|';
-    o << "= fieldFromInstruction"
-      << "(insn, " << EF.Base << ", " << EF.Width << ')';
-    if (OpInfo.numFields() != 1 || EF.Offset != 0)
+    o.indent(Indentation);
+    if (UseInsertBits)
+      o << "insertBits(tmp, ";
+    else
+      o << "tmp = ";
+    o << "fieldFromInstruction(insn, " << EF.Base << ", " << EF.Width << ')';
+    if (UseInsertBits)
+      o << ", " << EF.Offset << ", " << EF.Width << ')';
+    else if (EF.Offset != 0)
       o << " << " << EF.Offset;
     o << ";\n";
   }
@@ -2142,27 +2154,22 @@ static void emitFieldFromInstruction(formatted_raw_ostream &OS) {
   OS << "// Helper functions for extracting fields from encoded instructions.\n"
      << "// InsnType must either be integral or an APInt-like object that "
         "must:\n"
-     << "// * Have a static const max_size_in_bits equal to the number of bits "
-        "in the\n"
-     << "//   encoding.\n"
      << "// * be default-constructible and copy-constructible\n"
      << "// * be constructible from a uint64_t\n"
      << "// * be constructible from an APInt (this can be private)\n"
-     << "// * Support getBitsSet(loBit, hiBit)\n"
-     << "// * be convertible to uint64_t\n"
-     << "// * Support the ~, &, ==, !=, and |= operators with other objects of "
+     << "// * Support insertBits(bits, startBit, numBits)\n"
+     << "// * Support extractBitsAsZExtValue(numBits, startBit)\n"
+     << "// * be convertible to bool\n"
+     << "// * Support the ~, &, ==, and != operators with other objects of "
         "the same type\n"
-     << "// * Support shift (<<, >>) with signed and unsigned integers on the "
-        "RHS\n"
      << "// * Support put (<<) to raw_ostream&\n"
      << "template <typename InsnType>\n"
      << "#if defined(_MSC_VER) && !defined(__clang__)\n"
      << "__declspec(noinline)\n"
      << "#endif\n"
-     << "static InsnType fieldFromInstruction(InsnType insn, unsigned "
-        "startBit,\n"
-     << "                                     unsigned numBits, "
-        "std::true_type) {\n"
+     << "static std::enable_if_t<std::is_integral<InsnType>::value, InsnType>\n"
+     << "fieldFromInstruction(const InsnType &insn, unsigned startBit,\n"
+     << "                     unsigned numBits) {\n"
      << "  assert(startBit + numBits <= 64 && \"Cannot support >64-bit "
         "extractions!\");\n"
      << "  assert(startBit + numBits <= (sizeof(InsnType) * 8) &&\n"
@@ -2176,22 +2183,32 @@ static void emitFieldFromInstruction(formatted_raw_ostream &OS) {
      << "}\n"
      << "\n"
      << "template <typename InsnType>\n"
-     << "static InsnType fieldFromInstruction(InsnType insn, unsigned "
-        "startBit,\n"
-     << "                                     unsigned numBits, "
-        "std::false_type) {\n"
-     << "  assert(startBit + numBits <= InsnType::max_size_in_bits && "
-        "\"Instruction field out of bounds!\");\n"
-     << "  InsnType fieldMask = InsnType::getBitsSet(0, numBits);\n"
-     << "  return (insn >> startBit) & fieldMask;\n"
+     << "static std::enable_if_t<!std::is_integral<InsnType>::value, "
+        "uint64_t>\n"
+     << "fieldFromInstruction(const InsnType &insn, unsigned startBit,\n"
+     << "                     unsigned numBits) {\n"
+     << "  return insn.extractBitsAsZExtValue(numBits, startBit);\n"
+     << "}\n\n";
+}
+
+// emitInsertBits - Emit the templated helper function insertBits().
+static void emitInsertBits(formatted_raw_ostream &OS) {
+  OS << "// Helper function for inserting bits extracted from an encoded "
+        "instruction into\n"
+     << "// a field.\n"
+     << "template <typename InsnType>\n"
+     << "static std::enable_if_t<std::is_integral<InsnType>::value>\n"
+     << "insertBits(InsnType &field, InsnType bits, unsigned startBit, "
+        "unsigned numBits) {\n"
+     << "  assert(startBit + numBits <= sizeof field * 8);\n"
+     << "  field |= (InsnType)bits << startBit;\n"
      << "}\n"
      << "\n"
      << "template <typename InsnType>\n"
-     << "static InsnType fieldFromInstruction(InsnType insn, unsigned "
-        "startBit,\n"
-     << "                                     unsigned numBits) {\n"
-     << "  return fieldFromInstruction(insn, startBit, numBits, "
-        "std::is_integral<InsnType>());\n"
+     << "static std::enable_if_t<!std::is_integral<InsnType>::value>\n"
+     << "insertBits(InsnType &field, uint64_t bits, unsigned startBit, "
+        "unsigned numBits) {\n"
+     << "  field.insertBits(bits, startBit, numBits);\n"
      << "}\n\n";
 }
 
@@ -2394,6 +2411,7 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) {
   OS << "namespace llvm {\n\n";
 
   emitFieldFromInstruction(OS);
+  emitInsertBits(OS);
 
   Target.reverseBitsForLittleEndianEncoding();
 
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index fb953bdf52a1d4a6a90dead04f4f8387bff052f9..08760e970963e8da29b1d93ec58e4cc40b08443b 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -5351,7 +5351,7 @@ void GlobalISelEmitter::emitCxxPredicateFns(
     StringRef AdditionalDeclarations,
     std::function<bool(const Record *R)> Filter) {
   std::vector<const Record *> MatchedRecords;
-  const auto &Defs = RK.getAllDerivedDefinitions("PatFrag");
+  const auto &Defs = RK.getAllDerivedDefinitions("PatFrags");
   std::copy_if(Defs.begin(), Defs.end(), std::back_inserter(MatchedRecords),
                [&](Record *Record) {
                  return !Record->getValueAsString(CodeFieldName).empty() &&
diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index e05409db67d075f1c6070cc68d39339bc4be31c9..6acb630299c121403f31f7b19df410b23ead5439 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -108,6 +108,11 @@ addDagOperandMapping(Record *Rec, DagInit *Dag, CodeGenInstruction &Insn,
       OperandMap[BaseIdx + i].Kind = OpData::Imm;
       OperandMap[BaseIdx + i].Data.Imm = II->getValue();
       ++OpsAdded;
+    } else if (auto *BI = dyn_cast<BitsInit>(Dag->getArg(i))) {
+      auto *II = cast<IntInit>(BI->convertInitializerTo(IntRecTy::get()));
+      OperandMap[BaseIdx + i].Kind = OpData::Imm;
+      OperandMap[BaseIdx + i].Data.Imm = II->getValue();
+      ++OpsAdded;
     } else if (DagInit *SubDag = dyn_cast<DagInit>(Dag->getArg(i))) {
       // Just add the operands recursively. This is almost certainly
       // a constant value for a complex operand (> 1 MI operand).
diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index 7e062492db7540f8fd4ad5c0e9075f0ac11f4450..8885c10dc48969e7c0d2c5bc7944374f09f6c7a4 100644
--- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -1084,13 +1084,16 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target,
 
   for (const auto &RC : RegisterClasses) {
     assert(isInt<8>(RC.CopyCost) && "Copy cost too large.");
+    uint32_t RegSize = 0;
+    if (RC.RSI.isSimple())
+      RegSize = RC.RSI.getSimple().RegSize;
     OS << "  { " << RC.getName() << ", " << RC.getName() << "Bits, "
-       << RegClassStrings.get(RC.getName()) << ", "
-       << RC.getOrder().size() << ", sizeof(" << RC.getName() << "Bits), "
-       << RC.getQualifiedName() + "RegClassID" << ", "
-       << RC.CopyCost << ", "
-       << ( RC.Allocatable ? "true" : "false" ) << ", "
-       << ( RC.AllowsNoRegister ? "true" : "false" ) << " },\n";
+       << RegClassStrings.get(RC.getName()) << ", " << RC.getOrder().size()
+       << ", sizeof(" << RC.getName() << "Bits), "
+       << RC.getQualifiedName() + "RegClassID"
+       << ", " << RegSize << ", " << RC.CopyCost << ", "
+       << (RC.Allocatable ? "true" : "false") << ", "
+       << (RC.AllowsNoRegister ? "true" : "false") << " },\n";
   }
 
   OS << "};\n\n";
diff --git a/llvm/utils/TableGen/TableGen.cpp b/llvm/utils/TableGen/TableGen.cpp
index a4b16179cdf769cc8219e00a895a36ad01f1fed2..24c11c8bc8313f86e54691fe1c2691daed46b45d 100644
--- a/llvm/utils/TableGen/TableGen.cpp
+++ b/llvm/utils/TableGen/TableGen.cpp
@@ -57,7 +57,6 @@ enum ActionType {
   GenAutomata,
   GenDirectivesEnumDecl,
   GenDirectivesEnumImpl,
-  GenDirectivesEnumGen,
 };
 
 namespace llvm {
@@ -139,9 +138,7 @@ cl::opt<ActionType> Action(
         clEnumValN(GenDirectivesEnumDecl, "gen-directive-decl",
                    "Generate directive related declaration code (header file)"),
         clEnumValN(GenDirectivesEnumImpl, "gen-directive-impl",
-                   "Generate directive related implementation code"),
-        clEnumValN(GenDirectivesEnumGen, "gen-directive-gen",
-                   "Generate directive related implementation code part")));
+                   "Generate directive related implementation code")));
 
 cl::OptionCategory PrintEnumsCat("Options for -print-enums");
 cl::opt<std::string> Class("class", cl::desc("Print Enum list for this class"),
@@ -275,9 +272,6 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenDirectivesEnumImpl:
     EmitDirectivesImpl(Records, OS);
     break;
-  case GenDirectivesEnumGen:
-    EmitDirectivesGen(Records, OS);
-    break;
   }
 
   return false;
diff --git a/llvm/utils/TableGen/TableGenBackends.h b/llvm/utils/TableGen/TableGenBackends.h
index ed0057c697cf13aad8d2f5bb743a935a2f2d5a49..71db8dc77b0525cc8d16e570847d551443ec21fe 100644
--- a/llvm/utils/TableGen/TableGenBackends.h
+++ b/llvm/utils/TableGen/TableGenBackends.h
@@ -93,7 +93,6 @@ void EmitExegesis(RecordKeeper &RK, raw_ostream &OS);
 void EmitAutomata(RecordKeeper &RK, raw_ostream &OS);
 void EmitDirectivesDecl(RecordKeeper &RK, raw_ostream &OS);
 void EmitDirectivesImpl(RecordKeeper &RK, raw_ostream &OS);
-void EmitDirectivesGen(RecordKeeper &RK, raw_ostream &OS);
 
 } // End llvm namespace
 
diff --git a/llvm/utils/UpdateTestChecks/asm.py b/llvm/utils/UpdateTestChecks/asm.py
index 3cbedfeaccf24cf6600553bef9175385600a4889..b5546e7d2411af98a3190f56bc139069eaa9ee12 100644
--- a/llvm/utils/UpdateTestChecks/asm.py
+++ b/llvm/utils/UpdateTestChecks/asm.py
@@ -197,7 +197,7 @@ def scrub_asm_x86(asm, args):
     # Generically match a RIP-relative memory operand.
     asm = SCRUB_X86_RIP_RE.sub(r'{{.*}}(%rip)', asm)
   # Generically match a LCP symbol.
-  asm = SCRUB_X86_LCP_RE.sub(r'{{\.LCPI.*}}', asm)
+  asm = SCRUB_X86_LCP_RE.sub(r'{{\.LCPI[0-9]+_[0-9]+}}', asm)
   if getattr(args, 'extra_scrub', False):
     # Avoid generating different checks for 32- and 64-bit because of 'retl' vs 'retq'.
     asm = SCRUB_X86_RET_RE.sub(r'ret{{[l|q]}}', asm)
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index b3ca33f07428a95391dbf8aa9b418ec211c8c09c..45984751d6e77830900ede3a9196012fa5f71f60 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -16,6 +16,7 @@ else:
 
 
 _verbose = False
+_prefix_filecheck_ir_name = ''
 
 def parse_commandline_args(parser):
   parser.add_argument('--include-generated-funcs', action='store_true',
@@ -32,6 +33,8 @@ def parse_commandline_args(parser):
                       help='Deactivate CHECK line generation from this point forward')
   parser.add_argument('--replace-function-regex', nargs='+', default=[],
                       help='List of regular expressions to replace matching function names')
+  parser.add_argument('--prefix-filecheck-ir-name', default='',
+                      help='Add a prefix to FileCheck IR value names to avoid conflicts with scripted names')
   args = parser.parse_args()
   global _verbose
   _verbose = args.verbose
@@ -53,6 +56,9 @@ class TestInfo(object):
     self.argparse_callback = argparse_callback
     self.path = test
     self.args = args
+    if args.prefix_filecheck_ir_name:
+      global _prefix_filecheck_ir_name
+      _prefix_filecheck_ir_name = args.prefix_filecheck_ir_name
     self.argv = argv
     self.input_lines = input_lines
     self.run_lines = find_run_lines(test, self.input_lines)
@@ -362,8 +368,8 @@ class FunctionTestBuilder:
             func_repl = regex
             # Replace any capture groups with their matched strings.
             for g in match.groups():
-              func_repl = group_regex.sub(g, func_repl, count=1)
-            func = '{{' + func_repl + '}}'
+              func_repl = group_regex.sub(re.escape(g), func_repl, count=1)
+            func = re.sub(func_repl, '{{' + func_repl + '}}', func)
 
           # Replace all calls to regex matching functions.
           matches = re.finditer(regex, scrubbed_body)
@@ -371,7 +377,7 @@ class FunctionTestBuilder:
             func_repl = regex
             # Replace any capture groups with their matched strings.
             for g in match.groups():
-                func_repl = group_regex.sub(g, func_repl, count=1)
+                func_repl = group_regex.sub(re.escape(g), func_repl, count=1)
             # Substitute function call names that match the regex with the same
             # capture groups set.
             scrubbed_body = re.sub(func_repl, '{{' + func_repl + '}}', scrubbed_body)
@@ -504,19 +510,29 @@ def get_ir_prefix_from_ir_value_re_match(match):
         return nameless_values[idx].ir_regexp
     return nameless_values[idx].global_ir_prefix_regexp
 
-# Return true if this kind or IR value is "local", basically if it matches '%{{.*}}'.
+# Return true if this kind of IR value is "local", basically if it matches '%{{.*}}'.
 def is_local_def_ir_value_match(match):
     return nameless_values[get_idx_from_ir_value_match(match)].ir_prefix == '%'
 
-# Return true if this kind or IR value is "local", basically if it matches '%{{.*}}'.
+# Return true if this kind of IR value is "global", basically if it matches '#{{.*}}'.
 def is_global_scope_ir_value_match(match):
     return nameless_values[get_idx_from_ir_value_match(match)].global_ir_prefix is not None
 
+# Return true if var clashes with the scripted FileCheck check_prefix.
+def may_clash_with_default_check_prefix_name(check_prefix, var):
+  return check_prefix and re.match(r'^' + check_prefix + r'[0-9]+?$', var, re.IGNORECASE)
+
 # Create a FileCheck variable name based on an IR name.
 def get_value_name(var, check_prefix):
   var = var.replace('!', '')
+  # This is a nameless value, prepend check_prefix.
   if var.isdigit():
     var = check_prefix + var
+  else:
+    # This is a named value that clashes with the check_prefix, prepend with _prefix_filecheck_ir_name,
+    # if it has been defined.
+    if may_clash_with_default_check_prefix_name(check_prefix, var) and _prefix_filecheck_ir_name:
+      var = _prefix_filecheck_ir_name + var
   var = var.replace('.', '_')
   var = var.replace('-', '_')
   return var.upper()
@@ -546,8 +562,9 @@ def generalize_check_lines(lines, is_analyze, vars_seen, global_vars_seen):
     pre, check = get_ir_prefix_from_ir_value_match(match)
     var = get_name_from_ir_value_match(match)
     for nameless_value in nameless_values:
-        if nameless_value.check_prefix and re.match(r'^' + nameless_value.check_prefix + r'[0-9]+?$', var, re.IGNORECASE):
-            warn("Change IR value name '%s' to prevent possible conflict with scripted FileCheck name." % (var,))
+        if may_clash_with_default_check_prefix_name(nameless_value.check_prefix, var):
+          warn("Change IR value name '%s' or use -prefix-ir-filecheck-name to prevent possible conflict"
+            " with scripted FileCheck name." % (var,))
     key = (var, get_check_key_from_ir_value_match(match))
     is_local_def = is_local_def_ir_value_match(match)
     if is_local_def and key in vars_seen:
diff --git a/llvm/utils/gn/build/toolchain/target_flags.gni b/llvm/utils/gn/build/toolchain/target_flags.gni
index 0af52a0db6da4a32ad36ab49d11e701460a746c2..573e758a6d6f67e0262e58ed3aac09cc02dcc7a5 100644
--- a/llvm/utils/gn/build/toolchain/target_flags.gni
+++ b/llvm/utils/gn/build/toolchain/target_flags.gni
@@ -13,8 +13,11 @@ if (current_os == "android") {
   target_flags += [
     "--target=$llvm_current_triple",
     "--sysroot=$android_ndk_path/toolchains/llvm/prebuilt/linux-x86_64/sysroot",
-    "-B$android_ndk_path/toolchains/llvm/prebuilt/linux-x86_64",
+    "--gcc-toolchain=$android_ndk_path/toolchains/llvm/prebuilt/linux-x86_64",
   ]
+  if (current_cpu != "arm") {
+    target_flags += [ "-rtlib=libgcc" ]
+  }
   target_ldflags += [ "-static-libstdc++" ]
   if (current_cpu == "arm") {
     target_flags += [ "-march=armv7-a" ]
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/altera/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/altera/BUILD.gn
index dff088d817114c9ea406c7dbce4a5732770e1b76..2d40cf14af6dc2374d9cb8f29a5d099e0c16a0fe 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/altera/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/altera/BUILD.gn
@@ -16,5 +16,6 @@ static_library("altera") {
     "KernelNameRestrictionCheck.cpp",
     "SingleWorkItemBarrierCheck.cpp",
     "StructPackAlignCheck.cpp",
+    "UnrollLoopsCheck.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn
index fa1e24d32312862fb2cb8aa44299235f4b565a99..2c8d8cb5c0fc0e68a81cc1f9fc478eec5038e943 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn
@@ -18,7 +18,6 @@ static_library("readability") {
     "ContainerSizeEmptyCheck.cpp",
     "ConvertMemberFunctionsToStatic.cpp",
     "DeleteNullPointerCheck.cpp",
-    "DeletedDefaultCheck.cpp",
     "ElseAfterReturnCheck.cpp",
     "FunctionCognitiveComplexityCheck.cpp",
     "FunctionSizeCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index 479ba31eec6855353a6b69b451c28c3827f9a623..7c127c3a38f95db5dc468a3857d280ee4e7e0bc2 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -50,13 +50,6 @@ clang_tablegen("riscv_vector") {
   output_name = "riscv_vector.h"
 }
 
-# Generate riscv_vector_generic.h
-clang_tablegen("riscv_vector_generic") {
-  args = [ "-gen-riscv-vector-generic-header" ]
-  td_file = "//clang/include/clang/Basic/riscv_vector.td"
-  output_name = "riscv_vector_generic.h"
-}
-
 copy("tablegen_headers") {
   visibility = [ ":Headers" ]
   deps = [
@@ -67,7 +60,6 @@ copy("tablegen_headers") {
     ":arm_neon",
     ":arm_sve",
     ":riscv_vector",
-    ":riscv_vector_generic",
   ]
   sources = []
   foreach(dep, deps) {
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 6411d49b67de38bedaa3a3d3c2877a1b1443e5b0..9585e280856e29de93c918817f25b41683e4f363 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -56,6 +56,7 @@ write_lit_config("lit_site_cfg") {
     # that's also read here -- but that should happen after multi-toolchain
     # builds exist, to make sure it's a toolchain var.
     "CMAKE_CXX_COMPILER=c++",
+    "CMAKE_C_COMPILER=cc",
     "ENABLE_BACKTRACES=1",
     "LLVM_ENABLE_NEW_PASS_MANAGER=1",
     "LLVM_HOST_TRIPLE=$llvm_current_triple",
diff --git a/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn
index f25ead00165c065324f53bacd06decbbda61978e..a2f782d06e719a34cf582c87e90470b0b91d1995 100644
--- a/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn
@@ -18,6 +18,7 @@ unittest("ASTTests") {
     "ASTImporterFixtures.cpp",
     "ASTImporterGenericRedeclTest.cpp",
     "ASTImporterODRStrategiesTest.cpp",
+    "ASTImporterObjCTest.cpp",
     "ASTImporterTest.cpp",
     "ASTImporterVisibilityTest.cpp",
     "ASTTraverserTest.cpp",
diff --git a/llvm/utils/gn/secondary/compiler-rt/test/hwasan/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/hwasan/BUILD.gn
index e58637be0d80bec41504d2e00d1bec91ea45abde..de601b8d6122dc5ae776fc71ea491058f8f28820 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/hwasan/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/hwasan/BUILD.gn
@@ -84,10 +84,10 @@ if (supported_toolchains != []) {
     outputs = [ "$target_gen_dir/run-lit" ]  # Non-existing, so that ninja runs
                                              # it each time.
 
-    # Since check-hwasan is always dirty, //:default doesn't depend on it so that
-    # it's not part of the default ninja target.  Hence, check-hwasan shouldn't
-    # have any deps except :hwasan. so that the default target is sure to build
-    # all the deps.
+    # Since check-hwasan is always dirty, //:default doesn't depend on it so
+    # that it's not part of the default ninja target.  Hence, check-hwasan
+    # shouldn't have any deps except :hwasan. so that the default target is
+    # sure to build all the deps.
     deps = [ ":hwasan" ]
     testonly = true
 
diff --git a/llvm/utils/gn/secondary/lld/MachO/BUILD.gn b/llvm/utils/gn/secondary/lld/MachO/BUILD.gn
index d0fe2e128d9e02b1818f2eb4e8b7afe1a3f98cf6..98c38399b08b744078791f12c0025bf9108758e7 100644
--- a/llvm/utils/gn/secondary/lld/MachO/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/MachO/BUILD.gn
@@ -31,6 +31,7 @@ static_library("MachO2") {
     "InputFiles.cpp",
     "InputSection.cpp",
     "LTO.cpp",
+    "MapFile.cpp",
     "MergedOutputSection.cpp",
     "ObjC.cpp",
     "OutputSection.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn
index 995b7f29e52188cbe6d785f757c897eb02205b85..c6d80b2be3f6b27f8d535f35aad4f7a0d3671002 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn
@@ -8,7 +8,7 @@ tablegen("ACC") {
 
 tablegen("ACCcpp") {
   visibility = [ ":acc_gen" ]
-  args = [ "-gen-directive-gen" ]
+  args = [ "-gen-directive-impl" ]
   output_name = "ACC.inc"
   td_file = "ACC.td"
 }
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
index 04bfdd6f313472bfd6570d9918d8dea3da4632e0..d99b8b5a1e27dd451e8de6b7c664028751ec6677 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
@@ -9,7 +9,7 @@ tablegen("OMPh") {
 
 tablegen("OMP") {
   visibility = [ ":public_tablegen" ]
-  args = [ "-gen-directive-gen" ]
+  args = [ "-gen-directive-impl" ]
 }
 
 # Groups all tablegen() calls that create .inc files that are included in
diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn
index 51b8c269c2816bf82fcb471d7443d1186c314b97..f37bed799c7bc3106fc384a9e546cbc0e0cae162 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn
@@ -1,18 +1,8 @@
 import("//llvm/utils/TableGen/tablegen.gni")
 
-tablegen("ACCImpl") {
-  visibility = [ ":OpenACC" ]
-  args = [ "-gen-directive-impl" ]
-  td_file = "//llvm/include/llvm/Frontend/OpenACC/ACC.td"
-  output_name = "ACC.cpp"
-}
-
 static_library("OpenACC") {
   output_name = "LLVMFrontendOpenACC"
-  deps = [
-    ":ACCImpl",
-    "//llvm/lib/Support",
-  ]
+  deps = [ "//llvm/lib/Support" ]
   public_deps = [ "//llvm/include/llvm/Frontend/OpenACC:acc_gen" ]
-  sources = get_target_outputs(":ACCImpl")
+  sources = [ "ACC.cpp" ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
index 579e37c6d5cb7eb8ad65fb74325d2358e99c6dc6..d061eda5a557c1e1ab2895aa40c0260aa248f085 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
@@ -1,23 +1,16 @@
 import("//llvm/utils/TableGen/tablegen.gni")
 
-tablegen("OMPImpl") {
-  visibility = [ ":OpenMP" ]
-  args = [ "-gen-directive-impl" ]
-  td_file = "//llvm/include/llvm/Frontend/OpenMP/OMP.td"
-  output_name = "OMP.cpp"
-}
-
 static_library("OpenMP") {
   output_name = "LLVMFrontendOpenMP"
   deps = [
-    ":OMPImpl",
     "//llvm/lib/IR",
     "//llvm/lib/Support",
     "//llvm/lib/Transforms/Utils",
   ]
   public_deps = [ "//llvm/include/llvm/Frontend/OpenMP:public_tablegen" ]
   sources = [
-              "OMPContext.cpp",
-              "OMPIRBuilder.cpp",
-            ] + get_target_outputs(":OMPImpl")
+    "OMP.cpp",
+    "OMPContext.cpp",
+    "OMPIRBuilder.cpp",
+  ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 6b50aa58bd6bdfc97bf1a74d59e69c2233e4d69b..cd6cac3a5b709ea314fee9e0fc88d745858079da 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -203,7 +203,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "SIFrameLowering.cpp",
     "SIISelLowering.cpp",
     "SIInsertHardClauses.cpp",
-    "SIInsertSkips.cpp",
+    "SILateBranchLowering.cpp",
     "SIInsertWaitcnts.cpp",
     "SIInstrInfo.cpp",
     "SILoadStoreOptimizer.cpp",
@@ -222,7 +222,6 @@ static_library("LLVMAMDGPUCodeGen") {
     "SIPreEmitPeephole.cpp",
     "SIProgramInfo.cpp",
     "SIRegisterInfo.cpp",
-    "SIRemoveShortExecBranches.cpp",
     "SIShrinkInstructions.cpp",
     "SIWholeQuadMode.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn
index 36ac22dca8a1a747dd173e42c390c1fef534a8e4..2847ac1bd30878707e11f8dc33318645d00bc1f4 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn
@@ -98,9 +98,10 @@ static_library("LLVMARMCodeGen") {
     "ARMTargetTransformInfo.cpp",
     "MLxExpansionPass.cpp",
     "MVEGatherScatterLowering.cpp",
+    "MVELaneInterleavingPass.cpp",
+    "MVETPAndVPTOptimisationsPass.cpp",
     "MVETailPredication.cpp",
     "MVEVPTBlockPass.cpp",
-    "MVETPAndVPTOptimisationsPass.cpp",
     "Thumb1FrameLowering.cpp",
     "Thumb1InstrInfo.cpp",
     "Thumb2ITBlockPass.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni b/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni
index 102040c2fa82da7abb5d75986d46de329c8c6ab0..22da518ea1fb9e83e2074e37f5b6d240cda9427c 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni
@@ -49,6 +49,7 @@ llvm_build_ARM = false
 llvm_build_BPF = false
 llvm_build_Mips = false
 llvm_build_PowerPC = false
+llvm_build_SystemZ = false
 llvm_build_WebAssembly = false
 llvm_build_X86 = false
 foreach(target, llvm_targets_to_build) {
@@ -64,6 +65,8 @@ foreach(target, llvm_targets_to_build) {
     llvm_build_Mips = true
   } else if (target == "PowerPC") {
     llvm_build_PowerPC = true
+  } else if (target == "SystemZ") {
+    llvm_build_SystemZ = true
   } else if (target == "WebAssembly") {
     llvm_build_WebAssembly = true
   } else if (target == "X86") {
diff --git a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
index 4942f4b1445965e35f44acb3dfcc517268a47ef5..949816308a525df29f3c7fa3c180a93283686daf 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
@@ -80,6 +80,9 @@ group("unittests") {
       "tools/llvm-exegesis/PowerPC:LLVMExegesisPowerPCTests",
     ]
   }
+  if (llvm_build_SystemZ) {
+    deps += [ "MC/SystemZ:SystemZAsmLexerTests" ]
+  }
   if (llvm_build_WebAssembly) {
     deps += [ "Target/WebAssembly:WebAssemblyTests" ]
   }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/MC/SystemZ/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/MC/SystemZ/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..637fc342d489c85d7b68d42d8eae21d1556e178b
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/unittests/MC/SystemZ/BUILD.gn
@@ -0,0 +1,11 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("SystemZAsmLexerTests") {
+  deps = [
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/SystemZ",
+  ]
+  sources = [ "SystemZAsmLexerTest.cpp" ]
+}
diff --git a/llvm/utils/lit/lit/ProgressBar.py b/llvm/utils/lit/lit/ProgressBar.py
index 4f8bd3cc75e3aa99c75a5dd07a161f91f79f95c0..fd721db780b5c2388154b93ab9cca55e1d57f02b 100644
--- a/llvm/utils/lit/lit/ProgressBar.py
+++ b/llvm/utils/lit/lit/ProgressBar.py
@@ -253,7 +253,7 @@ class ProgressBar:
             elapsed = time.time() - self.startTime
             if percent > .0001 and elapsed > 1:
                 total = elapsed / percent
-                eta = int(total - elapsed)
+                eta = total - elapsed
                 h = eta//3600.
                 m = (eta//60) % 60
                 s = eta % 60
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index ce87cfa8abb580f16201d128e504c3de05822396..7cc610bf56bde249bfd45ee9a75072fcd5f843c6 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -3,6 +3,7 @@ import os
 from json import JSONEncoder
 
 from lit.BooleanExpression import BooleanExpression
+from lit.TestTimes import read_test_times
 
 # Test result codes.
 
@@ -207,6 +208,8 @@ class TestSuite:
         # The test suite configuration.
         self.config = config
 
+        self.test_times = read_test_times(self)
+
     def getSourcePath(self, components):
         return os.path.join(self.source_root, *components)
 
@@ -246,6 +249,18 @@ class Test:
         # The test result, once complete.
         self.result = None
 
+        # The previous test failure state, if applicable.
+        self.previous_failure = False
+
+        # The previous test elapsed time, if applicable.
+        self.previous_elapsed = 0.0
+
+        if '/'.join(path_in_suite) in suite.test_times:
+            time = suite.test_times['/'.join(path_in_suite)]
+            self.previous_elapsed = abs(time)
+            self.previous_failure = time < 0
+
+
     def setResult(self, result):
         assert self.result is None, "result already set"
         assert isinstance(result, Result), "unexpected result type"
@@ -395,15 +410,3 @@ class Test:
         )
         identifiers = set(filter(BooleanExpression.isIdentifier, tokens))
         return identifiers
-
-    def isEarlyTest(self):
-        """
-        isEarlyTest() -> bool
-
-        Check whether this test should be executed early in a particular run.
-        This can be used for test suites with long running tests to maximize
-        parallelism or where it is desirable to surface their failures early.
-        """
-        if '/'.join(self.path_in_suite) in self.suite.config.early_tests:
-            return True
-        return self.suite.config.is_early
diff --git a/llvm/utils/lit/lit/TestTimes.py b/llvm/utils/lit/lit/TestTimes.py
new file mode 100644
index 0000000000000000000000000000000000000000..621343f29d262276b2b64f835cb30637147c36ad
--- /dev/null
+++ b/llvm/utils/lit/lit/TestTimes.py
@@ -0,0 +1,41 @@
+import os
+
+
+def read_test_times(suite):
+    test_times = {}
+    test_times_file = os.path.join(suite.exec_root, '.lit_test_times.txt')
+    if not os.path.exists(test_times_file):
+        test_times_file = os.path.join(
+            suite.source_root, '.lit_test_times.txt')
+    if os.path.exists(test_times_file):
+        with open(test_times_file, 'r') as time_file:
+            for line in time_file:
+                time, path = line.split(maxsplit=1)
+                test_times[path.strip('\n')] = float(time)
+    return test_times
+
+
+def record_test_times(tests, lit_config):
+    times_by_suite = {}
+    for t in tests:
+        assert t.suite.test_times is None
+        if not t.result.elapsed:
+            continue
+        if not t.suite.exec_root in times_by_suite:
+            times_by_suite[t.suite.exec_root] = read_test_times(t.suite)
+        time = -t.result.elapsed if t.isFailure() else t.result.elapsed
+        # The "path" here is only used as a key into a dictionary. It is never
+        # used as an actual path to a filesystem API, therefore we use '/' as
+        # the canonical separator so that Unix and Windows machines can share
+        # timing data.
+        times_by_suite[t.suite.exec_root]['/'.join(t.path_in_suite)] = t.result.elapsed
+
+    for s, value in times_by_suite.items():
+        try:
+            path = os.path.join(s, '.lit_test_times.txt')
+            with open(path, 'w') as time_file:
+                for name, time in value.items():
+                    time_file.write(("%e" % time) + ' ' + name + '\n')
+        except:
+            lit_config.warning('Could not save test time: ' + path)
+            continue
diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py
index fafc754c1bc157230bdd4bfcfb6e25c656ae5d66..e1d13aa3a064512f676cded29049c09fa1eee2e4 100644
--- a/llvm/utils/lit/lit/TestingConfig.py
+++ b/llvm/utils/lit/lit/TestingConfig.py
@@ -33,6 +33,7 @@ class TestingConfig(object):
             pass_vars.append('INCLUDE')
             pass_vars.append('LIB')
             pass_vars.append('PATHEXT')
+            pass_vars.append('USERPROFILE')
             environment['PYTHONBUFFERED'] = '1'
 
         for var in pass_vars:
@@ -125,10 +126,6 @@ class TestingConfig(object):
         # require one of the features in this list if this list is non-empty.
         # Configurations can set this list to restrict the set of tests to run.
         self.limit_to_features = set(limit_to_features)
-        # Whether the suite should be tested early in a given run.
-        self.is_early = bool(is_early)
-        # List of tests to run early.
-        self.early_tests = {}
         self.parallelism_group = parallelism_group
         self._recursiveExpansionLimit = None
 
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 4d829659ea188babe5e60a51aedc0dd061f755e1..3eb1870bf16d9ce7127422ee8a19c61708379254 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -9,8 +9,7 @@ import lit.util
 
 
 class TestOrder(enum.Enum):
-    EARLY_TESTS_THEN_BY_NAME = enum.auto()
-    FAILING_FIRST = enum.auto()
+    DEFAULT = enum.auto()
     RANDOM = enum.auto()
 
 
@@ -155,7 +154,7 @@ def parse_args():
             help="Run tests in random order",
             action="store_true")
     selection_group.add_argument("-i", "--incremental",
-            help="Run modified and failing tests first (updates mtimes)",
+            help="Run failed tests first (DEPRECATED: now always enabled)",
             action="store_true")
     selection_group.add_argument("--filter",
             metavar="REGEX",
@@ -208,12 +207,13 @@ def parse_args():
     if opts.echoAllCommands:
         opts.showOutput = True
 
+    if opts.incremental:
+        print('WARNING: --incremental is deprecated. Failing tests now always run first.')
+
     if opts.shuffle:
         opts.order = TestOrder.RANDOM
-    elif opts.incremental:
-        opts.order = TestOrder.FAILING_FIRST
     else:
-        opts.order = TestOrder.EARLY_TESTS_THEN_BY_NAME
+        opts.order = TestOrder.DEFAULT
 
     if opts.numShards or opts.runShard:
         if not opts.numShards or not opts.runShard:
diff --git a/llvm/utils/lit/lit/discovery.py b/llvm/utils/lit/lit/discovery.py
index a185ae676d140c3f4efe63a5680c582be2baf877..22d6d8520283e02b72005daa7d857d19cafbfa42 100644
--- a/llvm/utils/lit/lit/discovery.py
+++ b/llvm/utils/lit/lit/discovery.py
@@ -281,6 +281,12 @@ def find_tests_for_inputs(lit_config, inputs, indirectlyRunCheck):
         if prev == len(tests):
             lit_config.warning('input %r contained no tests' % input)
 
+    # This data is no longer needed but keeping it around causes awful
+    # performance problems while the test suites run.
+    for k, suite in test_suite_cache.items():
+      if suite[0]:
+        suite[0].test_times = None
+
     # If there were any errors during test discovery, exit now.
     if lit_config.numErrors:
         sys.stderr.write('%d errors, exiting.\n' % lit_config.numErrors)
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 3543b287f25eae06884a4bb671f3b485862d03f8..51a05e8abaf427fb6be9cd6163e2a6fd76b3a6c9 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -5,8 +5,10 @@ def create_display(opts, tests, total_tests, workers):
     if opts.quiet:
         return NopDisplay()
 
-    of_total = (' of %d' % total_tests) if (tests != total_tests) else ''
-    header = '-- Testing: %d%s tests, %d workers --' % (tests, of_total, workers)
+    num_tests = len(tests)
+    of_total = (' of %d' % total_tests) if (num_tests != total_tests) else ''
+    header = '-- Testing: %d%s tests, %d workers --' % (
+        num_tests, of_total, workers)
 
     progress_bar = None
     if opts.succinct and opts.useProgressBar:
@@ -21,6 +23,44 @@ def create_display(opts, tests, total_tests, workers):
     return Display(opts, tests, header, progress_bar)
 
 
+class ProgressPredictor(object):
+    def __init__(self, tests):
+        self.completed = 0
+        self.time_elapsed = 0.0
+        self.predictable_tests_remaining = 0
+        self.predictable_time_remaining = 0.0
+        self.unpredictable_tests_remaining = 0
+
+        for test in tests:
+            if test.previous_elapsed:
+                self.predictable_tests_remaining += 1
+                self.predictable_time_remaining += test.previous_elapsed
+            else:
+                self.unpredictable_tests_remaining += 1
+
+    def update(self, test):
+        self.completed += 1
+        self.time_elapsed += test.result.elapsed
+
+        if test.previous_elapsed:
+            self.predictable_tests_remaining -= 1
+            self.predictable_time_remaining -= test.previous_elapsed
+        else:
+            self.unpredictable_tests_remaining -= 1
+
+        # NOTE: median would be more precise, but might be too slow.
+        average_test_time = (self.time_elapsed + self.predictable_time_remaining) / \
+            (self.completed + self.predictable_tests_remaining)
+        unpredictable_time_remaining = average_test_time * \
+            self.unpredictable_tests_remaining
+        total_time_remaining = self.predictable_time_remaining + unpredictable_time_remaining
+        total_time = self.time_elapsed + total_time_remaining
+
+        if total_time > 0:
+            return self.time_elapsed / total_time
+        return 0
+
+
 class NopDisplay(object):
     def print_header(self): pass
     def update(self, test): pass
@@ -30,8 +70,10 @@ class NopDisplay(object):
 class Display(object):
     def __init__(self, opts, tests, header, progress_bar):
         self.opts = opts
-        self.tests = tests
+        self.num_tests = len(tests)
         self.header = header
+        self.progress_predictor = ProgressPredictor(
+            tests) if progress_bar else None
         self.progress_bar = progress_bar
         self.completed = 0
 
@@ -55,7 +97,7 @@ class Display(object):
         if self.progress_bar:
             if test.isFailure():
                 self.progress_bar.barColor = 'RED'
-            percent = float(self.completed) / self.tests
+            percent = self.progress_predictor.update(test)
             self.progress_bar.update(percent, test.getFullName())
 
     def clear(self, interrupted):
@@ -66,7 +108,7 @@ class Display(object):
         # Show the test result line.
         test_name = test.getFullName()
         print('%s: %s (%d of %d)' % (test.result.code.name, test_name,
-                                     self.completed, self.tests))
+                                     self.completed, self.num_tests))
 
         # Show the test failure output, if requested.
         if (test.isFailure() and self.opts.showOutput) or \
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 3f265446be2e2ecd82fa6e23459aaf32111ed93e..47fe73388eaa76f8d623ac843c3b06cc299220f1 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -18,6 +18,7 @@ import lit.reports
 import lit.run
 import lit.Test
 import lit.util
+from lit.TestTimes import record_test_times
 
 
 def main(builtin_params={}):
@@ -105,6 +106,8 @@ def main(builtin_params={}):
     run_tests(selected_tests, lit_config, opts, len(discovered_tests))
     elapsed = time.time() - start
 
+    record_test_times(selected_tests, lit_config)
+
     if opts.time_tests:
         print_histogram(discovered_tests)
 
@@ -163,20 +166,12 @@ def print_discovered(tests, show_suites, show_tests):
 
 def determine_order(tests, order):
     from lit.cl_arguments import TestOrder
-    if order == TestOrder.EARLY_TESTS_THEN_BY_NAME:
-        tests.sort(key=lambda t: (not t.isEarlyTest(), t.getFullName()))
-    elif order == TestOrder.FAILING_FIRST:
-        def by_mtime(test):
-            return os.path.getmtime(test.getFilePath())
-        tests.sort(key=by_mtime, reverse=True)
-    elif order == TestOrder.RANDOM:
+    if order == TestOrder.RANDOM:
         import random
         random.shuffle(tests)
-
-
-def touch_file(test):
-    if test.isFailure():
-        os.utime(test.getFilePath(), None)
+    else:
+        assert order == TestOrder.DEFAULT, 'Unknown TestOrder value'
+        tests.sort(key=lambda t: (not t.previous_failure, -t.previous_elapsed, t.getFullName()))
 
 
 def filter_by_shard(tests, run, shards, lit_config):
@@ -210,15 +205,9 @@ def mark_excluded(discovered_tests, selected_tests):
 
 def run_tests(tests, lit_config, opts, discovered_tests):
     workers = min(len(tests), opts.workers)
-    display = lit.display.create_display(opts, len(tests), discovered_tests,
-                                         workers)
-
-    def progress_callback(test):
-        display.update(test)
-        if opts.order == lit.cl_arguments.TestOrder.FAILING_FIRST:
-            touch_file(test)
+    display = lit.display.create_display(opts, tests, discovered_tests, workers)
 
-    run = lit.run.Run(tests, lit_config, workers, progress_callback,
+    run = lit.run.Run(tests, lit_config, workers, display.update,
                       opts.max_failures, opts.timeout)
 
     display.print_header()
@@ -280,7 +269,7 @@ def print_results(tests, elapsed, opts):
         tests_by_code[test.result.code].append(test)
 
     for code in lit.Test.ResultCode.all_codes():
-        print_group(tests_by_code[code], code, opts.shown_codes)
+        print_group(sorted(tests_by_code[code], key=lambda t: t.getFullName()), code, opts.shown_codes)
 
     print_summary(tests_by_code, opts.quiet, elapsed)
 
diff --git a/llvm/utils/lit/tests/Inputs/reorder/.lit_test_times.txt b/llvm/utils/lit/tests/Inputs/reorder/.lit_test_times.txt
new file mode 100644
index 0000000000000000000000000000000000000000..73d55debc6052f4a464f7f354340c43a332d859e
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/reorder/.lit_test_times.txt
@@ -0,0 +1,4 @@
+42.0 not-executed.txt
+3.0 subdir/ccc.txt
+2.0 bbb.txt
+0.1 aaa.txt
diff --git a/llvm/utils/lit/tests/Inputs/early-tests/aaa.txt b/llvm/utils/lit/tests/Inputs/reorder/aaa.txt
similarity index 100%
rename from llvm/utils/lit/tests/Inputs/early-tests/aaa.txt
rename to llvm/utils/lit/tests/Inputs/reorder/aaa.txt
diff --git a/llvm/utils/lit/tests/Inputs/early-tests/bbb.txt b/llvm/utils/lit/tests/Inputs/reorder/bbb.txt
similarity index 100%
rename from llvm/utils/lit/tests/Inputs/early-tests/bbb.txt
rename to llvm/utils/lit/tests/Inputs/reorder/bbb.txt
diff --git a/llvm/utils/lit/tests/Inputs/early-tests/lit.cfg b/llvm/utils/lit/tests/Inputs/reorder/lit.cfg
similarity index 67%
rename from llvm/utils/lit/tests/Inputs/early-tests/lit.cfg
rename to llvm/utils/lit/tests/Inputs/reorder/lit.cfg
index db030510c249b57a372f40c58f596073f4986561..6320609a1e6c6d003c43c37406a9f44f29f878ca 100644
--- a/llvm/utils/lit/tests/Inputs/early-tests/lit.cfg
+++ b/llvm/utils/lit/tests/Inputs/reorder/lit.cfg
@@ -1,7 +1,6 @@
 import lit.formats
-config.name = 'early-tests'
+config.name = 'reorder'
 config.suffixes = ['.txt']
 config.test_format = lit.formats.ShTest()
 config.test_source_root = None
 config.test_exec_root = None
-config.early_tests = { "subdir/ccc.txt" }
diff --git a/llvm/utils/lit/tests/Inputs/early-tests/subdir/ccc.txt b/llvm/utils/lit/tests/Inputs/reorder/new-test.txt
similarity index 100%
rename from llvm/utils/lit/tests/Inputs/early-tests/subdir/ccc.txt
rename to llvm/utils/lit/tests/Inputs/reorder/new-test.txt
diff --git a/llvm/utils/lit/tests/Inputs/reorder/subdir/ccc.txt b/llvm/utils/lit/tests/Inputs/reorder/subdir/ccc.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b80b60b7a2794379dd3744c1e0a973be6ad5e604
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/reorder/subdir/ccc.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/custom-result-category.py b/llvm/utils/lit/tests/custom-result-category.py
index 8a374e393a9673a0a9a24e575435b38126abbe13..d4608b39a1c7d69b231f136fd6fdfa17b49ba2f5 100644
--- a/llvm/utils/lit/tests/custom-result-category.py
+++ b/llvm/utils/lit/tests/custom-result-category.py
@@ -1,6 +1,9 @@
 # UNSUPPORTED: system-windows
 # Test lit.main.add_result_category() extension API.
 
+# FIXME: this test depends on order of tests
+# RUN: rm -f %{inputs}/custom-result-category/.lit_test_times.txt
+
 # RUN: not %{lit} -j 1 %{inputs}/custom-result-category | FileCheck %s
 
 # CHECK: CUSTOM_PASS: custom-result-category :: test1.txt
diff --git a/llvm/utils/lit/tests/early-tests.py b/llvm/utils/lit/tests/early-tests.py
deleted file mode 100644
index b2ca9ac0a97d067510754957d54e23c2e8ccabda..0000000000000000000000000000000000000000
--- a/llvm/utils/lit/tests/early-tests.py
+++ /dev/null
@@ -1,9 +0,0 @@
-## Check that we can run tests early.
-
-# RUN: %{lit} -j1 %{inputs}/early-tests | FileCheck %s
-
-# CHECK:     -- Testing: 3 tests, 1 workers --
-# CHECK-NEXT: PASS: early-tests :: subdir/ccc.txt
-# CHECK-NEXT: PASS: early-tests :: aaa.txt
-# CHECK-NEXT: PASS: early-tests :: bbb.txt
-# CHECK:     Passed: 3
diff --git a/llvm/utils/lit/tests/googletest-format.py b/llvm/utils/lit/tests/googletest-format.py
index 8ea2f8320d86489cae73192b24343e7f0d01db67..09aa9dedabc3c473537fb80d10d72ec5e7ac18a0 100644
--- a/llvm/utils/lit/tests/googletest-format.py
+++ b/llvm/utils/lit/tests/googletest-format.py
@@ -1,5 +1,8 @@
 # Check the various features of the GoogleTest format.
-#
+
+# FIXME: this test depends on order of tests
+# RUN: rm -f %{inputs}/googletest-format/.lit_test_times.txt
+
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-format > %t.out
 # FIXME: Temporarily dump test output so we can debug failing tests on
 # buildbots.
diff --git a/llvm/utils/lit/tests/googletest-upstream-format.py b/llvm/utils/lit/tests/googletest-upstream-format.py
index a620877fc010bf5c78228663a346637b01a7ce99..517da4aa921e76b0e09bac01ee0be071cee75b0a 100644
--- a/llvm/utils/lit/tests/googletest-upstream-format.py
+++ b/llvm/utils/lit/tests/googletest-upstream-format.py
@@ -1,5 +1,8 @@
 # Check the various features of the GoogleTest format.
-#
+
+# FIXME: this test depends on order of tests
+# RUN: rm -f %{inputs}/googletest-upstream-format/.lit_test_times.txt
+
 # RUN: not %{lit} -j 1 -v %{inputs}/googletest-upstream-format > %t.out
 # RUN: FileCheck < %t.out %s
 #
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 135e29baa5a61febe2969f1d1e6bc3d3a2d24653..63c34516226d1a673cac5581357078e5a48e8b2e 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -6,10 +6,10 @@
 
 # END.
 
-# CHECK: FAIL: ignore-fail :: fail.txt
-# CHECK: UNRESOLVED: ignore-fail :: unresolved.txt
-# CHECK: XFAIL: ignore-fail :: xfail.txt
-# CHECK: XPASS: ignore-fail :: xpass.txt
+# CHECK-DAG: FAIL: ignore-fail :: fail.txt
+# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt
+# CHECK-DAG: XFAIL: ignore-fail :: xfail.txt
+# CHECK-DAG: XPASS: ignore-fail :: xpass.txt
 
 #      CHECK: Testing Time:
 # CHECK-NEXT:   Expectedly Failed : 1
diff --git a/llvm/utils/lit/tests/progress-bar.py b/llvm/utils/lit/tests/progress-bar.py
index ceeca68712bff1e4d46f317e7e8ea23059cc5719..e6dafffae879067a431a8987af11265c6a5c58f1 100644
--- a/llvm/utils/lit/tests/progress-bar.py
+++ b/llvm/utils/lit/tests/progress-bar.py
@@ -1,5 +1,8 @@
 # Check the simple progress bar.
-#
+
+# FIXME: this test depends on order of tests
+# RUN: rm -f %{inputs}/progress-bar/.lit_test_times.txt
+
 # RUN: not %{lit} -j 1 -s %{inputs}/progress-bar > %t.out
 # RUN: FileCheck < %t.out %s
 #
diff --git a/llvm/utils/lit/tests/reorder.py b/llvm/utils/lit/tests/reorder.py
new file mode 100644
index 0000000000000000000000000000000000000000..cac0a4eecaa67216ca731857a6de1035ed2ea5a8
--- /dev/null
+++ b/llvm/utils/lit/tests/reorder.py
@@ -0,0 +1,23 @@
+## Check that we can reorder test runs.
+
+# RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig
+# RUN: %{lit} -j1 %{inputs}/reorder > %t.out
+# RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.new
+# RUN: cp %{inputs}/reorder/.lit_test_times.txt.orig %{inputs}/reorder/.lit_test_times.txt
+# RUN: not diff %{inputs}/reorder/.lit_test_times.txt.new %{inputs}/reorder/.lit_test_times.txt.orig
+# RUN: FileCheck --check-prefix=TIMES --implicit-check-not= < %{inputs}/reorder/.lit_test_times.txt.new %s
+# RUN: FileCheck < %t.out %s
+# END.
+
+# TIMES: not-executed.txt
+# TIMES-NEXT: subdir/ccc.txt
+# TIMES-NEXT: bbb.txt
+# TIMES-NEXT: aaa.txt
+# TIMES-NEXT: new-test.txt
+
+# CHECK:     -- Testing: 4 tests, 1 workers --
+# CHECK-NEXT: PASS: reorder :: subdir/ccc.txt
+# CHECK-NEXT: PASS: reorder :: bbb.txt
+# CHECK-NEXT: PASS: reorder :: aaa.txt
+# CHECK-NEXT: PASS: reorder :: new-test.txt
+# CHECK:     Passed: 4
diff --git a/llvm/utils/lit/tests/shtest-env.py b/llvm/utils/lit/tests/shtest-env.py
index a4ddc0c9e31fb3fcdc7327cdef3ddb66601b6ada..ead217ace36168bb6402632061197f8300b55469 100644
--- a/llvm/utils/lit/tests/shtest-env.py
+++ b/llvm/utils/lit/tests/shtest-env.py
@@ -1,5 +1,8 @@
 # Check the env command
-#
+
+# FIXME: this test depends on order of tests
+# RUN: rm -f %{inputs}/shtest-env/.lit_test_times.txt
+
 # RUN: not %{lit} -j 1 -a -v %{inputs}/shtest-env \
 # RUN: | FileCheck -match-full-lines %s
 #
diff --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py
index 5c483972f2d0c01c647abca5e4a82855dc752a46..a15c0f9501740d0a7d4557e98104c5758e960191 100644
--- a/llvm/utils/lit/tests/shtest-format.py
+++ b/llvm/utils/lit/tests/shtest-format.py
@@ -1,5 +1,8 @@
 # Check the various features of the ShTest format.
-#
+
+# FIXME: this test depends on order of tests
+# RUN: rm -f %{inputs}/shtest-format/.lit_test_times.txt
+
 # RUN: rm -f %t.xml
 # RUN: not %{lit} -j 1 -v %{inputs}/shtest-format --xunit-xml-output %t.xml > %t.out
 # RUN: FileCheck < %t.out %s
diff --git a/llvm/utils/lit/tests/shtest-not.py b/llvm/utils/lit/tests/shtest-not.py
index d361ae8007fae1c86afcd12680be5b3f24105621..a391e7e60a3238ae00ec289e3145b16c946673c4 100644
--- a/llvm/utils/lit/tests/shtest-not.py
+++ b/llvm/utils/lit/tests/shtest-not.py
@@ -1,5 +1,8 @@
 # Check the not command
-#
+
+# FIXME: this test depends on order of tests
+# RUN: rm -f %{inputs}/shtest-not/.lit_test_times.txt
+
 # RUN: not %{lit} -j 1 -a -v %{inputs}/shtest-not \
 # RUN: | FileCheck -match-full-lines %s
 #
diff --git a/llvm/utils/lit/tests/shtest-run-at-line.py b/llvm/utils/lit/tests/shtest-run-at-line.py
index cd0e08137ee589b841df478f335fb9e16ae39ddb..bf47e6499cd2f93a82092b11709c38c0719f1e24 100644
--- a/llvm/utils/lit/tests/shtest-run-at-line.py
+++ b/llvm/utils/lit/tests/shtest-run-at-line.py
@@ -1,6 +1,9 @@
 # Check that -vv makes the line number of the failing RUN command clear.
 # (-v is actually sufficient in the case of the internal shell.)
-#
+
+# FIXME: this test depends on order of tests
+# RUN: rm -f %{inputs}/shtest-run-at-line/.lit_test_times.txt
+
 # RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out
 # RUN: FileCheck --input-file %t.out %s
 #
diff --git a/llvm/utils/lit/tests/shtest-shell.py b/llvm/utils/lit/tests/shtest-shell.py
index 4c247de15ddd11f8732f8368ac96626887a35ff2..13cf05f304fada07752a89288baf16a6db72321f 100644
--- a/llvm/utils/lit/tests/shtest-shell.py
+++ b/llvm/utils/lit/tests/shtest-shell.py
@@ -1,5 +1,8 @@
 # Check the internal shell handling component of the ShTest format.
-#
+
+# FIXME: this test depends on order of tests
+# RUN: rm -f %{inputs}/shtest-shell/.lit_test_times.txt
+
 # RUN: not %{lit} -j 1 -v %{inputs}/shtest-shell > %t.out
 # FIXME: Temporarily dump test output so we can debug failing tests on
 # buildbots.
@@ -8,6 +11,8 @@
 #
 # Test again in non-UTF shell to catch potential errors with python 2 seen
 # on stdout-encoding.txt
+# FIXME: lit's testing sets source_root == exec_root which complicates running lit more than once per test.
+# RUN: rm -f %{inputs}/shtest-shell/.lit_test_times.txt
 # RUN: env PYTHONIOENCODING=ascii not %{lit} -j 1 -a %{inputs}/shtest-shell > %t.ascii.out
 # FIXME: Temporarily dump test output so we can debug failing tests on
 # buildbots.
@@ -288,7 +293,7 @@
 
 # CHECK: FAIL: shtest-shell :: diff-r-error-0.txt
 # CHECK: *** TEST 'shtest-shell :: diff-r-error-0.txt' FAILED ***
-# CHECK: $ "diff" "-r" 
+# CHECK: $ "diff" "-r"
 # CHECK: # command output:
 # CHECK: Only in {{.*}}dir1: dir1unique
 # CHECK: Only in {{.*}}dir2: dir2unique
@@ -296,7 +301,7 @@
 
 # CHECK: FAIL: shtest-shell :: diff-r-error-1.txt
 # CHECK: *** TEST 'shtest-shell :: diff-r-error-1.txt' FAILED ***
-# CHECK: $ "diff" "-r" 
+# CHECK: $ "diff" "-r"
 # CHECK: # command output:
 # CHECK: *** {{.*}}dir1{{.*}}subdir{{.*}}f01
 # CHECK: --- {{.*}}dir2{{.*}}subdir{{.*}}f01
@@ -306,35 +311,35 @@
 
 # CHECK: FAIL: shtest-shell :: diff-r-error-2.txt
 # CHECK: *** TEST 'shtest-shell :: diff-r-error-2.txt' FAILED ***
-# CHECK: $ "diff" "-r" 
+# CHECK: $ "diff" "-r"
 # CHECK: # command output:
 # CHECK: Only in {{.*}}dir2: extrafile
 # CHECK: error: command failed with exit status: 1
 
 # CHECK: FAIL: shtest-shell :: diff-r-error-3.txt
 # CHECK: *** TEST 'shtest-shell :: diff-r-error-3.txt' FAILED ***
-# CHECK: $ "diff" "-r" 
+# CHECK: $ "diff" "-r"
 # CHECK: # command output:
 # CHECK: Only in {{.*}}dir1: extra_subdir
 # CHECK: error: command failed with exit status: 1
 
 # CHECK: FAIL: shtest-shell :: diff-r-error-4.txt
 # CHECK: *** TEST 'shtest-shell :: diff-r-error-4.txt' FAILED ***
-# CHECK: $ "diff" "-r" 
+# CHECK: $ "diff" "-r"
 # CHECK: # command output:
 # CHECK: File {{.*}}dir1{{.*}}extra_subdir is a directory while file {{.*}}dir2{{.*}}extra_subdir is a regular file
 # CHECK: error: command failed with exit status: 1
 
 # CHECK: FAIL: shtest-shell :: diff-r-error-5.txt
 # CHECK: *** TEST 'shtest-shell :: diff-r-error-5.txt' FAILED ***
-# CHECK: $ "diff" "-r" 
+# CHECK: $ "diff" "-r"
 # CHECK: # command output:
 # CHECK: Only in {{.*}}dir1: extra_subdir
 # CHECK: error: command failed with exit status: 1
 
 # CHECK: FAIL: shtest-shell :: diff-r-error-6.txt
 # CHECK: *** TEST 'shtest-shell :: diff-r-error-6.txt' FAILED ***
-# CHECK: $ "diff" "-r" 
+# CHECK: $ "diff" "-r"
 # CHECK: # command output:
 # CHECK: File {{.*}}dir1{{.*}}extra_file is a regular empty file while file {{.*}}dir2{{.*}}extra_file is a directory
 # CHECK: error: command failed with exit status: 1
diff --git a/llvm/utils/update_analyze_test_checks.py b/llvm/utils/update_analyze_test_checks.py
index 921a3bc085378140e58a8ec95a3bb2619ca73d4d..e2f78830edc2c9eb76381f6d51ada3f89c6da135 100755
--- a/llvm/utils/update_analyze_test_checks.py
+++ b/llvm/utils/update_analyze_test_checks.py
@@ -113,7 +113,8 @@ def main():
       flags = type('', (object,), {
             'verbose': args.verbose,
             'function_signature': False,
-            'check_attributes': False}),
+            'check_attributes': False,
+            'replace_function_regex': []}),
       scrubber_args = [])
 
     for prefixes, opt_args in prefix_list:
diff --git a/llvm/utils/update_cc_test_checks.py b/llvm/utils/update_cc_test_checks.py
index d084bc6d07950bfce7f4b9fd38e6cee4ff9bb558..d3af5308ac6a0c394671277ad76253422070914f 100755
--- a/llvm/utils/update_cc_test_checks.py
+++ b/llvm/utils/update_cc_test_checks.py
@@ -176,7 +176,7 @@ def config():
   return args, parser
 
 
-def get_function_body(builder, args, filename, clang_args, extra_commands, 
+def get_function_body(builder, args, filename, clang_args, extra_commands,
                       prefixes):
   # TODO Clean up duplication of asm/common build_function_body_dictionary
   # Invoke external tool and extract function bodies.
@@ -221,6 +221,13 @@ def main():
     # Build a list of clang command lines and check prefixes from RUN lines.
     run_list = []
     line2spell_and_mangled_list = collections.defaultdict(list)
+
+    subs = {
+      '%s' : ti.path,
+      '%t' : tempfile.NamedTemporaryFile().name,
+      '%S' : os.getcwd(),
+    }
+
     for l in ti.run_lines:
       commands = [cmd.strip() for cmd in l.split('|')]
 
@@ -234,15 +241,18 @@ def main():
       # Execute non-clang runline.
       if exec_args[0] not in SUBST:
         print('NOTE: Executing non-clang RUN line: ' + l, file=sys.stderr)
-        # Replace %s by `filename`.
-        exec_args = [i.replace('%s', ti.path) if '%s' in i else i for i in exec_args]
+        # Do lit-like substitutions.
+        for s in subs:
+          exec_args = [i.replace(s, subs[s]) if s in i else i for i in exec_args]
         exec_run_line(exec_args)
         continue
-      # This is a clang runline, apply %clang substitution rule, replace %s by `filename`,
+      # This is a clang runline, apply %clang substitution rule, do lit-like substitutions,
       # and append args.clang_args
       clang_args = exec_args
       clang_args[0:1] = SUBST[clang_args[0]]
-      clang_args = [i.replace('%s', ti.path) if '%s' in i else i for i in clang_args] + ti.args.clang_args
+      for s in subs:
+        clang_args = [i.replace(s, subs[s]) if s in i else i for i in clang_args]
+      clang_args += ti.args.clang_args
 
       # Extract -check-prefix in FileCheck args
       filecheck_cmd = commands[-1]
@@ -271,7 +281,7 @@ def main():
       common.debug('Extracted clang cmd: clang {}'.format(clang_args))
       common.debug('Extracted FileCheck prefixes: {}'.format(prefixes))
 
-      get_function_body(builder, ti.args, ti.path, clang_args, extra_commands, 
+      get_function_body(builder, ti.args, ti.path, clang_args, extra_commands,
                         prefixes)
 
       # Invoke clang -Xclang -ast-dump=json to get mapping from start lines to
@@ -315,7 +325,7 @@ def main():
                              prefixes,
                              func_dict, func)
 
-      common.add_checks_at_end(output_lines, run_list, builder.func_order(), 
+      common.add_checks_at_end(output_lines, run_list, builder.func_order(),
                                '//', lambda my_output_lines, prefixes, func:
                                check_generator(my_output_lines,
                                                prefixes, func))
diff --git a/llvm/utils/vim/syntax/llvm.vim b/llvm/utils/vim/syntax/llvm.vim
index 48d3daf581ca3c0c175c27d8782aea24269b1d0c..70918ffd160c567e9f73b03a7355b07264bb89e4 100644
--- a/llvm/utils/vim/syntax/llvm.vim
+++ b/llvm/utils/vim/syntax/llvm.vim
@@ -211,7 +211,7 @@ syn match   llvmNumber /-\?\<\d\+\>/
 syn match   llvmFloat  /-\?\<\d\+\.\d*\(e[+-]\d\+\)\?\>/
 syn match   llvmFloat  /\<0x\x\+\>/
 syn keyword llvmBoolean true false
-syn keyword llvmConstant zeroinitializer undef null none
+syn keyword llvmConstant zeroinitializer undef null none poison
 syn match   llvmComment /;.*$/
 syn region  llvmString start=/"/ skip=/\\"/ end=/"/
 syn match   llvmLabel /[-a-zA-Z$._][-a-zA-Z$._0-9]*:/
diff --git a/mlir/docs/Bindings/Python.md b/mlir/docs/Bindings/Python.md
index e5e8e6d77c30f121c76c758e340e3c2b9153d224..902c03916a8ea5b30e2dcca150a1c4cab6e3fe38 100644
--- a/mlir/docs/Bindings/Python.md
+++ b/mlir/docs/Bindings/Python.md
@@ -449,7 +449,7 @@ defaults on `OpView`):
   variadics. Used by `OpView._ods_build_default` to decode operand and result
   lists that contain lists.
 
-#### Builders
+#### Default Builder
 
 Presently, only a single, default builder is mapped to the `__init__` method.
 The intent is that this `__init__` method represents the *most specific* of
@@ -475,3 +475,90 @@ construction via a (nested in the case of variadic) sequence of `results` and
 `operands`. This can be used to get some default construction semantics for
 operations that are otherwise unsupported in Python, at the expense of having
 a very generic signature.
+
+#### Extending Generated Op Classes
+
+Note that this is a rather complex mechanism and this section errs on the side
+of explicitness. Users are encouraged to find an example and duplicate it if
+they don't feel the need to understand the subtlety. The `builtin` dialect
+provides some relatively simple examples.
+
+As mentioned above, the build system generates Python sources like
+`_{DIALECT_NAMESPACE}_ops_gen.py` for each dialect with Python bindings. It
+is often desirable to to use these generated classes as a starting point for
+further customization, so an extension mechanism is provided to make this
+easy (you are always free to do ad-hoc patching in your `{DIALECT_NAMESPACE}.py`
+file but we prefer a more standard mechanism that is applied uniformly).
+
+To provide extensions, add a `_{DIALECT_NAMESPACE}_ops_ext.py` file to the
+`dialects` module (i.e. adjacent to your `{DIALECT_NAMESPACE}.py` top-level
+and the `*_ops_gen.py` file). Using the `builtin` dialect and `FuncOp` as an
+example, the generated code will include an import like this:
+
+```python
+try:
+  from . import _builtin_ops_ext as _ods_ext_module
+except ImportError:
+  _ods_ext_module = None
+```
+
+Then for each generated concrete `OpView` subclass, it will apply a decorator
+like:
+
+```python
+@_ods_cext.register_operation(_Dialect)
+@_ods_extend_opview_class(_ods_ext_module)
+class FuncOp(_ods_ir.OpView):
+```
+
+See the `_ods_common.py` `extend_opview_class` function for details of the
+mechanism. At a high level:
+
+* If the extension module exists, locate an extension class for the op (in
+  this example, `FuncOp`):
+  * First by looking for an attribute with the exact name in the extension
+    module.
+  * Falling back to calling a `select_opview_mixin(parent_opview_cls)`
+    function defined in the extension module.
+* If a mixin class is found, a new subclass is dynamically created that multiply
+  inherits from `({_builtin_ops_ext.FuncOp}, _builtin_ops_gen.FuncOp)`.
+
+The mixin class should not inherit from anything (i.e. directly extends
+`object` only). The facility is typically used to define custom `__init__`
+methods, properties, instance methods and static methods. Due to the
+inheritance ordering, the mixin class can act as though it extends the
+generated `OpView` subclass in most contexts (i.e.
+`issubclass(_builtin_ops_ext.FuncOp, OpView)` will return `False` but usage
+generally allows you treat it as duck typed as an `OpView`).
+
+There are a couple of recommendations, given how the class hierarchy is
+defined:
+
+* For static methods that need to instantiate the actual "leaf" op (which
+  is dynamically generated and would result in circular dependencies to try
+  to reference by name), prefer to use `@classmethod` and the concrete
+  subclass will be provided as your first `cls` argument. See
+  `_builtin_ops_ext.FuncOp.from_py_func` as an example.
+* If seeking to replace the generated `__init__` method entirely, you may
+  actually want to invoke the super-super-class `mlir.ir.OpView` constructor
+  directly, as it takes an `mlir.ir.Operation`, which is likely what you
+  are constructing (i.e. the generated `__init__` method likely adds more
+  API constraints than you want to expose in a custom builder).
+
+A pattern that comes up frequently is wanting to provide a sugared `__init__`
+method which has optional or type-polymorphism/implicit conversions but to
+otherwise want to invoke the default op building logic. For such cases,
+it is recommended to use an idiom such as:
+
+```python
+  def __init__(self, sugar, spice, *, loc=None, ip=None):
+    ... massage into result_type, operands, attributes ...
+    OpView.__init__(self, self.build_generic(
+        results=[result_type],
+        operands=operands,
+        attributes=attributes,
+        loc=loc,
+        ip=ip))
+```
+
+Refer to the documentation for `build_generic` for more information.
diff --git a/mlir/docs/BufferDeallocationInternals.md b/mlir/docs/BufferDeallocationInternals.md
index 9626acad8825949131a468678d6a8e29209d48c3..7c731066d31e8428998903514c597d040a4ca175 100644
--- a/mlir/docs/BufferDeallocationInternals.md
+++ b/mlir/docs/BufferDeallocationInternals.md
@@ -25,6 +25,38 @@ should implement the `ReturnLike` trait to represent logical “value returns”
 Example dialects that are fully compatible are the “std” and “scf” dialects
 with respect to all implemented interfaces.
 
+During Bufferization, we convert immutable value types (tensors) to mutable
+types (memref). This conversion is done in several steps and in all of these
+steps the IR has to fulfill SSA like properties. The usage of memref has
+to be in the following consecutive order: allocation, write-buffer, read-
+buffer.
+In this case, there are only buffer reads allowed after the initial full
+buffer write is done. In particular, there must be no partial write to a
+buffer after the initial write has been finished. However, partial writes in
+the initializing is allowed (fill buffer step by step in a loop e.g.). This
+means, all buffer writes needs to dominate all buffer reads.
+
+Example for breaking the invariant:
+
+```mlir
+func @condBranch(%arg0: i1, %arg1: memref<2xf32>) {
+  %0 = memref.alloc() : memref<2xf32>
+  cond_br %arg0, ^bb1, ^bb2
+^bb1:
+  br ^bb3()
+^bb2:
+  partial_write(%0, %0)
+  br ^bb3()
+^bb3():
+  test.copy(%0, %arg1) : (memref<2xf32>, memref<2xf32>) -> ()
+  return
+}
+```
+
+The maintenance of the SSA like properties is only needed in the bufferization
+process. Afterwards, for example in optimization processes, the property is no
+longer needed.
+
 ## Detection of Buffer Allocations
 
 The first step of the BufferDeallocation transformation is to identify
@@ -101,11 +133,11 @@ func @condBranch(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 ^bb1:
   br ^bb3(%arg1 : memref<2xf32>)
 ^bb2:
-  %0 = alloc() : memref<2xf32>  // aliases: %1
+  %0 = memref.alloc() : memref<2xf32>  // aliases: %1
   use(%0)
   br ^bb3(%0 : memref<2xf32>)
 ^bb3(%1: memref<2xf32>):  // %1 could be %0 or %arg1
-  "linalg.copy"(%1, %arg2) : (memref<2xf32>, memref<2xf32>) -> ()
+  test.copy(%1, %arg2) : (memref<2xf32>, memref<2xf32>) -> ()
   return
 }
 ```
@@ -117,7 +149,7 @@ of code:
 
 ```mlir
 func @condBranch(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
-  %0 = alloc() : memref<2xf32>  // moved to bb0
+  %0 = memref.alloc() : memref<2xf32>  // moved to bb0
   cond_br %arg0, ^bb1, ^bb2
 ^bb1:
   br ^bb3(%arg1 : memref<2xf32>)
@@ -125,7 +157,7 @@ func @condBranch(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
    use(%0)
    br ^bb3(%0 : memref<2xf32>)
 ^bb3(%1: memref<2xf32>):
-  "linalg.copy"(%1, %arg2) : (memref<2xf32>, memref<2xf32>) -> ()
+  test.copy(%1, %arg2) : (memref<2xf32>, memref<2xf32>) -> ()
   return
 }
 ```
@@ -147,17 +179,17 @@ func @condBranchDynamicType(
 ^bb1:
   br ^bb3(%arg1 : memref<?xf32>)
 ^bb2(%0: index):
-  %1 = alloc(%0) : memref<?xf32>   // cannot be moved upwards to the data
+  %1 = memref.alloc(%0) : memref<?xf32>   // cannot be moved upwards to the data
                                    // dependency to %0
   use(%1)
   br ^bb3(%1 : memref<?xf32>)
 ^bb3(%2: memref<?xf32>):
-  "linalg.copy"(%2, %arg2) : (memref<?xf32>, memref<?xf32>) -> ()
+  test.copy(%2, %arg2) : (memref<?xf32>, memref<?xf32>) -> ()
   return
 }
 ```
 
-## Introduction of Copies
+## Introduction of Clones
 
 In order to guarantee that all allocated buffers are freed properly, we have to
 pay attention to the control flow and all potential aliases a buffer allocation
@@ -168,10 +200,10 @@ allocations have already been placed:
 
 ```mlir
 func @branch(%arg0: i1) {
-  %0 = alloc() : memref<2xf32>  // aliases: %2
+  %0 = memref.alloc() : memref<2xf32>  // aliases: %2
   cond_br %arg0, ^bb1, ^bb2
 ^bb1:
-  %1 = alloc() : memref<2xf32>  // resides here for demonstration purposes
+  %1 = memref.alloc() : memref<2xf32>  // resides here for demonstration purposes
                                 // aliases: %2
   br ^bb3(%1 : memref<2xf32>)
 ^bb2:
@@ -200,88 +232,31 @@ result:
 
 ```mlir
 func @branch(%arg0: i1) {
-  %0 = alloc() : memref<2xf32>
+  %0 = memref.alloc() : memref<2xf32>
   cond_br %arg0, ^bb1, ^bb2
 ^bb1:
-  %1 = alloc() : memref<2xf32>
-  %3 = alloc() : memref<2xf32>  // temp copy for %1
-  "linalg.copy"(%1, %3) : (memref<2xf32>, memref<2xf32>) -> ()
-  dealloc %1 : memref<2xf32> // %1 can be safely freed here
+  %1 = memref.alloc() : memref<2xf32>
+  %3 = memref.clone %1 : (memref<2xf32>) -> (memref<2xf32>)
+  memref.dealloc %1 : memref<2xf32> // %1 can be safely freed here
   br ^bb3(%3 : memref<2xf32>)
 ^bb2:
   use(%0)
-  %4 = alloc() : memref<2xf32>  // temp copy for %0
-  "linalg.copy"(%0, %4) : (memref<2xf32>, memref<2xf32>) -> ()
+  %4 = memref.clone %0 : (memref<2xf32>) -> (memref<2xf32>)
   br ^bb3(%4 : memref<2xf32>)
 ^bb3(%2: memref<2xf32>):
   …
-  dealloc %2 : memref<2xf32> // free temp buffer %2
-  dealloc %0 : memref<2xf32> // %0 can be safely freed here
+  memref.dealloc %2 : memref<2xf32> // free temp buffer %2
+  memref.dealloc %0 : memref<2xf32> // %0 can be safely freed here
   return
 }
 ```
 
 Note that a temporary buffer for %2 was introduced to free all allocations
 properly. Note further that the unnecessary allocation of %3 can be easily
-removed using one of the post-pass transformations.
-
-Reconsider the previously introduced sample demonstrating dynamically shaped
-types:
-
-```mlir
-func @condBranchDynamicType(
-  %arg0: i1,
-  %arg1: memref<?xf32>,
-  %arg2: memref<?xf32>,
-  %arg3: index) {
-  cond_br %arg0, ^bb1, ^bb2(%arg3: index)
-^bb1:
-  br ^bb3(%arg1 : memref<?xf32>)
-^bb2(%0: index):
-  %1 = alloc(%0) : memref<?xf32>  // aliases: %2
-  use(%1)
-  br ^bb3(%1 : memref<?xf32>)
-^bb3(%2: memref<?xf32>):
-  "linalg.copy"(%2, %arg2) : (memref<?xf32>, memref<?xf32>) -> ()
-  return
-}
-```
-
-In the presence of DSTs, we have to parameterize the allocations with
-additional dimension information of the source buffers, we want to copy from.
-BufferDeallocation automatically introduces all required operations to extract
-dimension specifications and wires them with the associated allocations:
+removed using one of the post-pass transformations or the canonicalization
+pass.
 
-```mlir
-func @condBranchDynamicType(
-  %arg0: i1,
-  %arg1: memref<?xf32>,
-  %arg2: memref<?xf32>,
-  %arg3: index) {
-  cond_br %arg0, ^bb1, ^bb2(%arg3 : index)
-^bb1:
-  %c0 = constant 0 : index
-  %0 = dim %arg1, %c0 : memref<?xf32>   // dimension operation to parameterize
-                                        // the following temp allocation
-  %1 = alloc(%0) : memref<?xf32>
-  "linalg.copy"(%arg1, %1) : (memref<?xf32>, memref<?xf32>) -> ()
-  br ^bb3(%1 : memref<?xf32>)
-^bb2(%2: index):
-  %3 = alloc(%2) : memref<?xf32>
-  use(%3)
-  %c0_0 = constant 0 : index
-  %4 = dim %3, %c0_0 : memref<?xf32>  // dimension operation to parameterize
-                                      // the following temp allocation
-  %5 = alloc(%4) : memref<?xf32>
-  "linalg.copy"(%3, %5) : (memref<?xf32>, memref<?xf32>) -> ()
-  dealloc %3 : memref<?xf32>  // %3 can be safely freed here
-  br ^bb3(%5 : memref<?xf32>)
-^bb3(%6: memref<?xf32>):
-  "linalg.copy"(%6, %arg2) : (memref<?xf32>, memref<?xf32>) -> ()
-  dealloc %6 : memref<?xf32>  // %6 can be safely freed here
-  return
-}
-```
+The presented example also works with dynamically shaped types.
 
 BufferDeallocation performs a fix-point iteration taking all aliases of all
 tracked allocations into account. We initialize the general iteration process
@@ -303,7 +278,7 @@ func @condBranchDynamicTypeNested(
 ^bb1:
   br ^bb6(%arg1 : memref<?xf32>)
 ^bb2(%0: index):
-  %1 = alloc(%0) : memref<?xf32>   // cannot be moved upwards due to the data
+  %1 = memref.alloc(%0) : memref<?xf32>   // cannot be moved upwards due to the data
                                    // dependency to %0
                                    // aliases: %2, %3, %4
   use(%1)
@@ -317,7 +292,7 @@ func @condBranchDynamicTypeNested(
 ^bb6(%3: memref<?xf32>):  // crit. alias of %arg1 and %2 (in other words %1)
   br ^bb7(%3 : memref<?xf32>)
 ^bb7(%4: memref<?xf32>):  // non-crit. alias of %3, since %3 dominates %4
-  "linalg.copy"(%4, %arg2) : (memref<?xf32>, memref<?xf32>) -> ()
+  test.copy(%4, %arg2) : (memref<?xf32>, memref<?xf32>) -> ()
   return
 }
 ```
@@ -334,13 +309,11 @@ func @condBranchDynamicTypeNested(
   %arg3: index) {
   cond_br %arg0, ^bb1, ^bb2(%arg3 : index)
 ^bb1:
-  %c0 = constant 0 : index
-  %d0 = dim %arg1, %c0 : memref<?xf32>
-  %5 = alloc(%d0) : memref<?xf32>  // temp buffer required due to alias %3
-  "linalg.copy"(%arg1, %5) : (memref<?xf32>, memref<?xf32>) -> ()
+  // temp buffer required due to alias %3
+  %5 = memref.clone %arg1 : (memref<?xf32>) -> (memref<?xf32>)
   br ^bb6(%5 : memref<?xf32>)
 ^bb2(%0: index):
-  %1 = alloc(%0) : memref<?xf32>
+  %1 = memref.alloc(%0) : memref<?xf32>
   use(%1)
   cond_br %arg0, ^bb3, ^bb4
 ^bb3:
@@ -348,17 +321,14 @@ func @condBranchDynamicTypeNested(
 ^bb4:
   br ^bb5(%1 : memref<?xf32>)
 ^bb5(%2: memref<?xf32>):
-  %c0_0 = constant 0 : index
-  %d1 = dim %2, %c0_0 : memref<?xf32>
-  %6 = alloc(%d1) : memref<?xf32>  // temp buffer required due to alias %3
-  "linalg.copy"(%1, %6) : (memref<?xf32>, memref<?xf32>) -> ()
-  dealloc %1 : memref<?xf32>
+  %6 = memref.clone %1 : (memref<?xf32>) -> (memref<?xf32>)
+  memref.dealloc %1 : memref<?xf32>
   br ^bb6(%6 : memref<?xf32>)
 ^bb6(%3: memref<?xf32>):
   br ^bb7(%3 : memref<?xf32>)
 ^bb7(%4: memref<?xf32>):
-  "linalg.copy"(%4, %arg2) : (memref<?xf32>, memref<?xf32>) -> ()
-  dealloc %3 : memref<?xf32>  // free %3, since %4 is a non-crit. alias of %3
+  test.copy(%4, %arg2) : (memref<?xf32>, memref<?xf32>) -> ()
+  memref.dealloc %3 : memref<?xf32>  // free %3, since %4 is a non-crit. alias of %3
   return
 }
 ```
@@ -367,7 +337,7 @@ Since %3 is a critical alias, BufferDeallocation introduces an additional
 temporary copy in all predecessor blocks. %3 has an additional (non-critical)
 alias %4 that extends the live range until the end of bb7. Therefore, we can
 free %3 after its last use, while taking all aliases into account. Note that %4
- does not need to be freed, since we did not introduce a copy for it.
+does not need to be freed, since we did not introduce a copy for it.
 
 The actual introduction of buffer copies is done after the fix-point iteration
 has been terminated and all critical aliases have been detected. A critical
@@ -413,7 +383,7 @@ infer the high-level control flow:
 func @inner_region_control_flow(
   %arg0 : index,
   %arg1 : index) -> memref<?x?xf32> {
-  %0 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %0 = memref.alloc(%arg0, %arg0) : memref<?x?xf32>
   %1 = custom.region_if %0 : memref<?x?xf32> -> (memref<?x?xf32>)
    then(%arg2 : memref<?x?xf32>) {  // aliases: %arg4, %1
     custom.region_if_yield %arg2 : memref<?x?xf32>
@@ -436,11 +406,11 @@ operation to determine the value of %2 at runtime which creates an alias:
 ```mlir
 func @nested_region_control_flow(%arg0 : index, %arg1 : index) -> memref<?x?xf32> {
   %0 = cmpi "eq", %arg0, %arg1 : index
-  %1 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %1 = memref.alloc(%arg0, %arg0) : memref<?x?xf32>
   %2 = scf.if %0 -> (memref<?x?xf32>) {
     scf.yield %1 : memref<?x?xf32>   // %2 will be an alias of %1
   } else {
-    %3 = alloc(%arg0, %arg1) : memref<?x?xf32>  // nested allocation in a div.
+    %3 = memref.alloc(%arg0, %arg1) : memref<?x?xf32>  // nested allocation in a div.
                                                 // branch
     use(%3)
     scf.yield %1 : memref<?x?xf32>   // %2 will be an alias of %1
@@ -457,13 +427,13 @@ alias of %1 which does not need to be tracked.
 ```mlir
 func @nested_region_control_flow(%arg0: index, %arg1: index) -> memref<?x?xf32> {
     %0 = cmpi "eq", %arg0, %arg1 : index
-    %1 = alloc(%arg0, %arg0) : memref<?x?xf32>
+    %1 = memref.alloc(%arg0, %arg0) : memref<?x?xf32>
     %2 = scf.if %0 -> (memref<?x?xf32>) {
       scf.yield %1 : memref<?x?xf32>
     } else {
-      %3 = alloc(%arg0, %arg1) : memref<?x?xf32>
+      %3 = memref.alloc(%arg0, %arg1) : memref<?x?xf32>
       use(%3)
-      dealloc %3 : memref<?x?xf32>  // %3 can be safely freed here
+      memref.dealloc %3 : memref<?x?xf32>  // %3 can be safely freed here
       scf.yield %1 : memref<?x?xf32>
     }
     return %2 : memref<?x?xf32>
@@ -482,12 +452,12 @@ above that uses a nested allocation:
 func @inner_region_control_flow_div(
   %arg0 : index,
   %arg1 : index) -> memref<?x?xf32> {
-  %0 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %0 = memref.alloc(%arg0, %arg0) : memref<?x?xf32>
   %1 = custom.region_if %0 : memref<?x?xf32> -> (memref<?x?xf32>)
    then(%arg2 : memref<?x?xf32>) {  // aliases: %arg4, %1
     custom.region_if_yield %arg2 : memref<?x?xf32>
    } else(%arg3 : memref<?x?xf32>) {
-    %2 = alloc(%arg0, %arg1) : memref<?x?xf32>  // aliases: %arg4, %1
+    %2 = memref.alloc(%arg0, %arg1) : memref<?x?xf32>  // aliases: %arg4, %1
     custom.region_if_yield %2 : memref<?x?xf32>
    } join(%arg4 : memref<?x?xf32>) {  // aliases: %1
     custom.region_if_yield %arg4 : memref<?x?xf32>
@@ -505,40 +475,22 @@ This causes BufferDeallocation to introduce additional copies:
 func @inner_region_control_flow_div(
   %arg0 : index,
   %arg1 : index) -> memref<?x?xf32> {
-  %0 = alloc(%arg0, %arg0) : memref<?x?xf32>
+  %0 = memref.alloc(%arg0, %arg0) : memref<?x?xf32>
   %1 = custom.region_if %0 : memref<?x?xf32> -> (memref<?x?xf32>)
    then(%arg2 : memref<?x?xf32>) {
-    %c0 = constant 0 : index  // determine dimension extents for temp allocation
-    %2 = dim %arg2, %c0 : memref<?x?xf32>
-    %c1 = constant 1 : index
-    %3 = dim %arg2, %c1 : memref<?x?xf32>
-    %4 = alloc(%2, %3) : memref<?x?xf32>  // temp buffer required due to critic.
-                                          // alias %arg4
-    linalg.copy(%arg2, %4) : memref<?x?xf32>, memref<?x?xf32>
+    %4 = memref.clone %arg2 : (memref<?x?xf32>) -> (memref<?x?xf32>)
     custom.region_if_yield %4 : memref<?x?xf32>
    } else(%arg3 : memref<?x?xf32>) {
-    %2 = alloc(%arg0, %arg1) : memref<?x?xf32>
-    %c0 = constant 0 : index  // determine dimension extents for temp allocation
-    %3 = dim %2, %c0 : memref<?x?xf32>
-    %c1 = constant 1 : index
-    %4 = dim %2, %c1 : memref<?x?xf32>
-    %5 = alloc(%3, %4) : memref<?x?xf32>  // temp buffer required due to critic.
-                                          // alias %arg4
-    linalg.copy(%2, %5) : memref<?x?xf32>, memref<?x?xf32>
-    dealloc %2 : memref<?x?xf32>
+    %2 = memref.alloc(%arg0, %arg1) : memref<?x?xf32>
+    %5 = memref.clone %2 : (memref<?x?xf32>) -> (memref<?x?xf32>)
+    memref.dealloc %2 : memref<?x?xf32>
     custom.region_if_yield %5 : memref<?x?xf32>
    } join(%arg4: memref<?x?xf32>) {
-    %c0 = constant 0 : index  // determine dimension extents for temp allocation
-    %2 = dim %arg4, %c0 : memref<?x?xf32>
-    %c1 = constant 1 : index
-    %3 = dim %arg4, %c1 : memref<?x?xf32>
-    %4 = alloc(%2, %3) : memref<?x?xf32>  // this allocation will be removed by
-                                          // applying the copy removal pass
-    linalg.copy(%arg4, %4) : memref<?x?xf32>, memref<?x?xf32>
-    dealloc %arg4 : memref<?x?xf32>
+    %4 = memref.clone %arg4 : (memref<?x?xf32>) -> (memref<?x?xf32>)
+    memref.dealloc %arg4 : memref<?x?xf32>
     custom.region_if_yield %4 : memref<?x?xf32>
    }
-  dealloc %0 : memref<?x?xf32>  // %0 can be safely freed here
+  memref.dealloc %0 : memref<?x?xf32>  // %0 can be safely freed here
   return %1 : memref<?x?xf32>
 }
 ```
@@ -568,7 +520,7 @@ func @loop_nested_if(
     iter_args(%iterBuf = %buf) -> memref<2xf32> {
     %1 = cmpi "eq", %i, %ub : index
     %2 = scf.if %1 -> (memref<2xf32>) {
-      %3 = alloc() : memref<2xf32>  // makes %2 a critical alias due to a
+      %3 = memref.alloc() : memref<2xf32>  // makes %2 a critical alias due to a
                                     // divergent allocation
       use(%3)
       scf.yield %3 : memref<2xf32>
@@ -577,7 +529,7 @@ func @loop_nested_if(
     }
     scf.yield %2 : memref<2xf32>
   }
-  "linalg.copy"(%0, %res) : (memref<2xf32>, memref<2xf32>) -> ()
+  test.copy(%0, %res) : (memref<2xf32>, memref<2xf32>) -> ()
   return
 }
 ```
@@ -602,31 +554,27 @@ func @loop_nested_if(
   %step: index,
   %buf: memref<2xf32>,
   %res: memref<2xf32>) {
-  %4 = alloc() : memref<2xf32>
-  "linalg.copy"(%buf, %4) : (memref<2xf32>, memref<2xf32>) -> ()
+  %4 = memref.clone %buf : (memref<2xf32>) -> (memref<2xf32>)
   %0 = scf.for %i = %lb to %ub step %step
     iter_args(%iterBuf = %4) -> memref<2xf32> {
     %1 = cmpi "eq", %i, %ub : index
     %2 = scf.if %1 -> (memref<2xf32>) {
-      %3 = alloc() : memref<2xf32> // makes %2 a critical alias
+      %3 = memref.alloc() : memref<2xf32> // makes %2 a critical alias
       use(%3)
-      %5 = alloc() : memref<2xf32> // temp copy due to crit. alias %2
-      "linalg.copy"(%3, %5) : memref<2xf32>, memref<2xf32>
-      dealloc %3 : memref<2xf32>
+      %5 = memref.clone %3 : (memref<2xf32>) -> (memref<2xf32>)
+      memref.dealloc %3 : memref<2xf32>
       scf.yield %5 : memref<2xf32>
     } else {
-      %6 = alloc() : memref<2xf32> // temp copy due to crit. alias %2
-      "linalg.copy"(%iterBuf, %6) : memref<2xf32>, memref<2xf32>
+      %6 = memref.clone %iterBuf : (memref<2xf32>) -> (memref<2xf32>)
       scf.yield %6 : memref<2xf32>
     }
-    %7 = alloc() : memref<2xf32> // temp copy due to crit. alias %iterBuf
-    "linalg.copy"(%2, %7) : memref<2xf32>, memref<2xf32>
-    dealloc %2 : memref<2xf32>
-    dealloc %iterBuf : memref<2xf32> // free backedge iteration variable
+    %7 = memref.clone %2 : (memref<2xf32>) -> (memref<2xf32>)
+    memref.dealloc %2 : memref<2xf32>
+    memref.dealloc %iterBuf : memref<2xf32> // free backedge iteration variable
     scf.yield %7 : memref<2xf32>
   }
-  "linalg.copy"(%0, %res) : (memref<2xf32>, memref<2xf32>) -> ()
-  dealloc %0 : memref<2xf32> // free temp copy %0
+  test.copy(%0, %res) : (memref<2xf32>, memref<2xf32>) -> ()
+  memref.dealloc %0 : memref<2xf32> // free temp copy %0
   return
 }
 ```
@@ -652,46 +600,37 @@ deallocations.
 
 In order to limit the complexity of the BufferDeallocation transformation, some
 tiny code-polishing/optimization transformations are not applied on-the-fly
-during placement. Currently, there is only the CopyRemoval transformation to
-remove unnecessary copy and allocation operations.
+during placement. Currently, a canonicalization pattern is added to the clone
+operation to reduce the appearance of unnecessary clones.
 
 Note: further transformations might be added to the post-pass phase in the
 future.
 
-## CopyRemoval Pass
-
-A common pattern that arises during placement is the introduction of
-unnecessary temporary copies that are used instead of the original source
-buffer. For this reason, there is a post-pass transformation that removes these
-allocations and copies via `-copy-removal`. This pass, besides removing
-unnecessary copy operations, will also remove the dead allocations and their
-corresponding deallocation operations. The CopyRemoval pass can currently be
-applied to operations that implement the `CopyOpInterface` in any of these two
-situations which are
+## Clone Canonicalization
 
-* reusing the source buffer of the copy operation.
-* reusing the target buffer of the copy operation.
+During placement of clones it may happen, that unnecessary clones are inserted.
+If these clones appear with their corresponding dealloc operation within the
+same block, we can use the canonicalizer to remove these unnecessary operations.
+Note, that this step needs to take place after the insertion of clones and
+deallocs in the buffer deallocation step. The canonicalization inludes both,
+the newly created target value from the clone operation and the source
+operation.
 
-## Reusing the Source Buffer of the Copy Operation
+## Canonicalization of the Source Buffer of the Clone Operation
 
-In this case, the source of the copy operation can be used instead of target.
-The unused allocation and deallocation operations that are defined for this
-copy operation are also removed. Here is a working example generated by the
-BufferDeallocation pass that allocates a buffer with dynamic size. A deeper
+In this case, the source of the clone operation can be used instead of its
+target. The unused allocation and deallocation operations that are defined for
+this clone operation are also removed. Here is a working example generated by
+the BufferDeallocation pass that allocates a buffer with dynamic size. A deeper
 analysis of this sample reveals that the highlighted operations are redundant
 and can be removed.
 
 ```mlir
 func @dynamic_allocation(%arg0: index, %arg1: index) -> memref<?x?xf32> {
-  %7 = alloc(%arg0, %arg1) : memref<?x?xf32>
-  %c0_0 = constant 0 : index
-  %8 = dim %7, %c0_0 : memref<?x?xf32>
-  %c1_1 = constant 1 : index
-  %9 = dim %7, %c1_1 : memref<?x?xf32>
-  %10 = alloc(%8, %9) : memref<?x?xf32>
-  linalg.copy(%7, %10) : memref<?x?xf32>, memref<?x?xf32>
-  dealloc %7 : memref<?x?xf32>
-  return %10 : memref<?x?xf32>
+  %1 = memref.alloc(%arg0, %arg1) : memref<?x?xf32>
+  %2 = memref.clone %1 : (memref<?x?xf32>) -> (memref<?x?xf32>)
+  memref.dealloc %1 : memref<?x?xf32>
+  return %2 : memref<?x?xf32>
 }
 ```
 
@@ -699,53 +638,39 @@ Will be transformed to:
 
 ```mlir
 func @dynamic_allocation(%arg0: index, %arg1: index) -> memref<?x?xf32> {
-  %7 = alloc(%arg0, %arg1) : memref<?x?xf32>
-  %c0_0 = constant 0 : index
-  %8 = dim %7, %c0_0 : memref<?x?xf32>
-  %c1_1 = constant 1 : index
-  %9 = dim %7, %c1_1 : memref<?x?xf32>
-  return %7 : memref<?x?xf32>
+  %1 = memref.alloc(%arg0, %arg1) : memref<?x?xf32>
+  return %1 : memref<?x?xf32>
 }
 ```
 
-In this case, the additional copy %10 can be replaced with its original source
-buffer %7. This also applies to the associated dealloc operation of %7.
-
-To limit the complexity of this transformation, it only removes copy operations
-when the following constraints are met:
+In this case, the additional copy %2 can be replaced with its original source
+buffer %1. This also applies to the associated dealloc operation of %1.
 
-* The copy operation, the defining operation for the target value, and the
-deallocation of the source value lie in the same block.
-* There are no users/aliases of the target value between the defining operation
-of the target value and its copy operation.
-* There are no users/aliases of the source value between its associated copy
-operation and the deallocation of the source value.
+## Canonicalization of the Target Buffer of the Clone Operation
 
-## Reusing the Target Buffer of the Copy Operation
+In this case, the target buffer of the clone operation can be used instead of
+its source. The unused deallocation operation that is defined for this clone
+operation is also removed.
 
-In this case, the target buffer of the copy operation can be used instead of
-its source. The unused allocation and deallocation operations that are defined
-for this copy operation are also removed.
-
-Consider the following example where a generic linalg operation writes the
-result to %temp and then copies %temp to %result. However, these two operations
-can be merged into a single step. Copy removal removes the copy operation and
-%temp, and replaces the uses of %temp with %result:
+Consider the following example where a generic test operation writes the result
+to %temp and then copies %temp to %result. However, these two operations
+can be merged into a single step. Canonicalization removes the clone operation
+and %temp, and replaces the uses of %temp with %result:
 
 ```mlir
 func @reuseTarget(%arg0: memref<2xf32>, %result: memref<2xf32>){
-  %temp = alloc() : memref<2xf32>
-  linalg.generic {
+  %temp = memref.alloc() : memref<2xf32>
+  test.generic {
     args_in = 1 : i64,
     args_out = 1 : i64,
     indexing_maps = [#map0, #map0],
     iterator_types = ["parallel"]} %arg0, %temp {
   ^bb0(%gen2_arg0: f32, %gen2_arg1: f32):
     %tmp2 = exp %gen2_arg0 : f32
-    linalg.yield %tmp2 : f32
+    test.yield %tmp2 : f32
   }: memref<2xf32>, memref<2xf32>
-  "linalg.copy"(%temp, %result) : (memref<2xf32>, memref<2xf32>) -> ()
-  dealloc %temp : memref<2xf32>
+  %result = memref.clone %temp : (memref<2xf32>) -> (memref<2xf32>)
+  memref.dealloc %temp : memref<2xf32>
   return
 }
 ```
@@ -754,33 +679,24 @@ Will be transformed to:
 
 ```mlir
 func @reuseTarget(%arg0: memref<2xf32>, %result: memref<2xf32>){
-  linalg.generic {
+  test.generic {
     args_in = 1 : i64,
     args_out = 1 : i64,
     indexing_maps = [#map0, #map0],
     iterator_types = ["parallel"]} %arg0, %result {
   ^bb0(%gen2_arg0: f32, %gen2_arg1: f32):
     %tmp2 = exp %gen2_arg0 : f32
-    linalg.yield %tmp2 : f32
+    test.yield %tmp2 : f32
   }: memref<2xf32>, memref<2xf32>
   return
 }
 ```
 
-Like before, several constraints to use the transformation apply:
-
-* The copy operation, the defining operation of the source value, and the
-deallocation of the source value lie in the same block.
-* There are no users/aliases of the target value between the defining operation
-of the source value and the copy operation.
-* There are no users/aliases of the source value between the copy operation and
-the deallocation of the source value.
-
 ## Known Limitations
 
-BufferDeallocation introduces additional copies using allocations from the
-“memref” dialect (“memref.alloc”). Analogous, all deallocations use the
-“memref” dialect-free operation “memref.dealloc”. The actual copy process is
-realized using “linalg.copy”. Furthermore, buffers are essentially immutable
-after their creation in a block. Another limitations are known in the case
-using unstructered control flow.
+BufferDeallocation introduces additional clones from “memref” dialect
+(“memref.clone”). Analogous, all deallocations use the “memref” dialect-free
+operation “memref.dealloc”. The actual copy process is realized using
+“test.copy”. Furthermore, buffers are essentially immutable after their
+creation in a block. Another limitations are known in the case using
+unstructered control flow.
diff --git a/mlir/docs/Bufferization.md b/mlir/docs/Bufferization.md
index de0648deea7a5030faad5c6147450e877516725e..eba93c7a6d348dffecbb13a787b5d203389816be 100644
--- a/mlir/docs/Bufferization.md
+++ b/mlir/docs/Bufferization.md
@@ -156,19 +156,19 @@ is very small, and follows the basic pattern of any dialect conversion pass.
 
 ```
 void mlir::populateTensorBufferizePatterns(
-    MLIRContext *context, BufferizeTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns) {
-  patterns.insert<BufferizeCastOp, BufferizeExtractOp>(typeConverter, context);
+    BufferizeTypeConverter &typeConverter, RewritePatternSet &patterns) {
+  patterns.add<BufferizeCastOp, BufferizeExtractOp>(typeConverter,
+                                                    patterns.getContext());
 }
 
 struct TensorBufferizePass : public TensorBufferizeBase<TensorBufferizePass> {
   void runOnFunction() override {
     auto *context = &getContext();
     BufferizeTypeConverter typeConverter;
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
     ConversionTarget target(*context);
 
-    populateTensorBufferizePatterns(context, typeConverter, patterns);
+    populateTensorBufferizePatterns(typeConverter, patterns);
     target.addIllegalOp<tensor::CastOp, tensor::ExtractOp>();
     target.addLegalDialect<StandardOpsDialect>();
 
@@ -180,7 +180,7 @@ struct TensorBufferizePass : public TensorBufferizeBase<TensorBufferizePass> {
 ```
 
 The pass has all the hallmarks of a dialect conversion pass that does type
-conversions: a `TypeConverter`, a `OwningRewritePatternList`, and a
+conversions: a `TypeConverter`, a `RewritePatternSet`, and a
 `ConversionTarget`, and a call to `applyPartialConversion`. Note that a function
 `populateTensorBufferizePatterns` is separated, so that power users can use the
 patterns independently, if necessary (such as to combine multiple sets of
diff --git a/mlir/docs/Canonicalization.md b/mlir/docs/Canonicalization.md
index 3e1c9d11ecabdeaf8fbb9f80746d46ba3302ff9b..143c9edfc976d5809832f6a12d2d9f16d7358cde 100644
--- a/mlir/docs/Canonicalization.md
+++ b/mlir/docs/Canonicalization.md
@@ -56,9 +56,9 @@ These transformations are applied to all levels of IR:
 ## Defining Canonicalizations
 
 Two mechanisms are available with which to define canonicalizations;
-`getCanonicalizationPatterns` and `fold`.
+general `RewritePattern`s and the `fold` method.
 
-### Canonicalizing with `getCanonicalizationPatterns`
+### Canonicalizing with `RewritePattern`s
 
 This mechanism allows for providing canonicalizations as a set of
 `RewritePattern`s, either imperatively defined in C++ or declaratively as
@@ -67,28 +67,41 @@ infrastructure allows for expressing many different types of canonicalizations.
 These transformations may be as simple as replacing a multiplication with a
 shift, or even replacing a conditional branch with an unconditional one.
 
-In [ODS](OpDefinitions.md), an operation can set the `hasCanonicalizer` bit to
-generate a declaration for the `getCanonicalizationPatterns` method.
+In [ODS](OpDefinitions.md), an operation can set the `hasCanonicalizer` bit or
+the `hasCanonicalizeMethod` bit to generate a declaration for the
+`getCanonicalizationPatterns` method:
 
 ```tablegen
 def MyOp : ... {
+  // I want to define a fully general set of patterns for this op.
   let hasCanonicalizer = 1;
 }
+
+def OtherOp : ... {
+  // A single "matchAndRewrite" style RewritePattern implemented as a method
+  // is good enough for me.
+  let hasCanonicalizeMethod = 1;
+}
 ```
 
 Canonicalization patterns can then be provided in the source file:
 
 ```c++
-void MyOp::getCanonicalizationPatterns(OwningRewritePatternList &patterns,
+void MyOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
                                        MLIRContext *context) {
-  patterns.insert<...>(...);
+  patterns.add<...>(...);
+}
+
+LogicalResult OtherOp::canonicalize(OtherOp op, PatternRewriter &rewriter) {
+  // patterns and rewrites go here.
+  return failure();
 }
 ```
 
 See the [quickstart guide](Tutorials/QuickstartRewrites.md) for information on
 defining operation rewrites.
 
-### Canonicalizing with `fold`
+### Canonicalizing with the `fold` method
 
 The `fold` mechanism is an intentionally limited, but powerful mechanism that
 allows for applying canonicalizations in many places throughout the compiler.
diff --git a/mlir/docs/DataLayout.md b/mlir/docs/DataLayout.md
index 4a57a2d6ca4c5cacab20779b9fb81c16127828a4..6d2aab864669f867a841cb5ecc3bb74aa71fa6ef 100644
--- a/mlir/docs/DataLayout.md
+++ b/mlir/docs/DataLayout.md
@@ -18,24 +18,26 @@ system. At the top level, it consists of:
     types.
 
 Built-in types are handled specially to decrease the overall query cost.
+Similarly, built-in `ModuleOp` supports data layouts without going through the
+interface.
 
 ## Usage
 
 ### Scoping
 
 Following MLIR's nested structure, data layout properties are _scoped_ to
-regions belonging to specific operations that implement the
-`DataLayoutOpInterface`. Such scoping operations partially control the data
-layout properties and may have attributes that affect them, typically organized
-in a data layout specification.
+regions belonging to either operations that implement the
+`DataLayoutOpInterface` or `ModuleOp` operations. Such scoping operations
+partially control the data layout properties and may have attributes that affect
+them, typically organized in a data layout specification.
 
 Types may have a different data layout in different scopes, including scopes
 that are nested in other scopes such as modules contained in other modules. At
 the same time, within the given scope excluding any nested scope, a given type
 has fixed data layout properties. Types are also expected to have a default,
 "natural" data layout in case they are used outside of any operation that
-provides data layout scope for them. This ensure data layout queries always have
-a valid result.
+provides data layout scope for them. This ensures that data layout queries
+always have a valid result.
 
 ### Compatibility and Transformations
 
@@ -70,6 +72,7 @@ public:
   explicit DataLayout(DataLayoutOpInterface scope);
 
   unsigned getTypeSize(Type type) const;
+  unsigned getTypeSizeInBits(Type type) const;
   unsigned getTypeABIAlignment(Type type) const;
   unsigned getTypePreferredAlignment(Type type) const;
 };
@@ -176,24 +179,37 @@ dialect is expected to implement the `DataLayoutDialectInterface`. This dialect
 provides hooks for verifying the validity of the entry value attributes and for
 and the compatibility of nested entries.
 
+### Bits and Bytes
+
+Two versions of hooks are provided for sizes: in bits and in bytes. The version
+in bytes has a default implementation that derives the size in bytes by rounding
+up the result of division of the size in bits by 8. Types exclusively targeting
+architectures with different assumptions can override this. Operations can
+redefine this for all types, providing scoped versions for cases of byte sizes
+other than eight without having to modify types, including built-in types.
+
 ### Query Dispatch
 
 The overall flow of a data layout property query is as follows.
 
--   The user constructs a `DataLayout` at the given scope. The constructor
+1.  The user constructs a `DataLayout` at the given scope. The constructor
     fetches the data layout specification and combines it with those of
     enclosing scopes (layouts are expected to be compatible).
--   The user calls `DataLayout::query(Type ty)`.
--   If `DataLayout` has a cached response, this response is returned
+2.  The user calls `DataLayout::query(Type ty)`.
+3.  If `DataLayout` has a cached response, this response is returned
     immediately.
--   Otherwise, the query is handed down by `DataLayout` to
-    `DataLayoutOpInterface::query(ty, *this, relevantEntries)` where the
-    relevant entries are computed as described above.
--   Unless the `query` hook is reimplemented by the op interface, the query is
+4.  Otherwise, the query is handed down by `DataLayout` to the closest layout
+    scoping operation. If it implements `DataLayoutOpInterface`, then the query
+    is forwarded to`DataLayoutOpInterface::query(ty, *this, relevantEntries)`
+    where the relevant entries are computed as described above. If it does not
+    implement `DataLayoutOpInterface`, it must be a `ModuleOp`, and the query is
+    forwarded to `DataLayoutTypeInterface::query(dataLayout, relevantEntries)`
+    after casting `ty` to the type interface.
+5.  Unless the `query` hook is reimplemented by the op interface, the query is
     handled further down to `DataLayoutTypeInterface::query(dataLayout,
     relevantEntries)` after casting `ty` to the type interface. If the type does
     not implement the interface, an unrecoverable fatal error is produced.
--   The type is expected to always provide the response, which is returned up
+6.  The type is expected to always provide the response, which is returned up
     the call stack and cached by the `DataLayout.`
 
 ## Default Implementation
@@ -201,6 +217,14 @@ The overall flow of a data layout property query is as follows.
 The default implementation of the data layout interfaces directly handles
 queries for a subset of built-in types.
 
+### Built-in Modules
+
+Built-in `ModuleOp` allows at most one attribute that implements
+`DataLayoutSpecInterface`. It does not implement the entire interface for
+efficiency and layering reasons. Instead, `DataLayout` can be constructed for
+`ModuleOp` and handles modules transparently alongside other operations that
+implement the interface.
+
 ### Built-in Types
 
 The following describes the default properties of built-in types.
@@ -229,6 +253,28 @@ with the
 [modeling of n-D vectors](https://mlir.llvm.org/docs/Dialects/Vector/#deeperdive).
 They **may change** in the future.
 
+#### `index` type
+
+Index type is an integer type used for target-specific size information in,
+e.g., `memref` operations. Its data layout is parameterized by a single integer
+data layout entry that specifies its bitwidth. For example,
+
+```
+module attributes { dlti.dl_spec = #dlti.dl_spec<
+  #dlti.dl_entry<index, 32>
+>} {}
+```
+
+specifies that `index` has 32 bits. All other layout properties of `index` match
+those of the integer type with the same bitwidth defined above.
+
+In absence of the corresponding entry, `index` is assumed to be a 64-bit
+integer.
+
+### Byte Size
+
+The default data layout assumes 8-bit bytes.
+
 ### DLTI Dialect
 
 The [DLTI](Dialects/DLTI.md) dialect provides the attributes implementing
diff --git a/mlir/docs/Diagnostics.md b/mlir/docs/Diagnostics.md
index 9e1e4f9156c3cb7646e4ccf892385add5d0d4cf0..6300dd3dc06c3f6262f7e5b9c0f3de952d0a6b8b 100644
--- a/mlir/docs/Diagnostics.md
+++ b/mlir/docs/Diagnostics.md
@@ -11,69 +11,9 @@ structure of the IR, operations, etc.
 ## Source Locations
 
 Source location information is extremely important for any compiler, because it
-provides a baseline for debuggability and error-reporting. MLIR provides several
-different location types depending on the situational need.
-
-### CallSite Location
-
-```
-callsite-location ::= 'callsite' '(' location 'at' location ')'
-```
-
-An instance of this location allows for representing a directed stack of
-location usages. This connects a location of a `callee` with the location of a
-`caller`.
-
-### FileLineCol Location
-
-```
-filelinecol-location ::= string-literal ':' integer-literal ':' integer-literal
-```
-
-An instance of this location represents a tuple of file, line number, and column
-number. This is similar to the type of location that you get from most source
-languages.
-
-### Fused Location
-
-```
-fused-location ::= `fused` fusion-metadata? '[' location (location ',')* ']'
-fusion-metadata ::= '<' attribute-value '>'
-```
-
-An instance of a `fused` location represents a grouping of several other source
-locations, with optional metadata that describes the context of the fusion.
-There are many places within a compiler in which several constructs may be fused
-together, e.g. pattern rewriting, that normally result partial or even total
-loss of location information. With `fused` locations, this is a non-issue.
-
-### Name Location
-
-```
-name-location ::= string-literal ('(' location ')')?
-```
-
-An instance of this location allows for attaching a name to a child location.
-This can be useful for representing the locations of variable, or node,
-definitions.
-
-### Opaque Location
-
-An instance of this location essentially contains a pointer to some data
-structure that is external to MLIR and an optional location that can be used if
-the first one is not suitable. Since it contains an external structure, only the
-optional location is used during serialization.
-
-### Unknown Location
-
-```
-unknown-location ::= `unknown`
-```
-
-Source location information is an extremely integral part of the MLIR
-infrastructure. As such, location information is always present in the IR, and
-must explicitly be set to unknown. Thus an instance of the `unknown` location,
-represents an unspecified source location.
+provides a baseline for debuggability and error-reporting. The
+[builtin dialect](Dialects/Builtin.md) provides several different location
+attributes types depending on the situational need.
 
 ## Diagnostic Engine
 
diff --git a/mlir/docs/Dialects/Builtin.md b/mlir/docs/Dialects/Builtin.md
new file mode 100644
index 0000000000000000000000000000000000000000..6a1bd365d2df71f5e2db3788210ccbd6debf943f
--- /dev/null
+++ b/mlir/docs/Dialects/Builtin.md
@@ -0,0 +1,32 @@
+# Builtin Dialect
+
+The builtin dialect contains a core set of Attributes, Operations, and Types
+that have wide applicability across a very large number of domains and
+abstractions. Many of the components of this dialect are also instrumental in
+the implementation of the core IR. As such, this dialect is implicitly loaded in
+every `MLIRContext`, and available directly to all users of MLIR.
+
+Given the far-reaching nature of this dialect and the fact that MLIR is
+extensible by design, any potential additions are heavily scrutinized.
+
+[TOC]
+
+## Attributes
+
+[include "Dialects/BuiltinAttributes.md"]
+
+## Location Attributes
+
+A subset of the builtin attribute values correspond to
+[source locations](../Diagnostics.md#source-locations), that may be attached to
+Operations.
+
+[include "Dialects/BuiltinLocationAttributes.md"]
+
+## Operations
+
+[include "Dialects/BuiltinOps.md"]
+
+## Types
+
+[include "Dialects/BuiltinTypes.md"]
diff --git a/mlir/docs/Interfaces.md b/mlir/docs/Interfaces.md
index 7f2920bae775b1b88c69253302d42037068fc1e3..927e1d89a1f5f30ceb1a3ab7d9c2bc6322e4833d 100644
--- a/mlir/docs/Interfaces.md
+++ b/mlir/docs/Interfaces.md
@@ -207,6 +207,45 @@ if (ExampleOpInterface example = dyn_cast<ExampleOpInterface>(op))
   llvm::errs() << "hook returned = " << example.exampleInterfaceHook() << "\n";
 ```
 
+#### Dialect Fallback for OpInterface
+
+Some dialects have an open ecosystem and don't register all of the possible
+operations. In such cases it is still possible to provide support for
+implementing an `OpInterface` for these operation. When an operation isn't
+registered or does not provide an implementation for an interface, the query
+will fallback to the dialect itself.
+
+A second model is used for such cases and automatically generated when
+using ODS (see below) with the name `FallbackModel`. This model can be implemented
+for a particular dialect:
+
+```c++
+// This is the implementation of a dialect fallback for `ExampleOpInterface`.
+struct FallbackExampleOpInterface
+    : public ExampleOpInterface::FallbackModel<
+          FallbackExampleOpInterface> {
+  static bool classof(Operation *op) { return true; }
+
+  unsigned exampleInterfaceHook(Operation *op) const;
+  unsigned exampleStaticInterfaceHook() const;
+};
+```
+
+A dialect can then instantiate this implementation and returns it on specific
+operations by overriding the `getRegisteredInterfaceForOp` method :
+
+```c++
+void *TestDialect::getRegisteredInterfaceForOp(TypeID typeID,
+                                               Identifier opName) {
+  if (typeID == TypeID::get<ExampleOpInterface>()) {
+    if (isSupported(opName))
+      return fallbackExampleOpInterface;
+    return nullptr;
+  }
+  return nullptr;
+}
+```
+
 #### Utilizing the ODS Framework
 
 Note: Before reading this section, the reader should have some familiarity with
diff --git a/mlir/docs/LLVMDialectMemRefConvention.md b/mlir/docs/LLVMDialectMemRefConvention.md
index 78ec6fb007526d6b20d2c98f09ede33ec3bb45d7..d528f0c01c217c01de2fc3bfd0418b9f38e21e42 100644
--- a/mlir/docs/LLVMDialectMemRefConvention.md
+++ b/mlir/docs/LLVMDialectMemRefConvention.md
@@ -232,29 +232,40 @@ struct MemRefDescriptor {
 };
 ```
 
+Furthermore, we also rewrite function results to pointer parameters if the
+rewritten function result has a struct type. The special result parameter is
+added as the first parameter and is of pointer-to-struct type.
+
 If enabled, the option will do the following. For _external_ functions declared
 in the MLIR module.
 
 1.  Declare a new function `_mlir_ciface_<original name>` where memref arguments
     are converted to pointer-to-struct and the remaining arguments are converted
-    as usual.
-1.  Add a body to the original function (making it non-external) that
-    1.  allocates a memref descriptor,
-    1.  populates it, and
-    1.  passes the pointer to it into the newly declared interface function,
+    as usual. Results are converted to a special argument if they are of struct
+    type.
+2.  Add a body to the original function (making it non-external) that
+    1.  allocates memref descriptors,
+    2.  populates them,
+    3.  potentially allocates space for the result struct, and
+    4.  passes the pointers to these into the newly declared interface function,
         then
-    1.  collects the result of the call and returns it to the caller.
+    5.  collects the result of the call (potentially from the result struct),
+        and
+    6.  returns it to the caller.
 
 For (non-external) functions defined in the MLIR module.
 
 1.  Define a new function `_mlir_ciface_<original name>` where memref arguments
     are converted to pointer-to-struct and the remaining arguments are converted
-    as usual.
-1.  Populate the body of the newly defined function with IR that
+    as usual. Results are converted to a special argument if they are of struct
+    type.
+2.  Populate the body of the newly defined function with IR that
     1.  loads descriptors from pointers;
-    1.  unpacks descriptor into individual non-aggregate values;
-    1.  passes these values into the original function;
-    1.  collects the result of the call and returns it to the caller.
+    2.  unpacks descriptor into individual non-aggregate values;
+    3.  passes these values into the original function;
+    4.  collects the results of the call and
+    5.  either copies the results into the result struct or returns them to the
+        caller.
 
 Examples:
 
@@ -342,6 +353,49 @@ llvm.func @_mlir_ciface_foo(%arg0: !llvm.memref_2d_ptr) {
 }
 ```
 
+```mlir
+func @foo(%arg0: memref<?x?xf32>) -> memref<?x?xf32> {
+  return %arg0 : memref<?x?xf32>
+}
+
+// Gets converted into the following
+// (using type alias for brevity):
+!llvm.memref_2d = type !llvm.struct<(ptr<f32>, ptr<f32>, i64,
+                                     array<2xi64>, array<2xi64>)>
+!llvm.memref_2d_ptr = type !llvm.ptr<struct<(ptr<f32>, ptr<f32>, i64,
+                                             array<2xi64>, array<2xi64>)>>
+
+// Function with unpacked arguments.
+llvm.func @foo(%arg0: !llvm.ptr<f32>, %arg1: !llvm.ptr<f32>, %arg2: i64,
+               %arg3: i64, %arg4: i64, %arg5: i64, %arg6: i64)
+    -> !llvm.memref_2d {
+  %0 = llvm.mlir.undef : !llvm.memref_2d
+  %1 = llvm.insertvalue %arg0, %0[0] : !llvm.memref_2d
+  %2 = llvm.insertvalue %arg1, %1[1] : !llvm.memref_2d
+  %3 = llvm.insertvalue %arg2, %2[2] : !llvm.memref_2d
+  %4 = llvm.insertvalue %arg3, %3[3, 0] : !llvm.memref_2d
+  %5 = llvm.insertvalue %arg5, %4[4, 0] : !llvm.memref_2d
+  %6 = llvm.insertvalue %arg4, %5[3, 1] : !llvm.memref_2d
+  %7 = llvm.insertvalue %arg6, %6[4, 1] : !llvm.memref_2d
+  llvm.return %7 : !llvm.memref_2d
+}
+
+// Interface function callable from C.
+llvm.func @_mlir_ciface_foo(%arg0: !llvm.memref_2d_ptr, %arg1: !llvm.memref_2d_ptr) {
+  %0 = llvm.load %arg1 : !llvm.memref_2d_ptr
+  %1 = llvm.extractvalue %0[0] : !llvm.memref_2d
+  %2 = llvm.extractvalue %0[1] : !llvm.memref_2d
+  %3 = llvm.extractvalue %0[2] : !llvm.memref_2d
+  %4 = llvm.extractvalue %0[3, 0] : !llvm.memref_2d
+  %5 = llvm.extractvalue %0[3, 1] : !llvm.memref_2d
+  %6 = llvm.extractvalue %0[4, 0] : !llvm.memref_2d
+  %7 = llvm.extractvalue %0[4, 1] : !llvm.memref_2d
+  %8 = llvm.call @foo(%1, %2, %3, %4, %5, %6, %7)
+    : (!llvm.ptr<f32>, !llvm.ptr<f32>, i64, i64, i64, i64, i64) -> !llvm.memref_2d
+  llvm.store %8, %arg0 : !llvm.memref_2d_ptr
+  llvm.return
+}
+
 Rationale: Introducing auxiliary functions for C-compatible interfaces is
 preferred to modifying the calling convention since it will minimize the effect
 of C compatibility on intra-module calls or calls between MLIR-generated
diff --git a/mlir/docs/LangRef.md b/mlir/docs/LangRef.md
index 7b58b63258a5e8c632221576d4bb882a8f396a8d..f380987e90aa54694956e8a5ca7ef3c0f2c63859 100644
--- a/mlir/docs/LangRef.md
+++ b/mlir/docs/LangRef.md
@@ -60,14 +60,13 @@ Operation](docs/Tutorials/Toy/Ch-2/#op-vs-operation-using-mlir-operations))
 
 One obvious application of MLIR is to represent an
 [SSA-based](https://en.wikipedia.org/wiki/Static_single_assignment_form) IR,
-like the LLVM core IR, with appropriate choice of Operation Types to define
-[Modules](#module), [Functions](#functions), Branches, Allocations, and
-verification constraints to ensure the SSA Dominance property. MLIR includes a
-'standard' dialect which defines just such structures. However, MLIR is
-intended to be general enough to represent other compiler-like data
-structures, such as Abstract Syntax Trees in a language frontend, generated
-instructions in a target-specific backend, or circuits in a High-Level
-Synthesis tool.
+like the LLVM core IR, with appropriate choice of operation types to define
+Modules, Functions, Branches, Memory Allocation, and verification constraints to
+ensure the SSA Dominance property. MLIR includes a collection of dialects which
+defines just such structures. However, MLIR is intended to be general enough to
+represent other compiler-like data structures, such as Abstract Syntax Trees in
+a language frontend, generated instructions in a target-specific backend, or
+circuits in a High-Level Synthesis tool.
 
 Here's an example of an MLIR module:
 
@@ -328,96 +327,12 @@ In addition to the basic syntax above, dialects may register known operations.
 This allows those dialects to support _custom assembly form_ for parsing and
 printing operations. In the operation sets listed below, we show both forms.
 
-### Terminator Operations
+### Builtin Operations
 
-These are a special category of operations that *must* terminate a block, e.g.
-[branches](Dialects/Standard.md#terminator-operations). These operations may
-also have a list of successors ([blocks](#blocks) and their arguments).
-
-Example:
-
-```mlir
-// Branch to ^bb1 or ^bb2 depending on the condition %cond.
-// Pass value %v to ^bb2, but not to ^bb1.
-"cond_br"(%cond)[^bb1, ^bb2(%v : index)] : (i1) -> ()
-```
-
-### Module
-
-```
-module ::= `module` symbol-ref-id? (`attributes` dictionary-attribute)? region
-```
-
-An MLIR Module represents a top-level container operation. It contains a single
-[SSACFG region](#control-flow-and-ssacfg-regions) containing a single block
-which can contain any operations. Operations within this region cannot
-implicitly capture values defined outside the module, i.e. Modules are
-[IsolatedFromAbove](Traits.md#isolatedfromabove). Modules have an optional
-[symbol name](SymbolsAndSymbolTables.md) which can be used to refer to them in
-operations.
-
-### Functions
-
-An MLIR Function is an operation with a name containing a single [SSACFG
-region](#control-flow-and-ssacfg-regions).  Operations within this region
-cannot implicitly capture values defined outside of the function,
-i.e. Functions are [IsolatedFromAbove](Traits.md#isolatedfromabove).  All
-external references must use function arguments or attributes that establish a
-symbolic connection (e.g. symbols referenced by name via a string attribute
-like [SymbolRefAttr](#symbol-reference-attribute)):
-
-```
-function ::= `func` function-signature function-attributes? function-body?
-
-function-signature ::= symbol-ref-id `(` argument-list `)`
-                       (`->` function-result-list)?
-
-argument-list ::= (named-argument (`,` named-argument)*) | /*empty*/
-argument-list ::= (type dictionary-attribute? (`,` type dictionary-attribute?)*)
-                | /*empty*/
-named-argument ::= value-id `:` type dictionary-attribute?
-
-function-result-list ::= function-result-list-parens
-                       | non-function-type
-function-result-list-parens ::= `(` `)`
-                              | `(` function-result-list-no-parens `)`
-function-result-list-no-parens ::= function-result (`,` function-result)*
-function-result ::= type dictionary-attribute?
-
-function-attributes ::= `attributes` dictionary-attribute
-function-body ::= region
-```
-
-An external function declaration (used when referring to a function declared
-in some other module) has no body. While the MLIR textual form provides a nice
-inline syntax for function arguments, they are internally represented as
-"block arguments" to the first block in the region.
-
-Only dialect attribute names may be specified in the attribute dictionaries
-for function arguments, results, or the function itself.
-
-Examples:
-
-```mlir
-// External function definitions.
-func @abort()
-func @scribble(i32, i64, memref<? x 128 x f32, #layout_map0>) -> f64
-
-// A function that returns its argument twice:
-func @count(%x: i64) -> (i64, i64)
-  attributes {fruit: "banana"} {
-  return %x, %x: i64, i64
-}
-
-// A function with an argument attribute
-func @example_fn_arg(%x: i32 {swift.self = unit})
-
-// A function with a result attribute
-func @example_fn_result() -> (f64 {dialectName.attrName = 0 : i64})
-
-// A function with an attribute
-func @example_fn_attr() attributes {dialectName.attrName = false}
-```
+The [builtin dialect](Dialects/Builtin.md) defines a select few operations that
+are widely applicable by MLIR dialects, such as a universal conversion cast
+operation that simplifies inter/intra dialect conversion. This dialect also
+defines a top-level `module` operation, that represents a useful IR container.
 
 ## Blocks
 
@@ -436,13 +351,18 @@ value-id-and-type-list ::= value-id-and-type (`,` value-id-and-type)*
 block-arg-list ::= `(` value-id-and-type-list? `)`
 ```
 
-A *Block* is an ordered list of operations, concluding with a single
-[terminator operation](#terminator-operations). In [SSACFG
+A *Block* is a list of operations. In [SSACFG
 regions](#control-flow-and-ssacfg-regions), each block represents a compiler
 [basic block](https://en.wikipedia.org/wiki/Basic_block) where instructions
 inside the block are executed in order and terminator operations implement
 control flow branches between basic blocks.
 
+A region with a single block may not include a [terminator
+operation](#terminator-operations). The enclosing op can opt-out of this
+requirement with the `NoTerminator` trait. The top-level `ModuleOp` is an
+example of such operation which defined this trait and whose block body does
+not have a terminator.
+
 Blocks in MLIR take a list of block arguments, notated in a function-like
 way. Block arguments are bound to values specified by the semantics of
 individual operations. Block arguments of the entry block of a region are also
@@ -701,14 +621,10 @@ defines the relation between the region results and the operation results.
 
 ## Type System
 
-Each value in MLIR has a type defined by the type system below. There are a
-number of primitive types (like integers) and also aggregate types for tensors
-and memory buffers. MLIR [builtin types](#builtin-types) do not include
-structures, arrays, or dictionaries.
-
-MLIR has an open type system (i.e. there is no fixed list of types), and types
-may have application-specific semantics. For example, MLIR supports a set of
-[dialect types](#dialect-types).
+Each value in MLIR has a type defined by the type system. MLIR has an open type
+system (i.e. there is no fixed list of types), and types may have
+application-specific semantics. MLIR dialects may define any number of types
+with no restrictions on the abstractions they represent.
 
 ```
 type ::= type-alias | dialect-type | builtin-type
@@ -806,497 +722,14 @@ the lighter syntax: `!foo.something<a%%123^^^>>>` because it contains characters
 that are not allowed in the lighter syntax, as well as unbalanced `<>`
 characters.
 
-See [here](Tutorials/DefiningAttributesAndTypes.md) to learn how to define dialect types.
+See [here](Tutorials/DefiningAttributesAndTypes.md) to learn how to define
+dialect types.
 
 ### Builtin Types
 
-Builtin types are a core set of [dialect types](#dialect-types) that are defined
-in a builtin dialect and thus available to all users of MLIR.
-
-```
-builtin-type ::=      complex-type
-                    | float-type
-                    | function-type
-                    | index-type
-                    | integer-type
-                    | memref-type
-                    | none-type
-                    | tensor-type
-                    | tuple-type
-                    | vector-type
-```
-
-#### Complex Type
-
-Syntax:
-
-```
-complex-type ::= `complex` `<` type `>`
-```
-
-The value of `complex` type represents a complex number with a parameterized
-element type, which is composed of a real and imaginary value of that element
-type. The element must be a floating point or integer scalar type.
-
-Examples:
-
-```mlir
-complex<f32>
-complex<i32>
-```
-
-#### Floating Point Types
-
-Syntax:
-
-```
-// Floating point.
-float-type ::= `f16` | `bf16` | `f32` | `f64` | `f80` | `f128`
-```
-
-MLIR supports float types of certain widths that are widely used as indicated
-above.
-
-#### Function Type
-
-Syntax:
-
-```
-// MLIR functions can return multiple values.
-function-result-type ::= type-list-parens
-                       | non-function-type
-
-function-type ::= type-list-parens `->` function-result-type
-```
-
-MLIR supports first-class functions: for example, the
-[`constant` operation](Dialects/Standard.md#stdconstant-constantop) produces the
-address of a function as a value. This value may be passed to and
-returned from functions, merged across control flow boundaries with
-[block arguments](#blocks), and called with the
-[`call_indirect` operation](Dialects/Standard.md#call-indirect-operation).
-
-Function types are also used to indicate the arguments and results of
-[operations](#operations).
-
-#### Index Type
-
-Syntax:
-
-```
-// Target word-sized integer.
-index-type ::= `index`
-```
-
-The `index` type is a signless integer whose size is equal to the natural
-machine word of the target
-([rationale](Rationale/Rationale.md#integer-signedness-semantics)) and is used
-by the affine constructs in MLIR. Unlike fixed-size integers, it cannot be used
-as an element of vector
-([rationale](Rationale/Rationale.md#index-type-disallowed-in-vector-types)).
-
-**Rationale:** integers of platform-specific bit widths are practical to express
-sizes, dimensionalities and subscripts.
-
-#### Integer Type
-
-Syntax:
-
-```
-// Sized integers like i1, i4, i8, i16, i32.
-signed-integer-type ::= `si` [1-9][0-9]*
-unsigned-integer-type ::= `ui` [1-9][0-9]*
-signless-integer-type ::= `i` [1-9][0-9]*
-integer-type ::= signed-integer-type |
-                 unsigned-integer-type |
-                 signless-integer-type
-```
-
-MLIR supports arbitrary precision integer types. Integer types have a designated
-width and may have signedness semantics.
-
-**Rationale:** low precision integers (like `i2`, `i4` etc) are useful for
-low-precision inference chips, and arbitrary precision integers are useful for
-hardware synthesis (where a 13 bit multiplier is a lot cheaper/smaller than a 16
-bit one).
-
-TODO: Need to decide on a representation for quantized integers
-([initial thoughts](Rationale/Rationale.md#quantized-integer-operations)).
-
-#### Memref Type
-
-Syntax:
-
-```
-memref-type ::= ranked-memref-type | unranked-memref-type
-
-ranked-memref-type ::= `memref` `<` dimension-list-ranked type
-                      (`,` layout-specification)? (`,` memory-space)? `>`
-
-unranked-memref-type ::= `memref` `<*x` type (`,` memory-space)? `>`
-
-stride-list ::= `[` (dimension (`,` dimension)*)? `]`
-strided-layout ::= `offset:` dimension `,` `strides: ` stride-list
-semi-affine-map-composition ::= (semi-affine-map `,` )* semi-affine-map
-layout-specification ::= semi-affine-map-composition | strided-layout
-memory-space ::= integer-literal /* | TODO: address-space-id */
-```
-
-A `memref` type is a reference to a region of memory (similar to a buffer
-pointer, but more powerful). The buffer pointed to by a memref can be allocated,
-aliased and deallocated. A memref can be used to read and write data from/to the
-memory region which it references. Memref types use the same shape specifier as
-tensor types. Note that `memref<f32>`, `memref<0 x f32>`, `memref<1 x 0 x f32>`,
-and `memref<0 x 1 x f32>` are all different types.
-
-A `memref` is allowed to have an unknown rank (e.g. `memref<*xf32>`). The
-purpose of unranked memrefs is to allow external library functions to receive
-memref arguments of any rank without versioning the functions based on the rank.
-Other uses of this type are disallowed or will have undefined behavior.
-
-##### Codegen of Unranked Memref
-
-Using unranked memref in codegen besides the case mentioned above is highly
-discouraged. Codegen is concerned with generating loop nests and specialized
-instructions for high-performance, unranked memref is concerned with hiding the
-rank and thus, the number of enclosing loops required to iterate over the data.
-However, if there is a need to code-gen unranked memref, one possible path is to
-cast into a static ranked type based on the dynamic rank. Another possible path
-is to emit a single while loop conditioned on a linear index and perform
-delinearization of the linear index to a dynamic array containing the (unranked)
-indices. While this is possible, it is expected to not be a good idea to perform
-this during codegen as the cost of the translations is expected to be
-prohibitive and optimizations at this level are not expected to be worthwhile.
-If expressiveness is the main concern, irrespective of performance, passing
-unranked memrefs to an external C++ library and implementing rank-agnostic logic
-there is expected to be significantly simpler.
-
-Unranked memrefs may provide expressiveness gains in the future and help bridge
-the gap with unranked tensors. Unranked memrefs will not be expected to be
-exposed to codegen but one may query the rank of an unranked memref (a special
-op will be needed for this purpose) and perform a switch and cast to a ranked
-memref as a prerequisite to codegen.
-
-Example:
-
-```mlir
-// With static ranks, we need a function for each possible argument type
-%A = alloc() : memref<16x32xf32>
-%B = alloc() : memref<16x32x64xf32>
-call @helper_2D(%A) : (memref<16x32xf32>)->()
-call @helper_3D(%B) : (memref<16x32x64xf32>)->()
-
-// With unknown rank, the functions can be unified under one unranked type
-%A = alloc() : memref<16x32xf32>
-%B = alloc() : memref<16x32x64xf32>
-// Remove rank info
-%A_u = memref_cast %A : memref<16x32xf32> -> memref<*xf32>
-%B_u = memref_cast %B : memref<16x32x64xf32> -> memref<*xf32>
-// call same function with dynamic ranks
-call @helper(%A_u) : (memref<*xf32>)->()
-call @helper(%B_u) : (memref<*xf32>)->()
-```
-
-The core syntax and representation of a layout specification is a
-[semi-affine map](Dialects/Affine.md#semi-affine-maps). Additionally, syntactic
-sugar is supported to make certain layout specifications more intuitive to read.
-For the moment, a `memref` supports parsing a strided form which is converted to
-a semi-affine map automatically.
-
-The memory space of a memref is specified by a target-specific attribute.
-It might be an integer value, string, dictionary or custom dialect attribute.
-The empty memory space (attribute is None) is target specific.
-
-The notionally dynamic value of a memref value includes the address of the
-buffer allocated, as well as the symbols referred to by the shape, layout map,
-and index maps.
-
-Examples of memref static type
-
-```mlir
-// Identity index/layout map
-#identity = affine_map<(d0, d1) -> (d0, d1)>
-
-// Column major layout.
-#col_major = affine_map<(d0, d1, d2) -> (d2, d1, d0)>
-
-// A 2-d tiled layout with tiles of size 128 x 256.
-#tiled_2d_128x256 = affine_map<(d0, d1) -> (d0 div 128, d1 div 256, d0 mod 128, d1 mod 256)>
-
-// A tiled data layout with non-constant tile sizes.
-#tiled_dynamic = affine_map<(d0, d1)[s0, s1] -> (d0 floordiv s0, d1 floordiv s1,
-                             d0 mod s0, d1 mod s1)>
-
-// A layout that yields a padding on two at either end of the minor dimension.
-#padded = affine_map<(d0, d1) -> (d0, (d1 + 2) floordiv 2, (d1 + 2) mod 2)>
-
-
-// The dimension list "16x32" defines the following 2D index space:
-//
-//   { (i, j) : 0 <= i < 16, 0 <= j < 32 }
-//
-memref<16x32xf32, #identity>
-
-// The dimension list "16x4x?" defines the following 3D index space:
-//
-//   { (i, j, k) : 0 <= i < 16, 0 <= j < 4, 0 <= k < N }
-//
-// where N is a symbol which represents the runtime value of the size of
-// the third dimension.
-//
-// %N here binds to the size of the third dimension.
-%A = alloc(%N) : memref<16x4x?xf32, #col_major>
-
-// A 2-d dynamic shaped memref that also has a dynamically sized tiled layout.
-// The memref index space is of size %M x %N, while %B1 and %B2 bind to the
-// symbols s0, s1 respectively of the layout map #tiled_dynamic. Data tiles of
-// size %B1 x %B2 in the logical space will be stored contiguously in memory.
-// The allocation size will be (%M ceildiv %B1) * %B1 * (%N ceildiv %B2) * %B2
-// f32 elements.
-%T = alloc(%M, %N) [%B1, %B2] : memref<?x?xf32, #tiled_dynamic>
-
-// A memref that has a two-element padding at either end. The allocation size
-// will fit 16 * 64 float elements of data.
-%P = alloc() : memref<16x64xf32, #padded>
-
-// Affine map with symbol 's0' used as offset for the first dimension.
-#imapS = affine_map<(d0, d1) [s0] -> (d0 + s0, d1)>
-// Allocate memref and bind the following symbols:
-// '%n' is bound to the dynamic second dimension of the memref type.
-// '%o' is bound to the symbol 's0' in the affine map of the memref type.
-%n = ...
-%o = ...
-%A = alloc (%n)[%o] : <16x?xf32, #imapS>
-```
-
-##### Index Space
-
-A memref dimension list defines an index space within which the memref can be
-indexed to access data.
-
-##### Index
-
-Data is accessed through a memref type using a multidimensional index into the
-multidimensional index space defined by the memref's dimension list.
-
-Examples
-
-```mlir
-// Allocates a memref with 2D index space:
-//   { (i, j) : 0 <= i < 16, 0 <= j < 32 }
-%A = alloc() : memref<16x32xf32, #imapA>
-
-// Loads data from memref '%A' using a 2D index: (%i, %j)
-%v = load %A[%i, %j] : memref<16x32xf32, #imapA>
-```
-
-##### Index Map
-
-An index map is a one-to-one
-[semi-affine map](Dialects/Affine.md#semi-affine-maps) that transforms a
-multidimensional index from one index space to another. For example, the
-following figure shows an index map which maps a 2-dimensional index from a 2x2
-index space to a 3x3 index space, using symbols `S0` and `S1` as offsets.
-
-![Index Map Example](/includes/img/index-map.svg)
-
-The number of domain dimensions and range dimensions of an index map can be
-different, but must match the number of dimensions of the input and output index
-spaces on which the map operates. The index space is always non-negative and
-integral. In addition, an index map must specify the size of each of its range
-dimensions onto which it maps. Index map symbols must be listed in order with
-symbols for dynamic dimension sizes first, followed by other required symbols.
-
-##### Layout Map
-
-A layout map is a [semi-affine map](Dialects/Affine.md#semi-affine-maps) which
-encodes logical to physical index space mapping, by mapping input dimensions to
-their ordering from most-major (slowest varying) to most-minor (fastest
-varying). Therefore, an identity layout map corresponds to a row-major layout.
-Identity layout maps do not contribute to the MemRef type identification and are
-discarded on construction. That is, a type with an explicit identity map is
-`memref<?x?xf32, (i,j)->(i,j)>` is strictly the same as the one without layout
-maps, `memref<?x?xf32>`.
-
-Layout map examples:
-
-```mlir
-// MxN matrix stored in row major layout in memory:
-#layout_map_row_major = (i, j) -> (i, j)
-
-// MxN matrix stored in column major layout in memory:
-#layout_map_col_major = (i, j) -> (j, i)
-
-// MxN matrix stored in a 2-d blocked/tiled layout with 64x64 tiles.
-#layout_tiled = (i, j) -> (i floordiv 64, j floordiv 64, i mod 64, j mod 64)
-```
-
-##### Affine Map Composition
-
-A memref specifies a semi-affine map composition as part of its type. A
-semi-affine map composition is a composition of semi-affine maps beginning with
-zero or more index maps, and ending with a layout map. The composition must be
-conformant: the number of dimensions of the range of one map, must match the
-number of dimensions of the domain of the next map in the composition.
-
-The semi-affine map composition specified in the memref type, maps from accesses
-used to index the memref in load/store operations to other index spaces (i.e.
-logical to physical index mapping). Each of the
-[semi-affine maps](Dialects/Affine.md) and thus its composition is required to
-be one-to-one.
-
-The semi-affine map composition can be used in dependence analysis, memory
-access pattern analysis, and for performance optimizations like vectorization,
-copy elision and in-place updates. If an affine map composition is not specified
-for the memref, the identity affine map is assumed.
-
-##### Strided MemRef
-
-A memref may specify strides as part of its type. A stride specification is a
-list of integer values that are either static or `?` (dynamic case). Strides
-encode the distance, in number of elements, in (linear) memory between
-successive entries along a particular dimension. A stride specification is
-syntactic sugar for an equivalent strided memref representation using
-semi-affine maps. For example, `memref<42x16xf32, offset: 33, strides: [1, 64]>`
-specifies a non-contiguous memory region of `42` by `16` `f32` elements such
-that:
-
-1.  the minimal size of the enclosing memory region must be `33 + 42 * 1 + 16 *
-    64 = 1066` elements;
-2.  the address calculation for accessing element `(i, j)` computes `33 + i +
-    64 * j`
-3.  the distance between two consecutive elements along the inner dimension is
-    `1` element and the distance between two consecutive elements along the
-    outer dimension is `64` elements.
-
-This corresponds to a column major view of the memory region and is internally
-represented as the type `memref<42x16xf32, (i, j) -> (33 + i + 64 * j)>`.
-
-The specification of strides must not alias: given an n-D strided memref,
-indices `(i1, ..., in)` and `(j1, ..., jn)` may not refer to the same memory
-address unless `i1 == j1, ..., in == jn`.
-
-Strided memrefs represent a view abstraction over preallocated data. They are
-constructed with special ops, yet to be introduced. Strided memrefs are a
-special subclass of memrefs with generic semi-affine map and correspond to a
-normalized memref descriptor when lowering to LLVM.
-
-#### None Type
-
-Syntax:
-
-```
-none-type ::= `none`
-```
-
-The `none` type is a unit type, i.e. a type with exactly one possible value,
-where its value does not have a defined dynamic representation.
-
-#### Tensor Type
-
-Syntax:
-
-```
-tensor-type ::= `tensor` `<` dimension-list type `>`
-
-dimension-list ::= dimension-list-ranked | (`*` `x`)
-dimension-list-ranked ::= (dimension `x`)*
-dimension ::= `?` | decimal-literal
-```
-
-Values with tensor type represents aggregate N-dimensional data values, and
-have a known element type. It may have an unknown rank (indicated by `*`) or may
-have a fixed rank with a list of dimensions. Each dimension may be a static
-non-negative decimal constant or be dynamically determined (indicated by `?`).
-
-The runtime representation of the MLIR tensor type is intentionally abstracted -
-you cannot control layout or get a pointer to the data. For low level buffer
-access, MLIR has a [`memref` type](#memref-type). This abstracted runtime
-representation holds both the tensor data values as well as information about
-the (potentially dynamic) shape of the tensor. The
-[`dim` operation](Dialects/Standard.md#dim-operation) returns the size of a
-dimension from a value of tensor type.
-
-Note: hexadecimal integer literals are not allowed in tensor type declarations
-to avoid confusion between `0xf32` and `0 x f32`. Zero sizes are allowed in
-tensors and treated as other sizes, e.g., `tensor<0 x 1 x i32>` and `tensor<1 x
-0 x i32>` are different types. Since zero sizes are not allowed in some other
-types, such tensors should be optimized away before lowering tensors to vectors.
-
-Examples:
-
-```mlir
-// Tensor with unknown rank.
-tensor<* x f32>
-
-// Known rank but unknown dimensions.
-tensor<? x ? x ? x ? x f32>
-
-// Partially known dimensions.
-tensor<? x ? x 13 x ? x f32>
-
-// Full static shape.
-tensor<17 x 4 x 13 x 4 x f32>
-
-// Tensor with rank zero. Represents a scalar.
-tensor<f32>
-
-// Zero-element dimensions are allowed.
-tensor<0 x 42 x f32>
-
-// Zero-element tensor of f32 type (hexadecimal literals not allowed here).
-tensor<0xf32>
-```
-
-#### Tuple Type
-
-Syntax:
-
-```
-tuple-type ::= `tuple` `<` (type ( `,` type)*)? `>`
-```
-
-The value of `tuple` type represents a fixed-size collection of elements, where
-each element may be of a different type.
-
-**Rationale:** Though this type is first class in the type system, MLIR provides
-no standard operations for operating on `tuple` types
-([rationale](Rationale/Rationale.md#tuple-types)).
-
-Examples:
-
-```mlir
-// Empty tuple.
-tuple<>
-
-// Single element
-tuple<f32>
-
-// Many elements.
-tuple<i32, f32, tensor<i1>, i5>
-```
-
-#### Vector Type
-
-Syntax:
-
-```
-vector-type ::= `vector` `<` static-dimension-list vector-element-type `>`
-vector-element-type ::= float-type | integer-type
-
-static-dimension-list ::= (decimal-literal `x`)+
-```
-
-The vector type represents a SIMD style vector, used by target-specific
-operation sets like AVX. While the most common use is for 1D vectors (e.g.
-vector<16 x f32>) we also support multidimensional registers on targets that
-support them (like TPUs).
-
-Vector shapes must be positive decimal integers.
-
-Note: hexadecimal integer literals are not allowed in vector type declarations,
-`vector<0x42xi32>` is invalid because it is interpreted as a 2D vector with
-shape `(0, 42)` and zero shapes are not allowed.
+The [builtin dialect](Dialects/Builtin.md) defines a set of types that are
+directly usable by any other dialect in MLIR. These types cover a range from
+primitive integer and floating-point types, function types, and more.
 
 ## Attributes
 
@@ -1401,263 +834,7 @@ attribute values.
 
 ### Builtin Attribute Values
 
-Builtin attributes are a core set of
-[dialect attribute values](#dialect-attribute-values) that are defined in a
-builtin dialect and thus available to all users of MLIR.
-
-```
-builtin-attribute ::=    affine-map-attribute
-                       | array-attribute
-                       | bool-attribute
-                       | dictionary-attribute
-                       | elements-attribute
-                       | float-attribute
-                       | integer-attribute
-                       | integer-set-attribute
-                       | string-attribute
-                       | symbol-ref-attribute
-                       | type-attribute
-                       | unit-attribute
-```
-
-#### AffineMap Attribute
-
-Syntax:
-
-```
-affine-map-attribute ::= `affine_map` `<` affine-map `>`
-```
-
-An affine-map attribute is an attribute that represents an affine-map object.
-
-#### Array Attribute
-
-Syntax:
-
-```
-array-attribute ::= `[` (attribute-value (`,` attribute-value)*)? `]`
-```
-
-An array attribute is an attribute that represents a collection of attribute
-values.
-
-#### Boolean Attribute
-
-Syntax:
-
-```
-bool-attribute ::= bool-literal
-```
-
-A boolean attribute is a literal attribute that represents a one-bit boolean
-value, true or false.
-
-#### Dictionary Attribute
-
-Syntax:
-
-```
-dictionary-attribute ::= `{` (attribute-entry (`,` attribute-entry)*)? `}`
-```
-
-A dictionary attribute is an attribute that represents a sorted collection of
-named attribute values. The elements are sorted by name, and each name must be
-unique within the collection.
-
-#### Elements Attributes
-
-Syntax:
-
-```
-elements-attribute ::= dense-elements-attribute
-                     | opaque-elements-attribute
-                     | sparse-elements-attribute
-```
-
-An elements attribute is a literal attribute that represents a constant
-[vector](#vector-type) or [tensor](#tensor-type) value.
-
-##### Dense Elements Attribute
-
-Syntax:
-
-```
-dense-elements-attribute ::= `dense` `<` attribute-value `>` `:`
-                             ( tensor-type | vector-type )
-```
-
-A dense elements attribute is an elements attribute where the storage for the
-constant vector or tensor value has been densely packed. The attribute supports
-storing integer or floating point elements, with integer/index/floating element
-types. It also support storing string elements with a custom dialect string
-element type.
-
-##### Opaque Elements Attribute
-
-Syntax:
-
-```
-opaque-elements-attribute ::= `opaque` `<` dialect-namespace  `,`
-                              hex-string-literal `>` `:`
-                              ( tensor-type | vector-type )
-```
-
-An opaque elements attribute is an elements attribute where the content of the
-value is opaque. The representation of the constant stored by this elements
-attribute is only understood, and thus decodable, by the dialect that created
-it.
-
-Note: The parsed string literal must be in hexadecimal form.
-
-##### Sparse Elements Attribute
-
-Syntax:
-
-```
-sparse-elements-attribute ::= `sparse` `<` attribute-value `,` attribute-value
-                              `>` `:` ( tensor-type | vector-type )
-```
-
-A sparse elements attribute is an elements attribute that represents a sparse
-vector or tensor object. This is where very few of the elements are non-zero.
-
-The attribute uses COO (coordinate list) encoding to represent the sparse
-elements of the elements attribute. The indices are stored via a 2-D tensor of
-64-bit integer elements with shape [N, ndims], which specifies the indices of
-the elements in the sparse tensor that contains non-zero values. The element
-values are stored via a 1-D tensor with shape [N], that supplies the
-corresponding values for the indices.
-
-Example:
-
-```mlir
-  sparse<[[0, 0], [1, 2]], [1, 5]> : tensor<3x4xi32>
-
-// This represents the following tensor:
-///  [[1, 0, 0, 0],
-///   [0, 0, 5, 0],
-///   [0, 0, 0, 0]]
-```
-
-#### Float Attribute
-
-Syntax:
-
-```
-float-attribute ::= (float-literal (`:` float-type)?)
-                  | (hexadecimal-literal `:` float-type)
-```
-
-A float attribute is a literal attribute that represents a floating point value
-of the specified [float type](#floating-point-types). It can be represented in
-the hexadecimal form where the hexadecimal value is interpreted as bits of the
-underlying binary representation. This form is useful for representing infinity
-and NaN floating point values. To avoid confusion with integer attributes,
-hexadecimal literals _must_ be followed by a float type to define a float
-attribute.
-
-Examples:
-
-```
-42.0         // float attribute defaults to f64 type
-42.0 : f32   // float attribute of f32 type
-0x7C00 : f16 // positive infinity
-0x7CFF : f16 // NaN (one of possible values)
-42 : f32     // Error: expected integer type
-```
-
-#### Integer Attribute
-
-Syntax:
-
-```
-integer-attribute ::= integer-literal ( `:` (index-type | integer-type) )?
-```
-
-An integer attribute is a literal attribute that represents an integral value of
-the specified integer or index type. The default type for this attribute, if one
-is not specified, is a 64-bit integer.
-
-##### Integer Set Attribute
-
-Syntax:
-
-```
-integer-set-attribute ::= `affine_set` `<` integer-set `>`
-```
-
-An integer-set attribute is an attribute that represents an integer-set object.
-
-#### String Attribute
-
-Syntax:
-
-```
-string-attribute ::= string-literal (`:` type)?
-```
-
-A string attribute is an attribute that represents a string literal value.
-
-#### Symbol Reference Attribute
-
-Syntax:
-
-```
-symbol-ref-attribute ::= symbol-ref-id (`::` symbol-ref-id)*
-```
-
-A symbol reference attribute is a literal attribute that represents a named
-reference to an operation that is nested within an operation with the
-`OpTrait::SymbolTable` trait. As such, this reference is given meaning by the
-nearest parent operation containing the `OpTrait::SymbolTable` trait. It may
-optionally contain a set of nested references that further resolve to a symbol
-nested within a different symbol table.
-
-This attribute can only be held internally by
-[array attributes](#array-attribute) and
-[dictionary attributes](#dictionary-attribute)(including the top-level operation
-attribute dictionary), i.e. no other attribute kinds such as Locations or
-extended attribute kinds.
-
-**Rationale:** Identifying accesses to global data is critical to
-enabling efficient multi-threaded compilation. Restricting global
-data access to occur through symbols and limiting the places that can
-legally hold a symbol reference simplifies reasoning about these data
-accesses.
-
-See [`Symbols And SymbolTables`](SymbolsAndSymbolTables.md) for more
-information.
-
-#### Type Attribute
-
-Syntax:
-
-```
-type-attribute ::= type
-```
-
-A type attribute is an attribute that represents a [type object](#type-system).
-
-#### Unit Attribute
-
-```
-unit-attribute ::= `unit`
-```
-
-A unit attribute is an attribute that represents a value of `unit` type. The
-`unit` type allows only one value forming a singleton set. This attribute value
-is used to represent attributes that only have meaning from their existence.
-
-One example of such an attribute could be the `swift.self` attribute. This
-attribute indicates that a function parameter is the self/context parameter. It
-could be represented as a [boolean attribute](#boolean-attribute)(true or
-false), but a value of false doesn't really bring any value. The parameter
-either is the self/context or it isn't.
-
-```mlir
-// A unit attribute defined with the `unit` value specifier.
-func @verbose_form(i1) attributes {dialectName.unitAttr = unit}
-
-// A unit attribute can also be defined without the value specifier.
-func @simple_form(i1) attributes {dialectName.unitAttr}
-```
+The [builtin dialect](Dialects/Builtin.md) defines a set of attribute values
+that are directly usable by any other dialect in MLIR. These types cover a range
+from primitive integer and floating-point values, attribute dictionaries, dense
+multi-dimensional arrays, and more.
diff --git a/mlir/docs/OpDefinitions.md b/mlir/docs/OpDefinitions.md
index 4128543974b3f9310b12f25c4f9ab96fd57103e2..e2203ff467fdb9141438b8a47e3c8f2a78990cdf 100644
--- a/mlir/docs/OpDefinitions.md
+++ b/mlir/docs/OpDefinitions.md
@@ -772,8 +772,13 @@ When a variable is optional, the provided value may be null.
 In certain situations operations may have "optional" information, e.g.
 attributes or an empty set of variadic operands. In these situations a section
 of the assembly format can be marked as `optional` based on the presence of this
-information. An optional group is defined by wrapping a set of elements within
-`()` followed by a `?` and has the following requirements:
+information. An optional group is defined as follows:
+
+```
+optional-group: `(` elements `)` (`:` `(` else-elements `)`)? `?`
+```
+
+The `elements` of an optional group have the following requirements:
 
 *   The first element of the group must either be a attribute, literal, operand,
     or region.
@@ -837,6 +842,32 @@ foo.op is_read_only
 foo.op
 ```
 
+##### Optional "else" Group
+
+Optional groups also have support for an "else" group of elements. These are
+elements that are parsed/printed if the `anchor` element of the optional group
+is *not* present. Unlike the main element group, the "else" group has no
+restriction on the first element and none of the elements may act as the
+`anchor` for the optional. An example is shown below:
+
+```tablegen
+def FooOp : ... {
+  let arguments = (ins UnitAttr:$foo);
+
+  let assemblyFormat = "attr-dict (`foo_is_present` $foo^):(`foo_is_absent`)?";
+}
+```
+
+would be formatted as such:
+
+```mlir
+// When the `foo` attribute is present:
+foo.op foo_is_present
+
+// When the `foo` attribute is not present:
+foo.op foo_is_absent
+```
+
 #### Requirements
 
 The format specification has a certain set of requirements that must be adhered
@@ -888,6 +919,13 @@ This boolean field indicate whether canonicalization patterns have been defined
 for this operation. If it is `1`, then `::getCanonicalizationPatterns()` should
 be defined.
 
+### `hasCanonicalizeMethod`
+
+When this boolean field is set to `true`, it indicates that the op implements a
+`canonicalize` method for simple "matchAndRewrite" style canonicalization
+patterns.  If `hasCanonicalizer` is 0, then an implementation of
+`::getCanonicalizationPatterns()` is implemented to call this function.
+
 ### `hasFolder`
 
 This boolean field indicate whether general folding rules have been defined for
@@ -1441,8 +1479,9 @@ need allocation in the storage constructor, there are two options:
 ### TypeParameter tablegen class
 
 This is used to further specify attributes about each of the types parameters.
-It includes documentation (`summary` and `syntax`), the C++ type to use, and a
-custom allocator to use in the storage constructor method.
+It includes documentation (`summary` and `syntax`), the C++ type to use, a
+custom allocator to use in the storage constructor method, and a custom
+comparator to decide if two instances of the parameter type are equal.
 
 ```tablegen
 // DO NOT DO THIS!
@@ -1472,6 +1511,11 @@ The `allocator` code block has the following substitutions:
 -   `$_allocator` is the TypeStorageAllocator in which to allocate objects.
 -   `$_dst` is the variable in which to place the allocated data.
 
+The `comparator` code block has the following substitutions:
+
+-   `$_lhs` is an instance of the parameter type.
+-   `$_rhs` is an instance of the parameter type.
+
 MLIR includes several specialized classes for common situations:
 
 -   `StringRefParameter<descriptionOfParam>` for StringRefs.
diff --git a/mlir/docs/PatternRewriter.md b/mlir/docs/PatternRewriter.md
index 590c9ffc95a1be37155afd4724c4c0cac26c9dd4..f452230584c0dbfe9375257c242c2d941ce1a615 100644
--- a/mlir/docs/PatternRewriter.md
+++ b/mlir/docs/PatternRewriter.md
@@ -154,10 +154,10 @@ creation, as well as many useful attribute and type construction methods.
 After a set of patterns have been defined, they are collected and provided to a
 specific driver for application. A driver consists of several high levels parts:
 
-*   Input `OwningRewritePatternList`
+*   Input `RewritePatternSet`
 
 The input patterns to a driver are provided in the form of an
-`OwningRewritePatternList`. This class provides a simplified API for building a
+`RewritePatternSet`. This class provides a simplified API for building a
 list of patterns.
 
 *   Driver-specific `PatternRewriter`
@@ -173,7 +173,7 @@ mutation directly.
 Each driver is responsible for defining its own operation visitation order as
 well as pattern cost model, but the final application is performed via a
 `PatternApplicator` class. This class takes as input the
-`OwningRewritePatternList` and transforms the patterns based upon a provided
+`RewritePatternSet` and transforms the patterns based upon a provided
 cost model. This cost model computes a final benefit for a given pattern, using
 whatever driver specific information necessary. After a cost model has been
 computed, the driver may begin to match patterns against operations using
@@ -189,8 +189,8 @@ public:
 };
 
 /// Populate the pattern list.
-void collectMyPatterns(OwningRewritePatternList &patterns, MLIRContext *ctx) {
-  patterns.insert<MyPattern>(/*benefit=*/1, ctx);
+void collectMyPatterns(RewritePatternSet &patterns, MLIRContext *ctx) {
+  patterns.add<MyPattern>(/*benefit=*/1, ctx);
 }
 
 /// Define a custom PatternRewriter for use by the driver.
@@ -203,7 +203,7 @@ public:
 
 /// Apply the custom driver to `op`.
 void applyMyPatternDriver(Operation *op,
-                          const OwningRewritePatternList &patterns) {
+                          const RewritePatternSet &patterns) {
   // Initialize the custom PatternRewriter.
   MyPatternRewriter rewriter(op->getContext());
 
diff --git a/mlir/docs/Traits.md b/mlir/docs/Traits.md
index 8a21827578121d04c4a9bdbecf3eff179bd34198..add2c80f1d6d52b378c7d172fd4371e7c9aea163 100644
--- a/mlir/docs/Traits.md
+++ b/mlir/docs/Traits.md
@@ -323,13 +323,20 @@ index expression that can express the equivalent of the memory-layout
 specification of the MemRef type. See [the -normalize-memrefs pass].
 (https://mlir.llvm.org/docs/Passes/#-normalize-memrefs-normalize-memrefs)
 
+### Single Block Region
+
+*   `OpTrait::SingleBlock` -- `SingleBlock`
+
+This trait provides APIs and verifiers for operations with regions that have a
+single block.
+
 ### Single Block with Implicit Terminator
 
-*   `OpTrait::SingleBlockImplicitTerminator<typename TerminatorOpType>` :
+*   `OpTrait::SingleBlockImplicitTerminator<typename TerminatorOpType>` --
     `SingleBlockImplicitTerminator<string op>`
 
-This trait provides APIs and verifiers for operations with regions that have a
-single block that must terminate with `TerminatorOpType`.
+This trait implies the `SingleBlock` above, but adds the additional requirement
+that the single block must terminate with `TerminatorOpType`.
 
 ### SymbolTable
 
@@ -344,3 +351,10 @@ This trait is used for operations that define a
 
 This trait provides verification and functionality for operations that are known
 to be [terminators](LangRef.md#terminator-operations).
+
+*   `OpTrait::NoTerminator` -- `NoTerminator`
+
+This trait removes the requirement on regions held by an operation to have
+[terminator operations](LangRef.md#terminator-operations) at the end of a block.
+This requires that these regions have a single block. An example of operation
+using this trait is the top-level `ModuleOp`.
diff --git a/mlir/docs/Tutorials/QuickstartRewrites.md b/mlir/docs/Tutorials/QuickstartRewrites.md
index 447f8a62f91eb0a45955d6a8465f2ae17abdb187..54e67214a473580349820a563294c8b4167233b4 100644
--- a/mlir/docs/Tutorials/QuickstartRewrites.md
+++ b/mlir/docs/Tutorials/QuickstartRewrites.md
@@ -155,14 +155,65 @@ add_public_tablegen_target(<name-of-the-cmake-target>)
 Then you can `#include` the generated file in any C++ implementation file you
 like. (You will also need to make sure the library depends on the CMake target
 defined in the above.) The generated file will have a `populateWithGenerated(
-MLIRContext *context, OwningRewritePatternList &patterns)` function that you can
+RewritePatternSet &patterns)` function that you can
 use to collect all the generated patterns inside `patterns` and then use
 `patterns` in any pass you would like.
 
-### C++ rewrite specification
+### Simple C++ `matchAndRewrite` style specifications
 
-In case patterns are not sufficient there is also the fully C++ way of
-expressing a rewrite:
+Many simple rewrites can be expressed with a `matchAndRewrite` style  of
+pattern, e.g. when converting a multiply by a power of two into a shift.  For
+these cases, the you can define the pattern as a simple function:
+
+```c++
+static LogicalResult
+convertTFLeakyRelu(TFLeakyReluOp op, PatternRewriter &rewriter) {
+  rewriter.replaceOpWithNewOp<TFL::LeakyReluOp>(
+      op, op->getResult(0).getType(), op->getOperand(0),
+      /*alpha=*/op->getAttrOfType<FloatAttr>("alpha"));
+  return success();
+}
+
+void populateRewrites(RewritePatternSet &patternSet) {
+  // Add it to a pattern set.
+  patternSet.add(convertTFLeakyRelu);
+}
+```
+
+ODS provides a simple way to define a function-style canonicalization for your
+operation.  In the TableGen definition of the op, specify
+`let hasCanonicalizeMethod = 1;` and then implement the `canonicalize` method in
+your .cpp file:
+
+```c++
+// Example from the CIRCT project which has a variadic integer multiply.
+LogicalResult circt::MulOp::canonicalize(MulOp op, PatternRewriter &rewriter) {
+  auto inputs = op.inputs();
+  APInt value;
+
+  // mul(x, c) -> shl(x, log2(c)), where c is a power of two.
+  if (inputs.size() == 2 && matchPattern(inputs.back(), m_RConstant(value)) &&
+      value.isPowerOf2()) {
+    auto shift = rewriter.create<rtl::ConstantOp>(op.getLoc(), op.getType(),
+                                                  value.exactLogBase2());
+    auto shlOp =
+        rewriter.create<comb::ShlOp>(op.getLoc(), inputs[0], shift);
+    rewriter.replaceOpWithNewOp<MulOp>(op, op.getType(),
+                                       ArrayRef<Value>(shlOp));
+    return success();
+  }
+
+  return failure();
+}
+```
+
+However, you may want the full generality of canonicalization patterns, for that
+you can specify an arbitrary list of `RewritePattern`s.
+
+### Fully general C++ `RewritePattern` specifications
+
+In case ODS patterns and `matchAndRewrite`-style functions are not sufficient
+you can also specify rewrites as a general set of `RewritePattern`s:
 
 ```c++
 /// Multi-step rewrite using "match" and "rewrite". This allows for separating
@@ -189,7 +240,7 @@ struct ConvertTFLeakyRelu : public RewritePattern {
       : RewritePattern("tf.LeakyRelu", 1, context) {}
 
   LogicalResult matchAndRewrite(Operation *op,
-                                     PatternRewriter &rewriter) const override {
+                                PatternRewriter &rewriter) const override {
     rewriter.replaceOpWithNewOp<TFL::LeakyReluOp>(
         op, op->getResult(0).getType(), op->getOperand(0),
         /*alpha=*/op->getAttrOfType<FloatAttr>("alpha"));
diff --git a/mlir/docs/Tutorials/Toy/Ch-2.md b/mlir/docs/Tutorials/Toy/Ch-2.md
index f684494f1b18800709323946a1a7d4ecf7ba88da..8386804da8c62b37855013f8f3396ea9ed07a248 100644
--- a/mlir/docs/Tutorials/Toy/Ch-2.md
+++ b/mlir/docs/Tutorials/Toy/Ch-2.md
@@ -24,7 +24,7 @@ pre-defined instructions (*operations* in MLIR terminology) or types.
 
 ## Interfacing with MLIR
 
-[Language reference](../../LangRef.md)
+[Language Reference](../../LangRef.md)
 
 MLIR is designed to be a completely extensible infrastructure; there is no
 closed set of attributes (think: constant metadata), operations, or types. MLIR
@@ -115,14 +115,12 @@ compiler passes - does not include locations in the output by default. The
 
 ### Opaque API
 
-MLIR is designed to allow most IR elements, such as attributes,
-operations, and types, to be customized. At the same time, IR
-elements can always be reduced to the above fundamental concepts. This
-allows MLIR to parse, represent, and
-[round-trip](../../../getting_started/Glossary.md#round-trip) IR for
-*any* operation. For example, we could place our Toy operation from
-above into an `.mlir` file and round-trip through *mlir-opt* without
-registering any dialect:
+MLIR is designed to allow all IR elements, such as attributes, operations, and
+types, to be customized. At the same time, IR elements can always be reduced to
+the above fundamental concepts. This allows MLIR to parse, represent, and
+[round-trip](../../../getting_started/Glossary.md#round-trip) IR for *any*
+operation. For example, we could place our Toy operation from above into an
+`.mlir` file and round-trip through *mlir-opt* without registering any dialect:
 
 ```mlir
 func @toy_func(%tensor: tensor<2x3xf64>) -> tensor<3x2xf64> {
@@ -131,16 +129,15 @@ func @toy_func(%tensor: tensor<2x3xf64>) -> tensor<3x2xf64> {
 }
 ```
 
-In the cases of unregistered attributes, operations, and types, MLIR
-will enforce some structural constraints (SSA, block termination,
-etc.), but otherwise they are completely opaque. For instance, MLIR
-has little information about whether an unregistered operation can
-operate on particular datatypes, how many operands it can take, or how
-many results it produces. This flexibility can be useful for
-bootstrapping purposes, but it is generally advised against in mature
+In the cases of unregistered attributes, operations, and types, MLIR will
+enforce some structural constraints (e.g. dominance, etc.), but otherwise they
+are completely opaque. For instance, MLIR has little information about whether
+an unregistered operation can operate on particular data types, how many
+operands it can take, or how many results it produces. This flexibility can be
+useful for bootstrapping purposes, but it is generally advised against in mature
 systems. Unregistered operations must be treated conservatively by
-transformations and analyses, and they are much harder to construct
-and manipulate.
+transformations and analyses, and they are much harder to construct and
+manipulate.
 
 This handling can be observed by crafting what should be an invalid IR for Toy
 and seeing it round-trip without tripping the verifier:
@@ -159,38 +156,84 @@ verifier, and add nicer APIs to manipulate our operations.
 ## Defining a Toy Dialect
 
 To effectively interface with MLIR, we will define a new Toy dialect. This
-dialect will model the structure of the Toy language, as well as
-provide an easy avenue for high-level analysis and transformation.
+dialect will model the structure of the Toy language, as well as provide an easy
+avenue for high-level analysis and transformation.
 
 ```c++
 /// This is the definition of the Toy dialect. A dialect inherits from
-/// mlir::Dialect and registers custom attributes, operations, and types (in its
-/// constructor). It can also override virtual methods to change some general
-/// behavior, which will be demonstrated in later chapters of the tutorial.
+/// mlir::Dialect and registers custom attributes, operations, and types. It can
+/// also override virtual methods to change some general behavior, which will be
+/// demonstrated in later chapters of the tutorial.
 class ToyDialect : public mlir::Dialect {
- public:
+public:
   explicit ToyDialect(mlir::MLIRContext *ctx);
 
-  /// Provide a utility accessor to the dialect namespace. This is used by
-  /// several utilities.
+  /// Provide a utility accessor to the dialect namespace.
   static llvm::StringRef getDialectNamespace() { return "toy"; }
+
+  /// An initializer called from the constructor of ToyDialect that is used to
+  /// register attributes, operations, types, and more within the Toy dialect.
+  void initialize();
 };
 ```
 
-The dialect can now be registered in the global registry:
+This is the C++ definition of a dialect, but MLIR also supports defining
+dialects declaratively via
+[tablegen](https://llvm.org/docs/TableGen/ProgRef.html). Using the declarative
+specification is much cleaner as it removes the need for a large portion of the
+boilerplate when defining a new dialect. It also enables easy generation of
+dialect documentation, which can be described directly alongside the dialect. In
+this declarative format, the toy dialect would be specified as:
+
+```tablegen
+// Provide a definition of the 'toy' dialect in the ODS framework so that we
+// can define our operations.
+def Toy_Dialect : Dialect {
+  // The namespace of our dialect, this corresponds 1-1 with the string we
+  // provided in `ToyDialect::getDialectNamespace`.
+  let name = "toy";
+
+  // A short one-line summary of our dialect.
+  let summary = "A high-level dialect for analyzing and optimizing the "
+                "Toy language";
+
+  // A much longer description of our dialect.
+  let description = [{
+    The Toy language is a tensor-based language that allows you to define
+    functions, perform some math computation, and print results. This dialect
+    provides a representation of the language that is amenable to analysis and
+    optimization.
+  }];
+
+  // The C++ namespace that the dialect class definition resides in.
+  let cppNamespace = "toy";
+}
+```
+
+To see what this generates, we can run the `mlir-tblgen` command with the
+`gen-dialect-decls` action like so:
+
+```shell
+${build_root}/bin/mlir-tblgen -gen-dialect-decls ${mlir_src_root}/examples/toy/Ch2/include/toy/Ops.td -I ${mlir_src_root}/include/
+```
+
+After the dialect has been defined, it can now be loaded into an MLIRContext:
 
 ```c++
-  mlir::registerDialect<ToyDialect>();
+  context.loadDialect<ToyDialect>();
 ```
 
-Any new `MLIRContext` created from now on will contain an instance of the Toy
-dialect and invoke specific hooks for things like parsing attributes and types.
+By default, an `MLIRContext` only loads the
+[Builtin Dialect](../../Dialects/Builtin.md), which provides a few core IR
+components, meaning that other dialects, such as our `Toy` dialect, must be
+explicitly loaded.
 
 ## Defining Toy Operations
 
-Now that we have a `Toy` dialect, we can start registering operations. This will
-allow for providing semantic information that the rest of the system can hook
-into. Let's walk through the creation of the `toy.constant` operation:
+Now that we have a `Toy` dialect, we can start defining the operations. This
+will allow for providing semantic information that the rest of the system can
+hook into. As an example, let's walk through the creation of a `toy.constant`
+operation. This operation will represent a constant value in the Toy language.
 
 ```mlir
  %4 = "toy.constant"() {value = dense<1.0> : tensor<2x3xf64>} : () -> tensor<2x3xf64>
@@ -198,41 +241,50 @@ into. Let's walk through the creation of the `toy.constant` operation:
 
 This operation takes zero operands, a
 [dense elements](../../LangRef.md#dense-elements-attribute) attribute named
-`value`, and returns a single result of
-[TensorType](../../LangRef.md#tensor-type). An operation inherits from the
+`value` to represent the constant value, and returns a single result of
+[TensorType](../../LangRef.md#tensor-type). An operation class inherits from the
 [CRTP](https://en.wikipedia.org/wiki/Curiously_recurring_template_pattern)
 `mlir::Op` class which also takes some optional [*traits*](../../Traits.md) to
-customize its behavior. These traits may provide additional accessors,
-verification, etc.
+customize its behavior. `Traits` are a mechanism with which we can inject
+additional behavior into an Operation, such as additional accessors,
+verification, and more. Let's look below at a possible definition for the
+constant operation that we have described above:
 
 ```c++
-class ConstantOp : public mlir::Op<ConstantOp,
-                     /// The ConstantOp takes no inputs.
+class ConstantOp : public mlir::Op<
+                     /// `mlir::Op` is a CRTP class, meaning that we provide the
+                     /// derived class as a template parameter.
+                     ConstantOp,
+                     /// The ConstantOp takes zero input operands.
                      mlir::OpTrait::ZeroOperands,
                      /// The ConstantOp returns a single result.
                      mlir::OpTrait::OneResult,
-                     /// The result of getType is `Type`.
-                     mlir::OpTraits::OneTypedResult<Type>::Impl> {
+                     /// We also provide a utility `getType` accessor that
+                     /// returns the TensorType of the single result.
+                     mlir::OpTraits::OneTypedResult<TensorType>::Impl> {
 
  public:
   /// Inherit the constructors from the base Op class.
   using Op::Op;
 
   /// Provide the unique name for this operation. MLIR will use this to register
-  /// the operation and uniquely identify it throughout the system.
+  /// the operation and uniquely identify it throughout the system. The name
+  /// provided here must be prefixed by the parent dialect namespace followed
+  /// by a `.`.
   static llvm::StringRef getOperationName() { return "toy.constant"; }
 
   /// Return the value of the constant by fetching it from the attribute.
   mlir::DenseElementsAttr getValue();
 
-  /// Operations can provide additional verification beyond the traits they
-  /// define. Here we will ensure that the specific invariants of the constant
-  /// operation are upheld, for example the result type must be of TensorType.
+  /// Operations may provide additional verification beyond what the attached
+  /// traits provide.  Here we will ensure that the specific invariants of the
+  /// constant operation are upheld, for example the result type must be
+  /// of TensorType and matches the type of the constant `value`.
   LogicalResult verify();
 
   /// Provide an interface to build this operation from a set of input values.
-  /// This interface is used by the builder to allow for easily generating
-  /// instances of this operation:
+  /// This interface is used by the `builder` classes to allow for easily
+  /// generating instances of this operation:
   ///   mlir::OpBuilder::create<ConstantOp>(...)
   /// This method populates the given `state` that MLIR uses to create
   /// operations. This state is a collection of all of the discrete elements
@@ -249,39 +301,35 @@ class ConstantOp : public mlir::Op<ConstantOp,
 };
 ```
 
-and we register this operation in the `ToyDialect` constructor:
+and we can register this operation in the `ToyDialect` initializer:
 
 ```c++
-ToyDialect::ToyDialect(mlir::MLIRContext *ctx)
-    : mlir::Dialect(getDialectNamespace(), ctx) {
+void ToyDialect::initialize() {
   addOperations<ConstantOp>();
 }
 ```
 
 ### Op vs Operation: Using MLIR Operations
 
-Now that we have defined an operation, we will want to access and
-transform it.  In MLIR, there are two main classes related to
-operations: `Operation` and `Op`.  The `Operation` class is used to
-generically model all operations.  It is 'opaque', in the sense that
-it does not describe the properties of particular operations or types
-of operations.  Instead, the 'Operation' class provides a general API
-into an operation instance.  On the other hand, each specific type of
-operation is represented by an `Op` derived class.  For instance
-`ConstantOp` represents a operation with zero inputs, and one output,
-which is always set to the same value.  `Op` derived classes act as
-smart pointer wrapper around a `Operation*`, provide
-operation-specific accessor methods, and type-safe properties of
-operations. This means that when we define our Toy operations, we are
-simply defining a clean, semantically useful interface for building
-and interfacing with the `Operation` class.  This is why our
-`ConstantOp` defines no class fields; all the data structures are
-stored in the referenced `Operation`.  A side effect is that we always
-pass around `Op` derived classes by value, instead of by reference or
-pointer (*passing by value* is a common idiom and applies similarly to
-attributes, types, etc).  Given a generic `Operation*` instance, we
-can always get a specific `Op` instance using LLVM's casting
-infrastructure:
+Now that we have defined an operation, we will want to access and transform it.
+In MLIR, there are two main classes related to operations: `Operation` and `Op`.
+The `Operation` class is used to generically model all operations. It is
+'opaque', in the sense that it does not describe the properties of particular
+operations or types of operations. Instead, the `Operation` class provides a
+general API into an operation instance. On the other hand, each specific type of
+operation is represented by an `Op` derived class. For instance `ConstantOp`
+represents a operation with zero inputs, and one output, which is always set to
+the same value. `Op` derived classes act as smart pointer wrapper around a
+`Operation*`, provide operation-specific accessor methods, and type-safe
+properties of operations. This means that when we define our Toy operations, we
+are simply defining a clean, semantically useful interface for building and
+interfacing with the `Operation` class. This is why our `ConstantOp` defines no
+class fields; all of the data for this operation is stored in the referenced
+`Operation`. A side effect of this design is that we always pass around `Op`
+derived classes "by-value", instead of by reference or pointer (*passing by
+value* is a common idiom in MLIR and applies similarly to attributes, types,
+etc). Given a generic `Operation*` instance, we can always get a specific `Op`
+instance using LLVM's casting infrastructure:
 
 ```c++
 void processConstantOp(mlir::Operation *operation) {
@@ -311,27 +359,9 @@ C++ API changes.
 
 Lets see how to define the ODS equivalent of our ConstantOp:
 
-The first thing to do is to define a link to the Toy dialect that we defined in
-C++. This is used to link all of the operations that we will define to our
-dialect:
-
-```tablegen
-// Provide a definition of the 'toy' dialect in the ODS framework so that we
-// can define our operations.
-def Toy_Dialect : Dialect {
-  // The namespace of our dialect, this corresponds 1-1 with the string we
-  // provided in `ToyDialect::getDialectNamespace`.
-  let name = "toy";
-
-  // The C++ namespace that the dialect class definition resides in.
-  let cppNamespace = "toy";
-}
-```
-
-Now that we have defined a link to the Toy dialect, we can start defining
-operations. Operations in ODS are defined by inheriting from the `Op` class. To
-simplify our operation definitions, we will define a base class for operations
-in the Toy dialect.
+Operations in ODS are defined by inheriting from the `Op` class. To simplify our
+operation definitions, we will define a base class for operations in the Toy
+dialect.
 
 ```tablegen
 // Base class for toy dialect operations. This operation inherits from the base
diff --git a/mlir/docs/Tutorials/Toy/Ch-3.md b/mlir/docs/Tutorials/Toy/Ch-3.md
index 7976d7c30db59922af5b09b8bda69a22d029bc0f..abdb419f534fa9abcef47c640ee28ba58607f56d 100644
--- a/mlir/docs/Tutorials/Toy/Ch-3.md
+++ b/mlir/docs/Tutorials/Toy/Ch-3.md
@@ -114,8 +114,8 @@ pattern with the canonicalization framework.
 ```c++
 // Register our patterns for rewrite by the Canonicalization framework.
 void TransposeOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<SimplifyRedundantTranspose>(context);
+    RewritePatternSet &results, MLIRContext *context) {
+  results.add<SimplifyRedundantTranspose>(context);
 }
 ```
 
diff --git a/mlir/docs/Tutorials/Toy/Ch-4.md b/mlir/docs/Tutorials/Toy/Ch-4.md
index c454a762e2b52c7b8eec74b0387701e18407b03d..938e44289280a6733a6ef3772fe99263c2d31cdb 100644
--- a/mlir/docs/Tutorials/Toy/Ch-4.md
+++ b/mlir/docs/Tutorials/Toy/Ch-4.md
@@ -99,7 +99,7 @@ We then register our dialect interface directly on the Toy dialect, similarly to
 how we did for operations.
 
 ```c++
-ToyDialect::ToyDialect(mlir::MLIRContext *ctx) : mlir::Dialect("toy", ctx) {
+void ToyDialect::initialize() {
   addInterfaces<ToyInlinerInterface>();
 }
 ```
diff --git a/mlir/docs/Tutorials/Toy/Ch-5.md b/mlir/docs/Tutorials/Toy/Ch-5.md
index b8964f93e1a3d4267f0a57940388681d492e6d6d..ef95fa4bb4e5bcf8eb178d06c3d00454d4536d02 100644
--- a/mlir/docs/Tutorials/Toy/Ch-5.md
+++ b/mlir/docs/Tutorials/Toy/Ch-5.md
@@ -147,8 +147,8 @@ void ToyToAffineLoweringPass::runOnFunction() {
 
   // Now that the conversion target has been defined, we just need to provide
   // the set of patterns that will lower the Toy operations.
-  mlir::OwningRewritePatternList patterns;
-  patterns.insert<..., TransposeOpLowering>(&getContext());
+  mlir::RewritePatternSet patterns(&getContext());
+  patterns.add<..., TransposeOpLowering>(&getContext());
 
   ...
 ```
diff --git a/mlir/docs/Tutorials/Toy/Ch-6.md b/mlir/docs/Tutorials/Toy/Ch-6.md
index bddd93688ddb8b6b44c0a79f5365ad42e1187e5e..9c5d838c447c13008b22881ff935b9fafa3b71c2 100644
--- a/mlir/docs/Tutorials/Toy/Ch-6.md
+++ b/mlir/docs/Tutorials/Toy/Ch-6.md
@@ -63,7 +63,7 @@ everything to the LLVM dialect.
 ```c++
   mlir::ConversionTarget target(getContext());
   target.addLegalDialect<mlir::LLVMDialect>();
-  target.addLegalOp<mlir::ModuleOp, mlir::ModuleTerminatorOp>();
+  target.addLegalOp<mlir::ModuleOp>();
 ```
 
 ### Type Converter
@@ -90,14 +90,14 @@ into LLVM dialect. These patterns allow for lowering the IR in multiple stages
 by relying on [transitive lowering](../../../getting_started/Glossary.md#transitive-lowering).
 
 ```c++
-  mlir::OwningRewritePatternList patterns;
+  mlir::RewritePatternSet patterns(&getContext());
   mlir::populateAffineToStdConversionPatterns(patterns, &getContext());
   mlir::populateLoopToStdConversionPatterns(patterns, &getContext());
   mlir::populateStdToLLVMConversionPatterns(typeConverter, patterns);
 
   // The only remaining operation, to lower from the `toy` dialect, is the
   // PrintOp.
-  patterns.insert<PrintOpLowering>(&getContext());
+  patterns.add<PrintOpLowering>(&getContext());
 ```
 
 ### Full Lowering
diff --git a/mlir/docs/Tutorials/Toy/Ch-7.md b/mlir/docs/Tutorials/Toy/Ch-7.md
index 315cf3237b407eb0b4c7c70ea361c7fac4bac37c..72dc8440e9ef39fc09ebbd5ef8bbbeca4d123a5e 100644
--- a/mlir/docs/Tutorials/Toy/Ch-7.md
+++ b/mlir/docs/Tutorials/Toy/Ch-7.md
@@ -177,12 +177,11 @@ public:
 };
 ```
 
-We register this type in the `ToyDialect` constructor in a similar way to how we
+We register this type in the `ToyDialect` initializer in a similar way to how we
 did with operations:
 
 ```c++
-ToyDialect::ToyDialect(mlir::MLIRContext *ctx)
-    : mlir::Dialect(getDialectNamespace(), ctx) {
+void ToyDialect::initialize() {
   addTypes<StructType>();
 }
 ```
@@ -193,12 +192,32 @@ storage class must be visible.)
 With this we can now use our `StructType` when generating MLIR from Toy. See
 examples/toy/Ch7/mlir/MLIRGen.cpp for more details.
 
+### Exposing to ODS
+
+After defining a new type, we should make the ODS framework aware of our Type so
+that we can use it in the operation definitions and auto-generate utilities
+within the Dialect. A simple example is shown below:
+
+```tablegen
+// Provide a definition for the Toy StructType for use in ODS. This allows for
+// using StructType in a similar way to Tensor or MemRef. We use `DialectType`
+// to demarcate the StructType as belonging to the Toy dialect.
+def Toy_StructType :
+    DialectType<Toy_Dialect, CPred<"$_self.isa<StructType>()">,
+                "Toy struct type">;
+
+// Provide a definition of the types that are used within the Toy dialect.
+def Toy_Type : AnyTypeOf<[F64Tensor, Toy_StructType]>;
+```
+
 ### Parsing and Printing
 
 At this point we can use our `StructType` during MLIR generation and
 transformation, but we can't output or parse `.mlir`. For this we need to add
 support for parsing and printing instances of the `StructType`. This can be done
 by overriding the `parseType` and `printType` methods on the `ToyDialect`.
+Declarations for these methods are automatically provided when the type is
+exposed to ODS as detailed in the previous section.
 
 ```c++
 class ToyDialect : public mlir::Dialect {
@@ -321,22 +340,8 @@ the IR. The next step is to add support for using it within our operations.
 
 #### Updating Existing Operations
 
-A few of our existing operations will need to be updated to handle `StructType`.
-The first step is to make the ODS framework aware of our Type so that we can use
-it in the operation definitions. A simple example is shown below:
-
-```tablegen
-// Provide a definition for the Toy StructType for use in ODS. This allows for
-// using StructType in a similar way to Tensor or MemRef.
-def Toy_StructType :
-    Type<CPred<"$_self.isa<StructType>()">, "Toy struct type">;
-
-// Provide a definition of the types that are used within the Toy dialect.
-def Toy_Type : AnyTypeOf<[F64Tensor, Toy_StructType]>;
-```
-
-We can then update our operations, e.g. `ReturnOp`, to also accept the
-`Toy_StructType`:
+A few of our existing operations, e.g. `ReturnOp`, will need to be updated to
+handle `Toy_StructType`.
 
 ```tablegen
 def ReturnOp : Toy_Op<"return", [Terminator, HasParent<"FuncOp">]> {
diff --git a/mlir/docs/Tutorials/UnderstandingTheIRStructure.md b/mlir/docs/Tutorials/UnderstandingTheIRStructure.md
index 69560d29561862b961636ee790f0848562efa74e..3b32d6a409715d939ba33f20005b49fcf7f28a1e 100644
--- a/mlir/docs/Tutorials/UnderstandingTheIRStructure.md
+++ b/mlir/docs/Tutorials/UnderstandingTheIRStructure.md
@@ -110,7 +110,6 @@ llvm-project/mlir/test/IR/print-ir-nesting.mlir`:
     "dialect.innerop6"() : () -> ()
     "dialect.innerop7"() : () -> ()
   }) {"other attribute" = 42 : i64} : () -> ()
-  "module_terminator"() : () -> ()
 }) : () -> ()
 ```
 
@@ -147,7 +146,6 @@ visiting op: 'module' with 0 operands and 0 results
              0 nested regions:
             visiting op: 'dialect.innerop7' with 0 operands and 0 results
              0 nested regions:
-      visiting op: 'module_terminator' with 0 operands and 0 results
        0 nested regions:
 ```
 
diff --git a/mlir/examples/toy/Ch2/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch2/include/toy/CMakeLists.txt
index c08f78b0e8c8a93390fb46c401499687fbc232a0..26a0eb1f8e1b49cfc614e596ba6c7e21691dfad5 100644
--- a/mlir/examples/toy/Ch2/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch2/include/toy/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
 mlir_tablegen(Ops.h.inc -gen-op-decls)
 mlir_tablegen(Ops.cpp.inc -gen-op-defs)
+mlir_tablegen(Dialect.h.inc -gen-dialect-decls)
 add_public_tablegen_target(ToyCh2OpsIncGen)
diff --git a/mlir/examples/toy/Ch2/include/toy/Dialect.h b/mlir/examples/toy/Ch2/include/toy/Dialect.h
index b9f9627eb691edd88a3ec5809f7e6b737130b5d2..0abd0cb1a1465e58e2c77b83d1af80bfc718cf2f 100644
--- a/mlir/examples/toy/Ch2/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch2/include/toy/Dialect.h
@@ -18,24 +18,9 @@
 #include "mlir/IR/Dialect.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
-namespace mlir {
-namespace toy {
-
-/// This is the definition of the Toy dialect. A dialect inherits from
-/// mlir::Dialect and registers custom attributes, operations, and types (in its
-/// constructor). It can also override some general behavior exposed via virtual
-/// methods.
-class ToyDialect : public mlir::Dialect {
-public:
-  explicit ToyDialect(mlir::MLIRContext *ctx);
-
-  /// Provide a utility accessor to the dialect namespace. This is used by
-  /// several utilities for casting between dialects.
-  static llvm::StringRef getDialectNamespace() { return "toy"; }
-};
-
-} // end namespace toy
-} // end namespace mlir
+/// Include the auto-generated header file containing the declaration of the toy
+/// dialect.
+#include "toy/Dialect.h.inc"
 
 /// Include the auto-generated header file containing the declarations of the
 /// toy operations.
diff --git a/mlir/examples/toy/Ch2/mlir/Dialect.cpp b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
index bd23c779d6e993c1d45b3787ea015dd906a760e3..9327aaf8431a48a9c4124095ba7a7027afd13759 100644
--- a/mlir/examples/toy/Ch2/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
@@ -24,10 +24,9 @@ using namespace mlir::toy;
 // ToyDialect
 //===----------------------------------------------------------------------===//
 
-/// Dialect creation, the instance will be owned by the context. This is the
-/// point of registration of custom types and operations for the dialect.
-ToyDialect::ToyDialect(mlir::MLIRContext *ctx)
-    : mlir::Dialect(getDialectNamespace(), ctx, TypeID::get<ToyDialect>()) {
+/// Dialect initialization, the instance will be owned by the context. This is
+/// the point of registration of types and operations for the dialect.
+void ToyDialect::initialize() {
   addOperations<
 #define GET_OP_LIST
 #include "toy/Ops.cpp.inc"
diff --git a/mlir/examples/toy/Ch3/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch3/include/toy/CMakeLists.txt
index e76780c1f79f3901338b5f2b8e57c13265f7f387..15bb98ca9600ac606ee437c2f7ad180d69b1f72d 100644
--- a/mlir/examples/toy/Ch3/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch3/include/toy/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
 mlir_tablegen(Ops.h.inc -gen-op-decls)
 mlir_tablegen(Ops.cpp.inc -gen-op-defs)
+mlir_tablegen(Dialect.h.inc -gen-dialect-decls)
 add_public_tablegen_target(ToyCh3OpsIncGen)
diff --git a/mlir/examples/toy/Ch3/include/toy/Dialect.h b/mlir/examples/toy/Ch3/include/toy/Dialect.h
index b9f9627eb691edd88a3ec5809f7e6b737130b5d2..0abd0cb1a1465e58e2c77b83d1af80bfc718cf2f 100644
--- a/mlir/examples/toy/Ch3/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch3/include/toy/Dialect.h
@@ -18,24 +18,9 @@
 #include "mlir/IR/Dialect.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
-namespace mlir {
-namespace toy {
-
-/// This is the definition of the Toy dialect. A dialect inherits from
-/// mlir::Dialect and registers custom attributes, operations, and types (in its
-/// constructor). It can also override some general behavior exposed via virtual
-/// methods.
-class ToyDialect : public mlir::Dialect {
-public:
-  explicit ToyDialect(mlir::MLIRContext *ctx);
-
-  /// Provide a utility accessor to the dialect namespace. This is used by
-  /// several utilities for casting between dialects.
-  static llvm::StringRef getDialectNamespace() { return "toy"; }
-};
-
-} // end namespace toy
-} // end namespace mlir
+/// Include the auto-generated header file containing the declaration of the toy
+/// dialect.
+#include "toy/Dialect.h.inc"
 
 /// Include the auto-generated header file containing the declarations of the
 /// toy operations.
diff --git a/mlir/examples/toy/Ch3/mlir/Dialect.cpp b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
index bd23c779d6e993c1d45b3787ea015dd906a760e3..9327aaf8431a48a9c4124095ba7a7027afd13759 100644
--- a/mlir/examples/toy/Ch3/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
@@ -24,10 +24,9 @@ using namespace mlir::toy;
 // ToyDialect
 //===----------------------------------------------------------------------===//
 
-/// Dialect creation, the instance will be owned by the context. This is the
-/// point of registration of custom types and operations for the dialect.
-ToyDialect::ToyDialect(mlir::MLIRContext *ctx)
-    : mlir::Dialect(getDialectNamespace(), ctx, TypeID::get<ToyDialect>()) {
+/// Dialect initialization, the instance will be owned by the context. This is
+/// the point of registration of types and operations for the dialect.
+void ToyDialect::initialize() {
   addOperations<
 #define GET_OP_LIST
 #include "toy/Ops.cpp.inc"
diff --git a/mlir/examples/toy/Ch3/mlir/ToyCombine.cpp b/mlir/examples/toy/Ch3/mlir/ToyCombine.cpp
index 0af4cbfc11f12f1acd7fea6d3656319d457119b2..5e74d95d573b518109395b560dc8294dd08e83de 100644
--- a/mlir/examples/toy/Ch3/mlir/ToyCombine.cpp
+++ b/mlir/examples/toy/Ch3/mlir/ToyCombine.cpp
@@ -54,15 +54,15 @@ struct SimplifyRedundantTranspose : public mlir::OpRewritePattern<TransposeOp> {
 
 /// Register our patterns as "canonicalization" patterns on the TransposeOp so
 /// that they can be picked up by the Canonicalization framework.
-void TransposeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void TransposeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
-  results.insert<SimplifyRedundantTranspose>(context);
+  results.add<SimplifyRedundantTranspose>(context);
 }
 
 /// Register our patterns as "canonicalization" patterns on the ReshapeOp so
 /// that they can be picked up by the Canonicalization framework.
-void ReshapeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ReshapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
-  results.insert<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
-                 FoldConstantReshapeOptPattern>(context);
+  results.add<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
+              FoldConstantReshapeOptPattern>(context);
 }
diff --git a/mlir/examples/toy/Ch4/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch4/include/toy/CMakeLists.txt
index f293ff63fc6445201673a971fd301a112f8fa8c8..f2de562aaa3534bcd1855f6a3cd33ca5c2ab0083 100644
--- a/mlir/examples/toy/Ch4/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch4/include/toy/CMakeLists.txt
@@ -2,6 +2,7 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
 mlir_tablegen(Ops.h.inc -gen-op-decls)
 mlir_tablegen(Ops.cpp.inc -gen-op-defs)
+mlir_tablegen(Dialect.h.inc -gen-dialect-decls)
 add_public_tablegen_target(ToyCh4OpsIncGen)
 
 # Most dialects should use add_mlir_interfaces().
diff --git a/mlir/examples/toy/Ch4/include/toy/Dialect.h b/mlir/examples/toy/Ch4/include/toy/Dialect.h
index 41d20fa7de22d6db25abe1517978fed6293ddcda..41b95e834c3882dfde6476478dd1ebe20411019d 100644
--- a/mlir/examples/toy/Ch4/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch4/include/toy/Dialect.h
@@ -21,24 +21,9 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "toy/ShapeInferenceInterface.h"
 
-namespace mlir {
-namespace toy {
-
-/// This is the definition of the Toy dialect. A dialect inherits from
-/// mlir::Dialect and registers custom attributes, operations, and types (in its
-/// constructor). It can also override some general behavior exposed via virtual
-/// methods.
-class ToyDialect : public mlir::Dialect {
-public:
-  explicit ToyDialect(mlir::MLIRContext *ctx);
-
-  /// Provide a utility accessor to the dialect namespace. This is used by
-  /// several utilities for casting between dialects.
-  static llvm::StringRef getDialectNamespace() { return "toy"; }
-};
-
-} // end namespace toy
-} // end namespace mlir
+/// Include the auto-generated header file containing the declaration of the toy
+/// dialect.
+#include "toy/Dialect.h.inc"
 
 /// Include the auto-generated header file containing the declarations of the
 /// toy operations.
diff --git a/mlir/examples/toy/Ch4/mlir/Dialect.cpp b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
index face96461d0863943a13d46951ea670bb3cf5a2b..dd82e0409ab1649fee6fa570796e939bc42015a8 100644
--- a/mlir/examples/toy/Ch4/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
@@ -79,10 +79,9 @@ struct ToyInlinerInterface : public DialectInlinerInterface {
 // ToyDialect
 //===----------------------------------------------------------------------===//
 
-/// Dialect creation, the instance will be owned by the context. This is the
-/// point of registration of custom types and operations for the dialect.
-ToyDialect::ToyDialect(mlir::MLIRContext *ctx)
-    : mlir::Dialect(getDialectNamespace(), ctx, TypeID::get<ToyDialect>()) {
+/// Dialect initialization, the instance will be owned by the context. This is
+/// the point of registration of types and operations for the dialect.
+void ToyDialect::initialize() {
   addOperations<
 #define GET_OP_LIST
 #include "toy/Ops.cpp.inc"
diff --git a/mlir/examples/toy/Ch4/mlir/ToyCombine.cpp b/mlir/examples/toy/Ch4/mlir/ToyCombine.cpp
index 0af4cbfc11f12f1acd7fea6d3656319d457119b2..5e74d95d573b518109395b560dc8294dd08e83de 100644
--- a/mlir/examples/toy/Ch4/mlir/ToyCombine.cpp
+++ b/mlir/examples/toy/Ch4/mlir/ToyCombine.cpp
@@ -54,15 +54,15 @@ struct SimplifyRedundantTranspose : public mlir::OpRewritePattern<TransposeOp> {
 
 /// Register our patterns as "canonicalization" patterns on the TransposeOp so
 /// that they can be picked up by the Canonicalization framework.
-void TransposeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void TransposeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
-  results.insert<SimplifyRedundantTranspose>(context);
+  results.add<SimplifyRedundantTranspose>(context);
 }
 
 /// Register our patterns as "canonicalization" patterns on the ReshapeOp so
 /// that they can be picked up by the Canonicalization framework.
-void ReshapeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ReshapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
-  results.insert<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
-                 FoldConstantReshapeOptPattern>(context);
+  results.add<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
+              FoldConstantReshapeOptPattern>(context);
 }
diff --git a/mlir/examples/toy/Ch5/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch5/include/toy/CMakeLists.txt
index 66ceb13f66556aa36c78564011d65280a57c9c2d..10313c9d91ccd69dc7ff9ca70ebc2729ffee7e19 100644
--- a/mlir/examples/toy/Ch5/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch5/include/toy/CMakeLists.txt
@@ -2,6 +2,7 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
 mlir_tablegen(Ops.h.inc -gen-op-decls)
 mlir_tablegen(Ops.cpp.inc -gen-op-defs)
+mlir_tablegen(Dialect.h.inc -gen-dialect-decls)
 add_public_tablegen_target(ToyCh5OpsIncGen)
 
 # Most dialects should use add_mlir_interfaces().
diff --git a/mlir/examples/toy/Ch5/include/toy/Dialect.h b/mlir/examples/toy/Ch5/include/toy/Dialect.h
index 41d20fa7de22d6db25abe1517978fed6293ddcda..41b95e834c3882dfde6476478dd1ebe20411019d 100644
--- a/mlir/examples/toy/Ch5/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch5/include/toy/Dialect.h
@@ -21,24 +21,9 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "toy/ShapeInferenceInterface.h"
 
-namespace mlir {
-namespace toy {
-
-/// This is the definition of the Toy dialect. A dialect inherits from
-/// mlir::Dialect and registers custom attributes, operations, and types (in its
-/// constructor). It can also override some general behavior exposed via virtual
-/// methods.
-class ToyDialect : public mlir::Dialect {
-public:
-  explicit ToyDialect(mlir::MLIRContext *ctx);
-
-  /// Provide a utility accessor to the dialect namespace. This is used by
-  /// several utilities for casting between dialects.
-  static llvm::StringRef getDialectNamespace() { return "toy"; }
-};
-
-} // end namespace toy
-} // end namespace mlir
+/// Include the auto-generated header file containing the declaration of the toy
+/// dialect.
+#include "toy/Dialect.h.inc"
 
 /// Include the auto-generated header file containing the declarations of the
 /// toy operations.
diff --git a/mlir/examples/toy/Ch5/mlir/Dialect.cpp b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
index ce5901363be8f84f73db0daf5b121e85097741b6..18d5985042faa3d9a224b024298190a729e6857d 100644
--- a/mlir/examples/toy/Ch5/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
@@ -79,10 +79,9 @@ struct ToyInlinerInterface : public DialectInlinerInterface {
 // ToyDialect
 //===----------------------------------------------------------------------===//
 
-/// Dialect creation, the instance will be owned by the context. This is the
-/// point of registration of custom types and operations for the dialect.
-ToyDialect::ToyDialect(mlir::MLIRContext *ctx)
-    : mlir::Dialect(getDialectNamespace(), ctx, TypeID::get<ToyDialect>()) {
+/// Dialect initialization, the instance will be owned by the context. This is
+/// the point of registration of types and operations for the dialect.
+void ToyDialect::initialize() {
   addOperations<
 #define GET_OP_LIST
 #include "toy/Ops.cpp.inc"
diff --git a/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp b/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
index f2332fd5437fa66a8855bd1c0bd42296d8dd2251..8acbc37c77f7c2690d15107ef5dc4bb45b407e8e 100644
--- a/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
+++ b/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
@@ -297,9 +297,9 @@ void ToyToAffineLoweringPass::runOnFunction() {
 
   // Now that the conversion target has been defined, we just need to provide
   // the set of patterns that will lower the Toy operations.
-  OwningRewritePatternList patterns;
-  patterns.insert<AddOpLowering, ConstantOpLowering, MulOpLowering,
-                  ReturnOpLowering, TransposeOpLowering>(&getContext());
+  RewritePatternSet patterns(&getContext());
+  patterns.add<AddOpLowering, ConstantOpLowering, MulOpLowering,
+               ReturnOpLowering, TransposeOpLowering>(&getContext());
 
   // With the target and rewrite patterns defined, we can now attempt the
   // conversion. The conversion will signal failure if any of our `illegal`
diff --git a/mlir/examples/toy/Ch5/mlir/ToyCombine.cpp b/mlir/examples/toy/Ch5/mlir/ToyCombine.cpp
index 0af4cbfc11f12f1acd7fea6d3656319d457119b2..5e74d95d573b518109395b560dc8294dd08e83de 100644
--- a/mlir/examples/toy/Ch5/mlir/ToyCombine.cpp
+++ b/mlir/examples/toy/Ch5/mlir/ToyCombine.cpp
@@ -54,15 +54,15 @@ struct SimplifyRedundantTranspose : public mlir::OpRewritePattern<TransposeOp> {
 
 /// Register our patterns as "canonicalization" patterns on the TransposeOp so
 /// that they can be picked up by the Canonicalization framework.
-void TransposeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void TransposeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
-  results.insert<SimplifyRedundantTranspose>(context);
+  results.add<SimplifyRedundantTranspose>(context);
 }
 
 /// Register our patterns as "canonicalization" patterns on the ReshapeOp so
 /// that they can be picked up by the Canonicalization framework.
-void ReshapeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ReshapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
-  results.insert<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
-                 FoldConstantReshapeOptPattern>(context);
+  results.add<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
+              FoldConstantReshapeOptPattern>(context);
 }
diff --git a/mlir/examples/toy/Ch6/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch6/include/toy/CMakeLists.txt
index cc5b7469e7e1f053562525feceb4ee0e3b6f3c72..4c54020302a5d95eee66eecdc33c08742f90b5c9 100644
--- a/mlir/examples/toy/Ch6/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch6/include/toy/CMakeLists.txt
@@ -2,6 +2,7 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
 mlir_tablegen(Ops.h.inc -gen-op-decls)
 mlir_tablegen(Ops.cpp.inc -gen-op-defs)
+mlir_tablegen(Dialect.h.inc -gen-dialect-decls)
 add_public_tablegen_target(ToyCh6OpsIncGen)
 
 # Most dialects should use add_mlir_interfaces().
diff --git a/mlir/examples/toy/Ch6/include/toy/Dialect.h b/mlir/examples/toy/Ch6/include/toy/Dialect.h
index 41d20fa7de22d6db25abe1517978fed6293ddcda..41b95e834c3882dfde6476478dd1ebe20411019d 100644
--- a/mlir/examples/toy/Ch6/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch6/include/toy/Dialect.h
@@ -21,24 +21,9 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "toy/ShapeInferenceInterface.h"
 
-namespace mlir {
-namespace toy {
-
-/// This is the definition of the Toy dialect. A dialect inherits from
-/// mlir::Dialect and registers custom attributes, operations, and types (in its
-/// constructor). It can also override some general behavior exposed via virtual
-/// methods.
-class ToyDialect : public mlir::Dialect {
-public:
-  explicit ToyDialect(mlir::MLIRContext *ctx);
-
-  /// Provide a utility accessor to the dialect namespace. This is used by
-  /// several utilities for casting between dialects.
-  static llvm::StringRef getDialectNamespace() { return "toy"; }
-};
-
-} // end namespace toy
-} // end namespace mlir
+/// Include the auto-generated header file containing the declaration of the toy
+/// dialect.
+#include "toy/Dialect.h.inc"
 
 /// Include the auto-generated header file containing the declarations of the
 /// toy operations.
diff --git a/mlir/examples/toy/Ch6/mlir/Dialect.cpp b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
index ce5901363be8f84f73db0daf5b121e85097741b6..18d5985042faa3d9a224b024298190a729e6857d 100644
--- a/mlir/examples/toy/Ch6/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
@@ -79,10 +79,9 @@ struct ToyInlinerInterface : public DialectInlinerInterface {
 // ToyDialect
 //===----------------------------------------------------------------------===//
 
-/// Dialect creation, the instance will be owned by the context. This is the
-/// point of registration of custom types and operations for the dialect.
-ToyDialect::ToyDialect(mlir::MLIRContext *ctx)
-    : mlir::Dialect(getDialectNamespace(), ctx, TypeID::get<ToyDialect>()) {
+/// Dialect initialization, the instance will be owned by the context. This is
+/// the point of registration of types and operations for the dialect.
+void ToyDialect::initialize() {
   addOperations<
 #define GET_OP_LIST
 #include "toy/Ops.cpp.inc"
diff --git a/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp b/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
index 250ad387594f3e1320737f861cdc91a2a6bbb751..c1ad4dc66e9960d27af3ba1a922817ad810a1e3b 100644
--- a/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
+++ b/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
@@ -296,9 +296,9 @@ void ToyToAffineLoweringPass::runOnFunction() {
 
   // Now that the conversion target has been defined, we just need to provide
   // the set of patterns that will lower the Toy operations.
-  OwningRewritePatternList patterns;
-  patterns.insert<AddOpLowering, ConstantOpLowering, MulOpLowering,
-                  ReturnOpLowering, TransposeOpLowering>(&getContext());
+  RewritePatternSet patterns(&getContext());
+  patterns.add<AddOpLowering, ConstantOpLowering, MulOpLowering,
+               ReturnOpLowering, TransposeOpLowering>(&getContext());
 
   // With the target and rewrite patterns defined, we can now attempt the
   // conversion. The conversion will signal failure if any of our `illegal`
diff --git a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
index bbe3e54f8ac5af9ca7ccb5af98da9948dcae6c81..f334faa29ca8837a35170c7b00cd04ce9235308f 100644
--- a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
@@ -174,7 +174,7 @@ void ToyToLLVMLoweringPass::runOnOperation() {
   // final target for this lowering. For this lowering, we are only targeting
   // the LLVM dialect.
   LLVMConversionTarget target(getContext());
-  target.addLegalOp<ModuleOp, ModuleTerminatorOp>();
+  target.addLegalOp<ModuleOp>();
 
   // During this lowering, we will also be lowering the MemRef types, that are
   // currently being operated on, to a representation in LLVM. To perform this
@@ -191,14 +191,14 @@ void ToyToLLVMLoweringPass::runOnOperation() {
   // lowerings. Transitive lowering, or A->B->C lowering, is when multiple
   // patterns must be applied to fully transform an illegal operation into a
   // set of legal ones.
-  OwningRewritePatternList patterns;
-  populateAffineToStdConversionPatterns(patterns, &getContext());
-  populateLoopToStdConversionPatterns(patterns, &getContext());
+  RewritePatternSet patterns(&getContext());
+  populateAffineToStdConversionPatterns(patterns);
+  populateLoopToStdConversionPatterns(patterns);
   populateStdToLLVMConversionPatterns(typeConverter, patterns);
 
   // The only remaining operation to lower from the `toy` dialect, is the
   // PrintOp.
-  patterns.insert<PrintOpLowering>(&getContext());
+  patterns.add<PrintOpLowering>(&getContext());
 
   // We want to completely lower to LLVM, so we use a `FullConversion`. This
   // ensures that only legal operations will remain after the conversion.
diff --git a/mlir/examples/toy/Ch6/mlir/ToyCombine.cpp b/mlir/examples/toy/Ch6/mlir/ToyCombine.cpp
index 0af4cbfc11f12f1acd7fea6d3656319d457119b2..5e74d95d573b518109395b560dc8294dd08e83de 100644
--- a/mlir/examples/toy/Ch6/mlir/ToyCombine.cpp
+++ b/mlir/examples/toy/Ch6/mlir/ToyCombine.cpp
@@ -54,15 +54,15 @@ struct SimplifyRedundantTranspose : public mlir::OpRewritePattern<TransposeOp> {
 
 /// Register our patterns as "canonicalization" patterns on the TransposeOp so
 /// that they can be picked up by the Canonicalization framework.
-void TransposeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void TransposeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
-  results.insert<SimplifyRedundantTranspose>(context);
+  results.add<SimplifyRedundantTranspose>(context);
 }
 
 /// Register our patterns as "canonicalization" patterns on the ReshapeOp so
 /// that they can be picked up by the Canonicalization framework.
-void ReshapeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ReshapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
-  results.insert<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
-                 FoldConstantReshapeOptPattern>(context);
+  results.add<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
+              FoldConstantReshapeOptPattern>(context);
 }
diff --git a/mlir/examples/toy/Ch7/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch7/include/toy/CMakeLists.txt
index 977ebb68e97ead4405e99c50c45f7ed70bfe11a0..3ff7633f8ddefdf262b2c03e4fb4d87517924b2c 100644
--- a/mlir/examples/toy/Ch7/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch7/include/toy/CMakeLists.txt
@@ -2,6 +2,7 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
 mlir_tablegen(Ops.h.inc -gen-op-decls)
 mlir_tablegen(Ops.cpp.inc -gen-op-defs)
+mlir_tablegen(Dialect.h.inc -gen-dialect-decls)
 add_public_tablegen_target(ToyCh7OpsIncGen)
 
 # Most dialects should use add_mlir_interfaces().
diff --git a/mlir/examples/toy/Ch7/include/toy/Dialect.h b/mlir/examples/toy/Ch7/include/toy/Dialect.h
index 1b754f3d108906a9b17bd7cf801c47d4d4295afb..0b88244dc6ef5c0f685eea9bef61ef2141ca8c56 100644
--- a/mlir/examples/toy/Ch7/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch7/include/toy/Dialect.h
@@ -26,34 +26,13 @@ namespace toy {
 namespace detail {
 struct StructTypeStorage;
 } // end namespace detail
-
-/// This is the definition of the Toy dialect. A dialect inherits from
-/// mlir::Dialect and registers custom attributes, operations, and types (in its
-/// constructor). It can also override some general behavior exposed via virtual
-/// methods.
-class ToyDialect : public mlir::Dialect {
-public:
-  explicit ToyDialect(mlir::MLIRContext *ctx);
-
-  /// A hook used to materialize constant values with the given type.
-  Operation *materializeConstant(OpBuilder &builder, Attribute value, Type type,
-                                 Location loc) override;
-
-  /// Parse an instance of a type registered to the toy dialect.
-  mlir::Type parseType(mlir::DialectAsmParser &parser) const override;
-
-  /// Print an instance of a type registered to the toy dialect.
-  void printType(mlir::Type type,
-                 mlir::DialectAsmPrinter &printer) const override;
-
-  /// Provide a utility accessor to the dialect namespace. This is used by
-  /// several utilities for casting between dialects.
-  static llvm::StringRef getDialectNamespace() { return "toy"; }
-};
-
 } // end namespace toy
 } // end namespace mlir
 
+/// Include the auto-generated header file containing the declaration of the toy
+/// dialect.
+#include "toy/Dialect.h.inc"
+
 //===----------------------------------------------------------------------===//
 // Toy Operations
 //===----------------------------------------------------------------------===//
diff --git a/mlir/examples/toy/Ch7/include/toy/Ops.td b/mlir/examples/toy/Ch7/include/toy/Ops.td
index 7ca473ea083c63fc4409c7beab872e71fd76dd4d..77371496e47bb2f4fcdba949db0604dc0b8be2c1 100644
--- a/mlir/examples/toy/Ch7/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch7/include/toy/Ops.td
@@ -23,6 +23,10 @@ include "toy/ShapeInferenceInterface.td"
 def Toy_Dialect : Dialect {
   let name = "toy";
   let cppNamespace = "::mlir::toy";
+
+  // We set this bit to generate a declaration of the `materializeConstant`
+  // method so that we can materialize constants for our toy operations.
+  let hasConstantMaterializer = 1;
 }
 
 // Base class for toy dialect operations. This operation inherits from the base
@@ -34,9 +38,11 @@ class Toy_Op<string mnemonic, list<OpTrait> traits = []> :
     Op<Toy_Dialect, mnemonic, traits>;
 
 // Provide a definition for the Toy StructType for use in ODS. This allows for
-// using StructType in a similar way to Tensor or MemRef.
+// using StructType in a similar way to Tensor or MemRef. We use `DialectType`
+// to demarcate the StructType as belonging to the Toy dialect.
 def Toy_StructType :
-    Type<CPred<"$_self.isa<StructType>()">, "Toy struct type">;
+    DialectType<Toy_Dialect, CPred<"$_self.isa<StructType>()">,
+                "Toy struct type">;
 
 // Provide a definition of the types that are used within the Toy dialect.
 def Toy_Type : AnyTypeOf<[F64Tensor, Toy_StructType]>;
diff --git a/mlir/examples/toy/Ch7/mlir/Dialect.cpp b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
index cbcf53f313c1f73916d5b450ec1fbc7f0bf279cd..28f5435f7e982cfb2f3a27576d5f2f8218161aa9 100644
--- a/mlir/examples/toy/Ch7/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
@@ -544,10 +544,9 @@ void ToyDialect::printType(mlir::Type type,
 // ToyDialect
 //===----------------------------------------------------------------------===//
 
-/// Dialect creation, the instance will be owned by the context. This is the
-/// point of registration of custom types and operations for the dialect.
-ToyDialect::ToyDialect(mlir::MLIRContext *ctx)
-    : mlir::Dialect(getDialectNamespace(), ctx, TypeID::get<ToyDialect>()) {
+/// Dialect initialization, the instance will be owned by the context. This is
+/// the point of registration of types and operations for the dialect.
+void ToyDialect::initialize() {
   addOperations<
 #define GET_OP_LIST
 #include "toy/Ops.cpp.inc"
diff --git a/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp b/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
index f2332fd5437fa66a8855bd1c0bd42296d8dd2251..8acbc37c77f7c2690d15107ef5dc4bb45b407e8e 100644
--- a/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
+++ b/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
@@ -297,9 +297,9 @@ void ToyToAffineLoweringPass::runOnFunction() {
 
   // Now that the conversion target has been defined, we just need to provide
   // the set of patterns that will lower the Toy operations.
-  OwningRewritePatternList patterns;
-  patterns.insert<AddOpLowering, ConstantOpLowering, MulOpLowering,
-                  ReturnOpLowering, TransposeOpLowering>(&getContext());
+  RewritePatternSet patterns(&getContext());
+  patterns.add<AddOpLowering, ConstantOpLowering, MulOpLowering,
+               ReturnOpLowering, TransposeOpLowering>(&getContext());
 
   // With the target and rewrite patterns defined, we can now attempt the
   // conversion. The conversion will signal failure if any of our `illegal`
diff --git a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
index bbe3e54f8ac5af9ca7ccb5af98da9948dcae6c81..f334faa29ca8837a35170c7b00cd04ce9235308f 100644
--- a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
@@ -174,7 +174,7 @@ void ToyToLLVMLoweringPass::runOnOperation() {
   // final target for this lowering. For this lowering, we are only targeting
   // the LLVM dialect.
   LLVMConversionTarget target(getContext());
-  target.addLegalOp<ModuleOp, ModuleTerminatorOp>();
+  target.addLegalOp<ModuleOp>();
 
   // During this lowering, we will also be lowering the MemRef types, that are
   // currently being operated on, to a representation in LLVM. To perform this
@@ -191,14 +191,14 @@ void ToyToLLVMLoweringPass::runOnOperation() {
   // lowerings. Transitive lowering, or A->B->C lowering, is when multiple
   // patterns must be applied to fully transform an illegal operation into a
   // set of legal ones.
-  OwningRewritePatternList patterns;
-  populateAffineToStdConversionPatterns(patterns, &getContext());
-  populateLoopToStdConversionPatterns(patterns, &getContext());
+  RewritePatternSet patterns(&getContext());
+  populateAffineToStdConversionPatterns(patterns);
+  populateLoopToStdConversionPatterns(patterns);
   populateStdToLLVMConversionPatterns(typeConverter, patterns);
 
   // The only remaining operation to lower from the `toy` dialect, is the
   // PrintOp.
-  patterns.insert<PrintOpLowering>(&getContext());
+  patterns.add<PrintOpLowering>(&getContext());
 
   // We want to completely lower to LLVM, so we use a `FullConversion`. This
   // ensures that only legal operations will remain after the conversion.
diff --git a/mlir/examples/toy/Ch7/mlir/ToyCombine.cpp b/mlir/examples/toy/Ch7/mlir/ToyCombine.cpp
index bfbd36b40fa032fafb11b4b7d7343fc0b32fa7fb..95072eeef1d2ee1243f63d93eb25895651b5c2b6 100644
--- a/mlir/examples/toy/Ch7/mlir/ToyCombine.cpp
+++ b/mlir/examples/toy/Ch7/mlir/ToyCombine.cpp
@@ -72,15 +72,15 @@ struct SimplifyRedundantTranspose : public mlir::OpRewritePattern<TransposeOp> {
 
 /// Register our patterns as "canonicalization" patterns on the TransposeOp so
 /// that they can be picked up by the Canonicalization framework.
-void TransposeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void TransposeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
-  results.insert<SimplifyRedundantTranspose>(context);
+  results.add<SimplifyRedundantTranspose>(context);
 }
 
 /// Register our patterns as "canonicalization" patterns on the ReshapeOp so
 /// that they can be picked up by the Canonicalization framework.
-void ReshapeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ReshapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
-  results.insert<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
-                 FoldConstantReshapeOptPattern>(context);
+  results.add<ReshapeReshapeOptPattern, RedundantReshapeOptPattern,
+              FoldConstantReshapeOptPattern>(context);
 }
diff --git a/mlir/include/mlir-c/Dialect/Linalg.h b/mlir/include/mlir-c/Dialect/Linalg.h
index 56258ac19af455e730da9bf06e70a7a812531107..be73a5c8c207a04de9e33471f36e32e5feafb5bf 100644
--- a/mlir/include/mlir-c/Dialect/Linalg.h
+++ b/mlir/include/mlir-c/Dialect/Linalg.h
@@ -11,6 +11,7 @@
 #define MLIR_C_DIALECT_LINALG_H
 
 #include "mlir-c/Registration.h"
+#include "mlir-c/Support.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -22,4 +23,6 @@ MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Linalg, linalg);
 }
 #endif
 
+#include "mlir/Dialect/Linalg/Passes.capi.h.inc"
+
 #endif // MLIR_C_DIALECT_LINALG_H
diff --git a/mlir/include/mlir/Conversion/AffineToStandard/AffineToStandard.h b/mlir/include/mlir/Conversion/AffineToStandard/AffineToStandard.h
index 4647cacdd9cd8f9640da3e0cf1abdccc0e5db29e..b8afecdaa93dee1c63f27a764488e2b7819f537a 100644
--- a/mlir/include/mlir/Conversion/AffineToStandard/AffineToStandard.h
+++ b/mlir/include/mlir/Conversion/AffineToStandard/AffineToStandard.h
@@ -18,15 +18,14 @@ class AffineMap;
 class AffineParallelOp;
 class Location;
 struct LogicalResult;
-class MLIRContext;
 class OpBuilder;
 class Pass;
 class RewritePattern;
 class Value;
 class ValueRange;
 
-// Owning list of rewriting patterns.
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Emit code that computes the given affine expression using standard
 /// arithmetic operations applied to the provided dimension and symbol values.
@@ -43,13 +42,11 @@ Optional<SmallVector<Value, 8>> expandAffineMap(OpBuilder &builder,
 /// Collect a set of patterns to convert from the Affine dialect to the Standard
 /// dialect, in particular convert structured affine control flow into CFG
 /// branch-based control flow.
-void populateAffineToStdConversionPatterns(OwningRewritePatternList &patterns,
-                                           MLIRContext *ctx);
+void populateAffineToStdConversionPatterns(RewritePatternSet &patterns);
 
 /// Collect a set of patterns to convert vector-related Affine ops to the Vector
 /// dialect.
-void populateAffineToVectorConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx);
+void populateAffineToVectorConversionPatterns(RewritePatternSet &patterns);
 
 /// Emit code that computes the lower bound of the given affine loop using
 /// standard arithmetic operations.
diff --git a/mlir/include/mlir/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.h b/mlir/include/mlir/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.h
index 8cba4e9be5d5cc64519b1b54ab71328a88a0fac2..1cbe3f69e36aee43155a1eea47cb7973ffc9cdd0 100644
--- a/mlir/include/mlir/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.h
+++ b/mlir/include/mlir/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.h
@@ -12,11 +12,12 @@
 namespace mlir {
 
 class LLVMTypeConverter;
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Collect a set of patterns to convert from the ArmSVE dialect to LLVM.
 void populateArmSVEToLLVMConversionPatterns(LLVMTypeConverter &converter,
-                                            OwningRewritePatternList &patterns);
+                                            RewritePatternSet &patterns);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h b/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h
index 938c5cbbebce375216a25b40324fbdfa62d408b2..0878c633ec4f8a52e390fe6274eceeefc2f6e9ee 100644
--- a/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h
+++ b/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h
@@ -18,8 +18,9 @@ class ModuleOp;
 template <typename T>
 class OperationPass;
 class MLIRContext;
-class OwningRewritePatternList;
 class TypeConverter;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Create a pass to convert Async operations to the LLVM dialect.
 std::unique_ptr<OperationPass<ModuleOp>> createConvertAsyncToLLVMPass();
@@ -33,8 +34,8 @@ std::unique_ptr<OperationPass<ModuleOp>> createConvertAsyncToLLVMPass();
 /// the TypeConverter, but otherwise don't care what type conversions are
 /// happening.
 void populateAsyncStructuralTypeConversionsAndLegality(
-    MLIRContext *context, TypeConverter &typeConverter,
-    OwningRewritePatternList &patterns, ConversionTarget &target);
+    TypeConverter &typeConverter, RewritePatternSet &patterns,
+    ConversionTarget &target);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h b/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h
index 3dab2a136b289c23162e64ae198bce5e4e266bc7..378eb006b969a808de2503207c37ad5c87bad43c 100644
--- a/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h
+++ b/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h
@@ -18,8 +18,8 @@ template <typename T>
 class OperationPass;
 
 /// Populate the given list with patterns that convert from Complex to LLVM.
-void populateComplexToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+void populateComplexToLLVMConversionPatterns(LLVMTypeConverter &converter,
+                                             RewritePatternSet &patterns);
 
 /// Create a pass to convert Complex operations to the LLVMIR dialect.
 std::unique_ptr<OperationPass<ModuleOp>> createConvertComplexToLLVMPass();
diff --git a/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h b/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h
index fb5e8202df63f670312b061bc40304e8f3c24972..708a3fe0b23ee3d51d8bd51f7e08fcc83eea0196 100644
--- a/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h
+++ b/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h
@@ -24,7 +24,8 @@ class Location;
 struct LogicalResult;
 class ModuleOp;
 class Operation;
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 template <typename T>
 class OperationPass;
@@ -43,53 +44,13 @@ using BlobGenerator =
 using LoweringCallback = std::function<std::unique_ptr<llvm::Module>(
     Operation *, llvm::LLVMContext &, StringRef)>;
 
-/// Creates a pass to convert a gpu.launch_func operation into a sequence of
-/// GPU runtime calls.
+/// Creates a pass to convert a GPU operations into a sequence of GPU runtime
+/// calls.
 ///
 /// This pass does not generate code to call GPU runtime APIs directly but
 /// instead uses a small wrapper library that exports a stable and conveniently
 /// typed ABI on top of GPU runtimes such as CUDA or ROCm (HIP).
-///
-/// A non-empty gpuBinaryAnnotation overrides the pass' command line option.
-std::unique_ptr<OperationPass<ModuleOp>>
-createGpuToLLVMConversionPass(StringRef gpuBinaryAnnotation = {});
-
-/// Collect a set of patterns to convert from the GPU dialect to LLVM.
-///
-/// A non-empty gpuBinaryAnnotation overrides the pass' command line option.
-void populateGpuToLLVMConversionPatterns(LLVMTypeConverter &converter,
-                                         OwningRewritePatternList &patterns,
-                                         StringRef gpuBinaryAnnotation = {});
-
-/// Creates a pass to convert kernel functions into GPU target object blobs.
-///
-/// This transformation takes the body of each function that is annotated with
-/// the 'gpu.kernel' attribute, copies it to a new LLVM module, compiles the
-/// module with help of the GPU backend to target object and then invokes
-/// the provided blobGenerator to produce a binary blob. Such blob is then
-/// attached as a string attribute to the kernel function.
-///
-/// Following callbacks are to be provided by user:
-/// - loweringCallback : lower the module to an LLVM module.
-/// - blobGenerator : build a blob executable on target GPU.
-///
-/// Information wrt LLVM backend are to be supplied by user:
-/// - triple : target triple to be used.
-/// - targetChip : mcpu to be used.
-/// - features : target-specific features to be used.
-///
-/// Information about result attribute is to be specified by user:
-/// - gpuBinaryAnnotation : the name of the attribute which contains the blob.
-///
-/// After the transformation, the body of the kernel function is removed (i.e.,
-/// it is turned into a declaration).
-///
-/// A non-empty gpuBinaryAnnotation overrides the pass' command line option.
-std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
-createConvertGPUKernelToBlobPass(LoweringCallback loweringCallback,
-                                 BlobGenerator blobGenerator, StringRef triple,
-                                 StringRef targetChip, StringRef features,
-                                 StringRef gpuBinaryAnnotation = {});
+std::unique_ptr<OperationPass<ModuleOp>> createGpuToLLVMConversionPass();
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h b/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
index 233b947bcfeded275e1cb50e475039ebe05c996a..a005fb50226f50b54630f5a4941bedfe28d1b780 100644
--- a/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
+++ b/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
@@ -13,10 +13,12 @@
 
 namespace mlir {
 class LLVMTypeConverter;
-class OwningRewritePatternList;
 class ConversionTarget;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
-template <typename OpT> class OperationPass;
+template <typename OpT>
+class OperationPass;
 
 namespace gpu {
 class GPUModuleOp;
@@ -27,7 +29,7 @@ void configureGpuToNVVMConversionLegality(ConversionTarget &target);
 
 /// Collect a set of patterns to convert from the GPU dialect to NVVM.
 void populateGpuToNVVMConversionPatterns(LLVMTypeConverter &converter,
-                                         OwningRewritePatternList &patterns);
+                                         RewritePatternSet &patterns);
 
 /// Creates a pass that lowers GPU dialect operations to NVVM counterparts. The
 /// index bitwidth used for the lowering of the device side index computations
diff --git a/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h b/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
index 5fa798bf2834278679eeab23f3bf48086a394f1d..bcec880dfc6d4c51b7b3ffea20094ba4064a069b 100644
--- a/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
+++ b/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
@@ -13,8 +13,9 @@
 
 namespace mlir {
 class LLVMTypeConverter;
-class OwningRewritePatternList;
 class ConversionTarget;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 template <typename OpT>
 class OperationPass;
@@ -25,7 +26,7 @@ class GPUModuleOp;
 
 /// Collect a set of patterns to convert from the GPU dialect to ROCDL.
 void populateGpuToROCDLConversionPatterns(LLVMTypeConverter &converter,
-                                          OwningRewritePatternList &patterns);
+                                          RewritePatternSet &patterns);
 
 /// Configure target to convert from the GPU dialect to ROCDL.
 void configureGpuToROCDLConversionLegality(ConversionTarget &target);
diff --git a/mlir/include/mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h b/mlir/include/mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h
index ad5dac0038972436253bbf96a0c3d77c287a9e6b..add196d441ea8d07a078e70fc0427dec814e2c11 100644
--- a/mlir/include/mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h
+++ b/mlir/include/mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h
@@ -21,9 +21,8 @@ class SPIRVTypeConverter;
 /// Appends to a pattern list additional patterns for translating GPU Ops to
 /// SPIR-V ops. For a gpu.func to be converted, it should have a
 /// spv.entry_point_abi attribute.
-void populateGPUToSPIRVPatterns(MLIRContext *context,
-                                SPIRVTypeConverter &typeConverter,
-                                OwningRewritePatternList &patterns);
+void populateGPUToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                RewritePatternSet &patterns);
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_GPUTOSPIRV_GPUTOSPIRV_H
diff --git a/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h b/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h
index 948c2a4be6f2d1e6789e254618b8d9fd28393271..a1f56048cd7f4df3aac1c2913de505dd1cc5f869 100644
--- a/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h
+++ b/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h
@@ -14,11 +14,12 @@
 namespace mlir {
 class MLIRContext;
 class ModuleOp;
-template <typename T> class OperationPass;
+template <typename T>
+class OperationPass;
 
 /// Populate the given list with patterns that convert from Linalg to LLVM.
 void populateLinalgToLLVMConversionPatterns(LLVMTypeConverter &converter,
-                                            OwningRewritePatternList &patterns);
+                                            RewritePatternSet &patterns);
 
 /// Create a pass to convert Linalg operations to the LLVMIR dialect.
 std::unique_ptr<OperationPass<ModuleOp>> createConvertLinalgToLLVMPass();
diff --git a/mlir/include/mlir/Conversion/LinalgToSPIRV/LinalgToSPIRV.h b/mlir/include/mlir/Conversion/LinalgToSPIRV/LinalgToSPIRV.h
index b2fc9e4e91385eee4f3c72fafca9f8263e7ae571..64b612ed6b129606c24225904fb7ad78a15ad2cc 100644
--- a/mlir/include/mlir/Conversion/LinalgToSPIRV/LinalgToSPIRV.h
+++ b/mlir/include/mlir/Conversion/LinalgToSPIRV/LinalgToSPIRV.h
@@ -15,14 +15,14 @@
 
 namespace mlir {
 class MLIRContext;
-class OwningRewritePatternList;
 class SPIRVTypeConverter;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Appends to a pattern list additional patterns for translating Linalg ops to
 /// SPIR-V ops.
-void populateLinalgToSPIRVPatterns(MLIRContext *context,
-                                   SPIRVTypeConverter &typeConverter,
-                                   OwningRewritePatternList &patterns);
+void populateLinalgToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                   RewritePatternSet &patterns);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/LinalgToStandard/LinalgToStandard.h b/mlir/include/mlir/Conversion/LinalgToStandard/LinalgToStandard.h
index 3a6c8bba614b95cf5d7e68aa6aef5bffc3c40cc4..eeb20c4806b9deeeec94b081818db49cf492d84d 100644
--- a/mlir/include/mlir/Conversion/LinalgToStandard/LinalgToStandard.h
+++ b/mlir/include/mlir/Conversion/LinalgToStandard/LinalgToStandard.h
@@ -30,12 +30,12 @@ namespace linalg {
 // or in an externally linked library.
 // This is a generic entry point for all LinalgOp, except for CopyOp and
 // IndexedGenericOp, for which omre specialized patterns are provided.
-class LinalgOpToLibraryCallRewrite : public RewritePattern {
+class LinalgOpToLibraryCallRewrite
+    : public OpInterfaceRewritePattern<LinalgOp> {
 public:
-  LinalgOpToLibraryCallRewrite()
-      : RewritePattern(/*benefit=*/1, MatchAnyOpTypeTag()) {}
+  using OpInterfaceRewritePattern<LinalgOp>::OpInterfaceRewritePattern;
 
-  LogicalResult matchAndRewrite(Operation *op,
+  LogicalResult matchAndRewrite(LinalgOp op,
                                 PatternRewriter &rewriter) const override;
 };
 
@@ -69,8 +69,7 @@ public:
 };
 
 /// Populate the given list with patterns that convert from Linalg to Standard.
-void populateLinalgToStandardConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx);
+void populateLinalgToStandardConversionPatterns(RewritePatternSet &patterns);
 
 } // namespace linalg
 
diff --git a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
index 4eae84cd0135cbeb2592ed6e4fc1930d9bb99062..b51b7f3b5938377d6d57cd079fb3fc937f700406 100644
--- a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
+++ b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
@@ -8,7 +8,7 @@
 #ifndef MLIR_CONVERSION_OPENMPTOLLVM_OPENMPTOLLVM_H_
 #define MLIR_CONVERSION_OPENMPTOLLVM_OPENMPTOLLVM_H_
 
-#include<memory>
+#include <memory>
 
 namespace mlir {
 class LLVMTypeConverter;
@@ -16,11 +16,12 @@ class MLIRContext;
 class ModuleOp;
 template <typename T>
 class OperationPass;
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Populate the given list with patterns that convert from OpenMP to LLVM.
 void populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter,
-                                            OwningRewritePatternList &patterns);
+                                            RewritePatternSet &patterns);
 
 /// Create a pass to convert OpenMP operations to the LLVMIR dialect.
 std::unique_ptr<OperationPass<ModuleOp>> createConvertOpenMPToLLVMPass();
diff --git a/mlir/include/mlir/Conversion/SCFToGPU/SCFToGPU.h b/mlir/include/mlir/Conversion/SCFToGPU/SCFToGPU.h
index d6316f663aa8bb029b35e22019e97e3184482a17..ac1ba0e2f24b4ef10ffa9a2c2cadec0d7587b981 100644
--- a/mlir/include/mlir/Conversion/SCFToGPU/SCFToGPU.h
+++ b/mlir/include/mlir/Conversion/SCFToGPU/SCFToGPU.h
@@ -15,8 +15,9 @@ class AffineForOp;
 class ConversionTarget;
 struct LogicalResult;
 class MLIRContext;
-class OwningRewritePatternList;
 class Value;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 namespace scf {
 class ForOp;
@@ -42,8 +43,7 @@ LogicalResult convertAffineLoopNestToGPULaunch(AffineForOp forOp,
 
 /// Adds the conversion pattern from `scf.parallel` to `gpu.launch` to the
 /// provided pattern list.
-void populateParallelLoopToGPUPatterns(OwningRewritePatternList &patterns,
-                                       MLIRContext *ctx);
+void populateParallelLoopToGPUPatterns(RewritePatternSet &patterns);
 
 /// Configures the rewrite target such that only `scf.parallel` operations that
 /// are not rewritten by the provided patterns are legal.
diff --git a/mlir/include/mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h b/mlir/include/mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h
index e0bab2759753b4e683ed94ec535e200befb342fe..284500df796c5b00d7277b37232a364b244400e6 100644
--- a/mlir/include/mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h
+++ b/mlir/include/mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h
@@ -15,13 +15,13 @@
 #include <memory>
 
 namespace mlir {
-class MLIRContext;
 class Pass;
 
 // Owning list of rewriting patterns.
-class OwningRewritePatternList;
 class SPIRVTypeConverter;
 struct ScfToSPIRVContextImpl;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 struct ScfToSPIRVContext {
   ScfToSPIRVContext();
@@ -35,10 +35,9 @@ private:
 
 /// Collects a set of patterns to lower from scf.for, scf.if, and
 /// loop.terminator to CFG operations within the SPIR-V dialect.
-void populateSCFToSPIRVPatterns(MLIRContext *context,
-                                SPIRVTypeConverter &typeConverter,
+void populateSCFToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
                                 ScfToSPIRVContext &scfToSPIRVContext,
-                                OwningRewritePatternList &patterns);
+                                RewritePatternSet &patterns);
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_SCFTOSPIRV_SCFTOSPIRV_H_
diff --git a/mlir/include/mlir/Conversion/SCFToStandard/SCFToStandard.h b/mlir/include/mlir/Conversion/SCFToStandard/SCFToStandard.h
index fd85a3d367e7de0b3e3adf3f6308acef232c146a..880d4ae6e7454aa4584f419197d33cc83ff4c416 100644
--- a/mlir/include/mlir/Conversion/SCFToStandard/SCFToStandard.h
+++ b/mlir/include/mlir/Conversion/SCFToStandard/SCFToStandard.h
@@ -14,18 +14,15 @@
 
 namespace mlir {
 struct LogicalResult;
-class MLIRContext;
 class Pass;
-class RewritePattern;
 
-// Owning list of rewriting patterns.
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Collect a set of patterns to lower from scf.for, scf.if, and
 /// loop.terminator to CFG operations within the Standard dialect, in particular
 /// convert structured control flow into CFG branch-based control flow.
-void populateLoopToStdConversionPatterns(OwningRewritePatternList &patterns,
-                                         MLIRContext *ctx);
+void populateLoopToStdConversionPatterns(RewritePatternSet &patterns);
 
 /// Creates a pass to convert scf.for, scf.if and loop.terminator ops to CFG.
 std::unique_ptr<Pass> createLowerToCFGPass();
diff --git a/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVM.h b/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVM.h
index 3ba24eaac1a0d7d63b902217b5fa3a6ac4a8bc58..c135eee27ee725a75dc585c16b5450892faa621b 100644
--- a/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVM.h
+++ b/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVM.h
@@ -40,20 +40,17 @@ void encodeBindAttribute(ModuleOp module);
 void populateSPIRVToLLVMTypeConversion(LLVMTypeConverter &typeConverter);
 
 /// Populates the given list with patterns that convert from SPIR-V to LLVM.
-void populateSPIRVToLLVMConversionPatterns(MLIRContext *context,
-                                           LLVMTypeConverter &typeConverter,
-                                           OwningRewritePatternList &patterns);
+void populateSPIRVToLLVMConversionPatterns(LLVMTypeConverter &typeConverter,
+                                           RewritePatternSet &patterns);
 
 /// Populates the given list with patterns for function conversion from SPIR-V
 /// to LLVM.
 void populateSPIRVToLLVMFunctionConversionPatterns(
-    MLIRContext *context, LLVMTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns);
+    LLVMTypeConverter &typeConverter, RewritePatternSet &patterns);
 
 /// Populates the given patterns for module conversion from SPIR-V to LLVM.
 void populateSPIRVToLLVMModuleConversionPatterns(
-    MLIRContext *context, LLVMTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns);
+    LLVMTypeConverter &typeConverter, RewritePatternSet &patterns);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/ShapeToStandard/ShapeToStandard.h b/mlir/include/mlir/Conversion/ShapeToStandard/ShapeToStandard.h
index 176f1018388133fdbd3c3a443739c6e211c039e7..a26d4dd2e314f9d2b27f47e4331cb0debb02bd8e 100644
--- a/mlir/include/mlir/Conversion/ShapeToStandard/ShapeToStandard.h
+++ b/mlir/include/mlir/Conversion/ShapeToStandard/ShapeToStandard.h
@@ -14,19 +14,18 @@
 namespace mlir {
 
 class FuncOp;
-class MLIRContext;
 class ModuleOp;
 template <typename T>
 class OperationPass;
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
-void populateShapeToStandardConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx);
+void populateShapeToStandardConversionPatterns(RewritePatternSet &patterns);
 
 std::unique_ptr<OperationPass<ModuleOp>> createConvertShapeToStandardPass();
 
 void populateConvertShapeConstraintsConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx);
+    RewritePatternSet &patterns);
 
 std::unique_ptr<OperationPass<FuncOp>> createConvertShapeConstraintsPass();
 
diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
index c37528178e833613d36f2fadc774badfff56f57e..43a1afcc3f892f0b9d12a489c6b0f13d496bc676 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
@@ -116,8 +116,10 @@ public:
                                    OpBuilder &builder);
 
   /// Converts the function type to a C-compatible format, in particular using
-  /// pointers to memref descriptors for arguments.
-  Type convertFunctionTypeCWrapper(FunctionType type);
+  /// pointers to memref descriptors for arguments. Also converts the return
+  /// type to a pointer argument if it is a struct. Returns true if this
+  /// was the case.
+  std::pair<Type, bool> convertFunctionTypeCWrapper(FunctionType type);
 
   /// Returns the data layout to use during and after conversion.
   const llvm::DataLayout &getDataLayout() { return options.dataLayout; }
@@ -127,7 +129,7 @@ public:
   Type getIndexType();
 
   /// Gets the bitwidth of the index type when converted to LLVM.
-  unsigned getIndexTypeBitwidth() { return options.indexBitwidth; }
+  unsigned getIndexTypeBitwidth() { return options.getIndexBitwidth(); }
 
   /// Gets the pointer bitwidth.
   unsigned getPointerBitwidth(unsigned addressSpace = 0);
diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
index ca623a3c050e02c83ae62070e68298e393d7c9af..a7a68ef0a9c62055097660f57bca8b587b01761d 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
@@ -14,11 +14,14 @@
 #include <memory>
 
 namespace mlir {
+class DataLayout;
 class LLVMTypeConverter;
+class MLIRContext;
 class ModuleOp;
 template <typename T>
 class OperationPass;
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Value to pass as bitwidth for the index type when the converter is expected
 /// to derive the bitwidth from the LLVM data layout.
@@ -26,10 +29,14 @@ static constexpr unsigned kDeriveIndexBitwidthFromDataLayout = 0;
 
 /// Options to control the Standard dialect to LLVM lowering. The struct is used
 /// to share lowering options between passes, patterns, and type converter.
-struct LowerToLLVMOptions {
+class LowerToLLVMOptions {
+public:
+  explicit LowerToLLVMOptions(MLIRContext *ctx);
+  explicit LowerToLLVMOptions(MLIRContext *ctx, DataLayout dl);
+
   bool useBarePtrCallConv = false;
   bool emitCWrappers = false;
-  unsigned indexBitwidth = kDeriveIndexBitwidthFromDataLayout;
+
   /// Use aligned_alloc for heap allocations.
   bool useAlignedAlloc = false;
 
@@ -38,45 +45,52 @@ struct LowerToLLVMOptions {
   // TODO: this should be replaced by MLIR data layout when one exists.
   llvm::DataLayout dataLayout = llvm::DataLayout("");
 
-  /// Get a statically allocated copy of the default LowerToLLVMOptions.
-  static const LowerToLLVMOptions &getDefaultOptions() {
-    static LowerToLLVMOptions options;
-    return options;
+  /// Set the index bitwidth to the given value.
+  void overrideIndexBitwidth(unsigned bitwidth) {
+    assert(bitwidth != kDeriveIndexBitwidthFromDataLayout &&
+           "can only override to a concrete bitwidth");
+    indexBitwidth = bitwidth;
   }
+
+  /// Get the index bitwidth.
+  unsigned getIndexBitwidth() const { return indexBitwidth; }
+
+private:
+  unsigned indexBitwidth;
 };
 
 /// Collect a set of patterns to convert memory-related operations from the
 /// Standard dialect to the LLVM dialect, excluding non-memory-related
 /// operations and FuncOp.
-void populateStdToLLVMMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+void populateStdToLLVMMemoryConversionPatterns(LLVMTypeConverter &converter,
+                                               RewritePatternSet &patterns);
 
 /// Collect a set of patterns to convert from the Standard dialect to the LLVM
 /// dialect, excluding the memory-related operations.
-void populateStdToLLVMNonMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+void populateStdToLLVMNonMemoryConversionPatterns(LLVMTypeConverter &converter,
+                                                  RewritePatternSet &patterns);
 
 /// Collect the default pattern to convert a FuncOp to the LLVM dialect. If
 /// `emitCWrappers` is set, the pattern will also produce functions
 /// that pass memref descriptors by pointer-to-structure in addition to the
 /// default unpacked form.
-void populateStdToLLVMFuncOpConversionPattern(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+void populateStdToLLVMFuncOpConversionPattern(LLVMTypeConverter &converter,
+                                              RewritePatternSet &patterns);
 
 /// Collect the patterns to convert from the Standard dialect to LLVM. The
 /// conversion patterns capture the LLVMTypeConverter and the LowerToLLVMOptions
 /// by reference meaning the references have to remain alive during the entire
 /// pattern lifetime.
 void populateStdToLLVMConversionPatterns(LLVMTypeConverter &converter,
-                                         OwningRewritePatternList &patterns);
+                                         RewritePatternSet &patterns);
 
 /// Creates a pass to convert the Standard dialect into the LLVMIR dialect.
 /// stdlib malloc/free is used by default for allocating memrefs allocated with
 /// memref.alloc, while LLVM's alloca is used for those allocated with
 /// memref.alloca.
+std::unique_ptr<OperationPass<ModuleOp>> createLowerToLLVMPass();
 std::unique_ptr<OperationPass<ModuleOp>>
-createLowerToLLVMPass(const LowerToLLVMOptions &options =
-                          LowerToLLVMOptions::getDefaultOptions());
+createLowerToLLVMPass(const LowerToLLVMOptions &options);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/StandardToSPIRV/StandardToSPIRV.h b/mlir/include/mlir/Conversion/StandardToSPIRV/StandardToSPIRV.h
index 87946d387f25bd4049f44162005f83670b757f5c..165ba0081b7769698bf731562b12b417963d0db8 100644
--- a/mlir/include/mlir/Conversion/StandardToSPIRV/StandardToSPIRV.h
+++ b/mlir/include/mlir/Conversion/StandardToSPIRV/StandardToSPIRV.h
@@ -21,9 +21,8 @@ class SPIRVTypeConverter;
 /// Appends to a pattern list additional patterns for translating standard ops
 /// to SPIR-V ops. Also adds the patterns to legalize ops not directly
 /// translated to SPIR-V dialect.
-void populateStandardToSPIRVPatterns(MLIRContext *context,
-                                     SPIRVTypeConverter &typeConverter,
-                                     OwningRewritePatternList &patterns);
+void populateStandardToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                     RewritePatternSet &patterns);
 
 /// Appends to a pattern list additional patterns for translating tensor ops
 /// to SPIR-V ops.
@@ -37,15 +36,14 @@ void populateStandardToSPIRVPatterns(MLIRContext *context,
 /// variables. SPIR-V consumers in GPU drivers may or may not optimize that
 /// away. So this has implications over register pressure. Therefore, a
 /// threshold is used to control when the patterns should kick in.
-void populateTensorToSPIRVPatterns(MLIRContext *context,
-                                   SPIRVTypeConverter &typeConverter,
+void populateTensorToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
                                    int64_t byteCountThreshold,
-                                   OwningRewritePatternList &patterns);
+                                   RewritePatternSet &patterns);
 
 /// Appends to a pattern list patterns to legalize ops that are not directly
 /// lowered to SPIR-V.
 void populateStdLegalizationPatternsForSPIRVLowering(
-    MLIRContext *context, OwningRewritePatternList &patterns);
+    RewritePatternSet &patterns);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h b/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
index 42493a57bcc2f752eaa65dc2213fae4d989351b5..8ee5316c10cc1b3f3942760f336aca6ec5f2539d 100644
--- a/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
+++ b/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
@@ -28,7 +28,7 @@ void addTosaToLinalgOnTensorsPasses(OpPassManager &pm);
 
 /// Populates conversion passes from TOSA dialect to Linalg dialect.
 void populateTosaToLinalgOnTensorsConversionPatterns(
-    MLIRContext *context, OwningRewritePatternList *patterns);
+    RewritePatternSet *patterns);
 
 } // namespace tosa
 } // namespace mlir
diff --git a/mlir/include/mlir/Conversion/TosaToSCF/TosaToSCF.h b/mlir/include/mlir/Conversion/TosaToSCF/TosaToSCF.h
index 68ed0e0b6525c00ea327187e58a30467b3718ac3..e3b2e04dd61fdbd8783abce1c0d31f21f019a0fd 100644
--- a/mlir/include/mlir/Conversion/TosaToSCF/TosaToSCF.h
+++ b/mlir/include/mlir/Conversion/TosaToSCF/TosaToSCF.h
@@ -20,8 +20,7 @@ namespace tosa {
 
 std::unique_ptr<Pass> createTosaToSCF();
 
-void populateTosaToSCFConversionPatterns(MLIRContext *context,
-                                         OwningRewritePatternList *patterns);
+void populateTosaToSCFConversionPatterns(RewritePatternSet *patterns);
 
 /// Populates passes to convert from TOSA to SCF.
 void addTosaToSCFPasses(OpPassManager &pm);
diff --git a/mlir/include/mlir/Conversion/TosaToStandard/TosaToStandard.h b/mlir/include/mlir/Conversion/TosaToStandard/TosaToStandard.h
index 82555003661e0e9fb3ea45a194beea3790dafc64..fc1284417896ceecc73d73e2cbf18b8490cac4f2 100644
--- a/mlir/include/mlir/Conversion/TosaToStandard/TosaToStandard.h
+++ b/mlir/include/mlir/Conversion/TosaToStandard/TosaToStandard.h
@@ -20,8 +20,10 @@ namespace tosa {
 
 std::unique_ptr<Pass> createTosaToStandard();
 
-void populateTosaToStandardConversionPatterns(
-    MLIRContext *context, OwningRewritePatternList *patterns);
+void populateTosaToStandardConversionPatterns(RewritePatternSet *patterns);
+
+void populateTosaRescaleToStandardConversionPatterns(
+    RewritePatternSet *patterns);
 
 /// Populates passes to convert from TOSA to Standard.
 void addTosaToStandardPasses(OpPassManager &pm);
diff --git a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
index 91ded03f84b06ded132cdc6a9026271004747e62..efd26ff8808c42b403621e693646ce17cf88f036 100644
--- a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
+++ b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
@@ -62,12 +62,12 @@ struct LowerVectorToLLVMOptions {
 /// Collect a set of patterns to convert from Vector contractions to LLVM Matrix
 /// Intrinsics. To lower to assembly, the LLVM flag -lower-matrix-intrinsics
 /// will be needed when invoking LLVM.
-void populateVectorToLLVMMatrixConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+void populateVectorToLLVMMatrixConversionPatterns(LLVMTypeConverter &converter,
+                                                  RewritePatternSet &patterns);
 
 /// Collect a set of patterns to convert from the Vector dialect to LLVM.
 void populateVectorToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
+    LLVMTypeConverter &converter, RewritePatternSet &patterns,
     bool reassociateFPReductions = false, bool enableIndexOptimizations = true);
 
 /// Create a pass to convert vector operations to the LLVMIR dialect.
diff --git a/mlir/include/mlir/Conversion/VectorToROCDL/VectorToROCDL.h b/mlir/include/mlir/Conversion/VectorToROCDL/VectorToROCDL.h
index 660de02ee36fa7e8ddbf9d8a9ae91e09fd202c99..2b935cdc3dab7c8f4bb1780b468e4e9cf9f1411c 100644
--- a/mlir/include/mlir/Conversion/VectorToROCDL/VectorToROCDL.h
+++ b/mlir/include/mlir/Conversion/VectorToROCDL/VectorToROCDL.h
@@ -12,14 +12,15 @@
 
 namespace mlir {
 class LLVMTypeConverter;
-class OwningRewritePatternList;
 class ModuleOp;
 template <typename OpT>
 class OperationPass;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Collect a set of patterns to convert from the GPU dialect to ROCDL.
-void populateVectorToROCDLConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+void populateVectorToROCDLConversionPatterns(LLVMTypeConverter &converter,
+                                             RewritePatternSet &patterns);
 
 /// Create a pass to convert vector operations to the ROCDL dialect.
 std::unique_ptr<OperationPass<ModuleOp>> createConvertVectorToROCDLPass();
diff --git a/mlir/include/mlir/Conversion/VectorToSCF/VectorToSCF.h b/mlir/include/mlir/Conversion/VectorToSCF/VectorToSCF.h
index f34a5762dfa387a2ed0b5461d92480f391aa3307..e8c7e651cc860a9524038429e9d004c3afe09ae0 100644
--- a/mlir/include/mlir/Conversion/VectorToSCF/VectorToSCF.h
+++ b/mlir/include/mlir/Conversion/VectorToSCF/VectorToSCF.h
@@ -13,8 +13,9 @@
 
 namespace mlir {
 class MLIRContext;
-class OwningRewritePatternList;
 class Pass;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Control whether unrolling is used when lowering vector transfer ops to SCF.
 ///
@@ -162,7 +163,7 @@ struct VectorTransferRewriter : public RewritePattern {
 
 /// Collect a set of patterns to convert from the Vector dialect to SCF + std.
 void populateVectorToSCFConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context,
+    RewritePatternSet &patterns,
     const VectorTransferToSCFOptions &options = VectorTransferToSCFOptions());
 
 /// Create a pass to convert a subset of vector ops to SCF.
diff --git a/mlir/include/mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h b/mlir/include/mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h
index 7908f6ed3b6f940084b8866317b05b548240ebf7..bfadb83a921e831c26f071de288bd5f5f0d66ed1 100644
--- a/mlir/include/mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h
+++ b/mlir/include/mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h
@@ -20,9 +20,8 @@ class SPIRVTypeConverter;
 
 /// Appends to a pattern list additional patterns for translating Vector Ops to
 /// SPIR-V ops.
-void populateVectorToSPIRVPatterns(MLIRContext *context,
-                                   SPIRVTypeConverter &typeConverter,
-                                   OwningRewritePatternList &patterns);
+void populateVectorToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                   RewritePatternSet &patterns);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/AMX/AMX.td b/mlir/include/mlir/Dialect/AMX/AMX.td
index 710387e70b55c53167d0b37c55233f8c03f19b1b..24052ed4f24d0f18e78e45ed2df0136a48249ba8 100644
--- a/mlir/include/mlir/Dialect/AMX/AMX.td
+++ b/mlir/include/mlir/Dialect/AMX/AMX.td
@@ -71,7 +71,7 @@ def TileZeroOp : AMX_Op<"tile_zero", [NoSideEffect]> {
     Example:
 
     ```mlir
-      %0 = amx.tilezero : vector<16x16xbf16>
+      %0 = amx.tile_zero : vector<16x16xbf16>
     ```
   }];
   let verifier = [{ return ::verify(*this); }];
@@ -100,7 +100,7 @@ def TileLoadOp : AMX_Op<"tile_load", [NoSideEffect]> {
     Example:
 
     ```mlir
-      %0 = amx.tileload %arg0[%c0, %c0] : memref<?x?xi8> into vector<16x64xi8>
+      %0 = amx.tile_load %arg0[%c0, %c0] : memref<?x?xi8> into vector<16x64xi8>
     ```
   }];
   let verifier = [{ return ::verify(*this); }];
@@ -131,7 +131,7 @@ def TileStoreOp : AMX_Op<"tile_store"> {
     Example:
 
     ```mlir
-      amx.tilestore %arg1[%c0, %c0], %0 : memref<?x?xi8>, vector<16x64xi8>
+      amx.tile_store %arg1[%c0, %c0], %0 : memref<?x?xi8>, vector<16x64xi8>
     ```
   }];
   let verifier = [{ return ::verify(*this); }];
@@ -165,7 +165,7 @@ def TileMulFOp : AMX_Op<"tile_mulf", [NoSideEffect, AllTypesMatch<["acc", "res"]
     Example:
 
     ```mlir
-      %0 = amx.tilemulf %a, %b, %c
+      %0 = amx.tile_mulf %a, %b, %c
         : vector<16x32xbf16>, vector<16x32xbf16>, vector<16x16xf32>
     ```
   }];
@@ -196,14 +196,14 @@ def TileMulIOp : AMX_Op<"tile_muli", [NoSideEffect, AllTypesMatch<["acc", "res"]
     into a "m x n" destination tile. Supports all "si32 <- s/ui8 x s/ui8"
     combinations (4 bytes packed into dwords in the columns of both the
     source operand tiles; the zero or sign extension is specified with
-    the attributes). The operation is eventually lowered into one of
-    the "tdpbssd", "tdpbsud", "tdpbusd", or "tdpbuud" instructions with
-    the corresponding tile configuration.
+    the attributes and default to sign extended). The operation is eventually
+    lowered into one of the "tdpbssd", "tdpbsud", "tdpbusd", or "tdpbuud"
+    instructions with the corresponding tile configuration.
 
     Example:
 
     ```mlir
-      %0 = amx.tilemuli %a, %b, %c [true, true]
+      %0 = amx.tile_muli %a zext, %b zext, %c 
         : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
     ```
   }];
@@ -211,7 +211,9 @@ def TileMulIOp : AMX_Op<"tile_muli", [NoSideEffect, AllTypesMatch<["acc", "res"]
   let arguments = (ins VectorOfRankAndType<[2], [I32, I8]>:$lhs,
                        VectorOfRankAndType<[2], [I32, I8]>:$rhs,
                        VectorOfRankAndType<[2], [I32, I8]>:$acc,
-                       BoolArrayAttr:$zext);
+                       UnitAttr:$isZextLhs,
+                       UnitAttr:$isZextRhs
+                       );
   let results = (outs VectorOfRankAndType<[2], [I32, I8]>:$res);
   let extraClassDeclaration = [{
     VectorType getLhsVectorType() {
@@ -224,7 +226,7 @@ def TileMulIOp : AMX_Op<"tile_muli", [NoSideEffect, AllTypesMatch<["acc", "res"]
       return res().getType().cast<VectorType>();
     }
   }];
-  let assemblyFormat = "$lhs `,` $rhs `,` $acc $zext attr-dict `:` "
+  let assemblyFormat = "$lhs (`zext` $isZextLhs^)? `,` $rhs (`zext` $isZextRhs^)? `,` $acc attr-dict `:` "
                        "type($lhs) `,` type($rhs) `,` type($acc) ";
 }
 
diff --git a/mlir/include/mlir/Dialect/AMX/Transforms.h b/mlir/include/mlir/Dialect/AMX/Transforms.h
index 11b3004292d4e32811dff865a6bea479550711b8..16dff0df1381770a571cc3495e9f0f0a0c753709 100644
--- a/mlir/include/mlir/Dialect/AMX/Transforms.h
+++ b/mlir/include/mlir/Dialect/AMX/Transforms.h
@@ -13,12 +13,13 @@ namespace mlir {
 
 class LLVMConversionTarget;
 class LLVMTypeConverter;
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Collect a set of patterns to lower AMX ops to ops that map to LLVM
 /// intrinsics.
-void populateAMXLegalizeForLLVMExportPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+void populateAMXLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter,
+                                              RewritePatternSet &patterns);
 
 /// Configure the target to support lowering AMX ops to ops that map to LLVM
 /// intrinsics.
diff --git a/mlir/include/mlir/Dialect/AVX512/Transforms.h b/mlir/include/mlir/Dialect/AVX512/Transforms.h
index 3506f50dc25825c656684df156410d430fb7425f..0ea3e627d78ca57985fc3c8e287ba72d536ee107 100644
--- a/mlir/include/mlir/Dialect/AVX512/Transforms.h
+++ b/mlir/include/mlir/Dialect/AVX512/Transforms.h
@@ -13,12 +13,13 @@ namespace mlir {
 
 class LLVMConversionTarget;
 class LLVMTypeConverter;
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Collect a set of patterns to lower AVX512 ops to ops that map to LLVM
 /// intrinsics.
-void populateAVX512LegalizeForLLVMExportPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+void populateAVX512LegalizeForLLVMExportPatterns(LLVMTypeConverter &converter,
+                                                 RewritePatternSet &patterns);
 
 /// Configure the target to support lowering AVX512 ops to ops that map to LLVM
 /// intrinsics.
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index 734075beb9d2609ca7c231483d3038b430b950ca..b930a8f4db6e1bd46d1307581d5e6092b656ffae 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -75,6 +75,11 @@ def AffineApplyOp : Affine_Op<"apply", [NoSideEffect]> {
     OpBuilder<(ins "AffineMap":$map, "ValueRange":$mapOperands),
     [{
       build($_builder, $_state, $_builder.getIndexType(), map, mapOperands);
+    }]>,
+    OpBuilder<(ins "ArrayRef<AffineExpr> ":$exprList,"ValueRange":$mapOperands),
+    [{
+      build($_builder, $_state, $_builder.getIndexType(),
+            AffineMap::inferFromExprList(exprList).front(), mapOperands);
     }]>
   ];
 
diff --git a/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
index b87c4b1cc12fec3f5ec3a67a94d8a8f6dfca224d..d57337bc8253c86d161114cabee74d5359eac404 100644
--- a/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
+++ b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
@@ -39,7 +39,7 @@ class ArmNeon_IntrOp<string mnemonic, list<int> overloadedResults,
                      list<int> overloadedOperands, int numResults,
                      list<OpTrait> traits = [], bit requiresAccessGroup = 0>
     : LLVM_IntrOpBase</*dialect=*/ArmNeon_Dialect,
-                      /*opName=*/mnemonic,
+                      /*opName=*/"intr." # mnemonic,
                       /*enumName=*/"aarch64_neon_" # !subst(".", "_", mnemonic),
                       /*overloadedResults=*/overloadedResults,
                       /*overloadedOperands=*/overloadedOperands,
@@ -53,6 +53,13 @@ class ArmNeon_OverloadedOneResultIntrOp<string mnemonic,
                                         list<OpTrait> traits = []>
   : ArmNeon_IntrOp<mnemonic, [0], [], 1, traits>;
 
+// ArmNeon dialect op that corresponds to an LLVM IR intrinsic with one
+// overloaded result and overloaded operands list.
+class ArmNeon_OverloadedOperandsWithOneResultIntrOp<string mnemonic,
+                                                    list<int> overloadedOperands,
+                                                    list<OpTrait> traits = []>
+  : ArmNeon_IntrOp<mnemonic, [0], overloadedOperands, 1, traits>;
+
 def SMullOp : ArmNeon_OverloadedOneResultIntrOp<"smull", [
        NoSideEffect,
        AllTypesMatch<["a", "b"]>,
@@ -82,5 +89,32 @@ def SMullOp : ArmNeon_OverloadedOneResultIntrOp<"smull", [
     "$a `,` $b attr-dict `:` type($a) `to` type($res)";
 }
 
+def SdotOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"sdot",[1], [
+      NoSideEffect,
+      AllTypesMatch<["b", "c"]>,
+      AllTypesMatch<["a", "res"]>,
+      TypesMatchWith<"res has the same number of elements as operand b",
+                     "b", "res",
+                     "VectorType::get({$_self.cast<VectorType>().getShape()[0] / 4},"
+                     "IntegerType::get($_self.getContext(), 32))">]> {
+  let summary = "sdot op";
+  let description = [{
+    Signed integer addition of dot product (vector). This instruction performs
+    the following operation on signed integer vectors: res = dot(b, c) + a,
+    where vector operands are partitioned into groups of four elements.
+
+    Source:
+    https://developer.arm.com/architectures/instruction-sets/simd-isas/neon/intrinsics
+  }];
+  // Supports either:
+  //   (vector<2xi32>, vector<8xi8>, vector<8xi8>) -> vector<2xi32>
+  //   (vector<4xi32>, vector<16xi8>, vector<16xi8>) -> vector<16xi32>
+  let arguments = (ins VectorOfLengthAndType<[4, 2], [I32]>:$a,
+                       VectorOfLengthAndType<[16, 8], [I8]>:$b,
+                       VectorOfLengthAndType<[16, 8], [I8]>:$c);
+  let results = (outs VectorOfLengthAndType<[4, 2], [I32]>:$res);
+  let assemblyFormat =
+    "$a `,` $b `,` $c attr-dict `:` type($b) `,` type($c) `to` type($res)";
+  }
 
 #endif // ARMNEON_OPS
diff --git a/mlir/include/mlir/Dialect/GPU/GPUDialect.h b/mlir/include/mlir/Dialect/GPU/GPUDialect.h
index 1e43ebeb55beb81a1ef0d74388ff7f7d4d02e322..26ab1717271467f7494d59300c7bacbe7fa23bac 100644
--- a/mlir/include/mlir/Dialect/GPU/GPUDialect.h
+++ b/mlir/include/mlir/Dialect/GPU/GPUDialect.h
@@ -14,6 +14,7 @@
 #ifndef MLIR_DIALECT_GPU_GPUDIALECT_H
 #define MLIR_DIALECT_GPU_GPUDIALECT_H
 
+#include "mlir/Dialect/DLTI/Traits.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
diff --git a/mlir/include/mlir/Dialect/GPU/GPUOps.td b/mlir/include/mlir/Dialect/GPU/GPUOps.td
index 5fb0793030b0615d53efb8c9a030439eb599f495..41206af46ae2d4a958ec7a6142abb67a4ad39921 100644
--- a/mlir/include/mlir/Dialect/GPU/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/GPUOps.td
@@ -13,9 +13,11 @@
 #ifndef GPU_OPS
 #define GPU_OPS
 
+include "mlir/Dialect/DLTI/DLTIBase.td"
 include "mlir/Dialect/GPU/GPUBase.td"
 include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
 include "mlir/IR/SymbolInterfaces.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 
 //===----------------------------------------------------------------------===//
@@ -694,7 +696,8 @@ def GPU_BarrierOp : GPU_Op<"barrier"> {
 }
 
 def GPU_GPUModuleOp : GPU_Op<"module", [
-  IsolatedFromAbove, SymbolTable, Symbol,
+  DataLayoutOpInterface, HasDefaultDLTIDataLayout, IsolatedFromAbove,
+  SymbolTable, Symbol,
   SingleBlockImplicitTerminator<"ModuleEndOp">
 ]> {
   let summary = "A top level compilation unit containing code to be run on a GPU.";
diff --git a/mlir/include/mlir/Dialect/GPU/Passes.h b/mlir/include/mlir/Dialect/GPU/Passes.h
index 6a6a2c0678b646d7bdb4ca8d88938dc4ac8f7ef4..a207c6b2279e152f3fff4085a3556d730f066f24 100644
--- a/mlir/include/mlir/Dialect/GPU/Passes.h
+++ b/mlir/include/mlir/Dialect/GPU/Passes.h
@@ -31,13 +31,11 @@ std::unique_ptr<OperationPass<ModuleOp>> createGpuKernelOutliningPass();
 std::unique_ptr<OperationPass<FuncOp>> createGpuAsyncRegionPass();
 
 /// Collect a set of patterns to rewrite all-reduce ops within the GPU dialect.
-void populateGpuAllReducePatterns(MLIRContext *context,
-                                  OwningRewritePatternList &patterns);
+void populateGpuAllReducePatterns(RewritePatternSet &patterns);
 
 /// Collect all patterns to rewrite ops within the GPU dialect.
-inline void populateGpuRewritePatterns(MLIRContext *context,
-                                       OwningRewritePatternList &patterns) {
-  populateGpuAllReducePatterns(context, patterns);
+inline void populateGpuRewritePatterns(RewritePatternSet &patterns) {
+  populateGpuAllReducePatterns(patterns);
 }
 
 namespace gpu {
@@ -90,6 +88,10 @@ protected:
 /// annotation.
 void registerGpuSerializeToCubinPass();
 
+/// Register pass to serialize GPU kernel functions to a HSAco binary
+/// annotation.
+void registerGpuSerializeToHsacoPass();
+
 /// Generate the code for registering passes.
 #define GEN_PASS_REGISTRATION
 #include "mlir/Dialect/GPU/Passes.h.inc"
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index ddc0ed32fd7ad5223c359bd128f03d46df8fbd71..3623565d169f56cb45e8ba177678f7fd13d2bf88 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -200,7 +200,7 @@ def LLVM_ICmpOp : LLVM_Op<"icmp", [NoSideEffect]> {
     OpBuilder<(ins "ICmpPredicate":$predicate, "Value":$lhs, "Value":$rhs),
     [{
       build($_builder, $_state, IntegerType::get(lhs.getType().getContext(), 1),
-            $_builder.getI64IntegerAttr(static_cast<int64_t>(predicate)), lhs, rhs);
+            predicate, lhs, rhs);
     }]>];
   let parser = [{ return parseCmpOp<ICmpPredicate>(parser, result); }];
   let printer = [{ printICmpOp(p, *this); }];
@@ -246,14 +246,6 @@ def LLVM_FCmpOp : LLVM_Op<"fcmp", [
   let llvmBuilder = [{
     $res = builder.CreateFCmp(getLLVMCmpPredicate($predicate), $lhs, $rhs);
   }];
-  let builders = [
-    OpBuilder<(ins "FCmpPredicate":$predicate, "Value":$lhs, "Value":$rhs,
-                  CArg<"FastmathFlags", "{}">:$fmf),
-    [{
-      build($_builder, $_state, IntegerType::get(lhs.getType().getContext(), 1),
-            $_builder.getI64IntegerAttr(static_cast<int64_t>(predicate)), lhs, rhs,
-            ::mlir::LLVM::FMFAttr::get($_builder.getContext(), fmf));
-    }]>];
   let parser = [{ return parseCmpOp<FCmpPredicate>(parser, result); }];
   let printer = [{ printFCmpOp(p, *this); }];
 }
diff --git a/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt b/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt
index d0edae3979e040b26f436f3f399435bd40cb9b97..2c74207d6c95e83f87ca50dba44ebd2e4db5bc70 100644
--- a/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt
@@ -2,6 +2,8 @@ add_subdirectory(IR)
 
 set(LLVM_TARGET_DEFINITIONS Passes.td)
 mlir_tablegen(Passes.h.inc -gen-pass-decls -name Linalg)
+mlir_tablegen(Passes.capi.h.inc -gen-pass-capi-header --prefix Linalg)
+mlir_tablegen(Passes.capi.cpp.inc -gen-pass-capi-impl --prefix Linalg)
 add_public_tablegen_target(MLIRLinalgPassIncGen)
 
 add_mlir_doc(Passes -gen-pass-doc LinalgPasses ./)
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
index a706d67d2988912091a99197345ef1dcd34c6d8d..5a906ff2dafdf23d77e92f9c63af3fac4874c334 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
@@ -15,6 +15,8 @@
 
 include "mlir/IR/OpBase.td"
 
+def LinalgOperand: AnyTypeOf<[AnyRankedTensor, AnyStridedMemRef]>;
+
 def Linalg_Dialect : Dialect {
   let name = "linalg";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
index 63bee92ded7ccd55e39cec999caf6aa8bb0ca98b..fe6720761abefebcf9a8fe5c8ed9f06697842aa6 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
@@ -496,21 +496,25 @@ def Linalg_TiledLoopOp : Linalg_Op<"tiled_loop", [
   let summary = "Linalg tiled loop operation";
   let description = [{
     This is a loop-like operation with additional properties. The arguments
-    also include the input and the output tensors and the attributes to specify
-    the iterator types. The body region of the loop contains `subtensor`
-    operations applied to every tensor argument of TiledLoopOp.
+    also include the input and the output tensors or memrefs and the attributes
+    to specify the iterator types.
+
+    Parsing TiledLoopOp will set all elements of the `iterator_types` attribute
+    to "parallel" type, when it is absent from the custom format.
+
+    Tensor-based version:
+
+    The body region of the loop contains `subtensor` operations applied to
+    every tensor argument of TiledLoopOp.
 
     The body region must contain exactly one block that terminates with
     `linalg.yield` with the operands resulting from `subtensor_insert`
     operations.
 
-    Parsing TiledLoopOp will set all elements of the `iterator_types` attribute
-    to "parallel" type, when it is absent from the custom format.
-
     Example:
 
     ```mlir
-    linalg.tiled_loop (%i) = (%c0) to (%c24) step (%c4)
+    %0 = linalg.tiled_loop (%i) = (%c0) to (%c24) step (%c4)
         ins(%lhs, %rhs : tensor<24x64xi8>, tensor<24x64xi8>)
         outs(%out : tensor<24x64xi8>)
         iterators("parallel") {
@@ -528,13 +532,40 @@ def Linalg_TiledLoopOp : Linalg_Op<"tiled_loop", [
       linalg.yield %result : tensor<24x64xi8>
     }
     ```
+
+    MemRef-based version:
+
+    The body region of the loop contains `subview` operations applied to
+    every memref argument of TiledLoopOp.
+
+    The body region must contain exactly one block that terminates with
+    `linalg.yield` with no operands.
+
+    Example:
+
+    ```mlir
+    linalg.tiled_loop (%i) = (%c0) to (%c24) step (%c4)
+        ins(%lhs, %rhs : memref<24x64xi8>, memref<24x64xi8>)
+        outs(%out : memref<24x64xi8>)
+        iterators("parallel") {
+      %lhs_sub = subview %lhs[%i, 0] [%c4, %c64] [1, 1]
+          : memref<24x64xi8> to memref<?x?xi8>
+      %rhs_sub = subview %rhs[%i, 0] [%c4, %c64] [1, 1]
+          : memref<24x64xi8> to memref<?x?xi8>
+      %out_sub = subview %out[%i, 0] [%c4, %c64] [1, 1]
+          : memref<24x64xi8> to memref<?x?xi8>
+
+      %result_sub = linalg.generic ...
+      linalg.yield
+    }
+    ```
   }];
 
   let arguments = (ins Variadic<Index>:$lowerBound,
                        Variadic<Index>:$upperBound,
                        Variadic<Index>:$step,
-                       Variadic<AnyRankedTensor>:$inputs,
-                       Variadic<AnyRankedTensor>:$outputs,
+                       Variadic<LinalgOperand>:$inputs,
+                       Variadic<LinalgOperand>:$outputs,
                        ArrayAttr:$iterator_types);
   let results = (outs Variadic<AnyRankedTensor>:$results);
   let regions = (region SizedRegion<1>:$region);
@@ -542,7 +573,7 @@ def Linalg_TiledLoopOp : Linalg_Op<"tiled_loop", [
   let builders = [
     OpBuilder<(ins "ValueRange":$lowerBounds, "ValueRange":$upperBounds,
       "ValueRange":$steps, "ValueRange":$inputs, "ValueRange":$outputs,
-      "ArrayRef<StringRef>":$iteratorTypes,
+      "ArrayAttr":$iteratorTypes,
       CArg<"function_ref<void (OpBuilder &, Location, ValueRange)>",
            "nullptr">:$bodyBuilderFn)>,
   ];
@@ -553,6 +584,9 @@ def Linalg_TiledLoopOp : Linalg_Op<"tiled_loop", [
     }
     unsigned getNumLoops() { return step().size(); }
   }];
+
+  let hasCanonicalizer = 1;
+  let hasFolder = 1;
 }
 
 
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index f87a1eaeac8f1b7ede67b2e372e8d4f7ee4710f4..69aa7659b81cfa39b46658e1d255700d65947b0d 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -496,8 +496,6 @@ def PoolingSumOp: SingleInputPoolingBase_Op<"pooling_sum"> {
 //===----------------------------------------------------------------------===//
 // Generic Linalg ops.
 //===----------------------------------------------------------------------===//
-def LinalgOperand: AnyTypeOf<[AnyRankedTensor, AnyStridedMemRef]>;
-
 class LinalgOperandOfRank<int rank>: Type<
   And<[
     LinalgOperand.predicate,
diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.h b/mlir/include/mlir/Dialect/Linalg/Passes.h
index 34e25689763c726304e58d9921f4fc1e80c95c86..18820d4316b91db8f22bf26e82a200fbd5ff046d 100644
--- a/mlir/include/mlir/Dialect/Linalg/Passes.h
+++ b/mlir/include/mlir/Dialect/Linalg/Passes.h
@@ -52,8 +52,7 @@ std::unique_ptr<OperationPass<FuncOp>> createLinalgBufferizePass();
 
 /// Populate patterns that convert `ElementwiseMappable` ops to linalg
 /// parallel loops.
-void populateElementwiseToLinalgConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx);
+void populateElementwiseToLinalgConversionPatterns(RewritePatternSet &patterns);
 
 /// Create a pass to conver named Linalg operations to Linalg generic
 /// operations.
@@ -67,14 +66,13 @@ std::unique_ptr<Pass> createLinalgDetensorizePass();
 /// producer (consumer) generic operation by expanding the dimensionality of the
 /// loop in the generic op.
 void populateFoldReshapeOpsByExpansionPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns);
+    RewritePatternSet &patterns, bool allowFoldingUnitDimReshapes = false);
 
 /// Patterns to fold a collapsing (expanding) tensor_reshape operation with its
 /// producer (consumer) generic/indexed_generic operation by linearizing the
 /// indexing map used to access the source (target) of the reshape operation in
 /// the generic/indexed_generic operation.
-void populateFoldReshapeOpsByLinearizationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns);
+void populateFoldReshapeOpsByLinearizationPatterns(RewritePatternSet &patterns);
 
 /// Patterns to fold a collapsing (expanding) tensor_reshape operation with its
 /// producer (consumer) generic/indexed_generic operation by linearizing the
@@ -83,16 +81,15 @@ void populateFoldReshapeOpsByLinearizationPatterns(
 /// the tensor reshape involved is collapsing (introducing) unit-extent
 /// dimensions.
 void populateFoldUnitDimsReshapeOpsByLinearizationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns);
+    RewritePatternSet &patterns);
 
 /// Patterns for fusing linalg operation on tensors.
-void populateLinalgTensorOpsFusionPatterns(MLIRContext *context,
-                                           OwningRewritePatternList &patterns);
+void populateLinalgTensorOpsFusionPatterns(
+    RewritePatternSet &patterns, bool allowFoldingUnitDimReshapes = false);
 
 /// Patterns to fold unit-extent dimensions in operands/results of linalg ops on
 /// tensors.
-void populateLinalgFoldUnitExtentDimsPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns);
+void populateLinalgFoldUnitExtentDimsPatterns(RewritePatternSet &patterns);
 
 //===----------------------------------------------------------------------===//
 // Registration
diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.td b/mlir/include/mlir/Dialect/Linalg/Passes.td
index aad11179be697534dc053551b1d82248dcd3afdc..786b9ec85dcf14358f521aa9b9c24e91babb0895 100644
--- a/mlir/include/mlir/Dialect/Linalg/Passes.td
+++ b/mlir/include/mlir/Dialect/Linalg/Passes.td
@@ -37,6 +37,12 @@ def LinalgFoldUnitExtentDims : FunctionPass<"linalg-fold-unit-extent-dims"> {
 def LinalgFusionOfTensorOps : Pass<"linalg-fusion-for-tensor-ops"> {
   let summary = "Fuse operations on RankedTensorType in linalg dialect";
   let constructor = "mlir::createLinalgFusionOfTensorOpsPass()";
+  let options = [
+    Option<"allowFoldingUnitDimReshapes", "allow-folding-unit-dim-reshapes",
+           "bool", /*default=*/"false",
+           "Allow fusing linalg.tensor_reshape ops that performs unit "
+           "dimension collapsing">
+  ];
   let dependentDialects = ["linalg::LinalgDialect", "AffineDialect"];
 }
 
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/CodegenStrategy.h b/mlir/include/mlir/Dialect/Linalg/Transforms/CodegenStrategy.h
index 872e76387d847cf061784d0f9ad36d095812fbfc..bee0d5a12800fe5871a40e8968f4196c28eb5dfd 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/CodegenStrategy.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/CodegenStrategy.h
@@ -24,7 +24,7 @@ struct Transformation {
   explicit Transformation(linalg::LinalgTransformationFilter::FilterFunction f)
       : filter(f) {}
   virtual ~Transformation() = default;
-  virtual OwningRewritePatternList
+  virtual RewritePatternSet
   buildRewritePatterns(MLIRContext *context,
                        linalg::LinalgTransformationFilter m) = 0;
   linalg::LinalgTransformationFilter::FilterFunction filter = nullptr;
@@ -35,38 +35,39 @@ template <template <typename> class PatternType, typename ConcreteOpType,
           typename OptionsType,
           typename = std::enable_if_t<std::is_member_function_pointer<
               decltype(&ConcreteOpType::getOperationName)>::value>>
-void sfinae_enqueue(OwningRewritePatternList &patternList, OptionsType options,
-                    MLIRContext *context, StringRef opName,
-                    linalg::LinalgTransformationFilter m) {
+void sfinae_enqueue(RewritePatternSet &patternList, OptionsType options,
+                    StringRef opName, linalg::LinalgTransformationFilter m) {
   assert(opName == ConcreteOpType::getOperationName() &&
          "explicit name must match ConcreteOpType::getOperationName");
-  patternList.insert<PatternType<ConcreteOpType>>(context, options, m);
+  patternList.add<PatternType<ConcreteOpType>>(patternList.getContext(),
+                                               options, m);
 }
 
 /// SFINAE: Enqueue helper for OpType that do not have a `getOperationName`
 /// (e.g. LinalgOp, other interfaces, Operation*).
 template <template <typename> class PatternType, typename OpType,
           typename OptionsType>
-void sfinae_enqueue(OwningRewritePatternList &patternList, OptionsType options,
-                    MLIRContext *context, StringRef opName,
-                    linalg::LinalgTransformationFilter m) {
+void sfinae_enqueue(RewritePatternSet &patternList, OptionsType options,
+                    StringRef opName, linalg::LinalgTransformationFilter m) {
   assert(!opName.empty() && "opName must not be empty");
-  patternList.insert<PatternType<OpType>>(opName, context, options, m);
+  patternList.add<PatternType<OpType>>(opName, patternList.getContext(),
+                                       options, m);
 }
 
 template <typename PatternType, typename OpType, typename OptionsType>
-void enqueue(OwningRewritePatternList &patternList, OptionsType options,
-             MLIRContext *context, StringRef opName,
-             linalg::LinalgTransformationFilter m) {
+void enqueue(RewritePatternSet &patternList, OptionsType options,
+             StringRef opName, linalg::LinalgTransformationFilter m) {
   if (!opName.empty())
-    patternList.insert<PatternType>(opName, context, options, m);
+    patternList.add<PatternType>(opName, patternList.getContext(), options, m);
   else
-    patternList.insert<PatternType>(m.addOpFilter<OpType>(), options);
+    patternList.add<PatternType>(patternList.getContext(),
+                                 m.addOpFilter<OpType>(), options);
 }
 
 /// Promotion transformation enqueues a particular stage-1 pattern for
 /// `Tile<LinalgOpType>`with the appropriate `options`.
-template <typename LinalgOpType> struct Tile : public Transformation {
+template <typename LinalgOpType>
+struct Tile : public Transformation {
   explicit Tile(linalg::LinalgTilingOptions options,
                 linalg::LinalgTransformationFilter::FilterFunction f = nullptr)
       : Transformation(f), opName(LinalgOpType::getOperationName()),
@@ -76,12 +77,12 @@ template <typename LinalgOpType> struct Tile : public Transformation {
        linalg::LinalgTransformationFilter::FilterFunction f = nullptr)
       : Transformation(f), opName(name), options(options) {}
 
-  OwningRewritePatternList
+  RewritePatternSet
   buildRewritePatterns(MLIRContext *context,
                        linalg::LinalgTransformationFilter m) override {
-    OwningRewritePatternList tilingPatterns;
+    RewritePatternSet tilingPatterns(context);
     sfinae_enqueue<linalg::LinalgTilingPattern, LinalgOpType>(
-        tilingPatterns, options, context, opName, m);
+        tilingPatterns, options, opName, m);
     return tilingPatterns;
   }
 
@@ -92,7 +93,8 @@ private:
 
 /// Promotion transformation enqueues a particular stage-1 pattern for
 /// `Promote<LinalgOpType>`with the appropriate `options`.
-template <typename LinalgOpType> struct Promote : public Transformation {
+template <typename LinalgOpType>
+struct Promote : public Transformation {
   explicit Promote(
       linalg::LinalgPromotionOptions options,
       linalg::LinalgTransformationFilter::FilterFunction f = nullptr)
@@ -103,12 +105,12 @@ template <typename LinalgOpType> struct Promote : public Transformation {
           linalg::LinalgTransformationFilter::FilterFunction f = nullptr)
       : Transformation(f), opName(name), options(options) {}
 
-  OwningRewritePatternList
+  RewritePatternSet
   buildRewritePatterns(MLIRContext *context,
                        linalg::LinalgTransformationFilter m) override {
-    OwningRewritePatternList promotionPatterns;
+    RewritePatternSet promotionPatterns(context);
     sfinae_enqueue<linalg::LinalgPromotionPattern, LinalgOpType>(
-        promotionPatterns, options, context, opName, m);
+        promotionPatterns, options, opName, m);
     return promotionPatterns;
   }
 
@@ -131,14 +133,14 @@ struct Vectorize : public Transformation {
             linalg::LinalgTransformationFilter::FilterFunction f = nullptr)
       : Transformation(f), opName(name), options(options) {}
 
-  OwningRewritePatternList
+  RewritePatternSet
   buildRewritePatterns(MLIRContext *context,
                        linalg::LinalgTransformationFilter m) override {
-    OwningRewritePatternList vectorizationPatterns;
+    RewritePatternSet vectorizationPatterns(context);
     enqueue<linalg::LinalgVectorizationPattern, LinalgOpType>(
-        vectorizationPatterns, options, context, opName, m);
-    vectorizationPatterns.insert<linalg::LinalgCopyVTRForwardingPattern,
-                                 linalg::LinalgCopyVTWForwardingPattern>(
+        vectorizationPatterns, options, opName, m);
+    vectorizationPatterns.add<linalg::LinalgCopyVTRForwardingPattern,
+                              linalg::LinalgCopyVTWForwardingPattern>(
         context, /*benefit=*/2);
     return vectorizationPatterns;
   }
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 54a4aec9f86700128ef408f2735492b1c68d8052..21e6cba9dc3ce3d028c533fc27918d801f2e4b60 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -19,7 +19,7 @@
 
 namespace mlir {
 class BufferizeTypeConverter;
-class FrozenRewritePatternList;
+class FrozenRewritePatternSet;
 
 namespace linalg {
 
@@ -33,13 +33,12 @@ using LinalgLoops = SmallVector<Operation *, 4>;
 
 /// Populates patterns for vectorization of all ConvN-D ops.
 void populateConvVectorizationPatterns(
-    MLIRContext *context, SmallVectorImpl<OwningRewritePatternList> &patterns,
+    MLIRContext *context, SmallVectorImpl<RewritePatternSet> &patterns,
     ArrayRef<int64_t> tileSizes);
 
 /// Populates the given list with patterns to bufferize linalg ops.
-void populateLinalgBufferizePatterns(MLIRContext *context,
-                                     BufferizeTypeConverter &converter,
-                                     OwningRewritePatternList &patterns);
+void populateLinalgBufferizePatterns(BufferizeTypeConverter &converter,
+                                     RewritePatternSet &patterns);
 
 /// Performs standalone tiling of a single LinalgOp by `tileSizes`.
 /// and permute the loop nest according to `interchangeVector`
@@ -442,10 +441,8 @@ struct LinalgTilingOptions {
 /// Canonicalization patterns relevant to apply after tiling patterns. These are
 /// applied automatically by the tiling pass but need to be applied manually
 /// when tiling is called programmatically.
-OwningRewritePatternList
-getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx);
-void populateLinalgTilingCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx);
+RewritePatternSet getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx);
+void populateLinalgTilingCanonicalizationPatterns(RewritePatternSet &patterns);
 
 /// Base pattern that applied the tiling transformation specified by `options`.
 /// Abort and return failure in 2 cases:
@@ -455,7 +452,7 @@ void populateLinalgTilingCanonicalizationPatterns(
 struct LinalgBaseTilingPattern : public RewritePattern {
   // Entry point to match any LinalgOp OpInterface.
   LinalgBaseTilingPattern(
-      LinalgTilingOptions options,
+      MLIRContext *context, LinalgTilingOptions options,
       LinalgTransformationFilter filter = LinalgTransformationFilter(),
       PatternBenefit benefit = 1);
   // Entry point to match a specific Linalg op.
@@ -647,7 +644,8 @@ struct LinalgVectorizationOptions {};
 
 struct LinalgBaseVectorizationPattern : public RewritePattern {
   /// MatchAnyOpTag-based constructor with a mandatory `filter`.
-  LinalgBaseVectorizationPattern(LinalgTransformationFilter filter,
+  LinalgBaseVectorizationPattern(MLIRContext *context,
+                                 LinalgTransformationFilter filter,
                                  PatternBenefit benefit = 1);
   /// Name-based constructor with an optional `filter`.
   LinalgBaseVectorizationPattern(
@@ -666,10 +664,10 @@ struct LinalgVectorizationPattern : public LinalgBaseVectorizationPattern {
   /// These constructors are available to anyone.
   /// MatchAnyOpTag-based constructor with a mandatory `filter`.
   LinalgVectorizationPattern(
-      LinalgTransformationFilter filter,
+      MLIRContext *context, LinalgTransformationFilter filter,
       LinalgVectorizationOptions options = LinalgVectorizationOptions(),
       PatternBenefit benefit = 1)
-      : LinalgBaseVectorizationPattern(filter, benefit) {}
+      : LinalgBaseVectorizationPattern(context, filter, benefit) {}
   /// Name-based constructor with an optional `filter`.
   LinalgVectorizationPattern(
       StringRef opName, MLIRContext *context,
@@ -691,37 +689,34 @@ template <
     typename OpType,
     typename = std::enable_if_t<detect_has_get_operation_name<OpType>::value>,
     typename = void>
-void insertVectorizationPatternImpl(OwningRewritePatternList &patternList,
-                                    MLIRContext *context,
+void insertVectorizationPatternImpl(RewritePatternSet &patternList,
                                     linalg::LinalgVectorizationOptions options,
                                     linalg::LinalgTransformationFilter f) {
-  patternList.insert<linalg::LinalgVectorizationPattern>(
-      OpType::getOperationName(), context, options, f);
+  patternList.add<linalg::LinalgVectorizationPattern>(
+      OpType::getOperationName(), patternList.getContext(), options, f);
 }
 
 /// SFINAE helper for single C++ class without a `getOperationName` method (e.g.
 /// an OpInterface).
 template <typename OpType, typename = std::enable_if_t<
                                !detect_has_get_operation_name<OpType>::value>>
-void insertVectorizationPatternImpl(OwningRewritePatternList &patternList,
-                                    MLIRContext *context,
+void insertVectorizationPatternImpl(RewritePatternSet &patternList,
                                     linalg::LinalgVectorizationOptions options,
                                     linalg::LinalgTransformationFilter f) {
-  patternList.insert<linalg::LinalgVectorizationPattern>(
-      f.addOpFilter<OpType>(), options);
+  patternList.add<linalg::LinalgVectorizationPattern>(
+      patternList.getContext(), f.addOpFilter<OpType>(), options);
 }
 
 /// Variadic helper function to insert vectorization patterns for C++ ops.
 template <typename... OpTypes>
-void insertVectorizationPatterns(OwningRewritePatternList &patternList,
-                                 MLIRContext *context,
+void insertVectorizationPatterns(RewritePatternSet &patternList,
                                  linalg::LinalgVectorizationOptions options,
                                  linalg::LinalgTransformationFilter f =
                                      linalg::LinalgTransformationFilter()) {
   // FIXME: In c++17 this can be simplified by using 'fold expressions'.
-  (void)std::initializer_list<int>{0, (insertVectorizationPatternImpl<OpTypes>(
-                                           patternList, context, options, f),
-                                       0)...};
+  (void)std::initializer_list<int>{
+      0,
+      (insertVectorizationPatternImpl<OpTypes>(patternList, options, f), 0)...};
 }
 
 ///
@@ -743,7 +738,7 @@ struct LinalgLoweringPattern : public RewritePattern {
       MLIRContext *context, LinalgLoweringType loweringType,
       LinalgTransformationFilter filter = LinalgTransformationFilter(),
       ArrayRef<unsigned> interchangeVector = {}, PatternBenefit benefit = 1)
-      : RewritePattern(OpTy::getOperationName(), {}, benefit, context),
+      : RewritePattern(OpTy::getOperationName(), benefit, context),
         filter(filter), loweringType(loweringType),
         interchangeVector(interchangeVector.begin(), interchangeVector.end()) {}
 
@@ -793,13 +788,13 @@ private:
 /// Populates `patterns` with patterns to convert spec-generated named ops to
 /// linalg.generic ops.
 void populateLinalgNamedOpsGeneralizationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns,
+    RewritePatternSet &patterns,
     LinalgTransformationFilter filter = LinalgTransformationFilter());
 
 /// Populates `patterns` with patterns to convert linalg.conv ops to
 /// linalg.generic ops.
 void populateLinalgConvGeneralizationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns,
+    RewritePatternSet &patterns,
     LinalgTransformationFilter filter = LinalgTransformationFilter());
 
 //===----------------------------------------------------------------------===//
@@ -893,6 +888,30 @@ struct AffineMinSCFCanonicalizationPattern
                                 PatternRewriter &rewriter) const override;
 };
 
+/// Helper struct to return the results of `substituteMin`.
+struct AffineMapAndOperands {
+  AffineMap map;
+  SmallVector<Value> dims;
+  SmallVector<Value> symbols;
+};
+/// Traverse the dims of the AffineMap of `affineMinOp` and substitute scf loop
+/// induction variables by new expressions involving the lower or upper bound:
+///   - If the AffineDimExpr mapped to a loop IV has a positive sign, it is
+///     replaced by the loop upper bound.
+///   - If the AffineDimExpr mapped to a loop IV has a negative sign, it is
+///     replaced by the loop lower bound.
+/// All loop induction variables are iteratively replaced, unless a
+/// `substituteOperation` hook is passed to more finely determine which
+/// operations are substituted.
+/// This is used as an intermediate step in computing bounding boxes and
+/// canonicalize AffineMinOps. All dim and symbol operands are assumed to have
+/// positive values (positive orthant assumptions).
+/// Return a new AffineMap, dims and symbols that have been canonicalized and
+/// simplified.
+AffineMapAndOperands substituteMin(
+    AffineMinOp affineMinOp,
+    llvm::function_ref<bool(Operation *)> substituteOperation = nullptr);
+
 /// Converts Convolution op into vector contraction.
 ///
 /// Conversion expects ConvOp to have dimensions marked in the *mask* as
@@ -946,8 +965,8 @@ public:
 //===----------------------------------------------------------------------===//
 /// Helper function to allow applying rewrite patterns, interleaved with more
 /// global transformations, in a staged fashion:
-///   1. the first stage consists of a list of FrozenRewritePatternList. Each
-///   FrozenRewritePatternList in this list is applied once, in order.
+///   1. the first stage consists of a list of FrozenRewritePatternSet. Each
+///   FrozenRewritePatternSet in this list is applied once, in order.
 ///   2. the second stage consists of a single OwningRewritePattern that is
 ///   applied greedily until convergence.
 ///   3. the third stage consists of applying a lambda, generally used for
@@ -955,8 +974,8 @@ public:
 ///   transformations where patterns can be ordered and applied at a finer
 ///   granularity than a sequence of traditional compiler passes.
 LogicalResult applyStagedPatterns(
-    Operation *op, ArrayRef<FrozenRewritePatternList> stage1Patterns,
-    const FrozenRewritePatternList &stage2Patterns,
+    Operation *op, ArrayRef<FrozenRewritePatternSet> stage1Patterns,
+    const FrozenRewritePatternSet &stage2Patterns,
     function_ref<LogicalResult(Operation *)> stage3Lambda = nullptr);
 
 //===----------------------------------------------------------------------===//
@@ -1036,12 +1055,11 @@ struct SparsificationOptions {
 
 /// Sets up sparsification rewriting rules with the given options.
 void populateSparsificationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns,
+    RewritePatternSet &patterns,
     const SparsificationOptions &options = SparsificationOptions());
 
 /// Sets up sparsification conversion rules with the given options.
-void populateSparsificationConversionPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns);
+void populateSparsificationConversionPatterns(RewritePatternSet &patterns);
 
 } // namespace linalg
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
index 2dc208f429f439a5ba4f45cba9621c3c0579018f..33efeddadc9e5beef07d73ee55ef8b4ab344034f 100644
--- a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
@@ -17,13 +17,9 @@
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/StandardOps/EDSC/Intrinsics.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
-
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SetVector.h"
 
-using mlir::edsc::intrinsics::AffineIndexedValue;
-using mlir::edsc::intrinsics::MemRefIndexedValue;
-
 namespace mlir {
 class AffineExpr;
 class AffineForOp;
@@ -34,33 +30,32 @@ class PatternRewriter;
 namespace linalg {
 class LinalgDependenceGraph;
 
-/// A struct containing the Linalg producer before and after fusion.
-/// When operating on tensors, `fusedProducer` may feed into a `tensor.cast` op
-/// before the consumer Linalg op, until enough canonicalizations have applied.
-struct FusionInfo {
-  LinalgOp originalProducer;
-  LinalgOp fusedProducer;
-};
+//===----------------------------------------------------------------------===//
+// General utilities
+//===----------------------------------------------------------------------===//
 
-/// A struct containing common matchers over linalg op's region.
-struct RegionMatcher {
-  enum class BinaryOpKind {
-    IAdd,
-  };
+/// Apply the permutation defined by `permutation` to `inVec`.
+/// Element `i` in `inVec` is mapped to location `j = permutation[i]`.
+/// E.g.: for an input vector `inVec = ['a', 'b', 'c']` and a permutation vector
+/// `permutation = [2, 0, 1]`, this function leaves `inVec = ['c', 'a', 'b']`.
+template <typename T, unsigned N>
+void applyPermutationToVector(SmallVector<T, N> &inVec,
+                              ArrayRef<unsigned> permutation) {
+  SmallVector<T, N> auxVec(inVec.size());
+  for (unsigned i = 0; i < permutation.size(); ++i)
+    auxVec[i] = inVec[permutation[i]];
+  inVec = auxVec;
+}
 
-  /// Matches the given linalg op if its body is performing binary operation on
-  /// int or float scalar values and returns the binary op kind.
-  ///
-  /// The linalg op's region is expected to be
-  /// ```
-  /// {
-  ///   ^bb(%a: <scalar-type>, %b: <scalar-type>):
-  ///     %0 = <binary-op> %a, %b: <scalar-type>
-  ///     linalg.yield %0: <scalar-type>
-  /// }
-  /// ```
-  static Optional<BinaryOpKind> matchAsScalarBinaryOp(GenericOp op);
-};
+/// If `size` comes from an AffineMinOp and one of the values of AffineMinOp
+/// is a constant then return a new value set to the smallest such constant.
+/// If `size` comes from a ConstantOp, return the constant.
+/// Otherwise return nullptr.
+IntegerAttr getSmallestBoundingIndex(Value size);
+
+//===----------------------------------------------------------------------===//
+// Iterator type utilities
+//===----------------------------------------------------------------------===//
 
 /// Checks if an iterator_type attribute is parallel.
 bool isParallelIteratorType(Attribute attr);
@@ -71,6 +66,10 @@ bool isReductionIteratorType(Attribute attr);
 /// Checks if an iterator_type attribute is parallel.
 bool isWindowIteratorType(Attribute attr);
 
+//===----------------------------------------------------------------------===//
+// Fusion utilities
+//===----------------------------------------------------------------------===//
+
 /// Checks whether the specific `producer` is the last write to exactly the
 /// whole `consumedView`. This checks structural dominance, that the dependence
 /// is a RAW without any interleaved write to any piece of `consumedView`.
@@ -84,6 +83,21 @@ bool isProducerLastWriteOfView(const LinalgDependenceGraph &graph,
 bool isFusableInto(const LinalgDependenceGraph &graph, LinalgOp consumer,
                    Value consumedView, LinalgOp producer);
 
+/// Creates subtensor/subview ops for all `tiledOperands` of the given
+/// `linalgOp` with `builder`, assuming `linalgOp` is being fused into a loop
+/// nest for tiling with the given induction variables `ivs` and tile sizes
+/// `tileSizes`. `sizeBounds` are the iteration space bounds for *all* the
+/// implicit loops in `linalgOp`.
+///
+/// Note that a constant zero in `tileSizes` means no tiling at that implicit
+/// loop. The number of non-zero values in `tileSizes` should be equal to the
+/// number of values in `ivs`.
+SmallVector<Value, 4> makeTiledShapes(OpBuilder &builder, Location loc,
+                                      LinalgOp linalgOp,
+                                      ArrayRef<Value> tiledOperands,
+                                      ValueRange ivs, ValueRange tileSizes,
+                                      ArrayRef<Value> sizeBounds);
+
 using FusableOpDependencesTy = llvm::MapVector<
     Operation *,
     SmallVector<LinalgDependenceGraph::LinalgDependenceGraphElem, 1>>;
@@ -91,6 +105,14 @@ FusableOpDependencesTy
 findAllFusableDependences(ArrayRef<LinalgOp> ops,
                           const LinalgDependenceGraph &dependenceGraph);
 
+/// A struct containing the Linalg producer before and after fusion.
+/// When operating on tensors, `fusedProducer` may feed into a `tensor.cast` op
+/// before the consumer Linalg op, until enough canonicalizations have applied.
+struct FusionInfo {
+  LinalgOp originalProducer;
+  LinalgOp fusedProducer;
+};
+
 /// Fuses producer into consumer if the producer is structurally feasible and
 /// the fusion would not violate dependencies.
 /// Implements the fusion part of the "tileAndFuse on buffers" transformation
@@ -119,24 +141,9 @@ Optional<FusionInfo> fuseProducerOfTensor(OpBuilder &b,
 Optional<SmallVector<Value, 1>> fuseTensorOps(PatternRewriter &rewriter,
                                               OpOperand &consumerOpOperand);
 
-/// Apply the permutation defined by `permutation` to `inVec`.
-/// Element `i` in `inVec` is mapped to location `j = permutation[i]`.
-/// E.g.: for an input vector `inVec = ['a', 'b', 'c']` and a permutation vector
-/// `permutation = [2, 0, 1]`, this function leaves `inVec = ['c', 'a', 'b']`.
-template <typename T, unsigned N>
-void applyPermutationToVector(SmallVector<T, N> &inVec,
-                              ArrayRef<unsigned> permutation) {
-  SmallVector<T, N> auxVec(inVec.size());
-  for (unsigned i = 0; i < permutation.size(); ++i)
-    auxVec[i] = inVec[permutation[i]];
-  inVec = auxVec;
-}
-
-/// If `size` comes from an AffineMinOp and one of the values of AffineMinOp
-/// is a constant then return a new value set to the smallest such constant.
-/// If `size` comes from a ConstantOp, return the constant.
-/// Otherwise return nullptr.
-IntegerAttr getSmallestBoundingIndex(Value size);
+//===----------------------------------------------------------------------===//
+// Distribution utilities
+//===----------------------------------------------------------------------===//
 
 /// Scheme used to distribute loops to processors.
 enum class DistributionMethod {
@@ -206,6 +213,34 @@ struct LinalgLoopDistributionOptions {
   SmallVector<DistributionMethod, 0> distributionMethod = {};
 };
 
+//===----------------------------------------------------------------------===//
+// Generic op region utilities
+//===----------------------------------------------------------------------===//
+
+/// A struct containing common matchers over linalg op's region.
+struct RegionMatcher {
+  enum class BinaryOpKind {
+    IAdd,
+  };
+
+  /// Matches the given linalg op if its body is performing binary operation on
+  /// int or float scalar values and returns the binary op kind.
+  ///
+  /// The linalg op's region is expected to be
+  /// ```
+  /// {
+  ///   ^bb(%a: <scalar-type>, %b: <scalar-type>):
+  ///     %0 = <binary-op> %a, %b: <scalar-type>
+  ///     linalg.yield %0: <scalar-type>
+  /// }
+  /// ```
+  static Optional<BinaryOpKind> matchAsScalarBinaryOp(GenericOp op);
+};
+
+//===----------------------------------------------------------------------===//
+// Loop nest utilities
+//===----------------------------------------------------------------------===//
+
 /// Utility class used to generate nested loops with ranges described by
 /// `loopRanges` and loop type described by the `iteratorTypes`. `bodyBuilderFn`
 /// is used to generate the body of the innermost loop. It is passed a range
@@ -214,7 +249,8 @@ template <typename LoopTy>
 struct GenerateLoopNest {
   using IndexedValueTy =
       typename std::conditional<std::is_same<LoopTy, AffineForOp>::value,
-                                AffineIndexedValue, MemRefIndexedValue>::type;
+                                edsc::intrinsics::AffineIndexedValue,
+                                edsc::intrinsics::MemRefIndexedValue>::type;
 
   static void
   doit(ArrayRef<Range> loopRanges, ValueRange iterArgInitValues,
diff --git a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
index c965bab3769bb085e143cbd08652667c9172ddae..10635667a5fccb411af7cd4881a1fdb95045258c 100644
--- a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
@@ -9,18 +9,13 @@
 #ifndef MLIR_DIALECT_MATH_TRANSFORMS_PASSES_H_
 #define MLIR_DIALECT_MATH_TRANSFORMS_PASSES_H_
 
-#include "mlir/Pass/Pass.h"
-#include "mlir/Transforms/Bufferize.h"
-
 namespace mlir {
 
-class OwningRewritePatternList;
+class RewritePatternSet;
 
-void populateExpandTanhPattern(OwningRewritePatternList &patterns,
-                               MLIRContext *ctx);
+void populateExpandTanhPattern(RewritePatternSet &patterns);
 
-void populateMathPolynomialApproximationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx);
+void populateMathPolynomialApproximationPatterns(RewritePatternSet &patterns);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h b/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
index 9c2b912c0df15543f119eefa14610e73784d3857..0542423977835ca9bd166f98d189752213d6ffa5 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
@@ -12,6 +12,7 @@
 #include "mlir/IR/Dialect.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "mlir/Interfaces/CastInterfaces.h"
+#include "mlir/Interfaces/CopyOpInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
 
diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index 4e4e2ecc25054e459752618a7715f3089bc21055..fe0fd7d0ff363bb78a239d6d5df902bfaf7ec26b 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -10,7 +10,9 @@
 #define MEMREF_OPS
 
 include "mlir/Dialect/MemRef/IR/MemRefBase.td"
+include "mlir/IR/OpBase.td"
 include "mlir/Interfaces/CastInterfaces.td"
+include "mlir/Interfaces/CopyOpInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 include "mlir/Interfaces/ViewLikeInterface.td"
 include "mlir/IR/SymbolInterfaces.td"
@@ -106,28 +108,6 @@ def AssumeAlignmentOp : MemRef_Op<"assume_alignment"> {
   let assemblyFormat = "$memref `,` $alignment attr-dict `:` type($memref)";
 }
 
-//===----------------------------------------------------------------------===//
-// BaseOpWithOffsetSizesAndStrides
-//===----------------------------------------------------------------------===//
-
-// Base class for ops with static/dynamic offset, sizes and strides
-// attributes/arguments.
-class BaseOpWithOffsetSizesAndStrides<string mnemonic, list<OpTrait> traits = []> :
-    MemRef_Op<mnemonic,
-           !listconcat(traits, [NoSideEffect, AttrSizedOperandSegments])> {
-  code extraBaseClassDeclaration = [{
-    /// Returns the dynamic sizes for this subview operation if specified.
-    operand_range getDynamicSizes() { return sizes(); }
-
-    /// Return the list of Range (i.e. offset, size, stride). Each
-    /// Range entry contains either the dynamic value or a ConstantIndexOp
-    /// constructed with `b` at location `loc`.
-    SmallVector<Range, 8> getOrCreateRanges(OpBuilder &b, Location loc) {
-      return mlir::getOrCreateRanges(*this, b, loc);
-    }
-  }];
-}
-
 //===----------------------------------------------------------------------===//
 // AllocOp
 //===----------------------------------------------------------------------===//
@@ -235,6 +215,9 @@ def MemRef_BufferCastOp : MemRef_Op<"buffer_cast",
     // Result type is tensor<4x?xf32>
     %12 = memref.buffer_cast %10 : memref<4x?xf32, #map0, 42>
     ```
+
+    Note, that mutating the result of the buffer cast operation leads to
+    undefined behavior.
   }];
 
   let arguments = (ins AnyTensor:$tensor);
@@ -333,6 +316,46 @@ def MemRef_CastOp : MemRef_Op<"cast", [
   let hasFolder = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// CloneOp
+//===----------------------------------------------------------------------===//
+
+def CloneOp : MemRef_Op<"clone", [
+    CopyOpInterface,
+    DeclareOpInterfaceMethods<MemoryEffectsOpInterface>
+  ]> {
+  let builders = [
+    OpBuilder<(ins "Value":$value), [{
+      return build($_builder, $_state, value.getType(), value);
+    }]>];
+
+  let description = [{
+    Clones the data in the input view into an implicitly defined output view.
+
+    Usage:
+
+    ```mlir
+    %arg1 = memref.clone %arg0 : memref<?xf32> to memref<?xf32>
+    ```
+
+    Note, that mutating the source or result of the clone operation leads to
+    undefined behavior.
+  }];
+
+  let arguments = (ins Arg<AnyMemRef, "", []>:$input);
+  let results = (outs Arg<AnyMemRef, "", []>:$output);
+
+  let extraClassDeclaration = [{
+    Value getSource() { return input();}
+    Value getTarget() { return output(); }
+  }];
+
+  let assemblyFormat = "$input attr-dict `:` type($input) `to` type($output)";
+
+  let hasFolder = 1;
+  let hasCanonicalizer = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // DeallocOp
 //===----------------------------------------------------------------------===//
@@ -627,8 +650,9 @@ def MemRef_PrefetchOp : MemRef_Op<"prefetch"> {
 //===----------------------------------------------------------------------===//
 
 def MemRef_ReinterpretCastOp:
-    BaseOpWithOffsetSizesAndStrides<"reinterpret_cast", [
-      NoSideEffect, ViewLikeOpInterface, OffsetSizeAndStrideOpInterface
+    BaseOpWithOffsetSizesAndStrides<MemRef_Dialect, "reinterpret_cast", [
+      NoSideEffect, AttrSizedOperandSegments, ViewLikeOpInterface,
+      OffsetSizeAndStrideOpInterface
     ]> {
   let summary = "memref reinterpret cast operation";
   let description = [{
@@ -855,8 +879,9 @@ def MemRef_StoreOp : MemRef_Op<"store",
 //===----------------------------------------------------------------------===//
 
 def SubViewOp : BaseOpWithOffsetSizesAndStrides<
-    "subview", [DeclareOpInterfaceMethods<ViewLikeOpInterface>,
-                NoSideEffect, OffsetSizeAndStrideOpInterface] >  {
+    MemRef_Dialect, "subview", [DeclareOpInterfaceMethods<ViewLikeOpInterface>,
+                NoSideEffect, AttrSizedOperandSegments,
+		OffsetSizeAndStrideOpInterface] >  {
   let summary = "memref subview operation";
   let description = [{
     The "subview" operation converts a memref type to another memref type
@@ -1109,6 +1134,9 @@ def TensorLoadOp : MemRef_Op<"tensor_load",
     // Produces a value of tensor<4x?xf32> type.
     %12 = memref.tensor_load %10 : memref<4x?xf32, #layout, memspace0>
     ```
+
+    If tensor load is used in the bufferization steps, mutating the source
+    buffer after loading leads to undefined behavior.
   }];
 
   let arguments = (ins Arg<AnyRankedOrUnrankedMemRef,
diff --git a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
new file mode 100644
index 0000000000000000000000000000000000000000..024fe5ebfbc35cdcd5dc33ee69d892a2b762242c
--- /dev/null
+++ b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
@@ -0,0 +1,29 @@
+//===- MemRefUtils.h - MemRef transformation utilities ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This header file defines prototypes for various transformation utilities for
+// the MemRefOps dialect. These are not passes by themselves but are used
+// either by passes, optimization sequences, or in turn by other transformation
+// utilities.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_MEMREF_UTILS_MEMREFUTILS_H
+#define MLIR_DIALECT_MEMREF_UTILS_MEMREFUTILS_H
+
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+
+namespace mlir {
+
+/// Finds the associated dealloc that can be linked to our allocation nodes (if
+/// any).
+Operation *findDealloc(Value allocValue);
+
+} // end namespace mlir
+
+#endif // MLIR_DIALECT_MEMREF_UTILS_MEMREFUTILS_H
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 24a06c4d0d0061fa6b21881f363a9255be5d9470..6c1f5c0e7f10b349e9831b1661f246736a8a6b3f 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -116,7 +116,8 @@ def TerminatorOp : OpenMP_Op<"terminator", [Terminator]> {
 // 2.9.2 Workshare Loop Construct
 //===----------------------------------------------------------------------===//
 
-def WsLoopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments]> {
+def WsLoopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments,
+                         AllTypesMatch<["lowerBound", "upperBound", "step"]>]> {
   let summary = "workshare loop construct";
   let description = [{
     The workshare loop construct specifies that the iterations of the loop(s)
@@ -130,13 +131,13 @@ def WsLoopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments]> {
     by "omp.yield" instruction without operands.
 
     ```
-      omp.wsloop (%i1, %i2) = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) {
-        %a = load %arrA[%i1, %i2] : memref<?x?xf32>
-        %b = load %arrB[%i1, %i2] : memref<?x?xf32>
-        %sum = addf %a, %b : f32
-        store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
-        omp.yield
-      }
+    omp.wsloop (%i1, %i2) : index = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) {
+      %a = load %arrA[%i1, %i2] : memref<?x?xf32>
+      %b = load %arrB[%i1, %i2] : memref<?x?xf32>
+      %sum = addf %a, %b : f32
+      store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
+      omp.yield
+    }
     ```
 
     `private_vars`, `firstprivate_vars`, `lastprivate_vars` and `linear_vars`
@@ -181,10 +182,23 @@ def WsLoopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments]> {
              OptionalAttr<OrderKind>:$order_val,
              UnitAttr:$inclusive);
 
+  let skipDefaultBuilders = 1;
+
   let builders = [
     OpBuilder<(ins "ValueRange":$lowerBound, "ValueRange":$upperBound,
-                  "ValueRange":$step,
-                  CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes)>
+               "ValueRange":$step,
+               CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes)>,
+    OpBuilder<(ins "TypeRange":$resultTypes, "ValueRange":$lowerBound,
+               "ValueRange":$upperBound, "ValueRange":$step,
+               "ValueRange":$privateVars, "ValueRange":$firstprivateVars,
+               "ValueRange":$lastprivate_vars, "ValueRange":$linear_vars,
+               "ValueRange":$linear_step_vars, "StringAttr":$schedule_val,
+               "Value":$schedule_chunk_var, "IntegerAttr":$collapse_val,
+               "UnitAttr":$nowait, "IntegerAttr":$ordered_val,
+               "StringAttr":$order_val, "UnitAttr":$inclusive, CArg<"bool",
+               "true">:$buildBody)>,
+    OpBuilder<(ins "TypeRange":$resultTypes, "ValueRange":$operands,
+               CArg<"ArrayRef<NamedAttribute>", "{}">:$attributes)>
   ];
 
   let regions = (region AnyRegion:$region);
@@ -193,6 +207,8 @@ def WsLoopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments]> {
     /// Returns the number of loops in the workshape loop nest.
     unsigned getNumLoops() { return lowerBound().size(); }
   }];
+  let parser = [{ return parseWsLoopOp(parser, result); }];
+  let printer = [{ return printWsLoopOp(p, *this); }];
 }
 
 def YieldOp : OpenMP_Op<"yield", [NoSideEffect, ReturnLike, Terminator,
diff --git a/mlir/include/mlir/Dialect/PDL/IR/PDLDialect.td b/mlir/include/mlir/Dialect/PDL/IR/PDLDialect.td
index afdf50673ed4e7276a082b31918fe31cf6560822..1c9de16af358b1e473ca47d0ee261068670e10f8 100644
--- a/mlir/include/mlir/Dialect/PDL/IR/PDLDialect.td
+++ b/mlir/include/mlir/Dialect/PDL/IR/PDLDialect.td
@@ -48,7 +48,7 @@ def PDL_Dialect : Dialect {
 
       %resultType = pdl.type
       %inputOperand = pdl.operand
-      %root, %results = pdl.operation "foo.op"(%inputOperand) -> %resultType
+      %root = pdl.operation "foo.op"(%inputOperand) -> %resultType
       pdl.rewrite %root {
         pdl.replace %root with (%inputOperand)
       }
diff --git a/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td b/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td
index 60590b1fcd01bca04c8928dff02e966cd39f00ec..32de9f438c00516efcbb1b956cee033ff4f041b8 100644
--- a/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td
+++ b/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td
@@ -29,17 +29,17 @@ class PDL_Op<string mnemonic, list<OpTrait> traits = []>
 }
 
 //===----------------------------------------------------------------------===//
-// pdl::ApplyConstraintOp
+// pdl::ApplyNativeConstraintOp
 //===----------------------------------------------------------------------===//
 
-def PDL_ApplyConstraintOp
-    : PDL_Op<"apply_constraint", [HasParent<"pdl::PatternOp">]> {
-  let summary = "Apply a generic constraint to a set of provided entities";
+def PDL_ApplyNativeConstraintOp
+    : PDL_Op<"apply_native_constraint", [HasParent<"pdl::PatternOp">]> {
+  let summary = "Apply a native constraint to a set of provided entities";
   let description = [{
-    `apply_constraint` operations apply a generic constraint, that has been
-    registered externally with the consumer of PDL, to a given set of entities.
-    The constraint is permitted to accept any number of constant valued
-    parameters.
+    `pdl.apply_native_constraint` operations apply a native C++ constraint, that
+    has been registered externally with the consumer of PDL, to a given set of
+    entities. The constraint is permitted to accept any number of constant
+    valued parameters.
 
     Example:
 
@@ -47,7 +47,7 @@ def PDL_ApplyConstraintOp
     // Apply `myConstraint` to the entities defined by `input`, `attr`, and
     // `op`. `42`, `"abc"`, and `i32` are constant parameters passed to the
     // constraint.
-    pdl.apply_constraint "myConstraint"[42, "abc", i32](%input, %attr, %op : !pdl.value, !pdl.attribute, !pdl.operation)
+    pdl.apply_native_constraint "myConstraint"[42, "abc", i32](%input, %attr, %op : !pdl.value, !pdl.attribute, !pdl.operation)
     ```
   }];
 
@@ -60,13 +60,65 @@ def PDL_ApplyConstraintOp
 
   let builders = [
     OpBuilder<(ins "StringRef":$name, CArg<"ValueRange", "{}">:$args,
-      CArg<"ArrayRef<Attribute>", "{}">:$params), [{
+                   CArg<"ArrayRef<Attribute>", "{}">:$params), [{
       build($_builder, $_state, $_builder.getStringAttr(name), args,
             params.empty() ? ArrayAttr() : $_builder.getArrayAttr(params));
     }]>,
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// pdl::ApplyNativeRewriteOp
+//===----------------------------------------------------------------------===//
+
+def PDL_ApplyNativeRewriteOp
+    : PDL_Op<"apply_native_rewrite", [HasParent<"pdl::RewriteOp">]> {
+  let summary = "Apply a native rewrite method inside of pdl.rewrite region";
+  let description = [{
+    `pdl.apply_native_rewrite` operations apply a native C++ function, that has
+    been registered externally with the consumer of PDL, to perform a rewrite
+    and optionally return a number of values. The native function may accept any
+    number of arguments and constant attribute parameters. This operation is
+    used within a pdl.rewrite region to enable the interleaving of native
+    rewrite methods with other pdl constructs.
+
+    Example:
+
+    ```mlir
+    // Apply a native rewrite method that returns an attribute.
+    %ret = pdl.apply_native_rewrite "myNativeFunc"[42, "gt"](%arg0, %arg1) : !pdl.attribute
+    ```
+
+    ```c++
+    // The native rewrite as defined in C++:
+    static void myNativeFunc(ArrayRef<PDLValue> args, ArrayAttr constantParams,
+                             PatternRewriter &rewriter,
+                             PDLResultList &results) {
+      Value arg0 = args[0].cast<Value>();
+      Value arg1 = args[1].cast<Value>();
+      IntegerAttr param0 = constantParams[0].cast<IntegerAttr>();
+      StringAttr param1 = constantParams[1].cast<StringAttr>();
+
+      // Just push back the first param attribute.
+      results.push_back(param0);
+    }
+
+    void registerNativeRewrite(PDLPatternModule &pdlModule) {
+      pdlModule.registerRewriteFunction("myNativeFunc", myNativeFunc);
+    }
+    ```
+  }];
+
+  let arguments = (ins StrAttr:$name,
+                       Variadic<PDL_AnyType>:$args,
+                       OptionalAttr<ArrayAttr>:$constParams);
+  let results = (outs Variadic<PDL_AnyType>:$results);
+  let assemblyFormat = [{
+    $name ($constParams^)? (`(` $args^ `:` type($args) `)`)? `:` type($results)
+    attr-dict
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // pdl::AttributeOp
 //===----------------------------------------------------------------------===//
@@ -113,39 +165,6 @@ def PDL_AttributeOp : PDL_Op<"attribute"> {
   ];
 }
 
-//===----------------------------------------------------------------------===//
-// pdl::CreateNativeOp
-//===----------------------------------------------------------------------===//
-
-def PDL_CreateNativeOp
-    : PDL_Op<"create_native", [HasParent<"pdl::RewriteOp">]> {
-  let summary = "Call a native creation method to construct an `Attribute`, "
-                "`Operation`, `Type`, or `Value`";
-  let description = [{
-    `pdl.create_native` operations invoke a native C++ function, that has been
-    registered externally with the consumer of PDL, to create an `Attribute`,
-    `Operation`, `Type`, or `Value`. The native function must produce a value
-    of the specified return type, and may accept any number of positional
-    arguments and constant attribute parameters.
-
-    Example:
-
-    ```mlir
-    %ret = pdl.create_native "myNativeFunc"[42, "gt"](%arg0, %arg1) : !pdl.attribute
-    ```
-  }];
-
-  let arguments = (ins StrAttr:$name,
-                       Variadic<PDL_AnyType>:$args,
-                       OptionalAttr<ArrayAttr>:$constParams);
-  let results = (outs PDL_AnyType:$result);
-  let assemblyFormat = [{
-    $name ($constParams^)? (`(` $args^ `:` type($args) `)`)? `:` type($result)
-    attr-dict
-  }];
-  let verifier = ?;
-}
-
 //===----------------------------------------------------------------------===//
 // pdl::EraseOp
 //===----------------------------------------------------------------------===//
@@ -177,9 +196,9 @@ def PDL_OperandOp : PDL_Op<"operand", [HasParent<"pdl::PatternOp">]> {
   let description = [{
     `pdl.operand` operations capture external operand edges into an operation
     node that originate from operations or block arguments not otherwise
-    specified within the pattern (e.g. via `pdl.operation`). These operations
-    define individual operands of a given operation. A `pdl.operand` may
-    partially constrain an operand by specifying an expected value type
+    specified within the pattern (i.e. via `pdl.result` or `pdl.results`). These
+    operations define individual operands of a given operation. A `pdl.operand`
+    may partially constrain an operand by specifying an expected value type
     (via a `pdl.type` operation).
 
     Example:
@@ -205,6 +224,44 @@ def PDL_OperandOp : PDL_Op<"operand", [HasParent<"pdl::PatternOp">]> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// pdl::OperandsOp
+//===----------------------------------------------------------------------===//
+
+def PDL_OperandsOp : PDL_Op<"operands", [HasParent<"pdl::PatternOp">]> {
+  let summary = "Define a range of input operands in a pattern";
+  let description = [{
+    `pdl.operands` operations capture external operand range edges into an
+    operation node that originate from operations or block arguments not
+    otherwise specified within the pattern (i.e. via `pdl.result` or
+    `pdl.results`). These operations define groups of input operands into a
+    given operation. A `pdl.operands` may partially constrain a set of input
+    operands by specifying expected value types (via `pdl.types` operations).
+
+    Example:
+
+    ```mlir
+    // Define a range of input operands:
+    %operands = pdl.operands
+
+    // Define a range of input operands with expected types:
+    %types = pdl.types : [i32, i64, i32]
+    %typed_operands = pdl.operands : %types
+    ```
+  }];
+
+  let arguments = (ins Optional<PDL_RangeOf<PDL_Type>>:$type);
+  let results = (outs PDL_RangeOf<PDL_Value>:$val);
+  let assemblyFormat = "(`:` $type^)? attr-dict";
+
+  let builders = [
+    OpBuilder<(ins), [{
+      build($_builder, $_state, RangeType::get($_builder.getType<ValueType>()),
+            Value());
+    }]>,
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // pdl::OperationOp
 //===----------------------------------------------------------------------===//
@@ -223,8 +280,16 @@ def PDL_OperationOp
     `pdl.operation`s are composed of a name, and a set of attribute, operand,
     and result type values, that map to what those that would be on a
     constructed instance of that operation. The results of a `pdl.operation` are
-    a handle to the operation itself, and a handle to each of the operation
-    result values.
+    a handle to the operation itself. Handles to the results of the operation
+    can be extracted via `pdl.result`.
+
+    Example:
+
+    ```mlir
+    // Define an instance of a `foo.op` operation.
+    %op = pdl.operation "foo.op"(%arg0, %arg1 : !pdl.value, !pdl.value)
+      {"attrA" = %attr0} -> (%type, %type : !pdl.type, !pdl.type)
+    ```
 
     When used within a matching context, the name of the operation may be
     omitted.
@@ -233,25 +298,85 @@ def PDL_OperationOp
     `pdl.rewrite`, all of the result types must be "inferable". This means that
     the type must be attributable to either a constant type value or the result
     type of another entity, such as an attribute, the result of a
-    `createNative`, or the result type of another operation. If the result type
-    value does not meet any of these criteria, the operation must provide the
-    `InferTypeOpInterface` to ensure that the result types can be inferred.
+    `apply_native_rewrite`, or the result type of another operation. If the
+    result type value does not meet any of these criteria, the operation must
+    override the `InferTypeOpInterface` to ensure that the result types can be
+    inferred.
 
-    Example:
+    The operands of the operation are interpreted in the following ways:
+
+    1) A single !pdl.range<value>:
+
+    In this case, the single range is treated as all of the operands of the
+    operation.
 
     ```mlir
-    // Define an instance of a `foo.op` operation.
-    %op, %results:4 = pdl.operation "foo.op"(%arg0, %arg1) {"attrA" = %attr0} -> %type, %type, %type, %type
+    // Define an instance with single range of operands.
+    %op = pdl.operation "std.return"(%allArgs : !pdl.range<value>)
+    ```
+
+    2) A variadic number of either !pdl.value or !pdl.range<value>:
+
+    In this case, the inputs are expected to correspond with the operand groups
+    defined on the operation in ODS.
+
+    ```tablgen
+    // Given the following operation definition in ODS:
+    def MyIndirectCallOp {
+      let results = (outs FunctionType:$call, Variadic<AnyType>:$args);
+    }
+    ```
+
+    ```mlir
+    // We can match the operands as so:
+    %op = pdl.operation "my.indirect_call"(%call, %args : !pdl.value, !pdl.range<value>)
+    ```
+
+    The results of the operation are interpreted in the following ways:
+
+    1) A single !pdl.range<type>:
+
+    In this case, the single range is treated as all of the result types of the
+    operation.
+
+    ```mlir
+    // Define an instance with single range of types.
+    %allResultTypes = pdl.types
+    %op = pdl.operation "unrealized_conversion_cast" -> (%allResultTypes : !pdl.types)
+    ```
+
+    2) A variadic number of either !pdl.type or !pdl.range<type>:
+
+    In this case, the inputs are expected to correspond with the result groups
+    defined on the operation in ODS.
+
+    ```tablgen
+    // Given the following operation definition in ODS:
+    def MyOp {
+      let results = (outs SomeType:$result, Variadic<SomeType>:$otherResults);
+    }
+    ```
+
+    ```mlir
+    // We can match the results as so:
+    %result = pdl.type
+    %otherResults = pdl.types
+    %op = pdl.operation "foo.op" -> (%result, %otherResults : !pdl.type, !pdl.range<type>)
     ```
   }];
 
   let arguments = (ins OptionalAttr<StrAttr>:$name,
-                       Variadic<PDL_Value>:$operands,
+                       Variadic<PDL_InstOrRangeOf<PDL_Value>>:$operands,
                        Variadic<PDL_Attribute>:$attributes,
                        StrArrayAttr:$attributeNames,
-                       Variadic<PDL_Type>:$types);
-  let results = (outs PDL_Operation:$op,
-                      Variadic<PDL_Value>:$results);
+                       Variadic<PDL_InstOrRangeOf<PDL_Type>>:$types);
+  let results = (outs PDL_Operation:$op);
+  let assemblyFormat = [{
+    ($name^)? (`(` $operands^ `:` type($operands) `)`)?
+    custom<OperationOpAttributes>($attributes, $attributeNames)
+    (`->` `(` $types^ `:` type($types) `)`)? attr-dict
+  }];
+
   let builders = [
     OpBuilder<(ins CArg<"Optional<StringRef>", "llvm::None">:$name,
       CArg<"ValueRange", "llvm::None">:$operandValues,
@@ -259,10 +384,9 @@ def PDL_OperationOp
       CArg<"ValueRange", "llvm::None">:$attrValues,
       CArg<"ValueRange", "llvm::None">:$resultTypes), [{
       auto nameAttr = name ? StringAttr() : $_builder.getStringAttr(*name);
-      build($_builder, $_state, $_builder.getType<OperationType>(), {}, nameAttr,
+      build($_builder, $_state, $_builder.getType<OperationType>(), nameAttr,
             operandValues, attrValues, $_builder.getStrArrayAttr(attrNames),
             resultTypes);
-      $_state.types.append(resultTypes.size(), $_builder.getType<ValueType>());
     }]>,
   ];
   let extraClassDeclaration = [{
@@ -293,7 +417,7 @@ def PDL_PatternOp : PDL_Op<"pattern", [IsolatedFromAbove, Symbol]> {
     pdl.pattern : benefit(1) {
       %resultType = pdl.type
       %inputOperand = pdl.operand
-      %root, %results = pdl.operation "foo.op"(%inputOperand) -> (%resultType)
+      %root = pdl.operation "foo.op"(%inputOperand) -> (%resultType)
       pdl.rewrite %root {
         pdl.replace %root with (%inputOperand)
       }
@@ -354,7 +478,10 @@ def PDL_ReplaceOp : PDL_Op<"replace", [
 
     ```mlir
     // Replace root node with 2 values:
-    pdl.replace %root with (%val0, %val1)
+    pdl.replace %root with (%val0, %val1 : !pdl.value, !pdl.value)
+
+    // Replace root node with a range of values:
+    pdl.replace %root with (%vals : !pdl.range<value>)
 
     // Replace root with another operation:
     pdl.replace %root with %otherOp
@@ -362,9 +489,85 @@ def PDL_ReplaceOp : PDL_Op<"replace", [
   }];
   let arguments = (ins PDL_Operation:$operation,
                        Optional<PDL_Operation>:$replOperation,
-                       Variadic<PDL_Value>:$replValues);
+                       Variadic<PDL_InstOrRangeOf<PDL_Value>>:$replValues);
   let assemblyFormat = [{
-    $operation `with` (`(` $replValues^ `)`)? ($replOperation^)? attr-dict
+    $operation `with` (`(` $replValues^ `:` type($replValues) `)`)?
+    ($replOperation^)? attr-dict
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// pdl::ResultOp
+//===----------------------------------------------------------------------===//
+
+def PDL_ResultOp : PDL_Op<"result"> {
+  let summary = "Extract a result from an operation";
+  let description = [{
+    `pdl.result` operations extract result edges from an operation node within
+    a pattern or rewrite region. The provided index is zero-based, and
+    represents the concrete result to extract, i.e. this is not the result index
+    as defined by the ODS definition of the operation.
+
+    Example:
+
+    ```mlir
+    // Extract a result:
+    %operation = pdl.operation ...
+    %pdl_result = pdl.result 1 of %operation
+
+    // Imagine the following IR being matched:
+    %result_0, %result_1 = foo.op ...
+
+    // If the example pattern snippet above were matching against `foo.op` in
+    // the IR snippet, `%pdl_result` would correspond to `%result_1`.
+    ```
+  }];
+
+  let arguments = (ins PDL_Operation:$parent, I32Attr:$index);
+  let results = (outs PDL_Value:$val);
+  let assemblyFormat = "$index `of` $parent attr-dict";
+  let verifier = ?;
+}
+
+//===----------------------------------------------------------------------===//
+// pdl::ResultsOp
+//===----------------------------------------------------------------------===//
+
+def PDL_ResultsOp : PDL_Op<"results"> {
+  let summary = "Extract a result group from an operation";
+  let description = [{
+    `pdl.results` operations extract a result group from an operation within a
+    pattern or rewrite region. If an index is provided, this operation extracts
+    a result group as defined by the ODS definition of the operation. In this
+    case the result of this operation may be either a single `pdl.value` or
+    a `pdl.range<value>`, depending on the constraint of the result in ODS. If
+    no index is provided, this operation extracts the full result range of the
+    operation.
+
+    Example:
+
+    ```mlir
+    // Extract all of the results of an operation:
+    %operation = pdl.operation ...
+    %results = pdl.results of %operation
+
+    // Extract the results in the first result group of an operation, which is
+    // variadic:
+    %operation = pdl.operation ...
+    %results = pdl.results 0 of %operation -> !pdl.range<value>
+
+    // Extract the results in the second result group of an operation, which is
+    // not variadic:
+    %operation = pdl.operation ...
+    %results = pdl.results 1 of %operation -> !pdl.value
+    ```
+  }];
+
+  let arguments = (ins PDL_Operation:$parent, OptionalAttr<I32Attr>:$index);
+  let results = (outs PDL_InstOrRangeOf<PDL_Value>:$val);
+  let assemblyFormat = [{
+    ($index^)? `of` $parent custom<ResultsValueType>(ref($index), type($val))
+    attr-dict
   }];
 }
 
@@ -379,13 +582,14 @@ def PDL_RewriteOp : PDL_Op<"rewrite", [
   let summary = "Specify the rewrite of a matched pattern";
   let description = [{
     `pdl.rewrite` operations terminate the region of a `pdl.pattern` and specify
-    the rewrite of a `pdl.pattern`, on the specified root operation. The
+    the main rewrite of a `pdl.pattern`, on the specified root operation. The
     rewrite is specified either via a string name (`name`) to an external
     rewrite function, or via the region body. The rewrite region, if specified,
     must contain a single block and terminate via the `pdl.rewrite_end`
     operation. If the rewrite is external, it also takes a set of constant
     parameters and a set of additional positional values defined within the
-    matcher as arguments.
+    matcher as arguments. If the rewrite is external, the root operation is
+    passed to the native function as the first argument.
 
     Example:
 
@@ -431,7 +635,7 @@ def PDL_RewriteEndOp : PDL_Op<"rewrite_end", [Terminator,
 def PDL_TypeOp : PDL_Op<"type"> {
   let summary = "Define a type handle within a pattern";
   let description = [{
-    `pdl.type` operations capture result type constraints of an `Attributes`,
+    `pdl.type` operations capture result type constraints of `Attributes`,
     `Values`, and `Operations`. Instances of this operation define, and
     partially constrain, results types of a given entity. A `pdl.type` may
     partially constrain the result by specifying a constant `Type`.
@@ -440,23 +644,44 @@ def PDL_TypeOp : PDL_Op<"type"> {
 
     ```mlir
     // Define a type:
-    %attr = pdl.type
+    %type = pdl.type
 
     // Define a type with a constant value:
-    %attr = pdl.type : i32
+    %type = pdl.type : i32
     ```
   }];
 
   let arguments = (ins OptionalAttr<TypeAttr>:$type);
   let results = (outs PDL_Type:$result);
   let assemblyFormat = "attr-dict (`:` $type^)?";
+}
 
-  let builders = [
-    OpBuilder<(ins CArg<"Type", "Type()">:$ty), [{
-      build($_builder, $_state, $_builder.getType<AttributeType>(),
-            ty ? TypeAttr::get(ty) : TypeAttr());
-    }]>,
-  ];
+//===----------------------------------------------------------------------===//
+// pdl::TypesOp
+//===----------------------------------------------------------------------===//
+
+def PDL_TypesOp : PDL_Op<"types"> {
+  let summary = "Define a range of type handles within a pattern";
+  let description = [{
+    `pdl.types` operations capture result type constraints of `Value`s, and
+    `Operation`s. Instances of this operation define results types of a given
+    entity. A `pdl.types` may partially constrain the results by specifying
+    an array of `Type`s.
+
+    Example:
+
+    ```mlir
+    // Define a range of types:
+    %types = pdl.types
+
+    // Define a range of types with a range of constant values:
+    %types = pdl.types : [i32, i64, i32]
+    ```
+  }];
+
+  let arguments = (ins OptionalAttr<TypeArrayAttr>:$types);
+  let results = (outs PDL_RangeOf<PDL_Type>:$result);
+  let assemblyFormat = "attr-dict (`:` $types^)?";
 }
 
 #endif // MLIR_DIALECT_PDL_IR_PDLOPS
diff --git a/mlir/include/mlir/Dialect/PDL/IR/PDLTypes.td b/mlir/include/mlir/Dialect/PDL/IR/PDLTypes.td
index c854616fbc8f3a5fb28efa6578551833985b334c..1e0578339ad8664fb7b8ec26fc6b28fc6a0213b5 100644
--- a/mlir/include/mlir/Dialect/PDL/IR/PDLTypes.td
+++ b/mlir/include/mlir/Dialect/PDL/IR/PDLTypes.td
@@ -101,4 +101,18 @@ def PDL_AnyType : Type<
   CPred<"$_self.isa<::mlir::pdl::PDLType>()">, "pdl type",
         "::mlir::pdl::PDLType">;
 
+// A range of positional values of one of the provided types.
+class PDL_RangeOf<Type positionalType> :
+  ContainerType<AnyTypeOf<[positionalType]>, PDL_Range.predicate,
+                "$_self.cast<::mlir::pdl::RangeType>().getElementType()",
+                "range", "::mlir::pdl::RangeType">,
+    BuildableType<"::mlir::pdl::RangeType::get(" # positionalType.builderCall #
+                  ")">;
+
+// Either a positional value or a range of positional values for a given type.
+class PDL_InstOrRangeOf<Type positionalType> :
+    AnyTypeOf<[positionalType, PDL_RangeOf<positionalType>],
+              "single element or range of " # positionalType.summary,
+              "::mlir::pdl::PDLType">;
+
 #endif // MLIR_DIALECT_PDL_IR_PDLTYPES
diff --git a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td
index 517a0f4f0af0ca353483965b5e7fc083cc1312c0..ff9b3dda0f489277e9075dcce2f1b5f3a9c0acb0 100644
--- a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td
+++ b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td
@@ -130,32 +130,35 @@ def PDLInterp_ApplyRewriteOp : PDLInterp_Op<"apply_rewrite"> {
   let description = [{
     `pdl_interp.apply_rewrite` operations invoke an external rewriter that has
     been registered with the interpreter to perform the rewrite after a
-    successful match. The rewrite is passed the root operation being matched, a
-    set of additional positional arguments generated within the matcher, and a
-    set of constant parameters.
+    successful match. The rewrite is passed a set of positional arguments,
+    and a set of constant parameters. The rewrite function may return any
+    number of results.
 
     Example:
 
     ```mlir
     // Rewriter operating solely on the root operation.
-    pdl_interp.apply_rewrite "rewriter" on %root
+    pdl_interp.apply_rewrite "rewriter"(%root : !pdl.operation)
+
+    // Rewriter operating solely on the root operation and return an attribute.
+    %attr = pdl_interp.apply_rewrite "rewriter"(%root : !pdl.operation) : !pdl.attribute
 
     // Rewriter operating on the root operation along with additional arguments
     // from the matcher.
-    pdl_interp.apply_rewrite "rewriter"(%value : !pdl.value) on %root
+    pdl_interp.apply_rewrite "rewriter"(%root : !pdl.operation, %value : !pdl.value)
 
     // Rewriter operating on the root operation along with additional arguments
     // and constant parameters.
-    pdl_interp.apply_rewrite "rewriter"[42](%value : !pdl.value) on %root
+    pdl_interp.apply_rewrite "rewriter"[42](%root : !pdl.operation, %value : !pdl.value)
     ```
   }];
   let arguments = (ins StrAttr:$name,
-                       PDL_Operation:$root,
                        Variadic<PDL_AnyType>:$args,
                        OptionalAttr<ArrayAttr>:$constParams);
+  let results = (outs Variadic<PDL_AnyType>:$results);
   let assemblyFormat = [{
-    $name ($constParams^)? (`(` $args^ `:` type($args) `)`)? `on` $root
-    attr-dict
+    $name ($constParams^)? (`(` $args^ `:` type($args) `)`)?
+    (`:` type($results)^)? attr-dict
   }];
 }
 
@@ -165,7 +168,7 @@ def PDLInterp_ApplyRewriteOp : PDLInterp_Op<"apply_rewrite"> {
 
 def PDLInterp_AreEqualOp
     : PDLInterp_PredicateOp<"are_equal", [NoSideEffect, SameTypeOperands]> {
-  let summary = "Check if two positional values are equivalent";
+  let summary = "Check if two positional values or ranges are equivalent";
   let description = [{
     `pdl_interp.are_equal` operations compare two positional values for
     equality. On success, this operation branches to the true destination,
@@ -238,19 +241,29 @@ def PDLInterp_CheckOperandCountOp
   let summary = "Check the number of operands of an `Operation`";
   let description = [{
     `pdl_interp.check_operand_count` operations compare the number of operands
-    of a given operation value with a constant. On success, this operation
-    branches to the true destination, otherwise the false destination is taken.
+    of a given operation value with a constant. The comparison is either exact
+    or at_least, with the latter used to compare against a minimum number of
+    expected operands. On success, this operation branches to the true
+    destination, otherwise the false destination is taken.
 
     Example:
 
     ```mlir
+    // Check for exact equality.
     pdl_interp.check_operand_count of %op is 2 -> ^matchDest, ^failureDest
+
+    // Check for at least N operands.
+    pdl_interp.check_operand_count of %op is at_least 2 -> ^matchDest, ^failureDest
     ```
   }];
 
   let arguments = (ins PDL_Operation:$operation,
-                       Confined<I32Attr, [IntNonNegative]>:$count);
-  let assemblyFormat = "`of` $operation `is` $count attr-dict `->` successors";
+                       Confined<I32Attr, [IntNonNegative]>:$count,
+                       UnitAttr:$compareAtLeast);
+  let assemblyFormat = [{
+    `of` $operation `is` (`at_least` $compareAtLeast^)? $count attr-dict
+    `->` successors
+  }];
 }
 
 //===----------------------------------------------------------------------===//
@@ -285,19 +298,29 @@ def PDLInterp_CheckResultCountOp
   let summary = "Check the number of results of an `Operation`";
   let description = [{
     `pdl_interp.check_result_count` operations compare the number of results
-    of a given operation value with a constant. On success, this operation
-    branches to the true destination, otherwise the false destination is taken.
+    of a given operation value with a constant. The comparison is either exact
+    or at_least, with the latter used to compare against a minimum number of
+    expected results. On success, this operation branches to the true
+    destination, otherwise the false destination is taken.
 
     Example:
 
     ```mlir
-    pdl_interp.check_result_count of %op is 0 -> ^matchDest, ^failureDest
+    // Check for exact equality.
+    pdl_interp.check_result_count of %op is 2 -> ^matchDest, ^failureDest
+
+    // Check for at least N results.
+    pdl_interp.check_result_count of %op is at_least 2 -> ^matchDest, ^failureDest
     ```
   }];
 
   let arguments = (ins PDL_Operation:$operation,
-                       Confined<I32Attr, [IntNonNegative]>:$count);
-  let assemblyFormat = "`of` $operation `is` $count attr-dict `->` successors";
+                       Confined<I32Attr, [IntNonNegative]>:$count,
+                       UnitAttr:$compareAtLeast);
+  let assemblyFormat = [{
+    `of` $operation `is` (`at_least` $compareAtLeast^)? $count attr-dict
+    `->` successors
+  }];
 }
 
 //===----------------------------------------------------------------------===//
@@ -323,6 +346,30 @@ def PDLInterp_CheckTypeOp
   let assemblyFormat = "$value `is` $type attr-dict `->` successors";
 }
 
+//===----------------------------------------------------------------------===//
+// pdl_interp::CheckTypesOp
+//===----------------------------------------------------------------------===//
+
+def PDLInterp_CheckTypesOp
+    : PDLInterp_PredicateOp<"check_types", [NoSideEffect]> {
+  let summary = "Compare a range of types to a range of known values";
+  let description = [{
+    `pdl_interp.check_types` operations compare a range of types with a
+    statically known range of types. On success, this operation branches
+    to the true destination, otherwise the false destination is taken.
+
+    Example:
+
+    ```mlir
+    pdl_interp.check_types %type are [i32, i64] -> ^matchDest, ^failureDest
+    ```
+  }];
+
+  let arguments = (ins PDL_RangeOf<PDL_Type>:$value,
+                       TypeArrayAttr:$types);
+  let assemblyFormat = "$value `are` $types attr-dict `->` successors";
+}
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::CreateAttributeOp
 //===----------------------------------------------------------------------===//
@@ -351,38 +398,6 @@ def PDLInterp_CreateAttributeOp
     }]>];
 }
 
-//===----------------------------------------------------------------------===//
-// pdl_interp::CreateNativeOp
-//===----------------------------------------------------------------------===//
-
-def PDLInterp_CreateNativeOp : PDLInterp_Op<"create_native"> {
-  let summary = "Call a native creation method to construct an `Attribute`, "
-                "`Operation`, `Type`, or `Value`";
-  let description = [{
-    `pdl_interp.create_native` operations invoke a native C++ function, that has
-    been registered externally with the consumer of PDL, to create an
-    `Attribute`, `Operation`, `Type`, or `Value`. The native function must
-    produce a value of the specified return type, and may accept any number of
-    positional arguments and constant attribute parameters.
-
-    Example:
-
-    ```mlir
-    %ret = pdl_interp.create_native "myNativeFunc"[42, "gt"](%arg0, %arg1 : !pdl.value, !pdl.value) : !pdl.attribute
-    ```
-  }];
-
-  let arguments = (ins StrAttr:$name,
-                       Variadic<PDL_AnyType>:$args,
-                       OptionalAttr<ArrayAttr>:$constParams);
-  let results = (outs PDL_AnyType:$result);
-  let assemblyFormat = [{
-    $name ($constParams^)? (`(` $args^ `:` type($args) `)`)? `:` type($result)
-    attr-dict
-  }];
-  let verifier = ?;
-}
-
 //===----------------------------------------------------------------------===//
 // pdl_interp::CreateOperationOp
 //===----------------------------------------------------------------------===//
@@ -392,21 +407,23 @@ def PDLInterp_CreateOperationOp
   let summary = "Create an instance of a specific `Operation`";
   let description = [{
     `pdl_interp.create_operation` operations create an `Operation` instance with
-    the specified attributes, operands, and result types.
+    the specified attributes, operands, and result types. See `pdl.operation`
+    for a more detailed description on the interpretation of the arguments to
+    this operation.
 
     Example:
 
     ```mlir
     // Create an instance of a `foo.op` operation.
-    %op = pdl_interp.create_operation "foo.op"(%arg0) {"attrA" = %attr0} -> %type, %type
+    %op = pdl_interp.create_operation "foo.op"(%arg0 : !pdl.value) {"attrA" = %attr0} -> (%type : !pdl.type)
     ```
   }];
 
   let arguments = (ins StrAttr:$name,
-                       Variadic<PDL_Value>:$operands,
+                       Variadic<PDL_InstOrRangeOf<PDL_Value>>:$operands,
                        Variadic<PDL_Attribute>:$attributes,
                        StrArrayAttr:$attributeNames,
-                       Variadic<PDL_Type>:$types);
+                       Variadic<PDL_InstOrRangeOf<PDL_Type>>:$types);
   let results = (outs PDL_Operation:$operation);
 
   let builders = [
@@ -415,9 +432,13 @@ def PDLInterp_CreateOperationOp
       "ArrayAttr":$attributeNames), [{
       build($_builder, $_state, $_builder.getType<pdl::OperationType>(), name,
             operands, attributes, attributeNames, types);
-    }]>];
-  let parser = [{ return ::parseCreateOperationOp(parser, result); }];
-  let printer = [{ ::print(p, *this); }];
+    }]>
+  ];
+  let assemblyFormat = [{
+    $name (`(` $operands^ `:` type($operands) `)`)?
+    custom<CreateOperationOpAttributes>($attributes, $attributeNames)
+    (`->` `(` $types^ `:` type($types) `)`)? attr-dict
+  }];
 }
 
 //===----------------------------------------------------------------------===//
@@ -448,6 +469,28 @@ def PDLInterp_CreateTypeOp : PDLInterp_Op<"create_type", [NoSideEffect]> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// pdl_interp::CreateTypesOp
+//===----------------------------------------------------------------------===//
+
+def PDLInterp_CreateTypesOp : PDLInterp_Op<"create_types", [NoSideEffect]> {
+  let summary = "Create an interpreter handle to a range of constant `Type`s";
+  let description = [{
+    `pdl_interp.create_types` operations generate a handle within the
+    interpreter for a specific range of constant type values.
+
+    Example:
+
+    ```mlir
+    pdl_interp.create_types [i64, i64]
+    ```
+  }];
+
+  let arguments = (ins TypeArrayAttr:$value);
+  let results = (outs PDL_RangeOf<PDL_Type>:$result);
+  let assemblyFormat = "$value attr-dict";
+}
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::EraseOp
 //===----------------------------------------------------------------------===//
@@ -552,19 +595,20 @@ def PDLInterp_GetDefiningOpOp
   let summary = "Get the defining operation of a `Value`";
   let description = [{
     `pdl_interp.get_defining_op` operations try to get the defining operation
-    of a specific value. If the value is not an operation result, null is
-    returned.
+    of a specific value or range of values. In the case of range, the defining
+    op of the first value is returned. If the value is not an operation result
+    or range of operand results, null is returned.
 
     Example:
 
     ```mlir
-    %op = pdl_interp.get_defining_op of %value
+    %op = pdl_interp.get_defining_op of %value : !pdl.value
     ```
   }];
 
-  let arguments = (ins PDL_Value:$value);
+  let arguments = (ins PDL_InstOrRangeOf<PDL_Value>:$value);
   let results = (outs PDL_Operation:$operation);
-  let assemblyFormat = "`of` $value attr-dict";
+  let assemblyFormat = "`of` $value `:` type($value) attr-dict";
 }
 
 //===----------------------------------------------------------------------===//
@@ -591,6 +635,49 @@ def PDLInterp_GetOperandOp : PDLInterp_Op<"get_operand", [NoSideEffect]> {
   let assemblyFormat = "$index `of` $operation attr-dict";
 }
 
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetOperandsOp
+//===----------------------------------------------------------------------===//
+
+def PDLInterp_GetOperandsOp : PDLInterp_Op<"get_operands", [NoSideEffect]> {
+  let summary = "Get a specified operand group from an `Operation`";
+  let description = [{
+    `pdl_interp.get_operands` operations try to get a specific operand
+    group from an operation. If the expected result is a single Value, null is
+    returned if the operand group is not of size 1. If a range is expected,
+    null is returned if the operand group is invalid. If no index is provided,
+    the returned operand group corresponds to all operands of the operation.
+
+    Example:
+
+    ```mlir
+    // Get the first group of operands from an operation, and expect a single
+    // element.
+    %operand = pdl_interp.get_operands 0 of %op : !pdl.value
+
+    // Get the first group of operands from an operation.
+    %operands = pdl_interp.get_operands 0 of %op : !pdl.range<value>
+
+    // Get all of the operands from an operation.
+    %operands = pdl_interp.get_operands of %op : !pdl.range<value>
+    ```
+  }];
+
+  let arguments = (ins
+    PDL_Operation:$operation,
+    OptionalAttr<Confined<I32Attr, [IntNonNegative]>>:$index
+  );
+  let results = (outs PDL_InstOrRangeOf<PDL_Value>:$value);
+  let assemblyFormat = "($index^)? `of` $operation `:` type($value) attr-dict";
+  let builders = [
+    OpBuilder<(ins "Type":$resultType, "Value":$operation,
+                   "Optional<unsigned>":$index), [{
+      build($_builder, $_state, resultType, operation,
+            index ? $_builder.getI32IntegerAttr(*index) : IntegerAttr());
+    }]>,
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::GetResultOp
 //===----------------------------------------------------------------------===//
@@ -615,59 +702,117 @@ def PDLInterp_GetResultOp : PDLInterp_Op<"get_result", [NoSideEffect]> {
   let assemblyFormat = "$index `of` $operation attr-dict";
 }
 
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetResultsOp
+//===----------------------------------------------------------------------===//
+
+def PDLInterp_GetResultsOp : PDLInterp_Op<"get_results", [NoSideEffect]> {
+  let summary = "Get a specified result group from an `Operation`";
+  let description = [{
+    `pdl_interp.get_results` operations try to get a specific result group
+    from an operation. If the expected result is a single Value, null is
+    returned if the result group is not of size 1. If a range is expected,
+    null is returned if the result group is invalid. If no index is provided,
+    the returned operand group corresponds to all results of the operation.
+
+    Example:
+
+    ```mlir
+    // Get the first group of results from an operation, and expect a single
+    // element.
+    %result = pdl_interp.get_results 0 of %op : !pdl.value
+
+    // Get the first group of results from an operation.
+    %results = pdl_interp.get_results 0 of %op : !pdl.range<value>
+
+    // Get all of the results from an operation.
+    %results = pdl_interp.get_results of %op : !pdl.range<value>
+    ```
+  }];
+
+  let arguments = (ins
+    PDL_Operation:$operation,
+    OptionalAttr<Confined<I32Attr, [IntNonNegative]>>:$index
+  );
+  let results = (outs PDL_InstOrRangeOf<PDL_Value>:$value);
+  let assemblyFormat = "($index^)? `of` $operation `:` type($value) attr-dict";
+  let builders = [
+    OpBuilder<(ins "Type":$resultType, "Value":$operation,
+                   "Optional<unsigned>":$index), [{
+      build($_builder, $_state, resultType, operation,
+            index ? $_builder.getI32IntegerAttr(*index) : IntegerAttr());
+    }]>,
+    OpBuilder<(ins "Value":$operation), [{
+      build($_builder, $_state,
+            pdl::RangeType::get($_builder.getType<pdl::ValueType>()), operation,
+            IntegerAttr());
+    }]>,
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::GetValueTypeOp
 //===----------------------------------------------------------------------===//
 
-// Get a type from the root operation, held in the rewriter context.
-def PDLInterp_GetValueTypeOp : PDLInterp_Op<"get_value_type", [NoSideEffect]> {
+def PDLInterp_GetValueTypeOp : PDLInterp_Op<"get_value_type", [NoSideEffect,
+     TypesMatchWith<"`value` type matches arity of `result`",
+                    "result", "value", "getGetValueTypeOpValueType($_self)">]> {
   let summary = "Get the result type of a specified `Value`";
   let description = [{
     `pdl_interp.get_value_type` operations get the resulting type of a specific
-    value.
+    value or range thereof.
 
     Example:
 
     ```mlir
-    %type = pdl_interp.get_value_type of %value
+    // Get the type of a single value.
+    %type = pdl_interp.get_value_type of %value : !pdl.type
+
+    // Get the types of a value range.
+    %type = pdl_interp.get_value_type of %values : !pdl.range<type>
     ```
   }];
 
-  let arguments = (ins PDL_Value:$value);
-  let results = (outs PDL_Type:$result);
-  let assemblyFormat = "`of` $value attr-dict";
+  let arguments = (ins PDL_InstOrRangeOf<PDL_Value>:$value);
+  let results = (outs PDL_InstOrRangeOf<PDL_Type>:$result);
+  let assemblyFormat = "`of` $value `:` type($result) attr-dict";
 
   let builders = [
     OpBuilder<(ins "Value":$value), [{
-      build($_builder, $_state, $_builder.getType<pdl::TypeType>(), value);
+      Type valType = value.getType();
+      Type typeType = $_builder.getType<pdl::TypeType>();
+      build($_builder, $_state,
+            valType.isa<pdl::RangeType>() ? pdl::RangeType::get(typeType)
+                                          : typeType,
+            value);
     }]>
   ];
 }
 
 //===----------------------------------------------------------------------===//
-// pdl_interp::InferredTypeOp
+// pdl_interp::InferredTypesOp
 //===----------------------------------------------------------------------===//
 
-def PDLInterp_InferredTypeOp : PDLInterp_Op<"inferred_type"> {
-  let summary = "Generate a handle to a Type that is \"inferred\"";
+def PDLInterp_InferredTypesOp : PDLInterp_Op<"inferred_types"> {
+  let summary = "Generate a handle to a range of Types that are \"inferred\"";
   let description = [{
-    `pdl_interp.inferred_type` operations generate a handle to a type that
-    should be inferred. This signals to other operations, such as
-    `pdl_interp.create_operation`, that this type should be inferred.
+    `pdl_interp.inferred_types` operations generate handles to ranges of types
+    that should be inferred. This signals to other operations, such as
+    `pdl_interp.create_operation`, that these types should be inferred.
 
     Example:
 
     ```mlir
-    pdl_interp.inferred_type
+    %types = pdl_interp.inferred_types
     ```
   }];
-  let results = (outs PDL_Type:$type);
+  let results = (outs PDL_RangeOf<PDL_Type>:$type);
   let assemblyFormat = "attr-dict";
-
   let builders = [
     OpBuilder<(ins), [{
-      build($_builder, $_state, $_builder.getType<pdl::TypeType>());
-    }]>,
+      build($_builder, $_state,
+            pdl::RangeType::get($_builder.getType<pdl::TypeType>()));
+    }]>
   ];
 }
 
@@ -679,7 +824,8 @@ def PDLInterp_IsNotNullOp
     : PDLInterp_PredicateOp<"is_not_null", [NoSideEffect]> {
   let summary = "Check if a positional value is non-null";
   let description = [{
-    `pdl_interp.is_not_null` operations check that a positional value exists. On
+    `pdl_interp.is_not_null` operations check that a positional value or range
+    exists. For ranges, this does not mean that the range was simply empty. On
     success, this operation branches to the true destination. Otherwise, the
     false destination is taken.
 
@@ -747,12 +893,15 @@ def PDLInterp_ReplaceOp : PDLInterp_Op<"replace"> {
 
     ```mlir
     // Replace root node with 2 values:
-    pdl_interp.replace %root with (%val0, %val1)
+    pdl_interp.replace %root with (%val0, %val1 : !pdl.type, !pdl.type)
     ```
   }];
   let arguments = (ins PDL_Operation:$operation,
-                       Variadic<PDL_Value>:$replValues);
-  let assemblyFormat = "$operation `with` `(` $replValues `)` attr-dict";
+                       Variadic<PDL_InstOrRangeOf<PDL_Value>>:$replValues);
+  let assemblyFormat = [{
+    $operation `with` ` ` `(` ($replValues^ `:` type($replValues))? `)`
+    attr-dict
+  }];
 }
 
 //===----------------------------------------------------------------------===//
@@ -814,7 +963,7 @@ def PDLInterp_SwitchOperandCountOp
 
   let builders = [
     OpBuilder<(ins "Value":$operation, "ArrayRef<int32_t>":$counts,
-      "Block *":$defaultDest, "BlockRange":$dests), [{
+                   "Block *":$defaultDest, "BlockRange":$dests), [{
     build($_builder, $_state, operation, $_builder.getI32VectorAttr(counts),
           defaultDest, dests);
   }]>];
@@ -884,7 +1033,7 @@ def PDLInterp_SwitchResultCountOp
 
   let builders = [
     OpBuilder<(ins "Value":$operation, "ArrayRef<int32_t>":$counts,
-      "Block *":$defaultDest, "BlockRange":$dests), [{
+                   "Block *":$defaultDest, "BlockRange":$dests), [{
     build($_builder, $_state, operation, $_builder.getI32VectorAttr(counts),
           defaultDest, dests);
   }]>];
@@ -915,9 +1064,9 @@ def PDLInterp_SwitchTypeOp : PDLInterp_SwitchOp<"switch_type", [NoSideEffect]> {
   }];
 
   let builders = [
-    OpBuilder<(ins "Value":$edge, "TypeRange":$types, "Block *":$defaultDest,
-      "BlockRange":$dests), [{
-      build($_builder, $_state, edge, $_builder.getTypeArrayAttr(types),
+    OpBuilder<(ins "Value":$edge, "ArrayRef<Attribute>":$types,
+                   "Block *":$defaultDest, "BlockRange":$dests), [{
+      build($_builder, $_state, edge, $_builder.getArrayAttr(types),
             defaultDest, dests);
     }]>,
   ];
@@ -927,4 +1076,45 @@ def PDLInterp_SwitchTypeOp : PDLInterp_SwitchOp<"switch_type", [NoSideEffect]> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// pdl_interp::SwitchTypesOp
+//===----------------------------------------------------------------------===//
+
+def PDLInterp_SwitchTypesOp : PDLInterp_SwitchOp<"switch_types",
+                                                 [NoSideEffect]> {
+  let summary = "Switch on a range of `Type` values";
+  let description = [{
+    `pdl_interp.switch_types` operations compare a range of types with a set of
+    statically known ranges. If the value matches one of the provided case
+    values the destination for that case value is taken, otherwise the default
+    destination is taken.
+
+    Example:
+
+    ```mlir
+    pdl_interp.switch_types %type is [[i32], [i64, i64]] -> ^i32Dest, ^i64Dest, ^defaultDest
+    ```
+  }];
+
+  let arguments = (ins
+    PDL_RangeOf<PDL_Type>:$value,
+    TypedArrayAttrBase<TypeArrayAttr, "type-array array attribute">:$caseValues
+  );
+  let assemblyFormat = [{
+    $value `to` $caseValues `(` $cases `)` attr-dict `->` $defaultDest
+  }];
+
+  let builders = [
+    OpBuilder<(ins "Value":$edge, "ArrayRef<Attribute>":$types,
+                   "Block *":$defaultDest, "BlockRange":$dests), [{
+      build($_builder, $_state, edge, $_builder.getArrayAttr(types),
+            defaultDest, dests);
+    }]>,
+  ];
+
+  let extraClassDeclaration = [{
+    auto getCaseTypes() { return caseValues().getAsRange<ArrayAttr>(); }
+  }];
+}
+
 #endif // MLIR_DIALECT_PDLINTERP_IR_PDLINTERPOPS
diff --git a/mlir/include/mlir/Dialect/SCF/Transforms.h b/mlir/include/mlir/Dialect/SCF/Transforms.h
index 456eb4e286598e21edf9cf98fdf80f5b712107c0..8ab13e42f477dc07bca6e4df9d7431da7ace0b21 100644
--- a/mlir/include/mlir/Dialect/SCF/Transforms.h
+++ b/mlir/include/mlir/Dialect/SCF/Transforms.h
@@ -19,9 +19,10 @@ namespace mlir {
 
 class ConversionTarget;
 class MLIRContext;
-class OwningRewritePatternList;
 class Region;
 class TypeConverter;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 namespace scf {
 
@@ -60,8 +61,8 @@ tileParallelLoop(ParallelOp op, llvm::ArrayRef<int64_t> tileSizes);
 /// corresponding scf.yield ops need to update their types accordingly to the
 /// TypeConverter, but otherwise don't care what type conversions are happening.
 void populateSCFStructuralTypeConversionsAndLegality(
-    MLIRContext *context, TypeConverter &typeConverter,
-    OwningRewritePatternList &patterns, ConversionTarget &target);
+    TypeConverter &typeConverter, RewritePatternSet &patterns,
+    ConversionTarget &target);
 
 } // namespace scf
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVEnums.h b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVEnums.h
index e5838eff06006aecb705106d944400077b51c06a..ac128ac928ea67ad290514f3c14b3e64265f159e 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVEnums.h
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVEnums.h
@@ -13,6 +13,7 @@
 #ifndef MLIR_DIALECT_SPIRV_IR_SPIRVENUMS_H_
 #define MLIR_DIALECT_SPIRV_IR_SPIRVENUMS_H_
 
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/StringRef.h"
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.h b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.h
index 1921dbbcfc70e7eb31675748fd81231b2d35aa30..7d6f0ce74031a7b2df15e1efa42c6e193b7f0787 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.h
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.h
@@ -24,7 +24,7 @@
 namespace mlir {
 namespace spirv {
 void populateSPIRVGLSLCanonicalizationPatterns(
-    mlir::OwningRewritePatternList &results, mlir::MLIRContext *context);
+    mlir::RewritePatternSet &results);
 } // namespace spirv
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVMemoryOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVMemoryOps.td
index 0cffdb5a6d069eefce5507c819643f360ac69c8e..77fa63fc492ffd830d8abc48a39d1b144851d1b4 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVMemoryOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVMemoryOps.td
@@ -184,7 +184,7 @@ def SPV_LoadOp : SPV_Op<"Load", []> {
 
   let builders = [
     OpBuilder<(ins "Value":$basePtr,
-      CArg<"IntegerAttr", "{}">:$memory_access,
+      CArg<"MemoryAccessAttr", "{}">:$memory_access,
       CArg<"IntegerAttr", "{}">:$alignment)>
   ];
 }
diff --git a/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h b/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
index 1ac7db13793bc0d87b2d28e61466f487fbbd5f1c..881f8e90fa0db81ed7a1a2ba862b7c8efa5cbd8b 100644
--- a/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
+++ b/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
@@ -67,9 +67,8 @@ private:
 /// `func` op to the SPIR-V dialect. These patterns do not handle shader
 /// interface/ABI; they convert function parameters to be of SPIR-V allowed
 /// types.
-void populateBuiltinFuncToSPIRVPatterns(MLIRContext *context,
-                                        SPIRVTypeConverter &typeConverter,
-                                        OwningRewritePatternList &patterns);
+void populateBuiltinFuncToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                        RewritePatternSet &patterns);
 
 namespace spirv {
 class AccessChainOp;
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index a17be3834dc2c1a1d1f4c4a231cac12f06c6eb61..34a12275eabeb8ca567cc5341cb370a9aa6a45bd 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -713,10 +713,10 @@ def Shape_AssumingAllOp : Shape_Op<"assuming_all", [Commutative, NoSideEffect]>
   let verifier = [{ return ::verify(*this); }];
 }
 
-def Shape_AssumingOp : Shape_Op<"assuming",
-                           [SingleBlockImplicitTerminator<"AssumingYieldOp">,
-                            DeclareOpInterfaceMethods<RegionBranchOpInterface>,
-                            RecursiveSideEffects]> {
+def Shape_AssumingOp : Shape_Op<"assuming", [
+    SingleBlockImplicitTerminator<"AssumingYieldOp">,
+    DeclareOpInterfaceMethods<RegionBranchOpInterface>,
+    RecursiveSideEffects]> {
   let summary = "Execute the region";
   let description = [{
     Executes the region assuming all witnesses are true.
@@ -742,6 +742,11 @@ def Shape_AssumingOp : Shape_Op<"assuming",
     static void inlineRegionIntoParent(AssumingOp &op, PatternRewriter &rewriter);
   }];
 
+  let builders = [
+    OpBuilder<(ins "Value":$witness,
+        CArg<"function_ref<SmallVector<Value, 2>(OpBuilder &, Location)>">)>
+  ];
+
   let hasCanonicalizer = 1;
 }
 
@@ -757,7 +762,9 @@ def Shape_AssumingYieldOp : Shape_Op<"assuming_yield",
 
   let arguments = (ins Variadic<AnyType>:$operands);
 
-  let builders = [OpBuilder<(ins), [{ /* nothing to do */ }]>];
+  let builders = [
+    OpBuilder<(ins), [{ /* nothing to do */ }]>,
+  ];
 
   let assemblyFormat = "attr-dict ($operands^ `:` type($operands))?";
 }
@@ -894,7 +901,7 @@ def Shape_CstrRequireOp : Shape_Op<"cstr_require", []> {
 
 def Shape_FunctionLibraryOp : Shape_Op<"function_library",
     [AffineScope, IsolatedFromAbove, NoRegionArguments, SymbolTable, Symbol,
-     SingleBlockImplicitTerminator<"ShapeFunctionLibraryTerminatorOp">]> {
+     NoTerminator, SingleBlock]> {
   let summary = "Represents shape functions and corresponding ops";
   let description = [{
     Represents a list of shape functions and the ops whose shape transfer
@@ -931,19 +938,4 @@ def Shape_FunctionLibraryOp : Shape_Op<"function_library",
   let parser = [{ return ::parse$cppClass(parser, result); }];
 }
 
-//===----------------------------------------------------------------------===//
-// ShapeFunctionLibraryTerminatorOp
-//===----------------------------------------------------------------------===//
-
-def ShapeFunctionLibraryTerminatorOp : Shape_Op<"fn_lib_terminator",
-    [Terminator, HasParent<"FunctionLibraryOp">]> {
-  let summary = "A pseudo op that marks the end of a shape function library";
-  let description = [{
-    `shape_fn_lib_terminator` is a special pseudo terminator operation for the
-    shape function library. It has no semantic meaning beyond keeping the body
-    well-formed.
-  }];
-  let assemblyFormat = "attr-dict";
-}
-
 #endif // SHAPE_OPS
diff --git a/mlir/include/mlir/Dialect/Shape/Transforms/Passes.h b/mlir/include/mlir/Dialect/Shape/Transforms/Passes.h
index 6df12998566ab409b9181baf9b2f4bfdaf0ea0b1..1cf83e6b0beff289aef61e26e408e407c262ccd9 100644
--- a/mlir/include/mlir/Dialect/Shape/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Shape/Transforms/Passes.h
@@ -28,8 +28,7 @@ namespace mlir {
 std::unique_ptr<Pass> createShapeToShapeLowering();
 
 /// Collects a set of patterns to rewrite ops within the Shape dialect.
-void populateShapeRewritePatterns(MLIRContext *context,
-                                  OwningRewritePatternList &patterns);
+void populateShapeRewritePatterns(RewritePatternSet &patterns);
 
 // Collects a set of patterns to replace all constraints with passing witnesses.
 // This is intended to then allow all ShapeConstraint related ops and data to
@@ -37,8 +36,7 @@ void populateShapeRewritePatterns(MLIRContext *context,
 // canonicalization and dead code elimination.
 //
 // After this pass, no cstr_ operations exist.
-void populateRemoveShapeConstraintsPatterns(OwningRewritePatternList &patterns,
-                                            MLIRContext *ctx);
+void populateRemoveShapeConstraintsPatterns(RewritePatternSet &patterns);
 std::unique_ptr<FunctionPass> createRemoveShapeConstraintsPass();
 
 /// Populates patterns for shape dialect structural type conversions and sets up
@@ -53,8 +51,8 @@ std::unique_ptr<FunctionPass> createRemoveShapeConstraintsPass();
 /// do for a structural type conversion is to update both of their types
 /// consistently to the new types prescribed by the TypeConverter.
 void populateShapeStructuralTypeConversionsAndLegality(
-    MLIRContext *context, TypeConverter &typeConverter,
-    OwningRewritePatternList &patterns, ConversionTarget &target);
+    TypeConverter &typeConverter, RewritePatternSet &patterns,
+    ConversionTarget &target);
 
 // Bufferizes shape dialect ops.
 //
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
index 48d1834f899c789625726a5a3accf5f199991bc7..ce3087a176b3c8c0fe97648a4be602ff0b97a66a 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
@@ -33,6 +33,7 @@ class AffineMap;
 class Builder;
 class FuncOp;
 class OpBuilder;
+class PatternRewriter;
 
 /// Return the list of Range (i.e. offset, size, stride). Each Range
 /// entry contains either the dynamic value or a ConstantIndexOp constructed
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
index 403335ad90e5cdf89d1bd34c2ec29904bff98213..fcfe8f1850e95840458e07c924acc2a73dda5924 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
@@ -182,24 +182,6 @@ class FloatTernaryOp<string mnemonic, list<OpTrait> traits = []> :
                   [DeclareOpInterfaceMethods<VectorUnrollOpInterface>])>,
     Arguments<(ins FloatLike:$a, FloatLike:$b, FloatLike:$c)>;
 
-// Base class for ops with static/dynamic offset, sizes and strides
-// attributes/arguments.
-class BaseOpWithOffsetSizesAndStrides<string mnemonic, list<OpTrait> traits = []> :
-    Std_Op<mnemonic,
-           !listconcat(traits, [NoSideEffect, AttrSizedOperandSegments])> {
-  code extraBaseClassDeclaration = [{
-    /// Returns the dynamic sizes for this subview operation if specified.
-    operand_range getDynamicSizes() { return sizes(); }
-
-    /// Return the list of Range (i.e. offset, size, stride). Each
-    /// Range entry contains either the dynamic value or a ConstantIndexOp
-    /// constructed with `b` at location `loc`.
-    SmallVector<Range, 8> getOrCreateRanges(OpBuilder &b, Location loc) {
-      return mlir::getOrCreateRanges(*this, b, loc);
-    }
-  }];
-}
-
 //===----------------------------------------------------------------------===//
 // AbsFOp
 //===----------------------------------------------------------------------===//
@@ -357,7 +339,7 @@ def AssertOp : Std_Op<"assert"> {
   // AssertOp is fully verified by its traits.
   let verifier = ?;
 
-  let hasCanonicalizer = 1;
+  let hasCanonicalizeMethod = 1;
 }
 
 //===----------------------------------------------------------------------===//
@@ -518,7 +500,7 @@ def BranchOp : Std_Op<"br",
     void eraseOperand(unsigned index);
   }];
 
-  let hasCanonicalizer = 1;
+  let hasCanonicalizeMethod = 1;
   let assemblyFormat = [{
     $dest (`(` $destOperands^ `:` type($destOperands) `)`)? attr-dict
   }];
@@ -647,7 +629,7 @@ def CallIndirectOp : Std_Op<"call_indirect", [
   }];
 
   let verifier = ?;
-  let hasCanonicalizer = 1;
+  let hasCanonicalizeMethod = 1;
 
   let assemblyFormat = "$callee `(` $operands `)` attr-dict `:` type($callee)";
 }
@@ -1815,7 +1797,8 @@ def SubIOp : IntBinaryOp<"subi"> {
 //===----------------------------------------------------------------------===//
 
 def SubTensorOp : BaseOpWithOffsetSizesAndStrides<
-    "subtensor", [OffsetSizeAndStrideOpInterface]> {
+    StandardOps_Dialect, "subtensor", [NoSideEffect, AttrSizedOperandSegments,
+                                       OffsetSizeAndStrideOpInterface]> {
   let summary = "subtensor operation";
   let description = [{
     The "subtensor" operation extract a tensor from another tensor as
@@ -1951,8 +1934,8 @@ def SubTensorOp : BaseOpWithOffsetSizesAndStrides<
 //===----------------------------------------------------------------------===//
 
 def SubTensorInsertOp : BaseOpWithOffsetSizesAndStrides<
-    "subtensor_insert",
-    [OffsetSizeAndStrideOpInterface,
+    StandardOps_Dialect, "subtensor_insert",
+    [NoSideEffect, AttrSizedOperandSegments, OffsetSizeAndStrideOpInterface,
      TypesMatchWith<"expected result type to match dest type",
                     "dest", "result", "$_self">]> {
   let summary = "subtensor_insert operation";
@@ -2087,6 +2070,8 @@ def TruncateIOp : Std_Op<"trunci", [NoSideEffect,
   let printer = [{
     return printStandardCastOp(this->getOperation(), p);
   }];
+
+  let hasFolder = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.h b/mlir/include/mlir/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.h
index 49895acd9d241cc6b660c18094b7c7a581daffac..c453e80ee71de3d0b017dd5c18d46bdc5d80ef3d 100644
--- a/mlir/include/mlir/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.h
+++ b/mlir/include/mlir/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.h
@@ -81,9 +81,10 @@ private:
 
 /// Populates the patterns needed to drive the conversion process for
 /// decomposing call graph types with the given `ValueDecomposer`.
-void populateDecomposeCallGraphTypesPatterns(
-    MLIRContext *context, TypeConverter &typeConverter,
-    ValueDecomposer &decomposer, OwningRewritePatternList &patterns);
+void populateDecomposeCallGraphTypesPatterns(MLIRContext *context,
+                                             TypeConverter &typeConverter,
+                                             ValueDecomposer &decomposer,
+                                             RewritePatternSet &patterns);
 
 } // end namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/StandardOps/Transforms/FuncConversions.h b/mlir/include/mlir/Dialect/StandardOps/Transforms/FuncConversions.h
index 1a0308d96259ff8d06b52317ba58be8591f60a53..b932d1e00983455038741a7d3e78868dc8c7139c 100644
--- a/mlir/include/mlir/Dialect/StandardOps/Transforms/FuncConversions.h
+++ b/mlir/include/mlir/Dialect/StandardOps/Transforms/FuncConversions.h
@@ -19,22 +19,21 @@ namespace mlir {
 class ConversionTarget;
 class MLIRContext;
 class Operation;
-class OwningRewritePatternList;
 class TypeConverter;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 /// Add a pattern to the given pattern list to convert the operand and result
 /// types of a CallOp with the given type converter.
-void populateCallOpTypeConversionPattern(OwningRewritePatternList &patterns,
-                                         MLIRContext *ctx,
+void populateCallOpTypeConversionPattern(RewritePatternSet &patterns,
                                          TypeConverter &converter);
 
 /// Add a pattern to the given pattern list to rewrite branch operations to use
 /// operands that have been legalized by the conversion framework. This can only
 /// be done if the branch operation implements the BranchOpInterface. Only
 /// needed for partial conversions.
-void populateBranchOpInterfaceTypeConversionPattern(
-    OwningRewritePatternList &patterns, MLIRContext *ctx,
-    TypeConverter &converter);
+void populateBranchOpInterfaceTypeConversionPattern(RewritePatternSet &patterns,
+                                                    TypeConverter &converter);
 
 /// Return true if op is a BranchOpInterface op whose operands are all legal
 /// according to converter.
@@ -43,8 +42,7 @@ bool isLegalForBranchOpInterfaceTypeConversionPattern(Operation *op,
 
 /// Add a pattern to the given pattern list to rewrite `return` ops to use
 /// operands that have been legalized by the conversion framework.
-void populateReturnOpTypeConversionPattern(OwningRewritePatternList &patterns,
-                                           MLIRContext *ctx,
+void populateReturnOpTypeConversionPattern(RewritePatternSet &patterns,
                                            TypeConverter &converter);
 
 /// For ReturnLike ops (except `return`), return True. If op is a `return` &&
diff --git a/mlir/include/mlir/Dialect/StandardOps/Transforms/Passes.h b/mlir/include/mlir/Dialect/StandardOps/Transforms/Passes.h
index a6fdca852c806d994595a09bc0bb966b8e669b8c..2b7f3da150cdf15e9c55f8d10a50b077ec94ae70 100644
--- a/mlir/include/mlir/Dialect/StandardOps/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/StandardOps/Transforms/Passes.h
@@ -19,11 +19,11 @@
 
 namespace mlir {
 
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
-void populateStdBufferizePatterns(MLIRContext *context,
-                                  BufferizeTypeConverter &typeConverter,
-                                  OwningRewritePatternList &patterns);
+void populateStdBufferizePatterns(BufferizeTypeConverter &typeConverter,
+                                  RewritePatternSet &patterns);
 
 /// Creates an instance of std bufferization pass.
 std::unique_ptr<Pass> createStdBufferizePass();
@@ -42,8 +42,7 @@ std::unique_ptr<Pass> createTensorConstantBufferizePass();
 std::unique_ptr<Pass> createStdExpandOpsPass();
 
 /// Collects a set of patterns to rewrite ops within the Std dialect.
-void populateStdExpandOpsPatterns(MLIRContext *context,
-                                  OwningRewritePatternList &patterns);
+void populateStdExpandOpsPatterns(RewritePatternSet &patterns);
 
 //===----------------------------------------------------------------------===//
 // Registration
diff --git a/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.h b/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.h
index 436b3fceb973f79c8344676fdf6b8778eb7434e3..6cb2758459ebabcced3114b78aa534efd425621d 100644
--- a/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.h
@@ -14,11 +14,11 @@
 
 namespace mlir {
 
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
-void populateTensorBufferizePatterns(MLIRContext *context,
-                                     BufferizeTypeConverter &typeConverter,
-                                     OwningRewritePatternList &patterns);
+void populateTensorBufferizePatterns(BufferizeTypeConverter &typeConverter,
+                                     RewritePatternSet &patterns);
 
 /// Creates an instance of `tensor` dialect bufferization pass.
 std::unique_ptr<Pass> createTensorBufferizePass();
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index c9790596ed88122da585c24e405b68924d8f6871..75a2926f4a1033b3006af2fcaed5a3ba6346ea25 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -1363,17 +1363,38 @@ def Tosa_GatherOp : Tosa_Op<"gather", [NoSideEffect]> {
 
   let description = [{
     Generate a tensor for which each element in the output is a subtensor of the
-    values tensor along the given axis, based on the value of indices.
+    values tensor based on the value of indices.
   }];
 
   let arguments = (ins
-    Tosa_Int32Or64Tensor:$indices,
-    Tosa_Tensor1Dto4D:$values,
-    I32Attr:$axis
+    Tosa_Tensor3D:$values,
+    2DTensorOf<[Tosa_Int32]>:$indices
   );
 
   let results = (outs
-    Tosa_Tensor1Dto4D:$output
+    Tosa_Tensor3D:$output
+  );
+}
+
+//===----------------------------------------------------------------------===//
+// Operator: scatter
+//===----------------------------------------------------------------------===//
+def Tosa_ScatterOp : Tosa_Op<"scatter", [NoSideEffect]> {
+  let summary = "Scatter operation,";
+
+  let description = [{
+    The values_out tensor is set to the values_in tensor with data modified as follows:
+    data from the input tensor is inserted at the positions specified by the indices tensor.
+  }];
+
+  let arguments = (ins
+    Tosa_Tensor3D:$values_in,
+    2DTensorOf<[Tosa_Int32]>:$indices,
+    Tosa_Tensor3D:$input
+  );
+
+  let results = (outs
+    Tosa_Tensor3D:$values_out
   );
 }
 
@@ -1402,6 +1423,8 @@ def Tosa_ResizeOp : Tosa_Op<"resize", [NoSideEffect]> {
     Tosa_IntArrayAttr2:$stride,
     Tosa_IntArrayAttr2:$offset,
     I32Attr:$shift,
+    Tosa_Fp32ArrayAttr2:$stride_fp,
+    Tosa_Fp32ArrayAttr2:$offset_fp,
     Tosa_ResizeTypeAttr:$mode
   );
 
@@ -1462,20 +1485,20 @@ def Tosa_RescaleOp: Tosa_Op<"rescale", [NoSideEffect]> {
   let description = [{
     Rescale quantized values into a new domain. Supported rescalings are:
     Mode                    Input   Output
-    signed 8 to 8           aint8   aint8
-    signed 8 to 16          aint8   int16
-    signed 8 to 32          aint8   int32
-    signed 16 to 8          int16   aint8
+    signed 8 to 8           int8    int8
+    signed 8 to 16          int8    int16
+    signed 8 to 32          int8    int32
+    signed 16 to 8          int16   int8
     signed 16 to 16         int16   int16
     signed 16 to 32         int16   int32
-    signed 32 to 8          int32   aint8
+    signed 32 to 8          int32   int8
     signed 32 to 16         int32   int16
     signed 32 to 32         int32   int32
-    signed 48 to 8          int48   aint8
+    signed 48 to 8          int48   int8
     signed 48 to 16         int48   int16
     signed 48 to 32         int48   int32
-    unsigned 8 to signed 8  uint8   aint8
-    signed 8 to unsigned 8  aint8   uint8
+    unsigned 8 to signed 8  uint8   int8
+    signed 8 to unsigned 8  int8    uint8
   }];
 
   let arguments = (ins
@@ -1494,6 +1517,30 @@ def Tosa_RescaleOp: Tosa_Op<"rescale", [NoSideEffect]> {
   );
 }
 
+def Tosa_ApplyScaleOp: Tosa_Op<"apply_scale", [NoSideEffect] # ElementwiseMappable.traits> {
+  let summary = "Rescale scalar operator for Tosa tensor operators";
+
+  let description = [{
+    Applies rescaling for fixed point values. This behavior is replicated in
+    multiple quantized operations (mul, convolution, rescale, matmul, pooling).
+
+    The commonplace implementation is to use i64 operations to avoid integer
+    overflow with target specific implementations can use native operations to
+    avoid wider than necessary types.
+  }];
+
+  let arguments = (ins
+    Tosa_Int32Like:$value,
+    Tosa_Int32Like:$multiplier,
+    Tosa_Int8Like:$shift,
+    BoolAttr:$double_round
+  );
+
+  let results = (outs
+    Tosa_Int32:$output
+  );
+}
+
 //===----------------------------------------------------------------------===//
 // TOSA Spec Section 2.13
 // Operator Class: Data Node Ops.
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
index fe3eee7168c6ac607b607025ceff64e40313d15c..b65201e8a9784d46b7e99c358f49ab67fcdd36bf 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
@@ -66,14 +66,12 @@ def Tosa_Int32Or64 : AnyTypeOf<[Tosa_Int32,
 //===----------------------------------------------------------------------===//
 // Name    Symmetry   Grouping                Sign
 //===----------------------------------------------------------------------===//
-// aint8 : asymmetric per tensor,             signed
 // uint8 : asymmetric per tensor ,            unsigned
 // int4  : symmetric  per channel,            signed
 // int8  : symmetric  per tensor/per channel, signed
 // int16 : symmetric  per tensor,             signed
 //===----------------------------------------------------------------------===//
-def Tosa_QuantizedInt	: AnyTypeOf<[Tosa_QuantizedType<"aint8", [8], 1>,
-                                     Tosa_QuantizedType<"uint8", [8], 0>,
+def Tosa_QuantizedInt	: AnyTypeOf<[ Tosa_QuantizedType<"uint8", [8], 0>,
                                      Tosa_QuantizedType<"int4", [4, 0], 1>,
                                      Tosa_QuantizedType<"int8", [8, 0], 1>,
                                      Tosa_QuantizedType<"int16", [16, 0], 1>]>;
@@ -114,6 +112,7 @@ class Tosa_TensorOfOrNone<list<Type> allowedTypes, string description = ""> :
 // Must be listed rank.
 def Tosa_Tensor1D : 1DTensorOf<[Tosa_AnyNumber]>;
 def Tosa_Tensor2D : 2DTensorOf<[Tosa_AnyNumber]>;
+def Tosa_Tensor3D : 3DTensorOf<[Tosa_AnyNumber]>;
 def Tosa_Tensor4D : 4DTensorOf<[Tosa_AnyNumber]>;
 def Tosa_Tensor5D : TensorRankOf<[Tosa_AnyNumber], [5]>;
 def Tosa_Tensor6D : TensorRankOf<[Tosa_AnyNumber], [6]>;
@@ -127,6 +126,21 @@ def Tosa_Tensor1Dto6D : TensorRankOf<[Tosa_AnyNumber], [1,2,3,4,5,6]>;
 def Tosa_TensorUpto4D : TensorRankOf<[Tosa_AnyNumber], [0,1,2,3,4]>;
 def Tosa_TensorUpto6D : TensorRankOf<[Tosa_AnyNumber], [0,1,2,3,4,5,6]>;
 
+//===----------------------------------------------------------------------===//
+// Generic scalar, vector, or tensor of a particular type.
+//===----------------------------------------------------------------------===//
+
+class Tosa_TypeLike<list<Type> types, string description = ""> : TypeConstraint<Or<[
+     AnyTypeOf<types>.predicate,
+     VectorOf<types>.predicate,
+     TensorOf<types>.predicate]>,
+     "signless-integer-32-like">;
+
+def Tosa_Int8Like : Tosa_TypeLike<[Tosa_Int8], "signless-integer-8-bit-like">;
+def Tosa_Int16Like : Tosa_TypeLike<[Tosa_Int16], "signless-integer-16-bit-like">;
+def Tosa_Int32Like : Tosa_TypeLike<[Tosa_Int32], "signless-integer-32-bit-like">;
+def Tosa_Int64Like : Tosa_TypeLike<[Tosa_Int64], "signless-integer-64-bit-like">;
+
 //===----------------------------------------------------------------------===//
 // Attribute predicates and classes.
 //===----------------------------------------------------------------------===//
@@ -134,6 +148,12 @@ class ArrayMaxCt<int n> : AttrConstraint<
     CPred<"$_self.cast<::mlir::ArrayAttr>().size() <= " # n>,
     "with at least " # n # " elements">;
 
+def Tosa_Fp32ArrayAttr2 : Confined<F32ArrayAttr, [ArrayCount<2>]>;
+def Tosa_Fp32ArrayAttr3 : Confined<F32ArrayAttr, [ArrayCount<3>]>;
+def Tosa_Fp32ArrayAttr4 : Confined<F32ArrayAttr, [ArrayCount<4>]>;
+def Tosa_Fp32ArrayAttr5 : Confined<F32ArrayAttr, [ArrayCount<5>]>;
+def Tosa_Fp32ArrayAttr6 : Confined<F32ArrayAttr, [ArrayCount<6>]>;
+
 def Tosa_IntArrayAttr2 : Confined<I64ArrayAttr, [ArrayCount<2>]>;
 def Tosa_IntArrayAttr3 : Confined<I64ArrayAttr, [ArrayCount<3>]>;
 def Tosa_IntArrayAttr4 : Confined<I64ArrayAttr, [ArrayCount<4>]>;
diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.h b/mlir/include/mlir/Dialect/Vector/VectorOps.h
index 9e486d038a48869274f7f15800a8ebb1e5923ee9..111cb7370eefcde2560a0d243c5866252d3a4039 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.h
@@ -28,7 +28,8 @@
 
 namespace mlir {
 class MLIRContext;
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 namespace vector {
 class VectorDialect;
@@ -39,11 +40,10 @@ struct BitmaskEnumStorage;
 
 /// Collect a set of vector-to-vector canonicalization patterns.
 void populateVectorToVectorCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context);
+    RewritePatternSet &patterns);
 
 /// Collect a set of vector-to-vector transformation patterns.
-void populateVectorToVectorTransformationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context);
+void populateVectorToVectorTransformationPatterns(RewritePatternSet &patterns);
 
 /// Collect a set of patterns to split transfer read/write ops.
 ///
@@ -54,7 +54,7 @@ void populateVectorToVectorTransformationPatterns(
 /// of being generic canonicalization patterns. Also one can let the
 /// `ignoreFilter` to return true to fail matching for fine-grained control.
 void populateSplitVectorTransferPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context,
+    RewritePatternSet &patterns,
     std::function<bool(Operation *)> ignoreFilter = nullptr);
 
 /// Collect a set of leading one dimension removal patterns.
@@ -63,16 +63,14 @@ void populateSplitVectorTransferPatterns(
 /// to expose more canonical forms of read/write/insert/extract operations.
 /// With them, there are more chances that we can cancel out extract-insert
 /// pairs or forward write-read pairs.
-void populateCastAwayVectorLeadingOneDimPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context);
+void populateCastAwayVectorLeadingOneDimPatterns(RewritePatternSet &patterns);
 
 /// Collect a set of patterns that bubble up/down bitcast ops.
 ///
 /// These patterns move vector.bitcast ops to be before insert ops or after
 /// extract ops where suitable. With them, bitcast will happen on smaller
 /// vectors and there are more chances to share extract/insert ops.
-void populateBubbleVectorBitCastOpPatterns(OwningRewritePatternList &patterns,
-                                           MLIRContext *context);
+void populateBubbleVectorBitCastOpPatterns(RewritePatternSet &patterns);
 
 /// Collect a set of vector slices transformation patterns:
 ///    ExtractSlicesOpLowering, InsertSlicesOpLowering
@@ -82,15 +80,13 @@ void populateBubbleVectorBitCastOpPatterns(OwningRewritePatternList &patterns,
 /// use for "slices" ops), this lowering removes all tuple related
 /// operations as well (through DCE and folding). If tuple values
 /// "leak" coming in, however, some tuple related ops will remain.
-void populateVectorSlicesLoweringPatterns(OwningRewritePatternList &patterns,
-                                          MLIRContext *context);
+void populateVectorSlicesLoweringPatterns(RewritePatternSet &patterns);
 
 /// Collect a set of transfer read/write lowering patterns.
 ///
 /// These patterns lower transfer ops to simpler ops like `vector.load`,
 /// `vector.store` and `vector.broadcast`.
-void populateVectorTransferLoweringPatterns(OwningRewritePatternList &patterns,
-                                            MLIRContext *context);
+void populateVectorTransferLoweringPatterns(RewritePatternSet &patterns);
 
 /// An attribute that specifies the combining function for `vector.contract`,
 /// and `vector.reduction`.
@@ -174,7 +170,7 @@ struct VectorTransformsOptions {
 /// These transformation express higher level vector ops in terms of more
 /// elementary extraction, insertion, reduction, product, and broadcast ops.
 void populateVectorContractLoweringPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context,
+    RewritePatternSet &patterns,
     VectorTransformsOptions vectorTransformOptions = VectorTransformsOptions());
 
 /// Returns the integer type required for subscripts in the vector dialect.
diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.td b/mlir/include/mlir/Dialect/Vector/VectorOps.td
index 69fb073bfcdf7ee9ffaf647d41743e8113277a80..f26a7719ff341646a5f55911dbdc63290e5a7d59 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.td
@@ -53,6 +53,7 @@ def CombiningKind : BitEnumAttr<
      COMBINING_KIND_MAX, COMBINING_KIND_AND, COMBINING_KIND_OR,
      COMBINING_KIND_XOR]> {
   let cppNamespace = "::mlir::vector";
+  let genSpecializedAttr = 0;
 }
 
 def Vector_CombiningKindAttr : DialectAttr<
diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
index ff3dbfdcad1f946bc51e27b7e1cee0b98dbe6a31..b765dafbce46f00c0be1126c9eb3bd22fcb1ed43 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
@@ -16,8 +16,9 @@
 
 namespace mlir {
 class MLIRContext;
-class OwningRewritePatternList;
 class VectorTransferOpInterface;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 namespace scf {
 class IfOp;
@@ -26,7 +27,7 @@ class IfOp;
 /// Collect a set of patterns to convert from the Vector dialect to itself.
 /// Should be merged with populateVectorToSCFLoweringPattern.
 void populateVectorToVectorConversionPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns,
+    MLIRContext *context, RewritePatternSet &patterns,
     ArrayRef<int64_t> coarseVectorShape = {},
     ArrayRef<int64_t> fineVectorShape = {});
 
@@ -122,7 +123,8 @@ struct UnrollVectorOptions {
 struct UnrollVectorPattern : public RewritePattern {
   using FilterConstraintType = std::function<LogicalResult(Operation *op)>;
   UnrollVectorPattern(MLIRContext *context, UnrollVectorOptions options)
-      : RewritePattern(/*benefit=*/1, MatchAnyOpTypeTag()), options(options) {}
+      : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/1, context),
+        options(options) {}
   LogicalResult matchAndRewrite(Operation *op,
                                 PatternRewriter &rewriter) const override {
     if (options.filterConstraint && failed(options.filterConstraint(op)))
@@ -215,7 +217,7 @@ struct VectorTransferFullPartialRewriter : public RewritePattern {
       FilterConstraintType filter =
           [](VectorTransferOpInterface op) { return success(); },
       PatternBenefit benefit = 1)
-      : RewritePattern(benefit, MatchAnyOpTypeTag()), options(options),
+      : RewritePattern(MatchAnyOpTypeTag(), benefit, context), options(options),
         filter(filter) {}
 
   /// Performs the rewrite.
diff --git a/mlir/include/mlir/IR/BuiltinAttributes.h b/mlir/include/mlir/IR/BuiltinAttributes.h
index 99bb2b68cc622e3f2e27921ebd4e921f0978c492..ac98bfef1b58444ed747a0ec0afb8ce48e77bb16 100644
--- a/mlir/include/mlir/IR/BuiltinAttributes.h
+++ b/mlir/include/mlir/IR/BuiltinAttributes.h
@@ -16,160 +16,14 @@
 
 namespace mlir {
 class AffineMap;
+class BoolAttr;
+class DenseIntElementsAttr;
 class FlatSymbolRefAttr;
 class FunctionType;
 class IntegerSet;
+class IntegerType;
 class Location;
 class ShapedType;
-} // namespace mlir
-
-//===----------------------------------------------------------------------===//
-// Tablegen Attribute Declarations
-//===----------------------------------------------------------------------===//
-
-#define GET_ATTRDEF_CLASSES
-#include "mlir/IR/BuiltinAttributes.h.inc"
-
-//===----------------------------------------------------------------------===//
-// C++ Attribute Declarations
-//===----------------------------------------------------------------------===//
-
-namespace mlir {
-namespace detail {
-
-struct IntegerAttributeStorage;
-struct FloatAttributeStorage;
-struct SymbolRefAttributeStorage;
-struct TypeAttributeStorage;
-
-/// Elements Attributes.
-struct DenseIntOrFPElementsAttributeStorage;
-struct DenseStringElementsAttributeStorage;
-struct OpaqueElementsAttributeStorage;
-struct SparseElementsAttributeStorage;
-} // namespace detail
-
-//===----------------------------------------------------------------------===//
-// FloatAttr
-//===----------------------------------------------------------------------===//
-
-class FloatAttr : public Attribute::AttrBase<FloatAttr, Attribute,
-                                             detail::FloatAttributeStorage> {
-public:
-  using Base::Base;
-  using Base::getChecked;
-  using ValueType = APFloat;
-
-  /// Return a float attribute for the specified value in the specified type.
-  /// These methods should only be used for simple constant values, e.g 1.0/2.0,
-  /// that are known-valid both as host double and the 'type' format.
-  static FloatAttr get(Type type, double value);
-  static FloatAttr getChecked(function_ref<InFlightDiagnostic()> emitError,
-                              Type type, double value);
-
-  /// Return a float attribute for the specified value in the specified type.
-  static FloatAttr get(Type type, const APFloat &value);
-  static FloatAttr getChecked(function_ref<InFlightDiagnostic()> emitError,
-                              Type type, const APFloat &value);
-
-  APFloat getValue() const;
-
-  /// This function is used to convert the value to a double, even if it loses
-  /// precision.
-  double getValueAsDouble() const;
-  static double getValueAsDouble(APFloat val);
-
-  /// Verify the construction invariants for a double value.
-  static LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
-                              Type type, double value);
-  static LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
-                              Type type, const APFloat &value);
-};
-
-//===----------------------------------------------------------------------===//
-// IntegerAttr
-//===----------------------------------------------------------------------===//
-
-class IntegerAttr
-    : public Attribute::AttrBase<IntegerAttr, Attribute,
-                                 detail::IntegerAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = APInt;
-
-  static IntegerAttr get(Type type, int64_t value);
-  static IntegerAttr get(Type type, const APInt &value);
-
-  APInt getValue() const;
-  /// Return the integer value as a 64-bit int. The attribute must be a signless
-  /// integer.
-  // TODO: Change callers to use getValue instead.
-  int64_t getInt() const;
-  /// Return the integer value as a signed 64-bit int. The attribute must be
-  /// a signed integer.
-  int64_t getSInt() const;
-  /// Return the integer value as a unsigned 64-bit int. The attribute must be
-  /// an unsigned integer.
-  uint64_t getUInt() const;
-
-  static LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
-                              Type type, int64_t value);
-  static LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
-                              Type type, const APInt &value);
-};
-
-//===----------------------------------------------------------------------===//
-// BoolAttr
-
-/// Special case of IntegerAttr to represent boolean integers, i.e., signless i1
-/// integers.
-class BoolAttr : public Attribute {
-public:
-  using Attribute::Attribute;
-  using ValueType = bool;
-
-  static BoolAttr get(MLIRContext *context, bool value);
-
-  /// Enable conversion to IntegerAttr. This uses conversion vs. inheritance to
-  /// avoid bringing in all of IntegerAttrs methods.
-  operator IntegerAttr() const { return IntegerAttr(impl); }
-
-  /// Return the boolean value of this attribute.
-  bool getValue() const;
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast.
-  static bool classof(Attribute attr);
-};
-
-//===----------------------------------------------------------------------===//
-// FlatSymbolRefAttr
-//===----------------------------------------------------------------------===//
-
-/// A symbol reference with a reference path containing a single element. This
-/// is used to refer to an operation within the current symbol table.
-class FlatSymbolRefAttr : public SymbolRefAttr {
-public:
-  using SymbolRefAttr::SymbolRefAttr;
-  using ValueType = StringRef;
-
-  /// Construct a symbol reference for the given value name.
-  static FlatSymbolRefAttr get(MLIRContext *ctx, StringRef value) {
-    return SymbolRefAttr::get(ctx, value);
-  }
-
-  /// Returns the name of the held symbol reference.
-  StringRef getValue() const { return getRootReference(); }
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast.
-  static bool classof(Attribute attr) {
-    SymbolRefAttr refAttr = attr.dyn_cast<SymbolRefAttr>();
-    return refAttr && refAttr.getNestedReferences().empty();
-  }
-
-private:
-  using SymbolRefAttr::get;
-  using SymbolRefAttr::getNestedReferences;
-};
 
 //===----------------------------------------------------------------------===//
 // Elements Attributes
@@ -751,88 +605,91 @@ protected:
   bool isValidComplex(int64_t dataEltSize, bool isInt, bool isSigned) const;
 };
 
-/// An attribute class for representing dense arrays of strings. The structure
-/// storing and querying a list of densely packed strings.
-class DenseStringElementsAttr
-    : public Attribute::AttrBase<DenseStringElementsAttr, DenseElementsAttr,
-                                 detail::DenseStringElementsAttributeStorage> {
-
+/// An attribute that represents a reference to a splat vector or tensor
+/// constant, meaning all of the elements have the same value.
+class SplatElementsAttr : public DenseElementsAttr {
 public:
-  using Base::Base;
-
-  /// Overload of the raw 'get' method that asserts that the given type is of
-  /// integer or floating-point type. This method is used to verify type
-  /// invariants that the templatized 'get' method cannot.
-  static DenseStringElementsAttr get(ShapedType type, ArrayRef<StringRef> data);
+  using DenseElementsAttr::DenseElementsAttr;
 
-protected:
-  friend DenseElementsAttr;
+  /// Method for support type inquiry through isa, cast and dyn_cast.
+  static bool classof(Attribute attr) {
+    auto denseAttr = attr.dyn_cast<DenseElementsAttr>();
+    return denseAttr && denseAttr.isSplat();
+  }
 };
 
-/// An attribute class for specializing behavior of Int and Floating-point
-/// densely packed string arrays.
-class DenseIntOrFPElementsAttr
-    : public Attribute::AttrBase<DenseIntOrFPElementsAttr, DenseElementsAttr,
-                                 detail::DenseIntOrFPElementsAttributeStorage> {
+} // namespace mlir
 
+//===----------------------------------------------------------------------===//
+// Tablegen Attribute Declarations
+//===----------------------------------------------------------------------===//
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/IR/BuiltinAttributes.h.inc"
+
+//===----------------------------------------------------------------------===//
+// C++ Attribute Declarations
+//===----------------------------------------------------------------------===//
+
+namespace mlir {
+//===----------------------------------------------------------------------===//
+// BoolAttr
+//===----------------------------------------------------------------------===//
+
+/// Special case of IntegerAttr to represent boolean integers, i.e., signless i1
+/// integers.
+class BoolAttr : public Attribute {
 public:
-  using Base::Base;
-
-  /// Convert endianess of input ArrayRef for big-endian(BE) machines. All of
-  /// the elements of `inRawData` has `type`. If `inRawData` is little endian
-  /// (LE), it is converted to big endian (BE). Conversely, if `inRawData` is
-  /// BE, converted to LE.
-  static void
-  convertEndianOfArrayRefForBEmachine(ArrayRef<char> inRawData,
-                                      MutableArrayRef<char> outRawData,
-                                      ShapedType type);
-
-  /// Convert endianess of input for big-endian(BE) machines. The number of
-  /// elements of `inRawData` is `numElements`, and each element has
-  /// `elementBitWidth` bits. If `inRawData` is little endian (LE), it is
-  /// converted to big endian (BE) and saved in `outRawData`. Conversely, if
-  /// `inRawData` is BE, converted to LE.
-  static void convertEndianOfCharForBEmachine(const char *inRawData,
-                                              char *outRawData,
-                                              size_t elementBitWidth,
-                                              size_t numElements);
+  using Attribute::Attribute;
+  using ValueType = bool;
 
-protected:
-  friend DenseElementsAttr;
+  static BoolAttr get(MLIRContext *context, bool value);
 
-  /// Constructs a dense elements attribute from an array of raw APFloat values.
-  /// Each APFloat value is expected to have the same bitwidth as the element
-  /// type of 'type'. 'type' must be a vector or tensor with static shape.
-  static DenseElementsAttr getRaw(ShapedType type, size_t storageWidth,
-                                  ArrayRef<APFloat> values, bool isSplat);
+  /// Enable conversion to IntegerAttr. This uses conversion vs. inheritance to
+  /// avoid bringing in all of IntegerAttrs methods.
+  operator IntegerAttr() const { return IntegerAttr(impl); }
 
-  /// Constructs a dense elements attribute from an array of raw APInt values.
-  /// Each APInt value is expected to have the same bitwidth as the element type
-  /// of 'type'. 'type' must be a vector or tensor with static shape.
-  static DenseElementsAttr getRaw(ShapedType type, size_t storageWidth,
-                                  ArrayRef<APInt> values, bool isSplat);
+  /// Return the boolean value of this attribute.
+  bool getValue() const;
 
-  /// Get or create a new dense elements attribute instance with the given raw
-  /// data buffer. 'type' must be a vector or tensor with static shape.
-  static DenseElementsAttr getRaw(ShapedType type, ArrayRef<char> data,
-                                  bool isSplat);
+  /// Methods for support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(Attribute attr);
+};
 
-  /// Overload of the raw 'get' method that asserts that the given type is of
-  /// complex type. This method is used to verify type invariants that the
-  /// templatized 'get' method cannot.
-  static DenseElementsAttr getRawComplex(ShapedType type, ArrayRef<char> data,
-                                         int64_t dataEltSize, bool isInt,
-                                         bool isSigned);
+//===----------------------------------------------------------------------===//
+// FlatSymbolRefAttr
+//===----------------------------------------------------------------------===//
 
-  /// Overload of the raw 'get' method that asserts that the given type is of
-  /// integer or floating-point type. This method is used to verify type
-  /// invariants that the templatized 'get' method cannot.
-  static DenseElementsAttr getRawIntOrFloat(ShapedType type,
-                                            ArrayRef<char> data,
-                                            int64_t dataEltSize, bool isInt,
-                                            bool isSigned);
+/// A symbol reference with a reference path containing a single element. This
+/// is used to refer to an operation within the current symbol table.
+class FlatSymbolRefAttr : public SymbolRefAttr {
+public:
+  using SymbolRefAttr::SymbolRefAttr;
+  using ValueType = StringRef;
+
+  /// Construct a symbol reference for the given value name.
+  static FlatSymbolRefAttr get(MLIRContext *ctx, StringRef value) {
+    return SymbolRefAttr::get(ctx, value);
+  }
+
+  /// Returns the name of the held symbol reference.
+  StringRef getValue() const { return getRootReference(); }
+
+  /// Methods for support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(Attribute attr) {
+    SymbolRefAttr refAttr = attr.dyn_cast<SymbolRefAttr>();
+    return refAttr && refAttr.getNestedReferences().empty();
+  }
+
+private:
+  using SymbolRefAttr::get;
+  using SymbolRefAttr::getNestedReferences;
 };
 
+//===----------------------------------------------------------------------===//
+// DenseFPElementsAttr
+//===----------------------------------------------------------------------===//
+
 /// An attribute that represents a reference to a dense float vector or tensor
 /// object. Each element is stored as a double.
 class DenseFPElementsAttr : public DenseIntOrFPElementsAttr {
@@ -869,6 +726,10 @@ public:
   static bool classof(Attribute attr);
 };
 
+//===----------------------------------------------------------------------===//
+// DenseIntElementsAttr
+//===----------------------------------------------------------------------===//
+
 /// An attribute that represents a reference to a dense integer vector or tensor
 /// object.
 class DenseIntElementsAttr : public DenseIntOrFPElementsAttr {
@@ -906,170 +767,27 @@ public:
   static bool classof(Attribute attr);
 };
 
-/// An opaque attribute that represents a reference to a vector or tensor
-/// constant with opaque content. This representation is for tensor constants
-/// which the compiler may not need to interpret. This attribute is always
-/// associated with a particular dialect, which provides a method to convert
-/// tensor representation to a non-opaque format.
-class OpaqueElementsAttr
-    : public Attribute::AttrBase<OpaqueElementsAttr, ElementsAttr,
-                                 detail::OpaqueElementsAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = StringRef;
-
-  static OpaqueElementsAttr get(Dialect *dialect, ShapedType type,
-                                StringRef bytes);
-
-  StringRef getValue() const;
-
-  /// Return the value at the given index. The 'index' is expected to refer to a
-  /// valid element.
-  Attribute getValue(ArrayRef<uint64_t> index) const;
-
-  /// Decodes the attribute value using dialect-specific decoding hook.
-  /// Returns false if decoding is successful. If not, returns true and leaves
-  /// 'result' argument unspecified.
-  bool decode(ElementsAttr &result);
-
-  /// Returns dialect associated with this opaque constant.
-  Dialect *getDialect() const;
-};
-
-/// An attribute that represents a reference to a sparse vector or tensor
-/// object.
-///
-/// This class uses COO (coordinate list) encoding to represent the sparse
-/// elements in an element attribute. Specifically, the sparse vector/tensor
-/// stores the indices and values as two separate dense elements attributes of
-/// tensor type (even if the sparse attribute is of vector type, in order to
-/// support empty lists). The dense elements attribute indices is a 2-D tensor
-/// of 64-bit integer elements with shape [N, ndims], which specifies the
-/// indices of the elements in the sparse tensor that contains nonzero values.
-/// The dense elements attribute values is a 1-D tensor with shape [N], and it
-/// supplies the corresponding values for the indices.
-///
-/// For example,
-/// `sparse<tensor<3x4xi32>, [[0, 0], [1, 2]], [1, 5]>` represents tensor
-/// [[1, 0, 0, 0],
-///  [0, 0, 5, 0],
-///  [0, 0, 0, 0]].
-class SparseElementsAttr
-    : public Attribute::AttrBase<SparseElementsAttr, ElementsAttr,
-                                 detail::SparseElementsAttributeStorage> {
-public:
-  using Base::Base;
-
-  template <typename T>
-  using iterator =
-      llvm::mapped_iterator<llvm::detail::value_sequence_iterator<ptrdiff_t>,
-                            std::function<T(ptrdiff_t)>>;
-
-  /// 'type' must be a vector or tensor with static shape.
-  static SparseElementsAttr get(ShapedType type, DenseElementsAttr indices,
-                                DenseElementsAttr values);
-
-  DenseIntElementsAttr getIndices() const;
-
-  DenseElementsAttr getValues() const;
-
-  /// Return the values of this attribute in the form of the given type 'T'. 'T'
-  /// may be any of Attribute, APInt, APFloat, c++ integer/float types, etc.
-  template <typename T> llvm::iterator_range<iterator<T>> getValues() const {
-    auto zeroValue = getZeroValue<T>();
-    auto valueIt = getValues().getValues<T>().begin();
-    const std::vector<ptrdiff_t> flatSparseIndices(getFlattenedSparseIndices());
-    // TODO: Move-capture flatSparseIndices when c++14 is available.
-    std::function<T(ptrdiff_t)> mapFn = [=](ptrdiff_t index) {
-      // Try to map the current index to one of the sparse indices.
-      for (unsigned i = 0, e = flatSparseIndices.size(); i != e; ++i)
-        if (flatSparseIndices[i] == index)
-          return *std::next(valueIt, i);
-      // Otherwise, return the zero value.
-      return zeroValue;
-    };
-    return llvm::map_range(llvm::seq<ptrdiff_t>(0, getNumElements()), mapFn);
-  }
-
-  /// Return the value of the element at the given index. The 'index' is
-  /// expected to refer to a valid element.
-  Attribute getValue(ArrayRef<uint64_t> index) const;
-
-private:
-  /// Get a zero APFloat for the given sparse attribute.
-  APFloat getZeroAPFloat() const;
-
-  /// Get a zero APInt for the given sparse attribute.
-  APInt getZeroAPInt() const;
-
-  /// Get a zero attribute for the given sparse attribute.
-  Attribute getZeroAttr() const;
-
-  /// Utility methods to generate a zero value of some type 'T'. This is used by
-  /// the 'iterator' class.
-  /// Get a zero for a given attribute type.
-  template <typename T>
-  typename std::enable_if<std::is_base_of<Attribute, T>::value, T>::type
-  getZeroValue() const {
-    return getZeroAttr().template cast<T>();
-  }
-  /// Get a zero for an APInt.
-  template <typename T>
-  typename std::enable_if<std::is_same<APInt, T>::value, T>::type
-  getZeroValue() const {
-    return getZeroAPInt();
-  }
-  template <typename T>
-  typename std::enable_if<std::is_same<std::complex<APInt>, T>::value, T>::type
-  getZeroValue() const {
-    APInt intZero = getZeroAPInt();
-    return {intZero, intZero};
-  }
-  /// Get a zero for an APFloat.
-  template <typename T>
-  typename std::enable_if<std::is_same<APFloat, T>::value, T>::type
-  getZeroValue() const {
-    return getZeroAPFloat();
-  }
-  template <typename T>
-  typename std::enable_if<std::is_same<std::complex<APFloat>, T>::value,
-                          T>::type
-  getZeroValue() const {
-    APFloat floatZero = getZeroAPFloat();
-    return {floatZero, floatZero};
-  }
-
-  /// Get a zero for an C++ integer, float, StringRef, or complex type.
-  template <typename T>
-  typename std::enable_if<
-      std::numeric_limits<T>::is_integer ||
-          DenseElementsAttr::is_valid_cpp_fp_type<T>::value ||
-          std::is_same<T, StringRef>::value ||
-          (detail::is_complex_t<T>::value &&
-           !llvm::is_one_of<T, std::complex<APInt>,
-                            std::complex<APFloat>>::value),
-      T>::type
-  getZeroValue() const {
-    return T();
-  }
-
-  /// Flatten, and return, all of the sparse indices in this attribute in
-  /// row-major order.
-  std::vector<ptrdiff_t> getFlattenedSparseIndices() const;
-};
-
-/// An attribute that represents a reference to a splat vector or tensor
-/// constant, meaning all of the elements have the same value.
-class SplatElementsAttr : public DenseElementsAttr {
-public:
-  using DenseElementsAttr::DenseElementsAttr;
+//===----------------------------------------------------------------------===//
+// SparseElementsAttr
+//===----------------------------------------------------------------------===//
 
-  /// Method for support type inquiry through isa, cast and dyn_cast.
-  static bool classof(Attribute attr) {
-    auto denseAttr = attr.dyn_cast<DenseElementsAttr>();
-    return denseAttr && denseAttr.isSplat();
-  }
-};
+template <typename T>
+auto SparseElementsAttr::getValues() const
+    -> llvm::iterator_range<iterator<T>> {
+  auto zeroValue = getZeroValue<T>();
+  auto valueIt = getValues().getValues<T>().begin();
+  const std::vector<ptrdiff_t> flatSparseIndices(getFlattenedSparseIndices());
+  // TODO: Move-capture flatSparseIndices when c++14 is available.
+  std::function<T(ptrdiff_t)> mapFn = [=](ptrdiff_t index) {
+    // Try to map the current index to one of the sparse indices.
+    for (unsigned i = 0, e = flatSparseIndices.size(); i != e; ++i)
+      if (flatSparseIndices[i] == index)
+        return *std::next(valueIt, i);
+    // Otherwise, return the zero value.
+    return zeroValue;
+  };
+  return llvm::map_range(llvm::seq<ptrdiff_t>(0, getNumElements()), mapFn);
+}
 
 namespace detail {
 /// This class represents a general iterator over the values of an ElementsAttr.
diff --git a/mlir/include/mlir/IR/BuiltinAttributes.td b/mlir/include/mlir/IR/BuiltinAttributes.td
index 530ab0b79e3f0557061eb78d9804630db5af9d42..45214535b1f8ced3474e4e1350b245dd7fc68c74 100644
--- a/mlir/include/mlir/IR/BuiltinAttributes.td
+++ b/mlir/include/mlir/IR/BuiltinAttributes.td
@@ -22,7 +22,8 @@ include "mlir/IR/BuiltinDialect.td"
 // to this file instead.
 
 // Base class for Builtin dialect attributes.
-class Builtin_Attr<string name> : AttrDef<Builtin_Dialect, name> {
+class Builtin_Attr<string name, string baseCppClass = "::mlir::Attribute">
+    : AttrDef<Builtin_Dialect, name, baseCppClass> {
   let mnemonic = ?;
 }
 
@@ -127,6 +128,151 @@ def Builtin_ArrayAttr : Builtin_Attr<"Array"> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// DenseIntOrFPElementsAttr
+//===----------------------------------------------------------------------===//
+
+def Builtin_DenseIntOrFPElementsAttr
+    : Builtin_Attr<"DenseIntOrFPElements", "DenseElementsAttr"> {
+  let summary = "An Attribute containing a dense multi-dimensional array of "
+                "integer or floating-point values";
+  let description = [{
+    Syntax:
+
+    ```
+    dense-intorfloat-elements-attribute ::= `dense` `<` attribute-value `>` `:`
+                                            ( tensor-type | vector-type )
+    ```
+
+    A dense int-or-float elements attribute is an elements attribute containing
+    a densely packed vector or tensor of integer or floating-point values. The
+    element type of this attribute is required to be either an `IntegerType` or
+    a `FloatType`.
+
+    Examples:
+
+    ```
+    // A splat tensor of integer values.
+    dense<10> : tensor<2xi32>
+    // A tensor of 2 float32 elements.
+    dense<[10.0, 11.0]> : tensor<2xf32>
+    ```
+  }];
+  let parameters = (ins AttributeSelfTypeParameter<"", "ShapedType">:$type,
+                        "ArrayRef<char>":$rawData);
+  let extraClassDeclaration = [{
+    /// Convert endianess of input ArrayRef for big-endian(BE) machines. All of
+    /// the elements of `inRawData` has `type`. If `inRawData` is little endian
+    /// (LE), it is converted to big endian (BE). Conversely, if `inRawData` is
+    /// BE, converted to LE.
+    static void
+    convertEndianOfArrayRefForBEmachine(ArrayRef<char> inRawData,
+                                        MutableArrayRef<char> outRawData,
+                                        ShapedType type);
+
+    /// Convert endianess of input for big-endian(BE) machines. The number of
+    /// elements of `inRawData` is `numElements`, and each element has
+    /// `elementBitWidth` bits. If `inRawData` is little endian (LE), it is
+    /// converted to big endian (BE) and saved in `outRawData`. Conversely, if
+    /// `inRawData` is BE, converted to LE.
+    static void convertEndianOfCharForBEmachine(const char *inRawData,
+                                                char *outRawData,
+                                                size_t elementBitWidth,
+                                                size_t numElements);
+
+  protected:
+    friend DenseElementsAttr;
+
+    /// Constructs a dense elements attribute from an array of raw APFloat
+    /// values. Each APFloat value is expected to have the same bitwidth as the
+    /// element type of 'type'. 'type' must be a vector or tensor with static
+    /// shape.
+    static DenseElementsAttr getRaw(ShapedType type, size_t storageWidth,
+                                    ArrayRef<APFloat> values, bool isSplat);
+
+    /// Constructs a dense elements attribute from an array of raw APInt values.
+    /// Each APInt value is expected to have the same bitwidth as the element
+    /// type of 'type'. 'type' must be a vector or tensor with static shape.
+    static DenseElementsAttr getRaw(ShapedType type, size_t storageWidth,
+                                    ArrayRef<APInt> values, bool isSplat);
+
+    /// Get or create a new dense elements attribute instance with the given raw
+    /// data buffer. 'type' must be a vector or tensor with static shape.
+    static DenseElementsAttr getRaw(ShapedType type, ArrayRef<char> data,
+                                    bool isSplat);
+
+    /// Overload of the raw 'get' method that asserts that the given type is of
+    /// complex type. This method is used to verify type invariants that the
+    /// templatized 'get' method cannot.
+    static DenseElementsAttr getRawComplex(ShapedType type, ArrayRef<char> data,
+                                           int64_t dataEltSize, bool isInt,
+                                           bool isSigned);
+
+    /// Overload of the raw 'get' method that asserts that the given type is of
+    /// integer or floating-point type. This method is used to verify type
+    /// invariants that the templatized 'get' method cannot.
+    static DenseElementsAttr getRawIntOrFloat(ShapedType type,
+                                              ArrayRef<char> data,
+                                              int64_t dataEltSize, bool isInt,
+                                              bool isSigned);
+
+  public:
+  }];
+  let genAccessors = 0;
+  let genStorageClass = 0;
+  let skipDefaultBuilders = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// DenseStringElementsAttr
+//===----------------------------------------------------------------------===//
+
+def Builtin_DenseStringElementsAttr
+    : Builtin_Attr<"DenseStringElements", "DenseElementsAttr"> {
+  let summary = "An Attribute containing a dense multi-dimensional array of "
+                "strings";
+  let description = [{
+    Syntax:
+
+    ```
+    dense-string-elements-attribute ::= `dense` `<` attribute-value `>` `:`
+                                        ( tensor-type | vector-type )
+    ```
+
+    A dense string elements attribute is an elements attribute containing a
+    densely packed vector or tensor of string values. There are no restrictions
+    placed on the element type of this attribute, enabling the use of dialect
+    specific string types.
+
+    Examples:
+
+    ```
+    // A splat tensor of strings.
+    dense<"example"> : tensor<2x!foo.string>
+    // A tensor of 2 string elements.
+    dense<["example1", "example2"]> : tensor<2x!foo.string>
+    ```
+  }];
+  let parameters = (ins AttributeSelfTypeParameter<"", "ShapedType">:$type,
+                        "ArrayRef<StringRef>":$value);
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "ShapedType":$type,
+                                        "ArrayRef<StringRef>":$values), [{
+      return $_get(type.getContext(), type, values,
+                   /* isSplat */(values.size() == 1));
+    }]>,
+  ];
+  let extraClassDeclaration = [{
+  protected:
+    friend DenseElementsAttr;
+
+  public:
+  }];
+  let genAccessors = 0;
+  let genStorageClass = 0;
+  let skipDefaultBuilders = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // DictionaryAttr
 //===----------------------------------------------------------------------===//
@@ -220,6 +366,147 @@ def Builtin_DictionaryAttr : Builtin_Attr<"Dictionary"> {
   let skipDefaultBuilders = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// FloatAttr
+//===----------------------------------------------------------------------===//
+
+def Builtin_FloatAttr : Builtin_Attr<"Float"> {
+  let summary = "An Attribute containing a floating-point value";
+  let description = [{
+    Syntax:
+
+    ```
+    float-attribute ::= (float-literal (`:` float-type)?)
+                      | (hexadecimal-literal `:` float-type)
+    ```
+
+    A float attribute is a literal attribute that represents a floating point
+    value of the specified [float type](#floating-point-types). It can be
+    represented in the hexadecimal form where the hexadecimal value is
+    interpreted as bits of the underlying binary representation. This form is
+    useful for representing infinity and NaN floating point values. To avoid
+    confusion with integer attributes, hexadecimal literals _must_ be followed
+    by a float type to define a float attribute.
+
+    Examples:
+
+    ```
+    42.0         // float attribute defaults to f64 type
+    42.0 : f32   // float attribute of f32 type
+    0x7C00 : f16 // positive infinity
+    0x7CFF : f16 // NaN (one of possible values)
+    42 : f32     // Error: expected integer type
+    ```
+  }];
+  let parameters = (ins AttributeSelfTypeParameter<"">:$type,
+                        APFloatParameter<"">:$value);
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "Type":$type,
+                                        "const APFloat &":$value), [{
+      return $_get(type.getContext(), type, value);
+    }]>,
+    AttrBuilderWithInferredContext<(ins "Type":$type, "double":$value), [{
+      if (type.isF64() || !type.isa<FloatType>())
+        return $_get(type.getContext(), type, APFloat(value));
+
+      // This handles, e.g., F16 because there is no APFloat constructor for it.
+      bool unused;
+      APFloat val(value);
+      val.convert(type.cast<FloatType>().getFloatSemantics(),
+                  APFloat::rmNearestTiesToEven, &unused);
+      return $_get(type.getContext(), type, val);
+    }]>
+  ];
+  let extraClassDeclaration = [{
+    using ValueType = APFloat;
+
+    /// This function is used to convert the value to a double, even if it loses
+    /// precision.
+    double getValueAsDouble() const;
+    static double getValueAsDouble(APFloat val);
+  }];
+  let genVerifyDecl = 1;
+  let skipDefaultBuilders = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// IntegerAttr
+//===----------------------------------------------------------------------===//
+
+def Builtin_IntegerAttr : Builtin_Attr<"Integer"> {
+  let summary = "An Attribute containing a integer value";
+  let description = [{
+    Syntax:
+
+    ```
+    integer-attribute ::= (integer-literal ( `:` (index-type | integer-type) )?)
+                          | `true` | `false`
+    ```
+
+    An integer attribute is a literal attribute that represents an integral
+    value of the specified integer or index type. `i1` integer attributes are
+    treated as `boolean` attributes, and use a unique assembly format of either
+    `true` or `false` depending on the value. The default type for non-boolean
+    integer attributes, if a type is not specified, is signless 64-bit integer.
+
+    Examples:
+
+    ```mlir
+    10 : i32
+    10    // : i64 is implied here.
+    true  // A bool, i.e. i1, value.
+    false // A bool, i.e. i1, value.
+    ```
+  }];
+  let parameters = (ins AttributeSelfTypeParameter<"">:$type, "APInt":$value);
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "Type":$type,
+                                        "const APInt &":$value), [{
+      if (type.isSignlessInteger(1))
+        return BoolAttr::get(type.getContext(), value.getBoolValue());
+      return $_get(type.getContext(), type, value);
+    }]>,
+    AttrBuilderWithInferredContext<(ins "Type":$type, "int64_t":$value), [{
+      // `index` has a defined internal storage width.
+      if (type.isIndex()) {
+        APInt apValue(IndexType::kInternalStorageBitWidth, value);
+        return $_get(type.getContext(), type, apValue);
+      }
+
+      IntegerType intTy = type.cast<IntegerType>();
+      APInt apValue(intTy.getWidth(), value, intTy.isSignedInteger());
+      return $_get(type.getContext(), type, apValue);
+    }]>
+  ];
+  let extraClassDeclaration = [{
+    using ValueType = APInt;
+
+    /// Return the integer value as a 64-bit int. The attribute must be a
+    /// signless integer.
+    // TODO: Change callers to use getValue instead.
+    int64_t getInt() const;
+    /// Return the integer value as a signed 64-bit int. The attribute must be
+    /// a signed integer.
+    int64_t getSInt() const;
+    /// Return the integer value as a unsigned 64-bit int. The attribute must be
+    /// an unsigned integer.
+    uint64_t getUInt() const;
+
+  private:
+    /// Return a boolean attribute. This is a special variant of the `get`
+    /// method that is used by the MLIRContext to cache the boolean IntegerAttr
+    /// instances.
+    static BoolAttr getBoolAttrUnchecked(IntegerType type, bool value);
+
+    /// Allow access to `getBoolAttrUnchecked`.
+    friend MLIRContext;
+
+  public:
+  }];
+  let genVerifyDecl = 1;
+  let skipDefaultBuilders = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // IntegerSetAttr
 //===----------------------------------------------------------------------===//
@@ -282,8 +569,212 @@ def Builtin_OpaqueAttr : Builtin_Attr<"Opaque"> {
       return $_get(dialect.getContext(), dialect, attrData, type);
     }]>
   ];
-  bit genVerifyDecl = 1;
-  // let skipDefaultBuilders = 1;
+  let genVerifyDecl = 1;
+  let skipDefaultBuilders = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// OpaqueElementsAttr
+//===----------------------------------------------------------------------===//
+
+def Builtin_OpaqueElementsAttr
+    : Builtin_Attr<"OpaqueElements", "ElementsAttr"> {
+  let summary = "An opaque representation of a multi-dimensional array";
+  let description = [{
+    Syntax:
+
+    ```
+    opaque-elements-attribute ::= `opaque` `<` dialect-namespace  `,`
+                                  hex-string-literal `>` `:`
+                                  ( tensor-type | vector-type )
+    ```
+
+    An opaque elements attribute is an elements attribute where the content of
+    the value is opaque. The representation of the constant stored by this
+    elements attribute is only understood, and thus decodable, by the dialect
+    that created it.
+
+    Note: The parsed string literal must be in hexadecimal form.
+
+    Examples:
+
+    ```mlir
+    opaque<"foo_dialect", "0xDEADBEEF"> : tensor<10xi32>
+    ```
+  }];
+
+  // TODO: Provide a way to avoid copying content of large opaque
+  // tensors This will likely require a new reference attribute kind.
+  let parameters = (ins "Identifier":$dialect,
+                        StringRefParameter<"">:$value,
+                        AttributeSelfTypeParameter<"", "ShapedType">:$type);
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "Identifier":$dialect,
+                                        "ShapedType":$type,
+                                        "StringRef":$value), [{
+      return $_get(dialect.getContext(), dialect, value, type);
+    }]>,
+    AttrBuilderWithInferredContext<(ins "Dialect *":$dialect,
+                                        "ShapedType":$type,
+                                        "StringRef":$value), [{
+      MLIRContext *ctxt = dialect->getContext();
+      Identifier dialectName = Identifier::get(dialect->getNamespace(), ctxt);
+      return $_get(ctxt, dialectName, value, type);
+    }]>
+  ];
+  let extraClassDeclaration = [{
+    using ValueType = StringRef;
+
+    /// Return the value at the given index. The 'index' is expected to refer to
+    /// a valid element.
+    Attribute getValue(ArrayRef<uint64_t> index) const;
+
+    /// Decodes the attribute value using dialect-specific decoding hook.
+    /// Returns false if decoding is successful. If not, returns true and leaves
+    /// 'result' argument unspecified.
+    bool decode(ElementsAttr &result);
+
+  }];
+  let genVerifyDecl = 1;
+  let skipDefaultBuilders = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// SparseElementsAttr
+//===----------------------------------------------------------------------===//
+
+def Builtin_SparseElementsAttr
+    : Builtin_Attr<"SparseElements", "ElementsAttr"> {
+  let summary = "An opaque representation of a multi-dimensional array";
+  let description = [{
+    Syntax:
+
+    ```
+    sparse-elements-attribute ::= `sparse` `<` attribute-value `,`
+                                  attribute-value `>` `:`
+                                  ( tensor-type | vector-type )
+    ```
+
+    A sparse elements attribute is an elements attribute that represents a
+    sparse vector or tensor object. This is where very few of the elements are
+    non-zero.
+
+    The attribute uses COO (coordinate list) encoding to represent the sparse
+    elements of the elements attribute. The indices are stored via a 2-D tensor
+    of 64-bit integer elements with shape [N, ndims], which specifies the
+    indices of the elements in the sparse tensor that contains non-zero values.
+    The element values are stored via a 1-D tensor with shape [N], that supplies
+    the corresponding values for the indices.
+
+    Example:
+
+    ```mlir
+    sparse<[[0, 0], [1, 2]], [1, 5]> : tensor<3x4xi32>
+
+    // This represents the following tensor:
+    ///  [[1, 0, 0, 0],
+    ///   [0, 0, 5, 0],
+    ///   [0, 0, 0, 0]]
+    ```
+  }];
+
+  let parameters = (ins AttributeSelfTypeParameter<"", "ShapedType">:$type,
+                        "DenseIntElementsAttr":$indices,
+                        "DenseElementsAttr":$values);
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "ShapedType":$type,
+                                        "DenseElementsAttr":$indices,
+                                        "DenseElementsAttr":$values), [{
+      assert(indices.getType().getElementType().isInteger(64) &&
+             "expected sparse indices to be 64-bit integer values");
+      assert((type.isa<RankedTensorType, VectorType>()) &&
+             "type must be ranked tensor or vector");
+      assert(type.hasStaticShape() && "type must have static shape");
+      return $_get(type.getContext(), type,
+                   indices.cast<DenseIntElementsAttr>(), values);
+    }]>,
+  ];
+  let extraClassDeclaration = [{
+    template <typename T>
+    using iterator =
+        llvm::mapped_iterator<llvm::detail::value_sequence_iterator<ptrdiff_t>,
+                              std::function<T(ptrdiff_t)>>;
+
+    /// Return the values of this attribute in the form of the given type 'T'.
+    /// 'T'  may be any of Attribute, APInt, APFloat, c++ integer/float types,
+    /// etc.
+    template <typename T> llvm::iterator_range<iterator<T>> getValues() const;
+
+    /// Return the value of the element at the given index. The 'index' is
+    /// expected to refer to a valid element.
+    Attribute getValue(ArrayRef<uint64_t> index) const;
+
+  private:
+    /// Get a zero APFloat for the given sparse attribute.
+    APFloat getZeroAPFloat() const;
+
+    /// Get a zero APInt for the given sparse attribute.
+    APInt getZeroAPInt() const;
+
+    /// Get a zero attribute for the given sparse attribute.
+    Attribute getZeroAttr() const;
+
+    /// Utility methods to generate a zero value of some type 'T'. This is used
+    /// by the 'iterator' class.
+    /// Get a zero for a given attribute type.
+    template <typename T>
+    typename std::enable_if<std::is_base_of<Attribute, T>::value, T>::type
+    getZeroValue() const {
+      return getZeroAttr().template cast<T>();
+    }
+    /// Get a zero for an APInt.
+    template <typename T>
+    typename std::enable_if<std::is_same<APInt, T>::value, T>::type
+    getZeroValue() const {
+      return getZeroAPInt();
+    }
+    template <typename T>
+    typename std::enable_if<std::is_same<std::complex<APInt>, T>::value,
+                            T>::type
+    getZeroValue() const {
+      APInt intZero = getZeroAPInt();
+      return {intZero, intZero};
+    }
+    /// Get a zero for an APFloat.
+    template <typename T>
+    typename std::enable_if<std::is_same<APFloat, T>::value, T>::type
+    getZeroValue() const {
+      return getZeroAPFloat();
+    }
+    template <typename T>
+    typename std::enable_if<std::is_same<std::complex<APFloat>, T>::value,
+                            T>::type
+    getZeroValue() const {
+      APFloat floatZero = getZeroAPFloat();
+      return {floatZero, floatZero};
+    }
+
+    /// Get a zero for an C++ integer, float, StringRef, or complex type.
+    template <typename T>
+    typename std::enable_if<
+        std::numeric_limits<T>::is_integer ||
+            DenseElementsAttr::is_valid_cpp_fp_type<T>::value ||
+            std::is_same<T, StringRef>::value ||
+            (detail::is_complex_t<T>::value &&
+             !llvm::is_one_of<T, std::complex<APInt>,
+                              std::complex<APFloat>>::value),
+        T>::type
+    getZeroValue() const {
+      return T();
+    }
+
+    /// Flatten, and return, all of the sparse indices in this attribute in
+    /// row-major order.
+    std::vector<ptrdiff_t> getFlattenedSparseIndices() const;
+
+  public:
+  }];
+  let skipDefaultBuilders = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/BuiltinOps.h b/mlir/include/mlir/IR/BuiltinOps.h
index c0163b148f3cc7ed8006d5ba055bea39f2fac437..bc341e624fcd47bb9c66d26c3b4026d932fb995a 100644
--- a/mlir/include/mlir/IR/BuiltinOps.h
+++ b/mlir/include/mlir/IR/BuiltinOps.h
@@ -15,9 +15,11 @@
 
 #include "mlir/IR/FunctionSupport.h"
 #include "mlir/IR/OwningOpRef.h"
+#include "mlir/IR/RegionKindInterface.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "mlir/Interfaces/CastInterfaces.h"
+#include "mlir/Interfaces/DataLayoutInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 
diff --git a/mlir/include/mlir/IR/BuiltinOps.td b/mlir/include/mlir/IR/BuiltinOps.td
index 3e61608ace1d5fb1ca4b72f8f8faea479498c7e5..f91f4e2e613a305ee0efb1fa858a50aed242975a 100644
--- a/mlir/include/mlir/IR/BuiltinOps.td
+++ b/mlir/include/mlir/IR/BuiltinOps.td
@@ -15,9 +15,11 @@
 #define BUILTIN_OPS
 
 include "mlir/IR/BuiltinDialect.td"
+include "mlir/IR/RegionKindInterface.td"
 include "mlir/IR/SymbolInterfaces.td"
 include "mlir/Interfaces/CallInterfaces.td"
 include "mlir/Interfaces/CastInterfaces.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 
 // Base class for Builtin dialect ops.
@@ -158,17 +160,17 @@ def FuncOp : Builtin_Op<"func", [
 //===----------------------------------------------------------------------===//
 
 def ModuleOp : Builtin_Op<"module", [
-  AffineScope, IsolatedFromAbove, NoRegionArguments, SymbolTable, Symbol,
-  SingleBlockImplicitTerminator<"ModuleTerminatorOp">
-]> {
+  AffineScope, IsolatedFromAbove, NoRegionArguments, SymbolTable, Symbol] 
+  # GraphRegionNoTerminator.traits> {
   let summary = "A top level container operation";
   let description = [{
     A `module` represents a top-level container operation. It contains a single
-    SSACFG region containing a single block which can contain any
-    operations. Operations within this region cannot implicitly capture values
-    defined outside the module, i.e. Modules are `IsolatedFromAbove`. Modules
-    have an optional symbol name which can be used to refer to them in
-    operations.
+   [graph region](#control-flow-and-ssacfg-regions) containing a single block
+   which can contain any operations and does not have a terminator. Operations
+   within this region cannot implicitly capture values defined outside the module,
+   i.e. Modules are [IsolatedFromAbove](Traits.md#isolatedfromabove). Modules have
+   an optional [symbol name](SymbolsAndSymbolTables.md) which can be used to refer
+   to them in operations.
 
     Example:
 
@@ -198,6 +200,12 @@ def ModuleOp : Builtin_Op<"module", [
 
     /// A ModuleOp may optionally define a symbol.
     bool isOptionalSymbol() { return true; }
+
+    //===------------------------------------------------------------------===//
+    // DataLayoutOpInterface Methods
+    //===------------------------------------------------------------------===//
+
+    DataLayoutSpecInterface getDataLayoutSpec();
   }];
   let verifier = [{ return ::verify(*this); }];
 
@@ -206,22 +214,6 @@ def ModuleOp : Builtin_Op<"module", [
   let skipDefaultBuilders = 1;
 }
 
-//===----------------------------------------------------------------------===//
-// ModuleTerminatorOp
-//===----------------------------------------------------------------------===//
-
-def ModuleTerminatorOp : Builtin_Op<"module_terminator", [
-  Terminator, HasParent<"ModuleOp">
-]> {
-  let summary = "A pseudo op that marks the end of a module";
-  let description = [{
-    `module_terminator` is a special terminator operation for the body of a
-    `module`, it has no semantic meaning beyond keeping the body of a `module`
-    well-formed.
-  }];
-  let assemblyFormat = "attr-dict";
-}
-
 //===----------------------------------------------------------------------===//
 // UnrealizedConversionCastOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/BuiltinTypes.td b/mlir/include/mlir/IR/BuiltinTypes.td
index 02f699ab3628f296224da2edc83225998db054cd..22d194db3b6852bc3f7e82adefd32a229a71670a 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.td
+++ b/mlir/include/mlir/IR/BuiltinTypes.td
@@ -131,7 +131,6 @@ def Builtin_Function : Builtin_Type<"Function"> {
 
     The function type can be thought of as a function signature. It consists of
     a list of formal parameter types and a list of formal result types.
-    ```
   }];
   let parameters = (ins "ArrayRef<Type>":$inputs, "ArrayRef<Type>":$results);
   let builders = [
diff --git a/mlir/include/mlir/IR/CMakeLists.txt b/mlir/include/mlir/IR/CMakeLists.txt
index e44e5dc218b1e5c8abcce9aabdd044f3e8c04fc7..963d6a87eee25fb8ed8a0ef1405fb2b24b73e48c 100644
--- a/mlir/include/mlir/IR/CMakeLists.txt
+++ b/mlir/include/mlir/IR/CMakeLists.txt
@@ -26,4 +26,7 @@ mlir_tablegen(BuiltinTypes.h.inc -gen-typedef-decls)
 mlir_tablegen(BuiltinTypes.cpp.inc -gen-typedef-defs)
 add_public_tablegen_target(MLIRBuiltinTypesIncGen)
 
-add_mlir_doc(BuiltinOps -gen-dialect-doc Builtin Dialects/)
+add_mlir_doc(BuiltinAttributes -gen-attrdef-doc BuiltinAttributes Dialects/)
+add_mlir_doc(BuiltinLocationAttributes -gen-attrdef-doc BuiltinLocationAttributes Dialects/)
+add_mlir_doc(BuiltinOps -gen-op-doc BuiltinOps Dialects/)
+add_mlir_doc(BuiltinTypes -gen-typedef-doc BuiltinTypes Dialects/)
diff --git a/mlir/include/mlir/IR/Dialect.h b/mlir/include/mlir/IR/Dialect.h
index 4a816ccb79c9545653e1c26d6bdbcd0700a8e88e..d3af95521d74b402ee3d1fe90bd428ef3fd3be8b 100644
--- a/mlir/include/mlir/IR/Dialect.h
+++ b/mlir/include/mlir/IR/Dialect.h
@@ -39,6 +39,12 @@ using InterfaceAllocatorFunction =
 ///
 class Dialect {
 public:
+  /// Type for a callback provided by the dialect to parse a custom operation.
+  /// This is used for the dialect to provide an alternative way to parse custom
+  /// operations, including unregistered ones.
+  using ParseOpHook =
+      function_ref<ParseResult(OpAsmParser &parser, OperationState &result)>;
+
   virtual ~Dialect();
 
   /// Utility function that returns if the given string is a valid dialect
@@ -97,6 +103,18 @@ public:
     llvm_unreachable("dialect has no registered type printing hook");
   }
 
+  /// Return the hook to parse an operation registered to this dialect, if any.
+  /// By default this will lookup for registered operations and return the
+  /// `parse()` method registered on the AbstractOperation. Dialects can
+  /// override this behavior and handle unregistered operations as well.
+  virtual Optional<ParseOpHook> getParseOperationHook(StringRef opName) const;
+
+  /// Print an operation registered to this dialect.
+  /// This hook is invoked for registered operation which don't override the
+  /// `print()` method to define their own custom assembly.
+  virtual LogicalResult printOperation(Operation *op,
+                                       OpAsmPrinter &printer) const;
+
   //===--------------------------------------------------------------------===//
   // Verification Hooks
   //===--------------------------------------------------------------------===//
@@ -140,6 +158,19 @@ public:
         getRegisteredInterface(InterfaceT::getInterfaceID()));
   }
 
+  /// Lookup an op interface for the given ID if one is registered, otherwise
+  /// nullptr.
+  virtual void *getRegisteredInterfaceForOp(TypeID interfaceID,
+                                            OperationName opName) {
+    return nullptr;
+  }
+  template <typename InterfaceT>
+  typename InterfaceT::Concept *
+  getRegisteredInterfaceForOp(OperationName opName) {
+    return static_cast<typename InterfaceT::Concept *>(
+        getRegisteredInterfaceForOp(InterfaceT::getInterfaceID(), opName));
+  }
+
 protected:
   /// The constructor takes a unique namespace for this dialect as well as the
   /// context to bind to.
diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td
index 70a5236d885f75107c1cba5306500852104f386f..3ea9bb41518eeedb770d00826cc42339a1c48bd8 100644
--- a/mlir/include/mlir/IR/OpBase.td
+++ b/mlir/include/mlir/IR/OpBase.td
@@ -272,6 +272,9 @@ class Dialect {
 
   // If this dialect overrides the hook for verifying region result attributes.
   bit hasRegionResultAttrVerify = 0;
+
+  // If this dialect overrides the hook for op interface fallback.
+  bit hasOperationInterfaceFallback = 0;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1142,7 +1145,9 @@ class BitEnumAttrCase<string sym, int val, string str = sym> :
 }
 
 // Additional information for an enum attribute.
-class EnumAttrInfo<string name, list<EnumAttrCaseInfo> cases> {
+class EnumAttrInfo<
+    string name, list<EnumAttrCaseInfo> cases, Attr baseClass> :
+      Attr<baseClass.predicate, baseClass.summary> {
   // The C++ enum class name
   string className = name;
 
@@ -1188,6 +1193,28 @@ class EnumAttrInfo<string name, list<EnumAttrCaseInfo> cases> {
   // static constexpr unsigned <fn-name>();
   // ```
   string maxEnumValFnName = "getMaxEnumValFor" # name;
+
+  // Generate specialized Attribute class
+  bit genSpecializedAttr = 1;
+  // The underlying Attribute class, which holds the enum value
+  Attr baseAttrClass = baseClass;
+  // The name of specialized Enum Attribute class
+  string specializedAttrClassName = name # Attr;
+
+  // Override Attr class fields for specialized class
+  let predicate = !if(genSpecializedAttr,
+    CPred<"$_self.isa<" # cppNamespace # "::" # specializedAttrClassName # ">()">,
+    baseAttrClass.predicate);
+  let storageType = !if(genSpecializedAttr,
+    cppNamespace # "::" # specializedAttrClassName,
+    baseAttrClass.storageType);
+  let returnType = !if(genSpecializedAttr,
+    cppNamespace # "::" # className,
+    baseAttrClass.returnType);
+  let constBuilderCall = !if(genSpecializedAttr,
+    cppNamespace # "::" # specializedAttrClassName # "::get($_builder.getContext(), $0)",
+    baseAttrClass.constBuilderCall);
+  let valueType = baseAttrClass.valueType;
 }
 
 // An enum attribute backed by StringAttr.
@@ -1195,47 +1222,44 @@ class EnumAttrInfo<string name, list<EnumAttrCaseInfo> cases> {
 // Op attributes of this kind are stored as StringAttr. Extra verification will
 // be generated on the string though: only the symbols of the allowed cases are
 // permitted as the string value.
-class StrEnumAttr<string name, string summary, list<StrEnumAttrCase> cases>
-  : EnumAttrInfo<name, cases>,
+class StrEnumAttr<string name, string summary, list<StrEnumAttrCase> cases> :
+  EnumAttrInfo<name, cases,
     StringBasedAttr<
       And<[StrAttr.predicate, Or<!foreach(case, cases, case.predicate)>]>,
       !if(!empty(summary), "allowed string cases: " #
           !interleave(!foreach(case, cases, "'" # case.symbol # "'"), ", "),
-          summary)>;
+          summary)>> {
+  // Disable specialized Attribute class for `StringAttr` backend by default.
+  let genSpecializedAttr = 0;
+}
 
 // An enum attribute backed by IntegerAttr.
 //
 // Op attributes of this kind are stored as IntegerAttr. Extra verification will
 // be generated on the integer though: only the values of the allowed cases are
 // permitted as the integer value.
-class IntEnumAttr<I intType, string name, string summary,
-                  list<IntEnumAttrCaseBase> cases> :
-    EnumAttrInfo<name, cases>,
-    SignlessIntegerAttrBase<intType,
-      !if(!empty(summary), "allowed " # intType.summary # " cases: " #
-          !interleave(!foreach(case, cases, case.value), ", "), summary)> {
+class IntEnumAttrBase<I intType, list<IntEnumAttrCaseBase> cases, string summary> :
+    SignlessIntegerAttrBase<intType, summary> {
   let predicate = And<[
-    SignlessIntegerAttrBase<intType, "">.predicate,
+    SignlessIntegerAttrBase<intType, summary>.predicate,
     Or<!foreach(case, cases, case.predicate)>]>;
 }
 
-class I32EnumAttr<string name, string summary,
-                  list<I32EnumAttrCase> cases> :
+class IntEnumAttr<I intType, string name, string summary,
+                  list<IntEnumAttrCaseBase> cases> :
+  EnumAttrInfo<name, cases,
+    IntEnumAttrBase<intType, cases,
+      !if(!empty(summary), "allowed " # intType.summary # " cases: " #
+          !interleave(!foreach(case, cases, case.value), ", "),
+          summary)>>;
+
+class I32EnumAttr<string name, string summary, list<I32EnumAttrCase> cases> :
     IntEnumAttr<I32, name, summary, cases> {
-  let returnType = cppNamespace # "::" # name;
   let underlyingType = "uint32_t";
-  let convertFromStorage = "static_cast<" # returnType # ">($_self.getInt())";
-  let constBuilderCall =
-          "$_builder.getI32IntegerAttr(static_cast<int32_t>($0))";
 }
-class I64EnumAttr<string name, string summary,
-                  list<I64EnumAttrCase> cases> :
+class I64EnumAttr<string name, string summary, list<I64EnumAttrCase> cases> :
     IntEnumAttr<I64, name, summary, cases> {
-  let returnType = cppNamespace # "::" # name;
   let underlyingType = "uint64_t";
-  let convertFromStorage = "static_cast<" # returnType # ">($_self.getInt())";
-  let constBuilderCall =
-          "$_builder.getI64IntegerAttr(static_cast<int64_t>($0))";
 }
 
 // A bit enum stored with 32-bit IntegerAttr.
@@ -1244,9 +1268,8 @@ class I64EnumAttr<string name, string summary,
 // be generated on the integer to make sure only allowed bit are set. Besides,
 // helper methods are generated to parse a string separated with a specified
 // delimiter to a symbol and vice versa.
-class BitEnumAttr<string name, string summary,
-                  list<BitEnumAttrCase> cases> :
-    EnumAttrInfo<name, cases>, SignlessIntegerAttrBase<I32, summary> {
+class BitEnumAttrBase<list<BitEnumAttrCase> cases, string summary> :
+    SignlessIntegerAttrBase<I32, summary> {
   let predicate = And<[
     I32Attr.predicate,
     // Make sure we don't have unknown bit set.
@@ -1254,12 +1277,11 @@ class BitEnumAttr<string name, string summary,
           # !interleave(!foreach(case, cases, case.value # "u"), "|") #
           ")))">
   ]>;
+}
 
-  let returnType = cppNamespace # "::" # name;
+class BitEnumAttr<string name, string summary, list<BitEnumAttrCase> cases> :
+    EnumAttrInfo<name, cases, BitEnumAttrBase<cases, summary>> {
   let underlyingType = "uint32_t";
-  let convertFromStorage = "static_cast<" # returnType # ">($_self.getInt())";
-  let constBuilderCall =
-          "$_builder.getI32IntegerAttr(static_cast<int32_t>($0))";
 
   // We need to return a string because we may concatenate symbols for multiple
   // bits together.
@@ -1439,7 +1461,7 @@ class TypedArrayAttrBase<Attr element, string summary>: ArrayAttrBase<
       CPred<"$_self.isa<::mlir::ArrayAttr>()">,
       // Guarantee all elements satisfy the constraints from `element`
       Concat<"::llvm::all_of($_self.cast<::mlir::ArrayAttr>(), "
-                            "[](::mlir::Attribute attr) { return ",
+                            "[&](::mlir::Attribute attr) { return ",
                                SubstLeaves<"$_self", "attr", element.predicate>,
                             "; })">]>,
     summary> {
@@ -1805,10 +1827,16 @@ def ElementwiseMappable {
   ];
 }
 
+// Op's regions have a single block.
+def SingleBlock : NativeOpTrait<"SingleBlock">;
+
 // Op's regions have a single block with the specified terminator.
 class SingleBlockImplicitTerminator<string op>
     : ParamNativeOpTrait<"SingleBlockImplicitTerminator", op>;
 
+// Op's regions don't have terminator.
+def NoTerminator : NativeOpTrait<"NoTerminator">;
+
 // Op's parent operation is the provided one.
 class HasParent<string op>
     : ParamNativeOpTrait<"HasParent", op>;
@@ -2122,11 +2150,12 @@ class Op<Dialect dialect, string mnemonic, list<OpTrait> props = []> {
   code verifier = ?;
 
   // Whether this op has associated canonicalization patterns.
-  // TODO: figure out a better way to write canonicalization patterns in
-  // TableGen rules directly instead of using this marker and C++
-  // implementations.
   bit hasCanonicalizer = 0;
 
+  // Whether this op has a static "canonicalize" method to perform "match and
+  // rewrite patterns".
+  bit hasCanonicalizeMethod = 0;
+
   // Whether this op has a folder.
   bit hasFolder = 0;
 
@@ -2139,6 +2168,35 @@ class Op<Dialect dialect, string mnemonic, list<OpTrait> props = []> {
   code extraClassDeclaration = ?;
 }
 
+// Base class for ops with static/dynamic offset, sizes and strides
+// attributes/arguments.
+class BaseOpWithOffsetSizesAndStrides<Dialect dialect, string mnemonic,
+                                      list<OpTrait> traits = []> :
+  Op<dialect, mnemonic, traits> {
+
+  // For every such op, there needs to be a:
+  //   * void print(OpAsmPrinter &p, ${C++ class of Op} op)
+  //   * LogicalResult verify(${C++ class of Op} op)
+  //   * ParseResult parse${C++ class of Op}(OpAsmParser &parser,
+  //                                         OperationState &result)
+  // functions.
+  let printer = [{ return ::print(p, *this); }];
+  let verifier = [{ return ::verify(*this); }];
+  let parser = [{ return ::parse$cppClass(parser, result); }];
+
+  code extraBaseClassDeclaration = [{
+    /// Returns the dynamic sizes for this subview operation if specified.
+    operand_range getDynamicSizes() { return sizes(); }
+
+    /// Return the list of Range (i.e. offset, size, stride). Each
+    /// Range entry contains either the dynamic value or a ConstantIndexOp
+    /// constructed with `b` at location `loc`.
+    SmallVector<Range, 8> getOrCreateRanges(OpBuilder &b, Location loc) {
+      return mlir::getOrCreateRanges(*this, b, loc);
+    }
+  }];
+}
+
 // The arguments of an op.
 class Arguments<dag args> {
   dag arguments = args;
@@ -2673,6 +2731,8 @@ class TypeDef<Dialect dialect, string name,
 class AttrOrTypeParameter<string type, string desc> {
   // Custom memory allocation code for storage constructor.
   code allocator = ?;
+  // Custom comparator used to compare two instances for equality.
+  code comparator = ?;
   // The C++ type of this parameter.
   string cppType = type;
   // One-line human-readable description of the argument.
@@ -2689,6 +2749,12 @@ class StringRefParameter<string desc = ""> :
   let allocator = [{$_dst = $_allocator.copyInto($_self);}];
 }
 
+// For APFloats, which require comparison.
+class APFloatParameter<string desc> :
+    AttrOrTypeParameter<"::llvm::APFloat", desc> {
+  let comparator = "$_lhs.bitwiseIsEqual($_rhs)";
+}
+
 // For standard ArrayRefs, which require allocation.
 class ArrayRefParameter<string arrayOf, string desc = ""> :
     AttrOrTypeParameter<"::llvm::ArrayRef<" # arrayOf # ">", desc> {
@@ -2715,7 +2781,8 @@ class ArrayRefOfSelfAllocationParameter<string arrayOf, string desc> :
 // This is a special parameter used for AttrDefs that represents a `mlir::Type`
 // that is also used as the value `Type` of the attribute. Only one parameter
 // of the attribute may be of this type.
-class AttributeSelfTypeParameter<string desc> :
-    AttrOrTypeParameter<"::mlir::Type", desc> {}
+class AttributeSelfTypeParameter<string desc,
+                                 string derivedType = "::mlir::Type"> :
+    AttrOrTypeParameter<derivedType, desc> {}
 
 #endif // OP_BASE
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index 8b3b052590a97c2408cb9a6ff7abd8121f1053dc..b488dc12a4818ffaf4e7c5b7029cf31bf9db99fe 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -19,6 +19,7 @@
 #ifndef MLIR_IR_OPDEFINITION_H
 #define MLIR_IR_OPDEFINITION_H
 
+#include "mlir/IR/Dialect.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 
@@ -122,27 +123,6 @@ public:
   /// The source location the operation was defined or derived from.
   Location getLoc() { return state->getLoc(); }
 
-  /// Return all of the attributes on this operation.
-  LLVM_ATTRIBUTE_DEPRECATED(
-      ArrayRef<NamedAttribute> getAttrs(),
-      "Use Operation::getAttrs() instead (replace '.' with '->').") {
-    return state->getAttrs();
-  }
-
-  /// Remove the attribute with the specified name if it exists. Return the
-  /// attribute that was erased, or nullptr if there was no attribute with such
-  /// name.
-  LLVM_ATTRIBUTE_DEPRECATED(
-      Attribute removeAttr(Identifier name),
-      "Use Operation::removeAttr() instead (replace '.' with '->').") {
-    return state->removeAttr(name);
-  }
-  LLVM_ATTRIBUTE_DEPRECATED(
-      Attribute removeAttr(StringRef name),
-      "Use Operation::removeAttr() instead (replace '.' with '->').") {
-    return state->removeAttr(Identifier::get(name, getContext()));
-  }
-
   /// Return true if there are no users of any results of this operation.
   bool use_empty() { return state->use_empty(); }
 
@@ -185,7 +165,7 @@ public:
 public:
   /// This hook returns any canonicalization pattern rewrites that the operation
   /// supports, for use by the canonicalization pass.
-  static void getCanonicalizationPatterns(OwningRewritePatternList &results,
+  static void getCanonicalizationPatterns(RewritePatternSet &results,
                                           MLIRContext *context) {}
 
 protected:
@@ -674,6 +654,11 @@ class VariadicResults
 //===----------------------------------------------------------------------===//
 // Terminator Traits
 
+/// This class indicates that the regions associated with this op don't have
+/// terminators.
+template <typename ConcreteType>
+class NoTerminator : public TraitBase<ConcreteType, NoTerminator> {};
+
 /// This class provides the API for ops that are known to be terminators.
 template <typename ConcreteType>
 class IsTerminator : public TraitBase<ConcreteType, IsTerminator> {
@@ -777,6 +762,87 @@ class VariadicSuccessors
     : public detail::MultiSuccessorTraitBase<ConcreteType, VariadicSuccessors> {
 };
 
+//===----------------------------------------------------------------------===//
+// SingleBlock
+
+/// This class provides APIs and verifiers for ops with regions having a single
+/// block.
+template <typename ConcreteType>
+struct SingleBlock : public TraitBase<ConcreteType, SingleBlock> {
+public:
+  static LogicalResult verifyTrait(Operation *op) {
+    for (unsigned i = 0, e = op->getNumRegions(); i < e; ++i) {
+      Region &region = op->getRegion(i);
+
+      // Empty regions are fine.
+      if (region.empty())
+        continue;
+
+      // Non-empty regions must contain a single basic block.
+      if (!llvm::hasSingleElement(region))
+        return op->emitOpError("expects region #")
+               << i << " to have 0 or 1 blocks";
+
+      if (!ConcreteType::template hasTrait<NoTerminator>()) {
+        Block &block = region.front();
+        if (block.empty())
+          return op->emitOpError() << "expects a non-empty block";
+      }
+    }
+    return success();
+  }
+
+  Block *getBody(unsigned idx = 0) {
+    Region &region = this->getOperation()->getRegion(idx);
+    assert(!region.empty() && "unexpected empty region");
+    return &region.front();
+  }
+  Region &getBodyRegion(unsigned idx = 0) {
+    return this->getOperation()->getRegion(idx);
+  }
+
+  //===------------------------------------------------------------------===//
+  // Single Region Utilities
+  //===------------------------------------------------------------------===//
+
+  /// The following are a set of methods only enabled when the parent
+  /// operation has a single region. Each of these methods take an additional
+  /// template parameter that represents the concrete operation so that we
+  /// can use SFINAE to disable the methods for non-single region operations.
+  template <typename OpT, typename T = void>
+  using enable_if_single_region =
+      typename std::enable_if_t<OpT::template hasTrait<OneRegion>(), T>;
+
+  template <typename OpT = ConcreteType>
+  enable_if_single_region<OpT, Block::iterator> begin() {
+    return getBody()->begin();
+  }
+  template <typename OpT = ConcreteType>
+  enable_if_single_region<OpT, Block::iterator> end() {
+    return getBody()->end();
+  }
+  template <typename OpT = ConcreteType>
+  enable_if_single_region<OpT, Operation &> front() {
+    return *begin();
+  }
+
+  /// Insert the operation into the back of the body.
+  template <typename OpT = ConcreteType>
+  enable_if_single_region<OpT> push_back(Operation *op) {
+    insert(Block::iterator(getBody()->end()), op);
+  }
+
+  /// Insert the operation at the given insertion point.
+  template <typename OpT = ConcreteType>
+  enable_if_single_region<OpT> insert(Operation *insertPt, Operation *op) {
+    insert(Block::iterator(insertPt), op);
+  }
+  template <typename OpT = ConcreteType>
+  enable_if_single_region<OpT> insert(Block::iterator insertPt, Operation *op) {
+    getBody()->getOperations().insert(insertPt, op);
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // SingleBlockImplicitTerminator
 
@@ -785,8 +851,9 @@ class VariadicSuccessors
 template <typename TerminatorOpType>
 struct SingleBlockImplicitTerminator {
   template <typename ConcreteType>
-  class Impl : public TraitBase<ConcreteType, Impl> {
+  class Impl : public SingleBlock<ConcreteType> {
   private:
+    using Base = SingleBlock<ConcreteType>;
     /// Builds a terminator operation without relying on OpBuilder APIs to avoid
     /// cyclic header inclusion.
     static Operation *buildTerminator(OpBuilder &builder, Location loc) {
@@ -800,22 +867,14 @@ struct SingleBlockImplicitTerminator {
     using ImplicitTerminatorOpT = TerminatorOpType;
 
     static LogicalResult verifyTrait(Operation *op) {
+      if (failed(Base::verifyTrait(op)))
+        return failure();
       for (unsigned i = 0, e = op->getNumRegions(); i < e; ++i) {
         Region &region = op->getRegion(i);
-
         // Empty regions are fine.
         if (region.empty())
           continue;
-
-        // Non-empty regions must contain a single basic block.
-        if (std::next(region.begin()) != region.end())
-          return op->emitOpError("expects region #")
-                 << i << " to have 0 or 1 blocks";
-
-        Block &block = region.front();
-        if (block.empty())
-          return op->emitOpError() << "expects a non-empty block";
-        Operation &terminator = block.back();
+        Operation &terminator = region.front().back();
         if (isa<TerminatorOpType>(terminator))
           continue;
 
@@ -848,40 +907,15 @@ struct SingleBlockImplicitTerminator {
                                            buildTerminator);
     }
 
-    Block *getBody(unsigned idx = 0) {
-      Region &region = this->getOperation()->getRegion(idx);
-      assert(!region.empty() && "unexpected empty region");
-      return &region.front();
-    }
-    Region &getBodyRegion(unsigned idx = 0) {
-      return this->getOperation()->getRegion(idx);
-    }
-
     //===------------------------------------------------------------------===//
     // Single Region Utilities
     //===------------------------------------------------------------------===//
+    using Base::getBody;
 
-    /// The following are a set of methods only enabled when the parent
-    /// operation has a single region. Each of these methods take an additional
-    /// template parameter that represents the concrete operation so that we
-    /// can use SFINAE to disable the methods for non-single region operations.
     template <typename OpT, typename T = void>
     using enable_if_single_region =
         typename std::enable_if_t<OpT::template hasTrait<OneRegion>(), T>;
 
-    template <typename OpT = ConcreteType>
-    enable_if_single_region<OpT, Block::iterator> begin() {
-      return getBody()->begin();
-    }
-    template <typename OpT = ConcreteType>
-    enable_if_single_region<OpT, Block::iterator> end() {
-      return getBody()->end();
-    }
-    template <typename OpT = ConcreteType>
-    enable_if_single_region<OpT, Operation &> front() {
-      return *begin();
-    }
-
     /// Insert the operation into the back of the body, before the terminator.
     template <typename OpT = ConcreteType>
     enable_if_single_region<OpT> push_back(Operation *op) {
@@ -906,6 +940,27 @@ struct SingleBlockImplicitTerminator {
   };
 };
 
+/// Check is an op defines the `ImplicitTerminatorOpT` member. This is intended
+/// to be used with `llvm::is_detected`.
+template <class T>
+using has_implicit_terminator_t = typename T::ImplicitTerminatorOpT;
+
+/// Support to check if an operation has the SingleBlockImplicitTerminator
+/// trait. We can't just use `hasTrait` because this class is templated on a
+/// specific terminator op.
+template <class Op, bool hasTerminator =
+                        llvm::is_detected<has_implicit_terminator_t, Op>::value>
+struct hasSingleBlockImplicitTerminator {
+  static constexpr bool value = std::is_base_of<
+      typename OpTrait::SingleBlockImplicitTerminator<
+          typename Op::ImplicitTerminatorOpT>::template Impl<Op>,
+      Op>::value;
+};
+template <class Op>
+struct hasSingleBlockImplicitTerminator<Op, false> {
+  static constexpr bool value = false;
+};
+
 //===----------------------------------------------------------------------===//
 // Misc Traits
 
@@ -1537,6 +1592,13 @@ public:
 #endif
     return false;
   }
+  /// Provide `classof` support for other OpBase derived classes, such as
+  /// Interfaces.
+  template <typename T>
+  static std::enable_if_t<std::is_base_of<OpState, T>::value, bool>
+  classof(const T *op) {
+    return classof(const_cast<T *>(op)->getOperation());
+  }
 
   /// Expose the type we are instantiated on to template machinery that may want
   /// to introspect traits on this operation.
@@ -1735,7 +1797,20 @@ protected:
   static typename InterfaceBase::Concept *getInterfaceFor(Operation *op) {
     // Access the raw interface from the abstract operation.
     auto *abstractOp = op->getAbstractOperation();
-    return abstractOp ? abstractOp->getInterface<ConcreteType>() : nullptr;
+    if (abstractOp) {
+      if (auto *opIface = abstractOp->getInterface<ConcreteType>())
+        return opIface;
+      // Fallback to the dialect to provide it with a chance to implement this
+      // interface for this operation.
+      return abstractOp->dialect.getRegisteredInterfaceForOp<ConcreteType>(
+          op->getName());
+    }
+    // Fallback to the dialect to provide it with a chance to implement this
+    // interface for this operation.
+    Dialect *dialect = op->getName().getDialect();
+    return dialect ? dialect->getRegisteredInterfaceForOp<ConcreteType>(
+                         op->getName())
+                   : nullptr;
   }
 
   /// Allow access to `getInterfaceFor`.
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index 99561c3a089beb3593e1945ba5cdbc10d78bf6a3..5250899a9b447850244e4ca87d39c3430fc4f58a 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -92,8 +92,13 @@ public:
   virtual void printGenericOp(Operation *op) = 0;
 
   /// Prints a region.
+  /// If 'printEntryBlockArgs' is false, the arguments of the
+  /// block are not printed. If 'printBlockTerminator' is false, the terminator
+  /// operation of the block is not printed. If printEmptyBlock is true, then
+  /// the block header is printed even if the block is empty.
   virtual void printRegion(Region &blocks, bool printEntryBlockArgs = true,
-                           bool printBlockTerminators = true) = 0;
+                           bool printBlockTerminators = true,
+                           bool printEmptyBlock = false) = 0;
 
   /// Renumber the arguments for the specified region to the same names as the
   /// SSA values in namesToUse.  This may only be used for IsolatedFromAbove
diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h
index 60af4b09e0e107eda59f08bb7d829f46809d253f..cb82ec9c471491b7184b7e9fe6597c43e334bf2c 100644
--- a/mlir/include/mlir/IR/OperationSupport.h
+++ b/mlir/include/mlir/IR/OperationSupport.h
@@ -51,9 +51,11 @@ class RewritePattern;
 class Type;
 class Value;
 class ValueRange;
-template <typename ValueRangeT> class ValueTypeRange;
+template <typename ValueRangeT>
+class ValueTypeRange;
 
-class OwningRewritePatternList;
+class RewritePatternSet;
+using OwningRewritePatternList = RewritePatternSet;
 
 //===----------------------------------------------------------------------===//
 // AbstractOperation
@@ -65,7 +67,7 @@ class OwningRewritePatternList;
 /// the concrete operation types.
 class AbstractOperation {
 public:
-  using GetCanonicalizationPatternsFn = void (*)(OwningRewritePatternList &,
+  using GetCanonicalizationPatternsFn = void (*)(RewritePatternSet &,
                                                  MLIRContext *);
   using FoldHookFn = LogicalResult (*)(Operation *, ArrayRef<Attribute>,
                                        SmallVectorImpl<OpFoldResult> &);
@@ -86,6 +88,9 @@ public:
   /// Use the specified object to parse this ops custom assembly format.
   ParseResult parseAssembly(OpAsmParser &parser, OperationState &result) const;
 
+  /// Return the static hook for parsing this operation assembly.
+  ParseAssemblyFn getParseAssemblyFn() const { return parseAssemblyFn; }
+
   /// This hook implements the AsmPrinter for this operation.
   void printAssembly(Operation *op, OpAsmPrinter &p) const {
     return printAssemblyFn(op, p);
@@ -124,7 +129,7 @@ public:
 
   /// This hook returns any canonicalization pattern rewrites that the operation
   /// supports, for use by the canonicalization pass.
-  void getCanonicalizationPatterns(OwningRewritePatternList &results,
+  void getCanonicalizationPatterns(RewritePatternSet &results,
                                    MLIRContext *context) const {
     return getCanonicalizationPatternsFn(results, context);
   }
@@ -132,15 +137,25 @@ public:
   /// Returns an instance of the concept object for the given interface if it
   /// was registered to this operation, null otherwise. This should not be used
   /// directly.
-  template <typename T> typename T::Concept *getInterface() const {
+  template <typename T>
+  typename T::Concept *getInterface() const {
     return interfaceMap.lookup<T>();
   }
 
+  /// Returns true if this operation has the given interface registered to it.
+  bool hasInterface(TypeID interfaceID) const {
+    return interfaceMap.contains(interfaceID);
+  }
+
   /// Returns true if the operation has a particular trait.
-  template <template <typename T> class Trait> bool hasTrait() const {
+  template <template <typename T> class Trait>
+  bool hasTrait() const {
     return hasTraitFn(TypeID::get<Trait>());
   }
 
+  /// Returns true if the operation has a particular trait.
+  bool hasTrait(TypeID traitID) const { return hasTraitFn(traitID); }
+
   /// Look up the specified operation in the specified MLIRContext and return a
   /// pointer to it if present.  Otherwise, return a null pointer.
   static const AbstractOperation *lookup(StringRef opName,
@@ -148,7 +163,8 @@ public:
 
   /// This constructor is used by Dialect objects when they register the list of
   /// operations they contain.
-  template <typename T> static void insert(Dialect &dialect) {
+  template <typename T>
+  static void insert(Dialect &dialect) {
     insert(T::getOperationName(), dialect, TypeID::get<T>(),
            T::getParseAssemblyFn(), T::getPrintAssemblyFn(),
            T::getVerifyInvariantsFn(), T::getFoldHookFn(),
@@ -220,7 +236,8 @@ public:
   void append(NamedAttribute attr) { push_back(attr); }
 
   /// Add an array of named attributes.
-  template <typename RangeT> void append(RangeT &&newAttributes) {
+  template <typename RangeT>
+  void append(RangeT &&newAttributes) {
     append(std::begin(newAttributes), std::end(newAttributes));
   }
 
@@ -851,7 +868,8 @@ LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
 
 namespace llvm {
 // Identifiers hash just like pointers, there is no need to hash the bytes.
-template <> struct DenseMapInfo<mlir::OperationName> {
+template <>
+struct DenseMapInfo<mlir::OperationName> {
   static mlir::OperationName getEmptyKey() {
     auto pointer = llvm::DenseMapInfo<void *>::getEmptyKey();
     return mlir::OperationName::getFromOpaquePointer(pointer);
@@ -871,7 +889,8 @@ template <> struct DenseMapInfo<mlir::OperationName> {
 /// The pointer inside of an identifier comes from a StringMap, so its alignment
 /// is always at least 4 and probably 8 (on 64-bit machines).  Allow LLVM to
 /// steal the low bits.
-template <> struct PointerLikeTypeTraits<mlir::OperationName> {
+template <>
+struct PointerLikeTypeTraits<mlir::OperationName> {
 public:
   static inline void *getAsVoidPointer(mlir::OperationName I) {
     return const_cast<void *>(I.getAsOpaquePointer());
diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h
index 8e1a5b98c31868e513fd4c48d483c98d19278b5b..5ee9418efa38fb729ea2917cb110c22dbd14faa9 100644
--- a/mlir/include/mlir/IR/PatternMatch.h
+++ b/mlir/include/mlir/IR/PatternMatch.h
@@ -68,6 +68,19 @@ private:
 /// used to interface with the metadata of a pattern, such as the benefit or
 /// root operation.
 class Pattern {
+  /// This enum represents the kind of value used to select the root operations
+  /// that match this pattern.
+  enum class RootKind {
+    /// The pattern root matches "any" operation.
+    Any,
+    /// The pattern root is matched using a concrete operation name.
+    OperationName,
+    /// The pattern root is matched using an interface ID.
+    InterfaceID,
+    /// The patter root is matched using a trait ID.
+    TraitID
+  };
+
 public:
   /// Return a list of operations that may be generated when rewriting an
   /// operation instance with this pattern.
@@ -75,7 +88,29 @@ public:
 
   /// Return the root node that this pattern matches. Patterns that can match
   /// multiple root types return None.
-  Optional<OperationName> getRootKind() const { return rootKind; }
+  Optional<OperationName> getRootKind() const {
+    if (rootKind == RootKind::OperationName)
+      return OperationName::getFromOpaquePointer(rootValue);
+    return llvm::None;
+  }
+
+  /// Return the interface ID used to match the root operation of this pattern.
+  /// If the pattern does not use an interface ID for deciding the root match,
+  /// this returns None.
+  Optional<TypeID> getRootInterfaceID() const {
+    if (rootKind == RootKind::InterfaceID)
+      return TypeID::getFromOpaquePointer(rootValue);
+    return llvm::None;
+  }
+
+  /// Return the trait ID used to match the root operation of this pattern.
+  /// If the pattern does not use a trait ID for deciding the root match, this
+  /// returns None.
+  Optional<TypeID> getRootTraitID() const {
+    if (rootKind == RootKind::TraitID)
+      return TypeID::getFromOpaquePointer(rootValue);
+    return llvm::None;
+  }
 
   /// Return the benefit (the inverse of "cost") of matching this pattern.  The
   /// benefit of a Pattern is always static - rewrites that may have dynamic
@@ -88,56 +123,85 @@ public:
   /// i.e. this pattern may generate IR that also matches this pattern, but is
   /// known to bound the recursion. This signals to a rewrite driver that it is
   /// safe to apply this pattern recursively to generated IR.
-  bool hasBoundedRewriteRecursion() const { return hasBoundedRecursion; }
+  bool hasBoundedRewriteRecursion() const {
+    return contextAndHasBoundedRecursion.getInt();
+  }
+
+  /// Return the MLIRContext used to create this pattern.
+  MLIRContext *getContext() const {
+    return contextAndHasBoundedRecursion.getPointer();
+  }
 
 protected:
   /// This class acts as a special tag that makes the desire to match "any"
   /// operation type explicit. This helps to avoid unnecessary usages of this
   /// feature, and ensures that the user is making a conscious decision.
   struct MatchAnyOpTypeTag {};
+  /// This class acts as a special tag that makes the desire to match any
+  /// operation that implements a given interface explicit. This helps to avoid
+  /// unnecessary usages of this feature, and ensures that the user is making a
+  /// conscious decision.
+  struct MatchInterfaceOpTypeTag {};
+  /// This class acts as a special tag that makes the desire to match any
+  /// operation that implements a given trait explicit. This helps to avoid
+  /// unnecessary usages of this feature, and ensures that the user is making a
+  /// conscious decision.
+  struct MatchTraitOpTypeTag {};
 
   /// Construct a pattern with a certain benefit that matches the operation
   /// with the given root name.
-  Pattern(StringRef rootName, PatternBenefit benefit, MLIRContext *context);
-  /// Construct a pattern with a certain benefit that matches any operation
-  /// type. `MatchAnyOpTypeTag` is just a tag to ensure that the "match any"
-  /// behavior is what the user actually desired, `MatchAnyOpTypeTag()` should
-  /// always be supplied here.
-  Pattern(PatternBenefit benefit, MatchAnyOpTypeTag tag);
-  /// Construct a pattern with a certain benefit that matches the operation with
-  /// the given root name. `generatedNames` contains the names of operations
-  /// that may be generated during a successful rewrite.
-  Pattern(StringRef rootName, ArrayRef<StringRef> generatedNames,
-          PatternBenefit benefit, MLIRContext *context);
+  Pattern(StringRef rootName, PatternBenefit benefit, MLIRContext *context,
+          ArrayRef<StringRef> generatedNames = {});
   /// Construct a pattern that may match any operation type. `generatedNames`
   /// contains the names of operations that may be generated during a successful
   /// rewrite. `MatchAnyOpTypeTag` is just a tag to ensure that the "match any"
   /// behavior is what the user actually desired, `MatchAnyOpTypeTag()` should
   /// always be supplied here.
-  Pattern(ArrayRef<StringRef> generatedNames, PatternBenefit benefit,
-          MLIRContext *context, MatchAnyOpTypeTag tag);
+  Pattern(MatchAnyOpTypeTag tag, PatternBenefit benefit, MLIRContext *context,
+          ArrayRef<StringRef> generatedNames = {});
+  /// Construct a pattern that may match any operation that implements the
+  /// interface defined by the provided `interfaceID`. `generatedNames` contains
+  /// the names of operations that may be generated during a successful rewrite.
+  /// `MatchInterfaceOpTypeTag` is just a tag to ensure that the "match
+  /// interface" behavior is what the user actually desired,
+  /// `MatchInterfaceOpTypeTag()` should always be supplied here.
+  Pattern(MatchInterfaceOpTypeTag tag, TypeID interfaceID,
+          PatternBenefit benefit, MLIRContext *context,
+          ArrayRef<StringRef> generatedNames = {});
+  /// Construct a pattern that may match any operation that implements the
+  /// trait defined by the provided `traitID`. `generatedNames` contains the
+  /// names of operations that may be generated during a successful rewrite.
+  /// `MatchTraitOpTypeTag` is just a tag to ensure that the "match trait"
+  /// behavior is what the user actually desired, `MatchTraitOpTypeTag()` should
+  /// always be supplied here.
+  Pattern(MatchTraitOpTypeTag tag, TypeID traitID, PatternBenefit benefit,
+          MLIRContext *context, ArrayRef<StringRef> generatedNames = {});
 
   /// Set the flag detailing if this pattern has bounded rewrite recursion or
   /// not.
   void setHasBoundedRewriteRecursion(bool hasBoundedRecursionArg = true) {
-    hasBoundedRecursion = hasBoundedRecursionArg;
+    contextAndHasBoundedRecursion.setInt(hasBoundedRecursionArg);
   }
 
 private:
-  /// A list of the potential operations that may be generated when rewriting
-  /// an op with this pattern.
-  SmallVector<OperationName, 2> generatedOps;
+  Pattern(const void *rootValue, RootKind rootKind,
+          ArrayRef<StringRef> generatedNames, PatternBenefit benefit,
+          MLIRContext *context);
 
-  /// The root operation of the pattern. If the pattern matches a specific
-  /// operation, this contains the name of that operation. Contains None
-  /// otherwise.
-  Optional<OperationName> rootKind;
+  /// The value used to match the root operation of the pattern.
+  const void *rootValue;
+  RootKind rootKind;
 
   /// The expected benefit of matching this pattern.
   const PatternBenefit benefit;
 
-  /// A boolean flag of whether this pattern has bounded recursion or not.
-  bool hasBoundedRecursion = false;
+  /// The context this pattern was created from, and a boolean flag indicating
+  /// whether this pattern has bounded recursion or not.
+  llvm::PointerIntPair<MLIRContext *, 1, bool> contextAndHasBoundedRecursion;
+
+  /// A list of the potential operations that may be generated when rewriting
+  /// an op with this pattern.
+  SmallVector<OperationName, 2> generatedOps;
 };
 
 //===----------------------------------------------------------------------===//
@@ -188,15 +252,13 @@ protected:
   virtual void anchor();
 };
 
-/// OpRewritePattern is a wrapper around RewritePattern that allows for
-/// matching and rewriting against an instance of a derived operation class as
-/// opposed to a raw Operation.
+namespace detail {
+/// OpOrInterfaceRewritePatternBase is a wrapper around RewritePattern that
+/// allows for matching and rewriting against an instance of a derived operation
+/// class or Interface.
 template <typename SourceOp>
-struct OpRewritePattern : public RewritePattern {
-  /// Patterns must specify the root operation name they match against, and can
-  /// also specify the benefit of the pattern matching.
-  OpRewritePattern(MLIRContext *context, PatternBenefit benefit = 1)
-      : RewritePattern(SourceOp::getOperationName(), benefit, context) {}
+struct OpOrInterfaceRewritePatternBase : public RewritePattern {
+  using RewritePattern::RewritePattern;
 
   /// Wrappers around the RewritePattern methods that pass the derived op type.
   void rewrite(Operation *op, PatternRewriter &rewriter) const final {
@@ -227,6 +289,43 @@ struct OpRewritePattern : public RewritePattern {
     return failure();
   }
 };
+} // namespace detail
+
+/// OpRewritePattern is a wrapper around RewritePattern that allows for
+/// matching and rewriting against an instance of a derived operation class as
+/// opposed to a raw Operation.
+template <typename SourceOp>
+struct OpRewritePattern
+    : public detail::OpOrInterfaceRewritePatternBase<SourceOp> {
+  /// Patterns must specify the root operation name they match against, and can
+  /// also specify the benefit of the pattern matching.
+  OpRewritePattern(MLIRContext *context, PatternBenefit benefit = 1)
+      : detail::OpOrInterfaceRewritePatternBase<SourceOp>(
+            SourceOp::getOperationName(), benefit, context) {}
+};
+
+/// OpInterfaceRewritePattern is a wrapper around RewritePattern that allows for
+/// matching and rewriting against an instance of an operation interface instead
+/// of a raw Operation.
+template <typename SourceOp>
+struct OpInterfaceRewritePattern
+    : public detail::OpOrInterfaceRewritePatternBase<SourceOp> {
+  OpInterfaceRewritePattern(MLIRContext *context, PatternBenefit benefit = 1)
+      : detail::OpOrInterfaceRewritePatternBase<SourceOp>(
+            Pattern::MatchInterfaceOpTypeTag(), SourceOp::getInterfaceID(),
+            benefit, context) {}
+};
+
+/// OpTraitRewritePattern is a wrapper around RewritePattern that allows for
+/// matching and rewriting against instances of an operation that possess a
+/// given trait.
+template <template <typename> class TraitType>
+class OpTraitRewritePattern : public RewritePattern {
+public:
+  OpTraitRewritePattern(MLIRContext *context, PatternBenefit benefit = 1)
+      : RewritePattern(Pattern::MatchTraitOpTypeTag(), TypeID::get<TraitType>(),
+                       benefit, context) {}
+};
 
 //===----------------------------------------------------------------------===//
 // PDLPatternModule
@@ -238,63 +337,96 @@ struct OpRewritePattern : public RewritePattern {
 /// Storage type of byte-code interpreter values. These are passed to constraint
 /// functions as arguments.
 class PDLValue {
-  /// The internal implementation type when the value is an Attribute,
-  /// Operation*, or Type. See `impl` below for more details.
-  using AttrOpTypeImplT = llvm::PointerUnion<Attribute, Operation *, Type>;
-
 public:
-  PDLValue(const PDLValue &other) : impl(other.impl) {}
-  PDLValue(std::nullptr_t = nullptr) : impl() {}
-  PDLValue(Attribute value) : impl(value) {}
-  PDLValue(Operation *value) : impl(value) {}
-  PDLValue(Type value) : impl(value) {}
-  PDLValue(Value value) : impl(value) {}
+  /// The underlying kind of a PDL value.
+  enum class Kind { Attribute, Operation, Type, TypeRange, Value, ValueRange };
+
+  /// Construct a new PDL value.
+  PDLValue(const PDLValue &other) = default;
+  PDLValue(std::nullptr_t = nullptr) : value(nullptr), kind(Kind::Attribute) {}
+  PDLValue(Attribute value)
+      : value(value.getAsOpaquePointer()), kind(Kind::Attribute) {}
+  PDLValue(Operation *value) : value(value), kind(Kind::Operation) {}
+  PDLValue(Type value) : value(value.getAsOpaquePointer()), kind(Kind::Type) {}
+  PDLValue(TypeRange *value) : value(value), kind(Kind::TypeRange) {}
+  PDLValue(Value value)
+      : value(value.getAsOpaquePointer()), kind(Kind::Value) {}
+  PDLValue(ValueRange *value) : value(value), kind(Kind::ValueRange) {}
 
   /// Returns true if the type of the held value is `T`.
   template <typename T>
-  std::enable_if_t<std::is_same<T, Value>::value, bool> isa() const {
-    return impl.is<Value>();
-  }
-  template <typename T>
-  std::enable_if_t<!std::is_same<T, Value>::value, bool> isa() const {
-    auto attrOpTypeImpl = impl.dyn_cast<AttrOpTypeImplT>();
-    return attrOpTypeImpl && attrOpTypeImpl.is<T>();
+  bool isa() const {
+    assert(value && "isa<> used on a null value");
+    return kind == getKindOf<T>();
   }
 
   /// Attempt to dynamically cast this value to type `T`, returns null if this
   /// value is not an instance of `T`.
-  template <typename T>
-  std::enable_if_t<std::is_same<T, Value>::value, T> dyn_cast() const {
-    return impl.dyn_cast<T>();
-  }
-  template <typename T>
-  std::enable_if_t<!std::is_same<T, Value>::value, T> dyn_cast() const {
-    auto attrOpTypeImpl = impl.dyn_cast<AttrOpTypeImplT>();
-    return attrOpTypeImpl && attrOpTypeImpl.dyn_cast<T>();
+  template <typename T,
+            typename ResultT = std::conditional_t<
+                std::is_convertible<T, bool>::value, T, Optional<T>>>
+  ResultT dyn_cast() const {
+    return isa<T>() ? castImpl<T>() : ResultT();
   }
 
   /// Cast this value to type `T`, asserts if this value is not an instance of
   /// `T`.
   template <typename T>
-  std::enable_if_t<std::is_same<T, Value>::value, T> cast() const {
-    return impl.get<T>();
-  }
-  template <typename T>
-  std::enable_if_t<!std::is_same<T, Value>::value, T> cast() const {
-    return impl.get<AttrOpTypeImplT>().get<T>();
+  T cast() const {
+    assert(isa<T>() && "expected value to be of type `T`");
+    return castImpl<T>();
   }
 
   /// Get an opaque pointer to the value.
-  void *getAsOpaquePointer() { return impl.getOpaqueValue(); }
+  const void *getAsOpaquePointer() const { return value; }
+
+  /// Return if this value is null or not.
+  explicit operator bool() const { return value; }
+
+  /// Return the kind of this value.
+  Kind getKind() const { return kind; }
 
   /// Print this value to the provided output stream.
-  void print(raw_ostream &os);
+  void print(raw_ostream &os) const;
 
 private:
-  /// The internal opaque representation of a PDLValue. We use a nested
-  /// PointerUnion structure here because `Value` only has 1 low bit
-  /// available, where as the remaining types all have 3.
-  llvm::PointerUnion<AttrOpTypeImplT, Value> impl;
+  /// Find the index of a given type in a range of other types.
+  template <typename...>
+  struct index_of_t;
+  template <typename T, typename... R>
+  struct index_of_t<T, T, R...> : std::integral_constant<size_t, 0> {};
+  template <typename T, typename F, typename... R>
+  struct index_of_t<T, F, R...>
+      : std::integral_constant<size_t, 1 + index_of_t<T, R...>::value> {};
+
+  /// Return the kind used for the given T.
+  template <typename T>
+  static Kind getKindOf() {
+    return static_cast<Kind>(index_of_t<T, Attribute, Operation *, Type,
+                                        TypeRange, Value, ValueRange>::value);
+  }
+
+  /// The internal implementation of `cast`, that returns the underlying value
+  /// as the given type `T`.
+  template <typename T>
+  std::enable_if_t<llvm::is_one_of<T, Attribute, Type, Value>::value, T>
+  castImpl() const {
+    return T::getFromOpaquePointer(value);
+  }
+  template <typename T>
+  std::enable_if_t<llvm::is_one_of<T, TypeRange, ValueRange>::value, T>
+  castImpl() const {
+    return *reinterpret_cast<T *>(const_cast<void *>(value));
+  }
+  template <typename T>
+  std::enable_if_t<std::is_pointer<T>::value, T> castImpl() const {
+    return reinterpret_cast<T>(const_cast<void *>(value));
+  }
+
+  /// The internal opaque representation of a PDLValue.
+  const void *value;
+  /// The kind of the opaque value.
+  Kind kind;
 };
 
 inline raw_ostream &operator<<(raw_ostream &os, PDLValue value) {
@@ -302,6 +434,85 @@ inline raw_ostream &operator<<(raw_ostream &os, PDLValue value) {
   return os;
 }
 
+//===----------------------------------------------------------------------===//
+// PDLResultList
+
+/// The class represents a list of PDL results, returned by a native rewrite
+/// method. It provides the mechanism with which to pass PDLValues back to the
+/// PDL bytecode.
+class PDLResultList {
+public:
+  /// Push a new Attribute value onto the result list.
+  void push_back(Attribute value) { results.push_back(value); }
+
+  /// Push a new Operation onto the result list.
+  void push_back(Operation *value) { results.push_back(value); }
+
+  /// Push a new Type onto the result list.
+  void push_back(Type value) { results.push_back(value); }
+
+  /// Push a new TypeRange onto the result list.
+  void push_back(TypeRange value) {
+    // The lifetime of a TypeRange can't be guaranteed, so we'll need to
+    // allocate a storage for it.
+    llvm::OwningArrayRef<Type> storage(value.size());
+    llvm::copy(value, storage.begin());
+    allocatedTypeRanges.emplace_back(std::move(storage));
+    typeRanges.push_back(allocatedTypeRanges.back());
+    results.push_back(&typeRanges.back());
+  }
+  void push_back(ValueTypeRange<OperandRange> value) {
+    typeRanges.push_back(value);
+    results.push_back(&typeRanges.back());
+  }
+  void push_back(ValueTypeRange<ResultRange> value) {
+    typeRanges.push_back(value);
+    results.push_back(&typeRanges.back());
+  }
+
+  /// Push a new Value onto the result list.
+  void push_back(Value value) { results.push_back(value); }
+
+  /// Push a new ValueRange onto the result list.
+  void push_back(ValueRange value) {
+    // The lifetime of a ValueRange can't be guaranteed, so we'll need to
+    // allocate a storage for it.
+    llvm::OwningArrayRef<Value> storage(value.size());
+    llvm::copy(value, storage.begin());
+    allocatedValueRanges.emplace_back(std::move(storage));
+    valueRanges.push_back(allocatedValueRanges.back());
+    results.push_back(&valueRanges.back());
+  }
+  void push_back(OperandRange value) {
+    valueRanges.push_back(value);
+    results.push_back(&valueRanges.back());
+  }
+  void push_back(ResultRange value) {
+    valueRanges.push_back(value);
+    results.push_back(&valueRanges.back());
+  }
+
+protected:
+  /// Create a new result list with the expected number of results.
+  PDLResultList(unsigned maxNumResults) {
+    // For now just reserve enough space for all of the results. We could do
+    // separate counts per range type, but it isn't really worth it unless there
+    // are a "large" number of results.
+    typeRanges.reserve(maxNumResults);
+    valueRanges.reserve(maxNumResults);
+  }
+
+  /// The PDL results held by this list.
+  SmallVector<PDLValue> results;
+  /// Memory used to store ranges held by the list.
+  SmallVector<TypeRange> typeRanges;
+  SmallVector<ValueRange> valueRanges;
+  /// Memory allocated to store ranges in the result list whose lifetime was
+  /// generated in the native function.
+  SmallVector<llvm::OwningArrayRef<Type>> allocatedTypeRanges;
+  SmallVector<llvm::OwningArrayRef<Value>> allocatedValueRanges;
+};
+
 //===----------------------------------------------------------------------===//
 // PDLPatternModule
 
@@ -311,16 +522,13 @@ inline raw_ostream &operator<<(raw_ostream &os, PDLValue value) {
 /// success if the constraint successfully held, failure otherwise.
 using PDLConstraintFunction = std::function<LogicalResult(
     ArrayRef<PDLValue>, ArrayAttr, PatternRewriter &)>;
-/// A native PDL creation function. This function creates a new PDLValue given
-/// a set of existing PDL values, a set of constant parameters specified in
-/// Attribute form, and a PatternRewriter. Returns the newly created PDLValue.
-using PDLCreateFunction =
-    std::function<PDLValue(ArrayRef<PDLValue>, ArrayAttr, PatternRewriter &)>;
-/// A native PDL rewrite function. This function rewrites the given root
-/// operation using the provided PatternRewriter. This method is only invoked
-/// when the corresponding match was successful.
-using PDLRewriteFunction = std::function<void(Operation *, ArrayRef<PDLValue>,
-                                              ArrayAttr, PatternRewriter &)>;
+/// A native PDL rewrite function. This function performs a rewrite on the
+/// given set of values and constant parameters. Any results from this rewrite
+/// that should be passed back to PDL should be added to the provided result
+/// list. This method is only invoked when the corresponding match was
+/// successful.
+using PDLRewriteFunction = std::function<void(
+    ArrayRef<PDLValue>, ArrayAttr, PatternRewriter &, PDLResultList &)>;
 /// A generic PDL pattern constraint function. This function applies a
 /// constraint to a given opaque PDLValue entity. The second parameter is a set
 /// of constant value parameters specified in Attribute form. Returns success if
@@ -367,9 +575,6 @@ public:
         });
   }
 
-  /// Register a creation function.
-  void registerCreateFunction(StringRef name, PDLCreateFunction createFn);
-
   /// Register a rewrite function.
   void registerRewriteFunction(StringRef name, PDLRewriteFunction rewriteFn);
 
@@ -380,13 +585,6 @@ public:
   llvm::StringMap<PDLConstraintFunction> takeConstraintFunctions() {
     return constraintFunctions;
   }
-  /// Return the set of the registered create functions.
-  const llvm::StringMap<PDLCreateFunction> &getCreateFunctions() const {
-    return createFunctions;
-  }
-  llvm::StringMap<PDLCreateFunction> takeCreateFunctions() {
-    return createFunctions;
-  }
   /// Return the set of the registered rewrite functions.
   const llvm::StringMap<PDLRewriteFunction> &getRewriteFunctions() const {
     return rewriteFunctions;
@@ -399,7 +597,6 @@ public:
   void clear() {
     pdlModule = nullptr;
     constraintFunctions.clear();
-    createFunctions.clear();
     rewriteFunctions.clear();
   }
 
@@ -409,7 +606,6 @@ private:
 
   /// The external functions referenced from within the PDL module.
   llvm::StringMap<PDLConstraintFunction> constraintFunctions;
-  llvm::StringMap<PDLCreateFunction> createFunctions;
   llvm::StringMap<PDLRewriteFunction> rewriteFunctions;
 };
 
@@ -618,21 +814,26 @@ public:
 };
 
 //===----------------------------------------------------------------------===//
-// OwningRewritePatternList
+// RewritePatternSet
 //===----------------------------------------------------------------------===//
 
-class OwningRewritePatternList {
+class RewritePatternSet {
   using NativePatternListT = std::vector<std::unique_ptr<RewritePattern>>;
 
 public:
-  OwningRewritePatternList() = default;
+  RewritePatternSet(MLIRContext *context) : context(context) {}
 
-  /// Construct a OwningRewritePatternList populated with the given pattern.
-  OwningRewritePatternList(std::unique_ptr<RewritePattern> pattern) {
+  /// Construct a RewritePatternSet populated with the given pattern.
+  RewritePatternSet(MLIRContext *context,
+                    std::unique_ptr<RewritePattern> pattern)
+      : context(context) {
     nativePatterns.emplace_back(std::move(pattern));
   }
-  OwningRewritePatternList(PDLPatternModule &&pattern)
-      : pdlPatterns(std::move(pattern)) {}
+  RewritePatternSet(PDLPatternModule &&pattern)
+      : context(pattern.getModule()->getContext()),
+        pdlPatterns(std::move(pattern)) {}
+
+  MLIRContext *getContext() const { return context; }
 
   /// Return the native patterns held in this list.
   NativePatternListT &getNativePatterns() { return nativePatterns; }
@@ -646,66 +847,156 @@ public:
     pdlPatterns.clear();
   }
 
+  //===--------------------------------------------------------------------===//
+  // 'add' methods for adding patterns to the set.
+  //===--------------------------------------------------------------------===//
+
+  /// Add an instance of each of the pattern types 'Ts' to the pattern list with
+  /// the given arguments. Return a reference to `this` for chaining insertions.
+  /// Note: ConstructorArg is necessary here to separate the two variadic lists.
+  template <typename... Ts, typename ConstructorArg,
+            typename... ConstructorArgs,
+            typename = std::enable_if_t<sizeof...(Ts) != 0>>
+  RewritePatternSet &add(ConstructorArg &&arg, ConstructorArgs &&... args) {
+    // The following expands a call to emplace_back for each of the pattern
+    // types 'Ts'. This magic is necessary due to a limitation in the places
+    // that a parameter pack can be expanded in c++11.
+    // FIXME: In c++17 this can be simplified by using 'fold expressions'.
+    (void)std::initializer_list<int>{0, (addImpl<Ts>(arg, args...), 0)...};
+    return *this;
+  }
+
+  /// Add an instance of each of the pattern types 'Ts'. Return a reference to
+  /// `this` for chaining insertions.
+  template <typename... Ts>
+  RewritePatternSet &add() {
+    (void)std::initializer_list<int>{0, (addImpl<Ts>(), 0)...};
+    return *this;
+  }
+
+  /// Add the given native pattern to the pattern list. Return a reference to
+  /// `this` for chaining insertions.
+  RewritePatternSet &add(std::unique_ptr<RewritePattern> pattern) {
+    nativePatterns.emplace_back(std::move(pattern));
+    return *this;
+  }
+
+  /// Add the given PDL pattern to the pattern list. Return a reference to
+  /// `this` for chaining insertions.
+  RewritePatternSet &add(PDLPatternModule &&pattern) {
+    pdlPatterns.mergeIn(std::move(pattern));
+    return *this;
+  }
+
+  // Add a matchAndRewrite style pattern represented as a C function pointer.
+  template <typename OpType>
+  RewritePatternSet &add(LogicalResult (*implFn)(OpType,
+                                                 PatternRewriter &rewriter)) {
+    struct FnPattern final : public OpRewritePattern<OpType> {
+      FnPattern(LogicalResult (*implFn)(OpType, PatternRewriter &rewriter),
+                MLIRContext *context)
+          : OpRewritePattern<OpType>(context), implFn(implFn) {}
+
+      LogicalResult matchAndRewrite(OpType op,
+                                    PatternRewriter &rewriter) const override {
+        return implFn(op, rewriter);
+      }
+
+    private:
+      LogicalResult (*implFn)(OpType, PatternRewriter &rewriter);
+    };
+    add(std::make_unique<FnPattern>(std::move(implFn), getContext()));
+    return *this;
+  }
+
   //===--------------------------------------------------------------------===//
   // Pattern Insertion
   //===--------------------------------------------------------------------===//
 
+  // TODO: These are soft deprecated in favor of the 'add' methods above.
+
   /// Add an instance of each of the pattern types 'Ts' to the pattern list with
   /// the given arguments. Return a reference to `this` for chaining insertions.
   /// Note: ConstructorArg is necessary here to separate the two variadic lists.
   template <typename... Ts, typename ConstructorArg,
             typename... ConstructorArgs,
             typename = std::enable_if_t<sizeof...(Ts) != 0>>
-  OwningRewritePatternList &insert(ConstructorArg &&arg,
-                                   ConstructorArgs &&...args) {
+  RewritePatternSet &insert(ConstructorArg &&arg, ConstructorArgs &&... args) {
     // The following expands a call to emplace_back for each of the pattern
     // types 'Ts'. This magic is necessary due to a limitation in the places
     // that a parameter pack can be expanded in c++11.
     // FIXME: In c++17 this can be simplified by using 'fold expressions'.
-    (void)std::initializer_list<int>{0, (insertImpl<Ts>(arg, args...), 0)...};
+    (void)std::initializer_list<int>{0, (addImpl<Ts>(arg, args...), 0)...};
     return *this;
   }
 
   /// Add an instance of each of the pattern types 'Ts'. Return a reference to
   /// `this` for chaining insertions.
-  template <typename... Ts> OwningRewritePatternList &insert() {
-    (void)std::initializer_list<int>{0, (insertImpl<Ts>(), 0)...};
+  template <typename... Ts>
+  RewritePatternSet &insert() {
+    (void)std::initializer_list<int>{0, (addImpl<Ts>(), 0)...};
     return *this;
   }
 
   /// Add the given native pattern to the pattern list. Return a reference to
   /// `this` for chaining insertions.
-  OwningRewritePatternList &insert(std::unique_ptr<RewritePattern> pattern) {
+  RewritePatternSet &insert(std::unique_ptr<RewritePattern> pattern) {
     nativePatterns.emplace_back(std::move(pattern));
     return *this;
   }
 
   /// Add the given PDL pattern to the pattern list. Return a reference to
   /// `this` for chaining insertions.
-  OwningRewritePatternList &insert(PDLPatternModule &&pattern) {
+  RewritePatternSet &insert(PDLPatternModule &&pattern) {
     pdlPatterns.mergeIn(std::move(pattern));
     return *this;
   }
 
+  // Add a matchAndRewrite style pattern represented as a C function pointer.
+  template <typename OpType>
+  RewritePatternSet &
+  insert(LogicalResult (*implFn)(OpType, PatternRewriter &rewriter)) {
+    struct FnPattern final : public OpRewritePattern<OpType> {
+      FnPattern(LogicalResult (*implFn)(OpType, PatternRewriter &rewriter),
+                MLIRContext *context)
+          : OpRewritePattern<OpType>(context), implFn(implFn) {}
+
+      LogicalResult matchAndRewrite(OpType op,
+                                    PatternRewriter &rewriter) const override {
+        return implFn(op, rewriter);
+      }
+
+    private:
+      LogicalResult (*implFn)(OpType, PatternRewriter &rewriter);
+    };
+    insert(std::make_unique<FnPattern>(std::move(implFn), getContext()));
+    return *this;
+  }
+
 private:
   /// Add an instance of the pattern type 'T'. Return a reference to `this` for
   /// chaining insertions.
   template <typename T, typename... Args>
   std::enable_if_t<std::is_base_of<RewritePattern, T>::value>
-  insertImpl(Args &&...args) {
+  addImpl(Args &&... args) {
     nativePatterns.emplace_back(
         std::make_unique<T>(std::forward<Args>(args)...));
   }
   template <typename T, typename... Args>
   std::enable_if_t<std::is_base_of<PDLPatternModule, T>::value>
-  insertImpl(Args &&...args) {
+  addImpl(Args &&... args) {
     pdlPatterns.mergeIn(T(std::forward<Args>(args)...));
   }
 
+  MLIRContext *const context;
   NativePatternListT nativePatterns;
   PDLPatternModule pdlPatterns;
 };
 
+// TODO: OwningRewritePatternList is soft-deprecated and will be removed in the
+// future.
+using OwningRewritePatternList = RewritePatternSet;
+
 } // end namespace mlir
 
 #endif // MLIR_PATTERN_MATCH_H
diff --git a/mlir/include/mlir/IR/Region.h b/mlir/include/mlir/IR/Region.h
index 8888862dd10c4178f1aaa30e99366ed13d81e828..bc35e2d231a504f2658649572cb4c2176fd43dc4 100644
--- a/mlir/include/mlir/IR/Region.h
+++ b/mlir/include/mlir/IR/Region.h
@@ -43,6 +43,10 @@ public:
 
   using BlockListType = llvm::iplist<Block>;
   BlockListType &getBlocks() { return blocks; }
+  Block &emplaceBlock() {
+    push_back(new Block);
+    return back();
+  }
 
   // Iteration over the blocks in the region.
   using iterator = BlockListType::iterator;
diff --git a/mlir/include/mlir/IR/RegionKindInterface.h b/mlir/include/mlir/IR/RegionKindInterface.h
index c1a1fa8074a71c3b362d5e00109ee2023a57c690..a4b77d65cd65445a6787b441da00af6849221e3a 100644
--- a/mlir/include/mlir/IR/RegionKindInterface.h
+++ b/mlir/include/mlir/IR/RegionKindInterface.h
@@ -28,6 +28,16 @@ enum class RegionKind {
   Graph,
 };
 
+namespace OpTrait {
+/// A trait that specifies that an operation only defines graph regions.
+template <typename ConcreteType>
+class HasOnlyGraphRegion : public TraitBase<ConcreteType, HasOnlyGraphRegion> {
+public:
+  static RegionKind getRegionKind(unsigned index) { return RegionKind::Graph; }
+  static bool hasSSADominance(unsigned index) { return false; }
+};
+} // namespace OpTrait
+
 } // namespace mlir
 
 #include "mlir/IR/RegionKindInterface.h.inc"
diff --git a/mlir/include/mlir/IR/RegionKindInterface.td b/mlir/include/mlir/IR/RegionKindInterface.td
index 1a6f739be172d58bd932550516735fb8fb37a8cc..59b235186455f9d8eee9a22ec5ea9d5518ba4a27 100644
--- a/mlir/include/mlir/IR/RegionKindInterface.td
+++ b/mlir/include/mlir/IR/RegionKindInterface.td
@@ -50,4 +50,17 @@ def RegionKindInterface : OpInterface<"RegionKindInterface"> {
   ];
 }
 
+def HasOnlyGraphRegion : NativeOpTrait<"HasOnlyGraphRegion">;
+
+// Op's regions that don't need a terminator: requires some other traits
+// so it defines a list that must be concatenated.
+def GraphRegionNoTerminator {
+  list<OpTrait> traits = [
+    NoTerminator,
+    SingleBlock,
+    RegionKindInterface,
+    HasOnlyGraphRegion
+  ];
+}
+
 #endif // MLIR_IR_REGIONKINDINTERFACE
diff --git a/mlir/include/mlir/IR/SymbolInterfaces.td b/mlir/include/mlir/IR/SymbolInterfaces.td
index a7b1fd8cfe6433529f062266f66565d0804b129b..0e8f5a13fe5556acb95147d5aed35d646b411fc6 100644
--- a/mlir/include/mlir/IR/SymbolInterfaces.td
+++ b/mlir/include/mlir/IR/SymbolInterfaces.td
@@ -178,10 +178,10 @@ def Symbol : OpInterface<"SymbolOpInterface"> {
   let extraClassDeclaration = [{
     /// Custom classof that handles the case where the symbol is optional.
     static bool classof(Operation *op) {
-      auto *concept = getInterfaceFor(op);
-      if (!concept)
+      auto *opConcept = getInterfaceFor(op);
+      if (!opConcept)
         return false;
-      return !concept->isOptionalSymbol(op) ||
+      return !opConcept->isOptionalSymbol(opConcept, op) ||
              op->getAttr(::mlir::SymbolTable::getSymbolAttrName());
     }
   }];
diff --git a/mlir/include/mlir/IR/TypeRange.h b/mlir/include/mlir/IR/TypeRange.h
index fe11fde587934c527fe1e61f9cec1a7dcc51e218..4fb40e127f9faa995a07815d3b4c049174b5bae8 100644
--- a/mlir/include/mlir/IR/TypeRange.h
+++ b/mlir/include/mlir/IR/TypeRange.h
@@ -82,6 +82,12 @@ inline ::llvm::hash_code hash_value(TypeRange arg) {
   return ::llvm::hash_combine_range(arg.begin(), arg.end());
 }
 
+/// Emit a type range to the given output stream.
+inline raw_ostream &operator<<(raw_ostream &os, const TypeRange &types) {
+  llvm::interleaveComma(types, os);
+  return os;
+}
+
 //===----------------------------------------------------------------------===//
 // ValueTypeRange
 
diff --git a/mlir/include/mlir/InitAllPasses.h b/mlir/include/mlir/InitAllPasses.h
index 029df07359594a5c6a624a1f0fc367d8e0e1b5f9..ab9629ac86c152d6326a317ec741bdbeaca2a213 100644
--- a/mlir/include/mlir/InitAllPasses.h
+++ b/mlir/include/mlir/InitAllPasses.h
@@ -52,6 +52,7 @@ inline void registerAllPasses() {
   registerAsyncPasses();
   registerGPUPasses();
   registerGpuSerializeToCubinPass();
+  registerGpuSerializeToHsacoPass();
   registerLinalgPasses();
   LLVM::registerLLVMPasses();
   quant::registerQuantPasses();
diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h
index f92048973e71438840b98e38947a02f81f9858d1..87cac054c55eca29b00a6d814e0ef84978c8dd69 100644
--- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h
+++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h
@@ -29,12 +29,19 @@ using DataLayoutEntryList = llvm::SmallVector<DataLayoutEntryInterface, 4>;
 using DataLayoutEntryListRef = llvm::ArrayRef<DataLayoutEntryInterface>;
 class DataLayoutOpInterface;
 class DataLayoutSpecInterface;
+class ModuleOp;
 
 namespace detail {
 /// Default handler for the type size request. Computes results for built-in
 /// types and dispatches to the DataLayoutTypeInterface for other types.
 unsigned getDefaultTypeSize(Type type, const DataLayout &dataLayout,
-                            ArrayRef<DataLayoutEntryInterface> params);
+                            DataLayoutEntryListRef params);
+
+/// Default handler for the type size in bits request. Computes results for
+/// built-in types and dispatches to the DataLayoutTypeInterface for other
+/// types.
+unsigned getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout,
+                                  DataLayoutEntryListRef params);
 
 /// Default handler for the required alignemnt request. Computes results for
 /// built-in types and dispatches to the DataLayoutTypeInterface for other
@@ -60,10 +67,11 @@ DataLayoutEntryList filterEntriesForType(DataLayoutEntryListRef entries,
 DataLayoutEntryInterface
 filterEntryForIdentifier(DataLayoutEntryListRef entries, Identifier id);
 
-/// Verifies that the operation implementing the data layout interface is valid.
-/// This calls the verifier of the spec attribute and checks if the layout is
-/// compatible with specs attached to the enclosing operations.
-LogicalResult verifyDataLayoutOp(DataLayoutOpInterface op);
+/// Verifies that the operation implementing the data layout interface, or a
+/// module operation, is valid. This calls the verifier of the spec attribute
+/// and checks if the layout is compatible with specs attached to the enclosing
+/// operations.
+LogicalResult verifyDataLayoutOp(Operation *op);
 
 /// Verifies that a data layout spec is valid. This dispatches to individual
 /// entry verifiers, and then to the verifiers implemented by the relevant type
@@ -132,11 +140,16 @@ public:
 /// mode, the cache validity is being checked in every request.
 class DataLayout {
 public:
+  explicit DataLayout();
   explicit DataLayout(DataLayoutOpInterface op);
+  explicit DataLayout(ModuleOp op);
 
   /// Returns the size of the given type in the current scope.
   unsigned getTypeSize(Type t) const;
 
+  /// Returns the size in bits of the given type in the current scope.
+  unsigned getTypeSizeInBits(Type t) const;
+
   /// Returns the required alignment of the given type in the current scope.
   unsigned getTypeABIAlignment(Type t) const;
 
@@ -159,10 +172,11 @@ private:
   /// Operation defining the scope of requests.
   // TODO: this is mutable because the generated interface method are not const.
   // Update the generator to support const methods and change this to const.
-  mutable DataLayoutOpInterface scope;
+  mutable Operation *scope;
 
   /// Caches for individual requests.
   mutable DenseMap<Type, unsigned> sizes;
+  mutable DenseMap<Type, unsigned> bitsizes;
   mutable DenseMap<Type, unsigned> abiAlignments;
   mutable DenseMap<Type, unsigned> preferredAlignments;
 };
diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td
index 19a0410f1961236fa362f3c463ec974ac22da0a5..62606995930cfe55013ad764c248c018b91eccff 100644
--- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td
+++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td
@@ -208,7 +208,23 @@ def DataLayoutOpInterface : OpInterface<"DataLayoutOpInterface"> {
                     "::mlir::DataLayoutEntryListRef":$params),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{
-        return ::mlir::detail::getDefaultTypeSize(type, dataLayout, params);
+        unsigned bits = ConcreteOp::getTypeSizeInBits(type, dataLayout, params);
+        return ::llvm::divideCeil(bits, 8);
+      }]
+    >,
+    StaticInterfaceMethod<
+      /*description=*/"Returns the size of the given type in bits computed "
+                      "using the relevant entries. The data layout object can "
+                      "be used for recursive queries.",
+      /*retTy=*/"unsigned",
+      /*methodName=*/"getTypeSizeInBits",
+      /*args=*/(ins "::mlir::Type":$type,
+                    "const ::mlir::DataLayout &":$dataLayout,
+                    "::mlir::DataLayoutEntryListRef":$params),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return ::mlir::detail::getDefaultTypeSizeInBits(type, dataLayout,
+                                                        params);
       }]
     >,
     StaticInterfaceMethod<
@@ -281,6 +297,18 @@ def DataLayoutTypeInterface : TypeInterface<"DataLayoutTypeInterface"> {
       /*description=*/"Returns the size of this type in bytes.",
       /*retTy=*/"unsigned",
       /*methodName=*/"getTypeSize",
+      /*args=*/(ins "const ::mlir::DataLayout &":$dataLayout,
+                    "::mlir::DataLayoutEntryListRef":$params),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        unsigned bits = $_type.getTypeSizeInBits(dataLayout, params);
+        return ::llvm::divideCeil(bits, 8);
+      }]
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the size of this type in bits.",
+      /*retTy=*/"unsigned",
+      /*methodName=*/"getTypeSizeInBits",
       /*args=*/(ins "const ::mlir::DataLayout &":$dataLayout,
                     "::mlir::DataLayoutEntryListRef":$params)
     >,
diff --git a/mlir/include/mlir/Parser.h b/mlir/include/mlir/Parser.h
index cec60474d23d2a7426b31cc83583214d6bb97051..907f31824628b0b4c702c6585e84ae01d7f9fa8a 100644
--- a/mlir/include/mlir/Parser.h
+++ b/mlir/include/mlir/Parser.h
@@ -25,6 +25,7 @@ class StringRef;
 
 namespace mlir {
 namespace detail {
+
 /// Given a block containing operations that have just been parsed, if the block
 /// contains a single operation of `ContainerOpT` type then remove it from the
 /// block and return it. If the block does not contain just that operation,
@@ -37,12 +38,11 @@ inline OwningOpRef<ContainerOpT> constructContainerOpForParserIfNecessary(
     Block *parsedBlock, MLIRContext *context, Location sourceFileLoc) {
   static_assert(
       ContainerOpT::template hasTrait<OpTrait::OneRegion>() &&
-          std::is_base_of<typename OpTrait::SingleBlockImplicitTerminator<
-                              typename ContainerOpT::ImplicitTerminatorOpT>::
-                              template Impl<ContainerOpT>,
-                          ContainerOpT>::value,
+          (ContainerOpT::template hasTrait<OpTrait::NoTerminator>() ||
+           OpTrait::template hasSingleBlockImplicitTerminator<
+               ContainerOpT>::value),
       "Expected `ContainerOpT` to have a single region with a single "
-      "block that has an implicit terminator");
+      "block that has an implicit terminator or does not require one");
 
   // Check to see if we parsed a single instance of this operation.
   if (llvm::hasSingleElement(*parsedBlock)) {
diff --git a/mlir/include/mlir/Rewrite/FrozenRewritePatternList.h b/mlir/include/mlir/Rewrite/FrozenRewritePatternList.h
deleted file mode 100644
index 0e583aab3dc468303afdc4d443f790b8c7eea887..0000000000000000000000000000000000000000
--- a/mlir/include/mlir/Rewrite/FrozenRewritePatternList.h
+++ /dev/null
@@ -1,71 +0,0 @@
-//===- FrozenRewritePatternList.h - FrozenRewritePatternList ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_REWRITE_FROZENREWRITEPATTERNLIST_H
-#define MLIR_REWRITE_FROZENREWRITEPATTERNLIST_H
-
-#include "mlir/IR/PatternMatch.h"
-
-namespace mlir {
-namespace detail {
-class PDLByteCode;
-} // end namespace detail
-
-/// This class represents a frozen set of patterns that can be processed by a
-/// pattern applicator. This class is designed to enable caching pattern lists
-/// such that they need not be continuously recomputed. Note that all copies of
-/// this class share the same compiled pattern list, allowing for a reduction in
-/// the number of duplicated patterns that need to be created.
-class FrozenRewritePatternList {
-  using NativePatternListT = std::vector<std::unique_ptr<RewritePattern>>;
-
-public:
-  /// Freeze the patterns held in `patterns`, and take ownership.
-  FrozenRewritePatternList();
-  FrozenRewritePatternList(OwningRewritePatternList &&patterns);
-  FrozenRewritePatternList(FrozenRewritePatternList &&patterns) = default;
-  FrozenRewritePatternList(const FrozenRewritePatternList &patterns) = default;
-  FrozenRewritePatternList &
-  operator=(const FrozenRewritePatternList &patterns) = default;
-  FrozenRewritePatternList &
-  operator=(FrozenRewritePatternList &&patterns) = default;
-  ~FrozenRewritePatternList();
-
-  /// Return the native patterns held by this list.
-  iterator_range<llvm::pointee_iterator<NativePatternListT::const_iterator>>
-  getNativePatterns() const {
-    const NativePatternListT &nativePatterns = impl->nativePatterns;
-    return llvm::make_pointee_range(nativePatterns);
-  }
-
-  /// Return the compiled PDL bytecode held by this list. Returns null if
-  /// there are no PDL patterns within the list.
-  const detail::PDLByteCode *getPDLByteCode() const {
-    return impl->pdlByteCode.get();
-  }
-
-private:
-  /// The internal implementation of the frozen pattern list.
-  struct Impl {
-    /// The set of native C++ rewrite patterns.
-    NativePatternListT nativePatterns;
-
-    /// The bytecode containing the compiled PDL patterns.
-    std::unique_ptr<detail::PDLByteCode> pdlByteCode;
-  };
-
-  /// A pointer to the internal pattern list. This uses a shared_ptr to avoid
-  /// the need to compile the same pattern list multiple times. For example,
-  /// during multi-threaded pass execution, all copies of a pass can share the
-  /// same pattern list.
-  std::shared_ptr<Impl> impl;
-};
-
-} // end namespace mlir
-
-#endif // MLIR_REWRITE_FROZENREWRITEPATTERNLIST_H
diff --git a/mlir/include/mlir/Rewrite/FrozenRewritePatternSet.h b/mlir/include/mlir/Rewrite/FrozenRewritePatternSet.h
new file mode 100644
index 0000000000000000000000000000000000000000..6791fbd7e3c09c1f431fd65f19a3580f00325aa0
--- /dev/null
+++ b/mlir/include/mlir/Rewrite/FrozenRewritePatternSet.h
@@ -0,0 +1,93 @@
+//===- FrozenRewritePatternSet.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_REWRITE_FROZENREWRITEPATTERNSET_H
+#define MLIR_REWRITE_FROZENREWRITEPATTERNSET_H
+
+#include "mlir/IR/PatternMatch.h"
+
+namespace mlir {
+namespace detail {
+class PDLByteCode;
+} // end namespace detail
+
+/// This class represents a frozen set of patterns that can be processed by a
+/// pattern applicator. This class is designed to enable caching pattern lists
+/// such that they need not be continuously recomputed. Note that all copies of
+/// this class share the same compiled pattern list, allowing for a reduction in
+/// the number of duplicated patterns that need to be created.
+class FrozenRewritePatternSet {
+  using NativePatternListT = std::vector<std::unique_ptr<RewritePattern>>;
+
+public:
+  /// A map of operation specific native patterns.
+  using OpSpecificNativePatternListT =
+      DenseMap<OperationName, std::vector<RewritePattern *>>;
+
+  /// Freeze the patterns held in `patterns`, and take ownership.
+  FrozenRewritePatternSet();
+  FrozenRewritePatternSet(RewritePatternSet &&patterns);
+  FrozenRewritePatternSet(FrozenRewritePatternSet &&patterns) = default;
+  FrozenRewritePatternSet(const FrozenRewritePatternSet &patterns) = default;
+  FrozenRewritePatternSet &
+  operator=(const FrozenRewritePatternSet &patterns) = default;
+  FrozenRewritePatternSet &
+  operator=(FrozenRewritePatternSet &&patterns) = default;
+  ~FrozenRewritePatternSet();
+
+  /// Return the op specific native patterns held by this list.
+  const OpSpecificNativePatternListT &getOpSpecificNativePatterns() const {
+    return impl->nativeOpSpecificPatternMap;
+  }
+
+  /// Return the "match any" native patterns held by this list.
+  iterator_range<llvm::pointee_iterator<NativePatternListT::const_iterator>>
+  getMatchAnyOpNativePatterns() const {
+    const NativePatternListT &nativeList = impl->nativeAnyOpPatterns;
+    return llvm::make_pointee_range(nativeList);
+  }
+
+  /// Return the compiled PDL bytecode held by this list. Returns null if
+  /// there are no PDL patterns within the list.
+  const detail::PDLByteCode *getPDLByteCode() const {
+    return impl->pdlByteCode.get();
+  }
+
+private:
+  /// The internal implementation of the frozen pattern list.
+  struct Impl {
+    /// The set of native C++ rewrite patterns that are matched to specific
+    /// operation kinds.
+    OpSpecificNativePatternListT nativeOpSpecificPatternMap;
+
+    /// The full op-specific native rewrite list. This allows for the map above
+    /// to contain duplicate patterns, e.g. for interfaces and traits.
+    NativePatternListT nativeOpSpecificPatternList;
+
+    /// The set of native C++ rewrite patterns that are matched to "any"
+    /// operation.
+    NativePatternListT nativeAnyOpPatterns;
+
+    /// The bytecode containing the compiled PDL patterns.
+    std::unique_ptr<detail::PDLByteCode> pdlByteCode;
+  };
+
+  /// A pointer to the internal pattern list. This uses a shared_ptr to avoid
+  /// the need to compile the same pattern list multiple times. For example,
+  /// during multi-threaded pass execution, all copies of a pass can share the
+  /// same pattern list.
+  std::shared_ptr<Impl> impl;
+};
+
+// TODO: FrozenRewritePatternList is soft-deprecated and will be removed in the
+// future.
+using FrozenRewritePatternList = FrozenRewritePatternSet;
+
+} // end namespace mlir
+
+#endif // MLIR_REWRITE_FROZENREWRITEPATTERNSET_H
diff --git a/mlir/include/mlir/Rewrite/PatternApplicator.h b/mlir/include/mlir/Rewrite/PatternApplicator.h
index 9d197175b47d742dce368e0c30a2e923c060bdaa..9314496ecda15565aecf7345f5aa6a3eafb5b21e 100644
--- a/mlir/include/mlir/Rewrite/PatternApplicator.h
+++ b/mlir/include/mlir/Rewrite/PatternApplicator.h
@@ -14,7 +14,7 @@
 #ifndef MLIR_REWRITE_PATTERNAPPLICATOR_H
 #define MLIR_REWRITE_PATTERNAPPLICATOR_H
 
-#include "mlir/Rewrite/FrozenRewritePatternList.h"
+#include "mlir/Rewrite/FrozenRewritePatternSet.h"
 
 namespace mlir {
 class PatternRewriter;
@@ -33,7 +33,7 @@ public:
   /// `impossibleToMatch`.
   using CostModel = function_ref<PatternBenefit(const Pattern &)>;
 
-  explicit PatternApplicator(const FrozenRewritePatternList &frozenPatternList);
+  explicit PatternApplicator(const FrozenRewritePatternSet &frozenPatternList);
   ~PatternApplicator();
 
   /// Attempt to match and rewrite the given op with any pattern, allowing a
@@ -65,7 +65,7 @@ public:
 
 private:
   /// The list that owns the patterns used within this applicator.
-  const FrozenRewritePatternList &frozenPatternList;
+  const FrozenRewritePatternSet &frozenPatternList;
   /// The set of patterns to match for each operation, stable sorted by benefit.
   DenseMap<OperationName, SmallVector<const RewritePattern *, 2>> patterns;
   /// The set of patterns that may match against any operation type, stable
diff --git a/mlir/include/mlir/Support/InterfaceSupport.h b/mlir/include/mlir/Support/InterfaceSupport.h
index b618e8effd4a99dac3713c3426688efae1ed1da3..6fc61170a59236b553584619c2210e9d736d5f69 100644
--- a/mlir/include/mlir/Support/InterfaceSupport.h
+++ b/mlir/include/mlir/Support/InterfaceSupport.h
@@ -71,6 +71,8 @@ class Interface : public BaseType {
 public:
   using Concept = typename Traits::Concept;
   template <typename T> using Model = typename Traits::template Model<T>;
+  template <typename T>
+  using FallbackModel = typename Traits::template FallbackModel<T>;
   using InterfaceBase =
       Interface<ConcreteType, ValueT, Traits, BaseType, BaseTrait>;
 
@@ -183,6 +185,9 @@ public:
     return reinterpret_cast<typename T::Concept *>(lookup(T::getInterfaceID()));
   }
 
+  /// Returns true if the interface map contains an interface for the given id.
+  bool contains(TypeID interfaceID) const { return lookup(interfaceID); }
+
 private:
   /// Compare two TypeID instances by comparing the underlying pointer.
   static bool compare(TypeID lhs, TypeID rhs) {
diff --git a/mlir/include/mlir/TableGen/AttrOrTypeDef.h b/mlir/include/mlir/TableGen/AttrOrTypeDef.h
index 84a5a04c909c37a5c402ada1332e8b892bba706e..5271fbae0eea3ddc9657e2870639e08595163ff2 100644
--- a/mlir/include/mlir/TableGen/AttrOrTypeDef.h
+++ b/mlir/include/mlir/TableGen/AttrOrTypeDef.h
@@ -183,6 +183,9 @@ public:
   // If specified, get the custom allocator code for this parameter.
   Optional<StringRef> getAllocator() const;
 
+  // If specified, get the custom comparator code for this parameter.
+  Optional<StringRef> getComparator() const;
+
   // Get the C++ type of this parameter.
   StringRef getCppType() const;
 
diff --git a/mlir/include/mlir/TableGen/Attribute.h b/mlir/include/mlir/TableGen/Attribute.h
index dc6c9692581c3fb521089206e319d8cdd0b9e3b3..a8292a982320daf7895e68cd1f6490ce3e5a63ce 100644
--- a/mlir/include/mlir/TableGen/Attribute.h
+++ b/mlir/include/mlir/TableGen/Attribute.h
@@ -202,6 +202,10 @@ public:
 
   // Returns all allowed cases for this enum attribute.
   std::vector<EnumAttrCase> getAllCases() const;
+
+  bool genSpecializedAttr() const;
+  llvm::Record *getBaseAttrClass() const;
+  StringRef getSpecializedAttrClassName() const;
 };
 
 class StructFieldAttr {
diff --git a/mlir/include/mlir/TableGen/Dialect.h b/mlir/include/mlir/TableGen/Dialect.h
index ee86a2504b3c924f210fa500cf4bd4992df2aef4..35a9e7ba4c9b6a4b67499e6d5752256f3c87e06b 100644
--- a/mlir/include/mlir/TableGen/Dialect.h
+++ b/mlir/include/mlir/TableGen/Dialect.h
@@ -63,6 +63,9 @@ public:
   /// Returns true if this dialect has a region result attribute verifier.
   bool hasRegionResultAttrVerify() const;
 
+  /// Returns true if this dialect has fallback interfaces for its operations.
+  bool hasOperationInterfaceFallback() const;
+
   // Returns whether two dialects are equal by checking the equality of the
   // underlying record.
   bool operator==(const Dialect &other) const;
diff --git a/mlir/include/mlir/TableGen/Format.h b/mlir/include/mlir/TableGen/Format.h
index 18a7a6f985b8017da062c175e72c043348fe1685..441e05c29f264672e4cc90373ac63099be8b6971 100644
--- a/mlir/include/mlir/TableGen/Format.h
+++ b/mlir/include/mlir/TableGen/Format.h
@@ -186,6 +186,20 @@ public:
   }
 };
 
+class FmtStrVecObject : public FmtObjectBase {
+public:
+  using StrFormatAdapter =
+      decltype(llvm::detail::build_format_adapter(std::declval<std::string>()));
+
+  FmtStrVecObject(StringRef fmt, const FmtContext *ctx,
+                  ArrayRef<std::string> params);
+  FmtStrVecObject(FmtStrVecObject const &that) = delete;
+  FmtStrVecObject(FmtStrVecObject &&that);
+
+private:
+  SmallVector<StrFormatAdapter, 16> parameters;
+};
+
 /// Formats text by substituting placeholders in format string with replacement
 /// parameters.
 ///
@@ -234,6 +248,11 @@ inline auto tgfmt(StringRef fmt, const FmtContext *ctx, Ts &&... vals)
           llvm::detail::build_format_adapter(std::forward<Ts>(vals))...));
 }
 
+inline FmtStrVecObject tgfmt(StringRef fmt, const FmtContext *ctx,
+                             ArrayRef<std::string> params) {
+  return FmtStrVecObject(fmt, ctx, params);
+}
+
 } // end namespace tblgen
 } // end namespace mlir
 
diff --git a/mlir/include/mlir/Transforms/BufferUtils.h b/mlir/include/mlir/Transforms/BufferUtils.h
index 70da6a025343f4d010097fc43ac23dc85c7c088b..e432fb8f53f55ebf2efe95d4c6ee9ae154f2d918 100644
--- a/mlir/include/mlir/Transforms/BufferUtils.h
+++ b/mlir/include/mlir/Transforms/BufferUtils.h
@@ -39,10 +39,6 @@ public:
   static Operation *getStartOperation(Value allocValue, Block *placementBlock,
                                       const Liveness &liveness);
 
-  /// Find an associated dealloc operation that is linked to the given
-  /// allocation node (if any).
-  static Operation *findDealloc(Value allocValue);
-
 public:
   /// Initializes the internal list by discovering all supported allocation
   /// nodes.
@@ -120,6 +116,24 @@ protected:
   Liveness liveness;
 };
 
+namespace memref {
+class GlobalOp;
+} // namespace memref
+
+// Support class to create global ops for tensor-valued constants in the
+// program. Globals are created lazily at the top of the `moduleOp` with pretty
+// names. Duplicates are avoided.
+class GlobalCreator {
+public:
+  explicit GlobalCreator(ModuleOp module) : moduleOp(module) {}
+  memref::GlobalOp getGlobalFor(ConstantOp constantOp);
+
+private:
+  ModuleOp moduleOp;
+  // This could use memref::GlobalOp key but we avoid introducing a new
+  // dependence to the memref dialect for this.
+  DenseMap<Attribute, Operation *> globals;
+};
 } // end namespace mlir
 
 #endif // MLIR_TRANSFORMS_BUFFERUTILS_H
diff --git a/mlir/include/mlir/Transforms/Bufferize.h b/mlir/include/mlir/Transforms/Bufferize.h
index 29e16c2ecb1cbc9ce03aad34f57f80033f56c4f1..22155f7de1d4f38d6ebed50c6b9a943be85a228b 100644
--- a/mlir/include/mlir/Transforms/Bufferize.h
+++ b/mlir/include/mlir/Transforms/Bufferize.h
@@ -56,8 +56,7 @@ void populateBufferizeMaterializationLegality(ConversionTarget &target);
 ///
 /// In particular, these are the tensor_load/buffer_cast ops.
 void populateEliminateBufferizeMaterializationsPatterns(
-    MLIRContext *context, BufferizeTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns);
+    BufferizeTypeConverter &typeConverter, RewritePatternSet &patterns);
 
 } // end namespace mlir
 
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 5cc5d8ae0586ec00049f96aa19a7236ac17999e1..d1bb6bc1033d5f4a8cbb68ca6420cb4790750371 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -13,7 +13,7 @@
 #ifndef MLIR_TRANSFORMS_DIALECTCONVERSION_H_
 #define MLIR_TRANSFORMS_DIALECTCONVERSION_H_
 
-#include "mlir/Rewrite/FrozenRewritePatternList.h"
+#include "mlir/Rewrite/FrozenRewritePatternSet.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringMap.h"
 
@@ -351,20 +351,12 @@ protected:
   /// See `RewritePattern::RewritePattern` for information on the other
   /// available constructors.
   using RewritePattern::RewritePattern;
-  /// Construct a conversion pattern that matches an operation with the given
-  /// root name. This constructor allows for providing a type converter to use
-  /// within the pattern.
-  ConversionPattern(StringRef rootName, PatternBenefit benefit,
-                    TypeConverter &typeConverter, MLIRContext *ctx)
-      : RewritePattern(rootName, benefit, ctx), typeConverter(&typeConverter) {}
-  /// Construct a conversion pattern that matches any operation type. This
-  /// constructor allows for providing a type converter to use within the
-  /// pattern. `MatchAnyOpTypeTag` is just a tag to ensure that the "match any"
-  /// behavior is what the user actually desired, `MatchAnyOpTypeTag()` should
-  /// always be supplied here.
-  ConversionPattern(PatternBenefit benefit, TypeConverter &typeConverter,
-                    MatchAnyOpTypeTag tag)
-      : RewritePattern(benefit, tag), typeConverter(&typeConverter) {}
+  /// Construct a conversion pattern with the given converter, and forward the
+  /// remaining arguments to RewritePattern.
+  template <typename... Args>
+  ConversionPattern(TypeConverter &typeConverter, Args &&... args)
+      : RewritePattern(std::forward<Args>(args)...),
+        typeConverter(&typeConverter) {}
 
 protected:
   /// An optional type converter for use by this pattern.
@@ -374,17 +366,13 @@ private:
   using RewritePattern::rewrite;
 };
 
-/// OpConversionPattern is a wrapper around ConversionPattern that allows for
-/// matching and rewriting against an instance of a derived operation class as
-/// opposed to a raw Operation.
+namespace detail {
+/// OpOrInterfaceConversionPatternBase is a wrapper around ConversionPattern
+/// that allows for matching and rewriting against an instance of a derived
+/// operation class or an Interface as opposed to a raw Operation.
 template <typename SourceOp>
-struct OpConversionPattern : public ConversionPattern {
-  OpConversionPattern(MLIRContext *context, PatternBenefit benefit = 1)
-      : ConversionPattern(SourceOp::getOperationName(), benefit, context) {}
-  OpConversionPattern(TypeConverter &typeConverter, MLIRContext *context,
-                      PatternBenefit benefit = 1)
-      : ConversionPattern(SourceOp::getOperationName(), benefit, typeConverter,
-                          context) {}
+struct OpOrInterfaceConversionPatternBase : public ConversionPattern {
+  using ConversionPattern::ConversionPattern;
 
   /// Wrappers around the ConversionPattern methods that pass the derived op
   /// type.
@@ -419,26 +407,57 @@ struct OpConversionPattern : public ConversionPattern {
 private:
   using ConversionPattern::matchAndRewrite;
 };
+} // namespace detail
+
+/// OpConversionPattern is a wrapper around ConversionPattern that allows for
+/// matching and rewriting against an instance of a derived operation class as
+/// opposed to a raw Operation.
+template <typename SourceOp>
+struct OpConversionPattern
+    : public detail::OpOrInterfaceConversionPatternBase<SourceOp> {
+  OpConversionPattern(MLIRContext *context, PatternBenefit benefit = 1)
+      : detail::OpOrInterfaceConversionPatternBase<SourceOp>(
+            SourceOp::getOperationName(), benefit, context) {}
+  OpConversionPattern(TypeConverter &typeConverter, MLIRContext *context,
+                      PatternBenefit benefit = 1)
+      : detail::OpOrInterfaceConversionPatternBase<SourceOp>(
+            typeConverter, SourceOp::getOperationName(), benefit, context) {}
+};
+
+/// OpInterfaceConversionPattern is a wrapper around ConversionPattern that
+/// allows for matching and rewriting against an instance of an OpInterface
+/// class as opposed to a raw Operation.
+template <typename SourceOp>
+struct OpInterfaceConversionPattern
+    : public detail::OpOrInterfaceConversionPatternBase<SourceOp> {
+  OpInterfaceConversionPattern(MLIRContext *context, PatternBenefit benefit = 1)
+      : detail::OpOrInterfaceConversionPatternBase<SourceOp>(
+            Pattern::MatchInterfaceOpTypeTag(), SourceOp::getInterfaceID(),
+            benefit, context) {}
+  OpInterfaceConversionPattern(TypeConverter &typeConverter,
+                               MLIRContext *context, PatternBenefit benefit = 1)
+      : detail::OpOrInterfaceConversionPatternBase<SourceOp>(
+            typeConverter, Pattern::MatchInterfaceOpTypeTag(),
+            SourceOp::getInterfaceID(), benefit, context) {}
+};
 
 /// Add a pattern to the given pattern list to convert the signature of a
 /// FunctionLike op with the given type converter. This only supports
 /// FunctionLike ops which use FunctionType to represent their type.
-void populateFunctionLikeTypeConversionPattern(
-    StringRef functionLikeOpName, OwningRewritePatternList &patterns,
-    MLIRContext *ctx, TypeConverter &converter);
+void populateFunctionLikeTypeConversionPattern(StringRef functionLikeOpName,
+                                               RewritePatternSet &patterns,
+                                               TypeConverter &converter);
 
 template <typename FuncOpT>
-void populateFunctionLikeTypeConversionPattern(
-    OwningRewritePatternList &patterns, MLIRContext *ctx,
-    TypeConverter &converter) {
+void populateFunctionLikeTypeConversionPattern(RewritePatternSet &patterns,
+                                               TypeConverter &converter) {
   populateFunctionLikeTypeConversionPattern(FuncOpT::getOperationName(),
-                                            patterns, ctx, converter);
+                                            patterns, converter);
 }
 
 /// Add a pattern to the given pattern list to convert the signature of a FuncOp
 /// with the given type converter.
-void populateFuncOpTypeConversionPattern(OwningRewritePatternList &patterns,
-                                         MLIRContext *ctx,
+void populateFuncOpTypeConversionPattern(RewritePatternSet &patterns,
                                          TypeConverter &converter);
 
 //===----------------------------------------------------------------------===//
@@ -604,22 +623,26 @@ public:
 
   /// Register a legality action for the given operation.
   void setOpAction(OperationName op, LegalizationAction action);
-  template <typename OpT> void setOpAction(LegalizationAction action) {
+  template <typename OpT>
+  void setOpAction(LegalizationAction action) {
     setOpAction(OperationName(OpT::getOperationName(), &ctx), action);
   }
 
   /// Register the given operations as legal.
-  template <typename OpT> void addLegalOp() {
+  template <typename OpT>
+  void addLegalOp() {
     setOpAction<OpT>(LegalizationAction::Legal);
   }
-  template <typename OpT, typename OpT2, typename... OpTs> void addLegalOp() {
+  template <typename OpT, typename OpT2, typename... OpTs>
+  void addLegalOp() {
     addLegalOp<OpT>();
     addLegalOp<OpT2, OpTs...>();
   }
 
   /// Register the given operation as dynamically legal, i.e. requiring custom
   /// handling by the target via 'isDynamicallyLegal'.
-  template <typename OpT> void addDynamicallyLegalOp() {
+  template <typename OpT>
+  void addDynamicallyLegalOp() {
     setOpAction<OpT>(LegalizationAction::Dynamic);
   }
   template <typename OpT, typename OpT2, typename... OpTs>
@@ -651,10 +674,12 @@ public:
 
   /// Register the given operation as illegal, i.e. this operation is known to
   /// not be supported by this target.
-  template <typename OpT> void addIllegalOp() {
+  template <typename OpT>
+  void addIllegalOp() {
     setOpAction<OpT>(LegalizationAction::Illegal);
   }
-  template <typename OpT, typename OpT2, typename... OpTs> void addIllegalOp() {
+  template <typename OpT, typename OpT2, typename... OpTs>
+  void addIllegalOp() {
     addIllegalOp<OpT>();
     addIllegalOp<OpT2, OpTs...>();
   }
@@ -692,7 +717,8 @@ public:
     SmallVector<StringRef, 2> dialectNames({name, names...});
     setDialectAction(dialectNames, LegalizationAction::Legal);
   }
-  template <typename... Args> void addLegalDialect() {
+  template <typename... Args>
+  void addLegalDialect() {
     SmallVector<StringRef, 2> dialectNames({Args::getDialectNamespace()...});
     setDialectAction(dialectNames, LegalizationAction::Legal);
   }
@@ -736,7 +762,8 @@ public:
     SmallVector<StringRef, 2> dialectNames({name, names...});
     setDialectAction(dialectNames, LegalizationAction::Illegal);
   }
-  template <typename... Args> void addIllegalDialect() {
+  template <typename... Args>
+  void addIllegalDialect() {
     SmallVector<StringRef, 2> dialectNames({Args::getDialectNamespace()...});
     setDialectAction(dialectNames, LegalizationAction::Illegal);
   }
@@ -836,11 +863,11 @@ private:
 /// the `unconvertedOps` set will not necessarily be complete.)
 LogicalResult
 applyPartialConversion(ArrayRef<Operation *> ops, ConversionTarget &target,
-                       const FrozenRewritePatternList &patterns,
+                       const FrozenRewritePatternSet &patterns,
                        DenseSet<Operation *> *unconvertedOps = nullptr);
 LogicalResult
 applyPartialConversion(Operation *op, ConversionTarget &target,
-                       const FrozenRewritePatternList &patterns,
+                       const FrozenRewritePatternSet &patterns,
                        DenseSet<Operation *> *unconvertedOps = nullptr);
 
 /// Apply a complete conversion on the given operations, and all nested
@@ -849,9 +876,9 @@ applyPartialConversion(Operation *op, ConversionTarget &target,
 /// within 'ops'.
 LogicalResult applyFullConversion(ArrayRef<Operation *> ops,
                                   ConversionTarget &target,
-                                  const FrozenRewritePatternList &patterns);
+                                  const FrozenRewritePatternSet &patterns);
 LogicalResult applyFullConversion(Operation *op, ConversionTarget &target,
-                                  const FrozenRewritePatternList &patterns);
+                                  const FrozenRewritePatternSet &patterns);
 
 /// Apply an analysis conversion on the given operations, and all nested
 /// operations. This method analyzes which operations would be successfully
@@ -863,10 +890,10 @@ LogicalResult applyFullConversion(Operation *op, ConversionTarget &target,
 /// the regions nested within 'ops'.
 LogicalResult applyAnalysisConversion(ArrayRef<Operation *> ops,
                                       ConversionTarget &target,
-                                      const FrozenRewritePatternList &patterns,
+                                      const FrozenRewritePatternSet &patterns,
                                       DenseSet<Operation *> &convertedOps);
 LogicalResult applyAnalysisConversion(Operation *op, ConversionTarget &target,
-                                      const FrozenRewritePatternList &patterns,
+                                      const FrozenRewritePatternSet &patterns,
                                       DenseSet<Operation *> &convertedOps);
 } // end namespace mlir
 
diff --git a/mlir/include/mlir/Transforms/FoldUtils.h b/mlir/include/mlir/Transforms/FoldUtils.h
index ad406cb18085d97d5010cddfd0f24c2a8e930b34..7f4166c12ded9e809dd0083c0a1b4ea3ee36c1e9 100644
--- a/mlir/include/mlir/Transforms/FoldUtils.h
+++ b/mlir/include/mlir/Transforms/FoldUtils.h
@@ -23,7 +23,6 @@ namespace mlir {
 class Operation;
 class Value;
 
-
 //===--------------------------------------------------------------------===//
 // OperationFolder
 //===--------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h b/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
index 4a084c57b6edf1fb0c12d859bbdc4ec5314d9cb8..3a76fbd3e0b02fdc2dd10df52c1c19a632e8c6e4 100644
--- a/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
+++ b/mlir/include/mlir/Transforms/GreedyPatternRewriteDriver.h
@@ -14,7 +14,7 @@
 #ifndef MLIR_TRANSFORMS_GREEDYPATTERNREWRITEDRIVER_H_
 #define MLIR_TRANSFORMS_GREEDYPATTERNREWRITEDRIVER_H_
 
-#include "mlir/Rewrite/FrozenRewritePatternList.h"
+#include "mlir/Rewrite/FrozenRewritePatternSet.h"
 
 namespace mlir {
 
@@ -35,25 +35,25 @@ namespace mlir {
 ///       before attempting to match any of the provided patterns.
 LogicalResult
 applyPatternsAndFoldGreedily(Operation *op,
-                             const FrozenRewritePatternList &patterns);
+                             const FrozenRewritePatternSet &patterns);
 
 /// Rewrite the regions of the specified operation, with a user-provided limit
 /// on iterations to attempt before reaching convergence.
 LogicalResult
 applyPatternsAndFoldGreedily(Operation *op,
-                             const FrozenRewritePatternList &patterns,
+                             const FrozenRewritePatternSet &patterns,
                              unsigned maxIterations);
 
 /// Rewrite the given regions, which must be isolated from above.
 LogicalResult
 applyPatternsAndFoldGreedily(MutableArrayRef<Region> regions,
-                             const FrozenRewritePatternList &patterns);
+                             const FrozenRewritePatternSet &patterns);
 
 /// Rewrite the given regions, with a user-provided limit on iterations to
 /// attempt before reaching convergence.
 LogicalResult
 applyPatternsAndFoldGreedily(MutableArrayRef<Region> regions,
-                             const FrozenRewritePatternList &patterns,
+                             const FrozenRewritePatternSet &patterns,
                              unsigned maxIterations);
 
 /// Applies the specified patterns on `op` alone while also trying to fold it,
@@ -62,7 +62,7 @@ applyPatternsAndFoldGreedily(MutableArrayRef<Region> regions,
 /// was folded away or erased as a result of becoming dead. Note: This does not
 /// apply any patterns recursively to the regions of `op`.
 LogicalResult applyOpPatternsAndFold(Operation *op,
-                                     const FrozenRewritePatternList &patterns,
+                                     const FrozenRewritePatternSet &patterns,
                                      bool *erased = nullptr);
 
 } // end namespace mlir
diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index 60ea4b188ae162b0b498b4305343fd36c0905efc..1d4234b38efca5815c2c8a2b17e00bd27bbf09ae 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -63,9 +63,6 @@ std::unique_ptr<Pass> createBufferResultsToOutParamsPass();
 /// Creates an instance of the Canonicalizer pass.
 std::unique_ptr<Pass> createCanonicalizerPass();
 
-/// Create a pass that removes unnecessary Copy operations.
-std::unique_ptr<Pass> createCopyRemovalPass();
-
 /// Creates a pass to perform common sub expression elimination.
 std::unique_ptr<Pass> createCSEPass();
 
diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td
index 2305c4a391912a202ef33f660fdca9831331ed93..0e14dcb873e73d1db4a6142e711cc4b3069b04c4 100644
--- a/mlir/include/mlir/Transforms/Passes.td
+++ b/mlir/include/mlir/Transforms/Passes.td
@@ -282,8 +282,6 @@ def BufferDeallocation : FunctionPass<"buffer-deallocation"> {
 
   }];
   let constructor = "mlir::createBufferDeallocationPass()";
-  // TODO: this pass likely shouldn't depend on Linalg?
-  let dependentDialects = ["linalg::LinalgDialect"];
 }
 
 def BufferHoisting : FunctionPass<"buffer-hoisting"> {
@@ -366,11 +364,6 @@ def Canonicalizer : Pass<"canonicalize"> {
   let dependentDialects = ["memref::MemRefDialect"];
 }
 
-def CopyRemoval : FunctionPass<"copy-removal"> {
-  let summary = "Remove the redundant copies from input IR";
-  let constructor = "mlir::createCopyRemovalPass()";
-}
-
 def CSE : Pass<"cse"> {
   let summary = "Eliminate common sub-expressions";
   let description = [{
diff --git a/mlir/include/mlir/Transforms/RegionUtils.h b/mlir/include/mlir/Transforms/RegionUtils.h
index 72c2f51c9e702a1f765654d453c62efe3591bbd1..c2124d8b70f03001b0010dd0374ff69282425e43 100644
--- a/mlir/include/mlir/Transforms/RegionUtils.h
+++ b/mlir/include/mlir/Transforms/RegionUtils.h
@@ -15,6 +15,7 @@
 #include "llvm/ADT/SetVector.h"
 
 namespace mlir {
+class RewriterBase;
 
 /// Check if all values in the provided range are defined above the `limit`
 /// region.  That is, if they are defined in a region that is a proper ancestor
@@ -53,8 +54,10 @@ void getUsedValuesDefinedAbove(MutableArrayRef<Region> regions,
 /// Run a set of structural simplifications over the given regions. This
 /// includes transformations like unreachable block elimination, dead argument
 /// elimination, as well as some other DCE. This function returns success if any
-/// of the regions were simplified, failure otherwise.
-LogicalResult simplifyRegions(MutableArrayRef<Region> regions);
+/// of the regions were simplified, failure otherwise. The provided rewriter is
+/// used to notify callers of operation and block deletion.
+LogicalResult simplifyRegions(RewriterBase &rewriter,
+                              MutableArrayRef<Region> regions);
 
 } // namespace mlir
 
diff --git a/mlir/lib/Bindings/Python/CMakeLists.txt b/mlir/lib/Bindings/Python/CMakeLists.txt
index 5f042ec57c290be1e8f7f5aadf0e4b2eabb990d9..43d6275d4d20c2c8eb73a05cde556c1d9dd697d0 100644
--- a/mlir/lib/Bindings/Python/CMakeLists.txt
+++ b/mlir/lib/Bindings/Python/CMakeLists.txt
@@ -70,7 +70,10 @@ add_mlir_python_extension(MLIRCoreBindingsPythonExtension _mlir
     python
   SOURCES
     MainModule.cpp
-    IRModules.cpp
+    IRAffine.cpp
+    IRAttributes.cpp
+    IRCore.cpp
+    IRTypes.cpp
     PybindUtils.cpp
     Pass.cpp
     ExecutionEngine.cpp
@@ -115,3 +118,11 @@ endif()
 
 add_subdirectory(Transforms)
 add_subdirectory(Conversions)
+
+add_mlir_python_extension(MLIRLinalgPassesBindingsPythonExtension _mlirLinalgPasses
+  INSTALL_DIR
+    python
+  SOURCES
+    LinalgPasses.cpp
+)
+add_dependencies(MLIRBindingsPythonExtension MLIRLinalgPassesBindingsPythonExtension)
diff --git a/mlir/lib/Bindings/Python/ExecutionEngine.cpp b/mlir/lib/Bindings/Python/ExecutionEngine.cpp
index f6f52e2e0aaee1e88df244dbc1f37d817066e63e..5ca9b1f681286f5c9593813b85c5d17693a3942b 100644
--- a/mlir/lib/Bindings/Python/ExecutionEngine.cpp
+++ b/mlir/lib/Bindings/Python/ExecutionEngine.cpp
@@ -8,7 +8,7 @@
 
 #include "ExecutionEngine.h"
 
-#include "IRModules.h"
+#include "IRModule.h"
 #include "mlir-c/Bindings/Python/Interop.h"
 #include "mlir-c/ExecutionEngine.h"
 
diff --git a/mlir/lib/Bindings/Python/IRAffine.cpp b/mlir/lib/Bindings/Python/IRAffine.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..73a57d95e15866057d87641f09e055bab946b6d0
--- /dev/null
+++ b/mlir/lib/Bindings/Python/IRAffine.cpp
@@ -0,0 +1,781 @@
+//===- IRAffine.cpp - Exports 'ir' module affine related bindings ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IRModule.h"
+
+#include "PybindUtils.h"
+
+#include "mlir-c/AffineMap.h"
+#include "mlir-c/Bindings/Python/Interop.h"
+#include "mlir-c/IntegerSet.h"
+
+namespace py = pybind11;
+using namespace mlir;
+using namespace mlir::python;
+
+using llvm::SmallVector;
+using llvm::StringRef;
+using llvm::Twine;
+
+static const char kDumpDocstring[] =
+    R"(Dumps a debug representation of the object to stderr.)";
+
+/// Attempts to populate `result` with the content of `list` casted to the
+/// appropriate type (Python and C types are provided as template arguments).
+/// Throws errors in case of failure, using "action" to describe what the caller
+/// was attempting to do.
+template <typename PyType, typename CType>
+static void pyListToVector(py::list list, llvm::SmallVectorImpl<CType> &result,
+                           StringRef action) {
+  result.reserve(py::len(list));
+  for (py::handle item : list) {
+    try {
+      result.push_back(item.cast<PyType>());
+    } catch (py::cast_error &err) {
+      std::string msg = (llvm::Twine("Invalid expression when ") + action +
+                         " (" + err.what() + ")")
+                            .str();
+      throw py::cast_error(msg);
+    } catch (py::reference_cast_error &err) {
+      std::string msg = (llvm::Twine("Invalid expression (None?) when ") +
+                         action + " (" + err.what() + ")")
+                            .str();
+      throw py::cast_error(msg);
+    }
+  }
+}
+
+template <typename PermutationTy>
+static bool isPermutation(std::vector<PermutationTy> permutation) {
+  llvm::SmallVector<bool, 8> seen(permutation.size(), false);
+  for (auto val : permutation) {
+    if (val < permutation.size()) {
+      if (seen[val])
+        return false;
+      seen[val] = true;
+      continue;
+    }
+    return false;
+  }
+  return true;
+}
+
+namespace {
+
+/// CRTP base class for Python MLIR affine expressions that subclass AffineExpr
+/// and should be castable from it. Intermediate hierarchy classes can be
+/// modeled by specifying BaseTy.
+template <typename DerivedTy, typename BaseTy = PyAffineExpr>
+class PyConcreteAffineExpr : public BaseTy {
+public:
+  // Derived classes must define statics for:
+  //   IsAFunctionTy isaFunction
+  //   const char *pyClassName
+  // and redefine bindDerived.
+  using ClassTy = py::class_<DerivedTy, BaseTy>;
+  using IsAFunctionTy = bool (*)(MlirAffineExpr);
+
+  PyConcreteAffineExpr() = default;
+  PyConcreteAffineExpr(PyMlirContextRef contextRef, MlirAffineExpr affineExpr)
+      : BaseTy(std::move(contextRef), affineExpr) {}
+  PyConcreteAffineExpr(PyAffineExpr &orig)
+      : PyConcreteAffineExpr(orig.getContext(), castFrom(orig)) {}
+
+  static MlirAffineExpr castFrom(PyAffineExpr &orig) {
+    if (!DerivedTy::isaFunction(orig)) {
+      auto origRepr = py::repr(py::cast(orig)).cast<std::string>();
+      throw SetPyError(PyExc_ValueError,
+                       Twine("Cannot cast affine expression to ") +
+                           DerivedTy::pyClassName + " (from " + origRepr + ")");
+    }
+    return orig;
+  }
+
+  static void bind(py::module &m) {
+    auto cls = ClassTy(m, DerivedTy::pyClassName);
+    cls.def(py::init<PyAffineExpr &>());
+    DerivedTy::bindDerived(cls);
+  }
+
+  /// Implemented by derived classes to add methods to the Python subclass.
+  static void bindDerived(ClassTy &m) {}
+};
+
+class PyAffineConstantExpr : public PyConcreteAffineExpr<PyAffineConstantExpr> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAConstant;
+  static constexpr const char *pyClassName = "AffineConstantExpr";
+  using PyConcreteAffineExpr::PyConcreteAffineExpr;
+
+  static PyAffineConstantExpr get(intptr_t value,
+                                  DefaultingPyMlirContext context) {
+    MlirAffineExpr affineExpr =
+        mlirAffineConstantExprGet(context->get(), static_cast<int64_t>(value));
+    return PyAffineConstantExpr(context->getRef(), affineExpr);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static("get", &PyAffineConstantExpr::get, py::arg("value"),
+                 py::arg("context") = py::none());
+    c.def_property_readonly("value", [](PyAffineConstantExpr &self) {
+      return mlirAffineConstantExprGetValue(self);
+    });
+  }
+};
+
+class PyAffineDimExpr : public PyConcreteAffineExpr<PyAffineDimExpr> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsADim;
+  static constexpr const char *pyClassName = "AffineDimExpr";
+  using PyConcreteAffineExpr::PyConcreteAffineExpr;
+
+  static PyAffineDimExpr get(intptr_t pos, DefaultingPyMlirContext context) {
+    MlirAffineExpr affineExpr = mlirAffineDimExprGet(context->get(), pos);
+    return PyAffineDimExpr(context->getRef(), affineExpr);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static("get", &PyAffineDimExpr::get, py::arg("position"),
+                 py::arg("context") = py::none());
+    c.def_property_readonly("position", [](PyAffineDimExpr &self) {
+      return mlirAffineDimExprGetPosition(self);
+    });
+  }
+};
+
+class PyAffineSymbolExpr : public PyConcreteAffineExpr<PyAffineSymbolExpr> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsASymbol;
+  static constexpr const char *pyClassName = "AffineSymbolExpr";
+  using PyConcreteAffineExpr::PyConcreteAffineExpr;
+
+  static PyAffineSymbolExpr get(intptr_t pos, DefaultingPyMlirContext context) {
+    MlirAffineExpr affineExpr = mlirAffineSymbolExprGet(context->get(), pos);
+    return PyAffineSymbolExpr(context->getRef(), affineExpr);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static("get", &PyAffineSymbolExpr::get, py::arg("position"),
+                 py::arg("context") = py::none());
+    c.def_property_readonly("position", [](PyAffineSymbolExpr &self) {
+      return mlirAffineSymbolExprGetPosition(self);
+    });
+  }
+};
+
+class PyAffineBinaryExpr : public PyConcreteAffineExpr<PyAffineBinaryExpr> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsABinary;
+  static constexpr const char *pyClassName = "AffineBinaryExpr";
+  using PyConcreteAffineExpr::PyConcreteAffineExpr;
+
+  PyAffineExpr lhs() {
+    MlirAffineExpr lhsExpr = mlirAffineBinaryOpExprGetLHS(get());
+    return PyAffineExpr(getContext(), lhsExpr);
+  }
+
+  PyAffineExpr rhs() {
+    MlirAffineExpr rhsExpr = mlirAffineBinaryOpExprGetRHS(get());
+    return PyAffineExpr(getContext(), rhsExpr);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def_property_readonly("lhs", &PyAffineBinaryExpr::lhs);
+    c.def_property_readonly("rhs", &PyAffineBinaryExpr::rhs);
+  }
+};
+
+class PyAffineAddExpr
+    : public PyConcreteAffineExpr<PyAffineAddExpr, PyAffineBinaryExpr> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAAdd;
+  static constexpr const char *pyClassName = "AffineAddExpr";
+  using PyConcreteAffineExpr::PyConcreteAffineExpr;
+
+  static PyAffineAddExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
+    MlirAffineExpr expr = mlirAffineAddExprGet(lhs, rhs);
+    return PyAffineAddExpr(lhs.getContext(), expr);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static("get", &PyAffineAddExpr::get);
+  }
+};
+
+class PyAffineMulExpr
+    : public PyConcreteAffineExpr<PyAffineMulExpr, PyAffineBinaryExpr> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAMul;
+  static constexpr const char *pyClassName = "AffineMulExpr";
+  using PyConcreteAffineExpr::PyConcreteAffineExpr;
+
+  static PyAffineMulExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
+    MlirAffineExpr expr = mlirAffineMulExprGet(lhs, rhs);
+    return PyAffineMulExpr(lhs.getContext(), expr);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static("get", &PyAffineMulExpr::get);
+  }
+};
+
+class PyAffineModExpr
+    : public PyConcreteAffineExpr<PyAffineModExpr, PyAffineBinaryExpr> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAMod;
+  static constexpr const char *pyClassName = "AffineModExpr";
+  using PyConcreteAffineExpr::PyConcreteAffineExpr;
+
+  static PyAffineModExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
+    MlirAffineExpr expr = mlirAffineModExprGet(lhs, rhs);
+    return PyAffineModExpr(lhs.getContext(), expr);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static("get", &PyAffineModExpr::get);
+  }
+};
+
+class PyAffineFloorDivExpr
+    : public PyConcreteAffineExpr<PyAffineFloorDivExpr, PyAffineBinaryExpr> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAFloorDiv;
+  static constexpr const char *pyClassName = "AffineFloorDivExpr";
+  using PyConcreteAffineExpr::PyConcreteAffineExpr;
+
+  static PyAffineFloorDivExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
+    MlirAffineExpr expr = mlirAffineFloorDivExprGet(lhs, rhs);
+    return PyAffineFloorDivExpr(lhs.getContext(), expr);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static("get", &PyAffineFloorDivExpr::get);
+  }
+};
+
+class PyAffineCeilDivExpr
+    : public PyConcreteAffineExpr<PyAffineCeilDivExpr, PyAffineBinaryExpr> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsACeilDiv;
+  static constexpr const char *pyClassName = "AffineCeilDivExpr";
+  using PyConcreteAffineExpr::PyConcreteAffineExpr;
+
+  static PyAffineCeilDivExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
+    MlirAffineExpr expr = mlirAffineCeilDivExprGet(lhs, rhs);
+    return PyAffineCeilDivExpr(lhs.getContext(), expr);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static("get", &PyAffineCeilDivExpr::get);
+  }
+};
+
+} // namespace
+
+bool PyAffineExpr::operator==(const PyAffineExpr &other) {
+  return mlirAffineExprEqual(affineExpr, other.affineExpr);
+}
+
+py::object PyAffineExpr::getCapsule() {
+  return py::reinterpret_steal<py::object>(
+      mlirPythonAffineExprToCapsule(*this));
+}
+
+PyAffineExpr PyAffineExpr::createFromCapsule(py::object capsule) {
+  MlirAffineExpr rawAffineExpr = mlirPythonCapsuleToAffineExpr(capsule.ptr());
+  if (mlirAffineExprIsNull(rawAffineExpr))
+    throw py::error_already_set();
+  return PyAffineExpr(
+      PyMlirContext::forContext(mlirAffineExprGetContext(rawAffineExpr)),
+      rawAffineExpr);
+}
+
+//------------------------------------------------------------------------------
+// PyAffineMap and utilities.
+//------------------------------------------------------------------------------
+namespace {
+
+/// A list of expressions contained in an affine map. Internally these are
+/// stored as a consecutive array leading to inexpensive random access. Both
+/// the map and the expression are owned by the context so we need not bother
+/// with lifetime extension.
+class PyAffineMapExprList
+    : public Sliceable<PyAffineMapExprList, PyAffineExpr> {
+public:
+  static constexpr const char *pyClassName = "AffineExprList";
+
+  PyAffineMapExprList(PyAffineMap map, intptr_t startIndex = 0,
+                      intptr_t length = -1, intptr_t step = 1)
+      : Sliceable(startIndex,
+                  length == -1 ? mlirAffineMapGetNumResults(map) : length,
+                  step),
+        affineMap(map) {}
+
+  intptr_t getNumElements() { return mlirAffineMapGetNumResults(affineMap); }
+
+  PyAffineExpr getElement(intptr_t pos) {
+    return PyAffineExpr(affineMap.getContext(),
+                        mlirAffineMapGetResult(affineMap, pos));
+  }
+
+  PyAffineMapExprList slice(intptr_t startIndex, intptr_t length,
+                            intptr_t step) {
+    return PyAffineMapExprList(affineMap, startIndex, length, step);
+  }
+
+private:
+  PyAffineMap affineMap;
+};
+} // end namespace
+
+bool PyAffineMap::operator==(const PyAffineMap &other) {
+  return mlirAffineMapEqual(affineMap, other.affineMap);
+}
+
+py::object PyAffineMap::getCapsule() {
+  return py::reinterpret_steal<py::object>(mlirPythonAffineMapToCapsule(*this));
+}
+
+PyAffineMap PyAffineMap::createFromCapsule(py::object capsule) {
+  MlirAffineMap rawAffineMap = mlirPythonCapsuleToAffineMap(capsule.ptr());
+  if (mlirAffineMapIsNull(rawAffineMap))
+    throw py::error_already_set();
+  return PyAffineMap(
+      PyMlirContext::forContext(mlirAffineMapGetContext(rawAffineMap)),
+      rawAffineMap);
+}
+
+//------------------------------------------------------------------------------
+// PyIntegerSet and utilities.
+//------------------------------------------------------------------------------
+namespace {
+
+class PyIntegerSetConstraint {
+public:
+  PyIntegerSetConstraint(PyIntegerSet set, intptr_t pos) : set(set), pos(pos) {}
+
+  PyAffineExpr getExpr() {
+    return PyAffineExpr(set.getContext(),
+                        mlirIntegerSetGetConstraint(set, pos));
+  }
+
+  bool isEq() { return mlirIntegerSetIsConstraintEq(set, pos); }
+
+  static void bind(py::module &m) {
+    py::class_<PyIntegerSetConstraint>(m, "IntegerSetConstraint")
+        .def_property_readonly("expr", &PyIntegerSetConstraint::getExpr)
+        .def_property_readonly("is_eq", &PyIntegerSetConstraint::isEq);
+  }
+
+private:
+  PyIntegerSet set;
+  intptr_t pos;
+};
+
+class PyIntegerSetConstraintList
+    : public Sliceable<PyIntegerSetConstraintList, PyIntegerSetConstraint> {
+public:
+  static constexpr const char *pyClassName = "IntegerSetConstraintList";
+
+  PyIntegerSetConstraintList(PyIntegerSet set, intptr_t startIndex = 0,
+                             intptr_t length = -1, intptr_t step = 1)
+      : Sliceable(startIndex,
+                  length == -1 ? mlirIntegerSetGetNumConstraints(set) : length,
+                  step),
+        set(set) {}
+
+  intptr_t getNumElements() { return mlirIntegerSetGetNumConstraints(set); }
+
+  PyIntegerSetConstraint getElement(intptr_t pos) {
+    return PyIntegerSetConstraint(set, pos);
+  }
+
+  PyIntegerSetConstraintList slice(intptr_t startIndex, intptr_t length,
+                                   intptr_t step) {
+    return PyIntegerSetConstraintList(set, startIndex, length, step);
+  }
+
+private:
+  PyIntegerSet set;
+};
+} // namespace
+
+bool PyIntegerSet::operator==(const PyIntegerSet &other) {
+  return mlirIntegerSetEqual(integerSet, other.integerSet);
+}
+
+py::object PyIntegerSet::getCapsule() {
+  return py::reinterpret_steal<py::object>(
+      mlirPythonIntegerSetToCapsule(*this));
+}
+
+PyIntegerSet PyIntegerSet::createFromCapsule(py::object capsule) {
+  MlirIntegerSet rawIntegerSet = mlirPythonCapsuleToIntegerSet(capsule.ptr());
+  if (mlirIntegerSetIsNull(rawIntegerSet))
+    throw py::error_already_set();
+  return PyIntegerSet(
+      PyMlirContext::forContext(mlirIntegerSetGetContext(rawIntegerSet)),
+      rawIntegerSet);
+}
+
+void mlir::python::populateIRAffine(py::module &m) {
+  //----------------------------------------------------------------------------
+  // Mapping of PyAffineExpr and derived classes.
+  //----------------------------------------------------------------------------
+  py::class_<PyAffineExpr>(m, "AffineExpr")
+      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR,
+                             &PyAffineExpr::getCapsule)
+      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyAffineExpr::createFromCapsule)
+      .def("__add__",
+           [](PyAffineExpr &self, PyAffineExpr &other) {
+             return PyAffineAddExpr::get(self, other);
+           })
+      .def("__mul__",
+           [](PyAffineExpr &self, PyAffineExpr &other) {
+             return PyAffineMulExpr::get(self, other);
+           })
+      .def("__mod__",
+           [](PyAffineExpr &self, PyAffineExpr &other) {
+             return PyAffineModExpr::get(self, other);
+           })
+      .def("__sub__",
+           [](PyAffineExpr &self, PyAffineExpr &other) {
+             auto negOne =
+                 PyAffineConstantExpr::get(-1, *self.getContext().get());
+             return PyAffineAddExpr::get(self,
+                                         PyAffineMulExpr::get(negOne, other));
+           })
+      .def("__eq__", [](PyAffineExpr &self,
+                        PyAffineExpr &other) { return self == other; })
+      .def("__eq__",
+           [](PyAffineExpr &self, py::object &other) { return false; })
+      .def("__str__",
+           [](PyAffineExpr &self) {
+             PyPrintAccumulator printAccum;
+             mlirAffineExprPrint(self, printAccum.getCallback(),
+                                 printAccum.getUserData());
+             return printAccum.join();
+           })
+      .def("__repr__",
+           [](PyAffineExpr &self) {
+             PyPrintAccumulator printAccum;
+             printAccum.parts.append("AffineExpr(");
+             mlirAffineExprPrint(self, printAccum.getCallback(),
+                                 printAccum.getUserData());
+             printAccum.parts.append(")");
+             return printAccum.join();
+           })
+      .def_property_readonly(
+          "context",
+          [](PyAffineExpr &self) { return self.getContext().getObject(); })
+      .def_static(
+          "get_add", &PyAffineAddExpr::get,
+          "Gets an affine expression containing a sum of two expressions.")
+      .def_static(
+          "get_mul", &PyAffineMulExpr::get,
+          "Gets an affine expression containing a product of two expressions.")
+      .def_static("get_mod", &PyAffineModExpr::get,
+                  "Gets an affine expression containing the modulo of dividing "
+                  "one expression by another.")
+      .def_static("get_floor_div", &PyAffineFloorDivExpr::get,
+                  "Gets an affine expression containing the rounded-down "
+                  "result of dividing one expression by another.")
+      .def_static("get_ceil_div", &PyAffineCeilDivExpr::get,
+                  "Gets an affine expression containing the rounded-up result "
+                  "of dividing one expression by another.")
+      .def_static("get_constant", &PyAffineConstantExpr::get, py::arg("value"),
+                  py::arg("context") = py::none(),
+                  "Gets a constant affine expression with the given value.")
+      .def_static(
+          "get_dim", &PyAffineDimExpr::get, py::arg("position"),
+          py::arg("context") = py::none(),
+          "Gets an affine expression of a dimension at the given position.")
+      .def_static(
+          "get_symbol", &PyAffineSymbolExpr::get, py::arg("position"),
+          py::arg("context") = py::none(),
+          "Gets an affine expression of a symbol at the given position.")
+      .def(
+          "dump", [](PyAffineExpr &self) { mlirAffineExprDump(self); },
+          kDumpDocstring);
+  PyAffineConstantExpr::bind(m);
+  PyAffineDimExpr::bind(m);
+  PyAffineSymbolExpr::bind(m);
+  PyAffineBinaryExpr::bind(m);
+  PyAffineAddExpr::bind(m);
+  PyAffineMulExpr::bind(m);
+  PyAffineModExpr::bind(m);
+  PyAffineFloorDivExpr::bind(m);
+  PyAffineCeilDivExpr::bind(m);
+
+  //----------------------------------------------------------------------------
+  // Mapping of PyAffineMap.
+  //----------------------------------------------------------------------------
+  py::class_<PyAffineMap>(m, "AffineMap")
+      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR,
+                             &PyAffineMap::getCapsule)
+      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyAffineMap::createFromCapsule)
+      .def("__eq__",
+           [](PyAffineMap &self, PyAffineMap &other) { return self == other; })
+      .def("__eq__", [](PyAffineMap &self, py::object &other) { return false; })
+      .def("__str__",
+           [](PyAffineMap &self) {
+             PyPrintAccumulator printAccum;
+             mlirAffineMapPrint(self, printAccum.getCallback(),
+                                printAccum.getUserData());
+             return printAccum.join();
+           })
+      .def("__repr__",
+           [](PyAffineMap &self) {
+             PyPrintAccumulator printAccum;
+             printAccum.parts.append("AffineMap(");
+             mlirAffineMapPrint(self, printAccum.getCallback(),
+                                printAccum.getUserData());
+             printAccum.parts.append(")");
+             return printAccum.join();
+           })
+      .def_property_readonly(
+          "context",
+          [](PyAffineMap &self) { return self.getContext().getObject(); },
+          "Context that owns the Affine Map")
+      .def(
+          "dump", [](PyAffineMap &self) { mlirAffineMapDump(self); },
+          kDumpDocstring)
+      .def_static(
+          "get",
+          [](intptr_t dimCount, intptr_t symbolCount, py::list exprs,
+             DefaultingPyMlirContext context) {
+            SmallVector<MlirAffineExpr> affineExprs;
+            pyListToVector<PyAffineExpr, MlirAffineExpr>(
+                exprs, affineExprs, "attempting to create an AffineMap");
+            MlirAffineMap map =
+                mlirAffineMapGet(context->get(), dimCount, symbolCount,
+                                 affineExprs.size(), affineExprs.data());
+            return PyAffineMap(context->getRef(), map);
+          },
+          py::arg("dim_count"), py::arg("symbol_count"), py::arg("exprs"),
+          py::arg("context") = py::none(),
+          "Gets a map with the given expressions as results.")
+      .def_static(
+          "get_constant",
+          [](intptr_t value, DefaultingPyMlirContext context) {
+            MlirAffineMap affineMap =
+                mlirAffineMapConstantGet(context->get(), value);
+            return PyAffineMap(context->getRef(), affineMap);
+          },
+          py::arg("value"), py::arg("context") = py::none(),
+          "Gets an affine map with a single constant result")
+      .def_static(
+          "get_empty",
+          [](DefaultingPyMlirContext context) {
+            MlirAffineMap affineMap = mlirAffineMapEmptyGet(context->get());
+            return PyAffineMap(context->getRef(), affineMap);
+          },
+          py::arg("context") = py::none(), "Gets an empty affine map.")
+      .def_static(
+          "get_identity",
+          [](intptr_t nDims, DefaultingPyMlirContext context) {
+            MlirAffineMap affineMap =
+                mlirAffineMapMultiDimIdentityGet(context->get(), nDims);
+            return PyAffineMap(context->getRef(), affineMap);
+          },
+          py::arg("n_dims"), py::arg("context") = py::none(),
+          "Gets an identity map with the given number of dimensions.")
+      .def_static(
+          "get_minor_identity",
+          [](intptr_t nDims, intptr_t nResults,
+             DefaultingPyMlirContext context) {
+            MlirAffineMap affineMap =
+                mlirAffineMapMinorIdentityGet(context->get(), nDims, nResults);
+            return PyAffineMap(context->getRef(), affineMap);
+          },
+          py::arg("n_dims"), py::arg("n_results"),
+          py::arg("context") = py::none(),
+          "Gets a minor identity map with the given number of dimensions and "
+          "results.")
+      .def_static(
+          "get_permutation",
+          [](std::vector<unsigned> permutation,
+             DefaultingPyMlirContext context) {
+            if (!isPermutation(permutation))
+              throw py::cast_error("Invalid permutation when attempting to "
+                                   "create an AffineMap");
+            MlirAffineMap affineMap = mlirAffineMapPermutationGet(
+                context->get(), permutation.size(), permutation.data());
+            return PyAffineMap(context->getRef(), affineMap);
+          },
+          py::arg("permutation"), py::arg("context") = py::none(),
+          "Gets an affine map that permutes its inputs.")
+      .def("get_submap",
+           [](PyAffineMap &self, std::vector<intptr_t> &resultPos) {
+             intptr_t numResults = mlirAffineMapGetNumResults(self);
+             for (intptr_t pos : resultPos) {
+               if (pos < 0 || pos >= numResults)
+                 throw py::value_error("result position out of bounds");
+             }
+             MlirAffineMap affineMap = mlirAffineMapGetSubMap(
+                 self, resultPos.size(), resultPos.data());
+             return PyAffineMap(self.getContext(), affineMap);
+           })
+      .def("get_major_submap",
+           [](PyAffineMap &self, intptr_t nResults) {
+             if (nResults >= mlirAffineMapGetNumResults(self))
+               throw py::value_error("number of results out of bounds");
+             MlirAffineMap affineMap =
+                 mlirAffineMapGetMajorSubMap(self, nResults);
+             return PyAffineMap(self.getContext(), affineMap);
+           })
+      .def("get_minor_submap",
+           [](PyAffineMap &self, intptr_t nResults) {
+             if (nResults >= mlirAffineMapGetNumResults(self))
+               throw py::value_error("number of results out of bounds");
+             MlirAffineMap affineMap =
+                 mlirAffineMapGetMinorSubMap(self, nResults);
+             return PyAffineMap(self.getContext(), affineMap);
+           })
+      .def_property_readonly(
+          "is_permutation",
+          [](PyAffineMap &self) { return mlirAffineMapIsPermutation(self); })
+      .def_property_readonly("is_projected_permutation",
+                             [](PyAffineMap &self) {
+                               return mlirAffineMapIsProjectedPermutation(self);
+                             })
+      .def_property_readonly(
+          "n_dims",
+          [](PyAffineMap &self) { return mlirAffineMapGetNumDims(self); })
+      .def_property_readonly(
+          "n_inputs",
+          [](PyAffineMap &self) { return mlirAffineMapGetNumInputs(self); })
+      .def_property_readonly(
+          "n_symbols",
+          [](PyAffineMap &self) { return mlirAffineMapGetNumSymbols(self); })
+      .def_property_readonly("results", [](PyAffineMap &self) {
+        return PyAffineMapExprList(self);
+      });
+  PyAffineMapExprList::bind(m);
+
+  //----------------------------------------------------------------------------
+  // Mapping of PyIntegerSet.
+  //----------------------------------------------------------------------------
+  py::class_<PyIntegerSet>(m, "IntegerSet")
+      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR,
+                             &PyIntegerSet::getCapsule)
+      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyIntegerSet::createFromCapsule)
+      .def("__eq__", [](PyIntegerSet &self,
+                        PyIntegerSet &other) { return self == other; })
+      .def("__eq__", [](PyIntegerSet &self, py::object other) { return false; })
+      .def("__str__",
+           [](PyIntegerSet &self) {
+             PyPrintAccumulator printAccum;
+             mlirIntegerSetPrint(self, printAccum.getCallback(),
+                                 printAccum.getUserData());
+             return printAccum.join();
+           })
+      .def("__repr__",
+           [](PyIntegerSet &self) {
+             PyPrintAccumulator printAccum;
+             printAccum.parts.append("IntegerSet(");
+             mlirIntegerSetPrint(self, printAccum.getCallback(),
+                                 printAccum.getUserData());
+             printAccum.parts.append(")");
+             return printAccum.join();
+           })
+      .def_property_readonly(
+          "context",
+          [](PyIntegerSet &self) { return self.getContext().getObject(); })
+      .def(
+          "dump", [](PyIntegerSet &self) { mlirIntegerSetDump(self); },
+          kDumpDocstring)
+      .def_static(
+          "get",
+          [](intptr_t numDims, intptr_t numSymbols, py::list exprs,
+             std::vector<bool> eqFlags, DefaultingPyMlirContext context) {
+            if (exprs.size() != eqFlags.size())
+              throw py::value_error(
+                  "Expected the number of constraints to match "
+                  "that of equality flags");
+            if (exprs.empty())
+              throw py::value_error("Expected non-empty list of constraints");
+
+            // Copy over to a SmallVector because std::vector has a
+            // specialization for booleans that packs data and does not
+            // expose a `bool *`.
+            SmallVector<bool, 8> flags(eqFlags.begin(), eqFlags.end());
+
+            SmallVector<MlirAffineExpr> affineExprs;
+            pyListToVector<PyAffineExpr>(exprs, affineExprs,
+                                         "attempting to create an IntegerSet");
+            MlirIntegerSet set = mlirIntegerSetGet(
+                context->get(), numDims, numSymbols, exprs.size(),
+                affineExprs.data(), flags.data());
+            return PyIntegerSet(context->getRef(), set);
+          },
+          py::arg("num_dims"), py::arg("num_symbols"), py::arg("exprs"),
+          py::arg("eq_flags"), py::arg("context") = py::none())
+      .def_static(
+          "get_empty",
+          [](intptr_t numDims, intptr_t numSymbols,
+             DefaultingPyMlirContext context) {
+            MlirIntegerSet set =
+                mlirIntegerSetEmptyGet(context->get(), numDims, numSymbols);
+            return PyIntegerSet(context->getRef(), set);
+          },
+          py::arg("num_dims"), py::arg("num_symbols"),
+          py::arg("context") = py::none())
+      .def("get_replaced",
+           [](PyIntegerSet &self, py::list dimExprs, py::list symbolExprs,
+              intptr_t numResultDims, intptr_t numResultSymbols) {
+             if (static_cast<intptr_t>(dimExprs.size()) !=
+                 mlirIntegerSetGetNumDims(self))
+               throw py::value_error(
+                   "Expected the number of dimension replacement expressions "
+                   "to match that of dimensions");
+             if (static_cast<intptr_t>(symbolExprs.size()) !=
+                 mlirIntegerSetGetNumSymbols(self))
+               throw py::value_error(
+                   "Expected the number of symbol replacement expressions "
+                   "to match that of symbols");
+
+             SmallVector<MlirAffineExpr> dimAffineExprs, symbolAffineExprs;
+             pyListToVector<PyAffineExpr>(
+                 dimExprs, dimAffineExprs,
+                 "attempting to create an IntegerSet by replacing dimensions");
+             pyListToVector<PyAffineExpr>(
+                 symbolExprs, symbolAffineExprs,
+                 "attempting to create an IntegerSet by replacing symbols");
+             MlirIntegerSet set = mlirIntegerSetReplaceGet(
+                 self, dimAffineExprs.data(), symbolAffineExprs.data(),
+                 numResultDims, numResultSymbols);
+             return PyIntegerSet(self.getContext(), set);
+           })
+      .def_property_readonly("is_canonical_empty",
+                             [](PyIntegerSet &self) {
+                               return mlirIntegerSetIsCanonicalEmpty(self);
+                             })
+      .def_property_readonly(
+          "n_dims",
+          [](PyIntegerSet &self) { return mlirIntegerSetGetNumDims(self); })
+      .def_property_readonly(
+          "n_symbols",
+          [](PyIntegerSet &self) { return mlirIntegerSetGetNumSymbols(self); })
+      .def_property_readonly(
+          "n_inputs",
+          [](PyIntegerSet &self) { return mlirIntegerSetGetNumInputs(self); })
+      .def_property_readonly("n_equalities",
+                             [](PyIntegerSet &self) {
+                               return mlirIntegerSetGetNumEqualities(self);
+                             })
+      .def_property_readonly("n_inequalities",
+                             [](PyIntegerSet &self) {
+                               return mlirIntegerSetGetNumInequalities(self);
+                             })
+      .def_property_readonly("constraints", [](PyIntegerSet &self) {
+        return PyIntegerSetConstraintList(self);
+      });
+  PyIntegerSetConstraint::bind(m);
+  PyIntegerSetConstraintList::bind(m);
+}
diff --git a/mlir/lib/Bindings/Python/IRAttributes.cpp b/mlir/lib/Bindings/Python/IRAttributes.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6f9206c1b9121b2f6ef5a9b0c90915279b5c88e1
--- /dev/null
+++ b/mlir/lib/Bindings/Python/IRAttributes.cpp
@@ -0,0 +1,761 @@
+//===- IRAttributes.cpp - Exports builtin and standard attributes ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IRModule.h"
+
+#include "PybindUtils.h"
+
+#include "mlir-c/BuiltinAttributes.h"
+#include "mlir-c/BuiltinTypes.h"
+
+namespace py = pybind11;
+using namespace mlir;
+using namespace mlir::python;
+
+using llvm::SmallVector;
+using llvm::StringRef;
+using llvm::Twine;
+
+namespace {
+
+static MlirStringRef toMlirStringRef(const std::string &s) {
+  return mlirStringRefCreate(s.data(), s.size());
+}
+
+/// CRTP base classes for Python attributes that subclass Attribute and should
+/// be castable from it (i.e. via something like StringAttr(attr)).
+/// By default, attribute class hierarchies are one level deep (i.e. a
+/// concrete attribute class extends PyAttribute); however, intermediate
+/// python-visible base classes can be modeled by specifying a BaseTy.
+template <typename DerivedTy, typename BaseTy = PyAttribute>
+class PyConcreteAttribute : public BaseTy {
+public:
+  // Derived classes must define statics for:
+  //   IsAFunctionTy isaFunction
+  //   const char *pyClassName
+  using ClassTy = py::class_<DerivedTy, BaseTy>;
+  using IsAFunctionTy = bool (*)(MlirAttribute);
+
+  PyConcreteAttribute() = default;
+  PyConcreteAttribute(PyMlirContextRef contextRef, MlirAttribute attr)
+      : BaseTy(std::move(contextRef), attr) {}
+  PyConcreteAttribute(PyAttribute &orig)
+      : PyConcreteAttribute(orig.getContext(), castFrom(orig)) {}
+
+  static MlirAttribute castFrom(PyAttribute &orig) {
+    if (!DerivedTy::isaFunction(orig)) {
+      auto origRepr = py::repr(py::cast(orig)).cast<std::string>();
+      throw SetPyError(PyExc_ValueError, Twine("Cannot cast attribute to ") +
+                                             DerivedTy::pyClassName +
+                                             " (from " + origRepr + ")");
+    }
+    return orig;
+  }
+
+  static void bind(py::module &m) {
+    auto cls = ClassTy(m, DerivedTy::pyClassName, py::buffer_protocol());
+    cls.def(py::init<PyAttribute &>(), py::keep_alive<0, 1>());
+    DerivedTy::bindDerived(cls);
+  }
+
+  /// Implemented by derived classes to add methods to the Python subclass.
+  static void bindDerived(ClassTy &m) {}
+};
+
+class PyAffineMapAttribute : public PyConcreteAttribute<PyAffineMapAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAAffineMap;
+  static constexpr const char *pyClassName = "AffineMapAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](PyAffineMap &affineMap) {
+          MlirAttribute attr = mlirAffineMapAttrGet(affineMap.get());
+          return PyAffineMapAttribute(affineMap.getContext(), attr);
+        },
+        py::arg("affine_map"), "Gets an attribute wrapping an AffineMap.");
+  }
+};
+
+class PyArrayAttribute : public PyConcreteAttribute<PyArrayAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAArray;
+  static constexpr const char *pyClassName = "ArrayAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  class PyArrayAttributeIterator {
+  public:
+    PyArrayAttributeIterator(PyAttribute attr) : attr(attr) {}
+
+    PyArrayAttributeIterator &dunderIter() { return *this; }
+
+    PyAttribute dunderNext() {
+      if (nextIndex >= mlirArrayAttrGetNumElements(attr.get())) {
+        throw py::stop_iteration();
+      }
+      return PyAttribute(attr.getContext(),
+                         mlirArrayAttrGetElement(attr.get(), nextIndex++));
+    }
+
+    static void bind(py::module &m) {
+      py::class_<PyArrayAttributeIterator>(m, "ArrayAttributeIterator")
+          .def("__iter__", &PyArrayAttributeIterator::dunderIter)
+          .def("__next__", &PyArrayAttributeIterator::dunderNext);
+    }
+
+  private:
+    PyAttribute attr;
+    int nextIndex = 0;
+  };
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](py::list attributes, DefaultingPyMlirContext context) {
+          SmallVector<MlirAttribute> mlirAttributes;
+          mlirAttributes.reserve(py::len(attributes));
+          for (auto attribute : attributes) {
+            try {
+              mlirAttributes.push_back(attribute.cast<PyAttribute>());
+            } catch (py::cast_error &err) {
+              std::string msg = std::string("Invalid attribute when attempting "
+                                            "to create an ArrayAttribute (") +
+                                err.what() + ")";
+              throw py::cast_error(msg);
+            } catch (py::reference_cast_error &err) {
+              // This exception seems thrown when the value is "None".
+              std::string msg =
+                  std::string("Invalid attribute (None?) when attempting to "
+                              "create an ArrayAttribute (") +
+                  err.what() + ")";
+              throw py::cast_error(msg);
+            }
+          }
+          MlirAttribute attr = mlirArrayAttrGet(
+              context->get(), mlirAttributes.size(), mlirAttributes.data());
+          return PyArrayAttribute(context->getRef(), attr);
+        },
+        py::arg("attributes"), py::arg("context") = py::none(),
+        "Gets a uniqued Array attribute");
+    c.def("__getitem__",
+          [](PyArrayAttribute &arr, intptr_t i) {
+            if (i >= mlirArrayAttrGetNumElements(arr))
+              throw py::index_error("ArrayAttribute index out of range");
+            return PyAttribute(arr.getContext(),
+                               mlirArrayAttrGetElement(arr, i));
+          })
+        .def("__len__",
+             [](const PyArrayAttribute &arr) {
+               return mlirArrayAttrGetNumElements(arr);
+             })
+        .def("__iter__", [](const PyArrayAttribute &arr) {
+          return PyArrayAttributeIterator(arr);
+        });
+  }
+};
+
+/// Float Point Attribute subclass - FloatAttr.
+class PyFloatAttribute : public PyConcreteAttribute<PyFloatAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAFloat;
+  static constexpr const char *pyClassName = "FloatAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](PyType &type, double value, DefaultingPyLocation loc) {
+          MlirAttribute attr = mlirFloatAttrDoubleGetChecked(loc, type, value);
+          // TODO: Rework error reporting once diagnostic engine is exposed
+          // in C API.
+          if (mlirAttributeIsNull(attr)) {
+            throw SetPyError(PyExc_ValueError,
+                             Twine("invalid '") +
+                                 py::repr(py::cast(type)).cast<std::string>() +
+                                 "' and expected floating point type.");
+          }
+          return PyFloatAttribute(type.getContext(), attr);
+        },
+        py::arg("type"), py::arg("value"), py::arg("loc") = py::none(),
+        "Gets an uniqued float point attribute associated to a type");
+    c.def_static(
+        "get_f32",
+        [](double value, DefaultingPyMlirContext context) {
+          MlirAttribute attr = mlirFloatAttrDoubleGet(
+              context->get(), mlirF32TypeGet(context->get()), value);
+          return PyFloatAttribute(context->getRef(), attr);
+        },
+        py::arg("value"), py::arg("context") = py::none(),
+        "Gets an uniqued float point attribute associated to a f32 type");
+    c.def_static(
+        "get_f64",
+        [](double value, DefaultingPyMlirContext context) {
+          MlirAttribute attr = mlirFloatAttrDoubleGet(
+              context->get(), mlirF64TypeGet(context->get()), value);
+          return PyFloatAttribute(context->getRef(), attr);
+        },
+        py::arg("value"), py::arg("context") = py::none(),
+        "Gets an uniqued float point attribute associated to a f64 type");
+    c.def_property_readonly(
+        "value",
+        [](PyFloatAttribute &self) {
+          return mlirFloatAttrGetValueDouble(self);
+        },
+        "Returns the value of the float point attribute");
+  }
+};
+
+/// Integer Attribute subclass - IntegerAttr.
+class PyIntegerAttribute : public PyConcreteAttribute<PyIntegerAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAInteger;
+  static constexpr const char *pyClassName = "IntegerAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](PyType &type, int64_t value) {
+          MlirAttribute attr = mlirIntegerAttrGet(type, value);
+          return PyIntegerAttribute(type.getContext(), attr);
+        },
+        py::arg("type"), py::arg("value"),
+        "Gets an uniqued integer attribute associated to a type");
+    c.def_property_readonly(
+        "value",
+        [](PyIntegerAttribute &self) {
+          return mlirIntegerAttrGetValueInt(self);
+        },
+        "Returns the value of the integer attribute");
+  }
+};
+
+/// Bool Attribute subclass - BoolAttr.
+class PyBoolAttribute : public PyConcreteAttribute<PyBoolAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsABool;
+  static constexpr const char *pyClassName = "BoolAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](bool value, DefaultingPyMlirContext context) {
+          MlirAttribute attr = mlirBoolAttrGet(context->get(), value);
+          return PyBoolAttribute(context->getRef(), attr);
+        },
+        py::arg("value"), py::arg("context") = py::none(),
+        "Gets an uniqued bool attribute");
+    c.def_property_readonly(
+        "value",
+        [](PyBoolAttribute &self) { return mlirBoolAttrGetValue(self); },
+        "Returns the value of the bool attribute");
+  }
+};
+
+class PyFlatSymbolRefAttribute
+    : public PyConcreteAttribute<PyFlatSymbolRefAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAFlatSymbolRef;
+  static constexpr const char *pyClassName = "FlatSymbolRefAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](std::string value, DefaultingPyMlirContext context) {
+          MlirAttribute attr =
+              mlirFlatSymbolRefAttrGet(context->get(), toMlirStringRef(value));
+          return PyFlatSymbolRefAttribute(context->getRef(), attr);
+        },
+        py::arg("value"), py::arg("context") = py::none(),
+        "Gets a uniqued FlatSymbolRef attribute");
+    c.def_property_readonly(
+        "value",
+        [](PyFlatSymbolRefAttribute &self) {
+          MlirStringRef stringRef = mlirFlatSymbolRefAttrGetValue(self);
+          return py::str(stringRef.data, stringRef.length);
+        },
+        "Returns the value of the FlatSymbolRef attribute as a string");
+  }
+};
+
+class PyStringAttribute : public PyConcreteAttribute<PyStringAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAString;
+  static constexpr const char *pyClassName = "StringAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](std::string value, DefaultingPyMlirContext context) {
+          MlirAttribute attr =
+              mlirStringAttrGet(context->get(), toMlirStringRef(value));
+          return PyStringAttribute(context->getRef(), attr);
+        },
+        py::arg("value"), py::arg("context") = py::none(),
+        "Gets a uniqued string attribute");
+    c.def_static(
+        "get_typed",
+        [](PyType &type, std::string value) {
+          MlirAttribute attr =
+              mlirStringAttrTypedGet(type, toMlirStringRef(value));
+          return PyStringAttribute(type.getContext(), attr);
+        },
+
+        "Gets a uniqued string attribute associated to a type");
+    c.def_property_readonly(
+        "value",
+        [](PyStringAttribute &self) {
+          MlirStringRef stringRef = mlirStringAttrGetValue(self);
+          return py::str(stringRef.data, stringRef.length);
+        },
+        "Returns the value of the string attribute");
+  }
+};
+
+// TODO: Support construction of bool elements.
+// TODO: Support construction of string elements.
+class PyDenseElementsAttribute
+    : public PyConcreteAttribute<PyDenseElementsAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsADenseElements;
+  static constexpr const char *pyClassName = "DenseElementsAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static PyDenseElementsAttribute
+  getFromBuffer(py::buffer array, bool signless,
+                DefaultingPyMlirContext contextWrapper) {
+    // Request a contiguous view. In exotic cases, this will cause a copy.
+    int flags = PyBUF_C_CONTIGUOUS | PyBUF_FORMAT;
+    Py_buffer *view = new Py_buffer();
+    if (PyObject_GetBuffer(array.ptr(), view, flags) != 0) {
+      delete view;
+      throw py::error_already_set();
+    }
+    py::buffer_info arrayInfo(view);
+
+    MlirContext context = contextWrapper->get();
+    // Switch on the types that can be bulk loaded between the Python and
+    // MLIR-C APIs.
+    // See: https://docs.python.org/3/library/struct.html#format-characters
+    if (arrayInfo.format == "f") {
+      // f32
+      assert(arrayInfo.itemsize == 4 && "mismatched array itemsize");
+      return PyDenseElementsAttribute(
+          contextWrapper->getRef(),
+          bulkLoad(context, mlirDenseElementsAttrFloatGet,
+                   mlirF32TypeGet(context), arrayInfo));
+    } else if (arrayInfo.format == "d") {
+      // f64
+      assert(arrayInfo.itemsize == 8 && "mismatched array itemsize");
+      return PyDenseElementsAttribute(
+          contextWrapper->getRef(),
+          bulkLoad(context, mlirDenseElementsAttrDoubleGet,
+                   mlirF64TypeGet(context), arrayInfo));
+    } else if (isSignedIntegerFormat(arrayInfo.format)) {
+      if (arrayInfo.itemsize == 4) {
+        // i32
+        MlirType elementType = signless ? mlirIntegerTypeGet(context, 32)
+                                        : mlirIntegerTypeSignedGet(context, 32);
+        return PyDenseElementsAttribute(contextWrapper->getRef(),
+                                        bulkLoad(context,
+                                                 mlirDenseElementsAttrInt32Get,
+                                                 elementType, arrayInfo));
+      } else if (arrayInfo.itemsize == 8) {
+        // i64
+        MlirType elementType = signless ? mlirIntegerTypeGet(context, 64)
+                                        : mlirIntegerTypeSignedGet(context, 64);
+        return PyDenseElementsAttribute(contextWrapper->getRef(),
+                                        bulkLoad(context,
+                                                 mlirDenseElementsAttrInt64Get,
+                                                 elementType, arrayInfo));
+      }
+    } else if (isUnsignedIntegerFormat(arrayInfo.format)) {
+      if (arrayInfo.itemsize == 4) {
+        // unsigned i32
+        MlirType elementType = signless
+                                   ? mlirIntegerTypeGet(context, 32)
+                                   : mlirIntegerTypeUnsignedGet(context, 32);
+        return PyDenseElementsAttribute(contextWrapper->getRef(),
+                                        bulkLoad(context,
+                                                 mlirDenseElementsAttrUInt32Get,
+                                                 elementType, arrayInfo));
+      } else if (arrayInfo.itemsize == 8) {
+        // unsigned i64
+        MlirType elementType = signless
+                                   ? mlirIntegerTypeGet(context, 64)
+                                   : mlirIntegerTypeUnsignedGet(context, 64);
+        return PyDenseElementsAttribute(contextWrapper->getRef(),
+                                        bulkLoad(context,
+                                                 mlirDenseElementsAttrUInt64Get,
+                                                 elementType, arrayInfo));
+      }
+    }
+
+    // TODO: Fall back to string-based get.
+    std::string message = "unimplemented array format conversion from format: ";
+    message.append(arrayInfo.format);
+    throw SetPyError(PyExc_ValueError, message);
+  }
+
+  static PyDenseElementsAttribute getSplat(PyType shapedType,
+                                           PyAttribute &elementAttr) {
+    auto contextWrapper =
+        PyMlirContext::forContext(mlirTypeGetContext(shapedType));
+    if (!mlirAttributeIsAInteger(elementAttr) &&
+        !mlirAttributeIsAFloat(elementAttr)) {
+      std::string message = "Illegal element type for DenseElementsAttr: ";
+      message.append(py::repr(py::cast(elementAttr)));
+      throw SetPyError(PyExc_ValueError, message);
+    }
+    if (!mlirTypeIsAShaped(shapedType) ||
+        !mlirShapedTypeHasStaticShape(shapedType)) {
+      std::string message =
+          "Expected a static ShapedType for the shaped_type parameter: ";
+      message.append(py::repr(py::cast(shapedType)));
+      throw SetPyError(PyExc_ValueError, message);
+    }
+    MlirType shapedElementType = mlirShapedTypeGetElementType(shapedType);
+    MlirType attrType = mlirAttributeGetType(elementAttr);
+    if (!mlirTypeEqual(shapedElementType, attrType)) {
+      std::string message =
+          "Shaped element type and attribute type must be equal: shaped=";
+      message.append(py::repr(py::cast(shapedType)));
+      message.append(", element=");
+      message.append(py::repr(py::cast(elementAttr)));
+      throw SetPyError(PyExc_ValueError, message);
+    }
+
+    MlirAttribute elements =
+        mlirDenseElementsAttrSplatGet(shapedType, elementAttr);
+    return PyDenseElementsAttribute(contextWrapper->getRef(), elements);
+  }
+
+  intptr_t dunderLen() { return mlirElementsAttrGetNumElements(*this); }
+
+  py::buffer_info accessBuffer() {
+    MlirType shapedType = mlirAttributeGetType(*this);
+    MlirType elementType = mlirShapedTypeGetElementType(shapedType);
+
+    if (mlirTypeIsAF32(elementType)) {
+      // f32
+      return bufferInfo(shapedType, mlirDenseElementsAttrGetFloatValue);
+    } else if (mlirTypeIsAF64(elementType)) {
+      // f64
+      return bufferInfo(shapedType, mlirDenseElementsAttrGetDoubleValue);
+    } else if (mlirTypeIsAInteger(elementType) &&
+               mlirIntegerTypeGetWidth(elementType) == 32) {
+      if (mlirIntegerTypeIsSignless(elementType) ||
+          mlirIntegerTypeIsSigned(elementType)) {
+        // i32
+        return bufferInfo(shapedType, mlirDenseElementsAttrGetInt32Value);
+      } else if (mlirIntegerTypeIsUnsigned(elementType)) {
+        // unsigned i32
+        return bufferInfo(shapedType, mlirDenseElementsAttrGetUInt32Value);
+      }
+    } else if (mlirTypeIsAInteger(elementType) &&
+               mlirIntegerTypeGetWidth(elementType) == 64) {
+      if (mlirIntegerTypeIsSignless(elementType) ||
+          mlirIntegerTypeIsSigned(elementType)) {
+        // i64
+        return bufferInfo(shapedType, mlirDenseElementsAttrGetInt64Value);
+      } else if (mlirIntegerTypeIsUnsigned(elementType)) {
+        // unsigned i64
+        return bufferInfo(shapedType, mlirDenseElementsAttrGetUInt64Value);
+      }
+    }
+
+    std::string message = "unimplemented array format.";
+    throw SetPyError(PyExc_ValueError, message);
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def("__len__", &PyDenseElementsAttribute::dunderLen)
+        .def_static("get", PyDenseElementsAttribute::getFromBuffer,
+                    py::arg("array"), py::arg("signless") = true,
+                    py::arg("context") = py::none(),
+                    "Gets from a buffer or ndarray")
+        .def_static("get_splat", PyDenseElementsAttribute::getSplat,
+                    py::arg("shaped_type"), py::arg("element_attr"),
+                    "Gets a DenseElementsAttr where all values are the same")
+        .def_property_readonly("is_splat",
+                               [](PyDenseElementsAttribute &self) -> bool {
+                                 return mlirDenseElementsAttrIsSplat(self);
+                               })
+        .def_buffer(&PyDenseElementsAttribute::accessBuffer);
+  }
+
+private:
+  template <typename ElementTy>
+  static MlirAttribute
+  bulkLoad(MlirContext context,
+           MlirAttribute (*ctor)(MlirType, intptr_t, ElementTy *),
+           MlirType mlirElementType, py::buffer_info &arrayInfo) {
+    SmallVector<int64_t, 4> shape(arrayInfo.shape.begin(),
+                                  arrayInfo.shape.begin() + arrayInfo.ndim);
+    auto shapedType =
+        mlirRankedTensorTypeGet(shape.size(), shape.data(), mlirElementType);
+    intptr_t numElements = arrayInfo.size;
+    const ElementTy *contents = static_cast<const ElementTy *>(arrayInfo.ptr);
+    return ctor(shapedType, numElements, contents);
+  }
+
+  static bool isUnsignedIntegerFormat(const std::string &format) {
+    if (format.empty())
+      return false;
+    char code = format[0];
+    return code == 'I' || code == 'B' || code == 'H' || code == 'L' ||
+           code == 'Q';
+  }
+
+  static bool isSignedIntegerFormat(const std::string &format) {
+    if (format.empty())
+      return false;
+    char code = format[0];
+    return code == 'i' || code == 'b' || code == 'h' || code == 'l' ||
+           code == 'q';
+  }
+
+  template <typename Type>
+  py::buffer_info bufferInfo(MlirType shapedType,
+                             Type (*value)(MlirAttribute, intptr_t)) {
+    intptr_t rank = mlirShapedTypeGetRank(shapedType);
+    // Prepare the data for the buffer_info.
+    // Buffer is configured for read-only access below.
+    Type *data = static_cast<Type *>(
+        const_cast<void *>(mlirDenseElementsAttrGetRawData(*this)));
+    // Prepare the shape for the buffer_info.
+    SmallVector<intptr_t, 4> shape;
+    for (intptr_t i = 0; i < rank; ++i)
+      shape.push_back(mlirShapedTypeGetDimSize(shapedType, i));
+    // Prepare the strides for the buffer_info.
+    SmallVector<intptr_t, 4> strides;
+    intptr_t strideFactor = 1;
+    for (intptr_t i = 1; i < rank; ++i) {
+      strideFactor = 1;
+      for (intptr_t j = i; j < rank; ++j) {
+        strideFactor *= mlirShapedTypeGetDimSize(shapedType, j);
+      }
+      strides.push_back(sizeof(Type) * strideFactor);
+    }
+    strides.push_back(sizeof(Type));
+    return py::buffer_info(data, sizeof(Type),
+                           py::format_descriptor<Type>::format(), rank, shape,
+                           strides, /*readonly=*/true);
+  }
+}; // namespace
+
+/// Refinement of the PyDenseElementsAttribute for attributes containing integer
+/// (and boolean) values. Supports element access.
+class PyDenseIntElementsAttribute
+    : public PyConcreteAttribute<PyDenseIntElementsAttribute,
+                                 PyDenseElementsAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsADenseIntElements;
+  static constexpr const char *pyClassName = "DenseIntElementsAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  /// Returns the element at the given linear position. Asserts if the index is
+  /// out of range.
+  py::int_ dunderGetItem(intptr_t pos) {
+    if (pos < 0 || pos >= dunderLen()) {
+      throw SetPyError(PyExc_IndexError,
+                       "attempt to access out of bounds element");
+    }
+
+    MlirType type = mlirAttributeGetType(*this);
+    type = mlirShapedTypeGetElementType(type);
+    assert(mlirTypeIsAInteger(type) &&
+           "expected integer element type in dense int elements attribute");
+    // Dispatch element extraction to an appropriate C function based on the
+    // elemental type of the attribute. py::int_ is implicitly constructible
+    // from any C++ integral type and handles bitwidth correctly.
+    // TODO: consider caching the type properties in the constructor to avoid
+    // querying them on each element access.
+    unsigned width = mlirIntegerTypeGetWidth(type);
+    bool isUnsigned = mlirIntegerTypeIsUnsigned(type);
+    if (isUnsigned) {
+      if (width == 1) {
+        return mlirDenseElementsAttrGetBoolValue(*this, pos);
+      }
+      if (width == 32) {
+        return mlirDenseElementsAttrGetUInt32Value(*this, pos);
+      }
+      if (width == 64) {
+        return mlirDenseElementsAttrGetUInt64Value(*this, pos);
+      }
+    } else {
+      if (width == 1) {
+        return mlirDenseElementsAttrGetBoolValue(*this, pos);
+      }
+      if (width == 32) {
+        return mlirDenseElementsAttrGetInt32Value(*this, pos);
+      }
+      if (width == 64) {
+        return mlirDenseElementsAttrGetInt64Value(*this, pos);
+      }
+    }
+    throw SetPyError(PyExc_TypeError, "Unsupported integer type");
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def("__getitem__", &PyDenseIntElementsAttribute::dunderGetItem);
+  }
+};
+
+class PyDictAttribute : public PyConcreteAttribute<PyDictAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsADictionary;
+  static constexpr const char *pyClassName = "DictAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  intptr_t dunderLen() { return mlirDictionaryAttrGetNumElements(*this); }
+
+  static void bindDerived(ClassTy &c) {
+    c.def("__len__", &PyDictAttribute::dunderLen);
+    c.def_static(
+        "get",
+        [](py::dict attributes, DefaultingPyMlirContext context) {
+          SmallVector<MlirNamedAttribute> mlirNamedAttributes;
+          mlirNamedAttributes.reserve(attributes.size());
+          for (auto &it : attributes) {
+            auto &mlir_attr = it.second.cast<PyAttribute &>();
+            auto name = it.first.cast<std::string>();
+            mlirNamedAttributes.push_back(mlirNamedAttributeGet(
+                mlirIdentifierGet(mlirAttributeGetContext(mlir_attr),
+                                  toMlirStringRef(name)),
+                mlir_attr));
+          }
+          MlirAttribute attr =
+              mlirDictionaryAttrGet(context->get(), mlirNamedAttributes.size(),
+                                    mlirNamedAttributes.data());
+          return PyDictAttribute(context->getRef(), attr);
+        },
+        py::arg("value"), py::arg("context") = py::none(),
+        "Gets an uniqued dict attribute");
+    c.def("__getitem__", [](PyDictAttribute &self, const std::string &name) {
+      MlirAttribute attr =
+          mlirDictionaryAttrGetElementByName(self, toMlirStringRef(name));
+      if (mlirAttributeIsNull(attr)) {
+        throw SetPyError(PyExc_KeyError,
+                         "attempt to access a non-existent attribute");
+      }
+      return PyAttribute(self.getContext(), attr);
+    });
+    c.def("__getitem__", [](PyDictAttribute &self, intptr_t index) {
+      if (index < 0 || index >= self.dunderLen()) {
+        throw SetPyError(PyExc_IndexError,
+                         "attempt to access out of bounds attribute");
+      }
+      MlirNamedAttribute namedAttr = mlirDictionaryAttrGetElement(self, index);
+      return PyNamedAttribute(
+          namedAttr.attribute,
+          std::string(mlirIdentifierStr(namedAttr.name).data));
+    });
+  }
+};
+
+/// Refinement of PyDenseElementsAttribute for attributes containing
+/// floating-point values. Supports element access.
+class PyDenseFPElementsAttribute
+    : public PyConcreteAttribute<PyDenseFPElementsAttribute,
+                                 PyDenseElementsAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsADenseFPElements;
+  static constexpr const char *pyClassName = "DenseFPElementsAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  py::float_ dunderGetItem(intptr_t pos) {
+    if (pos < 0 || pos >= dunderLen()) {
+      throw SetPyError(PyExc_IndexError,
+                       "attempt to access out of bounds element");
+    }
+
+    MlirType type = mlirAttributeGetType(*this);
+    type = mlirShapedTypeGetElementType(type);
+    // Dispatch element extraction to an appropriate C function based on the
+    // elemental type of the attribute. py::float_ is implicitly constructible
+    // from float and double.
+    // TODO: consider caching the type properties in the constructor to avoid
+    // querying them on each element access.
+    if (mlirTypeIsAF32(type)) {
+      return mlirDenseElementsAttrGetFloatValue(*this, pos);
+    }
+    if (mlirTypeIsAF64(type)) {
+      return mlirDenseElementsAttrGetDoubleValue(*this, pos);
+    }
+    throw SetPyError(PyExc_TypeError, "Unsupported floating-point type");
+  }
+
+  static void bindDerived(ClassTy &c) {
+    c.def("__getitem__", &PyDenseFPElementsAttribute::dunderGetItem);
+  }
+};
+
+class PyTypeAttribute : public PyConcreteAttribute<PyTypeAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAType;
+  static constexpr const char *pyClassName = "TypeAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](PyType value, DefaultingPyMlirContext context) {
+          MlirAttribute attr = mlirTypeAttrGet(value.get());
+          return PyTypeAttribute(context->getRef(), attr);
+        },
+        py::arg("value"), py::arg("context") = py::none(),
+        "Gets a uniqued Type attribute");
+    c.def_property_readonly("value", [](PyTypeAttribute &self) {
+      return PyType(self.getContext()->getRef(),
+                    mlirTypeAttrGetValue(self.get()));
+    });
+  }
+};
+
+/// Unit Attribute subclass. Unit attributes don't have values.
+class PyUnitAttribute : public PyConcreteAttribute<PyUnitAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAUnit;
+  static constexpr const char *pyClassName = "UnitAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](DefaultingPyMlirContext context) {
+          return PyUnitAttribute(context->getRef(),
+                                 mlirUnitAttrGet(context->get()));
+        },
+        py::arg("context") = py::none(), "Create a Unit attribute.");
+  }
+};
+
+} // namespace
+
+void mlir::python::populateIRAttributes(py::module &m) {
+  PyAffineMapAttribute::bind(m);
+  PyArrayAttribute::bind(m);
+  PyArrayAttribute::PyArrayAttributeIterator::bind(m);
+  PyBoolAttribute::bind(m);
+  PyDenseElementsAttribute::bind(m);
+  PyDenseFPElementsAttribute::bind(m);
+  PyDenseIntElementsAttribute::bind(m);
+  PyDictAttribute::bind(m);
+  PyFlatSymbolRefAttribute::bind(m);
+  PyFloatAttribute::bind(m);
+  PyIntegerAttribute::bind(m);
+  PyStringAttribute::bind(m);
+  PyTypeAttribute::bind(m);
+  PyUnitAttribute::bind(m);
+}
diff --git a/mlir/lib/Bindings/Python/IRModules.cpp b/mlir/lib/Bindings/Python/IRCore.cpp
similarity index 52%
rename from mlir/lib/Bindings/Python/IRModules.cpp
rename to mlir/lib/Bindings/Python/IRCore.cpp
index a544e52c26131831208c8e9dc37348ae3df14ebb..0a4c5fcb40c392cc6d2d7eebd4a4db1cb24c2854 100644
--- a/mlir/lib/Bindings/Python/IRModules.cpp
+++ b/mlir/lib/Bindings/Python/IRCore.cpp
@@ -6,16 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "IRModules.h"
+#include "IRModule.h"
 
 #include "Globals.h"
 #include "PybindUtils.h"
 
-#include "mlir-c/AffineMap.h"
 #include "mlir-c/Bindings/Python/Interop.h"
 #include "mlir-c/BuiltinAttributes.h"
 #include "mlir-c/BuiltinTypes.h"
-#include "mlir-c/IntegerSet.h"
 #include "mlir-c/Registration.h"
 #include "llvm/ADT/SmallVector.h"
 #include <pybind11/stl.h>
@@ -138,12 +136,6 @@ py::object classmethod(Func f, Args... args) {
   return py::reinterpret_borrow<py::object>((PyClassMethod_New(cf.ptr())));
 }
 
-/// Checks whether the given type is an integer or float type.
-static int mlirTypeIsAIntegerOrFloat(MlirType type) {
-  return mlirTypeIsAInteger(type) || mlirTypeIsABF16(type) ||
-         mlirTypeIsAF16(type) || mlirTypeIsAF32(type) || mlirTypeIsAF64(type);
-}
-
 static py::object
 createCustomDialectWrapper(const std::string &dialectNamespace,
                            py::object dialectDescriptor) {
@@ -161,21 +153,6 @@ static MlirStringRef toMlirStringRef(const std::string &s) {
   return mlirStringRefCreate(s.data(), s.size());
 }
 
-template <typename PermutationTy>
-static bool isPermutation(std::vector<PermutationTy> permutation) {
-  llvm::SmallVector<bool, 8> seen(permutation.size(), false);
-  for (auto val : permutation) {
-    if (val < permutation.size()) {
-      if (seen[val])
-        return false;
-      seen[val] = true;
-      continue;
-    }
-    return false;
-  }
-  return true;
-}
-
 //------------------------------------------------------------------------------
 // Collections.
 //------------------------------------------------------------------------------
@@ -1057,8 +1034,8 @@ PyOpView::buildGeneric(py::object cls, py::list resultTypeList,
   py::object operandSegmentSpecObj = cls.attr("_ODS_OPERAND_SEGMENTS");
   py::object resultSegmentSpecObj = cls.attr("_ODS_RESULT_SEGMENTS");
 
-  std::vector<uint64_t> operandSegmentLengths;
-  std::vector<uint64_t> resultSegmentLengths;
+  std::vector<uint32_t> operandSegmentLengths;
+  std::vector<uint32_t> resultSegmentLengths;
 
   // Validate/determine region count.
   auto opRegionSpec = py::cast<std::tuple<int, bool>>(cls.attr("_ODS_REGIONS"));
@@ -1270,8 +1247,8 @@ PyOpView::buildGeneric(py::object cls, py::list resultTypeList,
     // Add result_segment_sizes attribute.
     if (!resultSegmentLengths.empty()) {
       int64_t size = resultSegmentLengths.size();
-      MlirAttribute segmentLengthAttr = mlirDenseElementsAttrUInt64Get(
-          mlirVectorTypeGet(1, &size, mlirIntegerTypeGet(context->get(), 64)),
+      MlirAttribute segmentLengthAttr = mlirDenseElementsAttrUInt32Get(
+          mlirVectorTypeGet(1, &size, mlirIntegerTypeGet(context->get(), 32)),
           resultSegmentLengths.size(), resultSegmentLengths.data());
       (*attributes)["result_segment_sizes"] =
           PyAttribute(context, segmentLengthAttr);
@@ -1280,8 +1257,8 @@ PyOpView::buildGeneric(py::object cls, py::list resultTypeList,
     // Add operand_segment_sizes attribute.
     if (!operandSegmentLengths.empty()) {
       int64_t size = operandSegmentLengths.size();
-      MlirAttribute segmentLengthAttr = mlirDenseElementsAttrUInt64Get(
-          mlirVectorTypeGet(1, &size, mlirIntegerTypeGet(context->get(), 64)),
+      MlirAttribute segmentLengthAttr = mlirDenseElementsAttrUInt32Get(
+          mlirVectorTypeGet(1, &size, mlirIntegerTypeGet(context->get(), 32)),
           operandSegmentLengths.size(), operandSegmentLengths.data());
       (*attributes)["operand_segment_sizes"] =
           PyAttribute(context, segmentLengthAttr);
@@ -1466,7 +1443,8 @@ namespace {
 /// CRTP base class for Python MLIR values that subclass Value and should be
 /// castable from it. The value hierarchy is one level deep and is not supposed
 /// to accommodate other levels unless core MLIR changes.
-template <typename DerivedTy> class PyConcreteValue : public PyValue {
+template <typename DerivedTy>
+class PyConcreteValue : public PyValue {
 public:
   // Derived classes must define statics for:
   //   IsAFunctionTy isaFunction
@@ -1717,1907 +1695,169 @@ private:
 } // end namespace
 
 //------------------------------------------------------------------------------
-// Builtin attribute subclasses.
+// Populates the core exports of the 'ir' submodule.
 //------------------------------------------------------------------------------
 
-namespace {
-
-/// CRTP base classes for Python attributes that subclass Attribute and should
-/// be castable from it (i.e. via something like StringAttr(attr)).
-/// By default, attribute class hierarchies are one level deep (i.e. a
-/// concrete attribute class extends PyAttribute); however, intermediate
-/// python-visible base classes can be modeled by specifying a BaseTy.
-template <typename DerivedTy, typename BaseTy = PyAttribute>
-class PyConcreteAttribute : public BaseTy {
-public:
-  // Derived classes must define statics for:
-  //   IsAFunctionTy isaFunction
-  //   const char *pyClassName
-  using ClassTy = py::class_<DerivedTy, BaseTy>;
-  using IsAFunctionTy = bool (*)(MlirAttribute);
-
-  PyConcreteAttribute() = default;
-  PyConcreteAttribute(PyMlirContextRef contextRef, MlirAttribute attr)
-      : BaseTy(std::move(contextRef), attr) {}
-  PyConcreteAttribute(PyAttribute &orig)
-      : PyConcreteAttribute(orig.getContext(), castFrom(orig)) {}
-
-  static MlirAttribute castFrom(PyAttribute &orig) {
-    if (!DerivedTy::isaFunction(orig)) {
-      auto origRepr = py::repr(py::cast(orig)).cast<std::string>();
-      throw SetPyError(PyExc_ValueError, Twine("Cannot cast attribute to ") +
-                                             DerivedTy::pyClassName +
-                                             " (from " + origRepr + ")");
-    }
-    return orig;
-  }
-
-  static void bind(py::module &m) {
-    auto cls = ClassTy(m, DerivedTy::pyClassName, py::buffer_protocol());
-    cls.def(py::init<PyAttribute &>(), py::keep_alive<0, 1>());
-    DerivedTy::bindDerived(cls);
-  }
-
-  /// Implemented by derived classes to add methods to the Python subclass.
-  static void bindDerived(ClassTy &m) {}
-};
-
-class PyAffineMapAttribute : public PyConcreteAttribute<PyAffineMapAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAAffineMap;
-  static constexpr const char *pyClassName = "AffineMapAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](PyAffineMap &affineMap) {
-          MlirAttribute attr = mlirAffineMapAttrGet(affineMap.get());
-          return PyAffineMapAttribute(affineMap.getContext(), attr);
-        },
-        py::arg("affine_map"), "Gets an attribute wrapping an AffineMap.");
-  }
-};
-
-class PyArrayAttribute : public PyConcreteAttribute<PyArrayAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAArray;
-  static constexpr const char *pyClassName = "ArrayAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  class PyArrayAttributeIterator {
-  public:
-    PyArrayAttributeIterator(PyAttribute attr) : attr(attr) {}
-
-    PyArrayAttributeIterator &dunderIter() { return *this; }
-
-    PyAttribute dunderNext() {
-      if (nextIndex >= mlirArrayAttrGetNumElements(attr.get())) {
-        throw py::stop_iteration();
-      }
-      return PyAttribute(attr.getContext(),
-                         mlirArrayAttrGetElement(attr.get(), nextIndex++));
-    }
-
-    static void bind(py::module &m) {
-      py::class_<PyArrayAttributeIterator>(m, "ArrayAttributeIterator")
-          .def("__iter__", &PyArrayAttributeIterator::dunderIter)
-          .def("__next__", &PyArrayAttributeIterator::dunderNext);
-    }
-
-  private:
-    PyAttribute attr;
-    int nextIndex = 0;
-  };
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](py::list attributes, DefaultingPyMlirContext context) {
-          SmallVector<MlirAttribute> mlirAttributes;
-          mlirAttributes.reserve(py::len(attributes));
-          for (auto attribute : attributes) {
-            try {
-              mlirAttributes.push_back(attribute.cast<PyAttribute>());
-            } catch (py::cast_error &err) {
-              std::string msg = std::string("Invalid attribute when attempting "
-                                            "to create an ArrayAttribute (") +
-                                err.what() + ")";
-              throw py::cast_error(msg);
-            } catch (py::reference_cast_error &err) {
-              // This exception seems thrown when the value is "None".
-              std::string msg =
-                  std::string("Invalid attribute (None?) when attempting to "
-                              "create an ArrayAttribute (") +
-                  err.what() + ")";
-              throw py::cast_error(msg);
+void mlir::python::populateIRCore(py::module &m) {
+  //----------------------------------------------------------------------------
+  // Mapping of MlirContext
+  //----------------------------------------------------------------------------
+  py::class_<PyMlirContext>(m, "Context")
+      .def(py::init<>(&PyMlirContext::createNewContextForInit))
+      .def_static("_get_live_count", &PyMlirContext::getLiveCount)
+      .def("_get_context_again",
+           [](PyMlirContext &self) {
+             PyMlirContextRef ref = PyMlirContext::forContext(self.get());
+             return ref.releaseObject();
+           })
+      .def("_get_live_operation_count", &PyMlirContext::getLiveOperationCount)
+      .def("_get_live_module_count", &PyMlirContext::getLiveModuleCount)
+      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR,
+                             &PyMlirContext::getCapsule)
+      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyMlirContext::createFromCapsule)
+      .def("__enter__", &PyMlirContext::contextEnter)
+      .def("__exit__", &PyMlirContext::contextExit)
+      .def_property_readonly_static(
+          "current",
+          [](py::object & /*class*/) {
+            auto *context = PyThreadContextEntry::getDefaultContext();
+            if (!context)
+              throw SetPyError(PyExc_ValueError, "No current Context");
+            return context;
+          },
+          "Gets the Context bound to the current thread or raises ValueError")
+      .def_property_readonly(
+          "dialects",
+          [](PyMlirContext &self) { return PyDialects(self.getRef()); },
+          "Gets a container for accessing dialects by name")
+      .def_property_readonly(
+          "d", [](PyMlirContext &self) { return PyDialects(self.getRef()); },
+          "Alias for 'dialect'")
+      .def(
+          "get_dialect_descriptor",
+          [=](PyMlirContext &self, std::string &name) {
+            MlirDialect dialect = mlirContextGetOrLoadDialect(
+                self.get(), {name.data(), name.size()});
+            if (mlirDialectIsNull(dialect)) {
+              throw SetPyError(PyExc_ValueError,
+                               Twine("Dialect '") + name + "' not found");
             }
-          }
-          MlirAttribute attr = mlirArrayAttrGet(
-              context->get(), mlirAttributes.size(), mlirAttributes.data());
-          return PyArrayAttribute(context->getRef(), attr);
-        },
-        py::arg("attributes"), py::arg("context") = py::none(),
-        "Gets a uniqued Array attribute");
-    c.def("__getitem__",
-          [](PyArrayAttribute &arr, intptr_t i) {
-            if (i >= mlirArrayAttrGetNumElements(arr))
-              throw py::index_error("ArrayAttribute index out of range");
-            return PyAttribute(arr.getContext(),
-                               mlirArrayAttrGetElement(arr, i));
-          })
-        .def("__len__",
-             [](const PyArrayAttribute &arr) {
-               return mlirArrayAttrGetNumElements(arr);
-             })
-        .def("__iter__", [](const PyArrayAttribute &arr) {
-          return PyArrayAttributeIterator(arr);
-        });
-  }
-};
-
-/// Float Point Attribute subclass - FloatAttr.
-class PyFloatAttribute : public PyConcreteAttribute<PyFloatAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAFloat;
-  static constexpr const char *pyClassName = "FloatAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](PyType &type, double value, DefaultingPyLocation loc) {
-          MlirAttribute attr = mlirFloatAttrDoubleGetChecked(loc, type, value);
-          // TODO: Rework error reporting once diagnostic engine is exposed
-          // in C API.
-          if (mlirAttributeIsNull(attr)) {
-            throw SetPyError(PyExc_ValueError,
-                             Twine("invalid '") +
-                                 py::repr(py::cast(type)).cast<std::string>() +
-                                 "' and expected floating point type.");
-          }
-          return PyFloatAttribute(type.getContext(), attr);
-        },
-        py::arg("type"), py::arg("value"), py::arg("loc") = py::none(),
-        "Gets an uniqued float point attribute associated to a type");
-    c.def_static(
-        "get_f32",
-        [](double value, DefaultingPyMlirContext context) {
-          MlirAttribute attr = mlirFloatAttrDoubleGet(
-              context->get(), mlirF32TypeGet(context->get()), value);
-          return PyFloatAttribute(context->getRef(), attr);
-        },
-        py::arg("value"), py::arg("context") = py::none(),
-        "Gets an uniqued float point attribute associated to a f32 type");
-    c.def_static(
-        "get_f64",
-        [](double value, DefaultingPyMlirContext context) {
-          MlirAttribute attr = mlirFloatAttrDoubleGet(
-              context->get(), mlirF64TypeGet(context->get()), value);
-          return PyFloatAttribute(context->getRef(), attr);
-        },
-        py::arg("value"), py::arg("context") = py::none(),
-        "Gets an uniqued float point attribute associated to a f64 type");
-    c.def_property_readonly(
-        "value",
-        [](PyFloatAttribute &self) {
-          return mlirFloatAttrGetValueDouble(self);
-        },
-        "Returns the value of the float point attribute");
-  }
-};
-
-/// Integer Attribute subclass - IntegerAttr.
-class PyIntegerAttribute : public PyConcreteAttribute<PyIntegerAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAInteger;
-  static constexpr const char *pyClassName = "IntegerAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](PyType &type, int64_t value) {
-          MlirAttribute attr = mlirIntegerAttrGet(type, value);
-          return PyIntegerAttribute(type.getContext(), attr);
-        },
-        py::arg("type"), py::arg("value"),
-        "Gets an uniqued integer attribute associated to a type");
-    c.def_property_readonly(
-        "value",
-        [](PyIntegerAttribute &self) {
-          return mlirIntegerAttrGetValueInt(self);
-        },
-        "Returns the value of the integer attribute");
-  }
-};
-
-/// Bool Attribute subclass - BoolAttr.
-class PyBoolAttribute : public PyConcreteAttribute<PyBoolAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsABool;
-  static constexpr const char *pyClassName = "BoolAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](bool value, DefaultingPyMlirContext context) {
-          MlirAttribute attr = mlirBoolAttrGet(context->get(), value);
-          return PyBoolAttribute(context->getRef(), attr);
-        },
-        py::arg("value"), py::arg("context") = py::none(),
-        "Gets an uniqued bool attribute");
-    c.def_property_readonly(
-        "value",
-        [](PyBoolAttribute &self) { return mlirBoolAttrGetValue(self); },
-        "Returns the value of the bool attribute");
-  }
-};
-
-class PyFlatSymbolRefAttribute
-    : public PyConcreteAttribute<PyFlatSymbolRefAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAFlatSymbolRef;
-  static constexpr const char *pyClassName = "FlatSymbolRefAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](std::string value, DefaultingPyMlirContext context) {
-          MlirAttribute attr =
-              mlirFlatSymbolRefAttrGet(context->get(), toMlirStringRef(value));
-          return PyFlatSymbolRefAttribute(context->getRef(), attr);
-        },
-        py::arg("value"), py::arg("context") = py::none(),
-        "Gets a uniqued FlatSymbolRef attribute");
-    c.def_property_readonly(
-        "value",
-        [](PyFlatSymbolRefAttribute &self) {
-          MlirStringRef stringRef = mlirFlatSymbolRefAttrGetValue(self);
-          return py::str(stringRef.data, stringRef.length);
-        },
-        "Returns the value of the FlatSymbolRef attribute as a string");
-  }
-};
-
-class PyStringAttribute : public PyConcreteAttribute<PyStringAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAString;
-  static constexpr const char *pyClassName = "StringAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](std::string value, DefaultingPyMlirContext context) {
-          MlirAttribute attr =
-              mlirStringAttrGet(context->get(), toMlirStringRef(value));
-          return PyStringAttribute(context->getRef(), attr);
-        },
-        py::arg("value"), py::arg("context") = py::none(),
-        "Gets a uniqued string attribute");
-    c.def_static(
-        "get_typed",
-        [](PyType &type, std::string value) {
-          MlirAttribute attr =
-              mlirStringAttrTypedGet(type, toMlirStringRef(value));
-          return PyStringAttribute(type.getContext(), attr);
-        },
-
-        "Gets a uniqued string attribute associated to a type");
-    c.def_property_readonly(
-        "value",
-        [](PyStringAttribute &self) {
-          MlirStringRef stringRef = mlirStringAttrGetValue(self);
-          return py::str(stringRef.data, stringRef.length);
-        },
-        "Returns the value of the string attribute");
-  }
-};
-
-// TODO: Support construction of bool elements.
-// TODO: Support construction of string elements.
-class PyDenseElementsAttribute
-    : public PyConcreteAttribute<PyDenseElementsAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsADenseElements;
-  static constexpr const char *pyClassName = "DenseElementsAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  static PyDenseElementsAttribute
-  getFromBuffer(py::buffer array, bool signless,
-                DefaultingPyMlirContext contextWrapper) {
-    // Request a contiguous view. In exotic cases, this will cause a copy.
-    int flags = PyBUF_C_CONTIGUOUS | PyBUF_FORMAT;
-    Py_buffer *view = new Py_buffer();
-    if (PyObject_GetBuffer(array.ptr(), view, flags) != 0) {
-      delete view;
-      throw py::error_already_set();
-    }
-    py::buffer_info arrayInfo(view);
-
-    MlirContext context = contextWrapper->get();
-    // Switch on the types that can be bulk loaded between the Python and
-    // MLIR-C APIs.
-    // See: https://docs.python.org/3/library/struct.html#format-characters
-    if (arrayInfo.format == "f") {
-      // f32
-      assert(arrayInfo.itemsize == 4 && "mismatched array itemsize");
-      return PyDenseElementsAttribute(
-          contextWrapper->getRef(),
-          bulkLoad(context, mlirDenseElementsAttrFloatGet,
-                   mlirF32TypeGet(context), arrayInfo));
-    } else if (arrayInfo.format == "d") {
-      // f64
-      assert(arrayInfo.itemsize == 8 && "mismatched array itemsize");
-      return PyDenseElementsAttribute(
-          contextWrapper->getRef(),
-          bulkLoad(context, mlirDenseElementsAttrDoubleGet,
-                   mlirF64TypeGet(context), arrayInfo));
-    } else if (isSignedIntegerFormat(arrayInfo.format)) {
-      if (arrayInfo.itemsize == 4) {
-        // i32
-        MlirType elementType = signless ? mlirIntegerTypeGet(context, 32)
-                                        : mlirIntegerTypeSignedGet(context, 32);
-        return PyDenseElementsAttribute(contextWrapper->getRef(),
-                                        bulkLoad(context,
-                                                 mlirDenseElementsAttrInt32Get,
-                                                 elementType, arrayInfo));
-      } else if (arrayInfo.itemsize == 8) {
-        // i64
-        MlirType elementType = signless ? mlirIntegerTypeGet(context, 64)
-                                        : mlirIntegerTypeSignedGet(context, 64);
-        return PyDenseElementsAttribute(contextWrapper->getRef(),
-                                        bulkLoad(context,
-                                                 mlirDenseElementsAttrInt64Get,
-                                                 elementType, arrayInfo));
-      }
-    } else if (isUnsignedIntegerFormat(arrayInfo.format)) {
-      if (arrayInfo.itemsize == 4) {
-        // unsigned i32
-        MlirType elementType = signless
-                                   ? mlirIntegerTypeGet(context, 32)
-                                   : mlirIntegerTypeUnsignedGet(context, 32);
-        return PyDenseElementsAttribute(contextWrapper->getRef(),
-                                        bulkLoad(context,
-                                                 mlirDenseElementsAttrUInt32Get,
-                                                 elementType, arrayInfo));
-      } else if (arrayInfo.itemsize == 8) {
-        // unsigned i64
-        MlirType elementType = signless
-                                   ? mlirIntegerTypeGet(context, 64)
-                                   : mlirIntegerTypeUnsignedGet(context, 64);
-        return PyDenseElementsAttribute(contextWrapper->getRef(),
-                                        bulkLoad(context,
-                                                 mlirDenseElementsAttrUInt64Get,
-                                                 elementType, arrayInfo));
-      }
-    }
-
-    // TODO: Fall back to string-based get.
-    std::string message = "unimplemented array format conversion from format: ";
-    message.append(arrayInfo.format);
-    throw SetPyError(PyExc_ValueError, message);
-  }
-
-  static PyDenseElementsAttribute getSplat(PyType shapedType,
-                                           PyAttribute &elementAttr) {
-    auto contextWrapper =
-        PyMlirContext::forContext(mlirTypeGetContext(shapedType));
-    if (!mlirAttributeIsAInteger(elementAttr) &&
-        !mlirAttributeIsAFloat(elementAttr)) {
-      std::string message = "Illegal element type for DenseElementsAttr: ";
-      message.append(py::repr(py::cast(elementAttr)));
-      throw SetPyError(PyExc_ValueError, message);
-    }
-    if (!mlirTypeIsAShaped(shapedType) ||
-        !mlirShapedTypeHasStaticShape(shapedType)) {
-      std::string message =
-          "Expected a static ShapedType for the shaped_type parameter: ";
-      message.append(py::repr(py::cast(shapedType)));
-      throw SetPyError(PyExc_ValueError, message);
-    }
-    MlirType shapedElementType = mlirShapedTypeGetElementType(shapedType);
-    MlirType attrType = mlirAttributeGetType(elementAttr);
-    if (!mlirTypeEqual(shapedElementType, attrType)) {
-      std::string message =
-          "Shaped element type and attribute type must be equal: shaped=";
-      message.append(py::repr(py::cast(shapedType)));
-      message.append(", element=");
-      message.append(py::repr(py::cast(elementAttr)));
-      throw SetPyError(PyExc_ValueError, message);
-    }
-
-    MlirAttribute elements =
-        mlirDenseElementsAttrSplatGet(shapedType, elementAttr);
-    return PyDenseElementsAttribute(contextWrapper->getRef(), elements);
-  }
+            return PyDialectDescriptor(self.getRef(), dialect);
+          },
+          "Gets or loads a dialect by name, returning its descriptor object")
+      .def_property(
+          "allow_unregistered_dialects",
+          [](PyMlirContext &self) -> bool {
+            return mlirContextGetAllowUnregisteredDialects(self.get());
+          },
+          [](PyMlirContext &self, bool value) {
+            mlirContextSetAllowUnregisteredDialects(self.get(), value);
+          });
 
-  intptr_t dunderLen() { return mlirElementsAttrGetNumElements(*this); }
-
-  py::buffer_info accessBuffer() {
-    MlirType shapedType = mlirAttributeGetType(*this);
-    MlirType elementType = mlirShapedTypeGetElementType(shapedType);
-
-    if (mlirTypeIsAF32(elementType)) {
-      // f32
-      return bufferInfo(shapedType, mlirDenseElementsAttrGetFloatValue);
-    } else if (mlirTypeIsAF64(elementType)) {
-      // f64
-      return bufferInfo(shapedType, mlirDenseElementsAttrGetDoubleValue);
-    } else if (mlirTypeIsAInteger(elementType) &&
-               mlirIntegerTypeGetWidth(elementType) == 32) {
-      if (mlirIntegerTypeIsSignless(elementType) ||
-          mlirIntegerTypeIsSigned(elementType)) {
-        // i32
-        return bufferInfo(shapedType, mlirDenseElementsAttrGetInt32Value);
-      } else if (mlirIntegerTypeIsUnsigned(elementType)) {
-        // unsigned i32
-        return bufferInfo(shapedType, mlirDenseElementsAttrGetUInt32Value);
-      }
-    } else if (mlirTypeIsAInteger(elementType) &&
-               mlirIntegerTypeGetWidth(elementType) == 64) {
-      if (mlirIntegerTypeIsSignless(elementType) ||
-          mlirIntegerTypeIsSigned(elementType)) {
-        // i64
-        return bufferInfo(shapedType, mlirDenseElementsAttrGetInt64Value);
-      } else if (mlirIntegerTypeIsUnsigned(elementType)) {
-        // unsigned i64
-        return bufferInfo(shapedType, mlirDenseElementsAttrGetUInt64Value);
-      }
-    }
+  //----------------------------------------------------------------------------
+  // Mapping of PyDialectDescriptor
+  //----------------------------------------------------------------------------
+  py::class_<PyDialectDescriptor>(m, "DialectDescriptor")
+      .def_property_readonly("namespace",
+                             [](PyDialectDescriptor &self) {
+                               MlirStringRef ns =
+                                   mlirDialectGetNamespace(self.get());
+                               return py::str(ns.data, ns.length);
+                             })
+      .def("__repr__", [](PyDialectDescriptor &self) {
+        MlirStringRef ns = mlirDialectGetNamespace(self.get());
+        std::string repr("<DialectDescriptor ");
+        repr.append(ns.data, ns.length);
+        repr.append(">");
+        return repr;
+      });
 
-    std::string message = "unimplemented array format.";
-    throw SetPyError(PyExc_ValueError, message);
-  }
+  //----------------------------------------------------------------------------
+  // Mapping of PyDialects
+  //----------------------------------------------------------------------------
+  py::class_<PyDialects>(m, "Dialects")
+      .def("__getitem__",
+           [=](PyDialects &self, std::string keyName) {
+             MlirDialect dialect =
+                 self.getDialectForKey(keyName, /*attrError=*/false);
+             py::object descriptor =
+                 py::cast(PyDialectDescriptor{self.getContext(), dialect});
+             return createCustomDialectWrapper(keyName, std::move(descriptor));
+           })
+      .def("__getattr__", [=](PyDialects &self, std::string attrName) {
+        MlirDialect dialect =
+            self.getDialectForKey(attrName, /*attrError=*/true);
+        py::object descriptor =
+            py::cast(PyDialectDescriptor{self.getContext(), dialect});
+        return createCustomDialectWrapper(attrName, std::move(descriptor));
+      });
 
-  static void bindDerived(ClassTy &c) {
-    c.def("__len__", &PyDenseElementsAttribute::dunderLen)
-        .def_static("get", PyDenseElementsAttribute::getFromBuffer,
-                    py::arg("array"), py::arg("signless") = true,
-                    py::arg("context") = py::none(),
-                    "Gets from a buffer or ndarray")
-        .def_static("get_splat", PyDenseElementsAttribute::getSplat,
-                    py::arg("shaped_type"), py::arg("element_attr"),
-                    "Gets a DenseElementsAttr where all values are the same")
-        .def_property_readonly("is_splat",
-                               [](PyDenseElementsAttribute &self) -> bool {
-                                 return mlirDenseElementsAttrIsSplat(self);
-                               })
-        .def_buffer(&PyDenseElementsAttribute::accessBuffer);
-  }
+  //----------------------------------------------------------------------------
+  // Mapping of PyDialect
+  //----------------------------------------------------------------------------
+  py::class_<PyDialect>(m, "Dialect")
+      .def(py::init<py::object>(), "descriptor")
+      .def_property_readonly(
+          "descriptor", [](PyDialect &self) { return self.getDescriptor(); })
+      .def("__repr__", [](py::object self) {
+        auto clazz = self.attr("__class__");
+        return py::str("<Dialect ") +
+               self.attr("descriptor").attr("namespace") + py::str(" (class ") +
+               clazz.attr("__module__") + py::str(".") +
+               clazz.attr("__name__") + py::str(")>");
+      });
 
-private:
-  template <typename ElementTy>
-  static MlirAttribute
-  bulkLoad(MlirContext context,
-           MlirAttribute (*ctor)(MlirType, intptr_t, ElementTy *),
-           MlirType mlirElementType, py::buffer_info &arrayInfo) {
-    SmallVector<int64_t, 4> shape(arrayInfo.shape.begin(),
-                                  arrayInfo.shape.begin() + arrayInfo.ndim);
-    auto shapedType =
-        mlirRankedTensorTypeGet(shape.size(), shape.data(), mlirElementType);
-    intptr_t numElements = arrayInfo.size;
-    const ElementTy *contents = static_cast<const ElementTy *>(arrayInfo.ptr);
-    return ctor(shapedType, numElements, contents);
-  }
-
-  static bool isUnsignedIntegerFormat(const std::string &format) {
-    if (format.empty())
-      return false;
-    char code = format[0];
-    return code == 'I' || code == 'B' || code == 'H' || code == 'L' ||
-           code == 'Q';
-  }
-
-  static bool isSignedIntegerFormat(const std::string &format) {
-    if (format.empty())
-      return false;
-    char code = format[0];
-    return code == 'i' || code == 'b' || code == 'h' || code == 'l' ||
-           code == 'q';
-  }
-
-  template <typename Type>
-  py::buffer_info bufferInfo(MlirType shapedType,
-                             Type (*value)(MlirAttribute, intptr_t)) {
-    intptr_t rank = mlirShapedTypeGetRank(shapedType);
-    // Prepare the data for the buffer_info.
-    // Buffer is configured for read-only access below.
-    Type *data = static_cast<Type *>(
-        const_cast<void *>(mlirDenseElementsAttrGetRawData(*this)));
-    // Prepare the shape for the buffer_info.
-    SmallVector<intptr_t, 4> shape;
-    for (intptr_t i = 0; i < rank; ++i)
-      shape.push_back(mlirShapedTypeGetDimSize(shapedType, i));
-    // Prepare the strides for the buffer_info.
-    SmallVector<intptr_t, 4> strides;
-    intptr_t strideFactor = 1;
-    for (intptr_t i = 1; i < rank; ++i) {
-      strideFactor = 1;
-      for (intptr_t j = i; j < rank; ++j) {
-        strideFactor *= mlirShapedTypeGetDimSize(shapedType, j);
-      }
-      strides.push_back(sizeof(Type) * strideFactor);
-    }
-    strides.push_back(sizeof(Type));
-    return py::buffer_info(data, sizeof(Type),
-                           py::format_descriptor<Type>::format(), rank, shape,
-                           strides, /*readonly=*/true);
-  }
-}; // namespace
-
-/// Refinement of the PyDenseElementsAttribute for attributes containing integer
-/// (and boolean) values. Supports element access.
-class PyDenseIntElementsAttribute
-    : public PyConcreteAttribute<PyDenseIntElementsAttribute,
-                                 PyDenseElementsAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsADenseIntElements;
-  static constexpr const char *pyClassName = "DenseIntElementsAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  /// Returns the element at the given linear position. Asserts if the index is
-  /// out of range.
-  py::int_ dunderGetItem(intptr_t pos) {
-    if (pos < 0 || pos >= dunderLen()) {
-      throw SetPyError(PyExc_IndexError,
-                       "attempt to access out of bounds element");
-    }
-
-    MlirType type = mlirAttributeGetType(*this);
-    type = mlirShapedTypeGetElementType(type);
-    assert(mlirTypeIsAInteger(type) &&
-           "expected integer element type in dense int elements attribute");
-    // Dispatch element extraction to an appropriate C function based on the
-    // elemental type of the attribute. py::int_ is implicitly constructible
-    // from any C++ integral type and handles bitwidth correctly.
-    // TODO: consider caching the type properties in the constructor to avoid
-    // querying them on each element access.
-    unsigned width = mlirIntegerTypeGetWidth(type);
-    bool isUnsigned = mlirIntegerTypeIsUnsigned(type);
-    if (isUnsigned) {
-      if (width == 1) {
-        return mlirDenseElementsAttrGetBoolValue(*this, pos);
-      }
-      if (width == 32) {
-        return mlirDenseElementsAttrGetUInt32Value(*this, pos);
-      }
-      if (width == 64) {
-        return mlirDenseElementsAttrGetUInt64Value(*this, pos);
-      }
-    } else {
-      if (width == 1) {
-        return mlirDenseElementsAttrGetBoolValue(*this, pos);
-      }
-      if (width == 32) {
-        return mlirDenseElementsAttrGetInt32Value(*this, pos);
-      }
-      if (width == 64) {
-        return mlirDenseElementsAttrGetInt64Value(*this, pos);
-      }
-    }
-    throw SetPyError(PyExc_TypeError, "Unsupported integer type");
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def("__getitem__", &PyDenseIntElementsAttribute::dunderGetItem);
-  }
-};
-
-class PyDictAttribute : public PyConcreteAttribute<PyDictAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsADictionary;
-  static constexpr const char *pyClassName = "DictAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  intptr_t dunderLen() { return mlirDictionaryAttrGetNumElements(*this); }
-
-  static void bindDerived(ClassTy &c) {
-    c.def("__len__", &PyDictAttribute::dunderLen);
-    c.def_static(
-        "get",
-        [](py::dict attributes, DefaultingPyMlirContext context) {
-          SmallVector<MlirNamedAttribute> mlirNamedAttributes;
-          mlirNamedAttributes.reserve(attributes.size());
-          for (auto &it : attributes) {
-            auto &mlir_attr = it.second.cast<PyAttribute &>();
-            auto name = it.first.cast<std::string>();
-            mlirNamedAttributes.push_back(mlirNamedAttributeGet(
-                mlirIdentifierGet(mlirAttributeGetContext(mlir_attr),
-                                  toMlirStringRef(name)),
-                mlir_attr));
-          }
-          MlirAttribute attr =
-              mlirDictionaryAttrGet(context->get(), mlirNamedAttributes.size(),
-                                    mlirNamedAttributes.data());
-          return PyDictAttribute(context->getRef(), attr);
-        },
-        py::arg("value"), py::arg("context") = py::none(),
-        "Gets an uniqued dict attribute");
-    c.def("__getitem__", [](PyDictAttribute &self, const std::string &name) {
-      MlirAttribute attr =
-          mlirDictionaryAttrGetElementByName(self, toMlirStringRef(name));
-      if (mlirAttributeIsNull(attr)) {
-        throw SetPyError(PyExc_KeyError,
-                         "attempt to access a non-existent attribute");
-      }
-      return PyAttribute(self.getContext(), attr);
-    });
-    c.def("__getitem__", [](PyDictAttribute &self, intptr_t index) {
-      if (index < 0 || index >= self.dunderLen()) {
-        throw SetPyError(PyExc_IndexError,
-                         "attempt to access out of bounds attribute");
-      }
-      MlirNamedAttribute namedAttr = mlirDictionaryAttrGetElement(self, index);
-      return PyNamedAttribute(
-          namedAttr.attribute,
-          std::string(mlirIdentifierStr(namedAttr.name).data));
-    });
-  }
-};
-
-/// Refinement of PyDenseElementsAttribute for attributes containing
-/// floating-point values. Supports element access.
-class PyDenseFPElementsAttribute
-    : public PyConcreteAttribute<PyDenseFPElementsAttribute,
-                                 PyDenseElementsAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsADenseFPElements;
-  static constexpr const char *pyClassName = "DenseFPElementsAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  py::float_ dunderGetItem(intptr_t pos) {
-    if (pos < 0 || pos >= dunderLen()) {
-      throw SetPyError(PyExc_IndexError,
-                       "attempt to access out of bounds element");
-    }
-
-    MlirType type = mlirAttributeGetType(*this);
-    type = mlirShapedTypeGetElementType(type);
-    // Dispatch element extraction to an appropriate C function based on the
-    // elemental type of the attribute. py::float_ is implicitly constructible
-    // from float and double.
-    // TODO: consider caching the type properties in the constructor to avoid
-    // querying them on each element access.
-    if (mlirTypeIsAF32(type)) {
-      return mlirDenseElementsAttrGetFloatValue(*this, pos);
-    }
-    if (mlirTypeIsAF64(type)) {
-      return mlirDenseElementsAttrGetDoubleValue(*this, pos);
-    }
-    throw SetPyError(PyExc_TypeError, "Unsupported floating-point type");
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def("__getitem__", &PyDenseFPElementsAttribute::dunderGetItem);
-  }
-};
-
-class PyTypeAttribute : public PyConcreteAttribute<PyTypeAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAType;
-  static constexpr const char *pyClassName = "TypeAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](PyType value, DefaultingPyMlirContext context) {
-          MlirAttribute attr = mlirTypeAttrGet(value.get());
-          return PyTypeAttribute(context->getRef(), attr);
-        },
-        py::arg("value"), py::arg("context") = py::none(),
-        "Gets a uniqued Type attribute");
-    c.def_property_readonly("value", [](PyTypeAttribute &self) {
-      return PyType(self.getContext()->getRef(),
-                    mlirTypeAttrGetValue(self.get()));
-    });
-  }
-};
-
-/// Unit Attribute subclass. Unit attributes don't have values.
-class PyUnitAttribute : public PyConcreteAttribute<PyUnitAttribute> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAUnit;
-  static constexpr const char *pyClassName = "UnitAttr";
-  using PyConcreteAttribute::PyConcreteAttribute;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](DefaultingPyMlirContext context) {
-          return PyUnitAttribute(context->getRef(),
-                                 mlirUnitAttrGet(context->get()));
-        },
-        py::arg("context") = py::none(), "Create a Unit attribute.");
-  }
-};
-
-} // namespace
-
-//------------------------------------------------------------------------------
-// Builtin type subclasses.
-//------------------------------------------------------------------------------
-
-namespace {
-
-/// CRTP base classes for Python types that subclass Type and should be
-/// castable from it (i.e. via something like IntegerType(t)).
-/// By default, type class hierarchies are one level deep (i.e. a
-/// concrete type class extends PyType); however, intermediate python-visible
-/// base classes can be modeled by specifying a BaseTy.
-template <typename DerivedTy, typename BaseTy = PyType>
-class PyConcreteType : public BaseTy {
-public:
-  // Derived classes must define statics for:
-  //   IsAFunctionTy isaFunction
-  //   const char *pyClassName
-  using ClassTy = py::class_<DerivedTy, BaseTy>;
-  using IsAFunctionTy = bool (*)(MlirType);
-
-  PyConcreteType() = default;
-  PyConcreteType(PyMlirContextRef contextRef, MlirType t)
-      : BaseTy(std::move(contextRef), t) {}
-  PyConcreteType(PyType &orig)
-      : PyConcreteType(orig.getContext(), castFrom(orig)) {}
-
-  static MlirType castFrom(PyType &orig) {
-    if (!DerivedTy::isaFunction(orig)) {
-      auto origRepr = py::repr(py::cast(orig)).cast<std::string>();
-      throw SetPyError(PyExc_ValueError, Twine("Cannot cast type to ") +
-                                             DerivedTy::pyClassName +
-                                             " (from " + origRepr + ")");
-    }
-    return orig;
-  }
-
-  static void bind(py::module &m) {
-    auto cls = ClassTy(m, DerivedTy::pyClassName);
-    cls.def(py::init<PyType &>(), py::keep_alive<0, 1>());
-    DerivedTy::bindDerived(cls);
-  }
-
-  /// Implemented by derived classes to add methods to the Python subclass.
-  static void bindDerived(ClassTy &m) {}
-};
-
-class PyIntegerType : public PyConcreteType<PyIntegerType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAInteger;
-  static constexpr const char *pyClassName = "IntegerType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get_signless",
-        [](unsigned width, DefaultingPyMlirContext context) {
-          MlirType t = mlirIntegerTypeGet(context->get(), width);
-          return PyIntegerType(context->getRef(), t);
-        },
-        py::arg("width"), py::arg("context") = py::none(),
-        "Create a signless integer type");
-    c.def_static(
-        "get_signed",
-        [](unsigned width, DefaultingPyMlirContext context) {
-          MlirType t = mlirIntegerTypeSignedGet(context->get(), width);
-          return PyIntegerType(context->getRef(), t);
-        },
-        py::arg("width"), py::arg("context") = py::none(),
-        "Create a signed integer type");
-    c.def_static(
-        "get_unsigned",
-        [](unsigned width, DefaultingPyMlirContext context) {
-          MlirType t = mlirIntegerTypeUnsignedGet(context->get(), width);
-          return PyIntegerType(context->getRef(), t);
-        },
-        py::arg("width"), py::arg("context") = py::none(),
-        "Create an unsigned integer type");
-    c.def_property_readonly(
-        "width",
-        [](PyIntegerType &self) { return mlirIntegerTypeGetWidth(self); },
-        "Returns the width of the integer type");
-    c.def_property_readonly(
-        "is_signless",
-        [](PyIntegerType &self) -> bool {
-          return mlirIntegerTypeIsSignless(self);
-        },
-        "Returns whether this is a signless integer");
-    c.def_property_readonly(
-        "is_signed",
-        [](PyIntegerType &self) -> bool {
-          return mlirIntegerTypeIsSigned(self);
-        },
-        "Returns whether this is a signed integer");
-    c.def_property_readonly(
-        "is_unsigned",
-        [](PyIntegerType &self) -> bool {
-          return mlirIntegerTypeIsUnsigned(self);
-        },
-        "Returns whether this is an unsigned integer");
-  }
-};
-
-/// Index Type subclass - IndexType.
-class PyIndexType : public PyConcreteType<PyIndexType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAIndex;
-  static constexpr const char *pyClassName = "IndexType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](DefaultingPyMlirContext context) {
-          MlirType t = mlirIndexTypeGet(context->get());
-          return PyIndexType(context->getRef(), t);
-        },
-        py::arg("context") = py::none(), "Create a index type.");
-  }
-};
-
-/// Floating Point Type subclass - BF16Type.
-class PyBF16Type : public PyConcreteType<PyBF16Type> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsABF16;
-  static constexpr const char *pyClassName = "BF16Type";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](DefaultingPyMlirContext context) {
-          MlirType t = mlirBF16TypeGet(context->get());
-          return PyBF16Type(context->getRef(), t);
-        },
-        py::arg("context") = py::none(), "Create a bf16 type.");
-  }
-};
-
-/// Floating Point Type subclass - F16Type.
-class PyF16Type : public PyConcreteType<PyF16Type> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAF16;
-  static constexpr const char *pyClassName = "F16Type";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](DefaultingPyMlirContext context) {
-          MlirType t = mlirF16TypeGet(context->get());
-          return PyF16Type(context->getRef(), t);
-        },
-        py::arg("context") = py::none(), "Create a f16 type.");
-  }
-};
-
-/// Floating Point Type subclass - F32Type.
-class PyF32Type : public PyConcreteType<PyF32Type> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAF32;
-  static constexpr const char *pyClassName = "F32Type";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](DefaultingPyMlirContext context) {
-          MlirType t = mlirF32TypeGet(context->get());
-          return PyF32Type(context->getRef(), t);
-        },
-        py::arg("context") = py::none(), "Create a f32 type.");
-  }
-};
-
-/// Floating Point Type subclass - F64Type.
-class PyF64Type : public PyConcreteType<PyF64Type> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAF64;
-  static constexpr const char *pyClassName = "F64Type";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](DefaultingPyMlirContext context) {
-          MlirType t = mlirF64TypeGet(context->get());
-          return PyF64Type(context->getRef(), t);
-        },
-        py::arg("context") = py::none(), "Create a f64 type.");
-  }
-};
-
-/// None Type subclass - NoneType.
-class PyNoneType : public PyConcreteType<PyNoneType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsANone;
-  static constexpr const char *pyClassName = "NoneType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](DefaultingPyMlirContext context) {
-          MlirType t = mlirNoneTypeGet(context->get());
-          return PyNoneType(context->getRef(), t);
-        },
-        py::arg("context") = py::none(), "Create a none type.");
-  }
-};
-
-/// Complex Type subclass - ComplexType.
-class PyComplexType : public PyConcreteType<PyComplexType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAComplex;
-  static constexpr const char *pyClassName = "ComplexType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](PyType &elementType) {
-          // The element must be a floating point or integer scalar type.
-          if (mlirTypeIsAIntegerOrFloat(elementType)) {
-            MlirType t = mlirComplexTypeGet(elementType);
-            return PyComplexType(elementType.getContext(), t);
-          }
-          throw SetPyError(
-              PyExc_ValueError,
-              Twine("invalid '") +
-                  py::repr(py::cast(elementType)).cast<std::string>() +
-                  "' and expected floating point or integer type.");
-        },
-        "Create a complex type");
-    c.def_property_readonly(
-        "element_type",
-        [](PyComplexType &self) -> PyType {
-          MlirType t = mlirComplexTypeGetElementType(self);
-          return PyType(self.getContext(), t);
-        },
-        "Returns element type.");
-  }
-};
-
-class PyShapedType : public PyConcreteType<PyShapedType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAShaped;
-  static constexpr const char *pyClassName = "ShapedType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_property_readonly(
-        "element_type",
-        [](PyShapedType &self) {
-          MlirType t = mlirShapedTypeGetElementType(self);
-          return PyType(self.getContext(), t);
-        },
-        "Returns the element type of the shaped type.");
-    c.def_property_readonly(
-        "has_rank",
-        [](PyShapedType &self) -> bool { return mlirShapedTypeHasRank(self); },
-        "Returns whether the given shaped type is ranked.");
-    c.def_property_readonly(
-        "rank",
-        [](PyShapedType &self) {
-          self.requireHasRank();
-          return mlirShapedTypeGetRank(self);
-        },
-        "Returns the rank of the given ranked shaped type.");
-    c.def_property_readonly(
-        "has_static_shape",
-        [](PyShapedType &self) -> bool {
-          return mlirShapedTypeHasStaticShape(self);
-        },
-        "Returns whether the given shaped type has a static shape.");
-    c.def(
-        "is_dynamic_dim",
-        [](PyShapedType &self, intptr_t dim) -> bool {
-          self.requireHasRank();
-          return mlirShapedTypeIsDynamicDim(self, dim);
-        },
-        "Returns whether the dim-th dimension of the given shaped type is "
-        "dynamic.");
-    c.def(
-        "get_dim_size",
-        [](PyShapedType &self, intptr_t dim) {
-          self.requireHasRank();
-          return mlirShapedTypeGetDimSize(self, dim);
-        },
-        "Returns the dim-th dimension of the given ranked shaped type.");
-    c.def_static(
-        "is_dynamic_size",
-        [](int64_t size) -> bool { return mlirShapedTypeIsDynamicSize(size); },
-        "Returns whether the given dimension size indicates a dynamic "
-        "dimension.");
-    c.def(
-        "is_dynamic_stride_or_offset",
-        [](PyShapedType &self, int64_t val) -> bool {
-          self.requireHasRank();
-          return mlirShapedTypeIsDynamicStrideOrOffset(val);
-        },
-        "Returns whether the given value is used as a placeholder for dynamic "
-        "strides and offsets in shaped types.");
-  }
-
-private:
-  void requireHasRank() {
-    if (!mlirShapedTypeHasRank(*this)) {
-      throw SetPyError(
-          PyExc_ValueError,
-          "calling this method requires that the type has a rank.");
-    }
-  }
-};
-
-/// Vector Type subclass - VectorType.
-class PyVectorType : public PyConcreteType<PyVectorType, PyShapedType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAVector;
-  static constexpr const char *pyClassName = "VectorType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](std::vector<int64_t> shape, PyType &elementType,
-           DefaultingPyLocation loc) {
-          MlirType t = mlirVectorTypeGetChecked(loc, shape.size(), shape.data(),
-                                                elementType);
-          // TODO: Rework error reporting once diagnostic engine is exposed
-          // in C API.
-          if (mlirTypeIsNull(t)) {
-            throw SetPyError(
-                PyExc_ValueError,
-                Twine("invalid '") +
-                    py::repr(py::cast(elementType)).cast<std::string>() +
-                    "' and expected floating point or integer type.");
-          }
-          return PyVectorType(elementType.getContext(), t);
-        },
-        py::arg("shape"), py::arg("elementType"), py::arg("loc") = py::none(),
-        "Create a vector type");
-  }
-};
-
-/// Ranked Tensor Type subclass - RankedTensorType.
-class PyRankedTensorType
-    : public PyConcreteType<PyRankedTensorType, PyShapedType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsARankedTensor;
-  static constexpr const char *pyClassName = "RankedTensorType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](std::vector<int64_t> shape, PyType &elementType,
-           DefaultingPyLocation loc) {
-          MlirType t = mlirRankedTensorTypeGetChecked(
-              loc, shape.size(), shape.data(), elementType);
-          // TODO: Rework error reporting once diagnostic engine is exposed
-          // in C API.
-          if (mlirTypeIsNull(t)) {
-            throw SetPyError(
-                PyExc_ValueError,
-                Twine("invalid '") +
-                    py::repr(py::cast(elementType)).cast<std::string>() +
-                    "' and expected floating point, integer, vector or "
-                    "complex "
-                    "type.");
-          }
-          return PyRankedTensorType(elementType.getContext(), t);
-        },
-        py::arg("shape"), py::arg("element_type"), py::arg("loc") = py::none(),
-        "Create a ranked tensor type");
-  }
-};
-
-/// Unranked Tensor Type subclass - UnrankedTensorType.
-class PyUnrankedTensorType
-    : public PyConcreteType<PyUnrankedTensorType, PyShapedType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAUnrankedTensor;
-  static constexpr const char *pyClassName = "UnrankedTensorType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](PyType &elementType, DefaultingPyLocation loc) {
-          MlirType t = mlirUnrankedTensorTypeGetChecked(loc, elementType);
-          // TODO: Rework error reporting once diagnostic engine is exposed
-          // in C API.
-          if (mlirTypeIsNull(t)) {
-            throw SetPyError(
-                PyExc_ValueError,
-                Twine("invalid '") +
-                    py::repr(py::cast(elementType)).cast<std::string>() +
-                    "' and expected floating point, integer, vector or "
-                    "complex "
-                    "type.");
-          }
-          return PyUnrankedTensorType(elementType.getContext(), t);
-        },
-        py::arg("element_type"), py::arg("loc") = py::none(),
-        "Create a unranked tensor type");
-  }
-};
-
-class PyMemRefLayoutMapList;
-
-/// Ranked MemRef Type subclass - MemRefType.
-class PyMemRefType : public PyConcreteType<PyMemRefType, PyShapedType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsARankedTensor;
-  static constexpr const char *pyClassName = "MemRefType";
-  using PyConcreteType::PyConcreteType;
-
-  PyMemRefLayoutMapList getLayout();
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-         "get",
-         [](std::vector<int64_t> shape, PyType &elementType,
-            std::vector<PyAffineMap> layout, PyAttribute *memorySpace,
-            DefaultingPyLocation loc) {
-           SmallVector<MlirAffineMap> maps;
-           maps.reserve(layout.size());
-           for (PyAffineMap &map : layout)
-             maps.push_back(map);
-
-           MlirAttribute memSpaceAttr = {};
-           if (memorySpace)
-             memSpaceAttr = *memorySpace;
-
-           MlirType t = mlirMemRefTypeGetChecked(loc, elementType, shape.size(),
-                                                 shape.data(), maps.size(),
-                                                 maps.data(), memSpaceAttr);
-           // TODO: Rework error reporting once diagnostic engine is exposed
-           // in C API.
-           if (mlirTypeIsNull(t)) {
-             throw SetPyError(
-                 PyExc_ValueError,
-                 Twine("invalid '") +
-                     py::repr(py::cast(elementType)).cast<std::string>() +
-                     "' and expected floating point, integer, vector or "
-                     "complex "
-                     "type.");
-           }
-           return PyMemRefType(elementType.getContext(), t);
-         },
-         py::arg("shape"), py::arg("element_type"),
-         py::arg("layout") = py::list(), py::arg("memory_space") = py::none(),
-         py::arg("loc") = py::none(), "Create a memref type")
-        .def_property_readonly("layout", &PyMemRefType::getLayout,
-                               "The list of layout maps of the MemRef type.")
-        .def_property_readonly(
-            "memory_space",
-            [](PyMemRefType &self) -> PyAttribute {
-              MlirAttribute a = mlirMemRefTypeGetMemorySpace(self);
-              return PyAttribute(self.getContext(), a);
-            },
-            "Returns the memory space of the given MemRef type.");
-  }
-};
-
-/// A list of affine layout maps in a memref type. Internally, these are stored
-/// as consecutive elements, random access is cheap. Both the type and the maps
-/// are owned by the context, no need to worry about lifetime extension.
-class PyMemRefLayoutMapList
-    : public Sliceable<PyMemRefLayoutMapList, PyAffineMap> {
-public:
-  static constexpr const char *pyClassName = "MemRefLayoutMapList";
-
-  PyMemRefLayoutMapList(PyMemRefType type, intptr_t startIndex = 0,
-                        intptr_t length = -1, intptr_t step = 1)
-      : Sliceable(startIndex,
-                  length == -1 ? mlirMemRefTypeGetNumAffineMaps(type) : length,
-                  step),
-        memref(type) {}
-
-  intptr_t getNumElements() { return mlirMemRefTypeGetNumAffineMaps(memref); }
-
-  PyAffineMap getElement(intptr_t index) {
-    return PyAffineMap(memref.getContext(),
-                       mlirMemRefTypeGetAffineMap(memref, index));
-  }
-
-  PyMemRefLayoutMapList slice(intptr_t startIndex, intptr_t length,
-                              intptr_t step) {
-    return PyMemRefLayoutMapList(memref, startIndex, length, step);
-  }
-
-private:
-  PyMemRefType memref;
-};
-
-PyMemRefLayoutMapList PyMemRefType::getLayout() {
-  return PyMemRefLayoutMapList(*this);
-}
-
-/// Unranked MemRef Type subclass - UnrankedMemRefType.
-class PyUnrankedMemRefType
-    : public PyConcreteType<PyUnrankedMemRefType, PyShapedType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAUnrankedMemRef;
-  static constexpr const char *pyClassName = "UnrankedMemRefType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-         "get",
-         [](PyType &elementType, PyAttribute *memorySpace,
-            DefaultingPyLocation loc) {
-           MlirAttribute memSpaceAttr = {};
-           if (memorySpace)
-             memSpaceAttr = *memorySpace;
-
-           MlirType t =
-               mlirUnrankedMemRefTypeGetChecked(loc, elementType, memSpaceAttr);
-           // TODO: Rework error reporting once diagnostic engine is exposed
-           // in C API.
-           if (mlirTypeIsNull(t)) {
-             throw SetPyError(
-                 PyExc_ValueError,
-                 Twine("invalid '") +
-                     py::repr(py::cast(elementType)).cast<std::string>() +
-                     "' and expected floating point, integer, vector or "
-                     "complex "
-                     "type.");
-           }
-           return PyUnrankedMemRefType(elementType.getContext(), t);
-         },
-         py::arg("element_type"), py::arg("memory_space"),
-         py::arg("loc") = py::none(), "Create a unranked memref type")
-        .def_property_readonly(
-            "memory_space",
-            [](PyUnrankedMemRefType &self) -> PyAttribute {
-              MlirAttribute a = mlirMemRefTypeGetMemorySpace(self);
-              return PyAttribute(self.getContext(), a);
-            },
-            "Returns the memory space of the given Unranked MemRef type.");
-  }
-};
-
-/// Tuple Type subclass - TupleType.
-class PyTupleType : public PyConcreteType<PyTupleType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsATuple;
-  static constexpr const char *pyClassName = "TupleType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get_tuple",
-        [](py::list elementList, DefaultingPyMlirContext context) {
-          intptr_t num = py::len(elementList);
-          // Mapping py::list to SmallVector.
-          SmallVector<MlirType, 4> elements;
-          for (auto element : elementList)
-            elements.push_back(element.cast<PyType>());
-          MlirType t = mlirTupleTypeGet(context->get(), num, elements.data());
-          return PyTupleType(context->getRef(), t);
-        },
-        py::arg("elements"), py::arg("context") = py::none(),
-        "Create a tuple type");
-    c.def(
-        "get_type",
-        [](PyTupleType &self, intptr_t pos) -> PyType {
-          MlirType t = mlirTupleTypeGetType(self, pos);
-          return PyType(self.getContext(), t);
-        },
-        "Returns the pos-th type in the tuple type.");
-    c.def_property_readonly(
-        "num_types",
-        [](PyTupleType &self) -> intptr_t {
-          return mlirTupleTypeGetNumTypes(self);
-        },
-        "Returns the number of types contained in a tuple.");
-  }
-};
-
-/// Function type.
-class PyFunctionType : public PyConcreteType<PyFunctionType> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAFunction;
-  static constexpr const char *pyClassName = "FunctionType";
-  using PyConcreteType::PyConcreteType;
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static(
-        "get",
-        [](std::vector<PyType> inputs, std::vector<PyType> results,
-           DefaultingPyMlirContext context) {
-          SmallVector<MlirType, 4> inputsRaw(inputs.begin(), inputs.end());
-          SmallVector<MlirType, 4> resultsRaw(results.begin(), results.end());
-          MlirType t = mlirFunctionTypeGet(context->get(), inputsRaw.size(),
-                                           inputsRaw.data(), resultsRaw.size(),
-                                           resultsRaw.data());
-          return PyFunctionType(context->getRef(), t);
-        },
-        py::arg("inputs"), py::arg("results"), py::arg("context") = py::none(),
-        "Gets a FunctionType from a list of input and result types");
-    c.def_property_readonly(
-        "inputs",
-        [](PyFunctionType &self) {
-          MlirType t = self;
-          auto contextRef = self.getContext();
-          py::list types;
-          for (intptr_t i = 0, e = mlirFunctionTypeGetNumInputs(self); i < e;
-               ++i) {
-            types.append(PyType(contextRef, mlirFunctionTypeGetInput(t, i)));
-          }
-          return types;
-        },
-        "Returns the list of input types in the FunctionType.");
-    c.def_property_readonly(
-        "results",
-        [](PyFunctionType &self) {
-          auto contextRef = self.getContext();
-          py::list types;
-          for (intptr_t i = 0, e = mlirFunctionTypeGetNumResults(self); i < e;
-               ++i) {
-            types.append(
-                PyType(contextRef, mlirFunctionTypeGetResult(self, i)));
-          }
-          return types;
-        },
-        "Returns the list of result types in the FunctionType.");
-  }
-};
-
-} // namespace
-
-//------------------------------------------------------------------------------
-// PyAffineExpr and subclasses.
-//------------------------------------------------------------------------------
-
-namespace {
-/// CRTP base class for Python MLIR affine expressions that subclass AffineExpr
-/// and should be castable from it. Intermediate hierarchy classes can be
-/// modeled by specifying BaseTy.
-template <typename DerivedTy, typename BaseTy = PyAffineExpr>
-class PyConcreteAffineExpr : public BaseTy {
-public:
-  // Derived classes must define statics for:
-  //   IsAFunctionTy isaFunction
-  //   const char *pyClassName
-  // and redefine bindDerived.
-  using ClassTy = py::class_<DerivedTy, BaseTy>;
-  using IsAFunctionTy = bool (*)(MlirAffineExpr);
-
-  PyConcreteAffineExpr() = default;
-  PyConcreteAffineExpr(PyMlirContextRef contextRef, MlirAffineExpr affineExpr)
-      : BaseTy(std::move(contextRef), affineExpr) {}
-  PyConcreteAffineExpr(PyAffineExpr &orig)
-      : PyConcreteAffineExpr(orig.getContext(), castFrom(orig)) {}
-
-  static MlirAffineExpr castFrom(PyAffineExpr &orig) {
-    if (!DerivedTy::isaFunction(orig)) {
-      auto origRepr = py::repr(py::cast(orig)).cast<std::string>();
-      throw SetPyError(PyExc_ValueError,
-                       Twine("Cannot cast affine expression to ") +
-                           DerivedTy::pyClassName + " (from " + origRepr + ")");
-    }
-    return orig;
-  }
-
-  static void bind(py::module &m) {
-    auto cls = ClassTy(m, DerivedTy::pyClassName);
-    cls.def(py::init<PyAffineExpr &>());
-    DerivedTy::bindDerived(cls);
-  }
-
-  /// Implemented by derived classes to add methods to the Python subclass.
-  static void bindDerived(ClassTy &m) {}
-};
-
-class PyAffineConstantExpr : public PyConcreteAffineExpr<PyAffineConstantExpr> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAConstant;
-  static constexpr const char *pyClassName = "AffineConstantExpr";
-  using PyConcreteAffineExpr::PyConcreteAffineExpr;
-
-  static PyAffineConstantExpr get(intptr_t value,
-                                  DefaultingPyMlirContext context) {
-    MlirAffineExpr affineExpr =
-        mlirAffineConstantExprGet(context->get(), static_cast<int64_t>(value));
-    return PyAffineConstantExpr(context->getRef(), affineExpr);
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static("get", &PyAffineConstantExpr::get, py::arg("value"),
-                 py::arg("context") = py::none());
-    c.def_property_readonly("value", [](PyAffineConstantExpr &self) {
-      return mlirAffineConstantExprGetValue(self);
-    });
-  }
-};
-
-class PyAffineDimExpr : public PyConcreteAffineExpr<PyAffineDimExpr> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsADim;
-  static constexpr const char *pyClassName = "AffineDimExpr";
-  using PyConcreteAffineExpr::PyConcreteAffineExpr;
-
-  static PyAffineDimExpr get(intptr_t pos, DefaultingPyMlirContext context) {
-    MlirAffineExpr affineExpr = mlirAffineDimExprGet(context->get(), pos);
-    return PyAffineDimExpr(context->getRef(), affineExpr);
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static("get", &PyAffineDimExpr::get, py::arg("position"),
-                 py::arg("context") = py::none());
-    c.def_property_readonly("position", [](PyAffineDimExpr &self) {
-      return mlirAffineDimExprGetPosition(self);
-    });
-  }
-};
-
-class PyAffineSymbolExpr : public PyConcreteAffineExpr<PyAffineSymbolExpr> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsASymbol;
-  static constexpr const char *pyClassName = "AffineSymbolExpr";
-  using PyConcreteAffineExpr::PyConcreteAffineExpr;
-
-  static PyAffineSymbolExpr get(intptr_t pos, DefaultingPyMlirContext context) {
-    MlirAffineExpr affineExpr = mlirAffineSymbolExprGet(context->get(), pos);
-    return PyAffineSymbolExpr(context->getRef(), affineExpr);
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static("get", &PyAffineSymbolExpr::get, py::arg("position"),
-                 py::arg("context") = py::none());
-    c.def_property_readonly("position", [](PyAffineSymbolExpr &self) {
-      return mlirAffineSymbolExprGetPosition(self);
-    });
-  }
-};
-
-class PyAffineBinaryExpr : public PyConcreteAffineExpr<PyAffineBinaryExpr> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsABinary;
-  static constexpr const char *pyClassName = "AffineBinaryExpr";
-  using PyConcreteAffineExpr::PyConcreteAffineExpr;
-
-  PyAffineExpr lhs() {
-    MlirAffineExpr lhsExpr = mlirAffineBinaryOpExprGetLHS(get());
-    return PyAffineExpr(getContext(), lhsExpr);
-  }
-
-  PyAffineExpr rhs() {
-    MlirAffineExpr rhsExpr = mlirAffineBinaryOpExprGetRHS(get());
-    return PyAffineExpr(getContext(), rhsExpr);
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def_property_readonly("lhs", &PyAffineBinaryExpr::lhs);
-    c.def_property_readonly("rhs", &PyAffineBinaryExpr::rhs);
-  }
-};
-
-class PyAffineAddExpr
-    : public PyConcreteAffineExpr<PyAffineAddExpr, PyAffineBinaryExpr> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAAdd;
-  static constexpr const char *pyClassName = "AffineAddExpr";
-  using PyConcreteAffineExpr::PyConcreteAffineExpr;
-
-  static PyAffineAddExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
-    MlirAffineExpr expr = mlirAffineAddExprGet(lhs, rhs);
-    return PyAffineAddExpr(lhs.getContext(), expr);
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static("get", &PyAffineAddExpr::get);
-  }
-};
-
-class PyAffineMulExpr
-    : public PyConcreteAffineExpr<PyAffineMulExpr, PyAffineBinaryExpr> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAMul;
-  static constexpr const char *pyClassName = "AffineMulExpr";
-  using PyConcreteAffineExpr::PyConcreteAffineExpr;
-
-  static PyAffineMulExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
-    MlirAffineExpr expr = mlirAffineMulExprGet(lhs, rhs);
-    return PyAffineMulExpr(lhs.getContext(), expr);
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static("get", &PyAffineMulExpr::get);
-  }
-};
-
-class PyAffineModExpr
-    : public PyConcreteAffineExpr<PyAffineModExpr, PyAffineBinaryExpr> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAMod;
-  static constexpr const char *pyClassName = "AffineModExpr";
-  using PyConcreteAffineExpr::PyConcreteAffineExpr;
-
-  static PyAffineModExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
-    MlirAffineExpr expr = mlirAffineModExprGet(lhs, rhs);
-    return PyAffineModExpr(lhs.getContext(), expr);
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static("get", &PyAffineModExpr::get);
-  }
-};
-
-class PyAffineFloorDivExpr
-    : public PyConcreteAffineExpr<PyAffineFloorDivExpr, PyAffineBinaryExpr> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsAFloorDiv;
-  static constexpr const char *pyClassName = "AffineFloorDivExpr";
-  using PyConcreteAffineExpr::PyConcreteAffineExpr;
-
-  static PyAffineFloorDivExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
-    MlirAffineExpr expr = mlirAffineFloorDivExprGet(lhs, rhs);
-    return PyAffineFloorDivExpr(lhs.getContext(), expr);
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static("get", &PyAffineFloorDivExpr::get);
-  }
-};
-
-class PyAffineCeilDivExpr
-    : public PyConcreteAffineExpr<PyAffineCeilDivExpr, PyAffineBinaryExpr> {
-public:
-  static constexpr IsAFunctionTy isaFunction = mlirAffineExprIsACeilDiv;
-  static constexpr const char *pyClassName = "AffineCeilDivExpr";
-  using PyConcreteAffineExpr::PyConcreteAffineExpr;
-
-  static PyAffineCeilDivExpr get(PyAffineExpr lhs, PyAffineExpr rhs) {
-    MlirAffineExpr expr = mlirAffineCeilDivExprGet(lhs, rhs);
-    return PyAffineCeilDivExpr(lhs.getContext(), expr);
-  }
-
-  static void bindDerived(ClassTy &c) {
-    c.def_static("get", &PyAffineCeilDivExpr::get);
-  }
-};
-} // namespace
-
-bool PyAffineExpr::operator==(const PyAffineExpr &other) {
-  return mlirAffineExprEqual(affineExpr, other.affineExpr);
-}
-
-py::object PyAffineExpr::getCapsule() {
-  return py::reinterpret_steal<py::object>(
-      mlirPythonAffineExprToCapsule(*this));
-}
-
-PyAffineExpr PyAffineExpr::createFromCapsule(py::object capsule) {
-  MlirAffineExpr rawAffineExpr = mlirPythonCapsuleToAffineExpr(capsule.ptr());
-  if (mlirAffineExprIsNull(rawAffineExpr))
-    throw py::error_already_set();
-  return PyAffineExpr(
-      PyMlirContext::forContext(mlirAffineExprGetContext(rawAffineExpr)),
-      rawAffineExpr);
-}
-
-//------------------------------------------------------------------------------
-// PyAffineMap and utilities.
-//------------------------------------------------------------------------------
-
-namespace {
-/// A list of expressions contained in an affine map. Internally these are
-/// stored as a consecutive array leading to inexpensive random access. Both
-/// the map and the expression are owned by the context so we need not bother
-/// with lifetime extension.
-class PyAffineMapExprList
-    : public Sliceable<PyAffineMapExprList, PyAffineExpr> {
-public:
-  static constexpr const char *pyClassName = "AffineExprList";
-
-  PyAffineMapExprList(PyAffineMap map, intptr_t startIndex = 0,
-                      intptr_t length = -1, intptr_t step = 1)
-      : Sliceable(startIndex,
-                  length == -1 ? mlirAffineMapGetNumResults(map) : length,
-                  step),
-        affineMap(map) {}
-
-  intptr_t getNumElements() { return mlirAffineMapGetNumResults(affineMap); }
-
-  PyAffineExpr getElement(intptr_t pos) {
-    return PyAffineExpr(affineMap.getContext(),
-                        mlirAffineMapGetResult(affineMap, pos));
-  }
-
-  PyAffineMapExprList slice(intptr_t startIndex, intptr_t length,
-                            intptr_t step) {
-    return PyAffineMapExprList(affineMap, startIndex, length, step);
-  }
-
-private:
-  PyAffineMap affineMap;
-};
-} // end namespace
-
-bool PyAffineMap::operator==(const PyAffineMap &other) {
-  return mlirAffineMapEqual(affineMap, other.affineMap);
-}
-
-py::object PyAffineMap::getCapsule() {
-  return py::reinterpret_steal<py::object>(mlirPythonAffineMapToCapsule(*this));
-}
-
-PyAffineMap PyAffineMap::createFromCapsule(py::object capsule) {
-  MlirAffineMap rawAffineMap = mlirPythonCapsuleToAffineMap(capsule.ptr());
-  if (mlirAffineMapIsNull(rawAffineMap))
-    throw py::error_already_set();
-  return PyAffineMap(
-      PyMlirContext::forContext(mlirAffineMapGetContext(rawAffineMap)),
-      rawAffineMap);
-}
-
-//------------------------------------------------------------------------------
-// PyIntegerSet and utilities.
-//------------------------------------------------------------------------------
-
-class PyIntegerSetConstraint {
-public:
-  PyIntegerSetConstraint(PyIntegerSet set, intptr_t pos) : set(set), pos(pos) {}
-
-  PyAffineExpr getExpr() {
-    return PyAffineExpr(set.getContext(),
-                        mlirIntegerSetGetConstraint(set, pos));
-  }
-
-  bool isEq() { return mlirIntegerSetIsConstraintEq(set, pos); }
-
-  static void bind(py::module &m) {
-    py::class_<PyIntegerSetConstraint>(m, "IntegerSetConstraint")
-        .def_property_readonly("expr", &PyIntegerSetConstraint::getExpr)
-        .def_property_readonly("is_eq", &PyIntegerSetConstraint::isEq);
-  }
-
-private:
-  PyIntegerSet set;
-  intptr_t pos;
-};
-
-class PyIntegerSetConstraintList
-    : public Sliceable<PyIntegerSetConstraintList, PyIntegerSetConstraint> {
-public:
-  static constexpr const char *pyClassName = "IntegerSetConstraintList";
-
-  PyIntegerSetConstraintList(PyIntegerSet set, intptr_t startIndex = 0,
-                             intptr_t length = -1, intptr_t step = 1)
-      : Sliceable(startIndex,
-                  length == -1 ? mlirIntegerSetGetNumConstraints(set) : length,
-                  step),
-        set(set) {}
-
-  intptr_t getNumElements() { return mlirIntegerSetGetNumConstraints(set); }
-
-  PyIntegerSetConstraint getElement(intptr_t pos) {
-    return PyIntegerSetConstraint(set, pos);
-  }
-
-  PyIntegerSetConstraintList slice(intptr_t startIndex, intptr_t length,
-                                   intptr_t step) {
-    return PyIntegerSetConstraintList(set, startIndex, length, step);
-  }
-
-private:
-  PyIntegerSet set;
-};
-
-bool PyIntegerSet::operator==(const PyIntegerSet &other) {
-  return mlirIntegerSetEqual(integerSet, other.integerSet);
-}
-
-py::object PyIntegerSet::getCapsule() {
-  return py::reinterpret_steal<py::object>(
-      mlirPythonIntegerSetToCapsule(*this));
-}
-
-PyIntegerSet PyIntegerSet::createFromCapsule(py::object capsule) {
-  MlirIntegerSet rawIntegerSet = mlirPythonCapsuleToIntegerSet(capsule.ptr());
-  if (mlirIntegerSetIsNull(rawIntegerSet))
-    throw py::error_already_set();
-  return PyIntegerSet(
-      PyMlirContext::forContext(mlirIntegerSetGetContext(rawIntegerSet)),
-      rawIntegerSet);
-}
-
-/// Attempts to populate `result` with the content of `list` casted to the
-/// appropriate type (Python and C types are provided as template arguments).
-/// Throws errors in case of failure, using "action" to describe what the caller
-/// was attempting to do.
-template <typename PyType, typename CType>
-static void pyListToVector(py::list list, llvm::SmallVectorImpl<CType> &result,
-                           StringRef action) {
-  result.reserve(py::len(list));
-  for (py::handle item : list) {
-    try {
-      result.push_back(item.cast<PyType>());
-    } catch (py::cast_error &err) {
-      std::string msg = (llvm::Twine("Invalid expression when ") + action +
-                         " (" + err.what() + ")")
-                            .str();
-      throw py::cast_error(msg);
-    } catch (py::reference_cast_error &err) {
-      std::string msg = (llvm::Twine("Invalid expression (None?) when ") +
-                         action + " (" + err.what() + ")")
-                            .str();
-      throw py::cast_error(msg);
-    }
-  }
-}
-
-//------------------------------------------------------------------------------
-// Populates the pybind11 IR submodule.
-//------------------------------------------------------------------------------
-
-void mlir::python::populateIRSubmodule(py::module &m) {
-  //----------------------------------------------------------------------------
-  // Mapping of MlirContext
-  //----------------------------------------------------------------------------
-  py::class_<PyMlirContext>(m, "Context")
-      .def(py::init<>(&PyMlirContext::createNewContextForInit))
-      .def_static("_get_live_count", &PyMlirContext::getLiveCount)
-      .def("_get_context_again",
-           [](PyMlirContext &self) {
-             PyMlirContextRef ref = PyMlirContext::forContext(self.get());
-             return ref.releaseObject();
-           })
-      .def("_get_live_operation_count", &PyMlirContext::getLiveOperationCount)
-      .def("_get_live_module_count", &PyMlirContext::getLiveModuleCount)
-      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR,
-                             &PyMlirContext::getCapsule)
-      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyMlirContext::createFromCapsule)
-      .def("__enter__", &PyMlirContext::contextEnter)
-      .def("__exit__", &PyMlirContext::contextExit)
-      .def_property_readonly_static(
-          "current",
-          [](py::object & /*class*/) {
-            auto *context = PyThreadContextEntry::getDefaultContext();
-            if (!context)
-              throw SetPyError(PyExc_ValueError, "No current Context");
-            return context;
-          },
-          "Gets the Context bound to the current thread or raises ValueError")
-      .def_property_readonly(
-          "dialects",
-          [](PyMlirContext &self) { return PyDialects(self.getRef()); },
-          "Gets a container for accessing dialects by name")
-      .def_property_readonly(
-          "d", [](PyMlirContext &self) { return PyDialects(self.getRef()); },
-          "Alias for 'dialect'")
-      .def(
-          "get_dialect_descriptor",
-          [=](PyMlirContext &self, std::string &name) {
-            MlirDialect dialect = mlirContextGetOrLoadDialect(
-                self.get(), {name.data(), name.size()});
-            if (mlirDialectIsNull(dialect)) {
-              throw SetPyError(PyExc_ValueError,
-                               Twine("Dialect '") + name + "' not found");
-            }
-            return PyDialectDescriptor(self.getRef(), dialect);
-          },
-          "Gets or loads a dialect by name, returning its descriptor object")
-      .def_property(
-          "allow_unregistered_dialects",
-          [](PyMlirContext &self) -> bool {
-            return mlirContextGetAllowUnregisteredDialects(self.get());
-          },
-          [](PyMlirContext &self, bool value) {
-            mlirContextSetAllowUnregisteredDialects(self.get(), value);
-          });
-
-  //----------------------------------------------------------------------------
-  // Mapping of PyDialectDescriptor
-  //----------------------------------------------------------------------------
-  py::class_<PyDialectDescriptor>(m, "DialectDescriptor")
-      .def_property_readonly("namespace",
-                             [](PyDialectDescriptor &self) {
-                               MlirStringRef ns =
-                                   mlirDialectGetNamespace(self.get());
-                               return py::str(ns.data, ns.length);
-                             })
-      .def("__repr__", [](PyDialectDescriptor &self) {
-        MlirStringRef ns = mlirDialectGetNamespace(self.get());
-        std::string repr("<DialectDescriptor ");
-        repr.append(ns.data, ns.length);
-        repr.append(">");
-        return repr;
-      });
-
-  //----------------------------------------------------------------------------
-  // Mapping of PyDialects
-  //----------------------------------------------------------------------------
-  py::class_<PyDialects>(m, "Dialects")
-      .def("__getitem__",
-           [=](PyDialects &self, std::string keyName) {
-             MlirDialect dialect =
-                 self.getDialectForKey(keyName, /*attrError=*/false);
-             py::object descriptor =
-                 py::cast(PyDialectDescriptor{self.getContext(), dialect});
-             return createCustomDialectWrapper(keyName, std::move(descriptor));
-           })
-      .def("__getattr__", [=](PyDialects &self, std::string attrName) {
-        MlirDialect dialect =
-            self.getDialectForKey(attrName, /*attrError=*/true);
-        py::object descriptor =
-            py::cast(PyDialectDescriptor{self.getContext(), dialect});
-        return createCustomDialectWrapper(attrName, std::move(descriptor));
-      });
-
-  //----------------------------------------------------------------------------
-  // Mapping of PyDialect
-  //----------------------------------------------------------------------------
-  py::class_<PyDialect>(m, "Dialect")
-      .def(py::init<py::object>(), "descriptor")
-      .def_property_readonly(
-          "descriptor", [](PyDialect &self) { return self.getDescriptor(); })
-      .def("__repr__", [](py::object self) {
-        auto clazz = self.attr("__class__");
-        return py::str("<Dialect ") +
-               self.attr("descriptor").attr("namespace") + py::str(" (class ") +
-               clazz.attr("__module__") + py::str(".") +
-               clazz.attr("__name__") + py::str(")>");
-      });
-
-  //----------------------------------------------------------------------------
-  // Mapping of Location
-  //----------------------------------------------------------------------------
-  py::class_<PyLocation>(m, "Location")
-      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR, &PyLocation::getCapsule)
-      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyLocation::createFromCapsule)
-      .def("__enter__", &PyLocation::contextEnter)
-      .def("__exit__", &PyLocation::contextExit)
-      .def("__eq__",
-           [](PyLocation &self, PyLocation &other) -> bool {
-             return mlirLocationEqual(self, other);
-           })
-      .def("__eq__", [](PyLocation &self, py::object other) { return false; })
-      .def_property_readonly_static(
-          "current",
-          [](py::object & /*class*/) {
-            auto *loc = PyThreadContextEntry::getDefaultLocation();
-            if (!loc)
-              throw SetPyError(PyExc_ValueError, "No current Location");
-            return loc;
-          },
-          "Gets the Location bound to the current thread or raises ValueError")
-      .def_static(
-          "unknown",
-          [](DefaultingPyMlirContext context) {
-            return PyLocation(context->getRef(),
-                              mlirLocationUnknownGet(context->get()));
-          },
-          py::arg("context") = py::none(),
-          "Gets a Location representing an unknown location")
-      .def_static(
-          "file",
-          [](std::string filename, int line, int col,
-             DefaultingPyMlirContext context) {
-            return PyLocation(
-                context->getRef(),
-                mlirLocationFileLineColGet(
-                    context->get(), toMlirStringRef(filename), line, col));
-          },
-          py::arg("filename"), py::arg("line"), py::arg("col"),
-          py::arg("context") = py::none(), kContextGetFileLocationDocstring)
-      .def_property_readonly(
-          "context",
-          [](PyLocation &self) { return self.getContext().getObject(); },
-          "Context that owns the Location")
-      .def("__repr__", [](PyLocation &self) {
-        PyPrintAccumulator printAccum;
-        mlirLocationPrint(self, printAccum.getCallback(),
-                          printAccum.getUserData());
-        return printAccum.join();
-      });
+  //----------------------------------------------------------------------------
+  // Mapping of Location
+  //----------------------------------------------------------------------------
+  py::class_<PyLocation>(m, "Location")
+      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR, &PyLocation::getCapsule)
+      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyLocation::createFromCapsule)
+      .def("__enter__", &PyLocation::contextEnter)
+      .def("__exit__", &PyLocation::contextExit)
+      .def("__eq__",
+           [](PyLocation &self, PyLocation &other) -> bool {
+             return mlirLocationEqual(self, other);
+           })
+      .def("__eq__", [](PyLocation &self, py::object other) { return false; })
+      .def_property_readonly_static(
+          "current",
+          [](py::object & /*class*/) {
+            auto *loc = PyThreadContextEntry::getDefaultLocation();
+            if (!loc)
+              throw SetPyError(PyExc_ValueError, "No current Location");
+            return loc;
+          },
+          "Gets the Location bound to the current thread or raises ValueError")
+      .def_static(
+          "unknown",
+          [](DefaultingPyMlirContext context) {
+            return PyLocation(context->getRef(),
+                              mlirLocationUnknownGet(context->get()));
+          },
+          py::arg("context") = py::none(),
+          "Gets a Location representing an unknown location")
+      .def_static(
+          "file",
+          [](std::string filename, int line, int col,
+             DefaultingPyMlirContext context) {
+            return PyLocation(
+                context->getRef(),
+                mlirLocationFileLineColGet(
+                    context->get(), toMlirStringRef(filename), line, col));
+          },
+          py::arg("filename"), py::arg("line"), py::arg("col"),
+          py::arg("context") = py::none(), kContextGetFileLocationDocstring)
+      .def_property_readonly(
+          "context",
+          [](PyLocation &self) { return self.getContext().getObject(); },
+          "Context that owns the Location")
+      .def("__repr__", [](PyLocation &self) {
+        PyPrintAccumulator printAccum;
+        mlirLocationPrint(self, printAccum.getCallback(),
+                          printAccum.getUserData());
+        return printAccum.join();
+      });
 
   //----------------------------------------------------------------------------
   // Mapping of Module
@@ -4019,22 +2259,6 @@ void mlir::python::populateIRSubmodule(py::module &m) {
           py::keep_alive<0, 1>(),
           "The underlying generic attribute of the NamedAttribute binding");
 
-  // Builtin attribute bindings.
-  PyAffineMapAttribute::bind(m);
-  PyArrayAttribute::bind(m);
-  PyArrayAttribute::PyArrayAttributeIterator::bind(m);
-  PyBoolAttribute::bind(m);
-  PyDenseElementsAttribute::bind(m);
-  PyDenseFPElementsAttribute::bind(m);
-  PyDenseIntElementsAttribute::bind(m);
-  PyDictAttribute::bind(m);
-  PyFlatSymbolRefAttribute::bind(m);
-  PyFloatAttribute::bind(m);
-  PyIntegerAttribute::bind(m);
-  PyStringAttribute::bind(m);
-  PyTypeAttribute::bind(m);
-  PyUnitAttribute::bind(m);
-
   //----------------------------------------------------------------------------
   // Mapping of PyType.
   //----------------------------------------------------------------------------
@@ -4085,25 +2309,6 @@ void mlir::python::populateIRSubmodule(py::module &m) {
         return printAccum.join();
       });
 
-  // Builtin type bindings.
-  PyIntegerType::bind(m);
-  PyIndexType::bind(m);
-  PyBF16Type::bind(m);
-  PyF16Type::bind(m);
-  PyF32Type::bind(m);
-  PyF64Type::bind(m);
-  PyNoneType::bind(m);
-  PyComplexType::bind(m);
-  PyShapedType::bind(m);
-  PyVectorType::bind(m);
-  PyRankedTensorType::bind(m);
-  PyUnrankedTensorType::bind(m);
-  PyMemRefType::bind(m);
-  PyMemRefLayoutMapList::bind(m);
-  PyUnrankedMemRefType::bind(m);
-  PyTupleType::bind(m);
-  PyFunctionType::bind(m);
-
   //----------------------------------------------------------------------------
   // Mapping of Value.
   //----------------------------------------------------------------------------
@@ -4149,359 +2354,4 @@ void mlir::python::populateIRSubmodule(py::module &m) {
   PyOpResultList::bind(m);
   PyRegionIterator::bind(m);
   PyRegionList::bind(m);
-
-  //----------------------------------------------------------------------------
-  // Mapping of PyAffineExpr and derived classes.
-  //----------------------------------------------------------------------------
-  py::class_<PyAffineExpr>(m, "AffineExpr")
-      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR,
-                             &PyAffineExpr::getCapsule)
-      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyAffineExpr::createFromCapsule)
-      .def("__add__",
-           [](PyAffineExpr &self, PyAffineExpr &other) {
-             return PyAffineAddExpr::get(self, other);
-           })
-      .def("__mul__",
-           [](PyAffineExpr &self, PyAffineExpr &other) {
-             return PyAffineMulExpr::get(self, other);
-           })
-      .def("__mod__",
-           [](PyAffineExpr &self, PyAffineExpr &other) {
-             return PyAffineModExpr::get(self, other);
-           })
-      .def("__sub__",
-           [](PyAffineExpr &self, PyAffineExpr &other) {
-             auto negOne =
-                 PyAffineConstantExpr::get(-1, *self.getContext().get());
-             return PyAffineAddExpr::get(self,
-                                         PyAffineMulExpr::get(negOne, other));
-           })
-      .def("__eq__", [](PyAffineExpr &self,
-                        PyAffineExpr &other) { return self == other; })
-      .def("__eq__",
-           [](PyAffineExpr &self, py::object &other) { return false; })
-      .def("__str__",
-           [](PyAffineExpr &self) {
-             PyPrintAccumulator printAccum;
-             mlirAffineExprPrint(self, printAccum.getCallback(),
-                                 printAccum.getUserData());
-             return printAccum.join();
-           })
-      .def("__repr__",
-           [](PyAffineExpr &self) {
-             PyPrintAccumulator printAccum;
-             printAccum.parts.append("AffineExpr(");
-             mlirAffineExprPrint(self, printAccum.getCallback(),
-                                 printAccum.getUserData());
-             printAccum.parts.append(")");
-             return printAccum.join();
-           })
-      .def_property_readonly(
-          "context",
-          [](PyAffineExpr &self) { return self.getContext().getObject(); })
-      .def_static(
-          "get_add", &PyAffineAddExpr::get,
-          "Gets an affine expression containing a sum of two expressions.")
-      .def_static(
-          "get_mul", &PyAffineMulExpr::get,
-          "Gets an affine expression containing a product of two expressions.")
-      .def_static("get_mod", &PyAffineModExpr::get,
-                  "Gets an affine expression containing the modulo of dividing "
-                  "one expression by another.")
-      .def_static("get_floor_div", &PyAffineFloorDivExpr::get,
-                  "Gets an affine expression containing the rounded-down "
-                  "result of dividing one expression by another.")
-      .def_static("get_ceil_div", &PyAffineCeilDivExpr::get,
-                  "Gets an affine expression containing the rounded-up result "
-                  "of dividing one expression by another.")
-      .def_static("get_constant", &PyAffineConstantExpr::get, py::arg("value"),
-                  py::arg("context") = py::none(),
-                  "Gets a constant affine expression with the given value.")
-      .def_static(
-          "get_dim", &PyAffineDimExpr::get, py::arg("position"),
-          py::arg("context") = py::none(),
-          "Gets an affine expression of a dimension at the given position.")
-      .def_static(
-          "get_symbol", &PyAffineSymbolExpr::get, py::arg("position"),
-          py::arg("context") = py::none(),
-          "Gets an affine expression of a symbol at the given position.")
-      .def(
-          "dump", [](PyAffineExpr &self) { mlirAffineExprDump(self); },
-          kDumpDocstring);
-  PyAffineConstantExpr::bind(m);
-  PyAffineDimExpr::bind(m);
-  PyAffineSymbolExpr::bind(m);
-  PyAffineBinaryExpr::bind(m);
-  PyAffineAddExpr::bind(m);
-  PyAffineMulExpr::bind(m);
-  PyAffineModExpr::bind(m);
-  PyAffineFloorDivExpr::bind(m);
-  PyAffineCeilDivExpr::bind(m);
-
-  //----------------------------------------------------------------------------
-  // Mapping of PyAffineMap.
-  //----------------------------------------------------------------------------
-  py::class_<PyAffineMap>(m, "AffineMap")
-      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR,
-                             &PyAffineMap::getCapsule)
-      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyAffineMap::createFromCapsule)
-      .def("__eq__",
-           [](PyAffineMap &self, PyAffineMap &other) { return self == other; })
-      .def("__eq__", [](PyAffineMap &self, py::object &other) { return false; })
-      .def("__str__",
-           [](PyAffineMap &self) {
-             PyPrintAccumulator printAccum;
-             mlirAffineMapPrint(self, printAccum.getCallback(),
-                                printAccum.getUserData());
-             return printAccum.join();
-           })
-      .def("__repr__",
-           [](PyAffineMap &self) {
-             PyPrintAccumulator printAccum;
-             printAccum.parts.append("AffineMap(");
-             mlirAffineMapPrint(self, printAccum.getCallback(),
-                                printAccum.getUserData());
-             printAccum.parts.append(")");
-             return printAccum.join();
-           })
-      .def_property_readonly(
-          "context",
-          [](PyAffineMap &self) { return self.getContext().getObject(); },
-          "Context that owns the Affine Map")
-      .def(
-          "dump", [](PyAffineMap &self) { mlirAffineMapDump(self); },
-          kDumpDocstring)
-      .def_static(
-          "get",
-          [](intptr_t dimCount, intptr_t symbolCount, py::list exprs,
-             DefaultingPyMlirContext context) {
-            SmallVector<MlirAffineExpr> affineExprs;
-            pyListToVector<PyAffineExpr, MlirAffineExpr>(
-                exprs, affineExprs, "attempting to create an AffineMap");
-            MlirAffineMap map =
-                mlirAffineMapGet(context->get(), dimCount, symbolCount,
-                                 affineExprs.size(), affineExprs.data());
-            return PyAffineMap(context->getRef(), map);
-          },
-          py::arg("dim_count"), py::arg("symbol_count"), py::arg("exprs"),
-          py::arg("context") = py::none(),
-          "Gets a map with the given expressions as results.")
-      .def_static(
-          "get_constant",
-          [](intptr_t value, DefaultingPyMlirContext context) {
-            MlirAffineMap affineMap =
-                mlirAffineMapConstantGet(context->get(), value);
-            return PyAffineMap(context->getRef(), affineMap);
-          },
-          py::arg("value"), py::arg("context") = py::none(),
-          "Gets an affine map with a single constant result")
-      .def_static(
-          "get_empty",
-          [](DefaultingPyMlirContext context) {
-            MlirAffineMap affineMap = mlirAffineMapEmptyGet(context->get());
-            return PyAffineMap(context->getRef(), affineMap);
-          },
-          py::arg("context") = py::none(), "Gets an empty affine map.")
-      .def_static(
-          "get_identity",
-          [](intptr_t nDims, DefaultingPyMlirContext context) {
-            MlirAffineMap affineMap =
-                mlirAffineMapMultiDimIdentityGet(context->get(), nDims);
-            return PyAffineMap(context->getRef(), affineMap);
-          },
-          py::arg("n_dims"), py::arg("context") = py::none(),
-          "Gets an identity map with the given number of dimensions.")
-      .def_static(
-          "get_minor_identity",
-          [](intptr_t nDims, intptr_t nResults,
-             DefaultingPyMlirContext context) {
-            MlirAffineMap affineMap =
-                mlirAffineMapMinorIdentityGet(context->get(), nDims, nResults);
-            return PyAffineMap(context->getRef(), affineMap);
-          },
-          py::arg("n_dims"), py::arg("n_results"),
-          py::arg("context") = py::none(),
-          "Gets a minor identity map with the given number of dimensions and "
-          "results.")
-      .def_static(
-          "get_permutation",
-          [](std::vector<unsigned> permutation,
-             DefaultingPyMlirContext context) {
-            if (!isPermutation(permutation))
-              throw py::cast_error("Invalid permutation when attempting to "
-                                   "create an AffineMap");
-            MlirAffineMap affineMap = mlirAffineMapPermutationGet(
-                context->get(), permutation.size(), permutation.data());
-            return PyAffineMap(context->getRef(), affineMap);
-          },
-          py::arg("permutation"), py::arg("context") = py::none(),
-          "Gets an affine map that permutes its inputs.")
-      .def("get_submap",
-           [](PyAffineMap &self, std::vector<intptr_t> &resultPos) {
-             intptr_t numResults = mlirAffineMapGetNumResults(self);
-             for (intptr_t pos : resultPos) {
-               if (pos < 0 || pos >= numResults)
-                 throw py::value_error("result position out of bounds");
-             }
-             MlirAffineMap affineMap = mlirAffineMapGetSubMap(
-                 self, resultPos.size(), resultPos.data());
-             return PyAffineMap(self.getContext(), affineMap);
-           })
-      .def("get_major_submap",
-           [](PyAffineMap &self, intptr_t nResults) {
-             if (nResults >= mlirAffineMapGetNumResults(self))
-               throw py::value_error("number of results out of bounds");
-             MlirAffineMap affineMap =
-                 mlirAffineMapGetMajorSubMap(self, nResults);
-             return PyAffineMap(self.getContext(), affineMap);
-           })
-      .def("get_minor_submap",
-           [](PyAffineMap &self, intptr_t nResults) {
-             if (nResults >= mlirAffineMapGetNumResults(self))
-               throw py::value_error("number of results out of bounds");
-             MlirAffineMap affineMap =
-                 mlirAffineMapGetMinorSubMap(self, nResults);
-             return PyAffineMap(self.getContext(), affineMap);
-           })
-      .def_property_readonly(
-          "is_permutation",
-          [](PyAffineMap &self) { return mlirAffineMapIsPermutation(self); })
-      .def_property_readonly("is_projected_permutation",
-                             [](PyAffineMap &self) {
-                               return mlirAffineMapIsProjectedPermutation(self);
-                             })
-      .def_property_readonly(
-          "n_dims",
-          [](PyAffineMap &self) { return mlirAffineMapGetNumDims(self); })
-      .def_property_readonly(
-          "n_inputs",
-          [](PyAffineMap &self) { return mlirAffineMapGetNumInputs(self); })
-      .def_property_readonly(
-          "n_symbols",
-          [](PyAffineMap &self) { return mlirAffineMapGetNumSymbols(self); })
-      .def_property_readonly("results", [](PyAffineMap &self) {
-        return PyAffineMapExprList(self);
-      });
-  PyAffineMapExprList::bind(m);
-
-  //----------------------------------------------------------------------------
-  // Mapping of PyIntegerSet.
-  //----------------------------------------------------------------------------
-  py::class_<PyIntegerSet>(m, "IntegerSet")
-      .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR,
-                             &PyIntegerSet::getCapsule)
-      .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyIntegerSet::createFromCapsule)
-      .def("__eq__", [](PyIntegerSet &self,
-                        PyIntegerSet &other) { return self == other; })
-      .def("__eq__", [](PyIntegerSet &self, py::object other) { return false; })
-      .def("__str__",
-           [](PyIntegerSet &self) {
-             PyPrintAccumulator printAccum;
-             mlirIntegerSetPrint(self, printAccum.getCallback(),
-                                 printAccum.getUserData());
-             return printAccum.join();
-           })
-      .def("__repr__",
-           [](PyIntegerSet &self) {
-             PyPrintAccumulator printAccum;
-             printAccum.parts.append("IntegerSet(");
-             mlirIntegerSetPrint(self, printAccum.getCallback(),
-                                 printAccum.getUserData());
-             printAccum.parts.append(")");
-             return printAccum.join();
-           })
-      .def_property_readonly(
-          "context",
-          [](PyIntegerSet &self) { return self.getContext().getObject(); })
-      .def(
-          "dump", [](PyIntegerSet &self) { mlirIntegerSetDump(self); },
-          kDumpDocstring)
-      .def_static(
-          "get",
-          [](intptr_t numDims, intptr_t numSymbols, py::list exprs,
-             std::vector<bool> eqFlags, DefaultingPyMlirContext context) {
-            if (exprs.size() != eqFlags.size())
-              throw py::value_error(
-                  "Expected the number of constraints to match "
-                  "that of equality flags");
-            if (exprs.empty())
-              throw py::value_error("Expected non-empty list of constraints");
-
-            // Copy over to a SmallVector because std::vector has a
-            // specialization for booleans that packs data and does not
-            // expose a `bool *`.
-            SmallVector<bool, 8> flags(eqFlags.begin(), eqFlags.end());
-
-            SmallVector<MlirAffineExpr> affineExprs;
-            pyListToVector<PyAffineExpr>(exprs, affineExprs,
-                                         "attempting to create an IntegerSet");
-            MlirIntegerSet set = mlirIntegerSetGet(
-                context->get(), numDims, numSymbols, exprs.size(),
-                affineExprs.data(), flags.data());
-            return PyIntegerSet(context->getRef(), set);
-          },
-          py::arg("num_dims"), py::arg("num_symbols"), py::arg("exprs"),
-          py::arg("eq_flags"), py::arg("context") = py::none())
-      .def_static(
-          "get_empty",
-          [](intptr_t numDims, intptr_t numSymbols,
-             DefaultingPyMlirContext context) {
-            MlirIntegerSet set =
-                mlirIntegerSetEmptyGet(context->get(), numDims, numSymbols);
-            return PyIntegerSet(context->getRef(), set);
-          },
-          py::arg("num_dims"), py::arg("num_symbols"),
-          py::arg("context") = py::none())
-      .def("get_replaced",
-           [](PyIntegerSet &self, py::list dimExprs, py::list symbolExprs,
-              intptr_t numResultDims, intptr_t numResultSymbols) {
-             if (static_cast<intptr_t>(dimExprs.size()) !=
-                 mlirIntegerSetGetNumDims(self))
-               throw py::value_error(
-                   "Expected the number of dimension replacement expressions "
-                   "to match that of dimensions");
-             if (static_cast<intptr_t>(symbolExprs.size()) !=
-                 mlirIntegerSetGetNumSymbols(self))
-               throw py::value_error(
-                   "Expected the number of symbol replacement expressions "
-                   "to match that of symbols");
-
-             SmallVector<MlirAffineExpr> dimAffineExprs, symbolAffineExprs;
-             pyListToVector<PyAffineExpr>(
-                 dimExprs, dimAffineExprs,
-                 "attempting to create an IntegerSet by replacing dimensions");
-             pyListToVector<PyAffineExpr>(
-                 symbolExprs, symbolAffineExprs,
-                 "attempting to create an IntegerSet by replacing symbols");
-             MlirIntegerSet set = mlirIntegerSetReplaceGet(
-                 self, dimAffineExprs.data(), symbolAffineExprs.data(),
-                 numResultDims, numResultSymbols);
-             return PyIntegerSet(self.getContext(), set);
-           })
-      .def_property_readonly("is_canonical_empty",
-                             [](PyIntegerSet &self) {
-                               return mlirIntegerSetIsCanonicalEmpty(self);
-                             })
-      .def_property_readonly(
-          "n_dims",
-          [](PyIntegerSet &self) { return mlirIntegerSetGetNumDims(self); })
-      .def_property_readonly(
-          "n_symbols",
-          [](PyIntegerSet &self) { return mlirIntegerSetGetNumSymbols(self); })
-      .def_property_readonly(
-          "n_inputs",
-          [](PyIntegerSet &self) { return mlirIntegerSetGetNumInputs(self); })
-      .def_property_readonly("n_equalities",
-                             [](PyIntegerSet &self) {
-                               return mlirIntegerSetGetNumEqualities(self);
-                             })
-      .def_property_readonly("n_inequalities",
-                             [](PyIntegerSet &self) {
-                               return mlirIntegerSetGetNumInequalities(self);
-                             })
-      .def_property_readonly("constraints", [](PyIntegerSet &self) {
-        return PyIntegerSetConstraintList(self);
-      });
-  PyIntegerSetConstraint::bind(m);
-  PyIntegerSetConstraintList::bind(m);
 }
diff --git a/mlir/lib/Bindings/Python/IRModules.h b/mlir/lib/Bindings/Python/IRModule.h
similarity index 99%
rename from mlir/lib/Bindings/Python/IRModules.h
rename to mlir/lib/Bindings/Python/IRModule.h
index 8140d704300da683d905b515c6f00b1eec3ab96b..5c710abe789a16a521576c6933942e22e9bed851 100644
--- a/mlir/lib/Bindings/Python/IRModules.h
+++ b/mlir/lib/Bindings/Python/IRModule.h
@@ -747,7 +747,10 @@ private:
   MlirIntegerSet integerSet;
 };
 
-void populateIRSubmodule(pybind11::module &m);
+void populateIRAffine(pybind11::module &m);
+void populateIRAttributes(pybind11::module &m);
+void populateIRCore(pybind11::module &m);
+void populateIRTypes(pybind11::module &m);
 
 } // namespace python
 } // namespace mlir
diff --git a/mlir/lib/Bindings/Python/IRTypes.cpp b/mlir/lib/Bindings/Python/IRTypes.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..96f6bf6666c958b45c4fce2cb71b40298092de59
--- /dev/null
+++ b/mlir/lib/Bindings/Python/IRTypes.cpp
@@ -0,0 +1,678 @@
+//===- IRTypes.cpp - Exports builtin and standard types -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IRModule.h"
+
+#include "PybindUtils.h"
+
+#include "mlir-c/BuiltinTypes.h"
+
+namespace py = pybind11;
+using namespace mlir;
+using namespace mlir::python;
+
+using llvm::SmallVector;
+using llvm::Twine;
+
+namespace {
+
+/// Checks whether the given type is an integer or float type.
+static int mlirTypeIsAIntegerOrFloat(MlirType type) {
+  return mlirTypeIsAInteger(type) || mlirTypeIsABF16(type) ||
+         mlirTypeIsAF16(type) || mlirTypeIsAF32(type) || mlirTypeIsAF64(type);
+}
+
+/// CRTP base classes for Python types that subclass Type and should be
+/// castable from it (i.e. via something like IntegerType(t)).
+/// By default, type class hierarchies are one level deep (i.e. a
+/// concrete type class extends PyType); however, intermediate python-visible
+/// base classes can be modeled by specifying a BaseTy.
+template <typename DerivedTy, typename BaseTy = PyType>
+class PyConcreteType : public BaseTy {
+public:
+  // Derived classes must define statics for:
+  //   IsAFunctionTy isaFunction
+  //   const char *pyClassName
+  using ClassTy = py::class_<DerivedTy, BaseTy>;
+  using IsAFunctionTy = bool (*)(MlirType);
+
+  PyConcreteType() = default;
+  PyConcreteType(PyMlirContextRef contextRef, MlirType t)
+      : BaseTy(std::move(contextRef), t) {}
+  PyConcreteType(PyType &orig)
+      : PyConcreteType(orig.getContext(), castFrom(orig)) {}
+
+  static MlirType castFrom(PyType &orig) {
+    if (!DerivedTy::isaFunction(orig)) {
+      auto origRepr = py::repr(py::cast(orig)).cast<std::string>();
+      throw SetPyError(PyExc_ValueError, Twine("Cannot cast type to ") +
+                                             DerivedTy::pyClassName +
+                                             " (from " + origRepr + ")");
+    }
+    return orig;
+  }
+
+  static void bind(py::module &m) {
+    auto cls = ClassTy(m, DerivedTy::pyClassName);
+    cls.def(py::init<PyType &>(), py::keep_alive<0, 1>());
+    cls.def_static("isinstance", [](PyType &otherType) -> bool {
+      return DerivedTy::isaFunction(otherType);
+    });
+    DerivedTy::bindDerived(cls);
+  }
+
+  /// Implemented by derived classes to add methods to the Python subclass.
+  static void bindDerived(ClassTy &m) {}
+};
+
+class PyIntegerType : public PyConcreteType<PyIntegerType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAInteger;
+  static constexpr const char *pyClassName = "IntegerType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get_signless",
+        [](unsigned width, DefaultingPyMlirContext context) {
+          MlirType t = mlirIntegerTypeGet(context->get(), width);
+          return PyIntegerType(context->getRef(), t);
+        },
+        py::arg("width"), py::arg("context") = py::none(),
+        "Create a signless integer type");
+    c.def_static(
+        "get_signed",
+        [](unsigned width, DefaultingPyMlirContext context) {
+          MlirType t = mlirIntegerTypeSignedGet(context->get(), width);
+          return PyIntegerType(context->getRef(), t);
+        },
+        py::arg("width"), py::arg("context") = py::none(),
+        "Create a signed integer type");
+    c.def_static(
+        "get_unsigned",
+        [](unsigned width, DefaultingPyMlirContext context) {
+          MlirType t = mlirIntegerTypeUnsignedGet(context->get(), width);
+          return PyIntegerType(context->getRef(), t);
+        },
+        py::arg("width"), py::arg("context") = py::none(),
+        "Create an unsigned integer type");
+    c.def_property_readonly(
+        "width",
+        [](PyIntegerType &self) { return mlirIntegerTypeGetWidth(self); },
+        "Returns the width of the integer type");
+    c.def_property_readonly(
+        "is_signless",
+        [](PyIntegerType &self) -> bool {
+          return mlirIntegerTypeIsSignless(self);
+        },
+        "Returns whether this is a signless integer");
+    c.def_property_readonly(
+        "is_signed",
+        [](PyIntegerType &self) -> bool {
+          return mlirIntegerTypeIsSigned(self);
+        },
+        "Returns whether this is a signed integer");
+    c.def_property_readonly(
+        "is_unsigned",
+        [](PyIntegerType &self) -> bool {
+          return mlirIntegerTypeIsUnsigned(self);
+        },
+        "Returns whether this is an unsigned integer");
+  }
+};
+
+/// Index Type subclass - IndexType.
+class PyIndexType : public PyConcreteType<PyIndexType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAIndex;
+  static constexpr const char *pyClassName = "IndexType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](DefaultingPyMlirContext context) {
+          MlirType t = mlirIndexTypeGet(context->get());
+          return PyIndexType(context->getRef(), t);
+        },
+        py::arg("context") = py::none(), "Create a index type.");
+  }
+};
+
+/// Floating Point Type subclass - BF16Type.
+class PyBF16Type : public PyConcreteType<PyBF16Type> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsABF16;
+  static constexpr const char *pyClassName = "BF16Type";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](DefaultingPyMlirContext context) {
+          MlirType t = mlirBF16TypeGet(context->get());
+          return PyBF16Type(context->getRef(), t);
+        },
+        py::arg("context") = py::none(), "Create a bf16 type.");
+  }
+};
+
+/// Floating Point Type subclass - F16Type.
+class PyF16Type : public PyConcreteType<PyF16Type> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAF16;
+  static constexpr const char *pyClassName = "F16Type";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](DefaultingPyMlirContext context) {
+          MlirType t = mlirF16TypeGet(context->get());
+          return PyF16Type(context->getRef(), t);
+        },
+        py::arg("context") = py::none(), "Create a f16 type.");
+  }
+};
+
+/// Floating Point Type subclass - F32Type.
+class PyF32Type : public PyConcreteType<PyF32Type> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAF32;
+  static constexpr const char *pyClassName = "F32Type";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](DefaultingPyMlirContext context) {
+          MlirType t = mlirF32TypeGet(context->get());
+          return PyF32Type(context->getRef(), t);
+        },
+        py::arg("context") = py::none(), "Create a f32 type.");
+  }
+};
+
+/// Floating Point Type subclass - F64Type.
+class PyF64Type : public PyConcreteType<PyF64Type> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAF64;
+  static constexpr const char *pyClassName = "F64Type";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](DefaultingPyMlirContext context) {
+          MlirType t = mlirF64TypeGet(context->get());
+          return PyF64Type(context->getRef(), t);
+        },
+        py::arg("context") = py::none(), "Create a f64 type.");
+  }
+};
+
+/// None Type subclass - NoneType.
+class PyNoneType : public PyConcreteType<PyNoneType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsANone;
+  static constexpr const char *pyClassName = "NoneType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](DefaultingPyMlirContext context) {
+          MlirType t = mlirNoneTypeGet(context->get());
+          return PyNoneType(context->getRef(), t);
+        },
+        py::arg("context") = py::none(), "Create a none type.");
+  }
+};
+
+/// Complex Type subclass - ComplexType.
+class PyComplexType : public PyConcreteType<PyComplexType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAComplex;
+  static constexpr const char *pyClassName = "ComplexType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](PyType &elementType) {
+          // The element must be a floating point or integer scalar type.
+          if (mlirTypeIsAIntegerOrFloat(elementType)) {
+            MlirType t = mlirComplexTypeGet(elementType);
+            return PyComplexType(elementType.getContext(), t);
+          }
+          throw SetPyError(
+              PyExc_ValueError,
+              Twine("invalid '") +
+                  py::repr(py::cast(elementType)).cast<std::string>() +
+                  "' and expected floating point or integer type.");
+        },
+        "Create a complex type");
+    c.def_property_readonly(
+        "element_type",
+        [](PyComplexType &self) -> PyType {
+          MlirType t = mlirComplexTypeGetElementType(self);
+          return PyType(self.getContext(), t);
+        },
+        "Returns element type.");
+  }
+};
+
+class PyShapedType : public PyConcreteType<PyShapedType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAShaped;
+  static constexpr const char *pyClassName = "ShapedType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_property_readonly(
+        "element_type",
+        [](PyShapedType &self) {
+          MlirType t = mlirShapedTypeGetElementType(self);
+          return PyType(self.getContext(), t);
+        },
+        "Returns the element type of the shaped type.");
+    c.def_property_readonly(
+        "has_rank",
+        [](PyShapedType &self) -> bool { return mlirShapedTypeHasRank(self); },
+        "Returns whether the given shaped type is ranked.");
+    c.def_property_readonly(
+        "rank",
+        [](PyShapedType &self) {
+          self.requireHasRank();
+          return mlirShapedTypeGetRank(self);
+        },
+        "Returns the rank of the given ranked shaped type.");
+    c.def_property_readonly(
+        "has_static_shape",
+        [](PyShapedType &self) -> bool {
+          return mlirShapedTypeHasStaticShape(self);
+        },
+        "Returns whether the given shaped type has a static shape.");
+    c.def(
+        "is_dynamic_dim",
+        [](PyShapedType &self, intptr_t dim) -> bool {
+          self.requireHasRank();
+          return mlirShapedTypeIsDynamicDim(self, dim);
+        },
+        "Returns whether the dim-th dimension of the given shaped type is "
+        "dynamic.");
+    c.def(
+        "get_dim_size",
+        [](PyShapedType &self, intptr_t dim) {
+          self.requireHasRank();
+          return mlirShapedTypeGetDimSize(self, dim);
+        },
+        "Returns the dim-th dimension of the given ranked shaped type.");
+    c.def_static(
+        "is_dynamic_size",
+        [](int64_t size) -> bool { return mlirShapedTypeIsDynamicSize(size); },
+        "Returns whether the given dimension size indicates a dynamic "
+        "dimension.");
+    c.def(
+        "is_dynamic_stride_or_offset",
+        [](PyShapedType &self, int64_t val) -> bool {
+          self.requireHasRank();
+          return mlirShapedTypeIsDynamicStrideOrOffset(val);
+        },
+        "Returns whether the given value is used as a placeholder for dynamic "
+        "strides and offsets in shaped types.");
+  }
+
+private:
+  void requireHasRank() {
+    if (!mlirShapedTypeHasRank(*this)) {
+      throw SetPyError(
+          PyExc_ValueError,
+          "calling this method requires that the type has a rank.");
+    }
+  }
+};
+
+/// Vector Type subclass - VectorType.
+class PyVectorType : public PyConcreteType<PyVectorType, PyShapedType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAVector;
+  static constexpr const char *pyClassName = "VectorType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](std::vector<int64_t> shape, PyType &elementType,
+           DefaultingPyLocation loc) {
+          MlirType t = mlirVectorTypeGetChecked(loc, shape.size(), shape.data(),
+                                                elementType);
+          // TODO: Rework error reporting once diagnostic engine is exposed
+          // in C API.
+          if (mlirTypeIsNull(t)) {
+            throw SetPyError(
+                PyExc_ValueError,
+                Twine("invalid '") +
+                    py::repr(py::cast(elementType)).cast<std::string>() +
+                    "' and expected floating point or integer type.");
+          }
+          return PyVectorType(elementType.getContext(), t);
+        },
+        py::arg("shape"), py::arg("elementType"), py::arg("loc") = py::none(),
+        "Create a vector type");
+  }
+};
+
+/// Ranked Tensor Type subclass - RankedTensorType.
+class PyRankedTensorType
+    : public PyConcreteType<PyRankedTensorType, PyShapedType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsARankedTensor;
+  static constexpr const char *pyClassName = "RankedTensorType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](std::vector<int64_t> shape, PyType &elementType,
+           DefaultingPyLocation loc) {
+          MlirType t = mlirRankedTensorTypeGetChecked(
+              loc, shape.size(), shape.data(), elementType);
+          // TODO: Rework error reporting once diagnostic engine is exposed
+          // in C API.
+          if (mlirTypeIsNull(t)) {
+            throw SetPyError(
+                PyExc_ValueError,
+                Twine("invalid '") +
+                    py::repr(py::cast(elementType)).cast<std::string>() +
+                    "' and expected floating point, integer, vector or "
+                    "complex "
+                    "type.");
+          }
+          return PyRankedTensorType(elementType.getContext(), t);
+        },
+        py::arg("shape"), py::arg("element_type"), py::arg("loc") = py::none(),
+        "Create a ranked tensor type");
+  }
+};
+
+/// Unranked Tensor Type subclass - UnrankedTensorType.
+class PyUnrankedTensorType
+    : public PyConcreteType<PyUnrankedTensorType, PyShapedType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAUnrankedTensor;
+  static constexpr const char *pyClassName = "UnrankedTensorType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](PyType &elementType, DefaultingPyLocation loc) {
+          MlirType t = mlirUnrankedTensorTypeGetChecked(loc, elementType);
+          // TODO: Rework error reporting once diagnostic engine is exposed
+          // in C API.
+          if (mlirTypeIsNull(t)) {
+            throw SetPyError(
+                PyExc_ValueError,
+                Twine("invalid '") +
+                    py::repr(py::cast(elementType)).cast<std::string>() +
+                    "' and expected floating point, integer, vector or "
+                    "complex "
+                    "type.");
+          }
+          return PyUnrankedTensorType(elementType.getContext(), t);
+        },
+        py::arg("element_type"), py::arg("loc") = py::none(),
+        "Create a unranked tensor type");
+  }
+};
+
+class PyMemRefLayoutMapList;
+
+/// Ranked MemRef Type subclass - MemRefType.
+class PyMemRefType : public PyConcreteType<PyMemRefType, PyShapedType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsARankedTensor;
+  static constexpr const char *pyClassName = "MemRefType";
+  using PyConcreteType::PyConcreteType;
+
+  PyMemRefLayoutMapList getLayout();
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+         "get",
+         [](std::vector<int64_t> shape, PyType &elementType,
+            std::vector<PyAffineMap> layout, PyAttribute *memorySpace,
+            DefaultingPyLocation loc) {
+           SmallVector<MlirAffineMap> maps;
+           maps.reserve(layout.size());
+           for (PyAffineMap &map : layout)
+             maps.push_back(map);
+
+           MlirAttribute memSpaceAttr = {};
+           if (memorySpace)
+             memSpaceAttr = *memorySpace;
+
+           MlirType t = mlirMemRefTypeGetChecked(loc, elementType, shape.size(),
+                                                 shape.data(), maps.size(),
+                                                 maps.data(), memSpaceAttr);
+           // TODO: Rework error reporting once diagnostic engine is exposed
+           // in C API.
+           if (mlirTypeIsNull(t)) {
+             throw SetPyError(
+                 PyExc_ValueError,
+                 Twine("invalid '") +
+                     py::repr(py::cast(elementType)).cast<std::string>() +
+                     "' and expected floating point, integer, vector or "
+                     "complex "
+                     "type.");
+           }
+           return PyMemRefType(elementType.getContext(), t);
+         },
+         py::arg("shape"), py::arg("element_type"),
+         py::arg("layout") = py::list(), py::arg("memory_space") = py::none(),
+         py::arg("loc") = py::none(), "Create a memref type")
+        .def_property_readonly("layout", &PyMemRefType::getLayout,
+                               "The list of layout maps of the MemRef type.")
+        .def_property_readonly(
+            "memory_space",
+            [](PyMemRefType &self) -> PyAttribute {
+              MlirAttribute a = mlirMemRefTypeGetMemorySpace(self);
+              return PyAttribute(self.getContext(), a);
+            },
+            "Returns the memory space of the given MemRef type.");
+  }
+};
+
+/// A list of affine layout maps in a memref type. Internally, these are stored
+/// as consecutive elements, random access is cheap. Both the type and the maps
+/// are owned by the context, no need to worry about lifetime extension.
+class PyMemRefLayoutMapList
+    : public Sliceable<PyMemRefLayoutMapList, PyAffineMap> {
+public:
+  static constexpr const char *pyClassName = "MemRefLayoutMapList";
+
+  PyMemRefLayoutMapList(PyMemRefType type, intptr_t startIndex = 0,
+                        intptr_t length = -1, intptr_t step = 1)
+      : Sliceable(startIndex,
+                  length == -1 ? mlirMemRefTypeGetNumAffineMaps(type) : length,
+                  step),
+        memref(type) {}
+
+  intptr_t getNumElements() { return mlirMemRefTypeGetNumAffineMaps(memref); }
+
+  PyAffineMap getElement(intptr_t index) {
+    return PyAffineMap(memref.getContext(),
+                       mlirMemRefTypeGetAffineMap(memref, index));
+  }
+
+  PyMemRefLayoutMapList slice(intptr_t startIndex, intptr_t length,
+                              intptr_t step) {
+    return PyMemRefLayoutMapList(memref, startIndex, length, step);
+  }
+
+private:
+  PyMemRefType memref;
+};
+
+PyMemRefLayoutMapList PyMemRefType::getLayout() {
+  return PyMemRefLayoutMapList(*this);
+}
+
+/// Unranked MemRef Type subclass - UnrankedMemRefType.
+class PyUnrankedMemRefType
+    : public PyConcreteType<PyUnrankedMemRefType, PyShapedType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAUnrankedMemRef;
+  static constexpr const char *pyClassName = "UnrankedMemRefType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+         "get",
+         [](PyType &elementType, PyAttribute *memorySpace,
+            DefaultingPyLocation loc) {
+           MlirAttribute memSpaceAttr = {};
+           if (memorySpace)
+             memSpaceAttr = *memorySpace;
+
+           MlirType t =
+               mlirUnrankedMemRefTypeGetChecked(loc, elementType, memSpaceAttr);
+           // TODO: Rework error reporting once diagnostic engine is exposed
+           // in C API.
+           if (mlirTypeIsNull(t)) {
+             throw SetPyError(
+                 PyExc_ValueError,
+                 Twine("invalid '") +
+                     py::repr(py::cast(elementType)).cast<std::string>() +
+                     "' and expected floating point, integer, vector or "
+                     "complex "
+                     "type.");
+           }
+           return PyUnrankedMemRefType(elementType.getContext(), t);
+         },
+         py::arg("element_type"), py::arg("memory_space"),
+         py::arg("loc") = py::none(), "Create a unranked memref type")
+        .def_property_readonly(
+            "memory_space",
+            [](PyUnrankedMemRefType &self) -> PyAttribute {
+              MlirAttribute a = mlirMemRefTypeGetMemorySpace(self);
+              return PyAttribute(self.getContext(), a);
+            },
+            "Returns the memory space of the given Unranked MemRef type.");
+  }
+};
+
+/// Tuple Type subclass - TupleType.
+class PyTupleType : public PyConcreteType<PyTupleType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsATuple;
+  static constexpr const char *pyClassName = "TupleType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get_tuple",
+        [](py::list elementList, DefaultingPyMlirContext context) {
+          intptr_t num = py::len(elementList);
+          // Mapping py::list to SmallVector.
+          SmallVector<MlirType, 4> elements;
+          for (auto element : elementList)
+            elements.push_back(element.cast<PyType>());
+          MlirType t = mlirTupleTypeGet(context->get(), num, elements.data());
+          return PyTupleType(context->getRef(), t);
+        },
+        py::arg("elements"), py::arg("context") = py::none(),
+        "Create a tuple type");
+    c.def(
+        "get_type",
+        [](PyTupleType &self, intptr_t pos) -> PyType {
+          MlirType t = mlirTupleTypeGetType(self, pos);
+          return PyType(self.getContext(), t);
+        },
+        "Returns the pos-th type in the tuple type.");
+    c.def_property_readonly(
+        "num_types",
+        [](PyTupleType &self) -> intptr_t {
+          return mlirTupleTypeGetNumTypes(self);
+        },
+        "Returns the number of types contained in a tuple.");
+  }
+};
+
+/// Function type.
+class PyFunctionType : public PyConcreteType<PyFunctionType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAFunction;
+  static constexpr const char *pyClassName = "FunctionType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](std::vector<PyType> inputs, std::vector<PyType> results,
+           DefaultingPyMlirContext context) {
+          SmallVector<MlirType, 4> inputsRaw(inputs.begin(), inputs.end());
+          SmallVector<MlirType, 4> resultsRaw(results.begin(), results.end());
+          MlirType t = mlirFunctionTypeGet(context->get(), inputsRaw.size(),
+                                           inputsRaw.data(), resultsRaw.size(),
+                                           resultsRaw.data());
+          return PyFunctionType(context->getRef(), t);
+        },
+        py::arg("inputs"), py::arg("results"), py::arg("context") = py::none(),
+        "Gets a FunctionType from a list of input and result types");
+    c.def_property_readonly(
+        "inputs",
+        [](PyFunctionType &self) {
+          MlirType t = self;
+          auto contextRef = self.getContext();
+          py::list types;
+          for (intptr_t i = 0, e = mlirFunctionTypeGetNumInputs(self); i < e;
+               ++i) {
+            types.append(PyType(contextRef, mlirFunctionTypeGetInput(t, i)));
+          }
+          return types;
+        },
+        "Returns the list of input types in the FunctionType.");
+    c.def_property_readonly(
+        "results",
+        [](PyFunctionType &self) {
+          auto contextRef = self.getContext();
+          py::list types;
+          for (intptr_t i = 0, e = mlirFunctionTypeGetNumResults(self); i < e;
+               ++i) {
+            types.append(
+                PyType(contextRef, mlirFunctionTypeGetResult(self, i)));
+          }
+          return types;
+        },
+        "Returns the list of result types in the FunctionType.");
+  }
+};
+
+} // namespace
+
+void mlir::python::populateIRTypes(py::module &m) {
+  PyIntegerType::bind(m);
+  PyIndexType::bind(m);
+  PyBF16Type::bind(m);
+  PyF16Type::bind(m);
+  PyF32Type::bind(m);
+  PyF64Type::bind(m);
+  PyNoneType::bind(m);
+  PyComplexType::bind(m);
+  PyShapedType::bind(m);
+  PyVectorType::bind(m);
+  PyRankedTensorType::bind(m);
+  PyUnrankedTensorType::bind(m);
+  PyMemRefType::bind(m);
+  PyMemRefLayoutMapList::bind(m);
+  PyUnrankedMemRefType::bind(m);
+  PyTupleType::bind(m);
+  PyFunctionType::bind(m);
+}
diff --git a/mlir/lib/Bindings/Python/LinalgPasses.cpp b/mlir/lib/Bindings/Python/LinalgPasses.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3f230207a42114b6eb3ebcd7444b3996a8f957d8
--- /dev/null
+++ b/mlir/lib/Bindings/Python/LinalgPasses.cpp
@@ -0,0 +1,22 @@
+//===- LinalgPasses.cpp - Pybind module for the Linalg passes -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir-c/Dialect/Linalg.h"
+
+#include <pybind11/pybind11.h>
+
+// -----------------------------------------------------------------------------
+// Module initialization.
+// -----------------------------------------------------------------------------
+
+PYBIND11_MODULE(_mlirLinalgPasses, m) {
+  m.doc() = "MLIR Linalg Dialect Passes";
+
+  // Register all Linalg passes on load.
+  mlirRegisterLinalgPasses();
+}
diff --git a/mlir/lib/Bindings/Python/MainModule.cpp b/mlir/lib/Bindings/Python/MainModule.cpp
index 9bfe8b09f6db504de1645b9280732c34b68a1032..5fe0401afaeb62915431761e4d0e83734331a195 100644
--- a/mlir/lib/Bindings/Python/MainModule.cpp
+++ b/mlir/lib/Bindings/Python/MainModule.cpp
@@ -12,7 +12,7 @@
 
 #include "ExecutionEngine.h"
 #include "Globals.h"
-#include "IRModules.h"
+#include "IRModule.h"
 #include "Pass.h"
 
 namespace py = pybind11;
@@ -211,7 +211,10 @@ PYBIND11_MODULE(_mlir, m) {
 
   // Define and populate IR submodule.
   auto irModule = m.def_submodule("ir", "MLIR IR Bindings");
-  populateIRSubmodule(irModule);
+  populateIRCore(irModule);
+  populateIRAffine(irModule);
+  populateIRAttributes(irModule);
+  populateIRTypes(irModule);
 
   // Define and populate PassManager submodule.
   auto passModule =
diff --git a/mlir/lib/Bindings/Python/Pass.cpp b/mlir/lib/Bindings/Python/Pass.cpp
index dd57647f03270a46b3aef1c808a13d6361fdae6b..0e2f5bafb4655b846c0e7abe6e7a72f8ca6b9cd9 100644
--- a/mlir/lib/Bindings/Python/Pass.cpp
+++ b/mlir/lib/Bindings/Python/Pass.cpp
@@ -8,7 +8,7 @@
 
 #include "Pass.h"
 
-#include "IRModules.h"
+#include "IRModule.h"
 #include "mlir-c/Bindings/Python/Interop.h"
 #include "mlir-c/Pass.h"
 
diff --git a/mlir/lib/Bindings/Python/mlir/dialects/_builtin_ops_ext.py b/mlir/lib/Bindings/Python/mlir/dialects/_builtin_ops_ext.py
index b0789299139d9e568ccdece13eef49c357f92270..6598efe3e082830de64f7ac18472c836bca4caec 100644
--- a/mlir/lib/Bindings/Python/mlir/dialects/_builtin_ops_ext.py
+++ b/mlir/lib/Bindings/Python/mlir/dialects/_builtin_ops_ext.py
@@ -1,6 +1,11 @@
 #  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from typing import Optional, Sequence
+
+import inspect
+
 from ..ir import *
 
 
@@ -11,8 +16,6 @@ class ModuleOp:
     super().__init__(self.build_generic(results=[], operands=[], loc=loc,
                                         ip=ip))
     body = self.regions[0].blocks.append()
-    with InsertionPoint(body):
-      Operation.create("module_terminator")
 
   @property
   def body(self):
@@ -93,3 +96,99 @@ class FuncOp:
       raise IndexError('The function already has an entry block!')
     self.body.blocks.append(*self.type.inputs)
     return self.body.blocks[0]
+
+  @classmethod
+  def from_py_func(FuncOp,
+                   *inputs: Type,
+                   results: Optional[Sequence[Type]] = None,
+                   name: Optional[str] = None):
+    """Decorator to define an MLIR FuncOp specified as a python function.
+
+    Requires that an `mlir.ir.InsertionPoint` and `mlir.ir.Location` are
+    active for the current thread (i.e. established in a `with` block).
+
+    When applied as a decorator to a Python function, an entry block will
+    be constructed for the FuncOp with types as specified in `*inputs`. The
+    block arguments will be passed positionally to the Python function. In
+    addition, if the Python function accepts keyword arguments generally or
+    has a corresponding keyword argument, the following will be passed:
+      * `func_op`: The `func` op being defined.
+
+    By default, the function name will be the Python function `__name__`. This
+    can be overriden by passing the `name` argument to the decorator.
+
+    If `results` is not specified, then the decorator will implicitly
+    insert a `ReturnOp` with the `Value`'s returned from the decorated
+    function. It will also set the `FuncOp` type with the actual return
+    value types. If `results` is specified, then the decorated function
+    must return `None` and no implicit `ReturnOp` is added (nor are the result
+    types updated). The implicit behavior is intended for simple, single-block
+    cases, and users should specify result types explicitly for any complicated
+    cases.
+
+    The decorated function can further be called from Python and will insert
+    a `CallOp` at the then-current insertion point, returning either None (
+    if no return values), a unary Value (for one result), or a list of Values).
+    This mechanism cannot be used to emit recursive calls (by construction).
+    """
+
+    def decorator(f):
+      from . import std
+      # Introspect the callable for optional features.
+      sig = inspect.signature(f)
+      has_arg_func_op = False
+      for param in sig.parameters.values():
+        if param.kind == param.VAR_KEYWORD:
+          has_arg_func_op = True
+        if param.name == "func_op" and (param.kind
+                                        == param.POSITIONAL_OR_KEYWORD or
+                                        param.kind == param.KEYWORD_ONLY):
+          has_arg_func_op = True
+
+      # Emit the FuncOp.
+      implicit_return = results is None
+      symbol_name = name or f.__name__
+      function_type = FunctionType.get(
+          inputs=inputs, results=[] if implicit_return else results)
+      func_op = FuncOp(name=symbol_name, type=function_type)
+      with InsertionPoint(func_op.add_entry_block()):
+        func_args = func_op.entry_block.arguments
+        func_kwargs = {}
+        if has_arg_func_op:
+          func_kwargs["func_op"] = func_op
+        return_values = f(*func_args, **func_kwargs)
+        if not implicit_return:
+          return_types = list(results)
+          assert return_values is None, (
+              "Capturing a python function with explicit `results=` "
+              "requires that the wrapped function returns None.")
+        else:
+          # Coerce return values, add ReturnOp and rewrite func type.
+          if return_values is None:
+            return_values = []
+          elif isinstance(return_values, Value):
+            return_values = [return_values]
+          else:
+            return_values = list(return_values)
+          std.ReturnOp(return_values)
+          # Recompute the function type.
+          return_types = [v.type for v in return_values]
+          function_type = FunctionType.get(inputs=inputs, results=return_types)
+          func_op.attributes["type"] = TypeAttr.get(function_type)
+
+      def emit_call_op(*call_args):
+        call_op = std.CallOp(return_types, FlatSymbolRefAttr.get(symbol_name),
+                             call_args)
+        if return_types is None:
+          return None
+        elif len(return_types) == 1:
+          return call_op.result
+        else:
+          return call_op.results
+
+      wrapped = emit_call_op
+      wrapped.__name__ = f.__name__
+      wrapped.func_op = func_op
+      return wrapped
+
+    return decorator
diff --git a/mlir/lib/Bindings/Python/mlir/dialects/_linalg_ops_ext.py b/mlir/lib/Bindings/Python/mlir/dialects/_linalg_ops_ext.py
index 74390d487a6702bac1e04edb92c66af6beea2ba7..d35d10cc4b8eb36c96fe0a9e66869dc868a5d771 100644
--- a/mlir/lib/Bindings/Python/mlir/dialects/_linalg_ops_ext.py
+++ b/mlir/lib/Bindings/Python/mlir/dialects/_linalg_ops_ext.py
@@ -2,6 +2,48 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+from typing import Optional, Sequence, Union
+from ..ir import *
+from ._ods_common import get_default_loc_context
+
+
+class InitTensorOp:
+  """Extends the linalg.init_tensor op."""
+
+  def __init__(self,
+               sizes: Union[Sequence[int], Sequence[Value]],
+               element_type: Type,
+               *,
+               loc=None,
+               ip=None):
+    """Constructs an `init_tensor` with either static or dynamic sizes."""
+    context = get_default_loc_context(loc)
+    operands = []
+    attributes = {}
+    # TODO: Refactor the InitTensorOp to take an element type attribute and
+    # then use normal result type inference, unifying the Python and C++ side
+    # with a standard mechanism (versus stashing that in builders).
+    if sizes and isinstance(sizes[0], Value):
+      # Dynamic sizes.
+      operands.extend(sizes)
+      static_size_ints = [-1] * len(sizes)
+      result_type = RankedTensorType.get(static_size_ints, element_type)
+    else:
+      # Static sizes.
+      result_type = RankedTensorType.get(sizes, element_type)
+      static_size_ints = sizes
+
+    index_type = IndexType.get(context)
+    attributes["static_sizes"] = ArrayAttr.get(
+        [IntegerAttr.get(index_type, s) for s in static_size_ints],
+        context=context)
+    op = self.build_generic(results=[result_type],
+                            operands=operands,
+                            attributes=attributes,
+                            loc=loc,
+                            ip=ip)
+    OpView.__init__(self, op)
+
 
 class StructuredOpMixin:
   """All structured ops use the same mixin class."""
diff --git a/mlir/lib/Bindings/Python/mlir/dialects/_ods_common.py b/mlir/lib/Bindings/Python/mlir/dialects/_ods_common.py
index 6d37700ecdc4730a38d34d418c6670c5360dc465..d030440887414b4da63def80b154bf5bcf2750f1 100644
--- a/mlir/lib/Bindings/Python/mlir/dialects/_ods_common.py
+++ b/mlir/lib/Bindings/Python/mlir/dialects/_ods_common.py
@@ -17,14 +17,15 @@ def extend_opview_class(ext_module):
   """Decorator to extend an OpView class from an extension module.
 
   Extension modules can expose various entry-points:
+    Stand-alone class with the same name as a parent OpView class (i.e.
+    "ReturnOp"). A name-based match is attempted first before falling back
+    to a below mechanism.
+
     def select_opview_mixin(parent_opview_cls):
       If defined, allows an appropriate mixin class to be selected dynamically
       based on the parent OpView class. Should return NotImplemented if a
       decision is not made.
 
-    Stand-alone class with the same name as a parent OpView class (i.e.
-    "ReturnOp").
-
   Args:
     ext_module: A module from which to locate extensions. Can be None if not
       available.
@@ -38,16 +39,18 @@ def extend_opview_class(ext_module):
     if ext_module is None:
       return parent_opview_cls
     mixin_cls = NotImplemented
+    # First try to resolve by name.
     try:
-      select_mixin = getattr(ext_module, "select_opview_mixin")
+      mixin_cls = getattr(ext_module, parent_opview_cls.__name__)
     except AttributeError:
-      # Try to default resolve it.
+      # Fall back to a select_opview_mixin hook.
       try:
-        mixin_cls = getattr(ext_module, parent_opview_cls.__name__)
+        select_mixin = getattr(ext_module, "select_opview_mixin")
       except AttributeError:
         pass
-    else:
-      mixin_cls = select_mixin(parent_opview_cls)
+      else:
+        mixin_cls = select_mixin(parent_opview_cls)
+
     if mixin_cls is NotImplemented or mixin_cls is None:
       return parent_opview_cls
 
diff --git a/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/dump_oplib.py b/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/dump_oplib.py
index 98bf2e247ea18320f2b41700f23b12893212adb6..bacc0c302c5e3b4c90560ca48cf5c968c460ec09 100644
--- a/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/dump_oplib.py
+++ b/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/dump_oplib.py
@@ -14,7 +14,7 @@ this module). Loose module files can be specified via `--file <filepath>`.
 Sample usage:
   # Dump the YAML op definitions for the core named ops (as in the dialect
   # source tree).
-  python -m mlir.tools.linalg_opdsl.dump_oplib .ops.core_named_ops
+  python -m mlir.dialects.linalg.opdsl.dump_oplib .ops.core_named_ops
 
 Note: YAML output is emitted in "document list" format with each operation
 as its own "document". Practically, this means that each operation (or group
diff --git a/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/config.py b/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/config.py
index 115ea40619b8f5b74141100a2c773181b6885017..fdc6cfd9bab078863ce80171aca9b3582455cf46 100644
--- a/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/config.py
+++ b/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/config.py
@@ -21,6 +21,7 @@ from .yaml_helper import *
 __all__ = [
     "LinalgStructuredOpConfig",
     "LinalgOpConfig",
+    "TensorDefConfig",
 ]
 
 
@@ -51,17 +52,17 @@ class TensorDefConfig(YAMLObject):
     self.shape_map = shape_map
     self.indexing_map = None  # type: Optional[_ir.AffineMap]
 
-  def to_yaml_custom_dict(self):
-
-    def get_usage():
-      if self.tensor_def.output:
-        return "output"
-      else:
-        return "input"
+  @property
+  def usage(self) -> str:
+    if self.tensor_def.output:
+      return "output"
+    else:
+      return "input"
 
+  def to_yaml_custom_dict(self):
     return dict(
         name=self.tensor_def.tensor_name,
-        usage=get_usage(),
+        usage=self.usage,
         shape=_serialize_affine_map(self.shape_map),
         element_type_var=self.tensor_def.type_var.name,
     )
diff --git a/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/dsl.py b/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/dsl.py
index d367c5bdde077b9403dcbbcbaf299dae6b47fa7f..cbff41db2d889b27533683204763fae676873713 100644
--- a/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/dsl.py
+++ b/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/dsl.py
@@ -11,6 +11,8 @@ import threading
 
 from mlir import ir
 from .comprehension import *
+from .config import *
+from .emitter import *
 
 _CONTEXT = threading.local()
 
@@ -42,9 +44,34 @@ class DefinedOpCallable:
     self.op_name = op_name
     self.model = model
 
-  def __call__(self, *args, **kwargs):
-    # TODO: Upstream the emitter and invoke here
-    raise NotImplementedError("Linalg generic emission not yet implemented")
+  def __call__(self, *args, emit_generic: bool = True, **kwargs):
+    """Emits the corresponding op definition as IR.
+
+    Most arguments are passed through to the underlying emitter. The following
+    are interpreted here:
+      emit_generic: Emits a generic form as appropriate (default True). If
+        False, a named form is emitted (which must have been built in to the
+        compiler).
+    """
+    op_configs = LinalgOpConfig.from_linalg_op_def(self.model,
+                                                   context=ir.Context.current)
+
+    if len(op_configs) != 1:
+      # TODO: Support composite ops.
+      raise NotImplementedError(
+          f"Emission of composite linalg ops not supported: {op_configs}")
+
+    op_config = op_configs[0]
+    if op_config.structured_op:
+      if emit_generic:
+        return emit_generic_structured_op(op_config.structured_op, *args,
+                                          **kwargs)
+      else:
+        return emit_named_structured_op(op_config.structured_op, *args,
+                                        **kwargs)
+
+    raise NotImplementedError(
+        f"Emission of linalg op type not supported: {op_config}")
 
 
 def linalg_structured_op(dsl_func=None,
diff --git a/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/emitter.py b/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/emitter.py
new file mode 100644
index 0000000000000000000000000000000000000000..9a18993e9f6272c696f2de689edc5be3e6c15d57
--- /dev/null
+++ b/mlir/lib/Bindings/Python/mlir/dialects/linalg/opdsl/lang/emitter.py
@@ -0,0 +1,252 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from typing import Dict, Sequence
+
+from mlir.ir import *
+from mlir.dialects import linalg
+from mlir.dialects import std
+
+from .scalar_expr import *
+from .config import *
+
+__all__ = [
+    "emit_generic_structured_op",
+    "emit_named_structured_op",
+]
+
+
+def emit_generic_structured_op(op_config: LinalgStructuredOpConfig,
+                               *ins: Value,
+                               outs: Value = ()):
+  all_arg_defs = op_config.ordered_tensor_args
+  in_arg_defs = [arg for arg in all_arg_defs if arg.usage == "input"]
+  out_arg_defs = [arg for arg in all_arg_defs if arg.usage == "output"]
+
+  # Arity validation.
+  if len(ins) != len(in_arg_defs):
+    raise ValueError(f"Expected {len(in_arg_defs)} inputs but got "
+                     f"{len(ins)} for {op_config}")
+  if outs and len(outs) != len(out_arg_defs):
+    raise ValueError(f"Expected {len(out_arg_defs)} outputs but got "
+                     f"{len(outs)} for {op_config}")
+
+  outs, out_types = _infer_structured_outs(op_config, in_arg_defs, ins,
+                                           out_arg_defs, outs)
+
+  # Extract type vars for input/output based types.
+  type_mapping = dict()  # type: Dict[str, Type]
+  for arg_def, arg_element_type in zip(
+      in_arg_defs + out_arg_defs,
+      _get_shaped_element_types_from_values(*ins, *outs)):
+    tv_name = arg_def.tensor_def.type_var.name
+    type_mapping[tv_name] = arg_element_type
+
+  # Emit the generic op.
+  # TODO: Support emission of pure memref form.
+  indexing_maps_attr = ArrayAttr.get(
+      [AffineMapAttr.get(am) for am in op_config.indexing_maps])
+  iterator_types_attr = ArrayAttr.get(
+      [StringAttr.get(s) for s in op_config.iterator_types])
+  generic_op = linalg.GenericOp(
+      result_tensors=out_types,
+      inputs=ins,
+      outputs=outs,
+      indexing_maps=indexing_maps_attr,
+      iterator_types=iterator_types_attr,
+      doc=None,  # TODO: Make optional.
+      library_call=None,  # TODO: Make optional.
+      sparse=BoolAttr.get(False))  # TODO: Make optional.
+
+  # Construct the body.
+  block_arg_names = _get_tensor_def_names(*in_arg_defs, *out_arg_defs)
+  block_arg_types = _get_shaped_element_types_from_values(*ins, *outs)
+  block = generic_op.regions[0].blocks.append(*block_arg_types)
+  block_arg_mapping = dict(zip(block_arg_names, block.arguments))
+  with InsertionPoint(block):
+    body_builder = _BodyBuilder(type_mapping, block_arg_mapping)
+    for assignment in op_config.assignments:
+      body_builder.assign(assignment)
+    body_builder.yield_outputs(*_get_tensor_def_names(*out_arg_defs))
+
+  if len(out_arg_defs) == 1:
+    return generic_op.result
+  else:
+    return generic_op.results
+
+
+def emit_named_structured_op(op_config: LinalgStructuredOpConfig,
+                             *ins: Value,
+                             outs: Value = ()):
+  raise NotImplementedError(
+      f"Emission of named structured ops is not supported: {op_config}")
+
+
+class _BodyBuilder:
+  """Constructs a structured op body by evaluating assignments."""
+
+  def __init__(self, type_mapping: Dict[str, Type],
+               block_arg_mapping: Dict[str, Value]):
+    self.type_mapping = type_mapping
+    self.block_arg_mapping = block_arg_mapping
+    self.yield_mapping = dict()  # type: Dict[str, Value]
+
+  def assign(self, assignment: ScalarAssign):
+    if assignment.arg in self.yield_mapping:
+      raise ValueError(
+          f"Multiple assignments to the same argument are forbidden: "
+          f"{assignment}")
+    self.yield_mapping[assignment.arg] = self.expression(assignment.value)
+
+  def expression(self, expr: ScalarExpression) -> Value:
+    if expr.scalar_arg:
+      try:
+        return self.block_arg_mapping[expr.scalar_arg.arg]
+      except KeyError:
+        raise ValueError(f"Argument {expr.scalar_arg.arg} is not bound for "
+                         f"this structured op.")
+    elif expr.scalar_apply:
+      try:
+        fn = getattr(self, f"_eval_{expr.scalar_apply.fn_name}")
+      except AttributeError:
+        raise ValueError(
+            f"Function '{expr.scalar_apply.fn_name}' is not a known "
+            "scalar body function")
+      operand_values = [
+          self.expression(operand) for operand in expr.scalar_apply.operands
+      ]
+      return fn(*operand_values)
+    elif expr.symbolic_cast:
+      operand_value = self.expression(expr.symbolic_cast.operand)
+      return self.cast(expr.symbolic_cast.to_type.name, operand_value)
+    raise NotImplementedError(f"Unimplemented scalar body expression: {expr}")
+
+  def cast(self, type_var_name: str, operand: Value) -> Value:
+    try:
+      to_type = self.type_mapping[type_var_name]
+    except KeyError:
+      raise ValueError(f"Unbound type variable '{type_var_name}' ("
+                       f"expected one of {self.type_mappings.keys()}")
+    if operand.type == to_type:
+      return operand
+    if _is_integer_type(to_type):
+      return self._cast_to_integer(to_type, operand)
+    elif _is_floating_point_type(to_type):
+      return self._cast_to_floating_point(to_type, operand)
+
+    raise ValueError(f"Unable to cast body expression from {operand.type} to "
+                     f"{to_type}")
+
+  def _cast_to_integer(self, to_type: Type, operand: Value) -> Value:
+    to_width = IntegerType(to_type).width
+    operand_type = operand.type
+    if _is_floating_point_type(operand_type):
+      return std.FPToSIOp(to_type, operand).result
+    # Assume integer.
+    from_width = IntegerType(operand_type).width
+    if to_width > from_width:
+      return std.SignExtendIOp(to_type, operand).result
+    elif to_width < from_width:
+      return std.TruncateIOp(to_type, operand).result
+    raise ValueError(f"Unable to cast body expression from {operand_type} to "
+                     f"{to_type}")
+
+  def _cast_to_floating_point(self, to_type: Type, operand: Value) -> Value:
+    operand_type = operand.type
+    if _is_integer_type(operand_type):
+      return std.SIToFPOp(to_type, operand).result
+    # Assume FloatType.
+    to_width = _get_floating_point_width(to_type)
+    from_width = _get_floating_point_width(operand_type)
+    if to_width > from_width:
+      return std.FPExtOp(to_type, operand).result
+    elif to_width < from_width:
+      return std.FPTruncOp(to_type, operand).result
+    raise ValueError(f"Unable to cast body expression from {operand_type} to "
+                     f"{to_type}")
+
+  def yield_outputs(self, *output_names: str):
+    output_values = []
+    for n in output_names:
+      try:
+        output_values.append(self.yield_mapping[n])
+      except KeyError:
+        raise ValueError(f"Body assignments do not assign all outputs: "
+                         f"missing '{n}'")
+    linalg.YieldOp(output_values)
+
+  def _eval_add(self, lhs: Value, rhs: Value) -> Value:
+    if _is_floating_point_type(lhs.type):
+      return std.AddFOp(lhs.type, lhs, rhs).result
+    if _is_integer_type(lhs.type):
+      return std.AddIOp(lhs.type, lhs, rhs).result
+    raise NotImplementedError("Unsupported 'add' operand: {lhs}")
+
+  def _eval_mul(self, lhs: Value, rhs: Value) -> Value:
+    if _is_floating_point_type(lhs.type):
+      return std.MulFOp(lhs.type, lhs, rhs).result
+    if _is_integer_type(lhs.type):
+      return std.MulIOp(lhs.type, lhs, rhs).result
+    raise NotImplementedError("Unsupported 'mul' operand: {lhs}")
+
+
+def _infer_structured_outs(op_config: LinalgStructuredOpConfig,
+                           in_arg_defs: Sequence[TensorDefConfig],
+                           ins: Sequence[Value],
+                           out_arg_defs: Sequence[TensorDefConfig],
+                           outs: Sequence[Value]):
+  """Infers implicit outs and output types.
+
+  Respects existing contents of outs if not empty.
+
+  Returns:
+    normalized outs, output types
+  """
+  # If outs were explicitly provided, we accept them verbatim.
+  if outs:
+    return outs, [out.type for out in outs]
+
+  raise NotImplementedError(f"Output tensor inference not yet supported for "
+                            "structured ops")
+
+
+def _get_shaped_element_types_from_values(*values: Value) -> Sequence[Type]:
+  types = []
+  for v in values:
+    try:
+      t = ShapedType(v.type)
+    except Exception as e:
+      raise ValueError(f"Expected ShapedType but got {v}") from e
+    types.append(t.element_type)
+  return types
+
+
+def _get_tensor_def_names(
+    *tensor_def_configs: TensorDefConfig) -> Sequence[str]:
+  return [tdc.tensor_def.tensor_name for tdc in tensor_def_configs]
+
+
+def _is_floating_point_type(t: Type) -> bool:
+  # TODO: Create a FloatType in the Python API and implement the switch
+  # there.
+  return (F64Type.isinstance(t) or F32Type.isinstance(t) or
+          F16Type.isinstance(t) or BF16Type.isinstance(t))
+
+
+def _is_integer_type(t: Type) -> bool:
+  return IntegerType.isinstance(t)
+
+
+def _get_floating_point_width(t: Type) -> int:
+  # TODO: Create a FloatType in the Python API and implement the switch
+  # there.
+  if F64Type.isinstance(t):
+    return 64
+  if F32Type.isinstance(t):
+    return 32
+  if F16Type.isinstance(t):
+    return 16
+  if BF16Type.isinstance(t):
+    return 16
+  raise NotImplementedError(f"Unhandled floating point type switch {t}")
diff --git a/mlir/lib/Bindings/Python/mlir/dialects/linalg/passes/__init__.py b/mlir/lib/Bindings/Python/mlir/dialects/linalg/passes/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6555ad69a5231a54d3accbcac52fc759c6eb5dcb
--- /dev/null
+++ b/mlir/lib/Bindings/Python/mlir/dialects/linalg/passes/__init__.py
@@ -0,0 +1,6 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from ...._cext_loader import _load_extension
+_cextLinalgPasses = _load_extension("_mlirLinalgPasses")
diff --git a/mlir/lib/CAPI/Dialect/CMakeLists.txt b/mlir/lib/CAPI/Dialect/CMakeLists.txt
index d256309bf8f08fa19612023e8672c80084f28fba..41c659d6ab75469dfa64ba352e6627b6fd03930a 100644
--- a/mlir/lib/CAPI/Dialect/CMakeLists.txt
+++ b/mlir/lib/CAPI/Dialect/CMakeLists.txt
@@ -1,6 +1,7 @@
 # TODO: Make the check source feature optional as an argument on *_add_library.
 set(LLVM_OPTIONAL_SOURCES
   Linalg.cpp
+  LinalgPasses.cpp
   SCF.cpp
   Shape.cpp
   Standard.cpp
@@ -9,10 +10,16 @@ set(LLVM_OPTIONAL_SOURCES
 
 add_mlir_public_c_api_library(MLIRCAPILinalg
   Linalg.cpp
+  LinalgPasses.cpp
+
+  DEPENDS
+  MLIRLinalgPassIncGen
 
   LINK_LIBS PUBLIC
   MLIRCAPIIR
   MLIRLinalg
+  MLIRPass
+  MLIRLinalgTransforms
 )
 
 add_mlir_public_c_api_library(MLIRCAPISCF
diff --git a/mlir/lib/CAPI/Dialect/LinalgPasses.cpp b/mlir/lib/CAPI/Dialect/LinalgPasses.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6677476d8a183be5badfe99ddb792fe81d3a990f
--- /dev/null
+++ b/mlir/lib/CAPI/Dialect/LinalgPasses.cpp
@@ -0,0 +1,26 @@
+//===- LinalgPasses.cpp - C API for Linalg Dialect Passes -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/CAPI/Pass.h"
+#include "mlir/Dialect/Linalg/Passes.h"
+#include "mlir/Pass/Pass.h"
+
+// Must include the declarations as they carry important visibility attributes.
+#include "mlir/Dialect/Linalg/Passes.capi.h.inc"
+
+using namespace mlir;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "mlir/Dialect/Linalg/Passes.capi.cpp.inc"
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index c7b5183e07c9efb2a871c30f02108ed07e86453a..1ad07b2f6e068729e6fadbad1bd5e4e3c1f0767c 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -532,7 +532,8 @@ public:
                 : rewriter.create<ConstantIntOp>(loc, /*value=*/1, /*width=*/1);
 
     bool hasElseRegion = !op.elseRegion().empty();
-    auto ifOp = rewriter.create<scf::IfOp>(loc, cond, hasElseRegion);
+    auto ifOp = rewriter.create<scf::IfOp>(loc, op.getResultTypes(), cond,
+                                           hasElseRegion);
     rewriter.inlineRegionBefore(op.thenRegion(), &ifOp.thenRegion().back());
     rewriter.eraseBlock(&ifOp.thenRegion().back());
     if (hasElseRegion) {
@@ -540,8 +541,8 @@ public:
       rewriter.eraseBlock(&ifOp.elseRegion().back());
     }
 
-    // Ok, we're done!
-    rewriter.eraseOp(op);
+    // Replace the Affine IfOp finally.
+    rewriter.replaceOp(op, ifOp.results());
     return success();
   }
 };
@@ -745,10 +746,9 @@ public:
 
 } // end namespace
 
-void mlir::populateAffineToStdConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx) {
+void mlir::populateAffineToStdConversionPatterns(RewritePatternSet &patterns) {
   // clang-format off
-  patterns.insert<
+  patterns.add<
       AffineApplyLowering,
       AffineDmaStartLowering,
       AffineDmaWaitLowering,
@@ -760,25 +760,25 @@ void mlir::populateAffineToStdConversionPatterns(
       AffineStoreLowering,
       AffineForLowering,
       AffineIfLowering,
-      AffineYieldOpLowering>(ctx);
+      AffineYieldOpLowering>(patterns.getContext());
   // clang-format on
 }
 
 void mlir::populateAffineToVectorConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx) {
+    RewritePatternSet &patterns) {
   // clang-format off
-  patterns.insert<
+  patterns.add<
       AffineVectorLoadLowering,
-      AffineVectorStoreLowering>(ctx);
+      AffineVectorStoreLowering>(patterns.getContext());
   // clang-format on
 }
 
 namespace {
 class LowerAffinePass : public ConvertAffineToStandardBase<LowerAffinePass> {
   void runOnOperation() override {
-    OwningRewritePatternList patterns;
-    populateAffineToStdConversionPatterns(patterns, &getContext());
-    populateAffineToVectorConversionPatterns(patterns, &getContext());
+    RewritePatternSet patterns(&getContext());
+    populateAffineToStdConversionPatterns(patterns);
+    populateAffineToVectorConversionPatterns(patterns);
     ConversionTarget target(getContext());
     target.addLegalDialect<memref::MemRefDialect, scf::SCFDialect,
                            StandardOpsDialect, VectorDialect>();
diff --git a/mlir/lib/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.cpp b/mlir/lib/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.cpp
index 1d95f73327fd88ace489cc171d7da79741195248..7ac8fa2b6c99dcca2f989c9672f5b6d235a6d702 100644
--- a/mlir/lib/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.cpp
@@ -96,19 +96,19 @@ static Optional<Value> addUnrealizedCast(OpBuilder &builder,
 }
 
 /// Populate the given list with patterns that convert from ArmSVE to LLVM.
-void mlir::populateArmSVEToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+void mlir::populateArmSVEToLLVMConversionPatterns(LLVMTypeConverter &converter,
+                                                  RewritePatternSet &patterns) {
   converter.addConversion([&converter](ScalableVectorType svType) {
     return convertScalableVectorTypeToLLVM(svType, converter);
   });
   converter.addSourceMaterialization(addUnrealizedCast);
 
   // clang-format off
-  patterns.insert<ForwardOperands<CallOp>,
+  patterns.add<ForwardOperands<CallOp>,
                   ForwardOperands<CallIndirectOp>,
                   ForwardOperands<ReturnOp>>(converter,
                                              &converter.getContext());
-  patterns.insert<SdotOpLowering,
+  patterns.add<SdotOpLowering,
                   SmmlaOpLowering,
                   UdotOpLowering,
                   UmmlaOpLowering,
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index 3fe1c7fef2ec0f8d4a0c6e664dda56082d71b3a3..7a24b75640ec1d4f5c91b46c32a26178b4f23257 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -156,8 +156,8 @@ struct AsyncAPI {
 
 /// Adds Async Runtime C API declarations to the module.
 static void addAsyncRuntimeApiDeclarations(ModuleOp module) {
-  auto builder = ImplicitLocOpBuilder::atBlockTerminator(module.getLoc(),
-                                                         module.getBody());
+  auto builder =
+      ImplicitLocOpBuilder::atBlockEnd(module.getLoc(), module.getBody());
 
   auto addFuncDecl = [&](StringRef name, FunctionType type) {
     if (module.lookupSymbol(name))
@@ -207,8 +207,8 @@ static void addCRuntimeDeclarations(ModuleOp module) {
   using namespace mlir::LLVM;
 
   MLIRContext *ctx = module.getContext();
-  ImplicitLocOpBuilder builder(module.getLoc(),
-                               module.getBody()->getTerminator());
+  auto builder =
+      ImplicitLocOpBuilder::atBlockEnd(module.getLoc(), module.getBody());
 
   auto voidTy = LLVMVoidType::get(ctx);
   auto i64 = IntegerType::get(ctx, 64);
@@ -232,15 +232,14 @@ static void addResumeFunction(ModuleOp module) {
     return;
 
   MLIRContext *ctx = module.getContext();
-
-  OpBuilder moduleBuilder(module.getBody()->getTerminator());
-  Location loc = module.getLoc();
+  auto loc = module.getLoc();
+  auto moduleBuilder = ImplicitLocOpBuilder::atBlockEnd(loc, module.getBody());
 
   auto voidTy = LLVM::LLVMVoidType::get(ctx);
   auto i8Ptr = LLVM::LLVMPointerType::get(IntegerType::get(ctx, 8));
 
   auto resumeOp = moduleBuilder.create<LLVM::LLVMFuncOp>(
-      loc, kResume, LLVM::LLVMFunctionType::get(voidTy, {i8Ptr}));
+      kResume, LLVM::LLVMFunctionType::get(voidTy, {i8Ptr}));
   resumeOp.setPrivate();
 
   auto *block = resumeOp.addEntryBlock();
@@ -875,7 +874,7 @@ void ConvertAsyncToLLVMPass::runOnOperation() {
 
   // Convert async dialect types and operations to LLVM dialect.
   AsyncRuntimeTypeConverter converter;
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(ctx);
 
   // We use conversion to LLVM type to lower async.runtime load and store
   // operations.
@@ -883,28 +882,28 @@ void ConvertAsyncToLLVMPass::runOnOperation() {
   llvmConverter.addConversion(AsyncRuntimeTypeConverter::convertAsyncTypes);
 
   // Convert async types in function signatures and function calls.
-  populateFuncOpTypeConversionPattern(patterns, ctx, converter);
-  populateCallOpTypeConversionPattern(patterns, ctx, converter);
+  populateFuncOpTypeConversionPattern(patterns, converter);
+  populateCallOpTypeConversionPattern(patterns, converter);
 
   // Convert return operations inside async.execute regions.
-  patterns.insert<ReturnOpOpConversion>(converter, ctx);
+  patterns.add<ReturnOpOpConversion>(converter, ctx);
 
   // Lower async.runtime operations to the async runtime API calls.
-  patterns.insert<RuntimeSetAvailableOpLowering, RuntimeAwaitOpLowering,
-                  RuntimeAwaitAndResumeOpLowering, RuntimeResumeOpLowering,
-                  RuntimeAddToGroupOpLowering, RuntimeAddRefOpLowering,
-                  RuntimeDropRefOpLowering>(converter, ctx);
+  patterns.add<RuntimeSetAvailableOpLowering, RuntimeAwaitOpLowering,
+               RuntimeAwaitAndResumeOpLowering, RuntimeResumeOpLowering,
+               RuntimeAddToGroupOpLowering, RuntimeAddRefOpLowering,
+               RuntimeDropRefOpLowering>(converter, ctx);
 
   // Lower async.runtime operations that rely on LLVM type converter to convert
   // from async value payload type to the LLVM type.
-  patterns.insert<RuntimeCreateOpLowering, RuntimeStoreOpLowering,
-                  RuntimeLoadOpLowering>(llvmConverter, ctx);
+  patterns.add<RuntimeCreateOpLowering, RuntimeStoreOpLowering,
+               RuntimeLoadOpLowering>(llvmConverter, ctx);
 
   // Lower async coroutine operations to LLVM coroutine intrinsics.
-  patterns.insert<CoroIdOpConversion, CoroBeginOpConversion,
-                  CoroFreeOpConversion, CoroEndOpConversion,
-                  CoroSaveOpConversion, CoroSuspendOpConversion>(converter,
-                                                                 ctx);
+  patterns
+      .add<CoroIdOpConversion, CoroBeginOpConversion, CoroFreeOpConversion,
+           CoroEndOpConversion, CoroSaveOpConversion, CoroSuspendOpConversion>(
+          converter, ctx);
 
   ConversionTarget target(*ctx);
   target.addLegalOp<ConstantOp>();
@@ -985,16 +984,15 @@ std::unique_ptr<OperationPass<ModuleOp>> mlir::createConvertAsyncToLLVMPass() {
 }
 
 void mlir::populateAsyncStructuralTypeConversionsAndLegality(
-    MLIRContext *context, TypeConverter &typeConverter,
-    OwningRewritePatternList &patterns, ConversionTarget &target) {
+    TypeConverter &typeConverter, RewritePatternSet &patterns,
+    ConversionTarget &target) {
   typeConverter.addConversion([&](TokenType type) { return type; });
   typeConverter.addConversion([&](ValueType type) {
     return ValueType::get(typeConverter.convertType(type.getValueType()));
   });
 
-  patterns
-      .insert<ConvertExecuteOpTypes, ConvertAwaitOpTypes, ConvertYieldOpTypes>(
-          typeConverter, context);
+  patterns.add<ConvertExecuteOpTypes, ConvertAwaitOpTypes, ConvertYieldOpTypes>(
+      typeConverter, patterns.getContext());
 
   target.addDynamicallyLegalOp<AwaitOp, ExecuteOp, async::YieldOp>(
       [&](Operation *op) { return typeConverter.isLegal(op); });
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index 00ab63790fd40ea341cfcac4092001bda3a158a5..d5fb64a8eb650f466a2840ba6874620a74506d88 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -258,9 +258,9 @@ struct SubOpConversion : public ConvertOpToLLVMPattern<complex::SubOp> {
 } // namespace
 
 void mlir::populateComplexToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+    LLVMTypeConverter &converter, RewritePatternSet &patterns) {
   // clang-format off
-  patterns.insert<
+  patterns.add<
       AbsOpConversion,
       AddOpConversion,
       CreateOpConversion,
@@ -284,7 +284,7 @@ void ConvertComplexToLLVMPass::runOnOperation() {
   auto module = getOperation();
 
   // Convert to the LLVM IR dialect using the converter defined above.
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(&getContext());
   LLVMTypeConverter converter(&getContext());
   populateComplexToLLVMConversionPatterns(converter, patterns);
 
diff --git a/mlir/lib/Conversion/GPUCommon/CMakeLists.txt b/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
index 825bed600abadb3273748d18886584db3a68a85b..65ad9de0b20cad9e0506b2373f92d9e98e361745 100644
--- a/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
@@ -15,8 +15,7 @@ if (MLIR_ROCM_CONVERSIONS_ENABLED)
 endif()
 
 add_mlir_conversion_library(MLIRGPUToGPURuntimeTransforms
-  ConvertLaunchFuncToRuntimeCalls.cpp
-  ConvertKernelFuncToBlob.cpp
+  GPUToLLVMConversion.cpp
   GPUOpsLowering.cpp
 
   DEPENDS
@@ -37,4 +36,5 @@ add_mlir_conversion_library(MLIRGPUToGPURuntimeTransforms
   MLIRPass
   MLIRSupport
   MLIRStandardToLLVM
+  MLIRVectorToLLVM
 )
diff --git a/mlir/lib/Conversion/GPUCommon/ConvertKernelFuncToBlob.cpp b/mlir/lib/Conversion/GPUCommon/ConvertKernelFuncToBlob.cpp
deleted file mode 100644
index e8f9a7a469367db872278786f3dfcd44879ad9c9..0000000000000000000000000000000000000000
--- a/mlir/lib/Conversion/GPUCommon/ConvertKernelFuncToBlob.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-//===- ConvertKernelFuncToBlob.cpp - MLIR GPU lowering passes -------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements a pass to convert gpu kernel functions into a
-// corresponding binary blob that can be executed on a GPU. Currently
-// only translates the function itself but no dependencies.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
-
-#include "mlir/Dialect/GPU/GPUDialect.h"
-#include "mlir/Dialect/GPU/Passes.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/IR/Attributes.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Pass/PassRegistry.h"
-#include "mlir/Support/LogicalResult.h"
-
-#include "llvm/ADT/Optional.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/Mutex.h"
-#include "llvm/Support/TargetRegistry.h"
-#include "llvm/Support/TargetSelect.h"
-
-using namespace mlir;
-
-namespace {
-
-/// A pass converting tagged kernel modules to a blob with target instructions.
-///
-/// If tagged as a kernel module, each contained function is translated to
-/// user-specified IR. A user provided BlobGenerator then compiles the IR to
-/// GPU binary code, which is then attached as an attribute to the function.
-/// The function body is erased.
-class GpuKernelToBlobPass
-    : public PassWrapper<GpuKernelToBlobPass, gpu::SerializeToBlobPass> {
-public:
-  GpuKernelToBlobPass(LoweringCallback loweringCallback,
-                      BlobGenerator blobGenerator, StringRef triple,
-                      StringRef targetChip, StringRef features,
-                      StringRef gpuBinaryAnnotation)
-      : loweringCallback(loweringCallback), blobGenerator(blobGenerator) {
-    if (!triple.empty())
-      this->triple = triple.str();
-    if (!targetChip.empty())
-      this->chip = targetChip.str();
-    if (!features.empty())
-      this->features = features.str();
-    if (!gpuBinaryAnnotation.empty())
-      this->gpuBinaryAnnotation = gpuBinaryAnnotation.str();
-  }
-
-private:
-  // Translates the 'getOperation()' result to an LLVM module.
-  // Note: when this class is removed, this function no longer needs to be
-  // virtual.
-  std::unique_ptr<llvm::Module>
-  translateToLLVMIR(llvm::LLVMContext &llvmContext) override {
-    return loweringCallback(getOperation(), llvmContext, "LLVMDialectModule");
-  }
-
-  // Serializes the target ISA to binary form.
-  std::unique_ptr<std::vector<char>>
-  serializeISA(const std::string &isa) override {
-    return blobGenerator(isa, getOperation().getLoc(),
-                         getOperation().getName());
-  }
-
-  LoweringCallback loweringCallback;
-  BlobGenerator blobGenerator;
-};
-
-} // anonymous namespace
-
-std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
-mlir::createConvertGPUKernelToBlobPass(LoweringCallback loweringCallback,
-                                       BlobGenerator blobGenerator,
-                                       StringRef triple, StringRef targetChip,
-                                       StringRef features,
-                                       StringRef gpuBinaryAnnotation) {
-  return std::make_unique<GpuKernelToBlobPass>(loweringCallback, blobGenerator,
-                                               triple, targetChip, features,
-                                               gpuBinaryAnnotation);
-}
diff --git a/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
similarity index 95%
rename from mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp
rename to mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 44dfd730e44fe6d0251536051822362335070a5c..60a47a9d7befcd71d6c553c449d8795481129d0f 100644
--- a/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -18,6 +18,7 @@
 #include "../PassDetail.h"
 #include "mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h"
 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
+#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
 #include "mlir/Dialect/Async/IR/Async.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/GPU/Passes.h"
@@ -40,10 +41,7 @@ namespace {
 class GpuToLLVMConversionPass
     : public GpuToLLVMConversionPassBase<GpuToLLVMConversionPass> {
 public:
-  GpuToLLVMConversionPass(StringRef gpuBinaryAnnotation) {
-    if (!gpuBinaryAnnotation.empty())
-      this->gpuBinaryAnnotation = gpuBinaryAnnotation.str();
-  }
+  GpuToLLVMConversionPass() = default;
 
   GpuToLLVMConversionPass(const GpuToLLVMConversionPass &other)
       : GpuToLLVMConversionPassBase(other) {}
@@ -310,13 +308,28 @@ private:
 
 void GpuToLLVMConversionPass::runOnOperation() {
   LLVMTypeConverter converter(&getContext());
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(&getContext());
   LLVMConversionTarget target(getContext());
 
+  populateVectorToLLVMConversionPatterns(converter, patterns);
   populateStdToLLVMConversionPatterns(converter, patterns);
-  populateAsyncStructuralTypeConversionsAndLegality(&getContext(), converter,
-                                                    patterns, target);
-  populateGpuToLLVMConversionPatterns(converter, patterns, gpuBinaryAnnotation);
+  populateAsyncStructuralTypeConversionsAndLegality(converter, patterns,
+                                                    target);
+
+  converter.addConversion(
+      [context = &converter.getContext()](gpu::AsyncTokenType type) -> Type {
+        return LLVM::LLVMPointerType::get(IntegerType::get(context, 8));
+      });
+  patterns.add<ConvertAllocOpToGpuRuntimeCallPattern,
+               ConvertDeallocOpToGpuRuntimeCallPattern,
+               ConvertHostRegisterOpToGpuRuntimeCallPattern,
+               ConvertMemcpyOpToGpuRuntimeCallPattern,
+               ConvertWaitAsyncOpToGpuRuntimeCallPattern,
+               ConvertWaitOpToGpuRuntimeCallPattern,
+               ConvertAsyncYieldToGpuRuntimeCallPattern>(converter);
+  patterns.add<ConvertLaunchFuncOpToGpuRuntimeCallPattern>(converter,
+                                                           gpuBinaryAnnotation);
+  patterns.add<EraseGpuModuleOpPattern>(&converter.getContext());
 
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
@@ -329,7 +342,7 @@ LLVM::CallOp FunctionCallBuilder::create(Location loc, OpBuilder &builder,
   auto function = [&] {
     if (auto function = module.lookupSymbol<LLVM::LLVMFuncOp>(functionName))
       return function;
-    return OpBuilder(module.getBody()->getTerminator())
+    return OpBuilder::atBlockEnd(module.getBody())
         .create<LLVM::LLVMFuncOp>(loc, functionName, functionType);
   }();
   return builder.create<LLVM::CallOp>(
@@ -782,25 +795,6 @@ LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite(
 }
 
 std::unique_ptr<mlir::OperationPass<mlir::ModuleOp>>
-mlir::createGpuToLLVMConversionPass(StringRef gpuBinaryAnnotation) {
-  return std::make_unique<GpuToLLVMConversionPass>(gpuBinaryAnnotation);
-}
-
-void mlir::populateGpuToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    StringRef gpuBinaryAnnotation) {
-  converter.addConversion(
-      [context = &converter.getContext()](gpu::AsyncTokenType type) -> Type {
-        return LLVM::LLVMPointerType::get(IntegerType::get(context, 8));
-      });
-  patterns.insert<ConvertAllocOpToGpuRuntimeCallPattern,
-                  ConvertDeallocOpToGpuRuntimeCallPattern,
-                  ConvertHostRegisterOpToGpuRuntimeCallPattern,
-                  ConvertMemcpyOpToGpuRuntimeCallPattern,
-                  ConvertWaitAsyncOpToGpuRuntimeCallPattern,
-                  ConvertWaitOpToGpuRuntimeCallPattern,
-                  ConvertAsyncYieldToGpuRuntimeCallPattern>(converter);
-  patterns.insert<ConvertLaunchFuncOpToGpuRuntimeCallPattern>(
-      converter, gpuBinaryAnnotation);
-  patterns.insert<EraseGpuModuleOpPattern>(&converter.getContext());
+mlir::createGpuToLLVMConversionPass() {
+  return std::make_unique<GpuToLLVMConversionPass>();
 }
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 9e167125904a3a2f0ba3261fd367f9ad7e9817f9..e96934088f613000bc426f18686cbdb60acce0af 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -108,10 +108,12 @@ struct LowerGpuOpsToNVVMOpsPass
     gpu::GPUModuleOp m = getOperation();
 
     /// Customize the bitwidth used for the device side index computations.
-    LowerToLLVMOptions options = {/*useBarePtrCallConv =*/false,
-                                  /*emitCWrappers =*/true,
-                                  /*indexBitwidth =*/indexBitwidth,
-                                  /*useAlignedAlloc =*/false};
+    LowerToLLVMOptions options(
+        m.getContext(),
+        DataLayout(cast<DataLayoutOpInterface>(m.getOperation())));
+    options.emitCWrappers = true;
+    if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
+      options.overrideIndexBitwidth(indexBitwidth);
 
     /// MemRef conversion for GPU to NVVM lowering. The GPU dialect uses memory
     /// space 5 for private memory attributions, but NVVM represents private
@@ -125,12 +127,13 @@ struct LowerGpuOpsToNVVMOpsPass
       return converter.convertType(MemRefType::Builder(type).setMemorySpace(0));
     });
 
-    OwningRewritePatternList patterns, llvmPatterns;
+    RewritePatternSet patterns(m.getContext());
+    RewritePatternSet llvmPatterns(m.getContext());
 
     // Apply in-dialect lowering first. In-dialect lowering will replace ops
     // which need to be lowered further, which is not supported by a single
     // conversion pass.
-    populateGpuRewritePatterns(m.getContext(), patterns);
+    populateGpuRewritePatterns(patterns);
     (void)applyPatternsAndFoldGreedily(m, std::move(patterns));
 
     populateStdToLLVMConversionPatterns(converter, llvmPatterns);
@@ -157,62 +160,62 @@ void mlir::configureGpuToNVVMConversionLegality(ConversionTarget &target) {
   target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
 }
 
-void mlir::populateGpuToNVVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  populateWithGenerated(converter.getDialect()->getContext(), patterns);
+void mlir::populateGpuToNVVMConversionPatterns(LLVMTypeConverter &converter,
+                                               RewritePatternSet &patterns) {
+  populateWithGenerated(patterns);
   patterns
-      .insert<GPUIndexIntrinsicOpLowering<gpu::ThreadIdOp, NVVM::ThreadIdXOp,
-                                          NVVM::ThreadIdYOp, NVVM::ThreadIdZOp>,
-              GPUIndexIntrinsicOpLowering<gpu::BlockDimOp, NVVM::BlockDimXOp,
-                                          NVVM::BlockDimYOp, NVVM::BlockDimZOp>,
-              GPUIndexIntrinsicOpLowering<gpu::BlockIdOp, NVVM::BlockIdXOp,
-                                          NVVM::BlockIdYOp, NVVM::BlockIdZOp>,
-              GPUIndexIntrinsicOpLowering<gpu::GridDimOp, NVVM::GridDimXOp,
-                                          NVVM::GridDimYOp, NVVM::GridDimZOp>,
-              GPUShuffleOpLowering, GPUReturnOpLowering>(converter);
+      .add<GPUIndexIntrinsicOpLowering<gpu::ThreadIdOp, NVVM::ThreadIdXOp,
+                                       NVVM::ThreadIdYOp, NVVM::ThreadIdZOp>,
+           GPUIndexIntrinsicOpLowering<gpu::BlockDimOp, NVVM::BlockDimXOp,
+                                       NVVM::BlockDimYOp, NVVM::BlockDimZOp>,
+           GPUIndexIntrinsicOpLowering<gpu::BlockIdOp, NVVM::BlockIdXOp,
+                                       NVVM::BlockIdYOp, NVVM::BlockIdZOp>,
+           GPUIndexIntrinsicOpLowering<gpu::GridDimOp, NVVM::GridDimXOp,
+                                       NVVM::GridDimYOp, NVVM::GridDimZOp>,
+           GPUShuffleOpLowering, GPUReturnOpLowering>(converter);
 
   // Explicitly drop memory space when lowering private memory
   // attributions since NVVM models it as `alloca`s in the default
   // memory space and does not support `alloca`s with addrspace(5).
-  patterns.insert<GPUFuncOpLowering>(
+  patterns.add<GPUFuncOpLowering>(
       converter, /*allocaAddrSpace=*/0,
       Identifier::get(NVVM::NVVMDialect::getKernelFuncAttrName(),
                       &converter.getContext()));
 
-  patterns.insert<OpToFuncCallLowering<AbsFOp>>(converter, "__nv_fabsf",
-                                                "__nv_fabs");
-  patterns.insert<OpToFuncCallLowering<math::AtanOp>>(converter, "__nv_atanf",
-                                                      "__nv_atan");
-  patterns.insert<OpToFuncCallLowering<math::Atan2Op>>(converter, "__nv_atan2f",
-                                                       "__nv_atan2");
-  patterns.insert<OpToFuncCallLowering<CeilFOp>>(converter, "__nv_ceilf",
-                                                 "__nv_ceil");
-  patterns.insert<OpToFuncCallLowering<math::CosOp>>(converter, "__nv_cosf",
-                                                     "__nv_cos");
-  patterns.insert<OpToFuncCallLowering<math::ExpOp>>(converter, "__nv_expf",
-                                                     "__nv_exp");
-  patterns.insert<OpToFuncCallLowering<math::ExpM1Op>>(converter, "__nv_expm1f",
-                                                       "__nv_expm1");
-  patterns.insert<OpToFuncCallLowering<FloorFOp>>(converter, "__nv_floorf",
-                                                  "__nv_floor");
-  patterns.insert<OpToFuncCallLowering<math::LogOp>>(converter, "__nv_logf",
-                                                     "__nv_log");
-  patterns.insert<OpToFuncCallLowering<math::Log1pOp>>(converter, "__nv_log1pf",
-                                                       "__nv_log1p");
-  patterns.insert<OpToFuncCallLowering<math::Log10Op>>(converter, "__nv_log10f",
-                                                       "__nv_log10");
-  patterns.insert<OpToFuncCallLowering<math::Log2Op>>(converter, "__nv_log2f",
-                                                      "__nv_log2");
-  patterns.insert<OpToFuncCallLowering<math::PowFOp>>(converter, "__nv_powf",
-                                                      "__nv_pow");
-  patterns.insert<OpToFuncCallLowering<math::RsqrtOp>>(converter, "__nv_rsqrtf",
-                                                       "__nv_rsqrt");
-  patterns.insert<OpToFuncCallLowering<math::SinOp>>(converter, "__nv_sinf",
-                                                     "__nv_sin");
-  patterns.insert<OpToFuncCallLowering<math::SqrtOp>>(converter, "__nv_sqrtf",
-                                                      "__nv_sqrt");
-  patterns.insert<OpToFuncCallLowering<math::TanhOp>>(converter, "__nv_tanhf",
-                                                      "__nv_tanh");
+  patterns.add<OpToFuncCallLowering<AbsFOp>>(converter, "__nv_fabsf",
+                                             "__nv_fabs");
+  patterns.add<OpToFuncCallLowering<math::AtanOp>>(converter, "__nv_atanf",
+                                                   "__nv_atan");
+  patterns.add<OpToFuncCallLowering<math::Atan2Op>>(converter, "__nv_atan2f",
+                                                    "__nv_atan2");
+  patterns.add<OpToFuncCallLowering<CeilFOp>>(converter, "__nv_ceilf",
+                                              "__nv_ceil");
+  patterns.add<OpToFuncCallLowering<math::CosOp>>(converter, "__nv_cosf",
+                                                  "__nv_cos");
+  patterns.add<OpToFuncCallLowering<math::ExpOp>>(converter, "__nv_expf",
+                                                  "__nv_exp");
+  patterns.add<OpToFuncCallLowering<math::ExpM1Op>>(converter, "__nv_expm1f",
+                                                    "__nv_expm1");
+  patterns.add<OpToFuncCallLowering<FloorFOp>>(converter, "__nv_floorf",
+                                               "__nv_floor");
+  patterns.add<OpToFuncCallLowering<math::LogOp>>(converter, "__nv_logf",
+                                                  "__nv_log");
+  patterns.add<OpToFuncCallLowering<math::Log1pOp>>(converter, "__nv_log1pf",
+                                                    "__nv_log1p");
+  patterns.add<OpToFuncCallLowering<math::Log10Op>>(converter, "__nv_log10f",
+                                                    "__nv_log10");
+  patterns.add<OpToFuncCallLowering<math::Log2Op>>(converter, "__nv_log2f",
+                                                   "__nv_log2");
+  patterns.add<OpToFuncCallLowering<math::PowFOp>>(converter, "__nv_powf",
+                                                   "__nv_pow");
+  patterns.add<OpToFuncCallLowering<math::RsqrtOp>>(converter, "__nv_rsqrtf",
+                                                    "__nv_rsqrt");
+  patterns.add<OpToFuncCallLowering<math::SinOp>>(converter, "__nv_sinf",
+                                                  "__nv_sin");
+  patterns.add<OpToFuncCallLowering<math::SqrtOp>>(converter, "__nv_sqrtf",
+                                                   "__nv_sqrt");
+  patterns.add<OpToFuncCallLowering<math::TanhOp>>(converter, "__nv_tanhf",
+                                                   "__nv_tanh");
 }
 
 std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index d61c04702b5adbde3bc6670d4b116c3e3b93c593..27ad870691bfbf9af4c79e85cb9b737c2998b913 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -54,15 +54,18 @@ struct LowerGpuOpsToROCDLOpsPass
     gpu::GPUModuleOp m = getOperation();
 
     /// Customize the bitwidth used for the device side index computations.
-    LowerToLLVMOptions options = {/*useBarePtrCallConv =*/false,
-                                  /*emitCWrappers =*/true,
-                                  /*indexBitwidth =*/indexBitwidth,
-                                  /*useAlignedAlloc =*/false};
+    LowerToLLVMOptions options(
+        m.getContext(),
+        DataLayout(cast<DataLayoutOpInterface>(m.getOperation())));
+    options.emitCWrappers = true;
+    if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
+      options.overrideIndexBitwidth(indexBitwidth);
     LLVMTypeConverter converter(m.getContext(), options);
 
-    OwningRewritePatternList patterns, llvmPatterns;
+    RewritePatternSet patterns(m.getContext());
+    RewritePatternSet llvmPatterns(m.getContext());
 
-    populateGpuRewritePatterns(m.getContext(), patterns);
+    populateGpuRewritePatterns(patterns);
     (void)applyPatternsAndFoldGreedily(m, std::move(patterns));
 
     populateVectorToLLVMConversionPatterns(converter, llvmPatterns);
@@ -91,57 +94,57 @@ void mlir::configureGpuToROCDLConversionLegality(ConversionTarget &target) {
   target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
 }
 
-void mlir::populateGpuToROCDLConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  populateWithGenerated(converter.getDialect()->getContext(), patterns);
-  patterns.insert<
-      GPUIndexIntrinsicOpLowering<gpu::ThreadIdOp, ROCDL::ThreadIdXOp,
-                                  ROCDL::ThreadIdYOp, ROCDL::ThreadIdZOp>,
-      GPUIndexIntrinsicOpLowering<gpu::BlockDimOp, ROCDL::BlockDimXOp,
-                                  ROCDL::BlockDimYOp, ROCDL::BlockDimZOp>,
-      GPUIndexIntrinsicOpLowering<gpu::BlockIdOp, ROCDL::BlockIdXOp,
-                                  ROCDL::BlockIdYOp, ROCDL::BlockIdZOp>,
-      GPUIndexIntrinsicOpLowering<gpu::GridDimOp, ROCDL::GridDimXOp,
-                                  ROCDL::GridDimYOp, ROCDL::GridDimZOp>,
-      GPUReturnOpLowering>(converter);
-  patterns.insert<GPUFuncOpLowering>(
+void mlir::populateGpuToROCDLConversionPatterns(LLVMTypeConverter &converter,
+                                                RewritePatternSet &patterns) {
+  populateWithGenerated(patterns);
+  patterns
+      .add<GPUIndexIntrinsicOpLowering<gpu::ThreadIdOp, ROCDL::ThreadIdXOp,
+                                       ROCDL::ThreadIdYOp, ROCDL::ThreadIdZOp>,
+           GPUIndexIntrinsicOpLowering<gpu::BlockDimOp, ROCDL::BlockDimXOp,
+                                       ROCDL::BlockDimYOp, ROCDL::BlockDimZOp>,
+           GPUIndexIntrinsicOpLowering<gpu::BlockIdOp, ROCDL::BlockIdXOp,
+                                       ROCDL::BlockIdYOp, ROCDL::BlockIdZOp>,
+           GPUIndexIntrinsicOpLowering<gpu::GridDimOp, ROCDL::GridDimXOp,
+                                       ROCDL::GridDimYOp, ROCDL::GridDimZOp>,
+           GPUReturnOpLowering>(converter);
+  patterns.add<GPUFuncOpLowering>(
       converter, /*allocaAddrSpace=*/5,
       Identifier::get(ROCDL::ROCDLDialect::getKernelFuncAttrName(),
                       &converter.getContext()));
-  patterns.insert<OpToFuncCallLowering<AbsFOp>>(converter, "__ocml_fabs_f32",
-                                                "__ocml_fabs_f64");
-  patterns.insert<OpToFuncCallLowering<math::AtanOp>>(
-      converter, "__ocml_atan_f32", "__ocml_atan_f64");
-  patterns.insert<OpToFuncCallLowering<math::Atan2Op>>(
+  patterns.add<OpToFuncCallLowering<AbsFOp>>(converter, "__ocml_fabs_f32",
+                                             "__ocml_fabs_f64");
+  patterns.add<OpToFuncCallLowering<math::AtanOp>>(converter, "__ocml_atan_f32",
+                                                   "__ocml_atan_f64");
+  patterns.add<OpToFuncCallLowering<math::Atan2Op>>(
       converter, "__ocml_atan2_f32", "__ocml_atan2_f64");
-  patterns.insert<OpToFuncCallLowering<CeilFOp>>(converter, "__ocml_ceil_f32",
-                                                 "__ocml_ceil_f64");
-  patterns.insert<OpToFuncCallLowering<math::CosOp>>(
-      converter, "__ocml_cos_f32", "__ocml_cos_f64");
-  patterns.insert<OpToFuncCallLowering<math::ExpOp>>(
-      converter, "__ocml_exp_f32", "__ocml_exp_f64");
-  patterns.insert<OpToFuncCallLowering<math::ExpM1Op>>(
+  patterns.add<OpToFuncCallLowering<CeilFOp>>(converter, "__ocml_ceil_f32",
+                                              "__ocml_ceil_f64");
+  patterns.add<OpToFuncCallLowering<math::CosOp>>(converter, "__ocml_cos_f32",
+                                                  "__ocml_cos_f64");
+  patterns.add<OpToFuncCallLowering<math::ExpOp>>(converter, "__ocml_exp_f32",
+                                                  "__ocml_exp_f64");
+  patterns.add<OpToFuncCallLowering<math::ExpM1Op>>(
       converter, "__ocml_expm1_f32", "__ocml_expm1_f64");
-  patterns.insert<OpToFuncCallLowering<FloorFOp>>(converter, "__ocml_floor_f32",
-                                                  "__ocml_floor_f64");
-  patterns.insert<OpToFuncCallLowering<math::LogOp>>(
-      converter, "__ocml_log_f32", "__ocml_log_f64");
-  patterns.insert<OpToFuncCallLowering<math::Log10Op>>(
+  patterns.add<OpToFuncCallLowering<FloorFOp>>(converter, "__ocml_floor_f32",
+                                               "__ocml_floor_f64");
+  patterns.add<OpToFuncCallLowering<math::LogOp>>(converter, "__ocml_log_f32",
+                                                  "__ocml_log_f64");
+  patterns.add<OpToFuncCallLowering<math::Log10Op>>(
       converter, "__ocml_log10_f32", "__ocml_log10_f64");
-  patterns.insert<OpToFuncCallLowering<math::Log1pOp>>(
+  patterns.add<OpToFuncCallLowering<math::Log1pOp>>(
       converter, "__ocml_log1p_f32", "__ocml_log1p_f64");
-  patterns.insert<OpToFuncCallLowering<math::Log2Op>>(
-      converter, "__ocml_log2_f32", "__ocml_log2_f64");
-  patterns.insert<OpToFuncCallLowering<math::PowFOp>>(
-      converter, "__ocml_pow_f32", "__ocml_pow_f64");
-  patterns.insert<OpToFuncCallLowering<math::RsqrtOp>>(
+  patterns.add<OpToFuncCallLowering<math::Log2Op>>(converter, "__ocml_log2_f32",
+                                                   "__ocml_log2_f64");
+  patterns.add<OpToFuncCallLowering<math::PowFOp>>(converter, "__ocml_pow_f32",
+                                                   "__ocml_pow_f64");
+  patterns.add<OpToFuncCallLowering<math::RsqrtOp>>(
       converter, "__ocml_rsqrt_f32", "__ocml_rsqrt_f64");
-  patterns.insert<OpToFuncCallLowering<math::SinOp>>(
-      converter, "__ocml_sin_f32", "__ocml_sin_f64");
-  patterns.insert<OpToFuncCallLowering<math::SqrtOp>>(
-      converter, "__ocml_sqrt_f32", "__ocml_sqrt_f64");
-  patterns.insert<OpToFuncCallLowering<math::TanhOp>>(
-      converter, "__ocml_tanh_f32", "__ocml_tanh_f64");
+  patterns.add<OpToFuncCallLowering<math::SinOp>>(converter, "__ocml_sin_f32",
+                                                  "__ocml_sin_f64");
+  patterns.add<OpToFuncCallLowering<math::SqrtOp>>(converter, "__ocml_sqrt_f32",
+                                                   "__ocml_sqrt_f64");
+  patterns.add<OpToFuncCallLowering<math::TanhOp>>(converter, "__ocml_tanh_f32",
+                                                   "__ocml_tanh_f64");
 }
 
 std::unique_ptr<OperationPass<gpu::GPUModuleOp>>
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index 1e0a7663435b4b52ab090aabe2382218761996ff..c2cd4baea631c834e11f88e74e3c0d2d0a8e91d7 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -329,11 +329,10 @@ namespace {
 #include "GPUToSPIRV.cpp.inc"
 }
 
-void mlir::populateGPUToSPIRVPatterns(MLIRContext *context,
-                                      SPIRVTypeConverter &typeConverter,
-                                      OwningRewritePatternList &patterns) {
-  populateWithGenerated(context, patterns);
-  patterns.insert<
+void mlir::populateGPUToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                      RewritePatternSet &patterns) {
+  populateWithGenerated(patterns);
+  patterns.add<
       GPUFuncOpConversion, GPUModuleConversion, GPUReturnOpConversion,
       LaunchConfigConversion<gpu::BlockIdOp, spirv::BuiltIn::WorkgroupId>,
       LaunchConfigConversion<gpu::GridDimOp, spirv::BuiltIn::NumWorkgroups>,
@@ -345,5 +344,5 @@ void mlir::populateGPUToSPIRVPatterns(MLIRContext *context,
                                       spirv::BuiltIn::NumSubgroups>,
       SingleDimLaunchConfigConversion<gpu::SubgroupSizeOp,
                                       spirv::BuiltIn::SubgroupSize>,
-      WorkGroupSizeConversion>(typeConverter, context);
+      WorkGroupSizeConversion>(typeConverter, patterns.getContext());
 }
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
index 8edb42e9a0f63a22c8b1d22118ab4241fe64bb40..1f23f7ce380efc63fd8594f9cf0566f2a00dea3a 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
@@ -57,9 +57,9 @@ void GPUToSPIRVPass::runOnOperation() {
       spirv::SPIRVConversionTarget::get(targetAttr);
 
   SPIRVTypeConverter typeConverter(targetAttr);
-  OwningRewritePatternList patterns;
-  populateGPUToSPIRVPatterns(context, typeConverter, patterns);
-  populateStandardToSPIRVPatterns(context, typeConverter, patterns);
+  RewritePatternSet patterns(context);
+  populateGPUToSPIRVPatterns(typeConverter, patterns);
+  populateStandardToSPIRVPatterns(typeConverter, patterns);
 
   if (failed(applyFullConversion(kernelModules, *target, std::move(patterns))))
     return signalPassFailure();
diff --git a/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp b/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
index 5b62ca455dea305b67c8399759d60856a40e6f1b..ba65865b6095e968558b968ee6c6a22f6e161be1 100644
--- a/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
+++ b/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
@@ -99,7 +99,7 @@ void ConvertGpuLaunchFuncToVulkanLaunchFunc::runOnOperation() {
 
 LogicalResult ConvertGpuLaunchFuncToVulkanLaunchFunc::declareVulkanLaunchFunc(
     Location loc, gpu::LaunchFuncOp launchOp) {
-  OpBuilder builder(getOperation().getBody()->getTerminator());
+  auto builder = OpBuilder::atBlockEnd(getOperation().getBody());
 
   // Workgroup size is written into the kernel. So to properly modelling
   // vulkan launch, we have to skip local workgroup size configuration here.
diff --git a/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp b/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
index 47968ea458ce9fee5fd2e8019399c866df244ee5..118941539e0c2ba036f3c0ff2a2a8698115a3b0e 100644
--- a/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
+++ b/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
@@ -291,7 +291,7 @@ LogicalResult VulkanLaunchFuncToVulkanCallsPass::deduceMemRefRankAndType(
 
 void VulkanLaunchFuncToVulkanCallsPass::declareVulkanFunctions(Location loc) {
   ModuleOp module = getOperation();
-  OpBuilder builder(module.getBody()->getTerminator());
+  auto builder = OpBuilder::atBlockEnd(module.getBody());
 
   if (!module.lookupSymbol(kSetEntryPoint)) {
     builder.create<LLVM::LLVMFuncOp>(
diff --git a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
index 5c0eb5e371358c7a13955fc156d604cd139170fb..2eccac860bf24d28697370ddced0d8453366d72d 100644
--- a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
+++ b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
@@ -200,9 +200,9 @@ public:
 } // namespace
 
 /// Populate the given list with patterns that convert from Linalg to LLVM.
-void mlir::populateLinalgToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  patterns.insert<RangeOpConversion, ReshapeOpConversion, YieldOpConversion>(
+void mlir::populateLinalgToLLVMConversionPatterns(LLVMTypeConverter &converter,
+                                                  RewritePatternSet &patterns) {
+  patterns.add<RangeOpConversion, ReshapeOpConversion, YieldOpConversion>(
       converter);
 
   // Populate the type conversions for the linalg types.
@@ -221,13 +221,13 @@ void ConvertLinalgToLLVMPass::runOnOperation() {
   auto module = getOperation();
 
   // Convert to the LLVM IR dialect using the converter defined above.
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(&getContext());
   LLVMTypeConverter converter(&getContext());
   populateLinalgToLLVMConversionPatterns(converter, patterns);
 
   LLVMConversionTarget target(getContext());
   target.addIllegalOp<RangeOp>();
-  target.addLegalOp<ModuleOp, ModuleTerminatorOp, LLVM::DialectCastOp>();
+  target.addLegalOp<ModuleOp, LLVM::DialectCastOp>();
   if (failed(applyPartialConversion(module, target, std::move(patterns))))
     signalPassFailure();
 }
diff --git a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp
index 0db760b17d7c80a5f3a386db89ee7de469a73918..a94435c0435846c82462214230824ebc351ac100 100644
--- a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp
+++ b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp
@@ -203,8 +203,7 @@ LogicalResult SingleWorkgroupReduction::matchAndRewrite(
 // Pattern population
 //===----------------------------------------------------------------------===//
 
-void mlir::populateLinalgToSPIRVPatterns(MLIRContext *context,
-                                         SPIRVTypeConverter &typeConverter,
-                                         OwningRewritePatternList &patterns) {
-  patterns.insert<SingleWorkgroupReduction>(typeConverter, context);
+void mlir::populateLinalgToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                         RewritePatternSet &patterns) {
+  patterns.add<SingleWorkgroupReduction>(typeConverter, patterns.getContext());
 }
diff --git a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.cpp b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.cpp
index ddcc97d359ca9001d49ea93faf50e0ff58d96356..6038a9841cf95dc84d9eb9a4ea4fcf7e507bd6f5 100644
--- a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.cpp
@@ -30,12 +30,12 @@ void LinalgToSPIRVPass::runOnOperation() {
       spirv::SPIRVConversionTarget::get(targetAttr);
 
   SPIRVTypeConverter typeConverter(targetAttr);
-  OwningRewritePatternList patterns;
-  populateLinalgToSPIRVPatterns(context, typeConverter, patterns);
-  populateBuiltinFuncToSPIRVPatterns(context, typeConverter, patterns);
+  RewritePatternSet patterns(context);
+  populateLinalgToSPIRVPatterns(typeConverter, patterns);
+  populateBuiltinFuncToSPIRVPatterns(typeConverter, patterns);
 
   // Allow builtin ops.
-  target->addLegalOp<ModuleOp, ModuleTerminatorOp>();
+  target->addLegalOp<ModuleOp>();
   target->addDynamicallyLegalOp<FuncOp>([&](FuncOp op) {
     return typeConverter.isSignatureLegal(op.getType()) &&
            typeConverter.isLegal(&op.getBody());
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index bf947a4ed08847ee8eeeec26162045d9af137410..5260c503cce92149708eb349168749f6f60996ed 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -101,9 +101,9 @@ createTypeCanonicalizedMemRefOperands(OpBuilder &b, Location loc,
 }
 
 LogicalResult mlir::linalg::LinalgOpToLibraryCallRewrite::matchAndRewrite(
-    Operation *op, PatternRewriter &rewriter) const {
+    LinalgOp op, PatternRewriter &rewriter) const {
   // Only LinalgOp for which there is no specialized pattern go through this.
-  if (!isa<LinalgOp>(op) || isa<CopyOp>(op) || isa<IndexedGenericOp>(op))
+  if (isa<CopyOp>(op) || isa<IndexedGenericOp>(op))
     return failure();
 
   auto libraryCallName = getLibraryCallSymbolRef(op, rewriter);
@@ -192,15 +192,15 @@ mlir::linalg::IndexedGenericOpToLibraryCallRewrite::matchAndRewrite(
 
 /// Populate the given list with patterns that convert from Linalg to Standard.
 void mlir::linalg::populateLinalgToStandardConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx) {
+    RewritePatternSet &patterns) {
   // TODO: ConvOp conversion needs to export a descriptor with relevant
   // attribute values such as kernel striding and dilation.
   // clang-format off
-  patterns.insert<
+  patterns.add<
       CopyOpToLibraryCallRewrite,
       CopyTransposeRewrite,
-      IndexedGenericOpToLibraryCallRewrite>(ctx);
-  patterns.insert<LinalgOpToLibraryCallRewrite>();
+      IndexedGenericOpToLibraryCallRewrite,
+      LinalgOpToLibraryCallRewrite>(patterns.getContext());
   // clang-format on
 }
 
@@ -216,10 +216,10 @@ void ConvertLinalgToStandardPass::runOnOperation() {
   ConversionTarget target(getContext());
   target.addLegalDialect<AffineDialect, memref::MemRefDialect, scf::SCFDialect,
                          StandardOpsDialect>();
-  target.addLegalOp<ModuleOp, FuncOp, ModuleTerminatorOp, ReturnOp>();
+  target.addLegalOp<ModuleOp, FuncOp, ReturnOp>();
   target.addLegalOp<linalg::ReshapeOp, linalg::RangeOp>();
-  OwningRewritePatternList patterns;
-  populateLinalgToStandardConversionPatterns(patterns, &getContext());
+  RewritePatternSet patterns(&getContext());
+  populateLinalgToStandardConversionPatterns(patterns);
   if (failed(applyFullConversion(module, target, std::move(patterns))))
     signalPassFailure();
 }
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 7bc510068aefe90ee297731c6e886d108462fc6d..878e11ae6c5aad82f009a0097bb3c08f2692d48a 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -41,10 +41,10 @@ struct RegionOpConversion : public ConvertOpToLLVMPattern<OpType> {
 };
 } // namespace
 
-void mlir::populateOpenMPToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  patterns.insert<RegionOpConversion<omp::ParallelOp>,
-                  RegionOpConversion<omp::WsLoopOp>>(converter);
+void mlir::populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter,
+                                                  RewritePatternSet &patterns) {
+  patterns.add<RegionOpConversion<omp::ParallelOp>,
+               RegionOpConversion<omp::WsLoopOp>>(converter);
 }
 
 namespace {
@@ -58,7 +58,7 @@ void ConvertOpenMPToLLVMPass::runOnOperation() {
   auto module = getOperation();
 
   // Convert to OpenMP operations with LLVM IR dialect
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(&getContext());
   LLVMTypeConverter converter(&getContext());
   populateStdToLLVMConversionPatterns(converter, patterns);
   populateOpenMPToLLVMConversionPatterns(converter, patterns);
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index a225699e89f7940b999bc84bc186151265277fcf..57a0885c03c8791f630d73cfec8b2b92be20ed62 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -56,9 +56,8 @@ private:
 
   /// Create an interpreter switch predicate operation, with a provided default
   /// and several case destinations.
-  void generateSwitch(Block *currentBlock, Qualifier *question, Value val,
-                      Block *defaultDest,
-                      ArrayRef<std::pair<Qualifier *, Block *>> dests);
+  void generateSwitch(SwitchNode *switchNode, Block *currentBlock,
+                      Qualifier *question, Value val, Block *defaultDest);
 
   /// Create the interpreter operations to record a successful pattern match.
   void generateRecordMatch(Block *currentBlock, Block *nextBlock,
@@ -70,6 +69,9 @@ private:
                                  SmallVectorImpl<Position *> &usedMatchValues);
 
   /// Generate the rewriter code for the given operation.
+  void generateRewriter(pdl::ApplyNativeRewriteOp rewriteOp,
+                        DenseMap<Value, Value> &rewriteValues,
+                        function_ref<Value(Value)> mapRewriteValue);
   void generateRewriter(pdl::AttributeOp attrOp,
                         DenseMap<Value, Value> &rewriteValues,
                         function_ref<Value(Value)> mapRewriteValue);
@@ -79,15 +81,21 @@ private:
   void generateRewriter(pdl::OperationOp operationOp,
                         DenseMap<Value, Value> &rewriteValues,
                         function_ref<Value(Value)> mapRewriteValue);
-  void generateRewriter(pdl::CreateNativeOp createNativeOp,
+  void generateRewriter(pdl::ReplaceOp replaceOp,
+                        DenseMap<Value, Value> &rewriteValues,
+                        function_ref<Value(Value)> mapRewriteValue);
+  void generateRewriter(pdl::ResultOp resultOp,
                         DenseMap<Value, Value> &rewriteValues,
                         function_ref<Value(Value)> mapRewriteValue);
-  void generateRewriter(pdl::ReplaceOp replaceOp,
+  void generateRewriter(pdl::ResultsOp resultOp,
                         DenseMap<Value, Value> &rewriteValues,
                         function_ref<Value(Value)> mapRewriteValue);
   void generateRewriter(pdl::TypeOp typeOp,
                         DenseMap<Value, Value> &rewriteValues,
                         function_ref<Value(Value)> mapRewriteValue);
+  void generateRewriter(pdl::TypesOp typeOp,
+                        DenseMap<Value, Value> &rewriteValues,
+                        function_ref<Value(Value)> mapRewriteValue);
 
   /// Generate the values used for resolving the result types of an operation
   /// created within a dag rewriter region.
@@ -197,12 +205,7 @@ Block *PatternLowering::generateMatcher(MatcherNode &node) {
 
     // Generate code for a switch node.
   } else if (auto *switchNode = dyn_cast<SwitchNode>(&node)) {
-    // Collect the next blocks for all of the children and generate a switch.
-    llvm::MapVector<Qualifier *, Block *> children;
-    for (auto &it : switchNode->getChildren())
-      children.insert({it.first, generateMatcher(*it.second)});
-    generateSwitch(block, node.getQuestion(), val, nextBlock,
-                   children.takeVector());
+    generateSwitch(switchNode, block, node.getQuestion(), val, nextBlock);
 
     // Generate code for a success node.
   } else if (auto *successNode = dyn_cast<SuccessNode>(&node)) {
@@ -239,6 +242,14 @@ Value PatternLowering::getValueAt(Block *cur, Position *pos) {
         operandPos->getOperandNumber());
     break;
   }
+  case Predicates::OperandGroupPos: {
+    auto *operandPos = cast<OperandGroupPosition>(pos);
+    Type valueTy = builder.getType<pdl::ValueType>();
+    value = builder.create<pdl_interp::GetOperandsOp>(
+        loc, operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
+        parentVal, operandPos->getOperandGroupNumber());
+    break;
+  }
   case Predicates::AttributePos: {
     auto *attrPos = cast<AttributePosition>(pos);
     value = builder.create<pdl_interp::GetAttributeOp>(
@@ -247,10 +258,10 @@ Value PatternLowering::getValueAt(Block *cur, Position *pos) {
     break;
   }
   case Predicates::TypePos: {
-    if (parentVal.getType().isa<pdl::ValueType>())
-      value = builder.create<pdl_interp::GetValueTypeOp>(loc, parentVal);
-    else
+    if (parentVal.getType().isa<pdl::AttributeType>())
       value = builder.create<pdl_interp::GetAttributeTypeOp>(loc, parentVal);
+    else
+      value = builder.create<pdl_interp::GetValueTypeOp>(loc, parentVal);
     break;
   }
   case Predicates::ResultPos: {
@@ -260,6 +271,14 @@ Value PatternLowering::getValueAt(Block *cur, Position *pos) {
         resPos->getResultNumber());
     break;
   }
+  case Predicates::ResultGroupPos: {
+    auto *resPos = cast<ResultGroupPosition>(pos);
+    Type valueTy = builder.getType<pdl::ValueType>();
+    value = builder.create<pdl_interp::GetResultsOp>(
+        loc, resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy,
+        parentVal, resPos->getResultGroupNumber());
+    break;
+  }
   default:
     llvm_unreachable("Generating unknown Position getter");
     break;
@@ -274,7 +293,8 @@ void PatternLowering::generatePredicate(Block *currentBlock,
                                         Block *falseDest) {
   builder.setInsertionPointToEnd(currentBlock);
   Location loc = val.getLoc();
-  switch (question->getKind()) {
+  Predicates::Kind kind = question->getKind();
+  switch (kind) {
   case Predicates::IsNotNullQuestion:
     builder.create<pdl_interp::IsNotNullOp>(loc, val, trueDest, falseDest);
     break;
@@ -286,8 +306,12 @@ void PatternLowering::generatePredicate(Block *currentBlock,
   }
   case Predicates::TypeQuestion: {
     auto *ans = cast<TypeAnswer>(answer);
-    builder.create<pdl_interp::CheckTypeOp>(
-        loc, val, TypeAttr::get(ans->getValue()), trueDest, falseDest);
+    if (val.getType().isa<pdl::RangeType>())
+      builder.create<pdl_interp::CheckTypesOp>(
+          loc, val, ans->getValue().cast<ArrayAttr>(), trueDest, falseDest);
+    else
+      builder.create<pdl_interp::CheckTypeOp>(
+          loc, val, ans->getValue().cast<TypeAttr>(), trueDest, falseDest);
     break;
   }
   case Predicates::AttributeQuestion: {
@@ -296,18 +320,20 @@ void PatternLowering::generatePredicate(Block *currentBlock,
                                                  trueDest, falseDest);
     break;
   }
-  case Predicates::OperandCountQuestion: {
-    auto *unsignedAnswer = cast<UnsignedAnswer>(answer);
+  case Predicates::OperandCountAtLeastQuestion:
+  case Predicates::OperandCountQuestion:
     builder.create<pdl_interp::CheckOperandCountOp>(
-        loc, val, unsignedAnswer->getValue(), trueDest, falseDest);
+        loc, val, cast<UnsignedAnswer>(answer)->getValue(),
+        /*compareAtLeast=*/kind == Predicates::OperandCountAtLeastQuestion,
+        trueDest, falseDest);
     break;
-  }
-  case Predicates::ResultCountQuestion: {
-    auto *unsignedAnswer = cast<UnsignedAnswer>(answer);
+  case Predicates::ResultCountAtLeastQuestion:
+  case Predicates::ResultCountQuestion:
     builder.create<pdl_interp::CheckResultCountOp>(
-        loc, val, unsignedAnswer->getValue(), trueDest, falseDest);
+        loc, val, cast<UnsignedAnswer>(answer)->getValue(),
+        /*compareAtLeast=*/kind == Predicates::ResultCountAtLeastQuestion,
+        trueDest, falseDest);
     break;
-  }
   case Predicates::EqualToQuestion: {
     auto *equalToQuestion = cast<EqualToQuestion>(question);
     builder.create<pdl_interp::AreEqualOp>(
@@ -333,7 +359,7 @@ void PatternLowering::generatePredicate(Block *currentBlock,
 
 template <typename OpT, typename PredT, typename ValT = typename PredT::KeyTy>
 static void createSwitchOp(Value val, Block *defaultDest, OpBuilder &builder,
-                           ArrayRef<std::pair<Qualifier *, Block *>> dests) {
+                           llvm::MapVector<Qualifier *, Block *> &dests) {
   std::vector<ValT> values;
   std::vector<Block *> blocks;
   values.reserve(dests.size());
@@ -345,27 +371,83 @@ static void createSwitchOp(Value val, Block *defaultDest, OpBuilder &builder,
   builder.create<OpT>(val.getLoc(), val, values, defaultDest, blocks);
 }
 
-void PatternLowering::generateSwitch(
-    Block *currentBlock, Qualifier *question, Value val, Block *defaultDest,
-    ArrayRef<std::pair<Qualifier *, Block *>> dests) {
+void PatternLowering::generateSwitch(SwitchNode *switchNode,
+                                     Block *currentBlock, Qualifier *question,
+                                     Value val, Block *defaultDest) {
+  // If the switch question is not an exact answer, i.e. for the `at_least`
+  // cases, we generate a special block sequence.
+  Predicates::Kind kind = question->getKind();
+  if (kind == Predicates::OperandCountAtLeastQuestion ||
+      kind == Predicates::ResultCountAtLeastQuestion) {
+    // Order the children such that the cases are in reverse numerical order.
+    SmallVector<unsigned> sortedChildren(
+        llvm::seq<unsigned>(0, switchNode->getChildren().size()));
+    llvm::sort(sortedChildren, [&](unsigned lhs, unsigned rhs) {
+      return cast<UnsignedAnswer>(switchNode->getChild(lhs).first)->getValue() >
+             cast<UnsignedAnswer>(switchNode->getChild(rhs).first)->getValue();
+    });
+
+    // Build the destination for each child using the next highest child as a
+    // a failure destination. This essentially creates the following control
+    // flow:
+    //
+    // if (operand_count < 1)
+    //   goto failure
+    // if (child1.match())
+    //   ...
+    //
+    // if (operand_count < 2)
+    //   goto failure
+    // if (child2.match())
+    //   ...
+    //
+    // failure:
+    //   ...
+    //
+    failureBlockStack.push_back(defaultDest);
+    for (unsigned idx : sortedChildren) {
+      auto &child = switchNode->getChild(idx);
+      Block *childBlock = generateMatcher(*child.second);
+      Block *predicateBlock = builder.createBlock(childBlock);
+      generatePredicate(predicateBlock, question, child.first, val, childBlock,
+                        defaultDest);
+      failureBlockStack.back() = predicateBlock;
+    }
+    Block *firstPredicateBlock = failureBlockStack.pop_back_val();
+    currentBlock->getOperations().splice(currentBlock->end(),
+                                         firstPredicateBlock->getOperations());
+    firstPredicateBlock->erase();
+    return;
+  }
+
+  // Otherwise, generate each of the children and generate an interpreter
+  // switch.
+  llvm::MapVector<Qualifier *, Block *> children;
+  for (auto &it : switchNode->getChildren())
+    children.insert({it.first, generateMatcher(*it.second)});
   builder.setInsertionPointToEnd(currentBlock);
+
   switch (question->getKind()) {
   case Predicates::OperandCountQuestion:
     return createSwitchOp<pdl_interp::SwitchOperandCountOp, UnsignedAnswer,
-                          int32_t>(val, defaultDest, builder, dests);
+                          int32_t>(val, defaultDest, builder, children);
   case Predicates::ResultCountQuestion:
     return createSwitchOp<pdl_interp::SwitchResultCountOp, UnsignedAnswer,
-                          int32_t>(val, defaultDest, builder, dests);
+                          int32_t>(val, defaultDest, builder, children);
   case Predicates::OperationNameQuestion:
     return createSwitchOp<pdl_interp::SwitchOperationNameOp,
                           OperationNameAnswer>(val, defaultDest, builder,
-                                               dests);
+                                               children);
   case Predicates::TypeQuestion:
+    if (val.getType().isa<pdl::RangeType>()) {
+      return createSwitchOp<pdl_interp::SwitchTypesOp, TypeAnswer>(
+          val, defaultDest, builder, children);
+    }
     return createSwitchOp<pdl_interp::SwitchTypeOp, TypeAnswer>(
-        val, defaultDest, builder, dests);
+        val, defaultDest, builder, children);
   case Predicates::AttributeQuestion:
     return createSwitchOp<pdl_interp::SwitchAttributeOp, AttributeAnswer>(
-        val, defaultDest, builder, dests);
+        val, defaultDest, builder, children);
   default:
     llvm_unreachable("Generating unknown switch predicate.");
   }
@@ -433,6 +515,11 @@ SymbolRefAttr PatternLowering::generateRewriter(
         return newValue = builder.create<pdl_interp::CreateTypeOp>(
                    typeOp.getLoc(), type);
       }
+    } else if (pdl::TypesOp typeOp = dyn_cast<pdl::TypesOp>(oldOp)) {
+      if (ArrayAttr type = typeOp.typesAttr()) {
+        return newValue = builder.create<pdl_interp::CreateTypesOp>(
+                   typeOp.getLoc(), typeOp.getType(), type);
+      }
     }
 
     // Otherwise, add this as an input to the rewriter.
@@ -446,18 +533,19 @@ SymbolRefAttr PatternLowering::generateRewriter(
   // method.
   pdl::RewriteOp rewriter = pattern.getRewriter();
   if (StringAttr rewriteName = rewriter.nameAttr()) {
-    Value root = mapRewriteValue(rewriter.root());
-    SmallVector<Value, 4> args = llvm::to_vector<4>(
-        llvm::map_range(rewriter.externalArgs(), mapRewriteValue));
+    auto mappedArgs = llvm::map_range(rewriter.externalArgs(), mapRewriteValue);
+    SmallVector<Value, 4> args(1, mapRewriteValue(rewriter.root()));
+    args.append(mappedArgs.begin(), mappedArgs.end());
     builder.create<pdl_interp::ApplyRewriteOp>(
-        rewriter.getLoc(), rewriteName, root, args,
+        rewriter.getLoc(), /*resultTypes=*/TypeRange(), rewriteName, args,
         rewriter.externalConstParamsAttr());
   } else {
     // Otherwise this is a dag rewriter defined using PDL operations.
     for (Operation &rewriteOp : *rewriter.getBody()) {
       llvm::TypeSwitch<Operation *>(&rewriteOp)
-          .Case<pdl::AttributeOp, pdl::CreateNativeOp, pdl::EraseOp,
-                pdl::OperationOp, pdl::ReplaceOp, pdl::TypeOp>([&](auto op) {
+          .Case<pdl::ApplyNativeRewriteOp, pdl::AttributeOp, pdl::EraseOp,
+                pdl::OperationOp, pdl::ReplaceOp, pdl::ResultOp, pdl::ResultsOp,
+                pdl::TypeOp, pdl::TypesOp>([&](auto op) {
             this->generateRewriter(op, rewriteValues, mapRewriteValue);
           });
     }
@@ -474,6 +562,19 @@ SymbolRefAttr PatternLowering::generateRewriter(
       builder.getSymbolRefAttr(rewriterFunc));
 }
 
+void PatternLowering::generateRewriter(
+    pdl::ApplyNativeRewriteOp rewriteOp, DenseMap<Value, Value> &rewriteValues,
+    function_ref<Value(Value)> mapRewriteValue) {
+  SmallVector<Value, 2> arguments;
+  for (Value argument : rewriteOp.args())
+    arguments.push_back(mapRewriteValue(argument));
+  auto interpOp = builder.create<pdl_interp::ApplyRewriteOp>(
+      rewriteOp.getLoc(), rewriteOp.getResultTypes(), rewriteOp.nameAttr(),
+      arguments, rewriteOp.constParamsAttr());
+  for (auto it : llvm::zip(rewriteOp.results(), interpOp.results()))
+    rewriteValues[std::get<0>(it)] = std::get<1>(it);
+}
+
 void PatternLowering::generateRewriter(
     pdl::AttributeOp attrOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
@@ -511,67 +612,109 @@ void PatternLowering::generateRewriter(
       operationOp.attributeNames());
   rewriteValues[operationOp.op()] = createdOp;
 
-  // Make all of the new operation results available.
-  OperandRange resultTypes = operationOp.types();
-  for (auto it : llvm::enumerate(operationOp.results())) {
-    Value getResultVal = builder.create<pdl_interp::GetResultOp>(
-        loc, builder.getType<pdl::ValueType>(), createdOp, it.index());
-    rewriteValues[it.value()] = getResultVal;
-
-    // If any of the types have not been resolved, make those available as well.
-    Value &type = rewriteValues[resultTypes[it.index()]];
-    if (!type)
-      type = builder.create<pdl_interp::GetValueTypeOp>(loc, getResultVal);
+  // Generate accesses for any results that have their types constrained.
+  // Handle the case where there is a single range representing all of the
+  // result types.
+  OperandRange resultTys = operationOp.types();
+  if (resultTys.size() == 1 && resultTys[0].getType().isa<pdl::RangeType>()) {
+    Value &type = rewriteValues[resultTys[0]];
+    if (!type) {
+      auto results = builder.create<pdl_interp::GetResultsOp>(loc, createdOp);
+      type = builder.create<pdl_interp::GetValueTypeOp>(loc, results);
+    }
+    return;
   }
-}
 
-void PatternLowering::generateRewriter(
-    pdl::CreateNativeOp createNativeOp, DenseMap<Value, Value> &rewriteValues,
-    function_ref<Value(Value)> mapRewriteValue) {
-  SmallVector<Value, 2> arguments;
-  for (Value argument : createNativeOp.args())
-    arguments.push_back(mapRewriteValue(argument));
-  Value result = builder.create<pdl_interp::CreateNativeOp>(
-      createNativeOp.getLoc(), createNativeOp.result().getType(),
-      createNativeOp.nameAttr(), arguments, createNativeOp.constParamsAttr());
-  rewriteValues[createNativeOp] = result;
+  // Otherwise, populate the individual results.
+  bool seenVariableLength = false;
+  Type valueTy = builder.getType<pdl::ValueType>();
+  Type valueRangeTy = pdl::RangeType::get(valueTy);
+  for (auto it : llvm::enumerate(resultTys)) {
+    Value &type = rewriteValues[it.value()];
+    if (type)
+      continue;
+    bool isVariadic = it.value().getType().isa<pdl::RangeType>();
+    seenVariableLength |= isVariadic;
+
+    // After a variable length result has been seen, we need to use result
+    // groups because the exact index of the result is not statically known.
+    Value resultVal;
+    if (seenVariableLength)
+      resultVal = builder.create<pdl_interp::GetResultsOp>(
+          loc, isVariadic ? valueRangeTy : valueTy, createdOp, it.index());
+    else
+      resultVal = builder.create<pdl_interp::GetResultOp>(
+          loc, valueTy, createdOp, it.index());
+    type = builder.create<pdl_interp::GetValueTypeOp>(loc, resultVal);
+  }
 }
 
 void PatternLowering::generateRewriter(
     pdl::ReplaceOp replaceOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
+  SmallVector<Value, 4> replOperands;
+
   // If the replacement was another operation, get its results. `pdl` allows
   // for using an operation for simplicitly, but the interpreter isn't as
   // user facing.
-  ValueRange origOperands;
-  if (Value replOp = replaceOp.replOperation())
-    origOperands = cast<pdl::OperationOp>(replOp.getDefiningOp()).results();
-  else
-    origOperands = replaceOp.replValues();
+  if (Value replOp = replaceOp.replOperation()) {
+    // Don't use replace if we know the replaced operation has no results.
+    auto opOp = replaceOp.operation().getDefiningOp<pdl::OperationOp>();
+    if (!opOp || !opOp.types().empty()) {
+      replOperands.push_back(builder.create<pdl_interp::GetResultsOp>(
+          replOp.getLoc(), mapRewriteValue(replOp)));
+    }
+  } else {
+    for (Value operand : replaceOp.replValues())
+      replOperands.push_back(mapRewriteValue(operand));
+  }
 
   // If there are no replacement values, just create an erase instead.
-  if (origOperands.empty()) {
+  if (replOperands.empty()) {
     builder.create<pdl_interp::EraseOp>(replaceOp.getLoc(),
                                         mapRewriteValue(replaceOp.operation()));
     return;
   }
 
-  SmallVector<Value, 4> replOperands;
-  for (Value operand : origOperands)
-    replOperands.push_back(mapRewriteValue(operand));
   builder.create<pdl_interp::ReplaceOp>(
       replaceOp.getLoc(), mapRewriteValue(replaceOp.operation()), replOperands);
 }
 
+void PatternLowering::generateRewriter(
+    pdl::ResultOp resultOp, DenseMap<Value, Value> &rewriteValues,
+    function_ref<Value(Value)> mapRewriteValue) {
+  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultOp>(
+      resultOp.getLoc(), builder.getType<pdl::ValueType>(),
+      mapRewriteValue(resultOp.parent()), resultOp.index());
+}
+
+void PatternLowering::generateRewriter(
+    pdl::ResultsOp resultOp, DenseMap<Value, Value> &rewriteValues,
+    function_ref<Value(Value)> mapRewriteValue) {
+  rewriteValues[resultOp] = builder.create<pdl_interp::GetResultsOp>(
+      resultOp.getLoc(), resultOp.getType(), mapRewriteValue(resultOp.parent()),
+      resultOp.index());
+}
+
 void PatternLowering::generateRewriter(
     pdl::TypeOp typeOp, DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
   // If the type isn't constant, the users (e.g. OperationOp) will resolve this
   // type.
   if (TypeAttr typeAttr = typeOp.typeAttr()) {
-    Value newType =
+    rewriteValues[typeOp] =
         builder.create<pdl_interp::CreateTypeOp>(typeOp.getLoc(), typeAttr);
-    rewriteValues[typeOp] = newType;
+  }
+}
+
+void PatternLowering::generateRewriter(
+    pdl::TypesOp typeOp, DenseMap<Value, Value> &rewriteValues,
+    function_ref<Value(Value)> mapRewriteValue) {
+  // If the type isn't constant, the users (e.g. OperationOp) will resolve this
+  // type.
+  if (ArrayAttr typeAttr = typeOp.typesAttr()) {
+    rewriteValues[typeOp] = builder.create<pdl_interp::CreateTypesOp>(
+        typeOp.getLoc(), typeOp.getType(), typeAttr);
   }
 }
 
@@ -579,31 +722,41 @@ void PatternLowering::generateOperationResultTypeRewriter(
     pdl::OperationOp op, SmallVectorImpl<Value> &types,
     DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
-  // Functor that returns if the given use can be used to infer a type.
+  // Look for an operation that was replaced by `op`. The result types will be
+  // inferred from the results that were replaced.
   Block *rewriterBlock = op->getBlock();
-  auto getReplacedOperationFrom = [&](OpOperand &use) -> Operation * {
+  Value replacedOp;
+  for (OpOperand &use : op.op().getUses()) {
     // Check that the use corresponds to a ReplaceOp and that it is the
     // replacement value, not the operation being replaced.
     pdl::ReplaceOp replOpUser = dyn_cast<pdl::ReplaceOp>(use.getOwner());
     if (!replOpUser || use.getOperandNumber() == 0)
-      return nullptr;
+      continue;
     // Make sure the replaced operation was defined before this one.
-    Operation *replacedOp = replOpUser.operation().getDefiningOp();
-    if (replacedOp->getBlock() != rewriterBlock ||
-        replacedOp->isBeforeInBlock(op))
-      return replacedOp;
-    return nullptr;
-  };
+    Value replOpVal = replOpUser.operation();
+    Operation *replacedOp = replOpVal.getDefiningOp();
+    if (replacedOp->getBlock() == rewriterBlock &&
+        !replacedOp->isBeforeInBlock(op))
+      continue;
+
+    Value replacedOpResults = builder.create<pdl_interp::GetResultsOp>(
+        replacedOp->getLoc(), mapRewriteValue(replOpVal));
+    types.push_back(builder.create<pdl_interp::GetValueTypeOp>(
+        replacedOp->getLoc(), replacedOpResults));
+    return;
+  }
 
-  // If non-None/non-Null, this is an operation that is replaced by `op`.
-  // If Null, there is no full replacement operation for `op`.
-  // If None, a replacement operation hasn't been searched for.
-  Optional<Operation *> fullReplacedOperation;
-  bool hasTypeInference = op.hasTypeInference();
-  auto resultTypeValues = op.types();
+  // Check if the operation has type inference support.
+  if (op.hasTypeInference()) {
+    types.push_back(builder.create<pdl_interp::InferredTypesOp>(op.getLoc()));
+    return;
+  }
+
+  // Otherwise, handle inference for each of the result types individually.
+  OperandRange resultTypeValues = op.types();
   types.reserve(resultTypeValues.size());
-  for (auto it : llvm::enumerate(op.results())) {
-    Value result = it.value(), resultType = resultTypeValues[it.index()];
+  for (auto it : llvm::enumerate(resultTypeValues)) {
+    Value resultType = it.value();
 
     // Check for an already translated value.
     if (Value existingRewriteValue = rewriteValues.lookup(resultType)) {
@@ -617,35 +770,11 @@ void PatternLowering::generateOperationResultTypeRewriter(
       continue;
     }
 
-    // Check if the operation has type inference support.
-    if (hasTypeInference) {
-      types.push_back(builder.create<pdl_interp::InferredTypeOp>(op.getLoc()));
-      continue;
-    }
-
-    // Look for an operation that was replaced by `op`. The result type will be
-    // inferred from the result that was replaced. There is guaranteed to be a
-    // replacement for either the op, or this specific result. Note that this is
-    // guaranteed by the verifier of `pdl::OperationOp`.
-    Operation *replacedOp = nullptr;
-    if (!fullReplacedOperation.hasValue()) {
-      for (OpOperand &use : op.op().getUses())
-        if ((replacedOp = getReplacedOperationFrom(use)))
-          break;
-      fullReplacedOperation = replacedOp;
-    } else {
-      replacedOp = fullReplacedOperation.getValue();
-    }
-    // Infer from the result, as there was no fully replaced op.
-    if (!replacedOp) {
-      for (OpOperand &use : result.getUses())
-        if ((replacedOp = getReplacedOperationFrom(use)))
-          break;
-      assert(replacedOp && "expected replaced op to infer a result type from");
-    }
-
-    auto replOpOp = cast<pdl::OperationOp>(replacedOp);
-    types.push_back(mapRewriteValue(replOpOp.types()[it.index()]));
+    // The verifier asserts that the result types of each pdl.operation can be
+    // inferred. If we reach here, there is a bug either in the logic above or
+    // in the verifier for pdl.operation.
+    op->emitOpError() << "unable to infer result type for operation";
+    llvm_unreachable("unable to infer result type for operation");
   }
 }
 
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.cpp b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.cpp
index 3eaeb13cffc08a409d37e6c2318caab04fc33d50..8983ecb8d324c7c8eac5e4816e54478b009c7b36 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.cpp
@@ -17,6 +17,13 @@ using namespace mlir::pdl_to_pdl_interp;
 
 Position::~Position() {}
 
+/// Returns the depth of the first ancestor operation position.
+unsigned Position::getOperationDepth() const {
+  if (const auto *operationPos = dyn_cast<OperationPosition>(this))
+    return operationPos->getDepth();
+  return parent->getOperationDepth();
+}
+
 //===----------------------------------------------------------------------===//
 // AttributePosition
 
@@ -32,18 +39,8 @@ OperandPosition::OperandPosition(const KeyTy &key) : Base(key) {
 }
 
 //===----------------------------------------------------------------------===//
-// OperationPosition
-
-OperationPosition *OperationPosition::get(StorageUniquer &uniquer,
-                                          ArrayRef<unsigned> index) {
-  assert(!index.empty() && "expected at least two indices");
-
-  // Set the parent position if this isn't the root.
-  Position *parent = nullptr;
-  if (index.size() > 1) {
-    auto *node = OperationPosition::get(uniquer, index.drop_back());
-    parent = OperandPosition::get(uniquer, std::make_pair(node, index.back()));
-  }
-  return uniquer.get<OperationPosition>(
-      [parent](OperationPosition *node) { node->parent = parent; }, index);
+// OperandGroupPosition
+
+OperandGroupPosition::OperandGroupPosition(const KeyTy &key) : Base(key) {
+  parent = std::get<0>(key);
 }
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h
index b3919609a6405cfe33c3f29efaa65526d1e0f722..1c8fece05e07190580a691de0ca638ee0cac9774 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h
+++ b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h
@@ -45,8 +45,10 @@ enum Kind : unsigned {
   /// Positions, ordered by decreasing priority.
   OperationPos,
   OperandPos,
+  OperandGroupPos,
   AttributePos,
   ResultPos,
+  ResultGroupPos,
   TypePos,
 
   // Questions, ordered by dependency and decreasing priority.
@@ -54,7 +56,9 @@ enum Kind : unsigned {
   OperationNameQuestion,
   TypeQuestion,
   AttributeQuestion,
+  OperandCountAtLeastQuestion,
   OperandCountQuestion,
+  ResultCountAtLeastQuestion,
   ResultCountQuestion,
   EqualToQuestion,
   ConstraintQuestion,
@@ -129,21 +133,15 @@ struct OperationPosition;
 /// predicates, and assists generating bytecode and memory management.
 ///
 /// Operation positions form the base of other positions, which are formed
-/// relative to a parent operation, e.g. OperandPosition<[0] -> 1>. Operations
-/// are indexed by child index: [0, 1, 2] refers to the 3rd child of the 2nd
-/// child of the root operation.
-///
-/// Positions are linked to their parent position, which describes how to obtain
-/// a positional value. As a concrete example, getting OperationPosition<[0, 1]>
-/// would be `root->getOperand(1)->getDefiningOp()`, so its parent is
-/// OperandPosition<[0] -> 1>, whose parent is OperationPosition<[0]>.
+/// relative to a parent operation. Operations are anchored at Operand nodes,
+/// except for the root operation which is parentless.
 class Position : public StorageUniquer::BaseStorage {
 public:
   explicit Position(Predicates::Kind kind) : kind(kind) {}
   virtual ~Position();
 
-  /// Returns the base node position. This is an array of indices.
-  virtual ArrayRef<unsigned> getIndex() const = 0;
+  /// Returns the depth of the first ancestor operation position.
+  unsigned getOperationDepth() const;
 
   /// Returns the parent position. The root operation position has no parent.
   Position *getParent() const { return parent; }
@@ -170,9 +168,6 @@ struct AttributePosition
                            Predicates::AttributePos> {
   explicit AttributePosition(const KeyTy &key);
 
-  /// Returns the index of this position.
-  ArrayRef<unsigned> getIndex() const final { return parent->getIndex(); }
-
   /// Returns the attribute name of this position.
   Identifier getName() const { return key.second; }
 };
@@ -187,42 +182,61 @@ struct OperandPosition
                            Predicates::OperandPos> {
   explicit OperandPosition(const KeyTy &key);
 
-  /// Returns the index of this position.
-  ArrayRef<unsigned> getIndex() const final { return parent->getIndex(); }
-
   /// Returns the operand number of this position.
   unsigned getOperandNumber() const { return key.second; }
 };
 
+//===----------------------------------------------------------------------===//
+// OperandGroupPosition
+
+/// A position describing an operand group of an operation.
+struct OperandGroupPosition
+    : public PredicateBase<
+          OperandGroupPosition, Position,
+          std::tuple<OperationPosition *, Optional<unsigned>, bool>,
+          Predicates::OperandGroupPos> {
+  explicit OperandGroupPosition(const KeyTy &key);
+
+  /// Returns a hash suitable for the given keytype.
+  static llvm::hash_code hashKey(const KeyTy &key) {
+    return llvm::hash_value(key);
+  }
+
+  /// Returns the group number of this position. If None, this group refers to
+  /// all operands.
+  Optional<unsigned> getOperandGroupNumber() const { return std::get<1>(key); }
+
+  /// Returns if the operand group has unknown size. If false, the operand group
+  /// has at max one element.
+  bool isVariadic() const { return std::get<2>(key); }
+};
+
 //===----------------------------------------------------------------------===//
 // OperationPosition
 
 /// An operation position describes an operation node in the IR. Other position
 /// kinds are formed with respect to an operation position.
-struct OperationPosition
-    : public PredicateBase<OperationPosition, Position, ArrayRef<unsigned>,
-                           Predicates::OperationPos> {
-  using Base::Base;
+struct OperationPosition : public PredicateBase<OperationPosition, Position,
+                                                std::pair<Position *, unsigned>,
+                                                Predicates::OperationPos> {
+  explicit OperationPosition(const KeyTy &key) : Base(key) {
+    parent = key.first;
+  }
 
-  /// Gets the root position, which is always [0].
+  /// Gets the root position.
   static OperationPosition *getRoot(StorageUniquer &uniquer) {
-    return get(uniquer, ArrayRef<unsigned>(0));
+    return Base::get(uniquer, nullptr, 0);
   }
-  /// Gets a node position for the given index.
-  static OperationPosition *get(StorageUniquer &uniquer,
-                                ArrayRef<unsigned> index);
-
-  /// Constructs an instance with the given storage allocator.
-  static OperationPosition *construct(StorageUniquer::StorageAllocator &alloc,
-                                      ArrayRef<unsigned> key) {
-    return Base::construct(alloc, alloc.copyInto(key));
+  /// Gets an operation position with the given parent.
+  static OperationPosition *get(StorageUniquer &uniquer, Position *parent) {
+    return Base::get(uniquer, parent, parent->getOperationDepth() + 1);
   }
 
-  /// Returns the index of this position.
-  ArrayRef<unsigned> getIndex() const final { return key; }
+  /// Returns the depth of this position.
+  unsigned getDepth() const { return key.second; }
 
   /// Returns if this operation position corresponds to the root.
-  bool isRoot() const { return key.size() == 1 && key[0] == 0; }
+  bool isRoot() const { return getDepth() == 0; }
 };
 
 //===----------------------------------------------------------------------===//
@@ -235,13 +249,37 @@ struct ResultPosition
                            Predicates::ResultPos> {
   explicit ResultPosition(const KeyTy &key) : Base(key) { parent = key.first; }
 
-  /// Returns the index of this position.
-  ArrayRef<unsigned> getIndex() const final { return key.first->getIndex(); }
-
   /// Returns the result number of this position.
   unsigned getResultNumber() const { return key.second; }
 };
 
+//===----------------------------------------------------------------------===//
+// ResultGroupPosition
+
+/// A position describing a result group of an operation.
+struct ResultGroupPosition
+    : public PredicateBase<
+          ResultGroupPosition, Position,
+          std::tuple<OperationPosition *, Optional<unsigned>, bool>,
+          Predicates::ResultGroupPos> {
+  explicit ResultGroupPosition(const KeyTy &key) : Base(key) {
+    parent = std::get<0>(key);
+  }
+
+  /// Returns a hash suitable for the given keytype.
+  static llvm::hash_code hashKey(const KeyTy &key) {
+    return llvm::hash_value(key);
+  }
+
+  /// Returns the group number of this position. If None, this group refers to
+  /// all results.
+  Optional<unsigned> getResultGroupNumber() const { return std::get<1>(key); }
+
+  /// Returns if the result group has unknown size. If false, the result group
+  /// has at max one element.
+  bool isVariadic() const { return std::get<2>(key); }
+};
+
 //===----------------------------------------------------------------------===//
 // TypePosition
 
@@ -250,14 +288,11 @@ struct ResultPosition
 struct TypePosition : public PredicateBase<TypePosition, Position, Position *,
                                            Predicates::TypePos> {
   explicit TypePosition(const KeyTy &key) : Base(key) {
-    assert((isa<AttributePosition>(key) || isa<OperandPosition>(key) ||
-            isa<ResultPosition>(key)) &&
+    assert((isa<AttributePosition, OperandPosition, OperandGroupPosition,
+                ResultPosition, ResultGroupPosition>(key)) &&
            "expected parent to be an attribute, operand, or result");
     parent = key;
   }
-
-  /// Returns the index of this position.
-  ArrayRef<unsigned> getIndex() const final { return key->getIndex(); }
 };
 
 //===----------------------------------------------------------------------===//
@@ -311,8 +346,9 @@ struct TrueAnswer
   using Base::Base;
 };
 
-/// An Answer representing a `Type` value.
-struct TypeAnswer : public PredicateBase<TypeAnswer, Qualifier, Type,
+/// An Answer representing a `Type` value. The value is stored as either a
+/// TypeAttr, or an ArrayAttr of TypeAttr.
+struct TypeAnswer : public PredicateBase<TypeAnswer, Qualifier, Attribute,
                                          Predicates::TypeAnswer> {
   using Base::Base;
 };
@@ -365,6 +401,9 @@ struct IsNotNullQuestion
 struct OperandCountQuestion
     : public PredicateBase<OperandCountQuestion, Qualifier, void,
                            Predicates::OperandCountQuestion> {};
+struct OperandCountAtLeastQuestion
+    : public PredicateBase<OperandCountAtLeastQuestion, Qualifier, void,
+                           Predicates::OperandCountAtLeastQuestion> {};
 
 /// Compare the name of an operation with a known value.
 struct OperationNameQuestion
@@ -375,6 +414,9 @@ struct OperationNameQuestion
 struct ResultCountQuestion
     : public PredicateBase<ResultCountQuestion, Qualifier, void,
                            Predicates::ResultCountQuestion> {};
+struct ResultCountAtLeastQuestion
+    : public PredicateBase<ResultCountAtLeastQuestion, Qualifier, void,
+                           Predicates::ResultCountAtLeastQuestion> {};
 
 /// Compare the type of an attribute or value with a known type.
 struct TypeQuestion : public PredicateBase<TypeQuestion, Qualifier, void,
@@ -392,8 +434,10 @@ public:
     // Register the types of Positions with the uniquer.
     registerParametricStorageType<AttributePosition>();
     registerParametricStorageType<OperandPosition>();
+    registerParametricStorageType<OperandGroupPosition>();
     registerParametricStorageType<OperationPosition>();
     registerParametricStorageType<ResultPosition>();
+    registerParametricStorageType<ResultGroupPosition>();
     registerParametricStorageType<TypePosition>();
 
     // Register the types of Questions with the uniquer.
@@ -409,8 +453,10 @@ public:
     registerSingletonStorageType<AttributeQuestion>();
     registerSingletonStorageType<IsNotNullQuestion>();
     registerSingletonStorageType<OperandCountQuestion>();
+    registerSingletonStorageType<OperandCountAtLeastQuestion>();
     registerSingletonStorageType<OperationNameQuestion>();
     registerSingletonStorageType<ResultCountQuestion>();
+    registerSingletonStorageType<ResultCountAtLeastQuestion>();
     registerSingletonStorageType<TypeQuestion>();
   }
 };
@@ -433,10 +479,10 @@ public:
   Position *getRoot() { return OperationPosition::getRoot(uniquer); }
 
   /// Returns the parent position defining the value held by the given operand.
-  Position *getParent(OperandPosition *p) {
-    std::vector<unsigned> index = p->getIndex();
-    index.push_back(p->getOperandNumber());
-    return OperationPosition::get(uniquer, index);
+  OperationPosition *getOperandDefiningOp(Position *p) {
+    assert((isa<OperandPosition, OperandGroupPosition>(p)) &&
+           "expected operand position");
+    return OperationPosition::get(uniquer, p);
   }
 
   /// Returns an attribute position for an attribute of the given operation.
@@ -449,11 +495,29 @@ public:
     return OperandPosition::get(uniquer, p, operand);
   }
 
+  /// Returns a position for a group of operands of the given operation.
+  Position *getOperandGroup(OperationPosition *p, Optional<unsigned> group,
+                            bool isVariadic) {
+    return OperandGroupPosition::get(uniquer, p, group, isVariadic);
+  }
+  Position *getAllOperands(OperationPosition *p) {
+    return getOperandGroup(p, /*group=*/llvm::None, /*isVariadic=*/true);
+  }
+
   /// Returns a result position for a result of the given operation.
   Position *getResult(OperationPosition *p, unsigned result) {
     return ResultPosition::get(uniquer, p, result);
   }
 
+  /// Returns a position for a group of results of the given operation.
+  Position *getResultGroup(OperationPosition *p, Optional<unsigned> group,
+                           bool isVariadic) {
+    return ResultGroupPosition::get(uniquer, p, group, isVariadic);
+  }
+  Position *getAllResults(OperationPosition *p) {
+    return getResultGroup(p, /*group=*/llvm::None, /*isVariadic=*/true);
+  }
+
   /// Returns a type position for the given entity.
   Position *getType(Position *p) { return TypePosition::get(uniquer, p); }
 
@@ -496,6 +560,10 @@ public:
     return {OperandCountQuestion::get(uniquer),
             UnsignedAnswer::get(uniquer, count)};
   }
+  Predicate getOperandCountAtLeast(unsigned count) {
+    return {OperandCountAtLeastQuestion::get(uniquer),
+            UnsignedAnswer::get(uniquer, count)};
+  }
 
   /// Create a predicate comparing the name of an operation to a known value.
   Predicate getOperationName(StringRef name) {
@@ -509,10 +577,15 @@ public:
     return {ResultCountQuestion::get(uniquer),
             UnsignedAnswer::get(uniquer, count)};
   }
+  Predicate getResultCountAtLeast(unsigned count) {
+    return {ResultCountAtLeastQuestion::get(uniquer),
+            UnsignedAnswer::get(uniquer, count)};
+  }
 
   /// Create a predicate comparing the type of an attribute or value to a known
-  /// type.
-  Predicate getTypeConstraint(Type type) {
+  /// type. The value is stored as either a TypeAttr, or an ArrayAttr of
+  /// TypeAttr.
+  Predicate getTypeConstraint(Attribute type) {
     return {TypeQuestion::get(uniquer), TypeAnswer::get(uniquer, type)};
   }
 
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
index 22794aa4d991973cb0c3502afa2cdac17fb2050b..3061d464fe8b29430a752e37e3132d42a1ba35a5 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
+#include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
 using namespace mlir::pdl_to_pdl_interp;
@@ -20,151 +21,282 @@ using namespace mlir::pdl_to_pdl_interp;
 // Predicate List Building
 //===----------------------------------------------------------------------===//
 
+static void getTreePredicates(std::vector<PositionalPredicate> &predList,
+                              Value val, PredicateBuilder &builder,
+                              DenseMap<Value, Position *> &inputs,
+                              Position *pos);
+
 /// Compares the depths of two positions.
 static bool comparePosDepth(Position *lhs, Position *rhs) {
-  return lhs->getIndex().size() < rhs->getIndex().size();
+  return lhs->getOperationDepth() < rhs->getOperationDepth();
+}
+
+/// Returns the number of non-range elements within `values`.
+static unsigned getNumNonRangeValues(ValueRange values) {
+  return llvm::count_if(values.getTypes(),
+                        [](Type type) { return !type.isa<pdl::RangeType>(); });
 }
 
-/// Collect the tree predicates anchored at the given value.
 static void getTreePredicates(std::vector<PositionalPredicate> &predList,
                               Value val, PredicateBuilder &builder,
                               DenseMap<Value, Position *> &inputs,
-                              Position *pos) {
-  // Make sure this input value is accessible to the rewrite.
-  auto it = inputs.try_emplace(val, pos);
+                              AttributePosition *pos) {
+  assert(val.getType().isa<pdl::AttributeType>() && "expected attribute type");
+  pdl::AttributeOp attr = cast<pdl::AttributeOp>(val.getDefiningOp());
+  predList.emplace_back(pos, builder.getIsNotNull());
+
+  // If the attribute has a type or value, add a constraint.
+  if (Value type = attr.type())
+    getTreePredicates(predList, type, builder, inputs, builder.getType(pos));
+  else if (Attribute value = attr.valueAttr())
+    predList.emplace_back(pos, builder.getAttributeConstraint(value));
+}
 
-  // If this is an input value that has been visited in the tree, add a
-  // constraint to ensure that both instances refer to the same value.
-  if (!it.second &&
-      isa<pdl::AttributeOp, pdl::OperandOp, pdl::TypeOp>(val.getDefiningOp())) {
-    auto minMaxPositions = std::minmax(pos, it.first->second, comparePosDepth);
-    predList.emplace_back(minMaxPositions.second,
-                          builder.getEqualTo(minMaxPositions.first));
-    return;
-  }
+/// Collect all of the predicates for the given operand position.
+static void getOperandTreePredicates(std::vector<PositionalPredicate> &predList,
+                                     Value val, PredicateBuilder &builder,
+                                     DenseMap<Value, Position *> &inputs,
+                                     Position *pos) {
+  Type valueType = val.getType();
+  bool isVariadic = valueType.isa<pdl::RangeType>();
+
+  // If this is a typed operand, add a type constraint.
+  TypeSwitch<Operation *>(val.getDefiningOp())
+      .Case<pdl::OperandOp, pdl::OperandsOp>([&](auto op) {
+        // Prevent traversal into a null value if the operand has a proper
+        // index.
+        if (std::is_same<pdl::OperandOp, decltype(op)>::value ||
+            cast<OperandGroupPosition>(pos)->getOperandGroupNumber())
+          predList.emplace_back(pos, builder.getIsNotNull());
+
+        if (Value type = op.type())
+          getTreePredicates(predList, type, builder, inputs,
+                            builder.getType(pos));
+      })
+      .Case<pdl::ResultOp, pdl::ResultsOp>([&](auto op) {
+        Optional<unsigned> index = op.index();
+
+        // Prevent traversal into a null value if the result has a proper index.
+        if (index)
+          predList.emplace_back(pos, builder.getIsNotNull());
+
+        // Get the parent operation of this operand.
+        OperationPosition *parentPos = builder.getOperandDefiningOp(pos);
+        predList.emplace_back(parentPos, builder.getIsNotNull());
+
+        // Ensure that the operands match the corresponding results of the
+        // parent operation.
+        Position *resultPos = nullptr;
+        if (std::is_same<pdl::ResultOp, decltype(op)>::value)
+          resultPos = builder.getResult(parentPos, *index);
+        else
+          resultPos = builder.getResultGroup(parentPos, index, isVariadic);
+        predList.emplace_back(resultPos, builder.getEqualTo(pos));
+
+        // Collect the predicates of the parent operation.
+        getTreePredicates(predList, op.parent(), builder, inputs,
+                          (Position *)parentPos);
+      });
+}
 
-  // Check for a per-position predicate to apply.
-  switch (pos->getKind()) {
-  case Predicates::AttributePos: {
-    assert(val.getType().isa<pdl::AttributeType>() &&
-           "expected attribute type");
-    pdl::AttributeOp attr = cast<pdl::AttributeOp>(val.getDefiningOp());
-    predList.emplace_back(pos, builder.getIsNotNull());
+static void getTreePredicates(std::vector<PositionalPredicate> &predList,
+                              Value val, PredicateBuilder &builder,
+                              DenseMap<Value, Position *> &inputs,
+                              OperationPosition *pos) {
+  assert(val.getType().isa<pdl::OperationType>() && "expected operation");
+  pdl::OperationOp op = cast<pdl::OperationOp>(val.getDefiningOp());
+  OperationPosition *opPos = cast<OperationPosition>(pos);
 
-    // If the attribute has a type, add a type constraint.
-    if (Value type = attr.type()) {
-      getTreePredicates(predList, type, builder, inputs, builder.getType(pos));
+  // Ensure getDefiningOp returns a non-null operation.
+  if (!opPos->isRoot())
+    predList.emplace_back(pos, builder.getIsNotNull());
 
-      // Check for a constant value of the attribute.
-    } else if (Optional<Attribute> value = attr.value()) {
-      predList.emplace_back(pos, builder.getAttributeConstraint(*value));
-    }
-    break;
+  // Check that this is the correct root operation.
+  if (Optional<StringRef> opName = op.name())
+    predList.emplace_back(pos, builder.getOperationName(*opName));
+
+  // Check that the operation has the proper number of operands. If there are
+  // any variable length operands, we check a minimum instead of an exact count.
+  OperandRange operands = op.operands();
+  unsigned minOperands = getNumNonRangeValues(operands);
+  if (minOperands != operands.size()) {
+    if (minOperands)
+      predList.emplace_back(pos, builder.getOperandCountAtLeast(minOperands));
+  } else {
+    predList.emplace_back(pos, builder.getOperandCount(minOperands));
   }
-  case Predicates::OperandPos: {
-    assert(val.getType().isa<pdl::ValueType>() && "expected value type");
 
-    // Prevent traversal into a null value.
-    predList.emplace_back(pos, builder.getIsNotNull());
+  // Check that the operation has the proper number of results. If there are
+  // any variable length results, we check a minimum instead of an exact count.
+  OperandRange types = op.types();
+  unsigned minResults = getNumNonRangeValues(types);
+  if (minResults == types.size())
+    predList.emplace_back(pos, builder.getResultCount(types.size()));
+  else if (minResults)
+    predList.emplace_back(pos, builder.getResultCountAtLeast(minResults));
+
+  // Recurse into any attributes, operands, or results.
+  for (auto it : llvm::zip(op.attributeNames(), op.attributes())) {
+    getTreePredicates(
+        predList, std::get<1>(it), builder, inputs,
+        builder.getAttribute(opPos,
+                             std::get<0>(it).cast<StringAttr>().getValue()));
+  }
 
-    // If this is a typed operand, add a type constraint.
-    if (auto in = val.getDefiningOp<pdl::OperandOp>()) {
-      if (Value type = in.type()) {
-        getTreePredicates(predList, type, builder, inputs,
-                          builder.getType(pos));
-      }
-
-      // Otherwise, recurse into the parent node.
-    } else if (auto parentOp = val.getDefiningOp<pdl::OperationOp>()) {
-      getTreePredicates(predList, parentOp.op(), builder, inputs,
-                        builder.getParent(cast<OperandPosition>(pos)));
+  // Process the operands and results of the operation. For all values up to
+  // the first variable length value, we use the concrete operand/result
+  // number. After that, we use the "group" given that we can't know the
+  // concrete indices until runtime. If there is only one variadic operand
+  // group, we treat it as all of the operands/results of the operation.
+  /// Operands.
+  if (operands.size() == 1 && operands[0].getType().isa<pdl::RangeType>()) {
+    getTreePredicates(predList, operands.front(), builder, inputs,
+                      builder.getAllOperands(opPos));
+  } else {
+    bool foundVariableLength = false;
+    for (auto operandIt : llvm::enumerate(operands)) {
+      bool isVariadic = operandIt.value().getType().isa<pdl::RangeType>();
+      foundVariableLength |= isVariadic;
+
+      Position *pos =
+          foundVariableLength
+              ? builder.getOperandGroup(opPos, operandIt.index(), isVariadic)
+              : builder.getOperand(opPos, operandIt.index());
+      getTreePredicates(predList, operandIt.value(), builder, inputs, pos);
     }
-    break;
   }
-  case Predicates::OperationPos: {
-    assert(val.getType().isa<pdl::OperationType>() && "expected operation");
-    pdl::OperationOp op = cast<pdl::OperationOp>(val.getDefiningOp());
-    OperationPosition *opPos = cast<OperationPosition>(pos);
-
-    // Ensure getDefiningOp returns a non-null operation.
-    if (!opPos->isRoot())
-      predList.emplace_back(pos, builder.getIsNotNull());
-
-    // Check that this is the correct root operation.
-    if (Optional<StringRef> opName = op.name())
-      predList.emplace_back(pos, builder.getOperationName(*opName));
-
-    // Check that the operation has the proper number of operands and results.
-    OperandRange operands = op.operands();
-    ResultRange results = op.results();
-    predList.emplace_back(pos, builder.getOperandCount(operands.size()));
-    predList.emplace_back(pos, builder.getResultCount(results.size()));
-
-    // Recurse into any attributes, operands, or results.
-    for (auto it : llvm::zip(op.attributeNames(), op.attributes())) {
-      getTreePredicates(
-          predList, std::get<1>(it), builder, inputs,
-          builder.getAttribute(opPos,
-                               std::get<0>(it).cast<StringAttr>().getValue()));
-    }
-    for (auto operandIt : llvm::enumerate(operands))
-      getTreePredicates(predList, operandIt.value(), builder, inputs,
-                        builder.getOperand(opPos, operandIt.index()));
-
-    // Only recurse into results that are not referenced in the source tree.
-    for (auto resultIt : llvm::enumerate(results)) {
+  /// Results.
+  if (types.size() == 1 && types[0].getType().isa<pdl::RangeType>()) {
+    getTreePredicates(predList, types.front(), builder, inputs,
+                      builder.getType(builder.getAllResults(opPos)));
+  } else {
+    bool foundVariableLength = false;
+    for (auto &resultIt : llvm::enumerate(types)) {
+      bool isVariadic = resultIt.value().getType().isa<pdl::RangeType>();
+      foundVariableLength |= isVariadic;
+
+      auto *resultPos =
+          foundVariableLength
+              ? builder.getResultGroup(pos, resultIt.index(), isVariadic)
+              : builder.getResult(pos, resultIt.index());
+      predList.emplace_back(resultPos, builder.getIsNotNull());
       getTreePredicates(predList, resultIt.value(), builder, inputs,
-                        builder.getResult(opPos, resultIt.index()));
+                        builder.getType(resultPos));
     }
-    break;
   }
-  case Predicates::ResultPos: {
-    assert(val.getType().isa<pdl::ValueType>() && "expected value type");
-    pdl::OperationOp parentOp = cast<pdl::OperationOp>(val.getDefiningOp());
-
-    // Prevent traversing a null value.
-    predList.emplace_back(pos, builder.getIsNotNull());
+}
 
-    // Traverse the type constraint.
-    unsigned resultNo = cast<ResultPosition>(pos)->getResultNumber();
-    getTreePredicates(predList, parentOp.types()[resultNo], builder, inputs,
-                      builder.getType(pos));
-    break;
-  }
-  case Predicates::TypePos: {
-    assert(val.getType().isa<pdl::TypeType>() && "expected value type");
-    pdl::TypeOp typeOp = cast<pdl::TypeOp>(val.getDefiningOp());
-
-    // Check for a constraint on a constant type.
-    if (Optional<Type> type = typeOp.type())
-      predList.emplace_back(pos, builder.getTypeConstraint(*type));
-    break;
+static void getTreePredicates(std::vector<PositionalPredicate> &predList,
+                              Value val, PredicateBuilder &builder,
+                              DenseMap<Value, Position *> &inputs,
+                              TypePosition *pos) {
+  // Check for a constraint on a constant type.
+  if (pdl::TypeOp typeOp = val.getDefiningOp<pdl::TypeOp>()) {
+    if (Attribute type = typeOp.typeAttr())
+      predList.emplace_back(pos, builder.getTypeConstraint(type));
+  } else if (pdl::TypesOp typeOp = val.getDefiningOp<pdl::TypesOp>()) {
+    if (Attribute typeAttr = typeOp.typesAttr())
+      predList.emplace_back(pos, builder.getTypeConstraint(typeAttr));
   }
-  default:
-    llvm_unreachable("unknown position kind");
+}
+
+/// Collect the tree predicates anchored at the given value.
+static void getTreePredicates(std::vector<PositionalPredicate> &predList,
+                              Value val, PredicateBuilder &builder,
+                              DenseMap<Value, Position *> &inputs,
+                              Position *pos) {
+  // Make sure this input value is accessible to the rewrite.
+  auto it = inputs.try_emplace(val, pos);
+  if (!it.second) {
+    // If this is an input value that has been visited in the tree, add a
+    // constraint to ensure that both instances refer to the same value.
+    if (isa<pdl::AttributeOp, pdl::OperandOp, pdl::OperandsOp, pdl::OperationOp,
+            pdl::TypeOp>(val.getDefiningOp())) {
+      auto minMaxPositions =
+          std::minmax(pos, it.first->second, comparePosDepth);
+      predList.emplace_back(minMaxPositions.second,
+                            builder.getEqualTo(minMaxPositions.first));
+    }
+    return;
   }
+
+  TypeSwitch<Position *>(pos)
+      .Case<AttributePosition, OperationPosition, TypePosition>([&](auto *pos) {
+        getTreePredicates(predList, val, builder, inputs, pos);
+      })
+      .Case<OperandPosition, OperandGroupPosition>([&](auto *pos) {
+        getOperandTreePredicates(predList, val, builder, inputs, pos);
+      })
+      .Default([](auto *) { llvm_unreachable("unexpected position kind"); });
 }
 
 /// Collect all of the predicates related to constraints within the given
 /// pattern operation.
-static void collectConstraintPredicates(
-    pdl::PatternOp pattern, std::vector<PositionalPredicate> &predList,
-    PredicateBuilder &builder, DenseMap<Value, Position *> &inputs) {
-  for (auto op : pattern.body().getOps<pdl::ApplyConstraintOp>()) {
-    OperandRange arguments = op.args();
-    ArrayAttr parameters = op.constParamsAttr();
-
-    std::vector<Position *> allPositions;
-    allPositions.reserve(arguments.size());
-    for (Value arg : arguments)
-      allPositions.push_back(inputs.lookup(arg));
-
-    // Push the constraint to the furthest position.
-    Position *pos = *std::max_element(allPositions.begin(), allPositions.end(),
-                                      comparePosDepth);
-    PredicateBuilder::Predicate pred =
-        builder.getConstraint(op.name(), std::move(allPositions), parameters);
-    predList.emplace_back(pos, pred);
+static void getConstraintPredicates(pdl::ApplyNativeConstraintOp op,
+                                    std::vector<PositionalPredicate> &predList,
+                                    PredicateBuilder &builder,
+                                    DenseMap<Value, Position *> &inputs) {
+  OperandRange arguments = op.args();
+  ArrayAttr parameters = op.constParamsAttr();
+
+  std::vector<Position *> allPositions;
+  allPositions.reserve(arguments.size());
+  for (Value arg : arguments)
+    allPositions.push_back(inputs.lookup(arg));
+
+  // Push the constraint to the furthest position.
+  Position *pos = *std::max_element(allPositions.begin(), allPositions.end(),
+                                    comparePosDepth);
+  PredicateBuilder::Predicate pred =
+      builder.getConstraint(op.name(), std::move(allPositions), parameters);
+  predList.emplace_back(pos, pred);
+}
+
+static void getResultPredicates(pdl::ResultOp op,
+                                std::vector<PositionalPredicate> &predList,
+                                PredicateBuilder &builder,
+                                DenseMap<Value, Position *> &inputs) {
+  Position *&resultPos = inputs[op];
+  if (resultPos)
+    return;
+
+  // Ensure that the result isn't null.
+  auto *parentPos = cast<OperationPosition>(inputs.lookup(op.parent()));
+  resultPos = builder.getResult(parentPos, op.index());
+  predList.emplace_back(resultPos, builder.getIsNotNull());
+}
+
+static void getResultPredicates(pdl::ResultsOp op,
+                                std::vector<PositionalPredicate> &predList,
+                                PredicateBuilder &builder,
+                                DenseMap<Value, Position *> &inputs) {
+  Position *&resultPos = inputs[op];
+  if (resultPos)
+    return;
+
+  // Ensure that the result isn't null if the result has an index.
+  auto *parentPos = cast<OperationPosition>(inputs.lookup(op.parent()));
+  bool isVariadic = op.getType().isa<pdl::RangeType>();
+  Optional<unsigned> index = op.index();
+  resultPos = builder.getResultGroup(parentPos, index, isVariadic);
+  if (index)
+    predList.emplace_back(resultPos, builder.getIsNotNull());
+}
+
+/// Collect all of the predicates that cannot be determined via walking the
+/// tree.
+static void getNonTreePredicates(pdl::PatternOp pattern,
+                                 std::vector<PositionalPredicate> &predList,
+                                 PredicateBuilder &builder,
+                                 DenseMap<Value, Position *> &inputs) {
+  for (Operation &op : pattern.body().getOps()) {
+    TypeSwitch<Operation *>(&op)
+        .Case<pdl::ApplyNativeConstraintOp>([&](auto constraintOp) {
+          getConstraintPredicates(constraintOp, predList, builder, inputs);
+        })
+        .Case<pdl::ResultOp, pdl::ResultsOp>([&](auto resultOp) {
+          getResultPredicates(resultOp, predList, builder, inputs);
+        });
   }
 }
 
@@ -176,7 +308,7 @@ static void buildPredicateList(pdl::PatternOp pattern,
                                DenseMap<Value, Position *> &valueToPosition) {
   getTreePredicates(predList, pattern.getRewriter().root(), builder,
                     valueToPosition, builder.getRoot());
-  collectConstraintPredicates(pattern, predList, builder, valueToPosition);
+  getNonTreePredicates(pattern, predList, builder, valueToPosition);
 }
 
 //===----------------------------------------------------------------------===//
@@ -223,10 +355,10 @@ struct OrderedPredicate {
     // * lower position dependency
     // * lower predicate dependency
     auto *rhsPos = rhs.position;
-    return std::make_tuple(primary, secondary, rhsPos->getIndex().size(),
+    return std::make_tuple(primary, secondary, rhsPos->getOperationDepth(),
                            rhsPos->getKind(), rhs.question->getKind()) >
            std::make_tuple(rhs.primary, rhs.secondary,
-                           position->getIndex().size(), position->getKind(),
+                           position->getOperationDepth(), position->getKind(),
                            question->getKind());
   }
 };
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h
index 1621fa96747b98262db1102ef4b4dd46baa01e92..ac2fa98d7c7b1f3ca7398e37d44635ca19c96df6 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h
@@ -190,6 +190,12 @@ struct SwitchNode : public MatcherNode {
   using ChildMapT = llvm::MapVector<Qualifier *, std::unique_ptr<MatcherNode>>;
   ChildMapT &getChildren() { return children; }
 
+  /// Returns the child at the given index.
+  std::pair<Qualifier *, std::unique_ptr<MatcherNode>> &getChild(unsigned i) {
+    assert(i < children.size() && "invalid child index");
+    return *std::next(children.begin(), i);
+  }
+
 private:
   /// Switch predicate "answers" select the child. Answers that are not found
   /// default to the failure node.
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index 9f5e4abc42ca955f79120b43319b54b41103f78d..d13cebe3c3a2f66fbb6ca781c45e0313b2c91b4f 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -642,9 +642,8 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
   return success();
 }
 
-void mlir::populateParallelLoopToGPUPatterns(OwningRewritePatternList &patterns,
-                                             MLIRContext *ctx) {
-  patterns.insert<ParallelToGpuLaunchLowering>(ctx);
+void mlir::populateParallelLoopToGPUPatterns(RewritePatternSet &patterns) {
+  patterns.add<ParallelToGpuLaunchLowering>(patterns.getContext());
 }
 
 void mlir::configureParallelLoopToGPULegality(ConversionTarget &target) {
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
index 15075b53ab9024b5324a4bd68a7a9a78730b857e..43c6798091e73876ecf03a624204341c3c88c2ee 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
@@ -47,8 +47,8 @@ struct ForLoopMapper : public ConvertAffineForToGPUBase<ForLoopMapper> {
 struct ParallelLoopToGpuPass
     : public ConvertParallelLoopToGpuBase<ParallelLoopToGpuPass> {
   void runOnOperation() override {
-    OwningRewritePatternList patterns;
-    populateParallelLoopToGPUPatterns(patterns, &getContext());
+    RewritePatternSet patterns(&getContext());
+    populateParallelLoopToGPUPatterns(patterns);
     ConversionTarget target(getContext());
     target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
     configureParallelLoopToGPULegality(target);
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 3adb02af15c60426b9c055d4190296c73ffbdf0d..1dedc2c39d8f672f031dc164ec092bf733c1f124 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -90,9 +90,9 @@ static LogicalResult applyPatterns(FuncOp func) {
       [](scf::YieldOp op) { return !isa<scf::ParallelOp>(op->getParentOp()); });
   target.addLegalDialect<omp::OpenMPDialect>();
 
-  OwningRewritePatternList patterns;
-  patterns.insert<ParallelOpLowering>(func.getContext());
-  FrozenRewritePatternList frozen(std::move(patterns));
+  RewritePatternSet patterns(func.getContext());
+  patterns.add<ParallelOpLowering>(func.getContext());
+  FrozenRewritePatternSet frozen(std::move(patterns));
   return applyPartialConversion(func, target, frozen);
 }
 
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index 8e7540f4e76b28335e6079ba3d60b526a37f9052..08e3d3f727627018f84a920f84722b06453f12b3 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -155,9 +155,7 @@ ForOpConversion::matchAndRewrite(scf::ForOp forOp, ArrayRef<Value> operands,
   // header to merge.
   scf::ForOpAdaptor forOperands(operands);
   auto loc = forOp.getLoc();
-  auto loopControl = rewriter.getI32IntegerAttr(
-      static_cast<uint32_t>(spirv::LoopControl::None));
-  auto loopOp = rewriter.create<spirv::LoopOp>(loc, loopControl);
+  auto loopOp = rewriter.create<spirv::LoopOp>(loc, spirv::LoopControl::None);
   loopOp.addEntryAndMergeBlock();
 
   OpBuilder::InsertionGuard guard(rewriter);
@@ -238,11 +236,9 @@ IfOpConversion::matchAndRewrite(scf::IfOp ifOp, ArrayRef<Value> operands,
   scf::IfOpAdaptor ifOperands(operands);
   auto loc = ifOp.getLoc();
 
-  // Create `spv.mlir.selection` operation, selection header block and merge
-  // block.
-  auto selectionControl = rewriter.getI32IntegerAttr(
-      static_cast<uint32_t>(spirv::SelectionControl::None));
-  auto selectionOp = rewriter.create<spirv::SelectionOp>(loc, selectionControl);
+  // Create `spv.selection` operation, selection header block and merge block.
+  auto selectionOp =
+      rewriter.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
   auto *mergeBlock =
       rewriter.createBlock(&selectionOp.body(), selectionOp.body().end());
   rewriter.create<spirv::MergeOp>(loc);
@@ -323,10 +319,9 @@ LogicalResult TerminatorOpConversion::matchAndRewrite(
 // Hooks
 //===----------------------------------------------------------------------===//
 
-void mlir::populateSCFToSPIRVPatterns(MLIRContext *context,
-                                      SPIRVTypeConverter &typeConverter,
+void mlir::populateSCFToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
                                       ScfToSPIRVContext &scfToSPIRVContext,
-                                      OwningRewritePatternList &patterns) {
-  patterns.insert<ForOpConversion, IfOpConversion, TerminatorOpConversion>(
-      context, typeConverter, scfToSPIRVContext.getImpl());
+                                      RewritePatternSet &patterns) {
+  patterns.add<ForOpConversion, IfOpConversion, TerminatorOpConversion>(
+      patterns.getContext(), typeConverter, scfToSPIRVContext.getImpl());
 }
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
index b0d879974a56a065be8ef1dd86390186e5163165..637e6a7501b717fcc06107dbf44a3c0bb40a6cec 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
@@ -37,10 +37,10 @@ void SCFToSPIRVPass::runOnOperation() {
 
   SPIRVTypeConverter typeConverter(targetAttr);
   ScfToSPIRVContext scfContext;
-  OwningRewritePatternList patterns;
-  populateSCFToSPIRVPatterns(context, typeConverter, scfContext, patterns);
-  populateStandardToSPIRVPatterns(context, typeConverter, patterns);
-  populateBuiltinFuncToSPIRVPatterns(context, typeConverter, patterns);
+  RewritePatternSet patterns(context);
+  populateSCFToSPIRVPatterns(typeConverter, scfContext, patterns);
+  populateStandardToSPIRVPatterns(typeConverter, patterns);
+  populateBuiltinFuncToSPIRVPatterns(typeConverter, patterns);
 
   if (failed(applyPartialConversion(module, *target, std::move(patterns))))
     return signalPassFailure();
diff --git a/mlir/lib/Conversion/SCFToStandard/SCFToStandard.cpp b/mlir/lib/Conversion/SCFToStandard/SCFToStandard.cpp
index b8f3140dee735fd1547dd603d67d228acf405376..6efba3fc816ce68ed6afac4288efaf0fccfd86bf 100644
--- a/mlir/lib/Conversion/SCFToStandard/SCFToStandard.cpp
+++ b/mlir/lib/Conversion/SCFToStandard/SCFToStandard.cpp
@@ -568,16 +568,15 @@ DoWhileLowering::matchAndRewrite(WhileOp whileOp,
   return success();
 }
 
-void mlir::populateLoopToStdConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx) {
-  patterns.insert<ForLowering, IfLowering, ParallelLowering, WhileLowering>(
-      ctx);
-  patterns.insert<DoWhileLowering>(ctx, /*benefit=*/2);
+void mlir::populateLoopToStdConversionPatterns(RewritePatternSet &patterns) {
+  patterns.add<ForLowering, IfLowering, ParallelLowering, WhileLowering>(
+      patterns.getContext());
+  patterns.add<DoWhileLowering>(patterns.getContext(), /*benefit=*/2);
 }
 
 void SCFToStandardPass::runOnOperation() {
-  OwningRewritePatternList patterns;
-  populateLoopToStdConversionPatterns(patterns, &getContext());
+  RewritePatternSet patterns(&getContext());
+  populateLoopToStdConversionPatterns(patterns);
   // Configure conversion to lower out scf.for, scf.if, scf.parallel and
   // scf.while. Anything else is fine.
   ConversionTarget target(getContext());
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index d152a738dc0b630e23ab9dc532abd0a58269d41a..f064a6532c8a0765c81b3266345154bf92f6b401 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -273,15 +273,13 @@ public:
 
     // Specify options to lower Standard to LLVM and pull in the conversion
     // patterns.
-    LowerToLLVMOptions options = {
-        /*useBarePtrCallConv=*/false,
-        /*emitCWrappers=*/true,
-        /*indexBitwidth=*/kDeriveIndexBitwidthFromDataLayout};
+    LowerToLLVMOptions options(module.getContext());
+    options.emitCWrappers = true;
     auto *context = module.getContext();
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
     LLVMTypeConverter typeConverter(context, options);
     populateStdToLLVMConversionPatterns(typeConverter, patterns);
-    patterns.insert<GPULaunchLowering>(typeConverter);
+    patterns.add<GPULaunchLowering>(typeConverter);
 
     // Pull in SPIR-V type conversion patterns to convert SPIR-V global
     // variable's type to LLVM dialect type.
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index 871f54b3aa09bddca8ec5593f2f51c938c551c04..172f63ba2268890ae9f0acd4faf62137b199c85c 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -826,10 +826,8 @@ public:
       return failure();
 
     rewriter.template replaceOpWithNewOp<LLVM::FCmpOp>(
-        operation, dstType,
-        rewriter.getI64IntegerAttr(static_cast<int64_t>(predicate)),
-        operation.operand1(), operation.operand2(),
-        LLVM::FMFAttr::get(operation.getContext(), {}));
+        operation, dstType, predicate, operation.operand1(),
+        operation.operand2());
     return success();
   }
 };
@@ -849,9 +847,8 @@ public:
       return failure();
 
     rewriter.template replaceOpWithNewOp<LLVM::ICmpOp>(
-        operation, dstType,
-        rewriter.getI64IntegerAttr(static_cast<int64_t>(predicate)),
-        operation.operand1(), operation.operand2());
+        operation, dstType, predicate, operation.operand1(),
+        operation.operand2());
     return success();
   }
 };
@@ -1361,7 +1358,7 @@ public:
   matchAndRewrite(spirv::ModuleEndOp moduleEndOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
 
-    rewriter.replaceOpWithNewOp<ModuleTerminatorOp>(moduleEndOp);
+    rewriter.eraseOp(moduleEndOp);
     return success();
   }
 };
@@ -1388,9 +1385,8 @@ void mlir::populateSPIRVToLLVMTypeConversion(LLVMTypeConverter &typeConverter) {
 }
 
 void mlir::populateSPIRVToLLVMConversionPatterns(
-    MLIRContext *context, LLVMTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns) {
-  patterns.insert<
+    LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) {
+  patterns.add<
       // Arithmetic ops
       DirectConversionPattern<spirv::IAddOp, LLVM::AddOp>,
       DirectConversionPattern<spirv::IMulOp, LLVM::MulOp>,
@@ -1499,20 +1495,18 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       ShiftPattern<spirv::ShiftLeftLogicalOp, LLVM::ShlOp>,
 
       // Return ops
-      ReturnPattern, ReturnValuePattern>(context, typeConverter);
+      ReturnPattern, ReturnValuePattern>(patterns.getContext(), typeConverter);
 }
 
 void mlir::populateSPIRVToLLVMFunctionConversionPatterns(
-    MLIRContext *context, LLVMTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns) {
-  patterns.insert<FuncConversionPattern>(context, typeConverter);
+    LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) {
+  patterns.add<FuncConversionPattern>(patterns.getContext(), typeConverter);
 }
 
 void mlir::populateSPIRVToLLVMModuleConversionPatterns(
-    MLIRContext *context, LLVMTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns) {
-  patterns.insert<ModuleConversionPattern, ModuleEndConversionPattern>(
-      context, typeConverter);
+    LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) {
+  patterns.add<ModuleConversionPattern, ModuleEndConversionPattern>(
+      patterns.getContext(), typeConverter);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp
index 2a4113f307fa8db80dd57ceb91338778eae35517..293128c8446bda0bbc543e739aea2dd2b3d4a5d0 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp
@@ -36,22 +36,20 @@ void ConvertSPIRVToLLVMPass::runOnOperation() {
   // Encode global variable's descriptor set and binding if they exist.
   encodeBindAttribute(module);
 
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(context);
 
   populateSPIRVToLLVMTypeConversion(converter);
 
-  populateSPIRVToLLVMModuleConversionPatterns(context, converter, patterns);
-  populateSPIRVToLLVMConversionPatterns(context, converter, patterns);
-  populateSPIRVToLLVMFunctionConversionPatterns(context, converter, patterns);
+  populateSPIRVToLLVMModuleConversionPatterns(converter, patterns);
+  populateSPIRVToLLVMConversionPatterns(converter, patterns);
+  populateSPIRVToLLVMFunctionConversionPatterns(converter, patterns);
 
-  ConversionTarget target(getContext());
+  ConversionTarget target(*context);
   target.addIllegalDialect<spirv::SPIRVDialect>();
   target.addLegalDialect<LLVM::LLVMDialect>();
 
-  // Set `ModuleOp` and `ModuleTerminatorOp` as legal for `spv.module`
-  // conversion.
+  // Set `ModuleOp` as legal for `spv.module` conversion.
   target.addLegalOp<ModuleOp>();
-  target.addLegalOp<ModuleTerminatorOp>();
   if (failed(applyPartialConversion(module, target, std::move(patterns))))
     signalPassFailure();
 }
diff --git a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
index af976056acb0755553549f26e960c41e7d4313d4..d5388dfd4040c02d32a7a13c28a9df1b3934324b 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
@@ -37,10 +37,10 @@ public:
 } // namespace
 
 void mlir::populateConvertShapeConstraintsConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx) {
-  patterns.insert<CstrBroadcastableToRequire>(ctx);
-  patterns.insert<CstrEqToRequire>(ctx);
-  patterns.insert<ConvertCstrRequireOp>(ctx);
+    RewritePatternSet &patterns) {
+  patterns.add<CstrBroadcastableToRequire>(patterns.getContext());
+  patterns.add<CstrEqToRequire>(patterns.getContext());
+  patterns.add<ConvertCstrRequireOp>(patterns.getContext());
 }
 
 namespace {
@@ -54,8 +54,8 @@ class ConvertShapeConstraints
     auto func = getOperation();
     auto *context = &getContext();
 
-    OwningRewritePatternList patterns;
-    populateConvertShapeConstraintsConversionPatterns(patterns, context);
+    RewritePatternSet patterns(context);
+    populateConvertShapeConstraintsConversionPatterns(patterns);
 
     if (failed(applyPatternsAndFoldGreedily(func, std::move(patterns))))
       return signalPassFailure();
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index 2c06702e32c1100fd7e45ae5e46a8ed8ac5bd63f..e0342f6162c533c4def0f481a6f46700ad3e16cc 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -675,11 +675,11 @@ void ConvertShapeToStandardPass::runOnOperation() {
   ConversionTarget target(ctx);
   target.addLegalDialect<memref::MemRefDialect, StandardOpsDialect, SCFDialect,
                          tensor::TensorDialect>();
-  target.addLegalOp<CstrRequireOp, FuncOp, ModuleOp, ModuleTerminatorOp>();
+  target.addLegalOp<CstrRequireOp, FuncOp, ModuleOp>();
 
   // Setup conversion patterns.
-  OwningRewritePatternList patterns;
-  populateShapeToStandardConversionPatterns(patterns, &ctx);
+  RewritePatternSet patterns(&ctx);
+  populateShapeToStandardConversionPatterns(patterns);
 
   // Apply conversion.
   auto module = getOperation();
@@ -688,10 +688,10 @@ void ConvertShapeToStandardPass::runOnOperation() {
 }
 
 void mlir::populateShapeToStandardConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx) {
+    RewritePatternSet &patterns) {
   // clang-format off
-  populateWithGenerated(ctx, patterns);
-  patterns.insert<
+  populateWithGenerated(patterns);
+  patterns.add<
       AnyOpConversion,
       BinaryOpConversion<AddOp, AddIOp>,
       BinaryOpConversion<MulOp, MulIOp>,
@@ -705,7 +705,7 @@ void mlir::populateShapeToStandardConversionPatterns(
       ShapeEqOpConverter,
       ShapeOfOpConversion,
       SplitAtOpConversion,
-      ToExtentTensorOpConversion>(ctx);
+      ToExtentTensorOpConversion>(patterns.getContext());
   // clang-format on
 }
 
diff --git a/mlir/lib/Conversion/StandardToLLVM/CMakeLists.txt b/mlir/lib/Conversion/StandardToLLVM/CMakeLists.txt
index e1e13f0b1cc2514215c4fcf5a516ec64eccae3f1..13cf5cb16c9f6d6de858561fc481a9bd4ff2466d 100644
--- a/mlir/lib/Conversion/StandardToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/StandardToLLVM/CMakeLists.txt
@@ -12,6 +12,7 @@ add_mlir_conversion_library(MLIRStandardToLLVM
   Core
 
   LINK_LIBS PUBLIC
+  MLIRDataLayoutInterfaces
   MLIRLLVMIR
   MLIRMath
   MLIRMemRef
diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
index b3a2bb634f39de5311b43c3615e3522fa62fedb7..ddfb349cb816aa3fcd0a9b09d72e0ec97b1a8994 100644
--- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
@@ -26,6 +26,7 @@
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
+#include "mlir/Interfaces/DataLayoutInterfaces.h"
 #include "mlir/Support/LogicalResult.h"
 #include "mlir/Support/MathExtras.h"
 #include "mlir/Transforms/DialectConversion.h"
@@ -101,7 +102,7 @@ LogicalResult mlir::barePtrFuncArgTypeConverter(LLVMTypeConverter &converter,
 
 /// Create an LLVMTypeConverter using default LowerToLLVMOptions.
 LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx)
-    : LLVMTypeConverter(ctx, LowerToLLVMOptions::getDefaultOptions()) {}
+    : LLVMTypeConverter(ctx, LowerToLLVMOptions(ctx)) {}
 
 /// Create an LLVMTypeConverter using custom LowerToLLVMOptions.
 LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
@@ -109,8 +110,6 @@ LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
     : llvmDialect(ctx->getOrLoadDialect<LLVM::LLVMDialect>()),
       options(options) {
   assert(llvmDialect && "LLVM IR dialect is not registered");
-  if (options.indexBitwidth == kDeriveIndexBitwidthFromDataLayout)
-    this->options.indexBitwidth = options.dataLayout.getPointerSizeInBits();
 
   // Register conversions for the builtin types.
   addConversion([&](ComplexType type) { return convertComplexType(type); });
@@ -253,8 +252,24 @@ Type LLVMTypeConverter::convertFunctionSignature(
 
 /// Converts the function type to a C-compatible format, in particular using
 /// pointers to memref descriptors for arguments.
-Type LLVMTypeConverter::convertFunctionTypeCWrapper(FunctionType type) {
+std::pair<Type, bool>
+LLVMTypeConverter::convertFunctionTypeCWrapper(FunctionType type) {
   SmallVector<Type, 4> inputs;
+  bool resultIsNowArg = false;
+
+  Type resultType = type.getNumResults() == 0
+                        ? LLVM::LLVMVoidType::get(&getContext())
+                        : unwrap(packFunctionResults(type.getResults()));
+  if (!resultType)
+    return {};
+
+  if (auto structType = resultType.dyn_cast<LLVM::LLVMStructType>()) {
+    // Struct types cannot be safely returned via C interface. Make this a
+    // pointer argument, instead.
+    inputs.push_back(LLVM::LLVMPointerType::get(structType));
+    resultType = LLVM::LLVMVoidType::get(&getContext());
+    resultIsNowArg = true;
+  }
 
   for (Type t : type.getInputs()) {
     auto converted = convertType(t);
@@ -265,13 +280,7 @@ Type LLVMTypeConverter::convertFunctionTypeCWrapper(FunctionType type) {
     inputs.push_back(converted);
   }
 
-  Type resultType = type.getNumResults() == 0
-                        ? LLVM::LLVMVoidType::get(&getContext())
-                        : unwrap(packFunctionResults(type.getResults()));
-  if (!resultType)
-    return {};
-
-  return LLVM::LLVMFunctionType::get(resultType, inputs);
+  return {LLVM::LLVMFunctionType::get(resultType, inputs), resultIsNowArg};
 }
 
 static constexpr unsigned kAllocatedPtrPosInMemRefDescriptor = 0;
@@ -440,7 +449,7 @@ ConvertToLLVMPattern::ConvertToLLVMPattern(StringRef rootOpName,
                                            MLIRContext *context,
                                            LLVMTypeConverter &typeConverter,
                                            PatternBenefit benefit)
-    : ConversionPattern(rootOpName, benefit, typeConverter, context) {}
+    : ConversionPattern(typeConverter, rootOpName, benefit, context) {}
 
 //===----------------------------------------------------------------------===//
 // StructBuilder implementation
@@ -1212,8 +1221,11 @@ static void filterFuncAttributes(ArrayRef<NamedAttribute> attrs,
 /// Creates an auxiliary function with pointer-to-memref-descriptor-struct
 /// arguments instead of unpacked arguments. This function can be called from C
 /// by passing a pointer to a C struct corresponding to a memref descriptor.
+/// Similarly, returned memrefs are passed via pointers to a C struct that is
+/// passed as additional argument.
 /// Internally, the auxiliary function unpacks the descriptor into individual
-/// components and forwards them to `newFuncOp`.
+/// components and forwards them to `newFuncOp` and forwards the results to
+/// the extra arguments.
 static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
                                    LLVMTypeConverter &typeConverter,
                                    FuncOp funcOp, LLVM::LLVMFuncOp newFuncOp) {
@@ -1221,17 +1233,21 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
   SmallVector<NamedAttribute, 4> attributes;
   filterFuncAttributes(funcOp->getAttrs(), /*filterArgAttrs=*/false,
                        attributes);
+  Type wrapperFuncType;
+  bool resultIsNowArg;
+  std::tie(wrapperFuncType, resultIsNowArg) =
+      typeConverter.convertFunctionTypeCWrapper(type);
   auto wrapperFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
       loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(),
-      typeConverter.convertFunctionTypeCWrapper(type), LLVM::Linkage::External,
-      attributes);
+      wrapperFuncType, LLVM::Linkage::External, attributes);
 
   OpBuilder::InsertionGuard guard(rewriter);
   rewriter.setInsertionPointToStart(wrapperFuncOp.addEntryBlock());
 
   SmallVector<Value, 8> args;
+  size_t argOffset = resultIsNowArg ? 1 : 0;
   for (auto &en : llvm::enumerate(type.getInputs())) {
-    Value arg = wrapperFuncOp.getArgument(en.index());
+    Value arg = wrapperFuncOp.getArgument(en.index() + argOffset);
     if (auto memrefType = en.value().dyn_cast<MemRefType>()) {
       Value loaded = rewriter.create<LLVM::LoadOp>(loc, arg);
       MemRefDescriptor::unpack(rewriter, loc, loaded, memrefType, args);
@@ -1243,28 +1259,40 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
       continue;
     }
 
-    args.push_back(wrapperFuncOp.getArgument(en.index()));
+    args.push_back(arg);
   }
+
   auto call = rewriter.create<LLVM::CallOp>(loc, newFuncOp, args);
-  rewriter.create<LLVM::ReturnOp>(loc, call.getResults());
+
+  if (resultIsNowArg) {
+    rewriter.create<LLVM::StoreOp>(loc, call.getResult(0),
+                                   wrapperFuncOp.getArgument(0));
+    rewriter.create<LLVM::ReturnOp>(loc, ValueRange{});
+  } else {
+    rewriter.create<LLVM::ReturnOp>(loc, call.getResults());
+  }
 }
 
 /// Creates an auxiliary function with pointer-to-memref-descriptor-struct
 /// arguments instead of unpacked arguments. Creates a body for the (external)
 /// `newFuncOp` that allocates a memref descriptor on stack, packs the
 /// individual arguments into this descriptor and passes a pointer to it into
-/// the auxiliary function. This auxiliary external function is now compatible
-/// with functions defined in C using pointers to C structs corresponding to a
-/// memref descriptor.
+/// the auxiliary function. If the result of the function cannot be directly
+/// returned, we write it to a special first argument that provides a pointer
+/// to a corresponding struct. This auxiliary external function is now
+/// compatible with functions defined in C using pointers to C structs
+/// corresponding to a memref descriptor.
 static void wrapExternalFunction(OpBuilder &builder, Location loc,
                                  LLVMTypeConverter &typeConverter,
                                  FuncOp funcOp, LLVM::LLVMFuncOp newFuncOp) {
   OpBuilder::InsertionGuard guard(builder);
 
-  Type wrapperType =
+  Type wrapperType;
+  bool resultIsNowArg;
+  std::tie(wrapperType, resultIsNowArg) =
       typeConverter.convertFunctionTypeCWrapper(funcOp.getType());
   // This conversion can only fail if it could not convert one of the argument
-  // types. But since it has been applies to a non-wrapper function before, it
+  // types. But since it has been applied to a non-wrapper function before, it
   // should have failed earlier and not reach this point at all.
   assert(wrapperType && "unexpected type conversion failure");
 
@@ -1285,6 +1313,17 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   args.reserve(type.getNumInputs());
   ValueRange wrapperArgsRange(newFuncOp.getArguments());
 
+  if (resultIsNowArg) {
+    // Allocate the struct on the stack and pass the pointer.
+    Type resultType =
+        wrapperType.cast<LLVM::LLVMFunctionType>().getParamType(0);
+    Value one = builder.create<LLVM::ConstantOp>(
+        loc, typeConverter.convertType(builder.getIndexType()),
+        builder.getIntegerAttr(builder.getIndexType(), 1));
+    Value result = builder.create<LLVM::AllocaOp>(loc, resultType, one);
+    args.push_back(result);
+  }
+
   // Iterate over the inputs of the original function and pack values into
   // memref descriptors if the original type is a memref.
   for (auto &en : llvm::enumerate(type.getInputs())) {
@@ -1322,7 +1361,13 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   assert(wrapperArgsRange.empty() && "did not map some of the arguments");
 
   auto call = builder.create<LLVM::CallOp>(loc, wrapperFunc, args);
-  builder.create<LLVM::ReturnOp>(loc, call.getResults());
+
+  if (resultIsNowArg) {
+    Value result = builder.create<LLVM::LoadOp>(loc, args.front());
+    builder.create<LLVM::ReturnOp>(loc, ValueRange{result});
+  } else {
+    builder.create<LLVM::ReturnOp>(loc, call.getResults());
+  }
 }
 
 namespace {
@@ -2303,6 +2348,61 @@ struct GetGlobalMemrefOpLowering : public AllocLikeOpLowering {
   }
 };
 
+// A `log1p` is converted into `log(1 + ...)`.
+struct Log1pOpLowering : public ConvertOpToLLVMPattern<math::Log1pOp> {
+  using ConvertOpToLLVMPattern<math::Log1pOp>::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(math::Log1pOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    math::Log1pOp::Adaptor transformed(operands);
+    auto operandType = transformed.operand().getType();
+
+    if (!operandType || !LLVM::isCompatibleType(operandType))
+      return rewriter.notifyMatchFailure(op, "unsupported operand type");
+
+    auto loc = op.getLoc();
+    auto resultType = op.getResult().getType();
+    auto floatType = getElementTypeOrSelf(resultType).cast<FloatType>();
+    auto floatOne = rewriter.getFloatAttr(floatType, 1.0);
+
+    if (!operandType.isa<LLVM::LLVMArrayType>()) {
+      LLVM::ConstantOp one =
+          LLVM::isCompatibleVectorType(operandType)
+              ? rewriter.create<LLVM::ConstantOp>(
+                    loc, operandType,
+                    SplatElementsAttr::get(resultType.cast<ShapedType>(),
+                                           floatOne))
+              : rewriter.create<LLVM::ConstantOp>(loc, operandType, floatOne);
+
+      auto add = rewriter.create<LLVM::FAddOp>(loc, operandType, one,
+                                               transformed.operand());
+      rewriter.replaceOpWithNewOp<LLVM::LogOp>(op, operandType, add);
+      return success();
+    }
+
+    auto vectorType = resultType.dyn_cast<VectorType>();
+    if (!vectorType)
+      return rewriter.notifyMatchFailure(op, "expected vector result type");
+
+    return handleMultidimensionalVectors(
+        op.getOperation(), operands, *getTypeConverter(),
+        [&](Type llvm1DVectorTy, ValueRange operands) {
+          auto splatAttr = SplatElementsAttr::get(
+              mlir::VectorType::get(
+                  {LLVM::getVectorNumElements(llvm1DVectorTy).getFixedValue()},
+                  floatType),
+              floatOne);
+          auto one =
+              rewriter.create<LLVM::ConstantOp>(loc, llvm1DVectorTy, splatAttr);
+          auto add = rewriter.create<LLVM::FAddOp>(loc, llvm1DVectorTy, one,
+                                                   transformed.operand());
+          return rewriter.create<LLVM::LogOp>(loc, llvm1DVectorTy, add);
+        },
+        rewriter);
+  }
+};
+
 // A `rsqrt` is converted into `1 / sqrt`.
 struct RsqrtOpLowering : public ConvertOpToLLVMPattern<math::RsqrtOp> {
   using ConvertOpToLLVMPattern<math::RsqrtOp>::ConvertOpToLLVMPattern;
@@ -2968,8 +3068,7 @@ struct CmpIOpLowering : public ConvertOpToLLVMPattern<CmpIOp> {
 
     rewriter.replaceOpWithNewOp<LLVM::ICmpOp>(
         cmpiOp, typeConverter->convertType(cmpiOp.getResult().getType()),
-        rewriter.getI64IntegerAttr(static_cast<int64_t>(
-            convertCmpPredicate<LLVM::ICmpPredicate>(cmpiOp.getPredicate()))),
+        convertCmpPredicate<LLVM::ICmpPredicate>(cmpiOp.getPredicate()),
         transformed.lhs(), transformed.rhs());
 
     return success();
@@ -2984,12 +3083,10 @@ struct CmpFOpLowering : public ConvertOpToLLVMPattern<CmpFOp> {
                   ConversionPatternRewriter &rewriter) const override {
     CmpFOpAdaptor transformed(operands);
 
-    auto fmf = LLVM::FMFAttr::get(cmpfOp.getContext(), {});
     rewriter.replaceOpWithNewOp<LLVM::FCmpOp>(
         cmpfOp, typeConverter->convertType(cmpfOp.getResult().getType()),
-        rewriter.getI64IntegerAttr(static_cast<int64_t>(
-            convertCmpPredicate<LLVM::FCmpPredicate>(cmpfOp.getPredicate()))),
-        transformed.lhs(), transformed.rhs(), fmf);
+        convertCmpPredicate<LLVM::FCmpPredicate>(cmpfOp.getPredicate()),
+        transformed.lhs(), transformed.rhs());
 
     return success();
   }
@@ -3758,10 +3855,10 @@ private:
 
 /// Collect a set of patterns to convert from the Standard dialect to LLVM.
 void mlir::populateStdToLLVMNonMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+    LLVMTypeConverter &converter, RewritePatternSet &patterns) {
   // FIXME: this should be tablegen'ed
   // clang-format off
-  patterns.insert<
+  patterns.add<
       AbsFOpLowering,
       AddFOpLowering,
       AddIOpLowering,
@@ -3788,6 +3885,7 @@ void mlir::populateStdToLLVMNonMemoryConversionPatterns(
       GenericAtomicRMWOpLowering,
       LogOpLowering,
       Log10OpLowering,
+      Log1pOpLowering,
       Log2OpLowering,
       FPExtLowering,
       FPToSILowering,
@@ -3827,9 +3925,9 @@ void mlir::populateStdToLLVMNonMemoryConversionPatterns(
 }
 
 void mlir::populateStdToLLVMMemoryConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+    LLVMTypeConverter &converter, RewritePatternSet &patterns) {
   // clang-format off
-  patterns.insert<
+  patterns.add<
       AssumeAlignmentOpLowering,
       DeallocOpLowering,
       DimOpLowering,
@@ -3846,21 +3944,21 @@ void mlir::populateStdToLLVMMemoryConversionPatterns(
       ViewOpLowering>(converter);
   // clang-format on
   if (converter.getOptions().useAlignedAlloc)
-    patterns.insert<AlignedAllocOpLowering>(converter);
+    patterns.add<AlignedAllocOpLowering>(converter);
   else
-    patterns.insert<AllocOpLowering>(converter);
+    patterns.add<AllocOpLowering>(converter);
 }
 
 void mlir::populateStdToLLVMFuncOpConversionPattern(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+    LLVMTypeConverter &converter, RewritePatternSet &patterns) {
   if (converter.getOptions().useBarePtrCallConv)
-    patterns.insert<BarePtrFuncOpConversion>(converter);
+    patterns.add<BarePtrFuncOpConversion>(converter);
   else
-    patterns.insert<FuncOpConversion>(converter);
+    patterns.add<FuncOpConversion>(converter);
 }
 
-void mlir::populateStdToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+void mlir::populateStdToLLVMConversionPatterns(LLVMTypeConverter &converter,
+                                               RewritePatternSet &patterns) {
   populateStdToLLVMFuncOpConversionPattern(converter, patterns);
   populateStdToLLVMNonMemoryConversionPatterns(converter, patterns);
   populateStdToLLVMMemoryConversionPatterns(converter, patterns);
@@ -3975,12 +4073,16 @@ struct LLVMLoweringPass : public ConvertStandardToLLVMBase<LLVMLoweringPass> {
 
     ModuleOp m = getOperation();
 
-    LowerToLLVMOptions options = {useBarePtrCallConv, emitCWrappers,
-                                  indexBitwidth, useAlignedAlloc,
-                                  llvm::DataLayout(this->dataLayout)};
+    LowerToLLVMOptions options(&getContext(), DataLayout(m));
+    options.useBarePtrCallConv = useBarePtrCallConv;
+    options.emitCWrappers = emitCWrappers;
+    if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
+      options.overrideIndexBitwidth(indexBitwidth);
+    options.useAlignedAlloc = useAlignedAlloc;
+    options.dataLayout = llvm::DataLayout(this->dataLayout);
     LLVMTypeConverter typeConverter(&getContext(), options);
 
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(&getContext());
     populateStdToLLVMConversionPatterns(typeConverter, patterns);
 
     LLVMConversionTarget target(getContext());
@@ -3999,9 +4101,21 @@ mlir::LLVMConversionTarget::LLVMConversionTarget(MLIRContext &ctx)
   this->addIllegalOp<math::TanhOp>();
 }
 
+std::unique_ptr<OperationPass<ModuleOp>> mlir::createLowerToLLVMPass() {
+  return std::make_unique<LLVMLoweringPass>();
+}
+
 std::unique_ptr<OperationPass<ModuleOp>>
 mlir::createLowerToLLVMPass(const LowerToLLVMOptions &options) {
   return std::make_unique<LLVMLoweringPass>(
-      options.useBarePtrCallConv, options.emitCWrappers, options.indexBitwidth,
-      options.useAlignedAlloc, options.dataLayout);
+      options.useBarePtrCallConv, options.emitCWrappers,
+      options.getIndexBitwidth(), options.useAlignedAlloc, options.dataLayout);
+}
+
+mlir::LowerToLLVMOptions::LowerToLLVMOptions(MLIRContext *ctx)
+    : LowerToLLVMOptions(ctx, DataLayout()) {}
+
+mlir::LowerToLLVMOptions::LowerToLLVMOptions(MLIRContext *ctx,
+                                             mlir::DataLayout dl) {
+  indexBitwidth = dl.getTypeSizeInBits(IndexType::get(ctx));
 }
diff --git a/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp b/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
index 00bf6c0f157fa63fa72b81a3916f8ae58d1319c3..fc5f4746714148311a41affd449a3a961fe1f5d9 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
+++ b/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
@@ -193,11 +193,12 @@ StoreOpOfSubViewFolder<OpTy>::matchAndRewrite(OpTy storeOp,
 //===----------------------------------------------------------------------===//
 
 void mlir::populateStdLegalizationPatternsForSPIRVLowering(
-    MLIRContext *context, OwningRewritePatternList &patterns) {
-  patterns.insert<LoadOpOfSubViewFolder<memref::LoadOp>,
-                  LoadOpOfSubViewFolder<vector::TransferReadOp>,
-                  StoreOpOfSubViewFolder<memref::StoreOp>,
-                  StoreOpOfSubViewFolder<vector::TransferWriteOp>>(context);
+    RewritePatternSet &patterns) {
+  patterns.add<LoadOpOfSubViewFolder<memref::LoadOp>,
+               LoadOpOfSubViewFolder<vector::TransferReadOp>,
+               StoreOpOfSubViewFolder<memref::StoreOp>,
+               StoreOpOfSubViewFolder<vector::TransferWriteOp>>(
+      patterns.getContext());
 }
 
 //===----------------------------------------------------------------------===//
@@ -212,9 +213,8 @@ struct SPIRVLegalization final
 } // namespace
 
 void SPIRVLegalization::runOnOperation() {
-  OwningRewritePatternList patterns;
-  auto *context = &getContext();
-  populateStdLegalizationPatternsForSPIRVLowering(context, patterns);
+  RewritePatternSet patterns(&getContext());
+  populateStdLegalizationPatternsForSPIRVLowering(patterns);
   (void)applyPatternsAndFoldGreedily(getOperation()->getRegions(),
                                      std::move(patterns));
 }
diff --git a/mlir/lib/Conversion/StandardToSPIRV/StandardToSPIRV.cpp b/mlir/lib/Conversion/StandardToSPIRV/StandardToSPIRV.cpp
index ed1b72c607238309ae7ca80c913eee012321521c..ed66252e20ae591849a575fe6cfe85a3b45361ed 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/StandardToSPIRV.cpp
+++ b/mlir/lib/Conversion/StandardToSPIRV/StandardToSPIRV.cpp
@@ -1017,7 +1017,7 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp,
                                                    srcBits, dstBits, rewriter);
   Value spvLoadOp = rewriter.create<spirv::LoadOp>(
       loc, dstType, adjustedPtr,
-      loadOp->getAttrOfType<IntegerAttr>(
+      loadOp->getAttrOfType<spirv::MemoryAccessAttr>(
           spirv::attributeName<spirv::MemoryAccess>()),
       loadOp->getAttrOfType<IntegerAttr>("alignment"));
 
@@ -1224,10 +1224,11 @@ XOrOpPattern::matchAndRewrite(XOrOp xorOp, ArrayRef<Value> operands,
 //===----------------------------------------------------------------------===//
 
 namespace mlir {
-void populateStandardToSPIRVPatterns(MLIRContext *context,
-                                     SPIRVTypeConverter &typeConverter,
-                                     OwningRewritePatternList &patterns) {
-  patterns.insert<
+void populateStandardToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                     RewritePatternSet &patterns) {
+  MLIRContext *context = patterns.getContext();
+
+  patterns.add<
       // Math dialect operations.
       // TODO: Move to separate pass.
       UnaryAndBinaryOpPattern<math::CosOp, spirv::GLSLCosOp>,
@@ -1289,16 +1290,15 @@ void populateStandardToSPIRVPatterns(MLIRContext *context,
 
   // Give CmpFOpNanKernelPattern a higher benefit so it can prevail when Kernel
   // capability is available.
-  patterns.insert<CmpFOpNanKernelPattern>(typeConverter, context,
-                                          /*benefit=*/2);
+  patterns.add<CmpFOpNanKernelPattern>(typeConverter, context,
+                                       /*benefit=*/2);
 }
 
-void populateTensorToSPIRVPatterns(MLIRContext *context,
-                                   SPIRVTypeConverter &typeConverter,
+void populateTensorToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
                                    int64_t byteCountThreshold,
-                                   OwningRewritePatternList &patterns) {
-  patterns.insert<TensorExtractPattern>(typeConverter, context,
-                                        byteCountThreshold);
+                                   RewritePatternSet &patterns) {
+  patterns.add<TensorExtractPattern>(typeConverter, patterns.getContext(),
+                                     byteCountThreshold);
 }
 
 } // namespace mlir
diff --git a/mlir/lib/Conversion/StandardToSPIRV/StandardToSPIRVPass.cpp b/mlir/lib/Conversion/StandardToSPIRV/StandardToSPIRVPass.cpp
index ce8419b407190db81bc0d667b0e1487e00a827ff..c738537f743827711bbf91041527394ba726879c 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/StandardToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/StandardToSPIRV/StandardToSPIRVPass.cpp
@@ -35,11 +35,11 @@ void ConvertStandardToSPIRVPass::runOnOperation() {
       spirv::SPIRVConversionTarget::get(targetAttr);
 
   SPIRVTypeConverter typeConverter(targetAttr);
-  OwningRewritePatternList patterns;
-  populateStandardToSPIRVPatterns(context, typeConverter, patterns);
-  populateTensorToSPIRVPatterns(context, typeConverter,
+  RewritePatternSet patterns(context);
+  populateStandardToSPIRVPatterns(typeConverter, patterns);
+  populateTensorToSPIRVPatterns(typeConverter,
                                 /*byteCountThreshold=*/64, patterns);
-  populateBuiltinFuncToSPIRVPatterns(context, typeConverter, patterns);
+  populateBuiltinFuncToSPIRVPatterns(typeConverter, patterns);
 
   if (failed(applyPartialConversion(module, *target, std::move(patterns))))
     return signalPassFailure();
diff --git a/mlir/lib/Conversion/TosaToLinalg/CMakeLists.txt b/mlir/lib/Conversion/TosaToLinalg/CMakeLists.txt
index 8a53b9da025bb4d112fb14c7a8a55bdc711ebacb..7cc1721bb0fb8a5de3829075384411dd4b906532 100644
--- a/mlir/lib/Conversion/TosaToLinalg/CMakeLists.txt
+++ b/mlir/lib/Conversion/TosaToLinalg/CMakeLists.txt
@@ -14,7 +14,9 @@ add_mlir_conversion_library(MLIRTosaToLinalg
   MLIRLinalg
   MLIRLinalgUtils
   MLIRMath
+  MLIRMemRef
   MLIRPass
+  MLIRTensor
   MLIRTosa
   MLIRTosaTransforms
   MLIRSupport
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 2fe4aa31e482338b5d61900f692e28ef00922624..4dbd8879d6a0a6c1f0dcb1b56a9ef0ac4d19dd65 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -13,7 +13,9 @@
 #include "mlir/Conversion/TosaToLinalg/TosaToLinalg.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
 #include "mlir/Dialect/Math/IR/Math.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
@@ -31,22 +33,49 @@ static SmallVector<StringRef> getNParallelLoopsAttrs(unsigned nParallelLoops) {
 template <typename T>
 static mlir::ConstantOp
 createConstFromIntAttribute(Operation *op, std::string attrName,
-                            Type requiredAttrType, PatternRewriter &rewriter) {
+                            Type requiredAttrType, OpBuilder &rewriter) {
   auto castedN = static_cast<T>(
       op->getAttr(attrName).cast<IntegerAttr>().getValue().getSExtValue());
   return rewriter.create<mlir::ConstantOp>(
       op->getLoc(), IntegerAttr::get(requiredAttrType, castedN));
 }
 
+template <typename T>
+static void getValuesFromIntArrayAttribute(ArrayAttr attr,
+                                           SmallVector<T> &arrayValues) {
+  for (Attribute val : attr.getValue()) {
+    arrayValues.push_back(val.cast<IntegerAttr>().getValue().getSExtValue());
+  }
+}
+
+// Generates an affine map for parallel operations on a given type. This
+// performs implicit broadcasting across any dimension of size-1.
+static AffineMap createAffineMapForType(ShapedType type,
+                                        PatternRewriter &rewriter) {
+  unsigned rank = type.getRank();
+  auto shape = type.getShape();
+  SmallVector<AffineExpr, 4> dimExprs;
+  dimExprs.reserve(rank);
+  for (unsigned i = 0; i < rank; ++i) {
+    // If the dimension is one we can broadcast the input with a constant
+    // affine expression.
+    if (shape[i] == 1)
+      dimExprs.push_back(rewriter.getAffineConstantExpr(0));
+    else
+      dimExprs.push_back(rewriter.getAffineDimExpr(i));
+  }
+  return AffineMap::get(/*dimCount=*/rank, /*symbolCount=*/0, dimExprs,
+                        rewriter.getContext());
+}
+
 template <typename T, typename P>
-static mlir::SelectOp clampHelper(Operation *op, ValueRange args,
-                                  mlir::ConstantOp min, mlir::ConstantOp max,
-                                  P pred, PatternRewriter &rewriter) {
-  Location loc = op->getLoc();
-  auto smallerThanMin = rewriter.create<T>(loc, pred, args[0], min);
+static mlir::SelectOp clampHelper(Location loc, Value arg, mlir::ConstantOp min,
+                                  mlir::ConstantOp max, P pred,
+                                  OpBuilder &rewriter) {
+  auto smallerThanMin = rewriter.create<T>(loc, pred, arg, min);
   auto minOrArg =
-      rewriter.create<mlir::SelectOp>(loc, smallerThanMin, min, args[0]);
-  auto largerThanMax = rewriter.create<T>(loc, pred, max, args[0]);
+      rewriter.create<mlir::SelectOp>(loc, smallerThanMin, min, arg);
+  auto largerThanMax = rewriter.create<T>(loc, pred, max, arg);
   return rewriter.create<mlir::SelectOp>(loc, largerThanMax, max, minOrArg);
 }
 
@@ -87,12 +116,39 @@ createLinalgBodyCalculationForElementwiseOp(Operation *op, ValueRange args,
   }
 
   if (isa<tosa::MulOp>(op) && elementTy.isa<IntegerType>()) {
-    auto mul =
-        rewriter.create<mlir::MulIOp>(loc, resultTypes, args[0], args[1]);
-    auto constant =
-        rewriter.create<mlir::ConstantOp>(loc, elementTy, op->getAttr("shift"));
-    return rewriter.create<mlir::SignedShiftRightOp>(loc, resultTypes, mul,
-                                                     constant);
+    Value a = args[0];
+    Value b = args[1];
+    auto shift =
+        op->getAttr("shift").cast<IntegerAttr>().getValue().getSExtValue();
+    if (shift > 0) {
+      auto shiftConst =
+          rewriter.create<ConstantIntOp>(loc, shift, /*bitwidth=*/8);
+      if (!a.getType().isInteger(32))
+        a = rewriter.create<SignExtendIOp>(loc, rewriter.getI32Type(), a);
+
+      if (!b.getType().isInteger(32))
+        b = rewriter.create<SignExtendIOp>(loc, rewriter.getI32Type(), b);
+
+      auto result = rewriter.create<tosa::ApplyScaleOp>(
+          loc, rewriter.getI32Type(), a, b, shiftConst,
+          rewriter.getBoolAttr(false));
+
+      if (elementTy.isInteger(32))
+        return result;
+
+      return rewriter.create<TruncateIOp>(loc, elementTy, result);
+    }
+
+    int aWidth = a.getType().getIntOrFloatBitWidth();
+    int bWidth = b.getType().getIntOrFloatBitWidth();
+    int cWidth = resultTypes[0].getIntOrFloatBitWidth();
+
+    if (aWidth < cWidth)
+      a = rewriter.create<SignExtendIOp>(loc, resultTypes[0], a);
+    if (bWidth < cWidth)
+      b = rewriter.create<SignExtendIOp>(loc, resultTypes[0], b);
+
+    return rewriter.create<mlir::MulIOp>(loc, resultTypes, a, b);
   }
 
   // tosa::NegateOp
@@ -113,6 +169,14 @@ createLinalgBodyCalculationForElementwiseOp(Operation *op, ValueRange args,
   if (isa<tosa::BitwiseOrOp>(op) && elementTy.isa<IntegerType>())
     return rewriter.create<mlir::OrOp>(loc, resultTypes, args);
 
+  // tosa::BitwiseNotOp
+  if (isa<tosa::BitwiseNotOp>(op) && elementTy.isa<IntegerType>()) {
+    auto allOnesAttr = rewriter.getIntegerAttr(
+        elementTy, APInt::getAllOnesValue(elementTy.getIntOrFloatBitWidth()));
+    auto allOnes = rewriter.create<ConstantOp>(loc, allOnesAttr);
+    return rewriter.create<mlir::XOrOp>(loc, resultTypes, args[0], allOnes);
+  }
+
   // tosa::BitwiseXOrOp
   if (isa<tosa::BitwiseXorOp>(op) && elementTy.isa<IntegerType>())
     return rewriter.create<mlir::XOrOp>(loc, resultTypes, args);
@@ -121,10 +185,29 @@ createLinalgBodyCalculationForElementwiseOp(Operation *op, ValueRange args,
   if (isa<tosa::LogicalLeftShiftOp>(op) && elementTy.isa<IntegerType>())
     return rewriter.create<mlir::ShiftLeftOp>(loc, resultTypes, args);
 
-  // tosa::LogicalrightShiftOp
+  // tosa::LogicalRightShiftOp
   if (isa<tosa::LogicalRightShiftOp>(op) && elementTy.isa<IntegerType>())
     return rewriter.create<mlir::UnsignedShiftRightOp>(loc, resultTypes, args);
 
+  // tosa::LogicalAnd
+  if (isa<tosa::LogicalAndOp>(op) && elementTy.isInteger(1))
+    return rewriter.create<mlir::AndOp>(loc, resultTypes, args);
+
+  // tosa::LogicalNot
+  if (isa<tosa::LogicalNotOp>(op) && elementTy.isInteger(1)) {
+    auto one = rewriter.create<mlir::ConstantOp>(
+        loc, rewriter.getIntegerAttr(elementTy, 1));
+    return rewriter.create<mlir::XOrOp>(loc, resultTypes, args[0], one);
+  }
+
+  // tosa::LogicalOr
+  if (isa<tosa::LogicalOrOp>(op) && elementTy.isInteger(1))
+    return rewriter.create<mlir::OrOp>(loc, resultTypes, args);
+
+  // tosa::LogicalXor
+  if (isa<tosa::LogicalXorOp>(op) && elementTy.isInteger(1))
+    return rewriter.create<mlir::XOrOp>(loc, resultTypes, args);
+
   // tosa::PowOp
   if (isa<tosa::PowOp>(op) && elementTy.isa<FloatType>())
     return rewriter.create<mlir::math::PowFOp>(loc, resultTypes, args);
@@ -210,7 +293,7 @@ createLinalgBodyCalculationForElementwiseOp(Operation *op, ValueRange args,
                                                  op->getAttr("min_fp"));
     auto max = rewriter.create<mlir::ConstantOp>(loc, elementTy,
                                                  op->getAttr("max_fp"));
-    return clampHelper<mlir::CmpFOp>(op, args, min, max, CmpFPredicate::OLT,
+    return clampHelper<mlir::CmpFOp>(loc, args[0], min, max, CmpFPredicate::OLT,
                                      rewriter);
   }
 
@@ -219,7 +302,7 @@ createLinalgBodyCalculationForElementwiseOp(Operation *op, ValueRange args,
                                                     rewriter);
     auto max = createConstFromIntAttribute<int32_t>(op, "max_int", elementTy,
                                                     rewriter);
-    return clampHelper<mlir::CmpIOp>(op, args, min, max, CmpIPredicate::slt,
+    return clampHelper<mlir::CmpIOp>(loc, args[0], min, max, CmpIPredicate::slt,
                                      rewriter);
   }
 
@@ -229,7 +312,7 @@ createLinalgBodyCalculationForElementwiseOp(Operation *op, ValueRange args,
         rewriter.create<mlir::ConstantOp>(loc, FloatAttr::get(elementTy, 0));
     auto n = rewriter.create<mlir::ConstantOp>(loc, elementTy,
                                                op->getAttr("max_fp"));
-    return clampHelper<mlir::CmpFOp>(op, args, zero, n, CmpFPredicate::OLT,
+    return clampHelper<mlir::CmpFOp>(loc, args[0], zero, n, CmpFPredicate::OLT,
                                      rewriter);
   }
 
@@ -238,10 +321,71 @@ createLinalgBodyCalculationForElementwiseOp(Operation *op, ValueRange args,
         rewriter.create<mlir::ConstantOp>(loc, IntegerAttr::get(elementTy, 0));
     auto n = createConstFromIntAttribute<int32_t>(op, "max_int", elementTy,
                                                   rewriter);
-    return clampHelper<mlir::CmpIOp>(op, args, zero, n, CmpIPredicate::slt,
+    return clampHelper<mlir::CmpIOp>(loc, args[0], zero, n, CmpIPredicate::slt,
                                      rewriter);
   }
 
+  // tosa::CastOp
+  if (isa<tosa::CastOp>(op)) {
+    Type srcTy = elementTy;
+    Type dstTy = resultTypes.front();
+    bool bitExtend =
+        srcTy.getIntOrFloatBitWidth() < dstTy.getIntOrFloatBitWidth();
+
+    if (srcTy == dstTy)
+      return args.front();
+
+    if (srcTy.isa<FloatType>() && dstTy.isa<FloatType>() && bitExtend)
+      return rewriter.create<mlir::FPExtOp>(loc, resultTypes, args, mlir::None);
+
+    if (srcTy.isa<FloatType>() && dstTy.isa<FloatType>() && !bitExtend)
+      return rewriter.create<mlir::FPTruncOp>(loc, resultTypes, args,
+                                              mlir::None);
+
+    // 1-bit integers need to be treated as signless.
+    if (srcTy.isInteger(1) && mlir::UIToFPOp::areCastCompatible(srcTy, dstTy))
+      return rewriter.create<mlir::UIToFPOp>(loc, resultTypes, args,
+                                             mlir::None);
+
+    if (srcTy.isInteger(1) && dstTy.isa<IntegerType>() && bitExtend)
+      return rewriter.create<mlir::ZeroExtendIOp>(loc, resultTypes, args,
+                                                  mlir::None);
+
+    // All other si-to-fp conversions should be handled by SIToFP.
+    if (mlir::SIToFPOp::areCastCompatible(srcTy, dstTy))
+      return rewriter.create<mlir::SIToFPOp>(loc, resultTypes, args,
+                                             mlir::None);
+
+    // Casting to boolean, floats need to only be checked as not-equal to zero.
+    if (srcTy.isa<FloatType>() && dstTy.isInteger(1)) {
+      Value zero =
+          rewriter.create<ConstantOp>(loc, rewriter.getFloatAttr(srcTy, 0.0));
+      return rewriter.create<mlir::CmpFOp>(loc, CmpFPredicate::UNE,
+                                           args.front(), zero);
+    }
+
+    if (mlir::FPToSIOp::areCastCompatible(srcTy, dstTy))
+      return rewriter.create<mlir::FPToSIOp>(loc, resultTypes, args,
+                                             mlir::None);
+
+    // Casting to boolean, integers need to only be checked as not-equal to
+    // zero.
+    if (srcTy.isa<IntegerType>() && dstTy.isInteger(1)) {
+      Value zero =
+          rewriter.create<ConstantIntOp>(loc, 0, srcTy.getIntOrFloatBitWidth());
+      return rewriter.create<mlir::CmpIOp>(loc, CmpIPredicate::ne, args.front(),
+                                           zero);
+    }
+
+    if (srcTy.isa<IntegerType>() && dstTy.isa<IntegerType>() && bitExtend)
+      return rewriter.create<mlir::SignExtendIOp>(loc, resultTypes, args,
+                                                  mlir::None);
+
+    if (srcTy.isa<IntegerType>() && dstTy.isa<IntegerType>() && !bitExtend)
+      return rewriter.create<mlir::TruncateIOp>(loc, resultTypes, args,
+                                                mlir::None);
+  }
+
   (void)rewriter.notifyMatchFailure(
       op, "unhandled op for linalg body calculation for elementwise op");
   return nullptr;
@@ -289,21 +433,9 @@ elementwiseMatchAndRewriteHelper(Operation *operation,
   indexingMaps.reserve(operation->getNumOperands() + bodyResultTypes.size());
 
   // Input indexing maps may be broadcasted.
-  for (Type types : operation->getOperandTypes()) {
-    auto shape = types.cast<ShapedType>().getShape();
-    SmallVector<AffineExpr, 4> dimExprs;
-    dimExprs.reserve(nloops);
-    for (unsigned i = 0; i < nloops; ++i) {
-      // If the dimension is one we can broadcast the input with a constant
-      // affine expression.
-      if (shape[i] == 1)
-        dimExprs.push_back(rewriter.getAffineConstantExpr(0));
-      else
-        dimExprs.push_back(rewriter.getAffineDimExpr(i));
-    }
-    indexingMaps.push_back(AffineMap::get(/*dimCount=*/nloops,
-                                          /*symbolCount=*/0, dimExprs,
-                                          rewriter.getContext()));
+  for (Type type : operation->getOperandTypes()) {
+    indexingMaps.push_back(
+        createAffineMapForType(type.cast<ShapedType>(), rewriter));
   }
 
   indexingMaps.append(operation->getNumResults(),
@@ -317,13 +449,14 @@ elementwiseMatchAndRewriteHelper(Operation *operation,
         Value opResult = createLinalgBodyCalculationForElementwiseOp(
             operation, blockArgs.take_front(operation->getNumOperands()),
             bodyResultTypes, rewriter);
-        if (opResult) {
+        if (!opResult) {
           didEncounterError = true;
+          return;
         }
         nestedBuilder.create<linalg::YieldOp>(loc, opResult);
       });
 
-  if (!didEncounterError)
+  if (didEncounterError)
     return failure();
 
   rewriter.replaceOp(operation, linalgOp->getResults());
@@ -364,6 +497,15 @@ static Attribute createInitialValueForReduceOp(Operation *op, Type elementTy,
     return rewriter.getIntegerAttr(
         elementTy, APInt::getSignedMinValue(elementTy.getIntOrFloatBitWidth()));
 
+  if (isa<tosa::ArgMaxOp>(op) && elementTy.isa<FloatType>())
+    return rewriter.getFloatAttr(
+        elementTy, APFloat::getLargest(
+                       elementTy.cast<FloatType>().getFloatSemantics(), true));
+
+  if (isa<tosa::ArgMaxOp>(op) && elementTy.isa<IntegerType>())
+    return rewriter.getIntegerAttr(
+        elementTy, APInt::getSignedMinValue(elementTy.getIntOrFloatBitWidth()));
+
   return {};
 }
 
@@ -489,7 +631,85 @@ public:
   }
 };
 
-class ReshapeOpConverter : public OpConversionPattern<tosa::ReshapeOp> {
+class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
+public:
+  using OpConversionPattern<tosa::MatMulOp>::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(tosa::MatMulOp op, ArrayRef<Value> args,
+                  ConversionPatternRewriter &rewriter) const final {
+    tosa::MatMulOp::Adaptor adaptor(args);
+
+    Location loc = op.getLoc();
+
+    auto outputTy = op.getType().cast<ShapedType>();
+    auto outputElementTy = outputTy.getElementType();
+    auto zero_attr = rewriter.getZeroAttr(outputElementTy);
+    Value zero = rewriter.create<ConstantOp>(loc, zero_attr);
+    auto initTensor = rewriter.create<linalg::InitTensorOp>(
+        loc, outputTy.getShape(), outputTy.getElementType());
+    Value zeroTensor =
+        rewriter.create<linalg::FillOp>(loc, initTensor, zero).getResult(0);
+    rewriter.replaceOpWithNewOp<linalg::MatmulOp>(
+        op, TypeRange{op.getType()}, ValueRange{adaptor.a(), adaptor.b()},
+        ValueRange{zeroTensor});
+    return success();
+  }
+};
+
+class FullyConnectedConverter
+    : public OpConversionPattern<tosa::FullyConnectedOp> {
+public:
+  using OpConversionPattern<tosa::FullyConnectedOp>::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(tosa::FullyConnectedOp op, ArrayRef<Value> args,
+                  ConversionPatternRewriter &rewriter) const final {
+    tosa::FullyConnectedOp::Adaptor adaptor(args);
+
+    Location loc = op.getLoc();
+    auto outputTy = op.getType().cast<ShapedType>();
+    auto biasTy = op->getOperand(2).getType().cast<ShapedType>();
+
+    // Reshaping the bias from n to [1, n] for broadcasting
+    SmallVector<int64_t> biasShapeReshaped;
+    biasShapeReshaped.push_back(1);
+    biasShapeReshaped.push_back(biasTy.getShape()[0]);
+
+    RankedTensorType reshapedBias =
+        RankedTensorType::get(biasShapeReshaped, outputTy.getElementType());
+    auto reshapeResult =
+        rewriter.create<tosa::ReshapeOp>(loc, reshapedBias, args[2])
+            ->getResult(0);
+
+    // Creating maps for the output of MatMul and the bias
+    SmallVector<AffineMap, 4> indexingMaps;
+    indexingMaps.push_back(createAffineMapForType(reshapedBias, rewriter));
+    indexingMaps.push_back(rewriter.getMultiDimIdentityMap(outputTy.getRank()));
+
+    auto initTensor =
+        rewriter
+            .create<linalg::InitTensorOp>(loc, outputTy.getShape(),
+                                          outputTy.getElementType())
+            ->getResults();
+
+    auto linalgOp =
+        rewriter
+            .create<linalg::GenericOp>(
+                loc, outputTy, reshapeResult, initTensor, indexingMaps,
+                getNParallelLoopsAttrs(outputTy.getRank()),
+                [&](OpBuilder &nested_builder, Location nested_loc,
+                    ValueRange args) {
+                  nested_builder.create<linalg::YieldOp>(loc, *args.begin());
+                })
+            ->getResults();
+
+    rewriter.replaceOpWithNewOp<linalg::MatmulOp>(
+        op, TypeRange{op.getType()},
+        ValueRange{adaptor.input(), adaptor.weight()}, linalgOp);
+    return success();
+  }
+};
+
+class ReshapeConverter : public OpConversionPattern<tosa::ReshapeOp> {
 public:
   using OpConversionPattern<tosa::ReshapeOp>::OpConversionPattern;
 
@@ -501,6 +721,11 @@ public:
     ShapedType operandTy = operands.input1().getType().cast<ShapedType>();
     ShapedType resultTy = reshape.getType().template cast<ShapedType>();
 
+    if (operandTy == resultTy) {
+      rewriter.replaceOp(reshape, args[0]);
+      return success();
+    }
+
     if (!operandTy.hasStaticShape() || !resultTy.hasStaticShape())
       return failure();
 
@@ -631,6 +856,142 @@ public:
   }
 };
 
+class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
+public:
+  using OpRewritePattern<tosa::RescaleOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tosa::RescaleOp op,
+                                PatternRewriter &rewriter) const final {
+    auto loc = op.getLoc();
+    auto input = op.input();
+    auto inputTy = op.input().getType().cast<ShapedType>();
+    auto outputTy = op.output().getType().cast<ShapedType>();
+    unsigned rank = inputTy.getRank();
+
+    if (!outputTy.hasStaticShape())
+      return rewriter.notifyMatchFailure(
+          op, "tosa to linalg conversion expects statically shaped tensors");
+
+    // The shift and multiplier values.
+    SmallVector<int32_t> multiplierValues;
+    getValuesFromIntArrayAttribute(op.multiplier(), multiplierValues);
+
+    SmallVector<int8_t> shiftValues;
+    getValuesFromIntArrayAttribute(op.shift(), shiftValues);
+
+    // Double round only occurs if shift is greater than 31, check that this
+    // is ever true.
+    bool doubleRound =
+        op.double_round() &&
+        llvm::any_of(shiftValues, [](int32_t v) { return v > 31; });
+
+    // We need to broadcast along the last dimension, so make all dims 1.
+    SmallVector<int64_t> multiplierShape;
+    multiplierShape.resize(rank, 1);
+
+    SmallVector<int64_t> shiftShape;
+    shiftShape.resize(rank, 1);
+
+    // Set the channel dimension to match the number of shift/broadcast
+    // channels.
+    if (!multiplierShape.empty())
+      multiplierShape.back() = multiplierValues.size();
+    if (!shiftShape.empty())
+      shiftShape.back() = shiftValues.size();
+
+    // Create the tensor types.
+    auto multiplierType =
+        RankedTensorType::get(multiplierShape, rewriter.getI32Type());
+    auto shiftType =
+        RankedTensorType::get(shiftShape, rewriter.getIntegerType(8));
+
+    auto multiplierConst = rewriter.create<ConstantOp>(
+        loc, DenseIntElementsAttr::get(multiplierType, multiplierValues));
+
+    auto shiftConst = rewriter.create<ConstantOp>(
+        loc, DenseIntElementsAttr::get(shiftType, shiftValues));
+
+    // Construct the indexing maps needed for linalg.generic ops.
+    SmallVector<Type> bodyArgTypes = {getElementTypeOrSelf(inputTy),
+                                      rewriter.getI32Type(),
+                                      rewriter.getI32Type()};
+    Value initTensor = rewriter.create<linalg::InitTensorOp>(
+        loc, ArrayRef<Value>({}), outputTy.getShape(),
+        outputTy.getElementType());
+
+    SmallVector<AffineMap, 4> indexingMaps;
+
+    // Indexing map for input values.
+    indexingMaps.push_back(rewriter.getMultiDimIdentityMap(rank));
+
+    // Shift and multiplier will need to broadcast across their non channel
+    // values.
+    indexingMaps.push_back(createAffineMapForType(multiplierType, rewriter));
+    indexingMaps.push_back(createAffineMapForType(shiftType, rewriter));
+
+    // Indexing maps for output values.
+    indexingMaps.push_back(rewriter.getMultiDimIdentityMap(rank));
+
+    auto linalgOp = rewriter.create<linalg::GenericOp>(
+        loc, outputTy, ValueRange{input, multiplierConst, shiftConst},
+        ValueRange{initTensor}, indexingMaps, getNParallelLoopsAttrs(rank),
+        [&](OpBuilder &nestedBuilder, Location nestedLoc,
+            ValueRange blockArgs) {
+          // For now we do all of our math in 64-bit. This is not optimal but
+          // should be correct for now, consider computing correct bit depth
+          // later.
+          auto inputZp = createConstFromIntAttribute<int32_t>(
+              op, "input_zp", nestedBuilder.getI32Type(), nestedBuilder);
+          auto outputZp = createConstFromIntAttribute<int32_t>(
+              op, "output_zp", nestedBuilder.getI32Type(), nestedBuilder);
+
+          Value value = blockArgs[0];
+          Value multiplier = blockArgs[1];
+          Value shift = blockArgs[2];
+
+          if (value.getType().getIntOrFloatBitWidth() < 32) {
+            value = nestedBuilder.create<SignExtendIOp>(
+                nestedLoc, nestedBuilder.getI32Type(), value);
+          }
+
+          value = nestedBuilder.create<SubIOp>(nestedLoc, value, inputZp);
+
+          value = nestedBuilder.create<tosa::ApplyScaleOp>(
+              loc, nestedBuilder.getI32Type(), value, multiplier, shift,
+              nestedBuilder.getBoolAttr(doubleRound));
+
+          // Move to the new zero-point.
+          value = nestedBuilder.create<AddIOp>(nestedLoc, value, outputZp);
+
+          // Saturate to the output size.
+          IntegerType outIntType =
+              blockArgs.back().getType().cast<IntegerType>();
+          unsigned outBitWidth = outIntType.getWidth();
+          auto intMin = nestedBuilder.create<ConstantOp>(
+              loc, nestedBuilder.getIntegerAttr(
+                       nestedBuilder.getI32Type(),
+                       APInt::getSignedMinValue(outBitWidth).getSExtValue()));
+          auto intMax = nestedBuilder.create<ConstantOp>(
+              loc, nestedBuilder.getIntegerAttr(
+                       nestedBuilder.getI32Type(),
+                       APInt::getSignedMaxValue(outBitWidth).getSExtValue()));
+
+          value = clampHelper<mlir::CmpIOp>(nestedLoc, value, intMin, intMax,
+                                            CmpIPredicate::slt, nestedBuilder);
+
+          if (outIntType.getWidth() < 32) {
+            value =
+                nestedBuilder.create<TruncateIOp>(nestedLoc, outIntType, value);
+          }
+
+          nestedBuilder.create<linalg::YieldOp>(loc, value);
+        });
+
+    rewriter.replaceOp(op, linalgOp->getResults());
+    return success();
+  }
+};
+
 // At the codegen level any identity operations should be removed. Any cases
 // where identity is load-bearing (e.g. cross device computation) should be
 // handled before lowering to codegen.
@@ -657,11 +1018,369 @@ public:
   }
 };
 
+struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
+  using OpConversionPattern<tosa::ConcatOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(tosa::ConcatOp op, ArrayRef<Value> args,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto resultType = op.getType().dyn_cast<RankedTensorType>();
+    if (!resultType || !resultType.hasStaticShape()) {
+      return rewriter.notifyMatchFailure(op,
+                                         "expected static shaped tensor type");
+    }
+
+    Location loc = op.getLoc();
+    int axis = op.axis();
+    Value axisValue =
+        rewriter.create<ConstantOp>(loc, rewriter.getIndexAttr(axis));
+    int rank = resultType.getRank();
+    SmallVector<Value, 3> offsets, sizes, strides;
+    sizes.reserve(rank);
+    strides.resize(rank, rewriter.create<ConstantIndexOp>(loc, 1));
+    offsets.resize(rank, rewriter.create<ConstantIndexOp>(loc, 0));
+
+    for (int i = 0; i < rank; ++i) {
+      sizes.push_back(rewriter.create<memref::DimOp>(loc, args[0], i));
+    }
+
+    Value resultDimSize = sizes[axis];
+    for (auto arg : args.drop_front()) {
+      auto size = rewriter.create<memref::DimOp>(loc, arg, axisValue);
+      resultDimSize = rewriter.create<AddIOp>(loc, resultDimSize, size);
+    }
+    sizes[axis] = resultDimSize;
+
+    Value result = rewriter.create<linalg::InitTensorOp>(
+        loc, resultType.getShape(), resultType.getElementType());
+
+    for (auto arg : args) {
+      sizes[axis] = rewriter.create<memref::DimOp>(loc, arg, axisValue);
+      result = rewriter.create<SubTensorInsertOp>(loc, arg, result, offsets,
+                                                  sizes, strides);
+      offsets[axis] = rewriter.create<AddIOp>(loc, offsets[axis], sizes[axis]);
+    }
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
+class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
+public:
+  using OpRewritePattern<tosa::ReverseOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tosa::ReverseOp op,
+                                PatternRewriter &rewriter) const final {
+    auto loc = op.getLoc();
+    Value input = op.input();
+    auto inputTy = input.getType().template cast<ShapedType>();
+    auto resultTy = op.getType().template cast<ShapedType>();
+    auto rank = resultTy.getRank();
+    auto axis = op.axis();
+
+    if (!inputTy.hasStaticShape())
+      return rewriter.notifyMatchFailure(
+          op, "No initial value found for reduction operation");
+
+    // First fill the output buffer with the init value.
+    auto initTensor = rewriter
+                          .create<linalg::InitTensorOp>(
+                              loc, ArrayRef<Value>({}), inputTy.getShape(),
+                              inputTy.getElementType())
+                          .result();
+
+    SmallVector<AffineExpr, 2> inputExprs;
+    inputExprs.resize(resultTy.getRank());
+
+    for (int i = 0; i < rank; i++)
+      inputExprs[i] = rewriter.getAffineDimExpr(i);
+
+    inputExprs[axis] =
+        rewriter.getAffineConstantExpr(inputTy.getDimSize(axis) - 1) -
+        inputExprs[axis];
+
+    SmallVector<AffineMap, 2> affineMaps = {
+        AffineMap::get(resultTy.getRank(), /*symbolCount=*/0, inputExprs,
+                       rewriter.getContext()),
+        rewriter.getMultiDimIdentityMap(resultTy.getRank())};
+
+    rewriter.replaceOpWithNewOp<linalg::GenericOp>(
+        op, resultTy, op.input(), ValueRange{initTensor}, affineMaps,
+        getNParallelLoopsAttrs(resultTy.getRank()),
+        [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
+          nestedBuilder.create<linalg::YieldOp>(op.getLoc(), *args.begin());
+        });
+    return success();
+  }
+};
+
+// This converter translate a tile operation to a reshape, broadcast, reshape.
+// The first reshape minimally expands each tiled dimension to include a
+// proceding size-1 dim. This dim is then broadcasted to the appropriate
+// multiple.
+struct TileConverter : public OpConversionPattern<tosa::TileOp> {
+  using OpConversionPattern<tosa::TileOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(tosa::TileOp op, ArrayRef<Value> args,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op.getLoc();
+    auto input = op.input1();
+    auto inputTy = input.getType().cast<ShapedType>();
+    auto inputShape = inputTy.getShape();
+    auto resultTy = op.getType().cast<ShapedType>();
+    auto elementTy = inputTy.getElementType();
+    int64_t rank = inputTy.getRank();
+
+    if (!inputTy.hasStaticShape() || !resultTy.hasStaticShape())
+      return failure();
+
+    SmallVector<int64_t> multiples;
+    getValuesFromIntArrayAttribute(op.multiples(), multiples);
+
+    llvm::SmallVector<int64_t, 4> reshapeShape;
+    reshapeShape.reserve(rank * 2);
+    for (int i = 0; i < rank; i++) {
+      reshapeShape.push_back(1);
+      reshapeShape.push_back(inputShape[i]);
+    }
+
+    ShapedType reshapeTy = RankedTensorType::get(reshapeShape, elementTy);
+    Value reshape = rewriter.create<tosa::ReshapeOp>(
+        loc, reshapeTy, input, rewriter.getI64ArrayAttr(reshapeTy.getShape()));
+
+    // Broadcast the newly added dimensions to their appropriate multiple.
+    SmallVector<int64_t, 2> genericShape;
+    for (int i = 0; i < rank; i++) {
+      genericShape.push_back(multiples[i]);
+      genericShape.push_back(inputShape[i]);
+    }
+
+    auto initTensor = rewriter.create<linalg::InitTensorOp>(
+        op.getLoc(), ArrayRef<Value>({}), genericShape, elementTy);
+
+    SmallVector<AffineMap, 2> affineMaps = {
+        createAffineMapForType(reshapeTy, rewriter),
+        rewriter.getMultiDimIdentityMap(genericShape.size())};
+
+    auto genericOp = rewriter.create<linalg::GenericOp>(
+        loc, RankedTensorType::get(genericShape, elementTy), reshape,
+        ValueRange{initTensor}, affineMaps,
+        getNParallelLoopsAttrs(genericShape.size()),
+        [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
+          nestedBuilder.create<linalg::YieldOp>(op.getLoc(), *args.begin());
+        });
+
+    rewriter.replaceOpWithNewOp<tosa::ReshapeOp>(
+        op, resultTy, genericOp.getResult(0),
+        rewriter.getI64ArrayAttr(resultTy.getShape()));
+    return success();
+  }
+};
+
+class PadConverter : public OpRewritePattern<tosa::PadOp> {
+public:
+  using OpRewritePattern<tosa::PadOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tosa::PadOp padOp,
+                                PatternRewriter &rewriter) const final {
+    auto loc = padOp.getLoc();
+    auto input = padOp.input1();
+    auto padding = padOp.padding();
+
+    ShapedType inputTy = input.getType().cast<ShapedType>();
+    ShapedType paddingTy = padding.getType().cast<ShapedType>();
+    Type elementTy = inputTy.getElementType();
+    int64_t rank = inputTy.getRank();
+
+    if (!inputTy.hasStaticShape() || !paddingTy.hasStaticShape()) {
+      return rewriter.notifyMatchFailure(
+          padOp,
+          "Pad converter requires static shaped input / padding values.");
+    }
+
+    Value lowIndex = rewriter.create<ConstantOp>(loc, rewriter.getIndexAttr(0));
+    Value highIndex =
+        rewriter.create<ConstantOp>(loc, rewriter.getIndexAttr(1));
+
+    SmallVector<OpFoldResult, 3> lowValues;
+    SmallVector<OpFoldResult, 3> highValues;
+
+    lowValues.reserve(rank);
+    highValues.reserve(rank);
+
+    for (int i = 0; i < rank; i++) {
+      Value inputIndex = rewriter.createOrFold<ConstantIndexOp>(loc, i);
+      Value lowVal = rewriter.createOrFold<tensor::ExtractOp>(
+          loc, padding, ValueRange({inputIndex, lowIndex}));
+      Value highVal = rewriter.createOrFold<tensor::ExtractOp>(
+          loc, padding, ValueRange({inputIndex, highIndex}));
+
+      lowVal = rewriter.createOrFold<IndexCastOp>(loc, rewriter.getIndexType(),
+                                                  lowVal);
+      highVal = rewriter.createOrFold<IndexCastOp>(loc, rewriter.getIndexType(),
+                                                   highVal);
+
+      lowValues.push_back(lowVal);
+      highValues.push_back(highVal);
+    }
+
+    Attribute constantAttr;
+    if (elementTy.isa<FloatType>())
+      constantAttr = rewriter.getFloatAttr(elementTy, 0.0);
+    else if (elementTy.isa<IntegerType>() && !padOp.quantization_info())
+      constantAttr = rewriter.getIntegerAttr(elementTy, 0);
+    else if (elementTy.isa<IntegerType>() && padOp.quantization_info()) {
+      auto value = padOp.quantization_info().getValue().input_zp().getValue();
+      constantAttr = rewriter.getIntegerAttr(elementTy, value.getZExtValue());
+    }
+
+    if (!constantAttr) {
+      return rewriter.notifyMatchFailure(
+          padOp,
+          "tosa.pad to linalg lowering encountered an unknown element type");
+    }
+
+    Value constant = rewriter.create<ConstantOp>(loc, constantAttr);
+
+    auto newPadOp = linalg::PadTensorOp::createPadScalarOp(
+        padOp.getType(), input, constant, lowValues, highValues, loc, rewriter);
+
+    rewriter.replaceOp(padOp, newPadOp.getResult());
+    return success();
+  }
+};
+
+// Tosa argmax lowering represents the ArgMax op as an linalg.indexed_generic
+// op, producing two output buffers.
+//
+// The first output buffer contains the index of the found maximum value. It is
+// initialized to 0 and is resulting integer type.
+//
+// The second output buffer contains the maximum value found. It is initialized
+// to the minimum representable value of the input element type. After being
+// populated by indexed_generic, this buffer is disgarded as only the index is
+// requested.
+//
+// The indexed_generic op updates both the maximum value and index if the
+// current value exceeds the running max.
+class ArgMaxConverter : public OpRewritePattern<tosa::ArgMaxOp> {
+public:
+  using OpRewritePattern<tosa::ArgMaxOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tosa::ArgMaxOp argmaxOp,
+                                PatternRewriter &rewriter) const final {
+    auto loc = argmaxOp.getLoc();
+    Value input = argmaxOp.input();
+    auto inputTy = input.getType().cast<ShapedType>();
+    auto resultTy = argmaxOp.output().getType().cast<ShapedType>();
+    auto inElementTy = inputTy.getElementType();
+    auto outElementTy = resultTy.getElementType();
+    int axis = argmaxOp.axis();
+    auto resultMaxTy = RankedTensorType::get(resultTy.getShape(), inElementTy);
+
+    if (!inputTy.hasStaticShape())
+      return rewriter.notifyMatchFailure(
+          argmaxOp,
+          "tosa.arg_max to linalg.* requires statically shaped input");
+
+    if (!outElementTy.isa<IntegerType>())
+      return rewriter.notifyMatchFailure(
+          argmaxOp,
+          "tosa.arg_max to linalg.* requires integer-like result type");
+
+    // First fill the output buffer for the index.
+    auto initTensorIdx =
+        rewriter
+            .create<linalg::InitTensorOp>(loc, ArrayRef<Value>({}),
+                                          resultTy.getShape(), outElementTy)
+            .result();
+    auto fillValueIdx = rewriter.create<ConstantOp>(
+        loc, rewriter.getIntegerAttr(outElementTy, 0));
+    auto filledTensorIdx =
+        rewriter.create<linalg::FillOp>(loc, initTensorIdx, fillValueIdx)
+            .result();
+
+    // Second fill the output buffer for the running max.
+    auto initTensorMax =
+        rewriter
+            .create<linalg::InitTensorOp>(loc, ArrayRef<Value>({}),
+                                          resultTy.getShape(), inElementTy)
+            .result();
+    auto fillValueMaxAttr =
+        createInitialValueForReduceOp(argmaxOp, inElementTy, rewriter);
+
+    if (!fillValueMaxAttr)
+      return rewriter.notifyMatchFailure(
+          argmaxOp, "unsupported tosa.argmax element type");
+
+    auto fillValueMax = rewriter.create<ConstantOp>(loc, fillValueMaxAttr);
+    auto filledTensorMax =
+        rewriter.create<linalg::FillOp>(loc, initTensorMax, fillValueMax)
+            .result();
+
+    // We need to reduce along the arg-max axis, with parallel operations along
+    // the rest.
+    SmallVector<StringRef, 4> iteratorTypes;
+    iteratorTypes.resize(inputTy.getRank(), getParallelIteratorTypeName());
+    iteratorTypes[axis] = getReductionIteratorTypeName();
+
+    SmallVector<AffineExpr, 2> srcExprs;
+    SmallVector<AffineExpr, 2> dstExprs;
+    for (int i = 0, rank = inputTy.getRank(); i != rank; ++i) {
+      srcExprs.push_back(mlir::getAffineDimExpr(i, rewriter.getContext()));
+      if (axis != i)
+        dstExprs.push_back(mlir::getAffineDimExpr(i, rewriter.getContext()));
+    }
+
+    bool didEncounterError = false;
+    auto maps = AffineMap::inferFromExprList({srcExprs, dstExprs, dstExprs});
+    auto linalgOp = rewriter.create<linalg::IndexedGenericOp>(
+        loc, ArrayRef<Type>({resultTy, resultMaxTy}), input,
+        ValueRange({filledTensorIdx, filledTensorMax}), maps, iteratorTypes,
+        [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange ivs,
+            ValueRange blockArgs) {
+          auto newValue = blockArgs[0];
+          auto oldIndex = blockArgs[1];
+          auto oldValue = blockArgs[2];
+
+          Value newIndex = rewriter.create<IndexCastOp>(
+              nestedLoc, oldIndex.getType(), ivs[axis]);
+
+          Value predicate;
+          if (inElementTy.isa<FloatType>()) {
+            predicate = rewriter.create<mlir::CmpFOp>(
+                nestedLoc, CmpFPredicate::OGT, newValue, oldValue);
+          } else if (inElementTy.isa<IntegerType>()) {
+            predicate = rewriter.create<mlir::CmpIOp>(
+                nestedLoc, CmpIPredicate::sgt, newValue, oldValue);
+          } else {
+            didEncounterError = true;
+            return;
+          }
+
+          auto resultMax = rewriter.create<mlir::SelectOp>(nestedLoc, predicate,
+                                                           newValue, oldValue);
+          auto resultIndex = rewriter.create<mlir::SelectOp>(
+              nestedLoc, predicate, newIndex, oldIndex);
+          nestedBuilder.create<linalg::YieldOp>(
+              nestedLoc, ValueRange({resultIndex, resultMax}));
+        });
+
+    if (didEncounterError)
+      return rewriter.notifyMatchFailure(
+          argmaxOp, "unsupported tosa.argmax element type");
+
+    rewriter.replaceOp(argmaxOp, linalgOp.getResult(0));
+    return success();
+  }
+};
+
 } // namespace
 
 void mlir::tosa::populateTosaToLinalgOnTensorsConversionPatterns(
-    MLIRContext *context, OwningRewritePatternList *patterns) {
-  patterns->insert<
+    RewritePatternSet *patterns) {
+  patterns->add<
       PointwiseConverter<tosa::AddOp>, PointwiseConverter<tosa::SubOp>,
       PointwiseConverter<tosa::MulOp>, PointwiseConverter<tosa::NegateOp>,
       PointwiseConverter<tosa::PowOp>, PointwiseConverter<tosa::RsqrtOp>,
@@ -669,7 +1388,12 @@ void mlir::tosa::populateTosaToLinalgOnTensorsConversionPatterns(
       PointwiseConverter<tosa::AbsOp>, PointwiseConverter<tosa::TanhOp>,
       PointwiseConverter<tosa::BitwiseAndOp>,
       PointwiseConverter<tosa::BitwiseOrOp>,
+      PointwiseConverter<tosa::BitwiseNotOp>,
       PointwiseConverter<tosa::BitwiseXorOp>,
+      PointwiseConverter<tosa::LogicalAndOp>,
+      PointwiseConverter<tosa::LogicalNotOp>,
+      PointwiseConverter<tosa::LogicalOrOp>,
+      PointwiseConverter<tosa::LogicalXorOp>, PointwiseConverter<tosa::CastOp>,
       PointwiseConverter<tosa::LogicalLeftShiftOp>,
       PointwiseConverter<tosa::LogicalRightShiftOp>,
       PointwiseConverter<tosa::SelectOp>, PointwiseConverter<tosa::GreaterOp>,
@@ -680,6 +1404,8 @@ void mlir::tosa::populateTosaToLinalgOnTensorsConversionPatterns(
       IdentityNConverter<tosa::IdentityOp>,
       IdentityNConverter<tosa::IdentityNOp>, ReduceConverter<tosa::ReduceMinOp>,
       ReduceConverter<tosa::ReduceMaxOp>, ReduceConverter<tosa::ReduceSumOp>,
-      ReduceConverter<tosa::ReduceProdOp>, ReshapeOpConverter,
-      TransposeConverter>(context);
+      ReduceConverter<tosa::ReduceProdOp>, ArgMaxConverter, ConcatConverter, PadConverter,
+      ReshapeConverter, RescaleConverter, ReverseConverter, TileConverter,
+      TransposeConverter, MatMulConverter, FullyConnectedConverter>(
+        patterns->getContext());
 }
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
index 8ccf83529457ece7dc35b9adb8e21d1d2dd7c318..baf9e575a47368f6e15f2825a299a4a60c3372ff 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
@@ -14,7 +14,9 @@
 #include "mlir/Conversion/TosaToLinalg/TosaToLinalg.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
 #include "mlir/Dialect/Math/IR/Math.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Transforms/PassDetail.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
@@ -31,20 +33,28 @@ struct TosaToLinalgOnTensors
     : public TosaToLinalgOnTensorsBase<TosaToLinalgOnTensors> {
 public:
   void getDependentDialects(DialectRegistry &registry) const override {
-    registry
-        .insert<linalg::LinalgDialect, math::MathDialect, StandardOpsDialect>();
+    registry.insert<linalg::LinalgDialect, math::MathDialect,
+                    memref::MemRefDialect, StandardOpsDialect,
+                    tensor::TensorDialect>();
   }
 
   void runOnFunction() override {
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(&getContext());
     ConversionTarget target(getContext());
-    target.addLegalDialect<linalg::LinalgDialect, StandardOpsDialect>();
+    target.addLegalDialect<linalg::LinalgDialect, memref::MemRefDialect,
+                           StandardOpsDialect, tensor::TensorDialect>();
     target.addIllegalDialect<tosa::TosaDialect>();
+
+    // Not every TOSA op can be legalized to linalg.
+    target.addLegalOp<tosa::ApplyScaleOp>();
+    target.addLegalOp<tosa::IfOp>();
+    target.addLegalOp<tosa::ConstOp>();
+    target.addLegalOp<tosa::WhileOp>();
+
     target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
 
     FuncOp func = getFunction();
-    mlir::tosa::populateTosaToLinalgOnTensorsConversionPatterns(
-        func.getContext(), &patterns);
+    mlir::tosa::populateTosaToLinalgOnTensorsConversionPatterns(&patterns);
     if (failed(applyFullConversion(func, target, std::move(patterns))))
       signalPassFailure();
   }
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
index 55ed64b10322b58f464da9494a89d1c68a5030f2..ef5ccf93765b52679fd083b1eaa7a03b1d046a17 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
@@ -103,7 +103,7 @@ public:
 } // namespace
 
 void mlir::tosa::populateTosaToSCFConversionPatterns(
-    MLIRContext *context, OwningRewritePatternList *patterns) {
-  patterns->insert<IfOpConverter>(context);
-  patterns->insert<WhileOpConverter>(context);
+    RewritePatternSet *patterns) {
+  patterns->add<IfOpConverter>(patterns->getContext());
+  patterns->add<WhileOpConverter>(patterns->getContext());
 }
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
index f403a4658b979eae7644c735dcaadb2e573bc49b..6563a446aaa2166a9cf27235e03dce5924df77e6 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
@@ -29,15 +29,14 @@ namespace {
 struct TosaToSCF : public TosaToSCFBase<TosaToSCF> {
 public:
   void runOnOperation() override {
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(&getContext());
     ConversionTarget target(getContext());
     target.addLegalDialect<tensor::TensorDialect, scf::SCFDialect>();
     target.addIllegalOp<tosa::IfOp, tosa::WhileOp>();
     target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
 
     auto *op = getOperation();
-    mlir::tosa::populateTosaToSCFConversionPatterns(op->getContext(),
-                                                    &patterns);
+    mlir::tosa::populateTosaToSCFConversionPatterns(&patterns);
     if (failed(applyPartialConversion(op, target, std::move(patterns))))
       signalPassFailure();
   }
diff --git a/mlir/lib/Conversion/TosaToStandard/TosaToStandard.cpp b/mlir/lib/Conversion/TosaToStandard/TosaToStandard.cpp
index 21a8da291aee049f29256c458b1dd309615468de..668548d3dec39d6b4ccaf5af01500eb2515d9c95 100644
--- a/mlir/lib/Conversion/TosaToStandard/TosaToStandard.cpp
+++ b/mlir/lib/Conversion/TosaToStandard/TosaToStandard.cpp
@@ -32,9 +32,134 @@ public:
   }
 };
 
+class SliceOpConverter : public OpRewritePattern<tosa::SliceOp> {
+public:
+  using OpRewritePattern<tosa::SliceOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tosa::SliceOp sliceOp,
+                                PatternRewriter &rewriter) const final {
+    Value input = sliceOp.input();
+    SmallVector<int64_t> strides;
+    strides.resize(sliceOp.getType().template cast<ShapedType>().getRank(), 1);
+
+    rewriter.replaceOpWithNewOp<SubTensorOp>(
+        sliceOp, sliceOp.getType(), input, ValueRange({}), ValueRange({}),
+        ValueRange({}), sliceOp.start(), sliceOp.size(),
+        rewriter.getI64ArrayAttr(strides));
+    return success();
+  }
+};
+
+// This converts the TOSA ApplyScale operator to a set of StandardOps ops,
+// using 64-bit operations to perform the necessary multiply, bias, and shift.
+// Multiple types are used to use minimal bit width operations.
+class ApplyScaleOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
+public:
+  using OpRewritePattern<tosa::ApplyScaleOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tosa::ApplyScaleOp op,
+                                PatternRewriter &rewriter) const final {
+    Location loc = op.getLoc();
+    Value value32 = op.value();
+    Value multiplier32 = op.multiplier();
+    Value shift8 = op.shift();
+    bool doubleRound = op.double_round();
+
+    Value one8 = rewriter.create<ConstantOp>(
+        loc, rewriter.getIntegerAttr(rewriter.getIntegerType(8), 1));
+    Value one32 = rewriter.create<ConstantOp>(
+        loc, rewriter.getIntegerAttr(rewriter.getI32Type(), 1));
+    Value one64 = rewriter.create<ConstantOp>(
+        loc, rewriter.getIntegerAttr(rewriter.getI64Type(), 1));
+
+    Value shiftSubOne8 = rewriter.create<SubIOp>(loc, shift8, one8);
+
+    // The rounding value semantics below equate to the following code:
+    //    int64_t round = 1 << (shift - 1);
+    //    if (double_round) {
+    //      if (shift > 31 && value >= 0) round += 1<<30;
+    //      if (shift > 31 && value < 0) round -= 1<<30;
+    //    }
+    //
+    // Note that minimal bitwidth operators are used throughout the block.
+
+    Value shift32 = rewriter.create<mlir::SignExtendIOp>(
+        loc, rewriter.getI32Type(), shift8);
+
+    Value round64 = rewriter.create<mlir::ShiftLeftOp>(
+        loc, one64,
+        rewriter.create<SignExtendIOp>(loc, rewriter.getI64Type(),
+                                       shiftSubOne8));
+
+    // Double rounding is performing a round operation before the shift
+    if (doubleRound) {
+      Value zero32 = rewriter.create<ConstantOp>(
+          loc, rewriter.getZeroAttr(rewriter.getI32Type()));
+      Value thirty32 = rewriter.create<ConstantOp>(
+          loc, rewriter.getIntegerAttr(rewriter.getI32Type(), 30));
+
+      Value shiftThirty32 =
+          rewriter.create<mlir::ShiftLeftOp>(loc, one32, thirty32);
+      Value shiftThirty64 = rewriter.create<mlir::SignExtendIOp>(
+          loc, rewriter.getI64Type(), shiftThirty32);
+
+      // Round value needs to with be added or sbustracted depending on
+      Value roundAdd64 =
+          rewriter.create<mlir::AddIOp>(loc, round64, shiftThirty64);
+      Value roundSub64 =
+          rewriter.create<mlir::SubIOp>(loc, round64, shiftThirty64);
+
+      Value valueGreaterThanZero = rewriter.create<mlir::CmpIOp>(
+          loc, CmpIPredicate::sge, value32, zero32);
+
+      Value doubleRound64 = rewriter.create<mlir::SelectOp>(
+          loc, valueGreaterThanZero, roundAdd64, roundSub64);
+
+      // We only perform double rounding if the shift value is greater than 32.
+      Value thirtyTwo32 = rewriter.create<ConstantOp>(
+          loc, rewriter.getIntegerAttr(rewriter.getI32Type(), 32));
+      Value shiftGreaterThanThirtyTwo = rewriter.create<mlir::CmpIOp>(
+          loc, CmpIPredicate::sge, shift32, thirtyTwo32);
+      round64 = rewriter.create<mlir::SelectOp>(loc, shiftGreaterThanThirtyTwo,
+                                                doubleRound64, round64);
+    }
+
+    // The computation below equates to the following pseudocode:
+    //    int64_t result = (int64_t)value * multiplier + round;
+    //    result = result >> shift;
+    //
+    // Note that multiply and shift need to be perform in i64 to preserve bits.
+
+    Value value64 =
+        rewriter.create<SignExtendIOp>(loc, rewriter.getI64Type(), value32);
+    Value multiplier64 = rewriter.create<SignExtendIOp>(
+        loc, rewriter.getI64Type(), multiplier32);
+    Value shift64 =
+        rewriter.create<SignExtendIOp>(loc, rewriter.getI64Type(), shift8);
+
+    // Multiply as a pair of i64 values to guarantee the end value fits.
+    Value result64 = rewriter.create<MulIOp>(loc, value64, multiplier64);
+    result64 = rewriter.create<AddIOp>(loc, result64, round64);
+    result64 =
+        rewriter.create<mlir::SignedShiftRightOp>(loc, result64, shift64);
+
+    Value result32 = rewriter.create<mlir::TruncateIOp>(
+        loc, rewriter.getI32Type(), result64);
+
+    rewriter.replaceOp(op, result32);
+    return success();
+  }
+};
+
 } // namespace
 
 void mlir::tosa::populateTosaToStandardConversionPatterns(
-    MLIRContext *context, OwningRewritePatternList *patterns) {
-  patterns->insert<ConstOpConverter>(context);
+    RewritePatternSet *patterns) {
+  patterns->add<ApplyScaleOpConverter, ConstOpConverter, SliceOpConverter>(
+      patterns->getContext());
+}
+
+void mlir::tosa::populateTosaRescaleToStandardConversionPatterns(
+    RewritePatternSet *patterns) {
+  patterns->add<ApplyScaleOpConverter>(patterns->getContext());
 }
diff --git a/mlir/lib/Conversion/TosaToStandard/TosaToStandardPass.cpp b/mlir/lib/Conversion/TosaToStandard/TosaToStandardPass.cpp
index 225855e78bdaf72ff40542430a439034a1170977..af639cb42e52e7e12fa4621acb9f50c846c2633a 100644
--- a/mlir/lib/Conversion/TosaToStandard/TosaToStandardPass.cpp
+++ b/mlir/lib/Conversion/TosaToStandard/TosaToStandardPass.cpp
@@ -29,15 +29,16 @@ namespace {
 struct TosaToStandard : public TosaToStandardBase<TosaToStandard> {
 public:
   void runOnOperation() override {
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(&getContext());
     ConversionTarget target(getContext());
     target.addIllegalOp<tosa::ConstOp>();
-    target.addLegalOp<ConstantOp>();
+    target.addIllegalOp<tosa::SliceOp>();
+    target.addIllegalOp<tosa::ApplyScaleOp>();
+    target.addLegalDialect<StandardOpsDialect>();
 
-    auto *op = getOperation();
-    mlir::tosa::populateTosaToStandardConversionPatterns(op->getContext(),
-                                                         &patterns);
-    if (failed(applyPartialConversion(op, target, std::move(patterns))))
+    mlir::tosa::populateTosaToStandardConversionPatterns(&patterns);
+    if (failed(applyPartialConversion(getOperation(), target,
+                                      std::move(patterns))))
       signalPassFailure();
   }
 };
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 15553bbd9be5286456b3a87b28478d83451e0fb0..24c5092894f02b3f7a4c35ff24ea9aae2d8acf1b 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -1482,47 +1482,37 @@ public:
 
 /// Populate the given list with patterns that convert from Vector to LLVM.
 void mlir::populateVectorToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
+    LLVMTypeConverter &converter, RewritePatternSet &patterns,
     bool reassociateFPReductions, bool enableIndexOptimizations) {
   MLIRContext *ctx = converter.getDialect()->getContext();
-  // clang-format off
-  patterns.insert<VectorFMAOpNDRewritePattern,
-                  VectorInsertStridedSliceOpDifferentRankRewritePattern,
-                  VectorInsertStridedSliceOpSameRankRewritePattern,
-                  VectorExtractStridedSliceOpConversion>(ctx);
-  patterns.insert<VectorReductionOpConversion>(
-      converter, reassociateFPReductions);
-  patterns.insert<VectorCreateMaskOpConversion,
-                  VectorTransferConversion<TransferReadOp>,
-                  VectorTransferConversion<TransferWriteOp>>(
+  patterns.add<VectorFMAOpNDRewritePattern,
+               VectorInsertStridedSliceOpDifferentRankRewritePattern,
+               VectorInsertStridedSliceOpSameRankRewritePattern,
+               VectorExtractStridedSliceOpConversion>(ctx);
+  patterns.add<VectorReductionOpConversion>(converter, reassociateFPReductions);
+  patterns.add<VectorCreateMaskOpConversion,
+               VectorTransferConversion<TransferReadOp>,
+               VectorTransferConversion<TransferWriteOp>>(
       converter, enableIndexOptimizations);
   patterns
-      .insert<VectorBitCastOpConversion,
-              VectorShuffleOpConversion,
-              VectorExtractElementOpConversion,
-              VectorExtractOpConversion,
-              VectorFMAOp1DConversion,
-              VectorInsertElementOpConversion,
-              VectorInsertOpConversion,
-              VectorPrintOpConversion,
-              VectorTypeCastOpConversion,
-              VectorLoadStoreConversion<vector::LoadOp,
-                                        vector::LoadOpAdaptor>,
-              VectorLoadStoreConversion<vector::MaskedLoadOp,
-                                        vector::MaskedLoadOpAdaptor>,
-              VectorLoadStoreConversion<vector::StoreOp,
-                                        vector::StoreOpAdaptor>,
-              VectorLoadStoreConversion<vector::MaskedStoreOp,
-                                        vector::MaskedStoreOpAdaptor>,
-              VectorGatherOpConversion,
-              VectorScatterOpConversion,
-              VectorExpandLoadOpConversion,
-              VectorCompressStoreOpConversion>(converter);
-  // clang-format on
+      .add<VectorBitCastOpConversion, VectorShuffleOpConversion,
+           VectorExtractElementOpConversion, VectorExtractOpConversion,
+           VectorFMAOp1DConversion, VectorInsertElementOpConversion,
+           VectorInsertOpConversion, VectorPrintOpConversion,
+           VectorTypeCastOpConversion,
+           VectorLoadStoreConversion<vector::LoadOp, vector::LoadOpAdaptor>,
+           VectorLoadStoreConversion<vector::MaskedLoadOp,
+                                     vector::MaskedLoadOpAdaptor>,
+           VectorLoadStoreConversion<vector::StoreOp, vector::StoreOpAdaptor>,
+           VectorLoadStoreConversion<vector::MaskedStoreOp,
+                                     vector::MaskedStoreOpAdaptor>,
+           VectorGatherOpConversion, VectorScatterOpConversion,
+           VectorExpandLoadOpConversion, VectorCompressStoreOpConversion>(
+          converter);
 }
 
 void mlir::populateVectorToLLVMMatrixConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  patterns.insert<VectorMatmulOpConversion>(converter);
-  patterns.insert<VectorFlatTransposeOpConversion>(converter);
+    LLVMTypeConverter &converter, RewritePatternSet &patterns) {
+  patterns.add<VectorMatmulOpConversion>(converter);
+  patterns.add<VectorFlatTransposeOpConversion>(converter);
 }
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
index 85657742413f3e3a98e29c6e3189505a6c3222da..abddcd73af1eeebff3cb37a4c100a1371b38300a 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
@@ -61,16 +61,16 @@ void LowerVectorToLLVMPass::runOnOperation() {
   // Perform progressive lowering of operations on slices and
   // all contraction operations. Also applies folding and DCE.
   {
-    OwningRewritePatternList patterns;
-    populateVectorToVectorCanonicalizationPatterns(patterns, &getContext());
-    populateVectorSlicesLoweringPatterns(patterns, &getContext());
-    populateVectorContractLoweringPatterns(patterns, &getContext());
+    RewritePatternSet patterns(&getContext());
+    populateVectorToVectorCanonicalizationPatterns(patterns);
+    populateVectorSlicesLoweringPatterns(patterns);
+    populateVectorContractLoweringPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
   }
 
   // Convert to the LLVM IR dialect.
   LLVMTypeConverter converter(&getContext());
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(&getContext());
   populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
   populateVectorToLLVMConversionPatterns(
       converter, patterns, reassociateFPReductions, enableIndexOptimizations);
@@ -98,7 +98,7 @@ void LowerVectorToLLVMPass::runOnOperation() {
       return false;
     };
     // Remove any ArmSVE-specific types from function signatures and results.
-    populateFuncOpTypeConversionPattern(patterns, &getContext(), converter);
+    populateFuncOpTypeConversionPattern(patterns, converter);
     target.addDynamicallyLegalOp<FuncOp>([hasScalableVectorType](FuncOp op) {
       return !hasScalableVectorType(op.getType().getInputs()) &&
              !hasScalableVectorType(op.getType().getResults());
diff --git a/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp b/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
index 42c072626cf58483836e6a2c103ac678fbd934be..5ebed5f80e4a41b7fae704221a87b95fa884a97e 100644
--- a/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
+++ b/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
@@ -144,9 +144,9 @@ public:
 } // end anonymous namespace
 
 void mlir::populateVectorToROCDLConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  patterns.insert<VectorTransferConversion<TransferReadOp>,
-                  VectorTransferConversion<TransferWriteOp>>(converter);
+    LLVMTypeConverter &converter, RewritePatternSet &patterns) {
+  patterns.add<VectorTransferConversion<TransferReadOp>,
+               VectorTransferConversion<TransferWriteOp>>(converter);
 }
 
 namespace {
@@ -158,7 +158,7 @@ struct LowerVectorToROCDLPass
 
 void LowerVectorToROCDLPass::runOnOperation() {
   LLVMTypeConverter converter(&getContext());
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(&getContext());
 
   populateVectorToROCDLConversionPatterns(converter, patterns);
   populateStdToLLVMConversionPatterns(converter, patterns);
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index dce5b64284ec336a14f0a85a8e05ad73d52c8e14..f95193f76508054f08403eff6081e9edfa87dd46 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -116,11 +116,9 @@ public:
         VectorType::get(vectorType.getShape().take_back(minorRank),
                         vectorType.getElementType());
     /// Memref of minor vector type is used for individual transfers.
-    memRefMinorVectorType =
-        MemRefType::get(majorVectorType.getShape(), minorVectorType, {},
-                        xferOp.getShapedType()
-                            .template cast<MemRefType>()
-                            .getMemorySpaceAsInt());
+    memRefMinorVectorType = MemRefType::get(
+        majorVectorType.getShape(), minorVectorType, {},
+        xferOp.getShapedType().template cast<MemRefType>().getMemorySpace());
   }
 
   LogicalResult doReplace();
@@ -694,11 +692,10 @@ LogicalResult VectorTransferRewriter<TransferWriteOp>::matchAndRewrite(
 }
 
 void populateVectorToSCFConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context,
-    const VectorTransferToSCFOptions &options) {
-  patterns.insert<VectorTransferRewriter<vector::TransferReadOp>,
-                  VectorTransferRewriter<vector::TransferWriteOp>>(options,
-                                                                   context);
+    RewritePatternSet &patterns, const VectorTransferToSCFOptions &options) {
+  patterns.add<VectorTransferRewriter<vector::TransferReadOp>,
+               VectorTransferRewriter<vector::TransferWriteOp>>(
+      options, patterns.getContext());
 }
 
 } // namespace mlir
@@ -713,10 +710,9 @@ struct ConvertVectorToSCFPass
   }
 
   void runOnFunction() override {
-    OwningRewritePatternList patterns;
-    auto *context = getFunction().getContext();
+    RewritePatternSet patterns(getFunction().getContext());
     populateVectorToSCFConversionPatterns(
-        patterns, context, VectorTransferToSCFOptions().setUnroll(fullUnroll));
+        patterns, VectorTransferToSCFOptions().setUnroll(fullUnroll));
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 8d4fcbacca4a2def8dd7a67cf640e76633b62d7c..4cfcb4148cf0fddf86494ed3ca4c5cfb51510437 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -241,12 +241,12 @@ struct VectorInsertStridedSliceOpConvert final
 
 } // namespace
 
-void mlir::populateVectorToSPIRVPatterns(MLIRContext *context,
-                                         SPIRVTypeConverter &typeConverter,
-                                         OwningRewritePatternList &patterns) {
-  patterns.insert<VectorBitcastConvert, VectorBroadcastConvert,
-                  VectorExtractElementOpConvert, VectorExtractOpConvert,
-                  VectorExtractStridedSliceOpConvert, VectorFmaOpConvert,
-                  VectorInsertElementOpConvert, VectorInsertOpConvert,
-                  VectorInsertStridedSliceOpConvert>(typeConverter, context);
+void mlir::populateVectorToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                         RewritePatternSet &patterns) {
+  patterns.add<VectorBitcastConvert, VectorBroadcastConvert,
+               VectorExtractElementOpConvert, VectorExtractOpConvert,
+               VectorExtractStridedSliceOpConvert, VectorFmaOpConvert,
+               VectorInsertElementOpConvert, VectorInsertOpConvert,
+               VectorInsertStridedSliceOpConvert>(typeConverter,
+                                                  patterns.getContext());
 }
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
index 9a4d09f9ad0e6bca35d3c279fd0b4e9922fc1255..e170df2948fef69eaac9f02299ac062eb73bd813 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
@@ -37,10 +37,10 @@ void LowerVectorToSPIRVPass::runOnOperation() {
       spirv::SPIRVConversionTarget::get(targetAttr);
 
   SPIRVTypeConverter typeConverter(targetAttr);
-  OwningRewritePatternList patterns;
-  populateVectorToSPIRVPatterns(context, typeConverter, patterns);
+  RewritePatternSet patterns(context);
+  populateVectorToSPIRVPatterns(typeConverter, patterns);
 
-  target->addLegalOp<ModuleOp, ModuleTerminatorOp>();
+  target->addLegalOp<ModuleOp>();
   target->addLegalOp<FuncOp>();
 
   if (failed(applyFullConversion(module, *target, std::move(patterns))))
diff --git a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
index 5ebef7efe2131120df74881a6a2cef0673e4b7a6..ab98820b2ecbc387cd0004c13317ae72c11f0a5f 100644
--- a/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
+++ b/mlir/lib/Dialect/AMX/IR/AMXDialect.cpp
@@ -85,8 +85,6 @@ static LogicalResult verify(amx::TileMulFOp op) {
 }
 
 static LogicalResult verify(amx::TileMulIOp op) {
-  if (op.zext().size() != 2)
-    return op.emitOpError("unexpected zext length");
   VectorType aType = op.getLhsVectorType();
   VectorType bType = op.getRhsVectorType();
   VectorType cType = op.getVectorType();
diff --git a/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
index 6e082ce790fc0ed6b09d4d0a0fb4ee2c0c6a0779..d9cb5034a8dc0c8ad85b759f83a8ccb6ef890a20 100644
--- a/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/AMX/Transforms/LegalizeForLLVMExport.cpp
@@ -191,8 +191,8 @@ struct TileMulIConversion : public ConvertOpToLLVMPattern<TileMulIOp> {
         getTileSizes(rewriter, *getTypeConverter(), bType, op.getLoc());
     // Replace operation with intrinsic.
     Type resType = typeConverter->convertType(cType);
-    bool zexta = op.zext()[0].cast<BoolAttr>().getValue();
-    bool zextb = op.zext()[1].cast<BoolAttr>().getValue();
+    bool zexta = op.isZextLhs();
+    bool zextb = op.isZextRhs();
     if (zexta && zextb)
       rewriter.replaceOpWithNewOp<amx::x86_amx_tdpbuud>(
           op, resType, tsza.first, tszb.second, tsza.second, adaptor.acc(),
@@ -216,9 +216,9 @@ struct TileMulIConversion : public ConvertOpToLLVMPattern<TileMulIOp> {
 } // namespace
 
 void mlir::populateAMXLegalizeForLLVMExportPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  patterns.insert<TileZeroConversion, TileLoadConversion, TileStoreConversion,
-                  TileMulFConversion, TileMulIConversion>(converter);
+    LLVMTypeConverter &converter, RewritePatternSet &patterns) {
+  patterns.add<TileZeroConversion, TileLoadConversion, TileStoreConversion,
+               TileMulFConversion, TileMulIConversion>(converter);
 }
 
 void mlir::configureAMXLegalizeForExportTarget(LLVMConversionTarget &target) {
diff --git a/mlir/lib/Dialect/AVX512/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/AVX512/Transforms/LegalizeForLLVMExport.cpp
index cfe9f2b3ac028ae7c43768035c1523ad38a82ede..eaa6498f175293b781c098f2aaa5ffd15e717b99 100644
--- a/mlir/lib/Dialect/AVX512/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/AVX512/Transforms/LegalizeForLLVMExport.cpp
@@ -105,10 +105,10 @@ struct RegistryImpl {
   /// Registers the patterns specializing the "main" op to one of the
   /// "intrinsic" ops depending on elemental type.
   static void registerPatterns(LLVMTypeConverter &converter,
-                               OwningRewritePatternList &patterns) {
+                               RewritePatternSet &patterns) {
     patterns
-        .insert<LowerToIntrinsic<typename Args::MainOp, typename Args::Intr32Op,
-                                 typename Args::Intr64Op>...>(converter);
+        .add<LowerToIntrinsic<typename Args::MainOp, typename Args::Intr32Op,
+                              typename Args::Intr64Op>...>(converter);
   }
 
   /// Configures the conversion target to lower out "main" ops.
@@ -128,9 +128,9 @@ using Registry = RegistryImpl<
 
 /// Populate the given list with patterns that convert from AVX512 to LLVM.
 void mlir::populateAVX512LegalizeForLLVMExportPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+    LLVMTypeConverter &converter, RewritePatternSet &patterns) {
   Registry::registerPatterns(converter, patterns);
-  patterns.insert<MaskCompressOpConversion>(converter);
+  patterns.add<MaskCompressOpConversion>(converter);
 }
 
 void mlir::configureAVX512LegalizeForExportTarget(
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 7962ec21b5dedb63bcd6f3f769cbfed5fd42937b..2b12b3ad161674b3abf5b9c492c34e41ccab165a 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -905,9 +905,9 @@ void SimplifyAffineOp<AffineOpTy>::replaceAffineOp(
 }
 } // end anonymous namespace.
 
-void AffineApplyOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<SimplifyAffineOp<AffineApplyOp>>(context);
+void AffineApplyOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                MLIRContext *context) {
+  results.add<SimplifyAffineOp<AffineApplyOp>>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1611,9 +1611,9 @@ struct AffineForEmptyLoopFolder : public OpRewritePattern<AffineForOp> {
 };
 } // end anonymous namespace
 
-void AffineForOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void AffineForOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
-  results.insert<AffineForEmptyLoopFolder>(context);
+  results.add<AffineForEmptyLoopFolder>(context);
 }
 
 LogicalResult AffineForOp::fold(ArrayRef<Attribute> operands,
@@ -2033,9 +2033,9 @@ LogicalResult AffineIfOp::fold(ArrayRef<Attribute>,
   return failure();
 }
 
-void AffineIfOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void AffineIfOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                              MLIRContext *context) {
-  results.insert<SimplifyDeadElse>(context);
+  results.add<SimplifyDeadElse>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2149,9 +2149,9 @@ LogicalResult verify(AffineLoadOp op) {
   return success();
 }
 
-void AffineLoadOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<SimplifyAffineOp<AffineLoadOp>>(context);
+void AffineLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                               MLIRContext *context) {
+  results.add<SimplifyAffineOp<AffineLoadOp>>(context);
 }
 
 OpFoldResult AffineLoadOp::fold(ArrayRef<Attribute> cstOperands) {
@@ -2239,9 +2239,9 @@ LogicalResult verify(AffineStoreOp op) {
   return success();
 }
 
-void AffineStoreOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<SimplifyAffineOp<AffineStoreOp>>(context);
+void AffineStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                MLIRContext *context) {
+  results.add<SimplifyAffineOp<AffineStoreOp>>(context);
 }
 
 LogicalResult AffineStoreOp::fold(ArrayRef<Attribute> cstOperands,
@@ -2327,6 +2327,130 @@ static OpFoldResult foldMinMaxOp(T op, ArrayRef<Attribute> operands) {
   return IntegerAttr::get(IndexType::get(op.getContext()), *resultIt);
 }
 
+/// Remove duplicated expressions in affine min/max ops.
+template <typename T>
+struct DeduplicateAffineMinMaxExpressions : public OpRewritePattern<T> {
+  using OpRewritePattern<T>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(T affineOp,
+                                PatternRewriter &rewriter) const override {
+    AffineMap oldMap = affineOp.getAffineMap();
+
+    SmallVector<AffineExpr, 4> newExprs;
+    for (AffineExpr expr : oldMap.getResults()) {
+      // This is a linear scan over newExprs, but it should be fine given that
+      // we typically just have a few expressions per op.
+      if (!llvm::is_contained(newExprs, expr))
+        newExprs.push_back(expr);
+    }
+
+    if (newExprs.size() == oldMap.getNumResults())
+      return failure();
+
+    auto newMap = AffineMap::get(oldMap.getNumDims(), oldMap.getNumSymbols(),
+                                 newExprs, rewriter.getContext());
+    rewriter.replaceOpWithNewOp<T>(affineOp, newMap, affineOp.getMapOperands());
+
+    return success();
+  }
+};
+
+/// Merge an affine min/max op to its consumers if its consumer is also an
+/// affine min/max op.
+///
+/// This pattern requires the producer affine min/max op is bound to a
+/// dimension/symbol that is used as a standalone expression in the consumer
+/// affine op's map.
+///
+/// For example, a pattern like the following:
+///
+///   %0 = affine.min affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%sym1]
+///   %1 = affine.min affine_map<(d0)[s0] -> (s0 + 4, d0)> (%0)[%sym2]
+///
+/// Can be turned into:
+///
+///   %1 = affine.min affine_map<
+///          ()[s0, s1] -> (s0 + 4, s1 + 16, s1 * 8)> ()[%sym2, %sym1]
+template <typename T>
+struct MergeAffineMinMaxOp : public OpRewritePattern<T> {
+  using OpRewritePattern<T>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(T affineOp,
+                                PatternRewriter &rewriter) const override {
+    AffineMap oldMap = affineOp.getAffineMap();
+    ValueRange dimOperands =
+        affineOp.getMapOperands().take_front(oldMap.getNumDims());
+    ValueRange symOperands =
+        affineOp.getMapOperands().take_back(oldMap.getNumSymbols());
+
+    auto newDimOperands = llvm::to_vector<8>(dimOperands);
+    auto newSymOperands = llvm::to_vector<8>(symOperands);
+    SmallVector<AffineExpr, 4> newExprs;
+    SmallVector<T, 4> producerOps;
+
+    // Go over each expression to see whether it's a single dimension/symbol
+    // with the corresponding operand which is the result of another affine
+    // min/max op. If So it can be merged into this affine op.
+    for (AffineExpr expr : oldMap.getResults()) {
+      if (auto symExpr = expr.dyn_cast<AffineSymbolExpr>()) {
+        Value symValue = symOperands[symExpr.getPosition()];
+        if (auto producerOp = symValue.getDefiningOp<T>()) {
+          producerOps.push_back(producerOp);
+          continue;
+        }
+      } else if (auto dimExpr = expr.dyn_cast<AffineDimExpr>()) {
+        Value dimValue = dimOperands[dimExpr.getPosition()];
+        if (auto producerOp = dimValue.getDefiningOp<T>()) {
+          producerOps.push_back(producerOp);
+          continue;
+        }
+      }
+      // For the above cases we will remove the expression by merging the
+      // producer affine min/max's affine expressions. Otherwise we need to
+      // keep the existing expression.
+      newExprs.push_back(expr);
+    }
+
+    if (producerOps.empty())
+      return failure();
+
+    unsigned numUsedDims = oldMap.getNumDims();
+    unsigned numUsedSyms = oldMap.getNumSymbols();
+
+    // Now go over all producer affine ops and merge their expressions.
+    for (T producerOp : producerOps) {
+      AffineMap producerMap = producerOp.getAffineMap();
+      unsigned numProducerDims = producerMap.getNumDims();
+      unsigned numProducerSyms = producerMap.getNumSymbols();
+
+      // Collect all dimension/symbol values.
+      ValueRange dimValues =
+          producerOp.getMapOperands().take_front(numProducerDims);
+      ValueRange symValues =
+          producerOp.getMapOperands().take_back(numProducerSyms);
+      newDimOperands.append(dimValues.begin(), dimValues.end());
+      newSymOperands.append(symValues.begin(), symValues.end());
+
+      // For expressions we need to shift to avoid overlap.
+      for (AffineExpr expr : producerMap.getResults()) {
+        newExprs.push_back(expr.shiftDims(numProducerDims, numUsedDims)
+                               .shiftSymbols(numProducerSyms, numUsedSyms));
+      }
+
+      numUsedDims += numProducerDims;
+      numUsedSyms += numProducerSyms;
+    }
+
+    auto newMap = AffineMap::get(numUsedDims, numUsedSyms, newExprs,
+                                 rewriter.getContext());
+    auto newOperands =
+        llvm::to_vector<8>(llvm::concat<Value>(newDimOperands, newSymOperands));
+    rewriter.replaceOpWithNewOp<T>(affineOp, newMap, newOperands);
+
+    return success();
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // AffineMinOp
 //===----------------------------------------------------------------------===//
@@ -2338,9 +2462,11 @@ OpFoldResult AffineMinOp::fold(ArrayRef<Attribute> operands) {
   return foldMinMaxOp(*this, operands);
 }
 
-void AffineMinOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<SimplifyAffineOp<AffineMinOp>>(context);
+void AffineMinOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
+                                              MLIRContext *context) {
+  patterns.add<DeduplicateAffineMinMaxExpressions<AffineMinOp>,
+               MergeAffineMinMaxOp<AffineMinOp>, SimplifyAffineOp<AffineMinOp>>(
+      context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2354,9 +2480,11 @@ OpFoldResult AffineMaxOp::fold(ArrayRef<Attribute> operands) {
   return foldMinMaxOp(*this, operands);
 }
 
-void AffineMaxOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<SimplifyAffineOp<AffineMaxOp>>(context);
+void AffineMaxOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
+                                              MLIRContext *context) {
+  patterns.add<DeduplicateAffineMinMaxExpressions<AffineMaxOp>,
+               MergeAffineMinMaxOp<AffineMaxOp>, SimplifyAffineOp<AffineMaxOp>>(
+      context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2454,10 +2582,10 @@ static LogicalResult verify(AffinePrefetchOp op) {
   return success();
 }
 
-void AffinePrefetchOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
+void AffinePrefetchOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                   MLIRContext *context) {
   // prefetch(memrefcast) -> prefetch
-  results.insert<SimplifyAffineOp<AffinePrefetchOp>>(context);
+  results.add<SimplifyAffineOp<AffinePrefetchOp>>(context);
 }
 
 LogicalResult AffinePrefetchOp::fold(ArrayRef<Attribute> cstOperands,
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index e3834ea62367db84fde0ccb0c492d939e9c03482..851ec5051a6be0fd6f0bdb2ac04a5c9bcd03e8b3 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -227,10 +227,10 @@ void AffineDataCopyGeneration::runOnFunction() {
   // Promoting single iteration loops could lead to simplification of
   // contained load's/store's, and the latter could anyway also be
   // canonicalized.
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(&getContext());
   AffineLoadOp::getCanonicalizationPatterns(patterns, &getContext());
   AffineStoreOp::getCanonicalizationPatterns(patterns, &getContext());
-  FrozenRewritePatternList frozenPatterns(std::move(patterns));
+  FrozenRewritePatternSet frozenPatterns(std::move(patterns));
   for (Operation *op : copyOps)
     (void)applyOpPatternsAndFold(op, frozenPatterns);
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
index 918fec42b2fd675c2354beb522573a625f210f6a..8f59074e6b791ef4b6c886a6efe6d51ff74134b3 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
@@ -79,11 +79,11 @@ mlir::createSimplifyAffineStructuresPass() {
 void SimplifyAffineStructures::runOnFunction() {
   auto func = getFunction();
   simplifiedAttributes.clear();
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(func.getContext());
   AffineForOp::getCanonicalizationPatterns(patterns, func.getContext());
   AffineIfOp::getCanonicalizationPatterns(patterns, func.getContext());
   AffineApplyOp::getCanonicalizationPatterns(patterns, func.getContext());
-  FrozenRewritePatternList frozenPatterns(std::move(patterns));
+  FrozenRewritePatternSet frozenPatterns(std::move(patterns));
   func.walk([&](Operation *op) {
     for (auto attr : op->getAttrs()) {
       if (auto mapAttr = attr.second.dyn_cast<AffineMapAttr>())
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index acd854d2731cc5e395f274121bfb4edce86858b2..522cfd7fca9501a1f6eccfa649e9b8e614151cfa 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -188,10 +188,10 @@ LogicalResult mlir::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
   // effective (no unused operands). Since the pattern rewriter's folding is
   // entangled with application of patterns, we may fold/end up erasing the op,
   // in which case we return with `folded` being set.
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(ifOp.getContext());
   AffineIfOp::getCanonicalizationPatterns(patterns, ifOp.getContext());
   bool erased;
-  FrozenRewritePatternList frozenPatterns(std::move(patterns));
+  FrozenRewritePatternSet frozenPatterns(std::move(patterns));
   (void)applyOpPatternsAndFold(ifOp, frozenPatterns, &erased);
   if (erased) {
     if (folded)
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index f4f6e0bdf9560ba0819b3e561edb635a3bad17f1..3627635ed060621ad62aa31a452164124983c96f 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -270,8 +270,8 @@ AsyncParallelForRewrite::matchAndRewrite(scf::ParallelOp op,
 void AsyncParallelForPass::runOnFunction() {
   MLIRContext *ctx = &getContext();
 
-  OwningRewritePatternList patterns;
-  patterns.insert<AsyncParallelForRewrite>(ctx, numConcurrentAsyncExecute);
+  RewritePatternSet patterns(ctx);
+  patterns.add<AsyncParallelForRewrite>(ctx, numConcurrentAsyncExecute);
 
   if (failed(applyPatternsAndFoldGreedily(getFunction(), std::move(patterns))))
     signalPassFailure();
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
index a17da4286b71067daacf90443ac7f4a735274300..268c603c64466a588e0a874697d9da503a58b82f 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
@@ -199,7 +199,7 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
   // TODO: Derive outlined function name from the parent FuncOp (support
   // multiple nested async.execute operations).
   FuncOp func = FuncOp::create(loc, kAsyncFnPrefix, funcType, funcAttrs);
-  symbolTable.insert(func, Block::iterator(module.getBody()->getTerminator()));
+  symbolTable.insert(func);
 
   SymbolTable::setSymbolVisibility(func, SymbolTable::Visibility::Private);
 
@@ -485,14 +485,14 @@ void AsyncToAsyncRuntimePass::runOnOperation() {
 
   // Lower async operations to async.runtime operations.
   MLIRContext *ctx = module->getContext();
-  OwningRewritePatternList asyncPatterns;
+  RewritePatternSet asyncPatterns(ctx);
 
   // Async lowering does not use type converter because it must preserve all
   // types for async.runtime operations.
-  asyncPatterns.insert<CreateGroupOpLowering, AddToGroupOpLowering>(ctx);
-  asyncPatterns.insert<AwaitTokenOpLowering, AwaitValueOpLowering,
-                       AwaitAllOpLowering, YieldOpLowering>(ctx,
-                                                            outlinedFunctions);
+  asyncPatterns.add<CreateGroupOpLowering, AddToGroupOpLowering>(ctx);
+  asyncPatterns.add<AwaitTokenOpLowering, AwaitValueOpLowering,
+                    AwaitAllOpLowering, YieldOpLowering>(ctx,
+                                                         outlinedFunctions);
 
   // All high level async operations must be lowered to the runtime operations.
   ConversionTarget runtimeTarget(*ctx);
diff --git a/mlir/lib/Dialect/DLTI/DLTI.cpp b/mlir/lib/Dialect/DLTI/DLTI.cpp
index 4b9c3b523b6bf68aadb02305fe16cabb84455614..2567be64ac1ad5a0eb9d4d5d8f164e6296c49b89 100644
--- a/mlir/lib/Dialect/DLTI/DLTI.cpp
+++ b/mlir/lib/Dialect/DLTI/DLTI.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinDialect.h"
+#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -369,6 +370,8 @@ LogicalResult DLTIDialect::verifyOperationAttribute(Operation *op,
       return op->emitError() << "'" << DLTIDialect::kDataLayoutAttrName
                              << "' is expected to be a #dlti.dl_spec attribute";
     }
+    if (isa<ModuleOp>(op))
+      return detail::verifyDataLayoutOp(op);
     return success();
   }
 
diff --git a/mlir/lib/Dialect/GPU/CMakeLists.txt b/mlir/lib/Dialect/GPU/CMakeLists.txt
index d7fbfe0b5b61125261ee8404322703ee2321a2b0..c4895ccb30fa49401c38d6da883e4f6630c91cc8 100644
--- a/mlir/lib/Dialect/GPU/CMakeLists.txt
+++ b/mlir/lib/Dialect/GPU/CMakeLists.txt
@@ -6,6 +6,16 @@ if (MLIR_CUDA_CONVERSIONS_ENABLED)
   )
 endif()
 
+if (MLIR_ROCM_CONVERSIONS_ENABLED)
+  set(AMDGPU_LIBS
+    MCParser
+    AMDGPUAsmParser
+    AMDGPUCodeGen
+    AMDGPUDesc
+    AMDGPUInfo
+  )
+endif()
+
 add_mlir_dialect_library(MLIRGPU
   IR/GPUDialect.cpp
   Transforms/AllReduceLowering.cpp
@@ -15,6 +25,7 @@ add_mlir_dialect_library(MLIRGPU
   Transforms/ParallelLoopMapper.cpp
   Transforms/SerializeToBlob.cpp
   Transforms/SerializeToCubin.cpp
+  Transforms/SerializeToHsaco.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/GPU
@@ -23,6 +34,7 @@ add_mlir_dialect_library(MLIRGPU
   Core
   MC
   ${NVPTX_LIBS}
+  ${AMDGPU_LIBS}
 
   DEPENDS
   MLIRGPUOpsIncGen
@@ -33,6 +45,8 @@ add_mlir_dialect_library(MLIRGPU
 
   LINK_LIBS PUBLIC
   MLIRAsync
+  MLIRDataLayoutInterfaces
+  MLIRDLTI
   MLIREDSC
   MLIRIR
   MLIRMemRef
@@ -84,3 +98,58 @@ if(MLIR_CUDA_RUNNER_ENABLED)
   )
 
 endif()
+
+if(MLIR_ROCM_RUNNER_ENABLED)
+  if (NOT ("AMDGPU" IN_LIST LLVM_TARGETS_TO_BUILD))
+    message(SEND_ERROR
+      "Building mlir with ROCm support requires the AMDGPU backend")
+  endif()
+
+  # Ensure lld is enabled.
+  if (NOT "lld" IN_LIST LLVM_ENABLE_PROJECTS)
+    message(SEND_ERROR "lld is not enabled. Please revise LLVM_ENABLE_PROJECTS")
+  endif()
+
+  # Configure ROCm support.
+  if (NOT DEFINED ROCM_PATH)
+    if (NOT DEFINED ENV{ROCM_PATH})
+      set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCm has been installed")
+    else()
+      set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCm has been installed")
+    endif()
+    set(HIP_PATH "${ROCM_PATH}/hip" CACHE PATH " Path to which HIP has been installed")
+  endif()
+  set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})
+  find_package(HIP)
+  if (NOT HIP_FOUND)
+    message(SEND_ERROR "Building mlir with ROCm support requires a working ROCm and HIP install")
+  else()
+    message(STATUS "ROCm HIP version: ${HIP_VERSION}")
+  endif()
+
+  target_compile_definitions(obj.MLIRGPU
+    PRIVATE
+    __HIP_PLATFORM_HCC__
+    __ROCM_PATH__="${ROCM_PATH}"
+    MLIR_GPU_TO_HSACO_PASS_ENABLE=1
+  )
+
+  target_include_directories(obj.MLIRGPU
+    PRIVATE
+    ${MLIR_SOURCE_DIR}/../lld/include
+    ${HIP_PATH}/include
+    ${ROCM_PATH}/include
+  )
+
+  target_link_libraries(MLIRGPU
+    PRIVATE
+    lldELF
+    MLIRROCDLToLLVMIRTranslation
+  )
+
+  # Link lldELF also to libmlir.so. Create an alias that starts with LLVM
+  # because LINK_COMPONENTS elements are implicitly prefixed with LLVM.
+  add_library(LLVMAliasTolldELF ALIAS lldELF)
+  set_property(GLOBAL APPEND PROPERTY MLIR_LLVM_LINK_COMPONENTS AliasTolldELF)
+
+endif()
diff --git a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
index 8e9ec0b353d57c2f556bfcff320b63ffdee42909..13f455f1cd6e7c11e3ab00ee8e58d91b47ee8b79 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AllReduceLowering.cpp
@@ -401,7 +401,6 @@ struct GpuAllReduceConversion : public RewritePattern {
 };
 } // namespace
 
-void mlir::populateGpuAllReducePatterns(MLIRContext *context,
-                                        OwningRewritePatternList &patterns) {
-  patterns.insert<GpuAllReduceConversion>(context);
+void mlir::populateGpuAllReducePatterns(RewritePatternSet &patterns) {
+  patterns.add<GpuAllReduceConversion>(patterns.getContext());
 }
diff --git a/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp b/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
index 6ccb59aff35a3f11fe9d79149f8d948416491434..b032169188bcac3d721c318f4fd6f4b2e685a5c5 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
@@ -36,7 +36,7 @@ ParallelLoopDimMapping getParallelLoopDimMappingAttr(Processor processor,
   MLIRContext *context = map.getContext();
   OpBuilder builder(context);
   return ParallelLoopDimMapping::get(
-      builder.getI64IntegerAttr(static_cast<int32_t>(processor)),
+      ProcessorAttr::get(builder.getContext(), processor),
       AffineMapAttr::get(map), AffineMapAttr::get(bound), context);
 }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1369c1e575490aa8916382e97e5ef45a05135fad
--- /dev/null
+++ b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp
@@ -0,0 +1,284 @@
+//===- LowerGPUToHSACO.cpp - Convert GPU kernel to HSACO blob -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a pass that serializes a gpu module into HSAco blob and
+// adds that blob as a string attribute of the module.
+//
+//===----------------------------------------------------------------------===//
+#include "mlir/Dialect/GPU/Passes.h"
+
+#if MLIR_GPU_TO_HSACO_PASS_ENABLE
+#include "mlir/Pass/Pass.h"
+#include "mlir/Support/FileUtilities.h"
+#include "mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Export.h"
+
+#include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/WithColor.h"
+#include "llvm/Target/TargetOptions.h"
+
+#include "lld/Common/Driver.h"
+
+#include "hip/hip_version.h"
+
+#include <mutex>
+
+using namespace mlir;
+
+namespace {
+class SerializeToHsacoPass
+    : public PassWrapper<SerializeToHsacoPass, gpu::SerializeToBlobPass> {
+public:
+  SerializeToHsacoPass();
+
+private:
+  void getDependentDialects(DialectRegistry &registry) const override;
+
+  // Serializes ROCDL to HSACO.
+  std::unique_ptr<std::vector<char>>
+  serializeISA(const std::string &isa) override;
+
+  std::unique_ptr<SmallVectorImpl<char>> assembleIsa(const std::string &isa);
+  std::unique_ptr<std::vector<char>>
+  createHsaco(const SmallVectorImpl<char> &isaBinary);
+};
+} // namespace
+
+static std::string getDefaultChip() {
+  const char kDefaultChip[] = "gfx900";
+
+  // Locate rocm_agent_enumerator.
+  const char kRocmAgentEnumerator[] = "rocm_agent_enumerator";
+  llvm::ErrorOr<std::string> rocmAgentEnumerator = llvm::sys::findProgramByName(
+      kRocmAgentEnumerator, {__ROCM_PATH__ "/bin"});
+  if (!rocmAgentEnumerator) {
+    llvm::WithColor::warning(llvm::errs())
+        << kRocmAgentEnumerator << "couldn't be located under " << __ROCM_PATH__
+        << "/bin\n";
+    return kDefaultChip;
+  }
+
+  // Prepare temp file to hold the outputs.
+  int tempFd = -1;
+  SmallString<128> tempFilename;
+  if (llvm::sys::fs::createTemporaryFile("rocm_agent", "txt", tempFd,
+                                         tempFilename)) {
+    llvm::WithColor::warning(llvm::errs())
+        << "temporary file for " << kRocmAgentEnumerator << " creation error\n";
+    return kDefaultChip;
+  }
+  llvm::FileRemover cleanup(tempFilename);
+
+  // Invoke rocm_agent_enumerator.
+  std::string errorMessage;
+  SmallVector<StringRef, 2> args{"-t", "GPU"};
+  Optional<StringRef> redirects[3] = {{""}, tempFilename.str(), {""}};
+  int result =
+      llvm::sys::ExecuteAndWait(rocmAgentEnumerator.get(), args, llvm::None,
+                                redirects, 0, 0, &errorMessage);
+  if (result) {
+    llvm::WithColor::warning(llvm::errs())
+        << kRocmAgentEnumerator << " invocation error: " << errorMessage
+        << "\n";
+    return kDefaultChip;
+  }
+
+  // Load and parse the result.
+  auto gfxIsaList = openInputFile(tempFilename);
+  if (!gfxIsaList) {
+    llvm::WithColor::error(llvm::errs())
+        << "read ROCm agent list temp file error\n";
+    return kDefaultChip;
+  }
+  for (llvm::line_iterator lines(*gfxIsaList); !lines.is_at_end(); ++lines) {
+    // Skip the line with content "gfx000".
+    if (*lines == "gfx000")
+      continue;
+    // Use the first ISA version found.
+    return lines->str();
+  }
+
+  return kDefaultChip;
+}
+
+// Sets the 'option' to 'value' unless it already has a value.
+static void maybeSetOption(Pass::Option<std::string> &option,
+                           function_ref<std::string()> getValue) {
+  if (!option.hasValue())
+    option = getValue();
+}
+
+SerializeToHsacoPass::SerializeToHsacoPass() {
+  maybeSetOption(this->triple, [] { return "amdgcn-amd-amdhsa"; });
+  maybeSetOption(this->chip, [] {
+    static auto chip = getDefaultChip();
+    return chip;
+  });
+}
+
+void SerializeToHsacoPass::getDependentDialects(
+    DialectRegistry &registry) const {
+  registerROCDLDialectTranslation(registry);
+  gpu::SerializeToBlobPass::getDependentDialects(registry);
+}
+
+std::unique_ptr<SmallVectorImpl<char>>
+SerializeToHsacoPass::assembleIsa(const std::string &isa) {
+  auto loc = getOperation().getLoc();
+
+  SmallVector<char, 0> result;
+  llvm::raw_svector_ostream os(result);
+
+  llvm::Triple triple(llvm::Triple::normalize(this->triple));
+  std::string error;
+  const llvm::Target *target =
+      llvm::TargetRegistry::lookupTarget(triple.normalize(), error);
+  if (!target) {
+    emitError(loc, Twine("failed to lookup target: ") + error);
+    return {};
+  }
+
+  llvm::SourceMgr srcMgr;
+  srcMgr.AddNewSourceBuffer(llvm::MemoryBuffer::getMemBuffer(isa),
+                            llvm::SMLoc());
+
+  const llvm::MCTargetOptions mcOptions;
+  std::unique_ptr<llvm::MCRegisterInfo> mri(
+      target->createMCRegInfo(this->triple));
+  std::unique_ptr<llvm::MCAsmInfo> mai(
+      target->createMCAsmInfo(*mri, this->triple, mcOptions));
+  mai->setRelaxELFRelocations(true);
+
+  llvm::MCObjectFileInfo mofi;
+  llvm::MCContext ctx(mai.get(), mri.get(), &mofi, &srcMgr, &mcOptions);
+  mofi.InitMCObjectFileInfo(triple, false, ctx, false);
+
+  SmallString<128> cwd;
+  if (!llvm::sys::fs::current_path(cwd))
+    ctx.setCompilationDir(cwd);
+
+  std::unique_ptr<llvm::MCStreamer> mcStreamer;
+  std::unique_ptr<llvm::MCInstrInfo> mcii(target->createMCInstrInfo());
+  std::unique_ptr<llvm::MCSubtargetInfo> sti(
+      target->createMCSubtargetInfo(this->triple, this->chip, this->features));
+
+  llvm::MCCodeEmitter *ce = target->createMCCodeEmitter(*mcii, *mri, ctx);
+  llvm::MCAsmBackend *mab = target->createMCAsmBackend(*sti, *mri, mcOptions);
+  mcStreamer.reset(target->createMCObjectStreamer(
+      triple, ctx, std::unique_ptr<llvm::MCAsmBackend>(mab),
+      mab->createObjectWriter(os), std::unique_ptr<llvm::MCCodeEmitter>(ce),
+      *sti, mcOptions.MCRelaxAll, mcOptions.MCIncrementalLinkerCompatible,
+      /*DWARFMustBeAtTheEnd*/ false));
+  mcStreamer->setUseAssemblerInfoForParsing(true);
+
+  std::unique_ptr<llvm::MCAsmParser> parser(
+      createMCAsmParser(srcMgr, ctx, *mcStreamer, *mai));
+  std::unique_ptr<llvm::MCTargetAsmParser> tap(
+      target->createMCAsmParser(*sti, *parser, *mcii, mcOptions));
+
+  if (!tap) {
+    emitError(loc, "assembler initialization error");
+    return {};
+  }
+
+  parser->setTargetParser(*tap);
+  parser->Run(false);
+
+  return std::make_unique<SmallVector<char, 0>>(std::move(result));
+}
+
+std::unique_ptr<std::vector<char>>
+SerializeToHsacoPass::createHsaco(const SmallVectorImpl<char> &isaBinary) {
+  auto loc = getOperation().getLoc();
+
+  // Save the ISA binary to a temp file.
+  int tempIsaBinaryFd = -1;
+  SmallString<128> tempIsaBinaryFilename;
+  if (llvm::sys::fs::createTemporaryFile("kernel", "o", tempIsaBinaryFd,
+                                         tempIsaBinaryFilename)) {
+    emitError(loc, "temporary file for ISA binary creation error");
+    return {};
+  }
+  llvm::FileRemover cleanupIsaBinary(tempIsaBinaryFilename);
+  llvm::raw_fd_ostream tempIsaBinaryOs(tempIsaBinaryFd, true);
+  tempIsaBinaryOs << StringRef(isaBinary.data(), isaBinary.size());
+  tempIsaBinaryOs.close();
+
+  // Create a temp file for HSA code object.
+  int tempHsacoFD = -1;
+  SmallString<128> tempHsacoFilename;
+  if (llvm::sys::fs::createTemporaryFile("kernel", "hsaco", tempHsacoFD,
+                                         tempHsacoFilename)) {
+    emitError(loc, "temporary file for HSA code object creation error");
+    return {};
+  }
+  llvm::FileRemover cleanupHsaco(tempHsacoFilename);
+
+  {
+    static std::mutex mutex;
+    const std::lock_guard<std::mutex> lock(mutex);
+    // Invoke lld. Expect a true return value from lld.
+    if (!lld::elf::link({"ld.lld", "-shared", tempIsaBinaryFilename.c_str(),
+                         "-o", tempHsacoFilename.c_str()},
+                        /*canEarlyExit=*/false, llvm::outs(), llvm::errs())) {
+      emitError(loc, "lld invocation error");
+      return {};
+    }
+  }
+
+  // Load the HSA code object.
+  auto hsacoFile = openInputFile(tempHsacoFilename);
+  if (!hsacoFile) {
+    emitError(loc, "read HSA code object from temp file error");
+    return {};
+  }
+
+  StringRef buffer = hsacoFile->getBuffer();
+  return std::make_unique<std::vector<char>>(buffer.begin(), buffer.end());
+}
+
+std::unique_ptr<std::vector<char>>
+SerializeToHsacoPass::serializeISA(const std::string &isa) {
+  auto isaBinary = assembleIsa(isa);
+  if (!isaBinary)
+    return {};
+  return createHsaco(*isaBinary);
+}
+
+// Register pass to serialize GPU kernel functions to a HSACO binary annotation.
+void mlir::registerGpuSerializeToHsacoPass() {
+  PassRegistration<SerializeToHsacoPass> registerSerializeToHSACO(
+      "gpu-to-hsaco", "Lower GPU kernel function to HSACO binary annotations",
+      [] {
+        // Initialize LLVM AMDGPU backend.
+        LLVMInitializeAMDGPUAsmParser();
+        LLVMInitializeAMDGPUAsmPrinter();
+        LLVMInitializeAMDGPUTarget();
+        LLVMInitializeAMDGPUTargetInfo();
+        LLVMInitializeAMDGPUTargetMC();
+
+        return std::make_unique<SerializeToHsacoPass>();
+      });
+}
+#else  // MLIR_GPU_TO_HSACO_PASS_ENABLE
+void mlir::registerGpuSerializeToHsacoPass() {}
+#endif // MLIR_GPU_TO_HSACO_PASS_ENABLE
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 3b268d703a745668f44e17c9d2f5fe4ea546fd52..744f0276daa84a5c495fcc40a644db21b387f40e 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -842,10 +842,9 @@ struct FoldInitTensorWithTensorReshapeOp
 };
 } // namespace
 
-void InitTensorOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results
-      .insert<FoldInitTensorWithSubTensorOp, FoldInitTensorWithTensorReshapeOp,
+void InitTensorOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                               MLIRContext *context) {
+  results.add<FoldInitTensorWithSubTensorOp, FoldInitTensorWithTensorReshapeOp,
               ReplaceDimOfInitTensorOp, ReplaceStaticShapeDims>(context);
 }
 
@@ -1546,9 +1545,9 @@ static LogicalResult verify(ReshapeOp op) {
   return success();
 }
 
-void ReshapeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ReshapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
-  results.insert<CollapseReshapeOps<ReshapeOp>>(context);
+  results.add<CollapseReshapeOps<ReshapeOp>>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1659,12 +1658,33 @@ struct ReplaceDimOfReshapeOpResult : OpRewritePattern<memref::DimOp> {
     return success();
   }
 };
+
+/// Fold linalg.fill -> linalg.tensor_reshape chain.
+///
+/// For such op chains, we can create new linalg.fill ops with the result
+/// type of the linalg.tensor_reshape op.
+struct FoldFillWithTensorReshape : OpRewritePattern<TensorReshapeOp> {
+  using OpRewritePattern<TensorReshapeOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(TensorReshapeOp reshapeOp,
+                                PatternRewriter &rewriter) const override {
+    auto oldFill = reshapeOp.src().getDefiningOp<FillOp>();
+    if (!oldFill)
+      return failure();
+
+    auto newInit = rewriter.create<InitTensorOp>(
+        oldFill.getLoc(), reshapeOp.getResultType().getShape(),
+        reshapeOp.getResultType().getElementType());
+    rewriter.replaceOpWithNewOp<FillOp>(reshapeOp, newInit, oldFill.value());
+
+    return success();
+  }
+};
 } // namespace
 
-void TensorReshapeOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<CollapseReshapeOps<TensorReshapeOp>, FoldReshapeWithConstant,
-                 ReplaceDimOfReshapeOpResult>(context);
+void TensorReshapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                  MLIRContext *context) {
+  results.add<CollapseReshapeOps<TensorReshapeOp>, FoldFillWithTensorReshape,
+              FoldReshapeWithConstant, ReplaceDimOfReshapeOpResult>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1744,7 +1764,7 @@ static LogicalResult verify(linalg::YieldOp op) {
 void TiledLoopOp::build(
     OpBuilder &builder, OperationState &result, ValueRange lowerBounds,
     ValueRange upperBounds, ValueRange steps, ValueRange inputs,
-    ValueRange outputs, ArrayRef<StringRef> iteratorTypes,
+    ValueRange outputs, ArrayAttr iteratorTypes,
     function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
   result.addOperands(lowerBounds);
   result.addOperands(upperBounds);
@@ -1758,9 +1778,14 @@ void TiledLoopOp::build(
                                 static_cast<int32_t>(steps.size()),
                                 static_cast<int32_t>(inputs.size()),
                                 static_cast<int32_t>(outputs.size())}));
-  result.addAttribute(getIteratorTypesAttrName(),
-                      builder.getStrArrayAttr(iteratorTypes));
-  result.addTypes(outputs.getTypes());
+  result.addAttribute(getIteratorTypesAttrName(), iteratorTypes);
+
+  // Add output types for `RankedTensorType` output arguments.
+  for (Value output : outputs) {
+    Type outputType = output.getType();
+    if (outputType.isa<RankedTensorType>())
+      result.addTypes(outputType);
+  }
 
   OpBuilder::InsertionGuard guard(builder);
   unsigned numIVs = steps.size();
@@ -1771,8 +1796,8 @@ void TiledLoopOp::build(
   if (bodyBuilderFn) {
     builder.setInsertionPointToStart(bodyBlock);
     bodyBuilderFn(builder, result.location, bodyBlock->getArguments());
+    TiledLoopOp::ensureTerminator(*bodyRegion, builder, result.location);
   }
-  TiledLoopOp::ensureTerminator(*bodyRegion, builder, result.location);
 }
 
 static void print(OpAsmPrinter &p, TiledLoopOp op) {
@@ -1858,7 +1883,9 @@ static ParseResult parseTiledLoopOp(OpAsmParser &parser,
     if (parser.resolveOperands(outputs, outputTypes, outputsOperandsLoc,
                                result.operands))
       return failure();
-    result.addTypes(outputTypes);
+    for (Type outputType : outputTypes)
+      if (outputType.isa<RankedTensorType>())
+        result.addTypes(outputType);
   }
 
   // Parse attributes.
@@ -1916,6 +1943,87 @@ bool TiledLoopOp::isDefinedOutsideOfLoop(Value value) {
 
 static LogicalResult verify(TiledLoopOp op) { return success(); }
 
+namespace {
+
+// Folds away TiledLoopOp output tensors when the following conditions are met:
+// * result of `linalg.tiled_loop` has no uses
+// * output tensor is the argument of `linalg.yield`
+//
+// Example:
+//
+// %0 = linalg.tiled_loop ...  outs (%out, %out_buf:tensor<...>, memref<...>) {
+//   ...
+//   linalg.yield %out : tensor ...
+// }
+//
+// Becomes
+//
+// linalg.tiled_loop ...  outs (%out_buf:memref<...>) {
+//   ...
+//   linalg.yield
+// }
+struct TiledLoopResultsFolder : public OpRewritePattern<linalg::TiledLoopOp> {
+  using OpRewritePattern<linalg::TiledLoopOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(linalg::TiledLoopOp tiledLoop,
+                                PatternRewriter &rewriter) const final {
+    if (tiledLoop.getNumResults() == 0)
+      return failure();
+
+    Block *block = tiledLoop.getBody();
+    auto yieldOp = cast<linalg::YieldOp>(block->getTerminator());
+
+    // Match the pattern and collect output buffers that will replace the output
+    // tensors and also the ops that will be ignored when cloning the body.
+    SmallVector<Value, 2> newOutputOperands, newYieldArgs;
+    int resultId = 0;
+    for (Value out : tiledLoop.outputs()) {
+      if (!out.getType().isa<RankedTensorType>()) {
+        newOutputOperands.push_back(out);
+        continue;
+      }
+      Value result = tiledLoop.getResult(resultId);
+      Value yieldArg = yieldOp.getOperand(resultId);
+      if (yieldArg != out || !result.use_empty()) {
+        newOutputOperands.push_back(out);
+        newYieldArgs.push_back(yieldArg);
+      }
+      ++resultId;
+    }
+    if (newOutputOperands.size() == tiledLoop.outputs().size())
+      return failure();
+
+    Location loc = tiledLoop.getLoc();
+    auto newTiledLoop = rewriter.create<TiledLoopOp>(
+        loc, tiledLoop.lowerBound(), tiledLoop.upperBound(), tiledLoop.step(),
+        tiledLoop.inputs(), newOutputOperands, tiledLoop.iterator_types());
+
+    // Clone the region ignoring the def-chain for linalg.yield args:
+    // unnecessary `subtensor_insert`, `tensor_load` and `cast` ops.
+    BlockAndValueMapping bvm;
+    bvm.map(tiledLoop.getInductionVars(), newTiledLoop.getInductionVars());
+    OpBuilder innerBuilder =
+        OpBuilder::atBlockEnd(newTiledLoop.getBody(), rewriter.getListener());
+    for (auto &op : tiledLoop.getBody()->without_terminator())
+      innerBuilder.clone(op, bvm);
+    innerBuilder.create<linalg::YieldOp>(loc, newYieldArgs);
+    rewriter.eraseOp(tiledLoop);
+
+    return success();
+  }
+};
+} // namespace
+
+void TiledLoopOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+                                              MLIRContext *context) {
+  results.insert<TiledLoopResultsFolder>(context);
+}
+
+LogicalResult TiledLoopOp::fold(ArrayRef<Attribute>,
+                                SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+
 /////// Operations corresponding to library calls defined with Tablegen ////////
 
 template <typename LinalgPoolingOp>
@@ -2360,16 +2468,12 @@ static LogicalResult verifyNamedStructuredOp(NamedStructuredOpType op) {
 //===----------------------------------------------------------------------===//
 
 namespace {
-struct EraseDeadLinalgOp : public RewritePattern {
-  EraseDeadLinalgOp(PatternBenefit benefit = 1)
-      : RewritePattern(benefit, MatchAnyOpTypeTag()) {}
+struct EraseDeadLinalgOp : public OpInterfaceRewritePattern<LinalgOp> {
+  using OpInterfaceRewritePattern<LinalgOp>::OpInterfaceRewritePattern;
 
-  LogicalResult matchAndRewrite(Operation *op,
+  LogicalResult matchAndRewrite(LinalgOp op,
                                 PatternRewriter &rewriter) const override {
-    auto linalgOp = dyn_cast<LinalgOp>(op);
-    if (!linalgOp)
-      return failure();
-    for (Value v : linalgOp.getShapedOperands()) {
+    for (Value v : op.getShapedOperands()) {
       // Linalg "inputs" may be either tensor or memref type.
       // tensor<0xelt_type> is a convention that may not always mean
       // "0 iterations". Only erase in cases we see memref<...x0x...>.
@@ -2377,7 +2481,7 @@ struct EraseDeadLinalgOp : public RewritePattern {
       if (!mt)
         continue;
       if (llvm::is_contained(mt.getShape(), 0)) {
-        rewriter.eraseOp(linalgOp);
+        rewriter.eraseOp(op);
         return success();
       }
     }
@@ -2385,19 +2489,14 @@ struct EraseDeadLinalgOp : public RewritePattern {
   }
 };
 
-struct FoldTensorCastOp : public RewritePattern {
-  FoldTensorCastOp(PatternBenefit benefit = 1)
-      : RewritePattern(benefit, MatchAnyOpTypeTag()) {}
+struct FoldTensorCastOp : public OpInterfaceRewritePattern<LinalgOp> {
+  using OpInterfaceRewritePattern<LinalgOp>::OpInterfaceRewritePattern;
 
-  LogicalResult matchAndRewrite(Operation *op,
+  LogicalResult matchAndRewrite(LinalgOp op,
                                 PatternRewriter &rewriter) const override {
-    auto linalgOp = dyn_cast<LinalgOp>(op);
-    if (!linalgOp)
-      return failure();
-
     // If no operand comes from a tensor::CastOp and can be folded then fail.
     bool hasTensorCastOperand =
-        llvm::any_of(linalgOp.getShapedOperands(), [&](Value v) {
+        llvm::any_of(op.getShapedOperands(), [&](Value v) {
           if (v.isa<BlockArgument>())
             return false;
           auto castOp = v.getDefiningOp<tensor::CastOp>();
@@ -2411,23 +2510,23 @@ struct FoldTensorCastOp : public RewritePattern {
     SmallVector<Value, 4> newOperands;
     newOperands.reserve(op->getNumOperands());
     // Inputs may fold.
-    for (Value v : linalgOp.getInputs()) {
+    for (Value v : op.getInputs()) {
       auto tensorCastOp = v.getDefiningOp<tensor::CastOp>();
       newOperands.push_back(
           canFoldIntoConsumerOp(tensorCastOp) ? tensorCastOp.source() : v);
     }
     // Init tensors may fold, in which case the resultType must also change.
-    for (Value v : linalgOp.getOutputs()) {
+    for (Value v : op.getOutputs()) {
       auto tensorCastOp = v.getDefiningOp<tensor::CastOp>();
       bool fold = canFoldIntoConsumerOp(tensorCastOp);
       newOperands.push_back(fold ? tensorCastOp.getOperand() : v);
       newResultTypes.push_back(newOperands.back().getType());
     }
-    auto extraOperands = linalgOp.getAssumedNonShapedOperands();
+    auto extraOperands = op.getAssumedNonShapedOperands();
     newOperands.append(extraOperands.begin(), extraOperands.end());
     // Clone op.
     Operation *newOp =
-        linalgOp.clone(rewriter, op->getLoc(), newResultTypes, newOperands);
+        op.clone(rewriter, op->getLoc(), newResultTypes, newOperands);
     SmallVector<Value, 4> replacements;
     replacements.reserve(newOp->getNumResults());
     for (auto result : llvm::zip(op->getResults(), newOp->getResults())) {
@@ -2494,17 +2593,15 @@ struct ReplaceDimOfLinalgOpResult : public OpRewritePattern<memref::DimOp> {
 namespace {
 // Deduplicate redundant args of a linalg op.
 // An arg is redundant if it has the same Value and indexing map as another.
-struct DeduplicateInputs : public RewritePattern {
-  DeduplicateInputs(PatternBenefit benefit = 1)
-      : RewritePattern(benefit, MatchAnyOpTypeTag()) {}
+struct DeduplicateInputs : public OpInterfaceRewritePattern<LinalgOp> {
+  using OpInterfaceRewritePattern<LinalgOp>::OpInterfaceRewritePattern;
 
-  LogicalResult matchAndRewrite(Operation *op,
+  LogicalResult matchAndRewrite(LinalgOp op,
                                 PatternRewriter &rewriter) const override {
     // This pattern reduces the number of arguments of an op, which breaks
     // the invariants of semantically charged named ops.
     if (!isa<GenericOp, IndexedGenericOp>(op))
       return failure();
-    auto linalgOp = cast<LinalgOp>(op);
 
     // Associate each input to an equivalent "canonical" input that has the same
     // Value and indexing map.
@@ -2518,9 +2615,9 @@ struct DeduplicateInputs : public RewritePattern {
     // having a simple "inputIndex -> canonicalInputIndex" integer mapping is
     // convenient.
     SmallVector<int, 6> canonicalInputIndices;
-    for (int i = 0, e = linalgOp.getNumInputs(); i != e; i++) {
-      Value input = linalgOp.getInput(i);
-      AffineMap indexingMap = linalgOp.getInputIndexingMap(i);
+    for (int i = 0, e = op.getNumInputs(); i != e; i++) {
+      Value input = op.getInput(i);
+      AffineMap indexingMap = op.getInputIndexingMap(i);
       // STL-like maps have a convenient behavior for our use case here. In the
       // case of duplicate keys, the insertion is rejected, and the returned
       // iterator gives access to the value already in the map.
@@ -2529,20 +2626,20 @@ struct DeduplicateInputs : public RewritePattern {
     }
 
     // If there are no duplicate args, then bail out.
-    if (canonicalInput.size() == linalgOp.getNumInputs())
+    if (canonicalInput.size() == op.getNumInputs())
       return failure();
 
     // The operands for the newly canonicalized op.
     SmallVector<Value, 6> newOperands;
-    for (auto v : llvm::enumerate(linalgOp.getInputs()))
+    for (auto v : llvm::enumerate(op.getInputs()))
       if (canonicalInputIndices[v.index()] == static_cast<int>(v.index()))
         newOperands.push_back(v.value());
-    llvm::append_range(newOperands, linalgOp.getOutputs());
-    llvm::append_range(newOperands, linalgOp.getAssumedNonShapedOperands());
+    llvm::append_range(newOperands, op.getOutputs());
+    llvm::append_range(newOperands, op.getAssumedNonShapedOperands());
 
     // Clone the old op with new operands.
-    Operation *newOp = linalgOp.clone(rewriter, op->getLoc(),
-                                      op->getResultTypes(), newOperands);
+    Operation *newOp =
+        op.clone(rewriter, op->getLoc(), op->getResultTypes(), newOperands);
     auto newLinalgOp = cast<LinalgOp>(newOp);
 
     // Repair the indexing maps by filtering out the ones that have been
@@ -2567,7 +2664,7 @@ struct DeduplicateInputs : public RewritePattern {
     // Repair the payload entry block by RAUW'ing redundant arguments and
     // erasing them.
     Block &payload = newOp->getRegion(0).front();
-    for (int i = 0, e = linalgOp.getNumInputs(); i < e; i++) {
+    for (int i = 0, e = op.getNumInputs(); i < e; i++) {
       // Iterate in reverse, so that we erase later args first, preventing the
       // argument list from shifting unexpectedly and invalidating all our
       // indices.
@@ -2591,13 +2688,12 @@ struct DeduplicateInputs : public RewritePattern {
 /// 1) All iterator types are parallel
 /// 2) The body contains just a yield operation with the yielded values being
 ///    the arguments corresponding to the operands.
-struct RemoveIdentityLinalgOps : public RewritePattern {
-  RemoveIdentityLinalgOps(PatternBenefit benefit = 1)
-      : RewritePattern(benefit, MatchAnyOpTypeTag()) {}
+struct RemoveIdentityLinalgOps : public OpInterfaceRewritePattern<LinalgOp> {
+  using OpInterfaceRewritePattern<LinalgOp>::OpInterfaceRewritePattern;
 
-  LogicalResult matchAndRewrite(Operation *op,
+  LogicalResult matchAndRewrite(LinalgOp op,
                                 PatternRewriter &rewriter) const override {
-    if (auto copyOp = dyn_cast<CopyOp>(op)) {
+    if (auto copyOp = dyn_cast<CopyOp>(*op)) {
       assert(copyOp.hasBufferSemantics());
       if (copyOp.input() == copyOp.output() &&
           copyOp.inputPermutation() == copyOp.outputPermutation()) {
@@ -2608,11 +2704,10 @@ struct RemoveIdentityLinalgOps : public RewritePattern {
 
     if (!isa<GenericOp, IndexedGenericOp>(op))
       return failure();
-    LinalgOp genericOp = cast<LinalgOp>(op);
-    if (!genericOp.hasTensorSemantics())
+    if (!op.hasTensorSemantics())
       return failure();
     // Check all indexing maps are identity.
-    if (llvm::any_of(genericOp.getIndexingMaps(),
+    if (llvm::any_of(op.getIndexingMaps(),
                      [](AffineMap map) { return !map.isIdentity(); }))
       return failure();
 
@@ -2627,7 +2722,7 @@ struct RemoveIdentityLinalgOps : public RewritePattern {
 
     // Get the argument number of the returned values. That is the operand
     // number to use for replacing uses of this operation.
-    unsigned numIndexArgs = genericOp.getNumPayloadInductionVariables();
+    unsigned numIndexArgs = op.getNumPayloadInductionVariables();
     SmallVector<Value, 4> returnedArgs;
     for (Value yieldVal : yieldOp.values()) {
       auto yieldArg = yieldVal.dyn_cast<BlockArgument>();
@@ -2638,20 +2733,19 @@ struct RemoveIdentityLinalgOps : public RewritePattern {
         return failure();
       returnedArgs.push_back(op->getOperand(argumentNumber - numIndexArgs));
     }
-    if (returnedArgs.size() != genericOp.getOperation()->getNumResults())
+    if (returnedArgs.size() != op.getOperation()->getNumResults())
       return failure();
-    rewriter.replaceOp(genericOp, returnedArgs);
+    rewriter.replaceOp(op, returnedArgs);
     return success();
   }
 };
 } // namespace
 
 #define CANONICALIZERS_AND_FOLDERS(XXX)                                        \
-  void XXX::getCanonicalizationPatterns(OwningRewritePatternList &results,     \
+  void XXX::getCanonicalizationPatterns(RewritePatternSet &results,            \
                                         MLIRContext *context) {                \
-    results.insert<DeduplicateInputs, EraseDeadLinalgOp, FoldTensorCastOp,     \
-                   RemoveIdentityLinalgOps>();                                 \
-    results.insert<ReplaceDimOfLinalgOpResult>(context);                       \
+    results.add<DeduplicateInputs, EraseDeadLinalgOp, FoldTensorCastOp,        \
+                RemoveIdentityLinalgOps, ReplaceDimOfLinalgOpResult>(context); \
   }                                                                            \
                                                                                \
   LogicalResult XXX::fold(ArrayRef<Attribute>,                                 \
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
index 32b2ee706d195cebc5b8b2b502fa6fe177a44747..3ab86beb9367cb0f1e636706e9b0587e7d1a5791 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
@@ -32,8 +32,7 @@ static Value cloneMemref(Location loc, Value memref, OpBuilder &b) {
 }
 
 static LogicalResult
-allocateBuffersForResults(Location loc, LinalgOp linalgOp,
-                          linalg::GenericOpAdaptor &adaptor,
+allocateBuffersForResults(Location loc, LinalgOp linalgOp, ValueRange outputs,
                           SmallVectorImpl<Value> &resultBuffers, OpBuilder &b) {
   // Lazily compute loopRanges.
   SmallVector<Range, 4> loopRanges;
@@ -52,7 +51,7 @@ allocateBuffersForResults(Location loc, LinalgOp linalgOp,
     }
     auto tensorShape = tensorType.getShape();
     auto memrefType = MemRefType::get(tensorShape, tensorType.getElementType());
-    Value resultTensor = adaptor.outputs()[resultIndex];
+    Value resultTensor = outputs[resultIndex];
 
     // Clone output buffers whose value is actually used.
     if (linalgOp.payloadUsesValueFromOutputOperandIndex(resultIndex)) {
@@ -138,8 +137,7 @@ static void finalizeBufferAllocation(ConversionPatternRewriter &rewriter,
 
 namespace {
 
-/// Generic conversion pattern that matches any LinalgOp. This avoids template
-/// instantiating one pattern for each LinalgOp.
+/// Conversion pattern that replaces `linalg.init_tensor` with allocation.
 class BufferizeInitTensorOp : public OpConversionPattern<InitTensorOp> {
 public:
   using OpConversionPattern<InitTensorOp>::OpConversionPattern;
@@ -155,19 +153,37 @@ public:
   }
 };
 
+/// Conversion pattern that bufferizes `linalg.fill` operation.
+class BufferizeFillOp : public OpConversionPattern<FillOp> {
+public:
+  using OpConversionPattern<FillOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(FillOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    linalg::FillOpAdaptor adaptor(operands, op->getAttrDictionary());
+    if (!op.output().getType().isa<TensorType>())
+      return rewriter.notifyMatchFailure(op,
+                                         "operand must be of a tensor type");
+
+    rewriter.create<FillOp>(op.getLoc(), adaptor.output(), adaptor.value());
+    rewriter.replaceOp(op, adaptor.output());
+
+    return success();
+  }
+};
+
 /// Generic conversion pattern that matches any LinalgOp. This avoids template
 /// instantiating one pattern for each LinalgOp.
-class BufferizeAnyLinalgOp : public ConversionPattern {
+class BufferizeAnyLinalgOp : public OpInterfaceConversionPattern<LinalgOp> {
 public:
-  BufferizeAnyLinalgOp(TypeConverter &typeConverter)
-      : ConversionPattern(/*benefit=*/1, typeConverter, MatchAnyOpTypeTag()) {}
+  using OpInterfaceConversionPattern<LinalgOp>::OpInterfaceConversionPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(LinalgOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
-
-    LinalgOp linalgOp = dyn_cast<linalg::LinalgOp>(op);
-    if (!linalgOp)
+    // GenericOpAdaptor below expects an `operand_segment_sizes` attribute.
+    if (!op->hasAttr("operand_segment_sizes"))
       return failure();
 
     // We abuse the GenericOpAdaptor here.
@@ -175,32 +191,30 @@ public:
     // linalg::LinalgOp interface ops.
     linalg::GenericOpAdaptor adaptor(operands, op->getAttrDictionary());
 
-    Location loc = linalgOp.getLoc();
+    Location loc = op.getLoc();
     SmallVector<Value, 2> newOutputBuffers;
 
-    if (failed(allocateBuffersForResults(loc, linalgOp, adaptor,
+    if (failed(allocateBuffersForResults(loc, op, adaptor.outputs(),
                                          newOutputBuffers, rewriter))) {
-      linalgOp.emitOpError()
-          << "Failed to allocate buffers for tensor results.";
-      return failure();
+      return op.emitOpError()
+             << "Failed to allocate buffers for tensor results.";
     }
 
     // Delegate to the linalg generic pattern.
-    if (auto genericOp = dyn_cast<linalg::GenericOp>(op)) {
+    if (auto genericOp = dyn_cast<linalg::GenericOp>(*op)) {
       finalizeBufferAllocationForGenericOp<GenericOp>(
           rewriter, genericOp, adaptor.inputs(), newOutputBuffers);
       return success();
     }
 
     // Delegate to the linalg indexed generic pattern.
-    if (auto genericOp = dyn_cast<linalg::IndexedGenericOp>(op)) {
+    if (auto genericOp = dyn_cast<linalg::IndexedGenericOp>(*op)) {
       finalizeBufferAllocationForGenericOp<IndexedGenericOp>(
           rewriter, genericOp, adaptor.inputs(), newOutputBuffers);
       return success();
     }
 
-    finalizeBufferAllocation(rewriter, linalgOp, adaptor.inputs(),
-                             newOutputBuffers);
+    finalizeBufferAllocation(rewriter, op, adaptor.inputs(), newOutputBuffers);
     return success();
   }
 };
@@ -305,8 +319,8 @@ struct LinalgBufferizePass : public LinalgBufferizeBase<LinalgBufferizePass> {
     target.addDynamicallyLegalDialect<linalg::LinalgDialect>(isLegalOperation);
     target.addDynamicallyLegalOp<ConstantOp>(isLegalOperation);
 
-    OwningRewritePatternList patterns;
-    populateLinalgBufferizePatterns(&context, typeConverter, patterns);
+    RewritePatternSet patterns(&context);
+    populateLinalgBufferizePatterns(typeConverter, patterns);
     if (failed(applyPartialConversion(getOperation(), target,
                                       std::move(patterns))))
       signalPassFailure();
@@ -319,15 +333,15 @@ std::unique_ptr<OperationPass<FuncOp>> mlir::createLinalgBufferizePass() {
 }
 
 void mlir::linalg::populateLinalgBufferizePatterns(
-    MLIRContext *context, BufferizeTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns) {
-  patterns.insert<BufferizeAnyLinalgOp>(typeConverter);
+    BufferizeTypeConverter &typeConverter, RewritePatternSet &patterns) {
   // TODO: Drop this once tensor constants work in standard.
   // clang-format off
-  patterns.insert<
+  patterns.add<
+      BufferizeAnyLinalgOp,
+      BufferizeFillOp,
       BufferizeInitTensorOp,
       SubTensorOpConverter,
       SubTensorInsertOpConverter
-    >(typeConverter, context);
+    >(typeConverter, patterns.getContext());
   // clang-format on
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp b/mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp
index cd7b481aea44440aca0ad80eb5885d9fe1de7b98..e31a6b5210e3c8562fe27c96cee7b5f8d3a182f7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp
@@ -31,7 +31,7 @@ void mlir::linalg::CodegenStrategy::transform(FuncOp func) const {
   // Emplace patterns one at a time while also maintaining a simple chained
   // state transition.
   unsigned stepCount = 0;
-  SmallVector<FrozenRewritePatternList, 4> stage1Patterns;
+  SmallVector<FrozenRewritePatternSet, 4> stage1Patterns;
   auto zeroState = Identifier::get(std::to_string(stepCount), context);
   auto currentState = zeroState;
   for (const std::unique_ptr<Transformation> &t : transformationSequence) {
@@ -45,9 +45,9 @@ void mlir::linalg::CodegenStrategy::transform(FuncOp func) const {
     currentState = nextState;
   }
 
-  OwningRewritePatternList stage2Patterns =
+  RewritePatternSet stage2Patterns =
       linalg::getLinalgTilingCanonicalizationPatterns(context);
-  stage2Patterns.insert<AffineMinSCFCanonicalizationPattern>(context);
+  stage2Patterns.add<AffineMinSCFCanonicalizationPattern>(context);
 
   auto stage3Transforms = [&](Operation *op) {
     // Some of these may be too aggressive as a stage 3 that is applied on each
@@ -76,18 +76,18 @@ void mlir::linalg::CodegenStrategy::transform(FuncOp func) const {
 
   // Programmatic splitting of slow/fast path vector transfers.
   if (lateCodegenStrategyOptions.enableVectorTransferPartialRewrite) {
-    OwningRewritePatternList patterns;
-    patterns.insert<vector::VectorTransferFullPartialRewriter>(
+    RewritePatternSet patterns(context);
+    patterns.add<vector::VectorTransferFullPartialRewriter>(
         context, vectorTransformsOptions);
     (void)applyPatternsAndFoldGreedily(func, std::move(patterns));
   }
 
   // Programmatic controlled lowering of vector.contract only.
   if (lateCodegenStrategyOptions.enableVectorContractLowering) {
-    OwningRewritePatternList vectorContractLoweringPatterns;
+    RewritePatternSet vectorContractLoweringPatterns(context);
     vectorContractLoweringPatterns
-        .insert<ContractionOpToOuterProductOpLowering,
-                ContractionOpToMatmulOpLowering, ContractionOpLowering>(
+        .add<ContractionOpToOuterProductOpLowering,
+             ContractionOpToMatmulOpLowering, ContractionOpLowering>(
             vectorTransformsOptions, context);
     (void)applyPatternsAndFoldGreedily(
         func, std::move(vectorContractLoweringPatterns));
@@ -95,8 +95,8 @@ void mlir::linalg::CodegenStrategy::transform(FuncOp func) const {
 
   // Programmatic controlled lowering of vector.transfer only.
   if (lateCodegenStrategyOptions.enableVectorToSCFConversion) {
-    OwningRewritePatternList vectorToLoopsPatterns;
-    populateVectorToSCFConversionPatterns(vectorToLoopsPatterns, context,
+    RewritePatternSet vectorToLoopsPatterns(context);
+    populateVectorToSCFConversionPatterns(vectorToLoopsPatterns,
                                           vectorToSCFOptions);
     (void)applyPatternsAndFoldGreedily(func, std::move(vectorToLoopsPatterns));
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index 2d34468dae72e90b59ad1c6479f02e947ddd3105..85b9836d5d36995cf1fa6a1d248476699a1f6f4b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -83,7 +83,7 @@ public:
 struct FunctionNonEntryBlockConversion : public ConversionPattern {
   FunctionNonEntryBlockConversion(StringRef functionLikeOpName,
                                   MLIRContext *ctx, TypeConverter &converter)
-      : ConversionPattern(functionLikeOpName, /*benefit=*/1, converter, ctx) {}
+      : ConversionPattern(converter, functionLikeOpName, /*benefit=*/1, ctx) {}
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -163,7 +163,7 @@ struct LinalgDetensorize : public LinalgDetensorizeBase<LinalgDetensorize> {
   void runOnFunction() override {
     auto *context = &getContext();
     DetensorizeTypeConverter typeConverter;
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
     ConversionTarget target(*context);
 
     target.addDynamicallyLegalOp<GenericOp>([&](GenericOp op) {
@@ -194,19 +194,18 @@ struct LinalgDetensorize : public LinalgDetensorizeBase<LinalgDetensorize> {
                  op, typeConverter, /*returnOpAlwaysLegal*/ true);
     });
 
-    patterns.insert<DetensorizeGenericOp>(typeConverter, context);
-    patterns.insert<FunctionNonEntryBlockConversion>(FuncOp::getOperationName(),
-                                                     context, typeConverter);
+    patterns.add<DetensorizeGenericOp>(typeConverter, context);
+    patterns.add<FunctionNonEntryBlockConversion>(FuncOp::getOperationName(),
+                                                  context, typeConverter);
     // Since non-entry block arguments get detensorized, we also need to update
     // the control flow inside the function to reflect the correct types.
-    populateBranchOpInterfaceTypeConversionPattern(patterns, context,
-                                                   typeConverter);
+    populateBranchOpInterfaceTypeConversionPattern(patterns, typeConverter);
 
     if (failed(applyFullConversion(getFunction(), target, std::move(patterns))))
       signalPassFailure();
 
-    OwningRewritePatternList canonPatterns;
-    canonPatterns.insert<ExtractFromReshapeFromElements>(context);
+    RewritePatternSet canonPatterns(context);
+    canonPatterns.add<ExtractFromReshapeFromElements>(context);
     if (failed(applyPatternsAndFoldGreedily(getFunction(),
                                             std::move(canonPatterns))))
       signalPassFailure();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index c7b76404b2f88ecaa84b1ccf47d055d2ddd70ade..2d3e16fab9605406d8e46b047c45d2613b3a00bd 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -294,8 +294,8 @@ struct ReplaceUnitExtentTensors : public OpRewritePattern<GenericOpTy> {
   using OpRewritePattern<GenericOpTy>::OpRewritePattern;
   LogicalResult matchAndRewrite(GenericOpTy op,
                                 PatternRewriter &rewriter) const override {
-    // TODO: support init_tensors and reductions.
-    if (!op.hasTensorSemantics() || op.getNumInitTensors() != 0)
+    // TODO: support reductions.
+    if (!op.hasTensorSemantics())
       return failure();
 
     MLIRContext *context = rewriter.getContext();
@@ -490,14 +490,14 @@ struct FoldReshapeOpWithUnitExtent : OpRewritePattern<TensorReshapeOp> {
 /// Patterns that are used to canonicalize the use of unit-extent dims for
 /// broadcasting.
 void mlir::populateLinalgFoldUnitExtentDimsPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns) {
-  patterns
-      .insert<FoldUnitDimLoops<GenericOp>, FoldUnitDimLoops<IndexedGenericOp>,
-              ReplaceUnitExtentTensors<GenericOp>,
-              ReplaceUnitExtentTensors<IndexedGenericOp>>(context);
+    RewritePatternSet &patterns) {
+  auto *context = patterns.getContext();
+  patterns.add<FoldUnitDimLoops<GenericOp>, FoldUnitDimLoops<IndexedGenericOp>,
+               ReplaceUnitExtentTensors<GenericOp>,
+               ReplaceUnitExtentTensors<IndexedGenericOp>>(context);
   TensorReshapeOp::getCanonicalizationPatterns(patterns, context);
-  patterns.insert<FoldReshapeOpWithUnitExtent>(context);
-  populateFoldUnitDimsReshapeOpsByLinearizationPatterns(context, patterns);
+  patterns.add<FoldReshapeOpWithUnitExtent>(context);
+  populateFoldUnitDimsReshapeOpsByLinearizationPatterns(patterns);
 }
 
 namespace {
@@ -505,14 +505,15 @@ namespace {
 struct LinalgFoldUnitExtentDimsPass
     : public LinalgFoldUnitExtentDimsBase<LinalgFoldUnitExtentDimsPass> {
   void runOnFunction() override {
-    OwningRewritePatternList patterns;
     FuncOp funcOp = getFunction();
     MLIRContext *context = funcOp.getContext();
+    RewritePatternSet patterns(context);
     if (foldOneTripLoopsOnly)
-      patterns.insert<FoldUnitDimLoops<GenericOp>,
-                      FoldUnitDimLoops<IndexedGenericOp>>(context);
+      patterns
+          .add<FoldUnitDimLoops<GenericOp>, FoldUnitDimLoops<IndexedGenericOp>>(
+              context);
     else
-      populateLinalgFoldUnitExtentDimsPatterns(context, patterns);
+      populateLinalgFoldUnitExtentDimsPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(funcOp.getBody(), std::move(patterns));
   }
 };
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
index 1d50e067cff300579033c3ee966196d74e8436c3..86b7eafa4ecce6c07ce2ebf0f79dbb7025a67216 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
@@ -75,8 +75,8 @@ getOrCreateOperandsMatchingResultTypes(OpBuilder &b, Operation *op) {
 
 namespace {
 struct ConvertAnyElementwiseMappableOpOnRankedTensors : public RewritePattern {
-  ConvertAnyElementwiseMappableOpOnRankedTensors()
-      : RewritePattern(/*benefit=*/1, MatchAnyOpTypeTag()) {}
+  ConvertAnyElementwiseMappableOpOnRankedTensors(MLIRContext *context)
+      : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/1, context) {}
   LogicalResult matchAndRewrite(Operation *op,
                                 PatternRewriter &rewriter) const final {
     if (!isElementwiseMappableOpOnRankedTensors(op))
@@ -116,8 +116,9 @@ struct ConvertAnyElementwiseMappableOpOnRankedTensors : public RewritePattern {
 } // namespace
 
 void mlir::populateElementwiseToLinalgConversionPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *) {
-  patterns.insert<ConvertAnyElementwiseMappableOpOnRankedTensors>();
+    RewritePatternSet &patterns) {
+  patterns.add<ConvertAnyElementwiseMappableOpOnRankedTensors>(
+      patterns.getContext());
 }
 
 namespace {
@@ -128,9 +129,9 @@ class ConvertElementwiseToLinalgPass
     auto func = getOperation();
     auto *context = &getContext();
     ConversionTarget target(*context);
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
 
-    populateElementwiseToLinalgConversionPatterns(patterns, context);
+    populateElementwiseToLinalgConversionPatterns(patterns);
     target.markUnknownOpDynamicallyLegal([](Operation *op) {
       return !isElementwiseMappableOpOnRankedTensors(op);
     });
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index ffa2811c37d21b05148dedd4d9e1277384fa20eb..ea7fd62baad4e10ecf5376f17af4080e1116e0a0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -59,104 +59,6 @@ using llvm::dbgs;
 /// More advanced use cases, analyses as well as profitability heuristics are
 /// left for future work.
 
-// Fill `offset`, `sizes` and `strides` used to iterate over the shape indexed
-// by `permutationMap`.
-static void inferShapeComponents(AffineMap permutationMap,
-                                 ArrayRef<Range> loopRanges,
-                                 SmallVectorImpl<OpFoldResult> &offsets,
-                                 SmallVectorImpl<OpFoldResult> &sizes,
-                                 SmallVectorImpl<OpFoldResult> &strides) {
-  assert(permutationMap.isProjectedPermutation() &&
-         "expected some subset of a permutation map");
-  SmallVector<Range, 4> shapeRanges(permutationMap.getNumResults());
-  unsigned idx = 0;
-  for (AffineExpr e : permutationMap.getResults()) {
-    // loopToOperandRangesMaps are permutations-only, just swap indices.
-    unsigned loopPos = e.cast<AffineDimExpr>().getPosition();
-    shapeRanges[idx++] = loopRanges[loopPos];
-  }
-  // Construct a new subshape for the tile.
-  unsigned rank = shapeRanges.size();
-  offsets.reserve(rank);
-  sizes.reserve(rank);
-  strides.reserve(rank);
-  for (auto r : shapeRanges) {
-    offsets.push_back(r.offset);
-    sizes.push_back(r.size);
-    strides.push_back(r.stride);
-  }
-}
-
-// Return a cloned version of `op` that operates on `loopRanges`, assumed to be
-// a subset of the original loop ranges of `op`.
-// This is achieved by applying the `loopToOperandRangesMaps` permutation maps
-// to the `loopRanges` in order to obtain view ranges.
-static LinalgOp cloneWithLoopRanges(OpBuilder &b, Location loc, LinalgOp op,
-                                    ArrayRef<Range> loopRanges) {
-  SmallVector<Value, 8> clonedShapes;
-  clonedShapes.reserve(op.getNumShapedOperands());
-
-  // Iterate over the shape operands in order.
-  // Extract the subranges from the linearized ranges.
-  for (auto en : llvm::enumerate(op.getShapedOperands())) {
-    unsigned shapedOperandIdx = en.index();
-    AffineMap map = op.getIndexingMap(shapedOperandIdx);
-    LLVM_DEBUG(llvm::dbgs() << "shapedOperandIdx: " << shapedOperandIdx
-                            << " with indexingMap: " << map << "\n");
-    SmallVector<OpFoldResult, 4> offsets, sizes, strides;
-    inferShapeComponents(map, loopRanges, offsets, sizes, strides);
-    Value shape = en.value();
-    Value sub =
-        shape.getType().isa<MemRefType>()
-            ? b.create<memref::SubViewOp>(loc, shape, offsets, sizes, strides)
-                  .getResult()
-            : b.create<SubTensorOp>(loc, shape, offsets, sizes, strides)
-                  .getResult();
-    clonedShapes.push_back(sub);
-  }
-  // Append the other operands.
-  auto operands = op.getAssumedNonShapedOperands();
-  clonedShapes.append(operands.begin(), operands.end());
-
-  // Iterate over the results in order.
-  // Extract the subtensor type from the linearized range.
-  // Since we do not enforce any canonicalizations on the fly, this is always
-  // fully dynamic at construction time.
-  SmallVector<Type, 4> resultTypes;
-  resultTypes.reserve(op->getNumResults());
-  for (RankedTensorType t : op.getOutputTensorTypes()) {
-    unsigned rank = t.getRank();
-    SmallVector<int64_t, 4> staticOffsetsVector(
-        rank, ShapedType::kDynamicStrideOrOffset);
-    SmallVector<int64_t, 4> staticSizesVector(rank, ShapedType::kDynamicSize);
-    SmallVector<int64_t, 4> staticStridesVector(
-        rank, ShapedType::kDynamicStrideOrOffset);
-    resultTypes.push_back(SubTensorOp::inferResultType(
-        t.cast<RankedTensorType>(), staticOffsetsVector, staticSizesVector,
-        staticStridesVector));
-  }
-
-  Operation *clonedOp = op.clone(b, loc, resultTypes, clonedShapes);
-  // When the producer is an IndexedGenericOp, we have to transform its block
-  // IV arguments according to the tiling of the consumer, i.e. offset them by
-  // the values computed in `loopRanges`.
-  if (auto indexedGenericOp = dyn_cast<IndexedGenericOp>(clonedOp)) {
-    auto &block = indexedGenericOp.region().front();
-    OpBuilder::InsertionGuard g(b);
-    b.setInsertionPointToStart(&block);
-    for (unsigned i = 0, e = indexedGenericOp.getNumLoops(); i < e; ++i) {
-      Value oldIndex = block.getArgument(i);
-      // TODO: replace by an affine_apply.
-      AddIOp newIndex = b.create<AddIOp>(indexedGenericOp.getLoc(), oldIndex,
-                                         loopRanges[i].offset);
-      oldIndex.replaceAllUsesExcept(newIndex,
-                                    SmallPtrSet<Operation *, 1>{newIndex});
-    }
-  }
-
-  return clonedOp;
-}
-
 struct ShapeDimension {
   Value shape;
   unsigned dimension;
@@ -208,35 +110,86 @@ getShapeDefiningLoopRange(LinalgOp op, unsigned loopDepth,
   llvm_unreachable("Expect to be able to extract a shape defining loop range");
 }
 
-/// Fuse the producer by cloning the `producer`. The `fusedLoopsAndRanges`
+/// Fuses the producer by cloning the `producer`. The `fusedLoopsAndRanges`
 /// provides the loop range information for the fused loops. The rest are
 /// obtained from the producer itself, since they are not tiled + fused.
-static LinalgOp fuse(OpBuilder &b, LinalgOp producer,
+static LinalgOp fuse(OpBuilder &builder, LinalgOp producer,
                      const DenseMap<unsigned, Range> &fusedLoopsAndRanges) {
-
-  unsigned nPar = producer.getNumParallelLoops();
-  unsigned nRed = producer.getNumReductionLoops();
-  unsigned nWin = producer.getNumWindowLoops();
-  SmallVector<Range, 8> loopRanges(nPar + nRed + nWin);
-  for (auto fusedLoops : fusedLoopsAndRanges)
-    loopRanges[fusedLoops.first] = fusedLoops.second;
-
-  // Iterate over all dimensions. For the dimensions not identified by the
-  // producer map for `producerIdx`, we need to explicitly compute the shape
-  // that defines the loop ranges using the `producer`.
-  for (unsigned i = 0, nLoops = loopRanges.size(); i < nLoops; ++i) {
-    if (loopRanges[i].offset)
-      LLVM_DEBUG(llvm::dbgs()
-                 << "existing LoopRange: " << loopRanges[i] << "\n");
-    else {
+  SmallVector<Value, 8> ivs, tileSizes, sizeBounds;
+  SmallVector<Range, 8> loopRanges;
+  auto zero = std_constant_index(0);
+  auto one = std_constant_index(1);
+  Location loc = producer.getLoc();
+
+  for (unsigned i = 0, e = producer.getNumLoops(); i < e; ++i) {
+    auto it = fusedLoopsAndRanges.find(i);
+    if (it != fusedLoopsAndRanges.end()) {
+      ivs.push_back(it->second.offset);
+      tileSizes.push_back(it->second.size);
+      sizeBounds.push_back(nullptr);
+      loopRanges.push_back(it->second);
+      LLVM_DEBUG(llvm::dbgs() << "tiled loop#" << i << " with LoopRange "
+                              << loopRanges.back() << "\n");
+    } else {
       auto shapeDim = getShapeDefiningLoopRange(producer, i);
       Value dim = memref_dim(shapeDim.shape, shapeDim.dimension);
-      loopRanges[i] = Range{std_constant_index(0), dim, std_constant_index(1)};
-      LLVM_DEBUG(llvm::dbgs() << "new LoopRange: " << loopRanges[i] << "\n");
+      tileSizes.push_back(zero);
+      sizeBounds.push_back(dim);
+      loopRanges.push_back(Range{zero, dim, one});
+      LLVM_DEBUG(llvm::dbgs() << "full loop#" << i << " with LoopRange "
+                              << loopRanges.back() << "\n");
+    }
+  }
+
+  SmallVector<Value, 8> clonedShapes;
+  clonedShapes.reserve(producer.getNumShapedOperands());
+
+  // Compute subranges for all tensor input/output operands.
+  auto tiledOperands = llvm::to_vector<4>(producer.getShapedOperands());
+  clonedShapes.append(makeTiledShapes(builder, loc, producer, tiledOperands,
+                                      ivs, tileSizes, sizeBounds));
+
+  // Append the other operands.
+  auto operands = producer.getAssumedNonShapedOperands();
+  clonedShapes.append(operands.begin(), operands.end());
+
+  // Iterate over the results in order.
+  // Extract the subtensor type from the linearized range.
+  // Since we do not enforce any canonicalizations on the fly, this is always
+  // fully dynamic at construction time.
+  SmallVector<Type, 4> resultTypes;
+  resultTypes.reserve(producer->getNumResults());
+  for (RankedTensorType t : producer.getOutputTensorTypes()) {
+    unsigned rank = t.getRank();
+    SmallVector<int64_t, 4> staticOffsetsVector(
+        rank, ShapedType::kDynamicStrideOrOffset);
+    SmallVector<int64_t, 4> staticSizesVector(rank, ShapedType::kDynamicSize);
+    SmallVector<int64_t, 4> staticStridesVector(
+        rank, ShapedType::kDynamicStrideOrOffset);
+    resultTypes.push_back(SubTensorOp::inferResultType(
+        t.cast<RankedTensorType>(), staticOffsetsVector, staticSizesVector,
+        staticStridesVector));
+  }
+
+  Operation *clonedOp = producer.clone(builder, loc, resultTypes, clonedShapes);
+  // When the producer is an IndexedGenericOp, we have to transform its block
+  // IV arguments according to the tiling of the consumer, i.e. offset them by
+  // the values computed in `loopRanges`.
+  if (auto indexedGenericOp = dyn_cast<IndexedGenericOp>(clonedOp)) {
+    auto &block = indexedGenericOp.region().front();
+    OpBuilder::InsertionGuard g(builder);
+    builder.setInsertionPointToStart(&block);
+    for (unsigned i = 0, e = indexedGenericOp.getNumLoops(); i < e; ++i) {
+      Value oldIndex = block.getArgument(i);
+      // TODO: replace by an affine_apply.
+      AddIOp newIndex = builder.create<AddIOp>(indexedGenericOp.getLoc(),
+                                               oldIndex, loopRanges[i].offset);
+      oldIndex.replaceAllUsesExcept(newIndex,
+                                    SmallPtrSet<Operation *, 1>{newIndex});
     }
   }
 
-  return cloneWithLoopRanges(b, producer.getLoc(), producer, loopRanges);
+  return clonedOp;
 }
 
 /// Get the loop range for a dimension `dim` based on the `shapedOperand`. It is
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusionOnTensors.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusionOnTensors.cpp
index ad7ad116ce66e899619c6c530f25db51f8e19bb0..7e89a0887d0df1e95c41f88c28136a5736c63d76 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusionOnTensors.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusionOnTensors.cpp
@@ -37,6 +37,11 @@ static bool areTensorOpsFusable(LinalgOp producer, LinalgOp consumer,
   if (producer.getNumParallelLoops() != producer.getNumLoops())
     return false;
 
+  // Only allow fusing the producer of an input operand for now.
+  // TODO: allow fusing the producer of an output operand.
+  if (consumerIdx >= consumer.getNumInputs())
+    return false;
+
   // Get the consumer index map. The number of results of the consumer index
   // map must match the number of loops of the producer.
   AffineMap consumerIndexMap = consumer.getIndexingMap(consumerIdx);
@@ -120,60 +125,86 @@ static void generateFusedTensorOpRegion(PatternRewriter &rewriter,
        isa<IndexedGenericOp>(consumer.getOperation()))
           ? std::max(producer.getNumLoops(), consumer.getNumLoops())
           : 0;
-  // Firstly, add all the indices to the block arguments.
+
+  // 0. Firstly, add all the indices to the block arguments.
   for (unsigned i = 0, e = numFusedOpIndices; i < e; ++i)
     fusedBlock->addArgument(rewriter.getIndexType());
-  // Map the arguments for the unmodified args from the consumer.
-  for (auto consumerArg : llvm::enumerate(consumerBlock.getArguments())) {
-    if (consumerArg.index() == consumerIdx + numConsumerIndices) {
-      // Map the arguments for the args from the producer.
-      for (auto producerArg :
-           llvm::enumerate(producerBlock.getArguments().take_front(
-               producer.getNumInputs() + numProducerIndices))) {
-        // If producer is an indexed_generic op, map the indices from consumer
-        // loop to producer loop (because the fusedOp is built based on
-        // consumer's perspective).
-        if (producerArg.index() < numProducerIndices) {
-          auto newIndex = rewriter.create<mlir::AffineApplyOp>(
-              producer.getLoc(),
-              consumerToProducerLoopsMap.getSubMap(producerArg.index()),
-              fusedBlock->getArguments().take_front(numFusedOpIndices));
-          mapper.map(producerArg.value(), newIndex);
-        } else {
-          mapper.map(producerArg.value(),
-                     fusedBlock->addArgument(producerArg.value().getType()));
-        }
-      }
-      continue;
-    }
-
-    // If consumer is an indexed_generic op, map the indices to the block
-    // arguments directly. Otherwise, add the same type of argument and map to
-    // it.
-    if (consumerArg.index() < numConsumerIndices) {
-      mapper.map(consumerArg.value(),
-                 fusedBlock->getArgument(consumerArg.index()));
-    } else {
-      mapper.map(consumerArg.value(),
-                 fusedBlock->addArgument(consumerArg.value().getType()));
-    }
+  // 1. Map consumer indices to fusedBlock indices 1-1.
+  mapper.map(consumerBlock.getArguments().take_front(numConsumerIndices),
+             fusedBlock->getArguments().take_front(numConsumerIndices));
+  // 2. Embed producer indices into fusedBlock index space 1-1.
+  for (auto it :
+       llvm::zip(producerBlock.getArguments().take_front(numProducerIndices),
+                 fusedBlock->getArguments().take_front(numProducerIndices))) {
+    auto newIndex = rewriter.create<mlir::AffineApplyOp>(
+        producer.getLoc(),
+        consumerToProducerLoopsMap.getSubMap(std::get<0>(it).getArgNumber()),
+        fusedBlock->getArguments().take_front(numFusedOpIndices));
+    mapper.map(std::get<0>(it), newIndex);
   }
-
-  // Add operations from producer (except the yield operation) to the fused
+  // TODO: allow fusing the producer of an output operand.
+  assert(consumerIdx < consumer.getNumInputs() &&
+         "expected producer of input operand");
+  // 3. Consumer input operands up to consumerIdx (exclusive).
+  for (BlockArgument bbArg : consumerBlock.getArguments()
+                                 .drop_front(numConsumerIndices)
+                                 .take_front(consumerIdx)) // input assumption.
+    mapper.map(bbArg, fusedBlock->addArgument(bbArg.getType()));
+
+  // Replacing consumerIdx requires getting the cloned, yielded, value from
+  // the (cloned) producer block. This happens in step 9.
+
+  // 4. Splice in producer's input operands.
+  for (BlockArgument bbArg : producerBlock.getArguments()
+                                 .drop_front(numProducerIndices)
+                                 .take_front(producer.getNumInputs()))
+    mapper.map(bbArg, fusedBlock->addArgument(bbArg.getType()));
+  // 5. Remaining consumer's input operands (drop past index `consumerIdx`).
+  for (BlockArgument bbArg : consumerBlock.getArguments()
+                                 .drop_front(numConsumerIndices)
+                                 .take_front(consumer.getNumInputs())
+                                 .drop_front(consumerIdx + 1))
+    mapper.map(bbArg, fusedBlock->addArgument(bbArg.getType()));
+  // 6. All of consumer's output operands.
+  for (BlockArgument bbArg :
+       consumerBlock.getArguments().take_back(consumer.getNumOutputs()))
+    mapper.map(bbArg, fusedBlock->addArgument(bbArg.getType()));
+  // 7. All of producer's output operands except the one fused.
+  // TODO: allow fusion of multi-result producers.
+  assert(producer->getNumResults() == 1 && "expected single result producer");
+
+  // 8. Clone operations from producer (except the yield operation) to the fused
   // op.
-  for (auto &op : producerBlock.getOperations()) {
-    if (auto yieldOp = dyn_cast<linalg::YieldOp>(op)) {
-      // Lookup the value the yield operation is mapped to.
-      Value yieldVal = yieldOp.getOperand(0);
-      if (Value clonedVal = mapper.lookupOrNull(yieldVal))
-        mapper.map(consumerBlock.getArgument(consumerIdx + numConsumerIndices),
-                   clonedVal);
-      continue;
-    }
+  for (auto &op : producerBlock.without_terminator())
     rewriter.clone(op, mapper);
+  // 9. Now we can map the consumerBlock's `consumerIdx` block argument. Just
+  // forward the yield operand.
+  auto yieldOp = cast<linalg::YieldOp>(producerBlock.getTerminator());
+  // TODO: allow fusion of multi-result producers.
+  assert(producer->getNumResults() == 1 && "expected single result producer");
+  unsigned producerResultNumber = 0;
+  Value replacement =
+      mapper.lookupOrDefault(yieldOp.getOperand(producerResultNumber));
+  // Sanity checks, if replacement is not already in the mapper then it must be
+  // produced outside.
+  if (replacement == yieldOp.getOperand(producerResultNumber)) {
+    if (auto bb = replacement.dyn_cast<BlockArgument>())
+      assert(bb.getOwner() != &producerBlock &&
+             "yielded block argument must have been mapped");
+    else
+      assert(!producer->isAncestor(replacement.getDefiningOp()) &&
+             "yielded value must have been mapped");
   }
+  mapper.map(consumerBlock.getArgument(consumerIdx + numConsumerIndices),
+             replacement);
+  // 10. Clone operations from the consumer to the fused op.
   for (auto &op : consumerBlock.getOperations())
     rewriter.clone(op, mapper);
+
+  // Sanity checks.
+  assert(fusedBlock->getNumArguments() ==
+             fusedOp->getNumOperands() + numFusedOpIndices &&
+         "Ill-formed LinalgOp region");
 }
 
 static Optional<SmallVector<Value, 1>>
@@ -856,8 +887,6 @@ struct FoldProducerReshapeOpByLinearization
       op->setOperands(fusedOperands);
       op.indexing_mapsAttr(rewriter.getAffineMapArrayAttr(fusedIndexMaps));
       rewriter.finalizeRootUpdate(op);
-      if (reshapeOp.use_empty())
-        rewriter.eraseOp(reshapeOp);
       return success();
     }
     return failure();
@@ -868,9 +897,14 @@ struct FoldProducerReshapeOpByLinearization
 /// generic/indexed_generic op, when the reshape op is collapsing
 /// dimensions. The dimensionality of the loop in the consumer is expanded.
 template <typename GenericOpTy>
-struct FoldWithProducerReshapeOpByExpansion
+class FoldWithProducerReshapeOpByExpansion
     : public OpRewritePattern<GenericOpTy> {
-  using OpRewritePattern<GenericOpTy>::OpRewritePattern;
+public:
+  FoldWithProducerReshapeOpByExpansion(MLIRContext *context,
+                                       bool foldUnitDimReshapes,
+                                       PatternBenefit benefit = 1)
+      : OpRewritePattern<GenericOpTy>(context, benefit),
+        allowFoldingUnitDimReshapes(foldUnitDimReshapes) {}
 
   LogicalResult matchAndRewrite(GenericOpTy genericOp,
                                 PatternRewriter &rewriter) const override {
@@ -887,8 +921,9 @@ struct FoldWithProducerReshapeOpByExpansion
       if (reshapeOp.getSrcType().getRank() <
               reshapeOp.getResultType().getRank() ||
           !isFusableWithReshapeByDimExpansion(linalgOp, operand.index()) ||
-          isUnitDimExpansionOnly(reshapeOp.getSrcType().getShape(),
-                                 reshapeOp.getReassociationMaps()))
+          (!allowFoldingUnitDimReshapes &&
+           isUnitDimExpansionOnly(reshapeOp.getSrcType().getShape(),
+                                  reshapeOp.getReassociationMaps())))
         continue;
 
       Optional<SmallVector<Value, 1>> replacementValues =
@@ -897,12 +932,13 @@ struct FoldWithProducerReshapeOpByExpansion
       if (!replacementValues)
         return failure();
       rewriter.replaceOp(genericOp, replacementValues.getValue());
-      if (reshapeOp.use_empty())
-        rewriter.eraseOp(reshapeOp);
       return success();
     }
     return failure();
   }
+
+private:
+  bool allowFoldingUnitDimReshapes;
 };
 
 /// Pattern to fold tensor_reshape op with its producer. The corresponding index
@@ -963,8 +999,6 @@ struct FoldConsumerReshapeOpByLinearization
     rewriter.cloneRegionBefore(producer->getRegion(0), fusedRegion,
                                fusedRegion.begin());
     rewriter.replaceOp(reshapeOp, fusedOp->getResults());
-    if (producer.use_empty())
-      rewriter.eraseOp(producer);
     return success();
   }
 };
@@ -995,8 +1029,6 @@ struct FoldReshapeWithGenericOpByExpansion
     if (!replacementValues)
       return failure();
     rewriter.replaceOp(reshapeOp, replacementValues.getValue());
-    if (producer.use_empty())
-      rewriter.eraseOp(producer);
     return success();
   }
 };
@@ -1057,8 +1089,6 @@ struct FoldSplatConstants : public OpRewritePattern<LinalgOpTy> {
       rewriter.cloneRegionBefore(linalgOpRegion, fusedRegion,
                                  fusedRegion.begin(), mapping);
       rewriter.replaceOp(linalgOp, fusedOp->getResults());
-      if (constantOp.use_empty())
-        rewriter.eraseOp(constantOp);
       return success();
     }
     return failure();
@@ -1092,15 +1122,14 @@ struct FuseTensorOps : public OpRewritePattern<LinalgOpTy> {
                                 PatternRewriter &rewriter) const override {
     // Find the first operand that is defined by another generic op on tensors.
     for (OpOperand &opOperand : op.getShapedOpOperands()) {
-      Operation *producer = opOperand.get().getDefiningOp();
-      if (!producer)
+      LinalgOp producerOp =
+          dyn_cast_or_null<LinalgOp>(opOperand.get().getDefiningOp());
+      if (!producerOp || !producerOp.hasTensorSemantics())
         continue;
       Optional<SmallVector<Value, 1>> fusedOpResults =
           fuseTensorOps(rewriter, opOperand);
       if (fusedOpResults) {
         rewriter.replaceOp(op, *fusedOpResults);
-        if (producer->use_empty())
-          rewriter.eraseOp(producer);
         return success();
       }
     }
@@ -1112,9 +1141,10 @@ struct FuseTensorOps : public OpRewritePattern<LinalgOpTy> {
 struct FusionOfTensorOpsPass
     : public LinalgFusionOfTensorOpsBase<FusionOfTensorOpsPass> {
   void runOnOperation() override {
-    OwningRewritePatternList patterns;
     Operation *op = getOperation();
-    populateLinalgTensorOpsFusionPatterns(op->getContext(), patterns);
+    RewritePatternSet patterns(op->getContext());
+    populateLinalgTensorOpsFusionPatterns(patterns,
+                                          allowFoldingUnitDimReshapes);
     (void)applyPatternsAndFoldGreedily(op->getRegions(), std::move(patterns));
   }
 };
@@ -1125,9 +1155,9 @@ struct FoldReshapeOpsByLinearizationPass
     : public LinalgFoldReshapeOpsByLinearizationBase<
           FoldReshapeOpsByLinearizationPass> {
   void runOnOperation() override {
-    OwningRewritePatternList patterns;
     Operation *op = getOperation();
-    populateFoldReshapeOpsByLinearizationPatterns(op->getContext(), patterns);
+    RewritePatternSet patterns(op->getContext());
+    populateFoldReshapeOpsByLinearizationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(op->getRegions(), std::move(patterns));
   }
 };
@@ -1135,33 +1165,38 @@ struct FoldReshapeOpsByLinearizationPass
 } // namespace
 
 void mlir::populateFoldReshapeOpsByLinearizationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns) {
-  patterns.insert<FoldProducerReshapeOpByLinearization<GenericOp, false>,
-                  FoldProducerReshapeOpByLinearization<IndexedGenericOp, false>,
-                  FoldConsumerReshapeOpByLinearization<false>>(context);
+    RewritePatternSet &patterns) {
+  patterns.add<FoldProducerReshapeOpByLinearization<GenericOp, false>,
+               FoldProducerReshapeOpByLinearization<IndexedGenericOp, false>,
+               FoldConsumerReshapeOpByLinearization<false>>(
+      patterns.getContext());
 }
 
 void mlir::populateFoldUnitDimsReshapeOpsByLinearizationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns) {
-  patterns.insert<FoldProducerReshapeOpByLinearization<GenericOp, true>,
-                  FoldProducerReshapeOpByLinearization<IndexedGenericOp, true>,
-                  FoldConsumerReshapeOpByLinearization<true>>(context);
+    RewritePatternSet &patterns) {
+  patterns.add<FoldProducerReshapeOpByLinearization<GenericOp, true>,
+               FoldProducerReshapeOpByLinearization<IndexedGenericOp, true>,
+               FoldConsumerReshapeOpByLinearization<true>>(
+      patterns.getContext());
 }
 
 void mlir::populateFoldReshapeOpsByExpansionPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns) {
-  patterns.insert<FoldReshapeWithGenericOpByExpansion,
-                  FoldWithProducerReshapeOpByExpansion<GenericOp>,
-                  FoldWithProducerReshapeOpByExpansion<IndexedGenericOp>>(
-      context);
+    RewritePatternSet &patterns, bool allowFoldingUnitDimReshapes) {
+  patterns.add<FoldReshapeWithGenericOpByExpansion>(patterns.getContext());
+  patterns.add<FoldWithProducerReshapeOpByExpansion<GenericOp>,
+               FoldWithProducerReshapeOpByExpansion<IndexedGenericOp>>(
+      patterns.getContext(), allowFoldingUnitDimReshapes);
 }
 
 void mlir::populateLinalgTensorOpsFusionPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns) {
-  patterns.insert<FuseTensorOps<GenericOp>, FuseTensorOps<IndexedGenericOp>,
-                  FoldSplatConstants<GenericOp>,
-                  FoldSplatConstants<IndexedGenericOp>>(context);
-  populateFoldReshapeOpsByExpansionPatterns(context, patterns);
+    RewritePatternSet &patterns, bool allowFoldingUnitDimReshapes) {
+  auto *context = patterns.getContext();
+  patterns
+      .add<FuseTensorOps<GenericOp>, FuseTensorOps<IndexedGenericOp>,
+           FoldSplatConstants<GenericOp>, FoldSplatConstants<IndexedGenericOp>>(
+          context);
+  populateFoldReshapeOpsByExpansionPatterns(patterns,
+                                            allowFoldingUnitDimReshapes);
   GenericOp::getCanonicalizationPatterns(patterns, context);
   IndexedGenericOp::getCanonicalizationPatterns(patterns, context);
   TensorReshapeOp::getCanonicalizationPatterns(patterns, context);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
index 69de55c00cb790423c8ac2ec13d283813a08d57a..af3f393997e7cb0c5f14da74cd37616abda78374 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
@@ -104,7 +104,7 @@ struct LinalgNamedOpGeneralizationPattern : RewritePattern {
   LinalgNamedOpGeneralizationPattern(MLIRContext *context,
                                      linalg::LinalgTransformationFilter marker,
                                      PatternBenefit benefit = 1)
-      : RewritePattern(benefit, MatchAnyOpTypeTag()),
+      : RewritePattern(MatchAnyOpTypeTag(), benefit, context),
         marker(std::move(marker)) {}
 
   LogicalResult matchAndRewrite(Operation *rootOp,
@@ -143,9 +143,9 @@ struct LinalgGeneralizationPass
 
 void LinalgGeneralizationPass::runOnFunction() {
   FuncOp func = getFunction();
-  OwningRewritePatternList patterns;
-  linalg::populateLinalgConvGeneralizationPatterns(&getContext(), patterns);
-  linalg::populateLinalgNamedOpsGeneralizationPatterns(&getContext(), patterns);
+  RewritePatternSet patterns(&getContext());
+  linalg::populateLinalgConvGeneralizationPatterns(patterns);
+  linalg::populateLinalgNamedOpsGeneralizationPatterns(patterns);
   (void)applyPatternsAndFoldGreedily(func.getBody(), std::move(patterns));
 }
 
@@ -167,15 +167,14 @@ linalg::GenericOp GeneralizeConvOp::createGenericOp(linalg::ConvOp convOp,
 }
 
 void mlir::linalg::populateLinalgConvGeneralizationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns,
-    linalg::LinalgTransformationFilter marker) {
-  patterns.insert<GeneralizeConvOp>(context, marker);
+    RewritePatternSet &patterns, linalg::LinalgTransformationFilter marker) {
+  patterns.add<GeneralizeConvOp>(patterns.getContext(), marker);
 }
 
 void mlir::linalg::populateLinalgNamedOpsGeneralizationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns,
-    linalg::LinalgTransformationFilter marker) {
-  patterns.insert<LinalgNamedOpGeneralizationPattern>(context, marker);
+    RewritePatternSet &patterns, linalg::LinalgTransformationFilter marker) {
+  patterns.add<LinalgNamedOpGeneralizationPattern>(patterns.getContext(),
+                                                   marker);
 }
 
 std::unique_ptr<OperationPass<FuncOp>> mlir::createLinalgGeneralizationPass() {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index cc0cce7e74cc4f9dd6d27da07562a33c0d625d7f..b4a2182cf23a312fde79a5718f03ce90c8da59ad 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -13,7 +13,9 @@
 
 #include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
 #include "mlir/Analysis/SliceAnalysis.h"
+#include "mlir/Dialect/Affine/Utils.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
+#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/SCF/Utils.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
@@ -378,7 +380,7 @@ void mlir::linalg::hoistRedundantVectorTransfersOnTensor(FuncOp func) {
     // Apply canonicalization so the newForOp + yield folds immediately, thus
     // cleaning up the IR and potentially enabling more hoisting.
     if (changed) {
-      OwningRewritePatternList patterns;
+      RewritePatternSet patterns(func->getContext());
       scf::ForOp::getCanonicalizationPatterns(patterns, func->getContext());
       (void)applyPatternsAndFoldGreedily(func, std::move(patterns));
     }
@@ -503,6 +505,135 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
   }
 }
 
+/// Return success if `v` is a value that is only transitively defined by ops of
+/// type in `OpTypeList`.
+template <typename... OpTypeList>
+static bool backwardsSliceOnlyHasOpsOfType(scf::ForOp outerLimit, Value v) {
+  // Compute a backward slice up to, but not including, `outerLimit`.
+  llvm::SetVector<Operation *> backwardSlice;
+  getBackwardSlice(v, &backwardSlice, [&](Operation *op) {
+    return outerLimit->isProperAncestor(op);
+  });
+  // Traverse the backward slice and ensure we can perform the computation to
+  // hoist.
+  for (Operation *op : backwardSlice) {
+    if (isa<OpTypeList...>(op))
+      continue;
+    LLVM_DEBUG(DBGS() << "Abort: unadmissible op in slice " << *op << "\n");
+    return false;
+  }
+  return true;
+}
+
+bool isDefinedOutsideOrConstant(scf::ForOp outer, Value v) {
+  return outer.isDefinedOutsideOfLoop(v) || v.getDefiningOp<ConstantOp>();
+}
+
+/// Compute the tightest lower bound with quantities that are all defined
+/// outside of `outer`.
+/// Return null if such a bound cannot be computed.
+Value computeLoopIndependentLowerBound(OpBuilder &b, scf::ForOp outer,
+                                       Value v) {
+  if (isDefinedOutsideOrConstant(outer, v))
+    return v;
+  return Value();
+}
+
+/// Compute the tightest upper bound with quantities that are all defined
+/// outside of `outer`.
+/// Expects all ops in the backward slice of `v` up to `outer` to be either
+/// scf.for, affine.min or affine.apply.
+static Value computeLoopIndependentUpperBound(OpBuilder &b, scf::ForOp outer,
+                                              Value v) {
+  if (isDefinedOutsideOrConstant(outer, v))
+    return v;
+
+  LLVM_DEBUG(DBGS() << "Begin loopIndependentUpperBound for: " << v << "\n");
+
+  bool ok =
+      backwardsSliceOnlyHasOpsOfType<scf::ForOp, AffineMinOp, AffineApplyOp>(
+          outer, v);
+  assert(ok && "expected to only be defined by scf::ForOp and AffineMinOp");
+  (void)ok;
+
+  // Compute a backward slice up to, but not including, `outer`.
+  llvm::SetVector<Operation *> backwardSlice;
+  getBackwardSlice(v, &backwardSlice,
+                   [&](Operation *op) { return outer->isProperAncestor(op); });
+  backwardSlice.insert(v.getDefiningOp());
+
+  OpBuilder::InsertionGuard g(b);
+  b.setInsertionPoint(outer);
+  Value res = v;
+  BlockAndValueMapping bvm;
+  for (Operation *op : backwardSlice) {
+    if (isa<scf::ForOp>(op))
+      continue;
+    if (isa<AffineApplyOp>(op)) {
+      b.clone(*op, bvm);
+      continue;
+    }
+    auto sliceMinOp = cast<AffineMinOp>(op);
+    // Perform the substitution of the operands of AffineMinOp.
+    auto mapAndOperands = substituteMin(
+        sliceMinOp, [&](Operation *op) { return outer->isAncestor(op); });
+    SmallVector<Value> resultOperands = mapAndOperands.dims;
+    llvm::append_range(resultOperands, mapAndOperands.symbols);
+    AffineMap map = mapAndOperands.map;
+    canonicalizeMapAndOperands(&map, &resultOperands);
+    map = simplifyAffineMap(map);
+    res = b.create<AffineMinOp>(
+        outer->getLoc(), map,
+        llvm::to_vector<4>(llvm::map_range(resultOperands, [&](Value operand) {
+          return bvm.lookupOrDefault(operand);
+        })));
+    bvm.map(sliceMinOp, res);
+  }
+  LLVM_DEBUG(DBGS() << "End loopIndependentUpperBound with: " << res << "\n");
+  return res;
+}
+
+/// Return the number of iterations in the loop (ub - lb).ceilDiv(step).
+/// The returned Value is guaranteed not to depend on any loop comprised in
+/// [`outer`, `forOp`].
+/// Return null if such a loop-independent quantity cannot be computed.
+static Value buildLoopTripCount(OpBuilder &b, scf::ForOp outer,
+                                scf::ForOp forOp) {
+  MLIRContext *ctx = forOp->getContext();
+  AffineExpr lb, ub, step;
+  bindDims(ctx, lb, ub);
+  bindSymbols(ctx, step);
+  Value lbVal = computeLoopIndependentLowerBound(b, outer, forOp.lowerBound()),
+        ubVal = computeLoopIndependentUpperBound(b, outer, forOp.upperBound()),
+        stepVal = forOp.step();
+  if (!lbVal || !ubVal || !stepVal)
+    return Value();
+  auto loc = forOp->getLoc();
+  Value res = b.create<AffineApplyOp>(loc, (ub - lb).ceilDiv(step),
+                                      ValueRange{lbVal, ubVal, stepVal});
+  return res;
+}
+
+/// Return the current iteration number in the loop (iv - lb).ceilDiv(step).
+/// The returned Value is guaranteed not to depend on any loop comprised in
+/// [`outer`, `forOp`].
+/// Return null if such a loop-independent quantity cannot be computed.
+static Value buildLoopIterationCount(OpBuilder &b, scf::ForOp outer,
+                                     scf::ForOp forOp) {
+  MLIRContext *ctx = forOp->getContext();
+  AffineExpr iv, lb, step;
+  bindDims(ctx, iv, lb);
+  bindSymbols(ctx, step);
+  Value ivVal = forOp.getInductionVar(),
+        lbVal = computeLoopIndependentLowerBound(b, outer, forOp.lowerBound()),
+        stepVal = forOp.step();
+  if (!ivVal || !lbVal || !stepVal)
+    return Value();
+  auto loc = forOp->getLoc();
+  return b.create<AffineApplyOp>(loc, (iv - lb).ceilDiv(step),
+                                 ValueRange{ivVal, lbVal, stepVal});
+}
+
 /// Ensure prerequisites that guarantee pad op hoisting can occur.
 /// Return failure in the cases when we cannot perform hoisting; i.e. if either:
 ///   1. There exists a use of `padTensorOp` that is not a linalg input operand.
@@ -510,8 +641,10 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
 ///   3. There exists an op with a region that is dominated by
 ///   `outermostEnclosingForOp` and that isn't a LoopLikeInterface or a
 ///    LinalgOp.
-///   3. There exists an op with side effects that is dominated by
-///    `outermostEnclosingForOp` and that isn't a LoopLikeInterface.
+///   4. There exists an op with side effects that is dominated by
+///   `outermostEnclosingForOp` and that isn't a LoopLikeInterface.
+///   5. The lower bound, upper bound and step of all the loops involved in the
+///   hoisting can be
 ///
 /// While ensuring prerequisites:
 ///   1. Fill the `backwardSlice` to contain the topologically sorted ops
@@ -523,7 +656,8 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
 static LogicalResult
 hoistPaddingOnTensorsPrerequisites(linalg::PadTensorOp padTensorOp, int nLevels,
                                    llvm::SetVector<Operation *> &backwardSlice,
-                                   llvm::SetVector<Operation *> &packingLoops) {
+                                   llvm::SetVector<Operation *> &packingLoops,
+                                   SmallVector<Value> &dynamicTensorSizes) {
   // Bail on any use that isn't an input of a Linalg op.
   // Hoisting of inplace updates happens after vectorization.
   for (OpOperand &use : padTensorOp.result().getUses()) {
@@ -583,36 +717,39 @@ hoistPaddingOnTensorsPrerequisites(linalg::PadTensorOp padTensorOp, int nLevels,
   // `outermostEnclosingForOp`.
   assert(outermostEnclosingForOp == backwardSlice.front());
 
-  return success();
-}
-
-/// Return the number of iterations in the loop (ub - lb).ceilDiv(step).
-static Value buildLoopTripCount(OpBuilder &b, scf::ForOp forOp) {
-  MLIRContext *ctx = forOp->getContext();
-  AffineExpr lb, ub, step;
-  bindDims(ctx, lb, ub);
-  bindSymbols(ctx, step);
-  return b.create<AffineApplyOp>(
-      forOp->getLoc(), AffineMap::get(2, 1, {(ub - lb).ceilDiv(step)}, ctx),
-      ValueRange{forOp.lowerBound(), forOp.upperBound(), forOp.step()});
-}
+  scf::ForOp outer = cast<scf::ForOp>(outermostEnclosingForOp);
+  if (llvm::any_of(packingLoops, [&](Operation *op) {
+        scf::ForOp forOp = cast<scf::ForOp>(op);
+        Value lb = forOp.lowerBound(), ub = forOp.upperBound(),
+              step = forOp.step();
+        return !isDefinedOutsideOrConstant(outer, lb) ||
+               !(isDefinedOutsideOrConstant(outer, ub) ||
+                 backwardsSliceOnlyHasOpsOfType<scf::ForOp, AffineMinOp,
+                                                AffineApplyOp>(outer, ub)) ||
+               !isDefinedOutsideOrConstant(outer, step);
+      }))
+    return failure();
 
-/// Return the current iteration number in the loop (iv - lb).ceilDiv(step).
-static Value buildLoopIterationCount(OpBuilder &b, scf::ForOp forOp) {
-  MLIRContext *ctx = forOp->getContext();
-  AffineExpr iv, lb, step;
-  bindDims(ctx, iv, lb);
-  bindSymbols(ctx, step);
-  return b.create<AffineApplyOp>(
-      forOp->getLoc(), AffineMap::get(2, 1, {(iv - lb).ceilDiv(step)}, ctx),
-      ValueRange{forOp.getInductionVar(), forOp.lowerBound(), forOp.step()});
+  // IP just before the outermost loop considered that we hoist above.
+  OpBuilder b(outermostEnclosingForOp);
+  dynamicTensorSizes =
+      llvm::to_vector<4>(llvm::map_range(packingLoops, [&](Operation *op) {
+        return buildLoopTripCount(b, cast<scf::ForOp>(outermostEnclosingForOp),
+                                  cast<scf::ForOp>(op));
+      }));
+  // Assert all loop trip counts can be computed.
+  if (!llvm::all_of(dynamicTensorSizes, [](Value v) { return v; }))
+    llvm_unreachable("loop independence prerequisite not met");
+  return success();
 }
 
 LogicalResult mlir::linalg::hoistPaddingOnTensors(PadTensorOp &padTensorOp,
                                                   unsigned nLoops) {
+  SmallVector<Value> dynamicTensorSizes;
   llvm::SetVector<Operation *> backwardSlice, packingLoops;
   if (failed(hoistPaddingOnTensorsPrerequisites(padTensorOp, nLoops,
-                                                backwardSlice, packingLoops)))
+                                                backwardSlice, packingLoops,
+                                                dynamicTensorSizes)))
     return failure();
 
   // Update actual number of loops, which may be smaller.
@@ -636,12 +773,8 @@ LogicalResult mlir::linalg::hoistPaddingOnTensors(PadTensorOp &padTensorOp,
   llvm::append_range(packedShape, paddedTensorType.getShape());
   auto packedTensorType =
       RankedTensorType::get(packedShape, paddedTensorType.getElementType());
-  auto dynamicSizes =
-      llvm::to_vector<4>(llvm::map_range(packingLoops, [&](Operation *op) {
-        return buildLoopTripCount(b, cast<scf::ForOp>(op));
-      }));
   Value packedTensor = b.create<linalg::InitTensorOp>(
-      loc, dynamicSizes, packedTensorType.getShape(),
+      loc, dynamicTensorSizes, packedTensorType.getShape(),
       packedTensorType.getElementType());
 
   // Clone the operations involved in the backward slice, iteratively stepping
@@ -656,11 +789,19 @@ LogicalResult mlir::linalg::hoistPaddingOnTensors(PadTensorOp &padTensorOp,
   clonedLoopIvs.reserve(nLoops);
   leadingPackedTensorIndexings.reserve(nLoops);
   BlockAndValueMapping bvm;
-  // Stack step 1. iteratively clone loops and push `packedTensor`.
   // Insert `padTensorOp` into the backwardSlice so we clone it too.
   backwardSlice.insert(padTensorOp);
+  // Stack step 1. iteratively clone loops and push `packedTensor`.
   for (Operation *op : backwardSlice) {
-    if (op->getNumRegions() == 0 || isa<linalg::PadTensorOp>(op)) {
+    // Specifically sit out in the subtenso(packedTensor) case: this is the
+    // piece we seek to replace.
+    if (auto subTensor = dyn_cast<SubTensorOp>(op))
+      if (bvm.lookupOrDefault(subTensor.source()) == packedTensor)
+        continue;
+    auto effects = dyn_cast<MemoryEffectOpInterface>(op);
+    bool hasNoEffects = !effects || effects.hasNoEffect();
+    if (hasNoEffects &&
+        (op->getNumRegions() == 0 || isa<linalg::PadTensorOp>(op))) {
       b.clone(*op, bvm);
       continue;
     }
@@ -670,15 +811,25 @@ LogicalResult mlir::linalg::hoistPaddingOnTensors(PadTensorOp &padTensorOp,
     // Unused loop, just skip it.
     if (!packingLoops.contains(forOp))
       continue;
+
     auto clonedForOp =
-        b.create<scf::ForOp>(loc, forOp.lowerBound(), forOp.upperBound(),
-                             forOp.step(), packedTensor);
+        b.create<scf::ForOp>(loc, bvm.lookupOrDefault(forOp.lowerBound()),
+                             bvm.lookupOrDefault(forOp.upperBound()),
+                             bvm.lookupOrDefault(forOp.step()), packedTensor);
+    // Map the induction var, region args and results to the `clonedForOp`.
+    bvm.map(forOp.getInductionVar(), clonedForOp.getInductionVar());
+    bvm.map(forOp.getRegionIterArgs(), clonedForOp.getRegionIterArgs());
+    bvm.map(forOp.getResults(), clonedForOp.getResults());
     assert(clonedForOp->getNumRegions() == 1);
     clonedLoopIvs.push_back(clonedForOp.getInductionVar());
+
     b.setInsertionPointToStart(&clonedForOp->getRegion(0).front());
-    leadingPackedTensorIndexings.push_back(
-        buildLoopIterationCount(b, clonedForOp));
-    bvm.map(forOp.getInductionVar(), clonedLoopIvs.back());
+    Value loopIndependentIterationCount = buildLoopIterationCount(
+        b, cast<scf::ForOp>(outermostEnclosingForOp), clonedForOp);
+    // Assert the loop-independent iteration count can be computed.
+    if (!loopIndependentIterationCount)
+      llvm_unreachable("loop independence prerequisite not met");
+    leadingPackedTensorIndexings.push_back(loopIndependentIterationCount);
     packedTensor = clonedForOp.getRegionIterArgs().front();
   }
 
@@ -716,8 +867,13 @@ LogicalResult mlir::linalg::hoistPaddingOnTensors(PadTensorOp &padTensorOp,
   b.setInsertionPoint(padTensorOp);
   SmallVector<Value> loopIterationCounts =
       llvm::to_vector<4>(llvm::map_range(packingLoops, [&](Operation *loop) {
-        return buildLoopIterationCount(b, cast<scf::ForOp>(loop));
+        return buildLoopIterationCount(
+            b, cast<scf::ForOp>(outermostEnclosingForOp),
+            cast<scf::ForOp>(loop));
       }));
+  // Assert all loop iteration counts can be computed.
+  if (llvm::any_of(loopIterationCounts, [](Value v) { return !v; }))
+    llvm_unreachable("loop independence prerequisite not met");
   // offsets = [originalLoopIvs, 0 .. 0].
   offsets.assign(loopIterationCounts.begin(), loopIterationCounts.end());
   offsets.append(paddedRank, b.getIndexAttr(0));
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index d6423f4badd4fa8a056a95502365569ee4c3bf8e..a6e296b0ea1120e4e4b21949fa8a9f7daab5d23e 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -520,8 +520,9 @@ namespace {
 template <typename LoopType>
 class LinalgRewritePattern : public RewritePattern {
 public:
-  LinalgRewritePattern(ArrayRef<unsigned> interchangeVector)
-      : RewritePattern(/*benefit=*/1, MatchAnyOpTypeTag()),
+  LinalgRewritePattern(MLIRContext *context,
+                       ArrayRef<unsigned> interchangeVector)
+      : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/1, context),
         interchangeVector(interchangeVector.begin(), interchangeVector.end()) {}
 
   LogicalResult matchAndRewrite(Operation *op,
@@ -545,11 +546,11 @@ template <typename LoopType>
 static void lowerLinalgToLoopsImpl(FuncOp funcOp,
                                    ArrayRef<unsigned> interchangeVector) {
   MLIRContext *context = funcOp.getContext();
-  OwningRewritePatternList patterns;
-  patterns.insert<LinalgRewritePattern<LoopType>>(interchangeVector);
+  RewritePatternSet patterns(context);
+  patterns.add<LinalgRewritePattern<LoopType>>(context, interchangeVector);
   memref::DimOp::getCanonicalizationPatterns(patterns, context);
   AffineApplyOp::getCanonicalizationPatterns(patterns, context);
-  patterns.insert<FoldAffineOp>(context);
+  patterns.add<FoldAffineOp>(context);
   // Just apply the patterns greedily.
   (void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns));
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/SparseLowering.cpp b/mlir/lib/Dialect/Linalg/Transforms/SparseLowering.cpp
index d9c258083a52374aaecd110550d097c27d01d7c2..f4c84fc4366b0745ca26964ead039b53a4cd6c83 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/SparseLowering.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SparseLowering.cpp
@@ -137,8 +137,8 @@ public:
 /// Populates the given patterns list with conversion rules required for
 /// the sparsification of linear algebra operations.
 void linalg::populateSparsificationConversionPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns) {
-  patterns.insert<TensorFromPointerConverter, TensorToDimSizeConverter,
-                  TensorToPointersConverter, TensorToIndicesConverter,
-                  TensorToValuesConverter>(context);
+    RewritePatternSet &patterns) {
+  patterns.add<TensorFromPointerConverter, TensorToDimSizeConverter,
+               TensorToPointersConverter, TensorToIndicesConverter,
+               TensorToValuesConverter>(patterns.getContext());
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Sparsification.cpp b/mlir/lib/Dialect/Linalg/Transforms/Sparsification.cpp
index a940bd697f53ce35d87d47cb6ea1349f1ccb0ce1..c0c1970290fcaf50340e5222ab87d18d799a0017 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Sparsification.cpp
@@ -1361,7 +1361,6 @@ private:
 /// Populates the given patterns list with rewriting rules required for
 /// the sparsification of linear algebra operations.
 void linalg::populateSparsificationPatterns(
-    MLIRContext *context, OwningRewritePatternList &patterns,
-    const SparsificationOptions &options) {
-  patterns.insert<GenericOpSparsifier>(context, options);
+    RewritePatternSet &patterns, const SparsificationOptions &options) {
+  patterns.add<GenericOpSparsifier>(patterns.getContext(), options);
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index d638c609aa9efd7d4b7b24a4a27ce7324c86c5c6..0c29bc05cb6651de8e4ce3c7d74c748c0010fa5c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -23,7 +23,6 @@
 #include "mlir/Dialect/StandardOps/EDSC/Intrinsics.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/Transforms/FoldUtils.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
@@ -82,34 +81,6 @@ makeTiledLoopRanges(OpBuilder &b, Location loc, AffineMap map,
         Range{std_constant_index(0), shapeSizes[idx], tileSizes[idx]});
   return std::make_tuple(res, loopIndexToRangeIndex);
 }
-namespace {
-
-// Helper visitor to determine whether an AffineExpr is tiled.
-// This is achieved by traversing every AffineDimExpr with position `pos` and
-// checking whether the corresponding `tileSizes[pos]` is non-zero.
-// This also enforces only positive coefficients occur in multiplications.
-//
-// Example:
-//   `d0 + 2 * d1 + d3` is tiled by [0, 0, 0, 2] but not by [0, 0, 2, 0]
-//
-struct TileCheck : public AffineExprVisitor<TileCheck> {
-  TileCheck(ValueRange tileSizes) : isTiled(false), tileSizes(tileSizes) {}
-
-  void visitDimExpr(AffineDimExpr expr) {
-    isTiled |= !isZero(tileSizes[expr.getPosition()]);
-  }
-  void visitAffineBinaryOpExpr(AffineBinaryOpExpr expr) {
-    visit(expr.getLHS());
-    visit(expr.getRHS());
-    if (expr.getKind() == mlir::AffineExprKind::Mul)
-      assert(expr.getRHS().cast<AffineConstantExpr>().getValue() > 0 &&
-             "nonpositive multiplying coefficient");
-  }
-  bool isTiled;
-  ValueRange tileSizes;
-};
-
-} // namespace
 
 // IndexedGenericOp explicitly uses induction variables in the loop body. The
 // values of the indices that are used in the loop body for any given access of
@@ -201,117 +172,6 @@ static void transformIndexedGenericOpIndices(
   }
 }
 
-static bool isTiled(AffineExpr expr, ValueRange tileSizes) {
-  if (!expr)
-    return false;
-  TileCheck t(tileSizes);
-  t.visit(expr);
-  return t.isTiled;
-}
-
-// Checks whether the `map  varies with respect to a non-zero `tileSize`.
-static bool isTiled(AffineMap map, ValueRange tileSizes) {
-  if (!map)
-    return false;
-  for (unsigned r = 0; r < map.getNumResults(); ++r)
-    if (isTiled(map.getResult(r), tileSizes))
-      return true;
-  return false;
-}
-
-static SmallVector<Value, 4>
-makeTiledShapes(OpBuilder &b, Location loc, LinalgOp linalgOp,
-                ArrayRef<Value> tiledOperands, AffineMap map, ValueRange ivs,
-                ValueRange tileSizes, ValueRange allShapeSizes) {
-  assert(ivs.size() == static_cast<size_t>(llvm::count_if(
-                           llvm::make_range(tileSizes.begin(), tileSizes.end()),
-                           [](Value v) { return !isZero(v); })) &&
-         "expected as many ivs as non-zero sizes");
-
-  using namespace edsc::op;
-
-  auto shapeSizes = applyMapToValues(b, loc, map, allShapeSizes);
-  // Construct (potentially temporary) mins and maxes on which to apply maps
-  // that define tile subshapes.
-  SmallVector<Value, 8> lbs, subShapeSizes;
-  for (unsigned idx = 0, idxIvs = 0, e = tileSizes.size(); idx < e; ++idx) {
-    bool isTiled = !isZero(tileSizes[idx]);
-    lbs.push_back(isTiled ? ivs[idxIvs++] : (Value)std_constant_index(0));
-    // Before composing, we need to make range a closed interval.
-    Value size = isTiled ? tileSizes[idx] : shapeSizes[idx];
-    subShapeSizes.push_back(size - std_constant_index(1));
-  }
-
-  SmallVector<Value, 4> res;
-  res.reserve(tiledOperands.size());
-  for (auto en : llvm::enumerate(tiledOperands)) {
-    Value shapedOp = en.value();
-    ShapedType shapedType = shapedOp.getType().cast<ShapedType>();
-    unsigned rank = shapedType.getRank();
-    AffineMap map = linalgOp.getIndexingMap(en.index());
-    // If the shape is not tiled, we can use it as is.
-    if (!isTiled(map, tileSizes)) {
-      res.push_back(shapedOp);
-      continue;
-    }
-
-    // Construct a new subview / subtensor for the tile.
-    SmallVector<OpFoldResult, 4> offsets, sizes, strides;
-    offsets.reserve(rank);
-    sizes.reserve(rank);
-    strides.reserve(rank);
-    for (unsigned r = 0; r < rank; ++r) {
-      if (!isTiled(map.getSubMap({r}), tileSizes)) {
-        offsets.push_back(b.getIndexAttr(0));
-        sizes.push_back(memref_dim(shapedOp, r).value);
-        strides.push_back(b.getIndexAttr(1));
-        continue;
-      }
-
-      // Tiling creates a new slice at the proper index, the slice step is 1
-      // (i.e. the op does not subsample, stepping occurs in the loop).
-      auto m = map.getSubMap({r});
-      auto offset = applyMapToValues(b, loc, m, lbs).front();
-      offsets.push_back(offset);
-      auto closedIntSize = applyMapToValues(b, loc, m, subShapeSizes).front();
-      // Resulting size needs to be made half open interval again.
-      auto size = closedIntSize + std_constant_index(1);
-
-      // The size of the subview / subtensor should be trimmed to avoid
-      // out-of-bounds accesses, unless we statically know the subshape size
-      // divides the shape size evenly.
-      int64_t shapeSize = shapedType.getDimSize(r);
-      auto sizeCst = size.getDefiningOp<ConstantIndexOp>();
-      if (ShapedType::isDynamic(shapeSize) || !sizeCst ||
-          (shapeSize % sizeCst.getValue()) != 0) {
-        // Compute min(size, dim - offset) to avoid out-of-bounds accesses.
-        auto minMap = AffineMap::get(
-            /*dimCount=*/3, /*symbolCount=*/0,
-            {getAffineDimExpr(/*position=*/0, b.getContext()),
-             getAffineDimExpr(/*position=*/1, b.getContext()) -
-                 getAffineDimExpr(/*position=*/2, b.getContext())},
-            b.getContext());
-        Value d = memref_dim(shapedOp, r);
-        SmallVector<Value, 4> operands{size, d, offset};
-        fullyComposeAffineMapAndOperands(&minMap, &operands);
-        size = affine_min(b.getIndexType(), minMap, operands);
-      }
-
-      sizes.push_back(size);
-      strides.push_back(b.getIndexAttr(1));
-    }
-
-    if (shapedType.isa<MemRefType>())
-      res.push_back(
-          b.create<memref::SubViewOp>(loc, shapedOp, offsets, sizes, strides));
-    else
-      res.push_back(
-          b.create<SubTensorOp>(loc, shapedOp, offsets, sizes, strides));
-  }
-
-  return res;
-}
-
 template <typename LoopTy>
 static Optional<TiledLinalgOp>
 tileLinalgOpImpl(OpBuilder &b, LinalgOp op, ValueRange tileSizes,
@@ -401,9 +261,10 @@ tileLinalgOpImpl(OpBuilder &b, LinalgOp op, ValueRange tileSizes,
         assert(outputBuffers.empty() || iterArgs.empty());
         operands.append(outputBuffers.begin(), outputBuffers.end());
         operands.append(iterArgs.begin(), iterArgs.end());
-        SmallVector<Value, 4> tiledOperands =
-            makeTiledShapes(b, loc, op, operands, shapeSizesToLoopsMap,
-                            interchangedIvs, tileSizes, allShapeSizes);
+        auto sizeBounds =
+            applyMapToValues(b, loc, shapeSizesToLoopsMap, allShapeSizes);
+        SmallVector<Value, 4> tiledOperands = makeTiledShapes(
+            b, loc, op, operands, interchangedIvs, tileSizes, sizeBounds);
         auto nonShapedOperands = op.getAssumedNonShapedOperands();
         tiledOperands.append(nonShapedOperands.begin(),
                              nonShapedOperands.end());
@@ -511,15 +372,15 @@ class CanonicalizationPatternList;
 template <>
 class CanonicalizationPatternList<> {
 public:
-  static void insert(OwningRewritePatternList &patterns, MLIRContext *ctx) {}
+  static void insert(RewritePatternSet &patterns) {}
 };
 
 template <typename OpTy, typename... OpTypes>
 class CanonicalizationPatternList<OpTy, OpTypes...> {
 public:
-  static void insert(OwningRewritePatternList &patterns, MLIRContext *ctx) {
-    OpTy::getCanonicalizationPatterns(patterns, ctx);
-    CanonicalizationPatternList<OpTypes...>::insert(patterns, ctx);
+  static void insert(RewritePatternSet &patterns) {
+    OpTy::getCanonicalizationPatterns(patterns, patterns.getContext());
+    CanonicalizationPatternList<OpTypes...>::insert(patterns);
   }
 };
 
@@ -530,33 +391,35 @@ class RewritePatternList;
 template <>
 class RewritePatternList<> {
 public:
-  static void insert(OwningRewritePatternList &patterns,
-                     const LinalgTilingOptions &options, MLIRContext *ctx) {}
+  static void insert(RewritePatternSet &patterns,
+                     const LinalgTilingOptions &options) {}
 };
 
 template <typename OpTy, typename... OpTypes>
 class RewritePatternList<OpTy, OpTypes...> {
 public:
-  static void insert(OwningRewritePatternList &patterns,
-                     const LinalgTilingOptions &options, MLIRContext *ctx) {
-    patterns.insert<LinalgTilingPattern<OpTy>>(
+  static void insert(RewritePatternSet &patterns,
+                     const LinalgTilingOptions &options) {
+    auto *ctx = patterns.getContext();
+    patterns.add<LinalgTilingPattern<OpTy>>(
         ctx, options,
         LinalgTransformationFilter(ArrayRef<Identifier>{},
                                    Identifier::get("tiled", ctx)));
-    RewritePatternList<OpTypes...>::insert(patterns, options, ctx);
+    RewritePatternList<OpTypes...>::insert(patterns, options);
   }
 };
 } // namespace
 
-OwningRewritePatternList
+RewritePatternSet
 mlir::linalg::getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx) {
-  OwningRewritePatternList patterns;
-  populateLinalgTilingCanonicalizationPatterns(patterns, ctx);
+  RewritePatternSet patterns(ctx);
+  populateLinalgTilingCanonicalizationPatterns(patterns);
   return patterns;
 }
 
 void mlir::linalg::populateLinalgTilingCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx) {
+    RewritePatternSet &patterns) {
+  auto *ctx = patterns.getContext();
   AffineApplyOp::getCanonicalizationPatterns(patterns, ctx);
   AffineForOp::getCanonicalizationPatterns(patterns, ctx);
   AffineMinOp::getCanonicalizationPatterns(patterns, ctx);
@@ -571,17 +434,16 @@ void mlir::linalg::populateLinalgTilingCanonicalizationPatterns(
   CanonicalizationPatternList<
 #define GET_OP_LIST
 #include "mlir/Dialect/Linalg/IR/LinalgStructuredOps.cpp.inc"
-      >::insert(patterns, ctx);
+      >::insert(patterns);
 }
 
 /// Populate the given list with patterns that apply Linalg tiling.
-static void insertTilingPatterns(OwningRewritePatternList &patterns,
-                                 const LinalgTilingOptions &options,
-                                 MLIRContext *ctx) {
+static void insertTilingPatterns(RewritePatternSet &patterns,
+                                 const LinalgTilingOptions &options) {
   RewritePatternList<GenericOp, IndexedGenericOp,
 #define GET_OP_LIST
 #include "mlir/Dialect/Linalg/IR/LinalgStructuredOps.cpp.inc"
-                     >::insert(patterns, options, ctx);
+                     >::insert(patterns, options);
 }
 
 static void applyTilingToLoopPatterns(LinalgTilingLoopType loopType,
@@ -590,8 +452,8 @@ static void applyTilingToLoopPatterns(LinalgTilingLoopType loopType,
   auto options =
       LinalgTilingOptions().setTileSizes(tileSizes).setLoopType(loopType);
   MLIRContext *ctx = funcOp.getContext();
-  OwningRewritePatternList patterns;
-  insertTilingPatterns(patterns, options, ctx);
+  RewritePatternSet patterns(ctx);
+  insertTilingPatterns(patterns, options);
   (void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns));
   (void)applyPatternsAndFoldGreedily(
       funcOp, getLinalgTilingCanonicalizationPatterns(ctx));
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index c2e52c63eabdf3d0b737c1c8bf9d3a8bb72cb698..e7095a9f0b34e1f2e63479b3d7763df68e142ec1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -234,13 +234,13 @@ static LogicalResult rewriteAsPaddedOp(PatternRewriter &rewriter,
 mlir::linalg::LinalgBaseTilingPattern::LinalgBaseTilingPattern(
     StringRef opName, MLIRContext *context, LinalgTilingOptions options,
     LinalgTransformationFilter filter, PatternBenefit benefit)
-    : RewritePattern(opName, {}, benefit, context), filter(filter),
+    : RewritePattern(opName, benefit, context), filter(filter),
       options(options) {}
 
 mlir::linalg::LinalgBaseTilingPattern::LinalgBaseTilingPattern(
-    LinalgTilingOptions options, LinalgTransformationFilter filter,
-    PatternBenefit benefit)
-    : RewritePattern(benefit, MatchAnyOpTypeTag()), filter(filter),
+    MLIRContext *context, LinalgTilingOptions options,
+    LinalgTransformationFilter filter, PatternBenefit benefit)
+    : RewritePattern(MatchAnyOpTypeTag(), benefit, context), filter(filter),
       options(options) {}
 
 LogicalResult mlir::linalg::LinalgBaseTilingPattern::matchAndRewriteBase(
@@ -306,7 +306,7 @@ mlir::linalg::LinalgBaseTileAndFusePattern::LinalgBaseTileAndFusePattern(
     LinalgTilingOptions tilingOptions, LinalgFusionOptions fusionOptions,
     LinalgTransformationFilter filter, LinalgTransformationFilter fusedOpMarker,
     LinalgTransformationFilter originalOpMarker, PatternBenefit benefit)
-    : RewritePattern(opName, {}, benefit, context),
+    : RewritePattern(opName, benefit, context, {}),
       dependenceGraph(dependenceGraph), tilingOptions(tilingOptions),
       fusionOptions(fusionOptions), filter(filter),
       fusedOpMarker(fusedOpMarker), originalOpMarker(originalOpMarker) {}
@@ -401,7 +401,7 @@ mlir::linalg::LinalgBaseInterchangePattern::LinalgBaseInterchangePattern(
     StringRef opName, MLIRContext *context,
     ArrayRef<unsigned> interchangeVector, LinalgTransformationFilter filter,
     PatternBenefit benefit)
-    : RewritePattern(opName, {}, benefit, context), filter(filter),
+    : RewritePattern(opName, benefit, context, {}), filter(filter),
       interchangeVector(interchangeVector.begin(), interchangeVector.end()) {}
 
 LogicalResult mlir::linalg::LinalgBaseInterchangePattern::matchAndRewrite(
@@ -427,7 +427,7 @@ LogicalResult mlir::linalg::LinalgBaseInterchangePattern::matchAndRewrite(
 mlir::linalg::LinalgBasePromotionPattern::LinalgBasePromotionPattern(
     StringRef opName, MLIRContext *context, LinalgPromotionOptions options,
     LinalgTransformationFilter filter, PatternBenefit benefit)
-    : RewritePattern(opName, {}, benefit, context), filter(filter),
+    : RewritePattern(opName, benefit, context, {}), filter(filter),
       options(options) {}
 
 LogicalResult mlir::linalg::LinalgBasePromotionPattern::matchAndRewrite(
@@ -453,13 +453,14 @@ LogicalResult mlir::linalg::LinalgBasePromotionPattern::matchAndRewrite(
 }
 
 mlir::linalg::LinalgBaseVectorizationPattern::LinalgBaseVectorizationPattern(
-    LinalgTransformationFilter filter, PatternBenefit benefit)
-    : RewritePattern(benefit, MatchAnyOpTypeTag()), filter(filter) {}
+    MLIRContext *context, LinalgTransformationFilter filter,
+    PatternBenefit benefit)
+    : RewritePattern(MatchAnyOpTypeTag(), benefit, context), filter(filter) {}
 
 mlir::linalg::LinalgBaseVectorizationPattern::LinalgBaseVectorizationPattern(
     StringRef opName, MLIRContext *context, LinalgTransformationFilter filter,
     PatternBenefit benefit)
-    : RewritePattern(opName, {}, benefit, context), filter(filter) {}
+    : RewritePattern(opName, benefit, context, {}), filter(filter) {}
 
 LogicalResult mlir::linalg::LinalgBaseVectorizationPattern::matchAndRewrite(
     Operation *op, PatternRewriter &rewriter) const {
@@ -479,8 +480,8 @@ LogicalResult mlir::linalg::LinalgBaseVectorizationPattern::matchAndRewrite(
 }
 
 LogicalResult mlir::linalg::applyStagedPatterns(
-    Operation *op, ArrayRef<FrozenRewritePatternList> stage1Patterns,
-    const FrozenRewritePatternList &stage2Patterns,
+    Operation *op, ArrayRef<FrozenRewritePatternSet> stage1Patterns,
+    const FrozenRewritePatternSet &stage2Patterns,
     function_ref<LogicalResult(Operation *)> stage3Lambda) {
   unsigned iteration = 0;
   (void)iteration;
@@ -536,8 +537,10 @@ static AffineExpr substituteLoopInExpr(AffineExpr expr, AffineExpr dimExpr,
 
 /// Traverse the `dims` and substitute known min or max expressions in place of
 /// induction variables in `exprs`.
-static AffineMap substitute(AffineMap map, SmallVectorImpl<Value> &dims,
-                            SmallVectorImpl<Value> &symbols) {
+static AffineMap substitute(
+    AffineMap map, SmallVectorImpl<Value> &dims,
+    SmallVectorImpl<Value> &symbols,
+    llvm::function_ref<bool(Operation *)> substituteOperation = nullptr) {
   auto exprs = llvm::to_vector<4>(map.getResults());
   for (AffineExpr &expr : exprs) {
     bool substituted = true;
@@ -549,17 +552,19 @@ static AffineMap substitute(AffineMap map, SmallVectorImpl<Value> &dims,
         LLVM_DEBUG(DBGS() << "Subst: " << dim << " @ " << dimExpr << "\n");
         AffineExpr substitutedExpr;
         if (auto forOp = scf::getForInductionVarOwner(dim))
-          substitutedExpr = substituteLoopInExpr(
-              expr, dimExpr, forOp.lowerBound(), forOp.upperBound(),
-              forOp.step(), dims, symbols);
+          if (!substituteOperation || substituteOperation(forOp))
+            substitutedExpr = substituteLoopInExpr(
+                expr, dimExpr, forOp.lowerBound(), forOp.upperBound(),
+                forOp.step(), dims, symbols);
 
         if (auto parallelForOp = scf::getParallelForInductionVarOwner(dim))
-          for (unsigned idx = 0, e = parallelForOp.getNumLoops(); idx < e;
-               ++idx)
-            substitutedExpr = substituteLoopInExpr(
-                expr, dimExpr, parallelForOp.lowerBound()[idx],
-                parallelForOp.upperBound()[idx], parallelForOp.step()[idx],
-                dims, symbols);
+          if (!substituteOperation || substituteOperation(parallelForOp))
+            for (unsigned idx = 0, e = parallelForOp.getNumLoops(); idx < e;
+                 ++idx)
+              substitutedExpr = substituteLoopInExpr(
+                  expr, dimExpr, parallelForOp.lowerBound()[idx],
+                  parallelForOp.upperBound()[idx], parallelForOp.step()[idx],
+                  dims, symbols);
 
         if (!substitutedExpr)
           continue;
@@ -577,7 +582,12 @@ static AffineMap substitute(AffineMap map, SmallVectorImpl<Value> &dims,
     auto map = AffineMap::get(dims.size(), symbols.size(), exprs,
                               exprs.front().getContext());
 
-    LLVM_DEBUG(DBGS() << "Map to simplify: " << map << "\n");
+    LLVM_DEBUG({
+      DBGS() << "Map to simplify: " << map << "\n";
+      DBGS() << "Operands:\n";
+      for (Value v : operands)
+        DBGS() << v << "\n";
+    });
 
     // Pull in affine.apply operations and compose them fully into the
     // result.
@@ -596,14 +606,38 @@ static AffineMap substitute(AffineMap map, SmallVectorImpl<Value> &dims,
   return AffineMap::get(dims.size(), symbols.size(), exprs, map.getContext());
 }
 
+/// Traverse the dims of the AffineMap of `affineMinOp` and substitute scf loop
+/// induction variables by new expressions involving the lower or upper bound:
+///   - If the AffineDimExpr mapped to a loop IV has a positive sign, it is
+///     replaced by the loop upper bound.
+///   - If the AffineDimExpr mapped to a loop IV has a negative sign, it is
+///     replaced by the loop lower bound.
+/// All loop induction variables are iteratively replaced, unless a
+/// `substituteOperation` hook is passed to more finely determine which
+/// operations are substituted.
+/// This is used as an intermediate step in computing bounding boxes and
+/// canonicalize AffineMinOps. All dim and symbol operands are assumed to have
+/// positive values (positive orthant assumptions).
+/// Return a new AffineMap, dims and symbols that have been canonicalized and
+/// simplified.
+AffineMapAndOperands mlir::linalg::substituteMin(
+    AffineMinOp affineMinOp,
+    llvm::function_ref<bool(Operation *)> substituteOperation) {
+  AffineMapAndOperands res{affineMinOp.getAffineMap(),
+                           SmallVector<Value>(affineMinOp.getDimOperands()),
+                           SmallVector<Value>(affineMinOp.getSymbolOperands())};
+  res.map = substitute(affineMinOp.getAffineMap(), res.dims, res.symbols,
+                       substituteOperation);
+  return res;
+}
+
 LogicalResult AffineMinSCFCanonicalizationPattern::matchAndRewrite(
     AffineMinOp minOp, PatternRewriter &rewriter) const {
   LLVM_DEBUG(DBGS() << "Canonicalize AffineMinSCF: " << *minOp.getOperation()
                     << "\n");
 
-  SmallVector<Value, 4> dims(minOp.getDimOperands()),
-      symbols(minOp.getSymbolOperands());
-  AffineMap map = substitute(minOp.getAffineMap(), dims, symbols);
+  auto affineMapAndOperands = substituteMin(minOp);
+  AffineMap map = affineMapAndOperands.map;
 
   LLVM_DEBUG(DBGS() << "Resulting map: " << map << "\n");
 
@@ -638,8 +672,8 @@ LogicalResult AffineMinSCFCanonicalizationPattern::matchAndRewrite(
       rewriter.replaceOpWithNewOp<ConstantIndexOp>(minOp, cst.getValue());
     } else {
       auto resultMap = AffineMap::get(0, map.getNumSymbols(), {e}, ctx);
-      SmallVector<Value, 4> resultOperands = dims;
-      resultOperands.append(symbols.begin(), symbols.end());
+      SmallVector<Value> resultOperands = affineMapAndOperands.dims;
+      llvm::append_range(resultOperands, affineMapAndOperands.symbols);
       canonicalizeMapAndOperands(&resultMap, &resultOperands);
       resultMap = simplifyAffineMap(resultMap);
       rewriter.replaceOpWithNewOp<AffineApplyOp>(minOp, resultMap,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 880e7f38572431230d77b333d8f0e336a7352e40..10562d68a9e0a5de515559c7b1167abc76e56bcb 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -87,11 +87,14 @@ static VectorType extractVectorTypeFromShapedValue(Value v) {
 /// Build a vector.transfer_read from `source` at indices set to all `0`.
 /// If source has rank zero, build an memref.load.
 /// Return the produced value.
-static Value buildVectorRead(OpBuilder &builder, Value source) {
+static Value buildVectorRead(OpBuilder &builder, Value source,
+                             VectorType vectorType, AffineMap map) {
   edsc::ScopedContext scope(builder);
   auto shapedType = source.getType().cast<ShapedType>();
-  if (VectorType vectorType = extractVectorTypeFromShapedValue(source)) {
+  if (vectorType) {
     SmallVector<Value> indices(shapedType.getRank(), std_constant_index(0));
+    if (map)
+      return vector_transfer_read(vectorType, source, indices, map);
     return vector_transfer_read(vectorType, source, indices);
   }
   return memref_load(source);
@@ -238,9 +241,54 @@ vectorizeOneOp(OpBuilder &builder, Operation *op,
                              builder.createOperation(state)};
 }
 
+/// Detect whether `r` has only ConstantOp, ElementwiseMappable and YieldOp.
+static bool hasOnlyScalarElementwiseOp(Region &r) {
+  if (!llvm::hasSingleElement(r))
+    return false;
+  for (Operation &op : r.front()) {
+    if (!(isa<ConstantOp, linalg::YieldOp>(op) ||
+          OpTrait::hasElementwiseMappableTraits(&op)) ||
+        llvm::any_of(op.getResultTypes(),
+                     [](Type type) { return !type.isIntOrIndexOrFloat(); }))
+      return false;
+  }
+  return true;
+}
+
+// Return true if the op is an element-wise linalg op.
+static bool isElementwise(Operation *op) {
+  auto linalgOp = dyn_cast<linalg::LinalgOp>(op);
+  if (!linalgOp)
+    return false;
+  if (linalgOp.getNumLoops() != linalgOp.getNumParallelLoops())
+    return false;
+  // TODO: relax the restrictions on indexing map.
+  for (unsigned i = 0, e = linalgOp.getNumOutputs(); i < e; i++) {
+    if (!linalgOp.getOutputIndexingMap(i).isIdentity())
+      return false;
+  }
+  if (linalgOp->getNumRegions() != 1)
+    return false;
+  return hasOnlyScalarElementwiseOp(linalgOp->getRegion(0));
+}
+
+// Calculate the map to apply to transfer_read to convert the input shape into
+// the output shape.
+static AffineMap getTransferReadMap(LinalgOp linalgOp, unsigned argIndex) {
+  AffineMap linalgMap = linalgOp.getIndexingMap(argIndex);
+  MLIRContext *context = linalgMap.getContext();
+  AffineExpr zero = mlir::getAffineConstantExpr(0, context);
+  SmallVector<AffineExpr, 4> exprs(linalgMap.getNumInputs(), zero);
+  for (unsigned i : llvm::seq(unsigned(0), linalgMap.getNumResults())) {
+    exprs[linalgMap.getDimPosition(i)] = getAffineDimExpr(i, context);
+  }
+  return AffineMap::get(linalgMap.getNumResults(), /*symbolCount=*/0, exprs,
+                        context);
+}
+
 /// Generic vectorization function that rewrites the body of a `linalgOp` into
 /// vector form. Generic vectorization proceeds as follows:
-///   1. The region for the linalg op is created if necessary.
+///   1. Verify the `linalgOp` has one non-empty region.
 ///   2. Values defined above the region are mapped to themselves and will be
 ///   broadcasted on a per-need basis by their consumers.
 ///   3. Each region argument is vectorized into a vector.transfer_read (or 0-d
@@ -251,38 +299,35 @@ vectorizeOneOp(OpBuilder &builder, Operation *op,
 LogicalResult vectorizeAsLinalgGeneric(
     OpBuilder &builder, LinalgOp linalgOp, SmallVectorImpl<Value> &newResults,
     ArrayRef<CustomVectorizationHook> customVectorizationHooks = {}) {
-  // 1. Certain Linalg ops do not have a region but only a region builder.
-  // If so, build the region so we can vectorize.
-  std::unique_ptr<Region> owningRegion;
-  Region *region;
-  if (linalgOp->getNumRegions() > 0) {
-    region = &linalgOp->getRegion(0);
-  } else {
-    // RAII avoid remaining in block.
-    OpBuilder::InsertionGuard g(builder);
-    owningRegion = std::make_unique<Region>();
-    region = owningRegion.get();
-    Block *block = builder.createBlock(region);
-    auto elementTypes = llvm::to_vector<4>(
-        llvm::map_range(linalgOp.getShapedOperandTypes(),
-                        [](ShapedType t) { return t.getElementType(); }));
-    block->addArguments(elementTypes);
-    linalgOp.getRegionBuilder()(*block, /*captures=*/{});
-  }
-  Block *block = &region->front();
+  // 1. Fail to vectorize if the operation does not have one non-empty region.
+  if (linalgOp->getNumRegions() != 1 || linalgOp->getRegion(0).empty())
+    return failure();
+  auto &block = linalgOp->getRegion(0).front();
 
   BlockAndValueMapping bvm;
   // 2. Values defined above the region can only be broadcast for now. Make them
   // map to themselves.
   llvm::SetVector<Value> valuesSet;
-  mlir::getUsedValuesDefinedAbove(*region, valuesSet);
+  mlir::getUsedValuesDefinedAbove(linalgOp->getRegion(0), valuesSet);
   bvm.map(valuesSet.getArrayRef(), valuesSet.getArrayRef());
 
   // 3. Turn all BBArgs into vector.transfer_read / load.
   SmallVector<AffineMap> indexings;
-  for (auto bbarg : block->getArguments()) {
+  for (auto bbarg : block.getArguments()) {
     Value vectorArg = linalgOp.getShapedOperand(bbarg.getArgNumber());
-    Value vectorRead = buildVectorRead(builder, vectorArg);
+    AffineMap map;
+    VectorType vectorType = extractVectorTypeFromShapedValue(vectorArg);
+    if (isElementwise(linalgOp) &&
+        !linalgOp.getIndexingMap(bbarg.getArgNumber()).isMinorIdentity()) {
+      // Currently assume we don't support output permutations.
+      assert(linalgOp.getNumOutputs() > 0 &&
+             linalgOp.getOutputIndexingMap(0).isIdentity());
+      ArrayRef<int64_t> outputShape =
+          linalgOp.getOutputShapedType(0).getShape();
+      vectorType = VectorType::get(outputShape, vectorType.getElementType());
+      map = getTransferReadMap(linalgOp, bbarg.getArgNumber());
+    }
+    Value vectorRead = buildVectorRead(builder, vectorArg, vectorType, map);
     LLVM_DEBUG(dbgs() << "\n[" DEBUG_TYPE "]: new vectorized bbarg("
                       << bbarg.getArgNumber() << "): " << vectorRead);
     bvm.map(bbarg, vectorRead);
@@ -300,7 +345,7 @@ LogicalResult vectorizeAsLinalgGeneric(
   hooks.push_back(vectorizeYield);
 
   // 5. Iteratively call `vectorizeOneOp` to each op in the slice.
-  for (Operation &op : block->getOperations()) {
+  for (Operation &op : block.getOperations()) {
     VectorizationResult result = vectorizeOneOp(builder, &op, bvm, hooks);
     if (result.status == VectorizationStatus::Failure) {
       LLVM_DEBUG(dbgs() << "\n[" DEBUG_TYPE "]: failed to vectorize: " << op);
@@ -316,44 +361,6 @@ LogicalResult vectorizeAsLinalgGeneric(
   return success();
 }
 
-/// Detect whether `r` has only ConstantOp, ElementwiseMappable and YieldOp.
-static bool hasOnlyScalarElementwiseOp(Region &r) {
-  if (!llvm::hasSingleElement(r))
-    return false;
-  for (Operation &op : r.front()) {
-    if (!(isa<ConstantOp, linalg::YieldOp>(op) ||
-          OpTrait::hasElementwiseMappableTraits(&op)) ||
-        llvm::any_of(op.getResultTypes(),
-                     [](Type type) { return !type.isIntOrIndexOrFloat(); }))
-      return false;
-  }
-  return true;
-}
-
-// Return true if the op is an element-wise linalg op.
-static bool isElementwise(Operation *op) {
-  auto linalgOp = dyn_cast<linalg::LinalgOp>(op);
-  if (!linalgOp)
-    return false;
-  if (linalgOp.getNumLoops() != linalgOp.getNumParallelLoops())
-    return false;
-  // TODO: relax the restrictions on indexing map.
-  for (unsigned i = 0, e = linalgOp.getNumOutputs(); i < e; i++) {
-    if (!linalgOp.getOutputIndexingMap(i).isIdentity())
-      return false;
-  }
-  // Currently bound the input indexing map to minor identity as other
-  // permutations might require adding transpose ops to convert the vector read
-  // to the right shape.
-  for (unsigned i = 0, e = linalgOp.getNumInputs(); i < e; i++) {
-    if (!linalgOp.getInputIndexingMap(i).isMinorIdentity())
-      return false;
-  }
-  if (linalgOp->getNumRegions() != 1)
-    return false;
-  return hasOnlyScalarElementwiseOp(linalgOp->getRegion(0));
-}
-
 static LogicalResult vectorizeContraction(OpBuilder &builder, LinalgOp linalgOp,
                                           SmallVectorImpl<Value> &newResults) {
   assert(isaContractionOpInterface(linalgOp) &&
@@ -554,23 +561,21 @@ using ConvOpConst = ConvOpVectorization<ConvWOp, 1>;
 /// Inserts tiling, promotion and vectorization pattern for ConvOp
 /// conversion into corresponding pattern lists.
 template <typename ConvOp, unsigned N>
-static void
-populateVectorizationPatterns(OwningRewritePatternList &tilingPatterns,
-                              OwningRewritePatternList &promotionPatterns,
-                              OwningRewritePatternList &vectorizationPatterns,
-                              ArrayRef<int64_t> tileSizes,
-                              MLIRContext *context) {
+static void populateVectorizationPatterns(
+    RewritePatternSet &tilingPatterns, RewritePatternSet &promotionPatterns,
+    RewritePatternSet &vectorizationPatterns, ArrayRef<int64_t> tileSizes) {
+  auto *context = tilingPatterns.getContext();
   if (tileSizes.size() < N)
     return;
 
   constexpr static StringRef kTiledMarker = "TILED";
   constexpr static StringRef kPromotedMarker = "PROMOTED";
-  tilingPatterns.insert<LinalgTilingPattern<ConvOp>>(
+  tilingPatterns.add<LinalgTilingPattern<ConvOp>>(
       context, LinalgTilingOptions().setTileSizes(tileSizes),
       LinalgTransformationFilter(ArrayRef<Identifier>{},
                                  Identifier::get(kTiledMarker, context)));
 
-  promotionPatterns.insert<LinalgPromotionPattern<ConvOp>>(
+  promotionPatterns.add<LinalgPromotionPattern<ConvOp>>(
       context, LinalgPromotionOptions().setUseFullTileBuffersByDefault(true),
       LinalgTransformationFilter(Identifier::get(kTiledMarker, context),
                                  Identifier::get(kPromotedMarker, context)));
@@ -580,51 +585,53 @@ populateVectorizationPatterns(OwningRewritePatternList &tilingPatterns,
   std::transform(tileSizes.begin() + offset, tileSizes.end(), mask.begin(),
                  [](int64_t i) -> bool { return i > 1; });
 
-  vectorizationPatterns.insert<ConvOpVectorization<ConvOp, N>>(context, mask);
+  vectorizationPatterns.add<ConvOpVectorization<ConvOp, N>>(context, mask);
 }
 
 void mlir::linalg::populateConvVectorizationPatterns(
-    MLIRContext *context, SmallVectorImpl<OwningRewritePatternList> &patterns,
+    MLIRContext *context, SmallVectorImpl<RewritePatternSet> &patterns,
     ArrayRef<int64_t> tileSizes) {
-  OwningRewritePatternList tiling, promotion, vectorization;
+  RewritePatternSet tiling(context);
+  RewritePatternSet promotion(context);
+  RewritePatternSet vectorization(context);
   populateVectorizationPatterns<ConvWOp, 1>(tiling, promotion, vectorization,
-                                            tileSizes, context);
+                                            tileSizes);
 
   populateVectorizationPatterns<ConvNWCOp, 3>(tiling, promotion, vectorization,
-                                              tileSizes, context);
+                                              tileSizes);
   populateVectorizationPatterns<ConvInputNWCFilterWCFOp, 3>(
-      tiling, promotion, vectorization, tileSizes, context);
+      tiling, promotion, vectorization, tileSizes);
 
   populateVectorizationPatterns<ConvNCWOp, 3>(tiling, promotion, vectorization,
-                                              tileSizes, context);
+                                              tileSizes);
   populateVectorizationPatterns<ConvInputNCWFilterWCFOp, 3>(
-      tiling, promotion, vectorization, tileSizes, context);
+      tiling, promotion, vectorization, tileSizes);
 
   populateVectorizationPatterns<ConvHWOp, 2>(tiling, promotion, vectorization,
-                                             tileSizes, context);
+                                             tileSizes);
 
   populateVectorizationPatterns<ConvNHWCOp, 4>(tiling, promotion, vectorization,
-                                               tileSizes, context);
+                                               tileSizes);
   populateVectorizationPatterns<ConvInputNHWCFilterHWCFOp, 4>(
-      tiling, promotion, vectorization, tileSizes, context);
+      tiling, promotion, vectorization, tileSizes);
 
   populateVectorizationPatterns<ConvNCHWOp, 4>(tiling, promotion, vectorization,
-                                               tileSizes, context);
+                                               tileSizes);
   populateVectorizationPatterns<ConvInputNCHWFilterHWCFOp, 4>(
-      tiling, promotion, vectorization, tileSizes, context);
+      tiling, promotion, vectorization, tileSizes);
 
   populateVectorizationPatterns<ConvDHWOp, 3>(tiling, promotion, vectorization,
-                                              tileSizes, context);
+                                              tileSizes);
 
-  populateVectorizationPatterns<ConvNDHWCOp, 5>(
-      tiling, promotion, vectorization, tileSizes, context);
+  populateVectorizationPatterns<ConvNDHWCOp, 5>(tiling, promotion,
+                                                vectorization, tileSizes);
   populateVectorizationPatterns<ConvInputNDHWCFilterDHWCFOp, 5>(
-      tiling, promotion, vectorization, tileSizes, context);
+      tiling, promotion, vectorization, tileSizes);
 
-  populateVectorizationPatterns<ConvNCDHWOp, 5>(
-      tiling, promotion, vectorization, tileSizes, context);
+  populateVectorizationPatterns<ConvNCDHWOp, 5>(tiling, promotion,
+                                                vectorization, tileSizes);
   populateVectorizationPatterns<ConvInputNCDHWFilterDHWCFOp, 5>(
-      tiling, promotion, vectorization, tileSizes, context);
+      tiling, promotion, vectorization, tileSizes);
 
   patterns.push_back(std::move(tiling));
   patterns.push_back(std::move(promotion));
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index 8d91bd74712bfbec861bc01a4749d5b492754959..8c8b0cf1f7bf519834b496d0d1c971466bee7766 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -18,18 +18,78 @@
 #include "mlir/Dialect/Linalg/IR/LinalgTypes.h"
 #include "mlir/Dialect/SCF/EDSC/Builders.h"
 #include "mlir/Dialect/SCF/SCF.h"
+#include "mlir/Dialect/StandardOps/EDSC/Intrinsics.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/AffineExpr.h"
+#include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/LoopUtils.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "linalg-utils"
 
 using namespace mlir;
+using namespace mlir::edsc;
+using namespace mlir::edsc::intrinsics;
 using namespace mlir::linalg;
 using namespace mlir::scf;
 
+static bool isZero(Value v) {
+  if (auto cst = v.getDefiningOp<ConstantIndexOp>())
+    return cst.getValue() == 0;
+  return false;
+}
+
+namespace {
+
+// Helper visitor to determine whether an AffineExpr is tiled.
+// This is achieved by traversing every AffineDimExpr with position `pos` and
+// checking whether the corresponding `tileSizes[pos]` is non-zero.
+// This also enforces only positive coefficients occur in multiplications.
+//
+// Example:
+//   `d0 + 2 * d1 + d3` is tiled by [0, 0, 0, 2] but not by [0, 0, 2, 0]
+//
+struct TileCheck : public AffineExprVisitor<TileCheck> {
+  TileCheck(ValueRange tileSizes) : isTiled(false), tileSizes(tileSizes) {}
+
+  void visitDimExpr(AffineDimExpr expr) {
+    isTiled |= !isZero(tileSizes[expr.getPosition()]);
+  }
+  void visitAffineBinaryOpExpr(AffineBinaryOpExpr expr) {
+    visit(expr.getLHS());
+    visit(expr.getRHS());
+    if (expr.getKind() == mlir::AffineExprKind::Mul)
+      assert(expr.getRHS().cast<AffineConstantExpr>().getValue() > 0 &&
+             "nonpositive multiplying coefficient");
+  }
+  bool isTiled;
+  ValueRange tileSizes;
+};
+
+} // namespace
+
+static bool isTiled(AffineExpr expr, ValueRange tileSizes) {
+  if (!expr)
+    return false;
+  TileCheck t(tileSizes);
+  t.visit(expr);
+  return t.isTiled;
+}
+
+// Checks whether the `map  varies with respect to a non-zero `tileSize`.
+static bool isTiled(AffineMap map, ValueRange tileSizes) {
+  if (!map)
+    return false;
+  for (unsigned r = 0; r < map.getNumResults(); ++r)
+    if (isTiled(map.getResult(r), tileSizes))
+      return true;
+  return false;
+}
+
 Optional<RegionMatcher::BinaryOpKind>
 RegionMatcher::matchAsScalarBinaryOp(GenericOp op) {
   auto &region = op.region();
@@ -374,5 +434,117 @@ void GenerateLoopNest<scf::ParallelOp>::doit(
   assert(ivs.size() == iteratorTypes.size() && "did not generate enough loops");
 }
 
+SmallVector<Value, 4> makeTiledShapes(OpBuilder &builder, Location loc,
+                                      LinalgOp linalgOp,
+                                      ArrayRef<Value> tiledOperands,
+                                      ValueRange ivs, ValueRange tileSizes,
+                                      ArrayRef<Value> sizeBounds) {
+  assert(ivs.size() == static_cast<size_t>(llvm::count_if(
+                           llvm::make_range(tileSizes.begin(), tileSizes.end()),
+                           [](Value v) { return !isZero(v); })) &&
+         "expected as many ivs as non-zero sizes");
+
+  using namespace edsc::op;
+
+  // Construct (potentially temporary) mins and maxes on which to apply maps
+  // that define tile subshapes.
+  SmallVector<Value, 8> lbs, subShapeSizes;
+  for (unsigned idx = 0, idxIvs = 0, e = tileSizes.size(); idx < e; ++idx) {
+    LLVM_DEBUG(llvm::dbgs() << "makeTiledShapes: for loop#" << idx << "\n");
+    bool isTiled = !isZero(tileSizes[idx]);
+    lbs.push_back(isTiled ? ivs[idxIvs++] : (Value)std_constant_index(0));
+    // Before composing, we need to make range a closed interval.
+    Value size = isTiled ? tileSizes[idx] : sizeBounds[idx];
+    subShapeSizes.push_back(size - std_constant_index(1));
+    LLVM_DEBUG(llvm::dbgs() << "lb: " << lbs.back() << "\n");
+    LLVM_DEBUG(llvm::dbgs() << "size: " << subShapeSizes.back() << "\n");
+  }
+
+  MLIRContext *context = builder.getContext();
+  SmallVector<Value, 4> tiledShapes;
+  tiledShapes.reserve(tiledOperands.size());
+  for (auto en : llvm::enumerate(tiledOperands)) {
+    Value shapedOp = en.value();
+    LLVM_DEBUG(llvm::dbgs() << "makeTiledShapes: for operand " << shapedOp);
+    ShapedType shapedType = shapedOp.getType().cast<ShapedType>();
+    unsigned rank = shapedType.getRank();
+    AffineMap map = linalgOp.getIndexingMap(en.index());
+    // If the shape is not tiled, we can use it as is.
+    if (!isTiled(map, tileSizes)) {
+      tiledShapes.push_back(shapedOp);
+      LLVM_DEBUG(llvm::dbgs()
+                 << ": not tiled: use shape: " << shapedType << "\n");
+      continue;
+    }
+    LLVM_DEBUG(llvm::dbgs() << ": tiled: figure out subshape...\n");
+
+    // Construct a new subview / subtensor for the tile.
+    SmallVector<OpFoldResult, 4> offsets, sizes, strides;
+    offsets.reserve(rank);
+    sizes.reserve(rank);
+    strides.reserve(rank);
+    for (unsigned r = 0; r < rank; ++r) {
+      LLVM_DEBUG(llvm::dbgs() << "makeTiledShapes: for dim#" << r);
+      if (!isTiled(map.getSubMap({r}), tileSizes)) {
+        offsets.push_back(builder.getIndexAttr(0));
+        Value dim = memref_dim(shapedOp, r).value;
+        sizes.push_back(dim);
+        strides.push_back(builder.getIndexAttr(1));
+        LLVM_DEBUG(llvm::dbgs() << ": not tiled: use size: " << dim << "\n");
+        continue;
+      }
+      LLVM_DEBUG(llvm::dbgs() << ": tiled: figure out subsize...\n");
+
+      // Tiling creates a new slice at the proper index, the slice step is 1
+      // (i.e. the op does not subsample, stepping occurs in the loop).
+      auto m = map.getSubMap({r});
+      LLVM_DEBUG(llvm::dbgs() << "makeTiledShapes: submap: " << map << "\n");
+      auto offset = applyMapToValues(builder, loc, m, lbs).front();
+      offsets.push_back(offset);
+      auto closedIntSize =
+          applyMapToValues(builder, loc, m, subShapeSizes).front();
+      // Resulting size needs to be made half open interval again.
+      auto size = closedIntSize + std_constant_index(1);
+      LLVM_DEBUG(llvm::dbgs() << "makeTiledShapes: raw size: " << size << "\n");
+
+      // The size of the subview / subtensor should be trimmed to avoid
+      // out-of-bounds accesses, unless we statically know the subshape size
+      // divides the shape size evenly.
+      int64_t shapeSize = shapedType.getDimSize(r);
+      auto sizeCst = size.getDefiningOp<ConstantIndexOp>();
+      if (ShapedType::isDynamic(shapeSize) || !sizeCst ||
+          (shapeSize % sizeCst.getValue()) != 0) {
+        LLVM_DEBUG(llvm::dbgs() << "makeTiledShapes: shapeSize=" << shapeSize
+                                << ", size: " << size
+                                << ": make sure in bound with affine.min\n");
+        AffineExpr dim0, dim1, dim2;
+        bindDims(context, dim0, dim1, dim2);
+        // Compute min(size, dim - offset) to avoid out-of-bounds accesses.
+        auto minMap = AffineMap::get(
+            /*dimCount=*/3, /*symbolCount=*/0, {dim0, dim1 - dim2}, context);
+        Value d = memref_dim(shapedOp, r);
+        SmallVector<Value, 4> operands{size, d, offset};
+        fullyComposeAffineMapAndOperands(&minMap, &operands);
+        size = affine_min(builder.getIndexType(), minMap, operands);
+      }
+
+      sizes.push_back(size);
+      LLVM_DEBUG(llvm::dbgs()
+                 << "makeTiledShapes: new offset: " << offset << "\n");
+      LLVM_DEBUG(llvm::dbgs() << "makeTiledShapes: new size: " << size << "\n");
+      strides.push_back(builder.getIndexAttr(1));
+    }
+
+    if (shapedType.isa<MemRefType>())
+      tiledShapes.push_back(builder.create<memref::SubViewOp>(
+          loc, shapedOp, offsets, sizes, strides));
+    else
+      tiledShapes.push_back(
+          builder.create<SubTensorOp>(loc, shapedOp, offsets, sizes, strides));
+  }
+
+  return tiledShapes;
+}
+
 } // namespace linalg
 } // namespace mlir
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandTanh.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandTanh.cpp
index 06d51582a8ce2c6ff06ac2e92958d49a8aebf502..994501cfa0e546177ddf623b515c7f65ea14b17f 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandTanh.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandTanh.cpp
@@ -15,52 +15,42 @@
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/Transforms/DialectConversion.h"
-
 using namespace mlir;
 
-namespace {
 /// Expands tanh op into
 ///   1) 1-exp^{-2x} / 1+exp^{-2x}, if x => 0
 ///   2) exp^{2x}-1 / exp^{2x}+1  , if x < 0
-struct TanhOpConverter : public OpRewritePattern<math::TanhOp> {
-public:
-  using OpRewritePattern::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(math::TanhOp op,
-                                PatternRewriter &rewriter) const final {
-    auto floatType = op.operand().getType();
-    Location loc = op.getLoc();
-    auto floatOne = rewriter.getFloatAttr(floatType, 1.0);
-    auto floatTwo = rewriter.getFloatAttr(floatType, 2.0);
-    Value one = rewriter.create<ConstantOp>(loc, floatOne);
-    Value two = rewriter.create<ConstantOp>(loc, floatTwo);
-    Value doubledX = rewriter.create<MulFOp>(loc, op.operand(), two);
-
-    // Case 1: tanh(x) = 1-exp^{-2x} / 1+exp^{-2x}
-    Value negDoubledX = rewriter.create<NegFOp>(loc, doubledX);
-    Value exp2x = rewriter.create<math::ExpOp>(loc, negDoubledX);
-    Value dividend = rewriter.create<SubFOp>(loc, one, exp2x);
-    Value divisor = rewriter.create<AddFOp>(loc, one, exp2x);
-    Value positiveRes = rewriter.create<DivFOp>(loc, dividend, divisor);
-
-    // Case 2: tanh(x) = exp^{2x}-1 / exp^{2x}+1
-    exp2x = rewriter.create<math::ExpOp>(loc, doubledX);
-    dividend = rewriter.create<SubFOp>(loc, exp2x, one);
-    divisor = rewriter.create<AddFOp>(loc, exp2x, one);
-    Value negativeRes = rewriter.create<DivFOp>(loc, dividend, divisor);
-
-    // tanh(x) = x >= 0 ? positiveRes : negativeRes
-    auto floatZero = rewriter.getFloatAttr(floatType, 0.0);
-    Value zero = rewriter.create<ConstantOp>(loc, floatZero);
-    Value cmpRes =
-        rewriter.create<CmpFOp>(loc, CmpFPredicate::OGE, op.operand(), zero);
-    rewriter.replaceOpWithNewOp<SelectOp>(op, cmpRes, positiveRes, negativeRes);
-    return success();
-  }
-};
-} // namespace
+static LogicalResult convertTanhOp(math::TanhOp op, PatternRewriter &rewriter) {
+  auto floatType = op.operand().getType();
+  Location loc = op.getLoc();
+  auto floatOne = rewriter.getFloatAttr(floatType, 1.0);
+  auto floatTwo = rewriter.getFloatAttr(floatType, 2.0);
+  Value one = rewriter.create<ConstantOp>(loc, floatOne);
+  Value two = rewriter.create<ConstantOp>(loc, floatTwo);
+  Value doubledX = rewriter.create<MulFOp>(loc, op.operand(), two);
+
+  // Case 1: tanh(x) = 1-exp^{-2x} / 1+exp^{-2x}
+  Value negDoubledX = rewriter.create<NegFOp>(loc, doubledX);
+  Value exp2x = rewriter.create<math::ExpOp>(loc, negDoubledX);
+  Value dividend = rewriter.create<SubFOp>(loc, one, exp2x);
+  Value divisor = rewriter.create<AddFOp>(loc, one, exp2x);
+  Value positiveRes = rewriter.create<DivFOp>(loc, dividend, divisor);
+
+  // Case 2: tanh(x) = exp^{2x}-1 / exp^{2x}+1
+  exp2x = rewriter.create<math::ExpOp>(loc, doubledX);
+  dividend = rewriter.create<SubFOp>(loc, exp2x, one);
+  divisor = rewriter.create<AddFOp>(loc, exp2x, one);
+  Value negativeRes = rewriter.create<DivFOp>(loc, dividend, divisor);
+
+  // tanh(x) = x >= 0 ? positiveRes : negativeRes
+  auto floatZero = rewriter.getFloatAttr(floatType, 0.0);
+  Value zero = rewriter.create<ConstantOp>(loc, floatZero);
+  Value cmpRes =
+      rewriter.create<CmpFOp>(loc, CmpFPredicate::OGE, op.operand(), zero);
+  rewriter.replaceOpWithNewOp<SelectOp>(op, cmpRes, positiveRes, negativeRes);
+  return success();
+}
 
-void mlir::populateExpandTanhPattern(OwningRewritePatternList &patterns,
-                                     MLIRContext *ctx) {
-  patterns.insert<TanhOpConverter>(ctx);
+void mlir::populateExpandTanhPattern(RewritePatternSet &patterns) {
+  patterns.add(convertTanhOp);
 }
diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index f13e48e6ba27303d678e6c2f20f61cf212ffc12f..d9c78f527c8a2075b1b24ccaa44f60b4e47dd345 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -10,6 +10,7 @@
 // that do not rely on any of the library functions.
 //
 //===----------------------------------------------------------------------===//
+
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/Math/IR/Math.h"
@@ -17,9 +18,10 @@
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
+#include "mlir/Transforms/Bufferize.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include <limits.h>
+#include <climits>
 
 using namespace mlir;
 using namespace mlir::vector;
@@ -530,7 +532,7 @@ ExpApproximation::matchAndRewrite(math::ExpOp op,
 //----------------------------------------------------------------------------//
 
 void mlir::populateMathPolynomialApproximationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx) {
-  patterns.insert<TanhApproximation, LogApproximation, Log2Approximation,
-                  ExpApproximation>(ctx);
+    RewritePatternSet &patterns) {
+  patterns.add<TanhApproximation, LogApproximation, Log2Approximation,
+               ExpApproximation>(patterns.getContext());
 }
diff --git a/mlir/lib/Dialect/MemRef/CMakeLists.txt b/mlir/lib/Dialect/MemRef/CMakeLists.txt
index f33061b2d87cffc48b072763eb55d349e9070aba..737094373b0e1e4306c115f0c5ecc666c47bbc6e 100644
--- a/mlir/lib/Dialect/MemRef/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/CMakeLists.txt
@@ -1 +1,22 @@
-add_subdirectory(IR)
+add_mlir_dialect_library(MLIRMemRef
+  IR/MemRefDialect.cpp
+  IR/MemRefOps.cpp
+  Utils/MemRefUtils.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${PROJECT_SOURCE_DIR}/inlude/mlir/Dialect/MemRefDialect
+
+  DEPENDS
+  MLIRStandardOpsIncGen
+  MLIRMemRefOpsIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRDialect
+  MLIRIR
+  MLIRStandard
+  MLIRTensor
+  MLIRViewLikeInterface
+)
diff --git a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt b/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
deleted file mode 100644
index aa9d57beb105bb7e6a43d7811f46bf60da476710..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-add_mlir_dialect_library(MLIRMemRef
-  MemRefDialect.cpp
-  MemRefOps.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${PROJECT_SOURCE_DIR}/inlude/mlir/Dialect/MemRefDialect
-
-  DEPENDS
-  MLIRStandardOpsIncGen
-  MLIRMemRefOpsIncGen
-
-  LINK_COMPONENTS
-  Core
-
-  LINK_LIBS PUBLIC
-  MLIRDialect
-  MLIRIR
-  MLIRStandard
-  MLIRTensor
-  MLIRViewLikeInterface
-)
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index db8a96fa2bed18b5c385cd0473196dc358da5f4f..fc179b29c239285fc7a04f791b00d44e4b1d9cb8 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/StandardOps/Utils/Utils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
@@ -209,14 +210,14 @@ struct SimplifyDeadAlloc : public OpRewritePattern<AllocOp> {
 };
 } // end anonymous namespace.
 
-void AllocOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void AllocOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                           MLIRContext *context) {
-  results.insert<SimplifyAllocConst<AllocOp>, SimplifyDeadAlloc>(context);
+  results.add<SimplifyAllocConst<AllocOp>, SimplifyDeadAlloc>(context);
 }
 
-void AllocaOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void AllocaOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                            MLIRContext *context) {
-  results.insert<SimplifyAllocConst<AllocaOp>>(context);
+  results.add<SimplifyAllocConst<AllocaOp>>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -290,9 +291,9 @@ struct TensorLoadToMemRef : public OpRewritePattern<BufferCastOp> {
 
 } // namespace
 
-void BufferCastOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<BufferCast, TensorLoadToMemRef>(context);
+void BufferCastOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                               MLIRContext *context) {
+  results.add<BufferCast, TensorLoadToMemRef>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -420,7 +421,7 @@ bool CastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
         if (!checkCompatible(aStride.value(), bStrides[aStride.index()]))
           return false;
     }
-    if (aT.getMemorySpaceAsInt() != bT.getMemorySpaceAsInt())
+    if (aT.getMemorySpace() != bT.getMemorySpace())
       return false;
 
     // They must have the same rank, and any specified dimensions must match.
@@ -447,10 +448,8 @@ bool CastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
     if (aEltType != bEltType)
       return false;
 
-    auto aMemSpace =
-        (aT) ? aT.getMemorySpaceAsInt() : uaT.getMemorySpaceAsInt();
-    auto bMemSpace =
-        (bT) ? bT.getMemorySpaceAsInt() : ubT.getMemorySpaceAsInt();
+    auto aMemSpace = (aT) ? aT.getMemorySpace() : uaT.getMemorySpace();
+    auto bMemSpace = (bT) ? bT.getMemorySpace() : ubT.getMemorySpace();
     if (aMemSpace != bMemSpace)
       return false;
 
@@ -464,6 +463,76 @@ OpFoldResult CastOp::fold(ArrayRef<Attribute> operands) {
   return succeeded(foldMemRefCast(*this)) ? getResult() : Value();
 }
 
+//===----------------------------------------------------------------------===//
+// CloneOp
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(CloneOp op) { return success(); }
+
+void CloneOp::getEffects(
+    SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
+        &effects) {
+  effects.emplace_back(MemoryEffects::Read::get(), input(),
+                       SideEffects::DefaultResource::get());
+  effects.emplace_back(MemoryEffects::Write::get(), output(),
+                       SideEffects::DefaultResource::get());
+}
+
+namespace {
+/// Fold Dealloc operations that are deallocating an AllocOp that is only used
+/// by other Dealloc operations.
+struct SimplifyClones : public OpRewritePattern<CloneOp> {
+  using OpRewritePattern<CloneOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(CloneOp cloneOp,
+                                PatternRewriter &rewriter) const override {
+    if (cloneOp.use_empty()) {
+      rewriter.eraseOp(cloneOp);
+      return success();
+    }
+
+    Value source = cloneOp.input();
+
+    // Removes the clone operation and the corresponding dealloc and alloc
+    // operation (if any).
+    auto tryRemoveClone = [&](Operation *sourceOp, Operation *dealloc,
+                              Operation *alloc) {
+      if (!sourceOp || !dealloc || !alloc ||
+          alloc->getBlock() != dealloc->getBlock())
+        return false;
+      rewriter.replaceOp(cloneOp, source);
+      rewriter.eraseOp(dealloc);
+      return true;
+    };
+
+    // Removes unnecessary clones that are derived from the result of the clone
+    // op.
+    Operation *deallocOp = findDealloc(cloneOp.output());
+    Operation *sourceOp = source.getDefiningOp();
+    if (tryRemoveClone(sourceOp, deallocOp, sourceOp))
+      return success();
+
+    // Removes unnecessary clones that are derived from the source of the clone
+    // op.
+    deallocOp = findDealloc(source);
+    if (tryRemoveClone(sourceOp, deallocOp, cloneOp))
+      return success();
+
+    return failure();
+  }
+};
+
+} // end anonymous namespace.
+
+void CloneOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+                                          MLIRContext *context) {
+  results.insert<SimplifyClones>(context);
+}
+
+OpFoldResult CloneOp::fold(ArrayRef<Attribute> operands) {
+  return succeeded(foldMemRefCast(*this)) ? getResult() : Value();
+}
+
 //===----------------------------------------------------------------------===//
 // DeallocOp
 //===----------------------------------------------------------------------===//
@@ -498,9 +567,9 @@ static LogicalResult verify(DeallocOp op) {
   return success();
 }
 
-void DeallocOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void DeallocOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
-  results.insert<SimplifyDeadDealloc>(context);
+  results.add<SimplifyDeadDealloc>(context);
 }
 
 LogicalResult DeallocOp::fold(ArrayRef<Attribute> cstOperands,
@@ -617,6 +686,10 @@ OpFoldResult DimOp::fold(ArrayRef<Attribute> operands) {
     return *(alloc.getDynamicSizes().begin() +
              memrefType.getDynamicDimIndex(unsignedIndex));
 
+  if (auto alloca = dyn_cast_or_null<AllocaOp>(definingOp))
+    return *(alloca.getDynamicSizes().begin() +
+             memrefType.getDynamicDimIndex(unsignedIndex));
+
   if (auto view = dyn_cast_or_null<ViewOp>(definingOp))
     return *(view.getDynamicSizes().begin() +
              memrefType.getDynamicDimIndex(unsignedIndex));
@@ -673,10 +746,10 @@ struct DimOfCastOp : public OpRewritePattern<DimOp> {
 };
 } // end anonymous namespace.
 
-void DimOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void DimOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                         MLIRContext *context) {
-  results.insert<DimOfMemRefReshape, DimOfCastOp<BufferCastOp>,
-                 DimOfCastOp<tensor::CastOp>>(context);
+  results.add<DimOfMemRefReshape, DimOfCastOp<BufferCastOp>,
+              DimOfCastOp<tensor::CastOp>>(context);
 }
 
 // ---------------------------------------------------------------------------
@@ -1065,9 +1138,9 @@ struct LoadOfBufferCast : public OpRewritePattern<LoadOp> {
 };
 } // end anonymous namespace.
 
-void LoadOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void LoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                          MLIRContext *context) {
-  results.insert<LoadOfBufferCast>(context);
+  results.add<LoadOfBufferCast>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1200,7 +1273,7 @@ static LogicalResult verify(ReinterpretCastOp op) {
   // The source and result memrefs should be in the same memory space.
   auto srcType = op.source().getType().cast<BaseMemRefType>();
   auto resultType = op.getType().cast<MemRefType>();
-  if (srcType.getMemorySpaceAsInt() != resultType.getMemorySpaceAsInt())
+  if (srcType.getMemorySpace() != resultType.getMemorySpace())
     return op.emitError("different memory spaces specified for source type ")
            << srcType << " and result memref type " << resultType;
   if (srcType.getElementType() != resultType.getElementType())
@@ -1385,7 +1458,7 @@ Type SubViewOp::inferResultType(MemRefType sourceMemRefType,
       staticSizes, sourceMemRefType.getElementType(),
       makeStridedLinearLayoutMap(targetStrides, targetOffset,
                                  sourceMemRefType.getContext()),
-      sourceMemRefType.getMemorySpaceAsInt());
+      sourceMemRefType.getMemorySpace());
 }
 
 Type SubViewOp::inferResultType(MemRefType sourceMemRefType,
@@ -1431,7 +1504,7 @@ Type SubViewOp::inferRankReducedResultType(
       map = getProjectedMap(maps.front(), dimsToProject);
     inferredType =
         MemRefType::get(projectedShape, inferredType.getElementType(), map,
-                        inferredType.getMemorySpaceAsInt());
+                        inferredType.getMemorySpace());
   }
   return inferredType;
 }
@@ -1609,7 +1682,7 @@ isRankReducedType(Type originalType, Type candidateReducedType,
   // Strided layout logic is relevant for MemRefType only.
   MemRefType original = originalType.cast<MemRefType>();
   MemRefType candidateReduced = candidateReducedType.cast<MemRefType>();
-  if (original.getMemorySpaceAsInt() != candidateReduced.getMemorySpaceAsInt())
+  if (original.getMemorySpace() != candidateReduced.getMemorySpace())
     return SubViewVerificationResult::MemSpaceMismatch;
 
   llvm::SmallDenseSet<unsigned> unusedDims = optionalUnusedDimsMask.getValue();
@@ -1683,7 +1756,7 @@ static LogicalResult verify(SubViewOp op) {
   MemRefType subViewType = op.getType();
 
   // The base memref and the view memref should be in the same memory space.
-  if (baseType.getMemorySpaceAsInt() != subViewType.getMemorySpaceAsInt())
+  if (baseType.getMemorySpace() != subViewType.getMemorySpace())
     return op.emitError("different memory spaces specified for base memref "
                         "type ")
            << baseType << " and subview memref type " << subViewType;
@@ -1798,11 +1871,11 @@ struct SubViewCanonicalizer {
   }
 };
 
-void SubViewOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void SubViewOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
-  results.insert<OpWithOffsetSizesAndStridesConstantArgumentFolder<
-                     SubViewOp, SubViewCanonicalizer>,
-                 SubViewOpMemRefCastFolder>(context);
+  results.add<OpWithOffsetSizesAndStridesConstantArgumentFolder<
+                  SubViewOp, SubViewCanonicalizer>,
+              SubViewOpMemRefCastFolder>(context);
 }
 
 OpFoldResult SubViewOp::fold(ArrayRef<Attribute> operands) {
@@ -1975,7 +2048,7 @@ static LogicalResult verify(ViewOp op) {
     return op.emitError("unsupported map for result memref type ") << viewType;
 
   // The base memref and the view memref should be in the same memory space.
-  if (baseType.getMemorySpaceAsInt() != viewType.getMemorySpaceAsInt())
+  if (baseType.getMemorySpace() != viewType.getMemorySpace())
     return op.emitError("different memory spaces specified for base memref "
                         "type ")
            << baseType << " and view memref type " << viewType;
@@ -2081,9 +2154,9 @@ struct ViewOpMemrefCastFolder : public OpRewritePattern<ViewOp> {
 
 } // end anonymous namespace
 
-void ViewOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ViewOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                          MLIRContext *context) {
-  results.insert<ViewOpShapeFolder, ViewOpMemrefCastFolder>(context);
+  results.add<ViewOpShapeFolder, ViewOpMemrefCastFolder>(context);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..26a9a217134e28af2028c479ed8ac96dec82d0e7
--- /dev/null
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -0,0 +1,35 @@
+//===- Utils.cpp - Utilities to support the MemRef dialect ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements utilities for the MemRef dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+
+using namespace mlir;
+
+/// Finds associated deallocs that can be linked to our allocation nodes (if
+/// any).
+Operation *mlir::findDealloc(Value allocValue) {
+  auto userIt = llvm::find_if(allocValue.getUsers(), [&](Operation *user) {
+    auto effectInterface = dyn_cast<MemoryEffectOpInterface>(user);
+    if (!effectInterface)
+      return false;
+    // Try to find a free effect that is applied to one of our values
+    // that will be automatically freed by our pass.
+    SmallVector<MemoryEffects::EffectInstance, 2> effects;
+    effectInterface.getEffectsOnValue(allocValue, effects);
+    return llvm::any_of(effects, [&](MemoryEffects::EffectInstance &it) {
+      return isa<MemoryEffects::Free>(it.getEffect());
+    });
+  });
+  // Assign the associated dealloc operation (if any).
+  return userIt != allocValue.user_end() ? *userIt : nullptr;
+}
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 907ba65c07b7edd4bab16c024dc3c7644d85121e..06854cd99be14ec45798bd9ba3ae9f138fccb19d 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/OperationSupport.h"
 
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include <cstddef>
@@ -172,8 +173,8 @@ static void printParallelOp(OpAsmPrinter &p, ParallelOp op) {
 }
 
 /// Emit an error if the same clause is present more than once on an operation.
-static ParseResult allowedOnce(OpAsmParser &parser, llvm::StringRef clause,
-                               llvm::StringRef operation) {
+static ParseResult allowedOnce(OpAsmParser &parser, StringRef clause,
+                               StringRef operation) {
   return parser.emitError(parser.getNameLoc())
          << " at most one " << clause << " clause can appear on the "
          << operation << " operation";
@@ -213,7 +214,7 @@ static ParseResult parseParallelOp(OpAsmParser &parser,
   SmallVector<OpAsmParser::OperandType, 4> allocators;
   SmallVector<Type, 4> allocatorTypes;
   std::array<int, 8> segments{0, 0, 0, 0, 0, 0, 0, 0};
-  llvm::StringRef keyword;
+  StringRef keyword;
   bool defaultVal = false;
   bool procBind = false;
 
@@ -225,11 +226,11 @@ static ParseResult parseParallelOp(OpAsmParser &parser,
   const int copyinClausePos = 5;
   const int allocateClausePos = 6;
   const int allocatorPos = 7;
-  const llvm::StringRef opName = result.name.getStringRef();
+  const StringRef opName = result.name.getStringRef();
 
   while (succeeded(parser.parseOptionalKeyword(&keyword))) {
     if (keyword == "if") {
-      // Fail if there was already another if condition
+      // Fail if there was already another if condition.
       if (segments[ifClausePos])
         return allowedOnce(parser, "if", opName);
       if (parser.parseLParen() || parser.parseOperand(ifCond.first) ||
@@ -237,7 +238,7 @@ static ParseResult parseParallelOp(OpAsmParser &parser,
         return failure();
       segments[ifClausePos] = 1;
     } else if (keyword == "num_threads") {
-      // fail if there was already another num_threads clause
+      // Fail if there was already another num_threads clause.
       if (segments[numThreadsClausePos])
         return allowedOnce(parser, "num_threads", opName);
       if (parser.parseLParen() || parser.parseOperand(numThreads.first) ||
@@ -245,35 +246,35 @@ static ParseResult parseParallelOp(OpAsmParser &parser,
         return failure();
       segments[numThreadsClausePos] = 1;
     } else if (keyword == "private") {
-      // fail if there was already another private clause
+      // Fail if there was already another private clause.
       if (segments[privateClausePos])
         return allowedOnce(parser, "private", opName);
       if (parseOperandAndTypeList(parser, privates, privateTypes))
         return failure();
       segments[privateClausePos] = privates.size();
     } else if (keyword == "firstprivate") {
-      // fail if there was already another firstprivate clause
+      // Fail if there was already another firstprivate clause.
       if (segments[firstprivateClausePos])
         return allowedOnce(parser, "firstprivate", opName);
       if (parseOperandAndTypeList(parser, firstprivates, firstprivateTypes))
         return failure();
       segments[firstprivateClausePos] = firstprivates.size();
     } else if (keyword == "shared") {
-      // fail if there was already another shared clause
+      // Fail if there was already another shared clause.
       if (segments[sharedClausePos])
         return allowedOnce(parser, "shared", opName);
       if (parseOperandAndTypeList(parser, shareds, sharedTypes))
         return failure();
       segments[sharedClausePos] = shareds.size();
     } else if (keyword == "copyin") {
-      // fail if there was already another copyin clause
+      // Fail if there was already another copyin clause.
       if (segments[copyinClausePos])
         return allowedOnce(parser, "copyin", opName);
       if (parseOperandAndTypeList(parser, copyins, copyinTypes))
         return failure();
       segments[copyinClausePos] = copyins.size();
     } else if (keyword == "allocate") {
-      // fail if there was already another allocate clause
+      // Fail if there was already another allocate clause.
       if (segments[allocateClausePos])
         return allowedOnce(parser, "allocate", opName);
       if (parseAllocateAndAllocator(parser, allocates, allocateTypes,
@@ -282,27 +283,27 @@ static ParseResult parseParallelOp(OpAsmParser &parser,
       segments[allocateClausePos] = allocates.size();
       segments[allocatorPos] = allocators.size();
     } else if (keyword == "default") {
-      // fail if there was already another default clause
+      // Fail if there was already another default clause.
       if (defaultVal)
         return allowedOnce(parser, "default", opName);
       defaultVal = true;
-      llvm::StringRef defval;
+      StringRef defval;
       if (parser.parseLParen() || parser.parseKeyword(&defval) ||
           parser.parseRParen())
         return failure();
-      llvm::SmallString<16> attrval;
+      SmallString<16> attrval;
       // The def prefix is required for the attribute as "private" is a keyword
-      // in C++
+      // in C++.
       attrval += "def";
       attrval += defval;
       auto attr = parser.getBuilder().getStringAttr(attrval);
       result.addAttribute("default_val", attr);
     } else if (keyword == "proc_bind") {
-      // fail if there was already another proc_bind clause
+      // Fail if there was already another proc_bind clause.
       if (procBind)
         return allowedOnce(parser, "proc_bind", opName);
       procBind = true;
-      llvm::StringRef bind;
+      StringRef bind;
       if (parser.parseLParen() || parser.parseKeyword(&bind) ||
           parser.parseRParen())
         return failure();
@@ -315,48 +316,48 @@ static ParseResult parseParallelOp(OpAsmParser &parser,
     }
   }
 
-  // Add if parameter
+  // Add if parameter.
   if (segments[ifClausePos] &&
       parser.resolveOperand(ifCond.first, ifCond.second, result.operands))
     return failure();
 
-  // Add num_threads parameter
+  // Add num_threads parameter.
   if (segments[numThreadsClausePos] &&
       parser.resolveOperand(numThreads.first, numThreads.second,
                             result.operands))
     return failure();
 
-  // Add private parameters
+  // Add private parameters.
   if (segments[privateClausePos] &&
       parser.resolveOperands(privates, privateTypes, privates[0].location,
                              result.operands))
     return failure();
 
-  // Add firstprivate parameters
+  // Add firstprivate parameters.
   if (segments[firstprivateClausePos] &&
       parser.resolveOperands(firstprivates, firstprivateTypes,
                              firstprivates[0].location, result.operands))
     return failure();
 
-  // Add shared parameters
+  // Add shared parameters.
   if (segments[sharedClausePos] &&
       parser.resolveOperands(shareds, sharedTypes, shareds[0].location,
                              result.operands))
     return failure();
 
-  // Add copyin parameters
+  // Add copyin parameters.
   if (segments[copyinClausePos] &&
       parser.resolveOperands(copyins, copyinTypes, copyins[0].location,
                              result.operands))
     return failure();
 
-  // Add allocate parameters
+  // Add allocate parameters.
   if (segments[allocateClausePos] &&
       parser.resolveOperands(allocates, allocateTypes, allocates[0].location,
                              result.operands))
     return failure();
 
-  // Add allocator parameters
+  // Add allocator parameters.
   if (segments[allocatorPos] &&
       parser.resolveOperands(allocators, allocatorTypes, allocators[0].location,
                              result.operands))
@@ -373,6 +374,335 @@ static ParseResult parseParallelOp(OpAsmParser &parser,
   return success();
 }
 
+/// linear ::= `linear` `(` linear-list `)`
+/// linear-list := linear-val | linear-val linear-list
+/// linear-val := ssa-id-and-type `=` ssa-id-and-type
+static ParseResult
+parseLinearClause(OpAsmParser &parser,
+                  SmallVectorImpl<OpAsmParser::OperandType> &vars,
+                  SmallVectorImpl<Type> &types,
+                  SmallVectorImpl<OpAsmParser::OperandType> &stepVars) {
+  if (parser.parseLParen())
+    return failure();
+
+  do {
+    OpAsmParser::OperandType var;
+    Type type;
+    OpAsmParser::OperandType stepVar;
+    if (parser.parseOperand(var) || parser.parseEqual() ||
+        parser.parseOperand(stepVar) || parser.parseColonType(type))
+      return failure();
+
+    vars.push_back(var);
+    types.push_back(type);
+    stepVars.push_back(stepVar);
+  } while (succeeded(parser.parseOptionalComma()));
+
+  if (parser.parseRParen())
+    return failure();
+
+  return success();
+}
+
+/// schedule ::= `schedule` `(` sched-list `)`
+/// sched-list ::= sched-val | sched-val sched-list
+/// sched-val ::= sched-with-chunk | sched-wo-chunk
+/// sched-with-chunk ::= sched-with-chunk-types (`=` ssa-id-and-type)?
+/// sched-with-chunk-types ::= `static` | `dynamic` | `guided`
+/// sched-wo-chunk ::=  `auto` | `runtime`
+static ParseResult
+parseScheduleClause(OpAsmParser &parser, SmallString<8> &schedule,
+                    Optional<OpAsmParser::OperandType> &chunkSize) {
+  if (parser.parseLParen())
+    return failure();
+
+  StringRef keyword;
+  if (parser.parseKeyword(&keyword))
+    return failure();
+
+  schedule = keyword;
+  if (keyword == "static" || keyword == "dynamic" || keyword == "guided") {
+    if (succeeded(parser.parseOptionalEqual())) {
+      chunkSize = OpAsmParser::OperandType{};
+      if (parser.parseOperand(*chunkSize))
+        return failure();
+    } else {
+      chunkSize = llvm::NoneType::None;
+    }
+  } else if (keyword == "auto" || keyword == "runtime") {
+    chunkSize = llvm::NoneType::None;
+  } else {
+    return parser.emitError(parser.getNameLoc()) << " expected schedule kind";
+  }
+
+  if (parser.parseRParen())
+    return failure();
+
+  return success();
+}
+
+/// Parses an OpenMP Workshare Loop operation
+///
+/// operation ::= `omp.wsloop` loop-control clause-list
+/// loop-control ::= `(` ssa-id-list `)` `:` type `=`  loop-bounds
+/// loop-bounds := `(` ssa-id-list `)` to `(` ssa-id-list `)` steps
+/// steps := `step` `(`ssa-id-list`)`
+/// clause-list ::= clause | empty | clause-list
+/// clause ::= private | firstprivate | lastprivate | linear | schedule |
+//             collapse | nowait | ordered | order | inclusive
+/// private ::= `private` `(` ssa-id-and-type-list `)`
+/// firstprivate ::= `firstprivate` `(` ssa-id-and-type-list `)`
+/// lastprivate ::= `lastprivate` `(` ssa-id-and-type-list `)`
+/// linear ::= `linear` `(` linear-list `)`
+/// schedule ::= `schedule` `(` sched-list `)`
+/// collapse ::= `collapse` `(` ssa-id-and-type `)`
+/// nowait ::= `nowait`
+/// ordered ::= `ordered` `(` ssa-id-and-type `)`
+/// order ::= `order` `(` `concurrent` `)`
+/// inclusive ::= `inclusive`
+///
+static ParseResult parseWsLoopOp(OpAsmParser &parser, OperationState &result) {
+  Type loopVarType;
+  int numIVs;
+
+  // Parse an opening `(` followed by induction variables followed by `)`
+  SmallVector<OpAsmParser::OperandType> ivs;
+  if (parser.parseRegionArgumentList(ivs, /*requiredOperandCount=*/-1,
+                                     OpAsmParser::Delimiter::Paren))
+    return failure();
+
+  numIVs = static_cast<int>(ivs.size());
+
+  if (parser.parseColonType(loopVarType))
+    return failure();
+
+  // Parse loop bounds.
+  SmallVector<OpAsmParser::OperandType> lower;
+  if (parser.parseEqual() ||
+      parser.parseOperandList(lower, numIVs, OpAsmParser::Delimiter::Paren) ||
+      parser.resolveOperands(lower, loopVarType, result.operands))
+    return failure();
+
+  SmallVector<OpAsmParser::OperandType> upper;
+  if (parser.parseKeyword("to") ||
+      parser.parseOperandList(upper, numIVs, OpAsmParser::Delimiter::Paren) ||
+      parser.resolveOperands(upper, loopVarType, result.operands))
+    return failure();
+
+  // Parse step values.
+  SmallVector<OpAsmParser::OperandType> steps;
+  if (parser.parseKeyword("step") ||
+      parser.parseOperandList(steps, numIVs, OpAsmParser::Delimiter::Paren) ||
+      parser.resolveOperands(steps, loopVarType, result.operands))
+    return failure();
+
+  SmallVector<OpAsmParser::OperandType> privates;
+  SmallVector<Type> privateTypes;
+  SmallVector<OpAsmParser::OperandType> firstprivates;
+  SmallVector<Type> firstprivateTypes;
+  SmallVector<OpAsmParser::OperandType> lastprivates;
+  SmallVector<Type> lastprivateTypes;
+  SmallVector<OpAsmParser::OperandType> linears;
+  SmallVector<Type> linearTypes;
+  SmallVector<OpAsmParser::OperandType> linearSteps;
+  SmallString<8> schedule;
+  Optional<OpAsmParser::OperandType> scheduleChunkSize;
+  std::array<int, 9> segments{numIVs, numIVs, numIVs, 0, 0, 0, 0, 0, 0};
+
+  const StringRef opName = result.name.getStringRef();
+  StringRef keyword;
+
+  enum SegmentPos {
+    lbPos = 0,
+    ubPos,
+    stepPos,
+    privateClausePos,
+    firstprivateClausePos,
+    lastprivateClausePos,
+    linearClausePos,
+    linearStepPos,
+    scheduleClausePos,
+  };
+
+  while (succeeded(parser.parseOptionalKeyword(&keyword))) {
+    if (keyword == "private") {
+      if (segments[privateClausePos])
+        return allowedOnce(parser, "private", opName);
+      if (parseOperandAndTypeList(parser, privates, privateTypes))
+        return failure();
+      segments[privateClausePos] = privates.size();
+    } else if (keyword == "firstprivate") {
+      // fail if there was already another firstprivate clause
+      if (segments[firstprivateClausePos])
+        return allowedOnce(parser, "firstprivate", opName);
+      if (parseOperandAndTypeList(parser, firstprivates, firstprivateTypes))
+        return failure();
+      segments[firstprivateClausePos] = firstprivates.size();
+    } else if (keyword == "lastprivate") {
+      // fail if there was already another shared clause
+      if (segments[lastprivateClausePos])
+        return allowedOnce(parser, "lastprivate", opName);
+      if (parseOperandAndTypeList(parser, lastprivates, lastprivateTypes))
+        return failure();
+      segments[lastprivateClausePos] = lastprivates.size();
+    } else if (keyword == "linear") {
+      // fail if there was already another linear clause
+      if (segments[linearClausePos])
+        return allowedOnce(parser, "linear", opName);
+      if (parseLinearClause(parser, linears, linearTypes, linearSteps))
+        return failure();
+      segments[linearClausePos] = linears.size();
+      segments[linearStepPos] = linearSteps.size();
+    } else if (keyword == "schedule") {
+      if (!schedule.empty())
+        return allowedOnce(parser, "schedule", opName);
+      if (parseScheduleClause(parser, schedule, scheduleChunkSize))
+        return failure();
+      if (scheduleChunkSize) {
+        segments[scheduleClausePos] = 1;
+      }
+    } else if (keyword == "collapse") {
+      auto type = parser.getBuilder().getI64Type();
+      mlir::IntegerAttr attr;
+      if (parser.parseLParen() || parser.parseAttribute(attr, type) ||
+          parser.parseRParen())
+        return failure();
+      result.addAttribute("collapse_val", attr);
+    } else if (keyword == "nowait") {
+      auto attr = UnitAttr::get(parser.getBuilder().getContext());
+      result.addAttribute("nowait", attr);
+    } else if (keyword == "ordered") {
+      mlir::IntegerAttr attr;
+      if (succeeded(parser.parseOptionalLParen())) {
+        auto type = parser.getBuilder().getI64Type();
+        if (parser.parseAttribute(attr, type))
+          return failure();
+        if (parser.parseRParen())
+          return failure();
+      } else {
+        // Use 0 to represent no ordered parameter was specified
+        attr = parser.getBuilder().getI64IntegerAttr(0);
+      }
+      result.addAttribute("ordered_val", attr);
+    } else if (keyword == "order") {
+      StringRef order;
+      if (parser.parseLParen() || parser.parseKeyword(&order) ||
+          parser.parseRParen())
+        return failure();
+      auto attr = parser.getBuilder().getStringAttr(order);
+      result.addAttribute("order", attr);
+    } else if (keyword == "inclusive") {
+      auto attr = UnitAttr::get(parser.getBuilder().getContext());
+      result.addAttribute("inclusive", attr);
+    }
+  }
+
+  if (segments[privateClausePos]) {
+    parser.resolveOperands(privates, privateTypes, privates[0].location,
+                           result.operands);
+  }
+
+  if (segments[firstprivateClausePos]) {
+    parser.resolveOperands(firstprivates, firstprivateTypes,
+                           firstprivates[0].location, result.operands);
+  }
+
+  if (segments[lastprivateClausePos]) {
+    parser.resolveOperands(lastprivates, lastprivateTypes,
+                           lastprivates[0].location, result.operands);
+  }
+
+  if (segments[linearClausePos]) {
+    parser.resolveOperands(linears, linearTypes, linears[0].location,
+                           result.operands);
+    auto linearStepType = parser.getBuilder().getI32Type();
+    SmallVector<Type> linearStepTypes(linearSteps.size(), linearStepType);
+    parser.resolveOperands(linearSteps, linearStepTypes,
+                           linearSteps[0].location, result.operands);
+  }
+
+  if (!schedule.empty()) {
+    schedule[0] = llvm::toUpper(schedule[0]);
+    auto attr = parser.getBuilder().getStringAttr(schedule);
+    result.addAttribute("schedule_val", attr);
+    if (scheduleChunkSize) {
+      auto chunkSizeType = parser.getBuilder().getI32Type();
+      parser.resolveOperand(*scheduleChunkSize, chunkSizeType, result.operands);
+    }
+  }
+
+  result.addAttribute("operand_segment_sizes",
+                      parser.getBuilder().getI32VectorAttr(segments));
+
+  // Now parse the body.
+  Region *body = result.addRegion();
+  SmallVector<Type> ivTypes(numIVs, loopVarType);
+  if (parser.parseRegion(*body, ivs, ivTypes))
+    return failure();
+  return success();
+}
+
+static void printWsLoopOp(OpAsmPrinter &p, WsLoopOp op) {
+  auto args = op.getRegion().front().getArguments();
+  p << op.getOperationName() << " (" << args << ") : " << args[0].getType()
+    << " = (" << op.lowerBound() << ") to (" << op.upperBound() << ") step ("
+    << op.step() << ")";
+
+  // Print private, firstprivate, shared and copyin parameters
+  auto printDataVars = [&p](StringRef name, OperandRange vars) {
+    if (vars.empty())
+      return;
+
+    p << " " << name << "(";
+    llvm::interleaveComma(
+        vars, p, [&](const Value &v) { p << v << " : " << v.getType(); });
+    p << ")";
+  };
+  printDataVars("private", op.private_vars());
+  printDataVars("firstprivate", op.firstprivate_vars());
+  printDataVars("lastprivate", op.lastprivate_vars());
+
+  auto linearVars = op.linear_vars();
+  auto linearVarsSize = linearVars.size();
+  if (linearVarsSize) {
+    p << " "
+      << "linear"
+      << "(";
+    for (unsigned i = 0; i < linearVarsSize; ++i) {
+      std::string separator = i == linearVarsSize - 1 ? ")" : ", ";
+      p << linearVars[i];
+      if (op.linear_step_vars().size() > i)
+        p << " = " << op.linear_step_vars()[i];
+      p << " : " << linearVars[i].getType() << separator;
+    }
+  }
+
+  if (auto sched = op.schedule_val()) {
+    auto schedLower = sched->lower();
+    p << " schedule(" << schedLower;
+    if (auto chunk = op.schedule_chunk_var()) {
+      p << " = " << chunk;
+    }
+    p << ")";
+  }
+
+  if (auto collapse = op.collapse_val())
+    p << " collapse(" << collapse << ")";
+
+  if (op.nowait())
+    p << " nowait";
+
+  if (auto ordered = op.ordered_val()) {
+    p << " ordered(" << ordered << ")";
+  }
+
+  if (op.inclusive()) {
+    p << " inclusive";
+  }
+
+  p.printRegion(op.region(), /*printEntryBlockArgs=*/false);
+}
+
 //===----------------------------------------------------------------------===//
 // WsLoopOp
 //===----------------------------------------------------------------------===//
@@ -386,9 +716,71 @@ void WsLoopOp::build(OpBuilder &builder, OperationState &state,
         /*linear_vars=*/ValueRange(), /*linear_step_vars=*/ValueRange(),
         /*schedule_val=*/nullptr, /*schedule_chunk_var=*/nullptr,
         /*collapse_val=*/nullptr,
-        /*nowait=*/false, /*ordered_val=*/nullptr, /*order_val=*/nullptr,
-        /*inclusive=*/false);
+        /*nowait=*/nullptr, /*ordered_val=*/nullptr, /*order_val=*/nullptr,
+        /*inclusive=*/nullptr, /*buildBody=*/false);
+  state.addAttributes(attributes);
+}
+
+void WsLoopOp::build(OpBuilder &, OperationState &state, TypeRange resultTypes,
+                     ValueRange operands, ArrayRef<NamedAttribute> attributes) {
+  state.addOperands(operands);
   state.addAttributes(attributes);
+  (void)state.addRegion();
+  assert(resultTypes.size() == 0u && "mismatched number of return types");
+  state.addTypes(resultTypes);
+}
+
+void WsLoopOp::build(OpBuilder &builder, OperationState &result,
+                     TypeRange typeRange, ValueRange lowerBounds,
+                     ValueRange upperBounds, ValueRange steps,
+                     ValueRange privateVars, ValueRange firstprivateVars,
+                     ValueRange lastprivateVars, ValueRange linearVars,
+                     ValueRange linearStepVars, StringAttr scheduleVal,
+                     Value scheduleChunkVar, IntegerAttr collapseVal,
+                     UnitAttr nowait, IntegerAttr orderedVal,
+                     StringAttr orderVal, UnitAttr inclusive, bool buildBody) {
+  result.addOperands(lowerBounds);
+  result.addOperands(upperBounds);
+  result.addOperands(steps);
+  result.addOperands(privateVars);
+  result.addOperands(firstprivateVars);
+  result.addOperands(linearVars);
+  result.addOperands(linearStepVars);
+  if (scheduleChunkVar)
+    result.addOperands(scheduleChunkVar);
+
+  if (scheduleVal)
+    result.addAttribute("schedule_val", scheduleVal);
+  if (collapseVal)
+    result.addAttribute("collapse_val", collapseVal);
+  if (nowait)
+    result.addAttribute("nowait", nowait);
+  if (orderedVal)
+    result.addAttribute("ordered_val", orderedVal);
+  if (orderVal)
+    result.addAttribute("order", orderVal);
+  if (inclusive)
+    result.addAttribute("inclusive", inclusive);
+  result.addAttribute(
+      WsLoopOp::getOperandSegmentSizeAttr(),
+      builder.getI32VectorAttr(
+          {static_cast<int32_t>(lowerBounds.size()),
+           static_cast<int32_t>(upperBounds.size()),
+           static_cast<int32_t>(steps.size()),
+           static_cast<int32_t>(privateVars.size()),
+           static_cast<int32_t>(firstprivateVars.size()),
+           static_cast<int32_t>(lastprivateVars.size()),
+           static_cast<int32_t>(linearVars.size()),
+           static_cast<int32_t>(linearStepVars.size()),
+           static_cast<int32_t>(scheduleChunkVar != nullptr ? 1 : 0)}));
+
+  Region *bodyRegion = result.addRegion();
+  if (buildBody) {
+    OpBuilder::InsertionGuard guard(builder);
+    unsigned numIVs = steps.size();
+    SmallVector<Type, 8> argTypes(numIVs, steps.getType().front());
+    builder.createBlock(bodyRegion, {}, argTypes);
+  }
 }
 
 #define GET_OP_CLASSES
diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp
index beb43d7072f2ec0d92556c9d6601967cf3ecfcc1..8164c89dac5445ba3dabc30b4b52238464b852d5 100644
--- a/mlir/lib/Dialect/PDL/IR/PDL.cpp
+++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp
@@ -28,36 +28,67 @@ void PDLDialect::initialize() {
   registerTypes();
 }
 
+//===----------------------------------------------------------------------===//
+// PDL Operations
+//===----------------------------------------------------------------------===//
+
 /// Returns true if the given operation is used by a "binding" pdl operation
 /// within the main matcher body of a `pdl.pattern`.
+static bool hasBindingUseInMatcher(Operation *op, Block *matcherBlock) {
+  for (OpOperand &use : op->getUses()) {
+    Operation *user = use.getOwner();
+    if (user->getBlock() != matcherBlock)
+      continue;
+    if (isa<AttributeOp, OperandOp, OperandsOp, OperationOp>(user))
+      return true;
+    // Only the first operand of RewriteOp may be bound to, i.e. the root
+    // operation of the pattern.
+    if (isa<RewriteOp>(user) && use.getOperandNumber() == 0)
+      return true;
+    // A result by itself is not binding, it must also be bound.
+    if (isa<ResultOp, ResultsOp>(user) &&
+        hasBindingUseInMatcher(user, matcherBlock))
+      return true;
+  }
+  return false;
+}
+
+/// Returns success if the given operation is used by a "binding" pdl operation
+/// within the main matcher body of a `pdl.pattern`. On failure, emits an error
+/// with the given context message.
 static LogicalResult
 verifyHasBindingUseInMatcher(Operation *op,
                              StringRef bindableContextStr = "`pdl.operation`") {
   // If the pattern is not a pattern, there is nothing to do.
   if (!isa<PatternOp>(op->getParentOp()))
     return success();
-  Block *matcherBlock = op->getBlock();
-  for (Operation *user : op->getUsers()) {
-    if (user->getBlock() != matcherBlock)
-      continue;
-    if (isa<AttributeOp, OperandOp, OperationOp, RewriteOp>(user))
-      return success();
-  }
+  if (hasBindingUseInMatcher(op, op->getBlock()))
+    return success();
   return op->emitOpError()
          << "expected a bindable (i.e. " << bindableContextStr
          << ") user when defined in the matcher body of a `pdl.pattern`";
 }
 
 //===----------------------------------------------------------------------===//
-// pdl::ApplyConstraintOp
+// pdl::ApplyNativeConstraintOp
 //===----------------------------------------------------------------------===//
 
-static LogicalResult verify(ApplyConstraintOp op) {
+static LogicalResult verify(ApplyNativeConstraintOp op) {
   if (op.getNumOperands() == 0)
     return op.emitOpError("expected at least one argument");
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// pdl::ApplyNativeRewriteOp
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(ApplyNativeRewriteOp op) {
+  if (op.getNumOperands() == 0 && op.getNumResults() == 0)
+    return op.emitOpError("expected at least one argument or result");
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // pdl::AttributeOp
 //===----------------------------------------------------------------------===//
@@ -83,40 +114,23 @@ static LogicalResult verify(OperandOp op) {
 }
 
 //===----------------------------------------------------------------------===//
-// pdl::OperationOp
+// pdl::OperandsOp
 //===----------------------------------------------------------------------===//
 
-static ParseResult parseOperationOp(OpAsmParser &p, OperationState &state) {
-  Builder &builder = p.getBuilder();
-
-  // Parse the optional operation name.
-  bool startsWithOperands = succeeded(p.parseOptionalLParen());
-  bool startsWithAttributes =
-      !startsWithOperands && succeeded(p.parseOptionalLBrace());
-  bool startsWithOpName = false;
-  if (!startsWithAttributes && !startsWithOperands) {
-    StringAttr opName;
-    OptionalParseResult opNameResult =
-        p.parseOptionalAttribute(opName, "name", state.attributes);
-    startsWithOpName = opNameResult.hasValue();
-    if (startsWithOpName && failed(*opNameResult))
-      return failure();
-  }
+static LogicalResult verify(OperandsOp op) {
+  return verifyHasBindingUseInMatcher(op);
+}
 
-  // Parse the operands.
-  SmallVector<OpAsmParser::OperandType, 4> operands;
-  if (startsWithOperands ||
-      (!startsWithAttributes && succeeded(p.parseOptionalLParen()))) {
-    if (p.parseOperandList(operands) || p.parseRParen() ||
-        p.resolveOperands(operands, builder.getType<ValueType>(),
-                          state.operands))
-      return failure();
-  }
+//===----------------------------------------------------------------------===//
+// pdl::OperationOp
+//===----------------------------------------------------------------------===//
 
-  // Parse the attributes.
+static ParseResult parseOperationOpAttributes(
+    OpAsmParser &p, SmallVectorImpl<OpAsmParser::OperandType> &attrOperands,
+    ArrayAttr &attrNamesAttr) {
+  Builder &builder = p.getBuilder();
   SmallVector<Attribute, 4> attrNames;
-  if (startsWithAttributes || succeeded(p.parseOptionalLBrace())) {
-    SmallVector<OpAsmParser::OperandType, 4> attrOps;
+  if (succeeded(p.parseOptionalLBrace())) {
     do {
       StringAttr nameAttr;
       OpAsmParser::OperandType operand;
@@ -124,68 +138,29 @@ static ParseResult parseOperationOp(OpAsmParser &p, OperationState &state) {
           p.parseOperand(operand))
         return failure();
       attrNames.push_back(nameAttr);
-      attrOps.push_back(operand);
+      attrOperands.push_back(operand);
     } while (succeeded(p.parseOptionalComma()));
-
-    if (p.parseRBrace() ||
-        p.resolveOperands(attrOps, builder.getType<AttributeType>(),
-                          state.operands))
+    if (p.parseRBrace())
       return failure();
   }
-  state.addAttribute("attributeNames", builder.getArrayAttr(attrNames));
-  state.addTypes(builder.getType<OperationType>());
-
-  // Parse the result types.
-  SmallVector<OpAsmParser::OperandType, 4> opResultTypes;
-  if (succeeded(p.parseOptionalArrow())) {
-    if (p.parseOperandList(opResultTypes) ||
-        p.resolveOperands(opResultTypes, builder.getType<TypeType>(),
-                          state.operands))
-      return failure();
-    state.types.append(opResultTypes.size(), builder.getType<ValueType>());
-  }
-
-  if (p.parseOptionalAttrDict(state.attributes))
-    return failure();
-
-  int32_t operandSegmentSizes[] = {static_cast<int32_t>(operands.size()),
-                                   static_cast<int32_t>(attrNames.size()),
-                                   static_cast<int32_t>(opResultTypes.size())};
-  state.addAttribute("operand_segment_sizes",
-                     builder.getI32VectorAttr(operandSegmentSizes));
+  attrNamesAttr = builder.getArrayAttr(attrNames);
   return success();
 }
 
-static void print(OpAsmPrinter &p, OperationOp op) {
-  p << "pdl.operation ";
-  if (Optional<StringRef> name = op.name())
-    p << '"' << *name << '"';
-
-  auto operandValues = op.operands();
-  if (!operandValues.empty())
-    p << '(' << operandValues << ')';
-
-  // Emit the optional attributes.
-  ArrayAttr attrNames = op.attributeNames();
-  if (!attrNames.empty()) {
-    Operation::operand_range attrArgs = op.attributes();
-    p << " {";
-    interleaveComma(llvm::seq<int>(0, attrNames.size()), p,
-                    [&](int i) { p << attrNames[i] << " = " << attrArgs[i]; });
-    p << '}';
-  }
-
-  // Print the result type constraints of the operation.
-  if (!op.results().empty())
-    p << " -> " << op.types();
-  p.printOptionalAttrDict(op->getAttrs(),
-                          {"attributeNames", "name", "operand_segment_sizes"});
+static void printOperationOpAttributes(OpAsmPrinter &p, OperationOp op,
+                                       OperandRange attrArgs,
+                                       ArrayAttr attrNames) {
+  if (attrNames.empty())
+    return;
+  p << " {";
+  interleaveComma(llvm::seq<int>(0, attrNames.size()), p,
+                  [&](int i) { p << attrNames[i] << " = " << attrArgs[i]; });
+  p << '}';
 }
 
 /// Verifies that the result types of this operation, defined within a
 /// `pdl.rewrite`, can be inferred.
 static LogicalResult verifyResultTypesAreInferrable(OperationOp op,
-                                                    ResultRange opResults,
                                                     OperandRange resultTypes) {
   // Functor that returns if the given use can be used to infer a type.
   Block *rewriterBlock = op->getBlock();
@@ -207,18 +182,13 @@ static LogicalResult verifyResultTypesAreInferrable(OperationOp op,
     return success();
 
   // Otherwise, make sure each of the types can be inferred.
-  for (int i : llvm::seq<int>(0, opResults.size())) {
-    Operation *resultTypeOp = resultTypes[i].getDefiningOp();
+  for (auto it : llvm::enumerate(resultTypes)) {
+    Operation *resultTypeOp = it.value().getDefiningOp();
     assert(resultTypeOp && "expected valid result type operation");
 
-    // If the op was defined by a `create_native`, it is guaranteed to be
+    // If the op was defined by a `apply_native_rewrite`, it is guaranteed to be
     // usable.
-    if (isa<CreateNativeOp>(resultTypeOp))
-      continue;
-
-    // If the type is already constrained, there is nothing to do.
-    TypeOp typeOp = cast<TypeOp>(resultTypeOp);
-    if (typeOp.type())
+    if (isa<ApplyNativeRewriteOp>(resultTypeOp))
       continue;
 
     // If the type operation was defined in the matcher and constrains the
@@ -226,17 +196,19 @@ static LogicalResult verifyResultTypesAreInferrable(OperationOp op,
     auto constrainsInputOp = [rewriterBlock](Operation *user) {
       return user->getBlock() != rewriterBlock && isa<OperationOp>(user);
     };
-    if (llvm::any_of(typeOp.getResult().getUsers(), constrainsInputOp))
-      continue;
+    if (TypeOp typeOp = dyn_cast<TypeOp>(resultTypeOp)) {
+      if (typeOp.type() || llvm::any_of(typeOp->getUsers(), constrainsInputOp))
+        continue;
+    } else if (TypesOp typeOp = dyn_cast<TypesOp>(resultTypeOp)) {
+      if (typeOp.types() || llvm::any_of(typeOp->getUsers(), constrainsInputOp))
+        continue;
+    }
 
-    // Otherwise, check to see if any uses of the result can infer the type.
-    if (llvm::any_of(opResults[i].getUses(), canInferTypeFromUse))
-      continue;
     return op
         .emitOpError("must have inferable or constrained result types when "
                      "nested within `pdl.rewrite`")
         .attachNote()
-        .append("result type #", i, " was not constrained");
+        .append("result type #", it.index(), " was not constrained");
   }
   return success();
 }
@@ -256,19 +228,10 @@ static LogicalResult verify(OperationOp op) {
            << " values";
   }
 
-  OperandRange resultTypes = op.types();
-  auto opResults = op.results();
-  if (resultTypes.size() != opResults.size()) {
-    return op.emitOpError() << "expected the same number of result values and "
-                               "result type constraints, got "
-                            << opResults.size() << " results and "
-                            << resultTypes.size() << " constraints";
-  }
-
   // If the operation is within a rewrite body and doesn't have type inference,
   // ensure that the result types can be resolved.
   if (isWithinRewrite && !op.hasTypeInference()) {
-    if (failed(verifyResultTypesAreInferrable(op, opResults, resultTypes)))
+    if (failed(verifyResultTypesAreInferrable(op, op.types())))
       return failure();
   }
 
@@ -341,37 +304,39 @@ Optional<StringRef> PatternOp::getRootKind() {
 //===----------------------------------------------------------------------===//
 
 static LogicalResult verify(ReplaceOp op) {
-  auto sourceOp = cast<OperationOp>(op.operation().getDefiningOp());
-  auto sourceOpResults = sourceOp.results();
-  auto replValues = op.replValues();
-
-  if (Value replOpVal = op.replOperation()) {
-    auto replOp = cast<OperationOp>(replOpVal.getDefiningOp());
-    auto replOpResults = replOp.results();
-    if (sourceOpResults.size() != replOpResults.size()) {
-      return op.emitOpError()
-             << "expected source operation to have the same number of results "
-                "as the replacement operation, replacement operation provided "
-             << replOpResults.size() << " but expected "
-             << sourceOpResults.size();
-    }
+  if (op.replOperation() && !op.replValues().empty())
+    return op.emitOpError() << "expected no replacement values to be provided"
+                               " when the replacement operation is present";
+  return success();
+}
 
-    if (!replValues.empty()) {
-      return op.emitOpError() << "expected no replacement values to be provided"
-                                 " when the replacement operation is present";
-    }
+//===----------------------------------------------------------------------===//
+// pdl::ResultsOp
+//===----------------------------------------------------------------------===//
 
+static ParseResult parseResultsValueType(OpAsmParser &p, IntegerAttr index,
+                                         Type &resultType) {
+  if (!index) {
+    resultType = RangeType::get(p.getBuilder().getType<ValueType>());
     return success();
   }
+  if (p.parseArrow() || p.parseType(resultType))
+    return failure();
+  return success();
+}
 
-  if (sourceOpResults.size() != replValues.size()) {
-    return op.emitOpError()
-           << "expected source operation to have the same number of results "
-              "as the provided replacement values, found "
-           << replValues.size() << " replacement values but expected "
-           << sourceOpResults.size();
-  }
+static void printResultsValueType(OpAsmPrinter &p, ResultsOp op,
+                                  IntegerAttr index, Type resultType) {
+  if (index)
+    p << " -> " << resultType;
+}
 
+static LogicalResult verify(ResultsOp op) {
+  if (!op.index() && op.getType().isa<pdl::ValueType>()) {
+    return op.emitOpError() << "expected `pdl.range<value>` result type when "
+                               "no index is specified, but got: "
+                            << op.getType();
+  }
   return success();
 }
 
@@ -419,6 +384,14 @@ static LogicalResult verify(TypeOp op) {
       op, "`pdl.attribute`, `pdl.operand`, or `pdl.operation`");
 }
 
+//===----------------------------------------------------------------------===//
+// pdl::TypesOp
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(TypesOp op) {
+  return verifyHasBindingUseInMatcher(op, "`pdl.operands`, or `pdl.operation`");
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
index 8b9c27c63e8239e6d7c03127ba6dc78d34046d2f..a93f3c48503cfe4c86dbce89d96b9640a5ccdd17 100644
--- a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
+++ b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
@@ -29,28 +29,12 @@ void PDLInterpDialect::initialize() {
 // pdl_interp::CreateOperationOp
 //===----------------------------------------------------------------------===//
 
-static ParseResult parseCreateOperationOp(OpAsmParser &p,
-                                          OperationState &state) {
-  if (p.parseOptionalAttrDict(state.attributes))
-    return failure();
+static ParseResult parseCreateOperationOpAttributes(
+    OpAsmParser &p, SmallVectorImpl<OpAsmParser::OperandType> &attrOperands,
+    ArrayAttr &attrNamesAttr) {
   Builder &builder = p.getBuilder();
-
-  // Parse the operation name.
-  StringAttr opName;
-  if (p.parseAttribute(opName, "name", state.attributes))
-    return failure();
-
-  // Parse the operands.
-  SmallVector<OpAsmParser::OperandType, 4> operands;
-  if (p.parseLParen() || p.parseOperandList(operands) || p.parseRParen() ||
-      p.resolveOperands(operands, builder.getType<pdl::ValueType>(),
-                        state.operands))
-    return failure();
-
-  // Parse the attributes.
   SmallVector<Attribute, 4> attrNames;
   if (succeeded(p.parseOptionalLBrace())) {
-    SmallVector<OpAsmParser::OperandType, 4> attrOps;
     do {
       StringAttr nameAttr;
       OpAsmParser::OperandType operand;
@@ -58,60 +42,35 @@ static ParseResult parseCreateOperationOp(OpAsmParser &p,
           p.parseOperand(operand))
         return failure();
       attrNames.push_back(nameAttr);
-      attrOps.push_back(operand);
+      attrOperands.push_back(operand);
     } while (succeeded(p.parseOptionalComma()));
-
-    if (p.parseRBrace() ||
-        p.resolveOperands(attrOps, builder.getType<pdl::AttributeType>(),
-                          state.operands))
-      return failure();
-  }
-  state.addAttribute("attributeNames", builder.getArrayAttr(attrNames));
-  state.addTypes(builder.getType<pdl::OperationType>());
-
-  // Parse the result types.
-  SmallVector<OpAsmParser::OperandType, 4> opResultTypes;
-  if (p.parseArrow())
-    return failure();
-  if (succeeded(p.parseOptionalLParen())) {
-    if (p.parseRParen())
+    if (p.parseRBrace())
       return failure();
-  } else if (p.parseOperandList(opResultTypes) ||
-             p.resolveOperands(opResultTypes, builder.getType<pdl::TypeType>(),
-                               state.operands)) {
-    return failure();
   }
-
-  int32_t operandSegmentSizes[] = {static_cast<int32_t>(operands.size()),
-                                   static_cast<int32_t>(attrNames.size()),
-                                   static_cast<int32_t>(opResultTypes.size())};
-  state.addAttribute("operand_segment_sizes",
-                     builder.getI32VectorAttr(operandSegmentSizes));
+  attrNamesAttr = builder.getArrayAttr(attrNames);
   return success();
 }
 
-static void print(OpAsmPrinter &p, CreateOperationOp op) {
-  p << "pdl_interp.create_operation ";
-  p.printOptionalAttrDict(op->getAttrs(),
-                          {"attributeNames", "name", "operand_segment_sizes"});
-  p << '"' << op.name() << "\"(" << op.operands() << ')';
+static void printCreateOperationOpAttributes(OpAsmPrinter &p,
+                                             CreateOperationOp op,
+                                             OperandRange attrArgs,
+                                             ArrayAttr attrNames) {
+  if (attrNames.empty())
+    return;
+  p << " {";
+  interleaveComma(llvm::seq<int>(0, attrNames.size()), p,
+                  [&](int i) { p << attrNames[i] << " = " << attrArgs[i]; });
+  p << '}';
+}
 
-  // Emit the optional attributes.
-  ArrayAttr attrNames = op.attributeNames();
-  if (!attrNames.empty()) {
-    Operation::operand_range attrArgs = op.attributes();
-    p << " {";
-    interleaveComma(llvm::seq<int>(0, attrNames.size()), p,
-                    [&](int i) { p << attrNames[i] << " = " << attrArgs[i]; });
-    p << '}';
-  }
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetValueTypeOp
+//===----------------------------------------------------------------------===//
 
-  // Print the result type constraints of the operation.
-  auto types = op.types();
-  if (types.empty())
-    p << " -> ()";
-  else
-    p << " -> " << op.types();
+/// Given the result type of a `GetValueTypeOp`, return the expected input type.
+static Type getGetValueTypeOpValueType(Type type) {
+  Type valueTy = pdl::ValueType::get(type.getContext());
+  return type.isa<pdl::RangeType>() ? pdl::RangeType::get(valueTy) : valueTy;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp b/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
index f67020d1763de4f13d5a93d3f024297d69de0e36..6c5ef29e822838977aa054cdb9a1836780135340 100644
--- a/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
@@ -91,10 +91,10 @@ QuantizedConstRewrite::matchAndRewrite(QuantizeCastOp qbarrier,
 }
 
 void ConvertConstPass::runOnFunction() {
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(&getContext());
   auto func = getFunction();
   auto *context = &getContext();
-  patterns.insert<QuantizedConstRewrite>(context);
+  patterns.add<QuantizedConstRewrite>(context);
   (void)applyPatternsAndFoldGreedily(func, std::move(patterns));
 }
 
diff --git a/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp b/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
index daa1cda8ae99f4258e302c582dc62325ac19a6f8..c50d09a2c0653c62779595ae9ec65379b509fe67 100644
--- a/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
@@ -124,10 +124,10 @@ public:
 
 void ConvertSimulatedQuantPass::runOnFunction() {
   bool hadFailure = false;
-  OwningRewritePatternList patterns;
   auto func = getFunction();
+  RewritePatternSet patterns(func.getContext());
   auto ctx = func.getContext();
-  patterns.insert<ConstFakeQuantRewrite, ConstFakeQuantPerAxisRewrite>(
+  patterns.add<ConstFakeQuantRewrite, ConstFakeQuantPerAxisRewrite>(
       ctx, &hadFailure);
   (void)applyPatternsAndFoldGreedily(func, std::move(patterns));
   if (hadFailure)
diff --git a/mlir/lib/Dialect/SCF/SCF.cpp b/mlir/lib/Dialect/SCF/SCF.cpp
index 9c0df1b47c355a023f63aa3d45a1f4721723df9c..2d1ad054bf052738f80e95ceeffe46cb6c4d3a32 100644
--- a/mlir/lib/Dialect/SCF/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/SCF.cpp
@@ -254,7 +254,7 @@ ForOp mlir::scf::getForInductionVarOwner(Value val) {
 }
 
 /// Return operands used when entering the region at 'index'. These operands
-/// correspond to the loop iterator operands, i.e., those exclusing the
+/// correspond to the loop iterator operands, i.e., those excluding the
 /// induction variable. LoopOp only has one region, so 0 is the only valid value
 /// for `index`.
 OperandRange ForOp::getSuccessorEntryOperands(unsigned index) {
@@ -408,9 +408,16 @@ static void replaceOpWithRegion(PatternRewriter &rewriter, Operation *op,
 }
 
 namespace {
-// Fold away ForOp iter arguments that are also yielded by the op.
-// These arguments must be defined outside of the ForOp region and can just be
-// forwarded after simplifying the op inits, yields and returns.
+// Fold away ForOp iter arguments when:
+// 1) The op yields the iter arguments.
+// 2) The iter arguments have no use and the corresponding outer region
+// iterators (inputs) are yielded.
+// 3) The iter arguments have no use and the corresponding (operation) results
+// have no use.
+//
+// These arguments must be defined outside of
+// the ForOp region and can just be forwarded after simplifying the op inits,
+// yields and returns.
 //
 // The implementation uses `mergeBlockBefore` to steal the content of the
 // original ForOp and avoid cloning.
@@ -439,10 +446,19 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
     newResultValues.reserve(forOp.getNumResults());
     for (auto it : llvm::zip(forOp.getIterOperands(),   // iter from outside
                              forOp.getRegionIterArgs(), // iter inside region
+                             forOp.getResults(),        // op results
                              yieldOp.getOperands()      // iter yield
                              )) {
-      // Forwarded is `true` when the region `iter` argument is yielded.
-      bool forwarded = (std::get<1>(it) == std::get<2>(it));
+      // Forwarded is `true` when:
+      // 1) The region `iter` argument is yielded.
+      // 2) The region `iter` argument has no use, and the corresponding iter
+      // operand (input) is yielded.
+      // 3) The region `iter` argument has no use, and the corresponding op
+      // result has no use.
+      bool forwarded = ((std::get<1>(it) == std::get<3>(it)) ||
+                        (std::get<1>(it).use_empty() &&
+                         (std::get<0>(it) == std::get<3>(it) ||
+                          std::get<2>(it).use_empty())));
       keepMask.push_back(!forwarded);
       canonicalize |= forwarded;
       if (forwarded) {
@@ -451,7 +467,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
         continue;
       }
       newIterArgs.push_back(std::get<0>(it));
-      newYieldValues.push_back(std::get<2>(it));
+      newYieldValues.push_back(std::get<3>(it));
       newBlockTransferArgs.push_back(Value()); // placeholder with null value
       newResultValues.push_back(Value());      // placeholder with null value
     }
@@ -483,7 +499,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
            "unexpected argument size mismatch");
 
     // No results case: the scf::ForOp builder already created a zero
-    // reult terminator. Merge before this terminator and just get rid of the
+    // result terminator. Merge before this terminator and just get rid of the
     // original terminator that has been merged in.
     if (newIterArgs.empty()) {
       auto newYieldOp = cast<scf::YieldOp>(newBlock.getTerminator());
@@ -687,10 +703,10 @@ struct LastTensorLoadCanonicalization : public OpRewritePattern<ForOp> {
 };
 } // namespace
 
-void ForOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ForOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                         MLIRContext *context) {
-  results.insert<ForOpIterArgsFolder, SimplifyTrivialLoops,
-                 LastTensorLoadCanonicalization>(context);
+  results.add<ForOpIterArgsFolder, SimplifyTrivialLoops,
+              LastTensorLoadCanonicalization>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -918,11 +934,49 @@ struct RemoveStaticCondition : public OpRewritePattern<IfOp> {
     return success();
   }
 };
+
+struct ConvertTrivialIfToSelect : public OpRewritePattern<IfOp> {
+  using OpRewritePattern<IfOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(IfOp op,
+                                PatternRewriter &rewriter) const override {
+    if (op->getNumResults() == 0)
+      return failure();
+
+    if (!llvm::hasSingleElement(op.thenRegion().front()) ||
+        !llvm::hasSingleElement(op.elseRegion().front()))
+      return failure();
+
+    auto cond = op.condition();
+    auto thenYieldArgs =
+        cast<scf::YieldOp>(op.thenRegion().front().getTerminator())
+            .getOperands();
+    auto elseYieldArgs =
+        cast<scf::YieldOp>(op.elseRegion().front().getTerminator())
+            .getOperands();
+    SmallVector<Value> results(op->getNumResults());
+    assert(thenYieldArgs.size() == results.size());
+    assert(elseYieldArgs.size() == results.size());
+    for (auto it : llvm::enumerate(llvm::zip(thenYieldArgs, elseYieldArgs))) {
+      Value trueVal = std::get<0>(it.value());
+      Value falseVal = std::get<1>(it.value());
+      if (trueVal == falseVal)
+        results[it.index()] = trueVal;
+      else
+        results[it.index()] =
+            rewriter.create<SelectOp>(op.getLoc(), cond, trueVal, falseVal);
+    }
+
+    rewriter.replaceOp(op, results);
+    return success();
+  }
+};
 } // namespace
 
-void IfOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void IfOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                        MLIRContext *context) {
-  results.insert<RemoveUnusedResults, RemoveStaticCondition>(context);
+  results.add<RemoveUnusedResults, RemoveStaticCondition,
+              ConvertTrivialIfToSelect>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1221,10 +1275,9 @@ struct RemoveEmptyParallelLoops : public OpRewritePattern<ParallelOp> {
 
 } // namespace
 
-void ParallelOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ParallelOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                              MLIRContext *context) {
-  results.insert<CollapseSingleIterationLoops, RemoveEmptyParallelLoops>(
-      context);
+  results.add<CollapseSingleIterationLoops, RemoveEmptyParallelLoops>(context);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp b/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp
index aa25f47e48017ca9d6d1aa99f037e219d567e5d5..9ee80758924260ab34769e02a874054bf2c039f6 100644
--- a/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp
@@ -25,12 +25,12 @@ struct SCFBufferizePass : public SCFBufferizeBase<SCFBufferizePass> {
     auto *context = &getContext();
 
     BufferizeTypeConverter typeConverter;
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
     ConversionTarget target(*context);
 
     populateBufferizeMaterializationLegality(target);
-    populateSCFStructuralTypeConversionsAndLegality(context, typeConverter,
-                                                    patterns, target);
+    populateSCFStructuralTypeConversionsAndLegality(typeConverter, patterns,
+                                                    target);
     if (failed(applyPartialConversion(func, target, std::move(patterns))))
       return signalPassFailure();
   };
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 9197375cc237b6ce57b95779a851a0357f6e6e08..107c32779e926447eab9f048babc0c179fb19e7f 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -134,10 +134,10 @@ public:
 } // namespace
 
 void mlir::scf::populateSCFStructuralTypeConversionsAndLegality(
-    MLIRContext *context, TypeConverter &typeConverter,
-    OwningRewritePatternList &patterns, ConversionTarget &target) {
-  patterns.insert<ConvertForOpTypes, ConvertIfOpTypes, ConvertYieldOpTypes>(
-      typeConverter, context);
+    TypeConverter &typeConverter, RewritePatternSet &patterns,
+    ConversionTarget &target) {
+  patterns.add<ConvertForOpTypes, ConvertIfOpTypes, ConvertYieldOpTypes>(
+      typeConverter, patterns.getContext());
   target.addDynamicallyLegalOp<ForOp, IfOp>([&](Operation *op) {
     return typeConverter.isLegal(op->getResultTypes());
   });
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index b791695c1dd9e7f569ef11d8cbe0c5594d245b89..437c762b5f550ad87b81f049e702e026886a0b82 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -111,17 +111,17 @@ struct CombineChainedAccessChain
 } // end anonymous namespace
 
 void spirv::AccessChainOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<CombineChainedAccessChain>(context);
+    RewritePatternSet &results, MLIRContext *context) {
+  results.add<CombineChainedAccessChain>(context);
 }
 
 //===----------------------------------------------------------------------===//
 // spv.BitcastOp
 //===----------------------------------------------------------------------===//
 
-void spirv::BitcastOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<ConvertChainedBitcast>(context);
+void spirv::BitcastOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                   MLIRContext *context) {
+  results.add<ConvertChainedBitcast>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -230,10 +230,11 @@ OpFoldResult spirv::LogicalAndOp::fold(ArrayRef<Attribute> operands) {
 //===----------------------------------------------------------------------===//
 
 void spirv::LogicalNotOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<ConvertLogicalNotOfIEqual, ConvertLogicalNotOfINotEqual,
-                 ConvertLogicalNotOfLogicalEqual,
-                 ConvertLogicalNotOfLogicalNotEqual>(context);
+    RewritePatternSet &results, MLIRContext *context) {
+  results
+      .add<ConvertLogicalNotOfIEqual, ConvertLogicalNotOfINotEqual,
+           ConvertLogicalNotOfLogicalEqual, ConvertLogicalNotOfLogicalNotEqual>(
+          context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -415,7 +416,7 @@ LogicalResult ConvertSelectionOpToSelect::canCanonicalizeSelection(
 }
 } // end anonymous namespace
 
-void spirv::SelectionOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<ConvertSelectionOpToSelect>(context);
+void spirv::SelectionOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                     MLIRContext *context) {
+  results.add<ConvertSelectionOpToSelect>(context);
 }
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVEnums.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVEnums.cpp
index a289d9d2801b58991e1fe86a99a9eb9aa3caf3a8..a8519063a8186e2a213a1218590f3c41b78d32b4 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVEnums.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVEnums.cpp
@@ -12,6 +12,8 @@
 
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
 
+#include "mlir/IR/BuiltinTypes.h"
+
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.cpp
index 0aa413941efd2082fb29bb10aa265ba255390347..9c72dbb6f7aa3ff5633eb14d8988160320963922 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVGLSLCanonicalization.cpp
@@ -22,14 +22,14 @@ namespace {
 
 namespace mlir {
 namespace spirv {
-void populateSPIRVGLSLCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<ConvertComparisonIntoClampSPV_FOrdLessThanOp,
-                 ConvertComparisonIntoClampSPV_FOrdLessThanEqualOp,
-                 ConvertComparisonIntoClampSPV_SLessThanOp,
-                 ConvertComparisonIntoClampSPV_SLessThanEqualOp,
-                 ConvertComparisonIntoClampSPV_ULessThanOp,
-                 ConvertComparisonIntoClampSPV_ULessThanEqualOp>(context);
+void populateSPIRVGLSLCanonicalizationPatterns(RewritePatternSet &results) {
+  results.add<ConvertComparisonIntoClampSPV_FOrdLessThanOp,
+              ConvertComparisonIntoClampSPV_FOrdLessThanEqualOp,
+              ConvertComparisonIntoClampSPV_SLessThanOp,
+              ConvertComparisonIntoClampSPV_SLessThanEqualOp,
+              ConvertComparisonIntoClampSPV_ULessThanOp,
+              ConvertComparisonIntoClampSPV_ULessThanEqualOp>(
+      results.getContext());
 }
 } // namespace spirv
 } // namespace mlir
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index dd2dc3d7f1de291fa315859ee13bf02c2ccc793f..21bcfe458ab7fab169c6d6efb055453fb4401690 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -1659,7 +1659,7 @@ void spirv::EntryPointOp::build(OpBuilder &builder, OperationState &state,
                                 spirv::FuncOp function,
                                 ArrayRef<Attribute> interfaceVars) {
   build(builder, state,
-        builder.getI32IntegerAttr(static_cast<int32_t>(executionModel)),
+        spirv::ExecutionModelAttr::get(builder.getContext(), executionModel),
         builder.getSymbolRefAttr(function),
         builder.getArrayAttr(interfaceVars));
 }
@@ -1721,7 +1721,7 @@ void spirv::ExecutionModeOp::build(OpBuilder &builder, OperationState &state,
                                    spirv::ExecutionMode executionMode,
                                    ArrayRef<int32_t> params) {
   build(builder, state, builder.getSymbolRefAttr(function),
-        builder.getI32IntegerAttr(static_cast<int32_t>(executionMode)),
+        spirv::ExecutionModeAttr::get(builder.getContext(), executionMode),
         builder.getI32ArrayAttr(params));
 }
 
@@ -2243,10 +2243,10 @@ static LogicalResult verify(spirv::GroupNonUniformElectOp groupOp) {
 //===----------------------------------------------------------------------===//
 
 void spirv::LoadOp::build(OpBuilder &builder, OperationState &state,
-                          Value basePtr, IntegerAttr memory_access,
+                          Value basePtr, MemoryAccessAttr memoryAccess,
                           IntegerAttr alignment) {
   auto ptrType = basePtr.getType().cast<spirv::PointerType>();
-  build(builder, state, ptrType.getPointeeType(), basePtr, memory_access,
+  build(builder, state, ptrType.getPointeeType(), basePtr, memoryAccess,
         alignment);
 }
 
@@ -2784,9 +2784,8 @@ void spirv::SelectionOp::addMergeBlock() {
 spirv::SelectionOp spirv::SelectionOp::createIfThen(
     Location loc, Value condition,
     function_ref<void(OpBuilder &builder)> thenBody, OpBuilder &builder) {
-  auto selectionControl = builder.getI32IntegerAttr(
-      static_cast<uint32_t>(spirv::SelectionControl::None));
-  auto selectionOp = builder.create<spirv::SelectionOp>(loc, selectionControl);
+  auto selectionOp =
+      builder.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);
 
   selectionOp.addMergeBlock();
   Block *mergeBlock = selectionOp.getMergeBlock();
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/DecorateCompositeTypeLayoutPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/DecorateCompositeTypeLayoutPass.cpp
index c4954ca34b06bc82bad9d838702f1c36f975f0ba..372295a986afcacda4db9cd6c3d28885595c3430 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/DecorateCompositeTypeLayoutPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/DecorateCompositeTypeLayoutPass.cpp
@@ -74,10 +74,9 @@ public:
 };
 } // namespace
 
-static void populateSPIRVLayoutInfoPatterns(OwningRewritePatternList &patterns,
-                                            MLIRContext *ctx) {
-  patterns.insert<SPIRVGlobalVariableOpLayoutInfoDecoration,
-                  SPIRVAddressOfOpLayoutInfoDecoration>(ctx);
+static void populateSPIRVLayoutInfoPatterns(RewritePatternSet &patterns) {
+  patterns.add<SPIRVGlobalVariableOpLayoutInfoDecoration,
+               SPIRVAddressOfOpLayoutInfoDecoration>(patterns.getContext());
 }
 
 namespace {
@@ -90,8 +89,8 @@ class DecorateSPIRVCompositeTypeLayoutPass
 
 void DecorateSPIRVCompositeTypeLayoutPass::runOnOperation() {
   auto module = getOperation();
-  OwningRewritePatternList patterns;
-  populateSPIRVLayoutInfoPatterns(patterns, module.getContext());
+  RewritePatternSet patterns(module.getContext());
+  populateSPIRVLayoutInfoPatterns(patterns);
   ConversionTarget target(*(module.getContext()));
   target.addLegalDialect<spirv::SPIRVDialect>();
   target.addLegalOp<FuncOp>();
@@ -107,7 +106,7 @@ void DecorateSPIRVCompositeTypeLayoutPass::runOnOperation() {
 
   // TODO: Change the type for the indirect users such as spv.Load, spv.Store,
   // spv.FunctionCall and so on.
-  FrozenRewritePatternList frozenPatterns(std::move(patterns));
+  FrozenRewritePatternSet frozenPatterns(std::move(patterns));
   for (auto spirvModule : module.getOps<spirv::ModuleOp>())
     if (failed(applyFullConversion(spirvModule, target, frozenPatterns)))
       signalPassFailure();
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index d96892b480b2b4c5ea23450bde9fd60cab5a0b0f..cd2c9c52eeae8619e2644f64c0d9d396d5b9f283 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -246,8 +246,8 @@ void LowerABIAttributesPass::runOnOperation() {
     return builder.create<spirv::BitcastOp>(loc, type, inputs[0]).getResult();
   });
 
-  OwningRewritePatternList patterns;
-  patterns.insert<ProcessInterfaceVarABI>(typeConverter, context);
+  RewritePatternSet patterns(context);
+  patterns.add<ProcessInterfaceVarABI>(typeConverter, context);
 
   ConversionTarget target(*context);
   // "Legal" function ops should have no interface variable ABI attributes.
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index c544512950f05d969b3142dc14592d47c82a444e..7539e9a050765152707cd2175d3a3b7d5e4454d2 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -514,10 +514,9 @@ FuncOpConversion::matchAndRewrite(FuncOp funcOp, ArrayRef<Value> operands,
   return success();
 }
 
-void mlir::populateBuiltinFuncToSPIRVPatterns(
-    MLIRContext *context, SPIRVTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns) {
-  patterns.insert<FuncOpConversion>(typeConverter, context);
+void mlir::populateBuiltinFuncToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                              RewritePatternSet &patterns) {
+  patterns.add<FuncOpConversion>(typeConverter, patterns.getContext());
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index ed8dcfc135498df4238a1f9e335fcea8fa3a38d9..a1419322afb3a853e6eed95b7cfa062f103f35a8 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -270,10 +270,10 @@ struct AssumingWithTrue : public OpRewritePattern<AssumingOp> {
 };
 } // namespace
 
-void AssumingOp::getCanonicalizationPatterns(OwningRewritePatternList &patterns,
+void AssumingOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
                                              MLIRContext *context) {
   // If taking a passing witness, inline region.
-  patterns.insert<AssumingWithTrue>(context);
+  patterns.add<AssumingWithTrue>(context);
 }
 
 // See RegionBranchOpInterface in Interfaces/ControlFlowInterfaces.td
@@ -311,13 +311,34 @@ void AssumingOp::inlineRegionIntoParent(AssumingOp &op,
   rewriter.mergeBlocks(blockAfterAssuming, blockBeforeAssuming);
 }
 
+void AssumingOp::build(
+    OpBuilder &builder, OperationState &result, Value witness,
+    function_ref<SmallVector<Value, 2>(OpBuilder &, Location)> bodyBuilder) {
+
+  result.addOperands(witness);
+  Region *bodyRegion = result.addRegion();
+  bodyRegion->push_back(new Block);
+  Block &bodyBlock = bodyRegion->front();
+
+  // Build body.
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointToStart(&bodyBlock);
+  SmallVector<Value, 2> yieldValues = bodyBuilder(builder, result.location);
+  builder.create<AssumingYieldOp>(result.location, yieldValues);
+
+  SmallVector<Type, 2> assumingTypes;
+  for (Value v : yieldValues)
+    assumingTypes.push_back(v.getType());
+  result.addTypes(assumingTypes);
+}
+
 //===----------------------------------------------------------------------===//
 // AssumingAllOp
 //===----------------------------------------------------------------------===//
 
-void AssumingAllOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<AssumingAllOneOp>(context);
+void AssumingAllOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
+                                                MLIRContext *context) {
+  patterns.add<AssumingAllOneOp>(context);
 }
 
 OpFoldResult AssumingAllOp::fold(ArrayRef<Attribute> operands) {
@@ -414,11 +435,26 @@ struct RemoveDuplicateOperandsPattern : public OpRewritePattern<OpTy> {
     return failure();
   }
 };
+
+struct BroadcastForwardSingleOperandPattern
+    : public OpRewritePattern<BroadcastOp> {
+  using OpRewritePattern<BroadcastOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(BroadcastOp op,
+                                PatternRewriter &rewriter) const override {
+    if (op.getNumOperands() == 1) {
+      rewriter.replaceOp(op, op.shapes().front());
+      return success();
+    }
+    return failure();
+  }
+};
 } // namespace
 
-void BroadcastOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<RemoveDuplicateOperandsPattern<BroadcastOp>>(context);
+void BroadcastOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
+                                              MLIRContext *context) {
+  patterns.add<BroadcastForwardSingleOperandPattern,
+               RemoveDuplicateOperandsPattern<BroadcastOp>>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -485,9 +521,9 @@ static ParseResult parseConstShapeOp(OpAsmParser &parser,
 
 OpFoldResult ConstShapeOp::fold(ArrayRef<Attribute>) { return shapeAttr(); }
 
-void ConstShapeOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<TensorCastConstShape>(context);
+void ConstShapeOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
+                                               MLIRContext *context) {
+  patterns.add<TensorCastConstShape>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -513,11 +549,12 @@ LogicalResult getShapeVec(Value input, SmallVectorImpl<int64_t> &shapeValues) {
 } // namespace
 
 void CstrBroadcastableOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
+    RewritePatternSet &patterns, MLIRContext *context) {
   // Canonicalization patterns have overlap with the considerations during
   // folding in case additional shape information is inferred at some point that
   // does not result in folding.
-  patterns.insert<CstrBroadcastableEqOps>(context);
+  patterns.add<CstrBroadcastableEqOps,
+               RemoveDuplicateOperandsPattern<CstrBroadcastableOp>>(context);
 }
 
 // Return true if there is exactly one attribute not representing a scalar
@@ -555,9 +592,9 @@ OpFoldResult CstrBroadcastableOp::fold(ArrayRef<Attribute> operands) {
   // on the input shapes.
   if ([&] {
         SmallVector<SmallVector<int64_t, 6>, 6> extents;
-        for (const auto &shape : shapes()) {
+        for (auto shapeValue : shapes()) {
           extents.emplace_back();
-          if (failed(getShapeVec(shape, extents.back())))
+          if (failed(getShapeVec(shapeValue, extents.back())))
             return false;
         }
         return OpTrait::util::staticallyKnownBroadcastable(extents);
@@ -580,10 +617,10 @@ static LogicalResult verify(CstrBroadcastableOp op) {
 // CstrEqOp
 //===----------------------------------------------------------------------===//
 
-void CstrEqOp::getCanonicalizationPatterns(OwningRewritePatternList &patterns,
+void CstrEqOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
                                            MLIRContext *context) {
   // If inputs are equal, return passing witness
-  patterns.insert<CstrEqEqOps>(context);
+  patterns.add<CstrEqEqOps>(context);
 }
 
 OpFoldResult CstrEqOp::fold(ArrayRef<Attribute> operands) {
@@ -682,9 +719,9 @@ OpFoldResult IndexToSizeOp::fold(ArrayRef<Attribute> operands) {
   return {};
 }
 
-void IndexToSizeOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<SizeToIndexToSizeCanonicalization>(context);
+void IndexToSizeOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
+                                                MLIRContext *context) {
+  patterns.add<SizeToIndexToSizeCanonicalization>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -707,7 +744,6 @@ OpFoldResult FromExtentsOp::fold(ArrayRef<Attribute> operands) {
 
 void FunctionLibraryOp::build(OpBuilder &builder, OperationState &result,
                               StringRef name) {
-  ensureTerminator(*result.addRegion(), builder, result.location);
   result.attributes.push_back(builder.getNamedAttr(
       ::mlir::SymbolTable::getSymbolAttrName(), builder.getStringAttr(name)));
 }
@@ -736,8 +772,6 @@ ParseResult parseFunctionLibraryOp(OpAsmParser &parser,
   if (parser.parseRegion(*bodyRegion))
     return failure();
 
-  FunctionLibraryOp::ensureTerminator(*bodyRegion, parser.getBuilder(),
-                                      result.location);
   if (parser.parseKeyword("mapping"))
     return failure();
 
@@ -802,9 +836,9 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
 // IsBroadcastableOp
 //===----------------------------------------------------------------------===//
 
-void IsBroadcastableOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<RemoveDuplicateOperandsPattern<IsBroadcastableOp>>(context);
+void IsBroadcastableOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
+                                                    MLIRContext *context) {
+  patterns.add<RemoveDuplicateOperandsPattern<IsBroadcastableOp>>(context);
 }
 
 OpFoldResult IsBroadcastableOp::fold(ArrayRef<Attribute> operands) {
@@ -870,9 +904,9 @@ struct RankShapeOfCanonicalizationPattern
 };
 } // namespace
 
-void shape::RankOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<RankShapeOfCanonicalizationPattern>(context);
+void shape::RankOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
+                                                MLIRContext *context) {
+  patterns.add<RankShapeOfCanonicalizationPattern>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -955,9 +989,9 @@ struct ShapeOfWithTensor : public OpRewritePattern<shape::ShapeOfOp> {
 };
 } // namespace
 
-void ShapeOfOp::getCanonicalizationPatterns(OwningRewritePatternList &patterns,
+void ShapeOfOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
                                             MLIRContext *context) {
-  patterns.insert<ShapeOfWithTensor>(context);
+  patterns.add<ShapeOfWithTensor>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -972,9 +1006,9 @@ OpFoldResult SizeToIndexOp::fold(ArrayRef<Attribute> operands) {
   return impl::foldCastOp(*this);
 }
 
-void SizeToIndexOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<IndexToSizeToIndexCanonicalization>(context);
+void SizeToIndexOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
+                                                MLIRContext *context) {
+  patterns.add<IndexToSizeToIndexCanonicalization>(context);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Shape/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Shape/Transforms/Bufferize.cpp
index 36b5eac501c3d456d739d22f4f78e3731fc59d03..ea2f97d7d0569c5ccf5ff88b6e3adb2a64c5b063 100644
--- a/mlir/lib/Dialect/Shape/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/Bufferize.cpp
@@ -19,13 +19,13 @@ struct ShapeBufferizePass : public ShapeBufferizeBase<ShapeBufferizePass> {
   void runOnFunction() override {
     MLIRContext &ctx = getContext();
 
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(&ctx);
     BufferizeTypeConverter typeConverter;
-    ConversionTarget target(getContext());
+    ConversionTarget target(ctx);
 
     populateBufferizeMaterializationLegality(target);
-    populateShapeStructuralTypeConversionsAndLegality(&ctx, typeConverter,
-                                                      patterns, target);
+    populateShapeStructuralTypeConversionsAndLegality(typeConverter, patterns,
+                                                      target);
 
     if (failed(
             applyPartialConversion(getFunction(), target, std::move(patterns))))
diff --git a/mlir/lib/Dialect/Shape/Transforms/RemoveShapeConstraints.cpp b/mlir/lib/Dialect/Shape/Transforms/RemoveShapeConstraints.cpp
index 492abce92577bea54d4585e162decc0529227ef3..dc403b0ceacd8a3e608bddee3ab9f31df840ecec 100644
--- a/mlir/lib/Dialect/Shape/Transforms/RemoveShapeConstraints.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/RemoveShapeConstraints.cpp
@@ -46,8 +46,8 @@ class RemoveShapeConstraintsPass
   void runOnFunction() override {
     MLIRContext &ctx = getContext();
 
-    OwningRewritePatternList patterns;
-    populateRemoveShapeConstraintsPatterns(patterns, &ctx);
+    RewritePatternSet patterns(&ctx);
+    populateRemoveShapeConstraintsPatterns(patterns);
 
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
@@ -55,9 +55,9 @@ class RemoveShapeConstraintsPass
 
 } // namespace
 
-void mlir::populateRemoveShapeConstraintsPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *ctx) {
-  patterns.insert<RemoveCstrBroadcastableOp, RemoveCstrEqOp>(ctx);
+void mlir::populateRemoveShapeConstraintsPatterns(RewritePatternSet &patterns) {
+  patterns.add<RemoveCstrBroadcastableOp, RemoveCstrEqOp>(
+      patterns.getContext());
 }
 
 std::unique_ptr<FunctionPass> mlir::createRemoveShapeConstraintsPass() {
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index 6190ff351ebf48e5719ac5233a5661951ab45c8f..66c4e5048f105bc9c3b751e39c503e39d052a083 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -61,8 +61,8 @@ struct ShapeToShapeLowering
 void ShapeToShapeLowering::runOnFunction() {
   MLIRContext &ctx = getContext();
 
-  OwningRewritePatternList patterns;
-  populateShapeRewritePatterns(&ctx, patterns);
+  RewritePatternSet patterns(&ctx);
+  populateShapeRewritePatterns(patterns);
 
   ConversionTarget target(getContext());
   target.addLegalDialect<ShapeDialect, StandardOpsDialect>();
@@ -72,9 +72,8 @@ void ShapeToShapeLowering::runOnFunction() {
     signalPassFailure();
 }
 
-void mlir::populateShapeRewritePatterns(MLIRContext *context,
-                                        OwningRewritePatternList &patterns) {
-  patterns.insert<NumElementsOpConverter>(context);
+void mlir::populateShapeRewritePatterns(RewritePatternSet &patterns) {
+  patterns.add<NumElementsOpConverter>(patterns.getContext());
 }
 
 std::unique_ptr<Pass> mlir::createShapeToShapeLowering() {
diff --git a/mlir/lib/Dialect/Shape/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/Shape/Transforms/StructuralTypeConversions.cpp
index 041b54b3bd1445490cc589b447feb2b27cbef03f..b58fa4dfdc87d8d66dd03ef0723e90e82513f62b 100644
--- a/mlir/lib/Dialect/Shape/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/StructuralTypeConversions.cpp
@@ -57,10 +57,10 @@ public:
 } // namespace
 
 void mlir::populateShapeStructuralTypeConversionsAndLegality(
-    MLIRContext *context, TypeConverter &typeConverter,
-    OwningRewritePatternList &patterns, ConversionTarget &target) {
-  patterns.insert<ConvertAssumingOpTypes, ConvertAssumingYieldOpTypes>(
-      typeConverter, context);
+    TypeConverter &typeConverter, RewritePatternSet &patterns,
+    ConversionTarget &target) {
+  patterns.add<ConvertAssumingOpTypes, ConvertAssumingYieldOpTypes>(
+      typeConverter, patterns.getContext());
   target.addDynamicallyLegalOp<AssumingOp>([&](AssumingOp op) {
     return typeConverter.isLegal(op.getResultTypes());
   });
diff --git a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
index bd38e154bcf6f7d49d61efdcc4b0125584a60ab0..3dad958887b52f80edf5a08eee88aa3156445d30 100644
--- a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
+++ b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
@@ -308,25 +308,13 @@ OpFoldResult AndOp::fold(ArrayRef<Attribute> operands) {
 // AssertOp
 //===----------------------------------------------------------------------===//
 
-namespace {
-struct EraseRedundantAssertions : public OpRewritePattern<AssertOp> {
-  using OpRewritePattern<AssertOp>::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(AssertOp op,
-                                PatternRewriter &rewriter) const override {
-    // Erase assertion if argument is constant true.
-    if (matchPattern(op.arg(), m_One())) {
-      rewriter.eraseOp(op);
-      return success();
-    }
-    return failure();
+LogicalResult AssertOp::canonicalize(AssertOp op, PatternRewriter &rewriter) {
+  // Erase assertion if argument is constant true.
+  if (matchPattern(op.arg(), m_One())) {
+    rewriter.eraseOp(op);
+    return success();
   }
-};
-} // namespace
-
-void AssertOp::getCanonicalizationPatterns(OwningRewritePatternList &patterns,
-                                           MLIRContext *context) {
-  patterns.insert<EraseRedundantAssertions>(context);
+  return failure();
 }
 
 //===----------------------------------------------------------------------===//
@@ -498,26 +486,21 @@ static LogicalResult collapseBranch(Block *&successor,
   return success();
 }
 
-namespace {
 /// Simplify a branch to a block that has a single predecessor. This effectively
 /// merges the two blocks.
-struct SimplifyBrToBlockWithSinglePred : public OpRewritePattern<BranchOp> {
-  using OpRewritePattern<BranchOp>::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(BranchOp op,
-                                PatternRewriter &rewriter) const override {
-    // Check that the successor block has a single predecessor.
-    Block *succ = op.getDest();
-    Block *opParent = op->getBlock();
-    if (succ == opParent || !llvm::hasSingleElement(succ->getPredecessors()))
-      return failure();
+static LogicalResult
+simplifyBrToBlockWithSinglePred(BranchOp op, PatternRewriter &rewriter) {
+  // Check that the successor block has a single predecessor.
+  Block *succ = op.getDest();
+  Block *opParent = op->getBlock();
+  if (succ == opParent || !llvm::hasSingleElement(succ->getPredecessors()))
+    return failure();
 
-    // Merge the successor into the current block and erase the branch.
-    rewriter.mergeBlocks(succ, opParent, op.getOperands());
-    rewriter.eraseOp(op);
-    return success();
-  }
-};
+  // Merge the successor into the current block and erase the branch.
+  rewriter.mergeBlocks(succ, opParent, op.getOperands());
+  rewriter.eraseOp(op);
+  return success();
+}
 
 ///   br ^bb1
 /// ^bb1
@@ -525,27 +508,27 @@ struct SimplifyBrToBlockWithSinglePred : public OpRewritePattern<BranchOp> {
 ///
 ///  -> br ^bbN(...)
 ///
-struct SimplifyPassThroughBr : public OpRewritePattern<BranchOp> {
-  using OpRewritePattern<BranchOp>::OpRewritePattern;
+static LogicalResult simplifyPassThroughBr(BranchOp op,
+                                           PatternRewriter &rewriter) {
+  Block *dest = op.getDest();
+  ValueRange destOperands = op.getOperands();
+  SmallVector<Value, 4> destOperandStorage;
+
+  // Try to collapse the successor if it points somewhere other than this
+  // block.
+  if (dest == op->getBlock() ||
+      failed(collapseBranch(dest, destOperands, destOperandStorage)))
+    return failure();
 
-  LogicalResult matchAndRewrite(BranchOp op,
-                                PatternRewriter &rewriter) const override {
-    Block *dest = op.getDest();
-    ValueRange destOperands = op.getOperands();
-    SmallVector<Value, 4> destOperandStorage;
-
-    // Try to collapse the successor if it points somewhere other than this
-    // block.
-    if (dest == op->getBlock() ||
-        failed(collapseBranch(dest, destOperands, destOperandStorage)))
-      return failure();
+  // Create a new branch with the collapsed successor.
+  rewriter.replaceOpWithNewOp<BranchOp>(op, dest, destOperands);
+  return success();
+}
 
-    // Create a new branch with the collapsed successor.
-    rewriter.replaceOpWithNewOp<BranchOp>(op, dest, destOperands);
-    return success();
-  }
-};
-} // end anonymous namespace.
+LogicalResult BranchOp::canonicalize(BranchOp op, PatternRewriter &rewriter) {
+  return success(succeeded(simplifyBrToBlockWithSinglePred(op, rewriter)) ||
+                 succeeded(simplifyPassThroughBr(op, rewriter)));
+}
 
 Block *BranchOp::getDest() { return getSuccessor(); }
 
@@ -553,12 +536,6 @@ void BranchOp::setDest(Block *block) { return setSuccessor(block); }
 
 void BranchOp::eraseOperand(unsigned index) { (*this)->eraseOperand(index); }
 
-void BranchOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
-                                           MLIRContext *context) {
-  results.insert<SimplifyBrToBlockWithSinglePred, SimplifyPassThroughBr>(
-      context);
-}
-
 Optional<MutableOperandRange>
 BranchOp::getMutableSuccessorOperands(unsigned index) {
   assert(index == 0 && "invalid successor index");
@@ -609,31 +586,20 @@ FunctionType CallOp::getCalleeType() {
 //===----------------------------------------------------------------------===//
 // CallIndirectOp
 //===----------------------------------------------------------------------===//
-namespace {
-/// Fold indirect calls that have a constant function as the callee operand.
-struct SimplifyIndirectCallWithKnownCallee
-    : public OpRewritePattern<CallIndirectOp> {
-  using OpRewritePattern<CallIndirectOp>::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(CallIndirectOp indirectCall,
-                                PatternRewriter &rewriter) const override {
-    // Check that the callee is a constant callee.
-    SymbolRefAttr calledFn;
-    if (!matchPattern(indirectCall.getCallee(), m_Constant(&calledFn)))
-      return failure();
 
-    // Replace with a direct call.
-    rewriter.replaceOpWithNewOp<CallOp>(indirectCall, calledFn,
-                                        indirectCall.getResultTypes(),
-                                        indirectCall.getArgOperands());
-    return success();
-  }
-};
-} // end anonymous namespace.
+/// Fold indirect calls that have a constant function as the callee operand.
+LogicalResult CallIndirectOp::canonicalize(CallIndirectOp indirectCall,
+                                           PatternRewriter &rewriter) {
+  // Check that the callee is a constant callee.
+  SymbolRefAttr calledFn;
+  if (!matchPattern(indirectCall.getCallee(), m_Constant(&calledFn)))
+    return failure();
 
-void CallIndirectOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<SimplifyIndirectCallWithKnownCallee>(context);
+  // Replace with a direct call.
+  rewriter.replaceOpWithNewOp<CallOp>(indirectCall, calledFn,
+                                      indirectCall.getResultTypes(),
+                                      indirectCall.getArgOperands());
+  return success();
 }
 
 //===----------------------------------------------------------------------===//
@@ -965,11 +931,11 @@ struct SimplifyCondBranchFromCondBranchOnSameCondition
 };
 } // end anonymous namespace
 
-void CondBranchOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<SimplifyConstCondBranchPred, SimplifyPassThroughCondBranch,
-                 SimplifyCondBranchIdenticalSuccessors,
-                 SimplifyCondBranchFromCondBranchOnSameCondition>(context);
+void CondBranchOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                               MLIRContext *context) {
+  results.add<SimplifyConstCondBranchPred, SimplifyPassThroughCondBranch,
+              SimplifyCondBranchIdenticalSuccessors,
+              SimplifyCondBranchFromCondBranchOnSameCondition>(context);
 }
 
 Optional<MutableOperandRange>
@@ -1360,15 +1326,38 @@ static LogicalResult verify(ReturnOp op) {
 //===----------------------------------------------------------------------===//
 
 OpFoldResult SelectOp::fold(ArrayRef<Attribute> operands) {
+  auto trueVal = getTrueValue();
+  auto falseVal = getFalseValue();
+  if (trueVal == falseVal)
+    return trueVal;
+
   auto condition = getCondition();
 
   // select true, %0, %1 => %0
   if (matchPattern(condition, m_One()))
-    return getTrueValue();
+    return trueVal;
 
   // select false, %0, %1 => %1
   if (matchPattern(condition, m_Zero()))
-    return getFalseValue();
+    return falseVal;
+
+  if (auto cmp = dyn_cast_or_null<CmpIOp>(condition.getDefiningOp())) {
+    auto pred = cmp.predicate();
+    if (pred == mlir::CmpIPredicate::eq || pred == mlir::CmpIPredicate::ne) {
+      auto cmpLhs = cmp.lhs();
+      auto cmpRhs = cmp.rhs();
+
+      // %0 = cmpi eq, %arg0, %arg1
+      // %1 = select %0, %arg0, %arg1 => %arg1
+
+      // %0 = cmpi ne, %arg0, %arg1
+      // %1 = select %0, %arg0, %arg1 => %arg0
+
+      if ((cmpLhs == trueVal && cmpRhs == falseVal) ||
+          (cmpRhs == trueVal && cmpLhs == falseVal))
+        return pred == mlir::CmpIPredicate::ne ? trueVal : falseVal;
+    }
+  }
   return nullptr;
 }
 
@@ -1994,11 +1983,11 @@ struct SubTensorCanonicalizer {
   }
 };
 
-void SubTensorOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void SubTensorOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
-  results.insert<OpWithOffsetSizesAndStridesConstantArgumentFolder<
-                     SubTensorOp, SubTensorCanonicalizer>,
-                 SubTensorOpCastFolder>(context);
+  results.add<OpWithOffsetSizesAndStridesConstantArgumentFolder<
+                  SubTensorOp, SubTensorCanonicalizer>,
+              SubTensorOpCastFolder>(context);
 }
 
 //
@@ -2165,10 +2154,10 @@ struct SubTensorInsertOpCastFolder final
 };
 } // namespace
 
-void SubTensorInsertOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<SubTensorInsertOpConstantArgumentFolder,
-                 SubTensorInsertOpCastFolder>(context);
+void SubTensorInsertOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                    MLIRContext *context) {
+  results.add<SubTensorInsertOpConstantArgumentFolder,
+              SubTensorInsertOpCastFolder>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2192,6 +2181,16 @@ static LogicalResult verify(TruncateIOp op) {
   return success();
 }
 
+OpFoldResult TruncateIOp::fold(ArrayRef<Attribute> operands) {
+  // trunci(zexti(a)) -> a
+  // trunci(sexti(a)) -> a
+  if (matchPattern(getOperand(), m_Op<ZeroExtendIOp>()) ||
+      matchPattern(getOperand(), m_Op<SignExtendIOp>()))
+    return getOperand().getDefiningOp()->getOperand(0);
+
+  return nullptr;
+}
+
 //===----------------------------------------------------------------------===//
 // UnsignedDivIOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/Bufferize.cpp b/mlir/lib/Dialect/StandardOps/Transforms/Bufferize.cpp
index c2b9c938e0b893a3da62ba4bc82734f2cfe899b9..040bdc81f23b431d40d23d97650836530fec8a5b 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/Bufferize.cpp
@@ -54,10 +54,10 @@ public:
 };
 } // namespace
 
-void mlir::populateStdBufferizePatterns(MLIRContext *context,
-                                        BufferizeTypeConverter &typeConverter,
-                                        OwningRewritePatternList &patterns) {
-  patterns.insert<BufferizeDimOp, BufferizeSelectOp>(typeConverter, context);
+void mlir::populateStdBufferizePatterns(BufferizeTypeConverter &typeConverter,
+                                        RewritePatternSet &patterns) {
+  patterns.add<BufferizeDimOp, BufferizeSelectOp>(typeConverter,
+                                                  patterns.getContext());
 }
 
 namespace {
@@ -65,14 +65,14 @@ struct StdBufferizePass : public StdBufferizeBase<StdBufferizePass> {
   void runOnFunction() override {
     auto *context = &getContext();
     BufferizeTypeConverter typeConverter;
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
     ConversionTarget target(*context);
 
     target.addLegalDialect<memref::MemRefDialect>();
     target.addLegalDialect<StandardOpsDialect>();
     target.addLegalDialect<scf::SCFDialect>();
 
-    populateStdBufferizePatterns(context, typeConverter, patterns);
+    populateStdBufferizePatterns(typeConverter, patterns);
     // We only bufferize the case of tensor selected type and scalar condition,
     // as that boils down to a select over memref descriptors (don't need to
     // touch the data).
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.cpp b/mlir/lib/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.cpp
index 8d1d6befa66faafacbd45097405239b3026616a5..a3dd9a4be5ec8666f4f5f3ae444060258f8047e7 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.cpp
@@ -188,9 +188,9 @@ struct DecomposeCallGraphTypesForCallOp
 
 void mlir::populateDecomposeCallGraphTypesPatterns(
     MLIRContext *context, TypeConverter &typeConverter,
-    ValueDecomposer &decomposer, OwningRewritePatternList &patterns) {
-  patterns.insert<DecomposeCallGraphTypesForCallOp,
-                  DecomposeCallGraphTypesForFuncArgs,
-                  DecomposeCallGraphTypesForReturnOp>(typeConverter, context,
-                                                      decomposer);
+    ValueDecomposer &decomposer, RewritePatternSet &patterns) {
+  patterns
+      .add<DecomposeCallGraphTypesForCallOp, DecomposeCallGraphTypesForFuncArgs,
+           DecomposeCallGraphTypesForReturnOp>(typeConverter, context,
+                                               decomposer);
 }
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/StandardOps/Transforms/ExpandOps.cpp
index 98b261c8044bd067f9387bca1d1504cafdf0731e..fd1b36907ae1e4e398e270ead1e723b08f48bed2 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/ExpandOps.cpp
@@ -211,8 +211,8 @@ struct StdExpandOpsPass : public StdExpandOpsBase<StdExpandOpsPass> {
   void runOnFunction() override {
     MLIRContext &ctx = getContext();
 
-    OwningRewritePatternList patterns;
-    populateStdExpandOpsPatterns(&ctx, patterns);
+    RewritePatternSet patterns(&ctx);
+    populateStdExpandOpsPatterns(patterns);
 
     ConversionTarget target(getContext());
 
@@ -234,11 +234,10 @@ struct StdExpandOpsPass : public StdExpandOpsBase<StdExpandOpsPass> {
 
 } // namespace
 
-void mlir::populateStdExpandOpsPatterns(MLIRContext *context,
-                                        OwningRewritePatternList &patterns) {
-  patterns.insert<AtomicRMWOpConverter, MemRefReshapeOpConverter,
-                  SignedCeilDivIOpConverter, SignedFloorDivIOpConverter>(
-      context);
+void mlir::populateStdExpandOpsPatterns(RewritePatternSet &patterns) {
+  patterns.add<AtomicRMWOpConverter, MemRefReshapeOpConverter,
+               SignedCeilDivIOpConverter, SignedFloorDivIOpConverter>(
+      patterns.getContext());
 }
 
 std::unique_ptr<Pass> mlir::createStdExpandOpsPass() {
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/FuncBufferize.cpp b/mlir/lib/Dialect/StandardOps/Transforms/FuncBufferize.cpp
index d38a564ac2a9018ece7309e355c51b25cb62c3a2..1b07ccd9c1eddb1083ebd4dd38642fba9bf3ce5e 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/FuncBufferize.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/FuncBufferize.cpp
@@ -28,23 +28,21 @@ struct FuncBufferizePass : public FuncBufferizeBase<FuncBufferizePass> {
     auto *context = &getContext();
 
     BufferizeTypeConverter typeConverter;
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
     ConversionTarget target(*context);
 
-    populateFuncOpTypeConversionPattern(patterns, context, typeConverter);
+    populateFuncOpTypeConversionPattern(patterns, typeConverter);
     target.addDynamicallyLegalOp<FuncOp>([&](FuncOp op) {
       return typeConverter.isSignatureLegal(op.getType()) &&
              typeConverter.isLegal(&op.getBody());
     });
-    populateCallOpTypeConversionPattern(patterns, context, typeConverter);
+    populateCallOpTypeConversionPattern(patterns, typeConverter);
     target.addDynamicallyLegalOp<CallOp>(
         [&](CallOp op) { return typeConverter.isLegal(op); });
 
-    populateBranchOpInterfaceTypeConversionPattern(patterns, context,
-                                                   typeConverter);
-    populateReturnOpTypeConversionPattern(patterns, context, typeConverter);
-    target.addLegalOp<ModuleOp, ModuleTerminatorOp, memref::TensorLoadOp,
-                      memref::BufferCastOp>();
+    populateBranchOpInterfaceTypeConversionPattern(patterns, typeConverter);
+    populateReturnOpTypeConversionPattern(patterns, typeConverter);
+    target.addLegalOp<ModuleOp, memref::TensorLoadOp, memref::BufferCastOp>();
 
     target.markUnknownOpDynamicallyLegal([&](Operation *op) {
       return isNotBranchOpInterfaceOrReturnLikeOp(op) ||
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/FuncConversions.cpp b/mlir/lib/Dialect/StandardOps/Transforms/FuncConversions.cpp
index 4ba2069817a351e145d20bf686881de9ca472de5..bf2dcd69e9cad183f034cd51b86df955709da543 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/FuncConversions.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/FuncConversions.cpp
@@ -37,35 +37,30 @@ struct CallOpSignatureConversion : public OpConversionPattern<CallOp> {
 };
 } // end anonymous namespace
 
-void mlir::populateCallOpTypeConversionPattern(
-    OwningRewritePatternList &patterns, MLIRContext *ctx,
-    TypeConverter &converter) {
-  patterns.insert<CallOpSignatureConversion>(converter, ctx);
+void mlir::populateCallOpTypeConversionPattern(RewritePatternSet &patterns,
+                                               TypeConverter &converter) {
+  patterns.add<CallOpSignatureConversion>(converter, patterns.getContext());
 }
 
 namespace {
 /// Only needed to support partial conversion of functions where this pattern
 /// ensures that the branch operation arguments matches up with the succesor
 /// block arguments.
-class BranchOpInterfaceTypeConversion : public ConversionPattern {
+class BranchOpInterfaceTypeConversion
+    : public OpInterfaceConversionPattern<BranchOpInterface> {
 public:
-  BranchOpInterfaceTypeConversion(TypeConverter &typeConverter,
-                                  MLIRContext *ctx)
-      : ConversionPattern(/*benefit=*/1, typeConverter, MatchAnyOpTypeTag()) {}
+  using OpInterfaceConversionPattern<
+      BranchOpInterface>::OpInterfaceConversionPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(BranchOpInterface op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
-    auto branchOp = dyn_cast<BranchOpInterface>(op);
-    if (!branchOp)
-      return failure();
-
     // For a branch operation, only some operands go to the target blocks, so
     // only rewrite those.
     SmallVector<Value, 4> newOperands(op->operand_begin(), op->operand_end());
     for (int succIdx = 0, succEnd = op->getBlock()->getNumSuccessors();
          succIdx < succEnd; ++succIdx) {
-      auto successorOperands = branchOp.getSuccessorOperands(succIdx);
+      auto successorOperands = op.getSuccessorOperands(succIdx);
       if (!successorOperands)
         continue;
       for (int idx = successorOperands->getBeginOperandIndex(),
@@ -103,9 +98,9 @@ public:
 } // end anonymous namespace
 
 void mlir::populateBranchOpInterfaceTypeConversionPattern(
-    OwningRewritePatternList &patterns, MLIRContext *ctx,
-    TypeConverter &typeConverter) {
-  patterns.insert<BranchOpInterfaceTypeConversion>(typeConverter, ctx);
+    RewritePatternSet &patterns, TypeConverter &typeConverter) {
+  patterns.add<BranchOpInterfaceTypeConversion>(typeConverter,
+                                                patterns.getContext());
 }
 
 bool mlir::isLegalForBranchOpInterfaceTypeConversionPattern(
@@ -124,10 +119,9 @@ bool mlir::isLegalForBranchOpInterfaceTypeConversionPattern(
   return false;
 }
 
-void mlir::populateReturnOpTypeConversionPattern(
-    OwningRewritePatternList &patterns, MLIRContext *ctx,
-    TypeConverter &typeConverter) {
-  patterns.insert<ReturnOpTypeConversion>(typeConverter, ctx);
+void mlir::populateReturnOpTypeConversionPattern(RewritePatternSet &patterns,
+                                                 TypeConverter &typeConverter) {
+  patterns.add<ReturnOpTypeConversion>(typeConverter, patterns.getContext());
 }
 
 bool mlir::isLegalForReturnOpTypeConversionPattern(Operation *op,
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp b/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp
index 18c3be94685b61d2e5ae99df0110420563252ef3..b40e47c9441414d05737cf8fd3271dba97b5fd11 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp
@@ -15,64 +15,47 @@
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/StandardOps/Transforms/Passes.h"
 #include "mlir/IR/BlockAndValueMapping.h"
+#include "mlir/Transforms/BufferUtils.h"
 #include "mlir/Transforms/Bufferize.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 using namespace mlir;
 
-namespace {
-// This class creates global ops for all tensor-valued constants in the program.
-// It creates them with pretty names and makes sure that duplicate globals
-// aren't created.
-class GlobalCreator {
-public:
-  explicit GlobalCreator(ModuleOp module);
-  memref::GlobalOp getGlobalFor(Attribute attr) {
-    assert(globals.find(attr) != globals.end() && "unknown constant attr");
-    return globals[attr];
-  }
-
-private:
-  DenseMap<Attribute, memref::GlobalOp> globals;
-};
+memref::GlobalOp GlobalCreator::getGlobalFor(ConstantOp constantOp) {
+  auto type = constantOp.getType().cast<RankedTensorType>();
 
-GlobalCreator::GlobalCreator(ModuleOp module) {
   BufferizeTypeConverter typeConverter;
+
+  // If we already have a global for this constant value, no need to do
+  // anything else.
+  auto it = globals.find(constantOp.getValue());
+  if (it != globals.end())
+    return cast<memref::GlobalOp>(it->second);
+
   // Create a builder without an insertion point. We will insert using the
   // symbol table to guarantee unique names.
-  OpBuilder globalBuilder(module.getContext());
-  SymbolTable symbolTable(module);
-  module.walk([&](ConstantOp op) {
-    // We only want tensor constants for now.
-    auto type = op.getType().dyn_cast<RankedTensorType>();
-    if (!type)
-      return;
-    // If we already have a global for this constant value, no need to do
-    // anything else.
-    auto it = globals.find(op.getValue());
-    if (it != globals.end())
-      return;
+  OpBuilder globalBuilder(moduleOp.getContext());
+  SymbolTable symbolTable(moduleOp);
 
-    // Create a pretty name.
-    SmallString<64> buf;
-    llvm::raw_svector_ostream os(buf);
-    interleave(type.getShape(), os, "x");
-    os << "x" << type.getElementType();
+  // Create a pretty name.
+  SmallString<64> buf;
+  llvm::raw_svector_ostream os(buf);
+  interleave(type.getShape(), os, "x");
+  os << "x" << type.getElementType();
 
-    auto global = globalBuilder.create<memref::GlobalOp>(
-        op.getLoc(), (Twine("__constant_") + os.str()).str(),
-        /*sym_visibility=*/globalBuilder.getStringAttr("private"),
-        /*type=*/typeConverter.convertType(type),
-        /*initial_value=*/op.getValue().cast<ElementsAttr>(),
-        /*constant=*/true);
-    symbolTable.insert(global);
-    // The symbol table inserts at the end of the module, but globals are a bit
-    // nicer if they are at the beginning.
-    global->moveBefore(&module.front());
-    globals[op.getValue()] = global;
-  });
+  auto global = globalBuilder.create<memref::GlobalOp>(
+      constantOp.getLoc(), (Twine("__constant_") + os.str()).str(),
+      /*sym_visibility=*/globalBuilder.getStringAttr("private"),
+      /*type=*/typeConverter.convertType(type),
+      /*initial_value=*/constantOp.getValue().cast<ElementsAttr>(),
+      /*constant=*/true);
+  symbolTable.insert(global);
+  // The symbol table inserts at the end of the module, but globals are a bit
+  // nicer if they are at the beginning.
+  global->moveBefore(&moduleOp.front());
+  globals[constantOp.getValue()] = global;
+  return global;
 }
-} // namespace
 
 namespace {
 class BufferizeTensorConstantOp : public OpConversionPattern<ConstantOp> {
@@ -89,7 +72,7 @@ public:
     if (!type)
       return failure();
 
-    auto globalMemref = globals.getGlobalFor(op.value());
+    auto globalMemref = globals.getGlobalFor(op);
     rewriter.replaceOpWithNewOp<memref::GetGlobalOp>(op, globalMemref.type(),
                                                      globalMemref.getName());
     return success();
@@ -107,11 +90,11 @@ struct TensorConstantBufferizePass
 
     auto *context = &getContext();
     BufferizeTypeConverter typeConverter;
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
     ConversionTarget target(*context);
 
     target.addLegalDialect<memref::MemRefDialect>();
-    patterns.insert<BufferizeTensorConstantOp>(globals, typeConverter, context);
+    patterns.add<BufferizeTensorConstantOp>(globals, typeConverter, context);
     target.addDynamicallyLegalOp<ConstantOp>(
         [&](ConstantOp op) { return typeConverter.isLegal(op.getType()); });
     if (failed(applyPartialConversion(module, target, std::move(patterns))))
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 3da606131a411dc2053e7d206117893bc97074c3..9dc9240cc462302a8aae0334f5a567615898e781 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -177,9 +177,9 @@ struct ChainedTensorCast : public OpRewritePattern<CastOp> {
 
 } // namespace
 
-void CastOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void CastOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                          MLIRContext *context) {
-  results.insert<ChainedTensorCast>(context);
+  results.add<ChainedTensorCast>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -275,9 +275,9 @@ struct ExtractElementFromTensorFromElements
 
 } // namespace
 
-void FromElementsOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<ExtractElementFromTensorFromElements>(context);
+void FromElementsOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                 MLIRContext *context) {
+  results.add<ExtractElementFromTensorFromElements>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -435,11 +435,11 @@ struct ExtractFromTensorCast : public OpRewritePattern<tensor::ExtractOp> {
 
 } // namespace
 
-void GenerateOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void GenerateOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                              MLIRContext *context) {
   // TODO: Move extract patterns to tensor::ExtractOp.
-  results.insert<ExtractFromTensorGenerate, ExtractFromTensorCast,
-                 StaticTensorGenerate>(context);
+  results.add<ExtractFromTensorGenerate, ExtractFromTensorCast,
+              StaticTensorGenerate>(context);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Tensor/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Tensor/Transforms/Bufferize.cpp
index 1ef742e8442403116c52fdb8ae7d7a8f4d79a164..a52b5f69c08f3769522caca7f2fcc11109ff682b 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/Bufferize.cpp
@@ -138,10 +138,9 @@ public:
 } // namespace
 
 void mlir::populateTensorBufferizePatterns(
-    MLIRContext *context, BufferizeTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns) {
-  patterns.insert<BufferizeCastOp, BufferizeExtractOp, BufferizeFromElementsOp,
-                  BufferizeGenerateOp>(typeConverter, context);
+    BufferizeTypeConverter &typeConverter, RewritePatternSet &patterns) {
+  patterns.add<BufferizeCastOp, BufferizeExtractOp, BufferizeFromElementsOp,
+               BufferizeGenerateOp>(typeConverter, patterns.getContext());
 }
 
 namespace {
@@ -149,12 +148,12 @@ struct TensorBufferizePass : public TensorBufferizeBase<TensorBufferizePass> {
   void runOnFunction() override {
     auto *context = &getContext();
     BufferizeTypeConverter typeConverter;
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
     ConversionTarget target(*context);
 
     populateBufferizeMaterializationLegality(target);
 
-    populateTensorBufferizePatterns(context, typeConverter, patterns);
+    populateTensorBufferizePatterns(typeConverter, patterns);
     target.addIllegalOp<tensor::CastOp, tensor::ExtractOp,
                         tensor::FromElementsOp, tensor::GenerateOp>();
     target.addLegalDialect<memref::MemRefDialect>();
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
index 540a7902da93aae8374d25f952909720eeb0ca8b..e2e19c2d0d1e5261aa2ac85ff6e906f85ef7dd5b 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
@@ -251,20 +251,20 @@ struct TosaMakeBroadcastable
 public:
   void runOnFunction() override {
     auto func = getFunction();
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(func.getContext());
     MLIRContext *ctx = func.getContext();
     // Add the generated patterns to the list.
-    patterns.insert<ConvertTosaOp<tosa::AddOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::SubOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::MulOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::MaximumOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::MinimumOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::EqualOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::GreaterOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::GreaterEqualOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::LogicalLeftShiftOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::ArithmeticRightShiftOp>>(ctx);
-    patterns.insert<ConvertTosaOp<tosa::LogicalRightShiftOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::AddOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::SubOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::MulOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::MaximumOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::MinimumOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::EqualOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::GreaterOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::GreaterEqualOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::LogicalLeftShiftOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::ArithmeticRightShiftOp>>(ctx);
+    patterns.add<ConvertTosaOp<tosa::LogicalRightShiftOp>>(ctx);
     (void)applyPatternsAndFoldGreedily(func, std::move(patterns));
   }
 };
diff --git a/mlir/lib/Dialect/Vector/VectorOps.cpp b/mlir/lib/Dialect/Vector/VectorOps.cpp
index 6ca28ba681efae1632aa7443a4feb0d342d31653..9079f99ade8b1be02b8d293a43a61c4741f98d73 100644
--- a/mlir/lib/Dialect/Vector/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/VectorOps.cpp
@@ -714,11 +714,10 @@ struct CanonicalizeContractAdd : public OpRewritePattern<AddOpType> {
   }
 };
 
-void ContractionOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results
-      .insert<CanonicalizeContractAdd<AddIOp>, CanonicalizeContractAdd<AddFOp>>(
-          context);
+void ContractionOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                MLIRContext *context) {
+  results.add<CanonicalizeContractAdd<AddIOp>, CanonicalizeContractAdd<AddFOp>>(
+      context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1332,9 +1331,9 @@ public:
 
 } // namespace
 
-void BroadcastOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void BroadcastOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
-  results.insert<BroadcastToShapeCast>(context);
+  results.add<BroadcastToShapeCast>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2150,11 +2149,11 @@ public:
 } // end anonymous namespace
 
 void ExtractStridedSliceOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
+    RewritePatternSet &results, MLIRContext *context) {
   // Pattern to rewrite a ExtractStridedSliceOp(ConstantMaskOp) ->
   // ConstantMaskOp and ExtractStridedSliceOp(ConstantOp) -> ConstantOp.
-  results.insert<StridedSliceConstantMaskFolder, StridedSliceConstantFolder,
-                 StridedSliceBroadcast>(context);
+  results.add<StridedSliceConstantMaskFolder, StridedSliceConstantFolder,
+              StridedSliceBroadcast>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2294,8 +2293,7 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
 
 static void printTransferAttrs(OpAsmPrinter &p, VectorTransferOpInterface op) {
   SmallVector<StringRef, 2> elidedAttrs;
-  if (op.permutation_map() ==
-      getTransferMinorIdentityMap(op.getShapedType(), op.getVectorType()))
+  if (op.permutation_map().isMinorIdentity())
     elidedAttrs.push_back(op.getPermutationMapAttrName());
   bool elideMasked = true;
   if (auto maybeMasked = op.masked()) {
@@ -2779,9 +2777,9 @@ public:
 };
 } // namespace
 
-void MaskedLoadOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<MaskedLoadFolder>(context);
+void MaskedLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                               MLIRContext *context) {
+  results.add<MaskedLoadFolder>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2824,9 +2822,9 @@ public:
 };
 } // namespace
 
-void MaskedStoreOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<MaskedStoreFolder>(context);
+void MaskedStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                MLIRContext *context) {
+  results.add<MaskedStoreFolder>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2872,9 +2870,9 @@ public:
 };
 } // namespace
 
-void GatherOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void GatherOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                            MLIRContext *context) {
-  results.insert<GatherFolder>(context);
+  results.add<GatherFolder>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2918,9 +2916,9 @@ public:
 };
 } // namespace
 
-void ScatterOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ScatterOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
-  results.insert<ScatterFolder>(context);
+  results.add<ScatterFolder>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2966,9 +2964,9 @@ public:
 };
 } // namespace
 
-void ExpandLoadOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<ExpandLoadFolder>(context);
+void ExpandLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                               MLIRContext *context) {
+  results.add<ExpandLoadFolder>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3012,9 +3010,9 @@ public:
 };
 } // namespace
 
-void CompressStoreOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<CompressStoreFolder>(context);
+void CompressStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                                  MLIRContext *context) {
+  results.add<CompressStoreFolder>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3148,10 +3146,10 @@ public:
 
 } // namespace
 
-void ShapeCastOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+void ShapeCastOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
   // Pattern to rewrite a ShapeCastOp(ConstantOp) -> ConstantOp.
-  results.insert<ShapeCastConstantFolder>(context);
+  results.add<ShapeCastConstantFolder>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3237,7 +3235,7 @@ void TypeCastOp::build(OpBuilder &builder, OperationState &result,
       VectorType::get(extractShape(memRefType),
                       getElementTypeOrSelf(getElementTypeOrSelf(memRefType)));
   result.addTypes(
-      MemRefType::get({}, vectorType, {}, memRefType.getMemorySpaceAsInt()));
+      MemRefType::get({}, vectorType, {}, memRefType.getMemorySpace()));
 }
 
 static LogicalResult verify(TypeCastOp op) {
@@ -3246,8 +3244,8 @@ static LogicalResult verify(TypeCastOp op) {
     return op.emitOpError("expects operand to be a memref with no layout");
   if (!op.getResultMemRefType().getAffineMaps().empty())
     return op.emitOpError("expects result to be a memref with no layout");
-  if (op.getResultMemRefType().getMemorySpaceAsInt() !=
-      op.getMemRefType().getMemorySpaceAsInt())
+  if (op.getResultMemRefType().getMemorySpace() !=
+      op.getMemRefType().getMemorySpace())
     return op.emitOpError("expects result in same memory space");
 
   auto sourceType = op.getMemRefType();
@@ -3394,8 +3392,8 @@ public:
 } // end anonymous namespace
 
 void vector::TransposeOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<TransposeFolder>(context);
+    RewritePatternSet &results, MLIRContext *context) {
+  results.add<TransposeFolder>(context);
 }
 
 void vector::TransposeOp::getTransp(SmallVectorImpl<int64_t> &results) {
@@ -3529,17 +3527,18 @@ public:
 
 } // end anonymous namespace
 
-void CreateMaskOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<CreateMaskFolder>(context);
+void CreateMaskOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                               MLIRContext *context) {
+  results.add<CreateMaskFolder>(context);
 }
 
 void mlir::vector::populateVectorToVectorCanonicalizationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<CreateMaskFolder, MaskedLoadFolder, MaskedStoreFolder,
-                  GatherFolder, ScatterFolder, ExpandLoadFolder,
-                  CompressStoreFolder, StridedSliceConstantMaskFolder,
-                  TransposeFolder>(context);
+    RewritePatternSet &patterns) {
+  patterns
+      .add<CreateMaskFolder, MaskedLoadFolder, MaskedStoreFolder, GatherFolder,
+           ScatterFolder, ExpandLoadFolder, CompressStoreFolder,
+           StridedSliceConstantMaskFolder, TransposeFolder>(
+          patterns.getContext());
 }
 
 #define GET_OP_CLASSES
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index 57602a562d9a6b0190e5c97f6babbc264ebf5907..8766efa406c2883a46be5493d87bd1be703e2a91 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -2784,7 +2784,7 @@ struct TransferReadToVectorLoadLowering
       // If broadcasting is required and the number of loaded elements is 1 then
       // we can create `memref.load` instead of `vector.load`.
       loadOp = rewriter.create<memref::LoadOp>(read.getLoc(), read.source(),
-                                             read.indices());
+                                               read.indices());
     } else {
       // Otherwise create `vector.load`.
       loadOp = rewriter.create<vector::LoadOp>(read.getLoc(),
@@ -3263,60 +3263,60 @@ struct BubbleUpBitCastForStridedSliceInsert
 // TODO: Add pattern to rewrite ExtractSlices(ConstantMaskOp).
 // TODO: Add this as DRR pattern.
 void mlir::vector::populateVectorToVectorTransformationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<ShapeCastOpDecomposer, ShapeCastOpFolder, TupleGetFolderOp,
-                  TransferReadExtractPattern, TransferWriteInsertPattern>(
-      context);
+    RewritePatternSet &patterns) {
+  patterns.add<ShapeCastOpDecomposer, ShapeCastOpFolder, TupleGetFolderOp,
+               TransferReadExtractPattern, TransferWriteInsertPattern>(
+      patterns.getContext());
 }
 
 void mlir::vector::populateSplitVectorTransferPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context,
+    RewritePatternSet &patterns,
     std::function<bool(Operation *)> ignoreFilter) {
-  patterns.insert<SplitTransferReadOp, SplitTransferWriteOp>(context,
-                                                             ignoreFilter);
+  patterns.add<SplitTransferReadOp, SplitTransferWriteOp>(patterns.getContext(),
+                                                          ignoreFilter);
 }
 
 void mlir::vector::populateCastAwayVectorLeadingOneDimPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<CastAwayExtractStridedSliceLeadingOneDim,
-                  CastAwayInsertStridedSliceLeadingOneDim,
-                  CastAwayTransferReadLeadingOneDim,
-                  CastAwayTransferWriteLeadingOneDim, ShapeCastOpFolder>(
-      context);
+    RewritePatternSet &patterns) {
+  patterns.add<CastAwayExtractStridedSliceLeadingOneDim,
+               CastAwayInsertStridedSliceLeadingOneDim,
+               CastAwayTransferReadLeadingOneDim,
+               CastAwayTransferWriteLeadingOneDim, ShapeCastOpFolder>(
+      patterns.getContext());
 }
 
 void mlir::vector::populateBubbleVectorBitCastOpPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<BubbleDownVectorBitCastForExtract,
-                  BubbleDownBitCastForStridedSliceExtract,
-                  BubbleUpBitCastForStridedSliceInsert>(context);
+    RewritePatternSet &patterns) {
+  patterns.add<BubbleDownVectorBitCastForExtract,
+               BubbleDownBitCastForStridedSliceExtract,
+               BubbleUpBitCastForStridedSliceInsert>(patterns.getContext());
 }
 
 void mlir::vector::populateVectorSlicesLoweringPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<ExtractSlicesOpLowering, InsertSlicesOpLowering>(context);
+    RewritePatternSet &patterns) {
+  patterns.add<ExtractSlicesOpLowering, InsertSlicesOpLowering>(
+      patterns.getContext());
 }
 
 void mlir::vector::populateVectorContractLoweringPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context,
-    VectorTransformsOptions parameters) {
+    RewritePatternSet &patterns, VectorTransformsOptions parameters) {
   // clang-format off
-  patterns.insert<BroadcastOpLowering,
+  patterns.add<BroadcastOpLowering,
                   CreateMaskOpLowering,
                   ConstantMaskOpLowering,
                   OuterProductOpLowering,
                   ShapeCastOp2DDownCastRewritePattern,
                   ShapeCastOp2DUpCastRewritePattern,
-                  ShapeCastOpRewritePattern>(context);
-  patterns.insert<TransposeOpLowering,
+                  ShapeCastOpRewritePattern>(patterns.getContext());
+  patterns.add<TransposeOpLowering,
                   ContractionOpLowering,
                   ContractionOpToMatmulOpLowering,
-                  ContractionOpToOuterProductOpLowering>(parameters, context);
+                  ContractionOpToOuterProductOpLowering>(parameters, patterns.getContext());
   // clang-format on
 }
 
 void mlir::vector::populateVectorTransferLoweringPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<TransferReadToVectorLoadLowering,
-                  TransferWriteToVectorStoreLowering>(context);
+    RewritePatternSet &patterns) {
+  patterns.add<TransferReadToVectorLoadLowering,
+               TransferWriteToVectorStoreLowering>(patterns.getContext());
 }
diff --git a/mlir/lib/ExecutionEngine/CMakeLists.txt b/mlir/lib/ExecutionEngine/CMakeLists.txt
index b9176cf1e89b30fcdac98d86f29f0c19ce114c44..978bf1adedd523ab49f9adada82b6c7561131224 100644
--- a/mlir/lib/ExecutionEngine/CMakeLists.txt
+++ b/mlir/lib/ExecutionEngine/CMakeLists.txt
@@ -7,6 +7,7 @@ set(LLVM_OPTIONAL_SOURCES
   CudaRuntimeWrappers.cpp
   SparseUtils.cpp
   ExecutionEngine.cpp
+  RocmRuntimeWrappers.cpp
   RunnerUtils.cpp
   OptUtils.cpp
   JitRunner.cpp
@@ -136,3 +137,51 @@ if(MLIR_CUDA_RUNNER_ENABLED)
     ${CUDA_RUNTIME_LIBRARY}
   )
 endif()
+
+if(MLIR_ROCM_RUNNER_ENABLED)
+  # Configure ROCm support.
+  if (NOT DEFINED ROCM_PATH)
+    if (NOT DEFINED ENV{ROCM_PATH})
+      set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCm has been installed")
+    else()
+      set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCm has been installed")
+    endif()
+    set(HIP_PATH "${ROCM_PATH}/hip" CACHE PATH "Path to which HIP has been installed")
+  endif()
+  set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})
+  find_package(HIP)
+  if (NOT HIP_FOUND)
+    message(SEND_ERROR "Building mlir with ROCm support requires a working ROCm and HIP install")
+  else()
+    message(STATUS "ROCm HIP version: ${HIP_VERSION}")
+  endif()
+
+  # Locate HIP runtime library.
+  find_library(ROCM_RUNTIME_LIBRARY amdhip64
+               PATHS "${HIP_PATH}/lib")
+  if (NOT ROCM_RUNTIME_LIBRARY)
+    message(SEND_ERROR "Could not locate ROCm HIP runtime library")
+  else()
+    message(STATUS "ROCm HIP runtime lib: ${ROCM_RUNTIME_LIBRARY}")
+  endif()
+
+  add_mlir_library(mlir_rocm_runtime
+    SHARED
+    RocmRuntimeWrappers.cpp
+
+    EXCLUDE_FROM_LIBMLIR
+  )
+  target_compile_definitions(mlir_rocm_runtime
+    PRIVATE
+    __HIP_PLATFORM_HCC__
+  )
+  target_include_directories(mlir_rocm_runtime
+    PRIVATE
+    ${HIP_PATH}/include
+    ${ROCM_PATH}/include
+  )
+  target_link_libraries(mlir_rocm_runtime
+    PRIVATE
+    ${ROCM_RUNTIME_LIBRARY}
+  )
+endif()
diff --git a/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp b/mlir/lib/ExecutionEngine/RocmRuntimeWrappers.cpp
similarity index 91%
rename from mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp
rename to mlir/lib/ExecutionEngine/RocmRuntimeWrappers.cpp
index 361ba8f8529db8711f4acb8493513511c9af6436..399a373310609d93ac656001211ded3dcbda4a46 100644
--- a/mlir/tools/mlir-rocm-runner/rocm-runtime-wrappers.cpp
+++ b/mlir/lib/ExecutionEngine/RocmRuntimeWrappers.cpp
@@ -1,4 +1,4 @@
-//===- rocm-runtime-wrappers.cpp - MLIR ROCM runner wrapper library -------===//
+//===- RocmRuntimeWrappers.cpp - MLIR ROCM runtime wrapper library --------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -30,29 +30,25 @@
     fprintf(stderr, "'%s' failed with '%s'\n", #expr, name);                   \
   }(expr)
 
-// Static reference to HIP primary context for device ordinal 0.
-static hipCtx_t Context = [] {
-  HIP_REPORT_IF_ERROR(hipInit(/*flags=*/0));
-  hipDevice_t device;
-  HIP_REPORT_IF_ERROR(hipDeviceGet(&device, /*ordinal=*/0));
-  hipCtx_t context;
-  HIP_REPORT_IF_ERROR(hipDevicePrimaryCtxRetain(&context, device));
-  return context;
-}();
-
 // Sets the `Context` for the duration of the instance and restores the previous
 // context on destruction.
 class ScopedContext {
 public:
   ScopedContext() {
-    HIP_REPORT_IF_ERROR(hipCtxGetCurrent(&previous));
-    HIP_REPORT_IF_ERROR(hipCtxSetCurrent(Context));
+    // Static reference to HIP primary context for device ordinal 0.
+    static hipCtx_t context = [] {
+      HIP_REPORT_IF_ERROR(hipInit(/*flags=*/0));
+      hipDevice_t device;
+      HIP_REPORT_IF_ERROR(hipDeviceGet(&device, /*ordinal=*/0));
+      hipCtx_t ctx;
+      HIP_REPORT_IF_ERROR(hipDevicePrimaryCtxRetain(&ctx, device));
+      return ctx;
+    }();
+
+    HIP_REPORT_IF_ERROR(hipCtxPushCurrent(context));
   }
 
-  ~ScopedContext() { HIP_REPORT_IF_ERROR(hipCtxSetCurrent(previous)); }
-
-private:
-  hipCtx_t previous;
+  ~ScopedContext() { HIP_REPORT_IF_ERROR(hipCtxPopCurrent(nullptr)); }
 };
 
 extern "C" hipModule_t mgpuModuleLoad(void *data) {
diff --git a/mlir/lib/IR/AffineMap.cpp b/mlir/lib/IR/AffineMap.cpp
index 9de80e96d45105d83cce9e3126a90689913874f7..98ca45bbb6f6a8e22ec8fa635fbb2fb1ffb93e41 100644
--- a/mlir/lib/IR/AffineMap.cpp
+++ b/mlir/lib/IR/AffineMap.cpp
@@ -106,8 +106,9 @@ AffineMap AffineMap::getMinorIdentityMap(unsigned dims, unsigned results,
 }
 
 bool AffineMap::isMinorIdentity() const {
-  return *this ==
-         getMinorIdentityMap(getNumDims(), getNumResults(), getContext());
+  return getNumDims() >= getNumResults() &&
+         *this ==
+             getMinorIdentityMap(getNumDims(), getNumResults(), getContext());
 }
 
 /// Returns true if this affine map is a minor identity up to broadcasted
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index b6d327b1c78b55170679fea97204b7c12cb3985b..bae9c97b1b40f5f3f08d2c483098ed7612998f64 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -418,7 +418,8 @@ private:
 
   /// Print the given region.
   void printRegion(Region &region, bool printEntryBlockArgs,
-                   bool printBlockTerminators) override {
+                   bool printBlockTerminators,
+                   bool printEmptyBlock = false) override {
     if (region.empty())
       return;
 
@@ -1515,7 +1516,7 @@ static void printSymbolReference(StringRef symbolRef, raw_ostream &os) {
 // accept the string "elided". The first string must be a registered dialect
 // name and the latter must be a hex constant.
 static void printElidedElementsAttr(raw_ostream &os) {
-  os << R"(opaque<"", "0xDEADBEEF">)";
+  os << R"(opaque<"_", "0xDEADBEEF">)";
 }
 
 void ModulePrinter::printAttribute(Attribute attr,
@@ -1610,8 +1611,8 @@ void ModulePrinter::printAttribute(Attribute attr,
     if (printerFlags.shouldElideElementsAttr(opaqueAttr)) {
       printElidedElementsAttr(os);
     } else {
-      os << "opaque<\"" << opaqueAttr.getDialect()->getNamespace() << "\", ";
-      os << '"' << "0x" << llvm::toHex(opaqueAttr.getValue()) << "\">";
+      os << "opaque<\"" << opaqueAttr.getDialect() << "\", \"0x"
+         << llvm::toHex(opaqueAttr.getValue()) << "\">";
     }
 
   } else if (auto intOrFpEltAttr = attr.dyn_cast<DenseIntOrFPElementsAttr>()) {
@@ -2324,7 +2325,7 @@ public:
 
   /// Print the given region.
   void printRegion(Region &region, bool printEntryBlockArgs,
-                   bool printBlockTerminators) override;
+                   bool printBlockTerminators, bool printEmptyBlock) override;
 
   /// Renumber the arguments for the specified region to the same names as the
   /// SSA values in namesToUse. This may only be used for IsolatedFromAbove
@@ -2409,6 +2410,11 @@ void OperationPrinter::printOperation(Operation *op) {
       opInfo->printAssembly(op, *this);
       return;
     }
+    // Otherwise try to dispatch to the dialect, if available.
+    if (Dialect *dialect = op->getDialect()) {
+      if (succeeded(dialect->printOperation(op, *this)))
+        return;
+    }
   }
 
   // Otherwise print with the generic assembly form.
@@ -2435,7 +2441,7 @@ void OperationPrinter::printGenericOp(Operation *op) {
     os << " (";
     interleaveComma(op->getRegions(), [&](Region &region) {
       printRegion(region, /*printEntryBlockArgs=*/true,
-                  /*printBlockTerminators=*/true);
+                  /*printBlockTerminators=*/true, /*printEmptyBlock=*/true);
     });
     os << ')';
   }
@@ -2536,12 +2542,18 @@ void OperationPrinter::printSuccessorAndUseList(Block *successor,
 }
 
 void OperationPrinter::printRegion(Region &region, bool printEntryBlockArgs,
-                                   bool printBlockTerminators) {
+                                   bool printBlockTerminators,
+                                   bool printEmptyBlock) {
   os << " {" << newLine;
   if (!region.empty()) {
     auto *entryBlock = &region.front();
-    print(entryBlock, printEntryBlockArgs && entryBlock->getNumArguments() != 0,
-          printBlockTerminators);
+    // Force printing the block header if printEmptyBlock is set and the block
+    // is empty or if printEntryBlockArgs is set and there are arguments to
+    // print.
+    bool shouldAlwaysPrintBlockHeader =
+        (printEmptyBlock && entryBlock->empty()) ||
+        (printEntryBlockArgs && entryBlock->getNumArguments() != 0);
+    print(entryBlock, shouldAlwaysPrintBlockHeader, printBlockTerminators);
     for (auto &b : llvm::drop_begin(region.getBlocks(), 1))
       print(&b);
   }
diff --git a/mlir/lib/IR/AttributeDetail.h b/mlir/lib/IR/AttributeDetail.h
index 9499a0f84c84b09049c3779c05fa4d8ac69e6367..f62d886cacb62a9617c41f359b145abd75724b7e 100644
--- a/mlir/lib/IR/AttributeDetail.h
+++ b/mlir/lib/IR/AttributeDetail.h
@@ -27,113 +27,6 @@
 namespace mlir {
 namespace detail {
 
-/// An attribute representing a floating point value.
-struct FloatAttributeStorage final
-    : public AttributeStorage,
-      public llvm::TrailingObjects<FloatAttributeStorage, uint64_t> {
-  using KeyTy = std::pair<Type, APFloat>;
-
-  FloatAttributeStorage(const llvm::fltSemantics &semantics, Type type,
-                        size_t numObjects)
-      : AttributeStorage(type), semantics(semantics), numObjects(numObjects) {}
-
-  /// Key equality and hash functions.
-  bool operator==(const KeyTy &key) const {
-    return key.first == getType() && key.second.bitwiseIsEqual(getValue());
-  }
-  static unsigned hashKey(const KeyTy &key) {
-    return llvm::hash_combine(key.first, llvm::hash_value(key.second));
-  }
-
-  /// Construct a key with a type and double.
-  static KeyTy getKey(Type type, double value) {
-    if (type.isF64())
-      return KeyTy(type, APFloat(value));
-
-    // This handles, e.g., F16 because there is no APFloat constructor for it.
-    bool unused;
-    APFloat val(value);
-    val.convert(type.cast<FloatType>().getFloatSemantics(),
-                APFloat::rmNearestTiesToEven, &unused);
-    return KeyTy(type, val);
-  }
-
-  /// Construct a new storage instance.
-  static FloatAttributeStorage *construct(AttributeStorageAllocator &allocator,
-                                          const KeyTy &key) {
-    const auto &apint = key.second.bitcastToAPInt();
-
-    // Here one word's bitwidth equals to that of uint64_t.
-    auto elements = ArrayRef<uint64_t>(apint.getRawData(), apint.getNumWords());
-
-    auto byteSize =
-        FloatAttributeStorage::totalSizeToAlloc<uint64_t>(elements.size());
-    auto rawMem = allocator.allocate(byteSize, alignof(FloatAttributeStorage));
-    auto result = ::new (rawMem) FloatAttributeStorage(
-        key.second.getSemantics(), key.first, elements.size());
-    std::uninitialized_copy(elements.begin(), elements.end(),
-                            result->getTrailingObjects<uint64_t>());
-    return result;
-  }
-
-  /// Returns an APFloat representing the stored value.
-  APFloat getValue() const {
-    auto val = APInt(APFloat::getSizeInBits(semantics),
-                     {getTrailingObjects<uint64_t>(), numObjects});
-    return APFloat(semantics, val);
-  }
-
-  const llvm::fltSemantics &semantics;
-  size_t numObjects;
-};
-
-/// An attribute representing an integral value.
-struct IntegerAttributeStorage final
-    : public AttributeStorage,
-      public llvm::TrailingObjects<IntegerAttributeStorage, uint64_t> {
-  using KeyTy = std::pair<Type, APInt>;
-
-  IntegerAttributeStorage(Type type, size_t numObjects)
-      : AttributeStorage(type), numObjects(numObjects) {
-    assert((type.isIndex() || type.isa<IntegerType>()) && "invalid type");
-  }
-
-  /// Key equality and hash functions.
-  bool operator==(const KeyTy &key) const {
-    return key == KeyTy(getType(), getValue());
-  }
-  static unsigned hashKey(const KeyTy &key) {
-    return llvm::hash_combine(key.first, llvm::hash_value(key.second));
-  }
-
-  /// Construct a new storage instance.
-  static IntegerAttributeStorage *
-  construct(AttributeStorageAllocator &allocator, const KeyTy &key) {
-    Type type;
-    APInt value;
-    std::tie(type, value) = key;
-
-    auto elements = ArrayRef<uint64_t>(value.getRawData(), value.getNumWords());
-    auto size =
-        IntegerAttributeStorage::totalSizeToAlloc<uint64_t>(elements.size());
-    auto rawMem = allocator.allocate(size, alignof(IntegerAttributeStorage));
-    auto result = ::new (rawMem) IntegerAttributeStorage(type, elements.size());
-    std::uninitialized_copy(elements.begin(), elements.end(),
-                            result->getTrailingObjects<uint64_t>());
-    return result;
-  }
-
-  /// Returns an APInt representing the stored value.
-  APInt getValue() const {
-    if (getType().isIndex())
-      return APInt(64, {getTrailingObjects<uint64_t>(), numObjects});
-    return APInt(getType().getIntOrFloatBitWidth(),
-                 {getTrailingObjects<uint64_t>(), numObjects});
-  }
-
-  size_t numObjects;
-};
-
 //===----------------------------------------------------------------------===//
 // Elements Attributes
 //===----------------------------------------------------------------------===//
@@ -158,10 +51,9 @@ public:
 };
 
 /// An attribute representing a reference to a dense vector or tensor object.
-struct DenseIntOrFPElementsAttributeStorage
-    : public DenseElementsAttributeStorage {
-  DenseIntOrFPElementsAttributeStorage(ShapedType ty, ArrayRef<char> data,
-                                       bool isSplat = false)
+struct DenseIntOrFPElementsAttrStorage : public DenseElementsAttributeStorage {
+  DenseIntOrFPElementsAttrStorage(ShapedType ty, ArrayRef<char> data,
+                                  bool isSplat = false)
       : DenseElementsAttributeStorage(ty, isSplat), data(data) {}
 
   struct KeyTy {
@@ -287,7 +179,7 @@ struct DenseIntOrFPElementsAttributeStorage
   }
 
   /// Construct a new storage instance.
-  static DenseIntOrFPElementsAttributeStorage *
+  static DenseIntOrFPElementsAttrStorage *
   construct(AttributeStorageAllocator &allocator, KeyTy key) {
     // If the data buffer is non-empty, we copy it into the allocator with a
     // 64-bit alignment.
@@ -303,8 +195,8 @@ struct DenseIntOrFPElementsAttributeStorage
       copy = ArrayRef<char>(rawData, data.size());
     }
 
-    return new (allocator.allocate<DenseIntOrFPElementsAttributeStorage>())
-        DenseIntOrFPElementsAttributeStorage(key.type, copy, key.isSplat);
+    return new (allocator.allocate<DenseIntOrFPElementsAttrStorage>())
+        DenseIntOrFPElementsAttrStorage(key.type, copy, key.isSplat);
   }
 
   ArrayRef<char> data;
@@ -312,10 +204,9 @@ struct DenseIntOrFPElementsAttributeStorage
 
 /// An attribute representing a reference to a dense vector or tensor object
 /// containing strings.
-struct DenseStringElementsAttributeStorage
-    : public DenseElementsAttributeStorage {
-  DenseStringElementsAttributeStorage(ShapedType ty, ArrayRef<StringRef> data,
-                                      bool isSplat = false)
+struct DenseStringElementsAttrStorage : public DenseElementsAttributeStorage {
+  DenseStringElementsAttrStorage(ShapedType ty, ArrayRef<StringRef> data,
+                                 bool isSplat = false)
       : DenseElementsAttributeStorage(ty, isSplat), data(data) {}
 
   struct KeyTy {
@@ -385,14 +276,14 @@ struct DenseStringElementsAttributeStorage
   }
 
   /// Construct a new storage instance.
-  static DenseStringElementsAttributeStorage *
+  static DenseStringElementsAttrStorage *
   construct(AttributeStorageAllocator &allocator, KeyTy key) {
     // If the data buffer is non-empty, we copy it into the allocator with a
     // 64-bit alignment.
     ArrayRef<StringRef> copy, data = key.data;
     if (data.empty()) {
-      return new (allocator.allocate<DenseStringElementsAttributeStorage>())
-          DenseStringElementsAttributeStorage(key.type, copy, key.isSplat);
+      return new (allocator.allocate<DenseStringElementsAttrStorage>())
+          DenseStringElementsAttrStorage(key.type, copy, key.isSplat);
     }
 
     int numEntries = key.isSplat ? 1 : data.size();
@@ -421,72 +312,13 @@ struct DenseStringElementsAttributeStorage
     copy =
         ArrayRef<StringRef>(reinterpret_cast<StringRef *>(rawData), numEntries);
 
-    return new (allocator.allocate<DenseStringElementsAttributeStorage>())
-        DenseStringElementsAttributeStorage(key.type, copy, key.isSplat);
+    return new (allocator.allocate<DenseStringElementsAttrStorage>())
+        DenseStringElementsAttrStorage(key.type, copy, key.isSplat);
   }
 
   ArrayRef<StringRef> data;
 };
 
-/// An attribute representing a reference to a tensor constant with opaque
-/// content.
-struct OpaqueElementsAttributeStorage : public AttributeStorage {
-  using KeyTy = std::tuple<Type, Dialect *, StringRef>;
-
-  OpaqueElementsAttributeStorage(Type type, Dialect *dialect, StringRef bytes)
-      : AttributeStorage(type), dialect(dialect), bytes(bytes) {}
-
-  /// Key equality and hash functions.
-  bool operator==(const KeyTy &key) const {
-    return key == std::make_tuple(getType(), dialect, bytes);
-  }
-  static unsigned hashKey(const KeyTy &key) {
-    return llvm::hash_combine(std::get<0>(key), std::get<1>(key),
-                              std::get<2>(key));
-  }
-
-  /// Construct a new storage instance.
-  static OpaqueElementsAttributeStorage *
-  construct(AttributeStorageAllocator &allocator, KeyTy key) {
-    // TODO: Provide a way to avoid copying content of large opaque
-    // tensors This will likely require a new reference attribute kind.
-    return new (allocator.allocate<OpaqueElementsAttributeStorage>())
-        OpaqueElementsAttributeStorage(std::get<0>(key), std::get<1>(key),
-                                       allocator.copyInto(std::get<2>(key)));
-  }
-
-  Dialect *dialect;
-  StringRef bytes;
-};
-
-/// An attribute representing a reference to a sparse vector or tensor object.
-struct SparseElementsAttributeStorage : public AttributeStorage {
-  using KeyTy = std::tuple<Type, DenseIntElementsAttr, DenseElementsAttr>;
-
-  SparseElementsAttributeStorage(Type type, DenseIntElementsAttr indices,
-                                 DenseElementsAttr values)
-      : AttributeStorage(type), indices(indices), values(values) {}
-
-  /// Key equality and hash functions.
-  bool operator==(const KeyTy &key) const {
-    return key == std::make_tuple(getType(), indices, values);
-  }
-  static unsigned hashKey(const KeyTy &key) {
-    return llvm::hash_combine(std::get<0>(key), std::get<1>(key),
-                              std::get<2>(key));
-  }
-
-  /// Construct a new storage instance.
-  static SparseElementsAttributeStorage *
-  construct(AttributeStorageAllocator &allocator, KeyTy key) {
-    return new (allocator.allocate<SparseElementsAttributeStorage>())
-        SparseElementsAttributeStorage(std::get<0>(key), std::get<1>(key),
-                                       std::get<2>(key));
-  }
-
-  DenseIntElementsAttr indices;
-  DenseElementsAttr values;
-};
 } // namespace detail
 } // namespace mlir
 
diff --git a/mlir/lib/IR/Block.cpp b/mlir/lib/IR/Block.cpp
index 07e2e5c007fe6a86c976931db399db970df13164..d390d490b176d068c4299bb163a565fd9b195c96 100644
--- a/mlir/lib/IR/Block.cpp
+++ b/mlir/lib/IR/Block.cpp
@@ -314,9 +314,11 @@ unsigned PredecessorIterator::getSuccessorIndex() const {
 SuccessorRange::SuccessorRange() : SuccessorRange(nullptr, 0) {}
 
 SuccessorRange::SuccessorRange(Block *block) : SuccessorRange() {
-  if (Operation *term = block->getTerminator())
+  if (!llvm::hasSingleElement(*block->getParent())) {
+    Operation *term = block->getTerminator();
     if ((count = term->getNumSuccessors()))
       base = term->getBlockOperands().data();
+  }
 }
 
 SuccessorRange::SuccessorRange(Operation *term) : SuccessorRange() {
diff --git a/mlir/lib/IR/BuiltinAttributes.cpp b/mlir/lib/IR/BuiltinAttributes.cpp
index 5efb8f7c70fff7df644bfed1dbd68c780ff88bcc..947ee143c96324349cf4999bbe29af94d6f7d290 100644
--- a/mlir/lib/IR/BuiltinAttributes.cpp
+++ b/mlir/lib/IR/BuiltinAttributes.cpp
@@ -202,26 +202,6 @@ DictionaryAttr DictionaryAttr::getEmptyUnchecked(MLIRContext *context) {
 // FloatAttr
 //===----------------------------------------------------------------------===//
 
-FloatAttr FloatAttr::get(Type type, double value) {
-  return Base::get(type.getContext(), type, value);
-}
-
-FloatAttr FloatAttr::getChecked(function_ref<InFlightDiagnostic()> emitError,
-                                Type type, double value) {
-  return Base::getChecked(emitError, type.getContext(), type, value);
-}
-
-FloatAttr FloatAttr::get(Type type, const APFloat &value) {
-  return Base::get(type.getContext(), type, value);
-}
-
-FloatAttr FloatAttr::getChecked(function_ref<InFlightDiagnostic()> emitError,
-                                Type type, const APFloat &value) {
-  return Base::getChecked(emitError, type.getContext(), type, value);
-}
-
-APFloat FloatAttr::getValue() const { return getImpl()->getValue(); }
-
 double FloatAttr::getValueAsDouble() const {
   return getValueAsDouble(getValue());
 }
@@ -234,25 +214,11 @@ double FloatAttr::getValueAsDouble(APFloat value) {
   return value.convertToDouble();
 }
 
-/// Verify construction invariants.
-static LogicalResult
-verifyFloatTypeInvariants(function_ref<InFlightDiagnostic()> emitError,
-                          Type type) {
-  if (!type.isa<FloatType>())
-    return emitError() << "expected floating point type";
-  return success();
-}
-
-LogicalResult FloatAttr::verify(function_ref<InFlightDiagnostic()> emitError,
-                                Type type, double value) {
-  return verifyFloatTypeInvariants(emitError, type);
-}
-
 LogicalResult FloatAttr::verify(function_ref<InFlightDiagnostic()> emitError,
-                                Type type, const APFloat &value) {
+                                Type type, APFloat value) {
   // Verify that the type is correct.
-  if (failed(verifyFloatTypeInvariants(emitError, type)))
-    return failure();
+  if (!type.isa<FloatType>())
+    return emitError() << "expected floating point type";
 
   // Verify that the type semantics match that of the value.
   if (&type.cast<FloatType>().getFloatSemantics() != &value.getSemantics()) {
@@ -279,72 +245,47 @@ StringRef SymbolRefAttr::getLeafReference() const {
 // IntegerAttr
 //===----------------------------------------------------------------------===//
 
-IntegerAttr IntegerAttr::get(Type type, const APInt &value) {
-  if (type.isSignlessInteger(1))
-    return BoolAttr::get(type.getContext(), value.getBoolValue());
-  return Base::get(type.getContext(), type, value);
-}
-
-IntegerAttr IntegerAttr::get(Type type, int64_t value) {
-  // This uses 64 bit APInts by default for index type.
-  if (type.isIndex())
-    return get(type, APInt(IndexType::kInternalStorageBitWidth, value));
-
-  auto intType = type.cast<IntegerType>();
-  return get(type, APInt(intType.getWidth(), value, intType.isSignedInteger()));
-}
-
-APInt IntegerAttr::getValue() const { return getImpl()->getValue(); }
-
 int64_t IntegerAttr::getInt() const {
-  assert((getImpl()->getType().isIndex() ||
-          getImpl()->getType().isSignlessInteger()) &&
+  assert((getType().isIndex() || getType().isSignlessInteger()) &&
          "must be signless integer");
   return getValue().getSExtValue();
 }
 
 int64_t IntegerAttr::getSInt() const {
-  assert(getImpl()->getType().isSignedInteger() && "must be signed integer");
+  assert(getType().isSignedInteger() && "must be signed integer");
   return getValue().getSExtValue();
 }
 
 uint64_t IntegerAttr::getUInt() const {
-  assert(getImpl()->getType().isUnsignedInteger() &&
-         "must be unsigned integer");
+  assert(getType().isUnsignedInteger() && "must be unsigned integer");
   return getValue().getZExtValue();
 }
 
-static LogicalResult
-verifyIntegerTypeInvariants(function_ref<InFlightDiagnostic()> emitError,
-                            Type type) {
-  if (type.isa<IntegerType, IndexType>())
-    return success();
-  return emitError() << "expected integer or index type";
-}
-
 LogicalResult IntegerAttr::verify(function_ref<InFlightDiagnostic()> emitError,
-                                  Type type, int64_t value) {
-  return verifyIntegerTypeInvariants(emitError, type);
-}
-
-LogicalResult IntegerAttr::verify(function_ref<InFlightDiagnostic()> emitError,
-                                  Type type, const APInt &value) {
-  if (failed(verifyIntegerTypeInvariants(emitError, type)))
-    return failure();
-  if (auto integerType = type.dyn_cast<IntegerType>())
+                                  Type type, APInt value) {
+  if (IntegerType integerType = type.dyn_cast<IntegerType>()) {
     if (integerType.getWidth() != value.getBitWidth())
       return emitError() << "integer type bit width (" << integerType.getWidth()
                          << ") doesn't match value bit width ("
                          << value.getBitWidth() << ")";
-  return success();
+    return success();
+  }
+  if (type.isa<IndexType>())
+    return success();
+  return emitError() << "expected integer or index type";
+}
+
+BoolAttr IntegerAttr::getBoolAttrUnchecked(IntegerType type, bool value) {
+  auto attr = Base::get(type.getContext(), type, APInt(/*numBits=*/1, value));
+  return attr.cast<BoolAttr>();
 }
 
 //===----------------------------------------------------------------------===//
 // BoolAttr
 
 bool BoolAttr::getValue() const {
-  auto *storage = reinterpret_cast<IntegerAttributeStorage *>(impl);
-  return storage->getValue().getBoolValue();
+  auto *storage = reinterpret_cast<IntegerAttrStorage *>(impl);
+  return storage->value.getBoolValue();
 }
 
 bool BoolAttr::classof(Attribute attr) {
@@ -987,11 +928,11 @@ auto DenseElementsAttr::getComplexFloatValues() const
 
 /// Return the raw storage data held by this attribute.
 ArrayRef<char> DenseElementsAttr::getRawData() const {
-  return static_cast<DenseIntOrFPElementsAttributeStorage *>(impl)->data;
+  return static_cast<DenseIntOrFPElementsAttrStorage *>(impl)->data;
 }
 
 ArrayRef<StringRef> DenseElementsAttr::getRawStringData() const {
-  return static_cast<DenseStringElementsAttributeStorage *>(impl)->data;
+  return static_cast<DenseStringElementsAttrStorage *>(impl)->data;
 }
 
 /// Return a new DenseElementsAttr that has the same data as the current
@@ -1021,15 +962,6 @@ DenseElementsAttr DenseElementsAttr::mapValues(
   return cast<DenseFPElementsAttr>().mapValues(newElementType, mapping);
 }
 
-//===----------------------------------------------------------------------===//
-// DenseStringElementsAttr
-//===----------------------------------------------------------------------===//
-
-DenseStringElementsAttr
-DenseStringElementsAttr::get(ShapedType type, ArrayRef<StringRef> values) {
-  return Base::get(type.getContext(), type, values, (values.size() == 1));
-}
-
 //===----------------------------------------------------------------------===//
 // DenseIntOrFPElementsAttr
 //===----------------------------------------------------------------------===//
@@ -1254,15 +1186,6 @@ bool DenseIntElementsAttr::classof(Attribute attr) {
 // OpaqueElementsAttr
 //===----------------------------------------------------------------------===//
 
-OpaqueElementsAttr OpaqueElementsAttr::get(Dialect *dialect, ShapedType type,
-                                           StringRef bytes) {
-  assert(TensorType::isValidElementType(type.getElementType()) &&
-         "Input element type should be a valid tensor element type");
-  return Base::get(type.getContext(), type, dialect, bytes);
-}
-
-StringRef OpaqueElementsAttr::getValue() const { return getImpl()->bytes; }
-
 /// Return the value at the given index. If index does not refer to a valid
 /// element, then a null attribute is returned.
 Attribute OpaqueElementsAttr::getValue(ArrayRef<uint64_t> index) const {
@@ -1270,43 +1193,30 @@ Attribute OpaqueElementsAttr::getValue(ArrayRef<uint64_t> index) const {
   return Attribute();
 }
 
-Dialect *OpaqueElementsAttr::getDialect() const { return getImpl()->dialect; }
-
 bool OpaqueElementsAttr::decode(ElementsAttr &result) {
-  auto *d = getDialect();
-  if (!d)
+  Dialect *dialect = getDialect().getDialect();
+  if (!dialect)
     return true;
   auto *interface =
-      d->getRegisteredInterface<DialectDecodeAttributesInterface>();
+      dialect->getRegisteredInterface<DialectDecodeAttributesInterface>();
   if (!interface)
     return true;
   return failed(interface->decode(*this, result));
 }
 
+LogicalResult
+OpaqueElementsAttr::verify(function_ref<InFlightDiagnostic()> emitError,
+                           Identifier dialect, StringRef value,
+                           ShapedType type) {
+  if (!Dialect::isValidNamespace(dialect.strref()))
+    return emitError() << "invalid dialect namespace '" << dialect << "'";
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // SparseElementsAttr
 //===----------------------------------------------------------------------===//
 
-SparseElementsAttr SparseElementsAttr::get(ShapedType type,
-                                           DenseElementsAttr indices,
-                                           DenseElementsAttr values) {
-  assert(indices.getType().getElementType().isInteger(64) &&
-         "expected sparse indices to be 64-bit integer values");
-  assert((type.isa<RankedTensorType, VectorType>()) &&
-         "type must be ranked tensor or vector");
-  assert(type.hasStaticShape() && "type must have static shape");
-  return Base::get(type.getContext(), type,
-                   indices.cast<DenseIntElementsAttr>(), values);
-}
-
-DenseIntElementsAttr SparseElementsAttr::getIndices() const {
-  return getImpl()->indices;
-}
-
-DenseElementsAttr SparseElementsAttr::getValues() const {
-  return getImpl()->values;
-}
-
 /// Return the value of the element at the given index.
 Attribute SparseElementsAttr::getValue(ArrayRef<uint64_t> index) const {
   assert(isValidIndex(index) && "expected valid multi-dimensional index");
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 28aef1500a00c4a4217d5328bc4474fe0a904911..e1706f2c93152612cce7c08c1e29fe63a894d31d 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -209,7 +209,7 @@ FuncOp FuncOp::clone() {
 
 void ModuleOp::build(OpBuilder &builder, OperationState &state,
                      Optional<StringRef> name) {
-  ensureTerminator(*state.addRegion(), builder, state.location);
+  state.addRegion()->emplaceBlock();
   if (name) {
     state.attributes.push_back(builder.getNamedAttr(
         mlir::SymbolTable::getSymbolAttrName(), builder.getStringAttr(*name)));
@@ -222,6 +222,17 @@ ModuleOp ModuleOp::create(Location loc, Optional<StringRef> name) {
   return builder.create<ModuleOp>(loc, name);
 }
 
+DataLayoutSpecInterface ModuleOp::getDataLayoutSpec() {
+  // Take the first and only (if present) attribute that implements the
+  // interface. This needs a linear search, but is called only once per data
+  // layout object construction that is used for repeated queries.
+  for (Attribute attr : llvm::make_second_range(getOperation()->getAttrs())) {
+    if (auto spec = attr.dyn_cast<DataLayoutSpecInterface>())
+      return spec;
+  }
+  return {};
+}
+
 static LogicalResult verify(ModuleOp op) {
   // Check that none of the attributes are non-dialect attributes, except for
   // the symbol related attributes.
@@ -236,6 +247,23 @@ static LogicalResult verify(ModuleOp op) {
                               << attr.first << "'";
   }
 
+  // Check that there is at most one data layout spec attribute.
+  StringRef layoutSpecAttrName;
+  DataLayoutSpecInterface layoutSpec;
+  for (const NamedAttribute &na : op->getAttrs()) {
+    if (auto spec = na.second.dyn_cast<DataLayoutSpecInterface>()) {
+      if (layoutSpec) {
+        InFlightDiagnostic diag =
+            op.emitOpError() << "expects at most one data layout attribute";
+        diag.attachNote() << "'" << layoutSpecAttrName
+                          << "' is a data layout attribute";
+        diag.attachNote() << "'" << na.first << "' is a data layout attribute";
+      }
+      layoutSpecAttrName = na.first.strref();
+      layoutSpec = spec;
+    }
+  }
+
   return success();
 }
 
diff --git a/mlir/lib/IR/BuiltinTypes.cpp b/mlir/lib/IR/BuiltinTypes.cpp
index ee75994ec4f220c003142a80f20e66847175fbdd..99b85c362bb401feace6c5fcfac1243ba71834b1 100644
--- a/mlir/lib/IR/BuiltinTypes.cpp
+++ b/mlir/lib/IR/BuiltinTypes.cpp
@@ -673,26 +673,23 @@ LogicalResult mlir::getStridesAndOffset(MemRefType t,
                                         SmallVectorImpl<AffineExpr> &strides,
                                         AffineExpr &offset) {
   auto affineMaps = t.getAffineMaps();
-  // For now strides are only computed on a single affine map with a single
-  // result (i.e. the closed subset of linearization maps that are compatible
-  // with striding semantics).
-  // TODO: support more forms on a per-need basis.
-  if (affineMaps.size() > 1)
-    return failure();
-  if (affineMaps.size() == 1 && affineMaps[0].getNumResults() != 1)
-    return failure();
 
-  auto zero = getAffineConstantExpr(0, t.getContext());
-  auto one = getAffineConstantExpr(1, t.getContext());
-  offset = zero;
-  strides.assign(t.getRank(), zero);
+  if (!affineMaps.empty() && affineMaps.back().getNumResults() != 1)
+    return failure();
 
   AffineMap m;
   if (!affineMaps.empty()) {
-    m = affineMaps.front();
+    m = affineMaps.back();
+    for (size_t i = affineMaps.size() - 1; i > 0; --i)
+      m = m.compose(affineMaps[i - 1]);
     assert(!m.isIdentity() && "unexpected identity map");
   }
 
+  auto zero = getAffineConstantExpr(0, t.getContext());
+  auto one = getAffineConstantExpr(1, t.getContext());
+  offset = zero;
+  strides.assign(t.getRank(), zero);
+
   // Canonical case for empty map.
   if (!m) {
     // 0-D corner case, offset is already 0.
diff --git a/mlir/lib/IR/CMakeLists.txt b/mlir/lib/IR/CMakeLists.txt
index cef7068af4d8b4557c0dde3f4ab470a59326c59b..68367d69b68a844cf971f2eccd1cea5f6e180e0d 100644
--- a/mlir/lib/IR/CMakeLists.txt
+++ b/mlir/lib/IR/CMakeLists.txt
@@ -40,6 +40,7 @@ add_mlir_library(MLIRIR
   MLIRBuiltinTypesIncGen
   MLIRCallInterfacesIncGen
   MLIRCastInterfacesIncGen
+  MLIRDataLayoutInterfacesIncGen
   MLIROpAsmInterfaceIncGen
   MLIRRegionKindInterfaceIncGen
   MLIRSideEffectInterfacesIncGen
diff --git a/mlir/lib/IR/Dialect.cpp b/mlir/lib/IR/Dialect.cpp
index fc21152878e3630ce21ce36da10a084ef95928de..612c902d4707967b5592c15447e2e82363a3884d 100644
--- a/mlir/lib/IR/Dialect.cpp
+++ b/mlir/lib/IR/Dialect.cpp
@@ -136,6 +136,18 @@ Type Dialect::parseType(DialectAsmParser &parser) const {
   return Type();
 }
 
+Optional<Dialect::ParseOpHook>
+Dialect::getParseOperationHook(StringRef opName) const {
+  return None;
+}
+
+LogicalResult Dialect::printOperation(Operation *op,
+                                      OpAsmPrinter &printer) const {
+  assert(op->getDialect() == this &&
+         "Dialect hook invoked on non-dialect owned operation");
+  return failure();
+}
+
 /// Utility function that returns if the given string is a valid dialect
 /// namespace.
 bool Dialect::isValidNamespace(StringRef str) {
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index 464c1a7c842fe08e19fef2d635ebe4599b50b438..ddc88047b7ee4e2aa832d25d7f6e341cb815d1c1 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -390,17 +390,13 @@ MLIRContext::MLIRContext(const DialectRegistry &registry)
   //// Attributes.
   //// Note: These must be registered after the types as they may generate one
   //// of the above types internally.
+  /// Unknown Location Attribute.
+  impl->unknownLocAttr = AttributeUniquer::get<UnknownLoc>(this);
   /// Bool Attributes.
-  impl->falseAttr = AttributeUniquer::get<IntegerAttr>(
-                        this, impl->int1Ty, APInt(/*numBits=*/1, false))
-                        .cast<BoolAttr>();
-  impl->trueAttr = AttributeUniquer::get<IntegerAttr>(
-                       this, impl->int1Ty, APInt(/*numBits=*/1, true))
-                       .cast<BoolAttr>();
+  impl->falseAttr = IntegerAttr::getBoolAttrUnchecked(impl->int1Ty, false);
+  impl->trueAttr = IntegerAttr::getBoolAttrUnchecked(impl->int1Ty, true);
   /// Unit Attribute.
   impl->unitAttr = AttributeUniquer::get<UnitAttr>(this);
-  /// Unknown Location Attribute.
-  impl->unknownLocAttr = AttributeUniquer::get<UnknownLoc>(this);
   /// The empty dictionary attribute.
   impl->emptyDictionaryAttr = DictionaryAttr::getEmptyUnchecked(this);
 
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index f427e100d347853eaf2fa5063b3bc7f6f2b57aea..81002e89b3792b7309a11094e9f0bea54d38f2e5 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -1001,8 +1001,10 @@ static LogicalResult verifyValueSizeAttr(Operation *op, StringRef attrName,
     return op->emitOpError("requires 1D vector attribute '") << attrName << "'";
 
   auto sizeAttrType = sizeAttr.getType().dyn_cast<VectorType>();
-  if (!sizeAttrType || sizeAttrType.getRank() != 1)
-    return op->emitOpError("requires 1D vector attribute '") << attrName << "'";
+  if (!sizeAttrType || sizeAttrType.getRank() != 1 ||
+      !sizeAttrType.getElementType().isInteger(32))
+    return op->emitOpError("requires 1D vector of i32 attribute '")
+           << attrName << "'";
 
   if (llvm::any_of(sizeAttr.getIntValues(), [](const APInt &element) {
         return !element.isNonNegative();
diff --git a/mlir/lib/IR/PatternMatch.cpp b/mlir/lib/IR/PatternMatch.cpp
index 90e89a536405aec15c3a852d9857b043bcff8b9a..4482b5cb219af9d6471e8c7a9975a75c9ae28fd4 100644
--- a/mlir/lib/IR/PatternMatch.cpp
+++ b/mlir/lib/IR/PatternMatch.cpp
@@ -29,23 +29,49 @@ unsigned short PatternBenefit::getBenefit() const {
 // Pattern
 //===----------------------------------------------------------------------===//
 
+//===----------------------------------------------------------------------===//
+// OperationName Root Constructors
+
 Pattern::Pattern(StringRef rootName, PatternBenefit benefit,
+                 MLIRContext *context, ArrayRef<StringRef> generatedNames)
+    : Pattern(OperationName(rootName, context).getAsOpaquePointer(),
+              RootKind::OperationName, generatedNames, benefit, context) {}
+
+//===----------------------------------------------------------------------===//
+// MatchAnyOpTypeTag Root Constructors
+
+Pattern::Pattern(MatchAnyOpTypeTag tag, PatternBenefit benefit,
+                 MLIRContext *context, ArrayRef<StringRef> generatedNames)
+    : Pattern(nullptr, RootKind::Any, generatedNames, benefit, context) {}
+
+//===----------------------------------------------------------------------===//
+// MatchInterfaceOpTypeTag Root Constructors
+
+Pattern::Pattern(MatchInterfaceOpTypeTag tag, TypeID interfaceID,
+                 PatternBenefit benefit, MLIRContext *context,
+                 ArrayRef<StringRef> generatedNames)
+    : Pattern(interfaceID.getAsOpaquePointer(), RootKind::InterfaceID,
+              generatedNames, benefit, context) {}
+
+//===----------------------------------------------------------------------===//
+// MatchTraitOpTypeTag Root Constructors
+
+Pattern::Pattern(MatchTraitOpTypeTag tag, TypeID traitID,
+                 PatternBenefit benefit, MLIRContext *context,
+                 ArrayRef<StringRef> generatedNames)
+    : Pattern(traitID.getAsOpaquePointer(), RootKind::TraitID, generatedNames,
+              benefit, context) {}
+
+//===----------------------------------------------------------------------===//
+// General Constructors
+
+Pattern::Pattern(const void *rootValue, RootKind rootKind,
+                 ArrayRef<StringRef> generatedNames, PatternBenefit benefit,
                  MLIRContext *context)
-    : rootKind(OperationName(rootName, context)), benefit(benefit) {}
-Pattern::Pattern(PatternBenefit benefit, MatchAnyOpTypeTag tag)
-    : benefit(benefit) {}
-Pattern::Pattern(StringRef rootName, ArrayRef<StringRef> generatedNames,
-                 PatternBenefit benefit, MLIRContext *context)
-    : Pattern(rootName, benefit, context) {
-  generatedOps.reserve(generatedNames.size());
-  std::transform(generatedNames.begin(), generatedNames.end(),
-                 std::back_inserter(generatedOps), [context](StringRef name) {
-                   return OperationName(name, context);
-                 });
-}
-Pattern::Pattern(ArrayRef<StringRef> generatedNames, PatternBenefit benefit,
-                 MLIRContext *context, MatchAnyOpTypeTag tag)
-    : Pattern(benefit, tag) {
+    : rootValue(rootValue), rootKind(rootKind), benefit(benefit),
+      contextAndHasBoundedRecursion(context, false) {
+  if (generatedNames.empty())
+    return;
   generatedOps.reserve(generatedNames.size());
   std::transform(generatedNames.begin(), generatedNames.end(),
                  std::back_inserter(generatedOps), [context](StringRef name) {
@@ -73,22 +99,31 @@ void RewritePattern::anchor() {}
 // PDLValue
 //===----------------------------------------------------------------------===//
 
-void PDLValue::print(raw_ostream &os) {
-  if (!impl) {
-    os << "<Null-PDLValue>";
+void PDLValue::print(raw_ostream &os) const {
+  if (!value) {
+    os << "<NULL-PDLValue>";
     return;
   }
-  if (Value val = impl.dyn_cast<Value>()) {
-    os << val;
-    return;
+  switch (kind) {
+  case Kind::Attribute:
+    os << cast<Attribute>();
+    break;
+  case Kind::Operation:
+    os << *cast<Operation *>();
+    break;
+  case Kind::Type:
+    os << cast<Type>();
+    break;
+  case Kind::TypeRange:
+    llvm::interleaveComma(cast<TypeRange>(), os);
+    break;
+  case Kind::Value:
+    os << cast<Value>();
+    break;
+  case Kind::ValueRange:
+    llvm::interleaveComma(cast<ValueRange>(), os);
+    break;
   }
-  AttrOpTypeImplT aotImpl = impl.get<AttrOpTypeImplT>();
-  if (Attribute attr = aotImpl.dyn_cast<Attribute>())
-    os << attr;
-  else if (Operation *op = aotImpl.dyn_cast<Operation *>())
-    os << *op;
-  else
-    os << aotImpl.get<Type>();
 }
 
 //===----------------------------------------------------------------------===//
@@ -102,7 +137,6 @@ void PDLPatternModule::mergeIn(PDLPatternModule &&other) {
   // Steal the other state if we have no patterns.
   if (!pdlModule) {
     constraintFunctions = std::move(other.constraintFunctions);
-    createFunctions = std::move(other.createFunctions);
     rewriteFunctions = std::move(other.rewriteFunctions);
     pdlModule = std::move(other.pdlModule);
     return;
@@ -110,8 +144,6 @@ void PDLPatternModule::mergeIn(PDLPatternModule &&other) {
   // Steal the functions of the other module.
   for (auto &it : constraintFunctions)
     registerConstraintFunction(it.first(), std::move(it.second));
-  for (auto &it : createFunctions)
-    registerCreateFunction(it.first(), std::move(it.second));
   for (auto &it : rewriteFunctions)
     registerRewriteFunction(it.first(), std::move(it.second));
 
@@ -132,13 +164,7 @@ void PDLPatternModule::registerConstraintFunction(
   assert(it.second &&
          "constraint with the given name has already been registered");
 }
-void PDLPatternModule::registerCreateFunction(StringRef name,
-                                              PDLCreateFunction createFn) {
-  auto it = createFunctions.try_emplace(name, std::move(createFn));
-  (void)it;
-  assert(it.second && "native create function with the given name has "
-                      "already been registered");
-}
+
 void PDLPatternModule::registerRewriteFunction(StringRef name,
                                                PDLRewriteFunction rewriteFn) {
   auto it = rewriteFunctions.try_emplace(name, std::move(rewriteFn));
diff --git a/mlir/lib/IR/SymbolTable.cpp b/mlir/lib/IR/SymbolTable.cpp
index 4620a5bcb381253cda11f927e7f1b38e6b3f6382..8d5ba2e1622443f845b58928bf9b3a92652ef3da 100644
--- a/mlir/lib/IR/SymbolTable.cpp
+++ b/mlir/lib/IR/SymbolTable.cpp
@@ -161,11 +161,17 @@ void SymbolTable::insert(Operation *symbol, Block::iterator insertPt) {
   // TODO: consider if SymbolTable's constructor should behave the same.
   if (!symbol->getParentOp()) {
     auto &body = symbolTableOp->getRegion(0).front();
-    if (insertPt == Block::iterator() || insertPt == body.end())
-      insertPt = Block::iterator(body.getTerminator());
-
-    assert(insertPt->getParentOp() == symbolTableOp &&
-           "expected insertPt to be in the associated module operation");
+    if (insertPt == Block::iterator()) {
+      insertPt = Block::iterator(body.end());
+    } else {
+      assert((insertPt == body.end() ||
+              insertPt->getParentOp() == symbolTableOp) &&
+             "expected insertPt to be in the associated module operation");
+    }
+    // Insert before the terminator, if any.
+    if (insertPt == Block::iterator(body.end()) && !body.empty() &&
+        std::prev(body.end())->hasTrait<OpTrait::IsTerminator>())
+      insertPt = std::prev(body.end());
 
     body.getOperations().insert(insertPt, symbol);
   }
@@ -291,11 +297,14 @@ void SymbolTable::walkSymbolTables(
 Operation *SymbolTable::lookupSymbolIn(Operation *symbolTableOp,
                                        StringRef symbol) {
   assert(symbolTableOp->hasTrait<OpTrait::SymbolTable>());
+  Region &region = symbolTableOp->getRegion(0);
+  if (region.empty())
+    return nullptr;
 
   // Look for a symbol with the given name.
   Identifier symbolNameId = Identifier::get(SymbolTable::getSymbolAttrName(),
                                             symbolTableOp->getContext());
-  for (auto &op : symbolTableOp->getRegion(0).front().without_terminator())
+  for (auto &op : region.front())
     if (getNameIfSymbol(&op, symbolNameId) == symbol)
       return &op;
   return nullptr;
diff --git a/mlir/lib/IR/TypeDetail.h b/mlir/lib/IR/TypeDetail.h
index 5240f766e61c1791ba655152d0509843e6bf0baa..09bfbf0fcf37cc6bde6a7d2a493a8a578ac5004f 100644
--- a/mlir/lib/IR/TypeDetail.h
+++ b/mlir/lib/IR/TypeDetail.h
@@ -62,7 +62,9 @@ struct FunctionTypeStorage : public TypeStorage {
   /// The hash key used for uniquing.
   using KeyTy = std::pair<TypeRange, TypeRange>;
   bool operator==(const KeyTy &key) const {
-    return key == KeyTy(getInputs(), getResults());
+    if (std::get<0>(key) == getInputs())
+      return std::get<1>(key) == getResults();
+    return false;
   }
 
   /// Construction.
diff --git a/mlir/lib/IR/Verifier.cpp b/mlir/lib/IR/Verifier.cpp
index 6aadab97023fb3e139b7dbfeb33578f7385afd64..c01581d45b4cc1c5fb8db5f2ebc58cb983cd64b6 100644
--- a/mlir/lib/IR/Verifier.cpp
+++ b/mlir/lib/IR/Verifier.cpp
@@ -113,17 +113,36 @@ LogicalResult OperationVerifier::verifyRegion(Region &region) {
   return success();
 }
 
+/// Returns true if this block may be valid without terminator. That is if:
+/// - it does not have a parent region.
+/// - Or the parent region have a single block and:
+///    - This region does not have a parent op.
+///    - Or the parent op is unregistered.
+///    - Or the parent op has the NoTerminator trait.
+static bool mayNotHaveTerminator(Block *block) {
+  if (!block->getParent())
+    return true;
+  if (!llvm::hasSingleElement(*block->getParent()))
+    return false;
+  Operation *op = block->getParentOp();
+  return !op || op->mightHaveTrait<OpTrait::NoTerminator>();
+}
+
 LogicalResult OperationVerifier::verifyBlock(Block &block) {
   for (auto arg : block.getArguments())
     if (arg.getOwner() != &block)
       return emitError(block, "block argument not owned by block");
 
   // Verify that this block has a terminator.
-  if (block.empty())
-    return emitError(block, "block with no terminator");
+
+  if (block.empty()) {
+    if (mayNotHaveTerminator(&block))
+      return success();
+    return emitError(block, "empty block: expect at least a terminator");
+  }
 
   // Verify the non-terminator operations separately so that we can verify
-  // they has no successors.
+  // they have no successors.
   for (auto &op : llvm::make_range(block.begin(), std::prev(block.end()))) {
     if (op.getNumSuccessors() != 0)
       return op.emitError(
@@ -137,8 +156,13 @@ LogicalResult OperationVerifier::verifyBlock(Block &block) {
   Operation &terminator = block.back();
   if (failed(verifyOperation(terminator)))
     return failure();
+
+  if (mayNotHaveTerminator(&block))
+    return success();
+
   if (!terminator.mightHaveTrait<OpTrait::IsTerminator>())
-    return block.back().emitError("block with no terminator");
+    return block.back().emitError("block with no terminator, has ")
+           << terminator;
 
   // Verify that this block is not branching to a block of a different
   // region.
@@ -176,13 +200,14 @@ LogicalResult OperationVerifier::verifyOperation(Operation &op) {
   unsigned numRegions = op.getNumRegions();
   for (unsigned i = 0; i < numRegions; i++) {
     Region &region = op.getRegion(i);
+    RegionKind kind =
+        kindInterface ? kindInterface.getRegionKind(i) : RegionKind::SSACFG;
     // Check that Graph Regions only have a single basic block. This is
     // similar to the code in SingleBlockImplicitTerminator, but doesn't
     // require the trait to be specified. This arbitrary limitation is
     // designed to limit the number of cases that have to be handled by
     // transforms and conversions until the concept stabilizes.
-    if (op.isRegistered() && kindInterface &&
-        kindInterface.getRegionKind(i) == RegionKind::Graph) {
+    if (op.isRegistered() && kind == RegionKind::Graph) {
       // Empty regions are fine.
       if (region.empty())
         continue;
diff --git a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp
index 3d23aa8859c8ddd77330b71e094464bb5bd330a1..9f5c75a425fb72eff5df818a5c4f0023dfc1ef4f 100644
--- a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp
+++ b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp
@@ -8,9 +8,12 @@
 
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
 #include "mlir/IR/BuiltinDialect.h"
+#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Operation.h"
 
+#include "llvm/ADT/TypeSwitch.h"
+
 using namespace mlir;
 
 //===----------------------------------------------------------------------===//
@@ -28,21 +31,45 @@ static LLVM_ATTRIBUTE_NORETURN void reportMissingDataLayout(Type type) {
   llvm::report_fatal_error(os.str());
 }
 
+/// Returns the bitwidth of the index type if specified in the param list.
+/// Assumes 64-bit index otherwise.
+static unsigned getIndexBitwidth(DataLayoutEntryListRef params) {
+  if (params.empty())
+    return 64;
+  auto attr = params.front().getValue().cast<IntegerAttr>();
+  return attr.getValue().getZExtValue();
+}
+
 unsigned
 mlir::detail::getDefaultTypeSize(Type type, const DataLayout &dataLayout,
                                  ArrayRef<DataLayoutEntryInterface> params) {
+  unsigned bits = getDefaultTypeSizeInBits(type, dataLayout, params);
+  return llvm::divideCeil(bits, 8);
+}
+
+unsigned mlir::detail::getDefaultTypeSizeInBits(Type type,
+                                                const DataLayout &dataLayout,
+                                                DataLayoutEntryListRef params) {
   if (type.isa<IntegerType, FloatType>())
-    return llvm::divideCeil(type.getIntOrFloatBitWidth(), 8);
+    return type.getIntOrFloatBitWidth();
+
+  // Index is an integer of some bitwidth.
+  if (type.isa<IndexType>())
+    return dataLayout.getTypeSizeInBits(
+        IntegerType::get(type.getContext(), getIndexBitwidth(params)));
 
   // Sizes of vector types are rounded up to those of types with closest
-  // power-of-two number of elements.
+  // power-of-two number of elements in the innermost dimension. We also assume
+  // there is no bit-packing at the moment element sizes are taken in bytes and
+  // multiplied with 8 bits.
   // TODO: make this extensible.
   if (auto vecType = type.dyn_cast<VectorType>())
-    return llvm::PowerOf2Ceil(vecType.getNumElements()) *
-           dataLayout.getTypeSize(vecType.getElementType());
+    return vecType.getNumElements() / vecType.getShape().back() *
+           llvm::PowerOf2Ceil(vecType.getShape().back()) *
+           dataLayout.getTypeSize(vecType.getElementType()) * 8;
 
   if (auto typeInterface = type.dyn_cast<DataLayoutTypeInterface>())
-    return typeInterface.getTypeSize(dataLayout, params);
+    return typeInterface.getTypeSizeInBits(dataLayout, params);
 
   reportMissingDataLayout(type);
 }
@@ -54,6 +81,11 @@ unsigned mlir::detail::getDefaultABIAlignment(
   if (type.isa<FloatType, VectorType>())
     return llvm::PowerOf2Ceil(dataLayout.getTypeSize(type));
 
+  // Index is an integer of some bitwidth.
+  if (type.isa<IndexType>())
+    return dataLayout.getTypeABIAlignment(
+        IntegerType::get(type.getContext(), getIndexBitwidth(params)));
+
   if (auto intType = type.dyn_cast<IntegerType>()) {
     return intType.getWidth() < 64
                ? llvm::PowerOf2Ceil(llvm::divideCeil(intType.getWidth(), 8))
@@ -75,7 +107,7 @@ unsigned mlir::detail::getDefaultPreferredAlignment(
 
   // Preferred alignment is the cloest power-of-two number above for integers
   // (ABI alignment may be smaller).
-  if (auto intType = type.dyn_cast<IntegerType>())
+  if (type.isa<IntegerType, IndexType>())
     return llvm::PowerOf2Ceil(dataLayout.getTypeSize(type));
 
   if (auto typeInterface = type.dyn_cast<DataLayoutTypeInterface>())
@@ -105,65 +137,97 @@ mlir::detail::filterEntryForIdentifier(DataLayoutEntryListRef entries,
   return it == entries.end() ? DataLayoutEntryInterface() : *it;
 }
 
+static DataLayoutSpecInterface getSpec(Operation *operation) {
+  return llvm::TypeSwitch<Operation *, DataLayoutSpecInterface>(operation)
+      .Case<ModuleOp, DataLayoutOpInterface>(
+          [&](auto op) { return op.getDataLayoutSpec(); })
+      .Default([](Operation *) {
+        llvm_unreachable("expected an op with data layout spec");
+        return DataLayoutSpecInterface();
+      });
+}
+
 /// Populates `opsWithLayout` with the list of proper ancestors of `leaf` that
-/// implement the `DataLayoutOpInterface`.
-static void findProperAscendantsWithLayout(
-    Operation *leaf, SmallVectorImpl<DataLayoutOpInterface> &opsWithLayout) {
+/// are either modules or implement the `DataLayoutOpInterface`.
+static void
+collectParentLayouts(Operation *leaf,
+                     SmallVectorImpl<DataLayoutSpecInterface> &specs,
+                     SmallVectorImpl<Location> *opLocations = nullptr) {
   if (!leaf)
     return;
 
-  while (auto opLayout = leaf->getParentOfType<DataLayoutOpInterface>()) {
-    opsWithLayout.push_back(opLayout);
-    leaf = opLayout;
+  for (Operation *parent = leaf->getParentOp(); parent != nullptr;
+       parent = parent->getParentOp()) {
+    llvm::TypeSwitch<Operation *>(parent)
+        .Case<ModuleOp>([&](ModuleOp op) {
+          // Skip top-level module op unless it has a layout. Top-level module
+          // without layout is most likely the one implicitly added by the
+          // parser and it doesn't have location. Top-level null specification
+          // would have had the same effect as not having a specification at all
+          // (using type defaults).
+          if (!op->getParentOp() && !op.getDataLayoutSpec())
+            return;
+          specs.push_back(op.getDataLayoutSpec());
+          if (opLocations)
+            opLocations->push_back(op.getLoc());
+        })
+        .Case<DataLayoutOpInterface>([&](DataLayoutOpInterface op) {
+          specs.push_back(op.getDataLayoutSpec());
+          if (opLocations)
+            opLocations->push_back(op.getLoc());
+        });
   }
 }
 
 /// Returns a layout spec that is a combination of the layout specs attached
 /// to the given operation and all its ancestors.
-static DataLayoutSpecInterface
-getCombinedDataLayout(DataLayoutOpInterface leaf) {
+static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf) {
   if (!leaf)
     return {};
 
+  assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
+         "expected an op with data layout spec");
+
   SmallVector<DataLayoutOpInterface> opsWithLayout;
-  findProperAscendantsWithLayout(leaf, opsWithLayout);
+  SmallVector<DataLayoutSpecInterface> specs;
+  collectParentLayouts(leaf, specs);
 
   // Fast track if there are no ancestors.
-  if (opsWithLayout.empty())
-    return leaf.getDataLayoutSpec();
+  if (specs.empty())
+    return getSpec(leaf);
 
   // Create the list of non-null specs (null/missing specs can be safely
   // ignored) from the outermost to the innermost.
-  SmallVector<DataLayoutSpecInterface> specs;
-  specs.reserve(opsWithLayout.size());
-  for (DataLayoutOpInterface op : llvm::reverse(opsWithLayout))
-    if (DataLayoutSpecInterface current = op.getDataLayoutSpec())
-      specs.push_back(current);
+  auto nonNullSpecs = llvm::to_vector<2>(llvm::make_filter_range(
+      llvm::reverse(specs),
+      [](DataLayoutSpecInterface iface) { return iface != nullptr; }));
 
   // Combine the specs using the innermost as anchor.
-  if (DataLayoutSpecInterface current = leaf.getDataLayoutSpec())
-    return current.combineWith(specs);
-  if (specs.empty())
+  if (DataLayoutSpecInterface current = getSpec(leaf))
+    return current.combineWith(nonNullSpecs);
+  if (nonNullSpecs.empty())
     return {};
-  return specs.back().combineWith(llvm::makeArrayRef(specs).drop_back());
+  return nonNullSpecs.back().combineWith(
+      llvm::makeArrayRef(nonNullSpecs).drop_back());
 }
 
-LogicalResult mlir::detail::verifyDataLayoutOp(DataLayoutOpInterface op) {
-  DataLayoutSpecInterface spec = op.getDataLayoutSpec();
+LogicalResult mlir::detail::verifyDataLayoutOp(Operation *op) {
+  DataLayoutSpecInterface spec = getSpec(op);
   // The layout specification may be missing and it's fine.
   if (!spec)
     return success();
 
-  if (failed(spec.verifySpec(op.getLoc())))
+  if (failed(spec.verifySpec(op->getLoc())))
     return failure();
   if (!getCombinedDataLayout(op)) {
     InFlightDiagnostic diag =
-        op.emitError()
-        << "data layout is not a refinement of the layouts in enclosing ops";
-    SmallVector<DataLayoutOpInterface> opsWithLayout;
-    findProperAscendantsWithLayout(op, opsWithLayout);
-    for (DataLayoutOpInterface parent : opsWithLayout)
-      diag.attachNote(parent.getLoc()) << "enclosing op with data layout";
+        op->emitError()
+        << "data layout does not combine with layouts of enclosing ops";
+    SmallVector<DataLayoutSpecInterface> specs;
+    SmallVector<Location> opLocations;
+    collectParentLayouts(op, specs, &opLocations);
+    for (Location loc : opLocations)
+      diag.attachNote(loc) << "enclosing op with data layout";
     return diag;
   }
   return success();
@@ -173,33 +237,42 @@ LogicalResult mlir::detail::verifyDataLayoutOp(DataLayoutOpInterface op) {
 // DataLayout
 //===----------------------------------------------------------------------===//
 
-mlir::DataLayout::DataLayout(DataLayoutOpInterface op)
-    : originalLayout(getCombinedDataLayout(op)), scope(op) {
+template <typename OpTy>
+void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op) {
   if (!originalLayout) {
     assert((!op || !op.getDataLayoutSpec()) &&
            "could not compute layout information for an op (failed to "
            "combine attributes?)");
   }
+}
+
+mlir::DataLayout::DataLayout() : DataLayout(ModuleOp()) {}
 
+mlir::DataLayout::DataLayout(DataLayoutOpInterface op)
+    : originalLayout(getCombinedDataLayout(op)), scope(op) {
 #ifndef NDEBUG
-  SmallVector<DataLayoutOpInterface> opsWithLayout;
-  findProperAscendantsWithLayout(op, opsWithLayout);
-  layoutStack = llvm::to_vector<2>(
-      llvm::map_range(opsWithLayout, [](DataLayoutOpInterface iface) {
-        return iface.getDataLayoutSpec();
-      }));
+  checkMissingLayout(originalLayout, op);
+  collectParentLayouts(op, layoutStack);
+#endif
+}
+
+mlir::DataLayout::DataLayout(ModuleOp op)
+    : originalLayout(getCombinedDataLayout(op)), scope(op) {
+#ifndef NDEBUG
+  checkMissingLayout(originalLayout, op);
+  collectParentLayouts(op, layoutStack);
 #endif
 }
 
 void mlir::DataLayout::checkValid() const {
 #ifndef NDEBUG
-  SmallVector<DataLayoutOpInterface> opsWithLayout;
-  findProperAscendantsWithLayout(scope, opsWithLayout);
-  assert(opsWithLayout.size() == layoutStack.size() &&
+  SmallVector<DataLayoutSpecInterface> specs;
+  collectParentLayouts(scope, specs);
+  assert(specs.size() == layoutStack.size() &&
          "data layout object used, but no longer valid due to the change in "
          "number of nested layouts");
-  for (auto pair : llvm::zip(opsWithLayout, layoutStack)) {
-    Attribute newLayout = std::get<0>(pair).getDataLayoutSpec();
+  for (auto pair : llvm::zip(specs, layoutStack)) {
+    Attribute newLayout = std::get<0>(pair);
     Attribute origLayout = std::get<1>(pair);
     assert(newLayout == origLayout &&
            "data layout object used, but no longer valid "
@@ -228,30 +301,48 @@ static unsigned cachedLookup(Type t, DenseMap<Type, unsigned> &cache,
 unsigned mlir::DataLayout::getTypeSize(Type t) const {
   checkValid();
   return cachedLookup(t, sizes, [&](Type ty) {
-    return (scope && originalLayout)
-               ? scope.getTypeSize(
-                     ty, *this, originalLayout.getSpecForType(ty.getTypeID()))
-               : detail::getDefaultTypeSize(ty, *this, {});
+    DataLayoutEntryList list;
+    if (originalLayout)
+      list = originalLayout.getSpecForType(ty.getTypeID());
+    if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
+      return iface.getTypeSize(ty, *this, list);
+    return detail::getDefaultTypeSize(ty, *this, list);
+  });
+}
+
+unsigned mlir::DataLayout::getTypeSizeInBits(Type t) const {
+  checkValid();
+  return cachedLookup(t, bitsizes, [&](Type ty) {
+    DataLayoutEntryList list;
+    if (originalLayout)
+      list = originalLayout.getSpecForType(ty.getTypeID());
+    if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
+      return iface.getTypeSizeInBits(ty, *this, list);
+    return detail::getDefaultTypeSizeInBits(ty, *this, list);
   });
 }
 
 unsigned mlir::DataLayout::getTypeABIAlignment(Type t) const {
   checkValid();
   return cachedLookup(t, abiAlignments, [&](Type ty) {
-    return (scope && originalLayout)
-               ? scope.getTypeABIAlignment(
-                     ty, *this, originalLayout.getSpecForType(ty.getTypeID()))
-               : detail::getDefaultABIAlignment(ty, *this, {});
+    DataLayoutEntryList list;
+    if (originalLayout)
+      list = originalLayout.getSpecForType(ty.getTypeID());
+    if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
+      return iface.getTypeABIAlignment(ty, *this, list);
+    return detail::getDefaultABIAlignment(ty, *this, list);
   });
 }
 
 unsigned mlir::DataLayout::getTypePreferredAlignment(Type t) const {
   checkValid();
   return cachedLookup(t, preferredAlignments, [&](Type ty) {
-    return (scope && originalLayout)
-               ? scope.getTypePreferredAlignment(
-                     ty, *this, originalLayout.getSpecForType(ty.getTypeID()))
-               : detail::getDefaultPreferredAlignment(ty, *this, {});
+    DataLayoutEntryList list;
+    if (originalLayout)
+      list = originalLayout.getSpecForType(ty.getTypeID());
+    if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
+      return iface.getTypePreferredAlignment(ty, *this, list);
+    return detail::getDefaultPreferredAlignment(ty, *this, list);
   });
 }
 
@@ -285,6 +376,16 @@ LogicalResult mlir::detail::verifyDataLayoutSpec(DataLayoutSpecInterface spec,
 
   for (const auto &kvp : types) {
     auto sampleType = kvp.second.front().getKey().get<Type>();
+    if (sampleType.isa<IndexType>()) {
+      assert(kvp.second.size() == 1 &&
+             "expected one data layout entry for non-parametric 'index' type");
+      if (!kvp.second.front().getValue().isa<IntegerAttr>())
+        return emitError(loc)
+               << "expected integer attribute in the data layout entry for "
+               << sampleType;
+      continue;
+    }
+
     if (isa<BuiltinDialect>(&sampleType.getDialect()))
       return emitError(loc) << "unexpected data layout for a built-in type";
 
diff --git a/mlir/lib/Parser/AttributeParser.cpp b/mlir/lib/Parser/AttributeParser.cpp
index 98f74174e5a3ef81656917519d84c3e34d536066..a90c65ff1fb38ddb087ae3b038f3b73e80b7a393 100644
--- a/mlir/lib/Parser/AttributeParser.cpp
+++ b/mlir/lib/Parser/AttributeParser.cpp
@@ -307,20 +307,6 @@ Attribute Parser::parseFloatAttr(Type type, bool isNegative) {
   return FloatAttr::get(type, isNegative ? -val.getValue() : val.getValue());
 }
 
-/// Construct a float attribute bitwise equivalent to the integer literal.
-static Optional<APFloat> buildHexadecimalFloatLiteral(Parser *p, FloatType type,
-                                                      uint64_t value) {
-  if (type.isF64())
-    return APFloat(type.getFloatSemantics(), APInt(/*numBits=*/64, value));
-
-  APInt apInt(type.getWidth(), value);
-  if (apInt != value) {
-    p->emitError("hexadecimal float constant out of range for type");
-    return llvm::None;
-  }
-  return APFloat(type.getFloatSemantics(), apInt);
-}
-
 /// Construct an APint from a parsed value, a known attribute type and
 /// sign.
 static Optional<APInt> buildAttributeAPInt(Type type, bool isNegative,
@@ -369,10 +355,9 @@ static Optional<APInt> buildAttributeAPInt(Type type, bool isNegative,
 /// Parse a decimal or a hexadecimal literal, which can be either an integer
 /// or a float attribute.
 Attribute Parser::parseDecOrHexAttr(Type type, bool isNegative) {
-  // Remember if the literal is hexadecimal.
-  StringRef spelling = getToken().getSpelling();
-  auto loc = state.curToken.getLoc();
-  bool isHex = spelling.size() > 1 && spelling[1] == 'x';
+  Token tok = getToken();
+  StringRef spelling = tok.getSpelling();
+  llvm::SMLoc loc = tok.getLoc();
 
   consumeToken(Token::integer);
   if (!type) {
@@ -384,26 +369,12 @@ Attribute Parser::parseDecOrHexAttr(Type type, bool isNegative) {
   }
 
   if (auto floatType = type.dyn_cast<FloatType>()) {
-    if (isNegative)
-      return emitError(
-                 loc,
-                 "hexadecimal float literal should not have a leading minus"),
-             nullptr;
-    if (!isHex) {
-      emitError(loc, "unexpected decimal integer literal for a float attribute")
-              .attachNote()
-          << "add a trailing dot to make the literal a float";
-      return nullptr;
-    }
-
-    auto val = Token::getUInt64IntegerValue(spelling);
-    if (!val.hasValue())
-      return emitError("integer constant out of range for attribute"), nullptr;
-
-    // Construct a float attribute bitwise equivalent to the integer literal.
-    Optional<APFloat> apVal =
-        buildHexadecimalFloatLiteral(this, floatType, *val);
-    return apVal ? FloatAttr::get(floatType, *apVal) : Attribute();
+    Optional<APFloat> result;
+    if (failed(parseFloatFromIntegerLiteral(result, tok, isNegative,
+                                            floatType.getFloatSemantics(),
+                                            floatType.getWidth())))
+      return Attribute();
+    return FloatAttr::get(floatType, *result);
   }
 
   if (!type.isa<IntegerType, IndexType>())
@@ -638,19 +609,13 @@ TensorLiteralParser::getFloatAttrElements(llvm::SMLoc loc, FloatType eltTy,
 
     // Handle hexadecimal float literals.
     if (token.is(Token::integer) && token.getSpelling().startswith("0x")) {
-      if (isNegative) {
-        return p.emitError(token.getLoc())
-               << "hexadecimal float literal should not have a leading minus";
-      }
-      auto val = token.getUInt64IntegerValue();
-      if (!val.hasValue()) {
-        return p.emitError(
-            "hexadecimal float constant out of range for attribute");
-      }
-      Optional<APFloat> apVal = buildHexadecimalFloatLiteral(&p, eltTy, *val);
-      if (!apVal)
+      Optional<APFloat> result;
+      if (failed(p.parseFloatFromIntegerLiteral(result, token, isNegative,
+                                                eltTy.getFloatSemantics(),
+                                                eltTy.getWidth())))
         return failure();
-      floatValues.push_back(*apVal);
+
+      floatValues.push_back(*result);
       continue;
     }
 
@@ -862,16 +827,7 @@ Attribute Parser::parseOpaqueElementsAttr(Type attrType) {
   if (getToken().isNot(Token::string))
     return (emitError("expected dialect namespace"), nullptr);
 
-  auto name = getToken().getStringValue();
-  // Lazy load a dialect in the context if there is a possible namespace.
-  Dialect *dialect = builder.getContext()->getOrLoadDialect(name);
-
-  // TODO: Allow for having an unknown dialect on an opaque
-  // attribute. Otherwise, it can't be roundtripped without having the dialect
-  // registered.
-  if (!dialect)
-    return (emitError("no registered dialect with namespace '" + name + "'"),
-            nullptr);
+  std::string name = getToken().getStringValue();
   consumeToken(Token::string);
 
   if (parseToken(Token::comma, "expected ','"))
@@ -888,7 +844,7 @@ Attribute Parser::parseOpaqueElementsAttr(Type attrType) {
   std::string data;
   if (parseElementAttrHexValues(*this, hexTok, data))
     return nullptr;
-  return OpaqueElementsAttr::get(dialect, type, data);
+  return OpaqueElementsAttr::get(builder.getIdentifier(name), type, data);
 }
 
 /// Shaped type for elements attribute.
diff --git a/mlir/lib/Parser/DialectSymbolParser.cpp b/mlir/lib/Parser/DialectSymbolParser.cpp
index 46096a59f8ac3c1f0e8960fd6c7675634529f883..851f40bb0fe3f8e71d5d8e9ddd497fd2b7703673 100644
--- a/mlir/lib/Parser/DialectSymbolParser.cpp
+++ b/mlir/lib/Parser/DialectSymbolParser.cpp
@@ -63,21 +63,34 @@ public:
 
   /// Parse a floating point value from the stream.
   ParseResult parseFloat(double &result) override {
-    bool negative = parser.consumeIf(Token::minus);
+    bool isNegative = parser.consumeIf(Token::minus);
     Token curTok = parser.getToken();
+    llvm::SMLoc loc = curTok.getLoc();
 
     // Check for a floating point value.
     if (curTok.is(Token::floatliteral)) {
       auto val = curTok.getFloatingPointValue();
       if (!val.hasValue())
-        return emitError(curTok.getLoc(), "floating point value too large");
+        return emitError(loc, "floating point value too large");
       parser.consumeToken(Token::floatliteral);
-      result = negative ? -*val : *val;
+      result = isNegative ? -*val : *val;
       return success();
     }
 
-    // TODO: support hex floating point values.
-    return emitError(getCurrentLocation(), "expected floating point literal");
+    // Check for a hexadecimal float value.
+    if (curTok.is(Token::integer)) {
+      Optional<APFloat> apResult;
+      if (failed(parser.parseFloatFromIntegerLiteral(
+              apResult, curTok, isNegative, APFloat::IEEEdouble(),
+              /*typeSizeInBits=*/64)))
+        return failure();
+
+      parser.consumeToken(Token::integer);
+      result = apResult->convertToDouble();
+      return success();
+    }
+
+    return emitError(loc, "expected floating point literal");
   }
 
   /// Parse an optional integer value from the stream.
diff --git a/mlir/lib/Parser/Parser.cpp b/mlir/lib/Parser/Parser.cpp
index 736522415b499ba5fa962a72c31ca4c00b014840..381338d4226debcda51bbce688db5f0a1cccbd56 100644
--- a/mlir/lib/Parser/Parser.cpp
+++ b/mlir/lib/Parser/Parser.cpp
@@ -112,6 +112,41 @@ OptionalParseResult Parser::parseOptionalInteger(uint64_t &result) {
   return success();
 }
 
+/// Parse a floating point value from an integer literal token.
+ParseResult Parser::parseFloatFromIntegerLiteral(
+    Optional<APFloat> &result, const Token &tok, bool isNegative,
+    const llvm::fltSemantics &semantics, size_t typeSizeInBits) {
+  llvm::SMLoc loc = tok.getLoc();
+  StringRef spelling = tok.getSpelling();
+  bool isHex = spelling.size() > 1 && spelling[1] == 'x';
+  if (!isHex) {
+    return emitError(loc, "unexpected decimal integer literal for a "
+                          "floating point value")
+               .attachNote()
+           << "add a trailing dot to make the literal a float";
+  }
+  if (isNegative) {
+    return emitError(loc, "hexadecimal float literal should not have a "
+                          "leading minus");
+  }
+
+  Optional<uint64_t> value = tok.getUInt64IntegerValue();
+  if (!value.hasValue())
+    return emitError(loc, "hexadecimal float constant out of range for type");
+
+  if (&semantics == &APFloat::IEEEdouble()) {
+    result = APFloat(semantics, APInt(typeSizeInBits, *value));
+    return success();
+  }
+
+  APInt apInt(typeSizeInBits, *value);
+  if (apInt != *value)
+    return emitError(loc, "hexadecimal float constant out of range for type");
+  result = APFloat(semantics, apInt);
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // OperationParser
 //===----------------------------------------------------------------------===//
@@ -890,17 +925,18 @@ Operation *OperationParser::parseGenericOperation(Block *insertBlock,
 namespace {
 class CustomOpAsmParser : public OpAsmParser {
 public:
-  CustomOpAsmParser(SMLoc nameLoc,
-                    ArrayRef<OperationParser::ResultRecord> resultIDs,
-                    const AbstractOperation *opDefinition,
-                    OperationParser &parser)
-      : nameLoc(nameLoc), resultIDs(resultIDs), opDefinition(opDefinition),
+  CustomOpAsmParser(
+      SMLoc nameLoc, ArrayRef<OperationParser::ResultRecord> resultIDs,
+      function_ref<ParseResult(OpAsmParser &, OperationState &)> parseAssembly,
+      bool isIsolatedFromAbove, StringRef opName, OperationParser &parser)
+      : nameLoc(nameLoc), resultIDs(resultIDs), parseAssembly(parseAssembly),
+        isIsolatedFromAbove(isIsolatedFromAbove), opName(opName),
         parser(parser) {}
 
   /// Parse an instance of the operation described by 'opDefinition' into the
   /// provided operation state.
   ParseResult parseOperation(OperationState &opState) {
-    if (opDefinition->parseAssembly(*this, opState))
+    if (parseAssembly(*this, opState))
       return failure();
     // Verify that the parsed attributes does not have duplicate attributes.
     // This can happen if an attribute set during parsing is also specified in
@@ -929,8 +965,7 @@ public:
   /// Emit a diagnostic at the specified location and return failure.
   InFlightDiagnostic emitError(llvm::SMLoc loc, const Twine &message) override {
     emittedError = true;
-    return parser.emitError(loc, "custom op '" + opDefinition->name.strref() +
-                                     "' " + message);
+    return parser.emitError(loc, "custom op '" + opName + "' " + message);
   }
 
   llvm::SMLoc getCurrentLocation() override {
@@ -1455,8 +1490,7 @@ public:
     }
 
     // Try to parse the region.
-    assert((!enableNameShadowing ||
-            opDefinition->hasTrait<OpTrait::IsIsolatedFromAbove>()) &&
+    assert((!enableNameShadowing || isIsolatedFromAbove) &&
            "name shadowing is only allowed on isolated regions");
     if (parser.parseRegion(region, regionArguments, enableNameShadowing))
       return failure();
@@ -1621,7 +1655,9 @@ private:
   ArrayRef<OperationParser::ResultRecord> resultIDs;
 
   /// The abstract information of the operation.
-  const AbstractOperation *opDefinition;
+  function_ref<ParseResult(OpAsmParser &, OperationState &)> parseAssembly;
+  bool isIsolatedFromAbove;
+  StringRef opName;
 
   /// The main operation parser.
   OperationParser &parser;
@@ -1635,31 +1671,51 @@ Operation *
 OperationParser::parseCustomOperation(ArrayRef<ResultRecord> resultIDs) {
   llvm::SMLoc opLoc = getToken().getLoc();
   StringRef opName = getTokenSpelling();
-
   auto *opDefinition = AbstractOperation::lookup(opName, getContext());
-  if (!opDefinition) {
+  Dialect *dialect = nullptr;
+  if (opDefinition) {
+    dialect = &opDefinition->dialect;
+  } else {
     if (opName.contains('.')) {
       // This op has a dialect, we try to check if we can register it in the
       // context on the fly.
       StringRef dialectName = opName.split('.').first;
-      if (!getContext()->getLoadedDialect(dialectName) &&
-          getContext()->getOrLoadDialect(dialectName)) {
+      dialect = getContext()->getLoadedDialect(dialectName);
+      if (!dialect && (dialect = getContext()->getOrLoadDialect(dialectName)))
         opDefinition = AbstractOperation::lookup(opName, getContext());
-      }
     } else {
       // If the operation name has no namespace prefix we treat it as a standard
       // operation and prefix it with "std".
       // TODO: Would it be better to just build a mapping of the registered
       // operations in the standard dialect?
-      if (getContext()->getOrLoadDialect("std"))
+      if (getContext()->getOrLoadDialect("std")) {
         opDefinition = AbstractOperation::lookup(Twine("std." + opName).str(),
                                                  getContext());
+        if (opDefinition)
+          opName = opDefinition->name.strref();
+      }
     }
   }
 
-  if (!opDefinition) {
-    emitError(opLoc) << "custom op '" << opName << "' is unknown";
-    return nullptr;
+  // This is the actual hook for the custom op parsing, usually implemented by
+  // the op itself (`Op::parse()`). We retrieve it either from the
+  // AbstractOperation or from the Dialect.
+  std::function<ParseResult(OpAsmParser &, OperationState &)> parseAssemblyFn;
+  bool isIsolatedFromAbove = false;
+
+  if (opDefinition) {
+    parseAssemblyFn = opDefinition->getParseAssemblyFn();
+    isIsolatedFromAbove =
+        opDefinition->hasTrait<OpTrait::IsIsolatedFromAbove>();
+  } else {
+    Optional<Dialect::ParseOpHook> dialectHook;
+    if (dialect)
+      dialectHook = dialect->getParseOperationHook(opName);
+    if (!dialectHook.hasValue()) {
+      emitError(opLoc) << "custom op '" << opName << "' is unknown";
+      return nullptr;
+    }
+    parseAssemblyFn = *dialectHook;
   }
 
   consumeToken();
@@ -1674,9 +1730,10 @@ OperationParser::parseCustomOperation(ArrayRef<ResultRecord> resultIDs) {
   auto srcLocation = getEncodedSourceLocation(opLoc);
 
   // Have the op implementation take a crack and parsing this.
-  OperationState opState(srcLocation, opDefinition->name);
+  OperationState opState(srcLocation, opName);
   CleanupOpStateRegions guard{opState};
-  CustomOpAsmParser opAsmParser(opLoc, resultIDs, opDefinition, *this);
+  CustomOpAsmParser opAsmParser(opLoc, resultIDs, parseAssemblyFn,
+                                isIsolatedFromAbove, opName, *this);
   if (opAsmParser.parseOperation(opState))
     return nullptr;
 
@@ -2064,7 +2121,7 @@ ParseResult TopLevelOperationParser::parse(Block *topLevelBlock,
       auto &parsedOps = (*topLevelOp)->getRegion(0).front().getOperations();
       auto &destOps = topLevelBlock->getOperations();
       destOps.splice(destOps.empty() ? destOps.end() : std::prev(destOps.end()),
-                     parsedOps, parsedOps.begin(), std::prev(parsedOps.end()));
+                     parsedOps, parsedOps.begin(), parsedOps.end());
       return success();
     }
 
diff --git a/mlir/lib/Parser/Parser.h b/mlir/lib/Parser/Parser.h
index 0e9e4caff440ca0b35469e3d7190b3d06b94a570..8d1910e73843b72cc133025af462b8e47d2719f2 100644
--- a/mlir/lib/Parser/Parser.h
+++ b/mlir/lib/Parser/Parser.h
@@ -130,6 +130,12 @@ public:
   /// Parse an optional integer value from the stream.
   OptionalParseResult parseOptionalInteger(uint64_t &result);
 
+  /// Parse a floating point value from an integer literal token.
+  ParseResult parseFloatFromIntegerLiteral(Optional<APFloat> &result,
+                                           const Token &tok, bool isNegative,
+                                           const llvm::fltSemantics &semantics,
+                                           size_t typeSizeInBits);
+
   //===--------------------------------------------------------------------===//
   // Type Parsing
   //===--------------------------------------------------------------------===//
diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp
index 8507fd6d34510840e2df170ad76729713862051a..3f1f8ec0a7d082e1df7841381653e69ee3293bfc 100644
--- a/mlir/lib/Pass/Pass.cpp
+++ b/mlir/lib/Pass/Pass.cpp
@@ -381,6 +381,10 @@ LogicalResult OpToOpPassAdaptor::run(Pass *pass, Operation *op,
                 "nested under the current operation the pass is processing";
     assert(pipeline.getOpName() == root->getName().getStringRef());
 
+    // Before running, make sure to coalesce any adjacent pass adaptors in the
+    // pipeline.
+    pipeline.getImpl().coalesceAdjacentAdaptorPasses();
+
     // Initialize the user provided pipeline and execute the pipeline.
     if (failed(pipeline.initialize(root->getContext(), parentInitGeneration)))
       return failure();
@@ -860,7 +864,8 @@ void PassManager::enableVerifier(bool enabled) { verifyPasses = enabled; }
 LogicalResult PassManager::run(Operation *op) {
   MLIRContext *context = getContext();
   assert(op->getName().getIdentifier() == getOpName(*context) &&
-         "operation has a different name than the PassManager");
+         "operation has a different name than the PassManager or is from a "
+         "different context");
 
   // Before running, make sure to coalesce any adjacent pass adaptors in the
   // pipeline.
diff --git a/mlir/lib/Rewrite/ByteCode.cpp b/mlir/lib/Rewrite/ByteCode.cpp
index 1986b3f87d964b28facfbc5c59c5c6b97dc939ea..a81387f3f58e2799a919e4e1d518747fcc5abc1c 100644
--- a/mlir/lib/Rewrite/ByteCode.cpp
+++ b/mlir/lib/Rewrite/ByteCode.cpp
@@ -20,6 +20,9 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/FormatVariadic.h"
+#include <numeric>
 
 #define DEBUG_TYPE "pdl-bytecode"
 
@@ -42,10 +45,10 @@ PDLByteCodePattern PDLByteCodePattern::create(pdl_interp::RecordMatchOp matchOp,
 
   // Check to see if this is pattern matches a specific operation type.
   if (Optional<StringRef> rootKind = matchOp.rootKind())
-    return PDLByteCodePattern(rewriterAddr, *rootKind, generatedOps, benefit,
-                              ctx);
-  return PDLByteCodePattern(rewriterAddr, generatedOps, benefit, ctx,
-                            MatchAnyOpTypeTag());
+    return PDLByteCodePattern(rewriterAddr, *rootKind, benefit, ctx,
+                              generatedOps);
+  return PDLByteCodePattern(rewriterAddr, MatchAnyOpTypeTag(), benefit, ctx,
+                            generatedOps);
 }
 
 //===----------------------------------------------------------------------===//
@@ -60,6 +63,14 @@ void PDLByteCodeMutableState::updatePatternBenefit(unsigned patternIndex,
   currentPatternBenefits[patternIndex] = benefit;
 }
 
+/// Cleanup any allocated state after a full match/rewrite has been completed.
+/// This method should be called irregardless of whether the match+rewrite was a
+/// success or not.
+void PDLByteCodeMutableState::cleanupAfterMatchAndRewrite() {
+  allocatedTypeRangeMemory.clear();
+  allocatedValueRangeMemory.clear();
+}
+
 //===----------------------------------------------------------------------===//
 // Bytecode OpCodes
 //===----------------------------------------------------------------------===//
@@ -72,6 +83,8 @@ enum OpCode : ByteCodeField {
   ApplyRewrite,
   /// Check if two generic values are equal.
   AreEqual,
+  /// Check if two ranges are equal.
+  AreRangesEqual,
   /// Unconditional branch.
   Branch,
   /// Compare the operand count of an operation with a constant.
@@ -80,10 +93,12 @@ enum OpCode : ByteCodeField {
   CheckOperationName,
   /// Compare the result count of an operation with a constant.
   CheckResultCount,
-  /// Invoke a native creation method.
-  CreateNative,
+  /// Compare a range of types to a constant range of types.
+  CheckTypes,
   /// Create an operation.
   CreateOperation,
+  /// Create a range of types.
+  CreateTypes,
   /// Erase an operation.
   EraseOp,
   /// Terminate a matcher or rewrite sequence.
@@ -100,14 +115,20 @@ enum OpCode : ByteCodeField {
   GetOperand2,
   GetOperand3,
   GetOperandN,
+  /// Get a specific operand group of an operation.
+  GetOperands,
   /// Get a specific result of an operation.
   GetResult0,
   GetResult1,
   GetResult2,
   GetResult3,
   GetResultN,
+  /// Get a specific result group of an operation.
+  GetResults,
   /// Get the type of a value.
   GetValueType,
+  /// Get the types of a value range.
+  GetValueRangeTypes,
   /// Check if a generic value is not null.
   IsNotNull,
   /// Record a successful pattern match.
@@ -124,9 +145,9 @@ enum OpCode : ByteCodeField {
   SwitchResultCount,
   /// Compare a type with a set of constants.
   SwitchType,
+  /// Compare a range of types with a set of constants.
+  SwitchTypes,
 };
-
-enum class PDLValueKind { Attribute, Operation, Type, Value };
 } // end anonymous namespace
 
 //===----------------------------------------------------------------------===//
@@ -147,16 +168,17 @@ public:
             SmallVectorImpl<ByteCodeField> &rewriterByteCode,
             SmallVectorImpl<PDLByteCodePattern> &patterns,
             ByteCodeField &maxValueMemoryIndex,
+            ByteCodeField &maxTypeRangeMemoryIndex,
+            ByteCodeField &maxValueRangeMemoryIndex,
             llvm::StringMap<PDLConstraintFunction> &constraintFns,
-            llvm::StringMap<PDLCreateFunction> &createFns,
             llvm::StringMap<PDLRewriteFunction> &rewriteFns)
       : ctx(ctx), uniquedData(uniquedData), matcherByteCode(matcherByteCode),
         rewriterByteCode(rewriterByteCode), patterns(patterns),
-        maxValueMemoryIndex(maxValueMemoryIndex) {
+        maxValueMemoryIndex(maxValueMemoryIndex),
+        maxTypeRangeMemoryIndex(maxTypeRangeMemoryIndex),
+        maxValueRangeMemoryIndex(maxValueRangeMemoryIndex) {
     for (auto it : llvm::enumerate(constraintFns))
       constraintToMemIndex.try_emplace(it.value().first(), it.index());
-    for (auto it : llvm::enumerate(createFns))
-      nativeCreateToMemIndex.try_emplace(it.value().first(), it.index());
     for (auto it : llvm::enumerate(rewriteFns))
       externalRewriterToMemIndex.try_emplace(it.value().first(), it.index());
   }
@@ -171,6 +193,13 @@ public:
     return valueToMemIndex[value];
   }
 
+  /// Return the range memory index used to store the given range value.
+  ByteCodeField &getRangeStorageIndex(Value value) {
+    assert(valueToRangeIndex.count(value) &&
+           "expected range index to be assigned");
+    return valueToRangeIndex[value];
+  }
+
   /// Return an index to use when referring to the given data that is uniqued in
   /// the MLIR context.
   template <typename T>
@@ -202,24 +231,28 @@ private:
   void generate(pdl_interp::CheckOperationNameOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::CheckResultCountOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::CheckTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CheckTypesOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::CreateAttributeOp op, ByteCodeWriter &writer);
-  void generate(pdl_interp::CreateNativeOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::CreateOperationOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::CreateTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CreateTypesOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::EraseOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::FinalizeOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::GetAttributeOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::GetAttributeTypeOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::GetDefiningOpOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::GetOperandOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::GetOperandsOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::GetResultOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::GetResultsOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::GetValueTypeOp op, ByteCodeWriter &writer);
-  void generate(pdl_interp::InferredTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::InferredTypesOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::IsNotNullOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::RecordMatchOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::ReplaceOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::SwitchAttributeOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::SwitchTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::SwitchTypesOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::SwitchOperandCountOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::SwitchOperationNameOp op, ByteCodeWriter &writer);
   void generate(pdl_interp::SwitchResultCountOp op, ByteCodeWriter &writer);
@@ -227,6 +260,9 @@ private:
   /// Mapping from value to its corresponding memory index.
   DenseMap<Value, ByteCodeField> valueToMemIndex;
 
+  /// Mapping from a range value to its corresponding range storage index.
+  DenseMap<Value, ByteCodeField> valueToRangeIndex;
+
   /// Mapping from the name of an externally registered rewrite to its index in
   /// the bytecode registry.
   llvm::StringMap<ByteCodeField> externalRewriterToMemIndex;
@@ -235,10 +271,6 @@ private:
   /// in the bytecode registry.
   llvm::StringMap<ByteCodeField> constraintToMemIndex;
 
-  /// Mapping from the name of an externally registered creation method to its
-  /// index in the bytecode registry.
-  llvm::StringMap<ByteCodeField> nativeCreateToMemIndex;
-
   /// Mapping from rewriter function name to the bytecode address of the
   /// rewriter function in byte.
   llvm::StringMap<ByteCodeAddr> rewriterToAddr;
@@ -256,6 +288,8 @@ private:
   SmallVectorImpl<ByteCodeField> &rewriterByteCode;
   SmallVectorImpl<PDLByteCodePattern> &patterns;
   ByteCodeField &maxValueMemoryIndex;
+  ByteCodeField &maxTypeRangeMemoryIndex;
+  ByteCodeField &maxValueRangeMemoryIndex;
 };
 
 /// This class provides utilities for writing a bytecode stream.
@@ -291,19 +325,33 @@ struct ByteCodeWriter {
   /// Append a range of values that will be read as generic PDLValues.
   void appendPDLValueList(OperandRange values) {
     bytecode.push_back(values.size());
-    for (Value value : values) {
-      // Append the type of the value in addition to the value itself.
-      PDLValueKind kind =
-          TypeSwitch<Type, PDLValueKind>(value.getType())
-              .Case<pdl::AttributeType>(
-                  [](Type) { return PDLValueKind::Attribute; })
-              .Case<pdl::OperationType>(
-                  [](Type) { return PDLValueKind::Operation; })
-              .Case<pdl::TypeType>([](Type) { return PDLValueKind::Type; })
-              .Case<pdl::ValueType>([](Type) { return PDLValueKind::Value; });
-      bytecode.push_back(static_cast<ByteCodeField>(kind));
-      append(value);
-    }
+    for (Value value : values)
+      appendPDLValue(value);
+  }
+
+  /// Append a value as a PDLValue.
+  void appendPDLValue(Value value) {
+    appendPDLValueKind(value);
+    append(value);
+  }
+
+  /// Append the PDLValue::Kind of the given value.
+  void appendPDLValueKind(Value value) {
+    // Append the type of the value in addition to the value itself.
+    PDLValue::Kind kind =
+        TypeSwitch<Type, PDLValue::Kind>(value.getType())
+            .Case<pdl::AttributeType>(
+                [](Type) { return PDLValue::Kind::Attribute; })
+            .Case<pdl::OperationType>(
+                [](Type) { return PDLValue::Kind::Operation; })
+            .Case<pdl::RangeType>([](pdl::RangeType rangeTy) {
+              if (rangeTy.getElementType().isa<pdl::TypeType>())
+                return PDLValue::Kind::TypeRange;
+              return PDLValue::Kind::ValueRange;
+            })
+            .Case<pdl::TypeType>([](Type) { return PDLValue::Kind::Type; })
+            .Case<pdl::ValueType>([](Type) { return PDLValue::Kind::Value; });
+    bytecode.push_back(static_cast<ByteCodeField>(kind));
   }
 
   /// Check if the given class `T` has an iterator type.
@@ -344,6 +392,36 @@ struct ByteCodeWriter {
   /// The main generator producing PDL.
   Generator &generator;
 };
+
+/// This class represents a live range of PDL Interpreter values, containing
+/// information about when values are live within a match/rewrite.
+struct ByteCodeLiveRange {
+  using Set = llvm::IntervalMap<ByteCodeField, char, 16>;
+  using Allocator = Set::Allocator;
+
+  ByteCodeLiveRange(Allocator &alloc) : liveness(alloc) {}
+
+  /// Union this live range with the one provided.
+  void unionWith(const ByteCodeLiveRange &rhs) {
+    for (auto it = rhs.liveness.begin(), e = rhs.liveness.end(); it != e; ++it)
+      liveness.insert(it.start(), it.stop(), /*dummyValue*/ 0);
+  }
+
+  /// Returns true if this range overlaps with the one provided.
+  bool overlaps(const ByteCodeLiveRange &rhs) const {
+    return llvm::IntervalMapOverlaps<Set, Set>(liveness, rhs.liveness).valid();
+  }
+
+  /// A map representing the ranges of the match/rewrite that a value is live in
+  /// the interpreter.
+  llvm::IntervalMap<ByteCodeField, char, 16> liveness;
+
+  /// The type range storage index for this range.
+  Optional<unsigned> typeRangeIndex;
+
+  /// The value range storage index for this range.
+  Optional<unsigned> valueRangeIndex;
+};
 } // end anonymous namespace
 
 void Generator::generate(ModuleOp module) {
@@ -391,15 +469,30 @@ void Generator::allocateMemoryIndices(FuncOp matcherFunc,
   // Rewriters use simplistic allocation scheme that simply assigns an index to
   // each result.
   for (FuncOp rewriterFunc : rewriterModule.getOps<FuncOp>()) {
-    ByteCodeField index = 0;
+    ByteCodeField index = 0, typeRangeIndex = 0, valueRangeIndex = 0;
+    auto processRewriterValue = [&](Value val) {
+      valueToMemIndex.try_emplace(val, index++);
+      if (pdl::RangeType rangeType = val.getType().dyn_cast<pdl::RangeType>()) {
+        Type elementTy = rangeType.getElementType();
+        if (elementTy.isa<pdl::TypeType>())
+          valueToRangeIndex.try_emplace(val, typeRangeIndex++);
+        else if (elementTy.isa<pdl::ValueType>())
+          valueToRangeIndex.try_emplace(val, valueRangeIndex++);
+      }
+    };
+
     for (BlockArgument arg : rewriterFunc.getArguments())
-      valueToMemIndex.try_emplace(arg, index++);
+      processRewriterValue(arg);
     rewriterFunc.getBody().walk([&](Operation *op) {
       for (Value result : op->getResults())
-        valueToMemIndex.try_emplace(result, index++);
+        processRewriterValue(result);
     });
     if (index > maxValueMemoryIndex)
       maxValueMemoryIndex = index;
+    if (typeRangeIndex > maxTypeRangeMemoryIndex)
+      maxTypeRangeMemoryIndex = typeRangeIndex;
+    if (valueRangeIndex > maxValueRangeMemoryIndex)
+      maxValueRangeMemoryIndex = valueRangeIndex;
   }
 
   // The matcher function uses a more sophisticated numbering that tries to
@@ -414,9 +507,8 @@ void Generator::allocateMemoryIndices(FuncOp matcherFunc,
   });
 
   // Liveness info for each of the defs within the matcher.
-  using LivenessSet = llvm::IntervalMap<ByteCodeField, char, 16>;
-  LivenessSet::Allocator allocator;
-  DenseMap<Value, LivenessSet> valueDefRanges;
+  ByteCodeLiveRange::Allocator allocator;
+  DenseMap<Value, ByteCodeLiveRange> valueDefRanges;
 
   // Assign the root operation being matched to slot 0.
   BlockArgument rootOpArg = matcherFunc.getArgument(0);
@@ -435,10 +527,19 @@ void Generator::allocateMemoryIndices(FuncOp matcherFunc,
 
       // Set indices for the range of this block that the value is used.
       auto defRangeIt = valueDefRanges.try_emplace(value, allocator).first;
-      defRangeIt->second.insert(
+      defRangeIt->second.liveness.insert(
           opToIndex[firstUseOrDef],
           opToIndex[info->getEndOperation(value, firstUseOrDef)],
           /*dummyValue*/ 0);
+
+      // Check to see if this value is a range type.
+      if (auto rangeTy = value.getType().dyn_cast<pdl::RangeType>()) {
+        Type eleType = rangeTy.getElementType();
+        if (eleType.isa<pdl::TypeType>())
+          defRangeIt->second.typeRangeIndex = 0;
+        else if (eleType.isa<pdl::ValueType>())
+          defRangeIt->second.valueRangeIndex = 0;
+      }
     };
 
     // Process the live-ins of this block.
@@ -452,37 +553,59 @@ void Generator::allocateMemoryIndices(FuncOp matcherFunc,
   }
 
   // Greedily allocate memory slots using the computed def live ranges.
-  std::vector<LivenessSet> allocatedIndices;
+  std::vector<ByteCodeLiveRange> allocatedIndices;
+  ByteCodeField numIndices = 1, numTypeRanges = 0, numValueRanges = 0;
   for (auto &defIt : valueDefRanges) {
     ByteCodeField &memIndex = valueToMemIndex[defIt.first];
-    LivenessSet &defSet = defIt.second;
+    ByteCodeLiveRange &defRange = defIt.second;
 
     // Try to allocate to an existing index.
     for (auto existingIndexIt : llvm::enumerate(allocatedIndices)) {
-      LivenessSet &existingIndex = existingIndexIt.value();
-      llvm::IntervalMapOverlaps<LivenessSet, LivenessSet> overlaps(
-          defIt.second, existingIndex);
-      if (overlaps.valid())
-        continue;
-      // Union the range of the def within the existing index.
-      for (auto it = defSet.begin(), e = defSet.end(); it != e; ++it)
-        existingIndex.insert(it.start(), it.stop(), /*dummyValue*/ 0);
-      memIndex = existingIndexIt.index() + 1;
+      ByteCodeLiveRange &existingRange = existingIndexIt.value();
+      if (!defRange.overlaps(existingRange)) {
+        existingRange.unionWith(defRange);
+        memIndex = existingIndexIt.index() + 1;
+
+        if (defRange.typeRangeIndex) {
+          if (!existingRange.typeRangeIndex)
+            existingRange.typeRangeIndex = numTypeRanges++;
+          valueToRangeIndex[defIt.first] = *existingRange.typeRangeIndex;
+        } else if (defRange.valueRangeIndex) {
+          if (!existingRange.valueRangeIndex)
+            existingRange.valueRangeIndex = numValueRanges++;
+          valueToRangeIndex[defIt.first] = *existingRange.valueRangeIndex;
+        }
+        break;
+      }
     }
 
     // If no existing index could be used, add a new one.
     if (memIndex == 0) {
       allocatedIndices.emplace_back(allocator);
-      for (auto it = defSet.begin(), e = defSet.end(); it != e; ++it)
-        allocatedIndices.back().insert(it.start(), it.stop(), /*dummyValue*/ 0);
+      ByteCodeLiveRange &newRange = allocatedIndices.back();
+      newRange.unionWith(defRange);
+
+      // Allocate an index for type/value ranges.
+      if (defRange.typeRangeIndex) {
+        newRange.typeRangeIndex = numTypeRanges;
+        valueToRangeIndex[defIt.first] = numTypeRanges++;
+      } else if (defRange.valueRangeIndex) {
+        newRange.valueRangeIndex = numValueRanges;
+        valueToRangeIndex[defIt.first] = numValueRanges++;
+      }
+
       memIndex = allocatedIndices.size();
+      ++numIndices;
     }
   }
 
   // Update the max number of indices.
-  ByteCodeField numMatcherIndices = allocatedIndices.size() + 1;
-  if (numMatcherIndices > maxValueMemoryIndex)
-    maxValueMemoryIndex = numMatcherIndices;
+  if (numIndices > maxValueMemoryIndex)
+    maxValueMemoryIndex = numIndices;
+  if (numTypeRanges > maxTypeRangeMemoryIndex)
+    maxTypeRangeMemoryIndex = numTypeRanges;
+  if (numValueRanges > maxValueRangeMemoryIndex)
+    maxValueRangeMemoryIndex = numValueRanges;
 }
 
 void Generator::generate(Operation *op, ByteCodeWriter &writer) {
@@ -491,16 +614,18 @@ void Generator::generate(Operation *op, ByteCodeWriter &writer) {
             pdl_interp::AreEqualOp, pdl_interp::BranchOp,
             pdl_interp::CheckAttributeOp, pdl_interp::CheckOperandCountOp,
             pdl_interp::CheckOperationNameOp, pdl_interp::CheckResultCountOp,
-            pdl_interp::CheckTypeOp, pdl_interp::CreateAttributeOp,
-            pdl_interp::CreateNativeOp, pdl_interp::CreateOperationOp,
-            pdl_interp::CreateTypeOp, pdl_interp::EraseOp,
-            pdl_interp::FinalizeOp, pdl_interp::GetAttributeOp,
-            pdl_interp::GetAttributeTypeOp, pdl_interp::GetDefiningOpOp,
-            pdl_interp::GetOperandOp, pdl_interp::GetResultOp,
-            pdl_interp::GetValueTypeOp, pdl_interp::InferredTypeOp,
-            pdl_interp::IsNotNullOp, pdl_interp::RecordMatchOp,
-            pdl_interp::ReplaceOp, pdl_interp::SwitchAttributeOp,
-            pdl_interp::SwitchTypeOp, pdl_interp::SwitchOperandCountOp,
+            pdl_interp::CheckTypeOp, pdl_interp::CheckTypesOp,
+            pdl_interp::CreateAttributeOp, pdl_interp::CreateOperationOp,
+            pdl_interp::CreateTypeOp, pdl_interp::CreateTypesOp,
+            pdl_interp::EraseOp, pdl_interp::FinalizeOp,
+            pdl_interp::GetAttributeOp, pdl_interp::GetAttributeTypeOp,
+            pdl_interp::GetDefiningOpOp, pdl_interp::GetOperandOp,
+            pdl_interp::GetOperandsOp, pdl_interp::GetResultOp,
+            pdl_interp::GetResultsOp, pdl_interp::GetValueTypeOp,
+            pdl_interp::InferredTypesOp, pdl_interp::IsNotNullOp,
+            pdl_interp::RecordMatchOp, pdl_interp::ReplaceOp,
+            pdl_interp::SwitchAttributeOp, pdl_interp::SwitchTypeOp,
+            pdl_interp::SwitchTypesOp, pdl_interp::SwitchOperandCountOp,
             pdl_interp::SwitchOperationNameOp, pdl_interp::SwitchResultCountOp>(
           [&](auto interpOp) { this->generate(interpOp, writer); })
       .Default([](Operation *) {
@@ -522,11 +647,34 @@ void Generator::generate(pdl_interp::ApplyRewriteOp op,
   assert(externalRewriterToMemIndex.count(op.name()) &&
          "expected index for rewrite function");
   writer.append(OpCode::ApplyRewrite, externalRewriterToMemIndex[op.name()],
-                op.constParamsAttr(), op.root());
+                op.constParamsAttr());
   writer.appendPDLValueList(op.args());
+
+  ResultRange results = op.results();
+  writer.append(ByteCodeField(results.size()));
+  for (Value result : results) {
+    // In debug mode we also record the expected kind of the result, so that we
+    // can provide extra verification of the native rewrite function.
+#ifndef NDEBUG
+    writer.appendPDLValueKind(result);
+#endif
+
+    // Range results also need to append the range storage index.
+    if (result.getType().isa<pdl::RangeType>())
+      writer.append(getRangeStorageIndex(result));
+    writer.append(result);
+  }
 }
 void Generator::generate(pdl_interp::AreEqualOp op, ByteCodeWriter &writer) {
-  writer.append(OpCode::AreEqual, op.lhs(), op.rhs(), op.getSuccessors());
+  Value lhs = op.lhs();
+  if (lhs.getType().isa<pdl::RangeType>()) {
+    writer.append(OpCode::AreRangesEqual);
+    writer.appendPDLValueKind(lhs);
+    writer.append(op.lhs(), op.rhs(), op.getSuccessors());
+    return;
+  }
+
+  writer.append(OpCode::AreEqual, lhs, op.rhs(), op.getSuccessors());
 }
 void Generator::generate(pdl_interp::BranchOp op, ByteCodeWriter &writer) {
   writer.append(OpCode::Branch, SuccessorRange(op.getOperation()));
@@ -539,6 +687,7 @@ void Generator::generate(pdl_interp::CheckAttributeOp op,
 void Generator::generate(pdl_interp::CheckOperandCountOp op,
                          ByteCodeWriter &writer) {
   writer.append(OpCode::CheckOperandCount, op.operation(), op.count(),
+                static_cast<ByteCodeField>(op.compareAtLeast()),
                 op.getSuccessors());
 }
 void Generator::generate(pdl_interp::CheckOperationNameOp op,
@@ -549,28 +698,25 @@ void Generator::generate(pdl_interp::CheckOperationNameOp op,
 void Generator::generate(pdl_interp::CheckResultCountOp op,
                          ByteCodeWriter &writer) {
   writer.append(OpCode::CheckResultCount, op.operation(), op.count(),
+                static_cast<ByteCodeField>(op.compareAtLeast()),
                 op.getSuccessors());
 }
 void Generator::generate(pdl_interp::CheckTypeOp op, ByteCodeWriter &writer) {
   writer.append(OpCode::AreEqual, op.value(), op.type(), op.getSuccessors());
 }
+void Generator::generate(pdl_interp::CheckTypesOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::CheckTypes, op.value(), op.types(), op.getSuccessors());
+}
 void Generator::generate(pdl_interp::CreateAttributeOp op,
                          ByteCodeWriter &writer) {
   // Simply repoint the memory index of the result to the constant.
   getMemIndex(op.attribute()) = getMemIndex(op.value());
 }
-void Generator::generate(pdl_interp::CreateNativeOp op,
-                         ByteCodeWriter &writer) {
-  assert(nativeCreateToMemIndex.count(op.name()) &&
-         "expected index for creation function");
-  writer.append(OpCode::CreateNative, nativeCreateToMemIndex[op.name()],
-                op.result(), op.constParamsAttr());
-  writer.appendPDLValueList(op.args());
-}
 void Generator::generate(pdl_interp::CreateOperationOp op,
                          ByteCodeWriter &writer) {
   writer.append(OpCode::CreateOperation, op.operation(),
-                OperationName(op.name(), ctx), op.operands());
+                OperationName(op.name(), ctx));
+  writer.appendPDLValueList(op.operands());
 
   // Add the attributes.
   OperandRange attributes = op.attributes();
@@ -580,12 +726,16 @@ void Generator::generate(pdl_interp::CreateOperationOp op,
         Identifier::get(std::get<0>(it).cast<StringAttr>().getValue(), ctx),
         std::get<1>(it));
   }
-  writer.append(op.types());
+  writer.appendPDLValueList(op.types());
 }
 void Generator::generate(pdl_interp::CreateTypeOp op, ByteCodeWriter &writer) {
   // Simply repoint the memory index of the result to the constant.
   getMemIndex(op.result()) = getMemIndex(op.value());
 }
+void Generator::generate(pdl_interp::CreateTypesOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::CreateTypes, op.result(),
+                getRangeStorageIndex(op.result()), op.value());
+}
 void Generator::generate(pdl_interp::EraseOp op, ByteCodeWriter &writer) {
   writer.append(OpCode::EraseOp, op.operation());
 }
@@ -603,7 +753,8 @@ void Generator::generate(pdl_interp::GetAttributeTypeOp op,
 }
 void Generator::generate(pdl_interp::GetDefiningOpOp op,
                          ByteCodeWriter &writer) {
-  writer.append(OpCode::GetDefiningOp, op.operation(), op.value());
+  writer.append(OpCode::GetDefiningOp, op.operation());
+  writer.appendPDLValue(op.value());
 }
 void Generator::generate(pdl_interp::GetOperandOp op, ByteCodeWriter &writer) {
   uint32_t index = op.index();
@@ -613,6 +764,18 @@ void Generator::generate(pdl_interp::GetOperandOp op, ByteCodeWriter &writer) {
     writer.append(OpCode::GetOperandN, index);
   writer.append(op.operation(), op.value());
 }
+void Generator::generate(pdl_interp::GetOperandsOp op, ByteCodeWriter &writer) {
+  Value result = op.value();
+  Optional<uint32_t> index = op.index();
+  writer.append(OpCode::GetOperands,
+                index.getValueOr(std::numeric_limits<uint32_t>::max()),
+                op.operation());
+  if (result.getType().isa<pdl::RangeType>())
+    writer.append(getRangeStorageIndex(result));
+  else
+    writer.append(std::numeric_limits<ByteCodeField>::max());
+  writer.append(result);
+}
 void Generator::generate(pdl_interp::GetResultOp op, ByteCodeWriter &writer) {
   uint32_t index = op.index();
   if (index < 4)
@@ -621,13 +784,32 @@ void Generator::generate(pdl_interp::GetResultOp op, ByteCodeWriter &writer) {
     writer.append(OpCode::GetResultN, index);
   writer.append(op.operation(), op.value());
 }
+void Generator::generate(pdl_interp::GetResultsOp op, ByteCodeWriter &writer) {
+  Value result = op.value();
+  Optional<uint32_t> index = op.index();
+  writer.append(OpCode::GetResults,
+                index.getValueOr(std::numeric_limits<uint32_t>::max()),
+                op.operation());
+  if (result.getType().isa<pdl::RangeType>())
+    writer.append(getRangeStorageIndex(result));
+  else
+    writer.append(std::numeric_limits<ByteCodeField>::max());
+  writer.append(result);
+}
 void Generator::generate(pdl_interp::GetValueTypeOp op,
                          ByteCodeWriter &writer) {
-  writer.append(OpCode::GetValueType, op.result(), op.value());
+  if (op.getType().isa<pdl::RangeType>()) {
+    Value result = op.result();
+    writer.append(OpCode::GetValueRangeTypes, result,
+                  getRangeStorageIndex(result), op.value());
+  } else {
+    writer.append(OpCode::GetValueType, op.result(), op.value());
+  }
 }
-void Generator::generate(pdl_interp::InferredTypeOp op,
+
+void Generator::generate(pdl_interp::InferredTypesOp op,
                          ByteCodeWriter &writer) {
-  // InferType maps to a null type as a marker for inferring a result type.
+  // InferType maps to a null type as a marker for inferring result types.
   getMemIndex(op.type()) = getMemIndex(Type());
 }
 void Generator::generate(pdl_interp::IsNotNullOp op, ByteCodeWriter &writer) {
@@ -638,11 +820,12 @@ void Generator::generate(pdl_interp::RecordMatchOp op, ByteCodeWriter &writer) {
   patterns.emplace_back(PDLByteCodePattern::create(
       op, rewriterToAddr[op.rewriter().getLeafReference()]));
   writer.append(OpCode::RecordMatch, patternIndex,
-                SuccessorRange(op.getOperation()), op.matchedOps(),
-                op.inputs());
+                SuccessorRange(op.getOperation()), op.matchedOps());
+  writer.appendPDLValueList(op.inputs());
 }
 void Generator::generate(pdl_interp::ReplaceOp op, ByteCodeWriter &writer) {
-  writer.append(OpCode::ReplaceOp, op.operation(), op.replValues());
+  writer.append(OpCode::ReplaceOp, op.operation());
+  writer.appendPDLValueList(op.replValues());
 }
 void Generator::generate(pdl_interp::SwitchAttributeOp op,
                          ByteCodeWriter &writer) {
@@ -671,6 +854,10 @@ void Generator::generate(pdl_interp::SwitchTypeOp op, ByteCodeWriter &writer) {
   writer.append(OpCode::SwitchType, op.value(), op.caseValuesAttr(),
                 op.getSuccessors());
 }
+void Generator::generate(pdl_interp::SwitchTypesOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::SwitchTypes, op.value(), op.caseValuesAttr(),
+                op.getSuccessors());
+}
 
 //===----------------------------------------------------------------------===//
 // PDLByteCode
@@ -678,18 +865,16 @@ void Generator::generate(pdl_interp::SwitchTypeOp op, ByteCodeWriter &writer) {
 
 PDLByteCode::PDLByteCode(ModuleOp module,
                          llvm::StringMap<PDLConstraintFunction> constraintFns,
-                         llvm::StringMap<PDLCreateFunction> createFns,
                          llvm::StringMap<PDLRewriteFunction> rewriteFns) {
   Generator generator(module.getContext(), uniquedData, matcherByteCode,
                       rewriterByteCode, patterns, maxValueMemoryIndex,
-                      constraintFns, createFns, rewriteFns);
+                      maxTypeRangeCount, maxValueRangeCount, constraintFns,
+                      rewriteFns);
   generator.generate(module);
 
   // Initialize the external functions.
   for (auto &it : constraintFns)
     constraintFunctions.push_back(std::move(it.second));
-  for (auto &it : createFns)
-    createFunctions.push_back(std::move(it.second));
   for (auto &it : rewriteFns)
     rewriteFunctions.push_back(std::move(it.second));
 }
@@ -698,6 +883,8 @@ PDLByteCode::PDLByteCode(ModuleOp module,
 /// bytecode.
 void PDLByteCode::initializeMutableState(PDLByteCodeMutableState &state) const {
   state.memory.resize(maxValueMemoryIndex, nullptr);
+  state.typeRangeMemory.resize(maxTypeRangeCount, TypeRange());
+  state.valueRangeMemory.resize(maxValueRangeCount, ValueRange());
   state.currentPatternBenefits.reserve(patterns.size());
   for (const PDLByteCodePattern &pattern : patterns)
     state.currentPatternBenefits.push_back(pattern.getBenefit());
@@ -710,19 +897,25 @@ namespace {
 /// This class provides support for executing a bytecode stream.
 class ByteCodeExecutor {
 public:
-  ByteCodeExecutor(const ByteCodeField *curCodeIt,
-                   MutableArrayRef<const void *> memory,
-                   ArrayRef<const void *> uniquedMemory,
-                   ArrayRef<ByteCodeField> code,
-                   ArrayRef<PatternBenefit> currentPatternBenefits,
-                   ArrayRef<PDLByteCodePattern> patterns,
-                   ArrayRef<PDLConstraintFunction> constraintFunctions,
-                   ArrayRef<PDLCreateFunction> createFunctions,
-                   ArrayRef<PDLRewriteFunction> rewriteFunctions)
-      : curCodeIt(curCodeIt), memory(memory), uniquedMemory(uniquedMemory),
-        code(code), currentPatternBenefits(currentPatternBenefits),
-        patterns(patterns), constraintFunctions(constraintFunctions),
-        createFunctions(createFunctions), rewriteFunctions(rewriteFunctions) {}
+  ByteCodeExecutor(
+      const ByteCodeField *curCodeIt, MutableArrayRef<const void *> memory,
+      MutableArrayRef<TypeRange> typeRangeMemory,
+      std::vector<llvm::OwningArrayRef<Type>> &allocatedTypeRangeMemory,
+      MutableArrayRef<ValueRange> valueRangeMemory,
+      std::vector<llvm::OwningArrayRef<Value>> &allocatedValueRangeMemory,
+      ArrayRef<const void *> uniquedMemory, ArrayRef<ByteCodeField> code,
+      ArrayRef<PatternBenefit> currentPatternBenefits,
+      ArrayRef<PDLByteCodePattern> patterns,
+      ArrayRef<PDLConstraintFunction> constraintFunctions,
+      ArrayRef<PDLRewriteFunction> rewriteFunctions)
+      : curCodeIt(curCodeIt), memory(memory), typeRangeMemory(typeRangeMemory),
+        allocatedTypeRangeMemory(allocatedTypeRangeMemory),
+        valueRangeMemory(valueRangeMemory),
+        allocatedValueRangeMemory(allocatedValueRangeMemory),
+        uniquedMemory(uniquedMemory), code(code),
+        currentPatternBenefits(currentPatternBenefits), patterns(patterns),
+        constraintFunctions(constraintFunctions),
+        rewriteFunctions(rewriteFunctions) {}
 
   /// Start executing the code at the current bytecode index. `matches` is an
   /// optional field provided when this function is executed in a matching
@@ -736,20 +929,25 @@ private:
   void executeApplyConstraint(PatternRewriter &rewriter);
   void executeApplyRewrite(PatternRewriter &rewriter);
   void executeAreEqual();
+  void executeAreRangesEqual();
   void executeBranch();
   void executeCheckOperandCount();
   void executeCheckOperationName();
   void executeCheckResultCount();
-  void executeCreateNative(PatternRewriter &rewriter);
+  void executeCheckTypes();
   void executeCreateOperation(PatternRewriter &rewriter,
                               Location mainRewriteLoc);
+  void executeCreateTypes();
   void executeEraseOp(PatternRewriter &rewriter);
   void executeGetAttribute();
   void executeGetAttributeType();
   void executeGetDefiningOp();
   void executeGetOperand(unsigned index);
+  void executeGetOperands();
   void executeGetResult(unsigned index);
+  void executeGetResults();
   void executeGetValueType();
+  void executeGetValueRangeTypes();
   void executeIsNotNull();
   void executeRecordMatch(PatternRewriter &rewriter,
                           SmallVectorImpl<PDLByteCode::MatchResult> &matches);
@@ -759,6 +957,7 @@ private:
   void executeSwitchOperationName();
   void executeSwitchResultCount();
   void executeSwitchType();
+  void executeSwitchTypes();
 
   /// Read a value from the bytecode buffer, optionally skipping a certain
   /// number of prefix values. These methods always update the buffer to point
@@ -778,6 +977,19 @@ private:
       list.push_back(read<ValueT>());
   }
 
+  /// Read a list of values from the bytecode buffer. The values may be encoded
+  /// as either Value or ValueRange elements.
+  void readValueList(SmallVectorImpl<Value> &list) {
+    for (unsigned i = 0, e = read(); i != e; ++i) {
+      if (read<PDLValue::Kind>() == PDLValue::Kind::Value) {
+        list.push_back(read<Value>());
+      } else {
+        ValueRange *values = read<ValueRange *>();
+        list.append(values->begin(), values->end());
+      }
+    }
+  }
+
   /// Jump to a specific successor based on a predicate value.
   void selectJump(bool isTrue) { selectJump(size_t(isTrue ? 0 : 1)); }
   /// Jump to a specific successor based on a destination index.
@@ -786,8 +998,8 @@ private:
   }
 
   /// Handle a switch operation with the provided value and cases.
-  template <typename T, typename RangeT>
-  void handleSwitch(const T &value, RangeT &&cases) {
+  template <typename T, typename RangeT, typename Comparator = std::equal_to<T>>
+  void handleSwitch(const T &value, RangeT &&cases, Comparator cmp = {}) {
     LLVM_DEBUG({
       llvm::dbgs() << "  * Value: " << value << "\n"
                    << "  * Cases: ";
@@ -798,7 +1010,7 @@ private:
     // Check to see if the attribute value is within the case list. Jump to
     // the correct successor index based on the result.
     for (auto it = cases.begin(), e = cases.end(); it != e; ++it)
-      if (*it == value)
+      if (cmp(*it, value))
         return selectJump(size_t((it - cases.begin()) + 1));
     selectJump(size_t(0));
   }
@@ -810,7 +1022,9 @@ private:
     size_t index = *curCodeIt++;
 
     // If this type is an SSA value, it can only be stored in non-const memory.
-    if (llvm::is_one_of<T, Operation *, Value>::value || index < memory.size())
+    if (llvm::is_one_of<T, Operation *, TypeRange *, ValueRange *,
+                        Value>::value ||
+        index < memory.size())
       return memory[index];
 
     // Otherwise, if this index is not inbounds it is uniqued.
@@ -828,17 +1042,21 @@ private:
   }
   template <typename T>
   std::enable_if_t<std::is_same<PDLValue, T>::value, T> readImpl() {
-    switch (static_cast<PDLValueKind>(read())) {
-    case PDLValueKind::Attribute:
+    switch (read<PDLValue::Kind>()) {
+    case PDLValue::Kind::Attribute:
       return read<Attribute>();
-    case PDLValueKind::Operation:
+    case PDLValue::Kind::Operation:
       return read<Operation *>();
-    case PDLValueKind::Type:
+    case PDLValue::Kind::Type:
       return read<Type>();
-    case PDLValueKind::Value:
+    case PDLValue::Kind::Value:
       return read<Value>();
+    case PDLValue::Kind::TypeRange:
+      return read<TypeRange *>();
+    case PDLValue::Kind::ValueRange:
+      return read<ValueRange *>();
     }
-    llvm_unreachable("unhandled PDLValueKind");
+    llvm_unreachable("unhandled PDLValue::Kind");
   }
   template <typename T>
   std::enable_if_t<std::is_same<T, ByteCodeAddr>::value, T> readImpl() {
@@ -853,12 +1071,20 @@ private:
   std::enable_if_t<std::is_same<T, ByteCodeField>::value, T> readImpl() {
     return *curCodeIt++;
   }
+  template <typename T>
+  std::enable_if_t<std::is_same<T, PDLValue::Kind>::value, T> readImpl() {
+    return static_cast<PDLValue::Kind>(readImpl<ByteCodeField>());
+  }
 
   /// The underlying bytecode buffer.
   const ByteCodeField *curCodeIt;
 
   /// The current execution memory.
   MutableArrayRef<const void *> memory;
+  MutableArrayRef<TypeRange> typeRangeMemory;
+  std::vector<llvm::OwningArrayRef<Type>> &allocatedTypeRangeMemory;
+  MutableArrayRef<ValueRange> valueRangeMemory;
+  std::vector<llvm::OwningArrayRef<Value>> &allocatedValueRangeMemory;
 
   /// References to ByteCode data necessary for execution.
   ArrayRef<const void *> uniquedMemory;
@@ -866,9 +1092,30 @@ private:
   ArrayRef<PatternBenefit> currentPatternBenefits;
   ArrayRef<PDLByteCodePattern> patterns;
   ArrayRef<PDLConstraintFunction> constraintFunctions;
-  ArrayRef<PDLCreateFunction> createFunctions;
   ArrayRef<PDLRewriteFunction> rewriteFunctions;
 };
+
+/// This class is an instantiation of the PDLResultList that provides access to
+/// the returned results. This API is not on `PDLResultList` to avoid
+/// overexposing access to information specific solely to the ByteCode.
+class ByteCodeRewriteResultList : public PDLResultList {
+public:
+  ByteCodeRewriteResultList(unsigned maxNumResults)
+      : PDLResultList(maxNumResults) {}
+
+  /// Return the list of PDL results.
+  MutableArrayRef<PDLValue> getResults() { return results; }
+
+  /// Return the type ranges allocated by this list.
+  MutableArrayRef<llvm::OwningArrayRef<Type>> getAllocatedTypeRanges() {
+    return allocatedTypeRanges;
+  }
+
+  /// Return the value ranges allocated by this list.
+  MutableArrayRef<llvm::OwningArrayRef<Value>> getAllocatedValueRanges() {
+    return allocatedValueRanges;
+  }
+};
 } // end anonymous namespace
 
 void ByteCodeExecutor::executeApplyConstraint(PatternRewriter &rewriter) {
@@ -892,18 +1139,54 @@ void ByteCodeExecutor::executeApplyRewrite(PatternRewriter &rewriter) {
   LLVM_DEBUG(llvm::dbgs() << "Executing ApplyRewrite:\n");
   const PDLRewriteFunction &rewriteFn = rewriteFunctions[read()];
   ArrayAttr constParams = read<ArrayAttr>();
-  Operation *root = read<Operation *>();
   SmallVector<PDLValue, 16> args;
   readList<PDLValue>(args);
 
   LLVM_DEBUG({
-    llvm::dbgs() << "  * Root: " << *root << "\n  * Arguments: ";
+    llvm::dbgs() << "  * Arguments: ";
     llvm::interleaveComma(args, llvm::dbgs());
     llvm::dbgs() << "\n  * Parameters: " << constParams << "\n";
   });
 
-  // Invoke the native rewrite function.
-  rewriteFn(root, args, constParams, rewriter);
+  // Execute the rewrite function.
+  ByteCodeField numResults = read();
+  ByteCodeRewriteResultList results(numResults);
+  rewriteFn(args, constParams, rewriter, results);
+
+  assert(results.getResults().size() == numResults &&
+         "native PDL rewrite function returned unexpected number of results");
+
+  // Store the results in the bytecode memory.
+  for (PDLValue &result : results.getResults()) {
+    LLVM_DEBUG(llvm::dbgs() << "  * Result: " << result << "\n");
+
+// In debug mode we also verify the expected kind of the result.
+#ifndef NDEBUG
+    assert(result.getKind() == read<PDLValue::Kind>() &&
+           "native PDL rewrite function returned an unexpected type of result");
+#endif
+
+    // If the result is a range, we need to copy it over to the bytecodes
+    // range memory.
+    if (Optional<TypeRange> typeRange = result.dyn_cast<TypeRange>()) {
+      unsigned rangeIndex = read();
+      typeRangeMemory[rangeIndex] = *typeRange;
+      memory[read()] = &typeRangeMemory[rangeIndex];
+    } else if (Optional<ValueRange> valueRange =
+                   result.dyn_cast<ValueRange>()) {
+      unsigned rangeIndex = read();
+      valueRangeMemory[rangeIndex] = *valueRange;
+      memory[read()] = &valueRangeMemory[rangeIndex];
+    } else {
+      memory[read()] = result.getAsOpaquePointer();
+    }
+  }
+
+  // Copy over any underlying storage allocated for result ranges.
+  for (auto &it : results.getAllocatedTypeRanges())
+    allocatedTypeRangeMemory.push_back(std::move(it));
+  for (auto &it : results.getAllocatedValueRanges())
+    allocatedValueRangeMemory.push_back(std::move(it));
 }
 
 void ByteCodeExecutor::executeAreEqual() {
@@ -915,6 +1198,32 @@ void ByteCodeExecutor::executeAreEqual() {
   selectJump(lhs == rhs);
 }
 
+void ByteCodeExecutor::executeAreRangesEqual() {
+  LLVM_DEBUG(llvm::dbgs() << "Executing AreRangesEqual:\n");
+  PDLValue::Kind valueKind = read<PDLValue::Kind>();
+  const void *lhs = read<const void *>();
+  const void *rhs = read<const void *>();
+
+  switch (valueKind) {
+  case PDLValue::Kind::TypeRange: {
+    const TypeRange *lhsRange = reinterpret_cast<const TypeRange *>(lhs);
+    const TypeRange *rhsRange = reinterpret_cast<const TypeRange *>(rhs);
+    LLVM_DEBUG(llvm::dbgs() << "  * " << lhs << " == " << rhs << "\n\n");
+    selectJump(*lhsRange == *rhsRange);
+    break;
+  }
+  case PDLValue::Kind::ValueRange: {
+    const auto *lhsRange = reinterpret_cast<const ValueRange *>(lhs);
+    const auto *rhsRange = reinterpret_cast<const ValueRange *>(rhs);
+    LLVM_DEBUG(llvm::dbgs() << "  * " << lhs << " == " << rhs << "\n\n");
+    selectJump(*lhsRange == *rhsRange);
+    break;
+  }
+  default:
+    llvm_unreachable("unexpected `AreRangesEqual` value kind");
+  }
+}
+
 void ByteCodeExecutor::executeBranch() {
   LLVM_DEBUG(llvm::dbgs() << "Executing Branch\n");
   curCodeIt = &code[read<ByteCodeAddr>()];
@@ -924,10 +1233,16 @@ void ByteCodeExecutor::executeCheckOperandCount() {
   LLVM_DEBUG(llvm::dbgs() << "Executing CheckOperandCount:\n");
   Operation *op = read<Operation *>();
   uint32_t expectedCount = read<uint32_t>();
+  bool compareAtLeast = read();
 
   LLVM_DEBUG(llvm::dbgs() << "  * Found: " << op->getNumOperands() << "\n"
-                          << "  * Expected: " << expectedCount << "\n");
-  selectJump(op->getNumOperands() == expectedCount);
+                          << "  * Expected: " << expectedCount << "\n"
+                          << "  * Comparator: "
+                          << (compareAtLeast ? ">=" : "==") << "\n");
+  if (compareAtLeast)
+    selectJump(op->getNumOperands() >= expectedCount);
+  else
+    selectJump(op->getNumOperands() == expectedCount);
 }
 
 void ByteCodeExecutor::executeCheckOperationName() {
@@ -944,30 +1259,44 @@ void ByteCodeExecutor::executeCheckResultCount() {
   LLVM_DEBUG(llvm::dbgs() << "Executing CheckResultCount:\n");
   Operation *op = read<Operation *>();
   uint32_t expectedCount = read<uint32_t>();
+  bool compareAtLeast = read();
 
   LLVM_DEBUG(llvm::dbgs() << "  * Found: " << op->getNumResults() << "\n"
-                          << "  * Expected: " << expectedCount << "\n");
-  selectJump(op->getNumResults() == expectedCount);
+                          << "  * Expected: " << expectedCount << "\n"
+                          << "  * Comparator: "
+                          << (compareAtLeast ? ">=" : "==") << "\n");
+  if (compareAtLeast)
+    selectJump(op->getNumResults() >= expectedCount);
+  else
+    selectJump(op->getNumResults() == expectedCount);
 }
 
-void ByteCodeExecutor::executeCreateNative(PatternRewriter &rewriter) {
-  LLVM_DEBUG(llvm::dbgs() << "Executing CreateNative:\n");
-  const PDLCreateFunction &createFn = createFunctions[read()];
-  ByteCodeField resultIndex = read();
-  ArrayAttr constParams = read<ArrayAttr>();
-  SmallVector<PDLValue, 16> args;
-  readList<PDLValue>(args);
+void ByteCodeExecutor::executeCheckTypes() {
+  LLVM_DEBUG(llvm::dbgs() << "Executing AreEqual:\n");
+  TypeRange *lhs = read<TypeRange *>();
+  Attribute rhs = read<Attribute>();
+  LLVM_DEBUG(llvm::dbgs() << "  * " << lhs << " == " << rhs << "\n\n");
 
-  LLVM_DEBUG({
-    llvm::dbgs() << "  * Arguments: ";
-    llvm::interleaveComma(args, llvm::dbgs());
-    llvm::dbgs() << "\n  * Parameters: " << constParams << "\n";
-  });
+  selectJump(*lhs == rhs.cast<ArrayAttr>().getAsValueRange<TypeAttr>());
+}
+
+void ByteCodeExecutor::executeCreateTypes() {
+  LLVM_DEBUG(llvm::dbgs() << "Executing CreateTypes:\n");
+  unsigned memIndex = read();
+  unsigned rangeIndex = read();
+  ArrayAttr typesAttr = read<Attribute>().cast<ArrayAttr>();
 
-  PDLValue result = createFn(args, constParams, rewriter);
-  memory[resultIndex] = result.getAsOpaquePointer();
+  LLVM_DEBUG(llvm::dbgs() << "  * Types: " << typesAttr << "\n\n");
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Result: " << result << "\n");
+  // Allocate a buffer for this type range.
+  llvm::OwningArrayRef<Type> storage(typesAttr.size());
+  llvm::copy(typesAttr.getAsValueRange<TypeAttr>(), storage.begin());
+  allocatedTypeRangeMemory.emplace_back(std::move(storage));
+
+  // Assign this to the range slot and use the range as the value for the
+  // memory index.
+  typeRangeMemory[rangeIndex] = allocatedTypeRangeMemory.back();
+  memory[memIndex] = &typeRangeMemory[rangeIndex];
 }
 
 void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter,
@@ -976,37 +1305,39 @@ void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter,
 
   unsigned memIndex = read();
   OperationState state(mainRewriteLoc, read<OperationName>());
-  readList<Value>(state.operands);
+  readValueList(state.operands);
   for (unsigned i = 0, e = read(); i != e; ++i) {
     Identifier name = read<Identifier>();
     if (Attribute attr = read<Attribute>())
       state.addAttribute(name, attr);
   }
 
-  bool hasInferredTypes = false;
   for (unsigned i = 0, e = read(); i != e; ++i) {
-    Type resultType = read<Type>();
-    hasInferredTypes |= !resultType;
-    state.types.push_back(resultType);
-  }
+    if (read<PDLValue::Kind>() == PDLValue::Kind::Type) {
+      state.types.push_back(read<Type>());
+      continue;
+    }
+
+    // If we find a null range, this signals that the types are infered.
+    if (TypeRange *resultTypes = read<TypeRange *>()) {
+      state.types.append(resultTypes->begin(), resultTypes->end());
+      continue;
+    }
 
-  // Handle the case where the operation has inferred types.
-  if (hasInferredTypes) {
+    // Handle the case where the operation has inferred types.
     InferTypeOpInterface::Concept *concept =
         state.name.getAbstractOperation()->getInterface<InferTypeOpInterface>();
 
     // TODO: Handle failure.
-    SmallVector<Type, 2> inferredTypes;
+    state.types.clear();
     if (failed(concept->inferReturnTypes(
             state.getContext(), state.location, state.operands,
             state.attributes.getDictionary(state.getContext()), state.regions,
-            inferredTypes)))
+            state.types)))
       return;
-
-    for (unsigned i = 0, e = state.types.size(); i != e; ++i)
-      if (!state.types[i])
-        state.types[i] = inferredTypes[i];
+    break;
   }
+
   Operation *resultOp = rewriter.createOperation(state);
   memory[memIndex] = resultOp;
 
@@ -1056,11 +1387,21 @@ void ByteCodeExecutor::executeGetAttributeType() {
 void ByteCodeExecutor::executeGetDefiningOp() {
   LLVM_DEBUG(llvm::dbgs() << "Executing GetDefiningOp:\n");
   unsigned memIndex = read();
-  Value value = read<Value>();
-  Operation *op = value ? value.getDefiningOp() : nullptr;
+  Operation *op = nullptr;
+  if (read<PDLValue::Kind>() == PDLValue::Kind::Value) {
+    Value value = read<Value>();
+    if (value)
+      op = value.getDefiningOp();
+    LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n");
+  } else {
+    ValueRange *values = read<ValueRange *>();
+    if (values && !values->empty()) {
+      op = values->front().getDefiningOp();
+    }
+    LLVM_DEBUG(llvm::dbgs() << "  * Values: " << values << "\n");
+  }
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n"
-                          << "  * Result: " << *op << "\n");
+  LLVM_DEBUG(llvm::dbgs() << "  * Result: " << op << "\n");
   memory[memIndex] = op;
 }
 
@@ -1076,6 +1417,75 @@ void ByteCodeExecutor::executeGetOperand(unsigned index) {
   memory[memIndex] = operand.getAsOpaquePointer();
 }
 
+/// This function is the internal implementation of `GetResults` and
+/// `GetOperands` that provides support for extracting a value range from the
+/// given operation.
+template <template <typename> class AttrSizedSegmentsT, typename RangeT>
+static void *
+executeGetOperandsResults(RangeT values, Operation *op, unsigned index,
+                          ByteCodeField rangeIndex, StringRef attrSizedSegments,
+                          MutableArrayRef<ValueRange> &valueRangeMemory) {
+  // Check for the sentinel index that signals that all values should be
+  // returned.
+  if (index == std::numeric_limits<uint32_t>::max()) {
+    LLVM_DEBUG(llvm::dbgs() << "  * Getting all values\n");
+    // `values` is already the full value range.
+
+    // Otherwise, check to see if this operation uses AttrSizedSegments.
+  } else if (op->hasTrait<AttrSizedSegmentsT>()) {
+    LLVM_DEBUG(llvm::dbgs()
+               << "  * Extracting values from `" << attrSizedSegments << "`\n");
+
+    auto segmentAttr = op->getAttrOfType<DenseElementsAttr>(attrSizedSegments);
+    if (!segmentAttr || segmentAttr.getNumElements() <= index)
+      return nullptr;
+
+    auto segments = segmentAttr.getValues<int32_t>();
+    unsigned startIndex =
+        std::accumulate(segments.begin(), segments.begin() + index, 0);
+    values = values.slice(startIndex, *std::next(segments.begin(), index));
+
+    LLVM_DEBUG(llvm::dbgs() << "  * Extracting range[" << startIndex << ", "
+                            << *std::next(segments.begin(), index) << "]\n");
+
+    // Otherwise, assume this is the last operand group of the operation.
+    // FIXME: We currently don't support operations with
+    // SameVariadicOperandSize/SameVariadicResultSize here given that we don't
+    // have a way to detect it's presence.
+  } else if (values.size() >= index) {
+    LLVM_DEBUG(llvm::dbgs()
+               << "  * Treating values as trailing variadic range\n");
+    values = values.drop_front(index);
+
+    // If we couldn't detect a way to compute the values, bail out.
+  } else {
+    return nullptr;
+  }
+
+  // If the range index is valid, we are returning a range.
+  if (rangeIndex != std::numeric_limits<ByteCodeField>::max()) {
+    valueRangeMemory[rangeIndex] = values;
+    return &valueRangeMemory[rangeIndex];
+  }
+
+  // If a range index wasn't provided, the range is required to be non-variadic.
+  return values.size() != 1 ? nullptr : values.front().getAsOpaquePointer();
+}
+
+void ByteCodeExecutor::executeGetOperands() {
+  LLVM_DEBUG(llvm::dbgs() << "Executing GetOperands:\n");
+  unsigned index = read<uint32_t>();
+  Operation *op = read<Operation *>();
+  ByteCodeField rangeIndex = read();
+
+  void *result = executeGetOperandsResults<OpTrait::AttrSizedOperandSegments>(
+      op->getOperands(), op, index, rangeIndex, "operand_segment_sizes",
+      valueRangeMemory);
+  if (!result)
+    LLVM_DEBUG(llvm::dbgs() << "  * Invalid operand range\n");
+  memory[read()] = result;
+}
+
 void ByteCodeExecutor::executeGetResult(unsigned index) {
   Operation *op = read<Operation *>();
   unsigned memIndex = read();
@@ -1088,6 +1498,20 @@ void ByteCodeExecutor::executeGetResult(unsigned index) {
   memory[memIndex] = result.getAsOpaquePointer();
 }
 
+void ByteCodeExecutor::executeGetResults() {
+  LLVM_DEBUG(llvm::dbgs() << "Executing GetResults:\n");
+  unsigned index = read<uint32_t>();
+  Operation *op = read<Operation *>();
+  ByteCodeField rangeIndex = read();
+
+  void *result = executeGetOperandsResults<OpTrait::AttrSizedResultSegments>(
+      op->getResults(), op, index, rangeIndex, "result_segment_sizes",
+      valueRangeMemory);
+  if (!result)
+    LLVM_DEBUG(llvm::dbgs() << "  * Invalid result range\n");
+  memory[read()] = result;
+}
+
 void ByteCodeExecutor::executeGetValueType() {
   LLVM_DEBUG(llvm::dbgs() << "Executing GetValueType:\n");
   unsigned memIndex = read();
@@ -1099,6 +1523,28 @@ void ByteCodeExecutor::executeGetValueType() {
   memory[memIndex] = type.getAsOpaquePointer();
 }
 
+void ByteCodeExecutor::executeGetValueRangeTypes() {
+  LLVM_DEBUG(llvm::dbgs() << "Executing GetValueRangeTypes:\n");
+  unsigned memIndex = read();
+  unsigned rangeIndex = read();
+  ValueRange *values = read<ValueRange *>();
+  if (!values) {
+    LLVM_DEBUG(llvm::dbgs() << "  * Values: <NULL>\n\n");
+    memory[memIndex] = nullptr;
+    return;
+  }
+
+  LLVM_DEBUG({
+    llvm::dbgs() << "  * Values (" << values->size() << "): ";
+    llvm::interleaveComma(*values, llvm::dbgs());
+    llvm::dbgs() << "\n  * Result: ";
+    llvm::interleaveComma(values->getType(), llvm::dbgs());
+    llvm::dbgs() << "\n";
+  });
+  typeRangeMemory[rangeIndex] = values->getType();
+  memory[memIndex] = &typeRangeMemory[rangeIndex];
+}
+
 void ByteCodeExecutor::executeIsNotNull() {
   LLVM_DEBUG(llvm::dbgs() << "Executing IsNotNull:\n");
   const void *value = read<const void *>();
@@ -1137,7 +1583,30 @@ void ByteCodeExecutor::executeRecordMatch(
   LLVM_DEBUG(llvm::dbgs() << "  * Benefit: " << benefit.getBenefit() << "\n"
                           << "  * Location: " << matchLoc << "\n");
   matches.emplace_back(matchLoc, patterns[patternIndex], benefit);
-  readList<const void *>(matches.back().values);
+  PDLByteCode::MatchResult &match = matches.back();
+
+  // Record all of the inputs to the match. If any of the inputs are ranges, we
+  // will also need to remap the range pointer to memory stored in the match
+  // state.
+  unsigned numInputs = read();
+  match.values.reserve(numInputs);
+  match.typeRangeValues.reserve(numInputs);
+  match.valueRangeValues.reserve(numInputs);
+  for (unsigned i = 0; i < numInputs; ++i) {
+    switch (read<PDLValue::Kind>()) {
+    case PDLValue::Kind::TypeRange:
+      match.typeRangeValues.push_back(*read<TypeRange *>());
+      match.values.push_back(&match.typeRangeValues.back());
+      break;
+    case PDLValue::Kind::ValueRange:
+      match.valueRangeValues.push_back(*read<ValueRange *>());
+      match.values.push_back(&match.valueRangeValues.back());
+      break;
+    default:
+      match.values.push_back(read<const void *>());
+      break;
+    }
+  }
   curCodeIt = dest;
 }
 
@@ -1145,7 +1614,7 @@ void ByteCodeExecutor::executeReplaceOp(PatternRewriter &rewriter) {
   LLVM_DEBUG(llvm::dbgs() << "Executing ReplaceOp:\n");
   Operation *op = read<Operation *>();
   SmallVector<Value, 16> args;
-  readList<Value>(args);
+  readValueList(args);
 
   LLVM_DEBUG({
     llvm::dbgs() << "  * Operation: " << *op << "\n"
@@ -1218,6 +1687,19 @@ void ByteCodeExecutor::executeSwitchType() {
   handleSwitch(value, cases);
 }
 
+void ByteCodeExecutor::executeSwitchTypes() {
+  LLVM_DEBUG(llvm::dbgs() << "Executing SwitchTypes:\n");
+  TypeRange *value = read<TypeRange *>();
+  auto cases = read<ArrayAttr>().getAsRange<ArrayAttr>();
+  if (!value) {
+    LLVM_DEBUG(llvm::dbgs() << "Types: <NULL>\n");
+    return selectJump(size_t(0));
+  }
+  handleSwitch(*value, cases, [](ArrayAttr caseValue, const TypeRange &value) {
+    return value == caseValue.getAsValueRange<TypeAttr>();
+  });
+}
+
 void ByteCodeExecutor::execute(
     PatternRewriter &rewriter,
     SmallVectorImpl<PDLByteCode::MatchResult> *matches,
@@ -1234,6 +1716,9 @@ void ByteCodeExecutor::execute(
     case AreEqual:
       executeAreEqual();
       break;
+    case AreRangesEqual:
+      executeAreRangesEqual();
+      break;
     case Branch:
       executeBranch();
       break;
@@ -1246,12 +1731,15 @@ void ByteCodeExecutor::execute(
     case CheckResultCount:
       executeCheckResultCount();
       break;
-    case CreateNative:
-      executeCreateNative(rewriter);
+    case CheckTypes:
+      executeCheckTypes();
       break;
     case CreateOperation:
       executeCreateOperation(rewriter, *mainRewriteLoc);
       break;
+    case CreateTypes:
+      executeCreateTypes();
+      break;
     case EraseOp:
       executeEraseOp(rewriter);
       break;
@@ -1280,6 +1768,9 @@ void ByteCodeExecutor::execute(
       LLVM_DEBUG(llvm::dbgs() << "Executing GetOperandN:\n");
       executeGetOperand(read<uint32_t>());
       break;
+    case GetOperands:
+      executeGetOperands();
+      break;
     case GetResult0:
     case GetResult1:
     case GetResult2:
@@ -1293,9 +1784,15 @@ void ByteCodeExecutor::execute(
       LLVM_DEBUG(llvm::dbgs() << "Executing GetResultN:\n");
       executeGetResult(read<uint32_t>());
       break;
+    case GetResults:
+      executeGetResults();
+      break;
     case GetValueType:
       executeGetValueType();
       break;
+    case GetValueRangeTypes:
+      executeGetValueRangeTypes();
+      break;
     case IsNotNull:
       executeIsNotNull();
       break;
@@ -1322,6 +1819,9 @@ void ByteCodeExecutor::execute(
     case SwitchType:
       executeSwitchType();
       break;
+    case SwitchTypes:
+      executeSwitchTypes();
+      break;
     }
     LLVM_DEBUG(llvm::dbgs() << "\n");
   }
@@ -1336,10 +1836,12 @@ void PDLByteCode::match(Operation *op, PatternRewriter &rewriter,
   state.memory[0] = op;
 
   // The matcher function always starts at code address 0.
-  ByteCodeExecutor executor(matcherByteCode.data(), state.memory, uniquedData,
-                            matcherByteCode, state.currentPatternBenefits,
-                            patterns, constraintFunctions, createFunctions,
-                            rewriteFunctions);
+  ByteCodeExecutor executor(
+      matcherByteCode.data(), state.memory, state.typeRangeMemory,
+      state.allocatedTypeRangeMemory, state.valueRangeMemory,
+      state.allocatedValueRangeMemory, uniquedData, matcherByteCode,
+      state.currentPatternBenefits, patterns, constraintFunctions,
+      rewriteFunctions);
   executor.execute(rewriter, &matches);
 
   // Order the found matches by benefit.
@@ -1358,7 +1860,9 @@ void PDLByteCode::rewrite(PatternRewriter &rewriter, const MatchResult &match,
 
   ByteCodeExecutor executor(
       &rewriterByteCode[match.pattern->getRewriterAddr()], state.memory,
-      uniquedData, rewriterByteCode, state.currentPatternBenefits, patterns,
-      constraintFunctions, createFunctions, rewriteFunctions);
+      state.typeRangeMemory, state.allocatedTypeRangeMemory,
+      state.valueRangeMemory, state.allocatedValueRangeMemory, uniquedData,
+      rewriterByteCode, state.currentPatternBenefits, patterns,
+      constraintFunctions, rewriteFunctions);
   executor.execute(rewriter, /*matches=*/nullptr, match.location);
 }
diff --git a/mlir/lib/Rewrite/ByteCode.h b/mlir/lib/Rewrite/ByteCode.h
index 38dbbcd855ce52ebeba8548a30061ea2a5d77a87..c6f41be768deacbbae0fb170fad8cbff84ccc0bf 100644
--- a/mlir/lib/Rewrite/ByteCode.h
+++ b/mlir/lib/Rewrite/ByteCode.h
@@ -25,8 +25,7 @@ namespace detail {
 class PDLByteCode;
 
 /// Use generic bytecode types. ByteCodeField refers to the actual bytecode
-/// entries (set to uint8_t for "byte" bytecode). ByteCodeAddr refers to size of
-/// indices into the bytecode. Correctness is checked with static asserts.
+/// entries. ByteCodeAddr refers to size of indices into the bytecode.
 using ByteCodeField = uint16_t;
 using ByteCodeAddr = uint32_t;
 
@@ -62,14 +61,16 @@ private:
 /// threads/drivers.
 class PDLByteCodeMutableState {
 public:
-  /// Initialize the state from a bytecode instance.
-  void initialize(PDLByteCode &bytecode);
-
   /// Set the new benefit for a bytecode pattern. The `patternIndex` corresponds
   /// to the position of the pattern within the range returned by
   /// `PDLByteCode::getPatterns`.
   void updatePatternBenefit(unsigned patternIndex, PatternBenefit benefit);
 
+  /// Cleanup any allocated state after a match/rewrite has been completed. This
+  /// method should be called irregardless of whether the match+rewrite was a
+  /// success or not.
+  void cleanupAfterMatchAndRewrite();
+
 private:
   /// Allow access to data fields.
   friend class PDLByteCode;
@@ -78,6 +79,20 @@ private:
   /// of the bytecode.
   std::vector<const void *> memory;
 
+  /// A mutable block of memory used during the matching and rewriting phase of
+  /// the bytecode to store ranges of types.
+  std::vector<TypeRange> typeRangeMemory;
+  /// A set of type ranges that have been allocated by the byte code interpreter
+  /// to provide a guaranteed lifetime.
+  std::vector<llvm::OwningArrayRef<Type>> allocatedTypeRangeMemory;
+
+  /// A mutable block of memory used during the matching and rewriting phase of
+  /// the bytecode to store ranges of values.
+  std::vector<ValueRange> valueRangeMemory;
+  /// A set of value ranges that have been allocated by the byte code
+  /// interpreter to provide a guaranteed lifetime.
+  std::vector<llvm::OwningArrayRef<Value>> allocatedValueRangeMemory;
+
   /// The up-to-date benefits of the patterns held by the bytecode. The order
   /// of this array corresponds 1-1 with the array of patterns in `PDLByteCode`.
   std::vector<PatternBenefit> currentPatternBenefits;
@@ -98,11 +113,19 @@ public:
     MatchResult(Location loc, const PDLByteCodePattern &pattern,
                 PatternBenefit benefit)
         : location(loc), pattern(&pattern), benefit(benefit) {}
+    MatchResult(const MatchResult &) = delete;
+    MatchResult &operator=(const MatchResult &) = delete;
+    MatchResult(MatchResult &&other) = default;
+    MatchResult &operator=(MatchResult &&) = default;
 
     /// The location of operations to be replaced.
     Location location;
     /// Memory values defined in the matcher that are passed to the rewriter.
-    SmallVector<const void *, 4> values;
+    SmallVector<const void *> values;
+    /// Memory used for the range input values.
+    SmallVector<TypeRange, 0> typeRangeValues;
+    SmallVector<ValueRange, 0> valueRangeValues;
+
     /// The originating pattern that was matched. This is always non-null, but
     /// represented with a pointer to allow for assignment.
     const PDLByteCodePattern *pattern;
@@ -114,7 +137,6 @@ public:
   /// the PDL interpreter dialect.
   PDLByteCode(ModuleOp module,
               llvm::StringMap<PDLConstraintFunction> constraintFns,
-              llvm::StringMap<PDLCreateFunction> createFns,
               llvm::StringMap<PDLRewriteFunction> rewriteFns);
 
   /// Return the patterns held by the bytecode.
@@ -160,11 +182,14 @@ private:
 
   /// A set of user defined functions invoked via PDL.
   std::vector<PDLConstraintFunction> constraintFunctions;
-  std::vector<PDLCreateFunction> createFunctions;
   std::vector<PDLRewriteFunction> rewriteFunctions;
 
   /// The maximum memory index used by a value.
   ByteCodeField maxValueMemoryIndex = 0;
+
+  /// The maximum number of different types of ranges.
+  ByteCodeField maxTypeRangeCount = 0;
+  ByteCodeField maxValueRangeCount = 0;
 };
 
 } // end namespace detail
diff --git a/mlir/lib/Rewrite/CMakeLists.txt b/mlir/lib/Rewrite/CMakeLists.txt
index 5822789cc91625ecaa4a7db8b404165e5e1405d1..76bf64944d50c005cdb51e1540b7ce3755975b68 100644
--- a/mlir/lib/Rewrite/CMakeLists.txt
+++ b/mlir/lib/Rewrite/CMakeLists.txt
@@ -1,6 +1,6 @@
 add_mlir_library(MLIRRewrite
   ByteCode.cpp
-  FrozenRewritePatternList.cpp
+  FrozenRewritePatternSet.cpp
   PatternApplicator.cpp
 
   ADDITIONAL_HEADER_DIRS
diff --git a/mlir/lib/Rewrite/FrozenRewritePatternList.cpp b/mlir/lib/Rewrite/FrozenRewritePatternSet.cpp
similarity index 54%
rename from mlir/lib/Rewrite/FrozenRewritePatternList.cpp
rename to mlir/lib/Rewrite/FrozenRewritePatternSet.cpp
index 40f7aec44e5154cf0b11386faaa17f61999bfc5e..0b6a1cf2cdf3e6162d7b14a68fe427fc7f46990d 100644
--- a/mlir/lib/Rewrite/FrozenRewritePatternList.cpp
+++ b/mlir/lib/Rewrite/FrozenRewritePatternSet.cpp
@@ -1,4 +1,4 @@
-//===- FrozenRewritePatternList.cpp - Frozen Pattern List -------*- C++ -*-===//
+//===- FrozenRewritePatternSet.cpp - Frozen Pattern List -------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Rewrite/FrozenRewritePatternList.h"
+#include "mlir/Rewrite/FrozenRewritePatternSet.h"
 #include "ByteCode.h"
 #include "mlir/Conversion/PDLToPDLInterp/PDLToPDLInterp.h"
 #include "mlir/Dialect/PDL/IR/PDLOps.h"
@@ -47,16 +47,51 @@ static LogicalResult convertPDLToPDLInterp(ModuleOp pdlModule) {
 }
 
 //===----------------------------------------------------------------------===//
-// FrozenRewritePatternList
+// FrozenRewritePatternSet
 //===----------------------------------------------------------------------===//
 
-FrozenRewritePatternList::FrozenRewritePatternList()
+FrozenRewritePatternSet::FrozenRewritePatternSet()
     : impl(std::make_shared<Impl>()) {}
 
-FrozenRewritePatternList::FrozenRewritePatternList(
-    OwningRewritePatternList &&patterns)
+FrozenRewritePatternSet::FrozenRewritePatternSet(RewritePatternSet &&patterns)
     : impl(std::make_shared<Impl>()) {
-  impl->nativePatterns = std::move(patterns.getNativePatterns());
+  // Functor used to walk all of the operations registered in the context. This
+  // is useful for patterns that get applied to multiple operations, such as
+  // interface and trait based patterns.
+  std::vector<AbstractOperation *> abstractOps;
+  auto addToOpsWhen = [&](std::unique_ptr<RewritePattern> &pattern,
+                          function_ref<bool(AbstractOperation *)> callbackFn) {
+    if (abstractOps.empty())
+      abstractOps = pattern->getContext()->getRegisteredOperations();
+    for (AbstractOperation *absOp : abstractOps) {
+      if (callbackFn(absOp)) {
+        OperationName opName(absOp);
+        impl->nativeOpSpecificPatternMap[opName].push_back(pattern.get());
+      }
+    }
+    impl->nativeOpSpecificPatternList.push_back(std::move(pattern));
+  };
+
+  for (std::unique_ptr<RewritePattern> &pat : patterns.getNativePatterns()) {
+    if (Optional<OperationName> rootName = pat->getRootKind()) {
+      impl->nativeOpSpecificPatternMap[*rootName].push_back(pat.get());
+      impl->nativeOpSpecificPatternList.push_back(std::move(pat));
+      continue;
+    }
+    if (Optional<TypeID> interfaceID = pat->getRootInterfaceID()) {
+      addToOpsWhen(pat, [&](AbstractOperation *absOp) {
+        return absOp->hasInterface(*interfaceID);
+      });
+      continue;
+    }
+    if (Optional<TypeID> traitID = pat->getRootTraitID()) {
+      addToOpsWhen(pat, [&](AbstractOperation *absOp) {
+        return absOp->hasTrait(*traitID);
+      });
+      continue;
+    }
+    impl->nativeAnyOpPatterns.push_back(std::move(pat));
+  }
 
   // Generate the bytecode for the PDL patterns if any were provided.
   PDLPatternModule &pdlPatterns = patterns.getPDLPatterns();
@@ -70,7 +105,7 @@ FrozenRewritePatternList::FrozenRewritePatternList(
   // Generate the pdl bytecode.
   impl->pdlByteCode = std::make_unique<detail::PDLByteCode>(
       pdlModule, pdlPatterns.takeConstraintFunctions(),
-      pdlPatterns.takeCreateFunctions(), pdlPatterns.takeRewriteFunctions());
+      pdlPatterns.takeRewriteFunctions());
 }
 
-FrozenRewritePatternList::~FrozenRewritePatternList() {}
+FrozenRewritePatternSet::~FrozenRewritePatternSet() {}
diff --git a/mlir/lib/Rewrite/PatternApplicator.cpp b/mlir/lib/Rewrite/PatternApplicator.cpp
index 6f5e1f299f2602b1f1ddc1683bdf7163ad8ba7b0..0ece814bca47c7f16d72cc373038b92d6508e194 100644
--- a/mlir/lib/Rewrite/PatternApplicator.cpp
+++ b/mlir/lib/Rewrite/PatternApplicator.cpp
@@ -15,11 +15,13 @@
 #include "ByteCode.h"
 #include "llvm/Support/Debug.h"
 
+#define DEBUG_TYPE "pattern-match"
+
 using namespace mlir;
 using namespace mlir::detail;
 
 PatternApplicator::PatternApplicator(
-    const FrozenRewritePatternList &frozenPatternList)
+    const FrozenRewritePatternSet &frozenPatternList)
     : frozenPatternList(frozenPatternList) {
   if (const PDLByteCode *bytecode = frozenPatternList.getPDLByteCode()) {
     mutableByteCodeState = std::make_unique<PDLByteCodeMutableState>();
@@ -28,7 +30,14 @@ PatternApplicator::PatternApplicator(
 }
 PatternApplicator::~PatternApplicator() {}
 
-#define DEBUG_TYPE "pattern-match"
+/// Log a message for a pattern that is impossible to match.
+static void logImpossibleToMatch(const Pattern &pattern) {
+  LLVM_DEBUG({
+    llvm::dbgs() << "Ignoring pattern '" << pattern.getRootKind()
+                 << "' because it is impossible to match or cannot lead "
+                    "to legal IR (by cost model)\n";
+  });
+}
 
 void PatternApplicator::applyCostModel(CostModel model) {
   // Apply the cost model to the bytecode patterns first, and then the native
@@ -38,23 +47,24 @@ void PatternApplicator::applyCostModel(CostModel model) {
       mutableByteCodeState->updatePatternBenefit(it.index(), model(it.value()));
   }
 
-  // Separate patterns by root kind to simplify lookup later on.
+  // Copy over the patterns so that we can sort by benefit based on the cost
+  // model. Patterns that are already impossible to match are ignored.
   patterns.clear();
-  anyOpPatterns.clear();
-  for (const auto &pat : frozenPatternList.getNativePatterns()) {
-    // If the pattern is always impossible to match, just ignore it.
-    if (pat.getBenefit().isImpossibleToMatch()) {
-      LLVM_DEBUG({
-        llvm::dbgs()
-            << "Ignoring pattern '" << pat.getRootKind()
-            << "' because it is impossible to match (by pattern benefit)\n";
-      });
-      continue;
+  for (const auto &it : frozenPatternList.getOpSpecificNativePatterns()) {
+    for (const RewritePattern *pattern : it.second) {
+      if (pattern->getBenefit().isImpossibleToMatch())
+        logImpossibleToMatch(*pattern);
+      else
+        patterns[it.first].push_back(pattern);
     }
-    if (Optional<OperationName> opName = pat.getRootKind())
-      patterns[*opName].push_back(&pat);
+  }
+  anyOpPatterns.clear();
+  for (const RewritePattern &pattern :
+       frozenPatternList.getMatchAnyOpNativePatterns()) {
+    if (pattern.getBenefit().isImpossibleToMatch())
+      logImpossibleToMatch(pattern);
     else
-      anyOpPatterns.push_back(&pat);
+      anyOpPatterns.push_back(&pattern);
   }
 
   // Sort the patterns using the provided cost model.
@@ -66,11 +76,7 @@ void PatternApplicator::applyCostModel(CostModel model) {
     // Special case for one pattern in the list, which is the most common case.
     if (list.size() == 1) {
       if (model(*list.front()).isImpossibleToMatch()) {
-        LLVM_DEBUG({
-          llvm::dbgs() << "Ignoring pattern '" << list.front()->getRootKind()
-                       << "' because it is impossible to match or cannot lead "
-                          "to legal IR (by cost model)\n";
-        });
+        logImpossibleToMatch(*list.front());
         list.clear();
       }
       return;
@@ -84,14 +90,8 @@ void PatternApplicator::applyCostModel(CostModel model) {
     // Sort patterns with highest benefit first, and remove those that are
     // impossible to match.
     std::stable_sort(list.begin(), list.end(), cmp);
-    while (!list.empty() && benefits[list.back()].isImpossibleToMatch()) {
-      LLVM_DEBUG({
-        llvm::dbgs() << "Ignoring pattern '" << list.back()->getRootKind()
-                     << "' because it is impossible to match or cannot lead to "
-                        "legal IR (by cost model)\n";
-      });
-      list.pop_back();
-    }
+    while (!list.empty() && benefits[list.back()].isImpossibleToMatch())
+      logImpossibleToMatch(*list.pop_back_val());
   };
   for (auto &it : patterns)
     processPatternList(it.second);
@@ -100,7 +100,10 @@ void PatternApplicator::applyCostModel(CostModel model) {
 
 void PatternApplicator::walkAllPatterns(
     function_ref<void(const Pattern &)> walk) {
-  for (const Pattern &it : frozenPatternList.getNativePatterns())
+  for (const auto &it : frozenPatternList.getOpSpecificNativePatterns())
+    for (const auto &pattern : it.second)
+      walk(*pattern);
+  for (const Pattern &it : frozenPatternList.getMatchAnyOpNativePatterns())
     walk(it);
   if (const PDLByteCode *bytecode = frozenPatternList.getPDLByteCode()) {
     for (const Pattern &it : bytecode->getPatterns())
@@ -129,29 +132,40 @@ LogicalResult PatternApplicator::matchAndRewrite(
 
   // Process the patterns for that match the specific operation type, and any
   // operation type in an interleaved fashion.
-  auto opIt = opPatterns.begin(), opE = opPatterns.end();
-  auto anyIt = anyOpPatterns.begin(), anyE = anyOpPatterns.end();
-  auto pdlIt = pdlMatches.begin(), pdlE = pdlMatches.end();
-  while (true) {
+  unsigned opIt = 0, opE = opPatterns.size();
+  unsigned anyIt = 0, anyE = anyOpPatterns.size();
+  unsigned pdlIt = 0, pdlE = pdlMatches.size();
+  LogicalResult result = failure();
+  do {
     // Find the next pattern with the highest benefit.
     const Pattern *bestPattern = nullptr;
+    unsigned *bestPatternIt = &opIt;
     const PDLByteCode::MatchResult *pdlMatch = nullptr;
+
     /// Operation specific patterns.
-    if (opIt != opE)
-      bestPattern = *(opIt++);
+    if (opIt < opE)
+      bestPattern = opPatterns[opIt];
     /// Operation agnostic patterns.
-    if (anyIt != anyE &&
-        (!bestPattern || bestPattern->getBenefit() < (*anyIt)->getBenefit()))
-      bestPattern = *(anyIt++);
+    if (anyIt < anyE &&
+        (!bestPattern ||
+         bestPattern->getBenefit() < anyOpPatterns[anyIt]->getBenefit())) {
+      bestPatternIt = &anyIt;
+      bestPattern = anyOpPatterns[anyIt];
+    }
     /// PDL patterns.
-    if (pdlIt != pdlE &&
-        (!bestPattern || bestPattern->getBenefit() < pdlIt->benefit)) {
-      pdlMatch = pdlIt;
-      bestPattern = (pdlIt++)->pattern;
+    if (pdlIt < pdlE && (!bestPattern || bestPattern->getBenefit() <
+                                             pdlMatches[pdlIt].benefit)) {
+      bestPatternIt = &pdlIt;
+      pdlMatch = &pdlMatches[pdlIt];
+      bestPattern = pdlMatch->pattern;
     }
     if (!bestPattern)
       break;
 
+    // Update the pattern iterator on failure so that this pattern isn't
+    // attempted again.
+    ++(*bestPatternIt);
+
     // Check that the pattern can be applied.
     if (canApply && !canApply(*bestPattern))
       continue;
@@ -160,19 +174,25 @@ LogicalResult PatternApplicator::matchAndRewrite(
     // benefit, so if we match we can immediately rewrite. For PDL patterns, the
     // match has already been performed, we just need to rewrite.
     rewriter.setInsertionPoint(op);
-    LogicalResult result = success();
     if (pdlMatch) {
       bytecode->rewrite(rewriter, *pdlMatch, *mutableByteCodeState);
+      result = success(!onSuccess || succeeded(onSuccess(*bestPattern)));
+
     } else {
-      result = static_cast<const RewritePattern *>(bestPattern)
-                   ->matchAndRewrite(op, rewriter);
+      const auto *pattern = static_cast<const RewritePattern *>(bestPattern);
+      result = pattern->matchAndRewrite(op, rewriter);
+      if (succeeded(result) && onSuccess && failed(onSuccess(*pattern)))
+        result = failure();
     }
-    if (succeeded(result) && (!onSuccess || succeeded(onSuccess(*bestPattern))))
-      return success();
+    if (succeeded(result))
+      break;
 
     // Perform any necessary cleanups.
     if (onFailure)
       onFailure(*bestPattern);
-  }
-  return failure();
+  } while (true);
+
+  if (mutableByteCodeState)
+    mutableByteCodeState->cleanupAfterMatchAndRewrite();
+  return result;
 }
diff --git a/mlir/lib/TableGen/AttrOrTypeDef.cpp b/mlir/lib/TableGen/AttrOrTypeDef.cpp
index 037dc4d40cc5c8268dadf799c8e7e93be59e3dc0..1e4f5e4becdd65361500c77071b0ba7fb21f5bef 100644
--- a/mlir/lib/TableGen/AttrOrTypeDef.cpp
+++ b/mlir/lib/TableGen/AttrOrTypeDef.cpp
@@ -45,10 +45,6 @@ AttrOrTypeDef::AttrOrTypeDef(const llvm::Record *def) : def(def) {
       }
       builders.emplace_back(builder);
     }
-  } else if (skipDefaultBuilders()) {
-    PrintFatalError(
-        def->getLoc(),
-        "default builders are skipped and no custom builders provided");
   }
 }
 
@@ -177,22 +173,18 @@ Optional<StringRef> AttrOrTypeParameter::getAllocator() const {
   llvm::Init *parameterType = def->getArg(index);
   if (isa<llvm::StringInit>(parameterType))
     return Optional<StringRef>();
+  if (auto *param = dyn_cast<llvm::DefInit>(parameterType))
+    return param->getDef()->getValueAsOptionalString("allocator");
+  llvm::PrintFatalError("Parameters DAG arguments must be either strings or "
+                        "defs which inherit from AttrOrTypeParameter\n");
+}
 
-  if (auto *param = dyn_cast<llvm::DefInit>(parameterType)) {
-    llvm::RecordVal *code = param->getDef()->getValue("allocator");
-    if (!code)
-      return Optional<StringRef>();
-    if (llvm::StringInit *ci = dyn_cast<llvm::StringInit>(code->getValue()))
-      return ci->getValue();
-    if (isa<llvm::UnsetInit>(code->getValue()))
-      return Optional<StringRef>();
-
-    llvm::PrintFatalError(
-        param->getDef()->getLoc(),
-        "Record `" + def->getArgName(index)->getValue() +
-            "', field `printer' does not have a code initializer!");
-  }
-
+Optional<StringRef> AttrOrTypeParameter::getComparator() const {
+  llvm::Init *parameterType = def->getArg(index);
+  if (isa<llvm::StringInit>(parameterType))
+    return Optional<StringRef>();
+  if (auto *param = dyn_cast<llvm::DefInit>(parameterType))
+    return param->getDef()->getValueAsOptionalString("comparator");
   llvm::PrintFatalError("Parameters DAG arguments must be either strings or "
                         "defs which inherit from AttrOrTypeParameter\n");
 }
diff --git a/mlir/lib/TableGen/Attribute.cpp b/mlir/lib/TableGen/Attribute.cpp
index 99d9d8ab9b3afd0ed39955fa8121f8cd8d3d49c2..3b949b0460beff9daa9bac952bc56c5b7a008eab 100644
--- a/mlir/lib/TableGen/Attribute.cpp
+++ b/mlir/lib/TableGen/Attribute.cpp
@@ -231,6 +231,18 @@ std::vector<EnumAttrCase> EnumAttr::getAllCases() const {
   return cases;
 }
 
+bool EnumAttr::genSpecializedAttr() const {
+  return def->getValueAsBit("genSpecializedAttr");
+}
+
+llvm::Record *EnumAttr::getBaseAttrClass() const {
+  return def->getValueAsDef("baseAttrClass");
+}
+
+StringRef EnumAttr::getSpecializedAttrClassName() const {
+  return def->getValueAsString("specializedAttrClassName");
+}
+
 StructFieldAttr::StructFieldAttr(const llvm::Record *record) : def(record) {
   assert(def->isSubClassOf("StructFieldAttr") &&
          "must be subclass of TableGen 'StructFieldAttr' class");
diff --git a/mlir/lib/TableGen/Dialect.cpp b/mlir/lib/TableGen/Dialect.cpp
index 0c1de78ce60e18bc9fd02ae1d766bb5b916ca34d..6b88f8a40bc1ce7cf47729f2d425d5442c907b44 100644
--- a/mlir/lib/TableGen/Dialect.cpp
+++ b/mlir/lib/TableGen/Dialect.cpp
@@ -77,6 +77,10 @@ bool Dialect::hasRegionResultAttrVerify() const {
   return def->getValueAsBit("hasRegionResultAttrVerify");
 }
 
+bool Dialect::hasOperationInterfaceFallback() const {
+  return def->getValueAsBit("hasOperationInterfaceFallback");
+}
+
 bool Dialect::operator==(const Dialect &other) const {
   return def == other.def;
 }
diff --git a/mlir/lib/TableGen/Format.cpp b/mlir/lib/TableGen/Format.cpp
index 7d17a0aef3f9788819cbfc22b1a6619e4459efc9..10834510b76741ff8a76dff6407f5b5a43891c08 100644
--- a/mlir/lib/TableGen/Format.cpp
+++ b/mlir/lib/TableGen/Format.cpp
@@ -173,3 +173,22 @@ void FmtObjectBase::format(raw_ostream &s) const {
     adapters[repl.index]->format(s, /*Options=*/"");
   }
 }
+
+FmtStrVecObject::FmtStrVecObject(StringRef fmt, const FmtContext *ctx,
+                                 ArrayRef<std::string> params)
+    : FmtObjectBase(fmt, ctx, params.size()) {
+  parameters.reserve(params.size());
+  for (std::string p : params)
+    parameters.push_back(llvm::detail::build_format_adapter(std::move(p)));
+
+  adapters.reserve(parameters.size());
+  for (auto &p : parameters)
+    adapters.push_back(&p);
+}
+
+FmtStrVecObject::FmtStrVecObject(FmtStrVecObject &&that)
+    : FmtObjectBase(std::move(that)), parameters(std::move(that.parameters)) {
+  adapters.reserve(parameters.size());
+  for (auto &p : parameters)
+    adapters.push_back(&p);
+}
diff --git a/mlir/lib/TableGen/Predicate.cpp b/mlir/lib/TableGen/Predicate.cpp
index a37847f0d48930ccdfc1c8fbaa269c76f924e12d..dd71540c15d440bda2283625b178749866896cd7 100644
--- a/mlir/lib/TableGen/Predicate.cpp
+++ b/mlir/lib/TableGen/Predicate.cpp
@@ -133,6 +133,23 @@ namespace {
 using Subst = std::pair<StringRef, StringRef>;
 } // end anonymous namespace
 
+/// Perform the given substitutions on 'str' in-place.
+static void performSubstitutions(std::string &str,
+                                 ArrayRef<Subst> substitutions) {
+  // Apply all parent substitutions from innermost to outermost.
+  for (const auto &subst : llvm::reverse(substitutions)) {
+    auto pos = str.find(std::string(subst.first));
+    while (pos != std::string::npos) {
+      str.replace(pos, subst.first.size(), std::string(subst.second));
+      // Skip the newly inserted substring, which itself may consider the
+      // pattern to match.
+      pos += subst.second.size();
+      // Find the next possible match position.
+      pos = str.find(std::string(subst.first), pos);
+    }
+  }
+}
+
 // Build the predicate tree starting from the top-level predicate, which may
 // have children, and perform leaf substitutions inplace.  Note that after
 // substitution, nodes are still pointing to the original TableGen record.
@@ -147,19 +164,7 @@ buildPredicateTree(const Pred &root,
   rootNode->predicate = &root;
   if (!root.isCombined()) {
     rootNode->expr = root.getCondition();
-    // Apply all parent substitutions from innermost to outermost.
-    for (const auto &subst : llvm::reverse(substitutions)) {
-      auto pos = rootNode->expr.find(std::string(subst.first));
-      while (pos != std::string::npos) {
-        rootNode->expr.replace(pos, subst.first.size(),
-                               std::string(subst.second));
-        // Skip the newly inserted substring, which itself may consider the
-        // pattern to match.
-        pos += subst.second.size();
-        // Find the next possible match position.
-        pos = rootNode->expr.find(std::string(subst.first), pos);
-      }
-    }
+    performSubstitutions(rootNode->expr, substitutions);
     return rootNode;
   }
 
@@ -170,12 +175,14 @@ buildPredicateTree(const Pred &root,
     const auto &substPred = static_cast<const SubstLeavesPred &>(root);
     allSubstitutions.push_back(
         {substPred.getPattern(), substPred.getReplacement()});
-  }
-  // If the current predicate is a ConcatPred, record the prefix and suffix.
-  else if (rootNode->kind == PredCombinerKind::Concat) {
+
+    // If the current predicate is a ConcatPred, record the prefix and suffix.
+  } else if (rootNode->kind == PredCombinerKind::Concat) {
     const auto &concatPred = static_cast<const ConcatPred &>(root);
     rootNode->prefix = std::string(concatPred.getPrefix());
+    performSubstitutions(rootNode->prefix, substitutions);
     rootNode->suffix = std::string(concatPred.getSuffix());
+    performSubstitutions(rootNode->suffix, substitutions);
   }
 
   // Build child subtrees.
diff --git a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
index 28b58cffe17001bd4fdfb340863bd333fe6eb981..8044ff62f6427bae6b2b75b3edf54b16fccfc054 100644
--- a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
+++ b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
@@ -269,10 +269,11 @@ private:
 
   /// Globals are inserted before the first function, if any.
   Block::iterator getGlobalInsertPt() {
-    auto i = module.getBody()->begin();
-    while (!isa<LLVMFuncOp, ModuleTerminatorOp>(i))
-      ++i;
-    return i;
+    auto it = module.getBody()->begin();
+    auto endIt = module.getBody()->end();
+    while (it != endIt && !isa<LLVMFuncOp>(it))
+      ++it;
+    return it;
   }
 
   /// Functions are always inserted before the module terminator.
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 891f30b95b66602990f09b773b6ce0b3874b31b7..a8d9f44d702d033b462de4a60ad5ca4bc93bd543 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -364,6 +364,15 @@ static Block &getModuleBody(Operation *module) {
   return module->getRegion(0).front();
 }
 
+/// A helper method to decide if a constant must not be set as a global variable
+/// initializer.
+static bool shouldDropGlobalInitializer(llvm::GlobalValue::LinkageTypes linkage,
+                                        llvm::Constant *cst) {
+  return (linkage == llvm::GlobalVariable::ExternalLinkage &&
+          isa<llvm::UndefValue>(cst)) ||
+         linkage == llvm::GlobalVariable::ExternalWeakLinkage;
+}
+
 /// Create named global variables that correspond to llvm.mlir.global
 /// definitions.
 LogicalResult ModuleTranslation::convertGlobals() {
@@ -381,31 +390,39 @@ LogicalResult ModuleTranslation::convertGlobals() {
                                          *this))) {
         return failure();
       }
-    } else if (Block *initializer = op.getInitializerBlock()) {
-      llvm::IRBuilder<> builder(llvmModule->getContext());
-      for (auto &op : initializer->without_terminator()) {
-        if (failed(convertOperation(op, builder)) ||
-            !isa<llvm::Constant>(lookupValue(op.getResult(0))))
-          return emitError(op.getLoc(), "unemittable constant value");
-      }
-      ReturnOp ret = cast<ReturnOp>(initializer->getTerminator());
-      cst = cast<llvm::Constant>(lookupValue(ret.getOperand(0)));
     }
 
     auto linkage = convertLinkageToLLVM(op.linkage());
-    bool anyExternalLinkage =
-        ((linkage == llvm::GlobalVariable::ExternalLinkage &&
-          isa<llvm::UndefValue>(cst)) ||
-         linkage == llvm::GlobalVariable::ExternalWeakLinkage);
     auto addrSpace = op.addr_space();
     auto *var = new llvm::GlobalVariable(
         *llvmModule, type, op.constant(), linkage,
-        anyExternalLinkage ? nullptr : cst, op.sym_name(),
+        shouldDropGlobalInitializer(linkage, cst) ? nullptr : cst,
+        op.sym_name(),
         /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal, addrSpace);
 
     globalsMapping.try_emplace(op, var);
   }
 
+  // Convert global variable bodies. This is done after all global variables
+  // have been created in LLVM IR because a global body may refer to another
+  // global or itself. So all global variables need to be mapped first.
+  for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
+    if (Block *initializer = op.getInitializerBlock()) {
+      llvm::IRBuilder<> builder(llvmModule->getContext());
+      for (auto &op : initializer->without_terminator()) {
+        if (failed(convertOperation(op, builder)) ||
+            !isa<llvm::Constant>(lookupValue(op.getResult(0))))
+          return emitError(op.getLoc(), "unemittable constant value");
+      }
+      ReturnOp ret = cast<ReturnOp>(initializer->getTerminator());
+      llvm::Constant *cst =
+          cast<llvm::Constant>(lookupValue(ret.getOperand(0)));
+      auto *global = cast<llvm::GlobalVariable>(lookupGlobal(op));
+      if (!shouldDropGlobalInitializer(global->getLinkage(), cst))
+        global->setInitializer(cst);
+    }
+  }
+
   return success();
 }
 
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index 06e7f8136e9694b598855f2b7dce37e6d6d8d16b..6137fee34dcd7dd5d0b708f5ef6ac16e7648912b 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -331,7 +331,8 @@ Deserializer::processOp<spirv::EntryPointOp>(ArrayRef<uint32_t> words) {
     return emitError(unknownLoc,
                      "missing Execution Model specification in OpEntryPoint");
   }
-  auto execModel = opBuilder.getI32IntegerAttr(words[wordIndex++]);
+  auto execModel = spirv::ExecutionModelAttr::get(
+      context, static_cast<spirv::ExecutionModel>(words[wordIndex++]));
   if (wordIndex >= words.size()) {
     return emitError(unknownLoc, "missing <id> in OpEntryPoint");
   }
@@ -383,7 +384,8 @@ Deserializer::processOp<spirv::ExecutionModeOp>(ArrayRef<uint32_t> words) {
   if (wordIndex >= words.size()) {
     return emitError(unknownLoc, "missing Execution Mode in OpExecutionMode");
   }
-  auto execMode = opBuilder.getI32IntegerAttr(words[wordIndex++]);
+  auto execMode = spirv::ExecutionModeAttr::get(
+      context, static_cast<spirv::ExecutionMode>(words[wordIndex++]));
 
   // Get the values
   SmallVector<Attribute, 4> attrListElems;
@@ -417,8 +419,11 @@ Deserializer::processOp<spirv::ControlBarrierOp>(ArrayRef<uint32_t> operands) {
     argAttrs.push_back(argAttr);
   }
 
-  opBuilder.create<spirv::ControlBarrierOp>(unknownLoc, argAttrs[0],
-                                            argAttrs[1], argAttrs[2]);
+  opBuilder.create<spirv::ControlBarrierOp>(
+      unknownLoc, argAttrs[0].cast<spirv::ScopeAttr>(),
+      argAttrs[1].cast<spirv::ScopeAttr>(),
+      argAttrs[2].cast<spirv::MemorySemanticsAttr>());
+
   return success();
 }
 
@@ -483,8 +488,9 @@ Deserializer::processOp<spirv::MemoryBarrierOp>(ArrayRef<uint32_t> operands) {
     argAttrs.push_back(argAttr);
   }
 
-  opBuilder.create<spirv::MemoryBarrierOp>(unknownLoc, argAttrs[0],
-                                           argAttrs[1]);
+  opBuilder.create<spirv::MemoryBarrierOp>(
+      unknownLoc, argAttrs[0].cast<spirv::ScopeAttr>(),
+      argAttrs[1].cast<spirv::MemorySemanticsAttr>());
   return success();
 }
 
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index 171d9b7f63ea134fb5dbe9359f9bc99f78e00521..c54c16853719d2a116171de6d8aa2b30524d69dd 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -1640,7 +1640,7 @@ ControlFlowStructurizer::createSelectionOp(uint32_t selectionControl) {
   // merge block so that the newly created SelectionOp will be inserted there.
   OpBuilder builder(&mergeBlock->front());
 
-  auto control = builder.getI32IntegerAttr(selectionControl);
+  auto control = static_cast<spirv::SelectionControl>(selectionControl);
   auto selectionOp = builder.create<spirv::SelectionOp>(location, control);
   selectionOp.addMergeBlock();
 
@@ -1652,7 +1652,7 @@ spirv::LoopOp ControlFlowStructurizer::createLoopOp(uint32_t loopControl) {
   // merge block so that the newly created LoopOp will be inserted there.
   OpBuilder builder(&mergeBlock->front());
 
-  auto control = builder.getI32IntegerAttr(loopControl);
+  auto control = static_cast<spirv::LoopControl>(loopControl);
   auto loopOp = builder.create<spirv::LoopOp>(location, control);
   loopOp.addEntryAndMergeBlock();
 
diff --git a/mlir/lib/Transforms/BufferDeallocation.cpp b/mlir/lib/Transforms/BufferDeallocation.cpp
index aa837cb0e77c5a1a99d9790bb2feedad0e06ccee..3ba744d8e6efb7530cdedba3524ececc7739baaf 100644
--- a/mlir/lib/Transforms/BufferDeallocation.cpp
+++ b/mlir/lib/Transforms/BufferDeallocation.cpp
@@ -7,16 +7,15 @@
 //===----------------------------------------------------------------------===//
 //
 // This file implements logic for computing correct alloc and dealloc positions.
-// Furthermore, buffer placement also adds required new alloc and copy
-// operations to ensure that all buffers are deallocated. The main class is the
+// Furthermore, buffer deallocation also adds required new clone operations to
+// ensure that all buffers are deallocated. The main class is the
 // BufferDeallocationPass class that implements the underlying algorithm. In
 // order to put allocations and deallocations at safe positions, it is
 // significantly important to put them into the correct blocks. However, the
 // liveness analysis does not pay attention to aliases, which can occur due to
 // branches (and their associated block arguments) in general. For this purpose,
 // BufferDeallocation firstly finds all possible aliases for a single value
-// (using the BufferAliasAnalysis class). Consider the following
-// example:
+// (using the BufferAliasAnalysis class). Consider the following example:
 //
 // ^bb0(%arg0):
 //   cond_br %cond, ^bb1, ^bb2
@@ -30,16 +29,16 @@
 //
 // We should place the dealloc for %new_value in exit. However, we have to free
 // the buffer in the same block, because it cannot be freed in the post
-// dominator. However, this requires a new copy buffer for %arg1 that will
+// dominator. However, this requires a new clone buffer for %arg1 that will
 // contain the actual contents. Using the class BufferAliasAnalysis, we
 // will find out that %new_value has a potential alias %arg1. In order to find
 // the dealloc position we have to find all potential aliases, iterate over
 // their uses and find the common post-dominator block (note that additional
-// copies and buffers remove potential aliases and will influence the placement
+// clones and buffers remove potential aliases and will influence the placement
 // of the deallocs). In all cases, the computed block can be safely used to free
 // the %new_value buffer (may be exit or bb2) as it will die and we can use
 // liveness information to determine the exact operation after which we have to
-// insert the dealloc. However, the algorithm supports introducing copy buffers
+// insert the dealloc. However, the algorithm supports introducing clone buffers
 // and placing deallocs in safe locations to ensure that all buffers will be
 // freed in the end.
 //
@@ -52,10 +51,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "PassDetail.h"
-#include "mlir/Dialect/Linalg/IR/LinalgOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/Dialect/StandardOps/Utils/Utils.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
@@ -187,25 +184,25 @@ private:
 
 /// The buffer deallocation transformation which ensures that all allocs in the
 /// program have a corresponding de-allocation. As a side-effect, it might also
-/// introduce copies that in turn leads to additional allocs and de-allocations.
+/// introduce clones that in turn leads to additional deallocations.
 class BufferDeallocation : BufferPlacementTransformationBase {
 public:
   BufferDeallocation(Operation *op)
       : BufferPlacementTransformationBase(op), dominators(op),
         postDominators(op) {}
 
-  /// Performs the actual placement/creation of all temporary alloc, copy and
-  /// dealloc nodes.
+  /// Performs the actual placement/creation of all temporary clone and dealloc
+  /// nodes.
   void deallocate() {
-    // Add additional allocations and copies that are required.
-    introduceCopies();
+    // Add additional clones that are required.
+    introduceClones();
     // Place deallocations for all allocation entries.
     placeDeallocs();
   }
 
 private:
-  /// Introduces required allocs and copy operations to avoid memory leaks.
-  void introduceCopies() {
+  /// Introduces required clone operations to avoid memory leaks.
+  void introduceClones() {
     // Initialize the set of values that require a dedicated memory free
     // operation since their operands cannot be safely deallocated in a post
     // dominator.
@@ -214,7 +211,7 @@ private:
     SmallVector<std::tuple<Value, Block *>, 8> toProcess;
 
     // Check dominance relation for proper dominance properties. If the given
-    // value node does not dominate an alias, we will have to create a copy in
+    // value node does not dominate an alias, we will have to create a clone in
     // order to free all buffers that can potentially leak into a post
     // dominator.
     auto findUnsafeValues = [&](Value source, Block *definingBlock) {
@@ -255,7 +252,7 @@ private:
     // arguments at the correct locations.
     aliases.remove(valuesToFree);
 
-    // Add new allocs and additional copy operations.
+    // Add new allocs and additional clone operations.
     for (Value value : valuesToFree) {
       if (auto blockArg = value.dyn_cast<BlockArgument>())
         introduceBlockArgCopy(blockArg);
@@ -269,7 +266,7 @@ private:
     }
   }
 
-  /// Introduces temporary allocs in all predecessors and copies the source
+  /// Introduces temporary clones in all predecessors and copies the source
   /// values into the newly allocated buffers.
   void introduceBlockArgCopy(BlockArgument blockArg) {
     // Allocate a buffer for the current block argument in the block of
@@ -285,9 +282,9 @@ private:
       Value sourceValue =
           branchInterface.getSuccessorOperands(it.getSuccessorIndex())
               .getValue()[blockArg.getArgNumber()];
-      // Create a new alloc and copy at the current location of the terminator.
-      Value alloc = introduceBufferCopy(sourceValue, terminator);
-      // Wire new alloc and successor operand.
+      // Create a new clone at the current location of the terminator.
+      Value clone = introduceCloneBuffers(sourceValue, terminator);
+      // Wire new clone and successor operand.
       auto mutableOperands =
           branchInterface.getMutableSuccessorOperands(it.getSuccessorIndex());
       if (!mutableOperands.hasValue())
@@ -296,7 +293,7 @@ private:
       else
         mutableOperands.getValue()
             .slice(blockArg.getArgNumber(), 1)
-            .assign(alloc);
+            .assign(clone);
     }
 
     // Check whether the block argument has implicitly defined predecessors via
@@ -310,7 +307,7 @@ private:
         !(regionInterface = dyn_cast<RegionBranchOpInterface>(parentOp)))
       return;
 
-    introduceCopiesForRegionSuccessors(
+    introduceClonesForRegionSuccessors(
         regionInterface, argRegion->getParentOp()->getRegions(), blockArg,
         [&](RegionSuccessor &successorRegion) {
           // Find a predecessor of our argRegion.
@@ -318,7 +315,7 @@ private:
         });
 
     // Check whether the block argument belongs to an entry region of the
-    // parent operation. In this case, we have to introduce an additional copy
+    // parent operation. In this case, we have to introduce an additional clone
     // for buffer that is passed to the argument.
     SmallVector<RegionSuccessor, 2> successorRegions;
     regionInterface.getSuccessorRegions(/*index=*/llvm::None, successorRegions);
@@ -329,20 +326,20 @@ private:
     if (it == successorRegions.end())
       return;
 
-    // Determine the actual operand to introduce a copy for and rewire the
-    // operand to point to the copy instead.
+    // Determine the actual operand to introduce a clone for and rewire the
+    // operand to point to the clone instead.
     Value operand =
         regionInterface.getSuccessorEntryOperands(argRegion->getRegionNumber())
             [llvm::find(it->getSuccessorInputs(), blockArg).getIndex()];
-    Value copy = introduceBufferCopy(operand, parentOp);
+    Value clone = introduceCloneBuffers(operand, parentOp);
 
     auto op = llvm::find(parentOp->getOperands(), operand);
     assert(op != parentOp->getOperands().end() &&
            "parentOp does not contain operand");
-    parentOp->setOperand(op.getIndex(), copy);
+    parentOp->setOperand(op.getIndex(), clone);
   }
 
-  /// Introduces temporary allocs in front of all associated nested-region
+  /// Introduces temporary clones in front of all associated nested-region
   /// terminators and copies the source values into the newly allocated buffers.
   void introduceValueCopyForRegionResult(Value value) {
     // Get the actual result index in the scope of the parent terminator.
@@ -354,20 +351,20 @@ private:
       // its parent operation.
       return !successorRegion.getSuccessor();
     };
-    // Introduce a copy for all region "results" that are returned to the parent
-    // operation. This is required since the parent's result value has been
-    // considered critical. Therefore, the algorithm assumes that a copy of a
-    // previously allocated buffer is returned by the operation (like in the
-    // case of a block argument).
-    introduceCopiesForRegionSuccessors(regionInterface, operation->getRegions(),
+    // Introduce a clone for all region "results" that are returned to the
+    // parent operation. This is required since the parent's result value has
+    // been considered critical. Therefore, the algorithm assumes that a clone
+    // of a previously allocated buffer is returned by the operation (like in
+    // the case of a block argument).
+    introduceClonesForRegionSuccessors(regionInterface, operation->getRegions(),
                                        value, regionPredicate);
   }
 
-  /// Introduces buffer copies for all terminators in the given regions. The
+  /// Introduces buffer clones for all terminators in the given regions. The
   /// regionPredicate is applied to every successor region in order to restrict
-  /// the copies to specific regions.
+  /// the clones to specific regions.
   template <typename TPredicate>
-  void introduceCopiesForRegionSuccessors(
+  void introduceClonesForRegionSuccessors(
       RegionBranchOpInterface regionInterface, MutableArrayRef<Region> regions,
       Value argValue, const TPredicate &regionPredicate) {
     for (Region &region : regions) {
@@ -393,49 +390,37 @@ private:
       walkReturnOperations(&region, [&](Operation *terminator) {
         // Extract the source value from the current terminator.
         Value sourceValue = terminator->getOperand(operandIndex);
-        // Create a new alloc at the current location of the terminator.
-        Value alloc = introduceBufferCopy(sourceValue, terminator);
-        // Wire alloc and terminator operand.
-        terminator->setOperand(operandIndex, alloc);
+        // Create a new clone at the current location of the terminator.
+        Value clone = introduceCloneBuffers(sourceValue, terminator);
+        // Wire clone and terminator operand.
+        terminator->setOperand(operandIndex, clone);
       });
     }
   }
 
-  /// Creates a new memory allocation for the given source value and copies
+  /// Creates a new memory allocation for the given source value and clones
   /// its content into the newly allocated buffer. The terminator operation is
-  /// used to insert the alloc and copy operations at the right places.
-  Value introduceBufferCopy(Value sourceValue, Operation *terminator) {
-    // Avoid multiple copies of the same source value. This can happen in the
+  /// used to insert the clone operation at the right place.
+  Value introduceCloneBuffers(Value sourceValue, Operation *terminator) {
+    // Avoid multiple clones of the same source value. This can happen in the
     // presence of loops when a branch acts as a backedge while also having
     // another successor that returns to its parent operation. Note: that
     // copying copied buffers can introduce memory leaks since the invariant of
-    // BufferPlacement assumes that a buffer will be only copied once into a
-    // temporary buffer. Hence, the construction of copy chains introduces
+    // BufferDeallocation assumes that a buffer will be only cloned once into a
+    // temporary buffer. Hence, the construction of clone chains introduces
     // additional allocations that are not tracked automatically by the
     // algorithm.
-    if (copiedValues.contains(sourceValue))
+    if (clonedValues.contains(sourceValue))
       return sourceValue;
-    // Create a new alloc at the current location of the terminator.
-    auto memRefType = sourceValue.getType().cast<MemRefType>();
+    // Create a new clone operation that copies the contents of the old
+    // buffer to the new one.
     OpBuilder builder(terminator);
+    auto cloneOp =
+        builder.create<memref::CloneOp>(terminator->getLoc(), sourceValue);
 
-    // Extract information about dynamically shaped types by
-    // extracting their dynamic dimensions.
-    auto dynamicOperands =
-        getDynOperands(terminator->getLoc(), sourceValue, builder);
-
-    // TODO: provide a generic interface to create dialect-specific
-    // Alloc and CopyOp nodes.
-    auto alloc = builder.create<memref::AllocOp>(terminator->getLoc(),
-                                                 memRefType, dynamicOperands);
-
-    // Create a new copy operation that copies to contents of the old
-    // allocation to the new one.
-    builder.create<linalg::CopyOp>(terminator->getLoc(), sourceValue, alloc);
-
-    // Remember the copy of original source value.
-    copiedValues.insert(alloc);
-    return alloc;
+    // Remember the clone of original source value.
+    clonedValues.insert(cloneOp);
+    return cloneOp;
   }
 
   /// Finds correct dealloc positions according to the algorithm described at
@@ -513,8 +498,8 @@ private:
   /// position.
   PostDominanceInfo postDominators;
 
-  /// Stores already copied allocations to avoid additional copies of copies.
-  ValueSetT copiedValues;
+  /// Stores already cloned buffers to avoid additional clones of clones.
+  ValueSetT clonedValues;
 };
 
 //===----------------------------------------------------------------------===//
@@ -522,8 +507,8 @@ private:
 //===----------------------------------------------------------------------===//
 
 /// The actual buffer deallocation pass that inserts and moves dealloc nodes
-/// into the right positions. Furthermore, it inserts additional allocs and
-/// copies if necessary. It uses the algorithm described at the top of the file.
+/// into the right positions. Furthermore, it inserts additional clones if
+/// necessary. It uses the algorithm described at the top of the file.
 struct BufferDeallocationPass : BufferDeallocationBase<BufferDeallocationPass> {
 
   void runOnFunction() override {
@@ -540,7 +525,7 @@ struct BufferDeallocationPass : BufferDeallocationBase<BufferDeallocationPass> {
       return signalPassFailure();
     }
 
-    // Place all required temporary alloc, copy and dealloc nodes.
+    // Place all required temporary clone and dealloc nodes.
     BufferDeallocation deallocation(getFunction());
     deallocation.deallocate();
   }
diff --git a/mlir/lib/Transforms/BufferUtils.cpp b/mlir/lib/Transforms/BufferUtils.cpp
index ab39f57b3fcc336906a7b985e5b1827d659b7259..0cefd53d2d3478e6bb9863e408a6c88e89396e0c 100644
--- a/mlir/lib/Transforms/BufferUtils.cpp
+++ b/mlir/lib/Transforms/BufferUtils.cpp
@@ -12,7 +12,7 @@
 
 #include "mlir/Transforms/BufferUtils.h"
 #include "PassDetail.h"
-#include "mlir/Dialect/Linalg/IR/LinalgOps.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
@@ -49,25 +49,6 @@ Operation *BufferPlacementAllocs::getStartOperation(Value allocValue,
   return startOperation;
 }
 
-/// Finds associated deallocs that can be linked to our allocation nodes (if
-/// any).
-Operation *BufferPlacementAllocs::findDealloc(Value allocValue) {
-  auto userIt = llvm::find_if(allocValue.getUsers(), [&](Operation *user) {
-    auto effectInterface = dyn_cast<MemoryEffectOpInterface>(user);
-    if (!effectInterface)
-      return false;
-    // Try to find a free effect that is applied to one of our values
-    // that will be automatically freed by our pass.
-    SmallVector<MemoryEffects::EffectInstance, 2> effects;
-    effectInterface.getEffectsOnValue(allocValue, effects);
-    return llvm::any_of(effects, [&](MemoryEffects::EffectInstance &it) {
-      return isa<MemoryEffects::Free>(it.getEffect());
-    });
-  });
-  // Assign the associated dealloc operation (if any).
-  return userIt != allocValue.user_end() ? *userIt : nullptr;
-}
-
 /// Initializes the internal list by discovering all supported allocation
 /// nodes.
 BufferPlacementAllocs::BufferPlacementAllocs(Operation *op) { build(op); }
diff --git a/mlir/lib/Transforms/Bufferize.cpp b/mlir/lib/Transforms/Bufferize.cpp
index 74de8613646a06ef21a30554ec2bc0baf9ee6bf2..7ed7526549fb1f1a8caa631cb17fd363e627ce82 100644
--- a/mlir/lib/Transforms/Bufferize.cpp
+++ b/mlir/lib/Transforms/Bufferize.cpp
@@ -84,10 +84,9 @@ public:
 } // namespace
 
 void mlir::populateEliminateBufferizeMaterializationsPatterns(
-    MLIRContext *context, BufferizeTypeConverter &typeConverter,
-    OwningRewritePatternList &patterns) {
-  patterns.insert<BufferizeTensorLoadOp, BufferizeCastOp>(typeConverter,
-                                                          context);
+    BufferizeTypeConverter &typeConverter, RewritePatternSet &patterns) {
+  patterns.add<BufferizeTensorLoadOp, BufferizeCastOp>(typeConverter,
+                                                       patterns.getContext());
 }
 
 namespace {
@@ -101,11 +100,10 @@ struct FinalizingBufferizePass
     auto *context = &getContext();
 
     BufferizeTypeConverter typeConverter;
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
     ConversionTarget target(*context);
 
-    populateEliminateBufferizeMaterializationsPatterns(context, typeConverter,
-                                                       patterns);
+    populateEliminateBufferizeMaterializationsPatterns(typeConverter, patterns);
 
     // If all result types are legal, and all block arguments are legal (ensured
     // by func conversion above), then all types in the program are legal.
diff --git a/mlir/lib/Transforms/CMakeLists.txt b/mlir/lib/Transforms/CMakeLists.txt
index 36f9e5b832be9405395cfbade23fb775f1fb78d2..2b185fcf0b7ee0aabf6fa0e8b34391ff0b1a35d7 100644
--- a/mlir/lib/Transforms/CMakeLists.txt
+++ b/mlir/lib/Transforms/CMakeLists.txt
@@ -7,7 +7,6 @@ add_mlir_library(MLIRTransforms
   BufferUtils.cpp
   Bufferize.cpp
   Canonicalizer.cpp
-  CopyRemoval.cpp
   CSE.cpp
   Inliner.cpp
   LocationSnapshot.cpp
diff --git a/mlir/lib/Transforms/Canonicalizer.cpp b/mlir/lib/Transforms/Canonicalizer.cpp
index cd99681a0283ce95702e4e724eb3048eec00f2ba..2d987778f22f21e87af6763c096a9120c7a1402c 100644
--- a/mlir/lib/Transforms/Canonicalizer.cpp
+++ b/mlir/lib/Transforms/Canonicalizer.cpp
@@ -25,7 +25,7 @@ struct Canonicalizer : public CanonicalizerBase<Canonicalizer> {
   /// Initialize the canonicalizer by building the set of patterns used during
   /// execution.
   LogicalResult initialize(MLIRContext *context) override {
-    OwningRewritePatternList owningPatterns;
+    RewritePatternSet owningPatterns(context);
     for (auto *op : context->getRegisteredOperations())
       op->getCanonicalizationPatterns(owningPatterns, context);
     patterns = std::move(owningPatterns);
@@ -35,7 +35,7 @@ struct Canonicalizer : public CanonicalizerBase<Canonicalizer> {
     (void)applyPatternsAndFoldGreedily(getOperation()->getRegions(), patterns);
   }
 
-  FrozenRewritePatternList patterns;
+  FrozenRewritePatternSet patterns;
 };
 } // end anonymous namespace
 
diff --git a/mlir/lib/Transforms/CopyRemoval.cpp b/mlir/lib/Transforms/CopyRemoval.cpp
deleted file mode 100644
index c5a8da63295681841febd0469df6f61b57d96b2d..0000000000000000000000000000000000000000
--- a/mlir/lib/Transforms/CopyRemoval.cpp
+++ /dev/null
@@ -1,217 +0,0 @@
-//===- CopyRemoval.cpp - Removing the redundant copies --------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Interfaces/CopyOpInterface.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Transforms/Passes.h"
-
-using namespace mlir;
-using namespace MemoryEffects;
-
-namespace {
-
-//===----------------------------------------------------------------------===//
-// CopyRemovalPass
-//===----------------------------------------------------------------------===//
-
-/// This pass removes the redundant Copy operations. Additionally, it
-/// removes the leftover definition and deallocation operations by erasing the
-/// copy operation.
-class CopyRemovalPass : public PassWrapper<CopyRemovalPass, OperationPass<>> {
-public:
-  void runOnOperation() override {
-    getOperation()->walk([&](CopyOpInterface copyOp) {
-      reuseCopySourceAsTarget(copyOp);
-      reuseCopyTargetAsSource(copyOp);
-    });
-    for (std::pair<Value, Value> &pair : replaceList)
-      pair.first.replaceAllUsesWith(pair.second);
-    for (Operation *op : eraseList)
-      op->erase();
-  }
-
-private:
-  /// List of operations that need to be removed.
-  llvm::SmallPtrSet<Operation *, 4> eraseList;
-
-  /// List of values that need to be replaced with their counterparts.
-  llvm::SmallDenseSet<std::pair<Value, Value>, 4> replaceList;
-
-  /// Returns the allocation operation for `value` in `block` if it exists.
-  /// nullptr otherwise.
-  Operation *getAllocationOpInBlock(Value value, Block *block) {
-    assert(block && "Block cannot be null");
-    Operation *op = value.getDefiningOp();
-    if (op && op->getBlock() == block) {
-      auto effects = dyn_cast<MemoryEffectOpInterface>(op);
-      if (effects && effects.hasEffect<Allocate>())
-        return op;
-    }
-    return nullptr;
-  }
-
-  /// Returns the deallocation operation for `value` in `block` if it exists.
-  /// nullptr otherwise.
-  Operation *getDeallocationOpInBlock(Value value, Block *block) {
-    assert(block && "Block cannot be null");
-    auto valueUsers = value.getUsers();
-    auto it = llvm::find_if(valueUsers, [&](Operation *op) {
-      auto effects = dyn_cast<MemoryEffectOpInterface>(op);
-      return effects && op->getBlock() == block && effects.hasEffect<Free>();
-    });
-    return (it == valueUsers.end() ? nullptr : *it);
-  }
-
-  /// Returns true if an operation between start and end operations has memory
-  /// effect.
-  bool hasMemoryEffectOpBetween(Operation *start, Operation *end) {
-    assert((start || end) && "Start and end operations cannot be null");
-    assert(start->getBlock() == end->getBlock() &&
-           "Start and end operations should be in the same block.");
-    Operation *op = start->getNextNode();
-    while (op->isBeforeInBlock(end)) {
-      if (isa<MemoryEffectOpInterface>(op))
-        return true;
-      op = op->getNextNode();
-    }
-    return false;
-  };
-
-  /// Returns true if `val` value has at least a user between `start` and
-  /// `end` operations.
-  bool hasUsersBetween(Value val, Operation *start, Operation *end) {
-    assert((start || end) && "Start and end operations cannot be null");
-    Block *block = start->getBlock();
-    assert(block == end->getBlock() &&
-           "Start and end operations should be in the same block.");
-    return llvm::any_of(val.getUsers(), [&](Operation *op) {
-      return op->getBlock() == block && start->isBeforeInBlock(op) &&
-             op->isBeforeInBlock(end);
-    });
-  };
-
-  bool areOpsInTheSameBlock(ArrayRef<Operation *> operations) {
-    assert(!operations.empty() &&
-           "The operations list should contain at least a single operation");
-    Block *block = operations.front()->getBlock();
-    return llvm::none_of(
-        operations, [&](Operation *op) { return block != op->getBlock(); });
-  }
-
-  /// Input:
-  /// func(){
-  ///   %from = alloc()
-  ///   write_to(%from)
-  ///   %to = alloc()
-  ///   copy(%from,%to)
-  ///   dealloc(%from)
-  ///   return %to
-  /// }
-  ///
-  /// Output:
-  /// func(){
-  ///   %from = alloc()
-  ///   write_to(%from)
-  ///   return %from
-  /// }
-  /// Constraints:
-  /// 1) %to, copy and dealloc must all be defined and lie in the same block.
-  /// 2) This transformation cannot be applied if there is a single user/alias
-  /// of `to` value between the defining operation of `to` and the copy
-  /// operation.
-  /// 3) This transformation cannot be applied if there is a single user/alias
-  /// of `from` value between the copy operation and the deallocation of `from`.
-  /// TODO: Alias analysis is not available at the moment. Currently, we check
-  /// if there are any operations with memory effects between copy and
-  /// deallocation operations.
-  void reuseCopySourceAsTarget(CopyOpInterface copyOp) {
-    if (eraseList.count(copyOp))
-      return;
-
-    Value from = copyOp.getSource();
-    Value to = copyOp.getTarget();
-
-    Operation *copy = copyOp.getOperation();
-    Block *copyBlock = copy->getBlock();
-    Operation *fromDefiningOp = from.getDefiningOp();
-    Operation *fromFreeingOp = getDeallocationOpInBlock(from, copyBlock);
-    Operation *toDefiningOp = getAllocationOpInBlock(to, copyBlock);
-    if (!fromDefiningOp || !fromFreeingOp || !toDefiningOp ||
-        !areOpsInTheSameBlock({fromFreeingOp, toDefiningOp, copy}) ||
-        hasUsersBetween(to, toDefiningOp, copy) ||
-        hasUsersBetween(from, copy, fromFreeingOp) ||
-        hasMemoryEffectOpBetween(copy, fromFreeingOp))
-      return;
-
-    replaceList.insert({to, from});
-    eraseList.insert(copy);
-    eraseList.insert(toDefiningOp);
-    eraseList.insert(fromFreeingOp);
-  }
-
-  /// Input:
-  /// func(){
-  ///   %to = alloc()
-  ///   %from = alloc()
-  ///   write_to(%from)
-  ///   copy(%from,%to)
-  ///   dealloc(%from)
-  ///   return %to
-  /// }
-  ///
-  /// Output:
-  /// func(){
-  ///   %to = alloc()
-  ///   write_to(%to)
-  ///   return %to
-  /// }
-  /// Constraints:
-  /// 1) %from, copy and dealloc must all be defined and lie in the same block.
-  /// 2) This transformation cannot be applied if there is a single user/alias
-  /// of `to` value between the defining operation of `from` and the copy
-  /// operation.
-  /// 3) This transformation cannot be applied if there is a single user/alias
-  /// of `from` value between the copy operation and the deallocation of `from`.
-  /// TODO: Alias analysis is not available at the moment. Currently, we check
-  /// if there are any operations with memory effects between copy and
-  /// deallocation operations.
-  void reuseCopyTargetAsSource(CopyOpInterface copyOp) {
-    if (eraseList.count(copyOp))
-      return;
-
-    Value from = copyOp.getSource();
-    Value to = copyOp.getTarget();
-
-    Operation *copy = copyOp.getOperation();
-    Block *copyBlock = copy->getBlock();
-    Operation *fromDefiningOp = getAllocationOpInBlock(from, copyBlock);
-    Operation *fromFreeingOp = getDeallocationOpInBlock(from, copyBlock);
-    if (!fromDefiningOp || !fromFreeingOp ||
-        !areOpsInTheSameBlock({fromFreeingOp, fromDefiningOp, copy}) ||
-        hasUsersBetween(to, fromDefiningOp, copy) ||
-        hasUsersBetween(from, copy, fromFreeingOp) ||
-        hasMemoryEffectOpBetween(copy, fromFreeingOp))
-      return;
-
-    replaceList.insert({from, to});
-    eraseList.insert(copy);
-    eraseList.insert(fromDefiningOp);
-    eraseList.insert(fromFreeingOp);
-  }
-};
-
-} // end anonymous namespace
-
-//===----------------------------------------------------------------------===//
-// CopyRemovalPass construction
-//===----------------------------------------------------------------------===//
-
-std::unique_ptr<Pass> mlir::createCopyRemovalPass() {
-  return std::make_unique<CopyRemovalPass>();
-}
diff --git a/mlir/lib/Transforms/SymbolDCE.cpp b/mlir/lib/Transforms/SymbolDCE.cpp
index 5d94245489c03ac19c724c821fff30f07cdb1af0..2c65d635d4c4466a046bffcd962003a49a45d2f0 100644
--- a/mlir/lib/Transforms/SymbolDCE.cpp
+++ b/mlir/lib/Transforms/SymbolDCE.cpp
@@ -61,8 +61,7 @@ void SymbolDCE::runOnOperation() {
     if (!nestedSymbolTable->hasTrait<OpTrait::SymbolTable>())
       return;
     for (auto &block : nestedSymbolTable->getRegion(0)) {
-      for (Operation &op :
-           llvm::make_early_inc_range(block.without_terminator())) {
+      for (Operation &op : llvm::make_early_inc_range(block)) {
         if (isa<SymbolOpInterface>(&op) && !liveSymbols.count(&op))
           op.erase();
       }
@@ -84,7 +83,7 @@ LogicalResult SymbolDCE::computeLiveness(Operation *symbolTableOp,
   // are known to be live.
   for (auto &block : symbolTableOp->getRegion(0)) {
     // Add all non-symbols or symbols that can't be discarded.
-    for (Operation &op : block.without_terminator()) {
+    for (Operation &op : block) {
       SymbolOpInterface symbol = dyn_cast<SymbolOpInterface>(&op);
       if (!symbol) {
         worklist.push_back(&op);
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 5c99c5812b546e408fa78e097d18be82988aafc3..821d867b7d1dff6614169ccf3d6f82e16b3e48aa 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -75,7 +75,8 @@ computeConversionSet(iterator_range<Region::iterator> region,
 
 /// A utility function to log a successful result for the given reason.
 template <typename... Args>
-static void logSuccess(llvm::ScopedPrinter &os, StringRef fmt, Args &&...args) {
+static void logSuccess(llvm::ScopedPrinter &os, StringRef fmt,
+                       Args &&... args) {
   LLVM_DEBUG({
     os.unindent();
     os.startLine() << "} -> SUCCESS";
@@ -88,7 +89,8 @@ static void logSuccess(llvm::ScopedPrinter &os, StringRef fmt, Args &&...args) {
 
 /// A utility function to log a failure result for the given reason.
 template <typename... Args>
-static void logFailure(llvm::ScopedPrinter &os, StringRef fmt, Args &&...args) {
+static void logFailure(llvm::ScopedPrinter &os, StringRef fmt,
+                       Args &&... args) {
   LLVM_DEBUG({
     os.unindent();
     os.startLine() << "} -> FAILURE : "
@@ -1504,7 +1506,7 @@ public:
   using LegalizationAction = ConversionTarget::LegalizationAction;
 
   OperationLegalizer(ConversionTarget &targetInfo,
-                     const FrozenRewritePatternList &patterns);
+                     const FrozenRewritePatternSet &patterns);
 
   /// Returns true if the given operation is known to be illegal on the target.
   bool isIllegal(Operation *op) const;
@@ -1600,7 +1602,7 @@ private:
 } // namespace
 
 OperationLegalizer::OperationLegalizer(ConversionTarget &targetInfo,
-                                       const FrozenRewritePatternList &patterns)
+                                       const FrozenRewritePatternSet &patterns)
     : target(targetInfo), applicator(patterns) {
   // The set of patterns that can be applied to illegal operations to transform
   // them into legal ones.
@@ -2123,7 +2125,7 @@ enum OpConversionMode {
 // conversion mode.
 struct OperationConverter {
   explicit OperationConverter(ConversionTarget &target,
-                              const FrozenRewritePatternList &patterns,
+                              const FrozenRewritePatternSet &patterns,
                               OpConversionMode mode,
                               DenseSet<Operation *> *trackedOps = nullptr)
       : opLegalizer(target, patterns), mode(mode), trackedOps(trackedOps) {}
@@ -2580,7 +2582,7 @@ namespace {
 struct FunctionLikeSignatureConversion : public ConversionPattern {
   FunctionLikeSignatureConversion(StringRef functionLikeOpName,
                                   MLIRContext *ctx, TypeConverter &converter)
-      : ConversionPattern(functionLikeOpName, /*benefit=*/1, converter, ctx) {}
+      : ConversionPattern(converter, functionLikeOpName, /*benefit=*/1, ctx) {}
 
   /// Hook to implement combined matching and rewriting for FunctionLike ops.
   LogicalResult
@@ -2610,16 +2612,15 @@ struct FunctionLikeSignatureConversion : public ConversionPattern {
 } // end anonymous namespace
 
 void mlir::populateFunctionLikeTypeConversionPattern(
-    StringRef functionLikeOpName, OwningRewritePatternList &patterns,
-    MLIRContext *ctx, TypeConverter &converter) {
-  patterns.insert<FunctionLikeSignatureConversion>(functionLikeOpName, ctx,
-                                                   converter);
+    StringRef functionLikeOpName, RewritePatternSet &patterns,
+    TypeConverter &converter) {
+  patterns.add<FunctionLikeSignatureConversion>(
+      functionLikeOpName, patterns.getContext(), converter);
 }
 
-void mlir::populateFuncOpTypeConversionPattern(
-    OwningRewritePatternList &patterns, MLIRContext *ctx,
-    TypeConverter &converter) {
-  populateFunctionLikeTypeConversionPattern<FuncOp>(patterns, ctx, converter);
+void mlir::populateFuncOpTypeConversionPattern(RewritePatternSet &patterns,
+                                               TypeConverter &converter) {
+  populateFunctionLikeTypeConversionPattern<FuncOp>(patterns, converter);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2754,7 +2755,7 @@ auto ConversionTarget::getOpInfo(OperationName op) const
 LogicalResult
 mlir::applyPartialConversion(ArrayRef<Operation *> ops,
                              ConversionTarget &target,
-                             const FrozenRewritePatternList &patterns,
+                             const FrozenRewritePatternSet &patterns,
                              DenseSet<Operation *> *unconvertedOps) {
   OperationConverter opConverter(target, patterns, OpConversionMode::Partial,
                                  unconvertedOps);
@@ -2762,7 +2763,7 @@ mlir::applyPartialConversion(ArrayRef<Operation *> ops,
 }
 LogicalResult
 mlir::applyPartialConversion(Operation *op, ConversionTarget &target,
-                             const FrozenRewritePatternList &patterns,
+                             const FrozenRewritePatternSet &patterns,
                              DenseSet<Operation *> *unconvertedOps) {
   return applyPartialConversion(llvm::makeArrayRef(op), target, patterns,
                                 unconvertedOps);
@@ -2773,13 +2774,13 @@ mlir::applyPartialConversion(Operation *op, ConversionTarget &target,
 /// operation fails.
 LogicalResult
 mlir::applyFullConversion(ArrayRef<Operation *> ops, ConversionTarget &target,
-                          const FrozenRewritePatternList &patterns) {
+                          const FrozenRewritePatternSet &patterns) {
   OperationConverter opConverter(target, patterns, OpConversionMode::Full);
   return opConverter.convertOperations(ops);
 }
 LogicalResult
 mlir::applyFullConversion(Operation *op, ConversionTarget &target,
-                          const FrozenRewritePatternList &patterns) {
+                          const FrozenRewritePatternSet &patterns) {
   return applyFullConversion(llvm::makeArrayRef(op), target, patterns);
 }
 
@@ -2792,7 +2793,7 @@ mlir::applyFullConversion(Operation *op, ConversionTarget &target,
 LogicalResult
 mlir::applyAnalysisConversion(ArrayRef<Operation *> ops,
                               ConversionTarget &target,
-                              const FrozenRewritePatternList &patterns,
+                              const FrozenRewritePatternSet &patterns,
                               DenseSet<Operation *> &convertedOps) {
   OperationConverter opConverter(target, patterns, OpConversionMode::Analysis,
                                  &convertedOps);
@@ -2800,7 +2801,7 @@ mlir::applyAnalysisConversion(ArrayRef<Operation *> ops,
 }
 LogicalResult
 mlir::applyAnalysisConversion(Operation *op, ConversionTarget &target,
-                              const FrozenRewritePatternList &patterns,
+                              const FrozenRewritePatternSet &patterns,
                               DenseSet<Operation *> &convertedOps) {
   return applyAnalysisConversion(llvm::makeArrayRef(op), target, patterns,
                                  convertedOps);
diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
index 9ed3b3514db60da6dc1b5d4bc19f0ca7984fb45e..c82076bde0820c61bebb73d0600993ef0e05b547 100644
--- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
@@ -37,7 +37,7 @@ namespace {
 class GreedyPatternRewriteDriver : public PatternRewriter {
 public:
   explicit GreedyPatternRewriteDriver(MLIRContext *ctx,
-                                      const FrozenRewritePatternList &patterns)
+                                      const FrozenRewritePatternSet &patterns)
       : PatternRewriter(ctx), matcher(patterns), folder(ctx) {
     worklist.reserve(64);
 
@@ -107,14 +107,15 @@ private:
   // be re-added to the worklist. This function should be called when an
   // operation is modified or removed, as it may trigger further
   // simplifications.
-  template <typename Operands> void addToWorklist(Operands &&operands) {
+  template <typename Operands>
+  void addToWorklist(Operands &&operands) {
     for (Value operand : operands) {
       // If the use count of this operand is now < 2, we re-add the defining
       // operation to the worklist.
       // TODO: This is based on the fact that zero use operations
       // may be deleted, and that single use values often have more
       // canonicalization opportunities.
-      if (!operand.use_empty() && !operand.hasOneUse())
+      if (!operand || (!operand.use_empty() && !operand.hasOneUse()))
         continue;
       if (auto *defInst = operand.getDefiningOp())
         addToWorklist(defInst);
@@ -202,10 +203,7 @@ bool GreedyPatternRewriteDriver::simplify(MutableArrayRef<Region> regions,
 
     // After applying patterns, make sure that the CFG of each of the regions is
     // kept up to date.
-    if (succeeded(simplifyRegions(regions))) {
-      folder.clear();
-      changed = true;
-    }
+    changed |= succeeded(simplifyRegions(*this, regions));
   } while (changed && ++i < maxIterations);
   // Whether the rewrite converges, i.e. wasn't changed in the last iteration.
   return !changed;
@@ -219,12 +217,12 @@ bool GreedyPatternRewriteDriver::simplify(MutableArrayRef<Region> regions,
 ///
 LogicalResult
 mlir::applyPatternsAndFoldGreedily(Operation *op,
-                                   const FrozenRewritePatternList &patterns) {
+                                   const FrozenRewritePatternSet &patterns) {
   return applyPatternsAndFoldGreedily(op, patterns, maxPatternMatchIterations);
 }
 LogicalResult
 mlir::applyPatternsAndFoldGreedily(Operation *op,
-                                   const FrozenRewritePatternList &patterns,
+                                   const FrozenRewritePatternSet &patterns,
                                    unsigned maxIterations) {
   return applyPatternsAndFoldGreedily(op->getRegions(), patterns,
                                       maxIterations);
@@ -232,13 +230,13 @@ mlir::applyPatternsAndFoldGreedily(Operation *op,
 /// Rewrite the given regions, which must be isolated from above.
 LogicalResult
 mlir::applyPatternsAndFoldGreedily(MutableArrayRef<Region> regions,
-                                   const FrozenRewritePatternList &patterns) {
+                                   const FrozenRewritePatternSet &patterns) {
   return applyPatternsAndFoldGreedily(regions, patterns,
                                       maxPatternMatchIterations);
 }
 LogicalResult
 mlir::applyPatternsAndFoldGreedily(MutableArrayRef<Region> regions,
-                                   const FrozenRewritePatternList &patterns,
+                                   const FrozenRewritePatternSet &patterns,
                                    unsigned maxIterations) {
   if (regions.empty())
     return success();
@@ -273,7 +271,7 @@ namespace {
 class OpPatternRewriteDriver : public PatternRewriter {
 public:
   explicit OpPatternRewriteDriver(MLIRContext *ctx,
-                                  const FrozenRewritePatternList &patterns)
+                                  const FrozenRewritePatternSet &patterns)
       : PatternRewriter(ctx), matcher(patterns), folder(ctx) {
     // Apply a simple cost model based solely on pattern benefit.
     matcher.applyDefaultCostModel();
@@ -357,7 +355,7 @@ LogicalResult OpPatternRewriteDriver::simplifyLocally(Operation *op,
 /// folding. `erased` is set to true if the op is erased as a result of being
 /// folded, replaced, or dead.
 LogicalResult mlir::applyOpPatternsAndFold(
-    Operation *op, const FrozenRewritePatternList &patterns, bool *erased) {
+    Operation *op, const FrozenRewritePatternSet &patterns, bool *erased) {
   // Start the pattern driver.
   OpPatternRewriteDriver driver(op->getContext(), patterns);
   bool opErased;
diff --git a/mlir/lib/Transforms/Utils/LoopUtils.cpp b/mlir/lib/Transforms/Utils/LoopUtils.cpp
index a9b5979278ffc014191ed710e92dbabec782524c..5aaa8fa0daaded21229a29fc1f8f95b0e117a6fc 100644
--- a/mlir/lib/Transforms/Utils/LoopUtils.cpp
+++ b/mlir/lib/Transforms/Utils/LoopUtils.cpp
@@ -403,7 +403,7 @@ LogicalResult mlir::affineForOpBodySkew(AffineForOp forOp,
 
       if (res) {
         // Simplify/canonicalize the affine.for.
-        OwningRewritePatternList patterns;
+        RewritePatternSet patterns(res.getContext());
         AffineForOp::getCanonicalizationPatterns(patterns, res.getContext());
         bool erased;
         (void)applyOpPatternsAndFold(res, std::move(patterns), &erased);
diff --git a/mlir/lib/Transforms/Utils/RegionUtils.cpp b/mlir/lib/Transforms/Utils/RegionUtils.cpp
index 7dd064ef0341dbf98d0f22ec35448c3f85e91570..f0e03c537a9c9bba5b7c6bf827c5ddc36355e996 100644
--- a/mlir/lib/Transforms/Utils/RegionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/RegionUtils.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Transforms/RegionUtils.h"
 #include "mlir/IR/Block.h"
 #include "mlir/IR/Operation.h"
+#include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/RegionGraphTraits.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
@@ -75,7 +76,8 @@ void mlir::getUsedValuesDefinedAbove(MutableArrayRef<Region> regions,
 /// Erase the unreachable blocks within the provided regions. Returns success
 /// if any blocks were erased, failure otherwise.
 // TODO: We could likely merge this with the DCE algorithm below.
-static LogicalResult eraseUnreachableBlocks(MutableArrayRef<Region> regions) {
+static LogicalResult eraseUnreachableBlocks(RewriterBase &rewriter,
+                                            MutableArrayRef<Region> regions) {
   // Set of blocks found to be reachable within a given region.
   llvm::df_iterator_default_set<Block *, 16> reachable;
   // If any blocks were found to be dead.
@@ -108,7 +110,7 @@ static LogicalResult eraseUnreachableBlocks(MutableArrayRef<Region> regions) {
     for (Block &block : llvm::make_early_inc_range(*region)) {
       if (!reachable.count(&block)) {
         block.dropAllDefinedValueUses();
-        block.erase();
+        rewriter.eraseBlock(&block);
         erasedDeadBlocks = true;
         continue;
       }
@@ -305,32 +307,32 @@ static void eraseTerminatorSuccessorOperands(Operation *terminator,
   }
 }
 
-static LogicalResult deleteDeadness(MutableArrayRef<Region> regions,
+static LogicalResult deleteDeadness(RewriterBase &rewriter,
+                                    MutableArrayRef<Region> regions,
                                     LiveMap &liveMap) {
   bool erasedAnything = false;
   for (Region &region : regions) {
     if (region.empty())
       continue;
+    bool hasSingleBlock = llvm::hasSingleElement(region);
 
-    // We do the deletion in an order that deletes all uses before deleting
-    // defs.
-    // MLIR's SSA structural invariants guarantee that except for block
-    // arguments, the use-def graph is acyclic, so this is possible with a
-    // single walk of ops and then a final pass to clean up block arguments.
-    //
-    // To do this, we visit ops in an order that visits domtree children
-    // before domtree parents. A CFG post-order (with reverse iteration with a
-    // block) satisfies that without needing an explicit domtree calculation.
+    // Delete every operation that is not live. Graph regions may have cycles
+    // in the use-def graph, so we must explicitly dropAllUses() from each
+    // operation as we erase it. Visiting the operations in post-order
+    // guarantees that in SSA CFG regions value uses are removed before defs,
+    // which makes dropAllUses() a no-op.
     for (Block *block : llvm::post_order(&region.front())) {
-      eraseTerminatorSuccessorOperands(block->getTerminator(), liveMap);
+      if (!hasSingleBlock)
+        eraseTerminatorSuccessorOperands(block->getTerminator(), liveMap);
       for (Operation &childOp :
            llvm::make_early_inc_range(llvm::reverse(block->getOperations()))) {
         if (!liveMap.wasProvenLive(&childOp)) {
           erasedAnything = true;
-          childOp.erase();
+          childOp.dropAllUses();
+          rewriter.eraseOp(&childOp);
         } else {
-          erasedAnything |=
-              succeeded(deleteDeadness(childOp.getRegions(), liveMap));
+          erasedAnything |= succeeded(
+              deleteDeadness(rewriter, childOp.getRegions(), liveMap));
         }
       }
     }
@@ -362,7 +364,8 @@ static LogicalResult deleteDeadness(MutableArrayRef<Region> regions,
 //
 // This function returns success if any operations or arguments were deleted,
 // failure otherwise.
-static LogicalResult runRegionDCE(MutableArrayRef<Region> regions) {
+static LogicalResult runRegionDCE(RewriterBase &rewriter,
+                                  MutableArrayRef<Region> regions) {
   LiveMap liveMap;
   do {
     liveMap.resetChanged();
@@ -371,7 +374,7 @@ static LogicalResult runRegionDCE(MutableArrayRef<Region> regions) {
       propagateLiveness(region, liveMap);
   } while (liveMap.hasChanged());
 
-  return deleteDeadness(regions, liveMap);
+  return deleteDeadness(rewriter, regions, liveMap);
 }
 
 //===----------------------------------------------------------------------===//
@@ -459,7 +462,7 @@ public:
   LogicalResult addToCluster(BlockEquivalenceData &blockData);
 
   /// Try to merge all of the blocks within this cluster into the leader block.
-  LogicalResult merge();
+  LogicalResult merge(RewriterBase &rewriter);
 
 private:
   /// The equivalence data for the leader of the cluster.
@@ -553,7 +556,7 @@ static bool ableToUpdatePredOperands(Block *block) {
   return true;
 }
 
-LogicalResult BlockMergeCluster::merge() {
+LogicalResult BlockMergeCluster::merge(RewriterBase &rewriter) {
   // Don't consider clusters that don't have blocks to merge.
   if (blocksToMerge.empty())
     return failure();
@@ -616,7 +619,7 @@ LogicalResult BlockMergeCluster::merge() {
   // Replace all uses of the merged blocks with the leader and erase them.
   for (Block *block : blocksToMerge) {
     block->replaceAllUsesWith(leaderBlock);
-    block->erase();
+    rewriter.eraseBlock(block);
   }
   return success();
 }
@@ -624,7 +627,8 @@ LogicalResult BlockMergeCluster::merge() {
 /// Identify identical blocks within the given region and merge them, inserting
 /// new block arguments as necessary. Returns success if any blocks were merged,
 /// failure otherwise.
-static LogicalResult mergeIdenticalBlocks(Region &region) {
+static LogicalResult mergeIdenticalBlocks(RewriterBase &rewriter,
+                                          Region &region) {
   if (region.empty() || llvm::hasSingleElement(region))
     return failure();
 
@@ -662,7 +666,7 @@ static LogicalResult mergeIdenticalBlocks(Region &region) {
         clusters.emplace_back(std::move(data));
     }
     for (auto &cluster : clusters)
-      mergedAnyBlocks |= succeeded(cluster.merge());
+      mergedAnyBlocks |= succeeded(cluster.merge(rewriter));
   }
 
   return success(mergedAnyBlocks);
@@ -670,14 +674,15 @@ static LogicalResult mergeIdenticalBlocks(Region &region) {
 
 /// Identify identical blocks within the given regions and merge them, inserting
 /// new block arguments as necessary.
-static LogicalResult mergeIdenticalBlocks(MutableArrayRef<Region> regions) {
+static LogicalResult mergeIdenticalBlocks(RewriterBase &rewriter,
+                                          MutableArrayRef<Region> regions) {
   llvm::SmallSetVector<Region *, 1> worklist;
   for (auto &region : regions)
     worklist.insert(&region);
   bool anyChanged = false;
   while (!worklist.empty()) {
     Region *region = worklist.pop_back_val();
-    if (succeeded(mergeIdenticalBlocks(*region))) {
+    if (succeeded(mergeIdenticalBlocks(rewriter, *region))) {
       worklist.insert(region);
       anyChanged = true;
     }
@@ -700,10 +705,12 @@ static LogicalResult mergeIdenticalBlocks(MutableArrayRef<Region> regions) {
 /// includes transformations like unreachable block elimination, dead argument
 /// elimination, as well as some other DCE. This function returns success if any
 /// of the regions were simplified, failure otherwise.
-LogicalResult mlir::simplifyRegions(MutableArrayRef<Region> regions) {
-  bool eliminatedBlocks = succeeded(eraseUnreachableBlocks(regions));
-  bool eliminatedOpsOrArgs = succeeded(runRegionDCE(regions));
-  bool mergedIdenticalBlocks = succeeded(mergeIdenticalBlocks(regions));
+LogicalResult mlir::simplifyRegions(RewriterBase &rewriter,
+                                    MutableArrayRef<Region> regions) {
+  bool eliminatedBlocks = succeeded(eraseUnreachableBlocks(rewriter, regions));
+  bool eliminatedOpsOrArgs = succeeded(runRegionDCE(rewriter, regions));
+  bool mergedIdenticalBlocks =
+      succeeded(mergeIdenticalBlocks(rewriter, regions));
   return success(eliminatedBlocks || eliminatedOpsOrArgs ||
                  mergedIdenticalBlocks);
 }
diff --git a/mlir/test/Bindings/Python/context_managers.py b/mlir/test/Bindings/Python/context_managers.py
index 33a89381a4164884ab8c11a6624c75ced9f3e8b9..b93fcf70ac4820c68289b036afe530db2927784a 100644
--- a/mlir/test/Bindings/Python/context_managers.py
+++ b/mlir/test/Bindings/Python/context_managers.py
@@ -62,7 +62,7 @@ run(testLocationEnterExit)
 def testInsertionPointEnterExit():
   ctx1 = Context()
   m = Module.create(Location.unknown(ctx1))
-  ip = InsertionPoint.at_block_terminator(m.body)
+  ip = InsertionPoint(m.body)
 
   with ip:
     assert InsertionPoint.current is ip
diff --git a/mlir/test/Bindings/Python/dialects.py b/mlir/test/Bindings/Python/dialects.py
index 128f64cab199c79a9d3e412a1470358c99444f36..41f4239e2b660db931bc2f834caa8f180e61b44a 100644
--- a/mlir/test/Bindings/Python/dialects.py
+++ b/mlir/test/Bindings/Python/dialects.py
@@ -77,7 +77,7 @@ def testCustomOpView():
     ctx.allow_unregistered_dialects = True
     m = Module.create()
 
-    with InsertionPoint.at_block_terminator(m.body):
+    with InsertionPoint(m.body):
       f32 = F32Type.get()
       # Create via dialects context collection.
       input1 = createInput()
diff --git a/mlir/test/Bindings/Python/dialects/builtin.py b/mlir/test/Bindings/Python/dialects/builtin.py
index 447a255f60215c4aab4637a6dc93f7cb70c9a001..1f4847dce892c81a2f42eea3be3ebf0ec599e18d 100644
--- a/mlir/test/Bindings/Python/dialects/builtin.py
+++ b/mlir/test/Bindings/Python/dialects/builtin.py
@@ -8,9 +8,106 @@ import mlir.dialects.std as std
 def run(f):
   print("\nTEST:", f.__name__)
   f()
+  return f
+
+
+# CHECK-LABEL: TEST: testFromPyFunc
+@run
+def testFromPyFunc():
+  with Context() as ctx, Location.unknown() as loc:
+    m = builtin.ModuleOp()
+    f32 = F32Type.get()
+    f64 = F64Type.get()
+    with InsertionPoint(m.body):
+      # CHECK-LABEL: func @unary_return(%arg0: f64) -> f64
+      # CHECK: return %arg0 : f64
+      @builtin.FuncOp.from_py_func(f64)
+      def unary_return(a):
+        return a
+
+      # CHECK-LABEL: func @binary_return(%arg0: f32, %arg1: f64) -> (f32, f64)
+      # CHECK: return %arg0, %arg1 : f32, f64
+      @builtin.FuncOp.from_py_func(f32, f64)
+      def binary_return(a, b):
+        return a, b
+
+      # CHECK-LABEL: func @none_return(%arg0: f32, %arg1: f64)
+      # CHECK: return
+      @builtin.FuncOp.from_py_func(f32, f64)
+      def none_return(a, b):
+        pass
+
+      # CHECK-LABEL: func @call_unary
+      # CHECK: %0 = call @unary_return(%arg0) : (f64) -> f64
+      # CHECK: return %0 : f64
+      @builtin.FuncOp.from_py_func(f64)
+      def call_unary(a):
+        return unary_return(a)
+
+      # CHECK-LABEL: func @call_binary
+      # CHECK: %0:2 = call @binary_return(%arg0, %arg1) : (f32, f64) -> (f32, f64)
+      # CHECK: return %0#0, %0#1 : f32, f64
+      @builtin.FuncOp.from_py_func(f32, f64)
+      def call_binary(a, b):
+        return binary_return(a, b)
+
+      # CHECK-LABEL: func @call_none
+      # CHECK: call @none_return(%arg0, %arg1) : (f32, f64) -> ()
+      # CHECK: return
+      @builtin.FuncOp.from_py_func(f32, f64)
+      def call_none(a, b):
+        return none_return(a, b)
+
+      ## Variants and optional feature tests.
+      # CHECK-LABEL: func @from_name_arg
+      @builtin.FuncOp.from_py_func(f32, f64, name="from_name_arg")
+      def explicit_name(a, b):
+        return b
+
+      @builtin.FuncOp.from_py_func(f32, f64)
+      def positional_func_op(a, b, func_op):
+        assert isinstance(func_op, builtin.FuncOp)
+        return b
+
+      @builtin.FuncOp.from_py_func(f32, f64)
+      def kw_func_op(a, b=None, func_op=None):
+        assert isinstance(func_op, builtin.FuncOp)
+        return b
+
+      @builtin.FuncOp.from_py_func(f32, f64)
+      def kwargs_func_op(a, b=None, **kwargs):
+        assert isinstance(kwargs["func_op"], builtin.FuncOp)
+        return b
+
+      # CHECK-LABEL: func @explicit_results(%arg0: f32, %arg1: f64) -> f64
+      # CHECK: return %arg1 : f64
+      @builtin.FuncOp.from_py_func(f32, f64, results=[f64])
+      def explicit_results(a, b):
+        std.ReturnOp([b])
+
+  print(m)
+
+
+# CHECK-LABEL: TEST: testFromPyFuncErrors
+@run
+def testFromPyFuncErrors():
+  with Context() as ctx, Location.unknown() as loc:
+    m = builtin.ModuleOp()
+    f32 = F32Type.get()
+    f64 = F64Type.get()
+    with InsertionPoint(m.body):
+      try:
+
+        @builtin.FuncOp.from_py_func(f64, results=[f64])
+        def unary_return(a):
+          return a
+      except AssertionError as e:
+        # CHECK: Capturing a python function with explicit `results=` requires that the wrapped function returns None.
+        print(e)
 
 
 # CHECK-LABEL: TEST: testBuildFuncOp
+@run
 def testBuildFuncOp():
   ctx = Context()
   with Location.unknown(ctx) as loc:
@@ -64,6 +161,3 @@ def testBuildFuncOp():
   # CHECK:   return %arg0 : tensor<2x3x4xf32>
   # CHECK:  }
   print(m)
-
-
-run(testBuildFuncOp)
diff --git a/mlir/test/Bindings/Python/dialects/linalg/opdsl/emit_structured_generic.py b/mlir/test/Bindings/Python/dialects/linalg/opdsl/emit_structured_generic.py
new file mode 100644
index 0000000000000000000000000000000000000000..f27f79a4fb037ad24d92399228c77797b2707f34
--- /dev/null
+++ b/mlir/test/Bindings/Python/dialects/linalg/opdsl/emit_structured_generic.py
@@ -0,0 +1,146 @@
+# RUN: %PYTHON %s | FileCheck %s
+
+from typing import Optional, Sequence
+
+from mlir.ir import *
+from mlir.dialects import builtin
+from mlir.dialects import linalg
+from mlir.dialects import std
+
+from mlir.dialects.linalg.opdsl.lang import *
+
+
+@linalg_structured_op
+def matmul_mono(A=TensorDef(T, S.M, S.K),
+                B=TensorDef(T, S.K, S.N),
+                C=TensorDef(T, S.M, S.N, output=True)):
+  C[D.m, D.n] += A[D.m, D.k] * B[D.k, D.n]
+
+
+@linalg_structured_op
+def matmul_poly(A=TensorDef(TV.T1, S.M, S.K),
+                B=TensorDef(TV.T2, S.K, S.N),
+                C=TensorDef(U, S.M, S.N, output=True)):
+  C[D.m, D.n] += cast(U, A[D.m, D.k]) * cast(U, B[D.k, D.n])
+
+
+with Context() as ctx, Location.unknown():
+  module = Module.create()
+  f16 = F16Type.get()
+  f32 = F32Type.get()
+  f64 = F64Type.get()
+  i8 = IntegerType.get_signless(8)
+  i16 = IntegerType.get_signless(16)
+  i32 = IntegerType.get_signless(32)
+  with InsertionPoint(module.body):
+
+    # Note that these all have the same indexing maps. We verify the first and
+    # then do more permutation tests on casting and body generation
+    # behavior.
+    # CHECK: #[[$MAPA:.+]] = affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d2)>
+    # CHECK: #[[$MAPB:.+]] = affine_map<(d0, d1, d2)[s0, s1, s2] -> (d2, d1)>
+    # CHECK: #[[$MAPC:.+]] = affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d1)>
+
+    # CHECK-LABEL: func @test_matmul_mono
+    # CHECK-SAME:  %[[A:.+]]: tensor<4x16xf32>
+    # CHECK-SAME: %[[B:.+]]: tensor<16x8xf32>
+
+    # CHECK: %[[INITC:.+]] = linalg.init_tensor [4, 8] : tensor<4x8xf32>
+    # CHECK: linalg.generic
+    # CHECK-SAME: indexing_maps = [#[[$MAPA]], #[[$MAPB]], #[[$MAPC]]]
+    # CHECK-SAME: iterator_types = ["parallel", "parallel", "reduction"]
+    # CHECK-SAME: ins(%[[A]], %[[B]]
+    # CHECK-SAME: outs(%[[INITC]]
+
+    @builtin.FuncOp.from_py_func(RankedTensorType.get((4, 16), f32),
+                                 RankedTensorType.get((16, 8), f32))
+    def test_matmul_mono(lhs, rhs):
+      init_result = linalg.InitTensorOp([4, 8], f32)
+      return matmul_mono(lhs, rhs, outs=[init_result.result])
+
+    # CHECK-LABEL: @test_i8i8i32_matmul
+    # CHECK:      ^{{.*}}(%[[A_ARG:.+]]: i8, %[[B_ARG:.+]]: i8, %[[C_ARG:.+]]: i32)
+    # CHECK-NEXT:   %[[A_CAST:.+]] = sexti %[[A_ARG]] : i8 to i32
+    # CHECK-NEXT:   %[[B_CAST:.+]] = sexti %[[B_ARG]] : i8 to i32
+    # CHECK-NEXT:   %[[MUL:.+]] = muli %[[A_CAST]], %[[B_CAST]] : i32
+    # CHECK-NEXT:   %[[ADD:.+]] = addi %[[C_ARG]], %[[MUL]] : i32
+    # CHECK-NEXT:   linalg.yield %[[ADD]] : i32
+    # CHECK-NEXT: -> tensor<4x8xi32>
+    @builtin.FuncOp.from_py_func(RankedTensorType.get((4, 16), i8),
+                                 RankedTensorType.get((16, 8), i8),
+                                 RankedTensorType.get((4, 8), i32))
+    def test_i8i8i32_matmul(lhs, rhs, init_result):
+      return matmul_poly(lhs, rhs, outs=[init_result])
+
+    # CHECK-LABEL: @test_i8i16i32_matmul
+    # CHECK:      ^{{.*}}(%[[A_ARG:.+]]: i8, %[[B_ARG:.+]]: i16, %[[C_ARG:.+]]: i32)
+    # CHECK-NEXT:   %[[A_CAST:.+]] = sexti %[[A_ARG]] : i8 to i32
+    # CHECK-NEXT:   %[[B_CAST:.+]] = sexti %[[B_ARG]] : i16 to i32
+    # CHECK-NEXT:   %[[MUL:.+]] = muli %[[A_CAST]], %[[B_CAST]] : i32
+    # CHECK-NEXT:   %[[ADD:.+]] = addi %[[C_ARG]], %[[MUL]] : i32
+    # CHECK-NEXT:   linalg.yield %[[ADD]] : i32
+    # CHECK-NEXT: -> tensor<4x8xi32>
+    @builtin.FuncOp.from_py_func(RankedTensorType.get((4, 16), i8),
+                                 RankedTensorType.get((16, 8), i16),
+                                 RankedTensorType.get((4, 8), i32))
+    def test_i8i16i32_matmul(lhs, rhs, init_result):
+      return matmul_poly(lhs, rhs, outs=[init_result])
+
+    # CHECK-LABEL: @test_i32i32i16_matmul
+    # CHECK:      ^{{.*}}(%[[A_ARG:.+]]: i32, %[[B_ARG:.+]]: i32, %[[C_ARG:.+]]: i16)
+    # CHECK-NEXT:   %[[A_CAST:.+]] = trunci %[[A_ARG]] : i32 to i16
+    # CHECK-NEXT:   %[[B_CAST:.+]] = trunci %[[B_ARG]] : i32 to i16
+    # CHECK-NEXT:   %[[MUL:.+]] = muli %[[A_CAST]], %[[B_CAST]] : i16
+    # CHECK-NEXT:   %[[ADD:.+]] = addi %[[C_ARG]], %[[MUL]] : i16
+    # CHECK-NEXT:   linalg.yield %[[ADD]] : i16
+    # CHECK-NEXT: -> tensor<4x8xi16>
+    @builtin.FuncOp.from_py_func(RankedTensorType.get((4, 16), i32),
+                                 RankedTensorType.get((16, 8), i32),
+                                 RankedTensorType.get((4, 8), i16))
+    def test_i32i32i16_matmul(lhs, rhs, init_result):
+      return matmul_poly(lhs, rhs, outs=[init_result])
+
+    # CHECK-LABEL: @test_i8i8f32_matmul
+    # CHECK:      ^{{.*}}(%[[A_ARG:.+]]: i8, %[[B_ARG:.+]]: i8, %[[C_ARG:.+]]: f32)
+    # CHECK-NEXT:   %[[A_CAST:.+]] = sitofp %[[A_ARG]] : i8 to f32
+    # CHECK-NEXT:   %[[B_CAST:.+]] = sitofp %[[B_ARG]] : i8 to f32
+    # CHECK-NEXT:   %[[MUL:.+]] = mulf %[[A_CAST]], %[[B_CAST]] : f32
+    # CHECK-NEXT:   %[[ADD:.+]] = addf %[[C_ARG]], %[[MUL]] : f32
+    # CHECK-NEXT:   linalg.yield %[[ADD]] : f32
+    # CHECK-NEXT: -> tensor<4x8xf32>
+    @builtin.FuncOp.from_py_func(RankedTensorType.get((4, 16), i8),
+                                 RankedTensorType.get((16, 8), i8),
+                                 RankedTensorType.get((4, 8), f32))
+    def test_i8i8f32_matmul(lhs, rhs, init_result):
+      return matmul_poly(lhs, rhs, outs=[init_result])
+
+    # CHECK-LABEL: @test_f16f16f32_matmul
+    # CHECK:      ^{{.*}}(%[[A_ARG:.+]]: f16, %[[B_ARG:.+]]: f16, %[[C_ARG:.+]]: f32)
+    # CHECK-NEXT:   %[[A_CAST:.+]] = fpext %[[A_ARG]] : f16 to f32
+    # CHECK-NEXT:   %[[B_CAST:.+]] = fpext %[[B_ARG]] : f16 to f32
+    # CHECK-NEXT:   %[[MUL:.+]] = mulf %[[A_CAST]], %[[B_CAST]] : f32
+    # CHECK-NEXT:   %[[ADD:.+]] = addf %[[C_ARG]], %[[MUL]] : f32
+    # CHECK-NEXT:   linalg.yield %[[ADD]] : f32
+    # CHECK-NEXT: -> tensor<4x8xf32>
+    @builtin.FuncOp.from_py_func(RankedTensorType.get((4, 16), f16),
+                                 RankedTensorType.get((16, 8), f16),
+                                 RankedTensorType.get((4, 8), f32))
+    def test_f16f16f32_matmul(lhs, rhs, init_result):
+      return matmul_poly(lhs, rhs, outs=[init_result])
+
+    # CHECK-LABEL: @test_f64f64f32_matmul
+    # CHECK:      ^{{.*}}(%[[A_ARG:.+]]: f64, %[[B_ARG:.+]]: f64, %[[C_ARG:.+]]: f32)
+    # CHECK-NEXT:   %[[A_CAST:.+]] = fptrunc %[[A_ARG]] : f64 to f32
+    # CHECK-NEXT:   %[[B_CAST:.+]] = fptrunc %[[B_ARG]] : f64 to f32
+    # CHECK-NEXT:   %[[MUL:.+]] = mulf %[[A_CAST]], %[[B_CAST]] : f32
+    # CHECK-NEXT:   %[[ADD:.+]] = addf %[[C_ARG]], %[[MUL]] : f32
+    # CHECK-NEXT:   linalg.yield %[[ADD]] : f32
+    # CHECK-NEXT: -> tensor<4x8xf32>
+    @builtin.FuncOp.from_py_func(RankedTensorType.get((4, 16), f64),
+                                 RankedTensorType.get((16, 8), f64),
+                                 RankedTensorType.get((4, 8), f32))
+    def test_f64f64f32_matmul(lhs, rhs, init_result):
+      return matmul_poly(lhs, rhs, outs=[init_result])
+
+
+print(module)
diff --git a/mlir/test/Bindings/Python/dialects/linalg/ops.py b/mlir/test/Bindings/Python/dialects/linalg/ops.py
index 0615dd37bfdded62fdaacf247362301446b844e2..22ed09e0716ce952acab5bf7c94a1b784e8c71ac 100644
--- a/mlir/test/Bindings/Python/dialects/linalg/ops.py
+++ b/mlir/test/Bindings/Python/dialects/linalg/ops.py
@@ -9,15 +9,45 @@ from mlir.dialects import std
 def run(f):
   print("\nTEST:", f.__name__)
   f()
+  return f
+
+
+# CHECK-LABEL: TEST: testInitTensor
+@run
+def testInitTensor():
+  with Context() as ctx, Location.unknown():
+    module = Module.create()
+    f32 = F32Type.get()
+    with InsertionPoint(module.body):
+      # CHECK-LABEL: func @static_sizes
+      # CHECK: %0 = linalg.init_tensor [3, 4] : tensor<3x4xf32>
+      @builtin.FuncOp.from_py_func()
+      def static_sizes():
+        return linalg.InitTensorOp([3, 4], f32)
+
+      # CHECK-LABEL: func @dynamic_sizes
+      # CHECK: %0 = linalg.init_tensor [%arg0, %arg1] : tensor<?x?xf32>
+      @builtin.FuncOp.from_py_func(IndexType.get(), IndexType.get())
+      def dynamic_sizes(d0, d1):
+        return linalg.InitTensorOp([d0, d1], f32)
+
+      # CHECK-LABEL: func @zero_d
+      # CHECK: %0 = linalg.init_tensor [] : tensor<f32>
+      @builtin.FuncOp.from_py_func()
+      def zero_d():
+        return linalg.InitTensorOp([], f32)
+
+  print(module)
 
 
 # CHECK-LABEL: TEST: testStructuredOpOnTensors
+@run
 def testStructuredOpOnTensors():
   with Context() as ctx, Location.unknown():
     module = Module.create()
     f32 = F32Type.get()
     tensor_type = RankedTensorType.get((2, 3, 4), f32)
-    with InsertionPoint.at_block_terminator(module.body):
+    with InsertionPoint(module.body):
       func = builtin.FuncOp(name="matmul_test",
                             type=FunctionType.get(
                                 inputs=[tensor_type, tensor_type],
@@ -31,16 +61,14 @@ def testStructuredOpOnTensors():
   print(module)
 
 
-run(testStructuredOpOnTensors)
-
-
 # CHECK-LABEL: TEST: testStructuredOpOnBuffers
+@run
 def testStructuredOpOnBuffers():
   with Context() as ctx, Location.unknown():
     module = Module.create()
     f32 = F32Type.get()
     memref_type = MemRefType.get((2, 3, 4), f32)
-    with InsertionPoint.at_block_terminator(module.body):
+    with InsertionPoint(module.body):
       func = builtin.FuncOp(name="matmul_test",
                             type=FunctionType.get(
                                 inputs=[memref_type, memref_type, memref_type],
@@ -52,6 +80,3 @@ def testStructuredOpOnBuffers():
 
   # CHECK: linalg.matmul ins(%arg0, %arg1 : memref<2x3x4xf32>, memref<2x3x4xf32>) outs(%arg2 : memref<2x3x4xf32>)
   print(module)
-
-
-run(testStructuredOpOnBuffers)
diff --git a/mlir/test/Bindings/Python/insertion_point.py b/mlir/test/Bindings/Python/insertion_point.py
index 1a2de37428cc2b86d30d93736b4fd31a064a768c..2e53aa64b999f8989e9f9b793d35e238a7b4ec51 100644
--- a/mlir/test/Bindings/Python/insertion_point.py
+++ b/mlir/test/Bindings/Python/insertion_point.py
@@ -129,8 +129,13 @@ run(test_insert_at_block_terminator_missing)
 def test_insert_at_end_with_terminator_errors():
   with Context() as ctx, Location.unknown():
     ctx.allow_unregistered_dialects = True
-    m = Module.create()  # Module is created with a terminator.
-    with InsertionPoint(m.body):
+    module = Module.parse(r"""
+      func @foo() -> () {
+        return
+      }
+    """)
+    entry_block = module.body.operations[0].regions[0].blocks[0]
+    with InsertionPoint(entry_block):
       try:
         Operation.create("custom.op1", results=[], operands=[])
       except IndexError as e:
diff --git a/mlir/test/Bindings/Python/ir_operation.py b/mlir/test/Bindings/Python/ir_operation.py
index a71194b354cbead6e37ab5eb34532993ad69962c..847c1093cd37fcd8d8ad981b2accf7089e49c42d 100644
--- a/mlir/test/Bindings/Python/ir_operation.py
+++ b/mlir/test/Bindings/Python/ir_operation.py
@@ -64,7 +64,6 @@ def testTraverseOpRegionBlockIterators():
   # CHECK:         BLOCK 0:
   # CHECK:           OP 0: %0 = "custom.addi"
   # CHECK:           OP 1: return
-  # CHECK:    OP 1: module_terminator
   walk_operations("", op)
 
 run(testTraverseOpRegionBlockIterators)
@@ -101,7 +100,6 @@ def testTraverseOpRegionBlockIndices():
   # CHECK:         BLOCK 0:
   # CHECK:           OP 0: %0 = "custom.addi"
   # CHECK:           OP 1: return
-  # CHECK:    OP 1: module_terminator
   walk_operations("", module.operation)
 
 run(testTraverseOpRegionBlockIndices)
@@ -470,7 +468,7 @@ def testOperationPrint():
   print(bytes_value)
 
   # Test get_asm with options.
-  # CHECK: value = opaque<"", "0xDEADBEEF"> : tensor<4xi32>
+  # CHECK: value = opaque<"_", "0xDEADBEEF"> : tensor<4xi32>
   # CHECK: "std.return"(%arg0) : (i32) -> () -:4:7
   module.operation.print(large_elements_limit=2, enable_debug_info=True,
       pretty_debug_info=True, print_generic_op_form=True, use_local_scope=True)
@@ -546,9 +544,9 @@ run(testSingleResultProperty)
 def testPrintInvalidOperation():
   ctx = Context()
   with Location.unknown(ctx):
-    module = Operation.create("module", regions=1)
-    # This block does not have a terminator, it may crash the custom printer.
-    # Verify that we fallback to the generic printer for safety.
+    module = Operation.create("module", regions=2)
+    # This module has two region and is invalid verify that we fallback
+    # to the generic printer for safety.
     block = module.regions[0].blocks.append()
     # CHECK: // Verification failed, printing generic form
     # CHECK: "module"() ( {
diff --git a/mlir/test/Bindings/Python/ir_types.py b/mlir/test/Bindings/Python/ir_types.py
index 59b4b50b533d89977e0269cd59ff1a6b6d3e4c23..ea05c1561f74bb67227a629c154c0e48d1227ebd 100644
--- a/mlir/test/Bindings/Python/ir_types.py
+++ b/mlir/test/Bindings/Python/ir_types.py
@@ -59,6 +59,21 @@ def testTypeEq():
 run(testTypeEq)
 
 
+# CHECK-LABEL: TEST: testTypeIsInstance
+def testTypeIsInstance():
+  ctx = Context()
+  t1 = Type.parse("i32", ctx)
+  t2 = Type.parse("f32", ctx)
+  # CHECK: True
+  print(IntegerType.isinstance(t1))
+  # CHECK: False
+  print(F32Type.isinstance(t1))
+  # CHECK: True
+  print(F32Type.isinstance(t2))
+
+run(testTypeIsInstance)
+
+
 # CHECK-LABEL: TEST: testTypeEqDoesNotRaise
 def testTypeEqDoesNotRaise():
   ctx = Context()
diff --git a/mlir/test/Bindings/Python/ods_helpers.py b/mlir/test/Bindings/Python/ods_helpers.py
index 54f68a82fc01e862e13acc277a11de9cf684655f..5aa9bef22a6ba74bc1ea27488a28d75eae3b10fe 100644
--- a/mlir/test/Bindings/Python/ods_helpers.py
+++ b/mlir/test/Bindings/Python/ods_helpers.py
@@ -29,7 +29,7 @@ def testOdsBuildDefaultImplicitRegions():
   with Context() as ctx, Location.unknown():
     ctx.allow_unregistered_dialects = True
     m = Module.create()
-    with InsertionPoint.at_block_terminator(m.body):
+    with InsertionPoint(m.body):
       op = TestFixedRegionsOp.build_generic(results=[], operands=[])
       # CHECK: NUM_REGIONS: 2
       print(f"NUM_REGIONS: {len(op.regions)}")
@@ -84,7 +84,7 @@ def testOdsBuildDefaultNonVariadic():
   with Context() as ctx, Location.unknown():
     ctx.allow_unregistered_dialects = True
     m = Module.create()
-    with InsertionPoint.at_block_terminator(m.body):
+    with InsertionPoint(m.body):
       v0 = add_dummy_value()
       v1 = add_dummy_value()
       t0 = IntegerType.get_signless(8)
@@ -111,7 +111,7 @@ def testOdsBuildDefaultSizedVariadic():
   with Context() as ctx, Location.unknown():
     ctx.allow_unregistered_dialects = True
     m = Module.create()
-    with InsertionPoint.at_block_terminator(m.body):
+    with InsertionPoint(m.body):
       v0 = add_dummy_value()
       v1 = add_dummy_value()
       v2 = add_dummy_value()
@@ -125,8 +125,8 @@ def testOdsBuildDefaultSizedVariadic():
       # CHECK: %[[V2:.+]] = "custom.value"
       # CHECK: %[[V3:.+]] = "custom.value"
       # CHECK: "custom.test_op"(%[[V0]], %[[V1]], %[[V2]], %[[V3]])
-      # CHECK-SAME: operand_segment_sizes = dense<[1, 2, 1]> : vector<3xi64>
-      # CHECK-SAME: result_segment_sizes = dense<[2, 1, 1]> : vector<3xi64>
+      # CHECK-SAME: operand_segment_sizes = dense<[1, 2, 1]> : vector<3xi32>
+      # CHECK-SAME: result_segment_sizes = dense<[2, 1, 1]> : vector<3xi32>
       # CHECK-SAME: : (i32, i32, i32, i32) -> (i8, i16, i32, i64)
       op = TestOp.build_generic(
           results=[[t0, t1], t2, t3],
@@ -187,7 +187,7 @@ def testOdsBuildDefaultCastError():
   with Context() as ctx, Location.unknown():
     ctx.allow_unregistered_dialects = True
     m = Module.create()
-    with InsertionPoint.at_block_terminator(m.body):
+    with InsertionPoint(m.body):
       v0 = add_dummy_value()
       v1 = add_dummy_value()
       t0 = IntegerType.get_signless(8)
diff --git a/mlir/test/Bindings/Python/pass_manager.py b/mlir/test/Bindings/Python/pass_manager.py
index 61ff64f67be65ab56b80bf8e3b05fae15ff56e00..35e6d980c9f39d591361b8ba1e53d50eb6b1d7c5 100644
--- a/mlir/test/Bindings/Python/pass_manager.py
+++ b/mlir/test/Bindings/Python/pass_manager.py
@@ -91,6 +91,5 @@ def testRunPipeline():
 # CHECK: Operations encountered:
 # CHECK: func              , 1
 # CHECK: module            , 1
-# CHECK: module_terminator , 1
 # CHECK: std.return        , 1
 run(testRunPipeline)
diff --git a/mlir/test/CAPI/ir.c b/mlir/test/CAPI/ir.c
index b72a6e6cf2fdcd64146ed47ef028d1582a4e75e9..40ef39b19d26607f8ac7184035c5864425a52e92 100644
--- a/mlir/test/CAPI/ir.c
+++ b/mlir/test/CAPI/ir.c
@@ -293,7 +293,7 @@ int collectStats(MlirOperation operation) {
   fprintf(stderr, "Number of op results: %u\n", stats.numOpResults);
   // clang-format off
   // CHECK-LABEL: @stats
-  // CHECK: Number of operations: 13
+  // CHECK: Number of operations: 12
   // CHECK: Number of attributes: 4
   // CHECK: Number of blocks: 3
   // CHECK: Number of regions: 3
@@ -449,7 +449,7 @@ static void printFirstOfEach(MlirContext ctx, MlirOperation operation) {
   mlirOperationPrintWithFlags(operation, flags, printToStderr, NULL);
   fprintf(stderr, "\n");
   // clang-format off
-  // CHECK: Op print with all flags: %{{.*}} = "std.constant"() {elts = opaque<"", "0xDEADBEEF"> : tensor<4xi32>, value = 0 : index} : () -> index loc(unknown)
+  // CHECK: Op print with all flags: %{{.*}} = "std.constant"() {elts = opaque<"_", "0xDEADBEEF"> : tensor<4xi32>, value = 0 : index} : () -> index loc(unknown)
   // clang-format on
 
   mlirOpPrintingFlagsDestroy(flags);
diff --git a/mlir/test/CAPI/pass.c b/mlir/test/CAPI/pass.c
index b7b9e373feb24f1e167d25b06b9f42eecdfc3482..d73aba1c7379c3fda49c610629a1b45a24f9d691 100644
--- a/mlir/test/CAPI/pass.c
+++ b/mlir/test/CAPI/pass.c
@@ -42,7 +42,6 @@ void testRunPassOnModule() {
   // Run the print-op-stats pass on the top-level module:
   // CHECK-LABEL: Operations encountered:
   // CHECK: func              , 1
-  // CHECK: module_terminator , 1
   // CHECK: std.addi          , 1
   // CHECK: std.return        , 1
   {
@@ -84,7 +83,6 @@ void testRunPassOnNestedModule() {
 
   // Run the print-op-stats pass on functions under the top-level module:
   // CHECK-LABEL: Operations encountered:
-  // CHECK-NOT: module_terminator
   // CHECK: func              , 1
   // CHECK: std.addi          , 1
   // CHECK: std.return        , 1
@@ -101,7 +99,6 @@ void testRunPassOnNestedModule() {
   }
   // Run the print-op-stats pass on functions under the nested module:
   // CHECK-LABEL: Operations encountered:
-  // CHECK-NOT: module_terminator
   // CHECK: func              , 1
   // CHECK: std.addf          , 1
   // CHECK: std.return        , 1
diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt
index 69d123d02047ff3805f32f2b59497b50eda213f3..775a462db53d9d9787143afef0c8da80b67d373d 100644
--- a/mlir/test/CMakeLists.txt
+++ b/mlir/test/CMakeLists.txt
@@ -21,8 +21,7 @@ set(MLIR_DIALECT_LINALG_INTEGRATION_TEST_LIB_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTOR
 set(MLIR_RUNNER_UTILS_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 
 # Passed to lit.site.cfg.py.in to set up the path where to find the libraries
-# for the mlir rocm / spirv / vulkan runner tests.
-set(MLIR_ROCM_WRAPPER_LIBRARY_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
+# for the mlir spirv / vulkan runner tests.
 set(MLIR_SPIRV_WRAPPER_LIBRARY_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 set(MLIR_VULKAN_WRAPPER_LIBRARY_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 
@@ -75,6 +74,10 @@ if(MLIR_CUDA_RUNNER_ENABLED)
   list(APPEND MLIR_TEST_DEPENDS mlir_cuda_runtime)
 endif()
 
+if(MLIR_ROCM_RUNNER_ENABLED)
+  list(APPEND MLIR_TEST_DEPENDS mlir_rocm_runtime)
+endif()
+
 list(APPEND MLIR_TEST_DEPENDS MLIRUnitTests)
 
 if(LLVM_BUILD_EXAMPLES)
@@ -89,12 +92,6 @@ if(LLVM_BUILD_EXAMPLES)
     )
 endif()
 
-if(MLIR_ROCM_RUNNER_ENABLED)
-  list(APPEND MLIR_TEST_DEPENDS
-    mlir-rocm-runner
-  )
-endif()
-
 if(MLIR_SPIRV_CPU_RUNNER_ENABLED)
   add_subdirectory(mlir-spirv-cpu-runner)
   list(APPEND MLIR_TEST_DEPENDS
diff --git a/mlir/test/Conversion/AffineToStandard/lower-affine.mlir b/mlir/test/Conversion/AffineToStandard/lower-affine.mlir
index 74c54ebe45b0f59186c84ddacacb7dbec77fef36..4e358ae70134cf65b5bab6847675893e9b98c83d 100644
--- a/mlir/test/Conversion/AffineToStandard/lower-affine.mlir
+++ b/mlir/test/Conversion/AffineToStandard/lower-affine.mlir
@@ -239,6 +239,33 @@ func @nested_ifs() {
   return
 }
 
+// CHECK-LABEL: func @if_with_yield
+// CHECK-NEXT:   %[[c0_i64:.*]] = constant 0 : i64
+// CHECK-NEXT:   %[[c1_i64:.*]] = constant 1 : i64
+// CHECK-NEXT:   %[[v0:.*]] = call @get_idx() : () -> index
+// CHECK-NEXT:   %[[c0:.*]] = constant 0 : index
+// CHECK-NEXT:   %[[cm10:.*]] = constant -10 : index
+// CHECK-NEXT:   %[[v1:.*]] = addi %[[v0]], %[[cm10]] : index
+// CHECK-NEXT:   %[[v2:.*]] = cmpi sge, %[[v1]], %[[c0]] : index
+// CHECK-NEXT:   %[[v3:.*]] = scf.if %[[v2]] -> (i64) {
+// CHECK-NEXT:     scf.yield %[[c0_i64]] : i64
+// CHECK-NEXT:   } else {
+// CHECK-NEXT:     scf.yield %[[c1_i64]] : i64
+// CHECK-NEXT:   }
+// CHECK-NEXT:   return %[[v3]] : i64
+// CHECK-NEXT: }
+func @if_with_yield() -> (i64) {
+  %cst0 = constant 0 : i64
+  %cst1 = constant 1 : i64
+  %i = call @get_idx() : () -> (index)
+  %1 = affine.if #set2(%i) -> (i64) {
+      affine.yield %cst0 : i64
+  } else {
+      affine.yield %cst1 : i64
+  }
+  return %1 : i64
+}
+
 #setN = affine_set<(d0)[N,M,K,L] : (N - d0 + 1 >= 0, N - 1 >= 0, M - 1 >= 0, K - 1 >= 0, L - 42 == 0)>
 
 // CHECK-LABEL: func @multi_cond
diff --git a/mlir/test/Conversion/GPUToROCm/lower-rocdl-kernel-to-hsaco.mlir b/mlir/test/Conversion/GPUToROCm/lower-rocdl-kernel-to-hsaco.mlir
index 3d7deb906e77a52aa78b2bc0ba062b77f8586c64..fb19ac6491b35a989a285453fef3409249287950 100644
--- a/mlir/test/Conversion/GPUToROCm/lower-rocdl-kernel-to-hsaco.mlir
+++ b/mlir/test/Conversion/GPUToROCm/lower-rocdl-kernel-to-hsaco.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s --test-kernel-to-hsaco -split-input-file | FileCheck %s
+// RUN: mlir-opt %s --test-gpu-to-hsaco | FileCheck %s
 
-// CHECK: attributes {rocdl.hsaco = "HSACO"}
+// CHECK: gpu.module @foo attributes {gpu.binary = "HSACO"}
 gpu.module @foo {
   llvm.func @kernel(%arg0 : f32, %arg1 : !llvm.ptr<f32>)
     // CHECK: attributes  {gpu.kernel}
@@ -9,8 +9,7 @@ gpu.module @foo {
   }
 }
 
-// -----
-
+// CHECK: gpu.module @bar attributes {gpu.binary = "HSACO"}
 gpu.module @bar {
   // CHECK: func @kernel_a
   llvm.func @kernel_a()
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index c1fc82e51c50ad127fa48e6ef3653b849832c2b6..e0bb0134a14af0936c9b7966f6f6f26e2942c661 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -34,10 +34,8 @@ func @branch_loop() {
 func @wsloop(%arg0: index, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index) {
   // CHECK: omp.parallel
   omp.parallel {
-    // CHECK: omp.wsloop
-    // CHECK: (%[[ARG0]], %[[ARG1]], %[[ARG2]], %[[ARG3]], %[[ARG4]], %[[ARG5]])
+    // CHECK: omp.wsloop (%[[ARG6:.*]], %[[ARG7:.*]]) : i64 = (%[[ARG0]], %[[ARG1]]) to (%[[ARG2]], %[[ARG3]]) step (%[[ARG4]], %[[ARG5]]) {
     "omp.wsloop"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) ( {
-    // CHECK: ^{{.*}}(%[[ARG6:.*]]: i64, %[[ARG7:.*]]: i64):
     ^bb0(%arg6: index, %arg7: index):  // no predecessors
       // CHECK: "test.payload"(%[[ARG6]], %[[ARG7]]) : (i64, i64) -> ()
       "test.payload"(%arg6, %arg7) : (index, index) -> ()
diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir
index 9d87ba5a21f06f2a0331169a1868f6c2c91941b3..0af77a24efb4949436ef5fdfa425551485ec4017 100644
--- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir
+++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir
@@ -24,10 +24,10 @@ module @simple {
 
   // CHECK: module @rewriters
   // CHECK:   func @pdl_generated_rewriter(%[[REWRITE_ROOT:.*]]: !pdl.operation)
-  // CHECK:     pdl_interp.apply_rewrite "rewriter" on %[[REWRITE_ROOT]]
+  // CHECK:     pdl_interp.apply_rewrite "rewriter"(%[[REWRITE_ROOT]]
   // CHECK:     pdl_interp.finalize
   pdl.pattern : benefit(1) {
-    %root = pdl.operation "foo.op"()
+    %root = pdl.operation "foo.op"
     pdl.rewrite %root with "rewriter"
   }
 }
@@ -63,15 +63,16 @@ module @constraints {
   // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
   // CHECK-DAG:   %[[INPUT:.*]] = pdl_interp.get_operand 0 of %[[ROOT]]
   // CHECK-DAG:   %[[INPUT1:.*]] = pdl_interp.get_operand 1 of %[[ROOT]]
-  // CHECK:       pdl_interp.apply_constraint "multi_constraint" [true](%[[INPUT]], %[[INPUT1]] : !pdl.value, !pdl.value)
+  // CHECK-DAG:   %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[ROOT]]
+  // CHECK:       pdl_interp.apply_constraint "multi_constraint" [true](%[[INPUT]], %[[INPUT1]], %[[RESULT]]
 
   pdl.pattern : benefit(1) {
     %input0 = pdl.operand
     %input1 = pdl.operand
+    %root = pdl.operation(%input0, %input1 : !pdl.value, !pdl.value)
+    %result0 = pdl.result 0 of %root
 
-    pdl.apply_constraint "multi_constraint"[true](%input0, %input1 : !pdl.value, !pdl.value)
-
-    %root = pdl.operation(%input0, %input1)
+    pdl.apply_native_constraint "multi_constraint"[true](%input0, %input1, %result0 : !pdl.value, !pdl.value, !pdl.value)
     pdl.rewrite %root with "rewriter"
   }
 }
@@ -95,7 +96,60 @@ module @inputs {
   pdl.pattern : benefit(1) {
     %type = pdl.type : i64
     %input = pdl.operand : %type
-    %root = pdl.operation(%input, %input)
+    %root = pdl.operation(%input, %input : !pdl.value, !pdl.value)
+    pdl.rewrite %root with "rewriter"
+  }
+}
+
+// -----
+
+// CHECK-LABEL: module @variadic_inputs
+module @variadic_inputs {
+  // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
+  // CHECK-DAG: pdl_interp.check_operand_count of %[[ROOT]] is at_least 2
+
+  // The first operand has a known index.
+  // CHECK-DAG:   %[[INPUT:.*]] = pdl_interp.get_operand 0 of %[[ROOT]]
+  // CHECK-DAG:   pdl_interp.is_not_null %[[INPUT]] : !pdl.value
+
+  // The second operand is a group of unknown size, with a type constraint.
+  // CHECK-DAG:   %[[VAR_INPUTS:.*]] = pdl_interp.get_operands 1 of %[[ROOT]] : !pdl.range<value>
+  // CHECK-DAG:   pdl_interp.is_not_null %[[VAR_INPUTS]] : !pdl.range<value>
+
+  // CHECK-DAG:   %[[INPUT_TYPE:.*]] = pdl_interp.get_value_type of %[[VAR_INPUTS]] : !pdl.range<type>
+  // CHECK-DAG:   pdl_interp.check_types %[[INPUT_TYPE]] are [i64]
+
+  // The third operand is at an unknown offset due to operand 2, but is expected
+  // to be of size 1.
+  // CHECK-DAG:  %[[INPUT2:.*]] = pdl_interp.get_operands 2 of %[[ROOT]] : !pdl.value
+  // CHECK-DAG:  pdl_interp.are_equal %[[INPUT]], %[[INPUT2]] : !pdl.value
+  pdl.pattern : benefit(1) {
+    %types = pdl.types : [i64]
+    %inputs = pdl.operands : %types
+    %input = pdl.operand
+    %root = pdl.operation(%input, %inputs, %input : !pdl.value, !pdl.range<value>, !pdl.value)
+    pdl.rewrite %root with "rewriter"
+  }
+}
+
+// -----
+
+// CHECK-LABEL: module @single_operand_range
+module @single_operand_range {
+  // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
+
+  // Check that the operand range is treated as all of the operands of the
+  // operation.
+  // CHECK-DAG:   %[[RESULTS:.*]] = pdl_interp.get_operands of %[[ROOT]]
+  // CHECK-DAG:   %[[RESULT_TYPES:.*]] = pdl_interp.get_value_type of %[[RESULTS]] : !pdl.range<type>
+  // CHECK-DAG:   pdl_interp.check_types %[[RESULT_TYPES]] are [i64]
+
+  // The operand count is unknown, so there is no need to check for it.
+  // CHECK-NOT: pdl_interp.check_operand_count
+  pdl.pattern : benefit(1) {
+    %types = pdl.types : [i64]
+    %operands = pdl.operands : %types
+    %root = pdl.operation(%operands : !pdl.range<value>)
     pdl.rewrite %root with "rewriter"
   }
 }
@@ -107,43 +161,226 @@ module @results {
   // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
   // CHECK:   pdl_interp.check_result_count of %[[ROOT]] is 2
 
-  // Get the input and check the type.
+  // Get the result and check the type.
   // CHECK-DAG:   %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[ROOT]]
   // CHECK-DAG:   pdl_interp.is_not_null %[[RESULT]] : !pdl.value
   // CHECK-DAG:   %[[RESULT_TYPE:.*]] = pdl_interp.get_value_type of %[[RESULT]]
   // CHECK-DAG:   pdl_interp.check_type %[[RESULT_TYPE]] is i32
 
-  // Get the second operand and check that it is equal to the first.
-  // CHECK-DAG:  %[[RESULT1:.*]] = pdl_interp.get_result 1 of %[[ROOT]]
-  // CHECK-NOT: pdl_interp.get_value_type of %[[RESULT1]]
+  // The second result doesn't have any constraints, so we don't generate an
+  // access for it.
+  // CHECK-NOT:   pdl_interp.get_result 1 of %[[ROOT]]
   pdl.pattern : benefit(1) {
     %type1 = pdl.type : i32
     %type2 = pdl.type
-    %root, %results:2 = pdl.operation -> %type1, %type2
+    %root = pdl.operation -> (%type1, %type2 : !pdl.type, !pdl.type)
     pdl.rewrite %root with "rewriter"
   }
 }
 
 // -----
 
-// CHECK-LABEL: module @switch_result_types
-module @switch_result_types {
+// CHECK-LABEL: module @variadic_results
+module @variadic_results {
+  // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
+  // CHECK-DAG: pdl_interp.check_result_count of %[[ROOT]] is at_least 2
+
+  // The first result has a known index.
+  // CHECK-DAG:   %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[ROOT]]
+  // CHECK-DAG:   pdl_interp.is_not_null %[[RESULT]] : !pdl.value
+
+  // The second result is a group of unknown size, with a type constraint.
+  // CHECK-DAG:   %[[VAR_RESULTS:.*]] = pdl_interp.get_results 1 of %[[ROOT]] : !pdl.range<value>
+  // CHECK-DAG:   pdl_interp.is_not_null %[[VAR_RESULTS]] : !pdl.range<value>
+
+  // CHECK-DAG:   %[[RESULT_TYPE:.*]] = pdl_interp.get_value_type of %[[VAR_RESULTS]] : !pdl.range<type>
+  // CHECK-DAG:   pdl_interp.check_types %[[RESULT_TYPE]] are [i64]
+
+  // The third result is at an unknown offset due to result 1, but is expected
+  // to be of size 1.
+  // CHECK-DAG:  %[[RESULT2:.*]] = pdl_interp.get_results 2 of %[[ROOT]] : !pdl.value
+  // CHECK-DAG:   pdl_interp.is_not_null %[[RESULT2]] : !pdl.value
+  pdl.pattern : benefit(1) {
+    %types = pdl.types : [i64]
+    %type = pdl.type
+    %root = pdl.operation -> (%type, %types, %type : !pdl.type, !pdl.range<type>, !pdl.type)
+    pdl.rewrite %root with "rewriter"
+  }
+}
+
+// -----
+
+// CHECK-LABEL: module @single_result_range
+module @single_result_range {
+  // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
+
+  // Check that the result range is treated as all of the results of the
+  // operation.
+  // CHECK-DAG:   %[[RESULTS:.*]] = pdl_interp.get_results of %[[ROOT]]
+  // CHECK-DAG:   %[[RESULT_TYPES:.*]] = pdl_interp.get_value_type of %[[RESULTS]] : !pdl.range<type>
+  // CHECK-DAG:   pdl_interp.check_types %[[RESULT_TYPES]] are [i64]
+
+  // The result count is unknown, so there is no need to check for it.
+  // CHECK-NOT: pdl_interp.check_result_count
+  pdl.pattern : benefit(1) {
+    %types = pdl.types : [i64]
+    %root = pdl.operation -> (%types : !pdl.range<type>)
+    pdl.rewrite %root with "rewriter"
+  }
+}
+
+// -----
+
+// CHECK-LABEL: module @results_as_operands
+module @results_as_operands {
+  // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
+
+  // Get the first result and check it matches the first operand.
+  // CHECK-DAG:   %[[OPERAND_0:.*]] = pdl_interp.get_operand 0 of %[[ROOT]]
+  // CHECK-DAG:   %[[DEF_OP_0:.*]] = pdl_interp.get_defining_op of %[[OPERAND_0]]
+  // CHECK-DAG:   %[[RESULT_0:.*]] = pdl_interp.get_result 0 of %[[DEF_OP_0]]
+  // CHECK-DAG:   pdl_interp.are_equal %[[RESULT_0]], %[[OPERAND_0]]
+
+  // Get the second result and check it matches the second operand.
+  // CHECK-DAG:   %[[OPERAND_1:.*]] = pdl_interp.get_operand 1 of %[[ROOT]]
+  // CHECK-DAG:   %[[DEF_OP_1:.*]] = pdl_interp.get_defining_op of %[[OPERAND_1]]
+  // CHECK-DAG:   %[[RESULT_1:.*]] = pdl_interp.get_result 1 of %[[DEF_OP_1]]
+  // CHECK-DAG:   pdl_interp.are_equal %[[RESULT_1]], %[[OPERAND_1]]
+
+  // Check that the parent operation of both results is the same.
+  // CHECK-DAG:   pdl_interp.are_equal %[[DEF_OP_0]], %[[DEF_OP_1]]
+
+  pdl.pattern : benefit(1) {
+    %type1 = pdl.type : i32
+    %type2 = pdl.type
+    %inputOp = pdl.operation -> (%type1, %type2 : !pdl.type, !pdl.type)
+    %result1 = pdl.result 0 of %inputOp
+    %result2 = pdl.result 1 of %inputOp
+
+    %root = pdl.operation(%result1, %result2 : !pdl.value, !pdl.value)
+    pdl.rewrite %root with "rewriter"
+  }
+}
+
+// -----
+
+// CHECK-LABEL: module @single_result_range_as_operands
+module @single_result_range_as_operands {
+  // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
+  // CHECK-DAG:  %[[OPERANDS:.*]] = pdl_interp.get_operands of %[[ROOT]] : !pdl.range<value>
+  // CHECK-DAG:  %[[OP:.*]] = pdl_interp.get_defining_op of %[[OPERANDS]] : !pdl.range<value>
+  // CHECK-DAG:  pdl_interp.is_not_null %[[OP]]
+  // CHECK-DAG:  %[[RESULTS:.*]] = pdl_interp.get_results of %[[OP]] : !pdl.range<value>
+  // CHECK-DAG:  pdl_interp.are_equal %[[RESULTS]], %[[OPERANDS]] : !pdl.range<value>
+
+  pdl.pattern : benefit(1) {
+    %types = pdl.types
+    %inputOp = pdl.operation -> (%types : !pdl.range<type>)
+    %results = pdl.results of %inputOp
+
+    %root = pdl.operation(%results : !pdl.range<value>)
+    pdl.rewrite %root with "rewriter"
+  }
+}
+
+// -----
+
+// CHECK-LABEL: module @switch_single_result_type
+module @switch_single_result_type {
   // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
   // CHECK:   %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[ROOT]]
   // CHECK:   %[[RESULT_TYPE:.*]] = pdl_interp.get_value_type of %[[RESULT]]
   // CHECK:   pdl_interp.switch_type %[[RESULT_TYPE]] to [i32, i64]
   pdl.pattern : benefit(1) {
     %type = pdl.type : i32
-    %root, %result = pdl.operation -> %type
+    %root = pdl.operation -> (%type : !pdl.type)
     pdl.rewrite %root with "rewriter"
   }
   pdl.pattern : benefit(1) {
     %type = pdl.type : i64
-    %root, %result = pdl.operation -> %type
+    %root = pdl.operation -> (%type : !pdl.type)
     pdl.rewrite %root with "rewriter"
   }
 }
 
+// -----
+
+// CHECK-LABEL: module @switch_result_types
+module @switch_result_types {
+  // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
+  // CHECK:   %[[RESULTS:.*]] = pdl_interp.get_results of %[[ROOT]]
+  // CHECK:   %[[RESULT_TYPES:.*]] = pdl_interp.get_value_type of %[[RESULTS]]
+  // CHECK:   pdl_interp.switch_types %[[RESULT_TYPES]] to {{\[\[}}i32], [i64, i32]]
+  pdl.pattern : benefit(1) {
+    %types = pdl.types : [i32]
+    %root = pdl.operation -> (%types : !pdl.range<type>)
+    pdl.rewrite %root with "rewriter"
+  }
+  pdl.pattern : benefit(1) {
+    %types = pdl.types : [i64, i32]
+    %root = pdl.operation -> (%types : !pdl.range<type>)
+    pdl.rewrite %root with "rewriter"
+  }
+}
+
+// -----
+
+// CHECK-LABEL: module @switch_operand_count_at_least
+module @switch_operand_count_at_least {
+  // Check that when there are multiple "at_least" checks, the failure branch
+  // goes to the next one in increasing order.
+
+  // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
+  // CHECK: pdl_interp.check_operand_count of %[[ROOT]] is at_least 1 -> ^[[PATTERN_1_NEXT_BLOCK:.*]],
+  // CHECK: ^bb2:
+  // CHECK-NEXT: pdl_interp.check_operand_count of %[[ROOT]] is at_least 2
+  // CHECK: ^[[PATTERN_1_NEXT_BLOCK]]:
+  // CHECK-NEXT: {{.*}} -> ^{{.*}}, ^bb2
+  pdl.pattern : benefit(1) {
+    %operand = pdl.operand
+    %operands = pdl.operands
+    %root = pdl.operation(%operand, %operands : !pdl.value, !pdl.range<value>)
+    pdl.rewrite %root with "rewriter"
+  }
+  pdl.pattern : benefit(1) {
+    %operand = pdl.operand
+    %operand2 = pdl.operand
+    %operands = pdl.operands
+    %root = pdl.operation(%operand, %operand2, %operands : !pdl.value, !pdl.value, !pdl.range<value>)
+    pdl.rewrite %root with "rewriter"
+  }
+}
+
+// -----
+
+// CHECK-LABEL: module @switch_result_count_at_least
+module @switch_result_count_at_least {
+  // Check that when there are multiple "at_least" checks, the failure branch
+  // goes to the next one in increasing order.
+
+  // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation)
+  // CHECK: pdl_interp.check_result_count of %[[ROOT]] is at_least 1 -> ^[[PATTERN_1_NEXT_BLOCK:.*]],
+  // CHECK: ^[[PATTERN_2_BLOCK:[a-zA-Z_0-9]*]]:
+  // CHECK: pdl_interp.check_result_count of %[[ROOT]] is at_least 2
+  // CHECK: ^[[PATTERN_1_NEXT_BLOCK]]:
+  // CHECK-NEXT: pdl_interp.get_result
+  // CHECK-NEXT: pdl_interp.is_not_null {{.*}} -> ^{{.*}}, ^[[PATTERN_2_BLOCK]]
+  pdl.pattern : benefit(1) {
+    %type = pdl.type
+    %types = pdl.types
+    %root = pdl.operation -> (%type, %types : !pdl.type, !pdl.range<type>)
+    pdl.rewrite %root with "rewriter"
+  }
+  pdl.pattern : benefit(1) {
+    %type = pdl.type
+    %type2 = pdl.type
+    %types = pdl.types
+    %root = pdl.operation -> (%type, %type2, %types : !pdl.type, !pdl.type, !pdl.range<type>)
+    pdl.rewrite %root with "rewriter"
+  }
+}
+
+
 // -----
 
 // CHECK-LABEL: module @predicate_ordering
@@ -160,14 +397,14 @@ module @predicate_ordering  {
 
   pdl.pattern : benefit(1) {
     %resultType = pdl.type
-    pdl.apply_constraint "typeConstraint"[](%resultType : !pdl.type)
-    %root, %result = pdl.operation -> %resultType
+    pdl.apply_native_constraint "typeConstraint"[](%resultType : !pdl.type)
+    %root = pdl.operation -> (%resultType : !pdl.type)
     pdl.rewrite %root with "rewriter"
   }
 
   pdl.pattern : benefit(1) {
     %resultType = pdl.type
-    %apply, %applyRes = pdl.operation -> %resultType
+    %apply = pdl.operation -> (%resultType : !pdl.type)
     pdl.rewrite %apply with "rewriter"
   }
 }
diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir
index 4b6b1ae75700deea83600905ca26b0dd06ed89ff..58d1c3177dad93a3459e2d52e61e4d1f019cdc14 100644
--- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir
+++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir
@@ -6,10 +6,10 @@
 module @external {
   // CHECK: module @rewriters
   // CHECK:   func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation, %[[INPUT:.*]]: !pdl.value)
-  // CHECK:     pdl_interp.apply_rewrite "rewriter" [true](%[[INPUT]] : !pdl.value) on %[[ROOT]]
+  // CHECK:     pdl_interp.apply_rewrite "rewriter" [true](%[[ROOT]], %[[INPUT]] : !pdl.operation, !pdl.value)
   pdl.pattern : benefit(1) {
     %input = pdl.operand
-    %root = pdl.operation "foo.op"(%input)
+    %root = pdl.operation "foo.op"(%input : !pdl.value)
     pdl.rewrite %root with "rewriter"[true](%input : !pdl.value)
   }
 }
@@ -37,7 +37,7 @@ module @operation_attributes {
   // CHECK: module @rewriters
   // CHECK:   func @pdl_generated_rewriter(%[[ATTR:.*]]: !pdl.attribute, %[[ROOT:.*]]: !pdl.operation)
   // CHECK:     %[[ATTR1:.*]] = pdl_interp.create_attribute true
-  // CHECK:     pdl_interp.create_operation "foo.op"() {"attr" = %[[ATTR]], "attr1" = %[[ATTR1]]}
+  // CHECK:     pdl_interp.create_operation "foo.op" {"attr" = %[[ATTR]], "attr1" = %[[ATTR1]]}
   pdl.pattern : benefit(1) {
     %attr = pdl.attribute
     %root = pdl.operation "foo.op" {"attr" = %attr}
@@ -55,16 +55,17 @@ module @operation_attributes {
 module @operation_operands {
   // CHECK: module @rewriters
   // CHECK:   func @pdl_generated_rewriter(%[[OPERAND:.*]]: !pdl.value, %[[ROOT:.*]]: !pdl.operation)
-  // CHECK:     %[[NEWOP:.*]] = pdl_interp.create_operation "foo.op"(%[[OPERAND]])
+  // CHECK:     %[[NEWOP:.*]] = pdl_interp.create_operation "foo.op"(%[[OPERAND]] : !pdl.value)
   // CHECK:     %[[OPERAND1:.*]] = pdl_interp.get_result 0 of %[[NEWOP]]
-  // CHECK:     pdl_interp.create_operation "foo.op2"(%[[OPERAND1]])
+  // CHECK:     pdl_interp.create_operation "foo.op2"(%[[OPERAND1]] : !pdl.value)
   pdl.pattern : benefit(1) {
     %operand = pdl.operand
-    %root = pdl.operation "foo.op"(%operand)
+    %root = pdl.operation "foo.op"(%operand : !pdl.value)
     pdl.rewrite %root {
       %type = pdl.type : i32
-      %newOp, %result = pdl.operation "foo.op"(%operand) -> %type
-      %newOp1 = pdl.operation "foo.op2"(%result)
+      %newOp = pdl.operation "foo.op"(%operand : !pdl.value) -> (%type : !pdl.type)
+      %result = pdl.result 0 of %newOp
+      %newOp1 = pdl.operation "foo.op2"(%result : !pdl.value)
       pdl.erase %root
     }
   }
@@ -76,16 +77,17 @@ module @operation_operands {
 module @operation_operands {
   // CHECK: module @rewriters
   // CHECK:   func @pdl_generated_rewriter(%[[OPERAND:.*]]: !pdl.value, %[[ROOT:.*]]: !pdl.operation)
-  // CHECK:     %[[NEWOP:.*]] = pdl_interp.create_operation "foo.op"(%[[OPERAND]])
+  // CHECK:     %[[NEWOP:.*]] = pdl_interp.create_operation "foo.op"(%[[OPERAND]] : !pdl.value)
   // CHECK:     %[[OPERAND1:.*]] = pdl_interp.get_result 0 of %[[NEWOP]]
-  // CHECK:     pdl_interp.create_operation "foo.op2"(%[[OPERAND1]])
+  // CHECK:     pdl_interp.create_operation "foo.op2"(%[[OPERAND1]] : !pdl.value)
   pdl.pattern : benefit(1) {
     %operand = pdl.operand
-    %root = pdl.operation "foo.op"(%operand)
+    %root = pdl.operation "foo.op"(%operand : !pdl.value)
     pdl.rewrite %root {
       %type = pdl.type : i32
-      %newOp, %result = pdl.operation "foo.op"(%operand) -> %type
-      %newOp1 = pdl.operation "foo.op2"(%result)
+      %newOp = pdl.operation "foo.op"(%operand : !pdl.value) -> (%type : !pdl.type)
+      %result = pdl.result 0 of %newOp
+      %newOp1 = pdl.operation "foo.op2"(%result : !pdl.value)
       pdl.erase %root
     }
   }
@@ -93,18 +95,20 @@ module @operation_operands {
 
 // -----
 
-// CHECK-LABEL: module @operation_result_types
-module @operation_result_types {
+// CHECK-LABEL: module @operation_infer_types_from_replaceop
+module @operation_infer_types_from_replaceop {
   // CHECK: module @rewriters
-  // CHECK:   func @pdl_generated_rewriter(%[[TYPE:.*]]: !pdl.type, %[[TYPE1:.*]]: !pdl.type
-  // CHECK:     pdl_interp.create_operation "foo.op"() -> %[[TYPE]], %[[TYPE1]]
+  // CHECK:   func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation
+  // CHECK:     %[[RESULTS:.*]] = pdl_interp.get_results of %[[ROOT]]
+  // CHECK:     %[[RESULT_TYPES:.*]] = pdl_interp.get_value_type of %[[RESULTS]]
+  // CHECK:     pdl_interp.create_operation "foo.op" -> (%[[RESULT_TYPES]] : !pdl.range<type>)
   pdl.pattern : benefit(1) {
     %rootType = pdl.type
     %rootType1 = pdl.type
-    %root, %results:2 = pdl.operation "foo.op" -> %rootType, %rootType1
+    %root = pdl.operation "foo.op" -> (%rootType, %rootType1 : !pdl.type, !pdl.type)
     pdl.rewrite %root {
       %newType1 = pdl.type
-      %newOp, %newResults:2 = pdl.operation "foo.op" -> %rootType, %newType1
+      %newOp = pdl.operation "foo.op" -> (%rootType, %newType1 : !pdl.type, !pdl.type)
       pdl.replace %root with %newOp
     }
   }
@@ -112,21 +116,37 @@ module @operation_result_types {
 
 // -----
 
-// CHECK-LABEL: module @operation_result_types_infer_from_value_replacement
-module @operation_result_types_infer_from_value_replacement {
+// CHECK-LABEL: module @operation_infer_types_from_otherop_individual_results
+module @operation_infer_types_from_otherop_individual_results {
   // CHECK: module @rewriters
-  // CHECK:   func @pdl_generated_rewriter(%[[TYPE:.*]]: !pdl.type
-  // CHECK:     pdl_interp.create_operation "foo.op"() -> %[[TYPE]]
+  // CHECK:   func @pdl_generated_rewriter(%[[TYPE:.*]]: !pdl.type, %[[TYPES:.*]]: !pdl.range<type>
+  // CHECK:     pdl_interp.create_operation "foo.op" -> (%[[TYPE]], %[[TYPES]] : !pdl.type, !pdl.range<type>)
   pdl.pattern : benefit(1) {
     %rootType = pdl.type
-    %root, %result = pdl.operation "foo.op" -> %rootType
+    %rootTypes = pdl.types
+    %root = pdl.operation "foo.op" -> (%rootType, %rootTypes : !pdl.type, !pdl.range<type>)
     pdl.rewrite %root {
-      %newType = pdl.type
-      %newOp, %newResult = pdl.operation "foo.op" -> %newType
-      pdl.replace %root with (%newResult)
+      %newOp = pdl.operation "foo.op" -> (%rootType, %rootTypes : !pdl.type, !pdl.range<type>)
     }
   }
 }
+
+// -----
+
+// CHECK-LABEL: module @operation_infer_types_from_otherop_results
+module @operation_infer_types_from_otherop_results {
+  // CHECK: module @rewriters
+  // CHECK:   func @pdl_generated_rewriter(%[[TYPES:.*]]: !pdl.range<type>
+  // CHECK:     pdl_interp.create_operation "foo.op" -> (%[[TYPES]] : !pdl.range<type>)
+  pdl.pattern : benefit(1) {
+    %rootTypes = pdl.types
+    %root = pdl.operation "foo.op" -> (%rootTypes : !pdl.range<type>)
+    pdl.rewrite %root {
+      %newOp = pdl.operation "foo.op" -> (%rootTypes : !pdl.range<type>)
+    }
+  }
+}
+
 // -----
 
 // CHECK-LABEL: module @replace_with_op
@@ -134,13 +154,13 @@ module @replace_with_op {
   // CHECK: module @rewriters
   // CHECK:   func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation)
   // CHECK:     %[[NEWOP:.*]] = pdl_interp.create_operation
-  // CHECK:     %[[OP_RESULT:.*]] = pdl_interp.get_result 0 of %[[NEWOP]]
-  // CHECK:     pdl_interp.replace %[[ROOT]] with(%[[OP_RESULT]])
+  // CHECK:     %[[RESULTS:.*]] = pdl_interp.get_results of %[[NEWOP]]
+  // CHECK:     pdl_interp.replace %[[ROOT]] with (%[[RESULTS]] : !pdl.range<value>)
   pdl.pattern : benefit(1) {
     %type = pdl.type : i32
-    %root, %result = pdl.operation "foo.op" -> %type
+    %root = pdl.operation "foo.op" -> (%type : !pdl.type)
     pdl.rewrite %root {
-      %newOp, %newResult = pdl.operation "foo.op" -> %type
+      %newOp = pdl.operation "foo.op" -> (%type : !pdl.type)
       pdl.replace %root with %newOp
     }
   }
@@ -151,16 +171,21 @@ module @replace_with_op {
 // CHECK-LABEL: module @replace_with_values
 module @replace_with_values {
   // CHECK: module @rewriters
-  // CHECK:   func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation)
+  // CHECK:   func @pdl_generated_rewriter({{.*}}, %[[ROOT:.*]]: !pdl.operation)
   // CHECK:     %[[NEWOP:.*]] = pdl_interp.create_operation
-  // CHECK:     %[[OP_RESULT:.*]] = pdl_interp.get_result 0 of %[[NEWOP]]
-  // CHECK:     pdl_interp.replace %[[ROOT]] with(%[[OP_RESULT]])
+  // CHECK:     %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[NEWOP]]
+  // CHECK:     %[[RESULTS:.*]] = pdl_interp.get_results 1 of %[[NEWOP]] : !pdl.range<value>
+  // CHECK:     %[[RESULTS_2:.*]] = pdl_interp.get_results 2 of %[[NEWOP]] : !pdl.value
+  // CHECK:     pdl_interp.replace %[[ROOT]] with (%[[RESULT]], %[[RESULTS]], %[[RESULTS_2]] : !pdl.value, !pdl.range<value>, !pdl.value)
   pdl.pattern : benefit(1) {
-    %type = pdl.type : i32
-    %root, %result = pdl.operation "foo.op" -> %type
+    %types = pdl.types
+    %root = pdl.operation "foo.op" -> (%types : !pdl.range<type>)
     pdl.rewrite %root {
-      %newOp, %newResult = pdl.operation "foo.op" -> %type
-      pdl.replace %root with (%newResult)
+      %newOp = pdl.operation "foo.op" -> (%types : !pdl.range<type>)
+      %newResult = pdl.result 0 of %newOp
+      %newResults = pdl.results 1 of %newOp -> !pdl.range<value>
+      %newResults2 = pdl.results 2 of %newOp -> !pdl.value
+      pdl.replace %root with (%newResult, %newResults, %newResults2 : !pdl.value, !pdl.range<value>, !pdl.value)
     }
   }
 }
@@ -184,19 +209,18 @@ module @replace_with_no_results {
 
 // -----
 
-// CHECK-LABEL: module @create_native
-module @create_native {
+// CHECK-LABEL: module @apply_native_rewrite
+module @apply_native_rewrite {
   // CHECK: module @rewriters
   // CHECK:   func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation)
-  // CHECK:     %[[TYPE:.*]] = pdl_interp.create_native "functor" [true](%[[ROOT]] : !pdl.operation) : !pdl.type
-  // CHECK:     pdl_interp.create_operation "foo.op"() -> %[[TYPE]]
+  // CHECK:     %[[TYPE:.*]] = pdl_interp.apply_rewrite "functor" [true](%[[ROOT]] : !pdl.operation) : !pdl.type
+  // CHECK:     pdl_interp.create_operation "foo.op" -> (%[[TYPE]] : !pdl.type)
   pdl.pattern : benefit(1) {
     %type = pdl.type
-    %root, %result = pdl.operation "foo.op" -> %type
+    %root = pdl.operation "foo.op" -> (%type : !pdl.type)
     pdl.rewrite %root {
-      %newType = pdl.create_native "functor"[true](%root : !pdl.operation) : !pdl.type
-      %newOp, %newResult = pdl.operation "foo.op" -> %newType
-      pdl.replace %root with %newOp
+      %newType = pdl.apply_native_rewrite "functor"[true](%root : !pdl.operation) : !pdl.type
+      %newOp = pdl.operation "foo.op" -> (%newType : !pdl.type)
     }
   }
 }
diff --git a/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir b/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir
index 466bd6aa96af9373f72600ca36381fb6e4a03745..60a143a85006b13fe62e305920e8f5b7aff8077f 100644
--- a/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir
+++ b/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir
@@ -4,9 +4,9 @@
 func @parallel(%arg0: index, %arg1: index, %arg2: index,
           %arg3: index, %arg4: index, %arg5: index) {
   // CHECK: omp.parallel {
-  // CHECK:  "omp.wsloop"({{.*}}) ( {
+  // CHECK: omp.wsloop (%[[LVAR1:.*]], %[[LVAR2:.*]]) : index = (%arg0, %arg1) to (%arg2, %arg3) step (%arg4, %arg5) {
   scf.parallel (%i, %j) = (%arg0, %arg1) to (%arg2, %arg3) step (%arg4, %arg5) {
-    // CHECK:   test.payload
+    // CHECK: "test.payload"(%[[LVAR1]], %[[LVAR2]]) : (index, index) -> ()
     "test.payload"(%i, %j) : (index, index) -> ()
     // CHECK:   omp.yield
     // CHECK: }
@@ -20,12 +20,12 @@ func @parallel(%arg0: index, %arg1: index, %arg2: index,
 func @nested_loops(%arg0: index, %arg1: index, %arg2: index,
                    %arg3: index, %arg4: index, %arg5: index) {
   // CHECK: omp.parallel {
-  // CHECK:  "omp.wsloop"({{.*}}) ( {
+  // CHECK: omp.wsloop (%[[LVAR_OUT1:.*]]) : index = (%arg0) to (%arg2) step (%arg4) {
   // CHECK-NOT: omp.parallel
   scf.parallel (%i) = (%arg0) to (%arg2) step (%arg4) {
-    // CHECK:   "omp.wsloop"({{.*}}) ( {
+    // CHECK: omp.wsloop (%[[LVAR_IN1:.*]]) : index = (%arg1) to (%arg3) step (%arg5) {
     scf.parallel (%j) = (%arg1) to (%arg3) step (%arg5) {
-      // CHECK:   test.payload
+      // CHECK: "test.payload"(%[[LVAR_OUT1]], %[[LVAR_IN1]]) : (index, index) -> ()
       "test.payload"(%i, %j) : (index, index) -> ()
       // CHECK:   omp.yield
       // CHECK: }
@@ -41,9 +41,9 @@ func @nested_loops(%arg0: index, %arg1: index, %arg2: index,
 func @adjacent_loops(%arg0: index, %arg1: index, %arg2: index,
                      %arg3: index, %arg4: index, %arg5: index) {
   // CHECK: omp.parallel {
-  // CHECK:  "omp.wsloop"({{.*}}) ( {
+  // CHECK: omp.wsloop (%[[LVAR_AL1:.*]]) : index = (%arg0) to (%arg2) step (%arg4) {
   scf.parallel (%i) = (%arg0) to (%arg2) step (%arg4) {
-    // CHECK:   test.payload1
+    // CHECK: "test.payload1"(%[[LVAR_AL1]]) : (index) -> ()
     "test.payload1"(%i) : (index) -> ()
     // CHECK:   omp.yield
     // CHECK: }
@@ -52,9 +52,9 @@ func @adjacent_loops(%arg0: index, %arg1: index, %arg2: index,
   // CHECK: }
 
   // CHECK: omp.parallel {
-  // CHECK:  "omp.wsloop"({{.*}}) ( {
+  // CHECK: omp.wsloop (%[[LVAR_AL2:.*]]) : index = (%arg1) to (%arg3) step (%arg5) {
   scf.parallel (%j) = (%arg1) to (%arg3) step (%arg5) {
-    // CHECK:   test.payload2
+    // CHECK: "test.payload2"(%[[LVAR_AL2]]) : (index) -> ()
     "test.payload2"(%j) : (index) -> ()
     // CHECK:   omp.yield
     // CHECK: }
diff --git a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
index 6db4106d6e693e28ce5ce51ed83538628a94aad4..e0fc24ee13339db3927b918f0a97045f19db57aa 100644
--- a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
@@ -144,7 +144,7 @@ func @return_var_memref_caller(%arg0: memref<4x3xf32>) {
 }
 
 // CHECK-LABEL: llvm.func @return_var_memref
-func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> {
+func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> attributes { llvm.emit_c_interface } {
   // Match the construction of the unranked descriptor.
   // CHECK: %[[ALLOCA:.*]] = llvm.alloca
   // CHECK: %[[MEMORY:.*]] = llvm.bitcast %[[ALLOCA]]
@@ -177,6 +177,10 @@ func @return_var_memref(%arg0: memref<4x3xf32>) -> memref<*xf32> {
   return %0 : memref<*xf32>
 }
 
+// Check that the result memref is passed as parameter
+// CHECK-LABEL: @_mlir_ciface_return_var_memref
+// CHECK-SAME: (%{{.*}}: !llvm.ptr<struct<(i64, ptr<i8>)>>, %{{.*}}: !llvm.ptr<struct<(ptr<f32>, ptr<f32>, i64, array<2 x i64>, array<2 x i64>)>>)
+
 // CHECK-LABEL: llvm.func @return_two_var_memref_caller
 func @return_two_var_memref_caller(%arg0: memref<4x3xf32>) {
   // Only check that we create two different descriptors using different
@@ -206,7 +210,7 @@ func @return_two_var_memref_caller(%arg0: memref<4x3xf32>) {
 }
 
 // CHECK-LABEL: llvm.func @return_two_var_memref
-func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memref<*xf32>) {
+func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memref<*xf32>) attributes { llvm.emit_c_interface } {
   // Match the construction of the unranked descriptor.
   // CHECK: %[[ALLOCA:.*]] = llvm.alloca
   // CHECK: %[[MEMORY:.*]] = llvm.bitcast %[[ALLOCA]]
@@ -240,3 +244,8 @@ func @return_two_var_memref(%arg0: memref<4x3xf32>) -> (memref<*xf32>, memref<*x
   return %0, %0 : memref<*xf32>, memref<*xf32>
 }
 
+// Check that the result memrefs are passed as parameter
+// CHECK-LABEL: @_mlir_ciface_return_two_var_memref
+// CHECK-SAME: (%{{.*}}: !llvm.ptr<struct<(struct<(i64, ptr<i8>)>, struct<(i64, ptr<i8>)>)>>,
+// CHECK-SAME: %{{.*}}: !llvm.ptr<struct<(ptr<f32>, ptr<f32>, i64, array<2 x i64>, array<2 x i64>)>>)
+
diff --git a/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir b/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
index fcb5b1c8a5a257070e916e78ca588f3280861de8..5eca81dcad00341e1051ff2993df33c2fee2d89b 100644
--- a/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
@@ -12,6 +12,18 @@ func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) {
 
 // -----
 
+// CHECK-LABEL: func @log1p(
+// CHECK-SAME: f32
+func @log1p(%arg0 : f32) {
+  // CHECK: %[[ONE:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : f32
+  // CHECK: %[[ADD:.*]] = llvm.fadd %[[ONE]], %arg0 : f32
+  // CHECK: %[[LOG:.*]] = "llvm.intr.log"(%[[ADD]]) : (f32) -> f32
+  %0 = math.log1p %arg0 : f32
+  std.return
+}
+
+// -----
+
 // CHECK-LABEL: func @rsqrt(
 // CHECK-SAME: f32
 func @rsqrt(%arg0 : f32) {
diff --git a/mlir/test/Conversion/StandardToSPIRV/legalization.mlir b/mlir/test/Conversion/StandardToSPIRV/legalization.mlir
index 98b5d930eee1cbc1fae8cb9228e23bdf6a43ec15..e7fa980186e0a1b2a31b2ec36ec035528f00f2ff 100644
--- a/mlir/test/Conversion/StandardToSPIRV/legalization.mlir
+++ b/mlir/test/Conversion/StandardToSPIRV/legalization.mlir
@@ -67,9 +67,9 @@ func @fold_dynamic_stride_subview_with_store(%arg0 : memref<12x32xf32>, %arg1 :
 // CHECK-SAME: [[ARG0:%.*]]: memref<12x32xf32>, [[ARG1:%.*]]: index, [[ARG2:%.*]]: index, [[ARG3:%.*]]: index, [[ARG4:%.*]]: index
 func @fold_static_stride_subview_with_transfer_read(%arg0 : memref<12x32xf32>, %arg1 : index, %arg2 : index, %arg3 : index, %arg4 : index) -> vector<4xf32> {
   // CHECK-NOT: memref.subview
-  // CHECK: [[F1:%.*]] = constant 1.000000e+00 : f32
-  // CHECK: [[C2:%.*]] = constant 2 : index
-  // CHECK: [[C3:%.*]] = constant 3 : index
+  // CHECK-DAG: [[F1:%.*]] = constant 1.000000e+00 : f32
+  // CHECK-DAG: [[C2:%.*]] = constant 2 : index
+  // CHECK-DAG: [[C3:%.*]] = constant 3 : index
   // CHECK: [[STRIDE1:%.*]] = muli [[ARG3]], [[C2]] : index
   // CHECK: [[INDEX1:%.*]] = addi [[ARG1]], [[STRIDE1]] : index
   // CHECK: [[STRIDE2:%.*]] = muli [[ARG4]], [[C3]] : index
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
index d1868e7683cec2bac080086b159eb813e54a9b46..8dc968193829b3ffde6607c49a6e4844760e7a81 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
@@ -180,6 +180,48 @@ func @test_simple_f32(%arg0: tensor<1xf32>) -> () {
   // CHECK: select
   %18 = "tosa.reluN"(%0) {max_int = 5 : i64, max_fp = 5.0 : f32} : (tensor<1xf32>) -> tensor<1xf32>
 
+  // CHECK: linalg.generic
+  // CHECK: fptosi
+  %19 = "tosa.cast"(%0) : (tensor<1xf32>) -> tensor<1xi32>
+
+  // CHECK: linalg.generic
+  // CHECK: constant 0
+  // CHECK: cmpf
+  %20 = "tosa.cast"(%0) : (tensor<1xf32>) -> tensor<1xi1>
+
+  // CHECK: linalg.generic
+  // CHECK: fptrunc
+  %21 = "tosa.cast"(%0) : (tensor<1xf32>) -> tensor<1xf16>
+
+  // CHECK: linalg.generic
+  // CHECK: yield
+  %22 = "tosa.cast"(%0) : (tensor<1xf32>) -> tensor<1xf32>
+
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @test_simple_f16
+func @test_simple_f16(%arg0: tensor<1xf16>) -> () {
+
+  // CHECK: linalg.generic
+  // CHECK: fpext
+  %0 = "tosa.cast"(%arg0) : (tensor<1xf16>) -> tensor<1xf32>
+
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @test_simple_i16
+func @test_simple_i16(%arg0: tensor<1xi16>) -> () {
+  // CHECK: linalg.generic
+  // CHECK: sext
+  // CHECK: sext
+  // CHECK: muli
+  %0 = "tosa.mul"(%arg0, %arg0) {shift = 0 : i32} : (tensor<1xi16>, tensor<1xi16>) -> tensor<1xi32>
+
   return
 }
 
@@ -199,61 +241,111 @@ func @test_simple_i32(%arg0: tensor<1xi32>) -> () {
   // CHECK: muli
   %2 = "tosa.mul"(%arg0, %arg0) {shift = 0 : i32} : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
+  // CHECK: linalg.generic
+  // CHECK: constant 2
+  // CHECK: apply_scale
+  %3 = "tosa.mul"(%arg0, %arg0) {shift = 2 : i32} : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+
   // CHECK: linalg.generic
   // CHECK: muli
-  %3 = "tosa.negate"(%arg0) : (tensor<1xi32>) -> tensor<1xi32>
+  %4 = "tosa.negate"(%arg0) : (tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: and
-  %4 = "tosa.bitwise_and"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %5 = "tosa.bitwise_and"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: or
-  %5 = "tosa.bitwise_or"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %6 = "tosa.bitwise_or"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: xor
-  %6 = "tosa.bitwise_xor"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %7 = "tosa.bitwise_xor"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: shift_left
-  %7 = "tosa.logical_left_shift"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %8 = "tosa.logical_left_shift"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: shift_right_unsigned
-  %8 = "tosa.logical_right_shift"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %9 = "tosa.logical_right_shift"(%arg0, %arg0) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: cmpi
-  %9 = "tosa.greater"(%0, %1) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi1>
+  %10 = "tosa.greater"(%0, %1) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi1>
 
   // CHECK: linalg.generic
   // CHECK: cmpi
-  %10 = "tosa.greater_equal"(%0, %1) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi1>
+  %11 = "tosa.greater_equal"(%0, %1) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi1>
 
   // CHECK: linalg.generic
   // CHECK: select
-  %11 = "tosa.select"(%9, %0, %1) : (tensor<1xi1>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %12 = "tosa.select"(%10, %0, %1) : (tensor<1xi1>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: cmpi
   // CHECK: select
-  %12 = "tosa.maximum"(%0, %1) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %13 = "tosa.maximum"(%0, %1) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: cmpi
   // CHECK: select
-  %13 = "tosa.minimum"(%0, %1) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
+  %14 = "tosa.minimum"(%0, %1) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: cmpi
   // CHECK: select
-  %14 = "tosa.clamp"(%0) {min_int = 1 : i64, max_int = 5 : i64, min_fp = 1.0 : f32, max_fp = 5.0 : f32} : (tensor<1xi32>) -> tensor<1xi32>
+  %15 = "tosa.clamp"(%0) {min_int = 1 : i64, max_int = 5 : i64, min_fp = 1.0 : f32, max_fp = 5.0 : f32} : (tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: cmpi
   // CHECK: select
-  %15 = "tosa.reluN"(%0) {max_int = 5 : i64, max_fp = 5.0 : f32} : (tensor<1xi32>) -> tensor<1xi32>
+  %16 = "tosa.reluN"(%0) {max_int = 5 : i64, max_fp = 5.0 : f32} : (tensor<1xi32>) -> tensor<1xi32>
+
+  // CHECK: linalg.generic
+  // CHECK: trunci
+  %17 = "tosa.cast"(%0) : (tensor<1xi32>) -> tensor<1xi16>
+
+  // CHECK: linalg.generic
+  // CHECK: yield
+  %18 = "tosa.cast"(%0) : (tensor<1xi32>) -> tensor<1xi32>
+
+  // CHECK: linalg.generic
+  // CHECK: sexti
+  %19 = "tosa.cast"(%0) : (tensor<1xi32>) -> tensor<1xi64>
+
+  // CHECK: linalg.generic
+  // CHECK: constant 0
+  // CHECK: cmpi
+  %20 = "tosa.cast"(%0) : (tensor<1xi32>) -> tensor<1xi1>
+
+  // CHECK: linalg.generic
+  // CHECK: sitofp
+  %21 = "tosa.cast"(%0) : (tensor<1xi32>) -> tensor<1xf32>
+
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @test_bool
+func @test_bool(%arg0: tensor<1xi1>, %arg1: tensor<1xi1>) -> () {
+  // CHECK: linalg.generic
+  // CHECK: and
+  %0 = "tosa.logical_and"(%arg0, %arg1) : (tensor<1xi1>, tensor<1xi1>) -> tensor<1xi1>
+
+  // CHECK: linalg.generic
+  // CHECK: or
+  %1 = "tosa.logical_or"(%arg0, %arg1) : (tensor<1xi1>, tensor<1xi1>) -> tensor<1xi1>
+
+  // CHECK: linalg.generic
+  // CHECK: xor
+  %2 = "tosa.logical_xor"(%arg0, %arg1) : (tensor<1xi1>, tensor<1xi1>) -> tensor<1xi1>
+
+  // CHECK: linalg.generic
+  // CHECK: constant true
+  // CHECK: xor
+  %3 = "tosa.logical_not"(%arg0) : (tensor<1xi1>) -> tensor<1xi1>
 
   return
 }
@@ -433,3 +525,271 @@ func @reduce_int(%arg0: tensor<5x4xi32>) -> () {
   %4 = "tosa.reduce_max"(%arg0) {axis = 0 : i64} : (tensor<5x4xi32>) -> tensor<4xi32>
   return
 }
+
+// -----
+
+// CHECK-LABEL: @concat
+func @concat(%arg0: tensor<5x1xf32>, %arg1: tensor<6x1xf32>) -> () {
+  // CHECK: [[AXIS:%.+]] = constant 0
+  // CHECK: [[STRIDE:%.+]]   = constant 1
+  // CHECK: [[OFFSET:%.+]] = constant 0 : index
+  // CHECK: [[IDX0:%.+]] = constant 0 : index
+  // CHECK: [[ARG0_DIM0:%.+]] = memref.dim %arg0, [[IDX0]]
+  // CHECK: [[IDX1:%.+]] = constant 1 : index
+  // CHECK: [[ARG0_DIM1:%.+]] = memref.dim %arg0, [[IDX1]]
+  // CHECK: [[ARG1_AXIS:%.+]] = memref.dim %arg1, [[AXIS]]
+  // CHECK: [[RESULT_AXIS:%.+]] = addi [[ARG0_DIM0]], [[ARG1_AXIS]]
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [11, 1]
+  // CHECK: [[ARG0_DIM0:%.+]] = memref.dim %arg0, [[AXIS]]
+  // CHECK: [[INSERT0:%.+]] = subtensor_insert %arg0 into [[INIT]]{{\[}}[[OFFSET]], [[OFFSET]]] {{\[}}[[ARG0_DIM0]], [[ARG0_DIM1]]] {{\[}}[[STRIDE]], [[STRIDE]]]
+  // CHECK: [[NEW_OFFSET:%.+]] = addi [[OFFSET]], [[ARG0_DIM0]]
+  // CHECK: [[ARG1_DIM0:%.+]] = memref.dim %arg1, [[AXIS]]
+  // CHECK: [[INSERT1:%.+]] = subtensor_insert %arg1 into [[INSERT0]]{{\[}}[[NEW_OFFSET]], [[OFFSET]]] {{\[}}[[ARG1_DIM0]], [[ARG0_DIM1]]] {{\[}}[[STRIDE]], [[STRIDE]]]
+  %0 = "tosa.concat"(%arg0, %arg1) { axis = 0 : i64} : (tensor<5x1xf32>, tensor<6x1xf32>)  -> (tensor<11x1xf32>)
+
+  // CHECK: [[AXIS:%.+]] = constant 1
+  // CHECK: [[STRIDE:%.+]]   = constant 1
+  // CHECK: [[OFFSET:%.+]] = constant 0 : index
+  // CHECK: [[IDX0:%.+]] = constant 0 : index
+  // CHECK: [[ARG0_DIM0:%.+]] = memref.dim %arg0, [[IDX0]]
+  // CHECK: [[IDX1:%.+]] = constant 1 : index
+  // CHECK: [[ARG0_DIM1:%.+]] = memref.dim %arg0, [[IDX1]]
+  // CHECK: [[ARG1_AXIS:%.+]] = memref.dim %arg0, [[AXIS]]
+  // CHECK: [[RESULT_AXIS:%.+]] = addi [[ARG0_DIM1]], [[ARG1_AXIS]]
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [5, 2]
+  // CHECK: [[ARG0_DIM1:%.+]] = memref.dim %arg0, [[AXIS]]
+  // CHECK: [[INSERT0:%.+]] = subtensor_insert %arg0 into [[INIT]]{{\[}}[[OFFSET]], [[OFFSET]]] {{\[}}[[ARG0_DIM0]], [[ARG0_DIM1]]] {{\[}}[[STRIDE]], [[STRIDE]]]
+  // CHECK: [[NEW_OFFSET:%.+]] = addi [[OFFSET]], [[ARG0_DIM1]]
+  // CHECK: [[ARG1_DIM1:%.+]] = memref.dim %arg0, [[AXIS]]
+  // CHECK: [[INSERT1:%.+]] = subtensor_insert %arg0 into [[INSERT0]]{{\[}}[[OFFSET]], [[NEW_OFFSET]]] {{\[}}[[ARG0_DIM0]], [[ARG1_DIM1]]] {{\[}}[[STRIDE]], [[STRIDE]]]
+  %1 = "tosa.concat"(%arg0, %arg0) { axis = 1 : i64} : (tensor<5x1xf32>, tensor<5x1xf32>)  -> (tensor<5x2xf32>)
+  return
+}
+
+// -----
+
+// CHECK: #[[$MAP0:.*]] = affine_map<(d0) -> (d0)>
+// CHECK: #[[$MAP1:.*]] = affine_map<(d0) -> (0)>
+
+// CHECK-LABEL: @rescale
+func @rescale(%arg0 : tensor<1xi8>) -> (tensor<1xi8>) {
+  // CHECK: [[C0:%.+]] = constant dense<19689>
+  // CHECK: [[C1:%.+]] = constant dense<15>
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [1]
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP1]], #[[$MAP0]]], iterator_types = ["parallel"]} ins(%arg0, [[C0]], [[C1]] : tensor<1xi8>, tensor<1xi32>, tensor<1xi8>) outs([[INIT]] : tensor<1xi8>)
+  // CHECK: ^bb0([[IN:%.+]]: i8, [[MULTIPLIER:%.+]]: i32, [[SHIFT:%.+]]: i8, [[UNUSED:%.+]]: i8):
+  // CHECK: [[C243:%.+]] = constant 243
+  // CHECK: [[C252:%.+]] = constant 252
+
+  // CHECK-DAG: [[IN32:%.+]] = sexti [[IN]]
+  // CHECK-DAG: [[IN_ZEROED:%.+]] = subi [[IN32]], [[C243]]
+  // CHECK-DAG: [[SCALED:%.+]] = "tosa.apply_scale"([[IN_ZEROED]], [[MULTIPLIER]], [[SHIFT]]) {double_round = false}
+  // CHECK-DAG: [[SCALED_ZEROED:%.+]] = addi [[SCALED]], [[C252]]
+  // CHECK-DAG: [[CMIN:%.+]] = constant -128
+  // CHECK-DAG: [[CMAX:%.+]] = constant 127
+  // CHECK-DAG: [[MINLT:%.+]] = cmpi slt, [[SCALED_ZEROED]], [[CMIN]]
+  // CHECK-DAG: [[MAXLT:%.+]] = cmpi slt, [[CMAX]], [[SCALED_ZEROED]]
+  // CHECK-DAG: [[LOWER:%.+]] = select [[MINLT]], [[CMIN]], [[SCALED_ZEROED]]
+  // CHECK-DAG: [[BOUNDED:%.+]] = select [[MAXLT]], [[CMAX]], [[LOWER]]
+  // CHECK-DAG: [[TRUNC:%.+]] = trunci [[BOUNDED]]
+  // CHECK-DAG: linalg.yield [[TRUNC]]
+  %0 = "tosa.rescale"(%arg0) {input_zp = 243 : i32, output_zp = 252 : i32, multiplier = [19689 : i32], shift = [15 : i32], scale32 = false, double_round = false, per_channel = false} : (tensor<1xi8>)  -> (tensor<1xi8>)
+
+  // CHECK: return [[GENERIC]]
+  return %0 : tensor<1xi8>
+}
+
+// CHECK-LABEL: @rescaleDoubleRound
+func @rescaleDoubleRound(%arg0 : tensor<1xi8>) -> (tensor<1xi8>) {
+  // CHECK: linalg.generic
+  // CHECK: "tosa.apply_scale"
+  // CHECK-SAME:  {double_round = true}
+  %0 = "tosa.rescale"(%arg0) {input_zp = 243 : i32, output_zp = 252 : i32, multiplier = [19689 : i32], shift = [33 : i32], scale32 = true, double_round = true, per_channel = false} : (tensor<1xi8>)  -> (tensor<1xi8>)
+  return %0 : tensor<1xi8>
+}
+
+// CHECK-LABEL: @rescaleUnnecessaryDoubleRound
+func @rescaleUnnecessaryDoubleRound(%arg0 : tensor<1xi8>) -> (tensor<1xi8>) {
+  // CHECK: linalg.generic
+  // CHECK: "tosa.apply_scale"
+  // CHECK-SAME:  {double_round = false}
+  %0 = "tosa.rescale"(%arg0) {input_zp = 243 : i32, output_zp = 252 : i32, multiplier = [19689 : i32], shift = [15 : i32], scale32 = true, double_round = true, per_channel = false} : (tensor<1xi8>)  -> (tensor<1xi8>)
+  return %0 : tensor<1xi8>
+}
+
+// -----
+
+// CHECK: #[[$MAP0:.*]] = affine_map<(d0, d1) -> (-d0 + 4, d1)>
+// CHECK: #[[$MAP1:.*]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK: #[[$MAP2:.*]] = affine_map<(d0, d1) -> (d0, -d1 + 3)>
+
+// CHECK-LABEL: @reverse
+func @reverse(%arg0: tensor<5x4xi32>) -> () {
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [5, 4]
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel"]} ins(%arg0 : tensor<5x4xi32>) outs([[INIT]] : tensor<5x4xi32>) {
+  // CHECK: ^bb0(%arg1: i32, %arg2: i32):
+  // CHECK:   linalg.yield %arg1 : i32
+  %0 = "tosa.reverse"(%arg0) {axis = 0 : i64} : (tensor<5x4xi32>) -> tensor<5x4xi32>
+
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [5, 4]
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP2]], #[[$MAP1]]], iterator_types = ["parallel", "parallel"]} ins(%arg0 : tensor<5x4xi32>) outs([[INIT]] : tensor<5x4xi32>) {
+  // CHECK: ^bb0(%arg1: i32, %arg2: i32):
+  // CHECK:   linalg.yield %arg1 : i32
+  %1 = "tosa.reverse"(%arg0) {axis = 1 : i64} : (tensor<5x4xi32>) -> tensor<5x4xi32>
+  return
+}
+
+// -----
+
+// CHECK: #[[$MAP0:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+// CHECK: #[[$MAP1:.*]] = affine_map<(d0, d1, d2, d3) -> (d3)>
+// CHECK: #[[$MAP2:.*]] = affine_map<(d0, d1, d2, d3) -> (0, d1, 0, d3)>
+// CHECK: #[[$MAP3:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+// CHECK: #[[$MAP4:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1)>
+// CHECK: #[[$MAP5:.*]] = affine_map<(d0, d1, d2, d3) -> (d2, d3)>
+
+// CHECK-LABEL: @tile
+func @tile(%arg0 : tensor<2x3xi8>) -> () {
+  // CHECK: [[RESHAPE:%.+]] = linalg.tensor_reshape %arg0 [#[[$MAP0]], #[[$MAP1]]] : tensor<2x3xi8> into tensor<1x2x1x3xi8>
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [2, 2, 1, 3]
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP2]], #[[$MAP3]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins([[RESHAPE]] : tensor<1x2x1x3xi8>) outs([[INIT]] : tensor<2x2x1x3xi8>)
+  // CHECK:   linalg.yield %arg1 : i8
+  // CHECK: linalg.tensor_reshape [[GENERIC]] [#[[$MAP0]], #[[$MAP1]]]
+  %0 = "tosa.tile"(%arg0) {multiples = [2, 1]} : (tensor<2x3xi8>)  -> (tensor<4x3xi8>)
+
+  // CHECK: [[RESHAPE:%.+]] = linalg.tensor_reshape %arg0 [#[[$MAP0]], #[[$MAP1]]] : tensor<2x3xi8> into tensor<1x2x1x3xi8>
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [1, 2, 2, 3]
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP2]], #[[$MAP3]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins([[RESHAPE]] : tensor<1x2x1x3xi8>) outs([[INIT]] : tensor<1x2x2x3xi8>)
+  // CHECK:   linalg.yield %arg1 : i8
+  // CHECK: linalg.tensor_reshape [[GENERIC]] [#[[$MAP4]], #[[$MAP5]]]
+  %1 = "tosa.tile"(%arg0) {multiples = [1, 2]} : (tensor<2x3xi8>)  -> (tensor<2x6xi8>)
+
+  // CHECK: [[RESHAPE:%.+]] = linalg.tensor_reshape %arg0 [#[[$MAP0]], #[[$MAP1]]] : tensor<2x3xi8> into tensor<1x2x1x3xi8>
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [5, 2, 7, 3]
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP2]], #[[$MAP3]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins([[RESHAPE]] : tensor<1x2x1x3xi8>) outs([[INIT]] : tensor<5x2x7x3xi8>)
+  // CHECK:   linalg.yield %arg1 : i8
+  // CHECK: linalg.tensor_reshape [[GENERIC]] [#[[$MAP4]], #[[$MAP5]]]
+  %2 = "tosa.tile"(%arg0) {multiples = [5, 7]} : (tensor<2x3xi8>)  -> (tensor<10x21xi8>)
+
+  return
+}
+
+// -----
+
+
+// CHECK-LABEL: @matmul
+func @matmul(%arg0: tensor<5x3xf32>, %arg1: tensor<3x6xf32>, %arg2: tensor<6xf32>) -> (tensor<5x6xf32>) {
+  // CHECK: [[C0:%.+]] = constant 0
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [5, 6]
+  // CHECK: [[FILLED:%.+]] = linalg.fill([[INIT]], [[C0]]) : tensor<5x6xf32>, f32 -> tensor<5x6xf32>
+  // CHECK: linalg.matmul ins(%arg0, %arg1 : tensor<5x3xf32>, tensor<3x6xf32>) outs([[FILLED]] : tensor<5x6xf32>) -> tensor<5x6xf32>
+  %0 = "tosa.matmul"(%arg0, %arg1) : (tensor<5x3xf32>, tensor<3x6xf32>)  -> (tensor<5x6xf32>)
+  return %0 : tensor<5x6xf32>
+}
+
+// -----
+
+// CHECK: #[[$MAP0:.*]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK: #[[$MAP1:.*]] = affine_map<(d0, d1) -> (0, d1)>
+
+// CHECK-LABEL: @fully_connected
+func @fully_connected(%arg0: tensor<5x3xf32>, %arg1: tensor<3x6xf32>, %arg2: tensor<6xf32>) -> (tensor<5x6xf32>) {
+  // CHECK: [[RS:%.+]] = linalg.tensor_reshape %arg2 [#[[$MAP0]]]
+  // CHECK: [[INIT:%.+]] = linalg.init_tensor [5, 6]
+  // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP0]]], iterator_types = ["parallel", "parallel"]} ins([[RS]] : tensor<1x6xf32>) outs([[INIT]] : tensor<5x6xf32>) {
+  // CHECK: ^bb0([[IN:%.+]]: f32, [[MULTIPLIER:%.+]]: f32):
+  // CHECK: linalg.matmul ins(%arg0, %arg1 : tensor<5x3xf32>, tensor<3x6xf32>) outs([[GENERIC]] : tensor<5x6xf32>) -> tensor<5x6xf32>
+  %0 = "tosa.fully_connected"(%arg0, %arg1, %arg2) : (tensor<5x3xf32>, tensor<3x6xf32>, tensor<6xf32>)  -> (tensor<5x6xf32>)
+  return %0 : tensor<5x6xf32>
+}
+
+// -----
+
+func @pad_float(%arg0 : tensor<1x2xf32>) -> (tensor<4x9xf32>) {
+  %0 = constant dense<[[1, 2], [3, 4]]> : tensor<2x2xi32>
+  // CHECK: [[INDEX0:%.+]] = constant 0 : index
+  // CHECK: [[INDEX1:%.+]] = constant 1 : index
+  // CHECK: [[ROW0:%.+]] = constant 0 : index
+  // CHECK: [[LOW0:%.+]] = tensor.extract %cst{{\[}}[[ROW0]], [[INDEX0]]]
+  // CHECK: [[HIGH0:%.+]] = tensor.extract %cst{{\[}}[[ROW0]], [[INDEX1]]]
+  // CHECK: [[LOW0_IDX:%.+]] = index_cast %0
+  // CHECK: [[HIGH0_IDX:%.+]] = index_cast %1
+  // CHECK: [[ROW1:%.+]] = constant 1 : index
+  // CHECK: [[LOW1:%.+]] = tensor.extract %cst{{\[}}%c1_1, %c0]
+  // CHECK: [[HIGH1:%.+]] = tensor.extract %cst{{\[}}%c1_1, %c1]
+  // CHECK: [[LOW1_IDX:%.+]] = index_cast [[LOW1]]
+  // CHECK: [[HIGH1_IDX:%.+]] = index_cast [[HIGH1]]
+  // CHECK: [[CST:%.+]] = constant 0.000000e+00 : f32
+  // CHECK: %8 = linalg.pad_tensor %arg0 low{{\[}}[[LOW0_IDX]], [[LOW1_IDX]]] high{{\[}}[[HIGH0_IDX]], [[HIGH1_IDX]]]  {
+  // CHECK: ^bb0(%arg1: index, %arg2: index):  // no predecessors
+  // CHECK:   linalg.yield [[CST]]
+  // CHECK: } : tensor<1x2xf32> to tensor<4x9xf32>
+  %1 = "tosa.pad"(%arg0, %0)  : (tensor<1x2xf32>, tensor<2x2xi32>)  -> (tensor<4x9xf32>)
+  return %1 : tensor<4x9xf32>
+}
+
+func @pad_int(%arg0 : tensor<1x2xi32>) -> (tensor<4x9xi32>) {
+  %0 = constant dense<[[1, 2], [3, 4]]> : tensor<2x2xi32>
+  // CHECK: [[CST:%.+]] = constant 0 : i32
+  // CHECK: linalg.pad_tensor
+  // CHECK:   linalg.yield [[CST]]
+  %1 = "tosa.pad"(%arg0, %0)  : (tensor<1x2xi32>, tensor<2x2xi32>)  -> (tensor<4x9xi32>)
+  return %1 : tensor<4x9xi32>
+}
+
+func @pad_quant(%arg0 : tensor<1x2xi32>) -> (tensor<4x9xi32>) {
+  %0 = constant dense<[[1, 2], [3, 4]]> : tensor<2x2xi32>
+  // CHECK: [[CST:%.+]] = constant 42 : i32
+  // CHECK: linalg.pad_tensor
+  // CHECK:   linalg.yield [[CST]]
+  %1 = "tosa.pad"(%arg0, %0) { quantization_info = { input_zp = 42 : i32}} : (tensor<1x2xi32>, tensor<2x2xi32>)  -> (tensor<4x9xi32>)
+  return %1 : tensor<4x9xi32>
+}
+
+// -----
+
+// CHECK: #[[$MAP0:.*]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK: #[[$MAP1:.*]] = affine_map<(d0, d1) -> (d1)>
+// CHECK: #[[$MAP2:.*]] = affine_map<(d0, d1) -> (d0)>
+// CHECK: #[[$MAP3:.*]] = affine_map<(d0) -> (d0)>
+// CHECK: #[[$MAP4:.*]] = affine_map<(d0) -> ()>
+
+func @argmax(%arg0 : tensor<3x2xi32>, %arg1 : tensor<6xf32>) -> () {
+  // CHECK: [[IDX_INIT:%.+]] = linalg.init_tensor [2]
+  // CHECK: [[IDX_MIN:%.+]] = constant 0 : i32
+  // CHECK: [[IDX_FILL:%.+]] = linalg.fill([[IDX_INIT]], [[IDX_MIN]])
+  // CHECK: [[VAL_INIT:%.+]] = linalg.init_tensor [2]
+  // CHECK: [[VAL_MIN:%.+]] = constant -2147483648
+  // CHECK: [[VAL_FILL:%.+]] = linalg.fill([[VAL_INIT]], [[VAL_MIN]])
+  // CHECK: linalg.indexed_generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP1]]], iterator_types = ["reduction", "parallel"]} ins(%arg0 : tensor<3x2xi32>) outs([[IDX_FILL]], [[VAL_FILL]] : tensor<2xi32>, tensor<2xi32>)
+  // CHECK:   [[CAST:%.+]] = index_cast %arg2
+  // CHECK:   [[CMP:%.+]] = cmpi sgt, %arg4, %arg6
+  // CHECK:   [[SELECT_VAL:%.+]] = select [[CMP]], %arg4, %arg6
+  // CHECK:   [[SELECT_IDX:%.+]] = select [[CMP]], [[CAST]], %arg5
+  // CHECK:   linalg.yield [[SELECT_IDX]], [[SELECT_VAL]]
+  %0 = "tosa.argmax"(%arg0) { axis = 0 : i64} : (tensor<3x2xi32>)  -> (tensor<2xi32>)
+
+  // CHECK: [[IDX_INIT:%.+]] = linalg.init_tensor [3]
+  // CHECK: [[IDX_MIN:%.+]] = constant 0 : i32
+  // CHECK: [[IDX_FILL:%.+]] = linalg.fill([[IDX_INIT]], [[IDX_MIN]])
+  // CHECK: [[VAL_INIT:%.+]] = linalg.init_tensor [3]
+  // CHECK: [[VAL_MIN:%.+]] = constant -2147483648
+  // CHECK: [[VAL_FILL:%.+]] = linalg.fill([[VAL_INIT]], [[VAL_MIN]])
+  // CHECK: linalg.indexed_generic {indexing_maps = [#map0, #map2, #map2], iterator_types = ["parallel", "reduction"]} ins(%arg0 : tensor<3x2xi32>) outs([[IDX_FILL]], [[VAL_FILL]] : tensor<3xi32>, tensor<3xi32>)
+  // CHECK:   [[CAST:%.+]] = index_cast %arg3
+  // CHECK:   [[CMP:%.+]] = cmpi sgt, %arg4, %arg6
+  // CHECK:   [[SELECT_VAL:%.+]] = select [[CMP]], %arg4, %arg6
+  // CHECK:   [[SELECT_IDX:%.+]] = select [[CMP]], [[CAST]], %arg5
+  // CHECK:   linalg.yield [[SELECT_IDX]], [[SELECT_VAL]]
+  %1 = "tosa.argmax"(%arg0) { axis = 1 : i64} : (tensor<3x2xi32>)  -> (tensor<3xi32>)
+
+  // CHECK: constant -3.40282347E+38 : f32
+  // CHECK: index_cast
+  // CHECK: cmpf ogt
+  // CHECK: select
+  // CHECK: select
+  // CHECK: linalg.yield
+  %2 = "tosa.argmax"(%arg1) { axis = 0 : i64} : (tensor<6xf32>)  -> (tensor<i32>)
+
+  return
+}
diff --git a/mlir/test/Conversion/TosaToStandard/tosa-to-standard.mlir b/mlir/test/Conversion/TosaToStandard/tosa-to-standard.mlir
index 86304dcba8627b69af580bebf9ebbe2f8c6f4026..2c80c31cf297b1d8f7dfe7d4e25067604d1a6669 100644
--- a/mlir/test/Conversion/TosaToStandard/tosa-to-standard.mlir
+++ b/mlir/test/Conversion/TosaToStandard/tosa-to-standard.mlir
@@ -8,3 +8,47 @@ func @const_test() -> (tensor<i32>) {
   // CHECK: return [[C3]]
   return %0 : tensor<i32>
 }
+
+// -----
+
+func @slice(%arg0: tensor<6xf32>) ->() {
+  // CHECK: [[SLICE:%.+]] = subtensor %arg0[2] [1] [1]
+  %0 = "tosa.slice"(%arg0) {start = [2], size = [1]} : (tensor<6xf32>)  -> (tensor<1xf32>)
+  return
+}
+
+// -----
+
+func @apply_scale_test(%arg0 : i32, %arg1 : i32, %arg2 : i8) -> (i32) {
+  // CHECK: [[C1_8:%.+]] = constant 1 : i8
+  // CHECK: [[C1_32:%.+]] = constant 1 : i32
+  // CHECK: [[C1_64:%.+]] = constant 1 : i64
+  // CHECK: [[SHIFT_MINUS_ONE_8:%.+]] = subi %arg2, [[C1_8]]
+
+  // CHECK: [[SHIFT_32:%.+]] = sexti %arg2 : i8 to i32
+  // CHECK: [[SHIFT_MINUS_ONE_64:%.+]] = sexti [[SHIFT_MINUS_ONE_8]] : i8 to i64
+  // CHECK: [[SHIFTED_64:%.+]] = shift_left [[C1_64]], [[SHIFT_MINUS_ONE_64]]
+
+  // CHECK: [[C0_32:%.+]] = constant 0 : i32
+  // CHECK: [[C30_32:%.+]] = constant 30 : i32
+  // CHECK: [[SECOND_BIAS:%.+]] = shift_left [[C1_32]], [[C30_32]]
+  // CHECK: [[SECOND_BIAS_64:%.+]] = sexti [[SECOND_BIAS]] : i32 to i64
+  // CHECK: [[POSITIVE_ROUND:%.+]] = addi [[SHIFTED_64]], [[SECOND_BIAS_64]]
+  // CHECK: [[NEGATIVE_ROUND:%.+]] = subi [[SHIFTED_64]], [[SECOND_BIAS_64]]
+  // CHECK: [[VALUE_NEGATIVE:%.+]] = cmpi sge, %arg0, [[C0_32]] : i32
+  // CHECK: [[DOUBLE_ROUNDED:%.+]] = select [[VALUE_NEGATIVE]], [[POSITIVE_ROUND]], [[NEGATIVE_ROUND]] : i64
+  // CHECK: [[C32_32:%.+]] = constant 32 : i32
+  // CHECK: [[IS_32BIT_SHIFT:%.+]] = cmpi sge, [[SHIFT_32]], [[C32_32]]
+  // CHECK: [[ROUND:%.+]] = select [[IS_32BIT_SHIFT]], [[DOUBLE_ROUNDED]], [[SHIFTED_64]]
+
+  // CHECK: [[VAL_64:%.+]] = sexti %arg0 : i32 to i64
+  // CHECK: [[MULTIPLY_64:%.+]] = sexti %arg1 : i32 to i64
+  // CHECK: [[SHIFT_64:%.+]] = sexti %arg2 : i8 to i64
+  // CHECK: [[SCALED:%.+]] = muli [[VAL_64]], [[MULTIPLY_64]]
+  // CHECK: [[BIASED:%.+]] = addi [[SCALED]], [[ROUND]]
+  // CHECK: [[DOWNSHIFTED:%.+]] = shift_right_signed [[BIASED]], [[SHIFT_64]]
+  // CHECK: [[TRUNCATED:%.+]] = trunci [[DOWNSHIFTED]]
+
+  %0 = "tosa.apply_scale"(%arg0, %arg1, %arg2) {double_round = true} : (i32, i32, i8) -> i32
+  return %0 : i32
+}
diff --git a/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir b/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir
index 97e4f4c37dc368089ff9a4b0b1d03efef7582652..1ebacc8ef274a35dd05c6399d1fca1e25b8af8c0 100644
--- a/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir
+++ b/mlir/test/Conversion/VectorToSCF/vector-to-loops.mlir
@@ -200,18 +200,16 @@ func @materialize_write(%M: index, %N: index, %O: index, %P: index) {
 //  FULL-UNROLL-SAME:   %[[base:[a-zA-Z0-9]+]]: index
 
 func @transfer_read_progressive(%A : memref<?x?xf32>, %base: index) -> vector<3x15xf32> {
-  // CHECK: %[[cst:.*]] = constant 7.000000e+00 : f32
   %f7 = constant 7.0: f32
-
+  // CHECK-DAG: %[[C0:.*]] = constant 0 : index
   // CHECK-DAG: %[[splat:.*]] = constant dense<7.000000e+00> : vector<15xf32>
   // CHECK-DAG: %[[alloc:.*]] = memref.alloca() : memref<3xvector<15xf32>>
-  // CHECK-DAG: %[[C0:.*]] = constant 0 : index
   // CHECK-DAG: %[[dim:.*]] = memref.dim %[[A]], %[[C0]] : memref<?x?xf32>
   // CHECK: affine.for %[[I:.*]] = 0 to 3 {
   // CHECK:   %[[add:.*]] = affine.apply #[[$MAP0]](%[[I]])[%[[base]]]
   // CHECK:   %[[cond1:.*]] = cmpi slt, %[[add]], %[[dim]] : index
   // CHECK:   scf.if %[[cond1]] {
-  // CHECK:     %[[vec_1d:.*]] = vector.transfer_read %[[A]][%[[add]], %[[base]]], %[[cst]] : memref<?x?xf32>, vector<15xf32>
+  // CHECK:     %[[vec_1d:.*]] = vector.transfer_read %[[A]][%[[add]], %[[base]]], %cst : memref<?x?xf32>, vector<15xf32>
   // CHECK:     store %[[vec_1d]], %[[alloc]][%[[I]]] : memref<3xvector<15xf32>>
   // CHECK:   } else {
   // CHECK:     store %[[splat]], %[[alloc]][%[[I]]] : memref<3xvector<15xf32>>
@@ -219,14 +217,13 @@ func @transfer_read_progressive(%A : memref<?x?xf32>, %base: index) -> vector<3x
   // CHECK: %[[vmemref:.*]] = vector.type_cast %[[alloc]] : memref<3xvector<15xf32>> to memref<vector<3x15xf32>>
   // CHECK: %[[cst:.*]] = memref.load %[[vmemref]][] : memref<vector<3x15xf32>>
 
-  // FULL-UNROLL: %[[pad:.*]] = constant 7.000000e+00 : f32
   // FULL-UNROLL: %[[VEC0:.*]] = constant dense<7.000000e+00> : vector<3x15xf32>
   // FULL-UNROLL: %[[C0:.*]] = constant 0 : index
   // FULL-UNROLL: %[[SPLAT:.*]] = constant dense<7.000000e+00> : vector<15xf32>
   // FULL-UNROLL: %[[DIM:.*]] = memref.dim %[[A]], %[[C0]] : memref<?x?xf32>
   // FULL-UNROLL: cmpi slt, %[[base]], %[[DIM]] : index
   // FULL-UNROLL: %[[VEC1:.*]] = scf.if %{{.*}} -> (vector<3x15xf32>) {
-  // FULL-UNROLL:   vector.transfer_read %[[A]][%[[base]], %[[base]]], %[[pad]] : memref<?x?xf32>, vector<15xf32>
+  // FULL-UNROLL:   vector.transfer_read %[[A]][%[[base]], %[[base]]], %cst : memref<?x?xf32>, vector<15xf32>
   // FULL-UNROLL:   vector.insert %{{.*}}, %[[VEC0]] [0] : vector<15xf32> into vector<3x15xf32>
   // FULL-UNROLL:   scf.yield %{{.*}} : vector<3x15xf32>
   // FULL-UNROLL: } else {
@@ -236,7 +233,7 @@ func @transfer_read_progressive(%A : memref<?x?xf32>, %base: index) -> vector<3x
   // FULL-UNROLL: affine.apply #[[$MAP1]]()[%[[base]]]
   // FULL-UNROLL: cmpi slt, %{{.*}}, %[[DIM]] : index
   // FULL-UNROLL: %[[VEC2:.*]] = scf.if %{{.*}} -> (vector<3x15xf32>) {
-  // FULL-UNROLL:   vector.transfer_read %[[A]][%{{.*}}, %[[base]]], %[[pad]] : memref<?x?xf32>, vector<15xf32>
+  // FULL-UNROLL:   vector.transfer_read %[[A]][%{{.*}}, %[[base]]], %cst : memref<?x?xf32>, vector<15xf32>
   // FULL-UNROLL:   vector.insert %{{.*}}, %[[VEC1]] [1] : vector<15xf32> into vector<3x15xf32>
   // FULL-UNROLL:   scf.yield %{{.*}} : vector<3x15xf32>
   // FULL-UNROLL: } else {
@@ -246,7 +243,7 @@ func @transfer_read_progressive(%A : memref<?x?xf32>, %base: index) -> vector<3x
   // FULL-UNROLL: affine.apply #[[$MAP2]]()[%[[base]]]
   // FULL-UNROLL: cmpi slt, %{{.*}}, %[[DIM]] : index
   // FULL-UNROLL: %[[VEC3:.*]] = scf.if %{{.*}} -> (vector<3x15xf32>) {
-  // FULL-UNROLL:   vector.transfer_read %[[A]][%{{.*}}, %[[base]]], %[[pad]] : memref<?x?xf32>, vector<15xf32>
+  // FULL-UNROLL:   vector.transfer_read %[[A]][%{{.*}}, %[[base]]], %cst : memref<?x?xf32>, vector<15xf32>
   // FULL-UNROLL:   vector.insert %{{.*}}, %[[VEC2]] [2] : vector<15xf32> into vector<3x15xf32>
   // FULL-UNROLL:   scf.yield %{{.*}} : vector<3x15xf32>
   // FULL-UNROLL: } else {
@@ -380,16 +377,16 @@ func @transfer_read_minor_identity(%A : memref<?x?x?x?xf32>) -> vector<3x3xf32>
 
 // CHECK-LABEL: transfer_read_minor_identity(
 //  CHECK-SAME:   %[[A:.*]]: memref<?x?x?x?xf32>) -> vector<3x3xf32>
-//       CHECK:   %[[c0:.*]] = constant 0 : index
-//       CHECK:   %[[cst:.*]] = constant 0.000000e+00 : f32
-//       CHECK:   %[[c2:.*]] = constant 2 : index
-//       CHECK:   %[[cst0:.*]] = constant dense<0.000000e+00> : vector<3xf32>
+//  CHECK-DAG:   %[[c0:.*]] = constant 0 : index
+//  CHECK-DAG:   %cst = constant 0.000000e+00 : f32
+//       CHECK-DAG:   %[[c2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[cst0:.*]] = constant dense<0.000000e+00> : vector<3xf32>
 //       CHECK:   %[[m:.*]] = memref.alloca() : memref<3xvector<3xf32>>
 //       CHECK:   %[[d:.*]] = memref.dim %[[A]], %[[c2]] : memref<?x?x?x?xf32>
 //       CHECK:   affine.for %[[arg1:.*]] = 0 to 3 {
 //       CHECK:      %[[cmp:.*]] = cmpi slt, %[[arg1]], %[[d]] : index
 //       CHECK:      scf.if %[[cmp]] {
-//       CHECK:        %[[tr:.*]] = vector.transfer_read %[[A]][%[[c0]], %[[c0]], %[[arg1]], %[[c0]]], %[[cst]] : memref<?x?x?x?xf32>, vector<3xf32>
+//       CHECK:        %[[tr:.*]] = vector.transfer_read %[[A]][%c0, %c0, %[[arg1]], %c0], %cst : memref<?x?x?x?xf32>, vector<3xf32>
 //       CHECK:        store %[[tr]], %[[m]][%[[arg1]]] : memref<3xvector<3xf32>>
 //       CHECK:      } else {
 //       CHECK:        store %[[cst0]], %[[m]][%[[arg1]]] : memref<3xvector<3xf32>>
@@ -411,8 +408,8 @@ func @transfer_write_minor_identity(%A : vector<3x3xf32>, %B : memref<?x?x?x?xf3
 // CHECK-LABEL: transfer_write_minor_identity(
 //  CHECK-SAME:   %[[A:.*]]: vector<3x3xf32>,
 //  CHECK-SAME:   %[[B:.*]]: memref<?x?x?x?xf32>)
-//       CHECK:   %[[c0:.*]] = constant 0 : index
-//       CHECK:   %[[c2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[c2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[c0:.*]] = constant 0 : index
 //       CHECK:   %[[m:.*]] = memref.alloca() : memref<3xvector<3xf32>>
 //       CHECK:   %[[cast:.*]] = vector.type_cast %[[m]] : memref<3xvector<3xf32>> to memref<vector<3x3xf32>>
 //       CHECK:   store %[[A]], %[[cast]][] : memref<vector<3x3xf32>>
diff --git a/mlir/test/Dialect/AMX/invalid.mlir b/mlir/test/Dialect/AMX/invalid.mlir
index b3a7286b526ab3dffda82abf96b7ad966efbb098..6f147cf2851e691ea27650648240cdf35a82e579 100644
--- a/mlir/test/Dialect/AMX/invalid.mlir
+++ b/mlir/test/Dialect/AMX/invalid.mlir
@@ -46,13 +46,3 @@ func @multsize() {
   // expected-error@+1 {{'amx.tile_mulf' op bad mult shape: 4 x 4 x 4}}
   %3 = amx.tile_mulf %0, %1, %2 : vector<8x8xbf16>, vector<8x8xbf16>, vector<4x4xf32>
 }
-
-// -----
-
-func @zextsize() {
-  %0 = amx.tile_zero : vector<8x8xi8>
-  %1 = amx.tile_zero : vector<8x8xi8>
-  %2 = amx.tile_zero : vector<8x8xi32>
-  // expected-error@+1 {{'amx.tile_muli' op unexpected zext length}}
-  %3 = amx.tile_muli %0, %1, %2 [true] : vector<8x8xi8>, vector<8x8xi8>, vector<8x8xi32>
-}
diff --git a/mlir/test/Dialect/AMX/legalize-for-llvm.mlir b/mlir/test/Dialect/AMX/legalize-for-llvm.mlir
index f88d83d8f311c75c9e7d2ee175bb627a006f02f9..37382b34972dca0b5e738e3a35a2f6b3183e7564 100644
--- a/mlir/test/Dialect/AMX/legalize-for-llvm.mlir
+++ b/mlir/test/Dialect/AMX/legalize-for-llvm.mlir
@@ -17,13 +17,13 @@ func @muli(%arg0: memref<?x?xi8>, %arg1: memref<?x?xi32>) {
   %1 = amx.tile_zero : vector<16x64xi8>
   %2 = amx.tile_load %arg0[%0, %0] : memref<?x?xi8> into vector<16x64xi8>
   %3 = amx.tile_load %arg1[%0, %0] : memref<?x?xi32> into vector<16x16xi32>
-  %4 = amx.tile_muli %1, %2, %3 [true, true] : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
+  %4 = amx.tile_muli %1 zext, %2 zext, %3 : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
   amx.tile_store %arg1[%0, %0], %4 : memref<?x?xi32>, vector<16x16xi32>
-  %5 = amx.tile_muli %1, %2, %3 [false, false] : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
+  %5 = amx.tile_muli %1, %2, %3 : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
   amx.tile_store %arg1[%0, %0], %5 : memref<?x?xi32>, vector<16x16xi32>
-  %6 = amx.tile_muli %1, %2, %3 [true, false] : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
+  %6 = amx.tile_muli %1 zext, %2, %3 : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
   amx.tile_store %arg1[%0, %0], %6 : memref<?x?xi32>, vector<16x16xi32>
-  %7 = amx.tile_muli %1, %2, %3 [false, true] : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
+  %7 = amx.tile_muli %1, %2 zext, %3 : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
   amx.tile_store %arg1[%0, %0], %7  : memref<?x?xi32>, vector<16x16xi32>
   return
 }
diff --git a/mlir/test/Dialect/AMX/roundtrip.mlir b/mlir/test/Dialect/AMX/roundtrip.mlir
index 98b8024c194d6a59a09c7d2b4a364f2a869568f2..93f3ea4a297776fdd0e4294da6d02dfbd76f3074 100644
--- a/mlir/test/Dialect/AMX/roundtrip.mlir
+++ b/mlir/test/Dialect/AMX/roundtrip.mlir
@@ -28,14 +28,22 @@ func @tmulf(%arg0: memref<?x?xbf16>, %arg1: memref<?x?xf32>) {
 // CHECK: %[[x:.*]] = amx.tile_load %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xi8> into vector<16x64xi8>
 // CHECK: %[[y:.*]] = amx.tile_load %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xi8> into vector<16x64xi8>
 // CHECK: %[[z:.*]] = amx.tile_load %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xi32> into vector<16x16xi32>
-// CHECK: %[[m:.*]] = amx.tile_muli %[[x]], %[[y]], %[[z]] [true, true] : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
+// CHECK: %[[m:.*]] = amx.tile_muli %[[x]] zext, %[[y]] zext, %[[z]] : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
 // CHECK: amx.tile_store %{{.*}}[%{{.*}}, %{{.*}}], %[[m]] : memref<?x?xi32>, vector<16x16xi32>
+// Verify the parsing/printing of the sign-extension annotation.
+// CHECK: amx.tile_muli %{{.*}}, %{{.*}} zext, %{{.*}}
+// CHECK: amx.tile_muli %{{.*}} zext, %{{.*}}, %{{.*}}
+// CHECK: amx.tile_muli %{{.*}}, %{{.*}}, %{{.*}}
 func @tmuli(%arg0: memref<?x?xi8>, %arg1: memref<?x?xi8>, %arg2: memref<?x?xi32>) {
   %0 = constant 0 : index
   %1 = amx.tile_load %arg0[%0, %0] : memref<?x?xi8> into vector<16x64xi8>
   %2 = amx.tile_load %arg1[%0, %0] : memref<?x?xi8> into vector<16x64xi8>
   %3 = amx.tile_load %arg2[%0, %0] : memref<?x?xi32> into vector<16x16xi32>
-  %4 = amx.tile_muli %1, %2, %3 [true, true] : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
+  %4 = amx.tile_muli %1 zext, %2 zext, %3 : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
   amx.tile_store %arg2[%0, %0], %4 : memref<?x?xi32>, vector<16x16xi32>
+  // Verify the various `zext` combinations.
+  %5 = amx.tile_muli %1, %2 zext, %3 : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
+  %6 = amx.tile_muli %1 zext, %2, %3 : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
+  %7 = amx.tile_muli %1, %2, %3 : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32>
   return
 }
diff --git a/mlir/test/Dialect/Affine/canonicalize.mlir b/mlir/test/Dialect/Affine/canonicalize.mlir
index 91547420015f7ebec58c19a12cda1e7e2fc0fcca..8ede55a9d284cad818980a4c71f043f414bfc661 100644
--- a/mlir/test/Dialect/Affine/canonicalize.mlir
+++ b/mlir/test/Dialect/Affine/canonicalize.mlir
@@ -694,3 +694,193 @@ func @compose_affine_maps_div_symbol(%A : memref<i64>, %i0 : index, %i1 : index)
   }
   return
 }
+
+// -----
+
+// CHECK: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1, s0 * s1)>
+
+// CHECK: func @deduplicate_affine_min_expressions
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index)
+func @deduplicate_affine_min_expressions(%i0: index, %i1: index) -> index {
+  // CHECK:  affine.min #[[MAP]]()[%[[I0]], %[[I1]]]
+  %0 = affine.min affine_map<()[s0, s1] -> (s0 + s1, s0 * s1, s1 + s0, s0 * s1)> ()[%i0, %i1]
+  return %0: index
+}
+
+// -----
+
+// CHECK: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1, s0 * s1)>
+
+// CHECK: func @deduplicate_affine_max_expressions
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index)
+func @deduplicate_affine_max_expressions(%i0: index, %i1: index) -> index {
+  // CHECK:  affine.max #[[MAP]]()[%[[I0]], %[[I1]]]
+  %0 = affine.max affine_map<()[s0, s1] -> (s0 + s1, s0 * s1, s1 + s0, s0 * s1)> ()[%i0, %i1]
+  return %0: index
+}
+
+// -----
+
+// CHECK-DAG: #[[MAP0:.+]] = affine_map<()[s0, s1, s2] -> (s0 * 3, 16, -s1 + s2)>
+// CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0, s1, s2] -> (-s1 + 5, 16, -s0 + s2)>
+
+// CHECK: func @merge_affine_min_ops
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index, %[[I2:.+]]: index, %[[I3:.+]]: index)
+func @merge_affine_min_ops(%i0: index, %i1: index, %i2: index, %i3: index) -> (index, index) {
+  %0 = affine.min affine_map<(d0)[s0] -> (16, d0 - s0)> (%i0)[%i1]
+
+ // CHECK: affine.min #[[MAP0]]()[%[[I2]], %[[I1]], %[[I0]]]
+  %1 = affine.min affine_map<(d0)[s0] -> (3 * s0, d0)> (%0)[%i2] // Use as dim
+ // CHECK: affine.min #[[MAP1]]()[%[[I1]], %[[I3]], %[[I0]]]
+  %2 = affine.min affine_map<(d0)[s0] -> (s0, 5 - d0)> (%i3)[%0] // Use as symbol
+
+  return %1, %2: index, index
+}
+
+// -----
+
+// CHECK: #[[MAP:.+]] = affine_map<()[s0, s1, s2] -> (s0 + 7, s1 + 16, s1 * 8, s2 + 8, s2 * 4)>
+
+// CHECK: func @merge_multiple_affine_min_ops
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index, %[[I2:.+]]: index)
+func @merge_multiple_affine_min_ops(%i0: index, %i1: index, %i2: index) -> index {
+  %0 = affine.min affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  %1 = affine.min affine_map<()[s0] -> (s0 + 8, s0 * 4)> ()[%i1]
+  // CHECK: affine.min #[[MAP]]()[%[[I2]], %[[I0]], %[[I1]]]
+  %2 = affine.min affine_map<()[s0, s1, s2] -> (s0, 7 + s1, s2)> ()[%0, %i2, %1]
+  return %2: index
+}
+
+// -----
+
+// CHECK-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 2, s1 + 16, s1 * 8)>
+
+// CHECK: func @merge_multiple_uses_of_affine_min_ops
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index)
+func @merge_multiple_uses_of_affine_min_ops(%i0: index, %i1: index) -> index {
+  %0 = affine.min affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  // CHECK: affine.min #[[MAP]]()[%[[I1]], %[[I0]]]
+  %2 = affine.min affine_map<()[s0, s1, s2] -> (s0, s1, s2 * 2)> ()[%0, %0, %i1]
+  return %2: index
+}
+
+// -----
+
+// CHECK-DAG: #[[MAP0:.+]] = affine_map<()[s0] -> (s0 + 16, s0 * 8)>
+// CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0, s1, s2] -> (s0 + 1, s1 * 2, s2 + 16, s2 * 8)>
+
+// CHECK: func @merge_mixed_uses_of_affine_min_ops
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index)
+func @merge_mixed_uses_of_affine_min_ops(%i0: index, %i1: index) -> index {
+  // CHECK: %[[AFFINE:.+]] = affine.min #[[MAP0]]()[%[[I0]]]
+  %0 = affine.min affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  // %0 is bound to a symbol that is both a standalone expression and a part
+  // of other expressions.
+  // CHECK: affine.min #[[MAP1]]()[%[[AFFINE]], %[[I1]], %[[I0]]]
+  %2 = affine.min affine_map<()[s0, s1, s2] -> (s0, s1 + 1, s2 * 2)> ()[%0, %0, %i1]
+  return %2: index
+}
+
+// -----
+
+// CHECK-LABEL: func @dont_merge_affine_min_if_not_single_dim
+func @dont_merge_affine_min_if_not_single_dim(%i0: index, %i1: index, %i2: index) -> index {
+  // CHECK-COUNT-2: affine.min
+  %0 = affine.min affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  %1 = affine.min affine_map<(d0)[s0] -> (s0 + 4, 7 + d0)> (%0)[%i2]
+  return %1: index
+}
+
+
+// -----
+
+// CHECK-LABEL: func @dont_merge_affine_min_if_not_single_sym
+func @dont_merge_affine_min_if_not_single_sym(%i0: index, %i1: index, %i2: index) -> index {
+  // CHECK-COUNT-2: affine.min
+  %0 = affine.min affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  %1 = affine.min affine_map<()[s0, s1] -> (s0 + 4, 7 + s1)> ()[%0, %i2]
+  return %1: index
+}
+
+// -----
+
+// CHECK-DAG: #[[MAP0:.+]] = affine_map<()[s0, s1, s2] -> (s0 * 3, 16, -s1 + s2)>
+// CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0, s1, s2] -> (-s1 + 5, 16, -s0 + s2)>
+
+// CHECK: func @merge_affine_max_ops
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index, %[[I2:.+]]: index, %[[I3:.+]]: index)
+func @merge_affine_max_ops(%i0: index, %i1: index, %i2: index, %i3: index) -> (index, index) {
+  %0 = affine.max affine_map<(d0)[s0] -> (16, d0 - s0)> (%i0)[%i1]
+
+ // CHECK: affine.max #[[MAP0]]()[%[[I2]], %[[I1]], %[[I0]]]
+  %1 = affine.max affine_map<(d0)[s0] -> (3 * s0, d0)> (%0)[%i2] // Use as dim
+ // CHECK: affine.max #[[MAP1]]()[%[[I1]], %[[I3]], %[[I0]]]
+  %2 = affine.max affine_map<(d0)[s0] -> (s0, 5 - d0)> (%i3)[%0] // Use as symbol
+
+  return %1, %2: index, index
+}
+
+// -----
+
+// CHECK: #[[MAP:.+]] = affine_map<()[s0, s1, s2] -> (s0 + 7, s1 + 16, s1 * 8, s2 + 8, s2 * 4)>
+
+// CHECK: func @merge_multiple_affine_max_ops
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index, %[[I2:.+]]: index)
+func @merge_multiple_affine_max_ops(%i0: index, %i1: index, %i2: index) -> index {
+  %0 = affine.max affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  %1 = affine.max affine_map<()[s0] -> (s0 + 8, s0 * 4)> ()[%i1]
+  // CHECK: affine.max #[[MAP]]()[%[[I2]], %[[I0]], %[[I1]]]
+  %2 = affine.max affine_map<()[s0, s1, s2] -> (s0, 7 + s1, s2)> ()[%0, %i2, %1]
+  return %2: index
+}
+
+// -----
+
+// CHECK-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 2, s1 + 16, s1 * 8)>
+
+// CHECK: func @merge_multiple_uses_of_affine_max_ops
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index)
+func @merge_multiple_uses_of_affine_max_ops(%i0: index, %i1: index) -> index {
+  %0 = affine.max affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  // CHECK: affine.max #[[MAP]]()[%[[I1]], %[[I0]]]
+  %2 = affine.max affine_map<()[s0, s1, s2] -> (s0, s1, s2 * 2)> ()[%0, %0, %i1]
+  return %2: index
+}
+
+// -----
+
+// CHECK-DAG: #[[MAP0:.+]] = affine_map<()[s0] -> (s0 + 16, s0 * 8)>
+// CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0, s1, s2] -> (s0 + 1, s1 * 2, s2 + 16, s2 * 8)>
+
+// CHECK: func @merge_mixed_uses_of_affine_max_ops
+// CHECK-SAME: (%[[I0:.+]]: index, %[[I1:.+]]: index)
+func @merge_mixed_uses_of_affine_max_ops(%i0: index, %i1: index) -> index {
+  // CHECK: %[[AFFINE:.+]] = affine.max #[[MAP0]]()[%[[I0]]]
+  %0 = affine.max affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  // %0 is bound to a symbol that is both a standalone expression and a part
+  // of other expressions.
+  // CHECK: affine.max #[[MAP1]]()[%[[AFFINE]], %[[I1]], %[[I0]]]
+  %2 = affine.max affine_map<()[s0, s1, s2] -> (s0, s1 + 1, s2 * 2)> ()[%0, %0, %i1]
+  return %2: index
+}
+
+// -----
+
+// CHECK-LABEL: func @dont_merge_affine_max_if_not_single_dim
+func @dont_merge_affine_max_if_not_single_dim(%i0: index, %i1: index, %i2: index) -> index {
+  // CHECK-COUNT-2: affine.max
+  %0 = affine.max affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  %1 = affine.max affine_map<(d0)[s0] -> (s0 + 4, 7 + d0)> (%0)[%i2]
+  return %1: index
+}
+
+
+// -----
+
+// CHECK-LABEL: func @dont_merge_affine_max_if_not_single_sym
+func @dont_merge_affine_max_if_not_single_sym(%i0: index, %i1: index, %i2: index) -> index {
+  // CHECK-COUNT-2: affine.max
+  %0 = affine.max affine_map<()[s0] -> (s0 + 16, s0 * 8)> ()[%i0]
+  %1 = affine.max affine_map<()[s0, s1] -> (s0 + 4, 7 + s1)> ()[%0, %i2]
+  return %1: index
+}
diff --git a/mlir/test/Dialect/Affine/memref-stride-calculation.mlir b/mlir/test/Dialect/Affine/memref-stride-calculation.mlir
index d410e9d825bc308c80cef0f7f1fb6370210ab611..d05f4e0d158e5a8eb91fb9ede2d9aa587db59a8f 100644
--- a/mlir/test/Dialect/Affine/memref-stride-calculation.mlir
+++ b/mlir/test/Dialect/Affine/memref-stride-calculation.mlir
@@ -60,7 +60,8 @@ func @f(%0: index) {
 // CHECK: MemRefType offset: 123 strides:
 
   %100 = memref.alloc(%0, %0)[%0, %0] : memref<?x?x16xf32, affine_map<(i, j, k)[M, N]->(i + j, j, k)>, affine_map<(i, j, k)[M, N]->(M * i + N * j + k + 1)>>
-// CHECK: MemRefType memref<?x?x16xf32, affine_map<(d0, d1, d2)[s0, s1] -> (d0 + d1, d1, d2)>, affine_map<(d0, d1, d2)[s0, s1] -> (d0 * s0 + d1 * s1 + d2 + 1)>> cannot be converted to strided form
+  // CHECK: MemRefType offset: 1 strides: ?, ?, 1
+
   %101 = memref.alloc() : memref<3x4x5xf32, affine_map<(i, j, k)->(i floordiv 4 + j + k)>>
 // CHECK: MemRefType memref<3x4x5xf32, affine_map<(d0, d1, d2) -> (d0 floordiv 4 + d1 + d2)>> cannot be converted to strided form
   %102 = memref.alloc() : memref<3x4x5xf32, affine_map<(i, j, k)->(i ceildiv 4 + j + k)>>
diff --git a/mlir/test/Dialect/ArmNeon/roundtrip.mlir b/mlir/test/Dialect/ArmNeon/roundtrip.mlir
index 014da313a0893f232074eb355cce82c664fde074..2252d2c857fe71f6be1b08f691fb7bb872e9f96e 100644
--- a/mlir/test/Dialect/ArmNeon/roundtrip.mlir
+++ b/mlir/test/Dialect/ArmNeon/roundtrip.mlir
@@ -3,18 +3,25 @@
 // CHECK-LABEL: arm_neon_smull
 func @arm_neon_smull(%a: vector<8xi8>, %b: vector<8xi8>)
     -> (vector<8xi16>, vector<4xi32>, vector<2xi64>) {
-  // CHECK: arm_neon.smull {{.*}}: vector<8xi8> to vector<8xi16>
-  %0 = arm_neon.smull %a, %b : vector<8xi8> to vector<8xi16>
+  // CHECK: arm_neon.intr.smull {{.*}}: vector<8xi8> to vector<8xi16>
+  %0 = arm_neon.intr.smull %a, %b : vector<8xi8> to vector<8xi16>
   %00 = vector.extract_strided_slice %0 {offsets = [3], sizes = [4], strides = [1]}:
     vector<8xi16> to vector<4xi16>
 
-  // CHECK: arm_neon.smull {{.*}}: vector<4xi16> to vector<4xi32>
-  %1 = arm_neon.smull %00, %00 : vector<4xi16> to vector<4xi32>
+  // CHECK: arm_neon.intr.smull {{.*}}: vector<4xi16> to vector<4xi32>
+  %1 = arm_neon.intr.smull %00, %00 : vector<4xi16> to vector<4xi32>
   %11 = vector.extract_strided_slice %1 {offsets = [1], sizes = [2], strides = [1]}:
     vector<4xi32> to vector<2xi32>
 
-  // CHECK: arm_neon.smull {{.*}}: vector<2xi32> to vector<2xi64>
-  %2 = arm_neon.smull %11, %11 : vector<2xi32> to vector<2xi64>
+  // CHECK: arm_neon.intr.smull {{.*}}: vector<2xi32> to vector<2xi64>
+  %2 = arm_neon.intr.smull %11, %11 : vector<2xi32> to vector<2xi64>
 
   return %0, %1, %2 : vector<8xi16>, vector<4xi32>, vector<2xi64>
 }
+
+// CHECK-LABEL: arm_neon_sdot
+func @arm_neon_sdot(%a: vector<2xi32>, %b: vector<8xi8>, %c: vector<8xi8>) -> vector<2xi32> {
+  // CHECK: arm_neon.intr.sdot {{.*}}: vector<8xi8>, vector<8xi8> to vector<2xi32>
+  %0 = arm_neon.intr.sdot %a, %b, %c : vector<8xi8>, vector<8xi8> to vector<2xi32>
+  return %0 : vector<2xi32>
+}
diff --git a/mlir/test/Dialect/DLTI/invalid.mlir b/mlir/test/Dialect/DLTI/invalid.mlir
index 9f7ff7e36c37648522ddd24185dc755a6c572ca2..aa9a713b26b84430782e309ab200e3aedd33b83f 100644
--- a/mlir/test/Dialect/DLTI/invalid.mlir
+++ b/mlir/test/Dialect/DLTI/invalid.mlir
@@ -55,7 +55,7 @@
 
 // Mismatching entries don't combine.
 "test.op_with_data_layout"() ({
-  // expected-error@below {{data layout is not a refinement of the layouts in enclosing ops}}
+  // expected-error@below {{data layout does not combine with layouts of enclosing ops}}
   // expected-note@above {{enclosing op with data layout}}
   "test.op_with_data_layout"() { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"unknown.unknown", 32>> } : () -> ()
   "test.maybe_terminator_op"() : () -> ()
@@ -71,3 +71,22 @@
 
 // expected-error@below {{data layout specified for a type that does not support it}}
 "test.op_with_data_layout"() { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!test.test_type, 32>> } : () -> ()
+
+// -----
+
+// Mismatching entries are checked on module ops as well.
+module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"unknown.unknown", 33>>} {
+  // expected-error@below {{data layout does not combine with layouts of enclosing ops}}
+  // expected-note@above {{enclosing op with data layout}}
+  module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"unknown.unknown", 32>>} {
+  }
+}
+
+// -----
+
+// Mismatching entries are checked on a combination of modules and other ops.
+module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"unknown.unknown", 33>>} {
+  // expected-error@below {{data layout does not combine with layouts of enclosing ops}}
+  // expected-note@above {{enclosing op with data layout}}
+  "test.op_with_data_layout"() { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"unknown.unknown", 32>>} : () -> ()
+}
diff --git a/mlir/test/Dialect/Linalg/bufferize.mlir b/mlir/test/Dialect/Linalg/bufferize.mlir
index e0027b765d2502ce382db54ba4595997711f1ae4..b9a4362f5e340739904afc2cbc793d380fb5e56a 100644
--- a/mlir/test/Dialect/Linalg/bufferize.mlir
+++ b/mlir/test/Dialect/Linalg/bufferize.mlir
@@ -265,3 +265,16 @@ func @bufferize_subtensor_insert(%t : tensor<?x?xf32>, %st0 : tensor<2x3xf32>, %
   return %t0, %t1: tensor<?x?xf32>, tensor<?x?xf32>
 }
 
+// -----
+
+// CHECK-LABEL: func @bufferize_fill(
+// CHECK-SAME:    %[[IN:.*]]: tensor<?xf32>
+func @bufferize_fill(%arg0: tensor<?xf32>) -> tensor<?xf32> {
+  %c0 = constant 0.0 : f32
+  // CHECK: %[[MEMREF:.*]] = memref.buffer_cast %[[IN]] : memref<?xf32>
+  // CHECK: linalg.fill(%[[MEMREF]], %cst) : memref<?xf32>, f32
+  // CHECK: %[[TENSOR:.*]] = memref.tensor_load %[[MEMREF]] : memref<?xf32>
+  // CHECK: return %[[TENSOR]]
+  %0 = linalg.fill(%arg0, %c0) : tensor<?xf32>, f32 -> tensor<?xf32>
+  return %0 : tensor<?xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/canonicalize.mlir b/mlir/test/Dialect/Linalg/canonicalize.mlir
index 693e94f6369833e87f4577881fcca1bbc24546ab..44f9dbd49cd565a00ab34dc1e06bbf1d6dcf0557 100644
--- a/mlir/test/Dialect/Linalg/canonicalize.mlir
+++ b/mlir/test/Dialect/Linalg/canonicalize.mlir
@@ -802,3 +802,62 @@ func @self_copy(%arg0 : memref<2x3x?x4xf32>) {
 //   CHECK: return
   return
 }
+
+// -----
+
+// CHECK-LABEL: func @fold_fill_reshape()
+func @fold_fill_reshape() -> tensor<6x4xf32> {
+  %zero = constant 0.0 : f32
+  // CHECK: %[[INIT:.+]] = linalg.init_tensor [6, 4] : tensor<6x4xf32>
+  %init = linalg.init_tensor [1, 2, 3, 4] : tensor<1x2x3x4xf32>
+  // CHECK: %[[FILL:.+]] = linalg.fill(%[[INIT]], %cst) : tensor<6x4xf32>, f32 -> tensor<6x4xf32>
+  %fill = linalg.fill(%init, %zero) : tensor<1x2x3x4xf32>, f32 -> tensor<1x2x3x4xf32>
+  %reshape = linalg.tensor_reshape %fill [
+    affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>,
+    affine_map<(d0, d1, d2, d3) -> (d3)>] : tensor<1x2x3x4xf32> into tensor<6x4xf32>
+  // CHECK: return %[[FILL]] : tensor<6x4xf32>
+  return %reshape : tensor<6x4xf32>
+}
+
+// -----
+
+#map0 = affine_map<(d0) -> (24, -d0 + 192)>
+#map1 = affine_map<(d0, d1)[s0] -> (d0 * 192 + s0 + d1)>
+#map2 = affine_map<(d0) -> (16, -d0 + 192)>
+
+func private @foo(%A: memref<192x192xf32>, %B: memref<192x192xf32>,
+                  %C: memref<192x192xf32>) -> ()
+
+func @fold_tiled_loop_results(%A: memref<192x192xf32>, %B: memref<192x192xf32>,
+                              %C: memref<192x192xf32>,
+                              %C_tensor: tensor<192x192xf32>) {
+  %cst = constant 0.000000e+00 : f32
+  %c24 = constant 24 : index
+  %c16 = constant 16 : index
+  %c0 = constant 0 : index
+  %c192 = constant 192 : index
+  %useless = linalg.tiled_loop (%i, %j) = (%c0, %c0) to (%c192, %c192)
+      step (%c24, %c16)
+      ins (%A, %B: memref<192x192xf32>, memref<192x192xf32>)
+      outs (%C_tensor, %C :tensor<192x192xf32>, memref<192x192xf32>) {
+        call @foo(%A, %B, %C) : (memref<192x192xf32>, memref<192x192xf32>, memref<192x192xf32>)-> ()
+    linalg.yield %C_tensor : tensor<192x192xf32>
+  }
+  return
+}
+
+// CHECK-LABEL: func @fold_tiled_loop_results(
+// CHECK-SAME:    %[[A:.*]]: [[TY:.*]], %[[B:.*]]: [[TY]], %[[C:.*]]: [[TY]],
+// CHECK-SAME:    %[[C_TENSOR:.*]]: tensor<{{.*}}>) {
+// CHECK:  %[[C24:.*]] = constant 24 : index
+// CHECK:  %[[C16:.*]] = constant 16 : index
+// CHECK:  %[[C0:.*]] = constant 0 : index
+// CHECK:  %[[C192:.*]] = constant 192 : index
+
+// CHECK-NOT: %{{.*}} = linalg.tiled_loop
+// CHECK:  linalg.tiled_loop (%{{.*}}, %{{.*}}) = (%[[C0]], %[[C0]])
+// CHECK-SAME: to (%[[C192]], %[[C192]]) step (%[[C24]], %[[C16]])
+// CHECK-SAME: ins (%[[A]], %[[B]]: memref<192x192xf32>, memref<192x192xf32>)
+// CHECK-SAME: outs (%[[C]]:memref<192x192xf32>) {
+// CHECK-NEXT:   call @foo(%[[A]], %[[B]], %[[C]])
+// CHECK-NEXT:   linalg.yield
diff --git a/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir b/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir
index d40a91667500a9d3f454b59c5f4c704551351b19..cb5d1089eb85d0f8cd558e873218f76cff1f621f 100644
--- a/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir
+++ b/mlir/test/Dialect/Linalg/drop-unit-extent-dims.mlir
@@ -354,3 +354,39 @@ func @fold_unit_dim_tensor_reshape_op(%arg0 : tensor<5xf32>) -> tensor<2x5xf32>
 // CHECK-LABEL: func @fold_unit_dim_tensor_reshape_op
 //       CHECK:   %[[RESULT:.+]] = linalg.generic
 //       CHECK:   return %[[RESULT]]
+
+// -----
+
+func @fold_unit_dim_for_init_tensor(%input: tensor<1x1000xf32>) -> tensor<1xf32> {
+  %cst = constant 0.0 : f32
+  %init = linalg.init_tensor [1] : tensor<1xf32>
+  %fill = linalg.fill(%init, %cst) : tensor<1xf32>, f32 -> tensor<1xf32>
+  %add = linalg.generic {
+      indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>],
+      iterator_types = ["parallel", "reduction"]}
+    ins(%input : tensor<1x1000xf32>)outs(%fill : tensor<1xf32>) {
+  ^bb0(%arg1: f32, %arg2: f32):
+    %1823 = addf %arg1, %arg2 : f32
+    linalg.yield %1823 : f32
+  } -> tensor<1xf32>
+  return %add : tensor<1xf32>
+}
+
+
+//   CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1) -> (d0, d1)>
+//   CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0) -> (d0)>
+//   CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0) -> ()>
+
+//       CHECK: func @fold_unit_dim_for_init_tensor
+
+
+//       CHECK: %[[INPUT_RESHAPE:.+]] = linalg.tensor_reshape %{{.+}} [#[[MAP0]]] : tensor<1x1000xf32> into tensor<1000xf32>
+//       CHECK: %[[INIT:.+]] = linalg.init_tensor [] : tensor<f32>
+//       CHECK: %[[FILL:.+]] = linalg.fill(%[[INIT]], %cst) : tensor<f32>, f32 -> tensor<f32>
+//       CHECK: %[[GENERIC:.+]] = linalg.generic
+//  CHECK-SAME:     indexing_maps = [#[[MAP1]], #[[MAP2]]]
+//  CHECK-SAME:     iterator_types = ["reduction"]
+//  CHECK-SAME:   ins(%[[INPUT_RESHAPE]] : tensor<1000xf32>)
+//  CHECK-SAME:   outs(%[[FILL]] : tensor<f32>)
+//       CHECK: %[[GENERIC_RESHAPE:.+]] = linalg.tensor_reshape %[[GENERIC]] [] : tensor<f32> into tensor<1xf32>
+//       CHECK: return %[[GENERIC_RESHAPE:.+]] : tensor<1xf32>
diff --git a/mlir/test/Dialect/Linalg/fusion-pattern.mlir b/mlir/test/Dialect/Linalg/fusion-pattern.mlir
index aefeeb5e3adad9df6f390d73543bd07670a8e29e..0f4e3543a7f379b948679b41fbdf1953922eb0bf 100644
--- a/mlir/test/Dialect/Linalg/fusion-pattern.mlir
+++ b/mlir/test/Dialect/Linalg/fusion-pattern.mlir
@@ -16,6 +16,8 @@ module {
 //  CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
 //  CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0)[s0] -> (64, -d0 + s0)>
 //  CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0)[s0] -> (16, -d0 + s0)>
+//  CHECK-DAG: #[[MAP4:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 32, -d0 + s1)>
+//  CHECK-DAG: #[[MAP5:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 64, -d0 + s1)>
 //      CHECK: func @basic_fusion
 // CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: memref<?x?xf32>
 // CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]+]]: memref<?x?xf32>
@@ -47,8 +49,10 @@ module {
 //      CHECK:     %[[TILE_N_2:.+]] = affine.min #[[MAP2]](%[[IV1]])[%[[N_2]]]
 //      CHECK:     %[[SV3:.+]] = memref.subview %[[ARG2]][%[[IV0]], %[[IV1]]]
 // CHECK-SAME:       [%[[TILE_M_2]], %[[TILE_N_2]]]
+//      CHECK:     %[[TILE_M_3:.+]] = affine.min #[[MAP4]](%[[IV0]])[%[[M_2]], %[[M]]]
+//      CHECK:     %[[TILE_N_3:.+]] = affine.min #[[MAP5]](%[[IV1]])[%[[N_2]], %[[N]]]
 //      CHECK:     %[[SV3_2:.+]] = memref.subview %[[ARG2]][%[[IV0]], %[[IV1]]]
-// CHECK-SAME:       [%[[TILE_M]], %[[TILE_N]]]
+// CHECK-SAME:       [%[[TILE_M_3]], %[[TILE_N_3]]]
 //      CHECK:     linalg.fill(%[[SV3_2]], %[[CST]])
 // CHECK-SAME:       __internal_linalg_transform__ = "after_basic_fusion_producer"
 //      CHECK:     scf.for %[[IV2:.+]] = %[[C0]] to %[[K]] step %[[C16]] {
@@ -86,6 +90,7 @@ module {
 //  CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
 //  CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0)[s0] -> (32, -d0 + s0)>
 //  CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0)[s0] -> (16, -d0 + s0)>
+//  CHECK-DAG: #[[MAP4:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 64, -d0 + s1)>
 //      CHECK: func @rhs_fusion
 // CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: memref<?x?xf32>
 // CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]+]]: memref<?x?xf32>
@@ -112,10 +117,13 @@ module {
 //      CHECK:     %[[SV2:.+]] = memref.subview %[[ARG3]][0, %[[IV0]]]
 // CHECK-SAME:       [%[[M]], %[[TILE_N_2]]]
 //      CHECK:     %[[K_2:.+]] = memref.dim %[[ARG1]], %[[C0]]
+//      CHECK:     %[[N_3:.+]] = memref.dim %[[ARG1]], %[[C1]]
+//      CHECK:     %[[TILE_N_3:.+]] = affine.min #[[MAP4]](%[[IV0]])[%[[N_3]], %[[N]]]
 //      CHECK:     %[[SV3:.+]] = memref.subview %[[ARG1]][0, %[[IV0]]]
-// CHECK-SAME:       [%[[K_2]], %[[TILE_N]]]
+// CHECK-SAME:       [%[[K_2]], %[[TILE_N_3]]]
+//      CHECK:     %[[TILE_N_4:.+]] = affine.min #[[MAP4]](%[[IV0]])[%[[N]], %[[N]]]
 //      CHECK:     %[[SV3_2:.+]] = memref.subview %[[ARG2]][0, %[[IV0]]]
-// CHECK-SAME:       [%[[K_2]], %[[TILE_N]]]
+// CHECK-SAME:       [%[[K]], %[[TILE_N_4]]]
 //      CHECK:     linalg.copy(%[[SV3]], %[[SV3_2]])
 // CHECK-SAME:       __internal_linalg_transform__ = "after_rhs_fusion_producer"
 //  CHECK-NOT:     linalg.fill
@@ -164,6 +172,7 @@ module {
 //  CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
 //  CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0)[s0] -> (16, -d0 + s0)>
 //  CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0)[s0] -> (64, -d0 + s0)>
+//  CHECK-DAG: #[[MAP4:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 32, -d0 + s1)>
 //      CHECK: func @two_operand_fusion
 // CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: memref<?x?xf32>
 // CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]+]]: memref<?x?xf32>
@@ -191,13 +200,17 @@ module {
 //      CHECK:     %[[N:.+]] = memref.dim %[[ARG3]], %[[C1]]
 //      CHECK:     %[[SV2:.+]] = memref.subview %[[ARG3]][%[[IV0]], 0]
 // CHECK-SAME:       [%[[TILE_M_2]], %[[N]]]
+//      CHECK:     %[[TILE_M_3:.+]] = affine.min #[[MAP4]](%[[IV0]])[%[[M_2]], %[[M]]]
 //      CHECK:     %[[SV2_2:.+]] = memref.subview %[[ARG3]][%[[IV0]], 0]
-// CHECK-SAME:       [%[[TILE_M]], %[[N]]]
-//      CHECK:     %[[K_2:.+]] = memref.dim %[[ARG0]], %[[C1]]
+// CHECK-SAME:       [%[[TILE_M_3]], %[[N]]]
+//      CHECK:     %[[M_3:.+]] = memref.dim %[[ARG0]], %[[C0]]
+//      CHECK:     %[[TILE_M_4:.+]] = affine.min #[[MAP4]](%[[IV0]])[%[[M_3]], %[[M]]]
+//      CHECK:     %[[K_3:.+]] = memref.dim %[[ARG0]], %[[C1]]
 //      CHECK:     %[[SV3:.+]] = memref.subview %[[ARG0]][%[[IV0]], 0]
-// CHECK-SAME:       [%[[TILE_M]], %[[K_2]]]
+// CHECK-SAME:       [%[[TILE_M_4]], %[[K_3]]]
+//      CHECK:     %[[TILE_M_5:.+]] = affine.min #[[MAP4]](%[[IV0]])[%[[M]], %[[M]]]
 //      CHECK:     %[[SV3_2:.+]] = memref.subview %[[ARG1]][%[[IV0]], 0]
-// CHECK-SAME:       [%[[TILE_M]], %[[K_2]]]
+// CHECK-SAME:       [%[[TILE_M_5]], %[[K]]]
 //      CHECK:     linalg.copy(%[[SV3]], %[[SV3_2]])
 // CHECK-SAME:       __internal_linalg_transform__ = "after_two_operand_fusion_producer"
 //      CHECK:     linalg.fill(%[[SV2_2]], %[[CST]])
@@ -246,6 +259,7 @@ module {
 //  CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
 //  CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0)[s0] -> (16, -d0 + s0)>
 //  CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0)[s0] -> (64, -d0 + s0)>
+//  CHECK-DAG: #[[MAP4:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 32, -d0 + s1)>
 //      CHECK: func @matmul_fusion
 // CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: memref<?x?xf32>
 // CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]+]]: memref<?x?xf32>
@@ -271,23 +285,24 @@ module {
 //      CHECK:     %[[N:.+]] = memref.dim %[[ARG4]], %[[C1]]
 //      CHECK:     %[[SV2:.+]] = memref.subview %[[ARG4]][%[[IV0]], 0]
 // CHECK-SAME:       [%[[TILE_M_2]], %[[N]]]
-//      CHECK:     %[[K2_2:.+]] = memref.dim %[[ARG1]], %[[C1]]
+//      CHECK:     %[[M_3:.+]] = memref.dim %[[ARG0]], %[[C0]]
+//      CHECK:     %[[TILE_M_3:.+]] = affine.min #[[MAP4]](%[[IV0]])[%[[M_3]], %[[M]]]
 //      CHECK:     %[[K1:.+]] = memref.dim %[[ARG0]], %[[C1]]
 //      CHECK:     %[[SV3:.+]] = memref.subview %[[ARG0]][%[[IV0]], 0]
-// CHECK-SAME:       [%[[TILE_M]], %[[K1]]]
-//      CHECK:     %[[SV4:.+]] = memref.subview %[[ARG1]][0, 0] [%[[K1]], %[[K2_2]]]
+// CHECK-SAME:       [%[[TILE_M_3]], %[[K1]]]
+//      CHECK:     %[[TILE_M_4:.+]] = affine.min #[[MAP4]](%[[IV0]])[%[[M]], %[[M]]]
 //      CHECK:     %[[SV1_2:.+]] = memref.subview %[[ARG2]][%[[IV0]], 0]
-// CHECK-SAME:       [%[[TILE_M]], %[[K2_2]]]
+// CHECK-SAME:       [%[[TILE_M_4]], %[[K2]]]
 //      CHECK:     linalg.matmul
 // CHECK-SAME:         __internal_linalg_transform__ = "after_lhs_fusion_producer"
-// CHECK-SAME:         ins(%[[SV3]], %[[SV4]]
-// CHECK-SAME:           : memref<?x?xf32, #[[MAP1]]>, memref<?x?xf32, #[[MAP1]]>)
+// CHECK-SAME:         ins(%[[SV3]], %[[ARG1]]
+// CHECK-SAME:           : memref<?x?xf32, #[[MAP1]]>, memref<?x?xf32>)
 // CHECK-SAME:         outs(%[[SV1_2]] : memref<?x?xf32, #[[MAP1]]>)
-//  CHECK-DAG:     %[[N_2:.+]] = memref.dim %[[ARG3]], %[[C1]]
+//      CHECK:     %[[N_2:.+]] = memref.dim %[[ARG3]], %[[C1]]
 //      CHECK:     scf.parallel (%[[IV1:.+]]) =
 // CHECK-SAME:       (%[[C0]]) to (%[[N_2]]) step (%[[C64]]) {
-// CHECK-NEXT:       scf.for %[[IV2:.+]] = %[[C0]] to %[[K]] step %[[C16]] {
-//      CHECK:         %[[TILE_K:.+]] = affine.min #[[MAP2]](%[[IV2]])[%[[K]]]
+// CHECK-NEXT:       scf.for %[[IV2:.+]] = %[[C0]] to %[[K2]] step %[[C16]] {
+//      CHECK:         %[[TILE_K:.+]] = affine.min #[[MAP2]](%[[IV2]])[%[[K2]]]
 //      CHECK:         %[[SV6:.+]] = memref.subview %[[SV1]][0, %[[IV2]]]
 // CHECK-SAME:           [%[[TILE_M]], %[[TILE_K]]]
 //      CHECK:         %[[K_2:.+]] = memref.dim %[[ARG3]], %[[C0]]
@@ -348,10 +363,11 @@ module {
 //       CHECK:     %[[T6:.+]] = memref.subview %[[ARG2]][%[[ARG3]], %[[ARG4]]]
 //       CHECK:     %[[T8:.+]] = memref.subview %[[ARG0]][%[[ARG3]], 0]
 //       CHECK:     %[[T9:.+]] = memref.subview %[[ARG1]][0, %[[ARG4]]]
+//       CHECK:     %[[T10:.+]] = memref.subview %[[T2]][%[[ARG3]], %[[ARG4]]]
 //       CHECK:     linalg.matmul
 //  CHECK-SAME:       after_transpose_fusion_producer
 //  CHECK-SAME:       ins(%[[T8]], %[[T9]]
-//  CHECK-SAME:       outs(%[[T5]]
+//  CHECK-SAME:       outs(%[[T10]]
 //   CHECK-NOT:     linalg.matmul
 //       CHECK:     linalg.generic
 //  CHECK-SAME:       ins(%[[T5]], %[[T5]]
diff --git a/mlir/test/Dialect/Linalg/fusion-sequence.mlir b/mlir/test/Dialect/Linalg/fusion-sequence.mlir
index bec19b325a7bffcca64824f91d8bccdf8cfa1859..3321595cf9a4b97f91e9529b4eb06c565b41e6c4 100644
--- a/mlir/test/Dialect/Linalg/fusion-sequence.mlir
+++ b/mlir/test/Dialect/Linalg/fusion-sequence.mlir
@@ -36,18 +36,19 @@ module {
 //  CHECK-SAME:   %[[ARG3:[a-zA-Z0-9_]+]]: memref<?x?xf32>
 //       CHECK:   %[[TEMP:.+]] = memref.alloc(%{{.*}}, %{{.*}}) : memref<?x?xf32>
 //       CHECK:   scf.parallel (%[[IV0:.+]], %[[IV1:.+]]) = {{.*}} {
-//   CHECK-DAG:     %[[SV_TEMP:.+]] = memref.subview %[[TEMP]][%[[IV0]], %[[IV1]]]
+//       CHECK:     %[[SV_TEMP_1:.+]] = memref.subview %[[TEMP]][%[[IV0]], %[[IV1]]]
 //   CHECK-DAG:     %[[SV_ARG2:.+]] = memref.subview %[[ARG2]][%[[IV1]]]
 //   CHECK-DAG:     %[[SV_ARG3:.+]] = memref.subview %[[ARG3]][%[[IV0]], %[[IV1]]]
 //   CHECK-DAG:     %[[SV_ARG0:.+]] = memref.subview %[[ARG0]][%[[IV0]], 0]
 //   CHECK-DAG:     %[[SV_ARG1:.+]] = memref.subview %[[ARG1]][0, %[[IV1]]]
-//       CHECK:     linalg.fill(%[[SV_TEMP]], %{{.+}})
+//       CHECK:     %[[SV_TEMP_2:.+]] = memref.subview %[[TEMP]][%[[IV0]], %[[IV1]]]
+//       CHECK:     linalg.fill(%[[SV_TEMP_2]], %{{.+}})
 //       CHECK:     linalg.matmul
 //  CHECK-SAME:       ins(%[[SV_ARG0]], %[[SV_ARG1]]
 //  CHECK-SAME:         : memref<?x?xf32, #[[MAP2]]>, memref<?x?xf32, #[[MAP2]]>)
-//  CHECK-SAME:       outs(%[[SV_TEMP]] : memref<?x?xf32, #[[MAP2]]>)
+//  CHECK-SAME:       outs(%[[SV_TEMP_2]] : memref<?x?xf32, #[[MAP2]]>)
 //       CHECK:     linalg.generic
-//  CHECK-SAME:       ins(%[[SV_TEMP]], %[[SV_ARG2]]
+//  CHECK-SAME:       ins(%[[SV_TEMP_1]], %[[SV_ARG2]]
 //  CHECK-SAME:         : memref<?x?xf32, #[[MAP2]]>, memref<?xf32, #[[MAP3]]>)
 //  CHECK-SAME:       outs(%[[SV_ARG3]] : memref<?x?xf32, #[[MAP2]]>)
 //       CHECK:     scf.yield
@@ -83,6 +84,10 @@ module {
 
 //   CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0)[s0] -> (16, -d0 + s0)>
 //   CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
+//   CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 16, -d0 + s1)>
+//   CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0)[s0] -> (-d0 + s0, 16)>
+
+
 //       CHECK: func @sequence_of_matmul
 //  CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: memref<?x?xf32>
 //  CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]+]]: memref<?x?xf32>
@@ -100,37 +105,41 @@ module {
 //       CHECK:   scf.parallel (%[[IV0:.+]]) = (%[[C0]]) to (%[[M]])
 //  CHECK-SAME:     step (%[[C16]]) {
 //       CHECK:     %[[TILE_M:.+]] = affine.min #[[MAP0]](%[[IV0]])[%[[M]]]
-//       CHECK:     %[[SV_ALLOC2:.+]] = memref.subview %[[ALLOC2]][%[[IV0]], 0]
+//       CHECK:     %[[SV_ALLOC3:.+]] = memref.subview %[[ALLOC2]][%[[IV0]], 0]
 //  CHECK-SAME:       [%[[TILE_M]], %[[N2]]]
 //       CHECK:     %[[M_2:.+]] = memref.dim %[[ARG4]], %[[C0]]
 //       CHECK:     %[[TILE_M_2:.+]] = affine.min #[[MAP0]](%[[IV0]])[%[[M_2]]]
 //       CHECK:     %[[N3:.+]] = memref.dim %[[ARG4]], %[[C1]]
 //       CHECK:     %[[SV_ARG4:.+]] = memref.subview %[[ARG4]][%[[IV0]], 0]
 //  CHECK-SAME:       [%[[TILE_M_2]], %[[N3]]]
+//       CHECK:     %[[TILE_M_3:.+]] = affine.min #[[MAP2]](%[[IV0]])[%[[M_2]], %[[M]]]
 //       CHECK:     %[[SV_ARG4_2:.+]] = memref.subview %[[ARG4]][%[[IV0]], 0]
-//  CHECK-SAME:       [%[[TILE_M]], %[[N3]]]
+//  CHECK-SAME:       [%[[TILE_M_3]], %[[N3]]]
+//       CHECK:     %[[TILE_M_4:.+]] = affine.min #[[MAP3]](%[[IV0]])[%[[M]]]
 //       CHECK:     %[[SV_ALLOC1:.+]] = memref.subview %[[ALLOC1]][%[[IV0]], 0]
-//  CHECK-SAME:       [%[[TILE_M]], %[[N1]]]
-//       CHECK:     %[[SV_ARG2:.+]] = memref.subview %[[ARG2]][0, 0] [%[[N1]], %[[N2]]]
+//  CHECK-SAME:       [%[[TILE_M_4]], %[[N1]]]
+//       CHECK:     %[[SV_ALLOC2:.+]] = memref.subview %[[ALLOC2]][%[[IV0]], 0]
+//  CHECK-SAME:       [%[[TILE_M_4]], %[[N2]]]
+//       CHECK:     %[[TILE_M_5:.+]] = affine.min #[[MAP2]](%[[IV0]])[%[[M]], %[[M]]]
 //       CHECK:     %[[N0:.+]] = memref.dim %[[ARG0]], %[[C1]]
 //       CHECK:     %[[SV_ARG0:.+]] = memref.subview %[[ARG0]][%[[IV0]], 0]
-//  CHECK-SAME:       [%[[TILE_M:.+]], %[[N0]]]
-//       CHECK:     %[[SV_ARG1:.+]] = memref.subview %[[ARG1]][0, 0] [%[[N0]], %[[N1]]]
+//  CHECK-SAME:       [%[[TILE_M_5]], %[[N0]]]
 //       CHECK:     linalg.fill(%[[SV_ALLOC1]], %{{.+}})
-//       CHECK:     linalg.matmul ins(%[[SV_ARG0]], %[[SV_ARG1]]
-//  CHECK-SAME:        : memref<?x?xf32, #[[MAP1]]>, memref<?x?xf32, #[[MAP1]]>)
+//       CHECK:     linalg.matmul ins(%[[SV_ARG0]], %[[ARG1]]
+//  CHECK-SAME:        : memref<?x?xf32, #[[MAP1]]>, memref<?x?xf32>)
 //  CHECK-SAME:        outs(%[[SV_ALLOC1]] : memref<?x?xf32, #[[MAP1]]>)
 //       CHECK:     linalg.fill(%[[SV_ALLOC2]], %{{.+}})
-//       CHECK:     linalg.matmul ins(%[[SV_ALLOC1]], %[[SV_ARG2]]
-//  CHECK-SAME:        : memref<?x?xf32, #[[MAP1]]>, memref<?x?xf32, #[[MAP1]]>)
+//       CHECK:     linalg.matmul ins(%[[SV_ALLOC1]], %[[ARG2]]
+//  CHECK-SAME:        : memref<?x?xf32, #[[MAP1]]>, memref<?x?xf32>)
 //  CHECK-SAME:        outs(%[[SV_ALLOC2]] : memref<?x?xf32, #[[MAP1]]>)
 //       CHECK:     linalg.fill(%[[SV_ARG4_2]], %{{.+}})
-//       CHECK:     linalg.matmul ins(%[[SV_ALLOC2]], %[[ARG3]]
+//       CHECK:     linalg.matmul ins(%[[SV_ALLOC3]], %[[ARG3]]
 //  CHECK-SAME:        : memref<?x?xf32, #[[MAP1]]>, memref<?x?xf32>)
 //  CHECK-SAME:        outs(%[[SV_ARG4]] : memref<?x?xf32, #[[MAP1]]>)
 //       CHECK:     scf.yield
 //       CHECK:   }
 
+
 // -----
 
 module {
@@ -189,8 +198,8 @@ module {
 module {
   func @tensor_matmul_fusion(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>,
                              %arg2: tensor<?x?xf32>, %arg3: tensor<?x?xf32>,
-			     %arg4: tensor<?x?xf32>, %arg5: tensor<?x?xf32>,
-			     %arg6: tensor<?x?xf32>) -> tensor<?x?xf32> {
+           %arg4: tensor<?x?xf32>, %arg5: tensor<?x?xf32>,
+           %arg6: tensor<?x?xf32>) -> tensor<?x?xf32> {
     %0 = linalg.matmul ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
       outs(%arg2 : tensor<?x?xf32>) -> tensor<?x?xf32> // [M, N0] * [N0, N1]
     %1 = linalg.matmul ins(%0, %arg3 : tensor<?x?xf32>, tensor<?x?xf32>)
@@ -200,7 +209,11 @@ module {
     return %2 : tensor<?x?xf32>
   }
 }
-// CHECK-LABEL: func @tensor_matmul_fusion(
+
+//       CHECK: #[[MAP0:.+]] = affine_map<(d0, d1) -> (16, d0 - d1)>
+//       CHECK: #[[MAP1:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 16, -d0 + s1)>
+
+//       CHECK: func @tensor_matmul_fusion(
 //  CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
 //  CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
 //  CHECK-SAME:   %[[ARG2:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
@@ -210,36 +223,38 @@ module {
 //  CHECK-SAME:   %[[ARG6:[a-zA-Z0-9_]+]]: tensor<?x?xf32>) -> tensor<?x?xf32> {
 //   CHECK-DAG:   %[[C0:.+]] = constant 0 : index
 //   CHECK-DAG:   %[[C1:.+]] = constant 1 : index
+//       CHECK:   %[[M:.+]] = memref.dim %[[ARG0]], %c0 : tensor<?x?xf32>
 //       CHECK:   %[[R0:.+]] = scf.for %[[IV0:[a-zA-Z0-9_]+]] =
 //  CHECK-SAME:     iter_args(%[[ARG8:.+]] = %[[ARG6]]) -> (tensor<?x?xf32>) {
-//       CHECK:       %[[N3:.+]] = memref.dim %[[ARG8]], %[[C1]]
-//       CHECK:       %[[STARG6:.+]] = subtensor %[[ARG8]][%[[IV0]], 0]
-//  CHECK-SAME:         [%{{[a-zA-Z0-9_]+}}, %[[N3]]]
-//       CHECK:       %[[N2:.+]] = memref.dim %[[ARG3]], %[[C1]]
-//       CHECK:       %[[N1:.+]] = memref.dim %[[ARG1]], %[[C1]]
-//       CHECK:       %[[STARG3:.+]] = subtensor %[[ARG3]][0, 0]
-//  CHECK-SAME:         [%[[N1]], %[[N2]]]
-//       CHECK:       %[[STARG4:.+]] = subtensor %[[ARG4]][%[[IV0]], 0]
-//  CHECK-SAME:         [%{{[a-zA-Z0-9_]+}}, %[[N2]]]
-//       CHECK:       %[[N0:.+]] = memref.dim %[[ARG0]], %[[C1]]
-//       CHECK:       %[[STARG0:.+]] = subtensor %[[ARG0]][%[[IV0]], 0]
-//  CHECK-SAME:         [%{{[a-zA-Z0-9_]+}}, %[[N0]]]
-//       CHECK:       %[[STARG1:.+]] = subtensor %[[ARG1]][0, 0]
-//  CHECK-SAME:         [%[[N0]], %[[N1]]]
-//       CHECK:       %[[STARG2:.+]] = subtensor %[[ARG2]][%[[IV0]], 0]
-//  CHECK-SAME:         [%{{[a-zA-Z0-9_]+}}, %[[N1]]]
-//       CHECK:       %[[T0:.+]] = linalg.matmul
-//  CHECK-SAME:         ins(%[[STARG0]], %[[STARG1]]
-//  CHECK-SAME:         ) outs(%[[STARG2]] : tensor<?x?xf32>)
-//       CHECK:       %[[T1:.+]] = linalg.matmul
-//  CHECK-SAME:         ins(%[[T0]], %[[STARG3]]
-//  CHECK-SAME:         ) outs(%[[STARG4]] : tensor<?x?xf32>)
-//       CHECK:       %[[T2:.+]] = linalg.matmul
-//  CHECK-SAME:         ins(%[[T1]], %[[ARG5]]
-//  CHECK-SAME:         ) outs(%[[STARG6]] : tensor<?x?xf32>)
-//       CHECK:       %[[R1:.+]] = subtensor_insert %[[T2]]
-//  CHECK-SAME:         into %[[ARG8]][%[[IV0]], 0]
-//       CHECK:       scf.yield %[[R1]]
-//       CHECK:     }
-//       CHECK:     return %[[R0]]
+//       CHECK:     %[[M_1:.+]] = memref.dim %[[ARG8]], %[[C0]]
+//       CHECK:     %[[TILE_M_1:.+]] = affine.min #[[MAP0]](%[[M_1]], %[[IV0]])
+//       CHECK:     %[[N3:.+]] = memref.dim %[[ARG8]], %[[C1]]
+//       CHECK:     %[[STARG6:.+]] = subtensor %[[ARG8]][%[[IV0]], 0]
+//  CHECK-SAME:       [%[[TILE_M_1]], %[[N3]]]
+//       CHECK:     %[[M_2:.+]] = memref.dim %[[ARG4]], %[[C0]]
+//       CHECK:     %[[TILE_M_2:.+]] = affine.min #[[MAP1]](%[[IV0]])[%[[M_2]], %[[M]]]
+//       CHECK:     %[[N2:.+]] = memref.dim %[[ARG4]], %[[C1]]
+//       CHECK:     %[[STARG4:.+]] = subtensor %[[ARG4]][%[[IV0]], 0]
+//  CHECK-SAME:       [%[[TILE_M_2]], %[[N2]]]
+//       CHECK:     %[[TILE_M_3:.+]] = affine.min #[[MAP1]](%[[IV0]])[%[[M]], %[[M]]]
+//       CHECK:     %[[N0:.+]] = memref.dim %[[ARG0]], %[[C1]]
+//       CHECK:     %[[STARG0:.+]] = subtensor %[[ARG0]][%[[IV0]], 0]
+//  CHECK-SAME:       [%[[TILE_M_3]], %[[N0]]]
+//       CHECK:     %[[M_3:.+]] = memref.dim %[[ARG2]], %[[C0]]
+//       CHECK:     %[[TILE_M_4:.+]] = affine.min #[[MAP1]](%[[IV0]])[%[[M_3]], %[[M]]]
+//       CHECK:     %[[N1:.+]] = memref.dim %[[ARG2]], %[[C1]]
+//       CHECK:     %[[STARG2:.+]] = subtensor %[[ARG2]][%[[IV0]], 0]
+//  CHECK-SAME:       [%[[TILE_M_4]], %[[N1]]]
+//       CHECK:     %[[T0:.+]] = linalg.matmul
+//  CHECK-SAME:       ins(%[[STARG0]], %[[ARG1]] : tensor<?x?xf32>, tensor<?x?xf32>
+//  CHECK-SAME:       ) outs(%[[STARG2]] : tensor<?x?xf32>)
+//       CHECK:     %[[T1:.+]] = linalg.matmul
+//  CHECK-SAME:       ins(%[[T0]], %arg3 : tensor<?x?xf32>, tensor<?x?xf32>
+//  CHECK-SAME:       ) outs(%[[STARG4]] : tensor<?x?xf32>)
+//       CHECK:     %[[T2:.+]] = linalg.matmul
+//  CHECK-SAME:       ins(%[[T1]], %arg5 : tensor<?x?xf32>, tensor<?x?xf32>
+//  CHECK-SAME:       ) outs(%[[STARG6]] : tensor<?x?xf32>)
+//       CHECK:     %[[R1:.+]] = subtensor_insert %[[T2]]
+//  CHECK-SAME:       into %[[ARG8]][%[[IV0]], 0] [%[[TILE_M_1]], %[[N3]]]
+//       CHECK:     scf.yield %[[R1]] : tensor<?x?xf32>
 //       CHECK:   }
diff --git a/mlir/test/Dialect/Linalg/fusion-tensor-pattern.mlir b/mlir/test/Dialect/Linalg/fusion-tensor-pattern.mlir
index bd0d61c8580ec95715211ba87945bef07932c2c4..b24324bba110d24a636f20c0ddf985f425b9a711 100644
--- a/mlir/test/Dialect/Linalg/fusion-tensor-pattern.mlir
+++ b/mlir/test/Dialect/Linalg/fusion-tensor-pattern.mlir
@@ -12,17 +12,19 @@ module {
     return %1 : tensor<?x?xf32>
   }
 }
-//  CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0)[s0] -> (32, -d0 + s0)>
 //  CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1) -> (32, d0 - d1)>
 //  CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0)[s0] -> (16, -d0 + s0)>
 //  CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0)[s0] -> (64, -d0 + s0)>
 //  CHECK-DAG: #[[MAP4:.+]] = affine_map<(d0, d1) -> (64, d0 - d1)>
+//  CHECK-DAG: #[[MAP5:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 32, -d0 + s1)>
+
 //      CHECK: func @matmul_fusion
 // CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
 // CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
 // CHECK-SAME:   %[[ARG2:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
 // CHECK-SAME:   %[[ARG3:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
 // CHECK-SAME:   %[[ARG4:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
+
 //  CHECK-DAG:   %[[C0:.+]] = constant 0 : index
 //  CHECK-DAG:   %[[C1:.+]] = constant 1 : index
 //  CHECK-DAG:   %[[C32:.+]] = constant 32 : index
@@ -32,24 +34,25 @@ module {
 //      CHECK:   %[[RESULT:.+]] = scf.for %[[IV0:[a-zA-Z0-9]+]] =
 // CHECK-SAME:     %[[C0]] to %[[M]] step %[[C32]]
 // CHECK-SAME:     iter_args(%[[ARG6:.+]] = %[[ARG4]]) -> (tensor<?x?xf32>) {
-//      CHECK:     %[[TILE_M:.+]] = affine.min #[[MAP0]](%[[IV0]])[%[[M]]]
 //      CHECK:     %[[M_2:.+]] = memref.dim %[[ARG6]], %[[C0]]
 //      CHECK:     %[[TILE_M_2:.+]] = affine.min #[[MAP1]](%[[M_2]], %[[IV0]])
 //      CHECK:     %[[N3:.+]] = memref.dim %[[ARG6]], %[[C1]]
 //      CHECK:     %[[ST_ARG6:.+]] = subtensor %[[ARG6]][%[[IV0]], 0]
 // CHECK-SAME:       [%[[TILE_M_2]], %[[N3]]]
-//      CHECK:     %[[N2:.+]] = memref.dim %[[ARG1]], %[[C1]]
+//      CHECK:     %[[TILE_M_3:.+]] = affine.min #[[MAP5]](%[[IV0]])[%[[M]], %[[M]]]
 //      CHECK:     %[[N1:.+]] = memref.dim %[[ARG0]], %[[C1]]
 //      CHECK:     %[[ST_ARG0:.+]] = subtensor %[[ARG0]][%[[IV0]], 0]
-// CHECK-SAME:       [%[[TILE_M]], %[[N1]]]
-//      CHECK:     %[[ST_ARG1:.+]] = subtensor %[[ARG1]][0, 0]
-// CHECK-SAME:       [%[[N1]], %[[N2]]]
+// CHECK-SAME:       [%[[TILE_M_3]], %[[N1]]]
+//      CHECK:     %[[M_3:.+]] = memref.dim %[[ARG2]], %[[C0]]
+//      CHECK:     %[[TILE_M_4:.+]] = affine.min #[[MAP5]](%[[IV0]])[%[[M_3]], %[[M]]]
+//      CHECK:     %[[N2_2:.+]] = memref.dim %[[ARG2]], %[[C1]]
 //      CHECK:     %[[ST_ARG2:.+]] = subtensor %[[ARG2]][%[[IV0]], 0]
-// CHECK-SAME:       [%[[TILE_M]], %[[N2]]]
+// CHECK-SAME:       [%[[TILE_M_4]], %[[N2_2]]]
 //      CHECK:     %[[LHS:.+]] = linalg.matmul
 // CHECK-SAME:       __internal_linalg_transform__ = "after_lhs_fusion_producer"
-// CHECK-SAME:       ins(%[[ST_ARG0]], %[[ST_ARG1]] : tensor<?x?xf32>, tensor<?x?xf32>)
+// CHECK-SAME:       ins(%[[ST_ARG0]], %[[ARG1]] : tensor<?x?xf32>, tensor<?x?xf32>)
 // CHECK-SAME:       outs(%[[ST_ARG2]] : tensor<?x?xf32>)
+//      CHECK:     %[[N2:.+]] = memref.dim %[[ARG1]], %[[C1]]
 //      CHECK:     %[[N3_2:.+]] = memref.dim %[[ARG3]], %[[C1]]
 //      CHECK:     %[[YIELD0:.+]] = scf.for %[[IV1:[a-zA-Z0-9]+]] =
 // CHECK-SAME:       %[[C0]] to %[[N3_2]] step %[[C64]]
@@ -59,7 +62,7 @@ module {
 // CHECK-SAME:         iter_args(%[[ARG10:.+]] = %[[ARG8]]) -> (tensor<?x?xf32>) {
 //      CHECK:         %[[TILE_N2:.+]] = affine.min #[[MAP2]](%[[IV2]])[%[[N2]]]
 //      CHECK:         %[[ST_LHS:.+]] = subtensor %[[LHS]][0, %[[IV2]]]
-// CHECK-SAME:           [%[[TILE_M]], %[[TILE_N2]]]
+// CHECK-SAME:           [%[[TILE_M_3]], %[[TILE_N2]]]
 //      CHECK:         %[[N2_3:.+]] = memref.dim %[[ARG3]], %[[C0]]
 //      CHECK:         %[[TILE_N2_2:.+]] = affine.min #[[MAP2]](%[[IV2]])[%[[N2_3]]]
 //      CHECK:         %[[TILE_N3:.+]] = affine.min #[[MAP3]](%[[IV1]])[%[[N3_2]]]
diff --git a/mlir/test/Dialect/Linalg/fusion-tensor.mlir b/mlir/test/Dialect/Linalg/fusion-tensor.mlir
index a4071897b4d815b8ded8b29397259b56b162b624..13109bd98c19c68b751d943525ce255d29d22634 100644
--- a/mlir/test/Dialect/Linalg/fusion-tensor.mlir
+++ b/mlir/test/Dialect/Linalg/fusion-tensor.mlir
@@ -578,3 +578,41 @@ func @consumer_with_reduction(%arg0: tensor<1x10xf32>,
 //      CHECK:     %[[T4:.+]] = addf %[[T3]], %[[T2]] : f32
 //      CHECK:     linalg.yield %[[T4]]
 //      CHECK:   return %[[RES]]
+
+// -----
+
+// CHECK-LABEL: func @sigmoid_dynamic_dim(
+//       CHECK:   %[[RES:.*]] = linalg.generic
+//   CHECK-NOT:   linalg.generic
+//       CHECK:   return %[[RES]]
+func @sigmoid_dynamic_dim(%0: tensor<?x1xf32>) -> tensor<?x1xf32> {
+  %cp5 = constant 5.000000e-01 : f32
+  %c0 = constant 0 : index
+  %shape = shape.shape_of %0 : tensor<?x1xf32> -> tensor<?xindex>
+  %extend = shape.to_extent_tensor %shape : tensor<?xindex> -> tensor<2xindex>
+  %extracted = tensor.extract %extend[%c0] : tensor<2xindex>
+  %init0 = linalg.init_tensor [%extracted, 1] : tensor<?x1xf32>
+  %1 = linalg.generic {indexing_maps = [
+    affine_map<(d0, d1) -> (d0, d1)>],
+    iterator_types = ["parallel", "parallel"]
+  }
+     outs(%init0 : tensor<?x1xf32>) {
+    ^bb0(%a: f32):  // no predecessors
+      linalg.yield %cp5 : f32
+  } -> tensor<?x1xf32>
+  %d0 = memref.dim %0, %c0 : tensor<?x1xf32>
+  %init1 = linalg.init_tensor [%d0, 1] : tensor<?x1xf32>
+  %2 = linalg.generic {indexing_maps = [
+    affine_map<(d0, d1) -> (d0, d1)>,
+    affine_map<(d0, d1) -> (d0, d1)>,
+    affine_map<(d0, d1) -> (d0, d1)>],
+    iterator_types = ["parallel", "parallel"]
+  }
+      ins(%0, %1 : tensor<?x1xf32>, tensor<?x1xf32>)
+     outs(%init1 : tensor<?x1xf32>) {
+  ^bb0(%a: f32, %b: f32, %c: f32):  // no predecessors
+      %m = mulf %a, %b : f32
+      linalg.yield %m : f32
+  } -> tensor<?x1xf32>
+  return %2 : tensor<?x1xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/fusion.mlir b/mlir/test/Dialect/Linalg/fusion.mlir
index 14fb995a6cfca47961653ce1c1fbc7026b1d3135..8060e6ed255aeb35cbd3cba1343513a81adac94d 100644
--- a/mlir/test/Dialect/Linalg/fusion.mlir
+++ b/mlir/test/Dialect/Linalg/fusion.mlir
@@ -252,25 +252,36 @@ func @f5(%A: memref<?x?xf32, offset: 0, strides: [?, ?]>,
   }
   return %E : memref<?x?xf32, offset: 0, strides: [?, ?]>
 }
-// CHECK-LABEL: func @f5
-// CHECK:  (%[[A:.*]]:{{.*}}, %[[B:.*]]:{{.*}}, %[[C:.*]]:{{.*}}, %[[D:.*]]:{{.*}}, %[[E:.*]]:{{.*}})
+
+// CHECK-DAG: #[[BOUND_2_MAP:.+]] = affine_map<(d0)[s0] -> (2, -d0 + s0)>
+// CHECK-DAG: #[[BOUND_2_MAP_2:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 2, -d0 + s1)>
+// CHECK-DAG: #[[BOUND_4_MAP:.+]] = affine_map<(d0)[s0] -> (4, -d0 + s0)>
+// CHECK: func @f5
+// HECK-SAME:  (%[[A:.*]]:{{.*}}, %[[B:.*]]:{{.*}}, %[[C:.*]]:{{.*}}, %[[D:.*]]:{{.*}}, %[[E:.*]]:{{.*}})
 // CHECK-DAG:  %[[C0:.*]] = constant 0 : index
 // CHECK-DAG:  %[[C1:.*]] = constant 1 : index
-// CHECK-DAG:  %[[B_1:.*]] = memref.dim %[[B]], %[[C1:.*]] : memref<?x?xf32, #[[$strided2D]]>
-// CHECK-DAG:  %[[D_0:.*]] = memref.dim %[[D]], %[[C0:.*]] : memref<?x?xf32, #[[$strided2D]]>
-// CHECK-DAG:  %[[D_1:.*]] = memref.dim %[[D]], %[[C1:.*]] : memref<?x?xf32, #[[$strided2D]]>
+// CHECK-DAG:  %[[B_1:.*]] = memref.dim %[[B]], %[[C1]] : memref<?x?xf32, #[[$strided2D]]>
+// CHECK-DAG:  %[[C_0:.*]] = memref.dim %[[C]], %[[C0]] : memref<?x?xf32, #[[$strided2D]]>
+// CHECK-DAG:  %[[D_0:.*]] = memref.dim %[[D]], %[[C0]] : memref<?x?xf32, #[[$strided2D]]>
+// CHECK-DAG:  %[[D_1:.*]] = memref.dim %[[D]], %[[C1]] : memref<?x?xf32, #[[$strided2D]]>
 // CHECK-DAG:  %[[B_00:.*]] = memref.subview %[[B]][0, 0]{{.*}}
 //     CHECK:  scf.for %[[I:.*]] = %{{.*}} to %[[D_0]] step %{{.*}} {
-// CHECK-DAG:    %[[A_I0:.*]] = memref.subview %[[A]][%[[I]], 0]
-// CHECK-DAG:    %[[C_I0:.*]] = memref.subview %[[C]][%[[I]], 0]
+//     CHECK:    %[[BOUND_2_C0:.+]] = affine.min #[[BOUND_2_MAP]](%[[I]])[%[[C_0]]]
+//     CHECK:    %[[C_I0:.*]] = memref.subview %[[C]][%[[I]], 0] [%[[BOUND_2_C0]]
+//     CHECK:    %[[BOUND_2_D0:.+]] = affine.min #[[BOUND_2_MAP]](%[[I]])[%[[D_0]]]
+//     CHECK:    %[[A_I0:.*]] = memref.subview %[[A]][%[[I]], 0]
+//     CHECK:    %[[BOUND_ID_C0:.+]] = affine.min #[[BOUND_2_MAP_2]](%[[I]])[%[[C_0]], %[[C_0]]]
+//     CHECK:    %[[C_I0_OUT:.*]] = memref.subview %[[C]][%[[I]], 0] [%[[BOUND_ID_C0]]
 //     CHECK:    scf.for %[[J:.*]] = %{{.*}} to %[[B_1]] step %{{.*}} {
 //     CHECK:      %[[E_IJ:.*]] = memref.subview %[[E]][%[[I]], %[[J]]]
 //     CHECK:      scf.for %[[K:.*]] = %{{.*}} to %[[D_1]] step %{{.*}} {
-// CHECK-DAG:        %[[D_IK:.*]] = memref.subview %[[D]][%[[I]], %[[K]]]
-// CHECK-DAG:        %[[B_0K:.*]] = memref.subview %[[B]][0, %[[K]]]
-// CHECK-DAG:        %[[B_KJ:.*]] = memref.subview %[[B]][%[[K]], %[[J]]]
-//     CHECK:        linalg.matmul ins(%[[A_I0]], %[[B_00]]{{.*}} outs(%[[C_I0]]
-//     CHECK:        linalg.matmul ins(%[[C_I0]], %[[B_0K]]{{.*}} outs(%[[D_IK]]
+//     CHECK:        %[[D_IK:.*]] = memref.subview %[[D]][%[[I]], %[[K]]] [2, 4]
+//     CHECK:        %[[B_KJ:.*]] = memref.subview %[[B]][%[[K]], %[[J]]]
+//     CHECK:        %[[B_0K:.*]] = memref.subview %[[B]][0, %[[K]]]
+//     CHECK:        %[[BOUND_4_D1:.+]] = affine.min #[[BOUND_4_MAP]](%[[K]])[%[[D_1]]]
+//     CHECK:        %[[D_IK_OUT:.+]] = memref.subview %[[D]][%[[I]], %[[K]]] [%[[BOUND_2_D0]], %[[BOUND_4_D1]]]
+//     CHECK:        linalg.matmul ins(%[[A_I0]], %[[B_00]]{{.*}} outs(%[[C_I0_OUT]]
+//     CHECK:        linalg.matmul ins(%[[C_I0]], %[[B_0K]]{{.*}} outs(%[[D_IK_OUT]]
 //     CHECK:        linalg.matmul ins(%[[D_IK]], %[[B_KJ]]{{.*}} outs(%[[E_IJ]]
 
 // -----
diff --git a/mlir/test/Dialect/Linalg/hoist-padding.mlir b/mlir/test/Dialect/Linalg/hoist-padding.mlir
index 969974aa7e52ddf88366d7ae964a8408beb44d27..248aa6414ad8bfbef4ae846fbe021a6a4c0b3135 100644
--- a/mlir/test/Dialect/Linalg/hoist-padding.mlir
+++ b/mlir/test/Dialect/Linalg/hoist-padding.mlir
@@ -1,20 +1,29 @@
-// RUN: mlir-opt %s -test-linalg-transform-patterns=test-hoist-padding-2-level -canonicalize | FileCheck %s
+// Specific structural checks are performed on 2-level hoisting
+// RUN: mlir-opt %s -split-input-file -test-linalg-transform-patterns=test-hoist-padding=2 -canonicalize | FileCheck %s
 
-#map0 = affine_map<(d0)[s0] -> (2, -d0 + s0)>
-#map1 = affine_map<(d0)[s0] -> (4, -d0 + s0)>
-#map2 = affine_map<(d0)[s0] -> (3, -d0 + s0)>
-#map3 = affine_map<(d0, d1) -> (2, d0 - d1)>
-#map4 = affine_map<(d0, d1) -> (3, d0 - d1)>
+// IR verification is performed on [0-6]-level hoisting
+// RUN: mlir-opt %s -split-input-file -test-linalg-transform-patterns=test-hoist-padding=0 | FileCheck %s --check-prefix=VERIFIER-ONLY
+// RUN: mlir-opt %s -split-input-file -test-linalg-transform-patterns=test-hoist-padding=1 | FileCheck %s --check-prefix=VERIFIER-ONLY
+// RUN: mlir-opt %s -split-input-file -test-linalg-transform-patterns=test-hoist-padding=3 | FileCheck %s --check-prefix=VERIFIER-ONLY
+// RUN: mlir-opt %s -split-input-file -test-linalg-transform-patterns=test-hoist-padding=4 | FileCheck %s --check-prefix=VERIFIER-ONLY
+// RUN: mlir-opt %s -split-input-file -test-linalg-transform-patterns=test-hoist-padding=5 | FileCheck %s --check-prefix=VERIFIER-ONLY
+// RUN: mlir-opt %s -split-input-file -test-linalg-transform-patterns=test-hoist-padding=6 | FileCheck %s --check-prefix=VERIFIER-ONLY
 
 // CHECK-DAG: #[[$DIV3:[0-9a-z]+]] = affine_map<(d0) -> (d0 ceildiv 3)>
 // CHECK-DAG: #[[$DIV4:[0-9a-z]+]] = affine_map<(d0) -> (d0 ceildiv 4)>
 // CHECK-DAG: #[[$DIVS3:[0-9a-z]+]] = affine_map<()[s0] -> (s0 ceildiv 3)>
 // CHECK-DAG: #[[$DIVS4:[0-9a-z]+]] = affine_map<()[s0] -> (s0 ceildiv 4)>
+#map0 = affine_map<(d0)[s0] -> (2, -d0 + s0)>
+#map1 = affine_map<(d0)[s0] -> (4, -d0 + s0)>
+#map2 = affine_map<(d0)[s0] -> (3, -d0 + s0)>
+#map3 = affine_map<(d0, d1) -> (2, d0 - d1)>
+#map4 = affine_map<(d0, d1) -> (3, d0 - d1)>
 
 // CHECK-LABEL: func @matmul_tensors
 //  CHECK-SAME:   %[[TA:[0-9a-z]+]]: tensor
 //  CHECK-SAME:   %[[TB:[0-9a-z]+]]: tensor
 //  CHECK-SAME:   %[[TC:[0-9a-z]+]]: tensor
+// VERIFIER-ONLY-LABEL: func @matmul_tensors
 func @matmul_tensors(
   %arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>, %arg2: tensor<?x?xf32>)
   -> tensor<?x?xf32>
@@ -129,3 +138,153 @@ func @matmul_tensors(
   }
   return %3 : tensor<?x?xf32>
 }
+
+// -----
+
+// CHECK-DAG: #[[$MIN_REST8:[0-9a-z]+]] = affine_map<(d0)[s0] -> (8, -d0 + s0)>
+// CHECK-DAG: #[[$MIN_MOD4:[0-9a-z]+]] = affine_map<(d0) -> (4, d0 - ((d0 - 1) floordiv 4) * 4)>
+// CHECK-DAG: #[[$DIV4:[0-9a-z]+]] = affine_map<(d0) -> (d0 ceildiv 4)>
+// CHECK-DAG: #[[$DIV2:[0-9a-z]+]] = affine_map<(d0) -> (d0 ceildiv 2)>
+#map0 = affine_map<(d0)[s0] -> (8, -d0 + s0)>
+#map1 = affine_map<(d0, d1) -> (4, d0 - d1)>
+#map2 = affine_map<(d0, d1) -> (2, d0 - d1)>
+
+// CHECK-LABEL: func @dot
+// VERIFIER-ONLY-LABEL: func @dot
+func @dot(%arg0: tensor<?xf32>, %arg1: tensor<?xf32>, %arg2: tensor<f32>)
+    -> tensor<f32>
+{
+  %c8 = constant 8 : index
+  %c4 = constant 4 : index
+  %cst = constant 0.000000e+00 : f32
+  %c2 = constant 2 : index
+  %c0 = constant 0 : index
+  %1 = memref.dim %arg0, %c0 : tensor<?xf32>
+  %2 = memref.dim %arg0, %c0 : tensor<?xf32>
+  %3 = memref.dim %arg1, %c0 : tensor<?xf32>
+
+  //      CHECK: scf.for %[[I:[0-9a-z]+]] =
+  //
+  //      CHECK:   %[[MR8:.*]] = affine.min #[[$MIN_REST8]](%[[I]])
+  //      CHECK:   %[[D0:.*]] = affine.apply #[[$DIV4]](%[[MR8]])
+  //      CHECK:   %[[MM4:.*]] = affine.min #[[$MIN_MOD4]](%[[MR8]])
+  //      CHECK:   %[[D1:.*]] = affine.apply #[[$DIV2]](%[[MM4]])
+  // Init tensor and pack.
+  //      CHECK:   %[[INIT_PACKED_A:.*]] = linalg.init_tensor [%[[D0]], %[[D1]], 2] : tensor<?x?x2xf32>
+  //      CHECK:   %[[PACKED_A:.*]] = scf.for %[[II:[0-9a-z]+]] = {{.*}} iter_args(%{{.*}} = %[[INIT_PACKED_A]]) -> (tensor<?x?x2xf32>) {
+  //      CHECK:     scf.for %[[III:[0-9a-z]+]] =
+  //      CHECK:       subtensor_insert %{{.*}} into %{{.*}}[%{{.*}}, %{{.*}}, 0] [1, 1, 2] [1, 1, 1] : tensor<2xf32> into tensor<?x?x2xf32>
+  //
+  //      CHECK:   %[[D0_2:.*]] = affine.apply #[[$DIV4]](%[[MR8]])
+  //      CHECK:   %[[MM4_2:.*]] = affine.min #[[$MIN_MOD4]](%[[MR8]])
+  //      CHECK:   %[[D1_2:.*]] = affine.apply #[[$DIV2]](%[[MM4_2]])
+  // Init tensor and pack.
+  //      CHECK:   %[[INIT_PACKED_B:.*]] = linalg.init_tensor [%[[D0_2]], %[[D1_2]], 2] : tensor<?x?x2xf32>
+  //      CHECK:   %[[PACKED_B:.*]] = scf.for %[[II_2:[0-9a-z]+]] = {{.*}} iter_args(%{{.*}} = %[[INIT_PACKED_B]]) -> (tensor<?x?x2xf32>) {
+  //      CHECK:     scf.for %[[III_2:[0-9a-z]+]] =
+  //      CHECK:       subtensor_insert %{{.*}} into %{{.*}}[%{{.*}}, %{{.*}}, 0] [1, 1, 2] [1, 1, 1] : tensor<2xf32> into tensor<?x?x2xf32>
+  // Compute.
+  //      CHECK:   scf.for %[[II_3:[0-9a-z]+]] =
+  //      CHECK:     scf.for %[[III_3:[0-9a-z]+]] = {{.*}} iter_args(%[[C:.*]] = %{{.*}}) -> (tensor<f32>) {
+  //      CHECK:       %[[IDX0:.*]] = affine.apply #[[$DIV4]](%[[II_3]])
+  //      CHECK:       %[[IDX1:.*]] = affine.apply #[[$DIV2]](%[[III_3]])
+  //      CHECK:       %[[A:.*]] = subtensor %[[PACKED_A]][%[[IDX0]], %[[IDX1]], 0] [1, 1, 2] [1, 1, 1] : tensor<?x?x2xf32> to tensor<2xf32>
+  //      CHECK:       %[[IDX0_2:.*]] = affine.apply #[[$DIV4]](%[[II_3]])
+  //      CHECK:       %[[IDX1_2:.*]] = affine.apply #[[$DIV2]](%[[III_3]])
+  //      CHECK:       %[[B:.*]] = subtensor %[[PACKED_B]][%[[IDX0_2]], %[[IDX1_2]], 0] [1, 1, 2] [1, 1, 1] : tensor<?x?x2xf32> to tensor<2xf32>
+  //      CHECK:       linalg.dot ins(%[[A]], %[[B]] : tensor<2xf32>, tensor<2xf32>) outs(%[[C]] : tensor<f32>) -> tensor<f32>
+
+  %4 = scf.for %arg3 = %c0 to %1 step %c8 iter_args(%arg4 = %arg2) -> (tensor<f32>) {
+    %5 = affine.min #map0(%arg3)[%2]
+    %6 = subtensor %arg0[%arg3] [%5] [1] : tensor<?xf32> to tensor<?xf32>
+    %7 = affine.min #map0(%arg3)[%3]
+    %8 = subtensor %arg1[%arg3] [%7] [1] : tensor<?xf32> to tensor<?xf32>
+    %9 = scf.for %arg5 = %c0 to %5 step %c4 iter_args(%arg6 = %arg4) -> (tensor<f32>) {
+      %10 = affine.min #map1(%5, %arg5)
+      %11 = subtensor %6[%arg5] [%10] [1] : tensor<?xf32> to tensor<?xf32>
+      %12 = affine.min #map1(%7, %arg5)
+      %13 = subtensor %8[%arg5] [%12] [1] : tensor<?xf32> to tensor<?xf32>
+      %14 = scf.for %arg7 = %c0 to %10 step %c2 iter_args(%arg8 = %arg6) -> (tensor<f32>) {
+        %15 = affine.min #map2(%10, %arg7)
+        %16 = subtensor %11[%arg7] [%15] [1] : tensor<?xf32> to tensor<?xf32>
+        %17 = affine.min #map2(%12, %arg7)
+        %18 = subtensor %13[%arg7] [%17] [1] : tensor<?xf32> to tensor<?xf32>
+        %19 = subi %c2, %15 : index
+        %20 = linalg.pad_tensor %16 low[%c0] high[%19]  {
+        ^bb0(%arg9: index):  // no predecessors
+          linalg.yield %cst : f32
+        } : tensor<?xf32> to tensor<2xf32>
+        %21 = subi %c2, %17 : index
+        %22 = linalg.pad_tensor %18 low[%c0] high[%21]  {
+        ^bb0(%arg9: index):  // no predecessors
+          linalg.yield %cst : f32
+        } : tensor<?xf32> to tensor<2xf32>
+        %23 = linalg.dot ins(%20, %22 : tensor<2xf32>, tensor<2xf32>) outs(%arg8 : tensor<f32>) -> tensor<f32>
+        scf.yield %23 : tensor<f32>
+      }
+      scf.yield %14 : tensor<f32>
+    }
+    scf.yield %9 : tensor<f32>
+  }
+  return %4 : tensor<f32>
+}
+
+// -----
+
+// CHECK-LABEL: func @matmul_2d_tiling
+// VERIFIER-ONLY-LABEL: func @matmul_2d_tiling
+func @matmul_2d_tiling(%arg0: tensor<32x128xf32>, %arg1: tensor<128x64xf32>, %arg2: tensor<32x64xf32>) -> tensor<32x64xf32> {
+  %c128 = constant 128 : index
+  %c64 = constant 64 : index
+  %c32 = constant 32 : index
+  %c16 = constant 16 : index
+  %cst = constant 0.000000e+00 : f32
+  %c2 = constant 2 : index
+  %c4 = constant 4 : index
+  %c0 = constant 0 : index
+  %1 = scf.for %arg3 = %c0 to %c32 step %c16 iter_args(%arg4 = %arg2) -> (tensor<32x64xf32>) {
+    %2 = scf.for %arg5 = %c0 to %c64 step %c32 iter_args(%arg6 = %arg4) -> (tensor<32x64xf32>) {
+      %3 = scf.for %arg7 = %c0 to %c128 step %c32 iter_args(%arg8 = %arg6) -> (tensor<32x64xf32>) {
+        %4 = subtensor %arg0[%arg3, %arg7] [16, 32] [1, 1] : tensor<32x128xf32> to tensor<16x32xf32>
+        %5 = subtensor %arg1[%arg7, %arg5] [32, 32] [1, 1] : tensor<128x64xf32> to tensor<32x32xf32>
+        %6 = subtensor %arg8[%arg3, %arg5] [16, 32] [1, 1] : tensor<32x64xf32> to tensor<16x32xf32>
+        %7 = scf.for %arg9 = %c0 to %c16 step %c2 iter_args(%arg10 = %6) -> (tensor<16x32xf32>) {
+          %10 = scf.for %arg11 = %c0 to %c32 step %c4 iter_args(%arg12 = %arg10) -> (tensor<16x32xf32>) {
+            %11 = scf.for %arg13 = %c0 to %c32 step %c16 iter_args(%arg14 = %arg12) -> (tensor<16x32xf32>) {
+              %12 = subtensor %4[%arg9, %arg13] [2, 16] [1, 1] : tensor<16x32xf32> to tensor<2x16xf32>
+              %13 = tensor.cast %12 : tensor<2x16xf32> to tensor<?x?xf32>
+              %14 = subtensor %5[%arg13, %arg11] [16, 4] [1, 1] : tensor<32x32xf32> to tensor<16x4xf32>
+              %15 = tensor.cast %14 : tensor<16x4xf32> to tensor<?x?xf32>
+              %16 = subtensor %arg14[%arg9, %arg11] [2, 4] [1, 1] : tensor<16x32xf32> to tensor<2x4xf32>
+              %17 = tensor.cast %16 : tensor<2x4xf32> to tensor<?x?xf32>
+              %18 = linalg.pad_tensor %13 low[%c0, %c0] high[%c0, %c0]  {
+              ^bb0(%arg15: index, %arg16: index):  // no predecessors
+                linalg.yield %cst : f32
+              } : tensor<?x?xf32> to tensor<2x16xf32>
+              %19 = linalg.pad_tensor %15 low[%c0, %c0] high[%c0, %c0]  {
+              ^bb0(%arg15: index, %arg16: index):  // no predecessors
+                linalg.yield %cst : f32
+              } : tensor<?x?xf32> to tensor<16x4xf32>
+              %20 = linalg.pad_tensor %17 low[%c0, %c0] high[%c0, %c0]  {
+              ^bb0(%arg15: index, %arg16: index):  // no predecessors
+                linalg.yield %cst : f32
+              } : tensor<?x?xf32> to tensor<2x4xf32>
+              %21 = linalg.matmul ins(%18, %19 : tensor<2x16xf32>, tensor<16x4xf32>) outs(%20 : tensor<2x4xf32>) -> tensor<2x4xf32>
+              %22 = tensor.cast %21 : tensor<2x4xf32> to tensor<?x?xf32>
+              %23 = subtensor_insert %22 into %arg14[%arg9, %arg11] [%c2, %c4] [1, 1] : tensor<?x?xf32> into tensor<16x32xf32>
+              scf.yield %23 : tensor<16x32xf32>
+            }
+            scf.yield %11 : tensor<16x32xf32>
+          }
+          scf.yield %10 : tensor<16x32xf32>
+        }
+        %8 = tensor.cast %7 : tensor<16x32xf32> to tensor<?x?xf32>
+        %9 = subtensor_insert %8 into %arg8[%arg3, %arg5] [%c16, %c32] [1, 1] : tensor<?x?xf32> into tensor<32x64xf32>
+        scf.yield %9 : tensor<32x64xf32>
+      }
+      scf.yield %3 : tensor<32x64xf32>
+    }
+    scf.yield %2 : tensor<32x64xf32>
+  }
+  return %1 : tensor<32x64xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/reshape_fusion.mlir b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
index fbaf47c9ac4df5c5bfb84a1535bbb4e88c8104a5..d5dc176f1fdf8ece52ca9476d7dbebac58f99959 100644
--- a/mlir/test/Dialect/Linalg/reshape_fusion.mlir
+++ b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
@@ -1,4 +1,5 @@
-// RUN: mlir-opt %s -linalg-fusion-for-tensor-ops -split-input-file -verify-each=0 | FileCheck %s
+// RUN: mlir-opt %s -linalg-fusion-for-tensor-ops="allow-folding-unit-dim-reshapes=false" -split-input-file -verify-each=0 | FileCheck %s
+// RUN: mlir-opt %s -linalg-fusion-for-tensor-ops="allow-folding-unit-dim-reshapes=true" -split-input-file -verify-each=0 | FileCheck %s --check-prefix=FOLDUNITDIM
 
 #map0 = affine_map<(d0, d1, d2) -> (d2, d0, d1)>
 #map1 = affine_map<(d0, d1, d2) -> (d1, d2, d0)>
@@ -300,7 +301,7 @@ func @reshape_as_consumer_permutation
          %5 = addi %3, %4 : i32
          %6 = index_cast %arg2 : index to i32
          %7 = addi %5, %6 : i32
-	 linalg.yield %7 : i32
+         linalg.yield %7 : i32
        } -> tensor<6x4x210xi32>
   %d = linalg.tensor_reshape %c
          [affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1)>,
@@ -531,3 +532,11 @@ func @unit_dim_reshape_expansion_full
 //  CHECK-DAG:   linalg.tensor_reshape
 //  CHECK-DAG:   linalg.init_tensor
 //      CHECK:   linalg.generic
+// CHECK-SAME:     ins(%{{.+}}, %{{.+}} : tensor<?x2x4xf32>, tensor<?x2x4xf32>)
+
+//         FOLDUNITDIM: func @unit_dim_reshape_expansion_full
+//         FOLDUNITDIM:   linalg.init_tensor
+// FOLDUNITDIM-COUNT-2:   linalg.tensor_reshape
+//         FOLDUNITDIM:   linalg.generic
+//    FOLDUNITDIM-SAME:     ins(%{{.+}}, %{{.+}} : tensor<1x?x1x2x1x4xf32>, tensor<1x?x1x2x1x4xf32>)
+
diff --git a/mlir/test/Dialect/Linalg/roundtrip.mlir b/mlir/test/Dialect/Linalg/roundtrip.mlir
index ab2547952cfb8a99f7d0dcae5bc07fbb4d8e1fa2..084b8a339c0dee301a7c662304b0b3b1b06947c8 100644
--- a/mlir/test/Dialect/Linalg/roundtrip.mlir
+++ b/mlir/test/Dialect/Linalg/roundtrip.mlir
@@ -6,6 +6,8 @@
 // Test that we can lower all the way to LLVM without crashing, don't check results here.
 // DISABLED: mlir-opt %s --convert-linalg-to-llvm -o=/dev/null 2>&1
 
+// CHECK-DAG: #[[$id_2d:.*]] = affine_map<(d0, d1, d2) -> (d0, d2)>
+// CHECK-DAG: #[[$id_1d:.*]] = affine_map<(d0, d1, d2) -> (d1)>
 // CHECK-DAG: #[[$permute_0:.*]] = affine_map<(d0, d1, d2) -> (d0, d2, d1)>
 // CHECK-DAG: #[[$permute_1:.*]] = affine_map<(d0, d1, d2) -> (d2, d1, d0)>
 // CHECK-DAG: #[[$reshape5D01:.*]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1)>
@@ -881,3 +883,61 @@ func @tiled_loop_reduction(%input_3d: tensor<16x24x32xf32>,
 }
 // CHECK-LABEL: func @tiled_loop_reduction
 // CHECK: iterators[
+
+// -----
+
+#trait_6 = {
+  indexing_maps = [
+    #id_3d,
+    #id_2d,
+    #id_1d,
+    #id_1d
+  ],
+  iterator_types = ["reduction", "parallel", "reduction"]
+}
+#map_1 = affine_map<(d0, d1, d2)[s0] -> (d0 * 768 + s0 + d1 * 32 + d2)>
+#map_2 = affine_map<(d0, d1)[s0] -> (d0 * 32 + s0 + d1)>
+#map_3 = affine_map<(d0)[s0] -> (d0 + s0)>
+
+func @tiled_loop_on_buffers(%input_3d: memref<16x24x32xf32>,
+                            %input_2d: memref<16x32xf32>,
+                            %input_1d: memref<24xf32>,
+                            %output: memref<24xf32>) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %c2 = constant 2 : index
+  %c4 = constant 4 : index
+  %c8 = constant 8 : index
+  %X = memref.dim %input_3d, %c0 : memref<16x24x32xf32>
+  %Y = memref.dim %input_3d, %c1 : memref<16x24x32xf32>
+  %Z = memref.dim %input_3d, %c2 : memref<16x24x32xf32>
+  linalg.tiled_loop (%i, %j, %k) = (%c0, %c0, %c0)
+      to (%X, %Y, %Z) step (%c2, %c4, %c8)
+      ins(%input_3d, %input_2d: memref<16x24x32xf32>, memref<16x32xf32>)
+      outs( %output: memref<24xf32>)
+      iterators["reduction", "parallel", "reduction"] {
+    %sub_3d = memref.subview %input_3d[%i, %j, %k][2, 4, 8][1, 1, 1]
+      : memref<16x24x32xf32> to memref<2x4x8xf32, #map_1>
+    %sub_2d = memref.subview %input_2d[%i, %k][2, 8][1, 1]
+      : memref<16x32xf32> to memref<2x8xf32, #map_2>
+    %sub_1d = memref.subview %input_1d[%j] [4] [1]
+      : memref<24xf32> to memref<4xf32, #map_3>
+    %sub_out = memref.subview %output[%j] [4] [1]
+      : memref<24xf32> to memref<4xf32, #map_3>
+    linalg.generic #trait_6
+      ins(%sub_3d, %sub_2d, %sub_1d
+        : memref<2x4x8xf32, #map_1>,
+          memref<2x8xf32, #map_2>,
+          memref<4xf32, #map_3>)
+      outs(%sub_out : memref<4xf32, #map_3>)  {
+    ^bb0(%i3d: f32, %i2d: f32, %i1d: f32, %o: f32):
+      %0 = addf %i3d, %i2d : f32
+      %1 = addf %0, %i1d : f32
+      linalg.yield %1 : f32
+    }
+    linalg.yield
+  }
+  return
+}
+// CHECK-LABEL: func @tiled_loop_on_buffers
+// CHECK: iterators[
diff --git a/mlir/test/Dialect/Linalg/sparse_2d.mlir b/mlir/test/Dialect/Linalg/sparse_2d.mlir
index 24ccdfc20b14253cb35312883f89ad7edb3693a2..b9e14e3afb8e209275c1c2c41b169e7eab4e146d 100644
--- a/mlir/test/Dialect/Linalg/sparse_2d.mlir
+++ b/mlir/test/Dialect/Linalg/sparse_2d.mlir
@@ -1163,9 +1163,9 @@ func @sum_reduction(%arga: tensor<10x20xf32>, %argx: tensor<f32>) -> tensor<f32>
 // CHECK-LABEL:   func @scale(
 // CHECK-SAME:                %[[VAL_0:.*]]: tensor<?x?xf64>,
 // CHECK-SAME:                %[[VAL_1:.*]]: tensor<?x?xf64>) -> tensor<?x?xf64> {
-// CHECK:           %[[VAL_2:.*]] = constant 2.000000e+00 : f64
-// CHECK:           %[[VAL_3:.*]] = constant 0 : index
-// CHECK:           %[[VAL_4:.*]] = constant 1 : index
+// CHECK-DAG:           %[[VAL_3:.*]] = constant 0 : index
+// CHECK-DAG:           %[[VAL_4:.*]] = constant 1 : index
+// CHECK-DAG:           %[[VAL_2:.*]] = constant 2.000000e+00 : f64
 // CHECK:           %[[VAL_5:.*]] = linalg.sparse_pointers %[[VAL_0]], %[[VAL_4]] : tensor<?x?xf64> to memref<?xindex>
 // CHECK:           %[[VAL_6:.*]] = linalg.sparse_indices %[[VAL_0]], %[[VAL_4]] : tensor<?x?xf64> to memref<?xindex>
 // CHECK:           %[[VAL_7:.*]] = linalg.sparse_values %[[VAL_0]] : tensor<?x?xf64> to memref<?xf64>
diff --git a/mlir/test/Dialect/Linalg/tile-and-distribute.mlir b/mlir/test/Dialect/Linalg/tile-and-distribute.mlir
index 6f8c2b0ce76eb4e19644f5312d44bdc98dbf5ab1..a6756306cff8964baaf9757617020949a39745db 100644
--- a/mlir/test/Dialect/Linalg/tile-and-distribute.mlir
+++ b/mlir/test/Dialect/Linalg/tile-and-distribute.mlir
@@ -136,7 +136,7 @@ func @gemm5(%a : memref<?x?xf32>, %b : memref<?x?xf32>, %c : memref<?x?xf32>)
 //      CHECK: %[[STEPX:.*]] = affine.apply #[[MAP0]]()[%[[NBLOCKSX]]]
 //      CHECK: %[[INBOUNDS:.*]] = cmpi slt, %[[LBY]], %{{.*}}
 //      CHECK: scf.if %[[INBOUNDS]]
-//      CHECK:   scf.parallel (%[[ARG3.*]]) = (%[[LBX]]) to (%{{.*}}) step (%[[STEPX]])
+//      CHECK:   scf.parallel (%[[ARG3:.*]]) = (%[[LBX]]) to (%{{.*}}) step (%[[STEPX]])
 //      CHECK:     scf.for %[[ARG4:.*]] =
 //      CHECK:      %[[OFFSETY:.*]] = affine.apply #[[MAP0]]()[%[[BIDY]]]
 //      CHECK:       %[[SV1:.*]] = memref.subview %[[ARG0]][%[[OFFSETY]], %[[ARG4]]]
@@ -164,7 +164,7 @@ func @gemm6(%a : memref<?x?xf32>, %b : memref<?x?xf32>, %c : memref<?x?xf32>)
 //      CHECK: %[[BIDX:.*]] = "gpu.block_id"() {dimension = "x"}
 //      CHECK: %[[LBY:.*]] = affine.apply #[[MAP0]]()[%[[BIDY]]]
 //      CHECK: %[[STEPY:.*]] = affine.apply #[[MAP0]]()[%[[NBLOCKSY]]]
-//      CHECK: scf.parallel (%[[ARG3.*]]) = (%[[LBY]]) to (%{{.*}}) step (%[[STEPY]])
+//      CHECK: scf.parallel (%[[ARG3:.*]]) = (%[[LBY]]) to (%{{.*}}) step (%[[STEPY]])
 //      CHECK:   scf.for %[[ARG4:.*]] =
 //      CHECK:     %[[SV1:.*]] = memref.subview %[[ARG0]][%[[ARG3]], %[[ARG4]]]
 //      CHECK:     %[[OFFSETX:.*]] = affine.apply #[[MAP0]]()[%[[BIDX]]]
diff --git a/mlir/test/Dialect/Linalg/tile-and-fuse-tensors.mlir b/mlir/test/Dialect/Linalg/tile-and-fuse-tensors.mlir
index b742f8148dac3d3a01294481629811f12ac14d8f..200a237cfdc8b80e4c8b408de5fd87fbf741c0ac 100644
--- a/mlir/test/Dialect/Linalg/tile-and-fuse-tensors.mlir
+++ b/mlir/test/Dialect/Linalg/tile-and-fuse-tensors.mlir
@@ -1,11 +1,5 @@
 // RUN: mlir-opt %s -test-linalg-greedy-fusion -split-input-file | FileCheck %s
 
-#map0 = affine_map<(d0)[s0] -> (2, -d0 + s0)>
-#map1 = affine_map<(d0)[s0] -> (4, -d0 + s0)>
-#map2 = affine_map<(d0)[s0] -> (3, -d0 + s0)>
-#map3 = affine_map<(d0, d1) -> (2, d0 - d1)>
-#map4 = affine_map<(d0, d1) -> (3, d0 - d1)>
-
 func @matmul_tensors(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>, %arg2: tensor<?x?xf32>) -> tensor<?x?xf32> {
   %t0 = linalg.matmul ins(%arg0, %arg1: tensor<?x?xf32>, tensor<?x?xf32>)
                      outs(%arg2: tensor<?x?xf32>)
@@ -36,23 +30,255 @@ func @matmul_tensors(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>, %arg2: tens
   return %3 : tensor<?x?xf32>
 }
 
-// CHECK-LABEL: func @matmul_tensors(
+//       CHECK: #[[BOUND2_MAP:.+]] = affine_map<(d0)[s0] -> (2, -d0 + s0)>
+//       CHECK: #[[BOUND4_MAP:.+]] = affine_map<(d0)[s0] -> (4, -d0 + s0)>
+
+//       CHECK: func @matmul_tensors(
 //  CHECK-SAME: %[[A:[0-9a-z]*]]: tensor<?x?xf32>
 //  CHECK-SAME: %[[B:[0-9a-z]*]]: tensor<?x?xf32>
 //  CHECK-SAME: %[[C:[0-9a-z]*]]: tensor<?x?xf32>
+
 //   CHECK-DAG: %[[C0:.*]] = constant 0 : index
 //   CHECK-DAG: %[[C1:.*]] = constant 1 : index
+//   CHECK-DAG: %[[dA0:.*]] = memref.dim %[[A]], %[[C0]] : tensor<?x?xf32>
 //   CHECK-DAG: %[[dA1:.*]] = memref.dim %[[A]], %[[C1]] : tensor<?x?xf32>
+//   CHECK-DAG: %[[dB0:.*]] = memref.dim %[[B]], %[[C0]] : tensor<?x?xf32>
+//   CHECK-DAG: %[[dB1:.*]] = memref.dim %[[B]], %[[C1]] : tensor<?x?xf32>
+//   CHECK-DAG: %[[dC0:.*]] = memref.dim %[[C]], %[[C0]] : tensor<?x?xf32>
+//   CHECK-DAG: %[[dC1:.*]] = memref.dim %[[C]], %[[C1]] : tensor<?x?xf32>
 //       CHECK: scf.for %[[I:[0-9a-z]*]]
-//       CHECK:     %[[stA:.*]] = subtensor %[[A]][%[[I]], 0] [2, %[[dA1]]] [1, 1]  : tensor<?x?xf32> to tensor<2x?xf32>
+//       CHECK:   %[[sizeA0:.*]] = affine.min #[[BOUND2_MAP]](%[[I]])[%[[dA0]]]
+//       CHECK:   %[[stA:.*]] = subtensor %[[A]][%[[I]], 0] [%[[sizeA0]], %[[dA1]]] [1, 1]  : tensor<?x?xf32> to tensor<?x?xf32>
+//       CHECK:   %[[sizeC0:.*]] = affine.min #[[BOUND2_MAP]](%[[I]])[%[[dC0]]]
 //  CHECK-NEXT:   scf.for %[[J:[0-9a-z]*]]
 //  CHECK-NEXT:     scf.for %[[K:[0-9a-z]*]] {{.*}} iter_args(%[[RES:[0-9a-z]*]]
 //   CHECK-DAG:       %[[stB1:.*]] = subtensor %[[B]][%[[K]], %[[J]]] [4, 3] [1, 1]  : tensor<?x?xf32> to tensor<4x3xf32>
 //   CHECK-DAG:       %[[stF:.*]] = subtensor %[[RES]][%[[I]], %[[J]]] [2, 3] [1, 1]  : tensor<?x?xf32> to tensor<2x3xf32>
 //
 // subtensors of the producing matmul.
-//   CHECK-DAG:       %[[stB2:.*]] = subtensor %[[B]][0, %[[K]]] [%[[dA1]], 4] [1, 1]  : tensor<?x?xf32> to tensor<?x4xf32>
-//   CHECK-DAG:       %[[stC:.*]] = subtensor %[[C]][%[[I]], %[[K]]] [2, 4] [1, 1]  : tensor<?x?xf32> to tensor<2x4xf32>
-//       CHECK:       %[[stD:.*]] = linalg.matmul ins(%[[stA]], %[[stB2]] : tensor<2x?xf32>, tensor<?x4xf32>) outs(%[[stC]] : tensor<2x4xf32>)  -> tensor<2x4xf32>
-//  CHECK-NEXT:       %[[stG:.*]] = linalg.matmul ins(%[[stD]], %[[stB1]] : tensor<2x4xf32>, tensor<4x3xf32>) outs(%[[stF]] : tensor<2x3xf32>)  -> tensor<2x3xf32>
+//       CHECK:       %[[sizeB1:.*]] = affine.min #[[BOUND4_MAP]](%[[K]])[%[[dB1]]]
+//       CHECK:       %[[stB2:.*]] = subtensor %[[B]][0, %[[K]]] [%[[dB0]], %[[sizeB1]]] [1, 1]  : tensor<?x?xf32> to tensor<?x?xf32>
+//       CHECK:       %[[sizeC1:.*]] = affine.min #[[BOUND4_MAP]](%[[K]])[%[[dC1]]]
+//       CHECK:       %[[stC:.*]] = subtensor %[[C]][%[[I]], %[[K]]] [%[[sizeC0]], %[[sizeC1]]] [1, 1]  : tensor<?x?xf32> to tensor<?x?xf32>
+//       CHECK:       %[[stD:.*]] = linalg.matmul ins(%[[stA]], %[[stB2]] : tensor<?x?xf32>, tensor<?x?xf32>) outs(%[[stC]] : tensor<?x?xf32>)  -> tensor<?x?xf32>
+//       CHECK:       %[[CAST:.*]] = tensor.cast %[[stD]] : tensor<?x?xf32> to tensor<?x4xf32>
+//  CHECK-NEXT:       %[[stG:.*]] = linalg.matmul ins(%[[CAST]], %[[stB1]] : tensor<?x4xf32>, tensor<4x3xf32>) outs(%[[stF]] : tensor<2x3xf32>)  -> tensor<2x3xf32>
 //  CHECK-NEXT:       subtensor_insert %[[stG]] into %[[RES]][%[[I]], %[[J]]]
+
+// -----
+
+func @conv_tensors_static(%input: tensor<1x225x225x32xf32>, %filter: tensor<3x3x3x32xf32>, %elementwise: tensor<1x112x112x32xf32>) -> tensor<1x112x112x32xf32> {
+  %c112 = constant 112 : index
+  %c32 = constant 32 : index
+  %c16 = constant 16 : index
+  %c8 = constant 8 : index
+  %c4 = constant 4 : index
+  %c0 = constant 0 : index
+  %cst = constant 0.0 : f32
+
+  %init = linalg.init_tensor [1, 112, 112, 32] : tensor<1x112x112x32xf32>
+  %fill = linalg.fill(%init, %cst) : tensor<1x112x112x32xf32>, f32 -> tensor<1x112x112x32xf32>
+
+  %conv = linalg.conv_2d_input_nhwc_filter_hwcf
+    {dilations = dense<1> : tensor<2xi64>, strides = dense<2> : tensor<2xi64>}
+    ins(%input, %filter : tensor<1x225x225x32xf32>, tensor<3x3x3x32xf32>)
+    outs(%fill : tensor<1x112x112x32xf32>) -> tensor<1x112x112x32xf32>
+
+  %for0 = scf.for %iv0 = %c0 to %c112 step %c8 iter_args(%arg0 = %fill) -> tensor<1x112x112x32xf32> {
+    %for1 = scf.for %iv1 = %c0 to %c112 step %c16 iter_args(%arg1 = %arg0) -> tensor<1x112x112x32xf32> {
+      %for2 = scf.for %iv2 = %c0 to %c32 step %c4 iter_args(%arg2 = %arg1) -> tensor<1x112x112x32xf32> {
+        %0 = subtensor %conv[0, %iv0, %iv1, %iv2][1, 8, 16, 4][1, 1, 1, 1] : tensor<1x112x112x32xf32> to tensor<1x8x16x4xf32>
+        %1 = subtensor %elementwise[0, %iv0, %iv1, %iv2][1, 8, 16, 4][1, 1, 1, 1] : tensor<1x112x112x32xf32> to tensor<1x8x16x4xf32>
+        %2 = subtensor %arg2[0, %iv0, %iv1, %iv2][1, 8, 16, 4][1, 1, 1, 1] : tensor<1x112x112x32xf32> to tensor<1x8x16x4xf32>
+        %add = linalg.generic
+          {
+            indexing_maps = [
+              affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>,
+              affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>,
+              affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>],
+            iterator_types = ["parallel", "parallel", "parallel", "parallel"]
+          }
+          ins(%0, %1 : tensor<1x8x16x4xf32>, tensor<1x8x16x4xf32>) outs(%2 : tensor<1x8x16x4xf32>) {
+        ^bb0(%arg3: f32, %arg4: f32, %arg5: f32):
+          %result = addf %arg3, %arg4 : f32
+          linalg.yield %result : f32
+        } -> tensor<1x8x16x4xf32>
+
+        %insert = subtensor_insert %add into %arg2[0, %iv0, %iv1, %iv2] [1, 8, 16, 4] [1, 1, 1, 1]  : tensor<1x8x16x4xf32> into tensor<1x112x112x32xf32>
+        scf.yield %insert : tensor<1x112x112x32xf32>
+      }
+      scf.yield %for2 : tensor<1x112x112x32xf32>
+    }
+    scf.yield %for1 : tensor<1x112x112x32xf32>
+  }
+  return %for0 : tensor<1x112x112x32xf32>
+}
+
+//      CHECK: #[[MAP0:.+]] = affine_map<(d0) -> (d0 * 2)>
+//      CHECK: #[[MAP1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+
+//      CHECK: func @conv_tensors_static
+// CHECK-SAME: (%[[INPUT:.+]]: tensor<1x225x225x32xf32>, %[[FILTER:.+]]: tensor<3x3x3x32xf32>, %[[ELEM:.+]]: tensor<1x112x112x32xf32>)
+
+//      CHECK: %[[INIT:.+]] = linalg.init_tensor [1, 112, 112, 32] : tensor<1x112x112x32xf32>
+// CHECK-NEXT: %[[FILL:.+]] = linalg.fill(%[[INIT]], %cst) : tensor<1x112x112x32xf32>, f32 -> tensor<1x112x112x32xf32>
+
+// CHECK-NEXT: scf.for %[[IV0:.+]] = %{{.+}} to %{{.+}} step %{{.+}} iter_args(%[[ARG0:.+]] = %[[FILL]])
+// CHECK-NEXT:   %[[OFFSET_H:.+]] = affine.apply #[[MAP0]](%[[IV0]])
+// CHECK-NEXT:   scf.for %[[IV1:.+]] = %{{.+}} to %{{.+}} step %{{.+}} iter_args(%[[ARG1:.+]] = %[[ARG0]])
+// CHECK-NEXT:     %[[OFFSET_W:.+]] = affine.apply #[[MAP0]](%[[IV1]])
+// CHECK-NEXT:     %[[ST_INPUT:.+]] = subtensor %arg0[0, %[[OFFSET_H]], %[[OFFSET_W]], 0] [1, 17, 33, 32] [1, 1, 1, 1] : tensor<1x225x225x32xf32> to tensor<1x17x33x32xf32>
+// CHECK-NEXT:     scf.for %[[IV2:.+]] = %{{.+}} to %{{.+}} step %{{.+}} iter_args(%[[ARG2:.+]] = %[[ARG1]])
+// CHECK-NEXT:       %[[ST_ELEM:.+]] = subtensor %[[ELEM]][0, %[[IV0]], %[[IV1]], %[[IV2]]] [1, 8, 16, 4] [1, 1, 1, 1] : tensor<1x112x112x32xf32> to tensor<1x8x16x4xf32>
+// CHECK-NEXT:       %[[ST_ARG2:.+]] = subtensor %[[ARG2]][0, %[[IV0]], %[[IV1]], %[[IV2]]] [1, 8, 16, 4] [1, 1, 1, 1] : tensor<1x112x112x32xf32> to tensor<1x8x16x4xf32>
+// CHECK-NEXT:       %[[ST_FILTER:.+]] = subtensor %[[FILTER]][0, 0, 0, %[[IV2]]] [3, 3, 3, 4] [1, 1, 1, 1] : tensor<3x3x3x32xf32> to tensor<3x3x3x4xf32>
+// CHECK-NEXT:       %[[ST_FILL:.+]] = subtensor %[[FILL]][0, %[[IV0]], %[[IV1]], %[[IV2]]] [1, 8, 16, 4] [1, 1, 1, 1] : tensor<1x112x112x32xf32> to tensor<1x8x16x4xf32>
+// CHECK-NEXT:       %[[ST_CONV:.+]] = linalg.conv_2d_input_nhwc_filter_hwcf
+// CHECK-SAME:         ins(%[[ST_INPUT]], %[[ST_FILTER]] : tensor<1x17x33x32xf32>, tensor<3x3x3x4xf32>)
+// CHECK-SAME:         outs(%[[ST_FILL]] : tensor<1x8x16x4xf32>)
+// CHECK-NEXT:       %[[ADD:.+]] = linalg.generic
+// CHECK-SAME:         ins(%[[ST_CONV]], %[[ST_ELEM]] : tensor<1x8x16x4xf32>, tensor<1x8x16x4xf32>)
+// CHECK-SAME:         outs(%[[ST_ARG2]] : tensor<1x8x16x4xf32>)
+//      CHECK:       subtensor_insert %[[ADD]] into %[[ARG2]][0, %[[IV0]], %[[IV1]], %[[IV2]]] [1, 8, 16, 4]
+
+// -----
+
+#bound4_map = affine_map<(d0)[s0] -> (4, -d0 + s0)>
+#bound8_map = affine_map<(d0)[s0] -> (8, -d0 + s0)>
+#bound16_map = affine_map<(d0)[s0] -> (16, -d0 + s0)>
+
+func @conv_tensors_dynamic(%input: tensor<?x?x?x?xf32>, %filter: tensor<?x?x?x?xf32>, %elementwise: tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32> {
+  %cst = constant 0.0 : f32
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %c2 = constant 2 : index
+  %c3 = constant 3 : index
+  %c4 = constant 4 : index
+  %c8 = constant 8 : index
+  %c16 = constant 16 : index
+
+  %n = memref.dim %elementwise, %c0 : tensor<?x?x?x?xf32>
+  %oh = memref.dim %elementwise, %c1 : tensor<?x?x?x?xf32>
+  %ow = memref.dim %elementwise, %c2 : tensor<?x?x?x?xf32>
+  %oc = memref.dim %elementwise, %c3 : tensor<?x?x?x?xf32>
+
+  %init = linalg.init_tensor [%n, %oh, %ow, %oc] : tensor<?x?x?x?xf32>
+  %fill = linalg.fill(%init, %cst) : tensor<?x?x?x?xf32>, f32 -> tensor<?x?x?x?xf32>
+
+  %conv = linalg.conv_2d_input_nhwc_filter_hwcf
+    {dilations = dense<1> : tensor<2xi64>, strides = dense<2> : tensor<2xi64>}
+    ins(%input, %filter : tensor<?x?x?x?xf32>, tensor<?x?x?x?xf32>)
+    outs(%fill : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+
+  %for0 = scf.for %iv0 = %c0 to %oh step %c8 iter_args(%arg0 = %fill) -> tensor<?x?x?x?xf32> {
+    %for1 = scf.for %iv1 = %c0 to %ow step %c16 iter_args(%arg1 = %arg0) -> tensor<?x?x?x?xf32> {
+      %for2 = scf.for %iv2 = %c0 to %oc step %c4 iter_args(%arg2 = %arg1) -> tensor<?x?x?x?xf32> {
+        %for3 = scf.for %iv3 = %c0 to %oc step %c2 iter_args(%arg3 = %arg2) -> tensor<?x?x?x?xf32> {
+          %n_size = affine.min #bound8_map(%iv0)[%n]
+          %oh_size = affine.min #bound16_map(%iv1)[%oh]
+          %ow_size = affine.min #bound4_map(%iv2)[%ow]
+          %oc_size = affine.min #bound4_map(%iv2)[%oc]
+          %0 = subtensor %conv[%iv0, %iv1, %iv2, %iv3][%n_size, %oh_size, %ow_size, %oc_size][1, 1, 1, 1] : tensor<?x?x?x?xf32> to tensor<?x?x?x?xf32>
+          %1 = subtensor %elementwise[%iv0, %iv1, %iv2, %iv3][%n_size, %oh_size, %ow_size, %oc_size][1, 1, 1, 1] : tensor<?x?x?x?xf32> to tensor<?x?x?x?xf32>
+          %2 = subtensor %arg3[%iv0, %iv1, %iv2, %iv3][%n_size, %oh_size, %ow_size, %oc_size][1, 1, 1, 1] : tensor<?x?x?x?xf32> to tensor<?x?x?x?xf32>
+          %add = linalg.generic
+            {
+              indexing_maps = [
+                affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>,
+                affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>,
+                affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>],
+              iterator_types = ["parallel", "parallel", "parallel", "parallel"]
+            }
+            ins(%0, %1 : tensor<?x?x?x?xf32>, tensor<?x?x?x?xf32>) outs(%2 : tensor<?x?x?x?xf32>) {
+          ^bb0(%arg4: f32, %arg5: f32, %arg6: f32):
+            %result = addf %arg4, %arg5 : f32
+            linalg.yield %result : f32
+          } -> tensor<?x?x?x?xf32>
+
+          %insert = subtensor_insert %add into %arg3[%iv0, %iv1, %iv2, %iv3] [%n_size, %oh_size, %ow_size, %oc_size] [1, 1, 1, 1]  : tensor<?x?x?x?xf32> into tensor<?x?x?x?xf32>
+          scf.yield %insert : tensor<?x?x?x?xf32>
+        }
+        scf.yield %for3 : tensor<?x?x?x?xf32>
+      }
+      scf.yield %for2 : tensor<?x?x?x?xf32>
+    }
+    scf.yield %for1 : tensor<?x?x?x?xf32>
+  }
+  return %for0 : tensor<?x?x?x?xf32>
+}
+
+// -----
+
+// CHECK: #[[BOUND8_MAP:.+]] = affine_map<(d0)[s0] -> (8, -d0 + s0)>
+// CHECK: #[[BOUND8_MAP_2:.+]] = affine_map<(d0)[s0, s1] -> (-d0 + s0, 8, -d0 + s1)>
+// CHECK: #[[BOUND8_MAP_3:.+]] = affine_map<(d0)[s0] -> (-d0 + s0, 8)>
+// CHECK: #[[BOUND16_MAP:.+]] = affine_map<(d0)[s0] -> (16, -d0 + s0)>
+// CHECK: #[[X2_MAP:.+]] = affine_map<(d0) -> (d0 * 2)>
+// CHECK: #[[INPUT_BOUND:.+]] = affine_map<(d0, d1)[s0, s1] -> (d0 * 2 + s0 - 2, d1 * -2 + s1)>
+// CHECK: #[[BOUND16_MAP_2:.+]] = affine_map<(d0)[s0] -> (-d0 + s0, 16)>
+// CHECK: #[[BOUND4_MAP:.+]] = affine_map<(d0)[s0] -> (4, -d0 + s0)>
+// CHECK: #[[BOUND4_MAP_2:.+]] = affine_map<(d0)[s0] -> (-d0 + s0, 4)>
+// CHECK: #[[BOUND4_MAP_3:.+]] = affine_map<(d0, d1)[s0, s1] -> (-d0 + s0, 4, -d1 + s1)>
+// CHECK: #[[BOUND4_MAP_4:.+]] = affine_map<(d0, d1)[s0] -> (-d0 + s0, 4, -d1 + s0)>
+
+//      CHECK: func @conv_tensors_dynamic
+// CHECK-SAME: (%[[INPUT]]: tensor<?x?x?x?xf32>, %[[FILTER]]: tensor<?x?x?x?xf32>, %[[ELEM]]: tensor<?x?x?x?xf32>)
+
+//  CHECK-DAG:   %[[C0:.+]] = constant 0 : index
+//  CHECK-DAG:   %[[C1:.+]] = constant 1 : index
+//  CHECK-DAG:   %[[C2:.+]] = constant 2 : index
+//  CHECK-DAG:   %[[C3:.+]] = constant 3 : index
+
+//  CHECK-DAG:   %[[ELEM_N:.+]] = memref.dim %[[ELEM]], %[[C0]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[ELEM_OH:.+]] = memref.dim %[[ELEM]], %[[C1]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[ELEM_OW:.+]] = memref.dim %[[ELEM]], %[[C2]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[ELEM_OC:.+]] = memref.dim %[[ELEM]], %[[C3]] : tensor<?x?x?x?xf32>
+
+//      CHECK:   %[[INIT:.+]] = linalg.init_tensor [%[[ELEM_N]], %[[ELEM_OH]], %[[ELEM_OW]], %[[ELEM_OC]]] : tensor<?x?x?x?xf32>
+//      CHECK:   %[[FILL:.+]] = linalg.fill(%[[INIT]], %cst) : tensor<?x?x?x?xf32>, f32 -> tensor<?x?x?x?xf32>
+
+//  CHECK-DAG:   %[[FILTER_H:.+]] = memref.dim %[[FILTER]], %[[C0]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[FILTER_W:.+]] = memref.dim %[[FILTER]], %[[C1]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[INPUT_N:.+]] = memref.dim %[[INPUT]], %[[C0]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[INPUT_H:.+]] = memref.dim %[[INPUT]], %[[C1]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[INPUT_W:.+]] = memref.dim %[[INPUT]], %[[C2]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[INPUT_C:.+]] = memref.dim %[[INPUT]], %[[C3]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[FILTER_IC:.+]] = memref.dim %[[FILTER]], %[[C2]] : tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[FILTER_OC:.+]] = memref.dim %[[FILTER]], %[[C3]] : tensor<?x?x?x?xf32>
+
+//      CHECK:   scf.for %[[IV0:.+]] = %{{.+}} to %[[ELEM_OH]] step %{{.+}} iter_args(%{{.+}} = %[[FILL]])
+// CHECK-NEXT:     %[[SIZE_ELEM_N:.+]] = affine.min #[[BOUND8_MAP]](%[[IV0]])[%[[ELEM_N]]]
+// CHECK-NEXT:     %[[SIZE_INPUT_N:.+]] = affine.min #[[BOUND8_MAP_2]](%[[IV0]])[%[[INPUT_N]], %[[ELEM_N]]]
+// CHECK-NEXT:     %[[SIZE_ELEM_N_2:.+]] = affine.min #[[BOUND8_MAP_3]](%[[IV0]])[%[[ELEM_N]]]
+// CHECK-NEXT:     scf.for %[[IV1:.+]] = %{{.+}} to %[[ELEM_OW]]
+// CHECK-NEXT:       %[[SIZE_ELEM_OH:.+]] = affine.min #[[BOUND16_MAP]](%[[IV1]])[%[[ELEM_OH]]]
+// CHECK-NEXT:       %[[OFFSET_OH:.+]] = affine.apply #[[X2_MAP]](%[[IV1]])
+// CHECK-NEXT:       %[[SIZE_INPUT_H:.+]] = affine.min #[[INPUT_BOUND]](%[[SIZE_ELEM_OH]], %[[IV1]])[%[[FILTER_H]], %[[INPUT_H]]]
+// CHECK-NEXT:       %[[SIZE_ELEM_OH_2:.+]] = affine.min #[[BOUND16_MAP_2]](%[[IV1]])[%[[ELEM_OH]]]
+// CHECK-NEXT:       scf.for %[[IV2:.+]] = %{{.+}} to %[[ELEM_OC]]
+// CHECK-NEXT:         %[[SIZE_ELEM_OW:.+]] = affine.min #[[BOUND4_MAP]](%[[IV2]])[%[[ELEM_OW]]]
+// CHECK-NEXT:         %[[SIZE_ELEM_OC:.+]] = affine.min #[[BOUND4_MAP]](%[[IV2]])[%[[ELEM_OC]]]
+// CHECK-NEXT:         %[[OFFSET_OW:.+]] = affine.apply #[[X2_MAP]](%[[IV2]])
+// CHECK-NEXT:         %[[SIZE_INPUT_W:.+]] = affine.min #[[INPUT_BOUND]](%[[SIZE_ELEM_OW]], %[[IV2]])[%[[FILTER_W]], %[[INPUT_W]]]
+// CHECK-NEXT:         %[[ST_INPUT:.+]] = subtensor %[[INPUT]][%[[IV0]], %[[OFFSET_OH]], %[[OFFSET_OW]], 0]
+// CHECK-SAME:               [%[[SIZE_INPUT_N]], %[[SIZE_INPUT_H]], %[[SIZE_INPUT_W]], %[[INPUT_C]]]
+// CHECK-NEXT:         %[[SIZE_ELEM_OW_2:.+]] = affine.min #[[BOUND4_MAP_2]](%[[IV2]])[%[[ELEM_OW]]]
+// CHECK-NEXT:         scf.for %[[IV3:.+]] = %{{.+}} to %[[ELEM_OC]] step %{{.+}} iter_args(%[[ARG:[a-z0-9]+]]
+// CHECK-NEXT:           %[[ST_ELEM:.+]] = subtensor %[[ELEM]][%[[IV0]], %[[IV1]], %[[IV2]], %[[IV3]]]
+// CHECK-SAME:                 [%[[SIZE_ELEM_N]], %[[SIZE_ELEM_OH]], %[[SIZE_ELEM_OW]], %[[SIZE_ELEM_OC]]]
+// CHECK-NEXT:           %[[ST_ARG:.+]] = subtensor %[[ARG]][%[[IV0]], %[[IV1]], %[[IV2]], %[[IV3]]]
+// CHECK-SAME:                 [%[[SIZE_ELEM_N]], %[[SIZE_ELEM_OH]], %[[SIZE_ELEM_OW]], %[[SIZE_ELEM_OC]]]
+// CHECK-NEXT:           %[[SIZE_ELEM_OC_2:.+]] = affine.min #[[BOUND4_MAP_3]](%[[IV3]], %[[IV2]])[%[[FILTER_OC]], %[[ELEM_OC]]]
+// CHECK-NEXT:           %[[ST_FILTER:.+]] = subtensor %[[FILTER]][0, 0, 0, %[[IV3]]]
+// CHECK-SAME:                 [%[[FILTER_H]], %[[FILTER_W]], %[[FILTER_IC]], %[[SIZE_ELEM_OC_2]]]
+// CHECK-NEXT:           %[[SIZE_ELEM_OC_3:.+]] = affine.min #[[BOUND4_MAP_4]](%[[IV3]], %[[IV2]])[%[[ELEM_OC]]]
+// CHECK-NEXT:           %[[ST_FILL:.+]] = subtensor %[[FILL]][%[[IV0]], %[[IV1]], %[[IV2]], %[[IV3]]]
+// CHECK-SAME:                 [%[[SIZE_ELEM_N_2]], %[[SIZE_ELEM_OH_2]], %[[SIZE_ELEM_OW_2]], %[[SIZE_ELEM_OC_3]]]
+// CHECK-NEXT:           %[[ST_CONV:.+]] = linalg.conv_2d_input_nhwc_filter_hwcf
+// CHECK-SAME:                 ins(%[[ST_INPUT]], %[[ST_FILTER]] : tensor<?x?x?x?xf32>, tensor<?x?x?x?xf32>)
+// CHECK-SAME:                 outs(%[[ST_FILL]] : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+// CHECK-NEXT:           %[[ST_ADD:.+]] = linalg.generic
+// CHECK-SAME:                 ins(%[[ST_CONV]], %[[ST_ELEM]] : tensor<?x?x?x?xf32>, tensor<?x?x?x?xf32>)
+// CHECK-SAME:                 outs(%[[ST_ARG]] : tensor<?x?x?x?xf32>)
+//      CHECK:           subtensor_insert %[[ST_ADD]] into %[[ARG]][%[[IV0]], %[[IV1]], %[[IV2]], %[[IV3]]]
+// CHECK-SAME:                 [%[[SIZE_ELEM_N]], %[[SIZE_ELEM_OH]], %[[SIZE_ELEM_OW]], %[[SIZE_ELEM_OC]]]
diff --git a/mlir/test/Dialect/Linalg/vectorization.mlir b/mlir/test/Dialect/Linalg/vectorization.mlir
index c43bf07d775d4ab03f8197ac97c661b1262156f0..74ff4367724e56e157462017bb2d3c841e3236a9 100644
--- a/mlir/test/Dialect/Linalg/vectorization.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization.mlir
@@ -341,6 +341,42 @@ func @generic_vectorize_tensor(%arg0: tensor<4x256xf32>,
 
 // -----
 
+// Test different input maps.
+#matmul_trait = {
+  indexing_maps = [
+    affine_map<(d0, d1, d2, d3) -> (d1, d0)>,
+    affine_map<(d0, d1, d2, d3) -> (d3, d1)>,
+    affine_map<(d0, d1, d2, d3) -> (d3, d1, d0, d2)>,
+    affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+  ],
+  iterator_types = ["parallel", "parallel", "parallel", "parallel"]
+}
+
+// CHECK-DAG: #[[MAP0:.*]] = affine_map<(d0, d1) -> (d1, d0, 0, 0)>
+// CHECK-DAG: #[[MAP1:.*]] = affine_map<(d0, d1) -> (0, d1, 0, d0)>
+// CHECK-DAG: #[[MAP2:.*]] = affine_map<(d0, d1, d2, d3) -> (d2, d1, d3, d0)>
+//       CHECK: func @vectorization_transpose
+//       CHECK: vector.transfer_read {{.*}}{permutation_map = #[[MAP0]]} : memref<14x7xf32>, vector<7x14x8x16xf32>
+//       CHECK: vector.transfer_read {{.*}}{permutation_map = #[[MAP1]]} : memref<16x14xf32>, vector<7x14x8x16xf32>
+//       CHECK: vector.transfer_read {{.*}}{permutation_map = #[[MAP2]]} : memref<16x14x7x8xf32>, vector<7x14x8x16xf32>
+//       CHECK: addf {{.*}} : vector<7x14x8x16xf32>
+//       CHECK: addf {{.*}} : vector<7x14x8x16xf32>
+//       CHECK: vector.transfer_write {{.*}} : vector<7x14x8x16xf32>, memref<7x14x8x16xf32>
+func @vectorization_transpose(%A: memref<14x7xf32>, %B: memref<16x14xf32>,
+                         %C: memref<16x14x7x8xf32>, %D: memref<7x14x8x16xf32>) {
+  linalg.generic #matmul_trait
+    ins(%A, %B, %C : memref<14x7xf32>, memref<16x14xf32>, memref<16x14x7x8xf32>)
+   outs(%D : memref<7x14x8x16xf32>) {
+    ^bb(%a: f32, %b: f32, %c: f32, %d: f32) :
+      %e = addf %a, %b: f32
+      %f = addf %e, %c: f32
+      linalg.yield %f : f32
+  }
+  return
+}
+
+// -----
+
 // CHECK-LABEL: func @matmul_tensors
 //  CHECK-SAME: (%[[ARG0:.*]]: tensor<8x4xf32>, %[[ARG1:.*]]: tensor<4x12xf32>,
 //  CHECK-SAME:  %[[ARG2:.*]]: tensor<8x12xf32>) -> tensor<8x12xf32>
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 6b9be10c7693a7c237513d6eeb78bfa1005c2193..8f7f9c1ca69ca53532287098b411650e04170114 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -89,77 +89,192 @@ func @omp_parallel(%data_var : memref<i32>, %if_cond : i1, %num_threads : si32)
 }
 
 func @omp_parallel_pretty(%data_var : memref<i32>, %if_cond : i1, %num_threads : si32, %allocator : si32) -> () {
-  // CHECK: omp.parallel
-  omp.parallel {
-    omp.terminator
-  }
+ // CHECK: omp.parallel
+ omp.parallel {
+  omp.terminator
+ }
+
+ // CHECK: omp.parallel num_threads(%{{.*}} : si32)
+ omp.parallel num_threads(%num_threads : si32) {
+   omp.terminator
+ }
+
+ // CHECK: omp.parallel allocate(%{{.*}} : memref<i32> -> %{{.*}} : memref<i32>)
+ omp.parallel allocate(%data_var : memref<i32> -> %data_var : memref<i32>) {
+   omp.terminator
+ }
+
+ // CHECK: omp.parallel private(%{{.*}} : memref<i32>, %{{.*}} : memref<i32>) firstprivate(%{{.*}} : memref<i32>)
+ omp.parallel private(%data_var : memref<i32>, %data_var : memref<i32>) firstprivate(%data_var : memref<i32>) {
+   omp.terminator
+ }
+
+ // CHECK omp.parallel shared(%{{.*}} : memref<i32>) copyin(%{{.*}} : memref<i32>, %{{.*}} : memref<i32>)
+ omp.parallel shared(%data_var : memref<i32>) copyin(%data_var : memref<i32>, %data_var : memref<i32>) {
+   omp.parallel if(%if_cond: i1) {
+     omp.terminator
+   }
+   omp.terminator
+ }
+
+ // CHECK omp.parallel if(%{{.*}}) num_threads(%{{.*}} : si32) private(%{{.*}} : memref<i32>) proc_bind(close)
+ omp.parallel num_threads(%num_threads : si32) if(%if_cond: i1)
+              private(%data_var : memref<i32>) proc_bind(close) {
+   omp.terminator
+ }
+
+ return
+}
 
-  // CHECK: omp.parallel num_threads(%{{.*}} : si32)
-  omp.parallel num_threads(%num_threads : si32) {
-    omp.terminator
-  }
+// CHECK-LABEL: omp_wsloop
+func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memref<i32>, %linear_var : i32, %chunk_var : i32) -> () {
 
-  // CHECK: omp.parallel allocate(%{{.*}} : memref<i32> -> %{{.*}} : memref<i32>)
-  omp.parallel allocate(%data_var : memref<i32> -> %data_var : memref<i32>) {
-    omp.terminator
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) private(%{{.*}} : memref<i32>, %{{.*}} : memref<i32>) collapse(2) ordered(1)
+  "omp.wsloop" (%lb, %ub, %step, %data_var, %data_var) ({
+    ^bb0(%iv: index):
+      omp.yield
+  }) {operand_segment_sizes = dense<[1,1,1,2,0,0,0,0,0]> : vector<9xi32>, collapse_val = 2, ordered_val = 1} :
+    (index, index, index, memref<i32>, memref<i32>) -> ()
+
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) linear(%{{.*}} = %{{.*}} : memref<i32>) schedule(static)
+  "omp.wsloop" (%lb, %ub, %step, %data_var, %linear_var) ({
+    ^bb0(%iv: index):
+      omp.yield
+  }) {operand_segment_sizes = dense<[1,1,1,0,0,0,1,1,0]> : vector<9xi32>, schedule_val = "Static"} :
+    (index, index, index, memref<i32>, i32) -> ()
+
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) linear(%{{.*}} = %{{.*}} : memref<i32>, %{{.*}} = %{{.*}} : memref<i32>) schedule(static)
+  "omp.wsloop" (%lb, %ub, %step, %data_var, %data_var, %linear_var, %linear_var) ({
+    ^bb0(%iv: index):
+      omp.yield
+  }) {operand_segment_sizes = dense<[1,1,1,0,0,0,2,2,0]> : vector<9xi32>, schedule_val = "Static"} :
+    (index, index, index, memref<i32>, memref<i32>, i32, i32) -> ()
+
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) private(%{{.*}} : memref<i32>) firstprivate(%{{.*}} : memref<i32>) lastprivate(%{{.*}} : memref<i32>) linear(%{{.*}} = %{{.*}} : memref<i32>) schedule(dynamic = %{{.*}}) collapse(3) ordered(2)
+  "omp.wsloop" (%lb, %ub, %step, %data_var, %data_var, %data_var, %data_var, %linear_var, %chunk_var) ({
+    ^bb0(%iv: index):
+      omp.yield
+  }) {operand_segment_sizes = dense<[1,1,1,1,1,1,1,1,1]> : vector<9xi32>, schedule_val = "Dynamic", collapse_val = 3, ordered_val = 2} :
+    (index, index, index, memref<i32>, memref<i32>, memref<i32>, memref<i32>, i32, i32) -> ()
+
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) private(%{{.*}} : memref<i32>) schedule(auto) nowait
+  "omp.wsloop" (%lb, %ub, %step, %data_var) ({
+    ^bb0(%iv: index):
+      omp.yield
+  }) {operand_segment_sizes = dense<[1,1,1,1,0,0,0,0,0]> : vector<9xi32>, nowait, schedule_val = "Auto"} :
+    (index, index, index, memref<i32>) -> ()
+
+  return
+}
+
+// CHECK-LABEL: omp_wsloop_pretty
+func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index,
+                 %data_var : memref<i32>, %linear_var : i32, %chunk_var : i32) -> () {
+
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) private(%{{.*}} : memref<i32>)
+  omp.wsloop (%iv) : index = (%lb) to (%ub) step (%step) private(%data_var : memref<i32>) collapse(2) ordered(2) {
+    omp.yield
   }
 
-  // CHECK: omp.parallel private(%{{.*}} : memref<i32>, %{{.*}} : memref<i32>) firstprivate(%{{.*}} : memref<i32>)
-  omp.parallel private(%data_var : memref<i32>, %data_var : memref<i32>) firstprivate(%data_var : memref<i32>) {
-    omp.terminator
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) linear(%{{.*}} = %{{.*}} : memref<i32>) schedule(static)
+  omp.wsloop (%iv) : index = (%lb) to (%ub) step (%step) schedule(static) lastprivate(%data_var : memref<i32>) linear(%data_var = %linear_var : memref<i32>) {
+    omp.yield
   }
 
-  // CHECK omp.parallel shared(%{{.*}} : memref<i32>) copyin(%{{.*}} : memref<i32>, %{{.*}} : memref<i32>)
-  omp.parallel shared(%data_var : memref<i32>) copyin(%data_var : memref<i32>, %data_var : memref<i32>) {
-    omp.parallel if(%if_cond: i1) {
-      omp.terminator
-    }
-    omp.terminator
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) private(%{{.*}} : memref<i32>) firstprivate(%{{.*}} : memref<i32>) lastprivate(%{{.*}} : memref<i32>) linear(%{{.*}} = %{{.*}} : memref<i32>) schedule(static = %{{.*}}) collapse(3) ordered(2)
+  omp.wsloop (%iv) : index = (%lb) to (%ub) step (%step) ordered(2) private(%data_var : memref<i32>)
+     firstprivate(%data_var : memref<i32>) lastprivate(%data_var : memref<i32>) linear(%data_var = %linear_var : memref<i32>)
+     schedule(static = %chunk_var) collapse(3) {
+    omp.yield
   }
 
-  // CHECK omp.parallel if(%{{.*}}) num_threads(%{{.*}} : si32) private(%{{.*}} : memref<i32>) proc_bind(close)
-  omp.parallel num_threads(%num_threads : si32) if(%if_cond: i1)
-               private(%data_var : memref<i32>) proc_bind(close) {
-    omp.terminator
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) private({{.*}} : memref<i32>)
+  omp.wsloop (%iv) : index = (%lb) to (%ub) step (%step) private(%data_var : memref<i32>) {
+    omp.yield
   }
 
   return
 }
 
-func @omp_wsloop(%lb : index, %ub : index, %step : index,
-                 %data_var : memref<i32>, %linear_var : si32, %chunk_var : si32) -> () {
+// CHECK-LABEL: omp_wsloop_pretty_multi_block
+func @omp_wsloop_pretty_multi_block(%lb : index, %ub : index, %step : index, %data1 : memref<?xi32>, %data2 : memref<?xi32>) -> () {
 
-  // CHECK: "omp.wsloop"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}})
-  "omp.wsloop" (%lb, %ub, %step, %data_var) ({
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}})
+  omp.wsloop (%iv) : index = (%lb) to (%ub) step (%step) {
+    %1 = "test.payload"(%iv) : (index) -> (i32)
+    br ^bb1(%1: i32)
+  ^bb1(%arg: i32):
+    memref.store %arg, %data1[%iv] : memref<?xi32>
     omp.yield
-  }) {operand_segment_sizes = dense<[1,1,1,1,0,0,0,0,0]> : vector<9xi32>, collapse_val = 2, ordered_val = 1} :
-    (index, index, index, memref<i32>) -> ()
+  }
 
-  // CHECK: "omp.wsloop"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}})
-  "omp.wsloop" (%lb, %lb, %ub, %ub, %step, %step, %data_var) ({
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}})
+  omp.wsloop (%iv) : index = (%lb) to (%ub) step (%step) {
+    %c = "test.condition"(%iv) : (index) -> (i1)
+    %v1 = "test.payload"(%iv) : (index) -> (i32)
+    cond_br %c, ^bb1(%v1: i32), ^bb2(%v1: i32)
+  ^bb1(%arg0: i32):
+    memref.store %arg0, %data1[%iv] : memref<?xi32>
+    br ^bb3
+  ^bb2(%arg1: i32):
+    memref.store %arg1, %data2[%iv] : memref<?xi32>
+    br ^bb3
+  ^bb3:
     omp.yield
-  }) {operand_segment_sizes = dense<[2,2,2,1,0,0,0,0,0]> : vector<9xi32>, collapse_val = 2, ordered_val = 1} :
-    (index, index, index, index, index, index, memref<i32>) -> ()
-
+  }
 
-  // CHECK: "omp.wsloop"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}})
-  "omp.wsloop" (%lb, %ub, %step, %data_var, %linear_var) ({
+  // CHECK: omp.wsloop (%{{.*}}) : index = (%{{.*}}) to (%{{.*}}) step (%{{.*}})
+  omp.wsloop (%iv) : index = (%lb) to (%ub) step (%step) {
+    %c = "test.condition"(%iv) : (index) -> (i1)
+    %v1 = "test.payload"(%iv) : (index) -> (i32)
+    cond_br %c, ^bb1(%v1: i32), ^bb2(%v1: i32)
+  ^bb1(%arg0: i32):
+    memref.store %arg0, %data1[%iv] : memref<?xi32>
     omp.yield
-  }) {operand_segment_sizes = dense<[1,1,1,0,0,0,1,1,0]> : vector<9xi32>, schedule_val = "Static"} :
-    (index, index, index, memref<i32>, si32) -> ()
+  ^bb2(%arg1: i32):
+    memref.store %arg1, %data2[%iv] : memref<?xi32>
+    omp.yield
+  }
 
-  // CHECK: "omp.wsloop"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}})
-  "omp.wsloop" (%lb, %ub, %step, %data_var, %data_var, %data_var, %data_var, %linear_var, %chunk_var) ({
+  return
+}
+
+// CHECK-LABEL: omp_wsloop_pretty_non_index
+func @omp_wsloop_pretty_non_index(%lb1 : i32, %ub1 : i32, %step1 : i32, %lb2 : i64, %ub2 : i64, %step2 : i64,
+                           %data1 : memref<?xi32>, %data2 : memref<?xi64>) -> () {
+
+  // CHECK: omp.wsloop (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) step (%{{.*}})
+  omp.wsloop (%iv1) : i32 = (%lb1) to (%ub1) step (%step1) {
+    %1 = "test.payload"(%iv1) : (i32) -> (index)
+    br ^bb1(%1: index)
+  ^bb1(%arg1: index):
+    memref.store %iv1, %data1[%arg1] : memref<?xi32>
     omp.yield
-  }) {operand_segment_sizes = dense<[1,1,1,1,1,1,1,1,1]> : vector<9xi32>, schedule_val = "Dynamic", collapse_val = 3, ordered_val = 2} :
-    (index, index, index, memref<i32>, memref<i32>, memref<i32>, memref<i32>, si32, si32) -> ()
+  }
 
-  // CHECK: "omp.wsloop"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}})
-  "omp.wsloop" (%lb, %ub, %step, %data_var) ({
+  // CHECK: omp.wsloop (%{{.*}}) : i64 = (%{{.*}}) to (%{{.*}}) step (%{{.*}})
+  omp.wsloop (%iv2) : i64 = (%lb2) to (%ub2) step (%step2) {
+    %2 = "test.payload"(%iv2) : (i64) -> (index)
+    br ^bb1(%2: index)
+  ^bb1(%arg2: index):
+    memref.store %iv2, %data2[%arg2] : memref<?xi64>
     omp.yield
-  }) {operand_segment_sizes = dense<[1,1,1,1,0,0,0,0,0]> : vector<9xi32>, nowait, schedule_val = "Auto"} :
-    (index, index, index, memref<i32>) -> ()
+  }
+
+  return
+}
 
+// CHECK-LABEL: omp_wsloop_pretty_multiple
+func @omp_wsloop_pretty_multiple(%lb1 : i32, %ub1 : i32, %step1 : i32, %lb2 : i32, %ub2 : i32, %step2 : i32, %data1 : memref<?xi32>) -> () {
+
+  // CHECK: omp.wsloop (%{{.*}}, %{{.*}}) : i32 = (%{{.*}}, %{{.*}}) to (%{{.*}}, %{{.*}}) step (%{{.*}}, %{{.*}})
+  omp.wsloop (%iv1, %iv2) : i32 = (%lb1, %lb2) to (%ub1, %ub2) step (%step1, %step2) {
+    %1 = "test.payload"(%iv1) : (i32) -> (index)
+    %2 = "test.payload"(%iv2) : (i32) -> (index)
+    memref.store %iv1, %data1[%1] : memref<?xi32>
+    memref.store %iv2, %data1[%2] : memref<?xi32>
+    omp.yield
+  }
 
   return
 }
diff --git a/mlir/test/Dialect/PDL/invalid.mlir b/mlir/test/Dialect/PDL/invalid.mlir
index 0f4d9677827721ea1a3fc815dd23aa6cfded6dde..e371d84086704eae817dcabf66a36a7b9d211627 100644
--- a/mlir/test/Dialect/PDL/invalid.mlir
+++ b/mlir/test/Dialect/PDL/invalid.mlir
@@ -1,19 +1,33 @@
 // RUN: mlir-opt %s -split-input-file -verify-diagnostics
 
 //===----------------------------------------------------------------------===//
-// pdl::ApplyConstraintOp
+// pdl::ApplyNativeConstraintOp
 //===----------------------------------------------------------------------===//
 
 pdl.pattern : benefit(1) {
   %op = pdl.operation "foo.op"
 
   // expected-error@below {{expected at least one argument}}
-  "pdl.apply_constraint"() {name = "foo", params = []} : () -> ()
+  "pdl.apply_native_constraint"() {name = "foo", params = []} : () -> ()
   pdl.rewrite %op with "rewriter"
 }
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// pdl::ApplyNativeRewriteOp
+//===----------------------------------------------------------------------===//
+
+pdl.pattern : benefit(1) {
+  %op = pdl.operation "foo.op"
+  pdl.rewrite %op {
+    // expected-error@below {{expected at least one argument}}
+    "pdl.apply_native_rewrite"() {name = "foo", params = []} : () -> ()
+  }
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // pdl::AttributeOp
 //===----------------------------------------------------------------------===//
@@ -24,7 +38,7 @@ pdl.pattern : benefit(1) {
   // expected-error@below {{expected only one of [`type`, `value`] to be set}}
   %attr = pdl.attribute : %type 10
 
-  %op, %result = pdl.operation "foo.op" {"attr" = %attr} -> %type
+  %op = pdl.operation "foo.op" {"attr" = %attr} -> (%type : !pdl.type)
   pdl.rewrite %op with "rewriter"
 }
 
@@ -76,6 +90,20 @@ pdl.pattern : benefit(1) {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// pdl::OperandsOp
+//===----------------------------------------------------------------------===//
+
+pdl.pattern : benefit(1) {
+  // expected-error@below {{expected a bindable (i.e. `pdl.operation`) user when defined in the matcher body of a `pdl.pattern`}}
+  %unused = pdl.operands
+
+  %op = pdl.operation "foo.op"
+  pdl.rewrite %op with "rewriter"
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // pdl::OperationOp
 //===----------------------------------------------------------------------===//
@@ -102,13 +130,13 @@ pdl.pattern : benefit(1) {
 // -----
 
 pdl.pattern : benefit(1) {
-  %op = pdl.operation "foo.op"()
+  %op = pdl.operation "foo.op"
   pdl.rewrite %op {
     %type = pdl.type
 
     // expected-error@below {{op must have inferable or constrained result types when nested within `pdl.rewrite`}}
     // expected-note@below {{result type #0 was not constrained}}
-    %newOp, %result = pdl.operation "foo.op" -> %type
+    %newOp = pdl.operation "foo.op" -> (%type : !pdl.type)
   }
 }
 
@@ -147,28 +175,12 @@ pdl.pattern : benefit(1) {
 
 // -----
 
-//===----------------------------------------------------------------------===//
-// pdl::ReplaceOp
-//===----------------------------------------------------------------------===//
-
-pdl.pattern : benefit(1) {
-  %root = pdl.operation "foo.op"
-  pdl.rewrite %root {
-    %type = pdl.type : i32
-    %newOp, %newResult = pdl.operation "foo.op" -> %type
-
-    // expected-error@below {{to have the same number of results as the replacement operation}}
-    pdl.replace %root with %newOp
-  }
-}
-
-// -----
-
 pdl.pattern : benefit(1) {
   %type = pdl.type : i32
-  %root, %oldResult = pdl.operation "foo.op" -> %type
+  %root = pdl.operation "foo.op" -> (%type : !pdl.type)
   pdl.rewrite %root {
-    %newOp, %newResult = pdl.operation "foo.op" -> %type
+    %newOp = pdl.operation "foo.op" -> (%type : !pdl.type)
+    %newResult = pdl.result 0 of %newOp
 
     // expected-error@below {{expected no replacement values to be provided when the replacement operation is present}}
     "pdl.replace"(%root, %newOp, %newResult) {
@@ -179,15 +191,15 @@ pdl.pattern : benefit(1) {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// pdl::ResultsOp
+//===----------------------------------------------------------------------===//
+
 pdl.pattern : benefit(1) {
   %root = pdl.operation "foo.op"
-  pdl.rewrite %root {
-    %type = pdl.type : i32
-    %newOp, %newResult = pdl.operation "foo.op" -> %type
-
-    // expected-error@below {{to have the same number of results as the provided replacement values}}
-    pdl.replace %root with (%newResult)
-  }
+  // expected-error@below {{expected `pdl.range<value>` result type when no index is specified, but got: '!pdl.value'}}
+  %results = "pdl.results"(%root) : (!pdl.operation) -> !pdl.value
+  pdl.rewrite %root with "rewriter"
 }
 
 // -----
@@ -252,3 +264,17 @@ pdl.pattern : benefit(1) {
   %op = pdl.operation "foo.op"
   pdl.rewrite %op with "rewriter"
 }
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl::TypesOp
+//===----------------------------------------------------------------------===//
+
+pdl.pattern : benefit(1) {
+  // expected-error@below {{expected a bindable (i.e. `pdl.operands`, or `pdl.operation`) user when defined in the matcher body of a `pdl.pattern`}}
+  %unused = pdl.types
+
+  %op = pdl.operation "foo.op"
+  pdl.rewrite %op with "rewriter"
+}
diff --git a/mlir/test/Dialect/PDL/ops.mlir b/mlir/test/Dialect/PDL/ops.mlir
index 5b6a642daf83f920829af3867d535661c09bf9f9..07e98f9e5868dda7b41baeacef22fe0d8bc6b8db 100644
--- a/mlir/test/Dialect/PDL/ops.mlir
+++ b/mlir/test/Dialect/PDL/ops.mlir
@@ -8,11 +8,12 @@ pdl.pattern @operations : benefit(1) {
   // Operation with attributes and results.
   %attribute = pdl.attribute
   %type = pdl.type
-  %op0, %op0_result = pdl.operation {"attr" = %attribute} -> %type
+  %op0 = pdl.operation {"attr" = %attribute} -> (%type : !pdl.type)
+  %op0_result = pdl.result 0 of %op0
 
   // Operation with input.
   %input = pdl.operand
-  %root = pdl.operation(%op0_result, %input)
+  %root = pdl.operation(%op0_result, %input : !pdl.value, !pdl.value)
   pdl.rewrite %root with "rewriter"
 }
 
@@ -20,7 +21,7 @@ pdl.pattern @operations : benefit(1) {
 
 pdl.pattern @rewrite_with_args : benefit(1) {
   %input = pdl.operand
-  %root = pdl.operation(%input)
+  %root = pdl.operation(%input : !pdl.value)
   pdl.rewrite %root with "rewriter"(%input : !pdl.value)
 }
 
@@ -35,7 +36,7 @@ pdl.pattern @rewrite_with_params : benefit(1) {
 
 pdl.pattern @rewrite_with_args_and_params : benefit(1) {
   %input = pdl.operand
-  %root = pdl.operation(%input)
+  %root = pdl.operation(%input : !pdl.value)
   pdl.rewrite %root with "rewriter"["I am param"](%input : !pdl.value)
 }
 
@@ -46,10 +47,10 @@ pdl.pattern @rewrite_with_args_and_params : benefit(1) {
 pdl.pattern @infer_type_from_operation_replace : benefit(1) {
   %type1 = pdl.type : i32
   %type2 = pdl.type
-  %root, %results:2 = pdl.operation -> %type1, %type2
+  %root = pdl.operation -> (%type1, %type2 : !pdl.type, !pdl.type)
   pdl.rewrite %root {
     %type3 = pdl.type
-    %newOp, %newResults:2 = pdl.operation "foo.op" -> %type1, %type3
+    %newOp = pdl.operation "foo.op" -> (%type1, %type3 : !pdl.type, !pdl.type)
     pdl.replace %root with %newOp
   }
 }
@@ -57,27 +58,25 @@ pdl.pattern @infer_type_from_operation_replace : benefit(1) {
 // -----
 
 // Check that the result type of an operation within a rewrite can be inferred
-// from a pdl.replace.
-pdl.pattern @infer_type_from_result_replace : benefit(1) {
+// from types used within the match block.
+pdl.pattern @infer_type_from_type_used_in_match : benefit(1) {
   %type1 = pdl.type : i32
   %type2 = pdl.type
-  %root, %results:2 = pdl.operation -> %type1, %type2
+  %root = pdl.operation -> (%type1, %type2 : !pdl.type, !pdl.type)
   pdl.rewrite %root {
-    %type3 = pdl.type
-    %newOp, %newResults:2 = pdl.operation "foo.op" -> %type1, %type3
-    pdl.replace %root with (%newResults#0, %newResults#1)
+    %newOp = pdl.operation "foo.op" -> (%type1, %type2 : !pdl.type, !pdl.type)
   }
 }
 
 // -----
 
 // Check that the result type of an operation within a rewrite can be inferred
-// from a pdl.replace.
+// from types used within the match block.
 pdl.pattern @infer_type_from_type_used_in_match : benefit(1) {
-  %type1 = pdl.type : i32
-  %type2 = pdl.type
-  %root, %results:2 = pdl.operation -> %type1, %type2
+  %types = pdl.types
+  %root = pdl.operation -> (%types : !pdl.range<type>)
   pdl.rewrite %root {
-    %newOp, %newResults:2 = pdl.operation "foo.op" -> %type1, %type2
+    %otherTypes = pdl.types : [i32, i64]
+    %newOp = pdl.operation "foo.op" -> (%types, %otherTypes : !pdl.range<type>, !pdl.range<type>)
   }
 }
diff --git a/mlir/test/Dialect/PDLInterp/ops.mlir b/mlir/test/Dialect/PDLInterp/ops.mlir
index d76b17c394e8ec79ca81b057bfbb419a2e5b9868..072dfaddcda25af64051626e46f031f1af2c55b5 100644
--- a/mlir/test/Dialect/PDLInterp/ops.mlir
+++ b/mlir/test/Dialect/PDLInterp/ops.mlir
@@ -10,16 +10,16 @@ func @operations(%attribute: !pdl.attribute,
                  %input: !pdl.value,
                  %type: !pdl.type) {
   // attributes, operands, and results
-  %op0 = pdl_interp.create_operation "foo.op"(%input) {"attr" = %attribute} -> %type
+  %op0 = pdl_interp.create_operation "foo.op"(%input : !pdl.value) {"attr" = %attribute} -> (%type : !pdl.type)
 
   // attributes, and results
-  %op1 = pdl_interp.create_operation "foo.op"() {"attr" = %attribute} -> %type
+  %op1 = pdl_interp.create_operation "foo.op" {"attr" = %attribute} -> (%type : !pdl.type)
 
   // attributes
-  %op2 = pdl_interp.create_operation "foo.op"() {"attr" = %attribute, "attr1" = %attribute} -> ()
+  %op2 = pdl_interp.create_operation "foo.op" {"attr" = %attribute, "attr1" = %attribute}
 
   // operands, and results
-  %op3 = pdl_interp.create_operation "foo.op"(%input) -> %type
+  %op3 = pdl_interp.create_operation "foo.op"(%input : !pdl.value) -> (%type : !pdl.type)
 
   pdl_interp.finalize
 }
diff --git a/mlir/test/Dialect/Quant/convert-const.mlir b/mlir/test/Dialect/Quant/convert-const.mlir
index bb8f8cf61c9d26b185aa3af100dbf2c40608556e..fb6baa25ba4cc7e14eb40dd2b4c4982ca6a6b6ee 100644
--- a/mlir/test/Dialect/Quant/convert-const.mlir
+++ b/mlir/test/Dialect/Quant/convert-const.mlir
@@ -144,9 +144,9 @@ func @const_custom_storage_range_i8_fixedpoint() -> tensor<7xf32> {
 // CHECK-LABEL: zero_tensors_to_zero_points
 func @zero_tensors_to_zero_points() -> (tensor<7xf32>, tensor<7xf32>, tensor<7xf32>, tensor<7xf32>) {
 
-// CHECK: %[[cst:.*]] = constant dense<-127> : tensor<7xi8>
-// CHECK: %[[cst0:.*]] = constant dense<0> : tensor<7xi8>
-// CHECK: %[[cst1:.*]] = constant dense<1> : tensor<7xi8>
+// CHECK-DAG: %[[cst1:.*]] = constant dense<1> : tensor<7xi8>
+// CHECK-DAG: %[[cst:.*]] = constant dense<-127> : tensor<7xi8>
+// CHECK-DAG: %[[cst0:.*]] = constant dense<0> : tensor<7xi8>
 // CHECK: "quant.scast"(%[[cst0]]) : (tensor<7xi8>) -> tensor<7x!quant.uniform<i8:f32, 1.000000e+00>>
 // CHECK: "quant.scast"(%[[cst]]) : (tensor<7xi8>) -> tensor<7x!quant.uniform<i8<-127:127>:f32, 1.000000e+00:-127>>
 // CHECK: "quant.scast"(%[[cst0]]) : (tensor<7xi8>) -> tensor<7x!quant.uniform<u8:f32, 1.000000e+00>>
@@ -176,10 +176,10 @@ func @zero_tensors_to_zero_points() -> (tensor<7xf32>, tensor<7xf32>, tensor<7xf
 // CHECK-LABEL: per_axis_dense_quantization
 func @per_axis_dense_quantization() -> (tensor<2x3xf32>, tensor<2x3xf32>) {
 
-// CHECK-NEXT: %[[cst:.*]] = constant dense<{{\[}}[-128, 64, 127], [0, 1, 2]]> : tensor<2x3xi8>
-// CHECK-NEXT: %[[cst0:.*]] = constant dense<{{\[}}[-128, -1, 1], [127, 1, 3]]> : tensor<2x3xi8>
+// CHECK-DAG: %[[cst0:.*]] = constant dense<{{\[}}[-128, -1, 1], [127, 1, 3]]> : tensor<2x3xi8>
+// CHECK-DAG: %[[cst:.*]] = constant dense<{{\[}}[-128, 64, 127], [0, 1, 2]]> : tensor<2x3xi8>
 // CHECK: "quant.scast"(%[[cst]]) : (tensor<2x3xi8>) -> tensor<2x3x!quant.uniform<i8:f32:0, {7.812500e-03:128,1.000000e+00}>>
-// CHECK: "quant.scast"(%cst_0) : (tensor<2x3xi8>) -> tensor<2x3x!quant.uniform<i8:f32:1, {7.812500e-03:128,1.000000e+00,1.000000e+00:1}>>
+// CHECK: "quant.scast"(%[[cst0]]) : (tensor<2x3xi8>) -> tensor<2x3x!quant.uniform<i8:f32:1, {7.812500e-03:128,1.000000e+00,1.000000e+00:1}>>
 
   %cst = constant dense<[[-2.0, -0.5, 0.0], [0.0, 1.0, 2.0]]> : tensor<2x3xf32>
   %1 = "quant.qcast"(%cst) : (tensor<2x3xf32>) -> tensor<2x3x!quant.uniform<i8:f32:0, {7.812500e-03:128, 1.0}>>
diff --git a/mlir/test/Dialect/Quant/parse-uniform.mlir b/mlir/test/Dialect/Quant/parse-uniform.mlir
index 0e609a77d1fcf72525c5fd2bbd50c29efab512d2..3f9dad898361e1a41d4a2e46d01d8d2c0aa337c1 100644
--- a/mlir/test/Dialect/Quant/parse-uniform.mlir
+++ b/mlir/test/Dialect/Quant/parse-uniform.mlir
@@ -92,6 +92,15 @@ func @parse() -> !qalias {
   return %0 : !qalias
 }
 
+// -----
+// Expressed type: f32
+// CHECK: !quant.uniform<u8:f32, 0x41646ABBA0000000:128>
+!qalias = type !quant.uniform<u8:f32, 0x41646ABBA0000000:128>
+func @parse() -> !qalias {
+  %0 = "foo"() : () -> !qalias
+  return %0 : !qalias
+}
+
 // -----
 // Expressed type: f16
 // CHECK: !quant.uniform<u8:f16, 2.000000e+02>
diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir
index 8f76926bdff0e608911b060ad4509c986481cf6d..7c751623db86b501e9f096ddfb8c1c0c7cbdb619 100644
--- a/mlir/test/Dialect/SCF/canonicalize.mlir
+++ b/mlir/test/Dialect/SCF/canonicalize.mlir
@@ -21,12 +21,12 @@ func @single_iteration(%A: memref<?x?x?xi32>) {
 
 // CHECK-LABEL:   func @single_iteration(
 // CHECK-SAME:                        [[ARG0:%.*]]: memref<?x?x?xi32>) {
-// CHECK:           [[C0:%.*]] = constant 0 : index
-// CHECK:           [[C2:%.*]] = constant 2 : index
-// CHECK:           [[C3:%.*]] = constant 3 : index
-// CHECK:           [[C6:%.*]] = constant 6 : index
-// CHECK:           [[C7:%.*]] = constant 7 : index
-// CHECK:           [[C42:%.*]] = constant 42 : i32
+// CHECK-DAG:           [[C42:%.*]] = constant 42 : i32
+// CHECK-DAG:           [[C7:%.*]] = constant 7 : index
+// CHECK-DAG:           [[C6:%.*]] = constant 6 : index
+// CHECK-DAG:           [[C3:%.*]] = constant 3 : index
+// CHECK-DAG:           [[C2:%.*]] = constant 2 : index
+// CHECK-DAG:           [[C0:%.*]] = constant 0 : index
 // CHECK:           scf.parallel ([[V0:%.*]]) = ([[C3]]) to ([[C6]]) step ([[C2]]) {
 // CHECK:             memref.store [[C42]], [[ARG0]]{{\[}}[[C0]], [[V0]], [[C7]]] : memref<?x?x?xi32>
 // CHECK:             scf.yield
@@ -35,10 +35,12 @@ func @single_iteration(%A: memref<?x?x?xi32>) {
 
 // -----
 
+func private @side_effect()
 func @one_unused(%cond: i1) -> (index) {
   %c0 = constant 0 : index
   %c1 = constant 1 : index
   %0, %1 = scf.if %cond -> (index, index) {
+    call @side_effect() : () -> ()
     scf.yield %c0, %c1 : index, index
   } else {
     scf.yield %c0, %c1 : index, index
@@ -49,6 +51,7 @@ func @one_unused(%cond: i1) -> (index) {
 // CHECK-LABEL:   func @one_unused
 // CHECK:           [[C0:%.*]] = constant 1 : index
 // CHECK:           [[V0:%.*]] = scf.if %{{.*}} -> (index) {
+// CHECK:             call @side_effect() : () -> ()
 // CHECK:             scf.yield [[C0]] : index
 // CHECK:           } else
 // CHECK:             scf.yield [[C0]] : index
@@ -57,11 +60,13 @@ func @one_unused(%cond: i1) -> (index) {
 
 // -----
 
+func private @side_effect()
 func @nested_unused(%cond1: i1, %cond2: i1) -> (index) {
   %c0 = constant 0 : index
   %c1 = constant 1 : index
   %0, %1 = scf.if %cond1 -> (index, index) {
     %2, %3 = scf.if %cond2 -> (index, index) {
+      call @side_effect() : () -> ()
       scf.yield %c0, %c1 : index, index
     } else {
       scf.yield %c0, %c1 : index, index
@@ -77,6 +82,7 @@ func @nested_unused(%cond1: i1, %cond2: i1) -> (index) {
 // CHECK:           [[C0:%.*]] = constant 1 : index
 // CHECK:           [[V0:%.*]] = scf.if {{.*}} -> (index) {
 // CHECK:             [[V1:%.*]] = scf.if {{.*}} -> (index) {
+// CHECK:               call @side_effect() : () -> ()
 // CHECK:               scf.yield [[C0]] : index
 // CHECK:             } else
 // CHECK:               scf.yield [[C0]] : index
@@ -113,6 +119,96 @@ func @all_unused(%cond: i1) {
 
 // -----
 
+func @empty_if1(%cond: i1) {
+  scf.if %cond {
+    scf.yield
+  }
+  return
+}
+
+// CHECK-LABEL:   func @empty_if1
+// CHECK-NOT:       scf.if
+// CHECK:           return
+
+// -----
+
+func @empty_if2(%cond: i1) {
+  scf.if %cond {
+    scf.yield
+  } else {
+    scf.yield
+  }
+  return
+}
+
+// CHECK-LABEL:   func @empty_if2
+// CHECK-NOT:       scf.if
+// CHECK:           return
+
+// -----
+
+func @to_select1(%cond: i1) -> index {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %0 = scf.if %cond -> index {
+    scf.yield %c0 : index
+  } else {
+    scf.yield %c1 : index
+  }
+  return %0 : index
+}
+
+// CHECK-LABEL:   func @to_select1
+// CHECK:           [[C0:%.*]] = constant 0 : index
+// CHECK:           [[C1:%.*]] = constant 1 : index
+// CHECK:           [[V0:%.*]] = select {{.*}}, [[C0]], [[C1]]
+// CHECK:           return [[V0]] : index
+
+// -----
+
+func @to_select_same_val(%cond: i1) -> (index, index) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %0, %1 = scf.if %cond -> (index, index) {
+    scf.yield %c0, %c1 : index, index
+  } else {
+    scf.yield %c1, %c1 : index, index
+  }
+  return %0, %1 : index, index
+}
+
+// CHECK-LABEL:   func @to_select_same_val
+// CHECK:           [[C0:%.*]] = constant 0 : index
+// CHECK:           [[C1:%.*]] = constant 1 : index
+// CHECK:           [[V0:%.*]] = select {{.*}}, [[C0]], [[C1]]
+// CHECK:           return [[V0]], [[C1]] : index, index
+
+// -----
+
+func @to_select2(%cond: i1) -> (index, index) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %c2 = constant 2 : index
+  %c3 = constant 3 : index
+  %0, %1 = scf.if %cond -> (index, index) {
+    scf.yield %c0, %c1 : index, index
+  } else {
+    scf.yield %c2, %c3 : index, index
+  }
+  return %0, %1 : index, index
+}
+
+// CHECK-LABEL:   func @to_select2
+// CHECK:           [[C0:%.*]] = constant 0 : index
+// CHECK:           [[C1:%.*]] = constant 1 : index
+// CHECK:           [[C2:%.*]] = constant 2 : index
+// CHECK:           [[C3:%.*]] = constant 3 : index
+// CHECK:           [[V0:%.*]] = select {{.*}}, [[C0]], [[C2]]
+// CHECK:           [[V1:%.*]] = select {{.*}}, [[C1]], [[C3]]
+// CHECK:           return [[V0]], [[V1]] : index
+
+// -----
+
 func private @make_i32() -> i32
 
 func @for_yields_2(%lb : index, %ub : index, %step : index) -> i32 {
@@ -335,6 +431,7 @@ func @remove_empty_parallel_loop(%lb: index, %ub: index, %s: index) {
 }
 
 // -----
+
 func private @process(%0 : memref<128x128xf32>)
 func private @process_tensor(%0 : tensor<128x128xf32>) -> memref<128x128xf32>
 
@@ -382,3 +479,40 @@ func @last_value(%t0: tensor<128x128xf32>, %t1: tensor<128x128xf32>,
   // CHECK-NEXT: return %[[R0]], %[[R1]], %[[FOR_RES]] : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32>
   return %0#0, %0#1, %0#2 : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32>
 }
+
+// -----
+
+// CHECK-LABEL: fold_away_iter_with_no_use_and_yielded_input
+//  CHECK-SAME:   %[[A0:[0-9a-z]*]]: i32
+func @fold_away_iter_with_no_use_and_yielded_input(%arg0 : i32,
+                    %ub : index, %lb : index, %step : index) -> (i32, i32) {
+  // CHECK-NEXT: %[[C32:.*]] = constant 32 : i32
+  %cst = constant 32 : i32
+  // CHECK-NEXT: %[[FOR_RES:.*]] = scf.for {{.*}} iter_args({{.*}} = %[[A0]]) -> (i32) { 
+  %0:2 = scf.for %arg1 = %lb to %ub step %step iter_args(%arg2 = %arg0, %arg3 = %cst)
+    -> (i32, i32) {
+    %1 = addi %arg2, %cst : i32
+    scf.yield %1, %cst : i32, i32
+  }
+
+  // CHECK: return %[[FOR_RES]], %[[C32]] : i32, i32
+  return %0#0, %0#1 : i32, i32
+}
+
+// -----
+
+// CHECK-LABEL: fold_away_iter_and_result_with_no_use
+//  CHECK-SAME:   %[[A0:[0-9a-z]*]]: i32
+func @fold_away_iter_and_result_with_no_use(%arg0 : i32,
+                    %ub : index, %lb : index, %step : index) -> (i32) {
+  %cst = constant 32 : i32
+  // CHECK: %[[FOR_RES:.*]] = scf.for {{.*}} iter_args({{.*}} = %[[A0]]) -> (i32) {
+  %0:2 = scf.for %arg1 = %lb to %ub step %step iter_args(%arg2 = %arg0, %arg3 = %cst)
+    -> (i32, i32) {
+    %1 = addi %arg2, %cst : i32
+    scf.yield %1, %1 : i32, i32
+  }
+  
+  // CHECK: return %[[FOR_RES]] : i32
+  return %0#0 : i32
+}
diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
index cc1db79d1c37688b791631e36cf26f1eedd2e103..48c2502cb914bd808d2f824b5af253fd96fe1356 100644
--- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
+++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
@@ -92,9 +92,9 @@ func @convert_bitcast_multi_use(%arg0 : vector<2xf32>, %arg1 : !spv.ptr<i64, Uni
 
 // CHECK-LABEL: extract_vector
 func @extract_vector() -> (i32, i32, i32) {
-  // CHECK: spv.Constant 42 : i32
-  // CHECK: spv.Constant -33 : i32
-  // CHECK: spv.Constant 6 : i32
+  // CHECK-DAG: spv.Constant 6 : i32
+  // CHECK-DAG: spv.Constant -33 : i32
+  // CHECK-DAG: spv.Constant 42 : i32
   %0 = spv.Constant dense<[42, -33, 6]> : vector<3xi32>
   %1 = spv.CompositeExtract %0[0 : i32] : vector<3xi32>
   %2 = spv.CompositeExtract %0[1 : i32] : vector<3xi32>
@@ -106,8 +106,8 @@ func @extract_vector() -> (i32, i32, i32) {
 
 // CHECK-LABEL: extract_array_final
 func @extract_array_final() -> (i32, i32) {
-  // CHECK: spv.Constant 4 : i32
-  // CHECK: spv.Constant -5 : i32
+  // CHECK-DAG: spv.Constant -5 : i32
+  // CHECK-DAG: spv.Constant 4 : i32
   %0 = spv.Constant [dense<[4, -5]> : vector<2xi32>] : !spv.array<1 x vector<2xi32>>
   %1 = spv.CompositeExtract %0[0 : i32, 0 : i32] : !spv.array<1 x vector<2 x i32>>
   %2 = spv.CompositeExtract %0[0 : i32, 1 : i32] : !spv.array<1 x vector<2 x i32>>
@@ -192,9 +192,9 @@ func @const_fold_scalar_iadd_normal() -> (i32, i32, i32) {
   %c5 = spv.Constant 5 : i32
   %cn8 = spv.Constant -8 : i32
 
-  // CHECK: spv.Constant 10
-  // CHECK: spv.Constant -16
-  // CHECK: spv.Constant -3
+  // CHECK-DAG: spv.Constant -3
+  // CHECK-DAG: spv.Constant -16
+  // CHECK-DAG: spv.Constant 10
   %0 = spv.IAdd %c5, %c5 : i32
   %1 = spv.IAdd %cn8, %cn8 : i32
   %2 = spv.IAdd %c5, %cn8 : i32
@@ -210,17 +210,17 @@ func @const_fold_scalar_iadd_flow() -> (i32, i32, i32, i32) {
   %c5 = spv.Constant -1 : i32          //         : 0xffff ffff
   %c6 = spv.Constant -2 : i32          //         : 0xffff fffe
 
+  // 0x8000 0000 + 0xffff fffe = 0x1 7fff fffe -> 0x7fff fffe
+  // CHECK-DAG: spv.Constant 2147483646
+  // 0x8000 0000 + 0xffff ffff = 0x1 7fff ffff -> 0x7fff ffff
+  // CHECK-DAG: spv.Constant 2147483647
+  // 0x0000 0002 + 0xffff ffff = 0x1 0000 0001 -> 0x0000 0001
+  // CHECK-DAG: spv.Constant 1
   // 0x0000 0001 + 0xffff ffff = 0x1 0000 0000 -> 0x0000 0000
-  // CHECK: spv.Constant 0
+  // CHECK-DAG: spv.Constant 0
   %0 = spv.IAdd %c1, %c3 : i32
-  // 0x0000 0002 + 0xffff ffff = 0x1 0000 0001 -> 0x0000 0001
-  // CHECK: spv.Constant 1
-  %1 = spv.IAdd %c2, %c3 : i32
-  // 0x8000 0000 + 0xffff ffff = 0x1 7fff ffff -> 0x7fff ffff
-  // CHECK: spv.Constant 2147483647
+   %1 = spv.IAdd %c2, %c3 : i32
   %2 = spv.IAdd %c4, %c5 : i32
-  // 0x8000 0000 + 0xffff fffe = 0x1 7fff fffe -> 0x7fff fffe
-  // CHECK: spv.Constant 2147483646
   %3 = spv.IAdd %c4, %c6 : i32
   return %0, %1, %2, %3: i32, i32, i32, i32
 }
@@ -259,9 +259,9 @@ func @const_fold_scalar_imul_normal() -> (i32, i32, i32) {
   %cn8 = spv.Constant -8 : i32
   %c7 = spv.Constant 7 : i32
 
-  // CHECK: spv.Constant 35
-  // CHECK: spv.Constant -40
-  // CHECK: spv.Constant -56
+  // CHECK-DAG: spv.Constant -56
+  // CHECK-DAG: spv.Constant -40
+  // CHECK-DAG: spv.Constant 35
   %0 = spv.IMul %c7, %c5 : i32
   %1 = spv.IMul %c5, %cn8 : i32
   %2 = spv.IMul %cn8, %c7 : i32
@@ -275,13 +275,14 @@ func @const_fold_scalar_imul_flow() -> (i32, i32, i32) {
   %c3 = spv.Constant 4294967295 : i32  // 2^32 - 1 : 0xffff ffff
   %c4 = spv.Constant 2147483647 : i32  // 2^31 - 1 : 0x7fff ffff
 
+  // (0x7fff ffff << 2) = 0x1 ffff fffc -> 0xffff fffc
+  // CHECK-DAG: %[[CST4:.*]] = spv.Constant -4
+
   // (0xffff ffff << 1) = 0x1 ffff fffe -> 0xffff fffe
-  // CHECK: %[[CST2:.*]] = spv.Constant -2
+  // CHECK-DAG: %[[CST2:.*]] = spv.Constant -2
   %0 = spv.IMul %c1, %c3 : i32
   // (0x7fff ffff << 1) = 0x0 ffff fffe -> 0xffff fffe
   %1 = spv.IMul %c1, %c4 : i32
-  // (0x7fff ffff << 2) = 0x1 ffff fffc -> 0xffff fffc
-  // CHECK: %[[CST4:.*]] = spv.Constant -4
   %2 = spv.IMul %c4, %c2 : i32
   // CHECK: return %[[CST2]], %[[CST2]], %[[CST4]]
   return %0, %1, %2: i32, i32, i32
@@ -317,9 +318,9 @@ func @const_fold_scalar_isub_normal() -> (i32, i32, i32) {
   %cn8 = spv.Constant -8 : i32
   %c7 = spv.Constant 7 : i32
 
-  // CHECK: spv.Constant 2
-  // CHECK: spv.Constant 13
-  // CHECK: spv.Constant -15
+  // CHECK-DAG: spv.Constant -15
+  // CHECK-DAG: spv.Constant 13
+  // CHECK-DAG: spv.Constant 2
   %0 = spv.ISub %c7, %c5 : i32
   %1 = spv.ISub %c5, %cn8 : i32
   %2 = spv.ISub %cn8, %c7 : i32
@@ -335,17 +336,17 @@ func @const_fold_scalar_isub_flow() -> (i32, i32, i32, i32) {
   %c5 = spv.Constant -1 : i32          //          : 0xffff ffff
   %c6 = spv.Constant -2 : i32          //          : 0xffff fffe
 
+  // 0xffff ffff - 0x7fff ffff -> 0xffff ffff + 0x8000 0001 = 0x1 8000 0000
+  // CHECK-DAG: spv.Constant -2147483648
+  // 0x0000 0001 - 0xffff ffff -> 0x0000 0001 + 0x0000 0001 = 0x0000 0002
+  // CHECK-DAG: spv.Constant 2
   // 0x0000 0000 - 0xffff ffff -> 0x0000 0000 + 0x0000 0001 = 0x0000 0001
-  // CHECK: spv.Constant 1
+  // CHECK-DAG: spv.Constant 1
+  // 0xffff fffe - 0x7fff ffff -> 0xffff fffe + 0x8000 0001 = 0x1 7fff ffff
+  // CHECK-DAG: spv.Constant 2147483647
   %0 = spv.ISub %c1, %c3 : i32
-  // 0x0000 0001 - 0xffff ffff -> 0x0000 0001 + 0x0000 0001 = 0x0000 0002
-  // CHECK: spv.Constant 2
   %1 = spv.ISub %c2, %c3 : i32
-  // 0xffff ffff - 0x7fff ffff -> 0xffff ffff + 0x8000 0001 = 0x1 8000 0000
-  // CHECK: spv.Constant -2147483648
   %2 = spv.ISub %c5, %c4 : i32
-  // 0xffff fffe - 0x7fff ffff -> 0xffff fffe + 0x8000 0001 = 0x1 7fff ffff
-  // CHECK: spv.Constant 2147483647
   %3 = spv.ISub %c6, %c4 : i32
   return %0, %1, %2, %3: i32, i32, i32, i32
 }
@@ -545,12 +546,14 @@ func @canonicalize_selection_op_vector_type(%cond: i1) -> () {
 
 // -----
 
+// CHECK-LABEL: cannot_canonicalize_selection_op_0
+
 // Store to a different variables.
 func @cannot_canonicalize_selection_op_0(%cond: i1) -> () {
   %0 = spv.Constant dense<[0, 1, 2]> : vector<3xi32>
-  // CHECK: %[[SRC_VALUE_0:.*]] = spv.Constant dense<[1, 2, 3]> : vector<3xi32>
+  // CHECK-DAG: %[[SRC_VALUE_1:.*]] = spv.Constant dense<[2, 3, 4]> : vector<3xi32>
+  // CHECK-DAG: %[[SRC_VALUE_0:.*]] = spv.Constant dense<[1, 2, 3]> : vector<3xi32>
   %1 = spv.Constant dense<[1, 2, 3]> : vector<3xi32>
-  // CHECK: %[[SRC_VALUE_1:.*]] = spv.Constant dense<[2, 3, 4]> : vector<3xi32>
   %2 = spv.Constant dense<[2, 3, 4]> : vector<3xi32>
   // CHECK: %[[DST_VAR_0:.*]] = spv.Variable init({{%.*}}) : !spv.ptr<vector<3xi32>, Function>
   %3 = spv.Variable init(%0) : !spv.ptr<vector<3xi32>, Function>
@@ -582,6 +585,8 @@ func @cannot_canonicalize_selection_op_0(%cond: i1) -> () {
 
 // -----
 
+// CHECK-LABEL: cannot_canonicalize_selection_op_1
+
 // A conditional block consists of more than 2 operations.
 func @cannot_canonicalize_selection_op_1(%cond: i1) -> () {
   %0 = spv.Constant dense<[0, 1, 2]> : vector<3xi32>
@@ -618,6 +623,8 @@ func @cannot_canonicalize_selection_op_1(%cond: i1) -> () {
 
 // -----
 
+// CHECK-LABEL: cannot_canonicalize_selection_op_2
+
 // A control-flow goes into `^then` block from `^else` block.
 func @cannot_canonicalize_selection_op_2(%cond: i1) -> () {
   %0 = spv.Constant dense<[0, 1, 2]> : vector<3xi32>
@@ -650,11 +657,13 @@ func @cannot_canonicalize_selection_op_2(%cond: i1) -> () {
 
 // -----
 
+// CHECK-LABEL: cannot_canonicalize_selection_op_3
+
 // `spv.Return` as a block terminator.
 func @cannot_canonicalize_selection_op_3(%cond: i1) -> () {
   %0 = spv.Constant dense<[0, 1, 2]> : vector<3xi32>
-  // CHECK: %[[SRC_VALUE_0:.*]] = spv.Constant dense<[1, 2, 3]> : vector<3xi32>
   %1 = spv.Constant dense<[1, 2, 3]> : vector<3xi32>
+  // CHECK: %[[SRC_VALUE_0:.*]] = spv.Constant dense<[1, 2, 3]> : vector<3xi32>
   // CHECK: %[[SRC_VALUE_1:.*]] = spv.Constant dense<[2, 3, 4]> : vector<3xi32>
   %2 = spv.Constant dense<[2, 3, 4]> : vector<3xi32>
   // CHECK: %[[DST_VAR:.*]] = spv.Variable init({{%.*}}) : !spv.ptr<vector<3xi32>, Function>
@@ -682,6 +691,8 @@ func @cannot_canonicalize_selection_op_3(%cond: i1) -> () {
 
 // -----
 
+// CHECK-LABEL: cannot_canonicalize_selection_op_4
+
 // Different memory access attributes.
 func @cannot_canonicalize_selection_op_4(%cond: i1) -> () {
   %0 = spv.Constant dense<[0, 1, 2]> : vector<3xi32>
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index 53f27e4839cf3bead74b37970a78556424fb7fba..39f17e9d253f6d45ce9b3a2b84c002d80bbfb454 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -648,7 +648,7 @@ func @f() {
   // CHECK: shape.cstr_broadcastable
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %cs0 = shape.const_shape [8, 1] : !shape.shape
+  %cs0 = shape.const_shape [8, 1] : !shape.shape  
   %cs1 = shape.const_shape [1, 8] : !shape.shape
   %cs2 = shape.const_shape [1, -1] : !shape.shape
   %0 = shape.cstr_broadcastable %cs0, %cs1, %cs2 : !shape.shape, !shape.shape, !shape.shape
@@ -665,7 +665,7 @@ func @f() {
   // CHECK-NEXT: return
   %cs0 = shape.const_shape [8, 1] : !shape.shape
   %cs1 = shape.const_shape [1, -1] : !shape.shape
-  %cs2 = shape.const_shape [1, -1] : !shape.shape
+  %cs2 = shape.const_shape [8, -1] : !shape.shape
   %0 = shape.cstr_broadcastable %cs0, %cs1, %cs2 : !shape.shape, !shape.shape, !shape.shape
   "consume.witness"(%0) : (!shape.witness) -> ()
   return
@@ -1097,6 +1097,19 @@ func @is_broadcastable_on_duplicate_shapes(%a : !shape.shape, %b : !shape.shape)
 
 // -----
 
+// CHECK-LABEL: @cstr_broadcastable_on_duplicate_shapes
+// CHECK-SAME: (%[[A:.*]]: !shape.shape, %[[B:.*]]: !shape.shape)
+func @cstr_broadcastable_on_duplicate_shapes(%a : !shape.shape,
+    %b : !shape.shape) -> !shape.witness {
+  // CHECK: %[[RES:.*]] = shape.cstr_broadcastable %[[A]], %[[B]] :
+  // CHECK: return %[[RES]]
+  %0 = shape.cstr_broadcastable %a, %b, %a, %a, %a, %b : !shape.shape,
+      !shape.shape, !shape.shape, !shape.shape, !shape.shape, !shape.shape
+  return %0 : !shape.witness
+}
+
+// -----
+
 // CHECK-LABEL: @broadcast_on_same_shape
 // CHECK-SAME: (%[[SHAPE:.*]]: !shape.shape)
 func @broadcast_on_same_shape(%shape : !shape.shape) -> !shape.shape {
@@ -1119,3 +1132,15 @@ func @broadcast_on_duplicate_shapes(%a : !shape.shape, %b : !shape.shape)
       !shape.shape, !shape.shape, !shape.shape, !shape.shape -> !shape.shape
   return %0 : !shape.shape
 }
+
+// -----
+
+// CHECK-LABEL: @broadcast_on_single_operand
+// CHECK-SAME: (%[[A:.*]]: tensor<3xindex>)
+func @broadcast_on_single_operand(%a : tensor<3xindex>) {
+  // CHECK-NOT: broadcast
+  // CHECK: "use"(%[[A]])
+  %0 = shape.broadcast %a : tensor<3xindex> -> tensor<?xindex>
+  "use"(%0) : (tensor<?xindex>) -> ()
+  return
+}
diff --git a/mlir/test/Dialect/Standard/canonicalize.mlir b/mlir/test/Dialect/Standard/canonicalize.mlir
index 41ae2248b299dc72e3722f398cda4a0150e2ed9e..77022024ae486c0a1b5bd85a24cd10e3d2da538b 100644
--- a/mlir/test/Dialect/Standard/canonicalize.mlir
+++ b/mlir/test/Dialect/Standard/canonicalize.mlir
@@ -134,6 +134,34 @@ func @cmpi_equal_operands(%arg0: i64)
 
 // -----
 
+// Test case: Folding of memref.dim(memref.alloca(%size), %idx) -> %size
+// CHECK-LABEL: func @dim_of_alloca(
+//  CHECK-SAME:     %[[SIZE:[0-9a-z]+]]: index
+//  CHECK-NEXT:   return %[[SIZE]] : index
+func @dim_of_alloca(%size: index) -> index {
+  %0 = memref.alloca(%size) : memref<?xindex>
+  %c0 = constant 0 : index
+  %1 = memref.dim %0, %c0 : memref<?xindex>
+  return %1 : index
+}
+
+// -----
+
+// Test case: Folding of memref.dim(memref.alloca(rank(%v)), %idx) -> rank(%v)
+// CHECK-LABEL: func @dim_of_alloca_with_dynamic_size(
+//  CHECK-SAME:     %[[MEM:[0-9a-z]+]]: memref<*xf32>
+//  CHECK-NEXT:   %[[RANK:.*]] = rank %[[MEM]] : memref<*xf32>
+//  CHECK-NEXT:   return %[[RANK]] : index
+func @dim_of_alloca_with_dynamic_size(%arg0: memref<*xf32>) -> index {
+  %0 = rank %arg0 : memref<*xf32>
+  %1 = memref.alloca(%0) : memref<?xindex>
+  %c0 = constant 0 : index
+  %2 = memref.dim %1, %c0 : memref<?xindex>
+  return %2 : index
+}
+
+// -----
+
 // Test case: Folding of memref.dim(memref.reshape %v %shp, %idx) -> memref.load %shp[%idx]
 // CHECK-LABEL: func @dim_of_memref_reshape(
 //  CHECK-SAME:     %[[MEM:[0-9a-z]+]]: memref<*xf32>,
@@ -311,3 +339,32 @@ func @subtensor_insert_output_dest_canonicalize(%arg0 : tensor<2x3xi32>, %arg1 :
 //       CHECK:   %[[GENERATE:.+]] = tensor.generate
 //       CHECK:   %[[RESULT:.+]] = subtensor_insert %[[ARG0]] into %[[GENERATE]]
 //       CHECK:   return %[[RESULT]]
+
+// -----
+
+// CHECK-LABEL: @select_same_val
+//       CHECK:   return %arg1
+func @select_same_val(%arg0: i1, %arg1: i64) -> i64 {
+  %0 = select %arg0, %arg1, %arg1 : i64
+  return %0 : i64
+}
+
+// -----
+
+// CHECK-LABEL: @select_cmp_eq_select
+//       CHECK:   return %arg1
+func @select_cmp_eq_select(%arg0: i64, %arg1: i64) -> i64 {
+  %0 = cmpi eq, %arg0, %arg1 : i64
+  %1 = select %0, %arg0, %arg1 : i64
+  return %1 : i64
+}
+
+// -----
+
+// CHECK-LABEL: @select_cmp_ne_select
+//       CHECK:   return %arg0
+func @select_cmp_ne_select(%arg0: i64, %arg1: i64) -> i64 {
+  %0 = cmpi ne, %arg0, %arg1 : i64
+  %1 = select %0, %arg0, %arg1 : i64
+  return %1 : i64
+}
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index c274b6f8b1c9c7cc3276456e9a0d9173959d77d2..c8ad16ab9b14a71a6c2ac18db5f2b13dd774ac30 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -69,25 +69,25 @@ func @fold_extract(%arg0 : index) -> (f32, f16, f16, i32) {
   %const_0 = constant 0 : index
   %const_1 = constant 1 : index
   %const_3 = constant 3 : index
+  // CHECK-DAG: [[C64:%.+]] = constant 64 : i32
+  // CHECK-DAG: [[C0:%.+]] = constant 0.{{0*}}e+00 : f16
+  // CHECK-DAG: [[CM2:%.+]] = constant -2.{{0*}}e+00 : f16
 
   // Fold an extract into a splat.
-  // CHECK-NEXT: [[C4:%.+]] = constant 4.{{0*}}e+00 : f32
+  // CHECK-DAG: [[C4:%.+]] = constant 4.{{0*}}e+00 : f32
   %0 = constant dense<4.0> : tensor<4xf32>
   %ext_1 = tensor.extract %0[%arg0] : tensor<4xf32>
 
   // Fold an extract into a sparse with a sparse index.
-  // CHECK-NEXT: [[CM2:%.+]] = constant -2.{{0*}}e+00 : f16
   %1 = constant sparse<[[0, 0, 0], [1, 1, 1]],  [-5.0, -2.0]> : tensor<4x4x4xf16>
   %ext_2 = tensor.extract %1[%const_1, %const_1, %const_1] : tensor<4x4x4xf16>
 
   // Fold an extract into a sparse with a non sparse index.
-  // CHECK-NEXT: [[C0:%.+]] = constant 0.{{0*}}e+00 : f16
   %2 = constant sparse<[[1, 1, 1]],  [-2.0]> : tensor<1x1x1xf16>
   %ext_3 = tensor.extract %2[%const_0, %const_0, %const_0] : tensor<1x1x1xf16>
 
   // Fold an extract into a dense tensor.
-  // CHECK-NEXT: [[C64:%.+]] = constant 64 : i32
-  %3 = constant dense<[[[1, -2, 1, 36]], [[0, 2, -1, 64]]]> : tensor<2x1x4xi32>
+   %3 = constant dense<[[[1, -2, 1, 36]], [[0, 2, -1, 64]]]> : tensor<2x1x4xi32>
   %ext_4 = tensor.extract %3[%const_1, %const_0, %const_3] : tensor<2x1x4xi32>
 
   // CHECK-NEXT: return [[C4]], [[CM2]], [[C0]], [[C64]]
diff --git a/mlir/test/Dialect/Tosa/ops.mlir b/mlir/test/Dialect/Tosa/ops.mlir
index f22e6fc7527d3fe96cd87819a246ec03813fce1f..2ff3971b77d3e9d4e5a9b19d85f8bd4965f6c936 100644
--- a/mlir/test/Dialect/Tosa/ops.mlir
+++ b/mlir/test/Dialect/Tosa/ops.mlir
@@ -406,18 +406,24 @@ func @test_transpose(%arg0: tensor<13x21x3xf32>) -> tensor<3x13x21xf32> {
 
 // -----
 // CHECK-LABEL: gather
-func @test_gather(%arg0: tensor<13x21x3xi32>, %arg1: tensor<26xi32>) -> tensor<26x21x3xi32> {
-  %0 = "tosa.gather"(%arg0, %arg1) {axis = 0 : i32, batch_dims = 0 : i64} : (tensor<13x21x3xi32>, tensor<26xi32>) -> tensor<26x21x3xi32>
-  return %0 : tensor<26x21x3xi32>
-}
-
-// Test TBD
-// DISABLED-CHECK-LABEL: resize
-//func @test_resize(%arg0: tensor<1x32x32x8xf32>) -> tensor<1x64x64x8xf32> {
-//  %0 = "tosa.const"() {value = dense<64> : tensor<2xi32>} : () -> tensor<2xi32>
-//  %1 = "tosa.resize"(%arg0, %0) {align_corners = false, half_pixel_centers = true} : (tensor<1x32x32x8xf32>, tensor<2xi32>) -> tensor<1x64x64x8xf32>
-//  return %1 : tensor<1x64x64x8xf32>
-//}
+func @test_gather(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x26xi32>) -> tensor<13x26x3xf32> {
+  %0 = "tosa.gather"(%arg0, %arg1) : (tensor<13x21x3xf32>, tensor<13x26xi32>) -> tensor<13x26x3xf32>
+  return %0 : tensor<13x26x3xf32>
+}
+
+// -----
+// CHECK-LABEL: scatter
+func @test_scatter(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf32>) -> tensor<13x21x3xf32> {
+  %0 = "tosa.scatter"(%arg0, %arg1, %arg2) : (tensor<13x21x3xf32>, tensor<13x26xi32>, tensor<13x26x3xf32>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+// CHECK-LABEL: resize
+func @test_resize(%arg0: tensor<1x32x32x8xf32>) -> tensor<1x64x64x8xf32> {
+  %1 = "tosa.resize"(%arg0) {output_size = [64, 64], stride = [1024, 1024], offset = [0, 0], shift = 10 : i32, stride_fp = [0.0 : f32, 0.0 : f32], offset_fp = [0.0 : f32, 0.0 : f32], mode = "BILINEAR"} : (tensor<1x32x32x8xf32>) -> tensor<1x64x64x8xf32>
+  return %1 : tensor<1x64x64x8xf32>
+}
 
 // -----
 // CHECK-LABEL: cast
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index a68c7fba8e1c1bd1b99a978d821d07c60ee3d57c..c6ec156e1519444169d1c42b034a0d118666cb48 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -571,10 +571,10 @@ func @bitcast_folding(%I1: vector<4x8xf32>, %I2: vector<2xi32>) -> (vector<4x8xf
 }
 
 // CHECK-LABEL: func @bitcast_f16_to_f32
-//              bit pattern: 0x00000000
-//       CHECK: %[[CST0:.+]] = constant dense<0.000000e+00> : vector<4xf32>
 //              bit pattern: 0x40004000
-//       CHECK: %[[CST1:.+]] = constant dense<2.00390625> : vector<4xf32>
+//       CHECK-DAG: %[[CST1:.+]] = constant dense<2.00390625> : vector<4xf32>
+//              bit pattern: 0x00000000
+//       CHECK-DAG: %[[CST0:.+]] = constant dense<0.000000e+00> : vector<4xf32>
 //       CHECK: return %[[CST0]], %[[CST1]]
 func @bitcast_f16_to_f32() -> (vector<4xf32>, vector<4xf32>) {
   %cst0 = constant dense<0.0> : vector<8xf16> // bit pattern: 0x0000
@@ -612,8 +612,8 @@ func @broadcast_folding2() -> vector<4x16xi32> {
 // -----
 
 // CHECK-LABEL: shape_cast_constant
-//       CHECK: %[[CST0:.*]] = constant dense<2.000000e+00> : vector<20x2xf32>
-//       CHECK: %[[CST1:.*]] = constant dense<1> : vector<3x4x2xi32>
+//       CHECK-DAG: %[[CST1:.*]] = constant dense<1> : vector<3x4x2xi32>
+//       CHECK-DAG: %[[CST0:.*]] = constant dense<2.000000e+00> : vector<20x2xf32>
 //       CHECK: return %[[CST0]], %[[CST1]] : vector<20x2xf32>, vector<3x4x2xi32>
 func @shape_cast_constant() -> (vector<20x2xf32>, vector<3x4x2xi32>) {
   %cst = constant dense<2.000000e+00> : vector<5x4x2xf32>
@@ -626,8 +626,8 @@ func @shape_cast_constant() -> (vector<20x2xf32>, vector<3x4x2xi32>) {
 // -----
 
 // CHECK-LABEL: extract_strided_constant
-//       CHECK: %[[CST0:.*]] = constant dense<2.000000e+00> : vector<12x2xf32>
-//       CHECK: %[[CST1:.*]] = constant dense<1> : vector<2x13x3xi32>
+//       CHECK-DAG: %[[CST1:.*]] = constant dense<1> : vector<2x13x3xi32>
+//       CHECK-DAG: %[[CST0:.*]] = constant dense<2.000000e+00> : vector<12x2xf32>
 //       CHECK: return %[[CST0]], %[[CST1]] : vector<12x2xf32>, vector<2x13x3xi32>
 func @extract_strided_constant() -> (vector<12x2xf32>, vector<2x13x3xi32>) {
   %cst = constant dense<2.000000e+00> : vector<29x7xf32>
diff --git a/mlir/test/Dialect/Vector/vector-contract-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-transforms.mlir
index 3adb18c1a2ae6e290f714e307b0fb2ded8a5961f..bf13b273d328e8553d9693cc8952cf13db93640d 100644
--- a/mlir/test/Dialect/Vector/vector-contract-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-contract-transforms.mlir
@@ -431,8 +431,9 @@ func @nop_shape_cast(%arg0: vector<16xf32>) -> vector<16xf32> {
 }
 
 // CHECK-LABEL: func @cancel_shape_cast
-// CHECK-SAME: %[[A:.*]]: vector<16xf32>
-// CHECK:      return %[[A]] : vector<16xf32>
+// FIXME: PR49590
+// HECK-SAME: %[[A:.*]]: vector<16xf32>
+// HECK:      return %[[A]] : vector<16xf32>
 
 func @cancel_shape_cast(%arg0: vector<16xf32>) -> vector<16xf32> {
   %0 = vector.shape_cast %arg0 : vector<16xf32> to vector<4x4xf32>
@@ -444,8 +445,8 @@ func @cancel_shape_cast(%arg0: vector<16xf32>) -> vector<16xf32> {
 // llvm.matrix operations
 // CHECK-LABEL: func @shape_casts
 func @shape_casts(%a: vector<2x2xf32>) -> (vector<4xf32>, vector<2x2xf32>) {
-  // CHECK: %[[cst:.*]] = constant dense<0.000000e+00> : vector<4xf32>
-  // CHECK: %[[cst22:.*]] = constant dense<0.000000e+00> : vector<2x2xf32>
+  // CHECK-DAG: %[[cst22:.*]] = constant dense<0.000000e+00> : vector<2x2xf32>
+  // CHECK-DAG: %[[cst:.*]] = constant dense<0.000000e+00> : vector<4xf32>
   // CHECK: %[[ex0:.*]] = vector.extract %{{.*}}[0] : vector<2x2xf32>
   //
   // CHECK: %[[in0:.*]] = vector.insert_strided_slice %[[ex0]], %[[cst]]
diff --git a/mlir/test/Dialect/Vector/vector-flat-transforms.mlir b/mlir/test/Dialect/Vector/vector-flat-transforms.mlir
index c07d651d985edcb3e745fb6fd477ae8a57af835f..8d51d323a1a713caeff8ece628add191e5e2d8aa 100644
--- a/mlir/test/Dialect/Vector/vector-flat-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-flat-transforms.mlir
@@ -22,10 +22,12 @@ func @transpose44_44(%arg0: vector<4x4xf32>) -> vector<4x4xf32> {
 // Folds preceding shape cast as expected,
 // no following shape cast folding expected.
 //
+// FIXME: PR49590 - shape_cast not stable.
+//
 // CHECK-LABEL: func @transpose16_44(
 // CHECK-SAME:  %[[A:.*]]: vector<16xf32>
-// CHECK:       %[[T0:.*]] = vector.flat_transpose %[[A]] {columns = 4 : i32, rows = 4 : i32} : vector<16xf32> -> vector<16xf32>
-// CHECK:       %[[T1:.*]] = vector.extract_strided_slice %[[T0]] {offsets = [0], sizes = [4], strides = [1]} : vector<16xf32> to vector<4xf32>
+// HECK:       %[[T0:.*]] = vector.flat_transpose %[[A]] {columns = 4 : i32, rows = 4 : i32} : vector<16xf32> -> vector<16xf32>
+// HECK:       %[[T1:.*]] = vector.extract_strided_slice %[[T0]] {offsets = [0], sizes = [4], strides = [1]} : vector<16xf32> to vector<4xf32>
 //
 func @transpose16_44(%arg0: vector<16xf32>) -> vector<4x4xf32> {
   %0 = vector.shape_cast %arg0 : vector<16xf32> to vector<4x4xf32>
@@ -49,9 +51,11 @@ func @transpose44_16(%arg0: vector<4x4xf32>) -> vector<16xf32> {
 // Folds preceding shape cast as expected,
 // but FAILS to fold following cast.
 //
+// FIXME: PR49590 - shape_cast not stable.
+//
 // CHECK-LABEL: func @transpose16_16(
 // CHECK-SAME:  %[[A:.*]]: vector<16xf32>
-// CHECK:       %[[T0:.*]] = vector.flat_transpose %[[A]] {columns = 4 : i32, rows = 4 : i32} : vector<16xf32> -> vector<16xf32>
+// HECK:       %[[T0:.*]] = vector.flat_transpose %[[A]] {columns = 4 : i32, rows = 4 : i32} : vector<16xf32> -> vector<16xf32>
 //
 func @transpose16_16(%arg0: vector<16xf32>) -> vector<16xf32> {
   %0 = vector.shape_cast %arg0 : vector<16xf32> to vector<4x4xf32>
diff --git a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
index 21d749cc088fd7f4bd543a0a50a43e70286b8f56..74b64ee1e263b432bf01f77fa6586370c097411f 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
@@ -25,9 +25,8 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
   %c0 = constant 0 : index
   %f0 = constant 0.0 : f32
 
-  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
   //  CHECK-DAG: %[[c8:.*]] = constant 8 : index
-  //  CHECK-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
+  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
   // alloca for boundary full tile
   //      CHECK: %[[alloc:.*]] = memref.alloca() {alignment = 32 : i64} : memref<4x8xf32>
   // %i + 4 <= dim(%A, 0)
@@ -54,13 +53,12 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
   //      CHECK:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
   // CHECK-SAME:     memref<?x8xf32>, index, index
   //      CHECK: }
-  //      CHECK: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %[[cst]]
+  //      CHECK: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %cst
   // CHECK_SAME:   {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
 
   //  LINALG-DAG: %[[c0:.*]] = constant 0 : index
   //  LINALG-DAG: %[[c4:.*]] = constant 4 : index
   //  LINALG-DAG: %[[c8:.*]] = constant 8 : index
-  //  LINALG-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
   // alloca for boundary full tile
   //      LINALG: %[[alloc:.*]] = memref.alloca() {alignment = 32 : i64} : memref<4x8xf32>
   // %i + 4 <= dim(%A, 0)
@@ -77,7 +75,7 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
   //      LINALG:   scf.yield %[[A]], %[[i]], %[[j]] : memref<?x8xf32>, index, index
   //      LINALG: } else {
   //               slow path, fill tmp alloc and yield a memref_casted version of it
-  //      LINALG:   linalg.fill(%[[alloc]], %[[cst]]) : memref<4x8xf32>, f32
+  //      LINALG:   linalg.fill(%[[alloc]], %cst) : memref<4x8xf32>, f32
   //      LINALG:   %[[d0:.*]] = memref.dim %[[A]], %[[c0]] : memref<?x8xf32>
   //      LINALG:   %[[sv0:.*]] = affine.min #[[$bounds_map_4]](%[[d0]], %[[i]], %[[c4]])
   //      LINALG:   %[[sv1:.*]] = affine.min #[[$bounds_map_8]](%[[c8]], %[[j]], %[[c8]])
@@ -89,7 +87,7 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
   //      LINALG:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
   // LINALG-SAME:     memref<?x8xf32>, index, index
   //      LINALG: }
-  //      LINALG: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %[[cst]]
+  //      LINALG: %[[res:.*]] = vector.transfer_read %[[ifres]]#0[%[[ifres]]#1, %[[ifres]]#2], %cst
   // LINALG_SAME:   {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
   %1 = vector.transfer_read %A[%i, %j], %f0 : memref<?x8xf32>, vector<4x8xf32>
 
@@ -112,10 +110,9 @@ func @split_vector_transfer_read_strided_2d(
   %c0 = constant 0 : index
   %f0 = constant 0.0 : f32
 
-  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
   //  CHECK-DAG: %[[c7:.*]] = constant 7 : index
   //  CHECK-DAG: %[[c8:.*]] = constant 8 : index
-  //  CHECK-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
+  //  CHECK-DAG: %[[c0:.*]] = constant 0 : index
   // alloca for boundary full tile
   //      CHECK: %[[alloc:.*]] = memref.alloca() {alignment = 32 : i64} : memref<4x8xf32>
   // %i + 4 <= dim(%A, 0)
@@ -152,7 +149,6 @@ func @split_vector_transfer_read_strided_2d(
   //  LINALG-DAG: %[[c4:.*]] = constant 4 : index
   //  LINALG-DAG: %[[c7:.*]] = constant 7 : index
   //  LINALG-DAG: %[[c8:.*]] = constant 8 : index
-  //  LINALG-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
   // alloca for boundary full tile
   //      LINALG: %[[alloc:.*]] = memref.alloca() {alignment = 32 : i64} : memref<4x8xf32>
   // %i + 4 <= dim(%A, 0)
@@ -171,7 +167,7 @@ func @split_vector_transfer_read_strided_2d(
   // LINALG-SAME:     memref<?x8xf32, #[[$map_2d_stride_1]]>, index, index
   //      LINALG: } else {
   //               slow path, fill tmp alloc and yield a memref_casted version of it
-  //      LINALG:   linalg.fill(%[[alloc]], %[[cst]]) : memref<4x8xf32>, f32
+  //      LINALG:   linalg.fill(%[[alloc]], %cst) : memref<4x8xf32>, f32
   //      LINALG:   %[[sv0:.*]] = affine.min #[[$bounds_map_4]](%[[c7]], %[[i]], %[[c4]])
   //      LINALG:   %[[sv1:.*]] = affine.min #[[$bounds_map_8]](%[[c8]], %[[j]], %[[c8]])
   //      LINALG:   %[[sv:.*]] = memref.subview %[[A]][%[[i]], %[[j]]] [%[[sv0]], %[[sv1]]] [1, 1]
diff --git a/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir b/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir
index d5e9535acb8ee3d4d28de4c43e1a25b00298a367..15b68275decf9f55e1326686c568c04ea361a650 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir
@@ -1,8 +1,8 @@
 // RUN: mlir-opt %s -test-vector-transfer-unrolling-patterns | FileCheck %s
 
 // CHECK-LABEL: func @transfer_read_unroll
-//       CHECK:   %[[C0:.*]] = constant 0 : index
-//       CHECK:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C0:.*]] = constant 0 : index
 //       CHECK:   %[[VTR0:.*]] = vector.transfer_read {{.*}}[%[[C0]], %[[C0]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32>
 //  CHECK-NEXT:   %[[VTR1:.*]] = vector.transfer_read {{.*}}[%[[C0]], %[[C2]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32>
 //  CHECK-NEXT:   %[[VTR2:.*]] = vector.transfer_read {{.*}}[%[[C2]], %[[C0]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32>
@@ -19,8 +19,8 @@ func @transfer_read_unroll(%arg0 : memref<4x4xf32>) -> vector<4x4xf32> {
 }
 
 // CHECK-LABEL: func @transfer_write_unroll
-//       CHECK:   %[[C0:.*]] = constant 0 : index
-//       CHECK:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C0:.*]] = constant 0 : index
 //       CHECK:   %[[TUPL:.*]] = vector.extract_slices {{.*}}, [2, 2], [1, 1] : vector<4x4xf32> into tuple<vector<2x2xf32>, vector<2x2xf32>, vector<2x2xf32>, vector<2x2xf32>>
 //  CHECK-NEXT:   %[[T0:.*]] = vector.tuple_get %[[TUPL]], 0 : tuple<vector<2x2xf32>, vector<2x2xf32>, vector<2x2xf32>, vector<2x2xf32>>
 //  CHECK-NEXT:   vector.transfer_write %[[T0]], {{.*}}[%[[C0]], %[[C0]]] {{.*}} : vector<2x2xf32>, memref<4x4xf32>
@@ -39,8 +39,8 @@ func @transfer_write_unroll(%arg0 : memref<4x4xf32>, %arg1 : vector<4x4xf32>) {
 }
 
 // CHECK-LABEL: func @transfer_readwrite_unroll
-//       CHECK:   %[[C0:.*]] = constant 0 : index
-//       CHECK:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C0:.*]] = constant 0 : index
 //       CHECK:   %[[VTR0:.*]] = vector.transfer_read {{.*}}[%[[C0]], %[[C0]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32>
 //  CHECK-NEXT:   %[[VTR1:.*]] = vector.transfer_read {{.*}}[%[[C0]], %[[C2]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32>
 //  CHECK-NEXT:   %[[VTR2:.*]] = vector.transfer_read {{.*}}[%[[C2]], %[[C0]]], %{{.*}} : memref<4x4xf32>, vector<2x2xf32>
@@ -60,8 +60,8 @@ func @transfer_readwrite_unroll(%arg0 : memref<4x4xf32>) {
 }
 
 // CHECK-LABEL: func @transfer_read_unroll_tensor
-//       CHECK:   %[[C0:.*]] = constant 0 : index
-//       CHECK:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C0:.*]] = constant 0 : index
 //       CHECK:   %[[VTR0:.*]] = vector.transfer_read {{.*}}[%[[C0]], %[[C0]]], %{{.*}} : tensor<4x4xf32>, vector<2x2xf32>
 //  CHECK-NEXT:   %[[VTR1:.*]] = vector.transfer_read {{.*}}[%[[C0]], %[[C2]]], %{{.*}} : tensor<4x4xf32>, vector<2x2xf32>
 //  CHECK-NEXT:   %[[VTR2:.*]] = vector.transfer_read {{.*}}[%[[C2]], %[[C0]]], %{{.*}} : tensor<4x4xf32>, vector<2x2xf32>
@@ -78,8 +78,8 @@ func @transfer_read_unroll_tensor(%arg0 : tensor<4x4xf32>) -> vector<4x4xf32> {
 }
 
 // CHECK-LABEL: func @transfer_write_unroll_tensor
-//       CHECK:   %[[C0:.*]] = constant 0 : index
-//       CHECK:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C0:.*]] = constant 0 : index
 //       CHECK:   %[[TUPL:.*]] = vector.extract_slices {{.*}}, [2, 2], [1, 1] : vector<4x4xf32> into tuple<vector<2x2xf32>, vector<2x2xf32>, vector<2x2xf32>, vector<2x2xf32>>
 //  CHECK-NEXT:   %[[T0:.*]] = vector.tuple_get %[[TUPL]], 0 : tuple<vector<2x2xf32>, vector<2x2xf32>, vector<2x2xf32>, vector<2x2xf32>>
 //  CHECK-NEXT:   %[[VTW0:.*]] = vector.transfer_write %[[T0]], {{.*}}[%[[C0]], %[[C0]]] {{.*}} : vector<2x2xf32>, tensor<4x4xf32>
@@ -100,8 +100,8 @@ func @transfer_write_unroll_tensor(%arg0 : tensor<4x4xf32>,
 }
 
 // CHECK-LABEL: func @transfer_readwrite_unroll_tensor
-//       CHECK:   %[[C0:.*]] = constant 0 : index
-//       CHECK:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C0:.*]] = constant 0 : index
 //       CHECK:   %[[VTR0:.*]] = vector.transfer_read {{.*}}[%[[C0]], %[[C0]]], %{{.*}} : tensor<4x4xf32>, vector<2x2xf32>
 //  CHECK-NEXT:   %[[VTR1:.*]] = vector.transfer_read {{.*}}[%[[C0]], %[[C2]]], %{{.*}} : tensor<4x4xf32>, vector<2x2xf32>
 //  CHECK-NEXT:   %[[VTR2:.*]] = vector.transfer_read {{.*}}[%[[C2]], %[[C0]]], %{{.*}} : tensor<4x4xf32>, vector<2x2xf32>
diff --git a/mlir/test/Dialect/Vector/vector-transforms.mlir b/mlir/test/Dialect/Vector/vector-transforms.mlir
index da899389de74460991e22862b7c0fbc176467bbf..9388b67dd5322456cfdcaaf574342c97edd7bc87 100644
--- a/mlir/test/Dialect/Vector/vector-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-transforms.mlir
@@ -225,8 +225,8 @@ func @contraction4x4_ikj(%arg0 : vector<4x2xf32>, %arg1 : vector<2x4xf32>,
 
 // CHECK-LABEL: func @contraction4x4_ikj_xfer_read
 
-// CHECK:      %[[C0:.*]] = constant 0 : index
-// CHECK:      %[[C2:.*]] = constant 2 : index
+// CHECK-DAG:      %[[C2:.*]] = constant 2 : index
+// CHECK-DAG:      %[[C0:.*]] = constant 0 : index
 
 // Check LHS vector.transfer read is split for each user.
 
@@ -422,8 +422,8 @@ func @cancelling_shape_cast_ops(%arg0 : vector<2x4xf32>) -> vector<2x4xf32> {
 }
 
 // CHECK-LABEL: func @vector_transfers_vector_element_type
-//      CHECK: %[[C0:.*]] = constant 0 : index
-//      CHECK: %[[C1:.*]] = constant 1 : index
+//      CHECK-DAG: %[[C1:.*]] = constant 1 : index
+//      CHECK-DAG: %[[C0:.*]] = constant 0 : index
 //      CHECK: %[[VTR0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]], %[[C0]]], %{{.*}} {masked = [false, false]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
 // CHECK-NEXT: %[[VTR1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C1]], %[[C0]]], %{{.*}} {masked = [false, false]} : memref<6x2x1xvector<2x4xf32>>, vector<1x1x2x4xf32>
 // CHECK-NEXT: vector.transfer_write %[[VTR0]], %{{.*}}[%[[C0]], %[[C0]], %[[C0]]] {masked = [false, false]} : vector<1x1x2x4xf32>, memref<6x2x1xvector<2x4xf32>>
@@ -516,8 +516,8 @@ func @shape_cast_fold(%arg0 : vector<5x4x2xf32>, %arg1 : vector<3x4x2xf32>)
 
 // CHECK-LABEL: func @elementwise_unroll
 //  CHECK-SAME: (%[[ARG0:.*]]: memref<4x4xf32>, %[[ARG1:.*]]: memref<4x4xf32>)
-//       CHECK:   %[[C0:.*]] = constant 0 : index
-//       CHECK:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C2:.*]] = constant 2 : index
+//       CHECK-DAG:   %[[C0:.*]] = constant 0 : index
 //       CHECK:   %[[VT0:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]]], {{.*}} : memref<4x4xf32>, vector<2x2xf32>
 //       CHECK:   %[[VT1:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C2]]], {{.*}} : memref<4x4xf32>, vector<2x2xf32>
 //       CHECK:   %[[VT2:.*]] = vector.transfer_read %[[ARG0]][%[[C2]], %[[C0]]], {{.*}} : memref<4x4xf32>, vector<2x2xf32>
diff --git a/mlir/test/IR/invalid-module-op.mlir b/mlir/test/IR/invalid-module-op.mlir
index 520821a7b0b42d92a44b90c5ed3edd8bf3bb9d65..741a3a9b2dc94dd4af69a2a42630f52b7bd1bb84 100644
--- a/mlir/test/IR/invalid-module-op.mlir
+++ b/mlir/test/IR/invalid-module-op.mlir
@@ -19,31 +19,12 @@ func @module_op() {
   // expected-error@+1 {{region should have no arguments}}
   module {
   ^bb1(%arg: i32):
-    "module_terminator"() : () -> ()
-  }
-  return
-}
-
-// -----
-
-func @module_op() {
-  // expected-error@below {{expects regions to end with 'module_terminator'}}
-  // expected-note@below {{the absence of terminator implies 'module_terminator'}}
-  module {
-    return
   }
   return
 }
 
 // -----
 
-func @module_op() {
-  // expected-error@+1 {{expects parent op 'module'}}
-  "module_terminator"() : () -> ()
-}
-
-// -----
-
 // expected-error@+1 {{can only contain attributes with dialect-prefixed names}}
 module attributes {attr} {
 }
diff --git a/mlir/test/IR/invalid.mlir b/mlir/test/IR/invalid.mlir
index 419c986265219b0cd021d52ce87797645f9a7042..39e72d2cb8daaaed6b86b9f3f86a15eb04a350a3 100644
--- a/mlir/test/IR/invalid.mlir
+++ b/mlir/test/IR/invalid.mlir
@@ -120,7 +120,7 @@ func @block_redef() {
 
 // -----
 
-func @no_terminator() {   // expected-error {{block with no terminator}}
+func @no_terminator() {   // expected-error {{empty block: expect at least a terminator}}
 ^bb40:
   return
 ^bb41:
@@ -766,21 +766,14 @@ func @elementsattr_malformed_opaque() -> () {
 
 func @elementsattr_malformed_opaque1() -> () {
 ^bb0:
-  "foo"(){bar = opaque<"", "0xQZz123"> : tensor<1xi8>} : () -> () // expected-error {{expected string containing hex digits starting with `0x`}}
+  "foo"(){bar = opaque<"_", "0xQZz123"> : tensor<1xi8>} : () -> () // expected-error {{expected string containing hex digits starting with `0x`}}
 }
 
 // -----
 
 func @elementsattr_malformed_opaque2() -> () {
 ^bb0:
-  "foo"(){bar = opaque<"", "00abc"> : tensor<1xi8>} : () -> () // expected-error {{expected string containing hex digits starting with `0x`}}
-}
-
-// -----
-
-func @elementsattr_malformed_opaque3() -> () {
-^bb0:
-  "foo"(){bar = opaque<"t", "0xabc"> : tensor<1xi8>} : () -> () // expected-error {{no registered dialect with namespace 't'}}
+  "foo"(){bar = opaque<"_", "00abc"> : tensor<1xi8>} : () -> () // expected-error {{expected string containing hex digits starting with `0x`}}
 }
 
 // -----
@@ -881,7 +874,7 @@ func @type_alias_unknown(!unknown_alias) -> () { // expected-error {{undefined s
 func @complex_loops() {
   affine.for %i1 = 1 to 100 {
   // expected-error @+1 {{expected '"' in string literal}}
-  "opaqueIntTensor"(){bar = opaque<"", "0x686]> : tensor<2x1x4xi32>} : () -> ()
+  "opaqueIntTensor"(){bar = opaque<"_", "0x686]> : tensor<2x1x4xi32>} : () -> ()
 
 // -----
 
@@ -1198,7 +1191,7 @@ func @hexadecimal_float_literal_overflow() {
 // -----
 
 func @decimal_float_literal() {
-  // expected-error @+2 {{unexpected decimal integer literal for a float attribute}}
+  // expected-error @+2 {{unexpected decimal integer literal for a floating point value}}
   // expected-note @+1 {{add a trailing dot to make the literal a float}}
   "foo"() {value = 42 : f32} : () -> ()
 }
@@ -1251,7 +1244,7 @@ func @hexadecimal_float_too_wide_for_type_in_tensor() {
 
 // Check that we report an error when a value is too wide to be parsed.
 func @hexadecimal_float_too_wide_in_tensor() {
-  // expected-error @+1 {{hexadecimal float constant out of range for attribute}}
+  // expected-error @+1 {{hexadecimal float constant out of range for type}}
   "foo"() {bar = dense<0x7FFFFFF0000000000000> : tensor<2xf32>} : () -> ()
 }
 
diff --git a/mlir/test/IR/module-op.mlir b/mlir/test/IR/module-op.mlir
index 2e5bd6f9685e61fe659b8f2bb0ca5bf0ea114641..d99806c92a902be9bcaf50263920a4348fe79247 100644
--- a/mlir/test/IR/module-op.mlir
+++ b/mlir/test/IR/module-op.mlir
@@ -4,16 +4,14 @@
 module {
 }
 
-// CHECK: module {
-// CHECK-NEXT: }
-module {
-  "module_terminator"() : () -> ()
-}
+// -----
 
 // CHECK: module attributes {foo.attr = true} {
 module attributes {foo.attr = true} {
 }
 
+// -----
+
 // CHECK: module {
 module {
   // CHECK-NEXT: "foo.result_op"() : () -> i32
@@ -55,3 +53,12 @@ module @foo {
     }
   }
 }
+
+// -----
+
+// expected-error@below {{expects at most one data layout attribute}}
+// expected-note@below {{'test.another_attribute' is a data layout attribute}}
+// expected-note@below {{'test.random_attribute' is a data layout attribute}}
+module attributes { test.random_attribute = #dlti.dl_spec<>,
+                    test.another_attribute = #dlti.dl_spec<>} {
+}
diff --git a/mlir/test/IR/parser.mlir b/mlir/test/IR/parser.mlir
index d24f5f07637ecda9e8b3f08ee0d5966013bc1075..df6f21610823706c42c375df04529998cf40c846 100644
--- a/mlir/test/IR/parser.mlir
+++ b/mlir/test/IR/parser.mlir
@@ -1411,3 +1411,7 @@ test.graph_region {
   %2 = "bar"(%1) : (i64) -> i64
   "unregistered_terminator"() : () -> ()
 }) {sym_name = "unregistered_op_dominance_violation_ok", type = () -> i1} : () -> ()
+
+// This is an unregister operation, the printing/parsing is handled by the dialect.
+// CHECK: test.dialect_custom_printer custom_format
+test.dialect_custom_printer custom_format
diff --git a/mlir/test/IR/pretty-attributes.mlir b/mlir/test/IR/pretty-attributes.mlir
index d4ac8e7739350c1421cb5017ee77892602ec555f..280e32672ea5fe52b5ca3ace9bd8b1a718eeab5c 100644
--- a/mlir/test/IR/pretty-attributes.mlir
+++ b/mlir/test/IR/pretty-attributes.mlir
@@ -5,17 +5,17 @@
 // tensor which passes don't look at directly, this isn't an issue.
 // RUN: mlir-opt %s -mlir-elide-elementsattrs-if-larger=2 | mlir-opt
 
-// CHECK: opaque<"", "0xDEADBEEF"> : tensor<3xi32>
+// CHECK: opaque<"_", "0xDEADBEEF"> : tensor<3xi32>
 "test.dense_attr"() {foo.dense_attr = dense<[1, 2, 3]> : tensor<3xi32>} : () -> ()
 
 // CHECK: dense<[1, 2]> : tensor<2xi32>
 "test.non_elided_dense_attr"() {foo.dense_attr = dense<[1, 2]> : tensor<2xi32>} : () -> ()
 
-// CHECK: opaque<"", "0xDEADBEEF"> : vector<1x1x1xf16>
+// CHECK: opaque<"_", "0xDEADBEEF"> : vector<1x1x1xf16>
 "test.sparse_attr"() {foo.sparse_attr = sparse<[[1, 2, 3]],  -2.0> : vector<1x1x1xf16>} : () -> ()
 
-// CHECK: opaque<"", "0xDEADBEEF"> : tensor<100xf32>
-"test.opaque_attr"() {foo.opaque_attr = opaque<"", "0xEBFE"> : tensor<100xf32> } : () -> ()
+// CHECK: opaque<"_", "0xDEADBEEF"> : tensor<100xf32>
+"test.opaque_attr"() {foo.opaque_attr = opaque<"_", "0xEBFE"> : tensor<100xf32> } : () -> ()
 
 // CHECK: dense<1> : tensor<3xi32>
 "test.dense_splat"() {foo.dense_attr = dense<1> : tensor<3xi32>} : () -> ()
diff --git a/mlir/test/IR/print-ir-defuse.mlir b/mlir/test/IR/print-ir-defuse.mlir
index 78c5804119250a240587a3aa43e87c1532bfa344..55c8494f83c460acec16ab5d11259c57f305a3bc 100644
--- a/mlir/test/IR/print-ir-defuse.mlir
+++ b/mlir/test/IR/print-ir-defuse.mlir
@@ -18,8 +18,6 @@
 // CHECK: Has 0 results:
 // CHECK: Visiting op 'dialect.op3' with 0 operands:
 // CHECK: Has 0 results:
-// CHECK: Visiting op 'module_terminator' with 0 operands:
-// CHECK: Has 0 results:
 // CHECK: Visiting op 'module' with 0 operands:
 // CHECK: Has 0 results:
 
diff --git a/mlir/test/IR/print-ir-nesting.mlir b/mlir/test/IR/print-ir-nesting.mlir
index 4682753947550c2ec73a9d5fce4bed5db9e4afbd..92259a6e04561a6718ecf64206476eb268c6eb7a 100644
--- a/mlir/test/IR/print-ir-nesting.mlir
+++ b/mlir/test/IR/print-ir-nesting.mlir
@@ -3,7 +3,7 @@
 // CHECK: visiting op: 'module' with 0 operands and 0 results
 // CHECK:  1 nested regions:
 // CHECK:   Region with 1 blocks:
-// CHECK:     Block with 0 arguments, 0 successors, and 3 operations
+// CHECK:     Block with 0 arguments, 0 successors, and 2 operations
 module {
 
 
@@ -52,6 +52,4 @@ module {
     "dialect.innerop7"() : () -> ()
   }) : () -> ()
 
-// CHECK:       visiting op: 'module_terminator' with 0 operands and 0 results
-
 } // module
diff --git a/mlir/test/IR/region.mlir b/mlir/test/IR/region.mlir
index 465ae511aad2329c5abb84d4bd05026509a15391..fc0f2da36af4cc156415223907ddd2c6749d29d5 100644
--- a/mlir/test/IR/region.mlir
+++ b/mlir/test/IR/region.mlir
@@ -73,3 +73,31 @@ func @named_region_has_wrong_number_of_blocks() {
     }) : () -> ()
     return
 }
+
+// -----
+
+// Region with single block and not terminator.
+// CHECK: unregistered_without_terminator
+"test.unregistered_without_terminator"() ( {
+  ^bb0:  // no predecessors
+}) : () -> ()
+
+// -----
+
+// CHECK: test.single_no_terminator_op
+"test.single_no_terminator_op"() (
+  {
+    func @foo1() { return }
+    func @foo2() { return }
+  }
+) : () -> ()
+
+// CHECK: test.variadic_no_terminator_op
+"test.variadic_no_terminator_op"() (
+  {
+    func @foo1() { return }
+  },
+  {
+    func @foo2() { return }
+  }
+) : () -> ()
diff --git a/mlir/test/IR/test-side-effects.mlir b/mlir/test/IR/test-side-effects.mlir
index db55414da03be18e125c40f6048712ba22c49456..947faff99d265f34b5d6f6e2b8dbd673200ecb5b 100644
--- a/mlir/test/IR/test-side-effects.mlir
+++ b/mlir/test/IR/test-side-effects.mlir
@@ -30,3 +30,9 @@
 %4 = "test.side_effect_op"() {
   effect_parameter = affine_map<(i, j) -> (j, i)>
 } : () -> i32
+
+// Check with this unregistered operation to test the fallback on the dialect.
+// expected-remark@+1 {{found a parametric effect with affine_map<(d0, d1) -> (d1, d0)>}}
+%5 = "test.unregistered_side_effect_op"() {
+  effect_parameter = affine_map<(i, j) -> (j, i)>
+} : () -> i32
diff --git a/mlir/test/IR/traits.mlir b/mlir/test/IR/traits.mlir
index dc9e5106a8d7f1daa9ac93e17df7ab2a6755cc3c..3e1bb3bee0b39750c698e25e0cb7b42713c222b4 100644
--- a/mlir/test/IR/traits.mlir
+++ b/mlir/test/IR/traits.mlir
@@ -382,19 +382,26 @@ func @failedMissingOperandSizeAttr(%arg: i32) {
 // -----
 
 func @failedOperandSizeAttrWrongType(%arg: i32) {
-  // expected-error @+1 {{requires 1D vector attribute 'operand_segment_sizes'}}
+  // expected-error @+1 {{requires 1D vector of i32 attribute 'operand_segment_sizes'}}
   "test.attr_sized_operands"(%arg, %arg, %arg, %arg) {operand_segment_sizes = dense<[1, 1, 1, 1]>: tensor<4xi32>} : (i32, i32, i32, i32) -> ()
 }
 
 // -----
 
 func @failedOperandSizeAttrWrongRank(%arg: i32) {
-  // expected-error @+1 {{requires 1D vector attribute 'operand_segment_sizes'}}
+  // expected-error @+1 {{requires 1D vector of i32 attribute 'operand_segment_sizes'}}
   "test.attr_sized_operands"(%arg, %arg, %arg, %arg) {operand_segment_sizes = dense<[[1, 1], [1, 1]]>: vector<2x2xi32>} : (i32, i32, i32, i32) -> ()
 }
 
 // -----
 
+func @failedOperandSizeAttrWrongElementType(%arg: i32) {
+  // expected-error @+1 {{requires 1D vector of i32 attribute 'operand_segment_sizes'}}
+  "test.attr_sized_operands"(%arg, %arg, %arg, %arg) {operand_segment_sizes = dense<[1, 1, 1, 1]>: vector<4xi64>} : (i32, i32, i32, i32) -> ()
+}
+
+// -----
+
 func @failedOperandSizeAttrNegativeValue(%arg: i32) {
   // expected-error @+1 {{'operand_segment_sizes' attribute cannot have negative elements}}
   "test.attr_sized_operands"(%arg, %arg, %arg, %arg) {operand_segment_sizes = dense<[1, 1, -1, 1]>: vector<4xi32>} : (i32, i32, i32, i32) -> ()
@@ -432,19 +439,26 @@ func @failedMissingResultSizeAttr() {
 // -----
 
 func @failedResultSizeAttrWrongType() {
-  // expected-error @+1 {{requires 1D vector attribute 'result_segment_sizes'}}
+  // expected-error @+1 {{requires 1D vector of i32 attribute 'result_segment_sizes'}}
   %0:4 = "test.attr_sized_results"() {result_segment_sizes = dense<[1, 1, 1, 1]>: tensor<4xi32>} : () -> (i32, i32, i32, i32)
 }
 
 // -----
 
 func @failedResultSizeAttrWrongRank() {
-  // expected-error @+1 {{requires 1D vector attribute 'result_segment_sizes'}}
+  // expected-error @+1 {{requires 1D vector of i32 attribute 'result_segment_sizes'}}
   %0:4 = "test.attr_sized_results"() {result_segment_sizes = dense<[[1, 1], [1, 1]]>: vector<2x2xi32>} : () -> (i32, i32, i32, i32)
 }
 
 // -----
 
+func @failedResultSizeAttrWrongElementType() {
+  // expected-error @+1 {{requires 1D vector of i32 attribute 'result_segment_sizes'}}
+  %0:4 = "test.attr_sized_results"() {result_segment_sizes = dense<[1, 1, 1, 1]>: vector<4xi64>} : () -> (i32, i32, i32, i32)
+}
+
+// -----
+
 func @failedResultSizeAttrNegativeValue() {
   // expected-error @+1 {{'result_segment_sizes' attribute cannot have negative elements}}
   %0:4 = "test.attr_sized_results"() {result_segment_sizes = dense<[1, 1, -1, 1]>: vector<4xi32>} : () -> (i32, i32, i32, i32)
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir
index 73d866af972ca5fc6cda833a5c2af4fcc69411e6..d188b86b2edaa40fbd5bc3b51e0164d3eb772cce 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir
@@ -38,9 +38,9 @@ func @entry() {
   %c2 = constant 2: index
 
   // Set up memory.
-  %a = alloc() : memref<2x4xbf16>
-  %b = alloc() : memref<2x4xbf16>
-  %c = alloc() : memref<2x2xf32>
+  %a = memref.alloc() : memref<2x4xbf16>
+  %b = memref.alloc() : memref<2x4xbf16>
+  %c = memref.alloc() : memref<2x2xf32>
 
   %0 = std.constant dense<[[1.0, 2.0, 3.0, 4.0 ],
                            [5.0, 6.0, 7.0, 8.0 ]]> : vector<2x4xbf16>
@@ -75,9 +75,9 @@ func @entry() {
   }
 
   // Release resources.
-  dealloc %a : memref<2x4xbf16>
-  dealloc %b : memref<2x4xbf16>
-  dealloc %c : memref<2x2xf32>
+  memref.dealloc %a : memref<2x4xbf16>
+  memref.dealloc %b : memref<2x4xbf16>
+  memref.dealloc %c : memref<2x2xf32>
 
   return
 }
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..45e9816fa9d62a1847d853a7331e5cfefa73b165
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir
@@ -0,0 +1,197 @@
+// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-std -convert-vector-to-llvm="enable-amx" -convert-std-to-llvm | \
+// RUN: mlir-translate -mlir-to-llvmir | \
+// RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+// Note: To run this test, your CPU must support AMX.
+
+func @print(%arg0: memref<16x4xi32>) {
+  %iu = constant -1: i32
+  %c0 = constant 0: index
+  %c1 = constant 1: index
+  %c16 = constant 16: index
+  scf.for %i = %c0 to %c16 step %c1 {
+    %0 = vector.transfer_read %arg0[%i, %c0], %iu: memref<16x4xi32>, vector<4xi32>
+    vector.print %0 : vector<4xi32>
+  }
+  return
+}
+
+func @kernel1(%arg0: memref<16x16xi8>,
+              %arg1: memref<4x16xi8>,
+              %arg2: memref<16x4xi32>) {
+  %0 = constant 0 : index
+  %1 = amx.tile_load %arg0[%0, %0] : memref<16x16xi8>  into vector<16x16xi8>
+  %2 = amx.tile_load %arg1[%0, %0] : memref<4x16xi8>  into vector<4x16xi8>
+  %3 = amx.tile_zero : vector<16x4xi32>
+  %4 = amx.tile_muli %1, %2, %3 : vector<16x16xi8>, vector<4x16xi8>, vector<16x4xi32>
+  amx.tile_store %arg2[%0, %0], %4 : memref<16x4xi32>, vector<16x4xi32>
+  return
+}
+
+func @kernel2(%arg0: memref<16x16xi8>,
+              %arg1: memref<4x16xi8>,
+              %arg2: memref<16x4xi32>) {
+  %0 = constant 0 : index
+  %1 = amx.tile_load %arg0[%0, %0] : memref<16x16xi8>  into vector<16x16xi8>
+  %2 = amx.tile_load %arg1[%0, %0] : memref<4x16xi8>  into vector<4x16xi8>
+  %3 = amx.tile_zero : vector<16x4xi32>
+  %4 = amx.tile_muli %1, %2 zext, %3 : vector<16x16xi8>, vector<4x16xi8>, vector<16x4xi32>
+  amx.tile_store %arg2[%0, %0], %4 : memref<16x4xi32>, vector<16x4xi32>
+  return
+}
+
+func @kernel3(%arg0: memref<16x16xi8>,
+              %arg1: memref<4x16xi8>,
+              %arg2: memref<16x4xi32>) {
+  %0 = constant 0 : index
+  %1 = amx.tile_load %arg0[%0, %0] : memref<16x16xi8>  into vector<16x16xi8>
+  %2 = amx.tile_load %arg1[%0, %0] : memref<4x16xi8>  into vector<4x16xi8>
+  %3 = amx.tile_zero : vector<16x4xi32>
+  %4 = amx.tile_muli %1 zext, %2, %3 : vector<16x16xi8>, vector<4x16xi8>, vector<16x4xi32>
+  amx.tile_store %arg2[%0, %0], %4 : memref<16x4xi32>, vector<16x4xi32>
+  return
+}
+
+func @kernel4(%arg0: memref<16x16xi8>,
+              %arg1: memref<4x16xi8>,
+              %arg2: memref<16x4xi32>) {
+  %0 = constant 0 : index
+  %1 = amx.tile_load %arg0[%0, %0] : memref<16x16xi8>  into vector<16x16xi8>
+  %2 = amx.tile_load %arg1[%0, %0] : memref<4x16xi8>  into vector<4x16xi8>
+  %3 = amx.tile_zero : vector<16x4xi32>
+  %4 = amx.tile_muli %1 zext, %2 zext, %3 : vector<16x16xi8>, vector<4x16xi8>, vector<16x4xi32>
+  amx.tile_store %arg2[%0, %0], %4 : memref<16x4xi32>, vector<16x4xi32>
+  return
+}
+
+func @entry() {
+  %c0 = constant 0: index
+
+  // Set up memory.
+  %a = memref.alloc() : memref<16x16xi8>
+  %b = memref.alloc() : memref<4x16xi8>
+  %c = memref.alloc() : memref<16x4xi32>
+
+  %0 = std.constant dense<
+    [ [  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,  13,  14,  15 ],
+      [ 16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,  27,  28,  29,  30,  31 ],
+      [ 32,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,  45,  46,  47 ],
+      [ 48,  49,  50,  51,  52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63 ],
+      [ 64,  65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,  78,  79 ],
+      [ 80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,  91,  92,  93,  94,  95 ],
+      [ 96,  97,  98,  99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111 ],
+      [112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127 ],
+      [128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143 ],
+      [144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159 ],
+      [160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175 ],
+      [176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191 ],
+      [192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207 ],
+      [208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223 ],
+      [224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239 ],
+      [240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255 ] ]> : vector<16x16xi8>
+
+  %1 = std.constant dense<
+    [ [192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207 ],
+      [208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223 ],
+      [224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239 ],
+      [240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255 ] ]> : vector<4x16xi8>
+
+  vector.transfer_write %0, %a[%c0, %c0] : vector<16x16xi8>, memref<16x16xi8>
+  vector.transfer_write %1, %b[%c0, %c0] : vector<4x16xi8>,  memref<4x16xi8>
+
+  // Call kernel1 and verify result.
+  //
+  // CHECK:      ( -3320, -2840, -2360, -1880 )
+  // CHECK-NEXT: ( -13176, -11672, -10168, -8664 )
+  // CHECK-NEXT: ( -23032, -20504, -17976, -15448 )
+  // CHECK-NEXT: ( -32888, -29336, -25784, -22232 )
+  // CHECK-NEXT: ( -42744, -38168, -33592, -29016 )
+  // CHECK-NEXT: ( -52600, -47000, -41400, -35800 )
+  // CHECK-NEXT: ( -62456, -55832, -49208, -42584 )
+  // CHECK-NEXT: ( -72312, -64664, -57016, -49368 )
+  // CHECK-NEXT: ( 75528, 67816, 60104, 52392 )
+  // CHECK-NEXT: ( 65672, 58984, 52296, 45608 )
+  // CHECK-NEXT: ( 55816, 50152, 44488, 38824 )
+  // CHECK-NEXT: ( 45960, 41320, 36680, 32040 )
+  // CHECK-NEXT: ( 36104, 32488, 28872, 25256 )
+  // CHECK-NEXT: ( 26248, 23656, 21064, 18472 )
+  // CHECK-NEXT: ( 16392, 14824, 13256, 11688 )
+  // CHECK-NEXT: ( 6536, 5992, 5448, 4904 )
+  //
+  call @kernel1(%a, %b, %c) : (memref<16x16xi8>, memref<4x16xi8>, memref<16x4xi32>) -> ()
+  call @print(%c) : (memref<16x4xi32>) -> ()
+
+  // Call kernel2 and verify result.
+  //
+  // CHECK-NEXT: ( 27400, 27880, 28360, 28840 )
+  // CHECK-NEXT: ( 83080, 84584, 86088, 87592 )
+  // CHECK-NEXT: ( 138760, 141288, 143816, 146344 )
+  // CHECK-NEXT: ( 194440, 197992, 201544, 205096 )
+  // CHECK-NEXT: ( 250120, 254696, 259272, 263848 )
+  // CHECK-NEXT: ( 305800, 311400, 317000, 322600 )
+  // CHECK-NEXT: ( 361480, 368104, 374728, 381352 )
+  // CHECK-NEXT: ( 417160, 424808, 432456, 440104 )
+  // CHECK-NEXT: ( -418040, -425752, -433464, -441176 )
+  // CHECK-NEXT: ( -362360, -369048, -375736, -382424 )
+  // CHECK-NEXT: ( -306680, -312344, -318008, -323672 )
+  // CHECK-NEXT: ( -251000, -255640, -260280, -264920 )
+  // CHECK-NEXT: ( -195320, -198936, -202552, -206168 )
+  // CHECK-NEXT: ( -139640, -142232, -144824, -147416 )
+  // CHECK-NEXT: ( -83960, -85528, -87096, -88664 )
+  // CHECK-NEXT: ( -28280, -28824, -29368, -29912 )
+  //
+  call @kernel2(%a, %b, %c) : (memref<16x16xi8>, memref<4x16xi8>, memref<16x4xi32>) -> ()
+  call @print(%c) : (memref<16x4xi32>) -> ()
+
+  // Call kernel3 and verify result.
+  //
+  // CHECK-NEXT: ( -3320, -2840, -2360, -1880 )
+  // CHECK-NEXT: ( -13176, -11672, -10168, -8664 )
+  // CHECK-NEXT: ( -23032, -20504, -17976, -15448 )
+  // CHECK-NEXT: ( -32888, -29336, -25784, -22232 )
+  // CHECK-NEXT: ( -42744, -38168, -33592, -29016 )
+  // CHECK-NEXT: ( -52600, -47000, -41400, -35800 )
+  // CHECK-NEXT: ( -62456, -55832, -49208, -42584 )
+  // CHECK-NEXT: ( -72312, -64664, -57016, -49368 )
+  // CHECK-NEXT: ( -82168, -73496, -64824, -56152 )
+  // CHECK-NEXT: ( -92024, -82328, -72632, -62936 )
+  // CHECK-NEXT: ( -101880, -91160, -80440, -69720 )
+  // CHECK-NEXT: ( -111736, -99992, -88248, -76504 )
+  // CHECK-NEXT: ( -121592, -108824, -96056, -83288 )
+  // CHECK-NEXT: ( -131448, -117656, -103864, -90072 )
+  // CHECK-NEXT: ( -141304, -126488, -111672, -96856 )
+  // CHECK-NEXT: ( -151160, -135320, -119480, -103640 )
+  //
+  call @kernel3(%a, %b, %c) : (memref<16x16xi8>, memref<4x16xi8>, memref<16x4xi32>) -> ()
+  call @print(%c) : (memref<16x4xi32>) -> ()
+
+  // Call kernel4 and verify result.
+  //
+  // CHECK-NEXT: ( 27400, 27880, 28360, 28840 )
+  // CHECK-NEXT: ( 83080, 84584, 86088, 87592 )
+  // CHECK-NEXT: ( 138760, 141288, 143816, 146344 )
+  // CHECK-NEXT: ( 194440, 197992, 201544, 205096 )
+  // CHECK-NEXT: ( 250120, 254696, 259272, 263848 )
+  // CHECK-NEXT: ( 305800, 311400, 317000, 322600 )
+  // CHECK-NEXT: ( 361480, 368104, 374728, 381352 )
+  // CHECK-NEXT: ( 417160, 424808, 432456, 440104 )
+  // CHECK-NEXT: ( 472840, 481512, 490184, 498856 )
+  // CHECK-NEXT: ( 528520, 538216, 547912, 557608 )
+  // CHECK-NEXT: ( 584200, 594920, 605640, 616360 )
+  // CHECK-NEXT: ( 639880, 651624, 663368, 675112 )
+  // CHECK-NEXT: ( 695560, 708328, 721096, 733864 )
+  // CHECK-NEXT: ( 751240, 765032, 778824, 792616 )
+  // CHECK-NEXT: ( 806920, 821736, 836552, 851368 )
+  // CHECK-NEXT: ( 862600, 878440, 894280, 910120 )
+  //
+  call @kernel4(%a, %b, %c) : (memref<16x16xi8>, memref<4x16xi8>, memref<16x4xi32>) -> ()
+  call @print(%c) : (memref<16x4xi32>) -> ()
+
+  // Release resources.
+  memref.dealloc %a : memref<16x16xi8>
+  memref.dealloc %b : memref<4x16xi8>
+  memref.dealloc %c : memref<16x4xi32>
+
+  return
+}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir
index 59eff35d33cf2b0e72ca636652b20ab489ff72c3..df848a04eae733343784797fe9099f0bc25d129d 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir
@@ -13,7 +13,7 @@ func @kernel1(%arg0: memref<2x8xi8>,
   %1 = amx.tile_load %arg0[%0, %0] : memref<2x8xi8>  into vector<2x8xi8>
   %2 = amx.tile_load %arg1[%0, %0] : memref<2x8xi8>  into vector<2x8xi8>
   %3 = amx.tile_zero : vector<2x2xi32>
-  %4 = amx.tile_muli %1, %2, %3 [true, true] : vector<2x8xi8>, vector<2x8xi8>, vector<2x2xi32>
+  %4 = amx.tile_muli %1 zext, %2 zext, %3 : vector<2x8xi8>, vector<2x8xi8>, vector<2x2xi32>
   amx.tile_store %arg2[%0, %0], %4 : memref<2x2xi32>, vector<2x2xi32>
   return
 }
@@ -26,7 +26,7 @@ func @kernel2(%arg0: memref<2x8xi8>,
   %1 = amx.tile_load %arg0[%0, %0] : memref<2x8xi8>  into vector<2x8xi8>
   %2 = amx.tile_load %arg1[%0, %0] : memref<2x8xi8>  into vector<2x8xi8>
   %3 = amx.tile_load %arg2[%0, %0] : memref<2x2xi32> into vector<2x2xi32>
-  %4 = amx.tile_muli %1, %2, %3 [true, true] : vector<2x8xi8>, vector<2x8xi8>, vector<2x2xi32>
+  %4 = amx.tile_muli %1 zext, %2 zext, %3 : vector<2x8xi8>, vector<2x8xi8>, vector<2x2xi32>
   amx.tile_store %arg2[%0, %0], %4 : memref<2x2xi32>, vector<2x2xi32>
   return
 }
@@ -38,9 +38,9 @@ func @entry() {
   %c2 = constant 2: index
 
   // Set up memory.
-  %a = alloc() : memref<2x8xi8>
-  %b = alloc() : memref<2x8xi8>
-  %c = alloc() : memref<2x2xi32>
+  %a = memref.alloc() : memref<2x8xi8>
+  %b = memref.alloc() : memref<2x8xi8>
+  %c = memref.alloc() : memref<2x2xi32>
 
   %0 = std.constant dense<[[1 , 2,  3 , 4 , 5,  6,  7,  8],
                            [9, 10, 11, 12, 13, 14, 15, 16]]> : vector<2x8xi8>
@@ -75,9 +75,9 @@ func @entry() {
   }
 
   // Release resources.
-  dealloc %a : memref<2x8xi8>
-  dealloc %b : memref<2x8xi8>
-  dealloc %c : memref<2x2xi32>
+  memref.dealloc %a : memref<2x8xi8>
+  memref.dealloc %b : memref<2x8xi8>
+  memref.dealloc %c : memref<2x2xi32>
 
   return
 }
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..64cf39d47b805c056b8ef28a0ce1f4fb4a292e85
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir
@@ -0,0 +1,81 @@
+// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-std -convert-vector-to-llvm="enable-amx" -convert-std-to-llvm | \
+// RUN: mlir-translate -mlir-to-llvmir | \
+// RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+// Note: To run this test, your CPU must support AMX.
+
+func @print(%arg0: memref<4x32xf32>) {
+  %fu = constant -1.0: f32
+  %c0 = constant 0: index
+  %c1 = constant 1: index
+  %c4 = constant 4: index
+  scf.for %i = %c0 to %c4 step %c1 {
+    %0 = vector.transfer_read %arg0[%i, %c0], %fu: memref<4x32xf32>, vector<32xf32>
+    vector.print %0 : vector<32xf32>
+  }
+  return
+}
+
+func @kernel(%arg0: memref<4x32xf32>) {
+  %c0  = constant 0: index
+  %c2  = constant 2 : index
+  %c4  = constant 4 : index
+  %c16 = constant 16 : index
+  %c32 = constant 32 : index
+  scf.for %i = %c0 to %c4 step %c2 {
+    scf.for %j = %c0 to %c32 step %c16 {
+      %0 = amx.tile_zero : vector<2x16xf32>
+      amx.tile_store %arg0[%i, %j], %0 : memref<4x32xf32>, vector<2x16xf32>
+      call @print(%arg0) : (memref<4x32xf32>) -> ()
+    }
+  }
+  return
+}
+
+func @entry() {
+  %f1  = constant 1.0: f32
+  %c0  = constant 0: index
+  %c1  = constant 1: index
+  %c4  = constant 4 : index
+  %c32 = constant 32 : index
+
+  // Set up memory.
+  %a = memref.alloc() : memref<4x32xf32>
+  scf.for %i = %c0 to %c4 step %c1 {
+    scf.for %j = %c0 to %c32 step %c1 {
+      memref.store %f1, %a[%i, %j] : memref<4x32xf32>
+    }
+  }
+
+  // Call kernel.
+  call @kernel(%a) : (memref<4x32xf32>) -> ()
+
+  // Verify progress of blocked tilezero.
+  //
+  // CHECK:      ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+  // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+  // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+  //
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+  // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+  //
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+  //
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  //
+
+  // Release resources.
+  memref.dealloc %a : memref<4x32xf32>
+
+  return
+}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir
index f49c66e4ce4bc26da7bbf2fd213ae54a14519eec..1a833f9ef1726d4b4b8c4660b2e6e7f141977494 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir
@@ -20,10 +20,10 @@ func @entry() {
   %c19 = constant 19: index
 
   // Set up memory.
-  %a = alloc(%c19, %c19) : memref<?x?xi32>
+  %a = memref.alloc(%c19, %c19) : memref<?x?xi32>
   scf.for %i = %c0 to %c19 step %c1 {
     scf.for %j = %c0 to %c19 step %c1 {
-      store %i1, %a[%i, %j] : memref<?x?xi32>
+      memref.store %i1, %a[%i, %j] : memref<?x?xi32>
     }
   }
 
@@ -90,7 +90,7 @@ func @entry() {
   }
 
   // Release resources.
-  dealloc %a : memref<?x?xi32>
+  memref.dealloc %a : memref<?x?xi32>
 
   return
 }
diff --git a/mlir/test/Integration/GPU/CUDA/gpu-to-cubin.mlir b/mlir/test/Integration/GPU/CUDA/gpu-to-cubin.mlir
index ec9720f5566651c34c2d6c46f81945498791d927..da3991fb1c7393ed87afac84cc3603cf69baa007 100644
--- a/mlir/test/Integration/GPU/CUDA/gpu-to-cubin.mlir
+++ b/mlir/test/Integration/GPU/CUDA/gpu-to-cubin.mlir
@@ -5,6 +5,7 @@
 // RUN: | mlir-cpu-runner \
 // RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_cuda_runtime%shlibext \
 // RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_runner_utils%shlibext \
+// RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_c_runner_utils%shlibext \
 // RUN:   --entry-point-result=void \
 // RUN: | FileCheck %s
 
@@ -21,7 +22,10 @@ func @other_func(%arg0 : f32, %arg1 : memref<?xf32>) {
 }
 
 // CHECK: [1, 1, 1, 1, 1]
+// CHECK: ( 1, 1 )
 func @main() {
+  %v0 = constant 0.0 : f32
+  %c0 = constant 0: index
   %arg0 = memref.alloc() : memref<5xf32>
   %21 = constant 5 : i32
   %22 = memref.cast %arg0 : memref<5xf32> to memref<?xf32>
@@ -31,6 +35,8 @@ func @main() {
   %24 = constant 1.0 : f32
   call @other_func(%24, %22) : (f32, memref<?xf32>) -> ()
   call @print_memref_f32(%23) : (memref<*xf32>) -> ()
+  %val1 = vector.transfer_read %arg0[%c0], %v0: memref<5xf32>, vector<2xf32>
+  vector.print %val1: vector<2xf32>
   return
 }
 
diff --git a/mlir/test/Integration/GPU/CUDA/lit.local.cfg b/mlir/test/Integration/GPU/CUDA/lit.local.cfg
index b063ddda7e1d6c9e945d2d1b8ced0751c5f32dce..0bdebfedeee36814a029ba1623d27aa4539271ec 100644
--- a/mlir/test/Integration/GPU/CUDA/lit.local.cfg
+++ b/mlir/test/Integration/GPU/CUDA/lit.local.cfg
@@ -1,2 +1,2 @@
 if not config.enable_cuda_runner:
-  config.unsupported = True
\ No newline at end of file
+  config.unsupported = True
diff --git a/mlir/test/mlir-rocm-runner/gpu-to-hsaco.mlir b/mlir/test/Integration/GPU/ROCM/gpu-to-hsaco.mlir
similarity index 82%
rename from mlir/test/mlir-rocm-runner/gpu-to-hsaco.mlir
rename to mlir/test/Integration/GPU/ROCM/gpu-to-hsaco.mlir
index 3f2d44fca38f4df5abb49c6e8cff54527e0a1159..fdc525bd2659cbe86c8ca04193f80c40b6b011c5 100644
--- a/mlir/test/mlir-rocm-runner/gpu-to-hsaco.mlir
+++ b/mlir/test/Integration/GPU/ROCM/gpu-to-hsaco.mlir
@@ -1,5 +1,9 @@
-// RUN: mlir-rocm-runner %s \
-// RUN:   --shared-libs=%cuda_wrapper_library_dir/libcuda-runtime-wrappers%shlibext \
+// RUN: mlir-opt %s \
+// RUN:   -gpu-kernel-outlining \
+// RUN:   -pass-pipeline='gpu.module(strip-debuginfo,convert-gpu-to-rocdl,gpu-to-hsaco)' \
+// RUN:   -gpu-to-llvm \
+// RUN: | mlir-cpu-runner \
+// RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_rocm_runtime%shlibext \
 // RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_runner_utils%shlibext \
 // RUN:   --entry-point-result=void \
 // RUN: | FileCheck %s
diff --git a/mlir/test/mlir-rocm-runner/lit.local.cfg b/mlir/test/Integration/GPU/ROCM/lit.local.cfg
similarity index 100%
rename from mlir/test/mlir-rocm-runner/lit.local.cfg
rename to mlir/test/Integration/GPU/ROCM/lit.local.cfg
diff --git a/mlir/test/mlir-rocm-runner/two-modules.mlir b/mlir/test/Integration/GPU/ROCM/two-modules.mlir
similarity index 74%
rename from mlir/test/mlir-rocm-runner/two-modules.mlir
rename to mlir/test/Integration/GPU/ROCM/two-modules.mlir
index 7c0faae5d135059f40bfb15247e8dd8c89accc41..3c6c56b0091a8d8177e0068c6c0ed331749e4d2b 100644
--- a/mlir/test/mlir-rocm-runner/two-modules.mlir
+++ b/mlir/test/Integration/GPU/ROCM/two-modules.mlir
@@ -1,5 +1,9 @@
-// RUN: mlir-rocm-runner %s \
-// RUN:   --shared-libs=%cuda_wrapper_library_dir/libcuda-runtime-wrappers%shlibext \
+// RUN: mlir-opt %s \
+// RUN:   -gpu-kernel-outlining \
+// RUN:   -pass-pipeline='gpu.module(strip-debuginfo,convert-gpu-to-rocdl,gpu-to-hsaco)' \
+// RUN:   -gpu-to-llvm \
+// RUN: | mlir-cpu-runner \
+// RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_rocm_runtime%shlibext \
 // RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_runner_utils%shlibext \
 // RUN:   --entry-point-result=void \
 // RUN: | FileCheck %s
@@ -30,5 +34,5 @@ func @main() {
   return
 }
 
-func @mgpuMemGetDeviceMemRef1dInt32(%ptr : memref<?xi32>) -> (memref<?xi32>)
-func @print_memref_i32(%ptr : memref<*xi32>)
+func private @mgpuMemGetDeviceMemRef1dInt32(%ptr : memref<?xi32>) -> (memref<?xi32>)
+func private @print_memref_i32(%ptr : memref<*xi32>)
diff --git a/mlir/test/mlir-rocm-runner/vecadd.mlir b/mlir/test/Integration/GPU/ROCM/vecadd.mlir
similarity index 87%
rename from mlir/test/mlir-rocm-runner/vecadd.mlir
rename to mlir/test/Integration/GPU/ROCM/vecadd.mlir
index d4dc862c60b6c546af11492691cb9a5bf5d6dda8..917be3c93d08992b136e0715d9db08da14684a07 100644
--- a/mlir/test/mlir-rocm-runner/vecadd.mlir
+++ b/mlir/test/Integration/GPU/ROCM/vecadd.mlir
@@ -1,5 +1,10 @@
-// RUN: mlir-rocm-runner %s \
-// RUN:   --shared-libs=%cuda_wrapper_library_dir/libcuda-runtime-wrappers%shlibext \
+// RUN: mlir-opt %s \
+// RUN:   -convert-scf-to-std \
+// RUN:   -gpu-kernel-outlining \
+// RUN:   -pass-pipeline='gpu.module(strip-debuginfo,convert-gpu-to-rocdl,gpu-to-hsaco)' \
+// RUN:   -gpu-to-llvm \
+// RUN: | mlir-cpu-runner \
+// RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_rocm_runtime%shlibext \
 // RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_runner_utils%shlibext \
 // RUN:   --entry-point-result=void \
 // RUN: | FileCheck %s
diff --git a/mlir/test/mlir-rocm-runner/vector-transferops.mlir b/mlir/test/Integration/GPU/ROCM/vector-transferops.mlir
similarity index 90%
rename from mlir/test/mlir-rocm-runner/vector-transferops.mlir
rename to mlir/test/Integration/GPU/ROCM/vector-transferops.mlir
index eda541a2d814259520cc25fce4ec0750c9b6cf08..c2807b64c06413e56224a74b39437901778044ea 100644
--- a/mlir/test/mlir-rocm-runner/vector-transferops.mlir
+++ b/mlir/test/Integration/GPU/ROCM/vector-transferops.mlir
@@ -1,5 +1,10 @@
-// RUN: mlir-rocm-runner %s \
-// RUN:   --shared-libs=%cuda_wrapper_library_dir/libcuda-runtime-wrappers%shlibext \
+// RUN: mlir-opt %s \
+// RUN:   -convert-scf-to-std \
+// RUN:   -gpu-kernel-outlining \
+// RUN:   -pass-pipeline='gpu.module(strip-debuginfo,convert-gpu-to-rocdl,gpu-to-hsaco)' \
+// RUN:   -gpu-to-llvm \
+// RUN: | mlir-cpu-runner \
+// RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_rocm_runtime%shlibext \
 // RUN:   --shared-libs=%linalg_test_lib_dir/libmlir_runner_utils%shlibext \
 // RUN:   --entry-point-result=void \
 // RUN: | FileCheck %s
diff --git a/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir b/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..adabe560ffdfdce790dc4a0f0a0f2a97e965b05c
--- /dev/null
+++ b/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir
@@ -0,0 +1,15 @@
+// RUN: mlir-opt --test-data-layout-query %s | FileCheck %s
+
+module attributes { dlti.dl_spec = #dlti.dl_spec<
+      #dlti.dl_entry<!test.test_type_with_layout<10>, ["size", 12]>,
+      #dlti.dl_entry<!test.test_type_with_layout<20>, ["alignment", 32]>>} {
+  // CHECK-LABEL: @module_level_layout
+  func @module_level_layout() {
+     // CHECK: alignment = 32
+     // CHECK: bitsize = 12
+     // CHECK: preferred = 1
+     // CHECK: size = 2
+    "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
+    return
+  }
+}
diff --git a/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir b/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir
index 2976326fa59e2049fef8b0dcae60c854af8a8c7d..41f21d97cb41a16ee6f8c82b17e3596775e51b5d 100644
--- a/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir
+++ b/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir
@@ -3,10 +3,12 @@
 // CHECK-LABEL: @no_layout_builtin
 func @no_layout_builtin() {
   // CHECK: alignment = 4
+  // CHECK: bitsize = 32
   // CHECK: preferred = 4
   // CHECK: size = 4
   "test.data_layout_query"() : () -> i32
   // CHECK: alignment = 8
+  // CHECK: bitsize = 64
   // CHECK: preferred = 8
   // CHECK: size = 8
   "test.data_layout_query"() : () -> f64
@@ -16,6 +18,7 @@ func @no_layout_builtin() {
 // CHECK-LABEL: @no_layout_custom
 func @no_layout_custom() {
   // CHECK: alignment = 1
+  // CHECK: bitsize = 1
   // CHECK: preferred = 1
   // CHECK: size = 1
   "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
@@ -26,6 +29,7 @@ func @no_layout_custom() {
 func @layout_op_no_layout() {
   "test.op_with_data_layout"() ({
     // CHECK: alignment = 1
+    // CHECK: bitsize = 1
     // CHECK: preferred = 1
     // CHECK: size = 1
     "test.data_layout_query"() : () -> !test.test_type_with_layout<1000>
@@ -38,8 +42,9 @@ func @layout_op_no_layout() {
 func @layout_op() {
   "test.op_with_data_layout"() ({
     // CHECK: alignment = 20
+    // CHECK: bitsize = 10
     // CHECK: preferred = 1
-    // CHECK: size = 10
+    // CHECK: size = 2
     "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
     "test.maybe_terminator"() : () -> ()
   }) { dlti.dl_spec = #dlti.dl_spec<
@@ -55,8 +60,9 @@ func @nested_inner_only() {
   "test.op_with_data_layout"() ({
     "test.op_with_data_layout"() ({
       // CHECK: alignment = 20
+      // CHECK: bitsize = 10
       // CHECK: preferred = 1
-      // CHECK: size = 10
+      // CHECK: size = 2
       "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
       "test.maybe_terminator"() : () -> ()
     }) { dlti.dl_spec = #dlti.dl_spec<
@@ -74,8 +80,9 @@ func @nested_outer_only() {
   "test.op_with_data_layout"() ({
     "test.op_with_data_layout"() ({
       // CHECK: alignment = 20
+      // CHECK: bitsize = 10
       // CHECK: preferred = 1
-      // CHECK: size = 10
+      // CHECK: size = 2
       "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
       "test.maybe_terminator"() : () -> ()
     }) : () -> ()
@@ -93,8 +100,9 @@ func @nested_middle_only() {
     "test.op_with_data_layout"() ({
       "test.op_with_data_layout"() ({
         // CHECK: alignment = 20
+        // CHECK: bitsize = 10
         // CHECK: preferred = 1
-        // CHECK: size = 10
+        // CHECK: size = 2
         "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
         "test.maybe_terminator"() : () -> ()
     }) : () -> ()
@@ -114,8 +122,9 @@ func @nested_combine_with_missing() {
     "test.op_with_data_layout"() ({
       "test.op_with_data_layout"() ({
         // CHECK: alignment = 20
+        // CHECK: bitsize = 10
         // CHECK: preferred = 30
-        // CHECK: size = 10
+        // CHECK: size = 2
         "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
         "test.maybe_terminator"() : () -> ()
       }) : () -> ()
@@ -125,8 +134,9 @@ func @nested_combine_with_missing() {
         #dlti.dl_entry<!test.test_type_with_layout<20>, ["alignment", 20]>
       >} : () -> ()
     // CHECK: alignment = 1
+    // CHECK: bitsize = 42
     // CHECK: preferred = 30
-    // CHECK: size = 42
+    // CHECK: size = 6
     "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
     "test.maybe_terminator"() : () -> ()
   }) { dlti.dl_spec = #dlti.dl_spec<
@@ -142,8 +152,9 @@ func @nested_combine_all() {
     "test.op_with_data_layout"() ({
       "test.op_with_data_layout"() ({
         // CHECK: alignment = 20
+        // CHECK: bitsize = 3
         // CHECK: preferred = 30
-        // CHECK: size = 3
+        // CHECK: size = 1
         "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
         "test.maybe_terminator"() : () -> ()
       }) { dlti.dl_spec = #dlti.dl_spec<
@@ -151,8 +162,9 @@ func @nested_combine_all() {
           #dlti.dl_entry<!test.test_type_with_layout<30>, ["preferred", 30]>
         >} : () -> ()
       // CHECK: alignment = 20
+      // CHECK: bitsize = 10
       // CHECK: preferred = 30
-      // CHECK: size = 10
+      // CHECK: size = 2
       "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
       "test.maybe_terminator"() : () -> ()
     }) { dlti.dl_spec = #dlti.dl_spec<
@@ -160,8 +172,9 @@ func @nested_combine_all() {
         #dlti.dl_entry<!test.test_type_with_layout<20>, ["alignment", 20]>
       >} : () -> ()
     // CHECK: alignment = 1
+    // CHECK: bitsize = 42
     // CHECK: preferred = 30
-    // CHECK: size = 42
+    // CHECK: size = 6
     "test.data_layout_query"() : () -> !test.test_type_with_layout<10>
     "test.maybe_terminator"() : () -> ()
   }) { dlti.dl_spec = #dlti.dl_spec<
diff --git a/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir b/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..02adc7f54544ef93927731a28ef8f05461d70ac8
--- /dev/null
+++ b/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir
@@ -0,0 +1,29 @@
+// RUN: mlir-opt --split-input-file --verify-diagnostics --test-data-layout-query %s | FileCheck %s
+
+// expected-error@below {{expected integer attribute in the data layout entry for 'index'}}
+module attributes { dlti.dl_spec = #dlti.dl_spec<
+  #dlti.dl_entry<index, [32]>>} {
+}
+
+// -----
+
+// CHECK-LABEL: @index
+module @index attributes { dlti.dl_spec = #dlti.dl_spec<
+  #dlti.dl_entry<index, 32>>} {
+  func @query() {
+    // CHECK: bitsize = 32
+    "test.data_layout_query"() : () -> index
+    return
+  }
+}
+
+// -----
+
+// CHECK-LABEL: @index_default
+module @index_default {
+  func @query() {
+    // CHECK: bitsize = 64
+    "test.data_layout_query"() : () -> index
+    return
+  }
+}
diff --git a/mlir/test/Rewrite/pdl-bytecode.mlir b/mlir/test/Rewrite/pdl-bytecode.mlir
index b2a22d0a8749cccdc76965497722f6b866d96ca8..d630fa2aa14db3232362045d2d6338f27f6bf69f 100644
--- a/mlir/test/Rewrite/pdl-bytecode.mlir
+++ b/mlir/test/Rewrite/pdl-bytecode.mlir
@@ -25,7 +25,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.replaced_by_pattern"() -> ()
+      %op = pdl_interp.create_operation "test.replaced_by_pattern"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -40,6 +40,38 @@ module @ir attributes { test.apply_constraint_1 } {
 
 // -----
 
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    %results = pdl_interp.get_results of %root : !pdl.range<value>
+    %types = pdl_interp.get_value_type of %results : !pdl.range<type>
+    pdl_interp.apply_constraint "multi_entity_var_constraint"(%results, %types : !pdl.range<value>, !pdl.range<type>) -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.replaced_by_pattern"
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.apply_constraint_2
+// CHECK-NOT: "test.replaced_by_pattern"
+// CHECK: "test.replaced_by_pattern"
+module @ir attributes { test.apply_constraint_2 } {
+  "test.failure_op"() { test_attr } : () -> ()
+  "test.success_op"() : () -> (i32, i64)
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::ApplyRewriteOp
 //===----------------------------------------------------------------------===//
@@ -58,7 +90,7 @@ module @patterns {
   module @rewriters {
     func @success(%root : !pdl.operation) {
       %operand = pdl_interp.get_operand 0 of %root
-      pdl_interp.apply_rewrite "rewriter"[42](%operand : !pdl.value) on %root
+      pdl_interp.apply_rewrite "rewriter"[42](%root, %operand : !pdl.operation, !pdl.value)
       pdl_interp.finalize
     }
   }
@@ -72,6 +104,97 @@ module @ir attributes { test.apply_rewrite_1 } {
   %input = "test.op_input"() : () -> i32
   "test.op"(%input) : (i32) -> ()
 }
+
+// -----
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.apply_rewrite "creator"(%root : !pdl.operation) : !pdl.operation
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.apply_rewrite_2
+// CHECK: "test.success"
+module @ir attributes { test.apply_rewrite_2 } {
+  "test.op"() : () -> ()
+}
+
+// -----
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %operands, %types = pdl_interp.apply_rewrite "var_creator"(%root : !pdl.operation) : !pdl.range<value>, !pdl.range<type>
+      %op = pdl_interp.create_operation "test.success"(%operands : !pdl.range<value>) -> (%types : !pdl.range<type>)
+      pdl_interp.replace %root with (%operands : !pdl.range<value>)
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.apply_rewrite_3
+// CHECK: %[[OPERAND:.*]] = "test.producer"
+// CHECK: "test.success"(%[[OPERAND]]) : (i32) -> i32
+// CHECK: "test.consumer"(%[[OPERAND]])
+module @ir attributes { test.apply_rewrite_3 } {
+  %first_operand = "test.producer"() : () -> (i32)
+  %operand = "test.op"(%first_operand) : (i32) -> (i32)
+  "test.consumer"(%operand) : (i32) -> ()
+}
+
+// -----
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %type = pdl_interp.apply_rewrite "type_creator" : !pdl.type
+      %newOp = pdl_interp.create_operation "test.success" -> (%type : !pdl.type)
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.apply_rewrite_4
+// CHECK: "test.success"() : () -> f32
+module @ir attributes { test.apply_rewrite_4 } {
+  "test.op"() : () -> ()
+}
+
 // -----
 
 //===----------------------------------------------------------------------===//
@@ -93,7 +216,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -108,6 +231,40 @@ module @ir attributes { test.are_equal_1 } {
 
 // -----
 
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    %const_types = pdl_interp.create_types [i32, i64]
+    %results = pdl_interp.get_results of %root : !pdl.range<value>
+    %result_types = pdl_interp.get_value_type of %results : !pdl.range<type>
+    pdl_interp.are_equal %result_types, %const_types : !pdl.range<type> -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.are_equal_2
+// CHECK: "test.not_equal"
+// CHECK: "test.success"
+// CHECK-NOT: "test.op"
+module @ir attributes { test.are_equal_2 } {
+  "test.not_equal"() : () -> (i32)
+  "test.op"() : () -> (i32, i64)
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::BranchOp
 //===----------------------------------------------------------------------===//
@@ -128,7 +285,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -161,7 +318,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -182,7 +339,10 @@ module @ir attributes { test.check_attribute_1 } {
 
 module @patterns {
   func @matcher(%root : !pdl.operation) {
-    pdl_interp.check_operand_count of %root is 1 -> ^pat, ^end
+    pdl_interp.check_operand_count of %root is at_least 1 -> ^exact_check, ^end
+
+  ^exact_check:
+    pdl_interp.check_operand_count of %root is 2 -> ^pat, ^end
 
   ^pat:
     pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
@@ -193,7 +353,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -205,7 +365,7 @@ module @patterns {
 // CHECK: "test.success"
 module @ir attributes { test.check_operand_count_1 } {
   %operand = "test.op"() : () -> i32
-  "test.op"(%operand) : (i32) -> ()
+  "test.op"(%operand, %operand) : (i32, i32) -> ()
 }
 
 // -----
@@ -227,7 +387,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -248,7 +408,10 @@ module @ir attributes { test.check_operation_name_1 } {
 
 module @patterns {
   func @matcher(%root : !pdl.operation) {
-    pdl_interp.check_result_count of %root is 1 -> ^pat, ^end
+    pdl_interp.check_result_count of %root is at_least 1 -> ^exact_check, ^end
+
+  ^exact_check:
+    pdl_interp.check_result_count of %root is 2 -> ^pat, ^end
 
   ^pat:
     pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
@@ -259,7 +422,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -267,9 +430,12 @@ module @patterns {
 }
 
 // CHECK-LABEL: test.check_result_count_1
+// CHECK: "test.op"() : () -> i32
 // CHECK: "test.success"() : () -> ()
+// CHECK-NOT: "test.op"() : () -> (i32, i32)
 module @ir attributes { test.check_result_count_1 } {
   "test.op"() : () -> i32
+  "test.op"() : () -> (i32, i32)
 }
 
 // -----
@@ -296,7 +462,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -312,22 +478,16 @@ module @ir attributes { test.check_type_1 } {
 // -----
 
 //===----------------------------------------------------------------------===//
-// pdl_interp::CreateAttributeOp
-//===----------------------------------------------------------------------===//
-
-// Fully tested within the tests for other operations.
-
-//===----------------------------------------------------------------------===//
-// pdl_interp::CreateNativeOp
+// pdl_interp::CheckTypesOp
 //===----------------------------------------------------------------------===//
 
-// -----
-
 module @patterns {
   func @matcher(%root : !pdl.operation) {
-    pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end
+    %results = pdl_interp.get_results of %root : !pdl.range<value>
+    %result_types = pdl_interp.get_value_type of %results : !pdl.range<type>
+    pdl_interp.check_types %result_types are [i32] -> ^pat2, ^end
 
-  ^pat:
+  ^pat2:
     pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
 
   ^end:
@@ -336,19 +496,30 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_native "creator"(%root : !pdl.operation) : !pdl.operation
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
   }
 }
 
-// CHECK-LABEL: test.create_native_1
+// CHECK-LABEL: test.check_types_1
+// CHECK: "test.op"() : () -> (i32, i64)
 // CHECK: "test.success"
-module @ir attributes { test.create_native_1 } {
-  "test.op"() : () -> ()
+// CHECK-NOT: "test.op"() : () -> i32
+module @ir attributes { test.check_types_1 } {
+  "test.op"() : () -> (i32, i64)
+  "test.op"() : () -> i32
 }
 
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CreateAttributeOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::CreateOperationOp
 //===----------------------------------------------------------------------===//
@@ -378,7 +549,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -393,6 +564,12 @@ module @ir attributes { test.create_type_1 } {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// pdl_interp::CreateTypesOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::EraseOp
 //===----------------------------------------------------------------------===//
@@ -428,8 +605,8 @@ module @patterns {
   ^pat1:
     %operand0 = pdl_interp.get_operand 0 of %root
     %operand4 = pdl_interp.get_operand 4 of %root
-    %defOp0 = pdl_interp.get_defining_op of %operand0
-    %defOp4 = pdl_interp.get_defining_op of %operand4
+    %defOp0 = pdl_interp.get_defining_op of %operand0 : !pdl.value
+    %defOp4 = pdl_interp.get_defining_op of %operand4 : !pdl.value
     pdl_interp.are_equal %defOp0, %defOp4 : !pdl.operation -> ^pat2, ^end
 
   ^pat2:
@@ -441,7 +618,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -468,6 +645,110 @@ module @ir attributes { test.get_defining_op_1 } {
 
 // Fully tested within the tests for other operations.
 
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetOperandsOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operand_count of %root is 2 -> ^pat1, ^end
+
+  ^pat1:
+    %operands = pdl_interp.get_operands 0 of %root : !pdl.range<value>
+    %full_operands = pdl_interp.get_operands of %root : !pdl.range<value>
+    pdl_interp.are_equal %operands, %full_operands : !pdl.range<value> -> ^pat2, ^end
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.get_operands_1
+// CHECK: "test.success"
+module @ir attributes { test.get_operands_1 } {
+  %inputs:2 = "test.producer"() : () -> (i32, i32)
+  "test.op"(%inputs#0, %inputs#1) : (i32, i32) -> ()
+}
+
+// -----
+
+// Test all of the various combinations related to `AttrSizedOperandSegments`.
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.attr_sized_operands" -> ^pat1, ^end
+
+  ^pat1:
+    %operands_0 = pdl_interp.get_operands 0 of %root : !pdl.range<value>
+    pdl_interp.is_not_null %operands_0 : !pdl.range<value> -> ^pat2, ^end
+
+  ^pat2:
+    %operands_0_single = pdl_interp.get_operands 0 of %root : !pdl.value
+    pdl_interp.is_not_null %operands_0_single : !pdl.value -> ^end, ^pat3
+
+  ^pat3:
+    %operands_1 = pdl_interp.get_operands 1 of %root : !pdl.range<value>
+    pdl_interp.is_not_null %operands_1 : !pdl.range<value> -> ^pat4, ^end
+
+  ^pat4:
+    %operands_1_single = pdl_interp.get_operands 1 of %root : !pdl.value
+    pdl_interp.is_not_null %operands_1_single : !pdl.value -> ^end, ^pat5
+
+  ^pat5:
+    %operands_2 = pdl_interp.get_operands 2 of %root : !pdl.range<value>
+    pdl_interp.is_not_null %operands_2 : !pdl.range<value> -> ^pat6, ^end
+
+  ^pat6:
+    %operands_2_single = pdl_interp.get_operands 2 of %root : !pdl.value
+    pdl_interp.is_not_null %operands_2_single : !pdl.value -> ^pat7, ^end
+
+  ^pat7:
+    %invalid_operands = pdl_interp.get_operands 50 of %root : !pdl.value
+    pdl_interp.is_not_null %invalid_operands : !pdl.value -> ^end, ^pat8
+
+  ^pat8:
+    pdl_interp.record_match @rewriters::@success(%root, %operands_0, %operands_1, %operands_2, %operands_2_single : !pdl.operation, !pdl.range<value>, !pdl.range<value>, !pdl.range<value>, !pdl.value) : benefit(1), loc([%root]) -> ^end
+
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root: !pdl.operation, %operands_0: !pdl.range<value>, %operands_1: !pdl.range<value>, %operands_2: !pdl.range<value>, %operands_2_single: !pdl.value) {
+      %op0 = pdl_interp.create_operation "test.success"(%operands_0 : !pdl.range<value>)
+      %op1 = pdl_interp.create_operation "test.success"(%operands_1 : !pdl.range<value>)
+      %op2 = pdl_interp.create_operation "test.success"(%operands_2 : !pdl.range<value>)
+      %op3 = pdl_interp.create_operation "test.success"(%operands_2_single : !pdl.value)
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.get_operands_2
+// CHECK-NEXT:  %[[INPUTS:.*]]:5 = "test.producer"() : () -> (i32, i32, i32, i32, i32)
+// CHECK-NEXT:  "test.success"() : () -> ()
+// CHECK-NEXT:  "test.success"(%[[INPUTS]]#0, %[[INPUTS]]#1, %[[INPUTS]]#2, %[[INPUTS]]#3) : (i32, i32, i32, i32) -> ()
+// CHECK-NEXT:  "test.success"(%[[INPUTS]]#4) : (i32) -> ()
+// CHECK-NEXT:  "test.success"(%[[INPUTS]]#4) : (i32) -> ()
+module @ir attributes { test.get_operands_2 } {
+  %inputs:5 = "test.producer"() : () -> (i32, i32, i32, i32, i32)
+  "test.attr_sized_operands"(%inputs#0, %inputs#1, %inputs#2, %inputs#3, %inputs#4) {operand_segment_sizes = dense<[0, 4, 1, 0]> : vector<4xi32>} : (i32, i32, i32, i32, i32) -> ()
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::GetResultOp
 //===----------------------------------------------------------------------===//
@@ -479,8 +760,8 @@ module @patterns {
   ^pat1:
     %result0 = pdl_interp.get_result 0 of %root
     %result4 = pdl_interp.get_result 4 of %root
-    %result0_type = pdl_interp.get_value_type of %result0
-    %result4_type = pdl_interp.get_value_type of %result4
+    %result0_type = pdl_interp.get_value_type of %result0 : !pdl.type
+    %result4_type = pdl_interp.get_value_type of %result4 : !pdl.type
     pdl_interp.are_equal %result0_type, %result4_type : !pdl.type -> ^pat2, ^end
 
   ^pat2:
@@ -492,7 +773,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -509,6 +790,119 @@ module @ir attributes { test.get_result_1 } {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetResultsOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_result_count of %root is 5 -> ^pat1, ^end
+
+  ^pat1:
+    %results = pdl_interp.get_results 0 of %root : !pdl.range<value>
+    %full_results = pdl_interp.get_results of %root : !pdl.range<value>
+    pdl_interp.are_equal %results, %full_results : !pdl.range<value> -> ^pat2, ^end
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.get_results_1
+// CHECK: "test.success"
+module @ir attributes { test.get_results_1 } {
+  %a:5 = "test.producer"() : () -> (i32, i32, i32, i32, i32)
+}
+
+// -----
+
+// Test all of the various combinations related to `AttrSizedResultSegments`.
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.attr_sized_results" -> ^pat1, ^end
+
+  ^pat1:
+    %results_0 = pdl_interp.get_results 0 of %root : !pdl.range<value>
+    pdl_interp.is_not_null %results_0 : !pdl.range<value> -> ^pat2, ^end
+
+  ^pat2:
+    %results_0_single = pdl_interp.get_results 0 of %root : !pdl.value
+    pdl_interp.is_not_null %results_0_single : !pdl.value -> ^end, ^pat3
+
+  ^pat3:
+    %results_1 = pdl_interp.get_results 1 of %root : !pdl.range<value>
+    pdl_interp.is_not_null %results_1 : !pdl.range<value> -> ^pat4, ^end
+
+  ^pat4:
+    %results_1_single = pdl_interp.get_results 1 of %root : !pdl.value
+    pdl_interp.is_not_null %results_1_single : !pdl.value -> ^end, ^pat5
+
+  ^pat5:
+    %results_2 = pdl_interp.get_results 2 of %root : !pdl.range<value>
+    pdl_interp.is_not_null %results_2 : !pdl.range<value> -> ^pat6, ^end
+
+  ^pat6:
+    %results_2_single = pdl_interp.get_results 2 of %root : !pdl.value
+    pdl_interp.is_not_null %results_2_single : !pdl.value -> ^pat7, ^end
+
+  ^pat7:
+    %invalid_results = pdl_interp.get_results 50 of %root : !pdl.value
+    pdl_interp.is_not_null %invalid_results : !pdl.value -> ^end, ^pat8
+
+  ^pat8:
+    pdl_interp.record_match @rewriters::@success(%root, %results_0, %results_1, %results_2, %results_2_single : !pdl.operation, !pdl.range<value>, !pdl.range<value>, !pdl.range<value>, !pdl.value) : benefit(1), loc([%root]) -> ^end
+
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root: !pdl.operation, %results_0: !pdl.range<value>, %results_1: !pdl.range<value>, %results_2: !pdl.range<value>, %results_2_single: !pdl.value) {
+      %results_0_types = pdl_interp.get_value_type of %results_0 : !pdl.range<type>
+      %results_1_types = pdl_interp.get_value_type of %results_1 : !pdl.range<type>
+      %results_2_types = pdl_interp.get_value_type of %results_2 : !pdl.range<type>
+      %results_2_single_types = pdl_interp.get_value_type of %results_2_single : !pdl.type
+
+      %op0 = pdl_interp.create_operation "test.success" -> (%results_0_types : !pdl.range<type>)
+      %op1 = pdl_interp.create_operation "test.success" -> (%results_1_types : !pdl.range<type>)
+      %op2 = pdl_interp.create_operation "test.success" -> (%results_2_types : !pdl.range<type>)
+      %op3 = pdl_interp.create_operation "test.success" -> (%results_2_single_types : !pdl.type)
+
+      %new_results_0 = pdl_interp.get_results of %op0 : !pdl.range<value>
+      %new_results_1 = pdl_interp.get_results of %op1 : !pdl.range<value>
+      %new_results_2 = pdl_interp.get_results of %op2 : !pdl.range<value>
+
+      pdl_interp.replace %root with (%new_results_0, %new_results_1, %new_results_2 : !pdl.range<value>, !pdl.range<value>, !pdl.range<value>)
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.get_results_2
+// CHECK: "test.success"() : () -> ()
+// CHECK: %[[RESULTS_1:.*]]:4 = "test.success"() : () -> (i32, i32, i32, i32)
+// CHECK: %[[RESULTS_2:.*]] = "test.success"() : () -> i32
+// CHECK: %[[RESULTS_2_SINGLE:.*]] = "test.success"() : () -> i32
+// CHECK: "test.consumer"(%[[RESULTS_1]]#0, %[[RESULTS_1]]#1, %[[RESULTS_1]]#2, %[[RESULTS_1]]#3, %[[RESULTS_2]]) : (i32, i32, i32, i32, i32) -> ()
+module @ir attributes { test.get_results_2 } {
+  %results:5 = "test.attr_sized_results"() {result_segment_sizes = dense<[0, 4, 1, 0]> : vector<4xi32>} : () -> (i32, i32, i32, i32, i32)
+  "test.consumer"(%results#0, %results#1, %results#2, %results#3, %results#4) : (i32, i32, i32, i32, i32) -> ()
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::GetValueTypeOp
 //===----------------------------------------------------------------------===//
@@ -516,7 +910,7 @@ module @ir attributes { test.get_result_1 } {
 // Fully tested within the tests for other operations.
 
 //===----------------------------------------------------------------------===//
-// pdl_interp::InferredTypeOp
+// pdl_interp::InferredTypesOp
 //===----------------------------------------------------------------------===//
 
 // Fully tested within the tests for other operations.
@@ -552,7 +946,7 @@ module @patterns {
       pdl_interp.finalize
     }
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -567,6 +961,43 @@ module @ir attributes { test.record_match_1 } {
 
 // -----
 
+// Check that ranges are properly forwarded to the result.
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat1, ^end
+
+  ^pat1:
+    %operands = pdl_interp.get_operands of %root : !pdl.range<value>
+    %results = pdl_interp.get_results of %root : !pdl.range<value>
+    %types = pdl_interp.get_value_type of %results : !pdl.range<type>
+    pdl_interp.record_match @rewriters::@success(%operands, %types, %root : !pdl.range<value>, !pdl.range<type>, !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%operands: !pdl.range<value>, %types: !pdl.range<type>, %root: !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"(%operands : !pdl.range<value>) -> (%types : !pdl.range<type>)
+      %results = pdl_interp.get_results of %op : !pdl.range<value>
+      pdl_interp.replace %root with (%results : !pdl.range<value>)
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.record_match_2
+// CHECK: %[[OPERAND:.*]] = "test.producer"() : () -> i32
+// CHECK: %[[RESULTS:.*]]:2 = "test.success"(%[[OPERAND]]) : (i32) -> (i64, i32)
+// CHECK: "test.consumer"(%[[RESULTS]]#0, %[[RESULTS]]#1) : (i64, i32) -> ()
+module @ir attributes { test.record_match_2 } {
+  %input = "test.producer"() : () -> i32
+  %results:2 = "test.op"(%input) : (i32) -> (i64, i32)
+  "test.consumer"(%results#0, %results#1) : (i64, i32) -> ()
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // pdl_interp::ReplaceOp
 //===----------------------------------------------------------------------===//
@@ -585,7 +1016,7 @@ module @patterns {
   module @rewriters {
     func @success(%root : !pdl.operation) {
       %operand = pdl_interp.get_operand 0 of %root
-      pdl_interp.replace %root with (%operand)
+      pdl_interp.replace %root with (%operand : !pdl.value)
       pdl_interp.finalize
     }
   }
@@ -625,7 +1056,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -660,7 +1091,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -696,7 +1127,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -731,7 +1162,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -771,7 +1202,7 @@ module @patterns {
 
   module @rewriters {
     func @success(%root : !pdl.operation) {
-      %op = pdl_interp.create_operation "test.success"() -> ()
+      %op = pdl_interp.create_operation "test.success"
       pdl_interp.erase %root
       pdl_interp.finalize
     }
@@ -783,3 +1214,40 @@ module @patterns {
 module @ir attributes { test.switch_type_1 } {
   "test.op"() { test_attr = 10 : i32 } : () -> ()
 }
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::SwitchTypesOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    %results = pdl_interp.get_results of %root : !pdl.range<value>
+    %types = pdl_interp.get_value_type of %results : !pdl.range<type>
+    pdl_interp.switch_types %types to [[i64, i64], [i32]](^pat2, ^end) -> ^end
+
+  ^pat2:
+    pdl_interp.switch_types %types to [[i32], [i64, i32]](^end, ^end) -> ^pat3
+
+  ^pat3:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.switch_types_1
+// CHECK: "test.success"
+module @ir attributes { test.switch_types_1 } {
+  %results:2 = "test.op"() : () -> (i64, i64)
+}
diff --git a/mlir/test/Target/LLVMIR/arm-neon.mlir b/mlir/test/Target/LLVMIR/arm-neon.mlir
index 3cd7641c5798a42d36e4fc7e2fa3afec5b48810e..d99f573c8bec0cbefd9abf671deeee062304e65b 100644
--- a/mlir/test/Target/LLVMIR/arm-neon.mlir
+++ b/mlir/test/Target/LLVMIR/arm-neon.mlir
@@ -4,16 +4,16 @@
 llvm.func @arm_neon_smull(%arg0: vector<8xi8>, %arg1: vector<8xi8>) -> !llvm.struct<(vector<8xi16>, vector<4xi32>, vector<2xi64>)> {
   //      CHECK: %[[V0:.*]] = call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %{{.*}}, <8 x i8> %{{.*}})
   // CHECK-NEXT: %[[V00:.*]] = shufflevector <8 x i16> %3, <8 x i16> %[[V0]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-  %0 = arm_neon.smull %arg0, %arg1 : vector<8xi8> to vector<8xi16>
+  %0 = arm_neon.intr.smull %arg0, %arg1 : vector<8xi8> to vector<8xi16>
   %1 = llvm.shufflevector %0, %0 [3, 4, 5, 6] : vector<8xi16>, vector<8xi16>
 
   // CHECK-NEXT: %[[V1:.*]] = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %[[V00]], <4 x i16> %[[V00]])
   // CHECK-NEXT: %[[V11:.*]] = shufflevector <4 x i32> %[[V1]], <4 x i32> %[[V1]], <2 x i32> <i32 1, i32 2>
-  %2 = arm_neon.smull %1, %1 : vector<4xi16> to vector<4xi32>
+  %2 = arm_neon.intr.smull %1, %1 : vector<4xi16> to vector<4xi32>
   %3 = llvm.shufflevector %2, %2 [1, 2] : vector<4xi32>, vector<4xi32>
 
   // CHECK-NEXT: %[[V1:.*]] = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %[[V11]], <2 x i32> %[[V11]])
-  %4 = arm_neon.smull %3, %3 : vector<2xi32> to vector<2xi64>
+  %4 = arm_neon.intr.smull %3, %3 : vector<2xi32> to vector<2xi64>
 
   %5 = llvm.mlir.undef : !llvm.struct<(vector<8xi16>, vector<4xi32>, vector<2xi64>)>
   %6 = llvm.insertvalue %0, %5[0] : !llvm.struct<(vector<8xi16>, vector<4xi32>, vector<2xi64>)>
@@ -23,3 +23,19 @@ llvm.func @arm_neon_smull(%arg0: vector<8xi8>, %arg1: vector<8xi8>) -> !llvm.str
   //      CHECK: ret { <8 x i16>, <4 x i32>, <2 x i64> }
   llvm.return %8 : !llvm.struct<(vector<8xi16>, vector<4xi32>, vector<2xi64>)>
 }
+
+// CHECK-LABEL: arm_neon_sdot_i8i8
+llvm.func @arm_neon_sdot_i8i8(%a: vector<2xi32>, %b: vector<8xi8>, %c: vector<8xi8>) -> vector<2xi32> {
+  // CHECK: %[[V0:.*]] = call <2 x i32> @llvm.aarch64.neon.sdot.v2i32.v8i8(<2 x i32> %{{.*}}, <8 x i8> %{{.*}}, <8 x i8> %{{.*}})
+  // CHECK-NEXT: ret <2 x i32>
+  %0 = arm_neon.intr.sdot %a, %b, %c : vector<8xi8>, vector<8xi8> to vector<2xi32>
+  llvm.return %0 : vector<2xi32>
+}
+
+// CHECK-LABEL: arm_neon_sdot_i16i16
+llvm.func @arm_neon_sdot_i16i16(%a: vector<4xi32>, %b: vector<16xi8>, %c: vector<16xi8>) -> vector<4xi32> {
+  // CHECK: %[[V0:.*]] = call <4 x i32> @llvm.aarch64.neon.sdot.v4i32.v16i8(<4 x i32> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}})
+  // CHECK-NEXT: ret <4 x i32>
+  %0 = arm_neon.intr.sdot %a, %b, %c : vector<16xi8>, vector<16xi8> to vector<4xi32>
+  llvm.return %0 : vector<4xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index de4697c47edc1c2ca79007719b252ad69e44de6d..abf997cef4fd40f9aebf87f790d10e12391dd60e 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -1265,6 +1265,29 @@ llvm.mlir.global internal constant @taker_of_address() : !llvm.ptr<func<void ()>
 
 // -----
 
+// CHECK: @forward_use_of_address = linkonce global float* @address_declared_after_use
+llvm.mlir.global linkonce @forward_use_of_address() : !llvm.ptr<f32> {
+  %0 = llvm.mlir.addressof @address_declared_after_use : !llvm.ptr<f32>
+  llvm.return %0 : !llvm.ptr<f32>
+}
+
+llvm.mlir.global linkonce @address_declared_after_use() : f32
+
+// -----
+
+// CHECK: @take_self_address = linkonce global { i32, i32* } {{.*}} { i32, i32* }* @take_self_address
+llvm.mlir.global linkonce @take_self_address() : !llvm.struct<(i32, !llvm.ptr<i32>)> {
+  %z32 = llvm.mlir.constant(0 : i32) : i32
+  %0 = llvm.mlir.undef : !llvm.struct<(i32, !llvm.ptr<i32>)>
+  %1 = llvm.mlir.addressof @take_self_address : !llvm.ptr<!llvm.struct<(i32, !llvm.ptr<i32>)>>
+  %2 = llvm.getelementptr %1[%z32, %z32] : (!llvm.ptr<!llvm.struct<(i32, !llvm.ptr<i32>)>>, i32, i32) -> !llvm.ptr<i32>
+  %3 = llvm.insertvalue %z32, %0[0 : i32] : !llvm.struct<(i32, !llvm.ptr<i32>)>
+  %4 = llvm.insertvalue %2, %3[1 : i32] : !llvm.struct<(i32, !llvm.ptr<i32>)>
+  llvm.return %4 : !llvm.struct<(i32, !llvm.ptr<i32>)>
+}
+
+// -----
+
 // Check that branch weight attributes are exported properly as metadata.
 llvm.func @cond_br_weights(%cond : i1, %arg0 : i32,  %arg1 : i32) -> i32 {
   // CHECK: !prof ![[NODE:[0-9]+]]
diff --git a/mlir/test/Transforms/buffer-deallocation.mlir b/mlir/test/Transforms/buffer-deallocation.mlir
index 25197d14fba779f9dd36bc11088a5d9674d2f475..35f7bbf79c8f5ebb36965222dddadda325dd9fea 100644
--- a/mlir/test/Transforms/buffer-deallocation.mlir
+++ b/mlir/test/Transforms/buffer-deallocation.mlir
@@ -30,13 +30,11 @@ func @condBranch(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 }
 
 // CHECK-NEXT: cond_br
-//      CHECK: %[[ALLOC0:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy
+//      CHECK: %[[ALLOC0:.*]] = memref.clone
 // CHECK-NEXT: br ^bb3(%[[ALLOC0]]
-//      CHECK: %[[ALLOC1:.*]] = memref.alloc()
+//      CHECK: %[[ALLOC1:.*]] = memref.alloc
 // CHECK-NEXT: test.buffer_based
-//      CHECK: %[[ALLOC2:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy
+// CHECK-NEXT: %[[ALLOC2:.*]] = memref.clone %[[ALLOC1]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC1]]
 // CHECK-NEXT: br ^bb3(%[[ALLOC2]]
 //      CHECK: test.copy
@@ -77,16 +75,12 @@ func @condBranchDynamicType(
 }
 
 // CHECK-NEXT: cond_br
-//      CHECK: %[[DIM0:.*]] = memref.dim
-// CHECK-NEXT: %[[ALLOC0:.*]] = memref.alloc(%[[DIM0]])
-// CHECK-NEXT: linalg.copy(%{{.*}}, %[[ALLOC0]])
+//      CHECK: %[[ALLOC0:.*]] = memref.clone
 // CHECK-NEXT: br ^bb3(%[[ALLOC0]]
 //      CHECK: ^bb2(%[[IDX:.*]]:{{.*}})
 // CHECK-NEXT: %[[ALLOC1:.*]] = memref.alloc(%[[IDX]])
 // CHECK-NEXT: test.buffer_based
-//      CHECK: %[[DIM1:.*]] = memref.dim %[[ALLOC1]]
-// CHECK-NEXT: %[[ALLOC2:.*]] = memref.alloc(%[[DIM1]])
-// CHECK-NEXT: linalg.copy(%[[ALLOC1]], %[[ALLOC2]])
+// CHECK-NEXT: %[[ALLOC2:.*]] = memref.clone
 // CHECK-NEXT: memref.dealloc %[[ALLOC1]]
 // CHECK-NEXT: br ^bb3
 // CHECK-NEXT: ^bb3(%[[ALLOC3:.*]]:{{.*}})
@@ -142,12 +136,10 @@ func @condBranchDynamicTypeNested(
   return
 }
 
-// CHECK-NEXT: cond_br
-//      CHECK: ^bb1
-//      CHECK: %[[DIM0:.*]] = memref.dim
-// CHECK-NEXT: %[[ALLOC0:.*]] = memref.alloc(%[[DIM0]])
-// CHECK-NEXT: linalg.copy(%{{.*}}, %[[ALLOC0]])
-// CHECK-NEXT: br ^bb6
+// CHECK-NEXT: cond_br{{.*}}
+// CHECK-NEXT: ^bb1
+// CHECK-NEXT: %[[ALLOC0:.*]] = memref.clone
+// CHECK-NEXT: br ^bb6(%[[ALLOC0]]
 //      CHECK: ^bb2(%[[IDX:.*]]:{{.*}})
 // CHECK-NEXT: %[[ALLOC1:.*]] = memref.alloc(%[[IDX]])
 // CHECK-NEXT: test.buffer_based
@@ -157,9 +149,7 @@ func @condBranchDynamicTypeNested(
 //      CHECK: ^bb4:
 // CHECK-NEXT: br ^bb5(%[[ALLOC1]]{{.*}})
 // CHECK-NEXT: ^bb5(%[[ALLOC2:.*]]:{{.*}})
-//      CHECK: %[[DIM2:.*]] = memref.dim %[[ALLOC2]]
-// CHECK-NEXT: %[[ALLOC3:.*]] = memref.alloc(%[[DIM2]])
-// CHECK-NEXT: linalg.copy(%[[ALLOC2]], %[[ALLOC3]])
+// CHECK-NEXT: %[[ALLOC3:.*]] = memref.clone %[[ALLOC2]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC1]]
 // CHECK-NEXT: br ^bb6(%[[ALLOC3]]{{.*}})
 // CHECK-NEXT: ^bb6(%[[ALLOC4:.*]]:{{.*}})
@@ -208,13 +198,11 @@ func @criticalEdge(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
   return
 }
 
-// CHECK-NEXT: %[[ALLOC0:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy
+// CHECK-NEXT: %[[ALLOC0:.*]] = memref.clone
 // CHECK-NEXT: cond_br
 //      CHECK: %[[ALLOC1:.*]] = memref.alloc()
 // CHECK-NEXT: test.buffer_based
-//      CHECK: %[[ALLOC2:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy
+// CHECK-NEXT: %[[ALLOC2:.*]] = memref.clone %[[ALLOC1]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC1]]
 //      CHECK: test.copy
 // CHECK-NEXT: memref.dealloc
@@ -419,20 +407,17 @@ func @moving_alloc_and_inserting_missing_dealloc(
   return
 }
 
-// CHECK-NEXT: cond_br
-//      CHECK: ^bb1
-//      CHECK: ^bb1
+// CHECK-NEXT: cond_br{{.*}}
+// CHECK-NEXT: ^bb1
 //      CHECK: %[[ALLOC0:.*]] = memref.alloc()
 // CHECK-NEXT: test.buffer_based
-//      CHECK: %[[ALLOC1:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy
+// CHECK-NEXT: %[[ALLOC1:.*]] = memref.clone %[[ALLOC0]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC0]]
 // CHECK-NEXT: br ^bb3(%[[ALLOC1]]
 // CHECK-NEXT: ^bb2
 // CHECK-NEXT: %[[ALLOC2:.*]] = memref.alloc()
 // CHECK-NEXT: test.buffer_based
-//      CHECK: %[[ALLOC3:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy
+// CHECK-NEXT: %[[ALLOC3:.*]] = memref.clone %[[ALLOC2]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC2]]
 // CHECK-NEXT: br ^bb3(%[[ALLOC3]]
 // CHECK-NEXT: ^bb3(%[[ALLOC4:.*]]:{{.*}})
@@ -545,8 +530,7 @@ func @nested_regions_and_cond_branch(
 }
 //      CHECK: (%[[cond:.*]]: {{.*}}, %[[ARG1:.*]]: {{.*}}, %{{.*}}: {{.*}})
 // CHECK-NEXT:   cond_br %[[cond]], ^[[BB1:.*]], ^[[BB2:.*]]
-//      CHECK:   %[[ALLOC0:.*]] = memref.alloc()
-// CHECK-NEXT:   linalg.copy(%[[ARG1]], %[[ALLOC0]])
+//      CHECK:   %[[ALLOC0:.*]] = memref.clone %[[ARG1]]
 //      CHECK: ^[[BB2]]:
 //      CHECK:   %[[ALLOC1:.*]] = memref.alloc()
 // CHECK-NEXT:   test.region_buffer_based in(%[[ARG1]]{{.*}}out(%[[ALLOC1]]
@@ -554,12 +538,11 @@ func @nested_regions_and_cond_branch(
 // CHECK-NEXT:     test.buffer_based in(%[[ARG1]]{{.*}}out(%[[ALLOC2]]
 //      CHECK:     memref.dealloc %[[ALLOC2]]
 // CHECK-NEXT:     %{{.*}} = math.exp
-//      CHECK:   %[[ALLOC3:.*]] = memref.alloc()
-// CHECK-NEXT:   linalg.copy(%[[ALLOC1]], %[[ALLOC3]])
+//      CHECK:   %[[ALLOC3:.*]] = memref.clone %[[ALLOC1]]
 // CHECK-NEXT:   memref.dealloc %[[ALLOC1]]
 //      CHECK:  ^[[BB3:.*]]({{.*}}):
 //      CHECK:  test.copy
-// CHECK-NEXT:  dealloc
+// CHECK-NEXT:  memref.dealloc
 
 // -----
 
@@ -641,12 +624,10 @@ func @nested_region_control_flow_div(
 
 //      CHECK: %[[ALLOC0:.*]] = memref.alloc(%arg0, %arg0)
 // CHECK-NEXT: %[[ALLOC1:.*]] = scf.if
-//      CHECK: %[[ALLOC2:.*]] = memref.alloc
-// CHECK-NEXT: linalg.copy(%[[ALLOC0]], %[[ALLOC2]])
+// CHECK-NEXT: %[[ALLOC2:.*]] = memref.clone %[[ALLOC0]]
 //      CHECK: scf.yield %[[ALLOC2]]
 //      CHECK: %[[ALLOC3:.*]] = memref.alloc(%arg0, %arg1)
-//      CHECK: %[[ALLOC4:.*]] = memref.alloc
-// CHECK-NEXT: linalg.copy(%[[ALLOC3]], %[[ALLOC4]])
+// CHECK-NEXT: %[[ALLOC4:.*]] = memref.clone %[[ALLOC3]]
 //      CHECK: memref.dealloc %[[ALLOC3]]
 //      CHECK: scf.yield %[[ALLOC4]]
 //      CHECK: memref.dealloc %[[ALLOC0]]
@@ -823,20 +804,18 @@ func @nestedRegionsAndCondBranchAlloca(
 //      CHECK: (%[[cond:.*]]: {{.*}}, %[[ARG1:.*]]: {{.*}}, %{{.*}}: {{.*}})
 // CHECK-NEXT:   cond_br %[[cond]], ^[[BB1:.*]], ^[[BB2:.*]]
 //      CHECK: ^[[BB1]]:
-//      CHECK: %[[ALLOC0:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy
+//      CHECK: %[[ALLOC0:.*]] = memref.clone
 //      CHECK: ^[[BB2]]:
 //      CHECK:   %[[ALLOC1:.*]] = memref.alloc()
 // CHECK-NEXT:   test.region_buffer_based in(%[[ARG1]]{{.*}}out(%[[ALLOC1]]
 //      CHECK:     %[[ALLOCA:.*]] = memref.alloca()
 // CHECK-NEXT:     test.buffer_based in(%[[ARG1]]{{.*}}out(%[[ALLOCA]]
 //      CHECK:     %{{.*}} = math.exp
-//      CHECK:  %[[ALLOC2:.*]] = memref.alloc()
-// CHECK-NEXT:  linalg.copy
+//      CHECK:  %[[ALLOC2:.*]] = memref.clone %[[ALLOC1]]
 // CHECK-NEXT:  memref.dealloc %[[ALLOC1]]
 //      CHECK:  ^[[BB3:.*]]({{.*}}):
 //      CHECK:  test.copy
-// CHECK-NEXT:  dealloc
+// CHECK-NEXT:  memref.dealloc
 
 // -----
 
@@ -888,15 +867,13 @@ func @loop_alloc(
 
 //      CHECK: %[[ALLOC0:.*]] = memref.alloc()
 // CHECK-NEXT: memref.dealloc %[[ALLOC0]]
-// CHECK-NEXT: %[[ALLOC1:.*]] = memref.alloc()
-//      CHECK: linalg.copy(%arg3, %[[ALLOC1]])
+// CHECK-NEXT: %[[ALLOC1:.*]] = memref.clone %arg3
 //      CHECK: %[[ALLOC2:.*]] = scf.for {{.*}} iter_args
 // CHECK-SAME: (%[[IALLOC:.*]] = %[[ALLOC1]]
 //      CHECK:    cmpi
 //      CHECK:    memref.dealloc %[[IALLOC]]
 //      CHECK:    %[[ALLOC3:.*]] = memref.alloc()
-//      CHECK:    %[[ALLOC4:.*]] = memref.alloc()
-//      CHECK:    linalg.copy(%[[ALLOC3]], %[[ALLOC4]])
+//      CHECK:    %[[ALLOC4:.*]] = memref.clone %[[ALLOC3]]
 //      CHECK:    memref.dealloc %[[ALLOC3]]
 //      CHECK:    scf.yield %[[ALLOC4]]
 //      CHECK: }
@@ -974,25 +951,21 @@ func @loop_nested_if_alloc(
 }
 
 //      CHECK: %[[ALLOC0:.*]] = memref.alloc()
-//      CHECK: %[[ALLOC1:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%arg3, %[[ALLOC1]])
+// CHECK-NEXT: %[[ALLOC1:.*]] = memref.clone %arg3
 // CHECK-NEXT: %[[ALLOC2:.*]] = scf.for {{.*}} iter_args
 // CHECK-SAME: (%[[IALLOC:.*]] = %[[ALLOC1]]
 //      CHECK: memref.dealloc %[[IALLOC]]
 //      CHECK: %[[ALLOC3:.*]] = scf.if
 
 //      CHECK: %[[ALLOC4:.*]] = memref.alloc()
-// CHECK-NEXT: %[[ALLOC5:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[ALLOC4]], %[[ALLOC5]])
+// CHECK-NEXT: %[[ALLOC5:.*]] = memref.clone %[[ALLOC4]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC4]]
 // CHECK-NEXT: scf.yield %[[ALLOC5]]
 
-//      CHECK: %[[ALLOC6:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[ALLOC0]], %[[ALLOC6]])
+//      CHECK: %[[ALLOC6:.*]] = memref.clone %[[ALLOC0]]
 // CHECK-NEXT: scf.yield %[[ALLOC6]]
 
-//      CHECK: %[[ALLOC7:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[ALLOC3:.*]], %[[ALLOC7]])
+//      CHECK: %[[ALLOC7:.*]] = memref.clone %[[ALLOC3]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC3]]
 // CHECK-NEXT: scf.yield %[[ALLOC7]]
 
@@ -1040,17 +1013,14 @@ func @loop_nested_alloc(
 
 //      CHECK: %[[ALLOC0:.*]] = memref.alloc()
 // CHECK-NEXT: memref.dealloc %[[ALLOC0]]
-// CHECK-NEXT: %[[ALLOC1:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%arg3, %[[ALLOC1]])
+// CHECK-NEXT: %[[ALLOC1:.*]] = memref.clone %arg3
 // CHECK-NEXT: %[[VAL_7:.*]] = scf.for {{.*}} iter_args
 // CHECK-SAME: (%[[IALLOC0:.*]] = %[[ALLOC1]])
-//      CHECK: %[[ALLOC2:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[IALLOC0]], %[[ALLOC2]])
+// CHECK-NEXT: %[[ALLOC2:.*]] = memref.clone %[[IALLOC0]]
 // CHECK-NEXT: memref.dealloc %[[IALLOC0]]
 // CHECK-NEXT: %[[ALLOC3:.*]] = scf.for {{.*}} iter_args
 // CHECK-SAME: (%[[IALLOC1:.*]] = %[[ALLOC2]])
-//      CHECK: %[[ALLOC5:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[IALLOC1]], %[[ALLOC5]])
+// CHECK-NEXT: %[[ALLOC5:.*]] = memref.clone %[[IALLOC1]]
 // CHECK-NEXT: memref.dealloc %[[IALLOC1]]
 
 //      CHECK: %[[ALLOC6:.*]] = scf.for {{.*}} iter_args
@@ -1060,28 +1030,23 @@ func @loop_nested_alloc(
 //      CHECK: %[[ALLOC9:.*]] = scf.if
 
 //      CHECK: %[[ALLOC11:.*]] = memref.alloc()
-// CHECK-NEXT: %[[ALLOC12:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[ALLOC11]], %[[ALLOC12]])
+// CHECK-NEXT: %[[ALLOC12:.*]] = memref.clone %[[ALLOC11]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC11]]
 // CHECK-NEXT: scf.yield %[[ALLOC12]]
 
-//      CHECK: %[[ALLOC13:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[IALLOC2]], %[[ALLOC13]])
+//      CHECK: %[[ALLOC13:.*]] = memref.clone %[[IALLOC2]]
 // CHECK-NEXT: scf.yield %[[ALLOC13]]
 
 //      CHECK: memref.dealloc %[[IALLOC2]]
-// CHECK-NEXT: %[[ALLOC10:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[ALLOC9]], %[[ALLOC10]])
+// CHECK-NEXT: %[[ALLOC10:.*]] = memref.clone %[[ALLOC9]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC9]]
 // CHECK-NEXT: scf.yield %[[ALLOC10]]
 
-//      CHECK: %[[ALLOC7:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[ALLOC6]], %[[ALLOC7]])
+//      CHECK: %[[ALLOC7:.*]] = memref.clone %[[ALLOC6]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC6]]
 // CHECK-NEXT: scf.yield %[[ALLOC7]]
 
-//      CHECK: %[[ALLOC4:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[ALLOC3]], %[[ALLOC4]])
+//      CHECK: %[[ALLOC4:.*]] = memref.clone %[[ALLOC3]]
 // CHECK-NEXT: memref.dealloc %[[ALLOC3]]
 // CHECK-NEXT: scf.yield %[[ALLOC4]]
 
@@ -1183,8 +1148,7 @@ func @assumingOp(
 // CHECK-NEXT:    shape.assuming_yield %[[ARG1]]
 //      CHECK: %[[ASSUMING_RESULT:.*]] = shape.assuming %[[ARG0]]
 // CHECK-NEXT:    %[[TMP_ALLOC:.*]] = memref.alloc()
-// CHECK-NEXT:    %[[RETURNING_ALLOC:.*]] = memref.alloc()
-// CHECK-NEXT:    linalg.copy(%[[TMP_ALLOC]], %[[RETURNING_ALLOC]])
+// CHECK-NEXT:    %[[RETURNING_ALLOC:.*]] = memref.clone %[[TMP_ALLOC]]
 // CHECK-NEXT:    memref.dealloc %[[TMP_ALLOC]]
 // CHECK-NEXT:    shape.assuming_yield %[[RETURNING_ALLOC]]
 //      CHECK: test.copy(%[[ASSUMING_RESULT:.*]], %[[ARG2]])
diff --git a/mlir/test/Transforms/canonicalize-dce.mlir b/mlir/test/Transforms/canonicalize-dce.mlir
index 4a351dca426e9a2d1b72e43ef425c0f96c9d934e..e96bd65d389a995ec7718829a862cc4ab0204b90 100644
--- a/mlir/test/Transforms/canonicalize-dce.mlir
+++ b/mlir/test/Transforms/canonicalize-dce.mlir
@@ -156,3 +156,20 @@ func @f(
   "foo.print"(%t4) : (tensor<4xf32>) -> ()
   return
 }
+
+// -----
+
+// Test case: Test values with use-def cycles are deleted properly.
+
+// CHECK:      func @f()
+// CHECK-NEXT:   test.graph_region
+// CHECK-NEXT:     "test.terminator"() : () -> ()
+
+func @f() {
+  test.graph_region {
+    %0 = "math.exp"(%1) : (f32) -> f32
+    %1 = "math.exp"(%0) : (f32) -> f32
+    "test.terminator"() : ()->()
+  }
+  return
+}
diff --git a/mlir/test/Transforms/canonicalize.mlir b/mlir/test/Transforms/canonicalize.mlir
index 5009d0f4207d4ac89fa486c5d87cf45a11afb9e0..e54135f21b4bd1d887deb508eaa06027df051c55 100644
--- a/mlir/test/Transforms/canonicalize.mlir
+++ b/mlir/test/Transforms/canonicalize.mlir
@@ -572,7 +572,8 @@ func @indirect_call_folding() {
 //
 // CHECK-LABEL: @lowered_affine_mod
 func @lowered_affine_mod() -> (index, index) {
-// CHECK-NEXT: {{.*}} = constant 41 : index
+// CHECK-DAG: {{.*}} = constant 1 : index
+// CHECK-DAG: {{.*}} = constant 41 : index
   %c-43 = constant -43 : index
   %c42 = constant 42 : index
   %0 = remi_signed %c-43, %c42 : index
@@ -580,7 +581,6 @@ func @lowered_affine_mod() -> (index, index) {
   %1 = cmpi slt, %0, %c0 : index
   %2 = addi %0, %c42 : index
   %3 = select %1, %2, %0 : index
-// CHECK-NEXT: {{.*}} = constant 1 : index
   %c43 = constant 43 : index
   %c42_0 = constant 42 : index
   %4 = remi_signed %c43, %c42_0 : index
@@ -598,7 +598,8 @@ func @lowered_affine_mod() -> (index, index) {
 //
 // CHECK-LABEL: func @lowered_affine_floordiv
 func @lowered_affine_floordiv() -> (index, index) {
-// CHECK-NEXT: %c-2 = constant -2 : index
+// CHECK-DAG: %c1 = constant 1 : index
+// CHECK-DAG: %c-2 = constant -2 : index
   %c-43 = constant -43 : index
   %c42 = constant 42 : index
   %c0 = constant 0 : index
@@ -609,7 +610,6 @@ func @lowered_affine_floordiv() -> (index, index) {
   %3 = divi_signed %2, %c42 : index
   %4 = subi %c-1, %3 : index
   %5 = select %0, %4, %3 : index
-// CHECK-NEXT: %c1 = constant 1 : index
   %c43 = constant 43 : index
   %c42_0 = constant 42 : index
   %c0_1 = constant 0 : index
@@ -724,17 +724,17 @@ func @view(%arg0 : index) -> (f32, f32, f32, f32) {
 // CHECK-LABEL: func @subview
 // CHECK-SAME: %[[ARG0:.*]]: index, %[[ARG1:.*]]: index
 func @subview(%arg0 : index, %arg1 : index) -> (index, index) {
-  // CHECK: %[[C0:.*]] = constant 0 : index
+  // Folded but reappears after subview folding into dim.
+  // CHECK-DAG: %[[C0:.*]] = constant 0 : index
+  // CHECK-DAG: %[[C7:.*]] = constant 7 : index
+  // CHECK-DAG: %[[C11:.*]] = constant 11 : index
   %c0 = constant 0 : index
   // CHECK-NOT: constant 1 : index
   %c1 = constant 1 : index
   // CHECK-NOT: constant 2 : index
   %c2 = constant 2 : index
   // Folded but reappears after subview folding into dim.
-  // CHECK: %[[C7:.*]] = constant 7 : index
   %c7 = constant 7 : index
-  // Folded but reappears after subview folding into dim.
-  // CHECK: %[[C11:.*]] = constant 11 : index
   %c11 = constant 11 : index
   // CHECK-NOT: constant 15 : index
   %c15 = constant 15 : index
@@ -895,8 +895,8 @@ func @index_cast_fold() -> (i16, index) {
   %1 = index_cast %c4 : index to i16
   %c4_i16 = constant 4 : i16
   %2 = index_cast %c4_i16 : i16 to index
-  // CHECK: %[[C4_I16:.*]] = constant 4 : i16
-  // CHECK: %[[C4:.*]] = constant 4 : index
+  // CHECK-DAG: %[[C4:.*]] = constant 4 : index
+  // CHECK-DAG: %[[C4_I16:.*]] = constant 4 : i16
   // CHECK: return %[[C4_I16]], %[[C4]] : i16, index
   return %1, %2 : i16, index
 }
@@ -1059,3 +1059,148 @@ func @subtensor(%t: tensor<8x16x4xf32>, %arg0 : index, %arg1 : index)
   return %2 : tensor<?x?x?xf32>
 }
 
+// -----
+
+// CHECK-LABEL: func @fold_trunci
+// CHECK-SAME:    (%[[ARG0:[0-9a-z]*]]: i1)
+func @fold_trunci(%arg0: i1) -> i1 attributes {} {
+  // CHECK-NEXT: return %[[ARG0]] : i1
+  %0 = zexti %arg0 : i1 to i8
+  %1 = trunci %0 : i8 to i1
+  return %1 : i1
+}
+
+// -----
+
+// CHECK-LABEL: func @fold_trunci_vector
+// CHECK-SAME:    (%[[ARG0:[0-9a-z]*]]: vector<4xi1>)
+func @fold_trunci_vector(%arg0: vector<4xi1>) -> vector<4xi1> attributes {} {
+  // CHECK-NEXT: return %[[ARG0]] : vector<4xi1>
+  %0 = zexti %arg0 : vector<4xi1> to vector<4xi8>
+  %1 = trunci %0 : vector<4xi8> to vector<4xi1>
+  return %1 : vector<4xi1>
+}
+
+// -----
+
+// TODO Canonicalize this into:
+//   zexti %arg0 : i1 to i2
+
+// CHECK-LABEL: func @do_not_fold_trunci
+// CHECK-SAME:    (%[[ARG0:[0-9a-z]*]]: i1)
+func @do_not_fold_trunci(%arg0: i1) -> i2 attributes {} {
+  // CHECK-NEXT: zexti %[[ARG0]] : i1 to i8
+  // CHECK-NEXT: %[[RES:[0-9a-z]*]] = trunci %{{.*}} : i8 to i2
+  // CHECK-NEXT: return %[[RES]] : i2
+  %0 = zexti %arg0 : i1 to i8
+  %1 = trunci %0 : i8 to i2
+  return %1 : i2
+}
+
+// -----
+
+// CHECK-LABEL: func @do_not_fold_trunci_vector
+// CHECK-SAME:    (%[[ARG0:[0-9a-z]*]]: vector<4xi1>)
+func @do_not_fold_trunci_vector(%arg0: vector<4xi1>) -> vector<4xi2> attributes {} {
+  // CHECK-NEXT: zexti %[[ARG0]] : vector<4xi1> to vector<4xi8>
+  // CHECK-NEXT: %[[RES:[0-9a-z]*]] = trunci %{{.*}} : vector<4xi8> to vector<4xi2>
+  // CHECK-NEXT: return %[[RES]] : vector<4xi2>
+  %0 = zexti %arg0 : vector<4xi1> to vector<4xi8>
+  %1 = trunci %0 : vector<4xi8> to vector<4xi2>
+  return %1 : vector<4xi2>
+}
+
+// -----
+
+// CHECK-LABEL: func @fold_trunci_sexti
+// CHECK-SAME:    (%[[ARG0:[0-9a-z]*]]: i1)
+func @fold_trunci_sexti(%arg0: i1) -> i1 attributes {} {
+  // CHECK-NEXT: return %[[ARG0]] : i1
+  %0 = sexti %arg0 : i1 to i8
+  %1 = trunci %0 : i8 to i1
+  return %1 : i1
+}
+
+// CHECK-LABEL: func @simple_clone_elimination
+func @simple_clone_elimination() -> memref<5xf32> {
+  %ret = memref.alloc() : memref<5xf32>
+  %temp = memref.clone %ret : memref<5xf32> to memref<5xf32>
+  memref.dealloc %temp : memref<5xf32>
+  return %ret : memref<5xf32>
+}
+// CHECK-NEXT: %[[ret:.*]] = memref.alloc()
+// CHECK-NOT: %[[temp:.*]] = memref.clone
+// CHECK-NOT: memref.dealloc %[[temp]]
+// CHECK: return %[[ret]]
+
+// -----
+
+// CHECK-LABEL: func @clone_loop_alloc
+func @clone_loop_alloc(%arg0: index, %arg1: index, %arg2: index, %arg3: memref<2xf32>, %arg4: memref<2xf32>) {
+  %0 = memref.alloc() : memref<2xf32>
+  memref.dealloc %0 : memref<2xf32>
+  %1 = memref.clone %arg3 : memref<2xf32> to memref<2xf32>
+  %2 = scf.for %arg5 = %arg0 to %arg1 step %arg2 iter_args(%arg6 = %1) -> (memref<2xf32>) {
+    %3 = cmpi eq, %arg5, %arg1 : index
+    memref.dealloc %arg6 : memref<2xf32>
+    %4 = memref.alloc() : memref<2xf32>
+    %5 = memref.clone %4 : memref<2xf32> to memref<2xf32>
+    memref.dealloc %4 : memref<2xf32>
+    %6 = memref.clone %5 : memref<2xf32> to memref<2xf32>
+    memref.dealloc %5 : memref<2xf32>
+    scf.yield %6 : memref<2xf32>
+  }
+  linalg.copy(%2, %arg4) : memref<2xf32>, memref<2xf32>
+  memref.dealloc %2 : memref<2xf32>
+  return
+}
+
+// CHECK-NEXT: %[[ALLOC0:.*]] = memref.clone
+// CHECK-NEXT: %[[ALLOC1:.*]] = scf.for
+// CHECK-NEXT: memref.dealloc
+// CHECK-NEXT: %[[ALLOC2:.*]] = memref.alloc
+// CHECK-NEXT: scf.yield %[[ALLOC2]]
+// CHECK: linalg.copy(%[[ALLOC1]]
+// CHECK-NEXT: memref.dealloc %[[ALLOC1]]
+
+// -----
+
+// CHECK-LABEL: func @clone_nested_region
+func @clone_nested_region(%arg0: index, %arg1: index) -> memref<?x?xf32> {
+  %0 = cmpi eq, %arg0, %arg1 : index
+  %1 = memref.alloc(%arg0, %arg0) : memref<?x?xf32>
+  %2 = scf.if %0 -> (memref<?x?xf32>) {
+    %3 = scf.if %0 -> (memref<?x?xf32>) {
+      %9 = memref.clone %1 : memref<?x?xf32> to memref<?x?xf32>
+      scf.yield %9 : memref<?x?xf32>
+    } else {
+      %7 = memref.alloc(%arg0, %arg1) : memref<?x?xf32>
+      %10 = memref.clone %7 : memref<?x?xf32> to memref<?x?xf32>
+      memref.dealloc %7 : memref<?x?xf32>
+      scf.yield %10 : memref<?x?xf32>
+    }
+    %6 = memref.clone %3 : memref<?x?xf32> to memref<?x?xf32>
+    memref.dealloc %3 : memref<?x?xf32>
+    scf.yield %6 : memref<?x?xf32>
+  } else {
+    %3 = memref.alloc(%arg1, %arg1) : memref<?x?xf32>
+    %6 = memref.clone %3 : memref<?x?xf32> to memref<?x?xf32>
+    memref.dealloc %3 : memref<?x?xf32>
+    scf.yield %6 : memref<?x?xf32>
+  }
+  memref.dealloc %1 : memref<?x?xf32>
+  return %2 : memref<?x?xf32>
+}
+
+//      CHECK: %[[ALLOC1:.*]] = memref.alloc
+// CHECK-NEXT: %[[ALLOC2:.*]] = scf.if
+// CHECK-NEXT: %[[ALLOC3_1:.*]] = scf.if
+// CHECK-NEXT: %[[ALLOC4_1:.*]] = memref.clone %[[ALLOC1]]
+// CHECK-NEXT: scf.yield %[[ALLOC4_1]]
+//      CHECK: %[[ALLOC4_2:.*]] = memref.alloc
+// CHECK-NEXT: scf.yield %[[ALLOC4_2]]
+//      CHECK: scf.yield %[[ALLOC3_1]]
+//      CHECK: %[[ALLOC3_2:.*]] = memref.alloc
+// CHECK-NEXT: scf.yield %[[ALLOC3_2]]
+//      CHECK: memref.dealloc %[[ALLOC1]]
+// CHECK-NEXT: return %[[ALLOC2]]
diff --git a/mlir/test/Transforms/copy-removal.mlir b/mlir/test/Transforms/copy-removal.mlir
deleted file mode 100644
index a91c5c2b952871b7005642a3df0c6dfe4d32d6b4..0000000000000000000000000000000000000000
--- a/mlir/test/Transforms/copy-removal.mlir
+++ /dev/null
@@ -1,361 +0,0 @@
-// RUN: mlir-opt -copy-removal -split-input-file %s | FileCheck %s
-
-// All linalg copies except the linalg.copy(%1, %9) must be removed since the
-// defining operation of %1 and its DeallocOp have been defined in another block.
-
-// CHECK-LABEL: func @nested_region_control_flow_div_nested
-func @nested_region_control_flow_div_nested(%arg0: index, %arg1: index) -> memref<?x?xf32> {
-  %0 = cmpi eq, %arg0, %arg1 : index
-  %1 = memref.alloc(%arg0, %arg0) : memref<?x?xf32>
-  // CHECK: %{{.*}} = scf.if
-  %2 = scf.if %0 -> (memref<?x?xf32>) {
-    // CHECK: %[[PERCENT3:.*]] = scf.if
-    %3 = scf.if %0 -> (memref<?x?xf32>) {
-      %c0_0 = constant 0 : index
-      %7 = memref.dim %1, %c0_0 : memref<?x?xf32>
-      %c1_1 = constant 1 : index
-      %8 = memref.dim %1, %c1_1 : memref<?x?xf32>
-      %9 = memref.alloc(%7, %8) : memref<?x?xf32>
-      // CHECK: linalg.copy({{.*}}, %[[PERCENT9:.*]])
-      linalg.copy(%1, %9) : memref<?x?xf32>, memref<?x?xf32>
-      // CHECK: scf.yield %[[PERCENT9]]
-      scf.yield %9 : memref<?x?xf32>
-    } else {
-      // CHECK: %[[PERCENT7:.*]] = memref.alloc
-      %7 = memref.alloc(%arg0, %arg1) : memref<?x?xf32>
-      %c0_0 = constant 0 : index
-      %8 = memref.dim %7, %c0_0 : memref<?x?xf32>
-      %c1_1 = constant 1 : index
-      %9 = memref.dim %7, %c1_1 : memref<?x?xf32>
-      // CHECK-NOT: %{{.*}} = memref.alloc
-      // CHECK-NOT: linalg.copy(%[[PERCENT7]], %{{.*}})
-      // CHECK-NOT: memref.dealloc %[[PERCENT7]]
-      %10 = memref.alloc(%8, %9) : memref<?x?xf32>
-      linalg.copy(%7, %10) : memref<?x?xf32>, memref<?x?xf32>
-      memref.dealloc %7 : memref<?x?xf32>
-      // CHECK: scf.yield %[[PERCENT7]]
-      scf.yield %10 : memref<?x?xf32>
-    }
-    %c0 = constant 0 : index
-    %4 = memref.dim %3, %c0 : memref<?x?xf32>
-    %c1 = constant 1 : index
-    %5 = memref.dim %3, %c1 : memref<?x?xf32>
-    // CHECK-NOT: %{{.*}} = memref.alloc
-    // CHECK-NOT: linalg.copy(%[[PERCENT3]], %{{.*}})
-    // CHECK-NOT: memref.dealloc %[[PERCENT3]]
-    %6 = memref.alloc(%4, %5) : memref<?x?xf32>
-    linalg.copy(%3, %6) : memref<?x?xf32>, memref<?x?xf32>
-    memref.dealloc %3 : memref<?x?xf32>
-    // CHECK: scf.yield %[[PERCENT3]]
-    scf.yield %6 : memref<?x?xf32>
-  } else {
-    // CHECK: %[[PERCENT3:.*]] = memref.alloc
-    %3 = memref.alloc(%arg1, %arg1) : memref<?x?xf32>
-    %c0 = constant 0 : index
-    %4 = memref.dim %3, %c0 : memref<?x?xf32>
-    %c1 = constant 1 : index
-    %5 = memref.dim %3, %c1 : memref<?x?xf32>
-    // CHECK-NOT: %{{.*}} = memref.alloc
-    // CHECK-NOT: linalg.copy(%[[PERCENT3]], %{{.*}})
-    // CHECK-NOT: memref.dealloc %[[PERCENT3]]
-    %6 = memref.alloc(%4, %5) : memref<?x?xf32>
-    linalg.copy(%3, %6) : memref<?x?xf32>, memref<?x?xf32>
-    memref.dealloc %3 : memref<?x?xf32>
-    // CHECK: scf.yield %[[PERCENT3]]
-    scf.yield %6 : memref<?x?xf32>
-  }
-  memref.dealloc %1 : memref<?x?xf32>
-  return %2 : memref<?x?xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @simple_test
-func @simple_test() -> memref<5xf32> {
-  %temp = memref.alloc() : memref<5xf32>
-  %ret = memref.alloc() : memref<5xf32>
-  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
-  memref.dealloc %ret : memref<5xf32>
-  return %temp : memref<5xf32>
-}
-// CHECK-SAME: () -> memref<5xf32>
-// CHECK-NEXT: %[[ret:.*]] = memref.alloc()
-// CHECK-NOT: linalg.copy(%[[ret]], %{{.*}})
-// CHECK-NOT: memref.dealloc %[[ret]]
-// CHECK: return %[[ret]]
-
-// -----
-
-// It is legal to remove the copy operation that %ret has a usage before the copy
-// operation. The allocation of %temp and the deallocation of %ret should be also
-// removed.
-
-// CHECK-LABEL: func @test_with_ret_usage_before_copy
-func @test_with_ret_usage_before_copy() -> memref<5xf32> {
-  %ret = memref.alloc() : memref<5xf32>
-  %temp = memref.alloc() : memref<5xf32>
-  %c0 = constant 0 : index
-  %dimension = memref.dim %ret, %c0 : memref<5xf32>
-  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
-  memref.dealloc %ret : memref<5xf32>
-  return %temp : memref<5xf32>
-}
-// CHECK-NEXT: %[[ret:.*]] = memref.alloc()
-// CHECK-NOT: %{{.*}} = memref.alloc
-// CHECK-NEXT: %{{.*}} = constant
-// CHECK-NEXT: %[[DIM:.*]] = memref.dim %[[ret]]
-// CHECK-NOT: linalg.copy(%[[ret]], %{{.*}})
-// CHECK-NOT: memref.dealloc %[[ret]]
-// CHECK: return %[[ret]]
-
-// -----
-
-// It is illegal to remove a copy operation that %ret has a usage after copy
-// operation.
-
-// CHECK-LABEL: func @test_with_ret_usage_after_copy
-func @test_with_ret_usage_after_copy() -> memref<5xf32> {
-  %ret = memref.alloc() : memref<5xf32>
-  %temp = memref.alloc() : memref<5xf32>
-  // CHECK: linalg.copy
-  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
-  %c0 = constant 0 : index
-  %dimension = memref.dim %ret, %c0 : memref<5xf32>
-  memref.dealloc %ret : memref<5xf32>
-  return %temp : memref<5xf32>
-}
-
-// -----
-
-// It is illegal to remove a copy operation that %temp has a usage before copy
-// operation.
-
-// CHECK-LABEL: func @test_with_temp_usage_before_copy
-func @test_with_temp_usage_before_copy() -> memref<5xf32> {
-  %ret = memref.alloc() : memref<5xf32>
-  %temp = memref.alloc() : memref<5xf32>
-  %c0 = constant 0 : index
-  %dimension = memref.dim %temp, %c0 : memref<5xf32>
-  // CHECK: linalg.copy
-  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
-  memref.dealloc %ret : memref<5xf32>
-  return %temp : memref<5xf32>
-}
-
-// -----
-
-// It is legal to remove the copy operation that %temp has a usage after the copy
-// operation. The allocation of %temp and the deallocation of %ret could be also
-// removed.
-
-// However the following pattern is not handled by copy removal.
-//   %from = memref.alloc()
-//   %to = memref.alloc()
-//   copy(%from, %to)
-//   read_from(%from) + write_to(%something_else)
-//   memref.dealloc(%from)
-//   return %to
-// In particular, linalg.generic is a memoryEffectOp between copy and dealloc.
-// Since no alias analysis is performed and no distinction is made between reads
-// and writes, the linalg.generic with effects blocks copy removal.
-
-#map0 = affine_map<(d0) -> (d0)>
-
-// CHECK-LABEL: func @test_with_temp_usage_after_copy
-func @test_with_temp_usage_after_copy() -> memref<5xf32> {
-  %ret = memref.alloc() : memref<5xf32>
-  %res = memref.alloc() : memref<5xf32>
-  %temp = memref.alloc() : memref<5xf32>
-  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
-  linalg.generic {
-    indexing_maps = [#map0, #map0],
-    iterator_types = ["parallel"]}
-    ins(%temp : memref<5xf32>)
-   outs(%res : memref<5xf32>) {
-  ^bb0(%gen1_arg0: f32, %gen1_arg1: f32):
-    %tmp1 = math.exp %gen1_arg0 : f32
-    linalg.yield %tmp1 : f32
-  }
-  memref.dealloc %ret : memref<5xf32>
-  return %temp : memref<5xf32>
-}
-// CHECK-NEXT: %[[ret:.*]] = memref.alloc()
-// CHECK-NEXT: %[[res:.*]] = memref.alloc()
-// CHECK-NEXT: %[[temp:.*]] = memref.alloc()
-// CHECK-NEXT: linalg.copy(%[[ret]], %[[temp]])
-// CHECK-NEXT: linalg.generic
-//      CHECK: memref.dealloc %[[ret]]
-//      CHECK: return %[[temp]]
-
-// -----
-
-// CHECK-LABEL: func @make_allocation
-func @make_allocation() -> memref<5xf32> {
-  %mem = memref.alloc() : memref<5xf32>
-  return %mem : memref<5xf32>
-}
-
-// CHECK-LABEL: func @test_with_function_call
-func @test_with_function_call() -> memref<5xf32> {
-  // CHECK-NEXT: %[[ret:.*]] = call @make_allocation() : () -> memref<5xf32>
-  %ret = call @make_allocation() : () -> (memref<5xf32>)
-  // CHECK-NOT: %{{.*}} = memref.alloc
-  // CHECK-NOT: linalg.copy(%[[ret]], %{{.*}})
-  // CHECK-NOT: memref.dealloc %[[ret]]
-  %temp = memref.alloc() : memref<5xf32>
-  linalg.copy(%ret, %temp) : memref<5xf32>, memref<5xf32>
-  memref.dealloc %ret : memref<5xf32>
-  // CHECK: return %[[ret]]
-  return %temp : memref<5xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @multiple_deallocs_in_different_blocks
-func @multiple_deallocs_in_different_blocks(%cond : i1) -> memref<5xf32> {
-  // CHECK-NEXT: %[[PERCENT0:.*]] = memref.alloc()
-  %0 = memref.alloc() : memref<5xf32>
-  cond_br %cond, ^bb1, ^bb2
-^bb1:
-  memref.dealloc %0 : memref<5xf32>
-  // CHECK: br ^[[BB3:.*]](%[[PERCENT0]]
-  br ^bb3(%0 : memref<5xf32>)
-^bb2:
-  // CHECK-NOT: %{{.*}} = memref.alloc
-  // CHECK-NOT: linalg.copy(%[[PERCENT0]], %{{.*}})
-  // CHECK-NOT: memref.dealloc %[[PERCENT0]]
-  %temp = memref.alloc() : memref<5xf32>
-  linalg.copy(%0, %temp) : memref<5xf32>, memref<5xf32>
-  memref.dealloc %0 : memref<5xf32>
-  // CHECK: br ^[[BB3]](%[[PERCENT0]]
-  br ^bb3(%temp : memref<5xf32>)
-^bb3(%res : memref<5xf32>):
-  return %res : memref<5xf32>
-}
-
-// -----
-
-#map0 = affine_map<(d0) -> (d0)>
-
-// CHECK-LABEL: func @test_ReuseCopyTargetAsSource
-func @test_ReuseCopyTargetAsSource(%arg0: memref<2xf32>, %result: memref<2xf32>){
-  // CHECK-SAME: (%[[ARG0:.*]]: memref<2xf32>, %[[RES:.*]]: memref<2xf32>)
-  // CHECK-NOT: %{{.*}} = memref.alloc
-  %temp = memref.alloc() : memref<2xf32>
-  // CHECK-NEXT: linalg.generic
-  // CHECK-SAME: ins(%[[ARG0]]{{.*}}outs(%[[RES]]
-  // CHECK-NOT: linalg.copy(%{{.*}}, %[[RES]])
-  // CHECK-NOT: memref.dealloc %{{.*}}
-  linalg.generic {
-    indexing_maps = [#map0, #map0],
-    iterator_types = ["parallel"]}
-    ins(%arg0 : memref<2xf32>)
-   outs(%temp : memref<2xf32>) {
-  ^bb0(%gen2_arg0: f32, %gen2_arg1: f32):
-    %tmp2 = math.exp %gen2_arg0 : f32
-    linalg.yield %tmp2 : f32
-  }
-  linalg.copy(%temp, %result) : memref<2xf32>, memref<2xf32>
-  memref.dealloc %temp : memref<2xf32>
-  // CHECK: return
-  return
-}
-
-// -----
-
-// Copy operation must not be removed since an operation writes to %to value
-// before copy.
-
-#map0 = affine_map<(d0) -> (d0)>
-
-// CHECK-LABEL: func @test_ReuseCopyTargetAsSource
-func @test_ReuseCopyTargetAsSource(%arg0: memref<2xf32>){
-  %to = memref.alloc() : memref<2xf32>
-  %temp = memref.alloc() : memref<2xf32>
-  linalg.generic {
-    indexing_maps = [#map0, #map0],
-    iterator_types = ["parallel"]}
-    ins(%arg0 : memref<2xf32>)
-   outs(%temp : memref<2xf32>) {
-  ^bb0(%gen1_arg0: f32, %gen1_arg1: f32):
-    %tmp1 = math.exp %gen1_arg0 : f32
-    linalg.yield %tmp1 : f32
-  }
-  linalg.generic {
-    indexing_maps = [#map0, #map0],
-    iterator_types = ["parallel"]}
-    ins(%arg0 : memref<2xf32>)
-   outs(%to : memref<2xf32>) {
-  ^bb0(%gen2_arg0: f32, %gen2_arg1: f32):
-    %tmp2 = math.exp %gen2_arg0 : f32
-    linalg.yield %tmp2 : f32
-  }
-  // CHECK: linalg.copy
-  linalg.copy(%temp, %to) : memref<2xf32>, memref<2xf32>
-  memref.dealloc %temp : memref<2xf32>
-  return
-}
-
-// -----
-
-// The only redundant copy is linalg.copy(%4, %5)
-
-// CHECK-LABEL: func @loop_alloc
-func @loop_alloc(%arg0: index, %arg1: index, %arg2: index, %arg3: memref<2xf32>, %arg4: memref<2xf32>) {
-  // CHECK: %{{.*}} = memref.alloc()
-  %0 = memref.alloc() : memref<2xf32>
-  memref.dealloc %0 : memref<2xf32>
-  // CHECK: %{{.*}} = memref.alloc()
-  %1 = memref.alloc() : memref<2xf32>
-  // CHECK: linalg.copy
-  linalg.copy(%arg3, %1) : memref<2xf32>, memref<2xf32>
-  %2 = scf.for %arg5 = %arg0 to %arg1 step %arg2 iter_args(%arg6 = %1) -> (memref<2xf32>) {
-    %3 = cmpi eq, %arg5, %arg1 : index
-    // CHECK: memref.dealloc
-    memref.dealloc %arg6 : memref<2xf32>
-    // CHECK: %[[PERCENT4:.*]] = memref.alloc()
-    %4 = memref.alloc() : memref<2xf32>
-    // CHECK-NOT: memref.alloc
-    // CHECK-NOT: linalg.copy
-    // CHECK-NOT: memref.dealloc
-    %5 = memref.alloc() : memref<2xf32>
-    linalg.copy(%4, %5) : memref<2xf32>, memref<2xf32>
-    memref.dealloc %4 : memref<2xf32>
-    // CHECK: %[[PERCENT6:.*]] = memref.alloc()
-    %6 = memref.alloc() : memref<2xf32>
-    // CHECK: linalg.copy(%[[PERCENT4]], %[[PERCENT6]])
-    linalg.copy(%5, %6) : memref<2xf32>, memref<2xf32>
-    scf.yield %6 : memref<2xf32>
-  }
-  // CHECK: linalg.copy
-  linalg.copy(%2, %arg4) : memref<2xf32>, memref<2xf32>
-  memref.dealloc %2 : memref<2xf32>
-  return
-}
-
-// -----
-
-// The linalg.copy operation can be removed in addition to alloc and dealloc
-// operations. All uses of %0 is then replaced with %arg2.
-
-// CHECK-LABEL: func @check_with_affine_dialect
-func @check_with_affine_dialect(%arg0: memref<4xf32>, %arg1: memref<4xf32>, %arg2: memref<4xf32>) {
-  // CHECK-SAME: (%[[ARG0:.*]]: memref<4xf32>, %[[ARG1:.*]]: memref<4xf32>, %[[RES:.*]]: memref<4xf32>)
-  // CHECK-NOT: memref.alloc
-  %0 = memref.alloc() : memref<4xf32>
-  affine.for %arg3 = 0 to 4 {
-    %5 = affine.load %arg0[%arg3] : memref<4xf32>
-    %6 = affine.load %arg1[%arg3] : memref<4xf32>
-    %7 = cmpf ogt, %5, %6 : f32
-    // CHECK: %[[SELECT_RES:.*]] = select
-    %8 = select %7, %5, %6 : f32
-    // CHECK-NEXT: affine.store %[[SELECT_RES]], %[[RES]]
-    affine.store %8, %0[%arg3] : memref<4xf32>
-  }
-  // CHECK-NOT: linalg.copy
-  // CHECK-NOT: dealloc
-  linalg.copy(%0, %arg2) : memref<4xf32>, memref<4xf32>
-  memref.dealloc %0 : memref<4xf32>
-  //CHECK: return
-  return
-}
diff --git a/mlir/test/Transforms/parallel-loop-collapsing.mlir b/mlir/test/Transforms/parallel-loop-collapsing.mlir
index 2bd78be6b63a5630271bcd5800de3f4b134aec29..a6a9aa8f61bd48d056c7ae65e0290b2df324e27c 100644
--- a/mlir/test/Transforms/parallel-loop-collapsing.mlir
+++ b/mlir/test/Transforms/parallel-loop-collapsing.mlir
@@ -28,15 +28,15 @@ func @parallel_many_dims() {
   return
 }
 
-// CHECK: [[C3:%.*]] = constant 3 : index
-// CHECK: [[C6:%.*]] = constant 6 : index
-// CHECK: [[C9:%.*]] = constant 9 : index
-// CHECK: [[C10:%.*]] = constant 10 : index
-// CHECK: [[C4:%.*]] = constant 4 : index
-// CHECK: [[C12:%.*]] = constant 12 : index
-// CHECK: [[C0:%.*]] = constant 0 : index
-// CHECK: [[C1:%.*]] = constant 1 : index
-// CHECK: [[C2:%.*]] = constant 2 : index
+// CHECK-DAG: [[C12:%.*]] = constant 12 : index
+// CHECK-DAG: [[C10:%.*]] = constant 10 : index
+// CHECK-DAG: [[C9:%.*]] = constant 9 : index
+// CHECK-DAG: [[C6:%.*]] = constant 6 : index
+// CHECK-DAG: [[C4:%.*]] = constant 4 : index
+// CHECK-DAG: [[C3:%.*]] = constant 3 : index
+// CHECK-DAG: [[C2:%.*]] = constant 2 : index
+// CHECK-DAG: [[C1:%.*]] = constant 1 : index
+// CHECK-DAG: [[C0:%.*]] = constant 0 : index
 // CHECK: scf.parallel ([[NEW_I0:%.*]]) = ([[C0]]) to ([[C4]]) step ([[C1]]) {
 // CHECK:   [[V0:%.*]] = remi_signed [[NEW_I0]], [[C2]] : index
 // CHECK:   [[I0:%.*]] = divi_signed [[NEW_I0]], [[C2]] : index
diff --git a/mlir/test/Transforms/single-parallel-loop-collapsing.mlir b/mlir/test/Transforms/single-parallel-loop-collapsing.mlir
index 2a516c483c8977c3ef03f9bd9bd555358dc46e8d..496f73568977e65499e63309aa07326398dfff1a 100644
--- a/mlir/test/Transforms/single-parallel-loop-collapsing.mlir
+++ b/mlir/test/Transforms/single-parallel-loop-collapsing.mlir
@@ -14,13 +14,13 @@ func @collapse_to_single() {
 }
 
 // CHECK-LABEL: func @collapse_to_single() {
-// CHECK:         [[C7:%.*]] = constant 7 : index
-// CHECK:         [[C3:%.*]] = constant 3 : index
-// CHECK:         [[C4:%.*]] = constant 4 : index
-// CHECK:         [[C18:%.*]] = constant 18 : index
-// CHECK:         [[C6:%.*]] = constant 6 : index
-// CHECK:         [[C0:%.*]] = constant 0 : index
-// CHECK:         [[C1:%.*]] = constant 1 : index
+// CHECK-DAG:         [[C18:%.*]] = constant 18 : index
+// CHECK-DAG:         [[C6:%.*]] = constant 6 : index
+// CHECK-DAG:         [[C3:%.*]] = constant 3 : index
+// CHECK-DAG:         [[C7:%.*]] = constant 7 : index
+// CHECK-DAG:         [[C4:%.*]] = constant 4 : index
+// CHECK-DAG:         [[C1:%.*]] = constant 1 : index
+// CHECK-DAG:         [[C0:%.*]] = constant 0 : index
 // CHECK:         scf.parallel ([[NEW_I:%.*]]) = ([[C0]]) to ([[C18]]) step ([[C1]]) {
 // CHECK:           [[I0_COUNT:%.*]] = remi_signed [[NEW_I]], [[C6]] : index
 // CHECK:           [[I1_COUNT:%.*]] = divi_signed [[NEW_I]], [[C6]] : index
diff --git a/mlir/test/Transforms/test-canonicalize.mlir b/mlir/test/Transforms/test-canonicalize.mlir
index cc6af03a7818259f60a960f83289598ac4759348..c0033a2409ec2645956ead130dd7d157874f2806 100644
--- a/mlir/test/Transforms/test-canonicalize.mlir
+++ b/mlir/test/Transforms/test-canonicalize.mlir
@@ -52,6 +52,25 @@ func @test_commutative_multi(%arg0: i32, %arg1: i32) -> (i32, i32) {
   return %y, %z: i32, i32
 }
 
+
+// CHECK-LABEL: func @test_commutative_multi_cst
+func @test_commutative_multi_cst(%arg0: i32, %arg1: i32) -> (i32, i32) {
+  // CHECK-NEXT: %c42_i32 = constant 42 : i32
+  %c42_i32 = constant 42 : i32
+  %c42_i32_2 = constant 42 : i32
+  // CHECK-NEXT: %[[O0:.*]] = "test.op_commutative"(%arg0, %arg1, %c42_i32, %c42_i32) : (i32, i32, i32, i32) -> i32
+  %y = "test.op_commutative"(%c42_i32, %arg0, %arg1, %c42_i32_2) : (i32, i32, i32, i32) -> i32
+
+  %c42_i32_3 = constant 42 : i32
+
+  // CHECK-NEXT: %[[O1:.*]] = "test.op_commutative"(%arg0, %arg1, %c42_i32, %c42_i32) : (i32, i32, i32, i32) -> i32
+  %z = "test.op_commutative"(%arg0, %c42_i32_3, %c42_i32_2, %arg1): (i32, i32, i32, i32) -> i32
+  // CHECK-NEXT: return %[[O0]], %[[O1]]
+  return %y, %z: i32, i32
+}
+
+// CHECK-LABEL: func @typemismatch
+
 func @typemismatch() -> i32 {
   %c42 = constant 42.0 : f32
 
diff --git a/mlir/test/Transforms/test-legalizer-analysis.mlir b/mlir/test/Transforms/test-legalizer-analysis.mlir
index cd0b936ada78e41f65a434d261c0a41ce03329d3..ca3c72cae6ff906883865bf1cc86bfd15be449a8 100644
--- a/mlir/test/Transforms/test-legalizer-analysis.mlir
+++ b/mlir/test/Transforms/test-legalizer-analysis.mlir
@@ -1,6 +1,5 @@
 // RUN: mlir-opt -allow-unregistered-dialect -test-legalize-patterns -verify-diagnostics -test-legalize-mode=analysis %s | FileCheck %s
 // expected-remark@-2 {{op 'module' is legalizable}}
-// expected-remark@-3 {{op 'module_terminator' is legalizable}}
 
 // expected-remark@+1 {{op 'func' is legalizable}}
 func @test(%arg0: f32) {
diff --git a/mlir/test/Unit/lit.cfg.py b/mlir/test/Unit/lit.cfg.py
index ea14853e71d634b370a9e84600c318caa69b7790..d645971074f54d7a4b8dbf533088cb35c1bd5e68 100644
--- a/mlir/test/Unit/lit.cfg.py
+++ b/mlir/test/Unit/lit.cfg.py
@@ -13,9 +13,6 @@ config.name = 'MLIR-Unit'
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = []
 
-# is_early; Request to run this suite early.
-config.is_early = True
-
 # test_source_root: The root path where tests are located.
 # test_exec_root: The root path where tests should be run.
 config.test_exec_root = os.path.join(config.mlir_obj_root, 'unittests')
diff --git a/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp b/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp
index 48085573a2d931a773990da0263ea860626f196a..b2d454bb23e048241bc390485058719fd703d1bf 100644
--- a/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp
+++ b/mlir/test/lib/Dialect/Affine/TestAffineDataCopy.cpp
@@ -110,7 +110,7 @@ void TestAffineDataCopy::runOnFunction() {
   // Promoting single iteration loops could lead to simplification of
   // generated load's/store's, and the latter could anyway also be
   // canonicalized.
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(&getContext());
   for (auto op : copyOps) {
     patterns.clear();
     if (isa<AffineLoadOp>(op)) {
diff --git a/mlir/test/lib/Dialect/SPIRV/TestAvailability.cpp b/mlir/test/lib/Dialect/SPIRV/TestAvailability.cpp
index 99a6022a3f45e375e28b1eba5407323096fecd40..11cd05aa9bec863b01f764074fe409fb71c10dc4 100644
--- a/mlir/test/lib/Dialect/SPIRV/TestAvailability.cpp
+++ b/mlir/test/lib/Dialect/SPIRV/TestAvailability.cpp
@@ -139,18 +139,18 @@ void ConvertToTargetEnv::runOnFunction() {
 
   auto target = spirv::SPIRVConversionTarget::get(targetEnv);
 
-  OwningRewritePatternList patterns;
-  patterns.insert<ConvertToAtomCmpExchangeWeak, ConvertToBitReverse,
-                  ConvertToGroupNonUniformBallot, ConvertToModule,
-                  ConvertToSubgroupBallot>(context);
+  RewritePatternSet patterns(context);
+  patterns.add<ConvertToAtomCmpExchangeWeak, ConvertToBitReverse,
+               ConvertToGroupNonUniformBallot, ConvertToModule,
+               ConvertToSubgroupBallot>(context);
 
   if (failed(applyPartialConversion(fn, *target, std::move(patterns))))
     return signalPassFailure();
 }
 
 ConvertToAtomCmpExchangeWeak::ConvertToAtomCmpExchangeWeak(MLIRContext *context)
-    : RewritePattern("test.convert_to_atomic_compare_exchange_weak_op",
-                     {"spv.AtomicCompareExchangeWeak"}, 1, context) {}
+    : RewritePattern("test.convert_to_atomic_compare_exchange_weak_op", 1,
+                     context, {"spv.AtomicCompareExchangeWeak"}) {}
 
 LogicalResult
 ConvertToAtomCmpExchangeWeak::matchAndRewrite(Operation *op,
@@ -170,8 +170,8 @@ ConvertToAtomCmpExchangeWeak::matchAndRewrite(Operation *op,
 }
 
 ConvertToBitReverse::ConvertToBitReverse(MLIRContext *context)
-    : RewritePattern("test.convert_to_bit_reverse_op", {"spv.BitReverse"}, 1,
-                     context) {}
+    : RewritePattern("test.convert_to_bit_reverse_op", 1, context,
+                     {"spv.BitReverse"}) {}
 
 LogicalResult
 ConvertToBitReverse::matchAndRewrite(Operation *op,
@@ -185,8 +185,8 @@ ConvertToBitReverse::matchAndRewrite(Operation *op,
 
 ConvertToGroupNonUniformBallot::ConvertToGroupNonUniformBallot(
     MLIRContext *context)
-    : RewritePattern("test.convert_to_group_non_uniform_ballot_op",
-                     {"spv.GroupNonUniformBallot"}, 1, context) {}
+    : RewritePattern("test.convert_to_group_non_uniform_ballot_op", 1, context,
+                     {"spv.GroupNonUniformBallot"}) {}
 
 LogicalResult ConvertToGroupNonUniformBallot::matchAndRewrite(
     Operation *op, PatternRewriter &rewriter) const {
@@ -198,7 +198,7 @@ LogicalResult ConvertToGroupNonUniformBallot::matchAndRewrite(
 }
 
 ConvertToModule::ConvertToModule(MLIRContext *context)
-    : RewritePattern("test.convert_to_module_op", {"spv.module"}, 1, context) {}
+    : RewritePattern("test.convert_to_module_op", 1, context, {"spv.module"}) {}
 
 LogicalResult
 ConvertToModule::matchAndRewrite(Operation *op,
@@ -210,8 +210,8 @@ ConvertToModule::matchAndRewrite(Operation *op,
 }
 
 ConvertToSubgroupBallot::ConvertToSubgroupBallot(MLIRContext *context)
-    : RewritePattern("test.convert_to_subgroup_ballot_op",
-                     {"spv.SubgroupBallotKHR"}, 1, context) {}
+    : RewritePattern("test.convert_to_subgroup_ballot_op", 1, context,
+                     {"spv.SubgroupBallotKHR"}) {}
 
 LogicalResult
 ConvertToSubgroupBallot::matchAndRewrite(Operation *op,
diff --git a/mlir/test/lib/Dialect/SPIRV/TestGLSLCanonicalization.cpp b/mlir/test/lib/Dialect/SPIRV/TestGLSLCanonicalization.cpp
index d80f912f070c5cbb3ce862c7039ce347e4491e06..ba6c94bc46dbe717ecd62ee277f7711a9e0950e7 100644
--- a/mlir/test/lib/Dialect/SPIRV/TestGLSLCanonicalization.cpp
+++ b/mlir/test/lib/Dialect/SPIRV/TestGLSLCanonicalization.cpp
@@ -25,8 +25,8 @@ public:
 } // namespace
 
 void TestGLSLCanonicalizationPass::runOnOperation() {
-  OwningRewritePatternList patterns;
-  spirv::populateSPIRVGLSLCanonicalizationPatterns(patterns, &getContext());
+  RewritePatternSet patterns(&getContext());
+  spirv::populateSPIRVGLSLCanonicalizationPatterns(patterns);
   (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
 }
 
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
index 6da619652c07236839102dd6d18560ce823db1b1..c0285a3623fa1f87f9c9b63512aa114160b04812 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.cpp
+++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp
@@ -33,6 +33,16 @@ void mlir::test::registerTestDialect(DialectRegistry &registry) {
 
 namespace {
 
+/// Testing the correctness of some traits.
+static_assert(
+    llvm::is_detected<OpTrait::has_implicit_terminator_t,
+                      SingleBlockImplicitTerminatorOp>::value,
+    "has_implicit_terminator_t does not match SingleBlockImplicitTerminatorOp");
+static_assert(OpTrait::hasSingleBlockImplicitTerminator<
+                  SingleBlockImplicitTerminatorOp>::value,
+              "hasSingleBlockImplicitTerminator does not match "
+              "SingleBlockImplicitTerminatorOp");
+
 // Test support for interacting with the AsmPrinter.
 struct TestOpAsmInterface : public OpAsmDialectInterface {
   using OpAsmDialectInterface::OpAsmDialectInterface;
@@ -166,6 +176,29 @@ struct TestInlinerInterface : public DialectInlinerInterface {
 // TestDialect
 //===----------------------------------------------------------------------===//
 
+static void testSideEffectOpGetEffect(
+    Operation *op,
+    SmallVectorImpl<SideEffects::EffectInstance<TestEffects::Effect>> &effects);
+
+// This is the implementation of a dialect fallback for `TestEffectOpInterface`.
+struct TestOpEffectInterfaceFallback
+    : public TestEffectOpInterface::FallbackModel<
+          TestOpEffectInterfaceFallback> {
+  static bool classof(Operation *op) {
+    bool isSupportedOp =
+        op->getName().getStringRef() == "test.unregistered_side_effect_op";
+    assert(isSupportedOp && "Unexpected dispatch");
+    return isSupportedOp;
+  }
+
+  void
+  getEffects(Operation *op,
+             SmallVectorImpl<SideEffects::EffectInstance<TestEffects::Effect>>
+                 &effects) const {
+    testSideEffectOpGetEffect(op, effects);
+  }
+};
+
 void TestDialect::initialize() {
   registerAttributes();
   registerTypes();
@@ -176,6 +209,14 @@ void TestDialect::initialize() {
   addInterfaces<TestOpAsmInterface, TestDialectFoldInterface,
                 TestInlinerInterface>();
   allowUnknownOperations();
+
+  // Instantiate our fallback op interface that we'll use on specific
+  // unregistered op.
+  fallbackEffectOpInterfaces = new TestOpEffectInterfaceFallback;
+}
+TestDialect::~TestDialect() {
+  delete static_cast<TestOpEffectInterfaceFallback *>(
+      fallbackEffectOpInterfaces);
 }
 
 Operation *TestDialect::materializeConstant(OpBuilder &builder, Attribute value,
@@ -183,6 +224,14 @@ Operation *TestDialect::materializeConstant(OpBuilder &builder, Attribute value,
   return builder.create<TestOpConstant>(loc, type, value);
 }
 
+void *TestDialect::getRegisteredInterfaceForOp(TypeID typeID,
+                                               OperationName opName) {
+  if (opName.getIdentifier() == "test.unregistered_side_effect_op" &&
+      typeID == TypeID::get<TestEffectOpInterface>())
+    return fallbackEffectOpInterfaces;
+  return nullptr;
+}
+
 LogicalResult TestDialect::verifyOperationAttribute(Operation *op,
                                                     NamedAttribute namedAttr) {
   if (namedAttr.first == "test.invalid_attr")
@@ -208,6 +257,26 @@ TestDialect::verifyRegionResultAttribute(Operation *op, unsigned regionIndex,
   return success();
 }
 
+Optional<Dialect::ParseOpHook>
+TestDialect::getParseOperationHook(StringRef opName) const {
+  if (opName == "test.dialect_custom_printer") {
+    return ParseOpHook{[](OpAsmParser &parser, OperationState &state) {
+      return parser.parseKeyword("custom_format");
+    }};
+  }
+  return None;
+}
+
+LogicalResult TestDialect::printOperation(Operation *op,
+                                          OpAsmPrinter &printer) const {
+  StringRef opName = op->getName().getStringRef();
+  if (opName == "test.dialect_custom_printer") {
+    printer.getStream() << opName << " custom_format";
+    return success();
+  }
+  return failure();
+}
+
 //===----------------------------------------------------------------------===//
 // TestBranchOp
 //===----------------------------------------------------------------------===//
@@ -235,9 +304,9 @@ struct FoldToCallOpPattern : public OpRewritePattern<FoldToCallOp> {
 };
 } // end anonymous namespace
 
-void FoldToCallOp::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<FoldToCallOpPattern>(context);
+void FoldToCallOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                               MLIRContext *context) {
+  results.add<FoldToCallOpPattern>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -615,8 +684,8 @@ struct TestRemoveOpWithInnerOps
 } // end anonymous namespace
 
 void TestOpWithRegionPattern::getCanonicalizationPatterns(
-    OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<TestRemoveOpWithInnerOps>(context);
+    RewritePatternSet &results, MLIRContext *context) {
+  results.add<TestRemoveOpWithInnerOps>(context);
 }
 
 OpFoldResult TestOpWithRegionFold::fold(ArrayRef<Attribute> operands) {
@@ -696,6 +765,17 @@ struct TestResource : public SideEffects::Resource::Base<TestResource> {
 };
 } // end anonymous namespace
 
+static void testSideEffectOpGetEffect(
+    Operation *op,
+    SmallVectorImpl<SideEffects::EffectInstance<TestEffects::Effect>>
+        &effects) {
+  auto effectsAttr = op->getAttrOfType<AffineMapAttr>("effect_parameter");
+  if (!effectsAttr)
+    return;
+
+  effects.emplace_back(TestEffects::Concrete::get(), effectsAttr);
+}
+
 void SideEffectOp::getEffects(
     SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
   // Check for an effects attribute on the op instance.
@@ -734,11 +814,7 @@ void SideEffectOp::getEffects(
 
 void SideEffectOp::getEffects(
     SmallVectorImpl<TestEffects::EffectInstance> &effects) {
-  auto effectsAttr = (*this)->getAttrOfType<AffineMapAttr>("effect_parameter");
-  if (!effectsAttr)
-    return;
-
-  effects.emplace_back(TestEffects::Concrete::get(), effectsAttr);
+  testSideEffectOpGetEffect(getOperation(), effects);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 1968ebd46f6ff5c99e3f4f64582dc9669d448e24..25cd7f2ce67cd1f2bf5c56cb628570aaa3429560 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -29,6 +29,7 @@ def Test_Dialect : Dialect {
   let hasOperationAttrVerify = 1;
   let hasRegionArgAttrVerify = 1;
   let hasRegionResultAttrVerify = 1;
+  let hasOperationInterfaceFallback = 1;
   let dependentDialects = ["::mlir::DLTIDialect"];
 
   let extraClassDeclaration = [{
@@ -39,6 +40,18 @@ def Test_Dialect : Dialect {
                              Type type) const override;
     void printAttribute(Attribute attr,
                         DialectAsmPrinter &printer) const override;
+
+    // Provides a custom printing/parsing for some operations.
+    Optional<ParseOpHook>
+      getParseOperationHook(StringRef opName) const override;
+    LogicalResult printOperation(Operation *op,
+                                 OpAsmPrinter &printer) const override;
+
+    ~TestDialect();
+  private:
+    // Storage for a custom fallback interface.
+    void *fallbackEffectOpInterfaces;
+
   }];
 }
 
@@ -320,6 +333,22 @@ def SizedRegionOp : TEST_Op<"sized_region_op", []> {
   let regions = (region SizedRegion<2>:$my_region, SizedRegion<1>);
 }
 
+//===----------------------------------------------------------------------===//
+// NoTerminator Operation
+//===----------------------------------------------------------------------===//
+
+def SingleNoTerminatorOp : TEST_Op<"single_no_terminator_op", GraphRegionNoTerminator.traits> {
+  let regions = (region SizedRegion<1>:$my_region);
+
+  let assemblyFormat = "attr-dict `:` $my_region";
+}
+
+def VariadicNoTerminatorOp : TEST_Op<"variadic_no_terminator_op", GraphRegionNoTerminator.traits> {
+  let regions = (region VariadicRegion<SizedRegion<1>>:$my_regions);
+
+  let assemblyFormat = "attr-dict `:` $my_regions";
+}
+
 //===----------------------------------------------------------------------===//
 // Test Call Interfaces
 //===----------------------------------------------------------------------===//
@@ -1052,39 +1081,39 @@ def OneResultOp3 : TEST_Op<"one_result3"> {
 }
 
 // Test using multi-result op as a whole
-def : Pat<(ThreeResultOp MultiResultOpKind1),
-          (AnotherThreeResultOp MultiResultOpKind1)>;
+def : Pat<(ThreeResultOp MultiResultOpKind1:$kind),
+          (AnotherThreeResultOp $kind)>;
 
 // Test using multi-result op as a whole for partial replacement
-def : Pattern<(ThreeResultOp MultiResultOpKind2),
-              [(TwoResultOp MultiResultOpKind2),
-               (OneResultOp1 MultiResultOpKind2)]>;
-def : Pattern<(ThreeResultOp MultiResultOpKind3),
-              [(OneResultOp2 MultiResultOpKind3),
-               (AnotherTwoResultOp MultiResultOpKind3)]>;
+def : Pattern<(ThreeResultOp MultiResultOpKind2:$kind),
+              [(TwoResultOp $kind),
+               (OneResultOp1 $kind)]>;
+def : Pattern<(ThreeResultOp MultiResultOpKind3:$kind),
+              [(OneResultOp2 $kind),
+               (AnotherTwoResultOp $kind)]>;
 
 // Test using results separately in a multi-result op
-def : Pattern<(ThreeResultOp MultiResultOpKind4),
-              [(TwoResultOp:$res1__0 MultiResultOpKind4),
-               (OneResultOp1 MultiResultOpKind4),
-               (TwoResultOp:$res2__1 MultiResultOpKind4)]>;
+def : Pattern<(ThreeResultOp MultiResultOpKind4:$kind),
+              [(TwoResultOp:$res1__0 $kind),
+               (OneResultOp1 $kind),
+               (TwoResultOp:$res2__1 $kind)]>;
 
 // Test referencing a single value in the value pack
 // This rule only matches TwoResultOp if its second result has no use.
-def : Pattern<(TwoResultOp:$res MultiResultOpKind5),
-              [(OneResultOp2 MultiResultOpKind5),
-               (OneResultOp1 MultiResultOpKind5)],
+def : Pattern<(TwoResultOp:$res MultiResultOpKind5:$kind),
+              [(OneResultOp2 $kind),
+               (OneResultOp1 $kind)],
               [(HasNoUseOf:$res__1)]>;
 
 // Test using auxiliary ops for replacing multi-result op
 def : Pattern<
-    (ThreeResultOp MultiResultOpKind6), [
+    (ThreeResultOp MultiResultOpKind6:$kind), [
         // Auxiliary op generated to help building the final result but not
         // directly used to replace the source op's results.
-        (TwoResultOp:$interm MultiResultOpKind6),
+        (TwoResultOp:$interm $kind),
 
         (OneResultOp3 $interm__1),
-        (AnotherTwoResultOp MultiResultOpKind6)
+        (AnotherTwoResultOp $kind)
     ]>;
 
 //===----------------------------------------------------------------------===//
@@ -1645,6 +1674,11 @@ def FormatOptionalEnumAttr : TEST_Op<"format_optional_enum_attr"> {
   let assemblyFormat = "($attr^)? attr-dict";
 }
 
+def FormatOptionalWithElse : TEST_Op<"format_optional_else"> {
+  let arguments = (ins UnitAttr:$isFirstBranchPresent);
+  let assemblyFormat = "(`then` $isFirstBranchPresent^):(`else`)? attr-dict";
+}
+
 //===----------------------------------------------------------------------===//
 // Custom Directives
 
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index 53651de0235a775f66db6d2fe596a8412815cbe1..1cc0c62b8691f5b20564949abd1d59989de0508d 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -79,11 +79,11 @@ public:
 
 struct TestPatternDriver : public PassWrapper<TestPatternDriver, FunctionPass> {
   void runOnFunction() override {
-    mlir::OwningRewritePatternList patterns;
-    populateWithGenerated(&getContext(), patterns);
+    mlir::RewritePatternSet patterns(&getContext());
+    populateWithGenerated(patterns);
 
     // Verify named pattern is generated with expected name.
-    patterns.insert<FoldingPattern, TestNamedPatternRule>(&getContext());
+    patterns.add<FoldingPattern, TestNamedPatternRule>(&getContext());
 
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
@@ -325,7 +325,7 @@ struct TestUndoBlockErase : public ConversionPattern {
 /// This patterns erases a region operation that has had a type conversion.
 struct TestDropOpSignatureConversion : public ConversionPattern {
   TestDropOpSignatureConversion(MLIRContext *ctx, TypeConverter &converter)
-      : ConversionPattern("test.drop_region_op", 1, converter, ctx) {}
+      : ConversionPattern(converter, "test.drop_region_op", 1, ctx) {}
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
@@ -557,25 +557,23 @@ struct TestLegalizePatternDriver
 
   void runOnOperation() override {
     TestTypeConverter converter;
-    mlir::OwningRewritePatternList patterns;
-    populateWithGenerated(&getContext(), patterns);
-    patterns.insert<
-        TestRegionRewriteBlockMovement, TestRegionRewriteUndo, TestCreateBlock,
-        TestCreateIllegalBlock, TestUndoBlockArgReplace, TestUndoBlockErase,
-        TestPassthroughInvalidOp, TestSplitReturnType,
-        TestChangeProducerTypeI32ToF32, TestChangeProducerTypeF32ToF64,
-        TestChangeProducerTypeF32ToInvalid, TestUpdateConsumerType,
-        TestNonRootReplacement, TestBoundedRecursiveRewrite,
-        TestNestedOpCreationUndoRewrite>(&getContext());
-    patterns.insert<TestDropOpSignatureConversion>(&getContext(), converter);
-    mlir::populateFuncOpTypeConversionPattern(patterns, &getContext(),
-                                              converter);
-    mlir::populateCallOpTypeConversionPattern(patterns, &getContext(),
-                                              converter);
+    mlir::RewritePatternSet patterns(&getContext());
+    populateWithGenerated(patterns);
+    patterns
+        .add<TestRegionRewriteBlockMovement, TestRegionRewriteUndo,
+             TestCreateBlock, TestCreateIllegalBlock, TestUndoBlockArgReplace,
+             TestUndoBlockErase, TestPassthroughInvalidOp, TestSplitReturnType,
+             TestChangeProducerTypeI32ToF32, TestChangeProducerTypeF32ToF64,
+             TestChangeProducerTypeF32ToInvalid, TestUpdateConsumerType,
+             TestNonRootReplacement, TestBoundedRecursiveRewrite,
+             TestNestedOpCreationUndoRewrite>(&getContext());
+    patterns.add<TestDropOpSignatureConversion>(&getContext(), converter);
+    mlir::populateFuncOpTypeConversionPattern(patterns, converter);
+    mlir::populateCallOpTypeConversionPattern(patterns, converter);
 
     // Define the conversion target used for the test.
     ConversionTarget target(getContext());
-    target.addLegalOp<ModuleOp, ModuleTerminatorOp>();
+    target.addLegalOp<ModuleOp>();
     target.addLegalOp<LegalOpA, LegalOpB, TestCastOp, TestValidOp,
                       TerminatorOp>();
     target
@@ -700,11 +698,11 @@ struct OneVResOneVOperandOp1Converter
 struct TestRemappedValue
     : public mlir::PassWrapper<TestRemappedValue, FunctionPass> {
   void runOnFunction() override {
-    mlir::OwningRewritePatternList patterns;
-    patterns.insert<OneVResOneVOperandOp1Converter>(&getContext());
+    mlir::RewritePatternSet patterns(&getContext());
+    patterns.add<OneVResOneVOperandOp1Converter>(&getContext());
 
     mlir::ConversionTarget target(getContext());
-    target.addLegalOp<ModuleOp, ModuleTerminatorOp, FuncOp, TestReturnOp>();
+    target.addLegalOp<ModuleOp, FuncOp, TestReturnOp>();
     // We make OneVResOneVOperandOp1 legal only when it has more that one
     // operand. This will trigger the conversion that will replace one-operand
     // OneVResOneVOperandOp1 with two-operand OneVResOneVOperandOp1.
@@ -728,7 +726,8 @@ struct TestRemappedValue
 namespace {
 /// This pattern matches and removes any operation in the test dialect.
 struct RemoveTestDialectOps : public RewritePattern {
-  RemoveTestDialectOps() : RewritePattern(/*benefit=*/1, MatchAnyOpTypeTag()) {}
+  RemoveTestDialectOps(MLIRContext *context)
+      : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/1, context) {}
 
   LogicalResult matchAndRewrite(Operation *op,
                                 PatternRewriter &rewriter) const override {
@@ -742,8 +741,8 @@ struct RemoveTestDialectOps : public RewritePattern {
 struct TestUnknownRootOpDriver
     : public mlir::PassWrapper<TestUnknownRootOpDriver, FunctionPass> {
   void runOnFunction() override {
-    mlir::OwningRewritePatternList patterns;
-    patterns.insert<RemoveTestDialectOps>();
+    mlir::RewritePatternSet patterns(&getContext());
+    patterns.add<RemoveTestDialectOps>(&getContext());
 
     mlir::ConversionTarget target(getContext());
     target.addIllegalDialect<TestDialect>();
@@ -878,12 +877,11 @@ struct TestTypeConversionDriver
     });
 
     // Initialize the set of rewrite patterns.
-    OwningRewritePatternList patterns;
-    patterns.insert<TestTypeConsumerForward, TestTypeConversionProducer,
-                    TestSignatureConversionUndo>(converter, &getContext());
-    patterns.insert<TestTypeConversionAnotherProducer>(&getContext());
-    mlir::populateFuncOpTypeConversionPattern(patterns, &getContext(),
-                                              converter);
+    RewritePatternSet patterns(&getContext());
+    patterns.add<TestTypeConsumerForward, TestTypeConversionProducer,
+                 TestSignatureConversionUndo>(converter, &getContext());
+    patterns.add<TestTypeConversionAnotherProducer>(&getContext());
+    mlir::populateFuncOpTypeConversionPattern(patterns, converter);
 
     if (failed(applyPartialConversion(getOperation(), target,
                                       std::move(patterns))))
@@ -966,15 +964,13 @@ struct TestMergeBlocksPatternDriver
     : public PassWrapper<TestMergeBlocksPatternDriver,
                          OperationPass<ModuleOp>> {
   void runOnOperation() override {
-    mlir::OwningRewritePatternList patterns;
     MLIRContext *context = &getContext();
-    patterns
-        .insert<TestMergeBlock, TestUndoBlocksMerge, TestMergeSingleBlockOps>(
-            context);
+    mlir::RewritePatternSet patterns(context);
+    patterns.add<TestMergeBlock, TestUndoBlocksMerge, TestMergeSingleBlockOps>(
+        context);
     ConversionTarget target(*context);
-    target.addLegalOp<FuncOp, ModuleOp, ModuleTerminatorOp, TerminatorOp,
-                      TestBranchOp, TestTypeConsumerOp, TestTypeProducerOp,
-                      TestReturnOp>();
+    target.addLegalOp<FuncOp, ModuleOp, TerminatorOp, TestBranchOp,
+                      TestTypeConsumerOp, TestTypeProducerOp, TestReturnOp>();
     target.addIllegalOp<ILLegalOpF>();
 
     /// Expect the op to have a single block after legalization.
@@ -1035,9 +1031,9 @@ struct TestSelectiveReplacementPatternDriver
     : public PassWrapper<TestSelectiveReplacementPatternDriver,
                          OperationPass<>> {
   void runOnOperation() override {
-    mlir::OwningRewritePatternList patterns;
     MLIRContext *context = &getContext();
-    patterns.insert<TestSelectiveOpReplacementPattern>(context);
+    mlir::RewritePatternSet patterns(context);
+    patterns.add<TestSelectiveOpReplacementPattern>(context);
     (void)applyPatternsAndFoldGreedily(getOperation()->getRegions(),
                                        std::move(patterns));
   }
diff --git a/mlir/test/lib/Dialect/Test/TestTraits.cpp b/mlir/test/lib/Dialect/Test/TestTraits.cpp
index 87bd782f0f6f6924815d0af1096851bb2a563f9c..1e675aec0d8ed95f8903bfae23691bcf15828a81 100644
--- a/mlir/test/lib/Dialect/Test/TestTraits.cpp
+++ b/mlir/test/lib/Dialect/Test/TestTraits.cpp
@@ -34,7 +34,7 @@ namespace {
 struct TestTraitFolder : public PassWrapper<TestTraitFolder, FunctionPass> {
   void runOnFunction() override {
     (void)applyPatternsAndFoldGreedily(getFunction(),
-                                       OwningRewritePatternList());
+                                       RewritePatternSet(&getContext()));
   }
 };
 } // end anonymous namespace
diff --git a/mlir/test/lib/Dialect/Test/TestTypes.h b/mlir/test/lib/Dialect/Test/TestTypes.h
index 150e5bafad5268f9a95363ec96505062df1f2ad3..e2a65a2aa12dfa74c4f3fe26e04b66d0976ff351 100644
--- a/mlir/test/lib/Dialect/Test/TestTypes.h
+++ b/mlir/test/lib/Dialect/Test/TestTypes.h
@@ -135,8 +135,8 @@ public:
 
   unsigned getKey() { return getImpl()->key; }
 
-  unsigned getTypeSize(const DataLayout &dataLayout,
-                       DataLayoutEntryListRef params) const {
+  unsigned getTypeSizeInBits(const DataLayout &dataLayout,
+                             DataLayoutEntryListRef params) const {
     return extractKind(params, "size");
   }
 
diff --git a/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp b/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
index 416bbca45c07d5904c5144a844e6221fe9cb20d6..da890c9123055dece75356949fa69b59e1b3c831 100644
--- a/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
+++ b/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
@@ -183,12 +183,12 @@ struct TosaTestQuantUtilAPI
 };
 
 void TosaTestQuantUtilAPI::runOnFunction() {
-  OwningRewritePatternList patterns;
   auto *ctx = &getContext();
+  RewritePatternSet patterns(ctx);
   auto func = getFunction();
 
-  patterns.insert<ConvertTosaNegateOp>(ctx);
-  patterns.insert<ConvertTosaConv2DOp>(ctx);
+  patterns.add<ConvertTosaNegateOp>(ctx);
+  patterns.add<ConvertTosaConv2DOp>(ctx);
   (void)applyPatternsAndFoldGreedily(func, std::move(patterns));
 }
 
diff --git a/mlir/test/lib/Rewrite/TestPDLByteCode.cpp b/mlir/test/lib/Rewrite/TestPDLByteCode.cpp
index 3b23cb103675cf9e0925321f3df9637354d74027..b20224a50b83d02d9444f40e083bd8d3e2df0715 100644
--- a/mlir/test/lib/Rewrite/TestPDLByteCode.cpp
+++ b/mlir/test/lib/Rewrite/TestPDLByteCode.cpp
@@ -24,20 +24,45 @@ static LogicalResult customMultiEntityConstraint(ArrayRef<PDLValue> values,
                                                  PatternRewriter &rewriter) {
   return customSingleEntityConstraint(values[1], constantParams, rewriter);
 }
+static LogicalResult
+customMultiEntityVariadicConstraint(ArrayRef<PDLValue> values,
+                                    ArrayAttr constantParams,
+                                    PatternRewriter &rewriter) {
+  if (llvm::any_of(values, [](const PDLValue &value) { return !value; }))
+    return failure();
+  ValueRange operandValues = values[0].cast<ValueRange>();
+  TypeRange typeValues = values[1].cast<TypeRange>();
+  if (operandValues.size() != 2 || typeValues.size() != 2)
+    return failure();
+  return success();
+}
 
 // Custom creator invoked from PDL.
-static PDLValue customCreate(ArrayRef<PDLValue> args, ArrayAttr constantParams,
-                             PatternRewriter &rewriter) {
-  return rewriter.createOperation(
-      OperationState(args[0].cast<Operation *>()->getLoc(), "test.success"));
+static void customCreate(ArrayRef<PDLValue> args, ArrayAttr constantParams,
+                         PatternRewriter &rewriter, PDLResultList &results) {
+  results.push_back(rewriter.createOperation(
+      OperationState(args[0].cast<Operation *>()->getLoc(), "test.success")));
+}
+static void customVariadicResultCreate(ArrayRef<PDLValue> args,
+                                       ArrayAttr constantParams,
+                                       PatternRewriter &rewriter,
+                                       PDLResultList &results) {
+  Operation *root = args[0].cast<Operation *>();
+  results.push_back(root->getOperands());
+  results.push_back(root->getOperands().getTypes());
+}
+static void customCreateType(ArrayRef<PDLValue> args, ArrayAttr constantParams,
+                             PatternRewriter &rewriter,
+                             PDLResultList &results) {
+  results.push_back(rewriter.getF32Type());
 }
 
 /// Custom rewriter invoked from PDL.
-static void customRewriter(Operation *root, ArrayRef<PDLValue> args,
-                           ArrayAttr constantParams,
-                           PatternRewriter &rewriter) {
+static void customRewriter(ArrayRef<PDLValue> args, ArrayAttr constantParams,
+                           PatternRewriter &rewriter, PDLResultList &results) {
+  Operation *root = args[0].cast<Operation *>();
   OperationState successOpState(root->getLoc(), "test.success");
-  successOpState.addOperands(args[0].cast<Value>());
+  successOpState.addOperands(args[1].cast<Value>());
   successOpState.addAttribute("constantParams", constantParams);
   rewriter.createOperation(successOpState);
   rewriter.eraseOp(root);
@@ -63,10 +88,15 @@ struct TestPDLByteCodePass
                                           customMultiEntityConstraint);
     pdlPattern.registerConstraintFunction("single_entity_constraint",
                                           customSingleEntityConstraint);
-    pdlPattern.registerCreateFunction("creator", customCreate);
+    pdlPattern.registerConstraintFunction("multi_entity_var_constraint",
+                                          customMultiEntityVariadicConstraint);
+    pdlPattern.registerRewriteFunction("creator", customCreate);
+    pdlPattern.registerRewriteFunction("var_creator",
+                                       customVariadicResultCreate);
+    pdlPattern.registerRewriteFunction("type_creator", customCreateType);
     pdlPattern.registerRewriteFunction("rewriter", customRewriter);
 
-    OwningRewritePatternList patternList(std::move(pdlPattern));
+    RewritePatternSet patternList(std::move(pdlPattern));
 
     // Invoke the pattern driver with the provided patterns.
     (void)applyPatternsAndFoldGreedily(irModule.getBodyRegion(),
diff --git a/mlir/test/lib/Transforms/TestConvVectorization.cpp b/mlir/test/lib/Transforms/TestConvVectorization.cpp
index cda3542235da987b5db6a158b399a9d2a97518fd..3ccfa93e0a0d379593cee9c9525be84d74449bc5 100644
--- a/mlir/test/lib/Transforms/TestConvVectorization.cpp
+++ b/mlir/test/lib/Transforms/TestConvVectorization.cpp
@@ -56,17 +56,17 @@ void TestConvVectorization::runOnOperation() {
   ConversionTarget target(*context);
   target.addLegalDialect<AffineDialect, scf::SCFDialect, StandardOpsDialect,
                          VectorDialect>();
-  target.addLegalOp<ModuleOp, FuncOp, ModuleTerminatorOp, ReturnOp>();
+  target.addLegalOp<ModuleOp, FuncOp, ReturnOp>();
   target.addLegalOp<linalg::FillOp, linalg::YieldOp>();
 
-  SmallVector<OwningRewritePatternList, 4> stage1Patterns;
+  SmallVector<RewritePatternSet, 4> stage1Patterns;
   linalg::populateConvVectorizationPatterns(context, stage1Patterns, tileSizes);
-  SmallVector<FrozenRewritePatternList, 4> frozenStage1Patterns;
+  SmallVector<FrozenRewritePatternSet, 4> frozenStage1Patterns;
   llvm::move(stage1Patterns, std::back_inserter(frozenStage1Patterns));
 
-  OwningRewritePatternList stage2Patterns =
+  RewritePatternSet stage2Patterns =
       linalg::getLinalgTilingCanonicalizationPatterns(context);
-  stage2Patterns.insert<linalg::AffineMinSCFCanonicalizationPattern>(context);
+  stage2Patterns.add<linalg::AffineMinSCFCanonicalizationPattern>(context);
 
   auto stage3Transforms = [](Operation *op) {
     PassManager pm(op->getContext());
@@ -91,11 +91,11 @@ void TestConvVectorization::runOnOperation() {
   VectorTransformsOptions vectorTransformsOptions{
       VectorContractLowering::Dot, VectorTransposeLowering::EltWise};
 
-  OwningRewritePatternList vectorTransferPatterns;
+  RewritePatternSet vectorTransferPatterns(context);
   // Pattern is not applied because rank-reducing vector transfer is not yet
   // supported as can be seen in splitFullAndPartialTransferPrecondition,
   // VectorTransforms.cpp
-  vectorTransferPatterns.insert<VectorTransferFullPartialRewriter>(
+  vectorTransferPatterns.add<VectorTransferFullPartialRewriter>(
       context, vectorTransformsOptions);
   (void)applyPatternsAndFoldGreedily(module, std::move(vectorTransferPatterns));
 
@@ -106,15 +106,15 @@ void TestConvVectorization::runOnOperation() {
     llvm_unreachable("Unexpected failure in linalg to loops pass.");
 
   // Programmatic controlled lowering of vector.contract only.
-  OwningRewritePatternList vectorContractLoweringPatterns;
+  RewritePatternSet vectorContractLoweringPatterns(context);
   populateVectorContractLoweringPatterns(vectorContractLoweringPatterns,
-                                         context, vectorTransformsOptions);
+                                         vectorTransformsOptions);
   (void)applyPatternsAndFoldGreedily(module,
                                      std::move(vectorContractLoweringPatterns));
 
   // Programmatic controlled lowering of vector.transfer only.
-  OwningRewritePatternList vectorToLoopsPatterns;
-  populateVectorToSCFConversionPatterns(vectorToLoopsPatterns, context,
+  RewritePatternSet vectorToLoopsPatterns(context);
+  populateVectorToSCFConversionPatterns(vectorToLoopsPatterns,
                                         VectorTransferToSCFOptions());
   (void)applyPatternsAndFoldGreedily(module, std::move(vectorToLoopsPatterns));
 
diff --git a/mlir/test/lib/Transforms/TestConvertCallOp.cpp b/mlir/test/lib/Transforms/TestConvertCallOp.cpp
index 2fe29b44e00678d482b2360b42f16155198a3c44..6f69e54fd6b5e716456b98ace49aef107d7b3c0d 100644
--- a/mlir/test/lib/Transforms/TestConvertCallOp.cpp
+++ b/mlir/test/lib/Transforms/TestConvertCallOp.cpp
@@ -43,15 +43,15 @@ public:
     ModuleOp m = getOperation();
 
     // Populate type conversions.
-    LLVMTypeConverter type_converter(m.getContext());
-    type_converter.addConversion([&](test::TestType type) {
+    LLVMTypeConverter typeConverter(m.getContext());
+    typeConverter.addConversion([&](test::TestType type) {
       return LLVM::LLVMPointerType::get(IntegerType::get(m.getContext(), 8));
     });
 
     // Populate patterns.
-    OwningRewritePatternList patterns;
-    populateStdToLLVMConversionPatterns(type_converter, patterns);
-    patterns.insert<TestTypeProducerOpConverter>(type_converter);
+    RewritePatternSet patterns(m.getContext());
+    populateStdToLLVMConversionPatterns(typeConverter, patterns);
+    patterns.add<TestTypeProducerOpConverter>(typeConverter);
 
     // Set target.
     ConversionTarget target(getContext());
diff --git a/mlir/test/lib/Transforms/TestConvertGPUKernelToHsaco.cpp b/mlir/test/lib/Transforms/TestConvertGPUKernelToHsaco.cpp
index 58e890b907a467009fcac796f22d8e166be12b51..5a3cb33526f6025dea796554c0a425160706d236 100644
--- a/mlir/test/lib/Transforms/TestConvertGPUKernelToHsaco.cpp
+++ b/mlir/test/lib/Transforms/TestConvertGPUKernelToHsaco.cpp
@@ -6,11 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
-#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
+#include "mlir/Dialect/GPU/Passes.h"
+
 #include "mlir/Pass/Pass.h"
-#include "mlir/Pass/PassManager.h"
-#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Export.h"
 #include "llvm/Support/TargetSelect.h"
@@ -18,38 +16,54 @@
 using namespace mlir;
 
 #if MLIR_ROCM_CONVERSIONS_ENABLED
-static OwnedBlob compileIsaToHsacoForTesting(const std::string &, Location,
-                                             StringRef) {
-  const char data[] = "HSACO";
-  return std::make_unique<std::vector<char>>(data, data + sizeof(data) - 1);
+namespace {
+class TestSerializeToHsacoPass
+    : public PassWrapper<TestSerializeToHsacoPass, gpu::SerializeToBlobPass> {
+public:
+  TestSerializeToHsacoPass();
+
+private:
+  void getDependentDialects(DialectRegistry &registry) const override;
+
+  // Serializes ROCDL IR to HSACO.
+  std::unique_ptr<std::vector<char>>
+  serializeISA(const std::string &isa) override;
+};
+} // namespace
+
+TestSerializeToHsacoPass::TestSerializeToHsacoPass() {
+  this->triple = "amdgcn-amd-amdhsa";
+  this->chip = "gfx900";
+}
+
+void TestSerializeToHsacoPass::getDependentDialects(
+    DialectRegistry &registry) const {
+  registerROCDLDialectTranslation(registry);
+  gpu::SerializeToBlobPass::getDependentDialects(registry);
 }
 
-static std::unique_ptr<llvm::Module>
-translateModuleToROCDL(Operation *m, llvm::LLVMContext &llvmContext,
-                       StringRef moduleName) {
-  registerLLVMDialectTranslation(*m->getContext());
-  registerROCDLDialectTranslation(*m->getContext());
-  return translateModuleToLLVMIR(m, llvmContext, moduleName);
+std::unique_ptr<std::vector<char>>
+TestSerializeToHsacoPass::serializeISA(const std::string &) {
+  std::string data = "HSACO";
+  return std::make_unique<std::vector<char>>(data.begin(), data.end());
 }
 
 namespace mlir {
 namespace test {
-void registerTestConvertGPUKernelToHsacoPass() {
-  PassPipelineRegistration<>(
-      "test-kernel-to-hsaco",
-      "Convert all kernel functions to ROCm hsaco blobs",
-      [](OpPassManager &pm) {
+// Register test pass to serialize GPU module to a HSAco binary annotation.
+void registerTestGpuSerializeToHsacoPass() {
+  PassRegistration<TestSerializeToHsacoPass> registerSerializeToHsaco(
+      "test-gpu-to-hsaco",
+      "Lower GPU kernel function to HSAco binary annotations", [] {
         // Initialize LLVM AMDGPU backend.
         LLVMInitializeAMDGPUTarget();
         LLVMInitializeAMDGPUTargetInfo();
         LLVMInitializeAMDGPUTargetMC();
         LLVMInitializeAMDGPUAsmPrinter();
 
-        pm.addPass(createConvertGPUKernelToBlobPass(
-            translateModuleToROCDL, compileIsaToHsacoForTesting,
-            "amdgcn-amd-amdhsa", "gfx900", "-code-object-v3", "rocdl.hsaco"));
+        return std::make_unique<TestSerializeToHsacoPass>();
       });
 }
 } // namespace test
 } // namespace mlir
-#endif
+#endif // MLIR_ROCM_CONVERSIONS_ENABLED
diff --git a/mlir/test/lib/Transforms/TestDataLayoutQuery.cpp b/mlir/test/lib/Transforms/TestDataLayoutQuery.cpp
index c6823a92775f6cf05cc1ca160cb83f8280935f51..843714c640326430574983f5e7e37b2580388450 100644
--- a/mlir/test/lib/Transforms/TestDataLayoutQuery.cpp
+++ b/mlir/test/lib/Transforms/TestDataLayoutQuery.cpp
@@ -36,13 +36,22 @@ struct TestDataLayoutQuery
             scope, scope ? cast<DataLayoutOpInterface>(scope.getOperation())
                          : nullptr);
       }
+      auto module = op->getParentOfType<ModuleOp>();
+      if (!layouts.count(module))
+        layouts.try_emplace(module, module);
 
-      const DataLayout &layout = layouts.find(scope)->getSecond();
+      Operation *closest = (scope && module && module->isProperAncestor(scope))
+                               ? scope.getOperation()
+                               : module.getOperation();
+
+      const DataLayout &layout = layouts.find(closest)->getSecond();
       unsigned size = layout.getTypeSize(op.getType());
+      unsigned bitsize = layout.getTypeSizeInBits(op.getType());
       unsigned alignment = layout.getTypeABIAlignment(op.getType());
       unsigned preferred = layout.getTypePreferredAlignment(op.getType());
       op->setAttrs(
           {builder.getNamedAttr("size", builder.getIndexAttr(size)),
+           builder.getNamedAttr("bitsize", builder.getIndexAttr(bitsize)),
            builder.getNamedAttr("alignment", builder.getIndexAttr(alignment)),
            builder.getNamedAttr("preferred", builder.getIndexAttr(preferred))});
     });
diff --git a/mlir/test/lib/Transforms/TestDecomposeCallGraphTypes.cpp b/mlir/test/lib/Transforms/TestDecomposeCallGraphTypes.cpp
index 2dd2c34238529680dd6ae6b8f42f920f6d1fd1bf..f2e1fa264bcf9203d08d01939893c1bdf585ddae 100644
--- a/mlir/test/lib/Transforms/TestDecomposeCallGraphTypes.cpp
+++ b/mlir/test/lib/Transforms/TestDecomposeCallGraphTypes.cpp
@@ -33,7 +33,7 @@ struct TestDecomposeCallGraphTypes
     TypeConverter typeConverter;
     ConversionTarget target(*context);
     ValueDecomposer decomposer;
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(context);
 
     target.addLegalDialect<test::TestDialect>();
 
diff --git a/mlir/test/lib/Transforms/TestExpandTanh.cpp b/mlir/test/lib/Transforms/TestExpandTanh.cpp
index e67e89b38cdb026ed4596e341b04a1df1af818f3..0241bddc982bb31fe1740288c0983ba5dfbfb374 100644
--- a/mlir/test/lib/Transforms/TestExpandTanh.cpp
+++ b/mlir/test/lib/Transforms/TestExpandTanh.cpp
@@ -24,8 +24,8 @@ struct TestExpandTanhPass
 } // end anonymous namespace
 
 void TestExpandTanhPass::runOnFunction() {
-  OwningRewritePatternList patterns;
-  populateExpandTanhPattern(patterns, &getContext());
+  RewritePatternSet patterns(&getContext());
+  populateExpandTanhPattern(patterns);
   (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
 }
 
diff --git a/mlir/test/lib/Transforms/TestGpuRewrite.cpp b/mlir/test/lib/Transforms/TestGpuRewrite.cpp
index 44ffd38bdc23ef076c597c87441b63258fb6bd07..27ecae96707bfaee0124c5e072b43549c53eda58 100644
--- a/mlir/test/lib/Transforms/TestGpuRewrite.cpp
+++ b/mlir/test/lib/Transforms/TestGpuRewrite.cpp
@@ -25,8 +25,8 @@ struct TestGpuRewritePass
     registry.insert<StandardOpsDialect, memref::MemRefDialect>();
   }
   void runOnOperation() override {
-    OwningRewritePatternList patterns;
-    populateGpuRewritePatterns(&getContext(), patterns);
+    RewritePatternSet patterns(&getContext());
+    populateGpuRewritePatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
   }
 };
diff --git a/mlir/test/lib/Transforms/TestLinalgFusionTransforms.cpp b/mlir/test/lib/Transforms/TestLinalgFusionTransforms.cpp
index 1efc565a1c60f899fc0d97ca318d36524c3c88a2..3ef6ed5e4b4bad4451c4b0ae4ff4235c2b25dda7 100644
--- a/mlir/test/lib/Transforms/TestLinalgFusionTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestLinalgFusionTransforms.cpp
@@ -23,9 +23,9 @@ using namespace mlir::linalg;
 template <LinalgTilingLoopType LoopType>
 static void fillFusionPatterns(MLIRContext *context,
                                const LinalgDependenceGraph &dependenceGraph,
-                               OwningRewritePatternList &patterns) {
-  patterns.insert<LinalgTileAndFusePattern<MatmulOp>,
-                  LinalgTileAndFusePattern<ConvOp>>(
+                               RewritePatternSet &patterns) {
+  patterns.add<LinalgTileAndFusePattern<MatmulOp>,
+               LinalgTileAndFusePattern<ConvOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64, 16}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({2}),
@@ -39,7 +39,7 @@ static void fillFusionPatterns(MLIRContext *context,
           ArrayRef<Identifier>(),
           Identifier::get("after_basic_fusion_original", context)));
 
-  patterns.insert<LinalgTileAndFusePattern<MatmulOp>>(
+  patterns.add<LinalgTileAndFusePattern<MatmulOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64, 16}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({0}),
@@ -52,7 +52,7 @@ static void fillFusionPatterns(MLIRContext *context,
           ArrayRef<Identifier>(),
           Identifier::get("after_lhs_fusion_original", context)));
 
-  patterns.insert<LinalgTileAndFusePattern<MatmulOp>>(
+  patterns.add<LinalgTileAndFusePattern<MatmulOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64, 16}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({1}),
@@ -65,7 +65,7 @@ static void fillFusionPatterns(MLIRContext *context,
           ArrayRef<Identifier>(),
           Identifier::get("after_rhs_fusion_original", context)));
 
-  patterns.insert<LinalgTileAndFusePattern<MatmulOp>>(
+  patterns.add<LinalgTileAndFusePattern<MatmulOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64, 16}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({0, 2}),
@@ -79,7 +79,7 @@ static void fillFusionPatterns(MLIRContext *context,
           ArrayRef<Identifier>(),
           Identifier::get("after_two_operand_fusion_original", context)));
 
-  patterns.insert<LinalgTileAndFusePattern<GenericOp>>(
+  patterns.add<LinalgTileAndFusePattern<GenericOp>>(
       context, dependenceGraph,
       LinalgTilingOptions().setTileSizes({32, 64}).setLoopType(LoopType),
       LinalgFusionOptions().setIndicesToFuse({0, 1}),
@@ -109,7 +109,7 @@ struct TestLinalgFusionTransforms
   void runOnFunction() override {
     MLIRContext *context = &this->getContext();
     FuncOp funcOp = this->getFunction();
-    OwningRewritePatternList fusionPatterns;
+    RewritePatternSet fusionPatterns(context);
     Aliases alias;
     LinalgDependenceGraph dependenceGraph =
         LinalgDependenceGraph::buildDependenceGraph(alias, funcOp);
@@ -179,12 +179,16 @@ static LogicalResult fuseLinalgOpsGreedily(FuncOp f) {
 namespace {
 struct TestLinalgGreedyFusion
     : public PassWrapper<TestLinalgGreedyFusion, FunctionPass> {
+  void getDependentDialects(DialectRegistry &registry) const override {
+    registry.insert<AffineDialect, linalg::LinalgDialect, memref::MemRefDialect,
+                    scf::SCFDialect>();
+  }
   void runOnFunction() override {
     MLIRContext *context = &getContext();
-    OwningRewritePatternList patterns =
+    RewritePatternSet patterns =
         linalg::getLinalgTilingCanonicalizationPatterns(context);
-    patterns.insert<AffineMinSCFCanonicalizationPattern>(context);
-    FrozenRewritePatternList frozenPatterns(std::move(patterns));
+    patterns.add<AffineMinSCFCanonicalizationPattern>(context);
+    FrozenRewritePatternSet frozenPatterns(std::move(patterns));
     while (succeeded(fuseLinalgOpsGreedily(getFunction()))) {
       (void)applyPatternsAndFoldGreedily(getFunction(), frozenPatterns);
       PassManager pm(context);
diff --git a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
index 6cc390f54ca2e303612da8a83780bf5ffad6b8fb..fd8fb3bc6eff2f5c9d68d722a95b30ade0fe1f18 100644
--- a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
@@ -84,44 +84,44 @@ struct TestLinalgTransforms
   Option<bool> testTileAndPadPattern{
       *this, "test-tile-and-pad-pattern",
       llvm::cl::desc("Test tile and pad pattern"), llvm::cl::init(false)};
-  Option<bool> testHoistPadding2Levels{*this, "test-hoist-padding-2-level",
-                                       llvm::cl::desc("Test hoist padding"),
-                                       llvm::cl::init(false)};
+  Option<int> testHoistPadding{*this, "test-hoist-padding",
+                               llvm::cl::desc("Test hoist padding"),
+                               llvm::cl::init(0)};
 };
 } // end anonymous namespace
 
 static void applyPatterns(FuncOp funcOp) {
   MLIRContext *ctx = funcOp.getContext();
-  OwningRewritePatternList patterns;
+  RewritePatternSet patterns(ctx);
 
   //===--------------------------------------------------------------------===//
   // Linalg tiling patterns.
   //===--------------------------------------------------------------------===//
-  patterns.insert<LinalgTilingPattern<MatmulOp>>(
+  patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({2000, 3000, 4000}),
       LinalgTransformationFilter(Identifier::get("MEM", ctx),
                                  Identifier::get("L3", ctx)));
-  patterns.insert<LinalgTilingPattern<MatmulOp>>(
+  patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({200, 300, 400}),
       LinalgTransformationFilter(Identifier::get("L3", ctx),
                                  Identifier::get("L2", ctx)));
-  patterns.insert<LinalgTilingPattern<MatmulOp>>(
+  patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({20, 30, 40}),
       LinalgTransformationFilter(Identifier::get("L2", ctx),
                                  Identifier::get("L1", ctx)));
-  patterns.insert<LinalgTilingPattern<MatmulOp>>(
+  patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({2, 3, 4}),
       LinalgTransformationFilter(Identifier::get("L1", ctx),
                                  Identifier::get("REG", ctx)));
 
-  patterns.insert<LinalgTilingPattern<MatvecOp>>(
+  patterns.add<LinalgTilingPattern<MatvecOp>>(
       ctx,
       LinalgTilingOptions().setTileSizes({5, 6}).setLoopType(
           LinalgTilingLoopType::ParallelLoops),
       LinalgTransformationFilter(ArrayRef<Identifier>{},
                                  Identifier::get("L1", ctx)));
 
-  patterns.insert<LinalgTilingPattern<DotOp>>(
+  patterns.add<LinalgTilingPattern<DotOp>>(
       ctx, LinalgTilingOptions().setTileSizes(8000),
       LinalgTransformationFilter(
           ArrayRef<Identifier>{Identifier::get("MEM", ctx),
@@ -132,31 +132,31 @@ static void applyPatterns(FuncOp funcOp) {
   //===--------------------------------------------------------------------===//
   // Linalg tiling and permutation patterns.
   //===--------------------------------------------------------------------===//
-  patterns.insert<LinalgTilingPattern<MatmulOp>>(
+  patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx,
       LinalgTilingOptions()
           .setTileSizes({2000, 3000, 4000})
           .setInterchange({1, 2, 0}),
       LinalgTransformationFilter(Identifier::get("__with_perm__", ctx),
                                  Identifier::get("L2__with_perm__", ctx)));
-  patterns.insert<LinalgTilingPattern<MatmulOp>>(
+  patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx,
       LinalgTilingOptions()
           .setTileSizes({200, 300, 400})
           .setInterchange({1, 0, 2}),
       LinalgTransformationFilter(Identifier::get("L2__with_perm__", ctx),
                                  Identifier::get("L1__with_perm__", ctx)));
-  patterns.insert<LinalgTilingPattern<MatmulOp>>(
+  patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({20, 30, 40}),
       LinalgTransformationFilter(Identifier::get("L1__with_perm__", ctx),
                                  Identifier::get("REG__with_perm__", ctx)));
 
-  patterns.insert<LinalgTilingPattern<MatvecOp>>(
+  patterns.add<LinalgTilingPattern<MatvecOp>>(
       ctx, LinalgTilingOptions().setTileSizes({5, 6}).setInterchange({1, 0}),
       LinalgTransformationFilter(Identifier::get("__with_perm__", ctx),
                                  Identifier::get("L1__with_perm__", ctx)));
 
-  patterns.insert<LinalgTilingPattern<MatmulOp>>(
+  patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx,
       LinalgTilingOptions()
           .setTileSizes({16, 8, 4})
@@ -169,7 +169,7 @@ static void applyPatterns(FuncOp funcOp) {
   //===--------------------------------------------------------------------===//
   // Linalg to loops patterns.
   //===--------------------------------------------------------------------===//
-  patterns.insert<LinalgLoweringPattern<DotOp>>(
+  patterns.add<LinalgLoweringPattern<DotOp>>(
       ctx,
       /*loweringType=*/LinalgLoweringType::Loops,
       LinalgTransformationFilter(Identifier::get("REG", ctx)));
@@ -182,19 +182,19 @@ static void applyPatterns(FuncOp funcOp) {
   //===--------------------------------------------------------------------===//
   // Linalg to vector contraction patterns.
   //===--------------------------------------------------------------------===//
-  patterns.insert<LinalgVectorizationPattern>(
-      LinalgTransformationFilter(Identifier::get("VECTORIZE", ctx))
-          .addOpFilter<MatmulOp, FillOp, CopyOp, GenericOp>());
+  patterns.add<LinalgVectorizationPattern>(
+      ctx, LinalgTransformationFilter(Identifier::get("VECTORIZE", ctx))
+               .addOpFilter<MatmulOp, FillOp, CopyOp, GenericOp>());
 
   //===--------------------------------------------------------------------===//
   // Linalg generic permutation patterns.
   //===--------------------------------------------------------------------===//
-  patterns.insert<LinalgInterchangePattern<GenericOp>>(
+  patterns.add<LinalgInterchangePattern<GenericOp>>(
       ctx,
       /*interchangeVector=*/ArrayRef<unsigned>{1, 2, 0},
       LinalgTransformationFilter(ArrayRef<Identifier>{},
                                  Identifier::get("PERMUTED", ctx)));
-  patterns.insert<LinalgInterchangePattern<IndexedGenericOp>>(
+  patterns.add<LinalgInterchangePattern<IndexedGenericOp>>(
       ctx,
       /*interchangeVector=*/ArrayRef<unsigned>{1, 2, 0},
       LinalgTransformationFilter(ArrayRef<Identifier>{},
@@ -203,11 +203,11 @@ static void applyPatterns(FuncOp funcOp) {
   //===--------------------------------------------------------------------===//
   // Linalg subview operands promotion.
   //===--------------------------------------------------------------------===//
-  patterns.insert<LinalgPromotionPattern<MatmulOp>>(
+  patterns.add<LinalgPromotionPattern<MatmulOp>>(
       ctx, LinalgPromotionOptions().setUseFullTileBuffersByDefault(true),
       LinalgTransformationFilter(Identifier::get("_promote_views_", ctx),
                                  Identifier::get("_views_promoted_", ctx)));
-  patterns.insert<LinalgPromotionPattern<MatmulOp>>(
+  patterns.add<LinalgPromotionPattern<MatmulOp>>(
       ctx,
       LinalgPromotionOptions()
           .setOperandsToPromote({0})
@@ -215,7 +215,7 @@ static void applyPatterns(FuncOp funcOp) {
       LinalgTransformationFilter(
           Identifier::get("_promote_first_view_", ctx),
           Identifier::get("_first_view_promoted_", ctx)));
-  patterns.insert<LinalgPromotionPattern<FillOp>>(
+  patterns.add<LinalgPromotionPattern<FillOp>>(
       ctx,
       LinalgPromotionOptions()
           .setOperandsToPromote({0})
@@ -235,26 +235,31 @@ static void applyPatterns(FuncOp funcOp) {
 
 static void fillL1TilingAndMatmulToVectorPatterns(
     FuncOp funcOp, StringRef startMarker,
-    SmallVectorImpl<OwningRewritePatternList> &patternsVector) {
+    SmallVectorImpl<RewritePatternSet> &patternsVector) {
   MLIRContext *ctx = funcOp.getContext();
-  patternsVector.emplace_back(std::make_unique<LinalgTilingPattern<MatmulOp>>(
-      ctx,
-      LinalgTilingOptions().setTileSizes({8, 12, 16}).setInterchange({1, 0, 2}),
-      LinalgTransformationFilter(Identifier::get(startMarker, ctx),
-                                 Identifier::get("L1", ctx))));
+  patternsVector.emplace_back(
+      ctx, std::make_unique<LinalgTilingPattern<MatmulOp>>(
+               ctx,
+               LinalgTilingOptions()
+                   .setTileSizes({8, 12, 16})
+                   .setInterchange({1, 0, 2}),
+               LinalgTransformationFilter(Identifier::get(startMarker, ctx),
+                                          Identifier::get("L1", ctx))));
 
   patternsVector.emplace_back(
+      ctx,
       std::make_unique<LinalgPromotionPattern<MatmulOp>>(
           ctx, LinalgPromotionOptions().setUseFullTileBuffersByDefault(true),
           LinalgTransformationFilter(Identifier::get("L1", ctx),
                                      Identifier::get("VEC", ctx))));
 
-  patternsVector.emplace_back(std::make_unique<LinalgVectorizationPattern>(
-      MatmulOp::getOperationName(), ctx, LinalgVectorizationOptions(),
-      LinalgTransformationFilter(Identifier::get("VEC", ctx))));
-  patternsVector.back().insert<LinalgVectorizationPattern>(
-      LinalgTransformationFilter().addFilter(
-          [](Operation *op) { return success(isa<FillOp, CopyOp>(op)); }));
+  patternsVector.emplace_back(
+      ctx, std::make_unique<LinalgVectorizationPattern>(
+               MatmulOp::getOperationName(), ctx, LinalgVectorizationOptions(),
+               LinalgTransformationFilter(Identifier::get("VEC", ctx))));
+  patternsVector.back().add<LinalgVectorizationPattern>(
+      ctx, LinalgTransformationFilter().addFilter(
+               [](Operation *op) { return success(isa<FillOp, CopyOp>(op)); }));
 }
 
 //===----------------------------------------------------------------------===//
@@ -296,12 +301,12 @@ static LogicalResult copyCallBackFn(OpBuilder &b, Value src, Value dst,
 }
 
 static void fillPromotionCallBackPatterns(MLIRContext *ctx,
-                                          OwningRewritePatternList &patterns) {
-  patterns.insert<LinalgTilingPattern<MatmulOp>>(
+                                          RewritePatternSet &patterns) {
+  patterns.add<LinalgTilingPattern<MatmulOp>>(
       ctx, LinalgTilingOptions().setTileSizes({16, 16, 16}),
       LinalgTransformationFilter(Identifier::get("START", ctx),
                                  Identifier::get("PROMOTE", ctx)));
-  patterns.insert<LinalgPromotionPattern<MatmulOp>>(
+  patterns.add<LinalgPromotionPattern<MatmulOp>>(
       ctx,
       LinalgPromotionOptions()
           .setOperandsToPromote({0, 2})
@@ -330,14 +335,14 @@ getGpuProcIds(OpBuilder &b, Location loc, ArrayRef<Range> parallelLoopRanges) {
 }
 
 static void fillTileAndDistributePatterns(MLIRContext *context,
-                                          OwningRewritePatternList &patterns) {
+                                          RewritePatternSet &patterns) {
   {
     LinalgLoopDistributionOptions cyclicNprocsEqNiters;
     cyclicNprocsEqNiters.distributionMethod.resize(
         2, DistributionMethod::CyclicNumProcsEqNumIters);
     cyclicNprocsEqNiters.procInfo =
         getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
-    patterns.insert<LinalgTilingPattern<MatmulOp>>(
+    patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
@@ -354,7 +359,7 @@ static void fillTileAndDistributePatterns(MLIRContext *context,
         2, DistributionMethod::CyclicNumProcsGeNumIters);
     cyclicNprocsGeNiters.procInfo =
         getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
-    patterns.insert<LinalgTilingPattern<MatmulOp>>(
+    patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
@@ -371,7 +376,7 @@ static void fillTileAndDistributePatterns(MLIRContext *context,
                                                   DistributionMethod::Cyclic);
     cyclicNprocsDefault.procInfo =
         getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
-    patterns.insert<LinalgTilingPattern<MatmulOp>>(
+    patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
@@ -388,7 +393,7 @@ static void fillTileAndDistributePatterns(MLIRContext *context,
         DistributionMethod::CyclicNumProcsEqNumIters,
         DistributionMethod::CyclicNumProcsGeNumIters};
     cyclicNprocsMixed1.procInfo = getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
-    patterns.insert<LinalgTilingPattern<MatmulOp>>(
+    patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
@@ -405,7 +410,7 @@ static void fillTileAndDistributePatterns(MLIRContext *context,
         DistributionMethod::CyclicNumProcsGeNumIters,
         DistributionMethod::Cyclic};
     cyclicNprocsMixed2.procInfo = getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
-    patterns.insert<LinalgTilingPattern<MatmulOp>>(
+    patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
@@ -423,7 +428,7 @@ static void fillTileAndDistributePatterns(MLIRContext *context,
         DistributionMethod::CyclicNumProcsEqNumIters};
     cyclicNprocsMixed3.procInfo = getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
 
-    patterns.insert<LinalgTilingPattern<MatmulOp>>(
+    patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
@@ -440,7 +445,7 @@ static void fillTileAndDistributePatterns(MLIRContext *context,
                                                    DistributionMethod::Cyclic);
     cyclicNprocsEqNiters.procInfo =
         getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
-    patterns.insert<LinalgTilingPattern<MatmulOp>>(
+    patterns.add<LinalgTilingPattern<MatmulOp>>(
         context,
         LinalgTilingOptions()
             .setTileSizes({8, 8, 4})
@@ -457,49 +462,51 @@ applyMatmulToVectorPatterns(FuncOp funcOp,
                             bool testMatmulToVectorPatterns1dTiling,
                             bool testMatmulToVectorPatterns2dTiling) {
   MLIRContext *ctx = funcOp.getContext();
-  SmallVector<OwningRewritePatternList, 4> stage1Patterns;
+  SmallVector<RewritePatternSet, 4> stage1Patterns;
   if (testMatmulToVectorPatterns1dTiling) {
     fillL1TilingAndMatmulToVectorPatterns(funcOp, Identifier::get("START", ctx),
                                           stage1Patterns);
   } else if (testMatmulToVectorPatterns2dTiling) {
-    stage1Patterns.emplace_back(std::make_unique<LinalgTilingPattern<MatmulOp>>(
-        ctx,
-        LinalgTilingOptions()
-            .setTileSizes({768, 264, 768})
-            .setInterchange({1, 2, 0}),
-        LinalgTransformationFilter(Identifier::get("START", ctx),
-                                   Identifier::get("L2", ctx))));
+    stage1Patterns.emplace_back(
+        ctx, std::make_unique<LinalgTilingPattern<MatmulOp>>(
+                 ctx,
+                 LinalgTilingOptions()
+                     .setTileSizes({768, 264, 768})
+                     .setInterchange({1, 2, 0}),
+                 LinalgTransformationFilter(Identifier::get("START", ctx),
+                                            Identifier::get("L2", ctx))));
     fillL1TilingAndMatmulToVectorPatterns(funcOp, Identifier::get("L2", ctx),
                                           stage1Patterns);
   }
-  SmallVector<FrozenRewritePatternList, 4> frozenStage1Patterns;
+  SmallVector<FrozenRewritePatternSet, 4> frozenStage1Patterns;
   llvm::move(stage1Patterns, std::back_inserter(frozenStage1Patterns));
-  FrozenRewritePatternList stage2Patterns =
+  FrozenRewritePatternSet stage2Patterns =
       getLinalgTilingCanonicalizationPatterns(ctx);
   (void)applyStagedPatterns(funcOp, frozenStage1Patterns,
                             std::move(stage2Patterns));
 }
 
 static void applyVectorTransferForwardingPatterns(FuncOp funcOp) {
-  OwningRewritePatternList forwardPattern;
-  forwardPattern.insert<LinalgCopyVTRForwardingPattern>(funcOp.getContext());
-  forwardPattern.insert<LinalgCopyVTWForwardingPattern>(funcOp.getContext());
+  RewritePatternSet forwardPattern(funcOp.getContext());
+  forwardPattern.add<LinalgCopyVTRForwardingPattern>(funcOp.getContext());
+  forwardPattern.add<LinalgCopyVTWForwardingPattern>(funcOp.getContext());
   (void)applyPatternsAndFoldGreedily(funcOp, std::move(forwardPattern));
 }
 
 static void applyLinalgToVectorPatterns(FuncOp funcOp) {
-  OwningRewritePatternList patterns;
-  patterns.insert<LinalgVectorizationPattern>(
+  RewritePatternSet patterns(funcOp.getContext());
+  patterns.add<LinalgVectorizationPattern>(
+      funcOp.getContext(),
       LinalgTransformationFilter()
           .addOpFilter<ContractionOpInterface, FillOp, CopyOp, GenericOp>());
-  patterns.insert<PadTensorOpVectorizationPattern>(funcOp.getContext());
+  patterns.add<PadTensorOpVectorizationPattern>(funcOp.getContext());
   (void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns));
 }
 
 static void applyAffineMinSCFCanonicalizationPatterns(FuncOp funcOp) {
-  OwningRewritePatternList foldPattern;
-  foldPattern.insert<AffineMinSCFCanonicalizationPattern>(funcOp.getContext());
-  FrozenRewritePatternList frozenPatterns(std::move(foldPattern));
+  RewritePatternSet foldPattern(funcOp.getContext());
+  foldPattern.add<AffineMinSCFCanonicalizationPattern>(funcOp.getContext());
+  FrozenRewritePatternSet frozenPatterns(std::move(foldPattern));
 
   // Explicitly walk and apply the pattern locally to avoid more general folding
   // on the rest of the IR.
@@ -517,12 +524,12 @@ static Value getNeutralOfLinalgOp(OpBuilder &b, OpOperand &op) {
 
 static void applyTileAndPadPattern(FuncOp funcOp) {
   MLIRContext *context = funcOp.getContext();
-  OwningRewritePatternList tilingPattern;
+  RewritePatternSet tilingPattern(context);
   auto linalgTilingOptions =
       linalg::LinalgTilingOptions()
           .setTileSizes({2, 3, 4})
           .setPaddingValueComputationFunction(getNeutralOfLinalgOp);
-  tilingPattern.insert<linalg::LinalgTilingPattern<linalg::MatmulI8I8I32Op>>(
+  tilingPattern.add<linalg::LinalgTilingPattern<linalg::MatmulI8I8I32Op>>(
       context, linalgTilingOptions,
       linalg::LinalgTransformationFilter(
           Identifier::get("tile-and-pad", context)));
@@ -539,13 +546,13 @@ void TestLinalgTransforms::runOnFunction() {
   std::unique_ptr<void, decltype(lambda)> cleanupGuard{(void *)1, lambda};
 
   if (testPromotionOptions) {
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(&getContext());
     fillPromotionCallBackPatterns(&getContext(), patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
     return;
   }
   if (testTileAndDistributionOptions) {
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(&getContext());
     fillTileAndDistributePatterns(&getContext(), patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
     return;
@@ -564,9 +571,9 @@ void TestLinalgTransforms::runOnFunction() {
     return applyAffineMinSCFCanonicalizationPatterns(getFunction());
   if (testTileAndPadPattern)
     return applyTileAndPadPattern(getFunction());
-  if (testHoistPadding2Levels) {
-    getFunction().walk([](linalg::PadTensorOp padTensorOp) {
-      (void)linalg::hoistPaddingOnTensors(padTensorOp, 2);
+  if (testHoistPadding) {
+    getFunction().walk([&](linalg::PadTensorOp padTensorOp) {
+      (void)linalg::hoistPaddingOnTensors(padTensorOp, testHoistPadding);
     });
   }
 }
diff --git a/mlir/test/lib/Transforms/TestPolynomialApproximation.cpp b/mlir/test/lib/Transforms/TestPolynomialApproximation.cpp
index b4b8ac56497aa5749390c83002d1c6625e3c9b84..fed76a0de5477b345c0150dff84a5ec91515b6c5 100644
--- a/mlir/test/lib/Transforms/TestPolynomialApproximation.cpp
+++ b/mlir/test/lib/Transforms/TestPolynomialApproximation.cpp
@@ -32,8 +32,8 @@ struct TestMathPolynomialApproximationPass
 } // end anonymous namespace
 
 void TestMathPolynomialApproximationPass::runOnFunction() {
-  OwningRewritePatternList patterns;
-  populateMathPolynomialApproximationPatterns(patterns, &getContext());
+  RewritePatternSet patterns(&getContext());
+  populateMathPolynomialApproximationPatterns(patterns);
   (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
 }
 
diff --git a/mlir/test/lib/Transforms/TestSparsification.cpp b/mlir/test/lib/Transforms/TestSparsification.cpp
index a76b8664f76c9e3bd691519f1eaf420081dab6e0..22900f7edd1b02dd681bc855f160243d248e3c1c 100644
--- a/mlir/test/lib/Transforms/TestSparsification.cpp
+++ b/mlir/test/lib/Transforms/TestSparsification.cpp
@@ -101,25 +101,25 @@ struct TestSparsification
   /// Runs the test on a function.
   void runOnOperation() override {
     auto *ctx = &getContext();
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(ctx);
     // Translate strategy flags to strategy options.
     linalg::SparsificationOptions options(parallelOption(), vectorOption(),
                                           vectorLength, typeOption(ptrType),
                                           typeOption(indType), fastOutput);
     // Apply rewriting.
-    linalg::populateSparsificationPatterns(ctx, patterns, options);
-    vector::populateVectorToVectorCanonicalizationPatterns(patterns, ctx);
+    linalg::populateSparsificationPatterns(patterns, options);
+    vector::populateVectorToVectorCanonicalizationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
     // Lower sparse primitives to calls into runtime support library.
     if (lower) {
-      OwningRewritePatternList conversionPatterns;
+      RewritePatternSet conversionPatterns(ctx);
       ConversionTarget target(*ctx);
       target.addIllegalOp<linalg::SparseTensorFromPointerOp,
                           linalg::SparseTensorToPointersMemRefOp,
                           linalg::SparseTensorToIndicesMemRefOp,
                           linalg::SparseTensorToValuesMemRefOp>();
       target.addLegalOp<CallOp>();
-      linalg::populateSparsificationConversionPatterns(ctx, conversionPatterns);
+      linalg::populateSparsificationConversionPatterns(conversionPatterns);
       if (failed(applyPartialConversion(getOperation(), target,
                                         std::move(conversionPatterns))))
         signalPassFailure();
diff --git a/mlir/test/lib/Transforms/TestVectorTransforms.cpp b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
index f11ee1347f5772f792d8cf867c387bf873f63943..76c5c7c0e2a46e433983a9e4b8476f7028b1446d 100644
--- a/mlir/test/lib/Transforms/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
@@ -36,19 +36,19 @@ struct TestVectorToVectorConversion
                       llvm::cl::init(false)};
 
   void runOnFunction() override {
-    OwningRewritePatternList patterns;
     auto *ctx = &getContext();
+    RewritePatternSet patterns(ctx);
     if (unroll) {
-      patterns.insert<UnrollVectorPattern>(
+      patterns.add<UnrollVectorPattern>(
           ctx,
           UnrollVectorOptions().setNativeShapeFn(getShape).setFilterConstraint(
               filter));
     }
-    populateVectorToVectorCanonicalizationPatterns(patterns, ctx);
-    populateVectorToVectorTransformationPatterns(patterns, ctx);
-    populateBubbleVectorBitCastOpPatterns(patterns, ctx);
-    populateCastAwayVectorLeadingOneDimPatterns(patterns, ctx);
-    populateSplitVectorTransferPatterns(patterns, ctx);
+    populateVectorToVectorCanonicalizationPatterns(patterns);
+    populateVectorToVectorTransformationPatterns(patterns);
+    populateBubbleVectorBitCastOpPatterns(patterns);
+    populateCastAwayVectorLeadingOneDimPatterns(patterns);
+    populateSplitVectorTransferPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 
@@ -70,8 +70,8 @@ private:
 struct TestVectorSlicesConversion
     : public PassWrapper<TestVectorSlicesConversion, FunctionPass> {
   void runOnFunction() override {
-    OwningRewritePatternList patterns;
-    populateVectorSlicesLoweringPatterns(patterns, &getContext());
+    RewritePatternSet patterns(&getContext());
+    populateVectorSlicesLoweringPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
@@ -101,14 +101,14 @@ struct TestVectorContractionConversion
       llvm::cl::init(false)};
 
   void runOnFunction() override {
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(&getContext());
 
     // Test on one pattern in isolation.
     if (lowerToOuterProduct) {
       VectorContractLowering lowering = VectorContractLowering::OuterProduct;
       VectorTransformsOptions options{lowering};
-      patterns.insert<ContractionOpToOuterProductOpLowering>(options,
-                                                             &getContext());
+      patterns.add<ContractionOpToOuterProductOpLowering>(options,
+                                                          &getContext());
       (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
       return;
     }
@@ -117,7 +117,7 @@ struct TestVectorContractionConversion
     if (lowerToFilterOuterProduct) {
       VectorContractLowering lowering = VectorContractLowering::OuterProduct;
       VectorTransformsOptions options{lowering};
-      patterns.insert<ContractionOpToOuterProductOpLowering>(
+      patterns.add<ContractionOpToOuterProductOpLowering>(
           options, &getContext(), [](vector::ContractionOp op) {
             // Only lowers vector.contract where the lhs as a type vector<MxNx?>
             // where M is not 4.
@@ -138,7 +138,7 @@ struct TestVectorContractionConversion
     if (lowerToFlatTranspose)
       transposeLowering = VectorTransposeLowering::Flat;
     VectorTransformsOptions options{contractLowering, transposeLowering};
-    populateVectorContractLoweringPatterns(patterns, &getContext(), options);
+    populateVectorContractLoweringPatterns(patterns, options);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
@@ -149,8 +149,8 @@ struct TestVectorUnrollingPatterns
   TestVectorUnrollingPatterns(const TestVectorUnrollingPatterns &pass) {}
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
-    OwningRewritePatternList patterns;
-    patterns.insert<UnrollVectorPattern>(
+    RewritePatternSet patterns(ctx);
+    patterns.add<UnrollVectorPattern>(
         ctx, UnrollVectorOptions()
                  .setNativeShape(ArrayRef<int64_t>{2, 2})
                  .setFilterConstraint([](Operation *op) {
@@ -171,22 +171,22 @@ struct TestVectorUnrollingPatterns
         }
         return nativeShape;
       };
-      patterns.insert<UnrollVectorPattern>(
+      patterns.add<UnrollVectorPattern>(
           ctx, UnrollVectorOptions()
                    .setNativeShapeFn(nativeShapeFn)
                    .setFilterConstraint([](Operation *op) {
                      return success(isa<ContractionOp>(op));
                    }));
     } else {
-      patterns.insert<UnrollVectorPattern>(
+      patterns.add<UnrollVectorPattern>(
           ctx, UnrollVectorOptions()
                    .setNativeShape(ArrayRef<int64_t>{2, 2, 2})
                    .setFilterConstraint([](Operation *op) {
                      return success(isa<ContractionOp>(op));
                    }));
     }
-    populateVectorToVectorCanonicalizationPatterns(patterns, ctx);
-    populateVectorToVectorTransformationPatterns(patterns, ctx);
+    populateVectorToVectorCanonicalizationPatterns(patterns);
+    populateVectorToVectorTransformationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 
@@ -210,7 +210,7 @@ struct TestVectorDistributePatterns
 
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(ctx);
     FuncOp func = getFunction();
     func.walk([&](AddFOp op) {
       OpBuilder builder(op);
@@ -240,8 +240,8 @@ struct TestVectorDistributePatterns
         }
       }
     });
-    patterns.insert<PointwiseExtractPattern>(ctx);
-    populateVectorToVectorTransformationPatterns(patterns, ctx);
+    patterns.add<PointwiseExtractPattern>(ctx);
+    populateVectorToVectorTransformationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
@@ -260,7 +260,7 @@ struct TestVectorToLoopPatterns
       llvm::cl::init(32)};
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(ctx);
     FuncOp func = getFunction();
     func.walk([&](AddFOp op) {
       // Check that the operation type can be broken down into a loop.
@@ -300,8 +300,8 @@ struct TestVectorToLoopPatterns
       }
       return mlir::WalkResult::interrupt();
     });
-    patterns.insert<PointwiseExtractPattern>(ctx);
-    populateVectorToVectorTransformationPatterns(patterns, ctx);
+    patterns.add<PointwiseExtractPattern>(ctx);
+    populateVectorToVectorTransformationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
@@ -313,8 +313,8 @@ struct TestVectorTransferUnrollingPatterns
   }
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
-    OwningRewritePatternList patterns;
-    patterns.insert<UnrollVectorPattern>(
+    RewritePatternSet patterns(ctx);
+    patterns.add<UnrollVectorPattern>(
         ctx,
         UnrollVectorOptions()
             .setNativeShape(ArrayRef<int64_t>{2, 2})
@@ -322,8 +322,8 @@ struct TestVectorTransferUnrollingPatterns
               return success(
                   isa<vector::TransferReadOp, vector::TransferWriteOp>(op));
             }));
-    populateVectorToVectorCanonicalizationPatterns(patterns, ctx);
-    populateVectorToVectorTransformationPatterns(patterns, ctx);
+    populateVectorToVectorCanonicalizationPatterns(patterns);
+    populateVectorToVectorTransformationPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
@@ -347,13 +347,13 @@ struct TestVectorTransferFullPartialSplitPatterns
       llvm::cl::init(false)};
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
-    OwningRewritePatternList patterns;
+    RewritePatternSet patterns(ctx);
     VectorTransformsOptions options;
     if (useLinalgOps)
       options.setVectorTransferSplit(VectorTransferSplit::LinalgCopy);
     else
       options.setVectorTransferSplit(VectorTransferSplit::VectorTransfer);
-    patterns.insert<VectorTransferFullPartialRewriter>(ctx, options);
+    patterns.add<VectorTransferFullPartialRewriter>(ctx, options);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
@@ -369,8 +369,8 @@ struct TestVectorTransferLoweringPatterns
     registry.insert<memref::MemRefDialect>();
   }
   void runOnFunction() override {
-    OwningRewritePatternList patterns;
-    populateVectorTransferLoweringPatterns(patterns, &getContext());
+    RewritePatternSet patterns(&getContext());
+    populateVectorTransferLoweringPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
 };
diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py
index 4ba36202578d46eb9927db5670cae0e35812cccc..199d7222e1cf58e3289cc898ff33598f118fa96f 100644
--- a/mlir/test/lit.cfg.py
+++ b/mlir/test/lit.cfg.py
@@ -77,7 +77,6 @@ tools.extend([
     ToolSubst('toy-ch5', unresolved='ignore'),
     ToolSubst('%linalg_test_lib_dir', config.linalg_test_lib_dir, unresolved='ignore'),
     ToolSubst('%mlir_runner_utils_dir', config.mlir_runner_utils_dir, unresolved='ignore'),
-    ToolSubst('%rocm_wrapper_library_dir', config.rocm_wrapper_library_dir, unresolved='ignore'),
     ToolSubst('%spirv_wrapper_library_dir', config.spirv_wrapper_library_dir, unresolved='ignore'),
     ToolSubst('%vulkan_wrapper_library_dir', config.vulkan_wrapper_library_dir, unresolved='ignore'),
     ToolSubst('%mlir_integration_test_dir', config.mlir_integration_test_dir, unresolved='ignore'),
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index 0015c1369d7a1c50bf728d5a52bac2b173402779..dbc8460df5767f8e032524a6ca053a934230e861 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -39,7 +39,6 @@ config.build_examples = @LLVM_BUILD_EXAMPLES@
 config.run_cuda_tests = @MLIR_CUDA_CONVERSIONS_ENABLED@
 config.enable_cuda_runner = @MLIR_CUDA_RUNNER_ENABLED@
 config.run_rocm_tests = @MLIR_ROCM_CONVERSIONS_ENABLED@
-config.rocm_wrapper_library_dir = "@MLIR_ROCM_WRAPPER_LIBRARY_DIR@"
 config.enable_rocm_runner = @MLIR_ROCM_RUNNER_ENABLED@
 config.spirv_wrapper_library_dir = "@MLIR_SPIRV_WRAPPER_LIBRARY_DIR@"
 config.enable_spirv_cpu_runner = @MLIR_SPIRV_CPU_RUNNER_ENABLED@
diff --git a/mlir/test/mlir-reduce/dce-test.mlir b/mlir/test/mlir-reduce/dce-test.mlir
index f98f3cd6b1a8f33240cb808483070f9944e0bafc..21605689d647c8c9e0e38fc6b10dfc50f057c449 100644
--- a/mlir/test/mlir-reduce/dce-test.mlir
+++ b/mlir/test/mlir-reduce/dce-test.mlir
@@ -3,14 +3,14 @@
 // This input should be reduced by the pass pipeline so that only
 // the @simple1 function remains as the other functions should be
 // removed by the dead code elimination pass.
-// CHECK-LABEL: func @simple1(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 
-// CHECK-NOT: func @dead_nested_function
+// CHECK-NOT: func private @dead_private_function
 func private @dead_private_function()
 
-// CHECK-NOT: func @dead_nested_function
+// CHECK-NOT: func nested @dead_nested_function
 func nested @dead_nested_function()
 
+// CHECK-LABEL: func @simple1(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 func @simple1(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
   "test.crashOp" () : () -> ()
   return
diff --git a/mlir/test/mlir-tblgen/attrdefs.td b/mlir/test/mlir-tblgen/attrdefs.td
index 34cdcab7246e6d67cefc18c3ed3fee190c72152c..fc95fba3c91cdb0319c97d2280c28f6214760f7c 100644
--- a/mlir/test/mlir-tblgen/attrdefs.td
+++ b/mlir/test/mlir-tblgen/attrdefs.td
@@ -53,7 +53,7 @@ def B_CompoundAttrA : TestAttr<"CompoundA"> {
       ins
       "int":$widthOfSomething,
       "::mlir::test::SimpleTypeA": $exampleTdType,
-      "SomeCppStruct": $exampleCppType,
+      APFloatParameter<"">: $apFloat,
       ArrayRefParameter<"int", "Matrix dimensions">:$dims,
       AttributeSelfTypeParameter<"">:$inner
   );
@@ -61,8 +61,8 @@ def B_CompoundAttrA : TestAttr<"CompoundA"> {
   let genVerifyDecl = 1;
 
 // DECL-LABEL: class CompoundAAttr : public ::mlir::Attribute
-// DECL: static CompoundAAttr getChecked(llvm::function_ref<::mlir::InFlightDiagnostic()> emitError, ::mlir::MLIRContext *context, int widthOfSomething, ::mlir::test::SimpleTypeA exampleTdType, SomeCppStruct exampleCppType, ::llvm::ArrayRef<int> dims, ::mlir::Type inner);
-// DECL: static ::mlir::LogicalResult verify(::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError, int widthOfSomething, ::mlir::test::SimpleTypeA exampleTdType, SomeCppStruct exampleCppType, ::llvm::ArrayRef<int> dims, ::mlir::Type inner);
+// DECL: static CompoundAAttr getChecked(llvm::function_ref<::mlir::InFlightDiagnostic()> emitError, ::mlir::MLIRContext *context, int widthOfSomething, ::mlir::test::SimpleTypeA exampleTdType, ::llvm::APFloat apFloat, ::llvm::ArrayRef<int> dims, ::mlir::Type inner);
+// DECL: static ::mlir::LogicalResult verify(::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError, int widthOfSomething, ::mlir::test::SimpleTypeA exampleTdType, ::llvm::APFloat apFloat, ::llvm::ArrayRef<int> dims, ::mlir::Type inner);
 // DECL: static constexpr ::llvm::StringLiteral getMnemonic() {
 // DECL:   return ::llvm::StringLiteral("cmpnd_a");
 // DECL: }
@@ -71,7 +71,7 @@ def B_CompoundAttrA : TestAttr<"CompoundA"> {
 // DECL: void print(::mlir::DialectAsmPrinter &printer) const;
 // DECL: int getWidthOfSomething() const;
 // DECL: ::mlir::test::SimpleTypeA getExampleTdType() const;
-// DECL: SomeCppStruct getExampleCppType() const;
+// DECL: ::llvm::APFloat getApFloat() const;
 
 // Check that AttributeSelfTypeParameter is handled properly.
 // DEF-LABEL: struct CompoundAAttrStorage
@@ -79,13 +79,23 @@ def B_CompoundAttrA : TestAttr<"CompoundA"> {
 // DEF-NEXT: : ::mlir::AttributeStorage(inner),
 
 // DEF: bool operator==(const KeyTy &key) const {
-// DEF-NEXT: return key == KeyTy(widthOfSomething, exampleTdType, exampleCppType, dims, getType());
+// DEF-NEXT: if (!(widthOfSomething == std::get<0>(key)))
+// DEF-NEXT:   return false;
+// DEF-NEXT: if (!(exampleTdType == std::get<1>(key)))
+// DEF-NEXT:   return false;
+// DEF-NEXT: if (!(apFloat.bitwiseIsEqual(std::get<2>(key))))
+// DEF-NEXT:   return false;
+// DEF-NEXT: if (!(dims == std::get<3>(key)))
+// DEF-NEXT:   return false;
+// DEF-NEXT: if (!(getType() == std::get<4>(key)))
+// DEF-NEXT:   return false;
+// DEF-NEXT: return true;
 
 // DEF: static CompoundAAttrStorage *construct
 // DEF: return new (allocator.allocate<CompoundAAttrStorage>())
-// DEF-NEXT: CompoundAAttrStorage(widthOfSomething, exampleTdType, exampleCppType, dims, inner);
+// DEF-NEXT: CompoundAAttrStorage(widthOfSomething, exampleTdType, apFloat, dims, inner);
 
-// DEF: ::mlir::Type CompoundAAttr::getInner() const { return getImpl()->getType(); }
+// DEF: ::mlir::Type CompoundAAttr::getInner() const { return getImpl()->getType().cast<::mlir::Type>(); }
 }
 
 def C_IndexAttr : TestAttr<"Index"> {
diff --git a/mlir/test/mlir-tblgen/op-attribute.td b/mlir/test/mlir-tblgen/op-attribute.td
index 725afd9bc1aa36ead0329520588f739af4d0c761..987f417d867ce02b714eee5cb27980b295a2127d 100644
--- a/mlir/test/mlir-tblgen/op-attribute.td
+++ b/mlir/test/mlir-tblgen/op-attribute.td
@@ -136,7 +136,7 @@ def BOp : NS_Op<"b_op", []> {
 // DEF: if (!((tblgen_function_attr.isa<::mlir::FlatSymbolRefAttr>())))
 // DEF: if (!(((tblgen_some_type_attr.isa<::mlir::TypeAttr>())) && ((tblgen_some_type_attr.cast<::mlir::TypeAttr>().getValue().isa<SomeType>()))))
 // DEF: if (!((tblgen_array_attr.isa<::mlir::ArrayAttr>())))
-// DEF: if (!(((tblgen_some_attr_array.isa<::mlir::ArrayAttr>())) && (::llvm::all_of(tblgen_some_attr_array.cast<::mlir::ArrayAttr>(), [](::mlir::Attribute attr) { return (some-condition); }))))
+// DEF: if (!(((tblgen_some_attr_array.isa<::mlir::ArrayAttr>())) && (::llvm::all_of(tblgen_some_attr_array.cast<::mlir::ArrayAttr>(), [&](::mlir::Attribute attr) { return (some-condition); }))))
 // DEF: if (!(((tblgen_type_attr.isa<::mlir::TypeAttr>())) && ((tblgen_type_attr.cast<::mlir::TypeAttr>().getValue().isa<::mlir::Type>()))))
 
 // Test common attribute kind getters' return types
diff --git a/mlir/test/mlir-tblgen/op-decl-and-defs.td b/mlir/test/mlir-tblgen/op-decl-and-defs.td
index 5115cfed7db8a2805547d6eb4eff98d0f456c6fd..4fb9ecb39730a0151e2ce5525c0dd71458ffe6ac 100644
--- a/mlir/test/mlir-tblgen/op-decl-and-defs.td
+++ b/mlir/test/mlir-tblgen/op-decl-and-defs.td
@@ -96,7 +96,7 @@ def NS_AOp : NS_Op<"a_op", [IsolatedFromAbove, IsolatedFromAbove]> {
 // CHECK:   static ::mlir::ParseResult parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result);
 // CHECK:   void print(::mlir::OpAsmPrinter &p);
 // CHECK:   ::mlir::LogicalResult verify();
-// CHECK:   static void getCanonicalizationPatterns(::mlir::OwningRewritePatternList &results, ::mlir::MLIRContext *context);
+// CHECK:   static void getCanonicalizationPatterns(::mlir::RewritePatternSet &results, ::mlir::MLIRContext *context);
 // CHECK:   ::mlir::LogicalResult fold(::llvm::ArrayRef<::mlir::Attribute> operands, ::llvm::SmallVectorImpl<::mlir::OpFoldResult> &results);
 // CHECK:   // Display a graph for debugging purposes.
 // CHECK:   void displayGraph();
diff --git a/mlir/test/mlir-tblgen/op-format-spec.td b/mlir/test/mlir-tblgen/op-format-spec.td
index 4f5ca63c4e72a39e86fed6978698efba27784013..8c6bb09f34a37ee502df1c99d3f6024b0cadf042 100644
--- a/mlir/test/mlir-tblgen/op-format-spec.td
+++ b/mlir/test/mlir-tblgen/op-format-spec.td
@@ -390,6 +390,18 @@ def OptionalInvalidL : TestFormat_Op<[{
 def OptionalInvalidM : TestFormat_Op<[{
   (` `^)?
 }]>, Arguments<(ins)>;
+// CHECK: error: expected '(' to start else branch of optional group
+def OptionalInvalidN : TestFormat_Op<[{
+  ($arg^):
+}]>, Arguments<(ins Variadic<I64>:$arg)>;
+// CHECK: error: expected directive, literal, variable, or optional group
+def OptionalInvalidO : TestFormat_Op<[{
+  ($arg^):(`test`
+}]>, Arguments<(ins Variadic<I64>:$arg)>;
+// CHECK: error: expected '?' after optional group
+def OptionalInvalidP : TestFormat_Op<[{
+  ($arg^):(`test`)
+}]>, Arguments<(ins Variadic<I64>:$arg)>;
 
 // CHECK-NOT: error
 def OptionalValidA : TestFormat_Op<[{
diff --git a/mlir/test/mlir-tblgen/op-format.mlir b/mlir/test/mlir-tblgen/op-format.mlir
index 8043786faf08087f48089d2a112221988de78f07..e6f998fa4ac39cb3f21fe112ae79c90ac0eadd70 100644
--- a/mlir/test/mlir-tblgen/op-format.mlir
+++ b/mlir/test/mlir-tblgen/op-format.mlir
@@ -239,6 +239,16 @@ test.format_optional_result_b_op : i64 -> i64, i64
 // CHECK: test.format_optional_result_c_op : (i64) -> (i64, i64)
 test.format_optional_result_c_op : (i64) -> (i64, i64)
 
+//===----------------------------------------------------------------------===//
+// Format optional with else
+//===----------------------------------------------------------------------===//
+
+// CHECK: test.format_optional_else then
+test.format_optional_else then
+
+// CHECK: test.format_optional_else else
+test.format_optional_else else
+
 //===----------------------------------------------------------------------===//
 // Format custom directives
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/mlir-tblgen/rewriter-indexing.td b/mlir/test/mlir-tblgen/rewriter-indexing.td
index a6b4032857656f9270d7c53e0231f53953b98fe4..cbdeff9c743dae17e987ce1fdd62c399129efba7 100644
--- a/mlir/test/mlir-tblgen/rewriter-indexing.td
+++ b/mlir/test/mlir-tblgen/rewriter-indexing.td
@@ -58,3 +58,30 @@ def test2 : Pat<(COp $attr1, $op1, $attr2, (AOp $op2)),
 def test3 : Pat<(BOp $attr, (AOp:$a $input)),
                 (BOp $attr, (AOp $input), (location $a))>;
 
+def DOp : NS_Op<"d_op", []> {
+  let arguments = (ins
+    AnyInteger:$v1,
+    AnyInteger:$v2,
+    AnyInteger:$v3,
+    AnyInteger:$v4,
+    AnyInteger:$v5,
+    AnyInteger:$v6,
+    AnyInteger:$v7,
+    AnyInteger:$v8,
+    AnyInteger:$v9,
+    AnyInteger:$v10
+  );
+
+  let results = (outs AnyInteger);
+}
+
+def NativeBuilder :
+  NativeCodeCall<[{
+    nativeCall($_builder, $_loc, $0, $1, $2, $3, $4, $5, $6, $7, $8, $9)
+  }]>;
+
+// Check Pattern with large number of DAG arguments passed to NativeCodeCall
+// CHECK: struct test4 : public ::mlir::RewritePattern {
+// CHECK: nativeCall(rewriter, odsLoc, (*v1.begin()), (*v2.begin()), (*v3.begin()), (*v4.begin()), (*v5.begin()), (*v6.begin()), (*v7.begin()), (*v8.begin()), (*v9.begin()), (*v10.begin()))
+def test4 : Pat<(DOp $v1, $v2, $v3, $v4, $v5, $v6, $v7, $v8, $v9, $v10),
+                (NativeBuilder $v1, $v2, $v3, $v4, $v5, $v6, $v7, $v8, $v9, $v10)>;
diff --git a/mlir/tools/CMakeLists.txt b/mlir/tools/CMakeLists.txt
index 37793ce65ab13d011986e173657f6cd98da03271..ac9ca816732026190f9ed3d340e7d5c9a23a9ec1 100644
--- a/mlir/tools/CMakeLists.txt
+++ b/mlir/tools/CMakeLists.txt
@@ -1,7 +1,6 @@
 add_subdirectory(mlir-cpu-runner)
 add_subdirectory(mlir-opt)
 add_subdirectory(mlir-reduce)
-add_subdirectory(mlir-rocm-runner)
 add_subdirectory(mlir-shlib)
 add_subdirectory(mlir-spirv-cpu-runner)
 add_subdirectory(mlir-translate)
diff --git a/mlir/tools/mlir-cpu-runner/CMakeLists.txt b/mlir/tools/mlir-cpu-runner/CMakeLists.txt
index 1135a683eb6efaf1d6b793a806e991d33f75588c..cc3b468b2d38442e6ef36cadcbd5cdd3f35abc22 100644
--- a/mlir/tools/mlir-cpu-runner/CMakeLists.txt
+++ b/mlir/tools/mlir-cpu-runner/CMakeLists.txt
@@ -17,8 +17,7 @@ target_link_libraries(mlir-cpu-runner PRIVATE
   MLIRJitRunner
   MLIRLLVMIR
   MLIRLLVMToLLVMIRTranslation
-  MLIROpenMP
-  MLIROpenMPToLLVMIRTranslation
+  MLIRToLLVMIRTranslationRegistration
   MLIRParser
   MLIRTargetLLVMIRExport
   MLIRSupport
diff --git a/mlir/tools/mlir-cpu-runner/mlir-cpu-runner.cpp b/mlir/tools/mlir-cpu-runner/mlir-cpu-runner.cpp
index d22b3a20bc8b77578caf8a23195a0e6dfe0eeba7..b1009a1fae458fd3648b33d0ec73dc171854b525 100644
--- a/mlir/tools/mlir-cpu-runner/mlir-cpu-runner.cpp
+++ b/mlir/tools/mlir-cpu-runner/mlir-cpu-runner.cpp
@@ -16,8 +16,7 @@
 #include "mlir/ExecutionEngine/JitRunner.h"
 #include "mlir/ExecutionEngine/OptUtils.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
-#include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Dialect/All.h"
 
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/TargetSelect.h"
@@ -30,8 +29,7 @@ int main(int argc, char **argv) {
   mlir::initializeLLVMPasses();
 
   mlir::DialectRegistry registry;
-  mlir::registerLLVMDialectTranslation(registry);
-  mlir::registerOpenMPDialectTranslation(registry);
+  mlir::registerAllToLLVMIRTranslations(registry);
 
   return mlir::JitRunnerMain(argc, argv, registry);
 }
diff --git a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-gen.cpp b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-gen.cpp
index 7165a0fe89fef146e7f20b1d4131933fc91c791e..548a2e3bbe7aee91d8188a06177e100511c0cd53 100644
--- a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-gen.cpp
+++ b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-gen.cpp
@@ -1518,7 +1518,6 @@ LogicalResult TCParser::parseExpression(TensorUse currentDefinition,
       reductionDims.push_back(iter.cast<AffineDimExpr>().getPosition());
   }
 
-
   auto parseExpr = [&]() -> LogicalResult {
     std::unique_ptr<Expression> e;
     if (failed(parseExpression(currentDefinition, e, state)))
@@ -2074,10 +2073,10 @@ void TCParser::printCanonicalizersAndFolders(llvm::raw_ostream &os,
                                              StringRef cppOpName) {
   const char *canonicalizersAndFoldersFmt = R"FMT(
     void {0}::getCanonicalizationPatterns(
-        OwningRewritePatternList &results,
+        RewritePatternSet &results,
         MLIRContext *context) {{
-      results.insert<EraseDeadLinalgOp>();
-      results.insert<FoldTensorCastOp>();
+      results.add<EraseDeadLinalgOp>(context);
+      results.add<FoldTensorCastOp>(context);
     }
     LogicalResult {0}::fold(ArrayRef<Attribute>,
                             SmallVectorImpl<OpFoldResult> &) {{
diff --git a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
index cea46325d54cd7265f4cc67fc298de57b112c6e2..53a5807bd17973509f8a6c2d24ba9914e2a20f70 100644
--- a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
+++ b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
@@ -519,10 +519,10 @@ ArrayAttr {0}::iterator_types() {
 // {0}: Class name
 const char structuredOpCanonicalizersAndFoldersFormat[] = R"FMT(
 void {0}::getCanonicalizationPatterns(
-    OwningRewritePatternList &results,
+    RewritePatternSet &results,
     MLIRContext *context) {{
-  results.insert<EraseDeadLinalgOp>();
-  results.insert<FoldTensorCastOp>();
+  results.add<EraseDeadLinalgOp>(context);
+  results.add<FoldTensorCastOp>(context);
 }
 LogicalResult {0}::fold(ArrayRef<Attribute>,
                         SmallVectorImpl<OpFoldResult> &) {{
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 241cee572cb137016889717d173815582892fcf7..428b3d5063174d49e60bc9ffec006f80ceee51c3 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -65,7 +65,7 @@ void registerTestCallGraphPass();
 void registerTestConstantFold();
 void registerTestConvVectorization();
 void registerTestGpuSerializeToCubinPass();
-void registerTestConvertGPUKernelToHsacoPass();
+void registerTestGpuSerializeToHsacoPass();
 void registerTestDataLayoutQuery();
 void registerTestDecomposeCallGraphTypes();
 void registerTestDialect(DialectRegistry &);
@@ -140,7 +140,7 @@ void registerTestPasses() {
   test::registerTestGpuSerializeToCubinPass();
 #endif
 #if MLIR_ROCM_CONVERSIONS_ENABLED
-  test::registerTestConvertGPUKernelToHsacoPass();
+  test::registerTestGpuSerializeToHsacoPass();
 #endif
   test::registerTestConvVectorization();
   test::registerTestDecomposeCallGraphTypes();
diff --git a/mlir/tools/mlir-rocm-runner/CMakeLists.txt b/mlir/tools/mlir-rocm-runner/CMakeLists.txt
deleted file mode 100644
index d2381413e1583af67aa9a61da017ccbeeb083fe3..0000000000000000000000000000000000000000
--- a/mlir/tools/mlir-rocm-runner/CMakeLists.txt
+++ /dev/null
@@ -1,127 +0,0 @@
-set(LLVM_OPTIONAL_SOURCES
-  rocm-runtime-wrappers.cpp
-  mlir-rocm-runner.cpp
-  )
-
-if(MLIR_ROCM_RUNNER_ENABLED)
-  if (NOT ("AMDGPU" IN_LIST LLVM_TARGETS_TO_BUILD))
-    message(SEND_ERROR
-      "Building the mlir rocm runner requires the AMDGPU backend")
-  endif()
-
-  # Ensure lld is enabled.
-  if (NOT "lld" IN_LIST LLVM_ENABLE_PROJECTS)
-    message(SEND_ERROR "lld is not enabled. Please revise LLVM_ENABLE_PROJECTS")
-  endif()
-
-  # lld header files.
-  include_directories(${MLIR_SOURCE_DIR}/../lld/include)
-
-  # Configure ROCm support.
-  if (NOT DEFINED ROCM_PATH)
-    if (NOT DEFINED ENV{ROCM_PATH})
-      set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCm has been installed")
-    else()
-      set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCm has been installed")
-    endif()
-    set(HIP_PATH "${ROCM_PATH}/hip" CACHE PATH " Path to which HIP has been installed")
-  endif()
-  set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})
-  find_package(HIP)
-  if (NOT HIP_FOUND)
-    message(SEND_ERROR "Build the mlir rocm runner requires a working ROCm and HIP install")
-  else()
-    message(STATUS "ROCm HIP version: ${HIP_VERSION}")
-  endif()
-
-  # Set compile-time flags for ROCm path.
-  add_definitions(-D__ROCM_PATH__="${ROCM_PATH}")
-
-  # Locate HIP runtime library.
-  find_library(ROCM_RUNTIME_LIBRARY amdhip64
-               PATHS "${HIP_PATH}/lib")
-  if (NOT ROCM_RUNTIME_LIBRARY)
-    message(SEND_ERROR "Could not locate ROCm HIP runtime library")
-  else()
-    message(STATUS "ROCm HIP runtime lib: ${ROCM_RUNTIME_LIBRARY}")
-  endif()
-
-  # Set HIP compile-time flags.
-  add_definitions(-D__HIP_PLATFORM_HCC__)
-
-  add_mlir_library(rocm-runtime-wrappers
-    SHARED
-    rocm-runtime-wrappers.cpp
-
-    EXCLUDE_FROM_LIBMLIR
-  )
-  target_include_directories(rocm-runtime-wrappers
-    PRIVATE
-    "${HIP_PATH}/../include"
-    "${HIP_PATH}/include"
-  )
-  target_link_libraries(rocm-runtime-wrappers
-    PRIVATE
-    ${ROCM_RUNTIME_LIBRARY}
-  )
-
-  get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
-  set(LIBS
-    ${conversion_libs}
-    lldCommon
-    lldDriver
-    lldELF
-    MLIRJitRunner
-    MLIRAnalysis
-    MLIREDSC
-    MLIRExecutionEngine
-    MLIRGPU
-    MLIRIR
-    MLIRLLVMIR
-    MLIRLLVMToLLVMIRTranslation
-    MLIRParser
-    MLIRROCDLIR
-    MLIRStandard
-    MLIRSupport
-    MLIRTargetLLVMIRExport
-    MLIRROCDLToLLVMIRTranslation
-    MLIRTransforms
-    MLIRTranslation
-    ${ROCM_RUNTIME_LIBRARY}
-  )
-
-  # Manually expand the target library, since our MLIR libraries
-  # aren't plugged into the LLVM dependency tracking. If we don't
-  # do this then we can't insert the CodeGen library after ourselves
-  llvm_expand_pseudo_components(TARGET_LIBS AllTargetsCodeGens AllTargetsAsmParsers)
-  # Prepend LLVM in front of every target, this is how the library
-  # are named with CMake
-  SET(targets_to_link)
-  FOREACH(t ${TARGET_LIBS})
-    LIST(APPEND targets_to_link "LLVM${t}")
-  ENDFOREACH(t)
-
-  add_llvm_tool(mlir-rocm-runner
-    mlir-rocm-runner.cpp
-
-    DEPENDS
-    rocm-runtime-wrappers
-
-    LINK_COMPONENTS
-
-    Core
-    LTO
-    MC
-    MCParser
-    Option
-    Support
-    )
-  llvm_update_compile_flags(mlir-rocm-runner)
-  target_include_directories(mlir-rocm-runner
-    PRIVATE
-    "${HIP_PATH}/../include"
-    "${HIP_PATH}/include"
-  )
-  target_link_libraries(mlir-rocm-runner PRIVATE ${LIBS} ${targets_to_link})
-
-endif()
diff --git a/mlir/tools/mlir-rocm-runner/mlir-rocm-runner.cpp b/mlir/tools/mlir-rocm-runner/mlir-rocm-runner.cpp
deleted file mode 100644
index c2f9abbf73c8ad6dc3e625dbd32c465807323199..0000000000000000000000000000000000000000
--- a/mlir/tools/mlir-rocm-runner/mlir-rocm-runner.cpp
+++ /dev/null
@@ -1,349 +0,0 @@
-//===- mlir-rocm-runner.cpp - MLIR ROCM Execution Driver-------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This is a command line utility that executes an MLIR file on the GPU by
-// translating MLIR to ROCDL/LLVM IR before JIT-compiling and executing the
-// latter.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/STLExtras.h"
-
-#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
-#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
-#include "mlir/Conversion/SCFToStandard/SCFToStandard.h"
-#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
-#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
-#include "mlir/Dialect/GPU/GPUDialect.h"
-#include "mlir/Dialect/GPU/Passes.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/ExecutionEngine/JitRunner.h"
-#include "mlir/ExecutionEngine/OptUtils.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Pass/PassManager.h"
-#include "mlir/Support/FileUtilities.h"
-#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
-#include "mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h"
-#include "mlir/Target/LLVMIR/Export.h"
-#include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/Passes.h"
-#include "llvm/Support/ErrorOr.h"
-#include "llvm/Support/FileUtilities.h"
-#include "llvm/Support/InitLLVM.h"
-#include "llvm/Support/LineIterator.h"
-#include "llvm/Support/Program.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/TargetRegistry.h"
-#include "llvm/Support/TargetSelect.h"
-
-// MC headers.
-#include "llvm/MC/MCAsmBackend.h"
-#include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MCCodeEmitter.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCInstPrinter.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCObjectFileInfo.h"
-#include "llvm/MC/MCObjectWriter.h"
-#include "llvm/MC/MCParser/AsmLexer.h"
-#include "llvm/MC/MCParser/MCTargetAsmParser.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/MC/MCTargetOptionsCommandFlags.h"
-
-// lld headers.
-#include "lld/Common/Driver.h"
-
-// HIP headers.
-#include "hip/hip_version.h"
-
-#include <mutex>
-
-using namespace mlir;
-using namespace llvm;
-
-using Blob = SmallVector<char, 0>;
-
-static cl::opt<std::string> tripleName("triple", cl::desc("target triple"),
-                                       cl::value_desc("triple string"),
-                                       cl::init("amdgcn-amd-amdhsa"));
-
-static cl::opt<std::string> targetChip("target", cl::desc("target chip"),
-                                       cl::value_desc("AMDGPU ISA version"),
-                                       cl::init(""));
-
-static cl::opt<std::string> features("feature", cl::desc("target features"),
-                                     cl::value_desc("AMDGPU target features"),
-                                     cl::init(""));
-
-static constexpr const char kRunnerProgram[] = "mlir-rocm-runner";
-static constexpr const char kRocmAgentEnumerator[] = "rocm_agent_enumerator";
-static constexpr const char kDefaultTargetChip[] = "gfx900";
-
-static LogicalResult assembleIsa(const std::string isa, StringRef name,
-                                 Blob &result) {
-  raw_svector_ostream os(result);
-
-  std::string error;
-  Triple theTriple(Triple::normalize(tripleName));
-  const Target *theTarget =
-      TargetRegistry::lookupTarget(theTriple.normalize(), error);
-  if (!theTarget) {
-    WithColor::error(errs(), name) << error;
-    return failure();
-  }
-
-  SourceMgr srcMgr;
-  srcMgr.AddNewSourceBuffer(MemoryBuffer::getMemBuffer(isa), SMLoc());
-
-  const MCTargetOptions mcOptions;
-  std::unique_ptr<MCRegisterInfo> mri(theTarget->createMCRegInfo(tripleName));
-  std::unique_ptr<MCAsmInfo> mai(
-      theTarget->createMCAsmInfo(*mri, tripleName, mcOptions));
-  mai->setRelaxELFRelocations(true);
-
-  MCObjectFileInfo mofi;
-  MCContext ctx(mai.get(), mri.get(), &mofi, &srcMgr, &mcOptions);
-  mofi.InitMCObjectFileInfo(theTriple, false, ctx, false);
-
-  SmallString<128> cwd;
-  if (!sys::fs::current_path(cwd))
-    ctx.setCompilationDir(cwd);
-
-  std::unique_ptr<MCStreamer> mcStreamer;
-  std::unique_ptr<MCInstrInfo> mcii(theTarget->createMCInstrInfo());
-  std::unique_ptr<MCSubtargetInfo> sti(
-      theTarget->createMCSubtargetInfo(tripleName, targetChip, features));
-
-  MCCodeEmitter *ce = theTarget->createMCCodeEmitter(*mcii, *mri, ctx);
-  MCAsmBackend *mab = theTarget->createMCAsmBackend(*sti, *mri, mcOptions);
-  mcStreamer.reset(theTarget->createMCObjectStreamer(
-      theTriple, ctx, std::unique_ptr<MCAsmBackend>(mab),
-      mab->createObjectWriter(os), std::unique_ptr<MCCodeEmitter>(ce), *sti,
-      mcOptions.MCRelaxAll, mcOptions.MCIncrementalLinkerCompatible,
-      /*DWARFMustBeAtTheEnd*/ false));
-  mcStreamer->setUseAssemblerInfoForParsing(true);
-
-  std::unique_ptr<MCAsmParser> parser(
-      createMCAsmParser(srcMgr, ctx, *mcStreamer, *mai));
-  std::unique_ptr<MCTargetAsmParser> tap(
-      theTarget->createMCAsmParser(*sti, *parser, *mcii, mcOptions));
-
-  if (!tap) {
-    WithColor::error(errs(), name) << "assembler initialization error.\n";
-    return failure();
-  }
-
-  parser->setTargetParser(*tap);
-  parser->Run(false);
-
-  return success();
-}
-
-static std::mutex mutex;
-static LogicalResult createHsaco(const Blob &isaBlob, StringRef name,
-                                 Blob &hsacoBlob) {
-  // Save the ISA binary to a temp file.
-  int tempIsaBinaryFd = -1;
-  SmallString<128> tempIsaBinaryFilename;
-  std::error_code ec = sys::fs::createTemporaryFile(
-      "kernel", "o", tempIsaBinaryFd, tempIsaBinaryFilename);
-  if (ec) {
-    WithColor::error(errs(), name)
-        << "temporary file for ISA binary creation error.\n";
-    return failure();
-  }
-  FileRemover cleanupIsaBinary(tempIsaBinaryFilename);
-  raw_fd_ostream tempIsaBinaryOs(tempIsaBinaryFd, true);
-  tempIsaBinaryOs << isaBlob;
-  tempIsaBinaryOs.close();
-
-  // Create a temp file for HSA code object.
-  int tempHsacoFD = -1;
-  SmallString<128> tempHsacoFilename;
-  ec = sys::fs::createTemporaryFile("kernel", "hsaco", tempHsacoFD,
-                                    tempHsacoFilename);
-  if (ec) {
-    WithColor::error(errs(), name)
-        << "temporary file for HSA code object creation error.\n";
-    return failure();
-  }
-  FileRemover cleanupHsaco(tempHsacoFilename);
-
-  const std::lock_guard<std::mutex> lock(mutex);
-  // Invoke lld. Expect a true return value from lld.
-  bool ret = lld::elf::link({"ld.lld", "-shared", tempIsaBinaryFilename.c_str(),
-                             "-o", tempHsacoFilename.c_str()},
-                            /*canEarlyExit=*/false, llvm::outs(), llvm::errs());
-  if (!ret) {
-    WithColor::error(errs(), name) << "lld invocation error.\n";
-    return failure();
-  }
-
-  // Load the HSA code object.
-  auto hsacoFile = mlir::openInputFile(tempHsacoFilename);
-  if (!hsacoFile) {
-    WithColor::error(errs(), name)
-        << "read HSA code object from temp file error.\n";
-    return failure();
-  }
-  hsacoBlob.assign(hsacoFile->getBuffer().begin(),
-                   hsacoFile->getBuffer().end());
-
-  return success();
-}
-
-static std::unique_ptr<llvm::Module>
-compileModuleToROCDLIR(Operation *m, llvm::LLVMContext &llvmContext,
-                       StringRef name) {
-  auto llvmModule = translateModuleToROCDLIR(m, llvmContext, name);
-  // TODO: Link with ROCm-Device-Libs in case needed (ex: the Module
-  // depends on math functions).
-  return llvmModule;
-}
-
-static OwnedBlob compileISAToHsaco(const std::string isa, Location loc,
-                                   StringRef name) {
-  // ISA -> ISA in binary form via MC.
-  // Use lld to create HSA code object.
-  Blob isaBlob;
-  Blob hsacoBlob;
-
-  if (succeeded(assembleIsa(isa, name, isaBlob)) &&
-      succeeded(createHsaco(isaBlob, name, hsacoBlob)))
-    return std::make_unique<std::vector<char>>(hsacoBlob.begin(),
-                                               hsacoBlob.end());
-
-  WithColor::error(errs(), name) << "producing HSA code object error.\n";
-  return {};
-}
-
-static void configTargetChip() {
-  // Set targetChip to default value first.
-  targetChip = kDefaultTargetChip;
-
-  // Locate rocm_agent_enumerator.
-  llvm::ErrorOr<std::string> rocmAgentEnumerator = llvm::sys::findProgramByName(
-      kRocmAgentEnumerator, {__ROCM_PATH__ "/bin"});
-  std::error_code ec;
-  if ((ec = rocmAgentEnumerator.getError())) {
-    WithColor::warning(errs(), kRunnerProgram)
-        << kRocmAgentEnumerator << " couldn't be located under "
-        << __ROCM_PATH__ << ", set target as " << kDefaultTargetChip << "\n";
-    return;
-  }
-
-  // Prepare temp file to hold the outputs.
-  int tempFd = -1;
-  SmallString<128> tempFilename;
-  ec = sys::fs::createTemporaryFile("rocm_agent", "txt", tempFd, tempFilename);
-  if (ec) {
-    WithColor::warning(errs(), kRunnerProgram)
-        << "temporary file for " << kRocmAgentEnumerator
-        << " creation error, set target as " << kDefaultTargetChip << "\n";
-    return;
-  }
-  FileRemover cleanup(tempFilename);
-
-  // Invoke rocm_agent_enumerator.
-  std::string errorMessage;
-  SmallVector<StringRef, 2> args{"-t", "GPU"};
-  Optional<StringRef> redirects[3] = {{""}, tempFilename.str(), {""}};
-  int result =
-      llvm::sys::ExecuteAndWait(rocmAgentEnumerator.get(), args, llvm::None,
-                                redirects, 0, 0, &errorMessage);
-  if (result) {
-    WithColor::warning(errs(), kRunnerProgram)
-        << kRocmAgentEnumerator << " invocation error: " << errorMessage
-        << ", set target as " << kDefaultTargetChip << "\n";
-    return;
-  }
-
-  // Load and parse the result.
-  auto gfxIsaList = mlir::openInputFile(tempFilename);
-  if (!gfxIsaList) {
-    WithColor::error(errs(), kRunnerProgram)
-        << "read ROCm agent list temp file error, set target as "
-        << kDefaultTargetChip << "\n";
-    return;
-  }
-  for (line_iterator lines(*gfxIsaList); !lines.is_at_end(); ++lines) {
-    // Skip the line with content "gfx000".
-    if (*lines == "gfx000")
-      continue;
-    // Use the first ISA version found.
-    targetChip = lines->str();
-    break;
-  }
-}
-
-static void configTargetFeatures() {
-  if (features.size() > 0)
-    features += ",";
-  // After ROCm 3.5, adopt HSA code object V3.
-  if (HIP_VERSION_MAJOR >= 3 && HIP_VERSION_MINOR >= 5)
-    features += "+code-object-v3";
-  else
-    features += "-code-object-v3";
-}
-
-static LogicalResult runMLIRPasses(ModuleOp m) {
-  PassManager pm(m.getContext());
-  applyPassManagerCLOptions(pm);
-
-  // Configure target chip ISA version if it has not been specified.
-  if (!targetChip.size())
-    configTargetChip();
-
-  // Configure target features per ROCm / HIP version.
-  configTargetFeatures();
-
-  const char gpuBinaryAnnotation[] = "rocdl.hsaco";
-  pm.addPass(createLowerToCFGPass());
-  pm.addPass(createGpuKernelOutliningPass());
-  auto &kernelPm = pm.nest<gpu::GPUModuleOp>();
-  kernelPm.addPass(createStripDebugInfoPass());
-  kernelPm.addPass(createLowerGpuOpsToROCDLOpsPass());
-  kernelPm.addPass(createConvertGPUKernelToBlobPass(
-      compileModuleToROCDLIR, compileISAToHsaco, tripleName, targetChip,
-      features, gpuBinaryAnnotation));
-  pm.addPass(createGpuToLLVMConversionPass(gpuBinaryAnnotation));
-
-  return pm.run(m);
-}
-
-int main(int argc, char **argv) {
-  registerPassManagerCLOptions();
-  llvm::InitLLVM y(argc, argv);
-  llvm::InitializeAllTargetInfos();
-  llvm::InitializeAllTargetMCs();
-  llvm::InitializeAllAsmParsers();
-
-  // Initialize LLVM AMDGPU backend.
-  LLVMInitializeAMDGPUTarget();
-  LLVMInitializeAMDGPUTargetInfo();
-  LLVMInitializeAMDGPUTargetMC();
-  LLVMInitializeAMDGPUAsmPrinter();
-
-  mlir::initializeLLVMPasses();
-
-  mlir::JitRunnerConfig jitRunnerConfig;
-  jitRunnerConfig.mlirTransformer = runMLIRPasses;
-
-  mlir::DialectRegistry registry;
-  registry.insert<mlir::LLVM::LLVMDialect, mlir::gpu::GPUDialect,
-                  mlir::ROCDL::ROCDLDialect, mlir::StandardOpsDialect>();
-  mlir::registerLLVMDialectTranslation(registry);
-  mlir::registerROCDLDialectTranslation(registry);
-
-  return mlir::JitRunnerMain(argc, argv, registry, jitRunnerConfig);
-}
diff --git a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
index 2c3caaeeea254f363e58be0f261775790fe3091e..a951df92fe1803a79333ce7d211c4012c76c166b 100644
--- a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
+++ b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
@@ -432,22 +432,16 @@ static ::mlir::LogicalResult generated{0}Printer(
 /// {1}: Storage class c++ name.
 /// {2}: Parameters parameters.
 /// {3}: Parameter initializer string.
-/// {4}: Parameter name list.
-/// {5}: Parameter types.
-/// {6}: The name of the base value type, e.g. Attribute or Type.
+/// {4}: Parameter types.
+/// {5}: The name of the base value type, e.g. Attribute or Type.
 static const char *const defStorageClassBeginStr = R"(
 namespace {0} {{
-  struct {1} : public ::mlir::{6}Storage {{
+  struct {1} : public ::mlir::{5}Storage {{
     {1} ({2})
       : {3} {{ }
 
     /// The hash key is a tuple of the parameter types.
-    using KeyTy = std::tuple<{5}>;
-
-    /// Define the comparison function for the key type.
-    bool operator==(const KeyTy &key) const {{
-      return key == KeyTy({4});
-    }
+    using KeyTy = std::tuple<{4}>;
 )";
 
 /// The storage class' constructor template.
@@ -555,23 +549,34 @@ void DefGenerator::emitStorageClass(const AttrOrTypeDef &def) {
     });
   }
 
-  // Construct the parameter list that is used when a concrete instance of the
-  // storage exists.
-  auto nonStaticParameterNames = llvm::map_range(params, [](const auto &param) {
-    return isa<AttributeSelfTypeParameter>(param) ? "getType()"
-                                                  : param.getName();
-  });
-
-  // 1) Emit most of the storage class up until the hashKey body.
+  // * Emit most of the storage class up until the hashKey body.
   os << formatv(
       defStorageClassBeginStr, def.getStorageNamespace(),
       def.getStorageClassName(),
       ParamCommaFormatter(ParamCommaFormatter::EmitFormat::TypeNamePairs,
                           params, /*prependComma=*/false),
-      paramInitializer, llvm::join(nonStaticParameterNames, ", "),
-      parameterTypeList, valueType);
+      paramInitializer, parameterTypeList, valueType);
+
+  // * Emit the comparison method.
+  os << "  bool operator==(const KeyTy &key) const {\n";
+  for (auto it : llvm::enumerate(params)) {
+    os << "    if (!(";
+
+    // Build the comparator context.
+    bool isSelfType = isa<AttributeSelfTypeParameter>(it.value());
+    FmtContext context;
+    context.addSubst("_lhs", isSelfType ? "getType()" : it.value().getName())
+        .addSubst("_rhs", "std::get<" + Twine(it.index()) + ">(key)");
+
+    // Use the parameter specified comparator if possible, otherwise default to
+    // operator==.
+    Optional<StringRef> comparator = it.value().getComparator();
+    os << tgfmt(comparator ? *comparator : "$_lhs == $_rhs", &context);
+    os << "))\n      return false;\n";
+  }
+  os << "    return true;\n  }\n";
 
-  // 2) Emit the haskKey method.
+  // * Emit the haskKey method.
   os << "  static ::llvm::hash_code hashKey(const KeyTy &key) {\n";
 
   // Extract each parameter from the key.
@@ -581,7 +586,7 @@ void DefGenerator::emitStorageClass(const AttrOrTypeDef &def) {
       [&](unsigned it) { os << "std::get<" << it << ">(key)"; });
   os << ");\n    }\n";
 
-  // 3) Emit the construct method.
+  // * Emit the construct method.
 
   // If user wants to build the storage constructor themselves, declare it
   // here and then they can write the definition elsewhere.
@@ -611,7 +616,7 @@ void DefGenerator::emitStorageClass(const AttrOrTypeDef &def) {
                   llvm::join(parameterNames, ", "));
   }
 
-  // 4) Emit the parameters as storage class members.
+  // * Emit the parameters as storage class members.
   for (const AttrOrTypeParameter &parameter : params) {
     // Attribute value types are not stored as fields in the storage.
     if (!isa<AttributeSelfTypeParameter>(parameter))
@@ -771,15 +776,19 @@ void DefGenerator::emitDefDef(const AttrOrTypeDef &def) {
     // Generate accessor definitions only if we also generate the storage class.
     // Otherwise, let the user define the exact accessor definition.
     if (def.genAccessors() && def.genStorageClass()) {
-      for (const AttrOrTypeParameter &parameter : parameters) {
-        StringRef paramStorageName = isa<AttributeSelfTypeParameter>(parameter)
-                                         ? "getType()"
-                                         : parameter.getName();
-
-        SmallString<16> name = parameter.getName();
+      for (const AttrOrTypeParameter &param : parameters) {
+        SmallString<32> paramStorageName;
+        if (isa<AttributeSelfTypeParameter>(param)) {
+          Twine("getType().cast<" + param.getCppType() + ">()")
+              .toVector(paramStorageName);
+        } else {
+          paramStorageName = param.getName();
+        }
+
+        SmallString<16> name = param.getName();
         name[0] = llvm::toUpper(name[0]);
         os << formatv("{0} {3}::get{1}() const {{ return getImpl()->{2}; }\n",
-                      parameter.getCppType(), name, paramStorageName,
+                      param.getCppType(), name, paramStorageName,
                       def.getCppClassName());
       }
     }
diff --git a/mlir/tools/mlir-tblgen/DialectGen.cpp b/mlir/tools/mlir-tblgen/DialectGen.cpp
index 8eaf17cd7c3fd2c2b19c878fb6cfd0a48d59b4ef..151519519afda5d81175436be1a0583277b49497 100644
--- a/mlir/tools/mlir-tblgen/DialectGen.cpp
+++ b/mlir/tools/mlir-tblgen/DialectGen.cpp
@@ -143,6 +143,13 @@ static const char *const regionResultAttrVerifierDecl = R"(
         ::mlir::NamedAttribute attribute) override;
 )";
 
+/// The code block for the op interface fallback hook.
+static const char *const operationInterfaceFallbackDecl = R"(
+    /// Provides a hook for op interface.
+    void *getRegisteredInterfaceForOp(mlir::TypeID interfaceID,
+                                      mlir::OperationName opName) override;
+)";
+
 /// Generate the declaration for the given dialect class.
 static void emitDialectDecl(Dialect &dialect,
                             iterator_range<DialectFilterIterator> dialectAttrs,
@@ -181,6 +188,8 @@ static void emitDialectDecl(Dialect &dialect,
     os << regionArgAttrVerifierDecl;
   if (dialect.hasRegionResultAttrVerify())
     os << regionResultAttrVerifierDecl;
+  if (dialect.hasOperationInterfaceFallback())
+    os << operationInterfaceFallbackDecl;
   if (llvm::Optional<StringRef> extraDecl = dialect.getExtraClassDeclaration())
     os << *extraDecl;
 
diff --git a/mlir/tools/mlir-tblgen/EnumsGen.cpp b/mlir/tools/mlir-tblgen/EnumsGen.cpp
index e207e313b7307d3dc6a2bc734df77677c55983ce..aa8841abfd73b2b4c14d7f7388b85c0bd9d8ca7b 100644
--- a/mlir/tools/mlir-tblgen/EnumsGen.cpp
+++ b/mlir/tools/mlir-tblgen/EnumsGen.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/Attribute.h"
+#include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/GenInfo.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -22,12 +23,16 @@
 
 using llvm::formatv;
 using llvm::isDigit;
+using llvm::PrintFatalError;
 using llvm::raw_ostream;
 using llvm::Record;
 using llvm::RecordKeeper;
 using llvm::StringRef;
+using mlir::tblgen::Attribute;
 using mlir::tblgen::EnumAttr;
 using mlir::tblgen::EnumAttrCase;
+using mlir::tblgen::FmtContext;
+using mlir::tblgen::tgfmt;
 
 static std::string makeIdentifier(StringRef str) {
   if (!str.empty() && isDigit(static_cast<unsigned char>(str.front()))) {
@@ -303,6 +308,78 @@ static void emitUnderlyingToSymFnForIntEnum(const Record &enumDef,
      << "}\n\n";
 }
 
+static void emitSpecializedAttrDef(const Record &enumDef, raw_ostream &os) {
+  EnumAttr enumAttr(enumDef);
+  StringRef enumName = enumAttr.getEnumClassName();
+  StringRef symToStrFnName = enumAttr.getSymbolToStringFnName();
+  StringRef strToSymFnName = enumAttr.getStringToSymbolFnName();
+  StringRef attrClassName = enumAttr.getSpecializedAttrClassName();
+  llvm::Record *baseAttrDef = enumAttr.getBaseAttrClass();
+  Attribute baseAttr(baseAttrDef);
+
+  // Emit classof method
+
+  os << formatv("bool {0}::classof(::mlir::Attribute attr) {{\n",
+                attrClassName);
+
+  mlir::tblgen::Pred baseAttrPred = baseAttr.getPredicate();
+  if (baseAttrPred.isNull())
+    PrintFatalError("ERROR: baseAttrClass for EnumAttr has no Predicate\n");
+
+  std::string condition = baseAttrPred.getCondition();
+  FmtContext verifyCtx;
+  verifyCtx.withSelf("attr");
+  os << tgfmt("  return $0;\n", /*ctx=*/nullptr, tgfmt(condition, &verifyCtx));
+
+  os << "}\n";
+
+  // Emit get method
+
+  os << formatv("{0} {0}::get(::mlir::MLIRContext *context, {1} val) {{\n",
+                attrClassName, enumName);
+
+  if (enumAttr.isSubClassOf("StrEnumAttr")) {
+    os << formatv("  ::mlir::StringAttr baseAttr = "
+                  "::mlir::StringAttr::get(context, {0}(val));\n",
+                  symToStrFnName);
+  } else {
+    StringRef underlyingType = enumAttr.getUnderlyingType();
+
+    // Assuming that it is IntegerAttr constraint
+    int64_t bitwidth = 64;
+    if (baseAttrDef->getValue("valueType")) {
+      auto *valueTypeDef = baseAttrDef->getValueAsDef("valueType");
+      if (valueTypeDef->getValue("bitwidth"))
+        bitwidth = valueTypeDef->getValueAsInt("bitwidth");
+    }
+
+    os << formatv("  ::mlir::IntegerType intType = "
+                  "::mlir::IntegerType::get(context, {0});\n",
+                  bitwidth);
+    os << formatv("  ::mlir::IntegerAttr baseAttr = "
+                  "::mlir::IntegerAttr::get(intType, static_cast<{0}>(val));\n",
+                  underlyingType);
+  }
+  os << formatv("  return baseAttr.cast<{0}>();\n", attrClassName);
+
+  os << "}\n";
+
+  // Emit getValue method
+
+  os << formatv("{0} {1}::getValue() const {{\n", enumName, attrClassName);
+
+  if (enumAttr.isSubClassOf("StrEnumAttr")) {
+    os << formatv("  const auto res = {0}(::mlir::StringAttr::getValue());\n",
+                  strToSymFnName);
+    os << "  return res.getValue();\n";
+  } else {
+    os << formatv("  return static_cast<{0}>(::mlir::IntegerAttr::getInt());\n",
+                  enumName);
+  }
+
+  os << "}\n";
+}
+
 static void emitUnderlyingToSymFnForBitEnum(const Record &enumDef,
                                             raw_ostream &os) {
   EnumAttr enumAttr(enumDef);
@@ -391,6 +468,23 @@ inline ::llvm::Optional<{0}> symbolizeEnum<{0}>(::llvm::StringRef str) {
 )";
   os << formatv(symbolizeEnumStr, enumName, strToSymFnName);
 
+  const char *const attrClassDecl = R"(
+class {1} : public ::mlir::{2} {
+public:
+  using ValueType = {0};
+  using ::mlir::{2}::{2};
+  static bool classof(::mlir::Attribute attr);
+  static {1} get(::mlir::MLIRContext *context, {0} val);
+  {0} getValue() const;
+};
+)";
+  if (enumAttr.genSpecializedAttr()) {
+    StringRef attrClassName = enumAttr.getSpecializedAttrClassName();
+    StringRef baseAttrClassName =
+        enumAttr.isSubClassOf("StrEnumAttr") ? "StringAttr" : "IntegerAttr";
+    os << formatv(attrClassDecl, enumName, attrClassName, baseAttrClassName);
+  }
+
   for (auto ns : llvm::reverse(namespaces))
     os << "} // namespace " << ns << "\n";
 
@@ -428,6 +522,9 @@ static void emitEnumDef(const Record &enumDef, raw_ostream &os) {
     emitUnderlyingToSymFnForIntEnum(enumDef, os);
   }
 
+  if (enumAttr.genSpecializedAttr())
+    emitSpecializedAttrDef(enumDef, os);
+
   for (auto ns : llvm::reverse(namespaces))
     os << "} // namespace " << ns << "\n";
   os << "\n";
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index e137df4244f753715ad94dc3cba6309cb0c29452..ac3bd168f9caa80fefa52e5ea2fa16f614340323 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -90,13 +90,14 @@ const char *adapterSegmentSizeAttrInitCode = R"(
   auto sizeAttr = odsAttrs.get("{0}").cast<::mlir::DenseIntElementsAttr>();
 )";
 const char *opSegmentSizeAttrInitCode = R"(
-  auto sizeAttr = (*this)->getAttrOfType<::mlir::DenseIntElementsAttr>("{0}");
+  auto sizeAttr = (*this)->getAttr("{0}").cast<::mlir::DenseIntElementsAttr>();
 )";
 const char *attrSizedSegmentValueRangeCalcCode = R"(
+  auto sizeAttrValues = sizeAttr.getValues<uint32_t>();
   unsigned start = 0;
   for (unsigned i = 0; i < index; ++i)
-    start += (*(sizeAttr.begin() + i)).getZExtValue();
-  unsigned size = (*(sizeAttr.begin() + index)).getZExtValue();
+    start += *(sizeAttrValues.begin() + i);
+  unsigned size = *(sizeAttrValues.begin() + index);
   return {start, size};
 )";
 
@@ -1673,15 +1674,40 @@ void OpEmitter::genCodeForAddingArgAndRegionForBuilder(OpMethodBody &body,
 }
 
 void OpEmitter::genCanonicalizerDecls() {
-  if (!def.getValueAsBit("hasCanonicalizer"))
+  bool hasCanonicalizeMethod = def.getValueAsBit("hasCanonicalizeMethod");
+  if (hasCanonicalizeMethod) {
+    // static LogicResult FooOp::
+    // canonicalize(FooOp op, PatternRewriter &rewriter);
+    SmallVector<OpMethodParameter, 2> paramList;
+    paramList.emplace_back(op.getCppClassName(), "op");
+    paramList.emplace_back("::mlir::PatternRewriter &", "rewriter");
+    opClass.addMethodAndPrune("::mlir::LogicalResult", "canonicalize",
+                              OpMethod::MP_StaticDeclaration,
+                              std::move(paramList));
+  }
+
+  // We get a prototype for 'getCanonicalizationPatterns' if requested directly
+  // or if using a 'canonicalize' method.
+  bool hasCanonicalizer = def.getValueAsBit("hasCanonicalizer");
+  if (!hasCanonicalizeMethod && !hasCanonicalizer)
     return;
 
+  // We get a body for 'getCanonicalizationPatterns' when using a 'canonicalize'
+  // method, but not implementing 'getCanonicalizationPatterns' manually.
+  bool hasBody = hasCanonicalizeMethod && !hasCanonicalizer;
+
+  // Add a signature for getCanonicalizationPatterns if implemented by the
+  // dialect or if synthesized to call 'canonicalize'.
   SmallVector<OpMethodParameter, 2> paramList;
-  paramList.emplace_back("::mlir::OwningRewritePatternList &", "results");
+  paramList.emplace_back("::mlir::RewritePatternSet &", "results");
   paramList.emplace_back("::mlir::MLIRContext *", "context");
-  opClass.addMethodAndPrune("void", "getCanonicalizationPatterns",
-                            OpMethod::MP_StaticDeclaration,
-                            std::move(paramList));
+  auto kind = hasBody ? OpMethod::MP_Static : OpMethod::MP_StaticDeclaration;
+  auto *method = opClass.addMethodAndPrune(
+      "void", "getCanonicalizationPatterns", kind, std::move(paramList));
+
+  // If synthesizing the method, fill it it.
+  if (hasBody)
+    method->body() << "  results.add(canonicalize);\n";
 }
 
 void OpEmitter::genFolderDecls() {
diff --git a/mlir/tools/mlir-tblgen/OpDocGen.cpp b/mlir/tools/mlir-tblgen/OpDocGen.cpp
index 45325deb2b7992f45dfb9024387e440cd6a49d25..d869aed8cb7d450de9885463900ca7b28698b5cf 100644
--- a/mlir/tools/mlir-tblgen/OpDocGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDocGen.cpp
@@ -162,46 +162,51 @@ static void emitTypeDoc(const Type &type, raw_ostream &os) {
 // TypeDef Documentation
 //===----------------------------------------------------------------------===//
 
-/// Emit the assembly format of a type.
-static void emitTypeAssemblyFormat(TypeDef td, raw_ostream &os) {
+static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,
+                                            raw_ostream &os) {
   SmallVector<AttrOrTypeParameter, 4> parameters;
-  td.getParameters(parameters);
-  if (parameters.size() == 0) {
-    os << "\nSyntax: `!" << td.getDialect().getName() << "." << td.getMnemonic()
-       << "`\n";
+  def.getParameters(parameters);
+  if (parameters.empty()) {
+    os << "\nSyntax: `!" << def.getDialect().getName() << "."
+       << def.getMnemonic() << "`\n";
     return;
   }
 
-  os << "\nSyntax:\n\n```\n!" << td.getDialect().getName() << "."
-     << td.getMnemonic() << "<\n";
-  for (auto *it = parameters.begin(), *e = parameters.end(); it < e; ++it) {
-    os << "  " << it->getSyntax();
-    if (it < parameters.end() - 1)
+  os << "\nSyntax:\n\n```\n!" << def.getDialect().getName() << "."
+     << def.getMnemonic() << "<\n";
+  for (auto it : llvm::enumerate(parameters)) {
+    const AttrOrTypeParameter &param = it.value();
+    os << "  " << param.getSyntax();
+    if (it.index() < (parameters.size() - 1))
       os << ",";
-    os << "   # " << it->getName() << "\n";
+    os << "   # " << param.getName() << "\n";
   }
   os << ">\n```\n";
 }
 
-static void emitTypeDefDoc(TypeDef td, raw_ostream &os) {
-  os << llvm::formatv("### `{0}` ({1})\n", td.getName(), td.getCppClassName());
+static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
+  os << llvm::formatv("### {0}\n", def.getCppClassName());
 
-  // Emit the summary, syntax, and description if present.
-  if (td.hasSummary())
-    os << "\n" << td.getSummary() << "\n";
-  if (td.getMnemonic() && td.getPrinterCode() && *td.getPrinterCode() == "" &&
-      td.getParserCode() && *td.getParserCode() == "")
-    emitTypeAssemblyFormat(td, os);
-  if (td.hasDescription()) {
+  // Emit the summary if present.
+  if (def.hasSummary())
+    os << "\n" << def.getSummary() << "\n";
+
+  // Emit the syntax if present.
+  if (def.getMnemonic() && def.getPrinterCode() == StringRef() &&
+      def.getParserCode() == StringRef())
+    emitAttrOrTypeDefAssemblyFormat(def, os);
+
+  // Emit the description if present.
+  if (def.hasDescription()) {
     os << "\n";
-    mlir::tblgen::emitDescription(td.getDescription(), os);
+    mlir::tblgen::emitDescription(def.getDescription(), os);
   }
 
-  // Emit attribute documentation.
+  // Emit parameter documentation.
   SmallVector<AttrOrTypeParameter, 4> parameters;
-  td.getParameters(parameters);
+  def.getParameters(parameters);
   if (!parameters.empty()) {
-    os << "\n#### Type parameters:\n\n";
+    os << "\n#### Parameters:\n\n";
     os << "| Parameter | C++ type | Description |\n"
        << "| :-------: | :-------: | ----------- |\n";
     for (const auto &it : parameters) {
@@ -214,24 +219,35 @@ static void emitTypeDefDoc(TypeDef td, raw_ostream &os) {
   os << "\n";
 }
 
+static void emitAttrOrTypeDefDoc(const RecordKeeper &recordKeeper,
+                                 raw_ostream &os, StringRef recordTypeName) {
+  std::vector<llvm::Record *> defs =
+      recordKeeper.getAllDerivedDefinitions(recordTypeName);
+
+  os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
+  for (const llvm::Record *def : defs)
+    emitAttrOrTypeDefDoc(AttrOrTypeDef(def), os);
+}
+
 //===----------------------------------------------------------------------===//
 // Dialect Documentation
 //===----------------------------------------------------------------------===//
 
-static void emitDialectDoc(const Dialect &dialect, ArrayRef<Operator> ops,
-                           ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
-                           raw_ostream &os) {
-  os << "# ";
-  if (dialect.getName().empty())
-    os << "Builtin";
-  else
-    os << "'" << dialect.getName() << "'";
-  os << " Dialect\n\n";
+static void emitDialectDoc(const Dialect &dialect, ArrayRef<AttrDef> attrDefs,
+                           ArrayRef<Operator> ops, ArrayRef<Type> types,
+                           ArrayRef<TypeDef> typeDefs, raw_ostream &os) {
+  os << "# '" << dialect.getName() << "' Dialect\n\n";
   emitIfNotEmpty(dialect.getSummary(), os);
   emitIfNotEmpty(dialect.getDescription(), os);
 
   os << "[TOC]\n\n";
 
+  if (!attrDefs.empty()) {
+    os << "## Attribute definition\n\n";
+    for (const AttrDef &def : attrDefs)
+      emitAttrOrTypeDefDoc(def, os);
+  }
+
   // TODO: Add link between use and def for types
   if (!types.empty()) {
     os << "## Type constraint definition\n\n";
@@ -247,46 +263,68 @@ static void emitDialectDoc(const Dialect &dialect, ArrayRef<Operator> ops,
 
   if (!typeDefs.empty()) {
     os << "## Type definition\n\n";
-    for (const TypeDef &td : typeDefs)
-      emitTypeDefDoc(td, os);
+    for (const TypeDef &def : typeDefs)
+      emitAttrOrTypeDefDoc(def, os);
   }
 }
 
 static void emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  const auto &opDefs = recordKeeper.getAllDerivedDefinitions("Op");
-  const auto &typeDefs = recordKeeper.getAllDerivedDefinitions("DialectType");
-  const auto &typeDefDefs = recordKeeper.getAllDerivedDefinitions("TypeDef");
+  std::vector<Record *> opDefs = recordKeeper.getAllDerivedDefinitions("Op");
+  std::vector<Record *> typeDefs =
+      recordKeeper.getAllDerivedDefinitions("DialectType");
+  std::vector<Record *> typeDefDefs =
+      recordKeeper.getAllDerivedDefinitions("TypeDef");
+  std::vector<Record *> attrDefDefs =
+      recordKeeper.getAllDerivedDefinitions("AttrDef");
 
   std::set<Dialect> dialectsWithDocs;
-  std::map<Dialect, std::vector<Operator>> dialectOps;
-  std::map<Dialect, std::vector<Type>> dialectTypes;
-  std::map<Dialect, std::vector<TypeDef>> dialectTypeDefs;
+
+  llvm::StringMap<std::vector<AttrDef>> dialectAttrDefs;
+  llvm::StringMap<std::vector<Operator>> dialectOps;
+  llvm::StringMap<std::vector<Type>> dialectTypes;
+  llvm::StringMap<std::vector<TypeDef>> dialectTypeDefs;
+  for (auto *attrDef : attrDefDefs) {
+    AttrDef attr(attrDef);
+    dialectAttrDefs[attr.getDialect().getName()].push_back(attr);
+    dialectsWithDocs.insert(attr.getDialect());
+  }
   for (auto *opDef : opDefs) {
     Operator op(opDef);
-    dialectOps[op.getDialect()].push_back(op);
+    dialectOps[op.getDialect().getName()].push_back(op);
     dialectsWithDocs.insert(op.getDialect());
   }
   for (auto *typeDef : typeDefs) {
     Type type(typeDef);
     if (auto dialect = type.getDialect())
-      dialectTypes[dialect].push_back(type);
+      dialectTypes[dialect.getName()].push_back(type);
   }
   for (auto *typeDef : typeDefDefs) {
     TypeDef type(typeDef);
-    dialectTypeDefs[type.getDialect()].push_back(type);
+    dialectTypeDefs[type.getDialect().getName()].push_back(type);
     dialectsWithDocs.insert(type.getDialect());
   }
 
   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
-  for (auto dialect : dialectsWithDocs)
-    emitDialectDoc(dialect, dialectOps[dialect], dialectTypes[dialect],
-                   dialectTypeDefs[dialect], os);
+  for (const Dialect &dialect : dialectsWithDocs) {
+    StringRef dialectName = dialect.getName();
+    emitDialectDoc(dialect, dialectAttrDefs[dialectName],
+                   dialectOps[dialectName], dialectTypes[dialectName],
+                   dialectTypeDefs[dialectName], os);
+  }
 }
 
 //===----------------------------------------------------------------------===//
 // Gen Registration
 //===----------------------------------------------------------------------===//
 
+static mlir::GenRegistration
+    genAttrRegister("gen-attrdef-doc",
+                    "Generate dialect attribute documentation",
+                    [](const RecordKeeper &records, raw_ostream &os) {
+                      emitAttrOrTypeDefDoc(records, os, "AttrDef");
+                      return false;
+                    });
+
 static mlir::GenRegistration
     genOpRegister("gen-op-doc", "Generate dialect documentation",
                   [](const RecordKeeper &records, raw_ostream &os) {
@@ -294,6 +332,13 @@ static mlir::GenRegistration
                     return false;
                   });
 
+static mlir::GenRegistration
+    genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",
+                    [](const RecordKeeper &records, raw_ostream &os) {
+                      emitAttrOrTypeDefDoc(records, os, "TypeDef");
+                      return false;
+                    });
+
 static mlir::GenRegistration
     genRegister("gen-dialect-doc", "Generate dialect documentation",
                 [](const RecordKeeper &records, raw_ostream &os) {
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index f474bbfb4f20451b193fccd5000cc9841a4b2f06..93d2155c3c82153887c2085083373d9219d92f35 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -348,29 +348,41 @@ private:
 
 namespace {
 /// This class represents a group of elements that are optionally emitted based
-/// upon an optional variable of the operation.
+/// upon an optional variable of the operation, and a group of elements that are
+/// emotted when the anchor element is not present.
 class OptionalElement : public Element {
 public:
-  OptionalElement(std::vector<std::unique_ptr<Element>> &&elements,
+  OptionalElement(std::vector<std::unique_ptr<Element>> &&thenElements,
+                  std::vector<std::unique_ptr<Element>> &&elseElements,
                   unsigned anchor, unsigned parseStart)
-      : Element{Kind::Optional}, elements(std::move(elements)), anchor(anchor),
+      : Element{Kind::Optional}, thenElements(std::move(thenElements)),
+        elseElements(std::move(elseElements)), anchor(anchor),
         parseStart(parseStart) {}
   static bool classof(const Element *element) {
     return element->getKind() == Kind::Optional;
   }
 
-  /// Return the nested elements of this grouping.
-  auto getElements() const { return llvm::make_pointee_range(elements); }
+  /// Return the `then` elements of this grouping.
+  auto getThenElements() const {
+    return llvm::make_pointee_range(thenElements);
+  }
+
+  /// Return the `else` elements of this grouping.
+  auto getElseElements() const {
+    return llvm::make_pointee_range(elseElements);
+  }
 
   /// Return the anchor of this optional group.
-  Element *getAnchor() const { return elements[anchor].get(); }
+  Element *getAnchor() const { return thenElements[anchor].get(); }
 
   /// Return the index of the first element that needs to be parsed.
   unsigned getParseStart() const { return parseStart; }
 
 private:
-  /// The child elements of this optional.
-  std::vector<std::unique_ptr<Element>> elements;
+  /// The child elements of `then` branch of this optional.
+  std::vector<std::unique_ptr<Element>> thenElements;
+  /// The child elements of `else` branch of this optional.
+  std::vector<std::unique_ptr<Element>> elseElements;
   /// The index of the element that acts as the anchor for the optional group.
   unsigned anchor;
   /// The index of the first element that is parsed (is not a
@@ -437,6 +449,14 @@ struct OperationFormat {
         llvm::any_of(op.getTraits(), [](const OpTrait &trait) {
           return trait.getDef().isSubClassOf("SingleBlockImplicitTerminator");
         });
+
+    hasSingleBlockTrait =
+        hasImplicitTermTrait ||
+        llvm::any_of(op.getTraits(), [](const OpTrait &trait) {
+          if (auto *native = dyn_cast<NativeOpTrait>(&trait))
+            return native->getTrait() == "::mlir::OpTrait::SingleBlock";
+          return false;
+        });
   }
 
   /// Generate the operation parser from this format.
@@ -472,6 +492,9 @@ struct OperationFormat {
   /// trait.
   bool hasImplicitTermTrait;
 
+  /// A flag indicating if this operation has the SingleBlock trait.
+  bool hasSingleBlockTrait;
+
   /// A map of buildable types to indices.
   llvm::MapVector<StringRef, int, llvm::StringMap<int>> buildableTypes;
 
@@ -667,6 +690,14 @@ const char *regionListEnsureTerminatorParserCode = R"(
     ensureTerminator(*region, parser.getBuilder(), result.location);
 )";
 
+/// The code snippet used to ensure a list of regions have a block.
+///
+/// {0}: The name of the region list.
+const char *regionListEnsureSingleBlockParserCode = R"(
+  for (auto &region : {0}Regions)
+    if (region->empty()) region->emplaceBlock();
+)";
+
 /// The code snippet used to generate a parser call for an optional region.
 ///
 /// {0}: The name of the region.
@@ -693,6 +724,13 @@ const char *regionEnsureTerminatorParserCode = R"(
   ensureTerminator(*{0}Region, parser.getBuilder(), result.location);
 )";
 
+/// The code snippet used to ensure a region has a block.
+///
+/// {0}: The name of the region.
+const char *regionEnsureSingleBlockParserCode = R"(
+  if ({0}Region->empty()) {0}Region->emplaceBlock();
+)";
+
 /// The code snippet used to generate a parser call for a successor list.
 ///
 /// {0}: The name for the successor list.
@@ -792,7 +830,7 @@ static void genLiteralParser(StringRef value, OpMethodBody &body) {
 /// Generate the storage code required for parsing the given element.
 static void genElementParserStorage(Element *element, OpMethodBody &body) {
   if (auto *optional = dyn_cast<OptionalElement>(element)) {
-    auto elements = optional->getElements();
+    auto elements = optional->getThenElements();
 
     // If the anchor is a unit attribute, it won't be parsed directly so elide
     // it.
@@ -803,6 +841,8 @@ static void genElementParserStorage(Element *element, OpMethodBody &body) {
     for (auto &childElement : elements)
       if (&childElement != elidedAnchorElement)
         genElementParserStorage(&childElement, body);
+    for (auto &childElement : optional->getElseElements())
+      genElementParserStorage(&childElement, body);
 
   } else if (auto *custom = dyn_cast<CustomDirective>(element)) {
     for (auto &paramElement : custom->getArguments())
@@ -1094,8 +1134,8 @@ void OperationFormat::genElementParser(Element *element, OpMethodBody &body,
                                        FmtContext &attrTypeCtx) {
   /// Optional Group.
   if (auto *optional = dyn_cast<OptionalElement>(element)) {
-    auto elements =
-        llvm::drop_begin(optional->getElements(), optional->getParseStart());
+    auto elements = llvm::drop_begin(optional->getThenElements(),
+                                     optional->getParseStart());
 
     // Generate a special optional parser for the first element to gate the
     // parsing of the rest of the elements.
@@ -1120,6 +1160,9 @@ void OperationFormat::genElementParser(Element *element, OpMethodBody &body,
         body << "  if (!" << region->name << "Region->empty()) {\n  ";
         if (hasImplicitTermTrait)
           body << llvm::formatv(regionEnsureTerminatorParserCode, region->name);
+        else if (hasSingleBlockTrait)
+          body << llvm::formatv(regionEnsureSingleBlockParserCode,
+                                region->name);
       }
     }
 
@@ -1140,7 +1183,17 @@ void OperationFormat::genElementParser(Element *element, OpMethodBody &body,
       if (&childElement != elidedAnchorElement)
         genElementParser(&childElement, body, attrTypeCtx);
     }
-    body << "  }\n";
+    body << "  }";
+
+    // Generate the else elements.
+    auto elseElements = optional->getElseElements();
+    if (!elseElements.empty()) {
+      body << " else {\n";
+      for (Element &childElement : elseElements)
+        genElementParser(&childElement, body, attrTypeCtx);
+      body << "  }";
+    }
+    body << "\n";
 
     /// Literals.
   } else if (LiteralElement *literal = dyn_cast<LiteralElement>(element)) {
@@ -1193,11 +1246,14 @@ void OperationFormat::genElementParser(Element *element, OpMethodBody &body,
     bool isVariadic = region->getVar()->isVariadic();
     body << llvm::formatv(isVariadic ? regionListParserCode : regionParserCode,
                           region->getVar()->name);
-    if (hasImplicitTermTrait) {
+    if (hasImplicitTermTrait)
       body << llvm::formatv(isVariadic ? regionListEnsureTerminatorParserCode
                                        : regionEnsureTerminatorParserCode,
                             region->getVar()->name);
-    }
+    else if (hasSingleBlockTrait)
+      body << llvm::formatv(isVariadic ? regionListEnsureSingleBlockParserCode
+                                       : regionEnsureSingleBlockParserCode,
+                            region->getVar()->name);
 
   } else if (auto *successor = dyn_cast<SuccessorVariable>(element)) {
     bool isVariadic = successor->getVar()->isVariadic();
@@ -1222,6 +1278,8 @@ void OperationFormat::genElementParser(Element *element, OpMethodBody &body,
     body << llvm::formatv(regionListParserCode, "full");
     if (hasImplicitTermTrait)
       body << llvm::formatv(regionListEnsureTerminatorParserCode, "full");
+    else if (hasSingleBlockTrait)
+      body << llvm::formatv(regionListEnsureSingleBlockParserCode, "full");
 
   } else if (isa<SuccessorsDirective>(element)) {
     body << llvm::formatv(successorListParserCode, "full");
@@ -1778,7 +1836,7 @@ void OperationFormat::genElementPrinter(Element *element, OpMethodBody &body,
 
     // If the anchor is a unit attribute, we don't need to print it. When
     // parsing, we will add this attribute if this group is present.
-    auto elements = optional->getElements();
+    auto elements = optional->getThenElements();
     Element *elidedAnchorElement = nullptr;
     auto *anchorAttr = dyn_cast<AttributeVariable>(anchor);
     if (anchorAttr && anchorAttr != &*elements.begin() &&
@@ -1793,7 +1851,20 @@ void OperationFormat::genElementPrinter(Element *element, OpMethodBody &body,
                           lastWasPunctuation);
       }
     }
-    body << "  }\n";
+    body << "  }";
+
+    // Emit each of the else elements.
+    auto elseElements = optional->getElseElements();
+    if (!elseElements.empty()) {
+      body << " else {\n";
+      for (Element &childElement : elseElements) {
+        genElementPrinter(&childElement, body, op, shouldEmitSpace,
+                          lastWasPunctuation);
+      }
+      body << "  }";
+    }
+
+    body << "\n";
     return;
   }
 
@@ -1911,6 +1982,7 @@ public:
     l_paren,
     r_paren,
     caret,
+    colon,
     comma,
     equal,
     less,
@@ -2065,6 +2137,8 @@ Token FormatLexer::lexToken() {
   // Lex punctuation.
   case '^':
     return formToken(Token::caret, tokStart);
+  case ':':
+    return formToken(Token::colon, tokStart);
   case ',':
     return formToken(Token::comma, tokStart);
   case '=':
@@ -2393,8 +2467,11 @@ LogicalResult FormatParser::verifyAttributes(
 
     // Traverse into optional groups.
     if (auto *optional = dyn_cast<OptionalElement>(element)) {
-      auto elements = optional->getElements();
-      iteratorStack.emplace_back(elements.begin(), elements.end());
+      auto thenElements = optional->getThenElements();
+      iteratorStack.emplace_back(thenElements.begin(), thenElements.end());
+
+      auto elseElements = optional->getElseElements();
+      iteratorStack.emplace_back(elseElements.begin(), elseElements.end());
       return ::mlir::success();
     }
 
@@ -2795,13 +2872,31 @@ LogicalResult FormatParser::parseOptional(std::unique_ptr<Element> &element,
   consumeToken();
 
   // Parse the child elements for this optional group.
-  std::vector<std::unique_ptr<Element>> elements;
+  std::vector<std::unique_ptr<Element>> thenElements, elseElements;
   Optional<unsigned> anchorIdx;
   do {
-    if (failed(parseOptionalChildElement(elements, anchorIdx)))
+    if (failed(parseOptionalChildElement(thenElements, anchorIdx)))
       return ::mlir::failure();
   } while (curToken.getKind() != Token::r_paren);
   consumeToken();
+
+  // Parse the `else` elements of this optional group.
+  if (curToken.getKind() == Token::colon) {
+    consumeToken();
+    if (failed(parseToken(Token::l_paren, "expected '(' to start else branch "
+                                          "of optional group")))
+      return failure();
+    do {
+      llvm::SMLoc childLoc = curToken.getLoc();
+      elseElements.push_back({});
+      if (failed(parseElement(elseElements.back(), TopLevelContext)) ||
+          failed(verifyOptionalChildElement(elseElements.back().get(), childLoc,
+                                            /*isAnchor=*/false)))
+        return failure();
+    } while (curToken.getKind() != Token::r_paren);
+    consumeToken();
+  }
+
   if (failed(parseToken(Token::question, "expected '?' after optional group")))
     return ::mlir::failure();
 
@@ -2811,7 +2906,7 @@ LogicalResult FormatParser::parseOptional(std::unique_ptr<Element> &element,
 
   // The first parsable element of the group must be able to be parsed in an
   // optional fashion.
-  auto parseBegin = llvm::find_if_not(elements, [](auto &element) {
+  auto parseBegin = llvm::find_if_not(thenElements, [](auto &element) {
     return isa<WhitespaceElement>(element.get());
   });
   Element *firstElement = parseBegin->get();
@@ -2822,9 +2917,9 @@ LogicalResult FormatParser::parseOptional(std::unique_ptr<Element> &element,
                      "first parsable element of an operand group must be "
                      "an attribute, literal, operand, or region");
 
-  auto parseStart = parseBegin - elements.begin();
-  element = std::make_unique<OptionalElement>(std::move(elements), *anchorIdx,
-                                              parseStart);
+  auto parseStart = parseBegin - thenElements.begin();
+  element = std::make_unique<OptionalElement>(
+      std::move(thenElements), std::move(elseElements), *anchorIdx, parseStart);
   return ::mlir::success();
 }
 
diff --git a/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp b/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
index 2f8f22ca3f225871e9d063a56d085562017664a6..d7692d2b6f5e15f0fab85d18c4ad7f1bd82f15b2 100644
--- a/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
@@ -43,9 +43,13 @@ static void emitMethodNameAndArgs(const InterfaceMethod &method,
                                   raw_ostream &os, StringRef valueType,
                                   bool addThisArg, bool addConst) {
   os << method.getName() << '(';
-  if (addThisArg)
+  if (addThisArg) {
+    if (addConst)
+      os << "const ";
+    os << "const Concept *impl, ";
     emitCPPType(valueType, os)
         << "tablegen_opaque_val" << (method.arg_empty() ? "" : ", ");
+  }
   llvm::interleaveComma(method.getArguments(), os,
                         [&](const InterfaceMethod::Argument &arg) {
                           os << arg.type << " " << arg.name;
@@ -124,7 +128,9 @@ struct OpInterfaceGenerator : public InterfaceGenerator {
     interfaceBaseType = "OpInterface";
     valueTemplate = "ConcreteOp";
     StringRef castCode = "(llvm::cast<ConcreteOp>(tablegen_opaque_val))";
-    nonStaticMethodFmt.withOp(castCode).withSelf(castCode);
+    nonStaticMethodFmt.addSubst("_this", "impl")
+        .withOp(castCode)
+        .withSelf(castCode);
     traitMethodFmt.withOp("(*static_cast<ConcreteOp *>(this))");
   }
 };
@@ -167,6 +173,7 @@ static void emitInterfaceDef(Interface interface, StringRef valueType,
     // Forward to the method on the concrete operation type.
     os << " {\n      return getImpl()->" << method.getName() << '(';
     if (!method.isStatic()) {
+      os << "getImpl(), ";
       os << (isOpInterface ? "getOperation()" : "*this");
       os << (method.arg_empty() ? "" : ", ");
     }
@@ -197,8 +204,10 @@ void InterfaceGenerator::emitConceptDecl(Interface &interface) {
     os << "    ";
     emitCPPType(method.getReturnType(), os);
     os << "(*" << method.getName() << ")(";
-    if (!method.isStatic())
+    if (!method.isStatic()) {
+      os << "const Concept *impl, ";
       emitCPPType(valueType, os) << (method.arg_empty() ? "" : ", ");
+    }
     llvm::interleaveComma(
         method.getArguments(), os,
         [&](const InterfaceMethod::Argument &arg) { os << arg.type; });
@@ -208,23 +217,25 @@ void InterfaceGenerator::emitConceptDecl(Interface &interface) {
 }
 
 void InterfaceGenerator::emitModelDecl(Interface &interface) {
-  os << "  template<typename " << valueTemplate << ">\n";
-  os << "  class Model : public Concept {\n  public:\n";
-  os << "    Model() : Concept{";
-  llvm::interleaveComma(
-      interface.getMethods(), os,
-      [&](const InterfaceMethod &method) { os << method.getName(); });
-  os << "} {}\n\n";
-
-  // Insert each of the virtual method overrides.
-  for (auto &method : interface.getMethods()) {
-    emitCPPType(method.getReturnType(), os << "    static inline ");
-    emitMethodNameAndArgs(method, os, valueType,
-                          /*addThisArg=*/!method.isStatic(),
-                          /*addConst=*/false);
-    os << ";\n";
+  for (const char *modelClass : {"Model", "FallbackModel"}) {
+    os << "  template<typename " << valueTemplate << ">\n";
+    os << "  class " << modelClass << " : public Concept {\n  public:\n";
+    os << "    " << modelClass << "() : Concept{";
+    llvm::interleaveComma(
+        interface.getMethods(), os,
+        [&](const InterfaceMethod &method) { os << method.getName(); });
+    os << "} {}\n\n";
+
+    // Insert each of the virtual method overrides.
+    for (auto &method : interface.getMethods()) {
+      emitCPPType(method.getReturnType(), os << "    static inline ");
+      emitMethodNameAndArgs(method, os, valueType,
+                            /*addThisArg=*/!method.isStatic(),
+                            /*addConst=*/false);
+      os << ";\n";
+    }
+    os << "  };\n";
   }
-  os << "  };\n";
 }
 
 void InterfaceGenerator::emitModelMethodsDef(Interface &interface) {
@@ -261,6 +272,32 @@ void InterfaceGenerator::emitModelMethodsDef(Interface &interface) {
         [&](const InterfaceMethod::Argument &arg) { os << arg.name; });
     os << ");\n}\n";
   }
+
+  for (auto &method : interface.getMethods()) {
+    os << "template<typename " << valueTemplate << ">\n";
+    emitCPPType(method.getReturnType(), os);
+    os << "detail::" << interface.getName() << "InterfaceTraits::FallbackModel<"
+       << valueTemplate << ">::";
+    emitMethodNameAndArgs(method, os, valueType,
+                          /*addThisArg=*/!method.isStatic(),
+                          /*addConst=*/false);
+    os << " {\n  ";
+
+    // Forward to the method on the concrete Model implementation.
+    if (method.isStatic())
+      os << "return " << valueTemplate << "::";
+    else
+      os << "return static_cast<const " << valueTemplate << " *>(impl)->";
+
+    // Add the arguments to the call.
+    os << method.getName() << '(';
+    if (!method.isStatic())
+      os << "tablegen_opaque_val" << (method.arg_empty() ? "" : ", ");
+    llvm::interleaveComma(
+        method.getArguments(), os,
+        [&](const InterfaceMethod::Argument &arg) { os << arg.name; });
+    os << ");\n}\n";
+  }
 }
 
 void InterfaceGenerator::emitTraitDecl(Interface &interface,
diff --git a/mlir/tools/mlir-tblgen/RewriterGen.cpp b/mlir/tools/mlir-tblgen/RewriterGen.cpp
index 5781870e0df793f55f5082717708b93dc98bc96c..28889de1ea607357d053b778d690b9f310955a1f 100644
--- a/mlir/tools/mlir-tblgen/RewriterGen.cpp
+++ b/mlir/tools/mlir-tblgen/RewriterGen.cpp
@@ -251,12 +251,8 @@ void PatternEmitter::emitNativeCodeMatch(DagNode tree, StringRef opName,
 
   // TODO(suderman): iterate through arguments, determine their types, output
   // names.
-  SmallVector<std::string, 8> capture(8);
-  if (tree.getNumArgs() > 8) {
-    PrintFatalError(loc,
-                    "unsupported NativeCodeCall matcher argument numbers: " +
-                        Twine(tree.getNumArgs()));
-  }
+  SmallVector<std::string, 8> capture;
+  capture.push_back(opName.str());
 
   raw_indented_ostream::DelimitedScope scope(os);
 
@@ -274,7 +270,7 @@ void PatternEmitter::emitNativeCodeMatch(DagNode tree, StringRef opName,
       }
     }
 
-    capture[i] = std::move(argName);
+    capture.push_back(std::move(argName));
   }
 
   bool hasLocationDirective;
@@ -282,21 +278,20 @@ void PatternEmitter::emitNativeCodeMatch(DagNode tree, StringRef opName,
   std::tie(hasLocationDirective, locToUse) = getLocation(tree);
 
   auto fmt = tree.getNativeCodeTemplate();
-  auto nativeCodeCall = std::string(tgfmt(
-      fmt, &fmtCtx.addSubst("_loc", locToUse), opName, capture[0], capture[1],
-      capture[2], capture[3], capture[4], capture[5], capture[6], capture[7]));
+  auto nativeCodeCall =
+      std::string(tgfmt(fmt, &fmtCtx.addSubst("_loc", locToUse), capture));
 
   os << "if (failed(" << nativeCodeCall << ")) return ::mlir::failure();\n";
 
   for (int i = 0, e = tree.getNumArgs(); i != e; ++i) {
     auto name = tree.getArgName(i);
     if (!name.empty() && name != "_") {
-      os << formatv("{0} = {1};\n", name, capture[i]);
+      os << formatv("{0} = {1};\n", name, capture[i + 1]);
     }
   }
 
   for (int i = 0, e = tree.getNumArgs(); i != e; ++i) {
-    std::string argName = capture[i];
+    std::string argName = capture[i + 1];
 
     // Handle nested DAG construct first
     if (DagNode argTree = tree.getArgAsNestedDag(i)) {
@@ -631,8 +626,8 @@ void PatternEmitter::emit(StringRef rewriteName) {
                 make_range(locs.rbegin(), locs.rend()));
   os << formatv(R"(struct {0} : public ::mlir::RewritePattern {
   {0}(::mlir::MLIRContext *context)
-      : ::mlir::RewritePattern("{1}", {{)",
-                rewriteName, rootName);
+      : ::mlir::RewritePattern("{1}", {2}, context, {{)",
+                rewriteName, rootName, pattern.getBenefit());
   // Sort result operators by name.
   llvm::SmallVector<const Operator *, 4> sortedResultOps(resultOps.begin(),
                                                          resultOps.end());
@@ -642,7 +637,7 @@ void PatternEmitter::emit(StringRef rewriteName) {
   llvm::interleaveComma(sortedResultOps, os, [&](const Operator *op) {
     os << '"' << op->getOperationName() << '"';
   });
-  os << formatv(R"(}, {0}, context) {{})", pattern.getBenefit()) << "\n";
+  os << "}) {}\n";
 
   // Emit matchAndRewrite() function.
   {
@@ -915,29 +910,26 @@ std::string PatternEmitter::handleReplaceWithNativeCodeCall(DagNode tree,
   LLVM_DEBUG(llvm::dbgs() << '\n');
 
   auto fmt = tree.getNativeCodeTemplate();
-  // TODO: replace formatv arguments with the exact specified args.
-  SmallVector<std::string, 8> attrs(8);
-  if (tree.getNumArgs() > 8) {
-    PrintFatalError(loc,
-                    "unsupported NativeCodeCall replace argument numbers: " +
-                        Twine(tree.getNumArgs()));
-  }
+
+  SmallVector<std::string, 16> attrs;
+
   bool hasLocationDirective;
   std::string locToUse;
   std::tie(hasLocationDirective, locToUse) = getLocation(tree);
 
   for (int i = 0, e = tree.getNumArgs() - hasLocationDirective; i != e; ++i) {
     if (tree.isNestedDagArg(i)) {
-      attrs[i] = handleResultPattern(tree.getArgAsNestedDag(i), i, depth + 1);
+      attrs.push_back(
+          handleResultPattern(tree.getArgAsNestedDag(i), i, depth + 1));
     } else {
-      attrs[i] = handleOpArgument(tree.getArgAsLeaf(i), tree.getArgName(i));
+      attrs.push_back(
+          handleOpArgument(tree.getArgAsLeaf(i), tree.getArgName(i)));
     }
     LLVM_DEBUG(llvm::dbgs() << "NativeCodeCall argument #" << i
                             << " replacement: " << attrs[i] << "\n");
   }
-  return std::string(tgfmt(fmt, &fmtCtx.addSubst("_loc", locToUse), attrs[0],
-                           attrs[1], attrs[2], attrs[3], attrs[4], attrs[5],
-                           attrs[6], attrs[7]));
+
+  return std::string(tgfmt(fmt, &fmtCtx.addSubst("_loc", locToUse), attrs));
 }
 
 int PatternEmitter::getNodeValueCount(DagNode node) {
@@ -1298,10 +1290,10 @@ static void emitRewriters(const RecordKeeper &recordKeeper, raw_ostream &os) {
   }
 
   // Emit function to add the generated matchers to the pattern list.
-  os << "void LLVM_ATTRIBUTE_UNUSED populateWithGenerated(::mlir::MLIRContext "
-        "*context, ::mlir::OwningRewritePatternList &patterns) {\n";
+  os << "void LLVM_ATTRIBUTE_UNUSED populateWithGenerated("
+        "::mlir::RewritePatternSet &patterns) {\n";
   for (const auto &name : rewriterNames) {
-    os << "  patterns.insert<" << name << ">(context);\n";
+    os << "  patterns.add<" << name << ">(patterns.getContext());\n";
   }
   os << "}\n";
 }
diff --git a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
index bb02a7993e43b4f48688a7cd35499833e38e07c8..294432a0600a9ce7e096ad30883ba774703dca15 100644
--- a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
+++ b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
@@ -171,7 +171,7 @@ static void emitInterfaceDef(const Availability &availability,
   StringRef methodName = availability.getQueryFnName();
   os << availability.getQueryFnRetType() << " "
      << availability.getInterfaceClassName() << "::" << methodName << "() {\n"
-     << "  return getImpl()->" << methodName << "(getOperation());\n"
+     << "  return getImpl()->" << methodName << "(getImpl(), getOperation());\n"
      << "}\n";
 }
 
@@ -208,23 +208,25 @@ static void emitConceptDecl(const Availability &availability, raw_ostream &os) {
      << "    virtual ~Concept() = default;\n"
      << "    virtual " << availability.getQueryFnRetType() << " "
      << availability.getQueryFnName()
-     << "(Operation *tblgen_opaque_op) const = 0;\n"
+     << "(const Concept *impl, Operation *tblgen_opaque_op) const = 0;\n"
      << "  };\n";
 }
 
 static void emitModelDecl(const Availability &availability, raw_ostream &os) {
-  os << "  template<typename ConcreteOp>\n";
-  os << "  class Model : public Concept {\n"
-     << "  public:\n"
-     << "    " << availability.getQueryFnRetType() << " "
-     << availability.getQueryFnName()
-     << "(Operation *tblgen_opaque_op) const final {\n"
-     << "      auto op = llvm::cast<ConcreteOp>(tblgen_opaque_op);\n"
-     << "      (void)op;\n"
-     // Forward to the method on the concrete operation type.
-     << "      return op." << availability.getQueryFnName() << "();\n"
-     << "    }\n"
-     << "  };\n";
+  for (const char *modelClass : {"Model", "FallbackModel"}) {
+    os << "  template<typename ConcreteOp>\n";
+    os << "  class " << modelClass << " : public Concept {\n"
+       << "  public:\n"
+       << "    " << availability.getQueryFnRetType() << " "
+       << availability.getQueryFnName()
+       << "(const Concept *impl, Operation *tblgen_opaque_op) const final {\n"
+       << "      auto op = llvm::cast<ConcreteOp>(tblgen_opaque_op);\n"
+       << "      (void)op;\n"
+       // Forward to the method on the concrete operation type.
+       << "      return op." << availability.getQueryFnName() << "();\n"
+       << "    }\n"
+       << "  };\n";
+  }
 }
 
 static void emitInterfaceDecl(const Availability &availability,
diff --git a/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp b/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp
index 732c966605bfd4c1f1142f59ec9f0a5b5e6ce3ea..1ef697768ba25472cc0d59f818365895c67f761f 100644
--- a/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp
+++ b/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp
@@ -46,10 +46,8 @@ static LogicalResult runMLIRPasses(ModuleOp module) {
   modulePM.addPass(spirv::createLowerABIAttributesPass());
   modulePM.addPass(spirv::createUpdateVersionCapabilityExtensionPass());
   passManager.addPass(createConvertGpuLaunchFuncToVulkanLaunchFuncPass());
-  LowerToLLVMOptions llvmOptions = {
-      /*useBarePtrCallConv =*/false,
-      /*emitCWrappers = */ true,
-      /*indexBitwidth =*/kDeriveIndexBitwidthFromDataLayout};
+  LowerToLLVMOptions llvmOptions(module.getContext(), DataLayout(module));
+  llvmOptions.emitCWrappers = true;
   passManager.addPass(createLowerToLLVMPass(llvmOptions));
   passManager.addPass(createConvertVulkanLaunchFuncToVulkanCallsPass());
   return passManager.run(module);
diff --git a/mlir/unittests/CMakeLists.txt b/mlir/unittests/CMakeLists.txt
index 9dbf3bfb4d4e14f26e0c7359afabbbf058e4c180..a8e9212ee2559742e33438780c7dc096c25b2d91 100644
--- a/mlir/unittests/CMakeLists.txt
+++ b/mlir/unittests/CMakeLists.txt
@@ -10,5 +10,6 @@ add_subdirectory(ExecutionEngine)
 add_subdirectory(Interfaces)
 add_subdirectory(IR)
 add_subdirectory(Pass)
+add_subdirectory(Rewrite)
 add_subdirectory(SDBM)
 add_subdirectory(TableGen)
diff --git a/mlir/unittests/IR/CMakeLists.txt b/mlir/unittests/IR/CMakeLists.txt
index af21c1b8b43b45a913134f52da531b5749da0273..81be1831ff0b7587dc1eeb8ea122eadded41bfbc 100644
--- a/mlir/unittests/IR/CMakeLists.txt
+++ b/mlir/unittests/IR/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_mlir_unittest(MLIRIRTests
   AttributeTest.cpp
   DialectTest.cpp
+  MemRefTypeTest.cpp
   OperationSupportTest.cpp
   ShapedTypeTest.cpp
 )
diff --git a/mlir/unittests/IR/MemRefTypeTest.cpp b/mlir/unittests/IR/MemRefTypeTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b694e65062df8fbce99897c96e2d5eef47ee2653
--- /dev/null
+++ b/mlir/unittests/IR/MemRefTypeTest.cpp
@@ -0,0 +1,50 @@
+//===- MemRefTypeTest.cpp - MemRefType unit tests -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/AffineMap.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "gtest/gtest.h"
+
+using namespace mlir;
+using namespace mlir::detail;
+
+namespace {
+
+TEST(MemRefTypeTest, GetStridesAndOffset) {
+  MLIRContext context;
+
+  SmallVector<int64_t> shape({2, 3, 4});
+  Type f32 = FloatType::getF32(&context);
+
+  AffineMap map1 = makeStridedLinearLayoutMap({12, 4, 1}, 5, &context);
+  MemRefType type1 = MemRefType::get(shape, f32, {map1});
+  SmallVector<int64_t> strides1;
+  int64_t offset1 = -1;
+  LogicalResult res1 = getStridesAndOffset(type1, strides1, offset1);
+  ASSERT_TRUE(res1.succeeded());
+  ASSERT_EQ(3u, strides1.size());
+  EXPECT_EQ(12, strides1[0]);
+  EXPECT_EQ(4, strides1[1]);
+  EXPECT_EQ(1, strides1[2]);
+  ASSERT_EQ(5, offset1);
+
+  AffineMap map2 = AffineMap::getPermutationMap({1, 2, 0}, &context);
+  AffineMap map3 = makeStridedLinearLayoutMap({8, 2, 1}, 0, &context);
+  MemRefType type2 = MemRefType::get(shape, f32, {map2, map3});
+  SmallVector<int64_t> strides2;
+  int64_t offset2 = -1;
+  LogicalResult res2 = getStridesAndOffset(type2, strides2, offset2);
+  ASSERT_TRUE(res2.succeeded());
+  ASSERT_EQ(3u, strides2.size());
+  EXPECT_EQ(1, strides2[0]);
+  EXPECT_EQ(8, strides2[1]);
+  EXPECT_EQ(2, strides2[2]);
+  ASSERT_EQ(0, offset2);
+}
+
+} // end namespace
diff --git a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp
index 88a52f673107d4252a0a538cbf9eb3058279cd01..287839120ab269e2586fd6b2aea564177a5a50af 100644
--- a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp
+++ b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp
@@ -71,8 +71,8 @@ struct SingleQueryType
 
   static SingleQueryType get(MLIRContext *ctx) { return Base::get(ctx); }
 
-  unsigned getTypeSize(const DataLayout &layout,
-                       DataLayoutEntryListRef params) {
+  unsigned getTypeSizeInBits(const DataLayout &layout,
+                             DataLayoutEntryListRef params) const {
     static bool executed = false;
     if (executed)
       llvm::report_fatal_error("repeated call");
@@ -121,19 +121,20 @@ struct OpWithLayout : public Op<OpWithLayout, DataLayoutOpInterface::Trait> {
     return getOperation()->getAttrOfType<DataLayoutSpecInterface>(kAttrName);
   }
 
-  static unsigned getTypeSize(Type type, const DataLayout &dataLayout,
-                              DataLayoutEntryListRef params) {
+  static unsigned getTypeSizeInBits(Type type, const DataLayout &dataLayout,
+                                    DataLayoutEntryListRef params) {
     // Make a recursive query.
     if (type.isa<FloatType>())
-      return dataLayout.getTypeSize(
+      return dataLayout.getTypeSizeInBits(
           IntegerType::get(type.getContext(), type.getIntOrFloatBitWidth()));
 
     // Handle built-in types that are not handled by the default process.
     if (auto iType = type.dyn_cast<IntegerType>()) {
       for (DataLayoutEntryInterface entry : params)
         if (entry.getKey().dyn_cast<Type>() == type)
-          return entry.getValue().cast<IntegerAttr>().getValue().getZExtValue();
-      return iType.getIntOrFloatBitWidth();
+          return 8 *
+                 entry.getValue().cast<IntegerAttr>().getValue().getZExtValue();
+      return 8 * iType.getIntOrFloatBitWidth();
     }
 
     // Use the default process for everything else.
@@ -152,13 +153,30 @@ struct OpWithLayout : public Op<OpWithLayout, DataLayoutOpInterface::Trait> {
   }
 };
 
+struct OpWith7BitByte
+    : public Op<OpWith7BitByte, DataLayoutOpInterface::Trait> {
+  using Op::Op;
+
+  static StringRef getOperationName() { return "dltest.op_with_7bit_byte"; }
+
+  DataLayoutSpecInterface getDataLayoutSpec() {
+    return getOperation()->getAttrOfType<DataLayoutSpecInterface>(kAttrName);
+  }
+
+  // Bytes are assumed to be 7-bit here.
+  static unsigned getTypeSize(Type type, const DataLayout &dataLayout,
+                              DataLayoutEntryListRef params) {
+    return llvm::divideCeil(dataLayout.getTypeSizeInBits(type), 7);
+  }
+};
+
 /// A dialect putting all the above together.
 struct DLTestDialect : Dialect {
   explicit DLTestDialect(MLIRContext *ctx)
       : Dialect(getDialectNamespace(), ctx, TypeID::get<DLTestDialect>()) {
     ctx->getOrLoadDialect<DLTIDialect>();
     addAttributes<CustomDataLayoutSpec>();
-    addOperations<OpWithLayout>();
+    addOperations<OpWithLayout, OpWith7BitByte>();
     addTypes<SingleQueryType, TypeNoLayout>();
   }
   static StringRef getDialectNamespace() { return "dltest"; }
@@ -209,7 +227,7 @@ struct DLTestDialect : Dialect {
 
 TEST(DataLayout, FallbackDefault) {
   const char *ir = R"MLIR(
-"dltest.op_with_layout"() : () -> ()
+module {}
   )MLIR";
 
   DialectRegistry registry;
@@ -217,17 +235,40 @@ TEST(DataLayout, FallbackDefault) {
   MLIRContext ctx(registry);
 
   OwningModuleRef module = parseSourceString(ir, &ctx);
-  auto op =
-      cast<DataLayoutOpInterface>(module->getBody()->getOperations().front());
-  DataLayout layout(op);
+  DataLayout layout(module.get());
   EXPECT_EQ(layout.getTypeSize(IntegerType::get(&ctx, 42)), 6u);
   EXPECT_EQ(layout.getTypeSize(Float16Type::get(&ctx)), 2u);
+  EXPECT_EQ(layout.getTypeSizeInBits(IntegerType::get(&ctx, 42)), 42u);
+  EXPECT_EQ(layout.getTypeSizeInBits(Float16Type::get(&ctx)), 16u);
   EXPECT_EQ(layout.getTypeABIAlignment(IntegerType::get(&ctx, 42)), 8u);
   EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 2u);
   EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 8u);
   EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 2u);
 }
 
+TEST(DataLayout, NullSpec) {
+  const char *ir = R"MLIR(
+"dltest.op_with_layout"() : () -> ()
+  )MLIR";
+
+  DialectRegistry registry;
+  registry.insert<DLTIDialect, DLTestDialect>();
+  MLIRContext ctx(registry);
+
+  OwningModuleRef module = parseSourceString(ir, &ctx);
+  auto op =
+      cast<DataLayoutOpInterface>(module->getBody()->getOperations().front());
+  DataLayout layout(op);
+  EXPECT_EQ(layout.getTypeSize(IntegerType::get(&ctx, 42)), 42u);
+  EXPECT_EQ(layout.getTypeSize(Float16Type::get(&ctx)), 16u);
+  EXPECT_EQ(layout.getTypeSizeInBits(IntegerType::get(&ctx, 42)), 8u * 42u);
+  EXPECT_EQ(layout.getTypeSizeInBits(Float16Type::get(&ctx)), 8u * 16u);
+  EXPECT_EQ(layout.getTypeABIAlignment(IntegerType::get(&ctx, 42)), 64u);
+  EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 16u);
+  EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 128u);
+  EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 32u);
+}
+
 TEST(DataLayout, EmptySpec) {
   const char *ir = R"MLIR(
 "dltest.op_with_layout"() { dltest.layout = #dltest.spec< > } : () -> ()
@@ -243,6 +284,8 @@ TEST(DataLayout, EmptySpec) {
   DataLayout layout(op);
   EXPECT_EQ(layout.getTypeSize(IntegerType::get(&ctx, 42)), 42u);
   EXPECT_EQ(layout.getTypeSize(Float16Type::get(&ctx)), 16u);
+  EXPECT_EQ(layout.getTypeSizeInBits(IntegerType::get(&ctx, 42)), 8u * 42u);
+  EXPECT_EQ(layout.getTypeSizeInBits(Float16Type::get(&ctx)), 8u * 16u);
   EXPECT_EQ(layout.getTypeABIAlignment(IntegerType::get(&ctx, 42)), 64u);
   EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 16u);
   EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 128u);
@@ -267,6 +310,8 @@ TEST(DataLayout, SpecWithEntries) {
   DataLayout layout(op);
   EXPECT_EQ(layout.getTypeSize(IntegerType::get(&ctx, 42)), 5u);
   EXPECT_EQ(layout.getTypeSize(Float16Type::get(&ctx)), 6u);
+  EXPECT_EQ(layout.getTypeSizeInBits(IntegerType::get(&ctx, 42)), 40u);
+  EXPECT_EQ(layout.getTypeSizeInBits(Float16Type::get(&ctx)), 48u);
   EXPECT_EQ(layout.getTypeABIAlignment(IntegerType::get(&ctx, 42)), 8u);
   EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 8u);
   EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 16u);
@@ -274,6 +319,8 @@ TEST(DataLayout, SpecWithEntries) {
 
   EXPECT_EQ(layout.getTypeSize(IntegerType::get(&ctx, 32)), 32u);
   EXPECT_EQ(layout.getTypeSize(Float32Type::get(&ctx)), 32u);
+  EXPECT_EQ(layout.getTypeSizeInBits(IntegerType::get(&ctx, 32)), 256u);
+  EXPECT_EQ(layout.getTypeSizeInBits(Float32Type::get(&ctx)), 256u);
   EXPECT_EQ(layout.getTypeABIAlignment(IntegerType::get(&ctx, 32)), 32u);
   EXPECT_EQ(layout.getTypeABIAlignment(Float32Type::get(&ctx)), 32u);
   EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 32)), 64u);
@@ -355,3 +402,23 @@ TEST(DataLayout, UnimplementedTypeInterface) {
                "neither the scoping op nor the type class provide data layout "
                "information");
 }
+
+TEST(DataLayout, SevenBitByte) {
+  const char *ir = R"MLIR(
+"dltest.op_with_7bit_byte"() { dltest.layout = #dltest.spec<> } : () -> ()
+  )MLIR";
+
+  DialectRegistry registry;
+  registry.insert<DLTIDialect, DLTestDialect>();
+  MLIRContext ctx(registry);
+
+  OwningModuleRef module = parseSourceString(ir, &ctx);
+  auto op =
+      cast<DataLayoutOpInterface>(module->getBody()->getOperations().front());
+  DataLayout layout(op);
+
+  EXPECT_EQ(layout.getTypeSizeInBits(IntegerType::get(&ctx, 42)), 42u);
+  EXPECT_EQ(layout.getTypeSizeInBits(IntegerType::get(&ctx, 32)), 32u);
+  EXPECT_EQ(layout.getTypeSize(IntegerType::get(&ctx, 42)), 6u);
+  EXPECT_EQ(layout.getTypeSize(IntegerType::get(&ctx, 32)), 5u);
+}
diff --git a/mlir/unittests/Rewrite/CMakeLists.txt b/mlir/unittests/Rewrite/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c0df7d4eee85578e9f432e6dbef485a1137c3105
--- /dev/null
+++ b/mlir/unittests/Rewrite/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_mlir_unittest(MLIRRewriteTests
+  PatternBenefit.cpp
+)
+target_link_libraries(MLIRRewriteTests
+  PRIVATE
+  MLIRRewrite
+  MLIRTransformUtils)
diff --git a/mlir/unittests/Rewrite/PatternBenefit.cpp b/mlir/unittests/Rewrite/PatternBenefit.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..86b1aa13a8cac0ea3b9f9d4c6dc612902035028b
--- /dev/null
+++ b/mlir/unittests/Rewrite/PatternBenefit.cpp
@@ -0,0 +1,79 @@
+//===- PatternBenefit.cpp - RewritePattern benefit unit tests -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Rewrite/PatternApplicator.h"
+#include "gtest/gtest.h"
+
+using namespace mlir;
+
+namespace {
+TEST(PatternBenefitTest, BenefitOrder) {
+  // There was a bug which caused low-benefit op-specific patterns to never be
+  // called in presence of high-benefit op-agnostic pattern
+
+  MLIRContext context;
+
+  OpBuilder builder(&context);
+  auto module = ModuleOp::create(builder.getUnknownLoc());
+
+  struct Pattern1 : public OpRewritePattern<ModuleOp> {
+    Pattern1(mlir::MLIRContext *context, bool *called)
+        : OpRewritePattern<ModuleOp>(context, /*benefit*/ 1), called(called) {}
+
+    mlir::LogicalResult
+    matchAndRewrite(ModuleOp /*op*/,
+                    mlir::PatternRewriter & /*rewriter*/) const override {
+      *called = true;
+      return failure();
+    }
+
+  private:
+    bool *called;
+  };
+
+  struct Pattern2 : public RewritePattern {
+    Pattern2(MLIRContext *context, bool *called)
+        : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/2, context),
+          called(called) {}
+
+    mlir::LogicalResult
+    matchAndRewrite(Operation * /*op*/,
+                    mlir::PatternRewriter & /*rewriter*/) const override {
+      *called = true;
+      return failure();
+    }
+
+  private:
+    bool *called;
+  };
+
+  RewritePatternSet patterns(&context);
+
+  bool called1 = false;
+  bool called2 = false;
+
+  patterns.add<Pattern1>(&context, &called1);
+  patterns.add<Pattern2>(&context, &called2);
+
+  FrozenRewritePatternSet frozenPatterns(std::move(patterns));
+  PatternApplicator pa(frozenPatterns);
+  pa.applyDefaultCostModel();
+
+  class MyPatternRewriter : public PatternRewriter {
+  public:
+    MyPatternRewriter(MLIRContext *ctx) : PatternRewriter(ctx) {}
+  };
+
+  MyPatternRewriter rewriter(&context);
+  (void)pa.matchAndRewrite(module, rewriter);
+
+  EXPECT_TRUE(called1);
+  EXPECT_TRUE(called2);
+}
+} // namespace
diff --git a/mlir/unittests/TableGen/EnumsGenTest.cpp b/mlir/unittests/TableGen/EnumsGenTest.cpp
index a5580197a0a399c1d734079fb91d20680c3b39f6..a873658cdc3ab0608c7c467afd8fc2e4664ce8ae 100644
--- a/mlir/unittests/TableGen/EnumsGenTest.cpp
+++ b/mlir/unittests/TableGen/EnumsGenTest.cpp
@@ -6,21 +6,29 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/MLIRContext.h"
 #include "mlir/Support/LLVM.h"
+
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+
 #include "gmock/gmock.h"
+
 #include <type_traits>
 
 /// Pull in generated enum utility declarations and definitions.
 #include "EnumsGenTest.h.inc"
+
 #include "EnumsGenTest.cpp.inc"
 
 /// Test namespaces and enum class/utility names.
 using Outer::Inner::ConvertToEnum;
 using Outer::Inner::ConvertToString;
 using Outer::Inner::StrEnum;
+using Outer::Inner::StrEnumAttr;
 
 TEST(EnumsGenTest, GeneratedStrEnumDefinition) {
   EXPECT_EQ(0u, static_cast<uint64_t>(StrEnum::CaseA));
@@ -110,3 +118,41 @@ TEST(EnumsGenTest, GeneratedCustomStringToSymbolFn) {
   auto none = symbolizePrettyIntEnum("Case1");
   EXPECT_FALSE(none);
 }
+
+TEST(EnumsGenTest, GeneratedIntAttributeClass) {
+  mlir::MLIRContext ctx;
+  I32Enum rawVal = I32Enum::Case5;
+
+  I32EnumAttr enumAttr = I32EnumAttr::get(&ctx, rawVal);
+  EXPECT_NE(enumAttr, nullptr);
+  EXPECT_EQ(enumAttr.getValue(), rawVal);
+
+  mlir::Type intType = mlir::IntegerType::get(&ctx, 32);
+  mlir::Attribute intAttr = mlir::IntegerAttr::get(intType, 5);
+  EXPECT_TRUE(intAttr.isa<I32EnumAttr>());
+  EXPECT_EQ(intAttr, enumAttr);
+}
+
+TEST(EnumsGenTest, GeneratedStringAttributeClass) {
+  mlir::MLIRContext ctx;
+  StrEnum rawVal = StrEnum::CaseA;
+
+  StrEnumAttr enumAttr = StrEnumAttr::get(&ctx, rawVal);
+  EXPECT_NE(enumAttr, nullptr);
+  EXPECT_EQ(enumAttr.getValue(), rawVal);
+
+  mlir::Attribute strAttr = mlir::StringAttr::get(&ctx, "CaseA");
+  EXPECT_TRUE(strAttr.isa<StrEnumAttr>());
+  EXPECT_EQ(strAttr, enumAttr);
+}
+
+TEST(EnumsGenTest, GeneratedBitAttributeClass) {
+  mlir::MLIRContext ctx;
+
+  mlir::Type intType = mlir::IntegerType::get(&ctx, 32);
+  mlir::Attribute intAttr = mlir::IntegerAttr::get(
+      intType,
+      static_cast<uint32_t>(BitEnumWithNone::Bit1 | BitEnumWithNone::Bit3));
+  EXPECT_TRUE(intAttr.isa<BitEnumWithNoneAttr>());
+  EXPECT_TRUE(intAttr.isa<BitEnumWithoutNoneAttr>());
+}
diff --git a/mlir/unittests/TableGen/enums.td b/mlir/unittests/TableGen/enums.td
index b2c8f6f7cecf2cd93f30d5480460fe0cec14073c..cdcc18254bd925f8f1a0fe103d6d5614fda1ab5b 100644
--- a/mlir/unittests/TableGen/enums.td
+++ b/mlir/unittests/TableGen/enums.td
@@ -15,6 +15,7 @@ def StrEnum: StrEnumAttr<"StrEnum", "A test enum", [CaseA, CaseB]> {
   let cppNamespace = "Outer::Inner";
   let stringToSymbolFnName = "ConvertToEnum";
   let symbolToStringFnName = "ConvertToString";
+  let genSpecializedAttr = 1;
 }
 
 def Case5: I32EnumAttrCase<"Case5", 5>;
diff --git a/openmp/libomptarget/deviceRTLs/CMakeLists.txt b/openmp/libomptarget/deviceRTLs/CMakeLists.txt
index 8bbf987aaf20554b1c0e1dcac345eef48a52c90a..3df94eac0727e89c657ce3a5cf485b28bd2c1694 100644
--- a/openmp/libomptarget/deviceRTLs/CMakeLists.txt
+++ b/openmp/libomptarget/deviceRTLs/CMakeLists.txt
@@ -10,5 +10,4 @@
 #
 ##===----------------------------------------------------------------------===##
 
-add_subdirectory(amdgcn)
 add_subdirectory(nvptx)
diff --git a/openmp/libomptarget/deviceRTLs/amdgcn/CMakeLists.txt b/openmp/libomptarget/deviceRTLs/amdgcn/CMakeLists.txt
index 361e052ef499dc65fdf1939c074827b61a0ec1f8..8ba057206b826f7772f199122c45456d4b52dfa6 100644
--- a/openmp/libomptarget/deviceRTLs/amdgcn/CMakeLists.txt
+++ b/openmp/libomptarget/deviceRTLs/amdgcn/CMakeLists.txt
@@ -154,6 +154,6 @@ foreach(mcpu ${mcpus})
   add_custom_target(lib${libname}-${mcpu} ALL DEPENDS ${bc_libname})
 
   install(FILES ${OUTPUTDIR}/${bc_libname}
-     DESTINATION "${OPENMP_INSTALL_LIBDIR}"
+     DESTINATION "${OPENMP_INSTALL_LIBDIR}/libdevice"
   )
 endforeach()
diff --git a/openmp/libomptarget/plugins/amdgpu/impl/internal.h b/openmp/libomptarget/plugins/amdgpu/impl/internal.h
index 1b1d6932878578db37d1c117c80a862b248a43af..8ca66a9d478e0eb499945896623571b0368b8711 100644
--- a/openmp/libomptarget/plugins/amdgpu/impl/internal.h
+++ b/openmp/libomptarget/plugins/amdgpu/impl/internal.h
@@ -97,6 +97,10 @@ typedef struct atl_kernel_info_s {
   uint64_t kernel_object;
   uint32_t group_segment_size;
   uint32_t private_segment_size;
+  uint32_t sgpr_count;
+  uint32_t vgpr_count;
+  uint32_t sgpr_spill_count;
+  uint32_t vgpr_spill_count;
   uint32_t kernel_segment_size;
   uint32_t num_args;
   std::vector<uint64_t> arg_alignments;
diff --git a/openmp/libomptarget/plugins/amdgpu/impl/system.cpp b/openmp/libomptarget/plugins/amdgpu/impl/system.cpp
index da152b4045d1e97752ea0a05fd2dfc6151b63b33..d6cde1f699c2359f6e5ddc3686cee0b1e37dd2da 100644
--- a/openmp/libomptarget/plugins/amdgpu/impl/system.cpp
+++ b/openmp/libomptarget/plugins/amdgpu/impl/system.cpp
@@ -832,7 +832,31 @@ static hsa_status_t get_code_object_custom_metadata(void *binary,
     msgpack_errors += map_lookup_string(element, ".symbol", &symbolName);
     msgpackErrorCheck(strings lookup in kernel metadata, msgpack_errors);
 
-    atl_kernel_info_t info = {0, 0, 0, 0, 0, {}, {}, {}};
+    atl_kernel_info_t info = {0, 0, 0, 0, 0, 0, 0, 0, 0, {}, {}, {}};
+
+    uint64_t sgpr_count, vgpr_count, sgpr_spill_count, vgpr_spill_count;
+    msgpack_errors += map_lookup_uint64_t(element, ".sgpr_count", &sgpr_count);
+    msgpackErrorCheck(sgpr count metadata lookup in kernel metadata,
+                      msgpack_errors);
+    info.sgpr_count = sgpr_count;
+
+    msgpack_errors += map_lookup_uint64_t(element, ".vgpr_count", &vgpr_count);
+    msgpackErrorCheck(vgpr count metadata lookup in kernel metadata,
+                      msgpack_errors);
+    info.vgpr_count = vgpr_count;
+
+    msgpack_errors +=
+        map_lookup_uint64_t(element, ".sgpr_spill_count", &sgpr_spill_count);
+    msgpackErrorCheck(sgpr spill count metadata lookup in kernel metadata,
+                      msgpack_errors);
+    info.sgpr_spill_count = sgpr_spill_count;
+
+    msgpack_errors +=
+        map_lookup_uint64_t(element, ".vgpr_spill_count", &vgpr_spill_count);
+    msgpackErrorCheck(vgpr spill count metadata lookup in kernel metadata,
+                      msgpack_errors);
+    info.vgpr_spill_count = vgpr_spill_count;
+
     size_t kernel_explicit_args_size = 0;
     uint64_t kernel_segment_size;
     msgpack_errors += map_lookup_uint64_t(element, ".kernarg_segment_size",
diff --git a/openmp/libomptarget/plugins/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins/amdgpu/src/rtl.cpp
index 0e8df9e9ca601e13f65526df9684cb0a24386612..a6b426dc0557912f445886dee8325577ce034424 100644
--- a/openmp/libomptarget/plugins/amdgpu/src/rtl.cpp
+++ b/openmp/libomptarget/plugins/amdgpu/src/rtl.cpp
@@ -1759,6 +1759,19 @@ int32_t __tgt_rtl_run_target_team_region_locked(
 
   KernelTy *KernelInfo = (KernelTy *)tgt_entry_ptr;
 
+  std::string kernel_name = std::string(KernelInfo->Name);
+  uint32_t sgpr_count, vgpr_count, sgpr_spill_count, vgpr_spill_count;
+
+  {
+    assert(KernelInfoTable[device_id].find(kernel_name) !=
+           KernelInfoTable[device_id].end());
+    auto it = KernelInfoTable[device_id][kernel_name];
+    sgpr_count = it.sgpr_count;
+    vgpr_count = it.vgpr_count;
+    sgpr_spill_count = it.sgpr_spill_count;
+    vgpr_spill_count = it.vgpr_spill_count;
+  }
+
   /*
    * Set limit based on ThreadsPerGroup and GroupsPerDevice
    */
@@ -1780,10 +1793,12 @@ int32_t __tgt_rtl_run_target_team_region_locked(
     bool traceToStdout = print_kernel_trace & (RTL_TO_STDOUT | RTL_TIMING);
     fprintf(traceToStdout ? stdout : stderr,
             "DEVID:%2d SGN:%1d ConstWGSize:%-4d args:%2d teamsXthrds:(%4dX%4d) "
-            "reqd:(%4dX%4d) n:%s\n",
+            "reqd:(%4dX%4d) sgpr_count:%u vgpr_count:%u sgpr_spill_count:%u "
+            "vgpr_spill_count:%u tripcount:%lu n:%s\n",
             device_id, KernelInfo->ExecutionMode, KernelInfo->ConstWGSize,
             arg_num, num_groups, threadsPerGroup, num_teams, thread_limit,
-            KernelInfo->Name);
+            sgpr_count, vgpr_count, sgpr_spill_count, vgpr_spill_count,
+            loop_tripcount, KernelInfo->Name);
   }
 
   // Run on the device.
@@ -1812,7 +1827,6 @@ int32_t __tgt_rtl_run_target_team_region_locked(
     packet->reserved2 = 0;           // atmi writes id_ here
     packet->completion_signal = {0}; // may want a pool of signals
 
-    std::string kernel_name = std::string(KernelInfo->Name);
     {
       assert(KernelInfoTable[device_id].find(kernel_name) !=
              KernelInfoTable[device_id].end());
diff --git a/openmp/libomptarget/plugins/cuda/src/rtl.cpp b/openmp/libomptarget/plugins/cuda/src/rtl.cpp
index 3d0424f160377a5ef6b37b0d8f03f4dc24f22d8e..a2f3cf4ba0c828d06c9e558096b59318e8fbc434 100644
--- a/openmp/libomptarget/plugins/cuda/src/rtl.cpp
+++ b/openmp/libomptarget/plugins/cuda/src/rtl.cpp
@@ -1035,13 +1035,6 @@ public:
   int synchronize(const int DeviceId, __tgt_async_info *AsyncInfo) const {
     CUstream Stream = reinterpret_cast<CUstream>(AsyncInfo->Queue);
     CUresult Err = cuStreamSynchronize(Stream);
-    if (Err != CUDA_SUCCESS) {
-      REPORT("Error when synchronizing stream. stream = " DPxMOD
-             ", async info ptr = " DPxMOD "\n",
-             DPxPTR(Stream), DPxPTR(AsyncInfo));
-      CUDA_ERR_STRING(Err);
-      return OFFLOAD_FAIL;
-    }
 
     // Once the stream is synchronized, return it to stream pool and reset
     // AsyncInfo. This is to make sure the synchronization only works for its
@@ -1050,7 +1043,13 @@ public:
                                 reinterpret_cast<CUstream>(AsyncInfo->Queue));
     AsyncInfo->Queue = nullptr;
 
-    return OFFLOAD_SUCCESS;
+    if (Err != CUDA_SUCCESS) {
+      REPORT("Error when synchronizing stream. stream = " DPxMOD
+             ", async info ptr = " DPxMOD "\n",
+             DPxPTR(Stream), DPxPTR(AsyncInfo));
+      CUDA_ERR_STRING(Err);
+    }
+    return (Err == CUDA_SUCCESS) ? OFFLOAD_SUCCESS : OFFLOAD_FAIL;
   }
 };
 
diff --git a/openmp/runtime/src/dllexports b/openmp/runtime/src/dllexports
index 79bca795d91edde3670924c86291a413434871bb..c6be679494cecc07b710b88f37bd7efd66db4a86 100644
--- a/openmp/runtime/src/dllexports
+++ b/openmp/runtime/src/dllexports
@@ -390,8 +390,9 @@ kmpc_set_disp_num_buffers                   267
         __kmpc_taskred_init                 277
         __kmpc_taskred_modifier_init        278
         __kmpc_omp_target_task_alloc        279
+        __kmpc_error                        281
         __kmpc_masked                       282
-        __kmpc_end_masked      	            283
+        __kmpc_end_masked                   283
 %endif
 
 # User API entry points that have both lower- and upper- case versions for Fortran.
diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index dc33fdbc7ff320c55d0d8b8d441ba894a2557d7e..5aa3115dc5a451e069a08235a94f6abacde47235 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -455,6 +455,8 @@ AffHWSubsetManyDies          "KMP_HW_SUBSET ignored: too many Dies requested."
 AffUseGlobCpuidL             "%1$s: Affinity capable, using global cpuid leaf %2$d info"
 AffNotCapableUseLocCpuidL    "%1$s: Affinity not capable, using local cpuid leaf %2$d info"
 AffNotUsingHwloc             "%1$s: Affinity not capable, using hwloc."
+UserDirectedError            "%1$s: Encountered user-directed error: %2$s."
+UserDirectedWarning          "%1$s: Encountered user-directed warning: %2$s."
 FailedToCreateTeam           "Failed to create teams between lower bound (%1$d) and upper bound (%2$d)."
 
 # --------------------------------------------------------------------------------------------------
diff --git a/openmp/runtime/src/include/omp-tools.h.var b/openmp/runtime/src/include/omp-tools.h.var
index 961e767c63c95b4ad0d2e18cbe1ffb9c0df8110c..8e822750b53e2e416a45a0b5b1db6d3056668ee5 100644
--- a/openmp/runtime/src/include/omp-tools.h.var
+++ b/openmp/runtime/src/include/omp-tools.h.var
@@ -1099,6 +1099,13 @@ typedef void (*ompt_callback_error_t) (
   const void *codeptr_ra
 );
 
+typedef struct ompt_record_error_t {
+  ompt_severity_t severity;
+  const char *message;
+  size_t length;
+  const void *codeptr_ra;
+} ompt_record_error_t;
+
 typedef struct ompd_address_t {
   ompd_seg_t segment;
   ompd_addr_t address;
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index bf4c812a8875656a1a50dfffc5a80bb5d5727b7f..c37e1d9feb57d4b288345f0f9ce4d365bdd9fa70 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -4087,6 +4087,13 @@ extern void __kmp_hidden_helper_main_thread_release();
 #define KMP_GTID_TO_SHADOW_GTID(gtid)                                          \
   ((gtid) % (__kmp_hidden_helper_threads_num - 1) + 2)
 
+// Support for error directive
+typedef enum kmp_severity_t {
+  severity_warning = 1,
+  severity_fatal = 2
+} kmp_severity_t;
+extern void __kmpc_error(ident_t *loc, int severity, const char *message);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/openmp/runtime/src/kmp_barrier.cpp b/openmp/runtime/src/kmp_barrier.cpp
index b4fde2aa067d15a372e08ac674ffcfeb1364f2be..237d18a73dcd6cf7f478959622b92f4e555aff8e 100644
--- a/openmp/runtime/src/kmp_barrier.cpp
+++ b/openmp/runtime/src/kmp_barrier.cpp
@@ -300,7 +300,7 @@ static void __kmp_tree_barrier_gather(
   kmp_uint32 branch_factor = 1 << branch_bits;
   kmp_uint32 child;
   kmp_uint32 child_tid;
-  kmp_uint64 new_state;
+  kmp_uint64 new_state = 0;
 
   KA_TRACE(
       20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
@@ -873,7 +873,7 @@ static void __kmp_hierarchical_barrier_gather(
   kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
   kmp_uint32 nproc = this_thr->th.th_team_nproc;
   kmp_info_t **other_threads = team->t.t_threads;
-  kmp_uint64 new_state;
+  kmp_uint64 new_state = 0;
 
   int level = team->t.t_level;
   if (other_threads[0]
@@ -1629,6 +1629,7 @@ int __kmp_barrier_gomp_cancel(int gtid) {
 void __kmp_end_split_barrier(enum barrier_type bt, int gtid) {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_end_split_barrier);
   KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
+  KMP_DEBUG_ASSERT(bt < bs_last_barrier);
   int tid = __kmp_tid_from_gtid(gtid);
   kmp_info_t *this_thr = __kmp_threads[gtid];
   kmp_team_t *team = this_thr->th.th_team;
@@ -1670,6 +1671,9 @@ void __kmp_end_split_barrier(enum barrier_type bt, int gtid) {
 void __kmp_join_barrier(int gtid) {
   KMP_TIME_PARTITIONED_BLOCK(OMP_join_barrier);
   KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER);
+
+  KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
+
   kmp_info_t *this_thr = __kmp_threads[gtid];
   kmp_team_t *team;
   kmp_uint nproc;
@@ -1706,7 +1710,6 @@ void __kmp_join_barrier(int gtid) {
   KMP_MB();
 
   // Verify state
-  KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
   KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_team));
   KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_root));
   KMP_DEBUG_ASSERT(this_thr == team->t.t_threads[tid]);
@@ -1931,6 +1934,7 @@ void __kmp_fork_barrier(int gtid, int tid) {
 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
 
 #ifdef KMP_DEBUG
+    KMP_DEBUG_ASSERT(team);
     kmp_info_t **other_threads = team->t.t_threads;
     int i;
 
@@ -2011,7 +2015,7 @@ void __kmp_fork_barrier(int gtid, int tid) {
     if (KMP_MASTER_TID(ds_tid) &&
         (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
          ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
-      codeptr = team->t.ompt_team_info.master_return_address;
+      codeptr = team ? team->t.ompt_team_info.master_return_address : NULL;
     if (ompt_enabled.ompt_callback_sync_region_wait) {
       ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
           ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index 4f34f3ac87b0f794c9af18b9dc1431fb030ec623..59d0dec505348cabeaa1945c0da7504194093517 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -4357,3 +4357,35 @@ int __kmpc_pause_resource(kmp_pause_status_t level) {
   }
   return __kmp_pause_resource(level);
 }
+
+void __kmpc_error(ident_t *loc, int severity, const char *message) {
+  if (!__kmp_init_serial)
+    __kmp_serial_initialize();
+
+  KMP_ASSERT(severity == severity_warning || severity == severity_fatal);
+
+#if OMPT_SUPPORT
+  if (ompt_enabled.enabled && ompt_enabled.ompt_callback_error) {
+    ompt_callbacks.ompt_callback(ompt_callback_error)(
+        (ompt_severity_t)severity, message, KMP_STRLEN(message),
+        OMPT_GET_RETURN_ADDRESS(0));
+  }
+#endif // OMPT_SUPPORT
+
+  char *src_loc;
+  if (loc && loc->psource) {
+    kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, false);
+    src_loc =
+        __kmp_str_format("%s:%s:%s", str_loc.file, str_loc.line, str_loc.col);
+    __kmp_str_loc_free(&str_loc);
+  } else {
+    src_loc = __kmp_str_format("unknown");
+  }
+
+  if (severity == severity_warning)
+    KMP_WARNING(UserDirectedWarning, src_loc, message);
+  else
+    KMP_FATAL(UserDirectedError, src_loc, message);
+
+  __kmp_str_free(&src_loc);
+}
diff --git a/openmp/runtime/src/kmp_dispatch.cpp b/openmp/runtime/src/kmp_dispatch.cpp
index 7d8bf468c38bfe44a7a558726199182d9aaca41b..d8e935d75080726f7d67d715fdd44734ee117085 100644
--- a/openmp/runtime/src/kmp_dispatch.cpp
+++ b/openmp/runtime/src/kmp_dispatch.cpp
@@ -1847,6 +1847,8 @@ int __kmp_dispatch_next_algorithm(int gtid,
         "__kmp_dispatch_next_algorithm: T#%%d exit status:%%d p_last:%%d "
         "p_lb:%%%s p_ub:%%%s p_st:%%%s\n",
         traits_t<T>::spec, traits_t<T>::spec, traits_t<ST>::spec);
+    KMP_DEBUG_ASSERT(p_last);
+    KMP_DEBUG_ASSERT(p_st);
     KD_TRACE(10, (buff, gtid, status, *p_last, *p_lb, *p_ub, *p_st));
     __kmp_str_free(&buff);
   }
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 8f42a9d3fe0ce990d18534d832cf596f4885e2cd..7a4920e9950529f54a3a3863bd896c744f42bb6a 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -854,6 +854,12 @@ static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
   if (TCR_PTR(__kmp_threads[0]) == NULL) {
     --capacity;
   }
+  // If it is not for initializing the hidden helper team, we need to take
+  // __kmp_hidden_helper_threads_num out of the capacity because it is included
+  // in __kmp_threads_capacity.
+  if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) {
+    capacity -= __kmp_hidden_helper_threads_num;
+  }
   if (__kmp_nth + new_nthreads -
           (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
       capacity) {
@@ -3607,6 +3613,13 @@ int __kmp_register_root(int initial_thread) {
     --capacity;
   }
 
+  // If it is not for initializing the hidden helper team, we need to take
+  // __kmp_hidden_helper_threads_num out of the capacity because it is included
+  // in __kmp_threads_capacity.
+  if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) {
+    capacity -= __kmp_hidden_helper_threads_num;
+  }
+
   /* see if there are too many threads */
   if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
     if (__kmp_tp_cached) {
@@ -3639,7 +3652,7 @@ int __kmp_register_root(int initial_thread) {
     /* find an available thread slot */
     // Don't reassign the zero slot since we need that to only be used by
     // initial thread. Slots for hidden helper threads should also be skipped.
-    if (initial_thread && __kmp_threads[0] == NULL) {
+    if (initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
       gtid = 0;
     } else {
       for (gtid = __kmp_hidden_helper_threads_num + 1;
@@ -4042,12 +4055,12 @@ static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
   /* this_thr->th.th_info.ds.ds_gtid is setup in
      kmp_allocate_thread/create_worker.
      this_thr->th.th_serial_team is setup in __kmp_allocate_thread */
-  kmp_info_t *master = team->t.t_threads[0];
   KMP_DEBUG_ASSERT(this_thr != NULL);
   KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);
   KMP_DEBUG_ASSERT(team);
   KMP_DEBUG_ASSERT(team->t.t_threads);
   KMP_DEBUG_ASSERT(team->t.t_dispatch);
+  kmp_info_t *master = team->t.t_threads[0];
   KMP_DEBUG_ASSERT(master);
   KMP_DEBUG_ASSERT(master->th.th_root);
 
@@ -5738,7 +5751,7 @@ void *__kmp_launch_thread(kmp_info_t *this_thr) {
   }
 
 #if OMPT_SUPPORT
-  ompt_data_t *thread_data;
+  ompt_data_t *thread_data = nullptr;
   if (ompt_enabled.enabled) {
     thread_data = &(this_thr->th.ompt_thread_info.thread_data);
     *thread_data = ompt_data_none;
diff --git a/openmp/runtime/src/kmp_sched.cpp b/openmp/runtime/src/kmp_sched.cpp
index c04c93985cbe0a43b5b71daf69e88659d312692b..09e497e02914c6acc18ae731ce10a34572d078f7 100644
--- a/openmp/runtime/src/kmp_sched.cpp
+++ b/openmp/runtime/src/kmp_sched.cpp
@@ -167,6 +167,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
                               "signed?<%s>, loc = %%s\n",
                               traits_t<T>::spec, traits_t<T>::spec,
                               traits_t<ST>::spec, traits_t<T>::spec);
+      check_loc(loc);
       KD_TRACE(100,
                (buff, *plastiter, *plower, *pupper, *pstride, loc->psource));
       __kmp_str_free(&buff);
diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp
index 35c15ee2a2e640cb8a6d91fb96ee5308a6de5994..7aff8f742783fc6d1a7ead62d0261d16c3010d02 100644
--- a/openmp/runtime/src/kmp_settings.cpp
+++ b/openmp/runtime/src/kmp_settings.cpp
@@ -504,9 +504,10 @@ int __kmp_initial_threads_capacity(int req_nproc) {
     nth = (4 * __kmp_xproc);
 
   // If hidden helper task is enabled, we initialize the thread capacity with
-  // extra
-  // __kmp_hidden_helper_threads_num.
-  nth += __kmp_hidden_helper_threads_num;
+  // extra __kmp_hidden_helper_threads_num.
+  if (__kmp_enable_hidden_helper) {
+    nth += __kmp_hidden_helper_threads_num;
+  }
 
   if (nth > __kmp_max_nth)
     nth = __kmp_max_nth;
@@ -1713,6 +1714,7 @@ static void __kmp_stg_print_barrier_pattern(kmp_str_buf_t *buffer,
         __kmp_str_buf_print(buffer, "   %s='",
                             __kmp_barrier_pattern_env_name[i]);
       }
+      KMP_DEBUG_ASSERT(j < bs_last_barrier && k < bs_last_barrier);
       __kmp_str_buf_print(buffer, "%s,%s'\n", __kmp_barrier_pattern_name[j],
                           __kmp_barrier_pattern_name[k]);
     }
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index f448e215a608264ca4b348d35ad9ae7bc809aaec..d6409f7b7d470d1b9d8b39bdd0eb2730c2a33b69 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -1221,6 +1221,7 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
                 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t,
                 sizeof_shareds, task_entry));
 
+  KMP_DEBUG_ASSERT(parent_task);
   if (parent_task->td_flags.final) {
     if (flags->merged_if0) {
     }
@@ -1682,10 +1683,8 @@ kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid,
   if (UNLIKELY(ompt_enabled.enabled)) {
     parent = new_taskdata->td_parent;
     if (ompt_enabled.ompt_callback_task_create) {
-      ompt_data_t task_data = ompt_data_none;
       ompt_callbacks.ompt_callback(ompt_callback_task_create)(
-          parent ? &(parent->ompt_task_info.task_data) : &task_data,
-          parent ? &(parent->ompt_task_info.frame) : NULL,
+          &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame),
           &(new_taskdata->ompt_task_info.task_data), ompt_task_explicit, 0,
           OMPT_GET_RETURN_ADDRESS(0));
     }
@@ -1782,10 +1781,9 @@ kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid,
             OMPT_GET_FRAME_ADDRESS(0);
       }
       if (ompt_enabled.ompt_callback_task_create) {
-        ompt_data_t task_data = ompt_data_none;
         ompt_callbacks.ompt_callback(ompt_callback_task_create)(
-            parent ? &(parent->ompt_task_info.task_data) : &task_data,
-            parent ? &(parent->ompt_task_info.frame) : NULL,
+            &(parent->ompt_task_info.task_data),
+            &(parent->ompt_task_info.frame),
             &(new_taskdata->ompt_task_info.task_data),
             ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
             OMPT_LOAD_RETURN_ADDRESS(gtid));
@@ -1845,10 +1843,8 @@ kmp_int32 __kmp_omp_taskloop_task(ident_t *loc_ref, kmp_int32 gtid,
     if (!parent->ompt_task_info.frame.enter_frame.ptr)
       parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
     if (ompt_enabled.ompt_callback_task_create) {
-      ompt_data_t task_data = ompt_data_none;
       ompt_callbacks.ompt_callback(ompt_callback_task_create)(
-          parent ? &(parent->ompt_task_info.task_data) : &task_data,
-          parent ? &(parent->ompt_task_info.frame) : NULL,
+          &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame),
           &(new_taskdata->ompt_task_info.task_data),
           ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
           codeptr_ra);
@@ -1873,7 +1869,7 @@ template <bool ompt>
 static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
                                               void *frame_address,
                                               void *return_address) {
-  kmp_taskdata_t *taskdata;
+  kmp_taskdata_t *taskdata = nullptr;
   kmp_info_t *thread;
   int thread_finished = FALSE;
   KMP_SET_THREAD_STATE_BLOCK(TASKWAIT);
@@ -2007,7 +2003,7 @@ kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) {
 
 // __kmpc_omp_taskyield: switch to a different task
 kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
-  kmp_taskdata_t *taskdata;
+  kmp_taskdata_t *taskdata = NULL;
   kmp_info_t *thread;
   int thread_finished = FALSE;
 
@@ -2533,7 +2529,7 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
   kmp_team_t *team;
   ompt_data_t my_task_data;
   ompt_data_t my_parallel_data;
-  void *codeptr;
+  void *codeptr = nullptr;
   if (UNLIKELY(ompt_enabled.enabled)) {
     team = thread->th.th_team;
     my_task_data = taskdata->ompt_task_info.task_data;
@@ -3593,8 +3589,7 @@ void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
     KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created new task_team %p"
                   " for team %d at parity=%d\n",
                   __kmp_gtid_from_thread(this_thr),
-                  team->t.t_task_team[this_thr->th.th_task_state],
-                  ((team != NULL) ? team->t.t_id : -1),
+                  team->t.t_task_team[this_thr->th.th_task_state], team->t.t_id,
                   this_thr->th.th_task_state));
   }
 
@@ -3607,14 +3602,14 @@ void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
   // for serialized teams.
   if (team->t.t_nproc > 1) {
     int other_team = 1 - this_thr->th.th_task_state;
+    KMP_DEBUG_ASSERT(other_team >= 0 && other_team < 2);
     if (team->t.t_task_team[other_team] == NULL) { // setup other team as well
       team->t.t_task_team[other_team] =
           __kmp_allocate_task_team(this_thr, team);
       KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created second new "
                     "task_team %p for team %d at parity=%d\n",
                     __kmp_gtid_from_thread(this_thr),
-                    team->t.t_task_team[other_team],
-                    ((team != NULL) ? team->t.t_id : -1), other_team));
+                    team->t.t_task_team[other_team], team->t.t_id, other_team));
     } else { // Leave the old task team struct in place for the upcoming region;
       // adjust as needed
       kmp_task_team_t *task_team = team->t.t_task_team[other_team];
@@ -3632,8 +3627,7 @@ void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
       KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d reset next task_team "
                     "%p for team %d at parity=%d\n",
                     __kmp_gtid_from_thread(this_thr),
-                    team->t.t_task_team[other_team],
-                    ((team != NULL) ? team->t.t_id : -1), other_team));
+                    team->t.t_task_team[other_team], team->t.t_id, other_team));
     }
   }
 
@@ -3676,7 +3670,7 @@ void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) {
            ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
             "%p from Team #%d (parity=%d)\n",
             __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team,
-            ((team != NULL) ? team->t.t_id : -1), this_thr->th.th_task_state));
+            team->t.t_id, this_thr->th.th_task_state));
 }
 
 // __kmp_task_team_wait: Primary thread waits for outstanding tasks after the
diff --git a/openmp/runtime/src/kmp_version.cpp b/openmp/runtime/src/kmp_version.cpp
index 2c82a17b0bf586d6afbf8ed5e8fed44801efd5b1..db2454c0f4de8cacbc964089528384325533f127 100644
--- a/openmp/runtime/src/kmp_version.cpp
+++ b/openmp/runtime/src/kmp_version.cpp
@@ -51,6 +51,8 @@
 #define KMP_COMPILER "Intel(R) C++ Compiler 18.0"
 #elif __INTEL_COMPILER == 1900
 #define KMP_COMPILER "Intel(R) C++ Compiler 19.0"
+#elif __INTEL_COMPILER == 1910
+#define KMP_COMPILER "Intel(R) C++ Compiler 19.1"
 #elif __INTEL_COMPILER >= 9900
 #define KMP_COMPILER "Intel(R) C++ Compiler mainline"
 #endif
diff --git a/openmp/runtime/src/ompt-event-specific.h b/openmp/runtime/src/ompt-event-specific.h
index 799fa0d578eabef8d7bf2985b3fb252ddd0bb822..875d6921b7b70f569029ab8a8bf6a31279c01a35 100644
--- a/openmp/runtime/src/ompt-event-specific.h
+++ b/openmp/runtime/src/ompt-event-specific.h
@@ -106,6 +106,6 @@
 
 #define ompt_callback_dispatch_implemented ompt_event_UNIMPLEMENTED
 
-#define ompt_callback_error_implemented ompt_event_UNIMPLEMENTED
+#define ompt_callback_error_implemented ompt_event_MAY_ALWAYS_OPTIONAL
 
 #endif
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index ea0fa7a2a4c5ffa9c346d8e6aed179360936141e..f755887e896bb61bf6f290fa94b7743cc746f5ac 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -102,6 +102,14 @@ ompt_callbacks_internal_t ompt_callbacks;
 
 static ompt_start_tool_result_t *ompt_start_tool_result = NULL;
 
+#if KMP_OS_WINDOWS
+static HMODULE ompt_tool_module = NULL;
+#define OMPT_DLCLOSE(Lib) FreeLibrary(Lib)
+#else
+static void *ompt_tool_module = NULL;
+#define OMPT_DLCLOSE(Lib) dlclose(Lib)
+#endif
+
 /*****************************************************************************
  * forward declarations
  ****************************************************************************/
@@ -314,12 +322,14 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
             OMPT_VERBOSE_INIT_CONTINUED_PRINT("Success.\n");
             OMPT_VERBOSE_INIT_PRINT(
                 "Tool was started and is using the OMPT interface.\n");
+            ompt_tool_module = h;
             break;
           }
           OMPT_VERBOSE_INIT_CONTINUED_PRINT(
               "Found but not using the OMPT interface.\n");
           OMPT_VERBOSE_INIT_PRINT("Continuing search...\n");
         }
+        OMPT_DLCLOSE(h);
       }
       fname = __kmp_str_token(NULL, sep, &buf);
     }
@@ -495,6 +505,8 @@ void ompt_fini() {
     ompt_start_tool_result->finalize(&(ompt_start_tool_result->tool_data));
   }
 
+  if (ompt_tool_module)
+    OMPT_DLCLOSE(ompt_tool_module);
   memset(&ompt_enabled, 0, sizeof(ompt_enabled));
 }
 
@@ -675,6 +687,8 @@ OMPT_API_ROUTINE int ompt_get_place_proc_ids(int place_num, int ids_size,
 #else
   int i, count;
   int tmp_ids[ids_size];
+  for (int j = 0; j < ids_size; j++)
+    tmp_ids[j] = 0;
   if (!KMP_AFFINITY_CAPABLE())
     return 0;
   if (place_num < 0 || place_num >= (int)__kmp_affinity_num_masks)
diff --git a/openmp/runtime/src/ompt-specific.h b/openmp/runtime/src/ompt-specific.h
index 4d76dfcec593e17b66465f2d11df6547fd5d96fb..2fc7ee1c35bc8ccc89746c2e0a540cab625b0eaa 100644
--- a/openmp/runtime/src/ompt-specific.h
+++ b/openmp/runtime/src/ompt-specific.h
@@ -103,7 +103,8 @@ inline kmp_info_t *ompt_get_thread() {
 }
 
 inline void ompt_set_thread_state(kmp_info_t *thread, ompt_state_t state) {
-  thread->th.ompt_thread_info.state = state;
+  if (thread)
+    thread->th.ompt_thread_info.state = state;
 }
 
 inline const char *ompt_get_runtime_version() {
diff --git a/openmp/runtime/test/ompt/callback.h b/openmp/runtime/test/ompt/callback.h
index e426558bf869048f29697b241aba9b88cbafbdb2..c21b16741d33d4076e0762033f9535d431dc5b5c 100644
--- a/openmp/runtime/test/ompt/callback.h
+++ b/openmp/runtime/test/ompt/callback.h
@@ -1124,6 +1124,15 @@ on_ompt_callback_control_tool(
   return 0; //success
 }
 
+static void on_ompt_callback_error(ompt_severity_t severity,
+                                   const char *message, size_t length,
+                                   const void *codeptr_ra) {
+  printf("%" PRIu64 ": ompt_event_runtime_error: severity=%" PRIu32
+         ", message=%s, length=%" PRIu64 ", codeptr_ra=%p\n",
+         ompt_get_thread_data()->value, severity, message, (uint64_t)length,
+         codeptr_ra);
+}
+
 int ompt_initialize(
   ompt_function_lookup_t lookup,
   int initial_device_num,
@@ -1173,6 +1182,7 @@ int ompt_initialize(
   register_ompt_callback(ompt_callback_task_dependence);
   register_ompt_callback(ompt_callback_thread_begin);
   register_ompt_callback(ompt_callback_thread_end);
+  register_ompt_callback(ompt_callback_error);
   printf("0: NULL_POINTER=%p\n", (void*)NULL);
   return 1; //success
 }
diff --git a/openmp/runtime/test/ompt/misc/runtime_error.c b/openmp/runtime/test/ompt/misc/runtime_error.c
new file mode 100644
index 0000000000000000000000000000000000000000..ee9e2e832bfa98e1a8eceeb5c264c485b47dc49a
--- /dev/null
+++ b/openmp/runtime/test/ompt/misc/runtime_error.c
@@ -0,0 +1,35 @@
+// RUN: %libomp-compile-and-run 2>&1 | sort | FileCheck %s
+// REQUIRES: ompt
+
+#include <string.h>
+#include <stdio.h>
+#include "callback.h"
+
+// TODO: use error directive when compiler suppors
+typedef void ident_t;
+extern void __kmpc_error(ident_t *, int, const char *);
+
+int main() {
+#pragma omp parallel num_threads(2)
+  {
+    if (omp_get_thread_num() == 0) {
+      const char *msg = "User message goes here";
+      printf("0: Message length=%" PRIu64 "\n", (uint64_t)strlen(msg));
+      __kmpc_error(NULL, ompt_warning, msg);
+    }
+  }
+  return 0;
+}
+
+// CHECK: {{^}}0: Message length=[[LENGTH:[0-9]+]]
+// CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
+
+// CHECK: {{^}}[[PRIMARY_ID:[0-9]+]]: ompt_event_implicit_task_begin
+// CHECK: {{^}}[[PRIMARY_ID]]: ompt_event_runtime_error
+// CHECK-SAME: severity=1
+// CHECK-SAME: message=User message goes here
+// CHECK-SAME: length=[[LENGTH]]
+// CHECK-SAME: codeptr_ra={{0x[0-f]+}}
+
+// Message from runtime
+// CHECK: {{^}}OMP: Warning{{.*}}User message goes here
diff --git a/openmp/runtime/test/tasking/hidden_helper_task/capacity_mix_threads.cpp b/openmp/runtime/test/tasking/hidden_helper_task/capacity_mix_threads.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..776aee9d8e2cac91c9df3582e64b0189163dd19f
--- /dev/null
+++ b/openmp/runtime/test/tasking/hidden_helper_task/capacity_mix_threads.cpp
@@ -0,0 +1,45 @@
+// RUN: %libomp-cxx-compile-and-run
+
+#include <omp.h>
+
+#include <algorithm>
+#include <cassert>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+void dummy_root() {
+  // omp_get_max_threads() will do middle initialization
+  int nthreads = omp_get_max_threads();
+  std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+}
+
+int main(int argc, char *argv[]) {
+  const int N = std::min(std::max(std::max(32, 4 * omp_get_max_threads()),
+                                  4 * omp_get_num_procs()),
+                         std::numeric_limits<int>::max());
+
+  std::vector<int> data(N);
+
+  // Create a new thread to initialize the OpenMP RTL. The new thread will not
+  // be taken as the "initial thread".
+  std::thread root(dummy_root);
+
+#pragma omp parallel for num_threads(N)
+  for (unsigned i = 0; i < N; ++i) {
+    data[i] = i;
+  }
+
+#pragma omp parallel for num_threads(N + 1)
+  for (unsigned i = 0; i < N; ++i) {
+    data[i] += i;
+  }
+
+  for (unsigned i = 0; i < N; ++i) {
+    assert(data[i] == 2 * i);
+  }
+
+  root.join();
+
+  return 0;
+}
diff --git a/openmp/runtime/test/tasking/hidden_helper_task/capacity_nthreads.cpp b/openmp/runtime/test/tasking/hidden_helper_task/capacity_nthreads.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a9d394f729e9c5171d6c22746d19a4a6653d377e
--- /dev/null
+++ b/openmp/runtime/test/tasking/hidden_helper_task/capacity_nthreads.cpp
@@ -0,0 +1,31 @@
+// RUN: %libomp-cxx-compile-and-run
+
+#include <omp.h>
+
+#include <algorithm>
+#include <cassert>
+#include <vector>
+
+int main(int argc, char *argv[]) {
+  const int N = std::min(std::max(std::max(32, 4 * omp_get_max_threads()),
+                                  4 * omp_get_num_procs()),
+                         std::numeric_limits<int>::max());
+
+  std::vector<int> data(N);
+
+#pragma omp parallel for num_threads(N)
+  for (unsigned i = 0; i < N; ++i) {
+    data[i] = i;
+  }
+
+#pragma omp parallel for num_threads(N + 1)
+  for (unsigned i = 0; i < N; ++i) {
+    data[i] += i;
+  }
+
+  for (unsigned i = 0; i < N; ++i) {
+    assert(data[i] == 2 * i);
+  }
+
+  return 0;
+}
diff --git a/polly/include/polly/DeadCodeElimination.h b/polly/include/polly/DeadCodeElimination.h
new file mode 100644
index 0000000000000000000000000000000000000000..044bc2ae84ee40ae2ed73ee32c913b39a4931dc1
--- /dev/null
+++ b/polly/include/polly/DeadCodeElimination.h
@@ -0,0 +1,40 @@
+//===- DeadCodeElimination.h ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  Eliminate dead iterations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef POLLY_DEADCODEELIMINATION_H
+#define POLLY_DEADCODEELIMINATION_H
+
+#include "polly/ScopPass.h"
+
+namespace llvm {
+class PassRegistry;
+class Pass;
+class raw_ostream;
+} // namespace llvm
+
+namespace polly {
+llvm::Pass *createDeadCodeElimWrapperPass();
+
+struct DeadCodeElimPass : llvm::PassInfoMixin<DeadCodeElimPass> {
+  DeadCodeElimPass() {}
+
+  llvm::PreservedAnalyses run(Scop &S, ScopAnalysisManager &SAM,
+                              ScopStandardAnalysisResults &SAR, SPMUpdater &U);
+};
+
+} // namespace polly
+
+namespace llvm {
+void initializeDeadCodeElimWrapperPassPass(llvm::PassRegistry &);
+} // namespace llvm
+
+#endif /* POLLY_DEADCODEELIMINATION_H */
diff --git a/polly/include/polly/LinkAllPasses.h b/polly/include/polly/LinkAllPasses.h
index f59db175a8a5d57db7420307d1d2866b24cbc56b..1d1e455c299b8a7b49aa8c62517b83dfea7ef372 100644
--- a/polly/include/polly/LinkAllPasses.h
+++ b/polly/include/polly/LinkAllPasses.h
@@ -28,7 +28,7 @@ class PassRegistry;
 namespace polly {
 llvm::Pass *createCodePreparationPass();
 llvm::Pass *createScopInlinerPass();
-llvm::Pass *createDeadCodeElimPass();
+llvm::Pass *createDeadCodeElimWrapperPass();
 llvm::Pass *createDependenceInfoPass();
 llvm::Pass *createDependenceInfoWrapperPassPass();
 llvm::Pass *createDOTOnlyPrinterPass();
@@ -75,7 +75,7 @@ struct PollyForcePassLinking {
       return;
 
     polly::createCodePreparationPass();
-    polly::createDeadCodeElimPass();
+    polly::createDeadCodeElimWrapperPass();
     polly::createDependenceInfoPass();
     polly::createDOTOnlyPrinterPass();
     polly::createDOTOnlyViewerPass();
@@ -109,7 +109,7 @@ namespace llvm {
 class PassRegistry;
 void initializeCodePreparationPass(llvm::PassRegistry &);
 void initializeScopInlinerPass(llvm::PassRegistry &);
-void initializeDeadCodeElimPass(llvm::PassRegistry &);
+void initializeDeadCodeElimWrapperPassPass(llvm::PassRegistry &);
 void initializeJSONExporterPass(llvm::PassRegistry &);
 void initializeJSONImporterPass(llvm::PassRegistry &);
 void initializeIslAstInfoWrapperPassPass(llvm::PassRegistry &);
diff --git a/polly/lib/CodeGen/IslNodeBuilder.cpp b/polly/lib/CodeGen/IslNodeBuilder.cpp
index 1ce407d7782917ee91f38dbab18f3ff7220e9961..fd0d50ac223358423c50b63f9ecf5ed144cf4f06 100644
--- a/polly/lib/CodeGen/IslNodeBuilder.cpp
+++ b/polly/lib/CodeGen/IslNodeBuilder.cpp
@@ -428,9 +428,14 @@ void IslNodeBuilder::createMark(__isl_take isl_ast_node *Node) {
   }
 
   BandAttr *ChildLoopAttr = getLoopAttr(isl::manage_copy(Id));
+  BandAttr *AncestorLoopAttr;
   if (ChildLoopAttr) {
-    assert(!Annotator.getStagingAttrEnv() &&
-           "conflicting loop attr environments");
+    // Save current LoopAttr environment to restore again when leaving this
+    // subtree. This means there was no loop between the ancestor LoopAttr and
+    // this mark, i.e. the ancestor LoopAttr did not directly mark a loop. This
+    // can happen e.g. if the AST build peeled or unrolled the loop.
+    AncestorLoopAttr = Annotator.getStagingAttrEnv();
+
     Annotator.getStagingAttrEnv() = ChildLoopAttr;
   }
 
@@ -438,8 +443,8 @@ void IslNodeBuilder::createMark(__isl_take isl_ast_node *Node) {
 
   if (ChildLoopAttr) {
     assert(Annotator.getStagingAttrEnv() == ChildLoopAttr &&
-           "Nest  must not overwrite loop attr environment");
-    Annotator.getStagingAttrEnv() = nullptr;
+           "Nest must not overwrite loop attr environment");
+    Annotator.getStagingAttrEnv() = AncestorLoopAttr;
   }
 
   isl_id_free(Id);
diff --git a/polly/lib/Support/PollyPasses.def b/polly/lib/Support/PollyPasses.def
index f2111e3bd14f180696e5842da6403c74b322acb3..03b3dc89569c670f4b364f26af3b8e82029a3cf2 100644
--- a/polly/lib/Support/PollyPasses.def
+++ b/polly/lib/Support/PollyPasses.def
@@ -38,4 +38,5 @@ SCOP_PASS("print<polly-delicm>", DeLICMPrinterPass(outs()))
 SCOP_PASS("polly-prune-unprofitable", PruneUnprofitablePass())
 SCOP_PASS("polly-opt-isl", IslScheduleOptimizerPass())
 SCOP_PASS("print<polly-opt-isl>", IslScheduleOptimizerPrinterPass(outs()))
+SCOP_PASS("polly-dce", DeadCodeElimPass())
 #undef SCOP_PASS
diff --git a/polly/lib/Support/RegisterPasses.cpp b/polly/lib/Support/RegisterPasses.cpp
index ebbce7cc8d32a2d46f98105194daea6466491cb2..05cad2b5e52a6014b32f5f8a9d12430c97a09822 100644
--- a/polly/lib/Support/RegisterPasses.cpp
+++ b/polly/lib/Support/RegisterPasses.cpp
@@ -25,6 +25,7 @@
 #include "polly/CodeGen/IslAst.h"
 #include "polly/CodePreparation.h"
 #include "polly/DeLICM.h"
+#include "polly/DeadCodeElimination.h"
 #include "polly/DependenceInfo.h"
 #include "polly/ForwardOpTree.h"
 #include "polly/JSONExporter.h"
@@ -248,7 +249,7 @@ void initializePollyPasses(PassRegistry &Registry) {
   LLVMInitializeNVPTXAsmPrinter();
 #endif
   initializeCodePreparationPass(Registry);
-  initializeDeadCodeElimPass(Registry);
+  initializeDeadCodeElimWrapperPassPass(Registry);
   initializeDependenceInfoPass(Registry);
   initializeDependenceInfoWrapperPassPass(Registry);
   initializeJSONExporterPass(Registry);
@@ -336,7 +337,7 @@ static void registerPollyPasses(llvm::legacy::PassManagerBase &PM,
     PM.add(polly::createJSONImporterPass());
 
   if (DeadCodeElim)
-    PM.add(polly::createDeadCodeElimPass());
+    PM.add(polly::createDeadCodeElimWrapperPass());
 
   if (FullyIndexedStaticExpansion)
     PM.add(polly::createMaximalStaticExpansionPass());
@@ -517,7 +518,7 @@ static void buildDefaultPollyPipeline(FunctionPassManager &PM,
     SPM.addPass(JSONImportPass());
 
   if (DeadCodeElim)
-    report_fatal_error("Option -polly-run-dce not supported with NPM", false);
+    SPM.addPass(DeadCodeElimPass());
 
   if (FullyIndexedStaticExpansion)
     report_fatal_error("Option -polly-enable-mse not supported with NPM",
diff --git a/polly/lib/Transform/DeadCodeElimination.cpp b/polly/lib/Transform/DeadCodeElimination.cpp
index cc8923c2547e62c6b4e5ae67e4c7de4126f357eb..546f38653b9690a8e52a5e07f1bc745525222000 100644
--- a/polly/lib/Transform/DeadCodeElimination.cpp
+++ b/polly/lib/Transform/DeadCodeElimination.cpp
@@ -31,6 +31,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "polly/DeadCodeElimination.h"
 #include "polly/DependenceInfo.h"
 #include "polly/LinkAllPasses.h"
 #include "polly/Options.h"
@@ -50,43 +51,38 @@ cl::opt<int> DCEPreciseSteps(
              "before the actual dead code elimination."),
     cl::ZeroOrMore, cl::init(-1), cl::cat(PollyCategory));
 
-class DeadCodeElim : public ScopPass {
+class DeadCodeElimWrapperPass : public ScopPass {
 public:
   static char ID;
-  explicit DeadCodeElim() : ScopPass(ID) {}
+  explicit DeadCodeElimWrapperPass() : ScopPass(ID) {}
 
   /// Remove dead iterations from the schedule of @p S.
   bool runOnScop(Scop &S) override;
 
   /// Register all analyses and transformation required.
   void getAnalysisUsage(AnalysisUsage &AU) const override;
-
-private:
-  /// Return the set of live iterations.
-  ///
-  /// The set of live iterations are all iterations that write to memory and for
-  /// which we can not prove that there will be a later write that _must_
-  /// overwrite the same memory location and is consequently the only one that
-  /// is visible after the execution of the SCoP.
-  ///
-  isl::union_set getLiveOut(Scop &S);
-  bool eliminateDeadCode(Scop &S, int PreciseSteps);
 };
-} // namespace
 
-char DeadCodeElim::ID = 0;
+char DeadCodeElimWrapperPass::ID = 0;
 
-// To compute the live outs, we compute for the data-locations that are
-// must-written to the last statement that touches these locations. On top of
-// this we add all statements that perform may-write accesses.
-//
-// We could be more precise by removing may-write accesses for which we know
-// that they are overwritten by a must-write after. However, at the moment the
-// only may-writes we introduce access the full (unbounded) array, such that
-// bounded write accesses can not overwrite all of the data-locations. As
-// this means may-writes are in the current situation always live, there is
-// no point in trying to remove them from the live-out set.
-isl::union_set DeadCodeElim::getLiveOut(Scop &S) {
+/// Return the set of live iterations.
+///
+/// The set of live iterations are all iterations that write to memory and for
+/// which we can not prove that there will be a later write that _must_
+/// overwrite the same memory location and is consequently the only one that
+/// is visible after the execution of the SCoP.
+///
+/// To compute the live outs, we compute for the data-locations that are
+/// must-written to the last statement that touches these locations. On top of
+/// this we add all statements that perform may-write accesses.
+///
+/// We could be more precise by removing may-write accesses for which we know
+/// that they are overwritten by a must-write after. However, at the moment the
+/// only may-writes we introduce access the full (unbounded) array, such that
+/// bounded write accesses can not overwrite all of the data-locations. As
+/// this means may-writes are in the current situation always live, there is
+/// no point in trying to remove them from the live-out set.
+static isl::union_set getLiveOut(Scop &S) {
   isl::union_map Schedule = S.getSchedule();
   isl::union_map MustWrites = S.getMustWrites();
   isl::union_map WriteIterations = MustWrites.reverse();
@@ -110,10 +106,8 @@ isl::union_set DeadCodeElim::getLiveOut(Scop &S) {
 /// To ensure the set of live iterations does not get too complex we always
 /// combine a certain number of precise steps with one approximating step that
 /// simplifies the life set with an affine hull.
-bool DeadCodeElim::eliminateDeadCode(Scop &S, int PreciseSteps) {
-  DependenceInfo &DI = getAnalysis<DependenceInfo>();
-  const Dependences &D = DI.getDependences(Dependences::AL_Statement);
-
+static bool runDeadCodeElimination(Scop &S, int PreciseSteps,
+                                   const Dependences &D) {
   if (!D.hasValidDependences())
     return false;
 
@@ -147,29 +141,60 @@ bool DeadCodeElim::eliminateDeadCode(Scop &S, int PreciseSteps) {
 
   Live = Live.coalesce();
 
-  bool Changed = S.restrictDomains(Live);
+  return S.restrictDomains(Live);
+}
+
+bool DeadCodeElimWrapperPass::runOnScop(Scop &S) {
+  auto &DI = getAnalysis<DependenceInfo>();
+  const Dependences &Deps = DI.getDependences(Dependences::AL_Statement);
+
+  bool Changed = runDeadCodeElimination(S, DCEPreciseSteps, Deps);
 
   // FIXME: We can probably avoid the recomputation of all dependences by
   // updating them explicitly.
   if (Changed)
     DI.recomputeDependences(Dependences::AL_Statement);
-  return Changed;
-}
 
-bool DeadCodeElim::runOnScop(Scop &S) {
-  return eliminateDeadCode(S, DCEPreciseSteps);
+  return false;
 }
 
-void DeadCodeElim::getAnalysisUsage(AnalysisUsage &AU) const {
+void DeadCodeElimWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
   ScopPass::getAnalysisUsage(AU);
   AU.addRequired<DependenceInfo>();
 }
 
-Pass *polly::createDeadCodeElimPass() { return new DeadCodeElim(); }
+} // namespace
+
+Pass *polly::createDeadCodeElimWrapperPass() {
+  return new DeadCodeElimWrapperPass();
+}
+
+llvm::PreservedAnalyses DeadCodeElimPass::run(Scop &S, ScopAnalysisManager &SAM,
+                                              ScopStandardAnalysisResults &SAR,
+                                              SPMUpdater &U) {
+  DependenceAnalysis::Result &DA = SAM.getResult<DependenceAnalysis>(S, SAR);
+  const Dependences &Deps = DA.getDependences(Dependences::AL_Statement);
+
+  bool Changed = runDeadCodeElimination(S, DCEPreciseSteps, Deps);
+
+  // FIXME: We can probably avoid the recomputation of all dependences by
+  // updating them explicitly.
+  if (Changed)
+    DA.recomputeDependences(Dependences::AL_Statement);
+
+  if (!Changed)
+    return PreservedAnalyses::all();
+
+  PreservedAnalyses PA;
+  PA.preserveSet<AllAnalysesOn<Module>>();
+  PA.preserveSet<AllAnalysesOn<Function>>();
+  PA.preserveSet<AllAnalysesOn<Loop>>();
+  return PA;
+}
 
-INITIALIZE_PASS_BEGIN(DeadCodeElim, "polly-dce",
+INITIALIZE_PASS_BEGIN(DeadCodeElimWrapperPass, "polly-dce",
                       "Polly - Remove dead iterations", false, false)
 INITIALIZE_PASS_DEPENDENCY(DependenceInfo)
 INITIALIZE_PASS_DEPENDENCY(ScopInfoRegionPass)
-INITIALIZE_PASS_END(DeadCodeElim, "polly-dce", "Polly - Remove dead iterations",
-                    false, false)
+INITIALIZE_PASS_END(DeadCodeElimWrapperPass, "polly-dce",
+                    "Polly - Remove dead iterations", false, false)
diff --git a/polly/lib/Transform/ScheduleOptimizer.cpp b/polly/lib/Transform/ScheduleOptimizer.cpp
index 493448aeaa2fe876927e890d983fdbc10b2e8836..913828994cb332a0cf58adf70e1ec42de819b800 100644
--- a/polly/lib/Transform/ScheduleOptimizer.cpp
+++ b/polly/lib/Transform/ScheduleOptimizer.cpp
@@ -1217,7 +1217,8 @@ getMicroKernelParams(const TargetTransformInfo *TTI, MatMulInfoTy MMI) {
   long RegisterBitwidth = VectorRegisterBitwidth;
 
   if (RegisterBitwidth == -1)
-    RegisterBitwidth = TTI->getRegisterBitWidth(true);
+    RegisterBitwidth =
+        TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector);
   auto ElementSize = getMatMulTypeSize(MMI);
   assert(ElementSize > 0 && "The element size of the matrix multiplication "
                             "operands should be greater than zero.");
@@ -1722,6 +1723,7 @@ private:
 
 char IslScheduleOptimizerWrapperPass::ID = 0;
 
+#ifndef NDEBUG
 static void printSchedule(llvm::raw_ostream &OS, const isl::schedule &Schedule,
                           StringRef Desc) {
   isl::ctx Ctx = Schedule.get_ctx();
@@ -1733,6 +1735,7 @@ static void printSchedule(llvm::raw_ostream &OS, const isl::schedule &Schedule,
   free(Str);
   isl_printer_free(P);
 }
+#endif
 
 /// Collect statistics for the schedule tree.
 ///
diff --git a/polly/lib/Transform/ScheduleTreeTransform.cpp b/polly/lib/Transform/ScheduleTreeTransform.cpp
index c6f9c32172be482c616ce9feabcbff7b66d9c5b9..32cef0ff959d7ac8611aee8fa5a9af24b433df34 100644
--- a/polly/lib/Transform/ScheduleTreeTransform.cpp
+++ b/polly/lib/Transform/ScheduleTreeTransform.cpp
@@ -533,13 +533,13 @@ isl::schedule polly::applyFullUnroll(isl::schedule_node BandToUnroll) {
   PartialSchedUAff = PartialSchedUAff.intersect_domain(Domain);
   isl::union_map PartialSchedUMap = isl::union_map(PartialSchedUAff);
 
-  // Make consumable for the following code.
-  // Schedule at the beginning so it is at coordinate 0.
-  isl::union_set PartialSchedUSet = PartialSchedUMap.reverse().wrap();
+  // Enumerator only the scatter elements.
+  isl::union_set ScatterList = PartialSchedUMap.range();
 
-  SmallVector<isl::point, 16> Elts;
+  // Enumerate all loop iterations.
   // TODO: Diagnose if not enumerable or depends on a parameter.
-  PartialSchedUSet.foreach_point([&Elts](isl::point P) -> isl::stat {
+  SmallVector<isl::point, 16> Elts;
+  ScatterList.foreach_point([&Elts](isl::point P) -> isl::stat {
     Elts.push_back(P);
     return isl::stat::ok();
   });
@@ -554,12 +554,10 @@ isl::schedule polly::applyFullUnroll(isl::schedule_node BandToUnroll) {
   // Convert the points to a sequence of filters.
   isl::union_set_list List = isl::union_set_list::alloc(Ctx, Elts.size());
   for (isl::point P : Elts) {
-    isl::basic_set AsSet{P};
-
-    // Throw away the scatter dimension.
-    AsSet = AsSet.unwrap().range();
+    // Determine the domains that map this scatter element.
+    isl::union_set DomainFilter = PartialSchedUMap.intersect_range(P).domain();
 
-    List = List.add(AsSet);
+    List = List.add(DomainFilter);
   }
 
   // Replace original band with unrolled sequence.
diff --git a/polly/test/DeadCodeElimination/computeout.ll b/polly/test/DeadCodeElimination/computeout.ll
index 6b40a09c880762d5addd3b6d7c935d57a82fb97b..d18ad44f66b19262ed5a96b000026c217e275baa 100644
--- a/polly/test/DeadCodeElimination/computeout.ll
+++ b/polly/test/DeadCodeElimination/computeout.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -S %loadPolly -basic-aa -polly-dce -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt -S %loadPolly "-passes=scop(polly-dce,print<polly-ast>)" < %s | FileCheck %s
 ; RUN: opt -S %loadPolly -basic-aa -polly-dce -polly-ast -analyze -polly-dependences-computeout=1 < %s | FileCheck %s -check-prefix=TIMEOUT
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/DeadCodeElimination/dead_iteration_elimination.ll b/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
index cc9851ffab870bc7a519a24d1089f33a9cc2d6bb..92442c7f9ee9de24f3815213f8afa69358e4a366 100644
--- a/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
+++ b/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-dce-precise-steps=2 -polly-ast -analyze < %s | FileCheck %s
+; RUN: opt -S %loadPolly "-passes=scop(polly-dce,print<polly-ast>)" -polly-dependences-analysis-type=value-based -polly-dce-precise-steps=2 < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll
index a9577271119a765c050e91a0467456de82173ef6..dcd65b357d97f8389930ba78e7be6c15022300c4 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll
@@ -38,15 +38,25 @@ return:
 
 
 ; CHECK-LABEL: Printing analysis 'Polly - Optimize schedule of SCoP' for region: 'for => return' in function 'func':
+; CHECK: domain: "{ Stmt_body[i0] : 0 <= i0 <= 11 }"
+; CHECK    sequence:
+; CHECK:   - filter: "{ Stmt_body[i0] : 0 <= i0 <= 3 }"
+; CHECK        sequence:
 ; CHECK:       - filter: "{ Stmt_body[0] }"
-; CHECK:       - filter: "{ Stmt_body[1] }"
-; CHECK:       - filter: "{ Stmt_body[2] }"
-; CHECK:       - filter: "{ Stmt_body[3] }"
+; CHECK:       - filter: "{ Stmt_body[i0] : (-1 + i0) mod 4 = 0 }"
+; CHECK:       - filter: "{ Stmt_body[i0] : (2 + i0) mod 4 = 0 }"
+; CHECK:       - filter: "{ Stmt_body[i0] : (1 + i0) mod 4 = 0 }"
+; CHECK    sequence:
+; CHECK:   - filter: "{ Stmt_body[i0] : 4 <= i0 <= 7 }"
+; CHECK        sequence:
 ; CHECK:       - filter: "{ Stmt_body[4] }"
-; CHECK:       - filter: "{ Stmt_body[5] }"
-; CHECK:       - filter: "{ Stmt_body[6] }"
-; CHECK:       - filter: "{ Stmt_body[7] }"
+; CHECK:       - filter: "{ Stmt_body[i0] : (-1 + i0) mod 4 = 0 }"
+; CHECK:       - filter: "{ Stmt_body[i0] : (2 + i0) mod 4 = 0 }"
+; CHECK:       - filter: "{ Stmt_body[i0] : (1 + i0) mod 4 = 0 }"
+; CHECK    sequence:
+; CHECK:   - filter: "{ Stmt_body[i0] : 8 <= i0 <= 11 }"
+; CHECK        sequence:
 ; CHECK:       - filter: "{ Stmt_body[8] }"
-; CHECK:       - filter: "{ Stmt_body[9] }"
-; CHECK:       - filter: "{ Stmt_body[10] }"
-; CHECK:       - filter: "{ Stmt_body[11] }"
+; CHECK:       - filter: "{ Stmt_body[i0] : (-1 + i0) mod 4 = 0 }"
+; CHECK:       - filter: "{ Stmt_body[i0] : (2 + i0) mod 4 = 0 }"
+; CHECK:       - filter: "{ Stmt_body[i0] : (1 + i0) mod 4 = 0 }"
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index d4b462adbc93370ed7438e077229f0eb4b8eb336..1ffce323d951a52a675ecfa22f9ebda710a8642b 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -2,7 +2,7 @@
 cmake_minimum_required(VERSION 3.13.4)
 project(Runtimes C CXX ASM)
 
-set(LLVM_ALL_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind;openmp")
+set(LLVM_ALL_RUNTIMES "compiler-rt;libc;libcxx;libcxxabi;libunwind;openmp")
 set(LLVM_ENABLE_RUNTIMES "" CACHE STRING
   "Semicolon-separated list of runtimes to build (${LLVM_ALL_RUNTIMES}), or \"all\".")
 if(LLVM_ENABLE_RUNTIMES STREQUAL "all" )